diff options
Diffstat (limited to 'llvm/include')
395 files changed, 4743 insertions, 2889 deletions
diff --git a/llvm/include/llvm-c/Core.h b/llvm/include/llvm-c/Core.h index ae2bcb8444b4..ca3ca24487a5 100644 --- a/llvm/include/llvm-c/Core.h +++ b/llvm/include/llvm-c/Core.h @@ -4020,8 +4020,13 @@ LLVMValueRef LLVMBuildIsNull(LLVMBuilderRef, LLVMValueRef Val, const char *Name); LLVMValueRef LLVMBuildIsNotNull(LLVMBuilderRef, LLVMValueRef Val, const char *Name); -LLVMValueRef LLVMBuildPtrDiff(LLVMBuilderRef, LLVMValueRef LHS, - LLVMValueRef RHS, const char *Name); +LLVM_ATTRIBUTE_C_DEPRECATED( + LLVMValueRef LLVMBuildPtrDiff(LLVMBuilderRef, LLVMValueRef LHS, + LLVMValueRef RHS, const char *Name), + "Use LLVMBuildPtrDiff2 instead to support opaque pointers"); +LLVMValueRef LLVMBuildPtrDiff2(LLVMBuilderRef, LLVMTypeRef ElemTy, + LLVMValueRef LHS, LLVMValueRef RHS, + const char *Name); LLVMValueRef LLVMBuildFence(LLVMBuilderRef B, LLVMAtomicOrdering ordering, LLVMBool singleThread, const char *Name); LLVMValueRef LLVMBuildAtomicRMW(LLVMBuilderRef B, LLVMAtomicRMWBinOp op, diff --git a/llvm/include/llvm-c/DebugInfo.h b/llvm/include/llvm-c/DebugInfo.h index d7fb898b60d2..a515533f38e2 100644 --- a/llvm/include/llvm-c/DebugInfo.h +++ b/llvm/include/llvm-c/DebugInfo.h @@ -1102,7 +1102,7 @@ LLVMMetadataRef LLVMDIBuilderGetOrCreateArray(LLVMDIBuilderRef Builder, * \param Length Length of the address operation array. */ LLVMMetadataRef LLVMDIBuilderCreateExpression(LLVMDIBuilderRef Builder, - int64_t *Addr, size_t Length); + uint64_t *Addr, size_t Length); /** * Create a new descriptor for the specified variable that does not have an @@ -1112,7 +1112,7 @@ LLVMMetadataRef LLVMDIBuilderCreateExpression(LLVMDIBuilderRef Builder, */ LLVMMetadataRef LLVMDIBuilderCreateConstantValueExpression(LLVMDIBuilderRef Builder, - int64_t Value); + uint64_t Value); /** * Create a new descriptor for the specified variable. diff --git a/llvm/include/llvm/ADT/APInt.h b/llvm/include/llvm/ADT/APInt.h index c2660502a419..b1fc85d3c09d 100644 --- a/llvm/include/llvm/ADT/APInt.h +++ b/llvm/include/llvm/ADT/APInt.h @@ -417,7 +417,7 @@ public: bool isIntN(unsigned N) const { return getActiveBits() <= N; } /// Check if this APInt has an N-bits signed integer value. - bool isSignedIntN(unsigned N) const { return getMinSignedBits() <= N; } + bool isSignedIntN(unsigned N) const { return getSignificantBits() <= N; } /// Check if this APInt's value is a power of two greater than zero. /// @@ -1069,8 +1069,9 @@ public: /// /// \returns true if *this < RHS when considered signed. bool slt(int64_t RHS) const { - return (!isSingleWord() && getMinSignedBits() > 64) ? isNegative() - : getSExtValue() < RHS; + return (!isSingleWord() && getSignificantBits() > 64) + ? isNegative() + : getSExtValue() < RHS; } /// Unsigned less or equal comparison @@ -1139,8 +1140,9 @@ public: /// /// \returns true if *this > RHS when considered signed. bool sgt(int64_t RHS) const { - return (!isSingleWord() && getMinSignedBits() > 64) ? !isNegative() - : getSExtValue() > RHS; + return (!isSingleWord() && getSignificantBits() > 64) + ? !isNegative() + : getSExtValue() > RHS; } /// Unsigned greater or equal comparison @@ -1450,7 +1452,12 @@ public: /// returns the smallest bit width that will retain the negative value. For /// example, -1 can be written as 0b1 or 0xFFFFFFFFFF. 0b1 is shorter and so /// for -1, this function will always return 1. - unsigned getMinSignedBits() const { return BitWidth - getNumSignBits() + 1; } + unsigned getSignificantBits() const { + return BitWidth - getNumSignBits() + 1; + } + + /// NOTE: This is soft-deprecated. Please use `getSignificantBits()` instead. + unsigned getMinSignedBits() const { return getSignificantBits(); } /// Get zero extended value /// @@ -1472,7 +1479,7 @@ public: int64_t getSExtValue() const { if (isSingleWord()) return SignExtend64(U.VAL, BitWidth); - assert(getMinSignedBits() <= 64 && "Too many bits for int64_t"); + assert(getSignificantBits() <= 64 && "Too many bits for int64_t"); return int64_t(U.pVal[0]); } diff --git a/llvm/include/llvm/ADT/AllocatorList.h b/llvm/include/llvm/ADT/AllocatorList.h index 404a657f27de..04d0afc9d076 100644 --- a/llvm/include/llvm/ADT/AllocatorList.h +++ b/llvm/include/llvm/ADT/AllocatorList.h @@ -13,7 +13,6 @@ #include "llvm/ADT/iterator.h" #include "llvm/ADT/simple_ilist.h" #include "llvm/Support/Allocator.h" -#include <algorithm> #include <cassert> #include <cstddef> #include <iterator> diff --git a/llvm/include/llvm/ADT/Any.h b/llvm/include/llvm/ADT/Any.h index e513586845a1..1b4f2c2fa985 100644 --- a/llvm/include/llvm/ADT/Any.h +++ b/llvm/include/llvm/ADT/Any.h @@ -15,7 +15,8 @@ #ifndef LLVM_ADT_ANY_H #define LLVM_ADT_ANY_H -#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/STLForwardCompat.h" +#include "llvm/Support/Compiler.h" #include <cassert> #include <memory> diff --git a/llvm/include/llvm/ADT/ArrayRef.h b/llvm/include/llvm/ADT/ArrayRef.h index 61f85cfc812b..b6896395dae8 100644 --- a/llvm/include/llvm/ADT/ArrayRef.h +++ b/llvm/include/llvm/ADT/ArrayRef.h @@ -141,7 +141,7 @@ namespace llvm { template <typename U, typename A> ArrayRef(const std::vector<U *, A> &Vec, std::enable_if_t<std::is_convertible<U *const *, T const *>::value> - * = 0) + * = nullptr) : Data(Vec.data()), Length(Vec.size()) {} /// @} diff --git a/llvm/include/llvm/ADT/BitVector.h b/llvm/include/llvm/ADT/BitVector.h index cd1964cbdd98..fff4a8f578d2 100644 --- a/llvm/include/llvm/ADT/BitVector.h +++ b/llvm/include/llvm/ADT/BitVector.h @@ -444,6 +444,12 @@ public: return (Bits[Idx / BITWORD_SIZE] & Mask) != 0; } + /// Return the last element in the vector. + bool back() const { + assert(!empty() && "Getting last element of empty vector."); + return (*this)[size() - 1]; + } + bool test(unsigned Idx) const { return (*this)[Idx]; } @@ -465,6 +471,12 @@ public: set(OldSize); } + /// Pop one bit from the end of the vector. + void pop_back() { + assert(!empty() && "Empty vector has no element to pop."); + resize(size() - 1); + } + /// Test if any common bits are set. bool anyCommon(const BitVector &RHS) const { unsigned ThisWords = Bits.size(); diff --git a/llvm/include/llvm/ADT/CoalescingBitVector.h b/llvm/include/llvm/ADT/CoalescingBitVector.h index 18803ecf209f..6935c255a099 100644 --- a/llvm/include/llvm/ADT/CoalescingBitVector.h +++ b/llvm/include/llvm/ADT/CoalescingBitVector.h @@ -15,12 +15,12 @@ #define LLVM_ADT_COALESCINGBITVECTOR_H #include "llvm/ADT/IntervalMap.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/iterator_range.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" -#include <algorithm> #include <initializer_list> namespace llvm { diff --git a/llvm/include/llvm/ADT/CombinationGenerator.h b/llvm/include/llvm/ADT/CombinationGenerator.h index ab6afd555726..a0bec68eaad6 100644 --- a/llvm/include/llvm/ADT/CombinationGenerator.h +++ b/llvm/include/llvm/ADT/CombinationGenerator.h @@ -28,7 +28,7 @@ #define LLVM_ADT_COMBINATIONGENERATOR_H #include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/STLFunctionalExtras.h" #include "llvm/ADT/SmallVector.h" #include <cassert> #include <cstring> diff --git a/llvm/include/llvm/ADT/DenseSet.h b/llvm/include/llvm/ADT/DenseSet.h index edce7c43773c..e767211a0900 100644 --- a/llvm/include/llvm/ADT/DenseSet.h +++ b/llvm/include/llvm/ADT/DenseSet.h @@ -17,7 +17,6 @@ #include "llvm/ADT/DenseMapInfo.h" #include "llvm/Support/MathExtras.h" #include "llvm/Support/type_traits.h" -#include <algorithm> #include <cstddef> #include <initializer_list> #include <iterator> diff --git a/llvm/include/llvm/ADT/DepthFirstIterator.h b/llvm/include/llvm/ADT/DepthFirstIterator.h index d4f173ca7caa..42ac61d7cf52 100644 --- a/llvm/include/llvm/ADT/DepthFirstIterator.h +++ b/llvm/include/llvm/ADT/DepthFirstIterator.h @@ -38,7 +38,6 @@ #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/iterator_range.h" #include <iterator> -#include <set> #include <utility> #include <vector> @@ -231,7 +230,7 @@ iterator_range<df_iterator<T>> depth_first(const T& G) { } // Provide global definitions of external depth first iterators... -template <class T, class SetTy = std::set<typename GraphTraits<T>::NodeRef>> +template <class T, class SetTy = df_iterator_default_set<typename GraphTraits<T>::NodeRef>> struct df_ext_iterator : public df_iterator<T, SetTy, true> { df_ext_iterator(const df_iterator<T, SetTy, true> &V) : df_iterator<T, SetTy, true>(V) {} @@ -280,7 +279,7 @@ iterator_range<idf_iterator<T>> inverse_depth_first(const T& G) { } // Provide global definitions of external inverse depth first iterators... -template <class T, class SetTy = std::set<typename GraphTraits<T>::NodeRef>> +template <class T, class SetTy = df_iterator_default_set<typename GraphTraits<T>::NodeRef>> struct idf_ext_iterator : public idf_iterator<T, SetTy, true> { idf_ext_iterator(const idf_iterator<T, SetTy, true> &V) : idf_iterator<T, SetTy, true>(V) {} diff --git a/llvm/include/llvm/ADT/GenericCycleInfo.h b/llvm/include/llvm/ADT/GenericCycleInfo.h index aad704301e43..7768253e121d 100644 --- a/llvm/include/llvm/ADT/GenericCycleInfo.h +++ b/llvm/include/llvm/ADT/GenericCycleInfo.h @@ -41,8 +41,8 @@ namespace llvm { -template <typename ContexT> class GenericCycleInfo; -template <typename ContexT> class GenericCycleInfoCompute; +template <typename ContextT> class GenericCycleInfo; +template <typename ContextT> class GenericCycleInfoCompute; /// A possibly irreducible generalization of a \ref Loop. template <typename ContextT> class GenericCycle { diff --git a/llvm/include/llvm/ADT/ImmutableMap.h b/llvm/include/llvm/ADT/ImmutableMap.h index 81b21a7319a7..f0e898cafaf9 100644 --- a/llvm/include/llvm/ADT/ImmutableMap.h +++ b/llvm/include/llvm/ADT/ImmutableMap.h @@ -140,44 +140,7 @@ public: bool isEmpty() const { return !Root; } - //===--------------------------------------------------===// - // Foreach - A limited form of map iteration. - //===--------------------------------------------------===// - -private: - template <typename Callback> - struct CBWrapper { - Callback C; - - void operator()(value_type_ref V) { C(V.first,V.second); } - }; - - template <typename Callback> - struct CBWrapperRef { - Callback &C; - - CBWrapperRef(Callback& c) : C(c) {} - - void operator()(value_type_ref V) { C(V.first,V.second); } - }; - public: - template <typename Callback> - void foreach(Callback& C) { - if (Root) { - CBWrapperRef<Callback> CB(C); - Root->foreach(CB); - } - } - - template <typename Callback> - void foreach() { - if (Root) { - CBWrapper<Callback> CB; - Root->foreach(CB); - } - } - //===--------------------------------------------------===// // For testing. //===--------------------------------------------------===// @@ -264,7 +227,7 @@ public: : Root(X.getRootWithoutRetain()), Factory(F.getTreeFactory()) {} static inline ImmutableMapRef getEmptyMap(FactoryTy *F) { - return ImmutableMapRef(0, F); + return ImmutableMapRef(nullptr, F); } void manualRetain() { @@ -345,7 +308,7 @@ public: /// which key is the highest in the ordering of keys in the map. This /// method returns NULL if the map is empty. value_type* getMaxElement() const { - return Root ? &(Root->getMaxElement()->getValue()) : 0; + return Root ? &(Root->getMaxElement()->getValue()) : nullptr; } //===--------------------------------------------------===// diff --git a/llvm/include/llvm/ADT/ImmutableSet.h b/llvm/include/llvm/ADT/ImmutableSet.h index 48b253d3b75e..8cef5acbafaa 100644 --- a/llvm/include/llvm/ADT/ImmutableSet.h +++ b/llvm/include/llvm/ADT/ImmutableSet.h @@ -169,20 +169,6 @@ public: /// is logarithmic in the size of the tree. bool contains(key_type_ref K) { return (bool) find(K); } - /// foreach - A member template the accepts invokes operator() on a functor - /// object (specified by Callback) for every node/subtree in the tree. - /// Nodes are visited using an inorder traversal. - template <typename Callback> - void foreach(Callback& C) { - if (ImutAVLTree* L = getLeft()) - L->foreach(C); - - C(value); - - if (ImutAVLTree* R = getRight()) - R->foreach(C); - } - /// validateTree - A utility method that checks that the balancing and /// ordering invariants of the tree are satisfied. It is a recursive /// method that returns the height of the tree, which is then consumed @@ -1063,12 +1049,6 @@ public: /// This method runs in constant time. bool isSingleton() const { return getHeight() == 1; } - template <typename Callback> - void foreach(Callback& C) { if (Root) Root->foreach(C); } - - template <typename Callback> - void foreach() { if (Root) { Callback C; Root->foreach(C); } } - //===--------------------------------------------------===// // Iterators. //===--------------------------------------------------===// diff --git a/llvm/include/llvm/ADT/MapVector.h b/llvm/include/llvm/ADT/MapVector.h index f9540999381a..d281166b3e19 100644 --- a/llvm/include/llvm/ADT/MapVector.h +++ b/llvm/include/llvm/ADT/MapVector.h @@ -18,7 +18,6 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallVector.h" -#include <algorithm> #include <cassert> #include <cstddef> #include <iterator> diff --git a/llvm/include/llvm/ADT/Optional.h b/llvm/include/llvm/ADT/Optional.h index cdbf2e353241..7d6b3e92f6b2 100644 --- a/llvm/include/llvm/ADT/Optional.h +++ b/llvm/include/llvm/ADT/Optional.h @@ -21,7 +21,6 @@ #include "llvm/Support/Compiler.h" #include "llvm/Support/type_traits.h" #include <cassert> -#include <memory> #include <new> #include <utility> @@ -34,12 +33,12 @@ namespace optional_detail { /// Storage for any type. // // The specialization condition intentionally uses -// llvm::is_trivially_copy_constructible instead of -// std::is_trivially_copy_constructible. GCC versions prior to 7.4 may -// instantiate the copy constructor of `T` when -// std::is_trivially_copy_constructible is instantiated. This causes -// compilation to fail if we query the trivially copy constructible property of -// a class which is not copy constructible. +// llvm::is_trivially_{copy/move}_constructible instead of +// std::is_trivially_{copy/move}_constructible. GCC versions prior to 7.4 may +// instantiate the copy/move constructor of `T` when +// std::is_trivially_{copy/move}_constructible is instantiated. This causes +// compilation to fail if we query the trivially copy/move constructible +// property of a class which is not copy/move constructible. // // The current implementation of OptionalStorage insists that in order to use // the trivial specialization, the value_type must be trivially copy @@ -50,12 +49,13 @@ namespace optional_detail { // // The move constructible / assignable conditions emulate the remaining behavior // of std::is_trivially_copyable. -template <typename T, bool = (llvm::is_trivially_copy_constructible<T>::value && - std::is_trivially_copy_assignable<T>::value && - (std::is_trivially_move_constructible<T>::value || - !std::is_move_constructible<T>::value) && - (std::is_trivially_move_assignable<T>::value || - !std::is_move_assignable<T>::value))> +template <typename T, + bool = (llvm::is_trivially_copy_constructible<T>::value && + std::is_trivially_copy_assignable<T>::value && + (llvm::is_trivially_move_constructible<T>::value || + !std::is_move_constructible<T>::value) && + (std::is_trivially_move_assignable<T>::value || + !std::is_move_assignable<T>::value))> class OptionalStorage { union { char empty; diff --git a/llvm/include/llvm/ADT/PriorityWorklist.h b/llvm/include/llvm/ADT/PriorityWorklist.h index 01dd59a2e71a..e9fbf296973d 100644 --- a/llvm/include/llvm/ADT/PriorityWorklist.h +++ b/llvm/include/llvm/ADT/PriorityWorklist.h @@ -19,7 +19,6 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" #include "llvm/Support/Compiler.h" -#include <algorithm> #include <cassert> #include <cstddef> #include <iterator> diff --git a/llvm/include/llvm/ADT/STLExtras.h b/llvm/include/llvm/ADT/STLExtras.h index 2d38e153c79e..c3200c926518 100644 --- a/llvm/include/llvm/ADT/STLExtras.h +++ b/llvm/include/llvm/ADT/STLExtras.h @@ -16,8 +16,10 @@ #ifndef LLVM_ADT_STLEXTRAS_H #define LLVM_ADT_STLEXTRAS_H +#include "llvm/ADT/identity.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/STLForwardCompat.h" +#include "llvm/ADT/STLFunctionalExtras.h" #include "llvm/ADT/iterator.h" #include "llvm/ADT/iterator_range.h" #include "llvm/Config/abi-breaking.h" @@ -200,65 +202,6 @@ template <size_t I, typename... Ts> using TypeAtIndex = std::tuple_element_t<I, std::tuple<Ts...>>; //===----------------------------------------------------------------------===// -// Extra additions to <functional> -//===----------------------------------------------------------------------===// - -template <class Ty> struct identity { - using argument_type = Ty; - - Ty &operator()(Ty &self) const { - return self; - } - const Ty &operator()(const Ty &self) const { - return self; - } -}; - -/// An efficient, type-erasing, non-owning reference to a callable. This is -/// intended for use as the type of a function parameter that is not used -/// after the function in question returns. -/// -/// This class does not own the callable, so it is not in general safe to store -/// a function_ref. -template<typename Fn> class function_ref; - -template<typename Ret, typename ...Params> -class function_ref<Ret(Params...)> { - Ret (*callback)(intptr_t callable, Params ...params) = nullptr; - intptr_t callable; - - template<typename Callable> - static Ret callback_fn(intptr_t callable, Params ...params) { - return (*reinterpret_cast<Callable*>(callable))( - std::forward<Params>(params)...); - } - -public: - function_ref() = default; - function_ref(std::nullptr_t) {} - - template <typename Callable> - function_ref( - Callable &&callable, - // This is not the copy-constructor. - std::enable_if_t<!std::is_same<remove_cvref_t<Callable>, - function_ref>::value> * = nullptr, - // Functor must be callable and return a suitable type. - std::enable_if_t<std::is_void<Ret>::value || - std::is_convertible<decltype(std::declval<Callable>()( - std::declval<Params>()...)), - Ret>::value> * = nullptr) - : callback(callback_fn<typename std::remove_reference<Callable>::type>), - callable(reinterpret_cast<intptr_t>(&callable)) {} - - Ret operator()(Params ...params) const { - return callback(callable, std::forward<Params>(params)...); - } - - explicit operator bool() const { return callback; } -}; - -//===----------------------------------------------------------------------===// // Extra additions to <iterator> //===----------------------------------------------------------------------===// @@ -416,20 +359,14 @@ auto reverse(ContainerTy &&C, return make_range(C.rbegin(), C.rend()); } -// Returns a std::reverse_iterator wrapped around the given iterator. -template <typename IteratorTy> -std::reverse_iterator<IteratorTy> make_reverse_iterator(IteratorTy It) { - return std::reverse_iterator<IteratorTy>(It); -} - // Returns an iterator_range over the given container which iterates in reverse. // Note that the container must have begin()/end() methods which return // bidirectional iterators for this to work. template <typename ContainerTy> auto reverse(ContainerTy &&C, std::enable_if_t<!has_rbegin<ContainerTy>::value> * = nullptr) { - return make_range(llvm::make_reverse_iterator(std::end(C)), - llvm::make_reverse_iterator(std::begin(C))); + return make_range(std::make_reverse_iterator(std::end(C)), + std::make_reverse_iterator(std::begin(C))); } /// An iterator adaptor that filters the elements of given inner iterators. diff --git a/llvm/include/llvm/ADT/STLFunctionalExtras.h b/llvm/include/llvm/ADT/STLFunctionalExtras.h new file mode 100644 index 000000000000..ebe1b1521a5d --- /dev/null +++ b/llvm/include/llvm/ADT/STLFunctionalExtras.h @@ -0,0 +1,76 @@ +//===- llvm/ADT/STLFunctionalExtras.h - Extras for <functional> -*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains some extension to <functional>. +// +// No library is required when using these functions. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_ADT_STLFUNCTIONALEXTRAS_H +#define LLVM_ADT_STLFUNCTIONALEXTRAS_H + +#include "llvm/ADT/STLForwardCompat.h" + +#include <type_traits> +#include <utility> +#include <cstdint> + +namespace llvm { + +//===----------------------------------------------------------------------===// +// Extra additions to <functional> +//===----------------------------------------------------------------------===// + +/// An efficient, type-erasing, non-owning reference to a callable. This is +/// intended for use as the type of a function parameter that is not used +/// after the function in question returns. +/// +/// This class does not own the callable, so it is not in general safe to store +/// a function_ref. +template<typename Fn> class function_ref; + +template<typename Ret, typename ...Params> +class function_ref<Ret(Params...)> { + Ret (*callback)(intptr_t callable, Params ...params) = nullptr; + intptr_t callable; + + template<typename Callable> + static Ret callback_fn(intptr_t callable, Params ...params) { + return (*reinterpret_cast<Callable*>(callable))( + std::forward<Params>(params)...); + } + +public: + function_ref() = default; + function_ref(std::nullptr_t) {} + + template <typename Callable> + function_ref( + Callable &&callable, + // This is not the copy-constructor. + std::enable_if_t<!std::is_same<remove_cvref_t<Callable>, + function_ref>::value> * = nullptr, + // Functor must be callable and return a suitable type. + std::enable_if_t<std::is_void<Ret>::value || + std::is_convertible<decltype(std::declval<Callable>()( + std::declval<Params>()...)), + Ret>::value> * = nullptr) + : callback(callback_fn<typename std::remove_reference<Callable>::type>), + callable(reinterpret_cast<intptr_t>(&callable)) {} + + Ret operator()(Params ...params) const { + return callback(callable, std::forward<Params>(params)...); + } + + explicit operator bool() const { return callback; } +}; + +} // end namespace llvm + +#endif // LLVM_ADT_STLFUNCTIONALEXTRAS_H diff --git a/llvm/include/llvm/ADT/ScopedHashTable.h b/llvm/include/llvm/ADT/ScopedHashTable.h index a5e57c6a16c2..48544961d095 100644 --- a/llvm/include/llvm/ADT/ScopedHashTable.h +++ b/llvm/include/llvm/ADT/ScopedHashTable.h @@ -197,7 +197,7 @@ public: using iterator = ScopedHashTableIterator<K, V, KInfo>; - iterator end() { return iterator(0); } + iterator end() { return iterator(nullptr); } iterator begin(const K &Key) { typename DenseMap<K, ValTy*, KInfo>::iterator I = diff --git a/llvm/include/llvm/ADT/SetVector.h b/llvm/include/llvm/ADT/SetVector.h index 32bcd50966cc..82d5e98afb5d 100644 --- a/llvm/include/llvm/ADT/SetVector.h +++ b/llvm/include/llvm/ADT/SetVector.h @@ -23,7 +23,6 @@ #include "llvm/ADT/DenseSet.h" #include "llvm/ADT/STLExtras.h" #include "llvm/Support/Compiler.h" -#include <algorithm> #include <cassert> #include <iterator> #include <vector> diff --git a/llvm/include/llvm/ADT/SmallBitVector.h b/llvm/include/llvm/ADT/SmallBitVector.h index 51ee5dbbce05..17be317a10d7 100644 --- a/llvm/include/llvm/ADT/SmallBitVector.h +++ b/llvm/include/llvm/ADT/SmallBitVector.h @@ -462,6 +462,12 @@ public: return getPointer()->operator[](Idx); } + /// Return the last element in the vector. + bool back() const { + assert(!empty() && "Getting last element of empty vector."); + return (*this)[size() - 1]; + } + bool test(unsigned Idx) const { return (*this)[Idx]; } @@ -471,6 +477,12 @@ public: resize(size() + 1, Val); } + /// Pop one bit from the end of the vector. + void pop_back() { + assert(!empty() && "Empty vector has no element to pop."); + resize(size() - 1); + } + /// Test if any common bits are set. bool anyCommon(const SmallBitVector &RHS) const { if (isSmall() && RHS.isSmall()) diff --git a/llvm/include/llvm/ADT/SmallSet.h b/llvm/include/llvm/ADT/SmallSet.h index 0600e528ee69..fe4f74eac85d 100644 --- a/llvm/include/llvm/ADT/SmallSet.h +++ b/llvm/include/llvm/ADT/SmallSet.h @@ -16,6 +16,7 @@ #include "llvm/ADT/None.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/iterator.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/type_traits.h" diff --git a/llvm/include/llvm/ADT/SmallString.h b/llvm/include/llvm/ADT/SmallString.h index 56b0639b98cc..81243af1f97d 100644 --- a/llvm/include/llvm/ADT/SmallString.h +++ b/llvm/include/llvm/ADT/SmallString.h @@ -70,16 +70,16 @@ public: /// Append from a list of StringRefs. void append(std::initializer_list<StringRef> Refs) { - size_t SizeNeeded = this->size(); + size_t CurrentSize = this->size(); + size_t SizeNeeded = CurrentSize; for (const StringRef &Ref : Refs) SizeNeeded += Ref.size(); - this->reserve(SizeNeeded); - auto CurEnd = this->end(); + this->resize_for_overwrite(SizeNeeded); for (const StringRef &Ref : Refs) { - this->uninitialized_copy(Ref.begin(), Ref.end(), CurEnd); - CurEnd += Ref.size(); + std::copy(Ref.begin(), Ref.end(), this->begin() + CurrentSize); + CurrentSize += Ref.size(); } - this->set_size(SizeNeeded); + assert(CurrentSize == this->size()); } /// @} diff --git a/llvm/include/llvm/ADT/SmallVector.h b/llvm/include/llvm/ADT/SmallVector.h index 804567ebe3f1..466acb83d466 100644 --- a/llvm/include/llvm/ADT/SmallVector.h +++ b/llvm/include/llvm/ADT/SmallVector.h @@ -13,10 +13,7 @@ #ifndef LLVM_ADT_SMALLVECTOR_H #define LLVM_ADT_SMALLVECTOR_H -#include "llvm/ADT/iterator_range.h" #include "llvm/Support/Compiler.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/MemAlloc.h" #include "llvm/Support/type_traits.h" #include <algorithm> #include <cassert> @@ -34,6 +31,8 @@ namespace llvm { +template <typename IteratorT> class iterator_range; + /// This is all the stuff common to all SmallVectors. /// /// The template parameter specifies the type which should be used to hold the @@ -72,15 +71,11 @@ public: LLVM_NODISCARD bool empty() const { return !Size; } +protected: /// Set the array size to \p N, which the current array must have enough /// capacity for. /// /// This does not construct or destroy any elements in the vector. - /// - /// Clients can use this in conjunction with capacity() to write past the end - /// of the buffer when they know that more elements are available, and only - /// update the size later. This avoids the cost of value initializing elements - /// which will only be overwritten. void set_size(size_t N) { assert(N <= capacity()); Size = N; @@ -588,6 +583,9 @@ public: } private: + // Make set_size() private to avoid misuse in subclasses. + using SuperClass::set_size; + template <bool ForOverwrite> void resizeImpl(size_type N) { if (N == this->size()) return; diff --git a/llvm/include/llvm/ADT/SparseMultiSet.h b/llvm/include/llvm/ADT/SparseMultiSet.h index fec0a70a0bef..f63cef936433 100644 --- a/llvm/include/llvm/ADT/SparseMultiSet.h +++ b/llvm/include/llvm/ADT/SparseMultiSet.h @@ -20,7 +20,7 @@ #ifndef LLVM_ADT_SPARSEMULTISET_H #define LLVM_ADT_SPARSEMULTISET_H -#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/identity.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/SparseSet.h" #include <cassert> diff --git a/llvm/include/llvm/ADT/SparseSet.h b/llvm/include/llvm/ADT/SparseSet.h index d8acf1ee2f3a..e66d76ad88e1 100644 --- a/llvm/include/llvm/ADT/SparseSet.h +++ b/llvm/include/llvm/ADT/SparseSet.h @@ -19,7 +19,7 @@ #ifndef LLVM_ADT_SPARSESET_H #define LLVM_ADT_SPARSESET_H -#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/identity.h" #include "llvm/ADT/SmallVector.h" #include "llvm/Support/AllocatorBase.h" #include <cassert> diff --git a/llvm/include/llvm/ADT/StringExtras.h b/llvm/include/llvm/ADT/StringExtras.h index 2ca672e7855b..81a0954226d6 100644 --- a/llvm/include/llvm/ADT/StringExtras.h +++ b/llvm/include/llvm/ADT/StringExtras.h @@ -29,14 +29,15 @@ namespace llvm { -template<typename T> class SmallVectorImpl; class raw_ostream; /// hexdigit - Return the hexadecimal character for the /// given number \p X (which should be less than 16). inline char hexdigit(unsigned X, bool LowerCase = false) { - const char HexChar = LowerCase ? 'a' : 'A'; - return X < 10 ? '0' + X : HexChar + X - 10; + assert(X < 16); + static const char LUT[] = "0123456789ABCDEF"; + const uint8_t Offset = LowerCase ? 32 : 0; + return LUT[X] | Offset; } /// Given an array of c-style strings terminated by a null pointer, construct @@ -164,23 +165,26 @@ inline std::string utohexstr(uint64_t X, bool LowerCase = false) { /// Convert buffer \p Input to its hexadecimal representation. /// The returned string is double the size of \p Input. -inline std::string toHex(StringRef Input, bool LowerCase = false) { - static const char *const LUT = "0123456789ABCDEF"; - const uint8_t Offset = LowerCase ? 32 : 0; - size_t Length = Input.size(); - - std::string Output; - Output.reserve(2 * Length); - for (size_t i = 0; i < Length; ++i) { - const unsigned char c = Input[i]; - Output.push_back(LUT[c >> 4] | Offset); - Output.push_back(LUT[c & 15] | Offset); +inline void toHex(ArrayRef<uint8_t> Input, bool LowerCase, + SmallVectorImpl<char> &Output) { + const size_t Length = Input.size(); + Output.resize_for_overwrite(Length * 2); + + for (size_t i = 0; i < Length; i++) { + const uint8_t c = Input[i]; + Output[i * 2 ] = hexdigit(c >> 4, LowerCase); + Output[i * 2 + 1] = hexdigit(c & 15, LowerCase); } - return Output; } inline std::string toHex(ArrayRef<uint8_t> Input, bool LowerCase = false) { - return toHex(toStringRef(Input), LowerCase); + SmallString<16> Output; + toHex(Input, LowerCase, Output); + return std::string(Output); +} + +inline std::string toHex(StringRef Input, bool LowerCase = false) { + return toHex(arrayRefFromStringRef(Input), LowerCase); } /// Store the binary representation of the two provided values, \p MSB and diff --git a/llvm/include/llvm/ADT/StringMap.h b/llvm/include/llvm/ADT/StringMap.h index 669956d41e0c..562a2ff1a192 100644 --- a/llvm/include/llvm/ADT/StringMap.h +++ b/llvm/include/llvm/ADT/StringMap.h @@ -14,6 +14,7 @@ #define LLVM_ADT_STRINGMAP_H #include "llvm/ADT/StringMapEntry.h" +#include "llvm/ADT/iterator.h" #include "llvm/Support/AllocatorBase.h" #include "llvm/Support/PointerLikeTypeTraits.h" #include <initializer_list> diff --git a/llvm/include/llvm/ADT/StringMapEntry.h b/llvm/include/llvm/ADT/StringMapEntry.h index 8bfad5540230..120d4f3ca4bc 100644 --- a/llvm/include/llvm/ADT/StringMapEntry.h +++ b/llvm/include/llvm/ADT/StringMapEntry.h @@ -15,7 +15,9 @@ #ifndef LLVM_ADT_STRINGMAPENTRY_H #define LLVM_ADT_STRINGMAPENTRY_H +#include "llvm/ADT/None.h" #include "llvm/ADT/StringRef.h" +#include "llvm/ADT/STLFunctionalExtras.h" namespace llvm { diff --git a/llvm/include/llvm/ADT/StringRef.h b/llvm/include/llvm/ADT/StringRef.h index 3950910f0635..118def2f43e1 100644 --- a/llvm/include/llvm/ADT/StringRef.h +++ b/llvm/include/llvm/ADT/StringRef.h @@ -9,7 +9,8 @@ #ifndef LLVM_ADT_STRINGREF_H #define LLVM_ADT_STRINGREF_H -#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/DenseMapInfo.h" +#include "llvm/ADT/STLFunctionalExtras.h" #include "llvm/ADT/iterator_range.h" #include "llvm/Support/Compiler.h" #include <algorithm> @@ -877,6 +878,25 @@ namespace llvm { return ltrim(Chars).rtrim(Chars); } + /// Detect the line ending style of the string. + /// + /// If the string contains a line ending, return the line ending character + /// sequence that is detected. Otherwise return '\n' for unix line endings. + /// + /// \return - The line ending character sequence. + LLVM_NODISCARD + StringRef detectEOL() const { + size_t Pos = find('\r'); + if (Pos == npos) { + // If there is no carriage return, assume unix + return "\n"; + } + if (Pos + 1 < Length && Data[Pos + 1] == '\n') + return "\r\n"; // Windows + if (Pos > 0 && Data[Pos - 1] == '\n') + return "\n\r"; // You monster! + return "\r"; // Classic Mac + } /// @} }; @@ -959,13 +979,7 @@ namespace llvm { reinterpret_cast<const char *>(~static_cast<uintptr_t>(1)), 0); } - static unsigned getHashValue(StringRef Val) { - assert(Val.data() != getEmptyKey().data() && - "Cannot hash the empty key!"); - assert(Val.data() != getTombstoneKey().data() && - "Cannot hash the tombstone key!"); - return (unsigned)(hash_value(Val)); - } + static unsigned getHashValue(StringRef Val); static bool isEqual(StringRef LHS, StringRef RHS) { if (RHS.data() == getEmptyKey().data()) diff --git a/llvm/include/llvm/ADT/StringSwitch.h b/llvm/include/llvm/ADT/StringSwitch.h index 726d67c8f24a..4b7882d7ca10 100644 --- a/llvm/include/llvm/ADT/StringSwitch.h +++ b/llvm/include/llvm/ADT/StringSwitch.h @@ -12,6 +12,7 @@ #ifndef LLVM_ADT_STRINGSWITCH_H #define LLVM_ADT_STRINGSWITCH_H +#include "llvm/ADT/Optional.h" #include "llvm/ADT/StringRef.h" #include "llvm/Support/Compiler.h" #include <cassert> diff --git a/llvm/include/llvm/ADT/Triple.h b/llvm/include/llvm/ADT/Triple.h index 5dbd4f16bfd5..0f0a7b08b5d3 100644 --- a/llvm/include/llvm/ADT/Triple.h +++ b/llvm/include/llvm/ADT/Triple.h @@ -107,9 +107,11 @@ public: enum SubArchType { NoSubArch, + ARMSubArch_v9_3a, ARMSubArch_v9_2a, ARMSubArch_v9_1a, ARMSubArch_v9, + ARMSubArch_v8_8a, ARMSubArch_v8_7a, ARMSubArch_v8_6a, ARMSubArch_v8_5a, @@ -270,9 +272,7 @@ public: /// Default constructor is the same as an empty string and leaves all /// triple fields unknown. - Triple() - : Data(), Arch(), SubArch(), Vendor(), OS(), Environment(), - ObjectFormat() {} + Triple() : Arch(), SubArch(), Vendor(), OS(), Environment(), ObjectFormat() {} explicit Triple(const Twine &Str); Triple(const Twine &ArchStr, const Twine &VendorStr, const Twine &OSStr); diff --git a/llvm/include/llvm/ADT/identity.h b/llvm/include/llvm/ADT/identity.h new file mode 100644 index 000000000000..f07bb6fb39f6 --- /dev/null +++ b/llvm/include/llvm/ADT/identity.h @@ -0,0 +1,34 @@ +//===- llvm/ADT/Identity.h - Provide std::identity from C++20 ---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file provides an implementation of std::identity from C++20. +// +// No library is required when using these functions. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_ADT_IDENTITY_H +#define LLVM_ADT_IDENTITY_H + + +namespace llvm { + +template <class Ty> struct identity { + using argument_type = Ty; + + Ty &operator()(Ty &self) const { + return self; + } + const Ty &operator()(const Ty &self) const { + return self; + } +}; + +} // end namespace llvm + +#endif // LLVM_ADT_IDENTITY_H diff --git a/llvm/include/llvm/ADT/ilist.h b/llvm/include/llvm/ADT/ilist.h index d5a1f286b177..b3aa26f2454d 100644 --- a/llvm/include/llvm/ADT/ilist.h +++ b/llvm/include/llvm/ADT/ilist.h @@ -103,7 +103,7 @@ template <class TraitsT, class NodeT> struct HasGetNext { template <size_t N> struct SFINAE {}; template <class U> - static Yes &test(U *I, decltype(I->getNext(&make<NodeT>())) * = 0); + static Yes &test(U *I, decltype(I->getNext(&make<NodeT>())) * = nullptr); template <class> static No &test(...); public: @@ -117,7 +117,7 @@ template <class TraitsT> struct HasCreateSentinel { typedef char No[2]; template <class U> - static Yes &test(U *I, decltype(I->createSentinel()) * = 0); + static Yes &test(U *I, decltype(I->createSentinel()) * = nullptr); template <class> static No &test(...); public: diff --git a/llvm/include/llvm/Analysis/AliasAnalysis.h b/llvm/include/llvm/Analysis/AliasAnalysis.h index 2770a1a9b277..d4febe6c1db9 100644 --- a/llvm/include/llvm/Analysis/AliasAnalysis.h +++ b/llvm/include/llvm/Analysis/AliasAnalysis.h @@ -60,7 +60,6 @@ class CatchReturnInst; class DominatorTree; class FenceInst; class Function; -class InvokeInst; class LoopInfo; class PreservedAnalyses; class TargetLibraryInfo; @@ -679,7 +678,7 @@ public: /// Checks if functions with the specified behavior are known to only write /// memory (or not access memory at all). - static bool doesNotReadMemory(FunctionModRefBehavior MRB) { + static bool onlyWritesMemory(FunctionModRefBehavior MRB) { return !isRefSet(createModRefInfo(MRB)); } @@ -1268,6 +1267,14 @@ bool isIdentifiedObject(const Value *V); /// IdentifiedObjects. bool isIdentifiedFunctionLocal(const Value *V); +/// Return true if Object memory is not visible after an unwind, in the sense +/// that program semantics cannot depend on Object containing any particular +/// value on unwind. If the RequiresNoCaptureBeforeUnwind out parameter is set +/// to true, then the memory is only not visible if the object has not been +/// captured prior to the unwind. Otherwise it is not visible even if captured. +bool isNotVisibleOnUnwind(const Value *Object, + bool &RequiresNoCaptureBeforeUnwind); + /// A manager for alias analyses. /// /// This class can have analyses registered with it and when run, it will run diff --git a/llvm/include/llvm/Analysis/AliasAnalysisEvaluator.h b/llvm/include/llvm/Analysis/AliasAnalysisEvaluator.h index 972eceaa3ba9..043b1b7ca2dc 100644 --- a/llvm/include/llvm/Analysis/AliasAnalysisEvaluator.h +++ b/llvm/include/llvm/Analysis/AliasAnalysisEvaluator.h @@ -31,17 +31,15 @@ namespace llvm { class AAResults; class AAEvaluator : public PassInfoMixin<AAEvaluator> { - int64_t FunctionCount; - int64_t NoAliasCount, MayAliasCount, PartialAliasCount, MustAliasCount; - int64_t NoModRefCount, ModCount, RefCount, ModRefCount; - int64_t MustCount, MustRefCount, MustModCount, MustModRefCount; + int64_t FunctionCount = 0; + int64_t NoAliasCount = 0, MayAliasCount = 0, PartialAliasCount = 0; + int64_t MustAliasCount = 0; + int64_t NoModRefCount = 0, ModCount = 0, RefCount = 0, ModRefCount = 0; + int64_t MustCount = 0, MustRefCount = 0, MustModCount = 0; + int64_t MustModRefCount = 0; public: - AAEvaluator() - : FunctionCount(), NoAliasCount(), MayAliasCount(), PartialAliasCount(), - MustAliasCount(), NoModRefCount(), ModCount(), RefCount(), - ModRefCount(), MustCount(), MustRefCount(), MustModCount(), - MustModRefCount() {} + AAEvaluator() = default; AAEvaluator(AAEvaluator &&Arg) : FunctionCount(Arg.FunctionCount), NoAliasCount(Arg.NoAliasCount), MayAliasCount(Arg.MayAliasCount), diff --git a/llvm/include/llvm/Analysis/BasicAliasAnalysis.h b/llvm/include/llvm/Analysis/BasicAliasAnalysis.h index ed9d1ba4c5a7..97dda58109e9 100644 --- a/llvm/include/llvm/Analysis/BasicAliasAnalysis.h +++ b/llvm/include/llvm/Analysis/BasicAliasAnalysis.h @@ -25,7 +25,6 @@ namespace llvm { -struct AAMDNodes; class AssumptionCache; class BasicBlock; class DataLayout; @@ -58,7 +57,7 @@ public: BasicAAResult(const DataLayout &DL, const Function &F, const TargetLibraryInfo &TLI, AssumptionCache &AC, DominatorTree *DT = nullptr, PhiValues *PV = nullptr) - : AAResultBase(), DL(DL), F(F), TLI(TLI), AC(AC), DT(DT), PV(PV) {} + : DL(DL), F(F), TLI(TLI), AC(AC), DT(DT), PV(PV) {} BasicAAResult(const BasicAAResult &Arg) : AAResultBase(Arg), DL(Arg.DL), F(Arg.F), TLI(Arg.TLI), AC(Arg.AC), diff --git a/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h b/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h index f581b18bff17..858dd369dd0b 100644 --- a/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h +++ b/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h @@ -14,6 +14,7 @@ #ifndef LLVM_ANALYSIS_BLOCKFREQUENCYINFOIMPL_H #define LLVM_ANALYSIS_BLOCKFREQUENCYINFOIMPL_H +#include "llvm/ADT/BitVector.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/DenseSet.h" #include "llvm/ADT/GraphTraits.h" @@ -1451,7 +1452,7 @@ void BlockFrequencyInfoImpl<BT>::iterativeInference( // frequencies need to be updated based on the incoming edges. // The set is dynamic and changes after every update. Initially all blocks // with a positive frequency are active - auto IsActive = std::vector<bool>(Freq.size(), false); + auto IsActive = BitVector(Freq.size(), false); std::queue<size_t> ActiveSet; for (size_t I = 0; I < Freq.size(); I++) { if (Freq[I] > 0) { diff --git a/llvm/include/llvm/Analysis/CFLAliasAnalysisUtils.h b/llvm/include/llvm/Analysis/CFLAliasAnalysisUtils.h index 02f999a5b913..2eae2824bec3 100644 --- a/llvm/include/llvm/Analysis/CFLAliasAnalysisUtils.h +++ b/llvm/include/llvm/Analysis/CFLAliasAnalysisUtils.h @@ -50,8 +50,8 @@ static inline const Function *parentFunctionOfValue(const Value *Val) { if (auto *Arg = dyn_cast<Argument>(Val)) return Arg->getParent(); return nullptr; +} } // namespace cflaa } // namespace llvm -} #endif // LLVM_ANALYSIS_CFLALIASANALYSISUTILS_H diff --git a/llvm/include/llvm/Analysis/ConstantFolding.h b/llvm/include/llvm/Analysis/ConstantFolding.h index 45fb879f0c1f..37258c80e3a3 100644 --- a/llvm/include/llvm/Analysis/ConstantFolding.h +++ b/llvm/include/llvm/Analysis/ConstantFolding.h @@ -148,12 +148,11 @@ Constant *ConstantFoldLoadFromConstPtr(Constant *C, Type *Ty, APInt Offset, Constant *ConstantFoldLoadFromConstPtr(Constant *C, Type *Ty, const DataLayout &DL); -/// ConstantFoldLoadThroughGEPConstantExpr - Given a constant and a -/// getelementptr constantexpr, return the constant value being addressed by the -/// constant expression, or null if something is funny and we can't decide. -Constant *ConstantFoldLoadThroughGEPConstantExpr(Constant *C, ConstantExpr *CE, - Type *Ty, - const DataLayout &DL); +/// If C is a uniform value where all bits are the same (either all zero, all +/// ones, all undef or all poison), return the corresponding uniform value in +/// the new type. If the value is not uniform or the result cannot be +/// represented, return null. +Constant *ConstantFoldLoadFromUniformValue(Constant *C, Type *Ty); /// canConstantFoldCallTo - Return true if its even possible to fold a call to /// the specified function. diff --git a/llvm/include/llvm/Analysis/ConstraintSystem.h b/llvm/include/llvm/Analysis/ConstraintSystem.h index d5b8f208172b..d7800f578325 100644 --- a/llvm/include/llvm/Analysis/ConstraintSystem.h +++ b/llvm/include/llvm/Analysis/ConstraintSystem.h @@ -73,7 +73,7 @@ public: return R; } - bool isConditionImplied(SmallVector<int64_t, 8> R); + bool isConditionImplied(SmallVector<int64_t, 8> R) const; void popLastConstraint() { Constraints.pop_back(); } diff --git a/llvm/include/llvm/Analysis/DDG.h b/llvm/include/llvm/Analysis/DDG.h index 51dd4a738f00..4ea589ec7efc 100644 --- a/llvm/include/llvm/Analysis/DDG.h +++ b/llvm/include/llvm/Analysis/DDG.h @@ -52,7 +52,7 @@ public: }; DDGNode() = delete; - DDGNode(const NodeKind K) : DDGNodeBase(), Kind(K) {} + DDGNode(const NodeKind K) : Kind(K) {} DDGNode(const DDGNode &N) : DDGNodeBase(N), Kind(N.Kind) {} DDGNode(DDGNode &&N) : DDGNodeBase(std::move(N)), Kind(N.Kind) {} virtual ~DDGNode() = 0; diff --git a/llvm/include/llvm/Analysis/DOTGraphTraitsPass.h b/llvm/include/llvm/Analysis/DOTGraphTraitsPass.h index 59737744f576..d8021907b5b2 100644 --- a/llvm/include/llvm/Analysis/DOTGraphTraitsPass.h +++ b/llvm/include/llvm/Analysis/DOTGraphTraitsPass.h @@ -181,6 +181,25 @@ private: std::string Name; }; +template <typename GraphT> +void WriteDOTGraphToFile(Function &F, GraphT &&Graph, + std::string FileNamePrefix, bool IsSimple) { + std::string Filename = FileNamePrefix + "." + F.getName().str() + ".dot"; + std::error_code EC; + + errs() << "Writing '" << Filename << "'..."; + + raw_fd_ostream File(Filename, EC, sys::fs::OF_TextWithCRLF); + std::string GraphName = DOTGraphTraits<GraphT>::getGraphName(Graph); + std::string Title = GraphName + " for '" + F.getName().str() + "' function"; + + if (!EC) + WriteGraph(File, Graph, IsSimple, Title); + else + errs() << " error opening file for writing!"; + errs() << "\n"; +} + } // end namespace llvm #endif diff --git a/llvm/include/llvm/Analysis/DependenceAnalysis.h b/llvm/include/llvm/Analysis/DependenceAnalysis.h index 305c9b1d88f2..8c852e85b04a 100644 --- a/llvm/include/llvm/Analysis/DependenceAnalysis.h +++ b/llvm/include/llvm/Analysis/DependenceAnalysis.h @@ -74,12 +74,8 @@ namespace llvm { Dependence &operator=(Dependence &&) = default; public: - Dependence(Instruction *Source, - Instruction *Destination) : - Src(Source), - Dst(Destination), - NextPredecessor(nullptr), - NextSuccessor(nullptr) {} + Dependence(Instruction *Source, Instruction *Destination) + : Src(Source), Dst(Destination) {} virtual ~Dependence() {} /// Dependence::DVEntry - Each level in the distance/direction vector @@ -99,9 +95,10 @@ namespace llvm { bool PeelFirst : 1; // Peeling the first iteration will break dependence. bool PeelLast : 1; // Peeling the last iteration will break the dependence. bool Splitable : 1; // Splitting the loop will break dependence. - const SCEV *Distance; // NULL implies no distance available. - DVEntry() : Direction(ALL), Scalar(true), PeelFirst(false), - PeelLast(false), Splitable(false), Distance(nullptr) { } + const SCEV *Distance = nullptr; // NULL implies no distance available. + DVEntry() + : Direction(ALL), Scalar(true), PeelFirst(false), PeelLast(false), + Splitable(false) {} }; /// getSrc - Returns the source instruction for this dependence. @@ -200,7 +197,7 @@ namespace llvm { private: Instruction *Src, *Dst; - const Dependence *NextPredecessor, *NextSuccessor; + const Dependence *NextPredecessor = nullptr, *NextSuccessor = nullptr; friend class DependenceInfo; }; diff --git a/llvm/include/llvm/Analysis/DivergenceAnalysis.h b/llvm/include/llvm/Analysis/DivergenceAnalysis.h index 6f759a81fdef..c52b42ae8dc2 100644 --- a/llvm/include/llvm/Analysis/DivergenceAnalysis.h +++ b/llvm/include/llvm/Analysis/DivergenceAnalysis.h @@ -22,7 +22,6 @@ #include <vector> namespace llvm { -class Module; class Value; class Instruction; class Loop; @@ -147,7 +146,7 @@ class DivergenceInfo { // analysis can run indefinitely. We set ContainsIrreducible and no // analysis is actually performed on the function. All values in // this function are conservatively reported as divergent instead. - bool ContainsIrreducible; + bool ContainsIrreducible = false; std::unique_ptr<SyncDependenceAnalysis> SDA; std::unique_ptr<DivergenceAnalysisImpl> DA; diff --git a/llvm/include/llvm/Analysis/DomPrinter.h b/llvm/include/llvm/Analysis/DomPrinter.h index a177f877b295..e6df12d88072 100644 --- a/llvm/include/llvm/Analysis/DomPrinter.h +++ b/llvm/include/llvm/Analysis/DomPrinter.h @@ -14,6 +14,20 @@ #ifndef LLVM_ANALYSIS_DOMPRINTER_H #define LLVM_ANALYSIS_DOMPRINTER_H +#include "llvm/IR/PassManager.h" + +namespace llvm { +class DomTreePrinterPass : public PassInfoMixin<DomTreePrinterPass> { +public: + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); +}; + +class DomTreeOnlyPrinterPass : public PassInfoMixin<DomTreeOnlyPrinterPass> { +public: + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); +}; +} // namespace llvm + namespace llvm { class FunctionPass; FunctionPass *createDomPrinterPass(); diff --git a/llvm/include/llvm/Analysis/IRSimilarityIdentifier.h b/llvm/include/llvm/Analysis/IRSimilarityIdentifier.h index 51c5c620230b..7b81d5754930 100644 --- a/llvm/include/llvm/Analysis/IRSimilarityIdentifier.h +++ b/llvm/include/llvm/Analysis/IRSimilarityIdentifier.h @@ -121,13 +121,23 @@ struct IRInstructionData /// and is used when checking when two instructions are considered similar. /// If either instruction is not legal, the instructions are automatically not /// considered similar. - bool Legal; + bool Legal = false; /// This is only relevant if we are wrapping a CmpInst where we needed to /// change the predicate of a compare instruction from a greater than form /// to a less than form. It is None otherwise. Optional<CmpInst::Predicate> RevisedPredicate; + /// This is only relevant if we are wrapping a CallInst. If we are requiring + /// that the function calls have matching names as well as types, and the + /// call is not an indirect call, this will hold the name of the function. If + /// it is an indirect string, it will be the empty string. However, if this + /// requirement is not in place it will be the empty string regardless of the + /// function call type. The value held here is used to create the hash of the + /// instruction, and check to make sure two instructions are close to one + /// another. + Optional<std::string> CalleeName; + /// This structure holds the distances of how far "ahead of" or "behind" the /// target blocks of a branch, or the incoming blocks of a phi nodes are. /// If the value is negative, it means that the block was registered before @@ -168,6 +178,10 @@ struct IRInstructionData /// instruction. the IRInstructionData must be wrapping a CmpInst. CmpInst::Predicate getPredicate() const; + /// Get the callee name that the call instruction is using for hashing the + /// instruction. The IRInstructionData must be wrapping a CallInst. + StringRef getCalleeName() const; + /// A function that swaps the predicates to their less than form if they are /// in a greater than form. Otherwise, the predicate is unchanged. /// @@ -185,6 +199,31 @@ struct IRInstructionData void setBranchSuccessors(DenseMap<BasicBlock *, unsigned> &BasicBlockToInteger); + /// For an IRInstructionData containing a CallInst, set the function name + /// appropriately. This will be an empty string if it is an indirect call, + /// or we are not matching by name of the called function. It will be the + /// name of the function if \p MatchByName is true and it is not an indirect + /// call. We may decide not to match by name in order to expand the + /// size of the regions we can match. If a function name has the same type + /// signature, but the different name, the region of code is still almost the + /// same. Since function names can be treated as constants, the name itself + /// could be extrapolated away. However, matching by name provides a + /// specificity and more "identical" code than not matching by name. + /// + /// \param MatchByName - A flag to mark whether we are using the called + /// function name as a differentiating parameter. + void setCalleeName(bool MatchByName = true); + + /// For an IRInstructionData containing a PHINode, finds the + /// relative distances from the incoming basic block to the current block by + /// taking the difference of the number assigned to the current basic block + /// and the incoming basic block of the branch. + /// + /// \param BasicBlockToInteger - The mapping of basic blocks to their location + /// in the module. + void + setPHIPredecessors(DenseMap<BasicBlock *, unsigned> &BasicBlockToInteger); + /// Hashes \p Value based on its opcode, types, and operand types. /// Two IRInstructionData instances produce the same hash when they perform /// the same operation. @@ -223,12 +262,14 @@ struct IRInstructionData llvm::hash_value(ID.Inst->getType()), llvm::hash_value(ID.getPredicate()), llvm::hash_combine_range(OperTypes.begin(), OperTypes.end())); - else if (CallInst *CI = dyn_cast<CallInst>(ID.Inst)) + else if (isa<CallInst>(ID.Inst)) { + std::string FunctionName = *ID.CalleeName; return llvm::hash_combine( llvm::hash_value(ID.Inst->getOpcode()), llvm::hash_value(ID.Inst->getType()), - llvm::hash_value(CI->getCalledFunction()->getName().str()), + llvm::hash_value(ID.Inst->getType()), llvm::hash_value(FunctionName), llvm::hash_combine_range(OperTypes.begin(), OperTypes.end())); + } return llvm::hash_combine( llvm::hash_value(ID.Inst->getOpcode()), llvm::hash_value(ID.Inst->getType()), @@ -346,6 +387,10 @@ struct IRInstructionMapper { /// to be considered for similarity. bool HaveLegalRange = false; + /// Marks whether we should use exact function names, as well as types to + /// find similarity between calls. + bool EnableMatchCallsByName = false; + /// This allocator pointer is in charge of holding on to the IRInstructionData /// so it is not deallocated until whatever external tool is using it is done /// with the information. @@ -462,8 +507,11 @@ struct IRInstructionMapper { return Legal; return Illegal; } - // TODO: Determine a scheme to resolve when the labels are similar enough. - InstrType visitPHINode(PHINode &PN) { return Illegal; } + InstrType visitPHINode(PHINode &PN) { + if (EnableBranches) + return Legal; + return Illegal; + } // TODO: Handle allocas. InstrType visitAllocaInst(AllocaInst &AI) { return Illegal; } // We exclude variable argument instructions since variable arguments @@ -483,7 +531,10 @@ struct IRInstructionMapper { // is not an indirect call. InstrType visitCallInst(CallInst &CI) { Function *F = CI.getCalledFunction(); - if (!F || CI.isIndirectCall() || !F->hasName()) + bool IsIndirectCall = CI.isIndirectCall(); + if (IsIndirectCall && !EnableIndirectCalls) + return Illegal; + if (!F && !IsIndirectCall) return Illegal; return Legal; } @@ -498,6 +549,10 @@ struct IRInstructionMapper { // The flag variable that lets the classifier know whether we should // allow branches to be checked for similarity. bool EnableBranches = false; + + // The flag variable that lets the classifier know whether we should + // allow indirect calls to be considered legal instructions. + bool EnableIndirectCalls = false; }; /// Maps an Instruction to a member of InstrType. @@ -882,9 +937,12 @@ typedef std::vector<SimilarityGroup> SimilarityGroupList; /// analyzing the module. class IRSimilarityIdentifier { public: - IRSimilarityIdentifier(bool MatchBranches = true) + IRSimilarityIdentifier(bool MatchBranches = true, + bool MatchIndirectCalls = true, + bool MatchCallsWithName = false) : Mapper(&InstDataAllocator, &InstDataListAllocator), - EnableBranches(MatchBranches) {} + EnableBranches(MatchBranches), EnableIndirectCalls(MatchIndirectCalls), + EnableMatchingCallsByName(MatchCallsWithName) {} private: /// Map the instructions in the module to unsigned integers, using mapping @@ -964,6 +1022,15 @@ private: /// similarity, or only look within basic blocks. bool EnableBranches = true; + /// The flag variable that marks whether we allow indirect calls to be checked + /// for similarity, or exclude them as a legal instruction. + bool EnableIndirectCalls = true; + + /// The flag variable that marks whether we allow calls to be marked as + /// similar if they do not have the same name, only the same calling + /// convention, attributes and type signature. + bool EnableMatchingCallsByName = true; + /// The SimilarityGroups found with the most recent run of \ref /// findSimilarity. None if there is no recent run. Optional<SimilarityGroupList> SimilarityCandidates; diff --git a/llvm/include/llvm/Analysis/IVDescriptors.h b/llvm/include/llvm/Analysis/IVDescriptors.h index 9858a46d16a2..dec488a6f26d 100644 --- a/llvm/include/llvm/Analysis/IVDescriptors.h +++ b/llvm/include/llvm/Analysis/IVDescriptors.h @@ -77,10 +77,12 @@ public: RecurrenceDescriptor(Value *Start, Instruction *Exit, RecurKind K, FastMathFlags FMF, Instruction *ExactFP, Type *RT, bool Signed, bool Ordered, - SmallPtrSetImpl<Instruction *> &CI) + SmallPtrSetImpl<Instruction *> &CI, + unsigned MinWidthCastToRecurTy) : StartValue(Start), LoopExitInstr(Exit), Kind(K), FMF(FMF), ExactFPMathInst(ExactFP), RecurrenceType(RT), IsSigned(Signed), - IsOrdered(Ordered) { + IsOrdered(Ordered), + MinWidthCastToRecurrenceType(MinWidthCastToRecurTy) { CastInsts.insert(CI.begin(), CI.end()); } @@ -251,6 +253,11 @@ public: /// recurrence. const SmallPtrSet<Instruction *, 8> &getCastInsts() const { return CastInsts; } + /// Returns the minimum width used by the recurrence in bits. + unsigned getMinWidthCastToRecurrenceTypeInBits() const { + return MinWidthCastToRecurrenceType; + } + /// Returns true if all source operands of the recurrence are SExtInsts. bool isSigned() const { return IsSigned; } @@ -291,6 +298,8 @@ private: bool IsOrdered = false; // Instructions used for type-promoting the recurrence. SmallPtrSet<Instruction *, 8> CastInsts; + // The minimum width used by the recurrence. + unsigned MinWidthCastToRecurrenceType; }; /// A struct for saving information about induction variables. diff --git a/llvm/include/llvm/Analysis/IVUsers.h b/llvm/include/llvm/Analysis/IVUsers.h index e2026a4d5875..390d09848dde 100644 --- a/llvm/include/llvm/Analysis/IVUsers.h +++ b/llvm/include/llvm/Analysis/IVUsers.h @@ -28,7 +28,6 @@ class Value; class ScalarEvolution; class SCEV; class IVUsers; -class DataLayout; /// IVStrideUse - Keep track of one use of a strided induction variable. /// The Expr member keeps track of the expression, User is the actual user diff --git a/llvm/include/llvm/Analysis/InlineAdvisor.h b/llvm/include/llvm/Analysis/InlineAdvisor.h index 9f9bc3a5e71b..0103ee7f8386 100644 --- a/llvm/include/llvm/Analysis/InlineAdvisor.h +++ b/llvm/include/llvm/Analysis/InlineAdvisor.h @@ -10,6 +10,7 @@ #define LLVM_ANALYSIS_INLINEADVISOR_H #include "llvm/Analysis/InlineCost.h" +#include "llvm/Analysis/LazyCallGraph.h" #include "llvm/Analysis/Utils/ImportedFunctionsInliningStatistics.h" #include "llvm/Config/llvm-config.h" #include "llvm/IR/PassManager.h" @@ -65,7 +66,9 @@ public: /// Call after inlining succeeded, and did not result in deleting the callee. void recordInlining(); - /// Call after inlining succeeded, and resulted in deleting the callee. + /// Call after inlining succeeded, and results in the callee being + /// delete-able, meaning, it has no more users, and will be cleaned up + /// subsequently. void recordInliningWithCalleeDeleted(); /// Call after the decision for a call site was to not inline. @@ -159,14 +162,13 @@ public: /// This must be called when the Inliner pass is exited, as function passes /// may be run subsequently. This allows an implementation of InlineAdvisor - /// to prepare for a partial update. - virtual void onPassExit() {} + /// to prepare for a partial update, based on the optional SCC. + virtual void onPassExit(LazyCallGraph::SCC *SCC = nullptr) {} - /// Called when the module is invalidated. We let the advisor implementation - /// decide what to refresh - in the case of the development mode - /// implementation, for example, we wouldn't want to delete the whole object - /// and need to re-load the model evaluator. - virtual void onModuleInvalidated() {} + /// Support for printer pass + virtual void print(raw_ostream &OS) const { + OS << "Unimplemented InlineAdvisor print\n"; + } protected: InlineAdvisor(Module &M, FunctionAnalysisManager &FAM); @@ -178,19 +180,6 @@ protected: FunctionAnalysisManager &FAM; std::unique_ptr<ImportedFunctionsInliningStatistics> ImportedFunctionsStats; - /// We may want to defer deleting functions to after the inlining for a whole - /// module has finished. This allows us to reliably use function pointers as - /// unique identifiers, as an efficient implementation detail of the - /// InlineAdvisor. Otherwise, it is possible the memory allocator - /// re-allocate Function objects at the same address of a deleted Function; - /// and Functions are potentially created during the function passes called - /// after each SCC inlining (e.g. argument promotion does that). - void freeDeletedFunctions(); - - bool isFunctionDeleted(const Function *F) const { - return DeletedFunctions.count(F); - } - enum class MandatoryInliningKind { NotMandatory, Always, Never }; static MandatoryInliningKind getMandatoryKind(CallBase &CB, @@ -201,8 +190,6 @@ protected: private: friend class InlineAdvice; - void markFunctionAsDeleted(Function *F); - std::unordered_set<const Function *> DeletedFunctions; }; /// The default (manual heuristics) implementation of the InlineAdvisor. This @@ -217,8 +204,6 @@ public: private: std::unique_ptr<InlineAdvice> getAdviceImpl(CallBase &CB) override; - void onPassExit() override { freeDeletedFunctions(); } - InlineParams Params; }; @@ -232,8 +217,6 @@ public: Result(Module &M, ModuleAnalysisManager &MAM) : M(M), MAM(MAM) {} bool invalidate(Module &, const PreservedAnalyses &PA, ModuleAnalysisManager::Invalidator &) { - if (Advisor && !PA.areAllPreserved()) - Advisor->onModuleInvalidated(); // Check whether the analysis has been explicitly invalidated. Otherwise, // it's stateless and remains preserved. auto PAC = PA.getChecker<InlineAdvisorAnalysis>(); @@ -252,16 +235,23 @@ public: Result run(Module &M, ModuleAnalysisManager &MAM) { return Result(M, MAM); } }; -#ifdef LLVM_HAVE_TF_AOT +/// Printer pass for the FunctionPropertiesAnalysis results. +class InlineAdvisorAnalysisPrinterPass + : public PassInfoMixin<InlineAdvisorAnalysisPrinterPass> { + raw_ostream &OS; + +public: + explicit InlineAdvisorAnalysisPrinterPass(raw_ostream &OS) : OS(OS) {} + + PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM); +}; + std::unique_ptr<InlineAdvisor> getReleaseModeAdvisor(Module &M, ModuleAnalysisManager &MAM); -#endif -#ifdef LLVM_HAVE_TF_API std::unique_ptr<InlineAdvisor> getDevelopmentModeAdvisor(Module &M, ModuleAnalysisManager &MAM, std::function<bool(CallBase &)> GetDefaultAdvice); -#endif // Default (manual policy) decision making helper APIs. Shared with the legacy // pass manager inliner. diff --git a/llvm/include/llvm/Analysis/InlineCost.h b/llvm/include/llvm/Analysis/InlineCost.h index 776749b9a07f..f86ee5a14874 100644 --- a/llvm/include/llvm/Analysis/InlineCost.h +++ b/llvm/include/llvm/Analysis/InlineCost.h @@ -21,7 +21,6 @@ #include <climits> namespace llvm { -class AssumptionCacheTracker; class BlockFrequencyInfo; class CallBase; class DataLayout; diff --git a/llvm/include/llvm/Analysis/InlineOrder.h b/llvm/include/llvm/Analysis/InlineOrder.h index def3192356f4..feefa9b9ddd1 100644 --- a/llvm/include/llvm/Analysis/InlineOrder.h +++ b/llvm/include/llvm/Analysis/InlineOrder.h @@ -20,7 +20,6 @@ namespace llvm { class CallBase; class Function; -class Module; template <typename T> class InlineOrder { public: diff --git a/llvm/include/llvm/Analysis/InstSimplifyFolder.h b/llvm/include/llvm/Analysis/InstSimplifyFolder.h new file mode 100644 index 000000000000..54ef1ddf6085 --- /dev/null +++ b/llvm/include/llvm/Analysis/InstSimplifyFolder.h @@ -0,0 +1,255 @@ +//===- InstSimplifyFolder.h - InstSimplify folding helper --------*- C++-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines the InstSimplifyFolder class, a helper for IRBuilder. +// It provides IRBuilder with a set of methods for folding operations to +// existing values using InstructionSimplify. At the moment, only a subset of +// the implementation uses InstructionSimplify. The rest of the implementation +// only folds constants. +// +// The folder also applies target-specific constant folding. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_ANALYSIS_INSTSIMPLIFYFOLDER_H +#define LLVM_ANALYSIS_INSTSIMPLIFYFOLDER_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/Analysis/InstructionSimplify.h" +#include "llvm/Analysis/TargetFolder.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/IRBuilderFolder.h" +#include "llvm/IR/InstrTypes.h" +#include "llvm/IR/Instruction.h" + +namespace llvm { + +/// InstSimplifyFolder - Use InstructionSimplify to fold operations to existing +/// values. Also applies target-specific constant folding when not using +/// InstructionSimplify. +class InstSimplifyFolder final : public IRBuilderFolder { + TargetFolder ConstFolder; + SimplifyQuery SQ; + + virtual void anchor(); + +public: + InstSimplifyFolder(const DataLayout &DL) : ConstFolder(DL), SQ(DL) {} + + //===--------------------------------------------------------------------===// + // Value-based folders. + // + // Return an existing value or a constant if the operation can be simplified. + // Otherwise return nullptr. + //===--------------------------------------------------------------------===// + Value *FoldAdd(Value *LHS, Value *RHS, bool HasNUW = false, + bool HasNSW = false) const override { + return SimplifyAddInst(LHS, RHS, HasNUW, HasNSW, SQ); + } + + Value *FoldAnd(Value *LHS, Value *RHS) const override { + return SimplifyAndInst(LHS, RHS, SQ); + } + + Value *FoldOr(Value *LHS, Value *RHS) const override { + return SimplifyOrInst(LHS, RHS, SQ); + } + + Value *FoldICmp(CmpInst::Predicate P, Value *LHS, Value *RHS) const override { + return SimplifyICmpInst(P, LHS, RHS, SQ); + } + + Value *FoldGEP(Type *Ty, Value *Ptr, ArrayRef<Value *> IdxList, + bool IsInBounds = false) const override { + return SimplifyGEPInst(Ty, Ptr, IdxList, IsInBounds, SQ); + } + + Value *FoldSelect(Value *C, Value *True, Value *False) const override { + return SimplifySelectInst(C, True, False, SQ); + } + + //===--------------------------------------------------------------------===// + // Binary Operators + //===--------------------------------------------------------------------===// + + Value *CreateFAdd(Constant *LHS, Constant *RHS) const override { + return ConstFolder.CreateFAdd(LHS, RHS); + } + Value *CreateSub(Constant *LHS, Constant *RHS, bool HasNUW = false, + bool HasNSW = false) const override { + return ConstFolder.CreateSub(LHS, RHS, HasNUW, HasNSW); + } + Value *CreateFSub(Constant *LHS, Constant *RHS) const override { + return ConstFolder.CreateFSub(LHS, RHS); + } + Value *CreateMul(Constant *LHS, Constant *RHS, bool HasNUW = false, + bool HasNSW = false) const override { + return ConstFolder.CreateMul(LHS, RHS, HasNUW, HasNSW); + } + Value *CreateFMul(Constant *LHS, Constant *RHS) const override { + return ConstFolder.CreateFMul(LHS, RHS); + } + Value *CreateUDiv(Constant *LHS, Constant *RHS, + bool isExact = false) const override { + return ConstFolder.CreateUDiv(LHS, RHS, isExact); + } + Value *CreateSDiv(Constant *LHS, Constant *RHS, + bool isExact = false) const override { + return ConstFolder.CreateSDiv(LHS, RHS, isExact); + } + Value *CreateFDiv(Constant *LHS, Constant *RHS) const override { + return ConstFolder.CreateFDiv(LHS, RHS); + } + Value *CreateURem(Constant *LHS, Constant *RHS) const override { + return ConstFolder.CreateURem(LHS, RHS); + } + Value *CreateSRem(Constant *LHS, Constant *RHS) const override { + return ConstFolder.CreateSRem(LHS, RHS); + } + Value *CreateFRem(Constant *LHS, Constant *RHS) const override { + return ConstFolder.CreateFRem(LHS, RHS); + } + Value *CreateShl(Constant *LHS, Constant *RHS, bool HasNUW = false, + bool HasNSW = false) const override { + return ConstFolder.CreateShl(LHS, RHS, HasNUW, HasNSW); + } + Value *CreateLShr(Constant *LHS, Constant *RHS, + bool isExact = false) const override { + return ConstFolder.CreateLShr(LHS, RHS, isExact); + } + Value *CreateAShr(Constant *LHS, Constant *RHS, + bool isExact = false) const override { + return ConstFolder.CreateAShr(LHS, RHS, isExact); + } + Value *CreateXor(Constant *LHS, Constant *RHS) const override { + return ConstFolder.CreateXor(LHS, RHS); + } + + Value *CreateBinOp(Instruction::BinaryOps Opc, Constant *LHS, + Constant *RHS) const override { + return ConstFolder.CreateBinOp(Opc, LHS, RHS); + } + + //===--------------------------------------------------------------------===// + // Unary Operators + //===--------------------------------------------------------------------===// + + Value *CreateNeg(Constant *C, bool HasNUW = false, + bool HasNSW = false) const override { + return ConstFolder.CreateNeg(C, HasNUW, HasNSW); + } + Value *CreateFNeg(Constant *C) const override { + return ConstFolder.CreateFNeg(C); + } + Value *CreateNot(Constant *C) const override { + return ConstFolder.CreateNot(C); + } + + Value *CreateUnOp(Instruction::UnaryOps Opc, Constant *C) const override { + return ConstFolder.CreateUnOp(Opc, C); + } + + //===--------------------------------------------------------------------===// + // Cast/Conversion Operators + //===--------------------------------------------------------------------===// + + Value *CreateCast(Instruction::CastOps Op, Constant *C, + Type *DestTy) const override { + if (C->getType() == DestTy) + return C; // avoid calling Fold + return ConstFolder.CreateCast(Op, C, DestTy); + } + Value *CreateIntCast(Constant *C, Type *DestTy, + bool isSigned) const override { + if (C->getType() == DestTy) + return C; // avoid calling Fold + return ConstFolder.CreateIntCast(C, DestTy, isSigned); + } + Value *CreatePointerCast(Constant *C, Type *DestTy) const override { + if (C->getType() == DestTy) + return C; // avoid calling Fold + return ConstFolder.CreatePointerCast(C, DestTy); + } + Value *CreateFPCast(Constant *C, Type *DestTy) const override { + if (C->getType() == DestTy) + return C; // avoid calling Fold + return ConstFolder.CreateFPCast(C, DestTy); + } + Value *CreateBitCast(Constant *C, Type *DestTy) const override { + return ConstFolder.CreateBitCast(C, DestTy); + } + Value *CreateIntToPtr(Constant *C, Type *DestTy) const override { + return ConstFolder.CreateIntToPtr(C, DestTy); + } + Value *CreatePtrToInt(Constant *C, Type *DestTy) const override { + return ConstFolder.CreatePtrToInt(C, DestTy); + } + Value *CreateZExtOrBitCast(Constant *C, Type *DestTy) const override { + if (C->getType() == DestTy) + return C; // avoid calling Fold + return ConstFolder.CreateZExtOrBitCast(C, DestTy); + } + Value *CreateSExtOrBitCast(Constant *C, Type *DestTy) const override { + if (C->getType() == DestTy) + return C; // avoid calling Fold + return ConstFolder.CreateSExtOrBitCast(C, DestTy); + } + Value *CreateTruncOrBitCast(Constant *C, Type *DestTy) const override { + if (C->getType() == DestTy) + return C; // avoid calling Fold + return ConstFolder.CreateTruncOrBitCast(C, DestTy); + } + + Value *CreatePointerBitCastOrAddrSpaceCast(Constant *C, + Type *DestTy) const override { + if (C->getType() == DestTy) + return C; // avoid calling Fold + return ConstFolder.CreatePointerBitCastOrAddrSpaceCast(C, DestTy); + } + + //===--------------------------------------------------------------------===// + // Compare Instructions + //===--------------------------------------------------------------------===// + + Value *CreateFCmp(CmpInst::Predicate P, Constant *LHS, + Constant *RHS) const override { + return ConstFolder.CreateFCmp(P, LHS, RHS); + } + + //===--------------------------------------------------------------------===// + // Other Instructions + //===--------------------------------------------------------------------===// + + Value *CreateExtractElement(Constant *Vec, Constant *Idx) const override { + return ConstFolder.CreateExtractElement(Vec, Idx); + } + + Value *CreateInsertElement(Constant *Vec, Constant *NewElt, + Constant *Idx) const override { + return ConstFolder.CreateInsertElement(Vec, NewElt, Idx); + } + + Value *CreateShuffleVector(Constant *V1, Constant *V2, + ArrayRef<int> Mask) const override { + return ConstFolder.CreateShuffleVector(V1, V2, Mask); + } + + Value *CreateExtractValue(Constant *Agg, + ArrayRef<unsigned> IdxList) const override { + return ConstFolder.CreateExtractValue(Agg, IdxList); + } + + Value *CreateInsertValue(Constant *Agg, Constant *Val, + ArrayRef<unsigned> IdxList) const override { + return ConstFolder.CreateInsertValue(Agg, Val, IdxList); + } +}; + +} // end namespace llvm + +#endif // LLVM_ANALYSIS_INSTSIMPLIFYFOLDER_H diff --git a/llvm/include/llvm/Analysis/InstructionSimplify.h b/llvm/include/llvm/Analysis/InstructionSimplify.h index f0f8e4bc9175..8b49c115f101 100644 --- a/llvm/include/llvm/Analysis/InstructionSimplify.h +++ b/llvm/include/llvm/Analysis/InstructionSimplify.h @@ -63,7 +63,7 @@ class Value; /// results if the users specified it is safe to use. struct InstrInfoQuery { InstrInfoQuery(bool UMD) : UseInstrInfo(UMD) {} - InstrInfoQuery() : UseInstrInfo(true) {} + InstrInfoQuery() = default; bool UseInstrInfo = true; MDNode *getMetadata(const Instruction *I, unsigned KindID) const { @@ -248,8 +248,8 @@ Value *SimplifySelectInst(Value *Cond, Value *TrueVal, Value *FalseVal, const SimplifyQuery &Q); /// Given operands for a GetElementPtrInst, fold the result or return null. -Value *SimplifyGEPInst(Type *SrcTy, ArrayRef<Value *> Ops, bool InBounds, - const SimplifyQuery &Q); +Value *SimplifyGEPInst(Type *SrcTy, Value *Ptr, ArrayRef<Value *> Indices, + bool InBounds, const SimplifyQuery &Q); /// Given operands for an InsertValueInst, fold the result or return null. Value *SimplifyInsertValueInst(Value *Agg, Value *Val, ArrayRef<unsigned> Idxs, diff --git a/llvm/include/llvm/Analysis/LazyBlockFrequencyInfo.h b/llvm/include/llvm/Analysis/LazyBlockFrequencyInfo.h index 0e7dc943bacf..a6d8b76b12ae 100644 --- a/llvm/include/llvm/Analysis/LazyBlockFrequencyInfo.h +++ b/llvm/include/llvm/Analysis/LazyBlockFrequencyInfo.h @@ -22,7 +22,6 @@ namespace llvm { class AnalysisUsage; -class BranchProbabilityInfo; class Function; class LoopInfo; @@ -34,8 +33,7 @@ template <typename FunctionT, typename BranchProbabilityInfoPassT, typename LoopInfoT, typename BlockFrequencyInfoT> class LazyBlockFrequencyInfo { public: - LazyBlockFrequencyInfo() - : Calculated(false), F(nullptr), BPIPass(nullptr), LI(nullptr) {} + LazyBlockFrequencyInfo() = default; /// Set up the per-function input. void setAnalysis(const FunctionT *F, BranchProbabilityInfoPassT *BPIPass, @@ -68,10 +66,10 @@ public: private: BlockFrequencyInfoT BFI; - bool Calculated; - const FunctionT *F; - BranchProbabilityInfoPassT *BPIPass; - const LoopInfoT *LI; + bool Calculated = false; + const FunctionT *F = nullptr; + BranchProbabilityInfoPassT *BPIPass = nullptr; + const LoopInfoT *LI = nullptr; }; /// This is an alternative analysis pass to diff --git a/llvm/include/llvm/Analysis/LazyBranchProbabilityInfo.h b/llvm/include/llvm/Analysis/LazyBranchProbabilityInfo.h index 3c632f02905a..bad7423616b4 100644 --- a/llvm/include/llvm/Analysis/LazyBranchProbabilityInfo.h +++ b/llvm/include/llvm/Analysis/LazyBranchProbabilityInfo.h @@ -57,7 +57,7 @@ class LazyBranchProbabilityInfoPass : public FunctionPass { public: LazyBranchProbabilityInfo(const Function *F, const LoopInfo *LI, const TargetLibraryInfo *TLI) - : Calculated(false), F(F), LI(LI), TLI(TLI) {} + : F(F), LI(LI), TLI(TLI) {} /// Retrieve the BPI with the branch probabilities computed. BranchProbabilityInfo &getCalculated() { @@ -75,7 +75,7 @@ class LazyBranchProbabilityInfoPass : public FunctionPass { private: BranchProbabilityInfo BPI; - bool Calculated; + bool Calculated = false; const Function *F; const LoopInfo *LI; const TargetLibraryInfo *TLI; diff --git a/llvm/include/llvm/Analysis/LazyCallGraph.h b/llvm/include/llvm/Analysis/LazyCallGraph.h index 0580f4d7b226..eb8f66bada59 100644 --- a/llvm/include/llvm/Analysis/LazyCallGraph.h +++ b/llvm/include/llvm/Analysis/LazyCallGraph.h @@ -884,7 +884,9 @@ public: RefSCC *RC = nullptr; /// Build the begin iterator for a node. - postorder_ref_scc_iterator(LazyCallGraph &G) : G(&G), RC(getRC(G, 0)) {} + postorder_ref_scc_iterator(LazyCallGraph &G) : G(&G), RC(getRC(G, 0)) { + incrementUntilNonEmptyRefSCC(); + } /// Build the end iterator for a node. This is selected purely by overload. postorder_ref_scc_iterator(LazyCallGraph &G, IsAtEndT /*Nonce*/) : G(&G) {} @@ -899,6 +901,17 @@ public: return G.PostOrderRefSCCs[Index]; } + // Keep incrementing until RC is non-empty (or null). + void incrementUntilNonEmptyRefSCC() { + while (RC && RC->size() == 0) + increment(); + } + + void increment() { + assert(RC && "Cannot increment the end iterator!"); + RC = getRC(*G, G->RefSCCIndices.find(RC)->second + 1); + } + public: bool operator==(const postorder_ref_scc_iterator &Arg) const { return G == Arg.G && RC == Arg.RC; @@ -908,8 +921,8 @@ public: using iterator_facade_base::operator++; postorder_ref_scc_iterator &operator++() { - assert(RC && "Cannot increment the end iterator!"); - RC = getRC(*G, G->RefSCCIndices.find(RC)->second + 1); + increment(); + incrementUntilNonEmptyRefSCC(); return *this; } }; @@ -1190,7 +1203,7 @@ private: } }; -inline LazyCallGraph::Edge::Edge() : Value() {} +inline LazyCallGraph::Edge::Edge() {} inline LazyCallGraph::Edge::Edge(Node &N, Kind K) : Value(&N, K) {} inline LazyCallGraph::Edge::operator bool() const { diff --git a/llvm/include/llvm/Analysis/Loads.h b/llvm/include/llvm/Analysis/Loads.h index ced1943b81d9..3db501c51a17 100644 --- a/llvm/include/llvm/Analysis/Loads.h +++ b/llvm/include/llvm/Analysis/Loads.h @@ -24,7 +24,6 @@ class DominatorTree; class Instruction; class LoadInst; class Loop; -class MDNode; class MemoryLocation; class ScalarEvolution; class TargetLibraryInfo; diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h index 2b4edfac61fc..c83a04991b04 100644 --- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h +++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h @@ -26,7 +26,6 @@ class AAResults; class DataLayout; class Loop; class LoopAccessInfo; -class OptimizationRemarkEmitter; class raw_ostream; class SCEV; class SCEVUnionPredicate; @@ -170,10 +169,7 @@ public: }; MemoryDepChecker(PredicatedScalarEvolution &PSE, const Loop *L) - : PSE(PSE), InnermostLoop(L), AccessIdx(0), MaxSafeDepDistBytes(0), - MaxSafeVectorWidthInBits(-1U), - FoundNonConstantDistanceDependence(false), - Status(VectorizationSafetyStatus::Safe), RecordDependences(true) {} + : PSE(PSE), InnermostLoop(L) {} /// Register the location (instructions are given increasing numbers) /// of a write access. @@ -265,30 +261,30 @@ private: SmallVector<Instruction *, 16> InstMap; /// The program order index to be used for the next instruction. - unsigned AccessIdx; + unsigned AccessIdx = 0; // We can access this many bytes in parallel safely. - uint64_t MaxSafeDepDistBytes; + uint64_t MaxSafeDepDistBytes = 0; /// Number of elements (from consecutive iterations) that are safe to /// operate on simultaneously, multiplied by the size of the element in bits. /// The size of the element is taken from the memory access that is most /// restrictive. - uint64_t MaxSafeVectorWidthInBits; + uint64_t MaxSafeVectorWidthInBits = -1U; /// If we see a non-constant dependence distance we can still try to /// vectorize this loop with runtime checks. - bool FoundNonConstantDistanceDependence; + bool FoundNonConstantDistanceDependence = false; /// Result of the dependence checks, indicating whether the checked /// dependences are safe for vectorization, require RT checks or are known to /// be unsafe. - VectorizationSafetyStatus Status; + VectorizationSafetyStatus Status = VectorizationSafetyStatus::Safe; //// True if Dependences reflects the dependences in the //// loop. If false we exceeded MaxDependences and //// Dependences is invalid. - bool RecordDependences; + bool RecordDependences = true; /// Memory dependences collected during the analysis. Only valid if /// RecordDependences is true. @@ -396,7 +392,7 @@ public: AliasSetId(AliasSetId), Expr(Expr) {} }; - RuntimePointerChecking(ScalarEvolution *SE) : Need(false), SE(SE) {} + RuntimePointerChecking(ScalarEvolution *SE) : SE(SE) {} /// Reset the state of the pointer runtime information. void reset() { @@ -445,7 +441,7 @@ public: unsigned Depth = 0) const; /// This flag indicates if we need to add the runtime check. - bool Need; + bool Need = false; /// Information about the pointers that may require checking. SmallVector<PointerInfo, 2> Pointers; @@ -621,17 +617,17 @@ private: Loop *TheLoop; - unsigned NumLoads; - unsigned NumStores; + unsigned NumLoads = 0; + unsigned NumStores = 0; - uint64_t MaxSafeDepDistBytes; + uint64_t MaxSafeDepDistBytes = -1; /// Cache the result of analyzeLoop. - bool CanVecMem; - bool HasConvergentOp; + bool CanVecMem = false; + bool HasConvergentOp = false; /// Indicator that there are non vectorizable stores to a uniform address. - bool HasDependenceInvolvingLoopInvariantAddress; + bool HasDependenceInvolvingLoopInvariantAddress = false; /// The diagnostics report generated for the analysis. E.g. why we /// couldn't analyze the loop. diff --git a/llvm/include/llvm/Analysis/LoopAnalysisManager.h b/llvm/include/llvm/Analysis/LoopAnalysisManager.h index bc8a1e74e447..d07e6977fed1 100644 --- a/llvm/include/llvm/Analysis/LoopAnalysisManager.h +++ b/llvm/include/llvm/Analysis/LoopAnalysisManager.h @@ -87,7 +87,7 @@ typedef InnerAnalysisManagerProxy<LoopAnalysisManager, Function> template <> class LoopAnalysisManagerFunctionProxy::Result { public: explicit Result(LoopAnalysisManager &InnerAM, LoopInfo &LI) - : InnerAM(&InnerAM), LI(&LI), MSSAUsed(false) {} + : InnerAM(&InnerAM), LI(&LI) {} Result(Result &&Arg) : InnerAM(std::move(Arg.InnerAM)), LI(Arg.LI), MSSAUsed(Arg.MSSAUsed) { // We have to null out the analysis manager in the moved-from state @@ -136,7 +136,7 @@ public: private: LoopAnalysisManager *InnerAM; LoopInfo *LI; - bool MSSAUsed; + bool MSSAUsed = false; }; /// Provide a specialized run method for the \c LoopAnalysisManagerFunctionProxy diff --git a/llvm/include/llvm/Analysis/LoopInfo.h b/llvm/include/llvm/Analysis/LoopInfo.h index 15c9d911ab80..b2326c4714dd 100644 --- a/llvm/include/llvm/Analysis/LoopInfo.h +++ b/llvm/include/llvm/Analysis/LoopInfo.h @@ -557,21 +557,24 @@ public: /// If the given value is an instruction inside of the loop and it can be /// hoisted, do so to make it trivially loop-invariant. - /// Return true if the value after any hoisting is loop invariant. This - /// function can be used as a slightly more aggressive replacement for - /// isLoopInvariant. + /// Return true if \c V is already loop-invariant, and false if \c V can't + /// be made loop-invariant. If \c V is made loop-invariant, \c Changed is + /// set to true. This function can be used as a slightly more aggressive + /// replacement for isLoopInvariant. /// /// If InsertPt is specified, it is the point to hoist instructions to. /// If null, the terminator of the loop preheader is used. + /// bool makeLoopInvariant(Value *V, bool &Changed, Instruction *InsertPt = nullptr, MemorySSAUpdater *MSSAU = nullptr) const; /// If the given instruction is inside of the loop and it can be hoisted, do /// so to make it trivially loop-invariant. - /// Return true if the instruction after any hoisting is loop invariant. This - /// function can be used as a slightly more aggressive replacement for - /// isLoopInvariant. + /// Return true if \c I is already loop-invariant, and false if \c I can't + /// be made loop-invariant. If \c I is made loop-invariant, \c Changed is + /// set to true. This function can be used as a slightly more aggressive + /// replacement for isLoopInvariant. /// /// If InsertPt is specified, it is the point to hoist instructions to. /// If null, the terminator of the loop preheader is used. diff --git a/llvm/include/llvm/Analysis/LoopNestAnalysis.h b/llvm/include/llvm/Analysis/LoopNestAnalysis.h index 3d4a064cf7e3..852a6c438d43 100644 --- a/llvm/include/llvm/Analysis/LoopNestAnalysis.h +++ b/llvm/include/llvm/Analysis/LoopNestAnalysis.h @@ -102,12 +102,35 @@ public: return Loops[Index]; } + /// Get the loop index of the given loop \p L. + unsigned getLoopIndex(const Loop &L) const { + for (unsigned I = 0; I < getNumLoops(); ++I) + if (getLoop(I) == &L) + return I; + llvm_unreachable("Loop not in the loop nest"); + } + /// Return the number of loops in the nest. size_t getNumLoops() const { return Loops.size(); } /// Get the loops in the nest. ArrayRef<Loop *> getLoops() const { return Loops; } + /// Get the loops in the nest at the given \p Depth. + LoopVectorTy getLoopsAtDepth(unsigned Depth) const { + assert(Depth >= Loops.front()->getLoopDepth() && + Depth <= Loops.back()->getLoopDepth() && "Invalid depth"); + LoopVectorTy Result; + for (unsigned I = 0; I < getNumLoops(); ++I) { + Loop *L = getLoop(I); + if (L->getLoopDepth() == Depth) + Result.push_back(L); + else if (L->getLoopDepth() > Depth) + break; + } + return Result; + } + /// Retrieve a vector of perfect loop nests contained in the current loop /// nest. For example, given the following nest containing 4 loops, this /// member function would return {{L1,L2},{L3,L4}}. diff --git a/llvm/include/llvm/Analysis/MLInlineAdvisor.h b/llvm/include/llvm/Analysis/MLInlineAdvisor.h index a218561e61c7..05411d9c99a2 100644 --- a/llvm/include/llvm/Analysis/MLInlineAdvisor.h +++ b/llvm/include/llvm/Analysis/MLInlineAdvisor.h @@ -9,13 +9,13 @@ #ifndef LLVM_ANALYSIS_MLINLINEADVISOR_H #define LLVM_ANALYSIS_MLINLINEADVISOR_H -#include "llvm/Analysis/CallGraph.h" #include "llvm/Analysis/InlineAdvisor.h" +#include "llvm/Analysis/LazyCallGraph.h" #include "llvm/Analysis/MLModelRunner.h" #include "llvm/IR/PassManager.h" +#include <deque> #include <memory> -#include <unordered_map> namespace llvm { class Module; @@ -26,10 +26,10 @@ public: MLInlineAdvisor(Module &M, ModuleAnalysisManager &MAM, std::unique_ptr<MLModelRunner> ModelRunner); - CallGraph *callGraph() const { return CG.get(); } virtual ~MLInlineAdvisor() = default; void onPassEntry() override; + void onPassExit(LazyCallGraph::SCC *SCC) override; int64_t getIRSize(const Function &F) const { return F.getInstructionCount(); } void onSuccessfulInlining(const MLInlineAdvice &Advice, @@ -38,7 +38,6 @@ public: bool isForcedToStop() const { return ForceStop; } int64_t getLocalCalls(Function &F); const MLModelRunner &getModelRunner() const { return *ModelRunner.get(); } - void onModuleInvalidated() override { Invalid = true; } protected: std::unique_ptr<InlineAdvice> getAdviceImpl(CallBase &CB) override; @@ -51,20 +50,32 @@ protected: virtual std::unique_ptr<MLInlineAdvice> getAdviceFromModel(CallBase &CB, OptimizationRemarkEmitter &ORE); + // Get the initial 'level' of the function, or 0 if the function has been + // introduced afterwards. + // TODO: should we keep this updated? + unsigned getInitialFunctionLevel(const Function &F) const; + std::unique_ptr<MLModelRunner> ModelRunner; private: int64_t getModuleIRSize() const; - bool Invalid = true; - std::unique_ptr<CallGraph> CG; + void print(raw_ostream &OS) const override { + OS << "[MLInlineAdvisor] Nodes: " << NodeCount << " Edges: " << EdgeCount + << "\n"; + } + + LazyCallGraph &CG; int64_t NodeCount = 0; int64_t EdgeCount = 0; - std::map<const Function *, unsigned> FunctionLevels; + int64_t EdgesOfLastSeenNodes = 0; + + std::map<const LazyCallGraph::Node *, unsigned> FunctionLevels; const int32_t InitialIRSize = 0; int32_t CurrentIRSize = 0; - + std::deque<const LazyCallGraph::Node *> NodesInLastSCC; + DenseSet<const LazyCallGraph::Node *> AllNodes; bool ForceStop = false; }; diff --git a/llvm/include/llvm/Analysis/MLModelRunner.h b/llvm/include/llvm/Analysis/MLModelRunner.h index 90b3cc7e76e6..669c02af0b3b 100644 --- a/llvm/include/llvm/Analysis/MLModelRunner.h +++ b/llvm/include/llvm/Analysis/MLModelRunner.h @@ -41,15 +41,22 @@ public: getTensorUntyped(static_cast<size_t>(FeatureID))); } -protected: - MLModelRunner(LLVMContext &Ctx) : Ctx(Ctx) {} - virtual void *evaluateUntyped() = 0; virtual void *getTensorUntyped(size_t Index) = 0; const void *getTensorUntyped(size_t Index) const { return (const_cast<MLModelRunner *>(this))->getTensorUntyped(Index); } + enum class Kind : int { Unknown, Release, Development, NoOp }; + Kind getKind() const { return Type; } + +protected: + MLModelRunner(LLVMContext &Ctx, Kind Type) : Ctx(Ctx), Type(Type) { + assert(Type != Kind::Unknown); + } + virtual void *evaluateUntyped() = 0; + LLVMContext &Ctx; + const Kind Type; }; } // namespace llvm diff --git a/llvm/include/llvm/Analysis/MemoryBuiltins.h b/llvm/include/llvm/Analysis/MemoryBuiltins.h index 94495a518042..d5b60ee540e0 100644 --- a/llvm/include/llvm/Analysis/MemoryBuiltins.h +++ b/llvm/include/llvm/Analysis/MemoryBuiltins.h @@ -30,7 +30,6 @@ namespace llvm { class AllocaInst; class Argument; class CallInst; -class ConstantInt; class ConstantPointerNull; class DataLayout; class ExtractElementInst; @@ -45,7 +44,6 @@ class IntToPtrInst; class LLVMContext; class LoadInst; class PHINode; -class PointerType; class SelectInst; class Type; class UndefValue; @@ -54,118 +52,26 @@ class Value; /// Tests if a value is a call or invoke to a library function that /// allocates or reallocates memory (either malloc, calloc, realloc, or strdup /// like). -bool isAllocationFn(const Value *V, const TargetLibraryInfo *TLI, - bool LookThroughBitCast = false); +bool isAllocationFn(const Value *V, const TargetLibraryInfo *TLI); bool isAllocationFn(const Value *V, - function_ref<const TargetLibraryInfo &(Function &)> GetTLI, - bool LookThroughBitCast = false); - -/// Tests if a value is a call or invoke to a function that returns a -/// NoAlias pointer (including malloc/calloc/realloc/strdup-like functions). -bool isNoAliasFn(const Value *V, const TargetLibraryInfo *TLI, - bool LookThroughBitCast = false); - -/// Tests if a value is a call or invoke to a library function that -/// allocates uninitialized memory (such as malloc). -bool isMallocLikeFn(const Value *V, const TargetLibraryInfo *TLI, - bool LookThroughBitCast = false); -bool isMallocLikeFn(const Value *V, - function_ref<const TargetLibraryInfo &(Function &)> GetTLI, - bool LookThroughBitCast = false); - -/// Tests if a value is a call or invoke to a library function that -/// allocates uninitialized memory with alignment (such as aligned_alloc). -bool isAlignedAllocLikeFn(const Value *V, const TargetLibraryInfo *TLI, - bool LookThroughBitCast = false); -bool isAlignedAllocLikeFn( - const Value *V, function_ref<const TargetLibraryInfo &(Function &)> GetTLI, - bool LookThroughBitCast = false); - -/// Tests if a value is a call or invoke to a library function that -/// allocates zero-filled memory (such as calloc). -bool isCallocLikeFn(const Value *V, const TargetLibraryInfo *TLI, - bool LookThroughBitCast = false); + function_ref<const TargetLibraryInfo &(Function &)> GetTLI); /// Tests if a value is a call or invoke to a library function that /// allocates memory similar to malloc or calloc. -bool isMallocOrCallocLikeFn(const Value *V, const TargetLibraryInfo *TLI, - bool LookThroughBitCast = false); +bool isMallocOrCallocLikeFn(const Value *V, const TargetLibraryInfo *TLI); /// Tests if a value is a call or invoke to a library function that /// allocates memory (either malloc, calloc, or strdup like). -bool isAllocLikeFn(const Value *V, const TargetLibraryInfo *TLI, - bool LookThroughBitCast = false); +bool isAllocLikeFn(const Value *V, const TargetLibraryInfo *TLI); /// Tests if a value is a call or invoke to a library function that /// reallocates memory (e.g., realloc). -bool isReallocLikeFn(const Value *V, const TargetLibraryInfo *TLI, - bool LookThroughBitCast = false); +bool isReallocLikeFn(const Value *V, const TargetLibraryInfo *TLI); /// Tests if a function is a call or invoke to a library function that /// reallocates memory (e.g., realloc). bool isReallocLikeFn(const Function *F, const TargetLibraryInfo *TLI); -/// Tests if a value is a call or invoke to a library function that -/// allocates memory and throws if an allocation failed (e.g., new). -bool isOpNewLikeFn(const Value *V, const TargetLibraryInfo *TLI, - bool LookThroughBitCast = false); - -/// Tests if a value is a call or invoke to a library function that -/// allocates memory (strdup, strndup). -bool isStrdupLikeFn(const Value *V, const TargetLibraryInfo *TLI, - bool LookThroughBitCast = false); - -//===----------------------------------------------------------------------===// -// malloc Call Utility Functions. -// - -/// extractMallocCall - Returns the corresponding CallInst if the instruction -/// is a malloc call. Since CallInst::CreateMalloc() only creates calls, we -/// ignore InvokeInst here. -const CallInst * -extractMallocCall(const Value *I, - function_ref<const TargetLibraryInfo &(Function &)> GetTLI); -inline CallInst * -extractMallocCall(Value *I, - function_ref<const TargetLibraryInfo &(Function &)> GetTLI) { - return const_cast<CallInst *>(extractMallocCall((const Value *)I, GetTLI)); -} - -/// getMallocType - Returns the PointerType resulting from the malloc call. -/// The PointerType depends on the number of bitcast uses of the malloc call: -/// 0: PointerType is the malloc calls' return type. -/// 1: PointerType is the bitcast's result type. -/// >1: Unique PointerType cannot be determined, return NULL. -PointerType *getMallocType(const CallInst *CI, const TargetLibraryInfo *TLI); - -/// getMallocAllocatedType - Returns the Type allocated by malloc call. -/// The Type depends on the number of bitcast uses of the malloc call: -/// 0: PointerType is the malloc calls' return type. -/// 1: PointerType is the bitcast's result type. -/// >1: Unique PointerType cannot be determined, return NULL. -Type *getMallocAllocatedType(const CallInst *CI, const TargetLibraryInfo *TLI); - -/// getMallocArraySize - Returns the array size of a malloc call. If the -/// argument passed to malloc is a multiple of the size of the malloced type, -/// then return that multiple. For non-array mallocs, the multiple is -/// constant 1. Otherwise, return NULL for mallocs whose array size cannot be -/// determined. -Value *getMallocArraySize(CallInst *CI, const DataLayout &DL, - const TargetLibraryInfo *TLI, - bool LookThroughSExt = false); - -//===----------------------------------------------------------------------===// -// calloc Call Utility Functions. -// - -/// extractCallocCall - Returns the corresponding CallInst if the instruction -/// is a calloc call. -const CallInst *extractCallocCall(const Value *I, const TargetLibraryInfo *TLI); -inline CallInst *extractCallocCall(Value *I, const TargetLibraryInfo *TLI) { - return const_cast<CallInst*>(extractCallocCall((const Value*)I, TLI)); -} - - //===----------------------------------------------------------------------===// // free Call Utility Functions. // @@ -181,6 +87,37 @@ inline CallInst *isFreeCall(Value *I, const TargetLibraryInfo *TLI) { } //===----------------------------------------------------------------------===// +// Properties of allocation functions +// + +/// Return false if the allocation can have side effects on the program state +/// we are required to preserve beyond the effect of allocating a new object. +/// Ex: If our allocation routine has a counter for the number of objects +/// allocated, and the program prints it on exit, can the value change due +/// to optimization? Answer is highly language dependent. +/// Note: *Removable* really does mean removable; it does not mean observable. +/// A language (e.g. C++) can allow removing allocations without allowing +/// insertion or speculative execution of allocation routines. +bool isAllocRemovable(const CallBase *V, const TargetLibraryInfo *TLI); + +/// Gets the alignment argument for an aligned_alloc-like function +Value *getAllocAlignment(const CallBase *V, const TargetLibraryInfo *TLI); + +/// Return the size of the requested allocation. With a trivial mapper, this is +/// identical to calling getObjectSize(..., Exact). A mapper function can be +/// used to replace one Value* (operand to the allocation) with another. This +/// is useful when doing abstract interpretation. +Optional<APInt> getAllocSize(const CallBase *CB, + const TargetLibraryInfo *TLI, + std::function<const Value*(const Value*)> Mapper); + +/// If this allocation function initializes memory to a fixed value, return +/// said value in the requested type. Otherwise, return nullptr. +Constant *getInitialValueOfAllocation(const CallBase *Alloc, + const TargetLibraryInfo *TLI, + Type *Ty); + +//===----------------------------------------------------------------------===// // Utility functions to compute size of objects. // diff --git a/llvm/include/llvm/Analysis/MemoryLocation.h b/llvm/include/llvm/Analysis/MemoryLocation.h index 833fce1b1726..23e50f601e04 100644 --- a/llvm/include/llvm/Analysis/MemoryLocation.h +++ b/llvm/include/llvm/Analysis/MemoryLocation.h @@ -284,8 +284,7 @@ public: return T.isScalable() ? UnknownSize : T.getFixedSize(); } - MemoryLocation() - : Ptr(nullptr), Size(LocationSize::beforeOrAfterPointer()), AATags() {} + MemoryLocation() : Ptr(nullptr), Size(LocationSize::beforeOrAfterPointer()) {} explicit MemoryLocation(const Value *Ptr, LocationSize Size, const AAMDNodes &AATags = AAMDNodes()) diff --git a/llvm/include/llvm/Analysis/MemorySSA.h b/llvm/include/llvm/Analysis/MemorySSA.h index 48aeef371e3d..b41f5771bacd 100644 --- a/llvm/include/llvm/Analysis/MemorySSA.h +++ b/llvm/include/llvm/Analysis/MemorySSA.h @@ -106,7 +106,6 @@ namespace llvm { -class AllocaInst; class Function; class Instruction; class MemoryAccess; @@ -866,7 +865,7 @@ private: AccessList *getOrCreateAccessList(const BasicBlock *); DefsList *getOrCreateDefsList(const BasicBlock *); void renumberBlock(const BasicBlock *) const; - AliasAnalysis *AA; + AliasAnalysis *AA = nullptr; DominatorTree *DT; Function &F; @@ -893,7 +892,7 @@ private: std::unique_ptr<ClobberWalkerBase<AliasAnalysis>> WalkerBase; std::unique_ptr<CachingWalker<AliasAnalysis>> Walker; std::unique_ptr<SkipSelfWalker<AliasAnalysis>> SkipWalker; - unsigned NextID; + unsigned NextID = 0; }; /// Enables verification of MemorySSA. diff --git a/llvm/include/llvm/Analysis/MemorySSAUpdater.h b/llvm/include/llvm/Analysis/MemorySSAUpdater.h index 659e6aff6e28..3e5ebe9cb427 100644 --- a/llvm/include/llvm/Analysis/MemorySSAUpdater.h +++ b/llvm/include/llvm/Analysis/MemorySSAUpdater.h @@ -44,7 +44,6 @@ namespace llvm { class BasicBlock; -class BranchInst; class DominatorTree; class Instruction; class LoopBlocksRPO; diff --git a/llvm/include/llvm/Analysis/ModelUnderTrainingRunner.h b/llvm/include/llvm/Analysis/ModelUnderTrainingRunner.h index ca99d5d01eef..071ccf96fe5b 100644 --- a/llvm/include/llvm/Analysis/ModelUnderTrainingRunner.h +++ b/llvm/include/llvm/Analysis/ModelUnderTrainingRunner.h @@ -26,17 +26,11 @@ namespace llvm { /// sacrificed for ease of use while training. class ModelUnderTrainingRunner final : public MLModelRunner { public: - ModelUnderTrainingRunner(LLVMContext &Ctx, const std::string &ModelPath, - const std::vector<TensorSpec> &InputSpecs, - const std::vector<LoggedFeatureSpec> &OutputSpecs); - // Disallows copy and assign. ModelUnderTrainingRunner(const ModelUnderTrainingRunner &) = delete; ModelUnderTrainingRunner & operator=(const ModelUnderTrainingRunner &) = delete; - bool isValid() const { return !!Evaluator; } - const std::vector<LoggedFeatureSpec> &outputLoggedFeatureSpecs() const { return OutputSpecs; } @@ -45,13 +39,27 @@ public: lastEvaluationResult() const { return LastEvaluationResult; } + static bool classof(const MLModelRunner *R) { + return R->getKind() == MLModelRunner::Kind::Development; + } + + static std::unique_ptr<ModelUnderTrainingRunner> + createAndEnsureValid(LLVMContext &Ctx, const std::string &ModelPath, + StringRef DecisionName, + const std::vector<TensorSpec> &InputSpecs, + StringRef OutputSpecsPathOverride = ""); private: + ModelUnderTrainingRunner(LLVMContext &Ctx, const std::string &ModelPath, + const std::vector<TensorSpec> &InputSpecs, + const std::vector<LoggedFeatureSpec> &OutputSpecs); + std::unique_ptr<TFModelEvaluator> Evaluator; const std::vector<LoggedFeatureSpec> OutputSpecs; Optional<TFModelEvaluator::EvaluationResult> LastEvaluationResult; void *evaluateUntyped() override; void *getTensorUntyped(size_t Index) override; + bool isValid() const { return !!Evaluator; } }; } // namespace llvm diff --git a/llvm/include/llvm/Analysis/NoInferenceModelRunner.h b/llvm/include/llvm/Analysis/NoInferenceModelRunner.h index 60d6777c765b..5bcedf98865c 100644 --- a/llvm/include/llvm/Analysis/NoInferenceModelRunner.h +++ b/llvm/include/llvm/Analysis/NoInferenceModelRunner.h @@ -26,6 +26,10 @@ public: NoInferenceModelRunner(LLVMContext &Ctx, const std::vector<TensorSpec> &Inputs); + static bool classof(const MLModelRunner *R) { + return R->getKind() == MLModelRunner::Kind::NoOp; + } + private: void *evaluateUntyped() override { llvm_unreachable("We shouldn't call run on this model runner."); @@ -36,4 +40,4 @@ private: }; } // namespace llvm #endif // defined(LLVM_HAVE_TF_API) -#endif // defined(LLVM_ANALYSIS_NOINFERENCEMODELRUNNER_H) +#endif // LLVM_ANALYSIS_NOINFERENCEMODELRUNNER_H diff --git a/llvm/include/llvm/Analysis/ObjCARCAliasAnalysis.h b/llvm/include/llvm/Analysis/ObjCARCAliasAnalysis.h index b4f4e5f29768..d19a6394bd48 100644 --- a/llvm/include/llvm/Analysis/ObjCARCAliasAnalysis.h +++ b/llvm/include/llvm/Analysis/ObjCARCAliasAnalysis.h @@ -40,7 +40,7 @@ class ObjCARCAAResult : public AAResultBase<ObjCARCAAResult> { const DataLayout &DL; public: - explicit ObjCARCAAResult(const DataLayout &DL) : AAResultBase(), DL(DL) {} + explicit ObjCARCAAResult(const DataLayout &DL) : DL(DL) {} ObjCARCAAResult(ObjCARCAAResult &&Arg) : AAResultBase(std::move(Arg)), DL(Arg.DL) {} diff --git a/llvm/include/llvm/Analysis/ObjCARCInstKind.h b/llvm/include/llvm/Analysis/ObjCARCInstKind.h index 84565b9315c7..e332bcf88be7 100644 --- a/llvm/include/llvm/Analysis/ObjCARCInstKind.h +++ b/llvm/include/llvm/Analysis/ObjCARCInstKind.h @@ -28,7 +28,7 @@ namespace objcarc { enum class ARCInstKind { Retain, ///< objc_retain RetainRV, ///< objc_retainAutoreleasedReturnValue - ClaimRV, ///< objc_unsafeClaimAutoreleasedReturnValue + UnsafeClaimRV, ///< objc_unsafeClaimAutoreleasedReturnValue RetainBlock, ///< objc_retainBlock Release, ///< objc_release Autorelease, ///< objc_autorelease diff --git a/llvm/include/llvm/Analysis/ObjCARCUtil.h b/llvm/include/llvm/Analysis/ObjCARCUtil.h index 362dd6c29992..1d330ca58a87 100644 --- a/llvm/include/llvm/Analysis/ObjCARCUtil.h +++ b/llvm/include/llvm/Analysis/ObjCARCUtil.h @@ -48,15 +48,15 @@ inline Optional<Function *> getAttachedARCFunction(const CallBase *CB) { return cast<Function>(B->Inputs[0]); } -/// Check whether the function is retainRV/claimRV. +/// Check whether the function is retainRV/unsafeClaimRV. inline bool isRetainOrClaimRV(ARCInstKind Kind) { - return Kind == ARCInstKind::RetainRV || Kind == ARCInstKind::ClaimRV; + return Kind == ARCInstKind::RetainRV || Kind == ARCInstKind::UnsafeClaimRV; } /// This function returns the ARCInstKind of the function attached to operand /// bundle clang_arc_attachedcall. It returns None if the call doesn't have the /// operand bundle or the operand is null. Otherwise it returns either RetainRV -/// or ClaimRV. +/// or UnsafeClaimRV. inline ARCInstKind getAttachedARCFunctionKind(const CallBase *CB) { Optional<Function *> Fn = getAttachedARCFunction(CB); if (!Fn.hasValue()) diff --git a/llvm/include/llvm/Analysis/PHITransAddr.h b/llvm/include/llvm/Analysis/PHITransAddr.h index 54a07f053478..a23f8e61c303 100644 --- a/llvm/include/llvm/Analysis/PHITransAddr.h +++ b/llvm/include/llvm/Analysis/PHITransAddr.h @@ -40,7 +40,7 @@ class PHITransAddr { const DataLayout &DL; /// TLI - The target library info if known, otherwise null. - const TargetLibraryInfo *TLI; + const TargetLibraryInfo *TLI = nullptr; /// A cache of \@llvm.assume calls used by SimplifyInstruction. AssumptionCache *AC; @@ -50,7 +50,7 @@ class PHITransAddr { public: PHITransAddr(Value *addr, const DataLayout &DL, AssumptionCache *AC) - : Addr(addr), DL(DL), TLI(nullptr), AC(AC) { + : Addr(addr), DL(DL), AC(AC) { // If the address is an instruction, the whole thing is considered an input. if (Instruction *I = dyn_cast<Instruction>(Addr)) InstInputs.push_back(I); diff --git a/llvm/include/llvm/Analysis/ReleaseModeModelRunner.h b/llvm/include/llvm/Analysis/ReleaseModeModelRunner.h index b684f87ea5cb..1bf2e853980c 100644 --- a/llvm/include/llvm/Analysis/ReleaseModeModelRunner.h +++ b/llvm/include/llvm/Analysis/ReleaseModeModelRunner.h @@ -10,6 +10,10 @@ // Only inference is supported. // //===----------------------------------------------------------------------===// + +#ifndef LLVM_ANALYSIS_RELEASEMODEMODELRUNNER_H +#define LLVM_ANALYSIS_RELEASEMODEMODELRUNNER_H + #include "llvm/Analysis/MLModelRunner.h" #include <memory> @@ -29,7 +33,8 @@ public: ReleaseModeModelRunner(LLVMContext &Ctx, const FType &FeatureNames, StringRef DecisionName, StringRef FeedPrefix = "feed_", StringRef FetchPrefix = "fetch_") - : MLModelRunner(Ctx), CompiledModel(std::make_unique<TGen>()) { + : MLModelRunner(Ctx, MLModelRunner::Kind::Release), + CompiledModel(std::make_unique<TGen>()) { assert(CompiledModel && "The CompiledModel should be valid"); const size_t FeatureCount = FeatureNames.size(); @@ -49,6 +54,10 @@ public: virtual ~ReleaseModeModelRunner() = default; + static bool classof(const MLModelRunner *R) { + return R->getKind() == MLModelRunner::Kind::Release; + } + private: void *evaluateUntyped() override { CompiledModel->Run(); @@ -65,3 +74,5 @@ private: std::unique_ptr<TGen> CompiledModel; }; } // namespace llvm + +#endif // LLVM_ANALYSIS_RELEASEMODEMODELRUNNER_H diff --git a/llvm/include/llvm/Analysis/ReplayInlineAdvisor.h b/llvm/include/llvm/Analysis/ReplayInlineAdvisor.h index a0eb9af62205..dc2efeafb568 100644 --- a/llvm/include/llvm/Analysis/ReplayInlineAdvisor.h +++ b/llvm/include/llvm/Analysis/ReplayInlineAdvisor.h @@ -14,11 +14,9 @@ #include "llvm/IR/LLVMContext.h" namespace llvm { -class BasicBlock; class CallBase; class Function; class Module; -class OptimizationRemarkEmitter; struct CallSiteFormat { enum class Format : int { diff --git a/llvm/include/llvm/Analysis/ScalarEvolution.h b/llvm/include/llvm/Analysis/ScalarEvolution.h index df50611832ce..1e6dac44cf2b 100644 --- a/llvm/include/llvm/Analysis/ScalarEvolution.h +++ b/llvm/include/llvm/Analysis/ScalarEvolution.h @@ -568,6 +568,7 @@ public: const SCEV *getTruncateExpr(const SCEV *Op, Type *Ty, unsigned Depth = 0); const SCEV *getZeroExtendExpr(const SCEV *Op, Type *Ty, unsigned Depth = 0); const SCEV *getSignExtendExpr(const SCEV *Op, Type *Ty, unsigned Depth = 0); + const SCEV *getCastExpr(SCEVTypes Kind, const SCEV *Op, Type *Ty); const SCEV *getAnyExtendExpr(const SCEV *Op, Type *Ty); const SCEV *getAddExpr(SmallVectorImpl<const SCEV *> &Ops, SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap, @@ -629,14 +630,18 @@ public: const SCEV *getAbsExpr(const SCEV *Op, bool IsNSW); const SCEV *getMinMaxExpr(SCEVTypes Kind, SmallVectorImpl<const SCEV *> &Operands); + const SCEV *getSequentialMinMaxExpr(SCEVTypes Kind, + SmallVectorImpl<const SCEV *> &Operands); const SCEV *getSMaxExpr(const SCEV *LHS, const SCEV *RHS); const SCEV *getSMaxExpr(SmallVectorImpl<const SCEV *> &Operands); const SCEV *getUMaxExpr(const SCEV *LHS, const SCEV *RHS); const SCEV *getUMaxExpr(SmallVectorImpl<const SCEV *> &Operands); const SCEV *getSMinExpr(const SCEV *LHS, const SCEV *RHS); const SCEV *getSMinExpr(SmallVectorImpl<const SCEV *> &Operands); - const SCEV *getUMinExpr(const SCEV *LHS, const SCEV *RHS); - const SCEV *getUMinExpr(SmallVectorImpl<const SCEV *> &Operands); + const SCEV *getUMinExpr(const SCEV *LHS, const SCEV *RHS, + bool Sequential = false); + const SCEV *getUMinExpr(SmallVectorImpl<const SCEV *> &Operands, + bool Sequential = false); const SCEV *getUnknown(Value *V); const SCEV *getCouldNotCompute(); @@ -728,11 +733,13 @@ public: /// Promote the operands to the wider of the types using zero-extension, and /// then perform a umin operation with them. - const SCEV *getUMinFromMismatchedTypes(const SCEV *LHS, const SCEV *RHS); + const SCEV *getUMinFromMismatchedTypes(const SCEV *LHS, const SCEV *RHS, + bool Sequential = false); /// Promote the operands to the wider of the types using zero-extension, and /// then perform a umin operation with them. N-ary function. - const SCEV *getUMinFromMismatchedTypes(SmallVectorImpl<const SCEV *> &Ops); + const SCEV *getUMinFromMismatchedTypes(SmallVectorImpl<const SCEV *> &Ops, + bool Sequential = false); /// Transitively follow the chain of pointer-type operands until reaching a /// SCEV that does not have a single pointer operand. This returns a @@ -1713,6 +1720,15 @@ private: bool IsSubExpr, bool AllowPredicates = false); + /// Variant of previous which takes the components representing an ICmp + /// as opposed to the ICmpInst itself. Note that the prior version can + /// return more precise results in some cases and is preferred when caller + /// has a materialized ICmp. + ExitLimit computeExitLimitFromICmp(const Loop *L, ICmpInst::Predicate Pred, + const SCEV *LHS, const SCEV *RHS, + bool IsSubExpr, + bool AllowPredicates = false); + /// Compute the number of times the backedge of the specified loop will /// execute if its exit condition were a switch with a single exiting case /// to ExitingBB. @@ -1882,6 +1898,15 @@ private: const SCEV *FoundLHS, const SCEV *FoundRHS, unsigned Depth); + /// Test whether the condition described by Pred, LHS, and RHS is true + /// whenever the condition described by Pred, FoundLHS, and FoundRHS is + /// true. + /// + /// This routine tries to reason about shifts. + bool isImpliedCondOperandsViaShift(ICmpInst::Predicate Pred, const SCEV *LHS, + const SCEV *RHS, const SCEV *FoundLHS, + const SCEV *FoundRHS); + /// If we know that the specified Phi is in the header of its containing /// loop, we know the loop executes a constant number of times, and the PHI /// node is just a recurrence involving constants, fold it. diff --git a/llvm/include/llvm/Analysis/ScalarEvolutionAliasAnalysis.h b/llvm/include/llvm/Analysis/ScalarEvolutionAliasAnalysis.h index 20acb407ead0..ebd427354cee 100644 --- a/llvm/include/llvm/Analysis/ScalarEvolutionAliasAnalysis.h +++ b/llvm/include/llvm/Analysis/ScalarEvolutionAliasAnalysis.h @@ -27,7 +27,7 @@ class SCEVAAResult : public AAResultBase<SCEVAAResult> { ScalarEvolution &SE; public: - explicit SCEVAAResult(ScalarEvolution &SE) : AAResultBase(), SE(SE) {} + explicit SCEVAAResult(ScalarEvolution &SE) : SE(SE) {} SCEVAAResult(SCEVAAResult &&Arg) : AAResultBase(std::move(Arg)), SE(Arg.SE) {} AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB, diff --git a/llvm/include/llvm/Analysis/ScalarEvolutionDivision.h b/llvm/include/llvm/Analysis/ScalarEvolutionDivision.h index 24f0c51487bd..7d5902d31795 100644 --- a/llvm/include/llvm/Analysis/ScalarEvolutionDivision.h +++ b/llvm/include/llvm/Analysis/ScalarEvolutionDivision.h @@ -42,6 +42,7 @@ public: void visitUMaxExpr(const SCEVUMaxExpr *Numerator) {} void visitSMinExpr(const SCEVSMinExpr *Numerator) {} void visitUMinExpr(const SCEVUMinExpr *Numerator) {} + void visitSequentialUMinExpr(const SCEVSequentialUMinExpr *Numerator) {} void visitUnknown(const SCEVUnknown *Numerator) {} void visitCouldNotCompute(const SCEVCouldNotCompute *Numerator) {} diff --git a/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h b/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h index c0da311e4e48..cd8e5fab6766 100644 --- a/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h +++ b/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h @@ -35,883 +35,929 @@ class ConstantRange; class Loop; class Type; - enum SCEVTypes : unsigned short { - // These should be ordered in terms of increasing complexity to make the - // folders simpler. - scConstant, scTruncate, scZeroExtend, scSignExtend, scAddExpr, scMulExpr, - scUDivExpr, scAddRecExpr, scUMaxExpr, scSMaxExpr, scUMinExpr, scSMinExpr, - scPtrToInt, scUnknown, scCouldNotCompute - }; - - /// This class represents a constant integer value. - class SCEVConstant : public SCEV { - friend class ScalarEvolution; - - ConstantInt *V; - - SCEVConstant(const FoldingSetNodeIDRef ID, ConstantInt *v) : - SCEV(ID, scConstant, 1), V(v) {} - - public: - ConstantInt *getValue() const { return V; } - const APInt &getAPInt() const { return getValue()->getValue(); } - - Type *getType() const { return V->getType(); } - - /// Methods for support type inquiry through isa, cast, and dyn_cast: - static bool classof(const SCEV *S) { - return S->getSCEVType() == scConstant; - } - }; - - inline unsigned short computeExpressionSize(ArrayRef<const SCEV *> Args) { - APInt Size(16, 1); - for (auto *Arg : Args) - Size = Size.uadd_sat(APInt(16, Arg->getExpressionSize())); - return (unsigned short)Size.getZExtValue(); +enum SCEVTypes : unsigned short { + // These should be ordered in terms of increasing complexity to make the + // folders simpler. + scConstant, + scTruncate, + scZeroExtend, + scSignExtend, + scAddExpr, + scMulExpr, + scUDivExpr, + scAddRecExpr, + scUMaxExpr, + scSMaxExpr, + scUMinExpr, + scSMinExpr, + scSequentialUMinExpr, + scPtrToInt, + scUnknown, + scCouldNotCompute +}; + +/// This class represents a constant integer value. +class SCEVConstant : public SCEV { + friend class ScalarEvolution; + + ConstantInt *V; + + SCEVConstant(const FoldingSetNodeIDRef ID, ConstantInt *v) + : SCEV(ID, scConstant, 1), V(v) {} + +public: + ConstantInt *getValue() const { return V; } + const APInt &getAPInt() const { return getValue()->getValue(); } + + Type *getType() const { return V->getType(); } + + /// Methods for support type inquiry through isa, cast, and dyn_cast: + static bool classof(const SCEV *S) { return S->getSCEVType() == scConstant; } +}; + +inline unsigned short computeExpressionSize(ArrayRef<const SCEV *> Args) { + APInt Size(16, 1); + for (auto *Arg : Args) + Size = Size.uadd_sat(APInt(16, Arg->getExpressionSize())); + return (unsigned short)Size.getZExtValue(); +} + +/// This is the base class for unary cast operator classes. +class SCEVCastExpr : public SCEV { +protected: + std::array<const SCEV *, 1> Operands; + Type *Ty; + + SCEVCastExpr(const FoldingSetNodeIDRef ID, SCEVTypes SCEVTy, const SCEV *op, + Type *ty); + +public: + const SCEV *getOperand() const { return Operands[0]; } + const SCEV *getOperand(unsigned i) const { + assert(i == 0 && "Operand index out of range!"); + return Operands[0]; } + using op_iterator = std::array<const SCEV *, 1>::const_iterator; + using op_range = iterator_range<op_iterator>; - /// This is the base class for unary cast operator classes. - class SCEVCastExpr : public SCEV { - protected: - std::array<const SCEV *, 1> Operands; - Type *Ty; - - SCEVCastExpr(const FoldingSetNodeIDRef ID, SCEVTypes SCEVTy, const SCEV *op, - Type *ty); - - public: - const SCEV *getOperand() const { return Operands[0]; } - const SCEV *getOperand(unsigned i) const { - assert(i == 0 && "Operand index out of range!"); - return Operands[0]; - } - using op_iterator = std::array<const SCEV *, 1>::const_iterator; - using op_range = iterator_range<op_iterator>; - - op_range operands() const { - return make_range(Operands.begin(), Operands.end()); - } - size_t getNumOperands() const { return 1; } - Type *getType() const { return Ty; } - - /// Methods for support type inquiry through isa, cast, and dyn_cast: - static bool classof(const SCEV *S) { - return S->getSCEVType() == scPtrToInt || S->getSCEVType() == scTruncate || - S->getSCEVType() == scZeroExtend || - S->getSCEVType() == scSignExtend; - } - }; - - /// This class represents a cast from a pointer to a pointer-sized integer - /// value. - class SCEVPtrToIntExpr : public SCEVCastExpr { - friend class ScalarEvolution; + op_range operands() const { + return make_range(Operands.begin(), Operands.end()); + } + size_t getNumOperands() const { return 1; } + Type *getType() const { return Ty; } - SCEVPtrToIntExpr(const FoldingSetNodeIDRef ID, const SCEV *Op, Type *ITy); + /// Methods for support type inquiry through isa, cast, and dyn_cast: + static bool classof(const SCEV *S) { + return S->getSCEVType() == scPtrToInt || S->getSCEVType() == scTruncate || + S->getSCEVType() == scZeroExtend || S->getSCEVType() == scSignExtend; + } +}; - public: - /// Methods for support type inquiry through isa, cast, and dyn_cast: - static bool classof(const SCEV *S) { - return S->getSCEVType() == scPtrToInt; - } - }; +/// This class represents a cast from a pointer to a pointer-sized integer +/// value. +class SCEVPtrToIntExpr : public SCEVCastExpr { + friend class ScalarEvolution; - /// This is the base class for unary integral cast operator classes. - class SCEVIntegralCastExpr : public SCEVCastExpr { - protected: - SCEVIntegralCastExpr(const FoldingSetNodeIDRef ID, SCEVTypes SCEVTy, - const SCEV *op, Type *ty); - - public: - /// Methods for support type inquiry through isa, cast, and dyn_cast: - static bool classof(const SCEV *S) { - return S->getSCEVType() == scTruncate || - S->getSCEVType() == scZeroExtend || - S->getSCEVType() == scSignExtend; - } - }; + SCEVPtrToIntExpr(const FoldingSetNodeIDRef ID, const SCEV *Op, Type *ITy); - /// This class represents a truncation of an integer value to a - /// smaller integer value. - class SCEVTruncateExpr : public SCEVIntegralCastExpr { - friend class ScalarEvolution; +public: + /// Methods for support type inquiry through isa, cast, and dyn_cast: + static bool classof(const SCEV *S) { return S->getSCEVType() == scPtrToInt; } +}; - SCEVTruncateExpr(const FoldingSetNodeIDRef ID, - const SCEV *op, Type *ty); +/// This is the base class for unary integral cast operator classes. +class SCEVIntegralCastExpr : public SCEVCastExpr { +protected: + SCEVIntegralCastExpr(const FoldingSetNodeIDRef ID, SCEVTypes SCEVTy, + const SCEV *op, Type *ty); - public: - /// Methods for support type inquiry through isa, cast, and dyn_cast: - static bool classof(const SCEV *S) { - return S->getSCEVType() == scTruncate; - } - }; +public: + /// Methods for support type inquiry through isa, cast, and dyn_cast: + static bool classof(const SCEV *S) { + return S->getSCEVType() == scTruncate || S->getSCEVType() == scZeroExtend || + S->getSCEVType() == scSignExtend; + } +}; - /// This class represents a zero extension of a small integer value - /// to a larger integer value. - class SCEVZeroExtendExpr : public SCEVIntegralCastExpr { - friend class ScalarEvolution; +/// This class represents a truncation of an integer value to a +/// smaller integer value. +class SCEVTruncateExpr : public SCEVIntegralCastExpr { + friend class ScalarEvolution; - SCEVZeroExtendExpr(const FoldingSetNodeIDRef ID, - const SCEV *op, Type *ty); + SCEVTruncateExpr(const FoldingSetNodeIDRef ID, const SCEV *op, Type *ty); - public: - /// Methods for support type inquiry through isa, cast, and dyn_cast: - static bool classof(const SCEV *S) { - return S->getSCEVType() == scZeroExtend; - } - }; +public: + /// Methods for support type inquiry through isa, cast, and dyn_cast: + static bool classof(const SCEV *S) { return S->getSCEVType() == scTruncate; } +}; - /// This class represents a sign extension of a small integer value - /// to a larger integer value. - class SCEVSignExtendExpr : public SCEVIntegralCastExpr { - friend class ScalarEvolution; +/// This class represents a zero extension of a small integer value +/// to a larger integer value. +class SCEVZeroExtendExpr : public SCEVIntegralCastExpr { + friend class ScalarEvolution; - SCEVSignExtendExpr(const FoldingSetNodeIDRef ID, - const SCEV *op, Type *ty); + SCEVZeroExtendExpr(const FoldingSetNodeIDRef ID, const SCEV *op, Type *ty); - public: - /// Methods for support type inquiry through isa, cast, and dyn_cast: - static bool classof(const SCEV *S) { - return S->getSCEVType() == scSignExtend; - } - }; +public: + /// Methods for support type inquiry through isa, cast, and dyn_cast: + static bool classof(const SCEV *S) { + return S->getSCEVType() == scZeroExtend; + } +}; - /// This node is a base class providing common functionality for - /// n'ary operators. - class SCEVNAryExpr : public SCEV { - protected: - // Since SCEVs are immutable, ScalarEvolution allocates operand - // arrays with its SCEVAllocator, so this class just needs a simple - // pointer rather than a more elaborate vector-like data structure. - // This also avoids the need for a non-trivial destructor. - const SCEV *const *Operands; - size_t NumOperands; - - SCEVNAryExpr(const FoldingSetNodeIDRef ID, enum SCEVTypes T, - const SCEV *const *O, size_t N) - : SCEV(ID, T, computeExpressionSize(makeArrayRef(O, N))), Operands(O), - NumOperands(N) {} +/// This class represents a sign extension of a small integer value +/// to a larger integer value. +class SCEVSignExtendExpr : public SCEVIntegralCastExpr { + friend class ScalarEvolution; - public: - size_t getNumOperands() const { return NumOperands; } + SCEVSignExtendExpr(const FoldingSetNodeIDRef ID, const SCEV *op, Type *ty); - const SCEV *getOperand(unsigned i) const { - assert(i < NumOperands && "Operand index out of range!"); - return Operands[i]; - } +public: + /// Methods for support type inquiry through isa, cast, and dyn_cast: + static bool classof(const SCEV *S) { + return S->getSCEVType() == scSignExtend; + } +}; + +/// This node is a base class providing common functionality for +/// n'ary operators. +class SCEVNAryExpr : public SCEV { +protected: + // Since SCEVs are immutable, ScalarEvolution allocates operand + // arrays with its SCEVAllocator, so this class just needs a simple + // pointer rather than a more elaborate vector-like data structure. + // This also avoids the need for a non-trivial destructor. + const SCEV *const *Operands; + size_t NumOperands; + + SCEVNAryExpr(const FoldingSetNodeIDRef ID, enum SCEVTypes T, + const SCEV *const *O, size_t N) + : SCEV(ID, T, computeExpressionSize(makeArrayRef(O, N))), Operands(O), + NumOperands(N) {} + +public: + size_t getNumOperands() const { return NumOperands; } + + const SCEV *getOperand(unsigned i) const { + assert(i < NumOperands && "Operand index out of range!"); + return Operands[i]; + } - using op_iterator = const SCEV *const *; - using op_range = iterator_range<op_iterator>; + using op_iterator = const SCEV *const *; + using op_range = iterator_range<op_iterator>; - op_iterator op_begin() const { return Operands; } - op_iterator op_end() const { return Operands + NumOperands; } - op_range operands() const { - return make_range(op_begin(), op_end()); - } + op_iterator op_begin() const { return Operands; } + op_iterator op_end() const { return Operands + NumOperands; } + op_range operands() const { return make_range(op_begin(), op_end()); } - NoWrapFlags getNoWrapFlags(NoWrapFlags Mask = NoWrapMask) const { - return (NoWrapFlags)(SubclassData & Mask); - } + NoWrapFlags getNoWrapFlags(NoWrapFlags Mask = NoWrapMask) const { + return (NoWrapFlags)(SubclassData & Mask); + } - bool hasNoUnsignedWrap() const { - return getNoWrapFlags(FlagNUW) != FlagAnyWrap; - } + bool hasNoUnsignedWrap() const { + return getNoWrapFlags(FlagNUW) != FlagAnyWrap; + } - bool hasNoSignedWrap() const { - return getNoWrapFlags(FlagNSW) != FlagAnyWrap; - } + bool hasNoSignedWrap() const { + return getNoWrapFlags(FlagNSW) != FlagAnyWrap; + } - bool hasNoSelfWrap() const { - return getNoWrapFlags(FlagNW) != FlagAnyWrap; - } + bool hasNoSelfWrap() const { return getNoWrapFlags(FlagNW) != FlagAnyWrap; } - /// Methods for support type inquiry through isa, cast, and dyn_cast: - static bool classof(const SCEV *S) { - return S->getSCEVType() == scAddExpr || S->getSCEVType() == scMulExpr || - S->getSCEVType() == scSMaxExpr || S->getSCEVType() == scUMaxExpr || - S->getSCEVType() == scSMinExpr || S->getSCEVType() == scUMinExpr || - S->getSCEVType() == scAddRecExpr; - } - }; + /// Methods for support type inquiry through isa, cast, and dyn_cast: + static bool classof(const SCEV *S) { + return S->getSCEVType() == scAddExpr || S->getSCEVType() == scMulExpr || + S->getSCEVType() == scSMaxExpr || S->getSCEVType() == scUMaxExpr || + S->getSCEVType() == scSMinExpr || S->getSCEVType() == scUMinExpr || + S->getSCEVType() == scSequentialUMinExpr || + S->getSCEVType() == scAddRecExpr; + } +}; - /// This node is the base class for n'ary commutative operators. - class SCEVCommutativeExpr : public SCEVNAryExpr { - protected: - SCEVCommutativeExpr(const FoldingSetNodeIDRef ID, - enum SCEVTypes T, const SCEV *const *O, size_t N) +/// This node is the base class for n'ary commutative operators. +class SCEVCommutativeExpr : public SCEVNAryExpr { +protected: + SCEVCommutativeExpr(const FoldingSetNodeIDRef ID, enum SCEVTypes T, + const SCEV *const *O, size_t N) : SCEVNAryExpr(ID, T, O, N) {} - public: - /// Methods for support type inquiry through isa, cast, and dyn_cast: - static bool classof(const SCEV *S) { - return S->getSCEVType() == scAddExpr || S->getSCEVType() == scMulExpr || - S->getSCEVType() == scSMaxExpr || S->getSCEVType() == scUMaxExpr || - S->getSCEVType() == scSMinExpr || S->getSCEVType() == scUMinExpr; - } - - /// Set flags for a non-recurrence without clearing previously set flags. - void setNoWrapFlags(NoWrapFlags Flags) { - SubclassData |= Flags; - } - }; +public: + /// Methods for support type inquiry through isa, cast, and dyn_cast: + static bool classof(const SCEV *S) { + return S->getSCEVType() == scAddExpr || S->getSCEVType() == scMulExpr || + S->getSCEVType() == scSMaxExpr || S->getSCEVType() == scUMaxExpr || + S->getSCEVType() == scSMinExpr || S->getSCEVType() == scUMinExpr; + } - /// This node represents an addition of some number of SCEVs. - class SCEVAddExpr : public SCEVCommutativeExpr { - friend class ScalarEvolution; - - Type *Ty; - - SCEVAddExpr(const FoldingSetNodeIDRef ID, const SCEV *const *O, size_t N) - : SCEVCommutativeExpr(ID, scAddExpr, O, N) { - auto *FirstPointerTypedOp = find_if(operands(), [](const SCEV *Op) { - return Op->getType()->isPointerTy(); - }); - if (FirstPointerTypedOp != operands().end()) - Ty = (*FirstPointerTypedOp)->getType(); - else - Ty = getOperand(0)->getType(); - } + /// Set flags for a non-recurrence without clearing previously set flags. + void setNoWrapFlags(NoWrapFlags Flags) { SubclassData |= Flags; } +}; + +/// This node represents an addition of some number of SCEVs. +class SCEVAddExpr : public SCEVCommutativeExpr { + friend class ScalarEvolution; + + Type *Ty; + + SCEVAddExpr(const FoldingSetNodeIDRef ID, const SCEV *const *O, size_t N) + : SCEVCommutativeExpr(ID, scAddExpr, O, N) { + auto *FirstPointerTypedOp = find_if(operands(), [](const SCEV *Op) { + return Op->getType()->isPointerTy(); + }); + if (FirstPointerTypedOp != operands().end()) + Ty = (*FirstPointerTypedOp)->getType(); + else + Ty = getOperand(0)->getType(); + } - public: - Type *getType() const { return Ty; } +public: + Type *getType() const { return Ty; } - /// Methods for support type inquiry through isa, cast, and dyn_cast: - static bool classof(const SCEV *S) { - return S->getSCEVType() == scAddExpr; - } - }; + /// Methods for support type inquiry through isa, cast, and dyn_cast: + static bool classof(const SCEV *S) { return S->getSCEVType() == scAddExpr; } +}; - /// This node represents multiplication of some number of SCEVs. - class SCEVMulExpr : public SCEVCommutativeExpr { - friend class ScalarEvolution; +/// This node represents multiplication of some number of SCEVs. +class SCEVMulExpr : public SCEVCommutativeExpr { + friend class ScalarEvolution; - SCEVMulExpr(const FoldingSetNodeIDRef ID, - const SCEV *const *O, size_t N) + SCEVMulExpr(const FoldingSetNodeIDRef ID, const SCEV *const *O, size_t N) : SCEVCommutativeExpr(ID, scMulExpr, O, N) {} - public: - Type *getType() const { return getOperand(0)->getType(); } +public: + Type *getType() const { return getOperand(0)->getType(); } - /// Methods for support type inquiry through isa, cast, and dyn_cast: - static bool classof(const SCEV *S) { - return S->getSCEVType() == scMulExpr; - } - }; + /// Methods for support type inquiry through isa, cast, and dyn_cast: + static bool classof(const SCEV *S) { return S->getSCEVType() == scMulExpr; } +}; - /// This class represents a binary unsigned division operation. - class SCEVUDivExpr : public SCEV { - friend class ScalarEvolution; +/// This class represents a binary unsigned division operation. +class SCEVUDivExpr : public SCEV { + friend class ScalarEvolution; - std::array<const SCEV *, 2> Operands; + std::array<const SCEV *, 2> Operands; - SCEVUDivExpr(const FoldingSetNodeIDRef ID, const SCEV *lhs, const SCEV *rhs) - : SCEV(ID, scUDivExpr, computeExpressionSize({lhs, rhs})) { - Operands[0] = lhs; - Operands[1] = rhs; - } - - public: - const SCEV *getLHS() const { return Operands[0]; } - const SCEV *getRHS() const { return Operands[1]; } - size_t getNumOperands() const { return 2; } - const SCEV *getOperand(unsigned i) const { - assert((i == 0 || i == 1) && "Operand index out of range!"); - return i == 0 ? getLHS() : getRHS(); - } + SCEVUDivExpr(const FoldingSetNodeIDRef ID, const SCEV *lhs, const SCEV *rhs) + : SCEV(ID, scUDivExpr, computeExpressionSize({lhs, rhs})) { + Operands[0] = lhs; + Operands[1] = rhs; + } - using op_iterator = std::array<const SCEV *, 2>::const_iterator; - using op_range = iterator_range<op_iterator>; - op_range operands() const { - return make_range(Operands.begin(), Operands.end()); - } +public: + const SCEV *getLHS() const { return Operands[0]; } + const SCEV *getRHS() const { return Operands[1]; } + size_t getNumOperands() const { return 2; } + const SCEV *getOperand(unsigned i) const { + assert((i == 0 || i == 1) && "Operand index out of range!"); + return i == 0 ? getLHS() : getRHS(); + } - Type *getType() const { - // In most cases the types of LHS and RHS will be the same, but in some - // crazy cases one or the other may be a pointer. ScalarEvolution doesn't - // depend on the type for correctness, but handling types carefully can - // avoid extra casts in the SCEVExpander. The LHS is more likely to be - // a pointer type than the RHS, so use the RHS' type here. - return getRHS()->getType(); - } + using op_iterator = std::array<const SCEV *, 2>::const_iterator; + using op_range = iterator_range<op_iterator>; + op_range operands() const { + return make_range(Operands.begin(), Operands.end()); + } - /// Methods for support type inquiry through isa, cast, and dyn_cast: - static bool classof(const SCEV *S) { - return S->getSCEVType() == scUDivExpr; - } - }; + Type *getType() const { + // In most cases the types of LHS and RHS will be the same, but in some + // crazy cases one or the other may be a pointer. ScalarEvolution doesn't + // depend on the type for correctness, but handling types carefully can + // avoid extra casts in the SCEVExpander. The LHS is more likely to be + // a pointer type than the RHS, so use the RHS' type here. + return getRHS()->getType(); + } - /// This node represents a polynomial recurrence on the trip count - /// of the specified loop. This is the primary focus of the - /// ScalarEvolution framework; all the other SCEV subclasses are - /// mostly just supporting infrastructure to allow SCEVAddRecExpr - /// expressions to be created and analyzed. - /// - /// All operands of an AddRec are required to be loop invariant. - /// - class SCEVAddRecExpr : public SCEVNAryExpr { - friend class ScalarEvolution; - - const Loop *L; - - SCEVAddRecExpr(const FoldingSetNodeIDRef ID, - const SCEV *const *O, size_t N, const Loop *l) + /// Methods for support type inquiry through isa, cast, and dyn_cast: + static bool classof(const SCEV *S) { return S->getSCEVType() == scUDivExpr; } +}; + +/// This node represents a polynomial recurrence on the trip count +/// of the specified loop. This is the primary focus of the +/// ScalarEvolution framework; all the other SCEV subclasses are +/// mostly just supporting infrastructure to allow SCEVAddRecExpr +/// expressions to be created and analyzed. +/// +/// All operands of an AddRec are required to be loop invariant. +/// +class SCEVAddRecExpr : public SCEVNAryExpr { + friend class ScalarEvolution; + + const Loop *L; + + SCEVAddRecExpr(const FoldingSetNodeIDRef ID, const SCEV *const *O, size_t N, + const Loop *l) : SCEVNAryExpr(ID, scAddRecExpr, O, N), L(l) {} - public: - Type *getType() const { return getStart()->getType(); } - const SCEV *getStart() const { return Operands[0]; } - const Loop *getLoop() const { return L; } - - /// Constructs and returns the recurrence indicating how much this - /// expression steps by. If this is a polynomial of degree N, it - /// returns a chrec of degree N-1. We cannot determine whether - /// the step recurrence has self-wraparound. - const SCEV *getStepRecurrence(ScalarEvolution &SE) const { - if (isAffine()) return getOperand(1); - return SE.getAddRecExpr(SmallVector<const SCEV *, 3>(op_begin()+1, - op_end()), - getLoop(), FlagAnyWrap); - } - - /// Return true if this represents an expression A + B*x where A - /// and B are loop invariant values. - bool isAffine() const { - // We know that the start value is invariant. This expression is thus - // affine iff the step is also invariant. - return getNumOperands() == 2; - } +public: + Type *getType() const { return getStart()->getType(); } + const SCEV *getStart() const { return Operands[0]; } + const Loop *getLoop() const { return L; } + + /// Constructs and returns the recurrence indicating how much this + /// expression steps by. If this is a polynomial of degree N, it + /// returns a chrec of degree N-1. We cannot determine whether + /// the step recurrence has self-wraparound. + const SCEV *getStepRecurrence(ScalarEvolution &SE) const { + if (isAffine()) + return getOperand(1); + return SE.getAddRecExpr( + SmallVector<const SCEV *, 3>(op_begin() + 1, op_end()), getLoop(), + FlagAnyWrap); + } - /// Return true if this represents an expression A + B*x + C*x^2 - /// where A, B and C are loop invariant values. This corresponds - /// to an addrec of the form {L,+,M,+,N} - bool isQuadratic() const { - return getNumOperands() == 3; - } + /// Return true if this represents an expression A + B*x where A + /// and B are loop invariant values. + bool isAffine() const { + // We know that the start value is invariant. This expression is thus + // affine iff the step is also invariant. + return getNumOperands() == 2; + } - /// Set flags for a recurrence without clearing any previously set flags. - /// For AddRec, either NUW or NSW implies NW. Keep track of this fact here - /// to make it easier to propagate flags. - void setNoWrapFlags(NoWrapFlags Flags) { - if (Flags & (FlagNUW | FlagNSW)) - Flags = ScalarEvolution::setFlags(Flags, FlagNW); - SubclassData |= Flags; - } + /// Return true if this represents an expression A + B*x + C*x^2 + /// where A, B and C are loop invariant values. This corresponds + /// to an addrec of the form {L,+,M,+,N} + bool isQuadratic() const { return getNumOperands() == 3; } + + /// Set flags for a recurrence without clearing any previously set flags. + /// For AddRec, either NUW or NSW implies NW. Keep track of this fact here + /// to make it easier to propagate flags. + void setNoWrapFlags(NoWrapFlags Flags) { + if (Flags & (FlagNUW | FlagNSW)) + Flags = ScalarEvolution::setFlags(Flags, FlagNW); + SubclassData |= Flags; + } - /// Return the value of this chain of recurrences at the specified - /// iteration number. - const SCEV *evaluateAtIteration(const SCEV *It, ScalarEvolution &SE) const; - - /// Return the value of this chain of recurrences at the specified iteration - /// number. Takes an explicit list of operands to represent an AddRec. - static const SCEV *evaluateAtIteration(ArrayRef<const SCEV *> Operands, - const SCEV *It, ScalarEvolution &SE); - - /// Return the number of iterations of this loop that produce - /// values in the specified constant range. Another way of - /// looking at this is that it returns the first iteration number - /// where the value is not in the condition, thus computing the - /// exit count. If the iteration count can't be computed, an - /// instance of SCEVCouldNotCompute is returned. - const SCEV *getNumIterationsInRange(const ConstantRange &Range, - ScalarEvolution &SE) const; - - /// Return an expression representing the value of this expression - /// one iteration of the loop ahead. - const SCEVAddRecExpr *getPostIncExpr(ScalarEvolution &SE) const; - - /// Methods for support type inquiry through isa, cast, and dyn_cast: - static bool classof(const SCEV *S) { - return S->getSCEVType() == scAddRecExpr; - } - }; + /// Return the value of this chain of recurrences at the specified + /// iteration number. + const SCEV *evaluateAtIteration(const SCEV *It, ScalarEvolution &SE) const; + + /// Return the value of this chain of recurrences at the specified iteration + /// number. Takes an explicit list of operands to represent an AddRec. + static const SCEV *evaluateAtIteration(ArrayRef<const SCEV *> Operands, + const SCEV *It, ScalarEvolution &SE); + + /// Return the number of iterations of this loop that produce + /// values in the specified constant range. Another way of + /// looking at this is that it returns the first iteration number + /// where the value is not in the condition, thus computing the + /// exit count. If the iteration count can't be computed, an + /// instance of SCEVCouldNotCompute is returned. + const SCEV *getNumIterationsInRange(const ConstantRange &Range, + ScalarEvolution &SE) const; + + /// Return an expression representing the value of this expression + /// one iteration of the loop ahead. + const SCEVAddRecExpr *getPostIncExpr(ScalarEvolution &SE) const; + + /// Methods for support type inquiry through isa, cast, and dyn_cast: + static bool classof(const SCEV *S) { + return S->getSCEVType() == scAddRecExpr; + } +}; - /// This node is the base class min/max selections. - class SCEVMinMaxExpr : public SCEVCommutativeExpr { - friend class ScalarEvolution; +/// This node is the base class min/max selections. +class SCEVMinMaxExpr : public SCEVCommutativeExpr { + friend class ScalarEvolution; - static bool isMinMaxType(enum SCEVTypes T) { - return T == scSMaxExpr || T == scUMaxExpr || T == scSMinExpr || - T == scUMinExpr; - } + static bool isMinMaxType(enum SCEVTypes T) { + return T == scSMaxExpr || T == scUMaxExpr || T == scSMinExpr || + T == scUMinExpr; + } - protected: - /// Note: Constructing subclasses via this constructor is allowed - SCEVMinMaxExpr(const FoldingSetNodeIDRef ID, enum SCEVTypes T, - const SCEV *const *O, size_t N) - : SCEVCommutativeExpr(ID, T, O, N) { - assert(isMinMaxType(T)); - // Min and max never overflow - setNoWrapFlags((NoWrapFlags)(FlagNUW | FlagNSW)); - } +protected: + /// Note: Constructing subclasses via this constructor is allowed + SCEVMinMaxExpr(const FoldingSetNodeIDRef ID, enum SCEVTypes T, + const SCEV *const *O, size_t N) + : SCEVCommutativeExpr(ID, T, O, N) { + assert(isMinMaxType(T)); + // Min and max never overflow + setNoWrapFlags((NoWrapFlags)(FlagNUW | FlagNSW)); + } - public: - Type *getType() const { return getOperand(0)->getType(); } +public: + Type *getType() const { return getOperand(0)->getType(); } - static bool classof(const SCEV *S) { - return isMinMaxType(S->getSCEVType()); - } + static bool classof(const SCEV *S) { return isMinMaxType(S->getSCEVType()); } - static enum SCEVTypes negate(enum SCEVTypes T) { - switch (T) { - case scSMaxExpr: - return scSMinExpr; - case scSMinExpr: - return scSMaxExpr; - case scUMaxExpr: - return scUMinExpr; - case scUMinExpr: - return scUMaxExpr; - default: - llvm_unreachable("Not a min or max SCEV type!"); - } + static enum SCEVTypes negate(enum SCEVTypes T) { + switch (T) { + case scSMaxExpr: + return scSMinExpr; + case scSMinExpr: + return scSMaxExpr; + case scUMaxExpr: + return scUMinExpr; + case scUMinExpr: + return scUMaxExpr; + default: + llvm_unreachable("Not a min or max SCEV type!"); } - }; + } +}; + +/// This class represents a signed maximum selection. +class SCEVSMaxExpr : public SCEVMinMaxExpr { + friend class ScalarEvolution; + + SCEVSMaxExpr(const FoldingSetNodeIDRef ID, const SCEV *const *O, size_t N) + : SCEVMinMaxExpr(ID, scSMaxExpr, O, N) {} + +public: + /// Methods for support type inquiry through isa, cast, and dyn_cast: + static bool classof(const SCEV *S) { return S->getSCEVType() == scSMaxExpr; } +}; + +/// This class represents an unsigned maximum selection. +class SCEVUMaxExpr : public SCEVMinMaxExpr { + friend class ScalarEvolution; + + SCEVUMaxExpr(const FoldingSetNodeIDRef ID, const SCEV *const *O, size_t N) + : SCEVMinMaxExpr(ID, scUMaxExpr, O, N) {} + +public: + /// Methods for support type inquiry through isa, cast, and dyn_cast: + static bool classof(const SCEV *S) { return S->getSCEVType() == scUMaxExpr; } +}; + +/// This class represents a signed minimum selection. +class SCEVSMinExpr : public SCEVMinMaxExpr { + friend class ScalarEvolution; + + SCEVSMinExpr(const FoldingSetNodeIDRef ID, const SCEV *const *O, size_t N) + : SCEVMinMaxExpr(ID, scSMinExpr, O, N) {} + +public: + /// Methods for support type inquiry through isa, cast, and dyn_cast: + static bool classof(const SCEV *S) { return S->getSCEVType() == scSMinExpr; } +}; + +/// This class represents an unsigned minimum selection. +class SCEVUMinExpr : public SCEVMinMaxExpr { + friend class ScalarEvolution; + + SCEVUMinExpr(const FoldingSetNodeIDRef ID, const SCEV *const *O, size_t N) + : SCEVMinMaxExpr(ID, scUMinExpr, O, N) {} + +public: + /// Methods for support type inquiry through isa, cast, and dyn_cast: + static bool classof(const SCEV *S) { return S->getSCEVType() == scUMinExpr; } +}; + +/// This node is the base class for sequential/in-order min/max selections. +/// Note that their fundamental difference from SCEVMinMaxExpr's is that they +/// are early-returning upon reaching saturation point. +/// I.e. given `0 umin_seq poison`, the result will be `0`, +/// while the result of `0 umin poison` is `poison`. +class SCEVSequentialMinMaxExpr : public SCEVNAryExpr { + friend class ScalarEvolution; + + static bool isSequentialMinMaxType(enum SCEVTypes T) { + return T == scSequentialUMinExpr; + } - /// This class represents a signed maximum selection. - class SCEVSMaxExpr : public SCEVMinMaxExpr { - friend class ScalarEvolution; + /// Set flags for a non-recurrence without clearing previously set flags. + void setNoWrapFlags(NoWrapFlags Flags) { SubclassData |= Flags; } + +protected: + /// Note: Constructing subclasses via this constructor is allowed + SCEVSequentialMinMaxExpr(const FoldingSetNodeIDRef ID, enum SCEVTypes T, + const SCEV *const *O, size_t N) + : SCEVNAryExpr(ID, T, O, N) { + assert(isSequentialMinMaxType(T)); + // Min and max never overflow + setNoWrapFlags((NoWrapFlags)(FlagNUW | FlagNSW)); + } - SCEVSMaxExpr(const FoldingSetNodeIDRef ID, const SCEV *const *O, size_t N) - : SCEVMinMaxExpr(ID, scSMaxExpr, O, N) {} +public: + Type *getType() const { return getOperand(0)->getType(); } - public: - /// Methods for support type inquiry through isa, cast, and dyn_cast: - static bool classof(const SCEV *S) { - return S->getSCEVType() == scSMaxExpr; + static SCEVTypes getEquivalentNonSequentialSCEVType(SCEVTypes Ty) { + assert(isSequentialMinMaxType(Ty)); + switch (Ty) { + case scSequentialUMinExpr: + return scUMinExpr; + default: + llvm_unreachable("Not a sequential min/max type."); } - }; - - /// This class represents an unsigned maximum selection. - class SCEVUMaxExpr : public SCEVMinMaxExpr { - friend class ScalarEvolution; - - SCEVUMaxExpr(const FoldingSetNodeIDRef ID, const SCEV *const *O, size_t N) - : SCEVMinMaxExpr(ID, scUMaxExpr, O, N) {} + } - public: - /// Methods for support type inquiry through isa, cast, and dyn_cast: - static bool classof(const SCEV *S) { - return S->getSCEVType() == scUMaxExpr; - } - }; + SCEVTypes getEquivalentNonSequentialSCEVType() const { + return getEquivalentNonSequentialSCEVType(getSCEVType()); + } - /// This class represents a signed minimum selection. - class SCEVSMinExpr : public SCEVMinMaxExpr { - friend class ScalarEvolution; + static bool classof(const SCEV *S) { + return isSequentialMinMaxType(S->getSCEVType()); + } +}; - SCEVSMinExpr(const FoldingSetNodeIDRef ID, const SCEV *const *O, size_t N) - : SCEVMinMaxExpr(ID, scSMinExpr, O, N) {} +/// This class represents a sequential/in-order unsigned minimum selection. +class SCEVSequentialUMinExpr : public SCEVSequentialMinMaxExpr { + friend class ScalarEvolution; - public: - /// Methods for support type inquiry through isa, cast, and dyn_cast: - static bool classof(const SCEV *S) { - return S->getSCEVType() == scSMinExpr; - } - }; + SCEVSequentialUMinExpr(const FoldingSetNodeIDRef ID, const SCEV *const *O, + size_t N) + : SCEVSequentialMinMaxExpr(ID, scSequentialUMinExpr, O, N) {} - /// This class represents an unsigned minimum selection. - class SCEVUMinExpr : public SCEVMinMaxExpr { - friend class ScalarEvolution; +public: + /// Methods for support type inquiry through isa, cast, and dyn_cast: + static bool classof(const SCEV *S) { + return S->getSCEVType() == scSequentialUMinExpr; + } +}; + +/// This means that we are dealing with an entirely unknown SCEV +/// value, and only represent it as its LLVM Value. This is the +/// "bottom" value for the analysis. +class SCEVUnknown final : public SCEV, private CallbackVH { + friend class ScalarEvolution; + + /// The parent ScalarEvolution value. This is used to update the + /// parent's maps when the value associated with a SCEVUnknown is + /// deleted or RAUW'd. + ScalarEvolution *SE; + + /// The next pointer in the linked list of all SCEVUnknown + /// instances owned by a ScalarEvolution. + SCEVUnknown *Next; + + SCEVUnknown(const FoldingSetNodeIDRef ID, Value *V, ScalarEvolution *se, + SCEVUnknown *next) + : SCEV(ID, scUnknown, 1), CallbackVH(V), SE(se), Next(next) {} + + // Implement CallbackVH. + void deleted() override; + void allUsesReplacedWith(Value *New) override; + +public: + Value *getValue() const { return getValPtr(); } + + /// @{ + /// Test whether this is a special constant representing a type + /// size, alignment, or field offset in a target-independent + /// manner, and hasn't happened to have been folded with other + /// operations into something unrecognizable. This is mainly only + /// useful for pretty-printing and other situations where it isn't + /// absolutely required for these to succeed. + bool isSizeOf(Type *&AllocTy) const; + bool isAlignOf(Type *&AllocTy) const; + bool isOffsetOf(Type *&STy, Constant *&FieldNo) const; + /// @} + + Type *getType() const { return getValPtr()->getType(); } + + /// Methods for support type inquiry through isa, cast, and dyn_cast: + static bool classof(const SCEV *S) { return S->getSCEVType() == scUnknown; } +}; + +/// This class defines a simple visitor class that may be used for +/// various SCEV analysis purposes. +template <typename SC, typename RetVal = void> struct SCEVVisitor { + RetVal visit(const SCEV *S) { + switch (S->getSCEVType()) { + case scConstant: + return ((SC *)this)->visitConstant((const SCEVConstant *)S); + case scPtrToInt: + return ((SC *)this)->visitPtrToIntExpr((const SCEVPtrToIntExpr *)S); + case scTruncate: + return ((SC *)this)->visitTruncateExpr((const SCEVTruncateExpr *)S); + case scZeroExtend: + return ((SC *)this)->visitZeroExtendExpr((const SCEVZeroExtendExpr *)S); + case scSignExtend: + return ((SC *)this)->visitSignExtendExpr((const SCEVSignExtendExpr *)S); + case scAddExpr: + return ((SC *)this)->visitAddExpr((const SCEVAddExpr *)S); + case scMulExpr: + return ((SC *)this)->visitMulExpr((const SCEVMulExpr *)S); + case scUDivExpr: + return ((SC *)this)->visitUDivExpr((const SCEVUDivExpr *)S); + case scAddRecExpr: + return ((SC *)this)->visitAddRecExpr((const SCEVAddRecExpr *)S); + case scSMaxExpr: + return ((SC *)this)->visitSMaxExpr((const SCEVSMaxExpr *)S); + case scUMaxExpr: + return ((SC *)this)->visitUMaxExpr((const SCEVUMaxExpr *)S); + case scSMinExpr: + return ((SC *)this)->visitSMinExpr((const SCEVSMinExpr *)S); + case scUMinExpr: + return ((SC *)this)->visitUMinExpr((const SCEVUMinExpr *)S); + case scSequentialUMinExpr: + return ((SC *)this) + ->visitSequentialUMinExpr((const SCEVSequentialUMinExpr *)S); + case scUnknown: + return ((SC *)this)->visitUnknown((const SCEVUnknown *)S); + case scCouldNotCompute: + return ((SC *)this)->visitCouldNotCompute((const SCEVCouldNotCompute *)S); + } + llvm_unreachable("Unknown SCEV kind!"); + } - SCEVUMinExpr(const FoldingSetNodeIDRef ID, const SCEV *const *O, size_t N) - : SCEVMinMaxExpr(ID, scUMinExpr, O, N) {} + RetVal visitCouldNotCompute(const SCEVCouldNotCompute *S) { + llvm_unreachable("Invalid use of SCEVCouldNotCompute!"); + } +}; + +/// Visit all nodes in the expression tree using worklist traversal. +/// +/// Visitor implements: +/// // return true to follow this node. +/// bool follow(const SCEV *S); +/// // return true to terminate the search. +/// bool isDone(); +template <typename SV> class SCEVTraversal { + SV &Visitor; + SmallVector<const SCEV *, 8> Worklist; + SmallPtrSet<const SCEV *, 8> Visited; + + void push(const SCEV *S) { + if (Visited.insert(S).second && Visitor.follow(S)) + Worklist.push_back(S); + } - public: - /// Methods for support type inquiry through isa, cast, and dyn_cast: - static bool classof(const SCEV *S) { - return S->getSCEVType() == scUMinExpr; - } - }; +public: + SCEVTraversal(SV &V) : Visitor(V) {} - /// This means that we are dealing with an entirely unknown SCEV - /// value, and only represent it as its LLVM Value. This is the - /// "bottom" value for the analysis. - class SCEVUnknown final : public SCEV, private CallbackVH { - friend class ScalarEvolution; - - /// The parent ScalarEvolution value. This is used to update the - /// parent's maps when the value associated with a SCEVUnknown is - /// deleted or RAUW'd. - ScalarEvolution *SE; - - /// The next pointer in the linked list of all SCEVUnknown - /// instances owned by a ScalarEvolution. - SCEVUnknown *Next; - - SCEVUnknown(const FoldingSetNodeIDRef ID, Value *V, - ScalarEvolution *se, SCEVUnknown *next) : - SCEV(ID, scUnknown, 1), CallbackVH(V), SE(se), Next(next) {} - - // Implement CallbackVH. - void deleted() override; - void allUsesReplacedWith(Value *New) override; - - public: - Value *getValue() const { return getValPtr(); } - - /// @{ - /// Test whether this is a special constant representing a type - /// size, alignment, or field offset in a target-independent - /// manner, and hasn't happened to have been folded with other - /// operations into something unrecognizable. This is mainly only - /// useful for pretty-printing and other situations where it isn't - /// absolutely required for these to succeed. - bool isSizeOf(Type *&AllocTy) const; - bool isAlignOf(Type *&AllocTy) const; - bool isOffsetOf(Type *&STy, Constant *&FieldNo) const; - /// @} - - Type *getType() const { return getValPtr()->getType(); } - - /// Methods for support type inquiry through isa, cast, and dyn_cast: - static bool classof(const SCEV *S) { - return S->getSCEVType() == scUnknown; - } - }; + void visitAll(const SCEV *Root) { + push(Root); + while (!Worklist.empty() && !Visitor.isDone()) { + const SCEV *S = Worklist.pop_back_val(); - /// This class defines a simple visitor class that may be used for - /// various SCEV analysis purposes. - template<typename SC, typename RetVal=void> - struct SCEVVisitor { - RetVal visit(const SCEV *S) { switch (S->getSCEVType()) { case scConstant: - return ((SC*)this)->visitConstant((const SCEVConstant*)S); + case scUnknown: + continue; case scPtrToInt: - return ((SC *)this)->visitPtrToIntExpr((const SCEVPtrToIntExpr *)S); case scTruncate: - return ((SC*)this)->visitTruncateExpr((const SCEVTruncateExpr*)S); case scZeroExtend: - return ((SC*)this)->visitZeroExtendExpr((const SCEVZeroExtendExpr*)S); case scSignExtend: - return ((SC*)this)->visitSignExtendExpr((const SCEVSignExtendExpr*)S); + push(cast<SCEVCastExpr>(S)->getOperand()); + continue; case scAddExpr: - return ((SC*)this)->visitAddExpr((const SCEVAddExpr*)S); case scMulExpr: - return ((SC*)this)->visitMulExpr((const SCEVMulExpr*)S); - case scUDivExpr: - return ((SC*)this)->visitUDivExpr((const SCEVUDivExpr*)S); - case scAddRecExpr: - return ((SC*)this)->visitAddRecExpr((const SCEVAddRecExpr*)S); case scSMaxExpr: - return ((SC*)this)->visitSMaxExpr((const SCEVSMaxExpr*)S); case scUMaxExpr: - return ((SC*)this)->visitUMaxExpr((const SCEVUMaxExpr*)S); case scSMinExpr: - return ((SC *)this)->visitSMinExpr((const SCEVSMinExpr *)S); case scUMinExpr: - return ((SC *)this)->visitUMinExpr((const SCEVUMinExpr *)S); - case scUnknown: - return ((SC*)this)->visitUnknown((const SCEVUnknown*)S); + case scSequentialUMinExpr: + case scAddRecExpr: + for (const auto *Op : cast<SCEVNAryExpr>(S)->operands()) + push(Op); + continue; + case scUDivExpr: { + const SCEVUDivExpr *UDiv = cast<SCEVUDivExpr>(S); + push(UDiv->getLHS()); + push(UDiv->getRHS()); + continue; + } case scCouldNotCompute: - return ((SC*)this)->visitCouldNotCompute((const SCEVCouldNotCompute*)S); + llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!"); } llvm_unreachable("Unknown SCEV kind!"); } + } +}; - RetVal visitCouldNotCompute(const SCEVCouldNotCompute *S) { - llvm_unreachable("Invalid use of SCEVCouldNotCompute!"); - } - }; +/// Use SCEVTraversal to visit all nodes in the given expression tree. +template <typename SV> void visitAll(const SCEV *Root, SV &Visitor) { + SCEVTraversal<SV> T(Visitor); + T.visitAll(Root); +} - /// Visit all nodes in the expression tree using worklist traversal. - /// - /// Visitor implements: - /// // return true to follow this node. - /// bool follow(const SCEV *S); - /// // return true to terminate the search. - /// bool isDone(); - template<typename SV> - class SCEVTraversal { - SV &Visitor; - SmallVector<const SCEV *, 8> Worklist; - SmallPtrSet<const SCEV *, 8> Visited; - - void push(const SCEV *S) { - if (Visited.insert(S).second && Visitor.follow(S)) - Worklist.push_back(S); - } +/// Return true if any node in \p Root satisfies the predicate \p Pred. +template <typename PredTy> +bool SCEVExprContains(const SCEV *Root, PredTy Pred) { + struct FindClosure { + bool Found = false; + PredTy Pred; - public: - SCEVTraversal(SV& V): Visitor(V) {} - - void visitAll(const SCEV *Root) { - push(Root); - while (!Worklist.empty() && !Visitor.isDone()) { - const SCEV *S = Worklist.pop_back_val(); - - switch (S->getSCEVType()) { - case scConstant: - case scUnknown: - continue; - case scPtrToInt: - case scTruncate: - case scZeroExtend: - case scSignExtend: - push(cast<SCEVCastExpr>(S)->getOperand()); - continue; - case scAddExpr: - case scMulExpr: - case scSMaxExpr: - case scUMaxExpr: - case scSMinExpr: - case scUMinExpr: - case scAddRecExpr: - for (const auto *Op : cast<SCEVNAryExpr>(S)->operands()) - push(Op); - continue; - case scUDivExpr: { - const SCEVUDivExpr *UDiv = cast<SCEVUDivExpr>(S); - push(UDiv->getLHS()); - push(UDiv->getRHS()); - continue; - } - case scCouldNotCompute: - llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!"); - } - llvm_unreachable("Unknown SCEV kind!"); - } + FindClosure(PredTy Pred) : Pred(Pred) {} + + bool follow(const SCEV *S) { + if (!Pred(S)) + return true; + + Found = true; + return false; } + + bool isDone() const { return Found; } }; - /// Use SCEVTraversal to visit all nodes in the given expression tree. - template<typename SV> - void visitAll(const SCEV *Root, SV& Visitor) { - SCEVTraversal<SV> T(Visitor); - T.visitAll(Root); + FindClosure FC(Pred); + visitAll(Root, FC); + return FC.Found; +} + +/// This visitor recursively visits a SCEV expression and re-writes it. +/// The result from each visit is cached, so it will return the same +/// SCEV for the same input. +template <typename SC> +class SCEVRewriteVisitor : public SCEVVisitor<SC, const SCEV *> { +protected: + ScalarEvolution &SE; + // Memoize the result of each visit so that we only compute once for + // the same input SCEV. This is to avoid redundant computations when + // a SCEV is referenced by multiple SCEVs. Without memoization, this + // visit algorithm would have exponential time complexity in the worst + // case, causing the compiler to hang on certain tests. + DenseMap<const SCEV *, const SCEV *> RewriteResults; + +public: + SCEVRewriteVisitor(ScalarEvolution &SE) : SE(SE) {} + + const SCEV *visit(const SCEV *S) { + auto It = RewriteResults.find(S); + if (It != RewriteResults.end()) + return It->second; + auto *Visited = SCEVVisitor<SC, const SCEV *>::visit(S); + auto Result = RewriteResults.try_emplace(S, Visited); + assert(Result.second && "Should insert a new entry"); + return Result.first->second; } - /// Return true if any node in \p Root satisfies the predicate \p Pred. - template <typename PredTy> - bool SCEVExprContains(const SCEV *Root, PredTy Pred) { - struct FindClosure { - bool Found = false; - PredTy Pred; - - FindClosure(PredTy Pred) : Pred(Pred) {} + const SCEV *visitConstant(const SCEVConstant *Constant) { return Constant; } - bool follow(const SCEV *S) { - if (!Pred(S)) - return true; - - Found = true; - return false; - } + const SCEV *visitPtrToIntExpr(const SCEVPtrToIntExpr *Expr) { + const SCEV *Operand = ((SC *)this)->visit(Expr->getOperand()); + return Operand == Expr->getOperand() + ? Expr + : SE.getPtrToIntExpr(Operand, Expr->getType()); + } - bool isDone() const { return Found; } - }; - - FindClosure FC(Pred); - visitAll(Root, FC); - return FC.Found; - } - - /// This visitor recursively visits a SCEV expression and re-writes it. - /// The result from each visit is cached, so it will return the same - /// SCEV for the same input. - template<typename SC> - class SCEVRewriteVisitor : public SCEVVisitor<SC, const SCEV *> { - protected: - ScalarEvolution &SE; - // Memoize the result of each visit so that we only compute once for - // the same input SCEV. This is to avoid redundant computations when - // a SCEV is referenced by multiple SCEVs. Without memoization, this - // visit algorithm would have exponential time complexity in the worst - // case, causing the compiler to hang on certain tests. - DenseMap<const SCEV *, const SCEV *> RewriteResults; - - public: - SCEVRewriteVisitor(ScalarEvolution &SE) : SE(SE) {} - - const SCEV *visit(const SCEV *S) { - auto It = RewriteResults.find(S); - if (It != RewriteResults.end()) - return It->second; - auto* Visited = SCEVVisitor<SC, const SCEV *>::visit(S); - auto Result = RewriteResults.try_emplace(S, Visited); - assert(Result.second && "Should insert a new entry"); - return Result.first->second; - } + const SCEV *visitTruncateExpr(const SCEVTruncateExpr *Expr) { + const SCEV *Operand = ((SC *)this)->visit(Expr->getOperand()); + return Operand == Expr->getOperand() + ? Expr + : SE.getTruncateExpr(Operand, Expr->getType()); + } - const SCEV *visitConstant(const SCEVConstant *Constant) { - return Constant; - } + const SCEV *visitZeroExtendExpr(const SCEVZeroExtendExpr *Expr) { + const SCEV *Operand = ((SC *)this)->visit(Expr->getOperand()); + return Operand == Expr->getOperand() + ? Expr + : SE.getZeroExtendExpr(Operand, Expr->getType()); + } - const SCEV *visitPtrToIntExpr(const SCEVPtrToIntExpr *Expr) { - const SCEV *Operand = ((SC *)this)->visit(Expr->getOperand()); - return Operand == Expr->getOperand() - ? Expr - : SE.getPtrToIntExpr(Operand, Expr->getType()); - } + const SCEV *visitSignExtendExpr(const SCEVSignExtendExpr *Expr) { + const SCEV *Operand = ((SC *)this)->visit(Expr->getOperand()); + return Operand == Expr->getOperand() + ? Expr + : SE.getSignExtendExpr(Operand, Expr->getType()); + } - const SCEV *visitTruncateExpr(const SCEVTruncateExpr *Expr) { - const SCEV *Operand = ((SC*)this)->visit(Expr->getOperand()); - return Operand == Expr->getOperand() - ? Expr - : SE.getTruncateExpr(Operand, Expr->getType()); + const SCEV *visitAddExpr(const SCEVAddExpr *Expr) { + SmallVector<const SCEV *, 2> Operands; + bool Changed = false; + for (auto *Op : Expr->operands()) { + Operands.push_back(((SC *)this)->visit(Op)); + Changed |= Op != Operands.back(); } + return !Changed ? Expr : SE.getAddExpr(Operands); + } - const SCEV *visitZeroExtendExpr(const SCEVZeroExtendExpr *Expr) { - const SCEV *Operand = ((SC*)this)->visit(Expr->getOperand()); - return Operand == Expr->getOperand() - ? Expr - : SE.getZeroExtendExpr(Operand, Expr->getType()); + const SCEV *visitMulExpr(const SCEVMulExpr *Expr) { + SmallVector<const SCEV *, 2> Operands; + bool Changed = false; + for (auto *Op : Expr->operands()) { + Operands.push_back(((SC *)this)->visit(Op)); + Changed |= Op != Operands.back(); } + return !Changed ? Expr : SE.getMulExpr(Operands); + } - const SCEV *visitSignExtendExpr(const SCEVSignExtendExpr *Expr) { - const SCEV *Operand = ((SC*)this)->visit(Expr->getOperand()); - return Operand == Expr->getOperand() - ? Expr - : SE.getSignExtendExpr(Operand, Expr->getType()); - } + const SCEV *visitUDivExpr(const SCEVUDivExpr *Expr) { + auto *LHS = ((SC *)this)->visit(Expr->getLHS()); + auto *RHS = ((SC *)this)->visit(Expr->getRHS()); + bool Changed = LHS != Expr->getLHS() || RHS != Expr->getRHS(); + return !Changed ? Expr : SE.getUDivExpr(LHS, RHS); + } - const SCEV *visitAddExpr(const SCEVAddExpr *Expr) { - SmallVector<const SCEV *, 2> Operands; - bool Changed = false; - for (auto *Op : Expr->operands()) { - Operands.push_back(((SC*)this)->visit(Op)); - Changed |= Op != Operands.back(); - } - return !Changed ? Expr : SE.getAddExpr(Operands); + const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) { + SmallVector<const SCEV *, 2> Operands; + bool Changed = false; + for (auto *Op : Expr->operands()) { + Operands.push_back(((SC *)this)->visit(Op)); + Changed |= Op != Operands.back(); } + return !Changed ? Expr + : SE.getAddRecExpr(Operands, Expr->getLoop(), + Expr->getNoWrapFlags()); + } - const SCEV *visitMulExpr(const SCEVMulExpr *Expr) { - SmallVector<const SCEV *, 2> Operands; - bool Changed = false; - for (auto *Op : Expr->operands()) { - Operands.push_back(((SC*)this)->visit(Op)); - Changed |= Op != Operands.back(); - } - return !Changed ? Expr : SE.getMulExpr(Operands); + const SCEV *visitSMaxExpr(const SCEVSMaxExpr *Expr) { + SmallVector<const SCEV *, 2> Operands; + bool Changed = false; + for (auto *Op : Expr->operands()) { + Operands.push_back(((SC *)this)->visit(Op)); + Changed |= Op != Operands.back(); } + return !Changed ? Expr : SE.getSMaxExpr(Operands); + } - const SCEV *visitUDivExpr(const SCEVUDivExpr *Expr) { - auto *LHS = ((SC *)this)->visit(Expr->getLHS()); - auto *RHS = ((SC *)this)->visit(Expr->getRHS()); - bool Changed = LHS != Expr->getLHS() || RHS != Expr->getRHS(); - return !Changed ? Expr : SE.getUDivExpr(LHS, RHS); + const SCEV *visitUMaxExpr(const SCEVUMaxExpr *Expr) { + SmallVector<const SCEV *, 2> Operands; + bool Changed = false; + for (auto *Op : Expr->operands()) { + Operands.push_back(((SC *)this)->visit(Op)); + Changed |= Op != Operands.back(); } + return !Changed ? Expr : SE.getUMaxExpr(Operands); + } - const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) { - SmallVector<const SCEV *, 2> Operands; - bool Changed = false; - for (auto *Op : Expr->operands()) { - Operands.push_back(((SC*)this)->visit(Op)); - Changed |= Op != Operands.back(); - } - return !Changed ? Expr - : SE.getAddRecExpr(Operands, Expr->getLoop(), - Expr->getNoWrapFlags()); + const SCEV *visitSMinExpr(const SCEVSMinExpr *Expr) { + SmallVector<const SCEV *, 2> Operands; + bool Changed = false; + for (auto *Op : Expr->operands()) { + Operands.push_back(((SC *)this)->visit(Op)); + Changed |= Op != Operands.back(); } + return !Changed ? Expr : SE.getSMinExpr(Operands); + } - const SCEV *visitSMaxExpr(const SCEVSMaxExpr *Expr) { - SmallVector<const SCEV *, 2> Operands; - bool Changed = false; - for (auto *Op : Expr->operands()) { - Operands.push_back(((SC *)this)->visit(Op)); - Changed |= Op != Operands.back(); - } - return !Changed ? Expr : SE.getSMaxExpr(Operands); + const SCEV *visitUMinExpr(const SCEVUMinExpr *Expr) { + SmallVector<const SCEV *, 2> Operands; + bool Changed = false; + for (auto *Op : Expr->operands()) { + Operands.push_back(((SC *)this)->visit(Op)); + Changed |= Op != Operands.back(); } + return !Changed ? Expr : SE.getUMinExpr(Operands); + } - const SCEV *visitUMaxExpr(const SCEVUMaxExpr *Expr) { - SmallVector<const SCEV *, 2> Operands; - bool Changed = false; - for (auto *Op : Expr->operands()) { - Operands.push_back(((SC*)this)->visit(Op)); - Changed |= Op != Operands.back(); - } - return !Changed ? Expr : SE.getUMaxExpr(Operands); + const SCEV *visitSequentialUMinExpr(const SCEVSequentialUMinExpr *Expr) { + SmallVector<const SCEV *, 2> Operands; + bool Changed = false; + for (auto *Op : Expr->operands()) { + Operands.push_back(((SC *)this)->visit(Op)); + Changed |= Op != Operands.back(); } + return !Changed ? Expr : SE.getUMinExpr(Operands, /*Sequential=*/true); + } - const SCEV *visitSMinExpr(const SCEVSMinExpr *Expr) { - SmallVector<const SCEV *, 2> Operands; - bool Changed = false; - for (auto *Op : Expr->operands()) { - Operands.push_back(((SC *)this)->visit(Op)); - Changed |= Op != Operands.back(); - } - return !Changed ? Expr : SE.getSMinExpr(Operands); - } + const SCEV *visitUnknown(const SCEVUnknown *Expr) { return Expr; } - const SCEV *visitUMinExpr(const SCEVUMinExpr *Expr) { - SmallVector<const SCEV *, 2> Operands; - bool Changed = false; - for (auto *Op : Expr->operands()) { - Operands.push_back(((SC *)this)->visit(Op)); - Changed |= Op != Operands.back(); - } - return !Changed ? Expr : SE.getUMinExpr(Operands); - } + const SCEV *visitCouldNotCompute(const SCEVCouldNotCompute *Expr) { + return Expr; + } +}; + +using ValueToValueMap = DenseMap<const Value *, Value *>; +using ValueToSCEVMapTy = DenseMap<const Value *, const SCEV *>; + +/// The SCEVParameterRewriter takes a scalar evolution expression and updates +/// the SCEVUnknown components following the Map (Value -> SCEV). +class SCEVParameterRewriter : public SCEVRewriteVisitor<SCEVParameterRewriter> { +public: + static const SCEV *rewrite(const SCEV *Scev, ScalarEvolution &SE, + ValueToSCEVMapTy &Map) { + SCEVParameterRewriter Rewriter(SE, Map); + return Rewriter.visit(Scev); + } - const SCEV *visitUnknown(const SCEVUnknown *Expr) { - return Expr; - } + SCEVParameterRewriter(ScalarEvolution &SE, ValueToSCEVMapTy &M) + : SCEVRewriteVisitor(SE), Map(M) {} - const SCEV *visitCouldNotCompute(const SCEVCouldNotCompute *Expr) { + const SCEV *visitUnknown(const SCEVUnknown *Expr) { + auto I = Map.find(Expr->getValue()); + if (I == Map.end()) return Expr; - } - }; - - using ValueToValueMap = DenseMap<const Value *, Value *>; - using ValueToSCEVMapTy = DenseMap<const Value *, const SCEV *>; - - /// The SCEVParameterRewriter takes a scalar evolution expression and updates - /// the SCEVUnknown components following the Map (Value -> SCEV). - class SCEVParameterRewriter : public SCEVRewriteVisitor<SCEVParameterRewriter> { - public: - static const SCEV *rewrite(const SCEV *Scev, ScalarEvolution &SE, - ValueToSCEVMapTy &Map) { - SCEVParameterRewriter Rewriter(SE, Map); - return Rewriter.visit(Scev); - } - - SCEVParameterRewriter(ScalarEvolution &SE, ValueToSCEVMapTy &M) - : SCEVRewriteVisitor(SE), Map(M) {} - - const SCEV *visitUnknown(const SCEVUnknown *Expr) { - auto I = Map.find(Expr->getValue()); - if (I == Map.end()) - return Expr; - return I->second; - } + return I->second; + } - private: - ValueToSCEVMapTy ⤅ - }; +private: + ValueToSCEVMapTy ⤅ +}; - using LoopToScevMapT = DenseMap<const Loop *, const SCEV *>; +using LoopToScevMapT = DenseMap<const Loop *, const SCEV *>; - /// The SCEVLoopAddRecRewriter takes a scalar evolution expression and applies - /// the Map (Loop -> SCEV) to all AddRecExprs. - class SCEVLoopAddRecRewriter - : public SCEVRewriteVisitor<SCEVLoopAddRecRewriter> { - public: - SCEVLoopAddRecRewriter(ScalarEvolution &SE, LoopToScevMapT &M) - : SCEVRewriteVisitor(SE), Map(M) {} +/// The SCEVLoopAddRecRewriter takes a scalar evolution expression and applies +/// the Map (Loop -> SCEV) to all AddRecExprs. +class SCEVLoopAddRecRewriter + : public SCEVRewriteVisitor<SCEVLoopAddRecRewriter> { +public: + SCEVLoopAddRecRewriter(ScalarEvolution &SE, LoopToScevMapT &M) + : SCEVRewriteVisitor(SE), Map(M) {} - static const SCEV *rewrite(const SCEV *Scev, LoopToScevMapT &Map, - ScalarEvolution &SE) { - SCEVLoopAddRecRewriter Rewriter(SE, Map); - return Rewriter.visit(Scev); - } + static const SCEV *rewrite(const SCEV *Scev, LoopToScevMapT &Map, + ScalarEvolution &SE) { + SCEVLoopAddRecRewriter Rewriter(SE, Map); + return Rewriter.visit(Scev); + } - const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) { - SmallVector<const SCEV *, 2> Operands; - for (const SCEV *Op : Expr->operands()) - Operands.push_back(visit(Op)); + const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) { + SmallVector<const SCEV *, 2> Operands; + for (const SCEV *Op : Expr->operands()) + Operands.push_back(visit(Op)); - const Loop *L = Expr->getLoop(); - if (0 == Map.count(L)) - return SE.getAddRecExpr(Operands, L, Expr->getNoWrapFlags()); + const Loop *L = Expr->getLoop(); + if (0 == Map.count(L)) + return SE.getAddRecExpr(Operands, L, Expr->getNoWrapFlags()); - return SCEVAddRecExpr::evaluateAtIteration(Operands, Map[L], SE); - } + return SCEVAddRecExpr::evaluateAtIteration(Operands, Map[L], SE); + } - private: - LoopToScevMapT ⤅ - }; +private: + LoopToScevMapT ⤅ +}; } // end namespace llvm diff --git a/llvm/include/llvm/Analysis/SyncDependenceAnalysis.h b/llvm/include/llvm/Analysis/SyncDependenceAnalysis.h index 92459ea79ab4..cfc1e20255d1 100644 --- a/llvm/include/llvm/Analysis/SyncDependenceAnalysis.h +++ b/llvm/include/llvm/Analysis/SyncDependenceAnalysis.h @@ -20,6 +20,7 @@ #include "llvm/ADT/PostOrderIterator.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/Analysis/LoopInfo.h" +#include <map> #include <memory> #include <unordered_map> @@ -27,7 +28,6 @@ namespace llvm { class BasicBlock; class DominatorTree; -class Loop; class PostDominatorTree; using ConstBlockSet = SmallPtrSet<const BasicBlock *, 4>; diff --git a/llvm/include/llvm/Analysis/TargetFolder.h b/llvm/include/llvm/Analysis/TargetFolder.h index b23316ac3d9b..1df0530e40e6 100644 --- a/llvm/include/llvm/Analysis/TargetFolder.h +++ b/llvm/include/llvm/Analysis/TargetFolder.h @@ -43,13 +43,72 @@ public: explicit TargetFolder(const DataLayout &DL) : DL(DL) {} //===--------------------------------------------------------------------===// - // Binary Operators + // Value-based folders. + // + // Return an existing value or a constant if the operation can be simplified. + // Otherwise return nullptr. //===--------------------------------------------------------------------===// + Value *FoldAdd(Value *LHS, Value *RHS, bool HasNUW = false, + bool HasNSW = false) const override { + auto *LC = dyn_cast<Constant>(LHS); + auto *RC = dyn_cast<Constant>(RHS); + if (LC && RC) + return Fold(ConstantExpr::getAdd(LC, RC, HasNUW, HasNSW)); + return nullptr; + } - Constant *CreateAdd(Constant *LHS, Constant *RHS, - bool HasNUW = false, bool HasNSW = false) const override { - return Fold(ConstantExpr::getAdd(LHS, RHS, HasNUW, HasNSW)); + Value *FoldAnd(Value *LHS, Value *RHS) const override { + auto *LC = dyn_cast<Constant>(LHS); + auto *RC = dyn_cast<Constant>(RHS); + if (LC && RC) + return Fold(ConstantExpr::getAnd(LC, RC)); + return nullptr; + } + + Value *FoldOr(Value *LHS, Value *RHS) const override { + auto *LC = dyn_cast<Constant>(LHS); + auto *RC = dyn_cast<Constant>(RHS); + if (LC && RC) + return Fold(ConstantExpr::getOr(LC, RC)); + return nullptr; + } + + Value *FoldICmp(CmpInst::Predicate P, Value *LHS, Value *RHS) const override { + auto *LC = dyn_cast<Constant>(LHS); + auto *RC = dyn_cast<Constant>(RHS); + if (LC && RC) + return ConstantExpr::getCompare(P, LC, RC); + return nullptr; + } + + Value *FoldGEP(Type *Ty, Value *Ptr, ArrayRef<Value *> IdxList, + bool IsInBounds = false) const override { + if (auto *PC = dyn_cast<Constant>(Ptr)) { + // Every index must be constant. + if (any_of(IdxList, [](Value *V) { return !isa<Constant>(V); })) + return nullptr; + if (IsInBounds) + return Fold(ConstantExpr::getInBoundsGetElementPtr(Ty, PC, IdxList)); + else + return Fold(ConstantExpr::getGetElementPtr(Ty, PC, IdxList)); + } + return nullptr; } + + Value *FoldSelect(Value *C, Value *True, Value *False) const override { + auto *CC = dyn_cast<Constant>(C); + auto *TC = dyn_cast<Constant>(True); + auto *FC = dyn_cast<Constant>(False); + if (CC && TC && FC) + return Fold(ConstantExpr::getSelect(CC, TC, FC)); + + return nullptr; + } + + //===--------------------------------------------------------------------===// + // Binary Operators + //===--------------------------------------------------------------------===// + Constant *CreateFAdd(Constant *LHS, Constant *RHS) const override { return Fold(ConstantExpr::getFAdd(LHS, RHS)); } @@ -99,12 +158,6 @@ public: bool isExact = false) const override { return Fold(ConstantExpr::getAShr(LHS, RHS, isExact)); } - Constant *CreateAnd(Constant *LHS, Constant *RHS) const override { - return Fold(ConstantExpr::getAnd(LHS, RHS)); - } - Constant *CreateOr(Constant *LHS, Constant *RHS) const override { - return Fold(ConstantExpr::getOr(LHS, RHS)); - } Constant *CreateXor(Constant *LHS, Constant *RHS) const override { return Fold(ConstantExpr::getXor(LHS, RHS)); } @@ -134,42 +187,6 @@ public: } //===--------------------------------------------------------------------===// - // Memory Instructions - //===--------------------------------------------------------------------===// - - Constant *CreateGetElementPtr(Type *Ty, Constant *C, - ArrayRef<Constant *> IdxList) const override { - return Fold(ConstantExpr::getGetElementPtr(Ty, C, IdxList)); - } - Constant *CreateGetElementPtr(Type *Ty, Constant *C, - Constant *Idx) const override { - // This form of the function only exists to avoid ambiguous overload - // warnings about whether to convert Idx to ArrayRef<Constant *> or - // ArrayRef<Value *>. - return Fold(ConstantExpr::getGetElementPtr(Ty, C, Idx)); - } - Constant *CreateGetElementPtr(Type *Ty, Constant *C, - ArrayRef<Value *> IdxList) const override { - return Fold(ConstantExpr::getGetElementPtr(Ty, C, IdxList)); - } - - Constant *CreateInBoundsGetElementPtr( - Type *Ty, Constant *C, ArrayRef<Constant *> IdxList) const override { - return Fold(ConstantExpr::getInBoundsGetElementPtr(Ty, C, IdxList)); - } - Constant *CreateInBoundsGetElementPtr(Type *Ty, Constant *C, - Constant *Idx) const override { - // This form of the function only exists to avoid ambiguous overload - // warnings about whether to convert Idx to ArrayRef<Constant *> or - // ArrayRef<Value *>. - return Fold(ConstantExpr::getInBoundsGetElementPtr(Ty, C, Idx)); - } - Constant *CreateInBoundsGetElementPtr( - Type *Ty, Constant *C, ArrayRef<Value *> IdxList) const override { - return Fold(ConstantExpr::getInBoundsGetElementPtr(Ty, C, IdxList)); - } - - //===--------------------------------------------------------------------===// // Cast/Conversion Operators //===--------------------------------------------------------------------===// @@ -231,10 +248,6 @@ public: // Compare Instructions //===--------------------------------------------------------------------===// - Constant *CreateICmp(CmpInst::Predicate P, Constant *LHS, - Constant *RHS) const override { - return Fold(ConstantExpr::getCompare(P, LHS, RHS)); - } Constant *CreateFCmp(CmpInst::Predicate P, Constant *LHS, Constant *RHS) const override { return Fold(ConstantExpr::getCompare(P, LHS, RHS)); @@ -244,11 +257,6 @@ public: // Other Instructions //===--------------------------------------------------------------------===// - Constant *CreateSelect(Constant *C, Constant *True, - Constant *False) const override { - return Fold(ConstantExpr::getSelect(C, True, False)); - } - Constant *CreateExtractElement(Constant *Vec, Constant *Idx) const override { return Fold(ConstantExpr::getExtractElement(Vec, Idx)); } diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h index d9f5c9689d5c..34ef9cc61c4f 100644 --- a/llvm/include/llvm/Analysis/TargetTransformInfo.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h @@ -43,7 +43,6 @@ class BlockFrequencyInfo; class DominatorTree; class BranchInst; class CallBase; -class ExtractElementInst; class Function; class GlobalValue; class InstCombiner; @@ -664,6 +663,12 @@ public: bool isLegalMaskedScatter(Type *DataType, Align Alignment) const; /// Return true if the target supports masked gather. bool isLegalMaskedGather(Type *DataType, Align Alignment) const; + /// Return true if the target forces scalarizing of llvm.masked.gather + /// intrinsics. + bool forceScalarizeMaskedGather(VectorType *Type, Align Alignment) const; + /// Return true if the target forces scalarizing of llvm.masked.scatter + /// intrinsics. + bool forceScalarizeMaskedScatter(VectorType *Type, Align Alignment) const; /// Return true if the target supports masked compress store. bool isLegalMaskedCompressStore(Type *DataType) const; @@ -1359,10 +1364,12 @@ public: /// Flags describing the kind of vector reduction. struct ReductionFlags { - ReductionFlags() : IsMaxOp(false), IsSigned(false), NoNaN(false) {} - bool IsMaxOp; ///< If the op a min/max kind, true if it's a max operation. - bool IsSigned; ///< Whether the operation is a signed int reduction. - bool NoNaN; ///< If op is an fp min/max, whether NaNs may be present. + ReductionFlags() = default; + bool IsMaxOp = + false; ///< If the op a min/max kind, true if it's a max operation. + bool IsSigned = false; ///< Whether the operation is a signed int reduction. + bool NoNaN = + false; ///< If op is an fp min/max, whether NaNs may be present. }; /// \returns True if the target prefers reductions in loop. @@ -1545,6 +1552,10 @@ public: virtual bool isLegalNTLoad(Type *DataType, Align Alignment) = 0; virtual bool isLegalMaskedScatter(Type *DataType, Align Alignment) = 0; virtual bool isLegalMaskedGather(Type *DataType, Align Alignment) = 0; + virtual bool forceScalarizeMaskedGather(VectorType *DataType, + Align Alignment) = 0; + virtual bool forceScalarizeMaskedScatter(VectorType *DataType, + Align Alignment) = 0; virtual bool isLegalMaskedCompressStore(Type *DataType) = 0; virtual bool isLegalMaskedExpandLoad(Type *DataType) = 0; virtual bool enableOrderedReductions() = 0; @@ -1948,6 +1959,14 @@ public: bool isLegalMaskedGather(Type *DataType, Align Alignment) override { return Impl.isLegalMaskedGather(DataType, Alignment); } + bool forceScalarizeMaskedGather(VectorType *DataType, + Align Alignment) override { + return Impl.forceScalarizeMaskedGather(DataType, Alignment); + } + bool forceScalarizeMaskedScatter(VectorType *DataType, + Align Alignment) override { + return Impl.forceScalarizeMaskedScatter(DataType, Alignment); + } bool isLegalMaskedCompressStore(Type *DataType) override { return Impl.isLegalMaskedCompressStore(DataType); } diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h index 26a696f09b3d..4b9ef7c57ffc 100644 --- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h @@ -267,6 +267,15 @@ public: return false; } + bool forceScalarizeMaskedGather(VectorType *DataType, Align Alignment) const { + return false; + } + + bool forceScalarizeMaskedScatter(VectorType *DataType, + Align Alignment) const { + return false; + } + bool isLegalMaskedCompressStore(Type *DataType) const { return false; } bool isLegalMaskedExpandLoad(Type *DataType) const { return false; } @@ -624,6 +633,7 @@ public: case Intrinsic::coro_end: case Intrinsic::coro_frame: case Intrinsic::coro_size: + case Intrinsic::coro_align: case Intrinsic::coro_suspend: case Intrinsic::coro_subfn_addr: // These intrinsics don't actually represent code after lowering. @@ -1052,7 +1062,20 @@ public: } case Instruction::Load: { auto *LI = cast<LoadInst>(U); - return TargetTTI->getMemoryOpCost(Opcode, U->getType(), LI->getAlign(), + Type *LoadType = U->getType(); + // If there is a non-register sized type, the cost estimation may expand + // it to be several instructions to load into multiple registers on the + // target. But, if the only use of the load is a trunc instruction to a + // register sized type, the instruction selector can combine these + // instructions to be a single load. So, in this case, we use the + // destination type of the trunc instruction rather than the load to + // accurately estimate the cost of this load instruction. + if (CostKind == TTI::TCK_CodeSize && LI->hasOneUse() && + !LoadType->isVectorTy()) { + if (const TruncInst *TI = dyn_cast<TruncInst>(*LI->user_begin())) + LoadType = TI->getDestTy(); + } + return TargetTTI->getMemoryOpCost(Opcode, LoadType, LI->getAlign(), LI->getPointerAddressSpace(), CostKind, I); } diff --git a/llvm/include/llvm/Analysis/Utils/TFUtils.h b/llvm/include/llvm/Analysis/Utils/TFUtils.h index 012fca53a200..785b9fe949a5 100644 --- a/llvm/include/llvm/Analysis/Utils/TFUtils.h +++ b/llvm/include/llvm/Analysis/Utils/TFUtils.h @@ -178,8 +178,14 @@ public: // Flush the content of the log to the stream, clearing the stored data in the // process. + void flush(std::string *Str); void flush(raw_ostream &OS); + // Flush a set of logs that are produced from the same module, e.g. + // per-function regalloc traces, as a google::protobuf::Struct message. + static void flushLogs(raw_ostream &OS, + const StringMap<std::unique_ptr<Logger>> &Loggers); + private: std::vector<LoggedFeatureSpec> FeatureSpecs; TensorSpec RewardSpec; diff --git a/llvm/include/llvm/Analysis/ValueTracking.h b/llvm/include/llvm/Analysis/ValueTracking.h index b4f38a3e976f..5b39b0244339 100644 --- a/llvm/include/llvm/Analysis/ValueTracking.h +++ b/llvm/include/llvm/Analysis/ValueTracking.h @@ -33,7 +33,6 @@ class APInt; class AssumptionCache; class DominatorTree; class GEPOperator; -class IntrinsicInst; class LoadInst; class WithOverflowInst; struct KnownBits; @@ -203,23 +202,14 @@ constexpr unsigned MaxAnalysisRecursionDepth = 6; const DominatorTree *DT = nullptr, bool UseInstrInfo = true); - /// Get the minimum bit size for this Value \p Op as a signed integer. - /// i.e. x == sext(trunc(x to MinSignedBits) to bitwidth(x)). - /// Similar to the APInt::getMinSignedBits function. - unsigned ComputeMinSignedBits(const Value *Op, const DataLayout &DL, - unsigned Depth = 0, - AssumptionCache *AC = nullptr, - const Instruction *CxtI = nullptr, - const DominatorTree *DT = nullptr); - - /// This function computes the integer multiple of Base that equals V. If - /// successful, it returns true and returns the multiple in Multiple. If - /// unsuccessful, it returns false. Also, if V can be simplified to an - /// integer, then the simplified V is returned in Val. Look through sext only - /// if LookThroughSExt=true. - bool ComputeMultiple(Value *V, unsigned Base, Value *&Multiple, - bool LookThroughSExt = false, - unsigned Depth = 0); + /// Get the upper bound on bit size for this Value \p Op as a signed integer. + /// i.e. x == sext(trunc(x to MaxSignificantBits) to bitwidth(x)). + /// Similar to the APInt::getSignificantBits function. + unsigned ComputeMaxSignificantBits(const Value *Op, const DataLayout &DL, + unsigned Depth = 0, + AssumptionCache *AC = nullptr, + const Instruction *CxtI = nullptr, + const DominatorTree *DT = nullptr); /// Map a call instruction to an intrinsic ID. Libcalls which have equivalent /// intrinsics are treated as-if they were intrinsics. @@ -555,7 +545,8 @@ constexpr unsigned MaxAnalysisRecursionDepth = 6; /// Determine the possible constant range of an integer or vector of integer /// value. This is intended as a cheap, non-recursive check. - ConstantRange computeConstantRange(const Value *V, bool UseInstrInfo = true, + ConstantRange computeConstantRange(const Value *V, bool ForSigned, + bool UseInstrInfo = true, AssumptionCache *AC = nullptr, const Instruction *CtxI = nullptr, const DominatorTree *DT = nullptr, diff --git a/llvm/include/llvm/BinaryFormat/AMDGPUMetadataVerifier.h b/llvm/include/llvm/BinaryFormat/AMDGPUMetadataVerifier.h index 7332b2a7ea89..c94cafe7458b 100644 --- a/llvm/include/llvm/BinaryFormat/AMDGPUMetadataVerifier.h +++ b/llvm/include/llvm/BinaryFormat/AMDGPUMetadataVerifier.h @@ -16,9 +16,21 @@ #ifndef LLVM_BINARYFORMAT_AMDGPUMETADATAVERIFIER_H #define LLVM_BINARYFORMAT_AMDGPUMETADATAVERIFIER_H -#include "llvm/BinaryFormat/MsgPackDocument.h" +#include "llvm/ADT/None.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/STLFunctionalExtras.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/BinaryFormat/MsgPackReader.h" + +#include <cstddef> namespace llvm { + +namespace msgpack { + class DocNode; + class MapDocNode; +} + namespace AMDGPU { namespace HSAMD { namespace V3 { diff --git a/llvm/include/llvm/BinaryFormat/Dwarf.def b/llvm/include/llvm/BinaryFormat/Dwarf.def index 61f3f27ebb47..76e51b3157d3 100644 --- a/llvm/include/llvm/BinaryFormat/Dwarf.def +++ b/llvm/include/llvm/BinaryFormat/Dwarf.def @@ -495,7 +495,8 @@ HANDLE_DW_AT(0x2133, GNU_addr_base, 0, GNU) HANDLE_DW_AT(0x2134, GNU_pubnames, 0, GNU) HANDLE_DW_AT(0x2135, GNU_pubtypes, 0, GNU) HANDLE_DW_AT(0x2136, GNU_discriminator, 0, GNU) -HANDLE_DW_AT(0x2137, GNU_entry_view, 0, GNU) +HANDLE_DW_AT(0x2137, GNU_locviews, 0, GNU) +HANDLE_DW_AT(0x2138, GNU_entry_view, 0, GNU) HANDLE_DW_AT(0x2201, SUN_template, 0, SUN) // Conflicting: diff --git a/llvm/include/llvm/BinaryFormat/Dwarf.h b/llvm/include/llvm/BinaryFormat/Dwarf.h index a725aff39ac6..4473f506d371 100644 --- a/llvm/include/llvm/BinaryFormat/Dwarf.h +++ b/llvm/include/llvm/BinaryFormat/Dwarf.h @@ -19,7 +19,6 @@ #ifndef LLVM_BINARYFORMAT_DWARF_H #define LLVM_BINARYFORMAT_DWARF_H -#include "llvm/ADT/Optional.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/DataTypes.h" #include "llvm/Support/ErrorHandling.h" @@ -31,9 +30,11 @@ namespace llvm { class StringRef; +template<typename T> class Optional; namespace dwarf { + //===----------------------------------------------------------------------===// // DWARF constants as gleaned from the DWARF Debugging Information Format V.5 // reference manual http://www.dwarfstd.org/. @@ -649,6 +650,9 @@ struct FormParams { uint16_t Version; uint8_t AddrSize; DwarfFormat Format; + /// True if DWARF v2 output generally uses relocations for references + /// to other .debug_* sections. + bool DwarfUsesRelocationsAcrossSections = false; /// The definition of the size of form DW_FORM_ref_addr depends on the /// version. In DWARF v2 it's the size of an address; after that, it's the diff --git a/llvm/include/llvm/BinaryFormat/ELF.h b/llvm/include/llvm/BinaryFormat/ELF.h index 065661cbd188..8840929174d6 100644 --- a/llvm/include/llvm/BinaryFormat/ELF.h +++ b/llvm/include/llvm/BinaryFormat/ELF.h @@ -22,7 +22,6 @@ #include "llvm/ADT/StringRef.h" #include <cstdint> #include <cstring> -#include <string> namespace llvm { namespace ELF { @@ -656,7 +655,8 @@ enum : unsigned { EF_RISCV_FLOAT_ABI_SINGLE = 0x0002, EF_RISCV_FLOAT_ABI_DOUBLE = 0x0004, EF_RISCV_FLOAT_ABI_QUAD = 0x0006, - EF_RISCV_RVE = 0x0008 + EF_RISCV_RVE = 0x0008, + EF_RISCV_TSO = 0x0010, }; // ELF Relocation types for RISC-V diff --git a/llvm/include/llvm/BinaryFormat/MachO.h b/llvm/include/llvm/BinaryFormat/MachO.h index df2ba94c7896..ce3a5c46e0d1 100644 --- a/llvm/include/llvm/BinaryFormat/MachO.h +++ b/llvm/include/llvm/BinaryFormat/MachO.h @@ -489,6 +489,7 @@ enum { VM_PROT_READ = 0x1, VM_PROT_WRITE = 0x2, VM_PROT_EXECUTE = 0x4 }; // Values for platform field in build_version_command. enum PlatformType { + PLATFORM_UNKNOWN = 0, PLATFORM_MACOS = 1, PLATFORM_IOS = 2, PLATFORM_TVOS = 3, diff --git a/llvm/include/llvm/BinaryFormat/MsgPackReader.h b/llvm/include/llvm/BinaryFormat/MsgPackReader.h index 5035de03cc1b..5123fb65cf09 100644 --- a/llvm/include/llvm/BinaryFormat/MsgPackReader.h +++ b/llvm/include/llvm/BinaryFormat/MsgPackReader.h @@ -34,8 +34,7 @@ #define LLVM_BINARYFORMAT_MSGPACKREADER_H #include "llvm/Support/Error.h" -#include "llvm/Support/MemoryBuffer.h" -#include "llvm/Support/raw_ostream.h" +#include "llvm/Support/MemoryBufferRef.h" #include <cstdint> namespace llvm { diff --git a/llvm/include/llvm/BinaryFormat/MsgPackWriter.h b/llvm/include/llvm/BinaryFormat/MsgPackWriter.h index 254b52f18bd3..b3d3c3bcdef0 100644 --- a/llvm/include/llvm/BinaryFormat/MsgPackWriter.h +++ b/llvm/include/llvm/BinaryFormat/MsgPackWriter.h @@ -28,12 +28,13 @@ #ifndef LLVM_BINARYFORMAT_MSGPACKWRITER_H #define LLVM_BINARYFORMAT_MSGPACKWRITER_H -#include "llvm/BinaryFormat/MsgPack.h" #include "llvm/Support/EndianStream.h" -#include "llvm/Support/MemoryBuffer.h" -#include "llvm/Support/raw_ostream.h" +#include "llvm/Support/MemoryBufferRef.h" namespace llvm { + +class raw_ostream; + namespace msgpack { /// Writes MessagePack objects to an output stream, one at a time. diff --git a/llvm/include/llvm/Bitcode/LLVMBitCodes.h b/llvm/include/llvm/Bitcode/LLVMBitCodes.h index 7301618d337a..6d0f51ce9c6d 100644 --- a/llvm/include/llvm/Bitcode/LLVMBitCodes.h +++ b/llvm/include/llvm/Bitcode/LLVMBitCodes.h @@ -381,10 +381,14 @@ enum ConstantsCodes { CST_CODE_CE_UNOP = 25, // CE_UNOP: [opcode, opval] CST_CODE_POISON = 26, // POISON CST_CODE_DSO_LOCAL_EQUIVALENT = 27, // DSO_LOCAL_EQUIVALENT [gvty, gv] - CST_CODE_INLINEASM = 28, // INLINEASM: [sideeffect|alignstack| + CST_CODE_INLINEASM_OLD3 = 28, // INLINEASM: [sideeffect|alignstack| + // asmdialect|unwind, + // asmstr,conststr] + CST_CODE_NO_CFI_VALUE = 29, // NO_CFI [ fty, f ] + CST_CODE_INLINEASM = 30, // INLINEASM: [fnty, + // sideeffect|alignstack| // asmdialect|unwind, // asmstr,conststr] - CST_CODE_NO_CFI_VALUE = 29, // NO_CFI [ fty, f ] }; /// CastOpcodes - These are values used in the bitcode files to encode which diff --git a/llvm/include/llvm/CodeGen/AsmPrinter.h b/llvm/include/llvm/CodeGen/AsmPrinter.h index 281ecb8de251..d911bfd435ae 100644 --- a/llvm/include/llvm/CodeGen/AsmPrinter.h +++ b/llvm/include/llvm/CodeGen/AsmPrinter.h @@ -17,6 +17,7 @@ #include "llvm/ADT/MapVector.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/BinaryFormat/Dwarf.h" #include "llvm/CodeGen/AsmPrinterHandler.h" #include "llvm/CodeGen/DwarfStringPoolEntry.h" #include "llvm/CodeGen/MachineFunctionPass.h" @@ -231,6 +232,9 @@ public: /// Returns 4 for DWARF32 and 12 for DWARF64. unsigned int getUnitLengthFieldByteSize() const; + /// Returns information about the byte size of DW_FORM values. + dwarf::FormParams getDwarfFormParams() const; + bool isPositionIndependent() const; /// Return true if assembly output should contain comments. @@ -431,7 +435,8 @@ public: /// global value is specified, and if that global has an explicit alignment /// requested, it will override the alignment request if required for /// correctness. - void emitAlignment(Align Alignment, const GlobalObject *GV = nullptr) const; + void emitAlignment(Align Alignment, const GlobalObject *GV = nullptr, + unsigned MaxBytesToEmit = 0) const; /// Lower the specified LLVM Constant to an MCExpr. virtual const MCExpr *lowerConstant(const Constant *CV); diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h index 324b7dcfb3ac..0b2737628923 100644 --- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h +++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h @@ -875,14 +875,18 @@ public: switch (improveShuffleKindFromMask(Kind, Mask)) { case TTI::SK_Broadcast: - return getBroadcastShuffleOverhead(cast<FixedVectorType>(Tp)); + if (auto *FVT = dyn_cast<FixedVectorType>(Tp)) + return getBroadcastShuffleOverhead(FVT); + return InstructionCost::getInvalid(); case TTI::SK_Select: case TTI::SK_Splice: case TTI::SK_Reverse: case TTI::SK_Transpose: case TTI::SK_PermuteSingleSrc: case TTI::SK_PermuteTwoSrc: - return getPermuteShuffleOverhead(cast<FixedVectorType>(Tp)); + if (auto *FVT = dyn_cast<FixedVectorType>(Tp)) + return getPermuteShuffleOverhead(FVT); + return InstructionCost::getInvalid(); case TTI::SK_ExtractSubvector: return getExtractSubvectorOverhead(Tp, Index, cast<FixedVectorType>(SubTp)); diff --git a/llvm/include/llvm/CodeGen/CalcSpillWeights.h b/llvm/include/llvm/CodeGen/CalcSpillWeights.h index 0b6ed079b38e..bfd5bab3d1c0 100644 --- a/llvm/include/llvm/CodeGen/CalcSpillWeights.h +++ b/llvm/include/llvm/CodeGen/CalcSpillWeights.h @@ -80,6 +80,18 @@ class VirtRegMap; /// live intervals. void calculateSpillWeightsAndHints(); + /// Return the preferred allocation register for reg, given a COPY + /// instruction. + static Register copyHint(const MachineInstr *MI, unsigned Reg, + const TargetRegisterInfo &TRI, + const MachineRegisterInfo &MRI); + + /// Determine if all values in LI are rematerializable. + static bool isRematerializable(const LiveInterval &LI, + const LiveIntervals &LIS, + const VirtRegMap &VRM, + const TargetInstrInfo &TII); + protected: /// Helper function for weight calculations. /// (Re)compute LI's spill weight and allocation hint, or, for non null diff --git a/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h b/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h index 1fd07ca2c8d4..f6563971f981 100644 --- a/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h +++ b/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h @@ -159,7 +159,7 @@ protected: class AddIRPass { public: AddIRPass(ModulePassManager &MPM, bool DebugPM, bool Check = true) - : MPM(MPM), FPM() { + : MPM(MPM) { if (Check) AddingFunctionPasses = false; } diff --git a/llvm/include/llvm/CodeGen/DIE.h b/llvm/include/llvm/CodeGen/DIE.h index 9e94c401bfae..32df448b91a1 100644 --- a/llvm/include/llvm/CodeGen/DIE.h +++ b/llvm/include/llvm/CodeGen/DIE.h @@ -191,7 +191,7 @@ public: void setValue(uint64_t Val) { Integer = Val; } void emitValue(const AsmPrinter *Asm, dwarf::Form Form) const; - unsigned SizeOf(const AsmPrinter *AP, dwarf::Form Form) const; + unsigned sizeOf(const dwarf::FormParams &FormParams, dwarf::Form Form) const; void print(raw_ostream &O) const; }; @@ -208,7 +208,7 @@ public: const MCExpr *getValue() const { return Expr; } void emitValue(const AsmPrinter *AP, dwarf::Form Form) const; - unsigned SizeOf(const AsmPrinter *AP, dwarf::Form Form) const; + unsigned sizeOf(const dwarf::FormParams &FormParams, dwarf::Form Form) const; void print(raw_ostream &O) const; }; @@ -225,7 +225,7 @@ public: const MCSymbol *getValue() const { return Label; } void emitValue(const AsmPrinter *AP, dwarf::Form Form) const; - unsigned SizeOf(const AsmPrinter *AP, dwarf::Form Form) const; + unsigned sizeOf(const dwarf::FormParams &FormParams, dwarf::Form Form) const; void print(raw_ostream &O) const; }; @@ -243,8 +243,8 @@ public: /// EmitValue - Emit base type reference. void emitValue(const AsmPrinter *AP, dwarf::Form Form) const; - /// SizeOf - Determine size of the base type reference in bytes. - unsigned SizeOf(const AsmPrinter *AP, dwarf::Form Form) const; + /// sizeOf - Determine size of the base type reference in bytes. + unsigned sizeOf(const dwarf::FormParams &, dwarf::Form) const; void print(raw_ostream &O) const; uint64_t getIndex() const { return Index; } @@ -261,7 +261,7 @@ public: DIEDelta(const MCSymbol *Hi, const MCSymbol *Lo) : LabelHi(Hi), LabelLo(Lo) {} void emitValue(const AsmPrinter *AP, dwarf::Form Form) const; - unsigned SizeOf(const AsmPrinter *AP, dwarf::Form Form) const; + unsigned sizeOf(const dwarf::FormParams &FormParams, dwarf::Form Form) const; void print(raw_ostream &O) const; }; @@ -280,7 +280,7 @@ public: StringRef getString() const { return S.getString(); } void emitValue(const AsmPrinter *AP, dwarf::Form Form) const; - unsigned SizeOf(const AsmPrinter *AP, dwarf::Form Form) const; + unsigned sizeOf(const dwarf::FormParams &FormParams, dwarf::Form Form) const; void print(raw_ostream &O) const; }; @@ -302,7 +302,7 @@ public: StringRef getString() const { return S; } void emitValue(const AsmPrinter *AP, dwarf::Form Form) const; - unsigned SizeOf(const AsmPrinter *AP, dwarf::Form Form) const; + unsigned sizeOf(const dwarf::FormParams &, dwarf::Form) const; void print(raw_ostream &O) const; }; @@ -321,7 +321,7 @@ public: DIE &getEntry() const { return *Entry; } void emitValue(const AsmPrinter *AP, dwarf::Form Form) const; - unsigned SizeOf(const AsmPrinter *AP, dwarf::Form Form) const; + unsigned sizeOf(const dwarf::FormParams &FormParams, dwarf::Form Form) const; void print(raw_ostream &O) const; }; @@ -340,7 +340,7 @@ public: size_t getValue() const { return Index; } void emitValue(const AsmPrinter *AP, dwarf::Form Form) const; - unsigned SizeOf(const AsmPrinter *AP, dwarf::Form Form) const; + unsigned sizeOf(const dwarf::FormParams &FormParams, dwarf::Form Form) const; void print(raw_ostream &O) const; }; @@ -356,7 +356,7 @@ public: : Addr(Idx), Offset(Hi, Lo) {} void emitValue(const AsmPrinter *AP, dwarf::Form Form) const; - unsigned SizeOf(const AsmPrinter *AP, dwarf::Form Form) const; + unsigned sizeOf(const dwarf::FormParams &FormParams, dwarf::Form Form) const; void print(raw_ostream &O) const; }; @@ -506,7 +506,7 @@ public: void emitValue(const AsmPrinter *AP) const; /// Return the size of a value in bytes. - unsigned SizeOf(const AsmPrinter *AP) const; + unsigned sizeOf(const dwarf::FormParams &FormParams) const; void print(raw_ostream &O) const; void dump() const; @@ -774,8 +774,16 @@ public: unsigned getAbbrevNumber() const { return AbbrevNumber; } dwarf::Tag getTag() const { return Tag; } /// Get the compile/type unit relative offset of this DIE. - unsigned getOffset() const { return Offset; } - unsigned getSize() const { return Size; } + unsigned getOffset() const { + // A real Offset can't be zero because the unit headers are at offset zero. + assert(Offset && "Offset being queried before it's been computed."); + return Offset; + } + unsigned getSize() const { + // A real Size can't be zero because it includes the non-empty abbrev code. + assert(Size && "Size being queried before it's been ocmputed."); + return Size; + } bool hasChildren() const { return ForceChildren || !Children.empty(); } void setForceChildren(bool B) { ForceChildren = B; } @@ -817,12 +825,12 @@ public: /// properly refer to other DIE objects since all DIEs have calculated their /// offsets. /// - /// \param AP AsmPrinter to use when calculating sizes. + /// \param FormParams Used when calculating sizes. /// \param AbbrevSet the abbreviation used to unique DIE abbreviations. /// \param CUOffset the compile/type unit relative offset in bytes. /// \returns the offset for the DIE that follows this DIE within the /// current compile/type unit. - unsigned computeOffsetsAndAbbrevs(const AsmPrinter *AP, + unsigned computeOffsetsAndAbbrevs(const dwarf::FormParams &FormParams, DIEAbbrevSet &AbbrevSet, unsigned CUOffset); /// Climb up the parent chain to get the compile unit or type unit DIE that @@ -925,9 +933,8 @@ class DIELoc : public DIEValueList { public: DIELoc() = default; - /// ComputeSize - Calculate the size of the location expression. - /// - unsigned ComputeSize(const AsmPrinter *AP) const; + /// Calculate the size of the location expression. + unsigned computeSize(const dwarf::FormParams &FormParams) const; // TODO: move setSize() and Size to DIEValueList. void setSize(unsigned size) { Size = size; } @@ -948,7 +955,7 @@ public: } void emitValue(const AsmPrinter *Asm, dwarf::Form Form) const; - unsigned SizeOf(const AsmPrinter *AP, dwarf::Form Form) const; + unsigned sizeOf(const dwarf::FormParams &, dwarf::Form Form) const; void print(raw_ostream &O) const; }; @@ -962,9 +969,8 @@ class DIEBlock : public DIEValueList { public: DIEBlock() = default; - /// ComputeSize - Calculate the size of the location expression. - /// - unsigned ComputeSize(const AsmPrinter *AP) const; + /// Calculate the size of the location expression. + unsigned computeSize(const dwarf::FormParams &FormParams) const; // TODO: move setSize() and Size to DIEValueList. void setSize(unsigned size) { Size = size; } @@ -982,7 +988,7 @@ public: } void emitValue(const AsmPrinter *Asm, dwarf::Form Form) const; - unsigned SizeOf(const AsmPrinter *AP, dwarf::Form Form) const; + unsigned sizeOf(const dwarf::FormParams &, dwarf::Form Form) const; void print(raw_ostream &O) const; }; diff --git a/llvm/include/llvm/CodeGen/DbgEntityHistoryCalculator.h b/llvm/include/llvm/CodeGen/DbgEntityHistoryCalculator.h index e7425dd3dc04..2ac9d938d281 100644 --- a/llvm/include/llvm/CodeGen/DbgEntityHistoryCalculator.h +++ b/llvm/include/llvm/CodeGen/DbgEntityHistoryCalculator.h @@ -17,7 +17,6 @@ namespace llvm { -class DILocalVariable; class DILocation; class DINode; class MachineFunction; diff --git a/llvm/include/llvm/CodeGen/FaultMaps.h b/llvm/include/llvm/CodeGen/FaultMaps.h index 12d2872c8c5b..8a8b1d2e6008 100644 --- a/llvm/include/llvm/CodeGen/FaultMaps.h +++ b/llvm/include/llvm/CodeGen/FaultMaps.h @@ -18,7 +18,6 @@ namespace llvm { class AsmPrinter; class MCExpr; -class raw_ostream; class FaultMaps { public: diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h b/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h index 9c878d4b087b..3a4b3ee18e1b 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h @@ -95,7 +95,7 @@ public: bool IsFixed = true) : ArgInfo(Regs, OrigValue.getType(), OrigIndex, Flags, IsFixed, &OrigValue) {} - ArgInfo() : BaseArgInfo() {} + ArgInfo() {} }; struct CallLoweringInfo { @@ -388,12 +388,12 @@ protected: /// \p Handler to move them to the assigned locations. /// /// \return True if everything has succeeded, false otherwise. - bool determineAndHandleAssignments(ValueHandler &Handler, - ValueAssigner &Assigner, - SmallVectorImpl<ArgInfo> &Args, - MachineIRBuilder &MIRBuilder, - CallingConv::ID CallConv, bool IsVarArg, - Register ThisReturnReg = Register()) const; + bool + determineAndHandleAssignments(ValueHandler &Handler, ValueAssigner &Assigner, + SmallVectorImpl<ArgInfo> &Args, + MachineIRBuilder &MIRBuilder, + CallingConv::ID CallConv, bool IsVarArg, + ArrayRef<Register> ThisReturnRegs = None) const; /// Use \p Handler to insert code to handle the argument/return values /// represented by \p Args. It's expected determineAssignments previously @@ -402,7 +402,7 @@ protected: CCState &CCState, SmallVectorImpl<CCValAssign> &ArgLocs, MachineIRBuilder &MIRBuilder, - Register ThisReturnReg = Register()) const; + ArrayRef<Register> ThisReturnRegs = None) const; /// Check whether parameters to a call that are passed in callee saved /// registers are the same as from the calling function. This needs to be diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h index f3fa652b0175..45c27c25aea0 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h @@ -323,6 +323,11 @@ public: void applyCombineUnmergeConstant(MachineInstr &MI, SmallVectorImpl<APInt> &Csts); + /// Transform G_UNMERGE G_IMPLICIT_DEF -> G_IMPLICIT_DEF, G_IMPLICIT_DEF, ... + bool + matchCombineUnmergeUndef(MachineInstr &MI, + std::function<void(MachineIRBuilder &)> &MatchInfo); + /// Transform X, Y<dead> = G_UNMERGE Z -> X = G_TRUNC Z. bool matchCombineUnmergeWithDeadLanesToTrunc(MachineInstr &MI); void applyCombineUnmergeWithDeadLanesToTrunc(MachineInstr &MI); @@ -353,8 +358,8 @@ public: std::pair<Register, bool> &PtrRegAndCommute); // Transform G_PTR_ADD (G_PTRTOINT C1), C2 -> C1 + C2 - bool matchCombineConstPtrAddToI2P(MachineInstr &MI, int64_t &NewCst); - void applyCombineConstPtrAddToI2P(MachineInstr &MI, int64_t &NewCst); + bool matchCombineConstPtrAddToI2P(MachineInstr &MI, APInt &NewCst); + void applyCombineConstPtrAddToI2P(MachineInstr &MI, APInt &NewCst); /// Transform anyext(trunc(x)) to x. bool matchCombineAnyExtTrunc(MachineInstr &MI, Register &Reg); diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerInfo.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerInfo.h index 4a1a4ff2528a..e73f8489497e 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerInfo.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerInfo.h @@ -20,7 +20,6 @@ class GISelChangeObserver; class LegalizerInfo; class MachineInstr; class MachineIRBuilder; -class MachineRegisterInfo; // Contains information relevant to enabling/disabling various combines for a // pass. diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GISelWorkList.h b/llvm/include/llvm/CodeGen/GlobalISel/GISelWorkList.h index c5af64d2bcbe..7d198fada411 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/GISelWorkList.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/GISelWorkList.h @@ -15,7 +15,6 @@ namespace llvm { class MachineInstr; -class MachineFunction; // Worklist which mostly works similar to InstCombineWorkList, but on // MachineInstrs. The main difference with something like a SetVector is that diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h index 886b3af834d7..38d2fe28063a 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h @@ -487,7 +487,8 @@ public: // That is not done yet. if (ConvertOp == 0) return true; - return !DestTy.isVector() && OpTy.isVector(); + return !DestTy.isVector() && OpTy.isVector() && + DestTy == OpTy.getElementType(); case TargetOpcode::G_CONCAT_VECTORS: { if (ConvertOp == 0) return true; @@ -977,10 +978,13 @@ public: Builder.setInstr(MI); for (unsigned Idx = 0; Idx < NumDefs; ++Idx) { - Register MergeSrc = MergeI->getOperand(Idx + 1).getReg(); Register DefReg = MI.getOperand(Idx).getReg(); - Builder.buildInstr(ConvertOp, {DefReg}, {MergeSrc}); - UpdatedDefs.push_back(DefReg); + Register MergeSrc = MergeI->getOperand(Idx + 1).getReg(); + + if (!MRI.use_empty(DefReg)) { + Builder.buildInstr(ConvertOp, {DefReg}, {MergeSrc}); + UpdatedDefs.push_back(DefReg); + } } markInstAndDefDead(MI, *MergeI, DeadInsts); diff --git a/llvm/include/llvm/CodeGen/GlobalISel/Legalizer.h b/llvm/include/llvm/CodeGen/GlobalISel/Legalizer.h index 4871d8d32ebd..c19f1d5330ba 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/Legalizer.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/Legalizer.h @@ -25,7 +25,6 @@ namespace llvm { -class MachineRegisterInfo; class LostDebugLocObserver; class Legalizer : public MachineFunctionPass { diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h index 044f2e22cfdd..3b2f937375eb 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h @@ -30,7 +30,6 @@ namespace llvm { // Forward declarations. class LegalizerInfo; -class Legalizer; class MachineRegisterInfo; class GISelChangeObserver; class LostDebugLocObserver; diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h index 0b37539030b1..9507c3411b5c 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h @@ -38,7 +38,6 @@ class LegalizerHelper; class MachineInstr; class MachineRegisterInfo; class MCInstrInfo; -class GISelChangeObserver; namespace LegalizeActions { enum LegalizeAction : std::uint8_t { @@ -557,7 +556,7 @@ class LegalizeRuleSet { } public: - LegalizeRuleSet() : AliasOf(0), IsAliasedByAnother(false), Rules() {} + LegalizeRuleSet() : AliasOf(0), IsAliasedByAnother(false) {} bool isAliasedByAnother() { return IsAliasedByAnother; } void setIsAliasedByAnother() { IsAliasedByAnother = true; } diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LoadStoreOpt.h b/llvm/include/llvm/CodeGen/GlobalISel/LoadStoreOpt.h index 29575f386d7a..0845c001abdb 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/LoadStoreOpt.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/LoadStoreOpt.h @@ -30,7 +30,6 @@ namespace llvm { // Forward declarations. class MachineRegisterInfo; -class TargetTransformInfo; namespace GISelAddressing { /// Helper struct to store a base, index and offset that forms an address struct BaseIndexOffset { diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h b/llvm/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h index 28bb8de11762..daf1ff052983 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h @@ -13,6 +13,7 @@ #ifndef LLVM_CODEGEN_GLOBALISEL_MIPATTERNMATCH_H #define LLVM_CODEGEN_GLOBALISEL_MIPATTERNMATCH_H +#include "llvm/ADT/APInt.h" #include "llvm/CodeGen/GlobalISel/Utils.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/IR/InstrTypes.h" @@ -59,11 +60,26 @@ inline OneNonDBGUse_match<SubPat> m_OneNonDBGUse(const SubPat &SP) { return SP; } -struct ConstantMatch { - int64_t &CR; - ConstantMatch(int64_t &C) : CR(C) {} +template <typename ConstT> +inline Optional<ConstT> matchConstant(Register, const MachineRegisterInfo &); + +template <> +inline Optional<APInt> matchConstant(Register Reg, + const MachineRegisterInfo &MRI) { + return getIConstantVRegVal(Reg, MRI); +} + +template <> +inline Optional<int64_t> matchConstant(Register Reg, + const MachineRegisterInfo &MRI) { + return getIConstantVRegSExtVal(Reg, MRI); +} + +template <typename ConstT> struct ConstantMatch { + ConstT &CR; + ConstantMatch(ConstT &C) : CR(C) {} bool match(const MachineRegisterInfo &MRI, Register Reg) { - if (auto MaybeCst = getIConstantVRegSExtVal(Reg, MRI)) { + if (auto MaybeCst = matchConstant<ConstT>(Reg, MRI)) { CR = *MaybeCst; return true; } @@ -71,7 +87,12 @@ struct ConstantMatch { } }; -inline ConstantMatch m_ICst(int64_t &Cst) { return ConstantMatch(Cst); } +inline ConstantMatch<APInt> m_ICst(APInt &Cst) { + return ConstantMatch<APInt>(Cst); +} +inline ConstantMatch<int64_t> m_ICst(int64_t &Cst) { + return ConstantMatch<int64_t>(Cst); +} struct GCstAndRegMatch { Optional<ValueAndVReg> &ValReg; diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h index fde0cb3cf1af..c4c2fc076dd8 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h @@ -836,17 +836,38 @@ public: /// \return a MachineInstrBuilder for the newly created instruction. MachineInstrBuilder buildCopy(const DstOp &Res, const SrcOp &Op); + + /// Build and insert G_ASSERT_SEXT, G_ASSERT_ZEXT, or G_ASSERT_ALIGN + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildAssertOp(unsigned Opc, const DstOp &Res, const SrcOp &Op, + unsigned Val) { + return buildInstr(Opc, Res, Op).addImm(Val); + } + /// Build and insert \p Res = G_ASSERT_ZEXT Op, Size /// /// \return a MachineInstrBuilder for the newly created instruction. MachineInstrBuilder buildAssertZExt(const DstOp &Res, const SrcOp &Op, - unsigned Size); + unsigned Size) { + return buildAssertOp(TargetOpcode::G_ASSERT_ZEXT, Res, Op, Size); + } /// Build and insert \p Res = G_ASSERT_SEXT Op, Size /// /// \return a MachineInstrBuilder for the newly created instruction. MachineInstrBuilder buildAssertSExt(const DstOp &Res, const SrcOp &Op, - unsigned Size); + unsigned Size) { + return buildAssertOp(TargetOpcode::G_ASSERT_SEXT, Res, Op, Size); + } + + /// Build and insert \p Res = G_ASSERT_ALIGN Op, AlignVal + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildAssertAlign(const DstOp &Res, const SrcOp &Op, + Align AlignVal) { + return buildAssertOp(TargetOpcode::G_ASSERT_ALIGN, Res, Op, AlignVal.value()); + } /// Build and insert `Res = G_LOAD Addr, MMO`. /// diff --git a/llvm/include/llvm/CodeGen/GlobalISel/RegBankSelect.h b/llvm/include/llvm/CodeGen/GlobalISel/RegBankSelect.h index 5c693d8de521..45006eecfce6 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/RegBankSelect.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/RegBankSelect.h @@ -253,7 +253,7 @@ public: public: MBBInsertPoint(MachineBasicBlock &MBB, bool Beginning = true) - : InsertPoint(), MBB(MBB), Beginning(Beginning) { + : MBB(MBB), Beginning(Beginning) { // If we try to insert before phis, we should use the insertion // points on the incoming edges. assert((!Beginning || MBB.getFirstNonPHI() == MBB.begin()) && @@ -299,7 +299,7 @@ public: public: EdgeInsertPoint(MachineBasicBlock &Src, MachineBasicBlock &Dst, Pass &P) - : InsertPoint(), Src(Src), DstOrSplit(&Dst), P(P) {} + : Src(Src), DstOrSplit(&Dst), P(P) {} bool isSplit() const override { return Src.succ_size() > 1 && DstOrSplit->pred_size() > 1; diff --git a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h index 8fed79585fe9..aed915d2cc4b 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h @@ -44,7 +44,6 @@ class TargetLowering; class TargetPassConfig; class TargetRegisterInfo; class TargetRegisterClass; -class ConstantInt; class ConstantFP; class APFloat; class MachineIRBuilder; @@ -271,9 +270,10 @@ Optional<APFloat> ConstantFoldFPBinOp(unsigned Opcode, const Register Op1, /// If successful, returns the G_BUILD_VECTOR representing the folded vector /// constant. \p MIB should have an insertion point already set to create new /// G_CONSTANT instructions as needed. -Optional<MachineInstr *> -ConstantFoldVectorBinop(unsigned Opcode, const Register Op1, const Register Op2, - const MachineRegisterInfo &MRI, MachineIRBuilder &MIB); +Register ConstantFoldVectorBinop(unsigned Opcode, const Register Op1, + const Register Op2, + const MachineRegisterInfo &MRI, + MachineIRBuilder &MIB); Optional<APInt> ConstantFoldExtOp(unsigned Opcode, const Register Op1, uint64_t Imm, const MachineRegisterInfo &MRI); @@ -311,10 +311,11 @@ Align inferAlignFromPtrInfo(MachineFunction &MF, const MachinePointerInfo &MPO); /// /// If there is an existing live-in argument register, it will be returned. /// This will also ensure there is a valid copy -Register getFunctionLiveInPhysReg(MachineFunction &MF, const TargetInstrInfo &TII, +Register getFunctionLiveInPhysReg(MachineFunction &MF, + const TargetInstrInfo &TII, MCRegister PhysReg, const TargetRegisterClass &RC, - LLT RegTy = LLT()); + const DebugLoc &DL, LLT RegTy = LLT()); /// Return the least common multiple type of \p OrigTy and \p TargetTy, by changing the /// number of vector elements or scalar bitwidth. The intent is a diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h index fd106f55a43d..b07c7cd3db3a 100644 --- a/llvm/include/llvm/CodeGen/ISDOpcodes.h +++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h @@ -797,6 +797,10 @@ enum NodeType { /// The scalar width of the type given in operand 1 must be equal to, or /// smaller than, the scalar result type width. It may end up being smaller /// than the result width as a result of integer type legalization. + /// + /// After converting to the scalar integer type in operand 1, the value is + /// extended to the result VT. FP_TO_SINT_SAT sign extends and FP_TO_UINT_SAT + /// zero extends. FP_TO_SINT_SAT, FP_TO_UINT_SAT, diff --git a/llvm/include/llvm/CodeGen/IndirectThunks.h b/llvm/include/llvm/CodeGen/IndirectThunks.h index 90f9912f0ee0..a2cdd0a9e965 100644 --- a/llvm/include/llvm/CodeGen/IndirectThunks.h +++ b/llvm/include/llvm/CodeGen/IndirectThunks.h @@ -59,7 +59,7 @@ void ThunkInserter<Derived>::createThunkFunction(MachineModuleInfo &MMI, // Add Attributes so that we don't create a frame, unwind information, or // inline. - AttrBuilder B; + AttrBuilder B(Ctx); B.addAttribute(llvm::Attribute::NoUnwind); B.addAttribute(llvm::Attribute::Naked); F->addFnAttrs(B); diff --git a/llvm/include/llvm/CodeGen/LiveInterval.h b/llvm/include/llvm/CodeGen/LiveInterval.h index 923a45821dd4..51ffe2807434 100644 --- a/llvm/include/llvm/CodeGen/LiveInterval.h +++ b/llvm/include/llvm/CodeGen/LiveInterval.h @@ -724,7 +724,7 @@ namespace llvm { T *P; public: - SingleLinkedListIterator<T>(T *P) : P(P) {} + SingleLinkedListIterator(T *P) : P(P) {} SingleLinkedListIterator<T> &operator++() { P = P->Next; diff --git a/llvm/include/llvm/CodeGen/LiveRangeEdit.h b/llvm/include/llvm/CodeGen/LiveRangeEdit.h index fa4e80179eec..d80522f5bdac 100644 --- a/llvm/include/llvm/CodeGen/LiveRangeEdit.h +++ b/llvm/include/llvm/CodeGen/LiveRangeEdit.h @@ -34,9 +34,7 @@ namespace llvm { class AAResults; class LiveIntervals; -class MachineBlockFrequencyInfo; class MachineInstr; -class MachineLoopInfo; class MachineOperand; class TargetInstrInfo; class TargetRegisterInfo; diff --git a/llvm/include/llvm/CodeGen/MIRFormatter.h b/llvm/include/llvm/CodeGen/MIRFormatter.h index 12c90600f6df..3f145ff224ad 100644 --- a/llvm/include/llvm/CodeGen/MIRFormatter.h +++ b/llvm/include/llvm/CodeGen/MIRFormatter.h @@ -23,7 +23,6 @@ namespace llvm { class MachineFunction; class MachineInstr; struct PerFunctionMIParsingState; -struct SlotMapping; /// MIRFormater - Interface to format MIR operand based on target class MIRFormatter { diff --git a/llvm/include/llvm/CodeGen/MachineBasicBlock.h b/llvm/include/llvm/CodeGen/MachineBasicBlock.h index efe22ea8f332..638b6732a543 100644 --- a/llvm/include/llvm/CodeGen/MachineBasicBlock.h +++ b/llvm/include/llvm/CodeGen/MachineBasicBlock.h @@ -136,6 +136,10 @@ private: /// Alignment of the basic block. One if the basic block does not need to be /// aligned. Align Alignment; + /// Maximum amount of bytes that can be added to align the basic block. If the + /// alignment cannot be reached in this many bytes, no bytes are emitted. + /// Zero to represent no maximum. + unsigned MaxBytesForAlignment = 0; /// Indicate that this basic block is entered via an exception handler. bool IsEHPad = false; @@ -521,6 +525,19 @@ public: /// Set alignment of the basic block. void setAlignment(Align A) { Alignment = A; } + void setAlignment(Align A, unsigned MaxBytes) { + setAlignment(A); + setMaxBytesForAlignment(MaxBytes); + } + + /// Return the maximum amount of padding allowed for aligning the basic block. + unsigned getMaxBytesForAlignment() const { return MaxBytesForAlignment; } + + /// Set the maximum amount of padding allowed for aligning the basic block + void setMaxBytesForAlignment(unsigned MaxBytes) { + MaxBytesForAlignment = MaxBytes; + } + /// Returns true if the block is a landing pad. That is this basic block is /// entered via an exception handler. bool isEHPad() const { return IsEHPad; } diff --git a/llvm/include/llvm/CodeGen/MachineLoopUtils.h b/llvm/include/llvm/CodeGen/MachineLoopUtils.h index 2352fbca548d..b9bf93b71e25 100644 --- a/llvm/include/llvm/CodeGen/MachineLoopUtils.h +++ b/llvm/include/llvm/CodeGen/MachineLoopUtils.h @@ -10,7 +10,6 @@ #define LLVM_CODEGEN_MACHINELOOPUTILS_H namespace llvm { -class MachineLoop; class MachineBasicBlock; class MachineRegisterInfo; class TargetInstrInfo; diff --git a/llvm/include/llvm/CodeGen/MachineModuleInfo.h b/llvm/include/llvm/CodeGen/MachineModuleInfo.h index 860a86ee991b..c07606e89374 100644 --- a/llvm/include/llvm/CodeGen/MachineModuleInfo.h +++ b/llvm/include/llvm/CodeGen/MachineModuleInfo.h @@ -44,7 +44,6 @@ namespace llvm { class BasicBlock; -class CallInst; class Function; class LLVMTargetMachine; class MMIAddrLabelMap; diff --git a/llvm/include/llvm/CodeGen/MachinePassManager.h b/llvm/include/llvm/CodeGen/MachinePassManager.h index f967167c65e1..75b8a89c812e 100644 --- a/llvm/include/llvm/CodeGen/MachinePassManager.h +++ b/llvm/include/llvm/CodeGen/MachinePassManager.h @@ -40,10 +40,10 @@ class MachineFunctionAnalysisManager : public AnalysisManager<MachineFunction> { public: using Base = AnalysisManager<MachineFunction>; - MachineFunctionAnalysisManager() : Base(), FAM(nullptr), MAM(nullptr) {} + MachineFunctionAnalysisManager() : FAM(nullptr), MAM(nullptr) {} MachineFunctionAnalysisManager(FunctionAnalysisManager &FAM, ModuleAnalysisManager &MAM) - : Base(), FAM(&FAM), MAM(&MAM) {} + : FAM(&FAM), MAM(&MAM) {} MachineFunctionAnalysisManager(MachineFunctionAnalysisManager &&) = default; MachineFunctionAnalysisManager & operator=(MachineFunctionAnalysisManager &&) = default; @@ -135,7 +135,7 @@ public: MachineFunctionPassManager(bool DebugLogging = false, bool RequireCodeGenSCCOrder = false, bool VerifyMachineFunction = false) - : Base(), RequireCodeGenSCCOrder(RequireCodeGenSCCOrder), + : RequireCodeGenSCCOrder(RequireCodeGenSCCOrder), VerifyMachineFunction(VerifyMachineFunction) {} MachineFunctionPassManager(MachineFunctionPassManager &&) = default; MachineFunctionPassManager & diff --git a/llvm/include/llvm/CodeGen/MachineScheduler.h b/llvm/include/llvm/CodeGen/MachineScheduler.h index e368fd7d056a..267c4b595eec 100644 --- a/llvm/include/llvm/CodeGen/MachineScheduler.h +++ b/llvm/include/llvm/CodeGen/MachineScheduler.h @@ -425,10 +425,6 @@ protected: IntervalPressure BotPressure; RegPressureTracker BotRPTracker; - /// True if disconnected subregister components are already renamed. - /// The renaming is only done on demand if lane masks are tracked. - bool DisconnectedComponentsRenamed = false; - public: ScheduleDAGMILive(MachineSchedContext *C, std::unique_ptr<MachineSchedStrategy> S) diff --git a/llvm/include/llvm/CodeGen/Passes.h b/llvm/include/llvm/CodeGen/Passes.h index d5ad12fadfa0..616ab1034133 100644 --- a/llvm/include/llvm/CodeGen/Passes.h +++ b/llvm/include/llvm/CodeGen/Passes.h @@ -30,7 +30,6 @@ class MemoryBuffer; class ModulePass; class Pass; class TargetMachine; -class TargetRegisterClass; class raw_ostream; } // End llvm namespace @@ -551,6 +550,10 @@ namespace llvm { /// The pass transforms amx intrinsics to scalar operation if the function has /// optnone attribute or it is O0. FunctionPass *createX86LowerAMXIntrinsicsPass(); + + /// When learning an eviction policy, extract score(reward) information, + /// otherwise this does nothing + FunctionPass *createRegAllocScoringPass(); } // End llvm namespace #endif diff --git a/llvm/include/llvm/CodeGen/SDNodeProperties.td b/llvm/include/llvm/CodeGen/SDNodeProperties.td index d25e0bda26a9..3cb304f47f4b 100644 --- a/llvm/include/llvm/CodeGen/SDNodeProperties.td +++ b/llvm/include/llvm/CodeGen/SDNodeProperties.td @@ -1,4 +1,4 @@ -//===- SDNodeProperties.td - Common code for DAG isels ---*- tablegen -*-===// +//===- SDNodeProperties.td - Common code for DAG isels -----*- tablegen -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h index d21844555f5b..e31719bcff0b 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAG.h +++ b/llvm/include/llvm/CodeGen/SelectionDAG.h @@ -1350,13 +1350,9 @@ public: SDValue getIndexedLoadVP(SDValue OrigLoad, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM); SDValue getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, - SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, - Align Alignment, MachineMemOperand::Flags MMOFlags, - const AAMDNodes &AAInfo = AAMDNodes(), - bool IsCompressing = false); - SDValue getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, - SDValue Mask, SDValue EVL, MachineMemOperand *MMO, - bool IsCompressing = false); + SDValue Offset, SDValue Mask, SDValue EVL, EVT MemVT, + MachineMemOperand *MMO, ISD::MemIndexedMode AM, + bool IsTruncating = false, bool IsCompressing = false); SDValue getTruncStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, @@ -1833,18 +1829,18 @@ public: unsigned ComputeNumSignBits(SDValue Op, const APInt &DemandedElts, unsigned Depth = 0) const; - /// Get the minimum bit size for this Value \p Op as a signed integer. - /// i.e. x == sext(trunc(x to MinSignedBits) to bitwidth(x)). - /// Similar to the APInt::getMinSignedBits function. + /// Get the upper bound on bit size for this Value \p Op as a signed integer. + /// i.e. x == sext(trunc(x to MaxSignedBits) to bitwidth(x)). + /// Similar to the APInt::getSignificantBits function. /// Helper wrapper to ComputeNumSignBits. - unsigned ComputeMinSignedBits(SDValue Op, unsigned Depth = 0) const; + unsigned ComputeMaxSignificantBits(SDValue Op, unsigned Depth = 0) const; - /// Get the minimum bit size for this Value \p Op as a signed integer. - /// i.e. x == sext(trunc(x to MinSignedBits) to bitwidth(x)). - /// Similar to the APInt::getMinSignedBits function. + /// Get the upper bound on bit size for this Value \p Op as a signed integer. + /// i.e. x == sext(trunc(x to MaxSignedBits) to bitwidth(x)). + /// Similar to the APInt::getSignificantBits function. /// Helper wrapper to ComputeNumSignBits. - unsigned ComputeMinSignedBits(SDValue Op, const APInt &DemandedElts, - unsigned Depth = 0) const; + unsigned ComputeMaxSignificantBits(SDValue Op, const APInt &DemandedElts, + unsigned Depth = 0) const; /// Return true if this function can prove that \p Op is never poison /// and, if \p PoisonOnly is false, does not have undef bits. @@ -2001,6 +1997,9 @@ public: return SplitVector(N, DL, LoVT, HiVT); } + /// Split the explicit vector length parameter of a VP operation. + std::pair<SDValue, SDValue> SplitEVL(SDValue N, EVT VecVT, const SDLoc &DL); + /// Split the node's operand with EXTRACT_SUBVECTOR and /// return the low/high part. std::pair<SDValue, SDValue> SplitVectorOperand(const SDNode *N, unsigned OpNo) diff --git a/llvm/include/llvm/CodeGen/SelectionDAGAddressAnalysis.h b/llvm/include/llvm/CodeGen/SelectionDAGAddressAnalysis.h index 6a3d76be0ed6..0f3af915da64 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAGAddressAnalysis.h +++ b/llvm/include/llvm/CodeGen/SelectionDAGAddressAnalysis.h @@ -39,7 +39,7 @@ private: public: BaseIndexOffset() = default; BaseIndexOffset(SDValue Base, SDValue Index, bool IsIndexSignExt) - : Base(Base), Index(Index), Offset(), IsIndexSignExt(IsIndexSignExt) {} + : Base(Base), Index(Index), IsIndexSignExt(IsIndexSignExt) {} BaseIndexOffset(SDValue Base, SDValue Index, int64_t Offset, bool IsIndexSignExt) : Base(Base), Index(Index), Offset(Offset), diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h index 2855e1f1e587..cd62c47abce9 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h +++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h @@ -675,6 +675,9 @@ public: } } + /// Test if this node is a vector predication operation. + bool isVPOpcode() const { return ISD::isVPOpcode(getOpcode()); } + /// Test if this node has a post-isel opcode, directly /// corresponding to a MachineInstr opcode. bool isMachineOpcode() const { return NodeType < 0; } diff --git a/llvm/include/llvm/CodeGen/TailDuplicator.h b/llvm/include/llvm/CodeGen/TailDuplicator.h index 6862bb2c3f44..daaa27f72d52 100644 --- a/llvm/include/llvm/CodeGen/TailDuplicator.h +++ b/llvm/include/llvm/CodeGen/TailDuplicator.h @@ -26,7 +26,6 @@ namespace llvm { class MachineBasicBlock; -class MachineBlockFrequencyInfo; class MachineBranchProbabilityInfo; class MachineFunction; class MachineInstr; diff --git a/llvm/include/llvm/CodeGen/TargetFrameLowering.h b/llvm/include/llvm/CodeGen/TargetFrameLowering.h index a855a0797723..f2ca1590fc39 100644 --- a/llvm/include/llvm/CodeGen/TargetFrameLowering.h +++ b/llvm/include/llvm/CodeGen/TargetFrameLowering.h @@ -216,8 +216,8 @@ public: /// With basic block sections, emit callee saved frame moves for basic blocks /// that are in a different section. virtual void - emitCalleeSavedFrameMoves(MachineBasicBlock &MBB, - MachineBasicBlock::iterator MBBI) const {} + emitCalleeSavedFrameMovesFullCFA(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI) const {} /// Replace a StackProbe stub (if any) with the actual probe code inline virtual void inlineStackProbe(MachineFunction &MF, diff --git a/llvm/include/llvm/CodeGen/TargetInstrInfo.h b/llvm/include/llvm/CodeGen/TargetInstrInfo.h index 58b8e59b68d7..411811d08c18 100644 --- a/llvm/include/llvm/CodeGen/TargetInstrInfo.h +++ b/llvm/include/llvm/CodeGen/TargetInstrInfo.h @@ -130,7 +130,7 @@ public: } /// Given \p MO is a PhysReg use return if it can be ignored for the purpose - /// of instruction rematerialization. + /// of instruction rematerialization or sinking. virtual bool isIgnorableUse(const MachineOperand &MO) const { return false; } diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h index b2d82e0cc6e8..bec191570594 100644 --- a/llvm/include/llvm/CodeGen/TargetLowering.h +++ b/llvm/include/llvm/CodeGen/TargetLowering.h @@ -63,7 +63,6 @@ namespace llvm { -class BranchProbability; class CCState; class CCValAssign; class Constant; @@ -1802,11 +1801,14 @@ public: /// Return the preferred loop alignment. virtual Align getPrefLoopAlignment(MachineLoop *ML = nullptr) const; + /// Return the maximum amount of bytes allowed to be emitted when padding for + /// alignment + virtual unsigned + getMaxPermittedBytesForAlignment(MachineBasicBlock *MBB) const; + /// Should loops be aligned even when the function is marked OptSize (but not /// MinSize). - virtual bool alignLoopsWithOptSize() const { - return false; - } + virtual bool alignLoopsWithOptSize() const { return false; } /// If the target has a standard location for the stack protector guard, /// returns the address of that location. Otherwise, returns nullptr. @@ -1836,8 +1838,8 @@ public: virtual Function *getSSPStackGuardCheck(const Module &M) const; /// \returns true if a constant G_UBFX is legal on the target. - virtual bool isConstantUnsignedBitfieldExtactLegal(unsigned Opc, LLT Ty1, - LLT Ty2) const { + virtual bool isConstantUnsignedBitfieldExtractLegal(unsigned Opc, LLT Ty1, + LLT Ty2) const { return false; } @@ -2341,6 +2343,9 @@ protected: /// means the target does not care about loop alignment. The target may also /// override getPrefLoopAlignment to provide per-loop values. void setPrefLoopAlignment(Align Alignment) { PrefLoopAlignment = Alignment; } + void setMaxBytesForAlignment(unsigned MaxBytes) { + MaxBytesForAlignment = MaxBytes; + } /// Set the minimum stack alignment of an argument. void setMinStackArgumentAlignment(Align Alignment) { @@ -2521,6 +2526,8 @@ public: case ISD::SHL: case ISD::SRL: case ISD::SRA: + case ISD::ROTL: + case ISD::ROTR: case ISD::SDIV: case ISD::UDIV: case ISD::SREM: @@ -3030,6 +3037,8 @@ private: /// The preferred loop alignment (in log2 bot in bytes). Align PrefLoopAlignment; + /// The maximum amount of bytes permitted to be emitted for alignment. + unsigned MaxBytesForAlignment; /// Size in bits of the maximum atomics size the backend supports. /// Accesses larger than this will be expanded by AtomicExpandPass. @@ -3283,6 +3292,17 @@ public: return false; } + // Lets target to control the following reassociation of operands: (op (op x, + // c1), y) -> (op (op x, y), c1) where N0 is (op x, c1) and N1 is y. By + // default consider profitable any case where N0 has single use. This + // behavior reflects the condition replaced by this target hook call in the + // DAGCombiner. Any particular target can implement its own heuristic to + // restrict common combiner. + virtual bool isReassocProfitable(SelectionDAG &DAG, SDValue N0, + SDValue N1) const { + return N0.hasOneUse(); + } + virtual bool isSDNodeAlwaysUniform(const SDNode * N) const { return false; } diff --git a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h index 8483d078ca74..c3b842052ef5 100644 --- a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h +++ b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h @@ -1094,6 +1094,13 @@ public: inline MCRegister getSubReg(MCRegister Reg, unsigned Idx) const { return static_cast<const MCRegisterInfo *>(this)->getSubReg(Reg, Idx); } + + /// Some targets have non-allocatable registers that aren't technically part + /// of the explicit callee saved register list, but should be handled as such + /// in certain cases. + virtual bool isNonallocatableRegisterCalleeSave(MCRegister Reg) const { + return false; + } }; //===----------------------------------------------------------------------===// diff --git a/llvm/include/llvm/DWARFLinker/DWARFLinker.h b/llvm/include/llvm/DWARFLinker/DWARFLinker.h index 1c6d0b1ead86..4f1c666df35f 100644 --- a/llvm/include/llvm/DWARFLinker/DWARFLinker.h +++ b/llvm/include/llvm/DWARFLinker/DWARFLinker.h @@ -385,8 +385,8 @@ private: : Die(Die), Type(T), CU(CU), Flags(0), OtherInfo(OtherInfo) {} WorklistItem(unsigned AncestorIdx, CompileUnit &CU, unsigned Flags) - : Die(), Type(WorklistItemType::LookForParentDIEsToKeep), CU(CU), - Flags(Flags), AncestorIdx(AncestorIdx) {} + : Type(WorklistItemType::LookForParentDIEsToKeep), CU(CU), Flags(Flags), + AncestorIdx(AncestorIdx) {} }; /// returns true if we need to translate strings. diff --git a/llvm/include/llvm/DWARFLinker/DWARFLinkerCompileUnit.h b/llvm/include/llvm/DWARFLinker/DWARFLinkerCompileUnit.h index a6310bcb5df1..afba19ac7d42 100644 --- a/llvm/include/llvm/DWARFLinker/DWARFLinkerCompileUnit.h +++ b/llvm/include/llvm/DWARFLinker/DWARFLinkerCompileUnit.h @@ -148,9 +148,6 @@ public: return LocationAttributes; } - void setHasInterestingContent() { HasInterestingContent = true; } - bool hasInterestingContent() { return HasInterestingContent; } - /// Mark every DIE in this unit as kept. This function also /// marks variables as InDebugMap so that they appear in the /// reconstructed accelerator tables. @@ -298,9 +295,6 @@ private: /// Is this unit subject to the ODR rule? bool HasODR; - /// Did a DIE actually contain a valid reloc? - bool HasInterestingContent; - /// The DW_AT_language of this unit. uint16_t Language = 0; diff --git a/llvm/include/llvm/DebugInfo/CodeView/CodeView.h b/llvm/include/llvm/DebugInfo/CodeView/CodeView.h index 9d41cb9fdd2b..d4cb6ae7a28e 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/CodeView.h +++ b/llvm/include/llvm/DebugInfo/CodeView/CodeView.h @@ -162,6 +162,8 @@ enum SourceLanguage : uint8_t { MSIL = 0x0f, HLSL = 0x10, + Rust = 0x15, + /// The DMD & Swift compilers emit 'D' and 'S', respectively, for the CV /// source language. Microsoft does not have enumerators for them yet. D = 'D', diff --git a/llvm/include/llvm/DebugInfo/CodeView/TypeRecord.h b/llvm/include/llvm/DebugInfo/CodeView/TypeRecord.h index 3b6d1b0b1a70..01916bd18fb8 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/TypeRecord.h +++ b/llvm/include/llvm/DebugInfo/CodeView/TypeRecord.h @@ -111,7 +111,8 @@ public: } TypeIndex ContainingType; - PointerToMemberRepresentation Representation; + PointerToMemberRepresentation Representation = + PointerToMemberRepresentation::Unknown; }; class TypeRecord { @@ -160,8 +161,8 @@ public: TypeIndex getArgumentList() const { return ArgumentList; } TypeIndex ReturnType; - CallingConvention CallConv; - FunctionOptions Options; + CallingConvention CallConv = CallingConvention::NearC; + FunctionOptions Options = FunctionOptions::None; uint16_t ParameterCount = 0; TypeIndex ArgumentList; }; @@ -194,8 +195,8 @@ public: TypeIndex ReturnType; TypeIndex ClassType; TypeIndex ThisType; - CallingConvention CallConv; - FunctionOptions Options; + CallingConvention CallConv = CallingConvention::NearC; + FunctionOptions Options = FunctionOptions::None; uint16_t ParameterCount = 0; TypeIndex ArgumentList; int32_t ThisPointerAdjustment = 0; @@ -209,7 +210,7 @@ public: LabelRecord(LabelType Mode) : TypeRecord(TypeRecordKind::Label), Mode(Mode) {} - LabelType Mode; + LabelType Mode = LabelType::Near; }; // LF_MFUNC_ID @@ -454,7 +455,7 @@ public: StringRef getUniqueName() const { return UniqueName; } uint16_t MemberCount = 0; - ClassOptions Options; + ClassOptions Options = ClassOptions::None; TypeIndex FieldList; StringRef Name; StringRef UniqueName; @@ -585,7 +586,7 @@ public: uint32_t getAge() const { return Age; } StringRef getName() const { return Name; } - GUID Guid; + GUID Guid = {}; uint32_t Age = 0; StringRef Name; }; diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFContext.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFContext.h index 24714ac3d101..e82faf6eeb24 100644 --- a/llvm/include/llvm/DebugInfo/DWARF/DWARFContext.h +++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFContext.h @@ -52,6 +52,7 @@ class raw_ostream; /// information parsing. The actual data is supplied through DWARFObj. class DWARFContext : public DIContext { DWARFUnitVector NormalUnits; + Optional<DenseMap<uint64_t, DWARFTypeUnit*>> NormalTypeUnits; std::unique_ptr<DWARFUnitIndex> CUIndex; std::unique_ptr<DWARFGdbIndex> GdbIndex; std::unique_ptr<DWARFUnitIndex> TUIndex; @@ -70,6 +71,7 @@ class DWARFContext : public DIContext { std::unique_ptr<AppleAcceleratorTable> AppleObjC; DWARFUnitVector DWOUnits; + Optional<DenseMap<uint64_t, DWARFTypeUnit*>> DWOTypeUnits; std::unique_ptr<DWARFDebugAbbrev> AbbrevDWO; std::unique_ptr<DWARFDebugMacro> MacinfoDWO; std::unique_ptr<DWARFDebugMacro> MacroDWO; diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugInfoEntry.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugInfoEntry.h index c4370cb54113..6bdd23900182 100644 --- a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugInfoEntry.h +++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugInfoEntry.h @@ -16,7 +16,6 @@ namespace llvm { -class DataExtractor; class DWARFUnit; /// DWARFDebugInfoEntry - A DIE with only the minimum required data. diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFDie.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFDie.h index 8f93ebc4ebc0..f731d440a35b 100644 --- a/llvm/include/llvm/DebugInfo/DWARF/DWARFDie.h +++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFDie.h @@ -463,13 +463,17 @@ inline bool operator!=(const std::reverse_iterator<DWARFDie::iterator> &LHS, } inline std::reverse_iterator<DWARFDie::iterator> DWARFDie::rbegin() const { - return llvm::make_reverse_iterator(end()); + return std::make_reverse_iterator(end()); } inline std::reverse_iterator<DWARFDie::iterator> DWARFDie::rend() const { - return llvm::make_reverse_iterator(begin()); + return std::make_reverse_iterator(begin()); } +void dumpTypeQualifiedName(const DWARFDie &DIE, raw_ostream &OS); +void dumpTypeUnqualifiedName(const DWARFDie &DIE, raw_ostream &OS, + std::string *OriginalFullName = nullptr); + } // end namespace llvm #endif // LLVM_DEBUGINFO_DWARF_DWARFDIE_H diff --git a/llvm/include/llvm/DebugInfo/GSYM/LookupResult.h b/llvm/include/llvm/DebugInfo/GSYM/LookupResult.h index 211a352595fd..3dabbce32bb2 100644 --- a/llvm/include/llvm/DebugInfo/GSYM/LookupResult.h +++ b/llvm/include/llvm/DebugInfo/GSYM/LookupResult.h @@ -17,7 +17,6 @@ namespace llvm { class raw_ostream; namespace gsym { -struct FileEntry; struct SourceLocation { StringRef Name; ///< Function or symbol name. diff --git a/llvm/include/llvm/DebugInfo/GSYM/ObjectFileTransformer.h b/llvm/include/llvm/DebugInfo/GSYM/ObjectFileTransformer.h index bcde92e94fc5..dcbda39a7696 100644 --- a/llvm/include/llvm/DebugInfo/GSYM/ObjectFileTransformer.h +++ b/llvm/include/llvm/DebugInfo/GSYM/ObjectFileTransformer.h @@ -21,7 +21,6 @@ class ObjectFile; namespace gsym { -struct CUInfo; class GsymCreator; class ObjectFileTransformer { diff --git a/llvm/include/llvm/DebugInfo/GSYM/StringTable.h b/llvm/include/llvm/DebugInfo/GSYM/StringTable.h index 045c9e3f3ebd..6dd90499c203 100644 --- a/llvm/include/llvm/DebugInfo/GSYM/StringTable.h +++ b/llvm/include/llvm/DebugInfo/GSYM/StringTable.h @@ -20,7 +20,7 @@ namespace gsym { /// string at offset zero. Strings must be UTF8 NULL terminated strings. struct StringTable { StringRef Data; - StringTable() : Data() {} + StringTable() {} StringTable(StringRef D) : Data(D) {} StringRef operator[](size_t Offset) const { return getString(Offset); } StringRef getString(uint32_t Offset) const { diff --git a/llvm/include/llvm/DebugInfo/MSF/MSFBuilder.h b/llvm/include/llvm/DebugInfo/MSF/MSFBuilder.h index 282870f5b3f1..1a03d42ded92 100644 --- a/llvm/include/llvm/DebugInfo/MSF/MSFBuilder.h +++ b/llvm/include/llvm/DebugInfo/MSF/MSFBuilder.h @@ -20,7 +20,6 @@ namespace llvm { class FileBufferByteStream; -class WritableBinaryStream; namespace msf { class MSFBuilder { diff --git a/llvm/include/llvm/DebugInfo/MSF/MappedBlockStream.h b/llvm/include/llvm/DebugInfo/MSF/MappedBlockStream.h index 296a4840b779..b5f0596fceed 100644 --- a/llvm/include/llvm/DebugInfo/MSF/MappedBlockStream.h +++ b/llvm/include/llvm/DebugInfo/MSF/MappedBlockStream.h @@ -154,7 +154,7 @@ private: WritableBinaryStreamRef WriteInterface; }; -} // end namespace pdb +} // namespace msf } // end namespace llvm #endif // LLVM_DEBUGINFO_MSF_MAPPEDBLOCKSTREAM_H diff --git a/llvm/include/llvm/DebugInfo/PDB/DIA/DIASupport.h b/llvm/include/llvm/DebugInfo/PDB/DIA/DIASupport.h index 1a7c2f3aeeab..570b40c70578 100644 --- a/llvm/include/llvm/DebugInfo/PDB/DIA/DIASupport.h +++ b/llvm/include/llvm/DebugInfo/PDB/DIA/DIASupport.h @@ -27,7 +27,14 @@ // DIA headers must come after windows headers. #include <cvconst.h> +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wnon-virtual-dtor" +#endif #include <dia2.h> +#ifdef __clang__ +#pragma clang diagnostic pop +#endif #include <diacreate.h> #endif // LLVM_DEBUGINFO_PDB_DIA_DIASUPPORT_H diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolExe.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolExe.h index 1a9fb240a248..cde66d399243 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolExe.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolExe.h @@ -38,7 +38,7 @@ private: void dumpChildren(raw_ostream &OS, StringRef Label, PDB_SymType ChildType, int Indent) const; }; +} // namespace pdb } // namespace llvm -} #endif // LLVM_DEBUGINFO_PDB_PDBSYMBOLEXE_H diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolFunc.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolFunc.h index 6be27c8d3bc7..f50057c68406 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolFunc.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolFunc.h @@ -79,7 +79,7 @@ public: uint32_t getCompilandId() const; }; +} // namespace pdb } // namespace llvm -} #endif // LLVM_DEBUGINFO_PDB_PDBSYMBOLFUNC_H diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolFuncDebugEnd.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolFuncDebugEnd.h index 7152249cbd03..1cdc1811bb1a 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolFuncDebugEnd.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolFuncDebugEnd.h @@ -40,7 +40,7 @@ public: FORWARD_SYMBOL_METHOD(getVirtualAddress) }; +} // namespace pdb } // namespace llvm -} #endif // LLVM_DEBUGINFO_PDB_PDBSYMBOLFUNCDEBUGEND_H diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolFuncDebugStart.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolFuncDebugStart.h index 3125c271d2e8..021f27c7f0f7 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolFuncDebugStart.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolFuncDebugStart.h @@ -39,7 +39,7 @@ public: FORWARD_SYMBOL_METHOD(getVirtualAddress) }; +} // namespace pdb } // namespace llvm -} #endif // LLVM_DEBUGINFO_PDB_PDBSYMBOLFUNCDEBUGSTART_H diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolLabel.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolLabel.h index 3625e23f014f..33eb36696cc2 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolLabel.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolLabel.h @@ -39,7 +39,7 @@ public: FORWARD_SYMBOL_METHOD(getVirtualAddress) }; +} // namespace pdb } // namespace llvm -} #endif // LLVM_DEBUGINFO_PDB_PDBSYMBOLLABEL_H diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolPublicSymbol.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolPublicSymbol.h index e2b2545d78ec..f8dcb2ba9d5f 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolPublicSymbol.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolPublicSymbol.h @@ -37,7 +37,7 @@ public: FORWARD_SYMBOL_METHOD(getUndecoratedName) }; +} // namespace pdb } // namespace llvm -} #endif // LLVM_DEBUGINFO_PDB_PDBSYMBOLPUBLICSYMBOL_H diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolThunk.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolThunk.h index 274de8b0b16f..a5f795cc1303 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolThunk.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolThunk.h @@ -46,7 +46,7 @@ public: FORWARD_SYMBOL_METHOD(getVirtualBaseOffset) FORWARD_SYMBOL_METHOD(isVolatileType) }; +} // namespace pdb } // namespace llvm -} #endif // LLVM_DEBUGINFO_PDB_PDBSYMBOLTHUNK_H diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeArray.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeArray.h index c0215c9ee4b1..d4cd6e71423e 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeArray.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeArray.h @@ -34,7 +34,7 @@ public: FORWARD_SYMBOL_METHOD(isVolatileType) }; +} // namespace pdb } // namespace llvm -} #endif // LLVM_DEBUGINFO_PDB_PDBSYMBOLTYPEARRAY_H diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeBaseClass.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeBaseClass.h index bab292ee0d46..bd2dbc914725 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeBaseClass.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeBaseClass.h @@ -53,7 +53,7 @@ public: FORWARD_SYMBOL_METHOD(isVolatileType) }; +} // namespace pdb } // namespace llvm -} #endif // LLVM_DEBUGINFO_PDB_PDBSYMBOLTYPEBASECLASS_H diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeBuiltin.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeBuiltin.h index 7d94c3c97a2b..df6309b1545c 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeBuiltin.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeBuiltin.h @@ -30,7 +30,7 @@ public: FORWARD_SYMBOL_METHOD(isVolatileType) }; +} // namespace pdb } // namespace llvm -} #endif // LLVM_DEBUGINFO_PDB_PDBSYMBOLTYPEBUILTIN_H diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeCustom.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeCustom.h index dc647aff48d3..7bf0317ff1ca 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeCustom.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeCustom.h @@ -26,7 +26,7 @@ public: FORWARD_SYMBOL_METHOD(getOemSymbolId) }; +} // namespace pdb } // namespace llvm -} #endif // LLVM_DEBUGINFO_PDB_PDBSYMBOLTYPECUSTOM_H diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeDimension.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeDimension.h index 7a9e43785d67..5d742237bac4 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeDimension.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeDimension.h @@ -26,7 +26,7 @@ public: FORWARD_SYMBOL_METHOD(getUpperBoundId) }; +} // namespace pdb } // namespace llvm -} #endif // LLVM_DEBUGINFO_PDB_PDBSYMBOLTYPEDIMENSION_H diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeEnum.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeEnum.h index 3ac72801b202..0aab91039509 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeEnum.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeEnum.h @@ -46,7 +46,7 @@ public: FORWARD_SYMBOL_METHOD(isVolatileType) }; +} // namespace pdb } // namespace llvm -} #endif // LLVM_DEBUGINFO_PDB_PDBSYMBOLTYPEENUM_H diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeFriend.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeFriend.h index c4d9dd6308a3..d56a90662dae 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeFriend.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeFriend.h @@ -27,7 +27,7 @@ public: FORWARD_SYMBOL_ID_METHOD(getType) }; +} // namespace pdb } // namespace llvm -} #endif // LLVM_DEBUGINFO_PDB_PDBSYMBOLTYPEFRIEND_H diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeFunctionArg.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeFunctionArg.h index 22d3623496f2..559ceec5aace 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeFunctionArg.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeFunctionArg.h @@ -27,7 +27,7 @@ public: FORWARD_SYMBOL_ID_METHOD(getType) }; +} // namespace pdb } // namespace llvm -} #endif // LLVM_DEBUGINFO_PDB_PDBSYMBOLTYPEFUNCTIONARG_H diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeFunctionSig.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeFunctionSig.h index a1491ca2e415..ceb4bff5b7b4 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeFunctionSig.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeFunctionSig.h @@ -41,7 +41,7 @@ public: FORWARD_SYMBOL_METHOD(isVolatileType) }; +} // namespace pdb } // namespace llvm -} #endif // LLVM_DEBUGINFO_PDB_PDBSYMBOLTYPEFUNCTIONSIG_H diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeManaged.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeManaged.h index 6bc70bca82e7..5e7b83ce8004 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeManaged.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeManaged.h @@ -25,7 +25,7 @@ public: FORWARD_SYMBOL_METHOD(getName) }; +} // namespace pdb } // namespace llvm -} #endif // LLVM_DEBUGINFO_PDB_PDBSYMBOLTYPEMANAGED_H diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypePointer.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypePointer.h index b36f459e880c..da25eab50f9b 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypePointer.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypePointer.h @@ -37,7 +37,7 @@ public: FORWARD_SYMBOL_METHOD(isVolatileType) }; +} // namespace pdb } // namespace llvm -} #endif // LLVM_DEBUGINFO_PDB_PDBSYMBOLTYPEPOINTER_H diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeTypedef.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeTypedef.h index 2712d0617e0e..8dc29ca26192 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeTypedef.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeTypedef.h @@ -44,7 +44,7 @@ public: FORWARD_SYMBOL_METHOD(isVolatileType) }; +} // namespace pdb } // namespace llvm -} #endif // LLVM_DEBUGINFO_PDB_PDBSYMBOLTYPETYPEDEF_H diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeVTable.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeVTable.h index e8161d311ea7..d08728dafa76 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeVTable.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeVTable.h @@ -31,7 +31,7 @@ public: FORWARD_SYMBOL_METHOD(isVolatileType) }; +} // namespace pdb } // namespace llvm -} #endif // LLVM_DEBUGINFO_PDB_PDBSYMBOLTYPEVTABLE_H diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeVTableShape.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeVTableShape.h index 614060867042..c7e2ac148503 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeVTableShape.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolTypeVTableShape.h @@ -29,7 +29,7 @@ public: FORWARD_SYMBOL_METHOD(isVolatileType) }; +} // namespace pdb } // namespace llvm -} #endif // LLVM_DEBUGINFO_PDB_PDBSYMBOLTYPEVTABLESHAPE_H diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolUnknown.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolUnknown.h index cc29d38c2578..5b4909b800b9 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolUnknown.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolUnknown.h @@ -24,7 +24,7 @@ public: void dump(PDBSymDumper &Dumper) const override; }; +} // namespace pdb } // namespace llvm -} #endif // LLVM_DEBUGINFO_PDB_PDBSYMBOLUNKNOWN_H diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolUsingNamespace.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolUsingNamespace.h index fd812cb2f793..19a8f414eb43 100644 --- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbolUsingNamespace.h +++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbolUsingNamespace.h @@ -27,7 +27,7 @@ public: FORWARD_SYMBOL_METHOD(getName) }; +} // namespace pdb } // namespace llvm -} #endif // LLVM_DEBUGINFO_PDB_PDBSYMBOLUSINGNAMESPACE_H diff --git a/llvm/include/llvm/DebugInfo/Symbolize/DIPrinter.h b/llvm/include/llvm/DebugInfo/Symbolize/DIPrinter.h index 4bb11bf62593..779dc885372d 100644 --- a/llvm/include/llvm/DebugInfo/Symbolize/DIPrinter.h +++ b/llvm/include/llvm/DebugInfo/Symbolize/DIPrinter.h @@ -87,7 +87,7 @@ private: public: PlainPrinterBase(raw_ostream &OS, raw_ostream &ES, PrinterConfig &Config) - : DIPrinter(), OS(OS), ES(ES), Config(Config) {} + : OS(OS), ES(ES), Config(Config) {} void print(const Request &Request, const DILineInfo &Info) override; void print(const Request &Request, const DIInliningInfo &Info) override; @@ -138,7 +138,7 @@ private: public: JSONPrinter(raw_ostream &OS, PrinterConfig &Config) - : DIPrinter(), OS(OS), Config(Config) {} + : OS(OS), Config(Config) {} void print(const Request &Request, const DILineInfo &Info) override; void print(const Request &Request, const DIInliningInfo &Info) override; diff --git a/llvm/include/llvm/Debuginfod/Debuginfod.h b/llvm/include/llvm/Debuginfod/Debuginfod.h index fcb8ed3a9222..064cfa75b1a1 100644 --- a/llvm/include/llvm/Debuginfod/Debuginfod.h +++ b/llvm/include/llvm/Debuginfod/Debuginfod.h @@ -23,6 +23,8 @@ #include "llvm/Support/Error.h" #include "llvm/Support/MemoryBuffer.h" +#include <chrono> + namespace llvm { typedef ArrayRef<uint8_t> BuildIDRef; diff --git a/llvm/include/llvm/Debuginfod/HTTPClient.h b/llvm/include/llvm/Debuginfod/HTTPClient.h index 51de66629544..ca3b76ca9f3f 100644 --- a/llvm/include/llvm/Debuginfod/HTTPClient.h +++ b/llvm/include/llvm/Debuginfod/HTTPClient.h @@ -13,12 +13,14 @@ /// //===----------------------------------------------------------------------===// -#ifndef LLVM_SUPPORT_HTTP_CLIENT_H -#define LLVM_SUPPORT_HTTP_CLIENT_H +#ifndef LLVM_DEBUGINFOD_HTTPCLIENT_H +#define LLVM_DEBUGINFOD_HTTPCLIENT_H #include "llvm/Support/Error.h" #include "llvm/Support/MemoryBuffer.h" +#include <chrono> + namespace llvm { enum class HTTPMethod { GET }; @@ -116,4 +118,4 @@ public: } // end namespace llvm -#endif // LLVM_SUPPORT_HTTP_CLIENT_H +#endif // LLVM_DEBUGINFOD_HTTPCLIENT_H diff --git a/llvm/include/llvm/Demangle/ItaniumDemangle.h b/llvm/include/llvm/Demangle/ItaniumDemangle.h index 86f5c992b63d..28545ed06836 100644 --- a/llvm/include/llvm/Demangle/ItaniumDemangle.h +++ b/llvm/include/llvm/Demangle/ItaniumDemangle.h @@ -1,3 +1,5 @@ +// Do not edit! -*- read-only -*- +// See README.txt for instructions //===------------------------- ItaniumDemangle.h ----------------*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. @@ -11,8 +13,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_DEMANGLE_ITANIUMDEMANGLE_H -#define LLVM_DEMANGLE_ITANIUMDEMANGLE_H +#ifndef DEMANGLE_ITANIUMDEMANGLE_H +#define DEMANGLE_ITANIUMDEMANGLE_H // FIXME: (possibly) incomplete list of features that clang mangles that this // file does not yet support: @@ -21,12 +23,13 @@ #include "DemangleConfig.h" #include "StringView.h" #include "Utility.h" +#include <algorithm> #include <cassert> #include <cctype> #include <cstdio> #include <cstdlib> #include <cstring> -#include <numeric> +#include <limits> #include <utility> #define FOR_EACH_NODE_KIND(X) \ @@ -1210,7 +1213,8 @@ public: class ParameterPack final : public Node { NodeArray Data; - // Setup OutputBuffer for a pack expansion unless we're already expanding one. + // Setup OutputBuffer for a pack expansion, unless we're already expanding + // one. void initializePackExpansion(OutputBuffer &OB) const { if (OB.CurrentPackMax == std::numeric_limits<unsigned>::max()) { OB.CurrentPackMax = static_cast<unsigned>(Data.size()); @@ -2473,7 +2477,7 @@ template <typename Derived, typename Alloc> struct AbstractManglingParser { char consume() { return First != Last ? *First++ : '\0'; } - char look(unsigned Lookahead = 0) { + char look(unsigned Lookahead = 0) const { if (static_cast<size_t>(Last - First) <= Lookahead) return '\0'; return First[Lookahead]; @@ -2591,34 +2595,38 @@ Node *AbstractManglingParser<Derived, Alloc>::parseName(NameState *State) { if (look() == 'Z') return getDerived().parseLocalName(State); - // ::= <unscoped-template-name> <template-args> - if (look() == 'S' && look(1) != 't') { - Node *S = getDerived().parseSubstitution(); - if (S == nullptr) - return nullptr; - if (look() != 'I') - return nullptr; - Node *TA = getDerived().parseTemplateArgs(State != nullptr); - if (TA == nullptr) - return nullptr; - if (State) State->EndsWithTemplateArgs = true; - return make<NameWithTemplateArgs>(S, TA); + Node *Result = nullptr; + bool IsSubst = look() == 'S' && look(1) != 't'; + if (IsSubst) { + // A substitution must lead to: + // ::= <unscoped-template-name> <template-args> + Result = getDerived().parseSubstitution(); + } else { + // An unscoped name can be one of: + // ::= <unscoped-name> + // ::= <unscoped-template-name> <template-args> + Result = getDerived().parseUnscopedName(State); } - - Node *N = getDerived().parseUnscopedName(State); - if (N == nullptr) + if (Result == nullptr) return nullptr; - // ::= <unscoped-template-name> <template-args> + if (look() == 'I') { - Subs.push_back(N); + // ::= <unscoped-template-name> <template-args> + if (!IsSubst) + // An unscoped-template-name is substitutable. + Subs.push_back(Result); Node *TA = getDerived().parseTemplateArgs(State != nullptr); if (TA == nullptr) return nullptr; - if (State) State->EndsWithTemplateArgs = true; - return make<NameWithTemplateArgs>(N, TA); + if (State) + State->EndsWithTemplateArgs = true; + Result = make<NameWithTemplateArgs>(Result, TA); + } else if (IsSubst) { + // The substitution case must be followed by <template-args>. + return nullptr; } - // ::= <unscoped-name> - return N; + + return Result; } // <local-name> := Z <function encoding> E <entity name> [<discriminator>] @@ -2663,13 +2671,17 @@ Node *AbstractManglingParser<Derived, Alloc>::parseLocalName(NameState *State) { template <typename Derived, typename Alloc> Node * AbstractManglingParser<Derived, Alloc>::parseUnscopedName(NameState *State) { - if (consumeIf("StL") || consumeIf("St")) { - Node *R = getDerived().parseUnqualifiedName(State); - if (R == nullptr) - return nullptr; - return make<StdQualifiedName>(R); - } - return getDerived().parseUnqualifiedName(State); + bool IsStd = consumeIf("St"); + if (IsStd) + consumeIf('L'); + + Node *Result = getDerived().parseUnqualifiedName(State); + if (Result == nullptr) + return nullptr; + if (IsStd) + Result = make<StdQualifiedName>(Result); + + return Result; } // <unqualified-name> ::= <operator-name> [abi-tags] @@ -4064,9 +4076,9 @@ Node *AbstractManglingParser<Derived, Alloc>::parseType() { } // ::= <substitution> # See Compression below case 'S': { - if (look(1) && look(1) != 't') { - Node *Sub = getDerived().parseSubstitution(); - if (Sub == nullptr) + if (look(1) != 't') { + Result = getDerived().parseSubstitution(); + if (Result == nullptr) return nullptr; // Sub could be either of: @@ -4083,13 +4095,13 @@ Node *AbstractManglingParser<Derived, Alloc>::parseType() { Node *TA = getDerived().parseTemplateArgs(); if (TA == nullptr) return nullptr; - Result = make<NameWithTemplateArgs>(Sub, TA); - break; + Result = make<NameWithTemplateArgs>(Result, TA); + } else { + // If all we parsed was a substitution, don't re-insert into the + // substitution table. + return Result; } - - // If all we parsed was a substitution, don't re-insert into the - // substitution table. - return Sub; + break; } DEMANGLE_FALLTHROUGH; } @@ -5437,38 +5449,35 @@ Node *AbstractManglingParser<Derived, Alloc>::parseSubstitution() { if (!consumeIf('S')) return nullptr; - if (std::islower(look())) { - Node *SpecialSub; + if (look() >= 'a' && look() <= 'z') { + SpecialSubKind Kind; switch (look()) { case 'a': - ++First; - SpecialSub = make<SpecialSubstitution>(SpecialSubKind::allocator); + Kind = SpecialSubKind::allocator; break; case 'b': - ++First; - SpecialSub = make<SpecialSubstitution>(SpecialSubKind::basic_string); + Kind = SpecialSubKind::basic_string; break; - case 's': - ++First; - SpecialSub = make<SpecialSubstitution>(SpecialSubKind::string); + case 'd': + Kind = SpecialSubKind::iostream; break; case 'i': - ++First; - SpecialSub = make<SpecialSubstitution>(SpecialSubKind::istream); + Kind = SpecialSubKind::istream; break; case 'o': - ++First; - SpecialSub = make<SpecialSubstitution>(SpecialSubKind::ostream); + Kind = SpecialSubKind::ostream; break; - case 'd': - ++First; - SpecialSub = make<SpecialSubstitution>(SpecialSubKind::iostream); + case 's': + Kind = SpecialSubKind::string; break; default: return nullptr; } + ++First; + auto *SpecialSub = make<SpecialSubstitution>(Kind); if (!SpecialSub) return nullptr; + // Itanium C++ ABI 5.1.2: If a name that would use a built-in <substitution> // has ABI tags, the tags are appended to the substitution; the result is a // substitutable component. @@ -5747,4 +5756,4 @@ struct ManglingParser : AbstractManglingParser<ManglingParser<Alloc>, Alloc> { DEMANGLE_NAMESPACE_END -#endif // LLVM_DEMANGLE_ITANIUMDEMANGLE_H +#endif // DEMANGLE_ITANIUMDEMANGLE_H diff --git a/llvm/include/llvm/Demangle/MicrosoftDemangle.h b/llvm/include/llvm/Demangle/MicrosoftDemangle.h index 040313661601..6f2d0416901e 100644 --- a/llvm/include/llvm/Demangle/MicrosoftDemangle.h +++ b/llvm/include/llvm/Demangle/MicrosoftDemangle.h @@ -9,10 +9,8 @@ #ifndef LLVM_DEMANGLE_MICROSOFTDEMANGLE_H #define LLVM_DEMANGLE_MICROSOFTDEMANGLE_H -#include "llvm/Demangle/DemangleConfig.h" #include "llvm/Demangle/MicrosoftDemangleNodes.h" #include "llvm/Demangle/StringView.h" -#include "llvm/Demangle/Utility.h" #include <utility> diff --git a/llvm/include/llvm/Demangle/MicrosoftDemangleNodes.h b/llvm/include/llvm/Demangle/MicrosoftDemangleNodes.h index 46daa3885a06..8ad2472364b4 100644 --- a/llvm/include/llvm/Demangle/MicrosoftDemangleNodes.h +++ b/llvm/include/llvm/Demangle/MicrosoftDemangleNodes.h @@ -13,7 +13,6 @@ #ifndef LLVM_DEMANGLE_MICROSOFTDEMANGLENODES_H #define LLVM_DEMANGLE_MICROSOFTDEMANGLENODES_H -#include "llvm/Demangle/DemangleConfig.h" #include "llvm/Demangle/StringView.h" #include <array> #include <cstdint> @@ -283,9 +282,7 @@ struct StructorIdentifierNode; struct ThunkSignatureNode; struct PointerTypeNode; struct ArrayTypeNode; -struct CustomNode; struct TagTypeNode; -struct IntrinsicTypeNode; struct NodeArrayNode; struct QualifiedNameNode; struct TemplateParameterReferenceNode; diff --git a/llvm/include/llvm/Demangle/StringView.h b/llvm/include/llvm/Demangle/StringView.h index 378e85341637..323282f69c26 100644 --- a/llvm/include/llvm/Demangle/StringView.h +++ b/llvm/include/llvm/Demangle/StringView.h @@ -1,3 +1,5 @@ +// Do not edit! -*- read-only -*- +// See README.txt for instructions //===--- StringView.h -------------------------------------------*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. @@ -10,11 +12,10 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_DEMANGLE_STRINGVIEW_H -#define LLVM_DEMANGLE_STRINGVIEW_H +#ifndef DEMANGLE_STRINGVIEW_H +#define DEMANGLE_STRINGVIEW_H #include "DemangleConfig.h" -#include <algorithm> #include <cassert> #include <cstring> @@ -38,15 +39,16 @@ public: StringView substr(size_t Pos, size_t Len = npos) const { assert(Pos <= size()); - return StringView(begin() + Pos, std::min(Len, size() - Pos)); + if (Len > size() - Pos) + Len = size() - Pos; + return StringView(begin() + Pos, Len); } size_t find(char C, size_t From = 0) const { - size_t FindBegin = std::min(From, size()); // Avoid calling memchr with nullptr. - if (FindBegin < size()) { + if (From < size()) { // Just forward to memchr, which is faster than a hand-rolled loop. - if (const void *P = ::memchr(First + FindBegin, C, size() - FindBegin)) + if (const void *P = ::memchr(First + From, C, size() - From)) return size_t(static_cast<const char *>(P) - First); } return npos; @@ -98,7 +100,7 @@ public: bool startsWith(StringView Str) const { if (Str.size() > size()) return false; - return std::equal(Str.begin(), Str.end(), begin()); + return std::strncmp(Str.begin(), begin(), Str.size()) == 0; } const char &operator[](size_t Idx) const { return *(begin() + Idx); } @@ -111,7 +113,7 @@ public: inline bool operator==(const StringView &LHS, const StringView &RHS) { return LHS.size() == RHS.size() && - std::equal(LHS.begin(), LHS.end(), RHS.begin()); + std::strncmp(LHS.begin(), RHS.begin(), LHS.size()) == 0; } DEMANGLE_NAMESPACE_END diff --git a/llvm/include/llvm/Demangle/Utility.h b/llvm/include/llvm/Demangle/Utility.h index 4fea9351a4bf..bec019da8680 100644 --- a/llvm/include/llvm/Demangle/Utility.h +++ b/llvm/include/llvm/Demangle/Utility.h @@ -1,3 +1,5 @@ +// Do not edit! -*- read-only -*- +// See README.txt for instructions //===--- Utility.h ----------------------------------------------*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. @@ -10,14 +12,15 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_DEMANGLE_UTILITY_H -#define LLVM_DEMANGLE_UTILITY_H +#ifndef DEMANGLE_UTILITY_H +#define DEMANGLE_UTILITY_H #include "StringView.h" +#include <array> #include <cstdint> #include <cstdlib> #include <cstring> -#include <iterator> +#include <exception> #include <limits> DEMANGLE_NAMESPACE_BEGIN @@ -48,8 +51,8 @@ class OutputBuffer { return; } - char Temp[21]; - char *TempPtr = std::end(Temp); + std::array<char, 21> Temp; + char *TempPtr = Temp.data() + Temp.size(); while (N) { *--TempPtr = char('0' + N % 10); @@ -59,7 +62,7 @@ class OutputBuffer { // Add negative sign... if (isNeg) *--TempPtr = '-'; - this->operator<<(StringView(TempPtr, std::end(Temp))); + this->operator<<(StringView(TempPtr, Temp.data() + Temp.size())); } public: diff --git a/llvm/include/llvm/ExecutionEngine/JITEventListener.h b/llvm/include/llvm/ExecutionEngine/JITEventListener.h index 4eefd993de2b..effff2ea5cfa 100644 --- a/llvm/include/llvm/ExecutionEngine/JITEventListener.h +++ b/llvm/include/llvm/ExecutionEngine/JITEventListener.h @@ -24,7 +24,6 @@ namespace llvm { class IntelJITEventsWrapper; -class MachineFunction; class OProfileWrapper; namespace object { diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/EHFrameSupport.h b/llvm/include/llvm/ExecutionEngine/JITLink/EHFrameSupport.h index ec78d9db40b6..33eee7a75f39 100644 --- a/llvm/include/llvm/ExecutionEngine/JITLink/EHFrameSupport.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/EHFrameSupport.h @@ -25,25 +25,20 @@ namespace jitlink { class EHFrameRegistrar { public: virtual ~EHFrameRegistrar(); - virtual Error registerEHFrames(JITTargetAddress EHFrameSectionAddr, - size_t EHFrameSectionSize) = 0; - virtual Error deregisterEHFrames(JITTargetAddress EHFrameSectionAddr, - size_t EHFrameSectionSize) = 0; + virtual Error registerEHFrames(orc::ExecutorAddrRange EHFrameSection) = 0; + virtual Error deregisterEHFrames(orc::ExecutorAddrRange EHFrameSection) = 0; }; /// Registers / Deregisters EH-frames in the current process. class InProcessEHFrameRegistrar final : public EHFrameRegistrar { public: - Error registerEHFrames(JITTargetAddress EHFrameSectionAddr, - size_t EHFrameSectionSize) override; + Error registerEHFrames(orc::ExecutorAddrRange EHFrameSection) override; - Error deregisterEHFrames(JITTargetAddress EHFrameSectionAddr, - size_t EHFrameSectionSize) override; + Error deregisterEHFrames(orc::ExecutorAddrRange EHFrameSection) override; }; -using StoreFrameRangeFunction = - std::function<void(JITTargetAddress EHFrameSectionAddr, - size_t EHFrameSectionSize)>; +using StoreFrameRangeFunction = std::function<void( + orc::ExecutorAddr EHFrameSectionAddr, size_t EHFrameSectionSize)>; /// Creates a pass that records the address and size of the EH frame section. /// If no eh-frame section is found then the address and size will both be given diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h b/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h index 83d85953fce6..ddbb3e76f145 100644 --- a/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h @@ -104,10 +104,10 @@ class Addressable { friend class LinkGraph; protected: - Addressable(JITTargetAddress Address, bool IsDefined) + Addressable(orc::ExecutorAddr Address, bool IsDefined) : Address(Address), IsDefined(IsDefined), IsAbsolute(false) {} - Addressable(JITTargetAddress Address) + Addressable(orc::ExecutorAddr Address) : Address(Address), IsDefined(false), IsAbsolute(true) { assert(!(IsDefined && IsAbsolute) && "Block cannot be both defined and absolute"); @@ -119,8 +119,8 @@ public: Addressable(Addressable &&) = delete; Addressable &operator=(Addressable &&) = default; - JITTargetAddress getAddress() const { return Address; } - void setAddress(JITTargetAddress Address) { this->Address = Address; } + orc::ExecutorAddr getAddress() const { return Address; } + void setAddress(orc::ExecutorAddr Address) { this->Address = Address; } /// Returns true if this is a defined addressable, in which case you /// can downcast this to a Block. @@ -133,7 +133,7 @@ private: this->IsAbsolute = IsAbsolute; } - JITTargetAddress Address = 0; + orc::ExecutorAddr Address; uint64_t IsDefined : 1; uint64_t IsAbsolute : 1; @@ -152,7 +152,7 @@ class Block : public Addressable { private: /// Create a zero-fill defined addressable. - Block(Section &Parent, JITTargetAddress Size, JITTargetAddress Address, + Block(Section &Parent, orc::ExecutorAddrDiff Size, orc::ExecutorAddr Address, uint64_t Alignment, uint64_t AlignmentOffset) : Addressable(Address, true), Parent(&Parent), Size(Size) { assert(isPowerOf2_64(Alignment) && "Alignment must be power of 2"); @@ -168,7 +168,7 @@ private: /// Create a defined addressable for the given content. /// The Content is assumed to be non-writable, and will be copied when /// mutations are required. - Block(Section &Parent, ArrayRef<char> Content, JITTargetAddress Address, + Block(Section &Parent, ArrayRef<char> Content, orc::ExecutorAddr Address, uint64_t Alignment, uint64_t AlignmentOffset) : Addressable(Address, true), Parent(&Parent), Data(Content.data()), Size(Content.size()) { @@ -188,7 +188,7 @@ private: /// The standard way to achieve this is to allocate it on the Graph's /// allocator. Block(Section &Parent, MutableArrayRef<char> Content, - JITTargetAddress Address, uint64_t Alignment, uint64_t AlignmentOffset) + orc::ExecutorAddr Address, uint64_t Alignment, uint64_t AlignmentOffset) : Addressable(Address, true), Parent(&Parent), Data(Content.data()), Size(Content.size()) { assert(isPowerOf2_64(Alignment) && "Alignment must be power of 2"); @@ -328,7 +328,7 @@ public: /// Returns the address of the fixup for the given edge, which is equal to /// this block's address plus the edge's offset. - JITTargetAddress getFixupAddress(const Edge &E) const { + orc::ExecutorAddr getFixupAddress(const Edge &E) const { return getAddress() + E.getOffset(); } @@ -343,12 +343,17 @@ private: std::vector<Edge> Edges; }; -// Align a JITTargetAddress to conform with block alignment requirements. -inline JITTargetAddress alignToBlock(JITTargetAddress Addr, Block &B) { +// Align an address to conform with block alignment requirements. +inline uint64_t alignToBlock(uint64_t Addr, Block &B) { uint64_t Delta = (B.getAlignmentOffset() - Addr) % B.getAlignment(); return Addr + Delta; } +// Align a orc::ExecutorAddr to conform with block alignment requirements. +inline orc::ExecutorAddr alignToBlock(orc::ExecutorAddr Addr, Block &B) { + return orc::ExecutorAddr(alignToBlock(Addr.getValue(), B)); +} + /// Describes symbol linkage. This can be used to make resolve definition /// clashes. enum class Linkage : uint8_t { @@ -391,8 +396,8 @@ class Symbol { friend class LinkGraph; private: - Symbol(Addressable &Base, JITTargetAddress Offset, StringRef Name, - JITTargetAddress Size, Linkage L, Scope S, bool IsLive, + Symbol(Addressable &Base, orc::ExecutorAddrDiff Offset, StringRef Name, + orc::ExecutorAddrDiff Size, Linkage L, Scope S, bool IsLive, bool IsCallable) : Name(Name), Base(&Base), Offset(Offset), Size(Size) { assert(Offset <= MaxOffset && "Offset out of range"); @@ -403,7 +408,8 @@ private: } static Symbol &constructCommon(void *SymStorage, Block &Base, StringRef Name, - JITTargetAddress Size, Scope S, bool IsLive) { + orc::ExecutorAddrDiff Size, Scope S, + bool IsLive) { assert(SymStorage && "Storage cannot be null"); assert(!Name.empty() && "Common symbol name cannot be empty"); assert(Base.isDefined() && @@ -416,7 +422,7 @@ private: } static Symbol &constructExternal(void *SymStorage, Addressable &Base, - StringRef Name, JITTargetAddress Size, + StringRef Name, orc::ExecutorAddrDiff Size, Linkage L) { assert(SymStorage && "Storage cannot be null"); assert(!Base.isDefined() && @@ -428,7 +434,7 @@ private: } static Symbol &constructAbsolute(void *SymStorage, Addressable &Base, - StringRef Name, JITTargetAddress Size, + StringRef Name, orc::ExecutorAddrDiff Size, Linkage L, Scope S, bool IsLive) { assert(SymStorage && "Storage cannot be null"); assert(!Base.isDefined() && @@ -439,8 +445,8 @@ private: } static Symbol &constructAnonDef(void *SymStorage, Block &Base, - JITTargetAddress Offset, - JITTargetAddress Size, bool IsCallable, + orc::ExecutorAddrDiff Offset, + orc::ExecutorAddrDiff Size, bool IsCallable, bool IsLive) { assert(SymStorage && "Storage cannot be null"); assert((Offset + Size) <= Base.getSize() && @@ -452,9 +458,9 @@ private: } static Symbol &constructNamedDef(void *SymStorage, Block &Base, - JITTargetAddress Offset, StringRef Name, - JITTargetAddress Size, Linkage L, Scope S, - bool IsLive, bool IsCallable) { + orc::ExecutorAddrDiff Offset, StringRef Name, + orc::ExecutorAddrDiff Size, Linkage L, + Scope S, bool IsLive, bool IsCallable) { assert(SymStorage && "Storage cannot be null"); assert((Offset + Size) <= Base.getSize() && "Symbol extends past end of block"); @@ -552,16 +558,16 @@ public: } /// Returns the offset for this symbol within the underlying addressable. - JITTargetAddress getOffset() const { return Offset; } + orc::ExecutorAddrDiff getOffset() const { return Offset; } /// Returns the address of this symbol. - JITTargetAddress getAddress() const { return Base->getAddress() + Offset; } + orc::ExecutorAddr getAddress() const { return Base->getAddress() + Offset; } /// Returns the size of this symbol. - JITTargetAddress getSize() const { return Size; } + orc::ExecutorAddrDiff getSize() const { return Size; } /// Set the size of this symbol. - void setSize(JITTargetAddress Size) { + void setSize(orc::ExecutorAddrDiff Size) { assert(Base && "Cannot set size for null Symbol"); assert((Size == 0 || Base->isDefined()) && "Non-zero size can only be set for defined symbols"); @@ -622,7 +628,7 @@ private: void setBlock(Block &B) { Base = &B; } - void setOffset(uint64_t NewOffset) { + void setOffset(orc::ExecutorAddrDiff NewOffset) { assert(NewOffset <= MaxOffset && "Offset out of range"); Offset = NewOffset; } @@ -637,7 +643,7 @@ private: uint64_t S : 2; uint64_t IsLive : 1; uint64_t IsCallable : 1; - JITTargetAddress Size = 0; + orc::ExecutorAddrDiff Size = 0; }; raw_ostream &operator<<(raw_ostream &OS, const Symbol &A); @@ -783,13 +789,17 @@ public: assert((First || !Last) && "Last can not be null if start is non-null"); return !First; } - JITTargetAddress getStart() const { - return First ? First->getAddress() : 0; + orc::ExecutorAddr getStart() const { + return First ? First->getAddress() : orc::ExecutorAddr(); + } + orc::ExecutorAddr getEnd() const { + return Last ? Last->getAddress() + Last->getSize() : orc::ExecutorAddr(); } - JITTargetAddress getEnd() const { - return Last ? Last->getAddress() + Last->getSize() : 0; + orc::ExecutorAddrDiff getSize() const { return getEnd() - getStart(); } + + orc::ExecutorAddrRange getRange() const { + return orc::ExecutorAddrRange(getStart(), getEnd()); } - uint64_t getSize() const { return getEnd() - getStart(); } private: Block *First = nullptr; @@ -995,7 +1005,7 @@ public: /// Create a content block. Block &createContentBlock(Section &Parent, ArrayRef<char> Content, - uint64_t Address, uint64_t Alignment, + orc::ExecutorAddr Address, uint64_t Alignment, uint64_t AlignmentOffset) { return createBlock(Parent, Content, Address, Alignment, AlignmentOffset); } @@ -1003,15 +1013,17 @@ public: /// Create a content block with initially mutable data. Block &createMutableContentBlock(Section &Parent, MutableArrayRef<char> MutableContent, - uint64_t Address, uint64_t Alignment, + orc::ExecutorAddr Address, + uint64_t Alignment, uint64_t AlignmentOffset) { return createBlock(Parent, MutableContent, Address, Alignment, AlignmentOffset); } /// Create a zero-fill block. - Block &createZeroFillBlock(Section &Parent, uint64_t Size, uint64_t Address, - uint64_t Alignment, uint64_t AlignmentOffset) { + Block &createZeroFillBlock(Section &Parent, orc::ExecutorAddrDiff Size, + orc::ExecutorAddr Address, uint64_t Alignment, + uint64_t AlignmentOffset) { return createBlock(Parent, Size, Address, Alignment, AlignmentOffset); } @@ -1061,22 +1073,24 @@ public: /// present during lookup: Externals with strong linkage must be found or /// an error will be emitted. Externals with weak linkage are permitted to /// be undefined, in which case they are assigned a value of 0. - Symbol &addExternalSymbol(StringRef Name, uint64_t Size, Linkage L) { + Symbol &addExternalSymbol(StringRef Name, orc::ExecutorAddrDiff Size, + Linkage L) { assert(llvm::count_if(ExternalSymbols, [&](const Symbol *Sym) { return Sym->getName() == Name; }) == 0 && "Duplicate external symbol"); - auto &Sym = - Symbol::constructExternal(Allocator.Allocate<Symbol>(), - createAddressable(0, false), Name, Size, L); + auto &Sym = Symbol::constructExternal( + Allocator.Allocate<Symbol>(), + createAddressable(orc::ExecutorAddr(), false), Name, Size, L); ExternalSymbols.insert(&Sym); return Sym; } /// Add an absolute symbol. - Symbol &addAbsoluteSymbol(StringRef Name, JITTargetAddress Address, - uint64_t Size, Linkage L, Scope S, bool IsLive) { + Symbol &addAbsoluteSymbol(StringRef Name, orc::ExecutorAddr Address, + orc::ExecutorAddrDiff Size, Linkage L, Scope S, + bool IsLive) { assert(llvm::count_if(AbsoluteSymbols, [&](const Symbol *Sym) { return Sym->getName() == Name; @@ -1091,7 +1105,7 @@ public: /// Convenience method for adding a weak zero-fill symbol. Symbol &addCommonSymbol(StringRef Name, Scope S, Section &Section, - JITTargetAddress Address, uint64_t Size, + orc::ExecutorAddr Address, orc::ExecutorAddrDiff Size, uint64_t Alignment, bool IsLive) { assert(llvm::count_if(defined_symbols(), [&](const Symbol *Sym) { @@ -1107,8 +1121,8 @@ public: } /// Add an anonymous symbol. - Symbol &addAnonymousSymbol(Block &Content, JITTargetAddress Offset, - JITTargetAddress Size, bool IsCallable, + Symbol &addAnonymousSymbol(Block &Content, orc::ExecutorAddrDiff Offset, + orc::ExecutorAddrDiff Size, bool IsCallable, bool IsLive) { auto &Sym = Symbol::constructAnonDef(Allocator.Allocate<Symbol>(), Content, Offset, Size, IsCallable, IsLive); @@ -1117,9 +1131,9 @@ public: } /// Add a named symbol. - Symbol &addDefinedSymbol(Block &Content, JITTargetAddress Offset, - StringRef Name, JITTargetAddress Size, Linkage L, - Scope S, bool IsCallable, bool IsLive) { + Symbol &addDefinedSymbol(Block &Content, orc::ExecutorAddrDiff Offset, + StringRef Name, orc::ExecutorAddrDiff Size, + Linkage L, Scope S, bool IsCallable, bool IsLive) { assert((S == Scope::Local || llvm::count_if(defined_symbols(), [&](const Symbol *Sym) { return Sym->getName() == Name; @@ -1193,7 +1207,7 @@ public: assert(Sym.isDefined() && "Sym is not a defined symbol"); Section &Sec = Sym.getBlock().getSection(); Sec.removeSymbol(Sym); - Sym.makeExternal(createAddressable(0, false)); + Sym.makeExternal(createAddressable(orc::ExecutorAddr(), false)); } ExternalSymbols.insert(&Sym); } @@ -1203,7 +1217,7 @@ public: /// /// Symbol size, linkage, scope, and callability, and liveness will be left /// unchanged. Symbol offset will be reset to 0. - void makeAbsolute(Symbol &Sym, JITTargetAddress Address) { + void makeAbsolute(Symbol &Sym, orc::ExecutorAddr Address) { assert(!Sym.isAbsolute() && "Symbol is already absolute"); if (Sym.isExternal()) { assert(ExternalSymbols.count(&Sym) && @@ -1222,8 +1236,9 @@ public: /// Turn an absolute or external symbol into a defined one by attaching it to /// a block. Symbol must not already be defined. - void makeDefined(Symbol &Sym, Block &Content, JITTargetAddress Offset, - JITTargetAddress Size, Linkage L, Scope S, bool IsLive) { + void makeDefined(Symbol &Sym, Block &Content, orc::ExecutorAddrDiff Offset, + orc::ExecutorAddrDiff Size, Linkage L, Scope S, + bool IsLive) { assert(!Sym.isDefined() && "Sym is already a defined symbol"); if (Sym.isAbsolute()) { assert(AbsoluteSymbols.count(&Sym) && @@ -1255,15 +1270,15 @@ public: /// /// All other symbol attributes are unchanged. void transferDefinedSymbol(Symbol &Sym, Block &DestBlock, - JITTargetAddress NewOffset, - Optional<JITTargetAddress> ExplicitNewSize) { + orc::ExecutorAddrDiff NewOffset, + Optional<orc::ExecutorAddrDiff> ExplicitNewSize) { auto &OldSection = Sym.getBlock().getSection(); Sym.setBlock(DestBlock); Sym.setOffset(NewOffset); if (ExplicitNewSize) Sym.setSize(*ExplicitNewSize); else { - JITTargetAddress RemainingBlockSize = DestBlock.getSize() - NewOffset; + auto RemainingBlockSize = DestBlock.getSize() - NewOffset; if (Sym.getSize() > RemainingBlockSize) Sym.setSize(RemainingBlockSize); } @@ -1377,7 +1392,7 @@ public: /// /// Accessing this object after finalization will result in undefined /// behavior. - JITLinkMemoryManager::AllocActions &allocActions() { return AAs; } + orc::shared::AllocActions &allocActions() { return AAs; } /// Dump the graph. void dump(raw_ostream &OS); @@ -1395,7 +1410,7 @@ private: SectionList Sections; ExternalSymbolSet ExternalSymbols; ExternalSymbolSet AbsoluteSymbols; - JITLinkMemoryManager::AllocActions AAs; + orc::shared::AllocActions AAs; }; inline MutableArrayRef<char> Block::getMutableContent(LinkGraph &G) { @@ -1407,14 +1422,14 @@ inline MutableArrayRef<char> Block::getMutableContent(LinkGraph &G) { /// Enables easy lookup of blocks by addresses. class BlockAddressMap { public: - using AddrToBlockMap = std::map<JITTargetAddress, Block *>; + using AddrToBlockMap = std::map<orc::ExecutorAddr, Block *>; using const_iterator = AddrToBlockMap::const_iterator; /// A block predicate that always adds all blocks. static bool includeAllBlocks(const Block &B) { return true; } /// A block predicate that always includes blocks with non-null addresses. - static bool includeNonNull(const Block &B) { return B.getAddress(); } + static bool includeNonNull(const Block &B) { return !!B.getAddress(); } BlockAddressMap() = default; @@ -1478,7 +1493,7 @@ public: /// Returns the block starting at the given address, or nullptr if no such /// block exists. - Block *getBlockAt(JITTargetAddress Addr) const { + Block *getBlockAt(orc::ExecutorAddr Addr) const { auto I = AddrToBlock.find(Addr); if (I == AddrToBlock.end()) return nullptr; @@ -1487,7 +1502,7 @@ public: /// Returns the block covering the given address, or nullptr if no such block /// exists. - Block *getBlockCovering(JITTargetAddress Addr) const { + Block *getBlockCovering(orc::ExecutorAddr Addr) const { auto I = AddrToBlock.upper_bound(Addr); if (I == AddrToBlock.begin()) return nullptr; @@ -1504,10 +1519,11 @@ private: ExistingBlock.getAddress() + ExistingBlock.getSize(); return make_error<JITLinkError>( "Block at " + - formatv("{0:x16} -- {1:x16}", NewBlock.getAddress(), NewBlockEnd) + + formatv("{0:x16} -- {1:x16}", NewBlock.getAddress().getValue(), + NewBlockEnd.getValue()) + " overlaps " + - formatv("{0:x16} -- {1:x16}", ExistingBlock.getAddress(), - ExistingBlockEnd)); + formatv("{0:x16} -- {1:x16}", ExistingBlock.getAddress().getValue(), + ExistingBlockEnd.getValue())); } AddrToBlockMap AddrToBlock; @@ -1532,7 +1548,7 @@ public: /// Returns the list of symbols that start at the given address, or nullptr if /// no such symbols exist. - const SymbolVector *getSymbolsAt(JITTargetAddress Addr) const { + const SymbolVector *getSymbolsAt(orc::ExecutorAddr Addr) const { auto I = AddrToSymbols.find(Addr); if (I == AddrToSymbols.end()) return nullptr; @@ -1540,7 +1556,7 @@ public: } private: - std::map<JITTargetAddress, SymbolVector> AddrToSymbols; + std::map<orc::ExecutorAddr, SymbolVector> AddrToSymbols; }; /// A function for mutating LinkGraphs. diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h b/llvm/include/llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h index 62c271dfc0b2..abfeafe798e7 100644 --- a/llvm/include/llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h @@ -13,9 +13,12 @@ #ifndef LLVM_EXECUTIONENGINE_JITLINK_JITLINKMEMORYMANAGER_H #define LLVM_EXECUTIONENGINE_JITLINK_JITLINKMEMORYMANAGER_H +#include "llvm/ADT/FunctionExtras.h" +#include "llvm/ADT/SmallVector.h" #include "llvm/ExecutionEngine/JITLink/JITLinkDylib.h" #include "llvm/ExecutionEngine/JITLink/MemoryFlags.h" -#include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/ExecutionEngine/Orc/Shared/AllocationActions.h" +#include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/Error.h" #include "llvm/Support/MSVCErrorWorkarounds.h" @@ -39,46 +42,6 @@ class Section; /// and their implemetations should include any necessary synchronization. class JITLinkMemoryManager { public: - /// Represents a call to a graph-memory-management support function in the - /// executor. - /// - /// Support functions are called as: - /// - /// auto *Result = - /// ((char*(*)(const void*, size_t))FnAddr)( - /// (const void*)CtxAddr, (size_t)CtxSize) - /// - /// A null result is interpreted as success. - /// - /// A non-null result is interpreted as a heap-allocated string containing - /// an error message to report to the allocator (the allocator's - /// executor-side implementation code is responsible for freeing the error - /// string). - struct AllocActionCall { - JITTargetAddress FnAddr = 0; - JITTargetAddress CtxAddr = 0; - JITTargetAddress CtxSize = 0; - }; - - /// A pair of AllocActionCalls, one to be run at finalization time, one to be - /// run at deallocation time. - /// - /// AllocActionCallPairs should be constructed for paired operations (e.g. - /// __register_ehframe and __deregister_ehframe for eh-frame registration). - /// See comments for AllocActions for execution ordering. - /// - /// For unpaired operations one or the other member can be left unused, as - /// AllocationActionCalls with an FnAddr of zero will be skipped. - struct AllocActionCallPair { - AllocActionCall Finalize; - AllocActionCall Dealloc; - }; - - /// A vector of allocation actions to be run for this allocation. - /// - /// Finalize allocations will be run in order at finalize time. Dealloc - /// actions will be run in reverse order at deallocation time. - using AllocActions = std::vector<AllocActionCallPair>; /// Represents a finalized allocation. /// @@ -92,47 +55,49 @@ public: class FinalizedAlloc { friend class JITLinkMemoryManager; - public: - static constexpr JITTargetAddress InvalidAddr = ~JITTargetAddress(0); + static constexpr auto InvalidAddr = ~uint64_t(0); + public: FinalizedAlloc() = default; - explicit FinalizedAlloc(JITTargetAddress A) : A(A) { - assert(A != 0 && "Explicitly creating an invalid allocation?"); + explicit FinalizedAlloc(orc::ExecutorAddr A) : A(A) { + assert(A.getValue() != InvalidAddr && + "Explicitly creating an invalid allocation?"); } FinalizedAlloc(const FinalizedAlloc &) = delete; FinalizedAlloc(FinalizedAlloc &&Other) : A(Other.A) { - Other.A = InvalidAddr; + Other.A.setValue(InvalidAddr); } FinalizedAlloc &operator=(const FinalizedAlloc &) = delete; FinalizedAlloc &operator=(FinalizedAlloc &&Other) { - assert(A == InvalidAddr && + assert(A.getValue() == InvalidAddr && "Cannot overwrite active finalized allocation"); std::swap(A, Other.A); return *this; } ~FinalizedAlloc() { - assert(A == InvalidAddr && "Finalized allocation was not deallocated"); + assert(A.getValue() == InvalidAddr && + "Finalized allocation was not deallocated"); } /// FinalizedAllocs convert to false for default-constructed, and /// true otherwise. Default-constructed allocs need not be deallocated. - explicit operator bool() const { return A != InvalidAddr; } + explicit operator bool() const { return A.getValue() != InvalidAddr; } /// Returns the address associated with this finalized allocation. /// The allocation is unmodified. - JITTargetAddress getAddress() const { return A; } + orc::ExecutorAddr getAddress() const { return A; } /// Returns the address associated with this finalized allocation and /// resets this object to the default state. /// This should only be used by allocators when deallocating memory. - JITTargetAddress release() { - JITTargetAddress Tmp = A; - A = InvalidAddr; + orc::ExecutorAddr release() { + orc::ExecutorAddr Tmp = A; + A.setValue(InvalidAddr); return Tmp; } private: - JITTargetAddress A = InvalidAddr; + orc::ExecutorAddr A{InvalidAddr}; }; /// Represents an allocation which has not been finalized yet. @@ -262,7 +227,7 @@ public: Align Alignment; size_t ContentSize; uint64_t ZeroFillSize; - JITTargetAddress Addr; + orc::ExecutorAddr Addr; char *WorkingMem = nullptr; private: @@ -312,7 +277,7 @@ public: /// Returns a reference to the AllocActions in the graph. /// This convenience function saves callers from having to #include /// LinkGraph.h if all they need are allocation actions. - JITLinkMemoryManager::AllocActions &graphAllocActions(); + orc::shared::AllocActions &graphAllocActions(); private: LinkGraph &G; @@ -340,7 +305,7 @@ public: /// Describes the segment working memory and executor address. struct SegmentInfo { - JITTargetAddress Addr = 0; + orc::ExecutorAddr Addr; MutableArrayRef<char> WorkingMem; }; @@ -413,12 +378,12 @@ private: // There shouldn't need to be a heap alloc for this. struct FinalizedAllocInfo { sys::MemoryBlock StandardSegments; - std::vector<AllocActionCall> DeallocActions; + std::vector<orc::shared::WrapperFunctionCall> DeallocActions; }; - FinalizedAlloc - createFinalizedAlloc(sys::MemoryBlock StandardSegments, - std::vector<AllocActionCall> DeallocActions); + FinalizedAlloc createFinalizedAlloc( + sys::MemoryBlock StandardSegments, + std::vector<orc::shared::WrapperFunctionCall> DeallocActions); uint64_t PageSize; std::mutex FinalizedAllocsMutex; diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/MemoryFlags.h b/llvm/include/llvm/ExecutionEngine/JITLink/MemoryFlags.h index 8fdce93ebc56..e9771319ef06 100644 --- a/llvm/include/llvm/ExecutionEngine/JITLink/MemoryFlags.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/MemoryFlags.h @@ -16,6 +16,7 @@ #include "llvm/ADT/BitmaskEnum.h" #include "llvm/ADT/DenseMapInfo.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/Support/Memory.h" #include "llvm/Support/raw_ostream.h" diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/riscv.h b/llvm/include/llvm/ExecutionEngine/JITLink/riscv.h index b8d08d88c1c9..5abd4cf11dea 100644 --- a/llvm/include/llvm/ExecutionEngine/JITLink/riscv.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/riscv.h @@ -37,6 +37,13 @@ enum EdgeKind_riscv : Edge::Kind { /// R_RISCV_64, + /// Low 12 bits of PC-relative branch pointer value relocation + /// + /// Fixup expression: + /// Fixup <- (Target - Fixup + Addend) & 0xFFF + /// + R_RISCV_BRANCH, + /// High 20 bits of 32-bit pointer value relocation /// /// Fixup expression @@ -72,6 +79,12 @@ enum EdgeKind_riscv : Edge::Kind { /// Fixup <- (Target - Fixup + Addend) R_RISCV_CALL, + /// 32 bits PC relative relocation + /// + /// Fixup expression: + /// Fixup <- (Target - Fixup + Addend) + R_RISCV_32_PCREL, + /// PC relative GOT offset /// /// Fixup expression: @@ -82,8 +95,79 @@ enum EdgeKind_riscv : Edge::Kind { /// /// Fixup expression: /// Fixup <- (Target - Fixup + Addend) - R_RISCV_CALL_PLT + R_RISCV_CALL_PLT, + + /// 64 bits label addition + /// + /// Fixup expression: + /// Fixup <- (Target - *{8}Fixup + Addend) + R_RISCV_ADD64, + + /// 32 bits label addition + /// + /// Fixup expression: + /// Fixup <- (Target - *{4}Fixup + Addend) + R_RISCV_ADD32, + + /// 16 bits label addition + /// + /// Fixup expression + /// Fixup <- (Target - *{2}Fixup + Addend) + R_RISCV_ADD16, + + /// 8 bits label addition + /// + /// Fixup expression + /// Fixup <- (Target - *{1}Fixup + Addend) + R_RISCV_ADD8, + + /// 64 bits label subtraction + /// + /// Fixup expression + /// Fixup <- (Target - *{8}Fixup - Addend) + R_RISCV_SUB64, + + /// 32 bits label subtraction + /// + /// Fixup expression + /// Fixup <- (Target - *{4}Fixup - Addend) + R_RISCV_SUB32, + + /// 16 bits label subtraction + /// + /// Fixup expression + /// Fixup <- (Target - *{2}Fixup - Addend) + R_RISCV_SUB16, + + /// 8 bits label subtraction + /// + /// Fixup expression + /// Fixup <- (Target - *{1}Fixup - Addend) + R_RISCV_SUB8, + /// Local label assignment + /// + /// Fixup expression: + /// Fixup <- (Target + Addend) + R_RISCV_SET6, + + /// Local label assignment + /// + /// Fixup expression: + /// Fixup <- (Target + Addend) + R_RISCV_SET8, + + /// Local label assignment + /// + /// Fixup expression: + /// Fixup <- (Target + Addend) + R_RISCV_SET16, + + /// Local label assignment + /// + /// Fixup expression: + /// Fixup <- (Target + Addend) + R_RISCV_SET32, }; /// Returns a string name for the given riscv edge. For debugging purposes diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/x86_64.h b/llvm/include/llvm/ExecutionEngine/JITLink/x86_64.h index 3130ea381534..4a4e8d15be66 100644 --- a/llvm/include/llvm/ExecutionEngine/JITLink/x86_64.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/x86_64.h @@ -368,18 +368,18 @@ inline Error applyFixup(LinkGraph &G, Block &B, const Edge &E, char *BlockWorkingMem = B.getAlreadyMutableContent().data(); char *FixupPtr = BlockWorkingMem + E.getOffset(); - JITTargetAddress FixupAddress = B.getAddress() + E.getOffset(); + auto FixupAddress = B.getAddress() + E.getOffset(); switch (E.getKind()) { case Pointer64: { - uint64_t Value = E.getTarget().getAddress() + E.getAddend(); + uint64_t Value = E.getTarget().getAddress().getValue() + E.getAddend(); *(ulittle64_t *)FixupPtr = Value; break; } case Pointer32: { - uint64_t Value = E.getTarget().getAddress() + E.getAddend(); + uint64_t Value = E.getTarget().getAddress().getValue() + E.getAddend(); if (LLVM_LIKELY(isInRangeForImmU32(Value))) *(ulittle32_t *)FixupPtr = Value; else @@ -387,7 +387,7 @@ inline Error applyFixup(LinkGraph &G, Block &B, const Edge &E, break; } case Pointer32Signed: { - int64_t Value = E.getTarget().getAddress() + E.getAddend(); + int64_t Value = E.getTarget().getAddress().getValue() + E.getAddend(); if (LLVM_LIKELY(isInRangeForImmS32(Value))) *(little32_t *)FixupPtr = Value; else @@ -483,8 +483,8 @@ extern const char PointerJumpStubContent[6]; inline Symbol &createAnonymousPointer(LinkGraph &G, Section &PointerSection, Symbol *InitialTarget = nullptr, uint64_t InitialAddend = 0) { - auto &B = - G.createContentBlock(PointerSection, NullPointerContent, ~7ULL, 8, 0); + auto &B = G.createContentBlock(PointerSection, NullPointerContent, + orc::ExecutorAddr(~uint64_t(7)), 8, 0); if (InitialTarget) B.addEdge(Pointer64, 0, *InitialTarget, InitialAddend); return G.addAnonymousSymbol(B, 0, 8, false, false); @@ -498,8 +498,8 @@ inline Symbol &createAnonymousPointer(LinkGraph &G, Section &PointerSection, /// address: highest allowable: (~5U) inline Block &createPointerJumpStubBlock(LinkGraph &G, Section &StubSection, Symbol &PointerSymbol) { - auto &B = - G.createContentBlock(StubSection, PointerJumpStubContent, ~5ULL, 1, 0); + auto &B = G.createContentBlock(StubSection, PointerJumpStubContent, + orc::ExecutorAddr(~uint64_t(5)), 1, 0); B.addEdge(Delta32, 2, PointerSymbol, -4); return B; } @@ -552,8 +552,7 @@ public: "Fell through switch, but no new kind to set"); DEBUG_WITH_TYPE("jitlink", { dbgs() << " Fixing " << G.getEdgeKindName(E.getKind()) << " edge at " - << formatv("{0:x}", B->getFixupAddress(E)) << " (" - << formatv("{0:x}", B->getAddress()) << " + " + << B->getFixupAddress(E) << " (" << B->getAddress() << " + " << formatv("{0:x}", E.getOffset()) << ")\n"; }); E.setKind(KindToSet); @@ -586,8 +585,7 @@ public: if (E.getKind() == x86_64::BranchPCRel32 && !E.getTarget().isDefined()) { DEBUG_WITH_TYPE("jitlink", { dbgs() << " Fixing " << G.getEdgeKindName(E.getKind()) << " edge at " - << formatv("{0:x}", B->getFixupAddress(E)) << " (" - << formatv("{0:x}", B->getAddress()) << " + " + << B->getFixupAddress(E) << " (" << B->getAddress() << " + " << formatv("{0:x}", E.getOffset()) << ")\n"; }); // Set the edge kind to Branch32ToPtrJumpStubBypassable to enable it to diff --git a/llvm/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h b/llvm/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h index 30544e8a1748..30af7a628e91 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h @@ -51,13 +51,8 @@ #include <vector> namespace llvm { - -class Value; - namespace orc { -class ExtractingIRMaterializationUnit; - class CompileOnDemandLayer : public IRLayer { friend class PartitioningIRMaterializationUnit; diff --git a/llvm/include/llvm/ExecutionEngine/Orc/CompileUtils.h b/llvm/include/llvm/ExecutionEngine/Orc/CompileUtils.h index c7ba57228ab7..ee53f2383cb0 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/CompileUtils.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/CompileUtils.h @@ -20,7 +20,6 @@ namespace llvm { -class MCContext; class MemoryBuffer; class Module; class ObjectCache; diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Core.h b/llvm/include/llvm/ExecutionEngine/Orc/Core.h index b5f5636800df..d0168f79e3d8 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/Core.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/Core.h @@ -1120,32 +1120,33 @@ public: /// DFS order (based on linkage relationships). Each JITDylib will appear /// only once. /// - /// It is illegal to call this method on a defunct JITDylib and the client - /// is responsible for ensuring that they do not do so. - static std::vector<JITDylibSP> getDFSLinkOrder(ArrayRef<JITDylibSP> JDs); + /// If any JITDylib in the order is defunct then this method will return an + /// error, otherwise returns the order. + static Expected<std::vector<JITDylibSP>> + getDFSLinkOrder(ArrayRef<JITDylibSP> JDs); - /// Returns the given JITDylibs and all of their transitive dependensies in + /// Returns the given JITDylibs and all of their transitive dependencies in /// reverse DFS order (based on linkage relationships). Each JITDylib will /// appear only once. /// - /// It is illegal to call this method on a defunct JITDylib and the client - /// is responsible for ensuring that they do not do so. - static std::vector<JITDylibSP> + /// If any JITDylib in the order is defunct then this method will return an + /// error, otherwise returns the order. + static Expected<std::vector<JITDylibSP>> getReverseDFSLinkOrder(ArrayRef<JITDylibSP> JDs); /// Return this JITDylib and its transitive dependencies in DFS order /// based on linkage relationships. /// - /// It is illegal to call this method on a defunct JITDylib and the client - /// is responsible for ensuring that they do not do so. - std::vector<JITDylibSP> getDFSLinkOrder(); + /// If any JITDylib in the order is defunct then this method will return an + /// error, otherwise returns the order. + Expected<std::vector<JITDylibSP>> getDFSLinkOrder(); /// Rteurn this JITDylib and its transitive dependencies in reverse DFS order /// based on linkage relationships. /// - /// It is illegal to call this method on a defunct JITDylib and the client - /// is responsible for ensuring that they do not do so. - std::vector<JITDylibSP> getReverseDFSLinkOrder(); + /// If any JITDylib in the order is defunct then this method will return an + /// error, otherwise returns the order. + Expected<std::vector<JITDylibSP>> getReverseDFSLinkOrder(); private: using AsynchronousSymbolQuerySet = @@ -1309,6 +1310,10 @@ public: /// __dso_handle). virtual Error setupJITDylib(JITDylib &JD) = 0; + /// This method will be called outside the session lock each time a JITDylib + /// is removed to allow the Platform to remove any JITDylib-specific data. + virtual Error teardownJITDylib(JITDylib &JD) = 0; + /// This method will be called under the ExecutionSession lock each time a /// MaterializationUnit is added to a JITDylib. virtual Error notifyAdding(ResourceTracker &RT, @@ -1327,8 +1332,7 @@ public: const DenseMap<JITDylib *, SymbolLookupSet> &InitSyms); /// Performs an async lookup for the the given symbols in each of the given - /// JITDylibs, calling the given handler with the compound result map once - /// all lookups have completed. + /// JITDylibs, calling the given handler once all lookups have completed. static void lookupInitSymbolsAsync(unique_function<void(Error)> OnComplete, ExecutionSession &ES, diff --git a/llvm/include/llvm/ExecutionEngine/Orc/DebugUtils.h b/llvm/include/llvm/ExecutionEngine/Orc/DebugUtils.h index 4b4472e0ac4d..7eb98dfc741e 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/DebugUtils.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/DebugUtils.h @@ -73,9 +73,6 @@ raw_ostream &operator<<(raw_ostream &OS, /// Rendar a SymbolLookupFlags instance. raw_ostream &operator<<(raw_ostream &OS, const SymbolLookupFlags &LookupFlags); -/// Render a JITDylibLookupFlags instance. -raw_ostream &operator<<(raw_ostream &OS, const LookupKind &K); - /// Render a SymbolLookupSet entry. raw_ostream &operator<<(raw_ostream &OS, const SymbolLookupSet::value_type &KV); diff --git a/llvm/include/llvm/ExecutionEngine/Orc/DebuggerSupportPlugin.h b/llvm/include/llvm/ExecutionEngine/Orc/DebuggerSupportPlugin.h index af092b3287d3..d2bf8330695f 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/DebuggerSupportPlugin.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/DebuggerSupportPlugin.h @@ -10,8 +10,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_EXECUTIONENGINE_ORC_DEBUGGERSUPPORT_H -#define LLVM_EXECUTIONENGINE_ORC_DEBUGGERSUPPORT_H +#ifndef LLVM_EXECUTIONENGINE_ORC_DEBUGGERSUPPORTPLUGIN_H +#define LLVM_EXECUTIONENGINE_ORC_DEBUGGERSUPPORTPLUGIN_H #include "llvm/ExecutionEngine/Orc/Core.h" #include "llvm/ExecutionEngine/Orc/EPCDebugObjectRegistrar.h" @@ -61,4 +61,4 @@ private: } // namespace orc } // namespace llvm -#endif // LLVM_EXECUTIONENGINE_ORC_DEBUGGERSUPPORT_H +#endif // LLVM_EXECUTIONENGINE_ORC_DEBUGGERSUPPORTPLUGIN_H diff --git a/llvm/include/llvm/ExecutionEngine/Orc/ELFNixPlatform.h b/llvm/include/llvm/ExecutionEngine/Orc/ELFNixPlatform.h index 20da3e3b89eb..6b12fe990a8a 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/ELFNixPlatform.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/ELFNixPlatform.h @@ -101,6 +101,7 @@ public: ObjectLinkingLayer &getObjectLinkingLayer() const { return ObjLinkingLayer; } Error setupJITDylib(JITDylib &JD) override; + Error teardownJITDylib(JITDylib &JD) override; Error notifyAdding(ResourceTracker &RT, const MaterializationUnit &MU) override; Error notifyRemoving(ResourceTracker &RT) override; @@ -236,7 +237,7 @@ private: DenseMap<JITDylib *, ELFNixJITDylibInitializers> InitSeqs; std::vector<ELFPerObjectSectionsToRegister> BootstrapPOSRs; - DenseMap<JITTargetAddress, JITDylib *> HandleAddrToJITDylib; + DenseMap<ExecutorAddr, JITDylib *> HandleAddrToJITDylib; DenseMap<JITDylib *, uint64_t> JITDylibToPThreadKey; }; diff --git a/llvm/include/llvm/ExecutionEngine/Orc/EPCEHFrameRegistrar.h b/llvm/include/llvm/ExecutionEngine/Orc/EPCEHFrameRegistrar.h index 6d113a7bdf1a..3de55a11a563 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/EPCEHFrameRegistrar.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/EPCEHFrameRegistrar.h @@ -39,10 +39,8 @@ public: : ES(ES), RegisterEHFrameWrapperFnAddr(RegisterEHFrameWrapperFnAddr), DeregisterEHFrameWrapperFnAddr(DeregisterEHFRameWrapperFnAddr) {} - Error registerEHFrames(JITTargetAddress EHFrameSectionAddr, - size_t EHFrameSectionSize) override; - Error deregisterEHFrames(JITTargetAddress EHFrameSectionAddr, - size_t EHFrameSectionSize) override; + Error registerEHFrames(ExecutorAddrRange EHFrameSection) override; + Error deregisterEHFrames(ExecutorAddrRange EHFrameSection) override; private: ExecutionSession &ES; diff --git a/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.h b/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.h index b9825f17ec17..18656d03e441 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.h @@ -85,7 +85,7 @@ public: ExecutorAddr A; if (!SPSArgList<SPSExecutorAddr>::deserialize(IB, A)) return false; - FA = jitlink::JITLinkMemoryManager::FinalizedAlloc(A.getValue()); + FA = jitlink::JITLinkMemoryManager::FinalizedAlloc(A); return true; } }; diff --git a/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.h b/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.h index b6fdfb92ced3..29d282372b1f 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.h @@ -89,11 +89,6 @@ private: ExecutorAddr RemoteAddr; }; - struct EHFrame { - ExecutorAddr Addr; - uint64_t Size; - }; - // Group of section allocations to be allocated together in the executor. The // RemoteCodeAddr will stand in as the id of the group for deallocation // purposes. @@ -107,7 +102,7 @@ private: ExecutorAddrRange RemoteCode; ExecutorAddrRange RemoteROData; ExecutorAddrRange RemoteRWData; - std::vector<EHFrame> UnfinalizedEHFrames; + std::vector<ExecutorAddrRange> UnfinalizedEHFrames; std::vector<Alloc> CodeAllocs, RODataAllocs, RWDataAllocs; }; diff --git a/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h b/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h index 8e572ea1d0c1..85fa0bb88688 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h @@ -33,7 +33,6 @@ class ConstantArray; class GlobalVariable; class Function; class Module; -class TargetMachine; class Value; namespace orc { diff --git a/llvm/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h b/llvm/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h index f368c92b3702..4f1cde32ff18 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h @@ -19,7 +19,6 @@ #include <memory> namespace llvm { -class Module; namespace orc { /// A layer that applies a transform to emitted modules. diff --git a/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h b/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h index d7b5e2eda6ee..01f3f1b2ab63 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h @@ -97,6 +97,7 @@ public: ObjectLinkingLayer &getObjectLinkingLayer() const { return ObjLinkingLayer; } Error setupJITDylib(JITDylib &JD) override; + Error teardownJITDylib(JITDylib &JD) override; Error notifyAdding(ResourceTracker &RT, const MaterializationUnit &MU) override; Error notifyRemoving(ResourceTracker &RT) override; @@ -239,7 +240,7 @@ private: std::mutex PlatformMutex; DenseMap<JITDylib *, MachOJITDylibInitializers> InitSeqs; - DenseMap<JITTargetAddress, JITDylib *> HeaderAddrToJITDylib; + DenseMap<ExecutorAddr, JITDylib *> HeaderAddrToJITDylib; DenseMap<JITDylib *, uint64_t> JITDylibToPThreadKey; }; diff --git a/llvm/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h b/llvm/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h index 109922a46e26..12505fc97083 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h @@ -38,10 +38,6 @@ class LinkGraph; class Symbol; } // namespace jitlink -namespace object { -class ObjectFile; -} // namespace object - namespace orc { class ObjectLinkingLayerJITLinkContext; @@ -220,17 +216,11 @@ public: ResourceKey SrcKey) override; private: - - struct EHFrameRange { - JITTargetAddress Addr = 0; - size_t Size; - }; - std::mutex EHFramePluginMutex; ExecutionSession &ES; std::unique_ptr<jitlink::EHFrameRegistrar> Registrar; - DenseMap<MaterializationResponsibility *, EHFrameRange> InProcessLinks; - DenseMap<ResourceKey, std::vector<EHFrameRange>> EHFrameRanges; + DenseMap<MaterializationResponsibility *, ExecutorAddrRange> InProcessLinks; + DenseMap<ResourceKey, std::vector<ExecutorAddrRange>> EHFrameRanges; }; } // end namespace orc diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Shared/AllocationActions.h b/llvm/include/llvm/ExecutionEngine/Orc/Shared/AllocationActions.h new file mode 100644 index 000000000000..6469b87c816f --- /dev/null +++ b/llvm/include/llvm/ExecutionEngine/Orc/Shared/AllocationActions.h @@ -0,0 +1,101 @@ +//===- AllocationActions.h -- JITLink allocation support calls -*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Structures for making memory allocation support calls. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_SHARED_ALLOCATIONACTIONS_H +#define LLVM_EXECUTIONENGINE_ORC_SHARED_ALLOCATIONACTIONS_H + +#include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h" +#include "llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h" +#include "llvm/Support/Memory.h" + +#include <vector> + +namespace llvm { +namespace orc { +namespace shared { + +/// A pair of WrapperFunctionCalls, one to be run at finalization time, one to +/// be run at deallocation time. +/// +/// AllocActionCallPairs should be constructed for paired operations (e.g. +/// __register_ehframe and __deregister_ehframe for eh-frame registration). +/// See comments for AllocActions for execution ordering. +/// +/// For unpaired operations one or the other member can be left unused, as +/// AllocationActionCalls with an FnAddr of zero will be skipped. +struct AllocActionCallPair { + WrapperFunctionCall Finalize; + WrapperFunctionCall Dealloc; +}; + +/// A vector of allocation actions to be run for this allocation. +/// +/// Finalize allocations will be run in order at finalize time. Dealloc +/// actions will be run in reverse order at deallocation time. +using AllocActions = std::vector<AllocActionCallPair>; + +/// Returns the number of deallocaton actions in the given AllocActions array. +/// +/// This can be useful if clients want to pre-allocate room for deallocation +/// actions with the rest of their memory. +inline size_t numDeallocActions(const AllocActions &AAs) { + return llvm::count_if( + AAs, [](const AllocActionCallPair &P) { return !!P.Dealloc; }); +} + +/// Run finalize actions. +/// +/// If any finalize action fails then the corresponding dealloc actions will be +/// run in reverse order (not including the deallocation action for the failed +/// finalize action), and the error for the failing action will be returned. +/// +/// If all finalize actions succeed then a vector of deallocation actions will +/// be returned. The dealloc actions should be run by calling +/// runDeallocationActions. If this function succeeds then the AA argument will +/// be cleared before the function returns. +Expected<std::vector<WrapperFunctionCall>> +runFinalizeActions(AllocActions &AAs); + +/// Run deallocation actions. +/// Dealloc actions will be run in reverse order (from last element of DAs to +/// first). +Error runDeallocActions(ArrayRef<WrapperFunctionCall> DAs); + +using SPSAllocActionCallPair = + SPSTuple<SPSWrapperFunctionCall, SPSWrapperFunctionCall>; + +template <> +class SPSSerializationTraits<SPSAllocActionCallPair, + AllocActionCallPair> { + using AL = SPSAllocActionCallPair::AsArgList; + +public: + static size_t size(const AllocActionCallPair &AAP) { + return AL::size(AAP.Finalize, AAP.Dealloc); + } + + static bool serialize(SPSOutputBuffer &OB, + const AllocActionCallPair &AAP) { + return AL::serialize(OB, AAP.Finalize, AAP.Dealloc); + } + + static bool deserialize(SPSInputBuffer &IB, + AllocActionCallPair &AAP) { + return AL::deserialize(IB, AAP.Finalize, AAP.Dealloc); + } +}; + +} // end namespace shared +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_SHARED_ALLOCATIONACTIONS_H diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h b/llvm/include/llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h index 3c0b2b9edd52..dc080cfc79d1 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h @@ -13,7 +13,10 @@ #ifndef LLVM_EXECUTIONENGINE_ORC_SHARED_EXECUTORADDRESS_H #define LLVM_EXECUTIONENGINE_ORC_SHARED_EXECUTORADDRESS_H +#include "llvm/ADT/DenseMapInfo.h" #include "llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h" +#include "llvm/Support/FormatVariadic.h" +#include "llvm/Support/raw_ostream.h" #include <cassert> #include <type_traits> @@ -21,17 +24,7 @@ namespace llvm { namespace orc { -/// Represents the difference between two addresses in the executor process. -class ExecutorAddrDiff { -public: - ExecutorAddrDiff() = default; - explicit ExecutorAddrDiff(uint64_t Value) : Value(Value) {} - - uint64_t getValue() const { return Value; } - -private: - int64_t Value = 0; -}; +using ExecutorAddrDiff = uint64_t; /// Represents an address in the executor process. class ExecutorAddr { @@ -39,7 +32,7 @@ public: ExecutorAddr() = default; /// Create an ExecutorAddr from the given value. - explicit ExecutorAddr(uint64_t Addr) : Addr(Addr) {} + explicit constexpr ExecutorAddr(uint64_t Addr) : Addr(Addr) {} /// Create an ExecutorAddr from the given pointer. /// Warning: This should only be used when JITing in-process. @@ -98,13 +91,13 @@ public: ExecutorAddr operator++(int) { return ExecutorAddr(Addr++); } ExecutorAddr operator--(int) { return ExecutorAddr(Addr--); } - ExecutorAddr &operator+=(const ExecutorAddrDiff Delta) { - Addr += Delta.getValue(); + ExecutorAddr &operator+=(const ExecutorAddrDiff &Delta) { + Addr += Delta; return *this; } - ExecutorAddr &operator-=(const ExecutorAddrDiff Delta) { - Addr -= Delta.getValue(); + ExecutorAddr &operator-=(const ExecutorAddrDiff &Delta) { + Addr -= Delta; return *this; } @@ -121,13 +114,25 @@ inline ExecutorAddrDiff operator-(const ExecutorAddr &LHS, /// Adding an offset and an address yields an address. inline ExecutorAddr operator+(const ExecutorAddr &LHS, const ExecutorAddrDiff &RHS) { - return ExecutorAddr(LHS.getValue() + RHS.getValue()); + return ExecutorAddr(LHS.getValue() + RHS); } /// Adding an address and an offset yields an address. inline ExecutorAddr operator+(const ExecutorAddrDiff &LHS, const ExecutorAddr &RHS) { - return ExecutorAddr(LHS.getValue() + RHS.getValue()); + return ExecutorAddr(LHS + RHS.getValue()); +} + +/// Subtracting an offset from an address yields an address. +inline ExecutorAddr operator-(const ExecutorAddr &LHS, + const ExecutorAddrDiff &RHS) { + return ExecutorAddr(LHS.getValue() - RHS); +} + +/// Taking the modulus of an address and a diff yields a diff. +inline ExecutorAddrDiff operator%(const ExecutorAddr &LHS, + const ExecutorAddrDiff &RHS) { + return ExecutorAddrDiff(LHS.getValue() % RHS); } /// Represents an address range in the exceutor process. @@ -158,6 +163,14 @@ struct ExecutorAddrRange { ExecutorAddr End; }; +inline raw_ostream &operator<<(raw_ostream &OS, const ExecutorAddr &A) { + return OS << formatv("{0:x}", A.getValue()); +} + +inline raw_ostream &operator<<(raw_ostream &OS, const ExecutorAddrRange &R) { + return OS << formatv("{0:x} -- {1:x}", R.Start.getValue(), R.End.getValue()); +} + namespace shared { class SPSExecutorAddr {}; @@ -208,6 +221,26 @@ using SPSExecutorAddrRangeSequence = SPSSequence<SPSExecutorAddrRange>; } // End namespace shared. } // End namespace orc. + +// Provide DenseMapInfo for ExecutorAddrs. +template <> struct DenseMapInfo<orc::ExecutorAddr> { + static inline orc::ExecutorAddr getEmptyKey() { + return orc::ExecutorAddr(DenseMapInfo<uint64_t>::getEmptyKey()); + } + static inline orc::ExecutorAddr getTombstoneKey() { + return orc::ExecutorAddr(DenseMapInfo<uint64_t>::getTombstoneKey()); + } + + static unsigned getHashValue(const orc::ExecutorAddr &Addr) { + return DenseMapInfo<uint64_t>::getHashValue(Addr.getValue()); + } + + static bool isEqual(const orc::ExecutorAddr &LHS, + const orc::ExecutorAddr &RHS) { + return DenseMapInfo<uint64_t>::isEqual(LHS.getValue(), RHS.getValue()); + } +}; + } // End namespace llvm. #endif // LLVM_EXECUTIONENGINE_ORC_SHARED_EXECUTORADDRESS_H diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h b/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h index 3ef43f33d84c..96166ac20b2e 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h @@ -37,8 +37,8 @@ extern const char *MemoryWriteUInt32sWrapperName; extern const char *MemoryWriteUInt64sWrapperName; extern const char *MemoryWriteBuffersWrapperName; -extern const char *RegisterEHFrameSectionCustomDirectWrapperName; -extern const char *DeregisterEHFrameSectionCustomDirectWrapperName; +extern const char *RegisterEHFrameSectionWrapperName; +extern const char *DeregisterEHFrameSectionWrapperName; extern const char *RunAsMainWrapperName; diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h b/llvm/include/llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h index 9ac13a493e9d..302b60b80fd0 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h @@ -33,6 +33,7 @@ #define LLVM_EXECUTIONENGINE_ORC_SHARED_SIMPLEPACKEDSERIALIZATION_H #include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" #include "llvm/Support/Error.h" @@ -112,12 +113,22 @@ public: static bool serialize(SPSOutputBuffer &OB) { return true; } static bool deserialize(SPSInputBuffer &IB) { return true; } + + static bool serializeToSmallVector(SmallVectorImpl<char> &V) { return true; } + + static bool deserializeFromSmallVector(const SmallVectorImpl<char> &V) { + return true; + } }; // Non-empty list specialization for SPSArgList. template <typename SPSTagT, typename... SPSTagTs> class SPSArgList<SPSTagT, SPSTagTs...> { public: + // FIXME: This typedef is here to enable SPS arg serialization from + // JITLink. It can be removed once JITLink can access SPS directly. + using OutputBuffer = SPSOutputBuffer; + template <typename ArgT, typename... ArgTs> static size_t size(const ArgT &Arg, const ArgTs &...Args) { return SPSSerializationTraits<SPSTagT, ArgT>::size(Arg) + @@ -284,6 +295,40 @@ public: } }; +/// Trivial SmallVectorImpl<T> -> SPSSequence<char> serialization. +template <typename SPSElementTagT, typename T> +class TrivialSPSSequenceSerialization<SPSElementTagT, SmallVectorImpl<T>> { +public: + static constexpr bool available = true; +}; + +/// Trivial SPSSequence<SPSElementTagT> -> SmallVectorImpl<T> deserialization. +template <typename SPSElementTagT, typename T> +class TrivialSPSSequenceDeserialization<SPSElementTagT, SmallVectorImpl<T>> { +public: + static constexpr bool available = true; + + using element_type = typename SmallVectorImpl<T>::value_type; + + static void reserve(SmallVectorImpl<T> &V, uint64_t Size) { V.reserve(Size); } + static bool append(SmallVectorImpl<T> &V, T E) { + V.push_back(std::move(E)); + return true; + } +}; + +/// Trivial SmallVectorImpl<T> -> SPSSequence<char> serialization. +template <typename SPSElementTagT, typename T, unsigned N> +class TrivialSPSSequenceSerialization<SPSElementTagT, SmallVector<T, N>> + : public TrivialSPSSequenceSerialization<SPSElementTagT, + SmallVectorImpl<T>> {}; + +/// Trivial SPSSequence<SPSElementTagT> -> SmallVectorImpl<T> deserialization. +template <typename SPSElementTagT, typename T, unsigned N> +class TrivialSPSSequenceDeserialization<SPSElementTagT, SmallVector<T, N>> + : public TrivialSPSSequenceDeserialization<SPSElementTagT, + SmallVectorImpl<T>> {}; + /// Trivial ArrayRef<T> -> SPSSequence<SPSElementTagT> serialization. template <typename SPSElementTagT, typename T> class TrivialSPSSequenceSerialization<SPSElementTagT, ArrayRef<T>> { diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h b/llvm/include/llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h index 0e8b7e7d345a..d596a89a50b6 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h @@ -17,6 +17,7 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/StringRef.h" #include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/ExecutionEngine/Orc/Shared/AllocationActions.h" #include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h" #include "llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h" #include "llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h" @@ -69,50 +70,6 @@ inline std::string getWireProtectionFlagsStr(WireProtectionFlags WPF) { return Result; } -struct WrapperFunctionCall { - ExecutorAddr Func; - ExecutorAddrRange ArgData; - - WrapperFunctionCall() = default; - WrapperFunctionCall(ExecutorAddr Func, ExecutorAddr ArgData, - ExecutorAddrDiff ArgSize) - : Func(Func), ArgData(ArgData, ArgSize) {} - WrapperFunctionCall(ExecutorAddr Func, ExecutorAddrRange ArgData) - : Func(Func), ArgData(ArgData) {} - - shared::WrapperFunctionResult run() { - using FnTy = - shared::CWrapperFunctionResult(const char *ArgData, size_t ArgSize); - return shared::WrapperFunctionResult( - Func.toPtr<FnTy *>()(ArgData.Start.toPtr<const char *>(), - static_cast<size_t>(ArgData.size().getValue()))); - } - - /// Run call and deserialize result using SPS. - template <typename SPSRetT, typename RetT> Error runWithSPSRet(RetT &RetVal) { - auto WFR = run(); - if (const char *ErrMsg = WFR.getOutOfBandError()) - return make_error<StringError>(ErrMsg, inconvertibleErrorCode()); - shared::SPSInputBuffer IB(WFR.data(), WFR.size()); - if (!shared::SPSSerializationTraits<SPSRetT, RetT>::deserialize(IB, RetVal)) - return make_error<StringError>("Could not deserialize result from " - "serialized wrapper function call", - inconvertibleErrorCode()); - return Error::success(); - } - - /// Overload for SPS functions returning void. - Error runWithSPSRet() { - shared::SPSEmpty E; - return runWithSPSRet<shared::SPSEmpty>(E); - } -}; - -struct AllocationActionsPair { - WrapperFunctionCall Finalize; - WrapperFunctionCall Deallocate; -}; - struct SegFinalizeRequest { WireProtectionFlags Prot; ExecutorAddr Addr; @@ -122,7 +79,7 @@ struct SegFinalizeRequest { struct FinalizeRequest { std::vector<SegFinalizeRequest> Segments; - std::vector<AllocationActionsPair> Actions; + shared::AllocActions Actions; }; template <typename T> struct UIntWrite { @@ -167,17 +124,12 @@ namespace shared { class SPSMemoryProtectionFlags {}; -using SPSWrapperFunctionCall = SPSTuple<SPSExecutorAddr, SPSExecutorAddrRange>; - using SPSSegFinalizeRequest = SPSTuple<SPSMemoryProtectionFlags, SPSExecutorAddr, uint64_t, SPSSequence<char>>; -using SPSAllocationActionsPair = - SPSTuple<SPSWrapperFunctionCall, SPSWrapperFunctionCall>; - using SPSFinalizeRequest = SPSTuple<SPSSequence<SPSSegFinalizeRequest>, - SPSSequence<SPSAllocationActionsPair>>; + SPSSequence<SPSAllocActionCallPair>>; template <typename T> using SPSMemoryAccessUIntWrite = SPSTuple<SPSExecutorAddr, T>; @@ -213,48 +165,6 @@ public: }; template <> -class SPSSerializationTraits<SPSWrapperFunctionCall, - tpctypes::WrapperFunctionCall> { - using AL = SPSWrapperFunctionCall::AsArgList; - -public: - static size_t size(const tpctypes::WrapperFunctionCall &WFC) { - return AL::size(WFC.Func, WFC.ArgData); - } - - static bool serialize(SPSOutputBuffer &OB, - const tpctypes::WrapperFunctionCall &WFC) { - return AL::serialize(OB, WFC.Func, WFC.ArgData); - } - - static bool deserialize(SPSInputBuffer &IB, - tpctypes::WrapperFunctionCall &WFC) { - return AL::deserialize(IB, WFC.Func, WFC.ArgData); - } -}; - -template <> -class SPSSerializationTraits<SPSAllocationActionsPair, - tpctypes::AllocationActionsPair> { - using AL = SPSAllocationActionsPair::AsArgList; - -public: - static size_t size(const tpctypes::AllocationActionsPair &AAP) { - return AL::size(AAP.Finalize, AAP.Deallocate); - } - - static bool serialize(SPSOutputBuffer &OB, - const tpctypes::AllocationActionsPair &AAP) { - return AL::serialize(OB, AAP.Finalize, AAP.Deallocate); - } - - static bool deserialize(SPSInputBuffer &IB, - tpctypes::AllocationActionsPair &AAP) { - return AL::deserialize(IB, AAP.Finalize, AAP.Deallocate); - } -}; - -template <> class SPSSerializationTraits<SPSSegFinalizeRequest, tpctypes::SegFinalizeRequest> { using SFRAL = SPSSegFinalizeRequest::AsArgList; diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h b/llvm/include/llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h index bf841b1f706b..eb3fb084b28b 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h @@ -356,6 +356,15 @@ public: } }; +template <typename SPSRetTagT> +class ResultSerializer<SPSRetTagT, ErrorSuccess> { +public: + static WrapperFunctionResult serialize(ErrorSuccess Err) { + return serializeViaSPSToWrapperFunctionResult<SPSArgList<SPSRetTagT>>( + toSPSSerializable(std::move(Err))); + } +}; + template <typename SPSRetTagT, typename T> class ResultSerializer<SPSRetTagT, Expected<T>> { public: @@ -609,6 +618,117 @@ makeMethodWrapperHandler(RetT (ClassT::*Method)(ArgTs...)) { return MethodWrapperHandler<RetT, ClassT, ArgTs...>(Method); } +/// Represents a serialized wrapper function call. +/// Serializing calls themselves allows us to batch them: We can make one +/// "run-wrapper-functions" utility and send it a list of calls to run. +/// +/// The motivating use-case for this API is JITLink allocation actions, where +/// we want to run multiple functions to finalize linked memory without having +/// to make separate IPC calls for each one. +class WrapperFunctionCall { +public: + using ArgDataBufferType = SmallVector<char, 24>; + + /// Create a WrapperFunctionCall using the given SPS serializer to serialize + /// the arguments. + template <typename SPSSerializer, typename... ArgTs> + static Expected<WrapperFunctionCall> Create(ExecutorAddr FnAddr, + const ArgTs &...Args) { + ArgDataBufferType ArgData; + ArgData.resize(SPSSerializer::size(Args...)); + SPSOutputBuffer OB(&ArgData[0], ArgData.size()); + if (SPSSerializer::serialize(OB, Args...)) + return WrapperFunctionCall(FnAddr, std::move(ArgData)); + return make_error<StringError>("Cannot serialize arguments for " + "AllocActionCall", + inconvertibleErrorCode()); + } + + WrapperFunctionCall() = default; + + /// Create a WrapperFunctionCall from a target function and arg buffer. + WrapperFunctionCall(ExecutorAddr FnAddr, ArgDataBufferType ArgData) + : FnAddr(FnAddr), ArgData(std::move(ArgData)) {} + + /// Returns the address to be called. + const ExecutorAddr &getCallee() const { return FnAddr; } + + /// Returns the argument data. + const ArgDataBufferType &getArgData() const { return ArgData; } + + /// WrapperFunctionCalls convert to true if the callee is non-null. + explicit operator bool() const { return !!FnAddr; } + + /// Run call returning raw WrapperFunctionResult. + shared::WrapperFunctionResult run() const { + using FnTy = + shared::CWrapperFunctionResult(const char *ArgData, size_t ArgSize); + return shared::WrapperFunctionResult( + FnAddr.toPtr<FnTy *>()(ArgData.data(), ArgData.size())); + } + + /// Run call and deserialize result using SPS. + template <typename SPSRetT, typename RetT> + std::enable_if_t<!std::is_same<SPSRetT, void>::value, Error> + runWithSPSRet(RetT &RetVal) const { + auto WFR = run(); + if (const char *ErrMsg = WFR.getOutOfBandError()) + return make_error<StringError>(ErrMsg, inconvertibleErrorCode()); + shared::SPSInputBuffer IB(WFR.data(), WFR.size()); + if (!shared::SPSSerializationTraits<SPSRetT, RetT>::deserialize(IB, RetVal)) + return make_error<StringError>("Could not deserialize result from " + "serialized wrapper function call", + inconvertibleErrorCode()); + return Error::success(); + } + + /// Overload for SPS functions returning void. + template <typename SPSRetT> + std::enable_if_t<std::is_same<SPSRetT, void>::value, Error> + runWithSPSRet() const { + shared::SPSEmpty E; + return runWithSPSRet<shared::SPSEmpty>(E); + } + + /// Run call and deserialize an SPSError result. SPSError returns and + /// deserialization failures are merged into the returned error. + Error runWithSPSRetErrorMerged() const { + detail::SPSSerializableError RetErr; + if (auto Err = runWithSPSRet<SPSError>(RetErr)) + return Err; + return detail::fromSPSSerializable(std::move(RetErr)); + } + +private: + orc::ExecutorAddr FnAddr; + ArgDataBufferType ArgData; +}; + +using SPSWrapperFunctionCall = SPSTuple<SPSExecutorAddr, SPSSequence<char>>; + +template <> +class SPSSerializationTraits<SPSWrapperFunctionCall, WrapperFunctionCall> { +public: + static size_t size(const WrapperFunctionCall &WFC) { + return SPSWrapperFunctionCall::AsArgList::size(WFC.getCallee(), + WFC.getArgData()); + } + + static bool serialize(SPSOutputBuffer &OB, const WrapperFunctionCall &WFC) { + return SPSWrapperFunctionCall::AsArgList::serialize(OB, WFC.getCallee(), + WFC.getArgData()); + } + + static bool deserialize(SPSInputBuffer &IB, WrapperFunctionCall &WFC) { + ExecutorAddr FnAddr; + WrapperFunctionCall::ArgDataBufferType ArgData; + if (!SPSWrapperFunctionCall::AsArgList::deserialize(IB, FnAddr, ArgData)) + return false; + WFC = WrapperFunctionCall(FnAddr, std::move(ArgData)); + return true; + } +}; + } // end namespace shared } // end namespace orc } // end namespace llvm diff --git a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h index 735aa53e41fd..04790afa6e31 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h @@ -33,22 +33,6 @@ Error deregisterEHFrameSection(const void *EHFrameSectionAddr, } // end namespace orc } // end namespace llvm -/// An eh-frame registration utility suitable for use as a support function -/// call. This function expects the direct address and size of the eh-frame -/// section to register as its arguments (it does not treat its arguments as -/// pointers to an SPS-serialized arg buffer). -extern "C" llvm::orc::shared::CWrapperFunctionResult -llvm_orc_registerEHFrameSectionCustomDirectWrapper( - const char *EHFrameSectionAddr, uint64_t Size); - -/// An eh-frame deregistration utility suitable for use as a support function -/// call. This function expects the direct address and size of the eh-frame -/// section to register as its arguments (it does not treat its arguments as -/// pointers to an SPS-serialized arg buffer). -extern "C" llvm::orc::shared::CWrapperFunctionResult -llvm_orc_deregisterEHFrameSectionCustomDirectWrapper( - const char *EHFrameSectionAddr, uint64_t Size); - extern "C" llvm::orc::shared::CWrapperFunctionResult llvm_orc_registerEHFrameSectionWrapper(const char *Data, uint64_t Size); diff --git a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.h b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.h index 6858f6d4db6e..97b333c68b63 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.h @@ -43,7 +43,7 @@ public: private: struct Allocation { size_t Size = 0; - std::vector<tpctypes::WrapperFunctionCall> DeallocationActions; + std::vector<shared::WrapperFunctionCall> DeallocationActions; }; using AllocationsMap = DenseMap<void *, Allocation>; diff --git a/llvm/include/llvm/ExecutionEngine/RuntimeDyldChecker.h b/llvm/include/llvm/ExecutionEngine/RuntimeDyldChecker.h index 153ffef14e6f..37fe44d5fa69 100644 --- a/llvm/include/llvm/ExecutionEngine/RuntimeDyldChecker.h +++ b/llvm/include/llvm/ExecutionEngine/RuntimeDyldChecker.h @@ -127,7 +127,7 @@ public: JITTargetAddress getTargetAddress() const { return TargetAddress; } private: - const char *ContentPtr = 0; + const char *ContentPtr = nullptr; uint64_t Size = 0; JITTargetAddress TargetAddress = 0; }; diff --git a/llvm/include/llvm/FileCheck/FileCheck.h b/llvm/include/llvm/FileCheck/FileCheck.h index 6ed75e14ccb6..7a6c98db3029 100644 --- a/llvm/include/llvm/FileCheck/FileCheck.h +++ b/llvm/include/llvm/FileCheck/FileCheck.h @@ -80,8 +80,7 @@ class FileCheckType { std::bitset<FileCheckKindModifier::Size> Modifiers; public: - FileCheckType(FileCheckKind Kind = CheckNone) - : Kind(Kind), Count(1), Modifiers() {} + FileCheckType(FileCheckKind Kind = CheckNone) : Kind(Kind), Count(1) {} FileCheckType(const FileCheckType &) = default; FileCheckType &operator=(const FileCheckType &) = default; diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td index 18d577dff497..c5abb16dd9e5 100644 --- a/llvm/include/llvm/Frontend/OpenMP/OMP.td +++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td @@ -358,6 +358,7 @@ def OMPC_Unknown : Clause<"unknown"> { def OMPC_Link : Clause<"link"> { let flangClass = "OmpObjectList"; } +def OMPC_Indirect : Clause<"indirect"> {} def OMPC_Inbranch : Clause<"inbranch"> {} def OMPC_Notinbranch : Clause<"notinbranch"> {} def OMPC_Filter : Clause<"filter"> { @@ -973,12 +974,14 @@ def OMP_BeginDeclareTarget : Directive<"begin declare target"> { VersionedClause<OMPC_To>, VersionedClause<OMPC_Link>, VersionedClause<OMPC_DeviceType>, + VersionedClause<OMPC_Indirect> ]; } def OMP_DeclareTarget : Directive<"declare target"> { let allowedClauses = [ VersionedClause<OMPC_To>, - VersionedClause<OMPC_Link> + VersionedClause<OMPC_Link>, + VersionedClause<OMPC_Indirect> ]; } def OMP_EndDeclareTarget : Directive<"end declare target"> {} diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPAssume.h b/llvm/include/llvm/Frontend/OpenMP/OMPAssume.h new file mode 100644 index 000000000000..c7462ffe6bc0 --- /dev/null +++ b/llvm/include/llvm/Frontend/OpenMP/OMPAssume.h @@ -0,0 +1,55 @@ +//===- OpenMP/OMPAssume.h --- OpenMP assumption helper functions - C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// \file +/// +/// This file provides helper functions and classes to deal with OpenMP +/// assumptions, e.g., as used by `[begin/end] assumes` and `assume`. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FRONTEND_OPENMP_OMPASSUME_H +#define LLVM_FRONTEND_OPENMP_OMPASSUME_H + +#include "llvm/ADT/StringRef.h" + +namespace llvm { + +namespace omp { + +/// Helper to describe assume clauses. +struct AssumptionClauseMappingInfo { + /// The identifier describing the (beginning of the) clause. + llvm::StringLiteral Identifier; + /// Flag to determine if the identifier is a full name or the start of a name. + bool StartsWith; + /// Flag to determine if a directive lists follows. + bool HasDirectiveList; + /// Flag to determine if an expression follows. + bool HasExpression; +}; + +/// All known assume clauses. +static constexpr AssumptionClauseMappingInfo AssumptionClauseMappings[] = { +#define OMP_ASSUME_CLAUSE(Identifier, StartsWith, HasDirectiveList, \ + HasExpression) \ + {Identifier, StartsWith, HasDirectiveList, HasExpression}, +#include "llvm/Frontend/OpenMP/OMPKinds.def" +}; + +inline std::string getAllAssumeClauseOptions() { + std::string S; + for (const AssumptionClauseMappingInfo &ACMI : AssumptionClauseMappings) + S += (S.empty() ? "'" : "', '") + ACMI.Identifier.str(); + return S + "'"; +} + +} // namespace omp + +} // namespace llvm + +#endif // LLVM_FRONTEND_OPENMP_OMPASSUME_H diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPConstants.h b/llvm/include/llvm/Frontend/OpenMP/OMPConstants.h index d2f9bac16e5a..2178acc90e2c 100644 --- a/llvm/include/llvm/Frontend/OpenMP/OMPConstants.h +++ b/llvm/include/llvm/Frontend/OpenMP/OMPConstants.h @@ -20,14 +20,6 @@ #include "llvm/Frontend/OpenMP/OMP.h.inc" namespace llvm { -class Type; -class Module; -class ArrayType; -class StructType; -class PointerType; -class StringRef; -class FunctionType; - namespace omp { LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE(); @@ -80,33 +72,6 @@ enum class IdentFlag { #define OMP_IDENT_FLAG(Enum, ...) constexpr auto Enum = omp::IdentFlag::Enum; #include "llvm/Frontend/OpenMP/OMPKinds.def" -/// Helper to describe assume clauses. -struct AssumptionClauseMappingInfo { - /// The identifier describing the (beginning of the) clause. - llvm::StringLiteral Identifier; - /// Flag to determine if the identifier is a full name or the start of a name. - bool StartsWith; - /// Flag to determine if a directive lists follows. - bool HasDirectiveList; - /// Flag to determine if an expression follows. - bool HasExpression; -}; - -/// All known assume clauses. -static constexpr AssumptionClauseMappingInfo AssumptionClauseMappings[] = { -#define OMP_ASSUME_CLAUSE(Identifier, StartsWith, HasDirectiveList, \ - HasExpression) \ - {Identifier, StartsWith, HasDirectiveList, HasExpression}, -#include "llvm/Frontend/OpenMP/OMPKinds.def" -}; - -inline std::string getAllAssumeClauseOptions() { - std::string S; - for (const AssumptionClauseMappingInfo &ACMI : AssumptionClauseMappings) - S += (S.empty() ? "'" : "', '") + ACMI.Identifier.str(); - return S + "'"; -} - /// \note This needs to be kept in sync with kmp.h enum sched_type. /// Todo: Update kmp.h to include this file, and remove the enums in kmp.h /// To complete this, more enum values will need to be moved here. @@ -140,6 +105,14 @@ enum OMPTgtExecModeFlags : int8_t { LLVM_MARK_AS_BITMASK_ENUM(/* LargestValue */ OMP_TGT_EXEC_MODE_GENERIC_SPMD) }; +enum class AddressSpace : unsigned { + Generic = 0, + Global = 1, + Shared = 3, + Constant = 4, + Local = 5, +}; + } // end namespace omp } // end namespace llvm diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h index 9976d1961ed1..85dd28ec3159 100644 --- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h +++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h @@ -41,10 +41,7 @@ public: /// Finalize the underlying module, e.g., by outlining regions. /// \param Fn The function to be finalized. If not used, /// all functions are finalized. - /// \param AllowExtractorSinking Flag to include sinking instructions, - /// emitted by CodeExtractor, in the - /// outlined region. Default is false. - void finalize(Function *Fn = nullptr, bool AllowExtractorSinking = false); + void finalize(Function *Fn = nullptr); /// Add attributes known for \p FnID to \p Fn. void addAttributes(omp::RuntimeFunction FnID, Function &Fn); @@ -517,6 +514,12 @@ public: void unrollLoopPartial(DebugLoc DL, CanonicalLoopInfo *Loop, int32_t Factor, CanonicalLoopInfo **UnrolledCLI); + /// Add metadata to simd-ize a loop. + /// + /// \param DL Debug location for instructions added by unrolling. + /// \param Loop The loop to simd-ize. + void applySimd(DebugLoc DL, CanonicalLoopInfo *Loop); + /// Generator for '#omp flush' /// /// \param Loc The location where the flush directive was encountered @@ -663,30 +666,34 @@ public: Function *getOrCreateRuntimeFunctionPtr(omp::RuntimeFunction FnID); /// Return the (LLVM-IR) string describing the source location \p LocStr. - Constant *getOrCreateSrcLocStr(StringRef LocStr); + Constant *getOrCreateSrcLocStr(StringRef LocStr, uint32_t &SrcLocStrSize); /// Return the (LLVM-IR) string describing the default source location. - Constant *getOrCreateDefaultSrcLocStr(); + Constant *getOrCreateDefaultSrcLocStr(uint32_t &SrcLocStrSize); /// Return the (LLVM-IR) string describing the source location identified by /// the arguments. Constant *getOrCreateSrcLocStr(StringRef FunctionName, StringRef FileName, - unsigned Line, unsigned Column); + unsigned Line, unsigned Column, + uint32_t &SrcLocStrSize); /// Return the (LLVM-IR) string describing the DebugLoc \p DL. Use \p F as /// fallback if \p DL does not specify the function name. - Constant *getOrCreateSrcLocStr(DebugLoc DL, Function *F = nullptr); + Constant *getOrCreateSrcLocStr(DebugLoc DL, uint32_t &SrcLocStrSize, + Function *F = nullptr); /// Return the (LLVM-IR) string describing the source location \p Loc. - Constant *getOrCreateSrcLocStr(const LocationDescription &Loc); + Constant *getOrCreateSrcLocStr(const LocationDescription &Loc, + uint32_t &SrcLocStrSize); /// Return an ident_t* encoding the source location \p SrcLocStr and \p Flags. /// TODO: Create a enum class for the Reserve2Flags - Value *getOrCreateIdent(Constant *SrcLocStr, - omp::IdentFlag Flags = omp::IdentFlag(0), - unsigned Reserve2Flags = 0); + Constant *getOrCreateIdent(Constant *SrcLocStr, uint32_t SrcLocStrSize, + omp::IdentFlag Flags = omp::IdentFlag(0), + unsigned Reserve2Flags = 0); - /// Create a global flag \p Namein the module with initial value \p Value. + /// Create a hidden global flag \p Name in the module with initial value \p + /// Value. GlobalValue *createGlobalFlag(unsigned Value, StringRef Name); /// Generate control flow and cleanup for cancellation. @@ -754,7 +761,7 @@ public: StringMap<Constant *> SrcLocStrMap; /// Map to remember existing ident_t*. - DenseMap<std::pair<Constant *, uint64_t>, Value *> IdentMap; + DenseMap<std::pair<Constant *, uint64_t>, Constant *> IdentMap; /// Helper that contains information about regions we need to outline /// during finalization. @@ -762,6 +769,7 @@ public: using PostOutlineCBTy = std::function<void(Function &)>; PostOutlineCBTy PostOutlineCB; BasicBlock *EntryBB, *ExitBB; + SmallVector<Value *, 2> ExcludeArgsFromAggregate; /// Collect all blocks in between EntryBB and ExitBB in both the given /// vector and set. diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def index 08bf5981cdc3..d2b70edd4d87 100644 --- a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def +++ b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def @@ -382,6 +382,8 @@ __OMP_RTL(__kmpc_doacross_wait, false, Void, IdentPtr, Int32, Int64Ptr) __OMP_RTL(__kmpc_doacross_fini, false, Void, IdentPtr, Int32) __OMP_RTL(__kmpc_alloc, false, VoidPtr, /* Int */ Int32, SizeTy, VoidPtr) +__OMP_RTL(__kmpc_aligned_alloc, false, VoidPtr, /* Int */ Int32, SizeTy, SizeTy, + VoidPtr) __OMP_RTL(__kmpc_free, false, Void, /* Int */ Int32, VoidPtr, VoidPtr) __OMP_RTL(__kmpc_init_allocator, false, /* omp_allocator_handle_t */ VoidPtr, @@ -905,6 +907,8 @@ __OMP_RTL_ATTRS(__kmpc_free_shared, DeviceAllocAttrs, AttributeSet(), ParamAttrs(NoCaptureAttrs)) __OMP_RTL_ATTRS(__kmpc_alloc, DefaultAttrs, ReturnPtrAttrs, ParamAttrs()) +__OMP_RTL_ATTRS(__kmpc_aligned_alloc, DefaultAttrs, ReturnPtrAttrs, + ParamAttrs()) __OMP_RTL_ATTRS(__kmpc_free, AllocAttrs, AttributeSet(), ParamAttrs()) __OMP_RTL_ATTRS(__kmpc_init_allocator, DefaultAttrs, ReturnPtrAttrs, @@ -1130,6 +1134,8 @@ __OMP_TRAIT_PROPERTY(implementation, vendor, gnu) __OMP_TRAIT_PROPERTY(implementation, vendor, ibm) __OMP_TRAIT_PROPERTY(implementation, vendor, intel) __OMP_TRAIT_PROPERTY(implementation, vendor, llvm) +__OMP_TRAIT_PROPERTY(implementation, vendor, nec) +__OMP_TRAIT_PROPERTY(implementation, vendor, nvidia) __OMP_TRAIT_PROPERTY(implementation, vendor, pgi) __OMP_TRAIT_PROPERTY(implementation, vendor, ti) __OMP_TRAIT_PROPERTY(implementation, vendor, unknown) diff --git a/llvm/include/llvm/FuzzMutate/OpDescriptor.h b/llvm/include/llvm/FuzzMutate/OpDescriptor.h index d6c98cd949a2..43c810920766 100644 --- a/llvm/include/llvm/FuzzMutate/OpDescriptor.h +++ b/llvm/include/llvm/FuzzMutate/OpDescriptor.h @@ -146,7 +146,7 @@ static inline SourcePred sizedPtrType() { return false; if (const auto *PtrT = dyn_cast<PointerType>(V->getType())) - return PtrT->getElementType()->isSized(); + return PtrT->getPointerElementType()->isSized(); return false; }; auto Make = [](ArrayRef<Value *>, ArrayRef<Type *> Ts) { diff --git a/llvm/include/llvm/IR/Argument.h b/llvm/include/llvm/IR/Argument.h index 396ab6a9d01d..7cbfa2a7b6ce 100644 --- a/llvm/include/llvm/IR/Argument.h +++ b/llvm/include/llvm/IR/Argument.h @@ -162,7 +162,7 @@ public: /// Remove attributes from an argument. void removeAttr(Attribute::AttrKind Kind); - void removeAttrs(const AttrBuilder &B); + void removeAttrs(const AttributeMask &AM); /// Check if an argument has a given attribute. bool hasAttribute(Attribute::AttrKind Kind) const; diff --git a/llvm/include/llvm/IR/Attributes.h b/llvm/include/llvm/IR/Attributes.h index f64f15bd38ba..5e2cfe6d81ac 100644 --- a/llvm/include/llvm/IR/Attributes.h +++ b/llvm/include/llvm/IR/Attributes.h @@ -27,13 +27,14 @@ #include <bitset> #include <cassert> #include <cstdint> -#include <map> +#include <set> #include <string> #include <utility> namespace llvm { class AttrBuilder; +class AttributeMask; class AttributeImpl; class AttributeListImpl; class AttributeSetNode; @@ -320,7 +321,7 @@ public: /// Remove the specified attributes from this set. Returns a new set because /// attribute sets are immutable. LLVM_NODISCARD AttributeSet - removeAttributes(LLVMContext &C, const AttrBuilder &AttrsToRemove) const; + removeAttributes(LLVMContext &C, const AttributeMask &AttrsToRemove) const; /// Return the number of attributes in this set. unsigned getNumAttributes() const; @@ -580,7 +581,7 @@ public: /// Remove the specified attributes at the specified index from this /// attribute list. Returns a new list because attribute lists are immutable. LLVM_NODISCARD AttributeList removeAttributesAtIndex( - LLVMContext &C, unsigned Index, const AttrBuilder &AttrsToRemove) const; + LLVMContext &C, unsigned Index, const AttributeMask &AttrsToRemove) const; /// Remove all attributes at the specified index from this /// attribute list. Returns a new list because attribute lists are immutable. @@ -604,7 +605,7 @@ public: /// Remove the specified attribute at the function index from this /// attribute list. Returns a new list because attribute lists are immutable. LLVM_NODISCARD AttributeList - removeFnAttributes(LLVMContext &C, const AttrBuilder &AttrsToRemove) const { + removeFnAttributes(LLVMContext &C, const AttributeMask &AttrsToRemove) const { return removeAttributesAtIndex(C, FunctionIndex, AttrsToRemove); } @@ -630,8 +631,8 @@ public: /// Remove the specified attribute at the return value index from this /// attribute list. Returns a new list because attribute lists are immutable. - LLVM_NODISCARD AttributeList - removeRetAttributes(LLVMContext &C, const AttrBuilder &AttrsToRemove) const { + LLVM_NODISCARD AttributeList removeRetAttributes( + LLVMContext &C, const AttributeMask &AttrsToRemove) const { return removeAttributesAtIndex(C, ReturnIndex, AttrsToRemove); } @@ -652,8 +653,9 @@ public: /// Remove the specified attribute at the specified arg index from this /// attribute list. Returns a new list because attribute lists are immutable. - LLVM_NODISCARD AttributeList removeParamAttributes( - LLVMContext &C, unsigned ArgNo, const AttrBuilder &AttrsToRemove) const { + LLVM_NODISCARD AttributeList + removeParamAttributes(LLVMContext &C, unsigned ArgNo, + const AttributeMask &AttrsToRemove) const { return removeAttributesAtIndex(C, ArgNo + FirstArgIndex, AttrsToRemove); } @@ -929,42 +931,88 @@ template <> struct DenseMapInfo<AttributeList, void> { //===----------------------------------------------------------------------===// /// \class +/// This class stores enough information to efficiently remove some attributes +/// from an existing AttrBuilder, AttributeSet or AttributeList. +class AttributeMask { + std::bitset<Attribute::EndAttrKinds> Attrs; + std::set<SmallString<32>, std::less<>> TargetDepAttrs; + +public: + AttributeMask() = default; + AttributeMask(const AttributeMask &) = delete; + AttributeMask(AttributeMask &&) = default; + + AttributeMask(AttributeSet AS) { + for (Attribute A : AS) + addAttribute(A); + } + + /// Add an attribute to the mask. + AttributeMask &addAttribute(Attribute::AttrKind Val) { + assert((unsigned)Val < Attribute::EndAttrKinds && + "Attribute out of range!"); + Attrs[Val] = true; + return *this; + } + + /// Add the Attribute object to the builder. + AttributeMask &addAttribute(Attribute A) { + if (A.isStringAttribute()) + addAttribute(A.getKindAsString()); + else + addAttribute(A.getKindAsEnum()); + return *this; + } + + /// Add the target-dependent attribute to the builder. + AttributeMask &addAttribute(StringRef A) { + TargetDepAttrs.insert(A); + return *this; + } + + /// Return true if the builder has the specified attribute. + bool contains(Attribute::AttrKind A) const { + assert((unsigned)A < Attribute::EndAttrKinds && "Attribute out of range!"); + return Attrs[A]; + } + + /// Return true if the builder has the specified target-dependent + /// attribute. + bool contains(StringRef A) const { return TargetDepAttrs.count(A); } + + /// Return true if the mask contains the specified attribute. + bool contains(Attribute A) const { + if (A.isStringAttribute()) + return contains(A.getKindAsString()); + return contains(A.getKindAsEnum()); + } +}; + +//===----------------------------------------------------------------------===// +/// \class /// This class is used in conjunction with the Attribute::get method to /// create an Attribute object. The object itself is uniquified. The Builder's /// value, however, is not. So this can be used as a quick way to test for /// equality, presence of attributes, etc. class AttrBuilder { - std::bitset<Attribute::EndAttrKinds> Attrs; - std::map<SmallString<32>, SmallString<32>, std::less<>> TargetDepAttrs; - std::array<uint64_t, Attribute::NumIntAttrKinds> IntAttrs = {}; - std::array<Type *, Attribute::NumTypeAttrKinds> TypeAttrs = {}; - - Optional<unsigned> kindToIntIndex(Attribute::AttrKind Kind) const; - Optional<unsigned> kindToTypeIndex(Attribute::AttrKind Kind) const; + LLVMContext &Ctx; + SmallVector<Attribute, 8> Attrs; public: - AttrBuilder() = default; + AttrBuilder(LLVMContext &Ctx) : Ctx(Ctx) {} AttrBuilder(const AttrBuilder &) = delete; AttrBuilder(AttrBuilder &&) = default; - AttrBuilder(const Attribute &A) { + AttrBuilder(LLVMContext &Ctx, const Attribute &A) : Ctx(Ctx) { addAttribute(A); } - AttrBuilder(AttributeList AS, unsigned Idx); - AttrBuilder(AttributeSet AS); + AttrBuilder(LLVMContext &Ctx, AttributeSet AS); void clear(); /// Add an attribute to the builder. - AttrBuilder &addAttribute(Attribute::AttrKind Val) { - assert((unsigned)Val < Attribute::EndAttrKinds && - "Attribute out of range!"); - assert(Attribute::isEnumAttrKind(Val) && - "Adding integer/type attribute without an argument!"); - Attrs[Val] = true; - return *this; - } + AttrBuilder &addAttribute(Attribute::AttrKind Val); /// Add the Attribute object to the builder. AttrBuilder &addAttribute(Attribute A); @@ -975,42 +1023,49 @@ public: /// Remove an attribute from the builder. AttrBuilder &removeAttribute(Attribute::AttrKind Val); - /// Remove the attributes from the builder. - AttrBuilder &removeAttributes(AttributeList A, uint64_t WithoutIndex); - - /// Remove the target-dependent attribute to the builder. + /// Remove the target-dependent attribute from the builder. AttrBuilder &removeAttribute(StringRef A); - /// Add the attributes from the builder. + /// Remove the target-dependent attribute from the builder. + AttrBuilder &removeAttribute(Attribute A) { + if (A.isStringAttribute()) + return removeAttribute(A.getKindAsString()); + else + return removeAttribute(A.getKindAsEnum()); + } + + /// Add the attributes from the builder. Attributes in the passed builder + /// overwrite attributes in this builder if they have the same key. AttrBuilder &merge(const AttrBuilder &B); /// Remove the attributes from the builder. - AttrBuilder &remove(const AttrBuilder &B); + AttrBuilder &remove(const AttributeMask &AM); /// Return true if the builder has any attribute that's in the /// specified builder. - bool overlaps(const AttrBuilder &B) const; + bool overlaps(const AttributeMask &AM) const; /// Return true if the builder has the specified attribute. - bool contains(Attribute::AttrKind A) const { - assert((unsigned)A < Attribute::EndAttrKinds && "Attribute out of range!"); - return Attrs[A]; - } + bool contains(Attribute::AttrKind A) const; /// Return true if the builder has the specified target-dependent /// attribute. bool contains(StringRef A) const; /// Return true if the builder has IR-level attributes. - bool hasAttributes() const; - - /// Return true if the builder has any attribute that's in the - /// specified attribute. - bool hasAttributes(AttributeList A, uint64_t Index) const; + bool hasAttributes() const { return !Attrs.empty(); } /// Return true if the builder has an alignment attribute. bool hasAlignmentAttr() const; + /// Return Attribute with the given Kind. The returned attribute will be + /// invalid if the Kind is not present in the builder. + Attribute getAttribute(Attribute::AttrKind Kind) const; + + /// Return Attribute with the given Kind. The returned attribute will be + /// invalid if the Kind is not present in the builder. + Attribute getAttribute(StringRef Kind) const; + /// Return raw (possibly packed/encoded) value of integer attribute or 0 if /// not set. uint64_t getRawIntAttr(Attribute::AttrKind Kind) const; @@ -1136,30 +1191,7 @@ public: /// Attribute.getIntValue(). AttrBuilder &addVScaleRangeAttrFromRawRepr(uint64_t RawVScaleRangeRepr); - /// Return true if the builder contains no target-independent - /// attributes. - bool empty() const { return Attrs.none(); } - - // Iterators for target-dependent attributes. - using td_type = decltype(TargetDepAttrs)::value_type; - using td_iterator = decltype(TargetDepAttrs)::iterator; - using td_const_iterator = decltype(TargetDepAttrs)::const_iterator; - using td_range = iterator_range<td_iterator>; - using td_const_range = iterator_range<td_const_iterator>; - - td_iterator td_begin() { return TargetDepAttrs.begin(); } - td_iterator td_end() { return TargetDepAttrs.end(); } - - td_const_iterator td_begin() const { return TargetDepAttrs.begin(); } - td_const_iterator td_end() const { return TargetDepAttrs.end(); } - - td_range td_attrs() { return td_range(td_begin(), td_end()); } - - td_const_range td_attrs() const { - return td_const_range(td_begin(), td_end()); - } - - bool td_empty() const { return TargetDepAttrs.empty(); } + ArrayRef<Attribute> attrs() const { return Attrs; } bool operator==(const AttrBuilder &B) const; bool operator!=(const AttrBuilder &B) const { return !(*this == B); } @@ -1168,14 +1200,14 @@ public: namespace AttributeFuncs { /// Which attributes cannot be applied to a type. -AttrBuilder typeIncompatible(Type *Ty); +AttributeMask typeIncompatible(Type *Ty); /// Get param/return attributes which imply immediate undefined behavior if an /// invalid value is passed. For example, this includes noundef (where undef /// implies UB), but not nonnull (where null implies poison). It also does not /// include attributes like nocapture, which constrain the function /// implementation rather than the passed value. -AttrBuilder getUBImplyingAttributes(); +AttributeMask getUBImplyingAttributes(); /// \returns Return true if the two functions have compatible target-independent /// attributes for inlining purposes. diff --git a/llvm/include/llvm/IR/Comdat.h b/llvm/include/llvm/IR/Comdat.h index 01a047d36455..1701802e6977 100644 --- a/llvm/include/llvm/IR/Comdat.h +++ b/llvm/include/llvm/IR/Comdat.h @@ -16,10 +16,12 @@ #define LLVM_IR_COMDAT_H #include "llvm-c/Types.h" +#include "llvm/ADT/SmallPtrSet.h" #include "llvm/Support/CBindingWrapping.h" namespace llvm { +class GlobalObject; class raw_ostream; class StringRef; template <typename ValueTy> class StringMapEntry; @@ -46,15 +48,21 @@ public: StringRef getName() const; void print(raw_ostream &OS, bool IsForDebug = false) const; void dump() const; + const SmallPtrSetImpl<GlobalObject *> &getUsers() const { return Users; } private: friend class Module; + friend class GlobalObject; Comdat(); + void addUser(GlobalObject *GO); + void removeUser(GlobalObject *GO); // Points to the map in Module. StringMapEntry<Comdat> *Name = nullptr; SelectionKind SK = Any; + // Globals using this comdat. + SmallPtrSet<GlobalObject *, 2> Users; }; // Create wrappers for C Binding types (see CBindingWrapping.h). diff --git a/llvm/include/llvm/IR/Constant.h b/llvm/include/llvm/IR/Constant.h index c8999b71f3d1..a97372ebbad2 100644 --- a/llvm/include/llvm/IR/Constant.h +++ b/llvm/include/llvm/IR/Constant.h @@ -204,6 +204,12 @@ public: /// Constant::removeDeadConstantUsers, but doesn't remove dead constants. bool hasOneLiveUse() const; + /// Return true if the constant has no live uses. + /// + /// This returns the same result as calling Value::use_empty after + /// Constant::removeDeadConstantUsers, but doesn't remove dead constants. + bool hasZeroLiveUses() const; + const Constant *stripPointerCasts() const { return cast<Constant>(Value::stripPointerCasts()); } @@ -244,6 +250,8 @@ private: /// Determine what potential relocations may be needed by this constant. PossibleRelocationsTy getRelocationInfo() const; + + bool hasNLiveUses(unsigned N) const; }; } // end namespace llvm diff --git a/llvm/include/llvm/IR/ConstantFolder.h b/llvm/include/llvm/IR/ConstantFolder.h index da4a18e3c181..28dc63a5886e 100644 --- a/llvm/include/llvm/IR/ConstantFolder.h +++ b/llvm/include/llvm/IR/ConstantFolder.h @@ -17,10 +17,11 @@ #define LLVM_IR_CONSTANTFOLDER_H #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/IR/Constants.h" +#include "llvm/IR/IRBuilderFolder.h" #include "llvm/IR/InstrTypes.h" #include "llvm/IR/Instruction.h" -#include "llvm/IR/IRBuilderFolder.h" namespace llvm { @@ -32,14 +33,72 @@ public: explicit ConstantFolder() = default; //===--------------------------------------------------------------------===// - // Binary Operators + // Value-based folders. + // + // Return an existing value or a constant if the operation can be simplified. + // Otherwise return nullptr. //===--------------------------------------------------------------------===// - - Constant *CreateAdd(Constant *LHS, Constant *RHS, - bool HasNUW = false, bool HasNSW = false) const override { - return ConstantExpr::getAdd(LHS, RHS, HasNUW, HasNSW); + Value *FoldAdd(Value *LHS, Value *RHS, bool HasNUW = false, + bool HasNSW = false) const override { + auto *LC = dyn_cast<Constant>(LHS); + auto *RC = dyn_cast<Constant>(RHS); + if (LC && RC) + return ConstantExpr::getAdd(LC, RC, HasNUW, HasNSW); + return nullptr; + } + + Value *FoldAnd(Value *LHS, Value *RHS) const override { + auto *LC = dyn_cast<Constant>(LHS); + auto *RC = dyn_cast<Constant>(RHS); + if (LC && RC) + return ConstantExpr::getAnd(LC, RC); + return nullptr; + } + + Value *FoldOr(Value *LHS, Value *RHS) const override { + auto *LC = dyn_cast<Constant>(LHS); + auto *RC = dyn_cast<Constant>(RHS); + if (LC && RC) + return ConstantExpr::getOr(LC, RC); + return nullptr; + } + + Value *FoldICmp(CmpInst::Predicate P, Value *LHS, Value *RHS) const override { + auto *LC = dyn_cast<Constant>(LHS); + auto *RC = dyn_cast<Constant>(RHS); + if (LC && RC) + return ConstantExpr::getCompare(P, LC, RC); + return nullptr; + } + + Value *FoldGEP(Type *Ty, Value *Ptr, ArrayRef<Value *> IdxList, + bool IsInBounds = false) const override { + if (auto *PC = dyn_cast<Constant>(Ptr)) { + // Every index must be constant. + if (any_of(IdxList, [](Value *V) { return !isa<Constant>(V); })) + return nullptr; + + if (IsInBounds) + return ConstantExpr::getInBoundsGetElementPtr(Ty, PC, IdxList); + else + return ConstantExpr::getGetElementPtr(Ty, PC, IdxList); + } + return nullptr; + } + + Value *FoldSelect(Value *C, Value *True, Value *False) const override { + auto *CC = dyn_cast<Constant>(C); + auto *TC = dyn_cast<Constant>(True); + auto *FC = dyn_cast<Constant>(False); + if (CC && TC && FC) + return ConstantExpr::getSelect(CC, TC, FC); + return nullptr; } + //===--------------------------------------------------------------------===// + // Binary Operators + //===--------------------------------------------------------------------===// + Constant *CreateFAdd(Constant *LHS, Constant *RHS) const override { return ConstantExpr::getFAdd(LHS, RHS); } @@ -103,11 +162,7 @@ public: return ConstantExpr::getAShr(LHS, RHS, isExact); } - Constant *CreateAnd(Constant *LHS, Constant *RHS) const override { - return ConstantExpr::getAnd(LHS, RHS); - } - - Constant *CreateOr(Constant *LHS, Constant *RHS) const override { + Constant *CreateOr(Constant *LHS, Constant *RHS) const { return ConstantExpr::getOr(LHS, RHS); } @@ -142,46 +197,6 @@ public: } //===--------------------------------------------------------------------===// - // Memory Instructions - //===--------------------------------------------------------------------===// - - Constant *CreateGetElementPtr(Type *Ty, Constant *C, - ArrayRef<Constant *> IdxList) const override { - return ConstantExpr::getGetElementPtr(Ty, C, IdxList); - } - - Constant *CreateGetElementPtr(Type *Ty, Constant *C, - Constant *Idx) const override { - // This form of the function only exists to avoid ambiguous overload - // warnings about whether to convert Idx to ArrayRef<Constant *> or - // ArrayRef<Value *>. - return ConstantExpr::getGetElementPtr(Ty, C, Idx); - } - - Constant *CreateGetElementPtr(Type *Ty, Constant *C, - ArrayRef<Value *> IdxList) const override { - return ConstantExpr::getGetElementPtr(Ty, C, IdxList); - } - - Constant *CreateInBoundsGetElementPtr( - Type *Ty, Constant *C, ArrayRef<Constant *> IdxList) const override { - return ConstantExpr::getInBoundsGetElementPtr(Ty, C, IdxList); - } - - Constant *CreateInBoundsGetElementPtr(Type *Ty, Constant *C, - Constant *Idx) const override { - // This form of the function only exists to avoid ambiguous overload - // warnings about whether to convert Idx to ArrayRef<Constant *> or - // ArrayRef<Value *>. - return ConstantExpr::getInBoundsGetElementPtr(Ty, C, Idx); - } - - Constant *CreateInBoundsGetElementPtr( - Type *Ty, Constant *C, ArrayRef<Value *> IdxList) const override { - return ConstantExpr::getInBoundsGetElementPtr(Ty, C, IdxList); - } - - //===--------------------------------------------------------------------===// // Cast/Conversion Operators //===--------------------------------------------------------------------===// @@ -236,11 +251,6 @@ public: // Compare Instructions //===--------------------------------------------------------------------===// - Constant *CreateICmp(CmpInst::Predicate P, Constant *LHS, - Constant *RHS) const override { - return ConstantExpr::getCompare(P, LHS, RHS); - } - Constant *CreateFCmp(CmpInst::Predicate P, Constant *LHS, Constant *RHS) const override { return ConstantExpr::getCompare(P, LHS, RHS); @@ -250,11 +260,6 @@ public: // Other Instructions //===--------------------------------------------------------------------===// - Constant *CreateSelect(Constant *C, Constant *True, - Constant *False) const override { - return ConstantExpr::getSelect(C, True, False); - } - Constant *CreateExtractElement(Constant *Vec, Constant *Idx) const override { return ConstantExpr::getExtractElement(Vec, Idx); } diff --git a/llvm/include/llvm/IR/Constants.h b/llvm/include/llvm/IR/Constants.h index 65d453861628..fb884912b318 100644 --- a/llvm/include/llvm/IR/Constants.h +++ b/llvm/include/llvm/IR/Constants.h @@ -1196,13 +1196,6 @@ public: /// and the getIndices() method may be used. bool hasIndices() const; - /// Return true if this is a getelementptr expression and all - /// the index operands are compile-time known integers within the - /// corresponding notional static array extents. Note that this is - /// not equivalant to, a subset of, or a superset of the "inbounds" - /// property. - bool isGEPWithNoNotionalOverIndexing() const; - /// Select constant expr /// /// \param OnlyIfReducedTy see \a getWithOperands() docs. diff --git a/llvm/include/llvm/IR/DIBuilder.h b/llvm/include/llvm/IR/DIBuilder.h index 61c6dd885980..f36c9e620d43 100644 --- a/llvm/include/llvm/IR/DIBuilder.h +++ b/llvm/include/llvm/IR/DIBuilder.h @@ -46,6 +46,7 @@ namespace llvm { Function *DeclareFn; ///< llvm.dbg.declare Function *ValueFn; ///< llvm.dbg.value Function *LabelFn; ///< llvm.dbg.label + Function *AddrFn; ///< llvm.dbg.addr SmallVector<Metadata *, 4> AllEnumTypes; /// Track the RetainTypes, since they can be updated later on. @@ -86,12 +87,25 @@ namespace llvm { Instruction *insertLabel(DILabel *LabelInfo, const DILocation *DL, BasicBlock *InsertBB, Instruction *InsertBefore); + /// Internal helper with common code used by insertDbg{Value,Addr}Intrinsic. + Instruction *insertDbgIntrinsic(llvm::Function *Intrinsic, llvm::Value *Val, + DILocalVariable *VarInfo, + DIExpression *Expr, const DILocation *DL, + BasicBlock *InsertBB, + Instruction *InsertBefore); + /// Internal helper for insertDbgValueIntrinsic. Instruction * insertDbgValueIntrinsic(llvm::Value *Val, DILocalVariable *VarInfo, DIExpression *Expr, const DILocation *DL, BasicBlock *InsertBB, Instruction *InsertBefore); + /// Internal helper for insertDbgAddrIntrinsic. + Instruction * + insertDbgAddrIntrinsic(llvm::Value *Val, DILocalVariable *VarInfo, + DIExpression *Expr, const DILocation *DL, + BasicBlock *InsertBB, Instruction *InsertBefore); + public: /// Construct a builder for a module. /// @@ -698,7 +712,6 @@ namespace llvm { /// variable which has a complex address expression for its address. /// \param Addr An array of complex address operations. DIExpression *createExpression(ArrayRef<uint64_t> Addr = None); - DIExpression *createExpression(ArrayRef<int64_t> Addr); /// Create an expression for a variable that does not have an address, but /// does have a constant value. @@ -930,6 +943,30 @@ namespace llvm { const DILocation *DL, Instruction *InsertBefore); + /// Insert a new llvm.dbg.addr intrinsic call. + /// \param Addr llvm::Value of the address + /// \param VarInfo Variable's debug info descriptor. + /// \param Expr A complex location expression. + /// \param DL Debug info location. + /// \param InsertAtEnd Location for the new intrinsic. + Instruction *insertDbgAddrIntrinsic(llvm::Value *Addr, + DILocalVariable *VarInfo, + DIExpression *Expr, + const DILocation *DL, + BasicBlock *InsertAtEnd); + + /// Insert a new llvm.dbg.addr intrinsic call. + /// \param Addr llvm::Value of the address. + /// \param VarInfo Variable's debug info descriptor. + /// \param Expr A complex location expression. + /// \param DL Debug info location. + /// \param InsertBefore Location for the new intrinsic. + Instruction *insertDbgAddrIntrinsic(llvm::Value *Addr, + DILocalVariable *VarInfo, + DIExpression *Expr, + const DILocation *DL, + Instruction *InsertBefore); + /// Replace the vtable holder in the given type. /// /// If this creates a self reference, it may orphan some unresolved cycles diff --git a/llvm/include/llvm/IR/DebugInfoMetadata.h b/llvm/include/llvm/IR/DebugInfoMetadata.h index c04f07c534af..ba2568042c41 100644 --- a/llvm/include/llvm/IR/DebugInfoMetadata.h +++ b/llvm/include/llvm/IR/DebugInfoMetadata.h @@ -852,42 +852,48 @@ class DIStringType : public DIType { static DIStringType *getImpl(LLVMContext &Context, unsigned Tag, StringRef Name, Metadata *StringLength, - Metadata *StrLenExp, uint64_t SizeInBits, - uint32_t AlignInBits, unsigned Encoding, - StorageType Storage, bool ShouldCreate = true) { + Metadata *StrLenExp, Metadata *StrLocationExp, + uint64_t SizeInBits, uint32_t AlignInBits, + unsigned Encoding, StorageType Storage, + bool ShouldCreate = true) { return getImpl(Context, Tag, getCanonicalMDString(Context, Name), - StringLength, StrLenExp, SizeInBits, AlignInBits, Encoding, - Storage, ShouldCreate); + StringLength, StrLenExp, StrLocationExp, SizeInBits, + AlignInBits, Encoding, Storage, ShouldCreate); } static DIStringType *getImpl(LLVMContext &Context, unsigned Tag, MDString *Name, Metadata *StringLength, - Metadata *StrLenExp, uint64_t SizeInBits, - uint32_t AlignInBits, unsigned Encoding, - StorageType Storage, bool ShouldCreate = true); + Metadata *StrLenExp, Metadata *StrLocationExp, + uint64_t SizeInBits, uint32_t AlignInBits, + unsigned Encoding, StorageType Storage, + bool ShouldCreate = true); TempDIStringType cloneImpl() const { return getTemporary(getContext(), getTag(), getRawName(), getRawStringLength(), getRawStringLengthExp(), - getSizeInBits(), getAlignInBits(), getEncoding()); + getRawStringLocationExp(), getSizeInBits(), + getAlignInBits(), getEncoding()); } public: DEFINE_MDNODE_GET(DIStringType, (unsigned Tag, StringRef Name, uint64_t SizeInBits, uint32_t AlignInBits), - (Tag, Name, nullptr, nullptr, SizeInBits, AlignInBits, 0)) + (Tag, Name, nullptr, nullptr, nullptr, SizeInBits, + AlignInBits, 0)) DEFINE_MDNODE_GET(DIStringType, (unsigned Tag, MDString *Name, Metadata *StringLength, - Metadata *StringLengthExp, uint64_t SizeInBits, - uint32_t AlignInBits, unsigned Encoding), - (Tag, Name, StringLength, StringLengthExp, SizeInBits, - AlignInBits, Encoding)) + Metadata *StringLengthExp, Metadata *StringLocationExp, + uint64_t SizeInBits, uint32_t AlignInBits, + unsigned Encoding), + (Tag, Name, StringLength, StringLengthExp, + StringLocationExp, SizeInBits, AlignInBits, Encoding)) DEFINE_MDNODE_GET(DIStringType, (unsigned Tag, StringRef Name, Metadata *StringLength, - Metadata *StringLengthExp, uint64_t SizeInBits, - uint32_t AlignInBits, unsigned Encoding), - (Tag, Name, StringLength, StringLengthExp, SizeInBits, - AlignInBits, Encoding)) + Metadata *StringLengthExp, Metadata *StringLocationExp, + uint64_t SizeInBits, uint32_t AlignInBits, + unsigned Encoding), + (Tag, Name, StringLength, StringLengthExp, + StringLocationExp, SizeInBits, AlignInBits, Encoding)) TempDIStringType clone() const { return cloneImpl(); } @@ -903,11 +909,17 @@ public: return cast_or_null<DIExpression>(getRawStringLengthExp()); } + DIExpression *getStringLocationExp() const { + return cast_or_null<DIExpression>(getRawStringLocationExp()); + } + unsigned getEncoding() const { return Encoding; } Metadata *getRawStringLength() const { return getOperand(3); } Metadata *getRawStringLengthExp() const { return getOperand(4); } + + Metadata *getRawStringLocationExp() const { return getOperand(5); } }; /// Derived types. diff --git a/llvm/include/llvm/IR/DerivedTypes.h b/llvm/include/llvm/IR/DerivedTypes.h index 8a1b26e699e3..f52ce3cde318 100644 --- a/llvm/include/llvm/IR/DerivedTypes.h +++ b/llvm/include/llvm/IR/DerivedTypes.h @@ -667,9 +667,11 @@ public: unsigned AddressSpace) { if (PT->isOpaque()) return get(PT->getContext(), AddressSpace); - return get(PT->getElementType(), AddressSpace); + return get(PT->PointeeTy, AddressSpace); } + [[deprecated("Pointer element types are deprecated. You can *temporarily* " + "use Type::getPointerElementType() instead")]] Type *getElementType() const { assert(!isOpaque() && "Attempting to get element type of opaque pointer"); return PointeeTy; diff --git a/llvm/include/llvm/IR/Function.h b/llvm/include/llvm/IR/Function.h index 669418eacbb0..90095cd1bc77 100644 --- a/llvm/include/llvm/IR/Function.h +++ b/llvm/include/llvm/IR/Function.h @@ -364,7 +364,7 @@ public: /// Remove function attribute from this function. void removeFnAttr(StringRef Kind); - void removeFnAttrs(const AttrBuilder &Attrs); + void removeFnAttrs(const AttributeMask &Attrs); /// removes the attribute from the return value list of attributes. void removeRetAttr(Attribute::AttrKind Kind); @@ -373,7 +373,7 @@ public: void removeRetAttr(StringRef Kind); /// removes the attributes from the return value list of attributes. - void removeRetAttrs(const AttrBuilder &Attrs); + void removeRetAttrs(const AttributeMask &Attrs); /// removes the attribute from the list of attributes. void removeParamAttr(unsigned ArgNo, Attribute::AttrKind Kind); @@ -382,7 +382,7 @@ public: void removeParamAttr(unsigned ArgNo, StringRef Kind); /// removes the attribute from the list of attributes. - void removeParamAttrs(unsigned ArgNo, const AttrBuilder &Attrs); + void removeParamAttrs(unsigned ArgNo, const AttributeMask &Attrs); /// Return true if the function has the attribute. bool hasFnAttribute(Attribute::AttrKind Kind) const; @@ -509,10 +509,10 @@ public: } /// Determine if the function does not access or only writes memory. - bool doesNotReadMemory() const { + bool onlyWritesMemory() const { return doesNotAccessMemory() || hasFnAttribute(Attribute::WriteOnly); } - void setDoesNotReadMemory() { + void setOnlyWritesMemory() { addFnAttr(Attribute::WriteOnly); } diff --git a/llvm/include/llvm/IR/GetElementPtrTypeIterator.h b/llvm/include/llvm/IR/GetElementPtrTypeIterator.h index ed854e458da2..1fa996229749 100644 --- a/llvm/include/llvm/IR/GetElementPtrTypeIterator.h +++ b/llvm/include/llvm/IR/GetElementPtrTypeIterator.h @@ -32,8 +32,6 @@ class generic_gep_type_iterator { ItTy OpIt; PointerUnion<StructType *, Type *> CurTy; - enum : uint64_t { Unbounded = -1ull }; - uint64_t NumElements = Unbounded; generic_gep_type_iterator() = default; @@ -79,16 +77,11 @@ public: generic_gep_type_iterator &operator++() { // Preincrement Type *Ty = getIndexedType(); - if (auto *ATy = dyn_cast<ArrayType>(Ty)) { + if (auto *ATy = dyn_cast<ArrayType>(Ty)) CurTy = ATy->getElementType(); - NumElements = ATy->getNumElements(); - } else if (auto *VTy = dyn_cast<VectorType>(Ty)) { + else if (auto *VTy = dyn_cast<VectorType>(Ty)) CurTy = VTy->getElementType(); - if (isa<ScalableVectorType>(VTy)) - NumElements = Unbounded; - else - NumElements = cast<FixedVectorType>(VTy)->getNumElements(); - } else + else CurTy = dyn_cast<StructType>(Ty); ++OpIt; return *this; @@ -123,15 +116,6 @@ public: StructType *getStructTypeOrNull() const { return CurTy.dyn_cast<StructType *>(); } - - bool isBoundedSequential() const { - return isSequential() && NumElements != Unbounded; - } - - uint64_t getSequentialNumElements() const { - assert(isBoundedSequential()); - return NumElements; - } }; using gep_type_iterator = generic_gep_type_iterator<>; diff --git a/llvm/include/llvm/IR/GlobalObject.h b/llvm/include/llvm/IR/GlobalObject.h index e15cf718bb10..0bb9fd730059 100644 --- a/llvm/include/llvm/IR/GlobalObject.h +++ b/llvm/include/llvm/IR/GlobalObject.h @@ -22,7 +22,6 @@ namespace llvm { class Comdat; -class MDNode; class Metadata; class GlobalObject : public GlobalValue { @@ -48,6 +47,7 @@ protected: ObjComdat(nullptr) { setGlobalValueSubClassData(0); } + ~GlobalObject(); Comdat *ObjComdat; enum { @@ -122,7 +122,7 @@ public: bool hasComdat() const { return getComdat() != nullptr; } const Comdat *getComdat() const { return ObjComdat; } Comdat *getComdat() { return ObjComdat; } - void setComdat(Comdat *C) { ObjComdat = C; } + void setComdat(Comdat *C); using Value::addMetadata; using Value::clearMetadata; diff --git a/llvm/include/llvm/IR/GlobalVariable.h b/llvm/include/llvm/IR/GlobalVariable.h index 674d49eb9de6..e772964fcc6b 100644 --- a/llvm/include/llvm/IR/GlobalVariable.h +++ b/llvm/include/llvm/IR/GlobalVariable.h @@ -34,7 +34,6 @@ class Constant; class Module; template <typename ValueSubClass> class SymbolTableListTraits; -class DIGlobalVariable; class DIGlobalVariableExpression; class GlobalVariable : public GlobalObject, public ilist_node<GlobalVariable> { diff --git a/llvm/include/llvm/IR/IRBuilder.h b/llvm/include/llvm/IR/IRBuilder.h index bcf52278ccbb..53f517480ca1 100644 --- a/llvm/include/llvm/IR/IRBuilder.h +++ b/llvm/include/llvm/IR/IRBuilder.h @@ -1211,9 +1211,8 @@ private: public: Value *CreateAdd(Value *LHS, Value *RHS, const Twine &Name = "", bool HasNUW = false, bool HasNSW = false) { - if (auto *LC = dyn_cast<Constant>(LHS)) - if (auto *RC = dyn_cast<Constant>(RHS)) - return Insert(Folder.CreateAdd(LC, RC, HasNUW, HasNSW), Name); + if (auto *V = Folder.FoldAdd(LHS, RHS, HasNUW, HasNSW)) + return V; return CreateInsertNUWNSWBinOp(Instruction::Add, LHS, RHS, Name, HasNUW, HasNSW); } @@ -1360,12 +1359,8 @@ public: } Value *CreateAnd(Value *LHS, Value *RHS, const Twine &Name = "") { - if (auto *RC = dyn_cast<Constant>(RHS)) { - if (isa<ConstantInt>(RC) && cast<ConstantInt>(RC)->isMinusOne()) - return LHS; // LHS & -1 -> LHS - if (auto *LC = dyn_cast<Constant>(LHS)) - return Insert(Folder.CreateAnd(LC, RC), Name); - } + if (auto *V = Folder.FoldAnd(LHS, RHS)) + return V; return Insert(BinaryOperator::CreateAnd(LHS, RHS), Name); } @@ -1386,12 +1381,8 @@ public: } Value *CreateOr(Value *LHS, Value *RHS, const Twine &Name = "") { - if (auto *RC = dyn_cast<Constant>(RHS)) { - if (RC->isNullValue()) - return LHS; // LHS | 0 -> LHS - if (auto *LC = dyn_cast<Constant>(LHS)) - return Insert(Folder.CreateOr(LC, RC), Name); - } + if (auto *V = Folder.FoldOr(LHS, RHS)) + return V; return Insert(BinaryOperator::CreateOr(LHS, RHS), Name); } @@ -1571,6 +1562,15 @@ public: Cond2, Name); } + // NOTE: this is sequential, non-commutative, ordered reduction! + Value *CreateLogicalOr(ArrayRef<Value *> Ops) { + assert(!Ops.empty()); + Value *Accum = Ops[0]; + for (unsigned i = 1; i < Ops.size(); i++) + Accum = CreateLogicalOr(Accum, Ops[i]); + return Accum; + } + CallInst *CreateConstrainedFPBinOp( Intrinsic::ID ID, Value *L, Value *R, Instruction *FMFSource = nullptr, const Twine &Name = "", MDNode *FPMathTag = nullptr, @@ -1735,45 +1735,28 @@ public: Value *CreateGEP(Type *Ty, Value *Ptr, ArrayRef<Value *> IdxList, const Twine &Name = "") { - if (auto *PC = dyn_cast<Constant>(Ptr)) { - // Every index must be constant. - size_t i, e; - for (i = 0, e = IdxList.size(); i != e; ++i) - if (!isa<Constant>(IdxList[i])) - break; - if (i == e) - return Insert(Folder.CreateGetElementPtr(Ty, PC, IdxList), Name); - } + if (auto *V = Folder.FoldGEP(Ty, Ptr, IdxList, /*IsInBounds=*/false)) + return V; return Insert(GetElementPtrInst::Create(Ty, Ptr, IdxList), Name); } Value *CreateInBoundsGEP(Type *Ty, Value *Ptr, ArrayRef<Value *> IdxList, const Twine &Name = "") { - if (auto *PC = dyn_cast<Constant>(Ptr)) { - // Every index must be constant. - size_t i, e; - for (i = 0, e = IdxList.size(); i != e; ++i) - if (!isa<Constant>(IdxList[i])) - break; - if (i == e) - return Insert(Folder.CreateInBoundsGetElementPtr(Ty, PC, IdxList), - Name); - } + if (auto *V = Folder.FoldGEP(Ty, Ptr, IdxList, /*IsInBounds=*/true)) + return V; return Insert(GetElementPtrInst::CreateInBounds(Ty, Ptr, IdxList), Name); } Value *CreateGEP(Type *Ty, Value *Ptr, Value *Idx, const Twine &Name = "") { - if (auto *PC = dyn_cast<Constant>(Ptr)) - if (auto *IC = dyn_cast<Constant>(Idx)) - return Insert(Folder.CreateGetElementPtr(Ty, PC, IC), Name); + if (auto *V = Folder.FoldGEP(Ty, Ptr, {Idx}, /*IsInBounds=*/false)) + return V; return Insert(GetElementPtrInst::Create(Ty, Ptr, Idx), Name); } Value *CreateInBoundsGEP(Type *Ty, Value *Ptr, Value *Idx, const Twine &Name = "") { - if (auto *PC = dyn_cast<Constant>(Ptr)) - if (auto *IC = dyn_cast<Constant>(Idx)) - return Insert(Folder.CreateInBoundsGetElementPtr(Ty, PC, IC), Name); + if (auto *V = Folder.FoldGEP(Ty, Ptr, {Idx}, /*IsInBounds=*/true)) + return V; return Insert(GetElementPtrInst::CreateInBounds(Ty, Ptr, Idx), Name); } @@ -1781,8 +1764,8 @@ public: const Twine &Name = "") { Value *Idx = ConstantInt::get(Type::getInt32Ty(Context), Idx0); - if (auto *PC = dyn_cast<Constant>(Ptr)) - return Insert(Folder.CreateGetElementPtr(Ty, PC, Idx), Name); + if (auto *V = Folder.FoldGEP(Ty, Ptr, Idx, /*IsInBounds=*/false)) + return V; return Insert(GetElementPtrInst::Create(Ty, Ptr, Idx), Name); } @@ -1791,8 +1774,8 @@ public: const Twine &Name = "") { Value *Idx = ConstantInt::get(Type::getInt32Ty(Context), Idx0); - if (auto *PC = dyn_cast<Constant>(Ptr)) - return Insert(Folder.CreateInBoundsGetElementPtr(Ty, PC, Idx), Name); + if (auto *V = Folder.FoldGEP(Ty, Ptr, Idx, /*IsInBounds=*/true)) + return V; return Insert(GetElementPtrInst::CreateInBounds(Ty, Ptr, Idx), Name); } @@ -1804,8 +1787,8 @@ public: ConstantInt::get(Type::getInt32Ty(Context), Idx1) }; - if (auto *PC = dyn_cast<Constant>(Ptr)) - return Insert(Folder.CreateGetElementPtr(Ty, PC, Idxs), Name); + if (auto *V = Folder.FoldGEP(Ty, Ptr, Idxs, /*IsInBounds=*/false)) + return V; return Insert(GetElementPtrInst::Create(Ty, Ptr, Idxs), Name); } @@ -1817,8 +1800,8 @@ public: ConstantInt::get(Type::getInt32Ty(Context), Idx1) }; - if (auto *PC = dyn_cast<Constant>(Ptr)) - return Insert(Folder.CreateInBoundsGetElementPtr(Ty, PC, Idxs), Name); + if (auto *V = Folder.FoldGEP(Ty, Ptr, Idxs, /*IsInBounds=*/true)) + return V; return Insert(GetElementPtrInst::CreateInBounds(Ty, Ptr, Idxs), Name); } @@ -1827,8 +1810,8 @@ public: const Twine &Name = "") { Value *Idx = ConstantInt::get(Type::getInt64Ty(Context), Idx0); - if (auto *PC = dyn_cast<Constant>(Ptr)) - return Insert(Folder.CreateGetElementPtr(Ty, PC, Idx), Name); + if (auto *V = Folder.FoldGEP(Ty, Ptr, Idx, /*IsInBounds=*/false)) + return V; return Insert(GetElementPtrInst::Create(Ty, Ptr, Idx), Name); } @@ -1837,8 +1820,8 @@ public: const Twine &Name = "") { Value *Idx = ConstantInt::get(Type::getInt64Ty(Context), Idx0); - if (auto *PC = dyn_cast<Constant>(Ptr)) - return Insert(Folder.CreateInBoundsGetElementPtr(Ty, PC, Idx), Name); + if (auto *V = Folder.FoldGEP(Ty, Ptr, Idx, /*IsInBounds=*/true)) + return V; return Insert(GetElementPtrInst::CreateInBounds(Ty, Ptr, Idx), Name); } @@ -1850,8 +1833,8 @@ public: ConstantInt::get(Type::getInt64Ty(Context), Idx1) }; - if (auto *PC = dyn_cast<Constant>(Ptr)) - return Insert(Folder.CreateGetElementPtr(Ty, PC, Idxs), Name); + if (auto *V = Folder.FoldGEP(Ty, Ptr, Idxs, /*IsInBounds=*/false)) + return V; return Insert(GetElementPtrInst::Create(Ty, Ptr, Idxs), Name); } @@ -1863,8 +1846,8 @@ public: ConstantInt::get(Type::getInt64Ty(Context), Idx1) }; - if (auto *PC = dyn_cast<Constant>(Ptr)) - return Insert(Folder.CreateInBoundsGetElementPtr(Ty, PC, Idxs), Name); + if (auto *V = Folder.FoldGEP(Ty, Ptr, Idxs, /*IsInBounds=*/true)) + return V; return Insert(GetElementPtrInst::CreateInBounds(Ty, Ptr, Idxs), Name); } @@ -2215,9 +2198,8 @@ public: Value *CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name = "") { - if (auto *LC = dyn_cast<Constant>(LHS)) - if (auto *RC = dyn_cast<Constant>(RHS)) - return Insert(Folder.CreateICmp(P, LC, RC), Name); + if (auto *V = Folder.FoldICmp(P, LHS, RHS)) + return V; return Insert(new ICmpInst(P, LHS, RHS), Name); } @@ -2431,7 +2413,8 @@ public: /// This is intended to implement C-style pointer subtraction. As such, the /// pointers must be appropriately aligned for their element types and /// pointing into the same object. - Value *CreatePtrDiff(Value *LHS, Value *RHS, const Twine &Name = ""); + Value *CreatePtrDiff(Type *ElemTy, Value *LHS, Value *RHS, + const Twine &Name = ""); /// Create a launder.invariant.group intrinsic call. If Ptr type is /// different from pointer to i8, it's casted to pointer to i8 in the same diff --git a/llvm/include/llvm/IR/IRBuilderFolder.h b/llvm/include/llvm/IR/IRBuilderFolder.h index e781e8e094af..2827ab553adc 100644 --- a/llvm/include/llvm/IR/IRBuilderFolder.h +++ b/llvm/include/llvm/IR/IRBuilderFolder.h @@ -26,11 +26,30 @@ public: virtual ~IRBuilderFolder(); //===--------------------------------------------------------------------===// + // Value-based folders. + // + // Return an existing value or a constant if the operation can be simplified. + // Otherwise return nullptr. + //===--------------------------------------------------------------------===// + virtual Value *FoldAdd(Value *LHS, Value *RHS, bool HasNUW = false, + bool HasNSW = false) const = 0; + + virtual Value *FoldAnd(Value *LHS, Value *RHS) const = 0; + + virtual Value *FoldOr(Value *LHS, Value *RHS) const = 0; + + virtual Value *FoldICmp(CmpInst::Predicate P, Value *LHS, + Value *RHS) const = 0; + + virtual Value *FoldGEP(Type *Ty, Value *Ptr, ArrayRef<Value *> IdxList, + bool IsInBounds = false) const = 0; + + virtual Value *FoldSelect(Value *C, Value *True, Value *False) const = 0; + + //===--------------------------------------------------------------------===// // Binary Operators //===--------------------------------------------------------------------===// - virtual Value *CreateAdd(Constant *LHS, Constant *RHS, - bool HasNUW = false, bool HasNSW = false) const = 0; virtual Value *CreateFAdd(Constant *LHS, Constant *RHS) const = 0; virtual Value *CreateSub(Constant *LHS, Constant *RHS, bool HasNUW = false, bool HasNSW = false) const = 0; @@ -52,8 +71,6 @@ public: bool isExact = false) const = 0; virtual Value *CreateAShr(Constant *LHS, Constant *RHS, bool isExact = false) const = 0; - virtual Value *CreateAnd(Constant *LHS, Constant *RHS) const = 0; - virtual Value *CreateOr(Constant *LHS, Constant *RHS) const = 0; virtual Value *CreateXor(Constant *LHS, Constant *RHS) const = 0; virtual Value *CreateBinOp(Instruction::BinaryOps Opc, Constant *LHS, Constant *RHS) const = 0; @@ -69,29 +86,6 @@ public: virtual Value *CreateUnOp(Instruction::UnaryOps Opc, Constant *C) const = 0; //===--------------------------------------------------------------------===// - // Memory Instructions - //===--------------------------------------------------------------------===// - - virtual Value *CreateGetElementPtr(Type *Ty, Constant *C, - ArrayRef<Constant *> IdxList) const = 0; - // This form of the function only exists to avoid ambiguous overload - // warnings about whether to convert Idx to ArrayRef<Constant *> or - // ArrayRef<Value *>. - virtual Value *CreateGetElementPtr(Type *Ty, Constant *C, - Constant *Idx) const = 0; - virtual Value *CreateGetElementPtr(Type *Ty, Constant *C, - ArrayRef<Value *> IdxList) const = 0; - virtual Value *CreateInBoundsGetElementPtr( - Type *Ty, Constant *C, ArrayRef<Constant *> IdxList) const = 0; - // This form of the function only exists to avoid ambiguous overload - // warnings about whether to convert Idx to ArrayRef<Constant *> or - // ArrayRef<Value *>. - virtual Value *CreateInBoundsGetElementPtr(Type *Ty, Constant *C, - Constant *Idx) const = 0; - virtual Value *CreateInBoundsGetElementPtr( - Type *Ty, Constant *C, ArrayRef<Value *> IdxList) const = 0; - - //===--------------------------------------------------------------------===// // Cast/Conversion Operators //===--------------------------------------------------------------------===// @@ -114,8 +108,6 @@ public: // Compare Instructions //===--------------------------------------------------------------------===// - virtual Value *CreateICmp(CmpInst::Predicate P, Constant *LHS, - Constant *RHS) const = 0; virtual Value *CreateFCmp(CmpInst::Predicate P, Constant *LHS, Constant *RHS) const = 0; @@ -123,8 +115,6 @@ public: // Other Instructions //===--------------------------------------------------------------------===// - virtual Value *CreateSelect(Constant *C, Constant *True, - Constant *False) const = 0; virtual Value *CreateExtractElement(Constant *Vec, Constant *Idx) const = 0; virtual Value *CreateInsertElement(Constant *Vec, Constant *NewElt, Constant *Idx) const = 0; diff --git a/llvm/include/llvm/IR/InlineAsm.h b/llvm/include/llvm/IR/InlineAsm.h index 1a0767aca142..cf6b7af96980 100644 --- a/llvm/include/llvm/IR/InlineAsm.h +++ b/llvm/include/llvm/IR/InlineAsm.h @@ -171,6 +171,11 @@ public: /// selectAlternative - Point this constraint to the alternative constraint /// indicated by the index. void selectAlternative(unsigned index); + + /// Whether this constraint corresponds to an argument. + bool hasArg() const { + return Type == isInput || (Type == isOutput && isIndirect); + } }; /// ParseConstraints - Split up the constraint string into the specific diff --git a/llvm/include/llvm/IR/InstrTypes.h b/llvm/include/llvm/IR/InstrTypes.h index 143a87f4997d..b3d2a2c8ed9d 100644 --- a/llvm/include/llvm/IR/InstrTypes.h +++ b/llvm/include/llvm/IR/InstrTypes.h @@ -1544,7 +1544,7 @@ public: } /// Removes the attributes from the function - void removeFnAttrs(const AttrBuilder &AttrsToRemove) { + void removeFnAttrs(const AttributeMask &AttrsToRemove) { Attrs = Attrs.removeFnAttributes(getContext(), AttrsToRemove); } @@ -1559,7 +1559,7 @@ public: } /// Removes the attributes from the return value - void removeRetAttrs(const AttrBuilder &AttrsToRemove) { + void removeRetAttrs(const AttributeMask &AttrsToRemove) { Attrs = Attrs.removeRetAttributes(getContext(), AttrsToRemove); } @@ -1576,7 +1576,7 @@ public: } /// Removes the attributes from the given argument - void removeParamAttrs(unsigned ArgNo, const AttrBuilder &AttrsToRemove) { + void removeParamAttrs(unsigned ArgNo, const AttributeMask &AttrsToRemove) { Attrs = Attrs.removeParamAttributes(getContext(), ArgNo, AttrsToRemove); } @@ -1717,13 +1717,19 @@ public: // FIXME: Once this API is no longer duplicated in `CallSite`, rename this to // better indicate that this may return a conservative answer. - bool doesNotReadMemory(unsigned OpNo) const { + bool onlyWritesMemory(unsigned OpNo) const { return dataOperandHasImpliedAttr(OpNo, Attribute::WriteOnly) || dataOperandHasImpliedAttr(OpNo, Attribute::ReadNone); } /// Extract the alignment of the return value. - MaybeAlign getRetAlign() const { return Attrs.getRetAlignment(); } + MaybeAlign getRetAlign() const { + if (auto Align = Attrs.getRetAlignment()) + return Align; + if (const Function *F = getCalledFunction()) + return F->getAttributes().getRetAlignment(); + return None; + } /// Extract the alignment for a call or parameter (0=unknown). MaybeAlign getParamAlign(unsigned ArgNo) const { @@ -1818,16 +1824,16 @@ public: /// Determine if the call does not access or only reads memory. bool onlyReadsMemory() const { - return doesNotAccessMemory() || hasFnAttr(Attribute::ReadOnly); + return hasImpliedFnAttr(Attribute::ReadOnly); } void setOnlyReadsMemory() { addFnAttr(Attribute::ReadOnly); } /// Determine if the call does not access or only writes memory. - bool doesNotReadMemory() const { - return doesNotAccessMemory() || hasFnAttr(Attribute::WriteOnly); + bool onlyWritesMemory() const { + return hasImpliedFnAttr(Attribute::WriteOnly); } - void setDoesNotReadMemory() { addFnAttr(Attribute::WriteOnly); } + void setOnlyWritesMemory() { addFnAttr(Attribute::WriteOnly); } /// Determine if the call can access memmory only using pointers based /// on its arguments. @@ -2087,8 +2093,8 @@ public: /// Is the function attribute S disallowed by some operand bundle on /// this operand bundle user? bool isFnAttrDisallowedByOpBundle(StringRef S) const { - // Operand bundles only possibly disallow readnone, readonly and argmemonly - // attributes. All String attributes are fine. + // Operand bundles only possibly disallow memory access attributes. All + // String attributes are fine. return false; } @@ -2113,6 +2119,9 @@ public: case Attribute::ReadOnly: return hasClobberingOperandBundles(); + + case Attribute::WriteOnly: + return hasReadingOperandBundles(); } llvm_unreachable("switch has a default case!"); @@ -2285,6 +2294,26 @@ private: return hasFnAttrOnCalledFunction(Kind); } + /// A specialized version of hasFnAttrImpl for when the caller wants to + /// know if an attribute's semantics are implied, not whether the attribute + /// is actually present. This distinction only exists when checking whether + /// something is readonly or writeonly since readnone implies both. The case + /// which motivates the specialized code is a callee with readnone, and an + /// operand bundle on the call which disallows readnone but not either + /// readonly or writeonly. + bool hasImpliedFnAttr(Attribute::AttrKind Kind) const { + assert((Kind == Attribute::ReadOnly || Kind == Attribute::WriteOnly) && + "use hasFnAttrImpl instead"); + if (Attrs.hasFnAttr(Kind) || Attrs.hasFnAttr(Attribute::ReadNone)) + return true; + + if (isFnAttrDisallowedByOpBundle(Kind)) + return false; + + return hasFnAttrOnCalledFunction(Kind) || + hasFnAttrOnCalledFunction(Attribute::ReadNone); + } + /// Determine whether the return value has the given attribute. Supports /// Attribute::AttrKind and StringRef as \p AttrKind types. template <typename AttrKind> bool hasRetAttrImpl(AttrKind Kind) const { diff --git a/llvm/include/llvm/IR/IntrinsicInst.h b/llvm/include/llvm/IR/IntrinsicInst.h index 647a912b72f6..f4e571e86493 100644 --- a/llvm/include/llvm/IR/IntrinsicInst.h +++ b/llvm/include/llvm/IR/IntrinsicInst.h @@ -530,8 +530,8 @@ public: Value *getRHS() const { return const_cast<Value *>(getArgOperand(1)); } /// Returns the comparison predicate underlying the intrinsic. - ICmpInst::Predicate getPredicate() const { - switch (getIntrinsicID()) { + static ICmpInst::Predicate getPredicate(Intrinsic::ID ID) { + switch (ID) { case Intrinsic::umin: return ICmpInst::Predicate::ICMP_ULT; case Intrinsic::umax: @@ -545,8 +545,58 @@ public: } } + /// Returns the comparison predicate underlying the intrinsic. + ICmpInst::Predicate getPredicate() const { + return getPredicate(getIntrinsicID()); + } + /// Whether the intrinsic is signed or unsigned. - bool isSigned() const { return ICmpInst::isSigned(getPredicate()); }; + static bool isSigned(Intrinsic::ID ID) { + return ICmpInst::isSigned(getPredicate(ID)); + }; + + /// Whether the intrinsic is signed or unsigned. + bool isSigned() const { return isSigned(getIntrinsicID()); }; + + /// Min/max intrinsics are monotonic, they operate on a fixed-bitwidth values, + /// so there is a certain threshold value, upon reaching which, + /// their value can no longer change. Return said threshold. + static APInt getSaturationPoint(Intrinsic::ID ID, unsigned numBits) { + switch (ID) { + case Intrinsic::umin: + return APInt::getMinValue(numBits); + case Intrinsic::umax: + return APInt::getMaxValue(numBits); + case Intrinsic::smin: + return APInt::getSignedMinValue(numBits); + case Intrinsic::smax: + return APInt::getSignedMaxValue(numBits); + default: + llvm_unreachable("Invalid intrinsic"); + } + } + + /// Min/max intrinsics are monotonic, they operate on a fixed-bitwidth values, + /// so there is a certain threshold value, upon reaching which, + /// their value can no longer change. Return said threshold. + APInt getSaturationPoint(unsigned numBits) const { + return getSaturationPoint(getIntrinsicID(), numBits); + } + + /// Min/max intrinsics are monotonic, they operate on a fixed-bitwidth values, + /// so there is a certain threshold value, upon reaching which, + /// their value can no longer change. Return said threshold. + static Constant *getSaturationPoint(Intrinsic::ID ID, Type *Ty) { + return Constant::getIntegerValue( + Ty, getSaturationPoint(ID, Ty->getScalarSizeInBits())); + } + + /// Min/max intrinsics are monotonic, they operate on a fixed-bitwidth values, + /// so there is a certain threshold value, upon reaching which, + /// their value can no longer change. Return said threshold. + Constant *getSaturationPoint(Type *Ty) const { + return getSaturationPoint(getIntrinsicID(), Ty); + } }; /// This class represents an intrinsic that is based on a binary operation. @@ -1126,36 +1176,37 @@ public: Value *getSrc() const { return const_cast<Value *>(getArgOperand(1)); } }; -/// This represents the llvm.instrprof_increment intrinsic. -class InstrProfIncrementInst : public IntrinsicInst { +/// A base class for all instrprof intrinsics. +class InstrProfInstBase : public IntrinsicInst { public: - static bool classof(const IntrinsicInst *I) { - return I->getIntrinsicID() == Intrinsic::instrprof_increment; - } - static bool classof(const Value *V) { - return isa<IntrinsicInst>(V) && classof(cast<IntrinsicInst>(V)); - } - + // The name of the instrumented function. GlobalVariable *getName() const { return cast<GlobalVariable>( const_cast<Value *>(getArgOperand(0))->stripPointerCasts()); } - + // The hash of the CFG for the instrumented function. ConstantInt *getHash() const { return cast<ConstantInt>(const_cast<Value *>(getArgOperand(1))); } + // The number of counters for the instrumented function. + ConstantInt *getNumCounters() const; + // The index of the counter that this instruction acts on. + ConstantInt *getIndex() const; +}; - ConstantInt *getNumCounters() const { - return cast<ConstantInt>(const_cast<Value *>(getArgOperand(2))); +/// This represents the llvm.instrprof.increment intrinsic. +class InstrProfIncrementInst : public InstrProfInstBase { +public: + static bool classof(const IntrinsicInst *I) { + return I->getIntrinsicID() == Intrinsic::instrprof_increment; } - - ConstantInt *getIndex() const { - return cast<ConstantInt>(const_cast<Value *>(getArgOperand(3))); + static bool classof(const Value *V) { + return isa<IntrinsicInst>(V) && classof(cast<IntrinsicInst>(V)); } - Value *getStep() const; }; +/// This represents the llvm.instrprof.increment.step intrinsic. class InstrProfIncrementInstStep : public InstrProfIncrementInst { public: static bool classof(const IntrinsicInst *I) { @@ -1166,8 +1217,8 @@ public: } }; -/// This represents the llvm.instrprof_value_profile intrinsic. -class InstrProfValueProfileInst : public IntrinsicInst { +/// This represents the llvm.instrprof.value.profile intrinsic. +class InstrProfValueProfileInst : public InstrProfInstBase { public: static bool classof(const IntrinsicInst *I) { return I->getIntrinsicID() == Intrinsic::instrprof_value_profile; @@ -1176,15 +1227,6 @@ public: return isa<IntrinsicInst>(V) && classof(cast<IntrinsicInst>(V)); } - GlobalVariable *getName() const { - return cast<GlobalVariable>( - const_cast<Value *>(getArgOperand(0))->stripPointerCasts()); - } - - ConstantInt *getHash() const { - return cast<ConstantInt>(const_cast<Value *>(getArgOperand(1))); - } - Value *getTargetValue() const { return cast<Value>(const_cast<Value *>(getArgOperand(2))); } diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td index da580de3dbd3..3e40bbf39dd4 100644 --- a/llvm/include/llvm/IR/Intrinsics.td +++ b/llvm/include/llvm/IR/Intrinsics.td @@ -1272,6 +1272,7 @@ def int_coro_end_async def int_coro_frame : Intrinsic<[llvm_ptr_ty], [], [IntrNoMem]>; def int_coro_noop : Intrinsic<[llvm_ptr_ty], [], [IntrNoMem]>; def int_coro_size : Intrinsic<[llvm_anyint_ty], [], [IntrNoMem]>; +def int_coro_align : Intrinsic<[llvm_anyint_ty], [], [IntrNoMem]>; def int_coro_save : Intrinsic<[llvm_token_ty], [llvm_ptr_ty], []>; def int_coro_suspend : Intrinsic<[llvm_i8_ty], [llvm_token_ty, llvm_i1_ty], []>; @@ -1507,6 +1508,12 @@ def int_vp_select : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ], LLVMMatchType<0>, llvm_i32_ty]>; +def int_vp_merge : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ], + [ LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + LLVMMatchType<0>, + LLVMMatchType<0>, + llvm_i32_ty]>; + // Reductions let IntrProperties = [IntrSpeculatable, IntrNoMem, IntrNoSync, IntrWillReturn] in { def int_vp_reduce_fadd : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>], diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td index c586af45f34d..e610c28a5923 100644 --- a/llvm/include/llvm/IR/IntrinsicsAArch64.td +++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td @@ -162,6 +162,10 @@ let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". [LLVMMatchType<0>, llvm_anyint_ty, llvm_i32_ty], [IntrNoMem]>; + class AdvSIMD_3IntArg_Intrinsic + : DefaultAttrsIntrinsic<[llvm_anyint_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], + [IntrNoMem]>; class AdvSIMD_3VectorArg_Intrinsic : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], @@ -258,6 +262,9 @@ let TargetPrefix = "aarch64", IntrProperties = [IntrNoMem] in { def int_aarch64_neon_sqrdmulh_lane : AdvSIMD_2VectorArg_Lane_Intrinsic; def int_aarch64_neon_sqrdmulh_laneq : AdvSIMD_2VectorArg_Lane_Intrinsic; + def int_aarch64_neon_sqrdmlah : AdvSIMD_3IntArg_Intrinsic; + def int_aarch64_neon_sqrdmlsh : AdvSIMD_3IntArg_Intrinsic; + // Vector Polynominal Multiply def int_aarch64_neon_pmul : AdvSIMD_2VectorArg_Intrinsic; diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td index 2f2564702b87..c5d266eb57ec 100644 --- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td +++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td @@ -558,6 +558,9 @@ class AMDGPUSampleVariant<string ucmod, string lcmod, list<AMDGPUArg> extra_addr // {offset} {bias} {z-compare} list<AMDGPUArg> ExtraAddrArgs = extra_addr; + bit Offset = false; + bit Bias = false; + bit ZCompare = false; bit Gradients = false; // Name of the {lod} or {clamp} argument that is appended to the coordinates, @@ -571,6 +574,7 @@ defset list<AMDGPUSampleVariant> AMDGPUSampleVariants = { multiclass AMDGPUSampleHelper_Offset<string ucmod, string lcmod, list<AMDGPUArg> extra_addr> { def NAME#lcmod : AMDGPUSampleVariant<ucmod, lcmod, extra_addr>; + let Offset = true in def NAME#lcmod#_o : AMDGPUSampleVariant< ucmod#"_O", lcmod#"_o", !listconcat([AMDGPUArg<llvm_i32_ty, "offset">], extra_addr)>; } @@ -578,6 +582,7 @@ defset list<AMDGPUSampleVariant> AMDGPUSampleVariants = { multiclass AMDGPUSampleHelper_Compare<string ucmod, string lcmod, list<AMDGPUArg> extra_addr> { defm NAME : AMDGPUSampleHelper_Offset<ucmod, lcmod, extra_addr>; + let ZCompare = true in defm NAME : AMDGPUSampleHelper_Offset< "_C"#ucmod, "_c"#lcmod, !listconcat(extra_addr, [AMDGPUArg<llvm_float_ty, "zcompare">])>; } @@ -591,6 +596,7 @@ defset list<AMDGPUSampleVariant> AMDGPUSampleVariants = { defset list<AMDGPUSampleVariant> AMDGPUSampleVariantsNoGradients = { defm AMDGPUSample : AMDGPUSampleHelper_Clamp<"", "", []>; + let Bias = true in defm AMDGPUSample : AMDGPUSampleHelper_Clamp< "_B", "_b", [AMDGPUArg<llvm_anyfloat_ty, "bias">]>; let LodOrClamp = "lod" in @@ -618,6 +624,9 @@ class AMDGPUDimProfile<string opmod, list<LLVMType> RetTypes = []; list<AMDGPUArg> DataArgs = []; list<AMDGPUArg> ExtraAddrArgs = []; + bit Offset = false; + bit Bias = false; + bit ZCompare = false; bit Gradients = false; string LodClampMip = ""; @@ -652,6 +661,9 @@ class AMDGPUDimProfileCopy<AMDGPUDimProfile base> : AMDGPUDimProfile<base.OpMod, let RetTypes = base.RetTypes; let DataArgs = base.DataArgs; let ExtraAddrArgs = base.ExtraAddrArgs; + let Offset = base.Offset; + let Bias = base.Bias; + let ZCompare = base.ZCompare; let Gradients = base.Gradients; let LodClampMip = base.LodClampMip; } @@ -662,6 +674,9 @@ class AMDGPUDimSampleProfile<string opmod, let IsSample = true; let RetTypes = [llvm_any_ty]; let ExtraAddrArgs = sample.ExtraAddrArgs; + let Offset = sample.Offset; + let Bias = sample.Bias; + let ZCompare = sample.ZCompare; let Gradients = sample.Gradients; let LodClampMip = sample.LodOrClamp; } @@ -702,7 +717,10 @@ class AMDGPUDimGetResInfoProfile<AMDGPUDimProps dim> class AMDGPUImageDimIntrinsicEval<AMDGPUDimProfile P_> { int NumDataArgs = !size(P_.DataArgs); int NumDmaskArgs = !not(P_.IsAtomic); - int NumExtraAddrArgs = !size(P_.ExtraAddrArgs); + int NumOffsetArgs = !if(P_.Offset, 1, 0); + int NumBiasArgs = !if(P_.Bias, 1, 0); + int NumZCompareArgs = !if(P_.ZCompare, 1, 0); + int NumExtraAddrArgs = !add(NumOffsetArgs, NumBiasArgs, NumZCompareArgs); int NumVAddrArgs = !size(P_.AddrArgs); int NumGradientArgs = !if(P_.Gradients, !size(P_.Dim.GradientArgs), 0); int NumCoordArgs = !if(P_.IsSample, !size(P_.Dim.CoordSliceArgs), !size(P_.Dim.CoordSliceIntArgs)); @@ -710,6 +728,9 @@ class AMDGPUImageDimIntrinsicEval<AMDGPUDimProfile P_> { int NumSampArgs = !if(P_.IsSample, 2, 0); int DmaskArgIndex = NumDataArgs; int VAddrArgIndex = !add(DmaskArgIndex, NumDmaskArgs); + int OffsetArgIndex = VAddrArgIndex; + int BiasArgIndex = !add(VAddrArgIndex, NumOffsetArgs); + int ZCompareArgIndex = !add(BiasArgIndex, NumBiasArgs); int GradientArgIndex = !add(VAddrArgIndex, NumExtraAddrArgs); int CoordArgIndex = !add(GradientArgIndex, NumGradientArgs); int LodArgIndex = !add(VAddrArgIndex, NumVAddrArgs, -1); @@ -1514,12 +1535,6 @@ def int_amdgcn_writelane : [IntrNoMem, IntrConvergent, IntrWillReturn] >; -// FIXME: Deprecated. This is equivalent to llvm.fshr -def int_amdgcn_alignbit : Intrinsic<[llvm_i32_ty], - [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], - [IntrNoMem, IntrSpeculatable, IntrWillReturn] ->; - def int_amdgcn_alignbyte : GCCBuiltin<"__builtin_amdgcn_alignbyte">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem, IntrSpeculatable, IntrWillReturn] diff --git a/llvm/include/llvm/IR/IntrinsicsARM.td b/llvm/include/llvm/IR/IntrinsicsARM.td index cf375b9280db..a42484757592 100644 --- a/llvm/include/llvm/IR/IntrinsicsARM.td +++ b/llvm/include/llvm/IR/IntrinsicsARM.td @@ -764,6 +764,9 @@ def int_arm_neon_sha256h: SHA_3Arg_v4i32_Intrinsic; def int_arm_neon_sha256h2: SHA_3Arg_v4i32_Intrinsic; def int_arm_neon_sha256su1: SHA_3Arg_v4i32_Intrinsic; +def int_arm_neon_vqrdmlah : Neon_3Arg_Intrinsic; +def int_arm_neon_vqrdmlsh : Neon_3Arg_Intrinsic; + // Armv8.2-A dot product instructions class Neon_Dot_Intrinsic : Intrinsic<[llvm_anyvector_ty], diff --git a/llvm/include/llvm/IR/IntrinsicsNVVM.td b/llvm/include/llvm/IR/IntrinsicsNVVM.td index 6f55d1ef730e..41b28db56c75 100644 --- a/llvm/include/llvm/IR/IntrinsicsNVVM.td +++ b/llvm/include/llvm/IR/IntrinsicsNVVM.td @@ -1185,6 +1185,36 @@ let TargetPrefix = "nvvm" in { def int_nvvm_f2h_rn : GCCBuiltin<"__nvvm_f2h_rn">, DefaultAttrsIntrinsic<[llvm_i16_ty], [llvm_float_ty], [IntrNoMem, IntrSpeculatable]>; + def int_nvvm_ff2bf16x2_rn : GCCBuiltin<"__nvvm_ff2bf16x2_rn">, + Intrinsic<[llvm_i32_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem]>; + def int_nvvm_ff2bf16x2_rn_relu : GCCBuiltin<"__nvvm_ff2bf16x2_rn_relu">, + Intrinsic<[llvm_i32_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem]>; + def int_nvvm_ff2bf16x2_rz : GCCBuiltin<"__nvvm_ff2bf16x2_rz">, + Intrinsic<[llvm_i32_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem]>; + def int_nvvm_ff2bf16x2_rz_relu : GCCBuiltin<"__nvvm_ff2bf16x2_rz_relu">, + Intrinsic<[llvm_i32_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem]>; + + def int_nvvm_ff2f16x2_rn : GCCBuiltin<"__nvvm_ff2f16x2_rn">, + Intrinsic<[llvm_v2f16_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem]>; + def int_nvvm_ff2f16x2_rn_relu : GCCBuiltin<"__nvvm_ff2f16x2_rn_relu">, + Intrinsic<[llvm_v2f16_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem]>; + def int_nvvm_ff2f16x2_rz : GCCBuiltin<"__nvvm_ff2f16x2_rz">, + Intrinsic<[llvm_v2f16_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem]>; + def int_nvvm_ff2f16x2_rz_relu : GCCBuiltin<"__nvvm_ff2f16x2_rz_relu">, + Intrinsic<[llvm_v2f16_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem]>; + + def int_nvvm_f2bf16_rn : GCCBuiltin<"__nvvm_f2bf16_rn">, + Intrinsic<[llvm_i16_ty], [llvm_float_ty], [IntrNoMem]>; + def int_nvvm_f2bf16_rn_relu : GCCBuiltin<"__nvvm_f2bf16_rn_relu">, + Intrinsic<[llvm_i16_ty], [llvm_float_ty], [IntrNoMem]>; + def int_nvvm_f2bf16_rz : GCCBuiltin<"__nvvm_f2bf16_rz">, + Intrinsic<[llvm_i16_ty], [llvm_float_ty], [IntrNoMem]>; + def int_nvvm_f2bf16_rz_relu : GCCBuiltin<"__nvvm_f2bf16_rz_relu">, + Intrinsic<[llvm_i16_ty], [llvm_float_ty], [IntrNoMem]>; + + def int_nvvm_f2tf32_rna : GCCBuiltin<"__nvvm_f2tf32_rna">, + Intrinsic<[llvm_i32_ty], [llvm_float_ty], [IntrNoMem]>; + // // Bitcast // diff --git a/llvm/include/llvm/IR/IntrinsicsRISCV.td b/llvm/include/llvm/IR/IntrinsicsRISCV.td index 747049b1035b..6780436bd701 100644 --- a/llvm/include/llvm/IR/IntrinsicsRISCV.td +++ b/llvm/include/llvm/IR/IntrinsicsRISCV.td @@ -80,19 +80,28 @@ let TargetPrefix = "riscv" in { : Intrinsic<[llvm_any_ty], [LLVMMatchType<0>, LLVMMatchType<0>], [IntrNoMem, IntrSpeculatable, IntrWillReturn]>; + class BitManipGPRGPRGRIntrinsics + : Intrinsic<[llvm_any_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], + [IntrNoMem, IntrSpeculatable, IntrWillReturn]>; // Zbb def int_riscv_orc_b : BitManipGPRIntrinsics; - // Zbc + // Zbc or Zbkc def int_riscv_clmul : BitManipGPRGPRIntrinsics; def int_riscv_clmulh : BitManipGPRGPRIntrinsics; + + // Zbc def int_riscv_clmulr : BitManipGPRGPRIntrinsics; // Zbe def int_riscv_bcompress : BitManipGPRGPRIntrinsics; def int_riscv_bdecompress : BitManipGPRGPRIntrinsics; + // Zbf + def int_riscv_bfp : BitManipGPRGPRIntrinsics; + // Zbp def int_riscv_grev : BitManipGPRGPRIntrinsics; def int_riscv_gorc : BitManipGPRGPRIntrinsics; @@ -112,17 +121,37 @@ let TargetPrefix = "riscv" in { def int_riscv_crc32c_h : BitManipGPRIntrinsics; def int_riscv_crc32c_w : BitManipGPRIntrinsics; def int_riscv_crc32c_d : BitManipGPRIntrinsics; + + // Zbt + def int_riscv_fsl : BitManipGPRGPRGRIntrinsics; + def int_riscv_fsr : BitManipGPRGPRGRIntrinsics; + + // Zbkb + def int_riscv_brev8 : BitManipGPRIntrinsics; + def int_riscv_zip : BitManipGPRIntrinsics; + def int_riscv_unzip : BitManipGPRIntrinsics; + + // Zbkx + def int_riscv_xperm4 : BitManipGPRGPRIntrinsics; + def int_riscv_xperm8 : BitManipGPRGPRIntrinsics; } // TargetPrefix = "riscv" //===----------------------------------------------------------------------===// // Vectors +// The intrinsic does not have any operand that must be extended. +defvar NoSplatOperand = 0xF; + +// The intrinsic does not have a VL operand. +// (e.g., riscv_vmv_x_s and riscv_vfmv_f_s) +defvar NoVLOperand = 0x1F; + class RISCVVIntrinsic { // These intrinsics may accept illegal integer values in their llvm_any_ty - // operand, so they have to be extended. If set to zero then the intrinsic - // does not have any operand that must be extended. + // operand, so they have to be extended. Intrinsic IntrinsicID = !cast<Intrinsic>(NAME); - bits<4> SplatOperand = 0; + bits<4> SplatOperand = NoSplatOperand; + bits<5> VLOperand = NoVLOperand; } let TargetPrefix = "riscv" in { @@ -141,23 +170,54 @@ let TargetPrefix = "riscv" in { ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>]>; - // For unit stride load + // Versions without side effects: better optimizable and usable if only the + // returned vector length is important. + def int_riscv_vsetvli_opt : Intrinsic<[llvm_anyint_ty], + /* AVL */ [LLVMMatchType<0>, + /* VSEW */ LLVMMatchType<0>, + /* VLMUL */ LLVMMatchType<0>], + [IntrNoMem, + ImmArg<ArgIndex<1>>, + ImmArg<ArgIndex<2>>]>; + def int_riscv_vsetvlimax_opt : Intrinsic<[llvm_anyint_ty], + /* VSEW */ [LLVMMatchType<0>, + /* VLMUL */ LLVMMatchType<0>], + [IntrNoMem, + ImmArg<ArgIndex<0>>, + ImmArg<ArgIndex<1>>]>; + + // For unit stride mask load // Input: (pointer, vl) - class RISCVUSLoad + class RISCVUSMLoad : Intrinsic<[llvm_anyvector_ty], [LLVMPointerType<LLVMMatchType<0>>, llvm_anyint_ty], - [NoCapture<ArgIndex<0>>, IntrReadMem]>, RISCVVIntrinsic; + [NoCapture<ArgIndex<0>>, IntrReadMem]>, RISCVVIntrinsic { + let VLOperand = 1; + } + // For unit stride load + // Input: (passthru, pointer, vl) + class RISCVUSLoad + : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, + LLVMPointerType<LLVMMatchType<0>>, + llvm_anyint_ty], + [NoCapture<ArgIndex<1>>, IntrReadMem]>, RISCVVIntrinsic { + let VLOperand = 2; + } // For unit stride fault-only-first load - // Input: (pointer, vl) + // Input: (passthru, pointer, vl) // Output: (data, vl) // NOTE: We model this with default memory properties since we model writing // VL as a side effect. IntrReadMem, IntrHasSideEffects does not work. class RISCVUSLoadFF : Intrinsic<[llvm_anyvector_ty, llvm_anyint_ty], - [LLVMPointerType<LLVMMatchType<0>>, LLVMMatchType<1>], - [NoCapture<ArgIndex<0>>]>, - RISCVVIntrinsic; + [LLVMMatchType<0>, + LLVMPointerType<LLVMMatchType<0>>, LLVMMatchType<1>], + [NoCapture<ArgIndex<1>>]>, + RISCVVIntrinsic { + let VLOperand = 2; + } // For unit stride load with mask // Input: (maskedoff, pointer, mask, vl, ta) class RISCVUSLoadMask @@ -167,7 +227,9 @@ let TargetPrefix = "riscv" in { LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyint_ty, LLVMMatchType<1>], [NoCapture<ArgIndex<1>>, ImmArg<ArgIndex<4>>, IntrReadMem]>, - RISCVVIntrinsic; + RISCVVIntrinsic { + let VLOperand = 3; + } // For unit stride fault-only-first load with mask // Input: (maskedoff, pointer, mask, vl, ta) // Output: (data, vl) @@ -179,14 +241,19 @@ let TargetPrefix = "riscv" in { LLVMPointerType<LLVMMatchType<0>>, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMMatchType<1>, LLVMMatchType<1>], - [NoCapture<ArgIndex<1>>, ImmArg<ArgIndex<4>>]>, RISCVVIntrinsic; - // For strided load - // Input: (pointer, stride, vl) + [NoCapture<ArgIndex<1>>, ImmArg<ArgIndex<4>>]>, RISCVVIntrinsic { + let VLOperand = 3; + } + // For strided load with passthru operand + // Input: (passthru, pointer, stride, vl) class RISCVSLoad : Intrinsic<[llvm_anyvector_ty], - [LLVMPointerType<LLVMMatchType<0>>, + [LLVMMatchType<0>, + LLVMPointerType<LLVMMatchType<0>>, llvm_anyint_ty, LLVMMatchType<1>], - [NoCapture<ArgIndex<0>>, IntrReadMem]>, RISCVVIntrinsic; + [NoCapture<ArgIndex<1>>, IntrReadMem]>, RISCVVIntrinsic { + let VLOperand = 3; + } // For strided load with mask // Input: (maskedoff, pointer, stride, mask, vl, ta) class RISCVSLoadMask @@ -196,14 +263,19 @@ let TargetPrefix = "riscv" in { LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMMatchType<1>, LLVMMatchType<1>], [NoCapture<ArgIndex<1>>, ImmArg<ArgIndex<5>>, IntrReadMem]>, - RISCVVIntrinsic; - // For indexed load - // Input: (pointer, index, vl) + RISCVVIntrinsic { + let VLOperand = 4; + } + // For indexed load with passthru operand + // Input: (passthru, pointer, index, vl) class RISCVILoad : Intrinsic<[llvm_anyvector_ty], - [LLVMPointerType<LLVMMatchType<0>>, + [LLVMMatchType<0>, + LLVMPointerType<LLVMMatchType<0>>, llvm_anyvector_ty, llvm_anyint_ty], - [NoCapture<ArgIndex<0>>, IntrReadMem]>, RISCVVIntrinsic; + [NoCapture<ArgIndex<1>>, IntrReadMem]>, RISCVVIntrinsic { + let VLOperand = 3; + } // For indexed load with mask // Input: (maskedoff, pointer, index, mask, vl, ta) class RISCVILoadMask @@ -213,7 +285,9 @@ let TargetPrefix = "riscv" in { LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyint_ty, LLVMMatchType<2>], [NoCapture<ArgIndex<1>>, ImmArg<ArgIndex<5>>, IntrReadMem]>, - RISCVVIntrinsic; + RISCVVIntrinsic { + let VLOperand = 4; + } // For unit stride store // Input: (vector_in, pointer, vl) class RISCVUSStore @@ -221,7 +295,9 @@ let TargetPrefix = "riscv" in { [llvm_anyvector_ty, LLVMPointerType<LLVMMatchType<0>>, llvm_anyint_ty], - [NoCapture<ArgIndex<1>>, IntrWriteMem]>, RISCVVIntrinsic; + [NoCapture<ArgIndex<1>>, IntrWriteMem]>, RISCVVIntrinsic { + let VLOperand = 2; + } // For unit stride store with mask // Input: (vector_in, pointer, mask, vl) class RISCVUSStoreMask @@ -230,7 +306,9 @@ let TargetPrefix = "riscv" in { LLVMPointerType<LLVMMatchType<0>>, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyint_ty], - [NoCapture<ArgIndex<1>>, IntrWriteMem]>, RISCVVIntrinsic; + [NoCapture<ArgIndex<1>>, IntrWriteMem]>, RISCVVIntrinsic { + let VLOperand = 3; + } // For strided store // Input: (vector_in, pointer, stride, vl) class RISCVSStore @@ -238,7 +316,9 @@ let TargetPrefix = "riscv" in { [llvm_anyvector_ty, LLVMPointerType<LLVMMatchType<0>>, llvm_anyint_ty, LLVMMatchType<1>], - [NoCapture<ArgIndex<1>>, IntrWriteMem]>, RISCVVIntrinsic; + [NoCapture<ArgIndex<1>>, IntrWriteMem]>, RISCVVIntrinsic { + let VLOperand = 3; + } // For stride store with mask // Input: (vector_in, pointer, stirde, mask, vl) class RISCVSStoreMask @@ -246,7 +326,9 @@ let TargetPrefix = "riscv" in { [llvm_anyvector_ty, LLVMPointerType<LLVMMatchType<0>>, llvm_anyint_ty, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMMatchType<1>], - [NoCapture<ArgIndex<1>>, IntrWriteMem]>, RISCVVIntrinsic; + [NoCapture<ArgIndex<1>>, IntrWriteMem]>, RISCVVIntrinsic { + let VLOperand = 4; + } // For indexed store // Input: (vector_in, pointer, index, vl) class RISCVIStore @@ -254,7 +336,9 @@ let TargetPrefix = "riscv" in { [llvm_anyvector_ty, LLVMPointerType<LLVMMatchType<0>>, llvm_anyint_ty, llvm_anyint_ty], - [NoCapture<ArgIndex<1>>, IntrWriteMem]>, RISCVVIntrinsic; + [NoCapture<ArgIndex<1>>, IntrWriteMem]>, RISCVVIntrinsic { + let VLOperand = 3; + } // For indexed store with mask // Input: (vector_in, pointer, index, mask, vl) class RISCVIStoreMask @@ -262,13 +346,17 @@ let TargetPrefix = "riscv" in { [llvm_anyvector_ty, LLVMPointerType<LLVMMatchType<0>>, llvm_anyvector_ty, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyint_ty], - [NoCapture<ArgIndex<1>>, IntrWriteMem]>, RISCVVIntrinsic; + [NoCapture<ArgIndex<1>>, IntrWriteMem]>, RISCVVIntrinsic { + let VLOperand = 4; + } // For destination vector type is the same as source vector. // Input: (vector_in, vl) class RISCVUnaryAANoMask : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, llvm_anyint_ty], - [IntrNoMem]>, RISCVVIntrinsic; + [IntrNoMem]>, RISCVVIntrinsic { + let VLOperand = 1; + } // For destination vector type is the same as first source vector (with mask). // Input: (vector_in, mask, vl, ta) class RISCVUnaryAAMask @@ -276,24 +364,32 @@ let TargetPrefix = "riscv" in { [LLVMMatchType<0>, LLVMMatchType<0>, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyint_ty, LLVMMatchType<1>], - [ImmArg<ArgIndex<4>>, IntrNoMem]>, RISCVVIntrinsic; + [ImmArg<ArgIndex<4>>, IntrNoMem]>, RISCVVIntrinsic { + let VLOperand = 3; + } class RISCVUnaryAAMaskNoTA : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, LLVMMatchType<0>, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyint_ty], - [IntrNoMem]>, RISCVVIntrinsic; + [IntrNoMem]>, RISCVVIntrinsic { + let VLOperand = 3; + } // For destination vector type is the same as first and second source vector. // Input: (vector_in, vector_in, vl) class RISCVBinaryAAANoMask : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyint_ty], - [IntrNoMem]>, RISCVVIntrinsic; + [IntrNoMem]>, RISCVVIntrinsic { + let VLOperand = 2; + } // For destination vector type is the same as first and second source vector. // Input: (vector_in, int_vector_in, vl) class RISCVRGatherVVNoMask : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, LLVMVectorOfBitcastsToInt<0>, llvm_anyint_ty], - [IntrNoMem]>, RISCVVIntrinsic; + [IntrNoMem]>, RISCVVIntrinsic { + let VLOperand = 2; + } // For destination vector type is the same as first and second source vector. // Input: (vector_in, vector_in, int_vector_in, vl, ta) class RISCVRGatherVVMask @@ -301,22 +397,28 @@ let TargetPrefix = "riscv" in { [LLVMMatchType<0>, LLVMMatchType<0>, LLVMVectorOfBitcastsToInt<0>, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyint_ty, LLVMMatchType<1>], - [ImmArg<ArgIndex<5>>, IntrNoMem]>, RISCVVIntrinsic; + [ImmArg<ArgIndex<5>>, IntrNoMem]>, RISCVVIntrinsic { + let VLOperand = 4; + } // Input: (vector_in, int16_vector_in, vl) class RISCVRGatherEI16VVNoMask : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, LLVMScalarOrSameVectorWidth<0, llvm_i16_ty>, llvm_anyint_ty], - [IntrNoMem]>, RISCVVIntrinsic; + [IntrNoMem]>, RISCVVIntrinsic { + let VLOperand = 2; + } // For destination vector type is the same as first and second source vector. // Input: (vector_in, vector_in, int16_vector_in, vl, ta) class RISCVRGatherEI16VVMask : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, LLVMMatchType<0>, - LLVMScalarOrSameVectorWidth<0, llvm_i16_ty>, + LLVMScalarOrSameVectorWidth<0, llvm_i16_ty>, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyint_ty, LLVMMatchType<1>], - [ImmArg<ArgIndex<5>>, IntrNoMem]>, RISCVVIntrinsic; + [ImmArg<ArgIndex<5>>, IntrNoMem]>, RISCVVIntrinsic { + let VLOperand = 4; + } // For destination vector type is the same as first source vector, and the // second operand is XLen. // Input: (vector_in, xlen_in, vl) @@ -324,6 +426,7 @@ let TargetPrefix = "riscv" in { : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, llvm_anyint_ty, LLVMMatchType<1>], [IntrNoMem]>, RISCVVIntrinsic { + let VLOperand = 2; } // For destination vector type is the same as first source vector (with mask). // Second operand is XLen. @@ -334,6 +437,7 @@ let TargetPrefix = "riscv" in { LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMMatchType<1>, LLVMMatchType<1>], [ImmArg<ArgIndex<5>>, IntrNoMem]>, RISCVVIntrinsic { + let VLOperand = 4; } // For destination vector type is the same as first source vector. // Input: (vector_in, vector_in/scalar_in, vl) @@ -341,7 +445,8 @@ let TargetPrefix = "riscv" in { : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, llvm_any_ty, llvm_anyint_ty], [IntrNoMem]>, RISCVVIntrinsic { - let SplatOperand = 2; + let SplatOperand = 1; + let VLOperand = 2; } // For destination vector type is the same as first source vector (with mask). // Input: (maskedoff, vector_in, vector_in/scalar_in, mask, vl, ta) @@ -351,7 +456,8 @@ let TargetPrefix = "riscv" in { LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyint_ty, LLVMMatchType<2>], [ImmArg<ArgIndex<5>>, IntrNoMem]>, RISCVVIntrinsic { - let SplatOperand = 3; + let SplatOperand = 2; + let VLOperand = 4; } // For destination vector type is the same as first source vector. The // second source operand must match the destination type or be an XLen scalar. @@ -359,7 +465,9 @@ let TargetPrefix = "riscv" in { class RISCVBinaryAAShiftNoMask : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, llvm_any_ty, llvm_anyint_ty], - [IntrNoMem]>, RISCVVIntrinsic; + [IntrNoMem]>, RISCVVIntrinsic { + let VLOperand = 2; + } // For destination vector type is the same as first source vector (with mask). // The second source operand must match the destination type or be an XLen scalar. // Input: (maskedoff, vector_in, vector_in/scalar_in, mask, vl, ta) @@ -368,14 +476,17 @@ let TargetPrefix = "riscv" in { [LLVMMatchType<0>, LLVMMatchType<0>, llvm_any_ty, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyint_ty, LLVMMatchType<2>], - [ImmArg<ArgIndex<5>>, IntrNoMem]>, RISCVVIntrinsic; + [ImmArg<ArgIndex<5>>, IntrNoMem]>, RISCVVIntrinsic { + let VLOperand = 4; + } // For destination vector type is NOT the same as first source vector. // Input: (vector_in, vector_in/scalar_in, vl) class RISCVBinaryABXNoMask : Intrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty, llvm_any_ty, llvm_anyint_ty], [IntrNoMem]>, RISCVVIntrinsic { - let SplatOperand = 2; + let SplatOperand = 1; + let VLOperand = 2; } // For destination vector type is NOT the same as first source vector (with mask). // Input: (maskedoff, vector_in, vector_in/scalar_in, mask, vl, ta) @@ -385,7 +496,8 @@ let TargetPrefix = "riscv" in { LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyint_ty, LLVMMatchType<3>], [ImmArg<ArgIndex<5>>, IntrNoMem]>, RISCVVIntrinsic { - let SplatOperand = 3; + let SplatOperand = 2; + let VLOperand = 4; } // For destination vector type is NOT the same as first source vector. The // second source operand must match the destination type or be an XLen scalar. @@ -393,7 +505,9 @@ let TargetPrefix = "riscv" in { class RISCVBinaryABShiftNoMask : Intrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty, llvm_any_ty, llvm_anyint_ty], - [IntrNoMem]>, RISCVVIntrinsic; + [IntrNoMem]>, RISCVVIntrinsic { + let VLOperand = 2; + } // For destination vector type is NOT the same as first source vector (with mask). // The second source operand must match the destination type or be an XLen scalar. // Input: (maskedoff, vector_in, vector_in/scalar_in, mask, vl, ta) @@ -402,7 +516,9 @@ let TargetPrefix = "riscv" in { [LLVMMatchType<0>, llvm_anyvector_ty, llvm_any_ty, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyint_ty, LLVMMatchType<3>], - [ImmArg<ArgIndex<5>>, IntrNoMem]>, RISCVVIntrinsic; + [ImmArg<ArgIndex<5>>, IntrNoMem]>, RISCVVIntrinsic { + let VLOperand = 4; + } // For binary operations with V0 as input. // Input: (vector_in, vector_in/scalar_in, V0, vl) class RISCVBinaryWithV0 @@ -411,7 +527,8 @@ let TargetPrefix = "riscv" in { LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyint_ty], [IntrNoMem]>, RISCVVIntrinsic { - let SplatOperand = 2; + let SplatOperand = 1; + let VLOperand = 3; } // For binary operations with mask type output and V0 as input. // Output: (mask type output) @@ -422,7 +539,8 @@ let TargetPrefix = "riscv" in { LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyint_ty], [IntrNoMem]>, RISCVVIntrinsic { - let SplatOperand = 2; + let SplatOperand = 1; + let VLOperand = 3; } // For binary operations with mask type output. // Output: (mask type output) @@ -431,7 +549,8 @@ let TargetPrefix = "riscv" in { : Intrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], [llvm_anyvector_ty, llvm_any_ty, llvm_anyint_ty], [IntrNoMem]>, RISCVVIntrinsic { - let SplatOperand = 2; + let SplatOperand = 1; + let VLOperand = 2; } // For binary operations with mask type output without mask. // Output: (mask type output) @@ -440,7 +559,8 @@ let TargetPrefix = "riscv" in { : Intrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], [llvm_anyvector_ty, llvm_any_ty, llvm_anyint_ty], [IntrNoMem]>, RISCVVIntrinsic { - let SplatOperand = 2; + let SplatOperand = 1; + let VLOperand = 2; } // For binary operations with mask type output with mask. // Output: (mask type output) @@ -451,7 +571,8 @@ let TargetPrefix = "riscv" in { llvm_anyvector_ty, llvm_any_ty, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyint_ty], [IntrNoMem]>, RISCVVIntrinsic { - let SplatOperand = 3; + let SplatOperand = 2; + let VLOperand = 4; } // For FP classify operations. // Output: (bit mask type output) @@ -459,7 +580,9 @@ let TargetPrefix = "riscv" in { class RISCVClassifyNoMask : Intrinsic<[LLVMVectorOfBitcastsToInt<0>], [llvm_anyvector_ty, llvm_anyint_ty], - [IntrNoMem]>, RISCVVIntrinsic; + [IntrNoMem]>, RISCVVIntrinsic { + let VLOperand = 1; + } // For FP classify operations with mask. // Output: (bit mask type output) // Input: (maskedoff, vector_in, mask, vl) @@ -467,7 +590,9 @@ let TargetPrefix = "riscv" in { : Intrinsic<[LLVMVectorOfBitcastsToInt<0>], [LLVMVectorOfBitcastsToInt<0>, llvm_anyvector_ty, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyint_ty], - [IntrNoMem]>, RISCVVIntrinsic; + [IntrNoMem]>, RISCVVIntrinsic { + let VLOperand = 3; + } // For Saturating binary operations. // The destination vector type is the same as first source vector. // Input: (vector_in, vector_in/scalar_in, vl) @@ -475,7 +600,8 @@ let TargetPrefix = "riscv" in { : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, llvm_any_ty, llvm_anyint_ty], [IntrNoMem, IntrHasSideEffects]>, RISCVVIntrinsic { - let SplatOperand = 2; + let SplatOperand = 1; + let VLOperand = 2; } // For Saturating binary operations with mask. // The destination vector type is the same as first source vector. @@ -486,7 +612,8 @@ let TargetPrefix = "riscv" in { LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyint_ty, LLVMMatchType<2>], [ImmArg<ArgIndex<5>>, IntrNoMem, IntrHasSideEffects]>, RISCVVIntrinsic { - let SplatOperand = 3; + let SplatOperand = 2; + let VLOperand = 4; } // For Saturating binary operations. // The destination vector type is the same as first source vector. @@ -495,7 +622,9 @@ let TargetPrefix = "riscv" in { class RISCVSaturatingBinaryAAShiftNoMask : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, llvm_any_ty, llvm_anyint_ty], - [IntrNoMem, IntrHasSideEffects]>, RISCVVIntrinsic; + [IntrNoMem, IntrHasSideEffects]>, RISCVVIntrinsic { + let VLOperand = 2; + } // For Saturating binary operations with mask. // The destination vector type is the same as first source vector. // The second source operand matches the destination type or is an XLen scalar. @@ -505,7 +634,9 @@ let TargetPrefix = "riscv" in { [LLVMMatchType<0>, LLVMMatchType<0>, llvm_any_ty, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyint_ty, LLVMMatchType<2>], - [ImmArg<ArgIndex<5>>, IntrNoMem, IntrHasSideEffects]>, RISCVVIntrinsic; + [ImmArg<ArgIndex<5>>, IntrNoMem, IntrHasSideEffects]>, RISCVVIntrinsic { + let VLOperand = 4; + } // For Saturating binary operations. // The destination vector type is NOT the same as first source vector. // The second source operand matches the destination type or is an XLen scalar. @@ -513,7 +644,9 @@ let TargetPrefix = "riscv" in { class RISCVSaturatingBinaryABShiftNoMask : Intrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty, llvm_any_ty, llvm_anyint_ty], - [IntrNoMem, IntrHasSideEffects]>, RISCVVIntrinsic; + [IntrNoMem, IntrHasSideEffects]>, RISCVVIntrinsic { + let VLOperand = 2; + } // For Saturating binary operations with mask. // The destination vector type is NOT the same as first source vector (with mask). // The second source operand matches the destination type or is an XLen scalar. @@ -523,44 +656,54 @@ let TargetPrefix = "riscv" in { [LLVMMatchType<0>, llvm_anyvector_ty, llvm_any_ty, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyint_ty, LLVMMatchType<3>], - [ImmArg<ArgIndex<5>>, IntrNoMem, IntrHasSideEffects]>, RISCVVIntrinsic; + [ImmArg<ArgIndex<5>>, IntrNoMem, IntrHasSideEffects]>, RISCVVIntrinsic { + let VLOperand = 4; + } class RISCVTernaryAAAXNoMask : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyint_ty, LLVMMatchType<1>], - [IntrNoMem]>, RISCVVIntrinsic; + [IntrNoMem]>, RISCVVIntrinsic { + let VLOperand = 3; + } class RISCVTernaryAAAXMask : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyint_ty, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMMatchType<1>], - [IntrNoMem]>, RISCVVIntrinsic; + [IntrNoMem]>, RISCVVIntrinsic { + let VLOperand = 4; + } class RISCVTernaryAAXANoMask : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, llvm_any_ty, LLVMMatchType<0>, llvm_anyint_ty], [IntrNoMem]>, RISCVVIntrinsic { - let SplatOperand = 2; + let SplatOperand = 1; + let VLOperand = 3; } class RISCVTernaryAAXAMask : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, llvm_any_ty, LLVMMatchType<0>, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyint_ty], [IntrNoMem]>, RISCVVIntrinsic { - let SplatOperand = 2; + let SplatOperand = 1; + let VLOperand = 4; } class RISCVTernaryWideNoMask : Intrinsic< [llvm_anyvector_ty], [LLVMMatchType<0>, llvm_any_ty, llvm_anyvector_ty, llvm_anyint_ty], [IntrNoMem] >, RISCVVIntrinsic { - let SplatOperand = 2; + let SplatOperand = 1; + let VLOperand = 3; } class RISCVTernaryWideMask : Intrinsic< [llvm_anyvector_ty], [LLVMMatchType<0>, llvm_any_ty, llvm_anyvector_ty, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyint_ty], [IntrNoMem]>, RISCVVIntrinsic { - let SplatOperand = 2; + let SplatOperand = 1; + let VLOperand = 4; } // For Reduction ternary operations. // For destination vector type is the same as first and third source vector. @@ -569,7 +712,9 @@ let TargetPrefix = "riscv" in { : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<0>, llvm_anyint_ty], - [IntrNoMem]>, RISCVVIntrinsic; + [IntrNoMem]>, RISCVVIntrinsic { + let VLOperand = 3; + } // For Reduction ternary operations with mask. // For destination vector type is the same as first and third source vector. // The mask type come from second source vector. @@ -578,27 +723,35 @@ let TargetPrefix = "riscv" in { : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<0>, LLVMScalarOrSameVectorWidth<1, llvm_i1_ty>, llvm_anyint_ty], - [IntrNoMem]>, RISCVVIntrinsic; + [IntrNoMem]>, RISCVVIntrinsic { + let VLOperand = 4; + } // For unary operations with scalar type output without mask // Output: (scalar type) // Input: (vector_in, vl) class RISCVMaskUnarySOutNoMask : Intrinsic<[LLVMMatchType<1>], [llvm_anyvector_ty, llvm_anyint_ty], - [IntrNoMem]>, RISCVVIntrinsic; + [IntrNoMem]>, RISCVVIntrinsic { + let VLOperand = 1; + } // For unary operations with scalar type output with mask // Output: (scalar type) // Input: (vector_in, mask, vl) class RISCVMaskUnarySOutMask : Intrinsic<[LLVMMatchType<1>], [llvm_anyvector_ty, LLVMMatchType<0>, llvm_anyint_ty], - [IntrNoMem]>, RISCVVIntrinsic; + [IntrNoMem]>, RISCVVIntrinsic { + let VLOperand = 2; + } // For destination vector type is NOT the same as source vector. // Input: (vector_in, vl) class RISCVUnaryABNoMask : Intrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty, llvm_anyint_ty], - [IntrNoMem]>, RISCVVIntrinsic; + [IntrNoMem]>, RISCVVIntrinsic { + let VLOperand = 1; + } // For destination vector type is NOT the same as source vector (with mask). // Input: (maskedoff, vector_in, mask, vl, ta) class RISCVUnaryABMask @@ -606,14 +759,18 @@ let TargetPrefix = "riscv" in { [LLVMMatchType<0>, llvm_anyvector_ty, LLVMScalarOrSameVectorWidth<1, llvm_i1_ty>, llvm_anyint_ty, LLVMMatchType<2>], - [ImmArg<ArgIndex<4>>, IntrNoMem]>, RISCVVIntrinsic; + [ImmArg<ArgIndex<4>>, IntrNoMem]>, RISCVVIntrinsic { + let VLOperand = 3; + } // For unary operations with the same vector type in/out without mask // Output: (vector) // Input: (vector_in, vl) class RISCVUnaryNoMask : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, llvm_anyint_ty], - [IntrNoMem]>, RISCVVIntrinsic; + [IntrNoMem]>, RISCVVIntrinsic { + let VLOperand = 1; + } // For mask unary operations with mask type in/out with mask // Output: (mask type output) // Input: (mask type maskedoff, mask type vector_in, mask, vl) @@ -621,19 +778,25 @@ let TargetPrefix = "riscv" in { : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyint_ty], - [IntrNoMem]>, RISCVVIntrinsic; + [IntrNoMem]>, RISCVVIntrinsic { + let VLOperand = 3; + } // Output: (vector) // Input: (vl) class RISCVNullaryIntrinsic : Intrinsic<[llvm_anyvector_ty], [llvm_anyint_ty], - [IntrNoMem]>, RISCVVIntrinsic; + [IntrNoMem]>, RISCVVIntrinsic { + let VLOperand = 0; + } // For Conversion unary operations. // Input: (vector_in, vl) class RISCVConversionNoMask : Intrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty, llvm_anyint_ty], - [IntrNoMem]>, RISCVVIntrinsic; + [IntrNoMem]>, RISCVVIntrinsic { + let VLOperand = 1; + } // For Conversion unary operations with mask. // Input: (maskedoff, vector_in, mask, vl, ta) class RISCVConversionMask @@ -641,7 +804,9 @@ let TargetPrefix = "riscv" in { [LLVMMatchType<0>, llvm_anyvector_ty, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyint_ty, LLVMMatchType<2>], - [ImmArg<ArgIndex<4>>, IntrNoMem]>, RISCVVIntrinsic; + [ImmArg<ArgIndex<4>>, IntrNoMem]>, RISCVVIntrinsic { + let VLOperand = 3; + } // For unit stride segment load // Input: (pointer, vl) @@ -649,7 +814,9 @@ let TargetPrefix = "riscv" in { : Intrinsic<!listconcat([llvm_anyvector_ty], !listsplat(LLVMMatchType<0>, !add(nf, -1))), [LLVMPointerToElt<0>, llvm_anyint_ty], - [NoCapture<ArgIndex<0>>, IntrReadMem]>, RISCVVIntrinsic; + [NoCapture<ArgIndex<0>>, IntrReadMem]>, RISCVVIntrinsic { + let VLOperand = 1; + } // For unit stride segment load with mask // Input: (maskedoff, pointer, mask, vl, ta) class RISCVUSSegLoadMask<int nf> @@ -660,7 +827,9 @@ let TargetPrefix = "riscv" in { LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyint_ty, LLVMMatchType<1>]), [ImmArg<ArgIndex<!add(nf, 3)>>, NoCapture<ArgIndex<nf>>, IntrReadMem]>, - RISCVVIntrinsic; + RISCVVIntrinsic { + let VLOperand = !add(nf, 2); + } // For unit stride fault-only-first segment load // Input: (pointer, vl) @@ -671,7 +840,9 @@ let TargetPrefix = "riscv" in { : Intrinsic<!listconcat([llvm_anyvector_ty], !listsplat(LLVMMatchType<0>, !add(nf, -1)), [llvm_anyint_ty]), [LLVMPointerToElt<0>, LLVMMatchType<1>], - [NoCapture<ArgIndex<0>>]>, RISCVVIntrinsic; + [NoCapture<ArgIndex<0>>]>, RISCVVIntrinsic { + let VLOperand = 1; + } // For unit stride fault-only-first segment load with mask // Input: (maskedoff, pointer, mask, vl, ta) // Output: (data, vl) @@ -685,7 +856,9 @@ let TargetPrefix = "riscv" in { LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMMatchType<1>, LLVMMatchType<1>]), [ImmArg<ArgIndex<!add(nf, 3)>>, NoCapture<ArgIndex<nf>>]>, - RISCVVIntrinsic; + RISCVVIntrinsic { + let VLOperand = !add(nf, 2); + } // For stride segment load // Input: (pointer, offset, vl) @@ -693,7 +866,9 @@ let TargetPrefix = "riscv" in { : Intrinsic<!listconcat([llvm_anyvector_ty], !listsplat(LLVMMatchType<0>, !add(nf, -1))), [LLVMPointerToElt<0>, llvm_anyint_ty, LLVMMatchType<1>], - [NoCapture<ArgIndex<0>>, IntrReadMem]>, RISCVVIntrinsic; + [NoCapture<ArgIndex<0>>, IntrReadMem]>, RISCVVIntrinsic { + let VLOperand = 2; + } // For stride segment load with mask // Input: (maskedoff, pointer, offset, mask, vl, ta) class RISCVSSegLoadMask<int nf> @@ -705,7 +880,9 @@ let TargetPrefix = "riscv" in { LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMMatchType<1>, LLVMMatchType<1>]), [ImmArg<ArgIndex<!add(nf, 4)>>, NoCapture<ArgIndex<nf>>, IntrReadMem]>, - RISCVVIntrinsic; + RISCVVIntrinsic { + let VLOperand = !add(nf, 3); + } // For indexed segment load // Input: (pointer, index, vl) @@ -713,7 +890,9 @@ let TargetPrefix = "riscv" in { : Intrinsic<!listconcat([llvm_anyvector_ty], !listsplat(LLVMMatchType<0>, !add(nf, -1))), [LLVMPointerToElt<0>, llvm_anyvector_ty, llvm_anyint_ty], - [NoCapture<ArgIndex<0>>, IntrReadMem]>, RISCVVIntrinsic; + [NoCapture<ArgIndex<0>>, IntrReadMem]>, RISCVVIntrinsic { + let VLOperand = 2; + } // For indexed segment load with mask // Input: (maskedoff, pointer, index, mask, vl, ta) class RISCVISegLoadMask<int nf> @@ -725,7 +904,9 @@ let TargetPrefix = "riscv" in { LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyint_ty, LLVMMatchType<2>]), [ImmArg<ArgIndex<!add(nf, 4)>>, NoCapture<ArgIndex<nf>>, IntrReadMem]>, - RISCVVIntrinsic; + RISCVVIntrinsic { + let VLOperand = !add(nf, 3); + } // For unit stride segment store // Input: (value, pointer, vl) @@ -734,7 +915,9 @@ let TargetPrefix = "riscv" in { !listconcat([llvm_anyvector_ty], !listsplat(LLVMMatchType<0>, !add(nf, -1)), [LLVMPointerToElt<0>, llvm_anyint_ty]), - [NoCapture<ArgIndex<nf>>, IntrWriteMem]>, RISCVVIntrinsic; + [NoCapture<ArgIndex<nf>>, IntrWriteMem]>, RISCVVIntrinsic { + let VLOperand = !add(nf, 1); + } // For unit stride segment store with mask // Input: (value, pointer, mask, vl) class RISCVUSSegStoreMask<int nf> @@ -744,7 +927,9 @@ let TargetPrefix = "riscv" in { [LLVMPointerToElt<0>, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyint_ty]), - [NoCapture<ArgIndex<nf>>, IntrWriteMem]>, RISCVVIntrinsic; + [NoCapture<ArgIndex<nf>>, IntrWriteMem]>, RISCVVIntrinsic { + let VLOperand = !add(nf, 2); + } // For stride segment store // Input: (value, pointer, offset, vl) @@ -754,7 +939,9 @@ let TargetPrefix = "riscv" in { !listsplat(LLVMMatchType<0>, !add(nf, -1)), [LLVMPointerToElt<0>, llvm_anyint_ty, LLVMMatchType<1>]), - [NoCapture<ArgIndex<nf>>, IntrWriteMem]>, RISCVVIntrinsic; + [NoCapture<ArgIndex<nf>>, IntrWriteMem]>, RISCVVIntrinsic { + let VLOperand = !add(nf, 2); + } // For stride segment store with mask // Input: (value, pointer, offset, mask, vl) class RISCVSSegStoreMask<int nf> @@ -764,7 +951,9 @@ let TargetPrefix = "riscv" in { [LLVMPointerToElt<0>, llvm_anyint_ty, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMMatchType<1>]), - [NoCapture<ArgIndex<nf>>, IntrWriteMem]>, RISCVVIntrinsic; + [NoCapture<ArgIndex<nf>>, IntrWriteMem]>, RISCVVIntrinsic { + let VLOperand = !add(nf, 3); + } // For indexed segment store // Input: (value, pointer, offset, vl) @@ -774,7 +963,9 @@ let TargetPrefix = "riscv" in { !listsplat(LLVMMatchType<0>, !add(nf, -1)), [LLVMPointerToElt<0>, llvm_anyvector_ty, llvm_anyint_ty]), - [NoCapture<ArgIndex<nf>>, IntrWriteMem]>, RISCVVIntrinsic; + [NoCapture<ArgIndex<nf>>, IntrWriteMem]>, RISCVVIntrinsic { + let VLOperand = !add(nf, 2); + } // For indexed segment store with mask // Input: (value, pointer, offset, mask, vl) class RISCVISegStoreMask<int nf> @@ -784,7 +975,9 @@ let TargetPrefix = "riscv" in { [LLVMPointerToElt<0>, llvm_anyvector_ty, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyint_ty]), - [NoCapture<ArgIndex<nf>>, IntrWriteMem]>, RISCVVIntrinsic; + [NoCapture<ArgIndex<nf>>, IntrWriteMem]>, RISCVVIntrinsic { + let VLOperand = !add(nf, 3); + } multiclass RISCVUSLoad { def "int_riscv_" # NAME : RISCVUSLoad; @@ -955,7 +1148,7 @@ let TargetPrefix = "riscv" in { defm vsoxei : RISCVIStore; defm vsuxei : RISCVIStore; - def int_riscv_vlm : RISCVUSLoad; + def int_riscv_vlm : RISCVUSMLoad; def int_riscv_vsm : RISCVUSStore; defm vadd : RISCVBinaryAAX; @@ -1051,13 +1244,19 @@ let TargetPrefix = "riscv" in { def int_riscv_vmv_v_v : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, llvm_anyint_ty], - [IntrNoMem]>, RISCVVIntrinsic; + [IntrNoMem]>, RISCVVIntrinsic { + let VLOperand = 1; + } def int_riscv_vmv_v_x : Intrinsic<[llvm_anyint_ty], [LLVMVectorElementType<0>, llvm_anyint_ty], - [IntrNoMem]>, RISCVVIntrinsic; + [IntrNoMem]>, RISCVVIntrinsic { + let VLOperand = 1; + } def int_riscv_vfmv_v_f : Intrinsic<[llvm_anyfloat_ty], [LLVMVectorElementType<0>, llvm_anyint_ty], - [IntrNoMem]>, RISCVVIntrinsic; + [IntrNoMem]>, RISCVVIntrinsic { + let VLOperand = 1; + } def int_riscv_vmv_x_s : Intrinsic<[LLVMVectorElementType<0>], [llvm_anyint_ty], @@ -1065,7 +1264,9 @@ let TargetPrefix = "riscv" in { def int_riscv_vmv_s_x : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, LLVMVectorElementType<0>, llvm_anyint_ty], - [IntrNoMem]>, RISCVVIntrinsic; + [IntrNoMem]>, RISCVVIntrinsic { + let VLOperand = 2; + } def int_riscv_vfmv_f_s : Intrinsic<[LLVMVectorElementType<0>], [llvm_anyfloat_ty], @@ -1073,7 +1274,9 @@ let TargetPrefix = "riscv" in { def int_riscv_vfmv_s_f : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, LLVMVectorElementType<0>, llvm_anyint_ty], - [IntrNoMem]>, RISCVVIntrinsic; + [IntrNoMem]>, RISCVVIntrinsic { + let VLOperand = 2; + } defm vfmul : RISCVBinaryAAX; defm vfdiv : RISCVBinaryAAX; @@ -1210,7 +1413,9 @@ let TargetPrefix = "riscv" in { def int_riscv_viota : Intrinsic<[llvm_anyvector_ty], [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyint_ty], - [IntrNoMem]>, RISCVVIntrinsic; + [IntrNoMem]>, RISCVVIntrinsic { + let VLOperand = 1; + } // Output: (vector) // Input: (maskedoff, mask type vector_in, mask, vl) def int_riscv_viota_mask : Intrinsic<[llvm_anyvector_ty], @@ -1218,7 +1423,9 @@ let TargetPrefix = "riscv" in { LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyint_ty], - [IntrNoMem]>, RISCVVIntrinsic; + [IntrNoMem]>, RISCVVIntrinsic { + let VLOperand = 3; + } // Output: (vector) // Input: (vl) def int_riscv_vid : RISCVNullaryIntrinsic; @@ -1229,7 +1436,9 @@ let TargetPrefix = "riscv" in { [LLVMMatchType<0>, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyint_ty], - [IntrNoMem]>, RISCVVIntrinsic; + [IntrNoMem]>, RISCVVIntrinsic { + let VLOperand = 2; + } foreach nf = [2, 3, 4, 5, 6, 7, 8] in { defm vlseg # nf : RISCVUSSegLoad<nf>; @@ -1255,3 +1464,92 @@ let TargetPrefix = "riscv" in { llvm_anyint_ty, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], [NoCapture<ArgIndex<1>>, IntrWriteMem]>; } // TargetPrefix = "riscv" + +//===----------------------------------------------------------------------===// +// Scalar Cryptography +// +// These intrinsics will lower directly into the corresponding instructions +// added by the scalar cyptography extension, if the extension is present. + +let TargetPrefix = "riscv" in { + +class ScalarCryptoGprIntrinsicAny + : Intrinsic<[llvm_anyint_ty], + [LLVMMatchType<0>], + [IntrNoMem, IntrSpeculatable]>; + +class ScalarCryptoByteSelect32 + : Intrinsic<[llvm_i32_ty], + [llvm_i32_ty, llvm_i32_ty, llvm_i8_ty], + [IntrNoMem, IntrWillReturn, IntrSpeculatable, + ImmArg<ArgIndex<2>>]>; + +class ScalarCryptoGprGprIntrinsic32 + : Intrinsic<[llvm_i32_ty], + [llvm_i32_ty, llvm_i32_ty], + [IntrNoMem, IntrWillReturn, IntrSpeculatable]>; + +class ScalarCryptoGprGprIntrinsic64 + : Intrinsic<[llvm_i64_ty], + [llvm_i64_ty, llvm_i64_ty], + [IntrNoMem, IntrWillReturn, IntrSpeculatable]>; + +class ScalarCryptoGprIntrinsic64 + : Intrinsic<[llvm_i64_ty], + [llvm_i64_ty], + [IntrNoMem, IntrWillReturn, IntrSpeculatable]>; + +class ScalarCryptoByteSelectAny + : Intrinsic<[llvm_anyint_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i8_ty], + [IntrNoMem, IntrSpeculatable, IntrWillReturn, + ImmArg<ArgIndex<2>>, Returned<ArgIndex<0>>]>; + +// Zknd +def int_riscv_aes32dsi : ScalarCryptoByteSelect32; +def int_riscv_aes32dsmi : ScalarCryptoByteSelect32; + +def int_riscv_aes64ds : ScalarCryptoGprGprIntrinsic64; +def int_riscv_aes64dsm : ScalarCryptoGprGprIntrinsic64; + +def int_riscv_aes64im : ScalarCryptoGprIntrinsic64; + +// Zkne +def int_riscv_aes32esi : ScalarCryptoByteSelect32; +def int_riscv_aes32esmi : ScalarCryptoByteSelect32; + +def int_riscv_aes64es : ScalarCryptoGprGprIntrinsic64; +def int_riscv_aes64esm : ScalarCryptoGprGprIntrinsic64; + +// Zknd & Zkne +def int_riscv_aes64ks2 : ScalarCryptoGprGprIntrinsic64; +def int_riscv_aes64ks1i : Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i32_ty], + [IntrNoMem, IntrSpeculatable, + IntrWillReturn, ImmArg<ArgIndex<1>>]>; + +// Zknh +def int_riscv_sha256sig0 : ScalarCryptoGprIntrinsicAny; +def int_riscv_sha256sig1 : ScalarCryptoGprIntrinsicAny; +def int_riscv_sha256sum0 : ScalarCryptoGprIntrinsicAny; +def int_riscv_sha256sum1 : ScalarCryptoGprIntrinsicAny; + +def int_riscv_sha512sig0l : ScalarCryptoGprGprIntrinsic32; +def int_riscv_sha512sig0h : ScalarCryptoGprGprIntrinsic32; +def int_riscv_sha512sig1l : ScalarCryptoGprGprIntrinsic32; +def int_riscv_sha512sig1h : ScalarCryptoGprGprIntrinsic32; +def int_riscv_sha512sum0r : ScalarCryptoGprGprIntrinsic32; +def int_riscv_sha512sum1r : ScalarCryptoGprGprIntrinsic32; + +def int_riscv_sha512sig0 : ScalarCryptoGprIntrinsic64; +def int_riscv_sha512sig1 : ScalarCryptoGprIntrinsic64; +def int_riscv_sha512sum0 : ScalarCryptoGprIntrinsic64; +def int_riscv_sha512sum1 : ScalarCryptoGprIntrinsic64; + +// Zksed +def int_riscv_sm4ks : ScalarCryptoByteSelectAny; +def int_riscv_sm4ed : ScalarCryptoByteSelectAny; + +// Zksh +def int_riscv_sm3p0 : ScalarCryptoGprIntrinsicAny; +def int_riscv_sm3p1 : ScalarCryptoGprIntrinsicAny; +} // TargetPrefix = "riscv" diff --git a/llvm/include/llvm/IR/LLVMContext.h b/llvm/include/llvm/IR/LLVMContext.h index 1c902ebce5ad..d165a405ce22 100644 --- a/llvm/include/llvm/IR/LLVMContext.h +++ b/llvm/include/llvm/IR/LLVMContext.h @@ -15,6 +15,7 @@ #define LLVM_IR_LLVMCONTEXT_H #include "llvm-c/Types.h" +#include "llvm/ADT/Optional.h" #include "llvm/IR/DiagnosticHandler.h" #include "llvm/Support/CBindingWrapping.h" #include <cstdint> @@ -32,7 +33,6 @@ class Module; class OptPassGate; template <typename T> class SmallVectorImpl; template <typename T> class StringMapEntry; -class SMDiagnostic; class StringRef; class Twine; class LLVMRemarkStreamer; diff --git a/llvm/include/llvm/IR/LegacyPassManagers.h b/llvm/include/llvm/IR/LegacyPassManagers.h index 0bcb408d4929..311a407f1a19 100644 --- a/llvm/include/llvm/IR/LegacyPassManagers.h +++ b/llvm/include/llvm/IR/LegacyPassManagers.h @@ -90,7 +90,6 @@ template <typename T> class ArrayRef; class Module; class StringRef; class Value; -class Timer; class PMDataManager; // enums for debugging strings @@ -460,8 +459,7 @@ private: class FPPassManager : public ModulePass, public PMDataManager { public: static char ID; - explicit FPPassManager() - : ModulePass(ID), PMDataManager() { } + explicit FPPassManager() : ModulePass(ID) {} /// run - Execute all of the passes scheduled for execution. Keep track of /// whether any of the passes modifies the module, and if so, return true. diff --git a/llvm/include/llvm/IR/MatrixBuilder.h b/llvm/include/llvm/IR/MatrixBuilder.h index 6cc5797269e2..4c8286692ebf 100644 --- a/llvm/include/llvm/IR/MatrixBuilder.h +++ b/llvm/include/llvm/IR/MatrixBuilder.h @@ -68,7 +68,7 @@ public: // Deal with the pointer PointerType *PtrTy = cast<PointerType>(DataPtr->getType()); - Type *EltTy = PtrTy->getElementType(); + Type *EltTy = PtrTy->getPointerElementType(); auto *RetType = FixedVectorType::get(EltTy, Rows * Columns); diff --git a/llvm/include/llvm/IR/Module.h b/llvm/include/llvm/IR/Module.h index 4ddbd6ff14b3..7b834fbeeebf 100644 --- a/llvm/include/llvm/IR/Module.h +++ b/llvm/include/llvm/IR/Module.h @@ -47,9 +47,7 @@ class GVMaterializer; class LLVMContext; class MemoryBuffer; class ModuleSummaryIndex; -class Pass; class RandomNumberGenerator; -template <class PtrType> class SmallPtrSetImpl; class StructType; class VersionTuple; diff --git a/llvm/include/llvm/IR/NoFolder.h b/llvm/include/llvm/IR/NoFolder.h index dcffa6b2f9da..ec149747e3f4 100644 --- a/llvm/include/llvm/IR/NoFolder.h +++ b/llvm/include/llvm/IR/NoFolder.h @@ -38,18 +38,37 @@ public: explicit NoFolder() = default; //===--------------------------------------------------------------------===// - // Binary Operators + // Value-based folders. + // + // Return an existing value or a constant if the operation can be simplified. + // Otherwise return nullptr. //===--------------------------------------------------------------------===// + Value *FoldAdd(Value *LHS, Value *RHS, bool HasNUW = false, + bool HasNSW = false) const override { + return nullptr; + } - Instruction *CreateAdd(Constant *LHS, Constant *RHS, - bool HasNUW = false, - bool HasNSW = false) const override { - BinaryOperator *BO = BinaryOperator::CreateAdd(LHS, RHS); - if (HasNUW) BO->setHasNoUnsignedWrap(); - if (HasNSW) BO->setHasNoSignedWrap(); - return BO; + Value *FoldAnd(Value *LHS, Value *RHS) const override { return nullptr; } + + Value *FoldOr(Value *LHS, Value *RHS) const override { return nullptr; } + + Value *FoldICmp(CmpInst::Predicate P, Value *LHS, Value *RHS) const override { + return nullptr; + } + + Value *FoldGEP(Type *Ty, Value *Ptr, ArrayRef<Value *> IdxList, + bool IsInBounds = false) const override { + return nullptr; } + Value *FoldSelect(Value *C, Value *True, Value *False) const override { + return nullptr; + } + + //===--------------------------------------------------------------------===// + // Binary Operators + //===--------------------------------------------------------------------===// + Instruction *CreateFAdd(Constant *LHS, Constant *RHS) const override { return BinaryOperator::CreateFAdd(LHS, RHS); } @@ -132,14 +151,6 @@ public: return BinaryOperator::CreateExactAShr(LHS, RHS); } - Instruction *CreateAnd(Constant *LHS, Constant *RHS) const override { - return BinaryOperator::CreateAnd(LHS, RHS); - } - - Instruction *CreateOr(Constant *LHS, Constant *RHS) const override { - return BinaryOperator::CreateOr(LHS, RHS); - } - Instruction *CreateXor(Constant *LHS, Constant *RHS) const override { return BinaryOperator::CreateXor(LHS, RHS); } @@ -176,46 +187,6 @@ public: } //===--------------------------------------------------------------------===// - // Memory Instructions - //===--------------------------------------------------------------------===// - - Constant *CreateGetElementPtr(Type *Ty, Constant *C, - ArrayRef<Constant *> IdxList) const override { - return ConstantExpr::getGetElementPtr(Ty, C, IdxList); - } - - Constant *CreateGetElementPtr(Type *Ty, Constant *C, - Constant *Idx) const override { - // This form of the function only exists to avoid ambiguous overload - // warnings about whether to convert Idx to ArrayRef<Constant *> or - // ArrayRef<Value *>. - return ConstantExpr::getGetElementPtr(Ty, C, Idx); - } - - Instruction *CreateGetElementPtr(Type *Ty, Constant *C, - ArrayRef<Value *> IdxList) const override { - return GetElementPtrInst::Create(Ty, C, IdxList); - } - - Constant *CreateInBoundsGetElementPtr( - Type *Ty, Constant *C, ArrayRef<Constant *> IdxList) const override { - return ConstantExpr::getInBoundsGetElementPtr(Ty, C, IdxList); - } - - Constant *CreateInBoundsGetElementPtr(Type *Ty, Constant *C, - Constant *Idx) const override { - // This form of the function only exists to avoid ambiguous overload - // warnings about whether to convert Idx to ArrayRef<Constant *> or - // ArrayRef<Value *>. - return ConstantExpr::getInBoundsGetElementPtr(Ty, C, Idx); - } - - Instruction *CreateInBoundsGetElementPtr( - Type *Ty, Constant *C, ArrayRef<Value *> IdxList) const override { - return GetElementPtrInst::CreateInBounds(Ty, C, IdxList); - } - - //===--------------------------------------------------------------------===// // Cast/Conversion Operators //===--------------------------------------------------------------------===// @@ -270,11 +241,6 @@ public: // Compare Instructions //===--------------------------------------------------------------------===// - Instruction *CreateICmp(CmpInst::Predicate P, - Constant *LHS, Constant *RHS) const override { - return new ICmpInst(P, LHS, RHS); - } - Instruction *CreateFCmp(CmpInst::Predicate P, Constant *LHS, Constant *RHS) const override { return new FCmpInst(P, LHS, RHS); @@ -284,11 +250,6 @@ public: // Other Instructions //===--------------------------------------------------------------------===// - Instruction *CreateSelect(Constant *C, - Constant *True, Constant *False) const override { - return SelectInst::Create(C, True, False); - } - Instruction *CreateExtractElement(Constant *Vec, Constant *Idx) const override { return ExtractElementInst::Create(Vec, Idx); diff --git a/llvm/include/llvm/IR/PatternMatch.h b/llvm/include/llvm/IR/PatternMatch.h index 320deb80bb1f..f9f4f1603861 100644 --- a/llvm/include/llvm/IR/PatternMatch.h +++ b/llvm/include/llvm/IR/PatternMatch.h @@ -589,6 +589,9 @@ struct is_lowbit_mask { inline cst_pred_ty<is_lowbit_mask> m_LowBitMask() { return cst_pred_ty<is_lowbit_mask>(); } +inline api_pred_ty<is_lowbit_mask> m_LowBitMask(const APInt *&V) { + return V; +} struct icmp_pred_with_threshold { ICmpInst::Predicate Pred; diff --git a/llvm/include/llvm/IR/PseudoProbe.h b/llvm/include/llvm/IR/PseudoProbe.h index 51ba7e675efe..7d14213143c0 100644 --- a/llvm/include/llvm/IR/PseudoProbe.h +++ b/llvm/include/llvm/IR/PseudoProbe.h @@ -21,7 +21,6 @@ namespace llvm { class Instruction; -class BasicBlock; constexpr const char *PseudoProbeDescMetadataName = "llvm.pseudo_probe_desc"; diff --git a/llvm/include/llvm/IR/SSAContext.h b/llvm/include/llvm/IR/SSAContext.h index 9d9290a2c1d7..8879512610c2 100644 --- a/llvm/include/llvm/IR/SSAContext.h +++ b/llvm/include/llvm/IR/SSAContext.h @@ -26,7 +26,6 @@ class BasicBlock; class Function; class Instruction; class Value; -template <typename> class SmallVectorImpl; template <typename, bool> class DominatorTreeBase; template <> class GenericSSAContext<Function> { diff --git a/llvm/include/llvm/IR/Statepoint.h b/llvm/include/llvm/IR/Statepoint.h index c6251b9bf5c9..a254a67e6b1f 100644 --- a/llvm/include/llvm/IR/Statepoint.h +++ b/llvm/include/llvm/IR/Statepoint.h @@ -123,7 +123,7 @@ public: /// statepoint. Type *getActualReturnType() const { auto *CalleeTy = - cast<PointerType>(getActualCalledOperand()->getType())->getElementType(); + getActualCalledOperand()->getType()->getPointerElementType(); return cast<FunctionType>(CalleeTy)->getReturnType(); } diff --git a/llvm/include/llvm/IR/Type.h b/llvm/include/llvm/IR/Type.h index c899c46d4055..98c97375ad7b 100644 --- a/llvm/include/llvm/IR/Type.h +++ b/llvm/include/llvm/IR/Type.h @@ -366,7 +366,16 @@ public: return ContainedTys[0]; } + /// This method is deprecated without replacement. Pointer element types are + /// not available with opaque pointers. Type *getPointerElementType() const { + return getNonOpaquePointerElementType(); + } + + /// Only use this method in code that is not reachable with opaque pointers, + /// or part of deprecated methods that will be removed as part of the opaque + /// pointers transition. + Type *getNonOpaquePointerElementType() const { assert(getTypeID() == PointerTyID); assert(NumContainedTys && "Attempting to get element type of opaque pointer"); diff --git a/llvm/include/llvm/IR/TypeFinder.h b/llvm/include/llvm/IR/TypeFinder.h index a83f85ea84c3..dd2b70c65c2d 100644 --- a/llvm/include/llvm/IR/TypeFinder.h +++ b/llvm/include/llvm/IR/TypeFinder.h @@ -14,6 +14,7 @@ #define LLVM_IR_TYPEFINDER_H #include "llvm/ADT/DenseSet.h" +#include "llvm/IR/Attributes.h" #include <cstddef> #include <vector> @@ -32,6 +33,7 @@ class TypeFinder { // objects, we keep several helper maps. DenseSet<const Value*> VisitedConstants; DenseSet<const MDNode *> VisitedMetadata; + DenseSet<AttributeList> VisitedAttributes; DenseSet<Type*> VisitedTypes; std::vector<StructType*> StructTypes; @@ -74,6 +76,9 @@ private: /// incorporateMDNode - This method is used to walk the operands of an MDNode /// to find types hiding within. void incorporateMDNode(const MDNode *V); + + /// Incorporate types referenced by attributes. + void incorporateAttributes(AttributeList AL); }; } // end namespace llvm diff --git a/llvm/include/llvm/IR/VPIntrinsics.def b/llvm/include/llvm/IR/VPIntrinsics.def index 121c8bbc6c27..1abcbb874a8d 100644 --- a/llvm/include/llvm/IR/VPIntrinsics.def +++ b/llvm/include/llvm/IR/VPIntrinsics.def @@ -39,9 +39,9 @@ // same name. Since the operands are also the same, we open the property // scopes for both the VPIntrinsic and the SDNode at once. // \p VPSD The SelectionDAG Node id (eg VP_ADD). -// \p LEGALPOS The operand position of the SDNode that is used for legalizing -// this SDNode. This can be `-1`, in which case the return type of -// the SDNode is used. +// \p LEGALPOS The operand position of the SDNode that is used for legalizing. +// If LEGALPOS < 0, then the return type given by +// TheNode->getValueType(-1-LEGALPOS) is used. // \p TDNAME The name of the TableGen definition of this SDNode. // \p MASKPOS The mask operand position. // \p EVLPOS The explicit vector length operand position. @@ -349,6 +349,10 @@ BEGIN_REGISTER_VP(vp_select, 0, 3, VP_SELECT, -1) VP_PROPERTY_FUNCTIONAL_OPC(Select) END_REGISTER_VP(vp_select, VP_SELECT) +// llvm.vp.merge(mask,on_true,on_false,pivot) +BEGIN_REGISTER_VP(vp_merge, 0, 3, VP_MERGE, -1) +END_REGISTER_VP(vp_merge, VP_MERGE) + BEGIN_REGISTER_VP(experimental_vp_splice, 3, 5, EXPERIMENTAL_VP_SPLICE, -1) END_REGISTER_VP(experimental_vp_splice, EXPERIMENTAL_VP_SPLICE) diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h index 0c5ebc9a2f28..489ef045796f 100644 --- a/llvm/include/llvm/InitializePasses.h +++ b/llvm/include/llvm/InitializePasses.h @@ -380,6 +380,7 @@ void initializeReassociateLegacyPassPass(PassRegistry&); void initializeRedundantDbgInstEliminationPass(PassRegistry&); void initializeRegAllocEvictionAdvisorAnalysisPass(PassRegistry &); void initializeRegAllocFastPass(PassRegistry&); +void initializeRegAllocScoringPass(PassRegistry &); void initializeRegBankSelectPass(PassRegistry&); void initializeRegToMemLegacyPass(PassRegistry&); void initializeRegUsageInfoCollectorPass(PassRegistry&); diff --git a/llvm/include/llvm/LTO/LTO.h b/llvm/include/llvm/LTO/LTO.h index d2b0fef1ca47..0d085a88a193 100644 --- a/llvm/include/llvm/LTO/LTO.h +++ b/llvm/include/llvm/LTO/LTO.h @@ -35,7 +35,6 @@ class LLVMContext; class MemoryBufferRef; class Module; class raw_pwrite_stream; -class Target; class ToolOutputFile; /// Resolve linkage for prevailing symbols in the \p Index. Linkage changes diff --git a/llvm/include/llvm/LTO/legacy/LTOModule.h b/llvm/include/llvm/LTO/legacy/LTOModule.h index 01e63db4bab3..1b2de3b33385 100644 --- a/llvm/include/llvm/LTO/legacy/LTOModule.h +++ b/llvm/include/llvm/LTO/legacy/LTOModule.h @@ -40,8 +40,8 @@ private: struct NameAndAttributes { StringRef name; uint32_t attributes = 0; - bool isFunction = 0; - const GlobalValue *symbol = 0; + bool isFunction = false; + const GlobalValue *symbol = nullptr; }; std::unique_ptr<LLVMContext> OwnedContext; diff --git a/llvm/include/llvm/LTO/legacy/ThinLTOCodeGenerator.h b/llvm/include/llvm/LTO/legacy/ThinLTOCodeGenerator.h index b8daea9441e0..be1f3154029c 100644 --- a/llvm/include/llvm/LTO/legacy/ThinLTOCodeGenerator.h +++ b/llvm/include/llvm/LTO/legacy/ThinLTOCodeGenerator.h @@ -29,7 +29,6 @@ namespace llvm { class StringRef; -class LLVMContext; class TargetMachine; /// Helper to gather options relevant to the target machine creation diff --git a/llvm/include/llvm/Linker/Linker.h b/llvm/include/llvm/Linker/Linker.h index c9b1d42b3903..ac8041d8df1a 100644 --- a/llvm/include/llvm/Linker/Linker.h +++ b/llvm/include/llvm/Linker/Linker.h @@ -14,8 +14,6 @@ namespace llvm { class Module; -class StructType; -class Type; /// This class provides the core functionality of linking in LLVM. It keeps a /// pointer to the merged module so far. It doesn't take ownership of the diff --git a/llvm/include/llvm/MC/MCContext.h b/llvm/include/llvm/MC/MCContext.h index bde750759a0b..88d86d5b675a 100644 --- a/llvm/include/llvm/MC/MCContext.h +++ b/llvm/include/llvm/MC/MCContext.h @@ -814,10 +814,6 @@ namespace llvm { void diagnose(const SMDiagnostic &SMD); void reportError(SMLoc L, const Twine &Msg); void reportWarning(SMLoc L, const Twine &Msg); - // Unrecoverable error has occurred. Display the best diagnostic we can - // and bail via exit(1). For now, most MC backend errors are unrecoverable. - // FIXME: We should really do something about that. - [[noreturn]] void reportFatalError(SMLoc L, const Twine &Msg); const MCAsmMacro *lookupMacro(StringRef Name) { StringMap<MCAsmMacro>::iterator I = MacroMap.find(Name); diff --git a/llvm/include/llvm/MC/MCFixedLenDisassembler.h b/llvm/include/llvm/MC/MCFixedLenDisassembler.h index 218ae0d13189..1edf3899c130 100644 --- a/llvm/include/llvm/MC/MCFixedLenDisassembler.h +++ b/llvm/include/llvm/MC/MCFixedLenDisassembler.h @@ -27,7 +27,7 @@ enum DecoderOps { OPC_Fail // OPC_Fail() }; -} // namespace MCDecode +} // namespace MCD } // namespace llvm #endif diff --git a/llvm/include/llvm/MC/MCParser/MCAsmParser.h b/llvm/include/llvm/MC/MCParser/MCAsmParser.h index dbb76bd36591..29386ffc45ac 100644 --- a/llvm/include/llvm/MC/MCParser/MCAsmParser.h +++ b/llvm/include/llvm/MC/MCParser/MCAsmParser.h @@ -262,6 +262,7 @@ public: bool parseOptionalToken(AsmToken::TokenKind T); bool parseComma() { return parseToken(AsmToken::Comma, "expected comma"); } + bool parseRParen() { return parseToken(AsmToken::RParen, "expected ')'"); } bool parseEOL(); bool parseEOL(const Twine &ErrMsg); diff --git a/llvm/include/llvm/MC/MCPseudoProbe.h b/llvm/include/llvm/MC/MCPseudoProbe.h index abc9705f0851..17b7446baae8 100644 --- a/llvm/include/llvm/MC/MCPseudoProbe.h +++ b/llvm/include/llvm/MC/MCPseudoProbe.h @@ -60,7 +60,6 @@ namespace llvm { class MCSection; -class MCStreamer; class MCSymbol; class MCObjectStreamer; class raw_ostream; diff --git a/llvm/include/llvm/MC/MCStreamer.h b/llvm/include/llvm/MC/MCStreamer.h index 7bfbdb880098..3d6c512bfe73 100644 --- a/llvm/include/llvm/MC/MCStreamer.h +++ b/llvm/include/llvm/MC/MCStreamer.h @@ -27,6 +27,7 @@ #include "llvm/Support/Error.h" #include "llvm/Support/MD5.h" #include "llvm/Support/SMLoc.h" +#include "llvm/Support/ARMTargetParser.h" #include "llvm/Support/TargetParser.h" #include "llvm/Support/VersionTuple.h" #include <cassert> @@ -40,7 +41,6 @@ namespace llvm { class AssemblerConstantPools; class MCAsmBackend; -class MCCodeEmitter; class MCContext; struct MCDwarfFrameInfo; class MCExpr; diff --git a/llvm/include/llvm/MC/MCTargetOptions.h b/llvm/include/llvm/MC/MCTargetOptions.h index 3510eeca8953..db50dc6749e2 100644 --- a/llvm/include/llvm/MC/MCTargetOptions.h +++ b/llvm/include/llvm/MC/MCTargetOptions.h @@ -62,7 +62,6 @@ public: std::string ABIName; std::string AssemblyLanguage; std::string SplitDwarfFile; - std::string COFFOutputFilename; const char *Argv0 = nullptr; ArrayRef<std::string> CommandLineArgs; diff --git a/llvm/include/llvm/MC/SubtargetFeature.h b/llvm/include/llvm/MC/SubtargetFeature.h index cc36b25a4965..032e2a7df1f2 100644 --- a/llvm/include/llvm/MC/SubtargetFeature.h +++ b/llvm/include/llvm/MC/SubtargetFeature.h @@ -18,6 +18,7 @@ #define LLVM_MC_SUBTARGETFEATURE_H #include "llvm/ADT/StringRef.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/Support/MathExtras.h" #include <array> #include <bitset> diff --git a/llvm/include/llvm/MCA/CustomBehaviour.h b/llvm/include/llvm/MCA/CustomBehaviour.h index 395b07cf722b..5b993c6a5345 100644 --- a/llvm/include/llvm/MCA/CustomBehaviour.h +++ b/llvm/include/llvm/MCA/CustomBehaviour.h @@ -43,6 +43,10 @@ public: virtual ~InstrPostProcess() {} + /// This method can be overriden by targets to modify the mca::Instruction + /// object after it has been lowered from the MCInst. + /// This is generally a less disruptive alternative to modifying the + /// scheduling model. virtual void postProcessInstruction(std::unique_ptr<Instruction> &Inst, const MCInst &MCI) {} }; diff --git a/llvm/include/llvm/MCA/HardwareUnits/LSUnit.h b/llvm/include/llvm/MCA/HardwareUnits/LSUnit.h index 7eddd067aa0c..c05f770df8eb 100644 --- a/llvm/include/llvm/MCA/HardwareUnits/LSUnit.h +++ b/llvm/include/llvm/MCA/HardwareUnits/LSUnit.h @@ -55,7 +55,7 @@ public: MemoryGroup() : NumPredecessors(0), NumExecutingPredecessors(0), NumExecutedPredecessors(0), NumInstructions(0), NumExecuting(0), - NumExecuted(0), CriticalPredecessor(), CriticalMemoryInstruction() {} + NumExecuted(0), CriticalPredecessor() {} MemoryGroup(MemoryGroup &&) = default; size_t getNumSuccessors() const { diff --git a/llvm/include/llvm/MCA/HardwareUnits/ResourceManager.h b/llvm/include/llvm/MCA/HardwareUnits/ResourceManager.h index b679b0d7d537..7467fd6754f0 100644 --- a/llvm/include/llvm/MCA/HardwareUnits/ResourceManager.h +++ b/llvm/include/llvm/MCA/HardwareUnits/ResourceManager.h @@ -118,8 +118,8 @@ class DefaultResourceStrategy final : public ResourceStrategy { public: DefaultResourceStrategy(uint64_t UnitMask) - : ResourceStrategy(), ResourceUnitMask(UnitMask), - NextInSequenceMask(UnitMask), RemovedFromNextInSequence(0) {} + : ResourceUnitMask(UnitMask), NextInSequenceMask(UnitMask), + RemovedFromNextInSequence(0) {} virtual ~DefaultResourceStrategy() = default; uint64_t select(uint64_t ReadyMask) override; diff --git a/llvm/include/llvm/MCA/Instruction.h b/llvm/include/llvm/MCA/Instruction.h index 3eb32186d551..33e3c8a2e630 100644 --- a/llvm/include/llvm/MCA/Instruction.h +++ b/llvm/include/llvm/MCA/Instruction.h @@ -406,7 +406,7 @@ public: bool operator<(const CycleSegment &Other) const { return Begin < Other.Begin; } - CycleSegment &operator--(void) { + CycleSegment &operator--() { if (Begin) Begin--; if (End) @@ -517,9 +517,14 @@ class InstructionBase { // Instruction opcode which can be used by mca::CustomBehaviour unsigned Opcode; + // Flags used by the LSUnit. + bool IsALoadBarrier; + bool IsAStoreBarrier; + public: InstructionBase(const InstrDesc &D, const unsigned Opcode) - : Desc(D), IsOptimizableMove(false), Operands(0), Opcode(Opcode) {} + : Desc(D), IsOptimizableMove(false), Operands(0), Opcode(Opcode), + IsALoadBarrier(false), IsAStoreBarrier(false) {} SmallVectorImpl<WriteState> &getDefs() { return Defs; } ArrayRef<WriteState> getDefs() const { return Defs; } @@ -530,6 +535,10 @@ public: unsigned getLatency() const { return Desc.MaxLatency; } unsigned getNumMicroOps() const { return Desc.NumMicroOps; } unsigned getOpcode() const { return Opcode; } + bool isALoadBarrier() const { return IsALoadBarrier; } + bool isAStoreBarrier() const { return IsAStoreBarrier; } + void setLoadBarrier(bool IsBarrier) { IsALoadBarrier = IsBarrier; } + void setStoreBarrier(bool IsBarrier) { IsAStoreBarrier = IsBarrier; } /// Return the MCAOperand which corresponds to index Idx within the original /// MCInst. diff --git a/llvm/include/llvm/MCA/Stages/EntryStage.h b/llvm/include/llvm/MCA/Stages/EntryStage.h index 1c133898d603..4c50838bef4b 100644 --- a/llvm/include/llvm/MCA/Stages/EntryStage.h +++ b/llvm/include/llvm/MCA/Stages/EntryStage.h @@ -36,7 +36,7 @@ class EntryStage final : public Stage { EntryStage &operator=(const EntryStage &Other) = delete; public: - EntryStage(SourceMgr &SM) : CurrentInstruction(), SM(SM), NumRetired(0) { } + EntryStage(SourceMgr &SM) : SM(SM), NumRetired(0) {} bool isAvailable(const InstRef &IR) const override; bool hasWorkToComplete() const override; diff --git a/llvm/include/llvm/MCA/Stages/ExecuteStage.h b/llvm/include/llvm/MCA/Stages/ExecuteStage.h index 4c09ca8255ff..03a78a8b6b85 100644 --- a/llvm/include/llvm/MCA/Stages/ExecuteStage.h +++ b/llvm/include/llvm/MCA/Stages/ExecuteStage.h @@ -49,7 +49,7 @@ class ExecuteStage final : public Stage { public: ExecuteStage(Scheduler &S) : ExecuteStage(S, false) {} ExecuteStage(Scheduler &S, bool ShouldPerformBottleneckAnalysis) - : Stage(), HWS(S), NumDispatchedOpcodes(0), NumIssuedOpcodes(0), + : HWS(S), NumDispatchedOpcodes(0), NumIssuedOpcodes(0), EnablePressureEvents(ShouldPerformBottleneckAnalysis) {} // This stage works under the assumption that the Pipeline will eventually diff --git a/llvm/include/llvm/MCA/Stages/InOrderIssueStage.h b/llvm/include/llvm/MCA/Stages/InOrderIssueStage.h index 42f386a13d85..40bc3b5aed94 100644 --- a/llvm/include/llvm/MCA/Stages/InOrderIssueStage.h +++ b/llvm/include/llvm/MCA/Stages/InOrderIssueStage.h @@ -38,7 +38,7 @@ struct StallInfo { unsigned CyclesLeft; StallKind Kind; - StallInfo() : IR(), CyclesLeft(), Kind(StallKind::DEFAULT) {} + StallInfo() : CyclesLeft(), Kind(StallKind::DEFAULT) {} StallKind getStallKind() const { return Kind; } unsigned getCyclesLeft() const { return CyclesLeft; } diff --git a/llvm/include/llvm/MCA/Stages/InstructionTables.h b/llvm/include/llvm/MCA/Stages/InstructionTables.h index 35b21b0ba94d..9617fd49db6e 100644 --- a/llvm/include/llvm/MCA/Stages/InstructionTables.h +++ b/llvm/include/llvm/MCA/Stages/InstructionTables.h @@ -32,7 +32,7 @@ class InstructionTables final : public Stage { public: InstructionTables(const MCSchedModel &Model) - : Stage(), SM(Model), Masks(Model.getNumProcResourceKinds()) { + : SM(Model), Masks(Model.getNumProcResourceKinds()) { computeProcResourceMasks(Model, Masks); } diff --git a/llvm/include/llvm/MCA/Stages/RetireStage.h b/llvm/include/llvm/MCA/Stages/RetireStage.h index b635a01db85e..aafe2815df15 100644 --- a/llvm/include/llvm/MCA/Stages/RetireStage.h +++ b/llvm/include/llvm/MCA/Stages/RetireStage.h @@ -36,7 +36,7 @@ class RetireStage final : public Stage { public: RetireStage(RetireControlUnit &R, RegisterFile &F, LSUnitBase &LS) - : Stage(), RCU(R), PRF(F), LSU(LS) {} + : RCU(R), PRF(F), LSU(LS) {} bool hasWorkToComplete() const override { return !RCU.isEmpty(); } Error cycleStart() override; diff --git a/llvm/include/llvm/Object/Archive.h b/llvm/include/llvm/Object/Archive.h index 5b024c7baebc..5a5fc90f18bd 100644 --- a/llvm/include/llvm/Object/Archive.h +++ b/llvm/include/llvm/Object/Archive.h @@ -32,50 +32,127 @@ namespace llvm { namespace object { +const char ArchiveMagic[] = "!<arch>\n"; +const char ThinArchiveMagic[] = "!<thin>\n"; +const char BigArchiveMagic[] = "<bigaf>\n"; + class Archive; -class ArchiveMemberHeader { +class AbstractArchiveMemberHeader { +protected: + AbstractArchiveMemberHeader(const Archive *Parent) : Parent(Parent){}; + public: friend class Archive; - - ArchiveMemberHeader(Archive const *Parent, const char *RawHeaderPtr, - uint64_t Size, Error *Err); - // ArchiveMemberHeader() = default; + virtual std::unique_ptr<AbstractArchiveMemberHeader> clone() const = 0; + virtual ~AbstractArchiveMemberHeader(){}; /// Get the name without looking up long names. - Expected<StringRef> getRawName() const; + virtual Expected<StringRef> getRawName() const = 0; + virtual StringRef getRawAccessMode() const = 0; + virtual StringRef getRawLastModified() const = 0; + virtual StringRef getRawUID() const = 0; + virtual StringRef getRawGID() const = 0; /// Get the name looking up long names. - Expected<StringRef> getName(uint64_t Size) const; + virtual Expected<StringRef> getName(uint64_t Size) const = 0; + virtual Expected<uint64_t> getSize() const = 0; + virtual uint64_t getOffset() const = 0; - Expected<uint64_t> getSize() const; + /// Get next file member location. + virtual Expected<const char *> getNextChildLoc() const = 0; + virtual Expected<bool> isThin() const = 0; Expected<sys::fs::perms> getAccessMode() const; Expected<sys::TimePoint<std::chrono::seconds>> getLastModified() const; + Expected<unsigned> getUID() const; + Expected<unsigned> getGID() const; + + /// Returns the size in bytes of the format-defined member header of the + /// concrete archive type. + virtual uint64_t getSizeOf() const = 0; + + const Archive *Parent; +}; + +template <typename T> +class CommonArchiveMemberHeader : public AbstractArchiveMemberHeader { +public: + CommonArchiveMemberHeader(const Archive *Parent, const T *RawHeaderPtr) + : AbstractArchiveMemberHeader(Parent), ArMemHdr(RawHeaderPtr){}; + StringRef getRawAccessMode() const override; + StringRef getRawLastModified() const override; + StringRef getRawUID() const override; + StringRef getRawGID() const override; + + uint64_t getOffset() const override; + uint64_t getSizeOf() const override { return sizeof(T); } + + T const *ArMemHdr; +}; - StringRef getRawLastModified() const { - return StringRef(ArMemHdr->LastModified, sizeof(ArMemHdr->LastModified)) - .rtrim(' '); +struct UnixArMemHdrType { + char Name[16]; + char LastModified[12]; + char UID[6]; + char GID[6]; + char AccessMode[8]; + char Size[10]; ///< Size of data, not including header or padding. + char Terminator[2]; +}; + +class ArchiveMemberHeader : public CommonArchiveMemberHeader<UnixArMemHdrType> { +public: + ArchiveMemberHeader(const Archive *Parent, const char *RawHeaderPtr, + uint64_t Size, Error *Err); + + std::unique_ptr<AbstractArchiveMemberHeader> clone() const override { + return std::make_unique<ArchiveMemberHeader>(*this); } - Expected<unsigned> getUID() const; - Expected<unsigned> getGID() const; + Expected<StringRef> getRawName() const override; - // This returns the size of the private struct ArMemHdrType - uint64_t getSizeOf() const { return sizeof(ArMemHdrType); } + Expected<StringRef> getName(uint64_t Size) const override; + Expected<uint64_t> getSize() const override; + Expected<const char *> getNextChildLoc() const override; + Expected<bool> isThin() const override; +}; -private: - struct ArMemHdrType { - char Name[16]; - char LastModified[12]; - char UID[6]; - char GID[6]; - char AccessMode[8]; - char Size[10]; ///< Size of data, not including header or padding. +// File Member Header +struct BigArMemHdrType { + char Size[20]; // File member size in decimal + char NextOffset[20]; // Next member offset in decimal + char PrevOffset[20]; // Previous member offset in decimal + char LastModified[12]; + char UID[12]; + char GID[12]; + char AccessMode[12]; + char NameLen[4]; // File member name length in decimal + union { + char Name[2]; // Start of member name char Terminator[2]; }; - Archive const *Parent; - ArMemHdrType const *ArMemHdr; +}; + +// Define file member header of AIX big archive. +class BigArchiveMemberHeader + : public CommonArchiveMemberHeader<BigArMemHdrType> { + +public: + BigArchiveMemberHeader(Archive const *Parent, const char *RawHeaderPtr, + uint64_t Size, Error *Err); + std::unique_ptr<AbstractArchiveMemberHeader> clone() const override { + return std::make_unique<BigArchiveMemberHeader>(*this); + } + + Expected<StringRef> getRawName() const override; + Expected<uint64_t> getRawNameSize() const; + + Expected<StringRef> getName(uint64_t Size) const override; + Expected<uint64_t> getSize() const override; + Expected<const char *> getNextChildLoc() const override; + Expected<uint64_t> getNextOffset() const; + Expected<bool> isThin() const override { return false; } }; class Archive : public Binary { @@ -84,10 +161,10 @@ class Archive : public Binary { public: class Child { friend Archive; - friend ArchiveMemberHeader; + friend AbstractArchiveMemberHeader; const Archive *Parent; - ArchiveMemberHeader Header; + std::unique_ptr<AbstractArchiveMemberHeader> Header; /// Includes header but not padding byte. StringRef Data; /// Offset from Data to the start of the file. @@ -99,6 +176,44 @@ public: Child(const Archive *Parent, const char *Start, Error *Err); Child(const Archive *Parent, StringRef Data, uint16_t StartOfFile); + Child(const Child &C) + : Parent(C.Parent), Data(C.Data), StartOfFile(C.StartOfFile) { + if (C.Header) + Header = C.Header->clone(); + } + + Child(Child &&C) { + Parent = std::move(C.Parent); + Header = std::move(C.Header); + Data = C.Data; + StartOfFile = C.StartOfFile; + } + + Child &operator=(Child &&C) noexcept { + if (&C == this) + return *this; + + Parent = std::move(C.Parent); + Header = std::move(C.Header); + Data = C.Data; + StartOfFile = C.StartOfFile; + + return *this; + } + + Child &operator=(const Child &C) { + if (&C == this) + return *this; + + Parent = C.Parent; + if (C.Header) + Header = C.Header->clone(); + Data = C.Data; + StartOfFile = C.StartOfFile; + + return *this; + } + bool operator==(const Child &other) const { assert(!Parent || !other.Parent || Parent == other.Parent); return Data.begin() == other.Data.begin(); @@ -109,19 +224,21 @@ public: Expected<StringRef> getName() const; Expected<std::string> getFullName() const; - Expected<StringRef> getRawName() const { return Header.getRawName(); } + Expected<StringRef> getRawName() const { return Header->getRawName(); } Expected<sys::TimePoint<std::chrono::seconds>> getLastModified() const { - return Header.getLastModified(); + return Header->getLastModified(); } - StringRef getRawLastModified() const { return Header.getRawLastModified(); } + StringRef getRawLastModified() const { + return Header->getRawLastModified(); + } - Expected<unsigned> getUID() const { return Header.getUID(); } - Expected<unsigned> getGID() const { return Header.getGID(); } + Expected<unsigned> getUID() const { return Header->getUID(); } + Expected<unsigned> getGID() const { return Header->getGID(); } Expected<sys::fs::perms> getAccessMode() const { - return Header.getAccessMode(); + return Header->getAccessMode(); } /// \return the size of the archive member without the header or padding. @@ -218,7 +335,7 @@ public: /// Size field is 10 decimal digits long static const uint64_t MaxMemberSize = 9999999999; - enum Kind { K_GNU, K_GNU64, K_BSD, K_DARWIN, K_DARWIN64, K_COFF }; + enum Kind { K_GNU, K_GNU64, K_BSD, K_DARWIN, K_DARWIN64, K_COFF, K_AIXBIG }; Kind kind() const { return (Kind)Format; } bool isThin() const { return IsThin; } @@ -236,7 +353,6 @@ public: return make_range(symbol_begin(), symbol_end()); } - // Cast methods. static bool classof(Binary const *v) { return v->isArchive(); } // check if a symbol is in the archive @@ -247,24 +363,55 @@ public: StringRef getSymbolTable() const { return SymbolTable; } StringRef getStringTable() const { return StringTable; } uint32_t getNumberOfSymbols() const; + virtual uint64_t getFirstChildOffset() const { return getArchiveMagicLen(); } std::vector<std::unique_ptr<MemoryBuffer>> takeThinBuffers() { return std::move(ThinBuffers); } + std::unique_ptr<AbstractArchiveMemberHeader> + createArchiveMemberHeader(const char *RawHeaderPtr, uint64_t Size, + Error *Err) const; + +protected: + uint64_t getArchiveMagicLen() const; + void setFirstRegular(const Child &C); + private: StringRef SymbolTable; StringRef StringTable; StringRef FirstRegularData; uint16_t FirstRegularStartOfFile = -1; - void setFirstRegular(const Child &C); unsigned Format : 3; unsigned IsThin : 1; mutable std::vector<std::unique_ptr<MemoryBuffer>> ThinBuffers; }; +class BigArchive : public Archive { + /// Fixed-Length Header. + struct FixLenHdr { + char Magic[sizeof(BigArchiveMagic) - 1]; ///< Big archive magic string. + char MemOffset[20]; ///< Offset to member table. + char GlobSymOffset[20]; ///< Offset to global symbol table. + char + GlobSym64Offset[20]; ///< Offset global symbol table for 64-bit objects. + char FirstChildOffset[20]; ///< Offset to first archive member. + char LastChildOffset[20]; ///< Offset to last archive member. + char FreeOffset[20]; ///< Offset to first mem on free list. + }; + + const FixLenHdr *ArFixLenHdr; + uint64_t FirstChildOffset = 0; + uint64_t LastChildOffset = 0; + +public: + BigArchive(MemoryBufferRef Source, Error &Err); + uint64_t getFirstChildOffset() const override { return FirstChildOffset; } + uint64_t getLastChildOffset() const { return LastChildOffset; } +}; + } // end namespace object } // end namespace llvm diff --git a/llvm/include/llvm/Object/ELFObjectFile.h b/llvm/include/llvm/Object/ELFObjectFile.h index 716b94d92d03..e2d2784d4f23 100644 --- a/llvm/include/llvm/Object/ELFObjectFile.h +++ b/llvm/include/llvm/Object/ELFObjectFile.h @@ -34,6 +34,7 @@ #include "llvm/Support/Error.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/ScopedPrinter.h" #include <cassert> #include <cstdint> #include <system_error> diff --git a/llvm/include/llvm/Object/Error.h b/llvm/include/llvm/Object/Error.h index 1fc1f6603a36..af334fc42658 100644 --- a/llvm/include/llvm/Object/Error.h +++ b/llvm/include/llvm/Object/Error.h @@ -22,8 +22,6 @@ class Twine; namespace object { -class Binary; - const std::error_category &object_category(); enum class object_error { diff --git a/llvm/include/llvm/Object/IRObjectFile.h b/llvm/include/llvm/Object/IRObjectFile.h index 338b1941eca1..db47960237a0 100644 --- a/llvm/include/llvm/Object/IRObjectFile.h +++ b/llvm/include/llvm/Object/IRObjectFile.h @@ -20,10 +20,7 @@ namespace llvm { class BitcodeModule; -class Mangler; class Module; -class GlobalValue; -class Triple; namespace object { class ObjectFile; diff --git a/llvm/include/llvm/Object/MachOUniversal.h b/llvm/include/llvm/Object/MachOUniversal.h index 9bcacb510108..e87eb31aad4e 100644 --- a/llvm/include/llvm/Object/MachOUniversal.h +++ b/llvm/include/llvm/Object/MachOUniversal.h @@ -22,7 +22,6 @@ namespace llvm { class StringRef; -class Module; class LLVMContext; namespace object { diff --git a/llvm/include/llvm/Object/ObjectFile.h b/llvm/include/llvm/Object/ObjectFile.h index 267fe3046738..12704b1fc88e 100644 --- a/llvm/include/llvm/Object/ObjectFile.h +++ b/llvm/include/llvm/Object/ObjectFile.h @@ -31,7 +31,6 @@ namespace llvm { -class ARMAttributeParser; class SubtargetFeatures; namespace object { diff --git a/llvm/include/llvm/Object/XCOFFObjectFile.h b/llvm/include/llvm/Object/XCOFFObjectFile.h index 94136afc45ea..ac911e534f34 100644 --- a/llvm/include/llvm/Object/XCOFFObjectFile.h +++ b/llvm/include/llvm/Object/XCOFFObjectFile.h @@ -348,6 +348,57 @@ struct XCOFFSectAuxEntForStat { uint8_t Pad[10]; }; // 32-bit XCOFF file only. +struct XCOFFFunctionAuxEnt32 { + support::ubig32_t OffsetToExceptionTbl; + support::ubig32_t SizeOfFunction; + support::ubig32_t PtrToLineNum; + support::big32_t SymIdxOfNextBeyond; + uint8_t Pad[2]; +}; + +struct XCOFFFunctionAuxEnt64 { + support::ubig64_t PtrToLineNum; + support::ubig32_t SizeOfFunction; + support::big32_t SymIdxOfNextBeyond; + uint8_t Pad; + XCOFF::SymbolAuxType AuxType; // Contains _AUX_FCN; Type of auxiliary entry +}; + +struct XCOFFExceptionAuxEnt { + support::ubig64_t OffsetToExceptionTbl; + support::ubig32_t SizeOfFunction; + support::big32_t SymIdxOfNextBeyond; + uint8_t Pad; + XCOFF::SymbolAuxType AuxType; // Contains _AUX_EXCEPT; Type of auxiliary entry +}; + +struct XCOFFBlockAuxEnt32 { + uint8_t ReservedZeros1[2]; + support::ubig16_t LineNumHi; + support::ubig16_t LineNumLo; + uint8_t ReservedZeros2[12]; +}; + +struct XCOFFBlockAuxEnt64 { + support::ubig32_t LineNum; + uint8_t Pad[13]; + XCOFF::SymbolAuxType AuxType; // Contains _AUX_SYM; Type of auxiliary entry +}; + +struct XCOFFSectAuxEntForDWARF32 { + support::ubig32_t LengthOfSectionPortion; + uint8_t Pad1[4]; + support::ubig32_t NumberOfRelocEnt; + uint8_t Pad2[6]; +}; + +struct XCOFFSectAuxEntForDWARF64 { + support::ubig64_t LengthOfSectionPortion; + support::ubig64_t NumberOfRelocEnt; + uint8_t Pad; + XCOFF::SymbolAuxType AuxType; // Contains _AUX_SECT; Type of Auxillary entry +}; + template <typename AddressType> struct XCOFFRelocation { // Masks for packing/unpacking the r_rsize field of relocations. diff --git a/llvm/include/llvm/ObjectYAML/DWARFEmitter.h b/llvm/include/llvm/ObjectYAML/DWARFEmitter.h index eb56d1e29326..30bb16deb810 100644 --- a/llvm/include/llvm/ObjectYAML/DWARFEmitter.h +++ b/llvm/include/llvm/ObjectYAML/DWARFEmitter.h @@ -26,7 +26,6 @@ class raw_ostream; namespace DWARFYAML { struct Data; -struct PubSection; Error emitDebugAbbrev(raw_ostream &OS, const Data &DI); Error emitDebugStr(raw_ostream &OS, const Data &DI); diff --git a/llvm/include/llvm/ObjectYAML/XCOFFYAML.h b/llvm/include/llvm/ObjectYAML/XCOFFYAML.h index aa1bc396f134..39bc334e1a89 100644 --- a/llvm/include/llvm/ObjectYAML/XCOFFYAML.h +++ b/llvm/include/llvm/ObjectYAML/XCOFFYAML.h @@ -82,6 +82,110 @@ struct Section { std::vector<Relocation> Relocations; }; +enum AuxSymbolType : uint8_t { + AUX_EXCEPT = 255, + AUX_FCN = 254, + AUX_SYM = 253, + AUX_FILE = 252, + AUX_CSECT = 251, + AUX_SECT = 250, + AUX_STAT = 249 +}; + +struct AuxSymbolEnt { + AuxSymbolType Type; + + explicit AuxSymbolEnt(AuxSymbolType T) : Type(T) {} + virtual ~AuxSymbolEnt(); +}; + +struct FileAuxEnt : AuxSymbolEnt { + Optional<StringRef> FileNameOrString; + Optional<XCOFF::CFileStringType> FileStringType; + + FileAuxEnt() : AuxSymbolEnt(AuxSymbolType::AUX_FILE) {} + static bool classof(const AuxSymbolEnt *S) { + return S->Type == AuxSymbolType::AUX_FILE; + } +}; + +struct CsectAuxEnt : AuxSymbolEnt { + // Only for XCOFF32. + Optional<uint32_t> SectionOrLength; + Optional<uint32_t> StabInfoIndex; + Optional<uint16_t> StabSectNum; + // Only for XCOFF64. + Optional<uint32_t> SectionOrLengthLo; + Optional<uint32_t> SectionOrLengthHi; + // Common fields for both XCOFF32 and XCOFF64. + Optional<uint32_t> ParameterHashIndex; + Optional<uint16_t> TypeChkSectNum; + Optional<uint8_t> SymbolAlignmentAndType; + Optional<XCOFF::StorageMappingClass> StorageMappingClass; + + CsectAuxEnt() : AuxSymbolEnt(AuxSymbolType::AUX_CSECT) {} + static bool classof(const AuxSymbolEnt *S) { + return S->Type == AuxSymbolType::AUX_CSECT; + } +}; + +struct FunctionAuxEnt : AuxSymbolEnt { + Optional<uint32_t> OffsetToExceptionTbl; // Only for XCOFF32. + Optional<uint64_t> PtrToLineNum; + Optional<uint32_t> SizeOfFunction; + Optional<int32_t> SymIdxOfNextBeyond; + + FunctionAuxEnt() : AuxSymbolEnt(AuxSymbolType::AUX_FCN) {} + static bool classof(const AuxSymbolEnt *S) { + return S->Type == AuxSymbolType::AUX_FCN; + } +}; + +struct ExcpetionAuxEnt : AuxSymbolEnt { + Optional<uint64_t> OffsetToExceptionTbl; + Optional<uint32_t> SizeOfFunction; + Optional<int32_t> SymIdxOfNextBeyond; + + ExcpetionAuxEnt() : AuxSymbolEnt(AuxSymbolType::AUX_EXCEPT) {} + static bool classof(const AuxSymbolEnt *S) { + return S->Type == AuxSymbolType::AUX_EXCEPT; + } +}; // Only for XCOFF64. + +struct BlockAuxEnt : AuxSymbolEnt { + // Only for XCOFF32. + Optional<uint16_t> LineNumHi; + Optional<uint16_t> LineNumLo; + // Only for XCOFF64. + Optional<uint32_t> LineNum; + + BlockAuxEnt() : AuxSymbolEnt(AuxSymbolType::AUX_SYM) {} + static bool classof(const AuxSymbolEnt *S) { + return S->Type == AuxSymbolType::AUX_SYM; + } +}; + +struct SectAuxEntForDWARF : AuxSymbolEnt { + Optional<uint32_t> LengthOfSectionPortion; + Optional<uint32_t> NumberOfRelocEnt; + + SectAuxEntForDWARF() : AuxSymbolEnt(AuxSymbolType::AUX_SECT) {} + static bool classof(const AuxSymbolEnt *S) { + return S->Type == AuxSymbolType::AUX_SECT; + } +}; + +struct SectAuxEntForStat : AuxSymbolEnt { + Optional<uint32_t> SectionLength; + Optional<uint16_t> NumberOfRelocEnt; + Optional<uint16_t> NumberOfLineNum; + + SectAuxEntForStat() : AuxSymbolEnt(AuxSymbolType::AUX_STAT) {} + static bool classof(const AuxSymbolEnt *S) { + return S->Type == AuxSymbolType::AUX_STAT; + } +}; // Only for XCOFF32. + struct Symbol { StringRef SymbolName; llvm::yaml::Hex64 Value; // Symbol value; storage class-dependent. @@ -89,7 +193,8 @@ struct Symbol { Optional<uint16_t> SectionIndex; llvm::yaml::Hex16 Type; XCOFF::StorageClass StorageClass; - uint8_t NumberOfAuxEntries; + Optional<uint8_t> NumberOfAuxEntries; + std::vector<std::unique_ptr<AuxSymbolEnt>> AuxEntries; }; struct StringTable { @@ -114,6 +219,7 @@ struct Object { LLVM_YAML_IS_SEQUENCE_VECTOR(XCOFFYAML::Symbol) LLVM_YAML_IS_SEQUENCE_VECTOR(XCOFFYAML::Relocation) LLVM_YAML_IS_SEQUENCE_VECTOR(XCOFFYAML::Section) +LLVM_YAML_IS_SEQUENCE_VECTOR(std::unique_ptr<llvm::XCOFFYAML::AuxSymbolEnt>) namespace llvm { namespace yaml { @@ -126,6 +232,18 @@ template <> struct ScalarEnumerationTraits<XCOFF::StorageClass> { static void enumeration(IO &IO, XCOFF::StorageClass &Value); }; +template <> struct ScalarEnumerationTraits<XCOFF::StorageMappingClass> { + static void enumeration(IO &IO, XCOFF::StorageMappingClass &Value); +}; + +template <> struct ScalarEnumerationTraits<XCOFF::CFileStringType> { + static void enumeration(IO &IO, XCOFF::CFileStringType &Type); +}; + +template <> struct ScalarEnumerationTraits<XCOFFYAML::AuxSymbolType> { + static void enumeration(IO &IO, XCOFFYAML::AuxSymbolType &Type); +}; + template <> struct MappingTraits<XCOFFYAML::FileHeader> { static void mapping(IO &IO, XCOFFYAML::FileHeader &H); }; @@ -134,6 +252,10 @@ template <> struct MappingTraits<XCOFFYAML::AuxiliaryHeader> { static void mapping(IO &IO, XCOFFYAML::AuxiliaryHeader &AuxHdr); }; +template <> struct MappingTraits<std::unique_ptr<XCOFFYAML::AuxSymbolEnt>> { + static void mapping(IO &IO, std::unique_ptr<XCOFFYAML::AuxSymbolEnt> &AuxSym); +}; + template <> struct MappingTraits<XCOFFYAML::Symbol> { static void mapping(IO &IO, XCOFFYAML::Symbol &S); }; diff --git a/llvm/include/llvm/Passes/StandardInstrumentations.h b/llvm/include/llvm/Passes/StandardInstrumentations.h index 6cab4ce7d138..9eb754a4d824 100644 --- a/llvm/include/llvm/Passes/StandardInstrumentations.h +++ b/llvm/include/llvm/Passes/StandardInstrumentations.h @@ -432,7 +432,7 @@ public: } // Return the label of the basic block reached on a transition on \p S. - const StringRef getSuccessorLabel(StringRef S) const { + StringRef getSuccessorLabel(StringRef S) const { assert(Successors.count(S) == 1 && "Expected to find successor."); return Successors.find(S)->getValue(); } diff --git a/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h b/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h index d3a5d44ce8dd..e1f45019b1a9 100644 --- a/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h +++ b/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h @@ -702,7 +702,7 @@ public: LineCoverageIterator(const CoverageData &CD, unsigned Line) : CD(CD), WrappedSegment(nullptr), Next(CD.begin()), Ended(false), - Line(Line), Segments(), Stats() { + Line(Line) { this->operator++(); } diff --git a/llvm/include/llvm/ProfileData/Coverage/CoverageMappingReader.h b/llvm/include/llvm/ProfileData/Coverage/CoverageMappingReader.h index 242ffdd16e3f..39c0045369be 100644 --- a/llvm/include/llvm/ProfileData/Coverage/CoverageMappingReader.h +++ b/llvm/include/llvm/ProfileData/Coverage/CoverageMappingReader.h @@ -56,10 +56,10 @@ public: using reference = value_type &; CoverageMappingIterator() - : Reader(nullptr), Record(), ReadErr(coveragemap_error::success) {} + : Reader(nullptr), ReadErr(coveragemap_error::success) {} CoverageMappingIterator(CoverageMappingReader *Reader) - : Reader(Reader), Record(), ReadErr(coveragemap_error::success) { + : Reader(Reader), ReadErr(coveragemap_error::success) { increment(); } diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h index 6c5efb2f6d5d..4d3bb0e8ff10 100644 --- a/llvm/include/llvm/ProfileData/InstrProf.h +++ b/llvm/include/llvm/ProfileData/InstrProf.h @@ -293,7 +293,6 @@ enum class instrprof_error { missing_debug_info_for_correlation, unexpected_debug_info_for_correlation, unable_to_correlate_profile, - unsupported_debug_format, unknown_function, invalid_prof, hash_mismatch, @@ -530,6 +529,12 @@ public: /// Return the name section data. inline StringRef getNameData() const { return Data; } + + /// Dump the symbols in this table. + void dumpNames(raw_ostream &OS) const { + for (StringRef S : NameTab.keys()) + OS << S << "\n"; + } }; Error InstrProfSymtab::create(StringRef D, uint64_t BaseAddr) { diff --git a/llvm/include/llvm/ProfileData/InstrProfCorrelator.h b/llvm/include/llvm/ProfileData/InstrProfCorrelator.h index eae7b4e0322c..135936b99f24 100644 --- a/llvm/include/llvm/ProfileData/InstrProfCorrelator.h +++ b/llvm/include/llvm/ProfileData/InstrProfCorrelator.h @@ -12,6 +12,7 @@ #ifndef LLVM_PROFILEDATA_INSTRPROFCORRELATOR_H #define LLVM_PROFILEDATA_INSTRPROFCORRELATOR_H +#include "llvm/ADT/DenseSet.h" #include "llvm/DebugInfo/DWARF/DWARFContext.h" #include "llvm/Object/Binary.h" #include "llvm/Object/ObjectFile.h" @@ -34,6 +35,20 @@ public: /// to their functions. virtual Error correlateProfileData() = 0; + /// Return the number of ProfileData elements. + llvm::Optional<size_t> getDataSize() const; + + /// Return a pointer to the names string that this class constructs. + const char *getNamesPointer() const { return Names.c_str(); } + + /// Return the number of bytes in the names string. + size_t getNamesSize() const { return Names.size(); } + + /// Return the size of the counters section in bytes. + uint64_t getCountersSectionSize() const { + return Ctx->CountersSectionEnd - Ctx->CountersSectionStart; + } + static const char *FunctionNameAttributeName; static const char *CFGHashAttributeName; static const char *NumCountersAttributeName; @@ -58,6 +73,9 @@ protected: InstrProfCorrelator(InstrProfCorrelatorKind K, std::unique_ptr<Context> Ctx) : Ctx(std::move(Ctx)), Kind(K) {} + std::string Names; + std::vector<std::string> NamesVec; + private: static llvm::Expected<std::unique_ptr<InstrProfCorrelator>> get(std::unique_ptr<MemoryBuffer> Buffer); @@ -82,22 +100,12 @@ public: /// Return the number of ProfileData elements. size_t getDataSize() const { return Data.size(); } - /// Return a pointer to the compressed names string that this class - /// constructs. - const char *getCompressedNamesPointer() const { - return CompressedNames.c_str(); - } - - /// Return the number of bytes in the compressed names string. - size_t getCompressedNamesSize() const { return CompressedNames.size(); } - static llvm::Expected<std::unique_ptr<InstrProfCorrelatorImpl<IntPtrT>>> get(std::unique_ptr<InstrProfCorrelator::Context> Ctx, const object::ObjectFile &Obj); protected: std::vector<RawInstrProf::ProfileData<IntPtrT>> Data; - std::string CompressedNames; Error correlateProfileData() override; virtual void correlateProfileDataImpl() = 0; @@ -109,7 +117,7 @@ private: InstrProfCorrelatorImpl(InstrProfCorrelatorKind Kind, std::unique_ptr<InstrProfCorrelator::Context> Ctx) : InstrProfCorrelator(Kind, std::move(Ctx)){}; - std::vector<std::string> Names; + llvm::DenseSet<IntPtrT> CounterOffsets; // Byte-swap the value if necessary. template <class T> T maybeSwap(T Value) const { @@ -138,7 +146,7 @@ private: static bool isDIEOfProbe(const DWARFDie &Die); /// Iterate over DWARF DIEs to find those that symbolize instrumentation - /// probes and construct the ProfileData vector and CompressedNames string. + /// probes and construct the ProfileData vector and Names string. /// /// Here is some example DWARF for an instrumentation probe we are looking /// for: diff --git a/llvm/include/llvm/ProfileData/InstrProfData.inc b/llvm/include/llvm/ProfileData/InstrProfData.inc index 44719126b596..0544b6b2ef71 100644 --- a/llvm/include/llvm/ProfileData/InstrProfData.inc +++ b/llvm/include/llvm/ProfileData/InstrProfData.inc @@ -128,8 +128,10 @@ INSTR_PROF_VALUE_NODE(PtrToNodeT, llvm::Type::getInt8PtrTy(Ctx), Next, \ INSTR_PROF_RAW_HEADER(uint64_t, Magic, __llvm_profile_get_magic()) INSTR_PROF_RAW_HEADER(uint64_t, Version, __llvm_profile_get_version()) INSTR_PROF_RAW_HEADER(uint64_t, BinaryIdsSize, __llvm_write_binary_ids(NULL)) +/* FIXME: A more accurate name is NumData */ INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize) INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesBeforeCounters, PaddingBytesBeforeCounters) +/* FIXME: A more accurate name is NumCounters */ INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize) INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterCounters, PaddingBytesAfterCounters) INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) @@ -644,6 +646,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \ (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129 +/* FIXME: Please remedy the fixme in the header before bumping the version. */ /* Raw profile format version (start from 1). */ #define INSTR_PROF_RAW_VERSION 8 /* Indexed profile format version (start from 1). */ diff --git a/llvm/include/llvm/ProfileData/InstrProfReader.h b/llvm/include/llvm/ProfileData/InstrProfReader.h index c615e8533178..1326cbf0e1ce 100644 --- a/llvm/include/llvm/ProfileData/InstrProfReader.h +++ b/llvm/include/llvm/ProfileData/InstrProfReader.h @@ -233,7 +233,8 @@ private: uint64_t NamesDelta; const RawInstrProf::ProfileData<IntPtrT> *Data; const RawInstrProf::ProfileData<IntPtrT> *DataEnd; - const uint64_t *CountersStart; + const char *CountersStart; + const char *CountersEnd; const char *NamesStart; const char *NamesEnd; // After value profile is all read, this pointer points to @@ -310,6 +311,15 @@ private: bool atEnd() const { return Data == DataEnd; } void advanceData() { + // `CountersDelta` is a constant zero when using debug info correlation. + if (!Correlator) { + // The initial CountersDelta is the in-memory address difference between + // the data and counts sections: + // start(__llvm_prf_cnts) - start(__llvm_prf_data) + // As we advance to the next record, we maintain the correct CountersDelta + // with respect to the next record. + CountersDelta -= sizeof(*Data); + } Data++; ValueDataStart += CurValueDataSize; } @@ -319,20 +329,11 @@ private: return (const char *)ValueDataStart; } - /// Get the offset of \p CounterPtr from the start of the counters section of - /// the profile. The offset has units of "number of counters", i.e. increasing - /// the offset by 1 corresponds to an increase in the *byte offset* by 8. - ptrdiff_t getCounterOffset(IntPtrT CounterPtr) const { - return (swap(CounterPtr) - CountersDelta) / sizeof(uint64_t); - } - - const uint64_t *getCounter(ptrdiff_t Offset) const { - return CountersStart + Offset; - } - StringRef getName(uint64_t NameRef) const { return Symtab->getFuncName(swap(NameRef)); } + + int getCounterTypeSize() const { return sizeof(uint64_t); } }; using RawInstrProfReader32 = RawInstrProfReader<uint32_t>; diff --git a/llvm/include/llvm/ProfileData/MemProfData.inc b/llvm/include/llvm/ProfileData/MemProfData.inc index d64227e4ba31..f2cb3738f053 100644 --- a/llvm/include/llvm/ProfileData/MemProfData.inc +++ b/llvm/include/llvm/ProfileData/MemProfData.inc @@ -1,5 +1,5 @@ -#ifndef MEMPROF_DATA_INC -#define MEMPROF_DATA_INC +#ifndef LLVM_PROFILEDATA_MEMPROFDATA_INC +#define LLVM_PROFILEDATA_MEMPROFDATA_INC /*===-- MemProfData.inc - MemProf profiling runtime structures -*- C++ -*-=== *\ |* |* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h index dc6522f4ec4c..bad2139fe8f0 100644 --- a/llvm/include/llvm/ProfileData/SampleProf.h +++ b/llvm/include/llvm/ProfileData/SampleProf.h @@ -195,19 +195,21 @@ enum class SecProfSummaryFlags : uint32_t { /// The common profile is usually merged from profiles collected /// from running other targets. SecFlagPartial = (1 << 0), - /// SecFlagContext means this is context-sensitive profile for + /// SecFlagContext means this is context-sensitive flat profile for /// CSSPGO SecFlagFullContext = (1 << 1), /// SecFlagFSDiscriminator means this profile uses flow-sensitive /// discriminators. - SecFlagFSDiscriminator = (1 << 2) + SecFlagFSDiscriminator = (1 << 2), + /// SecFlagIsCSNested means this is context-sensitive nested profile for + /// CSSPGO + SecFlagIsCSNested = (1 << 4), }; enum class SecFuncMetadataFlags : uint32_t { SecFlagInvalid = 0, SecFlagIsProbeBased = (1 << 0), SecFlagHasAttribute = (1 << 1), - SecFlagIsCSNested = (1 << 2), }; enum class SecFuncOffsetFlags : uint32_t { @@ -448,7 +450,7 @@ static inline hash_code hash_value(const SampleContextFrame &arg) { arg.Location.Discriminator); } -using SampleContextFrameVector = SmallVector<SampleContextFrame, 10>; +using SampleContextFrameVector = SmallVector<SampleContextFrame, 1>; using SampleContextFrames = ArrayRef<SampleContextFrame>; struct SampleContextFrameHash { diff --git a/llvm/include/llvm/Remarks/RemarkSerializer.h b/llvm/include/llvm/Remarks/RemarkSerializer.h index 97fd224ea082..90e556df87e7 100644 --- a/llvm/include/llvm/Remarks/RemarkSerializer.h +++ b/llvm/include/llvm/Remarks/RemarkSerializer.h @@ -48,7 +48,7 @@ struct RemarkSerializer { RemarkSerializer(Format SerializerFormat, raw_ostream &OS, SerializerMode Mode) - : SerializerFormat(SerializerFormat), OS(OS), Mode(Mode), StrTab() {} + : SerializerFormat(SerializerFormat), OS(OS), Mode(Mode) {} /// This is just an interface. virtual ~RemarkSerializer() = default; diff --git a/llvm/include/llvm/Support/AArch64TargetParser.def b/llvm/include/llvm/Support/AArch64TargetParser.def index 48e82fa55a0f..26f4bae53119 100644 --- a/llvm/include/llvm/Support/AArch64TargetParser.def +++ b/llvm/include/llvm/Support/AArch64TargetParser.def @@ -58,6 +58,13 @@ AARCH64_ARCH("armv8.7-a", ARMV8_7A, "8.7-A", "v8.7a", AArch64::AEK_RDM | AArch64::AEK_RCPC | AArch64::AEK_DOTPROD | AArch64::AEK_SM4 | AArch64::AEK_SHA3 | AArch64::AEK_BF16 | AArch64::AEK_SHA2 | AArch64::AEK_AES | AArch64::AEK_I8MM)) +AARCH64_ARCH("armv8.8-a", ARMV8_8A, "8.8-A", "v8.8a", + ARMBuildAttrs::CPUArch::v8_A, FK_CRYPTO_NEON_FP_ARMV8, + (AArch64::AEK_CRC | AArch64::AEK_FP | + AArch64::AEK_SIMD | AArch64::AEK_RAS | AArch64::AEK_LSE | + AArch64::AEK_RDM | AArch64::AEK_RCPC | AArch64::AEK_DOTPROD | + AArch64::AEK_SM4 | AArch64::AEK_SHA3 | AArch64::AEK_BF16 | + AArch64::AEK_SHA2 | AArch64::AEK_AES | AArch64::AEK_I8MM)) AARCH64_ARCH("armv9-a", ARMV9A, "9-A", "v9a", ARMBuildAttrs::CPUArch::v8_A, FK_NEON_FP_ARMV8, (AArch64::AEK_CRC | AArch64::AEK_FP | @@ -76,6 +83,12 @@ AARCH64_ARCH("armv9.2-a", ARMV9_2A, "9.2-A", "v9.2a", AArch64::AEK_SIMD | AArch64::AEK_RAS | AArch64::AEK_LSE | AArch64::AEK_RDM | AArch64::AEK_RCPC | AArch64::AEK_DOTPROD | AArch64::AEK_SVE2)) +AARCH64_ARCH("armv9.3-a", ARMV9_3A, "9.3-A", "v9.3", + ARMBuildAttrs::CPUArch::v8_A, FK_CRYPTO_NEON_FP_ARMV8, + (AArch64::AEK_CRC | AArch64::AEK_FP | + AArch64::AEK_SIMD | AArch64::AEK_RAS | AArch64::AEK_LSE | + AArch64::AEK_RDM | AArch64::AEK_RCPC | AArch64::AEK_DOTPROD | + AArch64::AEK_SVE2)) // For v8-R, we do not enable crypto and align with GCC that enables a more // minimal set of optional architecture extensions. AARCH64_ARCH("armv8-r", ARMV8R, "8-R", "v8r", @@ -90,47 +103,50 @@ AARCH64_ARCH("armv8-r", ARMV8R, "8-R", "v8r", #define AARCH64_ARCH_EXT_NAME(NAME, ID, FEATURE, NEGFEATURE) #endif // FIXME: This would be nicer were it tablegen -AARCH64_ARCH_EXT_NAME("invalid", AArch64::AEK_INVALID, nullptr, nullptr) -AARCH64_ARCH_EXT_NAME("none", AArch64::AEK_NONE, nullptr, nullptr) -AARCH64_ARCH_EXT_NAME("crc", AArch64::AEK_CRC, "+crc", "-crc") -AARCH64_ARCH_EXT_NAME("lse", AArch64::AEK_LSE, "+lse", "-lse") -AARCH64_ARCH_EXT_NAME("rdm", AArch64::AEK_RDM, "+rdm", "-rdm") -AARCH64_ARCH_EXT_NAME("crypto", AArch64::AEK_CRYPTO, "+crypto","-crypto") -AARCH64_ARCH_EXT_NAME("sm4", AArch64::AEK_SM4, "+sm4", "-sm4") -AARCH64_ARCH_EXT_NAME("sha3", AArch64::AEK_SHA3, "+sha3", "-sha3") -AARCH64_ARCH_EXT_NAME("sha2", AArch64::AEK_SHA2, "+sha2", "-sha2") -AARCH64_ARCH_EXT_NAME("aes", AArch64::AEK_AES, "+aes", "-aes") -AARCH64_ARCH_EXT_NAME("dotprod", AArch64::AEK_DOTPROD, "+dotprod","-dotprod") -AARCH64_ARCH_EXT_NAME("fp", AArch64::AEK_FP, "+fp-armv8", "-fp-armv8") -AARCH64_ARCH_EXT_NAME("simd", AArch64::AEK_SIMD, "+neon", "-neon") -AARCH64_ARCH_EXT_NAME("fp16", AArch64::AEK_FP16, "+fullfp16", "-fullfp16") -AARCH64_ARCH_EXT_NAME("fp16fml", AArch64::AEK_FP16FML, "+fp16fml", "-fp16fml") -AARCH64_ARCH_EXT_NAME("profile", AArch64::AEK_PROFILE, "+spe", "-spe") -AARCH64_ARCH_EXT_NAME("ras", AArch64::AEK_RAS, "+ras", "-ras") -AARCH64_ARCH_EXT_NAME("sve", AArch64::AEK_SVE, "+sve", "-sve") -AARCH64_ARCH_EXT_NAME("sve2", AArch64::AEK_SVE2, "+sve2", "-sve2") -AARCH64_ARCH_EXT_NAME("sve2-aes", AArch64::AEK_SVE2AES, "+sve2-aes", "-sve2-aes") -AARCH64_ARCH_EXT_NAME("sve2-sm4", AArch64::AEK_SVE2SM4, "+sve2-sm4", "-sve2-sm4") -AARCH64_ARCH_EXT_NAME("sve2-sha3", AArch64::AEK_SVE2SHA3, "+sve2-sha3", "-sve2-sha3") +AARCH64_ARCH_EXT_NAME("invalid", AArch64::AEK_INVALID, nullptr, nullptr) +AARCH64_ARCH_EXT_NAME("none", AArch64::AEK_NONE, nullptr, nullptr) +AARCH64_ARCH_EXT_NAME("crc", AArch64::AEK_CRC, "+crc", "-crc") +AARCH64_ARCH_EXT_NAME("lse", AArch64::AEK_LSE, "+lse", "-lse") +AARCH64_ARCH_EXT_NAME("rdm", AArch64::AEK_RDM, "+rdm", "-rdm") +AARCH64_ARCH_EXT_NAME("crypto", AArch64::AEK_CRYPTO, "+crypto", "-crypto") +AARCH64_ARCH_EXT_NAME("sm4", AArch64::AEK_SM4, "+sm4", "-sm4") +AARCH64_ARCH_EXT_NAME("sha3", AArch64::AEK_SHA3, "+sha3", "-sha3") +AARCH64_ARCH_EXT_NAME("sha2", AArch64::AEK_SHA2, "+sha2", "-sha2") +AARCH64_ARCH_EXT_NAME("aes", AArch64::AEK_AES, "+aes", "-aes") +AARCH64_ARCH_EXT_NAME("dotprod", AArch64::AEK_DOTPROD, "+dotprod", "-dotprod") +AARCH64_ARCH_EXT_NAME("fp", AArch64::AEK_FP, "+fp-armv8", "-fp-armv8") +AARCH64_ARCH_EXT_NAME("simd", AArch64::AEK_SIMD, "+neon", "-neon") +AARCH64_ARCH_EXT_NAME("fp16", AArch64::AEK_FP16, "+fullfp16", "-fullfp16") +AARCH64_ARCH_EXT_NAME("fp16fml", AArch64::AEK_FP16FML, "+fp16fml", "-fp16fml") +AARCH64_ARCH_EXT_NAME("profile", AArch64::AEK_PROFILE, "+spe", "-spe") +AARCH64_ARCH_EXT_NAME("ras", AArch64::AEK_RAS, "+ras", "-ras") +AARCH64_ARCH_EXT_NAME("sve", AArch64::AEK_SVE, "+sve", "-sve") +AARCH64_ARCH_EXT_NAME("sve2", AArch64::AEK_SVE2, "+sve2", "-sve2") +AARCH64_ARCH_EXT_NAME("sve2-aes", AArch64::AEK_SVE2AES, "+sve2-aes", "-sve2-aes") +AARCH64_ARCH_EXT_NAME("sve2-sm4", AArch64::AEK_SVE2SM4, "+sve2-sm4", "-sve2-sm4") +AARCH64_ARCH_EXT_NAME("sve2-sha3", AArch64::AEK_SVE2SHA3, "+sve2-sha3", "-sve2-sha3") AARCH64_ARCH_EXT_NAME("sve2-bitperm", AArch64::AEK_SVE2BITPERM, "+sve2-bitperm", "-sve2-bitperm") -AARCH64_ARCH_EXT_NAME("rcpc", AArch64::AEK_RCPC, "+rcpc", "-rcpc") -AARCH64_ARCH_EXT_NAME("rng", AArch64::AEK_RAND, "+rand", "-rand") -AARCH64_ARCH_EXT_NAME("memtag", AArch64::AEK_MTE, "+mte", "-mte") -AARCH64_ARCH_EXT_NAME("ssbs", AArch64::AEK_SSBS, "+ssbs", "-ssbs") -AARCH64_ARCH_EXT_NAME("sb", AArch64::AEK_SB, "+sb", "-sb") -AARCH64_ARCH_EXT_NAME("predres", AArch64::AEK_PREDRES, "+predres", "-predres") -AARCH64_ARCH_EXT_NAME("bf16", AArch64::AEK_BF16, "+bf16", "-bf16") -AARCH64_ARCH_EXT_NAME("i8mm", AArch64::AEK_I8MM, "+i8mm", "-i8mm") -AARCH64_ARCH_EXT_NAME("f32mm", AArch64::AEK_F32MM, "+f32mm", "-f32mm") -AARCH64_ARCH_EXT_NAME("f64mm", AArch64::AEK_F64MM, "+f64mm", "-f64mm") -AARCH64_ARCH_EXT_NAME("tme", AArch64::AEK_TME, "+tme", "-tme") -AARCH64_ARCH_EXT_NAME("ls64", AArch64::AEK_LS64, "+ls64", "-ls64") -AARCH64_ARCH_EXT_NAME("brbe", AArch64::AEK_BRBE, "+brbe", "-brbe") -AARCH64_ARCH_EXT_NAME("pauth", AArch64::AEK_PAUTH, "+pauth", "-pauth") -AARCH64_ARCH_EXT_NAME("flagm", AArch64::AEK_FLAGM, "+flagm", "-flagm") -AARCH64_ARCH_EXT_NAME("sme", AArch64::AEK_SME, "+sme", "-sme") -AARCH64_ARCH_EXT_NAME("sme-f64", AArch64::AEK_SMEF64, "+sme-f64", "-sme-f64") -AARCH64_ARCH_EXT_NAME("sme-i64", AArch64::AEK_SMEI64, "+sme-i64", "-sme-i64") +AARCH64_ARCH_EXT_NAME("rcpc", AArch64::AEK_RCPC, "+rcpc", "-rcpc") +AARCH64_ARCH_EXT_NAME("rng", AArch64::AEK_RAND, "+rand", "-rand") +AARCH64_ARCH_EXT_NAME("memtag", AArch64::AEK_MTE, "+mte", "-mte") +AARCH64_ARCH_EXT_NAME("ssbs", AArch64::AEK_SSBS, "+ssbs", "-ssbs") +AARCH64_ARCH_EXT_NAME("sb", AArch64::AEK_SB, "+sb", "-sb") +AARCH64_ARCH_EXT_NAME("predres", AArch64::AEK_PREDRES, "+predres", "-predres") +AARCH64_ARCH_EXT_NAME("bf16", AArch64::AEK_BF16, "+bf16", "-bf16") +AARCH64_ARCH_EXT_NAME("i8mm", AArch64::AEK_I8MM, "+i8mm", "-i8mm") +AARCH64_ARCH_EXT_NAME("f32mm", AArch64::AEK_F32MM, "+f32mm", "-f32mm") +AARCH64_ARCH_EXT_NAME("f64mm", AArch64::AEK_F64MM, "+f64mm", "-f64mm") +AARCH64_ARCH_EXT_NAME("tme", AArch64::AEK_TME, "+tme", "-tme") +AARCH64_ARCH_EXT_NAME("ls64", AArch64::AEK_LS64, "+ls64", "-ls64") +AARCH64_ARCH_EXT_NAME("brbe", AArch64::AEK_BRBE, "+brbe", "-brbe") +AARCH64_ARCH_EXT_NAME("pauth", AArch64::AEK_PAUTH, "+pauth", "-pauth") +AARCH64_ARCH_EXT_NAME("flagm", AArch64::AEK_FLAGM, "+flagm", "-flagm") +AARCH64_ARCH_EXT_NAME("sme", AArch64::AEK_SME, "+sme", "-sme") +AARCH64_ARCH_EXT_NAME("sme-f64", AArch64::AEK_SMEF64, "+sme-f64", "-sme-f64") +AARCH64_ARCH_EXT_NAME("sme-i64", AArch64::AEK_SMEI64, "+sme-i64", "-sme-i64") +AARCH64_ARCH_EXT_NAME("hbc", AArch64::AEK_HBC, "+hbc", "-hbc") +AARCH64_ARCH_EXT_NAME("mops", AArch64::AEK_MOPS, "+mops", "-mops") +AARCH64_ARCH_EXT_NAME("pmuv3", AArch64::AEK_PERFMON, "+perfmon", "-perfmon") #undef AARCH64_ARCH_EXT_NAME #ifndef AARCH64_CPU_NAME diff --git a/llvm/include/llvm/Support/AArch64TargetParser.h b/llvm/include/llvm/Support/AArch64TargetParser.h index 15bb428f19bc..d094c704d291 100644 --- a/llvm/include/llvm/Support/AArch64TargetParser.h +++ b/llvm/include/llvm/Support/AArch64TargetParser.h @@ -14,7 +14,6 @@ #ifndef LLVM_SUPPORT_AARCH64TARGETPARSER_H #define LLVM_SUPPORT_AARCH64TARGETPARSER_H -#include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" #include "llvm/Support/ARMTargetParser.h" #include <vector> @@ -69,6 +68,9 @@ enum ArchExtKind : uint64_t { AEK_SME = 1ULL << 37, AEK_SMEF64 = 1ULL << 38, AEK_SMEI64 = 1ULL << 39, + AEK_HBC = 1ULL << 40, + AEK_MOPS = 1ULL << 41, + AEK_PERFMON = 1ULL << 42, }; enum class ArchKind { diff --git a/llvm/include/llvm/Support/ARMAttributeParser.h b/llvm/include/llvm/Support/ARMAttributeParser.h index b46a4d9f690f..cbb5701540e1 100644 --- a/llvm/include/llvm/Support/ARMAttributeParser.h +++ b/llvm/include/llvm/Support/ARMAttributeParser.h @@ -11,14 +11,12 @@ #include "ARMBuildAttributes.h" #include "ELFAttributeParser.h" -#include "ScopedPrinter.h" -#include "llvm/ADT/ArrayRef.h" -#include "llvm/Support/DataExtractor.h" -#include "llvm/Support/Endian.h" +#include "llvm/ADT/StringRef.h" #include "llvm/Support/Error.h" namespace llvm { -class StringRef; + +class ScopedPrinter; class ARMAttributeParser : public ELFAttributeParser { struct DisplayHandler { diff --git a/llvm/include/llvm/Support/ARMTargetParser.def b/llvm/include/llvm/Support/ARMTargetParser.def index 7d29808f0501..433d7fdc2c3b 100644 --- a/llvm/include/llvm/Support/ARMTargetParser.def +++ b/llvm/include/llvm/Support/ARMTargetParser.def @@ -122,6 +122,12 @@ ARM_ARCH("armv8.7-a", ARMV8_7A, "8.7-A", "v8.7a", (ARM::AEK_SEC | ARM::AEK_MP | ARM::AEK_VIRT | ARM::AEK_HWDIVARM | ARM::AEK_HWDIVTHUMB | ARM::AEK_DSP | ARM::AEK_CRC | ARM::AEK_RAS | ARM::AEK_DOTPROD | ARM::AEK_BF16 | ARM::AEK_I8MM)) +ARM_ARCH("armv8.8-a", ARMV8_8A, "8.8-A", "v8.8a", + ARMBuildAttrs::CPUArch::v8_A, FK_CRYPTO_NEON_FP_ARMV8, + (ARM::AEK_SEC | ARM::AEK_MP | ARM::AEK_VIRT | ARM::AEK_HWDIVARM | + ARM::AEK_HWDIVTHUMB | ARM::AEK_DSP | ARM::AEK_CRC | ARM::AEK_RAS | + ARM::AEK_DOTPROD | ARM::AEK_BF16 | ARM::AEK_SHA2 | ARM::AEK_AES | + ARM::AEK_I8MM)) ARM_ARCH("armv9-a", ARMV9A, "9-A", "v9a", ARMBuildAttrs::CPUArch::v8_A, FK_NEON_FP_ARMV8, (ARM::AEK_SEC | ARM::AEK_MP | ARM::AEK_VIRT | ARM::AEK_HWDIVARM | @@ -137,6 +143,11 @@ ARM_ARCH("armv9.2-a", ARMV9_2A, "9.2-A", "v9.2a", (ARM::AEK_SEC | ARM::AEK_MP | ARM::AEK_VIRT | ARM::AEK_HWDIVARM | ARM::AEK_HWDIVTHUMB | ARM::AEK_DSP | ARM::AEK_CRC | ARM::AEK_RAS | ARM::AEK_DOTPROD | ARM::AEK_BF16 | ARM::AEK_I8MM)) +ARM_ARCH("armv9.3-a", ARMV9_3A, "9.3-A", "v9.3a", + ARMBuildAttrs::CPUArch::v8_A, FK_CRYPTO_NEON_FP_ARMV8, + (ARM::AEK_SEC | ARM::AEK_MP | ARM::AEK_VIRT | ARM::AEK_HWDIVARM | + ARM::AEK_HWDIVTHUMB | ARM::AEK_DSP | ARM::AEK_CRC | ARM::AEK_RAS | + ARM::AEK_DOTPROD | ARM::AEK_BF16 | ARM::AEK_I8MM)) ARM_ARCH("armv8-r", ARMV8R, "8-R", "v8r", ARMBuildAttrs::CPUArch::v8_R, FK_NEON_FP_ARMV8, (ARM::AEK_MP | ARM::AEK_VIRT | ARM::AEK_HWDIVARM | ARM::AEK_HWDIVTHUMB | diff --git a/llvm/include/llvm/Support/ARMTargetParser.h b/llvm/include/llvm/Support/ARMTargetParser.h index b40704c24e87..1ada6daaad3b 100644 --- a/llvm/include/llvm/Support/ARMTargetParser.h +++ b/llvm/include/llvm/Support/ARMTargetParser.h @@ -14,7 +14,6 @@ #ifndef LLVM_SUPPORT_ARMTARGETPARSER_H #define LLVM_SUPPORT_ARMTARGETPARSER_H -#include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" #include "llvm/Support/ARMBuildAttributes.h" #include <vector> diff --git a/llvm/include/llvm/Support/Allocator.h b/llvm/include/llvm/Support/Allocator.h index 9e8ce4e36197..ec5ed06b7fa4 100644 --- a/llvm/include/llvm/Support/Allocator.h +++ b/llvm/include/llvm/Support/Allocator.h @@ -22,16 +22,12 @@ #include "llvm/Support/Alignment.h" #include "llvm/Support/AllocatorBase.h" #include "llvm/Support/Compiler.h" -#include "llvm/Support/ErrorHandling.h" #include "llvm/Support/MathExtras.h" -#include "llvm/Support/MemAlloc.h" #include <algorithm> #include <cassert> #include <cstddef> #include <cstdint> -#include <cstdlib> #include <iterator> -#include <type_traits> #include <utility> namespace llvm { @@ -141,8 +137,10 @@ public: } /// Allocate space at the specified alignment. - LLVM_ATTRIBUTE_RETURNS_NONNULL LLVM_ATTRIBUTE_RETURNS_NOALIAS void * - Allocate(size_t Size, Align Alignment) { + // This method is *not* marked noalias, because + // SpecificBumpPtrAllocator::DestroyAll() loops over all allocations, and + // that loop is not based on the Allocate() return value. + LLVM_ATTRIBUTE_RETURNS_NONNULL void *Allocate(size_t Size, Align Alignment) { // Keep track of how many bytes we've allocated. BytesAllocated += Size; @@ -198,7 +196,7 @@ public: return AlignedPtr; } - inline LLVM_ATTRIBUTE_RETURNS_NONNULL LLVM_ATTRIBUTE_RETURNS_NOALIAS void * + inline LLVM_ATTRIBUTE_RETURNS_NONNULL void * Allocate(size_t Size, size_t Alignment) { assert(Alignment > 0 && "0-byte alignment is not allowed. Use 1 instead."); return Allocate(Size, Align(Alignment)); diff --git a/llvm/include/llvm/Support/AllocatorBase.h b/llvm/include/llvm/Support/AllocatorBase.h index e5549d111622..eccced1d1ff4 100644 --- a/llvm/include/llvm/Support/AllocatorBase.h +++ b/llvm/include/llvm/Support/AllocatorBase.h @@ -21,6 +21,7 @@ #include "llvm/Support/Compiler.h" #include "llvm/Support/MemAlloc.h" +#include <type_traits> namespace llvm { diff --git a/llvm/include/llvm/Support/BinaryByteStream.h b/llvm/include/llvm/Support/BinaryByteStream.h index 7d8b6d2dc43d..dc4adba26f16 100644 --- a/llvm/include/llvm/Support/BinaryByteStream.h +++ b/llvm/include/llvm/Support/BinaryByteStream.h @@ -17,7 +17,6 @@ #include "llvm/Support/Error.h" #include "llvm/Support/FileOutputBuffer.h" #include "llvm/Support/MemoryBuffer.h" -#include <algorithm> #include <cstdint> #include <cstring> #include <memory> diff --git a/llvm/include/llvm/Support/BinaryStreamArray.h b/llvm/include/llvm/Support/BinaryStreamArray.h index 85d29be26ca9..c3e0db4dcff0 100644 --- a/llvm/include/llvm/Support/BinaryStreamArray.h +++ b/llvm/include/llvm/Support/BinaryStreamArray.h @@ -323,7 +323,7 @@ public: FixedStreamArrayIterator(const FixedStreamArray<T> &Array, uint32_t Index) : Array(Array), Index(Index) {} - FixedStreamArrayIterator<T>(const FixedStreamArrayIterator<T> &Other) + FixedStreamArrayIterator(const FixedStreamArrayIterator<T> &Other) : Array(Other.Array), Index(Other.Index) {} FixedStreamArrayIterator<T> & operator=(const FixedStreamArrayIterator<T> &Other) { diff --git a/llvm/include/llvm/Support/BinaryStreamReader.h b/llvm/include/llvm/Support/BinaryStreamReader.h index 29b4b09b848c..c664ac48daad 100644 --- a/llvm/include/llvm/Support/BinaryStreamReader.h +++ b/llvm/include/llvm/Support/BinaryStreamReader.h @@ -10,7 +10,6 @@ #define LLVM_SUPPORT_BINARYSTREAMREADER_H #include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringRef.h" #include "llvm/Support/Alignment.h" #include "llvm/Support/BinaryStreamArray.h" @@ -18,7 +17,6 @@ #include "llvm/Support/ConvertUTF.h" #include "llvm/Support/Endian.h" #include "llvm/Support/Error.h" -#include "llvm/Support/type_traits.h" #include <type_traits> namespace llvm { diff --git a/llvm/include/llvm/Support/BinaryStreamRef.h b/llvm/include/llvm/Support/BinaryStreamRef.h index e0aaab82ffab..bc8c6a496ecf 100644 --- a/llvm/include/llvm/Support/BinaryStreamRef.h +++ b/llvm/include/llvm/Support/BinaryStreamRef.h @@ -14,7 +14,6 @@ #include "llvm/Support/BinaryStream.h" #include "llvm/Support/BinaryStreamError.h" #include "llvm/Support/Error.h" -#include <algorithm> #include <cstdint> #include <memory> diff --git a/llvm/include/llvm/Support/BinaryStreamWriter.h b/llvm/include/llvm/Support/BinaryStreamWriter.h index 3054f4ac7ef0..c05b0420aaa3 100644 --- a/llvm/include/llvm/Support/BinaryStreamWriter.h +++ b/llvm/include/llvm/Support/BinaryStreamWriter.h @@ -10,7 +10,6 @@ #define LLVM_SUPPORT_BINARYSTREAMWRITER_H #include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringRef.h" #include "llvm/Support/BinaryStreamArray.h" #include "llvm/Support/BinaryStreamError.h" diff --git a/llvm/include/llvm/Support/BlockFrequency.h b/llvm/include/llvm/Support/BlockFrequency.h index 18fb60e1904b..bf0ad46ab499 100644 --- a/llvm/include/llvm/Support/BlockFrequency.h +++ b/llvm/include/llvm/Support/BlockFrequency.h @@ -13,12 +13,11 @@ #ifndef LLVM_SUPPORT_BLOCKFREQUENCY_H #define LLVM_SUPPORT_BLOCKFREQUENCY_H -#include "llvm/Support/BranchProbability.h" -#include "llvm/Support/DataTypes.h" +#include <cstdint> namespace llvm { -class raw_ostream; +class BranchProbability; // This class represents Block Frequency as a 64-bit value. class BlockFrequency { diff --git a/llvm/include/llvm/Support/BranchProbability.h b/llvm/include/llvm/Support/BranchProbability.h index 6c7ad1fe2a52..6f071c15421f 100644 --- a/llvm/include/llvm/Support/BranchProbability.h +++ b/llvm/include/llvm/Support/BranchProbability.h @@ -16,7 +16,6 @@ #include "llvm/Support/DataTypes.h" #include <algorithm> #include <cassert> -#include <climits> #include <numeric> namespace llvm { diff --git a/llvm/include/llvm/Support/Caching.h b/llvm/include/llvm/Support/Caching.h index 5c30a822ef38..bef23ae757f2 100644 --- a/llvm/include/llvm/Support/Caching.h +++ b/llvm/include/llvm/Support/Caching.h @@ -62,10 +62,11 @@ using AddBufferFn = std::function<void(unsigned Task, std::unique_ptr<MemoryBuffer> MB)>; /// Create a local file system cache which uses the given cache name, temporary -/// file prefix, cache directory and file callback. This function also creates -/// the cache directory if it does not already exist. The cache name appears in -/// error messages for errors during caching. The temporary file prefix is used -/// in the temporary file naming scheme used when writing files atomically. +/// file prefix, cache directory and file callback. This function does not +/// immediately create the cache directory if it does not yet exist; this is +/// done lazily the first time a file is added. The cache name appears in error +/// messages for errors during caching. The temporary file prefix is used in the +/// temporary file naming scheme used when writing files atomically. Expected<FileCache> localCache( Twine CacheNameRef, Twine TempFilePrefixRef, Twine CacheDirectoryPathRef, AddBufferFn AddBuffer = [](size_t Task, std::unique_ptr<MemoryBuffer> MB) { diff --git a/llvm/include/llvm/Support/Chrono.h b/llvm/include/llvm/Support/Chrono.h index 629a37a90aae..9c2bd45d2803 100644 --- a/llvm/include/llvm/Support/Chrono.h +++ b/llvm/include/llvm/Support/Chrono.h @@ -14,6 +14,7 @@ #include <chrono> #include <ctime> +#include <ratio> namespace llvm { diff --git a/llvm/include/llvm/Support/CodeGenCoverage.h b/llvm/include/llvm/Support/CodeGenCoverage.h index a5b1796ca422..2acdd6a36a51 100644 --- a/llvm/include/llvm/Support/CodeGenCoverage.h +++ b/llvm/include/llvm/Support/CodeGenCoverage.h @@ -14,7 +14,6 @@ #include "llvm/ADT/BitVector.h" namespace llvm { -class LLVMContext; class MemoryBuffer; class CodeGenCoverage { diff --git a/llvm/include/llvm/Support/CommandLine.h b/llvm/include/llvm/Support/CommandLine.h index 2ee02010ff1d..120ab1840915 100644 --- a/llvm/include/llvm/Support/CommandLine.h +++ b/llvm/include/llvm/Support/CommandLine.h @@ -1550,8 +1550,9 @@ public: } template <class T> void addValue(const T &V) { - assert(Location != 0 && "cl::location(...) not specified for a command " - "line option with external storage!"); + assert(Location != nullptr && + "cl::location(...) not specified for a command " + "line option with external storage!"); Location->push_back(V); } }; @@ -1754,8 +1755,9 @@ public: } template <class T> void addValue(const T &V) { - assert(Location != 0 && "cl::location(...) not specified for a command " - "line option with external storage!"); + assert(Location != nullptr && + "cl::location(...) not specified for a command " + "line option with external storage!"); *Location |= Bit(V); } @@ -2080,7 +2082,8 @@ void tokenizeConfigFile(StringRef Source, StringSaver &Saver, /// /// It reads content of the specified file, tokenizes it and expands "@file" /// commands resolving file names in them relative to the directory where -/// CfgFilename resides. +/// CfgFilename resides. It also expands "<CFGDIR>" to the base path of the +/// current config file. /// bool readConfigFile(StringRef CfgFileName, StringSaver &Saver, SmallVectorImpl<const char *> &Argv); @@ -2100,13 +2103,15 @@ bool readConfigFile(StringRef CfgFileName, StringSaver &Saver, /// with nullptrs in the Argv vector. /// \param [in] RelativeNames true if names of nested response files must be /// resolved relative to including file. +/// \param [in] ExpandBasePath If true, "<CFGDIR>" expands to the base path of +/// the current response file. /// \param [in] FS File system used for all file access when running the tool. /// \param [in] CurrentDir Path used to resolve relative rsp files. If set to /// None, process' cwd is used instead. /// \return true if all @files were expanded successfully or there were none. bool ExpandResponseFiles(StringSaver &Saver, TokenizerCallback Tokenizer, SmallVectorImpl<const char *> &Argv, bool MarkEOLs, - bool RelativeNames, + bool RelativeNames, bool ExpandBasePath, llvm::Optional<llvm::StringRef> CurrentDir, llvm::vfs::FileSystem &FS); @@ -2115,7 +2120,7 @@ bool ExpandResponseFiles(StringSaver &Saver, TokenizerCallback Tokenizer, bool ExpandResponseFiles( StringSaver &Saver, TokenizerCallback Tokenizer, SmallVectorImpl<const char *> &Argv, bool MarkEOLs = false, - bool RelativeNames = false, + bool RelativeNames = false, bool ExpandBasePath = false, llvm::Optional<llvm::StringRef> CurrentDir = llvm::None); /// A convenience helper which concatenates the options specified by the diff --git a/llvm/include/llvm/Support/Compiler.h b/llvm/include/llvm/Support/Compiler.h index b31ba6bc7fc2..f4c277fae7cc 100644 --- a/llvm/include/llvm/Support/Compiler.h +++ b/llvm/include/llvm/Support/Compiler.h @@ -17,9 +17,6 @@ #include "llvm/Config/llvm-config.h" -#ifdef __cplusplus -#include <new> -#endif #include <stddef.h> #if defined(_MSC_VER) diff --git a/llvm/include/llvm/Support/ConvertUTF.h b/llvm/include/llvm/Support/ConvertUTF.h index 1add185330fa..374cdb907fdc 100644 --- a/llvm/include/llvm/Support/ConvertUTF.h +++ b/llvm/include/llvm/Support/ConvertUTF.h @@ -91,7 +91,10 @@ #include <cstddef> #include <string> + +#if defined(_WIN32) #include <system_error> +#endif // Wrap everything in namespace llvm so that programs can link with llvm and // their own version of the unicode libraries. diff --git a/llvm/include/llvm/Support/CrashRecoveryContext.h b/llvm/include/llvm/Support/CrashRecoveryContext.h index 2604ccb38431..f60e7335e197 100644 --- a/llvm/include/llvm/Support/CrashRecoveryContext.h +++ b/llvm/include/llvm/Support/CrashRecoveryContext.h @@ -9,7 +9,7 @@ #ifndef LLVM_SUPPORT_CRASHRECOVERYCONTEXT_H #define LLVM_SUPPORT_CRASHRECOVERYCONTEXT_H -#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/STLFunctionalExtras.h" namespace llvm { class CrashRecoveryContextCleanup; diff --git a/llvm/include/llvm/Support/DivisionByConstantInfo.h b/llvm/include/llvm/Support/DivisionByConstantInfo.h index 5bb326178c3e..896bc679885e 100644 --- a/llvm/include/llvm/Support/DivisionByConstantInfo.h +++ b/llvm/include/llvm/Support/DivisionByConstantInfo.h @@ -10,8 +10,8 @@ /// //===----------------------------------------------------------------------===// -#ifndef LLVM_SUPPORT_DIVISON_BY_CONSTANT_INFO_H -#define LLVM_SUPPORT_DIVISON_BY_CONSTANT_INFO_H +#ifndef LLVM_SUPPORT_DIVISIONBYCONSTANTINFO_H +#define LLVM_SUPPORT_DIVISIONBYCONSTANTINFO_H #include "llvm/ADT/APInt.h" diff --git a/llvm/include/llvm/Support/Duration.h b/llvm/include/llvm/Support/Duration.h new file mode 100644 index 000000000000..a5a0e2a3357a --- /dev/null +++ b/llvm/include/llvm/Support/Duration.h @@ -0,0 +1,28 @@ +//===--- Duration.h - wrapper around std::chrono::Duration ------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// The sole purpose of this file is to avoid the dependency on <chrono> in +// raw_ostream. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_DURATION_H +#define LLVM_SUPPORT_DURATION_H + +#include <chrono> + +namespace llvm { +class Duration { + std::chrono::milliseconds Value; + public: + Duration(std::chrono::milliseconds Value) : Value(Value) {} + std::chrono::milliseconds getDuration() const { return Value; } +}; +} + +#endif diff --git a/llvm/include/llvm/Support/ELFAttributeParser.h b/llvm/include/llvm/Support/ELFAttributeParser.h index 8bf87b2d84f0..3062dfffff68 100644 --- a/llvm/include/llvm/Support/ELFAttributeParser.h +++ b/llvm/include/llvm/Support/ELFAttributeParser.h @@ -10,15 +10,16 @@ #define LLVM_SUPPORT_ELFATTRIBUTEPARSER_H #include "ELFAttributes.h" -#include "ScopedPrinter.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/Support/DataExtractor.h" +#include "llvm/Support/Endian.h" #include "llvm/Support/Error.h" #include <unordered_map> namespace llvm { class StringRef; +class ScopedPrinter; class ELFAttributeParser { StringRef vendor; diff --git a/llvm/include/llvm/Support/Error.h b/llvm/include/llvm/Support/Error.h index e2002b89ada2..881049b15b0d 100644 --- a/llvm/include/llvm/Support/Error.h +++ b/llvm/include/llvm/Support/Error.h @@ -14,7 +14,6 @@ #define LLVM_SUPPORT_ERROR_H #include "llvm-c/Error.h" -#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringExtras.h" #include "llvm/ADT/Twine.h" @@ -26,7 +25,6 @@ #include "llvm/Support/ErrorOr.h" #include "llvm/Support/Format.h" #include "llvm/Support/raw_ostream.h" -#include <algorithm> #include <cassert> #include <cstdint> #include <cstdlib> diff --git a/llvm/include/llvm/Support/ExtensibleRTTI.h b/llvm/include/llvm/Support/ExtensibleRTTI.h index 21055247e932..d3193be6f529 100644 --- a/llvm/include/llvm/Support/ExtensibleRTTI.h +++ b/llvm/include/llvm/Support/ExtensibleRTTI.h @@ -62,8 +62,6 @@ namespace llvm { -template <typename ThisT, typename ParentT> class RTTIExtends; - /// Base class for the extensible RTTI hierarchy. /// /// This class defines virtual methods, dynamicClassID and isA, that enable diff --git a/llvm/include/llvm/Support/FileCollector.h b/llvm/include/llvm/Support/FileCollector.h index 264fb55c9dba..232dc8658aa3 100644 --- a/llvm/include/llvm/Support/FileCollector.h +++ b/llvm/include/llvm/Support/FileCollector.h @@ -9,7 +9,6 @@ #ifndef LLVM_SUPPORT_FILECOLLECTOR_H #define LLVM_SUPPORT_FILECOLLECTOR_H -#include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringSet.h" #include "llvm/Support/VirtualFileSystem.h" diff --git a/llvm/include/llvm/Support/FileSystem.h b/llvm/include/llvm/Support/FileSystem.h index 1a049533b82b..033482977a10 100644 --- a/llvm/include/llvm/Support/FileSystem.h +++ b/llvm/include/llvm/Support/FileSystem.h @@ -1014,6 +1014,25 @@ file_t getStderrHandle(); /// @returns The number of bytes read, or error. Expected<size_t> readNativeFile(file_t FileHandle, MutableArrayRef<char> Buf); +/// Default chunk size for \a readNativeFileToEOF(). +enum : size_t { DefaultReadChunkSize = 4 * 4096 }; + +/// Reads from \p FileHandle until EOF, appending to \p Buffer in chunks of +/// size \p ChunkSize. +/// +/// This calls \a readNativeFile() in a loop. On Error, previous chunks that +/// were read successfully are left in \p Buffer and returned. +/// +/// Note: For reading the final chunk at EOF, \p Buffer's capacity needs extra +/// storage of \p ChunkSize. +/// +/// \param FileHandle File to read from. +/// \param Buffer Where to put the file content. +/// \param ChunkSize Size of chunks. +/// \returns The error if EOF was not found. +Error readNativeFileToEOF(file_t FileHandle, SmallVectorImpl<char> &Buffer, + ssize_t ChunkSize = DefaultReadChunkSize); + /// Reads \p Buf.size() bytes from \p FileHandle at offset \p Offset into \p /// Buf. If 'pread' is available, this will use that, otherwise it will use /// 'lseek'. Returns the number of bytes actually read. Returns 0 when reaching @@ -1279,6 +1298,7 @@ private: } void unmapImpl(); + void dontNeedImpl(); std::error_code init(sys::fs::file_t FD, uint64_t Offset, mapmode Mode); @@ -1308,6 +1328,7 @@ public: unmapImpl(); copyFrom(mapped_file_region()); } + void dontNeed() { dontNeedImpl(); } size_t size() const; char *data() const; diff --git a/llvm/include/llvm/Support/FileUtilities.h b/llvm/include/llvm/Support/FileUtilities.h index 04efdced32a4..f8a37fe1177d 100644 --- a/llvm/include/llvm/Support/FileUtilities.h +++ b/llvm/include/llvm/Support/FileUtilities.h @@ -15,10 +15,10 @@ #define LLVM_SUPPORT_FILEUTILITIES_H #include "llvm/ADT/StringRef.h" -#include "llvm/Support/Errc.h" -#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/Error.h" #include "llvm/Support/FileSystem.h" -#include "llvm/Support/Path.h" + +#include <system_error> namespace llvm { diff --git a/llvm/include/llvm/Support/FormatVariadic.h b/llvm/include/llvm/Support/FormatVariadic.h index 89575f01b717..a872afb5e45e 100644 --- a/llvm/include/llvm/Support/FormatVariadic.h +++ b/llvm/include/llvm/Support/FormatVariadic.h @@ -29,16 +29,17 @@ #include "llvm/ADT/Optional.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallString.h" +#include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" #include "llvm/Support/FormatCommon.h" #include "llvm/Support/FormatProviders.h" #include "llvm/Support/FormatVariadicDetails.h" #include "llvm/Support/raw_ostream.h" +#include <array> #include <cstddef> #include <string> #include <tuple> #include <utility> -#include <vector> namespace llvm { diff --git a/llvm/include/llvm/Support/FormatVariadicDetails.h b/llvm/include/llvm/Support/FormatVariadicDetails.h index 08f8fc61f69b..2cafc120c1d7 100644 --- a/llvm/include/llvm/Support/FormatVariadicDetails.h +++ b/llvm/include/llvm/Support/FormatVariadicDetails.h @@ -10,6 +10,7 @@ #define LLVM_SUPPORT_FORMATVARIADICDETAILS_H #include "llvm/ADT/StringRef.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/Support/raw_ostream.h" #include <type_traits> diff --git a/llvm/include/llvm/Support/GraphWriter.h b/llvm/include/llvm/Support/GraphWriter.h index 1c0f5f702c6d..515057e7e312 100644 --- a/llvm/include/llvm/Support/GraphWriter.h +++ b/llvm/include/llvm/Support/GraphWriter.h @@ -28,8 +28,6 @@ #include "llvm/Support/DOTGraphTraits.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/raw_ostream.h" -#include <algorithm> -#include <cstddef> #include <iterator> #include <string> #include <type_traits> diff --git a/llvm/include/llvm/Support/ItaniumManglingCanonicalizer.h b/llvm/include/llvm/Support/ItaniumManglingCanonicalizer.h index 8e1b3d631983..aa7997a0228b 100644 --- a/llvm/include/llvm/Support/ItaniumManglingCanonicalizer.h +++ b/llvm/include/llvm/Support/ItaniumManglingCanonicalizer.h @@ -14,7 +14,6 @@ #ifndef LLVM_SUPPORT_ITANIUMMANGLINGCANONICALIZER_H #define LLVM_SUPPORT_ITANIUMMANGLINGCANONICALIZER_H -#include <cstddef> #include <cstdint> namespace llvm { diff --git a/llvm/include/llvm/Support/JSON.h b/llvm/include/llvm/Support/JSON.h index 469f50be40e0..719e8b60d0fa 100644 --- a/llvm/include/llvm/Support/JSON.h +++ b/llvm/include/llvm/Support/JSON.h @@ -49,6 +49,7 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" +#include "llvm/ADT/STLFunctionalExtras.h" #include "llvm/Support/Error.h" #include "llvm/Support/FormatVariadic.h" #include "llvm/Support/raw_ostream.h" diff --git a/llvm/include/llvm/Support/KnownBits.h b/llvm/include/llvm/Support/KnownBits.h index 1f32760a6fd1..5ef0ba31f785 100644 --- a/llvm/include/llvm/Support/KnownBits.h +++ b/llvm/include/llvm/Support/KnownBits.h @@ -249,7 +249,17 @@ public: return countMinLeadingZeros(); if (isNegative()) return countMinLeadingOnes(); - return 0; + // Every value has at least 1 sign bit. + return 1; + } + + /// Returns the maximum number of bits needed to represent all possible + /// signed values with these known bits. This is the inverse of the minimum + /// number of known sign bits. Examples for bitwidth 5: + /// 110?? --> 4 + /// 0000? --> 2 + unsigned countMaxSignificantBits() const { + return getBitWidth() - countMinSignBits() + 1; } /// Returns the maximum number of trailing zero bits possible. @@ -282,6 +292,9 @@ public: return getBitWidth() - Zero.countPopulation(); } + /// Returns the maximum number of bits needed to represent all possible + /// unsigned values with these known bits. This is the inverse of the + /// minimum number of leading zeros. unsigned countMaxActiveBits() const { return getBitWidth() - countMinLeadingZeros(); } diff --git a/llvm/include/llvm/Support/LowLevelTypeImpl.h b/llvm/include/llvm/Support/LowLevelTypeImpl.h index 2071a08d8711..dd286f5228fe 100644 --- a/llvm/include/llvm/Support/LowLevelTypeImpl.h +++ b/llvm/include/llvm/Support/LowLevelTypeImpl.h @@ -33,7 +33,6 @@ namespace llvm { -class DataLayout; class Type; class raw_ostream; diff --git a/llvm/include/llvm/Support/MD5.h b/llvm/include/llvm/Support/MD5.h index 3b960cd4fd88..70d046601346 100644 --- a/llvm/include/llvm/Support/MD5.h +++ b/llvm/include/llvm/Support/MD5.h @@ -88,7 +88,7 @@ public: /// Translates the bytes in \p Res to a hex string that is /// deposited into \p Str. The result will be of length 32. - static void stringifyResult(MD5Result &Result, SmallString<32> &Str); + static void stringifyResult(MD5Result &Result, SmallVectorImpl<char> &Str); /// Computes the hash for a given bytes. static std::array<uint8_t, 16> hash(ArrayRef<uint8_t> Data); diff --git a/llvm/include/llvm/Support/MachineValueType.h b/llvm/include/llvm/Support/MachineValueType.h index ce10a4c58dfe..643c2d8ce981 100644 --- a/llvm/include/llvm/Support/MachineValueType.h +++ b/llvm/include/llvm/Support/MachineValueType.h @@ -848,7 +848,11 @@ namespace llvm { } unsigned getVectorNumElements() const { - // TODO: Check that this isn't a scalable vector. + if (isScalableVector()) + llvm::reportInvalidSizeRequest( + "Possible incorrect use of MVT::getVectorNumElements() for " + "scalable vector. Scalable flag may be dropped, use " + "MVT::getVectorElementCount() instead"); return getVectorMinNumElements(); } diff --git a/llvm/include/llvm/Support/MemoryBuffer.h b/llvm/include/llvm/Support/MemoryBuffer.h index c9ceeedbf3dc..6385805eba1d 100644 --- a/llvm/include/llvm/Support/MemoryBuffer.h +++ b/llvm/include/llvm/Support/MemoryBuffer.h @@ -74,6 +74,13 @@ public: /// from. virtual StringRef getBufferIdentifier() const { return "Unknown buffer"; } + /// For read-only MemoryBuffer_MMap, mark the buffer as unused in the near + /// future and the kernel can free resources associated with it. Further + /// access is supported but may be expensive. This calls + /// madvise(MADV_DONTNEED) on read-only file mappings on *NIX systems. This + /// function should not be called on a writable buffer. + virtual void dontNeedIfMmap() {} + /// Open the specified file as a MemoryBuffer, returning a new MemoryBuffer /// if successful, otherwise returning null. /// diff --git a/llvm/include/llvm/Support/Parallel.h b/llvm/include/llvm/Support/Parallel.h index 5c3b26d5754c..04caf5eac961 100644 --- a/llvm/include/llvm/Support/Parallel.h +++ b/llvm/include/llvm/Support/Parallel.h @@ -130,64 +130,6 @@ void parallel_sort(RandomAccessIterator Start, RandomAccessIterator End, // improving to take the number of available cores into account.) enum { MaxTasksPerGroup = 1024 }; -template <class IterTy, class FuncTy> -void parallel_for_each(IterTy Begin, IterTy End, FuncTy Fn) { - // If we have zero or one items, then do not incur the overhead of spinning up - // a task group. They are surprisingly expensive, and because they do not - // support nested parallelism, a single entry task group can block parallel - // execution underneath them. - auto NumItems = std::distance(Begin, End); - if (NumItems <= 1) { - if (NumItems) - Fn(*Begin); - return; - } - - // Limit the number of tasks to MaxTasksPerGroup to limit job scheduling - // overhead on large inputs. - ptrdiff_t TaskSize = NumItems / MaxTasksPerGroup; - if (TaskSize == 0) - TaskSize = 1; - - TaskGroup TG; - while (TaskSize < std::distance(Begin, End)) { - TG.spawn([=, &Fn] { std::for_each(Begin, Begin + TaskSize, Fn); }); - Begin += TaskSize; - } - std::for_each(Begin, End, Fn); -} - -template <class IndexTy, class FuncTy> -void parallel_for_each_n(IndexTy Begin, IndexTy End, FuncTy Fn) { - // If we have zero or one items, then do not incur the overhead of spinning up - // a task group. They are surprisingly expensive, and because they do not - // support nested parallelism, a single entry task group can block parallel - // execution underneath them. - auto NumItems = End - Begin; - if (NumItems <= 1) { - if (NumItems) - Fn(Begin); - return; - } - - // Limit the number of tasks to MaxTasksPerGroup to limit job scheduling - // overhead on large inputs. - ptrdiff_t TaskSize = NumItems / MaxTasksPerGroup; - if (TaskSize == 0) - TaskSize = 1; - - TaskGroup TG; - IndexTy I = Begin; - for (; I + TaskSize < End; I += TaskSize) { - TG.spawn([=, &Fn] { - for (IndexTy J = I, E = I + TaskSize; J != E; ++J) - Fn(J); - }); - } - for (IndexTy J = I; J < End; ++J) - Fn(J); -} - template <class IterTy, class ResultTy, class ReduceFuncTy, class TransformFuncTy> ResultTy parallel_transform_reduce(IterTy Begin, IterTy End, ResultTy Init, @@ -251,27 +193,11 @@ void parallelSort(RandomAccessIterator Start, RandomAccessIterator End, llvm::sort(Start, End, Comp); } +void parallelForEachN(size_t Begin, size_t End, function_ref<void(size_t)> Fn); + template <class IterTy, class FuncTy> void parallelForEach(IterTy Begin, IterTy End, FuncTy Fn) { -#if LLVM_ENABLE_THREADS - if (parallel::strategy.ThreadsRequested != 1) { - parallel::detail::parallel_for_each(Begin, End, Fn); - return; - } -#endif - std::for_each(Begin, End, Fn); -} - -template <class FuncTy> -void parallelForEachN(size_t Begin, size_t End, FuncTy Fn) { -#if LLVM_ENABLE_THREADS - if (parallel::strategy.ThreadsRequested != 1) { - parallel::detail::parallel_for_each_n(Begin, End, Fn); - return; - } -#endif - for (size_t I = Begin; I != End; ++I) - Fn(I); + parallelForEachN(0, End - Begin, [&](size_t I) { Fn(Begin[I]); }); } template <class IterTy, class ResultTy, class ReduceFuncTy, diff --git a/llvm/include/llvm/Support/RISCVISAInfo.h b/llvm/include/llvm/Support/RISCVISAInfo.h index 1ba4d449b709..b450c1df3558 100644 --- a/llvm/include/llvm/Support/RISCVISAInfo.h +++ b/llvm/include/llvm/Support/RISCVISAInfo.h @@ -9,8 +9,6 @@ #ifndef LLVM_SUPPORT_RISCVISAINFO_H #define LLVM_SUPPORT_RISCVISAINFO_H -#include "llvm/ADT/Optional.h" -#include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" #include "llvm/Support/Error.h" @@ -61,9 +59,13 @@ public: unsigned getXLen() const { return XLen; }; unsigned getFLen() const { return FLen; }; + unsigned getMinVLen() const { return MinVLen; } + unsigned getMaxELen() const { return MaxELen; } + unsigned getMaxELenFp() const { return MaxELenFp; } bool hasExtension(StringRef Ext) const; std::string toString() const; + std::vector<std::string> toFeatureVector() const; static bool isSupportedExtensionFeature(StringRef Ext); static bool isSupportedExtension(StringRef Ext); @@ -71,10 +73,13 @@ public: unsigned MinorVersion); private: - RISCVISAInfo(unsigned XLen) : XLen(XLen), FLen(0) {} + RISCVISAInfo(unsigned XLen) + : XLen(XLen), FLen(0), MinVLen(0), MaxELen(0), MaxELenFp(0) {} unsigned XLen; unsigned FLen; + unsigned MinVLen; + unsigned MaxELen, MaxELenFp; OrderedExtensionMap Exts; @@ -85,6 +90,8 @@ private: void updateImplication(); void updateFLen(); + void updateMinVLen(); + void updateMaxELen(); }; } // namespace llvm diff --git a/llvm/include/llvm/Support/ScopedPrinter.h b/llvm/include/llvm/Support/ScopedPrinter.h index 865337e3cc7f..9bde4f455a2d 100644 --- a/llvm/include/llvm/Support/ScopedPrinter.h +++ b/llvm/include/llvm/Support/ScopedPrinter.h @@ -18,7 +18,6 @@ #include "llvm/Support/Endian.h" #include "llvm/Support/JSON.h" #include "llvm/Support/raw_ostream.h" -#include <algorithm> namespace llvm { @@ -123,7 +122,7 @@ public: void indent(int Levels = 1) { IndentLevel += Levels; } void unindent(int Levels = 1) { - IndentLevel = std::max(0, IndentLevel - Levels); + IndentLevel = IndentLevel > Levels ? IndentLevel - Levels : 0; } void resetIndent() { IndentLevel = 0; } @@ -799,7 +798,7 @@ struct DelimitedScope { }; struct DictScope : DelimitedScope { - explicit DictScope() : DelimitedScope() {} + explicit DictScope() {} explicit DictScope(ScopedPrinter &W) : DelimitedScope(W) { W.objectBegin(); } DictScope(ScopedPrinter &W, StringRef N) : DelimitedScope(W) { @@ -818,7 +817,7 @@ struct DictScope : DelimitedScope { }; struct ListScope : DelimitedScope { - explicit ListScope() : DelimitedScope() {} + explicit ListScope() {} explicit ListScope(ScopedPrinter &W) : DelimitedScope(W) { W.arrayBegin(); } ListScope(ScopedPrinter &W, StringRef N) : DelimitedScope(W) { diff --git a/llvm/include/llvm/Support/SymbolRemappingReader.h b/llvm/include/llvm/Support/SymbolRemappingReader.h index 820cf9e02192..4fdaf87be082 100644 --- a/llvm/include/llvm/Support/SymbolRemappingReader.h +++ b/llvm/include/llvm/Support/SymbolRemappingReader.h @@ -62,10 +62,11 @@ #include "llvm/ADT/StringRef.h" #include "llvm/Support/Error.h" #include "llvm/Support/ItaniumManglingCanonicalizer.h" -#include "llvm/Support/MemoryBuffer.h" namespace llvm { +class MemoryBuffer; + class SymbolRemappingParseError : public ErrorInfo<SymbolRemappingParseError> { public: SymbolRemappingParseError(StringRef File, int64_t Line, const Twine &Message) diff --git a/llvm/include/llvm/Support/TargetOpcodes.def b/llvm/include/llvm/Support/TargetOpcodes.def index b34b885ddc35..428cbb44705d 100644 --- a/llvm/include/llvm/Support/TargetOpcodes.def +++ b/llvm/include/llvm/Support/TargetOpcodes.def @@ -228,10 +228,11 @@ HANDLE_TARGET_OPCODE(ICALL_BRANCH_FUNNEL) /// generate code. These instructions only act as optimization hints. HANDLE_TARGET_OPCODE(G_ASSERT_SEXT) HANDLE_TARGET_OPCODE(G_ASSERT_ZEXT) +HANDLE_TARGET_OPCODE(G_ASSERT_ALIGN) HANDLE_TARGET_OPCODE_MARKER(PRE_ISEL_GENERIC_OPTIMIZATION_HINT_START, G_ASSERT_SEXT) HANDLE_TARGET_OPCODE_MARKER(PRE_ISEL_GENERIC_OPTIMIZATION_HINT_END, - G_ASSERT_ZEXT) + G_ASSERT_ALIGN) /// Generic ADD instruction. This is an integer add. HANDLE_TARGET_OPCODE(G_ADD) diff --git a/llvm/include/llvm/Support/TargetParser.h b/llvm/include/llvm/Support/TargetParser.h index 01e25a0ea857..02a8d72483db 100644 --- a/llvm/include/llvm/Support/TargetParser.h +++ b/llvm/include/llvm/Support/TargetParser.h @@ -16,14 +16,14 @@ // FIXME: vector is used because that's what clang uses for subtarget feature // lists, but SmallVector would probably be better -#include "llvm/ADT/Triple.h" -#include "llvm/Support/AArch64TargetParser.h" -#include "llvm/Support/ARMTargetParser.h" #include "llvm/Support/RISCVISAInfo.h" #include <vector> namespace llvm { + class StringRef; +template <typename T> class SmallVectorImpl; +class Triple; // Target specific information in their own namespaces. // (ARM/AArch64/X86 are declared in ARM/AArch64/X86TargetParser.h) @@ -158,12 +158,7 @@ enum CPUKind : unsigned { enum FeatureKind : unsigned { FK_INVALID = 0, FK_NONE = 1, - FK_STDEXTM = 1 << 2, - FK_STDEXTA = 1 << 3, - FK_STDEXTF = 1 << 4, - FK_STDEXTD = 1 << 5, - FK_STDEXTC = 1 << 6, - FK_64BIT = 1 << 7, + FK_64BIT = 1 << 2, }; bool checkCPUKind(CPUKind Kind, bool IsRV64); diff --git a/llvm/include/llvm/Support/ThreadPool.h b/llvm/include/llvm/Support/ThreadPool.h index aecff122d3cb..868dd2819f83 100644 --- a/llvm/include/llvm/Support/ThreadPool.h +++ b/llvm/include/llvm/Support/ThreadPool.h @@ -19,7 +19,6 @@ #include <future> -#include <atomic> #include <condition_variable> #include <functional> #include <memory> diff --git a/llvm/include/llvm/Support/TimeProfiler.h b/llvm/include/llvm/Support/TimeProfiler.h index 84794a25f78e..378253dc2ab9 100644 --- a/llvm/include/llvm/Support/TimeProfiler.h +++ b/llvm/include/llvm/Support/TimeProfiler.h @@ -10,10 +10,12 @@ #define LLVM_SUPPORT_TIMEPROFILER_H #include "llvm/Support/Error.h" -#include "llvm/Support/raw_ostream.h" +#include "llvm/ADT/STLFunctionalExtras.h" namespace llvm { +class raw_pwrite_stream; + struct TimeTraceProfiler; TimeTraceProfiler *getTimeTraceProfilerInstance(); diff --git a/llvm/include/llvm/Support/Timer.h b/llvm/include/llvm/Support/Timer.h index c5874ed35698..eb49e805b40d 100644 --- a/llvm/include/llvm/Support/Timer.h +++ b/llvm/include/llvm/Support/Timer.h @@ -13,13 +13,12 @@ #include "llvm/ADT/StringRef.h" #include "llvm/Support/DataTypes.h" #include <cassert> +#include <memory> #include <string> -#include <utility> #include <vector> namespace llvm { -class Timer; class TimerGroup; class raw_ostream; diff --git a/llvm/include/llvm/Support/TrigramIndex.h b/llvm/include/llvm/Support/TrigramIndex.h index 0be6a1012718..f772deca0301 100644 --- a/llvm/include/llvm/Support/TrigramIndex.h +++ b/llvm/include/llvm/Support/TrigramIndex.h @@ -33,7 +33,6 @@ #include <vector> namespace llvm { -class StringRef; class TrigramIndex { public: diff --git a/llvm/include/llvm/Support/TypeSize.h b/llvm/include/llvm/Support/TypeSize.h index 7d1274735a37..6bddb602e8c1 100644 --- a/llvm/include/llvm/Support/TypeSize.h +++ b/llvm/include/llvm/Support/TypeSize.h @@ -17,7 +17,7 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/Support/MathExtras.h" -#include "llvm/Support/WithColor.h" +#include "llvm/Support/raw_ostream.h" #include <algorithm> #include <array> diff --git a/llvm/include/llvm/Support/VirtualFileSystem.h b/llvm/include/llvm/Support/VirtualFileSystem.h index 78bcdbf3e932..f5dde334b0a7 100644 --- a/llvm/include/llvm/Support/VirtualFileSystem.h +++ b/llvm/include/llvm/Support/VirtualFileSystem.h @@ -19,6 +19,7 @@ #include "llvm/ADT/Optional.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" +#include "llvm/ADT/STLFunctionalExtras.h" #include "llvm/Support/Chrono.h" #include "llvm/Support/ErrorOr.h" #include "llvm/Support/FileSystem.h" @@ -419,6 +420,21 @@ namespace detail { class InMemoryDirectory; class InMemoryFile; +class InMemoryNode; + +struct NewInMemoryNodeInfo { + llvm::sys::fs::UniqueID DirUID; + StringRef Path; + StringRef Name; + time_t ModificationTime; + std::unique_ptr<llvm::MemoryBuffer> Buffer; + uint32_t User; + uint32_t Group; + llvm::sys::fs::file_type Type; + llvm::sys::fs::perms Perms; + + Status makeStatus() const; +}; } // namespace detail @@ -428,14 +444,15 @@ class InMemoryFileSystem : public FileSystem { std::string WorkingDirectory; bool UseNormalizedPaths = true; - /// If HardLinkTarget is non-null, a hardlink is created to the To path which - /// must be a file. If it is null then it adds the file as the public addFile. + using MakeNodeFn = llvm::function_ref<std::unique_ptr<detail::InMemoryNode>( + detail::NewInMemoryNodeInfo)>; + + /// Create node with \p MakeNode and add it into this filesystem at \p Path. bool addFile(const Twine &Path, time_t ModificationTime, std::unique_ptr<llvm::MemoryBuffer> Buffer, Optional<uint32_t> User, Optional<uint32_t> Group, Optional<llvm::sys::fs::file_type> Type, - Optional<llvm::sys::fs::perms> Perms, - const detail::InMemoryFile *HardLinkTarget); + Optional<llvm::sys::fs::perms> Perms, MakeNodeFn MakeNode); public: explicit InMemoryFileSystem(bool UseNormalizedPaths = true); @@ -547,6 +564,9 @@ class RedirectingFileSystemParser; /// } /// \endverbatim /// +/// The roots may be absolute or relative. If relative they will be made +/// absolute against the current working directory. +/// /// All configuration options are optional. /// 'case-sensitive': <boolean, default=(true for Posix, false for Windows)> /// 'use-external-names': <boolean, default=true> diff --git a/llvm/include/llvm/Support/X86TargetParser.h b/llvm/include/llvm/Support/X86TargetParser.h index bfa3e23dbd9d..612046f3b2d9 100644 --- a/llvm/include/llvm/Support/X86TargetParser.h +++ b/llvm/include/llvm/Support/X86TargetParser.h @@ -14,10 +14,10 @@ #define LLVM_SUPPORT_X86TARGETPARSER_H #include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringMap.h" namespace llvm { +template <typename T> class SmallVectorImpl; class StringRef; namespace X86 { diff --git a/llvm/include/llvm/Support/YAMLTraits.h b/llvm/include/llvm/Support/YAMLTraits.h index bea232e6e000..7ad73543fc6e 100644 --- a/llvm/include/llvm/Support/YAMLTraits.h +++ b/llvm/include/llvm/Support/YAMLTraits.h @@ -9,6 +9,7 @@ #ifndef LLVM_SUPPORT_YAMLTRAITS_H #define LLVM_SUPPORT_YAMLTRAITS_H +#include "llvm/ADT/BitVector.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringExtras.h" @@ -18,17 +19,12 @@ #include "llvm/Support/AlignOf.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/Endian.h" -#include "llvm/Support/Regex.h" #include "llvm/Support/SMLoc.h" #include "llvm/Support/SourceMgr.h" -#include "llvm/Support/VersionTuple.h" #include "llvm/Support/YAMLParser.h" #include "llvm/Support/raw_ostream.h" #include <cassert> #include <cctype> -#include <cstddef> -#include <cstdint> -#include <iterator> #include <map> #include <memory> #include <new> @@ -38,6 +34,9 @@ #include <vector> namespace llvm { + +class VersionTuple; + namespace yaml { enum class NodeKind : uint8_t { @@ -1523,7 +1522,7 @@ private: std::error_code EC; BumpPtrAllocator StringAllocator; document_iterator DocIterator; - std::vector<bool> BitValuesUsed; + llvm::BitVector BitValuesUsed; HNode *CurrentNode = nullptr; bool ScalarMatchFound = false; bool AllowUnknownKeys = false; diff --git a/llvm/include/llvm/Support/raw_ostream.h b/llvm/include/llvm/Support/raw_ostream.h index 98c26ef0b1e5..58adb41cb0ef 100644 --- a/llvm/include/llvm/Support/raw_ostream.h +++ b/llvm/include/llvm/Support/raw_ostream.h @@ -15,9 +15,9 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" +#include "llvm/ADT/Optional.h" #include "llvm/Support/DataTypes.h" #include <cassert> -#include <chrono> #include <cstddef> #include <cstdint> #include <cstring> @@ -30,6 +30,7 @@ namespace llvm { +class Duration; class formatv_object_base; class format_object_base; class FormattedString; @@ -444,6 +445,7 @@ class raw_fd_ostream : public raw_pwrite_stream { int FD; bool ShouldClose; bool SupportsSeeking = false; + bool IsRegularFile = false; mutable Optional<bool> HasColors; #ifdef _WIN32 @@ -514,6 +516,8 @@ public: bool supportsSeeking() const { return SupportsSeeking; } + bool isRegularFile() const { return IsRegularFile; } + /// Flushes the stream and repositions the underlying file descriptor position /// to the offset specified from the beginning of the file. uint64_t seek(uint64_t off); @@ -571,7 +575,7 @@ public: /// /// It is used as @ref lock. LLVM_NODISCARD - Expected<sys::fs::FileLocker> tryLockFor(std::chrono::milliseconds Timeout); + Expected<sys::fs::FileLocker> tryLockFor(Duration const& Timeout); }; /// This returns a reference to a raw_fd_ostream for standard output. Use it @@ -622,6 +626,9 @@ public: /// A raw_ostream that writes to an std::string. This is a simple adaptor /// class. This class does not encounter output errors. +/// raw_string_ostream operates without a buffer, delegating all memory +/// management to the std::string. Thus the std::string is always up-to-date, +/// may be used directly and there is no need to call flush(). class raw_string_ostream : public raw_ostream { std::string &OS; @@ -636,14 +643,11 @@ public: explicit raw_string_ostream(std::string &O) : OS(O) { SetUnbuffered(); } - ~raw_string_ostream() override; - /// Flushes the stream contents to the target string and returns the string's - /// reference. - std::string& str() { - flush(); - return OS; - } + /// Returns the string's reference. In most cases it is better to simply use + /// the underlying std::string directly. + /// TODO: Consider removing this API. + std::string &str() { return OS; } void reserveExtraSpace(uint64_t ExtraSize) override { OS.reserve(tell() + ExtraSize); diff --git a/llvm/include/llvm/TableGen/Record.h b/llvm/include/llvm/TableGen/Record.h index 5869a5cf0423..add05bd078d6 100644 --- a/llvm/include/llvm/TableGen/Record.h +++ b/llvm/include/llvm/TableGen/Record.h @@ -44,7 +44,6 @@ struct RecordContext; } // namespace detail class ListRecTy; -struct MultiClass; class Record; class RecordKeeper; class RecordVal; diff --git a/llvm/include/llvm/Target/CGPassBuilderOption.h b/llvm/include/llvm/Target/CGPassBuilderOption.h index 11b904e6e7fe..f84889392d13 100644 --- a/llvm/include/llvm/Target/CGPassBuilderOption.h +++ b/llvm/include/llvm/Target/CGPassBuilderOption.h @@ -19,7 +19,6 @@ #include "llvm/Target/TargetOptions.h" namespace llvm { -class TargetMachine; enum class RunOutliner { TargetDefault, AlwaysOutline, NeverOutline }; enum class RegAllocType { Default, Basic, Fast, Greedy, PBQP }; diff --git a/llvm/include/llvm/Target/GenericOpcodes.td b/llvm/include/llvm/Target/GenericOpcodes.td index 72c974834a2f..2af20ab6a53f 100644 --- a/llvm/include/llvm/Target/GenericOpcodes.td +++ b/llvm/include/llvm/Target/GenericOpcodes.td @@ -1434,3 +1434,10 @@ def G_ASSERT_SEXT : GenericInstruction { let InOperandList = (ins type0:$src, untyped_imm_0:$sz); let hasSideEffects = false; } + +// Asserts that a value has at least the given alignment. +def G_ASSERT_ALIGN : GenericInstruction { + let OutOperandList = (outs type0:$dst); + let InOperandList = (ins type0:$src, untyped_imm_0:$align); + let hasSideEffects = false; +} diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td index 1d189c6dea6d..4859cf6b57b7 100644 --- a/llvm/include/llvm/Target/GlobalISel/Combine.td +++ b/llvm/include/llvm/Target/GlobalISel/Combine.td @@ -391,7 +391,7 @@ def add_p2i_to_ptradd : GICombineRule< >; // Fold (ptr_add (int2ptr C1), C2) -> C1 + C2 -def const_ptradd_to_i2p_matchinfo : GIDefMatchData<"int64_t">; +def const_ptradd_to_i2p_matchinfo : GIDefMatchData<"APInt">; def const_ptradd_to_i2p: GICombineRule< (defs root:$root, const_ptradd_to_i2p_matchinfo:$info), (match (wip_match_opcode G_PTR_ADD):$root, @@ -535,6 +535,14 @@ def unmerge_cst : GICombineRule< (apply [{ Helper.applyCombineUnmergeConstant(*${d}, ${info}); }]) >; +// Fold (unmerge undef) -> undef, undef, ... +def unmerge_undef : GICombineRule< + (defs root:$root, build_fn_matchinfo:$info), + (match (wip_match_opcode G_UNMERGE_VALUES): $root, + [{ return Helper.matchCombineUnmergeUndef(*${root}, ${info}); }]), + (apply [{ Helper.applyBuildFn(*${root}, ${info}); }]) +>; + // Transform x,y<dead> = unmerge z -> x = trunc z. def unmerge_dead_to_trunc : GICombineRule< (defs root:$d), @@ -844,7 +852,8 @@ def undef_combines : GICombineGroup<[undef_to_fp_zero, undef_to_int_zero, propagate_undef_any_op, propagate_undef_all_ops, propagate_undef_shuffle_mask, - erase_undef_store]>; + erase_undef_store, + unmerge_undef]>; def identity_combines : GICombineGroup<[select_same_val, right_identity_zero, binop_same_val, binop_left_to_zero, diff --git a/llvm/include/llvm/Target/Target.td b/llvm/include/llvm/Target/Target.td index 7ae690b83770..d8faa63ee877 100644 --- a/llvm/include/llvm/Target/Target.td +++ b/llvm/include/llvm/Target/Target.td @@ -993,7 +993,7 @@ class InstrInfo { // by default, and TableGen will infer their value from the instruction // pattern when possible. // - // Normally, TableGen will issue an error it it can't infer the value of a + // Normally, TableGen will issue an error if it can't infer the value of a // property that hasn't been set explicitly. When guessInstructionProperties // is set, it will guess a safe value instead. // diff --git a/llvm/include/llvm/Target/TargetLoweringObjectFile.h b/llvm/include/llvm/Target/TargetLoweringObjectFile.h index 752032d3d04d..392ee4334cb5 100644 --- a/llvm/include/llvm/Target/TargetLoweringObjectFile.h +++ b/llvm/include/llvm/Target/TargetLoweringObjectFile.h @@ -221,7 +221,7 @@ public: } /// Returns the register used as static base in RWPI variants. - virtual const MCRegister getStaticBase() const { return MCRegister::NoRegister; } + virtual MCRegister getStaticBase() const { return MCRegister::NoRegister; } /// Get the target specific RWPI relocation. virtual const MCExpr *getIndirectSymViaRWPI(const MCSymbol *Sym) const { diff --git a/llvm/include/llvm/Target/TargetOptions.h b/llvm/include/llvm/Target/TargetOptions.h index c639f326abc9..a636c4822832 100644 --- a/llvm/include/llvm/Target/TargetOptions.h +++ b/llvm/include/llvm/Target/TargetOptions.h @@ -140,9 +140,9 @@ namespace llvm { EnableMachineFunctionSplitter(false), SupportsDefaultOutlining(false), EmitAddrsig(false), EmitCallSiteInfo(false), SupportsDebugEntryValues(false), EnableDebugEntryValues(false), - ValueTrackingVariableLocations(false), - ForceDwarfFrameSection(false), XRayOmitFunctionIndex(false), - DebugStrictDwarf(false), + ValueTrackingVariableLocations(false), ForceDwarfFrameSection(false), + XRayOmitFunctionIndex(false), DebugStrictDwarf(false), + Hotpatch(false), FPDenormalMode(DenormalMode::IEEE, DenormalMode::IEEE) {} /// DisableFramePointerElim - This returns true if frame pointer elimination @@ -342,6 +342,9 @@ namespace llvm { /// By default, it is set to false. unsigned DebugStrictDwarf : 1; + /// Emit the hotpatch flag in CodeView debug. + unsigned Hotpatch : 1; + /// Name of the stack usage file (i.e., .su file) if user passes /// -fstack-usage. If empty, it can be implied that -fstack-usage is not /// passed on the command line. diff --git a/llvm/include/llvm/Testing/Support/Annotations.h b/llvm/include/llvm/Testing/Support/Annotations.h index cc99d1061520..4e442269600d 100644 --- a/llvm/include/llvm/Testing/Support/Annotations.h +++ b/llvm/include/llvm/Testing/Support/Annotations.h @@ -16,6 +16,8 @@ namespace llvm { +class raw_ostream; + /// Annotations lets you mark points and ranges inside source code, for tests: /// /// Annotations Example(R"cpp( diff --git a/llvm/include/llvm/TextAPI/InterfaceFile.h b/llvm/include/llvm/TextAPI/InterfaceFile.h index 6ef4db2ae158..5f07397adaca 100644 --- a/llvm/include/llvm/TextAPI/InterfaceFile.h +++ b/llvm/include/llvm/TextAPI/InterfaceFile.h @@ -19,11 +19,7 @@ #include "llvm/ADT/Hashing.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/iterator.h" -#include "llvm/BinaryFormat/MachO.h" -#include "llvm/BinaryFormat/Magic.h" #include "llvm/Support/Allocator.h" -#include "llvm/Support/Error.h" -#include "llvm/TextAPI/Architecture.h" #include "llvm/TextAPI/ArchitectureSet.h" #include "llvm/TextAPI/PackedVersion.h" #include "llvm/TextAPI/Platform.h" diff --git a/llvm/include/llvm/TextAPI/Platform.h b/llvm/include/llvm/TextAPI/Platform.h index f7affc3ae980..d4225ca533fc 100644 --- a/llvm/include/llvm/TextAPI/Platform.h +++ b/llvm/include/llvm/TextAPI/Platform.h @@ -18,29 +18,14 @@ namespace llvm { namespace MachO { -/// Defines the list of MachO platforms. -enum class PlatformKind : unsigned { - unknown, - macOS = MachO::PLATFORM_MACOS, - iOS = MachO::PLATFORM_IOS, - tvOS = MachO::PLATFORM_TVOS, - watchOS = MachO::PLATFORM_WATCHOS, - bridgeOS = MachO::PLATFORM_BRIDGEOS, - macCatalyst = MachO::PLATFORM_MACCATALYST, - iOSSimulator = MachO::PLATFORM_IOSSIMULATOR, - tvOSSimulator = MachO::PLATFORM_TVOSSIMULATOR, - watchOSSimulator = MachO::PLATFORM_WATCHOSSIMULATOR, - driverKit = MachO::PLATFORM_DRIVERKIT, -}; +using PlatformSet = SmallSet<PlatformType, 3>; -using PlatformSet = SmallSet<PlatformKind, 3>; - -PlatformKind mapToPlatformKind(PlatformKind Platform, bool WantSim); -PlatformKind mapToPlatformKind(const Triple &Target); +PlatformType mapToPlatformType(PlatformType Platform, bool WantSim); +PlatformType mapToPlatformType(const Triple &Target); PlatformSet mapToPlatformSet(ArrayRef<Triple> Targets); -StringRef getPlatformName(PlatformKind Platform); -PlatformKind getPlatformFromName(StringRef Name); -std::string getOSAndEnvironmentName(PlatformKind Platform, +StringRef getPlatformName(PlatformType Platform); +PlatformType getPlatformFromName(StringRef Name); +std::string getOSAndEnvironmentName(PlatformType Platform, std::string Version = ""); } // end namespace MachO. diff --git a/llvm/include/llvm/TextAPI/Target.h b/llvm/include/llvm/TextAPI/Target.h index c2588b9d5a21..fbb76295f706 100644 --- a/llvm/include/llvm/TextAPI/Target.h +++ b/llvm/include/llvm/TextAPI/Target.h @@ -9,13 +9,15 @@ #ifndef LLVM_TEXTAPI_TARGET_H #define LLVM_TEXTAPI_TARGET_H -#include "llvm/ADT/Triple.h" #include "llvm/Support/Error.h" #include "llvm/TextAPI/Architecture.h" #include "llvm/TextAPI/ArchitectureSet.h" #include "llvm/TextAPI/Platform.h" namespace llvm { + +class Triple; + namespace MachO { // This is similar to a llvm Triple, but the triple doesn't have all the @@ -24,17 +26,17 @@ namespace MachO { class Target { public: Target() = default; - Target(Architecture Arch, PlatformKind Platform) + Target(Architecture Arch, PlatformType Platform) : Arch(Arch), Platform(Platform) {} explicit Target(const llvm::Triple &Triple) - : Arch(mapToArchitecture(Triple)), Platform(mapToPlatformKind(Triple)) {} + : Arch(mapToArchitecture(Triple)), Platform(mapToPlatformType(Triple)) {} static llvm::Expected<Target> create(StringRef Target); operator std::string() const; Architecture Arch; - PlatformKind Platform; + PlatformType Platform; }; inline bool operator==(const Target &LHS, const Target &RHS) { diff --git a/llvm/include/llvm/Transforms/Coroutines/CoroSplit.h b/llvm/include/llvm/Transforms/Coroutines/CoroSplit.h index ffca51578551..7623c9c0eb68 100644 --- a/llvm/include/llvm/Transforms/Coroutines/CoroSplit.h +++ b/llvm/include/llvm/Transforms/Coroutines/CoroSplit.h @@ -22,14 +22,14 @@ namespace llvm { struct CoroSplitPass : PassInfoMixin<CoroSplitPass> { - CoroSplitPass(bool ReuseFrameSlot = false) : ReuseFrameSlot(ReuseFrameSlot) {} + CoroSplitPass(bool OptimizeFrame = false) : OptimizeFrame(OptimizeFrame) {} PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM, LazyCallGraph &CG, CGSCCUpdateResult &UR); static bool isRequired() { return true; } // Would be true if the Optimization level isn't O0. - bool ReuseFrameSlot; + bool OptimizeFrame; }; } // end namespace llvm diff --git a/llvm/include/llvm/Transforms/IPO/Attributor.h b/llvm/include/llvm/Transforms/IPO/Attributor.h index d4cbc9bd20b7..d56a43ec7961 100644 --- a/llvm/include/llvm/Transforms/IPO/Attributor.h +++ b/llvm/include/llvm/Transforms/IPO/Attributor.h @@ -133,7 +133,6 @@ struct InformationCache; struct AAIsDead; struct AttributorCallGraph; -class AAManager; class AAResults; class Function; @@ -173,7 +172,8 @@ combineOptionalValuesInAAValueLatice(const Optional<Value *> &A, const Optional<Value *> &B, Type *Ty); /// Return the initial value of \p Obj with type \p Ty if that is a constant. -Constant *getInitialValueForObj(Value &Obj, Type &Ty); +Constant *getInitialValueForObj(Value &Obj, Type &Ty, + const TargetLibraryInfo *TLI); /// Collect all potential underlying objects of \p Ptr at position \p CtxI in /// \p Objects. Assumed information is used and dependences onto \p QueryingAA @@ -1623,10 +1623,17 @@ public: /// /// This method will evaluate \p Pred on all (transitive) uses of the /// associated value and return true if \p Pred holds every time. + /// If uses are skipped in favor of equivalent ones, e.g., if we look through + /// memory, the \p EquivalentUseCB will be used to give the caller an idea + /// what original used was replaced by a new one (or new ones). The visit is + /// cut short if \p EquivalentUseCB returns false and the function will return + /// false as well. bool checkForAllUses(function_ref<bool(const Use &, bool &)> Pred, const AbstractAttribute &QueryingAA, const Value &V, bool CheckBBLivenessOnly = false, - DepClassTy LivenessDepClass = DepClassTy::OPTIONAL); + DepClassTy LivenessDepClass = DepClassTy::OPTIONAL, + function_ref<bool(const Use &OldU, const Use &NewU)> + EquivalentUseCB = nullptr); /// Emit a remark generically. /// @@ -2354,11 +2361,11 @@ private: }; /// Simple wrapper for a single bit (boolean) state. -struct BooleanState : public IntegerStateBase<bool, 1, 0> { - using super = IntegerStateBase<bool, 1, 0>; +struct BooleanState : public IntegerStateBase<bool, true, false> { + using super = IntegerStateBase<bool, true, false>; using base_t = IntegerStateBase::base_t; - BooleanState() : super() {} + BooleanState() {} BooleanState(base_t Assumed) : super(Assumed) {} /// Set the assumed value to \p Value but never below the known one. diff --git a/llvm/include/llvm/Transforms/IPO/IROutliner.h b/llvm/include/llvm/Transforms/IPO/IROutliner.h index 110c0b4dcf16..ed74c8ed0e96 100644 --- a/llvm/include/llvm/Transforms/IPO/IROutliner.h +++ b/llvm/include/llvm/Transforms/IPO/IROutliner.h @@ -95,6 +95,10 @@ struct OutlinableRegion { /// required for the following basic blocks in this case. bool EndsInBranch = false; + /// The PHIBlocks with their corresponding return block based on the return + /// value as the key. + DenseMap<Value *, BasicBlock *> PHIBlocks; + /// Mapping of the argument number in the deduplicated function /// to a given constant, which is used when creating the arguments to the call /// to the newly created deduplicated function. This is handled separately @@ -182,7 +186,14 @@ public: IROutliner(function_ref<TargetTransformInfo &(Function &)> GTTI, function_ref<IRSimilarityIdentifier &(Module &)> GIRSI, function_ref<OptimizationRemarkEmitter &(Function &)> GORE) - : getTTI(GTTI), getIRSI(GIRSI), getORE(GORE) {} + : getTTI(GTTI), getIRSI(GIRSI), getORE(GORE) { + + // Check that the DenseMap implementation has not changed. + assert(DenseMapInfo<unsigned>::getEmptyKey() == (unsigned)-1 && + "DenseMapInfo<unsigned>'s empty key isn't -1!"); + assert(DenseMapInfo<unsigned>::getTombstoneKey() == (unsigned)-2 && + "DenseMapInfo<unsigned>'s tombstone key isn't -2!"); + } bool run(Module &M); private: @@ -331,8 +342,7 @@ private: bool visitBranchInst(BranchInst &BI) { return EnableBranches; } - // TODO: Determine a scheme to resolve when the labels are similar enough. - bool visitPHINode(PHINode &PN) { return false; } + bool visitPHINode(PHINode &PN) { return EnableBranches; } // TODO: Handle allocas. bool visitAllocaInst(AllocaInst &AI) { return false; } // VAArg instructions are not allowed since this could cause difficulty when @@ -354,7 +364,15 @@ private: // that they have a name in these cases. bool visitCallInst(CallInst &CI) { Function *F = CI.getCalledFunction(); - if (!F || CI.isIndirectCall() || !F->hasName()) + bool IsIndirectCall = CI.isIndirectCall(); + if (IsIndirectCall && !EnableIndirectCalls) + return false; + if (!F && !IsIndirectCall) + return false; + // Returning twice can cause issues with the state of the function call + // that were not expected when the function was used, so we do not include + // the call in outlined functions. + if (CI.canReturnTwice()) return false; return true; } @@ -373,6 +391,10 @@ private: // The flag variable that marks whether we should allow branch instructions // to be outlined. bool EnableBranches = false; + + // The flag variable that marks whether we should allow indirect calls + // to be outlined. + bool EnableIndirectCalls = true; }; /// A InstVisitor used to exclude certain instructions from being outlined. diff --git a/llvm/include/llvm/Transforms/IPO/ModuleInliner.h b/llvm/include/llvm/Transforms/IPO/ModuleInliner.h index 963d74d71003..7474e48aafaf 100644 --- a/llvm/include/llvm/Transforms/IPO/ModuleInliner.h +++ b/llvm/include/llvm/Transforms/IPO/ModuleInliner.h @@ -18,9 +18,6 @@ namespace llvm { -class AssumptionCacheTracker; -class ProfileSummaryInfo; - /// The module inliner pass for the new pass manager. /// /// This pass wires together the inlining utilities and the inline cost diff --git a/llvm/include/llvm/Transforms/IPO/OpenMPOpt.h b/llvm/include/llvm/Transforms/IPO/OpenMPOpt.h index c6ac5bd46f57..bf08336663b6 100644 --- a/llvm/include/llvm/Transforms/IPO/OpenMPOpt.h +++ b/llvm/include/llvm/Transforms/IPO/OpenMPOpt.h @@ -21,7 +21,7 @@ namespace omp { using Kernel = Function *; /// Set of kernels in the module -using KernelSet = SmallPtrSet<Kernel, 4>; +using KernelSet = SetVector<Kernel>; /// Helper to determine if \p M contains OpenMP. bool containsOpenMP(Module &M); diff --git a/llvm/include/llvm/Transforms/IPO/PassManagerBuilder.h b/llvm/include/llvm/Transforms/IPO/PassManagerBuilder.h index 7f321a688aff..3b944878a810 100644 --- a/llvm/include/llvm/Transforms/IPO/PassManagerBuilder.h +++ b/llvm/include/llvm/Transforms/IPO/PassManagerBuilder.h @@ -24,7 +24,6 @@ namespace llvm { class ModuleSummaryIndex; class Pass; class TargetLibraryInfoImpl; -class TargetMachine; // The old pass manager infrastructure is hidden in a legacy namespace now. namespace legacy { diff --git a/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h b/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h index c6aee439b5a0..f8cb6dc73a6f 100644 --- a/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h +++ b/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h @@ -480,9 +480,9 @@ public: return llvm::ComputeNumSignBits(Op, DL, Depth, &AC, CxtI, &DT); } - unsigned ComputeMinSignedBits(const Value *Op, unsigned Depth = 0, - const Instruction *CxtI = nullptr) const { - return llvm::ComputeMinSignedBits(Op, DL, Depth, &AC, CxtI, &DT); + unsigned ComputeMaxSignificantBits(const Value *Op, unsigned Depth = 0, + const Instruction *CxtI = nullptr) const { + return llvm::ComputeMaxSignificantBits(Op, DL, Depth, &AC, CxtI, &DT); } OverflowResult computeOverflowForUnsignedMul(const Value *LHS, diff --git a/llvm/include/llvm/Transforms/Instrumentation/AddressSanitizerCommon.h b/llvm/include/llvm/Transforms/Instrumentation/AddressSanitizerCommon.h index 6c351e3f8e1f..5a0fb835606a 100644 --- a/llvm/include/llvm/Transforms/Instrumentation/AddressSanitizerCommon.h +++ b/llvm/include/llvm/Transforms/Instrumentation/AddressSanitizerCommon.h @@ -26,6 +26,7 @@ class InterestingMemoryOperand { public: Use *PtrUse; bool IsWrite; + Type *OpType; uint64_t TypeSize; MaybeAlign Alignment; // The mask Value, if we're looking at a masked load/store. @@ -34,7 +35,8 @@ public: InterestingMemoryOperand(Instruction *I, unsigned OperandNo, bool IsWrite, class Type *OpType, MaybeAlign Alignment, Value *MaybeMask = nullptr) - : IsWrite(IsWrite), Alignment(Alignment), MaybeMask(MaybeMask) { + : IsWrite(IsWrite), OpType(OpType), Alignment(Alignment), + MaybeMask(MaybeMask) { const DataLayout &DL = I->getModule()->getDataLayout(); TypeSize = DL.getTypeStoreSizeInBits(OpType); PtrUse = &I->getOperandUse(OperandNo); diff --git a/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h b/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h index 94b156f3b137..64523d7d073c 100644 --- a/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h +++ b/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h @@ -100,7 +100,7 @@ private: /// /// If the counter array doesn't yet exist, the profile data variables /// referring to them will also be created. - GlobalVariable *getOrCreateRegionCounters(InstrProfIncrementInst *Inc); + GlobalVariable *getOrCreateRegionCounters(InstrProfInstBase *Inc); /// Emit the section with compressed function names. void emitNameData(); diff --git a/llvm/include/llvm/Transforms/Instrumentation/MemorySanitizer.h b/llvm/include/llvm/Transforms/Instrumentation/MemorySanitizer.h index d47beb93397e..e5779dc775ba 100644 --- a/llvm/include/llvm/Transforms/Instrumentation/MemorySanitizer.h +++ b/llvm/include/llvm/Transforms/Instrumentation/MemorySanitizer.h @@ -19,11 +19,15 @@ namespace llvm { struct MemorySanitizerOptions { - MemorySanitizerOptions() : MemorySanitizerOptions(0, false, false){}; - MemorySanitizerOptions(int TrackOrigins, bool Recover, bool Kernel); + MemorySanitizerOptions() : MemorySanitizerOptions(0, false, false, false){}; + MemorySanitizerOptions(int TrackOrigins, bool Recover, bool Kernel) + : MemorySanitizerOptions(TrackOrigins, Recover, Kernel, false) {} + MemorySanitizerOptions(int TrackOrigins, bool Recover, bool Kernel, + bool EagerChecks); bool Kernel; int TrackOrigins; bool Recover; + bool EagerChecks; }; // Insert MemorySanitizer instrumentation (detection of uninitialized reads) diff --git a/llvm/include/llvm/Transforms/Scalar/GVN.h b/llvm/include/llvm/Transforms/Scalar/GVN.h index cbe5057b9cde..9e660c92124e 100644 --- a/llvm/include/llvm/Transforms/Scalar/GVN.h +++ b/llvm/include/llvm/Transforms/Scalar/GVN.h @@ -39,11 +39,9 @@ class AssumptionCache; class BasicBlock; class BranchInst; class CallInst; -class Constant; class ExtractValueInst; class Function; class FunctionPass; -class IntrinsicInst; class LoadInst; class LoopInfo; class MemDepResult; diff --git a/llvm/include/llvm/Transforms/Scalar/InstSimplifyPass.h b/llvm/include/llvm/Transforms/Scalar/InstSimplifyPass.h index f5781e085f7b..09a4a95401d8 100644 --- a/llvm/include/llvm/Transforms/Scalar/InstSimplifyPass.h +++ b/llvm/include/llvm/Transforms/Scalar/InstSimplifyPass.h @@ -18,8 +18,6 @@ namespace llvm { -class FunctionPass; - /// Run instruction simplification across each instruction in the function. /// /// Instruction simplification has useful constraints in some contexts: diff --git a/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h b/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h index 419729271a23..7ba9d65cae55 100644 --- a/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h +++ b/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h @@ -435,8 +435,7 @@ public: bool UseBlockFrequencyInfo = false, bool UseBranchProbabilityInfo = false, bool LoopNestMode = false) - : Pass(std::move(Pass)), LoopCanonicalizationFPM(), - UseMemorySSA(UseMemorySSA), + : Pass(std::move(Pass)), UseMemorySSA(UseMemorySSA), UseBlockFrequencyInfo(UseBlockFrequencyInfo), UseBranchProbabilityInfo(UseBranchProbabilityInfo), LoopNestMode(LoopNestMode) { diff --git a/llvm/include/llvm/Transforms/Scalar/LoopReroll.h b/llvm/include/llvm/Transforms/Scalar/LoopReroll.h index 6ae309e48a28..496e8df85ea0 100644 --- a/llvm/include/llvm/Transforms/Scalar/LoopReroll.h +++ b/llvm/include/llvm/Transforms/Scalar/LoopReroll.h @@ -14,8 +14,6 @@ namespace llvm { -class Function; - class LoopRerollPass : public PassInfoMixin<LoopRerollPass> { public: PreservedAnalyses run(Loop &L, LoopAnalysisManager &AM, diff --git a/llvm/include/llvm/Transforms/Scalar/LoopUnrollAndJamPass.h b/llvm/include/llvm/Transforms/Scalar/LoopUnrollAndJamPass.h index 6125fc7636a0..72663d3d62a8 100644 --- a/llvm/include/llvm/Transforms/Scalar/LoopUnrollAndJamPass.h +++ b/llvm/include/llvm/Transforms/Scalar/LoopUnrollAndJamPass.h @@ -13,7 +13,6 @@ #include "llvm/Transforms/Scalar/LoopPassManager.h" namespace llvm { -class Function; /// A simple loop rotation transformation. class LoopUnrollAndJamPass : public PassInfoMixin<LoopUnrollAndJamPass> { diff --git a/llvm/include/llvm/Transforms/Scalar/SCCP.h b/llvm/include/llvm/Transforms/Scalar/SCCP.h index 2d7c94918699..cd4100447880 100644 --- a/llvm/include/llvm/Transforms/Scalar/SCCP.h +++ b/llvm/include/llvm/Transforms/Scalar/SCCP.h @@ -32,8 +32,6 @@ namespace llvm { -class PostDominatorTree; - /// This pass performs function-level constant propagation and merging. class SCCPPass : public PassInfoMixin<SCCPPass> { public: diff --git a/llvm/include/llvm/Transforms/Scalar/SROA.h b/llvm/include/llvm/Transforms/Scalar/SROA.h index f1a43435d89a..b74c45e71d95 100644 --- a/llvm/include/llvm/Transforms/Scalar/SROA.h +++ b/llvm/include/llvm/Transforms/Scalar/SROA.h @@ -27,7 +27,6 @@ class AllocaInst; class AssumptionCache; class DominatorTree; class Function; -class Instruction; class LLVMContext; class PHINode; class SelectInst; diff --git a/llvm/include/llvm/Transforms/Scalar/WarnMissedTransforms.h b/llvm/include/llvm/Transforms/Scalar/WarnMissedTransforms.h index 2d5942a3f569..04a5f7e6ff38 100644 --- a/llvm/include/llvm/Transforms/Scalar/WarnMissedTransforms.h +++ b/llvm/include/llvm/Transforms/Scalar/WarnMissedTransforms.h @@ -17,8 +17,6 @@ namespace llvm { class Function; -class Loop; -class LPMUpdater; // New pass manager boilerplate. class WarnMissedTransformationsPass diff --git a/llvm/include/llvm/Transforms/Utils/AssumeBundleBuilder.h b/llvm/include/llvm/Transforms/Utils/AssumeBundleBuilder.h index 5c06af9bc84c..a497722eece6 100644 --- a/llvm/include/llvm/Transforms/Utils/AssumeBundleBuilder.h +++ b/llvm/include/llvm/Transforms/Utils/AssumeBundleBuilder.h @@ -22,7 +22,6 @@ #include "llvm/IR/PassManager.h" namespace llvm { -class IntrinsicInst; class AssumptionCache; class DominatorTree; diff --git a/llvm/include/llvm/Transforms/Utils/Cloning.h b/llvm/include/llvm/Transforms/Utils/Cloning.h index 5a1f322b2054..fdc55bea99e7 100644 --- a/llvm/include/llvm/Transforms/Utils/Cloning.h +++ b/llvm/include/llvm/Transforms/Utils/Cloning.h @@ -33,13 +33,11 @@ class AAResults; class AllocaInst; class BasicBlock; class BlockFrequencyInfo; -class CallInst; class CallGraph; class DebugInfoFinder; class DominatorTree; class Function; class Instruction; -class InvokeInst; class Loop; class LoopInfo; class Module; diff --git a/llvm/include/llvm/Transforms/Utils/CodeExtractor.h b/llvm/include/llvm/Transforms/Utils/CodeExtractor.h index f08173e45a5b..8aed3d0e40d9 100644 --- a/llvm/include/llvm/Transforms/Utils/CodeExtractor.h +++ b/llvm/include/llvm/Transforms/Utils/CodeExtractor.h @@ -168,7 +168,7 @@ public: /// /// Based on the blocks used when constructing the code extractor, /// determine whether it is eligible for extraction. - /// + /// /// Checks that varargs handling (with vastart and vaend) is only done in /// the outlined blocks. bool isEligible() const; @@ -214,6 +214,10 @@ public: /// original block will be added to the outline region. BasicBlock *findOrCreateBlockForHoisting(BasicBlock *CommonExitBlock); + /// Exclude a value from aggregate argument passing when extracting a code + /// region, passing it instead as a scalar. + void excludeArgFromAggregate(Value *Arg); + private: struct LifetimeMarkerInfo { bool SinkLifeStart = false; @@ -222,6 +226,8 @@ public: Instruction *LifeEnd = nullptr; }; + ValueSet ExcludeArgsFromAggregate; + LifetimeMarkerInfo getLifetimeMarkers(const CodeExtractorAnalysisCache &CEAC, Instruction *Addr, BasicBlock *ExitBlock) const; diff --git a/llvm/include/llvm/Transforms/Utils/CodeLayout.h b/llvm/include/llvm/Transforms/Utils/CodeLayout.h index 987a5651a8b6..a0e5f8c7d014 100644 --- a/llvm/include/llvm/Transforms/Utils/CodeLayout.h +++ b/llvm/include/llvm/Transforms/Utils/CodeLayout.h @@ -20,8 +20,6 @@ namespace llvm { -class MachineBasicBlock; - /// Find a layout of nodes (basic blocks) of a given CFG optimizing jump /// locality and thus processor I-cache utilization. This is achieved via /// increasing the number of fall-through jumps and co-locating frequently diff --git a/llvm/include/llvm/Transforms/Utils/CtorUtils.h b/llvm/include/llvm/Transforms/Utils/CtorUtils.h index 3625ee662b1c..3ef3ba244b43 100644 --- a/llvm/include/llvm/Transforms/Utils/CtorUtils.h +++ b/llvm/include/llvm/Transforms/Utils/CtorUtils.h @@ -17,7 +17,6 @@ namespace llvm { -class GlobalVariable; class Function; class Module; diff --git a/llvm/include/llvm/Transforms/Utils/Evaluator.h b/llvm/include/llvm/Transforms/Utils/Evaluator.h index 1b93b0af86e2..99e826bf855f 100644 --- a/llvm/include/llvm/Transforms/Utils/Evaluator.h +++ b/llvm/include/llvm/Transforms/Utils/Evaluator.h @@ -36,6 +36,49 @@ class TargetLibraryInfo; /// be iterated over after the evaluation is complete. Once an evaluation call /// fails, the evaluation object should not be reused. class Evaluator { + struct MutableAggregate; + + /// The evaluator represents values either as a Constant*, or as a + /// MutableAggregate, which allows changing individual aggregate elements + /// without creating a new interned Constant. + class MutableValue { + PointerUnion<Constant *, MutableAggregate *> Val; + void clear(); + bool makeMutable(); + + public: + MutableValue(Constant *C) { Val = C; } + MutableValue(const MutableValue &) = delete; + MutableValue(MutableValue &&Other) { + Val = Other.Val; + Other.Val = nullptr; + } + ~MutableValue() { clear(); } + + Type *getType() const { + if (auto *C = Val.dyn_cast<Constant *>()) + return C->getType(); + return Val.get<MutableAggregate *>()->Ty; + } + + Constant *toConstant() const { + if (auto *C = Val.dyn_cast<Constant *>()) + return C; + return Val.get<MutableAggregate *>()->toConstant(); + } + + Constant *read(Type *Ty, APInt Offset, const DataLayout &DL) const; + bool write(Constant *V, APInt Offset, const DataLayout &DL); + }; + + struct MutableAggregate { + Type *Ty; + SmallVector<MutableValue> Elements; + + MutableAggregate(Type *Ty) : Ty(Ty) {} + Constant *toConstant() const; + }; + public: Evaluator(const DataLayout &DL, const TargetLibraryInfo *TLI) : DL(DL), TLI(TLI) { @@ -57,8 +100,11 @@ public: bool EvaluateFunction(Function *F, Constant *&RetVal, const SmallVectorImpl<Constant*> &ActualArgs); - const DenseMap<Constant *, Constant *> &getMutatedMemory() const { - return MutatedMemory; + DenseMap<GlobalVariable *, Constant *> getMutatedInitializers() const { + DenseMap<GlobalVariable *, Constant *> Result; + for (auto &Pair : MutatedMemory) + Result[Pair.first] = Pair.second.toConstant(); + return Result; } const SmallPtrSetImpl<GlobalVariable *> &getInvariants() const { @@ -81,7 +127,7 @@ private: } /// Casts call result to a type of bitcast call expression - Constant *castCallResultIfNeeded(Value *CallExpr, Constant *RV); + Constant *castCallResultIfNeeded(Type *ReturnType, Constant *RV); /// Given call site return callee and list of its formal arguments Function *getCalleeWithFormalArgs(CallBase &CB, @@ -106,7 +152,7 @@ private: /// For each store we execute, we update this map. Loads check this to get /// the most up-to-date value. If evaluation is successful, this state is /// committed to the process. - DenseMap<Constant*, Constant*> MutatedMemory; + DenseMap<GlobalVariable *, MutableValue> MutatedMemory; /// To 'execute' an alloca, we create a temporary global variable to represent /// its body. This vector is needed so we can delete the temporary globals diff --git a/llvm/include/llvm/Transforms/Utils/GlobalStatus.h b/llvm/include/llvm/Transforms/Utils/GlobalStatus.h index 78d7845c4353..775dd23d8f23 100644 --- a/llvm/include/llvm/Transforms/Utils/GlobalStatus.h +++ b/llvm/include/llvm/Transforms/Utils/GlobalStatus.h @@ -73,10 +73,6 @@ struct GlobalStatus { const Function *AccessingFunction = nullptr; bool HasMultipleAccessingFunctions = false; - /// Set to true if this global has a user that is not an instruction (e.g. a - /// constant expr or GV initializer). - bool HasNonInstructionUser = false; - /// Set to the strongest atomic ordering requirement. AtomicOrdering Ordering = AtomicOrdering::NotAtomic; diff --git a/llvm/include/llvm/Transforms/Utils/Local.h b/llvm/include/llvm/Transforms/Utils/Local.h index a914c6e0925f..873127554b47 100644 --- a/llvm/include/llvm/Transforms/Utils/Local.h +++ b/llvm/include/llvm/Transforms/Utils/Local.h @@ -42,9 +42,7 @@ class BasicBlock; class BranchInst; class CallBase; class CallInst; -class DbgDeclareInst; class DbgVariableIntrinsic; -class DbgValueInst; class DIBuilder; class DomTreeUpdater; class Function; @@ -243,7 +241,7 @@ inline Align getKnownAlignment(Value *V, const DataLayout &DL, CallInst *createCallMatchingInvoke(InvokeInst *II); /// This function converts the specified invoek into a normall call. -void changeToCall(InvokeInst *II, DomTreeUpdater *DTU = nullptr); +CallInst *changeToCall(InvokeInst *II, DomTreeUpdater *DTU = nullptr); ///===---------------------------------------------------------------------===// /// Dbg Intrinsic utilities diff --git a/llvm/include/llvm/Transforms/Utils/LoopPeel.h b/llvm/include/llvm/Transforms/Utils/LoopPeel.h index 6f1b4a880457..7b6595c192de 100644 --- a/llvm/include/llvm/Transforms/Utils/LoopPeel.h +++ b/llvm/include/llvm/Transforms/Utils/LoopPeel.h @@ -32,7 +32,7 @@ gatherPeelingPreferences(Loop *L, ScalarEvolution &SE, void computePeelCount(Loop *L, unsigned LoopSize, TargetTransformInfo::PeelingPreferences &PP, - unsigned &TripCount, DominatorTree &DT, + unsigned TripCount, DominatorTree &DT, ScalarEvolution &SE, unsigned Threshold = UINT_MAX); } // end namespace llvm diff --git a/llvm/include/llvm/Transforms/Utils/LoopUtils.h b/llvm/include/llvm/Transforms/Utils/LoopUtils.h index e0a9115f61b0..3a712d78df67 100644 --- a/llvm/include/llvm/Transforms/Utils/LoopUtils.h +++ b/llvm/include/llvm/Transforms/Utils/LoopUtils.h @@ -37,7 +37,6 @@ class MemorySSAUpdater; class OptimizationRemarkEmitter; class PredIteratorCache; class ScalarEvolution; -class ScalarEvolutionExpander; class SCEV; class SCEVExpander; class TargetLibraryInfo; diff --git a/llvm/include/llvm/Transforms/Utils/MemoryOpRemark.h b/llvm/include/llvm/Transforms/Utils/MemoryOpRemark.h index e5f8a46eaf23..8dc0f1e26a92 100644 --- a/llvm/include/llvm/Transforms/Utils/MemoryOpRemark.h +++ b/llvm/include/llvm/Transforms/Utils/MemoryOpRemark.h @@ -27,8 +27,6 @@ class Instruction; class IntrinsicInst; class Value; class OptimizationRemarkEmitter; -class OptimizationRemarkMissed; -class OptimizationRemarkAnalysis; class StoreInst; // FIXME: Once we get to more remarks like this one, we need to re-evaluate how diff --git a/llvm/include/llvm/Transforms/Utils/ModuleUtils.h b/llvm/include/llvm/Transforms/Utils/ModuleUtils.h index f53ba546dc5c..9bbe8ea7e1e8 100644 --- a/llvm/include/llvm/Transforms/Utils/ModuleUtils.h +++ b/llvm/include/llvm/Transforms/Utils/ModuleUtils.h @@ -92,7 +92,7 @@ void appendToCompilerUsed(Module &M, ArrayRef<GlobalValue *> Values); /// DeadComdatFunctions are those where every member of the comdat is listed /// and thus removing them is safe (provided *all* are removed). void filterDeadComdatFunctions( - Module &M, SmallVectorImpl<Function *> &DeadComdatFunctions); + SmallVectorImpl<Function *> &DeadComdatFunctions); /// Produce a unique identifier for this module by taking the MD5 sum of /// the names of the module's strong external symbols that are not comdat diff --git a/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h b/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h index efc3cc775e11..277eb7acf238 100644 --- a/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h +++ b/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h @@ -18,6 +18,7 @@ #include "llvm/ADT/Optional.h" #include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/Analysis/InstSimplifyFolder.h" #include "llvm/Analysis/ScalarEvolutionExpressions.h" #include "llvm/Analysis/ScalarEvolutionNormalization.h" #include "llvm/Analysis/TargetFolder.h" @@ -122,7 +123,7 @@ class SCEVExpander : public SCEVVisitor<SCEVExpander, Value *> { /// "expanded" form. bool LSRMode; - typedef IRBuilder<TargetFolder, IRBuilderCallbackInserter> BuilderType; + typedef IRBuilder<InstSimplifyFolder, IRBuilderCallbackInserter> BuilderType; BuilderType Builder; // RAII object that stores the current insertion point and restores it when @@ -178,7 +179,7 @@ public: : SE(se), DL(DL), IVName(name), PreserveLCSSA(PreserveLCSSA), IVIncInsertLoop(nullptr), IVIncInsertPos(nullptr), CanonicalMode(true), LSRMode(false), - Builder(se.getContext(), TargetFolder(DL), + Builder(se.getContext(), InstSimplifyFolder(DL), IRBuilderCallbackInserter( [this](Instruction *I) { rememberInstruction(I); })) { #ifdef LLVM_ENABLE_ABI_BREAKING_CHECKS @@ -450,6 +451,14 @@ private: /// Determine the most "relevant" loop for the given SCEV. const Loop *getRelevantLoop(const SCEV *); + Value *expandSMaxExpr(const SCEVNAryExpr *S); + + Value *expandUMaxExpr(const SCEVNAryExpr *S); + + Value *expandSMinExpr(const SCEVNAryExpr *S); + + Value *expandUMinExpr(const SCEVNAryExpr *S); + Value *visitConstant(const SCEVConstant *S) { return S->getValue(); } Value *visitPtrToIntExpr(const SCEVPtrToIntExpr *S); @@ -476,6 +485,8 @@ private: Value *visitUMinExpr(const SCEVUMinExpr *S); + Value *visitSequentialUMinExpr(const SCEVSequentialUMinExpr *S); + Value *visitUnknown(const SCEVUnknown *S) { return S->getValue(); } void rememberInstruction(Value *I); @@ -504,15 +515,13 @@ private: class SCEVExpanderCleaner { SCEVExpander &Expander; - DominatorTree &DT; - /// Indicates whether the result of the expansion is used. If false, the /// instructions added during expansion are removed. bool ResultUsed; public: - SCEVExpanderCleaner(SCEVExpander &Expander, DominatorTree &DT) - : Expander(Expander), DT(DT), ResultUsed(false) {} + SCEVExpanderCleaner(SCEVExpander &Expander) + : Expander(Expander), ResultUsed(false) {} ~SCEVExpanderCleaner() { cleanup(); } diff --git a/llvm/include/llvm/Transforms/Utils/UnifyFunctionExitNodes.h b/llvm/include/llvm/Transforms/Utils/UnifyFunctionExitNodes.h index 20b360212506..461669d6a217 100644 --- a/llvm/include/llvm/Transforms/Utils/UnifyFunctionExitNodes.h +++ b/llvm/include/llvm/Transforms/Utils/UnifyFunctionExitNodes.h @@ -19,8 +19,6 @@ namespace llvm { -class BasicBlock; - class UnifyFunctionExitNodesLegacyPass : public FunctionPass { public: static char ID; // Pass identification, replacement for typeid diff --git a/llvm/include/llvm/XRay/BlockIndexer.h b/llvm/include/llvm/XRay/BlockIndexer.h index eabc4e3f5c6e..77af77e5ec26 100644 --- a/llvm/include/llvm/XRay/BlockIndexer.h +++ b/llvm/include/llvm/XRay/BlockIndexer.h @@ -41,7 +41,7 @@ private: Block CurrentBlock{0, 0, nullptr, {}}; public: - explicit BlockIndexer(Index &I) : RecordVisitor(), Indices(I) {} + explicit BlockIndexer(Index &I) : Indices(I) {} Error visit(BufferExtents &) override; Error visit(WallclockRecord &) override; diff --git a/llvm/include/llvm/XRay/BlockPrinter.h b/llvm/include/llvm/XRay/BlockPrinter.h index 9215d64e73b9..2f9fed668069 100644 --- a/llvm/include/llvm/XRay/BlockPrinter.h +++ b/llvm/include/llvm/XRay/BlockPrinter.h @@ -36,8 +36,7 @@ class BlockPrinter : public RecordVisitor { State CurrentState = State::Start; public: - explicit BlockPrinter(raw_ostream &O, RecordPrinter &P) - : RecordVisitor(), OS(O), RP(P) {} + explicit BlockPrinter(raw_ostream &O, RecordPrinter &P) : OS(O), RP(P) {} Error visit(BufferExtents &) override; Error visit(WallclockRecord &) override; diff --git a/llvm/include/llvm/XRay/FDRRecordConsumer.h b/llvm/include/llvm/XRay/FDRRecordConsumer.h index 91020b0a4fef..8fff9fb86158 100644 --- a/llvm/include/llvm/XRay/FDRRecordConsumer.h +++ b/llvm/include/llvm/XRay/FDRRecordConsumer.h @@ -30,7 +30,7 @@ class LogBuilderConsumer : public RecordConsumer { public: explicit LogBuilderConsumer(std::vector<std::unique_ptr<Record>> &R) - : RecordConsumer(), Records(R) {} + : Records(R) {} Error consume(std::unique_ptr<Record> R) override; }; @@ -42,8 +42,7 @@ class PipelineConsumer : public RecordConsumer { std::vector<RecordVisitor *> Visitors; public: - PipelineConsumer(std::initializer_list<RecordVisitor *> V) - : RecordConsumer(), Visitors(V) {} + PipelineConsumer(std::initializer_list<RecordVisitor *> V) : Visitors(V) {} Error consume(std::unique_ptr<Record> R) override; }; diff --git a/llvm/include/llvm/XRay/FDRRecords.h b/llvm/include/llvm/XRay/FDRRecords.h index 9c318805d61b..8af88f5b0e13 100644 --- a/llvm/include/llvm/XRay/FDRRecords.h +++ b/llvm/include/llvm/XRay/FDRRecords.h @@ -424,7 +424,7 @@ public: static constexpr uint16_t DefaultVersion = 5u; explicit RecordInitializer(DataExtractor &DE, uint64_t &OP, uint16_t V) - : RecordVisitor(), E(DE), OffsetPtr(OP), Version(V) {} + : E(DE), OffsetPtr(OP), Version(V) {} explicit RecordInitializer(DataExtractor &DE, uint64_t &OP) : RecordInitializer(DE, OP, DefaultVersion) {} diff --git a/llvm/include/llvm/XRay/FDRTraceExpander.h b/llvm/include/llvm/XRay/FDRTraceExpander.h index 4a0bd24cfa9b..197c123fff1e 100644 --- a/llvm/include/llvm/XRay/FDRTraceExpander.h +++ b/llvm/include/llvm/XRay/FDRTraceExpander.h @@ -36,7 +36,7 @@ class TraceExpander : public RecordVisitor { public: explicit TraceExpander(function_ref<void(const XRayRecord &)> F, uint16_t L) - : RecordVisitor(), C(std::move(F)), LogVersion(L) {} + : C(std::move(F)), LogVersion(L) {} Error visit(BufferExtents &) override; Error visit(WallclockRecord &) override; diff --git a/llvm/include/llvm/XRay/RecordPrinter.h b/llvm/include/llvm/XRay/RecordPrinter.h index f7b809c062f2..8ca4794dce5e 100644 --- a/llvm/include/llvm/XRay/RecordPrinter.h +++ b/llvm/include/llvm/XRay/RecordPrinter.h @@ -25,7 +25,7 @@ class RecordPrinter : public RecordVisitor { public: explicit RecordPrinter(raw_ostream &O, std::string D) - : RecordVisitor(), OS(O), Delim(std::move(D)) {} + : OS(O), Delim(std::move(D)) {} explicit RecordPrinter(raw_ostream &O) : RecordPrinter(O, ""){}; diff --git a/llvm/include/llvm/module.modulemap b/llvm/include/llvm/module.modulemap index b0f7f2120606..25c7aeee148e 100644 --- a/llvm/include/llvm/module.modulemap +++ b/llvm/include/llvm/module.modulemap @@ -251,6 +251,7 @@ module LLVM_intrinsic_gen { module IR_Function { header "IR/Function.h" export * } module IR_InstrTypes { header "IR/InstrTypes.h" export * } module IR_Instructions { header "IR/Instructions.h" export * } + module IR_TypeFinder { header "IR/TypeFinder.h" export * } // Intrinsics.h |