diff options
Diffstat (limited to 'llvm/include/llvm/CodeGen/GlobalISel')
24 files changed, 1785 insertions, 681 deletions
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CSEInfo.h b/llvm/include/llvm/CodeGen/GlobalISel/CSEInfo.h index f76dec57c840..dd213bf68e62 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/CSEInfo.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/CSEInfo.h @@ -5,9 +5,9 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// -// +/// \file /// Provides analysis for continuously CSEing during GISel passes. -// +/// //===----------------------------------------------------------------------===// #ifndef LLVM_CODEGEN_GLOBALISEL_CSEINFO_H #define LLVM_CODEGEN_GLOBALISEL_CSEINFO_H diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h b/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h index 26ae7129f04a..6bdaddd9c6f5 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h @@ -17,11 +17,13 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SmallVector.h" #include "llvm/CodeGen/CallingConvLower.h" +#include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineOperand.h" #include "llvm/CodeGen/TargetCallingConv.h" #include "llvm/IR/Attributes.h" #include "llvm/IR/CallingConv.h" #include "llvm/IR/Type.h" +#include "llvm/IR/Value.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/MachineValueType.h" #include <cstdint> @@ -37,7 +39,6 @@ class MachineIRBuilder; struct MachinePointerInfo; class MachineRegisterInfo; class TargetLowering; -class Value; class CallLowering { const TargetLowering *TLI; @@ -64,10 +65,23 @@ public: // if the argument was an incoming arg. SmallVector<Register, 2> OrigRegs; - ArgInfo(ArrayRef<Register> Regs, Type *Ty, + /// Optionally track the original IR value for the argument. This may not be + /// meaningful in all contexts. This should only be used on for forwarding + /// through to use for aliasing information in MachinePointerInfo for memory + /// arguments. + const Value *OrigValue = nullptr; + + /// Index original Function's argument. + unsigned OrigArgIndex; + + /// Sentinel value for implicit machine-level input arguments. + static const unsigned NoArgIndex = UINT_MAX; + + ArgInfo(ArrayRef<Register> Regs, Type *Ty, unsigned OrigIndex, ArrayRef<ISD::ArgFlagsTy> Flags = ArrayRef<ISD::ArgFlagsTy>(), - bool IsFixed = true) - : BaseArgInfo(Ty, Flags, IsFixed), Regs(Regs.begin(), Regs.end()) { + bool IsFixed = true, const Value *OrigValue = nullptr) + : BaseArgInfo(Ty, Flags, IsFixed), Regs(Regs.begin(), Regs.end()), + OrigValue(OrigValue), OrigArgIndex(OrigIndex) { if (!Regs.empty() && Flags.empty()) this->Flags.push_back(ISD::ArgFlagsTy()); // FIXME: We should have just one way of saying "no register". @@ -76,6 +90,11 @@ public: "only void types should have no register"); } + ArgInfo(ArrayRef<Register> Regs, const Value &OrigValue, unsigned OrigIndex, + ArrayRef<ISD::ArgFlagsTy> Flags = ArrayRef<ISD::ArgFlagsTy>(), + bool IsFixed = true) + : ArgInfo(Regs, OrigValue.getType(), OrigIndex, Flags, IsFixed, &OrigValue) {} + ArgInfo() : BaseArgInfo() {} }; @@ -91,7 +110,7 @@ public: ArgInfo OrigRet; /// List of descriptors of the arguments passed to the function. - SmallVector<ArgInfo, 8> OrigArgs; + SmallVector<ArgInfo, 32> OrigArgs; /// Valid if the call has a swifterror inout parameter, and contains the /// vreg that the swifterror should be copied into after the call. @@ -129,10 +148,85 @@ public: /// returns. However, once a decision has been made on where an /// argument should go, exactly what happens can vary slightly. This /// class abstracts the differences. + /// + /// ValueAssigner should not depend on any specific function state, and + /// only determine the types and locations for arguments. + struct ValueAssigner { + ValueAssigner(bool IsIncoming, CCAssignFn *AssignFn_, + CCAssignFn *AssignFnVarArg_ = nullptr) + : AssignFn(AssignFn_), AssignFnVarArg(AssignFnVarArg_), + IsIncomingArgumentHandler(IsIncoming) { + + // Some targets change the handler depending on whether the call is + // varargs or not. If + if (!AssignFnVarArg) + AssignFnVarArg = AssignFn; + } + + virtual ~ValueAssigner() = default; + + /// Returns true if the handler is dealing with incoming arguments, + /// i.e. those that move values from some physical location to vregs. + bool isIncomingArgumentHandler() const { + return IsIncomingArgumentHandler; + } + + /// Wrap call to (typically tablegenerated CCAssignFn). This may be + /// overridden to track additional state information as arguments are + /// assigned or apply target specific hacks around the legacy + /// infrastructure. + virtual bool assignArg(unsigned ValNo, EVT OrigVT, MVT ValVT, MVT LocVT, + CCValAssign::LocInfo LocInfo, const ArgInfo &Info, + ISD::ArgFlagsTy Flags, CCState &State) { + if (getAssignFn(State.isVarArg())(ValNo, ValVT, LocVT, LocInfo, Flags, + State)) + return true; + StackOffset = State.getNextStackOffset(); + return false; + } + + /// Assignment function to use for a general call. + CCAssignFn *AssignFn; + + /// Assignment function to use for a variadic call. This is usually the same + /// as AssignFn on most targets. + CCAssignFn *AssignFnVarArg; + + /// Stack offset for next argument. At the end of argument evaluation, this + /// is typically the total stack size. + uint64_t StackOffset = 0; + + /// Select the appropriate assignment function depending on whether this is + /// a variadic call. + CCAssignFn *getAssignFn(bool IsVarArg) const { + return IsVarArg ? AssignFnVarArg : AssignFn; + } + + private: + const bool IsIncomingArgumentHandler; + virtual void anchor(); + }; + + struct IncomingValueAssigner : public ValueAssigner { + IncomingValueAssigner(CCAssignFn *AssignFn_, + CCAssignFn *AssignFnVarArg_ = nullptr) + : ValueAssigner(true, AssignFn_, AssignFnVarArg_) {} + }; + + struct OutgoingValueAssigner : public ValueAssigner { + OutgoingValueAssigner(CCAssignFn *AssignFn_, + CCAssignFn *AssignFnVarArg_ = nullptr) + : ValueAssigner(false, AssignFn_, AssignFnVarArg_) {} + }; + struct ValueHandler { + MachineIRBuilder &MIRBuilder; + MachineRegisterInfo &MRI; + const bool IsIncomingArgumentHandler; + ValueHandler(bool IsIncoming, MachineIRBuilder &MIRBuilder, - MachineRegisterInfo &MRI, CCAssignFn *AssignFn) - : MIRBuilder(MIRBuilder), MRI(MRI), AssignFn(AssignFn), + MachineRegisterInfo &MRI) + : MIRBuilder(MIRBuilder), MRI(MRI), IsIncomingArgumentHandler(IsIncoming) {} virtual ~ValueHandler() = default; @@ -148,8 +242,18 @@ public: /// direct SP manipulation, depending on the context. \p MPO /// should be initialized to an appropriate description of the /// address created. - virtual Register getStackAddress(uint64_t Size, int64_t Offset, - MachinePointerInfo &MPO) = 0; + virtual Register getStackAddress(uint64_t MemSize, int64_t Offset, + MachinePointerInfo &MPO, + ISD::ArgFlagsTy Flags) = 0; + + /// Return the in-memory size to write for the argument at \p VA. This may + /// be smaller than the allocated stack slot size. + /// + /// This is overridable primarily for targets to maintain compatibility with + /// hacks around the existing DAG call lowering infrastructure. + virtual LLT getStackValueStoreType(const DataLayout &DL, + const CCValAssign &VA, + ISD::ArgFlagsTy Flags) const; /// The specified value has been assigned to a physical register, /// handle the appropriate COPY (either to or from) and mark any @@ -161,59 +265,64 @@ public: /// location. Load or store it there, with appropriate extension /// if necessary. virtual void assignValueToAddress(Register ValVReg, Register Addr, - uint64_t Size, MachinePointerInfo &MPO, + LLT MemTy, MachinePointerInfo &MPO, CCValAssign &VA) = 0; - /// An overload which takes an ArgInfo if additional information about - /// the arg is needed. - virtual void assignValueToAddress(const ArgInfo &Arg, Register Addr, - uint64_t Size, MachinePointerInfo &MPO, + /// An overload which takes an ArgInfo if additional information about the + /// arg is needed. \p ValRegIndex is the index in \p Arg.Regs for the value + /// to store. + virtual void assignValueToAddress(const ArgInfo &Arg, unsigned ValRegIndex, + Register Addr, LLT MemTy, + MachinePointerInfo &MPO, CCValAssign &VA) { - assert(Arg.Regs.size() == 1); - assignValueToAddress(Arg.Regs[0], Addr, Size, MPO, VA); + assignValueToAddress(Arg.Regs[ValRegIndex], Addr, MemTy, MPO, VA); } /// Handle custom values, which may be passed into one or more of \p VAs. /// \return The number of \p VAs that have been assigned after the first /// one, and which should therefore be skipped from further /// processing. - virtual unsigned assignCustomValue(const ArgInfo &Arg, + virtual unsigned assignCustomValue(ArgInfo &Arg, ArrayRef<CCValAssign> VAs) { // This is not a pure virtual method because not all targets need to worry // about custom values. llvm_unreachable("Custom values not supported"); } + /// Do a memory copy of \p MemSize bytes from \p SrcPtr to \p DstPtr. This + /// is necessary for outgoing stack-passed byval arguments. + void + copyArgumentMemory(const ArgInfo &Arg, Register DstPtr, Register SrcPtr, + const MachinePointerInfo &DstPtrInfo, Align DstAlign, + const MachinePointerInfo &SrcPtrInfo, Align SrcAlign, + uint64_t MemSize, CCValAssign &VA) const; + /// Extend a register to the location type given in VA, capped at extending /// to at most MaxSize bits. If MaxSizeBits is 0 then no maximum is set. Register extendRegister(Register ValReg, CCValAssign &VA, unsigned MaxSizeBits = 0); - - virtual bool assignArg(unsigned ValNo, MVT ValVT, MVT LocVT, - CCValAssign::LocInfo LocInfo, const ArgInfo &Info, - ISD::ArgFlagsTy Flags, CCState &State) { - return AssignFn(ValNo, ValVT, LocVT, LocInfo, Flags, State); - } - - MachineIRBuilder &MIRBuilder; - MachineRegisterInfo &MRI; - CCAssignFn *AssignFn; - - private: - bool IsIncomingArgumentHandler; - virtual void anchor(); }; + /// Base class for ValueHandlers used for arguments coming into the current + /// function, or for return values received from a call. struct IncomingValueHandler : public ValueHandler { - IncomingValueHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI, - CCAssignFn *AssignFn) - : ValueHandler(true, MIRBuilder, MRI, AssignFn) {} + IncomingValueHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI) + : ValueHandler(/*IsIncoming*/ true, MIRBuilder, MRI) {} + + /// Insert G_ASSERT_ZEXT/G_ASSERT_SEXT or other hint instruction based on \p + /// VA, returning the new register if a hint was inserted. + Register buildExtensionHint(CCValAssign &VA, Register SrcReg, LLT NarrowTy); + + /// Provides a default implementation for argument handling. + void assignValueToReg(Register ValVReg, Register PhysReg, + CCValAssign &VA) override; }; + /// Base class for ValueHandlers used for arguments passed to a function call, + /// or for return values. struct OutgoingValueHandler : public ValueHandler { - OutgoingValueHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI, - CCAssignFn *AssignFn) - : ValueHandler(false, MIRBuilder, MRI, AssignFn) {} + OutgoingValueHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI) + : ValueHandler(/*IsIncoming*/ false, MIRBuilder, MRI) {} }; protected: @@ -243,44 +352,51 @@ protected: void setArgFlags(ArgInfo &Arg, unsigned OpIdx, const DataLayout &DL, const FuncInfoTy &FuncInfo) const; - /// Generate instructions for packing \p SrcRegs into one big register - /// corresponding to the aggregate type \p PackedTy. + /// Break \p OrigArgInfo into one or more pieces the calling convention can + /// process, returned in \p SplitArgs. For example, this should break structs + /// down into individual fields. /// - /// \param SrcRegs should contain one virtual register for each base type in - /// \p PackedTy, as returned by computeValueLLTs. + /// If \p Offsets is non-null, it points to a vector to be filled in + /// with the in-memory offsets of each of the individual values. + void splitToValueTypes(const ArgInfo &OrigArgInfo, + SmallVectorImpl<ArgInfo> &SplitArgs, + const DataLayout &DL, CallingConv::ID CallConv, + SmallVectorImpl<uint64_t> *Offsets = nullptr) const; + + /// Analyze the argument list in \p Args, using \p Assigner to populate \p + /// CCInfo. This will determine the types and locations to use for passed or + /// returned values. This may resize fields in \p Args if the value is split + /// across multiple registers or stack slots. /// - /// \return The packed register. - Register packRegs(ArrayRef<Register> SrcRegs, Type *PackedTy, - MachineIRBuilder &MIRBuilder) const; - - /// Generate instructions for unpacking \p SrcReg into the \p DstRegs - /// corresponding to the aggregate type \p PackedTy. + /// This is independent of the function state and can be used + /// to determine how a call would pass arguments without needing to change the + /// function. This can be used to check if arguments are suitable for tail + /// call lowering. /// - /// \param DstRegs should contain one virtual register for each base type in - /// \p PackedTy, as returned by computeValueLLTs. - void unpackRegs(ArrayRef<Register> DstRegs, Register SrcReg, Type *PackedTy, - MachineIRBuilder &MIRBuilder) const; + /// \return True if everything has succeeded, false otherwise. + bool determineAssignments(ValueAssigner &Assigner, + SmallVectorImpl<ArgInfo> &Args, + CCState &CCInfo) const; - /// Invoke Handler::assignArg on each of the given \p Args and then use + /// Invoke ValueAssigner::assignArg on each of the given \p Args and then use /// \p Handler to move them to the assigned locations. /// /// \return True if everything has succeeded, false otherwise. - bool handleAssignments(MachineIRBuilder &MIRBuilder, - SmallVectorImpl<ArgInfo> &Args, - ValueHandler &Handler) const; - bool handleAssignments(CCState &CCState, + bool determineAndHandleAssignments(ValueHandler &Handler, + ValueAssigner &Assigner, + SmallVectorImpl<ArgInfo> &Args, + MachineIRBuilder &MIRBuilder, + CallingConv::ID CallConv, bool IsVarArg, + Register ThisReturnReg = Register()) const; + + /// Use \p Handler to insert code to handle the argument/return values + /// represented by \p Args. It's expected determineAssignments previously + /// processed these arguments to populate \p CCState and \p ArgLocs. + bool handleAssignments(ValueHandler &Handler, SmallVectorImpl<ArgInfo> &Args, + CCState &CCState, SmallVectorImpl<CCValAssign> &ArgLocs, MachineIRBuilder &MIRBuilder, - SmallVectorImpl<ArgInfo> &Args, - ValueHandler &Handler) const; - - /// Analyze passed or returned values from a call, supplied in \p ArgInfo, - /// incorporating info about the passed values into \p CCState. - /// - /// Used to check if arguments are suitable for tail call lowering. - bool analyzeArgInfo(CCState &CCState, SmallVectorImpl<ArgInfo> &Args, - CCAssignFn &AssignFnFixed, - CCAssignFn &AssignFnVarArg) const; + Register ThisReturnReg = Register()) const; /// Check whether parameters to a call that are passed in callee saved /// registers are the same as from the calling function. This needs to be @@ -296,18 +412,14 @@ protected: /// \p Info is the CallLoweringInfo for the call. /// \p MF is the MachineFunction for the caller. /// \p InArgs contains the results of the call. - /// \p CalleeAssignFnFixed is the CCAssignFn to be used for the callee for - /// fixed arguments. - /// \p CalleeAssignFnVarArg is similar, but for varargs. - /// \p CallerAssignFnFixed is the CCAssignFn to be used for the caller for - /// fixed arguments. - /// \p CallerAssignFnVarArg is similar, but for varargs. + /// \p CalleeAssigner specifies the target's handling of the argument types + /// for the callee. + /// \p CallerAssigner specifies the target's handling of the + /// argument types for the caller. bool resultsCompatible(CallLoweringInfo &Info, MachineFunction &MF, SmallVectorImpl<ArgInfo> &InArgs, - CCAssignFn &CalleeAssignFnFixed, - CCAssignFn &CalleeAssignFnVarArg, - CCAssignFn &CallerAssignFnFixed, - CCAssignFn &CallerAssignFnVarArg) const; + ValueAssigner &CalleeAssigner, + ValueAssigner &CallerAssigner) const; public: CallLowering(const TargetLowering *TLI) : TLI(TLI) {} @@ -399,7 +511,9 @@ public: return false; } - virtual bool fallBackToDAGISel(const Function &F) const { return false; } + virtual bool fallBackToDAGISel(const MachineFunction &MF) const { + return false; + } /// This hook must be implemented to lower the incoming (formal) /// arguments, described by \p VRegs, for GlobalISel. Each argument @@ -456,6 +570,14 @@ public: ArrayRef<Register> ResRegs, ArrayRef<ArrayRef<Register>> ArgRegs, Register SwiftErrorVReg, std::function<unsigned()> GetCalleeReg) const; + + /// For targets which want to use big-endian can enable it with + /// enableBigEndian() hook + virtual bool enableBigEndian() const { return false; } + + /// For targets which support the "returned" parameter attribute, returns + /// true if the given type is a valid one to use with "returned". + virtual bool isTypeIsValidForThisReturn(EVT Ty) const { return false; } }; } // end namespace llvm diff --git a/llvm/include/llvm/CodeGen/GlobalISel/Combiner.h b/llvm/include/llvm/CodeGen/GlobalISel/Combiner.h index efe8bdf93664..795686980842 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/Combiner.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/Combiner.h @@ -5,10 +5,10 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// -// +/// \file /// This contains common code to drive combines. Combiner Passes will need to /// setup a CombinerInfo and call combineMachineFunction. -// +/// //===----------------------------------------------------------------------===// #ifndef LLVM_CODEGEN_GLOBALISEL_COMBINER_H @@ -43,4 +43,4 @@ protected: } // End namespace llvm. -#endif // LLVM_CODEGEN_GLOBALISEL_GICOMBINER_H +#endif // LLVM_CODEGEN_GLOBALISEL_COMBINER_H diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h index 8570f5ca5dd5..56459b68dce0 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h @@ -5,20 +5,21 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===--------------------------------------------------------------------===// -// +/// \file /// This contains common combine transformations that may be used in a combine /// pass,or by the target elsewhere. /// Targets can pick individual opcode transformations from the helper or use /// tryCombine which invokes all transformations. All of the transformations /// return true if the MachineInstruction changed and false otherwise. -// +/// //===--------------------------------------------------------------------===// -#ifndef LLVM_CODEGEN_GLOBALISEL_COMBINER_HELPER_H -#define LLVM_CODEGEN_GLOBALISEL_COMBINER_HELPER_H +#ifndef LLVM_CODEGEN_GLOBALISEL_COMBINERHELPER_H +#define LLVM_CODEGEN_GLOBALISEL_COMBINERHELPER_H #include "llvm/ADT/APFloat.h" #include "llvm/ADT/DenseMap.h" +#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" #include "llvm/CodeGen/LowLevelType.h" #include "llvm/CodeGen/Register.h" #include "llvm/Support/Alignment.h" @@ -150,16 +151,21 @@ public: void applyCombineIndexedLoadStore(MachineInstr &MI, IndexedLoadStoreMatchInfo &MatchInfo); bool matchSextTruncSextLoad(MachineInstr &MI); - bool applySextTruncSextLoad(MachineInstr &MI); + void applySextTruncSextLoad(MachineInstr &MI); /// Match sext_inreg(load p), imm -> sextload p bool matchSextInRegOfLoad(MachineInstr &MI, std::tuple<Register, unsigned> &MatchInfo); - bool applySextInRegOfLoad(MachineInstr &MI, std::tuple<Register, unsigned> &MatchInfo); + void applySextInRegOfLoad(MachineInstr &MI, std::tuple<Register, unsigned> &MatchInfo); + + /// Try to combine G_[SU]DIV and G_[SU]REM into a single G_[SU]DIVREM + /// when their source operands are identical. + bool matchCombineDivRem(MachineInstr &MI, MachineInstr *&OtherMI); + void applyCombineDivRem(MachineInstr &MI, MachineInstr *&OtherMI); /// If a brcond's true block is not the fallthrough, make it so by inverting /// the condition and swapping operands. - bool matchOptBrCondByInvertingCond(MachineInstr &MI); - void applyOptBrCondByInvertingCond(MachineInstr &MI); + bool matchOptBrCondByInvertingCond(MachineInstr &MI, MachineInstr *&BrCond); + void applyOptBrCondByInvertingCond(MachineInstr &MI, MachineInstr *&BrCond); /// If \p MI is G_CONCAT_VECTORS, try to combine it. /// Returns true if MI changed. @@ -234,92 +240,98 @@ public: bool tryCombineMemCpyFamily(MachineInstr &MI, unsigned MaxLen = 0); bool matchPtrAddImmedChain(MachineInstr &MI, PtrAddChain &MatchInfo); - bool applyPtrAddImmedChain(MachineInstr &MI, PtrAddChain &MatchInfo); + void applyPtrAddImmedChain(MachineInstr &MI, PtrAddChain &MatchInfo); /// Fold (shift (shift base, x), y) -> (shift base (x+y)) bool matchShiftImmedChain(MachineInstr &MI, RegisterImmPair &MatchInfo); - bool applyShiftImmedChain(MachineInstr &MI, RegisterImmPair &MatchInfo); + void applyShiftImmedChain(MachineInstr &MI, RegisterImmPair &MatchInfo); /// If we have a shift-by-constant of a bitwise logic op that itself has a /// shift-by-constant operand with identical opcode, we may be able to convert /// that into 2 independent shifts followed by the logic op. bool matchShiftOfShiftedLogic(MachineInstr &MI, ShiftOfShiftedLogic &MatchInfo); - bool applyShiftOfShiftedLogic(MachineInstr &MI, + void applyShiftOfShiftedLogic(MachineInstr &MI, ShiftOfShiftedLogic &MatchInfo); /// Transform a multiply by a power-of-2 value to a left shift. bool matchCombineMulToShl(MachineInstr &MI, unsigned &ShiftVal); - bool applyCombineMulToShl(MachineInstr &MI, unsigned &ShiftVal); + void applyCombineMulToShl(MachineInstr &MI, unsigned &ShiftVal); // Transform a G_SHL with an extended source into a narrower shift if // possible. bool matchCombineShlOfExtend(MachineInstr &MI, RegisterImmPair &MatchData); - bool applyCombineShlOfExtend(MachineInstr &MI, + void applyCombineShlOfExtend(MachineInstr &MI, const RegisterImmPair &MatchData); + /// Fold away a merge of an unmerge of the corresponding values. + bool matchCombineMergeUnmerge(MachineInstr &MI, Register &MatchInfo); + /// Reduce a shift by a constant to an unmerge and a shift on a half sized /// type. This will not produce a shift smaller than \p TargetShiftSize. bool matchCombineShiftToUnmerge(MachineInstr &MI, unsigned TargetShiftSize, unsigned &ShiftVal); - bool applyCombineShiftToUnmerge(MachineInstr &MI, const unsigned &ShiftVal); + void applyCombineShiftToUnmerge(MachineInstr &MI, const unsigned &ShiftVal); bool tryCombineShiftToUnmerge(MachineInstr &MI, unsigned TargetShiftAmount); /// Transform <ty,...> G_UNMERGE(G_MERGE ty X, Y, Z) -> ty X, Y, Z. bool matchCombineUnmergeMergeToPlainValues(MachineInstr &MI, SmallVectorImpl<Register> &Operands); - bool + void applyCombineUnmergeMergeToPlainValues(MachineInstr &MI, SmallVectorImpl<Register> &Operands); /// Transform G_UNMERGE Constant -> Constant1, Constant2, ... bool matchCombineUnmergeConstant(MachineInstr &MI, SmallVectorImpl<APInt> &Csts); - bool applyCombineUnmergeConstant(MachineInstr &MI, + void applyCombineUnmergeConstant(MachineInstr &MI, SmallVectorImpl<APInt> &Csts); /// Transform X, Y<dead> = G_UNMERGE Z -> X = G_TRUNC Z. bool matchCombineUnmergeWithDeadLanesToTrunc(MachineInstr &MI); - bool applyCombineUnmergeWithDeadLanesToTrunc(MachineInstr &MI); + void applyCombineUnmergeWithDeadLanesToTrunc(MachineInstr &MI); /// Transform X, Y = G_UNMERGE(G_ZEXT(Z)) -> X = G_ZEXT(Z); Y = G_CONSTANT 0 bool matchCombineUnmergeZExtToZExt(MachineInstr &MI); - bool applyCombineUnmergeZExtToZExt(MachineInstr &MI); + void applyCombineUnmergeZExtToZExt(MachineInstr &MI); /// Transform fp_instr(cst) to constant result of the fp operation. bool matchCombineConstantFoldFpUnary(MachineInstr &MI, Optional<APFloat> &Cst); - bool applyCombineConstantFoldFpUnary(MachineInstr &MI, + void applyCombineConstantFoldFpUnary(MachineInstr &MI, Optional<APFloat> &Cst); /// Transform IntToPtr(PtrToInt(x)) to x if cast is in the same address space. bool matchCombineI2PToP2I(MachineInstr &MI, Register &Reg); - bool applyCombineI2PToP2I(MachineInstr &MI, Register &Reg); + void applyCombineI2PToP2I(MachineInstr &MI, Register &Reg); /// Transform PtrToInt(IntToPtr(x)) to x. bool matchCombineP2IToI2P(MachineInstr &MI, Register &Reg); - bool applyCombineP2IToI2P(MachineInstr &MI, Register &Reg); + void applyCombineP2IToI2P(MachineInstr &MI, Register &Reg); /// Transform G_ADD (G_PTRTOINT x), y -> G_PTRTOINT (G_PTR_ADD x, y) /// Transform G_ADD y, (G_PTRTOINT x) -> G_PTRTOINT (G_PTR_ADD x, y) bool matchCombineAddP2IToPtrAdd(MachineInstr &MI, std::pair<Register, bool> &PtrRegAndCommute); - bool applyCombineAddP2IToPtrAdd(MachineInstr &MI, + void applyCombineAddP2IToPtrAdd(MachineInstr &MI, std::pair<Register, bool> &PtrRegAndCommute); // Transform G_PTR_ADD (G_PTRTOINT C1), C2 -> C1 + C2 bool matchCombineConstPtrAddToI2P(MachineInstr &MI, int64_t &NewCst); - bool applyCombineConstPtrAddToI2P(MachineInstr &MI, int64_t &NewCst); + void applyCombineConstPtrAddToI2P(MachineInstr &MI, int64_t &NewCst); /// Transform anyext(trunc(x)) to x. bool matchCombineAnyExtTrunc(MachineInstr &MI, Register &Reg); - bool applyCombineAnyExtTrunc(MachineInstr &MI, Register &Reg); + void applyCombineAnyExtTrunc(MachineInstr &MI, Register &Reg); + + /// Transform zext(trunc(x)) to x. + bool matchCombineZextTrunc(MachineInstr &MI, Register &Reg); /// Transform [asz]ext([asz]ext(x)) to [asz]ext x. bool matchCombineExtOfExt(MachineInstr &MI, std::tuple<Register, unsigned> &MatchInfo); - bool applyCombineExtOfExt(MachineInstr &MI, + void applyCombineExtOfExt(MachineInstr &MI, std::tuple<Register, unsigned> &MatchInfo); /// Transform fneg(fneg(x)) to x. @@ -327,23 +339,23 @@ public: /// Match fabs(fabs(x)) to fabs(x). bool matchCombineFAbsOfFAbs(MachineInstr &MI, Register &Src); - bool applyCombineFAbsOfFAbs(MachineInstr &MI, Register &Src); + void applyCombineFAbsOfFAbs(MachineInstr &MI, Register &Src); /// Transform trunc ([asz]ext x) to x or ([asz]ext x) or (trunc x). bool matchCombineTruncOfExt(MachineInstr &MI, std::pair<Register, unsigned> &MatchInfo); - bool applyCombineTruncOfExt(MachineInstr &MI, + void applyCombineTruncOfExt(MachineInstr &MI, std::pair<Register, unsigned> &MatchInfo); /// Transform trunc (shl x, K) to shl (trunc x), /// K => K < VT.getScalarSizeInBits(). bool matchCombineTruncOfShl(MachineInstr &MI, std::pair<Register, Register> &MatchInfo); - bool applyCombineTruncOfShl(MachineInstr &MI, + void applyCombineTruncOfShl(MachineInstr &MI, std::pair<Register, Register> &MatchInfo); /// Transform G_MUL(x, -1) to G_SUB(0, x) - bool applyCombineMulByNegativeOne(MachineInstr &MI); + void applyCombineMulByNegativeOne(MachineInstr &MI); /// Return true if any explicit use operand on \p MI is defined by a /// G_IMPLICIT_DEF. @@ -372,6 +384,9 @@ public: /// Replace an instruction with a G_CONSTANT with value \p C. bool replaceInstWithConstant(MachineInstr &MI, int64_t C); + /// Replace an instruction with a G_CONSTANT with value \p C. + bool replaceInstWithConstant(MachineInstr &MI, APInt C); + /// Replace an instruction with a G_IMPLICIT_DEF. bool replaceInstWithUndef(MachineInstr &MI); @@ -410,7 +425,7 @@ public: /// Return true if MI is a G_ADD which can be simplified to a G_SUB. bool matchSimplifyAddToSub(MachineInstr &MI, std::tuple<Register, Register> &MatchInfo); - bool applySimplifyAddToSub(MachineInstr &MI, + void applySimplifyAddToSub(MachineInstr &MI, std::tuple<Register, Register> &MatchInfo); /// Match (logic_op (op x...), (op y...)) -> (op (logic_op x, y)) @@ -419,14 +434,19 @@ public: InstructionStepsMatchInfo &MatchInfo); /// Replace \p MI with a series of instructions described in \p MatchInfo. - bool applyBuildInstructionSteps(MachineInstr &MI, + void applyBuildInstructionSteps(MachineInstr &MI, InstructionStepsMatchInfo &MatchInfo); /// Match ashr (shl x, C), C -> sext_inreg (C) bool matchAshrShlToSextInreg(MachineInstr &MI, std::tuple<Register, int64_t> &MatchInfo); - bool applyAshShlToSextInreg(MachineInstr &MI, + void applyAshShlToSextInreg(MachineInstr &MI, std::tuple<Register, int64_t> &MatchInfo); + + /// Fold and(and(x, C1), C2) -> C1&C2 ? and(x, C1&C2) : 0 + bool matchOverlappingAnd(MachineInstr &MI, + std::function<void(MachineIRBuilder &)> &MatchInfo); + /// \return true if \p MI is a G_AND instruction whose operands are x and y /// where x & y == x or x & y == y. (E.g., one of operands is all-ones value.) /// @@ -449,27 +469,27 @@ public: /// Combine inverting a result of a compare into the opposite cond code. bool matchNotCmp(MachineInstr &MI, SmallVectorImpl<Register> &RegsToNegate); - bool applyNotCmp(MachineInstr &MI, SmallVectorImpl<Register> &RegsToNegate); + void applyNotCmp(MachineInstr &MI, SmallVectorImpl<Register> &RegsToNegate); /// Fold (xor (and x, y), y) -> (and (not x), y) ///{ bool matchXorOfAndWithSameReg(MachineInstr &MI, std::pair<Register, Register> &MatchInfo); - bool applyXorOfAndWithSameReg(MachineInstr &MI, + void applyXorOfAndWithSameReg(MachineInstr &MI, std::pair<Register, Register> &MatchInfo); ///} /// Combine G_PTR_ADD with nullptr to G_INTTOPTR bool matchPtrAddZero(MachineInstr &MI); - bool applyPtrAddZero(MachineInstr &MI); + void applyPtrAddZero(MachineInstr &MI); /// Combine G_UREM x, (known power of 2) to an add and bitmasking. - bool applySimplifyURemByPow2(MachineInstr &MI); + void applySimplifyURemByPow2(MachineInstr &MI); bool matchCombineInsertVecElts(MachineInstr &MI, SmallVectorImpl<Register> &MatchInfo); - bool applyCombineInsertVecElts(MachineInstr &MI, + void applyCombineInsertVecElts(MachineInstr &MI, SmallVectorImpl<Register> &MatchInfo); /// Match expression trees of the form @@ -483,23 +503,76 @@ public: /// bswap. bool matchLoadOrCombine(MachineInstr &MI, std::function<void(MachineIRBuilder &)> &MatchInfo); - bool applyLoadOrCombine(MachineInstr &MI, + + bool matchExtendThroughPhis(MachineInstr &MI, MachineInstr *&ExtMI); + void applyExtendThroughPhis(MachineInstr &MI, MachineInstr *&ExtMI); + + bool matchExtractVecEltBuildVec(MachineInstr &MI, Register &Reg); + void applyExtractVecEltBuildVec(MachineInstr &MI, Register &Reg); + + bool matchExtractAllEltsFromBuildVector( + MachineInstr &MI, + SmallVectorImpl<std::pair<Register, MachineInstr *>> &MatchInfo); + void applyExtractAllEltsFromBuildVector( + MachineInstr &MI, + SmallVectorImpl<std::pair<Register, MachineInstr *>> &MatchInfo); + + /// Use a function which takes in a MachineIRBuilder to perform a combine. + /// By default, it erases the instruction \p MI from the function. + void applyBuildFn(MachineInstr &MI, + std::function<void(MachineIRBuilder &)> &MatchInfo); + /// Use a function which takes in a MachineIRBuilder to perform a combine. + /// This variant does not erase \p MI after calling the build function. + void applyBuildFnNoErase(MachineInstr &MI, + std::function<void(MachineIRBuilder &)> &MatchInfo); + + bool matchFunnelShiftToRotate(MachineInstr &MI); + void applyFunnelShiftToRotate(MachineInstr &MI); + bool matchRotateOutOfRange(MachineInstr &MI); + void applyRotateOutOfRange(MachineInstr &MI); + + /// \returns true if a G_ICMP instruction \p MI can be replaced with a true + /// or false constant based off of KnownBits information. + bool matchICmpToTrueFalseKnownBits(MachineInstr &MI, int64_t &MatchInfo); + + bool matchBitfieldExtractFromSExtInReg( + MachineInstr &MI, std::function<void(MachineIRBuilder &)> &MatchInfo); + /// Match: and (lshr x, cst), mask -> ubfx x, cst, width + bool matchBitfieldExtractFromAnd( + MachineInstr &MI, std::function<void(MachineIRBuilder &)> &MatchInfo); + + /// Reassociate pointer calculations with G_ADD involved, to allow better + /// addressing mode usage. + bool matchReassocPtrAdd(MachineInstr &MI, std::function<void(MachineIRBuilder &)> &MatchInfo); + + /// Do constant folding when opportunities are exposed after MIR building. + bool matchConstantFold(MachineInstr &MI, APInt &MatchInfo); + /// Try to transform \p MI by using all of the above /// combine functions. Returns true if changed. bool tryCombine(MachineInstr &MI); + /// Emit loads and stores that perform the given memcpy. + /// Assumes \p MI is a G_MEMCPY_INLINE + /// TODO: implement dynamically sized inline memcpy, + /// and rename: s/bool tryEmit/void emit/ + bool tryEmitMemcpyInline(MachineInstr &MI); + private: // Memcpy family optimization helpers. + bool tryEmitMemcpyInline(MachineInstr &MI, Register Dst, Register Src, + uint64_t KnownLen, Align DstAlign, Align SrcAlign, + bool IsVolatile); bool optimizeMemcpy(MachineInstr &MI, Register Dst, Register Src, - unsigned KnownLen, Align DstAlign, Align SrcAlign, - bool IsVolatile); + uint64_t KnownLen, uint64_t Limit, Align DstAlign, + Align SrcAlign, bool IsVolatile); bool optimizeMemmove(MachineInstr &MI, Register Dst, Register Src, - unsigned KnownLen, Align DstAlign, Align SrcAlign, + uint64_t KnownLen, Align DstAlign, Align SrcAlign, bool IsVolatile); bool optimizeMemset(MachineInstr &MI, Register Dst, Register Val, - unsigned KnownLen, Align DstAlign, bool IsVolatile); + uint64_t KnownLen, Align DstAlign, bool IsVolatile); /// Given a non-indexed load or store instruction \p MI, find an offset that /// can be usefully and legally folded into it as a post-indexing operation. @@ -533,11 +606,18 @@ private: /// at to the index of the load. /// \param [in] MemSizeInBits - The number of bits each load should produce. /// - /// \returns The lowest-index load found and the lowest index on success. - Optional<std::pair<MachineInstr *, int64_t>> findLoadOffsetsForLoadOrCombine( + /// \returns On success, a 3-tuple containing lowest-index load found, the + /// lowest index, and the last load in the sequence. + Optional<std::tuple<GZExtLoad *, int64_t, GZExtLoad *>> + findLoadOffsetsForLoadOrCombine( SmallDenseMap<int64_t, int64_t, 8> &MemOffset2Idx, const SmallVector<Register, 8> &RegsToVisit, const unsigned MemSizeInBits); + + /// Examines the G_PTR_ADD instruction \p PtrAdd and determines if performing + /// a re-association of its operands would break an existing legal addressing + /// mode that the address computation currently represents. + bool reassociationCanBreakAddressingModePattern(MachineInstr &PtrAdd); }; } // namespace llvm diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerInfo.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerInfo.h index e95a5e21f832..4a1a4ff2528a 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerInfo.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerInfo.h @@ -5,13 +5,13 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// -// +/// \file /// Interface for Targets to specify which operations are combined how and when. /// //===----------------------------------------------------------------------===// -#ifndef LLVM_CODEGEN_GLOBALISEL_COMBINER_INFO_H -#define LLVM_CODEGEN_GLOBALISEL_COMBINER_INFO_H +#ifndef LLVM_CODEGEN_GLOBALISEL_COMBINERINFO_H +#define LLVM_CODEGEN_GLOBALISEL_COMBINERINFO_H #include <cassert> namespace llvm { diff --git a/llvm/include/llvm/CodeGen/GlobalISel/ConstantFoldingMIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/ConstantFoldingMIRBuilder.h deleted file mode 100644 index df196bfbd437..000000000000 --- a/llvm/include/llvm/CodeGen/GlobalISel/ConstantFoldingMIRBuilder.h +++ /dev/null @@ -1,72 +0,0 @@ -//===-- llvm/CodeGen/GlobalISel/ConstantFoldingMIRBuilder.h --*- C++ -*-==// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -/// \file -/// This file implements a version of MachineIRBuilder which does trivial -/// constant folding. -//===----------------------------------------------------------------------===// -#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" -#include "llvm/CodeGen/GlobalISel/Utils.h" - -namespace llvm { - -/// An MIRBuilder which does trivial constant folding of binary ops. -/// Calls to buildInstr will also try to constant fold binary ops. -class ConstantFoldingMIRBuilder : public MachineIRBuilder { -public: - // Pull in base class constructors. - using MachineIRBuilder::MachineIRBuilder; - - virtual ~ConstantFoldingMIRBuilder() = default; - - // Try to provide an overload for buildInstr for binary ops in order to - // constant fold. - MachineInstrBuilder buildInstr(unsigned Opc, ArrayRef<DstOp> DstOps, - ArrayRef<SrcOp> SrcOps, - Optional<unsigned> Flags = None) override { - switch (Opc) { - default: - break; - case TargetOpcode::G_ADD: - case TargetOpcode::G_AND: - case TargetOpcode::G_ASHR: - case TargetOpcode::G_LSHR: - case TargetOpcode::G_MUL: - case TargetOpcode::G_OR: - case TargetOpcode::G_SHL: - case TargetOpcode::G_SUB: - case TargetOpcode::G_XOR: - case TargetOpcode::G_UDIV: - case TargetOpcode::G_SDIV: - case TargetOpcode::G_UREM: - case TargetOpcode::G_SREM: { - assert(DstOps.size() == 1 && "Invalid dst ops"); - assert(SrcOps.size() == 2 && "Invalid src ops"); - const DstOp &Dst = DstOps[0]; - const SrcOp &Src0 = SrcOps[0]; - const SrcOp &Src1 = SrcOps[1]; - if (auto MaybeCst = - ConstantFoldBinOp(Opc, Src0.getReg(), Src1.getReg(), *getMRI())) - return buildConstant(Dst, MaybeCst->getSExtValue()); - break; - } - case TargetOpcode::G_SEXT_INREG: { - assert(DstOps.size() == 1 && "Invalid dst ops"); - assert(SrcOps.size() == 2 && "Invalid src ops"); - const DstOp &Dst = DstOps[0]; - const SrcOp &Src0 = SrcOps[0]; - const SrcOp &Src1 = SrcOps[1]; - if (auto MaybeCst = - ConstantFoldExtOp(Opc, Src0.getReg(), Src1.getImm(), *getMRI())) - return buildConstant(Dst, MaybeCst->getSExtValue()); - break; - } - } - return MachineIRBuilder::buildInstr(Opc, DstOps, SrcOps); - } -}; -} // namespace llvm diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GISelChangeObserver.h b/llvm/include/llvm/CodeGen/GlobalISel/GISelChangeObserver.h index dd7f04a33f4b..79d71b2c8982 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/GISelChangeObserver.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/GISelChangeObserver.h @@ -5,11 +5,12 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// -// +/// \file /// This contains common code to allow clients to notify changes to machine /// instr. -// +/// //===----------------------------------------------------------------------===// + #ifndef LLVM_CODEGEN_GLOBALISEL_GISELCHANGEOBSERVER_H #define LLVM_CODEGEN_GLOBALISEL_GISELCHANGEOBSERVER_H diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GISelKnownBits.h b/llvm/include/llvm/CodeGen/GlobalISel/GISelKnownBits.h index eafed3760738..035c5a08feef 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/GISelKnownBits.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/GISelKnownBits.h @@ -5,13 +5,14 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// -// +/// \file /// Provides analysis for querying information about KnownBits during GISel /// passes. -// +/// //===----------------------------------------------------------------------===// -#ifndef LLVM_CODEGEN_GLOBALISEL_KNOWNBITSINFO_H -#define LLVM_CODEGEN_GLOBALISEL_KNOWNBITSINFO_H + +#ifndef LLVM_CODEGEN_GLOBALISEL_GISELKNOWNBITS_H +#define LLVM_CODEGEN_GLOBALISEL_GISELKNOWNBITS_H #include "llvm/ADT/DenseMap.h" #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h" @@ -128,4 +129,4 @@ public: }; } // namespace llvm -#endif // ifdef +#endif // LLVM_CODEGEN_GLOBALISEL_GISELKNOWNBITS_H diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GISelWorkList.h b/llvm/include/llvm/CodeGen/GlobalISel/GISelWorkList.h index 9e7ade3ee329..c5af64d2bcbe 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/GISelWorkList.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/GISelWorkList.h @@ -6,8 +6,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_GISEL_WORKLIST_H -#define LLVM_GISEL_WORKLIST_H +#ifndef LLVM_CODEGEN_GLOBALISEL_GISELWORKLIST_H +#define LLVM_CODEGEN_GLOBALISEL_GISELWORKLIST_H #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallVector.h" diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h new file mode 100644 index 000000000000..1162134b2ad2 --- /dev/null +++ b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h @@ -0,0 +1,200 @@ +//===- llvm/CodeGen/GlobalISel/GenericMachineInstrs.h -----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// \file +/// Declares convenience wrapper classes for interpreting MachineInstr instances +/// as specific generic operations. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_GLOBALISEL_GENERICMACHINEINSTRS_H +#define LLVM_CODEGEN_GLOBALISEL_GENERICMACHINEINSTRS_H + +#include "llvm/CodeGen/MachineInstr.h" +#include "llvm/CodeGen/MachineMemOperand.h" +#include "llvm/CodeGen/TargetOpcodes.h" +#include "llvm/Support/Casting.h" + +namespace llvm { + +/// A base class for all GenericMachineInstrs. +class GenericMachineInstr : public MachineInstr { +public: + GenericMachineInstr() = delete; + + /// Access the Idx'th operand as a register and return it. + /// This assumes that the Idx'th operand is a Register type. + Register getReg(unsigned Idx) const { return getOperand(Idx).getReg(); } + + static bool classof(const MachineInstr *MI) { + return isPreISelGenericOpcode(MI->getOpcode()); + } +}; + +/// Represents any type of generic load or store. +/// G_LOAD, G_STORE, G_ZEXTLOAD, G_SEXTLOAD. +class GLoadStore : public GenericMachineInstr { +public: + /// Get the source register of the pointer value. + Register getPointerReg() const { return getOperand(1).getReg(); } + + /// Get the MachineMemOperand on this instruction. + MachineMemOperand &getMMO() const { return **memoperands_begin(); } + + /// Returns true if the attached MachineMemOperand has the atomic flag set. + bool isAtomic() const { return getMMO().isAtomic(); } + /// Returns true if the attached MachineMemOpeand as the volatile flag set. + bool isVolatile() const { return getMMO().isVolatile(); } + /// Returns true if the memory operation is neither atomic or volatile. + bool isSimple() const { return !isAtomic() && !isVolatile(); } + /// Returns true if this memory operation doesn't have any ordering + /// constraints other than normal aliasing. Volatile and (ordered) atomic + /// memory operations can't be reordered. + bool isUnordered() const { return getMMO().isUnordered(); } + + /// Returns the size in bytes of the memory access. + uint64_t getMemSize() { return getMMO().getSize(); + } /// Returns the size in bits of the memory access. + uint64_t getMemSizeInBits() { return getMMO().getSizeInBits(); } + + static bool classof(const MachineInstr *MI) { + switch (MI->getOpcode()) { + case TargetOpcode::G_LOAD: + case TargetOpcode::G_STORE: + case TargetOpcode::G_ZEXTLOAD: + case TargetOpcode::G_SEXTLOAD: + return true; + default: + return false; + } + } +}; + +/// Represents any generic load, including sign/zero extending variants. +class GAnyLoad : public GLoadStore { +public: + /// Get the definition register of the loaded value. + Register getDstReg() const { return getOperand(0).getReg(); } + + static bool classof(const MachineInstr *MI) { + switch (MI->getOpcode()) { + case TargetOpcode::G_LOAD: + case TargetOpcode::G_ZEXTLOAD: + case TargetOpcode::G_SEXTLOAD: + return true; + default: + return false; + } + } +}; + +/// Represents a G_LOAD. +class GLoad : public GAnyLoad { +public: + static bool classof(const MachineInstr *MI) { + return MI->getOpcode() == TargetOpcode::G_LOAD; + } +}; + +/// Represents either a G_SEXTLOAD or G_ZEXTLOAD. +class GExtLoad : public GAnyLoad { +public: + static bool classof(const MachineInstr *MI) { + return MI->getOpcode() == TargetOpcode::G_SEXTLOAD || + MI->getOpcode() == TargetOpcode::G_ZEXTLOAD; + } +}; + +/// Represents a G_SEXTLOAD. +class GSExtLoad : public GExtLoad { +public: + static bool classof(const MachineInstr *MI) { + return MI->getOpcode() == TargetOpcode::G_SEXTLOAD; + } +}; + +/// Represents a G_ZEXTLOAD. +class GZExtLoad : public GExtLoad { +public: + static bool classof(const MachineInstr *MI) { + return MI->getOpcode() == TargetOpcode::G_ZEXTLOAD; + } +}; + +/// Represents a G_STORE. +class GStore : public GLoadStore { +public: + /// Get the stored value register. + Register getValueReg() const { return getOperand(0).getReg(); } + + static bool classof(const MachineInstr *MI) { + return MI->getOpcode() == TargetOpcode::G_STORE; + } +}; + +/// Represents a G_UNMERGE_VALUES. +class GUnmerge : public GenericMachineInstr { +public: + /// Returns the number of def registers. + unsigned getNumDefs() const { return getNumOperands() - 1; } + /// Get the unmerge source register. + Register getSourceReg() const { return getOperand(getNumDefs()).getReg(); } + + static bool classof(const MachineInstr *MI) { + return MI->getOpcode() == TargetOpcode::G_UNMERGE_VALUES; + } +}; + +/// Represents G_BUILD_VECTOR, G_CONCAT_VECTORS or G_MERGE_VALUES. +/// All these have the common property of generating a single value from +/// multiple sources. +class GMergeLikeOp : public GenericMachineInstr { +public: + /// Returns the number of source registers. + unsigned getNumSources() const { return getNumOperands() - 1; } + /// Returns the I'th source register. + Register getSourceReg(unsigned I) const { return getReg(I + 1); } + + static bool classof(const MachineInstr *MI) { + switch (MI->getOpcode()) { + case TargetOpcode::G_MERGE_VALUES: + case TargetOpcode::G_CONCAT_VECTORS: + case TargetOpcode::G_BUILD_VECTOR: + return true; + default: + return false; + } + } +}; + +/// Represents a G_MERGE_VALUES. +class GMerge : public GMergeLikeOp { +public: + static bool classof(const MachineInstr *MI) { + return MI->getOpcode() == TargetOpcode::G_MERGE_VALUES; + } +}; + +/// Represents a G_CONCAT_VECTORS. +class GConcatVectors : public GMergeLikeOp { +public: + static bool classof(const MachineInstr *MI) { + return MI->getOpcode() == TargetOpcode::G_CONCAT_VECTORS; + } +}; + +/// Represents a G_BUILD_VECTOR. +class GBuildVector : public GMergeLikeOp { +public: + static bool classof(const MachineInstr *MI) { + return MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR; + } +}; + +} // namespace llvm + +#endif // LLVM_CODEGEN_GLOBALISEL_GENERICMACHINEINSTRS_H
\ No newline at end of file diff --git a/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelect.h b/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelect.h index 1af46e0a9e76..4a72621ec61e 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelect.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelect.h @@ -17,6 +17,10 @@ #include "llvm/CodeGen/MachineFunctionPass.h" namespace llvm { + +class BlockFrequencyInfo; +class ProfileSummaryInfo; + /// This pass is responsible for selecting generic machine instructions to /// target-specific instructions. It relies on the InstructionSelector provided /// by the target. @@ -43,9 +47,16 @@ public: MachineFunctionProperties::Property::Selected); } + InstructionSelect(CodeGenOpt::Level OL); InstructionSelect(); bool runOnMachineFunction(MachineFunction &MF) override; + +protected: + BlockFrequencyInfo *BFI = nullptr; + ProfileSummaryInfo *PSI = nullptr; + + CodeGenOpt::Level OptLevel = CodeGenOpt::None; }; } // End namespace llvm. diff --git a/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h b/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h index 5b8243a93e7f..03f4f3bf0b19 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h @@ -18,6 +18,11 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/Analysis/BlockFrequencyInfo.h" +#include "llvm/Analysis/ProfileSummaryInfo.h" +#include "llvm/CodeGen/GlobalISel/Utils.h" +#include "llvm/CodeGen/MachineBasicBlock.h" +#include "llvm/CodeGen/MachineFunction.h" #include "llvm/Support/CodeGenCoverage.h" #include "llvm/Support/LowLevelTypeImpl.h" #include <bitset> @@ -39,7 +44,6 @@ class MachineOperand; class MachineRegisterInfo; class RegisterBankInfo; class TargetInstrInfo; -class TargetRegisterClass; class TargetRegisterInfo; /// Container class for CodeGen predicate results. @@ -136,6 +140,11 @@ enum { /// - InsnID - Instruction ID /// - The predicate to test GIM_CheckAPFloatImmPredicate, + /// Check an immediate predicate on the specified instruction + /// - InsnID - Instruction ID + /// - OpIdx - Operand index + /// - The predicate to test + GIM_CheckImmOperandPredicate, /// Check a memory operation has the specified atomic ordering. /// - InsnID - Instruction ID /// - Ordering - The AtomicOrdering value @@ -430,18 +439,25 @@ public: CodeGenCoverage *CoverageInfo = nullptr; GISelKnownBits *KnownBits = nullptr; MachineFunction *MF = nullptr; + ProfileSummaryInfo *PSI = nullptr; + BlockFrequencyInfo *BFI = nullptr; + // For some predicates, we need to track the current MBB. + MachineBasicBlock *CurMBB = nullptr; virtual void setupGeneratedPerFunctionState(MachineFunction &MF) { llvm_unreachable("TableGen should have emitted implementation"); } /// Setup per-MF selector state. - virtual void setupMF(MachineFunction &mf, - GISelKnownBits &KB, - CodeGenCoverage &covinfo) { + virtual void setupMF(MachineFunction &mf, GISelKnownBits *KB, + CodeGenCoverage &covinfo, ProfileSummaryInfo *psi, + BlockFrequencyInfo *bfi) { CoverageInfo = &covinfo; - KnownBits = &KB; + KnownBits = KB; MF = &mf; + PSI = psi; + BFI = bfi; + CurMBB = nullptr; setupGeneratedPerFunctionState(mf); } @@ -464,6 +480,12 @@ protected: MatcherState(unsigned MaxRenderers); }; + bool shouldOptForSize(const MachineFunction *MF) const { + const auto &F = MF->getFunction(); + return F.hasOptSize() || F.hasMinSize() || + (PSI && BFI && CurMBB && llvm::shouldOptForSize(*CurMBB, PSI, BFI)); + } + public: template <class PredicateBitset, class ComplexMatcherMemFn, class CustomRendererFn> diff --git a/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h b/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h index 82e26b0bc355..bc9f952146c2 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h @@ -263,22 +263,27 @@ bool InstructionSelector::executeMatchTable( } break; } - case GIM_CheckI64ImmPredicate: { + case GIM_CheckI64ImmPredicate: + case GIM_CheckImmOperandPredicate: { int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t OpIdx = MatcherOpcode == GIM_CheckImmOperandPredicate + ? MatchTable[CurrentIdx++] + : 1; int64_t Predicate = MatchTable[CurrentIdx++]; DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), - dbgs() - << CurrentIdx << ": GIM_CheckI64ImmPredicate(MIs[" - << InsnID << "], Predicate=" << Predicate << ")\n"); + dbgs() << CurrentIdx << ": GIM_CheckImmPredicate(MIs[" + << InsnID << "]->getOperand(" << OpIdx + << "), Predicate=" << Predicate << ")\n"); assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); - assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT && - "Expected G_CONSTANT"); + assert((State.MIs[InsnID]->getOperand(OpIdx).isImm() || + State.MIs[InsnID]->getOperand(OpIdx).isCImm()) && + "Expected immediate operand"); assert(Predicate > GIPFP_I64_Invalid && "Expected a valid predicate"); int64_t Value = 0; - if (State.MIs[InsnID]->getOperand(1).isCImm()) - Value = State.MIs[InsnID]->getOperand(1).getCImm()->getSExtValue(); - else if (State.MIs[InsnID]->getOperand(1).isImm()) - Value = State.MIs[InsnID]->getOperand(1).getImm(); + if (State.MIs[InsnID]->getOperand(OpIdx).isCImm()) + Value = State.MIs[InsnID]->getOperand(OpIdx).getCImm()->getSExtValue(); + else if (State.MIs[InsnID]->getOperand(OpIdx).isImm()) + Value = State.MIs[InsnID]->getOperand(OpIdx).getImm(); else llvm_unreachable("Expected Imm or CImm operand"); @@ -385,7 +390,7 @@ bool InstructionSelector::executeMatchTable( return false; for (const auto &MMO : State.MIs[InsnID]->memoperands()) - if (MMO->getOrdering() != Ordering) + if (MMO->getMergedOrdering() != Ordering) if (handleReject() == RejectAndGiveUp) return false; break; @@ -403,7 +408,7 @@ bool InstructionSelector::executeMatchTable( return false; for (const auto &MMO : State.MIs[InsnID]->memoperands()) - if (!isAtLeastOrStrongerThan(MMO->getOrdering(), Ordering)) + if (!isAtLeastOrStrongerThan(MMO->getMergedOrdering(), Ordering)) if (handleReject() == RejectAndGiveUp) return false; break; @@ -421,7 +426,7 @@ bool InstructionSelector::executeMatchTable( return false; for (const auto &MMO : State.MIs[InsnID]->memoperands()) - if (!isStrongerThan(Ordering, MMO->getOrdering())) + if (!isStrongerThan(Ordering, MMO->getMergedOrdering())) if (handleReject() == RejectAndGiveUp) return false; break; @@ -590,7 +595,7 @@ bool InstructionSelector::executeMatchTable( case GIM_CheckPointerToAny: { int64_t InsnID = MatchTable[CurrentIdx++]; int64_t OpIdx = MatchTable[CurrentIdx++]; - int64_t SizeInBits = MatchTable[CurrentIdx++]; + uint64_t SizeInBits = MatchTable[CurrentIdx++]; DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs[" diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegacyLegalizerInfo.h b/llvm/include/llvm/CodeGen/GlobalISel/LegacyLegalizerInfo.h new file mode 100644 index 000000000000..b1f2103da309 --- /dev/null +++ b/llvm/include/llvm/CodeGen/GlobalISel/LegacyLegalizerInfo.h @@ -0,0 +1,481 @@ +//===- llvm/CodeGen/GlobalISel/LegacyLegalizerInfo.h ------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// \file +/// Interface for Targets to specify which operations they can successfully +/// select and how the others should be expanded most efficiently. +/// This implementation has been deprecated for a long time but it still in use +/// in a few places. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_GLOBALISEL_LEGACYLEGALIZERINFO_H +#define LLVM_CODEGEN_GLOBALISEL_LEGACYLEGALIZERINFO_H + +#include "llvm/ADT/DenseMap.h" +#include "llvm/CodeGen/TargetOpcodes.h" +#include "llvm/Support/LowLevelTypeImpl.h" +#include <unordered_map> + +namespace llvm { +struct LegalityQuery; + +namespace LegacyLegalizeActions { +enum LegacyLegalizeAction : std::uint8_t { + /// The operation is expected to be selectable directly by the target, and + /// no transformation is necessary. + Legal, + + /// The operation should be synthesized from multiple instructions acting on + /// a narrower scalar base-type. For example a 64-bit add might be + /// implemented in terms of 32-bit add-with-carry. + NarrowScalar, + + /// The operation should be implemented in terms of a wider scalar + /// base-type. For example a <2 x s8> add could be implemented as a <2 + /// x s32> add (ignoring the high bits). + WidenScalar, + + /// The (vector) operation should be implemented by splitting it into + /// sub-vectors where the operation is legal. For example a <8 x s64> add + /// might be implemented as 4 separate <2 x s64> adds. + FewerElements, + + /// The (vector) operation should be implemented by widening the input + /// vector and ignoring the lanes added by doing so. For example <2 x i8> is + /// rarely legal, but you might perform an <8 x i8> and then only look at + /// the first two results. + MoreElements, + + /// Perform the operation on a different, but equivalently sized type. + Bitcast, + + /// The operation itself must be expressed in terms of simpler actions on + /// this target. E.g. a SREM replaced by an SDIV and subtraction. + Lower, + + /// The operation should be implemented as a call to some kind of runtime + /// support library. For example this usually happens on machines that don't + /// support floating-point operations natively. + Libcall, + + /// The target wants to do something special with this combination of + /// operand and type. A callback will be issued when it is needed. + Custom, + + /// This operation is completely unsupported on the target. A programming + /// error has occurred. + Unsupported, + + /// Sentinel value for when no action was found in the specified table. + NotFound, +}; +} // end namespace LegacyLegalizeActions +raw_ostream &operator<<(raw_ostream &OS, + LegacyLegalizeActions::LegacyLegalizeAction Action); + +/// Legalization is decided based on an instruction's opcode, which type slot +/// we're considering, and what the existing type is. These aspects are gathered +/// together for convenience in the InstrAspect class. +struct InstrAspect { + unsigned Opcode; + unsigned Idx = 0; + LLT Type; + + InstrAspect(unsigned Opcode, LLT Type) : Opcode(Opcode), Type(Type) {} + InstrAspect(unsigned Opcode, unsigned Idx, LLT Type) + : Opcode(Opcode), Idx(Idx), Type(Type) {} + + bool operator==(const InstrAspect &RHS) const { + return Opcode == RHS.Opcode && Idx == RHS.Idx && Type == RHS.Type; + } +}; + +/// The result of a query. It either indicates a final answer of Legal or +/// Unsupported or describes an action that must be taken to make an operation +/// more legal. +struct LegacyLegalizeActionStep { + /// The action to take or the final answer. + LegacyLegalizeActions::LegacyLegalizeAction Action; + /// If describing an action, the type index to change. Otherwise zero. + unsigned TypeIdx; + /// If describing an action, the new type for TypeIdx. Otherwise LLT{}. + LLT NewType; + + LegacyLegalizeActionStep(LegacyLegalizeActions::LegacyLegalizeAction Action, + unsigned TypeIdx, const LLT NewType) + : Action(Action), TypeIdx(TypeIdx), NewType(NewType) {} + + bool operator==(const LegacyLegalizeActionStep &RHS) const { + return std::tie(Action, TypeIdx, NewType) == + std::tie(RHS.Action, RHS.TypeIdx, RHS.NewType); + } +}; + + +class LegacyLegalizerInfo { +public: + using SizeAndAction = + std::pair<uint16_t, LegacyLegalizeActions::LegacyLegalizeAction>; + using SizeAndActionsVec = std::vector<SizeAndAction>; + using SizeChangeStrategy = + std::function<SizeAndActionsVec(const SizeAndActionsVec &v)>; + + LegacyLegalizerInfo(); + + static bool needsLegalizingToDifferentSize( + const LegacyLegalizeActions::LegacyLegalizeAction Action) { + using namespace LegacyLegalizeActions; + switch (Action) { + case NarrowScalar: + case WidenScalar: + case FewerElements: + case MoreElements: + case Unsupported: + return true; + default: + return false; + } + } + + /// Compute any ancillary tables needed to quickly decide how an operation + /// should be handled. This must be called after all "set*Action"methods but + /// before any query is made or incorrect results may be returned. + void computeTables(); + + /// More friendly way to set an action for common types that have an LLT + /// representation. + /// The LegacyLegalizeAction must be one for which + /// NeedsLegalizingToDifferentSize returns false. + void setAction(const InstrAspect &Aspect, + LegacyLegalizeActions::LegacyLegalizeAction Action) { + assert(!needsLegalizingToDifferentSize(Action)); + TablesInitialized = false; + const unsigned OpcodeIdx = Aspect.Opcode - FirstOp; + if (SpecifiedActions[OpcodeIdx].size() <= Aspect.Idx) + SpecifiedActions[OpcodeIdx].resize(Aspect.Idx + 1); + SpecifiedActions[OpcodeIdx][Aspect.Idx][Aspect.Type] = Action; + } + + /// The setAction calls record the non-size-changing legalization actions + /// to take on specificly-sized types. The SizeChangeStrategy defines what + /// to do when the size of the type needs to be changed to reach a legally + /// sized type (i.e., one that was defined through a setAction call). + /// e.g. + /// setAction ({G_ADD, 0, LLT::scalar(32)}, Legal); + /// setLegalizeScalarToDifferentSizeStrategy( + /// G_ADD, 0, widenToLargerTypesAndNarrowToLargest); + /// will end up defining getAction({G_ADD, 0, T}) to return the following + /// actions for different scalar types T: + /// LLT::scalar(1)..LLT::scalar(31): {WidenScalar, 0, LLT::scalar(32)} + /// LLT::scalar(32): {Legal, 0, LLT::scalar(32)} + /// LLT::scalar(33)..: {NarrowScalar, 0, LLT::scalar(32)} + /// + /// If no SizeChangeAction gets defined, through this function, + /// the default is unsupportedForDifferentSizes. + void setLegalizeScalarToDifferentSizeStrategy(const unsigned Opcode, + const unsigned TypeIdx, + SizeChangeStrategy S) { + const unsigned OpcodeIdx = Opcode - FirstOp; + if (ScalarSizeChangeStrategies[OpcodeIdx].size() <= TypeIdx) + ScalarSizeChangeStrategies[OpcodeIdx].resize(TypeIdx + 1); + ScalarSizeChangeStrategies[OpcodeIdx][TypeIdx] = S; + } + + /// See also setLegalizeScalarToDifferentSizeStrategy. + /// This function allows to set the SizeChangeStrategy for vector elements. + void setLegalizeVectorElementToDifferentSizeStrategy(const unsigned Opcode, + const unsigned TypeIdx, + SizeChangeStrategy S) { + const unsigned OpcodeIdx = Opcode - FirstOp; + if (VectorElementSizeChangeStrategies[OpcodeIdx].size() <= TypeIdx) + VectorElementSizeChangeStrategies[OpcodeIdx].resize(TypeIdx + 1); + VectorElementSizeChangeStrategies[OpcodeIdx][TypeIdx] = S; + } + + /// A SizeChangeStrategy for the common case where legalization for a + /// particular operation consists of only supporting a specific set of type + /// sizes. E.g. + /// setAction ({G_DIV, 0, LLT::scalar(32)}, Legal); + /// setAction ({G_DIV, 0, LLT::scalar(64)}, Legal); + /// setLegalizeScalarToDifferentSizeStrategy( + /// G_DIV, 0, unsupportedForDifferentSizes); + /// will result in getAction({G_DIV, 0, T}) to return Legal for s32 and s64, + /// and Unsupported for all other scalar types T. + static SizeAndActionsVec + unsupportedForDifferentSizes(const SizeAndActionsVec &v) { + using namespace LegacyLegalizeActions; + return increaseToLargerTypesAndDecreaseToLargest(v, Unsupported, + Unsupported); + } + + /// A SizeChangeStrategy for the common case where legalization for a + /// particular operation consists of widening the type to a large legal type, + /// unless there is no such type and then instead it should be narrowed to the + /// largest legal type. + static SizeAndActionsVec + widenToLargerTypesAndNarrowToLargest(const SizeAndActionsVec &v) { + using namespace LegacyLegalizeActions; + assert(v.size() > 0 && + "At least one size that can be legalized towards is needed" + " for this SizeChangeStrategy"); + return increaseToLargerTypesAndDecreaseToLargest(v, WidenScalar, + NarrowScalar); + } + + static SizeAndActionsVec + widenToLargerTypesUnsupportedOtherwise(const SizeAndActionsVec &v) { + using namespace LegacyLegalizeActions; + return increaseToLargerTypesAndDecreaseToLargest(v, WidenScalar, + Unsupported); + } + + static SizeAndActionsVec + narrowToSmallerAndUnsupportedIfTooSmall(const SizeAndActionsVec &v) { + using namespace LegacyLegalizeActions; + return decreaseToSmallerTypesAndIncreaseToSmallest(v, NarrowScalar, + Unsupported); + } + + static SizeAndActionsVec + narrowToSmallerAndWidenToSmallest(const SizeAndActionsVec &v) { + using namespace LegacyLegalizeActions; + assert(v.size() > 0 && + "At least one size that can be legalized towards is needed" + " for this SizeChangeStrategy"); + return decreaseToSmallerTypesAndIncreaseToSmallest(v, NarrowScalar, + WidenScalar); + } + + /// A SizeChangeStrategy for the common case where legalization for a + /// particular vector operation consists of having more elements in the + /// vector, to a type that is legal. Unless there is no such type and then + /// instead it should be legalized towards the widest vector that's still + /// legal. E.g. + /// setAction({G_ADD, LLT::vector(8, 8)}, Legal); + /// setAction({G_ADD, LLT::vector(16, 8)}, Legal); + /// setAction({G_ADD, LLT::vector(2, 32)}, Legal); + /// setAction({G_ADD, LLT::vector(4, 32)}, Legal); + /// setLegalizeVectorElementToDifferentSizeStrategy( + /// G_ADD, 0, moreToWiderTypesAndLessToWidest); + /// will result in the following getAction results: + /// * getAction({G_ADD, LLT::vector(8,8)}) returns + /// (Legal, vector(8,8)). + /// * getAction({G_ADD, LLT::vector(9,8)}) returns + /// (MoreElements, vector(16,8)). + /// * getAction({G_ADD, LLT::vector(8,32)}) returns + /// (FewerElements, vector(4,32)). + static SizeAndActionsVec + moreToWiderTypesAndLessToWidest(const SizeAndActionsVec &v) { + using namespace LegacyLegalizeActions; + return increaseToLargerTypesAndDecreaseToLargest(v, MoreElements, + FewerElements); + } + + /// Helper function to implement many typical SizeChangeStrategy functions. + static SizeAndActionsVec increaseToLargerTypesAndDecreaseToLargest( + const SizeAndActionsVec &v, + LegacyLegalizeActions::LegacyLegalizeAction IncreaseAction, + LegacyLegalizeActions::LegacyLegalizeAction DecreaseAction); + /// Helper function to implement many typical SizeChangeStrategy functions. + static SizeAndActionsVec decreaseToSmallerTypesAndIncreaseToSmallest( + const SizeAndActionsVec &v, + LegacyLegalizeActions::LegacyLegalizeAction DecreaseAction, + LegacyLegalizeActions::LegacyLegalizeAction IncreaseAction); + + LegacyLegalizeActionStep getAction(const LegalityQuery &Query) const; + + unsigned getOpcodeIdxForOpcode(unsigned Opcode) const; + +private: + /// Determine what action should be taken to legalize the given generic + /// instruction opcode, type-index and type. Requires computeTables to have + /// been called. + /// + /// \returns a pair consisting of the kind of legalization that should be + /// performed and the destination type. + std::pair<LegacyLegalizeActions::LegacyLegalizeAction, LLT> + getAspectAction(const InstrAspect &Aspect) const; + + /// The SizeAndActionsVec is a representation mapping between all natural + /// numbers and an Action. The natural number represents the bit size of + /// the InstrAspect. For example, for a target with native support for 32-bit + /// and 64-bit additions, you'd express that as: + /// setScalarAction(G_ADD, 0, + /// {{1, WidenScalar}, // bit sizes [ 1, 31[ + /// {32, Legal}, // bit sizes [32, 33[ + /// {33, WidenScalar}, // bit sizes [33, 64[ + /// {64, Legal}, // bit sizes [64, 65[ + /// {65, NarrowScalar} // bit sizes [65, +inf[ + /// }); + /// It may be that only 64-bit pointers are supported on your target: + /// setPointerAction(G_PTR_ADD, 0, LLT:pointer(1), + /// {{1, Unsupported}, // bit sizes [ 1, 63[ + /// {64, Legal}, // bit sizes [64, 65[ + /// {65, Unsupported}, // bit sizes [65, +inf[ + /// }); + void setScalarAction(const unsigned Opcode, const unsigned TypeIndex, + const SizeAndActionsVec &SizeAndActions) { + const unsigned OpcodeIdx = Opcode - FirstOp; + SmallVector<SizeAndActionsVec, 1> &Actions = ScalarActions[OpcodeIdx]; + setActions(TypeIndex, Actions, SizeAndActions); + } + void setPointerAction(const unsigned Opcode, const unsigned TypeIndex, + const unsigned AddressSpace, + const SizeAndActionsVec &SizeAndActions) { + const unsigned OpcodeIdx = Opcode - FirstOp; + if (AddrSpace2PointerActions[OpcodeIdx].find(AddressSpace) == + AddrSpace2PointerActions[OpcodeIdx].end()) + AddrSpace2PointerActions[OpcodeIdx][AddressSpace] = {{}}; + SmallVector<SizeAndActionsVec, 1> &Actions = + AddrSpace2PointerActions[OpcodeIdx].find(AddressSpace)->second; + setActions(TypeIndex, Actions, SizeAndActions); + } + + /// If an operation on a given vector type (say <M x iN>) isn't explicitly + /// specified, we proceed in 2 stages. First we legalize the underlying scalar + /// (so that there's at least one legal vector with that scalar), then we + /// adjust the number of elements in the vector so that it is legal. The + /// desired action in the first step is controlled by this function. + void setScalarInVectorAction(const unsigned Opcode, const unsigned TypeIndex, + const SizeAndActionsVec &SizeAndActions) { + unsigned OpcodeIdx = Opcode - FirstOp; + SmallVector<SizeAndActionsVec, 1> &Actions = + ScalarInVectorActions[OpcodeIdx]; + setActions(TypeIndex, Actions, SizeAndActions); + } + + /// See also setScalarInVectorAction. + /// This function let's you specify the number of elements in a vector that + /// are legal for a legal element size. + void setVectorNumElementAction(const unsigned Opcode, + const unsigned TypeIndex, + const unsigned ElementSize, + const SizeAndActionsVec &SizeAndActions) { + const unsigned OpcodeIdx = Opcode - FirstOp; + if (NumElements2Actions[OpcodeIdx].find(ElementSize) == + NumElements2Actions[OpcodeIdx].end()) + NumElements2Actions[OpcodeIdx][ElementSize] = {{}}; + SmallVector<SizeAndActionsVec, 1> &Actions = + NumElements2Actions[OpcodeIdx].find(ElementSize)->second; + setActions(TypeIndex, Actions, SizeAndActions); + } + + /// A partial SizeAndActionsVec potentially doesn't cover all bit sizes, + /// i.e. it's OK if it doesn't start from size 1. + static void checkPartialSizeAndActionsVector(const SizeAndActionsVec& v) { + using namespace LegacyLegalizeActions; +#ifndef NDEBUG + // The sizes should be in increasing order + int prev_size = -1; + for(auto SizeAndAction: v) { + assert(SizeAndAction.first > prev_size); + prev_size = SizeAndAction.first; + } + // - for every Widen action, there should be a larger bitsize that + // can be legalized towards (e.g. Legal, Lower, Libcall or Custom + // action). + // - for every Narrow action, there should be a smaller bitsize that + // can be legalized towards. + int SmallestNarrowIdx = -1; + int LargestWidenIdx = -1; + int SmallestLegalizableToSameSizeIdx = -1; + int LargestLegalizableToSameSizeIdx = -1; + for(size_t i=0; i<v.size(); ++i) { + switch (v[i].second) { + case FewerElements: + case NarrowScalar: + if (SmallestNarrowIdx == -1) + SmallestNarrowIdx = i; + break; + case WidenScalar: + case MoreElements: + LargestWidenIdx = i; + break; + case Unsupported: + break; + default: + if (SmallestLegalizableToSameSizeIdx == -1) + SmallestLegalizableToSameSizeIdx = i; + LargestLegalizableToSameSizeIdx = i; + } + } + if (SmallestNarrowIdx != -1) { + assert(SmallestLegalizableToSameSizeIdx != -1); + assert(SmallestNarrowIdx > SmallestLegalizableToSameSizeIdx); + } + if (LargestWidenIdx != -1) + assert(LargestWidenIdx < LargestLegalizableToSameSizeIdx); +#endif + } + + /// A full SizeAndActionsVec must cover all bit sizes, i.e. must start with + /// from size 1. + static void checkFullSizeAndActionsVector(const SizeAndActionsVec& v) { +#ifndef NDEBUG + // Data structure invariant: The first bit size must be size 1. + assert(v.size() >= 1); + assert(v[0].first == 1); + checkPartialSizeAndActionsVector(v); +#endif + } + + /// Sets actions for all bit sizes on a particular generic opcode, type + /// index and scalar or pointer type. + void setActions(unsigned TypeIndex, + SmallVector<SizeAndActionsVec, 1> &Actions, + const SizeAndActionsVec &SizeAndActions) { + checkFullSizeAndActionsVector(SizeAndActions); + if (Actions.size() <= TypeIndex) + Actions.resize(TypeIndex + 1); + Actions[TypeIndex] = SizeAndActions; + } + + static SizeAndAction findAction(const SizeAndActionsVec &Vec, + const uint32_t Size); + + /// Returns the next action needed to get the scalar or pointer type closer + /// to being legal + /// E.g. findLegalAction({G_REM, 13}) should return + /// (WidenScalar, 32). After that, findLegalAction({G_REM, 32}) will + /// probably be called, which should return (Lower, 32). + /// This is assuming the setScalarAction on G_REM was something like: + /// setScalarAction(G_REM, 0, + /// {{1, WidenScalar}, // bit sizes [ 1, 31[ + /// {32, Lower}, // bit sizes [32, 33[ + /// {33, NarrowScalar} // bit sizes [65, +inf[ + /// }); + std::pair<LegacyLegalizeActions::LegacyLegalizeAction, LLT> + findScalarLegalAction(const InstrAspect &Aspect) const; + + /// Returns the next action needed towards legalizing the vector type. + std::pair<LegacyLegalizeActions::LegacyLegalizeAction, LLT> + findVectorLegalAction(const InstrAspect &Aspect) const; + + static const int FirstOp = TargetOpcode::PRE_ISEL_GENERIC_OPCODE_START; + static const int LastOp = TargetOpcode::PRE_ISEL_GENERIC_OPCODE_END; + + // Data structures used temporarily during construction of legality data: + using TypeMap = DenseMap<LLT, LegacyLegalizeActions::LegacyLegalizeAction>; + SmallVector<TypeMap, 1> SpecifiedActions[LastOp - FirstOp + 1]; + SmallVector<SizeChangeStrategy, 1> + ScalarSizeChangeStrategies[LastOp - FirstOp + 1]; + SmallVector<SizeChangeStrategy, 1> + VectorElementSizeChangeStrategies[LastOp - FirstOp + 1]; + bool TablesInitialized; + + // Data structures used by getAction: + SmallVector<SizeAndActionsVec, 1> ScalarActions[LastOp - FirstOp + 1]; + SmallVector<SizeAndActionsVec, 1> ScalarInVectorActions[LastOp - FirstOp + 1]; + std::unordered_map<uint16_t, SmallVector<SizeAndActionsVec, 1>> + AddrSpace2PointerActions[LastOp - FirstOp + 1]; + std::unordered_map<uint16_t, SmallVector<SizeAndActionsVec, 1>> + NumElements2Actions[LastOp - FirstOp + 1]; +}; + +} // end namespace llvm + +#endif // define LLVM_CODEGEN_GLOBALISEL_LEGACYLEGALIZERINFO_H diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h index e7bda3b4bd97..44a48927d35a 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h @@ -11,6 +11,11 @@ // at the end of the legalization. //===----------------------------------------------------------------------===// +#ifndef LLVM_CODEGEN_GLOBALISEL_LEGALIZATIONARTIFACTCOMBINER_H +#define LLVM_CODEGEN_GLOBALISEL_LEGALIZATIONARTIFACTCOMBINER_H + +#include "llvm/ADT/SmallBitVector.h" +#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" #include "llvm/CodeGen/GlobalISel/Legalizer.h" #include "llvm/CodeGen/GlobalISel/LegalizerInfo.h" #include "llvm/CodeGen/GlobalISel/MIPatternMatch.h" @@ -78,7 +83,6 @@ public: } // Try to fold aext(g_constant) when the larger constant type is legal. - // Can't use MIPattern because we don't have a specific constant in mind. auto *SrcMI = MRI.getVRegDef(SrcReg); if (SrcMI->getOpcode() == TargetOpcode::G_CONSTANT) { const LLT DstTy = MRI.getType(DstReg); @@ -139,7 +143,6 @@ public: } // Try to fold zext(g_constant) when the larger constant type is legal. - // Can't use MIPattern because we don't have a specific constant in mind. auto *SrcMI = MRI.getVRegDef(SrcReg); if (SrcMI->getOpcode() == TargetOpcode::G_CONSTANT) { const LLT DstTy = MRI.getType(DstReg); @@ -194,6 +197,20 @@ public: return true; } + // Try to fold sext(g_constant) when the larger constant type is legal. + auto *SrcMI = MRI.getVRegDef(SrcReg); + if (SrcMI->getOpcode() == TargetOpcode::G_CONSTANT) { + const LLT DstTy = MRI.getType(DstReg); + if (isInstLegal({TargetOpcode::G_CONSTANT, {DstTy}})) { + auto &CstVal = SrcMI->getOperand(1); + Builder.buildConstant( + DstReg, CstVal.getCImm()->getValue().sext(DstTy.getSizeInBits())); + UpdatedDefs.push_back(DstReg); + markInstAndDefDead(MI, *SrcMI, DeadInsts); + return true; + } + } + return tryFoldImplicitDef(MI, DeadInsts, UpdatedDefs); } @@ -208,7 +225,6 @@ public: Register SrcReg = lookThroughCopyInstrs(MI.getOperand(1).getReg()); // Try to fold trunc(g_constant) when the smaller constant type is legal. - // Can't use MIPattern because we don't have a specific constant in mind. auto *SrcMI = MRI.getVRegDef(SrcReg); if (SrcMI->getOpcode() == TargetOpcode::G_CONSTANT) { const LLT DstTy = MRI.getType(DstReg); @@ -224,8 +240,8 @@ public: // Try to fold trunc(merge) to directly use the source of the merge. // This gets rid of large, difficult to legalize, merges - if (SrcMI->getOpcode() == TargetOpcode::G_MERGE_VALUES) { - const Register MergeSrcReg = SrcMI->getOperand(1).getReg(); + if (auto *SrcMerge = dyn_cast<GMerge>(SrcMI)) { + const Register MergeSrcReg = SrcMerge->getSourceReg(0); const LLT MergeSrcTy = MRI.getType(MergeSrcReg); const LLT DstTy = MRI.getType(DstReg); @@ -269,7 +285,7 @@ public: "trunc(merge) should require less inputs than merge"); SmallVector<Register, 8> SrcRegs(NumSrcs); for (unsigned i = 0; i < NumSrcs; ++i) - SrcRegs[i] = SrcMI->getOperand(i + 1).getReg(); + SrcRegs[i] = SrcMerge->getSourceReg(i); Builder.buildMerge(DstReg, SrcRegs); UpdatedDefs.push_back(DstReg); @@ -278,7 +294,7 @@ public: return false; } - markInstAndDefDead(MI, *SrcMI, DeadInsts); + markInstAndDefDead(MI, *SrcMerge, DeadInsts); return true; } @@ -370,7 +386,8 @@ public: unsigned UnmergeNumElts = DestTy.isVector() ? CastSrcTy.getNumElements() / NumDefs : 1; - LLT UnmergeTy = CastSrcTy.changeNumElements(UnmergeNumElts); + LLT UnmergeTy = CastSrcTy.changeElementCount( + ElementCount::getFixed(UnmergeNumElts)); if (isInstUnsupported( {TargetOpcode::G_UNMERGE_VALUES, {UnmergeTy, CastSrcTy}})) @@ -518,29 +535,267 @@ public: return DefIdx; } - bool tryCombineUnmergeValues(MachineInstr &MI, + /// This class provides utilities for finding source registers of specific + /// bit ranges in an artifact. The routines can look through the source + /// registers if they're other artifacts to try to find a non-artifact source + /// of a value. + class ArtifactValueFinder { + MachineRegisterInfo &MRI; + MachineIRBuilder &MIB; + const LegalizerInfo &LI; + + private: + /// Given an concat_vector op \p Concat and a start bit and size, try to + /// find the origin of the value defined by that start position and size. + /// + /// \returns A register if a value can be found, otherwise an empty + /// Register. + Register findValueFromConcat(GConcatVectors &Concat, unsigned StartBit, + unsigned Size) { + assert(Size > 0); + + // Find the source operand that provides the bits requested. + Register Src1Reg = Concat.getSourceReg(0); + unsigned SrcSize = MRI.getType(Src1Reg).getSizeInBits(); + + // Operand index of the source that provides the start of the bit range. + unsigned StartSrcIdx = (StartBit / SrcSize) + 1; + // Offset into the source at which the bit range starts. + unsigned InRegOffset = StartBit % SrcSize; + // Check that the bits don't span multiple sources. + // FIXME: we might be able return multiple sources? Or create an + // appropriate concat to make it fit. + if (InRegOffset + Size > SrcSize) + return Register(); + + // If the bits exactly cover a single source, then return the operand as + // our value reg. + Register SrcReg = Concat.getReg(StartSrcIdx); + if (InRegOffset == 0 && Size == SrcSize) + return SrcReg; // A source operand matches exactly. + + return findValueFromDef(SrcReg, InRegOffset, Size); + } + + /// Given an build_vector op \p BV and a start bit and size, try to find + /// the origin of the value defined by that start position and size. + /// + /// \returns A register if a value can be found, otherwise an empty + /// Register. + Register findValueFromBuildVector(GBuildVector &BV, unsigned StartBit, + unsigned Size) { + assert(Size > 0); + + // Find the source operand that provides the bits requested. + Register Src1Reg = BV.getSourceReg(0); + unsigned SrcSize = MRI.getType(Src1Reg).getSizeInBits(); + + // Operand index of the source that provides the start of the bit range. + unsigned StartSrcIdx = (StartBit / SrcSize) + 1; + // Offset into the source at which the bit range starts. + unsigned InRegOffset = StartBit % SrcSize; + + if (InRegOffset != 0) + return Register(); // Give up, bits don't start at a scalar source. + if (Size < SrcSize) + return Register(); // Scalar source is too large for requested bits. + + // If the bits cover multiple sources evenly, then create a new + // build_vector to synthesize the required size, if that's been requested. + if (Size > SrcSize) { + if (Size % SrcSize > 0) + return Register(); // Isn't covered exactly by sources. + + unsigned NumSrcsUsed = Size / SrcSize; + LLT SrcTy = MRI.getType(Src1Reg); + LLT NewBVTy = LLT::fixed_vector(NumSrcsUsed, SrcTy); + + // Check if the resulting build vector would be legal. + LegalizeActionStep ActionStep = + LI.getAction({TargetOpcode::G_BUILD_VECTOR, {NewBVTy, SrcTy}}); + if (ActionStep.Action != LegalizeActions::Legal) + return Register(); + + SmallVector<Register> NewSrcs; + for (unsigned SrcIdx = StartSrcIdx; SrcIdx < StartSrcIdx + NumSrcsUsed; + ++SrcIdx) + NewSrcs.push_back(BV.getReg(SrcIdx)); + MIB.setInstrAndDebugLoc(BV); + return MIB.buildBuildVector(NewBVTy, NewSrcs).getReg(0); + } + // A single source is requested, just return it. + return BV.getReg(StartSrcIdx); + } + + /// Given an G_INSERT op \p MI and a start bit and size, try to find + /// the origin of the value defined by that start position and size. + /// + /// \returns A register if a value can be found, otherwise an empty + /// Register. + Register findValueFromInsert(MachineInstr &MI, unsigned StartBit, + unsigned Size) { + assert(MI.getOpcode() == TargetOpcode::G_INSERT); + assert(Size > 0); + + Register ContainerSrcReg = MI.getOperand(1).getReg(); + Register InsertedReg = MI.getOperand(2).getReg(); + LLT InsertedRegTy = MRI.getType(InsertedReg); + unsigned InsertOffset = MI.getOperand(3).getImm(); + + // There are 4 possible container/insertreg + requested bit-range layouts + // that the instruction and query could be representing. + // For: %_ = G_INSERT %CONTAINER, %INS, InsOff (abbrev. to 'IO') + // and a start bit 'SB', with size S, giving an end bit 'EB', we could + // have... + // Scenario A: + // -------------------------- + // | INS | CONTAINER | + // -------------------------- + // | | + // SB EB + // + // Scenario B: + // -------------------------- + // | INS | CONTAINER | + // -------------------------- + // | | + // SB EB + // + // Scenario C: + // -------------------------- + // | CONTAINER | INS | + // -------------------------- + // | | + // SB EB + // + // Scenario D: + // -------------------------- + // | CONTAINER | INS | + // -------------------------- + // | | + // SB EB + // + // So therefore, A and D are requesting data from the INS operand, while + // B and C are requesting from the container operand. + + unsigned InsertedEndBit = InsertOffset + InsertedRegTy.getSizeInBits(); + unsigned EndBit = StartBit + Size; + unsigned NewStartBit; + Register SrcRegToUse; + if (EndBit <= InsertOffset || InsertedEndBit <= StartBit) { + SrcRegToUse = ContainerSrcReg; + NewStartBit = StartBit; + return findValueFromDef(SrcRegToUse, NewStartBit, Size); + } + if (InsertOffset <= StartBit && EndBit <= InsertedEndBit) { + SrcRegToUse = InsertedReg; + NewStartBit = StartBit - InsertOffset; + return findValueFromDef(SrcRegToUse, NewStartBit, Size); + } + // The bit range spans both the inserted and container regions. + return Register(); + } + + public: + ArtifactValueFinder(MachineRegisterInfo &Mri, MachineIRBuilder &Builder, + const LegalizerInfo &Info) + : MRI(Mri), MIB(Builder), LI(Info) {} + + /// Try to find a source of the value defined in the def \p DefReg, starting + /// at position \p StartBit with size \p Size. + /// \returns an empty Register if no value could be found, or \p DefReg if + /// if that was the best we could do. + Register findValueFromDef(Register DefReg, unsigned StartBit, + unsigned Size) { + MachineInstr *Def = getDefIgnoringCopies(DefReg, MRI); + // If the instruction has a single def, then simply delegate the search. + // For unmerge however with multiple defs, we need to compute the offset + // into the source of the unmerge. + switch (Def->getOpcode()) { + case TargetOpcode::G_CONCAT_VECTORS: + return findValueFromConcat(cast<GConcatVectors>(*Def), StartBit, Size); + case TargetOpcode::G_UNMERGE_VALUES: { + unsigned DefStartBit = 0; + unsigned DefSize = MRI.getType(DefReg).getSizeInBits(); + for (const auto &MO : Def->defs()) { + if (MO.getReg() == DefReg) + break; + DefStartBit += DefSize; + } + Register SrcReg = Def->getOperand(Def->getNumOperands() - 1).getReg(); + Register SrcOriginReg = + findValueFromDef(SrcReg, StartBit + DefStartBit, Size); + if (SrcOriginReg) + return SrcOriginReg; + // Failed to find a further value. If the StartBit and Size perfectly + // covered the requested DefReg, return that since it's better than + // nothing. + if (StartBit == 0 && Size == DefSize) + return DefReg; + return Register(); + } + case TargetOpcode::G_BUILD_VECTOR: + return findValueFromBuildVector(cast<GBuildVector>(*Def), StartBit, + Size); + case TargetOpcode::G_INSERT: + return findValueFromInsert(*Def, StartBit, Size); + default: + return Register(); + } + } + }; + + bool tryCombineUnmergeValues(GUnmerge &MI, SmallVectorImpl<MachineInstr *> &DeadInsts, SmallVectorImpl<Register> &UpdatedDefs, GISelChangeObserver &Observer) { - assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES); - - unsigned NumDefs = MI.getNumOperands() - 1; - Register SrcReg = MI.getOperand(NumDefs).getReg(); + unsigned NumDefs = MI.getNumDefs(); + Register SrcReg = MI.getSourceReg(); MachineInstr *SrcDef = getDefIgnoringCopies(SrcReg, MRI); if (!SrcDef) return false; - LLT OpTy = MRI.getType(MI.getOperand(NumDefs).getReg()); - LLT DestTy = MRI.getType(MI.getOperand(0).getReg()); + LLT OpTy = MRI.getType(SrcReg); + LLT DestTy = MRI.getType(MI.getReg(0)); + unsigned SrcDefIdx = getDefIndex(*SrcDef, SrcReg); + + Builder.setInstrAndDebugLoc(MI); + + auto tryCombineViaValueFinder = [&]() { + ArtifactValueFinder ValueFinder(MRI, Builder, LI); + + SmallBitVector DeadDefs(NumDefs); + for (unsigned DefIdx = 0; DefIdx < NumDefs; ++DefIdx) { + Register DefReg = MI.getReg(DefIdx); + Register FoundVal = + ValueFinder.findValueFromDef(DefReg, 0, DestTy.getSizeInBits()); + if (!FoundVal || FoundVal == DefReg) + continue; + if (MRI.getType(FoundVal) != DestTy) + continue; + + replaceRegOrBuildCopy(DefReg, FoundVal, MRI, Builder, UpdatedDefs, + Observer); + // We only want to replace the uses, not the def of the old reg. + Observer.changingInstr(MI); + MI.getOperand(DefIdx).setReg(DefReg); + Observer.changedInstr(MI); + DeadDefs[DefIdx] = true; + } + if (DeadDefs.all()) { + markInstAndDefDead(MI, *SrcDef, DeadInsts, SrcDefIdx); + return true; + } + return false; + }; - if (SrcDef->getOpcode() == TargetOpcode::G_UNMERGE_VALUES) { + if (auto *SrcUnmerge = dyn_cast<GUnmerge>(SrcDef)) { // %0:_(<4 x s16>) = G_FOO // %1:_(<2 x s16>), %2:_(<2 x s16>) = G_UNMERGE_VALUES %0 // %3:_(s16), %4:_(s16) = G_UNMERGE_VALUES %1 // // %3:_(s16), %4:_(s16), %5:_(s16), %6:_(s16) = G_UNMERGE_VALUES %0 - const unsigned NumSrcOps = SrcDef->getNumOperands(); - Register SrcUnmergeSrc = SrcDef->getOperand(NumSrcOps - 1).getReg(); + Register SrcUnmergeSrc = SrcUnmerge->getSourceReg(); LLT SrcUnmergeSrcTy = MRI.getType(SrcUnmergeSrc); // If we need to decrease the number of vector elements in the result type @@ -558,23 +813,21 @@ public: return false; break; default: - return false; + return tryCombineViaValueFinder(); } - Builder.setInstrAndDebugLoc(MI); auto NewUnmerge = Builder.buildUnmerge(DestTy, SrcUnmergeSrc); // TODO: Should we try to process out the other defs now? If the other // defs of the source unmerge are also unmerged, we end up with a separate // unmerge for each one. - unsigned SrcDefIdx = getDefIndex(*SrcDef, SrcReg); for (unsigned I = 0; I != NumDefs; ++I) { - Register Def = MI.getOperand(I).getReg(); + Register Def = MI.getReg(I); replaceRegOrBuildCopy(Def, NewUnmerge.getReg(SrcDefIdx * NumDefs + I), MRI, Builder, UpdatedDefs, Observer); } - markInstAndDefDead(MI, *SrcDef, DeadInsts, SrcDefIdx); + markInstAndDefDead(MI, *SrcUnmerge, DeadInsts, SrcDefIdx); return true; } @@ -592,7 +845,11 @@ public: ConvertOp, OpTy, DestTy)) { // We might have a chance to combine later by trying to combine // unmerge(cast) first - return tryFoldUnmergeCast(MI, *SrcDef, DeadInsts, UpdatedDefs); + if (tryFoldUnmergeCast(MI, *SrcDef, DeadInsts, UpdatedDefs)) + return true; + + // Try using the value finder. + return tryCombineViaValueFinder(); } const unsigned NumMergeRegs = MergeI->getNumOperands() - 1; @@ -614,7 +871,7 @@ public: SmallVector<Register, 8> DstRegs; for (unsigned j = 0, DefIdx = Idx * NewNumDefs; j < NewNumDefs; ++j, ++DefIdx) - DstRegs.push_back(MI.getOperand(DefIdx).getReg()); + DstRegs.push_back(MI.getReg(DefIdx)); if (ConvertOp) { LLT MergeSrcTy = MRI.getType(MergeI->getOperand(1).getReg()); @@ -671,7 +928,7 @@ public: ++j, ++Idx) Regs.push_back(MergeI->getOperand(Idx).getReg()); - Register DefReg = MI.getOperand(DefIdx).getReg(); + Register DefReg = MI.getReg(DefIdx); Builder.buildMerge(DefReg, Regs); UpdatedDefs.push_back(DefReg); } @@ -713,17 +970,6 @@ public: return true; } - static bool isMergeLikeOpcode(unsigned Opc) { - switch (Opc) { - case TargetOpcode::G_MERGE_VALUES: - case TargetOpcode::G_BUILD_VECTOR: - case TargetOpcode::G_CONCAT_VECTORS: - return true; - default: - return false; - } - } - bool tryCombineExtract(MachineInstr &MI, SmallVectorImpl<MachineInstr *> &DeadInsts, SmallVectorImpl<Register> &UpdatedDefs) { @@ -743,7 +989,7 @@ public: Register SrcReg = lookThroughCopyInstrs(MI.getOperand(1).getReg()); MachineInstr *MergeI = MRI.getVRegDef(SrcReg); - if (!MergeI || !isMergeLikeOpcode(MergeI->getOpcode())) + if (!MergeI || !isa<GMergeLikeOp>(MergeI)) return false; Register DstReg = MI.getOperand(0).getReg(); @@ -805,8 +1051,8 @@ public: Changed = tryCombineSExt(MI, DeadInsts, UpdatedDefs); break; case TargetOpcode::G_UNMERGE_VALUES: - Changed = - tryCombineUnmergeValues(MI, DeadInsts, UpdatedDefs, WrapperObserver); + Changed = tryCombineUnmergeValues(cast<GUnmerge>(MI), DeadInsts, + UpdatedDefs, WrapperObserver); break; case TargetOpcode::G_MERGE_VALUES: case TargetOpcode::G_BUILD_VECTOR: @@ -1009,3 +1255,5 @@ private: }; } // namespace llvm + +#endif // LLVM_CODEGEN_GLOBALISEL_LEGALIZATIONARTIFACTCOMBINER_H diff --git a/llvm/include/llvm/CodeGen/GlobalISel/Legalizer.h b/llvm/include/llvm/CodeGen/GlobalISel/Legalizer.h index 690e84f79a6b..4871d8d32ebd 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/Legalizer.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/Legalizer.h @@ -17,8 +17,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_CODEGEN_GLOBALISEL_LEGALIZEMACHINEIRPASS_H -#define LLVM_CODEGEN_GLOBALISEL_LEGALIZEMACHINEIRPASS_H +#ifndef LLVM_CODEGEN_GLOBALISEL_LEGALIZER_H +#define LLVM_CODEGEN_GLOBALISEL_LEGALIZER_H #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" #include "llvm/CodeGen/MachineFunctionPass.h" diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h index c3b494e94ff1..67141f3a6326 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h @@ -17,10 +17,11 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_CODEGEN_GLOBALISEL_MACHINELEGALIZEHELPER_H -#define LLVM_CODEGEN_GLOBALISEL_MACHINELEGALIZEHELPER_H +#ifndef LLVM_CODEGEN_GLOBALISEL_LEGALIZERHELPER_H +#define LLVM_CODEGEN_GLOBALISEL_LEGALIZERHELPER_H #include "llvm/CodeGen/GlobalISel/CallLowering.h" +#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" #include "llvm/CodeGen/LowLevelType.h" #include "llvm/CodeGen/MachineFunctionPass.h" @@ -32,6 +33,7 @@ class LegalizerInfo; class Legalizer; class MachineRegisterInfo; class GISelChangeObserver; +class LostDebugLocObserver; class TargetLowering; class LegalizerHelper { @@ -78,10 +80,11 @@ public: /// /// Considered as an opaque blob, the legal code will use and define the same /// registers as \p MI. - LegalizeResult legalizeInstrStep(MachineInstr &MI); + LegalizeResult legalizeInstrStep(MachineInstr &MI, + LostDebugLocObserver &LocObserver); /// Legalize an instruction by emiting a runtime library call instead. - LegalizeResult libcall(MachineInstr &MI); + LegalizeResult libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver); /// Legalize an instruction by reducing the width of the underlying scalar /// type. @@ -170,10 +173,12 @@ private: widenScalarExtract(MachineInstr &MI, unsigned TypeIdx, LLT WideTy); LegalizeResult widenScalarInsert(MachineInstr &MI, unsigned TypeIdx, LLT WideTy); - LegalizeResult widenScalarAddoSubo(MachineInstr &MI, unsigned TypeIdx, - LLT WideTy); + LegalizeResult widenScalarAddSubOverflow(MachineInstr &MI, unsigned TypeIdx, + LLT WideTy); LegalizeResult widenScalarAddSubShlSat(MachineInstr &MI, unsigned TypeIdx, LLT WideTy); + LegalizeResult widenScalarMulo(MachineInstr &MI, unsigned TypeIdx, + LLT WideTy); /// Helper function to split a wide generic register into bitwise blocks with /// the given Type (which implies the number of blocks needed). The generic @@ -247,6 +252,10 @@ private: void changeOpcode(MachineInstr &MI, unsigned NewOpcode); + LegalizeResult tryNarrowPow2Reduction(MachineInstr &MI, Register SrcReg, + LLT SrcTy, LLT NarrowTy, + unsigned ScalarOpc); + public: /// Return the alignment to use for a stack temporary object with the given /// type. @@ -285,6 +294,8 @@ public: LegalizeResult moreElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx, LLT MoreTy); + LegalizeResult moreElementsVectorShuffle(MachineInstr &MI, unsigned TypeIdx, + LLT MoreTy); LegalizeResult fewerElementsVectorUnmergeValues(MachineInstr &MI, unsigned TypeIdx, @@ -295,8 +306,11 @@ public: unsigned TypeIdx, LLT NarrowTy); - LegalizeResult - reduceLoadStoreWidth(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy); + LegalizeResult fewerElementsVectorMulo(MachineInstr &MI, unsigned TypeIdx, + LLT NarrowTy); + + LegalizeResult reduceLoadStoreWidth(GLoadStore &MI, unsigned TypeIdx, + LLT NarrowTy); /// Legalize an instruction by reducing the operation width, either by /// narrowing the type of the operation or by reducing the number of elements @@ -314,8 +328,17 @@ public: LegalizeResult narrowScalarShiftByConstant(MachineInstr &MI, const APInt &Amt, LLT HalfTy, LLT ShiftAmtTy); + LegalizeResult fewerElementsVectorReductions(MachineInstr &MI, + unsigned TypeIdx, LLT NarrowTy); + + LegalizeResult fewerElementsVectorShuffle(MachineInstr &MI, unsigned TypeIdx, + LLT NarrowTy); + LegalizeResult narrowScalarShift(MachineInstr &MI, unsigned TypeIdx, LLT Ty); + LegalizeResult narrowScalarAddSub(MachineInstr &MI, unsigned TypeIdx, + LLT NarrowTy); LegalizeResult narrowScalarMul(MachineInstr &MI, LLT Ty); + LegalizeResult narrowScalarFPTOI(MachineInstr &MI, unsigned TypeIdx, LLT Ty); LegalizeResult narrowScalarExtract(MachineInstr &MI, unsigned TypeIdx, LLT Ty); LegalizeResult narrowScalarInsert(MachineInstr &MI, unsigned TypeIdx, LLT Ty); @@ -335,9 +358,14 @@ public: LLT CastTy); LegalizeResult lowerBitcast(MachineInstr &MI); - LegalizeResult lowerLoad(MachineInstr &MI); - LegalizeResult lowerStore(MachineInstr &MI); + LegalizeResult lowerLoad(GAnyLoad &MI); + LegalizeResult lowerStore(GStore &MI); LegalizeResult lowerBitCount(MachineInstr &MI); + LegalizeResult lowerFunnelShiftWithInverse(MachineInstr &MI); + LegalizeResult lowerFunnelShiftAsShifts(MachineInstr &MI); + LegalizeResult lowerFunnelShift(MachineInstr &MI); + LegalizeResult lowerRotateWithReverseRotate(MachineInstr &MI); + LegalizeResult lowerRotate(MachineInstr &MI); LegalizeResult lowerU64ToF32BitOps(MachineInstr &MI); LegalizeResult lowerUITOFP(MachineInstr &MI); @@ -371,7 +399,9 @@ public: LegalizeResult lowerReadWriteRegister(MachineInstr &MI); LegalizeResult lowerSMULH_UMULH(MachineInstr &MI); LegalizeResult lowerSelect(MachineInstr &MI); - + LegalizeResult lowerDIVREM(MachineInstr &MI); + LegalizeResult lowerAbsToAddXor(MachineInstr &MI); + LegalizeResult lowerAbsToMaxNeg(MachineInstr &MI); }; /// Helper function that creates a libcall to the given \p Name using the given @@ -388,9 +418,9 @@ createLibcall(MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall, ArrayRef<CallLowering::ArgInfo> Args); /// Create a libcall to memcpy et al. -LegalizerHelper::LegalizeResult createMemLibcall(MachineIRBuilder &MIRBuilder, - MachineRegisterInfo &MRI, - MachineInstr &MI); +LegalizerHelper::LegalizeResult +createMemLibcall(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI, + MachineInstr &MI, LostDebugLocObserver &LocObserver); } // End namespace llvm. diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h index c0a89b6ae619..4fdfabbfb161 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h @@ -5,10 +5,10 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// -// +/// \file /// Interface for Targets to specify which operations they can successfully /// select and how the others should be expanded most efficiently. -// +/// //===----------------------------------------------------------------------===// #ifndef LLVM_CODEGEN_GLOBALISEL_LEGALIZERINFO_H @@ -20,6 +20,7 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallBitVector.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/CodeGen/GlobalISel/LegacyLegalizerInfo.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/TargetOpcodes.h" #include "llvm/Support/CommandLine.h" @@ -100,23 +101,6 @@ raw_ostream &operator<<(raw_ostream &OS, LegalizeActions::LegalizeAction Action) using LegalizeActions::LegalizeAction; -/// Legalization is decided based on an instruction's opcode, which type slot -/// we're considering, and what the existing type is. These aspects are gathered -/// together for convenience in the InstrAspect class. -struct InstrAspect { - unsigned Opcode; - unsigned Idx = 0; - LLT Type; - - InstrAspect(unsigned Opcode, LLT Type) : Opcode(Opcode), Type(Type) {} - InstrAspect(unsigned Opcode, unsigned Idx, LLT Type) - : Opcode(Opcode), Idx(Idx), Type(Type) {} - - bool operator==(const InstrAspect &RHS) const { - return Opcode == RHS.Opcode && Idx == RHS.Idx && Type == RHS.Type; - } -}; - /// The LegalityQuery object bundles together all the information that's needed /// to decide whether a given operation is legal or not. /// For efficiency, it doesn't make a copy of Types so care must be taken not @@ -126,7 +110,7 @@ struct LegalityQuery { ArrayRef<LLT> Types; struct MemDesc { - uint64_t SizeInBits; + LLT MemoryTy; uint64_t AlignInBits; AtomicOrdering Ordering; }; @@ -159,6 +143,45 @@ struct LegalizeActionStep { const LLT NewType) : Action(Action), TypeIdx(TypeIdx), NewType(NewType) {} + LegalizeActionStep(LegacyLegalizeActionStep Step) + : TypeIdx(Step.TypeIdx), NewType(Step.NewType) { + switch (Step.Action) { + case LegacyLegalizeActions::Legal: + Action = LegalizeActions::Legal; + break; + case LegacyLegalizeActions::NarrowScalar: + Action = LegalizeActions::NarrowScalar; + break; + case LegacyLegalizeActions::WidenScalar: + Action = LegalizeActions::WidenScalar; + break; + case LegacyLegalizeActions::FewerElements: + Action = LegalizeActions::FewerElements; + break; + case LegacyLegalizeActions::MoreElements: + Action = LegalizeActions::MoreElements; + break; + case LegacyLegalizeActions::Bitcast: + Action = LegalizeActions::Bitcast; + break; + case LegacyLegalizeActions::Lower: + Action = LegalizeActions::Lower; + break; + case LegacyLegalizeActions::Libcall: + Action = LegalizeActions::Libcall; + break; + case LegacyLegalizeActions::Custom: + Action = LegalizeActions::Custom; + break; + case LegacyLegalizeActions::Unsupported: + Action = LegalizeActions::Unsupported; + break; + case LegacyLegalizeActions::NotFound: + Action = LegalizeActions::NotFound; + break; + } + } + bool operator==(const LegalizeActionStep &RHS) const { return std::tie(Action, TypeIdx, NewType) == std::tie(RHS.Action, RHS.TypeIdx, RHS.NewType); @@ -173,13 +196,12 @@ namespace LegalityPredicates { struct TypePairAndMemDesc { LLT Type0; LLT Type1; - uint64_t MemSize; + LLT MemTy; uint64_t Align; bool operator==(const TypePairAndMemDesc &Other) const { return Type0 == Other.Type0 && Type1 == Other.Type1 && - Align == Other.Align && - MemSize == Other.MemSize; + Align == Other.Align && MemTy == Other.MemTy; } /// \returns true if this memory access is legal with for the access described @@ -187,7 +209,9 @@ struct TypePairAndMemDesc { bool isCompatible(const TypePairAndMemDesc &Other) const { return Type0 == Other.Type0 && Type1 == Other.Type1 && Align >= Other.Align && - MemSize == Other.MemSize; + // FIXME: This perhaps should be stricter, but the current legality + // rules are written only considering the size. + MemTy.getSizeInBits() == Other.MemTy.getSizeInBits(); } }; @@ -956,6 +980,23 @@ public: }); } + /// Conditionally narrow the scalar or elt to match the size of another. + LegalizeRuleSet &maxScalarEltSameAsIf(LegalityPredicate Predicate, + unsigned TypeIdx, + unsigned SmallTypeIdx) { + typeIdx(TypeIdx); + return narrowScalarIf( + [=](const LegalityQuery &Query) { + return Query.Types[SmallTypeIdx].getScalarSizeInBits() < + Query.Types[TypeIdx].getScalarSizeInBits() && + Predicate(Query); + }, + [=](const LegalityQuery &Query) { + LLT T = Query.Types[SmallTypeIdx]; + return std::make_pair(TypeIdx, T); + }); + } + /// Add more elements to the vector to reach the next power of two. /// No effect if the type is not a vector or the element count is a power of /// two. @@ -981,7 +1022,7 @@ public: [=](const LegalityQuery &Query) { LLT VecTy = Query.Types[TypeIdx]; return std::make_pair( - TypeIdx, LLT::vector(MinElements, VecTy.getElementType())); + TypeIdx, LLT::fixed_vector(MinElements, VecTy.getElementType())); }); } /// Limit the number of elements in EltTy vectors to at most MaxElements. @@ -998,7 +1039,8 @@ public: }, [=](const LegalityQuery &Query) { LLT VecTy = Query.Types[TypeIdx]; - LLT NewTy = LLT::scalarOrVector(MaxElements, VecTy.getElementType()); + LLT NewTy = LLT::scalarOrVector(ElementCount::getFixed(MaxElements), + VecTy.getElementType()); return std::make_pair(TypeIdx, NewTy); }); } @@ -1040,179 +1082,20 @@ public: class LegalizerInfo { public: - LegalizerInfo(); virtual ~LegalizerInfo() = default; + const LegacyLegalizerInfo &getLegacyLegalizerInfo() const { + return LegacyInfo; + } + LegacyLegalizerInfo &getLegacyLegalizerInfo() { return LegacyInfo; } + unsigned getOpcodeIdxForOpcode(unsigned Opcode) const; unsigned getActionDefinitionsIdx(unsigned Opcode) const; - /// Compute any ancillary tables needed to quickly decide how an operation - /// should be handled. This must be called after all "set*Action"methods but - /// before any query is made or incorrect results may be returned. - void computeTables(); - /// Perform simple self-diagnostic and assert if there is anything obviously /// wrong with the actions set up. void verify(const MCInstrInfo &MII) const; - static bool needsLegalizingToDifferentSize(const LegalizeAction Action) { - using namespace LegalizeActions; - switch (Action) { - case NarrowScalar: - case WidenScalar: - case FewerElements: - case MoreElements: - case Unsupported: - return true; - default: - return false; - } - } - - using SizeAndAction = std::pair<uint16_t, LegalizeAction>; - using SizeAndActionsVec = std::vector<SizeAndAction>; - using SizeChangeStrategy = - std::function<SizeAndActionsVec(const SizeAndActionsVec &v)>; - - /// More friendly way to set an action for common types that have an LLT - /// representation. - /// The LegalizeAction must be one for which NeedsLegalizingToDifferentSize - /// returns false. - void setAction(const InstrAspect &Aspect, LegalizeAction Action) { - assert(!needsLegalizingToDifferentSize(Action)); - TablesInitialized = false; - const unsigned OpcodeIdx = Aspect.Opcode - FirstOp; - if (SpecifiedActions[OpcodeIdx].size() <= Aspect.Idx) - SpecifiedActions[OpcodeIdx].resize(Aspect.Idx + 1); - SpecifiedActions[OpcodeIdx][Aspect.Idx][Aspect.Type] = Action; - } - - /// The setAction calls record the non-size-changing legalization actions - /// to take on specificly-sized types. The SizeChangeStrategy defines what - /// to do when the size of the type needs to be changed to reach a legally - /// sized type (i.e., one that was defined through a setAction call). - /// e.g. - /// setAction ({G_ADD, 0, LLT::scalar(32)}, Legal); - /// setLegalizeScalarToDifferentSizeStrategy( - /// G_ADD, 0, widenToLargerTypesAndNarrowToLargest); - /// will end up defining getAction({G_ADD, 0, T}) to return the following - /// actions for different scalar types T: - /// LLT::scalar(1)..LLT::scalar(31): {WidenScalar, 0, LLT::scalar(32)} - /// LLT::scalar(32): {Legal, 0, LLT::scalar(32)} - /// LLT::scalar(33)..: {NarrowScalar, 0, LLT::scalar(32)} - /// - /// If no SizeChangeAction gets defined, through this function, - /// the default is unsupportedForDifferentSizes. - void setLegalizeScalarToDifferentSizeStrategy(const unsigned Opcode, - const unsigned TypeIdx, - SizeChangeStrategy S) { - const unsigned OpcodeIdx = Opcode - FirstOp; - if (ScalarSizeChangeStrategies[OpcodeIdx].size() <= TypeIdx) - ScalarSizeChangeStrategies[OpcodeIdx].resize(TypeIdx + 1); - ScalarSizeChangeStrategies[OpcodeIdx][TypeIdx] = S; - } - - /// See also setLegalizeScalarToDifferentSizeStrategy. - /// This function allows to set the SizeChangeStrategy for vector elements. - void setLegalizeVectorElementToDifferentSizeStrategy(const unsigned Opcode, - const unsigned TypeIdx, - SizeChangeStrategy S) { - const unsigned OpcodeIdx = Opcode - FirstOp; - if (VectorElementSizeChangeStrategies[OpcodeIdx].size() <= TypeIdx) - VectorElementSizeChangeStrategies[OpcodeIdx].resize(TypeIdx + 1); - VectorElementSizeChangeStrategies[OpcodeIdx][TypeIdx] = S; - } - - /// A SizeChangeStrategy for the common case where legalization for a - /// particular operation consists of only supporting a specific set of type - /// sizes. E.g. - /// setAction ({G_DIV, 0, LLT::scalar(32)}, Legal); - /// setAction ({G_DIV, 0, LLT::scalar(64)}, Legal); - /// setLegalizeScalarToDifferentSizeStrategy( - /// G_DIV, 0, unsupportedForDifferentSizes); - /// will result in getAction({G_DIV, 0, T}) to return Legal for s32 and s64, - /// and Unsupported for all other scalar types T. - static SizeAndActionsVec - unsupportedForDifferentSizes(const SizeAndActionsVec &v) { - using namespace LegalizeActions; - return increaseToLargerTypesAndDecreaseToLargest(v, Unsupported, - Unsupported); - } - - /// A SizeChangeStrategy for the common case where legalization for a - /// particular operation consists of widening the type to a large legal type, - /// unless there is no such type and then instead it should be narrowed to the - /// largest legal type. - static SizeAndActionsVec - widenToLargerTypesAndNarrowToLargest(const SizeAndActionsVec &v) { - using namespace LegalizeActions; - assert(v.size() > 0 && - "At least one size that can be legalized towards is needed" - " for this SizeChangeStrategy"); - return increaseToLargerTypesAndDecreaseToLargest(v, WidenScalar, - NarrowScalar); - } - - static SizeAndActionsVec - widenToLargerTypesUnsupportedOtherwise(const SizeAndActionsVec &v) { - using namespace LegalizeActions; - return increaseToLargerTypesAndDecreaseToLargest(v, WidenScalar, - Unsupported); - } - - static SizeAndActionsVec - narrowToSmallerAndUnsupportedIfTooSmall(const SizeAndActionsVec &v) { - using namespace LegalizeActions; - return decreaseToSmallerTypesAndIncreaseToSmallest(v, NarrowScalar, - Unsupported); - } - - static SizeAndActionsVec - narrowToSmallerAndWidenToSmallest(const SizeAndActionsVec &v) { - using namespace LegalizeActions; - assert(v.size() > 0 && - "At least one size that can be legalized towards is needed" - " for this SizeChangeStrategy"); - return decreaseToSmallerTypesAndIncreaseToSmallest(v, NarrowScalar, - WidenScalar); - } - - /// A SizeChangeStrategy for the common case where legalization for a - /// particular vector operation consists of having more elements in the - /// vector, to a type that is legal. Unless there is no such type and then - /// instead it should be legalized towards the widest vector that's still - /// legal. E.g. - /// setAction({G_ADD, LLT::vector(8, 8)}, Legal); - /// setAction({G_ADD, LLT::vector(16, 8)}, Legal); - /// setAction({G_ADD, LLT::vector(2, 32)}, Legal); - /// setAction({G_ADD, LLT::vector(4, 32)}, Legal); - /// setLegalizeVectorElementToDifferentSizeStrategy( - /// G_ADD, 0, moreToWiderTypesAndLessToWidest); - /// will result in the following getAction results: - /// * getAction({G_ADD, LLT::vector(8,8)}) returns - /// (Legal, vector(8,8)). - /// * getAction({G_ADD, LLT::vector(9,8)}) returns - /// (MoreElements, vector(16,8)). - /// * getAction({G_ADD, LLT::vector(8,32)}) returns - /// (FewerElements, vector(4,32)). - static SizeAndActionsVec - moreToWiderTypesAndLessToWidest(const SizeAndActionsVec &v) { - using namespace LegalizeActions; - return increaseToLargerTypesAndDecreaseToLargest(v, MoreElements, - FewerElements); - } - - /// Helper function to implement many typical SizeChangeStrategy functions. - static SizeAndActionsVec - increaseToLargerTypesAndDecreaseToLargest(const SizeAndActionsVec &v, - LegalizeAction IncreaseAction, - LegalizeAction DecreaseAction); - /// Helper function to implement many typical SizeChangeStrategy functions. - static SizeAndActionsVec - decreaseToSmallerTypesAndIncreaseToSmallest(const SizeAndActionsVec &v, - LegalizeAction DecreaseAction, - LegalizeAction IncreaseAction); - /// Get the action definitions for the given opcode. Use this to run a /// LegalityQuery through the definitions. const LegalizeRuleSet &getActionDefinitions(unsigned Opcode) const; @@ -1289,191 +1172,11 @@ public: virtual unsigned getExtOpcodeForWideningConstant(LLT SmallTy) const; private: - /// Determine what action should be taken to legalize the given generic - /// instruction opcode, type-index and type. Requires computeTables to have - /// been called. - /// - /// \returns a pair consisting of the kind of legalization that should be - /// performed and the destination type. - std::pair<LegalizeAction, LLT> - getAspectAction(const InstrAspect &Aspect) const; - - /// The SizeAndActionsVec is a representation mapping between all natural - /// numbers and an Action. The natural number represents the bit size of - /// the InstrAspect. For example, for a target with native support for 32-bit - /// and 64-bit additions, you'd express that as: - /// setScalarAction(G_ADD, 0, - /// {{1, WidenScalar}, // bit sizes [ 1, 31[ - /// {32, Legal}, // bit sizes [32, 33[ - /// {33, WidenScalar}, // bit sizes [33, 64[ - /// {64, Legal}, // bit sizes [64, 65[ - /// {65, NarrowScalar} // bit sizes [65, +inf[ - /// }); - /// It may be that only 64-bit pointers are supported on your target: - /// setPointerAction(G_PTR_ADD, 0, LLT:pointer(1), - /// {{1, Unsupported}, // bit sizes [ 1, 63[ - /// {64, Legal}, // bit sizes [64, 65[ - /// {65, Unsupported}, // bit sizes [65, +inf[ - /// }); - void setScalarAction(const unsigned Opcode, const unsigned TypeIndex, - const SizeAndActionsVec &SizeAndActions) { - const unsigned OpcodeIdx = Opcode - FirstOp; - SmallVector<SizeAndActionsVec, 1> &Actions = ScalarActions[OpcodeIdx]; - setActions(TypeIndex, Actions, SizeAndActions); - } - void setPointerAction(const unsigned Opcode, const unsigned TypeIndex, - const unsigned AddressSpace, - const SizeAndActionsVec &SizeAndActions) { - const unsigned OpcodeIdx = Opcode - FirstOp; - if (AddrSpace2PointerActions[OpcodeIdx].find(AddressSpace) == - AddrSpace2PointerActions[OpcodeIdx].end()) - AddrSpace2PointerActions[OpcodeIdx][AddressSpace] = {{}}; - SmallVector<SizeAndActionsVec, 1> &Actions = - AddrSpace2PointerActions[OpcodeIdx].find(AddressSpace)->second; - setActions(TypeIndex, Actions, SizeAndActions); - } - - /// If an operation on a given vector type (say <M x iN>) isn't explicitly - /// specified, we proceed in 2 stages. First we legalize the underlying scalar - /// (so that there's at least one legal vector with that scalar), then we - /// adjust the number of elements in the vector so that it is legal. The - /// desired action in the first step is controlled by this function. - void setScalarInVectorAction(const unsigned Opcode, const unsigned TypeIndex, - const SizeAndActionsVec &SizeAndActions) { - unsigned OpcodeIdx = Opcode - FirstOp; - SmallVector<SizeAndActionsVec, 1> &Actions = - ScalarInVectorActions[OpcodeIdx]; - setActions(TypeIndex, Actions, SizeAndActions); - } - - /// See also setScalarInVectorAction. - /// This function let's you specify the number of elements in a vector that - /// are legal for a legal element size. - void setVectorNumElementAction(const unsigned Opcode, - const unsigned TypeIndex, - const unsigned ElementSize, - const SizeAndActionsVec &SizeAndActions) { - const unsigned OpcodeIdx = Opcode - FirstOp; - if (NumElements2Actions[OpcodeIdx].find(ElementSize) == - NumElements2Actions[OpcodeIdx].end()) - NumElements2Actions[OpcodeIdx][ElementSize] = {{}}; - SmallVector<SizeAndActionsVec, 1> &Actions = - NumElements2Actions[OpcodeIdx].find(ElementSize)->second; - setActions(TypeIndex, Actions, SizeAndActions); - } - - /// A partial SizeAndActionsVec potentially doesn't cover all bit sizes, - /// i.e. it's OK if it doesn't start from size 1. - static void checkPartialSizeAndActionsVector(const SizeAndActionsVec& v) { - using namespace LegalizeActions; -#ifndef NDEBUG - // The sizes should be in increasing order - int prev_size = -1; - for(auto SizeAndAction: v) { - assert(SizeAndAction.first > prev_size); - prev_size = SizeAndAction.first; - } - // - for every Widen action, there should be a larger bitsize that - // can be legalized towards (e.g. Legal, Lower, Libcall or Custom - // action). - // - for every Narrow action, there should be a smaller bitsize that - // can be legalized towards. - int SmallestNarrowIdx = -1; - int LargestWidenIdx = -1; - int SmallestLegalizableToSameSizeIdx = -1; - int LargestLegalizableToSameSizeIdx = -1; - for(size_t i=0; i<v.size(); ++i) { - switch (v[i].second) { - case FewerElements: - case NarrowScalar: - if (SmallestNarrowIdx == -1) - SmallestNarrowIdx = i; - break; - case WidenScalar: - case MoreElements: - LargestWidenIdx = i; - break; - case Unsupported: - break; - default: - if (SmallestLegalizableToSameSizeIdx == -1) - SmallestLegalizableToSameSizeIdx = i; - LargestLegalizableToSameSizeIdx = i; - } - } - if (SmallestNarrowIdx != -1) { - assert(SmallestLegalizableToSameSizeIdx != -1); - assert(SmallestNarrowIdx > SmallestLegalizableToSameSizeIdx); - } - if (LargestWidenIdx != -1) - assert(LargestWidenIdx < LargestLegalizableToSameSizeIdx); -#endif - } - - /// A full SizeAndActionsVec must cover all bit sizes, i.e. must start with - /// from size 1. - static void checkFullSizeAndActionsVector(const SizeAndActionsVec& v) { -#ifndef NDEBUG - // Data structure invariant: The first bit size must be size 1. - assert(v.size() >= 1); - assert(v[0].first == 1); - checkPartialSizeAndActionsVector(v); -#endif - } - - /// Sets actions for all bit sizes on a particular generic opcode, type - /// index and scalar or pointer type. - void setActions(unsigned TypeIndex, - SmallVector<SizeAndActionsVec, 1> &Actions, - const SizeAndActionsVec &SizeAndActions) { - checkFullSizeAndActionsVector(SizeAndActions); - if (Actions.size() <= TypeIndex) - Actions.resize(TypeIndex + 1); - Actions[TypeIndex] = SizeAndActions; - } - - static SizeAndAction findAction(const SizeAndActionsVec &Vec, - const uint32_t Size); - - /// Returns the next action needed to get the scalar or pointer type closer - /// to being legal - /// E.g. findLegalAction({G_REM, 13}) should return - /// (WidenScalar, 32). After that, findLegalAction({G_REM, 32}) will - /// probably be called, which should return (Lower, 32). - /// This is assuming the setScalarAction on G_REM was something like: - /// setScalarAction(G_REM, 0, - /// {{1, WidenScalar}, // bit sizes [ 1, 31[ - /// {32, Lower}, // bit sizes [32, 33[ - /// {33, NarrowScalar} // bit sizes [65, +inf[ - /// }); - std::pair<LegalizeAction, LLT> - findScalarLegalAction(const InstrAspect &Aspect) const; - - /// Returns the next action needed towards legalizing the vector type. - std::pair<LegalizeAction, LLT> - findVectorLegalAction(const InstrAspect &Aspect) const; - static const int FirstOp = TargetOpcode::PRE_ISEL_GENERIC_OPCODE_START; static const int LastOp = TargetOpcode::PRE_ISEL_GENERIC_OPCODE_END; - // Data structures used temporarily during construction of legality data: - using TypeMap = DenseMap<LLT, LegalizeAction>; - SmallVector<TypeMap, 1> SpecifiedActions[LastOp - FirstOp + 1]; - SmallVector<SizeChangeStrategy, 1> - ScalarSizeChangeStrategies[LastOp - FirstOp + 1]; - SmallVector<SizeChangeStrategy, 1> - VectorElementSizeChangeStrategies[LastOp - FirstOp + 1]; - bool TablesInitialized; - - // Data structures used by getAction: - SmallVector<SizeAndActionsVec, 1> ScalarActions[LastOp - FirstOp + 1]; - SmallVector<SizeAndActionsVec, 1> ScalarInVectorActions[LastOp - FirstOp + 1]; - std::unordered_map<uint16_t, SmallVector<SizeAndActionsVec, 1>> - AddrSpace2PointerActions[LastOp - FirstOp + 1]; - std::unordered_map<uint16_t, SmallVector<SizeAndActionsVec, 1>> - NumElements2Actions[LastOp - FirstOp + 1]; - LegalizeRuleSet RulesForOpcode[LastOp - FirstOp + 1]; + LegacyLegalizerInfo LegacyInfo; }; #ifndef NDEBUG diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LostDebugLocObserver.h b/llvm/include/llvm/CodeGen/GlobalISel/LostDebugLocObserver.h index cd2a871e9579..1667d0145b04 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/LostDebugLocObserver.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/LostDebugLocObserver.h @@ -5,9 +5,9 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// -// +/// \file /// Tracks DebugLocs between checkpoints and verifies that they are transferred. -// +/// //===----------------------------------------------------------------------===// #ifndef LLVM_CODEGEN_GLOBALISEL_LOSTDEBUGLOCOBSERVER_H #define LLVM_CODEGEN_GLOBALISEL_LOSTDEBUGLOCOBSERVER_H @@ -47,4 +47,4 @@ private: }; } // namespace llvm -#endif // ifndef LLVM_CODEGEN_GLOBALISEL_LOSTDEBUGLOCOBSERVER_H +#endif // LLVM_CODEGEN_GLOBALISEL_LOSTDEBUGLOCOBSERVER_H diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h b/llvm/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h index 55d6d365fbb4..4c6b47ab9bc8 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h @@ -5,12 +5,13 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// -// +/// \file /// Contains matchers for matching SSA Machine Instructions. -// +/// //===----------------------------------------------------------------------===// -#ifndef LLVM_GMIR_PATTERNMATCH_H -#define LLVM_GMIR_PATTERNMATCH_H + +#ifndef LLVM_CODEGEN_GLOBALISEL_MIPATTERNMATCH_H +#define LLVM_CODEGEN_GLOBALISEL_MIPATTERNMATCH_H #include "llvm/CodeGen/GlobalISel/Utils.h" #include "llvm/CodeGen/MachineRegisterInfo.h" @@ -24,6 +25,11 @@ bool mi_match(Reg R, const MachineRegisterInfo &MRI, Pattern &&P) { return P.match(MRI, R); } +template <typename Pattern> +bool mi_match(MachineInstr &MI, const MachineRegisterInfo &MRI, Pattern &&P) { + return P.match(MRI, &MI); +} + // TODO: Extend for N use. template <typename SubPatternT> struct OneUse_match { SubPatternT SubPat; @@ -67,6 +73,22 @@ struct ConstantMatch { inline ConstantMatch m_ICst(int64_t &Cst) { return ConstantMatch(Cst); } +struct ICstRegMatch { + Register &CR; + ICstRegMatch(Register &C) : CR(C) {} + bool match(const MachineRegisterInfo &MRI, Register Reg) { + if (auto MaybeCst = getConstantVRegValWithLookThrough( + Reg, MRI, /*LookThroughInstrs*/ true, + /*HandleFConstants*/ false)) { + CR = MaybeCst->VReg; + return true; + } + return false; + } +}; + +inline ICstRegMatch m_ICst(Register &Reg) { return ICstRegMatch(Reg); } + /// Matcher for a specific constant value. struct SpecificConstantMatch { int64_t RequestedVal; @@ -165,6 +187,11 @@ template <> struct bind_helper<MachineInstr *> { return true; return false; } + static bool bind(const MachineRegisterInfo &MRI, MachineInstr *&MI, + MachineInstr *Inst) { + MI = Inst; + return MI; + } }; template <> struct bind_helper<LLT> { @@ -228,6 +255,43 @@ struct BinaryOp_match { } }; +// Helper for (commutative) binary generic MI that checks Opcode. +template <typename LHS_P, typename RHS_P, bool Commutable = false> +struct BinaryOpc_match { + unsigned Opc; + LHS_P L; + RHS_P R; + + BinaryOpc_match(unsigned Opcode, const LHS_P &LHS, const RHS_P &RHS) + : Opc(Opcode), L(LHS), R(RHS) {} + template <typename OpTy> + bool match(const MachineRegisterInfo &MRI, OpTy &&Op) { + MachineInstr *TmpMI; + if (mi_match(Op, MRI, m_MInstr(TmpMI))) { + if (TmpMI->getOpcode() == Opc && TmpMI->getNumDefs() == 1 && + TmpMI->getNumOperands() == 3) { + return (L.match(MRI, TmpMI->getOperand(1).getReg()) && + R.match(MRI, TmpMI->getOperand(2).getReg())) || + (Commutable && (R.match(MRI, TmpMI->getOperand(1).getReg()) && + L.match(MRI, TmpMI->getOperand(2).getReg()))); + } + } + return false; + } +}; + +template <typename LHS, typename RHS> +inline BinaryOpc_match<LHS, RHS, false> m_BinOp(unsigned Opcode, const LHS &L, + const RHS &R) { + return BinaryOpc_match<LHS, RHS, false>(Opcode, L, R); +} + +template <typename LHS, typename RHS> +inline BinaryOpc_match<LHS, RHS, true> +m_CommutativeBinOp(unsigned Opcode, const LHS &L, const RHS &R) { + return BinaryOpc_match<LHS, RHS, true>(Opcode, L, R); +} + template <typename LHS, typename RHS> inline BinaryOp_match<LHS, RHS, TargetOpcode::G_ADD, true> m_GAdd(const LHS &L, const RHS &R) { @@ -306,6 +370,18 @@ m_GAShr(const LHS &L, const RHS &R) { return BinaryOp_match<LHS, RHS, TargetOpcode::G_ASHR, false>(L, R); } +template <typename LHS, typename RHS> +inline BinaryOp_match<LHS, RHS, TargetOpcode::G_SMAX, false> +m_GSMax(const LHS &L, const RHS &R) { + return BinaryOp_match<LHS, RHS, TargetOpcode::G_SMAX, false>(L, R); +} + +template <typename LHS, typename RHS> +inline BinaryOp_match<LHS, RHS, TargetOpcode::G_SMIN, false> +m_GSMin(const LHS &L, const RHS &R) { + return BinaryOp_match<LHS, RHS, TargetOpcode::G_SMIN, false>(L, R); +} + // Helper for unary instructions (G_[ZSA]EXT/G_TRUNC) etc template <typename SrcTy, unsigned Opcode> struct UnaryOp_match { SrcTy L; @@ -468,6 +544,13 @@ m_GInsertVecElt(const Src0Ty &Src0, const Src1Ty &Src1, const Src2Ty &Src2) { TargetOpcode::G_INSERT_VECTOR_ELT>(Src0, Src1, Src2); } +template <typename Src0Ty, typename Src1Ty, typename Src2Ty> +inline TernaryOp_match<Src0Ty, Src1Ty, Src2Ty, TargetOpcode::G_SELECT> +m_GISelect(const Src0Ty &Src0, const Src1Ty &Src1, const Src2Ty &Src2) { + return TernaryOp_match<Src0Ty, Src1Ty, Src2Ty, TargetOpcode::G_SELECT>( + Src0, Src1, Src2); +} + /// Matches a register negated by a G_SUB. /// G_SUB 0, %negated_reg template <typename SrcTy> @@ -484,7 +567,7 @@ m_Not(const SrcTy &&Src) { return m_GXor(Src, m_AllOnesInt()); } -} // namespace GMIPatternMatch +} // namespace MIPatternMatch } // namespace llvm #endif diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h index 1ab4cd704824..9b652d8e16bc 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h @@ -205,14 +205,6 @@ private: SrcType Ty; }; -class FlagsOp { - Optional<unsigned> Flags; - -public: - explicit FlagsOp(unsigned F) : Flags(F) {} - FlagsOp() : Flags(None) {} - Optional<unsigned> getFlags() const { return Flags; } -}; /// Helper class to build MachineInstr. /// It keeps internally the insertion point and debug location for all /// the new instructions we want to create. @@ -363,7 +355,7 @@ public: void setDebugLoc(const DebugLoc &DL) { this->State.DL = DL; } /// Get the current instruction's debug location. - DebugLoc getDebugLoc() { return State.DL; } + const DebugLoc &getDebugLoc() { return State.DL; } /// Build and insert <empty> = \p Opcode <empty>. /// The insertion point is the one set by the last call of either @@ -712,6 +704,12 @@ public: MachineInstrBuilder buildExtOrTrunc(unsigned ExtOpc, const DstOp &Res, const SrcOp &Op); + /// Build and inserts \p Res = \p G_AND \p Op, \p LowBitsSet(ImmOp) + /// Since there is no G_ZEXT_INREG like G_SEXT_INREG, the instruction is + /// emulated using G_AND. + MachineInstrBuilder buildZExtInReg(const DstOp &Res, const SrcOp &Op, + int64_t ImmOp); + /// Build and insert an appropriate cast between two registers of equal size. MachineInstrBuilder buildCast(const DstOp &Dst, const SrcOp &Src); @@ -810,6 +808,18 @@ public: /// \return a MachineInstrBuilder for the newly created instruction. MachineInstrBuilder buildCopy(const DstOp &Res, const SrcOp &Op); + /// Build and insert \p Res = G_ASSERT_ZEXT Op, Size + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildAssertZExt(const DstOp &Res, const SrcOp &Op, + unsigned Size); + + /// Build and insert \p Res = G_ASSERT_SEXT Op, Size + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildAssertSExt(const DstOp &Res, const SrcOp &Op, + unsigned Size); + /// Build and insert `Res = G_LOAD Addr, MMO`. /// /// Loads the value stored at \p Addr. Puts the result in \p Res. @@ -1426,6 +1436,13 @@ public: return buildInstr(TargetOpcode::G_SMULH, {Dst}, {Src0, Src1}, Flags); } + /// Build and insert \p Res = G_UREM \p Op0, \p Op1 + MachineInstrBuilder buildURem(const DstOp &Dst, const SrcOp &Src0, + const SrcOp &Src1, + Optional<unsigned> Flags = None) { + return buildInstr(TargetOpcode::G_UREM, {Dst}, {Src0, Src1}, Flags); + } + MachineInstrBuilder buildFMul(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, Optional<unsigned> Flags = None) { @@ -1501,8 +1518,9 @@ public: /// /// \return a MachineInstrBuilder for the newly created instruction. MachineInstrBuilder buildOr(const DstOp &Dst, const SrcOp &Src0, - const SrcOp &Src1) { - return buildInstr(TargetOpcode::G_OR, {Dst}, {Src0, Src1}); + const SrcOp &Src1, + Optional<unsigned> Flags = None) { + return buildInstr(TargetOpcode::G_OR, {Dst}, {Src0, Src1}, Flags); } /// Build and insert \p Res = G_XOR \p Op0, \p Op1 @@ -1797,6 +1815,56 @@ public: MachineInstrBuilder buildVecReduceUMin(const DstOp &Dst, const SrcOp &Src) { return buildInstr(TargetOpcode::G_VECREDUCE_UMIN, {Dst}, {Src}); } + + /// Build and insert G_MEMCPY or G_MEMMOVE + MachineInstrBuilder buildMemTransferInst(unsigned Opcode, const SrcOp &DstPtr, + const SrcOp &SrcPtr, + const SrcOp &Size, + MachineMemOperand &DstMMO, + MachineMemOperand &SrcMMO) { + auto MIB = buildInstr( + Opcode, {}, {DstPtr, SrcPtr, Size, SrcOp(INT64_C(0) /*isTailCall*/)}); + MIB.addMemOperand(&DstMMO); + MIB.addMemOperand(&SrcMMO); + return MIB; + } + + MachineInstrBuilder buildMemCpy(const SrcOp &DstPtr, const SrcOp &SrcPtr, + const SrcOp &Size, MachineMemOperand &DstMMO, + MachineMemOperand &SrcMMO) { + return buildMemTransferInst(TargetOpcode::G_MEMCPY, DstPtr, SrcPtr, Size, + DstMMO, SrcMMO); + } + + /// Build and insert \p Dst = G_SBFX \p Src, \p LSB, \p Width. + MachineInstrBuilder buildSbfx(const DstOp &Dst, const SrcOp &Src, + const SrcOp &LSB, const SrcOp &Width) { + return buildInstr(TargetOpcode::G_SBFX, {Dst}, {Src, LSB, Width}); + } + + /// Build and insert \p Dst = G_UBFX \p Src, \p LSB, \p Width. + MachineInstrBuilder buildUbfx(const DstOp &Dst, const SrcOp &Src, + const SrcOp &LSB, const SrcOp &Width) { + return buildInstr(TargetOpcode::G_UBFX, {Dst}, {Src, LSB, Width}); + } + + /// Build and insert \p Dst = G_ROTR \p Src, \p Amt + MachineInstrBuilder buildRotateRight(const DstOp &Dst, const SrcOp &Src, + const SrcOp &Amt) { + return buildInstr(TargetOpcode::G_ROTR, {Dst}, {Src, Amt}); + } + + /// Build and insert \p Dst = G_ROTL \p Src, \p Amt + MachineInstrBuilder buildRotateLeft(const DstOp &Dst, const SrcOp &Src, + const SrcOp &Amt) { + return buildInstr(TargetOpcode::G_ROTL, {Dst}, {Src, Amt}); + } + + /// Build and insert \p Dst = G_BITREVERSE \p Src + MachineInstrBuilder buildBitReverse(const DstOp &Dst, const SrcOp &Src) { + return buildInstr(TargetOpcode::G_BITREVERSE, {Dst}, {Src}); + } + virtual MachineInstrBuilder buildInstr(unsigned Opc, ArrayRef<DstOp> DstOps, ArrayRef<SrcOp> SrcOps, Optional<unsigned> Flags = None); diff --git a/llvm/include/llvm/CodeGen/GlobalISel/RegBankSelect.h b/llvm/include/llvm/CodeGen/GlobalISel/RegBankSelect.h index d9d076ba312c..5c693d8de521 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/RegBankSelect.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/RegBankSelect.h @@ -6,9 +6,10 @@ // //===----------------------------------------------------------------------===// // -/// \file This file describes the interface of the MachineFunctionPass +/// \file +/// This file describes the interface of the MachineFunctionPass /// responsible for assigning the generic virtual registers to register bank. - +/// /// By default, the reg bank selector relies on local decisions to /// assign the register bank. In other words, it looks at one instruction /// at a time to decide where the operand of that instruction should live. diff --git a/llvm/include/llvm/CodeGen/GlobalISel/RegisterBank.h b/llvm/include/llvm/CodeGen/GlobalISel/RegisterBank.h index f528d1a46012..5440d97728b4 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/RegisterBank.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/RegisterBank.h @@ -10,8 +10,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_CODEGEN_GLOBALISEL_REGBANK_H -#define LLVM_CODEGEN_GLOBALISEL_REGBANK_H +#ifndef LLVM_CODEGEN_GLOBALISEL_REGISTERBANK_H +#define LLVM_CODEGEN_GLOBALISEL_REGISTERBANK_H #include "llvm/ADT/BitVector.h" diff --git a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h index 68553ab5b1a8..818475a48abb 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h @@ -15,6 +15,7 @@ #define LLVM_CODEGEN_GLOBALISEL_UTILS_H #include "llvm/ADT/StringRef.h" +#include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/Register.h" #include "llvm/Support/Alignment.h" #include "llvm/Support/LowLevelTypeImpl.h" @@ -23,6 +24,7 @@ namespace llvm { class AnalysisUsage; +class BlockFrequencyInfo; class GISelKnownBits; class MachineFunction; class MachineInstr; @@ -32,15 +34,50 @@ class MachineOptimizationRemarkMissed; struct MachinePointerInfo; class MachineRegisterInfo; class MCInstrDesc; +class ProfileSummaryInfo; class RegisterBankInfo; class TargetInstrInfo; class TargetLowering; class TargetPassConfig; class TargetRegisterInfo; class TargetRegisterClass; +class ConstantInt; class ConstantFP; class APFloat; +// Convenience macros for dealing with vector reduction opcodes. +#define GISEL_VECREDUCE_CASES_ALL \ + case TargetOpcode::G_VECREDUCE_SEQ_FADD: \ + case TargetOpcode::G_VECREDUCE_SEQ_FMUL: \ + case TargetOpcode::G_VECREDUCE_FADD: \ + case TargetOpcode::G_VECREDUCE_FMUL: \ + case TargetOpcode::G_VECREDUCE_FMAX: \ + case TargetOpcode::G_VECREDUCE_FMIN: \ + case TargetOpcode::G_VECREDUCE_ADD: \ + case TargetOpcode::G_VECREDUCE_MUL: \ + case TargetOpcode::G_VECREDUCE_AND: \ + case TargetOpcode::G_VECREDUCE_OR: \ + case TargetOpcode::G_VECREDUCE_XOR: \ + case TargetOpcode::G_VECREDUCE_SMAX: \ + case TargetOpcode::G_VECREDUCE_SMIN: \ + case TargetOpcode::G_VECREDUCE_UMAX: \ + case TargetOpcode::G_VECREDUCE_UMIN: + +#define GISEL_VECREDUCE_CASES_NONSEQ \ + case TargetOpcode::G_VECREDUCE_FADD: \ + case TargetOpcode::G_VECREDUCE_FMUL: \ + case TargetOpcode::G_VECREDUCE_FMAX: \ + case TargetOpcode::G_VECREDUCE_FMIN: \ + case TargetOpcode::G_VECREDUCE_ADD: \ + case TargetOpcode::G_VECREDUCE_MUL: \ + case TargetOpcode::G_VECREDUCE_AND: \ + case TargetOpcode::G_VECREDUCE_OR: \ + case TargetOpcode::G_VECREDUCE_XOR: \ + case TargetOpcode::G_VECREDUCE_SMAX: \ + case TargetOpcode::G_VECREDUCE_SMIN: \ + case TargetOpcode::G_VECREDUCE_UMAX: \ + case TargetOpcode::G_VECREDUCE_UMIN: + /// Try to constrain Reg to the specified register class. If this fails, /// create a new virtual register in the correct class. /// @@ -153,6 +190,8 @@ getConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs = true, bool HandleFConstants = true, bool LookThroughAnyExt = false); +const ConstantInt *getConstantIntVRegVal(Register VReg, + const MachineRegisterInfo &MRI); const ConstantFP* getConstantFPVRegVal(Register VReg, const MachineRegisterInfo &MRI); @@ -171,11 +210,15 @@ struct DefinitionAndSourceRegister { /// Find the def instruction for \p Reg, and underlying value Register folding /// away any copies. +/// +/// Also walks through hints such as G_ASSERT_ZEXT. Optional<DefinitionAndSourceRegister> getDefSrcRegIgnoringCopies(Register Reg, const MachineRegisterInfo &MRI); /// Find the def instruction for \p Reg, folding away any trivial copies. May /// return nullptr if \p Reg is not a generic virtual register. +/// +/// Also walks through hints such as G_ASSERT_ZEXT. MachineInstr *getDefIgnoringCopies(Register Reg, const MachineRegisterInfo &MRI); @@ -183,8 +226,19 @@ MachineInstr *getDefIgnoringCopies(Register Reg, /// will be an output register of the instruction that getDefIgnoringCopies /// returns. May return an invalid register if \p Reg is not a generic virtual /// register. -Register getSrcRegIgnoringCopies(Register Reg, - const MachineRegisterInfo &MRI); +/// +/// Also walks through hints such as G_ASSERT_ZEXT. +Register getSrcRegIgnoringCopies(Register Reg, const MachineRegisterInfo &MRI); + +// Templated variant of getOpcodeDef returning a MachineInstr derived T. +/// See if Reg is defined by an single def instruction of type T +/// Also try to do trivial folding if it's a COPY with +/// same types. Returns null otherwise. +template <class T> +T *getOpcodeDef(Register Reg, const MachineRegisterInfo &MRI) { + MachineInstr *DefMI = getDefIgnoringCopies(Reg, MRI); + return dyn_cast_or_null<T>(DefMI); +} /// Returns an APFloat from Val converted to the appropriate size. APFloat getAPFloatFromSize(double Val, unsigned Size); @@ -196,10 +250,17 @@ void getSelectionDAGFallbackAnalysisUsage(AnalysisUsage &AU); Optional<APInt> ConstantFoldBinOp(unsigned Opcode, const Register Op1, const Register Op2, const MachineRegisterInfo &MRI); +Optional<APFloat> ConstantFoldFPBinOp(unsigned Opcode, const Register Op1, + const Register Op2, + const MachineRegisterInfo &MRI); Optional<APInt> ConstantFoldExtOp(unsigned Opcode, const Register Op1, uint64_t Imm, const MachineRegisterInfo &MRI); +Optional<APFloat> ConstantFoldIntToFloat(unsigned Opcode, LLT DstTy, + Register Src, + const MachineRegisterInfo &MRI); + /// Test if the given value is known to have exactly one bit set. This differs /// from computeKnownBits in that it doesn't necessarily determine which bit is /// set. @@ -252,6 +313,31 @@ LLT getLCMType(LLT OrigTy, LLT TargetTy); LLVM_READNONE LLT getGCDType(LLT OrigTy, LLT TargetTy); +/// Represents a value which can be a Register or a constant. +/// +/// This is useful in situations where an instruction may have an interesting +/// register operand or interesting constant operand. For a concrete example, +/// \see getVectorSplat. +class RegOrConstant { + int64_t Cst; + Register Reg; + bool IsReg; + +public: + explicit RegOrConstant(Register Reg) : Reg(Reg), IsReg(true) {} + explicit RegOrConstant(int64_t Cst) : Cst(Cst), IsReg(false) {} + bool isReg() const { return IsReg; } + bool isCst() const { return !IsReg; } + Register getReg() const { + assert(isReg() && "Expected a register!"); + return Reg; + } + int64_t getCst() const { + assert(isCst() && "Expected a constant!"); + return Cst; + } +}; + /// \returns The splat index of a G_SHUFFLE_VECTOR \p MI when \p MI is a splat. /// If \p MI is not a splat, returns None. Optional<int> getSplatIndex(MachineInstr &MI); @@ -270,6 +356,35 @@ bool isBuildVectorAllZeros(const MachineInstr &MI, bool isBuildVectorAllOnes(const MachineInstr &MI, const MachineRegisterInfo &MRI); +/// \returns a value when \p MI is a vector splat. The splat can be either a +/// Register or a constant. +/// +/// Examples: +/// +/// \code +/// %reg = COPY $physreg +/// %reg_splat = G_BUILD_VECTOR %reg, %reg, ..., %reg +/// \endcode +/// +/// If called on the G_BUILD_VECTOR above, this will return a RegOrConstant +/// containing %reg. +/// +/// \code +/// %cst = G_CONSTANT iN 4 +/// %constant_splat = G_BUILD_VECTOR %cst, %cst, ..., %cst +/// \endcode +/// +/// In the above case, this will return a RegOrConstant containing 4. +Optional<RegOrConstant> getVectorSplat(const MachineInstr &MI, + const MachineRegisterInfo &MRI); + +/// Attempt to match a unary predicate against a scalar/splat constant or every +/// element of a constant G_BUILD_VECTOR. If \p ConstVal is null, the source +/// value was undef. +bool matchUnaryPredicate(const MachineRegisterInfo &MRI, Register Reg, + std::function<bool(const Constant *ConstVal)> Match, + bool AllowUndefs = false); + /// Returns true if given the TargetLowering's boolean contents information, /// the value \p Val contains a true value. bool isConstTrueVal(const TargetLowering &TLI, int64_t Val, bool IsVector, @@ -278,5 +393,10 @@ bool isConstTrueVal(const TargetLowering &TLI, int64_t Val, bool IsVector, /// Returns an integer representing true, as defined by the /// TargetBooleanContents. int64_t getICmpTrueVal(const TargetLowering &TLI, bool IsVector, bool IsFP); + +/// Returns true if the given block should be optimized for size. +bool shouldOptForSize(const MachineBasicBlock &MBB, ProfileSummaryInfo *PSI, + BlockFrequencyInfo *BFI); + } // End namespace llvm. #endif |