diff options
Diffstat (limited to 'contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp')
| -rw-r--r-- | contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 946 |
1 files changed, 593 insertions, 353 deletions
diff --git a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 592c09c10fb0..2090762e2ff4 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -28,6 +28,7 @@ #include "llvm/Analysis/MemoryLocation.h" #include "llvm/Analysis/ProfileSummaryInfo.h" #include "llvm/Analysis/ValueTracking.h" +#include "llvm/CodeGen/FunctionLoweringInfo.h" #include "llvm/CodeGen/ISDOpcodes.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineConstantPool.h" @@ -138,6 +139,15 @@ bool ConstantFPSDNode::isValueValidForType(EVT VT, //===----------------------------------------------------------------------===// bool ISD::isConstantSplatVector(const SDNode *N, APInt &SplatVal) { + if (N->getOpcode() == ISD::SPLAT_VECTOR) { + unsigned EltSize = + N->getValueType(0).getVectorElementType().getSizeInBits(); + if (auto *Op0 = dyn_cast<ConstantSDNode>(N->getOperand(0))) { + SplatVal = Op0->getAPIntValue().truncOrSelf(EltSize); + return true; + } + } + auto *BV = dyn_cast<BuildVectorSDNode>(N); if (!BV) return false; @@ -154,11 +164,16 @@ bool ISD::isConstantSplatVector(const SDNode *N, APInt &SplatVal) { // FIXME: AllOnes and AllZeros duplicate a lot of code. Could these be // specializations of the more general isConstantSplatVector()? -bool ISD::isBuildVectorAllOnes(const SDNode *N) { +bool ISD::isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly) { // Look through a bit convert. while (N->getOpcode() == ISD::BITCAST) N = N->getOperand(0).getNode(); + if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) { + APInt SplatVal; + return isConstantSplatVector(N, SplatVal) && SplatVal.isAllOnesValue(); + } + if (N->getOpcode() != ISD::BUILD_VECTOR) return false; unsigned i = 0, e = N->getNumOperands(); @@ -198,11 +213,16 @@ bool ISD::isBuildVectorAllOnes(const SDNode *N) { return true; } -bool ISD::isBuildVectorAllZeros(const SDNode *N) { +bool ISD::isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly) { // Look through a bit convert. while (N->getOpcode() == ISD::BITCAST) N = N->getOperand(0).getNode(); + if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) { + APInt SplatVal; + return isConstantSplatVector(N, SplatVal) && SplatVal.isNullValue(); + } + if (N->getOpcode() != ISD::BUILD_VECTOR) return false; bool IsAllUndef = true; @@ -235,6 +255,14 @@ bool ISD::isBuildVectorAllZeros(const SDNode *N) { return true; } +bool ISD::isBuildVectorAllOnes(const SDNode *N) { + return isConstantSplatVectorAllOnes(N, /*BuildVectorOnly*/ true); +} + +bool ISD::isBuildVectorAllZeros(const SDNode *N) { + return isConstantSplatVectorAllZeros(N, /*BuildVectorOnly*/ true); +} + bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) { if (N->getOpcode() != ISD::BUILD_VECTOR) return false; @@ -278,7 +306,8 @@ bool ISD::matchUnaryPredicate(SDValue Op, return Match(Cst); // FIXME: Add support for vector UNDEF cases? - if (ISD::BUILD_VECTOR != Op.getOpcode()) + if (ISD::BUILD_VECTOR != Op.getOpcode() && + ISD::SPLAT_VECTOR != Op.getOpcode()) return false; EVT SVT = Op.getValueType().getScalarType(); @@ -332,6 +361,76 @@ bool ISD::matchBinaryPredicate( return true; } +ISD::NodeType ISD::getVecReduceBaseOpcode(unsigned VecReduceOpcode) { + switch (VecReduceOpcode) { + default: + llvm_unreachable("Expected VECREDUCE opcode"); + case ISD::VECREDUCE_FADD: + case ISD::VECREDUCE_SEQ_FADD: + return ISD::FADD; + case ISD::VECREDUCE_FMUL: + case ISD::VECREDUCE_SEQ_FMUL: + return ISD::FMUL; + case ISD::VECREDUCE_ADD: + return ISD::ADD; + case ISD::VECREDUCE_MUL: + return ISD::MUL; + case ISD::VECREDUCE_AND: + return ISD::AND; + case ISD::VECREDUCE_OR: + return ISD::OR; + case ISD::VECREDUCE_XOR: + return ISD::XOR; + case ISD::VECREDUCE_SMAX: + return ISD::SMAX; + case ISD::VECREDUCE_SMIN: + return ISD::SMIN; + case ISD::VECREDUCE_UMAX: + return ISD::UMAX; + case ISD::VECREDUCE_UMIN: + return ISD::UMIN; + case ISD::VECREDUCE_FMAX: + return ISD::FMAXNUM; + case ISD::VECREDUCE_FMIN: + return ISD::FMINNUM; + } +} + +bool ISD::isVPOpcode(unsigned Opcode) { + switch (Opcode) { + default: + return false; +#define BEGIN_REGISTER_VP_SDNODE(SDOPC, ...) \ + case ISD::SDOPC: \ + return true; +#include "llvm/IR/VPIntrinsics.def" + } +} + +/// The operand position of the vector mask. +Optional<unsigned> ISD::getVPMaskIdx(unsigned Opcode) { + switch (Opcode) { + default: + return None; +#define BEGIN_REGISTER_VP_SDNODE(SDOPC, LEGALPOS, TDNAME, MASKPOS, ...) \ + case ISD::SDOPC: \ + return MASKPOS; +#include "llvm/IR/VPIntrinsics.def" + } +} + +/// The operand position of the explicit vector length parameter. +Optional<unsigned> ISD::getVPExplicitVectorLengthIdx(unsigned Opcode) { + switch (Opcode) { + default: + return None; +#define BEGIN_REGISTER_VP_SDNODE(SDOPC, LEGALPOS, TDNAME, MASKPOS, EVLPOS) \ + case ISD::SDOPC: \ + return EVLPOS; +#include "llvm/IR/VPIntrinsics.def" + } +} + ISD::NodeType ISD::getExtForLoadExtType(bool IsFP, ISD::LoadExtType ExtType) { switch (ExtType) { case ISD::EXTLOAD: @@ -536,6 +635,11 @@ static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) { ID.AddInteger(cast<LifetimeSDNode>(N)->getOffset()); } break; + case ISD::PSEUDO_PROBE: + ID.AddInteger(cast<PseudoProbeSDNode>(N)->getGuid()); + ID.AddInteger(cast<PseudoProbeSDNode>(N)->getIndex()); + ID.AddInteger(cast<PseudoProbeSDNode>(N)->getAttributes()); + break; case ISD::JumpTable: case ISD::TargetJumpTable: ID.AddInteger(cast<JumpTableSDNode>(N)->getIndex()); @@ -1229,7 +1333,7 @@ SDValue SelectionDAG::getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isT, bool isO) { EVT EltVT = VT.getScalarType(); assert((EltVT.getSizeInBits() >= 64 || - (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) && + (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) && "getConstant with a uint64_t value that doesn't fit in the type!"); return getConstant(APInt(EltVT.getSizeInBits(), Val), DL, VT, isT, isO); } @@ -1251,10 +1355,10 @@ SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL, // inserted value (the type does not need to match the vector element type). // Any extra bits introduced will be truncated away. if (VT.isVector() && TLI->getTypeAction(*getContext(), EltVT) == - TargetLowering::TypePromoteInteger) { - EltVT = TLI->getTypeToTransformTo(*getContext(), EltVT); - APInt NewVal = Elt->getValue().zextOrTrunc(EltVT.getSizeInBits()); - Elt = ConstantInt::get(*getContext(), NewVal); + TargetLowering::TypePromoteInteger) { + EltVT = TLI->getTypeToTransformTo(*getContext(), EltVT); + APInt NewVal = Elt->getValue().zextOrTrunc(EltVT.getSizeInBits()); + Elt = ConstantInt::get(*getContext(), NewVal); } // In other cases the element type is illegal and needs to be expanded, for // example v2i64 on MIPS32. In this case, find the nearest legal type, split @@ -1264,7 +1368,7 @@ SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL, // only legalize if the DAG tells us we must produce legal types. else if (NewNodesMustHaveLegalTypes && VT.isVector() && TLI->getTypeAction(*getContext(), EltVT) == - TargetLowering::TypeExpandInteger) { + TargetLowering::TypeExpandInteger) { const APInt &NewVal = Elt->getValue(); EVT ViaEltVT = TLI->getTypeToTransformTo(*getContext(), EltVT); unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits(); @@ -1278,9 +1382,9 @@ SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL, SmallVector<SDValue, 2> EltParts; for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i) { - EltParts.push_back(getConstant(NewVal.lshr(i * ViaEltSizeInBits) - .zextOrTrunc(ViaEltSizeInBits), DL, - ViaEltVT, isT, isO)); + EltParts.push_back(getConstant( + NewVal.lshr(i * ViaEltSizeInBits).zextOrTrunc(ViaEltSizeInBits), DL, + ViaEltVT, isT, isO)); } // EltParts is currently in little endian order. If we actually want @@ -1297,9 +1401,10 @@ SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL, SmallVector<SDValue, 8> Ops; for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) - Ops.insert(Ops.end(), EltParts.begin(), EltParts.end()); + llvm::append_range(Ops, EltParts); - SDValue V = getNode(ISD::BITCAST, DL, VT, getBuildVector(ViaVecVT, DL, Ops)); + SDValue V = + getNode(ISD::BITCAST, DL, VT, getBuildVector(ViaVecVT, DL, Ops)); return V; } @@ -1380,7 +1485,9 @@ SDValue SelectionDAG::getConstantFP(const ConstantFP &V, const SDLoc &DL, } SDValue Result(N, 0); - if (VT.isVector()) + if (VT.isScalableVector()) + Result = getSplatVector(VT, DL, Result); + else if (VT.isVector()) Result = getSplatBuildVector(VT, DL, Result); NewSDValueDbgMsg(Result, "Creating fp constant: ", this); return Result; @@ -2023,7 +2130,14 @@ Align SelectionDAG::getReducedAlign(EVT VT, bool UseABI) { SDValue SelectionDAG::CreateStackTemporary(TypeSize Bytes, Align Alignment) { MachineFrameInfo &MFI = MF->getFrameInfo(); - int FrameIdx = MFI.CreateStackObject(Bytes, Alignment, false); + const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering(); + int StackID = 0; + if (Bytes.isScalable()) + StackID = TFI->getStackIDForScalableVectors(); + // The stack id gives an indication of whether the object is scalable or + // not, so it's safe to pass in the minimum size here. + int FrameIdx = MFI.CreateStackObject(Bytes.getKnownMinSize(), Alignment, + false, nullptr, StackID); return getFrameIndex(FrameIdx, TLI->getFrameIndexTy(getDataLayout())); } @@ -2035,7 +2149,14 @@ SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) { } SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) { - TypeSize Bytes = std::max(VT1.getStoreSize(), VT2.getStoreSize()); + TypeSize VT1Size = VT1.getStoreSize(); + TypeSize VT2Size = VT2.getStoreSize(); + assert(VT1Size.isScalable() == VT2Size.isScalable() && + "Don't know how to choose the maximum size when creating a stack " + "temporary"); + TypeSize Bytes = + VT1Size.getKnownMinSize() > VT2Size.getKnownMinSize() ? VT1Size : VT2Size; + Type *Ty1 = VT1.getTypeForEVT(*getContext()); Type *Ty2 = VT2.getTypeForEVT(*getContext()); const DataLayout &DL = getDataLayout(); @@ -2204,6 +2325,10 @@ SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, SDValue N2, /// SimplifyMultipleUseDemandedBits and not generate any new nodes. SDValue SelectionDAG::GetDemandedBits(SDValue V, const APInt &DemandedBits) { EVT VT = V.getValueType(); + + if (VT.isScalableVector()) + return SDValue(); + APInt DemandedElts = VT.isVector() ? APInt::getAllOnesValue(VT.getVectorNumElements()) : APInt(1, 1); @@ -2221,7 +2346,6 @@ SDValue SelectionDAG::GetDemandedBits(SDValue V, const APInt &DemandedBits, default: return TLI->SimplifyMultipleUseDemandedBits(V, DemandedBits, DemandedElts, *this, 0); - break; case ISD::Constant: { const APInt &CVal = cast<ConstantSDNode>(V)->getAPIntValue(); APInt NewVal = CVal & DemandedBits; @@ -2247,18 +2371,6 @@ SDValue SelectionDAG::GetDemandedBits(SDValue V, const APInt &DemandedBits, V.getOperand(1)); } break; - case ISD::AND: { - // X & -1 -> X (ignoring bits which aren't demanded). - // Also handle the case where masked out bits in X are known to be zero. - if (ConstantSDNode *RHSC = isConstOrConstSplat(V.getOperand(1))) { - const APInt &AndVal = RHSC->getAPIntValue(); - if (DemandedBits.isSubsetOf(AndVal) || - DemandedBits.isSubsetOf(computeKnownBits(V.getOperand(0)).Zero | - AndVal)) - return V.getOperand(0); - } - break; - } } return SDValue(); } @@ -2298,17 +2410,23 @@ bool SelectionDAG::MaskedValueIsAllOnes(SDValue V, const APInt &Mask, /// sense to specify which elements are demanded or undefined, therefore /// they are simply ignored. bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts, - APInt &UndefElts) { + APInt &UndefElts, unsigned Depth) { EVT VT = V.getValueType(); assert(VT.isVector() && "Vector type expected"); if (!VT.isScalableVector() && !DemandedElts) return false; // No demanded elts, better to assume we don't know anything. + if (Depth >= MaxRecursionDepth) + return false; // Limit search depth. + // Deal with some common cases here that work for both fixed and scalable // vector types. switch (V.getOpcode()) { case ISD::SPLAT_VECTOR: + UndefElts = V.getOperand(0).isUndef() + ? APInt::getAllOnesValue(DemandedElts.getBitWidth()) + : APInt(DemandedElts.getBitWidth(), 0); return true; case ISD::ADD: case ISD::SUB: @@ -2316,13 +2434,17 @@ bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts, APInt UndefLHS, UndefRHS; SDValue LHS = V.getOperand(0); SDValue RHS = V.getOperand(1); - if (isSplatValue(LHS, DemandedElts, UndefLHS) && - isSplatValue(RHS, DemandedElts, UndefRHS)) { + if (isSplatValue(LHS, DemandedElts, UndefLHS, Depth + 1) && + isSplatValue(RHS, DemandedElts, UndefRHS, Depth + 1)) { UndefElts = UndefLHS | UndefRHS; return true; } break; } + case ISD::TRUNCATE: + case ISD::SIGN_EXTEND: + case ISD::ZERO_EXTEND: + return isSplatValue(V.getOperand(0), DemandedElts, UndefElts, Depth + 1); } // We don't support other cases than those above for scalable vectors at @@ -2377,7 +2499,7 @@ bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts, unsigned NumSrcElts = Src.getValueType().getVectorNumElements(); APInt UndefSrcElts; APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx); - if (isSplatValue(Src, DemandedSrcElts, UndefSrcElts)) { + if (isSplatValue(Src, DemandedSrcElts, UndefSrcElts, Depth + 1)) { UndefElts = UndefSrcElts.extractBits(NumElts, Idx); return true; } @@ -2574,15 +2696,11 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, if (auto *C = dyn_cast<ConstantSDNode>(Op)) { // We know all of the bits for a constant! - Known.One = C->getAPIntValue(); - Known.Zero = ~Known.One; - return Known; + return KnownBits::makeConstant(C->getAPIntValue()); } if (auto *C = dyn_cast<ConstantFPSDNode>(Op)) { // We know all of the bits for a constant fp! - Known.One = C->getValueAPF().bitcastToAPInt(); - Known.Zero = ~Known.One; - return Known; + return KnownBits::makeConstant(C->getValueAPF().bitcastToAPInt()); } if (Depth >= MaxRecursionDepth) @@ -2617,8 +2735,7 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, } // Known bits are the values that are shared by every demanded element. - Known.One &= Known2.One; - Known.Zero &= Known2.Zero; + Known = KnownBits::commonBits(Known, Known2); // If we don't know any bits, early out. if (Known.isUnknown()) @@ -2655,8 +2772,7 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, if (!!DemandedLHS) { SDValue LHS = Op.getOperand(0); Known2 = computeKnownBits(LHS, DemandedLHS, Depth + 1); - Known.One &= Known2.One; - Known.Zero &= Known2.Zero; + Known = KnownBits::commonBits(Known, Known2); } // If we don't know any bits, early out. if (Known.isUnknown()) @@ -2664,8 +2780,7 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, if (!!DemandedRHS) { SDValue RHS = Op.getOperand(1); Known2 = computeKnownBits(RHS, DemandedRHS, Depth + 1); - Known.One &= Known2.One; - Known.Zero &= Known2.Zero; + Known = KnownBits::commonBits(Known, Known2); } break; } @@ -2681,8 +2796,7 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, if (!!DemandedSub) { SDValue Sub = Op.getOperand(i); Known2 = computeKnownBits(Sub, DemandedSub, Depth + 1); - Known.One &= Known2.One; - Known.Zero &= Known2.Zero; + Known = KnownBits::commonBits(Known, Known2); } // If we don't know any bits, early out. if (Known.isUnknown()) @@ -2710,8 +2824,7 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, } if (!!DemandedSrcElts) { Known2 = computeKnownBits(Src, DemandedSrcElts, Depth + 1); - Known.One &= Known2.One; - Known.Zero &= Known2.Zero; + Known = KnownBits::commonBits(Known, Known2); } break; } @@ -2830,35 +2943,13 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, case ISD::MUL: { Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); - - // If low bits are zero in either operand, output low known-0 bits. - // Also compute a conservative estimate for high known-0 bits. - // More trickiness is possible, but this is sufficient for the - // interesting case of alignment computation. - unsigned TrailZ = Known.countMinTrailingZeros() + - Known2.countMinTrailingZeros(); - unsigned LeadZ = std::max(Known.countMinLeadingZeros() + - Known2.countMinLeadingZeros(), - BitWidth) - BitWidth; - - Known.resetAll(); - Known.Zero.setLowBits(std::min(TrailZ, BitWidth)); - Known.Zero.setHighBits(std::min(LeadZ, BitWidth)); + Known = KnownBits::computeForMul(Known, Known2); break; } case ISD::UDIV: { - // For the purposes of computing leading zeros we can conservatively - // treat a udiv as a logical right shift by the power of 2 known to - // be less than the denominator. - Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); - unsigned LeadZ = Known2.countMinLeadingZeros(); - + Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); - unsigned RHSMaxLeadingZeros = Known2.countMaxLeadingZeros(); - if (RHSMaxLeadingZeros != BitWidth) - LeadZ = std::min(BitWidth, LeadZ + BitWidth - RHSMaxLeadingZeros - 1); - - Known.Zero.setHighBits(LeadZ); + Known = KnownBits::udiv(Known, Known2); break; } case ISD::SELECT: @@ -2870,8 +2961,7 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth+1); // Only known if known in both the LHS and RHS. - Known.One &= Known2.One; - Known.Zero &= Known2.Zero; + Known = KnownBits::commonBits(Known, Known2); break; case ISD::SELECT_CC: Known = computeKnownBits(Op.getOperand(3), DemandedElts, Depth+1); @@ -2881,8 +2971,7 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, Known2 = computeKnownBits(Op.getOperand(2), DemandedElts, Depth+1); // Only known if known in both the LHS and RHS. - Known.One &= Known2.One; - Known.Zero &= Known2.Zero; + Known = KnownBits::commonBits(Known, Known2); break; case ISD::SMULO: case ISD::UMULO: @@ -2911,19 +3000,8 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, } case ISD::SHL: Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); - - if (const APInt *ShAmt = getValidShiftAmountConstant(Op, DemandedElts)) { - unsigned Shift = ShAmt->getZExtValue(); - Known.Zero <<= Shift; - Known.One <<= Shift; - // Low bits are known zero. - Known.Zero.setLowBits(Shift); - break; - } - - // No matter the shift amount, the trailing zeros will stay zero. - Known.Zero = APInt::getLowBitsSet(BitWidth, Known.countMinTrailingZeros()); - Known.One.clearAllBits(); + Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); + Known = KnownBits::shl(Known, Known2); // Minimum shift low bits are known zero. if (const APInt *ShMinAmt = @@ -2932,19 +3010,8 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, break; case ISD::SRL: Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); - - if (const APInt *ShAmt = getValidShiftAmountConstant(Op, DemandedElts)) { - unsigned Shift = ShAmt->getZExtValue(); - Known.Zero.lshrInPlace(Shift); - Known.One.lshrInPlace(Shift); - // High bits are known zero. - Known.Zero.setHighBits(Shift); - break; - } - - // No matter the shift amount, the leading zeros will stay zero. - Known.Zero = APInt::getHighBitsSet(BitWidth, Known.countMinLeadingZeros()); - Known.One.clearAllBits(); + Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); + Known = KnownBits::lshr(Known, Known2); // Minimum shift high bits are known zero. if (const APInt *ShMinAmt = @@ -2952,13 +3019,10 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, Known.Zero.setHighBits(ShMinAmt->getZExtValue()); break; case ISD::SRA: - if (const APInt *ShAmt = getValidShiftAmountConstant(Op, DemandedElts)) { - Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); - unsigned Shift = ShAmt->getZExtValue(); - // Sign extend known zero/one bit (else is unknown). - Known.Zero.ashrInPlace(Shift); - Known.One.ashrInPlace(Shift); - } + Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); + Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); + Known = KnownBits::ashr(Known, Known2); + // TODO: Add minimum shift high known sign bits. break; case ISD::FSHL: case ISD::FSHR: @@ -2993,38 +3057,9 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, } break; case ISD::SIGN_EXTEND_INREG: { - EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT(); - unsigned EBits = EVT.getScalarSizeInBits(); - - // Sign extension. Compute the demanded bits in the result that are not - // present in the input. - APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - EBits); - - APInt InSignMask = APInt::getSignMask(EBits); - APInt InputDemandedBits = APInt::getLowBitsSet(BitWidth, EBits); - - // If the sign extended bits are demanded, we know that the sign - // bit is demanded. - InSignMask = InSignMask.zext(BitWidth); - if (NewBits.getBoolValue()) - InputDemandedBits |= InSignMask; - Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); - Known.One &= InputDemandedBits; - Known.Zero &= InputDemandedBits; - - // If the sign bit of the input is known set or clear, then we know the - // top bits of the result. - if (Known.Zero.intersects(InSignMask)) { // Input sign bit known clear - Known.Zero |= NewBits; - Known.One &= ~NewBits; - } else if (Known.One.intersects(InSignMask)) { // Input sign bit known set - Known.One |= NewBits; - Known.Zero &= ~NewBits; - } else { // Input sign bit unknown - Known.Zero &= ~NewBits; - Known.One &= ~NewBits; - } + EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT(); + Known = Known.sextInReg(EVT.getScalarSizeInBits()); break; } case ISD::CTTZ: @@ -3052,6 +3087,11 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, Known.Zero.setBitsFrom(Log2_32(PossibleOnes) + 1); break; } + case ISD::PARITY: { + // Parity returns 0 everywhere but the LSB. + Known.Zero.setBitsFrom(1); + break; + } case ISD::LOAD: { LoadSDNode *LD = cast<LoadSDNode>(Op); const Constant *Cst = TLI->getTargetConstantFromLoad(LD); @@ -3095,13 +3135,10 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, } } else if (BitWidth == CstTy->getPrimitiveSizeInBits()) { if (auto *CInt = dyn_cast<ConstantInt>(Cst)) { - const APInt &Value = CInt->getValue(); - Known.One = Value; - Known.Zero = ~Value; + Known = KnownBits::makeConstant(CInt->getValue()); } else if (auto *CFP = dyn_cast<ConstantFP>(Cst)) { - APInt Value = CFP->getValueAPF().bitcastToAPInt(); - Known.One = Value; - Known.Zero = ~Value; + Known = + KnownBits::makeConstant(CFP->getValueAPF().bitcastToAPInt()); } } } @@ -3241,53 +3278,16 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, Known = KnownBits::computeForAddCarry(Known, Known2, Carry); break; } - case ISD::SREM: - if (ConstantSDNode *Rem = isConstOrConstSplat(Op.getOperand(1))) { - const APInt &RA = Rem->getAPIntValue().abs(); - if (RA.isPowerOf2()) { - APInt LowBits = RA - 1; - Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); - - // The low bits of the first operand are unchanged by the srem. - Known.Zero = Known2.Zero & LowBits; - Known.One = Known2.One & LowBits; - - // If the first operand is non-negative or has all low bits zero, then - // the upper bits are all zero. - if (Known2.isNonNegative() || LowBits.isSubsetOf(Known2.Zero)) - Known.Zero |= ~LowBits; - - // If the first operand is negative and not all low bits are zero, then - // the upper bits are all one. - if (Known2.isNegative() && LowBits.intersects(Known2.One)) - Known.One |= ~LowBits; - assert((Known.Zero & Known.One) == 0&&"Bits known to be one AND zero?"); - } - } + case ISD::SREM: { + Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); + Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); + Known = KnownBits::srem(Known, Known2); break; + } case ISD::UREM: { - if (ConstantSDNode *Rem = isConstOrConstSplat(Op.getOperand(1))) { - const APInt &RA = Rem->getAPIntValue(); - if (RA.isPowerOf2()) { - APInt LowBits = (RA - 1); - Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); - - // The upper bits are all zero, the lower ones are unchanged. - Known.Zero = Known2.Zero | ~LowBits; - Known.One = Known2.One & LowBits; - break; - } - } - - // Since the result is less than or equal to either operand, any leading - // zero bits in either operand must also exist in the result. Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); - - uint32_t Leaders = - std::max(Known.countMinLeadingZeros(), Known2.countMinLeadingZeros()); - Known.resetAll(); - Known.Zero.setHighBits(Leaders); + Known = KnownBits::urem(Known, Known2); break; } case ISD::EXTRACT_ELEMENT: { @@ -3307,6 +3307,9 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, SDValue InVec = Op.getOperand(0); SDValue EltNo = Op.getOperand(1); EVT VecVT = InVec.getValueType(); + // computeKnownBits not yet implemented for scalable vectors. + if (VecVT.isScalableVector()) + break; const unsigned EltBitWidth = VecVT.getScalarSizeInBits(); const unsigned NumSrcElts = VecVT.getVectorNumElements(); @@ -3347,73 +3350,39 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, Known.Zero.setAllBits(); if (DemandedVal) { Known2 = computeKnownBits(InVal, Depth + 1); - Known.One &= Known2.One.zextOrTrunc(BitWidth); - Known.Zero &= Known2.Zero.zextOrTrunc(BitWidth); + Known = KnownBits::commonBits(Known, Known2.zextOrTrunc(BitWidth)); } if (!!DemandedVecElts) { Known2 = computeKnownBits(InVec, DemandedVecElts, Depth + 1); - Known.One &= Known2.One; - Known.Zero &= Known2.Zero; + Known = KnownBits::commonBits(Known, Known2); } break; } case ISD::BITREVERSE: { Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); - Known.Zero = Known2.Zero.reverseBits(); - Known.One = Known2.One.reverseBits(); + Known = Known2.reverseBits(); break; } case ISD::BSWAP: { Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); - Known.Zero = Known2.Zero.byteSwap(); - Known.One = Known2.One.byteSwap(); + Known = Known2.byteSwap(); break; } case ISD::ABS: { Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); - - // If the source's MSB is zero then we know the rest of the bits already. - if (Known2.isNonNegative()) { - Known.Zero = Known2.Zero; - Known.One = Known2.One; - break; - } - - // We only know that the absolute values's MSB will be zero iff there is - // a set bit that isn't the sign bit (otherwise it could be INT_MIN). - Known2.One.clearSignBit(); - if (Known2.One.getBoolValue()) { - Known.Zero = APInt::getSignMask(BitWidth); - break; - } + Known = Known2.abs(); break; } case ISD::UMIN: { Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); - - // UMIN - we know that the result will have the maximum of the - // known zero leading bits of the inputs. - unsigned LeadZero = Known.countMinLeadingZeros(); - LeadZero = std::max(LeadZero, Known2.countMinLeadingZeros()); - - Known.Zero &= Known2.Zero; - Known.One &= Known2.One; - Known.Zero.setHighBits(LeadZero); + Known = KnownBits::umin(Known, Known2); break; } case ISD::UMAX: { Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); - - // UMAX - we know that the result will have the maximum of the - // known one leading bits of the inputs. - unsigned LeadOne = Known.countMinLeadingOnes(); - LeadOne = std::max(LeadOne, Known2.countMinLeadingOnes()); - - Known.Zero &= Known2.Zero; - Known.One &= Known2.One; - Known.One.setHighBits(LeadOne); + Known = KnownBits::umax(Known, Known2); break; } case ISD::SMIN: @@ -3447,12 +3416,12 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, } } - // Fallback - just get the shared known bits of the operands. Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); - if (Known.isUnknown()) break; // Early-out Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); - Known.Zero &= Known2.Zero; - Known.One &= Known2.One; + if (IsMax) + Known = KnownBits::smax(Known, Known2); + else + Known = KnownBits::smin(Known, Known2); break; } case ISD::FrameIndex: @@ -4395,11 +4364,16 @@ static SDValue foldCONCAT_VECTORS(const SDLoc &DL, EVT VT, for (SDValue Op : Elts) SVT = (SVT.bitsLT(Op.getValueType()) ? Op.getValueType() : SVT); - if (SVT.bitsGT(VT.getScalarType())) - for (SDValue &Op : Elts) - Op = DAG.getTargetLoweringInfo().isZExtFree(Op.getValueType(), SVT) - ? DAG.getZExtOrTrunc(Op, DL, SVT) - : DAG.getSExtOrTrunc(Op, DL, SVT); + if (SVT.bitsGT(VT.getScalarType())) { + for (SDValue &Op : Elts) { + if (Op.isUndef()) + Op = DAG.getUNDEF(SVT); + else + Op = DAG.getTargetLoweringInfo().isZExtFree(Op.getValueType(), SVT) + ? DAG.getZExtOrTrunc(Op, DL, SVT) + : DAG.getSExtOrTrunc(Op, DL, SVT); + } + } SDValue V = DAG.getBuildVector(VT, DL, Elts); NewSDValueDbgMsg(V, "New node fold concat vectors: ", &DAG); @@ -4425,6 +4399,14 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT) { } SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, + SDValue Operand) { + SDNodeFlags Flags; + if (Inserter) + Flags = Inserter->getFlags(); + return getNode(Opcode, DL, VT, Operand, Flags); +} + +SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue Operand, const SDNodeFlags Flags) { // Constant fold unary operations with an integer constant operand. Even // opaque constant will be folded, because the folding of unary operations @@ -4625,8 +4607,8 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, Operand.getValueType().isFloatingPoint() && "Invalid FP cast!"); if (Operand.getValueType() == VT) return Operand; // noop conversion. assert((!VT.isVector() || - VT.getVectorNumElements() == - Operand.getValueType().getVectorNumElements()) && + VT.getVectorElementCount() == + Operand.getValueType().getVectorElementCount()) && "Vector element count mismatch!"); assert(Operand.getValueType().bitsLT(VT) && "Invalid fpext node, dst < src!"); @@ -4811,6 +4793,25 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, case ISD::VSCALE: assert(VT == Operand.getValueType() && "Unexpected VT!"); break; + case ISD::CTPOP: + if (Operand.getValueType().getScalarType() == MVT::i1) + return Operand; + break; + case ISD::CTLZ: + case ISD::CTTZ: + if (Operand.getValueType().getScalarType() == MVT::i1) + return getNOT(DL, Operand, Operand.getValueType()); + break; + case ISD::VECREDUCE_SMIN: + case ISD::VECREDUCE_UMAX: + if (Operand.getValueType().getScalarType() == MVT::i1) + return getNode(ISD::VECREDUCE_OR, DL, VT, Operand); + break; + case ISD::VECREDUCE_SMAX: + case ISD::VECREDUCE_UMIN: + if (Operand.getValueType().getScalarType() == MVT::i1) + return getNode(ISD::VECREDUCE_AND, DL, VT, Operand); + break; } SDNode *N; @@ -5233,6 +5234,14 @@ SDValue SelectionDAG::getAssertAlign(const SDLoc &DL, SDValue Val, Align A) { } SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, + SDValue N1, SDValue N2) { + SDNodeFlags Flags; + if (Inserter) + Flags = Inserter->getFlags(); + return getNode(Opcode, DL, VT, N1, N2, Flags); +} + +SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1, SDValue N2, const SDNodeFlags Flags) { ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1); ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2); @@ -5312,10 +5321,6 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, case ISD::MULHS: case ISD::SDIV: case ISD::SREM: - case ISD::SMIN: - case ISD::SMAX: - case ISD::UMIN: - case ISD::UMAX: case ISD::SADDSAT: case ISD::SSUBSAT: case ISD::UADDSAT: @@ -5324,6 +5329,22 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, assert(N1.getValueType() == N2.getValueType() && N1.getValueType() == VT && "Binary operator types must match!"); break; + case ISD::SMIN: + case ISD::UMAX: + assert(VT.isInteger() && "This operator does not apply to FP types!"); + assert(N1.getValueType() == N2.getValueType() && + N1.getValueType() == VT && "Binary operator types must match!"); + if (VT.isVector() && VT.getVectorElementType() == MVT::i1) + return getNode(ISD::OR, DL, VT, N1, N2); + break; + case ISD::SMAX: + case ISD::UMIN: + assert(VT.isInteger() && "This operator does not apply to FP types!"); + assert(N1.getValueType() == N2.getValueType() && + N1.getValueType() == VT && "Binary operator types must match!"); + if (VT.isVector() && VT.getVectorElementType() == MVT::i1) + return getNode(ISD::AND, DL, VT, N1, N2); + break; case ISD::FADD: case ISD::FSUB: case ISD::FMUL: @@ -5365,8 +5386,8 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, // amounts. This catches things like trying to shift an i1024 value by an // i8, which is easy to fall into in generic code that uses // TLI.getShiftAmount(). - assert(N2.getValueType().getScalarSizeInBits().getFixedSize() >= - Log2_32_Ceil(VT.getScalarSizeInBits().getFixedSize()) && + assert(N2.getValueType().getScalarSizeInBits() >= + Log2_32_Ceil(VT.getScalarSizeInBits()) && "Invalid use of small shift amount with oversized value!"); // Always fold shifts of i1 values so the code generator doesn't need to @@ -5562,6 +5583,11 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, (VT.getVectorMinNumElements() + N2C->getZExtValue()) <= N1VT.getVectorMinNumElements()) && "Extract subvector overflow!"); + assert(N2C->getAPIntValue().getBitWidth() == + TLI->getVectorIdxTy(getDataLayout()) + .getSizeInBits() + .getFixedSize() && + "Constant index for EXTRACT_SUBVECTOR has an invalid size"); // Trivial extraction. if (VT == N1VT) @@ -5573,8 +5599,8 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, // EXTRACT_SUBVECTOR of CONCAT_VECTOR can be simplified if the pieces of // the concat have the same type as the extract. - if (N2C && N1.getOpcode() == ISD::CONCAT_VECTORS && - N1.getNumOperands() > 0 && VT == N1.getOperand(0).getValueType()) { + if (N1.getOpcode() == ISD::CONCAT_VECTORS && N1.getNumOperands() > 0 && + VT == N1.getOperand(0).getValueType()) { unsigned Factor = VT.getVectorMinNumElements(); return N1.getOperand(N2C->getZExtValue() / Factor); } @@ -5671,6 +5697,14 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, } SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, + SDValue N1, SDValue N2, SDValue N3) { + SDNodeFlags Flags; + if (Inserter) + Flags = Inserter->getFlags(); + return getNode(Opcode, DL, VT, N1, N2, N3, Flags); +} + +SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1, SDValue N2, SDValue N3, const SDNodeFlags Flags) { // Perform various simplifications. @@ -5940,11 +5974,20 @@ static SDValue getMemsetStringVal(EVT VT, const SDLoc &dl, SelectionDAG &DAG, return SDValue(nullptr, 0); } -SDValue SelectionDAG::getMemBasePlusOffset(SDValue Base, int64_t Offset, +SDValue SelectionDAG::getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags) { EVT VT = Base.getValueType(); - return getMemBasePlusOffset(Base, getConstant(Offset, DL, VT), DL, Flags); + SDValue Index; + + if (Offset.isScalable()) + Index = getVScale(DL, Base.getValueType(), + APInt(Base.getValueSizeInBits().getFixedSize(), + Offset.getKnownMinSize())); + else + Index = getConstant(Offset.getFixedSize(), DL, VT); + + return getMemBasePlusOffset(Base, Index, DL, Flags); } SDValue SelectionDAG::getMemBasePlusOffset(SDValue Ptr, SDValue Offset, @@ -6039,7 +6082,8 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, SrcAlign = Alignment; assert(SrcAlign && "SrcAlign must be set"); ConstantDataArraySlice Slice; - bool CopyFromConstant = isMemSrcFromConstant(Src, Slice); + // If marked as volatile, perform a copy even when marked as constant. + bool CopyFromConstant = !isVol && isMemSrcFromConstant(Src, Slice); bool isZeroConstant = CopyFromConstant && Slice.Array == nullptr; unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize); const MemOp Op = isZeroConstant @@ -6111,8 +6155,9 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, Value = getMemsetStringVal(VT, dl, DAG, TLI, SubSlice); if (Value.getNode()) { Store = DAG.getStore( - Chain, dl, Value, DAG.getMemBasePlusOffset(Dst, DstOff, dl), - DstPtrInfo.getWithOffset(DstOff), Alignment.value(), MMOFlags); + Chain, dl, Value, + DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl), + DstPtrInfo.getWithOffset(DstOff), Alignment, MMOFlags); OutChains.push_back(Store); } } @@ -6132,16 +6177,17 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, if (isDereferenceable) SrcMMOFlags |= MachineMemOperand::MODereferenceable; - Value = DAG.getExtLoad(ISD::EXTLOAD, dl, NVT, Chain, - DAG.getMemBasePlusOffset(Src, SrcOff, dl), - SrcPtrInfo.getWithOffset(SrcOff), VT, - commonAlignment(*SrcAlign, SrcOff).value(), - SrcMMOFlags); + Value = DAG.getExtLoad( + ISD::EXTLOAD, dl, NVT, Chain, + DAG.getMemBasePlusOffset(Src, TypeSize::Fixed(SrcOff), dl), + SrcPtrInfo.getWithOffset(SrcOff), VT, + commonAlignment(*SrcAlign, SrcOff), SrcMMOFlags); OutLoadChains.push_back(Value.getValue(1)); Store = DAG.getTruncStore( - Chain, dl, Value, DAG.getMemBasePlusOffset(Dst, DstOff, dl), - DstPtrInfo.getWithOffset(DstOff), VT, Alignment.value(), MMOFlags); + Chain, dl, Value, + DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl), + DstPtrInfo.getWithOffset(DstOff), VT, Alignment, MMOFlags); OutStoreChains.push_back(Store); } SrcOff += VTSize; @@ -6261,9 +6307,10 @@ static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, if (isDereferenceable) SrcMMOFlags |= MachineMemOperand::MODereferenceable; - Value = DAG.getLoad( - VT, dl, Chain, DAG.getMemBasePlusOffset(Src, SrcOff, dl), - SrcPtrInfo.getWithOffset(SrcOff), SrcAlign->value(), SrcMMOFlags); + Value = + DAG.getLoad(VT, dl, Chain, + DAG.getMemBasePlusOffset(Src, TypeSize::Fixed(SrcOff), dl), + SrcPtrInfo.getWithOffset(SrcOff), *SrcAlign, SrcMMOFlags); LoadValues.push_back(Value); LoadChains.push_back(Value.getValue(1)); SrcOff += VTSize; @@ -6275,9 +6322,10 @@ static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, unsigned VTSize = VT.getSizeInBits() / 8; SDValue Store; - Store = DAG.getStore( - Chain, dl, LoadValues[i], DAG.getMemBasePlusOffset(Dst, DstOff, dl), - DstPtrInfo.getWithOffset(DstOff), Alignment.value(), MMOFlags); + Store = + DAG.getStore(Chain, dl, LoadValues[i], + DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl), + DstPtrInfo.getWithOffset(DstOff), Alignment, MMOFlags); OutChains.push_back(Store); DstOff += VTSize; } @@ -6375,8 +6423,9 @@ static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl, } assert(Value.getValueType() == VT && "Value with wrong type."); SDValue Store = DAG.getStore( - Chain, dl, Value, DAG.getMemBasePlusOffset(Dst, DstOff, dl), - DstPtrInfo.getWithOffset(DstOff), Alignment.value(), + Chain, dl, Value, + DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl), + DstPtrInfo.getWithOffset(DstOff), Alignment, isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone); OutChains.push_back(Store); DstOff += VT.getSizeInBits() / 8; @@ -6390,7 +6439,7 @@ static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI, unsigned AS) { // Lowering memcpy / memset / memmove intrinsics to calls is only valid if all // pointer operands can be losslessly bitcasted to pointers of address space 0 - if (AS != 0 && !TLI->isNoopAddrSpaceCast(AS, 0)) { + if (AS != 0 && !TLI->getTargetMachine().isNoopAddrSpaceCast(AS, 0)) { report_fatal_error("cannot lower memory intrinsic in address space " + Twine(AS)); } @@ -6882,6 +6931,30 @@ SDValue SelectionDAG::getLifetimeNode(bool IsStart, const SDLoc &dl, return V; } +SDValue SelectionDAG::getPseudoProbeNode(const SDLoc &Dl, SDValue Chain, + uint64_t Guid, uint64_t Index, + uint32_t Attr) { + const unsigned Opcode = ISD::PSEUDO_PROBE; + const auto VTs = getVTList(MVT::Other); + SDValue Ops[] = {Chain}; + FoldingSetNodeID ID; + AddNodeIDNode(ID, Opcode, VTs, Ops); + ID.AddInteger(Guid); + ID.AddInteger(Index); + void *IP = nullptr; + if (SDNode *E = FindNodeOrInsertPos(ID, Dl, IP)) + return SDValue(E, 0); + + auto *N = newSDNode<PseudoProbeSDNode>( + Opcode, Dl.getIROrder(), Dl.getDebugLoc(), VTs, Guid, Index, Attr); + createOperands(N, Ops); + CSEMap.InsertNode(N, IP); + InsertNode(N); + SDValue V(N, 0); + NewSDValueDbgMsg(V, "Creating new node: ", this); + return V; +} + /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a /// MachinePointerInfo record from it. This is particularly useful because the /// code generator has many cases where it doesn't bother passing in a @@ -6962,7 +7035,7 @@ SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, assert(VT.isVector() == MemVT.isVector() && "Cannot use an ext load to convert to or from a vector!"); assert((!VT.isVector() || - VT.getVectorNumElements() == MemVT.getVectorNumElements()) && + VT.getVectorElementCount() == MemVT.getVectorElementCount()) && "Cannot use an ext load to change the number of vector elements!"); } @@ -7041,8 +7114,7 @@ SDValue SelectionDAG::getIndexedLoad(SDValue OrigLoad, const SDLoc &dl, ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable); return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl, LD->getChain(), Base, Offset, LD->getPointerInfo(), - LD->getMemoryVT(), LD->getAlignment(), MMOFlags, - LD->getAAInfo()); + LD->getMemoryVT(), LD->getAlign(), MMOFlags, LD->getAAInfo()); } SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val, @@ -7112,7 +7184,8 @@ SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, MachineFunction &MF = getMachineFunction(); MachineMemOperand *MMO = MF.getMachineMemOperand( - PtrInfo, MMOFlags, SVT.getStoreSize(), Alignment, AAInfo); + PtrInfo, MMOFlags, MemoryLocation::getSizeOrUnknown(SVT.getStoreSize()), + Alignment, AAInfo); return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO); } @@ -7133,7 +7206,7 @@ SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, assert(VT.isVector() == SVT.isVector() && "Cannot use trunc store to convert to or from a vector!"); assert((!VT.isVector() || - VT.getVectorNumElements() == SVT.getVectorNumElements()) && + VT.getVectorElementCount() == SVT.getVectorElementCount()) && "Cannot use trunc store to change the number of vector elements!"); SDVTList VTs = getVTList(MVT::Other); @@ -7285,14 +7358,15 @@ SDValue SelectionDAG::getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl, SDValue SelectionDAG::getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef<SDValue> Ops, MachineMemOperand *MMO, - ISD::MemIndexType IndexType) { + ISD::MemIndexType IndexType, + ISD::LoadExtType ExtTy) { assert(Ops.size() == 6 && "Incompatible number of operands"); FoldingSetNodeID ID; AddNodeIDNode(ID, ISD::MGATHER, VTs, Ops); ID.AddInteger(VT.getRawBits()); ID.AddInteger(getSyntheticNodeSubclassData<MaskedGatherSDNode>( - dl.getIROrder(), VTs, VT, MMO, IndexType)); + dl.getIROrder(), VTs, VT, MMO, IndexType, ExtTy)); ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); void *IP = nullptr; if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { @@ -7300,17 +7374,22 @@ SDValue SelectionDAG::getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl, return SDValue(E, 0); } + IndexType = TLI->getCanonicalIndexType(IndexType, VT, Ops[4]); auto *N = newSDNode<MaskedGatherSDNode>(dl.getIROrder(), dl.getDebugLoc(), - VTs, VT, MMO, IndexType); + VTs, VT, MMO, IndexType, ExtTy); createOperands(N, Ops); assert(N->getPassThru().getValueType() == N->getValueType(0) && "Incompatible type of the PassThru value in MaskedGatherSDNode"); - assert(N->getMask().getValueType().getVectorNumElements() == - N->getValueType(0).getVectorNumElements() && + assert(N->getMask().getValueType().getVectorElementCount() == + N->getValueType(0).getVectorElementCount() && "Vector width mismatch between mask and data"); - assert(N->getIndex().getValueType().getVectorNumElements() >= - N->getValueType(0).getVectorNumElements() && + assert(N->getIndex().getValueType().getVectorElementCount().isScalable() == + N->getValueType(0).getVectorElementCount().isScalable() && + "Scalable flags of index and data do not match"); + assert(ElementCount::isKnownGE( + N->getIndex().getValueType().getVectorElementCount(), + N->getValueType(0).getVectorElementCount()) && "Vector width mismatch between index and data"); assert(isa<ConstantSDNode>(N->getScale()) && cast<ConstantSDNode>(N->getScale())->getAPIntValue().isPowerOf2() && @@ -7326,29 +7405,37 @@ SDValue SelectionDAG::getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl, SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef<SDValue> Ops, MachineMemOperand *MMO, - ISD::MemIndexType IndexType) { + ISD::MemIndexType IndexType, + bool IsTrunc) { assert(Ops.size() == 6 && "Incompatible number of operands"); FoldingSetNodeID ID; AddNodeIDNode(ID, ISD::MSCATTER, VTs, Ops); ID.AddInteger(VT.getRawBits()); ID.AddInteger(getSyntheticNodeSubclassData<MaskedScatterSDNode>( - dl.getIROrder(), VTs, VT, MMO, IndexType)); + dl.getIROrder(), VTs, VT, MMO, IndexType, IsTrunc)); ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); void *IP = nullptr; if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { cast<MaskedScatterSDNode>(E)->refineAlignment(MMO); return SDValue(E, 0); } + + IndexType = TLI->getCanonicalIndexType(IndexType, VT, Ops[4]); auto *N = newSDNode<MaskedScatterSDNode>(dl.getIROrder(), dl.getDebugLoc(), - VTs, VT, MMO, IndexType); + VTs, VT, MMO, IndexType, IsTrunc); createOperands(N, Ops); - assert(N->getMask().getValueType().getVectorNumElements() == - N->getValue().getValueType().getVectorNumElements() && + assert(N->getMask().getValueType().getVectorElementCount() == + N->getValue().getValueType().getVectorElementCount() && "Vector width mismatch between mask and data"); - assert(N->getIndex().getValueType().getVectorNumElements() >= - N->getValue().getValueType().getVectorNumElements() && + assert( + N->getIndex().getValueType().getVectorElementCount().isScalable() == + N->getValue().getValueType().getVectorElementCount().isScalable() && + "Scalable flags of index and data do not match"); + assert(ElementCount::isKnownGE( + N->getIndex().getValueType().getVectorElementCount(), + N->getValue().getValueType().getVectorElementCount()) && "Vector width mismatch between index and data"); assert(isa<ConstantSDNode>(N->getScale()) && cast<ConstantSDNode>(N->getScale())->getAPIntValue().isPowerOf2() && @@ -7452,6 +7539,11 @@ SDValue SelectionDAG::simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y, if (YC->getValueAPF().isExactlyValue(1.0)) return X; + // X * 0.0 --> 0.0 + if (Opcode == ISD::FMUL && Flags.hasNoNaNs() && Flags.hasNoSignedZeros()) + if (YC->getValueAPF().isZero()) + return getConstantFP(0.0, SDLoc(Y), Y.getValueType()); + return SDValue(); } @@ -7478,6 +7570,14 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, } SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, + ArrayRef<SDValue> Ops) { + SDNodeFlags Flags; + if (Inserter) + Flags = Inserter->getFlags(); + return getNode(Opcode, DL, VT, Ops, Flags); +} + +SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef<SDValue> Ops, const SDNodeFlags Flags) { unsigned NumOps = Ops.size(); switch (NumOps) { @@ -7549,6 +7649,14 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, } SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, + ArrayRef<SDValue> Ops) { + SDNodeFlags Flags; + if (Inserter) + Flags = Inserter->getFlags(); + return getNode(Opcode, DL, VTList, Ops, Flags); +} + +SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, ArrayRef<SDValue> Ops, const SDNodeFlags Flags) { if (VTList.NumVTs == 1) return getNode(Opcode, DL, VTList.VTs[0], Ops); @@ -8245,6 +8353,14 @@ SDValue SelectionDAG::getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, /// getNodeIfExists - Get the specified node if it's already available, or /// else return NULL. SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList, + ArrayRef<SDValue> Ops) { + SDNodeFlags Flags; + if (Inserter) + Flags = Inserter->getFlags(); + return getNodeIfExists(Opcode, VTList, Ops, Flags); +} + +SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList, ArrayRef<SDValue> Ops, const SDNodeFlags Flags) { if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) { @@ -8259,6 +8375,19 @@ SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList, return nullptr; } +/// doesNodeExist - Check if a node exists without modifying its flags. +bool SelectionDAG::doesNodeExist(unsigned Opcode, SDVTList VTList, + ArrayRef<SDValue> Ops) { + if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) { + FoldingSetNodeID ID; + AddNodeIDNode(ID, Opcode, VTList, Ops); + void *IP = nullptr; + if (FindNodeOrInsertPos(ID, SDLoc(), IP)) + return true; + } + return false; +} + /// getDbgValue - Creates a SDDbgValue node. /// /// SDNode @@ -8676,21 +8805,31 @@ namespace { } // end anonymous namespace -void SelectionDAG::updateDivergence(SDNode * N) -{ - if (TLI->isSDNodeAlwaysUniform(N)) - return; - bool IsDivergent = TLI->isSDNodeSourceOfDivergence(N, FLI, DA); +bool SelectionDAG::calculateDivergence(SDNode *N) { + if (TLI->isSDNodeAlwaysUniform(N)) { + assert(!TLI->isSDNodeSourceOfDivergence(N, FLI, DA) && + "Conflicting divergence information!"); + return false; + } + if (TLI->isSDNodeSourceOfDivergence(N, FLI, DA)) + return true; for (auto &Op : N->ops()) { - if (Op.Val.getValueType() != MVT::Other) - IsDivergent |= Op.getNode()->isDivergent(); + if (Op.Val.getValueType() != MVT::Other && Op.getNode()->isDivergent()) + return true; } - if (N->SDNodeBits.IsDivergent != IsDivergent) { - N->SDNodeBits.IsDivergent = IsDivergent; - for (auto U : N->uses()) { - updateDivergence(U); + return false; +} + +void SelectionDAG::updateDivergence(SDNode *N) { + SmallVector<SDNode *, 16> Worklist(1, N); + do { + N = Worklist.pop_back_val(); + bool IsDivergent = calculateDivergence(N); + if (N->SDNodeBits.IsDivergent != IsDivergent) { + N->SDNodeBits.IsDivergent = IsDivergent; + llvm::append_range(Worklist, N->uses()); } - } + } while (!Worklist.empty()); } void SelectionDAG::CreateTopologicalOrder(std::vector<SDNode *> &Order) { @@ -8716,26 +8855,9 @@ void SelectionDAG::CreateTopologicalOrder(std::vector<SDNode *> &Order) { void SelectionDAG::VerifyDAGDiverence() { std::vector<SDNode *> TopoOrder; CreateTopologicalOrder(TopoOrder); - const TargetLowering &TLI = getTargetLoweringInfo(); - DenseMap<const SDNode *, bool> DivergenceMap; - for (auto &N : allnodes()) { - DivergenceMap[&N] = false; - } - for (auto N : TopoOrder) { - bool IsDivergent = DivergenceMap[N]; - bool IsSDNodeDivergent = TLI.isSDNodeSourceOfDivergence(N, FLI, DA); - for (auto &Op : N->ops()) { - if (Op.Val.getValueType() != MVT::Other) - IsSDNodeDivergent |= DivergenceMap[Op.getNode()]; - } - if (!IsDivergent && IsSDNodeDivergent && !TLI.isSDNodeAlwaysUniform(N)) { - DivergenceMap[N] = true; - } - } - for (auto &N : allnodes()) { - (void)N; - assert(DivergenceMap[&N] == N.isDivergent() && - "Divergence bit inconsistency detected\n"); + for (auto *N : TopoOrder) { + assert(calculateDivergence(N) == N->isDivergent() && + "Divergence bit inconsistency detected"); } } #endif @@ -8904,25 +9026,32 @@ void SelectionDAG::AddDbgLabel(SDDbgLabel *DB) { DbgInfo->add(DB); } -SDValue SelectionDAG::makeEquivalentMemoryOrdering(LoadSDNode *OldLoad, - SDValue NewMemOp) { - assert(isa<MemSDNode>(NewMemOp.getNode()) && "Expected a memop node"); +SDValue SelectionDAG::makeEquivalentMemoryOrdering(SDValue OldChain, + SDValue NewMemOpChain) { + assert(isa<MemSDNode>(NewMemOpChain) && "Expected a memop node"); + assert(NewMemOpChain.getValueType() == MVT::Other && "Expected a token VT"); // The new memory operation must have the same position as the old load in // terms of memory dependency. Create a TokenFactor for the old load and new // memory operation and update uses of the old load's output chain to use that // TokenFactor. - SDValue OldChain = SDValue(OldLoad, 1); - SDValue NewChain = SDValue(NewMemOp.getNode(), 1); - if (OldChain == NewChain || !OldLoad->hasAnyUseOfValue(1)) - return NewChain; + if (OldChain == NewMemOpChain || OldChain.use_empty()) + return NewMemOpChain; - SDValue TokenFactor = - getNode(ISD::TokenFactor, SDLoc(OldLoad), MVT::Other, OldChain, NewChain); + SDValue TokenFactor = getNode(ISD::TokenFactor, SDLoc(OldChain), MVT::Other, + OldChain, NewMemOpChain); ReplaceAllUsesOfValueWith(OldChain, TokenFactor); - UpdateNodeOperands(TokenFactor.getNode(), OldChain, NewChain); + UpdateNodeOperands(TokenFactor.getNode(), OldChain, NewMemOpChain); return TokenFactor; } +SDValue SelectionDAG::makeEquivalentMemoryOrdering(LoadSDNode *OldLoad, + SDValue NewMemOp) { + assert(isa<MemSDNode>(NewMemOp.getNode()) && "Expected a memop node"); + SDValue OldChain = SDValue(OldLoad, 1); + SDValue NewMemOpChain = NewMemOp.getValue(1); + return makeEquivalentMemoryOrdering(OldChain, NewMemOpChain); +} + SDValue SelectionDAG::getSymbolFunctionGlobalAddress(SDValue Op, Function **OutFunction) { assert(isa<ExternalSymbolSDNode>(Op) && "Node should be an ExternalSymbol"); @@ -9006,6 +9135,18 @@ ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, bool AllowUndefs, if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N)) return CN; + // SplatVectors can truncate their operands. Ignore that case here unless + // AllowTruncation is set. + if (N->getOpcode() == ISD::SPLAT_VECTOR) { + EVT VecEltVT = N->getValueType(0).getVectorElementType(); + if (auto *CN = dyn_cast<ConstantSDNode>(N->getOperand(0))) { + EVT CVT = CN->getValueType(0); + assert(CVT.bitsGE(VecEltVT) && "Illegal splat_vector element extension"); + if (AllowTruncation || CVT == VecEltVT) + return CN; + } + } + if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N)) { BitVector UndefElements; ConstantSDNode *CN = BV->getConstantSplatNode(&UndefElements); @@ -9059,6 +9200,10 @@ ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N, bool AllowUndefs) { return CN; } + if (N.getOpcode() == ISD::SPLAT_VECTOR) + if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(N.getOperand(0))) + return CN; + return nullptr; } @@ -9220,8 +9365,7 @@ bool SDNode::areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N) { bool Seen = false; for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) { SDNode *User = *I; - if (llvm::any_of(Nodes, - [&User](const SDNode *Node) { return User == Node; })) + if (llvm::is_contained(Nodes, User)) Seen = true; else return false; @@ -9232,7 +9376,7 @@ bool SDNode::areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N) { /// isOperand - Return true if this node is an operand of N. bool SDValue::isOperandOf(const SDNode *N) const { - return any_of(N->op_values(), [this](SDValue Op) { return *this == Op; }); + return is_contained(N->op_values(), *this); } bool SDNode::isOperandOf(const SDNode *N) const { @@ -9616,24 +9760,24 @@ std::pair<EVT, EVT> SelectionDAG::GetDependentSplitDestVTs(const EVT &VT, const EVT &EnvVT, bool *HiIsEmpty) const { EVT EltTp = VT.getVectorElementType(); - bool IsScalable = VT.isScalableVector(); // Examples: // custom VL=8 with enveloping VL=8/8 yields 8/0 (hi empty) // custom VL=9 with enveloping VL=8/8 yields 8/1 // custom VL=10 with enveloping VL=8/8 yields 8/2 // etc. - unsigned VTNumElts = VT.getVectorNumElements(); - unsigned EnvNumElts = EnvVT.getVectorNumElements(); + ElementCount VTNumElts = VT.getVectorElementCount(); + ElementCount EnvNumElts = EnvVT.getVectorElementCount(); + assert(VTNumElts.isScalable() == EnvNumElts.isScalable() && + "Mixing fixed width and scalable vectors when enveloping a type"); EVT LoVT, HiVT; - if (VTNumElts > EnvNumElts) { + if (VTNumElts.getKnownMinValue() > EnvNumElts.getKnownMinValue()) { LoVT = EnvVT; - HiVT = EVT::getVectorVT(*getContext(), EltTp, VTNumElts - EnvNumElts, - IsScalable); + HiVT = EVT::getVectorVT(*getContext(), EltTp, VTNumElts - EnvNumElts); *HiIsEmpty = false; } else { // Flag that hi type has zero storage size, but return split envelop type // (this would be easier if vector types with zero elements were allowed). - LoVT = EVT::getVectorVT(*getContext(), EltTp, VTNumElts, IsScalable); + LoVT = EVT::getVectorVT(*getContext(), EltTp, VTNumElts); HiVT = EnvVT; *HiIsEmpty = true; } @@ -9768,16 +9912,16 @@ bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, APInt &SplatUndef, SDValue BuildVectorSDNode::getSplatValue(const APInt &DemandedElts, BitVector *UndefElements) const { + unsigned NumOps = getNumOperands(); if (UndefElements) { UndefElements->clear(); - UndefElements->resize(getNumOperands()); + UndefElements->resize(NumOps); } - assert(getNumOperands() == DemandedElts.getBitWidth() && - "Unexpected vector size"); + assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size"); if (!DemandedElts) return SDValue(); SDValue Splatted; - for (unsigned i = 0, e = getNumOperands(); i != e; ++i) { + for (unsigned i = 0; i != NumOps; ++i) { if (!DemandedElts[i]) continue; SDValue Op = getOperand(i); @@ -9806,6 +9950,58 @@ SDValue BuildVectorSDNode::getSplatValue(BitVector *UndefElements) const { return getSplatValue(DemandedElts, UndefElements); } +bool BuildVectorSDNode::getRepeatedSequence(const APInt &DemandedElts, + SmallVectorImpl<SDValue> &Sequence, + BitVector *UndefElements) const { + unsigned NumOps = getNumOperands(); + Sequence.clear(); + if (UndefElements) { + UndefElements->clear(); + UndefElements->resize(NumOps); + } + assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size"); + if (!DemandedElts || NumOps < 2 || !isPowerOf2_32(NumOps)) + return false; + + // Set the undefs even if we don't find a sequence (like getSplatValue). + if (UndefElements) + for (unsigned I = 0; I != NumOps; ++I) + if (DemandedElts[I] && getOperand(I).isUndef()) + (*UndefElements)[I] = true; + + // Iteratively widen the sequence length looking for repetitions. + for (unsigned SeqLen = 1; SeqLen < NumOps; SeqLen *= 2) { + Sequence.append(SeqLen, SDValue()); + for (unsigned I = 0; I != NumOps; ++I) { + if (!DemandedElts[I]) + continue; + SDValue &SeqOp = Sequence[I % SeqLen]; + SDValue Op = getOperand(I); + if (Op.isUndef()) { + if (!SeqOp) + SeqOp = Op; + continue; + } + if (SeqOp && !SeqOp.isUndef() && SeqOp != Op) { + Sequence.clear(); + break; + } + SeqOp = Op; + } + if (!Sequence.empty()) + return true; + } + + assert(Sequence.empty() && "Failed to empty non-repeating sequence pattern"); + return false; +} + +bool BuildVectorSDNode::getRepeatedSequence(SmallVectorImpl<SDValue> &Sequence, + BitVector *UndefElements) const { + APInt DemandedElts = APInt::getAllOnesValue(getNumOperands()); + return getRepeatedSequence(DemandedElts, Sequence, UndefElements); +} + ConstantSDNode * BuildVectorSDNode::getConstantSplatNode(const APInt &DemandedElts, BitVector *UndefElements) const { @@ -9878,7 +10074,7 @@ bool ShuffleVectorSDNode::isSplatMask(const int *Mask, EVT VT) { // Returns the SDNode if it is a constant integer BuildVector // or constant integer. -SDNode *SelectionDAG::isConstantIntBuildVectorOrConstantInt(SDValue N) { +SDNode *SelectionDAG::isConstantIntBuildVectorOrConstantInt(SDValue N) const { if (isa<ConstantSDNode>(N)) return N.getNode(); if (ISD::isBuildVectorOfConstantSDNodes(N.getNode())) @@ -9889,10 +10085,15 @@ SDNode *SelectionDAG::isConstantIntBuildVectorOrConstantInt(SDValue N) { if (GA->getOpcode() == ISD::GlobalAddress && TLI->isOffsetFoldingLegal(GA)) return GA; + if ((N.getOpcode() == ISD::SPLAT_VECTOR) && + isa<ConstantSDNode>(N.getOperand(0))) + return N.getNode(); return nullptr; } -SDNode *SelectionDAG::isConstantFPBuildVectorOrConstantFP(SDValue N) { +// Returns the SDNode if it is a constant float BuildVector +// or constant float. +SDNode *SelectionDAG::isConstantFPBuildVectorOrConstantFP(SDValue N) const { if (isa<ConstantFPSDNode>(N)) return N.getNode(); @@ -9914,13 +10115,14 @@ void SelectionDAG::createOperands(SDNode *Node, ArrayRef<SDValue> Vals) { Ops[I].setUser(Node); Ops[I].setInitial(Vals[I]); if (Ops[I].Val.getValueType() != MVT::Other) // Skip Chain. It does not carry divergence. - IsDivergent = IsDivergent || Ops[I].getNode()->isDivergent(); + IsDivergent |= Ops[I].getNode()->isDivergent(); } Node->NumOperands = Vals.size(); Node->OperandList = Ops; - IsDivergent |= TLI->isSDNodeSourceOfDivergence(Node, FLI, DA); - if (!TLI->isSDNodeAlwaysUniform(Node)) + if (!TLI->isSDNodeAlwaysUniform(Node)) { + IsDivergent |= TLI->isSDNodeSourceOfDivergence(Node, FLI, DA); Node->SDNodeBits.IsDivergent = IsDivergent; + } checkForCycles(Node); } @@ -9937,6 +10139,44 @@ SDValue SelectionDAG::getTokenFactor(const SDLoc &DL, return getNode(ISD::TokenFactor, DL, MVT::Other, Vals); } +SDValue SelectionDAG::getNeutralElement(unsigned Opcode, const SDLoc &DL, + EVT VT, SDNodeFlags Flags) { + switch (Opcode) { + default: + return SDValue(); + case ISD::ADD: + case ISD::OR: + case ISD::XOR: + case ISD::UMAX: + return getConstant(0, DL, VT); + case ISD::MUL: + return getConstant(1, DL, VT); + case ISD::AND: + case ISD::UMIN: + return getAllOnesConstant(DL, VT); + case ISD::SMAX: + return getConstant(APInt::getSignedMinValue(VT.getSizeInBits()), DL, VT); + case ISD::SMIN: + return getConstant(APInt::getSignedMaxValue(VT.getSizeInBits()), DL, VT); + case ISD::FADD: + return getConstantFP(-0.0, DL, VT); + case ISD::FMUL: + return getConstantFP(1.0, DL, VT); + case ISD::FMINNUM: + case ISD::FMAXNUM: { + // Neutral element for fminnum is NaN, Inf or FLT_MAX, depending on FMF. + const fltSemantics &Semantics = EVTToAPFloatSemantics(VT); + APFloat NeutralAF = !Flags.hasNoNaNs() ? APFloat::getQNaN(Semantics) : + !Flags.hasNoInfs() ? APFloat::getInf(Semantics) : + APFloat::getLargest(Semantics); + if (Opcode == ISD::FMAXNUM) + NeutralAF.changeSign(); + + return getConstantFP(NeutralAF, DL, VT); + } + } +} + #ifndef NDEBUG static void checkForCyclesHelper(const SDNode *N, SmallPtrSetImpl<const SDNode*> &Visited, |