diff options
Diffstat (limited to 'contrib/llvm-project/llvm/lib/Target/X86')
21 files changed, 1233 insertions, 825 deletions
diff --git a/contrib/llvm-project/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp b/contrib/llvm-project/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp index bc5f562d9589..051f6caa8c04 100644 --- a/contrib/llvm-project/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp +++ b/contrib/llvm-project/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp @@ -108,6 +108,8 @@ class X86AsmParser : public MCTargetAsmParser { // Does this instruction use apx extended register? bool UseApxExtendedReg = false; + // Is this instruction explicitly required not to update flags? + bool ForcedNoFlag = false; private: SMLoc consumeToken() { @@ -2312,8 +2314,7 @@ bool X86AsmParser::ParseIntelDotOperator(IntelExprStateMachine &SM, // Drop the optional '.'. StringRef DotDispStr = Tok.getString(); - if (DotDispStr.starts_with(".")) - DotDispStr = DotDispStr.drop_front(1); + DotDispStr.consume_front("."); StringRef TrailingDot; // .Imm gets lexed as a real. @@ -3126,6 +3127,7 @@ bool X86AsmParser::ParseInstruction(ParseInstructionInfo &Info, StringRef Name, ForcedVEXEncoding = VEXEncoding_Default; ForcedDispEncoding = DispEncoding_Default; UseApxExtendedReg = false; + ForcedNoFlag = false; // Parse pseudo prefixes. while (true) { @@ -3150,6 +3152,8 @@ bool X86AsmParser::ParseInstruction(ParseInstructionInfo &Info, StringRef Name, ForcedDispEncoding = DispEncoding_Disp8; else if (Prefix == "disp32") ForcedDispEncoding = DispEncoding_Disp32; + else if (Prefix == "nf") + ForcedNoFlag = true; else return Error(NameLoc, "unknown prefix"); @@ -3997,6 +4001,8 @@ unsigned X86AsmParser::checkTargetMatchPredicate(MCInst &Inst) { if (UseApxExtendedReg && !X86II::canUseApxExtendedReg(MCID)) return Match_Unsupported; + if (ForcedNoFlag != !!(MCID.TSFlags & X86II::EVEX_NF)) + return Match_Unsupported; if (ForcedVEXEncoding == VEXEncoding_EVEX && (MCID.TSFlags & X86II::EncodingMask) != X86II::EVEX) diff --git a/contrib/llvm-project/llvm/lib/Target/X86/Disassembler/X86Disassembler.cpp b/contrib/llvm-project/llvm/lib/Target/X86/Disassembler/X86Disassembler.cpp index 59e2008f5632..347dc0d4ed43 100644 --- a/contrib/llvm-project/llvm/lib/Target/X86/Disassembler/X86Disassembler.cpp +++ b/contrib/llvm-project/llvm/lib/Target/X86/Disassembler/X86Disassembler.cpp @@ -1169,7 +1169,11 @@ static int getInstructionID(struct InternalInstruction *insn, attrMask |= ATTR_EVEXKZ; if (bFromEVEX4of4(insn->vectorExtensionPrefix[3])) attrMask |= ATTR_EVEXB; - if (aaaFromEVEX4of4(insn->vectorExtensionPrefix[3])) + // nf bit is the MSB of aaa + if (nfFromEVEX4of4(insn->vectorExtensionPrefix[3]) && + insn->opcodeType == MAP4) + attrMask |= ATTR_EVEXNF; + else if (aaaFromEVEX4of4(insn->vectorExtensionPrefix[3])) attrMask |= ATTR_EVEXK; if (lFromEVEX4of4(insn->vectorExtensionPrefix[3])) attrMask |= ATTR_VEXL; diff --git a/contrib/llvm-project/llvm/lib/Target/X86/Disassembler/X86DisassemblerDecoder.h b/contrib/llvm-project/llvm/lib/Target/X86/Disassembler/X86DisassemblerDecoder.h index decc45091941..4c7b1c094522 100644 --- a/contrib/llvm-project/llvm/lib/Target/X86/Disassembler/X86DisassemblerDecoder.h +++ b/contrib/llvm-project/llvm/lib/Target/X86/Disassembler/X86DisassemblerDecoder.h @@ -103,6 +103,7 @@ namespace X86Disassembler { #define bFromEVEX4of4(evex) bitFromOffset4(evex) #define v2FromEVEX4of4(evex) invertedBitFromOffset3(evex) #define aaaFromEVEX4of4(evex) threeBitsFromOffset0(evex) +#define nfFromEVEX4of4(evex) bitFromOffset2(evex) // These enums represent Intel registers for use by the decoder. #define REGS_8BIT \ diff --git a/contrib/llvm-project/llvm/lib/Target/X86/MCTargetDesc/X86BaseInfo.h b/contrib/llvm-project/llvm/lib/Target/X86/MCTargetDesc/X86BaseInfo.h index b0fcaef5f4b0..e006dd877360 100644 --- a/contrib/llvm-project/llvm/lib/Target/X86/MCTargetDesc/X86BaseInfo.h +++ b/contrib/llvm-project/llvm/lib/Target/X86/MCTargetDesc/X86BaseInfo.h @@ -870,7 +870,10 @@ enum : uint64_t { ExplicitVEXPrefix = 2ULL << ExplicitOpPrefixShift, /// For instructions that are promoted to EVEX space for EGPR. ExplicitEVEXPrefix = 3ULL << ExplicitOpPrefixShift, - ExplicitOpPrefixMask = 3ULL << ExplicitOpPrefixShift + ExplicitOpPrefixMask = 3ULL << ExplicitOpPrefixShift, + /// EVEX_NF - Set if this instruction has EVEX.NF field set. + EVEX_NFShift = ExplicitOpPrefixShift + 2, + EVEX_NF = 1ULL << EVEX_NFShift }; /// \returns true if the instruction with given opcode is a prefix. @@ -992,6 +995,12 @@ inline unsigned getOperandBias(const MCInstrDesc &Desc) { } } +/// \returns true if the instruction has a NDD (new data destination). +inline bool hasNewDataDest(uint64_t TSFlags) { + return (TSFlags & X86II::OpMapMask) == X86II::T_MAP4 && + (TSFlags & X86II::EVEX_B) && (TSFlags & X86II::VEX_4V); +} + /// \returns operand # for the first field of the memory operand or -1 if no /// memory operands. /// NOTE: This ignores tied operands. If there is a tied register which is @@ -1018,7 +1027,7 @@ inline int getMemoryOperandNo(uint64_t TSFlags) { return -1; case X86II::MRMDestMem: case X86II::MRMDestMemFSIB: - return 0; + return hasNewDataDest(TSFlags); case X86II::MRMSrcMem: case X86II::MRMSrcMemFSIB: // Start from 1, skip any registers encoded in VEX_VVVV or I8IMM, or a diff --git a/contrib/llvm-project/llvm/lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.cpp b/contrib/llvm-project/llvm/lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.cpp index cab2f0a2e1c1..1947313a9dfb 100644 --- a/contrib/llvm-project/llvm/lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.cpp +++ b/contrib/llvm-project/llvm/lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.cpp @@ -369,6 +369,9 @@ void X86InstPrinterCommon::printInstFlags(const MCInst *MI, raw_ostream &O, else if (Flags & X86::IP_HAS_REPEAT) O << "\trep\t"; + if (TSFlags & X86II::EVEX_NF) + O << "\t{nf}"; + // These all require a pseudo prefix if ((Flags & X86::IP_USE_VEX) || (TSFlags & X86II::ExplicitOpPrefixMask) == X86II::ExplicitVEXPrefix) diff --git a/contrib/llvm-project/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp b/contrib/llvm-project/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp index 9e1f1eb97e70..924956295e7c 100644 --- a/contrib/llvm-project/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp +++ b/contrib/llvm-project/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp @@ -251,6 +251,7 @@ public: void setAAA(const MCInst &MI, unsigned OpNum) { EVEX_aaa = getRegEncoding(MI, OpNum); } + void setNF(bool V) { EVEX_aaa |= V << 2; } X86OpcodePrefixHelper(const MCRegisterInfo &MRI) : W(0), R(0), X(0), B(0), M(0), R2(0), X2(0), B2(0), VEX_4V(0), VEX_L(0), @@ -987,9 +988,11 @@ X86MCCodeEmitter::emitVEXOpcodePrefix(int MemOperand, const MCInst &MI, } Prefix.setW(TSFlags & X86II::REX_W); + Prefix.setNF(TSFlags & X86II::EVEX_NF); bool HasEVEX_K = TSFlags & X86II::EVEX_K; bool HasVEX_4V = TSFlags & X86II::VEX_4V; + bool IsND = X86II::hasNewDataDest(TSFlags); // IsND implies HasVEX_4V bool HasEVEX_RC = TSFlags & X86II::EVEX_RC; switch (TSFlags & X86II::OpMapMask) { @@ -1049,6 +1052,7 @@ X86MCCodeEmitter::emitVEXOpcodePrefix(int MemOperand, const MCInst &MI, bool EncodeRC = false; uint8_t EVEX_rc = 0; + unsigned CurOp = X86II::getOperandBias(Desc); switch (TSFlags & X86II::FormMask) { @@ -1073,16 +1077,21 @@ X86MCCodeEmitter::emitVEXOpcodePrefix(int MemOperand, const MCInst &MI, // MemAddr, src1(VEX_4V), src2(ModR/M) // MemAddr, src1(ModR/M), imm8 // + // NDD: + // dst(VEX_4V), MemAddr, src1(ModR/M) Prefix.setBB2(MI, MemOperand + X86::AddrBaseReg); Prefix.setXX2(MI, MemOperand + X86::AddrIndexReg); Prefix.setV2(MI, MemOperand + X86::AddrIndexReg, HasVEX_4V); + if (IsND) + Prefix.set4VV2(MI, CurOp++); + CurOp += X86::AddrNumOperands; if (HasEVEX_K) Prefix.setAAA(MI, CurOp++); - if (HasVEX_4V) + if (!IsND && HasVEX_4V) Prefix.set4VV2(MI, CurOp++); Prefix.setRR2(MI, CurOp++); @@ -1098,12 +1107,18 @@ X86MCCodeEmitter::emitVEXOpcodePrefix(int MemOperand, const MCInst &MI, // // FMA4: // dst(ModR/M.reg), src1(VEX_4V), src2(ModR/M), src3(Imm[7:4]) + // + // NDD: + // dst(VEX_4V), src1(ModR/M), MemAddr + if (IsND) + Prefix.set4VV2(MI, CurOp++); + Prefix.setRR2(MI, CurOp++); if (HasEVEX_K) Prefix.setAAA(MI, CurOp++); - if (HasVEX_4V) + if (!IsND && HasVEX_4V) Prefix.set4VV2(MI, CurOp++); Prefix.setBB2(MI, MemOperand + X86::AddrBaseReg); @@ -1160,12 +1175,17 @@ X86MCCodeEmitter::emitVEXOpcodePrefix(int MemOperand, const MCInst &MI, // // FMA4: // dst(ModR/M.reg), src1(VEX_4V), src2(Imm[7:4]), src3(ModR/M), + // + // NDD: + // dst(VEX_4V), src1(ModR/M.reg), src2(ModR/M) + if (IsND) + Prefix.set4VV2(MI, CurOp++); Prefix.setRR2(MI, CurOp++); if (HasEVEX_K) Prefix.setAAA(MI, CurOp++); - if (HasVEX_4V) + if (!IsND && HasVEX_4V) Prefix.set4VV2(MI, CurOp++); Prefix.setBB2(MI, CurOp); @@ -1209,6 +1229,11 @@ X86MCCodeEmitter::emitVEXOpcodePrefix(int MemOperand, const MCInst &MI, // dst(ModR/M), src(ModR/M) // dst(ModR/M), src(ModR/M), imm8 // dst(ModR/M), src1(VEX_4V), src2(ModR/M) + // + // NDD: + // dst(VEX_4V), src1(ModR/M), src2(ModR/M) + if (IsND) + Prefix.set4VV2(MI, CurOp++); Prefix.setBB2(MI, CurOp); Prefix.setX(MI, CurOp, 4); ++CurOp; @@ -1216,7 +1241,7 @@ X86MCCodeEmitter::emitVEXOpcodePrefix(int MemOperand, const MCInst &MI, if (HasEVEX_K) Prefix.setAAA(MI, CurOp++); - if (HasVEX_4V) + if (!IsND && HasVEX_4V) Prefix.set4VV2(MI, CurOp++); Prefix.setRR2(MI, CurOp++); @@ -1508,6 +1533,8 @@ void X86MCCodeEmitter::encodeInstruction(const MCInst &MI, unsigned OpcodeOffset = 0; + bool IsND = X86II::hasNewDataDest(TSFlags); + uint64_t Form = TSFlags & X86II::FormMask; switch (Form) { default: @@ -1576,6 +1603,8 @@ void X86MCCodeEmitter::encodeInstruction(const MCInst &MI, if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV) ++SrcRegNum; + if (IsND) // Skip the NDD operand encoded in EVEX_VVVV + ++CurOp; emitRegModRMByte(MI.getOperand(CurOp), getX86RegNum(MI.getOperand(SrcRegNum)), CB); @@ -1602,6 +1631,9 @@ void X86MCCodeEmitter::encodeInstruction(const MCInst &MI, if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV) ++SrcRegNum; + if (IsND) // Skip new data destination + ++CurOp; + bool ForceSIB = (Form == X86II::MRMDestMemFSIB); emitMemModRMByte(MI, CurOp, getX86RegNum(MI.getOperand(SrcRegNum)), TSFlags, Kind, StartByte, CB, Fixups, STI, ForceSIB); @@ -1669,6 +1701,9 @@ void X86MCCodeEmitter::encodeInstruction(const MCInst &MI, case X86II::MRMSrcMem: { unsigned FirstMemOp = CurOp + 1; + if (IsND) // Skip new data destination + CurOp++; + if (HasEVEX_K) // Skip writemask ++FirstMemOp; diff --git a/contrib/llvm-project/llvm/lib/Target/X86/X86.td b/contrib/llvm-project/llvm/lib/Target/X86/X86.td index 5fd6828f4312..e89ddcc570c9 100644 --- a/contrib/llvm-project/llvm/lib/Target/X86/X86.td +++ b/contrib/llvm-project/llvm/lib/Target/X86/X86.td @@ -1256,11 +1256,6 @@ def ProcessorFeatures { list<SubtargetFeature> SRFFeatures = !listconcat(ADLFeatures, SRFAdditionalFeatures); - // Grandridge - list<SubtargetFeature> GRRAdditionalFeatures = [FeatureRAOINT]; - list<SubtargetFeature> GRRFeatures = - !listconcat(SRFFeatures, GRRAdditionalFeatures); - // Arrowlake S list<SubtargetFeature> ARLSAdditionalFeatures = [FeatureAVXVNNIINT16, FeatureSHA512, @@ -1706,10 +1701,10 @@ foreach P = ["goldmont_plus", "goldmont-plus"] in { } def : ProcModel<"tremont", SLMModel, ProcessorFeatures.TRMFeatures, ProcessorFeatures.TRMTuning>; -def : ProcModel<"sierraforest", AlderlakePModel, ProcessorFeatures.SRFFeatures, - ProcessorFeatures.TRMTuning>; -def : ProcModel<"grandridge", AlderlakePModel, ProcessorFeatures.GRRFeatures, +foreach P = ["sierraforest", "grandridge"] in { + def : ProcModel<P, AlderlakePModel, ProcessorFeatures.SRFFeatures, ProcessorFeatures.TRMTuning>; +} // "Arrandale" along with corei3 and corei5 foreach P = ["nehalem", "corei7", "core_i7_sse4_2"] in { diff --git a/contrib/llvm-project/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp b/contrib/llvm-project/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp index 77a997588c4f..73b10cf3067e 100644 --- a/contrib/llvm-project/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp +++ b/contrib/llvm-project/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp @@ -487,7 +487,7 @@ namespace { // from PatFrags in tablegen. bool isUnneededShiftMask(SDNode *N, unsigned Width) const { assert(N->getOpcode() == ISD::AND && "Unexpected opcode"); - const APInt &Val = cast<ConstantSDNode>(N->getOperand(1))->getAPIntValue(); + const APInt &Val = N->getConstantOperandAPInt(1); if (Val.countr_one() >= Width) return true; @@ -5233,7 +5233,7 @@ void X86DAGToDAGISel::Select(SDNode *Node) { break; case X86ISD::VPTERNLOG: { - uint8_t Imm = cast<ConstantSDNode>(Node->getOperand(3))->getZExtValue(); + uint8_t Imm = Node->getConstantOperandVal(3); if (matchVPTERNLOG(Node, Node, Node, Node, Node->getOperand(0), Node->getOperand(1), Node->getOperand(2), Imm)) return; diff --git a/contrib/llvm-project/llvm/lib/Target/X86/X86ISelLowering.cpp b/contrib/llvm-project/llvm/lib/Target/X86/X86ISelLowering.cpp index 63bdf24d6b4f..1e4b1361f98a 100644 --- a/contrib/llvm-project/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/contrib/llvm-project/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -2267,6 +2267,8 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM, setOperationAction(ISD::FDIV, VT, Expand); setOperationAction(ISD::BUILD_VECTOR, VT, Custom); setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); + setOperationAction(ISD::INSERT_SUBVECTOR, VT, Legal); + setOperationAction(ISD::CONCAT_VECTORS, VT, Custom); } setOperationAction(ISD::FP_ROUND, MVT::v8bf16, Custom); addLegalFPImmediate(APFloat::getZero(APFloat::BFloat())); @@ -2282,6 +2284,8 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM, setOperationAction(ISD::BUILD_VECTOR, MVT::v32bf16, Custom); setOperationAction(ISD::FP_ROUND, MVT::v16bf16, Custom); setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v32bf16, Custom); + setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v32bf16, Legal); + setOperationAction(ISD::CONCAT_VECTORS, MVT::v32bf16, Custom); } if (!Subtarget.useSoftFloat() && Subtarget.hasVLX()) { @@ -3737,9 +3741,11 @@ static SDValue getZeroVector(MVT VT, const X86Subtarget &Subtarget, // type. This ensures they get CSE'd. But if the integer type is not // available, use a floating-point +0.0 instead. SDValue Vec; + const TargetLowering &TLI = DAG.getTargetLoweringInfo(); if (!Subtarget.hasSSE2() && VT.is128BitVector()) { Vec = DAG.getConstantFP(+0.0, dl, MVT::v4f32); - } else if (VT.isFloatingPoint()) { + } else if (VT.isFloatingPoint() && + TLI.isTypeLegal(VT.getVectorElementType())) { Vec = DAG.getConstantFP(+0.0, dl, VT); } else if (VT.getVectorElementType() == MVT::i1) { assert((Subtarget.hasBWI() || VT.getVectorNumElements() <= 16) && @@ -31752,7 +31758,7 @@ static SDValue LowerCVTPS2PH(SDValue Op, SelectionDAG &DAG) { static SDValue LowerPREFETCH(SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) { - unsigned IsData = cast<ConstantSDNode>(Op.getOperand(4))->getZExtValue(); + unsigned IsData = Op.getConstantOperandVal(4); // We don't support non-data prefetch without PREFETCHI. // Just preserve the chain. diff --git a/contrib/llvm-project/llvm/lib/Target/X86/X86InsertPrefetch.cpp b/contrib/llvm-project/llvm/lib/Target/X86/X86InsertPrefetch.cpp index 6c23928228d2..9aa70dff5f93 100644 --- a/contrib/llvm-project/llvm/lib/Target/X86/X86InsertPrefetch.cpp +++ b/contrib/llvm-project/llvm/lib/Target/X86/X86InsertPrefetch.cpp @@ -135,8 +135,7 @@ bool X86InsertPrefetch::findPrefetchInfo(const FunctionSamples *TopSamples, int64_t D = static_cast<int64_t>(S_V.second); unsigned IID = 0; for (const auto &HintType : HintTypes) { - if (Name.starts_with(HintType.first)) { - Name = Name.drop_front(HintType.first.size()); + if (Name.consume_front(HintType.first)) { IID = HintType.second; break; } diff --git a/contrib/llvm-project/llvm/lib/Target/X86/X86InstrAMX.td b/contrib/llvm-project/llvm/lib/Target/X86/X86InstrAMX.td index 7f3e193d9a1b..c47bee070e04 100644 --- a/contrib/llvm-project/llvm/lib/Target/X86/X86InstrAMX.td +++ b/contrib/llvm-project/llvm/lib/Target/X86/X86InstrAMX.td @@ -14,35 +14,45 @@ //===----------------------------------------------------------------------===// // AMX instructions -let Predicates = [HasAMXTILE, In64BitMode] in { - let SchedRW = [WriteSystem] in { - let hasSideEffects = 1, - Defs = [TMM0,TMM1,TMM2,TMM3,TMM4,TMM5,TMM6,TMM7] in - def LDTILECFG : I <0x49, MRM0m, (outs), (ins opaquemem:$src), - "ldtilecfg\t$src", - [(int_x86_ldtilecfg addr:$src)]>, VEX, T8; - let hasSideEffects = 1 in - def STTILECFG : I <0x49, MRM0m, (outs), (ins opaquemem:$src), - "sttilecfg\t$src", - [(int_x86_sttilecfg addr:$src)]>, VEX, T8, PD; - let mayLoad = 1 in - def TILELOADD : I<0x4b, MRMSrcMemFSIB, (outs TILE:$dst), - (ins sibmem:$src), - "tileloadd\t{$src, $dst|$dst, $src}", []>, - VEX, T8, XD; - let mayLoad = 1 in - def TILELOADDT1 : I<0x4b, MRMSrcMemFSIB, (outs TILE:$dst), - (ins sibmem:$src), - "tileloaddt1\t{$src, $dst|$dst, $src}", []>, - VEX, T8, PD; +multiclass AMX_TILE_COMMON<string Suffix, Predicate HasEGPR> { +let Predicates = [HasAMXTILE, HasEGPR, In64BitMode] in { + let hasSideEffects = 1, + Defs = [TMM0,TMM1,TMM2,TMM3,TMM4,TMM5,TMM6,TMM7] in + def LDTILECFG#Suffix : I<0x49, MRM0m, (outs), (ins opaquemem:$src), + "ldtilecfg\t$src", + [(int_x86_ldtilecfg addr:$src)]>, + T8, PS; + let hasSideEffects = 1 in + def STTILECFG#Suffix : I<0x49, MRM0m, (outs), (ins opaquemem:$src), + "sttilecfg\t$src", + [(int_x86_sttilecfg addr:$src)]>, + T8, PD; + let mayLoad = 1 in + def TILELOADD#Suffix : I<0x4b, MRMSrcMemFSIB, (outs TILE:$dst), + (ins sibmem:$src), + "tileloadd\t{$src, $dst|$dst, $src}", []>, + T8, XD; + let mayLoad = 1 in + def TILELOADDT1#Suffix : I<0x4b, MRMSrcMemFSIB, (outs TILE:$dst), + (ins sibmem:$src), + "tileloaddt1\t{$src, $dst|$dst, $src}", []>, + T8, PD; + let mayStore = 1 in + def TILESTORED#Suffix : I<0x4b, MRMDestMemFSIB, (outs), + (ins sibmem:$dst, TILE:$src), + "tilestored\t{$src, $dst|$dst, $src}", []>, + T8, XS; +} +} + +let SchedRW = [WriteSystem] in { + defm "" : AMX_TILE_COMMON<"", NoEGPR>, VEX; + defm "" : AMX_TILE_COMMON<"_EVEX", HasEGPR>, EVEX, NoCD8; + + let Predicates = [HasAMXTILE, In64BitMode] in { let Defs = [TMM0,TMM1,TMM2,TMM3,TMM4,TMM5,TMM6,TMM7] in def TILERELEASE : I<0x49, MRM_C0, (outs), (ins), - "tilerelease", [(int_x86_tilerelease)]>, VEX, T8; - let mayStore = 1 in - def TILESTORED : I<0x4b, MRMDestMemFSIB, (outs), - (ins sibmem:$dst, TILE:$src), - "tilestored\t{$src, $dst|$dst, $src}", []>, - VEX, T8, XS; + "tilerelease", [(int_x86_tilerelease)]>, VEX, T8, PS; def TILEZERO : I<0x49, MRMr0, (outs TILE:$dst), (ins), "tilezero\t$dst", []>, VEX, T8, XD; @@ -82,8 +92,8 @@ let Predicates = [HasAMXTILE, In64BitMode] in { def PTILEZERO : PseudoI<(outs), (ins u8imm:$src), [(int_x86_tilezero timm:$src)]>; } - } // SchedRW -} // HasAMXTILE + } // Predicates +} // SchedRW let Predicates = [HasAMXINT8, In64BitMode] in { let SchedRW = [WriteSystem] in { diff --git a/contrib/llvm-project/llvm/lib/Target/X86/X86InstrAVX512.td b/contrib/llvm-project/llvm/lib/Target/X86/X86InstrAVX512.td index 7c3c1d5fe42b..c3a673f97d34 100644 --- a/contrib/llvm-project/llvm/lib/Target/X86/X86InstrAVX512.td +++ b/contrib/llvm-project/llvm/lib/Target/X86/X86InstrAVX512.td @@ -1447,6 +1447,17 @@ def : Pat<(vselect_mask VK8WM:$mask, (VBROADCASTI32X4Z256rmk VR256X:$src0, VK8WM:$mask, addr:$src)>; } +let Predicates = [HasBF16] in { + def : Pat<(v32bf16 (X86SubVBroadcastld256 addr:$src)), + (VBROADCASTF64X4rm addr:$src)>; + def : Pat<(v32bf16 (X86SubVBroadcastld128 addr:$src)), + (VBROADCASTF32X4rm addr:$src)>; +} + +let Predicates = [HasBF16, HasVLX] in + def : Pat<(v16bf16 (X86SubVBroadcastld128 addr:$src)), + (VBROADCASTF32X4Z256rm addr:$src)>; + let Predicates = [HasVLX, HasDQI] in { defm VBROADCASTI64X2Z128 : avx512_subvec_broadcast_rm_dq<0x5a, "vbroadcasti64x2", X86SubVBroadcastld128, v4i64x_info, v2i64x_info>, VEX_W1X, diff --git a/contrib/llvm-project/llvm/lib/Target/X86/X86InstrArithmetic.td b/contrib/llvm-project/llvm/lib/Target/X86/X86InstrArithmetic.td index 936db48bb9df..6b0c1b8c28c9 100644 --- a/contrib/llvm-project/llvm/lib/Target/X86/X86InstrArithmetic.td +++ b/contrib/llvm-project/llvm/lib/Target/X86/X86InstrArithmetic.td @@ -44,591 +44,298 @@ def PLEA32r : PseudoI<(outs GR32:$dst), (ins anymem:$src), []>; def PLEA64r : PseudoI<(outs GR64:$dst), (ins anymem:$src), []>; } -// BinOpRR - Instructions that read "reg, reg". -class BinOpRR<bits<8> o, string m, X86TypeInfo t, dag out, list<dag> p> - : ITy<o, MRMDestReg, t, out, (ins t.RegClass:$src1, t.RegClass:$src2), m, - binop_args, p>, Sched<[WriteALU]>; -// BinOpRR_F - Instructions that read "reg, reg" and write EFLAGS only. -class BinOpRR_F<bits<8> o, string m, X86TypeInfo t, SDPatternOperator node> - : BinOpRR<o, m, t, (outs), - [(set EFLAGS, (node t.RegClass:$src1, t.RegClass:$src2))]>, - DefEFLAGS; -// BinOpRR_F_Rev - Reversed encoding of BinOpRR_F -class BinOpRR_F_Rev<bits<8> o, string m, X86TypeInfo t> - : BinOpRR_F<o, m, t, null_frag>, DisassembleOnly { - let Form = MRMSrcReg; -} -// BinOpRR_RF - Instructions that read "reg, reg", and write "reg", EFLAGS. -class BinOpRR_RF<bits<8> o, string m, X86TypeInfo t, SDPatternOperator node> - : BinOpRR<o, m, t, (outs t.RegClass:$dst), - [(set t.RegClass:$dst, EFLAGS, - (node t.RegClass:$src1, t.RegClass:$src2))]>, DefEFLAGS; -// BinOpRR_RF_Rev - Reversed encoding of BinOpRR_RF. -class BinOpRR_RF_Rev<bits<8> o, string m, X86TypeInfo t> - : BinOpRR_RF<o, m, t, null_frag>, DisassembleOnly { - let Form = MRMSrcReg; -} -// BinOpRRF_RF - Instructions that read "reg, reg", write "reg" and read/write -// EFLAGS. -class BinOpRRF_RF<bits<8> o, string m, X86TypeInfo t, SDPatternOperator node> - : BinOpRR<o, m, t, (outs t.RegClass:$dst), - [(set t.RegClass:$dst, EFLAGS, - (node t.RegClass:$src1, t.RegClass:$src2, - EFLAGS))]>, DefEFLAGS, UseEFLAGS { - let SchedRW = [WriteADC]; -} -// BinOpRRF_RF_Rev - Reversed encoding of BinOpRRF_RF -class BinOpRRF_RF_Rev<bits<8> o, string m, X86TypeInfo t> - : BinOpRRF_RF<o, m, t, null_frag>, DisassembleOnly { - let Form = MRMSrcReg; -} - -// BinOpRM - Instructions that read "reg, [mem]". -class BinOpRM<bits<8> o, string m, X86TypeInfo t, dag out, list<dag> p> - : ITy<o, MRMSrcMem, t, out, (ins t.RegClass:$src1, t.MemOperand:$src2), m, - binop_args, p>, - Sched<[WriteALU.Folded, WriteALU.ReadAfterFold]> { - let mayLoad = 1; -} -// BinOpRM_F - Instructions that read "reg, [mem]" and write EFLAGS only. -class BinOpRM_F<bits<8> o, string m, X86TypeInfo t, SDNode node> - : BinOpRM<o, m, t, (outs), - [(set EFLAGS, (node t.RegClass:$src1, - (t.LoadNode addr:$src2)))]>, DefEFLAGS; -// BinOpRM_RF - Instructions that read "reg, reg", and write "reg", EFLAGS. -class BinOpRM_RF<bits<8> o, string m, X86TypeInfo t, SDPatternOperator node> - : BinOpRM<o, m, t, (outs t.RegClass:$dst), - [(set t.RegClass:$dst, EFLAGS, (node t.RegClass:$src1, - (t.LoadNode addr:$src2)))]>, DefEFLAGS; -// BinOpRMF_RF - Instructions that read "reg, [mem]", write "reg" and read/write -// EFLAGS. -class BinOpRMF_RF<bits<8> o, string m, X86TypeInfo t, SDPatternOperator node> - : BinOpRM<o, m, t, (outs t.RegClass:$dst), - [(set t.RegClass:$dst, EFLAGS, - (node t.RegClass:$src1, (t.LoadNode addr:$src2), EFLAGS))]>, - DefEFLAGS, UseEFLAGS { - let SchedRW = [WriteADC.Folded, WriteADC.ReadAfterFold, - // base, scale, index, offset, segment. - ReadDefault, ReadDefault, ReadDefault, ReadDefault, ReadDefault, - // implicit register read. - WriteADC.ReadAfterFold]; -} - -// BinOpRI - Instructions that read "reg, imm". -class BinOpRI<bits<8> o, string m, X86TypeInfo t, Format f, dag out, list<dag> p> - : ITy<o, f, t, out, (ins t.RegClass:$src1, t.ImmOperand:$src2), m, - binop_args, p>, Sched<[WriteALU]> { - let ImmT = t.ImmEncoding; -} -// BinOpRI_F - Instructions that read "reg, imm" and write EFLAGS only. -class BinOpRI_F<bits<8> o, string m, X86TypeInfo t, SDPatternOperator node, - Format f> - : BinOpRI<o, m, t, f, (outs), - [(set EFLAGS, (node t.RegClass:$src1, - t.ImmOperator:$src2))]>, DefEFLAGS; -// BinOpRI_RF - Instructions that read "reg, imm" and write "reg", EFLAGS. -class BinOpRI_RF<bits<8> o, string m, X86TypeInfo t, SDNode node, Format f> - : BinOpRI<o, m, t, f, (outs t.RegClass:$dst), - [(set t.RegClass:$dst, EFLAGS, - (node t.RegClass:$src1, t.ImmOperator:$src2))]>, DefEFLAGS; -// BinOpRIF_RF - Instructions that read "reg, imm", write "reg" and read/write -// EFLAGS. -class BinOpRIF_RF<bits<8> o, string m, X86TypeInfo t, SDNode node, Format f> - : BinOpRI<o, m, t, f, (outs t.RegClass:$dst), - [(set t.RegClass:$dst, EFLAGS, - (node t.RegClass:$src1, t.ImmOperator:$src2, - EFLAGS))]>, DefEFLAGS, UseEFLAGS { - let SchedRW = [WriteADC]; -} -// BinOpRI8 - Instructions that read "reg, imm8". -class BinOpRI8<bits<8> o, string m, X86TypeInfo t, Format f, dag out> - : ITy<o, f, t, out, (ins t.RegClass:$src1, t.Imm8Operand:$src2), m, - binop_args, []>, Sched<[WriteALU]> { - let ImmT = Imm8; -} -// BinOpRI8_F - Instructions that read "reg, imm8" and write EFLAGS only. -class BinOpRI8_F<bits<8> o, string m, X86TypeInfo t, Format f> - : BinOpRI8<o, m, t, f, (outs)>, DefEFLAGS; -// BinOpRI8_RF - Instructions that read "reg, imm8" and write "reg", EFLAGS. -class BinOpRI8_RF<bits<8> o, string m, X86TypeInfo t, Format f> - : BinOpRI8<o, m, t, f, (outs t.RegClass:$dst)>, DefEFLAGS; -// BinOpRI8F_RF - Instructions that read "reg, imm", write "reg" and read/write -// EFLAGS. -class BinOpRI8F_RF<bits<8> o, string m, X86TypeInfo t, Format f> - : BinOpRI8<o, m, t, f, (outs t.RegClass:$dst)>, DefEFLAGS, UseEFLAGS { - let SchedRW = [WriteADC]; -} - -// BinOpMR - Instructions that read "[mem], reg". -class BinOpMR<bits<8> o, string m, X86TypeInfo t, list<dag> p> - : ITy<o, MRMDestMem, t, (outs), (ins t.MemOperand:$src1, t.RegClass:$src2), m, - binop_args, p> { - let mayLoad = 1; -} -// BinOpMR_F - Instructions that read "[mem], imm8" and write EFLAGS only. -class BinOpMR_F<bits<8> o, string m, X86TypeInfo t, SDPatternOperator node> - : BinOpMR<o, m, t, - [(set EFLAGS, (node (t.LoadNode addr:$src1), t.RegClass:$src2))]>, - Sched<[WriteALU.Folded, ReadDefault, ReadDefault, ReadDefault, - ReadDefault, ReadDefault, WriteALU.ReadAfterFold]>, DefEFLAGS; -// BinOpMR_MF - Instructions that read "[mem], reg" and write "[mem]", EFLAGS. -class BinOpMR_MF<bits<8> o, string m, X86TypeInfo t, SDNode node> - : BinOpMR<o, m, t, - [(store (node (load addr:$src1), t.RegClass:$src2), addr:$src1), - (implicit EFLAGS)]>, - Sched<[WriteALURMW, - // base, scale, index, offset, segment - ReadDefault, ReadDefault, ReadDefault, ReadDefault, ReadDefault, - WriteALU.ReadAfterFold]>, // reg - DefEFLAGS { - let mayStore = 1; -} -// BinOpMRF_MF - Instructions that read "[mem], reg", write "[mem]" and -// read/write EFLAGS. -class BinOpMRF_MF<bits<8> o, string m, X86TypeInfo t, SDNode node> - : BinOpMR<o, m, t, - [(store (node (load addr:$src1), t.RegClass:$src2, EFLAGS), - addr:$src1), (implicit EFLAGS)]>, - Sched<[WriteADCRMW, - // base, scale, index, offset, segment - ReadDefault, ReadDefault, ReadDefault, - ReadDefault, ReadDefault, - WriteALU.ReadAfterFold, // reg - WriteALU.ReadAfterFold]>, // EFLAGS - DefEFLAGS, UseEFLAGS { - let mayStore = 1; -} - -// BinOpMI - Instructions that read "[mem], imm". -class BinOpMI<bits<8> o, string m, X86TypeInfo t, Format f, list<dag> p> - : ITy<o, f, t, (outs), (ins t.MemOperand:$src1, t.ImmOperand:$src2), m, - binop_args, p> { - let ImmT = t.ImmEncoding; - let mayLoad = 1; -} -// BinOpMI_F - Instructions that read "[mem], imm" and write EFLAGS only. -class BinOpMI_F<bits<8> o, string m, X86TypeInfo t, SDPatternOperator node, - Format f> - : BinOpMI<o, m, t, f, - [(set EFLAGS, (node (t.LoadNode addr:$src1), t.ImmOperator:$src2))]>, - Sched<[WriteALU.Folded]>, DefEFLAGS; -// BinOpMI_MF - Instructions that read "[mem], imm" and write "[mem]", EFLAGS. -class BinOpMI_MF<bits<8> o, string m, X86TypeInfo t, SDNode node, Format f> - : BinOpMI<o, m, t, f, - [(store (node (t.VT (load addr:$src1)), - t.ImmOperator:$src2), addr:$src1), (implicit EFLAGS)]>, - Sched<[WriteALURMW]>, DefEFLAGS { - let mayStore = 1; -} -// BinOpMIF_MF - Instructions that read "[mem], imm", write "[mem]" and -// read/write EFLAGS. -class BinOpMIF_MF<bits<8> o, string m, X86TypeInfo t, SDNode node, Format f> - : BinOpMI<o, m, t, f, - [(store (node (t.VT (load addr:$src1)), - t.ImmOperator:$src2, EFLAGS), addr:$src1), (implicit EFLAGS)]>, - Sched<[WriteADCRMW]>, DefEFLAGS, UseEFLAGS { - let mayStore = 1; -} - -// BinOpMI8 - Instructions that read "[mem], imm8". -class BinOpMI8<string m, X86TypeInfo t, Format f> - : ITy<0x83, f, t, (outs), (ins t.MemOperand:$src1, t.Imm8Operand:$src2), m, - binop_args, []> { - let ImmT = Imm8; - let mayLoad = 1; -} -// BinOpMI8_F - Instructions that read "[mem], imm8" and write EFLAGS only. -class BinOpMI8_F<string m, X86TypeInfo t, Format f> - : BinOpMI8<m, t, f>, Sched<[WriteALU.Folded]>, DefEFLAGS; -// BinOpMI8_MF - Instructions that read "[mem], imm8" and write "[mem]", EFLAGS. -class BinOpMI8_MF<string m, X86TypeInfo t, Format f> - : BinOpMI8<m, t, f>, Sched<[WriteALURMW]>, DefEFLAGS { - let mayStore = 1; -} -// BinOpMI8F_MF - Instructions that read "[mem], imm8", write "[mem]" and -// read/write EFLAGS. -class BinOpMI8F_MF<string m, X86TypeInfo t, Format f> - : BinOpMI8<m, t, f>, Sched<[WriteADCRMW]>, DefEFLAGS, UseEFLAGS { - let mayStore = 1; -} - -// BinOpAI - Instructions that read "a-reg imm" (Accumulator register). -class BinOpAI<bits<8> o, string m, X86TypeInfo t, Register areg, string args> - : ITy<o, RawFrm, t, (outs), (ins t.ImmOperand:$src), m, args, []>, - Sched<[WriteALU]> { - let ImmT = t.ImmEncoding; - let Uses = [areg]; -} -// BinOpAI_F - Instructions that read "a-reg imm" and write EFLAGS only. -class BinOpAI_F<bits<8> o, string m, X86TypeInfo t, Register areg, string args> - : BinOpAI<o, m, t, areg, args>, DefEFLAGS; - -// BinOpAI_AF - Instructions that read "a-reg imm" and write a-reg/EFLAGS. -class BinOpAI_AF<bits<8> o, string m, X86TypeInfo t, Register areg, - string args> : BinOpAI<o, m, t, areg, args> { - let Defs = [areg, EFLAGS]; -} -// BinOpAIF_AF - Instructions that read "a-reg imm", write a-reg and read/write -// EFLAGS. -class BinOpAIF_AF<bits<8> o, string m, X86TypeInfo t, Register areg, - string args> : BinOpAI<o, m, t, areg, args> { - let Uses = [areg, EFLAGS]; - let Defs = [areg, EFLAGS]; - let SchedRW = [WriteADC]; +//===----------------------------------------------------------------------===// +// MUL/IMUL and DIV/IDIV Instructions +// +class MulDivOpR<bits<8> o, Format f, string m, X86TypeInfo t, + X86FoldableSchedWrite sched, list<dag> p> + : UnaryOpR<o, f, m, "$src1", t, (outs), p> { + let SchedRW = [sched]; } -// UnaryOpR - Instructions that read "reg" and write "reg". -class UnaryOpR<bits<8> o, Format f, string m, X86TypeInfo t, list<dag> p> - : ITy<o, f, t, (outs t.RegClass:$dst), - (ins t.RegClass:$src1), m, "$dst", p>, Sched<[WriteALU]>; - -// UnaryOpM - Instructions that read "[mem]" and writes "[mem]". -class UnaryOpM<bits<8> o, Format f, string m, X86TypeInfo t, list<dag> p> - : ITy<o, f, t, (outs), (ins t.MemOperand:$dst), m, "$dst", p>, - Sched<[WriteALURMW]> { - let mayLoad = 1; - let mayStore = 1; +class MulDivOpM<bits<8> o, Format f, string m, X86TypeInfo t, + X86FoldableSchedWrite sched, list<dag> p> + : UnaryOpM<o, f, m, "$src1", t, (outs), p> { + let SchedRW = + [sched.Folded, + // Memory operand. + ReadDefault, ReadDefault, ReadDefault, ReadDefault, ReadDefault, + // Register reads (implicit or explicit). + sched.ReadAfterFold, sched.ReadAfterFold]; } -// INCDECR - Instructions like "inc reg". -class INCDECR<Format f, string m, X86TypeInfo t, SDPatternOperator node> - : UnaryOpR<0xFF, f, m, t, - [(set t.RegClass:$dst, EFLAGS, (node t.RegClass:$src1, 1))]>, - DefEFLAGS { - let isConvertibleToThreeAddress = 1; // Can xform into LEA. +multiclass Mul<bits<8> o, string m, Format RegMRM, Format MemMRM, SDPatternOperator node> { + // AL is really implied by AX, but the registers in Defs must match the + // SDNode results (i8, i32). + // + // FIXME: Used for 8-bit mul, ignore result upper 8 bits. + // This probably ought to be moved to a def : Pat<> if the + // syntax can be accepted. + let Defs = [AL,EFLAGS,AX], Uses = [AL] in + def 8r : MulDivOpR<o, RegMRM, m, Xi8, WriteIMul8, + [(set AL, (node AL, GR8:$src1)), (implicit EFLAGS)]>; + let Defs = [AX,DX,EFLAGS], Uses = [AX] in + def 16r : MulDivOpR<o, RegMRM, m, Xi16, WriteIMul16, []>, OpSize16; + let Defs = [EAX,EDX,EFLAGS], Uses = [EAX] in + def 32r : MulDivOpR<o, RegMRM, m, Xi32, WriteIMul32, []>, OpSize32; + let Defs = [RAX,RDX,EFLAGS], Uses = [RAX] in + def 64r : MulDivOpR<o, RegMRM, m, Xi64, WriteIMul64, []>; + let Defs = [AL,EFLAGS,AX], Uses = [AL] in + def 8m : MulDivOpM<o, MemMRM, m, Xi8, WriteIMul8, + [(set AL, (node AL, (loadi8 addr:$src1))), (implicit EFLAGS)]>; + let Defs = [AX,DX,EFLAGS], Uses = [AX] in + def 16m : MulDivOpM<o, MemMRM, m, Xi16, WriteIMul16, []>, OpSize16; + let Defs = [EAX,EDX,EFLAGS], Uses = [EAX] in + def 32m : MulDivOpM<o, MemMRM, m, Xi32, WriteIMul32, []>, OpSize32; + let Defs = [RAX,RDX,EFLAGS], Uses = [RAX] in + def 64m : MulDivOpM<o, MemMRM, m, Xi64, WriteIMul64, []>, Requires<[In64BitMode]>; } -// INCDECM - Instructions like "inc [mem]". -class INCDECM<Format f, string m, X86TypeInfo t, int num> - : UnaryOpM<0xFF, f, m, t, - [(store (add (t.LoadNode addr:$dst), num), addr:$dst), - (implicit EFLAGS)]>, DefEFLAGS; - -// INCDECR_ALT - Instructions like "inc reg" short forms. -class INCDECR_ALT<bits<8> o, string m, X86TypeInfo t> - : UnaryOpR<o, AddRegFrm, m, t, []>, DefEFLAGS { - // Short forms only valid in 32-bit mode. Selected during MCInst lowering. - let Predicates = [Not64BitMode]; +defm MUL : Mul<0xF7, "mul", MRM4r, MRM4m, mul>; +defm IMUL : Mul<0xF7, "imul", MRM5r, MRM5m, null_frag>; + +multiclass Div<bits<8> o, string m, Format RegMRM, Format MemMRM> { + defvar sched8 = !if(!eq(m, "div"), WriteDiv8, WriteIDiv8); + defvar sched16 = !if(!eq(m, "div"), WriteDiv16, WriteIDiv16); + defvar sched32 = !if(!eq(m, "div"), WriteDiv32, WriteIDiv32); + defvar sched64 = !if(!eq(m, "div"), WriteDiv64, WriteIDiv64); + let Defs = [AL,AH,EFLAGS], Uses = [AX] in + def 8r : MulDivOpR<o, RegMRM, m, Xi8, sched8, []>; + let Defs = [AX,DX,EFLAGS], Uses = [AX,DX] in + def 16r : MulDivOpR<o, RegMRM, m, Xi16, sched16, []>, OpSize16; + let Defs = [EAX,EDX,EFLAGS], Uses = [EAX,EDX] in + def 32r : MulDivOpR<o, RegMRM, m, Xi32, sched32, []>, OpSize32; + let Defs = [RAX,RDX,EFLAGS], Uses = [RAX,RDX] in + def 64r : MulDivOpR<o, RegMRM, m, Xi64, sched64, []>; + let Defs = [AL,AH,EFLAGS], Uses = [AX] in + def 8m : MulDivOpM<o, MemMRM, m, Xi8, sched8, []>; + let Defs = [AX,DX,EFLAGS], Uses = [AX,DX] in + def 16m : MulDivOpM<o, MemMRM, m, Xi16, sched16, []>, OpSize16; + let Defs = [EAX,EDX,EFLAGS], Uses = [EAX,EDX] in + def 32m : MulDivOpM<o, MemMRM, m, Xi32, sched32, []>, OpSize32; + let Defs = [RAX,RDX,EFLAGS], Uses = [RAX,RDX] in + def 64m : MulDivOpM<o, MemMRM, m, Xi64, sched64, []>, Requires<[In64BitMode]>; } - -// MulOpR - Instructions like "mul reg". -class MulOpR<bits<8> o, Format f, string m, X86TypeInfo t, - X86FoldableSchedWrite sched, list<dag> p> - : ITy<o, f, t, (outs), (ins t.RegClass:$src), m, "$src", p>, Sched<[sched]>; - -// MulOpM - Instructions like "mul [mem]". -class MulOpM<bits<8> o, Format f, string m, X86TypeInfo t, - X86FoldableSchedWrite sched, list<dag> p> - : ITy<o, f, t, (outs), (ins t.MemOperand:$src), m, - "$src", p>, SchedLoadReg<sched> { - let mayLoad = 1; +let hasSideEffects = 1 in { // so that we don't speculatively execute +defm DIV: Div<0xF7, "div", MRM6r, MRM6m>; +defm IDIV: Div<0xF7, "idiv", MRM7r, MRM7m>; } -// NegOpR - Instructions like "neg reg". -class NegOpR<bits<8> o, string m, X86TypeInfo t> - : UnaryOpR<o, MRM3r, m, t, - [(set t.RegClass:$dst, (ineg t.RegClass:$src1)), - (implicit EFLAGS)]>, DefEFLAGS; - -// NegOpM - Instructions like "neg [mem]". -class NegOpM<bits<8> o, string m, X86TypeInfo t> - : UnaryOpM<o, MRM3m, m, t, - [(store (ineg (t.LoadNode addr:$dst)), addr:$dst), - (implicit EFLAGS)]>, DefEFLAGS; - -// NOTE: NOT does not set EFLAGS! -// NotOpR - Instructions like "not reg". -class NotOpR<bits<8> o, string m, X86TypeInfo t> - : UnaryOpR<o, MRM2r, m, t, [(set t.RegClass:$dst, (not t.RegClass:$src1))]>; - -// NotOpM - Instructions like "neg [mem]". -class NotOpM<bits<8> o, string m, X86TypeInfo t> - : UnaryOpM<o, MRM2m, m, t, - [(store (not (t.LoadNode addr:$dst)), addr:$dst)]>; - -// IMulOpRR - Instructions like "imul reg, reg, i8". -class IMulOpRR<bits<8> o, string m, X86TypeInfo t, X86FoldableSchedWrite sched> - : BinOpRR_RF<o, m, t, X86smul_flag>, TB { +class IMulOpRR<X86TypeInfo t, X86FoldableSchedWrite sched> + : BinOpRR_RF<0xAF, "imul", t, X86smul_flag>, TB { let Form = MRMSrcReg; let SchedRW = [sched]; // X = IMUL Y, Z --> X = IMUL Z, Y let isCommutable = 1; } - -// IMulOpRM - Instructions like "imul reg, reg, [mem]". -class IMulOpRM<bits<8> o, string m, X86TypeInfo t, X86FoldableSchedWrite sched> - : BinOpRM_RF<o, m, t, X86smul_flag>, TB { +class IMulOpRM<X86TypeInfo t, X86FoldableSchedWrite sched> + : BinOpRM_RF<0xAF, "imul", t, X86smul_flag>, TB { let Form = MRMSrcMem; let SchedRW = [sched.Folded, sched.ReadAfterFold]; } -// IMulOpRRI8 - Instructions like "imul reg, reg, i8". -class IMulOpRRI8<bits<8> o, string m, X86TypeInfo t, - X86FoldableSchedWrite sched> - : ITy<o, MRMSrcReg, t, (outs t.RegClass:$dst), - (ins t.RegClass:$src1, t.Imm8Operand:$src2), m, - "{$src2, $src1, $dst|$dst, $src1, $src2}", []>, Sched<[sched]>, DefEFLAGS { - let ImmT = Imm8; -} +def IMUL16rr : IMulOpRR<Xi16, WriteIMul16Reg>, OpSize16; +def IMUL32rr : IMulOpRR<Xi32, WriteIMul32Reg>, OpSize32; +def IMUL64rr : IMulOpRR<Xi64, WriteIMul64Reg>; +def IMUL16rm : IMulOpRM<Xi16, WriteIMul16Reg>, OpSize16; +def IMUL32rm : IMulOpRM<Xi32, WriteIMul32Reg>, OpSize32; +def IMUL64rm : IMulOpRM<Xi64, WriteIMul64Reg>; -// IMulOpRRI - Instructions like "imul reg, reg, i16/i32/i64". -class IMulOpRRI<bits<8> o, string m, X86TypeInfo t, - X86FoldableSchedWrite sched> - : ITy<o, MRMSrcReg, t, (outs t.RegClass:$dst), - (ins t.RegClass:$src1, t.ImmOperand:$src2), m, - "{$src2, $src1, $dst|$dst, $src1, $src2}", - [(set t.RegClass:$dst, EFLAGS, (X86smul_flag t.RegClass:$src1, - t.ImmNoSuOperator:$src2))]>, - Sched<[sched]>, DefEFLAGS { - let ImmT = t.ImmEncoding; +class IMulOpRI8_R<X86TypeInfo t, X86FoldableSchedWrite sched> + : BinOpRI8<0x6B, "imul", binop_ndd_args, t, MRMSrcReg, + (outs t.RegClass:$dst)>, DefEFLAGS { + let SchedRW = [sched]; } - -// IMulOpRMI8 - Instructions like "imul reg, [mem], i8". -class IMulOpRMI8<bits<8> o, string m, X86TypeInfo t, - X86FoldableSchedWrite sched> - : ITy<o, MRMSrcMem, t, (outs t.RegClass:$dst), - (ins t.MemOperand:$src1, t.Imm8Operand:$src2), m, - "{$src2, $src1, $dst|$dst, $src1, $src2}", []>, Sched<[sched.Folded]>, +class IMulOpRI_R<X86TypeInfo t, X86FoldableSchedWrite sched> + : BinOpRI<0x69, "imul", binop_ndd_args, t, MRMSrcReg, + (outs t.RegClass:$dst), + [(set t.RegClass:$dst, EFLAGS, (X86smul_flag t.RegClass:$src1, + t.ImmNoSuOperator:$src2))]>, DefEFLAGS { + let SchedRW = [sched]; +} +class IMulOpMI8_R<X86TypeInfo t, X86FoldableSchedWrite sched> + : BinOpMI8<"imul", binop_ndd_args, t, MRMSrcMem, (outs t.RegClass:$dst)>, DefEFLAGS { - let ImmT = Imm8; - let mayLoad = 1; + let Opcode = 0x6B; + let SchedRW = [sched.Folded]; } - -// IMulOpRMI - Instructions like "imul reg, [mem], i16/i32/i64". -class IMulOpRMI<bits<8> o, string m, X86TypeInfo t, - X86FoldableSchedWrite sched> - : ITy<o, MRMSrcMem, t, (outs t.RegClass:$dst), - (ins t.MemOperand:$src1, t.ImmOperand:$src2), m, - "{$src2, $src1, $dst|$dst, $src1, $src2}", - [(set t.RegClass:$dst, EFLAGS, - (X86smul_flag (t.LoadNode addr:$src1), t.ImmNoSuOperator:$src2))]>, - Sched<[sched.Folded]>, DefEFLAGS { - let ImmT = t.ImmEncoding; +class IMulOpMI_R<X86TypeInfo t, X86FoldableSchedWrite sched> + : BinOpMI<0x69, "imul", binop_ndd_args, t, MRMSrcMem, + (outs t.RegClass:$dst), + [(set t.RegClass:$dst, EFLAGS, (X86smul_flag (t.LoadNode addr:$src1), + t.ImmNoSuOperator:$src2))]>, + DefEFLAGS { + let SchedRW = [sched.Folded]; } +def IMUL16rri8 : IMulOpRI8_R<Xi16, WriteIMul16Imm>, OpSize16; +def IMUL32rri8 : IMulOpRI8_R<Xi32, WriteIMul32Imm>, OpSize32; +def IMUL64rri8 : IMulOpRI8_R<Xi64, WriteIMul64Imm>; +def IMUL16rri : IMulOpRI_R<Xi16, WriteIMul16Imm>, OpSize16; +def IMUL32rri : IMulOpRI_R<Xi32, WriteIMul32Imm>, OpSize32; +def IMUL64rri32 : IMulOpRI_R<Xi64, WriteIMul64Imm>; + +def IMUL16rmi8 : IMulOpMI8_R<Xi16, WriteIMul16Imm>, OpSize16; +def IMUL32rmi8 : IMulOpMI8_R<Xi32, WriteIMul32Imm>, OpSize32; +def IMUL64rmi8 : IMulOpMI8_R<Xi64, WriteIMul64Imm>; +def IMUL16rmi : IMulOpMI_R<Xi16, WriteIMul16Imm>, OpSize16; +def IMUL32rmi : IMulOpMI_R<Xi32, WriteIMul32Imm>, OpSize32; +def IMUL64rmi32 : IMulOpMI_R<Xi64, WriteIMul64Imm>; -let Constraints = "$src1 = $dst" in { -def INC16r_alt : INCDECR_ALT<0x40, "inc", Xi16>, OpSize16; -def INC32r_alt : INCDECR_ALT<0x40, "inc", Xi32>, OpSize32; -def INC8r : INCDECR<MRM0r, "inc", Xi8, X86add_flag_nocf>; -def INC16r : INCDECR<MRM0r, "inc", Xi16, X86add_flag_nocf>, OpSize16; -def INC32r : INCDECR<MRM0r, "inc", Xi32, X86add_flag_nocf>, OpSize32; -def INC64r : INCDECR<MRM0r, "inc", Xi64, X86add_flag_nocf>; - -def DEC16r_alt : INCDECR_ALT<0x48, "dec", Xi16>, OpSize16; -def DEC32r_alt : INCDECR_ALT<0x48, "dec", Xi32>, OpSize32; -def DEC8r : INCDECR<MRM1r, "dec", Xi8, X86sub_flag_nocf>; -def DEC16r : INCDECR<MRM1r, "dec", Xi16, X86sub_flag_nocf>, OpSize16; -def DEC32r : INCDECR<MRM1r, "dec", Xi32, X86sub_flag_nocf>, OpSize32; -def DEC64r : INCDECR<MRM1r, "dec", Xi64, X86sub_flag_nocf>; +//===----------------------------------------------------------------------===// +// INC and DEC Instructions +// +class IncOpR_RF<X86TypeInfo t> : UnaryOpR_RF<0xFF, MRM0r, "inc", t, null_frag> { + let Pattern = [(set t.RegClass:$dst, EFLAGS, + (X86add_flag_nocf t.RegClass:$src1, 1))]; +} +class DecOpR_RF<X86TypeInfo t> : UnaryOpR_RF<0xFF, MRM1r, "dec", t, null_frag> { + let Pattern = [(set t.RegClass:$dst, EFLAGS, + (X86sub_flag_nocf t.RegClass:$src1, 1))]; +} +class IncOpM_M<X86TypeInfo t> : UnaryOpM_MF<0xFF, MRM0m, "inc", t, null_frag> { + let Pattern = [(store (add (t.LoadNode addr:$src1), 1), addr:$src1), + (implicit EFLAGS)]; +} +class DecOpM_M<X86TypeInfo t> : UnaryOpM_MF<0xFF, MRM1m, "dec", t, null_frag> { + let Pattern = [(store (add (t.LoadNode addr:$src1), -1), addr:$src1), + (implicit EFLAGS)]; +} +// IncDec_Alt - Instructions like "inc reg" short forms. +// Short forms only valid in 32-bit mode. Selected during MCInst lowering. +class IncDec_Alt<bits<8> o, string m, X86TypeInfo t> + : UnaryOpR_RF<o, AddRegFrm, m, t, null_frag>, Requires<[Not64BitMode]>; + +let isConvertibleToThreeAddress = 1 in { +def INC16r_alt : IncDec_Alt<0x40, "inc", Xi16>, OpSize16; +def INC32r_alt : IncDec_Alt<0x40, "inc", Xi32>, OpSize32; +def DEC16r_alt : IncDec_Alt<0x48, "dec", Xi16>, OpSize16; +def DEC32r_alt : IncDec_Alt<0x48, "dec", Xi32>, OpSize32; +def INC8r : IncOpR_RF<Xi8>; +def INC16r : IncOpR_RF<Xi16>, OpSize16; +def INC32r : IncOpR_RF<Xi32>, OpSize32; +def INC64r : IncOpR_RF<Xi64>; +def DEC8r : DecOpR_RF<Xi8>; +def DEC16r : DecOpR_RF<Xi16>, OpSize16; +def DEC32r : DecOpR_RF<Xi32>, OpSize32; +def DEC64r : DecOpR_RF<Xi64>; } - let Predicates = [UseIncDec] in { -def INC8m : INCDECM<MRM0m, "inc", Xi8, 1>; -def INC16m : INCDECM<MRM0m, "inc", Xi16, 1>, OpSize16; -def INC32m : INCDECM<MRM0m, "inc", Xi32, 1>, OpSize32; -def DEC8m : INCDECM<MRM1m, "dec", Xi8, -1>; -def DEC16m : INCDECM<MRM1m, "dec", Xi16, -1>, OpSize16; -def DEC32m : INCDECM<MRM1m, "dec", Xi32, -1>, OpSize32; +def INC8m : IncOpM_M<Xi8>; +def INC16m : IncOpM_M<Xi16>, OpSize16; +def INC32m : IncOpM_M<Xi32>, OpSize32; +def DEC8m : DecOpM_M<Xi8>; +def DEC16m : DecOpM_M<Xi16>, OpSize16; +def DEC32m : DecOpM_M<Xi32>, OpSize32; } let Predicates = [UseIncDec, In64BitMode] in { -def INC64m : INCDECM<MRM0m, "inc", Xi64, 1>; -def DEC64m : INCDECM<MRM1m, "dec", Xi64, -1>; +def INC64m : IncOpM_M<Xi64>; +def DEC64m : DecOpM_M<Xi64>; } -// Extra precision multiplication - -// AL is really implied by AX, but the registers in Defs must match the -// SDNode results (i8, i32). -// AL,AH = AL*GR8 -let Defs = [AL,EFLAGS,AX], Uses = [AL] in -def MUL8r : MulOpR<0xF6, MRM4r, "mul", Xi8, WriteIMul8, - // FIXME: Used for 8-bit mul, ignore result upper 8 bits. - // This probably ought to be moved to a def : Pat<> if the - // syntax can be accepted. - [(set AL, (mul AL, GR8:$src)), (implicit EFLAGS)]>; -// AX,DX = AX*GR16 -let Defs = [AX,DX,EFLAGS], Uses = [AX] in -def MUL16r : MulOpR<0xF7, MRM4r, "mul", Xi16, WriteIMul16, []>, OpSize16; -// EAX,EDX = EAX*GR32 -let Defs = [EAX,EDX,EFLAGS], Uses = [EAX] in -def MUL32r : MulOpR<0xF7, MRM4r, "mul", Xi32, WriteIMul32, - [/*(set EAX, EDX, EFLAGS, (X86umul_flag EAX, GR32:$src))*/]>, OpSize32; -// RAX,RDX = RAX*GR64 -let Defs = [RAX,RDX,EFLAGS], Uses = [RAX] in -def MUL64r : MulOpR<0xF7, MRM4r, "mul", Xi64, WriteIMul64, - [/*(set RAX, RDX, EFLAGS, (X86umul_flag RAX, GR64:$src))*/]>; -// AL,AH = AL*[mem8] -let Defs = [AL,EFLAGS,AX], Uses = [AL] in -def MUL8m : MulOpM<0xF6, MRM4m, "mul", Xi8, WriteIMul8, - // FIXME: Used for 8-bit mul, ignore result upper 8 bits. - // This probably ought to be moved to a def : Pat<> if the - // syntax can be accepted. - [(set AL, (mul AL, (loadi8 addr:$src))), - (implicit EFLAGS)]>; -// AX,DX = AX*[mem16] -let Defs = [AX,DX,EFLAGS], Uses = [AX] in -def MUL16m : MulOpM<0xF7, MRM4m, "mul", Xi16, WriteIMul16, []>, OpSize16; -// EAX,EDX = EAX*[mem32] -let Defs = [EAX,EDX,EFLAGS], Uses = [EAX] in -def MUL32m : MulOpM<0xF7, MRM4m, "mul", Xi32, WriteIMul32, []>, OpSize32; -// RAX,RDX = RAX*[mem64] -let Defs = [RAX,RDX,EFLAGS], Uses = [RAX] in -def MUL64m : MulOpM<0xF7, MRM4m, "mul", Xi64, WriteIMul64, []>, - Requires<[In64BitMode]>; - -// AL,AH = AL*GR8 -let Defs = [AL,EFLAGS,AX], Uses = [AL] in -def IMUL8r : MulOpR<0xF6, MRM5r, "imul", Xi8, WriteIMul8, []>; -// AX,DX = AX*GR16 -let Defs = [AX,DX,EFLAGS], Uses = [AX] in -def IMUL16r : MulOpR<0xF7, MRM5r, "imul", Xi16, WriteIMul16, []>, OpSize16; -// EAX,EDX = EAX*GR32 -let Defs = [EAX,EDX,EFLAGS], Uses = [EAX] in -def IMUL32r : MulOpR<0xF7, MRM5r, "imul", Xi32, WriteIMul32, []>, OpSize32; -// RAX,RDX = RAX*GR64 -let Defs = [RAX,RDX,EFLAGS], Uses = [RAX] in -def IMUL64r : MulOpR<0xF7, MRM5r, "imul", Xi64, WriteIMul64, []>; - -// AL,AH = AL*[mem8] -let Defs = [AL,EFLAGS,AX], Uses = [AL] in -def IMUL8m : MulOpM<0xF6, MRM5m, "imul", Xi8, WriteIMul8, []>; -// AX,DX = AX*[mem16] -let Defs = [AX,DX,EFLAGS], Uses = [AX] in -def IMUL16m : MulOpM<0xF7, MRM5m, "imul", Xi16, WriteIMul16, []>, OpSize16; -// EAX,EDX = EAX*[mem32] -let Defs = [EAX,EDX,EFLAGS], Uses = [EAX] in -def IMUL32m : MulOpM<0xF7, MRM5m, "imul", Xi32, WriteIMul32, []>, OpSize32; -// RAX,RDX = RAX*[mem64] -let Defs = [RAX,RDX,EFLAGS], Uses = [RAX] in -def IMUL64m : MulOpM<0xF7, MRM5m, "imul", Xi64, WriteIMul64, []>, - Requires<[In64BitMode]>; - -let Constraints = "$src1 = $dst" in { -// Register-Register Signed Integer Multiply -def IMUL16rr : IMulOpRR<0xAF, "imul", Xi16, WriteIMul16Reg>, OpSize16; -def IMUL32rr : IMulOpRR<0xAF, "imul", Xi32, WriteIMul32Reg>, OpSize32; -def IMUL64rr : IMulOpRR<0xAF, "imul", Xi64, WriteIMul64Reg>; - -// Register-Memory Signed Integer Multiply -def IMUL16rm : IMulOpRM<0xAF, "imul", Xi16, WriteIMul16Reg>, OpSize16; -def IMUL32rm : IMulOpRM<0xAF, "imul", Xi32, WriteIMul32Reg>, OpSize32; -def IMUL64rm : IMulOpRM<0xAF, "imul", Xi64, WriteIMul64Reg>; +//===----------------------------------------------------------------------===// +// NEG and NOT Instructions +// +class NegOpR_R<X86TypeInfo t, bit ndd = 0> + : UnaryOpR_R<0xF7, MRM3r, "neg", t, ineg, ndd>; +class NegOpR_RF<X86TypeInfo t, bit ndd = 0> + : UnaryOpR_RF<0xF7, MRM3r, "neg", t, ineg, ndd>; +class NegOpM_M<X86TypeInfo t> : UnaryOpM_M<0xF7, MRM3m, "neg", t, null_frag>; +class NegOpM_MF<X86TypeInfo t> : UnaryOpM_MF<0xF7, MRM3m, "neg", t, ineg>; +class NegOpM_R<X86TypeInfo t> : UnaryOpM_R<0xF7, MRM3m, "neg", t, null_frag>; +class NegOpM_RF<X86TypeInfo t> : UnaryOpM_RF<0xF7, MRM3m, "neg", t, ineg>; + +class NotOpR_R<X86TypeInfo t, bit ndd = 0> + : UnaryOpR_R<0xF7, MRM2r, "not", t, not, ndd>; +class NotOpM_M<X86TypeInfo t> : UnaryOpM_M<0xF7, MRM2m, "not", t, not>; +class NotOpM_R<X86TypeInfo t> : UnaryOpM_R<0xF7, MRM2m, "not", t, not>; + +let Predicates = [NoNDD] in { +def NEG8r : NegOpR_RF<Xi8>; +def NEG16r : NegOpR_RF<Xi16>, OpSize16; +def NEG32r : NegOpR_RF<Xi32>, OpSize32; +def NEG64r : NegOpR_RF<Xi64>; +def NOT8r : NotOpR_R<Xi8>; +def NOT16r : NotOpR_R<Xi16>, OpSize16; +def NOT32r : NotOpR_R<Xi32>, OpSize32; +def NOT64r : NotOpR_R<Xi64>; } -// Surprisingly enough, these are not two address instructions! -// NOTE: These are order specific, we want the ri8 forms to be listed -// first so that they are slightly preferred to the ri forms. - -// Register-Integer Signed Integer Multiply -// GR16 = GR16*I8 -def IMUL16rri8 : IMulOpRRI8<0x6B, "imul", Xi16, WriteIMul16Imm>, OpSize16; -// GR16 = GR16*I16 -def IMUL16rri : IMulOpRRI<0x69, "imul", Xi16, WriteIMul16Imm>, OpSize16; -// GR32 = GR32*I8 -def IMUL32rri8 : IMulOpRRI8<0x6B, "imul", Xi32, WriteIMul32Imm>, OpSize32; -// GR32 = GR32*I32 -def IMUL32rri : IMulOpRRI<0x69, "imul", Xi32, WriteIMul32Imm>, OpSize32; -// GR64 = GR64*I8 -def IMUL64rri8 : IMulOpRRI8<0x6B, "imul", Xi64, WriteIMul64Imm>; -// GR64 = GR64*I32 -def IMUL64rri32 : IMulOpRRI<0x69, "imul", Xi64, WriteIMul64Imm>; - -// Memory-Integer Signed Integer Multiply -// GR16 = [mem16]*I8 -def IMUL16rmi8 : IMulOpRMI8<0x6B, "imul", Xi16, WriteIMul16Imm>, OpSize16; -// GR16 = [mem16]*I16 -def IMUL16rmi : IMulOpRMI<0x69, "imul", Xi16, WriteIMul16Imm>, OpSize16; -// GR32 = [mem32]*I8 -def IMUL32rmi8 : IMulOpRMI8<0x6B, "imul", Xi32, WriteIMul32Imm>, OpSize32; -// GR32 = [mem32]*I32 -def IMUL32rmi : IMulOpRMI<0x69, "imul", Xi32, WriteIMul32Imm>, OpSize32; -// GR64 = [mem64]*I8 -def IMUL64rmi8 : IMulOpRMI8<0x6B, "imul", Xi64, WriteIMul64Imm>; -// GR64 = [mem64]*I32 -def IMUL64rmi32 : IMulOpRMI<0x69, "imul", Xi64, WriteIMul64Imm>; - -// unsigned division/remainder -let hasSideEffects = 1 in { // so that we don't speculatively execute -let Defs = [AL,AH,EFLAGS], Uses = [AX] in -// AX/r8 = AL,AH -def DIV8r : MulOpR<0xF6, MRM6r, "div", Xi8, WriteDiv8, []>; -let Defs = [AX,DX,EFLAGS], Uses = [AX,DX] in -// DX:AX/r16 = AX,DX -def DIV16r : MulOpR<0xF7, MRM6r, "div", Xi16, WriteDiv16, []>, OpSize16; -let Defs = [EAX,EDX,EFLAGS], Uses = [EAX,EDX] in -// EDX:EAX/r32 = EAX,EDX -def DIV32r : MulOpR<0xF7, MRM6r, "div", Xi32, WriteDiv32, []>, OpSize32; -// RDX:RAX/r64 = RAX,RDX -let Defs = [RAX,RDX,EFLAGS], Uses = [RAX,RDX] in -def DIV64r : MulOpR<0xF7, MRM6r, "div", Xi64, WriteDiv64, []>; - -let mayLoad = 1 in { -let Defs = [AL,AH,EFLAGS], Uses = [AX] in -// AX/[mem8] = AL,AH -def DIV8m : MulOpM<0xF6, MRM6m, "div", Xi8, WriteDiv8, []>; -let Defs = [AX,DX,EFLAGS], Uses = [AX,DX] in -// DX:AX/[mem16] = AX,DX -def DIV16m : MulOpM<0xF7, MRM6m, "div", Xi16, WriteDiv16, []>, OpSize16; -let Defs = [EAX,EDX,EFLAGS], Uses = [EAX,EDX] in // EDX:EAX/[mem32] = EAX,EDX -def DIV32m : MulOpM<0xF7, MRM6m, "div", Xi32, WriteDiv32, []>, OpSize32; -// RDX:RAX/[mem64] = RAX,RDX -let Defs = [RAX,RDX,EFLAGS], Uses = [RAX,RDX] in -def DIV64m : MulOpM<0xF7, MRM6m, "div", Xi64, WriteDiv64, []>, - Requires<[In64BitMode]>; +let Predicates = [HasNDD, In64BitMode] in { +def NEG8r_ND : NegOpR_RF<Xi8, 1>; +def NEG16r_ND : NegOpR_RF<Xi16, 1>, PD; +def NEG32r_ND : NegOpR_RF<Xi32, 1>; +def NEG64r_ND : NegOpR_RF<Xi64, 1>; + +def NOT8r_ND : NotOpR_R<Xi8, 1>; +def NOT16r_ND : NotOpR_R<Xi16, 1>, PD; +def NOT32r_ND : NotOpR_R<Xi32, 1>; +def NOT64r_ND : NotOpR_R<Xi64, 1>; + +def NEG8r_NF_ND : NegOpR_R<Xi8, 1>, EVEX_NF; +def NEG16r_NF_ND : NegOpR_R<Xi16, 1>, EVEX_NF, PD; +def NEG32r_NF_ND : NegOpR_R<Xi32, 1>, EVEX_NF; +def NEG64r_NF_ND : NegOpR_R<Xi64, 1>, EVEX_NF; } -// Signed division/remainder. -let Defs = [AL,AH,EFLAGS], Uses = [AX] in -// AX/r8 = AL,AH -def IDIV8r : MulOpR<0xF6, MRM7r, "idiv", Xi8, WriteIDiv8, []>; -let Defs = [AX,DX,EFLAGS], Uses = [AX,DX] in -// DX:AX/r16 = AX,DX -def IDIV16r: MulOpR<0xF7, MRM7r, "idiv", Xi16, WriteIDiv16, []>, OpSize16; -let Defs = [EAX,EDX,EFLAGS], Uses = [EAX,EDX] in -// EDX:EAX/r32 = EAX,EDX -def IDIV32r: MulOpR<0xF7, MRM7r, "idiv", Xi32, WriteIDiv32, []>, OpSize32; -// RDX:RAX/r64 = RAX,RDX -let Defs = [RAX,RDX,EFLAGS], Uses = [RAX,RDX] in -def IDIV64r: MulOpR<0xF7, MRM7r, "idiv", Xi64, WriteIDiv64, []>; - -let Defs = [AL,AH,EFLAGS], Uses = [AX] in -// AX/[mem8] = AL,AH -def IDIV8m : MulOpM<0xF6, MRM7m, "idiv", Xi8, WriteIDiv8, []>; -let Defs = [AX,DX,EFLAGS], Uses = [AX,DX] in -// DX:AX/[mem16] = AX,DX -def IDIV16m: MulOpM<0xF7, MRM7m, "idiv", Xi16, WriteIDiv16, []>, OpSize16; -let Defs = [EAX,EDX,EFLAGS], Uses = [EAX,EDX] in -// EDX:EAX/[mem32] = EAX,EDX -def IDIV32m: MulOpM<0xF7, MRM7m, "idiv", Xi32, WriteIDiv32, []>, OpSize32; -let Defs = [RAX,RDX,EFLAGS], Uses = [RAX,RDX] in // RDX:RAX/[mem64] = RAX,RDX -// RDX:RAX/[mem64] = RAX,RDX -def IDIV64m: MulOpM<0xF7, MRM7m, "idiv", Xi64, WriteIDiv64, []>, - Requires<[In64BitMode]>; -} // hasSideEffects = 1 - -let Constraints = "$src1 = $dst" in { -def NEG8r : NegOpR<0xF6, "neg", Xi8>; -def NEG16r : NegOpR<0xF7, "neg", Xi16>, OpSize16; -def NEG32r : NegOpR<0xF7, "neg", Xi32>, OpSize32; -def NEG64r : NegOpR<0xF7, "neg", Xi64>; +def NEG8m : NegOpM_MF<Xi8>; +def NEG16m : NegOpM_MF<Xi16>, OpSize16; +def NEG32m : NegOpM_MF<Xi32>, OpSize32; +def NEG64m : NegOpM_MF<Xi64>, Requires<[In64BitMode]>; + +let Predicates = [HasNDD, In64BitMode] in { +def NEG8m_ND : NegOpM_RF<Xi8>; +def NEG16m_ND : NegOpM_RF<Xi16>, PD; +def NEG32m_ND : NegOpM_RF<Xi32>; +def NEG64m_ND : NegOpM_RF<Xi64>; + +def NEG8m_NF_ND : NegOpM_R<Xi8>, EVEX_NF; +def NEG16m_NF_ND : NegOpM_R<Xi16>, EVEX_NF, PD; +def NEG32m_NF_ND : NegOpM_R<Xi32>, EVEX_NF; +def NEG64m_NF_ND : NegOpM_R<Xi64>, EVEX_NF; } -def NEG8m : NegOpM<0xF6, "neg", Xi8>; -def NEG16m : NegOpM<0xF7, "neg", Xi16>, OpSize16; -def NEG32m : NegOpM<0xF7, "neg", Xi32>, OpSize32; -def NEG64m : NegOpM<0xF7, "neg", Xi64>, Requires<[In64BitMode]>; +def NOT8m : NotOpM_M<Xi8>; +def NOT16m : NotOpM_M<Xi16>, OpSize16; +def NOT32m : NotOpM_M<Xi32>, OpSize32; +def NOT64m : NotOpM_M<Xi64>, Requires<[In64BitMode]>; -let Constraints = "$src1 = $dst" in { -def NOT8r : NotOpR<0xF6, "not", Xi8>; -def NOT16r : NotOpR<0xF7, "not", Xi16>, OpSize16; -def NOT32r : NotOpR<0xF7, "not", Xi32>, OpSize32; -def NOT64r : NotOpR<0xF7, "not", Xi64>; +let Predicates = [HasNDD, In64BitMode] in { +def NOT8m_ND : NotOpM_R<Xi8>; +def NOT16m_ND : NotOpM_R<Xi16>, PD; +def NOT32m_ND : NotOpM_R<Xi32>; +def NOT64m_ND : NotOpM_R<Xi64>; } -def NOT8m : NotOpM<0xF6, "not", Xi8>; -def NOT16m : NotOpM<0xF7, "not", Xi16>, OpSize16; -def NOT32m : NotOpM<0xF7, "not", Xi32>, OpSize32; -def NOT64m : NotOpM<0xF7, "not", Xi64>, Requires<[In64BitMode]>; +let Predicates = [In64BitMode], Pattern = [(null_frag)] in { +def NEG8r_NF : NegOpR_R<Xi8>, NF; +def NEG16r_NF : NegOpR_R<Xi16>, NF, PD; +def NEG32r_NF : NegOpR_R<Xi32>, NF; +def NEG64r_NF : NegOpR_R<Xi64>, NF; +def NEG8m_NF : NegOpM_M<Xi8>, NF; +def NEG16m_NF : NegOpM_M<Xi16>, NF, PD; +def NEG32m_NF : NegOpM_M<Xi32>, NF; +def NEG64m_NF : NegOpM_M<Xi64>, NF; + +def NEG8r_EVEX : NegOpR_RF<Xi8>, PL; +def NEG16r_EVEX : NegOpR_RF<Xi16>, PL, PD; +def NEG32r_EVEX : NegOpR_RF<Xi32>, PL; +def NEG64r_EVEX : NegOpR_RF<Xi64>, PL; + +def NOT8r_EVEX : NotOpR_R<Xi8>, PL; +def NOT16r_EVEX : NotOpR_R<Xi16>, PL, PD; +def NOT32r_EVEX : NotOpR_R<Xi32>, PL; +def NOT64r_EVEX : NotOpR_R<Xi64>, PL; + +def NEG8m_EVEX : NegOpM_MF<Xi8>, PL; +def NEG16m_EVEX : NegOpM_MF<Xi16>, PL, PD; +def NEG32m_EVEX : NegOpM_MF<Xi32>, PL; +def NEG64m_EVEX : NegOpM_MF<Xi64>, PL; + +def NOT8m_EVEX : NotOpM_M<Xi8>, PL; +def NOT16m_EVEX : NotOpM_M<Xi16>, PL, PD; +def NOT32m_EVEX : NotOpM_M<Xi32>, PL; +def NOT64m_EVEX : NotOpM_M<Xi64>, PL; +} /// ArithBinOp_RF - This is an arithmetic binary operator where the pattern is /// defined with "(set GPR:$dst, EFLAGS, (...". @@ -640,61 +347,204 @@ multiclass ArithBinOp_RF<bits<8> BaseOpc, bits<8> BaseOpc2, bits<8> BaseOpc4, SDNode opnodeflag, SDNode opnode, bit CommutableRR, bit ConvertibleToThreeAddress, bit ConvertibleToThreeAddressRR> { - let Constraints = "$src1 = $dst" in { - let isCommutable = CommutableRR, - isConvertibleToThreeAddress = ConvertibleToThreeAddressRR in { - def NAME#8rr : BinOpRR_RF<BaseOpc, mnemonic, Xi8 , opnodeflag>; - def NAME#16rr : BinOpRR_RF<BaseOpc, mnemonic, Xi16, opnodeflag>, OpSize16; - def NAME#32rr : BinOpRR_RF<BaseOpc, mnemonic, Xi32, opnodeflag>, OpSize32; - def NAME#64rr : BinOpRR_RF<BaseOpc, mnemonic, Xi64, opnodeflag>; + let isCommutable = CommutableRR, + isConvertibleToThreeAddress = ConvertibleToThreeAddressRR in { + let Predicates = [NoNDD] in { + def NAME#8rr : BinOpRR_RF<BaseOpc, mnemonic, Xi8 , opnodeflag>; + def NAME#16rr : BinOpRR_RF<BaseOpc, mnemonic, Xi16, opnodeflag>, OpSize16; + def NAME#32rr : BinOpRR_RF<BaseOpc, mnemonic, Xi32, opnodeflag>, OpSize32; + def NAME#64rr : BinOpRR_RF<BaseOpc, mnemonic, Xi64, opnodeflag>; + } + let Predicates = [HasNDD, In64BitMode] in { + def NAME#8rr_ND : BinOpRR_RF<BaseOpc, mnemonic, Xi8 , opnodeflag, 1>; + def NAME#16rr_ND : BinOpRR_RF<BaseOpc, mnemonic, Xi16, opnodeflag, 1>, PD; + def NAME#32rr_ND : BinOpRR_RF<BaseOpc, mnemonic, Xi32, opnodeflag, 1>; + def NAME#64rr_ND : BinOpRR_RF<BaseOpc, mnemonic, Xi64, opnodeflag, 1>; + def NAME#8rr_NF_ND : BinOpRR_R<BaseOpc, mnemonic, Xi8, 1>, EVEX_NF; + def NAME#16rr_NF_ND : BinOpRR_R<BaseOpc, mnemonic, Xi16, 1>, EVEX_NF, PD; + def NAME#32rr_NF_ND : BinOpRR_R<BaseOpc, mnemonic, Xi32, 1>, EVEX_NF; + def NAME#64rr_NF_ND : BinOpRR_R<BaseOpc, mnemonic, Xi64, 1>, EVEX_NF; + } + let Predicates = [In64BitMode] in { + def NAME#8rr_NF : BinOpRR_R<BaseOpc, mnemonic, Xi8>, NF; + def NAME#16rr_NF : BinOpRR_R<BaseOpc, mnemonic, Xi16>, NF, PD; + def NAME#32rr_NF : BinOpRR_R<BaseOpc, mnemonic, Xi32>, NF; + def NAME#64rr_NF : BinOpRR_R<BaseOpc, mnemonic, Xi64>, NF; + def NAME#8rr_EVEX : BinOpRR_RF<BaseOpc, mnemonic, Xi8 , null_frag>, PL; + def NAME#16rr_EVEX : BinOpRR_RF<BaseOpc, mnemonic, Xi16, null_frag>, PL, PD; + def NAME#32rr_EVEX : BinOpRR_RF<BaseOpc, mnemonic, Xi32, null_frag>, PL; + def NAME#64rr_EVEX : BinOpRR_RF<BaseOpc, mnemonic, Xi64, null_frag>, PL; } + } - def NAME#8rr_REV : BinOpRR_RF_Rev<BaseOpc2, mnemonic, Xi8>; - def NAME#16rr_REV : BinOpRR_RF_Rev<BaseOpc2, mnemonic, Xi16>, OpSize16; - def NAME#32rr_REV : BinOpRR_RF_Rev<BaseOpc2, mnemonic, Xi32>, OpSize32; - def NAME#64rr_REV : BinOpRR_RF_Rev<BaseOpc2, mnemonic, Xi64>; + def NAME#8rr_REV : BinOpRR_RF_Rev<BaseOpc2, mnemonic, Xi8>; + def NAME#16rr_REV : BinOpRR_RF_Rev<BaseOpc2, mnemonic, Xi16>, OpSize16; + def NAME#32rr_REV : BinOpRR_RF_Rev<BaseOpc2, mnemonic, Xi32>, OpSize32; + def NAME#64rr_REV : BinOpRR_RF_Rev<BaseOpc2, mnemonic, Xi64>; + let Predicates = [In64BitMode] in { + def NAME#8rr_EVEX_REV : BinOpRR_RF_Rev<BaseOpc2, mnemonic, Xi8>, PL; + def NAME#16rr_EVEX_REV : BinOpRR_RF_Rev<BaseOpc2, mnemonic, Xi16>, PL, PD; + def NAME#32rr_EVEX_REV : BinOpRR_RF_Rev<BaseOpc2, mnemonic, Xi32>, PL; + def NAME#64rr_EVEX_REV : BinOpRR_RF_Rev<BaseOpc2, mnemonic, Xi64>, PL; + def NAME#8rr_ND_REV : BinOpRR_RF_Rev<BaseOpc2, mnemonic, Xi8, 1>; + def NAME#16rr_ND_REV : BinOpRR_RF_Rev<BaseOpc2, mnemonic, Xi16, 1>, PD; + def NAME#32rr_ND_REV : BinOpRR_RF_Rev<BaseOpc2, mnemonic, Xi32, 1>; + def NAME#64rr_ND_REV : BinOpRR_RF_Rev<BaseOpc2, mnemonic, Xi64, 1>; + def NAME#8rr_NF_REV : BinOpRR_R_Rev<BaseOpc2, mnemonic, Xi8>, NF; + def NAME#16rr_NF_REV : BinOpRR_R_Rev<BaseOpc2, mnemonic, Xi16>, NF, PD; + def NAME#32rr_NF_REV : BinOpRR_R_Rev<BaseOpc2, mnemonic, Xi32>, NF; + def NAME#64rr_NF_REV : BinOpRR_R_Rev<BaseOpc2, mnemonic, Xi64>, NF; + def NAME#8rr_NF_ND_REV : BinOpRR_R_Rev<BaseOpc2, mnemonic, Xi8, 1>, EVEX_NF; + def NAME#16rr_NF_ND_REV : BinOpRR_R_Rev<BaseOpc2, mnemonic, Xi16, 1>, EVEX_NF, PD; + def NAME#32rr_NF_ND_REV : BinOpRR_R_Rev<BaseOpc2, mnemonic, Xi32, 1>, EVEX_NF; + def NAME#64rr_NF_ND_REV : BinOpRR_R_Rev<BaseOpc2, mnemonic, Xi64, 1>, EVEX_NF; + } - def NAME#8rm : BinOpRM_RF<BaseOpc2, mnemonic, Xi8 , opnodeflag>; - def NAME#16rm : BinOpRM_RF<BaseOpc2, mnemonic, Xi16, opnodeflag>, OpSize16; - def NAME#32rm : BinOpRM_RF<BaseOpc2, mnemonic, Xi32, opnodeflag>, OpSize32; - def NAME#64rm : BinOpRM_RF<BaseOpc2, mnemonic, Xi64, opnodeflag>; + let Predicates = [NoNDD] in { + def NAME#8rm : BinOpRM_RF<BaseOpc2, mnemonic, Xi8 , opnodeflag>; + def NAME#16rm : BinOpRM_RF<BaseOpc2, mnemonic, Xi16, opnodeflag>, OpSize16; + def NAME#32rm : BinOpRM_RF<BaseOpc2, mnemonic, Xi32, opnodeflag>, OpSize32; + def NAME#64rm : BinOpRM_RF<BaseOpc2, mnemonic, Xi64, opnodeflag>; + } + let Predicates = [HasNDD, In64BitMode] in { + def NAME#8rm_ND : BinOpRM_RF<BaseOpc2, mnemonic, Xi8 , opnodeflag, 1>; + def NAME#16rm_ND : BinOpRM_RF<BaseOpc2, mnemonic, Xi16, opnodeflag, 1>, PD; + def NAME#32rm_ND : BinOpRM_RF<BaseOpc2, mnemonic, Xi32, opnodeflag, 1>; + def NAME#64rm_ND : BinOpRM_RF<BaseOpc2, mnemonic, Xi64, opnodeflag, 1>; + def NAME#8rm_NF_ND : BinOpRM_R<BaseOpc2, mnemonic, Xi8, 1>, EVEX_NF; + def NAME#16rm_NF_ND : BinOpRM_R<BaseOpc2, mnemonic, Xi16, 1>, EVEX_NF, PD; + def NAME#32rm_NF_ND : BinOpRM_R<BaseOpc2, mnemonic, Xi32, 1>, EVEX_NF; + def NAME#64rm_NF_ND : BinOpRM_R<BaseOpc2, mnemonic, Xi64, 1>, EVEX_NF; + } + let Predicates = [In64BitMode] in { + def NAME#8rm_NF : BinOpRM_R<BaseOpc2, mnemonic, Xi8>, NF; + def NAME#16rm_NF : BinOpRM_R<BaseOpc2, mnemonic, Xi16>, NF, PD; + def NAME#32rm_NF : BinOpRM_R<BaseOpc2, mnemonic, Xi32>, NF; + def NAME#64rm_NF : BinOpRM_R<BaseOpc2, mnemonic, Xi64>, NF; + def NAME#8rm_EVEX : BinOpRM_RF<BaseOpc2, mnemonic, Xi8 , null_frag>, PL; + def NAME#16rm_EVEX : BinOpRM_RF<BaseOpc2, mnemonic, Xi16, null_frag>, PL, PD; + def NAME#32rm_EVEX : BinOpRM_RF<BaseOpc2, mnemonic, Xi32, null_frag>, PL; + def NAME#64rm_EVEX : BinOpRM_RF<BaseOpc2, mnemonic, Xi64, null_frag>, PL; + } let isConvertibleToThreeAddress = ConvertibleToThreeAddress in { - def NAME#8ri : BinOpRI_RF<0x80, mnemonic, Xi8 , opnodeflag, RegMRM>; - // NOTE: These are order specific, we want the ri8 forms to be listed - // first so that they are slightly preferred to the ri forms. - def NAME#16ri8 : BinOpRI8_RF<0x83, mnemonic, Xi16, RegMRM>, OpSize16; - def NAME#32ri8 : BinOpRI8_RF<0x83, mnemonic, Xi32, RegMRM>, OpSize32; - def NAME#64ri8 : BinOpRI8_RF<0x83, mnemonic, Xi64, RegMRM>; - - def NAME#16ri : BinOpRI_RF<0x81, mnemonic, Xi16, opnodeflag, RegMRM>, OpSize16; - def NAME#32ri : BinOpRI_RF<0x81, mnemonic, Xi32, opnodeflag, RegMRM>, OpSize32; - def NAME#64ri32: BinOpRI_RF<0x81, mnemonic, Xi64, opnodeflag, RegMRM>; + let Predicates = [NoNDD] in { + // NOTE: These are order specific, we want the ri8 forms to be listed + // first so that they are slightly preferred to the ri forms. + def NAME#16ri8 : BinOpRI8_RF<0x83, mnemonic, Xi16, RegMRM>, OpSize16; + def NAME#32ri8 : BinOpRI8_RF<0x83, mnemonic, Xi32, RegMRM>, OpSize32; + def NAME#64ri8 : BinOpRI8_RF<0x83, mnemonic, Xi64, RegMRM>; + def NAME#8ri : BinOpRI_RF<0x80, mnemonic, Xi8 , opnodeflag, RegMRM>; + def NAME#16ri : BinOpRI_RF<0x81, mnemonic, Xi16, opnodeflag, RegMRM>, OpSize16; + def NAME#32ri : BinOpRI_RF<0x81, mnemonic, Xi32, opnodeflag, RegMRM>, OpSize32; + def NAME#64ri32: BinOpRI_RF<0x81, mnemonic, Xi64, opnodeflag, RegMRM>; + } + let Predicates = [HasNDD, In64BitMode] in { + def NAME#16ri8_ND : BinOpRI8_RF<0x83, mnemonic, Xi16, RegMRM, 1>, PD; + def NAME#32ri8_ND : BinOpRI8_RF<0x83, mnemonic, Xi32, RegMRM, 1>; + def NAME#64ri8_ND : BinOpRI8_RF<0x83, mnemonic, Xi64, RegMRM, 1>; + def NAME#8ri_ND : BinOpRI_RF<0x80, mnemonic, Xi8 , opnodeflag, RegMRM, 1>; + def NAME#16ri_ND : BinOpRI_RF<0x81, mnemonic, Xi16, opnodeflag, RegMRM, 1>, PD; + def NAME#32ri_ND : BinOpRI_RF<0x81, mnemonic, Xi32, opnodeflag, RegMRM, 1>; + def NAME#64ri32_ND: BinOpRI_RF<0x81, mnemonic, Xi64, opnodeflag, RegMRM, 1>; + def NAME#16ri8_NF_ND : BinOpRI8_R<0x83, mnemonic, Xi16, RegMRM, 1>, EVEX_NF, PD; + def NAME#32ri8_NF_ND : BinOpRI8_R<0x83, mnemonic, Xi32, RegMRM, 1>, EVEX_NF; + def NAME#64ri8_NF_ND : BinOpRI8_R<0x83, mnemonic, Xi64, RegMRM, 1>, EVEX_NF; + def NAME#8ri_NF_ND : BinOpRI_R<0x80, mnemonic, Xi8, RegMRM, 1>, EVEX_NF; + def NAME#16ri_NF_ND : BinOpRI_R<0x81, mnemonic, Xi16, RegMRM, 1>, EVEX_NF, PD; + def NAME#32ri_NF_ND : BinOpRI_R<0x81, mnemonic, Xi32, RegMRM, 1>, EVEX_NF; + def NAME#64ri32_NF_ND : BinOpRI_R<0x81, mnemonic, Xi64, RegMRM, 1>, EVEX_NF; + } + let Predicates = [In64BitMode] in { + def NAME#16ri8_NF : BinOpRI8_R<0x83, mnemonic, Xi16, RegMRM>, NF, PD; + def NAME#32ri8_NF : BinOpRI8_R<0x83, mnemonic, Xi32, RegMRM>, NF; + def NAME#64ri8_NF : BinOpRI8_R<0x83, mnemonic, Xi64, RegMRM>, NF; + def NAME#8ri_NF : BinOpRI_R<0x80, mnemonic, Xi8, RegMRM>, NF; + def NAME#16ri_NF : BinOpRI_R<0x81, mnemonic, Xi16, RegMRM>, NF, PD; + def NAME#32ri_NF : BinOpRI_R<0x81, mnemonic, Xi32, RegMRM>, NF; + def NAME#64ri32_NF : BinOpRI_R<0x81, mnemonic, Xi64, RegMRM>, NF; + def NAME#16ri8_EVEX : BinOpRI8_RF<0x83, mnemonic, Xi16, RegMRM>, PL, PD; + def NAME#32ri8_EVEX : BinOpRI8_RF<0x83, mnemonic, Xi32, RegMRM>, PL; + def NAME#64ri8_EVEX : BinOpRI8_RF<0x83, mnemonic, Xi64, RegMRM>, PL; + def NAME#8ri_EVEX : BinOpRI_RF<0x80, mnemonic, Xi8 , null_frag, RegMRM>, PL; + def NAME#16ri_EVEX : BinOpRI_RF<0x81, mnemonic, Xi16, null_frag, RegMRM>, PL, PD; + def NAME#32ri_EVEX : BinOpRI_RF<0x81, mnemonic, Xi32, null_frag, RegMRM>, PL; + def NAME#64ri32_EVEX: BinOpRI_RF<0x81, mnemonic, Xi64, null_frag, RegMRM>, PL; + } } - } // Constraints = "$src1 = $dst" - def NAME#8mr : BinOpMR_MF<BaseOpc, mnemonic, Xi8 , opnode>; - def NAME#16mr : BinOpMR_MF<BaseOpc, mnemonic, Xi16, opnode>, OpSize16; - def NAME#32mr : BinOpMR_MF<BaseOpc, mnemonic, Xi32, opnode>, OpSize32; - def NAME#64mr : BinOpMR_MF<BaseOpc, mnemonic, Xi64, opnode>; + def NAME#8mr : BinOpMR_MF<BaseOpc, mnemonic, Xi8 , opnode>; + def NAME#16mr : BinOpMR_MF<BaseOpc, mnemonic, Xi16, opnode>, OpSize16; + def NAME#32mr : BinOpMR_MF<BaseOpc, mnemonic, Xi32, opnode>, OpSize32; + def NAME#64mr : BinOpMR_MF<BaseOpc, mnemonic, Xi64, opnode>; + let Predicates = [HasNDD, In64BitMode] in { + def NAME#8mr_ND : BinOpMR_RF<BaseOpc, mnemonic, Xi8 , opnode>; + def NAME#16mr_ND : BinOpMR_RF<BaseOpc, mnemonic, Xi16, opnode>, PD; + def NAME#32mr_ND : BinOpMR_RF<BaseOpc, mnemonic, Xi32, opnode>; + def NAME#64mr_ND : BinOpMR_RF<BaseOpc, mnemonic, Xi64, opnode>; + def NAME#8mr_NF_ND : BinOpMR_R<BaseOpc, mnemonic, Xi8>, EVEX_NF; + def NAME#16mr_NF_ND : BinOpMR_R<BaseOpc, mnemonic, Xi16>, EVEX_NF, PD; + def NAME#32mr_NF_ND : BinOpMR_R<BaseOpc, mnemonic, Xi32>, EVEX_NF; + def NAME#64mr_NF_ND : BinOpMR_R<BaseOpc, mnemonic, Xi64>, EVEX_NF; + } + let Predicates = [In64BitMode] in { + def NAME#8mr_NF : BinOpMR_M<BaseOpc, mnemonic, Xi8>, NF; + def NAME#16mr_NF : BinOpMR_M<BaseOpc, mnemonic, Xi16>, NF, PD; + def NAME#32mr_NF : BinOpMR_M<BaseOpc, mnemonic, Xi32>, NF; + def NAME#64mr_NF : BinOpMR_M<BaseOpc, mnemonic, Xi64>, NF; + def NAME#8mr_EVEX : BinOpMR_MF<BaseOpc, mnemonic, Xi8 , null_frag>, PL; + def NAME#16mr_EVEX : BinOpMR_MF<BaseOpc, mnemonic, Xi16, null_frag>, PL, PD; + def NAME#32mr_EVEX : BinOpMR_MF<BaseOpc, mnemonic, Xi32, null_frag>, PL; + def NAME#64mr_EVEX : BinOpMR_MF<BaseOpc, mnemonic, Xi64, null_frag>, PL; + } // NOTE: These are order specific, we want the mi8 forms to be listed // first so that they are slightly preferred to the mi forms. def NAME#16mi8 : BinOpMI8_MF<mnemonic, Xi16, MemMRM>, OpSize16; def NAME#32mi8 : BinOpMI8_MF<mnemonic, Xi32, MemMRM>, OpSize32; let Predicates = [In64BitMode] in - def NAME#64mi8 : BinOpMI8_MF<mnemonic, Xi64, MemMRM>; - + def NAME#64mi8 : BinOpMI8_MF<mnemonic, Xi64, MemMRM>; def NAME#8mi : BinOpMI_MF<0x80, mnemonic, Xi8 , opnode, MemMRM>; def NAME#16mi : BinOpMI_MF<0x81, mnemonic, Xi16, opnode, MemMRM>, OpSize16; def NAME#32mi : BinOpMI_MF<0x81, mnemonic, Xi32, opnode, MemMRM>, OpSize32; let Predicates = [In64BitMode] in - def NAME#64mi32 : BinOpMI_MF<0x81, mnemonic, Xi64, opnode, MemMRM>; + def NAME#64mi32 : BinOpMI_MF<0x81, mnemonic, Xi64, opnode, MemMRM>; + let Predicates = [HasNDD, In64BitMode] in { + def NAME#16mi8_ND : BinOpMI8_RF<mnemonic, Xi16, MemMRM>, PD; + def NAME#32mi8_ND : BinOpMI8_RF<mnemonic, Xi32, MemMRM>; + def NAME#64mi8_ND : BinOpMI8_RF<mnemonic, Xi64, MemMRM>; + def NAME#8mi_ND : BinOpMI_RF<0x80, mnemonic, Xi8 , opnode, MemMRM>; + def NAME#16mi_ND : BinOpMI_RF<0x81, mnemonic, Xi16, opnode, MemMRM>, PD; + def NAME#32mi_ND : BinOpMI_RF<0x81, mnemonic, Xi32, opnode, MemMRM>; + def NAME#64mi32_ND : BinOpMI_RF<0x81, mnemonic, Xi64, opnode, MemMRM>; + def NAME#16mi8_NF_ND : BinOpMI8_R<mnemonic, Xi16, MemMRM>, NF, PD; + def NAME#32mi8_NF_ND : BinOpMI8_R<mnemonic, Xi32, MemMRM>, NF; + def NAME#64mi8_NF_ND : BinOpMI8_R<mnemonic, Xi64, MemMRM>, NF; + def NAME#8mi_NF_ND : BinOpMI_R<0x80, mnemonic, Xi8, MemMRM>, NF; + def NAME#16mi_NF_ND : BinOpMI_R<0x81, mnemonic, Xi16, MemMRM>, NF, PD; + def NAME#32mi_NF_ND : BinOpMI_R<0x81, mnemonic, Xi32, MemMRM>, NF; + def NAME#64mi32_NF_ND : BinOpMI_R<0x81, mnemonic, Xi64, MemMRM>, NF; + } + let Predicates = [In64BitMode] in { + def NAME#16mi8_NF : BinOpMI8_M<mnemonic, Xi16, MemMRM>, NF, PD; + def NAME#32mi8_NF : BinOpMI8_M<mnemonic, Xi32, MemMRM>, NF; + def NAME#64mi8_NF : BinOpMI8_M<mnemonic, Xi64, MemMRM>, NF; + def NAME#8mi_NF : BinOpMI_M<0x80, mnemonic, Xi8, MemMRM>, NF; + def NAME#16mi_NF : BinOpMI_M<0x81, mnemonic, Xi16, MemMRM>, NF, PD; + def NAME#32mi_NF : BinOpMI_M<0x81, mnemonic, Xi32, MemMRM>, NF; + def NAME#64mi32_NF : BinOpMI_M<0x81, mnemonic, Xi64, MemMRM>, NF; + def NAME#16mi8_EVEX : BinOpMI8_MF<mnemonic, Xi16, MemMRM>, PL, PD; + def NAME#32mi8_EVEX : BinOpMI8_MF<mnemonic, Xi32, MemMRM>, PL; + def NAME#64mi8_EVEX : BinOpMI8_MF<mnemonic, Xi64, MemMRM>, PL; + def NAME#8mi_EVEX : BinOpMI_MF<0x80, mnemonic, Xi8 , null_frag, MemMRM>, PL; + def NAME#16mi_EVEX : BinOpMI_MF<0x81, mnemonic, Xi16, null_frag, MemMRM>, PL, PD; + def NAME#32mi_EVEX : BinOpMI_MF<0x81, mnemonic, Xi32, null_frag, MemMRM>, PL; + def NAME#64mi32_EVEX : BinOpMI_MF<0x81, mnemonic, Xi64, null_frag, MemMRM>, PL; + } // These are for the disassembler since 0x82 opcode behaves like 0x80, but // not in 64-bit mode. let Predicates = [Not64BitMode] in { - let Constraints = "$src1 = $dst" in def NAME#8ri8 : BinOpRI8_RF<0x82, mnemonic, Xi8, RegMRM>, DisassembleOnly; def NAME#8mi8 : BinOpMI8_MF<mnemonic, Xi8, MemMRM>, DisassembleOnly; } @@ -719,62 +569,153 @@ multiclass ArithBinOp_RFF<bits<8> BaseOpc, bits<8> BaseOpc2, bits<8> BaseOpc4, string mnemonic, Format RegMRM, Format MemMRM, SDNode opnode, bit CommutableRR, bit ConvertibleToThreeAddress> { - let Constraints = "$src1 = $dst" in { - let isCommutable = CommutableRR in { - def NAME#8rr : BinOpRRF_RF<BaseOpc, mnemonic, Xi8 , opnode>; + let isCommutable = CommutableRR in { + let Predicates = [NoNDD] in { + def NAME#8rr : BinOpRRF_RF<BaseOpc, mnemonic, Xi8 , opnode>; let isConvertibleToThreeAddress = ConvertibleToThreeAddress in { - def NAME#16rr : BinOpRRF_RF<BaseOpc, mnemonic, Xi16, opnode>, OpSize16; - def NAME#32rr : BinOpRRF_RF<BaseOpc, mnemonic, Xi32, opnode>, OpSize32; - def NAME#64rr : BinOpRRF_RF<BaseOpc, mnemonic, Xi64, opnode>; - } // isConvertibleToThreeAddress + def NAME#16rr : BinOpRRF_RF<BaseOpc, mnemonic, Xi16, opnode>, OpSize16; + def NAME#32rr : BinOpRRF_RF<BaseOpc, mnemonic, Xi32, opnode>, OpSize32; + def NAME#64rr : BinOpRRF_RF<BaseOpc, mnemonic, Xi64, opnode>; + } + } + let Predicates = [HasNDD, In64BitMode] in { + def NAME#8rr_ND : BinOpRRF_RF<BaseOpc, mnemonic, Xi8 , opnode, 1>; + let isConvertibleToThreeAddress = ConvertibleToThreeAddress in { + def NAME#16rr_ND : BinOpRRF_RF<BaseOpc, mnemonic, Xi16, opnode, 1>, PD; + def NAME#32rr_ND : BinOpRRF_RF<BaseOpc, mnemonic, Xi32, opnode, 1>; + def NAME#64rr_ND : BinOpRRF_RF<BaseOpc, mnemonic, Xi64, opnode, 1>; + } + } } // isCommutable + let Predicates = [In64BitMode] in { + def NAME#8rr_EVEX : BinOpRRF_RF<BaseOpc, mnemonic, Xi8 , null_frag>, PL; + def NAME#16rr_EVEX : BinOpRRF_RF<BaseOpc, mnemonic, Xi16, null_frag>, PL, PD; + def NAME#32rr_EVEX : BinOpRRF_RF<BaseOpc, mnemonic, Xi32, null_frag>, PL; + def NAME#64rr_EVEX : BinOpRRF_RF<BaseOpc, mnemonic, Xi64, null_frag>, PL; + } + def NAME#8rr_REV : BinOpRRF_RF_Rev<BaseOpc2, mnemonic, Xi8>; def NAME#16rr_REV : BinOpRRF_RF_Rev<BaseOpc2, mnemonic, Xi16>, OpSize16; def NAME#32rr_REV : BinOpRRF_RF_Rev<BaseOpc2, mnemonic, Xi32>, OpSize32; def NAME#64rr_REV : BinOpRRF_RF_Rev<BaseOpc2, mnemonic, Xi64>; + let Predicates = [In64BitMode] in { + def NAME#8rr_ND_REV : BinOpRRF_RF_Rev<BaseOpc2, mnemonic, Xi8, 1>; + def NAME#16rr_ND_REV : BinOpRRF_RF_Rev<BaseOpc2, mnemonic, Xi16, 1>, PD; + def NAME#32rr_ND_REV : BinOpRRF_RF_Rev<BaseOpc2, mnemonic, Xi32, 1>; + def NAME#64rr_ND_REV : BinOpRRF_RF_Rev<BaseOpc2, mnemonic, Xi64, 1>; + def NAME#8rr_EVEX_REV : BinOpRRF_RF_Rev<BaseOpc2, mnemonic, Xi8>, PL; + def NAME#16rr_EVEX_REV : BinOpRRF_RF_Rev<BaseOpc2, mnemonic, Xi16>, PL, PD; + def NAME#32rr_EVEX_REV : BinOpRRF_RF_Rev<BaseOpc2, mnemonic, Xi32>, PL; + def NAME#64rr_EVEX_REV : BinOpRRF_RF_Rev<BaseOpc2, mnemonic, Xi64>, PL; + } + + let Predicates = [NoNDD] in { + def NAME#8rm : BinOpRMF_RF<BaseOpc2, mnemonic, Xi8 , opnode>; + def NAME#16rm : BinOpRMF_RF<BaseOpc2, mnemonic, Xi16, opnode>, OpSize16; + def NAME#32rm : BinOpRMF_RF<BaseOpc2, mnemonic, Xi32, opnode>, OpSize32; + def NAME#64rm : BinOpRMF_RF<BaseOpc2, mnemonic, Xi64, opnode>; + } + let Predicates = [HasNDD, In64BitMode] in { + def NAME#8rm_ND : BinOpRMF_RF<BaseOpc2, mnemonic, Xi8 , opnode, 1>; + def NAME#16rm_ND : BinOpRMF_RF<BaseOpc2, mnemonic, Xi16, opnode, 1>, PD; + def NAME#32rm_ND : BinOpRMF_RF<BaseOpc2, mnemonic, Xi32, opnode, 1>; + def NAME#64rm_ND : BinOpRMF_RF<BaseOpc2, mnemonic, Xi64, opnode, 1>; + } + let Predicates = [In64BitMode] in { + def NAME#8rm_EVEX : BinOpRMF_RF<BaseOpc2, mnemonic, Xi8 , opnode>, PL; + def NAME#16rm_EVEX : BinOpRMF_RF<BaseOpc2, mnemonic, Xi16, opnode>, PL, PD; + def NAME#32rm_EVEX : BinOpRMF_RF<BaseOpc2, mnemonic, Xi32, opnode>, PL; + def NAME#64rm_EVEX : BinOpRMF_RF<BaseOpc2, mnemonic, Xi64, opnode>, PL; + } - def NAME#8rm : BinOpRMF_RF<BaseOpc2, mnemonic, Xi8 , opnode>; - def NAME#16rm : BinOpRMF_RF<BaseOpc2, mnemonic, Xi16, opnode>, OpSize16; - def NAME#32rm : BinOpRMF_RF<BaseOpc2, mnemonic, Xi32, opnode>, OpSize32; - def NAME#64rm : BinOpRMF_RF<BaseOpc2, mnemonic, Xi64, opnode>; + let Predicates = [NoNDD] in { + def NAME#8ri : BinOpRIF_RF<0x80, mnemonic, Xi8 , opnode, RegMRM>; + let isConvertibleToThreeAddress = ConvertibleToThreeAddress in { + // NOTE: These are order specific, we want the ri8 forms to be listed + // first so that they are slightly preferred to the ri forms. + def NAME#16ri8 : BinOpRI8F_RF<0x83, mnemonic, Xi16, RegMRM>, OpSize16; + def NAME#32ri8 : BinOpRI8F_RF<0x83, mnemonic, Xi32, RegMRM>, OpSize32; + def NAME#64ri8 : BinOpRI8F_RF<0x83, mnemonic, Xi64, RegMRM>; + + def NAME#16ri : BinOpRIF_RF<0x81, mnemonic, Xi16, opnode, RegMRM>, OpSize16; + def NAME#32ri : BinOpRIF_RF<0x81, mnemonic, Xi32, opnode, RegMRM>, OpSize32; + def NAME#64ri32: BinOpRIF_RF<0x81, mnemonic, Xi64, opnode, RegMRM>; + } + } - def NAME#8ri : BinOpRIF_RF<0x80, mnemonic, Xi8 , opnode, RegMRM>; + let Predicates = [HasNDD, In64BitMode] in { + def NAME#8ri_ND : BinOpRIF_RF<0x80, mnemonic, Xi8 , opnode, RegMRM, 1>; let isConvertibleToThreeAddress = ConvertibleToThreeAddress in { - // NOTE: These are order specific, we want the ri8 forms to be listed - // first so that they are slightly preferred to the ri forms. - def NAME#16ri8 : BinOpRI8F_RF<0x83, mnemonic, Xi16, RegMRM>, OpSize16; - def NAME#32ri8 : BinOpRI8F_RF<0x83, mnemonic, Xi32, RegMRM>, OpSize32; - def NAME#64ri8 : BinOpRI8F_RF<0x83, mnemonic, Xi64, RegMRM>; - - def NAME#16ri : BinOpRIF_RF<0x81, mnemonic, Xi16, opnode, RegMRM>, OpSize16; - def NAME#32ri : BinOpRIF_RF<0x81, mnemonic, Xi32, opnode, RegMRM>, OpSize32; - def NAME#64ri32: BinOpRIF_RF<0x81, mnemonic, Xi64, opnode, RegMRM>; + def NAME#16ri8_ND : BinOpRI8F_RF<0x83, mnemonic, Xi16, RegMRM, 1>, PD; + def NAME#32ri8_ND : BinOpRI8F_RF<0x83, mnemonic, Xi32, RegMRM, 1>; + def NAME#64ri8_ND : BinOpRI8F_RF<0x83, mnemonic, Xi64, RegMRM, 1>; + def NAME#16ri_ND : BinOpRIF_RF<0x81, mnemonic, Xi16, opnode, RegMRM, 1>, PD; + def NAME#32ri_ND : BinOpRIF_RF<0x81, mnemonic, Xi32, opnode, RegMRM, 1>; + def NAME#64ri32_ND: BinOpRIF_RF<0x81, mnemonic, Xi64, opnode, RegMRM, 1>; } - } // Constraints = "$src1 = $dst" + } + let Predicates = [In64BitMode] in { + def NAME#8ri_EVEX : BinOpRIF_RF<0x80, mnemonic, Xi8 , opnode, RegMRM>, PL; + def NAME#16ri8_EVEX : BinOpRI8F_RF<0x83, mnemonic, Xi16, RegMRM>, PL, PD; + def NAME#32ri8_EVEX : BinOpRI8F_RF<0x83, mnemonic, Xi32, RegMRM>, PL; + def NAME#64ri8_EVEX : BinOpRI8F_RF<0x83, mnemonic, Xi64, RegMRM>, PL; + def NAME#16ri_EVEX : BinOpRIF_RF<0x81, mnemonic, Xi16, opnode, RegMRM>, PL, PD; + def NAME#32ri_EVEX : BinOpRIF_RF<0x81, mnemonic, Xi32, opnode, RegMRM>, PL; + def NAME#64ri32_EVEX: BinOpRIF_RF<0x81, mnemonic, Xi64, opnode, RegMRM>, PL; + } def NAME#8mr : BinOpMRF_MF<BaseOpc, mnemonic, Xi8 , opnode>; def NAME#16mr : BinOpMRF_MF<BaseOpc, mnemonic, Xi16, opnode>, OpSize16; def NAME#32mr : BinOpMRF_MF<BaseOpc, mnemonic, Xi32, opnode>, OpSize32; def NAME#64mr : BinOpMRF_MF<BaseOpc, mnemonic, Xi64, opnode>; + let Predicates = [HasNDD, In64BitMode] in { + def NAME#8mr_ND : BinOpMRF_RF<BaseOpc, mnemonic, Xi8 , opnode>; + def NAME#16mr_ND : BinOpMRF_RF<BaseOpc, mnemonic, Xi16, opnode>, PD; + def NAME#32mr_ND : BinOpMRF_RF<BaseOpc, mnemonic, Xi32, opnode>; + def NAME#64mr_ND : BinOpMRF_RF<BaseOpc, mnemonic, Xi64, opnode>; + } + let Predicates = [In64BitMode] in { + def NAME#8mr_EVEX : BinOpMRF_MF<BaseOpc, mnemonic, Xi8 , null_frag>, PL; + def NAME#16mr_EVEX : BinOpMRF_MF<BaseOpc, mnemonic, Xi16, null_frag>, PL, PD; + def NAME#32mr_EVEX : BinOpMRF_MF<BaseOpc, mnemonic, Xi32, null_frag>, PL; + def NAME#64mr_EVEX : BinOpMRF_MF<BaseOpc, mnemonic, Xi64, null_frag>, PL; + } // NOTE: These are order specific, we want the mi8 forms to be listed // first so that they are slightly preferred to the mi forms. + def NAME#8mi : BinOpMIF_MF<0x80, mnemonic, Xi8 , opnode, MemMRM>; def NAME#16mi8 : BinOpMI8F_MF<mnemonic, Xi16, MemMRM>, OpSize16; def NAME#32mi8 : BinOpMI8F_MF<mnemonic, Xi32, MemMRM>, OpSize32; let Predicates = [In64BitMode] in - def NAME#64mi8 : BinOpMI8F_MF<mnemonic, Xi64, MemMRM>; - - def NAME#8mi : BinOpMIF_MF<0x80, mnemonic, Xi8 , opnode, MemMRM>; + def NAME#64mi8 : BinOpMI8F_MF<mnemonic, Xi64, MemMRM>; def NAME#16mi : BinOpMIF_MF<0x81, mnemonic, Xi16, opnode, MemMRM>, OpSize16; def NAME#32mi : BinOpMIF_MF<0x81, mnemonic, Xi32, opnode, MemMRM>, OpSize32; let Predicates = [In64BitMode] in - def NAME#64mi32 : BinOpMIF_MF<0x81, mnemonic, Xi64, opnode, MemMRM>; + def NAME#64mi32 : BinOpMIF_MF<0x81, mnemonic, Xi64, opnode, MemMRM>; + + let Predicates = [HasNDD, In64BitMode] in { + def NAME#8mi_ND : BinOpMIF_RF<0x80, mnemonic, Xi8 , opnode, MemMRM>; + def NAME#16mi8_ND : BinOpMI8F_RF<mnemonic, Xi16, MemMRM>, PD; + def NAME#32mi8_ND : BinOpMI8F_RF<mnemonic, Xi32, MemMRM>; + def NAME#64mi8_ND : BinOpMI8F_RF<mnemonic, Xi64, MemMRM>; + def NAME#16mi_ND : BinOpMIF_RF<0x81, mnemonic, Xi16, opnode, MemMRM>, PD; + def NAME#32mi_ND : BinOpMIF_RF<0x81, mnemonic, Xi32, opnode, MemMRM>; + def NAME#64mi32_ND : BinOpMIF_RF<0x81, mnemonic, Xi64, opnode, MemMRM>; + } + let Predicates = [In64BitMode] in { + def NAME#8mi_EVEX : BinOpMIF_MF<0x80, mnemonic, Xi8 , opnode, MemMRM>, PL; + def NAME#16mi8_EVEX : BinOpMI8F_MF<mnemonic, Xi16, MemMRM>, PL, PD; + def NAME#32mi8_EVEX : BinOpMI8F_MF<mnemonic, Xi32, MemMRM>, PL; + def NAME#64mi8_EVEX : BinOpMI8F_MF<mnemonic, Xi64, MemMRM>, PL; + def NAME#16mi_EVEX : BinOpMIF_MF<0x81, mnemonic, Xi16, opnode, MemMRM>, PL, PD; + def NAME#32mi_EVEX : BinOpMIF_MF<0x81, mnemonic, Xi32, opnode, MemMRM>, PL; + def NAME#64mi32_EVEX : BinOpMIF_MF<0x81, mnemonic, Xi64, opnode, MemMRM>, PL; + } // These are for the disassembler since 0x82 opcode behaves like 0x80, but // not in 64-bit mode. let Predicates = [Not64BitMode] in { - let Constraints = "$src1 = $dst" in def NAME#8ri8 : BinOpRI8F_RF<0x82, mnemonic, Xi8, RegMRM>, DisassembleOnly; def NAME#8mi8 : BinOpMI8F_MF<mnemonic, Xi8, MemMRM>, DisassembleOnly; } @@ -1089,36 +1030,30 @@ def : Pat<(X86testpat (loadi64 addr:$src1), i64relocImmSExt32_su:$src2), //===----------------------------------------------------------------------===// // ANDN Instruction // -multiclass bmi_andn<string mnemonic, RegisterClass RC, X86MemOperand x86memop, - PatFrag ld_frag, X86FoldableSchedWrite sched> { -let Predicates = [HasBMI, NoEGPR] in { - def rr : I<0xF2, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2), - !strconcat(mnemonic, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), - [(set RC:$dst, EFLAGS, (X86and_flag (not RC:$src1), RC:$src2))]>, - VEX, VVVV, Sched<[sched]>; - def rm : I<0xF2, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, x86memop:$src2), - !strconcat(mnemonic, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), - [(set RC:$dst, EFLAGS, - (X86and_flag (not RC:$src1), (ld_frag addr:$src2)))]>, - VEX, VVVV, Sched<[sched.Folded, sched.ReadAfterFold]>; -} -let Predicates = [HasBMI, HasEGPR, In64BitMode] in { - def rr_EVEX : I<0xF2, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2), - !strconcat(mnemonic, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), - [(set RC:$dst, EFLAGS, (X86and_flag (not RC:$src1), RC:$src2))]>, - EVEX, VVVV, Sched<[sched]>; - def rm_EVEX : I<0xF2, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, x86memop:$src2), - !strconcat(mnemonic, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), - [(set RC:$dst, EFLAGS, - (X86and_flag (not RC:$src1), (ld_frag addr:$src2)))]>, - EVEX, VVVV, Sched<[sched.Folded, sched.ReadAfterFold]>; -} +multiclass AndN<X86TypeInfo t, string suffix> { + defvar andn_rr_p = + [(set t.RegClass:$dst, EFLAGS, (X86and_flag (not t.RegClass:$src1), + t.RegClass:$src2))]; + defvar andn_rm_p = + [(set t.RegClass:$dst, EFLAGS, (X86and_flag (not t.RegClass:$src1), + (t.LoadNode addr:$src2)))]; + def rr#suffix : ITy<0xF2, MRMSrcReg, t, (outs t.RegClass:$dst), + (ins t.RegClass:$src1, t.RegClass:$src2), "andn", + binop_ndd_args, andn_rr_p>, VVVV, Sched<[WriteALU]>, + T8, DefEFLAGS; + def rm#suffix : ITy<0xF2, MRMSrcMem, t, (outs t.RegClass:$dst), + (ins t.RegClass:$src1, t.MemOperand:$src2), "andn", + binop_ndd_args, andn_rm_p>, VVVV, + Sched<[WriteALU.Folded, WriteALU.ReadAfterFold]>, + T8, DefEFLAGS; } // Complexity is reduced to give and with immediate a chance to match first. -let Defs = [EFLAGS], AddedComplexity = -6 in { - defm ANDN32 : bmi_andn<"andn{l}", GR32, i32mem, loadi32, WriteALU>, T8; - defm ANDN64 : bmi_andn<"andn{q}", GR64, i64mem, loadi64, WriteALU>, T8, REX_W; +let AddedComplexity = -6 in { +defm ANDN32 : AndN<Xi32, "">, VEX, Requires<[HasBMI, NoEGPR]>; +defm ANDN64 : AndN<Xi64, "">, VEX, REX_W, Requires<[HasBMI, NoEGPR]>; +defm ANDN32 : AndN<Xi32, "_EVEX">, EVEX, Requires<[HasBMI, HasEGPR, In64BitMode]>; +defm ANDN64 : AndN<Xi64, "_EVEX">, EVEX, REX_W, Requires<[HasBMI, HasEGPR, In64BitMode]>; } let Predicates = [HasBMI], AddedComplexity = -6 in { @@ -1135,78 +1070,63 @@ let Predicates = [HasBMI], AddedComplexity = -6 in { //===----------------------------------------------------------------------===// // MULX Instruction // -multiclass bmi_mulx<string mnemonic, RegisterClass RC, X86MemOperand x86memop, - X86FoldableSchedWrite sched> { -let hasSideEffects = 0 in { -let Predicates = [HasBMI2, NoEGPR] in { - def rr : I<0xF6, MRMSrcReg, (outs RC:$dst1, RC:$dst2), (ins RC:$src), - !strconcat(mnemonic, "\t{$src, $dst2, $dst1|$dst1, $dst2, $src}"), - []>, T8, XD, VEX, VVVV, Sched<[WriteIMulH, sched]>; - +multiclass MulX<X86TypeInfo t, X86FoldableSchedWrite sched> { + defvar mulx_args = "{$src, $dst2, $dst1|$dst1, $dst2, $src}"; + defvar mulx_rm_sched = + [WriteIMulHLd, sched.Folded, + // Memory operand. + ReadDefault, ReadDefault, ReadDefault, ReadDefault, ReadDefault, + // Implicit read of EDX/RDX + sched.ReadAfterFold]; + + def rr : ITy<0xF6, MRMSrcReg, t, (outs t.RegClass:$dst1, t.RegClass:$dst2), + (ins t.RegClass:$src), "mulx", mulx_args, []>, T8, XD, VEX, + VVVV, Sched<[WriteIMulH, sched]>; let mayLoad = 1 in - def rm : I<0xF6, MRMSrcMem, (outs RC:$dst1, RC:$dst2), (ins x86memop:$src), - !strconcat(mnemonic, "\t{$src, $dst2, $dst1|$dst1, $dst2, $src}"), - []>, T8, XD, VEX, VVVV, - Sched<[WriteIMulHLd, sched.Folded, - // Memory operand. - ReadDefault, ReadDefault, ReadDefault, ReadDefault, ReadDefault, - // Implicit read of EDX/RDX - sched.ReadAfterFold]>; - + def rm : ITy<0xF6, MRMSrcMem, t, (outs t.RegClass:$dst1, t.RegClass:$dst2), + (ins t.MemOperand:$src), "mulx", mulx_args, []>, T8, XD, VEX, + VVVV, Sched<mulx_rm_sched>; + + let Predicates = [In64BitMode] in { + def rr_EVEX : ITy<0xF6, MRMSrcReg, t, + (outs t.RegClass:$dst1, t.RegClass:$dst2), + (ins t.RegClass:$src), "mulx", mulx_args, []>, T8, XD, + EVEX, VVVV, Sched<[WriteIMulH, sched]>; + let mayLoad = 1 in + def rm_EVEX : ITy<0xF6, MRMSrcMem, t, + (outs t.RegClass:$dst1, t.RegClass:$dst2), + (ins t.MemOperand:$src), "mulx", mulx_args, []>, T8, XD, + EVEX, VVVV, Sched<mulx_rm_sched>; + } // Pseudo instructions to be used when the low result isn't used. The // instruction is defined to keep the high if both destinations are the same. - def Hrr : PseudoI<(outs RC:$dst), (ins RC:$src), - []>, Sched<[sched]>; - + def Hrr : PseudoI<(outs t.RegClass:$dst), (ins t.RegClass:$src), []>, + Sched<[sched]>; let mayLoad = 1 in - def Hrm : PseudoI<(outs RC:$dst), (ins x86memop:$src), - []>, Sched<[sched.Folded]>; -} -let Predicates = [HasBMI2, HasEGPR, In64BitMode] in - def rr#_EVEX : I<0xF6, MRMSrcReg, (outs RC:$dst1, RC:$dst2), (ins RC:$src), - !strconcat(mnemonic, "\t{$src, $dst2, $dst1|$dst1, $dst2, $src}"), - []>, T8, XD, EVEX, VVVV, Sched<[WriteIMulH, sched]>; -let Predicates = [HasBMI2, HasEGPR, In64BitMode], mayLoad = 1 in - def rm#_EVEX : I<0xF6, MRMSrcMem, (outs RC:$dst1, RC:$dst2), (ins x86memop:$src), - !strconcat(mnemonic, "\t{$src, $dst2, $dst1|$dst1, $dst2, $src}"), - []>, T8, XD, EVEX, VVVV, - Sched<[WriteIMulHLd, sched.Folded, - // Memory operand. - ReadDefault, ReadDefault, ReadDefault, ReadDefault, ReadDefault, - // Implicit read of EDX/RDX - sched.ReadAfterFold]>; -} + def Hrm : PseudoI<(outs t.RegClass:$dst), (ins t.MemOperand:$src), []>, + Sched<[sched.Folded]>; } let Uses = [EDX] in - defm MULX32 : bmi_mulx<"mulx{l}", GR32, i32mem, WriteMULX32>; +defm MULX32 : MulX<Xi32, WriteMULX32>; + let Uses = [RDX] in - defm MULX64 : bmi_mulx<"mulx{q}", GR64, i64mem, WriteMULX64>, REX_W; +defm MULX64 : MulX<Xi64, WriteMULX64>, REX_W; //===----------------------------------------------------------------------===// // ADCX and ADOX Instructions // // We don't have patterns for these as there is no advantage over ADC for // most code. -class ADCOXOpRR <string m, X86TypeInfo t> - : BinOpRRF_RF<0xF6, m, t, null_frag> { - let Form = MRMSrcReg; - let isCommutable = 1; +let Form = MRMSrcReg in { +def ADCX32rr : BinOpRRF_RF<0xF6, "adcx", Xi32, null_frag>, T8, PD; +def ADCX64rr : BinOpRRF_RF<0xF6, "adcx", Xi64, null_frag>, T8, PD; +def ADOX32rr : BinOpRRF_RF<0xF6, "adox", Xi32, null_frag>, T8, XS; +def ADOX64rr : BinOpRRF_RF<0xF6, "adox", Xi64, null_frag>, T8, XS; } - -class ADCOXOpRM <string m, X86TypeInfo t> - : BinOpRMF_RF<0xF6, m, t, null_frag> { - let Form = MRMSrcMem; -} - -let OpSize = OpSizeFixed, Constraints = "$src1 = $dst", - Predicates = [HasADX] in { -def ADCX32rr : ADCOXOpRR<"adcx", Xi32>, T8, PD; -def ADCX64rr : ADCOXOpRR<"adcx", Xi64>, T8, PD; -def ADOX32rr : ADCOXOpRR<"adox", Xi32>, T8, XS; -def ADOX64rr : ADCOXOpRR<"adox", Xi64>, T8, XS; -def ADCX32rm : ADCOXOpRM<"adcx", Xi32>, T8, PD; -def ADCX64rm : ADCOXOpRM<"adcx", Xi64>, T8, PD; -def ADOX32rm : ADCOXOpRM<"adox", Xi32>, T8, XS; -def ADOX64rm : ADCOXOpRM<"adox", Xi64>, T8, XS; +let Form = MRMSrcMem in { +def ADCX32rm : BinOpRMF_RF<0xF6, "adcx", Xi32, null_frag>, T8, PD; +def ADCX64rm : BinOpRMF_RF<0xF6, "adcx", Xi64, null_frag>, T8, PD; +def ADOX32rm : BinOpRMF_RF<0xF6, "adox", Xi32, null_frag>, T8, XS; +def ADOX64rm : BinOpRMF_RF<0xF6, "adox", Xi64, null_frag>, T8, XS; } diff --git a/contrib/llvm-project/llvm/lib/Target/X86/X86InstrFormats.td b/contrib/llvm-project/llvm/lib/Target/X86/X86InstrFormats.td index 07e5576960d6..6e76b44b66a3 100644 --- a/contrib/llvm-project/llvm/lib/Target/X86/X86InstrFormats.td +++ b/contrib/llvm-project/llvm/lib/Target/X86/X86InstrFormats.td @@ -256,6 +256,7 @@ class X86Inst<bits<8> opcod, Format f, ImmType i, dag outs, dag ins, bit hasEVEX_Z = 0; // Does this inst set the EVEX_Z field? bit hasEVEX_L2 = 0; // Does this inst set the EVEX_L2 field? bit hasEVEX_B = 0; // Does this inst set the EVEX_B field? + bit hasEVEX_NF = 0; // Does this inst set the EVEX_NF field? bits<3> CD8_Form = 0; // Compressed disp8 form - vector-width. // Declare it int rather than bits<4> so that all bits are defined when // assigning to bits<7>. @@ -309,4 +310,5 @@ class X86Inst<bits<8> opcod, Format f, ImmType i, dag outs, dag ins, let TSFlags{48} = hasEVEX_RC; let TSFlags{49} = hasNoTrackPrefix; let TSFlags{51-50} = explicitOpPrefixBits; + let TSFlags{52} = hasEVEX_NF; } diff --git a/contrib/llvm-project/llvm/lib/Target/X86/X86InstrPredicates.td b/contrib/llvm-project/llvm/lib/Target/X86/X86InstrPredicates.td index 8653f15d8602..94fa6e45ded9 100644 --- a/contrib/llvm-project/llvm/lib/Target/X86/X86InstrPredicates.td +++ b/contrib/llvm-project/llvm/lib/Target/X86/X86InstrPredicates.td @@ -10,6 +10,8 @@ def TruePredicate : Predicate<"true">; def HasEGPR : Predicate<"Subtarget->hasEGPR()">; def NoEGPR : Predicate<"!Subtarget->hasEGPR()">; +def HasNDD : Predicate<"Subtarget->hasNDD()">; +def NoNDD : Predicate<"!Subtarget->hasNDD()">; def HasCMOV : Predicate<"Subtarget->canUseCMOV()">; def NoCMOV : Predicate<"!Subtarget->canUseCMOV()">; def HasNOPL : Predicate<"Subtarget->hasNOPL()">; @@ -100,7 +102,6 @@ def HasIFMA : Predicate<"Subtarget->hasIFMA()">; def HasAVXIFMA : Predicate<"Subtarget->hasAVXIFMA()">; def NoVLX_Or_NoIFMA : Predicate<"!Subtarget->hasVLX() || !Subtarget->hasIFMA()">; def HasRTM : Predicate<"Subtarget->hasRTM()">; -def HasADX : Predicate<"Subtarget->hasADX()">; def HasSHA : Predicate<"Subtarget->hasSHA()">; def HasSHA512 : Predicate<"Subtarget->hasSHA512()">; def HasSGX : Predicate<"Subtarget->hasSGX()">; diff --git a/contrib/llvm-project/llvm/lib/Target/X86/X86InstrSSE.td b/contrib/llvm-project/llvm/lib/Target/X86/X86InstrSSE.td index df1f0b5b4ca7..e8a1a2b83886 100644 --- a/contrib/llvm-project/llvm/lib/Target/X86/X86InstrSSE.td +++ b/contrib/llvm-project/llvm/lib/Target/X86/X86InstrSSE.td @@ -6655,49 +6655,51 @@ let Defs = [ECX, EFLAGS], Uses = [EAX, EDX], hasSideEffects = 0 in { // SSE4.2 - CRC Instructions //===----------------------------------------------------------------------===// +// NOTE: 'HasCRC32' is used as CRC32 instructions are GPR only and not directly +// controlled by the SSE42 flag. +// // No CRC instructions have AVX equivalents -// crc intrinsic instruction -// This set of instructions are only rm, the only difference is the size -// of r and m. -class SS42I_crc32r<bits<8> opc, string asm, RegisterClass RCOut, - RegisterClass RCIn, SDPatternOperator Int> : - CRC32I<opc, MRMSrcReg, (outs RCOut:$dst), (ins RCOut:$src1, RCIn:$src2), - !strconcat(asm, "\t{$src2, $src1|$src1, $src2}"), - [(set RCOut:$dst, (Int RCOut:$src1, RCIn:$src2))]>, - Sched<[WriteCRC32]>; - -class SS42I_crc32m<bits<8> opc, string asm, RegisterClass RCOut, - X86MemOperand x86memop, SDPatternOperator Int> : - CRC32I<opc, MRMSrcMem, (outs RCOut:$dst), (ins RCOut:$src1, x86memop:$src2), - !strconcat(asm, "\t{$src2, $src1|$src1, $src2}"), - [(set RCOut:$dst, (Int RCOut:$src1, (load addr:$src2)))]>, - Sched<[WriteCRC32.Folded, WriteCRC32.ReadAfterFold]>; - -let Constraints = "$src1 = $dst" in { - def CRC32r32m8 : SS42I_crc32m<0xF0, "crc32{b}", GR32, i8mem, - int_x86_sse42_crc32_32_8>; - def CRC32r32r8 : SS42I_crc32r<0xF0, "crc32{b}", GR32, GR8, - int_x86_sse42_crc32_32_8>; - def CRC32r32m16 : SS42I_crc32m<0xF1, "crc32{w}", GR32, i16mem, - int_x86_sse42_crc32_32_16>, OpSize16; - def CRC32r32r16 : SS42I_crc32r<0xF1, "crc32{w}", GR32, GR16, - int_x86_sse42_crc32_32_16>, OpSize16; - def CRC32r32m32 : SS42I_crc32m<0xF1, "crc32{l}", GR32, i32mem, - int_x86_sse42_crc32_32_32>, OpSize32; - def CRC32r32r32 : SS42I_crc32r<0xF1, "crc32{l}", GR32, GR32, - int_x86_sse42_crc32_32_32>, OpSize32; - def CRC32r64m64 : SS42I_crc32m<0xF1, "crc32{q}", GR64, i64mem, - int_x86_sse42_crc32_64_64>, REX_W; - def CRC32r64r64 : SS42I_crc32r<0xF1, "crc32{q}", GR64, GR64, - int_x86_sse42_crc32_64_64>, REX_W; - let hasSideEffects = 0 in { - let mayLoad = 1 in - def CRC32r64m8 : SS42I_crc32m<0xF0, "crc32{b}", GR64, i8mem, - null_frag>, REX_W; - def CRC32r64r8 : SS42I_crc32r<0xF0, "crc32{b}", GR64, GR8, - null_frag>, REX_W; - } +class Crc32r<X86TypeInfo t, RegisterClass rc, SDPatternOperator node> + : ITy<0xF1, MRMSrcReg, t, (outs rc:$dst), (ins rc:$src1, t.RegClass:$src2), + "crc32", binop_args, [(set rc:$dst, (node rc:$src1, t.RegClass:$src2))]>, + Sched<[WriteCRC32]>, NoCD8 { + let Constraints = "$src1 = $dst"; +} + +class Crc32m<X86TypeInfo t, RegisterClass rc, SDPatternOperator node> + : ITy<0xF1, MRMSrcMem, t, (outs rc:$dst), (ins rc:$src1, t.MemOperand:$src2), + "crc32", binop_args, [(set rc:$dst, (node rc:$src1, (load addr:$src2)))]>, + Sched<[WriteCRC32.Folded, WriteCRC32.ReadAfterFold]>, NoCD8 { + let Constraints = "$src1 = $dst"; +} + +let Predicates = [HasCRC32, NoEGPR], OpMap = T8, OpPrefix = XD in { + def CRC32r32r8 : Crc32r<Xi8, GR32, int_x86_sse42_crc32_32_8>; + def CRC32r32m8 : Crc32m<Xi8, GR32, int_x86_sse42_crc32_32_8>; + def CRC32r32r16 : Crc32r<Xi16, GR32, int_x86_sse42_crc32_32_16>, OpSize16; + def CRC32r32m16 : Crc32m<Xi16, GR32, int_x86_sse42_crc32_32_16>, OpSize16; + def CRC32r32r32 : Crc32r<Xi32, GR32, int_x86_sse42_crc32_32_32>, OpSize32; + def CRC32r32m32 : Crc32m<Xi32, GR32, int_x86_sse42_crc32_32_32>, OpSize32; + def CRC32r64r64 : Crc32r<Xi64, GR64, int_x86_sse42_crc32_64_64>; + def CRC32r64m64 : Crc32m<Xi64, GR64, int_x86_sse42_crc32_64_64>; + def CRC32r64r8 : Crc32r<Xi8, GR64, null_frag>, REX_W; + let mayLoad = 1 in + def CRC32r64m8 : Crc32m<Xi8, GR64, null_frag>, REX_W; +} + +let Predicates = [HasCRC32, HasEGPR, In64BitMode], OpMap = T_MAP4, OpEnc = EncEVEX in { + def CRC32r32r8_EVEX : Crc32r<Xi8, GR32, int_x86_sse42_crc32_32_8>; + def CRC32r32m8_EVEX : Crc32m<Xi8, GR32, int_x86_sse42_crc32_32_8>; + def CRC32r32r16_EVEX : Crc32r<Xi16, GR32, int_x86_sse42_crc32_32_16>, PD; + def CRC32r32m16_EVEX : Crc32m<Xi16, GR32, int_x86_sse42_crc32_32_16>, PD; + def CRC32r32r32_EVEX : Crc32r<Xi32, GR32, int_x86_sse42_crc32_32_32>; + def CRC32r32m32_EVEX : Crc32m<Xi32, GR32, int_x86_sse42_crc32_32_32>; + def CRC32r64r64_EVEX : Crc32r<Xi64, GR64, int_x86_sse42_crc32_64_64>; + def CRC32r64m64_EVEX : Crc32m<Xi64, GR64, int_x86_sse42_crc32_64_64>; + def CRC32r64r8_EVEX : Crc32r<Xi8, GR64, null_frag>, REX_W; + let mayLoad = 1 in + def CRC32r64m8_EVEX : Crc32m<Xi8, GR64, null_frag>, REX_W; } //===----------------------------------------------------------------------===// @@ -7160,6 +7162,10 @@ def : Pat<(v32i8 (X86SubVBroadcastld128 addr:$src)), (VBROADCASTF128rm addr:$src)>; } +let Predicates = [HasAVXNECONVERT, NoVLX] in + def : Pat<(v16bf16 (X86SubVBroadcastld128 addr:$src)), + (VBROADCASTF128rm addr:$src)>; + //===----------------------------------------------------------------------===// // VPERM2F128 - Permute Floating-Point Values in 128-bit chunks // @@ -7905,6 +7911,9 @@ let Predicates = [HasAVX2, NoVLX] in { defm : vinsert_lowering<"VINSERTI128", "VPERM2I128", v16i8, v32i8, loadv16i8, loadv32i8>; } +let Predicates = [HasAVXNECONVERT, NoVLX] in + defm : vinsert_lowering<"VINSERTI128", "VPERM2I128", v8bf16, v16bf16, loadv8bf16, loadv16bf16>; + //===----------------------------------------------------------------------===// // VEXTRACTI128 - Extract packed integer values // @@ -7927,6 +7936,9 @@ let Predicates = [HasAVX2, NoVLX] in { defm : vextract_lowering<"VEXTRACTI128", v32i8, v16i8>; } +let Predicates = [HasAVXNECONVERT, NoVLX] in + defm : vextract_lowering<"VEXTRACTI128", v16bf16, v8bf16>; + //===----------------------------------------------------------------------===// // VPMASKMOV - Conditional SIMD Integer Packed Loads and Stores // diff --git a/contrib/llvm-project/llvm/lib/Target/X86/X86InstrSystem.td b/contrib/llvm-project/llvm/lib/Target/X86/X86InstrSystem.td index efb58c6102dd..699e5847e63f 100644 --- a/contrib/llvm-project/llvm/lib/Target/X86/X86InstrSystem.td +++ b/contrib/llvm-project/llvm/lib/Target/X86/X86InstrSystem.td @@ -446,11 +446,11 @@ let Predicates = [HasUSERMSR], mayLoad = 1 in { } let Predicates = [HasUSERMSR], mayStore = 1 in { def UWRMSRrr : I<0xf8, MRMSrcReg, (outs), (ins GR64:$src1, GR64:$src2), - "uwrmsr\t{$src1, $src2|$src2, $src1}", + "uwrmsr\t{$src2, $src1|$src1, $src2}", [(int_x86_uwrmsr GR64:$src1, GR64:$src2)]>, T8, XS; def UWRMSRir : Ii32<0xf8, MRM0r, (outs), (ins GR64:$src, i64i32imm:$imm), "uwrmsr\t{$src, $imm|$imm, $src}", - [(int_x86_uwrmsr GR64:$src, i64immSExt32_su:$imm)]>, T_MAP7, XS, VEX; + [(int_x86_uwrmsr i64immSExt32_su:$imm, GR64:$src)]>, T_MAP7, XS, VEX; } let Defs = [RAX, RDX], Uses = [ECX] in def RDPMC : I<0x33, RawFrm, (outs), (ins), "rdpmc", []>, TB; diff --git a/contrib/llvm-project/llvm/lib/Target/X86/X86InstrUtils.td b/contrib/llvm-project/llvm/lib/Target/X86/X86InstrUtils.td index 9499753143d9..da85922a018d 100644 --- a/contrib/llvm-project/llvm/lib/Target/X86/X86InstrUtils.td +++ b/contrib/llvm-project/llvm/lib/Target/X86/X86InstrUtils.td @@ -39,17 +39,19 @@ class PS { Prefix OpPrefix = PS; } class PD { Prefix OpPrefix = PD; } class XD { Prefix OpPrefix = XD; } class XS { Prefix OpPrefix = XS; } -class VEX { Encoding OpEnc = EncVEX; } +class XOP { Encoding OpEnc = EncXOP; } +class VEX { Encoding OpEnc = EncVEX; } +class EVEX { Encoding OpEnc = EncEVEX; } class WIG { bit IgnoresW = 1; } // Special version of REX_W that can be changed to VEX.W==0 for EVEX2VEX. class VEX_W1X { bit hasREX_W = 1; bit EVEX_W1_VEX_W0 = 1; } class VEX_L { bit hasVEX_L = 1; } class VEX_LIG { bit ignoresVEX_L = 1; } class VVVV { bit hasVEX_4V = 1; } -class EVEX { Encoding OpEnc = EncEVEX; } class EVEX_K { bit hasEVEX_K = 1; } class EVEX_KZ : EVEX_K { bit hasEVEX_Z = 1; } class EVEX_B { bit hasEVEX_B = 1; } +class EVEX_NF { bit hasEVEX_NF = 1; } class EVEX_RC { bit hasEVEX_RC = 1; } class EVEX_V512 { bit hasEVEX_L2 = 1; bit hasVEX_L = 0; } class EVEX_V256 { bit hasEVEX_L2 = 0; bit hasVEX_L = 1; } @@ -63,7 +65,7 @@ class EVEX_CD8<int esize, CD8VForm form> { bits<3> CD8_Form = form.Value; } class NoCD8 { bits<7> CD8_Scale = 0; } -class XOP { Encoding OpEnc = EncXOP; } + class EVEX2VEXOverride<string VEXInstrName> { string EVEX2VEXOverride = VEXInstrName; } @@ -99,16 +101,24 @@ class DisassembleOnly { bit ForceDisassemble = 1; } - -// SchedModel info for instruction that loads one value and gets the second -// (and possibly third) value from a register. -// This is used for instructions that put the memory operands before other -// uses. -class SchedLoadReg<X86FoldableSchedWrite Sched> : Sched<[Sched.Folded, - // Memory operand. - ReadDefault, ReadDefault, ReadDefault, ReadDefault, ReadDefault, - // Register reads (implicit or explicit). - Sched.ReadAfterFold, Sched.ReadAfterFold]>; +defvar unaryop_args = "$src1"; +defvar unaryop_ndd_args = "{$src1, $dst|$dst, $src1}"; +defvar binop_args = "{$src2, $src1|$src1, $src2}"; +defvar binop_ndd_args = "{$src2, $src1, $dst|$dst, $src1, $src2}"; +defvar tie_dst_src1 = "$src1 = $dst"; + +// NDD - Helper for new data destination instructions +class NDD<bit ndd> { + string Constraints = !if(!eq(ndd, 0), tie_dst_src1, ""); + Encoding OpEnc = !if(!eq(ndd, 0), EncNormal, EncEVEX); + bit hasEVEX_B = ndd; + bit hasVEX_4V = ndd; + Map OpMap = !if(!eq(ndd, 0), OB, T_MAP4); +} +// NF - Helper for NF (no flags update) instructions +class NF: T_MAP4, EVEX, EVEX_NF, NoCD8; +// PL - Helper for promoted legacy instructions +class PL: T_MAP4, EVEX, NoCD8, ExplicitEVEXPrefix; //===----------------------------------------------------------------------===// // X86 Type infomation definitions @@ -723,13 +733,6 @@ class SS42AI<bits<8> o, Format F, dag outs, dag ins, string asm, : Ii8<o, F, outs, ins, asm, pattern, SSEPackedInt>, TA, PD, Requires<[UseSSE42]>; -// CRC32I - SSE 4.2 CRC32 instructions. -// NOTE: 'HasCRC32' is used as CRC32 instructions are GPR only and not directly -// controlled by the SSE42 flag. -class CRC32I<bits<8> o, Format F, dag outs, dag ins, string asm, - list<dag> pattern> - : I<o, F, outs, ins, asm, pattern>, T8, XD, Requires<[HasCRC32]>; - // AVX Instruction Templates: // Instructions introduced in AVX (no SSE equivalent forms) // @@ -957,15 +960,380 @@ class MMXIi8<bits<8> o, Format F, dag outs, dag ins, string asm, /// 2. Infers whether the instruction should have a 0x40 REX_W prefix. /// 3. Infers whether the low bit of the opcode should be 0 (for i8 operations) /// or 1 (for i16,i32,i64 operations). -class ITy<bits<8> opcode, Format f, X86TypeInfo typeinfo, dag outs, dag ins, - string mnemonic, string args, list<dag> pattern> - : I<{opcode{7}, opcode{6}, opcode{5}, opcode{4}, - opcode{3}, opcode{2}, opcode{1}, - !if(!eq(typeinfo.HasEvenOpcode, 1), 0, opcode{0})}, f, outs, ins, - !strconcat(mnemonic, "{", typeinfo.InstrSuffix, "}\t", args), pattern> { - +class ITy<bits<8> o, Format f, X86TypeInfo t, dag outs, dag ins, string m, + string args, list<dag> p> + : I<{o{7}, o{6}, o{5}, o{4}, o{3}, o{2}, o{1}, + !if(!eq(t.HasEvenOpcode, 1), 0, o{0})}, f, outs, ins, + !strconcat(m, "{", t.InstrSuffix, "}\t", args), p> { let hasSideEffects = 0; - let hasREX_W = typeinfo.HasREX_W; + let hasREX_W = t.HasREX_W; } -defvar binop_args = "{$src2, $src1|$src1, $src2}"; +// BinOpRR - Instructions that read "reg, reg". +class BinOpRR<bits<8> o, string m, string args, X86TypeInfo t, dag out, list<dag> p> + : ITy<o, MRMDestReg, t, out, (ins t.RegClass:$src1, t.RegClass:$src2), m, + args, p>, Sched<[WriteALU]>; +// BinOpRR_F - Instructions that read "reg, reg" and write EFLAGS only. +class BinOpRR_F<bits<8> o, string m, X86TypeInfo t, SDPatternOperator node> + : BinOpRR<o, m, binop_args, t, (outs), + [(set EFLAGS, (node t.RegClass:$src1, t.RegClass:$src2))]>, + DefEFLAGS; +// BinOpRR_F_Rev - Reversed encoding of BinOpRR_F +class BinOpRR_F_Rev<bits<8> o, string m, X86TypeInfo t> + : BinOpRR_F<o, m, t, null_frag>, DisassembleOnly { + let Form = MRMSrcReg; +} +// BinOpRR_R - Instructions that read "reg, reg" and write "reg". +class BinOpRR_R<bits<8> o, string m, X86TypeInfo t, bit ndd = 0> + : BinOpRR<o, m, !if(!eq(ndd, 0), binop_args, binop_ndd_args), t, + (outs t.RegClass:$dst), []>, NDD<ndd>; +// BinOpRR_R_Rev - Reversed encoding of BinOpRR_R +class BinOpRR_R_Rev<bits<8> o, string m, X86TypeInfo t, bit ndd = 0> + : BinOpRR_R<o, m, t, ndd>, DisassembleOnly { + let Form = MRMSrcReg; +} +// BinOpRR_RF - Instructions that read "reg, reg", and write "reg", EFLAGS. +class BinOpRR_RF<bits<8> o, string m, X86TypeInfo t, SDPatternOperator node, bit ndd = 0> + : BinOpRR<o, m, !if(!eq(ndd, 0), binop_args, binop_ndd_args), t, + (outs t.RegClass:$dst), + [(set t.RegClass:$dst, EFLAGS, + (node t.RegClass:$src1, t.RegClass:$src2))]>, DefEFLAGS, NDD<ndd>; +// BinOpRR_RF_Rev - Reversed encoding of BinOpRR_RF. +class BinOpRR_RF_Rev<bits<8> o, string m, X86TypeInfo t, bit ndd = 0> + : BinOpRR_RF<o, m, t, null_frag, ndd>, DisassembleOnly { + let Form = MRMSrcReg; +} +// BinOpRRF_RF - Instructions that read "reg, reg", write "reg" and read/write +// EFLAGS. +class BinOpRRF_RF<bits<8> o, string m, X86TypeInfo t, SDPatternOperator node, bit ndd = 0> + : BinOpRR<o, m, !if(!eq(ndd, 0), binop_args, binop_ndd_args), t, (outs t.RegClass:$dst), + [(set t.RegClass:$dst, EFLAGS, + (node t.RegClass:$src1, t.RegClass:$src2, + EFLAGS))]>, DefEFLAGS, UseEFLAGS, NDD<ndd> { + let SchedRW = [WriteADC]; +} +// BinOpRRF_RF_Rev - Reversed encoding of BinOpRRF_RF +class BinOpRRF_RF_Rev<bits<8> o, string m, X86TypeInfo t, bit ndd = 0> + : BinOpRRF_RF<o, m, t, null_frag, ndd>, DisassembleOnly { + let Form = MRMSrcReg; +} + +// BinOpRM - Instructions that read "reg, [mem]". +class BinOpRM<bits<8> o, string m, string args, X86TypeInfo t, dag out, list<dag> p> + : ITy<o, MRMSrcMem, t, out, (ins t.RegClass:$src1, t.MemOperand:$src2), m, + args, p>, + Sched<[WriteALU.Folded, WriteALU.ReadAfterFold]> { + let mayLoad = 1; +} +// BinOpRM_F - Instructions that read "reg, [mem]" and write EFLAGS only. +class BinOpRM_F<bits<8> o, string m, X86TypeInfo t, SDNode node> + : BinOpRM<o, m, binop_args, t, (outs), + [(set EFLAGS, (node t.RegClass:$src1, + (t.LoadNode addr:$src2)))]>, DefEFLAGS; +// BinOpRM_R - Instructions that read "reg, [mem]", and write "reg". +class BinOpRM_R<bits<8> o, string m, X86TypeInfo t, bit ndd = 0> + : BinOpRM<o, m, !if(!eq(ndd, 0), binop_args, binop_ndd_args), t, (outs t.RegClass:$dst), + []>, NDD<ndd>; +// BinOpRM_RF - Instructions that read "reg, [mem]", and write "reg", EFLAGS. +class BinOpRM_RF<bits<8> o, string m, X86TypeInfo t, SDPatternOperator node, bit ndd = 0> + : BinOpRM<o, m, !if(!eq(ndd, 0), binop_args, binop_ndd_args), t, (outs t.RegClass:$dst), + [(set t.RegClass:$dst, EFLAGS, (node t.RegClass:$src1, + (t.LoadNode addr:$src2)))]>, DefEFLAGS, NDD<ndd>; +// BinOpRMF_RF - Instructions that read "reg, [mem]", write "reg" and read/write +// EFLAGS. +class BinOpRMF_RF<bits<8> o, string m, X86TypeInfo t, SDPatternOperator node, bit ndd = 0> + : BinOpRM<o, m, !if(!eq(ndd, 0), binop_args, binop_ndd_args), t, (outs t.RegClass:$dst), + [(set t.RegClass:$dst, EFLAGS, + (node t.RegClass:$src1, (t.LoadNode addr:$src2), EFLAGS))]>, + DefEFLAGS, UseEFLAGS, NDD<ndd> { + let SchedRW = [WriteADC.Folded, WriteADC.ReadAfterFold, + // base, scale, index, offset, segment. + ReadDefault, ReadDefault, ReadDefault, ReadDefault, ReadDefault, + // implicit register read. + WriteADC.ReadAfterFold]; +} + +// BinOpRI - Instructions that read "reg, imm". +class BinOpRI<bits<8> o, string m, string args, X86TypeInfo t, Format f, dag out, list<dag> p> + : ITy<o, f, t, out, (ins t.RegClass:$src1, t.ImmOperand:$src2), m, + args, p>, Sched<[WriteALU]> { + let ImmT = t.ImmEncoding; +} +// BinOpRI_F - Instructions that read "reg, imm" and write EFLAGS only. +class BinOpRI_F<bits<8> o, string m, X86TypeInfo t, SDPatternOperator node, + Format f> + : BinOpRI<o, m, binop_args, t, f, (outs), + [(set EFLAGS, (node t.RegClass:$src1, + t.ImmOperator:$src2))]>, DefEFLAGS; +// BinOpRI_R - Instructions that read "reg, imm" and write "reg". +class BinOpRI_R<bits<8> o, string m, X86TypeInfo t, Format f, bit ndd = 0> + : BinOpRI<o, m, !if(!eq(ndd, 0), binop_args, binop_ndd_args), t, f, (outs t.RegClass:$dst), + []>, NDD<ndd>; +// BinOpRI_RF - Instructions that read "reg, imm" and write "reg", EFLAGS. +class BinOpRI_RF<bits<8> o, string m, X86TypeInfo t, SDPatternOperator node, Format f, bit ndd = 0> + : BinOpRI<o, m, !if(!eq(ndd, 0), binop_args, binop_ndd_args), t, f, (outs t.RegClass:$dst), + [(set t.RegClass:$dst, EFLAGS, + (node t.RegClass:$src1, t.ImmOperator:$src2))]>, DefEFLAGS, NDD<ndd>; +// BinOpRIF_RF - Instructions that read "reg, imm", write "reg" and read/write +// EFLAGS. +class BinOpRIF_RF<bits<8> o, string m, X86TypeInfo t, SDNode node, Format f, bit ndd = 0> + : BinOpRI<o, m, !if(!eq(ndd, 0), binop_args, binop_ndd_args), t, f, (outs t.RegClass:$dst), + [(set t.RegClass:$dst, EFLAGS, + (node t.RegClass:$src1, t.ImmOperator:$src2, + EFLAGS))]>, DefEFLAGS, UseEFLAGS, NDD<ndd> { + let SchedRW = [WriteADC]; +} +// BinOpRI8 - Instructions that read "reg, imm8". +class BinOpRI8<bits<8> o, string m, string args, X86TypeInfo t, Format f, dag out> + : ITy<o, f, t, out, (ins t.RegClass:$src1, t.Imm8Operand:$src2), m, + args, []>, Sched<[WriteALU]> { + let ImmT = Imm8; +} +// BinOpRI8_F - Instructions that read "reg, imm8" and write EFLAGS only. +class BinOpRI8_F<bits<8> o, string m, X86TypeInfo t, Format f> + : BinOpRI8<o, m, binop_args, t, f, (outs)>, DefEFLAGS; +// BinOpRI8_R - Instructions that read "reg, imm8" and write "reg". +class BinOpRI8_R<bits<8> o, string m, X86TypeInfo t, Format f, bit ndd = 0> + : BinOpRI8<o, m, !if(!eq(ndd, 0), binop_args, binop_ndd_args), t, f, (outs t.RegClass:$dst)>, NDD<ndd>; +// BinOpRI8_RF - Instructions that read "reg, imm8" and write "reg", EFLAGS. +class BinOpRI8_RF<bits<8> o, string m, X86TypeInfo t, Format f, bit ndd = 0> + : BinOpRI8<o, m, !if(!eq(ndd, 0), binop_args, binop_ndd_args), t, f, (outs t.RegClass:$dst)>, DefEFLAGS, NDD<ndd>; +// BinOpRI8F_RF - Instructions that read "reg, imm", write "reg" and read/write +// EFLAGS. +class BinOpRI8F_RF<bits<8> o, string m, X86TypeInfo t, Format f, bit ndd = 0> + : BinOpRI8<o, m, !if(!eq(ndd, 0), binop_args, binop_ndd_args), t, f, (outs t.RegClass:$dst)>, DefEFLAGS, UseEFLAGS, NDD<ndd> { + let SchedRW = [WriteADC]; +} + +// BinOpMR - Instructions that read "[mem], reg". +class BinOpMR<bits<8> o, string m, string args, X86TypeInfo t, dag out, list<dag> p> + : ITy<o, MRMDestMem, t, out, (ins t.MemOperand:$src1, t.RegClass:$src2), m, + args, p> { + let mayLoad = 1; + let SchedRW = [WriteALU.Folded, WriteALU.ReadAfterFold]; +} +// BinOpMR_R - Instructions that read "[mem], reg", and write "reg". +class BinOpMR_R<bits<8> o, string m, X86TypeInfo t> + : BinOpMR<o, m, binop_ndd_args, t, (outs t.RegClass:$dst), []>, NDD<1>; +// BinOpMR_RF - Instructions that read "[mem], reg", and write "reg", EFLAGS. +class BinOpMR_RF<bits<8> o, string m, X86TypeInfo t, SDPatternOperator node> + : BinOpMR<o, m, binop_ndd_args, t, (outs t.RegClass:$dst), + [(set t.RegClass:$dst, EFLAGS, (node (t.LoadNode addr:$src1), + t.RegClass:$src2))]>, DefEFLAGS, NDD<1>; +// BinOpMR_F - Instructions that read "[mem], imm8" and write EFLAGS only. +class BinOpMR_F<bits<8> o, string m, X86TypeInfo t, SDPatternOperator node> + : BinOpMR<o, m, binop_args, t, (outs), + [(set EFLAGS, (node (t.LoadNode addr:$src1), t.RegClass:$src2))]>, + Sched<[WriteALU.Folded, ReadDefault, ReadDefault, ReadDefault, + ReadDefault, ReadDefault, WriteALU.ReadAfterFold]>, DefEFLAGS; +// BinOpMR_M - Instructions that read "[mem], reg" and write "[mem]". +class BinOpMR_M<bits<8> o, string m, X86TypeInfo t> + : BinOpMR<o, m, binop_args, t, (outs), []>, + Sched<[WriteALURMW, + // base, scale, index, offset, segment + ReadDefault, ReadDefault, ReadDefault, ReadDefault, ReadDefault]> { + let mayStore = 1; +} +// BinOpMR_MF - Instructions that read "[mem], reg" and write "[mem]", EFLAGS. +class BinOpMR_MF<bits<8> o, string m, X86TypeInfo t, SDPatternOperator node> + : BinOpMR<o, m, binop_args, t, (outs), + [(store (node (load addr:$src1), t.RegClass:$src2), addr:$src1), + (implicit EFLAGS)]>, + Sched<[WriteALURMW, + // base, scale, index, offset, segment + ReadDefault, ReadDefault, ReadDefault, ReadDefault, ReadDefault, + WriteALU.ReadAfterFold]>, // reg + DefEFLAGS { + let mayStore = 1; +} +// BinOpMRF_RF - Instructions that read "[mem], reg", write "reg" and +// read/write EFLAGS. +class BinOpMRF_RF<bits<8> o, string m, X86TypeInfo t, SDPatternOperator node> + : BinOpMR<o, m, binop_ndd_args, t, (outs t.RegClass:$dst), + [(set t.RegClass:$dst, EFLAGS, (node (load addr:$src1), + t.RegClass:$src2, EFLAGS))]>, DefEFLAGS, UseEFLAGS, NDD<1>, + Sched<[WriteADC.Folded, WriteADC.ReadAfterFold]>; +// BinOpMRF_MF - Instructions that read "[mem], reg", write "[mem]" and +// read/write EFLAGS. +class BinOpMRF_MF<bits<8> o, string m, X86TypeInfo t, SDPatternOperator node> + : BinOpMR<o, m, binop_args, t, (outs), + [(store (node (load addr:$src1), t.RegClass:$src2, EFLAGS), + addr:$src1), (implicit EFLAGS)]>, + Sched<[WriteADCRMW, + // base, scale, index, offset, segment + ReadDefault, ReadDefault, ReadDefault, + ReadDefault, ReadDefault, + WriteALU.ReadAfterFold, // reg + WriteALU.ReadAfterFold]>, // EFLAGS + DefEFLAGS, UseEFLAGS { + let mayStore = 1; +} + +// BinOpMI - Instructions that read "[mem], imm". +class BinOpMI<bits<8> o, string m, string args, X86TypeInfo t, Format f, dag out, list<dag> p> + : ITy<o, f, t, out, (ins t.MemOperand:$src1, t.ImmOperand:$src2), m, + args, p> { + let ImmT = t.ImmEncoding; + let mayLoad = 1; +} +// BinOpMI_F - Instructions that read "[mem], imm" and write EFLAGS only. +class BinOpMI_F<bits<8> o, string m, X86TypeInfo t, SDPatternOperator node, + Format f> + : BinOpMI<o, m, binop_args, t, f, (outs), + [(set EFLAGS, (node (t.LoadNode addr:$src1), t.ImmOperator:$src2))]>, + Sched<[WriteALU.Folded]>, DefEFLAGS; +// BinOpMI_R - Instructions that read "[mem], imm" and write "reg". +class BinOpMI_R<bits<8> o, string m, X86TypeInfo t, Format f> + : BinOpMI<o, m, binop_ndd_args, t, f, (outs t.RegClass:$dst), []>, + Sched<[WriteALU.Folded, WriteALU.ReadAfterFold]>, NDD<1>; +// BinOpMI_R - Instructions that read "[mem], imm" and write "reg", EFLAGS. +class BinOpMI_RF<bits<8> o, string m, X86TypeInfo t, SDPatternOperator node, + Format f> + : BinOpMI<o, m, binop_ndd_args, t, f, (outs t.RegClass:$dst), + [(set t.RegClass:$dst, EFLAGS, (node (t.LoadNode addr:$src1), t.ImmOperator:$src2))]>, + Sched<[WriteALU.Folded, WriteALU.ReadAfterFold]>, DefEFLAGS, NDD<1>; +// BinOpMI_M - Instructions that read "[mem], imm" and write "[mem]". +class BinOpMI_M<bits<8> o, string m, X86TypeInfo t, Format f> + : BinOpMI<o, m, binop_args, t, f, (outs), []>, Sched<[WriteALURMW]> { + let mayStore = 1; +} +// BinOpMI_MF - Instructions that read "[mem], imm" and write "[mem]", EFLAGS. +class BinOpMI_MF<bits<8> o, string m, X86TypeInfo t, SDPatternOperator node, Format f> + : BinOpMI<o, m, binop_args, t, f, (outs), + [(store (node (t.VT (load addr:$src1)), + t.ImmOperator:$src2), addr:$src1), (implicit EFLAGS)]>, + Sched<[WriteALURMW]>, DefEFLAGS { + let mayStore = 1; +} +// BinOpMIF_RF - Instructions that read "[mem], imm", write "reg" and +// read/write EFLAGS. +class BinOpMIF_RF<bits<8> o, string m, X86TypeInfo t, SDNode node, Format f> + : BinOpMI<o, m, binop_ndd_args, t, f, (outs t.RegClass:$dst), + [(set t.RegClass:$dst, EFLAGS, (node (t.VT (load addr:$src1)), + t.ImmOperator:$src2, EFLAGS))]>, + Sched<[WriteADC.Folded, WriteADC.ReadAfterFold]>, DefEFLAGS, UseEFLAGS, NDD<1>; +// BinOpMIF_MF - Instructions that read "[mem], imm", write "[mem]" and +// read/write EFLAGS. +class BinOpMIF_MF<bits<8> o, string m, X86TypeInfo t, SDNode node, Format f> + : BinOpMI<o, m, binop_args, t, f, (outs), + [(store (node (t.VT (load addr:$src1)), + t.ImmOperator:$src2, EFLAGS), addr:$src1), (implicit EFLAGS)]>, + Sched<[WriteADCRMW]>, DefEFLAGS, UseEFLAGS { + let mayStore = 1; +} + +// BinOpMI8 - Instructions that read "[mem], imm8". +class BinOpMI8<string m, string args, X86TypeInfo t, Format f, dag out> + : ITy<0x83, f, t, out, (ins t.MemOperand:$src1, t.Imm8Operand:$src2), m, + args, []> { + let ImmT = Imm8; + let mayLoad = 1; +} +// BinOpMI8_F - Instructions that read "[mem], imm8" and write EFLAGS only. +class BinOpMI8_F<string m, X86TypeInfo t, Format f> + : BinOpMI8<m, binop_args, t, f, (outs)>, Sched<[WriteALU.Folded]>, DefEFLAGS; +// BinOpMI8_R - Instructions that read "[mem], imm8" and write "reg". +class BinOpMI8_R<string m, X86TypeInfo t, Format f> + : BinOpMI8<m, binop_ndd_args, t, f, (outs t.RegClass:$dst)>, Sched<[WriteALU.Folded, WriteALU.ReadAfterFold]>, NDD<1>; +// BinOpMI8_RF - Instructions that read "[mem], imm8" and write "reg"/EFLAGS. +class BinOpMI8_RF<string m, X86TypeInfo t, Format f> + : BinOpMI8<m, binop_ndd_args, t, f, (outs t.RegClass:$dst)>, Sched<[WriteALU.Folded, WriteALU.ReadAfterFold]>, DefEFLAGS, NDD<1>; +// BinOpMI8_M - Instructions that read "[mem], imm8" and write "[mem]". +class BinOpMI8_M<string m, X86TypeInfo t, Format f> + : BinOpMI8<m, binop_args, t, f, (outs)>, Sched<[WriteALURMW]> { + let mayStore = 1; +} +// BinOpMI8_MF - Instructions that read "[mem], imm8" and write "[mem]", EFLAGS. +class BinOpMI8_MF<string m, X86TypeInfo t, Format f> + : BinOpMI8<m, binop_args, t, f, (outs)>, Sched<[WriteALURMW]>, DefEFLAGS { + let mayStore = 1; +} +// BinOpMI8F_RF - Instructions that read "[mem], imm8", write "reg" and +// read/write EFLAGS. +class BinOpMI8F_RF<string m, X86TypeInfo t, Format f> + : BinOpMI8<m, binop_ndd_args, t, f, (outs t.RegClass:$dst)>, + Sched<[WriteADC.Folded, WriteADC.ReadAfterFold]>, DefEFLAGS, UseEFLAGS, NDD<1>; +// BinOpMI8F_MF - Instructions that read "[mem], imm8", write "[mem]" and +// read/write EFLAGS. +class BinOpMI8F_MF<string m, X86TypeInfo t, Format f> + : BinOpMI8<m, binop_args, t, f, (outs)>, Sched<[WriteADCRMW]>, DefEFLAGS, UseEFLAGS { + let mayStore = 1; +} + +// BinOpAI - Instructions that read "a-reg imm" (Accumulator register). +class BinOpAI<bits<8> o, string m, X86TypeInfo t, Register areg, string args> + : ITy<o, RawFrm, t, (outs), (ins t.ImmOperand:$src), m, args, []>, + Sched<[WriteALU]> { + let ImmT = t.ImmEncoding; + let Uses = [areg]; +} +// BinOpAI_F - Instructions that read "a-reg imm" and write EFLAGS only. +class BinOpAI_F<bits<8> o, string m, X86TypeInfo t, Register areg, string args> + : BinOpAI<o, m, t, areg, args>, DefEFLAGS; + +// BinOpAI_AF - Instructions that read "a-reg imm" and write a-reg/EFLAGS. +class BinOpAI_AF<bits<8> o, string m, X86TypeInfo t, Register areg, + string args> : BinOpAI<o, m, t, areg, args> { + let Defs = [areg, EFLAGS]; +} +// BinOpAIF_AF - Instructions that read "a-reg imm", write a-reg and read/write +// EFLAGS. +class BinOpAIF_AF<bits<8> o, string m, X86TypeInfo t, Register areg, + string args> : BinOpAI<o, m, t, areg, args> { + let Uses = [areg, EFLAGS]; + let Defs = [areg, EFLAGS]; + let SchedRW = [WriteADC]; +} + +// UnaryOpR - Instructions that read "reg". +class UnaryOpR<bits<8> o, Format f, string m, string args, X86TypeInfo t, + dag out, list<dag> p> + : ITy<o, f, t, out, (ins t.RegClass:$src1), m, args, p>, Sched<[WriteALU]>; +// UnaryOpR_R - Instructions that read "reg" and write "reg". +class UnaryOpR_R<bits<8> o, Format f, string m, X86TypeInfo t, + SDPatternOperator node, bit ndd = 0> + : UnaryOpR<o, f, m, !if(!eq(ndd, 0), unaryop_args, unaryop_ndd_args), t, + (outs t.RegClass:$dst), + [(set t.RegClass:$dst, (node t.RegClass:$src1))]>, NDD<ndd>; +// UnaryOpR_RF - Instructions that read "reg" and write "reg"/EFLAGS. +class UnaryOpR_RF<bits<8> o, Format f, string m, X86TypeInfo t, + SDPatternOperator node, bit ndd = 0> + : UnaryOpR<o, f, m, !if(!eq(ndd, 0), unaryop_args, unaryop_ndd_args), t, + (outs t.RegClass:$dst), + [(set t.RegClass:$dst, (node t.RegClass:$src1)), + (implicit EFLAGS)]>, DefEFLAGS, NDD<ndd>; + +// UnaryOpM - Instructions that read "[mem]". +class UnaryOpM<bits<8> o, Format f, string m, string args, X86TypeInfo t, + dag out, list<dag> p> + : ITy<o, f, t, out, (ins t.MemOperand:$src1), m, args, p> { + let mayLoad = 1; +} +// UnaryOpM_R - Instructions that read "[mem]" and writes "reg". +class UnaryOpM_R<bits<8> o, Format f, string m, X86TypeInfo t, + SDPatternOperator node> + : UnaryOpM<o, f, m, unaryop_ndd_args, t, (outs t.RegClass:$dst), + [(set t.RegClass:$dst, (node (t.LoadNode addr:$src1)))]>, + Sched<[WriteALU.Folded, WriteALU.ReadAfterFold]>, NDD<1>; +// UnaryOpM_RF - Instructions that read "[mem]" and writes "reg"/EFLAGS. +class UnaryOpM_RF<bits<8> o, Format f, string m, X86TypeInfo t, + SDPatternOperator node> + : UnaryOpM<o, f, m, unaryop_ndd_args, t, (outs t.RegClass:$dst), + [(set t.RegClass:$dst, EFLAGS, (node (t.LoadNode addr:$src1)))]>, + Sched<[WriteALU.Folded, WriteALU.ReadAfterFold]>, DefEFLAGS, NDD<1>; +// UnaryOpM_M - Instructions that read "[mem]" and writes "[mem]". +class UnaryOpM_M<bits<8> o, Format f, string m, X86TypeInfo t, + SDPatternOperator node> + : UnaryOpM<o, f, m, unaryop_args, t, (outs), + [(store (node (t.LoadNode addr:$src1)), addr:$src1)]>, + Sched<[WriteALURMW]>{ + let mayStore = 1; +} +// UnaryOpM_MF - Instructions that read "[mem]" and writes "[mem]"/EFLAGS. +class UnaryOpM_MF<bits<8> o, Format f, string m, X86TypeInfo t, + SDPatternOperator node> + : UnaryOpM<o, f, m, unaryop_args, t, (outs), + [(store (node (t.LoadNode addr:$src1)), addr:$src1), + (implicit EFLAGS)]>, Sched<[WriteALURMW]>, DefEFLAGS { + let mayStore = 1; +} diff --git a/contrib/llvm-project/llvm/lib/Target/X86/X86InstrVecCompiler.td b/contrib/llvm-project/llvm/lib/Target/X86/X86InstrVecCompiler.td index 70bd77bba03a..bbd19cf8d5b2 100644 --- a/contrib/llvm-project/llvm/lib/Target/X86/X86InstrVecCompiler.td +++ b/contrib/llvm-project/llvm/lib/Target/X86/X86InstrVecCompiler.td @@ -130,6 +130,9 @@ let Predicates = [HasAVX, NoVLX] in { defm : subvec_zero_lowering<"DQA", VR128, v32i8, v16i8, sub_xmm>; } +let Predicates = [HasAVXNECONVERT, NoVLX] in + defm : subvec_zero_lowering<"DQA", VR128, v16bf16, v8bf16, sub_xmm>; + let Predicates = [HasVLX] in { defm : subvec_zero_lowering<"APDZ128", VR128X, v4f64, v2f64, sub_xmm>; defm : subvec_zero_lowering<"APSZ128", VR128X, v8f32, v4f32, sub_xmm>; @@ -175,6 +178,12 @@ let Predicates = [HasFP16, HasVLX] in { defm : subvec_zero_lowering<"APSZ256", VR256X, v32f16, v16f16, sub_ymm>; } +let Predicates = [HasBF16, HasVLX] in { + defm : subvec_zero_lowering<"APSZ128", VR128X, v16bf16, v8bf16, sub_xmm>; + defm : subvec_zero_lowering<"APSZ128", VR128X, v32bf16, v8bf16, sub_xmm>; + defm : subvec_zero_lowering<"APSZ256", VR256X, v32bf16, v16bf16, sub_ymm>; +} + class maskzeroupper<ValueType vt, RegisterClass RC> : PatLeaf<(vt RC:$src), [{ return isMaskZeroExtended(N); diff --git a/contrib/llvm-project/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/contrib/llvm-project/llvm/lib/Target/X86/X86TargetTransformInfo.cpp index 8a04987e768a..49631f38017a 100644 --- a/contrib/llvm-project/llvm/lib/Target/X86/X86TargetTransformInfo.cpp +++ b/contrib/llvm-project/llvm/lib/Target/X86/X86TargetTransformInfo.cpp @@ -1459,6 +1459,15 @@ InstructionCost X86TTIImpl::getArithmeticInstrCost( Args, CxtI); } +InstructionCost +X86TTIImpl::getAltInstrCost(VectorType *VecTy, unsigned Opcode0, + unsigned Opcode1, const SmallBitVector &OpcodeMask, + TTI::TargetCostKind CostKind) const { + if (isLegalAltInstr(VecTy, Opcode0, Opcode1, OpcodeMask)) + return TTI::TCC_Basic; + return InstructionCost::getInvalid(); +} + InstructionCost X86TTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *BaseTp, ArrayRef<int> Mask, @@ -3724,10 +3733,10 @@ X86TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, { ISD::BITREVERSE, MVT::v8i16, { 8, 13, 10, 16 } }, { ISD::BITREVERSE, MVT::v32i8, { 13, 15, 17, 26 } }, // 2 x 128-bit Op + extract/insert { ISD::BITREVERSE, MVT::v16i8, { 7, 7, 9, 13 } }, - { ISD::BSWAP, MVT::v4i64, { 5, 7, 5, 10 } }, - { ISD::BSWAP, MVT::v2i64, { 2, 3, 1, 3 } }, - { ISD::BSWAP, MVT::v8i32, { 5, 7, 5, 10 } }, - { ISD::BSWAP, MVT::v4i32, { 2, 3, 1, 3 } }, + { ISD::BSWAP, MVT::v4i64, { 5, 6, 5, 10 } }, + { ISD::BSWAP, MVT::v2i64, { 2, 2, 1, 3 } }, + { ISD::BSWAP, MVT::v8i32, { 5, 6, 5, 10 } }, + { ISD::BSWAP, MVT::v4i32, { 2, 2, 1, 3 } }, { ISD::BSWAP, MVT::v16i16, { 5, 6, 5, 10 } }, { ISD::BSWAP, MVT::v8i16, { 2, 2, 1, 3 } }, { ISD::CTLZ, MVT::v4i64, { 29, 33, 49, 58 } }, // 2 x 128-bit Op + extract/insert @@ -3804,6 +3813,9 @@ X86TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, { ISD::FSQRT, MVT::v2f64, { 67, 71, 1, 5 } }, // sqrtpd }; static const CostKindTblEntry SLMCostTbl[] = { + { ISD::BSWAP, MVT::v2i64, { 5, 5, 1, 5 } }, + { ISD::BSWAP, MVT::v4i32, { 5, 5, 1, 5 } }, + { ISD::BSWAP, MVT::v8i16, { 5, 5, 1, 5 } }, { ISD::FSQRT, MVT::f32, { 20, 20, 1, 1 } }, // sqrtss { ISD::FSQRT, MVT::v4f32, { 40, 41, 1, 5 } }, // sqrtps { ISD::FSQRT, MVT::f64, { 35, 35, 1, 1 } }, // sqrtsd @@ -3842,9 +3854,9 @@ X86TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, { ISD::BITREVERSE, MVT::v4i32, { 16, 20, 11, 21 } }, { ISD::BITREVERSE, MVT::v8i16, { 16, 20, 11, 21 } }, { ISD::BITREVERSE, MVT::v16i8, { 11, 12, 10, 16 } }, - { ISD::BSWAP, MVT::v2i64, { 5, 5, 1, 5 } }, - { ISD::BSWAP, MVT::v4i32, { 5, 5, 1, 5 } }, - { ISD::BSWAP, MVT::v8i16, { 5, 5, 1, 5 } }, + { ISD::BSWAP, MVT::v2i64, { 2, 3, 1, 5 } }, + { ISD::BSWAP, MVT::v4i32, { 2, 3, 1, 5 } }, + { ISD::BSWAP, MVT::v8i16, { 2, 3, 1, 5 } }, { ISD::CTLZ, MVT::v2i64, { 18, 28, 28, 35 } }, { ISD::CTLZ, MVT::v4i32, { 15, 20, 22, 28 } }, { ISD::CTLZ, MVT::v8i16, { 13, 17, 16, 22 } }, diff --git a/contrib/llvm-project/llvm/lib/Target/X86/X86TargetTransformInfo.h b/contrib/llvm-project/llvm/lib/Target/X86/X86TargetTransformInfo.h index 0fa0d240a548..07a3fff4f84b 100644 --- a/contrib/llvm-project/llvm/lib/Target/X86/X86TargetTransformInfo.h +++ b/contrib/llvm-project/llvm/lib/Target/X86/X86TargetTransformInfo.h @@ -140,6 +140,11 @@ public: TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None}, ArrayRef<const Value *> Args = ArrayRef<const Value *>(), const Instruction *CxtI = nullptr); + InstructionCost getAltInstrCost(VectorType *VecTy, unsigned Opcode0, + unsigned Opcode1, + const SmallBitVector &OpcodeMask, + TTI::TargetCostKind CostKind) const; + InstructionCost getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp, ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index, |