diff options
Diffstat (limited to 'llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td')
| -rw-r--r-- | llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td | 2462 |
1 files changed, 1336 insertions, 1126 deletions
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td index 06e4d053d5d7..0284ff6d1c6b 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td @@ -7,7 +7,7 @@ //===----------------------------------------------------------------------===// /// /// This file contains the required infrastructure to support code generation -/// for the standard 'V' (Vector) extension, version 0.9. This version is still +/// for the standard 'V' (Vector) extension, version 0.10. This version is still /// experimental as the 'V' extension hasn't been ratified yet. /// /// This file is included from RISCVInstrInfoV.td @@ -20,39 +20,20 @@ def riscv_vmv_x_s : SDNode<"RISCVISD::VMV_X_S", def riscv_read_vlenb : SDNode<"RISCVISD::READ_VLENB", SDTypeProfile<1, 0, [SDTCisVT<0, XLenVT>]>>; -def riscv_vleff : SDNode<"RISCVISD::VLEFF", - SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisPtrTy<1>, - SDTCisVT<2, XLenVT>]>, - [SDNPHasChain, SDNPOutGlue, SDNPMayLoad, - SDNPSideEffect]>; -def riscv_vleff_mask : SDNode<"RISCVISD::VLEFF_MASK", - SDTypeProfile<1, 4, [SDTCisVec<0>, - SDTCisSameAs<0, 1>, - SDTCisPtrTy<2>, - SDTCVecEltisVT<3, i1>, - SDTCisVT<4, XLenVT>]>, - [SDNPHasChain, SDNPOutGlue, SDNPMayLoad, - SDNPSideEffect]>; -def riscv_read_vl : SDNode<"RISCVISD::READ_VL", - SDTypeProfile<1, 0, [SDTCisVT<0, XLenVT>]>, - [SDNPInGlue]>; +// Operand that is allowed to be a register or a 5 bit immediate. +// This allows us to pick between VSETIVLI and VSETVLI opcodes using the same +// pseudo instructions. +def AVL : RegisterOperand<GPR> { + let OperandNamespace = "RISCVOp"; + let OperandType = "OPERAND_AVL"; +} // X0 has special meaning for vsetvl/vsetvli. // rd | rs1 | AVL value | Effect on vl //-------------------------------------------------------------- // !X0 | X0 | VLMAX | Set vl to VLMAX // X0 | X0 | Value in vl | Keep current vl, just change vtype. -def NoX0 : SDNodeXForm<undef, -[{ - auto *C = dyn_cast<ConstantSDNode>(N); - if (C && C->isNullValue()) { - SDLoc DL(N); - return SDValue(CurDAG->getMachineNode(RISCV::ADDI, DL, Subtarget->getXLenVT(), - CurDAG->getRegister(RISCV::X0, Subtarget->getXLenVT()), - CurDAG->getTargetConstant(0, DL, Subtarget->getXLenVT())), 0); - } - return SDValue(N, 0); -}]>; +def VLOp : ComplexPattern<XLenVT, 1, "selectVLOp">; def DecImm : SDNodeXForm<imm, [{ return CurDAG->getTargetConstant(N->getSExtValue() - 1, SDLoc(N), @@ -64,7 +45,7 @@ def DecImm : SDNodeXForm<imm, [{ //===----------------------------------------------------------------------===// // This class describes information associated to the LMUL. -class LMULInfo<int lmul, VReg regclass, VReg wregclass, +class LMULInfo<int lmul, int oct, VReg regclass, VReg wregclass, VReg f2regclass, VReg f4regclass, VReg f8regclass, string mx> { bits<3> value = lmul; // This is encoded as the vlmul field of vtype. VReg vrclass = regclass; @@ -73,22 +54,39 @@ class LMULInfo<int lmul, VReg regclass, VReg wregclass, VReg f4vrclass = f4regclass; VReg f2vrclass = f2regclass; string MX = mx; + int octuple = oct; } // Associate LMUL with tablegen records of register classes. -def V_M1 : LMULInfo<0b000, VR, VRM2, VR, VR, VR, "M1">; -def V_M2 : LMULInfo<0b001, VRM2, VRM4, VR, VR, VR, "M2">; -def V_M4 : LMULInfo<0b010, VRM4, VRM8, VRM2, VR, VR, "M4">; -def V_M8 : LMULInfo<0b011, VRM8,/*NoVReg*/VR, VRM4, VRM2, VR, "M8">; +def V_M1 : LMULInfo<0b000, 8, VR, VRM2, VR, VR, VR, "M1">; +def V_M2 : LMULInfo<0b001, 16, VRM2, VRM4, VR, VR, VR, "M2">; +def V_M4 : LMULInfo<0b010, 32, VRM4, VRM8, VRM2, VR, VR, "M4">; +def V_M8 : LMULInfo<0b011, 64, VRM8,/*NoVReg*/VR, VRM4, VRM2, VR, "M8">; -def V_MF8 : LMULInfo<0b101, VR, VR,/*NoVReg*/VR,/*NoVReg*/VR,/*NoVReg*/VR, "MF8">; -def V_MF4 : LMULInfo<0b110, VR, VR, VR,/*NoVReg*/VR,/*NoVReg*/VR, "MF4">; -def V_MF2 : LMULInfo<0b111, VR, VR, VR, VR,/*NoVReg*/VR, "MF2">; +def V_MF8 : LMULInfo<0b101, 1, VR, VR,/*NoVReg*/VR,/*NoVReg*/VR,/*NoVReg*/VR, "MF8">; +def V_MF4 : LMULInfo<0b110, 2, VR, VR, VR,/*NoVReg*/VR,/*NoVReg*/VR, "MF4">; +def V_MF2 : LMULInfo<0b111, 4, VR, VR, VR, VR,/*NoVReg*/VR, "MF2">; // Used to iterate over all possible LMULs. def MxList { list<LMULInfo> m = [V_MF8, V_MF4, V_MF2, V_M1, V_M2, V_M4, V_M8]; } +// Used for widening and narrowing instructions as it doesn't contain M8. +def MxListW { + list<LMULInfo> m = [V_MF8, V_MF4, V_MF2, V_M1, V_M2, V_M4]; +} +// Use for zext/sext.vf2 +def MxListVF2 { + list<LMULInfo> m = [V_MF4, V_MF2, V_M1, V_M2, V_M4, V_M8]; +} +// Use for zext/sext.vf4 +def MxListVF4 { + list<LMULInfo> m = [V_MF2, V_M1, V_M2, V_M4, V_M8]; +} +// Use for zext/sext.vf8 +def MxListVF8 { + list<LMULInfo> m = [V_M1, V_M2, V_M4, V_M8]; +} class FPR_Info<RegisterClass regclass, string fx> { RegisterClass fprclass = regclass; @@ -102,6 +100,10 @@ def SCALAR_F64 : FPR_Info<FPR64, "F64">; def FPList { list<FPR_Info> fpinfo = [SCALAR_F16, SCALAR_F32, SCALAR_F64]; } +// Used for widening instructions. It excludes F64. +def FPListW { + list<FPR_Info> fpinfo = [SCALAR_F16, SCALAR_F32]; +} class MxSet<int eew> { list<LMULInfo> m = !cond(!eq(eew, 8) : [V_MF8, V_MF4, V_MF2, V_M1, V_M2, V_M4, V_M8], @@ -117,18 +119,8 @@ class NFSet<LMULInfo m> { true: [2, 3, 4, 5, 6, 7, 8]); } -class shift_amount<int num> { - int val = !if(!eq(num, 1), 0, !add(1, shift_amount<!srl(num, 1)>.val)); -} - -class octuple_from_str<string MX> { - int ret = !cond(!eq(MX, "MF8") : 1, - !eq(MX, "MF4") : 2, - !eq(MX, "MF2") : 4, - !eq(MX, "M1") : 8, - !eq(MX, "M2") : 16, - !eq(MX, "M4") : 32, - !eq(MX, "M8") : 64); +class log2<int num> { + int val = !if(!eq(num, 1), 0, !add(1, log2<!srl(num, 1)>.val)); } class octuple_to_str<int octuple> { @@ -142,6 +134,8 @@ class octuple_to_str<int octuple> { "NoDef"))))))); } +def VLOpFrag : PatFrag<(ops), (XLenVT (VLOp (XLenVT AVL:$vl)))>; + // Output pattern for X0 used to represent VLMAX in the pseudo instructions. def VLMax : OutPatFrag<(ops), (XLenVT X0)>; @@ -165,6 +159,7 @@ class VTypeInfo<ValueType Vec, ValueType Mas, int Sew, VReg Reg, LMULInfo M, ValueType Vector = Vec; ValueType Mask = Mas; int SEW = Sew; + int Log2SEW = log2<Sew>.val; VReg RegClass = Reg; LMULInfo LMul = M; ValueType Scalar = Scal; @@ -190,14 +185,16 @@ class GroupVTypeInfo<ValueType Vec, ValueType VecM1, ValueType Mas, int Sew, defset list<VTypeInfo> AllVectors = { defset list<VTypeInfo> AllIntegerVectors = { defset list<VTypeInfo> NoGroupIntegerVectors = { - def VI8MF8: VTypeInfo<vint8mf8_t, vbool64_t, 8, VR, V_MF8>; - def VI8MF4: VTypeInfo<vint8mf4_t, vbool32_t, 8, VR, V_MF4>; - def VI8MF2: VTypeInfo<vint8mf2_t, vbool16_t, 8, VR, V_MF2>; + defset list<VTypeInfo> FractionalGroupIntegerVectors = { + def VI8MF8: VTypeInfo<vint8mf8_t, vbool64_t, 8, VR, V_MF8>; + def VI8MF4: VTypeInfo<vint8mf4_t, vbool32_t, 8, VR, V_MF4>; + def VI8MF2: VTypeInfo<vint8mf2_t, vbool16_t, 8, VR, V_MF2>; + def VI16MF4: VTypeInfo<vint16mf4_t, vbool64_t, 16, VR, V_MF4>; + def VI16MF2: VTypeInfo<vint16mf2_t, vbool32_t, 16, VR, V_MF2>; + def VI32MF2: VTypeInfo<vint32mf2_t, vbool64_t, 32, VR, V_MF2>; + } def VI8M1: VTypeInfo<vint8m1_t, vbool8_t, 8, VR, V_M1>; - def VI16MF4: VTypeInfo<vint16mf4_t, vbool64_t, 16, VR, V_MF4>; - def VI16MF2: VTypeInfo<vint16mf2_t, vbool32_t, 16, VR, V_MF2>; def VI16M1: VTypeInfo<vint16m1_t, vbool16_t, 16, VR, V_M1>; - def VI32MF2: VTypeInfo<vint32mf2_t, vbool64_t, 32, VR, V_MF2>; def VI32M1: VTypeInfo<vint32m1_t, vbool32_t, 32, VR, V_M1>; def VI64M1: VTypeInfo<vint64m1_t, vbool64_t, 64, VR, V_M1>; } @@ -222,13 +219,13 @@ defset list<VTypeInfo> AllVectors = { defset list<VTypeInfo> AllFloatVectors = { defset list<VTypeInfo> NoGroupFloatVectors = { - def VF16MF4: VTypeInfo<vfloat16mf4_t, vbool64_t, 16, VR, V_MF4, f16, FPR16>; - def VF16MF2: VTypeInfo<vfloat16mf2_t, vbool32_t, 16, VR, V_MF2, f16, FPR16>; + defset list<VTypeInfo> FractionalGroupFloatVectors = { + def VF16MF4: VTypeInfo<vfloat16mf4_t, vbool64_t, 16, VR, V_MF4, f16, FPR16>; + def VF16MF2: VTypeInfo<vfloat16mf2_t, vbool32_t, 16, VR, V_MF2, f16, FPR16>; + def VF32MF2: VTypeInfo<vfloat32mf2_t,vbool64_t, 32, VR, V_MF2, f32, FPR32>; + } def VF16M1: VTypeInfo<vfloat16m1_t, vbool16_t, 16, VR, V_M1, f16, FPR16>; - - def VF32MF2: VTypeInfo<vfloat32mf2_t,vbool64_t, 32, VR, V_MF2, f32, FPR32>; def VF32M1: VTypeInfo<vfloat32m1_t, vbool32_t, 32, VR, V_M1, f32, FPR32>; - def VF64M1: VTypeInfo<vfloat64m1_t, vbool64_t, 64, VR, V_M1, f64, FPR64>; } @@ -270,8 +267,10 @@ class GetIntVTypeInfo<VTypeInfo vti> class MTypeInfo<ValueType Mas, LMULInfo M, string Bx> { ValueType Mask = Mas; // {SEW, VLMul} values set a valid VType to deal with this mask type. - // we assume SEW=8 and set corresponding LMUL. - int SEW = 8; + // we assume SEW=1 and set corresponding LMUL. vsetvli insertion will + // look for SEW=1 to optimize based on surrounding instructions. + int SEW = 1; + int Log2SEW = 0; LMULInfo LMul = M; string BX = Bx; // Appendix of mask operations. // The pattern fragment which produces the AVL operand, representing the @@ -413,29 +412,149 @@ def RISCVVPseudosTable : GenericTable { let Fields = [ "Pseudo", "BaseInstr" ]; let PrimaryKey = [ "Pseudo" ]; let PrimaryKeyName = "getPseudoInfo"; + let PrimaryKeyEarlyOut = true; } def RISCVVIntrinsicsTable : GenericTable { let FilterClass = "RISCVVIntrinsic"; let CppTypeName = "RISCVVIntrinsicInfo"; - let Fields = ["IntrinsicID", "ExtendOperand"]; + let Fields = ["IntrinsicID", "SplatOperand"]; let PrimaryKey = ["IntrinsicID"]; let PrimaryKeyName = "getRISCVVIntrinsicInfo"; } -class RISCVZvlsseg<string IntrName, bits<11> S, bits<3> L, bits<3> IL = V_M1.value> { - Intrinsic IntrinsicID = !cast<Intrinsic>(IntrName); - bits<11> SEW = S; +class RISCVVLE<bit M, bit Str, bit F, bits<3> S, bits<3> L> { + bits<1> Masked = M; + bits<1> Strided = Str; + bits<1> FF = F; + bits<3> Log2SEW = S; + bits<3> LMUL = L; + Pseudo Pseudo = !cast<Pseudo>(NAME); +} + +def RISCVVLETable : GenericTable { + let FilterClass = "RISCVVLE"; + let CppTypeName = "VLEPseudo"; + let Fields = ["Masked", "Strided", "FF", "Log2SEW", "LMUL", "Pseudo"]; + let PrimaryKey = ["Masked", "Strided", "FF", "Log2SEW", "LMUL"]; + let PrimaryKeyName = "getVLEPseudo"; +} + +class RISCVVSE<bit M, bit Str, bits<3> S, bits<3> L> { + bits<1> Masked = M; + bits<1> Strided = Str; + bits<3> Log2SEW = S; + bits<3> LMUL = L; + Pseudo Pseudo = !cast<Pseudo>(NAME); +} + +def RISCVVSETable : GenericTable { + let FilterClass = "RISCVVSE"; + let CppTypeName = "VSEPseudo"; + let Fields = ["Masked", "Strided", "Log2SEW", "LMUL", "Pseudo"]; + let PrimaryKey = ["Masked", "Strided", "Log2SEW", "LMUL"]; + let PrimaryKeyName = "getVSEPseudo"; +} + +class RISCVVLX_VSX<bit M, bit O, bits<3> S, bits<3> L, bits<3> IL> { + bits<1> Masked = M; + bits<1> Ordered = O; + bits<3> Log2SEW = S; bits<3> LMUL = L; bits<3> IndexLMUL = IL; Pseudo Pseudo = !cast<Pseudo>(NAME); } -def RISCVZvlssegTable : GenericTable { - let FilterClass = "RISCVZvlsseg"; - let Fields = ["IntrinsicID", "SEW", "LMUL", "IndexLMUL", "Pseudo"]; - let PrimaryKey = ["IntrinsicID", "SEW", "LMUL", "IndexLMUL"]; - let PrimaryKeyName = "getPseudo"; +class RISCVVLX<bit M, bit O, bits<3> S, bits<3> L, bits<3> IL> : + RISCVVLX_VSX<M, O, S, L, IL>; +class RISCVVSX<bit M, bit O, bits<3> S, bits<3> L, bits<3> IL> : + RISCVVLX_VSX<M, O, S, L, IL>; + +class RISCVVLX_VSXTable : GenericTable { + let CppTypeName = "VLX_VSXPseudo"; + let Fields = ["Masked", "Ordered", "Log2SEW", "LMUL", "IndexLMUL", "Pseudo"]; + let PrimaryKey = ["Masked", "Ordered", "Log2SEW", "LMUL", "IndexLMUL"]; +} + +def RISCVVLXTable : RISCVVLX_VSXTable { + let FilterClass = "RISCVVLX"; + let PrimaryKeyName = "getVLXPseudo"; +} + +def RISCVVSXTable : RISCVVLX_VSXTable { + let FilterClass = "RISCVVSX"; + let PrimaryKeyName = "getVSXPseudo"; +} + +class RISCVVLSEG<bits<4> N, bit M, bit Str, bit F, bits<3> S, bits<3> L> { + bits<4> NF = N; + bits<1> Masked = M; + bits<1> Strided = Str; + bits<1> FF = F; + bits<3> Log2SEW = S; + bits<3> LMUL = L; + Pseudo Pseudo = !cast<Pseudo>(NAME); +} + +def RISCVVLSEGTable : GenericTable { + let FilterClass = "RISCVVLSEG"; + let CppTypeName = "VLSEGPseudo"; + let Fields = ["NF", "Masked", "Strided", "FF", "Log2SEW", "LMUL", "Pseudo"]; + let PrimaryKey = ["NF", "Masked", "Strided", "FF", "Log2SEW", "LMUL"]; + let PrimaryKeyName = "getVLSEGPseudo"; +} + +class RISCVVLXSEG<bits<4> N, bit M, bit O, bits<3> S, bits<3> L, bits<3> IL> { + bits<4> NF = N; + bits<1> Masked = M; + bits<1> Ordered = O; + bits<3> Log2SEW = S; + bits<3> LMUL = L; + bits<3> IndexLMUL = IL; + Pseudo Pseudo = !cast<Pseudo>(NAME); +} + +def RISCVVLXSEGTable : GenericTable { + let FilterClass = "RISCVVLXSEG"; + let CppTypeName = "VLXSEGPseudo"; + let Fields = ["NF", "Masked", "Ordered", "Log2SEW", "LMUL", "IndexLMUL", "Pseudo"]; + let PrimaryKey = ["NF", "Masked", "Ordered", "Log2SEW", "LMUL", "IndexLMUL"]; + let PrimaryKeyName = "getVLXSEGPseudo"; +} + +class RISCVVSSEG<bits<4> N, bit M, bit Str, bits<3> S, bits<3> L> { + bits<4> NF = N; + bits<1> Masked = M; + bits<1> Strided = Str; + bits<3> Log2SEW = S; + bits<3> LMUL = L; + Pseudo Pseudo = !cast<Pseudo>(NAME); +} + +def RISCVVSSEGTable : GenericTable { + let FilterClass = "RISCVVSSEG"; + let CppTypeName = "VSSEGPseudo"; + let Fields = ["NF", "Masked", "Strided", "Log2SEW", "LMUL", "Pseudo"]; + let PrimaryKey = ["NF", "Masked", "Strided", "Log2SEW", "LMUL"]; + let PrimaryKeyName = "getVSSEGPseudo"; +} + +class RISCVVSXSEG<bits<4> N, bit M, bit O, bits<3> S, bits<3> L, bits<3> IL> { + bits<4> NF = N; + bits<1> Masked = M; + bits<1> Ordered = O; + bits<3> Log2SEW = S; + bits<3> LMUL = L; + bits<3> IndexLMUL = IL; + Pseudo Pseudo = !cast<Pseudo>(NAME); +} + +def RISCVVSXSEGTable : GenericTable { + let FilterClass = "RISCVVSXSEG"; + let CppTypeName = "VSXSEGPseudo"; + let Fields = ["NF", "Masked", "Ordered", "Log2SEW", "LMUL", "IndexLMUL", "Pseudo"]; + let PrimaryKey = ["NF", "Masked", "Ordered", "Log2SEW", "LMUL", "IndexLMUL"]; + let PrimaryKeyName = "getVSXSEGPseudo"; } //===----------------------------------------------------------------------===// @@ -458,39 +577,13 @@ class PseudoToVInst<string PseudoInst> { !subst("_B32", "", !subst("_B64", "", !subst("_MASK", "", + !subst("_COMMUTABLE", "", + !subst("_TA", "", + !subst("_TIED", "", !subst("F16", "F", !subst("F32", "F", !subst("F64", "F", - !subst("Pseudo", "", PseudoInst))))))))))))))))))); -} - -class ToLowerCase<string Upper> { - string L = !subst("FF", "ff", - !subst("VLSEG", "vlseg", - !subst("VLSSEG", "vlsseg", - !subst("VSSEG", "vsseg", - !subst("VSSSEG", "vssseg", - !subst("VLOXSEG", "vloxseg", - !subst("VLUXSEG", "vluxseg", - !subst("VSOXSEG", "vsoxseg", - !subst("VSUXSEG", "vsuxseg", Upper))))))))); -} - -// Example: PseudoVLSEG2E32_V_M2 -> int_riscv_vlseg2 -// Example: PseudoVLSEG2E32_V_M2_MASK -> int_riscv_vlseg2_mask -class PseudoToIntrinsic<string PseudoInst, bit IsMasked> { - string Intrinsic = !strconcat("int_riscv_", - ToLowerCase< - !subst("E8", "", - !subst("E16", "", - !subst("E32", "", - !subst("E64", "", - !subst("EI8", "", - !subst("EI16", "", - !subst("EI32", "", - !subst("EI64", "", - !subst("_V", "", PseudoToVInst<PseudoInst>.VInst)))))))))>.L, - !if(IsMasked, "_mask", "")); + !subst("Pseudo", "", PseudoInst)))))))))))))))))))))); } // The destination vector register group for a masked vector instruction cannot @@ -502,7 +595,18 @@ class GetVRegNoV0<VReg VRegClass> { !eq(VRegClass, VRM2) : VRM2NoV0, !eq(VRegClass, VRM4) : VRM4NoV0, !eq(VRegClass, VRM8) : VRM8NoV0, - !eq(1, 1) : VRegClass); + !eq(VRegClass, VRN2M1) : VRN2M1NoV0, + !eq(VRegClass, VRN2M2) : VRN2M2NoV0, + !eq(VRegClass, VRN2M4) : VRN2M4NoV0, + !eq(VRegClass, VRN3M1) : VRN3M1NoV0, + !eq(VRegClass, VRN3M2) : VRN3M2NoV0, + !eq(VRegClass, VRN4M1) : VRN4M1NoV0, + !eq(VRegClass, VRN4M2) : VRN4M2NoV0, + !eq(VRegClass, VRN5M1) : VRN5M1NoV0, + !eq(VRegClass, VRN6M1) : VRN6M1NoV0, + !eq(VRegClass, VRN7M1) : VRN7M1NoV0, + !eq(VRegClass, VRN8M1) : VRN8M1NoV0, + true : VRegClass); } // Join strings in list using separator and ignoring empty elements @@ -521,158 +625,151 @@ class VPseudo<Instruction instr, LMULInfo m, dag outs, dag ins> : let VLMul = m.value; } -class VPseudoUSLoadNoMask<VReg RetClass>: +class VPseudoUSLoadNoMask<VReg RetClass, int EEW, bit isFF> : Pseudo<(outs RetClass:$rd), - (ins GPR:$rs1, GPR:$vl, ixlenimm:$sew),[]>, - RISCVVPseudo { + (ins GPR:$rs1, AVL:$vl, ixlenimm:$sew),[]>, + RISCVVPseudo, + RISCVVLE</*Masked*/0, /*Strided*/0, /*FF*/isFF, log2<EEW>.val, VLMul> { let mayLoad = 1; let mayStore = 0; let hasSideEffects = 0; - let usesCustomInserter = 1; - let Uses = [VL, VTYPE]; let HasVLOp = 1; let HasSEWOp = 1; let HasDummyMask = 1; let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst); } -class VPseudoUSLoadMask<VReg RetClass>: +class VPseudoUSLoadMask<VReg RetClass, int EEW, bit isFF> : Pseudo<(outs GetVRegNoV0<RetClass>.R:$rd), (ins GetVRegNoV0<RetClass>.R:$merge, GPR:$rs1, - VMaskOp:$vm, GPR:$vl, ixlenimm:$sew),[]>, - RISCVVPseudo { + VMaskOp:$vm, AVL:$vl, ixlenimm:$sew),[]>, + RISCVVPseudo, + RISCVVLE</*Masked*/1, /*Strided*/0, /*FF*/isFF, log2<EEW>.val, VLMul> { let mayLoad = 1; let mayStore = 0; let hasSideEffects = 0; - let usesCustomInserter = 1; let Constraints = "$rd = $merge"; - let Uses = [VL, VTYPE]; let HasVLOp = 1; let HasSEWOp = 1; let HasMergeOp = 1; let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst); } -class VPseudoSLoadNoMask<VReg RetClass>: +class VPseudoSLoadNoMask<VReg RetClass, int EEW>: Pseudo<(outs RetClass:$rd), - (ins GPR:$rs1, GPR:$rs2, GPR:$vl, ixlenimm:$sew),[]>, - RISCVVPseudo { + (ins GPR:$rs1, GPR:$rs2, AVL:$vl, ixlenimm:$sew),[]>, + RISCVVPseudo, + RISCVVLE</*Masked*/0, /*Strided*/1, /*FF*/0, log2<EEW>.val, VLMul> { let mayLoad = 1; let mayStore = 0; let hasSideEffects = 0; - let usesCustomInserter = 1; - let Uses = [VL, VTYPE]; let HasVLOp = 1; let HasSEWOp = 1; let HasDummyMask = 1; let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst); } -class VPseudoSLoadMask<VReg RetClass>: +class VPseudoSLoadMask<VReg RetClass, int EEW>: Pseudo<(outs GetVRegNoV0<RetClass>.R:$rd), (ins GetVRegNoV0<RetClass>.R:$merge, GPR:$rs1, GPR:$rs2, - VMaskOp:$vm, GPR:$vl, ixlenimm:$sew),[]>, - RISCVVPseudo { + VMaskOp:$vm, AVL:$vl, ixlenimm:$sew),[]>, + RISCVVPseudo, + RISCVVLE</*Masked*/1, /*Strided*/1, /*FF*/0, log2<EEW>.val, VLMul> { let mayLoad = 1; let mayStore = 0; let hasSideEffects = 0; - let usesCustomInserter = 1; let Constraints = "$rd = $merge"; - let Uses = [VL, VTYPE]; let HasVLOp = 1; let HasSEWOp = 1; let HasMergeOp = 1; let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst); } -class VPseudoILoadNoMask<VReg RetClass, VReg IdxClass>: +class VPseudoILoadNoMask<VReg RetClass, VReg IdxClass, int EEW, bits<3> LMUL, + bit Ordered, bit EarlyClobber>: Pseudo<(outs RetClass:$rd), - (ins GPR:$rs1, IdxClass:$rs2, GPR:$vl, ixlenimm:$sew),[]>, - RISCVVPseudo { + (ins GPR:$rs1, IdxClass:$rs2, AVL:$vl, ixlenimm:$sew),[]>, + RISCVVPseudo, + RISCVVLX</*Masked*/0, Ordered, log2<EEW>.val, VLMul, LMUL> { let mayLoad = 1; let mayStore = 0; let hasSideEffects = 0; - let usesCustomInserter = 1; - let Uses = [VL, VTYPE]; let HasVLOp = 1; let HasSEWOp = 1; let HasDummyMask = 1; + let Constraints = !if(!eq(EarlyClobber, 1), "@earlyclobber $rd", ""); let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst); } -class VPseudoILoadMask<VReg RetClass, VReg IdxClass>: +class VPseudoILoadMask<VReg RetClass, VReg IdxClass, int EEW, bits<3> LMUL, + bit Ordered, bit EarlyClobber>: Pseudo<(outs GetVRegNoV0<RetClass>.R:$rd), (ins GetVRegNoV0<RetClass>.R:$merge, GPR:$rs1, IdxClass:$rs2, - VMaskOp:$vm, GPR:$vl, ixlenimm:$sew),[]>, - RISCVVPseudo { + VMaskOp:$vm, AVL:$vl, ixlenimm:$sew),[]>, + RISCVVPseudo, + RISCVVLX</*Masked*/1, Ordered, log2<EEW>.val, VLMul, LMUL> { let mayLoad = 1; let mayStore = 0; let hasSideEffects = 0; - let usesCustomInserter = 1; - let Constraints = "$rd = $merge"; - let Uses = [VL, VTYPE]; + let Constraints = !if(!eq(EarlyClobber, 1), "@earlyclobber $rd, $rd = $merge", "$rd = $merge"); let HasVLOp = 1; let HasSEWOp = 1; let HasMergeOp = 1; let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst); } -class VPseudoUSStoreNoMask<VReg StClass>: +class VPseudoUSStoreNoMask<VReg StClass, int EEW>: Pseudo<(outs), - (ins StClass:$rd, GPR:$rs1, GPR:$vl, ixlenimm:$sew),[]>, - RISCVVPseudo { + (ins StClass:$rd, GPR:$rs1, AVL:$vl, ixlenimm:$sew),[]>, + RISCVVPseudo, + RISCVVSE</*Masked*/0, /*Strided*/0, log2<EEW>.val, VLMul> { let mayLoad = 0; let mayStore = 1; let hasSideEffects = 0; - let usesCustomInserter = 1; - let Uses = [VL, VTYPE]; let HasVLOp = 1; let HasSEWOp = 1; let HasDummyMask = 1; let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst); } -class VPseudoUSStoreMask<VReg StClass>: +class VPseudoUSStoreMask<VReg StClass, int EEW>: Pseudo<(outs), - (ins StClass:$rd, GPR:$rs1, VMaskOp:$vm, GPR:$vl, ixlenimm:$sew),[]>, - RISCVVPseudo { + (ins StClass:$rd, GPR:$rs1, VMaskOp:$vm, AVL:$vl, ixlenimm:$sew),[]>, + RISCVVPseudo, + RISCVVSE</*Masked*/1, /*Strided*/0, log2<EEW>.val, VLMul> { let mayLoad = 0; let mayStore = 1; let hasSideEffects = 0; - let usesCustomInserter = 1; - let Uses = [VL, VTYPE]; let HasVLOp = 1; let HasSEWOp = 1; let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst); } -class VPseudoSStoreNoMask<VReg StClass>: +class VPseudoSStoreNoMask<VReg StClass, int EEW>: Pseudo<(outs), - (ins StClass:$rd, GPR:$rs1, GPR:$rs2, GPR:$vl, ixlenimm:$sew),[]>, - RISCVVPseudo { + (ins StClass:$rd, GPR:$rs1, GPR:$rs2, AVL:$vl, ixlenimm:$sew),[]>, + RISCVVPseudo, + RISCVVSE</*Masked*/0, /*Strided*/1, log2<EEW>.val, VLMul> { let mayLoad = 0; let mayStore = 1; let hasSideEffects = 0; - let usesCustomInserter = 1; - let Uses = [VL, VTYPE]; let HasVLOp = 1; let HasSEWOp = 1; let HasDummyMask = 1; let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst); } -class VPseudoSStoreMask<VReg StClass>: +class VPseudoSStoreMask<VReg StClass, int EEW>: Pseudo<(outs), - (ins StClass:$rd, GPR:$rs1, GPR:$rs2, VMaskOp:$vm, GPR:$vl, ixlenimm:$sew),[]>, - RISCVVPseudo { + (ins StClass:$rd, GPR:$rs1, GPR:$rs2, VMaskOp:$vm, AVL:$vl, ixlenimm:$sew),[]>, + RISCVVPseudo, + RISCVVSE</*Masked*/1, /*Strided*/1, log2<EEW>.val, VLMul> { let mayLoad = 0; let mayStore = 1; let hasSideEffects = 0; - let usesCustomInserter = 1; - let Uses = [VL, VTYPE]; let HasVLOp = 1; let HasSEWOp = 1; let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst); @@ -682,13 +779,11 @@ class VPseudoSStoreMask<VReg StClass>: class VPseudoUnaryNoDummyMask<VReg RetClass, DAGOperand Op2Class> : Pseudo<(outs RetClass:$rd), - (ins Op2Class:$rs1, GPR:$vl, ixlenimm:$sew), []>, + (ins Op2Class:$rs1, AVL:$vl, ixlenimm:$sew), []>, RISCVVPseudo { let mayLoad = 0; let mayStore = 0; let hasSideEffects = 0; - let usesCustomInserter = 1; - let Uses = [VL, VTYPE]; let HasVLOp = 1; let HasSEWOp = 1; let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst); @@ -696,13 +791,11 @@ class VPseudoUnaryNoDummyMask<VReg RetClass, class VPseudoNullaryNoMask<VReg RegClass>: Pseudo<(outs RegClass:$rd), - (ins GPR:$vl, ixlenimm:$sew), + (ins AVL:$vl, ixlenimm:$sew), []>, RISCVVPseudo { let mayLoad = 0; let mayStore = 0; let hasSideEffects = 0; - let usesCustomInserter = 1; - let Uses = [VL, VTYPE]; let HasVLOp = 1; let HasSEWOp = 1; let HasDummyMask = 1; @@ -711,14 +804,12 @@ class VPseudoNullaryNoMask<VReg RegClass>: class VPseudoNullaryMask<VReg RegClass>: Pseudo<(outs GetVRegNoV0<RegClass>.R:$rd), - (ins GetVRegNoV0<RegClass>.R:$merge, VMaskOp:$vm, GPR:$vl, + (ins GetVRegNoV0<RegClass>.R:$merge, VMaskOp:$vm, AVL:$vl, ixlenimm:$sew), []>, RISCVVPseudo { let mayLoad = 0; let mayStore = 0; let hasSideEffects = 0; - let usesCustomInserter = 1; let Constraints ="$rd = $merge"; - let Uses = [VL, VTYPE]; let HasVLOp = 1; let HasSEWOp = 1; let HasMergeOp = 1; @@ -728,13 +819,11 @@ class VPseudoNullaryMask<VReg RegClass>: // Nullary for pseudo instructions. They are expanded in // RISCVExpandPseudoInsts pass. class VPseudoNullaryPseudoM<string BaseInst> - : Pseudo<(outs VR:$rd), (ins GPR:$vl, ixlenimm:$sew), []>, + : Pseudo<(outs VR:$rd), (ins AVL:$vl, ixlenimm:$sew), []>, RISCVVPseudo { let mayLoad = 0; let mayStore = 0; let hasSideEffects = 0; - let usesCustomInserter = 1; - let Uses = [VL, VTYPE]; let HasVLOp = 1; let HasSEWOp = 1; // BaseInstr is not used in RISCVExpandPseudoInsts pass. @@ -745,14 +834,12 @@ class VPseudoNullaryPseudoM<string BaseInst> // RetClass could be GPR or VReg. class VPseudoUnaryNoMask<DAGOperand RetClass, VReg OpClass, string Constraint = ""> : Pseudo<(outs RetClass:$rd), - (ins OpClass:$rs2, GPR:$vl, ixlenimm:$sew), []>, + (ins OpClass:$rs2, AVL:$vl, ixlenimm:$sew), []>, RISCVVPseudo { let mayLoad = 0; let mayStore = 0; let hasSideEffects = 0; - let usesCustomInserter = 1; let Constraints = Constraint; - let Uses = [VL, VTYPE]; let HasVLOp = 1; let HasSEWOp = 1; let HasDummyMask = 1; @@ -762,14 +849,12 @@ class VPseudoUnaryNoMask<DAGOperand RetClass, VReg OpClass, string Constraint = class VPseudoUnaryMask<VReg RetClass, VReg OpClass, string Constraint = ""> : Pseudo<(outs GetVRegNoV0<RetClass>.R:$rd), (ins GetVRegNoV0<RetClass>.R:$merge, OpClass:$rs2, - VMaskOp:$vm, GPR:$vl, ixlenimm:$sew), []>, + VMaskOp:$vm, AVL:$vl, ixlenimm:$sew), []>, RISCVVPseudo { let mayLoad = 0; let mayStore = 0; let hasSideEffects = 0; - let usesCustomInserter = 1; let Constraints = Join<[Constraint, "$rd = $merge"], ",">.ret; - let Uses = [VL, VTYPE]; let HasVLOp = 1; let HasSEWOp = 1; let HasMergeOp = 1; @@ -779,32 +864,13 @@ class VPseudoUnaryMask<VReg RetClass, VReg OpClass, string Constraint = ""> : // mask unary operation without maskedoff class VPseudoMaskUnarySOutMask: Pseudo<(outs GPR:$rd), - (ins VR:$rs1, VMaskOp:$vm, GPR:$vl, ixlenimm:$sew), []>, - RISCVVPseudo { - let mayLoad = 0; - let mayStore = 0; - let hasSideEffects = 0; - let usesCustomInserter = 1; - let Uses = [VL, VTYPE]; - let HasVLOp = 1; - let HasSEWOp = 1; - let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst); -} - -// Masked mask operation have no $rd=$merge constraints -class VPseudoUnaryMOutMask: - Pseudo<(outs VR:$rd), - (ins VR:$merge, VR:$rs1, VMaskOp:$vm, GPR:$vl, ixlenimm:$sew), []>, + (ins VR:$rs1, VMaskOp:$vm, AVL:$vl, ixlenimm:$sew), []>, RISCVVPseudo { let mayLoad = 0; let mayStore = 0; let hasSideEffects = 0; - let usesCustomInserter = 1; - let Constraints = "$rd = $merge"; - let Uses = [VL, VTYPE]; let HasVLOp = 1; let HasSEWOp = 1; - let HasMergeOp = 1; let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst); } @@ -814,15 +880,13 @@ class VPseudoUnaryAnyMask<VReg RetClass, Pseudo<(outs RetClass:$rd), (ins RetClass:$merge, Op1Class:$rs2, - VR:$vm, GPR:$vl, ixlenimm:$sew), + VR:$vm, AVL:$vl, ixlenimm:$sew), []>, RISCVVPseudo { let mayLoad = 0; let mayStore = 0; let hasSideEffects = 0; - let usesCustomInserter = 1; let Constraints = "@earlyclobber $rd, $rd = $merge"; - let Uses = [VL, VTYPE]; let HasVLOp = 1; let HasSEWOp = 1; let HasMergeOp = 1; @@ -834,70 +898,125 @@ class VPseudoBinaryNoMask<VReg RetClass, DAGOperand Op2Class, string Constraint> : Pseudo<(outs RetClass:$rd), - (ins Op1Class:$rs2, Op2Class:$rs1, GPR:$vl, ixlenimm:$sew), []>, + (ins Op1Class:$rs2, Op2Class:$rs1, AVL:$vl, ixlenimm:$sew), []>, RISCVVPseudo { let mayLoad = 0; let mayStore = 0; let hasSideEffects = 0; - let usesCustomInserter = 1; let Constraints = Constraint; - let Uses = [VL, VTYPE]; let HasVLOp = 1; let HasSEWOp = 1; let HasDummyMask = 1; let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst); } -class VPseudoIStoreNoMask<VReg StClass, VReg IdxClass>: +class VPseudoTiedBinaryNoMask<VReg RetClass, + DAGOperand Op2Class, + string Constraint> : + Pseudo<(outs RetClass:$rd), + (ins RetClass:$rs2, Op2Class:$rs1, AVL:$vl, ixlenimm:$sew), []>, + RISCVVPseudo { + let mayLoad = 0; + let mayStore = 0; + let hasSideEffects = 0; + let Constraints = Join<[Constraint, "$rd = $rs2"], ",">.ret; + let HasVLOp = 1; + let HasSEWOp = 1; + let HasDummyMask = 1; + let ForceTailAgnostic = 1; + let isConvertibleToThreeAddress = 1; + let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst); +} + +class VPseudoIStoreNoMask<VReg StClass, VReg IdxClass, int EEW, bits<3> LMUL, + bit Ordered>: Pseudo<(outs), - (ins StClass:$rd, GPR:$rs1, IdxClass:$rs2, GPR:$vl, ixlenimm:$sew),[]>, - RISCVVPseudo { + (ins StClass:$rd, GPR:$rs1, IdxClass:$rs2, AVL:$vl, ixlenimm:$sew),[]>, + RISCVVPseudo, + RISCVVSX</*Masked*/0, Ordered, log2<EEW>.val, VLMul, LMUL> { let mayLoad = 0; let mayStore = 1; let hasSideEffects = 0; - let usesCustomInserter = 1; - let Uses = [VL, VTYPE]; let HasVLOp = 1; let HasSEWOp = 1; let HasDummyMask = 1; let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst); } -class VPseudoIStoreMask<VReg StClass, VReg IdxClass>: +class VPseudoIStoreMask<VReg StClass, VReg IdxClass, int EEW, bits<3> LMUL, + bit Ordered>: Pseudo<(outs), - (ins StClass:$rd, GPR:$rs1, IdxClass:$rs2, VMaskOp:$vm, GPR:$vl, ixlenimm:$sew),[]>, - RISCVVPseudo { + (ins StClass:$rd, GPR:$rs1, IdxClass:$rs2, VMaskOp:$vm, AVL:$vl, ixlenimm:$sew),[]>, + RISCVVPseudo, + RISCVVSX</*Masked*/1, Ordered, log2<EEW>.val, VLMul, LMUL> { let mayLoad = 0; let mayStore = 1; let hasSideEffects = 0; - let usesCustomInserter = 1; - let Uses = [VL, VTYPE]; let HasVLOp = 1; let HasSEWOp = 1; let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst); } class VPseudoBinaryMask<VReg RetClass, - VReg Op1Class, + RegisterClass Op1Class, DAGOperand Op2Class, string Constraint> : Pseudo<(outs GetVRegNoV0<RetClass>.R:$rd), (ins GetVRegNoV0<RetClass>.R:$merge, Op1Class:$rs2, Op2Class:$rs1, - VMaskOp:$vm, GPR:$vl, ixlenimm:$sew), []>, + VMaskOp:$vm, AVL:$vl, ixlenimm:$sew), []>, + RISCVVPseudo { + let mayLoad = 0; + let mayStore = 0; + let hasSideEffects = 0; + let Constraints = Join<[Constraint, "$rd = $merge"], ",">.ret; + let HasVLOp = 1; + let HasSEWOp = 1; + let HasMergeOp = 1; + let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst); +} + +// Like VPseudoBinaryMask, but output can be V0. +class VPseudoBinaryMOutMask<VReg RetClass, + RegisterClass Op1Class, + DAGOperand Op2Class, + string Constraint> : + Pseudo<(outs RetClass:$rd), + (ins RetClass:$merge, + Op1Class:$rs2, Op2Class:$rs1, + VMaskOp:$vm, AVL:$vl, ixlenimm:$sew), []>, RISCVVPseudo { let mayLoad = 0; let mayStore = 0; let hasSideEffects = 0; - let usesCustomInserter = 1; let Constraints = Join<[Constraint, "$rd = $merge"], ",">.ret; - let Uses = [VL, VTYPE]; let HasVLOp = 1; let HasSEWOp = 1; let HasMergeOp = 1; let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst); } +// Special version of VPseudoBinaryMask where we pretend the first source is +// tied to the destination so we can workaround the earlyclobber constraint. +// This allows maskedoff and rs2 to be the same register. +class VPseudoTiedBinaryMask<VReg RetClass, + DAGOperand Op2Class, + string Constraint> : + Pseudo<(outs GetVRegNoV0<RetClass>.R:$rd), + (ins GetVRegNoV0<RetClass>.R:$merge, + Op2Class:$rs1, + VMaskOp:$vm, AVL:$vl, ixlenimm:$sew), []>, + RISCVVPseudo { + let mayLoad = 0; + let mayStore = 0; + let hasSideEffects = 0; + let Constraints = Join<[Constraint, "$rd = $merge"], ",">.ret; + let HasVLOp = 1; + let HasSEWOp = 1; + let HasMergeOp = 0; // Merge is also rs2. + let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst); +} + class VPseudoBinaryCarryIn<VReg RetClass, VReg Op1Class, DAGOperand Op2Class, @@ -906,16 +1025,14 @@ class VPseudoBinaryCarryIn<VReg RetClass, string Constraint> : Pseudo<(outs RetClass:$rd), !if(CarryIn, - (ins Op1Class:$rs2, Op2Class:$rs1, VMV0:$carry, GPR:$vl, + (ins Op1Class:$rs2, Op2Class:$rs1, VMV0:$carry, AVL:$vl, ixlenimm:$sew), - (ins Op1Class:$rs2, Op2Class:$rs1, GPR:$vl, ixlenimm:$sew)), []>, + (ins Op1Class:$rs2, Op2Class:$rs1, AVL:$vl, ixlenimm:$sew)), []>, RISCVVPseudo { let mayLoad = 0; let mayStore = 0; let hasSideEffects = 0; - let usesCustomInserter = 1; let Constraints = Constraint; - let Uses = [VL, VTYPE]; let HasVLOp = 1; let HasSEWOp = 1; let HasMergeOp = 0; @@ -924,20 +1041,18 @@ class VPseudoBinaryCarryIn<VReg RetClass, } class VPseudoTernaryNoMask<VReg RetClass, - VReg Op1Class, + RegisterClass Op1Class, DAGOperand Op2Class, string Constraint> : Pseudo<(outs RetClass:$rd), (ins RetClass:$rs3, Op1Class:$rs1, Op2Class:$rs2, - GPR:$vl, ixlenimm:$sew), + AVL:$vl, ixlenimm:$sew), []>, RISCVVPseudo { let mayLoad = 0; let mayStore = 0; let hasSideEffects = 0; - let usesCustomInserter = 1; let Constraints = Join<[Constraint, "$rd = $rs3"], ",">.ret; - let Uses = [VL, VTYPE]; let HasVLOp = 1; let HasSEWOp = 1; let HasMergeOp = 1; @@ -951,14 +1066,12 @@ class VPseudoAMOWDNoMask<VReg RetClass, (ins GPR:$rs1, Op1Class:$vs2, GetVRegNoV0<RetClass>.R:$vd, - GPR:$vl, ixlenimm:$sew), []>, + AVL:$vl, ixlenimm:$sew), []>, RISCVVPseudo { let mayLoad = 1; let mayStore = 1; let hasSideEffects = 1; - let usesCustomInserter = 1; let Constraints = "$vd_wd = $vd"; - let Uses = [VL, VTYPE]; let HasVLOp = 1; let HasSEWOp = 1; let HasDummyMask = 1; @@ -971,14 +1084,12 @@ class VPseudoAMOWDMask<VReg RetClass, (ins GPR:$rs1, Op1Class:$vs2, GetVRegNoV0<RetClass>.R:$vd, - VMaskOp:$vm, GPR:$vl, ixlenimm:$sew), []>, + VMaskOp:$vm, AVL:$vl, ixlenimm:$sew), []>, RISCVVPseudo { let mayLoad = 1; let mayStore = 1; let hasSideEffects = 1; - let usesCustomInserter = 1; let Constraints = "$vd_wd = $vd"; - let Uses = [VL, VTYPE]; let HasVLOp = 1; let HasSEWOp = 1; let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst); @@ -991,17 +1102,15 @@ multiclass VPseudoAMOEI<int eew> { // therefore only [32, 64] is allowed here. foreach sew = [32, 64] in { foreach lmul = MxSet<sew>.m in { - defvar octuple_lmul = octuple_from_str<lmul.MX>.ret; + defvar octuple_lmul = lmul.octuple; // Calculate emul = eew * lmul / sew - defvar octuple_emul = !srl(!mul(eew, octuple_lmul), shift_amount<sew>.val); + defvar octuple_emul = !srl(!mul(eew, octuple_lmul), log2<sew>.val); if !and(!ge(octuple_emul, 1), !le(octuple_emul, 64)) then { defvar emulMX = octuple_to_str<octuple_emul>.ret; - defvar lmulMX = octuple_to_str<octuple_lmul>.ret; defvar emul= !cast<LMULInfo>("V_" # emulMX); - defvar lmul = !cast<LMULInfo>("V_" # lmulMX); let VLMul = lmul.value in { - def "_WD_" # lmulMX # "_" # emulMX : VPseudoAMOWDNoMask<lmul.vrclass, emul.vrclass>; - def "_WD_" # lmulMX # "_" # emulMX # "_MASK" : VPseudoAMOWDMask<lmul.vrclass, emul.vrclass>; + def "_WD_" # lmul.MX # "_" # emulMX : VPseudoAMOWDNoMask<lmul.vrclass, emul.vrclass>; + def "_WD_" # lmul.MX # "_" # emulMX # "_MASK" : VPseudoAMOWDMask<lmul.vrclass, emul.vrclass>; } } } @@ -1010,281 +1119,312 @@ multiclass VPseudoAMOEI<int eew> { multiclass VPseudoAMO { foreach eew = EEWList in - defm "EI" # eew : VPseudoAMOEI<eew>; + defm "EI" # eew : VPseudoAMOEI<eew>; } -class VPseudoUSSegLoadNoMask<VReg RetClass, bits<11> EEW>: +class VPseudoUSSegLoadNoMask<VReg RetClass, int EEW, bits<4> NF, bit isFF>: Pseudo<(outs RetClass:$rd), - (ins GPR:$rs1, GPR:$vl, ixlenimm:$sew),[]>, + (ins GPR:$rs1, AVL:$vl, ixlenimm:$sew),[]>, RISCVVPseudo, - RISCVZvlsseg<PseudoToIntrinsic<NAME, false>.Intrinsic, EEW, VLMul> { + RISCVVLSEG<NF, /*Masked*/0, /*Strided*/0, /*FF*/isFF, log2<EEW>.val, VLMul> { let mayLoad = 1; let mayStore = 0; let hasSideEffects = 0; - let usesCustomInserter = 1; - let Uses = [VL, VTYPE]; let HasVLOp = 1; let HasSEWOp = 1; let HasDummyMask = 1; let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst); } -class VPseudoUSSegLoadMask<VReg RetClass, bits<11> EEW>: +class VPseudoUSSegLoadMask<VReg RetClass, int EEW, bits<4> NF, bit isFF>: Pseudo<(outs GetVRegNoV0<RetClass>.R:$rd), (ins GetVRegNoV0<RetClass>.R:$merge, GPR:$rs1, - VMaskOp:$vm, GPR:$vl, ixlenimm:$sew),[]>, + VMaskOp:$vm, AVL:$vl, ixlenimm:$sew),[]>, RISCVVPseudo, - RISCVZvlsseg<PseudoToIntrinsic<NAME, true>.Intrinsic, EEW, VLMul> { + RISCVVLSEG<NF, /*Masked*/1, /*Strided*/0, /*FF*/isFF, log2<EEW>.val, VLMul> { let mayLoad = 1; let mayStore = 0; let hasSideEffects = 0; - let usesCustomInserter = 1; let Constraints = "$rd = $merge"; - let Uses = [VL, VTYPE]; let HasVLOp = 1; let HasSEWOp = 1; let HasMergeOp = 1; let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst); } -class VPseudoSSegLoadNoMask<VReg RetClass, bits<11> EEW>: +class VPseudoSSegLoadNoMask<VReg RetClass, int EEW, bits<4> NF>: Pseudo<(outs RetClass:$rd), - (ins GPR:$rs1, GPR:$offset, GPR:$vl, ixlenimm:$sew),[]>, + (ins GPR:$rs1, GPR:$offset, AVL:$vl, ixlenimm:$sew),[]>, RISCVVPseudo, - RISCVZvlsseg<PseudoToIntrinsic<NAME, false>.Intrinsic, EEW, VLMul> { + RISCVVLSEG<NF, /*Masked*/0, /*Strided*/1, /*FF*/0, log2<EEW>.val, VLMul> { + let mayLoad = 1; let mayLoad = 1; let mayStore = 0; let hasSideEffects = 0; - let usesCustomInserter = 1; - let Uses = [VL, VTYPE]; let HasVLOp = 1; let HasSEWOp = 1; let HasDummyMask = 1; let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst); } -class VPseudoSSegLoadMask<VReg RetClass, bits<11> EEW>: +class VPseudoSSegLoadMask<VReg RetClass, int EEW, bits<4> NF>: Pseudo<(outs GetVRegNoV0<RetClass>.R:$rd), (ins GetVRegNoV0<RetClass>.R:$merge, GPR:$rs1, - GPR:$offset, VMaskOp:$vm, GPR:$vl, ixlenimm:$sew),[]>, + GPR:$offset, VMaskOp:$vm, AVL:$vl, ixlenimm:$sew),[]>, RISCVVPseudo, - RISCVZvlsseg<PseudoToIntrinsic<NAME, true>.Intrinsic, EEW, VLMul> { + RISCVVLSEG<NF, /*Masked*/1, /*Strided*/1, /*FF*/0, log2<EEW>.val, VLMul> { let mayLoad = 1; let mayStore = 0; let hasSideEffects = 0; - let usesCustomInserter = 1; let Constraints = "$rd = $merge"; - let Uses = [VL, VTYPE]; let HasVLOp = 1; let HasSEWOp = 1; let HasMergeOp = 1; let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst); } -class VPseudoISegLoadNoMask<VReg RetClass, VReg IdxClass, bits<11> EEW, bits<3> LMUL>: +class VPseudoISegLoadNoMask<VReg RetClass, VReg IdxClass, int EEW, bits<3> LMUL, + bits<4> NF, bit Ordered>: Pseudo<(outs RetClass:$rd), - (ins GPR:$rs1, IdxClass:$offset, GPR:$vl, ixlenimm:$sew),[]>, + (ins GPR:$rs1, IdxClass:$offset, AVL:$vl, ixlenimm:$sew),[]>, RISCVVPseudo, - RISCVZvlsseg<PseudoToIntrinsic<NAME, false>.Intrinsic, EEW, VLMul, LMUL> { + RISCVVLXSEG<NF, /*Masked*/0, Ordered, log2<EEW>.val, VLMul, LMUL> { let mayLoad = 1; let mayStore = 0; let hasSideEffects = 0; - let usesCustomInserter = 1; // For vector indexed segment loads, the destination vector register groups // cannot overlap the source vector register group let Constraints = "@earlyclobber $rd"; - let Uses = [VL, VTYPE]; let HasVLOp = 1; let HasSEWOp = 1; let HasDummyMask = 1; let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst); } -class VPseudoISegLoadMask<VReg RetClass, VReg IdxClass, bits<11> EEW, bits<3> LMUL>: +class VPseudoISegLoadMask<VReg RetClass, VReg IdxClass, int EEW, bits<3> LMUL, + bits<4> NF, bit Ordered>: Pseudo<(outs GetVRegNoV0<RetClass>.R:$rd), (ins GetVRegNoV0<RetClass>.R:$merge, GPR:$rs1, - IdxClass:$offset, VMaskOp:$vm, GPR:$vl, ixlenimm:$sew),[]>, + IdxClass:$offset, VMaskOp:$vm, AVL:$vl, ixlenimm:$sew),[]>, RISCVVPseudo, - RISCVZvlsseg<PseudoToIntrinsic<NAME, true>.Intrinsic, EEW, VLMul, LMUL> { + RISCVVLXSEG<NF, /*Masked*/1, Ordered, log2<EEW>.val, VLMul, LMUL> { let mayLoad = 1; let mayStore = 0; let hasSideEffects = 0; - let usesCustomInserter = 1; // For vector indexed segment loads, the destination vector register groups // cannot overlap the source vector register group let Constraints = "@earlyclobber $rd, $rd = $merge"; - let Uses = [VL, VTYPE]; let HasVLOp = 1; let HasSEWOp = 1; let HasMergeOp = 1; let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst); } -class VPseudoUSSegStoreNoMask<VReg ValClass, bits<11> EEW>: +class VPseudoUSSegStoreNoMask<VReg ValClass, int EEW, bits<4> NF>: Pseudo<(outs), - (ins ValClass:$rd, GPR:$rs1, GPR:$vl, ixlenimm:$sew),[]>, + (ins ValClass:$rd, GPR:$rs1, AVL:$vl, ixlenimm:$sew),[]>, RISCVVPseudo, - RISCVZvlsseg<PseudoToIntrinsic<NAME, false>.Intrinsic, EEW, VLMul> { + RISCVVSSEG<NF, /*Masked*/0, /*Strided*/0, log2<EEW>.val, VLMul> { let mayLoad = 0; let mayStore = 1; let hasSideEffects = 0; - let usesCustomInserter = 1; - let Uses = [VL, VTYPE]; let HasVLOp = 1; let HasSEWOp = 1; let HasDummyMask = 1; let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst); } -class VPseudoUSSegStoreMask<VReg ValClass, bits<11> EEW>: +class VPseudoUSSegStoreMask<VReg ValClass, int EEW, bits<4> NF>: Pseudo<(outs), (ins ValClass:$rd, GPR:$rs1, - VMaskOp:$vm, GPR:$vl, ixlenimm:$sew),[]>, + VMaskOp:$vm, AVL:$vl, ixlenimm:$sew),[]>, RISCVVPseudo, - RISCVZvlsseg<PseudoToIntrinsic<NAME, true>.Intrinsic, EEW, VLMul> { + RISCVVSSEG<NF, /*Masked*/1, /*Strided*/0, log2<EEW>.val, VLMul> { let mayLoad = 0; let mayStore = 1; let hasSideEffects = 0; - let usesCustomInserter = 1; - let Uses = [VL, VTYPE]; let HasVLOp = 1; let HasSEWOp = 1; let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst); } -class VPseudoSSegStoreNoMask<VReg ValClass, bits<11> EEW>: +class VPseudoSSegStoreNoMask<VReg ValClass, int EEW, bits<4> NF>: Pseudo<(outs), - (ins ValClass:$rd, GPR:$rs1, GPR: $offset, GPR:$vl, ixlenimm:$sew),[]>, + (ins ValClass:$rd, GPR:$rs1, GPR: $offset, AVL:$vl, ixlenimm:$sew),[]>, RISCVVPseudo, - RISCVZvlsseg<PseudoToIntrinsic<NAME, false>.Intrinsic, EEW, VLMul> { + RISCVVSSEG<NF, /*Masked*/0, /*Strided*/1, log2<EEW>.val, VLMul> { let mayLoad = 0; let mayStore = 1; let hasSideEffects = 0; - let usesCustomInserter = 1; - let Uses = [VL, VTYPE]; let HasVLOp = 1; let HasSEWOp = 1; let HasDummyMask = 1; let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst); } -class VPseudoSSegStoreMask<VReg ValClass, bits<11> EEW>: +class VPseudoSSegStoreMask<VReg ValClass, int EEW, bits<4> NF>: Pseudo<(outs), (ins ValClass:$rd, GPR:$rs1, GPR: $offset, - VMaskOp:$vm, GPR:$vl, ixlenimm:$sew),[]>, + VMaskOp:$vm, AVL:$vl, ixlenimm:$sew),[]>, RISCVVPseudo, - RISCVZvlsseg<PseudoToIntrinsic<NAME, true>.Intrinsic, EEW, VLMul> { + RISCVVSSEG<NF, /*Masked*/1, /*Strided*/1, log2<EEW>.val, VLMul> { let mayLoad = 0; let mayStore = 1; let hasSideEffects = 0; - let usesCustomInserter = 1; - let Uses = [VL, VTYPE]; let HasVLOp = 1; let HasSEWOp = 1; let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst); } -class VPseudoISegStoreNoMask<VReg ValClass, VReg IdxClass, bits<11> EEW, bits<3> LMUL>: +class VPseudoISegStoreNoMask<VReg ValClass, VReg IdxClass, int EEW, bits<3> LMUL, + bits<4> NF, bit Ordered>: Pseudo<(outs), (ins ValClass:$rd, GPR:$rs1, IdxClass: $index, - GPR:$vl, ixlenimm:$sew),[]>, + AVL:$vl, ixlenimm:$sew),[]>, RISCVVPseudo, - RISCVZvlsseg<PseudoToIntrinsic<NAME, false>.Intrinsic, EEW, VLMul, LMUL> { + RISCVVSXSEG<NF, /*Masked*/0, Ordered, log2<EEW>.val, VLMul, LMUL> { let mayLoad = 0; let mayStore = 1; let hasSideEffects = 0; - let usesCustomInserter = 1; - let Uses = [VL, VTYPE]; let HasVLOp = 1; let HasSEWOp = 1; let HasDummyMask = 1; let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst); } -class VPseudoISegStoreMask<VReg ValClass, VReg IdxClass, bits<11> EEW, bits<3> LMUL>: +class VPseudoISegStoreMask<VReg ValClass, VReg IdxClass, int EEW, bits<3> LMUL, + bits<4> NF, bit Ordered>: Pseudo<(outs), (ins ValClass:$rd, GPR:$rs1, IdxClass: $index, - VMaskOp:$vm, GPR:$vl, ixlenimm:$sew),[]>, + VMaskOp:$vm, AVL:$vl, ixlenimm:$sew),[]>, RISCVVPseudo, - RISCVZvlsseg<PseudoToIntrinsic<NAME, true>.Intrinsic, EEW, VLMul, LMUL> { + RISCVVSXSEG<NF, /*Masked*/1, Ordered, log2<EEW>.val, VLMul, LMUL> { let mayLoad = 0; let mayStore = 1; let hasSideEffects = 0; - let usesCustomInserter = 1; - let Uses = [VL, VTYPE]; let HasVLOp = 1; let HasSEWOp = 1; let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst); } -multiclass VPseudoUSLoad { - foreach lmul = MxList.m in { - defvar LInfo = lmul.MX; - defvar vreg = lmul.vrclass; - let VLMul = lmul.value in { - def "_V_" # LInfo : VPseudoUSLoadNoMask<vreg>; - def "_V_" # LInfo # "_MASK" : VPseudoUSLoadMask<vreg>; +multiclass VPseudoUSLoad<bit isFF> { + foreach eew = EEWList in { + foreach lmul = MxSet<eew>.m in { + defvar LInfo = lmul.MX; + defvar vreg = lmul.vrclass; + defvar FFStr = !if(isFF, "FF", ""); + let VLMul = lmul.value in { + def "E" # eew # FFStr # "_V_" # LInfo : + VPseudoUSLoadNoMask<vreg, eew, isFF>; + def "E" # eew # FFStr # "_V_" # LInfo # "_MASK" : + VPseudoUSLoadMask<vreg, eew, isFF>; + } + } + } +} + +multiclass VPseudoLoadMask { + foreach mti = AllMasks in { + let VLMul = mti.LMul.value in { + def "_V_" # mti.BX : VPseudoUSLoadNoMask<VR, /*EEW*/1, /*isFF*/0>; } } } multiclass VPseudoSLoad { - foreach lmul = MxList.m in { - defvar LInfo = lmul.MX; - defvar vreg = lmul.vrclass; - let VLMul = lmul.value in { - def "_V_" # LInfo : VPseudoSLoadNoMask<vreg>; - def "_V_" # LInfo # "_MASK" : VPseudoSLoadMask<vreg>; + foreach eew = EEWList in { + foreach lmul = MxSet<eew>.m in { + defvar LInfo = lmul.MX; + defvar vreg = lmul.vrclass; + let VLMul = lmul.value in { + def "E" # eew # "_V_" # LInfo : VPseudoSLoadNoMask<vreg, eew>; + def "E" # eew # "_V_" # LInfo # "_MASK" : VPseudoSLoadMask<vreg, eew>; + } } } } -multiclass VPseudoILoad { - foreach lmul = MxList.m in - foreach idx_lmul = MxList.m in { - defvar LInfo = lmul.MX; - defvar Vreg = lmul.vrclass; - defvar IdxLInfo = idx_lmul.MX; - defvar IdxVreg = idx_lmul.vrclass; - let VLMul = lmul.value in { - def "_V_" # IdxLInfo # "_" # LInfo : VPseudoILoadNoMask<Vreg, IdxVreg>; - def "_V_" # IdxLInfo # "_" # LInfo # "_MASK" : VPseudoILoadMask<Vreg, IdxVreg>; +multiclass VPseudoILoad<bit Ordered> { + foreach eew = EEWList in { + foreach sew = EEWList in { + foreach lmul = MxSet<sew>.m in { + defvar octuple_lmul = lmul.octuple; + // Calculate emul = eew * lmul / sew + defvar octuple_emul = !srl(!mul(eew, octuple_lmul), log2<sew>.val); + if !and(!ge(octuple_emul, 1), !le(octuple_emul, 64)) then { + defvar LInfo = lmul.MX; + defvar IdxLInfo = octuple_to_str<octuple_emul>.ret; + defvar idx_lmul = !cast<LMULInfo>("V_" # IdxLInfo); + defvar Vreg = lmul.vrclass; + defvar IdxVreg = idx_lmul.vrclass; + defvar HasConstraint = !ne(sew, eew); + let VLMul = lmul.value in { + def "EI" # eew # "_V_" # IdxLInfo # "_" # LInfo : + VPseudoILoadNoMask<Vreg, IdxVreg, eew, idx_lmul.value, Ordered, HasConstraint>; + def "EI" # eew # "_V_" # IdxLInfo # "_" # LInfo # "_MASK" : + VPseudoILoadMask<Vreg, IdxVreg, eew, idx_lmul.value, Ordered, HasConstraint>; + } + } + } } } } multiclass VPseudoUSStore { - foreach lmul = MxList.m in { - defvar LInfo = lmul.MX; - defvar vreg = lmul.vrclass; - let VLMul = lmul.value in { - def "_V_" # LInfo : VPseudoUSStoreNoMask<vreg>; - def "_V_" # LInfo # "_MASK" : VPseudoUSStoreMask<vreg>; + foreach eew = EEWList in { + foreach lmul = MxSet<eew>.m in { + defvar LInfo = lmul.MX; + defvar vreg = lmul.vrclass; + let VLMul = lmul.value in { + def "E" # eew # "_V_" # LInfo : VPseudoUSStoreNoMask<vreg, eew>; + def "E" # eew # "_V_" # LInfo # "_MASK" : VPseudoUSStoreMask<vreg, eew>; + } + } + } +} + +multiclass VPseudoStoreMask { + foreach mti = AllMasks in { + let VLMul = mti.LMul.value in { + def "_V_" # mti.BX : VPseudoUSStoreNoMask<VR, /*EEW*/1>; } } } multiclass VPseudoSStore { - foreach lmul = MxList.m in { - defvar LInfo = lmul.MX; - defvar vreg = lmul.vrclass; - let VLMul = lmul.value in { - def "_V_" # LInfo : VPseudoSStoreNoMask<vreg>; - def "_V_" # LInfo # "_MASK" : VPseudoSStoreMask<vreg>; + foreach eew = EEWList in { + foreach lmul = MxSet<eew>.m in { + defvar LInfo = lmul.MX; + defvar vreg = lmul.vrclass; + let VLMul = lmul.value in { + def "E" # eew # "_V_" # LInfo : VPseudoSStoreNoMask<vreg, eew>; + def "E" # eew # "_V_" # LInfo # "_MASK" : VPseudoSStoreMask<vreg, eew>; + } } } } -multiclass VPseudoIStore { - foreach lmul = MxList.m in - foreach idx_lmul = MxList.m in { - defvar LInfo = lmul.MX; - defvar Vreg = lmul.vrclass; - defvar IdxLInfo = idx_lmul.MX; - defvar IdxVreg = idx_lmul.vrclass; - let VLMul = lmul.value in { - def "_V_" # IdxLInfo # "_" # LInfo : VPseudoIStoreNoMask<Vreg, IdxVreg>; - def "_V_" # IdxLInfo # "_" # LInfo # "_MASK" : VPseudoIStoreMask<Vreg, IdxVreg>; +multiclass VPseudoIStore<bit Ordered> { + foreach eew = EEWList in { + foreach sew = EEWList in { + foreach lmul = MxSet<sew>.m in { + defvar octuple_lmul = lmul.octuple; + // Calculate emul = eew * lmul / sew + defvar octuple_emul = !srl(!mul(eew, octuple_lmul), log2<sew>.val); + if !and(!ge(octuple_emul, 1), !le(octuple_emul, 64)) then { + defvar LInfo = lmul.MX; + defvar IdxLInfo = octuple_to_str<octuple_emul>.ret; + defvar idx_lmul = !cast<LMULInfo>("V_" # IdxLInfo); + defvar Vreg = lmul.vrclass; + defvar IdxVreg = idx_lmul.vrclass; + let VLMul = lmul.value in { + def "EI" # eew # "_V_" # IdxLInfo # "_" # LInfo : + VPseudoIStoreNoMask<Vreg, IdxVreg, eew, idx_lmul.value, Ordered>; + def "EI" # eew # "_V_" # IdxLInfo # "_" # LInfo # "_MASK" : + VPseudoIStoreMask<Vreg, IdxVreg, eew, idx_lmul.value, Ordered>; + } + } + } } } } @@ -1357,6 +1497,20 @@ multiclass VPseudoBinary<VReg RetClass, } } +multiclass VPseudoBinaryM<VReg RetClass, + VReg Op1Class, + DAGOperand Op2Class, + LMULInfo MInfo, + string Constraint = ""> { + let VLMul = MInfo.value in { + def "_" # MInfo.MX : VPseudoBinaryNoMask<RetClass, Op1Class, Op2Class, + Constraint>; + let ForceTailAgnostic = true in + def "_" # MInfo.MX # "_MASK" : VPseudoBinaryMOutMask<RetClass, Op1Class, + Op2Class, Constraint>; + } +} + multiclass VPseudoBinaryEmul<VReg RetClass, VReg Op1Class, DAGOperand Op2Class, @@ -1371,6 +1525,18 @@ multiclass VPseudoBinaryEmul<VReg RetClass, } } +multiclass VPseudoTiedBinary<VReg RetClass, + DAGOperand Op2Class, + LMULInfo MInfo, + string Constraint = ""> { + let VLMul = MInfo.value in { + def "_" # MInfo.MX # "_TIED": VPseudoTiedBinaryNoMask<RetClass, Op2Class, + Constraint>; + def "_" # MInfo.MX # "_MASK_TIED" : VPseudoTiedBinaryMask<RetClass, Op2Class, + Constraint>; + } +} + multiclass VPseudoBinaryV_VV<string Constraint = ""> { foreach m = MxList.m in defm _VV : VPseudoBinary<m.vrclass, m.vrclass, m.vrclass, m, Constraint>; @@ -1379,9 +1545,9 @@ multiclass VPseudoBinaryV_VV<string Constraint = ""> { multiclass VPseudoBinaryV_VV_EEW<int eew, string Constraint = ""> { foreach m = MxList.m in { foreach sew = EEWList in { - defvar octuple_lmul = octuple_from_str<m.MX>.ret; + defvar octuple_lmul = m.octuple; // emul = lmul * eew / sew - defvar octuple_emul = !srl(!mul(octuple_lmul, eew), shift_amount<sew>.val); + defvar octuple_emul = !srl(!mul(octuple_lmul, eew), log2<sew>.val); if !and(!ge(octuple_emul, 1), !le(octuple_emul, 64)) then { defvar emulMX = octuple_to_str<octuple_emul>.ret; defvar emul = !cast<LMULInfo>("V_" # emulMX); @@ -1423,61 +1589,67 @@ multiclass VPseudoBinaryM_MM { // at least 1, and the overlap is in the highest-numbered part of the // destination register group is legal. Otherwise, it is illegal. multiclass VPseudoBinaryW_VV { - foreach m = MxList.m[0-5] in + foreach m = MxListW.m in defm _VV : VPseudoBinary<m.wvrclass, m.vrclass, m.vrclass, m, "@earlyclobber $rd">; } multiclass VPseudoBinaryW_VX { - foreach m = MxList.m[0-5] in + foreach m = MxListW.m in defm "_VX" : VPseudoBinary<m.wvrclass, m.vrclass, GPR, m, "@earlyclobber $rd">; } multiclass VPseudoBinaryW_VF { - foreach m = MxList.m[0-5] in - foreach f = FPList.fpinfo[0-1] in + foreach m = MxListW.m in + foreach f = FPListW.fpinfo in defm "_V" # f.FX : VPseudoBinary<m.wvrclass, m.vrclass, f.fprclass, m, "@earlyclobber $rd">; } multiclass VPseudoBinaryW_WV { - foreach m = MxList.m[0-5] in + foreach m = MxListW.m in { defm _WV : VPseudoBinary<m.wvrclass, m.wvrclass, m.vrclass, m, "@earlyclobber $rd">; + defm _WV : VPseudoTiedBinary<m.wvrclass, m.vrclass, m, + "@earlyclobber $rd">; + } } multiclass VPseudoBinaryW_WX { - foreach m = MxList.m[0-5] in - defm "_WX" : VPseudoBinary<m.wvrclass, m.wvrclass, GPR, m, - "@earlyclobber $rd">; + foreach m = MxListW.m in + defm "_WX" : VPseudoBinary<m.wvrclass, m.wvrclass, GPR, m>; } multiclass VPseudoBinaryW_WF { - foreach m = MxList.m[0-5] in - foreach f = FPList.fpinfo[0-1] in + foreach m = MxListW.m in + foreach f = FPListW.fpinfo in defm "_W" # f.FX : VPseudoBinary<m.wvrclass, m.wvrclass, - f.fprclass, m, - "@earlyclobber $rd">; + f.fprclass, m>; } +// Narrowing instructions like vnsrl/vnsra/vnclip(u) don't need @earlyclobber +// if the source and destination have an LMUL<=1. This matches this overlap +// exception from the spec. +// "The destination EEW is smaller than the source EEW and the overlap is in the +// lowest-numbered part of the source register group." multiclass VPseudoBinaryV_WV { - foreach m = MxList.m[0-5] in + foreach m = MxListW.m in defm _WV : VPseudoBinary<m.vrclass, m.wvrclass, m.vrclass, m, - "@earlyclobber $rd">; + !if(!ge(m.octuple, 8), "@earlyclobber $rd", "")>; } multiclass VPseudoBinaryV_WX { - foreach m = MxList.m[0-5] in + foreach m = MxListW.m in defm _WX : VPseudoBinary<m.vrclass, m.wvrclass, GPR, m, - "@earlyclobber $rd">; + !if(!ge(m.octuple, 8), "@earlyclobber $rd", "")>; } multiclass VPseudoBinaryV_WI { - foreach m = MxList.m[0-5] in + foreach m = MxListW.m in defm _WI : VPseudoBinary<m.vrclass, m.wvrclass, uimm5, m, - "@earlyclobber $rd">; + !if(!ge(m.octuple, 8), "@earlyclobber $rd", "")>; } // For vadc and vsbc, the instruction encoding is reserved if the destination @@ -1552,7 +1724,7 @@ multiclass VPseudoUnaryV_V { multiclass PseudoUnaryV_VF2 { defvar constraints = "@earlyclobber $rd"; - foreach m = MxList.m[1-6] in + foreach m = MxListVF2.m in { let VLMul = m.value in { def "_" # m.MX : VPseudoUnaryNoMask<m.vrclass, m.f2vrclass, constraints>; @@ -1564,7 +1736,7 @@ multiclass PseudoUnaryV_VF2 { multiclass PseudoUnaryV_VF4 { defvar constraints = "@earlyclobber $rd"; - foreach m = MxList.m[2-6] in + foreach m = MxListVF4.m in { let VLMul = m.value in { def "_" # m.MX : VPseudoUnaryNoMask<m.vrclass, m.f4vrclass, constraints>; @@ -1576,7 +1748,7 @@ multiclass PseudoUnaryV_VF4 { multiclass PseudoUnaryV_VF8 { defvar constraints = "@earlyclobber $rd"; - foreach m = MxList.m[3-6] in + foreach m = MxListVF8.m in { let VLMul = m.value in { def "_" # m.MX : VPseudoUnaryNoMask<m.vrclass, m.f8vrclass, constraints>; @@ -1586,31 +1758,42 @@ multiclass PseudoUnaryV_VF8 { } } -// The destination EEW is 1. +// The destination EEW is 1 since "For the purposes of register group overlap +// constraints, mask elements have EEW=1." // The source EEW is 8, 16, 32, or 64. // When the destination EEW is different from source EEW, we need to use // @earlyclobber to avoid the overlap between destination and source registers. +// We don't need @earlyclobber for LMUL<=1 since that matches this overlap +// exception from the spec +// "The destination EEW is smaller than the source EEW and the overlap is in the +// lowest-numbered part of the source register group". +// With LMUL<=1 the source and dest occupy a single register so any overlap +// is in the lowest-numbered part. multiclass VPseudoBinaryM_VV { foreach m = MxList.m in - defm _VV : VPseudoBinary<VR, m.vrclass, m.vrclass, m, "@earlyclobber $rd">; + defm _VV : VPseudoBinaryM<VR, m.vrclass, m.vrclass, m, + !if(!ge(m.octuple, 16), "@earlyclobber $rd", "")>; } multiclass VPseudoBinaryM_VX { foreach m = MxList.m in defm "_VX" : - VPseudoBinary<VR, m.vrclass, GPR, m, "@earlyclobber $rd">; + VPseudoBinaryM<VR, m.vrclass, GPR, m, + !if(!ge(m.octuple, 16), "@earlyclobber $rd", "")>; } multiclass VPseudoBinaryM_VF { foreach m = MxList.m in foreach f = FPList.fpinfo in defm "_V" # f.FX : - VPseudoBinary<VR, m.vrclass, f.fprclass, m, "@earlyclobber $rd">; + VPseudoBinaryM<VR, m.vrclass, f.fprclass, m, + !if(!ge(m.octuple, 16), "@earlyclobber $rd", "")>; } multiclass VPseudoBinaryM_VI { foreach m = MxList.m in - defm _VI : VPseudoBinary<VR, m.vrclass, simm5, m, "@earlyclobber $rd">; + defm _VI : VPseudoBinaryM<VR, m.vrclass, simm5, m, + !if(!ge(m.octuple, 16), "@earlyclobber $rd", "")>; } multiclass VPseudoBinaryV_VV_VX_VI<Operand ImmType = simm5, string Constraint = ""> { @@ -1694,8 +1877,8 @@ multiclass VPseudoBinaryV_WV_WX_WI { } multiclass VPseudoTernary<VReg RetClass, - VReg Op1Class, - RegisterClass Op2Class, + RegisterClass Op1Class, + DAGOperand Op2Class, LMULInfo MInfo, string Constraint = ""> { let VLMul = MInfo.value in { @@ -1705,8 +1888,16 @@ multiclass VPseudoTernary<VReg RetClass, } multiclass VPseudoTernaryV_VV<string Constraint = ""> { - foreach m = MxList.m in + foreach m = MxList.m in { defm _VV : VPseudoTernary<m.vrclass, m.vrclass, m.vrclass, m, Constraint>; + + // Add a commutable version for use by IR mul+add. + let isCommutable = 1, ForceTailAgnostic = true, VLMul = m.value in + def "_VV_" # m.MX # "_COMMUTABLE" : VPseudoTernaryNoMask<m.vrclass, + m.vrclass, + m.vrclass, + Constraint>; + } } multiclass VPseudoTernaryV_VX<string Constraint = ""> { @@ -1715,35 +1906,68 @@ multiclass VPseudoTernaryV_VX<string Constraint = ""> { } multiclass VPseudoTernaryV_VX_AAXA<string Constraint = ""> { - foreach m = MxList.m in + foreach m = MxList.m in { defm "_VX" : VPseudoTernary<m.vrclass, GPR, m.vrclass, m, Constraint>; + + // Add a commutable version for use by IR mul+add. + let isCommutable = 1, ForceTailAgnostic = true, VLMul = m.value in + def "_VX_" # m.MX # "_COMMUTABLE" : + VPseudoTernaryNoMask<m.vrclass, GPR, m.vrclass, Constraint>; + } } multiclass VPseudoTernaryV_VF_AAXA<string Constraint = ""> { - foreach m = MxList.m in - foreach f = FPList.fpinfo in + foreach m = MxList.m in { + foreach f = FPList.fpinfo in { defm "_V" # f.FX : VPseudoTernary<m.vrclass, f.fprclass, m.vrclass, m, Constraint>; + + // Add a commutable version for use by IR mul+add. + let isCommutable = 1, ForceTailAgnostic = true, VLMul = m.value in + def "_V" # f.FX # "_" # m.MX # "_COMMUTABLE" : + VPseudoTernaryNoMask<m.vrclass, f.fprclass, m.vrclass, Constraint>; + } + } } multiclass VPseudoTernaryW_VV { defvar constraint = "@earlyclobber $rd"; - foreach m = MxList.m[0-5] in + foreach m = MxListW.m in { defm _VV : VPseudoTernary<m.wvrclass, m.vrclass, m.vrclass, m, constraint>; + + // Add a tail agnostic version for us by IR mul+add. + let ForceTailAgnostic = true, VLMul = m.value in + def "_VV_" # m.MX # "_TA" : VPseudoTernaryNoMask<m.wvrclass, + m.vrclass, + m.vrclass, + constraint>; + } } multiclass VPseudoTernaryW_VX { defvar constraint = "@earlyclobber $rd"; - foreach m = MxList.m[0-5] in + foreach m = MxListW.m in { defm "_VX" : VPseudoTernary<m.wvrclass, GPR, m.vrclass, m, constraint>; + + // Add a tail agnostic version for use by IR mul+add. + let ForceTailAgnostic = true, VLMul = m.value in + def "_VX_" # m.MX # "_TA" : + VPseudoTernaryNoMask<m.wvrclass, GPR, m.vrclass, constraint>; + } } multiclass VPseudoTernaryW_VF { defvar constraint = "@earlyclobber $rd"; - foreach m = MxList.m[0-5] in - foreach f = FPList.fpinfo[0-1] in + foreach m = MxListW.m in + foreach f = FPListW.fpinfo in { defm "_V" # f.FX : VPseudoTernary<m.wvrclass, f.fprclass, m.vrclass, m, constraint>; + + // Add a tail agnostic version for use by IR mul+add. + let ForceTailAgnostic = true, VLMul = m.value in + def "_V" # f.FX # "_" # m.MX # "_TA" : + VPseudoTernaryNoMask<m.vrclass, f.fprclass, m.vrclass, constraint>; + } } multiclass VPseudoTernaryV_VI<Operand ImmType = simm5, string Constraint = ""> { @@ -1799,7 +2023,6 @@ multiclass VPseudoBinaryM_VX_VI { multiclass VPseudoReductionV_VS { foreach m = MxList.m in { - let WritesElement0 = 1 in defm _VS : VPseudoTernary<V_M1.vrclass, m.vrclass, V_M1.vrclass, m>; } } @@ -1822,13 +2045,13 @@ multiclass VPseudoConversionV_V { multiclass VPseudoConversionW_V { defvar constraint = "@earlyclobber $rd"; - foreach m = MxList.m[0-5] in + foreach m = MxListW.m in defm _V : VPseudoConversion<m.wvrclass, m.vrclass, m, constraint>; } multiclass VPseudoConversionV_W { defvar constraint = "@earlyclobber $rd"; - foreach m = MxList.m[0-5] in + foreach m = MxListW.m in defm _W : VPseudoConversion<m.vrclass, m.wvrclass, m, constraint>; } @@ -1841,9 +2064,9 @@ multiclass VPseudoUSSegLoad<bit isFF> { defvar vreg = SegRegClass<lmul, nf>.RC; defvar FFStr = !if(isFF, "FF", ""); def nf # "E" # eew # FFStr # "_V_" # LInfo : - VPseudoUSSegLoadNoMask<vreg, eew>; + VPseudoUSSegLoadNoMask<vreg, eew, nf, isFF>; def nf # "E" # eew # FFStr # "_V_" # LInfo # "_MASK" : - VPseudoUSSegLoadMask<vreg, eew>; + VPseudoUSSegLoadMask<vreg, eew, nf, isFF>; } } } @@ -1857,28 +2080,37 @@ multiclass VPseudoSSegLoad { let VLMul = lmul.value in { foreach nf = NFSet<lmul>.L in { defvar vreg = SegRegClass<lmul, nf>.RC; - def nf # "E" # eew # "_V_" # LInfo : VPseudoSSegLoadNoMask<vreg, eew>; - def nf # "E" # eew # "_V_" # LInfo # "_MASK" : VPseudoSSegLoadMask<vreg, eew>; + def nf # "E" # eew # "_V_" # LInfo : VPseudoSSegLoadNoMask<vreg, eew, nf>; + def nf # "E" # eew # "_V_" # LInfo # "_MASK" : VPseudoSSegLoadMask<vreg, eew, nf>; } } } } } -multiclass VPseudoISegLoad { - foreach idx_eew = EEWList in { // EEW for index argument. - foreach idx_lmul = MxSet<idx_eew>.m in { // LMUL for index argument. - foreach val_lmul = MxList.m in { // LMUL for the value. - defvar IdxLInfo = idx_lmul.MX; - defvar IdxVreg = idx_lmul.vrclass; - defvar ValLInfo = val_lmul.MX; - let VLMul = val_lmul.value in { - foreach nf = NFSet<val_lmul>.L in { - defvar ValVreg = SegRegClass<val_lmul, nf>.RC; - def nf # "EI" # idx_eew # "_V_" # IdxLInfo # "_" # ValLInfo : - VPseudoISegLoadNoMask<ValVreg, IdxVreg, idx_eew, idx_lmul.value>; - def nf # "EI" # idx_eew # "_V_" # IdxLInfo # "_" # ValLInfo # "_MASK" : - VPseudoISegLoadMask<ValVreg, IdxVreg, idx_eew, idx_lmul.value>; +multiclass VPseudoISegLoad<bit Ordered> { + foreach idx_eew = EEWList in { + foreach sew = EEWList in { + foreach val_lmul = MxSet<sew>.m in { + defvar octuple_lmul = val_lmul.octuple; + // Calculate emul = eew * lmul / sew + defvar octuple_emul = !srl(!mul(idx_eew, octuple_lmul), log2<sew>.val); + if !and(!ge(octuple_emul, 1), !le(octuple_emul, 64)) then { + defvar ValLInfo = val_lmul.MX; + defvar IdxLInfo = octuple_to_str<octuple_emul>.ret; + defvar idx_lmul = !cast<LMULInfo>("V_" # IdxLInfo); + defvar Vreg = val_lmul.vrclass; + defvar IdxVreg = idx_lmul.vrclass; + let VLMul = val_lmul.value in { + foreach nf = NFSet<val_lmul>.L in { + defvar ValVreg = SegRegClass<val_lmul, nf>.RC; + def nf # "EI" # idx_eew # "_V_" # IdxLInfo # "_" # ValLInfo : + VPseudoISegLoadNoMask<ValVreg, IdxVreg, idx_eew, idx_lmul.value, + nf, Ordered>; + def nf # "EI" # idx_eew # "_V_" # IdxLInfo # "_" # ValLInfo # "_MASK" : + VPseudoISegLoadMask<ValVreg, IdxVreg, idx_eew, idx_lmul.value, + nf, Ordered>; + } } } } @@ -1893,8 +2125,8 @@ multiclass VPseudoUSSegStore { let VLMul = lmul.value in { foreach nf = NFSet<lmul>.L in { defvar vreg = SegRegClass<lmul, nf>.RC; - def nf # "E" # eew # "_V_" # LInfo : VPseudoUSSegStoreNoMask<vreg, eew>; - def nf # "E" # eew # "_V_" # LInfo # "_MASK" : VPseudoUSSegStoreMask<vreg, eew>; + def nf # "E" # eew # "_V_" # LInfo : VPseudoUSSegStoreNoMask<vreg, eew, nf>; + def nf # "E" # eew # "_V_" # LInfo # "_MASK" : VPseudoUSSegStoreMask<vreg, eew, nf>; } } } @@ -1908,28 +2140,37 @@ multiclass VPseudoSSegStore { let VLMul = lmul.value in { foreach nf = NFSet<lmul>.L in { defvar vreg = SegRegClass<lmul, nf>.RC; - def nf # "E" # eew # "_V_" # LInfo : VPseudoSSegStoreNoMask<vreg, eew>; - def nf # "E" # eew # "_V_" # LInfo # "_MASK" : VPseudoSSegStoreMask<vreg, eew>; + def nf # "E" # eew # "_V_" # LInfo : VPseudoSSegStoreNoMask<vreg, eew, nf>; + def nf # "E" # eew # "_V_" # LInfo # "_MASK" : VPseudoSSegStoreMask<vreg, eew, nf>; } } } } } -multiclass VPseudoISegStore { - foreach idx_eew = EEWList in { // EEW for index argument. - foreach idx_lmul = MxSet<idx_eew>.m in { // LMUL for index argument. - foreach val_lmul = MxList.m in { // LMUL for the value. - defvar IdxLInfo = idx_lmul.MX; - defvar IdxVreg = idx_lmul.vrclass; - defvar ValLInfo = val_lmul.MX; - let VLMul = val_lmul.value in { - foreach nf = NFSet<val_lmul>.L in { - defvar ValVreg = SegRegClass<val_lmul, nf>.RC; - def nf # "EI" # idx_eew # "_V_" # IdxLInfo # "_" # ValLInfo : - VPseudoISegStoreNoMask<ValVreg, IdxVreg, idx_eew, idx_lmul.value>; - def nf # "EI" # idx_eew # "_V_" # IdxLInfo # "_" # ValLInfo # "_MASK" : - VPseudoISegStoreMask<ValVreg, IdxVreg, idx_eew, idx_lmul.value>; +multiclass VPseudoISegStore<bit Ordered> { + foreach idx_eew = EEWList in { + foreach sew = EEWList in { + foreach val_lmul = MxSet<sew>.m in { + defvar octuple_lmul = val_lmul.octuple; + // Calculate emul = eew * lmul / sew + defvar octuple_emul = !srl(!mul(idx_eew, octuple_lmul), log2<sew>.val); + if !and(!ge(octuple_emul, 1), !le(octuple_emul, 64)) then { + defvar ValLInfo = val_lmul.MX; + defvar IdxLInfo = octuple_to_str<octuple_emul>.ret; + defvar idx_lmul = !cast<LMULInfo>("V_" # IdxLInfo); + defvar Vreg = val_lmul.vrclass; + defvar IdxVreg = idx_lmul.vrclass; + let VLMul = val_lmul.value in { + foreach nf = NFSet<val_lmul>.L in { + defvar ValVreg = SegRegClass<val_lmul, nf>.RC; + def nf # "EI" # idx_eew # "_V_" # IdxLInfo # "_" # ValLInfo : + VPseudoISegStoreNoMask<ValVreg, IdxVreg, idx_eew, idx_lmul.value, + nf, Ordered>; + def nf # "EI" # idx_eew # "_V_" # IdxLInfo # "_" # ValLInfo # "_MASK" : + VPseudoISegStoreMask<ValVreg, IdxVreg, idx_eew, idx_lmul.value, + nf, Ordered>; + } } } } @@ -1951,10 +2192,10 @@ class VPatUnaryNoMask<string intrinsic_name, VReg op2_reg_class> : Pat<(result_type (!cast<Intrinsic>(intrinsic_name) (op2_type op2_reg_class:$rs2), - (XLenVT GPR:$vl))), + VLOpFrag)), (!cast<Instruction>(inst#"_"#kind#"_"#vlmul.MX) (op2_type op2_reg_class:$rs2), - (NoX0 GPR:$vl), sew)>; + GPR:$vl, sew)>; class VPatUnaryMask<string intrinsic_name, string inst, @@ -1970,21 +2211,21 @@ class VPatUnaryMask<string intrinsic_name, (result_type result_reg_class:$merge), (op2_type op2_reg_class:$rs2), (mask_type V0), - (XLenVT GPR:$vl))), + VLOpFrag)), (!cast<Instruction>(inst#"_"#kind#"_"#vlmul.MX#"_MASK") (result_type result_reg_class:$merge), (op2_type op2_reg_class:$rs2), - (mask_type V0), (NoX0 GPR:$vl), sew)>; + (mask_type V0), GPR:$vl, sew)>; class VPatMaskUnaryNoMask<string intrinsic_name, string inst, MTypeInfo mti> : Pat<(mti.Mask (!cast<Intrinsic>(intrinsic_name) (mti.Mask VR:$rs2), - (XLenVT GPR:$vl))), + VLOpFrag)), (!cast<Instruction>(inst#"_M_"#mti.BX) (mti.Mask VR:$rs2), - (NoX0 GPR:$vl), mti.SEW)>; + GPR:$vl, mti.Log2SEW)>; class VPatMaskUnaryMask<string intrinsic_name, string inst, @@ -1993,11 +2234,11 @@ class VPatMaskUnaryMask<string intrinsic_name, (mti.Mask VR:$merge), (mti.Mask VR:$rs2), (mti.Mask V0), - (XLenVT GPR:$vl))), + VLOpFrag)), (!cast<Instruction>(inst#"_M_"#mti.BX#"_MASK") (mti.Mask VR:$merge), (mti.Mask VR:$rs2), - (mti.Mask V0), (NoX0 GPR:$vl), mti.SEW)>; + (mti.Mask V0), GPR:$vl, mti.Log2SEW)>; class VPatUnaryAnyMask<string intrinsic, string inst, @@ -2013,12 +2254,12 @@ class VPatUnaryAnyMask<string intrinsic, (result_type result_reg_class:$merge), (op1_type op1_reg_class:$rs1), (mask_type VR:$rs2), - (XLenVT GPR:$vl))), + VLOpFrag)), (!cast<Instruction>(inst#"_"#kind#"_"#vlmul.MX) (result_type result_reg_class:$merge), (op1_type op1_reg_class:$rs1), (mask_type VR:$rs2), - (NoX0 GPR:$vl), sew)>; + GPR:$vl, sew)>; class VPatBinaryNoMask<string intrinsic_name, string inst, @@ -2031,11 +2272,29 @@ class VPatBinaryNoMask<string intrinsic_name, Pat<(result_type (!cast<Intrinsic>(intrinsic_name) (op1_type op1_reg_class:$rs1), (op2_type op2_kind:$rs2), - (XLenVT GPR:$vl))), + VLOpFrag)), + (!cast<Instruction>(inst) + (op1_type op1_reg_class:$rs1), + (op2_type op2_kind:$rs2), + GPR:$vl, sew)>; + +// Same as above but source operands are swapped. +class VPatBinaryNoMaskSwapped<string intrinsic_name, + string inst, + ValueType result_type, + ValueType op1_type, + ValueType op2_type, + int sew, + VReg op1_reg_class, + DAGOperand op2_kind> : + Pat<(result_type (!cast<Intrinsic>(intrinsic_name) + (op2_type op2_kind:$rs2), + (op1_type op1_reg_class:$rs1), + VLOpFrag)), (!cast<Instruction>(inst) (op1_type op1_reg_class:$rs1), (op2_type op2_kind:$rs2), - (NoX0 GPR:$vl), sew)>; + GPR:$vl, sew)>; class VPatBinaryMask<string intrinsic_name, string inst, @@ -2052,12 +2311,70 @@ class VPatBinaryMask<string intrinsic_name, (op1_type op1_reg_class:$rs1), (op2_type op2_kind:$rs2), (mask_type V0), - (XLenVT GPR:$vl))), + VLOpFrag)), + (!cast<Instruction>(inst#"_MASK") + (result_type result_reg_class:$merge), + (op1_type op1_reg_class:$rs1), + (op2_type op2_kind:$rs2), + (mask_type V0), GPR:$vl, sew)>; + +// Same as above but source operands are swapped. +class VPatBinaryMaskSwapped<string intrinsic_name, + string inst, + ValueType result_type, + ValueType op1_type, + ValueType op2_type, + ValueType mask_type, + int sew, + VReg result_reg_class, + VReg op1_reg_class, + DAGOperand op2_kind> : + Pat<(result_type (!cast<Intrinsic>(intrinsic_name#"_mask") + (result_type result_reg_class:$merge), + (op2_type op2_kind:$rs2), + (op1_type op1_reg_class:$rs1), + (mask_type V0), + VLOpFrag)), (!cast<Instruction>(inst#"_MASK") (result_type result_reg_class:$merge), (op1_type op1_reg_class:$rs1), (op2_type op2_kind:$rs2), - (mask_type V0), (NoX0 GPR:$vl), sew)>; + (mask_type V0), GPR:$vl, sew)>; + +class VPatTiedBinaryNoMask<string intrinsic_name, + string inst, + ValueType result_type, + ValueType op2_type, + int sew, + VReg result_reg_class, + DAGOperand op2_kind> : + Pat<(result_type (!cast<Intrinsic>(intrinsic_name) + (result_type result_reg_class:$rs1), + (op2_type op2_kind:$rs2), + VLOpFrag)), + (!cast<Instruction>(inst#"_TIED") + (result_type result_reg_class:$rs1), + (op2_type op2_kind:$rs2), + GPR:$vl, sew)>; + +class VPatTiedBinaryMask<string intrinsic_name, + string inst, + ValueType result_type, + ValueType op2_type, + ValueType mask_type, + int sew, + VReg result_reg_class, + DAGOperand op2_kind> : + Pat<(result_type (!cast<Intrinsic>(intrinsic_name#"_mask") + (result_type result_reg_class:$merge), + (result_type result_reg_class:$merge), + (op2_type op2_kind:$rs2), + (mask_type V0), + VLOpFrag)), + (!cast<Instruction>(inst#"_MASK_TIED") + (result_type result_reg_class:$merge), + (op2_type op2_kind:$rs2), + (mask_type V0), GPR:$vl, sew)>; class VPatTernaryNoMask<string intrinsic, string inst, @@ -2075,12 +2392,12 @@ class VPatTernaryNoMask<string intrinsic, (result_type result_reg_class:$rs3), (op1_type op1_reg_class:$rs1), (op2_type op2_kind:$rs2), - (XLenVT GPR:$vl))), + VLOpFrag)), (!cast<Instruction>(inst#"_"#kind#"_"#vlmul.MX) result_reg_class:$rs3, (op1_type op1_reg_class:$rs1), op2_kind:$rs2, - (NoX0 GPR:$vl), sew)>; + GPR:$vl, sew)>; class VPatTernaryMask<string intrinsic, string inst, @@ -2099,13 +2416,13 @@ class VPatTernaryMask<string intrinsic, (op1_type op1_reg_class:$rs1), (op2_type op2_kind:$rs2), (mask_type V0), - (XLenVT GPR:$vl))), + VLOpFrag)), (!cast<Instruction>(inst#"_"#kind#"_"#vlmul.MX # "_MASK") result_reg_class:$rs3, (op1_type op1_reg_class:$rs1), op2_kind:$rs2, (mask_type V0), - (NoX0 GPR:$vl), sew)>; + GPR:$vl, sew)>; class VPatAMOWDNoMask<string intrinsic_name, string inst, @@ -2119,10 +2436,10 @@ class VPatAMOWDNoMask<string intrinsic_name, GPR:$rs1, (op1_type op1_reg_class:$vs2), (result_type vlmul.vrclass:$vd), - (XLenVT GPR:$vl))), + VLOpFrag)), (!cast<Instruction>(inst # "_WD_" # vlmul.MX # "_" # emul.MX) $rs1, $vs2, $vd, - (NoX0 GPR:$vl), sew)>; + GPR:$vl, sew)>; class VPatAMOWDMask<string intrinsic_name, string inst, @@ -2138,164 +2455,23 @@ class VPatAMOWDMask<string intrinsic_name, (op1_type op1_reg_class:$vs2), (result_type vlmul.vrclass:$vd), (mask_type V0), - (XLenVT GPR:$vl))), + VLOpFrag)), (!cast<Instruction>(inst # "_WD_" # vlmul.MX # "_" # emul.MX # "_MASK") $rs1, $vs2, $vd, - (mask_type V0), (NoX0 GPR:$vl), sew)>; - -multiclass VPatUSLoad<string intrinsic, - string inst, - LLVMType type, - LLVMType mask_type, - int sew, - LMULInfo vlmul, - VReg reg_class> -{ - defvar Intr = !cast<Intrinsic>(intrinsic); - defvar Pseudo = !cast<Instruction>(inst#"_V_"#vlmul.MX); - def : Pat<(type (Intr GPR:$rs1, GPR:$vl)), - (Pseudo $rs1, (NoX0 GPR:$vl), sew)>; - defvar IntrMask = !cast<Intrinsic>(intrinsic # "_mask"); - defvar PseudoMask = !cast<Instruction>(inst#"_V_"#vlmul.MX#"_MASK"); - def : Pat<(type (IntrMask (type GetVRegNoV0<reg_class>.R:$merge), - GPR:$rs1, (mask_type V0), GPR:$vl)), - (PseudoMask $merge, - $rs1, (mask_type V0), (NoX0 GPR:$vl), sew)>; -} - -multiclass VPatUSLoadFF<string inst, - LLVMType type, - LLVMType mask_type, - int sew, - LMULInfo vlmul, - VReg reg_class> -{ - defvar Pseudo = !cast<Instruction>(inst#"_V_"#vlmul.MX); - def : Pat<(type (riscv_vleff GPR:$rs1, GPR:$vl)), - (Pseudo $rs1, (NoX0 GPR:$vl), sew)>; - defvar PseudoMask = !cast<Instruction>(inst#"_V_"#vlmul.MX#"_MASK"); - def : Pat<(type (riscv_vleff_mask (type GetVRegNoV0<reg_class>.R:$merge), - GPR:$rs1, (mask_type V0), GPR:$vl)), - (PseudoMask $merge, - $rs1, (mask_type V0), (NoX0 GPR:$vl), sew)>; -} - -multiclass VPatSLoad<string intrinsic, - string inst, - LLVMType type, - LLVMType mask_type, - int sew, - LMULInfo vlmul, - VReg reg_class> -{ - defvar Intr = !cast<Intrinsic>(intrinsic); - defvar Pseudo = !cast<Instruction>(inst#"_V_"#vlmul.MX); - def : Pat<(type (Intr GPR:$rs1, GPR:$rs2, GPR:$vl)), - (Pseudo $rs1, $rs2, (NoX0 GPR:$vl), sew)>; - defvar IntrMask = !cast<Intrinsic>(intrinsic # "_mask"); - defvar PseudoMask = !cast<Instruction>(inst#"_V_"#vlmul.MX#"_MASK"); - def : Pat<(type (IntrMask (type GetVRegNoV0<reg_class>.R:$merge), - GPR:$rs1, GPR:$rs2, (mask_type V0), GPR:$vl)), - (PseudoMask $merge, - $rs1, $rs2, (mask_type V0), (NoX0 GPR:$vl), sew)>; -} - -multiclass VPatILoad<string intrinsic, - string inst, - LLVMType type, - LLVMType idx_type, - LLVMType mask_type, - int sew, - LMULInfo vlmul, - LMULInfo idx_vlmul, - VReg reg_class, - VReg idx_reg_class> -{ - defvar Intr = !cast<Intrinsic>(intrinsic); - defvar Pseudo = !cast<Instruction>(inst#"_V_"#idx_vlmul.MX#"_"#vlmul.MX); - def : Pat<(type (Intr GPR:$rs1, (idx_type idx_reg_class:$rs2), GPR:$vl)), - (Pseudo $rs1, $rs2, (NoX0 GPR:$vl), sew)>; - - defvar IntrMask = !cast<Intrinsic>(intrinsic # "_mask"); - defvar PseudoMask = !cast<Instruction>(inst#"_V_"#idx_vlmul.MX#"_"#vlmul.MX#"_MASK"); - def : Pat<(type (IntrMask (type GetVRegNoV0<reg_class>.R:$merge), - GPR:$rs1, (idx_type idx_reg_class:$rs2), - (mask_type V0), GPR:$vl)), - (PseudoMask $merge, - $rs1, $rs2, (mask_type V0), (NoX0 GPR:$vl), sew)>; -} - -multiclass VPatUSStore<string intrinsic, - string inst, - LLVMType type, - LLVMType mask_type, - int sew, - LMULInfo vlmul, - VReg reg_class> -{ - defvar Intr = !cast<Intrinsic>(intrinsic); - defvar Pseudo = !cast<Instruction>(inst#"_V_"#vlmul.MX); - def : Pat<(Intr (type reg_class:$rs3), GPR:$rs1, GPR:$vl), - (Pseudo $rs3, $rs1, (NoX0 GPR:$vl), sew)>; - defvar IntrMask = !cast<Intrinsic>(intrinsic # "_mask"); - defvar PseudoMask = !cast<Instruction>(inst#"_V_"#vlmul.MX#"_MASK"); - def : Pat<(IntrMask (type reg_class:$rs3), GPR:$rs1, (mask_type V0), GPR:$vl), - (PseudoMask $rs3, $rs1, (mask_type V0), (NoX0 GPR:$vl), sew)>; -} - -multiclass VPatSStore<string intrinsic, - string inst, - LLVMType type, - LLVMType mask_type, - int sew, - LMULInfo vlmul, - VReg reg_class> -{ - defvar Intr = !cast<Intrinsic>(intrinsic); - defvar Pseudo = !cast<Instruction>(inst#"_V_"#vlmul.MX); - def : Pat<(Intr (type reg_class:$rs3), GPR:$rs1, GPR:$rs2, GPR:$vl), - (Pseudo $rs3, $rs1, $rs2, (NoX0 GPR:$vl), sew)>; - defvar IntrMask = !cast<Intrinsic>(intrinsic # "_mask"); - defvar PseudoMask = !cast<Instruction>(inst#"_V_"#vlmul.MX#"_MASK"); - def : Pat<(IntrMask (type reg_class:$rs3), GPR:$rs1, GPR:$rs2, (mask_type V0), GPR:$vl), - (PseudoMask $rs3, $rs1, $rs2, (mask_type V0), (NoX0 GPR:$vl), sew)>; -} - -multiclass VPatIStore<string intrinsic, - string inst, - LLVMType type, - LLVMType idx_type, - LLVMType mask_type, - int sew, - LMULInfo vlmul, - LMULInfo idx_vlmul, - VReg reg_class, - VReg idx_reg_class> -{ - defvar Intr = !cast<Intrinsic>(intrinsic); - defvar Pseudo = !cast<Instruction>(inst#"_V_"#idx_vlmul.MX#"_"#vlmul.MX); - def : Pat<(Intr (type reg_class:$rs3), GPR:$rs1, - (idx_type idx_reg_class:$rs2), GPR:$vl), - (Pseudo $rs3, $rs1, $rs2, (NoX0 GPR:$vl), sew)>; - defvar IntrMask = !cast<Intrinsic>(intrinsic # "_mask"); - defvar PseudoMask = !cast<Instruction>(inst#"_V_"#idx_vlmul.MX#"_"#vlmul.MX#"_MASK"); - def : Pat<(IntrMask (type reg_class:$rs3), GPR:$rs1, - (idx_type idx_reg_class:$rs2), (mask_type V0), GPR:$vl), - (PseudoMask $rs3, $rs1, $rs2, (mask_type V0), (NoX0 GPR:$vl), sew)>; -} + (mask_type V0), GPR:$vl, sew)>; multiclass VPatUnaryS_M<string intrinsic_name, string inst> { foreach mti = AllMasks in { def : Pat<(XLenVT (!cast<Intrinsic>(intrinsic_name) - (mti.Mask VR:$rs1), GPR:$vl)), + (mti.Mask VR:$rs1), VLOpFrag)), (!cast<Instruction>(inst#"_M_"#mti.BX) $rs1, - (NoX0 GPR:$vl), mti.SEW)>; + GPR:$vl, mti.Log2SEW)>; def : Pat<(XLenVT (!cast<Intrinsic>(intrinsic_name # "_mask") - (mti.Mask VR:$rs1), (mti.Mask V0), GPR:$vl)), + (mti.Mask VR:$rs1), (mti.Mask V0), VLOpFrag)), (!cast<Instruction>(inst#"_M_"#mti.BX#"_MASK") $rs1, - (mti.Mask V0), (NoX0 GPR:$vl), mti.SEW)>; + (mti.Mask V0), GPR:$vl, mti.Log2SEW)>; } } @@ -2304,7 +2480,7 @@ multiclass VPatUnaryV_V_AnyMask<string intrinsic, string instruction, foreach vti = vtilist in { def : VPatUnaryAnyMask<intrinsic, instruction, "VM", vti.Vector, vti.Vector, vti.Mask, - vti.SEW, vti.LMul, vti.RegClass, + vti.Log2SEW, vti.LMul, vti.RegClass, vti.RegClass>; } } @@ -2322,9 +2498,9 @@ multiclass VPatUnaryV_M<string intrinsic, string instruction> { foreach vti = AllIntegerVectors in { def : VPatUnaryNoMask<intrinsic, instruction, "M", vti.Vector, vti.Mask, - vti.SEW, vti.LMul, VR>; + vti.Log2SEW, vti.LMul, VR>; def : VPatUnaryMask<intrinsic, instruction, "M", vti.Vector, vti.Mask, - vti.Mask, vti.SEW, vti.LMul, vti.RegClass, VR>; + vti.Mask, vti.Log2SEW, vti.LMul, vti.RegClass, VR>; } } @@ -2337,10 +2513,10 @@ multiclass VPatUnaryV_VF<string intrinsic, string instruction, string suffix, defvar fti = vtiTofti.Fti; def : VPatUnaryNoMask<intrinsic, instruction, suffix, vti.Vector, fti.Vector, - vti.SEW, vti.LMul, fti.RegClass>; + vti.Log2SEW, vti.LMul, fti.RegClass>; def : VPatUnaryMask<intrinsic, instruction, suffix, vti.Vector, fti.Vector, vti.Mask, - vti.SEW, vti.LMul, vti.RegClass, fti.RegClass>; + vti.Log2SEW, vti.LMul, vti.RegClass, fti.RegClass>; } } @@ -2349,10 +2525,10 @@ multiclass VPatUnaryV_V<string intrinsic, string instruction, foreach vti = vtilist in { def : VPatUnaryNoMask<intrinsic, instruction, "V", vti.Vector, vti.Vector, - vti.SEW, vti.LMul, vti.RegClass>; + vti.Log2SEW, vti.LMul, vti.RegClass>; def : VPatUnaryMask<intrinsic, instruction, "V", vti.Vector, vti.Vector, vti.Mask, - vti.SEW, vti.LMul, vti.RegClass, vti.RegClass>; + vti.Log2SEW, vti.LMul, vti.RegClass, vti.RegClass>; } } @@ -2360,24 +2536,24 @@ multiclass VPatNullaryV<string intrinsic, string instruction> { foreach vti = AllIntegerVectors in { def : Pat<(vti.Vector (!cast<Intrinsic>(intrinsic) - (XLenVT GPR:$vl))), + VLOpFrag)), (!cast<Instruction>(instruction#"_V_" # vti.LMul.MX) - (NoX0 GPR:$vl), vti.SEW)>; + GPR:$vl, vti.Log2SEW)>; def : Pat<(vti.Vector (!cast<Intrinsic>(intrinsic # "_mask") (vti.Vector vti.RegClass:$merge), - (vti.Mask V0), (XLenVT GPR:$vl))), + (vti.Mask V0), VLOpFrag)), (!cast<Instruction>(instruction#"_V_" # vti.LMul.MX # "_MASK") vti.RegClass:$merge, (vti.Mask V0), - (NoX0 GPR:$vl), vti.SEW)>; + GPR:$vl, vti.Log2SEW)>; } } multiclass VPatNullaryM<string intrinsic, string inst> { foreach mti = AllMasks in def : Pat<(mti.Mask (!cast<Intrinsic>(intrinsic) - (XLenVT GPR:$vl))), + (XLenVT (VLOp (XLenVT (XLenVT GPR:$vl)))))), (!cast<Instruction>(inst#"_M_"#mti.BX) - (NoX0 GPR:$vl), mti.SEW)>; + GPR:$vl, mti.Log2SEW)>; } multiclass VPatBinary<string intrinsic, @@ -2398,6 +2574,24 @@ multiclass VPatBinary<string intrinsic, op2_kind>; } +multiclass VPatBinarySwapped<string intrinsic, + string inst, + ValueType result_type, + ValueType op1_type, + ValueType op2_type, + ValueType mask_type, + int sew, + VReg result_reg_class, + VReg op1_reg_class, + DAGOperand op2_kind> +{ + def : VPatBinaryNoMaskSwapped<intrinsic, inst, result_type, op1_type, op2_type, + sew, op1_reg_class, op2_kind>; + def : VPatBinaryMaskSwapped<intrinsic, inst, result_type, op1_type, op2_type, + mask_type, sew, result_reg_class, op1_reg_class, + op2_kind>; +} + multiclass VPatBinaryCarryIn<string intrinsic, string inst, string kind, @@ -2414,11 +2608,11 @@ multiclass VPatBinaryCarryIn<string intrinsic, (op1_type op1_reg_class:$rs1), (op2_type op2_kind:$rs2), (mask_type V0), - (XLenVT GPR:$vl))), + VLOpFrag)), (!cast<Instruction>(inst#"_"#kind#"_"#vlmul.MX) (op1_type op1_reg_class:$rs1), (op2_type op2_kind:$rs2), - (mask_type V0), (NoX0 GPR:$vl), sew)>; + (mask_type V0), GPR:$vl, sew)>; } multiclass VPatBinaryMaskOut<string intrinsic, @@ -2435,11 +2629,11 @@ multiclass VPatBinaryMaskOut<string intrinsic, def : Pat<(result_type (!cast<Intrinsic>(intrinsic) (op1_type op1_reg_class:$rs1), (op2_type op2_kind:$rs2), - (XLenVT GPR:$vl))), + VLOpFrag)), (!cast<Instruction>(inst#"_"#kind#"_"#vlmul.MX) (op1_type op1_reg_class:$rs1), (op2_type op2_kind:$rs2), - (NoX0 GPR:$vl), sew)>; + GPR:$vl, sew)>; } multiclass VPatConversion<string intrinsic, @@ -2464,7 +2658,7 @@ multiclass VPatBinaryV_VV<string intrinsic, string instruction, foreach vti = vtilist in defm : VPatBinary<intrinsic, instruction # "_VV_" # vti.LMul.MX, vti.Vector, vti.Vector, vti.Vector,vti.Mask, - vti.SEW, vti.RegClass, + vti.Log2SEW, vti.RegClass, vti.RegClass, vti.RegClass>; } @@ -2474,7 +2668,7 @@ multiclass VPatBinaryV_VV_INT<string intrinsic, string instruction, defvar ivti = GetIntVTypeInfo<vti>.Vti; defm : VPatBinary<intrinsic, instruction # "_VV_" # vti.LMul.MX, vti.Vector, vti.Vector, ivti.Vector, vti.Mask, - vti.SEW, vti.RegClass, + vti.Log2SEW, vti.RegClass, vti.RegClass, vti.RegClass>; } } @@ -2484,15 +2678,15 @@ multiclass VPatBinaryV_VV_INT_EEW<string intrinsic, string instruction, foreach vti = vtilist in { // emul = lmul * eew / sew defvar vlmul = vti.LMul; - defvar octuple_lmul = octuple_from_str<vlmul.MX>.ret; - defvar octuple_emul = !srl(!mul(octuple_lmul, eew), shift_amount<vti.SEW>.val); + defvar octuple_lmul = vlmul.octuple; + defvar octuple_emul = !srl(!mul(octuple_lmul, eew), vti.Log2SEW); if !and(!ge(octuple_emul, 1), !le(octuple_emul, 64)) then { defvar emul_str = octuple_to_str<octuple_emul>.ret; defvar ivti = !cast<VTypeInfo>("VI" # eew # emul_str); defvar inst = instruction # "_VV_" # vti.LMul.MX # "_" # emul_str; defm : VPatBinary<intrinsic, inst, vti.Vector, vti.Vector, ivti.Vector, vti.Mask, - vti.SEW, vti.RegClass, + vti.Log2SEW, vti.RegClass, vti.RegClass, ivti.RegClass>; } } @@ -2504,7 +2698,7 @@ multiclass VPatBinaryV_VX<string intrinsic, string instruction, defvar kind = "V"#vti.ScalarSuffix; defm : VPatBinary<intrinsic, instruction#"_"#kind#"_"#vti.LMul.MX, vti.Vector, vti.Vector, vti.Scalar, vti.Mask, - vti.SEW, vti.RegClass, + vti.Log2SEW, vti.RegClass, vti.RegClass, vti.ScalarRegClass>; } } @@ -2514,7 +2708,7 @@ multiclass VPatBinaryV_VX_INT<string intrinsic, string instruction, foreach vti = vtilist in defm : VPatBinary<intrinsic, instruction # "_VX_" # vti.LMul.MX, vti.Vector, vti.Vector, XLenVT, vti.Mask, - vti.SEW, vti.RegClass, + vti.Log2SEW, vti.RegClass, vti.RegClass, GPR>; } @@ -2523,7 +2717,7 @@ multiclass VPatBinaryV_VI<string intrinsic, string instruction, foreach vti = vtilist in defm : VPatBinary<intrinsic, instruction # "_VI_" # vti.LMul.MX, vti.Vector, vti.Vector, XLenVT, vti.Mask, - vti.SEW, vti.RegClass, + vti.Log2SEW, vti.RegClass, vti.RegClass, imm_type>; } @@ -2531,7 +2725,7 @@ multiclass VPatBinaryM_MM<string intrinsic, string instruction> { foreach mti = AllMasks in def : VPatBinaryNoMask<intrinsic, instruction # "_MM_" # mti.LMul.MX, mti.Mask, mti.Mask, mti.Mask, - mti.SEW, VR, VR>; + mti.Log2SEW, VR, VR>; } multiclass VPatBinaryW_VV<string intrinsic, string instruction, @@ -2541,7 +2735,7 @@ multiclass VPatBinaryW_VV<string intrinsic, string instruction, defvar Wti = VtiToWti.Wti; defm : VPatBinary<intrinsic, instruction # "_VV_" # Vti.LMul.MX, Wti.Vector, Vti.Vector, Vti.Vector, Vti.Mask, - Vti.SEW, Wti.RegClass, + Vti.Log2SEW, Wti.RegClass, Vti.RegClass, Vti.RegClass>; } } @@ -2554,7 +2748,7 @@ multiclass VPatBinaryW_VX<string intrinsic, string instruction, defvar kind = "V"#Vti.ScalarSuffix; defm : VPatBinary<intrinsic, instruction#"_"#kind#"_"#Vti.LMul.MX, Wti.Vector, Vti.Vector, Vti.Scalar, Vti.Mask, - Vti.SEW, Wti.RegClass, + Vti.Log2SEW, Wti.RegClass, Vti.RegClass, Vti.ScalarRegClass>; } } @@ -2564,10 +2758,17 @@ multiclass VPatBinaryW_WV<string intrinsic, string instruction, foreach VtiToWti = vtilist in { defvar Vti = VtiToWti.Vti; defvar Wti = VtiToWti.Wti; - defm : VPatBinary<intrinsic, instruction # "_WV_" # Vti.LMul.MX, - Wti.Vector, Wti.Vector, Vti.Vector, Vti.Mask, - Vti.SEW, Wti.RegClass, - Wti.RegClass, Vti.RegClass>; + def : VPatTiedBinaryNoMask<intrinsic, instruction # "_WV_" # Vti.LMul.MX, + Wti.Vector, Vti.Vector, + Vti.Log2SEW, Wti.RegClass, Vti.RegClass>; + let AddedComplexity = 1 in + def : VPatTiedBinaryMask<intrinsic, instruction # "_WV_" # Vti.LMul.MX, + Wti.Vector, Vti.Vector, Vti.Mask, + Vti.Log2SEW, Wti.RegClass, Vti.RegClass>; + def : VPatBinaryMask<intrinsic, instruction # "_WV_" # Vti.LMul.MX, + Wti.Vector, Wti.Vector, Vti.Vector, Vti.Mask, + Vti.Log2SEW, Wti.RegClass, + Wti.RegClass, Vti.RegClass>; } } @@ -2579,7 +2780,7 @@ multiclass VPatBinaryW_WX<string intrinsic, string instruction, defvar kind = "W"#Vti.ScalarSuffix; defm : VPatBinary<intrinsic, instruction#"_"#kind#"_"#Vti.LMul.MX, Wti.Vector, Wti.Vector, Vti.Scalar, Vti.Mask, - Vti.SEW, Wti.RegClass, + Vti.Log2SEW, Wti.RegClass, Wti.RegClass, Vti.ScalarRegClass>; } } @@ -2591,7 +2792,7 @@ multiclass VPatBinaryV_WV<string intrinsic, string instruction, defvar Wti = VtiToWti.Wti; defm : VPatBinary<intrinsic, instruction # "_WV_" # Vti.LMul.MX, Vti.Vector, Wti.Vector, Vti.Vector, Vti.Mask, - Vti.SEW, Vti.RegClass, + Vti.Log2SEW, Vti.RegClass, Wti.RegClass, Vti.RegClass>; } } @@ -2604,7 +2805,7 @@ multiclass VPatBinaryV_WX<string intrinsic, string instruction, defvar kind = "W"#Vti.ScalarSuffix; defm : VPatBinary<intrinsic, instruction#"_"#kind#"_"#Vti.LMul.MX, Vti.Vector, Wti.Vector, Vti.Scalar, Vti.Mask, - Vti.SEW, Vti.RegClass, + Vti.Log2SEW, Vti.RegClass, Wti.RegClass, Vti.ScalarRegClass>; } } @@ -2616,7 +2817,7 @@ multiclass VPatBinaryV_WI<string intrinsic, string instruction, defvar Wti = VtiToWti.Wti; defm : VPatBinary<intrinsic, instruction # "_WI_" # Vti.LMul.MX, Vti.Vector, Wti.Vector, XLenVT, Vti.Mask, - Vti.SEW, Vti.RegClass, + Vti.Log2SEW, Vti.RegClass, Wti.RegClass, uimm5>; } } @@ -2628,7 +2829,7 @@ multiclass VPatBinaryV_VM<string intrinsic, string instruction, defm : VPatBinaryCarryIn<intrinsic, instruction, "VVM", !if(CarryOut, vti.Mask, vti.Vector), vti.Vector, vti.Vector, vti.Mask, - vti.SEW, vti.LMul, + vti.Log2SEW, vti.LMul, vti.RegClass, vti.RegClass>; } @@ -2640,7 +2841,7 @@ multiclass VPatBinaryV_XM<string intrinsic, string instruction, "V"#vti.ScalarSuffix#"M", !if(CarryOut, vti.Mask, vti.Vector), vti.Vector, vti.Scalar, vti.Mask, - vti.SEW, vti.LMul, + vti.Log2SEW, vti.LMul, vti.RegClass, vti.ScalarRegClass>; } @@ -2650,7 +2851,7 @@ multiclass VPatBinaryV_IM<string intrinsic, string instruction, defm : VPatBinaryCarryIn<intrinsic, instruction, "VIM", !if(CarryOut, vti.Mask, vti.Vector), vti.Vector, XLenVT, vti.Mask, - vti.SEW, vti.LMul, + vti.Log2SEW, vti.LMul, vti.RegClass, simm5>; } @@ -2658,7 +2859,7 @@ multiclass VPatBinaryV_V<string intrinsic, string instruction> { foreach vti = AllIntegerVectors in defm : VPatBinaryMaskOut<intrinsic, instruction, "VV", vti.Mask, vti.Vector, vti.Vector, - vti.SEW, vti.LMul, + vti.Log2SEW, vti.LMul, vti.RegClass, vti.RegClass>; } @@ -2666,7 +2867,7 @@ multiclass VPatBinaryV_X<string intrinsic, string instruction> { foreach vti = AllIntegerVectors in defm : VPatBinaryMaskOut<intrinsic, instruction, "VX", vti.Mask, vti.Vector, XLenVT, - vti.SEW, vti.LMul, + vti.Log2SEW, vti.LMul, vti.RegClass, GPR>; } @@ -2674,7 +2875,7 @@ multiclass VPatBinaryV_I<string intrinsic, string instruction> { foreach vti = AllIntegerVectors in defm : VPatBinaryMaskOut<intrinsic, instruction, "VI", vti.Mask, vti.Vector, XLenVT, - vti.SEW, vti.LMul, + vti.Log2SEW, vti.LMul, vti.RegClass, simm5>; } @@ -2683,17 +2884,26 @@ multiclass VPatBinaryM_VV<string intrinsic, string instruction, foreach vti = vtilist in defm : VPatBinary<intrinsic, instruction # "_VV_" # vti.LMul.MX, vti.Mask, vti.Vector, vti.Vector, vti.Mask, - vti.SEW, VR, + vti.Log2SEW, VR, vti.RegClass, vti.RegClass>; } +multiclass VPatBinarySwappedM_VV<string intrinsic, string instruction, + list<VTypeInfo> vtilist> { + foreach vti = vtilist in + defm : VPatBinarySwapped<intrinsic, instruction # "_VV_" # vti.LMul.MX, + vti.Mask, vti.Vector, vti.Vector, vti.Mask, + vti.Log2SEW, VR, + vti.RegClass, vti.RegClass>; +} + multiclass VPatBinaryM_VX<string intrinsic, string instruction, list<VTypeInfo> vtilist> { foreach vti = vtilist in { defvar kind = "V"#vti.ScalarSuffix; defm : VPatBinary<intrinsic, instruction#"_"#kind#"_"#vti.LMul.MX, vti.Mask, vti.Vector, vti.Scalar, vti.Mask, - vti.SEW, VR, + vti.Log2SEW, VR, vti.RegClass, vti.ScalarRegClass>; } } @@ -2703,92 +2913,68 @@ multiclass VPatBinaryM_VI<string intrinsic, string instruction, foreach vti = vtilist in defm : VPatBinary<intrinsic, instruction # "_VI_" # vti.LMul.MX, vti.Mask, vti.Vector, XLenVT, vti.Mask, - vti.SEW, VR, + vti.Log2SEW, VR, vti.RegClass, simm5>; } multiclass VPatBinaryV_VV_VX_VI<string intrinsic, string instruction, list<VTypeInfo> vtilist, Operand ImmType = simm5> -{ - defm "" : VPatBinaryV_VV<intrinsic, instruction, vtilist>; - defm "" : VPatBinaryV_VX<intrinsic, instruction, vtilist>; - defm "" : VPatBinaryV_VI<intrinsic, instruction, vtilist, ImmType>; -} + : VPatBinaryV_VV<intrinsic, instruction, vtilist>, + VPatBinaryV_VX<intrinsic, instruction, vtilist>, + VPatBinaryV_VI<intrinsic, instruction, vtilist, ImmType>; multiclass VPatBinaryV_VV_VX<string intrinsic, string instruction, list<VTypeInfo> vtilist> -{ - defm "" : VPatBinaryV_VV<intrinsic, instruction, vtilist>; - defm "" : VPatBinaryV_VX<intrinsic, instruction, vtilist>; -} + : VPatBinaryV_VV<intrinsic, instruction, vtilist>, + VPatBinaryV_VX<intrinsic, instruction, vtilist>; multiclass VPatBinaryV_VX_VI<string intrinsic, string instruction, list<VTypeInfo> vtilist> -{ - defm "" : VPatBinaryV_VX<intrinsic, instruction, vtilist>; - defm "" : VPatBinaryV_VI<intrinsic, instruction, vtilist, simm5>; -} + : VPatBinaryV_VX<intrinsic, instruction, vtilist>, + VPatBinaryV_VI<intrinsic, instruction, vtilist, simm5>; multiclass VPatBinaryW_VV_VX<string intrinsic, string instruction, list<VTypeInfoToWide> vtilist> -{ - defm "" : VPatBinaryW_VV<intrinsic, instruction, vtilist>; - defm "" : VPatBinaryW_VX<intrinsic, instruction, vtilist>; -} + : VPatBinaryW_VV<intrinsic, instruction, vtilist>, + VPatBinaryW_VX<intrinsic, instruction, vtilist>; multiclass VPatBinaryW_WV_WX<string intrinsic, string instruction, list<VTypeInfoToWide> vtilist> -{ - defm "" : VPatBinaryW_WV<intrinsic, instruction, vtilist>; - defm "" : VPatBinaryW_WX<intrinsic, instruction, vtilist>; -} + : VPatBinaryW_WV<intrinsic, instruction, vtilist>, + VPatBinaryW_WX<intrinsic, instruction, vtilist>; multiclass VPatBinaryV_WV_WX_WI<string intrinsic, string instruction, list<VTypeInfoToWide> vtilist> -{ - defm "" : VPatBinaryV_WV<intrinsic, instruction, vtilist>; - defm "" : VPatBinaryV_WX<intrinsic, instruction, vtilist>; - defm "" : VPatBinaryV_WI<intrinsic, instruction, vtilist>; -} + : VPatBinaryV_WV<intrinsic, instruction, vtilist>, + VPatBinaryV_WX<intrinsic, instruction, vtilist>, + VPatBinaryV_WI<intrinsic, instruction, vtilist>; multiclass VPatBinaryV_VM_XM_IM<string intrinsic, string instruction> -{ - defm "" : VPatBinaryV_VM<intrinsic, instruction>; - defm "" : VPatBinaryV_XM<intrinsic, instruction>; - defm "" : VPatBinaryV_IM<intrinsic, instruction>; -} + : VPatBinaryV_VM<intrinsic, instruction>, + VPatBinaryV_XM<intrinsic, instruction>, + VPatBinaryV_IM<intrinsic, instruction>; multiclass VPatBinaryM_VM_XM_IM<string intrinsic, string instruction> -{ - defm "" : VPatBinaryV_VM<intrinsic, instruction, /*CarryOut=*/1>; - defm "" : VPatBinaryV_XM<intrinsic, instruction, /*CarryOut=*/1>; - defm "" : VPatBinaryV_IM<intrinsic, instruction, /*CarryOut=*/1>; -} + : VPatBinaryV_VM<intrinsic, instruction, /*CarryOut=*/1>, + VPatBinaryV_XM<intrinsic, instruction, /*CarryOut=*/1>, + VPatBinaryV_IM<intrinsic, instruction, /*CarryOut=*/1>; multiclass VPatBinaryM_V_X_I<string intrinsic, string instruction> -{ - defm "" : VPatBinaryV_V<intrinsic, instruction>; - defm "" : VPatBinaryV_X<intrinsic, instruction>; - defm "" : VPatBinaryV_I<intrinsic, instruction>; -} + : VPatBinaryV_V<intrinsic, instruction>, + VPatBinaryV_X<intrinsic, instruction>, + VPatBinaryV_I<intrinsic, instruction>; multiclass VPatBinaryV_VM_XM<string intrinsic, string instruction> -{ - defm "" : VPatBinaryV_VM<intrinsic, instruction>; - defm "" : VPatBinaryV_XM<intrinsic, instruction>; -} + : VPatBinaryV_VM<intrinsic, instruction>, + VPatBinaryV_XM<intrinsic, instruction>; multiclass VPatBinaryM_VM_XM<string intrinsic, string instruction> -{ - defm "" : VPatBinaryV_VM<intrinsic, instruction, /*CarryOut=*/1>; - defm "" : VPatBinaryV_XM<intrinsic, instruction, /*CarryOut=*/1>; -} + : VPatBinaryV_VM<intrinsic, instruction, /*CarryOut=*/1>, + VPatBinaryV_XM<intrinsic, instruction, /*CarryOut=*/1>; multiclass VPatBinaryM_V_X<string intrinsic, string instruction> -{ - defm "" : VPatBinaryV_V<intrinsic, instruction>; - defm "" : VPatBinaryV_X<intrinsic, instruction>; -} + : VPatBinaryV_V<intrinsic, instruction>, + VPatBinaryV_X<intrinsic, instruction>; multiclass VPatTernary<string intrinsic, string inst, @@ -2815,7 +3001,7 @@ multiclass VPatTernaryV_VV<string intrinsic, string instruction, foreach vti = vtilist in defm : VPatTernary<intrinsic, instruction, "VV", vti.Vector, vti.Vector, vti.Vector, vti.Mask, - vti.SEW, vti.LMul, vti.RegClass, + vti.Log2SEW, vti.LMul, vti.RegClass, vti.RegClass, vti.RegClass>; } @@ -2824,7 +3010,7 @@ multiclass VPatTernaryV_VX<string intrinsic, string instruction, foreach vti = vtilist in defm : VPatTernary<intrinsic, instruction, "VX", vti.Vector, vti.Vector, XLenVT, vti.Mask, - vti.SEW, vti.LMul, vti.RegClass, + vti.Log2SEW, vti.LMul, vti.RegClass, vti.RegClass, GPR>; } @@ -2834,7 +3020,7 @@ multiclass VPatTernaryV_VX_AAXA<string intrinsic, string instruction, defm : VPatTernary<intrinsic, instruction, "V"#vti.ScalarSuffix, vti.Vector, vti.Scalar, vti.Vector, vti.Mask, - vti.SEW, vti.LMul, vti.RegClass, + vti.Log2SEW, vti.LMul, vti.RegClass, vti.ScalarRegClass, vti.RegClass>; } @@ -2843,7 +3029,7 @@ multiclass VPatTernaryV_VI<string intrinsic, string instruction, foreach vti = vtilist in defm : VPatTernary<intrinsic, instruction, "VI", vti.Vector, vti.Vector, XLenVT, vti.Mask, - vti.SEW, vti.LMul, vti.RegClass, + vti.Log2SEW, vti.LMul, vti.RegClass, vti.RegClass, Imm_type>; } @@ -2854,7 +3040,7 @@ multiclass VPatTernaryW_VV<string intrinsic, string instruction, defvar wti = vtiToWti.Wti; defm : VPatTernary<intrinsic, instruction, "VV", wti.Vector, vti.Vector, vti.Vector, - vti.Mask, vti.SEW, vti.LMul, + vti.Mask, vti.Log2SEW, vti.LMul, wti.RegClass, vti.RegClass, vti.RegClass>; } } @@ -2867,58 +3053,47 @@ multiclass VPatTernaryW_VX<string intrinsic, string instruction, defm : VPatTernary<intrinsic, instruction, "V"#vti.ScalarSuffix, wti.Vector, vti.Scalar, vti.Vector, - vti.Mask, vti.SEW, vti.LMul, + vti.Mask, vti.Log2SEW, vti.LMul, wti.RegClass, vti.ScalarRegClass, vti.RegClass>; } } multiclass VPatTernaryV_VV_VX_AAXA<string intrinsic, string instruction, - list<VTypeInfo> vtilist> { - defm "" : VPatTernaryV_VV<intrinsic, instruction, vtilist>; - defm "" : VPatTernaryV_VX_AAXA<intrinsic, instruction, vtilist>; -} + list<VTypeInfo> vtilist> + : VPatTernaryV_VV<intrinsic, instruction, vtilist>, + VPatTernaryV_VX_AAXA<intrinsic, instruction, vtilist>; multiclass VPatTernaryV_VX_VI<string intrinsic, string instruction, - list<VTypeInfo> vtilist, Operand Imm_type = simm5> { - defm "" : VPatTernaryV_VX<intrinsic, instruction, vtilist>; - defm "" : VPatTernaryV_VI<intrinsic, instruction, vtilist, Imm_type>; -} + list<VTypeInfo> vtilist, Operand Imm_type = simm5> + : VPatTernaryV_VX<intrinsic, instruction, vtilist>, + VPatTernaryV_VI<intrinsic, instruction, vtilist, Imm_type>; multiclass VPatBinaryM_VV_VX_VI<string intrinsic, string instruction, list<VTypeInfo> vtilist> -{ - defm "" : VPatBinaryM_VV<intrinsic, instruction, vtilist>; - defm "" : VPatBinaryM_VX<intrinsic, instruction, vtilist>; - defm "" : VPatBinaryM_VI<intrinsic, instruction, vtilist>; -} + : VPatBinaryM_VV<intrinsic, instruction, vtilist>, + VPatBinaryM_VX<intrinsic, instruction, vtilist>, + VPatBinaryM_VI<intrinsic, instruction, vtilist>; multiclass VPatTernaryW_VV_VX<string intrinsic, string instruction, - list<VTypeInfoToWide> vtilist> { - defm "" : VPatTernaryW_VV<intrinsic, instruction, vtilist>; - defm "" : VPatTernaryW_VX<intrinsic, instruction, vtilist>; -} + list<VTypeInfoToWide> vtilist> + : VPatTernaryW_VV<intrinsic, instruction, vtilist>, + VPatTernaryW_VX<intrinsic, instruction, vtilist>; multiclass VPatBinaryM_VV_VX<string intrinsic, string instruction, list<VTypeInfo> vtilist> -{ - defm "" : VPatBinaryM_VV<intrinsic, instruction, vtilist>; - defm "" : VPatBinaryM_VX<intrinsic, instruction, vtilist>; -} + : VPatBinaryM_VV<intrinsic, instruction, vtilist>, + VPatBinaryM_VX<intrinsic, instruction, vtilist>; multiclass VPatBinaryM_VX_VI<string intrinsic, string instruction, list<VTypeInfo> vtilist> -{ - defm "" : VPatBinaryM_VX<intrinsic, instruction, vtilist>; - defm "" : VPatBinaryM_VI<intrinsic, instruction, vtilist>; -} + : VPatBinaryM_VX<intrinsic, instruction, vtilist>, + VPatBinaryM_VI<intrinsic, instruction, vtilist>; multiclass VPatBinaryV_VV_VX_VI_INT<string intrinsic, string instruction, list<VTypeInfo> vtilist, Operand ImmType = simm5> -{ - defm "" : VPatBinaryV_VV_INT<intrinsic, instruction, vtilist>; - defm "" : VPatBinaryV_VX_INT<intrinsic, instruction, vtilist>; - defm "" : VPatBinaryV_VI<intrinsic, instruction, vtilist, ImmType>; -} + : VPatBinaryV_VV_INT<intrinsic#"_vv", instruction, vtilist>, + VPatBinaryV_VX_INT<intrinsic#"_vx", instruction, vtilist>, + VPatBinaryV_VI<intrinsic#"_vx", instruction, vtilist, ImmType>; multiclass VPatReductionV_VS<string intrinsic, string instruction, bit IsFloat = 0> { foreach vti = !if(IsFloat, NoGroupFloatVectors, NoGroupIntegerVectors) in @@ -2927,7 +3102,7 @@ multiclass VPatReductionV_VS<string intrinsic, string instruction, bit IsFloat = defm : VPatTernary<intrinsic, instruction, "VS", vectorM1.Vector, vti.Vector, vectorM1.Vector, vti.Mask, - vti.SEW, vti.LMul, + vti.Log2SEW, vti.LMul, VR, vti.RegClass, VR>; } foreach gvti = !if(IsFloat, GroupFloatVectors, GroupIntegerVectors) in @@ -2935,7 +3110,7 @@ multiclass VPatReductionV_VS<string intrinsic, string instruction, bit IsFloat = defm : VPatTernary<intrinsic, instruction, "VS", gvti.VectorM1, gvti.Vector, gvti.VectorM1, gvti.Mask, - gvti.SEW, gvti.LMul, + gvti.Log2SEW, gvti.LMul, VR, gvti.RegClass, VR>; } } @@ -2949,7 +3124,7 @@ multiclass VPatReductionW_VS<string intrinsic, string instruction, bit IsFloat = defm : VPatTernary<intrinsic, instruction, "VS", wtiM1.Vector, vti.Vector, wtiM1.Vector, vti.Mask, - vti.SEW, vti.LMul, + vti.Log2SEW, vti.LMul, wtiM1.RegClass, vti.RegClass, wtiM1.RegClass>; } @@ -2964,7 +3139,7 @@ multiclass VPatConversionVI_VF<string intrinsic, defvar ivti = GetIntVTypeInfo<fvti>.Vti; defm : VPatConversion<intrinsic, instruction, "V", - ivti.Vector, fvti.Vector, ivti.Mask, fvti.SEW, + ivti.Vector, fvti.Vector, ivti.Mask, fvti.Log2SEW, fvti.LMul, ivti.RegClass, fvti.RegClass>; } } @@ -2977,7 +3152,7 @@ multiclass VPatConversionVF_VI<string intrinsic, defvar ivti = GetIntVTypeInfo<fvti>.Vti; defm : VPatConversion<intrinsic, instruction, "V", - fvti.Vector, ivti.Vector, fvti.Mask, ivti.SEW, + fvti.Vector, ivti.Vector, fvti.Mask, ivti.Log2SEW, ivti.LMul, fvti.RegClass, ivti.RegClass>; } } @@ -2989,7 +3164,7 @@ multiclass VPatConversionWI_VF<string intrinsic, string instruction> { defvar iwti = GetIntVTypeInfo<fvtiToFWti.Wti>.Vti; defm : VPatConversion<intrinsic, instruction, "V", - iwti.Vector, fvti.Vector, iwti.Mask, fvti.SEW, + iwti.Vector, fvti.Vector, iwti.Mask, fvti.Log2SEW, fvti.LMul, iwti.RegClass, fvti.RegClass>; } } @@ -3001,7 +3176,7 @@ multiclass VPatConversionWF_VI<string intrinsic, string instruction> { defvar fwti = vtiToWti.Wti; defm : VPatConversion<intrinsic, instruction, "V", - fwti.Vector, vti.Vector, fwti.Mask, vti.SEW, + fwti.Vector, vti.Vector, fwti.Mask, vti.Log2SEW, vti.LMul, fwti.RegClass, vti.RegClass>; } } @@ -3013,7 +3188,7 @@ multiclass VPatConversionWF_VF <string intrinsic, string instruction> { defvar fwti = fvtiToFWti.Wti; defm : VPatConversion<intrinsic, instruction, "V", - fwti.Vector, fvti.Vector, fwti.Mask, fvti.SEW, + fwti.Vector, fvti.Vector, fwti.Mask, fvti.Log2SEW, fvti.LMul, fwti.RegClass, fvti.RegClass>; } } @@ -3025,7 +3200,7 @@ multiclass VPatConversionVI_WF <string intrinsic, string instruction> { defvar fwti = vtiToWti.Wti; defm : VPatConversion<intrinsic, instruction, "W", - vti.Vector, fwti.Vector, vti.Mask, vti.SEW, + vti.Vector, fwti.Vector, vti.Mask, vti.Log2SEW, vti.LMul, vti.RegClass, fwti.RegClass>; } } @@ -3037,7 +3212,7 @@ multiclass VPatConversionVF_WI <string intrinsic, string instruction> { defvar iwti = GetIntVTypeInfo<fvtiToFWti.Wti>.Vti; defm : VPatConversion<intrinsic, instruction, "W", - fvti.Vector, iwti.Vector, fvti.Mask, fvti.SEW, + fvti.Vector, iwti.Vector, fvti.Mask, fvti.Log2SEW, fvti.LMul, fvti.RegClass, iwti.RegClass>; } } @@ -3049,7 +3224,7 @@ multiclass VPatConversionVF_WF <string intrinsic, string instruction> { defvar fwti = fvtiToFWti.Wti; defm : VPatConversion<intrinsic, instruction, "W", - fvti.Vector, fwti.Vector, fvti.Mask, fvti.SEW, + fvti.Vector, fwti.Vector, fvti.Mask, fvti.Log2SEW, fvti.LMul, fvti.RegClass, fwti.RegClass>; } } @@ -3076,16 +3251,16 @@ multiclass VPatAMOV_WD<string intrinsic, foreach eew = EEWList in { foreach vti = vtilist in { if !or(!eq(vti.SEW, 32), !eq(vti.SEW, 64)) then { - defvar octuple_lmul = octuple_from_str<vti.LMul.MX>.ret; + defvar octuple_lmul = vti.LMul.octuple; // Calculate emul = eew * lmul / sew - defvar octuple_emul = !srl(!mul(eew, octuple_lmul), shift_amount<vti.SEW>.val); + defvar octuple_emul = !srl(!mul(eew, octuple_lmul), vti.Log2SEW); if !and(!ge(octuple_emul, 1), !le(octuple_emul, 64)) then { defvar emulMX = octuple_to_str<octuple_emul>.ret; defvar offsetVti = !cast<VTypeInfo>("VI" # eew # emulMX); defvar inst_ei = inst # "EI" # eew; defm : VPatAMOWD<intrinsic, inst_ei, vti.Vector, offsetVti.Vector, - vti.Mask, vti.SEW, vti.LMul, offsetVti.LMul, offsetVti.RegClass>; + vti.Mask, vti.Log2SEW, vti.LMul, offsetVti.LMul, offsetVti.RegClass>; } } } @@ -3115,8 +3290,35 @@ let hasSideEffects = 0, mayLoad = 0, mayStore = 0, isCodeGenOnly = 1 in { let hasSideEffects = 0, mayLoad = 0, mayStore = 0, isCodeGenOnly = 1, Uses = [VL] in -def PseudoReadVL : Pseudo<(outs GPR:$rd), (ins), - [(set GPR:$rd, (riscv_read_vl))]>; +def PseudoReadVL : Pseudo<(outs GPR:$rd), (ins), []>; + +let hasSideEffects = 0, mayLoad = 0, mayStore = 1, isCodeGenOnly = 1 in { + def PseudoVSPILL_M1 : VPseudo<VS1R_V, V_M1, (outs), (ins VR:$rs1, GPR:$rs2)>; + def PseudoVSPILL_M2 : VPseudo<VS2R_V, V_M2, (outs), (ins VRM2:$rs1, GPR:$rs2)>; + def PseudoVSPILL_M4 : VPseudo<VS4R_V, V_M4, (outs), (ins VRM4:$rs1, GPR:$rs2)>; + def PseudoVSPILL_M8 : VPseudo<VS8R_V, V_M8, (outs), (ins VRM8:$rs1, GPR:$rs2)>; +} + +let hasSideEffects = 0, mayLoad = 1, mayStore = 0, isCodeGenOnly = 1 in { + def PseudoVRELOAD_M1 : VPseudo<VL1RE8_V, V_M1, (outs VR:$rs1), (ins GPR:$rs2)>; + def PseudoVRELOAD_M2 : VPseudo<VL2RE8_V, V_M2, (outs VRM2:$rs1), (ins GPR:$rs2)>; + def PseudoVRELOAD_M4 : VPseudo<VL4RE8_V, V_M4, (outs VRM4:$rs1), (ins GPR:$rs2)>; + def PseudoVRELOAD_M8 : VPseudo<VL8RE8_V, V_M8, (outs VRM8:$rs1), (ins GPR:$rs2)>; +} + +foreach lmul = MxList.m in { + foreach nf = NFSet<lmul>.L in { + defvar vreg = SegRegClass<lmul, nf>.RC; + let hasSideEffects = 0, mayLoad = 0, mayStore = 1, isCodeGenOnly = 1 in { + def "PseudoVSPILL" # nf # "_" # lmul.MX : + Pseudo<(outs), (ins vreg:$rs1, GPR:$rs2, GPR:$vlenb), []>; + } + let hasSideEffects = 0, mayLoad = 1, mayStore = 0, isCodeGenOnly = 1 in { + def "PseudoVRELOAD" # nf # "_" # lmul.MX : + Pseudo<(outs vreg:$rs1), (ins GPR:$rs2, GPR:$vlenb), []>; + } + } +} //===----------------------------------------------------------------------===// // 6. Configuration-Setting Instructions @@ -3125,7 +3327,7 @@ def PseudoReadVL : Pseudo<(outs GPR:$rd), (ins), // Pseudos. let hasSideEffects = 1, mayLoad = 0, mayStore = 0, Defs = [VL, VTYPE] in { def PseudoVSETVLI : Pseudo<(outs GPR:$rd), (ins GPR:$rs1, VTypeIOp:$vtypei), []>; - +def PseudoVSETIVLI : Pseudo<(outs GPR:$rd), (ins uimm5:$rs1, VTypeIOp:$vtypei), []>; } //===----------------------------------------------------------------------===// @@ -3137,32 +3339,29 @@ def PseudoVSETVLI : Pseudo<(outs GPR:$rd), (ins GPR:$rs1, VTypeIOp:$vtypei), []> //===----------------------------------------------------------------------===// // Pseudos Unit-Stride Loads and Stores -foreach eew = EEWList in { - defm PseudoVLE # eew : VPseudoUSLoad; - defm PseudoVSE # eew : VPseudoUSStore; -} +defm PseudoVL : VPseudoUSLoad</*isFF=*/false>; +defm PseudoVS : VPseudoUSStore; + +defm PseudoVLE1 : VPseudoLoadMask; +defm PseudoVSE1 : VPseudoStoreMask; //===----------------------------------------------------------------------===// // 7.5 Vector Strided Instructions //===----------------------------------------------------------------------===// // Vector Strided Loads and Stores -foreach eew = EEWList in { - defm PseudoVLSE # eew : VPseudoSLoad; - defm PseudoVSSE # eew : VPseudoSStore; -} +defm PseudoVLS : VPseudoSLoad; +defm PseudoVSS : VPseudoSStore; //===----------------------------------------------------------------------===// // 7.6 Vector Indexed Instructions //===----------------------------------------------------------------------===// // Vector Indexed Loads and Stores -foreach eew = EEWList in { - defm PseudoVLUXEI # eew : VPseudoILoad; - defm PseudoVLOXEI # eew : VPseudoILoad; - defm PseudoVSOXEI # eew : VPseudoIStore; - defm PseudoVSUXEI # eew : VPseudoIStore; -} +defm PseudoVLUX : VPseudoILoad</*Ordered=*/false>; +defm PseudoVLOX : VPseudoILoad</*Ordered=*/true>; +defm PseudoVSOX : VPseudoIStore</*Ordered=*/true>; +defm PseudoVSUX : VPseudoIStore</*Ordered=*/false>; //===----------------------------------------------------------------------===// // 7.7. Unit-stride Fault-Only-First Loads @@ -3170,25 +3369,23 @@ foreach eew = EEWList in { // vleff may update VL register let hasSideEffects = 1, Defs = [VL] in -foreach eew = EEWList in { - defm PseudoVLE # eew # FF : VPseudoUSLoad; -} +defm PseudoVL : VPseudoUSLoad</*isFF=*/true>; //===----------------------------------------------------------------------===// // 7.8. Vector Load/Store Segment Instructions //===----------------------------------------------------------------------===// -defm PseudoVLSEG : VPseudoUSSegLoad</*fault-only-first*/false>; +defm PseudoVLSEG : VPseudoUSSegLoad</*isFF=*/false>; defm PseudoVLSSEG : VPseudoSSegLoad; -defm PseudoVLOXSEG : VPseudoISegLoad; -defm PseudoVLUXSEG : VPseudoISegLoad; +defm PseudoVLOXSEG : VPseudoISegLoad</*Ordered=*/true>; +defm PseudoVLUXSEG : VPseudoISegLoad</*Ordered=*/false>; defm PseudoVSSEG : VPseudoUSSegStore; defm PseudoVSSSEG : VPseudoSSegStore; -defm PseudoVSOXSEG : VPseudoISegStore; -defm PseudoVSUXSEG : VPseudoISegStore; +defm PseudoVSOXSEG : VPseudoISegStore</*Ordered=*/true>; +defm PseudoVSUXSEG : VPseudoISegStore</*Ordered=*/false>; // vlseg<nf>e<eew>ff.v may update VL register let hasSideEffects = 1, Defs = [VL] in -defm PseudoVLSEG : VPseudoUSSegLoad</*fault-only-first*/true>; +defm PseudoVLSEG : VPseudoUSSegLoad</*isFF=*/true>; //===----------------------------------------------------------------------===// // 8. Vector AMO Operations @@ -3214,6 +3411,53 @@ defm PseudoVADD : VPseudoBinaryV_VV_VX_VI; defm PseudoVSUB : VPseudoBinaryV_VV_VX; defm PseudoVRSUB : VPseudoBinaryV_VX_VI; +foreach vti = AllIntegerVectors in { + // Match vrsub with 2 vector operands to vsub.vv by swapping operands. This + // Occurs when legalizing vrsub.vx intrinsics for i64 on RV32 since we need + // to use a more complex splat sequence. Add the pattern for all VTs for + // consistency. + def : Pat<(vti.Vector (int_riscv_vrsub (vti.Vector vti.RegClass:$rs2), + (vti.Vector vti.RegClass:$rs1), + VLOpFrag)), + (!cast<Instruction>("PseudoVSUB_VV_"#vti.LMul.MX) vti.RegClass:$rs1, + vti.RegClass:$rs2, + GPR:$vl, + vti.Log2SEW)>; + def : Pat<(vti.Vector (int_riscv_vrsub_mask (vti.Vector vti.RegClass:$merge), + (vti.Vector vti.RegClass:$rs2), + (vti.Vector vti.RegClass:$rs1), + (vti.Mask V0), + VLOpFrag)), + (!cast<Instruction>("PseudoVSUB_VV_"#vti.LMul.MX#"_MASK") + vti.RegClass:$merge, + vti.RegClass:$rs1, + vti.RegClass:$rs2, + (vti.Mask V0), + GPR:$vl, + vti.Log2SEW)>; + + // Match VSUB with a small immediate to vadd.vi by negating the immediate. + def : Pat<(vti.Vector (int_riscv_vsub (vti.Vector vti.RegClass:$rs1), + (vti.Scalar simm5_plus1:$rs2), + VLOpFrag)), + (!cast<Instruction>("PseudoVADD_VI_"#vti.LMul.MX) vti.RegClass:$rs1, + (NegImm simm5_plus1:$rs2), + GPR:$vl, + vti.Log2SEW)>; + def : Pat<(vti.Vector (int_riscv_vsub_mask (vti.Vector vti.RegClass:$merge), + (vti.Vector vti.RegClass:$rs1), + (vti.Scalar simm5_plus1:$rs2), + (vti.Mask V0), + VLOpFrag)), + (!cast<Instruction>("PseudoVADD_VI_"#vti.LMul.MX#"_MASK") + vti.RegClass:$merge, + vti.RegClass:$rs1, + (NegImm simm5_plus1:$rs2), + (vti.Mask V0), + GPR:$vl, + vti.Log2SEW)>; +} + //===----------------------------------------------------------------------===// // 12.2. Vector Widening Integer Add/Subtract //===----------------------------------------------------------------------===// @@ -3327,12 +3571,12 @@ defm PseudoVWMACCSU : VPseudoTernaryW_VV_VX; defm PseudoVWMACCUS : VPseudoTernaryW_VX; //===----------------------------------------------------------------------===// -// 12.16. Vector Integer Merge Instructions +// 12.15. Vector Integer Merge Instructions //===----------------------------------------------------------------------===// defm PseudoVMERGE : VPseudoBinaryV_VM_XM_IM; //===----------------------------------------------------------------------===// -// 12.17. Vector Integer Move Instructions +// 12.16. Vector Integer Move Instructions //===----------------------------------------------------------------------===// defm PseudoVMV_V : VPseudoUnaryV_V_X_I_NoDummyMask; @@ -3349,7 +3593,7 @@ let Defs = [VXSAT], hasSideEffects = 1 in { //===----------------------------------------------------------------------===// // 13.2. Vector Single-Width Averaging Add and Subtract //===----------------------------------------------------------------------===// -let Uses = [VL, VTYPE, VXRM], hasSideEffects = 1 in { +let Uses = [VXRM], hasSideEffects = 1 in { defm PseudoVAADDU : VPseudoBinaryV_VV_VX; defm PseudoVAADD : VPseudoBinaryV_VV_VX; defm PseudoVASUBU : VPseudoBinaryV_VV_VX; @@ -3359,14 +3603,14 @@ let Uses = [VL, VTYPE, VXRM], hasSideEffects = 1 in { //===----------------------------------------------------------------------===// // 13.3. Vector Single-Width Fractional Multiply with Rounding and Saturation //===----------------------------------------------------------------------===// -let Uses = [VL, VTYPE, VXRM], Defs = [VXSAT], hasSideEffects = 1 in { +let Uses = [VXRM], Defs = [VXSAT], hasSideEffects = 1 in { defm PseudoVSMUL : VPseudoBinaryV_VV_VX; } //===----------------------------------------------------------------------===// // 13.4. Vector Single-Width Scaling Shift Instructions //===----------------------------------------------------------------------===// -let Uses = [VL, VTYPE, VXRM], hasSideEffects = 1 in { +let Uses = [VXRM], hasSideEffects = 1 in { defm PseudoVSSRL : VPseudoBinaryV_VV_VX_VI<uimm5>; defm PseudoVSSRA : VPseudoBinaryV_VV_VX_VI<uimm5>; } @@ -3374,7 +3618,7 @@ let Uses = [VL, VTYPE, VXRM], hasSideEffects = 1 in { //===----------------------------------------------------------------------===// // 13.5. Vector Narrowing Fixed-Point Clip Instructions //===----------------------------------------------------------------------===// -let Uses = [VL, VTYPE, VXRM], Defs = [VXSAT], hasSideEffects = 1 in { +let Uses = [VXRM], Defs = [VXSAT], hasSideEffects = 1 in { defm PseudoVNCLIP : VPseudoBinaryV_WV_WX_WI; defm PseudoVNCLIPU : VPseudoBinaryV_WV_WX_WI; } @@ -3437,12 +3681,12 @@ defm PseudoVFSQRT : VPseudoUnaryV_V; //===----------------------------------------------------------------------===// // 14.9. Vector Floating-Point Reciprocal Square-Root Estimate Instruction //===----------------------------------------------------------------------===// -defm PseudoVFRSQRTE7 : VPseudoUnaryV_V; +defm PseudoVFRSQRT7 : VPseudoUnaryV_V; //===----------------------------------------------------------------------===// // 14.10. Vector Floating-Point Reciprocal Estimate Instruction //===----------------------------------------------------------------------===// -defm PseudoVFRECE7 : VPseudoUnaryV_V; +defm PseudoVFREC7 : VPseudoUnaryV_V; //===----------------------------------------------------------------------===// // 14.11. Vector Floating-Point Min/Max Instructions @@ -3570,7 +3814,7 @@ defm PseudoVMNOR: VPseudoBinaryM_MM; defm PseudoVMORNOT: VPseudoBinaryM_MM; defm PseudoVMXNOR: VPseudoBinaryM_MM; -// Pseudo insturctions +// Pseudo instructions defm PseudoVMCLR : VPseudoNullaryPseudoM<"VMXOR">; defm PseudoVMSET : VPseudoNullaryPseudoM<"VMXNOR">; @@ -3620,19 +3864,18 @@ defm PseudoVID : VPseudoMaskNullaryV; //===----------------------------------------------------------------------===// let Predicates = [HasStdExtV] in { -let mayLoad = 0, mayStore = 0, hasSideEffects = 0, usesCustomInserter = 1, - Uses = [VL, VTYPE] in { +let mayLoad = 0, mayStore = 0, hasSideEffects = 0 in { foreach m = MxList.m in { let VLMul = m.value in { let HasSEWOp = 1, BaseInstr = VMV_X_S in def PseudoVMV_X_S # "_" # m.MX: Pseudo<(outs GPR:$rd), (ins m.vrclass:$rs2, ixlenimm:$sew), []>, RISCVVPseudo; - let HasVLOp = 1, HasSEWOp = 1, BaseInstr = VMV_S_X, WritesElement0 = 1, + let HasVLOp = 1, HasSEWOp = 1, BaseInstr = VMV_S_X, Constraints = "$rd = $rs1" in def PseudoVMV_S_X # "_" # m.MX: Pseudo<(outs m.vrclass:$rd), (ins m.vrclass:$rs1, GPR:$rs2, - GPR:$vl, ixlenimm:$sew), + AVL:$vl, ixlenimm:$sew), []>, RISCVVPseudo; } } @@ -3644,8 +3887,7 @@ let mayLoad = 0, mayStore = 0, hasSideEffects = 0, usesCustomInserter = 1, //===----------------------------------------------------------------------===// let Predicates = [HasStdExtV, HasStdExtF] in { -let mayLoad = 0, mayStore = 0, hasSideEffects = 0, usesCustomInserter = 1, - Uses = [VL, VTYPE] in { +let mayLoad = 0, mayStore = 0, hasSideEffects = 0 in { foreach m = MxList.m in { foreach f = FPList.fpinfo in { let VLMul = m.value in { @@ -3655,12 +3897,12 @@ let mayLoad = 0, mayStore = 0, hasSideEffects = 0, usesCustomInserter = 1, (ins m.vrclass:$rs2, ixlenimm:$sew), []>, RISCVVPseudo; - let HasVLOp = 1, HasSEWOp = 1, BaseInstr = VFMV_S_F, WritesElement0 = 1, + let HasVLOp = 1, HasSEWOp = 1, BaseInstr = VFMV_S_F, Constraints = "$rd = $rs1" in def "PseudoVFMV_S_" # f.FX # "_" # m.MX : Pseudo<(outs m.vrclass:$rd), (ins m.vrclass:$rs1, f.fprclass:$rs2, - GPR:$vl, ixlenimm:$sew), + AVL:$vl, ixlenimm:$sew), []>, RISCVVPseudo; } } @@ -3697,100 +3939,24 @@ defm PseudoVCOMPRESS : VPseudoUnaryV_V_AnyMask; //===----------------------------------------------------------------------===// // Patterns. //===----------------------------------------------------------------------===// -let Predicates = [HasStdExtV] in { - -//===----------------------------------------------------------------------===// -// 7. Vector Loads and Stores -//===----------------------------------------------------------------------===// - -//===----------------------------------------------------------------------===// -// 7.4 Vector Unit-Stride Instructions -//===----------------------------------------------------------------------===// - -foreach vti = AllVectors in -{ - defm : VPatUSLoad<"int_riscv_vle", - "PseudoVLE" # vti.SEW, - vti.Vector, vti.Mask, vti.SEW, vti.LMul, vti.RegClass>; - defm : VPatUSLoadFF<"PseudoVLE" # vti.SEW # "FF", - vti.Vector, vti.Mask, vti.SEW, vti.LMul, vti.RegClass>; - defm : VPatUSStore<"int_riscv_vse", - "PseudoVSE" # vti.SEW, - vti.Vector, vti.Mask, vti.SEW, vti.LMul, vti.RegClass>; -} - -//===----------------------------------------------------------------------===// -// 7.5 Vector Strided Instructions -//===----------------------------------------------------------------------===// - -foreach vti = AllVectors in -{ - defm : VPatSLoad<"int_riscv_vlse", - "PseudoVLSE" # vti.SEW, - vti.Vector, vti.Mask, vti.SEW, vti.LMul, vti.RegClass>; - defm : VPatSStore<"int_riscv_vsse", - "PseudoVSSE" # vti.SEW, - vti.Vector, vti.Mask, vti.SEW, vti.LMul, vti.RegClass>; -} - -//===----------------------------------------------------------------------===// -// 7.6 Vector Indexed Instructions -//===----------------------------------------------------------------------===// - -foreach vti = AllVectors in -foreach eew = EEWList in { - defvar vlmul = vti.LMul; - defvar octuple_lmul = octuple_from_str<vti.LMul.MX>.ret; - defvar log_sew = shift_amount<vti.SEW>.val; - // The data vector register group has EEW=SEW, EMUL=LMUL, while the offset - // vector register group has EEW encoding in the instruction and EMUL=(EEW/SEW)*LMUL. - // calculate octuple elmul which is (eew * octuple_lmul) >> log_sew - defvar octuple_elmul = !srl(!mul(eew, octuple_lmul), log_sew); - // legal octuple elmul should be more than 0 and less than equal 64 - if !gt(octuple_elmul, 0) then { - if !le(octuple_elmul, 64) then { - defvar elmul_str = octuple_to_str<octuple_elmul>.ret; - defvar elmul =!cast<LMULInfo>("V_" # elmul_str); - defvar idx_vti = !cast<VTypeInfo>("VI" # eew # elmul_str); - - defm : VPatILoad<"int_riscv_vluxei", - "PseudoVLUXEI"#eew, - vti.Vector, idx_vti.Vector, vti.Mask, vti.SEW, - vlmul, elmul, vti.RegClass, idx_vti.RegClass>; - defm : VPatILoad<"int_riscv_vloxei", - "PseudoVLOXEI"#eew, - vti.Vector, idx_vti.Vector, vti.Mask, vti.SEW, - vlmul, elmul, vti.RegClass, idx_vti.RegClass>; - defm : VPatIStore<"int_riscv_vsoxei", - "PseudoVSOXEI"#eew, - vti.Vector, idx_vti.Vector, vti.Mask, vti.SEW, - vlmul, elmul, vti.RegClass, idx_vti.RegClass>; - defm : VPatIStore<"int_riscv_vsuxei", - "PseudoVSUXEI"#eew, - vti.Vector, idx_vti.Vector, vti.Mask, vti.SEW, - vlmul, elmul, vti.RegClass, idx_vti.RegClass>; - } - } -} -} // Predicates = [HasStdExtV] //===----------------------------------------------------------------------===// // 8. Vector AMO Operations //===----------------------------------------------------------------------===// let Predicates = [HasStdExtZvamo] in { - defm "" : VPatAMOV_WD<"int_riscv_vamoswap", "PseudoVAMOSWAP", AllIntegerVectors>; - defm "" : VPatAMOV_WD<"int_riscv_vamoadd", "PseudoVAMOADD", AllIntegerVectors>; - defm "" : VPatAMOV_WD<"int_riscv_vamoxor", "PseudoVAMOXOR", AllIntegerVectors>; - defm "" : VPatAMOV_WD<"int_riscv_vamoand", "PseudoVAMOAND", AllIntegerVectors>; - defm "" : VPatAMOV_WD<"int_riscv_vamoor", "PseudoVAMOOR", AllIntegerVectors>; - defm "" : VPatAMOV_WD<"int_riscv_vamomin", "PseudoVAMOMIN", AllIntegerVectors>; - defm "" : VPatAMOV_WD<"int_riscv_vamomax", "PseudoVAMOMAX", AllIntegerVectors>; - defm "" : VPatAMOV_WD<"int_riscv_vamominu", "PseudoVAMOMINU", AllIntegerVectors>; - defm "" : VPatAMOV_WD<"int_riscv_vamomaxu", "PseudoVAMOMAXU", AllIntegerVectors>; + defm : VPatAMOV_WD<"int_riscv_vamoswap", "PseudoVAMOSWAP", AllIntegerVectors>; + defm : VPatAMOV_WD<"int_riscv_vamoadd", "PseudoVAMOADD", AllIntegerVectors>; + defm : VPatAMOV_WD<"int_riscv_vamoxor", "PseudoVAMOXOR", AllIntegerVectors>; + defm : VPatAMOV_WD<"int_riscv_vamoand", "PseudoVAMOAND", AllIntegerVectors>; + defm : VPatAMOV_WD<"int_riscv_vamoor", "PseudoVAMOOR", AllIntegerVectors>; + defm : VPatAMOV_WD<"int_riscv_vamomin", "PseudoVAMOMIN", AllIntegerVectors>; + defm : VPatAMOV_WD<"int_riscv_vamomax", "PseudoVAMOMAX", AllIntegerVectors>; + defm : VPatAMOV_WD<"int_riscv_vamominu", "PseudoVAMOMINU", AllIntegerVectors>; + defm : VPatAMOV_WD<"int_riscv_vamomaxu", "PseudoVAMOMAXU", AllIntegerVectors>; } // Predicates = [HasStdExtZvamo] let Predicates = [HasStdExtZvamo, HasStdExtF] in { - defm "" : VPatAMOV_WD<"int_riscv_vamoswap", "PseudoVAMOSWAP", AllFloatVectors>; + defm : VPatAMOV_WD<"int_riscv_vamoswap", "PseudoVAMOSWAP", AllFloatVectors>; } // Predicates = [HasStdExtZvamo, HasStdExtF] //===----------------------------------------------------------------------===// @@ -3801,255 +3967,317 @@ let Predicates = [HasStdExtV] in { //===----------------------------------------------------------------------===// // 12.1. Vector Single-Width Integer Add and Subtract //===----------------------------------------------------------------------===// -defm "" : VPatBinaryV_VV_VX_VI<"int_riscv_vadd", "PseudoVADD", AllIntegerVectors>; -defm "" : VPatBinaryV_VV_VX<"int_riscv_vsub", "PseudoVSUB", AllIntegerVectors>; -defm "" : VPatBinaryV_VX_VI<"int_riscv_vrsub", "PseudoVRSUB", AllIntegerVectors>; +defm : VPatBinaryV_VV_VX_VI<"int_riscv_vadd", "PseudoVADD", AllIntegerVectors>; +defm : VPatBinaryV_VV_VX<"int_riscv_vsub", "PseudoVSUB", AllIntegerVectors>; +defm : VPatBinaryV_VX_VI<"int_riscv_vrsub", "PseudoVRSUB", AllIntegerVectors>; //===----------------------------------------------------------------------===// // 12.2. Vector Widening Integer Add/Subtract //===----------------------------------------------------------------------===// -defm "" : VPatBinaryW_VV_VX<"int_riscv_vwaddu", "PseudoVWADDU", AllWidenableIntVectors>; -defm "" : VPatBinaryW_VV_VX<"int_riscv_vwsubu", "PseudoVWSUBU", AllWidenableIntVectors>; -defm "" : VPatBinaryW_VV_VX<"int_riscv_vwadd", "PseudoVWADD", AllWidenableIntVectors>; -defm "" : VPatBinaryW_VV_VX<"int_riscv_vwsub", "PseudoVWSUB", AllWidenableIntVectors>; -defm "" : VPatBinaryW_WV_WX<"int_riscv_vwaddu_w", "PseudoVWADDU", AllWidenableIntVectors>; -defm "" : VPatBinaryW_WV_WX<"int_riscv_vwsubu_w", "PseudoVWSUBU", AllWidenableIntVectors>; -defm "" : VPatBinaryW_WV_WX<"int_riscv_vwadd_w", "PseudoVWADD", AllWidenableIntVectors>; -defm "" : VPatBinaryW_WV_WX<"int_riscv_vwsub_w", "PseudoVWSUB", AllWidenableIntVectors>; +defm : VPatBinaryW_VV_VX<"int_riscv_vwaddu", "PseudoVWADDU", AllWidenableIntVectors>; +defm : VPatBinaryW_VV_VX<"int_riscv_vwsubu", "PseudoVWSUBU", AllWidenableIntVectors>; +defm : VPatBinaryW_VV_VX<"int_riscv_vwadd", "PseudoVWADD", AllWidenableIntVectors>; +defm : VPatBinaryW_VV_VX<"int_riscv_vwsub", "PseudoVWSUB", AllWidenableIntVectors>; +defm : VPatBinaryW_WV_WX<"int_riscv_vwaddu_w", "PseudoVWADDU", AllWidenableIntVectors>; +defm : VPatBinaryW_WV_WX<"int_riscv_vwsubu_w", "PseudoVWSUBU", AllWidenableIntVectors>; +defm : VPatBinaryW_WV_WX<"int_riscv_vwadd_w", "PseudoVWADD", AllWidenableIntVectors>; +defm : VPatBinaryW_WV_WX<"int_riscv_vwsub_w", "PseudoVWSUB", AllWidenableIntVectors>; //===----------------------------------------------------------------------===// // 12.3. Vector Integer Extension //===----------------------------------------------------------------------===// -defm "" : VPatUnaryV_VF<"int_riscv_vzext", "PseudoVZEXT", "VF2", - AllFractionableVF2IntVectors>; -defm "" : VPatUnaryV_VF<"int_riscv_vzext", "PseudoVZEXT", "VF4", - AllFractionableVF4IntVectors>; -defm "" : VPatUnaryV_VF<"int_riscv_vzext", "PseudoVZEXT", "VF8", - AllFractionableVF8IntVectors>; -defm "" : VPatUnaryV_VF<"int_riscv_vsext", "PseudoVSEXT", "VF2", - AllFractionableVF2IntVectors>; -defm "" : VPatUnaryV_VF<"int_riscv_vsext", "PseudoVSEXT", "VF4", - AllFractionableVF4IntVectors>; -defm "" : VPatUnaryV_VF<"int_riscv_vsext", "PseudoVSEXT", "VF8", - AllFractionableVF8IntVectors>; +defm : VPatUnaryV_VF<"int_riscv_vzext", "PseudoVZEXT", "VF2", + AllFractionableVF2IntVectors>; +defm : VPatUnaryV_VF<"int_riscv_vzext", "PseudoVZEXT", "VF4", + AllFractionableVF4IntVectors>; +defm : VPatUnaryV_VF<"int_riscv_vzext", "PseudoVZEXT", "VF8", + AllFractionableVF8IntVectors>; +defm : VPatUnaryV_VF<"int_riscv_vsext", "PseudoVSEXT", "VF2", + AllFractionableVF2IntVectors>; +defm : VPatUnaryV_VF<"int_riscv_vsext", "PseudoVSEXT", "VF4", + AllFractionableVF4IntVectors>; +defm : VPatUnaryV_VF<"int_riscv_vsext", "PseudoVSEXT", "VF8", + AllFractionableVF8IntVectors>; //===----------------------------------------------------------------------===// // 12.4. Vector Integer Add-with-Carry / Subtract-with-Borrow Instructions //===----------------------------------------------------------------------===// -defm "" : VPatBinaryV_VM_XM_IM<"int_riscv_vadc", "PseudoVADC">; -defm "" : VPatBinaryM_VM_XM_IM<"int_riscv_vmadc_carry_in", "PseudoVMADC">; -defm "" : VPatBinaryM_V_X_I<"int_riscv_vmadc", "PseudoVMADC">; +defm : VPatBinaryV_VM_XM_IM<"int_riscv_vadc", "PseudoVADC">; +defm : VPatBinaryM_VM_XM_IM<"int_riscv_vmadc_carry_in", "PseudoVMADC">; +defm : VPatBinaryM_V_X_I<"int_riscv_vmadc", "PseudoVMADC">; -defm "" : VPatBinaryV_VM_XM<"int_riscv_vsbc", "PseudoVSBC">; -defm "" : VPatBinaryM_VM_XM<"int_riscv_vmsbc_borrow_in", "PseudoVMSBC">; -defm "" : VPatBinaryM_V_X<"int_riscv_vmsbc", "PseudoVMSBC">; +defm : VPatBinaryV_VM_XM<"int_riscv_vsbc", "PseudoVSBC">; +defm : VPatBinaryM_VM_XM<"int_riscv_vmsbc_borrow_in", "PseudoVMSBC">; +defm : VPatBinaryM_V_X<"int_riscv_vmsbc", "PseudoVMSBC">; //===----------------------------------------------------------------------===// // 12.5. Vector Bitwise Logical Instructions //===----------------------------------------------------------------------===// -defm "" : VPatBinaryV_VV_VX_VI<"int_riscv_vand", "PseudoVAND", AllIntegerVectors>; -defm "" : VPatBinaryV_VV_VX_VI<"int_riscv_vor", "PseudoVOR", AllIntegerVectors>; -defm "" : VPatBinaryV_VV_VX_VI<"int_riscv_vxor", "PseudoVXOR", AllIntegerVectors>; +defm : VPatBinaryV_VV_VX_VI<"int_riscv_vand", "PseudoVAND", AllIntegerVectors>; +defm : VPatBinaryV_VV_VX_VI<"int_riscv_vor", "PseudoVOR", AllIntegerVectors>; +defm : VPatBinaryV_VV_VX_VI<"int_riscv_vxor", "PseudoVXOR", AllIntegerVectors>; //===----------------------------------------------------------------------===// // 12.6. Vector Single-Width Bit Shift Instructions //===----------------------------------------------------------------------===// -defm "" : VPatBinaryV_VV_VX_VI<"int_riscv_vsll", "PseudoVSLL", AllIntegerVectors, - uimm5>; -defm "" : VPatBinaryV_VV_VX_VI<"int_riscv_vsrl", "PseudoVSRL", AllIntegerVectors, - uimm5>; -defm "" : VPatBinaryV_VV_VX_VI<"int_riscv_vsra", "PseudoVSRA", AllIntegerVectors, - uimm5>; +defm : VPatBinaryV_VV_VX_VI<"int_riscv_vsll", "PseudoVSLL", AllIntegerVectors, + uimm5>; +defm : VPatBinaryV_VV_VX_VI<"int_riscv_vsrl", "PseudoVSRL", AllIntegerVectors, + uimm5>; +defm : VPatBinaryV_VV_VX_VI<"int_riscv_vsra", "PseudoVSRA", AllIntegerVectors, + uimm5>; //===----------------------------------------------------------------------===// // 12.7. Vector Narrowing Integer Right Shift Instructions //===----------------------------------------------------------------------===// -defm "" : VPatBinaryV_WV_WX_WI<"int_riscv_vnsrl", "PseudoVNSRL", AllWidenableIntVectors>; -defm "" : VPatBinaryV_WV_WX_WI<"int_riscv_vnsra", "PseudoVNSRA", AllWidenableIntVectors>; +defm : VPatBinaryV_WV_WX_WI<"int_riscv_vnsrl", "PseudoVNSRL", AllWidenableIntVectors>; +defm : VPatBinaryV_WV_WX_WI<"int_riscv_vnsra", "PseudoVNSRA", AllWidenableIntVectors>; //===----------------------------------------------------------------------===// // 12.8. Vector Integer Comparison Instructions //===----------------------------------------------------------------------===// -defm "" : VPatBinaryM_VV_VX_VI<"int_riscv_vmseq", "PseudoVMSEQ", AllIntegerVectors>; -defm "" : VPatBinaryM_VV_VX_VI<"int_riscv_vmsne", "PseudoVMSNE", AllIntegerVectors>; -defm "" : VPatBinaryM_VV_VX<"int_riscv_vmsltu", "PseudoVMSLTU", AllIntegerVectors>; -defm "" : VPatBinaryM_VV_VX<"int_riscv_vmslt", "PseudoVMSLT", AllIntegerVectors>; -defm "" : VPatBinaryM_VV_VX_VI<"int_riscv_vmsleu", "PseudoVMSLEU", AllIntegerVectors>; -defm "" : VPatBinaryM_VV_VX_VI<"int_riscv_vmsle", "PseudoVMSLE", AllIntegerVectors>; +defm : VPatBinaryM_VV_VX_VI<"int_riscv_vmseq", "PseudoVMSEQ", AllIntegerVectors>; +defm : VPatBinaryM_VV_VX_VI<"int_riscv_vmsne", "PseudoVMSNE", AllIntegerVectors>; +defm : VPatBinaryM_VV_VX<"int_riscv_vmsltu", "PseudoVMSLTU", AllIntegerVectors>; +defm : VPatBinaryM_VV_VX<"int_riscv_vmslt", "PseudoVMSLT", AllIntegerVectors>; +defm : VPatBinaryM_VV_VX_VI<"int_riscv_vmsleu", "PseudoVMSLEU", AllIntegerVectors>; +defm : VPatBinaryM_VV_VX_VI<"int_riscv_vmsle", "PseudoVMSLE", AllIntegerVectors>; -defm "" : VPatBinaryM_VX_VI<"int_riscv_vmsgtu", "PseudoVMSGTU", AllIntegerVectors>; -defm "" : VPatBinaryM_VX_VI<"int_riscv_vmsgt", "PseudoVMSGT", AllIntegerVectors>; +defm : VPatBinaryM_VX_VI<"int_riscv_vmsgtu", "PseudoVMSGTU", AllIntegerVectors>; +defm : VPatBinaryM_VX_VI<"int_riscv_vmsgt", "PseudoVMSGT", AllIntegerVectors>; + +// Match vmsgt with 2 vector operands to vmslt with the operands swapped. +defm : VPatBinarySwappedM_VV<"int_riscv_vmsgtu", "PseudoVMSLTU", AllIntegerVectors>; +defm : VPatBinarySwappedM_VV<"int_riscv_vmsgt", "PseudoVMSLT", AllIntegerVectors>; + +defm : VPatBinarySwappedM_VV<"int_riscv_vmsgeu", "PseudoVMSLEU", AllIntegerVectors>; +defm : VPatBinarySwappedM_VV<"int_riscv_vmsge", "PseudoVMSLE", AllIntegerVectors>; // Match vmslt(u).vx intrinsics to vmsle(u).vi if the scalar is -15 to 16. This // avoids the user needing to know that there is no vmslt(u).vi instruction. -// This is limited to vmslt(u).vx as there is no vmsge().vx intrinsic or -// instruction. +// Similar for vmsge(u).vx intrinsics using vmslt(u).vi. foreach vti = AllIntegerVectors in { def : Pat<(vti.Mask (int_riscv_vmslt (vti.Vector vti.RegClass:$rs1), - (vti.Scalar simm5_plus1:$rs2), GPR:$vl)), + (vti.Scalar simm5_plus1:$rs2), + VLOpFrag)), (!cast<Instruction>("PseudoVMSLE_VI_"#vti.LMul.MX) vti.RegClass:$rs1, (DecImm simm5_plus1:$rs2), - (NoX0 GPR:$vl), - vti.SEW)>; - def : Pat<(vti.Mask (int_riscv_vmslt_mask (vti.Mask V0), + GPR:$vl, + vti.Log2SEW)>; + def : Pat<(vti.Mask (int_riscv_vmslt_mask (vti.Mask VR:$merge), (vti.Vector vti.RegClass:$rs1), (vti.Scalar simm5_plus1:$rs2), - (vti.Mask VR:$merge), - GPR:$vl)), + (vti.Mask V0), + VLOpFrag)), (!cast<Instruction>("PseudoVMSLE_VI_"#vti.LMul.MX#"_MASK") VR:$merge, vti.RegClass:$rs1, (DecImm simm5_plus1:$rs2), (vti.Mask V0), - (NoX0 GPR:$vl), - vti.SEW)>; + GPR:$vl, + vti.Log2SEW)>; def : Pat<(vti.Mask (int_riscv_vmsltu (vti.Vector vti.RegClass:$rs1), - (vti.Scalar simm5_plus1:$rs2), GPR:$vl)), + (vti.Scalar simm5_plus1:$rs2), + VLOpFrag)), (!cast<Instruction>("PseudoVMSLEU_VI_"#vti.LMul.MX) vti.RegClass:$rs1, (DecImm simm5_plus1:$rs2), - (NoX0 GPR:$vl), - vti.SEW)>; - def : Pat<(vti.Mask (int_riscv_vmsltu_mask (vti.Mask V0), + GPR:$vl, + vti.Log2SEW)>; + def : Pat<(vti.Mask (int_riscv_vmsltu_mask (vti.Mask VR:$merge), (vti.Vector vti.RegClass:$rs1), (vti.Scalar simm5_plus1:$rs2), - (vti.Mask VR:$merge), - GPR:$vl)), + (vti.Mask V0), + VLOpFrag)), (!cast<Instruction>("PseudoVMSLEU_VI_"#vti.LMul.MX#"_MASK") VR:$merge, vti.RegClass:$rs1, (DecImm simm5_plus1:$rs2), (vti.Mask V0), - (NoX0 GPR:$vl), - vti.SEW)>; + GPR:$vl, + vti.Log2SEW)>; // Special cases to avoid matching vmsltu.vi 0 (always false) to // vmsleu.vi -1 (always true). Instead match to vmsne.vv. def : Pat<(vti.Mask (int_riscv_vmsltu (vti.Vector vti.RegClass:$rs1), - (vti.Scalar 0), GPR:$vl)), + (vti.Scalar 0), VLOpFrag)), (!cast<Instruction>("PseudoVMSNE_VV_"#vti.LMul.MX) vti.RegClass:$rs1, vti.RegClass:$rs1, - (NoX0 GPR:$vl), - vti.SEW)>; - def : Pat<(vti.Mask (int_riscv_vmsltu_mask (vti.Mask V0), - (vti.Vector vti.RegClass:$rs1), - (vti.Scalar 0), - (vti.Mask VR:$merge), - GPR:$vl)), + GPR:$vl, + vti.Log2SEW)>; + def : Pat<(vti.Mask (int_riscv_vmsltu_mask (vti.Mask VR:$merge), + (vti.Vector vti.RegClass:$rs1), + (vti.Scalar 0), + (vti.Mask V0), + VLOpFrag)), (!cast<Instruction>("PseudoVMSNE_VV_"#vti.LMul.MX#"_MASK") VR:$merge, vti.RegClass:$rs1, vti.RegClass:$rs1, (vti.Mask V0), - (NoX0 GPR:$vl), - vti.SEW)>; + GPR:$vl, + vti.Log2SEW)>; + + def : Pat<(vti.Mask (int_riscv_vmsge (vti.Vector vti.RegClass:$rs1), + (vti.Scalar simm5_plus1:$rs2), + VLOpFrag)), + (!cast<Instruction>("PseudoVMSGT_VI_"#vti.LMul.MX) vti.RegClass:$rs1, + (DecImm simm5_plus1:$rs2), + GPR:$vl, + vti.Log2SEW)>; + def : Pat<(vti.Mask (int_riscv_vmsge_mask (vti.Mask VR:$merge), + (vti.Vector vti.RegClass:$rs1), + (vti.Scalar simm5_plus1:$rs2), + (vti.Mask V0), + VLOpFrag)), + (!cast<Instruction>("PseudoVMSGT_VI_"#vti.LMul.MX#"_MASK") + VR:$merge, + vti.RegClass:$rs1, + (DecImm simm5_plus1:$rs2), + (vti.Mask V0), + GPR:$vl, + vti.Log2SEW)>; + + def : Pat<(vti.Mask (int_riscv_vmsgeu (vti.Vector vti.RegClass:$rs1), + (vti.Scalar simm5_plus1:$rs2), + VLOpFrag)), + (!cast<Instruction>("PseudoVMSGTU_VI_"#vti.LMul.MX) vti.RegClass:$rs1, + (DecImm simm5_plus1:$rs2), + GPR:$vl, + vti.Log2SEW)>; + def : Pat<(vti.Mask (int_riscv_vmsgeu_mask (vti.Mask VR:$merge), + (vti.Vector vti.RegClass:$rs1), + (vti.Scalar simm5_plus1:$rs2), + (vti.Mask V0), + VLOpFrag)), + (!cast<Instruction>("PseudoVMSGTU_VI_"#vti.LMul.MX#"_MASK") + VR:$merge, + vti.RegClass:$rs1, + (DecImm simm5_plus1:$rs2), + (vti.Mask V0), + GPR:$vl, + vti.Log2SEW)>; + + // Special cases to avoid matching vmsgeu.vi 0 (always true) to + // vmsgtu.vi -1 (always false). Instead match to vmsne.vv. + def : Pat<(vti.Mask (int_riscv_vmsgeu (vti.Vector vti.RegClass:$rs1), + (vti.Scalar 0), VLOpFrag)), + (!cast<Instruction>("PseudoVMSEQ_VV_"#vti.LMul.MX) vti.RegClass:$rs1, + vti.RegClass:$rs1, + GPR:$vl, + vti.Log2SEW)>; + def : Pat<(vti.Mask (int_riscv_vmsgeu_mask (vti.Mask VR:$merge), + (vti.Vector vti.RegClass:$rs1), + (vti.Scalar 0), + (vti.Mask V0), + VLOpFrag)), + (!cast<Instruction>("PseudoVMSEQ_VV_"#vti.LMul.MX#"_MASK") + VR:$merge, + vti.RegClass:$rs1, + vti.RegClass:$rs1, + (vti.Mask V0), + GPR:$vl, + vti.Log2SEW)>; } //===----------------------------------------------------------------------===// // 12.9. Vector Integer Min/Max Instructions //===----------------------------------------------------------------------===// -defm "" : VPatBinaryV_VV_VX<"int_riscv_vminu", "PseudoVMINU", AllIntegerVectors>; -defm "" : VPatBinaryV_VV_VX<"int_riscv_vmin", "PseudoVMIN", AllIntegerVectors>; -defm "" : VPatBinaryV_VV_VX<"int_riscv_vmaxu", "PseudoVMAXU", AllIntegerVectors>; -defm "" : VPatBinaryV_VV_VX<"int_riscv_vmax", "PseudoVMAX", AllIntegerVectors>; +defm : VPatBinaryV_VV_VX<"int_riscv_vminu", "PseudoVMINU", AllIntegerVectors>; +defm : VPatBinaryV_VV_VX<"int_riscv_vmin", "PseudoVMIN", AllIntegerVectors>; +defm : VPatBinaryV_VV_VX<"int_riscv_vmaxu", "PseudoVMAXU", AllIntegerVectors>; +defm : VPatBinaryV_VV_VX<"int_riscv_vmax", "PseudoVMAX", AllIntegerVectors>; //===----------------------------------------------------------------------===// // 12.10. Vector Single-Width Integer Multiply Instructions //===----------------------------------------------------------------------===// -defm "" : VPatBinaryV_VV_VX<"int_riscv_vmul", "PseudoVMUL", AllIntegerVectors>; -defm "" : VPatBinaryV_VV_VX<"int_riscv_vmulh", "PseudoVMULH", AllIntegerVectors>; -defm "" : VPatBinaryV_VV_VX<"int_riscv_vmulhu", "PseudoVMULHU", AllIntegerVectors>; -defm "" : VPatBinaryV_VV_VX<"int_riscv_vmulhsu", "PseudoVMULHSU", AllIntegerVectors>; +defm : VPatBinaryV_VV_VX<"int_riscv_vmul", "PseudoVMUL", AllIntegerVectors>; +defm : VPatBinaryV_VV_VX<"int_riscv_vmulh", "PseudoVMULH", AllIntegerVectors>; +defm : VPatBinaryV_VV_VX<"int_riscv_vmulhu", "PseudoVMULHU", AllIntegerVectors>; +defm : VPatBinaryV_VV_VX<"int_riscv_vmulhsu", "PseudoVMULHSU", AllIntegerVectors>; //===----------------------------------------------------------------------===// // 12.11. Vector Integer Divide Instructions //===----------------------------------------------------------------------===// -defm "" : VPatBinaryV_VV_VX<"int_riscv_vdivu", "PseudoVDIVU", AllIntegerVectors>; -defm "" : VPatBinaryV_VV_VX<"int_riscv_vdiv", "PseudoVDIV", AllIntegerVectors>; -defm "" : VPatBinaryV_VV_VX<"int_riscv_vremu", "PseudoVREMU", AllIntegerVectors>; -defm "" : VPatBinaryV_VV_VX<"int_riscv_vrem", "PseudoVREM", AllIntegerVectors>; +defm : VPatBinaryV_VV_VX<"int_riscv_vdivu", "PseudoVDIVU", AllIntegerVectors>; +defm : VPatBinaryV_VV_VX<"int_riscv_vdiv", "PseudoVDIV", AllIntegerVectors>; +defm : VPatBinaryV_VV_VX<"int_riscv_vremu", "PseudoVREMU", AllIntegerVectors>; +defm : VPatBinaryV_VV_VX<"int_riscv_vrem", "PseudoVREM", AllIntegerVectors>; //===----------------------------------------------------------------------===// // 12.12. Vector Widening Integer Multiply Instructions //===----------------------------------------------------------------------===// -defm "" : VPatBinaryW_VV_VX<"int_riscv_vwmul", "PseudoVWMUL", AllWidenableIntVectors>; -defm "" : VPatBinaryW_VV_VX<"int_riscv_vwmulu", "PseudoVWMULU", AllWidenableIntVectors>; -defm "" : VPatBinaryW_VV_VX<"int_riscv_vwmulsu", "PseudoVWMULSU", AllWidenableIntVectors>; +defm : VPatBinaryW_VV_VX<"int_riscv_vwmul", "PseudoVWMUL", AllWidenableIntVectors>; +defm : VPatBinaryW_VV_VX<"int_riscv_vwmulu", "PseudoVWMULU", AllWidenableIntVectors>; +defm : VPatBinaryW_VV_VX<"int_riscv_vwmulsu", "PseudoVWMULSU", AllWidenableIntVectors>; //===----------------------------------------------------------------------===// // 12.13. Vector Single-Width Integer Multiply-Add Instructions //===----------------------------------------------------------------------===// -defm "" : VPatTernaryV_VV_VX_AAXA<"int_riscv_vmadd", "PseudoVMADD", AllIntegerVectors>; -defm "" : VPatTernaryV_VV_VX_AAXA<"int_riscv_vnmsub", "PseudoVNMSUB", AllIntegerVectors>; -defm "" : VPatTernaryV_VV_VX_AAXA<"int_riscv_vmacc", "PseudoVMACC", AllIntegerVectors>; -defm "" : VPatTernaryV_VV_VX_AAXA<"int_riscv_vnmsac", "PseudoVNMSAC", AllIntegerVectors>; +defm : VPatTernaryV_VV_VX_AAXA<"int_riscv_vmadd", "PseudoVMADD", AllIntegerVectors>; +defm : VPatTernaryV_VV_VX_AAXA<"int_riscv_vnmsub", "PseudoVNMSUB", AllIntegerVectors>; +defm : VPatTernaryV_VV_VX_AAXA<"int_riscv_vmacc", "PseudoVMACC", AllIntegerVectors>; +defm : VPatTernaryV_VV_VX_AAXA<"int_riscv_vnmsac", "PseudoVNMSAC", AllIntegerVectors>; //===----------------------------------------------------------------------===// // 12.14. Vector Widening Integer Multiply-Add Instructions //===----------------------------------------------------------------------===// -defm "" : VPatTernaryW_VV_VX<"int_riscv_vwmaccu", "PseudoVWMACCU", AllWidenableIntVectors>; -defm "" : VPatTernaryW_VV_VX<"int_riscv_vwmacc", "PseudoVWMACC", AllWidenableIntVectors>; -defm "" : VPatTernaryW_VV_VX<"int_riscv_vwmaccsu", "PseudoVWMACCSU", AllWidenableIntVectors>; -defm "" : VPatTernaryW_VX<"int_riscv_vwmaccus", "PseudoVWMACCUS", AllWidenableIntVectors>; +defm : VPatTernaryW_VV_VX<"int_riscv_vwmaccu", "PseudoVWMACCU", AllWidenableIntVectors>; +defm : VPatTernaryW_VV_VX<"int_riscv_vwmacc", "PseudoVWMACC", AllWidenableIntVectors>; +defm : VPatTernaryW_VV_VX<"int_riscv_vwmaccsu", "PseudoVWMACCSU", AllWidenableIntVectors>; +defm : VPatTernaryW_VX<"int_riscv_vwmaccus", "PseudoVWMACCUS", AllWidenableIntVectors>; //===----------------------------------------------------------------------===// -// 12.16. Vector Integer Merge Instructions +// 12.15. Vector Integer Merge Instructions //===----------------------------------------------------------------------===// -defm "" : VPatBinaryV_VM_XM_IM<"int_riscv_vmerge", "PseudoVMERGE">; +defm : VPatBinaryV_VM_XM_IM<"int_riscv_vmerge", "PseudoVMERGE">; //===----------------------------------------------------------------------===// -// 12.17. Vector Integer Move Instructions +// 12.16. Vector Integer Move Instructions //===----------------------------------------------------------------------===// foreach vti = AllVectors in { def : Pat<(vti.Vector (int_riscv_vmv_v_v (vti.Vector vti.RegClass:$rs1), - GPR:$vl)), + VLOpFrag)), (!cast<Instruction>("PseudoVMV_V_V_"#vti.LMul.MX) - $rs1, (NoX0 GPR:$vl), vti.SEW)>; -} + $rs1, GPR:$vl, vti.Log2SEW)>; -foreach vti = AllIntegerVectors in { - def : Pat<(vti.Vector (int_riscv_vmv_v_x GPR:$rs2, GPR:$vl)), - (!cast<Instruction>("PseudoVMV_V_X_"#vti.LMul.MX) - $rs2, (NoX0 GPR:$vl), vti.SEW)>; - def : Pat<(vti.Vector (int_riscv_vmv_v_x simm5:$imm5, GPR:$vl)), - (!cast<Instruction>("PseudoVMV_V_I_"#vti.LMul.MX) - simm5:$imm5, (NoX0 GPR:$vl), vti.SEW)>; + // vmv.v.x/vmv.v.i are handled in RISCInstrVInstrInfoVVLPatterns.td } //===----------------------------------------------------------------------===// // 13.1. Vector Single-Width Saturating Add and Subtract //===----------------------------------------------------------------------===// -defm "" : VPatBinaryV_VV_VX_VI<"int_riscv_vsaddu", "PseudoVSADDU", AllIntegerVectors>; -defm "" : VPatBinaryV_VV_VX_VI<"int_riscv_vsadd", "PseudoVSADD", AllIntegerVectors>; -defm "" : VPatBinaryV_VV_VX<"int_riscv_vssubu", "PseudoVSSUBU", AllIntegerVectors>; -defm "" : VPatBinaryV_VV_VX<"int_riscv_vssub", "PseudoVSSUB", AllIntegerVectors>; +defm : VPatBinaryV_VV_VX_VI<"int_riscv_vsaddu", "PseudoVSADDU", AllIntegerVectors>; +defm : VPatBinaryV_VV_VX_VI<"int_riscv_vsadd", "PseudoVSADD", AllIntegerVectors>; +defm : VPatBinaryV_VV_VX<"int_riscv_vssubu", "PseudoVSSUBU", AllIntegerVectors>; +defm : VPatBinaryV_VV_VX<"int_riscv_vssub", "PseudoVSSUB", AllIntegerVectors>; //===----------------------------------------------------------------------===// // 13.2. Vector Single-Width Averaging Add and Subtract //===----------------------------------------------------------------------===// -defm "" : VPatBinaryV_VV_VX<"int_riscv_vaaddu", "PseudoVAADDU", AllIntegerVectors>; -defm "" : VPatBinaryV_VV_VX<"int_riscv_vaadd", "PseudoVAADD", AllIntegerVectors>; -defm "" : VPatBinaryV_VV_VX<"int_riscv_vasubu", "PseudoVASUBU", AllIntegerVectors>; -defm "" : VPatBinaryV_VV_VX<"int_riscv_vasub", "PseudoVASUB", AllIntegerVectors>; +defm : VPatBinaryV_VV_VX<"int_riscv_vaaddu", "PseudoVAADDU", AllIntegerVectors>; +defm : VPatBinaryV_VV_VX<"int_riscv_vaadd", "PseudoVAADD", AllIntegerVectors>; +defm : VPatBinaryV_VV_VX<"int_riscv_vasubu", "PseudoVASUBU", AllIntegerVectors>; +defm : VPatBinaryV_VV_VX<"int_riscv_vasub", "PseudoVASUB", AllIntegerVectors>; //===----------------------------------------------------------------------===// // 13.3. Vector Single-Width Fractional Multiply with Rounding and Saturation //===----------------------------------------------------------------------===// -defm "" : VPatBinaryV_VV_VX<"int_riscv_vsmul", "PseudoVSMUL", AllIntegerVectors>; +defm : VPatBinaryV_VV_VX<"int_riscv_vsmul", "PseudoVSMUL", AllIntegerVectors>; //===----------------------------------------------------------------------===// // 13.4. Vector Single-Width Scaling Shift Instructions //===----------------------------------------------------------------------===// -defm "" : VPatBinaryV_VV_VX_VI<"int_riscv_vssrl", "PseudoVSSRL", AllIntegerVectors, - uimm5>; -defm "" : VPatBinaryV_VV_VX_VI<"int_riscv_vssra", "PseudoVSSRA", AllIntegerVectors, - uimm5>; +defm : VPatBinaryV_VV_VX_VI<"int_riscv_vssrl", "PseudoVSSRL", AllIntegerVectors, + uimm5>; +defm : VPatBinaryV_VV_VX_VI<"int_riscv_vssra", "PseudoVSSRA", AllIntegerVectors, + uimm5>; //===----------------------------------------------------------------------===// // 13.5. Vector Narrowing Fixed-Point Clip Instructions //===----------------------------------------------------------------------===// -defm "" : VPatBinaryV_WV_WX_WI<"int_riscv_vnclipu", "PseudoVNCLIPU", AllWidenableIntVectors>; -defm "" : VPatBinaryV_WV_WX_WI<"int_riscv_vnclip", "PseudoVNCLIP", AllWidenableIntVectors>; +defm : VPatBinaryV_WV_WX_WI<"int_riscv_vnclipu", "PseudoVNCLIPU", AllWidenableIntVectors>; +defm : VPatBinaryV_WV_WX_WI<"int_riscv_vnclip", "PseudoVNCLIP", AllWidenableIntVectors>; } // Predicates = [HasStdExtV] @@ -4057,196 +4285,180 @@ let Predicates = [HasStdExtV, HasStdExtF] in { //===----------------------------------------------------------------------===// // 14.2. Vector Single-Width Floating-Point Add/Subtract Instructions //===----------------------------------------------------------------------===// -defm "" : VPatBinaryV_VV_VX<"int_riscv_vfadd", "PseudoVFADD", AllFloatVectors>; -defm "" : VPatBinaryV_VV_VX<"int_riscv_vfsub", "PseudoVFSUB", AllFloatVectors>; -defm "" : VPatBinaryV_VX<"int_riscv_vfrsub", "PseudoVFRSUB", AllFloatVectors>; +defm : VPatBinaryV_VV_VX<"int_riscv_vfadd", "PseudoVFADD", AllFloatVectors>; +defm : VPatBinaryV_VV_VX<"int_riscv_vfsub", "PseudoVFSUB", AllFloatVectors>; +defm : VPatBinaryV_VX<"int_riscv_vfrsub", "PseudoVFRSUB", AllFloatVectors>; //===----------------------------------------------------------------------===// // 14.3. Vector Widening Floating-Point Add/Subtract Instructions //===----------------------------------------------------------------------===// -defm "" : VPatBinaryW_VV_VX<"int_riscv_vfwadd", "PseudoVFWADD", AllWidenableFloatVectors>; -defm "" : VPatBinaryW_VV_VX<"int_riscv_vfwsub", "PseudoVFWSUB", AllWidenableFloatVectors>; -defm "" : VPatBinaryW_WV_WX<"int_riscv_vfwadd_w", "PseudoVFWADD", AllWidenableFloatVectors>; -defm "" : VPatBinaryW_WV_WX<"int_riscv_vfwsub_w", "PseudoVFWSUB", AllWidenableFloatVectors>; +defm : VPatBinaryW_VV_VX<"int_riscv_vfwadd", "PseudoVFWADD", AllWidenableFloatVectors>; +defm : VPatBinaryW_VV_VX<"int_riscv_vfwsub", "PseudoVFWSUB", AllWidenableFloatVectors>; +defm : VPatBinaryW_WV_WX<"int_riscv_vfwadd_w", "PseudoVFWADD", AllWidenableFloatVectors>; +defm : VPatBinaryW_WV_WX<"int_riscv_vfwsub_w", "PseudoVFWSUB", AllWidenableFloatVectors>; //===----------------------------------------------------------------------===// // 14.4. Vector Single-Width Floating-Point Multiply/Divide Instructions //===----------------------------------------------------------------------===// -defm "" : VPatBinaryV_VV_VX<"int_riscv_vfmul", "PseudoVFMUL", AllFloatVectors>; -defm "" : VPatBinaryV_VV_VX<"int_riscv_vfdiv", "PseudoVFDIV", AllFloatVectors>; -defm "" : VPatBinaryV_VX<"int_riscv_vfrdiv", "PseudoVFRDIV", AllFloatVectors>; +defm : VPatBinaryV_VV_VX<"int_riscv_vfmul", "PseudoVFMUL", AllFloatVectors>; +defm : VPatBinaryV_VV_VX<"int_riscv_vfdiv", "PseudoVFDIV", AllFloatVectors>; +defm : VPatBinaryV_VX<"int_riscv_vfrdiv", "PseudoVFRDIV", AllFloatVectors>; //===----------------------------------------------------------------------===// // 14.5. Vector Widening Floating-Point Multiply //===----------------------------------------------------------------------===// -defm "" : VPatBinaryW_VV_VX<"int_riscv_vfwmul", "PseudoVFWMUL", AllWidenableFloatVectors>; +defm : VPatBinaryW_VV_VX<"int_riscv_vfwmul", "PseudoVFWMUL", AllWidenableFloatVectors>; //===----------------------------------------------------------------------===// // 14.6. Vector Single-Width Floating-Point Fused Multiply-Add Instructions //===----------------------------------------------------------------------===// -defm "" : VPatTernaryV_VV_VX_AAXA<"int_riscv_vfmacc", "PseudoVFMACC", AllFloatVectors>; -defm "" : VPatTernaryV_VV_VX_AAXA<"int_riscv_vfnmacc", "PseudoVFNMACC", AllFloatVectors>; -defm "" : VPatTernaryV_VV_VX_AAXA<"int_riscv_vfmsac", "PseudoVFMSAC", AllFloatVectors>; -defm "" : VPatTernaryV_VV_VX_AAXA<"int_riscv_vfnmsac", "PseudoVFNMSAC", AllFloatVectors>; -defm "" : VPatTernaryV_VV_VX_AAXA<"int_riscv_vfmadd", "PseudoVFMADD", AllFloatVectors>; -defm "" : VPatTernaryV_VV_VX_AAXA<"int_riscv_vfnmadd", "PseudoVFNMADD", AllFloatVectors>; -defm "" : VPatTernaryV_VV_VX_AAXA<"int_riscv_vfmsub", "PseudoVFMSUB", AllFloatVectors>; -defm "" : VPatTernaryV_VV_VX_AAXA<"int_riscv_vfnmsub", "PseudoVFNMSUB", AllFloatVectors>; +defm : VPatTernaryV_VV_VX_AAXA<"int_riscv_vfmacc", "PseudoVFMACC", AllFloatVectors>; +defm : VPatTernaryV_VV_VX_AAXA<"int_riscv_vfnmacc", "PseudoVFNMACC", AllFloatVectors>; +defm : VPatTernaryV_VV_VX_AAXA<"int_riscv_vfmsac", "PseudoVFMSAC", AllFloatVectors>; +defm : VPatTernaryV_VV_VX_AAXA<"int_riscv_vfnmsac", "PseudoVFNMSAC", AllFloatVectors>; +defm : VPatTernaryV_VV_VX_AAXA<"int_riscv_vfmadd", "PseudoVFMADD", AllFloatVectors>; +defm : VPatTernaryV_VV_VX_AAXA<"int_riscv_vfnmadd", "PseudoVFNMADD", AllFloatVectors>; +defm : VPatTernaryV_VV_VX_AAXA<"int_riscv_vfmsub", "PseudoVFMSUB", AllFloatVectors>; +defm : VPatTernaryV_VV_VX_AAXA<"int_riscv_vfnmsub", "PseudoVFNMSUB", AllFloatVectors>; //===----------------------------------------------------------------------===// // 14.7. Vector Widening Floating-Point Fused Multiply-Add Instructions //===----------------------------------------------------------------------===// -defm "" : VPatTernaryW_VV_VX<"int_riscv_vfwmacc", "PseudoVFWMACC", AllWidenableFloatVectors>; -defm "" : VPatTernaryW_VV_VX<"int_riscv_vfwnmacc", "PseudoVFWNMACC", AllWidenableFloatVectors>; -defm "" : VPatTernaryW_VV_VX<"int_riscv_vfwmsac", "PseudoVFWMSAC", AllWidenableFloatVectors>; -defm "" : VPatTernaryW_VV_VX<"int_riscv_vfwnmsac", "PseudoVFWNMSAC", AllWidenableFloatVectors>; +defm : VPatTernaryW_VV_VX<"int_riscv_vfwmacc", "PseudoVFWMACC", AllWidenableFloatVectors>; +defm : VPatTernaryW_VV_VX<"int_riscv_vfwnmacc", "PseudoVFWNMACC", AllWidenableFloatVectors>; +defm : VPatTernaryW_VV_VX<"int_riscv_vfwmsac", "PseudoVFWMSAC", AllWidenableFloatVectors>; +defm : VPatTernaryW_VV_VX<"int_riscv_vfwnmsac", "PseudoVFWNMSAC", AllWidenableFloatVectors>; //===----------------------------------------------------------------------===// // 14.8. Vector Floating-Point Square-Root Instruction //===----------------------------------------------------------------------===// -defm "" : VPatUnaryV_V<"int_riscv_vfsqrt", "PseudoVFSQRT", AllFloatVectors>; +defm : VPatUnaryV_V<"int_riscv_vfsqrt", "PseudoVFSQRT", AllFloatVectors>; //===----------------------------------------------------------------------===// // 14.9. Vector Floating-Point Reciprocal Square-Root Estimate Instruction //===----------------------------------------------------------------------===// -defm "" : VPatUnaryV_V<"int_riscv_vfrsqrte7", "PseudoVFRSQRTE7", AllFloatVectors>; +defm : VPatUnaryV_V<"int_riscv_vfrsqrt7", "PseudoVFRSQRT7", AllFloatVectors>; //===----------------------------------------------------------------------===// // 14.10. Vector Floating-Point Reciprocal Estimate Instruction //===----------------------------------------------------------------------===// -defm "" : VPatUnaryV_V<"int_riscv_vfrece7", "PseudoVFRECE7", AllFloatVectors>; +defm : VPatUnaryV_V<"int_riscv_vfrec7", "PseudoVFREC7", AllFloatVectors>; //===----------------------------------------------------------------------===// // 14.11. Vector Floating-Point Min/Max Instructions //===----------------------------------------------------------------------===// -defm "" : VPatBinaryV_VV_VX<"int_riscv_vfmin", "PseudoVFMIN", AllFloatVectors>; -defm "" : VPatBinaryV_VV_VX<"int_riscv_vfmax", "PseudoVFMAX", AllFloatVectors>; +defm : VPatBinaryV_VV_VX<"int_riscv_vfmin", "PseudoVFMIN", AllFloatVectors>; +defm : VPatBinaryV_VV_VX<"int_riscv_vfmax", "PseudoVFMAX", AllFloatVectors>; //===----------------------------------------------------------------------===// // 14.12. Vector Floating-Point Sign-Injection Instructions //===----------------------------------------------------------------------===// -defm "" : VPatBinaryV_VV_VX<"int_riscv_vfsgnj", "PseudoVFSGNJ", AllFloatVectors>; -defm "" : VPatBinaryV_VV_VX<"int_riscv_vfsgnjn", "PseudoVFSGNJN", AllFloatVectors>; -defm "" : VPatBinaryV_VV_VX<"int_riscv_vfsgnjx", "PseudoVFSGNJX", AllFloatVectors>; +defm : VPatBinaryV_VV_VX<"int_riscv_vfsgnj", "PseudoVFSGNJ", AllFloatVectors>; +defm : VPatBinaryV_VV_VX<"int_riscv_vfsgnjn", "PseudoVFSGNJN", AllFloatVectors>; +defm : VPatBinaryV_VV_VX<"int_riscv_vfsgnjx", "PseudoVFSGNJX", AllFloatVectors>; //===----------------------------------------------------------------------===// // 14.13. Vector Floating-Point Compare Instructions //===----------------------------------------------------------------------===// -defm "" : VPatBinaryM_VV_VX<"int_riscv_vmfeq", "PseudoVMFEQ", AllFloatVectors>; -defm "" : VPatBinaryM_VV_VX<"int_riscv_vmfle", "PseudoVMFLE", AllFloatVectors>; -defm "" : VPatBinaryM_VV_VX<"int_riscv_vmflt", "PseudoVMFLT", AllFloatVectors>; -defm "" : VPatBinaryM_VV_VX<"int_riscv_vmfne", "PseudoVMFNE", AllFloatVectors>; -defm "" : VPatBinaryM_VX<"int_riscv_vmfgt", "PseudoVMFGT", AllFloatVectors>; -defm "" : VPatBinaryM_VX<"int_riscv_vmfge", "PseudoVMFGE", AllFloatVectors>; +defm : VPatBinaryM_VV_VX<"int_riscv_vmfeq", "PseudoVMFEQ", AllFloatVectors>; +defm : VPatBinaryM_VV_VX<"int_riscv_vmfle", "PseudoVMFLE", AllFloatVectors>; +defm : VPatBinaryM_VV_VX<"int_riscv_vmflt", "PseudoVMFLT", AllFloatVectors>; +defm : VPatBinaryM_VV_VX<"int_riscv_vmfne", "PseudoVMFNE", AllFloatVectors>; +defm : VPatBinaryM_VX<"int_riscv_vmfgt", "PseudoVMFGT", AllFloatVectors>; +defm : VPatBinaryM_VX<"int_riscv_vmfge", "PseudoVMFGE", AllFloatVectors>; +defm : VPatBinarySwappedM_VV<"int_riscv_vmfgt", "PseudoVMFLT", AllFloatVectors>; +defm : VPatBinarySwappedM_VV<"int_riscv_vmfge", "PseudoVMFLE", AllFloatVectors>; //===----------------------------------------------------------------------===// // 14.14. Vector Floating-Point Classify Instruction //===----------------------------------------------------------------------===// -defm "" : VPatConversionVI_VF<"int_riscv_vfclass", "PseudoVFCLASS">; +defm : VPatConversionVI_VF<"int_riscv_vfclass", "PseudoVFCLASS">; //===----------------------------------------------------------------------===// // 14.15. Vector Floating-Point Merge Instruction //===----------------------------------------------------------------------===// // We can use vmerge.vvm to support vector-vector vfmerge. -defm "" : VPatBinaryV_VM<"int_riscv_vfmerge", "PseudoVMERGE", - /*CarryOut = */0, /*vtilist=*/AllFloatVectors>; -defm "" : VPatBinaryV_XM<"int_riscv_vfmerge", "PseudoVFMERGE", - /*CarryOut = */0, /*vtilist=*/AllFloatVectors>; +defm : VPatBinaryV_VM<"int_riscv_vfmerge", "PseudoVMERGE", + /*CarryOut = */0, /*vtilist=*/AllFloatVectors>; +defm : VPatBinaryV_XM<"int_riscv_vfmerge", "PseudoVFMERGE", + /*CarryOut = */0, /*vtilist=*/AllFloatVectors>; foreach fvti = AllFloatVectors in { defvar instr = !cast<Instruction>("PseudoVMERGE_VIM_"#fvti.LMul.MX); def : Pat<(fvti.Vector (int_riscv_vfmerge (fvti.Vector fvti.RegClass:$rs2), (fvti.Scalar (fpimm0)), - (fvti.Mask V0), (XLenVT GPR:$vl))), - (instr fvti.RegClass:$rs2, 0, (fvti.Mask V0), (NoX0 GPR:$vl), fvti.SEW)>; -} - -//===----------------------------------------------------------------------===// -// 14.16. Vector Floating-Point Move Instruction -//===----------------------------------------------------------------------===// -foreach fvti = AllFloatVectors in { - // If we're splatting fpimm0, use vmv.v.x vd, x0. - def : Pat<(fvti.Vector (int_riscv_vfmv_v_f - (fvti.Scalar (fpimm0)), GPR:$vl)), - (!cast<Instruction>("PseudoVMV_V_I_"#fvti.LMul.MX) - 0, (NoX0 GPR:$vl), fvti.SEW)>; - - def : Pat<(fvti.Vector (int_riscv_vfmv_v_f - (fvti.Scalar fvti.ScalarRegClass:$rs2), GPR:$vl)), - (!cast<Instruction>("PseudoVFMV_V_" # fvti.ScalarSuffix # "_" # - fvti.LMul.MX) - (fvti.Scalar fvti.ScalarRegClass:$rs2), - (NoX0 GPR:$vl), fvti.SEW)>; + (fvti.Mask V0), VLOpFrag)), + (instr fvti.RegClass:$rs2, 0, (fvti.Mask V0), GPR:$vl, fvti.Log2SEW)>; } //===----------------------------------------------------------------------===// // 14.17. Single-Width Floating-Point/Integer Type-Convert Instructions //===----------------------------------------------------------------------===// -defm "" : VPatConversionVI_VF<"int_riscv_vfcvt_xu_f_v", "PseudoVFCVT_XU_F">; -defm "" : VPatConversionVI_VF<"int_riscv_vfcvt_rtz_xu_f_v", "PseudoVFCVT_RTZ_XU_F">; -defm "" : VPatConversionVI_VF<"int_riscv_vfcvt_x_f_v", "PseudoVFCVT_X_F">; -defm "" : VPatConversionVI_VF<"int_riscv_vfcvt_rtz_x_f_v", "PseudoVFCVT_RTZ_X_F">; -defm "" : VPatConversionVF_VI<"int_riscv_vfcvt_f_x_v", "PseudoVFCVT_F_X">; -defm "" : VPatConversionVF_VI<"int_riscv_vfcvt_f_xu_v", "PseudoVFCVT_F_XU">; +defm : VPatConversionVI_VF<"int_riscv_vfcvt_xu_f_v", "PseudoVFCVT_XU_F">; +defm : VPatConversionVI_VF<"int_riscv_vfcvt_rtz_xu_f_v", "PseudoVFCVT_RTZ_XU_F">; +defm : VPatConversionVI_VF<"int_riscv_vfcvt_x_f_v", "PseudoVFCVT_X_F">; +defm : VPatConversionVI_VF<"int_riscv_vfcvt_rtz_x_f_v", "PseudoVFCVT_RTZ_X_F">; +defm : VPatConversionVF_VI<"int_riscv_vfcvt_f_x_v", "PseudoVFCVT_F_X">; +defm : VPatConversionVF_VI<"int_riscv_vfcvt_f_xu_v", "PseudoVFCVT_F_XU">; //===----------------------------------------------------------------------===// // 14.18. Widening Floating-Point/Integer Type-Convert Instructions //===----------------------------------------------------------------------===// -defm "" : VPatConversionWI_VF<"int_riscv_vfwcvt_xu_f_v", "PseudoVFWCVT_XU_F">; -defm "" : VPatConversionWI_VF<"int_riscv_vfwcvt_x_f_v", "PseudoVFWCVT_X_F">; -defm "" : VPatConversionWI_VF<"int_riscv_vfwcvt_rtz_xu_f_v", "PseudoVFWCVT_RTZ_XU_F">; -defm "" : VPatConversionWI_VF<"int_riscv_vfwcvt_rtz_x_f_v", "PseudoVFWCVT_RTZ_X_F">; -defm "" : VPatConversionWF_VI<"int_riscv_vfwcvt_f_xu_v", "PseudoVFWCVT_F_XU">; -defm "" : VPatConversionWF_VI<"int_riscv_vfwcvt_f_x_v", "PseudoVFWCVT_F_X">; -defm "" : VPatConversionWF_VF<"int_riscv_vfwcvt_f_f_v", "PseudoVFWCVT_F_F">; +defm : VPatConversionWI_VF<"int_riscv_vfwcvt_xu_f_v", "PseudoVFWCVT_XU_F">; +defm : VPatConversionWI_VF<"int_riscv_vfwcvt_x_f_v", "PseudoVFWCVT_X_F">; +defm : VPatConversionWI_VF<"int_riscv_vfwcvt_rtz_xu_f_v", "PseudoVFWCVT_RTZ_XU_F">; +defm : VPatConversionWI_VF<"int_riscv_vfwcvt_rtz_x_f_v", "PseudoVFWCVT_RTZ_X_F">; +defm : VPatConversionWF_VI<"int_riscv_vfwcvt_f_xu_v", "PseudoVFWCVT_F_XU">; +defm : VPatConversionWF_VI<"int_riscv_vfwcvt_f_x_v", "PseudoVFWCVT_F_X">; +defm : VPatConversionWF_VF<"int_riscv_vfwcvt_f_f_v", "PseudoVFWCVT_F_F">; //===----------------------------------------------------------------------===// // 14.19. Narrowing Floating-Point/Integer Type-Convert Instructions //===----------------------------------------------------------------------===// -defm "" : VPatConversionVI_WF<"int_riscv_vfncvt_xu_f_w", "PseudoVFNCVT_XU_F">; -defm "" : VPatConversionVI_WF<"int_riscv_vfncvt_x_f_w", "PseudoVFNCVT_X_F">; -defm "" : VPatConversionVI_WF<"int_riscv_vfncvt_rtz_xu_f_w", "PseudoVFNCVT_RTZ_XU_F">; -defm "" : VPatConversionVI_WF<"int_riscv_vfncvt_rtz_x_f_w", "PseudoVFNCVT_RTZ_X_F">; -defm "" : VPatConversionVF_WI <"int_riscv_vfncvt_f_xu_w", "PseudoVFNCVT_F_XU">; -defm "" : VPatConversionVF_WI <"int_riscv_vfncvt_f_x_w", "PseudoVFNCVT_F_X">; -defm "" : VPatConversionVF_WF<"int_riscv_vfncvt_f_f_w", "PseudoVFNCVT_F_F">; -defm "" : VPatConversionVF_WF<"int_riscv_vfncvt_rod_f_f_w", "PseudoVFNCVT_ROD_F_F">; +defm : VPatConversionVI_WF<"int_riscv_vfncvt_xu_f_w", "PseudoVFNCVT_XU_F">; +defm : VPatConversionVI_WF<"int_riscv_vfncvt_x_f_w", "PseudoVFNCVT_X_F">; +defm : VPatConversionVI_WF<"int_riscv_vfncvt_rtz_xu_f_w", "PseudoVFNCVT_RTZ_XU_F">; +defm : VPatConversionVI_WF<"int_riscv_vfncvt_rtz_x_f_w", "PseudoVFNCVT_RTZ_X_F">; +defm : VPatConversionVF_WI <"int_riscv_vfncvt_f_xu_w", "PseudoVFNCVT_F_XU">; +defm : VPatConversionVF_WI <"int_riscv_vfncvt_f_x_w", "PseudoVFNCVT_F_X">; +defm : VPatConversionVF_WF<"int_riscv_vfncvt_f_f_w", "PseudoVFNCVT_F_F">; +defm : VPatConversionVF_WF<"int_riscv_vfncvt_rod_f_f_w", "PseudoVFNCVT_ROD_F_F">; } // Predicates = [HasStdExtV, HasStdExtF] let Predicates = [HasStdExtV] in { //===----------------------------------------------------------------------===// // 15.1. Vector Single-Width Integer Reduction Instructions //===----------------------------------------------------------------------===// -defm "" : VPatReductionV_VS<"int_riscv_vredsum", "PseudoVREDSUM">; -defm "" : VPatReductionV_VS<"int_riscv_vredand", "PseudoVREDAND">; -defm "" : VPatReductionV_VS<"int_riscv_vredor", "PseudoVREDOR">; -defm "" : VPatReductionV_VS<"int_riscv_vredxor", "PseudoVREDXOR">; -defm "" : VPatReductionV_VS<"int_riscv_vredminu", "PseudoVREDMINU">; -defm "" : VPatReductionV_VS<"int_riscv_vredmin", "PseudoVREDMIN">; -defm "" : VPatReductionV_VS<"int_riscv_vredmaxu", "PseudoVREDMAXU">; -defm "" : VPatReductionV_VS<"int_riscv_vredmax", "PseudoVREDMAX">; +defm : VPatReductionV_VS<"int_riscv_vredsum", "PseudoVREDSUM">; +defm : VPatReductionV_VS<"int_riscv_vredand", "PseudoVREDAND">; +defm : VPatReductionV_VS<"int_riscv_vredor", "PseudoVREDOR">; +defm : VPatReductionV_VS<"int_riscv_vredxor", "PseudoVREDXOR">; +defm : VPatReductionV_VS<"int_riscv_vredminu", "PseudoVREDMINU">; +defm : VPatReductionV_VS<"int_riscv_vredmin", "PseudoVREDMIN">; +defm : VPatReductionV_VS<"int_riscv_vredmaxu", "PseudoVREDMAXU">; +defm : VPatReductionV_VS<"int_riscv_vredmax", "PseudoVREDMAX">; //===----------------------------------------------------------------------===// // 15.2. Vector Widening Integer Reduction Instructions //===----------------------------------------------------------------------===// -defm "" : VPatReductionW_VS<"int_riscv_vwredsumu", "PseudoVWREDSUMU">; -defm "" : VPatReductionW_VS<"int_riscv_vwredsum", "PseudoVWREDSUM">; +defm : VPatReductionW_VS<"int_riscv_vwredsumu", "PseudoVWREDSUMU">; +defm : VPatReductionW_VS<"int_riscv_vwredsum", "PseudoVWREDSUM">; } // Predicates = [HasStdExtV] let Predicates = [HasStdExtV, HasStdExtF] in { //===----------------------------------------------------------------------===// // 15.3. Vector Single-Width Floating-Point Reduction Instructions //===----------------------------------------------------------------------===// -defm "" : VPatReductionV_VS<"int_riscv_vfredosum", "PseudoVFREDOSUM", /*IsFloat=*/1>; -defm "" : VPatReductionV_VS<"int_riscv_vfredsum", "PseudoVFREDSUM", /*IsFloat=*/1>; -defm "" : VPatReductionV_VS<"int_riscv_vfredmin", "PseudoVFREDMIN", /*IsFloat=*/1>; -defm "" : VPatReductionV_VS<"int_riscv_vfredmax", "PseudoVFREDMAX", /*IsFloat=*/1>; +defm : VPatReductionV_VS<"int_riscv_vfredosum", "PseudoVFREDOSUM", /*IsFloat=*/1>; +defm : VPatReductionV_VS<"int_riscv_vfredsum", "PseudoVFREDSUM", /*IsFloat=*/1>; +defm : VPatReductionV_VS<"int_riscv_vfredmin", "PseudoVFREDMIN", /*IsFloat=*/1>; +defm : VPatReductionV_VS<"int_riscv_vfredmax", "PseudoVFREDMAX", /*IsFloat=*/1>; //===----------------------------------------------------------------------===// // 15.4. Vector Widening Floating-Point Reduction Instructions //===----------------------------------------------------------------------===// -defm "" : VPatReductionW_VS<"int_riscv_vfwredsum", "PseudoVFWREDSUM", /*IsFloat=*/1>; -defm "" : VPatReductionW_VS<"int_riscv_vfwredosum", "PseudoVFWREDOSUM", /*IsFloat=*/1>; +defm : VPatReductionW_VS<"int_riscv_vfwredsum", "PseudoVFWREDSUM", /*IsFloat=*/1>; +defm : VPatReductionW_VS<"int_riscv_vfwredosum", "PseudoVFWREDOSUM", /*IsFloat=*/1>; } // Predicates = [HasStdExtV, HasStdExtF] @@ -4258,53 +4470,53 @@ let Predicates = [HasStdExtV] in { //===----------------------------------------------------------------------===// // 16.1 Vector Mask-Register Logical Instructions //===----------------------------------------------------------------------===// -defm "" : VPatBinaryM_MM<"int_riscv_vmand", "PseudoVMAND">; -defm "" : VPatBinaryM_MM<"int_riscv_vmnand", "PseudoVMNAND">; -defm "" : VPatBinaryM_MM<"int_riscv_vmandnot", "PseudoVMANDNOT">; -defm "" : VPatBinaryM_MM<"int_riscv_vmxor", "PseudoVMXOR">; -defm "" : VPatBinaryM_MM<"int_riscv_vmor", "PseudoVMOR">; -defm "" : VPatBinaryM_MM<"int_riscv_vmnor", "PseudoVMNOR">; -defm "" : VPatBinaryM_MM<"int_riscv_vmornot", "PseudoVMORNOT">; -defm "" : VPatBinaryM_MM<"int_riscv_vmxnor", "PseudoVMXNOR">; +defm : VPatBinaryM_MM<"int_riscv_vmand", "PseudoVMAND">; +defm : VPatBinaryM_MM<"int_riscv_vmnand", "PseudoVMNAND">; +defm : VPatBinaryM_MM<"int_riscv_vmandnot", "PseudoVMANDNOT">; +defm : VPatBinaryM_MM<"int_riscv_vmxor", "PseudoVMXOR">; +defm : VPatBinaryM_MM<"int_riscv_vmor", "PseudoVMOR">; +defm : VPatBinaryM_MM<"int_riscv_vmnor", "PseudoVMNOR">; +defm : VPatBinaryM_MM<"int_riscv_vmornot", "PseudoVMORNOT">; +defm : VPatBinaryM_MM<"int_riscv_vmxnor", "PseudoVMXNOR">; // pseudo instructions -defm "" : VPatNullaryM<"int_riscv_vmclr", "PseudoVMCLR">; -defm "" : VPatNullaryM<"int_riscv_vmset", "PseudoVMSET">; +defm : VPatNullaryM<"int_riscv_vmclr", "PseudoVMCLR">; +defm : VPatNullaryM<"int_riscv_vmset", "PseudoVMSET">; //===----------------------------------------------------------------------===// // 16.2. Vector mask population count vpopc //===----------------------------------------------------------------------===// -defm "" : VPatUnaryS_M<"int_riscv_vpopc", "PseudoVPOPC">; +defm : VPatUnaryS_M<"int_riscv_vpopc", "PseudoVPOPC">; //===----------------------------------------------------------------------===// // 16.3. vfirst find-first-set mask bit //===----------------------------------------------------------------------===// -defm "" : VPatUnaryS_M<"int_riscv_vfirst", "PseudoVFIRST">; +defm : VPatUnaryS_M<"int_riscv_vfirst", "PseudoVFIRST">; //===----------------------------------------------------------------------===// // 16.4. vmsbf.m set-before-first mask bit //===----------------------------------------------------------------------===// -defm "" : VPatUnaryM_M<"int_riscv_vmsbf", "PseudoVMSBF">; +defm : VPatUnaryM_M<"int_riscv_vmsbf", "PseudoVMSBF">; //===----------------------------------------------------------------------===// // 16.5. vmsif.m set-including-first mask bit //===----------------------------------------------------------------------===// -defm "" : VPatUnaryM_M<"int_riscv_vmsif", "PseudoVMSIF">; +defm : VPatUnaryM_M<"int_riscv_vmsif", "PseudoVMSIF">; //===----------------------------------------------------------------------===// // 16.6. vmsof.m set-only-first mask bit //===----------------------------------------------------------------------===// -defm "" : VPatUnaryM_M<"int_riscv_vmsof", "PseudoVMSOF">; +defm : VPatUnaryM_M<"int_riscv_vmsof", "PseudoVMSOF">; //===----------------------------------------------------------------------===// // 16.8. Vector Iota Instruction //===----------------------------------------------------------------------===// -defm "" : VPatUnaryV_M<"int_riscv_viota", "PseudoVIOTA">; +defm : VPatUnaryV_M<"int_riscv_viota", "PseudoVIOTA">; //===----------------------------------------------------------------------===// // 16.9. Vector Element Index Instruction //===----------------------------------------------------------------------===// -defm "" : VPatNullaryV<"int_riscv_vid", "PseudoVID">; +defm : VPatNullaryV<"int_riscv_vid", "PseudoVID">; } // Predicates = [HasStdExtV] @@ -4319,11 +4531,8 @@ defm "" : VPatNullaryV<"int_riscv_vid", "PseudoVID">; let Predicates = [HasStdExtV] in { foreach vti = AllIntegerVectors in { def : Pat<(riscv_vmv_x_s (vti.Vector vti.RegClass:$rs2)), - (!cast<Instruction>("PseudoVMV_X_S_" # vti.LMul.MX) $rs2, vti.SEW)>; - def : Pat<(vti.Vector (int_riscv_vmv_s_x (vti.Vector vti.RegClass:$rs1), - GPR:$rs2, GPR:$vl)), - (!cast<Instruction>("PseudoVMV_S_X_" # vti.LMul.MX) - (vti.Vector $rs1), $rs2, (NoX0 GPR:$vl), vti.SEW)>; + (!cast<Instruction>("PseudoVMV_X_S_" # vti.LMul.MX) $rs2, vti.Log2SEW)>; + // vmv.s.x is handled with a custom node in RISCVInstrInfoVVLPatterns.td } } // Predicates = [HasStdExtV] @@ -4336,15 +4545,15 @@ foreach fvti = AllFloatVectors in { defvar instr = !cast<Instruction>("PseudoVFMV_"#fvti.ScalarSuffix#"_S_" # fvti.LMul.MX); def : Pat<(fvti.Scalar (int_riscv_vfmv_f_s (fvti.Vector fvti.RegClass:$rs2))), - (instr $rs2, fvti.SEW)>; + (instr $rs2, fvti.Log2SEW)>; def : Pat<(fvti.Vector (int_riscv_vfmv_s_f (fvti.Vector fvti.RegClass:$rs1), - (fvti.Scalar fvti.ScalarRegClass:$rs2), GPR:$vl)), + (fvti.Scalar fvti.ScalarRegClass:$rs2), VLOpFrag)), (!cast<Instruction>("PseudoVFMV_S_"#fvti.ScalarSuffix#"_" # fvti.LMul.MX) (fvti.Vector $rs1), (fvti.Scalar fvti.ScalarRegClass:$rs2), - (NoX0 GPR:$vl), fvti.SEW)>; + GPR:$vl, fvti.Log2SEW)>; } } // Predicates = [HasStdExtV, HasStdExtF] @@ -4352,46 +4561,47 @@ foreach fvti = AllFloatVectors in { // 17.3. Vector Slide Instructions //===----------------------------------------------------------------------===// let Predicates = [HasStdExtV] in { - defm "" : VPatTernaryV_VX_VI<"int_riscv_vslideup", "PseudoVSLIDEUP", AllIntegerVectors, uimm5>; - defm "" : VPatTernaryV_VX_VI<"int_riscv_vslidedown", "PseudoVSLIDEDOWN", AllIntegerVectors, uimm5>; - defm "" : VPatBinaryV_VX<"int_riscv_vslide1up", "PseudoVSLIDE1UP", AllIntegerVectors>; - defm "" : VPatBinaryV_VX<"int_riscv_vslide1down", "PseudoVSLIDE1DOWN", AllIntegerVectors>; + defm : VPatTernaryV_VX_VI<"int_riscv_vslideup", "PseudoVSLIDEUP", AllIntegerVectors, uimm5>; + defm : VPatTernaryV_VX_VI<"int_riscv_vslidedown", "PseudoVSLIDEDOWN", AllIntegerVectors, uimm5>; + defm : VPatBinaryV_VX<"int_riscv_vslide1up", "PseudoVSLIDE1UP", AllIntegerVectors>; + defm : VPatBinaryV_VX<"int_riscv_vslide1down", "PseudoVSLIDE1DOWN", AllIntegerVectors>; } // Predicates = [HasStdExtV] let Predicates = [HasStdExtV, HasStdExtF] in { - defm "" : VPatTernaryV_VX_VI<"int_riscv_vslideup", "PseudoVSLIDEUP", AllFloatVectors, uimm5>; - defm "" : VPatTernaryV_VX_VI<"int_riscv_vslidedown", "PseudoVSLIDEDOWN", AllFloatVectors, uimm5>; - defm "" : VPatBinaryV_VX<"int_riscv_vfslide1up", "PseudoVFSLIDE1UP", AllFloatVectors>; - defm "" : VPatBinaryV_VX<"int_riscv_vfslide1down", "PseudoVFSLIDE1DOWN", AllFloatVectors>; + defm : VPatTernaryV_VX_VI<"int_riscv_vslideup", "PseudoVSLIDEUP", AllFloatVectors, uimm5>; + defm : VPatTernaryV_VX_VI<"int_riscv_vslidedown", "PseudoVSLIDEDOWN", AllFloatVectors, uimm5>; + defm : VPatBinaryV_VX<"int_riscv_vfslide1up", "PseudoVFSLIDE1UP", AllFloatVectors>; + defm : VPatBinaryV_VX<"int_riscv_vfslide1down", "PseudoVFSLIDE1DOWN", AllFloatVectors>; } // Predicates = [HasStdExtV, HasStdExtF] //===----------------------------------------------------------------------===// // 17.4. Vector Register Gather Instructions //===----------------------------------------------------------------------===// let Predicates = [HasStdExtV] in { - defm "" : VPatBinaryV_VV_VX_VI_INT<"int_riscv_vrgather", "PseudoVRGATHER", - AllIntegerVectors, uimm5>; - defm "" : VPatBinaryV_VV_INT_EEW<"int_riscv_vrgatherei16", "PseudoVRGATHEREI16", - /* eew */ 16, AllIntegerVectors>; + defm : VPatBinaryV_VV_VX_VI_INT<"int_riscv_vrgather", "PseudoVRGATHER", + AllIntegerVectors, uimm5>; + defm : VPatBinaryV_VV_INT_EEW<"int_riscv_vrgatherei16_vv", "PseudoVRGATHEREI16", + /* eew */ 16, AllIntegerVectors>; } // Predicates = [HasStdExtV] let Predicates = [HasStdExtV, HasStdExtF] in { - defm "" : VPatBinaryV_VV_VX_VI_INT<"int_riscv_vrgather", "PseudoVRGATHER", - AllFloatVectors, uimm5>; - defm "" : VPatBinaryV_VV_INT_EEW<"int_riscv_vrgatherei16", "PseudoVRGATHEREI16", - /* eew */ 16, AllFloatVectors>; + defm : VPatBinaryV_VV_VX_VI_INT<"int_riscv_vrgather", "PseudoVRGATHER", + AllFloatVectors, uimm5>; + defm : VPatBinaryV_VV_INT_EEW<"int_riscv_vrgatherei16_vv", "PseudoVRGATHEREI16", + /* eew */ 16, AllFloatVectors>; } // Predicates = [HasStdExtV, HasStdExtF] //===----------------------------------------------------------------------===// // 17.5. Vector Compress Instruction //===----------------------------------------------------------------------===// let Predicates = [HasStdExtV] in { - defm "" : VPatUnaryV_V_AnyMask<"int_riscv_vcompress", "PseudoVCOMPRESS", AllIntegerVectors>; + defm : VPatUnaryV_V_AnyMask<"int_riscv_vcompress", "PseudoVCOMPRESS", AllIntegerVectors>; } // Predicates = [HasStdExtV] let Predicates = [HasStdExtV, HasStdExtF] in { - defm "" : VPatUnaryV_V_AnyMask<"int_riscv_vcompress", "PseudoVCOMPRESS", AllFloatVectors>; + defm : VPatUnaryV_V_AnyMask<"int_riscv_vcompress", "PseudoVCOMPRESS", AllFloatVectors>; } // Predicates = [HasStdExtV, HasStdExtF] // Include the non-intrinsic ISel patterns include "RISCVInstrInfoVSDPatterns.td" +include "RISCVInstrInfoVVLPatterns.td" |