diff options
Diffstat (limited to 'contrib/llvm-project/clang/include/clang/Basic/riscv_vector.td')
| -rw-r--r-- | contrib/llvm-project/clang/include/clang/Basic/riscv_vector.td | 2612 |
1 files changed, 2612 insertions, 0 deletions
diff --git a/contrib/llvm-project/clang/include/clang/Basic/riscv_vector.td b/contrib/llvm-project/clang/include/clang/Basic/riscv_vector.td new file mode 100644 index 000000000000..8bde08105250 --- /dev/null +++ b/contrib/llvm-project/clang/include/clang/Basic/riscv_vector.td @@ -0,0 +1,2612 @@ +//==--- riscv_vector.td - RISC-V V-ext Builtin function list --------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines the builtins for RISC-V V-extension. See: +// +// https://github.com/riscv/rvv-intrinsic-doc +// +//===----------------------------------------------------------------------===// + +include "riscv_vector_common.td" + +defvar TypeList = ["c","s","i","l","x","f","d"]; +defvar EEWList = [["8", "(Log2EEW:3)"], + ["16", "(Log2EEW:4)"], + ["32", "(Log2EEW:5)"], + ["64", "(Log2EEW:6)"]]; + +class IsFloat<string type> { + bit val = !or(!eq(type, "x"), !eq(type, "f"), !eq(type, "d")); +} + +let SupportOverloading = false, + MaskedPolicyScheme = NonePolicy in { + class RVVVLEMaskBuiltin : RVVOutBuiltin<"m", "mPCUe", "c"> { + let Name = "vlm_v"; + let IRName = "vlm"; + let HasMasked = false; + } +} + +let SupportOverloading = false, + UnMaskedPolicyScheme = HasPassthruOperand in { + multiclass RVVVLEBuiltin<list<string> types> { + let Name = NAME # "_v", + IRName = "vle", + MaskedIRName ="vle_mask" in { + foreach type = types in { + def : RVVOutBuiltin<"v", "vPCe", type>; + if !not(IsFloat<type>.val) then { + def : RVVOutBuiltin<"Uv", "UvPCUe", type>; + } + } + } + } +} + +multiclass RVVVLEFFBuiltin<list<string> types> { + let Name = NAME # "_v", + IRName = "vleff", + MaskedIRName = "vleff_mask", + SupportOverloading = false, + UnMaskedPolicyScheme = HasPassthruOperand, + ManualCodegen = [{ + { + if (IsMasked) { + // Move mask to right before vl. + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); + if ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); + IntrinsicTypes = {ResultType, Ops[4]->getType()}; + } else { + if (PolicyAttrs & RVV_VTA) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + IntrinsicTypes = {ResultType, Ops[3]->getType()}; + } + Value *NewVL = Ops[2]; + Ops.erase(Ops.begin() + 2); + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); + llvm::Value *LoadValue = Builder.CreateCall(F, Ops, ""); + llvm::Value *V = Builder.CreateExtractValue(LoadValue, {0}); + // Store new_vl. + clang::CharUnits Align; + if (IsMasked) + Align = CGM.getNaturalPointeeTypeAlignment(E->getArg(E->getNumArgs()-2)->getType()); + else + Align = CGM.getNaturalPointeeTypeAlignment(E->getArg(1)->getType()); + llvm::Value *Val = Builder.CreateExtractValue(LoadValue, {1}); + Builder.CreateStore(Val, Address(NewVL, Val->getType(), Align)); + return V; + } + }] in { + foreach type = types in { + def : RVVBuiltin<"v", "vPCePz", type>; + // Skip floating types for unsigned versions. + if !not(IsFloat<type>.val) then { + def : RVVBuiltin<"Uv", "UvPCUePz", type>; + } + } + } +} + +multiclass RVVVLSEBuiltin<list<string> types> { + let Name = NAME # "_v", + IRName = "vlse", + MaskedIRName ="vlse_mask", + SupportOverloading = false, + UnMaskedPolicyScheme = HasPassthruOperand in { + foreach type = types in { + def : RVVOutBuiltin<"v", "vPCet", type>; + if !not(IsFloat<type>.val) then { + def : RVVOutBuiltin<"Uv", "UvPCUet", type>; + } + } + } +} + +multiclass RVVIndexedLoad<string op> { + let UnMaskedPolicyScheme = HasPassthruOperand in { + foreach type = TypeList in { + foreach eew_list = EEWList[0-2] in { + defvar eew = eew_list[0]; + defvar eew_type = eew_list[1]; + let Name = op # eew # "_v", IRName = op, MaskedIRName = op # "_mask", + RequiredFeatures = !if(!eq(type, "x"), ["Zvfhmin"], + []<string>) in { + def: RVVOutOp1Builtin<"v", "vPCe" # eew_type # "Uv", type>; + if !not(IsFloat<type>.val) then { + def: RVVOutOp1Builtin<"Uv", "UvPCUe" # eew_type # "Uv", type>; + } + } + } + defvar eew64 = "64"; + defvar eew64_type = "(Log2EEW:6)"; + let Name = op # eew64 # "_v", IRName = op, MaskedIRName = op # "_mask", + RequiredFeatures = !if(!eq(type, "x"), ["Zvfhmin", "RV64"], + ["RV64"]) in { + def: RVVOutOp1Builtin<"v", "vPCe" # eew64_type # "Uv", type>; + if !not(IsFloat<type>.val) then { + def: RVVOutOp1Builtin<"Uv", "UvPCUe" # eew64_type # "Uv", type>; + } + } + } + } +} + +let HasMaskedOffOperand = false, + MaskedPolicyScheme = NonePolicy, + ManualCodegen = [{ + if (IsMasked) { + // Builtin: (mask, ptr, value, vl). Intrinsic: (value, ptr, mask, vl) + std::swap(Ops[0], Ops[2]); + } else { + // Builtin: (ptr, value, vl). Intrinsic: (value, ptr, vl) + std::swap(Ops[0], Ops[1]); + } + if (IsMasked) + IntrinsicTypes = {Ops[0]->getType(), Ops[3]->getType()}; + else + IntrinsicTypes = {Ops[0]->getType(), Ops[2]->getType()}; + }] in { + class RVVVSEMaskBuiltin : RVVBuiltin<"m", "0PUem", "c"> { + let Name = "vsm_v"; + let IRName = "vsm"; + let HasMasked = false; + } + multiclass RVVVSEBuiltin<list<string> types> { + let Name = NAME # "_v", + IRName = "vse", + MaskedIRName = "vse_mask" in { + foreach type = types in { + def : RVVBuiltin<"v", "0Pev", type>; + if !not(IsFloat<type>.val) then { + def : RVVBuiltin<"Uv", "0PUeUv", type>; + } + } + } + } +} + +multiclass RVVVSSEBuiltin<list<string> types> { + let Name = NAME # "_v", + IRName = "vsse", + MaskedIRName = "vsse_mask", + HasMaskedOffOperand = false, + MaskedPolicyScheme = NonePolicy, + ManualCodegen = [{ + if (IsMasked) { + // Builtin: (mask, ptr, stride, value, vl). Intrinsic: (value, ptr, stride, mask, vl) + std::swap(Ops[0], Ops[3]); + } else { + // Builtin: (ptr, stride, value, vl). Intrinsic: (value, ptr, stride, vl) + std::rotate(Ops.begin(), Ops.begin() + 2, Ops.begin() + 3); + } + if (IsMasked) + IntrinsicTypes = {Ops[0]->getType(), Ops[4]->getType()}; + else + IntrinsicTypes = {Ops[0]->getType(), Ops[3]->getType()}; + }] in { + foreach type = types in { + def : RVVBuiltin<"v", "0Petv", type>; + if !not(IsFloat<type>.val) then { + def : RVVBuiltin<"Uv", "0PUetUv", type>; + } + } + } +} + +multiclass RVVIndexedStore<string op> { + let HasMaskedOffOperand = false, + MaskedPolicyScheme = NonePolicy, + ManualCodegen = [{ + if (IsMasked) { + // Builtin: (mask, ptr, index, value, vl). Intrinsic: (value, ptr, index, mask, vl) + std::swap(Ops[0], Ops[3]); + } else { + // Builtin: (ptr, index, value, vl). Intrinsic: (value, ptr, index, vl) + std::rotate(Ops.begin(), Ops.begin() + 2, Ops.begin() + 3); + } + if (IsMasked) + IntrinsicTypes = {Ops[0]->getType(), Ops[2]->getType(), Ops[4]->getType()}; + else + IntrinsicTypes = {Ops[0]->getType(), Ops[2]->getType(), Ops[3]->getType()}; + }] in { + foreach type = TypeList in { + foreach eew_list = EEWList[0-2] in { + defvar eew = eew_list[0]; + defvar eew_type = eew_list[1]; + let Name = op # eew # "_v", IRName = op, MaskedIRName = op # "_mask", + RequiredFeatures = !if(!eq(type, "x"), ["Zvfhmin"], + []<string>) in { + def : RVVBuiltin<"v", "0Pe" # eew_type # "Uvv", type>; + if !not(IsFloat<type>.val) then { + def : RVVBuiltin<"Uv", "0PUe" # eew_type # "UvUv", type>; + } + } + } + defvar eew64 = "64"; + defvar eew64_type = "(Log2EEW:6)"; + let Name = op # eew64 # "_v", IRName = op, MaskedIRName = op # "_mask", + RequiredFeatures = !if(!eq(type, "x"), ["Zvfhmin", "RV64"], + ["RV64"]) in { + def : RVVBuiltin<"v", "0Pe" # eew64_type # "Uvv", type>; + if !not(IsFloat<type>.val) then { + def : RVVBuiltin<"Uv", "0PUe" # eew64_type # "UvUv", type>; + } + } + } + } +} + +defvar NFList = [2, 3, 4, 5, 6, 7, 8]; +/* +A segment load builtin has different variants. + +Therefore a segment unit-stride load builtin can have 4 variants, +1. When unmasked and the policies are all specified as agnostic: +(Address0, ..., Address{NF - 1}, Ptr, VL) +2. When masked and the policies are all specified as agnostic: +(Address0, ..., Address{NF - 1}, Mask, Ptr, VL) +3. When unmasked and one of the policies is specified as undisturbed: +(Address0, ..., Address{NF - 1}, Maskedoff0, ..., Maskedoff{NF - 1}, + Ptr, VL) +4. When masked and one of the policies is specified as undisturbed: +(Address0, ..., Address{NF - 1}, Mask, Maskedoff0, ..., Maskedoff{NF - 1}, + Ptr, VL) + +Other variants of segment load builtin share the same structure, but they +have their own extra parameter. + +The segment unit-stride fault-only-first load builtin has a 'NewVL' +operand after the 'Ptr' operand. +1. When unmasked and the policies are all specified as agnostic: +(Address0, ..., Address{NF - 1}, Ptr, NewVL, VL) +2. When masked and the policies are all specified as agnostic: +(Address0, ..., Address{NF - 1}, Mask, Ptr, NewVL, VL) +3. When unmasked and one of the policies is specified as undisturbed: +(Address0, ..., Address{NF - 1}, Maskedoff0, ..., Maskedoff{NF - 1}, + Ptr, NewVL, VL) +4. When masked and one of the policies is specified as undisturbed: +(Address0, ..., Address{NF - 1}, Mask, Maskedoff0, ..., Maskedoff{NF - 1}, + Ptr, NewVL, VL) + +The segment strided load builtin has a 'Stride' operand after the 'Ptr' +operand. +1. When unmasked and the policies are all specified as agnostic: +(Address0, ..., Address{NF - 1}, Ptr, Stride, VL) +2. When masked and the policies are all specified as agnostic: +(Address0, ..., Address{NF - 1}, Mask, Ptr, Stride, VL) +3. When unmasked and one of the policies is specified as undisturbed: +(Address0, ..., Address{NF - 1}, Maskedoff0, ..., Maskedoff{NF - 1}, + Ptr, Stride, VL) +4. When masked and one of the policies is specified as undisturbed: +(Address0, ..., Address{NF - 1}, Mask, Maskedoff0, ..., Maskedoff{NF - 1}, + Ptr, Stride, VL) + +The segment indexed load builtin has a 'Idx' operand after the 'Ptr' operand. +1. When unmasked and the policies are all specified as agnostic: +(Address0, ..., Address{NF - 1}, Ptr, Idx, VL) +2. When masked and the policies are all specified as agnostic: +(Address0, ..., Address{NF - 1}, Mask, Ptr, Idx, VL) +3. When unmasked and one of the policies is specified as undisturbed: +(Address0, ..., Address{NF - 1}, Maskedoff0, ..., Maskedoff{NF - 1}, + Ptr, Idx, VL) +4. When masked and one of the policies is specified as undisturbed: +(Address0, ..., Address{NF - 1}, Mask, Maskedoff0, ..., Maskedoff{NF - 1}, + Ptr, Idx, VL) + +Segment load intrinsics has different variants similar to their builtins. + +Segment unit-stride load intrinsic, + Masked: (Vector0, ..., Vector{NF - 1}, Ptr, Mask, VL, Policy) + Unmasked: (Vector0, ..., Vector{NF - 1}, Ptr, VL) +Segment unit-stride fault-only-first load intrinsic, + Masked: (Vector0, ..., Vector{NF - 1}, Ptr, Mask, VL, Policy) + Unmasked: (Vector0, ..., Vector{NF - 1}, Ptr, VL) +Segment strided load intrinsic, + Masked: (Vector0, ..., Vector{NF - 1}, Ptr, Stride, Mask, VL, Policy) + Unmasked: (Vector0, ..., Vector{NF - 1}, Ptr, Stride, VL) +Segment indexed load intrinsic, + Masked: (Vector0, ..., Vector{NF - 1}, Ptr, Index, Mask, VL, Policy) + Unmasked: (Vector0, ..., Vector{NF - 1}, Ptr, Index, VL) + +The Vector(s) is poison when the policy behavior allows us to not care +about any masked-off elements. +*/ + +class PVString<int nf, bit signed> { + string S = + !cond(!eq(nf, 2): !if(signed, "PvPv", "PUvPUv"), + !eq(nf, 3): !if(signed, "PvPvPv", "PUvPUvPUv"), + !eq(nf, 4): !if(signed, "PvPvPvPv", "PUvPUvPUvPUv"), + !eq(nf, 5): !if(signed, "PvPvPvPvPv", "PUvPUvPUvPUvPUv"), + !eq(nf, 6): !if(signed, "PvPvPvPvPvPv", "PUvPUvPUvPUvPUvPUv"), + !eq(nf, 7): !if(signed, "PvPvPvPvPvPvPv", "PUvPUvPUvPUvPUvPUvPUv"), + !eq(nf, 8): !if(signed, "PvPvPvPvPvPvPvPv", "PUvPUvPUvPUvPUvPUvPUvPUv")); +} + +class VString<int nf, bit signed> { + string S = !cond(!eq(nf, 2): !if(signed, "vv", "UvUv"), + !eq(nf, 3): !if(signed, "vvv", "UvUvUv"), + !eq(nf, 4): !if(signed, "vvvv", "UvUvUvUv"), + !eq(nf, 5): !if(signed, "vvvvv", "UvUvUvUvUv"), + !eq(nf, 6): !if(signed, "vvvvvv", "UvUvUvUvUvUv"), + !eq(nf, 7): !if(signed, "vvvvvvv", "UvUvUvUvUvUvUv"), + !eq(nf, 8): !if(signed, "vvvvvvvv", "UvUvUvUvUvUvUvUv")); +} + + +class FixedVString<int fixed_lmul, int num, string vec> { + string V = "(LFixedLog2LMUL:" # fixed_lmul # ")" # vec; + string S = !interleave(!listsplat(V, num), ""); +} + +multiclass RVVNonTupleVCreateBuiltin<int dst_lmul, list<int> src_lmul_list> { + defvar dst_v = FixedVString<dst_lmul, 1, "v">.V; + defvar dst_uv = FixedVString<dst_lmul, 1, "Uv">.V; + foreach src_lmul = src_lmul_list in { + defvar num = !shl(1, !sub(dst_lmul, src_lmul)); + + defvar src_v = FixedVString<src_lmul, num, "v">.V; + defvar src_s = FixedVString<src_lmul, num, "v">.S; + def vcreate # src_v # dst_v : RVVBuiltin<src_v # dst_v, + dst_v # src_s, + "csilxfd", dst_v>; + + defvar src_uv = FixedVString<src_lmul, num, "Uv">.V; + defvar src_us = FixedVString<src_lmul, num, "Uv">.S; + def vcreate_u # src_uv # dst_uv : RVVBuiltin<src_uv # dst_uv, + dst_uv # src_us, + "csil", dst_uv>; + } +} + +multiclass RVVPseudoUnaryBuiltin<string IR, string type_range> { + let Name = NAME, + IRName = IR, + MaskedIRName = IR # "_mask", + UnMaskedPolicyScheme = HasPassthruOperand, + ManualCodegen = [{ + { + if (IsMasked) { + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); + if ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + } else { + if (PolicyAttrs & RVV_VTA) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + } + auto ElemTy = cast<llvm::VectorType>(ResultType)->getElementType(); + Ops.insert(Ops.begin() + 2, llvm::Constant::getNullValue(ElemTy)); + + if (IsMasked) { + Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); + // maskedoff, op1, op2, mask, vl, policy + IntrinsicTypes = {ResultType, ElemTy, Ops[4]->getType()}; + } else { + // passthru, op1, op2, vl + IntrinsicTypes = {ResultType, ElemTy, Ops[3]->getType()}; + } + break; + } + }] in { + def : RVVBuiltin<"v", "vv", type_range>; + } +} + +multiclass RVVPseudoVNotBuiltin<string IR, string type_range> { + let Name = NAME, + IRName = IR, + MaskedIRName = IR # "_mask", + UnMaskedPolicyScheme = HasPassthruOperand, + ManualCodegen = [{ + { + if (IsMasked) { + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); + if ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + } else { + if (PolicyAttrs & RVV_VTA) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + } + auto ElemTy = cast<llvm::VectorType>(ResultType)->getElementType(); + Ops.insert(Ops.begin() + 2, + llvm::Constant::getAllOnesValue(ElemTy)); + if (IsMasked) { + Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); + // maskedoff, op1, po2, mask, vl, policy + IntrinsicTypes = {ResultType, + ElemTy, + Ops[4]->getType()}; + } else { + // passthru, op1, op2, vl + IntrinsicTypes = {ResultType, + ElemTy, + Ops[3]->getType()}; + } + break; + } + }] in { + def : RVVBuiltin<"v", "vv", type_range>; + def : RVVBuiltin<"Uv", "UvUv", type_range>; + } +} + +multiclass RVVPseudoMaskBuiltin<string IR, string type_range> { + let Name = NAME, + IRName = IR, + HasMasked = false, + ManualCodegen = [{ + { + // op1, vl + IntrinsicTypes = {ResultType, + Ops[1]->getType()}; + Ops.insert(Ops.begin() + 1, Ops[0]); + break; + } + }] in { + def : RVVBuiltin<"m", "mm", type_range>; + } +} + +multiclass RVVPseudoVFUnaryBuiltin<string IR, string type_range> { + let Name = NAME, + IRName = IR, + MaskedIRName = IR # "_mask", + UnMaskedPolicyScheme = HasPassthruOperand, + ManualCodegen = [{ + { + if (IsMasked) { + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); + if ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + Ops.insert(Ops.begin() + 2, Ops[1]); + Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); + // maskedoff, op1, op2, mask, vl + IntrinsicTypes = {ResultType, + Ops[2]->getType(), + Ops.back()->getType()}; + } else { + if (PolicyAttrs & RVV_VTA) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + // op1, po2, vl + IntrinsicTypes = {ResultType, + Ops[1]->getType(), Ops[2]->getType()}; + Ops.insert(Ops.begin() + 2, Ops[1]); + break; + } + break; + } + }] in { + def : RVVBuiltin<"v", "vv", type_range>; + } +} + +multiclass RVVPseudoVWCVTBuiltin<string IR, string MName, string type_range, + list<list<string>> suffixes_prototypes> { + let Name = NAME, + OverloadedName = MName, + IRName = IR, + MaskedIRName = IR # "_mask", + UnMaskedPolicyScheme = HasPassthruOperand, + ManualCodegen = [{ + { + if (IsMasked) { + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); + if ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + } else { + if (PolicyAttrs & RVV_VTA) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + } + auto ElemTy = cast<llvm::VectorType>(ResultType)->getElementType(); + Ops.insert(Ops.begin() + 2, llvm::Constant::getNullValue(ElemTy)); + if (IsMasked) { + Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); + // maskedoff, op1, op2, mask, vl, policy + IntrinsicTypes = {ResultType, + Ops[1]->getType(), + ElemTy, + Ops[4]->getType()}; + } else { + // passtru, op1, op2, vl + IntrinsicTypes = {ResultType, + Ops[1]->getType(), + ElemTy, + Ops[3]->getType()}; + } + break; + } + }] in { + foreach s_p = suffixes_prototypes in { + def : RVVBuiltin<s_p[0], s_p[1], type_range>; + } + } +} + +multiclass RVVPseudoVNCVTBuiltin<string IR, string MName, string type_range, + list<list<string>> suffixes_prototypes> { + let Name = NAME, + OverloadedName = MName, + IRName = IR, + MaskedIRName = IR # "_mask", + UnMaskedPolicyScheme = HasPassthruOperand, + ManualCodegen = [{ + { + if (IsMasked) { + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); + if ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + } else { + if (PolicyAttrs & RVV_VTA) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + } + Ops.insert(Ops.begin() + 2, llvm::Constant::getNullValue(Ops.back()->getType())); + if (IsMasked) { + Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); + // maskedoff, op1, xlen, mask, vl + IntrinsicTypes = {ResultType, + Ops[1]->getType(), + Ops[4]->getType(), + Ops[4]->getType()}; + } else { + // passthru, op1, xlen, vl + IntrinsicTypes = {ResultType, + Ops[1]->getType(), + Ops[3]->getType(), + Ops[3]->getType()}; + } + break; + } + }] in { + foreach s_p = suffixes_prototypes in { + def : RVVBuiltin<s_p[0], s_p[1], type_range>; + } + } +} + +let HeaderCode = +[{ +#define __riscv_vlenb() __builtin_rvv_vlenb() +}] in +def vlenb_macro: RVVHeader; + +let HasBuiltinAlias = false, HasVL = false, HasMasked = false, + UnMaskedPolicyScheme = NonePolicy, MaskedPolicyScheme = NonePolicy, + Log2LMUL = [0], IRName = "", + ManualCodegen = [{ + { + LLVMContext &Context = CGM.getLLVMContext(); + llvm::MDBuilder MDHelper(Context); + + llvm::Metadata *Ops[] = {llvm::MDString::get(Context, "vlenb")}; + llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops); + llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName); + llvm::Function *F = + CGM.getIntrinsic(llvm::Intrinsic::read_register, {SizeTy}); + return Builder.CreateCall(F, Metadata); + } + }] in +{ + def vlenb : RVVBuiltin<"", "u", "i">; +} + +// 6. Configuration-Setting Instructions +// 6.1. vsetvli/vsetvl instructions + +// vsetvl/vsetvlmax are a macro because they require constant integers in SEW +// and LMUL. +let HeaderCode = +[{ +#define __riscv_vsetvl_e8mf4(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 6) +#define __riscv_vsetvl_e8mf2(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 7) +#define __riscv_vsetvl_e8m1(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 0) +#define __riscv_vsetvl_e8m2(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 1) +#define __riscv_vsetvl_e8m4(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 2) +#define __riscv_vsetvl_e8m8(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 3) + +#define __riscv_vsetvl_e16mf2(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 7) +#define __riscv_vsetvl_e16m1(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 0) +#define __riscv_vsetvl_e16m2(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 1) +#define __riscv_vsetvl_e16m4(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 2) +#define __riscv_vsetvl_e16m8(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 3) + +#define __riscv_vsetvl_e32m1(avl) __builtin_rvv_vsetvli((size_t)(avl), 2, 0) +#define __riscv_vsetvl_e32m2(avl) __builtin_rvv_vsetvli((size_t)(avl), 2, 1) +#define __riscv_vsetvl_e32m4(avl) __builtin_rvv_vsetvli((size_t)(avl), 2, 2) +#define __riscv_vsetvl_e32m8(avl) __builtin_rvv_vsetvli((size_t)(avl), 2, 3) + +#if __riscv_v_elen >= 64 +#define __riscv_vsetvl_e8mf8(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 5) +#define __riscv_vsetvl_e16mf4(avl) __builtin_rvv_vsetvli((size_t)(avl), 1, 6) +#define __riscv_vsetvl_e32mf2(avl) __builtin_rvv_vsetvli((size_t)(avl), 2, 7) + +#define __riscv_vsetvl_e64m1(avl) __builtin_rvv_vsetvli((size_t)(avl), 3, 0) +#define __riscv_vsetvl_e64m2(avl) __builtin_rvv_vsetvli((size_t)(avl), 3, 1) +#define __riscv_vsetvl_e64m4(avl) __builtin_rvv_vsetvli((size_t)(avl), 3, 2) +#define __riscv_vsetvl_e64m8(avl) __builtin_rvv_vsetvli((size_t)(avl), 3, 3) +#endif + +#define __riscv_vsetvlmax_e8mf4() __builtin_rvv_vsetvlimax(0, 6) +#define __riscv_vsetvlmax_e8mf2() __builtin_rvv_vsetvlimax(0, 7) +#define __riscv_vsetvlmax_e8m1() __builtin_rvv_vsetvlimax(0, 0) +#define __riscv_vsetvlmax_e8m2() __builtin_rvv_vsetvlimax(0, 1) +#define __riscv_vsetvlmax_e8m4() __builtin_rvv_vsetvlimax(0, 2) +#define __riscv_vsetvlmax_e8m8() __builtin_rvv_vsetvlimax(0, 3) + +#define __riscv_vsetvlmax_e16mf2() __builtin_rvv_vsetvlimax(1, 7) +#define __riscv_vsetvlmax_e16m1() __builtin_rvv_vsetvlimax(1, 0) +#define __riscv_vsetvlmax_e16m2() __builtin_rvv_vsetvlimax(1, 1) +#define __riscv_vsetvlmax_e16m4() __builtin_rvv_vsetvlimax(1, 2) +#define __riscv_vsetvlmax_e16m8() __builtin_rvv_vsetvlimax(1, 3) + +#define __riscv_vsetvlmax_e32m1() __builtin_rvv_vsetvlimax(2, 0) +#define __riscv_vsetvlmax_e32m2() __builtin_rvv_vsetvlimax(2, 1) +#define __riscv_vsetvlmax_e32m4() __builtin_rvv_vsetvlimax(2, 2) +#define __riscv_vsetvlmax_e32m8() __builtin_rvv_vsetvlimax(2, 3) + +#if __riscv_v_elen >= 64 +#define __riscv_vsetvlmax_e8mf8() __builtin_rvv_vsetvlimax(0, 5) +#define __riscv_vsetvlmax_e16mf4() __builtin_rvv_vsetvlimax(1, 6) +#define __riscv_vsetvlmax_e32mf2() __builtin_rvv_vsetvlimax(2, 7) + +#define __riscv_vsetvlmax_e64m1() __builtin_rvv_vsetvlimax(3, 0) +#define __riscv_vsetvlmax_e64m2() __builtin_rvv_vsetvlimax(3, 1) +#define __riscv_vsetvlmax_e64m4() __builtin_rvv_vsetvlimax(3, 2) +#define __riscv_vsetvlmax_e64m8() __builtin_rvv_vsetvlimax(3, 3) +#endif + +}] in +def vsetvl_macro: RVVHeader; + +let HasBuiltinAlias = false, + HasVL = false, + HasMasked = false, + MaskedPolicyScheme = NonePolicy, + Log2LMUL = [0], + ManualCodegen = [{IntrinsicTypes = {ResultType};}] in // Set XLEN type +{ + def vsetvli : RVVBuiltin<"", "zzKzKz", "i">; + def vsetvlimax : RVVBuiltin<"", "zKzKz", "i">; +} + +// 7. Vector Loads and Stores +// 7.4. Vector Unit-Stride Instructions +def vlm: RVVVLEMaskBuiltin; +defm vle8: RVVVLEBuiltin<["c"]>; +defm vle16: RVVVLEBuiltin<["s"]>; +let Name = "vle16_v", RequiredFeatures = ["Zvfhmin"] in + defm vle16_h: RVVVLEBuiltin<["x"]>; +defm vle32: RVVVLEBuiltin<["i","f"]>; +defm vle64: RVVVLEBuiltin<["l","d"]>; + +def vsm : RVVVSEMaskBuiltin; +defm vse8 : RVVVSEBuiltin<["c"]>; +defm vse16: RVVVSEBuiltin<["s"]>; +let Name = "vse16_v", RequiredFeatures = ["Zvfhmin"] in + defm vse16_h: RVVVSEBuiltin<["x"]>; +defm vse32: RVVVSEBuiltin<["i","f"]>; +defm vse64: RVVVSEBuiltin<["l","d"]>; + +// 7.5. Vector Strided Instructions +defm vlse8: RVVVLSEBuiltin<["c"]>; +defm vlse16: RVVVLSEBuiltin<["s"]>; +let Name = "vlse16_v", RequiredFeatures = ["Zvfhmin"] in + defm vlse16_h: RVVVLSEBuiltin<["x"]>; +defm vlse32: RVVVLSEBuiltin<["i","f"]>; +defm vlse64: RVVVLSEBuiltin<["l","d"]>; + +defm vsse8 : RVVVSSEBuiltin<["c"]>; +defm vsse16: RVVVSSEBuiltin<["s"]>; +let Name = "vsse16_v", RequiredFeatures = ["Zvfhmin"] in + defm vsse16_h: RVVVSSEBuiltin<["x"]>; +defm vsse32: RVVVSSEBuiltin<["i","f"]>; +defm vsse64: RVVVSSEBuiltin<["l","d"]>; + +// 7.6. Vector Indexed Instructions +defm : RVVIndexedLoad<"vluxei">; +defm : RVVIndexedLoad<"vloxei">; + +defm : RVVIndexedStore<"vsuxei">; +defm : RVVIndexedStore<"vsoxei">; + +// 7.7. Unit-stride Fault-Only-First Loads +defm vle8ff: RVVVLEFFBuiltin<["c"]>; +defm vle16ff: RVVVLEFFBuiltin<["s"]>; +let Name = "vle16ff_v", RequiredFeatures = ["Zvfhmin"] in + defm vle16ff: RVVVLEFFBuiltin<["x"]>; +defm vle32ff: RVVVLEFFBuiltin<["i", "f"]>; +defm vle64ff: RVVVLEFFBuiltin<["l", "d"]>; + +multiclass RVVUnitStridedSegLoadTuple<string op> { + foreach type = TypeList in { + defvar eew = !cond(!eq(type, "c") : "8", + !eq(type, "s") : "16", + !eq(type, "i") : "32", + !eq(type, "l") : "64", + !eq(type, "x") : "16", + !eq(type, "f") : "32", + !eq(type, "d") : "64"); + foreach nf = NFList in { + let Name = op # nf # "e" # eew # "_v", + IRName = op # nf, + MaskedIRName = op # nf # "_mask", + NF = nf, + RequiredFeatures = !if(!eq(type, "x"), ["Zvfhmin"], + []<string>), + ManualCodegen = [{ + { + llvm::Type *ElementVectorType = cast<StructType>(ResultType)->elements()[0]; + IntrinsicTypes = {ElementVectorType, Ops.back()->getType()}; + SmallVector<llvm::Value*, 12> Operands; + + bool NoPassthru = + (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) | + (!IsMasked && (PolicyAttrs & RVV_VTA)); + unsigned Offset = IsMasked ? NoPassthru ? 1 : 2 : NoPassthru ? 0 : 1; + + if (NoPassthru) { // Push poison into passthru + Operands.append(NF, llvm::PoisonValue::get(ElementVectorType)); + } else { // Push intrinsics operands into passthru + llvm::Value *PassthruOperand = IsMasked ? Ops[1] : Ops[0]; + for (unsigned I = 0; I < NF; ++I) + Operands.push_back(Builder.CreateExtractValue(PassthruOperand, {I})); + } + + Operands.push_back(Ops[Offset]); // Ptr + if (IsMasked) + Operands.push_back(Ops[0]); + Operands.push_back(Ops[Offset + 1]); // VL + if (IsMasked) + Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); + + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); + + llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); + if (ReturnValue.isNull()) + return LoadValue; + else + return Builder.CreateStore(LoadValue, ReturnValue.getValue()); + } + }] in { + defvar T = "(Tuple:" # nf # ")"; + def : RVVBuiltin<T # "v", T # "vPCe", type>; + if !not(IsFloat<type>.val) then { + def : RVVBuiltin<T # "Uv", T # "UvPCUe", type>; + } + } + } + } +} + +multiclass RVVUnitStridedSegStoreTuple<string op> { + foreach type = TypeList in { + defvar eew = !cond(!eq(type, "c") : "8", + !eq(type, "s") : "16", + !eq(type, "i") : "32", + !eq(type, "l") : "64", + !eq(type, "x") : "16", + !eq(type, "f") : "32", + !eq(type, "d") : "64"); + foreach nf = NFList in { + let Name = op # nf # "e" # eew # "_v", + IRName = op # nf, + MaskedIRName = op # nf # "_mask", + NF = nf, + HasMaskedOffOperand = false, + RequiredFeatures = !if(!eq(type, "x"), ["Zvfhmin"], + []<string>), + ManualCodegen = [{ + { + // Masked + // Builtin: (mask, ptr, v_tuple, vl) + // Intrinsic: (val0, val1, ..., ptr, mask, vl) + // Unmasked + // Builtin: (ptr, v_tuple, vl) + // Intrinsic: (val0, val1, ..., ptr, vl) + unsigned Offset = IsMasked ? 1 : 0; + llvm::Value *VTupleOperand = Ops[Offset + 1]; + + SmallVector<llvm::Value*, 12> Operands; + for (unsigned I = 0; I < NF; ++I) { + llvm::Value *V = Builder.CreateExtractValue(VTupleOperand, {I}); + Operands.push_back(V); + } + Operands.push_back(Ops[Offset]); // Ptr + if (IsMasked) + Operands.push_back(Ops[0]); + Operands.push_back(Ops[Offset + 2]); // VL + + IntrinsicTypes = {Operands[0]->getType(), Operands.back()->getType()}; + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); + return Builder.CreateCall(F, Operands, ""); + } + }] in { + defvar T = "(Tuple:" # nf # ")"; + def : RVVBuiltin<T # "v", "0Pe" # T # "v", type>; + if !not(IsFloat<type>.val) then { + def : RVVBuiltin<T # "Uv", "0PUe" # T # "Uv", type>; + } + } + } + } +} + +multiclass RVVUnitStridedSegLoadFFTuple<string op> { + foreach type = TypeList in { + defvar eew = !cond(!eq(type, "c") : "8", + !eq(type, "s") : "16", + !eq(type, "i") : "32", + !eq(type, "l") : "64", + !eq(type, "x") : "16", + !eq(type, "f") : "32", + !eq(type, "d") : "64"); + foreach nf = NFList in { + let Name = op # nf # "e" # eew # "ff_v", + IRName = op # nf # "ff", + MaskedIRName = op # nf # "ff_mask", + NF = nf, + RequiredFeatures = !if(!eq(type, "x"), ["Zvfhmin"], + []<string>), + ManualCodegen = [{ + { + llvm::Type *ElementVectorType = cast<StructType>(ResultType)->elements()[0]; + IntrinsicTypes = {ElementVectorType, Ops.back()->getType()}; + SmallVector<llvm::Value*, 12> Operands; + + bool NoPassthru = + (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) | + (!IsMasked && (PolicyAttrs & RVV_VTA)); + unsigned Offset = IsMasked ? NoPassthru ? 1 : 2 : NoPassthru ? 0 : 1; + + if (NoPassthru) { // Push poison into passthru + Operands.append(NF, llvm::PoisonValue::get(ElementVectorType)); + } else { // Push intrinsics operands into passthru + llvm::Value *PassthruOperand = IsMasked ? Ops[1] : Ops[0]; + for (unsigned I = 0; I < NF; ++I) + Operands.push_back(Builder.CreateExtractValue(PassthruOperand, {I})); + } + + Operands.push_back(Ops[Offset]); // Ptr + if (IsMasked) + Operands.push_back(Ops[0]); + Operands.push_back(Ops[Offset + 2]); // vl + if (IsMasked) + Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); + + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); + + llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); + // Get alignment from the new vl operand + clang::CharUnits Align = + CGM.getNaturalPointeeTypeAlignment(E->getArg(Offset + 1)->getType()); + + llvm::Value *ReturnTuple = llvm::PoisonValue::get(ResultType); + for (unsigned I = 0; I < NF; ++I) { + llvm::Value *V = Builder.CreateExtractValue(LoadValue, {I}); + ReturnTuple = Builder.CreateInsertValue(ReturnTuple, V, {I}); + } + + // Store new_vl + llvm::Value *V = Builder.CreateExtractValue(LoadValue, {NF}); + Builder.CreateStore(V, Address(Ops[Offset + 1], V->getType(), Align)); + + if (ReturnValue.isNull()) + return ReturnTuple; + else + return Builder.CreateStore(ReturnTuple, ReturnValue.getValue()); + } + }] in { + defvar T = "(Tuple:" # nf # ")"; + def : RVVBuiltin<T # "v", T # "vPCePz", type>; + if !not(IsFloat<type>.val) then { + def : RVVBuiltin<T # "Uv", T # "UvPCUePz", type>; + } + } + } + } +} + +multiclass RVVStridedSegLoadTuple<string op> { + foreach type = TypeList in { + defvar eew = !cond(!eq(type, "c") : "8", + !eq(type, "s") : "16", + !eq(type, "i") : "32", + !eq(type, "l") : "64", + !eq(type, "x") : "16", + !eq(type, "f") : "32", + !eq(type, "d") : "64"); + foreach nf = NFList in { + let Name = op # nf # "e" # eew # "_v", + IRName = op # nf, + MaskedIRName = op # nf # "_mask", + NF = nf, + RequiredFeatures = !if(!eq(type, "x"), ["Zvfhmin"], + []<string>), + ManualCodegen = [{ + { + llvm::Type *ElementVectorType = cast<StructType>(ResultType)->elements()[0]; + IntrinsicTypes = {ElementVectorType, Ops.back()->getType()}; + SmallVector<llvm::Value*, 12> Operands; + + bool NoPassthru = + (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) | + (!IsMasked && (PolicyAttrs & RVV_VTA)); + unsigned Offset = IsMasked ? NoPassthru ? 1 : 2 : NoPassthru ? 0 : 1; + + if (NoPassthru) { // Push poison into passthru + Operands.append(NF, llvm::PoisonValue::get(ElementVectorType)); + } else { // Push intrinsics operands into passthru + llvm::Value *PassthruOperand = IsMasked ? Ops[1] : Ops[0]; + for (unsigned I = 0; I < NF; ++I) + Operands.push_back(Builder.CreateExtractValue(PassthruOperand, {I})); + } + + Operands.push_back(Ops[Offset]); // Ptr + Operands.push_back(Ops[Offset + 1]); // Stride + if (IsMasked) + Operands.push_back(Ops[0]); + Operands.push_back(Ops[Offset + 2]); // VL + if (IsMasked) + Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); + + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); + llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); + + if (ReturnValue.isNull()) + return LoadValue; + else + return Builder.CreateStore(LoadValue, ReturnValue.getValue()); + } + }] in { + defvar T = "(Tuple:" # nf # ")"; + def : RVVBuiltin<T # "v", T # "vPCet", type>; + if !not(IsFloat<type>.val) then { + def : RVVBuiltin<T # "Uv", T # "UvPCUet", type>; + } + } + } + } +} + +multiclass RVVStridedSegStoreTuple<string op> { + foreach type = TypeList in { + defvar eew = !cond(!eq(type, "c") : "8", + !eq(type, "s") : "16", + !eq(type, "i") : "32", + !eq(type, "l") : "64", + !eq(type, "x") : "16", + !eq(type, "f") : "32", + !eq(type, "d") : "64"); + foreach nf = NFList in { + let Name = op # nf # "e" # eew # "_v", + IRName = op # nf, + MaskedIRName = op # nf # "_mask", + NF = nf, + HasMaskedOffOperand = false, + MaskedPolicyScheme = NonePolicy, + RequiredFeatures = !if(!eq(type, "x"), ["Zvfhmin"], + []<string>), + ManualCodegen = [{ + { + // Masked + // Builtin: (mask, ptr, stride, v_tuple, vl) + // Intrinsic: (val0, val1, ..., ptr, stride, mask, vl) + // Unmasked + // Builtin: (ptr, stride, v_tuple, vl) + // Intrinsic: (val0, val1, ..., ptr, stride, vl) + unsigned Offset = IsMasked ? 1 : 0; + llvm::Value *VTupleOperand = Ops[Offset + 2]; + + SmallVector<llvm::Value*, 12> Operands; + for (unsigned I = 0; I < NF; ++I) { + llvm::Value *V = Builder.CreateExtractValue(VTupleOperand, {I}); + Operands.push_back(V); + } + Operands.push_back(Ops[Offset]); // Ptr + Operands.push_back(Ops[Offset + 1]); // Stride + if (IsMasked) + Operands.push_back(Ops[0]); + Operands.push_back(Ops[Offset + 3]); // VL + + IntrinsicTypes = {Operands[0]->getType(), Operands.back()->getType()}; + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); + return Builder.CreateCall(F, Operands, ""); + } + }] in { + defvar T = "(Tuple:" # nf # ")"; + def : RVVBuiltin<T # "v", "0Pet" # T # "v", type>; + if !not(IsFloat<type>.val) then { + def : RVVBuiltin<T # "Uv", "0PUet" # T # "Uv", type>; + } + } + } + } +} + +multiclass RVVIndexedSegLoadTuple<string op> { + foreach type = TypeList in { + foreach eew_info = EEWList in { + defvar eew = eew_info[0]; + defvar eew_type = eew_info[1]; + foreach nf = NFList in { + let Name = op # nf # "ei" # eew # "_v", + IRName = op # nf, + MaskedIRName = op # nf # "_mask", + NF = nf, + RequiredFeatures = !if(!eq(type, "x"), ["Zvfhmin"], + []<string>), + ManualCodegen = [{ + { + llvm::Type *ElementVectorType = cast<StructType>(ResultType)->elements()[0]; + IntrinsicTypes = {ElementVectorType, Ops.back()->getType()}; + SmallVector<llvm::Value*, 12> Operands; + + bool NoPassthru = + (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) | + (!IsMasked && (PolicyAttrs & RVV_VTA)); + unsigned Offset = IsMasked ? NoPassthru ? 1 : 2 : NoPassthru ? 0 : 1; + + if (NoPassthru) { // Push poison into passthru + Operands.append(NF, llvm::PoisonValue::get(ElementVectorType)); + } else { // Push intrinsics operands into passthru + llvm::Value *PassthruOperand = IsMasked ? Ops[1] : Ops[0]; + for (unsigned I = 0; I < NF; ++I) + Operands.push_back(Builder.CreateExtractValue(PassthruOperand, {I})); + } + + Operands.push_back(Ops[Offset]); // Ptr + Operands.push_back(Ops[Offset + 1]); // Idx + if (IsMasked) + Operands.push_back(Ops[0]); + Operands.push_back(Ops[Offset + 2]); // VL + if (IsMasked) + Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); + + IntrinsicTypes = {ElementVectorType, Ops[Offset + 1]->getType(), + Ops.back()->getType()}; + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); + llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); + + if (ReturnValue.isNull()) + return LoadValue; + else + return Builder.CreateStore(LoadValue, ReturnValue.getValue()); + } + }] in { + defvar T = "(Tuple:" # nf # ")"; + def : RVVBuiltin<T # "v", T # "vPCe" # eew_type # "Uv", type>; + if !not(IsFloat<type>.val) then { + def : RVVBuiltin<T # "Uv", T # "UvPCUe" # eew_type # "Uv", type>; + } + } + } + } + } +} + +multiclass RVVIndexedSegStoreTuple<string op> { + foreach type = TypeList in { + foreach eew_info = EEWList in { + defvar eew = eew_info[0]; + defvar eew_type = eew_info[1]; + foreach nf = NFList in { + let Name = op # nf # "ei" # eew # "_v", + IRName = op # nf, + MaskedIRName = op # nf # "_mask", + NF = nf, + HasMaskedOffOperand = false, + MaskedPolicyScheme = NonePolicy, + RequiredFeatures = !if(!eq(type, "x"), ["Zvfhmin"], + []<string>), + ManualCodegen = [{ + { + // Masked + // Builtin: (mask, ptr, index, v_tuple, vl) + // Intrinsic: (val0, val1, ..., ptr, index, mask, vl) + // Unmasked + // Builtin: (ptr, index, v_tuple, vl) + // Intrinsic: (val0, val1, ..., ptr, index, vl) + unsigned Offset = IsMasked ? 1 : 0; + llvm::Value *VTupleOperand = Ops[Offset + 2]; + + SmallVector<llvm::Value*, 12> Operands; + for (unsigned I = 0; I < NF; ++I) { + llvm::Value *V = Builder.CreateExtractValue(VTupleOperand, {I}); + Operands.push_back(V); + } + Operands.push_back(Ops[Offset]); // Ptr + Operands.push_back(Ops[Offset + 1]); // Idx + if (IsMasked) + Operands.push_back(Ops[0]); + Operands.push_back(Ops[Offset + 3]); // VL + + IntrinsicTypes = {Operands[0]->getType(), Ops[Offset + 1]->getType(), + Operands.back()->getType()}; + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); + return Builder.CreateCall(F, Operands, ""); + } + }] in { + defvar T = "(Tuple:" # nf # ")"; + def : RVVBuiltin<T # "v", "0Pe" # eew_type # "Uv" # T # "v", type>; + if !not(IsFloat<type>.val) then { + def : RVVBuiltin<T # "Uv", "0PUe" # eew_type # "Uv" # T # "Uv", type>; + } + } + } + } + } +} + +// 7.8 Vector Load/Store Segment Instructions +let UnMaskedPolicyScheme = HasPassthruOperand, + IsTuple = true in { + defm : RVVUnitStridedSegLoadTuple<"vlseg">; + defm : RVVUnitStridedSegLoadFFTuple<"vlseg">; + defm : RVVStridedSegLoadTuple<"vlsseg">; + defm : RVVIndexedSegLoadTuple<"vluxseg">; + defm : RVVIndexedSegLoadTuple<"vloxseg">; +} + +let UnMaskedPolicyScheme = NonePolicy, + MaskedPolicyScheme = NonePolicy, + IsTuple = true in { +defm : RVVUnitStridedSegStoreTuple<"vsseg">; +defm : RVVStridedSegStoreTuple<"vssseg">; +defm : RVVIndexedSegStoreTuple<"vsuxseg">; +defm : RVVIndexedSegStoreTuple<"vsoxseg">; +} + +// 11. Vector Integer Arithmetic Instructions +// 11.1. Vector Single-Width Integer Add and Subtract +let UnMaskedPolicyScheme = HasPassthruOperand in { +defm vadd : RVVIntBinBuiltinSet; +defm vsub : RVVIntBinBuiltinSet; +defm vrsub : RVVOutOp1BuiltinSet<"vrsub", "csil", + [["vx", "v", "vve"], + ["vx", "Uv", "UvUvUe"]]>; +} +defm vneg_v : RVVPseudoUnaryBuiltin<"vrsub", "csil">; + +// 11.2. Vector Widening Integer Add/Subtract +// Widening unsigned integer add/subtract, 2*SEW = SEW +/- SEW +let UnMaskedPolicyScheme = HasPassthruOperand in { +defm vwaddu : RVVUnsignedWidenBinBuiltinSet; +defm vwsubu : RVVUnsignedWidenBinBuiltinSet; +// Widening signed integer add/subtract, 2*SEW = SEW +/- SEW +defm vwadd : RVVSignedWidenBinBuiltinSet; +defm vwsub : RVVSignedWidenBinBuiltinSet; +// Widening unsigned integer add/subtract, 2*SEW = 2*SEW +/- SEW +defm vwaddu : RVVUnsignedWidenOp0BinBuiltinSet; +defm vwsubu : RVVUnsignedWidenOp0BinBuiltinSet; +// Widening signed integer add/subtract, 2*SEW = 2*SEW +/- SEW +defm vwadd : RVVSignedWidenOp0BinBuiltinSet; +defm vwsub : RVVSignedWidenOp0BinBuiltinSet; +} +defm vwcvtu_x_x_v : RVVPseudoVWCVTBuiltin<"vwaddu", "vwcvtu_x", "csi", + [["Uw", "UwUv"]]>; +defm vwcvt_x_x_v : RVVPseudoVWCVTBuiltin<"vwadd", "vwcvt_x", "csi", + [["w", "wv"]]>; + +// 11.3. Vector Integer Extension +let UnMaskedPolicyScheme = HasPassthruOperand in { +let Log2LMUL = [-3, -2, -1, 0, 1, 2] in { + def vsext_vf2 : RVVIntExt<"vsext", "w", "wv", "csi">; + def vzext_vf2 : RVVIntExt<"vzext", "Uw", "UwUv", "csi">; +} +let Log2LMUL = [-3, -2, -1, 0, 1] in { + def vsext_vf4 : RVVIntExt<"vsext", "q", "qv", "cs">; + def vzext_vf4 : RVVIntExt<"vzext", "Uq", "UqUv", "cs">; +} +let Log2LMUL = [-3, -2, -1, 0] in { + def vsext_vf8 : RVVIntExt<"vsext", "o", "ov", "c">; + def vzext_vf8 : RVVIntExt<"vzext", "Uo", "UoUv", "c">; +} +} + +// 11.4. Vector Integer Add-with-Carry / Subtract-with-Borrow Instructions +let HasMasked = false, MaskedPolicyScheme = NonePolicy in { + let UnMaskedPolicyScheme = HasPassthruOperand in { + defm vadc : RVVCarryinBuiltinSet; + defm vsbc : RVVCarryinBuiltinSet; + } + defm vmadc : RVVCarryOutInBuiltinSet<"vmadc_carry_in">; + defm vmadc : RVVIntMaskOutBuiltinSet; + defm vmsbc : RVVCarryOutInBuiltinSet<"vmsbc_borrow_in">; + defm vmsbc : RVVIntMaskOutBuiltinSet; +} + +// 11.5. Vector Bitwise Logical Instructions +let UnMaskedPolicyScheme = HasPassthruOperand in { +defm vand : RVVIntBinBuiltinSet; +defm vxor : RVVIntBinBuiltinSet; +defm vor : RVVIntBinBuiltinSet; +} +defm vnot_v : RVVPseudoVNotBuiltin<"vxor", "csil">; + +// 11.6. Vector Single-Width Shift Instructions +let UnMaskedPolicyScheme = HasPassthruOperand in { +defm vsll : RVVShiftBuiltinSet; +defm vsrl : RVVUnsignedShiftBuiltinSet; +defm vsra : RVVSignedShiftBuiltinSet; + +// 11.7. Vector Narrowing Integer Right Shift Instructions +defm vnsrl : RVVUnsignedNShiftBuiltinSet; +defm vnsra : RVVSignedNShiftBuiltinSet; +} +defm vncvt_x_x_w : RVVPseudoVNCVTBuiltin<"vnsrl", "vncvt_x", "csi", + [["v", "vw"], + ["Uv", "UvUw"]]>; + +// 11.8. Vector Integer Compare Instructions +let MaskedPolicyScheme = HasPassthruOperand, + HasTailPolicy = false in { +defm vmseq : RVVIntMaskOutBuiltinSet; +defm vmsne : RVVIntMaskOutBuiltinSet; +defm vmsltu : RVVUnsignedMaskOutBuiltinSet; +defm vmslt : RVVSignedMaskOutBuiltinSet; +defm vmsleu : RVVUnsignedMaskOutBuiltinSet; +defm vmsle : RVVSignedMaskOutBuiltinSet; +defm vmsgtu : RVVUnsignedMaskOutBuiltinSet; +defm vmsgt : RVVSignedMaskOutBuiltinSet; +defm vmsgeu : RVVUnsignedMaskOutBuiltinSet; +defm vmsge : RVVSignedMaskOutBuiltinSet; +} + +// 11.9. Vector Integer Min/Max Instructions +let UnMaskedPolicyScheme = HasPassthruOperand in { +defm vminu : RVVUnsignedBinBuiltinSet; +defm vmin : RVVSignedBinBuiltinSet; +defm vmaxu : RVVUnsignedBinBuiltinSet; +defm vmax : RVVSignedBinBuiltinSet; + +// 11.10. Vector Single-Width Integer Multiply Instructions +defm vmul : RVVIntBinBuiltinSet; +defm vmulh : RVVSignedBinBuiltinSet; +defm vmulhu : RVVUnsignedBinBuiltinSet; +defm vmulhsu : RVVOutOp1BuiltinSet<"vmulhsu", "csil", + [["vv", "v", "vvUv"], + ["vx", "v", "vvUe"]]>; + +// 11.11. Vector Integer Divide Instructions +defm vdivu : RVVUnsignedBinBuiltinSet; +defm vdiv : RVVSignedBinBuiltinSet; +defm vremu : RVVUnsignedBinBuiltinSet; +defm vrem : RVVSignedBinBuiltinSet; +} + +// 11.12. Vector Widening Integer Multiply Instructions +let Log2LMUL = [-3, -2, -1, 0, 1, 2], UnMaskedPolicyScheme = HasPassthruOperand in { +defm vwmul : RVVOutOp0Op1BuiltinSet<"vwmul", "csi", + [["vv", "w", "wvv"], + ["vx", "w", "wve"]]>; +defm vwmulu : RVVOutOp0Op1BuiltinSet<"vwmulu", "csi", + [["vv", "Uw", "UwUvUv"], + ["vx", "Uw", "UwUvUe"]]>; +defm vwmulsu : RVVOutOp0Op1BuiltinSet<"vwmulsu", "csi", + [["vv", "w", "wvUv"], + ["vx", "w", "wvUe"]]>; +} + +// 11.13. Vector Single-Width Integer Multiply-Add Instructions +let UnMaskedPolicyScheme = HasPolicyOperand in { +defm vmacc : RVVIntTerBuiltinSet; +defm vnmsac : RVVIntTerBuiltinSet; +defm vmadd : RVVIntTerBuiltinSet; +defm vnmsub : RVVIntTerBuiltinSet; + +// 11.14. Vector Widening Integer Multiply-Add Instructions +let HasMaskedOffOperand = false, + Log2LMUL = [-3, -2, -1, 0, 1, 2] in { +defm vwmaccu : RVVOutOp1Op2BuiltinSet<"vwmaccu", "csi", + [["vv", "Uw", "UwUwUvUv"], + ["vx", "Uw", "UwUwUeUv"]]>; +defm vwmacc : RVVOutOp1Op2BuiltinSet<"vwmacc", "csi", + [["vv", "w", "wwvv"], + ["vx", "w", "wwev"]]>; +defm vwmaccsu : RVVOutOp1Op2BuiltinSet<"vwmaccsu", "csi", + [["vv", "w", "wwvUv"], + ["vx", "w", "wweUv"]]>; +defm vwmaccus : RVVOutOp1Op2BuiltinSet<"vwmaccus", "csi", + [["vx", "w", "wwUev"]]>; +} +} + +// 11.15. Vector Integer Merge Instructions +// C/C++ Operand: (mask, op1, op2, vl), Intrinsic: (passthru, op1, op2, mask, vl) +let HasMasked = false, + UnMaskedPolicyScheme = HasPassthruOperand, + MaskedPolicyScheme = NonePolicy, + ManualCodegen = [{ + // insert poison passthru + if (PolicyAttrs & RVV_VTA) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + IntrinsicTypes = {ResultType, Ops[2]->getType(), Ops.back()->getType()}; + }] in { + defm vmerge : RVVOutOp1BuiltinSet<"vmerge", "csil", + [["vvm", "v", "vvvm"], + ["vxm", "v", "vvem"], + ["vvm", "Uv", "UvUvUvm"], + ["vxm", "Uv", "UvUvUem"]]>; +} + +// 11.16. Vector Integer Move Instructions +let HasMasked = false, + UnMaskedPolicyScheme = HasPassthruOperand, + MaskedPolicyScheme = NonePolicy, + OverloadedName = "vmv_v" in { + defm vmv_v : RVVOutBuiltinSet<"vmv_v_v", "csil", + [["v", "Uv", "UvUv"]]>; + defm vmv_v : RVVOutBuiltinSet<"vmv_v_v", "csilfd", + [["v", "v", "vv"]]>; + let RequiredFeatures = ["Zvfhmin"] in + defm vmv_v : RVVOutBuiltinSet<"vmv_v_v", "x", + [["v", "v", "vv"]]>; + let SupportOverloading = false in + defm vmv_v : RVVOutBuiltinSet<"vmv_v_x", "csil", + [["x", "v", "ve"], + ["x", "Uv", "UvUe"]]>; +} + +// 12. Vector Fixed-Point Arithmetic Instructions +let HeaderCode = +[{ +enum __RISCV_VXRM { + __RISCV_VXRM_RNU = 0, + __RISCV_VXRM_RNE = 1, + __RISCV_VXRM_RDN = 2, + __RISCV_VXRM_ROD = 3, +}; +}] in +def vxrm_enum : RVVHeader; + +// 12.1. Vector Single-Width Saturating Add and Subtract +let UnMaskedPolicyScheme = HasPassthruOperand in { +defm vsaddu : RVVUnsignedBinBuiltinSet; +defm vsadd : RVVSignedBinBuiltinSet; +defm vssubu : RVVUnsignedBinBuiltinSet; +defm vssub : RVVSignedBinBuiltinSet; + +let ManualCodegen = [{ + { + // LLVM intrinsic + // Unmasked: (passthru, op0, op1, round_mode, vl) + // Masked: (passthru, vector_in, vector_in/scalar_in, mask, vxrm, vl, policy) + + SmallVector<llvm::Value*, 7> Operands; + bool HasMaskedOff = !( + (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) || + (!IsMasked && PolicyAttrs & RVV_VTA)); + unsigned Offset = IsMasked ? + (HasMaskedOff ? 2 : 1) : (HasMaskedOff ? 1 : 0); + + if (!HasMaskedOff) + Operands.push_back(llvm::PoisonValue::get(ResultType)); + else + Operands.push_back(Ops[IsMasked ? 1 : 0]); + + Operands.push_back(Ops[Offset]); // op0 + Operands.push_back(Ops[Offset + 1]); // op1 + + if (IsMasked) + Operands.push_back(Ops[0]); // mask + + Operands.push_back(Ops[Offset + 2]); // vxrm + Operands.push_back(Ops[Offset + 3]); // vl + + if (IsMasked) + Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); + + IntrinsicTypes = {ResultType, Ops[Offset + 1]->getType(), Ops.back()->getType()}; + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); + return Builder.CreateCall(F, Operands, ""); + } +}] in { + // 12.2. Vector Single-Width Averaging Add and Subtract + defm vaaddu : RVVUnsignedBinBuiltinSetRoundingMode; + defm vaadd : RVVSignedBinBuiltinSetRoundingMode; + defm vasubu : RVVUnsignedBinBuiltinSetRoundingMode; + defm vasub : RVVSignedBinBuiltinSetRoundingMode; + + // 12.3. Vector Single-Width Fractional Multiply with Rounding and Saturation + defm vsmul : RVVSignedBinBuiltinSetRoundingMode; + + // 12.4. Vector Single-Width Scaling Shift Instructions + defm vssrl : RVVUnsignedShiftBuiltinSetRoundingMode; + defm vssra : RVVSignedShiftBuiltinSetRoundingMode; +} + +let ManualCodegen = [{ + { + // LLVM intrinsic + // Unmasked: (passthru, op0, op1, round_mode, vl) + // Masked: (passthru, vector_in, vector_in/scalar_in, mask, vxrm, vl, policy) + + SmallVector<llvm::Value*, 7> Operands; + bool HasMaskedOff = !( + (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) || + (!IsMasked && PolicyAttrs & RVV_VTA)); + unsigned Offset = IsMasked ? + (HasMaskedOff ? 2 : 1) : (HasMaskedOff ? 1 : 0); + + if (!HasMaskedOff) + Operands.push_back(llvm::PoisonValue::get(ResultType)); + else + Operands.push_back(Ops[IsMasked ? 1 : 0]); + + Operands.push_back(Ops[Offset]); // op0 + Operands.push_back(Ops[Offset + 1]); // op1 + + if (IsMasked) + Operands.push_back(Ops[0]); // mask + + Operands.push_back(Ops[Offset + 2]); // vxrm + Operands.push_back(Ops[Offset + 3]); // vl + + if (IsMasked) + Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); + + IntrinsicTypes = {ResultType, Ops[Offset]->getType(), Ops[Offset + 1]->getType(), + Ops.back()->getType()}; + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); + return Builder.CreateCall(F, Operands, ""); + } +}] in { + // 12.5. Vector Narrowing Fixed-Point Clip Instructions + defm vnclipu : RVVUnsignedNShiftBuiltinSetRoundingMode; + defm vnclip : RVVSignedNShiftBuiltinSetRoundingMode; +} +} + +// 13. Vector Floating-Point Instructions +let HeaderCode = +[{ +enum __RISCV_FRM { + __RISCV_FRM_RNE = 0, + __RISCV_FRM_RTZ = 1, + __RISCV_FRM_RDN = 2, + __RISCV_FRM_RUP = 3, + __RISCV_FRM_RMM = 4, +}; +}] in def frm_enum : RVVHeader; + +let UnMaskedPolicyScheme = HasPassthruOperand in { +let ManualCodegen = [{ + { + // LLVM intrinsic + // Unmasked: (passthru, op0, op1, round_mode, vl) + // Masked: (passthru, vector_in, vector_in/scalar_in, mask, frm, vl, policy) + + SmallVector<llvm::Value*, 7> Operands; + bool HasMaskedOff = !( + (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) || + (!IsMasked && PolicyAttrs & RVV_VTA)); + bool HasRoundModeOp = IsMasked ? + (HasMaskedOff ? Ops.size() == 6 : Ops.size() == 5) : + (HasMaskedOff ? Ops.size() == 5 : Ops.size() == 4); + + unsigned Offset = IsMasked ? + (HasMaskedOff ? 2 : 1) : (HasMaskedOff ? 1 : 0); + + if (!HasMaskedOff) + Operands.push_back(llvm::PoisonValue::get(ResultType)); + else + Operands.push_back(Ops[IsMasked ? 1 : 0]); + + Operands.push_back(Ops[Offset]); // op0 + Operands.push_back(Ops[Offset + 1]); // op1 + + if (IsMasked) + Operands.push_back(Ops[0]); // mask + + if (HasRoundModeOp) { + Operands.push_back(Ops[Offset + 2]); // frm + Operands.push_back(Ops[Offset + 3]); // vl + } else { + Operands.push_back(ConstantInt::get(Ops[Offset + 2]->getType(), 7)); // frm + Operands.push_back(Ops[Offset + 2]); // vl + } + + if (IsMasked) + Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); + + IntrinsicTypes = {ResultType, Ops[Offset + 1]->getType(), + Operands.back()->getType()}; + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); + return Builder.CreateCall(F, Operands, ""); + } +}] in { + let HasFRMRoundModeOp = true in { + // 13.2. Vector Single-Width Floating-Point Add/Subtract Instructions + defm vfadd : RVVFloatingBinBuiltinSetRoundingMode; + defm vfsub : RVVFloatingBinBuiltinSetRoundingMode; + defm vfrsub : RVVFloatingBinVFBuiltinSetRoundingMode; + + // 13.3. Vector Widening Floating-Point Add/Subtract Instructions + // Widening FP add/subtract, 2*SEW = 2*SEW +/- SEW + defm vfwadd : RVVFloatingWidenOp0BinBuiltinSetRoundingMode; + defm vfwsub : RVVFloatingWidenOp0BinBuiltinSetRoundingMode; + + // 13.4. Vector Single-Width Floating-Point Multiply/Divide Instructions + defm vfmul : RVVFloatingBinBuiltinSetRoundingMode; + defm vfdiv : RVVFloatingBinBuiltinSetRoundingMode; + defm vfrdiv : RVVFloatingBinVFBuiltinSetRoundingMode; + } + // 13.2. Vector Single-Width Floating-Point Add/Subtract Instructions + defm vfadd : RVVFloatingBinBuiltinSet; + defm vfsub : RVVFloatingBinBuiltinSet; + defm vfrsub : RVVFloatingBinVFBuiltinSet; + + // 13.3. Vector Widening Floating-Point Add/Subtract Instructions + // Widening FP add/subtract, 2*SEW = 2*SEW +/- SEW + defm vfwadd : RVVFloatingWidenOp0BinBuiltinSet; + defm vfwsub : RVVFloatingWidenOp0BinBuiltinSet; + + // 13.4. Vector Single-Width Floating-Point Multiply/Divide Instructions + defm vfmul : RVVFloatingBinBuiltinSet; + defm vfdiv : RVVFloatingBinBuiltinSet; + defm vfrdiv : RVVFloatingBinVFBuiltinSet; +} + +let ManualCodegen = [{ + { + // LLVM intrinsic + // Unmasked: (passthru, op0, op1, round_mode, vl) + // Masked: (passthru, vector_in, vector_in/scalar_in, mask, frm, vl, policy) + + SmallVector<llvm::Value*, 7> Operands; + bool HasMaskedOff = !( + (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) || + (!IsMasked && PolicyAttrs & RVV_VTA)); + bool HasRoundModeOp = IsMasked ? + (HasMaskedOff ? Ops.size() == 6 : Ops.size() == 5) : + (HasMaskedOff ? Ops.size() == 5 : Ops.size() == 4); + + unsigned Offset = IsMasked ? + (HasMaskedOff ? 2 : 1) : (HasMaskedOff ? 1 : 0); + + if (!HasMaskedOff) + Operands.push_back(llvm::PoisonValue::get(ResultType)); + else + Operands.push_back(Ops[IsMasked ? 1 : 0]); + + Operands.push_back(Ops[Offset]); // op0 + Operands.push_back(Ops[Offset + 1]); // op1 + + if (IsMasked) + Operands.push_back(Ops[0]); // mask + + if (HasRoundModeOp) { + Operands.push_back(Ops[Offset + 2]); // frm + Operands.push_back(Ops[Offset + 3]); // vl + } else { + Operands.push_back(ConstantInt::get(Ops[Offset + 2]->getType(), 7)); // frm + Operands.push_back(Ops[Offset + 2]); // vl + } + + if (IsMasked) + Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); + + IntrinsicTypes = {ResultType, Ops[Offset]->getType(), Ops[Offset + 1]->getType(), + Ops.back()->getType()}; + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); + return Builder.CreateCall(F, Operands, ""); + } +}] in { + let HasFRMRoundModeOp = true in { + // 13.3. Vector Widening Floating-Point Add/Subtract Instructions + // Widening FP add/subtract, 2*SEW = SEW +/- SEW + defm vfwadd : RVVFloatingWidenBinBuiltinSetRoundingMode; + defm vfwsub : RVVFloatingWidenBinBuiltinSetRoundingMode; + + // 13.5. Vector Widening Floating-Point Multiply + let Log2LMUL = [-2, -1, 0, 1, 2] in { + defm vfwmul : RVVOutOp0Op1BuiltinSet<"vfwmul", "xf", + [["vv", "w", "wvvu"], + ["vf", "w", "wveu"]]>; + } + } + // 13.3. Vector Widening Floating-Point Add/Subtract Instructions + // Widening FP add/subtract, 2*SEW = SEW +/- SEW + defm vfwadd : RVVFloatingWidenBinBuiltinSet; + defm vfwsub : RVVFloatingWidenBinBuiltinSet; + + // 13.5. Vector Widening Floating-Point Multiply + let Log2LMUL = [-2, -1, 0, 1, 2] in { + defm vfwmul : RVVOutOp0Op1BuiltinSet<"vfwmul", "xf", + [["vv", "w", "wvv"], + ["vf", "w", "wve"]]>; + } +} +} + + +let UnMaskedPolicyScheme = HasPolicyOperand in { +let ManualCodegen = [{ + { + // LLVM intrinsic + // Unmasked: (passthru, op0, op1, round_mode, vl) + // Masked: (passthru, vector_in, vector_in/scalar_in, mask, frm, vl, policy) + + SmallVector<llvm::Value*, 7> Operands; + bool HasRoundModeOp = IsMasked ? Ops.size() == 6 : Ops.size() == 5; + + unsigned Offset = IsMasked ? 2 : 1; + + Operands.push_back(Ops[IsMasked ? 1 : 0]); // passthrough + + Operands.push_back(Ops[Offset]); // op0 + Operands.push_back(Ops[Offset + 1]); // op1 + + if (IsMasked) + Operands.push_back(Ops[0]); // mask + + if (HasRoundModeOp) { + Operands.push_back(Ops[Offset + 2]); // frm + Operands.push_back(Ops[Offset + 3]); // vl + } else { + Operands.push_back(ConstantInt::get(Ops[Offset + 2]->getType(), 7)); // frm + Operands.push_back(Ops[Offset + 2]); // vl + } + + Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); + + IntrinsicTypes = {ResultType, Ops[Offset]->getType(), + Operands.back()->getType()}; + + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); + + return Builder.CreateCall(F, Operands, ""); + } +}] in { + let HasFRMRoundModeOp = 1 in { + // 13.6. Vector Single-Width Floating-Point Fused Multiply-Add Instructions + defm vfmacc : RVVFloatingTerBuiltinSetRoundingMode; + defm vfnmacc : RVVFloatingTerBuiltinSetRoundingMode; + defm vfmsac : RVVFloatingTerBuiltinSetRoundingMode; + defm vfnmsac : RVVFloatingTerBuiltinSetRoundingMode; + defm vfmadd : RVVFloatingTerBuiltinSetRoundingMode; + defm vfnmadd : RVVFloatingTerBuiltinSetRoundingMode; + defm vfmsub : RVVFloatingTerBuiltinSetRoundingMode; + defm vfnmsub : RVVFloatingTerBuiltinSetRoundingMode; + } + // 13.6. Vector Single-Width Floating-Point Fused Multiply-Add Instructions + defm vfmacc : RVVFloatingTerBuiltinSet; + defm vfnmacc : RVVFloatingTerBuiltinSet; + defm vfmsac : RVVFloatingTerBuiltinSet; + defm vfnmsac : RVVFloatingTerBuiltinSet; + defm vfmadd : RVVFloatingTerBuiltinSet; + defm vfnmadd : RVVFloatingTerBuiltinSet; + defm vfmsub : RVVFloatingTerBuiltinSet; + defm vfnmsub : RVVFloatingTerBuiltinSet; +} + +let ManualCodegen = [{ + { + // LLVM intrinsic + // Unmasked: (passthru, op0, op1, round_mode, vl) + // Masked: (passthru, vector_in, vector_in/scalar_in, mask, frm, vl, policy) + + SmallVector<llvm::Value*, 7> Operands; + bool HasRoundModeOp = IsMasked ? Ops.size() == 6 : Ops.size() == 5; + + unsigned Offset = IsMasked ? 2 : 1; + + Operands.push_back(Ops[IsMasked ? 1 : 0]); // passthrough + + Operands.push_back(Ops[Offset]); // op0 + Operands.push_back(Ops[Offset + 1]); // op1 + + if (IsMasked) + Operands.push_back(Ops[0]); // mask + + if (HasRoundModeOp) { + Operands.push_back(Ops[Offset + 2]); // frm + Operands.push_back(Ops[Offset + 3]); // vl + } else { + Operands.push_back(ConstantInt::get(Ops[Offset + 2]->getType(), 7)); // frm + Operands.push_back(Ops[Offset + 2]); // vl + } + + Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); + + IntrinsicTypes = {ResultType, Ops[Offset]->getType(), Ops[Offset + 1]->getType(), + Operands.back()->getType()}; + + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); + + return Builder.CreateCall(F, Operands, ""); + } +}] in { + let HasFRMRoundModeOp = 1 in { + // 13.7. Vector Widening Floating-Point Fused Multiply-Add Instructions + defm vfwmacc : RVVFloatingWidenTerBuiltinSetRoundingMode; + defm vfwnmacc : RVVFloatingWidenTerBuiltinSetRoundingMode; + defm vfwmsac : RVVFloatingWidenTerBuiltinSetRoundingMode; + defm vfwnmsac : RVVFloatingWidenTerBuiltinSetRoundingMode; + } + // 13.7. Vector Widening Floating-Point Fused Multiply-Add Instructions + defm vfwmacc : RVVFloatingWidenTerBuiltinSet; + defm vfwnmacc : RVVFloatingWidenTerBuiltinSet; + defm vfwmsac : RVVFloatingWidenTerBuiltinSet; + defm vfwnmsac : RVVFloatingWidenTerBuiltinSet; +} + +} + +let UnMaskedPolicyScheme = HasPassthruOperand in { +let ManualCodegen = [{ + { + // LLVM intrinsic + // Unmasked: (passthru, op0, round_mode, vl) + // Masked: (passthru, op0, mask, frm, vl, policy) + + SmallVector<llvm::Value*, 7> Operands; + bool HasMaskedOff = !( + (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) || + (!IsMasked && PolicyAttrs & RVV_VTA)); + bool HasRoundModeOp = IsMasked ? + (HasMaskedOff ? Ops.size() == 5 : Ops.size() == 4) : + (HasMaskedOff ? Ops.size() == 4 : Ops.size() == 3); + + unsigned Offset = IsMasked ? + (HasMaskedOff ? 2 : 1) : (HasMaskedOff ? 1 : 0); + + if (!HasMaskedOff) + Operands.push_back(llvm::PoisonValue::get(ResultType)); + else + Operands.push_back(Ops[IsMasked ? 1 : 0]); + + Operands.push_back(Ops[Offset]); // op0 + + if (IsMasked) + Operands.push_back(Ops[0]); // mask + + if (HasRoundModeOp) { + Operands.push_back(Ops[Offset + 1]); // frm + Operands.push_back(Ops[Offset + 2]); // vl + } else { + Operands.push_back(ConstantInt::get(Ops[Offset + 1]->getType(), 7)); // frm + Operands.push_back(Ops[Offset + 1]); // vl + } + + if (IsMasked) + Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); + + IntrinsicTypes = {ResultType, Operands.back()->getType()}; + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); + return Builder.CreateCall(F, Operands, ""); + } +}] in { + let HasFRMRoundModeOp = 1 in { + // 13.8. Vector Floating-Point Square-Root Instruction + defm vfsqrt : RVVOutBuiltinSet<"vfsqrt", "xfd", [["v", "v", "vvu"]]>; + + // 13.10. Vector Floating-Point Reciprocal Estimate Instruction + defm vfrec7 : RVVOutBuiltinSet<"vfrec7", "xfd", [["v", "v", "vvu"]]>; + } + // 13.8. Vector Floating-Point Square-Root Instruction + defm vfsqrt : RVVOutBuiltinSet<"vfsqrt", "xfd", [["v", "v", "vv"]]>; + + // 13.10. Vector Floating-Point Reciprocal Estimate Instruction + defm vfrec7 : RVVOutBuiltinSet<"vfrec7", "xfd", [["v", "v", "vv"]]>; +} + +// 13.9. Vector Floating-Point Reciprocal Square-Root Estimate Instruction +def vfrsqrt7 : RVVFloatingUnaryVVBuiltin; + +// 13.11. Vector Floating-Point MIN/MAX Instructions +defm vfmin : RVVFloatingBinBuiltinSet; +defm vfmax : RVVFloatingBinBuiltinSet; + +// 13.12. Vector Floating-Point Sign-Injection Instructions +defm vfsgnj : RVVFloatingBinBuiltinSet; +defm vfsgnjn : RVVFloatingBinBuiltinSet; +defm vfsgnjx : RVVFloatingBinBuiltinSet; +} +defm vfneg_v : RVVPseudoVFUnaryBuiltin<"vfsgnjn", "xfd">; +defm vfabs_v : RVVPseudoVFUnaryBuiltin<"vfsgnjx", "xfd">; + +// 13.13. Vector Floating-Point Compare Instructions +let MaskedPolicyScheme = HasPassthruOperand, + HasTailPolicy = false in { +defm vmfeq : RVVFloatingMaskOutBuiltinSet; +defm vmfne : RVVFloatingMaskOutBuiltinSet; +defm vmflt : RVVFloatingMaskOutBuiltinSet; +defm vmfle : RVVFloatingMaskOutBuiltinSet; +defm vmfgt : RVVFloatingMaskOutBuiltinSet; +defm vmfge : RVVFloatingMaskOutBuiltinSet; +} + +// 13.14. Vector Floating-Point Classify Instruction +let Name = "vfclass_v", UnMaskedPolicyScheme = HasPassthruOperand in + def vfclass : RVVOp0Builtin<"Uv", "Uvv", "xfd">; + +// 13.15. Vector Floating-Point Merge Instruction +// C/C++ Operand: (mask, op1, op2, vl), Builtin: (op1, op2, mask, vl) +let HasMasked = false, + UnMaskedPolicyScheme = HasPassthruOperand, + MaskedPolicyScheme = NonePolicy, + ManualCodegen = [{ + // insert poison passthru + if (PolicyAttrs & RVV_VTA) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + IntrinsicTypes = {ResultType, Ops[2]->getType(), Ops.back()->getType()}; + }] in { + defm vmerge : RVVOutOp1BuiltinSet<"vmerge", "fd", + [["vvm", "v", "vvvm"]]>; + let RequiredFeatures = ["Zvfhmin"] in + defm vmerge : RVVOutOp1BuiltinSet<"vmerge", "x", + [["vvm", "v", "vvvm"]]>; + defm vfmerge : RVVOutOp1BuiltinSet<"vfmerge", "xfd", + [["vfm", "v", "vvem"]]>; +} + +// 13.16. Vector Floating-Point Move Instruction +let HasMasked = false, + UnMaskedPolicyScheme = HasPassthruOperand, + SupportOverloading = false, + MaskedPolicyScheme = NonePolicy, + OverloadedName = "vfmv_v" in + defm vfmv_v : RVVOutBuiltinSet<"vfmv_v_f", "xfd", + [["f", "v", "ve"]]>; + +// 13.17. Single-Width Floating-Point/Integer Type-Convert Instructions +let UnMaskedPolicyScheme = HasPassthruOperand in { +def vfcvt_rtz_xu_f_v : RVVConvToUnsignedBuiltin<"vfcvt_rtz_xu">; +def vfcvt_rtz_x_f_v : RVVConvToSignedBuiltin<"vfcvt_rtz_x">; + +// 13.18. Widening Floating-Point/Integer Type-Convert Instructions +let Log2LMUL = [-3, -2, -1, 0, 1, 2] in { + def vfwcvt_rtz_xu_f_v : RVVConvToWidenUnsignedBuiltin<"vfwcvt_rtz_xu">; + def vfwcvt_rtz_x_f_v : RVVConvToWidenSignedBuiltin<"vfwcvt_rtz_x">; + def vfwcvt_f_xu_v : RVVConvBuiltin<"Fw", "FwUv", "csi", "vfwcvt_f">; + def vfwcvt_f_x_v : RVVConvBuiltin<"Fw", "Fwv", "csi", "vfwcvt_f">; + def vfwcvt_f_f_v : RVVConvBuiltin<"w", "wv", "f", "vfwcvt_f">; + let RequiredFeatures = ["Zvfhmin"] in + def vfwcvt_f_f_v_fp16 : RVVConvBuiltin<"w", "wv", "x", "vfwcvt_f"> { + let Name = "vfwcvt_f_f_v"; + let IRName = "vfwcvt_f_f_v"; + let MaskedIRName = "vfwcvt_f_f_v_mask"; + } +} + +// 13.19. Narrowing Floating-Point/Integer Type-Convert Instructions +let Log2LMUL = [-3, -2, -1, 0, 1, 2] in { + def vfncvt_rtz_xu_f_w : RVVConvToNarrowingUnsignedBuiltin<"vfncvt_rtz_xu">; + def vfncvt_rtz_x_f_w : RVVConvToNarrowingSignedBuiltin<"vfncvt_rtz_x">; + def vfncvt_rod_f_f_w : RVVConvBuiltin<"v", "vw", "xf", "vfncvt_rod_f">; +} +let ManualCodegen = [{ + { + // LLVM intrinsic + // Unmasked: (passthru, op0, frm, vl) + // Masked: (passthru, op0, mask, frm, vl, policy) + SmallVector<llvm::Value*, 7> Operands; + bool HasMaskedOff = !( + (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) || + (!IsMasked && PolicyAttrs & RVV_VTA)); + bool HasRoundModeOp = IsMasked ? + (HasMaskedOff ? Ops.size() == 5 : Ops.size() == 4) : + (HasMaskedOff ? Ops.size() == 4 : Ops.size() == 3); + + unsigned Offset = IsMasked ? + (HasMaskedOff ? 2 : 1) : (HasMaskedOff ? 1 : 0); + + if (!HasMaskedOff) + Operands.push_back(llvm::PoisonValue::get(ResultType)); + else + Operands.push_back(Ops[IsMasked ? 1 : 0]); + + Operands.push_back(Ops[Offset]); // op0 + + if (IsMasked) + Operands.push_back(Ops[0]); // mask + + if (HasRoundModeOp) { + Operands.push_back(Ops[Offset + 1]); // frm + Operands.push_back(Ops[Offset + 2]); // vl + } else { + Operands.push_back(ConstantInt::get(Ops[Offset + 1]->getType(), 7)); // frm + Operands.push_back(Ops[Offset + 1]); // vl + } + + if (IsMasked) + Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); + + IntrinsicTypes = {ResultType, Ops[Offset]->getType(), + Operands.back()->getType()}; + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); + return Builder.CreateCall(F, Operands, ""); + } +}] in { + let HasFRMRoundModeOp = 1 in { + // 14.17. Single-Width Floating-Point/Integer Type-Convert Instructions + let OverloadedName = "vfcvt_x" in + defm : + RVVConvBuiltinSet<"vfcvt_x_f_v", "xfd", [["Iv", "Ivvu"]]>; + let OverloadedName = "vfcvt_xu" in + defm : + RVVConvBuiltinSet<"vfcvt_xu_f_v", "xfd", [["Uv", "Uvvu"]]>; + let OverloadedName = "vfcvt_f" in { + defm : + RVVConvBuiltinSet<"vfcvt_f_x_v", "sil", [["Fv", "Fvvu"]]>; + defm : + RVVConvBuiltinSet<"vfcvt_f_xu_v", "sil", [["Fv", "FvUvu"]]>; + } + + // 13.18. Widening Floating-Point/Integer Type-Convert Instructions + let Log2LMUL = [-3, -2, -1, 0, 1, 2] in { + let OverloadedName = "vfwcvt_x" in + defm : + RVVConvBuiltinSet<"vfwcvt_x_f_v", "xf", [["Iw", "Iwvu"]]>; + let OverloadedName = "vfwcvt_xu" in + defm : + RVVConvBuiltinSet<"vfwcvt_xu_f_v", "xf", [["Uw", "Uwvu"]]>; + } + // 13.19. Narrowing Floating-Point/Integer Type-Convert Instructions + let Log2LMUL = [-3, -2, -1, 0, 1, 2] in { + let OverloadedName = "vfncvt_x" in + defm : + RVVConvBuiltinSet<"vfncvt_x_f_w", "csi", [["Iv", "IvFwu"]]>; + let OverloadedName = "vfncvt_xu" in + defm : + RVVConvBuiltinSet<"vfncvt_xu_f_w", "csi", [["Uv", "UvFwu"]]>; + let OverloadedName = "vfncvt_f" in { + defm : + RVVConvBuiltinSet<"vfncvt_f_x_w", "csi", [["Fv", "Fvwu"]]>; + defm : + RVVConvBuiltinSet<"vfncvt_f_xu_w", "csi", [["Fv", "FvUwu"]]>; + } + let OverloadedName = "vfncvt_f" in { + defm : RVVConvBuiltinSet<"vfncvt_f_f_w", "f", [["v", "vwu"]]>; + let RequiredFeatures = ["Zvfhmin"] in + defm : RVVConvBuiltinSet<"vfncvt_f_f_w", "x", [["v", "vwu"]]>; + } + } + } + + // 13.17. Single-Width Floating-Point/Integer Type-Convert Instructions + let OverloadedName = "vfcvt_x" in + defm : + RVVConvBuiltinSet<"vfcvt_x_f_v", "xfd", [["Iv", "Ivv"]]>; + let OverloadedName = "vfcvt_xu" in + defm : + RVVConvBuiltinSet<"vfcvt_xu_f_v", "xfd", [["Uv", "Uvv"]]>; + let OverloadedName = "vfcvt_f" in { + defm : + RVVConvBuiltinSet<"vfcvt_f_x_v", "sil", [["Fv", "Fvv"]]>; + defm : + RVVConvBuiltinSet<"vfcvt_f_xu_v", "sil", [["Fv", "FvUv"]]>; + } + + // 13.18. Widening Floating-Point/Integer Type-Convert Instructions + let Log2LMUL = [-3, -2, -1, 0, 1, 2] in { + let OverloadedName = "vfwcvt_x" in + defm : + RVVConvBuiltinSet<"vfwcvt_x_f_v", "xf", [["Iw", "Iwv"]]>; + let OverloadedName = "vfwcvt_xu" in + defm : + RVVConvBuiltinSet<"vfwcvt_xu_f_v", "xf", [["Uw", "Uwv"]]>; + } + // 13.19. Narrowing Floating-Point/Integer Type-Convert Instructions + let Log2LMUL = [-3, -2, -1, 0, 1, 2] in { + let OverloadedName = "vfncvt_x" in + defm : + RVVConvBuiltinSet<"vfncvt_x_f_w", "csi", [["Iv", "IvFw"]]>; + let OverloadedName = "vfncvt_xu" in + defm : + RVVConvBuiltinSet<"vfncvt_xu_f_w", "csi", [["Uv", "UvFw"]]>; + let OverloadedName = "vfncvt_f" in { + defm : + RVVConvBuiltinSet<"vfncvt_f_x_w", "csi", [["Fv", "Fvw"]]>; + defm : + RVVConvBuiltinSet<"vfncvt_f_xu_w", "csi", [["Fv", "FvUw"]]>; + } + let OverloadedName = "vfncvt_f" in { + defm : RVVConvBuiltinSet<"vfncvt_f_f_w", "f", [["v", "vw"]]>; + let RequiredFeatures = ["Zvfhmin"] in + defm : RVVConvBuiltinSet<"vfncvt_f_f_w", "x", [["v", "vw"]]>; + } + } +} +} + +// 14. Vector Reduction Operations +// 14.1. Vector Single-Width Integer Reduction Instructions +let UnMaskedPolicyScheme = HasPassthruOperand, + MaskedPolicyScheme = HasPassthruOperand, + HasMaskPolicy = false in { +defm vredsum : RVVIntReductionBuiltinSet; +defm vredmaxu : RVVUnsignedReductionBuiltin; +defm vredmax : RVVSignedReductionBuiltin; +defm vredminu : RVVUnsignedReductionBuiltin; +defm vredmin : RVVSignedReductionBuiltin; +defm vredand : RVVIntReductionBuiltinSet; +defm vredor : RVVIntReductionBuiltinSet; +defm vredxor : RVVIntReductionBuiltinSet; + +// 14.2. Vector Widening Integer Reduction Instructions +// Vector Widening Integer Reduction Operations +let HasMaskedOffOperand = true in { + defm vwredsum : RVVOutOp0BuiltinSet<"vwredsum", "csi", + [["vs", "vSw", "SwvSw"]]>; + defm vwredsumu : RVVOutOp0BuiltinSet<"vwredsumu", "csi", + [["vs", "UvUSw", "USwUvUSw"]]>; +} + +// 14.3. Vector Single-Width Floating-Point Reduction Instructions +defm vfredmax : RVVFloatingReductionBuiltin; +defm vfredmin : RVVFloatingReductionBuiltin; +let ManualCodegen = [{ + { + // LLVM intrinsic + // Unmasked: (passthru, op0, op1, round_mode, vl) + // Masked: (passthru, vector_in, vector_in/scalar_in, mask, frm, vl, policy) + + SmallVector<llvm::Value*, 7> Operands; + bool HasMaskedOff = !( + (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) || + (!IsMasked && PolicyAttrs & RVV_VTA)); + bool HasRoundModeOp = IsMasked ? + (HasMaskedOff ? Ops.size() == 6 : Ops.size() == 5) : + (HasMaskedOff ? Ops.size() == 5 : Ops.size() == 4); + + unsigned Offset = IsMasked ? + (HasMaskedOff ? 2 : 1) : (HasMaskedOff ? 1 : 0); + + if (!HasMaskedOff) + Operands.push_back(llvm::PoisonValue::get(ResultType)); + else + Operands.push_back(Ops[IsMasked ? 1 : 0]); + + Operands.push_back(Ops[Offset]); // op0 + Operands.push_back(Ops[Offset + 1]); // op1 + + if (IsMasked) + Operands.push_back(Ops[0]); // mask + + if (HasRoundModeOp) { + Operands.push_back(Ops[Offset + 2]); // frm + Operands.push_back(Ops[Offset + 3]); // vl + } else { + Operands.push_back(ConstantInt::get(Ops[Offset + 2]->getType(), 7)); // frm + Operands.push_back(Ops[Offset + 2]); // vl + } + + IntrinsicTypes = {ResultType, Ops[Offset]->getType(), + Ops.back()->getType()}; + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); + return Builder.CreateCall(F, Operands, ""); + } +}] in { + let HasFRMRoundModeOp = 1 in { + // 14.3. Vector Single-Width Floating-Point Reduction Instructions + defm vfredusum : RVVFloatingReductionBuiltinRoundingMode; + defm vfredosum : RVVFloatingReductionBuiltinRoundingMode; + + // 14.4. Vector Widening Floating-Point Reduction Instructions + defm vfwredusum : RVVFloatingWidenReductionBuiltinRoundingMode; + defm vfwredosum : RVVFloatingWidenReductionBuiltinRoundingMode; + } + // 14.3. Vector Single-Width Floating-Point Reduction Instructions + defm vfredusum : RVVFloatingReductionBuiltin; + defm vfredosum : RVVFloatingReductionBuiltin; + + // 14.4. Vector Widening Floating-Point Reduction Instructions + defm vfwredusum : RVVFloatingWidenReductionBuiltin; + defm vfwredosum : RVVFloatingWidenReductionBuiltin; +} +} + +// 15. Vector Mask Instructions +// 15.1. Vector Mask-Register Logical Instructions +def vmand : RVVMaskBinBuiltin; +def vmnand : RVVMaskBinBuiltin; +def vmandn : RVVMaskBinBuiltin; +def vmxor : RVVMaskBinBuiltin; +def vmor : RVVMaskBinBuiltin; +def vmnor : RVVMaskBinBuiltin; +def vmorn : RVVMaskBinBuiltin; +def vmxnor : RVVMaskBinBuiltin; +// pseudoinstructions +def vmclr : RVVMaskNullaryBuiltin; +def vmset : RVVMaskNullaryBuiltin; +defm vmmv_m : RVVPseudoMaskBuiltin<"vmand", "c">; +defm vmnot_m : RVVPseudoMaskBuiltin<"vmnand", "c">; + +let MaskedPolicyScheme = NonePolicy in { +// 15.2. Vector count population in mask vcpop.m +def vcpop : RVVMaskOp0Builtin<"um">; + +// 15.3. vfirst find-first-set mask bit +def vfirst : RVVMaskOp0Builtin<"lm">; +} + +let MaskedPolicyScheme = HasPassthruOperand, + HasTailPolicy = false in { +// 15.4. vmsbf.m set-before-first mask bit +def vmsbf : RVVMaskUnaryBuiltin; + +// 15.5. vmsif.m set-including-first mask bit +def vmsif : RVVMaskUnaryBuiltin; + +// 15.6. vmsof.m set-only-first mask bit +def vmsof : RVVMaskUnaryBuiltin; +} + +let UnMaskedPolicyScheme = HasPassthruOperand, SupportOverloading = false in { + // 15.8. Vector Iota Instruction + defm viota : RVVOutBuiltinSet<"viota", "csil", [["m", "Uv", "Uvm"]]>; + + // 15.9. Vector Element Index Instruction + defm vid : RVVOutBuiltinSet<"vid", "csil", [["v", "v", "v"], + ["v", "Uv", "Uv"]]>; +} + +// 16. Vector Permutation Instructions +// 16.1. Integer Scalar Move Instructions +let HasMasked = false, MaskedPolicyScheme = NonePolicy in { + let HasVL = false, OverloadedName = "vmv_x" in + defm vmv_x : RVVOp0BuiltinSet<"vmv_x_s", "csil", + [["s", "ve", "ev"], + ["s", "UvUe", "UeUv"]]>; + let OverloadedName = "vmv_s", + UnMaskedPolicyScheme = HasPassthruOperand, + SupportOverloading = false in + defm vmv_s : RVVOutBuiltinSet<"vmv_s_x", "csil", + [["x", "v", "ve"], + ["x", "Uv", "UvUe"]]>; +} + +// 16.2. Floating-Point Scalar Move Instructions +let HasMasked = false, MaskedPolicyScheme = NonePolicy in { + let HasVL = false, OverloadedName = "vfmv_f" in + defm vfmv_f : RVVOp0BuiltinSet<"vfmv_f_s", "xfd", + [["s", "ve", "ev"]]>; + let OverloadedName = "vfmv_s", + UnMaskedPolicyScheme = HasPassthruOperand, + SupportOverloading = false in + defm vfmv_s : RVVOutBuiltinSet<"vfmv_s_f", "xfd", + [["f", "v", "ve"], + ["x", "Uv", "UvUe"]]>; +} + +// 16.3. Vector Slide Instructions +// 16.3.1. Vector Slideup Instructions +defm vslideup : RVVSlideUpBuiltinSet; +// 16.3.2. Vector Slidedown Instructions +defm vslidedown : RVVSlideDownBuiltinSet; + +// 16.3.3. Vector Slide1up Instructions +let UnMaskedPolicyScheme = HasPassthruOperand in { +defm vslide1up : RVVSlideOneBuiltinSet; +defm vfslide1up : RVVFloatingBinVFBuiltinSet; + +// 16.3.4. Vector Slide1down Instruction +defm vslide1down : RVVSlideOneBuiltinSet; +defm vfslide1down : RVVFloatingBinVFBuiltinSet; + +// 16.4. Vector Register Gather Instructions +// signed and floating type +defm vrgather : RVVOutBuiltinSet<"vrgather_vv", "csilxfd", + [["vv", "v", "vvUv"]]>; +defm vrgather : RVVOutBuiltinSet<"vrgather_vx", "csilxfd", + [["vx", "v", "vvz"]]>; +defm vrgatherei16 : RVVOutBuiltinSet<"vrgatherei16_vv", "csilxfd", + [["vv", "v", "vv(Log2EEW:4)Uv"]]>; +// unsigned type +defm vrgather : RVVOutBuiltinSet<"vrgather_vv", "csil", + [["vv", "Uv", "UvUvUv"]]>; +defm vrgather : RVVOutBuiltinSet<"vrgather_vx", "csil", + [["vx", "Uv", "UvUvz"]]>; +defm vrgatherei16 : RVVOutBuiltinSet<"vrgatherei16_vv", "csil", + [["vv", "Uv", "UvUv(Log2EEW:4)Uv"]]>; +} + +// 16.5. Vector Compress Instruction +let HasMasked = false, + UnMaskedPolicyScheme = HasPassthruOperand, + MaskedPolicyScheme = NonePolicy, + ManualCodegen = [{ + // insert poison passthru + if (PolicyAttrs & RVV_VTA) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + IntrinsicTypes = {ResultType, Ops.back()->getType()}; + }] in { + // signed and floating type + defm vcompress : RVVOutBuiltinSet<"vcompress", "csilxfd", + [["vm", "v", "vvm"]]>; + // unsigned type + defm vcompress : RVVOutBuiltinSet<"vcompress", "csil", + [["vm", "Uv", "UvUvm"]]>; +} + +// Miscellaneous +let HasMasked = false, HasVL = false, IRName = "" in { + let Name = "vreinterpret_v", MaskedPolicyScheme = NonePolicy, + ManualCodegen = [{ + if (ResultType->isIntOrIntVectorTy(1) || + Ops[0]->getType()->isIntOrIntVectorTy(1)) { + assert(isa<ScalableVectorType>(ResultType) && + isa<ScalableVectorType>(Ops[0]->getType())); + + LLVMContext &Context = CGM.getLLVMContext(); + ScalableVectorType *Boolean64Ty = + ScalableVectorType::get(llvm::Type::getInt1Ty(Context), 64); + + if (ResultType->isIntOrIntVectorTy(1)) { + // Casting from m1 vector integer -> vector boolean + // Ex: <vscale x 8 x i8> + // --(bitcast)--------> <vscale x 64 x i1> + // --(vector_extract)-> <vscale x 8 x i1> + llvm::Value *BitCast = Builder.CreateBitCast(Ops[0], Boolean64Ty); + return Builder.CreateExtractVector(ResultType, BitCast, + ConstantInt::get(Int64Ty, 0)); + } else { + // Casting from vector boolean -> m1 vector integer + // Ex: <vscale x 1 x i1> + // --(vector_insert)-> <vscale x 64 x i1> + // --(bitcast)-------> <vscale x 8 x i8> + llvm::Value *Boolean64Val = + Builder.CreateInsertVector(Boolean64Ty, + llvm::PoisonValue::get(Boolean64Ty), + Ops[0], + ConstantInt::get(Int64Ty, 0)); + return Builder.CreateBitCast(Boolean64Val, ResultType); + } + } + return Builder.CreateBitCast(Ops[0], ResultType); + }] in { + // Reinterpret between different type under the same SEW and LMUL + def vreinterpret_i_u : RVVBuiltin<"Uvv", "vUv", "csil", "v">; + def vreinterpret_i_f : RVVBuiltin<"Fvv", "vFv", "il", "v">; + def vreinterpret_u_i : RVVBuiltin<"vUv", "Uvv", "csil", "Uv">; + def vreinterpret_u_f : RVVBuiltin<"FvUv", "UvFv", "il", "Uv">; + def vreinterpret_f_i : RVVBuiltin<"vFv", "Fvv", "il", "Fv">; + def vreinterpret_f_u : RVVBuiltin<"UvFv", "FvUv", "il", "Fv">; + let RequiredFeatures = ["Zvfhmin"] in { + def vreinterpret_i_h : RVVBuiltin<"Fvv", "vFv", "s", "v">; + def vreinterpret_u_h : RVVBuiltin<"FvUv", "UvFv", "s", "Uv">; + def vreinterpret_h_i : RVVBuiltin<"vFv", "Fvv", "s", "Fv">; + def vreinterpret_h_u : RVVBuiltin<"UvFv", "FvUv", "s", "Fv">; + } + + // Reinterpret between different SEW under the same LMUL + foreach dst_sew = ["(FixedSEW:8)", "(FixedSEW:16)", "(FixedSEW:32)", + "(FixedSEW:64)"] in { + def vreinterpret_i_ # dst_sew : RVVBuiltin<"v" # dst_sew # "v", + dst_sew # "vv", "csil", dst_sew # "v">; + def vreinterpret_u_ # dst_sew : RVVBuiltin<"Uv" # dst_sew # "Uv", + dst_sew # "UvUv", "csil", dst_sew # "Uv">; + } + + // Existing users of FixedSEW - the reinterpretation between different SEW + // and same LMUL has the implicit assumption that if FixedSEW is set to the + // given element width, then the type will be identified as invalid, thus + // skipping definition of reinterpret of SEW=8 to SEW=8. However this blocks + // our usage here of defining all possible combinations of a fixed SEW to + // any boolean. So we need to separately define SEW=8 here. + // Reinterpret from LMUL=1 integer type to vector boolean type + def vreintrepret_m1_b8_signed : + RVVBuiltin<"Svm", + "mSv", + "c", "m">; + def vreintrepret_m1_b8_usigned : + RVVBuiltin<"USvm", + "mUSv", + "c", "m">; + + // Reinterpret from vector boolean type to LMUL=1 integer type + def vreintrepret_b8_m1_signed : + RVVBuiltin<"mSv", + "Svm", + "c", "Sv">; + def vreintrepret_b8_m1_usigned : + RVVBuiltin<"mUSv", + "USvm", + "c", "USv">; + + foreach dst_sew = ["16", "32", "64"] in { + // Reinterpret from LMUL=1 integer type to vector boolean type + def vreinterpret_m1_b # dst_sew # _signed: + RVVBuiltin<"(FixedSEW:" # dst_sew # ")Svm", + "m(FixedSEW:" # dst_sew # ")Sv", + "c", "m">; + def vreinterpret_m1_b # dst_sew # _unsigned: + RVVBuiltin<"(FixedSEW:" # dst_sew # ")USvm", + "m(FixedSEW:" # dst_sew # ")USv", + "c", "m">; + // Reinterpret from vector boolean type to LMUL=1 integer type + def vreinterpret_b # dst_sew # _m1_signed: + RVVBuiltin<"m(FixedSEW:" # dst_sew # ")Sv", + "(FixedSEW:" # dst_sew # ")Svm", + "c", "(FixedSEW:" # dst_sew # ")Sv">; + def vreinterpret_b # dst_sew # _m1_unsigned: + RVVBuiltin<"m(FixedSEW:" # dst_sew # ")USv", + "(FixedSEW:" # dst_sew # ")USvm", + "c", "(FixedSEW:" # dst_sew # ")USv">; + } + } + + let Name = "vundefined", SupportOverloading = false, + MaskedPolicyScheme = NonePolicy, + ManualCodegen = [{ + return llvm::PoisonValue::get(ResultType); + }] in { + def vundefined : RVVBuiltin<"v", "v", "csilxfd">; + def vundefined_u : RVVBuiltin<"Uv", "Uv", "csil">; + + foreach nf = NFList in { + let NF = nf in { + defvar T = "(Tuple:" # nf # ")"; + def : RVVBuiltin<T # "v", T # "v", "csilxfd">; + def : RVVBuiltin<T # "Uv", T # "Uv", "csil">; + } + } + + } + + // LMUL truncation + // C/C++ Operand: VecTy, IR Operand: VecTy, Index + let Name = "vlmul_trunc_v", OverloadedName = "vlmul_trunc", + MaskedPolicyScheme = NonePolicy, + ManualCodegen = [{ { + return Builder.CreateExtractVector(ResultType, Ops[0], + ConstantInt::get(Int64Ty, 0)); + } }] in { + foreach dst_lmul = ["(SFixedLog2LMUL:-3)", "(SFixedLog2LMUL:-2)", "(SFixedLog2LMUL:-1)", + "(SFixedLog2LMUL:0)", "(SFixedLog2LMUL:1)", "(SFixedLog2LMUL:2)"] in { + def vlmul_trunc # dst_lmul : RVVBuiltin<"v" # dst_lmul # "v", + dst_lmul # "vv", "csilxfd", dst_lmul # "v">; + def vlmul_trunc_u # dst_lmul : RVVBuiltin<"Uv" # dst_lmul # "Uv", + dst_lmul # "UvUv", "csil", dst_lmul # "Uv">; + } + } + + // LMUL extension + // C/C++ Operand: SubVecTy, IR Operand: VecTy, SubVecTy, Index + let Name = "vlmul_ext_v", OverloadedName = "vlmul_ext", + MaskedPolicyScheme = NonePolicy, + ManualCodegen = [{ + return Builder.CreateInsertVector(ResultType, + llvm::PoisonValue::get(ResultType), + Ops[0], ConstantInt::get(Int64Ty, 0)); + }] in { + foreach dst_lmul = ["(LFixedLog2LMUL:-2)", "(LFixedLog2LMUL:-1)", "(LFixedLog2LMUL:-0)", + "(LFixedLog2LMUL:1)", "(LFixedLog2LMUL:2)", "(LFixedLog2LMUL:3)"] in { + def vlmul_ext # dst_lmul : RVVBuiltin<"v" # dst_lmul # "v", + dst_lmul # "vv", "csilxfd", dst_lmul # "v">; + def vlmul_ext_u # dst_lmul : RVVBuiltin<"Uv" # dst_lmul # "Uv", + dst_lmul # "UvUv", "csil", dst_lmul # "Uv">; + } + } + + let Name = "vget_v", MaskedPolicyScheme = NonePolicy, + ManualCodegen = [{ + { + if (isa<StructType>(Ops[0]->getType())) // For tuple type + // Extract value from index (operand 1) of vtuple (operand 0) + return Builder.CreateExtractValue( + Ops[0], + {(unsigned)cast<ConstantInt>(Ops[1])->getZExtValue()}); + auto *VecTy = cast<ScalableVectorType>(ResultType); + auto *OpVecTy = cast<ScalableVectorType>(Ops[0]->getType()); + // Mask to only valid indices. + unsigned MaxIndex = OpVecTy->getMinNumElements() / VecTy->getMinNumElements(); + assert(isPowerOf2_32(MaxIndex)); + Ops[1] = Builder.CreateZExt(Ops[1], Builder.getInt64Ty()); + Ops[1] = Builder.CreateAnd(Ops[1], MaxIndex - 1); + Ops[1] = Builder.CreateMul(Ops[1], + ConstantInt::get(Ops[1]->getType(), + VecTy->getMinNumElements())); + return Builder.CreateExtractVector(ResultType, Ops[0], Ops[1]); + } + }] in { + foreach dst_lmul = ["(SFixedLog2LMUL:0)", "(SFixedLog2LMUL:1)", "(SFixedLog2LMUL:2)"] in { + def : RVVBuiltin<"v" # dst_lmul # "v", dst_lmul # "vvKz", "csilxfd", dst_lmul # "v">; + def : RVVBuiltin<"Uv" # dst_lmul # "Uv", dst_lmul # "UvUvKz", "csil", dst_lmul # "Uv">; + } + foreach nf = NFList in { + defvar T = "(Tuple:" # nf # ")"; + def : RVVBuiltin<T # "vv", "v" # T # "vKz", "csilxfd", "v">; + def : RVVBuiltin<T # "UvUv", "Uv" # T # "UvKz", "csil", "Uv">; + } + } + + let Name = "vset_v", MaskedPolicyScheme = NonePolicy, + ManualCodegen = [{ + { + if (isa<StructType>(ResultType)) // For tuple type + // Insert value (operand 2) into index (operand 1) of vtuple (operand 0) + return Builder.CreateInsertValue( + Ops[0], Ops[2], + {(unsigned)cast<ConstantInt>(Ops[1])->getZExtValue()}); + auto *ResVecTy = cast<ScalableVectorType>(ResultType); + auto *VecTy = cast<ScalableVectorType>(Ops[2]->getType()); + // Mask to only valid indices. + unsigned MaxIndex = ResVecTy->getMinNumElements() / VecTy->getMinNumElements(); + assert(isPowerOf2_32(MaxIndex)); + Ops[1] = Builder.CreateZExt(Ops[1], Builder.getInt64Ty()); + Ops[1] = Builder.CreateAnd(Ops[1], MaxIndex - 1); + Ops[1] = Builder.CreateMul(Ops[1], + ConstantInt::get(Ops[1]->getType(), + VecTy->getMinNumElements())); + return Builder.CreateInsertVector(ResultType, Ops[0], Ops[2], Ops[1]); + } + }] in { + foreach dst_lmul = ["(LFixedLog2LMUL:1)", "(LFixedLog2LMUL:2)", "(LFixedLog2LMUL:3)"] in { + def : RVVBuiltin<"v" # dst_lmul # "v", dst_lmul # "v" # dst_lmul # "vKzv", "csilxfd">; + def : RVVBuiltin<"Uv" # dst_lmul # "Uv", dst_lmul # "Uv" # dst_lmul #"UvKzUv", "csil">; + } + foreach nf = NFList in { + defvar T = "(Tuple:" # nf # ")"; + def : RVVBuiltin<"v" # T # "v", T # "v" # T # "vKzv", "csilxfd">; + def : RVVBuiltin<"Uv" # T # "Uv", T # "Uv" # T # "UvKzUv", "csil">; + } + } + + let Name = "vcreate_v", + UnMaskedPolicyScheme = NonePolicy, + MaskedPolicyScheme = NonePolicy, + SupportOverloading = false, + ManualCodegen = [{ + { + if (isa<StructType>(ResultType)) { + unsigned NF = cast<StructType>(ResultType)->getNumElements(); + llvm::Value *ReturnTuple = llvm::PoisonValue::get(ResultType); + for (unsigned I = 0; I < NF; ++I) { + ReturnTuple = Builder.CreateInsertValue(ReturnTuple, Ops[I], {I}); + } + return ReturnTuple; + } + llvm::Value *ReturnVector = llvm::PoisonValue::get(ResultType); + auto *VecTy = cast<ScalableVectorType>(Ops[0]->getType()); + for (unsigned I = 0, N = Ops.size(); I < N; ++I) { + llvm::Value *Idx = + ConstantInt::get(Builder.getInt64Ty(), + VecTy->getMinNumElements() * I); + ReturnVector = + Builder.CreateInsertVector(ResultType, ReturnVector, Ops[I], Idx); + } + return ReturnVector; + } + }] in { + + defm : RVVNonTupleVCreateBuiltin<1, [0]>; + defm : RVVNonTupleVCreateBuiltin<2, [0, 1]>; + defm : RVVNonTupleVCreateBuiltin<3, [0, 1, 2]>; + + foreach nf = NFList in { + let NF = nf in { + defvar T = "(Tuple:" # nf # ")"; + defvar V = VString<nf, /*signed=*/true>.S; + defvar UV = VString<nf, /*signed=*/false>.S; + def : RVVBuiltin<T # "v", T # "v" # V, "csilxfd">; + def : RVVBuiltin<T # "Uv", T # "Uv" # UV, "csil">; + } + } + } +} + +multiclass RVVOutBuiltinSetZvbb { + let OverloadedName = NAME in + defm "" : RVVOutBuiltinSet<NAME, "csil", [["v", "v", "vv"], + ["v", "Uv", "UvUv"]]>; +} + +multiclass RVVOutBuiltinSetZvk<bit HasVV = 1, bit HasVS = 1> { + // vaesz only has 'vs' and vgmul only has 'vv' and they do not have ambiguous + // prototypes like other zvkned instructions (e.g. vaesdf), so we don't + // need to encode the operand mnemonics into its intrinsic function name. + if HasVV then { + defvar name = NAME # !if(!eq(NAME, "vgmul"), "", "_vv"); + let OverloadedName = name in + defm "" : RVVOutBuiltinSet<NAME # "_vv", "i", + [["vv", "Uv", "UvUvUv"]]>; + } + + if HasVS then { + foreach vs2_lmul = ["(SEFixedLog2LMUL:-1)", "(SEFixedLog2LMUL:0)", + "(SEFixedLog2LMUL:1)", "(SEFixedLog2LMUL:2)", + "(SEFixedLog2LMUL:3)"] in { + defvar name = NAME # !if(!eq(NAME, "vaesz"), "", "_vs"); + let OverloadedName = name, IRName = NAME # "_vs", Name = NAME # "_vs", + IntrinsicTypes = [-1, 1] in + def NAME # vs2_lmul + : RVVBuiltin<vs2_lmul # "UvUv", "UvUv" # vs2_lmul # "Uv", "i">; + } + } +} + +multiclass RVVOutOp2BuiltinSetVVZvk<string type_range = "i"> + : RVVOutOp2BuiltinSet<NAME, type_range, [["vv", "Uv", "UvUvUvUv"]]>; + +multiclass RVVOutOp2BuiltinSetVIZvk<string type_range = "i"> + : RVVOutOp2BuiltinSet<NAME, type_range, [["vi", "Uv", "UvUvUvKz"]]>; + +multiclass RVVSignedWidenBinBuiltinSetVwsll + : RVVWidenBuiltinSet<NAME, "csi", + [["vv", "Uw", "UwUvUv"], + ["vx", "Uw", "UwUvz"]]>; + +let UnMaskedPolicyScheme = HasPassthruOperand in { + // zvkb + let RequiredFeatures = ["Zvkb", "Experimental"] in { + defm vandn : RVVUnsignedBinBuiltinSet; + defm vbrev8 : RVVOutBuiltinSetZvbb; + defm vrev8 : RVVOutBuiltinSetZvbb; + defm vrol : RVVUnsignedShiftBuiltinSet; + defm vror : RVVUnsignedShiftBuiltinSet; + } + + // zvbb + let RequiredFeatures = ["Zvbb", "Experimental"] in { + defm vbrev : RVVOutBuiltinSetZvbb; + defm vclz : RVVOutBuiltinSetZvbb; + defm vctz : RVVOutBuiltinSetZvbb; + defm vcpopv : RVVOutBuiltinSetZvbb; + let OverloadedName = "vwsll" in + defm vwsll : RVVSignedWidenBinBuiltinSetVwsll; + } + + // zvbc + let RequiredFeatures = ["Zvbc", "Experimental"] in { + defm vclmul : RVVInt64BinBuiltinSet; + defm vclmulh : RVVInt64BinBuiltinSet; + } +} + +let UnMaskedPolicyScheme = HasPolicyOperand, HasMasked = false in { + // zvkg + let RequiredFeatures = ["Zvkg", "Experimental"] in { + defm vghsh : RVVOutOp2BuiltinSetVVZvk; + defm vgmul : RVVOutBuiltinSetZvk<HasVV=1, HasVS=0>; + } + + // zvkned + let RequiredFeatures = ["Zvkned", "Experimental"] in { + defm vaesdf : RVVOutBuiltinSetZvk; + defm vaesdm : RVVOutBuiltinSetZvk; + defm vaesef : RVVOutBuiltinSetZvk; + defm vaesem : RVVOutBuiltinSetZvk; + let UnMaskedPolicyScheme = HasPassthruOperand in + defm vaeskf1 : RVVOutOp1BuiltinSet<"vaeskf1", "i", [["vi", "Uv", "UvUvKz"]]>; + defm vaeskf2 : RVVOutOp2BuiltinSetVIZvk; + defm vaesz : RVVOutBuiltinSetZvk<HasVV=0>; + } + + // zvknha + let RequiredFeatures = ["Zvknha", "Experimental"] in { + defm vsha2ch : RVVOutOp2BuiltinSetVVZvk<"i">; + defm vsha2cl : RVVOutOp2BuiltinSetVVZvk<"i">; + defm vsha2ms : RVVOutOp2BuiltinSetVVZvk<"i">; + } + + // zvknhb + let RequiredFeatures = ["Zvknhb", "Experimental"] in { + defm vsha2ch : RVVOutOp2BuiltinSetVVZvk<"il">; + defm vsha2cl : RVVOutOp2BuiltinSetVVZvk<"il">; + defm vsha2ms : RVVOutOp2BuiltinSetVVZvk<"il">; + } + + // zvksed + let RequiredFeatures = ["Zvksed", "Experimental"] in { + let UnMaskedPolicyScheme = HasPassthruOperand in + defm vsm4k : RVVOutOp1BuiltinSet<"vsm4k", "i", [["vi", "Uv", "UvUvKz"]]>; + defm vsm4r : RVVOutBuiltinSetZvk; + } + + // zvksh + let RequiredFeatures = ["Zvksh", "Experimental"] in { + defm vsm3c : RVVOutOp2BuiltinSetVIZvk; + let UnMaskedPolicyScheme = HasPassthruOperand in + defm vsm3me : RVVOutOp1BuiltinSet<"vsm3me", "i", [["vv", "Uv", "UvUvUv"]]>; + } +} |