diff options
Diffstat (limited to 'contrib/llvm-project/llvm/lib/IR/IRBuilder.cpp')
| -rw-r--r-- | contrib/llvm-project/llvm/lib/IR/IRBuilder.cpp | 1305 |
1 files changed, 1305 insertions, 0 deletions
diff --git a/contrib/llvm-project/llvm/lib/IR/IRBuilder.cpp b/contrib/llvm-project/llvm/lib/IR/IRBuilder.cpp new file mode 100644 index 000000000000..d0c622fe2389 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/IR/IRBuilder.cpp @@ -0,0 +1,1305 @@ +//===- IRBuilder.cpp - Builder for LLVM Instrs ----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the IRBuilder class, which is used as a convenient way +// to create LLVM instructions with a consistent and simplified interface. +// +//===----------------------------------------------------------------------===// + +#include "llvm/IR/IRBuilder.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/None.h" +#include "llvm/IR/Constant.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/DebugInfoMetadata.h" +#include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/GlobalValue.h" +#include "llvm/IR/GlobalVariable.h" +#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/Intrinsics.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/NoFolder.h" +#include "llvm/IR/Operator.h" +#include "llvm/IR/Statepoint.h" +#include "llvm/IR/Type.h" +#include "llvm/IR/Value.h" +#include "llvm/Support/Casting.h" +#include <cassert> +#include <cstdint> +#include <vector> + +using namespace llvm; + +/// CreateGlobalString - Make a new global variable with an initializer that +/// has array of i8 type filled in with the nul terminated string value +/// specified. If Name is specified, it is the name of the global variable +/// created. +GlobalVariable *IRBuilderBase::CreateGlobalString(StringRef Str, + const Twine &Name, + unsigned AddressSpace, + Module *M) { + Constant *StrConstant = ConstantDataArray::getString(Context, Str); + if (!M) + M = BB->getParent()->getParent(); + auto *GV = new GlobalVariable( + *M, StrConstant->getType(), true, GlobalValue::PrivateLinkage, + StrConstant, Name, nullptr, GlobalVariable::NotThreadLocal, AddressSpace); + GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global); + GV->setAlignment(Align(1)); + return GV; +} + +Type *IRBuilderBase::getCurrentFunctionReturnType() const { + assert(BB && BB->getParent() && "No current function!"); + return BB->getParent()->getReturnType(); +} + +Value *IRBuilderBase::getCastedInt8PtrValue(Value *Ptr) { + auto *PT = cast<PointerType>(Ptr->getType()); + if (PT->isOpaqueOrPointeeTypeMatches(getInt8Ty())) + return Ptr; + + // Otherwise, we need to insert a bitcast. + return CreateBitCast(Ptr, getInt8PtrTy(PT->getAddressSpace())); +} + +DebugLoc IRBuilderBase::getCurrentDebugLocation() const { + for (auto &KV : MetadataToCopy) + if (KV.first == LLVMContext::MD_dbg) + return {cast<DILocation>(KV.second)}; + + return {}; +} +void IRBuilderBase::SetInstDebugLocation(Instruction *I) const { + for (const auto &KV : MetadataToCopy) + if (KV.first == LLVMContext::MD_dbg) { + I->setDebugLoc(DebugLoc(KV.second)); + return; + } +} + +static CallInst *createCallHelper(Function *Callee, ArrayRef<Value *> Ops, + IRBuilderBase *Builder, + const Twine &Name = "", + Instruction *FMFSource = nullptr, + ArrayRef<OperandBundleDef> OpBundles = {}) { + CallInst *CI = Builder->CreateCall(Callee, Ops, OpBundles, Name); + if (FMFSource) + CI->copyFastMathFlags(FMFSource); + return CI; +} + +Value *IRBuilderBase::CreateVScale(Constant *Scaling, const Twine &Name) { + assert(isa<ConstantInt>(Scaling) && "Expected constant integer"); + if (cast<ConstantInt>(Scaling)->isZero()) + return Scaling; + Module *M = GetInsertBlock()->getParent()->getParent(); + Function *TheFn = + Intrinsic::getDeclaration(M, Intrinsic::vscale, {Scaling->getType()}); + CallInst *CI = createCallHelper(TheFn, {}, this, Name); + return cast<ConstantInt>(Scaling)->getSExtValue() == 1 + ? CI + : CreateMul(CI, Scaling); +} + +Value *IRBuilderBase::CreateStepVector(Type *DstType, const Twine &Name) { + Type *STy = DstType->getScalarType(); + if (isa<ScalableVectorType>(DstType)) { + Type *StepVecType = DstType; + // TODO: We expect this special case (element type < 8 bits) to be + // temporary - once the intrinsic properly supports < 8 bits this code + // can be removed. + if (STy->getScalarSizeInBits() < 8) + StepVecType = + VectorType::get(getInt8Ty(), cast<ScalableVectorType>(DstType)); + Value *Res = CreateIntrinsic(Intrinsic::experimental_stepvector, + {StepVecType}, {}, nullptr, Name); + if (StepVecType != DstType) + Res = CreateTrunc(Res, DstType); + return Res; + } + + unsigned NumEls = cast<FixedVectorType>(DstType)->getNumElements(); + + // Create a vector of consecutive numbers from zero to VF. + SmallVector<Constant *, 8> Indices; + for (unsigned i = 0; i < NumEls; ++i) + Indices.push_back(ConstantInt::get(STy, i)); + + // Add the consecutive indices to the vector value. + return ConstantVector::get(Indices); +} + +CallInst *IRBuilderBase::CreateMemSet(Value *Ptr, Value *Val, Value *Size, + MaybeAlign Align, bool isVolatile, + MDNode *TBAATag, MDNode *ScopeTag, + MDNode *NoAliasTag) { + Ptr = getCastedInt8PtrValue(Ptr); + Value *Ops[] = {Ptr, Val, Size, getInt1(isVolatile)}; + Type *Tys[] = { Ptr->getType(), Size->getType() }; + Module *M = BB->getParent()->getParent(); + Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memset, Tys); + + CallInst *CI = createCallHelper(TheFn, Ops, this); + + if (Align) + cast<MemSetInst>(CI)->setDestAlignment(*Align); + + // Set the TBAA info if present. + if (TBAATag) + CI->setMetadata(LLVMContext::MD_tbaa, TBAATag); + + if (ScopeTag) + CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag); + + if (NoAliasTag) + CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag); + + return CI; +} + +CallInst *IRBuilderBase::CreateMemSetInline(Value *Dst, MaybeAlign DstAlign, + Value *Val, Value *Size, + bool IsVolatile, MDNode *TBAATag, + MDNode *ScopeTag, + MDNode *NoAliasTag) { + Dst = getCastedInt8PtrValue(Dst); + Value *Ops[] = {Dst, Val, Size, getInt1(IsVolatile)}; + Type *Tys[] = {Dst->getType(), Size->getType()}; + Module *M = BB->getParent()->getParent(); + Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memset_inline, Tys); + + CallInst *CI = createCallHelper(TheFn, Ops, this); + + if (DstAlign) + cast<MemSetInlineInst>(CI)->setDestAlignment(*DstAlign); + + // Set the TBAA info if present. + if (TBAATag) + CI->setMetadata(LLVMContext::MD_tbaa, TBAATag); + + if (ScopeTag) + CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag); + + if (NoAliasTag) + CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag); + + return CI; +} + +CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemSet( + Value *Ptr, Value *Val, Value *Size, Align Alignment, uint32_t ElementSize, + MDNode *TBAATag, MDNode *ScopeTag, MDNode *NoAliasTag) { + + Ptr = getCastedInt8PtrValue(Ptr); + Value *Ops[] = {Ptr, Val, Size, getInt32(ElementSize)}; + Type *Tys[] = {Ptr->getType(), Size->getType()}; + Module *M = BB->getParent()->getParent(); + Function *TheFn = Intrinsic::getDeclaration( + M, Intrinsic::memset_element_unordered_atomic, Tys); + + CallInst *CI = createCallHelper(TheFn, Ops, this); + + cast<AtomicMemSetInst>(CI)->setDestAlignment(Alignment); + + // Set the TBAA info if present. + if (TBAATag) + CI->setMetadata(LLVMContext::MD_tbaa, TBAATag); + + if (ScopeTag) + CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag); + + if (NoAliasTag) + CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag); + + return CI; +} + +CallInst *IRBuilderBase::CreateMemTransferInst( + Intrinsic::ID IntrID, Value *Dst, MaybeAlign DstAlign, Value *Src, + MaybeAlign SrcAlign, Value *Size, bool isVolatile, MDNode *TBAATag, + MDNode *TBAAStructTag, MDNode *ScopeTag, MDNode *NoAliasTag) { + Dst = getCastedInt8PtrValue(Dst); + Src = getCastedInt8PtrValue(Src); + + Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)}; + Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() }; + Module *M = BB->getParent()->getParent(); + Function *TheFn = Intrinsic::getDeclaration(M, IntrID, Tys); + + CallInst *CI = createCallHelper(TheFn, Ops, this); + + auto* MCI = cast<MemTransferInst>(CI); + if (DstAlign) + MCI->setDestAlignment(*DstAlign); + if (SrcAlign) + MCI->setSourceAlignment(*SrcAlign); + + // Set the TBAA info if present. + if (TBAATag) + CI->setMetadata(LLVMContext::MD_tbaa, TBAATag); + + // Set the TBAA Struct info if present. + if (TBAAStructTag) + CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag); + + if (ScopeTag) + CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag); + + if (NoAliasTag) + CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag); + + return CI; +} + +CallInst *IRBuilderBase::CreateMemCpyInline( + Value *Dst, MaybeAlign DstAlign, Value *Src, MaybeAlign SrcAlign, + Value *Size, bool IsVolatile, MDNode *TBAATag, MDNode *TBAAStructTag, + MDNode *ScopeTag, MDNode *NoAliasTag) { + Dst = getCastedInt8PtrValue(Dst); + Src = getCastedInt8PtrValue(Src); + + Value *Ops[] = {Dst, Src, Size, getInt1(IsVolatile)}; + Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()}; + Function *F = BB->getParent(); + Module *M = F->getParent(); + Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memcpy_inline, Tys); + + CallInst *CI = createCallHelper(TheFn, Ops, this); + + auto *MCI = cast<MemCpyInlineInst>(CI); + if (DstAlign) + MCI->setDestAlignment(*DstAlign); + if (SrcAlign) + MCI->setSourceAlignment(*SrcAlign); + + // Set the TBAA info if present. + if (TBAATag) + MCI->setMetadata(LLVMContext::MD_tbaa, TBAATag); + + // Set the TBAA Struct info if present. + if (TBAAStructTag) + MCI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag); + + if (ScopeTag) + MCI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag); + + if (NoAliasTag) + MCI->setMetadata(LLVMContext::MD_noalias, NoAliasTag); + + return CI; +} + +CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemCpy( + Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size, + uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag, + MDNode *ScopeTag, MDNode *NoAliasTag) { + assert(DstAlign >= ElementSize && + "Pointer alignment must be at least element size"); + assert(SrcAlign >= ElementSize && + "Pointer alignment must be at least element size"); + Dst = getCastedInt8PtrValue(Dst); + Src = getCastedInt8PtrValue(Src); + + Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)}; + Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()}; + Module *M = BB->getParent()->getParent(); + Function *TheFn = Intrinsic::getDeclaration( + M, Intrinsic::memcpy_element_unordered_atomic, Tys); + + CallInst *CI = createCallHelper(TheFn, Ops, this); + + // Set the alignment of the pointer args. + auto *AMCI = cast<AtomicMemCpyInst>(CI); + AMCI->setDestAlignment(DstAlign); + AMCI->setSourceAlignment(SrcAlign); + + // Set the TBAA info if present. + if (TBAATag) + CI->setMetadata(LLVMContext::MD_tbaa, TBAATag); + + // Set the TBAA Struct info if present. + if (TBAAStructTag) + CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag); + + if (ScopeTag) + CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag); + + if (NoAliasTag) + CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag); + + return CI; +} + +CallInst *IRBuilderBase::CreateMemMove(Value *Dst, MaybeAlign DstAlign, + Value *Src, MaybeAlign SrcAlign, + Value *Size, bool isVolatile, + MDNode *TBAATag, MDNode *ScopeTag, + MDNode *NoAliasTag) { + Dst = getCastedInt8PtrValue(Dst); + Src = getCastedInt8PtrValue(Src); + + Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)}; + Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() }; + Module *M = BB->getParent()->getParent(); + Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memmove, Tys); + + CallInst *CI = createCallHelper(TheFn, Ops, this); + + auto *MMI = cast<MemMoveInst>(CI); + if (DstAlign) + MMI->setDestAlignment(*DstAlign); + if (SrcAlign) + MMI->setSourceAlignment(*SrcAlign); + + // Set the TBAA info if present. + if (TBAATag) + CI->setMetadata(LLVMContext::MD_tbaa, TBAATag); + + if (ScopeTag) + CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag); + + if (NoAliasTag) + CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag); + + return CI; +} + +CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemMove( + Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size, + uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag, + MDNode *ScopeTag, MDNode *NoAliasTag) { + assert(DstAlign >= ElementSize && + "Pointer alignment must be at least element size"); + assert(SrcAlign >= ElementSize && + "Pointer alignment must be at least element size"); + Dst = getCastedInt8PtrValue(Dst); + Src = getCastedInt8PtrValue(Src); + + Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)}; + Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()}; + Module *M = BB->getParent()->getParent(); + Function *TheFn = Intrinsic::getDeclaration( + M, Intrinsic::memmove_element_unordered_atomic, Tys); + + CallInst *CI = createCallHelper(TheFn, Ops, this); + + // Set the alignment of the pointer args. + CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), DstAlign)); + CI->addParamAttr(1, Attribute::getWithAlignment(CI->getContext(), SrcAlign)); + + // Set the TBAA info if present. + if (TBAATag) + CI->setMetadata(LLVMContext::MD_tbaa, TBAATag); + + // Set the TBAA Struct info if present. + if (TBAAStructTag) + CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag); + + if (ScopeTag) + CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag); + + if (NoAliasTag) + CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag); + + return CI; +} + +static CallInst *getReductionIntrinsic(IRBuilderBase *Builder, Intrinsic::ID ID, + Value *Src) { + Module *M = Builder->GetInsertBlock()->getParent()->getParent(); + Value *Ops[] = {Src}; + Type *Tys[] = { Src->getType() }; + auto Decl = Intrinsic::getDeclaration(M, ID, Tys); + return createCallHelper(Decl, Ops, Builder); +} + +CallInst *IRBuilderBase::CreateFAddReduce(Value *Acc, Value *Src) { + Module *M = GetInsertBlock()->getParent()->getParent(); + Value *Ops[] = {Acc, Src}; + auto Decl = Intrinsic::getDeclaration(M, Intrinsic::vector_reduce_fadd, + {Src->getType()}); + return createCallHelper(Decl, Ops, this); +} + +CallInst *IRBuilderBase::CreateFMulReduce(Value *Acc, Value *Src) { + Module *M = GetInsertBlock()->getParent()->getParent(); + Value *Ops[] = {Acc, Src}; + auto Decl = Intrinsic::getDeclaration(M, Intrinsic::vector_reduce_fmul, + {Src->getType()}); + return createCallHelper(Decl, Ops, this); +} + +CallInst *IRBuilderBase::CreateAddReduce(Value *Src) { + return getReductionIntrinsic(this, Intrinsic::vector_reduce_add, Src); +} + +CallInst *IRBuilderBase::CreateMulReduce(Value *Src) { + return getReductionIntrinsic(this, Intrinsic::vector_reduce_mul, Src); +} + +CallInst *IRBuilderBase::CreateAndReduce(Value *Src) { + return getReductionIntrinsic(this, Intrinsic::vector_reduce_and, Src); +} + +CallInst *IRBuilderBase::CreateOrReduce(Value *Src) { + return getReductionIntrinsic(this, Intrinsic::vector_reduce_or, Src); +} + +CallInst *IRBuilderBase::CreateXorReduce(Value *Src) { + return getReductionIntrinsic(this, Intrinsic::vector_reduce_xor, Src); +} + +CallInst *IRBuilderBase::CreateIntMaxReduce(Value *Src, bool IsSigned) { + auto ID = + IsSigned ? Intrinsic::vector_reduce_smax : Intrinsic::vector_reduce_umax; + return getReductionIntrinsic(this, ID, Src); +} + +CallInst *IRBuilderBase::CreateIntMinReduce(Value *Src, bool IsSigned) { + auto ID = + IsSigned ? Intrinsic::vector_reduce_smin : Intrinsic::vector_reduce_umin; + return getReductionIntrinsic(this, ID, Src); +} + +CallInst *IRBuilderBase::CreateFPMaxReduce(Value *Src) { + return getReductionIntrinsic(this, Intrinsic::vector_reduce_fmax, Src); +} + +CallInst *IRBuilderBase::CreateFPMinReduce(Value *Src) { + return getReductionIntrinsic(this, Intrinsic::vector_reduce_fmin, Src); +} + +CallInst *IRBuilderBase::CreateLifetimeStart(Value *Ptr, ConstantInt *Size) { + assert(isa<PointerType>(Ptr->getType()) && + "lifetime.start only applies to pointers."); + Ptr = getCastedInt8PtrValue(Ptr); + if (!Size) + Size = getInt64(-1); + else + assert(Size->getType() == getInt64Ty() && + "lifetime.start requires the size to be an i64"); + Value *Ops[] = { Size, Ptr }; + Module *M = BB->getParent()->getParent(); + Function *TheFn = + Intrinsic::getDeclaration(M, Intrinsic::lifetime_start, {Ptr->getType()}); + return createCallHelper(TheFn, Ops, this); +} + +CallInst *IRBuilderBase::CreateLifetimeEnd(Value *Ptr, ConstantInt *Size) { + assert(isa<PointerType>(Ptr->getType()) && + "lifetime.end only applies to pointers."); + Ptr = getCastedInt8PtrValue(Ptr); + if (!Size) + Size = getInt64(-1); + else + assert(Size->getType() == getInt64Ty() && + "lifetime.end requires the size to be an i64"); + Value *Ops[] = { Size, Ptr }; + Module *M = BB->getParent()->getParent(); + Function *TheFn = + Intrinsic::getDeclaration(M, Intrinsic::lifetime_end, {Ptr->getType()}); + return createCallHelper(TheFn, Ops, this); +} + +CallInst *IRBuilderBase::CreateInvariantStart(Value *Ptr, ConstantInt *Size) { + + assert(isa<PointerType>(Ptr->getType()) && + "invariant.start only applies to pointers."); + Ptr = getCastedInt8PtrValue(Ptr); + if (!Size) + Size = getInt64(-1); + else + assert(Size->getType() == getInt64Ty() && + "invariant.start requires the size to be an i64"); + + Value *Ops[] = {Size, Ptr}; + // Fill in the single overloaded type: memory object type. + Type *ObjectPtr[1] = {Ptr->getType()}; + Module *M = BB->getParent()->getParent(); + Function *TheFn = + Intrinsic::getDeclaration(M, Intrinsic::invariant_start, ObjectPtr); + return createCallHelper(TheFn, Ops, this); +} + +CallInst * +IRBuilderBase::CreateAssumption(Value *Cond, + ArrayRef<OperandBundleDef> OpBundles) { + assert(Cond->getType() == getInt1Ty() && + "an assumption condition must be of type i1"); + + Value *Ops[] = { Cond }; + Module *M = BB->getParent()->getParent(); + Function *FnAssume = Intrinsic::getDeclaration(M, Intrinsic::assume); + return createCallHelper(FnAssume, Ops, this, "", nullptr, OpBundles); +} + +Instruction *IRBuilderBase::CreateNoAliasScopeDeclaration(Value *Scope) { + Module *M = BB->getModule(); + auto *FnIntrinsic = Intrinsic::getDeclaration( + M, Intrinsic::experimental_noalias_scope_decl, {}); + return createCallHelper(FnIntrinsic, {Scope}, this); +} + +/// Create a call to a Masked Load intrinsic. +/// \p Ty - vector type to load +/// \p Ptr - base pointer for the load +/// \p Alignment - alignment of the source location +/// \p Mask - vector of booleans which indicates what vector lanes should +/// be accessed in memory +/// \p PassThru - pass-through value that is used to fill the masked-off lanes +/// of the result +/// \p Name - name of the result variable +CallInst *IRBuilderBase::CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment, + Value *Mask, Value *PassThru, + const Twine &Name) { + auto *PtrTy = cast<PointerType>(Ptr->getType()); + assert(Ty->isVectorTy() && "Type should be vector"); + assert(PtrTy->isOpaqueOrPointeeTypeMatches(Ty) && "Wrong element type"); + assert(Mask && "Mask should not be all-ones (null)"); + if (!PassThru) + PassThru = UndefValue::get(Ty); + Type *OverloadedTypes[] = { Ty, PtrTy }; + Value *Ops[] = {Ptr, getInt32(Alignment.value()), Mask, PassThru}; + return CreateMaskedIntrinsic(Intrinsic::masked_load, Ops, + OverloadedTypes, Name); +} + +/// Create a call to a Masked Store intrinsic. +/// \p Val - data to be stored, +/// \p Ptr - base pointer for the store +/// \p Alignment - alignment of the destination location +/// \p Mask - vector of booleans which indicates what vector lanes should +/// be accessed in memory +CallInst *IRBuilderBase::CreateMaskedStore(Value *Val, Value *Ptr, + Align Alignment, Value *Mask) { + auto *PtrTy = cast<PointerType>(Ptr->getType()); + Type *DataTy = Val->getType(); + assert(DataTy->isVectorTy() && "Val should be a vector"); + assert(PtrTy->isOpaqueOrPointeeTypeMatches(DataTy) && "Wrong element type"); + assert(Mask && "Mask should not be all-ones (null)"); + Type *OverloadedTypes[] = { DataTy, PtrTy }; + Value *Ops[] = {Val, Ptr, getInt32(Alignment.value()), Mask}; + return CreateMaskedIntrinsic(Intrinsic::masked_store, Ops, OverloadedTypes); +} + +/// Create a call to a Masked intrinsic, with given intrinsic Id, +/// an array of operands - Ops, and an array of overloaded types - +/// OverloadedTypes. +CallInst *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id, + ArrayRef<Value *> Ops, + ArrayRef<Type *> OverloadedTypes, + const Twine &Name) { + Module *M = BB->getParent()->getParent(); + Function *TheFn = Intrinsic::getDeclaration(M, Id, OverloadedTypes); + return createCallHelper(TheFn, Ops, this, Name); +} + +/// Create a call to a Masked Gather intrinsic. +/// \p Ty - vector type to gather +/// \p Ptrs - vector of pointers for loading +/// \p Align - alignment for one element +/// \p Mask - vector of booleans which indicates what vector lanes should +/// be accessed in memory +/// \p PassThru - pass-through value that is used to fill the masked-off lanes +/// of the result +/// \p Name - name of the result variable +CallInst *IRBuilderBase::CreateMaskedGather(Type *Ty, Value *Ptrs, + Align Alignment, Value *Mask, + Value *PassThru, + const Twine &Name) { + auto *VecTy = cast<VectorType>(Ty); + ElementCount NumElts = VecTy->getElementCount(); + auto *PtrsTy = cast<VectorType>(Ptrs->getType()); + assert(cast<PointerType>(PtrsTy->getElementType()) + ->isOpaqueOrPointeeTypeMatches( + cast<VectorType>(Ty)->getElementType()) && + "Element type mismatch"); + assert(NumElts == PtrsTy->getElementCount() && "Element count mismatch"); + + if (!Mask) + Mask = Constant::getAllOnesValue( + VectorType::get(Type::getInt1Ty(Context), NumElts)); + + if (!PassThru) + PassThru = UndefValue::get(Ty); + + Type *OverloadedTypes[] = {Ty, PtrsTy}; + Value *Ops[] = {Ptrs, getInt32(Alignment.value()), Mask, PassThru}; + + // We specify only one type when we create this intrinsic. Types of other + // arguments are derived from this type. + return CreateMaskedIntrinsic(Intrinsic::masked_gather, Ops, OverloadedTypes, + Name); +} + +/// Create a call to a Masked Scatter intrinsic. +/// \p Data - data to be stored, +/// \p Ptrs - the vector of pointers, where the \p Data elements should be +/// stored +/// \p Align - alignment for one element +/// \p Mask - vector of booleans which indicates what vector lanes should +/// be accessed in memory +CallInst *IRBuilderBase::CreateMaskedScatter(Value *Data, Value *Ptrs, + Align Alignment, Value *Mask) { + auto *PtrsTy = cast<VectorType>(Ptrs->getType()); + auto *DataTy = cast<VectorType>(Data->getType()); + ElementCount NumElts = PtrsTy->getElementCount(); + +#ifndef NDEBUG + auto *PtrTy = cast<PointerType>(PtrsTy->getElementType()); + assert(NumElts == DataTy->getElementCount() && + PtrTy->isOpaqueOrPointeeTypeMatches(DataTy->getElementType()) && + "Incompatible pointer and data types"); +#endif + + if (!Mask) + Mask = Constant::getAllOnesValue( + VectorType::get(Type::getInt1Ty(Context), NumElts)); + + Type *OverloadedTypes[] = {DataTy, PtrsTy}; + Value *Ops[] = {Data, Ptrs, getInt32(Alignment.value()), Mask}; + + // We specify only one type when we create this intrinsic. Types of other + // arguments are derived from this type. + return CreateMaskedIntrinsic(Intrinsic::masked_scatter, Ops, OverloadedTypes); +} + +template <typename T0> +static std::vector<Value *> +getStatepointArgs(IRBuilderBase &B, uint64_t ID, uint32_t NumPatchBytes, + Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs) { + std::vector<Value *> Args; + Args.push_back(B.getInt64(ID)); + Args.push_back(B.getInt32(NumPatchBytes)); + Args.push_back(ActualCallee); + Args.push_back(B.getInt32(CallArgs.size())); + Args.push_back(B.getInt32(Flags)); + llvm::append_range(Args, CallArgs); + // GC Transition and Deopt args are now always handled via operand bundle. + // They will be removed from the signature of gc.statepoint shortly. + Args.push_back(B.getInt32(0)); + Args.push_back(B.getInt32(0)); + // GC args are now encoded in the gc-live operand bundle + return Args; +} + +template<typename T1, typename T2, typename T3> +static std::vector<OperandBundleDef> +getStatepointBundles(Optional<ArrayRef<T1>> TransitionArgs, + Optional<ArrayRef<T2>> DeoptArgs, + ArrayRef<T3> GCArgs) { + std::vector<OperandBundleDef> Rval; + if (DeoptArgs) { + SmallVector<Value*, 16> DeoptValues; + llvm::append_range(DeoptValues, *DeoptArgs); + Rval.emplace_back("deopt", DeoptValues); + } + if (TransitionArgs) { + SmallVector<Value*, 16> TransitionValues; + llvm::append_range(TransitionValues, *TransitionArgs); + Rval.emplace_back("gc-transition", TransitionValues); + } + if (GCArgs.size()) { + SmallVector<Value*, 16> LiveValues; + llvm::append_range(LiveValues, GCArgs); + Rval.emplace_back("gc-live", LiveValues); + } + return Rval; +} + +template <typename T0, typename T1, typename T2, typename T3> +static CallInst *CreateGCStatepointCallCommon( + IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes, + FunctionCallee ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs, + Optional<ArrayRef<T1>> TransitionArgs, Optional<ArrayRef<T2>> DeoptArgs, + ArrayRef<T3> GCArgs, const Twine &Name) { + Module *M = Builder->GetInsertBlock()->getParent()->getParent(); + // Fill in the one generic type'd argument (the function is also vararg) + Function *FnStatepoint = + Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_statepoint, + {ActualCallee.getCallee()->getType()}); + + std::vector<Value *> Args = getStatepointArgs( + *Builder, ID, NumPatchBytes, ActualCallee.getCallee(), Flags, CallArgs); + + CallInst *CI = Builder->CreateCall( + FnStatepoint, Args, + getStatepointBundles(TransitionArgs, DeoptArgs, GCArgs), Name); + CI->addParamAttr(2, + Attribute::get(Builder->getContext(), Attribute::ElementType, + ActualCallee.getFunctionType())); + return CI; +} + +CallInst *IRBuilderBase::CreateGCStatepointCall( + uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee, + ArrayRef<Value *> CallArgs, Optional<ArrayRef<Value *>> DeoptArgs, + ArrayRef<Value *> GCArgs, const Twine &Name) { + return CreateGCStatepointCallCommon<Value *, Value *, Value *, Value *>( + this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None), + CallArgs, None /* No Transition Args */, DeoptArgs, GCArgs, Name); +} + +CallInst *IRBuilderBase::CreateGCStatepointCall( + uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee, + uint32_t Flags, ArrayRef<Value *> CallArgs, + Optional<ArrayRef<Use>> TransitionArgs, Optional<ArrayRef<Use>> DeoptArgs, + ArrayRef<Value *> GCArgs, const Twine &Name) { + return CreateGCStatepointCallCommon<Value *, Use, Use, Value *>( + this, ID, NumPatchBytes, ActualCallee, Flags, CallArgs, TransitionArgs, + DeoptArgs, GCArgs, Name); +} + +CallInst *IRBuilderBase::CreateGCStatepointCall( + uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee, + ArrayRef<Use> CallArgs, Optional<ArrayRef<Value *>> DeoptArgs, + ArrayRef<Value *> GCArgs, const Twine &Name) { + return CreateGCStatepointCallCommon<Use, Value *, Value *, Value *>( + this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None), + CallArgs, None, DeoptArgs, GCArgs, Name); +} + +template <typename T0, typename T1, typename T2, typename T3> +static InvokeInst *CreateGCStatepointInvokeCommon( + IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes, + FunctionCallee ActualInvokee, BasicBlock *NormalDest, + BasicBlock *UnwindDest, uint32_t Flags, ArrayRef<T0> InvokeArgs, + Optional<ArrayRef<T1>> TransitionArgs, Optional<ArrayRef<T2>> DeoptArgs, + ArrayRef<T3> GCArgs, const Twine &Name) { + Module *M = Builder->GetInsertBlock()->getParent()->getParent(); + // Fill in the one generic type'd argument (the function is also vararg) + Function *FnStatepoint = + Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_statepoint, + {ActualInvokee.getCallee()->getType()}); + + std::vector<Value *> Args = + getStatepointArgs(*Builder, ID, NumPatchBytes, ActualInvokee.getCallee(), + Flags, InvokeArgs); + + InvokeInst *II = Builder->CreateInvoke( + FnStatepoint, NormalDest, UnwindDest, Args, + getStatepointBundles(TransitionArgs, DeoptArgs, GCArgs), Name); + II->addParamAttr(2, + Attribute::get(Builder->getContext(), Attribute::ElementType, + ActualInvokee.getFunctionType())); + return II; +} + +InvokeInst *IRBuilderBase::CreateGCStatepointInvoke( + uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee, + BasicBlock *NormalDest, BasicBlock *UnwindDest, + ArrayRef<Value *> InvokeArgs, Optional<ArrayRef<Value *>> DeoptArgs, + ArrayRef<Value *> GCArgs, const Twine &Name) { + return CreateGCStatepointInvokeCommon<Value *, Value *, Value *, Value *>( + this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest, + uint32_t(StatepointFlags::None), InvokeArgs, None /* No Transition Args*/, + DeoptArgs, GCArgs, Name); +} + +InvokeInst *IRBuilderBase::CreateGCStatepointInvoke( + uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee, + BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags, + ArrayRef<Value *> InvokeArgs, Optional<ArrayRef<Use>> TransitionArgs, + Optional<ArrayRef<Use>> DeoptArgs, ArrayRef<Value *> GCArgs, + const Twine &Name) { + return CreateGCStatepointInvokeCommon<Value *, Use, Use, Value *>( + this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest, Flags, + InvokeArgs, TransitionArgs, DeoptArgs, GCArgs, Name); +} + +InvokeInst *IRBuilderBase::CreateGCStatepointInvoke( + uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee, + BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef<Use> InvokeArgs, + Optional<ArrayRef<Value *>> DeoptArgs, ArrayRef<Value *> GCArgs, + const Twine &Name) { + return CreateGCStatepointInvokeCommon<Use, Value *, Value *, Value *>( + this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest, + uint32_t(StatepointFlags::None), InvokeArgs, None, DeoptArgs, GCArgs, + Name); +} + +CallInst *IRBuilderBase::CreateGCResult(Instruction *Statepoint, + Type *ResultType, const Twine &Name) { + Intrinsic::ID ID = Intrinsic::experimental_gc_result; + Module *M = BB->getParent()->getParent(); + Type *Types[] = {ResultType}; + Function *FnGCResult = Intrinsic::getDeclaration(M, ID, Types); + + Value *Args[] = {Statepoint}; + return createCallHelper(FnGCResult, Args, this, Name); +} + +CallInst *IRBuilderBase::CreateGCRelocate(Instruction *Statepoint, + int BaseOffset, int DerivedOffset, + Type *ResultType, const Twine &Name) { + Module *M = BB->getParent()->getParent(); + Type *Types[] = {ResultType}; + Function *FnGCRelocate = + Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_relocate, Types); + + Value *Args[] = {Statepoint, getInt32(BaseOffset), getInt32(DerivedOffset)}; + return createCallHelper(FnGCRelocate, Args, this, Name); +} + +CallInst *IRBuilderBase::CreateGCGetPointerBase(Value *DerivedPtr, + const Twine &Name) { + Module *M = BB->getParent()->getParent(); + Type *PtrTy = DerivedPtr->getType(); + Function *FnGCFindBase = Intrinsic::getDeclaration( + M, Intrinsic::experimental_gc_get_pointer_base, {PtrTy, PtrTy}); + return createCallHelper(FnGCFindBase, {DerivedPtr}, this, Name); +} + +CallInst *IRBuilderBase::CreateGCGetPointerOffset(Value *DerivedPtr, + const Twine &Name) { + Module *M = BB->getParent()->getParent(); + Type *PtrTy = DerivedPtr->getType(); + Function *FnGCGetOffset = Intrinsic::getDeclaration( + M, Intrinsic::experimental_gc_get_pointer_offset, {PtrTy}); + return createCallHelper(FnGCGetOffset, {DerivedPtr}, this, Name); +} + +CallInst *IRBuilderBase::CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V, + Instruction *FMFSource, + const Twine &Name) { + Module *M = BB->getModule(); + Function *Fn = Intrinsic::getDeclaration(M, ID, {V->getType()}); + return createCallHelper(Fn, {V}, this, Name, FMFSource); +} + +CallInst *IRBuilderBase::CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS, + Value *RHS, + Instruction *FMFSource, + const Twine &Name) { + Module *M = BB->getModule(); + Function *Fn = Intrinsic::getDeclaration(M, ID, { LHS->getType() }); + return createCallHelper(Fn, {LHS, RHS}, this, Name, FMFSource); +} + +CallInst *IRBuilderBase::CreateIntrinsic(Intrinsic::ID ID, + ArrayRef<Type *> Types, + ArrayRef<Value *> Args, + Instruction *FMFSource, + const Twine &Name) { + Module *M = BB->getModule(); + Function *Fn = Intrinsic::getDeclaration(M, ID, Types); + return createCallHelper(Fn, Args, this, Name, FMFSource); +} + +CallInst *IRBuilderBase::CreateConstrainedFPBinOp( + Intrinsic::ID ID, Value *L, Value *R, Instruction *FMFSource, + const Twine &Name, MDNode *FPMathTag, + Optional<RoundingMode> Rounding, + Optional<fp::ExceptionBehavior> Except) { + Value *RoundingV = getConstrainedFPRounding(Rounding); + Value *ExceptV = getConstrainedFPExcept(Except); + + FastMathFlags UseFMF = FMF; + if (FMFSource) + UseFMF = FMFSource->getFastMathFlags(); + + CallInst *C = CreateIntrinsic(ID, {L->getType()}, + {L, R, RoundingV, ExceptV}, nullptr, Name); + setConstrainedFPCallAttr(C); + setFPAttrs(C, FPMathTag, UseFMF); + return C; +} + +Value *IRBuilderBase::CreateNAryOp(unsigned Opc, ArrayRef<Value *> Ops, + const Twine &Name, MDNode *FPMathTag) { + if (Instruction::isBinaryOp(Opc)) { + assert(Ops.size() == 2 && "Invalid number of operands!"); + return CreateBinOp(static_cast<Instruction::BinaryOps>(Opc), + Ops[0], Ops[1], Name, FPMathTag); + } + if (Instruction::isUnaryOp(Opc)) { + assert(Ops.size() == 1 && "Invalid number of operands!"); + return CreateUnOp(static_cast<Instruction::UnaryOps>(Opc), + Ops[0], Name, FPMathTag); + } + llvm_unreachable("Unexpected opcode!"); +} + +CallInst *IRBuilderBase::CreateConstrainedFPCast( + Intrinsic::ID ID, Value *V, Type *DestTy, + Instruction *FMFSource, const Twine &Name, MDNode *FPMathTag, + Optional<RoundingMode> Rounding, + Optional<fp::ExceptionBehavior> Except) { + Value *ExceptV = getConstrainedFPExcept(Except); + + FastMathFlags UseFMF = FMF; + if (FMFSource) + UseFMF = FMFSource->getFastMathFlags(); + + CallInst *C; + bool HasRoundingMD = false; + switch (ID) { + default: + break; +#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \ + case Intrinsic::INTRINSIC: \ + HasRoundingMD = ROUND_MODE; \ + break; +#include "llvm/IR/ConstrainedOps.def" + } + if (HasRoundingMD) { + Value *RoundingV = getConstrainedFPRounding(Rounding); + C = CreateIntrinsic(ID, {DestTy, V->getType()}, {V, RoundingV, ExceptV}, + nullptr, Name); + } else + C = CreateIntrinsic(ID, {DestTy, V->getType()}, {V, ExceptV}, nullptr, + Name); + + setConstrainedFPCallAttr(C); + + if (isa<FPMathOperator>(C)) + setFPAttrs(C, FPMathTag, UseFMF); + return C; +} + +Value *IRBuilderBase::CreateFCmpHelper( + CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name, + MDNode *FPMathTag, bool IsSignaling) { + if (IsFPConstrained) { + auto ID = IsSignaling ? Intrinsic::experimental_constrained_fcmps + : Intrinsic::experimental_constrained_fcmp; + return CreateConstrainedFPCmp(ID, P, LHS, RHS, Name); + } + + if (auto *LC = dyn_cast<Constant>(LHS)) + if (auto *RC = dyn_cast<Constant>(RHS)) + return Insert(Folder.CreateFCmp(P, LC, RC), Name); + return Insert(setFPAttrs(new FCmpInst(P, LHS, RHS), FPMathTag, FMF), Name); +} + +CallInst *IRBuilderBase::CreateConstrainedFPCmp( + Intrinsic::ID ID, CmpInst::Predicate P, Value *L, Value *R, + const Twine &Name, Optional<fp::ExceptionBehavior> Except) { + Value *PredicateV = getConstrainedFPPredicate(P); + Value *ExceptV = getConstrainedFPExcept(Except); + + CallInst *C = CreateIntrinsic(ID, {L->getType()}, + {L, R, PredicateV, ExceptV}, nullptr, Name); + setConstrainedFPCallAttr(C); + return C; +} + +CallInst *IRBuilderBase::CreateConstrainedFPCall( + Function *Callee, ArrayRef<Value *> Args, const Twine &Name, + Optional<RoundingMode> Rounding, + Optional<fp::ExceptionBehavior> Except) { + llvm::SmallVector<Value *, 6> UseArgs; + + append_range(UseArgs, Args); + bool HasRoundingMD = false; + switch (Callee->getIntrinsicID()) { + default: + break; +#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \ + case Intrinsic::INTRINSIC: \ + HasRoundingMD = ROUND_MODE; \ + break; +#include "llvm/IR/ConstrainedOps.def" + } + if (HasRoundingMD) + UseArgs.push_back(getConstrainedFPRounding(Rounding)); + UseArgs.push_back(getConstrainedFPExcept(Except)); + + CallInst *C = CreateCall(Callee, UseArgs, Name); + setConstrainedFPCallAttr(C); + return C; +} + +Value *IRBuilderBase::CreateSelect(Value *C, Value *True, Value *False, + const Twine &Name, Instruction *MDFrom) { + if (auto *V = Folder.FoldSelect(C, True, False)) + return V; + + SelectInst *Sel = SelectInst::Create(C, True, False); + if (MDFrom) { + MDNode *Prof = MDFrom->getMetadata(LLVMContext::MD_prof); + MDNode *Unpred = MDFrom->getMetadata(LLVMContext::MD_unpredictable); + Sel = addBranchMetadata(Sel, Prof, Unpred); + } + if (isa<FPMathOperator>(Sel)) + setFPAttrs(Sel, nullptr /* MDNode* */, FMF); + return Insert(Sel, Name); +} + +Value *IRBuilderBase::CreatePtrDiff(Type *ElemTy, Value *LHS, Value *RHS, + const Twine &Name) { + assert(LHS->getType() == RHS->getType() && + "Pointer subtraction operand types must match!"); + assert(cast<PointerType>(LHS->getType()) + ->isOpaqueOrPointeeTypeMatches(ElemTy) && + "Pointer type must match element type"); + Value *LHS_int = CreatePtrToInt(LHS, Type::getInt64Ty(Context)); + Value *RHS_int = CreatePtrToInt(RHS, Type::getInt64Ty(Context)); + Value *Difference = CreateSub(LHS_int, RHS_int); + return CreateExactSDiv(Difference, ConstantExpr::getSizeOf(ElemTy), + Name); +} + +Value *IRBuilderBase::CreateLaunderInvariantGroup(Value *Ptr) { + assert(isa<PointerType>(Ptr->getType()) && + "launder.invariant.group only applies to pointers."); + // FIXME: we could potentially avoid casts to/from i8*. + auto *PtrType = Ptr->getType(); + auto *Int8PtrTy = getInt8PtrTy(PtrType->getPointerAddressSpace()); + if (PtrType != Int8PtrTy) + Ptr = CreateBitCast(Ptr, Int8PtrTy); + Module *M = BB->getParent()->getParent(); + Function *FnLaunderInvariantGroup = Intrinsic::getDeclaration( + M, Intrinsic::launder_invariant_group, {Int8PtrTy}); + + assert(FnLaunderInvariantGroup->getReturnType() == Int8PtrTy && + FnLaunderInvariantGroup->getFunctionType()->getParamType(0) == + Int8PtrTy && + "LaunderInvariantGroup should take and return the same type"); + + CallInst *Fn = CreateCall(FnLaunderInvariantGroup, {Ptr}); + + if (PtrType != Int8PtrTy) + return CreateBitCast(Fn, PtrType); + return Fn; +} + +Value *IRBuilderBase::CreateStripInvariantGroup(Value *Ptr) { + assert(isa<PointerType>(Ptr->getType()) && + "strip.invariant.group only applies to pointers."); + + // FIXME: we could potentially avoid casts to/from i8*. + auto *PtrType = Ptr->getType(); + auto *Int8PtrTy = getInt8PtrTy(PtrType->getPointerAddressSpace()); + if (PtrType != Int8PtrTy) + Ptr = CreateBitCast(Ptr, Int8PtrTy); + Module *M = BB->getParent()->getParent(); + Function *FnStripInvariantGroup = Intrinsic::getDeclaration( + M, Intrinsic::strip_invariant_group, {Int8PtrTy}); + + assert(FnStripInvariantGroup->getReturnType() == Int8PtrTy && + FnStripInvariantGroup->getFunctionType()->getParamType(0) == + Int8PtrTy && + "StripInvariantGroup should take and return the same type"); + + CallInst *Fn = CreateCall(FnStripInvariantGroup, {Ptr}); + + if (PtrType != Int8PtrTy) + return CreateBitCast(Fn, PtrType); + return Fn; +} + +Value *IRBuilderBase::CreateVectorReverse(Value *V, const Twine &Name) { + auto *Ty = cast<VectorType>(V->getType()); + if (isa<ScalableVectorType>(Ty)) { + Module *M = BB->getParent()->getParent(); + Function *F = Intrinsic::getDeclaration( + M, Intrinsic::experimental_vector_reverse, Ty); + return Insert(CallInst::Create(F, V), Name); + } + // Keep the original behaviour for fixed vector + SmallVector<int, 8> ShuffleMask; + int NumElts = Ty->getElementCount().getKnownMinValue(); + for (int i = 0; i < NumElts; ++i) + ShuffleMask.push_back(NumElts - i - 1); + return CreateShuffleVector(V, ShuffleMask, Name); +} + +Value *IRBuilderBase::CreateVectorSplice(Value *V1, Value *V2, int64_t Imm, + const Twine &Name) { + assert(isa<VectorType>(V1->getType()) && "Unexpected type"); + assert(V1->getType() == V2->getType() && + "Splice expects matching operand types!"); + + if (auto *VTy = dyn_cast<ScalableVectorType>(V1->getType())) { + Module *M = BB->getParent()->getParent(); + Function *F = Intrinsic::getDeclaration( + M, Intrinsic::experimental_vector_splice, VTy); + + Value *Ops[] = {V1, V2, getInt32(Imm)}; + return Insert(CallInst::Create(F, Ops), Name); + } + + unsigned NumElts = cast<FixedVectorType>(V1->getType())->getNumElements(); + assert(((-Imm <= NumElts) || (Imm < NumElts)) && + "Invalid immediate for vector splice!"); + + // Keep the original behaviour for fixed vector + unsigned Idx = (NumElts + Imm) % NumElts; + SmallVector<int, 8> Mask; + for (unsigned I = 0; I < NumElts; ++I) + Mask.push_back(Idx + I); + + return CreateShuffleVector(V1, V2, Mask); +} + +Value *IRBuilderBase::CreateVectorSplat(unsigned NumElts, Value *V, + const Twine &Name) { + auto EC = ElementCount::getFixed(NumElts); + return CreateVectorSplat(EC, V, Name); +} + +Value *IRBuilderBase::CreateVectorSplat(ElementCount EC, Value *V, + const Twine &Name) { + assert(EC.isNonZero() && "Cannot splat to an empty vector!"); + + // First insert it into a poison vector so we can shuffle it. + Type *I32Ty = getInt32Ty(); + Value *Poison = PoisonValue::get(VectorType::get(V->getType(), EC)); + V = CreateInsertElement(Poison, V, ConstantInt::get(I32Ty, 0), + Name + ".splatinsert"); + + // Shuffle the value across the desired number of elements. + SmallVector<int, 16> Zeros; + Zeros.resize(EC.getKnownMinValue()); + return CreateShuffleVector(V, Zeros, Name + ".splat"); +} + +Value *IRBuilderBase::CreateExtractInteger( + const DataLayout &DL, Value *From, IntegerType *ExtractedTy, + uint64_t Offset, const Twine &Name) { + auto *IntTy = cast<IntegerType>(From->getType()); + assert(DL.getTypeStoreSize(ExtractedTy) + Offset <= + DL.getTypeStoreSize(IntTy) && + "Element extends past full value"); + uint64_t ShAmt = 8 * Offset; + Value *V = From; + if (DL.isBigEndian()) + ShAmt = 8 * (DL.getTypeStoreSize(IntTy) - + DL.getTypeStoreSize(ExtractedTy) - Offset); + if (ShAmt) { + V = CreateLShr(V, ShAmt, Name + ".shift"); + } + assert(ExtractedTy->getBitWidth() <= IntTy->getBitWidth() && + "Cannot extract to a larger integer!"); + if (ExtractedTy != IntTy) { + V = CreateTrunc(V, ExtractedTy, Name + ".trunc"); + } + return V; +} + +Value *IRBuilderBase::CreatePreserveArrayAccessIndex( + Type *ElTy, Value *Base, unsigned Dimension, unsigned LastIndex, + MDNode *DbgInfo) { + auto *BaseType = Base->getType(); + assert(isa<PointerType>(BaseType) && + "Invalid Base ptr type for preserve.array.access.index."); + assert(cast<PointerType>(BaseType)->isOpaqueOrPointeeTypeMatches(ElTy) && + "Pointer element type mismatch"); + + Value *LastIndexV = getInt32(LastIndex); + Constant *Zero = ConstantInt::get(Type::getInt32Ty(Context), 0); + SmallVector<Value *, 4> IdxList(Dimension, Zero); + IdxList.push_back(LastIndexV); + + Type *ResultType = + GetElementPtrInst::getGEPReturnType(ElTy, Base, IdxList); + + Module *M = BB->getParent()->getParent(); + Function *FnPreserveArrayAccessIndex = Intrinsic::getDeclaration( + M, Intrinsic::preserve_array_access_index, {ResultType, BaseType}); + + Value *DimV = getInt32(Dimension); + CallInst *Fn = + CreateCall(FnPreserveArrayAccessIndex, {Base, DimV, LastIndexV}); + Fn->addParamAttr( + 0, Attribute::get(Fn->getContext(), Attribute::ElementType, ElTy)); + if (DbgInfo) + Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo); + + return Fn; +} + +Value *IRBuilderBase::CreatePreserveUnionAccessIndex( + Value *Base, unsigned FieldIndex, MDNode *DbgInfo) { + assert(isa<PointerType>(Base->getType()) && + "Invalid Base ptr type for preserve.union.access.index."); + auto *BaseType = Base->getType(); + + Module *M = BB->getParent()->getParent(); + Function *FnPreserveUnionAccessIndex = Intrinsic::getDeclaration( + M, Intrinsic::preserve_union_access_index, {BaseType, BaseType}); + + Value *DIIndex = getInt32(FieldIndex); + CallInst *Fn = + CreateCall(FnPreserveUnionAccessIndex, {Base, DIIndex}); + if (DbgInfo) + Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo); + + return Fn; +} + +Value *IRBuilderBase::CreatePreserveStructAccessIndex( + Type *ElTy, Value *Base, unsigned Index, unsigned FieldIndex, + MDNode *DbgInfo) { + auto *BaseType = Base->getType(); + assert(isa<PointerType>(BaseType) && + "Invalid Base ptr type for preserve.struct.access.index."); + assert(cast<PointerType>(BaseType)->isOpaqueOrPointeeTypeMatches(ElTy) && + "Pointer element type mismatch"); + + Value *GEPIndex = getInt32(Index); + Constant *Zero = ConstantInt::get(Type::getInt32Ty(Context), 0); + Type *ResultType = + GetElementPtrInst::getGEPReturnType(ElTy, Base, {Zero, GEPIndex}); + + Module *M = BB->getParent()->getParent(); + Function *FnPreserveStructAccessIndex = Intrinsic::getDeclaration( + M, Intrinsic::preserve_struct_access_index, {ResultType, BaseType}); + + Value *DIIndex = getInt32(FieldIndex); + CallInst *Fn = CreateCall(FnPreserveStructAccessIndex, + {Base, GEPIndex, DIIndex}); + Fn->addParamAttr( + 0, Attribute::get(Fn->getContext(), Attribute::ElementType, ElTy)); + if (DbgInfo) + Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo); + + return Fn; +} + +CallInst *IRBuilderBase::CreateAlignmentAssumptionHelper(const DataLayout &DL, + Value *PtrValue, + Value *AlignValue, + Value *OffsetValue) { + SmallVector<Value *, 4> Vals({PtrValue, AlignValue}); + if (OffsetValue) + Vals.push_back(OffsetValue); + OperandBundleDefT<Value *> AlignOpB("align", Vals); + return CreateAssumption(ConstantInt::getTrue(getContext()), {AlignOpB}); +} + +CallInst *IRBuilderBase::CreateAlignmentAssumption(const DataLayout &DL, + Value *PtrValue, + unsigned Alignment, + Value *OffsetValue) { + assert(isa<PointerType>(PtrValue->getType()) && + "trying to create an alignment assumption on a non-pointer?"); + assert(Alignment != 0 && "Invalid Alignment"); + auto *PtrTy = cast<PointerType>(PtrValue->getType()); + Type *IntPtrTy = getIntPtrTy(DL, PtrTy->getAddressSpace()); + Value *AlignValue = ConstantInt::get(IntPtrTy, Alignment); + return CreateAlignmentAssumptionHelper(DL, PtrValue, AlignValue, OffsetValue); +} + +CallInst *IRBuilderBase::CreateAlignmentAssumption(const DataLayout &DL, + Value *PtrValue, + Value *Alignment, + Value *OffsetValue) { + assert(isa<PointerType>(PtrValue->getType()) && + "trying to create an alignment assumption on a non-pointer?"); + return CreateAlignmentAssumptionHelper(DL, PtrValue, Alignment, OffsetValue); +} + +IRBuilderDefaultInserter::~IRBuilderDefaultInserter() = default; +IRBuilderCallbackInserter::~IRBuilderCallbackInserter() = default; +IRBuilderFolder::~IRBuilderFolder() = default; +void ConstantFolder::anchor() {} +void NoFolder::anchor() {} |