aboutsummaryrefslogtreecommitdiff
path: root/contrib/llvm-project/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'contrib/llvm-project/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp')
-rw-r--r--contrib/llvm-project/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp386
1 files changed, 284 insertions, 102 deletions
diff --git a/contrib/llvm-project/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/contrib/llvm-project/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 6d415c9c7f90..9a67a8d05a4d 100644
--- a/contrib/llvm-project/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/contrib/llvm-project/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -15,16 +15,21 @@
#include "llvm/ADT/ScopeExit.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/SmallVector.h"
+#include "llvm/Analysis/AliasAnalysis.h"
+#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/BranchProbabilityInfo.h"
#include "llvm/Analysis/Loads.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/CodeGen/Analysis.h"
+#include "llvm/CodeGen/GlobalISel/CSEInfo.h"
+#include "llvm/CodeGen/GlobalISel/CSEMIRBuilder.h"
#include "llvm/CodeGen/GlobalISel/CallLowering.h"
#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
#include "llvm/CodeGen/GlobalISel/InlineAsmLowering.h"
#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
#include "llvm/CodeGen/LowLevelType.h"
+#include "llvm/CodeGen/LowLevelTypeUtils.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
@@ -39,6 +44,7 @@
#include "llvm/CodeGen/TargetFrameLowering.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/CodeGen/TargetLowering.h"
+#include "llvm/CodeGen/TargetOpcodes.h"
#include "llvm/CodeGen/TargetPassConfig.h"
#include "llvm/CodeGen/TargetRegisterInfo.h"
#include "llvm/CodeGen/TargetSubtargetInfo.h"
@@ -47,7 +53,6 @@
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
-#include "llvm/IR/DebugInfo.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/Function.h"
@@ -60,6 +65,7 @@
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/Metadata.h"
#include "llvm/IR/PatternMatch.h"
+#include "llvm/IR/Statepoint.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/User.h"
#include "llvm/IR/Value.h"
@@ -70,7 +76,6 @@
#include "llvm/Support/CodeGen.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/LowLevelTypeImpl.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetIntrinsicInfo.h"
@@ -78,9 +83,9 @@
#include "llvm/Transforms/Utils/MemoryOpRemark.h"
#include <algorithm>
#include <cassert>
-#include <cstddef>
#include <cstdint>
#include <iterator>
+#include <optional>
#include <string>
#include <utility>
#include <vector>
@@ -151,11 +156,11 @@ public:
LLVM_DEBUG(dbgs() << "Checking DILocation from " << *CurrInst
<< " was copied to " << MI);
#endif
- // We allow insts in the entry block to have a debug loc line of 0 because
+ // We allow insts in the entry block to have no debug loc because
// they could have originated from constants, and we don't want a jumpy
// debug experience.
assert((CurrInst->getDebugLoc() == MI.getDebugLoc() ||
- MI.getDebugLoc().getLine() == 0) &&
+ (MI.getParent()->isEntryBlock() && !MI.getDebugLoc())) &&
"Line info was not transferred to all instructions");
}
};
@@ -167,8 +172,11 @@ void IRTranslator::getAnalysisUsage(AnalysisUsage &AU) const {
AU.addRequired<StackProtector>();
AU.addRequired<TargetPassConfig>();
AU.addRequired<GISelCSEAnalysisWrapperPass>();
- if (OptLevel != CodeGenOpt::None)
+ AU.addRequired<AssumptionCacheTracker>();
+ if (OptLevel != CodeGenOpt::None) {
AU.addRequired<BranchProbabilityInfoWrapperPass>();
+ AU.addRequired<AAResultsWrapperPass>();
+ }
AU.addRequired<TargetLibraryInfoWrapperPass>();
AU.addPreserved<TargetLibraryInfoWrapperPass>();
getSelectionDAGFallbackAnalysisUsage(AU);
@@ -293,7 +301,7 @@ bool IRTranslator::translateBinaryOp(unsigned Opcode, const User &U,
Register Op0 = getOrCreateVReg(*U.getOperand(0));
Register Op1 = getOrCreateVReg(*U.getOperand(1));
Register Res = getOrCreateVReg(U);
- uint16_t Flags = 0;
+ uint32_t Flags = 0;
if (isa<Instruction>(U)) {
const Instruction &I = cast<Instruction>(U);
Flags = MachineInstr::copyFlagsFromInstruction(I);
@@ -307,7 +315,7 @@ bool IRTranslator::translateUnaryOp(unsigned Opcode, const User &U,
MachineIRBuilder &MIRBuilder) {
Register Op0 = getOrCreateVReg(*U.getOperand(0));
Register Res = getOrCreateVReg(U);
- uint16_t Flags = 0;
+ uint32_t Flags = 0;
if (isa<Instruction>(U)) {
const Instruction &I = cast<Instruction>(U);
Flags = MachineInstr::copyFlagsFromInstruction(I);
@@ -338,7 +346,7 @@ bool IRTranslator::translateCompare(const User &U,
MIRBuilder.buildCopy(
Res, getOrCreateVReg(*Constant::getAllOnesValue(U.getType())));
else {
- uint16_t Flags = 0;
+ uint32_t Flags = 0;
if (CI)
Flags = MachineInstr::copyFlagsFromInstruction(*CI);
MIRBuilder.buildFCmp(Pred, Res, Op0, Op1, Flags);
@@ -685,7 +693,7 @@ bool IRTranslator::translateSwitch(const User &U, MachineIRBuilder &MIB) {
BranchProbabilityInfo *BPI = FuncInfo.BPI;
CaseClusterVector Clusters;
Clusters.reserve(SI.getNumCases());
- for (auto &I : SI.cases()) {
+ for (const auto &I : SI.cases()) {
MachineBasicBlock *Succ = &getMBB(*I.getCaseSuccessor());
assert(Succ && "Could not find successor mbb in mapping");
const ConstantInt *CaseVal = I.getCaseValue();
@@ -837,8 +845,8 @@ void IRTranslator::emitSwitchCase(SwitchCG::CaseBlock &CB,
// For conditional branch lowering, we might try to do something silly like
// emit an G_ICMP to compare an existing G_ICMP i1 result with true. If so,
// just re-use the existing condition vreg.
- if (MRI->getType(CondLHS).getSizeInBits() == 1 && CI &&
- CI->getZExtValue() == 1 && CB.PredInfo.Pred == CmpInst::ICMP_EQ) {
+ if (MRI->getType(CondLHS).getSizeInBits() == 1 && CI && CI->isOne() &&
+ CB.PredInfo.Pred == CmpInst::ICMP_EQ) {
Cond = CondLHS;
} else {
Register CondRHS = getOrCreateVReg(*CB.CmpRHS);
@@ -1011,7 +1019,7 @@ void IRTranslator::emitBitTestHeader(SwitchCG::BitTestBlock &B,
LLT MaskTy = SwitchOpTy;
if (MaskTy.getSizeInBits() > PtrTy.getSizeInBits() ||
- !isPowerOf2_32(MaskTy.getSizeInBits()))
+ !llvm::has_single_bit<uint32_t>(MaskTy.getSizeInBits()))
MaskTy = LLT::scalar(PtrTy.getSizeInBits());
else {
// Ensure that the type will fit the mask value.
@@ -1062,19 +1070,19 @@ void IRTranslator::emitBitTestCase(SwitchCG::BitTestBlock &BB,
LLT SwitchTy = getLLTForMVT(BB.RegVT);
Register Cmp;
- unsigned PopCount = countPopulation(B.Mask);
+ unsigned PopCount = llvm::popcount(B.Mask);
if (PopCount == 1) {
// Testing for a single bit; just compare the shift count with what it
// would need to be to shift a 1 bit in that position.
auto MaskTrailingZeros =
- MIB.buildConstant(SwitchTy, countTrailingZeros(B.Mask));
+ MIB.buildConstant(SwitchTy, llvm::countr_zero(B.Mask));
Cmp =
MIB.buildICmp(ICmpInst::ICMP_EQ, LLT::scalar(1), Reg, MaskTrailingZeros)
.getReg(0);
} else if (PopCount == BB.Range) {
// There is only one zero bit in the range, test for it directly.
auto MaskTrailingOnes =
- MIB.buildConstant(SwitchTy, countTrailingOnes(B.Mask));
+ MIB.buildConstant(SwitchTy, llvm::countr_one(B.Mask));
Cmp = MIB.buildICmp(CmpInst::ICMP_NE, LLT::scalar(1), Reg, MaskTrailingOnes)
.getReg(0);
} else {
@@ -1276,26 +1284,37 @@ static bool isSwiftError(const Value *V) {
bool IRTranslator::translateLoad(const User &U, MachineIRBuilder &MIRBuilder) {
const LoadInst &LI = cast<LoadInst>(U);
- if (DL->getTypeStoreSize(LI.getType()) == 0)
+
+ unsigned StoreSize = DL->getTypeStoreSize(LI.getType());
+ if (StoreSize == 0)
return true;
ArrayRef<Register> Regs = getOrCreateVRegs(LI);
ArrayRef<uint64_t> Offsets = *VMap.getOffsets(LI);
Register Base = getOrCreateVReg(*LI.getPointerOperand());
+ AAMDNodes AAInfo = LI.getAAMetadata();
- Type *OffsetIRTy = DL->getIntPtrType(LI.getPointerOperandType());
+ const Value *Ptr = LI.getPointerOperand();
+ Type *OffsetIRTy = DL->getIndexType(Ptr->getType());
LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
- if (CLI->supportSwiftError() && isSwiftError(LI.getPointerOperand())) {
+ if (CLI->supportSwiftError() && isSwiftError(Ptr)) {
assert(Regs.size() == 1 && "swifterror should be single pointer");
- Register VReg = SwiftError.getOrCreateVRegUseAt(&LI, &MIRBuilder.getMBB(),
- LI.getPointerOperand());
+ Register VReg =
+ SwiftError.getOrCreateVRegUseAt(&LI, &MIRBuilder.getMBB(), Ptr);
MIRBuilder.buildCopy(Regs[0], VReg);
return true;
}
auto &TLI = *MF->getSubtarget().getTargetLowering();
- MachineMemOperand::Flags Flags = TLI.getLoadMemOperandFlags(LI, *DL);
+ MachineMemOperand::Flags Flags =
+ TLI.getLoadMemOperandFlags(LI, *DL, AC, LibInfo);
+ if (AA && !(Flags & MachineMemOperand::MOInvariant)) {
+ if (AA->pointsToConstantMemory(
+ MemoryLocation(Ptr, LocationSize::precise(StoreSize), AAInfo))) {
+ Flags |= MachineMemOperand::MOInvariant;
+ }
+ }
const MDNode *Ranges =
Regs.size() == 1 ? LI.getMetadata(LLVMContext::MD_range) : nullptr;
@@ -1307,7 +1326,7 @@ bool IRTranslator::translateLoad(const User &U, MachineIRBuilder &MIRBuilder) {
Align BaseAlign = getMemOpAlign(LI);
auto MMO = MF->getMachineMemOperand(
Ptr, Flags, MRI->getType(Regs[i]),
- commonAlignment(BaseAlign, Offsets[i] / 8), LI.getAAMetadata(), Ranges,
+ commonAlignment(BaseAlign, Offsets[i] / 8), AAInfo, Ranges,
LI.getSyncScopeID(), LI.getOrdering());
MIRBuilder.buildLoad(Regs[i], Addr, *MMO);
}
@@ -1324,7 +1343,7 @@ bool IRTranslator::translateStore(const User &U, MachineIRBuilder &MIRBuilder) {
ArrayRef<uint64_t> Offsets = *VMap.getOffsets(*SI.getValueOperand());
Register Base = getOrCreateVReg(*SI.getPointerOperand());
- Type *OffsetIRTy = DL->getIntPtrType(SI.getPointerOperandType());
+ Type *OffsetIRTy = DL->getIndexType(SI.getPointerOperandType());
LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
if (CLI->supportSwiftError() && isSwiftError(SI.getPointerOperand())) {
@@ -1401,7 +1420,7 @@ bool IRTranslator::translateInsertValue(const User &U,
ArrayRef<uint64_t> DstOffsets = *VMap.getOffsets(U);
ArrayRef<Register> SrcRegs = getOrCreateVRegs(*Src);
ArrayRef<Register> InsertedRegs = getOrCreateVRegs(*U.getOperand(1));
- auto InsertedIt = InsertedRegs.begin();
+ auto *InsertedIt = InsertedRegs.begin();
for (unsigned i = 0; i < DstRegs.size(); ++i) {
if (DstOffsets[i] >= Offset && InsertedIt != InsertedRegs.end())
@@ -1420,7 +1439,7 @@ bool IRTranslator::translateSelect(const User &U,
ArrayRef<Register> Op0Regs = getOrCreateVRegs(*U.getOperand(1));
ArrayRef<Register> Op1Regs = getOrCreateVRegs(*U.getOperand(2));
- uint16_t Flags = 0;
+ uint32_t Flags = 0;
if (const SelectInst *SI = dyn_cast<SelectInst>(&U))
Flags = MachineInstr::copyFlagsFromInstruction(*SI);
@@ -1450,8 +1469,14 @@ bool IRTranslator::translateBitCast(const User &U,
MachineIRBuilder &MIRBuilder) {
// If we're bitcasting to the source type, we can reuse the source vreg.
if (getLLTForType(*U.getOperand(0)->getType(), *DL) ==
- getLLTForType(*U.getType(), *DL))
+ getLLTForType(*U.getType(), *DL)) {
+ // If the source is a ConstantInt then it was probably created by
+ // ConstantHoisting and we should leave it alone.
+ if (isa<ConstantInt>(U.getOperand(0)))
+ return translateCast(TargetOpcode::G_CONSTANT_FOLD_BARRIER, U,
+ MIRBuilder);
return translateCopy(U, *U.getOperand(0), MIRBuilder);
+ }
return translateCast(TargetOpcode::G_BITCAST, U, MIRBuilder);
}
@@ -1470,7 +1495,7 @@ bool IRTranslator::translateGetElementPtr(const User &U,
Register BaseReg = getOrCreateVReg(Op0);
Type *PtrIRTy = Op0.getType();
LLT PtrTy = getLLTForType(*PtrIRTy, *DL);
- Type *OffsetIRTy = DL->getIntPtrType(PtrIRTy);
+ Type *OffsetIRTy = DL->getIndexType(PtrIRTy);
LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
// Normalize Vector GEP - all scalar operands should be converted to the
@@ -1495,7 +1520,7 @@ bool IRTranslator::translateGetElementPtr(const User &U,
.getReg(0);
PtrIRTy = FixedVectorType::get(PtrIRTy, VectorWidth);
PtrTy = getLLTForType(*PtrIRTy, *DL);
- OffsetIRTy = DL->getIntPtrType(PtrIRTy);
+ OffsetIRTy = DL->getIndexType(PtrIRTy);
OffsetTy = getLLTForType(*OffsetIRTy, *DL);
}
@@ -1564,9 +1589,9 @@ bool IRTranslator::translateGetElementPtr(const User &U,
bool IRTranslator::translateMemFunc(const CallInst &CI,
MachineIRBuilder &MIRBuilder,
unsigned Opcode) {
-
+ const Value *SrcPtr = CI.getArgOperand(1);
// If the source is undef, then just emit a nop.
- if (isa<UndefValue>(CI.getArgOperand(1)))
+ if (isa<UndefValue>(SrcPtr))
return true;
SmallVector<Register, 3> SrcRegs;
@@ -1596,15 +1621,20 @@ bool IRTranslator::translateMemFunc(const CallInst &CI,
unsigned IsVol =
cast<ConstantInt>(CI.getArgOperand(CI.arg_size() - 1))->getZExtValue();
+ ConstantInt *CopySize = nullptr;
+
if (auto *MCI = dyn_cast<MemCpyInst>(&CI)) {
DstAlign = MCI->getDestAlign().valueOrOne();
SrcAlign = MCI->getSourceAlign().valueOrOne();
+ CopySize = dyn_cast<ConstantInt>(MCI->getArgOperand(2));
} else if (auto *MCI = dyn_cast<MemCpyInlineInst>(&CI)) {
DstAlign = MCI->getDestAlign().valueOrOne();
SrcAlign = MCI->getSourceAlign().valueOrOne();
+ CopySize = dyn_cast<ConstantInt>(MCI->getArgOperand(2));
} else if (auto *MMI = dyn_cast<MemMoveInst>(&CI)) {
DstAlign = MMI->getDestAlign().valueOrOne();
SrcAlign = MMI->getSourceAlign().valueOrOne();
+ CopySize = dyn_cast<ConstantInt>(MMI->getArgOperand(2));
} else {
auto *MSI = cast<MemSetInst>(&CI);
DstAlign = MSI->getDestAlign().valueOrOne();
@@ -1618,14 +1648,31 @@ bool IRTranslator::translateMemFunc(const CallInst &CI,
}
// Create mem operands to store the alignment and volatile info.
- auto VolFlag = IsVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
- ICall.addMemOperand(MF->getMachineMemOperand(
- MachinePointerInfo(CI.getArgOperand(0)),
- MachineMemOperand::MOStore | VolFlag, 1, DstAlign));
+ MachineMemOperand::Flags LoadFlags = MachineMemOperand::MOLoad;
+ MachineMemOperand::Flags StoreFlags = MachineMemOperand::MOStore;
+ if (IsVol) {
+ LoadFlags |= MachineMemOperand::MOVolatile;
+ StoreFlags |= MachineMemOperand::MOVolatile;
+ }
+
+ AAMDNodes AAInfo = CI.getAAMetadata();
+ if (AA && CopySize &&
+ AA->pointsToConstantMemory(MemoryLocation(
+ SrcPtr, LocationSize::precise(CopySize->getZExtValue()), AAInfo))) {
+ LoadFlags |= MachineMemOperand::MOInvariant;
+
+ // FIXME: pointsToConstantMemory probably does not imply dereferenceable,
+ // but the previous usage implied it did. Probably should check
+ // isDereferenceableAndAlignedPointer.
+ LoadFlags |= MachineMemOperand::MODereferenceable;
+ }
+
+ ICall.addMemOperand(
+ MF->getMachineMemOperand(MachinePointerInfo(CI.getArgOperand(0)),
+ StoreFlags, 1, DstAlign, AAInfo));
if (Opcode != TargetOpcode::G_MEMSET)
ICall.addMemOperand(MF->getMachineMemOperand(
- MachinePointerInfo(CI.getArgOperand(1)),
- MachineMemOperand::MOLoad | VolFlag, 1, SrcAlign));
+ MachinePointerInfo(SrcPtr), LoadFlags, 1, SrcAlign, AAInfo));
return true;
}
@@ -1719,6 +1766,8 @@ unsigned IRTranslator::getSimpleIntrinsicOpcode(Intrinsic::ID ID) {
return TargetOpcode::G_FLOG2;
case Intrinsic::log10:
return TargetOpcode::G_FLOG10;
+ case Intrinsic::ldexp:
+ return TargetOpcode::G_FLDEXP;
case Intrinsic::nearbyint:
return TargetOpcode::G_FNEARBYINT;
case Intrinsic::pow:
@@ -1786,7 +1835,7 @@ bool IRTranslator::translateSimpleIntrinsic(const CallInst &CI,
// Yes. Let's translate it.
SmallVector<llvm::SrcOp, 4> VRegs;
- for (auto &Arg : CI.args())
+ for (const auto &Arg : CI.args())
VRegs.push_back(getOrCreateVReg(*Arg));
MIRBuilder.buildInstr(Op, {getOrCreateVReg(CI)}, VRegs,
@@ -1811,6 +1860,8 @@ static unsigned getConstrainedOpcode(Intrinsic::ID ID) {
return TargetOpcode::G_STRICT_FMA;
case Intrinsic::experimental_constrained_sqrt:
return TargetOpcode::G_STRICT_FSQRT;
+ case Intrinsic::experimental_constrained_ldexp:
+ return TargetOpcode::G_STRICT_FLDEXP;
default:
return 0;
}
@@ -1818,13 +1869,13 @@ static unsigned getConstrainedOpcode(Intrinsic::ID ID) {
bool IRTranslator::translateConstrainedFPIntrinsic(
const ConstrainedFPIntrinsic &FPI, MachineIRBuilder &MIRBuilder) {
- fp::ExceptionBehavior EB = FPI.getExceptionBehavior().getValue();
+ fp::ExceptionBehavior EB = *FPI.getExceptionBehavior();
unsigned Opcode = getConstrainedOpcode(FPI.getIntrinsicID());
if (!Opcode)
return false;
- unsigned Flags = MachineInstr::copyFlagsFromInstruction(FPI);
+ uint32_t Flags = MachineInstr::copyFlagsFromInstruction(FPI);
if (EB == fp::ExceptionBehavior::ebIgnore)
Flags |= MachineInstr::NoFPExcept;
@@ -1839,14 +1890,66 @@ bool IRTranslator::translateConstrainedFPIntrinsic(
return true;
}
+std::optional<MCRegister> IRTranslator::getArgPhysReg(Argument &Arg) {
+ auto VRegs = getOrCreateVRegs(Arg);
+ if (VRegs.size() != 1)
+ return std::nullopt;
+
+ // Arguments are lowered as a copy of a livein physical register.
+ auto *VRegDef = MF->getRegInfo().getVRegDef(VRegs[0]);
+ if (!VRegDef || !VRegDef->isCopy())
+ return std::nullopt;
+ return VRegDef->getOperand(1).getReg().asMCReg();
+}
+
+bool IRTranslator::translateIfEntryValueArgument(const DbgValueInst &DebugInst,
+ MachineIRBuilder &MIRBuilder) {
+ auto *Arg = dyn_cast<Argument>(DebugInst.getValue());
+ if (!Arg)
+ return false;
+
+ const DIExpression *Expr = DebugInst.getExpression();
+ if (!Expr->isEntryValue())
+ return false;
+
+ std::optional<MCRegister> PhysReg = getArgPhysReg(*Arg);
+ if (!PhysReg) {
+ LLVM_DEBUG(dbgs() << "Dropping dbg.value: expression is entry_value but "
+ "couldn't find a physical register\n"
+ << DebugInst << "\n");
+ return true;
+ }
+
+ MIRBuilder.buildDirectDbgValue(*PhysReg, DebugInst.getVariable(),
+ DebugInst.getExpression());
+ return true;
+}
+
+bool IRTranslator::translateIfEntryValueArgument(
+ const DbgDeclareInst &DebugInst) {
+ auto *Arg = dyn_cast<Argument>(DebugInst.getAddress());
+ if (!Arg)
+ return false;
+
+ const DIExpression *Expr = DebugInst.getExpression();
+ if (!Expr->isEntryValue())
+ return false;
+
+ std::optional<MCRegister> PhysReg = getArgPhysReg(*Arg);
+ if (!PhysReg)
+ return false;
+
+ MF->setVariableDbgInfo(DebugInst.getVariable(), Expr, *PhysReg,
+ DebugInst.getDebugLoc());
+ return true;
+}
+
bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
MachineIRBuilder &MIRBuilder) {
if (auto *MI = dyn_cast<AnyMemIntrinsic>(&CI)) {
if (ORE->enabled()) {
- const Function &F = *MI->getParent()->getParent();
- auto &TLI = getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
- if (MemoryOpRemark::canHandle(MI, TLI)) {
- MemoryOpRemark R(*ORE, "gisel-irtranslator-memsize", *DL, TLI);
+ if (MemoryOpRemark::canHandle(MI, *LibInfo)) {
+ MemoryOpRemark R(*ORE, "gisel-irtranslator-memsize", *DL, *LibInfo);
R.visit(MI);
}
}
@@ -1907,12 +2010,16 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
// instructions (in fact, they get ignored if they *do* exist).
MF->setVariableDbgInfo(DI.getVariable(), DI.getExpression(),
getOrCreateFrameIndex(*AI), DI.getDebugLoc());
- } else {
- // A dbg.declare describes the address of a source variable, so lower it
- // into an indirect DBG_VALUE.
- MIRBuilder.buildIndirectDbgValue(getOrCreateVReg(*Address),
- DI.getVariable(), DI.getExpression());
+ return true;
}
+
+ if (translateIfEntryValueArgument(DI))
+ return true;
+
+ // A dbg.declare describes the address of a source variable, so lower it
+ // into an indirect DBG_VALUE.
+ MIRBuilder.buildIndirectDbgValue(getOrCreateVReg(*Address),
+ DI.getVariable(), DI.getExpression());
return true;
}
case Intrinsic::dbg_label: {
@@ -1953,16 +2060,32 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
// DI cannot produce a valid DBG_VALUE, so produce an undef DBG_VALUE to
// terminate any prior location.
MIRBuilder.buildIndirectDbgValue(0, DI.getVariable(), DI.getExpression());
- } else if (const auto *CI = dyn_cast<Constant>(V)) {
+ return true;
+ }
+ if (const auto *CI = dyn_cast<Constant>(V)) {
MIRBuilder.buildConstDbgValue(*CI, DI.getVariable(), DI.getExpression());
- } else {
- for (Register Reg : getOrCreateVRegs(*V)) {
- // FIXME: This does not handle register-indirect values at offset 0. The
- // direct/indirect thing shouldn't really be handled by something as
- // implicit as reg+noreg vs reg+imm in the first place, but it seems
- // pretty baked in right now.
- MIRBuilder.buildDirectDbgValue(Reg, DI.getVariable(), DI.getExpression());
- }
+ return true;
+ }
+ if (auto *AI = dyn_cast<AllocaInst>(V);
+ AI && AI->isStaticAlloca() && DI.getExpression()->startsWithDeref()) {
+ // If the value is an alloca and the expression starts with a
+ // dereference, track a stack slot instead of a register, as registers
+ // may be clobbered.
+ auto ExprOperands = DI.getExpression()->getElements();
+ auto *ExprDerefRemoved =
+ DIExpression::get(AI->getContext(), ExprOperands.drop_front());
+ MIRBuilder.buildFIDbgValue(getOrCreateFrameIndex(*AI), DI.getVariable(),
+ ExprDerefRemoved);
+ return true;
+ }
+ if (translateIfEntryValueArgument(DI, MIRBuilder))
+ return true;
+ for (Register Reg : getOrCreateVRegs(*V)) {
+ // FIXME: This does not handle register-indirect values at offset 0. The
+ // direct/indirect thing shouldn't really be handled by something as
+ // implicit as reg+noreg vs reg+imm in the first place, but it seems
+ // pretty baked in right now.
+ MIRBuilder.buildDirectDbgValue(Reg, DI.getVariable(), DI.getExpression());
}
return true;
}
@@ -2052,6 +2175,13 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
getOrCreateVReg(*CI.getArgOperand(0)),
MachineInstr::copyFlagsFromInstruction(CI));
return true;
+ case Intrinsic::frexp: {
+ ArrayRef<Register> VRegs = getOrCreateVRegs(CI);
+ MIRBuilder.buildFFrexp(VRegs[0], VRegs[1],
+ getOrCreateVReg(*CI.getArgOperand(0)),
+ MachineInstr::copyFlagsFromInstruction(CI));
+ return true;
+ }
case Intrinsic::memcpy_inline:
return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMCPY_INLINE);
case Intrinsic::memcpy:
@@ -2077,9 +2207,14 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
getStackGuard(getOrCreateVReg(CI), MIRBuilder);
return true;
case Intrinsic::stackprotector: {
+ const TargetLowering &TLI = *MF->getSubtarget().getTargetLowering();
LLT PtrTy = getLLTForType(*CI.getArgOperand(0)->getType(), *DL);
- Register GuardVal = MRI->createGenericVirtualRegister(PtrTy);
- getStackGuard(GuardVal, MIRBuilder);
+ Register GuardVal;
+ if (TLI.useLoadStackGuardNode()) {
+ GuardVal = MRI->createGenericVirtualRegister(PtrTy);
+ getStackGuard(GuardVal, MIRBuilder);
+ } else
+ GuardVal = getOrCreateVReg(*CI.getArgOperand(0)); // The guard's value.
AllocaInst *Slot = cast<AllocaInst>(CI.getArgOperand(1));
int FI = getOrCreateFrameIndex(*Slot);
@@ -2252,6 +2387,34 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
Info.OrigRet = {Register(), Type::getVoidTy(CI.getContext()), 0};
return CLI->lowerCall(MIRBuilder, Info);
}
+ case Intrinsic::fptrunc_round: {
+ uint32_t Flags = MachineInstr::copyFlagsFromInstruction(CI);
+
+ // Convert the metadata argument to a constant integer
+ Metadata *MD = cast<MetadataAsValue>(CI.getArgOperand(1))->getMetadata();
+ std::optional<RoundingMode> RoundMode =
+ convertStrToRoundingMode(cast<MDString>(MD)->getString());
+
+ // Add the Rounding mode as an integer
+ MIRBuilder
+ .buildInstr(TargetOpcode::G_INTRINSIC_FPTRUNC_ROUND,
+ {getOrCreateVReg(CI)},
+ {getOrCreateVReg(*CI.getArgOperand(0))}, Flags)
+ .addImm((int)*RoundMode);
+
+ return true;
+ }
+ case Intrinsic::is_fpclass: {
+ Value *FpValue = CI.getOperand(0);
+ ConstantInt *TestMaskValue = cast<ConstantInt>(CI.getOperand(1));
+
+ MIRBuilder
+ .buildInstr(TargetOpcode::G_IS_FPCLASS, {getOrCreateVReg(CI)},
+ {getOrCreateVReg(*FpValue)})
+ .addImm(TestMaskValue->getZExtValue());
+
+ return true;
+ }
#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
case Intrinsic::INTRINSIC:
#include "llvm/IR/ConstrainedOps.def"
@@ -2284,14 +2447,14 @@ bool IRTranslator::translateCallBase(const CallBase &CB,
SmallVector<ArrayRef<Register>, 8> Args;
Register SwiftInVReg = 0;
Register SwiftErrorVReg = 0;
- for (auto &Arg : CB.args()) {
+ for (const auto &Arg : CB.args()) {
if (CLI->supportSwiftError() && isSwiftError(Arg)) {
assert(SwiftInVReg == 0 && "Expected only one swift error argument");
LLT Ty = getLLTForType(*Arg->getType(), *DL);
SwiftInVReg = MRI->createGenericVirtualRegister(Ty);
MIRBuilder.buildCopy(SwiftInVReg, SwiftError.getOrCreateVRegUseAt(
&CB, &MIRBuilder.getMBB(), Arg));
- Args.emplace_back(makeArrayRef(SwiftInVReg));
+ Args.emplace_back(ArrayRef(SwiftInVReg));
SwiftErrorVReg =
SwiftError.getOrCreateVRegDefAt(&CB, &MIRBuilder.getMBB(), Arg);
continue;
@@ -2301,10 +2464,8 @@ bool IRTranslator::translateCallBase(const CallBase &CB,
if (auto *CI = dyn_cast<CallInst>(&CB)) {
if (ORE->enabled()) {
- const Function &F = *CI->getParent()->getParent();
- auto &TLI = getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
- if (MemoryOpRemark::canHandle(CI, TLI)) {
- MemoryOpRemark R(*ORE, "gisel-irtranslator-memsize", *DL, TLI);
+ if (MemoryOpRemark::canHandle(CI, *LibInfo)) {
+ MemoryOpRemark R(*ORE, "gisel-irtranslator-memsize", *DL, *LibInfo);
R.visit(CI);
}
}
@@ -2342,6 +2503,10 @@ bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) {
if (CI.countOperandBundlesOfType(LLVMContext::OB_cfguardtarget))
return false;
+ // FIXME: support statepoints and related.
+ if (isa<GCStatepointInst, GCRelocateInst, GCResultInst>(U))
+ return false;
+
if (CI.isInlineAsm())
return translateInlineAsm(CI, MIRBuilder);
@@ -2373,7 +2538,7 @@ bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) {
if (isa<FPMathOperator>(CI))
MIB->copyIRFlags(CI);
- for (auto &Arg : enumerate(CI.args())) {
+ for (const auto &Arg : enumerate(CI.args())) {
// If this is required to be an immediate, don't materialize it in a
// register.
if (CI.paramHasAttr(Arg.index(), Attribute::ImmArg)) {
@@ -2409,13 +2574,21 @@ bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) {
TargetLowering::IntrinsicInfo Info;
// TODO: Add a GlobalISel version of getTgtMemIntrinsic.
if (TLI.getTgtMemIntrinsic(Info, CI, *MF, ID)) {
- Align Alignment = Info.align.getValueOr(
+ Align Alignment = Info.align.value_or(
DL->getABITypeAlign(Info.memVT.getTypeForEVT(F->getContext())));
LLT MemTy = Info.memVT.isSimple()
? getLLTForMVT(Info.memVT.getSimpleVT())
: LLT::scalar(Info.memVT.getStoreSizeInBits());
- MIB.addMemOperand(MF->getMachineMemOperand(MachinePointerInfo(Info.ptrVal),
- Info.flags, MemTy, Alignment));
+
+ // TODO: We currently just fallback to address space 0 if getTgtMemIntrinsic
+ // didn't yield anything useful.
+ MachinePointerInfo MPI;
+ if (Info.ptrVal)
+ MPI = MachinePointerInfo(Info.ptrVal, Info.offset);
+ else if (Info.fallbackAddressSpace)
+ MPI = MachinePointerInfo(*Info.fallbackAddressSpace);
+ MIB.addMemOperand(
+ MF->getMachineMemOperand(MPI, Info.flags, MemTy, Alignment, CI.getAAMetadata()));
}
return true;
@@ -2505,14 +2678,12 @@ bool IRTranslator::translateInvoke(const User &U,
bool LowerInlineAsm = I.isInlineAsm();
bool NeedEHLabel = true;
- // If it can't throw then use a fast-path without emitting EH labels.
- if (LowerInlineAsm)
- NeedEHLabel = (cast<InlineAsm>(I.getCalledOperand()))->canThrow();
// Emit the actual call, bracketed by EH_LABELs so that the MF knows about
// the region covered by the try.
MCSymbol *BeginSymbol = nullptr;
if (NeedEHLabel) {
+ MIRBuilder.buildInstr(TargetOpcode::G_INVOKE_REGION_START);
BeginSymbol = Context.createTempSymbol();
MIRBuilder.buildInstr(TargetOpcode::EH_LABEL).addSym(BeginSymbol);
}
@@ -2747,7 +2918,7 @@ bool IRTranslator::translateExtractElement(const User &U,
Register Idx;
if (auto *CI = dyn_cast<ConstantInt>(U.getOperand(1))) {
if (CI->getBitWidth() != PreferredVecIdxWidth) {
- APInt NewIdx = CI->getValue().sextOrTrunc(PreferredVecIdxWidth);
+ APInt NewIdx = CI->getValue().zextOrTrunc(PreferredVecIdxWidth);
auto *NewIdxCI = ConstantInt::get(CI->getContext(), NewIdx);
Idx = getOrCreateVReg(*NewIdxCI);
}
@@ -2756,7 +2927,7 @@ bool IRTranslator::translateExtractElement(const User &U,
Idx = getOrCreateVReg(*U.getOperand(1));
if (MRI->getType(Idx).getSizeInBits() != PreferredVecIdxWidth) {
const LLT VecIdxTy = LLT::scalar(PreferredVecIdxWidth);
- Idx = MIRBuilder.buildSExtOrTrunc(VecIdxTy, Idx).getReg(0);
+ Idx = MIRBuilder.buildZExtOrTrunc(VecIdxTy, Idx).getReg(0);
}
MIRBuilder.buildExtractVectorElement(Res, Val, Idx);
return true;
@@ -2867,6 +3038,18 @@ bool IRTranslator::translateAtomicRMW(const User &U,
case AtomicRMWInst::FSub:
Opcode = TargetOpcode::G_ATOMICRMW_FSUB;
break;
+ case AtomicRMWInst::FMax:
+ Opcode = TargetOpcode::G_ATOMICRMW_FMAX;
+ break;
+ case AtomicRMWInst::FMin:
+ Opcode = TargetOpcode::G_ATOMICRMW_FMIN;
+ break;
+ case AtomicRMWInst::UIncWrap:
+ Opcode = TargetOpcode::G_ATOMICRMW_UINC_WRAP;
+ break;
+ case AtomicRMWInst::UDecWrap:
+ Opcode = TargetOpcode::G_ATOMICRMW_UDEC_WRAP;
+ break;
}
MIRBuilder.buildAtomicRMW(
@@ -2920,7 +3103,7 @@ void IRTranslator::finishPendingPhis() {
for (unsigned i = 0; i < PI->getNumIncomingValues(); ++i) {
auto IRPred = PI->getIncomingBlock(i);
ArrayRef<Register> ValRegs = getOrCreateVRegs(*PI->getIncomingValue(i));
- for (auto Pred : getMachinePredBBs({IRPred, PI->getParent()})) {
+ for (auto *Pred : getMachinePredBBs({IRPred, PI->getParent()})) {
if (SeenPreds.count(Pred) || !PhiMBB->isPredecessor(Pred))
continue;
SeenPreds.insert(Pred);
@@ -2934,17 +3117,9 @@ void IRTranslator::finishPendingPhis() {
}
}
-bool IRTranslator::valueIsSplit(const Value &V,
- SmallVectorImpl<uint64_t> *Offsets) {
- SmallVector<LLT, 4> SplitTys;
- if (Offsets && !Offsets->empty())
- Offsets->clear();
- computeValueLLTs(*DL, *V.getType(), SplitTys, Offsets);
- return SplitTys.size() > 1;
-}
-
bool IRTranslator::translate(const Instruction &Inst) {
CurBuilder->setDebugLoc(Inst.getDebugLoc());
+ CurBuilder->setPCSections(Inst.getMetadata(LLVMContext::MD_pcsections));
auto &TLI = *MF->getSubtarget().getTargetLowering();
if (TLI.fallBackToDAGISel(Inst))
@@ -2962,11 +3137,9 @@ bool IRTranslator::translate(const Instruction &Inst) {
bool IRTranslator::translate(const Constant &C, Register Reg) {
// We only emit constants into the entry block from here. To prevent jumpy
- // debug behaviour set the line to 0.
+ // debug behaviour remove debug line.
if (auto CurrInstDL = CurBuilder->getDL())
- EntryBuilder->setDebugLoc(DILocation::get(C.getContext(), 0, 0,
- CurrInstDL.getScope(),
- CurrInstDL.getInlinedAt()));
+ EntryBuilder->setDebugLoc(DebugLoc());
if (auto CI = dyn_cast<ConstantInt>(&C))
EntryBuilder->buildConstant(Reg, *CI);
@@ -2984,7 +3157,7 @@ bool IRTranslator::translate(const Constant &C, Register Reg) {
// Return the scalar if it is a <1 x Ty> vector.
unsigned NumElts = CAZ->getElementCount().getFixedValue();
if (NumElts == 1)
- return translateCopy(C, *CAZ->getElementValue(0u), *EntryBuilder.get());
+ return translateCopy(C, *CAZ->getElementValue(0u), *EntryBuilder);
SmallVector<Register, 4> Ops;
for (unsigned I = 0; I < NumElts; ++I) {
Constant &Elt = *CAZ->getElementValue(I);
@@ -2994,8 +3167,7 @@ bool IRTranslator::translate(const Constant &C, Register Reg) {
} else if (auto CV = dyn_cast<ConstantDataVector>(&C)) {
// Return the scalar if it is a <1 x Ty> vector.
if (CV->getNumElements() == 1)
- return translateCopy(C, *CV->getElementAsConstant(0),
- *EntryBuilder.get());
+ return translateCopy(C, *CV->getElementAsConstant(0), *EntryBuilder);
SmallVector<Register, 4> Ops;
for (unsigned i = 0; i < CV->getNumElements(); ++i) {
Constant &Elt = *CV->getElementAsConstant(i);
@@ -3013,7 +3185,7 @@ bool IRTranslator::translate(const Constant &C, Register Reg) {
}
} else if (auto CV = dyn_cast<ConstantVector>(&C)) {
if (CV->getNumOperands() == 1)
- return translateCopy(C, *CV->getOperand(0), *EntryBuilder.get());
+ return translateCopy(C, *CV->getOperand(0), *EntryBuilder);
SmallVector<Register, 4> Ops;
for (unsigned i = 0; i < CV->getNumOperands(); ++i) {
Ops.push_back(getOrCreateVReg(*CV->getOperand(i)));
@@ -3255,14 +3427,13 @@ bool IRTranslator::emitSPDescriptorFailure(StackProtectorDescriptor &SPD,
return false;
}
- // On PS4, the "return address" must still be within the calling function,
- // even if it's at the very end, so emit an explicit TRAP here.
- // Passing 'true' for doesNotReturn above won't generate the trap for us.
+ // On PS4/PS5, the "return address" must still be within the calling
+ // function, even if it's at the very end, so emit an explicit TRAP here.
// WebAssembly needs an unreachable instruction after a non-returning call,
// because the function return type can be different from __stack_chk_fail's
// return type (void).
const TargetMachine &TM = MF->getTarget();
- if (TM.getTargetTriple().isPS4CPU() || TM.getTargetTriple().isWasm()) {
+ if (TM.getTargetTriple().isPS() || TM.getTargetTriple().isWasm()) {
LLVM_DEBUG(dbgs() << "Unhandled trap emission for stack protector fail\n");
return false;
}
@@ -3331,11 +3502,17 @@ bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) {
TM.resetTargetOptions(F);
EnableOpts = OptLevel != CodeGenOpt::None && !skipFunction(F);
FuncInfo.MF = MF;
- if (EnableOpts)
+ if (EnableOpts) {
+ AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
FuncInfo.BPI = &getAnalysis<BranchProbabilityInfoWrapperPass>().getBPI();
- else
+ } else {
+ AA = nullptr;
FuncInfo.BPI = nullptr;
+ }
+ AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(
+ MF->getFunction());
+ LibInfo = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
FuncInfo.CanLowerReturn = CLI->checkReturnTypeForCallConv(*MF);
const auto &TLI = *MF->getSubtarget().getTargetLowering();
@@ -3380,7 +3557,7 @@ bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) {
MF->push_back(MBB);
if (BB.hasAddressTaken())
- MBB->setHasAddressTaken();
+ MBB->setAddressTakenIRBlock(const_cast<BasicBlock *>(&BB));
if (!HasMustTailInVarArgFn)
HasMustTailInVarArgFn = checkForMustTailInVarArgFn(IsVarArg, BB);
@@ -3413,7 +3590,7 @@ bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) {
}
}
- if (!CLI->lowerFormalArguments(*EntryBuilder.get(), F, VRegArgs, FuncInfo)) {
+ if (!CLI->lowerFormalArguments(*EntryBuilder, F, VRegArgs, FuncInfo)) {
OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
F.getSubprogram(), &F.getEntryBlock());
R << "unable to lower arguments: " << ore::NV("Prototype", F.getType());
@@ -3469,8 +3646,13 @@ bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) {
return false;
}
- if (!finalizeBasicBlock(*BB, MBB))
+ if (!finalizeBasicBlock(*BB, MBB)) {
+ OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
+ BB->getTerminator()->getDebugLoc(), BB);
+ R << "unable to translate basic block";
+ reportTranslationError(*MF, *TPC, *ORE, R);
return false;
+ }
}
#ifndef NDEBUG
WrapperObserver.removeObserver(&Verifier);