diff options
Diffstat (limited to 'contrib/llvm-project/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp')
-rw-r--r-- | contrib/llvm-project/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 386 |
1 files changed, 284 insertions, 102 deletions
diff --git a/contrib/llvm-project/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/contrib/llvm-project/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp index 6d415c9c7f90..9a67a8d05a4d 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -15,16 +15,21 @@ #include "llvm/ADT/ScopeExit.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/Analysis/AliasAnalysis.h" +#include "llvm/Analysis/AssumptionCache.h" #include "llvm/Analysis/BranchProbabilityInfo.h" #include "llvm/Analysis/Loads.h" #include "llvm/Analysis/OptimizationRemarkEmitter.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/CodeGen/Analysis.h" +#include "llvm/CodeGen/GlobalISel/CSEInfo.h" +#include "llvm/CodeGen/GlobalISel/CSEMIRBuilder.h" #include "llvm/CodeGen/GlobalISel/CallLowering.h" #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h" #include "llvm/CodeGen/GlobalISel/InlineAsmLowering.h" #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" #include "llvm/CodeGen/LowLevelType.h" +#include "llvm/CodeGen/LowLevelTypeUtils.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" @@ -39,6 +44,7 @@ #include "llvm/CodeGen/TargetFrameLowering.h" #include "llvm/CodeGen/TargetInstrInfo.h" #include "llvm/CodeGen/TargetLowering.h" +#include "llvm/CodeGen/TargetOpcodes.h" #include "llvm/CodeGen/TargetPassConfig.h" #include "llvm/CodeGen/TargetRegisterInfo.h" #include "llvm/CodeGen/TargetSubtargetInfo.h" @@ -47,7 +53,6 @@ #include "llvm/IR/Constant.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DataLayout.h" -#include "llvm/IR/DebugInfo.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/Function.h" @@ -60,6 +65,7 @@ #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Metadata.h" #include "llvm/IR/PatternMatch.h" +#include "llvm/IR/Statepoint.h" #include "llvm/IR/Type.h" #include "llvm/IR/User.h" #include "llvm/IR/Value.h" @@ -70,7 +76,6 @@ #include "llvm/Support/CodeGen.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/LowLevelTypeImpl.h" #include "llvm/Support/MathExtras.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetIntrinsicInfo.h" @@ -78,9 +83,9 @@ #include "llvm/Transforms/Utils/MemoryOpRemark.h" #include <algorithm> #include <cassert> -#include <cstddef> #include <cstdint> #include <iterator> +#include <optional> #include <string> #include <utility> #include <vector> @@ -151,11 +156,11 @@ public: LLVM_DEBUG(dbgs() << "Checking DILocation from " << *CurrInst << " was copied to " << MI); #endif - // We allow insts in the entry block to have a debug loc line of 0 because + // We allow insts in the entry block to have no debug loc because // they could have originated from constants, and we don't want a jumpy // debug experience. assert((CurrInst->getDebugLoc() == MI.getDebugLoc() || - MI.getDebugLoc().getLine() == 0) && + (MI.getParent()->isEntryBlock() && !MI.getDebugLoc())) && "Line info was not transferred to all instructions"); } }; @@ -167,8 +172,11 @@ void IRTranslator::getAnalysisUsage(AnalysisUsage &AU) const { AU.addRequired<StackProtector>(); AU.addRequired<TargetPassConfig>(); AU.addRequired<GISelCSEAnalysisWrapperPass>(); - if (OptLevel != CodeGenOpt::None) + AU.addRequired<AssumptionCacheTracker>(); + if (OptLevel != CodeGenOpt::None) { AU.addRequired<BranchProbabilityInfoWrapperPass>(); + AU.addRequired<AAResultsWrapperPass>(); + } AU.addRequired<TargetLibraryInfoWrapperPass>(); AU.addPreserved<TargetLibraryInfoWrapperPass>(); getSelectionDAGFallbackAnalysisUsage(AU); @@ -293,7 +301,7 @@ bool IRTranslator::translateBinaryOp(unsigned Opcode, const User &U, Register Op0 = getOrCreateVReg(*U.getOperand(0)); Register Op1 = getOrCreateVReg(*U.getOperand(1)); Register Res = getOrCreateVReg(U); - uint16_t Flags = 0; + uint32_t Flags = 0; if (isa<Instruction>(U)) { const Instruction &I = cast<Instruction>(U); Flags = MachineInstr::copyFlagsFromInstruction(I); @@ -307,7 +315,7 @@ bool IRTranslator::translateUnaryOp(unsigned Opcode, const User &U, MachineIRBuilder &MIRBuilder) { Register Op0 = getOrCreateVReg(*U.getOperand(0)); Register Res = getOrCreateVReg(U); - uint16_t Flags = 0; + uint32_t Flags = 0; if (isa<Instruction>(U)) { const Instruction &I = cast<Instruction>(U); Flags = MachineInstr::copyFlagsFromInstruction(I); @@ -338,7 +346,7 @@ bool IRTranslator::translateCompare(const User &U, MIRBuilder.buildCopy( Res, getOrCreateVReg(*Constant::getAllOnesValue(U.getType()))); else { - uint16_t Flags = 0; + uint32_t Flags = 0; if (CI) Flags = MachineInstr::copyFlagsFromInstruction(*CI); MIRBuilder.buildFCmp(Pred, Res, Op0, Op1, Flags); @@ -685,7 +693,7 @@ bool IRTranslator::translateSwitch(const User &U, MachineIRBuilder &MIB) { BranchProbabilityInfo *BPI = FuncInfo.BPI; CaseClusterVector Clusters; Clusters.reserve(SI.getNumCases()); - for (auto &I : SI.cases()) { + for (const auto &I : SI.cases()) { MachineBasicBlock *Succ = &getMBB(*I.getCaseSuccessor()); assert(Succ && "Could not find successor mbb in mapping"); const ConstantInt *CaseVal = I.getCaseValue(); @@ -837,8 +845,8 @@ void IRTranslator::emitSwitchCase(SwitchCG::CaseBlock &CB, // For conditional branch lowering, we might try to do something silly like // emit an G_ICMP to compare an existing G_ICMP i1 result with true. If so, // just re-use the existing condition vreg. - if (MRI->getType(CondLHS).getSizeInBits() == 1 && CI && - CI->getZExtValue() == 1 && CB.PredInfo.Pred == CmpInst::ICMP_EQ) { + if (MRI->getType(CondLHS).getSizeInBits() == 1 && CI && CI->isOne() && + CB.PredInfo.Pred == CmpInst::ICMP_EQ) { Cond = CondLHS; } else { Register CondRHS = getOrCreateVReg(*CB.CmpRHS); @@ -1011,7 +1019,7 @@ void IRTranslator::emitBitTestHeader(SwitchCG::BitTestBlock &B, LLT MaskTy = SwitchOpTy; if (MaskTy.getSizeInBits() > PtrTy.getSizeInBits() || - !isPowerOf2_32(MaskTy.getSizeInBits())) + !llvm::has_single_bit<uint32_t>(MaskTy.getSizeInBits())) MaskTy = LLT::scalar(PtrTy.getSizeInBits()); else { // Ensure that the type will fit the mask value. @@ -1062,19 +1070,19 @@ void IRTranslator::emitBitTestCase(SwitchCG::BitTestBlock &BB, LLT SwitchTy = getLLTForMVT(BB.RegVT); Register Cmp; - unsigned PopCount = countPopulation(B.Mask); + unsigned PopCount = llvm::popcount(B.Mask); if (PopCount == 1) { // Testing for a single bit; just compare the shift count with what it // would need to be to shift a 1 bit in that position. auto MaskTrailingZeros = - MIB.buildConstant(SwitchTy, countTrailingZeros(B.Mask)); + MIB.buildConstant(SwitchTy, llvm::countr_zero(B.Mask)); Cmp = MIB.buildICmp(ICmpInst::ICMP_EQ, LLT::scalar(1), Reg, MaskTrailingZeros) .getReg(0); } else if (PopCount == BB.Range) { // There is only one zero bit in the range, test for it directly. auto MaskTrailingOnes = - MIB.buildConstant(SwitchTy, countTrailingOnes(B.Mask)); + MIB.buildConstant(SwitchTy, llvm::countr_one(B.Mask)); Cmp = MIB.buildICmp(CmpInst::ICMP_NE, LLT::scalar(1), Reg, MaskTrailingOnes) .getReg(0); } else { @@ -1276,26 +1284,37 @@ static bool isSwiftError(const Value *V) { bool IRTranslator::translateLoad(const User &U, MachineIRBuilder &MIRBuilder) { const LoadInst &LI = cast<LoadInst>(U); - if (DL->getTypeStoreSize(LI.getType()) == 0) + + unsigned StoreSize = DL->getTypeStoreSize(LI.getType()); + if (StoreSize == 0) return true; ArrayRef<Register> Regs = getOrCreateVRegs(LI); ArrayRef<uint64_t> Offsets = *VMap.getOffsets(LI); Register Base = getOrCreateVReg(*LI.getPointerOperand()); + AAMDNodes AAInfo = LI.getAAMetadata(); - Type *OffsetIRTy = DL->getIntPtrType(LI.getPointerOperandType()); + const Value *Ptr = LI.getPointerOperand(); + Type *OffsetIRTy = DL->getIndexType(Ptr->getType()); LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL); - if (CLI->supportSwiftError() && isSwiftError(LI.getPointerOperand())) { + if (CLI->supportSwiftError() && isSwiftError(Ptr)) { assert(Regs.size() == 1 && "swifterror should be single pointer"); - Register VReg = SwiftError.getOrCreateVRegUseAt(&LI, &MIRBuilder.getMBB(), - LI.getPointerOperand()); + Register VReg = + SwiftError.getOrCreateVRegUseAt(&LI, &MIRBuilder.getMBB(), Ptr); MIRBuilder.buildCopy(Regs[0], VReg); return true; } auto &TLI = *MF->getSubtarget().getTargetLowering(); - MachineMemOperand::Flags Flags = TLI.getLoadMemOperandFlags(LI, *DL); + MachineMemOperand::Flags Flags = + TLI.getLoadMemOperandFlags(LI, *DL, AC, LibInfo); + if (AA && !(Flags & MachineMemOperand::MOInvariant)) { + if (AA->pointsToConstantMemory( + MemoryLocation(Ptr, LocationSize::precise(StoreSize), AAInfo))) { + Flags |= MachineMemOperand::MOInvariant; + } + } const MDNode *Ranges = Regs.size() == 1 ? LI.getMetadata(LLVMContext::MD_range) : nullptr; @@ -1307,7 +1326,7 @@ bool IRTranslator::translateLoad(const User &U, MachineIRBuilder &MIRBuilder) { Align BaseAlign = getMemOpAlign(LI); auto MMO = MF->getMachineMemOperand( Ptr, Flags, MRI->getType(Regs[i]), - commonAlignment(BaseAlign, Offsets[i] / 8), LI.getAAMetadata(), Ranges, + commonAlignment(BaseAlign, Offsets[i] / 8), AAInfo, Ranges, LI.getSyncScopeID(), LI.getOrdering()); MIRBuilder.buildLoad(Regs[i], Addr, *MMO); } @@ -1324,7 +1343,7 @@ bool IRTranslator::translateStore(const User &U, MachineIRBuilder &MIRBuilder) { ArrayRef<uint64_t> Offsets = *VMap.getOffsets(*SI.getValueOperand()); Register Base = getOrCreateVReg(*SI.getPointerOperand()); - Type *OffsetIRTy = DL->getIntPtrType(SI.getPointerOperandType()); + Type *OffsetIRTy = DL->getIndexType(SI.getPointerOperandType()); LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL); if (CLI->supportSwiftError() && isSwiftError(SI.getPointerOperand())) { @@ -1401,7 +1420,7 @@ bool IRTranslator::translateInsertValue(const User &U, ArrayRef<uint64_t> DstOffsets = *VMap.getOffsets(U); ArrayRef<Register> SrcRegs = getOrCreateVRegs(*Src); ArrayRef<Register> InsertedRegs = getOrCreateVRegs(*U.getOperand(1)); - auto InsertedIt = InsertedRegs.begin(); + auto *InsertedIt = InsertedRegs.begin(); for (unsigned i = 0; i < DstRegs.size(); ++i) { if (DstOffsets[i] >= Offset && InsertedIt != InsertedRegs.end()) @@ -1420,7 +1439,7 @@ bool IRTranslator::translateSelect(const User &U, ArrayRef<Register> Op0Regs = getOrCreateVRegs(*U.getOperand(1)); ArrayRef<Register> Op1Regs = getOrCreateVRegs(*U.getOperand(2)); - uint16_t Flags = 0; + uint32_t Flags = 0; if (const SelectInst *SI = dyn_cast<SelectInst>(&U)) Flags = MachineInstr::copyFlagsFromInstruction(*SI); @@ -1450,8 +1469,14 @@ bool IRTranslator::translateBitCast(const User &U, MachineIRBuilder &MIRBuilder) { // If we're bitcasting to the source type, we can reuse the source vreg. if (getLLTForType(*U.getOperand(0)->getType(), *DL) == - getLLTForType(*U.getType(), *DL)) + getLLTForType(*U.getType(), *DL)) { + // If the source is a ConstantInt then it was probably created by + // ConstantHoisting and we should leave it alone. + if (isa<ConstantInt>(U.getOperand(0))) + return translateCast(TargetOpcode::G_CONSTANT_FOLD_BARRIER, U, + MIRBuilder); return translateCopy(U, *U.getOperand(0), MIRBuilder); + } return translateCast(TargetOpcode::G_BITCAST, U, MIRBuilder); } @@ -1470,7 +1495,7 @@ bool IRTranslator::translateGetElementPtr(const User &U, Register BaseReg = getOrCreateVReg(Op0); Type *PtrIRTy = Op0.getType(); LLT PtrTy = getLLTForType(*PtrIRTy, *DL); - Type *OffsetIRTy = DL->getIntPtrType(PtrIRTy); + Type *OffsetIRTy = DL->getIndexType(PtrIRTy); LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL); // Normalize Vector GEP - all scalar operands should be converted to the @@ -1495,7 +1520,7 @@ bool IRTranslator::translateGetElementPtr(const User &U, .getReg(0); PtrIRTy = FixedVectorType::get(PtrIRTy, VectorWidth); PtrTy = getLLTForType(*PtrIRTy, *DL); - OffsetIRTy = DL->getIntPtrType(PtrIRTy); + OffsetIRTy = DL->getIndexType(PtrIRTy); OffsetTy = getLLTForType(*OffsetIRTy, *DL); } @@ -1564,9 +1589,9 @@ bool IRTranslator::translateGetElementPtr(const User &U, bool IRTranslator::translateMemFunc(const CallInst &CI, MachineIRBuilder &MIRBuilder, unsigned Opcode) { - + const Value *SrcPtr = CI.getArgOperand(1); // If the source is undef, then just emit a nop. - if (isa<UndefValue>(CI.getArgOperand(1))) + if (isa<UndefValue>(SrcPtr)) return true; SmallVector<Register, 3> SrcRegs; @@ -1596,15 +1621,20 @@ bool IRTranslator::translateMemFunc(const CallInst &CI, unsigned IsVol = cast<ConstantInt>(CI.getArgOperand(CI.arg_size() - 1))->getZExtValue(); + ConstantInt *CopySize = nullptr; + if (auto *MCI = dyn_cast<MemCpyInst>(&CI)) { DstAlign = MCI->getDestAlign().valueOrOne(); SrcAlign = MCI->getSourceAlign().valueOrOne(); + CopySize = dyn_cast<ConstantInt>(MCI->getArgOperand(2)); } else if (auto *MCI = dyn_cast<MemCpyInlineInst>(&CI)) { DstAlign = MCI->getDestAlign().valueOrOne(); SrcAlign = MCI->getSourceAlign().valueOrOne(); + CopySize = dyn_cast<ConstantInt>(MCI->getArgOperand(2)); } else if (auto *MMI = dyn_cast<MemMoveInst>(&CI)) { DstAlign = MMI->getDestAlign().valueOrOne(); SrcAlign = MMI->getSourceAlign().valueOrOne(); + CopySize = dyn_cast<ConstantInt>(MMI->getArgOperand(2)); } else { auto *MSI = cast<MemSetInst>(&CI); DstAlign = MSI->getDestAlign().valueOrOne(); @@ -1618,14 +1648,31 @@ bool IRTranslator::translateMemFunc(const CallInst &CI, } // Create mem operands to store the alignment and volatile info. - auto VolFlag = IsVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone; - ICall.addMemOperand(MF->getMachineMemOperand( - MachinePointerInfo(CI.getArgOperand(0)), - MachineMemOperand::MOStore | VolFlag, 1, DstAlign)); + MachineMemOperand::Flags LoadFlags = MachineMemOperand::MOLoad; + MachineMemOperand::Flags StoreFlags = MachineMemOperand::MOStore; + if (IsVol) { + LoadFlags |= MachineMemOperand::MOVolatile; + StoreFlags |= MachineMemOperand::MOVolatile; + } + + AAMDNodes AAInfo = CI.getAAMetadata(); + if (AA && CopySize && + AA->pointsToConstantMemory(MemoryLocation( + SrcPtr, LocationSize::precise(CopySize->getZExtValue()), AAInfo))) { + LoadFlags |= MachineMemOperand::MOInvariant; + + // FIXME: pointsToConstantMemory probably does not imply dereferenceable, + // but the previous usage implied it did. Probably should check + // isDereferenceableAndAlignedPointer. + LoadFlags |= MachineMemOperand::MODereferenceable; + } + + ICall.addMemOperand( + MF->getMachineMemOperand(MachinePointerInfo(CI.getArgOperand(0)), + StoreFlags, 1, DstAlign, AAInfo)); if (Opcode != TargetOpcode::G_MEMSET) ICall.addMemOperand(MF->getMachineMemOperand( - MachinePointerInfo(CI.getArgOperand(1)), - MachineMemOperand::MOLoad | VolFlag, 1, SrcAlign)); + MachinePointerInfo(SrcPtr), LoadFlags, 1, SrcAlign, AAInfo)); return true; } @@ -1719,6 +1766,8 @@ unsigned IRTranslator::getSimpleIntrinsicOpcode(Intrinsic::ID ID) { return TargetOpcode::G_FLOG2; case Intrinsic::log10: return TargetOpcode::G_FLOG10; + case Intrinsic::ldexp: + return TargetOpcode::G_FLDEXP; case Intrinsic::nearbyint: return TargetOpcode::G_FNEARBYINT; case Intrinsic::pow: @@ -1786,7 +1835,7 @@ bool IRTranslator::translateSimpleIntrinsic(const CallInst &CI, // Yes. Let's translate it. SmallVector<llvm::SrcOp, 4> VRegs; - for (auto &Arg : CI.args()) + for (const auto &Arg : CI.args()) VRegs.push_back(getOrCreateVReg(*Arg)); MIRBuilder.buildInstr(Op, {getOrCreateVReg(CI)}, VRegs, @@ -1811,6 +1860,8 @@ static unsigned getConstrainedOpcode(Intrinsic::ID ID) { return TargetOpcode::G_STRICT_FMA; case Intrinsic::experimental_constrained_sqrt: return TargetOpcode::G_STRICT_FSQRT; + case Intrinsic::experimental_constrained_ldexp: + return TargetOpcode::G_STRICT_FLDEXP; default: return 0; } @@ -1818,13 +1869,13 @@ static unsigned getConstrainedOpcode(Intrinsic::ID ID) { bool IRTranslator::translateConstrainedFPIntrinsic( const ConstrainedFPIntrinsic &FPI, MachineIRBuilder &MIRBuilder) { - fp::ExceptionBehavior EB = FPI.getExceptionBehavior().getValue(); + fp::ExceptionBehavior EB = *FPI.getExceptionBehavior(); unsigned Opcode = getConstrainedOpcode(FPI.getIntrinsicID()); if (!Opcode) return false; - unsigned Flags = MachineInstr::copyFlagsFromInstruction(FPI); + uint32_t Flags = MachineInstr::copyFlagsFromInstruction(FPI); if (EB == fp::ExceptionBehavior::ebIgnore) Flags |= MachineInstr::NoFPExcept; @@ -1839,14 +1890,66 @@ bool IRTranslator::translateConstrainedFPIntrinsic( return true; } +std::optional<MCRegister> IRTranslator::getArgPhysReg(Argument &Arg) { + auto VRegs = getOrCreateVRegs(Arg); + if (VRegs.size() != 1) + return std::nullopt; + + // Arguments are lowered as a copy of a livein physical register. + auto *VRegDef = MF->getRegInfo().getVRegDef(VRegs[0]); + if (!VRegDef || !VRegDef->isCopy()) + return std::nullopt; + return VRegDef->getOperand(1).getReg().asMCReg(); +} + +bool IRTranslator::translateIfEntryValueArgument(const DbgValueInst &DebugInst, + MachineIRBuilder &MIRBuilder) { + auto *Arg = dyn_cast<Argument>(DebugInst.getValue()); + if (!Arg) + return false; + + const DIExpression *Expr = DebugInst.getExpression(); + if (!Expr->isEntryValue()) + return false; + + std::optional<MCRegister> PhysReg = getArgPhysReg(*Arg); + if (!PhysReg) { + LLVM_DEBUG(dbgs() << "Dropping dbg.value: expression is entry_value but " + "couldn't find a physical register\n" + << DebugInst << "\n"); + return true; + } + + MIRBuilder.buildDirectDbgValue(*PhysReg, DebugInst.getVariable(), + DebugInst.getExpression()); + return true; +} + +bool IRTranslator::translateIfEntryValueArgument( + const DbgDeclareInst &DebugInst) { + auto *Arg = dyn_cast<Argument>(DebugInst.getAddress()); + if (!Arg) + return false; + + const DIExpression *Expr = DebugInst.getExpression(); + if (!Expr->isEntryValue()) + return false; + + std::optional<MCRegister> PhysReg = getArgPhysReg(*Arg); + if (!PhysReg) + return false; + + MF->setVariableDbgInfo(DebugInst.getVariable(), Expr, *PhysReg, + DebugInst.getDebugLoc()); + return true; +} + bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder) { if (auto *MI = dyn_cast<AnyMemIntrinsic>(&CI)) { if (ORE->enabled()) { - const Function &F = *MI->getParent()->getParent(); - auto &TLI = getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F); - if (MemoryOpRemark::canHandle(MI, TLI)) { - MemoryOpRemark R(*ORE, "gisel-irtranslator-memsize", *DL, TLI); + if (MemoryOpRemark::canHandle(MI, *LibInfo)) { + MemoryOpRemark R(*ORE, "gisel-irtranslator-memsize", *DL, *LibInfo); R.visit(MI); } } @@ -1907,12 +2010,16 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID, // instructions (in fact, they get ignored if they *do* exist). MF->setVariableDbgInfo(DI.getVariable(), DI.getExpression(), getOrCreateFrameIndex(*AI), DI.getDebugLoc()); - } else { - // A dbg.declare describes the address of a source variable, so lower it - // into an indirect DBG_VALUE. - MIRBuilder.buildIndirectDbgValue(getOrCreateVReg(*Address), - DI.getVariable(), DI.getExpression()); + return true; } + + if (translateIfEntryValueArgument(DI)) + return true; + + // A dbg.declare describes the address of a source variable, so lower it + // into an indirect DBG_VALUE. + MIRBuilder.buildIndirectDbgValue(getOrCreateVReg(*Address), + DI.getVariable(), DI.getExpression()); return true; } case Intrinsic::dbg_label: { @@ -1953,16 +2060,32 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID, // DI cannot produce a valid DBG_VALUE, so produce an undef DBG_VALUE to // terminate any prior location. MIRBuilder.buildIndirectDbgValue(0, DI.getVariable(), DI.getExpression()); - } else if (const auto *CI = dyn_cast<Constant>(V)) { + return true; + } + if (const auto *CI = dyn_cast<Constant>(V)) { MIRBuilder.buildConstDbgValue(*CI, DI.getVariable(), DI.getExpression()); - } else { - for (Register Reg : getOrCreateVRegs(*V)) { - // FIXME: This does not handle register-indirect values at offset 0. The - // direct/indirect thing shouldn't really be handled by something as - // implicit as reg+noreg vs reg+imm in the first place, but it seems - // pretty baked in right now. - MIRBuilder.buildDirectDbgValue(Reg, DI.getVariable(), DI.getExpression()); - } + return true; + } + if (auto *AI = dyn_cast<AllocaInst>(V); + AI && AI->isStaticAlloca() && DI.getExpression()->startsWithDeref()) { + // If the value is an alloca and the expression starts with a + // dereference, track a stack slot instead of a register, as registers + // may be clobbered. + auto ExprOperands = DI.getExpression()->getElements(); + auto *ExprDerefRemoved = + DIExpression::get(AI->getContext(), ExprOperands.drop_front()); + MIRBuilder.buildFIDbgValue(getOrCreateFrameIndex(*AI), DI.getVariable(), + ExprDerefRemoved); + return true; + } + if (translateIfEntryValueArgument(DI, MIRBuilder)) + return true; + for (Register Reg : getOrCreateVRegs(*V)) { + // FIXME: This does not handle register-indirect values at offset 0. The + // direct/indirect thing shouldn't really be handled by something as + // implicit as reg+noreg vs reg+imm in the first place, but it seems + // pretty baked in right now. + MIRBuilder.buildDirectDbgValue(Reg, DI.getVariable(), DI.getExpression()); } return true; } @@ -2052,6 +2175,13 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID, getOrCreateVReg(*CI.getArgOperand(0)), MachineInstr::copyFlagsFromInstruction(CI)); return true; + case Intrinsic::frexp: { + ArrayRef<Register> VRegs = getOrCreateVRegs(CI); + MIRBuilder.buildFFrexp(VRegs[0], VRegs[1], + getOrCreateVReg(*CI.getArgOperand(0)), + MachineInstr::copyFlagsFromInstruction(CI)); + return true; + } case Intrinsic::memcpy_inline: return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMCPY_INLINE); case Intrinsic::memcpy: @@ -2077,9 +2207,14 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID, getStackGuard(getOrCreateVReg(CI), MIRBuilder); return true; case Intrinsic::stackprotector: { + const TargetLowering &TLI = *MF->getSubtarget().getTargetLowering(); LLT PtrTy = getLLTForType(*CI.getArgOperand(0)->getType(), *DL); - Register GuardVal = MRI->createGenericVirtualRegister(PtrTy); - getStackGuard(GuardVal, MIRBuilder); + Register GuardVal; + if (TLI.useLoadStackGuardNode()) { + GuardVal = MRI->createGenericVirtualRegister(PtrTy); + getStackGuard(GuardVal, MIRBuilder); + } else + GuardVal = getOrCreateVReg(*CI.getArgOperand(0)); // The guard's value. AllocaInst *Slot = cast<AllocaInst>(CI.getArgOperand(1)); int FI = getOrCreateFrameIndex(*Slot); @@ -2252,6 +2387,34 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID, Info.OrigRet = {Register(), Type::getVoidTy(CI.getContext()), 0}; return CLI->lowerCall(MIRBuilder, Info); } + case Intrinsic::fptrunc_round: { + uint32_t Flags = MachineInstr::copyFlagsFromInstruction(CI); + + // Convert the metadata argument to a constant integer + Metadata *MD = cast<MetadataAsValue>(CI.getArgOperand(1))->getMetadata(); + std::optional<RoundingMode> RoundMode = + convertStrToRoundingMode(cast<MDString>(MD)->getString()); + + // Add the Rounding mode as an integer + MIRBuilder + .buildInstr(TargetOpcode::G_INTRINSIC_FPTRUNC_ROUND, + {getOrCreateVReg(CI)}, + {getOrCreateVReg(*CI.getArgOperand(0))}, Flags) + .addImm((int)*RoundMode); + + return true; + } + case Intrinsic::is_fpclass: { + Value *FpValue = CI.getOperand(0); + ConstantInt *TestMaskValue = cast<ConstantInt>(CI.getOperand(1)); + + MIRBuilder + .buildInstr(TargetOpcode::G_IS_FPCLASS, {getOrCreateVReg(CI)}, + {getOrCreateVReg(*FpValue)}) + .addImm(TestMaskValue->getZExtValue()); + + return true; + } #define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \ case Intrinsic::INTRINSIC: #include "llvm/IR/ConstrainedOps.def" @@ -2284,14 +2447,14 @@ bool IRTranslator::translateCallBase(const CallBase &CB, SmallVector<ArrayRef<Register>, 8> Args; Register SwiftInVReg = 0; Register SwiftErrorVReg = 0; - for (auto &Arg : CB.args()) { + for (const auto &Arg : CB.args()) { if (CLI->supportSwiftError() && isSwiftError(Arg)) { assert(SwiftInVReg == 0 && "Expected only one swift error argument"); LLT Ty = getLLTForType(*Arg->getType(), *DL); SwiftInVReg = MRI->createGenericVirtualRegister(Ty); MIRBuilder.buildCopy(SwiftInVReg, SwiftError.getOrCreateVRegUseAt( &CB, &MIRBuilder.getMBB(), Arg)); - Args.emplace_back(makeArrayRef(SwiftInVReg)); + Args.emplace_back(ArrayRef(SwiftInVReg)); SwiftErrorVReg = SwiftError.getOrCreateVRegDefAt(&CB, &MIRBuilder.getMBB(), Arg); continue; @@ -2301,10 +2464,8 @@ bool IRTranslator::translateCallBase(const CallBase &CB, if (auto *CI = dyn_cast<CallInst>(&CB)) { if (ORE->enabled()) { - const Function &F = *CI->getParent()->getParent(); - auto &TLI = getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F); - if (MemoryOpRemark::canHandle(CI, TLI)) { - MemoryOpRemark R(*ORE, "gisel-irtranslator-memsize", *DL, TLI); + if (MemoryOpRemark::canHandle(CI, *LibInfo)) { + MemoryOpRemark R(*ORE, "gisel-irtranslator-memsize", *DL, *LibInfo); R.visit(CI); } } @@ -2342,6 +2503,10 @@ bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) { if (CI.countOperandBundlesOfType(LLVMContext::OB_cfguardtarget)) return false; + // FIXME: support statepoints and related. + if (isa<GCStatepointInst, GCRelocateInst, GCResultInst>(U)) + return false; + if (CI.isInlineAsm()) return translateInlineAsm(CI, MIRBuilder); @@ -2373,7 +2538,7 @@ bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) { if (isa<FPMathOperator>(CI)) MIB->copyIRFlags(CI); - for (auto &Arg : enumerate(CI.args())) { + for (const auto &Arg : enumerate(CI.args())) { // If this is required to be an immediate, don't materialize it in a // register. if (CI.paramHasAttr(Arg.index(), Attribute::ImmArg)) { @@ -2409,13 +2574,21 @@ bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) { TargetLowering::IntrinsicInfo Info; // TODO: Add a GlobalISel version of getTgtMemIntrinsic. if (TLI.getTgtMemIntrinsic(Info, CI, *MF, ID)) { - Align Alignment = Info.align.getValueOr( + Align Alignment = Info.align.value_or( DL->getABITypeAlign(Info.memVT.getTypeForEVT(F->getContext()))); LLT MemTy = Info.memVT.isSimple() ? getLLTForMVT(Info.memVT.getSimpleVT()) : LLT::scalar(Info.memVT.getStoreSizeInBits()); - MIB.addMemOperand(MF->getMachineMemOperand(MachinePointerInfo(Info.ptrVal), - Info.flags, MemTy, Alignment)); + + // TODO: We currently just fallback to address space 0 if getTgtMemIntrinsic + // didn't yield anything useful. + MachinePointerInfo MPI; + if (Info.ptrVal) + MPI = MachinePointerInfo(Info.ptrVal, Info.offset); + else if (Info.fallbackAddressSpace) + MPI = MachinePointerInfo(*Info.fallbackAddressSpace); + MIB.addMemOperand( + MF->getMachineMemOperand(MPI, Info.flags, MemTy, Alignment, CI.getAAMetadata())); } return true; @@ -2505,14 +2678,12 @@ bool IRTranslator::translateInvoke(const User &U, bool LowerInlineAsm = I.isInlineAsm(); bool NeedEHLabel = true; - // If it can't throw then use a fast-path without emitting EH labels. - if (LowerInlineAsm) - NeedEHLabel = (cast<InlineAsm>(I.getCalledOperand()))->canThrow(); // Emit the actual call, bracketed by EH_LABELs so that the MF knows about // the region covered by the try. MCSymbol *BeginSymbol = nullptr; if (NeedEHLabel) { + MIRBuilder.buildInstr(TargetOpcode::G_INVOKE_REGION_START); BeginSymbol = Context.createTempSymbol(); MIRBuilder.buildInstr(TargetOpcode::EH_LABEL).addSym(BeginSymbol); } @@ -2747,7 +2918,7 @@ bool IRTranslator::translateExtractElement(const User &U, Register Idx; if (auto *CI = dyn_cast<ConstantInt>(U.getOperand(1))) { if (CI->getBitWidth() != PreferredVecIdxWidth) { - APInt NewIdx = CI->getValue().sextOrTrunc(PreferredVecIdxWidth); + APInt NewIdx = CI->getValue().zextOrTrunc(PreferredVecIdxWidth); auto *NewIdxCI = ConstantInt::get(CI->getContext(), NewIdx); Idx = getOrCreateVReg(*NewIdxCI); } @@ -2756,7 +2927,7 @@ bool IRTranslator::translateExtractElement(const User &U, Idx = getOrCreateVReg(*U.getOperand(1)); if (MRI->getType(Idx).getSizeInBits() != PreferredVecIdxWidth) { const LLT VecIdxTy = LLT::scalar(PreferredVecIdxWidth); - Idx = MIRBuilder.buildSExtOrTrunc(VecIdxTy, Idx).getReg(0); + Idx = MIRBuilder.buildZExtOrTrunc(VecIdxTy, Idx).getReg(0); } MIRBuilder.buildExtractVectorElement(Res, Val, Idx); return true; @@ -2867,6 +3038,18 @@ bool IRTranslator::translateAtomicRMW(const User &U, case AtomicRMWInst::FSub: Opcode = TargetOpcode::G_ATOMICRMW_FSUB; break; + case AtomicRMWInst::FMax: + Opcode = TargetOpcode::G_ATOMICRMW_FMAX; + break; + case AtomicRMWInst::FMin: + Opcode = TargetOpcode::G_ATOMICRMW_FMIN; + break; + case AtomicRMWInst::UIncWrap: + Opcode = TargetOpcode::G_ATOMICRMW_UINC_WRAP; + break; + case AtomicRMWInst::UDecWrap: + Opcode = TargetOpcode::G_ATOMICRMW_UDEC_WRAP; + break; } MIRBuilder.buildAtomicRMW( @@ -2920,7 +3103,7 @@ void IRTranslator::finishPendingPhis() { for (unsigned i = 0; i < PI->getNumIncomingValues(); ++i) { auto IRPred = PI->getIncomingBlock(i); ArrayRef<Register> ValRegs = getOrCreateVRegs(*PI->getIncomingValue(i)); - for (auto Pred : getMachinePredBBs({IRPred, PI->getParent()})) { + for (auto *Pred : getMachinePredBBs({IRPred, PI->getParent()})) { if (SeenPreds.count(Pred) || !PhiMBB->isPredecessor(Pred)) continue; SeenPreds.insert(Pred); @@ -2934,17 +3117,9 @@ void IRTranslator::finishPendingPhis() { } } -bool IRTranslator::valueIsSplit(const Value &V, - SmallVectorImpl<uint64_t> *Offsets) { - SmallVector<LLT, 4> SplitTys; - if (Offsets && !Offsets->empty()) - Offsets->clear(); - computeValueLLTs(*DL, *V.getType(), SplitTys, Offsets); - return SplitTys.size() > 1; -} - bool IRTranslator::translate(const Instruction &Inst) { CurBuilder->setDebugLoc(Inst.getDebugLoc()); + CurBuilder->setPCSections(Inst.getMetadata(LLVMContext::MD_pcsections)); auto &TLI = *MF->getSubtarget().getTargetLowering(); if (TLI.fallBackToDAGISel(Inst)) @@ -2962,11 +3137,9 @@ bool IRTranslator::translate(const Instruction &Inst) { bool IRTranslator::translate(const Constant &C, Register Reg) { // We only emit constants into the entry block from here. To prevent jumpy - // debug behaviour set the line to 0. + // debug behaviour remove debug line. if (auto CurrInstDL = CurBuilder->getDL()) - EntryBuilder->setDebugLoc(DILocation::get(C.getContext(), 0, 0, - CurrInstDL.getScope(), - CurrInstDL.getInlinedAt())); + EntryBuilder->setDebugLoc(DebugLoc()); if (auto CI = dyn_cast<ConstantInt>(&C)) EntryBuilder->buildConstant(Reg, *CI); @@ -2984,7 +3157,7 @@ bool IRTranslator::translate(const Constant &C, Register Reg) { // Return the scalar if it is a <1 x Ty> vector. unsigned NumElts = CAZ->getElementCount().getFixedValue(); if (NumElts == 1) - return translateCopy(C, *CAZ->getElementValue(0u), *EntryBuilder.get()); + return translateCopy(C, *CAZ->getElementValue(0u), *EntryBuilder); SmallVector<Register, 4> Ops; for (unsigned I = 0; I < NumElts; ++I) { Constant &Elt = *CAZ->getElementValue(I); @@ -2994,8 +3167,7 @@ bool IRTranslator::translate(const Constant &C, Register Reg) { } else if (auto CV = dyn_cast<ConstantDataVector>(&C)) { // Return the scalar if it is a <1 x Ty> vector. if (CV->getNumElements() == 1) - return translateCopy(C, *CV->getElementAsConstant(0), - *EntryBuilder.get()); + return translateCopy(C, *CV->getElementAsConstant(0), *EntryBuilder); SmallVector<Register, 4> Ops; for (unsigned i = 0; i < CV->getNumElements(); ++i) { Constant &Elt = *CV->getElementAsConstant(i); @@ -3013,7 +3185,7 @@ bool IRTranslator::translate(const Constant &C, Register Reg) { } } else if (auto CV = dyn_cast<ConstantVector>(&C)) { if (CV->getNumOperands() == 1) - return translateCopy(C, *CV->getOperand(0), *EntryBuilder.get()); + return translateCopy(C, *CV->getOperand(0), *EntryBuilder); SmallVector<Register, 4> Ops; for (unsigned i = 0; i < CV->getNumOperands(); ++i) { Ops.push_back(getOrCreateVReg(*CV->getOperand(i))); @@ -3255,14 +3427,13 @@ bool IRTranslator::emitSPDescriptorFailure(StackProtectorDescriptor &SPD, return false; } - // On PS4, the "return address" must still be within the calling function, - // even if it's at the very end, so emit an explicit TRAP here. - // Passing 'true' for doesNotReturn above won't generate the trap for us. + // On PS4/PS5, the "return address" must still be within the calling + // function, even if it's at the very end, so emit an explicit TRAP here. // WebAssembly needs an unreachable instruction after a non-returning call, // because the function return type can be different from __stack_chk_fail's // return type (void). const TargetMachine &TM = MF->getTarget(); - if (TM.getTargetTriple().isPS4CPU() || TM.getTargetTriple().isWasm()) { + if (TM.getTargetTriple().isPS() || TM.getTargetTriple().isWasm()) { LLVM_DEBUG(dbgs() << "Unhandled trap emission for stack protector fail\n"); return false; } @@ -3331,11 +3502,17 @@ bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) { TM.resetTargetOptions(F); EnableOpts = OptLevel != CodeGenOpt::None && !skipFunction(F); FuncInfo.MF = MF; - if (EnableOpts) + if (EnableOpts) { + AA = &getAnalysis<AAResultsWrapperPass>().getAAResults(); FuncInfo.BPI = &getAnalysis<BranchProbabilityInfoWrapperPass>().getBPI(); - else + } else { + AA = nullptr; FuncInfo.BPI = nullptr; + } + AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache( + MF->getFunction()); + LibInfo = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F); FuncInfo.CanLowerReturn = CLI->checkReturnTypeForCallConv(*MF); const auto &TLI = *MF->getSubtarget().getTargetLowering(); @@ -3380,7 +3557,7 @@ bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) { MF->push_back(MBB); if (BB.hasAddressTaken()) - MBB->setHasAddressTaken(); + MBB->setAddressTakenIRBlock(const_cast<BasicBlock *>(&BB)); if (!HasMustTailInVarArgFn) HasMustTailInVarArgFn = checkForMustTailInVarArgFn(IsVarArg, BB); @@ -3413,7 +3590,7 @@ bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) { } } - if (!CLI->lowerFormalArguments(*EntryBuilder.get(), F, VRegArgs, FuncInfo)) { + if (!CLI->lowerFormalArguments(*EntryBuilder, F, VRegArgs, FuncInfo)) { OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure", F.getSubprogram(), &F.getEntryBlock()); R << "unable to lower arguments: " << ore::NV("Prototype", F.getType()); @@ -3469,8 +3646,13 @@ bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) { return false; } - if (!finalizeBasicBlock(*BB, MBB)) + if (!finalizeBasicBlock(*BB, MBB)) { + OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure", + BB->getTerminator()->getDebugLoc(), BB); + R << "unable to translate basic block"; + reportTranslationError(*MF, *TPC, *ORE, R); return false; + } } #ifndef NDEBUG WrapperObserver.removeObserver(&Verifier); |