Lines Matching refs:MIRBuilder
92 : MIRBuilder(Builder), Observer(Observer), MRI(MF.getRegInfo()), in LegalizerHelper()
95 MIRBuilder.setChangeObserver(Observer); in LegalizerHelper()
101 : MIRBuilder(B), Observer(Observer), MRI(MF.getRegInfo()), LI(LI), in LegalizerHelper()
103 MIRBuilder.setChangeObserver(Observer); in LegalizerHelper()
109 MIRBuilder.setInstrAndDebugLoc(MI); in legalizeInstrStep()
153 MIRBuilder.buildUnmerge(VRegs, Reg); in extractParts()
171 MIRBuilder.buildUnmerge(VRegs, Reg); in extractParts()
188 MIRBuilder.buildExtract(NewReg, Reg, MainSize * I); in extractParts()
195 MIRBuilder.buildExtract(NewReg, Reg, Offset); in extractParts()
210 MIRBuilder.buildMerge(DstReg, PartRegs); in insertParts()
215 MIRBuilder.buildConcatVectors(DstReg, PartRegs); in insertParts()
217 MIRBuilder.buildBuildVector(DstReg, PartRegs); in insertParts()
225 MIRBuilder.buildUndef(CurResultReg); in insertParts()
230 MIRBuilder.buildInsert(NewResultReg, CurResultReg, PartReg, Offset); in insertParts()
240 MIRBuilder.buildInsert(NewResultReg, CurResultReg, LeftoverRegs[I], Offset); in insertParts()
267 auto Unmerge = MIRBuilder.buildUnmerge(GCDTy, SrcReg); in extractGCDType()
295 PadReg = MIRBuilder.buildConstant(GCDTy, 0).getReg(0); in buildLCMMergePieces()
297 PadReg = MIRBuilder.buildUndef(GCDTy).getReg(0); in buildLCMMergePieces()
303 MIRBuilder.buildConstant(LLT::scalar(64), GCDTy.getSizeInBits() - 1); in buildLCMMergePieces()
304 PadReg = MIRBuilder.buildAShr(GCDTy, VRegs.back(), ShiftAmt).getReg(0); in buildLCMMergePieces()
342 AllPadReg = MIRBuilder.buildUndef(NarrowTy).getReg(0); in buildLCMMergePieces()
344 AllPadReg = MIRBuilder.buildConstant(NarrowTy, 0).getReg(0); in buildLCMMergePieces()
360 Remerge[I] = MIRBuilder.buildMerge(NarrowTy, SubMerge).getReg(0); in buildLCMMergePieces()
379 MIRBuilder.buildMerge(DstReg, RemergeRegs); in buildWidenedRemergeToDst()
383 auto Remerge = MIRBuilder.buildMerge(LCMTy, RemergeRegs); in buildWidenedRemergeToDst()
385 MIRBuilder.buildTrunc(DstReg, Remerge); in buildWidenedRemergeToDst()
396 MIRBuilder.buildUnmerge(UnmergeDefs, in buildWidenedRemergeToDst()
397 MIRBuilder.buildMerge(LCMTy, RemergeRegs)); in buildWidenedRemergeToDst()
525 llvm::createLibcall(MachineIRBuilder &MIRBuilder, const char *Name, in createLibcall() argument
529 auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering(); in createLibcall()
536 if (!CLI.lowerCall(MIRBuilder, Info)) in createLibcall()
543 llvm::createLibcall(MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall, in createLibcall() argument
546 auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering(); in createLibcall()
549 return createLibcall(MIRBuilder, Name, Result, Args, CC); in createLibcall()
554 simpleLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, unsigned Size, in simpleLibcall() argument
561 return createLibcall(MIRBuilder, Libcall, {MI.getOperand(0).getReg(), OpType}, in simpleLibcall()
566 llvm::createMemLibcall(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI, in createMemLibcall() argument
568 auto &Ctx = MIRBuilder.getMF().getFunction().getContext(); in createMemLibcall()
585 auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering(); in createMemLibcall()
586 auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering(); in createMemLibcall()
608 isLibCallInTailPosition(MIRBuilder.getTII(), MI); in createMemLibcall()
611 if (!CLI.lowerCall(MIRBuilder, Info)) in createMemLibcall()
654 conversionLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, Type *ToType, in conversionLibcall() argument
657 return createLibcall(MIRBuilder, Libcall, {MI.getOperand(0).getReg(), ToType}, in conversionLibcall()
665 auto &Ctx = MIRBuilder.getMF().getFunction().getContext(); in libcall()
676 auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy); in libcall()
708 auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy); in libcall()
719 LegalizeResult Status = conversionLibcall(MI, MIRBuilder, ToTy, FromTy ); in libcall()
732 MI, MIRBuilder, in libcall()
747 MI, MIRBuilder, in libcall()
757 LegalizeResult Result = createMemLibcall(MIRBuilder, *MIRBuilder.getMRI(), MI); in libcall()
791 Register ImplicitReg = MIRBuilder.buildUndef(ImplicitTy).getReg(0); in narrowScalar()
792 MIRBuilder.buildAnyExt(DstReg, ImplicitReg); in narrowScalar()
802 DstRegs.push_back(MIRBuilder.buildUndef(NarrowTy).getReg(0)); in narrowScalar()
805 MIRBuilder.buildBuildVector(DstReg, DstRegs); in narrowScalar()
807 MIRBuilder.buildMerge(DstReg, DstRegs); in narrowScalar()
821 auto K = MIRBuilder.buildConstant(NarrowTy, in narrowScalar()
831 auto K = MIRBuilder.buildConstant( in narrowScalar()
857 auto Unmerge = MIRBuilder.buildUnmerge(NarrowTy, MI.getOperand(1)); in narrowScalar()
858 MIRBuilder.buildCopy(MI.getOperand(0), Unmerge.getReg(0)); in narrowScalar()
884 MIRBuilder.buildUAddo(DstReg, CarryOut, Src1Regs[i], Src2Regs[i]); in narrowScalar()
886 MIRBuilder.buildUAdde(DstReg, CarryOut, Src1Regs[i], in narrowScalar()
895 MIRBuilder.buildBuildVector(DstReg, DstRegs); in narrowScalar()
897 MIRBuilder.buildMerge(DstReg, DstRegs); in narrowScalar()
915 MIRBuilder.buildInstr(TargetOpcode::G_USUBO, {DstReg, BorrowOut}, in narrowScalar()
923 MIRBuilder.buildInstr(TargetOpcode::G_USUBE, {DstReg, BorrowOut}, in narrowScalar()
929 MIRBuilder.buildMerge(MI.getOperand(0), DstRegs); in narrowScalar()
949 MIRBuilder.buildLoad(TmpReg, MI.getOperand(1), MMO); in narrowScalar()
950 MIRBuilder.buildAnyExt(DstReg, TmpReg); in narrowScalar()
966 MIRBuilder.buildLoad(TmpReg, PtrReg, MMO); in narrowScalar()
968 MIRBuilder.buildLoadInstr(MI.getOpcode(), TmpReg, PtrReg, MMO); in narrowScalar()
972 MIRBuilder.buildZExt(DstReg, TmpReg); in narrowScalar()
974 MIRBuilder.buildSExt(DstReg, TmpReg); in narrowScalar()
996 MIRBuilder.buildTrunc(TmpReg, SrcReg); in narrowScalar()
997 MIRBuilder.buildStore(TmpReg, MI.getOperand(1), MMO); in narrowScalar()
1070 MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator()); in narrowScalar()
1075 MIRBuilder.setInsertPt(MBB, MI); in narrowScalar()
1079 MIRBuilder.buildInstr(TargetOpcode::G_PHI).addDef(DstRegs[i]); in narrowScalar()
1083 MIRBuilder.setInsertPt(MBB, MBB.getFirstNonPHI()); in narrowScalar()
1084 MIRBuilder.buildMerge(MI.getOperand(0), DstRegs); in narrowScalar()
1108 MIRBuilder.buildUnmerge({LHSL, LHSH}, MI.getOperand(2)); in narrowScalar()
1112 MIRBuilder.buildUnmerge({RHSL, RHSH}, MI.getOperand(3)); in narrowScalar()
1119 MachineInstrBuilder XorL = MIRBuilder.buildXor(NarrowTy, LHSL, RHSL); in narrowScalar()
1120 MachineInstrBuilder XorH = MIRBuilder.buildXor(NarrowTy, LHSH, RHSH); in narrowScalar()
1121 MachineInstrBuilder Or = MIRBuilder.buildOr(NarrowTy, XorL, XorH); in narrowScalar()
1122 MachineInstrBuilder Zero = MIRBuilder.buildConstant(NarrowTy, 0); in narrowScalar()
1123 MIRBuilder.buildICmp(Pred, MI.getOperand(0), Or, Zero); in narrowScalar()
1125 MachineInstrBuilder CmpH = MIRBuilder.buildICmp(Pred, ResTy, LHSH, RHSH); in narrowScalar()
1127 MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, ResTy, LHSH, RHSH); in narrowScalar()
1128 MachineInstrBuilder CmpLU = MIRBuilder.buildICmp( in narrowScalar()
1130 MIRBuilder.buildSelect(MI.getOperand(0), CmpHEQ, CmpLU, CmpH); in narrowScalar()
1149 auto TruncMIB = MIRBuilder.buildTrunc(NarrowTy, MO1); in narrowScalar()
1154 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); in narrowScalar()
1155 MIRBuilder.buildSExt(MO2, DstExt); in narrowScalar()
1182 MIRBuilder.buildUnmerge(SrcRegs, MI.getOperand(1)); in narrowScalar()
1185 MIRBuilder.buildConstant(NarrowTy, NarrowTy.getScalarSizeInBits() - 1) in narrowScalar()
1202 MIRBuilder.buildAShr(NarrowTy, PartialExtensionReg, AshrCstReg) in narrowScalar()
1207 MIRBuilder in narrowScalar()
1218 MIRBuilder.buildMerge(DstReg, DstRegs); in narrowScalar()
1233 auto DstPart = MIRBuilder.buildInstr(MI.getOpcode(), {NarrowTy}, in narrowScalar()
1238 MIRBuilder.buildMerge(MI.getOperand(0), DstRegs); in narrowScalar()
1284 const DataLayout &DL = MIRBuilder.getDataLayout(); in coerceToScalar()
1289 return MIRBuilder.buildPtrToInt(NewTy, Val).getReg(0); in coerceToScalar()
1297 NewVal = MIRBuilder.buildPtrToInt(NewTy, NewVal).getReg(0); in coerceToScalar()
1298 return MIRBuilder.buildBitcast(NewTy, NewVal).getReg(0); in coerceToScalar()
1304 auto ExtB = MIRBuilder.buildInstr(ExtOpcode, {WideTy}, {MO}); in widenScalarSrc()
1311 auto ExtB = MIRBuilder.buildTrunc(NarrowTy, MO); in narrowScalarSrc()
1319 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); in widenScalarDst()
1320 MIRBuilder.buildInstr(TruncOpcode, {MO}, {DstExt}); in widenScalarDst()
1328 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); in narrowScalarDst()
1329 MIRBuilder.buildInstr(ExtOpcode, {MO}, {DstTrunc}); in narrowScalarDst()
1336 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); in moreElementsVectorDst()
1355 Register ImpDef = MIRBuilder.buildUndef(OldTy).getReg(0); in moreElementsVectorSrc()
1359 auto Concat = MIRBuilder.buildConcatVectors(MoreTy, Parts); in moreElementsVectorSrc()
1365 Register ImpDef = MIRBuilder.buildUndef(MoreTy).getReg(0); in moreElementsVectorSrc()
1366 MIRBuilder.buildInsert(MoreReg, ImpDef, MO.getReg(), 0); in moreElementsVectorSrc()
1372 Op.setReg(MIRBuilder.buildBitcast(CastTy, Op).getReg(0)); in bitcastSrc()
1378 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); in bitcastDst()
1379 MIRBuilder.buildBitcast(MO, CastDst); in bitcastDst()
1407 Register ResultReg = MIRBuilder.buildZExt(WideTy, Src1).getReg(0); in widenScalarMergeValues()
1415 auto ZextInput = MIRBuilder.buildZExt(WideTy, SrcReg); in widenScalarMergeValues()
1420 auto ShiftAmt = MIRBuilder.buildConstant(WideTy, Offset); in widenScalarMergeValues()
1421 auto Shl = MIRBuilder.buildShl(WideTy, ZextInput, ShiftAmt); in widenScalarMergeValues()
1422 MIRBuilder.buildOr(NextResult, ResultReg, Shl); in widenScalarMergeValues()
1427 MIRBuilder.buildTrunc(DstReg, ResultReg); in widenScalarMergeValues()
1429 MIRBuilder.buildIntToPtr(DstReg, ResultReg); in widenScalarMergeValues()
1470 auto Unmerge = MIRBuilder.buildUnmerge(GCDTy, SrcReg); in widenScalarMergeValues()
1478 Register UndefReg = MIRBuilder.buildUndef(GCDTy).getReg(0); in widenScalarMergeValues()
1488 auto Merge = MIRBuilder.buildMerge(WideTy, Slicer.take_front(PartsPerGCD)); in widenScalarMergeValues()
1495 MIRBuilder.buildMerge(DstReg, NewMergeRegs); in widenScalarMergeValues()
1497 auto FinalMerge = MIRBuilder.buildMerge(WideDstTy, NewMergeRegs); in widenScalarMergeValues()
1498 MIRBuilder.buildTrunc(DstReg, FinalMerge.getReg(0)); in widenScalarMergeValues()
1523 Register Undef = MIRBuilder.buildUndef(WideTy).getReg(0); in widenWithUnmerge()
1526 UnmergeSrc = MIRBuilder.buildMerge(LCMTy, MergeParts).getReg(0); in widenWithUnmerge()
1535 MIRBuilder.buildUnmerge(UnmergeResults, UnmergeSrc); in widenWithUnmerge()
1558 const DataLayout &DL = MIRBuilder.getDataLayout(); in widenScalarUnmergeValues()
1566 SrcReg = MIRBuilder.buildPtrToInt(SrcTy, SrcReg).getReg(0); in widenScalarUnmergeValues()
1574 SrcReg = MIRBuilder.buildAnyExt(WideTy, SrcReg).getReg(0); in widenScalarUnmergeValues()
1581 MIRBuilder.buildTrunc(Dst0Reg, SrcReg); in widenScalarUnmergeValues()
1583 auto ShiftAmt = MIRBuilder.buildConstant(SrcTy, DstSize * I); in widenScalarUnmergeValues()
1584 auto Shr = MIRBuilder.buildLShr(SrcTy, SrcReg, ShiftAmt); in widenScalarUnmergeValues()
1585 MIRBuilder.buildTrunc(MI.getOperand(I), Shr); in widenScalarUnmergeValues()
1603 WideSrc = MIRBuilder.buildAnyExt(LCMTy, WideSrc).getReg(0); in widenScalarUnmergeValues()
1606 auto Unmerge = MIRBuilder.buildUnmerge(WideTy, WideSrc); in widenScalarUnmergeValues()
1633 auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_UNMERGE_VALUES); in widenScalarUnmergeValues()
1659 MIRBuilder.buildMerge(MI.getOperand(I).getReg(), RemergeParts); in widenScalarUnmergeValues()
1686 const DataLayout &DL = MIRBuilder.getDataLayout(); in widenScalarExtract()
1691 Src = MIRBuilder.buildPtrToInt(SrcAsIntTy, Src); in widenScalarExtract()
1700 MIRBuilder.buildTrunc(DstReg, in widenScalarExtract()
1701 MIRBuilder.buildAnyExtOrTrunc(WideTy, Src)); in widenScalarExtract()
1709 Src = MIRBuilder.buildAnyExt(WideTy, Src); in widenScalarExtract()
1713 auto LShr = MIRBuilder.buildLShr( in widenScalarExtract()
1714 ShiftTy, Src, MIRBuilder.buildConstant(ShiftTy, Offset)); in widenScalarExtract()
1715 MIRBuilder.buildTrunc(DstReg, LShr); in widenScalarExtract()
1782 auto LHS = MIRBuilder.buildAnyExt(WideTy, MI.getOperand(1)); in widenScalarAddSubShlSat()
1783 auto RHS = IsShift ? MIRBuilder.buildZExt(WideTy, MI.getOperand(2)) in widenScalarAddSubShlSat()
1784 : MIRBuilder.buildAnyExt(WideTy, MI.getOperand(2)); in widenScalarAddSubShlSat()
1785 auto ShiftK = MIRBuilder.buildConstant(WideTy, SHLAmount); in widenScalarAddSubShlSat()
1786 auto ShiftL = MIRBuilder.buildShl(WideTy, LHS, ShiftK); in widenScalarAddSubShlSat()
1787 auto ShiftR = IsShift ? RHS : MIRBuilder.buildShl(WideTy, RHS, ShiftK); in widenScalarAddSubShlSat()
1789 auto WideInst = MIRBuilder.buildInstr(MI.getOpcode(), {WideTy}, in widenScalarAddSubShlSat()
1794 auto Result = IsSigned ? MIRBuilder.buildAShr(WideTy, WideInst, ShiftK) in widenScalarAddSubShlSat()
1795 : MIRBuilder.buildLShr(WideTy, WideInst, ShiftK); in widenScalarAddSubShlSat()
1797 MIRBuilder.buildTrunc(DstReg, Result); in widenScalarAddSubShlSat()
1819 auto LHSZext = MIRBuilder.buildZExt(WideTy, MI.getOperand(2)); in widenScalar()
1820 auto RHSZext = MIRBuilder.buildZExt(WideTy, MI.getOperand(3)); in widenScalar()
1825 auto NewOp = MIRBuilder.buildInstr(Opcode, {WideTy}, {LHSZext, RHSZext}); in widenScalar()
1829 auto AndOp = MIRBuilder.buildAnd( in widenScalar()
1830 WideTy, NewOp, MIRBuilder.buildConstant(WideTy, Mask)); in widenScalar()
1832 MIRBuilder.buildICmp(CmpInst::ICMP_NE, MI.getOperand(1), NewOp, AndOp); in widenScalar()
1834 MIRBuilder.buildTrunc(MI.getOperand(0), NewOp); in widenScalar()
1860 auto MIBSrc = MIRBuilder.buildZExt(WideTy, SrcReg); in widenScalar()
1868 MIBSrc = MIRBuilder.buildOr( in widenScalar()
1869 WideTy, MIBSrc, MIRBuilder.buildConstant(WideTy, TopBit)); in widenScalar()
1873 auto MIBNewOp = MIRBuilder.buildInstr(MI.getOpcode(), {WideTy}, {MIBSrc}); in widenScalar()
1879 MIBNewOp = MIRBuilder.buildSub( in widenScalar()
1880 WideTy, MIBNewOp, MIRBuilder.buildConstant(WideTy, SizeDiff)); in widenScalar()
1883 MIRBuilder.buildZExtOrTrunc(MI.getOperand(0), MIBNewOp); in widenScalar()
1898 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); in widenScalar()
1902 MIRBuilder.buildConstant(ShiftAmtReg, DiffBits); in widenScalar()
1903 MIRBuilder.buildLShr(ShrReg, DstExt, ShiftAmtReg); in widenScalar()
1905 MIRBuilder.buildTrunc(DstReg, ShrReg); in widenScalar()
1919 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); in widenScalar()
1921 auto ShiftAmt = MIRBuilder.buildConstant(WideTy, DiffBits); in widenScalar()
1922 auto Shift = MIRBuilder.buildLShr(WideTy, DstExt, ShiftAmt); in widenScalar()
1923 MIRBuilder.buildTrunc(DstReg, Shift); in widenScalar()
2019 widenScalarSrc(MI, WideTy, 1, MIRBuilder.getBoolExtOp(IsVec, false)); in widenScalar()
2082 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext(); in widenScalar()
2101 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext(); in widenScalar()
2134 widenScalarSrc(MI, WideTy, 0, MIRBuilder.getBoolExtOp(false, false)); in widenScalar()
2177 MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator()); in widenScalar()
2182 MIRBuilder.setInsertPt(MBB, --MBB.getFirstNonPHI()); in widenScalar()
2311 MI.setDesc(MIRBuilder.getTII().get(TargetOpcode::G_BUILD_VECTOR_TRUNC)); in widenScalar()
2392 getUnmergePieces(SrcRegs, MIRBuilder, Src, SrcPartTy); in lowerBitcast()
2394 SrcReg = MIRBuilder.buildBitcast(DstCastTy, SrcReg).getReg(0); in lowerBitcast()
2396 getUnmergePieces(SrcRegs, MIRBuilder, Src, SrcEltTy); in lowerBitcast()
2398 MIRBuilder.buildMerge(Dst, SrcRegs); in lowerBitcast()
2405 getUnmergePieces(SrcRegs, MIRBuilder, Src, DstTy.getElementType()); in lowerBitcast()
2406 MIRBuilder.buildMerge(Dst, SrcRegs); in lowerBitcast()
2459 Register CastVec = MIRBuilder.buildBitcast(CastTy, SrcVec).getReg(0); in bitcastExtractVectorElt()
2481 auto NewEltsPerOldEltK = MIRBuilder.buildConstant(IdxTy, NewEltsPerOldElt); in bitcastExtractVectorElt()
2484 auto NewBaseIdx = MIRBuilder.buildMul(IdxTy, Idx, NewEltsPerOldEltK); in bitcastExtractVectorElt()
2487 auto IdxOffset = MIRBuilder.buildConstant(IdxTy, I); in bitcastExtractVectorElt()
2488 auto TmpIdx = MIRBuilder.buildAdd(IdxTy, NewBaseIdx, IdxOffset); in bitcastExtractVectorElt()
2489 auto Elt = MIRBuilder.buildExtractVectorElement(NewEltTy, CastVec, TmpIdx); in bitcastExtractVectorElt()
2493 auto NewVec = MIRBuilder.buildBuildVector(MidTy, NewOps); in bitcastExtractVectorElt()
2494 MIRBuilder.buildBitcast(Dst, NewVec); in bitcastExtractVectorElt()
2523 auto Log2Ratio = MIRBuilder.buildConstant(IdxTy, Log2EltRatio); in bitcastExtractVectorElt()
2526 auto ScaledIdx = MIRBuilder.buildLShr(IdxTy, Idx, Log2Ratio); in bitcastExtractVectorElt()
2530 WideElt = MIRBuilder.buildExtractVectorElement(NewEltTy, CastVec, in bitcastExtractVectorElt()
2536 MIRBuilder, Idx, NewEltSize, OldEltSize); in bitcastExtractVectorElt()
2539 auto ExtractedBits = MIRBuilder.buildLShr(NewEltTy, WideElt, OffsetBits); in bitcastExtractVectorElt()
2540 MIRBuilder.buildTrunc(Dst, ExtractedBits); in bitcastExtractVectorElt()
2603 Register CastVec = MIRBuilder.buildBitcast(CastTy, SrcVec).getReg(0); in bitcastInsertVectorElt()
2615 auto Log2Ratio = MIRBuilder.buildConstant(IdxTy, Log2EltRatio); in bitcastInsertVectorElt()
2618 auto ScaledIdx = MIRBuilder.buildLShr(IdxTy, Idx, Log2Ratio); in bitcastInsertVectorElt()
2622 ExtractedElt = MIRBuilder.buildExtractVectorElement(NewEltTy, CastVec, in bitcastInsertVectorElt()
2628 MIRBuilder, Idx, NewEltSize, OldEltSize); in bitcastInsertVectorElt()
2630 Register InsertedElt = buildBitFieldInsert(MIRBuilder, ExtractedElt, in bitcastInsertVectorElt()
2633 InsertedElt = MIRBuilder.buildInsertVectorElement( in bitcastInsertVectorElt()
2637 MIRBuilder.buildBitcast(Dst, InsertedElt); in bitcastInsertVectorElt()
2676 MachineFunction &MF = MIRBuilder.getMF(); in lowerLoad()
2687 auto LargeLoad = MIRBuilder.buildLoadInstr( in lowerLoad()
2690 auto OffsetCst = MIRBuilder.buildConstant( in lowerLoad()
2694 MIRBuilder.buildPtrAdd(PtrAddReg, PtrReg, OffsetCst.getReg(0)); in lowerLoad()
2695 auto SmallLoad = MIRBuilder.buildLoad(SmallLdReg, SmallPtr.getReg(0), in lowerLoad()
2698 auto ShiftAmt = MIRBuilder.buildConstant(AnyExtTy, LargeSplitSize); in lowerLoad()
2699 auto Shift = MIRBuilder.buildShl(AnyExtTy, SmallLoad, ShiftAmt); in lowerLoad()
2700 auto Or = MIRBuilder.buildOr(AnyExtTy, Shift, LargeLoad); in lowerLoad()
2701 MIRBuilder.buildTrunc(DstReg, {Or.getReg(0)}); in lowerLoad()
2706 MIRBuilder.buildLoad(DstReg, PtrReg, MMO); in lowerLoad()
2714 MIRBuilder.buildLoad(TmpReg, PtrReg, MMO); in lowerLoad()
2719 MIRBuilder.buildAnyExtOrTrunc(DstReg, TmpReg); in lowerLoad()
2722 MIRBuilder.buildSExt(DstReg, TmpReg); in lowerLoad()
2725 MIRBuilder.buildZExt(DstReg, TmpReg); in lowerLoad()
2757 auto ExtVal = MIRBuilder.buildAnyExt(ExtendTy, SrcReg); in lowerStore()
2762 auto ShiftAmt = MIRBuilder.buildConstant(ExtendTy, LargeSplitSize); in lowerStore()
2763 auto SmallVal = MIRBuilder.buildLShr(ExtendTy, ExtVal, ShiftAmt); in lowerStore()
2767 auto OffsetCst = MIRBuilder.buildConstant( in lowerStore()
2771 MIRBuilder.buildPtrAdd(PtrAddReg, PtrReg, OffsetCst.getReg(0)); in lowerStore()
2773 MachineFunction &MF = MIRBuilder.getMF(); in lowerStore()
2778 MIRBuilder.buildStore(ExtVal.getReg(0), PtrReg, *LargeMMO); in lowerStore()
2779 MIRBuilder.buildStore(SmallVal.getReg(0), SmallPtr.getReg(0), *SmallMMO); in lowerStore()
2844 MI.setDesc(MIRBuilder.getTII().get(NewOpcode)); in changeOpcode()
2861 MIRBuilder.buildInstr(MI.getOpcode() == G_SREM ? G_SDIV : G_UDIV, {Ty}, in lower()
2864 auto Prod = MIRBuilder.buildMul(Ty, Quot, MI.getOperand(2)); in lower()
2865 MIRBuilder.buildSub(MI.getOperand(0), MI.getOperand(1), Prod); in lower()
2890 const auto &TII = MIRBuilder.getTII(); in lower()
2895 auto HiPart = MIRBuilder.buildInstr(Opcode, {Ty}, {LHS, RHS}); in lower()
2896 auto Zero = MIRBuilder.buildConstant(Ty, 0); in lower()
2899 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); in lower()
2904 auto ShiftAmt = MIRBuilder.buildConstant(Ty, Ty.getSizeInBits() - 1); in lower()
2905 auto Shifted = MIRBuilder.buildAShr(Ty, Res, ShiftAmt); in lower()
2906 MIRBuilder.buildICmp(CmpInst::ICMP_NE, Overflow, HiPart, Shifted); in lower()
2908 MIRBuilder.buildICmp(CmpInst::ICMP_NE, Overflow, HiPart, Zero); in lower()
2921 MIRBuilder.buildConstant(Ty, APInt::getSignMask(Ty.getSizeInBits())); in lower()
2923 MIRBuilder.buildXor(Res, SubByReg, SignMask); in lower()
2939 MIRBuilder.buildFNeg(Neg, RHS); in lower()
2940 MIRBuilder.buildFAdd(Res, LHS, Neg, MI.getFlags()); in lower()
2962 MIRBuilder.buildAtomicCmpXchg(OldValRes, Addr, CmpVal, NewVal, in lower()
2964 MIRBuilder.buildICmp(CmpInst::ICMP_EQ, SuccessRes, OldValRes, CmpVal); in lower()
2986 MIRBuilder.buildAdd(Res, LHS, RHS); in lower()
2987 MIRBuilder.buildICmp(CmpInst::ICMP_ULT, CarryOut, Res, RHS); in lower()
3000 auto TmpRes = MIRBuilder.buildAdd(Ty, LHS, RHS); in lower()
3001 auto ZExtCarryIn = MIRBuilder.buildZExt(Ty, CarryIn); in lower()
3002 MIRBuilder.buildAdd(Res, TmpRes, ZExtCarryIn); in lower()
3003 MIRBuilder.buildICmp(CmpInst::ICMP_ULT, CarryOut, Res, LHS); in lower()
3014 MIRBuilder.buildSub(Res, LHS, RHS); in lower()
3015 MIRBuilder.buildICmp(CmpInst::ICMP_ULT, BorrowOut, LHS, RHS); in lower()
3029 auto TmpRes = MIRBuilder.buildSub(Ty, LHS, RHS); in lower()
3030 auto ZExtBorrowIn = MIRBuilder.buildZExt(Ty, BorrowIn); in lower()
3031 MIRBuilder.buildSub(Res, TmpRes, ZExtBorrowIn); in lower()
3033 auto LHS_EQ_RHS = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, CondTy, LHS, RHS); in lower()
3034 auto LHS_ULT_RHS = MIRBuilder.buildICmp(CmpInst::ICMP_ULT, CondTy, LHS, RHS); in lower()
3035 MIRBuilder.buildSelect(BorrowOut, LHS_EQ_RHS, BorrowIn, LHS_ULT_RHS); in lower()
3075 auto MIBSz = MIRBuilder.buildConstant(DstTy, DstTy.getScalarSizeInBits() - SizeInBits); in lower()
3076 MIRBuilder.buildShl(TmpRes, SrcReg, MIBSz->getOperand(0)); in lower()
3077 MIRBuilder.buildAShr(DstReg, TmpRes, MIBSz->getOperand(0)); in lower()
3131 MIRBuilder.buildConstant(DstTy, DstTy.getScalarSizeInBits() - 1); in lower()
3133 MIRBuilder.buildAShr(DstTy, OpReg, ShiftAmt); in lower()
3134 auto Add = MIRBuilder.buildAdd(DstTy, OpReg, Shift); in lower()
3135 MIRBuilder.buildXor(MI.getOperand(0).getReg(), Add, Shift); in lower()
3157 MachineFunction &MF = MIRBuilder.getMF(); in createStackTemporary()
3158 const DataLayout &DL = MIRBuilder.getDataLayout(); in createStackTemporary()
3165 return MIRBuilder.buildFrameIndex(FramePtrTy, FrameIdx); in createStackTemporary()
3194 Index = clampDynamicVectorIndex(MIRBuilder, Index, VecTy); in getVectorElementPointer()
3197 auto Mul = MIRBuilder.buildMul(IdxTy, Index, in getVectorElementPointer()
3198 MIRBuilder.buildConstant(IdxTy, EltSize)); in getVectorElementPointer()
3201 return MIRBuilder.buildPtrAdd(PtrTy, VecPtr, Mul).getReg(0); in getVectorElementPointer()
3212 auto NewUndef = MIRBuilder.buildUndef(NarrowTy); in fewerElementsVectorImplicitDef()
3275 NewInsts.push_back(MIRBuilder.buildInstrNoInsert(MI.getOpcode()) in fewerElementsVectorMultiEltType()
3283 NewInsts.push_back(MIRBuilder.buildInstrNoInsert(MI.getOpcode()) in fewerElementsVectorMultiEltType()
3307 MIRBuilder.insertInstr(MIB); in fewerElementsVectorMultiEltType()
3348 MIRBuilder.buildInstr(MI.getOpcode(), {DstReg}, {SrcRegs[I]}); in fewerElementsVectorCasts()
3355 MIRBuilder.buildConcatVectors(DstReg, DstRegs); in fewerElementsVectorCasts()
3357 MIRBuilder.buildBuildVector(DstReg, DstRegs); in fewerElementsVectorCasts()
3413 MIRBuilder.buildICmp(Pred, DstReg, Src1Regs[I], Src2Regs[I]); in fewerElementsVectorCmp()
3416 = MIRBuilder.buildFCmp(Pred, DstReg, Src1Regs[I], Src2Regs[I]); in fewerElementsVectorCmp()
3422 MIRBuilder.buildConcatVectors(DstReg, DstRegs); in fewerElementsVectorCmp()
3424 MIRBuilder.buildBuildVector(DstReg, DstRegs); in fewerElementsVectorCmp()
3488 MIRBuilder.buildSelect(DstReg, CondTy.isVector() ? Src0Regs[i] : CondReg, in fewerElementsVectorSelect()
3494 MIRBuilder.buildConcatVectors(DstReg, DstRegs); in fewerElementsVectorSelect()
3496 MIRBuilder.buildBuildVector(DstReg, DstRegs); in fewerElementsVectorSelect()
3527 NewInsts.push_back(MIRBuilder.buildInstr(TargetOpcode::G_PHI) in fewerElementsVectorPhi()
3536 MIRBuilder.setInsertPt(*MBB, MBB->getFirstNonPHI()); in fewerElementsVectorPhi()
3548 MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator()); in fewerElementsVectorPhi()
3593 auto Unmerge = MIRBuilder.buildUnmerge(GCDTy, SrcReg); in fewerElementsVectorUnmergeValues()
3598 auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_UNMERGE_VALUES); in fewerElementsVectorUnmergeValues()
3675 MIRBuilder.buildUndef(DstReg); in fewerElementsVectorExtractInsertVectorElt()
3692 MIRBuilder.buildConstant(IdxTy, IdxVal - NewNumElts * PartIdx); in fewerElementsVectorExtractInsertVectorElt()
3698 auto InsertPart = MIRBuilder.buildInsertVectorElement( in fewerElementsVectorExtractInsertVectorElt()
3706 MIRBuilder.buildExtractVectorElement(DstReg, VecParts[PartIdx], NewIdx); in fewerElementsVectorExtractInsertVectorElt()
3775 MachineFunction &MF = MIRBuilder.getMF(); in reduceLoadStoreWidth()
3783 MIRBuilder.materializePtrAdd(NewAddrReg, AddrReg, OffsetTy, ByteOffset); in reduceLoadStoreWidth()
3791 MIRBuilder.buildLoad(Dst, NewAddrReg, *NewMMO); in reduceLoadStoreWidth()
3793 MIRBuilder.buildStore(ValRegs[Idx], NewAddrReg, *NewMMO); in reduceLoadStoreWidth()
3856 SrcReg = MIRBuilder.buildBitcast(SrcTy, SrcReg).getReg(0); in reduceOperationWidth()
3897 auto Inst = MIRBuilder.buildInstr(Opc, {NarrowDstTy}, InputRegs, Flags); in reduceOperationWidth()
3906 MIRBuilder.buildUndef(NarrowDstTy).getReg(0)); in reduceOperationWidth()
3920 MIRBuilder.buildBitcast(DstReg, MergeDstReg); in reduceOperationWidth()
3940 R = MIRBuilder.buildSExtInReg(NarrowTy, R, Imm).getReg(0); in fewerElementsVectorSextInReg()
4076 MIRBuilder.buildUnmerge({InL, InH}, MI.getOperand(1)); in narrowScalarShiftByConstant()
4079 MIRBuilder.buildMerge(MI.getOperand(0), {InL, InH}); in narrowScalarShiftByConstant()
4091 Lo = Hi = MIRBuilder.buildConstant(NVT, 0); in narrowScalarShiftByConstant()
4093 Lo = MIRBuilder.buildConstant(NVT, 0); in narrowScalarShiftByConstant()
4094 Hi = MIRBuilder.buildShl(NVT, InL, in narrowScalarShiftByConstant()
4095 MIRBuilder.buildConstant(AmtTy, Amt - NVTBits)); in narrowScalarShiftByConstant()
4097 Lo = MIRBuilder.buildConstant(NVT, 0); in narrowScalarShiftByConstant()
4100 Lo = MIRBuilder.buildShl(NVT, InL, MIRBuilder.buildConstant(AmtTy, Amt)); in narrowScalarShiftByConstant()
4102 MIRBuilder.buildShl(NVT, InH, MIRBuilder.buildConstant(AmtTy, Amt)); in narrowScalarShiftByConstant()
4103 auto OrRHS = MIRBuilder.buildLShr( in narrowScalarShiftByConstant()
4104 NVT, InL, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits)); in narrowScalarShiftByConstant()
4105 Hi = MIRBuilder.buildOr(NVT, OrLHS, OrRHS); in narrowScalarShiftByConstant()
4109 Lo = Hi = MIRBuilder.buildConstant(NVT, 0); in narrowScalarShiftByConstant()
4111 Lo = MIRBuilder.buildLShr(NVT, InH, in narrowScalarShiftByConstant()
4112 MIRBuilder.buildConstant(AmtTy, Amt - NVTBits)); in narrowScalarShiftByConstant()
4113 Hi = MIRBuilder.buildConstant(NVT, 0); in narrowScalarShiftByConstant()
4116 Hi = MIRBuilder.buildConstant(NVT, 0); in narrowScalarShiftByConstant()
4118 auto ShiftAmtConst = MIRBuilder.buildConstant(AmtTy, Amt); in narrowScalarShiftByConstant()
4120 auto OrLHS = MIRBuilder.buildLShr(NVT, InL, ShiftAmtConst); in narrowScalarShiftByConstant()
4121 auto OrRHS = MIRBuilder.buildShl( in narrowScalarShiftByConstant()
4122 NVT, InH, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits)); in narrowScalarShiftByConstant()
4124 Lo = MIRBuilder.buildOr(NVT, OrLHS, OrRHS); in narrowScalarShiftByConstant()
4125 Hi = MIRBuilder.buildLShr(NVT, InH, ShiftAmtConst); in narrowScalarShiftByConstant()
4129 Hi = Lo = MIRBuilder.buildAShr( in narrowScalarShiftByConstant()
4130 NVT, InH, MIRBuilder.buildConstant(AmtTy, NVTBits - 1)); in narrowScalarShiftByConstant()
4132 Lo = MIRBuilder.buildAShr(NVT, InH, in narrowScalarShiftByConstant()
4133 MIRBuilder.buildConstant(AmtTy, Amt - NVTBits)); in narrowScalarShiftByConstant()
4134 Hi = MIRBuilder.buildAShr(NVT, InH, in narrowScalarShiftByConstant()
4135 MIRBuilder.buildConstant(AmtTy, NVTBits - 1)); in narrowScalarShiftByConstant()
4138 Hi = MIRBuilder.buildAShr(NVT, InH, in narrowScalarShiftByConstant()
4139 MIRBuilder.buildConstant(AmtTy, NVTBits - 1)); in narrowScalarShiftByConstant()
4141 auto ShiftAmtConst = MIRBuilder.buildConstant(AmtTy, Amt); in narrowScalarShiftByConstant()
4143 auto OrLHS = MIRBuilder.buildLShr(NVT, InL, ShiftAmtConst); in narrowScalarShiftByConstant()
4144 auto OrRHS = MIRBuilder.buildShl( in narrowScalarShiftByConstant()
4145 NVT, InH, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits)); in narrowScalarShiftByConstant()
4147 Lo = MIRBuilder.buildOr(NVT, OrLHS, OrRHS); in narrowScalarShiftByConstant()
4148 Hi = MIRBuilder.buildAShr(NVT, InH, ShiftAmtConst); in narrowScalarShiftByConstant()
4152 MIRBuilder.buildMerge(MI.getOperand(0), {Lo, Hi}); in narrowScalarShiftByConstant()
4196 auto NewBits = MIRBuilder.buildConstant(ShiftAmtTy, NewBitSize); in narrowScalarShift()
4200 MIRBuilder.buildUnmerge({InL, InH}, MI.getOperand(1)); in narrowScalarShift()
4202 auto AmtExcess = MIRBuilder.buildSub(ShiftAmtTy, Amt, NewBits); in narrowScalarShift()
4203 auto AmtLack = MIRBuilder.buildSub(ShiftAmtTy, NewBits, Amt); in narrowScalarShift()
4205 auto Zero = MIRBuilder.buildConstant(ShiftAmtTy, 0); in narrowScalarShift()
4206 auto IsShort = MIRBuilder.buildICmp(ICmpInst::ICMP_ULT, CondTy, Amt, NewBits); in narrowScalarShift()
4207 auto IsZero = MIRBuilder.buildICmp(ICmpInst::ICMP_EQ, CondTy, Amt, Zero); in narrowScalarShift()
4213 auto LoS = MIRBuilder.buildShl(HalfTy, InL, Amt); in narrowScalarShift()
4215 auto LoOr = MIRBuilder.buildLShr(HalfTy, InL, AmtLack); in narrowScalarShift()
4216 auto HiOr = MIRBuilder.buildShl(HalfTy, InH, Amt); in narrowScalarShift()
4217 auto HiS = MIRBuilder.buildOr(HalfTy, LoOr, HiOr); in narrowScalarShift()
4220 auto LoL = MIRBuilder.buildConstant(HalfTy, 0); // Lo part is zero. in narrowScalarShift()
4221 auto HiL = MIRBuilder.buildShl(HalfTy, InL, AmtExcess); // Hi from Lo part. in narrowScalarShift()
4223 auto Lo = MIRBuilder.buildSelect(HalfTy, IsShort, LoS, LoL); in narrowScalarShift()
4224 auto Hi = MIRBuilder.buildSelect( in narrowScalarShift()
4225 HalfTy, IsZero, InH, MIRBuilder.buildSelect(HalfTy, IsShort, HiS, HiL)); in narrowScalarShift()
4234 auto HiS = MIRBuilder.buildInstr(MI.getOpcode(), {HalfTy}, {InH, Amt}); in narrowScalarShift()
4236 auto LoOr = MIRBuilder.buildLShr(HalfTy, InL, Amt); in narrowScalarShift()
4237 auto HiOr = MIRBuilder.buildShl(HalfTy, InH, AmtLack); in narrowScalarShift()
4238 auto LoS = MIRBuilder.buildOr(HalfTy, LoOr, HiOr); in narrowScalarShift()
4243 HiL = MIRBuilder.buildConstant(HalfTy, 0); // Hi part is zero. in narrowScalarShift()
4245 auto ShiftAmt = MIRBuilder.buildConstant(ShiftAmtTy, NewBitSize - 1); in narrowScalarShift()
4246 HiL = MIRBuilder.buildAShr(HalfTy, InH, ShiftAmt); // Sign of Hi part. in narrowScalarShift()
4248 auto LoL = MIRBuilder.buildInstr(MI.getOpcode(), {HalfTy}, in narrowScalarShift()
4251 auto Lo = MIRBuilder.buildSelect( in narrowScalarShift()
4252 HalfTy, IsZero, InL, MIRBuilder.buildSelect(HalfTy, IsShort, LoS, LoL)); in narrowScalarShift()
4254 auto Hi = MIRBuilder.buildSelect(HalfTy, IsShort, HiS, HiL); in narrowScalarShift()
4264 MIRBuilder.buildMerge(DstReg, ResultRegs); in narrowScalarShift()
4277 MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator()); in moreElementsVectorPhi()
4282 MIRBuilder.setInsertPt(MBB, --MBB.getFirstNonPHI()); in moreElementsVectorPhi()
4365 auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_UNMERGE_VALUES); in moreElementsVector()
4388 MachineIRBuilder &B = MIRBuilder; in multiplyRegisters()
4477 MIRBuilder.buildMerge(DstReg, DstRegs); in narrowScalarMul()
4532 MIRBuilder.buildExtract(SegReg, SrcRegs[i], ExtractOffset); in narrowScalarExtract()
4540 MIRBuilder.buildBuildVector(DstReg, DstRegs); in narrowScalarExtract()
4542 MIRBuilder.buildMerge(DstReg, DstRegs); in narrowScalarExtract()
4544 MIRBuilder.buildCopy(DstReg, DstRegs[0]); in narrowScalarExtract()
4606 MIRBuilder.buildExtract(SegReg, OpReg, ExtractOffset); in narrowScalarInsert()
4610 MIRBuilder.buildInsert(DstReg, SrcRegs[i], SegReg, InsertOffset); in narrowScalarInsert()
4617 MIRBuilder.buildBuildVector(DstReg, DstRegs); in narrowScalarInsert()
4619 MIRBuilder.buildMerge(DstReg, DstRegs); in narrowScalarInsert()
4646 auto Inst = MIRBuilder.buildInstr(MI.getOpcode(), {NarrowTy}, in narrowScalarBasic()
4652 auto Inst = MIRBuilder.buildInstr( in narrowScalarBasic()
4715 auto Select = MIRBuilder.buildSelect(NarrowTy, in narrowScalarSelect()
4721 auto Select = MIRBuilder.buildSelect( in narrowScalarSelect()
4748 MachineIRBuilder &B = MIRBuilder; in narrowScalarCTLZ()
4784 MachineIRBuilder &B = MIRBuilder; in narrowScalarCTTZ()
4817 auto UnmergeSrc = MIRBuilder.buildUnmerge(NarrowTy, MI.getOperand(1)); in narrowScalarCTPOP()
4819 auto LoCTPOP = MIRBuilder.buildCTPOP(DstTy, UnmergeSrc.getReg(0)); in narrowScalarCTPOP()
4820 auto HiCTPOP = MIRBuilder.buildCTPOP(DstTy, UnmergeSrc.getReg(1)); in narrowScalarCTPOP()
4821 MIRBuilder.buildAdd(DstReg, HiCTPOP, LoCTPOP); in narrowScalarCTPOP()
4833 const auto &TII = MIRBuilder.getTII(); in lowerBitCount()
4857 auto CtlzZU = MIRBuilder.buildCTLZ_ZERO_UNDEF(DstTy, SrcReg); in lowerBitCount()
4858 auto ZeroSrc = MIRBuilder.buildConstant(SrcTy, 0); in lowerBitCount()
4859 auto ICmp = MIRBuilder.buildICmp( in lowerBitCount()
4861 auto LenConst = MIRBuilder.buildConstant(DstTy, Len); in lowerBitCount()
4862 MIRBuilder.buildSelect(DstReg, ICmp, LenConst, CtlzZU); in lowerBitCount()
4880 auto MIBShiftAmt = MIRBuilder.buildConstant(SrcTy, 1ULL << i); in lowerBitCount()
4881 auto MIBOp = MIRBuilder.buildOr( in lowerBitCount()
4882 SrcTy, Op, MIRBuilder.buildLShr(SrcTy, Op, MIBShiftAmt)); in lowerBitCount()
4885 auto MIBPop = MIRBuilder.buildCTPOP(DstTy, Op); in lowerBitCount()
4886 MIRBuilder.buildSub(MI.getOperand(0), MIRBuilder.buildConstant(DstTy, Len), in lowerBitCount()
4908 auto CttzZU = MIRBuilder.buildCTTZ_ZERO_UNDEF(DstTy, SrcReg); in lowerBitCount()
4909 auto Zero = MIRBuilder.buildConstant(SrcTy, 0); in lowerBitCount()
4910 auto ICmp = MIRBuilder.buildICmp( in lowerBitCount()
4912 auto LenConst = MIRBuilder.buildConstant(DstTy, Len); in lowerBitCount()
4913 MIRBuilder.buildSelect(DstReg, ICmp, LenConst, CttzZU); in lowerBitCount()
4921 auto MIBCstNeg1 = MIRBuilder.buildConstant(SrcTy, -1); in lowerBitCount()
4922 auto MIBNot = MIRBuilder.buildXor(SrcTy, SrcReg, MIBCstNeg1); in lowerBitCount()
4923 auto MIBTmp = MIRBuilder.buildAnd( in lowerBitCount()
4924 SrcTy, MIBNot, MIRBuilder.buildAdd(SrcTy, SrcReg, MIBCstNeg1)); in lowerBitCount()
4927 auto MIBCstLen = MIRBuilder.buildConstant(SrcTy, Len); in lowerBitCount()
4928 MIRBuilder.buildSub(MI.getOperand(0), MIBCstLen, in lowerBitCount()
4929 MIRBuilder.buildCTLZ(SrcTy, MIBTmp)); in lowerBitCount()
4941 MachineIRBuilder &B = MIRBuilder; in lowerBitCount()
5015 auto Zero32 = MIRBuilder.buildConstant(S32, 0); in lowerU64ToF32BitOps()
5016 auto Zero64 = MIRBuilder.buildConstant(S64, 0); in lowerU64ToF32BitOps()
5018 auto LZ = MIRBuilder.buildCTLZ_ZERO_UNDEF(S32, Src); in lowerU64ToF32BitOps()
5020 auto K = MIRBuilder.buildConstant(S32, 127U + 63U); in lowerU64ToF32BitOps()
5021 auto Sub = MIRBuilder.buildSub(S32, K, LZ); in lowerU64ToF32BitOps()
5023 auto NotZero = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, Src, Zero64); in lowerU64ToF32BitOps()
5024 auto E = MIRBuilder.buildSelect(S32, NotZero, Sub, Zero32); in lowerU64ToF32BitOps()
5026 auto Mask0 = MIRBuilder.buildConstant(S64, (-1ULL) >> 1); in lowerU64ToF32BitOps()
5027 auto ShlLZ = MIRBuilder.buildShl(S64, Src, LZ); in lowerU64ToF32BitOps()
5029 auto U = MIRBuilder.buildAnd(S64, ShlLZ, Mask0); in lowerU64ToF32BitOps()
5031 auto Mask1 = MIRBuilder.buildConstant(S64, 0xffffffffffULL); in lowerU64ToF32BitOps()
5032 auto T = MIRBuilder.buildAnd(S64, U, Mask1); in lowerU64ToF32BitOps()
5034 auto UShl = MIRBuilder.buildLShr(S64, U, MIRBuilder.buildConstant(S64, 40)); in lowerU64ToF32BitOps()
5035 auto ShlE = MIRBuilder.buildShl(S32, E, MIRBuilder.buildConstant(S32, 23)); in lowerU64ToF32BitOps()
5036 auto V = MIRBuilder.buildOr(S32, ShlE, MIRBuilder.buildTrunc(S32, UShl)); in lowerU64ToF32BitOps()
5038 auto C = MIRBuilder.buildConstant(S64, 0x8000000000ULL); in lowerU64ToF32BitOps()
5039 auto RCmp = MIRBuilder.buildICmp(CmpInst::ICMP_UGT, S1, T, C); in lowerU64ToF32BitOps()
5040 auto TCmp = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, S1, T, C); in lowerU64ToF32BitOps()
5041 auto One = MIRBuilder.buildConstant(S32, 1); in lowerU64ToF32BitOps()
5043 auto VTrunc1 = MIRBuilder.buildAnd(S32, V, One); in lowerU64ToF32BitOps()
5044 auto Select0 = MIRBuilder.buildSelect(S32, TCmp, VTrunc1, Zero32); in lowerU64ToF32BitOps()
5045 auto R = MIRBuilder.buildSelect(S32, RCmp, One, Select0); in lowerU64ToF32BitOps()
5046 MIRBuilder.buildAdd(Dst, V, R); in lowerU64ToF32BitOps()
5059 auto True = MIRBuilder.buildFConstant(DstTy, 1.0); in lowerUITOFP()
5060 auto False = MIRBuilder.buildFConstant(DstTy, 0.0); in lowerUITOFP()
5061 MIRBuilder.buildSelect(Dst, Src, True, False); in lowerUITOFP()
5091 auto True = MIRBuilder.buildFConstant(DstTy, -1.0); in lowerSITOFP()
5092 auto False = MIRBuilder.buildFConstant(DstTy, 0.0); in lowerSITOFP()
5093 MIRBuilder.buildSelect(Dst, Src, True, False); in lowerSITOFP()
5108 auto SignBit = MIRBuilder.buildConstant(S64, 63); in lowerSITOFP()
5109 auto S = MIRBuilder.buildAShr(S64, L, SignBit); in lowerSITOFP()
5111 auto LPlusS = MIRBuilder.buildAdd(S64, L, S); in lowerSITOFP()
5112 auto Xor = MIRBuilder.buildXor(S64, LPlusS, S); in lowerSITOFP()
5113 auto R = MIRBuilder.buildUITOFP(S32, Xor); in lowerSITOFP()
5115 auto RNeg = MIRBuilder.buildFNeg(S32, R); in lowerSITOFP()
5116 auto SignNotZero = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, S, in lowerSITOFP()
5117 MIRBuilder.buildConstant(S64, 0)); in lowerSITOFP()
5118 MIRBuilder.buildSelect(Dst, SignNotZero, RNeg, R); in lowerSITOFP()
5149 MachineInstrBuilder FPTOSI = MIRBuilder.buildFPTOSI(DstTy, Src); in lowerFPTOUI()
5151 MachineInstrBuilder Threshold = MIRBuilder.buildFConstant(SrcTy, TwoPExpFP); in lowerFPTOUI()
5154 MachineInstrBuilder FSub = MIRBuilder.buildFSub(SrcTy, Src, Threshold); in lowerFPTOUI()
5155 MachineInstrBuilder ResLowBits = MIRBuilder.buildFPTOSI(DstTy, FSub); in lowerFPTOUI()
5156 MachineInstrBuilder ResHighBit = MIRBuilder.buildConstant(DstTy, TwoPExpInt); in lowerFPTOUI()
5157 MachineInstrBuilder Res = MIRBuilder.buildXor(DstTy, ResLowBits, ResHighBit); in lowerFPTOUI()
5162 MIRBuilder.buildFCmp(CmpInst::FCMP_ULT, S1, Src, Threshold); in lowerFPTOUI()
5163 MIRBuilder.buildSelect(Dst, FCMP, FPTOSI, Res); in lowerFPTOUI()
5187 auto ExponentMask = MIRBuilder.buildConstant(SrcTy, 0x7F800000); in lowerFPTOSI()
5188 auto ExponentLoBit = MIRBuilder.buildConstant(SrcTy, 23); in lowerFPTOSI()
5190 auto AndExpMask = MIRBuilder.buildAnd(SrcTy, Src, ExponentMask); in lowerFPTOSI()
5191 auto ExponentBits = MIRBuilder.buildLShr(SrcTy, AndExpMask, ExponentLoBit); in lowerFPTOSI()
5193 auto SignMask = MIRBuilder.buildConstant(SrcTy, in lowerFPTOSI()
5195 auto AndSignMask = MIRBuilder.buildAnd(SrcTy, Src, SignMask); in lowerFPTOSI()
5196 auto SignLowBit = MIRBuilder.buildConstant(SrcTy, SrcEltBits - 1); in lowerFPTOSI()
5197 auto Sign = MIRBuilder.buildAShr(SrcTy, AndSignMask, SignLowBit); in lowerFPTOSI()
5198 Sign = MIRBuilder.buildSExt(DstTy, Sign); in lowerFPTOSI()
5200 auto MantissaMask = MIRBuilder.buildConstant(SrcTy, 0x007FFFFF); in lowerFPTOSI()
5201 auto AndMantissaMask = MIRBuilder.buildAnd(SrcTy, Src, MantissaMask); in lowerFPTOSI()
5202 auto K = MIRBuilder.buildConstant(SrcTy, 0x00800000); in lowerFPTOSI()
5204 auto R = MIRBuilder.buildOr(SrcTy, AndMantissaMask, K); in lowerFPTOSI()
5205 R = MIRBuilder.buildZExt(DstTy, R); in lowerFPTOSI()
5207 auto Bias = MIRBuilder.buildConstant(SrcTy, 127); in lowerFPTOSI()
5208 auto Exponent = MIRBuilder.buildSub(SrcTy, ExponentBits, Bias); in lowerFPTOSI()
5209 auto SubExponent = MIRBuilder.buildSub(SrcTy, Exponent, ExponentLoBit); in lowerFPTOSI()
5210 auto ExponentSub = MIRBuilder.buildSub(SrcTy, ExponentLoBit, Exponent); in lowerFPTOSI()
5212 auto Shl = MIRBuilder.buildShl(DstTy, R, SubExponent); in lowerFPTOSI()
5213 auto Srl = MIRBuilder.buildLShr(DstTy, R, ExponentSub); in lowerFPTOSI()
5216 auto CmpGt = MIRBuilder.buildICmp(CmpInst::ICMP_SGT, in lowerFPTOSI()
5219 R = MIRBuilder.buildSelect(DstTy, CmpGt, Shl, Srl); in lowerFPTOSI()
5221 auto XorSign = MIRBuilder.buildXor(DstTy, R, Sign); in lowerFPTOSI()
5222 auto Ret = MIRBuilder.buildSub(DstTy, XorSign, Sign); in lowerFPTOSI()
5224 auto ZeroSrcTy = MIRBuilder.buildConstant(SrcTy, 0); in lowerFPTOSI()
5226 auto ExponentLt0 = MIRBuilder.buildICmp(CmpInst::ICMP_SLT, in lowerFPTOSI()
5229 auto ZeroDstTy = MIRBuilder.buildConstant(DstTy, 0); in lowerFPTOSI()
5230 MIRBuilder.buildSelect(Dst, ExponentLt0, ZeroDstTy, Ret); in lowerFPTOSI()
5251 auto Unmerge = MIRBuilder.buildUnmerge(S32, Src); in lowerFPTRUNC_F64_TO_F16()
5255 auto E = MIRBuilder.buildLShr(S32, UH, MIRBuilder.buildConstant(S32, 20)); in lowerFPTRUNC_F64_TO_F16()
5256 E = MIRBuilder.buildAnd(S32, E, MIRBuilder.buildConstant(S32, ExpMask)); in lowerFPTRUNC_F64_TO_F16()
5260 E = MIRBuilder.buildAdd( in lowerFPTRUNC_F64_TO_F16()
5261 S32, E, MIRBuilder.buildConstant(S32, -ExpBiasf64 + ExpBiasf16)); in lowerFPTRUNC_F64_TO_F16()
5263 auto M = MIRBuilder.buildLShr(S32, UH, MIRBuilder.buildConstant(S32, 8)); in lowerFPTRUNC_F64_TO_F16()
5264 M = MIRBuilder.buildAnd(S32, M, MIRBuilder.buildConstant(S32, 0xffe)); in lowerFPTRUNC_F64_TO_F16()
5266 auto MaskedSig = MIRBuilder.buildAnd(S32, UH, in lowerFPTRUNC_F64_TO_F16()
5267 MIRBuilder.buildConstant(S32, 0x1ff)); in lowerFPTRUNC_F64_TO_F16()
5268 MaskedSig = MIRBuilder.buildOr(S32, MaskedSig, U); in lowerFPTRUNC_F64_TO_F16()
5270 auto Zero = MIRBuilder.buildConstant(S32, 0); in lowerFPTRUNC_F64_TO_F16()
5271 auto SigCmpNE0 = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, MaskedSig, Zero); in lowerFPTRUNC_F64_TO_F16()
5272 auto Lo40Set = MIRBuilder.buildZExt(S32, SigCmpNE0); in lowerFPTRUNC_F64_TO_F16()
5273 M = MIRBuilder.buildOr(S32, M, Lo40Set); in lowerFPTRUNC_F64_TO_F16()
5276 auto Bits0x200 = MIRBuilder.buildConstant(S32, 0x0200); in lowerFPTRUNC_F64_TO_F16()
5277 auto CmpM_NE0 = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, M, Zero); in lowerFPTRUNC_F64_TO_F16()
5278 auto SelectCC = MIRBuilder.buildSelect(S32, CmpM_NE0, Bits0x200, Zero); in lowerFPTRUNC_F64_TO_F16()
5280 auto Bits0x7c00 = MIRBuilder.buildConstant(S32, 0x7c00); in lowerFPTRUNC_F64_TO_F16()
5281 auto I = MIRBuilder.buildOr(S32, SelectCC, Bits0x7c00); in lowerFPTRUNC_F64_TO_F16()
5284 auto EShl12 = MIRBuilder.buildShl(S32, E, MIRBuilder.buildConstant(S32, 12)); in lowerFPTRUNC_F64_TO_F16()
5285 auto N = MIRBuilder.buildOr(S32, M, EShl12); in lowerFPTRUNC_F64_TO_F16()
5288 auto One = MIRBuilder.buildConstant(S32, 1); in lowerFPTRUNC_F64_TO_F16()
5289 auto OneSubExp = MIRBuilder.buildSub(S32, One, E); in lowerFPTRUNC_F64_TO_F16()
5290 auto B = MIRBuilder.buildSMax(S32, OneSubExp, Zero); in lowerFPTRUNC_F64_TO_F16()
5291 B = MIRBuilder.buildSMin(S32, B, MIRBuilder.buildConstant(S32, 13)); in lowerFPTRUNC_F64_TO_F16()
5293 auto SigSetHigh = MIRBuilder.buildOr(S32, M, in lowerFPTRUNC_F64_TO_F16()
5294 MIRBuilder.buildConstant(S32, 0x1000)); in lowerFPTRUNC_F64_TO_F16()
5296 auto D = MIRBuilder.buildLShr(S32, SigSetHigh, B); in lowerFPTRUNC_F64_TO_F16()
5297 auto D0 = MIRBuilder.buildShl(S32, D, B); in lowerFPTRUNC_F64_TO_F16()
5299 auto D0_NE_SigSetHigh = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, in lowerFPTRUNC_F64_TO_F16()
5301 auto D1 = MIRBuilder.buildZExt(S32, D0_NE_SigSetHigh); in lowerFPTRUNC_F64_TO_F16()
5302 D = MIRBuilder.buildOr(S32, D, D1); in lowerFPTRUNC_F64_TO_F16()
5304 auto CmpELtOne = MIRBuilder.buildICmp(CmpInst::ICMP_SLT, S1, E, One); in lowerFPTRUNC_F64_TO_F16()
5305 auto V = MIRBuilder.buildSelect(S32, CmpELtOne, D, N); in lowerFPTRUNC_F64_TO_F16()
5307 auto VLow3 = MIRBuilder.buildAnd(S32, V, MIRBuilder.buildConstant(S32, 7)); in lowerFPTRUNC_F64_TO_F16()
5308 V = MIRBuilder.buildLShr(S32, V, MIRBuilder.buildConstant(S32, 2)); in lowerFPTRUNC_F64_TO_F16()
5310 auto VLow3Eq3 = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, S1, VLow3, in lowerFPTRUNC_F64_TO_F16()
5311 MIRBuilder.buildConstant(S32, 3)); in lowerFPTRUNC_F64_TO_F16()
5312 auto V0 = MIRBuilder.buildZExt(S32, VLow3Eq3); in lowerFPTRUNC_F64_TO_F16()
5314 auto VLow3Gt5 = MIRBuilder.buildICmp(CmpInst::ICMP_SGT, S1, VLow3, in lowerFPTRUNC_F64_TO_F16()
5315 MIRBuilder.buildConstant(S32, 5)); in lowerFPTRUNC_F64_TO_F16()
5316 auto V1 = MIRBuilder.buildZExt(S32, VLow3Gt5); in lowerFPTRUNC_F64_TO_F16()
5318 V1 = MIRBuilder.buildOr(S32, V0, V1); in lowerFPTRUNC_F64_TO_F16()
5319 V = MIRBuilder.buildAdd(S32, V, V1); in lowerFPTRUNC_F64_TO_F16()
5321 auto CmpEGt30 = MIRBuilder.buildICmp(CmpInst::ICMP_SGT, S1, in lowerFPTRUNC_F64_TO_F16()
5322 E, MIRBuilder.buildConstant(S32, 30)); in lowerFPTRUNC_F64_TO_F16()
5323 V = MIRBuilder.buildSelect(S32, CmpEGt30, in lowerFPTRUNC_F64_TO_F16()
5324 MIRBuilder.buildConstant(S32, 0x7c00), V); in lowerFPTRUNC_F64_TO_F16()
5326 auto CmpEGt1039 = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, S1, in lowerFPTRUNC_F64_TO_F16()
5327 E, MIRBuilder.buildConstant(S32, 1039)); in lowerFPTRUNC_F64_TO_F16()
5328 V = MIRBuilder.buildSelect(S32, CmpEGt1039, I, V); in lowerFPTRUNC_F64_TO_F16()
5331 auto Sign = MIRBuilder.buildLShr(S32, UH, MIRBuilder.buildConstant(S32, 16)); in lowerFPTRUNC_F64_TO_F16()
5332 Sign = MIRBuilder.buildAnd(S32, Sign, MIRBuilder.buildConstant(S32, 0x8000)); in lowerFPTRUNC_F64_TO_F16()
5335 V = MIRBuilder.buildOr(S32, Sign, V); in lowerFPTRUNC_F64_TO_F16()
5337 MIRBuilder.buildTrunc(Dst, V); in lowerFPTRUNC_F64_TO_F16()
5366 auto CvtSrc1 = MIRBuilder.buildSITOFP(Ty, Src1); in lowerFPOWI()
5367 MIRBuilder.buildFPow(Dst, Src0, CvtSrc1, MI.getFlags()); in lowerFPOWI()
5395 auto Cmp = MIRBuilder.buildICmp(Pred, CmpType, Src0, Src1); in lowerMinMax()
5396 MIRBuilder.buildSelect(Dst, Cmp, Src0, Src1); in lowerMinMax()
5414 auto SignBitMask = MIRBuilder.buildConstant( in lowerFCopySign()
5417 auto NotSignBitMask = MIRBuilder.buildConstant( in lowerFCopySign()
5420 auto And0 = MIRBuilder.buildAnd(Src0Ty, Src0, NotSignBitMask); in lowerFCopySign()
5424 auto And1 = MIRBuilder.buildAnd(Src1Ty, Src1, SignBitMask); in lowerFCopySign()
5425 Or = MIRBuilder.buildOr(Dst, And0, And1); in lowerFCopySign()
5427 auto ShiftAmt = MIRBuilder.buildConstant(Src0Ty, Src0Size - Src1Size); in lowerFCopySign()
5428 auto Zext = MIRBuilder.buildZExt(Src0Ty, Src1); in lowerFCopySign()
5429 auto Shift = MIRBuilder.buildShl(Src0Ty, Zext, ShiftAmt); in lowerFCopySign()
5430 auto And1 = MIRBuilder.buildAnd(Src0Ty, Shift, SignBitMask); in lowerFCopySign()
5431 Or = MIRBuilder.buildOr(Dst, And0, And1); in lowerFCopySign()
5433 auto ShiftAmt = MIRBuilder.buildConstant(Src1Ty, Src1Size - Src0Size); in lowerFCopySign()
5434 auto Shift = MIRBuilder.buildLShr(Src1Ty, Src1, ShiftAmt); in lowerFCopySign()
5435 auto Trunc = MIRBuilder.buildTrunc(Src0Ty, Shift); in lowerFCopySign()
5436 auto And1 = MIRBuilder.buildAnd(Src0Ty, Trunc, SignBitMask); in lowerFCopySign()
5437 Or = MIRBuilder.buildOr(Dst, And0, And1); in lowerFCopySign()
5468 Src0 = MIRBuilder.buildFCanonicalize(Ty, Src0, MI.getFlags()).getReg(0); in lowerFMinNumMaxNum()
5471 Src1 = MIRBuilder.buildFCanonicalize(Ty, Src1, MI.getFlags()).getReg(0); in lowerFMinNumMaxNum()
5476 MIRBuilder.buildInstr(NewOp, {Dst}, {Src0, Src1}, MI.getFlags()); in lowerFMinNumMaxNum()
5487 auto Mul = MIRBuilder.buildFMul(Ty, MI.getOperand(1), MI.getOperand(2), in lowerFMad()
5489 MIRBuilder.buildFAdd(DstReg, Mul, MI.getOperand(3), Flags); in lowerFMad()
5508 auto T = MIRBuilder.buildIntrinsicTrunc(Ty, X, Flags); in lowerIntrinsicRound()
5510 auto Diff = MIRBuilder.buildFSub(Ty, X, T, Flags); in lowerIntrinsicRound()
5511 auto AbsDiff = MIRBuilder.buildFAbs(Ty, Diff, Flags); in lowerIntrinsicRound()
5512 auto Zero = MIRBuilder.buildFConstant(Ty, 0.0); in lowerIntrinsicRound()
5513 auto One = MIRBuilder.buildFConstant(Ty, 1.0); in lowerIntrinsicRound()
5514 auto Half = MIRBuilder.buildFConstant(Ty, 0.5); in lowerIntrinsicRound()
5515 auto SignOne = MIRBuilder.buildFCopysign(Ty, One, X); in lowerIntrinsicRound()
5517 auto Cmp = MIRBuilder.buildFCmp(CmpInst::FCMP_OGE, CondTy, AbsDiff, Half, in lowerIntrinsicRound()
5519 auto Sel = MIRBuilder.buildSelect(Ty, Cmp, SignOne, Zero, Flags); in lowerIntrinsicRound()
5521 MIRBuilder.buildFAdd(DstReg, T, Sel, Flags); in lowerIntrinsicRound()
5539 auto Trunc = MIRBuilder.buildIntrinsicTrunc(Ty, SrcReg, Flags); in lowerFFloor()
5540 auto Zero = MIRBuilder.buildFConstant(Ty, 0.0); in lowerFFloor()
5542 auto Lt0 = MIRBuilder.buildFCmp(CmpInst::FCMP_OLT, CondTy, in lowerFFloor()
5544 auto NeTrunc = MIRBuilder.buildFCmp(CmpInst::FCMP_ONE, CondTy, in lowerFFloor()
5546 auto And = MIRBuilder.buildAnd(CondTy, Lt0, NeTrunc); in lowerFFloor()
5547 auto AddVal = MIRBuilder.buildSITOFP(Ty, And); in lowerFFloor()
5549 MIRBuilder.buildFAdd(DstReg, Trunc, AddVal, Flags); in lowerFFloor()
5564 Register ResultReg = MIRBuilder.buildZExt(WideTy, Src0Reg).getReg(0); in lowerMergeValues()
5570 auto ZextInput = MIRBuilder.buildZExt(WideTy, SrcReg); in lowerMergeValues()
5575 auto ShiftAmt = MIRBuilder.buildConstant(WideTy, Offset); in lowerMergeValues()
5576 auto Shl = MIRBuilder.buildShl(WideTy, ZextInput, ShiftAmt); in lowerMergeValues()
5577 MIRBuilder.buildOr(NextResult, ResultReg, Shl); in lowerMergeValues()
5582 if (MIRBuilder.getDataLayout().isNonIntegralAddressSpace( in lowerMergeValues()
5588 MIRBuilder.buildIntToPtr(DstReg, ResultReg); in lowerMergeValues()
5611 MIRBuilder.buildTrunc(Dst0Reg, SrcReg); in lowerUnmergeValues()
5616 auto ShiftAmt = MIRBuilder.buildConstant(IntTy, Offset); in lowerUnmergeValues()
5617 auto Shift = MIRBuilder.buildLShr(IntTy, SrcReg, ShiftAmt); in lowerUnmergeValues()
5618 MIRBuilder.buildTrunc(MI.getOperand(I), Shift); in lowerUnmergeValues()
5659 MIRBuilder.buildStore(SrcVec, StackTemp, PtrInfo, VecAlign); in lowerExtractInsertVectorElt()
5678 MIRBuilder.buildStore(InsertVal, EltPtr, PtrInfo, EltAlign); in lowerExtractInsertVectorElt()
5681 MIRBuilder.buildLoad(DstReg, StackTemp, PtrInfo, VecAlign); in lowerExtractInsertVectorElt()
5683 MIRBuilder.buildLoad(DstReg, EltPtr, PtrInfo, EltAlign); in lowerExtractInsertVectorElt()
5709 Val = MIRBuilder.buildUndef(DstTy).getReg(0); in lowerShuffleVector()
5712 MIRBuilder.buildCopy(DstReg, Val); in lowerShuffleVector()
5724 Undef = MIRBuilder.buildUndef(EltTy).getReg(0); in lowerShuffleVector()
5735 auto IdxK = MIRBuilder.buildConstant(IdxTy, ExtractIdx); in lowerShuffleVector()
5736 auto Extract = MIRBuilder.buildExtractVectorElement(EltTy, SrcVec, IdxK); in lowerShuffleVector()
5741 MIRBuilder.buildBuildVector(DstReg, BuildVec); in lowerShuffleVector()
5761 auto SPTmp = MIRBuilder.buildCopy(PtrTy, SPReg); in lowerDynStackAlloc()
5762 SPTmp = MIRBuilder.buildCast(IntPtrTy, SPTmp); in lowerDynStackAlloc()
5767 auto Alloc = MIRBuilder.buildSub(IntPtrTy, SPTmp, AllocSize); in lowerDynStackAlloc()
5771 auto AlignCst = MIRBuilder.buildConstant(IntPtrTy, AlignMask); in lowerDynStackAlloc()
5772 Alloc = MIRBuilder.buildAnd(IntPtrTy, Alloc, AlignCst); in lowerDynStackAlloc()
5775 SPTmp = MIRBuilder.buildCast(PtrTy, Alloc); in lowerDynStackAlloc()
5776 MIRBuilder.buildCopy(SPReg, SPTmp); in lowerDynStackAlloc()
5777 MIRBuilder.buildCopy(Dst, SPTmp); in lowerDynStackAlloc()
5798 Src = MIRBuilder.buildBitcast(SrcIntTy, Src).getReg(0); in lowerExtract()
5802 MIRBuilder.buildTrunc(Dst, Src); in lowerExtract()
5804 auto ShiftAmt = MIRBuilder.buildConstant(SrcIntTy, Offset); in lowerExtract()
5805 auto Shr = MIRBuilder.buildLShr(SrcIntTy, Src, ShiftAmt); in lowerExtract()
5806 MIRBuilder.buildTrunc(Dst, Shr); in lowerExtract()
5829 const DataLayout &DL = MIRBuilder.getDataLayout(); in lowerInsert()
5842 Src = MIRBuilder.buildCast(IntDstTy, Src).getReg(0); in lowerInsert()
5847 InsertSrc = MIRBuilder.buildPtrToInt(IntInsertTy, InsertSrc).getReg(0); in lowerInsert()
5850 Register ExtInsSrc = MIRBuilder.buildZExt(IntDstTy, InsertSrc).getReg(0); in lowerInsert()
5852 auto ShiftAmt = MIRBuilder.buildConstant(IntDstTy, Offset); in lowerInsert()
5853 ExtInsSrc = MIRBuilder.buildShl(IntDstTy, ExtInsSrc, ShiftAmt).getReg(0); in lowerInsert()
5859 auto Mask = MIRBuilder.buildConstant(IntDstTy, MaskVal); in lowerInsert()
5860 auto MaskedSrc = MIRBuilder.buildAnd(IntDstTy, Src, Mask); in lowerInsert()
5861 auto Or = MIRBuilder.buildOr(IntDstTy, MaskedSrc, ExtInsSrc); in lowerInsert()
5863 MIRBuilder.buildCast(Dst, Or); in lowerInsert()
5880 MIRBuilder.buildAdd(Dst0, LHS, RHS); in lowerSADDO_SSUBO()
5882 MIRBuilder.buildSub(Dst0, LHS, RHS); in lowerSADDO_SSUBO()
5886 auto Zero = MIRBuilder.buildConstant(Ty, 0); in lowerSADDO_SSUBO()
5895 MIRBuilder.buildICmp(CmpInst::ICMP_SLT, BoolTy, Dst0, LHS); in lowerSADDO_SSUBO()
5896 auto ConditionRHS = MIRBuilder.buildICmp( in lowerSADDO_SSUBO()
5899 MIRBuilder.buildXor(Dst1, ConditionRHS, ResultLowerThanLHS); in lowerSADDO_SSUBO()
5951 MIRBuilder.buildConstant(Ty, APInt::getSignedMaxValue(NumBits)); in lowerAddSubSatToMinMax()
5953 MIRBuilder.buildConstant(Ty, APInt::getSignedMinValue(NumBits)); in lowerAddSubSatToMinMax()
5956 auto Zero = MIRBuilder.buildConstant(Ty, 0); in lowerAddSubSatToMinMax()
5957 Hi = MIRBuilder.buildSub(Ty, MaxVal, MIRBuilder.buildSMax(Ty, LHS, Zero)); in lowerAddSubSatToMinMax()
5958 Lo = MIRBuilder.buildSub(Ty, MinVal, MIRBuilder.buildSMin(Ty, LHS, Zero)); in lowerAddSubSatToMinMax()
5960 auto NegOne = MIRBuilder.buildConstant(Ty, -1); in lowerAddSubSatToMinMax()
5961 Lo = MIRBuilder.buildSub(Ty, MIRBuilder.buildSMax(Ty, LHS, NegOne), in lowerAddSubSatToMinMax()
5963 Hi = MIRBuilder.buildSub(Ty, MIRBuilder.buildSMin(Ty, LHS, NegOne), in lowerAddSubSatToMinMax()
5967 MIRBuilder.buildSMin(Ty, MIRBuilder.buildSMax(Ty, Lo, RHS), Hi); in lowerAddSubSatToMinMax()
5968 MIRBuilder.buildInstr(BaseOp, {Res}, {LHS, RHSClamped}); in lowerAddSubSatToMinMax()
5972 Register Not = IsAdd ? MIRBuilder.buildNot(Ty, LHS).getReg(0) : LHS; in lowerAddSubSatToMinMax()
5973 auto Min = MIRBuilder.buildUMin(Ty, Not, RHS); in lowerAddSubSatToMinMax()
5974 MIRBuilder.buildInstr(BaseOp, {Res}, {LHS, Min}); in lowerAddSubSatToMinMax()
6017 MIRBuilder.buildInstr(OverflowOp, {Ty, BoolTy}, {LHS, RHS}); in lowerAddSubSatToAddoSubo()
6029 auto ShiftAmount = MIRBuilder.buildConstant(Ty, NumBits - 1); in lowerAddSubSatToAddoSubo()
6030 auto Sign = MIRBuilder.buildAShr(Ty, Tmp, ShiftAmount); in lowerAddSubSatToAddoSubo()
6032 MIRBuilder.buildConstant(Ty, APInt::getSignedMinValue(NumBits)); in lowerAddSubSatToAddoSubo()
6033 Clamp = MIRBuilder.buildAdd(Ty, Sign, MinVal); in lowerAddSubSatToAddoSubo()
6041 Clamp = MIRBuilder.buildConstant(Ty, IsAdd ? -1 : 0); in lowerAddSubSatToAddoSubo()
6043 MIRBuilder.buildSelect(Res, Ov, Clamp, Tmp); in lowerAddSubSatToAddoSubo()
6062 auto Result = MIRBuilder.buildShl(Ty, LHS, RHS); in lowerShlSat()
6063 auto Orig = IsSigned ? MIRBuilder.buildAShr(Ty, Result, RHS) in lowerShlSat()
6064 : MIRBuilder.buildLShr(Ty, Result, RHS); in lowerShlSat()
6068 auto SatMin = MIRBuilder.buildConstant(Ty, APInt::getSignedMinValue(BW)); in lowerShlSat()
6069 auto SatMax = MIRBuilder.buildConstant(Ty, APInt::getSignedMaxValue(BW)); in lowerShlSat()
6070 auto Cmp = MIRBuilder.buildICmp(CmpInst::ICMP_SLT, BoolTy, LHS, in lowerShlSat()
6071 MIRBuilder.buildConstant(Ty, 0)); in lowerShlSat()
6072 SatVal = MIRBuilder.buildSelect(Ty, Cmp, SatMin, SatMax); in lowerShlSat()
6074 SatVal = MIRBuilder.buildConstant(Ty, APInt::getMaxValue(BW)); in lowerShlSat()
6076 auto Ov = MIRBuilder.buildICmp(CmpInst::ICMP_NE, BoolTy, LHS, Orig); in lowerShlSat()
6077 MIRBuilder.buildSelect(Res, Ov, SatVal, Result); in lowerShlSat()
6092 auto ShiftAmt = MIRBuilder.buildConstant(Ty, BaseShiftAmt); in lowerBswap()
6093 auto LSByteShiftedLeft = MIRBuilder.buildShl(Ty, Src, ShiftAmt); in lowerBswap()
6094 auto MSByteShiftedRight = MIRBuilder.buildLShr(Ty, Src, ShiftAmt); in lowerBswap()
6095 auto Res = MIRBuilder.buildOr(Ty, MSByteShiftedRight, LSByteShiftedLeft); in lowerBswap()
6101 auto Mask = MIRBuilder.buildConstant(Ty, APMask); in lowerBswap()
6102 auto ShiftAmt = MIRBuilder.buildConstant(Ty, BaseShiftAmt - 16 * i); in lowerBswap()
6104 auto LoByte = MIRBuilder.buildAnd(Ty, Src, Mask); in lowerBswap()
6105 auto LoShiftedLeft = MIRBuilder.buildShl(Ty, LoByte, ShiftAmt); in lowerBswap()
6106 Res = MIRBuilder.buildOr(Ty, Res, LoShiftedLeft); in lowerBswap()
6108 auto SrcShiftedRight = MIRBuilder.buildLShr(Ty, Src, ShiftAmt); in lowerBswap()
6109 auto HiShiftedRight = MIRBuilder.buildAnd(Ty, SrcShiftedRight, Mask); in lowerBswap()
6110 Res = MIRBuilder.buildOr(Ty, Res, HiShiftedRight); in lowerBswap()
6137 MIRBuilder.buildInstr(TargetOpcode::G_BSWAP, {Ty}, {Src}); in lowerBitreverse()
6143 SwapN(4, Ty, MIRBuilder, BSWAP, APInt::getSplat(Size, APInt(8, 0xF0))); in lowerBitreverse()
6149 SwapN(2, Ty, MIRBuilder, Swap4, APInt::getSplat(Size, APInt(8, 0xCC))); in lowerBitreverse()
6154 SwapN(1, Dst, MIRBuilder, Swap2, APInt::getSplat(Size, APInt(8, 0xAA))); in lowerBitreverse()
6162 MachineFunction &MF = MIRBuilder.getMF(); in lowerReadWriteRegister()
6178 MIRBuilder.buildCopy(ValReg, PhysReg); in lowerReadWriteRegister()
6180 MIRBuilder.buildCopy(PhysReg, ValReg); in lowerReadWriteRegister()
6195 auto LHS = MIRBuilder.buildInstr(ExtOp, {WideTy}, {MI.getOperand(1)}); in lowerSMULH_UMULH()
6196 auto RHS = MIRBuilder.buildInstr(ExtOp, {WideTy}, {MI.getOperand(2)}); in lowerSMULH_UMULH()
6197 auto Mul = MIRBuilder.buildMul(WideTy, LHS, RHS); in lowerSMULH_UMULH()
6200 auto ShiftAmt = MIRBuilder.buildConstant(WideTy, SizeInBits); in lowerSMULH_UMULH()
6201 auto Shifted = MIRBuilder.buildInstr(ShiftOp, {WideTy}, {Mul, ShiftAmt}); in lowerSMULH_UMULH()
6202 MIRBuilder.buildTrunc(Result, Shifted); in lowerSMULH_UMULH()
6225 MaskElt = MIRBuilder.buildSExt(DstTy.getElementType(), MaskElt).getReg(0); in lowerSelect()
6227 auto ShufSplat = MIRBuilder.buildShuffleSplat(DstTy, MaskElt); in lowerSelect()
6238 auto NotMask = MIRBuilder.buildNot(MaskTy, MaskReg); in lowerSelect()
6239 auto NewOp1 = MIRBuilder.buildAnd(MaskTy, Op1Reg, MaskReg); in lowerSelect()
6240 auto NewOp2 = MIRBuilder.buildAnd(MaskTy, Op2Reg, NotMask); in lowerSelect()
6241 MIRBuilder.buildOr(DstReg, NewOp1, NewOp2); in lowerSelect()