Lines Matching +full:robust +full:- +full:predicates
1 //===-- X86FastISel.cpp - X86 FastISel implementation ---------------------===//
8 //===----------------------------------------------------------------------===//
10 // This file defines the X86-specific support for the FastISel class. Much
11 // of the target-specific code is generated by tablegen in the file
14 //===----------------------------------------------------------------------===//
49 /// Subtarget - Keep a pointer to the X86Subtarget around so that we can
53 /// X86ScalarSSEf32, X86ScalarSSEf64 - Select between SSE or x87
64 Subtarget = &funcInfo.MF->getSubtarget<X86Subtarget>(); in X86FastISel()
65 X86ScalarSSEf64 = Subtarget->hasSSE2(); in X86FastISel()
66 X86ScalarSSEf32 = Subtarget->hasSSE1(); in X86FastISel()
137 return Subtarget->getInstrInfo(); in getInstrInfo()
154 /// isScalarFPTypeInSSEReg - Return true if the specified scalar FP type is
183 // Floating-point Predicates in getX86ConditionCode()
185 case CmpInst::FCMP_OLT: NeedSwap = true; // fall-through in getX86ConditionCode()
187 case CmpInst::FCMP_OLE: NeedSwap = true; // fall-through in getX86ConditionCode()
189 case CmpInst::FCMP_UGT: NeedSwap = true; // fall-through in getX86ConditionCode()
191 case CmpInst::FCMP_UGE: NeedSwap = true; // fall-through in getX86ConditionCode()
196 case CmpInst::FCMP_OEQ: // fall-through in getX86ConditionCode()
199 // Integer Predicates in getX86ConditionCode()
221 // 0 - EQ in getX86SSEConditionCode()
222 // 1 - LT in getX86SSEConditionCode()
223 // 2 - LE in getX86SSEConditionCode()
224 // 3 - UNORD in getX86SSEConditionCode()
225 // 4 - NEQ in getX86SSEConditionCode()
226 // 5 - NLT in getX86SSEConditionCode()
227 // 6 - NLE in getX86SSEConditionCode()
228 // 7 - ORD in getX86SSEConditionCode()
232 case CmpInst::FCMP_OGT: NeedSwap = true; // fall-through in getX86SSEConditionCode()
234 case CmpInst::FCMP_OGE: NeedSwap = true; // fall-through in getX86SSEConditionCode()
238 case CmpInst::FCMP_ULE: NeedSwap = true; // fall-through in getX86SSEConditionCode()
240 case CmpInst::FCMP_ULT: NeedSwap = true; // fall-through in getX86SSEConditionCode()
258 AM.IndexReg = constrainOperandRegClass(MIB->getDesc(), AM.IndexReg, in addFullAddress()
259 MIB->getNumOperands() + in addFullAddress()
272 if (!isa<IntrinsicInst>(EV->getAggregateOperand())) in foldX86XALUIntrinsic()
275 const auto *II = cast<IntrinsicInst>(EV->getAggregateOperand()); in foldX86XALUIntrinsic()
277 const Function *Callee = II->getCalledFunction(); in foldX86XALUIntrinsic()
279 cast<StructType>(Callee->getReturnType())->getTypeAtIndex(0U); in foldX86XALUIntrinsic()
287 switch (II->getIntrinsicID()) { in foldX86XALUIntrinsic()
298 if (II->getParent() != I->getParent()) in foldX86XALUIntrinsic()
304 for (auto Itr = std::prev(Start); Itr != End; --Itr) { in foldX86XALUIntrinsic()
312 if (EVI->getAggregateOperand() != II) in foldX86XALUIntrinsic()
327 // For now, require SSE/SSE2 for performing floating-point operations, in isTypeLegal()
336 // We only handle legal types. For example, on x86-32 the instruction in isTypeLegal()
337 // selector contains all of the 64-bit instructions from x86-64, in isTypeLegal()
345 /// X86FastEmitLoad - Emit a machine instruction to load a value of type VT.
346 /// The address is either pre-computed, i.e. Ptr, or a GlobalAddress, i.e. GV.
351 bool HasSSE41 = Subtarget->hasSSE41(); in X86FastEmitLoad()
352 bool HasAVX = Subtarget->hasAVX(); in X86FastEmitLoad()
353 bool HasAVX2 = Subtarget->hasAVX2(); in X86FastEmitLoad()
354 bool IsNonTemporal = MMO && MMO->isNonTemporal(); in X86FastEmitLoad()
375 // Must be in x86-64 mode. in X86FastEmitLoad()
458 assert(Subtarget->hasAVX512()); in X86FastEmitLoad()
466 assert(Subtarget->hasAVX512()); in X86FastEmitLoad()
477 assert(Subtarget->hasAVX512()); in X86FastEmitLoad()
478 // Note: There are a lot more choices based on type with AVX-512, but in X86FastEmitLoad()
493 MIB->addMemOperand(*FuncInfo.MF, MMO); in X86FastEmitLoad()
497 /// X86FastEmitStore - Emit a machine instruction to store a value Val of
498 /// type VT. The address is either pre-computed, consisted of a base ptr, Ptr
504 bool HasSSE2 = Subtarget->hasSSE2(); in X86FastEmitStore()
505 bool HasSSE4A = Subtarget->hasSSE4A(); in X86FastEmitStore()
506 bool HasAVX = Subtarget->hasAVX(); in X86FastEmitStore()
507 bool IsNonTemporal = MMO && MMO->isNonTemporal(); in X86FastEmitStore()
529 // Must be in x86-64 mode. in X86FastEmitStore()
605 assert(Subtarget->hasAVX512()); in X86FastEmitStore()
612 assert(Subtarget->hasAVX512()); in X86FastEmitStore()
622 assert(Subtarget->hasAVX512()); in X86FastEmitStore()
623 // Note: There are a lot more choices based on type with AVX-512, but in X86FastEmitStore()
639 ValReg = constrainOperandRegClass(Desc, ValReg, Desc.getNumOperands() - 1); in X86FastEmitStore()
644 MIB->addMemOperand(*FuncInfo.MF, MMO); in X86FastEmitStore()
654 Val = Constant::getNullValue(DL.getIntPtrType(Val->getContext())); in X86FastEmitStore()
667 // Must be a 32-bit sign extended value. in X86FastEmitStore()
668 if (isInt<32>(CI->getSExtValue())) in X86FastEmitStore()
676 addFullAddress(MIB, AM).addImm(Signed ? (uint64_t) CI->getSExtValue() in X86FastEmitStore()
677 : CI->getZExtValue()); in X86FastEmitStore()
679 MIB->addMemOperand(*FuncInfo.MF, MMO); in X86FastEmitStore()
692 /// X86FastEmitExtend - Emit a machine instruction to extend a value Src of
715 if (GV->isThreadLocal()) in handleConstantAddresses()
718 // RIP-relative addresses can't have additional register operands, so if in handleConstantAddresses()
721 if (!Subtarget->isPICStyleRIPRel() || in handleConstantAddresses()
727 unsigned char GVFlags = Subtarget->classifyGlobalReference(GV); in handleConstantAddresses()
732 AM.Base.Reg = getInstrInfo()->getGlobalBaseReg(FuncInfo.MF); in handleConstantAddresses()
738 if (Subtarget->isPICStyleRIPRel()) { in handleConstantAddresses()
739 // Use rip-relative addressing if we can. Above we verified that the in handleConstantAddresses()
752 if (I != LocalValueMap.end() && I->second != 0) { in handleConstantAddresses()
753 LoadReg = I->second; in handleConstantAddresses()
763 // Prepare for inserting code in the local-value area. in handleConstantAddresses()
770 if (Subtarget->isPICStyleRIPRel()) in handleConstantAddresses()
798 if (!AM.GV || !Subtarget->isPICStyleRIPRel()) { in handleConstantAddresses()
813 /// X86SelectAddress - Attempt to fill in an address from the given value.
825 FuncInfo.MBBMap[I->getParent()] == FuncInfo.MBB) { in X86SelectAddress()
826 Opcode = I->getOpcode(); in X86SelectAddress()
830 Opcode = C->getOpcode(); in X86SelectAddress()
834 if (PointerType *Ty = dyn_cast<PointerType>(V->getType())) in X86SelectAddress()
835 if (Ty->getAddressSpace() > 255) in X86SelectAddress()
844 return X86SelectAddress(U->getOperand(0), AM); in X86SelectAddress()
847 // Look past no-op inttoptrs. in X86SelectAddress()
848 if (TLI.getValueType(DL, U->getOperand(0)->getType()) == in X86SelectAddress()
850 return X86SelectAddress(U->getOperand(0), AM); in X86SelectAddress()
854 // Look past no-op ptrtoints. in X86SelectAddress()
855 if (TLI.getValueType(DL, U->getType()) == TLI.getPointerTy(DL)) in X86SelectAddress()
856 return X86SelectAddress(U->getOperand(0), AM); in X86SelectAddress()
866 AM.Base.FrameIndex = SI->second; in X86SelectAddress()
874 if (const ConstantInt *CI = dyn_cast<ConstantInt>(U->getOperand(1))) { in X86SelectAddress()
875 uint64_t Disp = (int32_t)AM.Disp + (uint64_t)CI->getSExtValue(); in X86SelectAddress()
876 // They have to fit in the 32-bit signed displacement field though. in X86SelectAddress()
879 return X86SelectAddress(U->getOperand(0), AM); in X86SelectAddress()
888 // Pattern-match simple GEPs. in X86SelectAddress()
895 for (User::const_op_iterator i = U->op_begin() + 1, e = U->op_end(); in X86SelectAddress()
900 Disp += SL->getElementOffset(cast<ConstantInt>(Op)->getZExtValue()); in X86SelectAddress()
909 // Constant-offset addressing. in X86SelectAddress()
910 Disp += CI->getSExtValue() * S; in X86SelectAddress()
916 cast<ConstantInt>(cast<AddOperator>(Op)->getOperand(1)); in X86SelectAddress()
917 Disp += CI->getSExtValue() * S; in X86SelectAddress()
919 Op = cast<AddOperator>(Op)->getOperand(0); in X86SelectAddress()
923 (!AM.GV || !Subtarget->isPICStyleRIPRel()) && in X86SelectAddress()
925 // Scaled-index addressing. in X86SelectAddress()
947 dyn_cast<GetElementPtrInst>(U->getOperand(0))) { in X86SelectAddress()
948 // Ok, the GEP indices were covered by constant-offset and scaled-index in X86SelectAddress()
952 } else if (X86SelectAddress(U->getOperand(0), AM)) { in X86SelectAddress()
974 /// X86SelectCallAddress - Attempt to fill in an address from the given value.
1005 Opcode = I->getOpcode(); in X86SelectCallAddress()
1007 InMBB = I->getParent() == FuncInfo.MBB->getBasicBlock(); in X86SelectCallAddress()
1009 Opcode = C->getOpcode(); in X86SelectCallAddress()
1018 return X86SelectCallAddress(U->getOperand(0), AM); in X86SelectCallAddress()
1022 // Look past no-op inttoptrs if its operand is in the same BB. in X86SelectCallAddress()
1024 TLI.getValueType(DL, U->getOperand(0)->getType()) == in X86SelectCallAddress()
1026 return X86SelectCallAddress(U->getOperand(0), AM); in X86SelectCallAddress()
1030 // Look past no-op ptrtoints if its operand is in the same BB. in X86SelectCallAddress()
1031 if (InMBB && TLI.getValueType(DL, U->getType()) == TLI.getPointerTy(DL)) in X86SelectCallAddress()
1032 return X86SelectCallAddress(U->getOperand(0), AM); in X86SelectCallAddress()
1042 // RIP-relative addresses can't have additional register operands. in X86SelectCallAddress()
1043 if (Subtarget->isPICStyleRIPRel() && in X86SelectCallAddress()
1048 if (GV->hasDLLImportStorageClass()) in X86SelectCallAddress()
1053 if (GVar->isThreadLocal()) in X86SelectCallAddress()
1061 if (Subtarget->isPICStyleRIPRel()) { in X86SelectCallAddress()
1062 // Use rip-relative addressing if we can. Above we verified that the in X86SelectCallAddress()
1067 AM.GVOpFlags = Subtarget->classifyLocalReference(nullptr); in X86SelectCallAddress()
1074 if (!AM.GV || !Subtarget->isPICStyleRIPRel()) { in X86SelectCallAddress()
1090 /// X86SelectStore - Select and emit code to implement store instructions.
1095 if (S->isAtomic()) in X86SelectStore()
1098 const Value *PtrV = I->getOperand(1); in X86SelectStore()
1103 if (Arg->hasSwiftErrorAttr()) in X86SelectStore()
1108 if (Alloca->isSwiftError()) in X86SelectStore()
1113 const Value *Val = S->getValueOperand(); in X86SelectStore()
1114 const Value *Ptr = S->getPointerOperand(); in X86SelectStore()
1117 if (!isTypeLegal(Val->getType(), VT, /*AllowI1=*/true)) in X86SelectStore()
1120 unsigned Alignment = S->getAlignment(); in X86SelectStore()
1121 unsigned ABIAlignment = DL.getABITypeAlignment(Val->getType()); in X86SelectStore()
1133 /// X86SelectRet - Select and emit code to implement ret instructions.
1136 const Function &F = *I->getParent()->getParent(); in X86SelectRet()
1138 FuncInfo.MF->getInfo<X86MachineFunctionInfo>(); in X86SelectRet()
1159 if (Subtarget->isCallingConvWin64(CC)) in X86SelectRet()
1163 if (!isUInt<16>(X86MFInfo->getBytesToPopOnReturn())) in X86SelectRet()
1166 // fastcc with -tailcallopt is intended to provide a guaranteed in X86SelectRet()
1178 if (Ret->getNumOperands() > 0) { in X86SelectRet()
1184 CCState CCInfo(CC, F.isVarArg(), *FuncInfo.MF, ValLocs, I->getContext()); in X86SelectRet()
1187 const Value *RV = Ret->getOperand(0); in X86SelectRet()
1205 // The calling-convention tables for x87 returns don't tell in X86SelectRet()
1211 EVT SrcVT = TLI.getValueType(DL, RV->getType()); in X86SelectRet()
1238 // Avoid a cross-class copy. This is very unlikely. in X86SelectRet()
1239 if (!SrcRC->contains(DstReg)) in X86SelectRet()
1256 unsigned Reg = X86MFInfo->getSRetReturnReg(); in X86SelectRet()
1259 unsigned RetReg = Subtarget->is64Bit() ? X86::RAX : X86::EAX; in X86SelectRet()
1267 if (X86MFInfo->getBytesToPopOnReturn()) { in X86SelectRet()
1269 TII.get(Subtarget->is64Bit() ? X86::RETIQ : X86::RETIL)) in X86SelectRet()
1270 .addImm(X86MFInfo->getBytesToPopOnReturn()); in X86SelectRet()
1273 TII.get(Subtarget->is64Bit() ? X86::RETQ : X86::RETL)); in X86SelectRet()
1280 /// X86SelectLoad - Select and emit code to implement load instructions.
1286 if (LI->isAtomic()) in X86SelectLoad()
1289 const Value *SV = I->getOperand(0); in X86SelectLoad()
1294 if (Arg->hasSwiftErrorAttr()) in X86SelectLoad()
1299 if (Alloca->isSwiftError()) in X86SelectLoad()
1305 if (!isTypeLegal(LI->getType(), VT, /*AllowI1=*/true)) in X86SelectLoad()
1308 const Value *Ptr = LI->getPointerOperand(); in X86SelectLoad()
1314 unsigned Alignment = LI->getAlignment(); in X86SelectLoad()
1315 unsigned ABIAlignment = DL.getABITypeAlignment(LI->getType()); in X86SelectLoad()
1329 bool HasAVX = Subtarget->hasAVX(); in X86ChooseCmpOpcode()
1330 bool X86ScalarSSEf32 = Subtarget->hasSSE1(); in X86ChooseCmpOpcode()
1331 bool X86ScalarSSEf64 = Subtarget->hasSSE2(); in X86ChooseCmpOpcode()
1349 int64_t Val = RHSC->getSExtValue(); in X86ChooseCmpImmediateOpcode()
1367 // 64-bit comparisons are only valid if the immediate fits in a 32-bit sext in X86ChooseCmpImmediateOpcode()
1382 Op1 = Constant::getNullValue(DL.getIntPtrType(Op0->getContext())); in X86FastEmitCompare()
1391 .addImm(Op1C->getSExtValue()); in X86FastEmitCompare()
1412 if (!isTypeLegal(I->getOperand(0)->getType(), VT)) in X86SelectCmp()
1415 if (I->getType()->isIntegerTy(1) && Subtarget->hasAVX512()) in X86SelectCmp()
1446 const Value *LHS = CI->getOperand(0); in X86SelectCmp()
1447 const Value *RHS = CI->getOperand(1); in X86SelectCmp()
1454 if (RHSC && RHSC->isNullValue()) in X86SelectCmp()
1472 if (!X86FastEmitCompare(LHS, RHS, VT, I->getDebugLoc())) in X86SelectCmp()
1497 if (!X86FastEmitCompare(LHS, RHS, VT, I->getDebugLoc())) in X86SelectCmp()
1506 EVT DstVT = TLI.getValueType(DL, I->getType()); in X86SelectZExt()
1510 unsigned ResultReg = getRegForValue(I->getOperand(0)); in X86SelectZExt()
1514 // Handle zero-extension from i1 to i8, which is common. in X86SelectZExt()
1515 MVT SrcVT = TLI.getSimpleValueType(DL, I->getOperand(0)->getType()); in X86SelectZExt()
1526 // Handle extension to 64-bits via sub-register shenanigans. in X86SelectZExt()
1556 // Unconditional branches are selected by tablegen-generated code. in X86SelectBranch()
1559 MachineBasicBlock *TrueMBB = FuncInfo.MBBMap[BI->getSuccessor(0)]; in X86SelectBranch()
1560 MachineBasicBlock *FalseMBB = FuncInfo.MBBMap[BI->getSuccessor(1)]; in X86SelectBranch()
1566 if (const CmpInst *CI = dyn_cast<CmpInst>(BI->getCondition())) { in X86SelectBranch()
1567 if (CI->hasOneUse() && CI->getParent() == I->getParent()) { in X86SelectBranch()
1568 EVT VT = TLI.getValueType(DL, CI->getOperand(0)->getType()); in X86SelectBranch()
1578 const Value *CmpLHS = CI->getOperand(0); in X86SelectBranch()
1579 const Value *CmpRHS = CI->getOperand(1); in X86SelectBranch()
1587 if (CmpRHSC && CmpRHSC->isNullValue()) in X86SelectBranch()
1592 if (FuncInfo.MBB->isLayoutSuccessor(TrueMBB)) { in X86SelectBranch()
1606 std::swap(TrueMBB, FalseMBB); // fall-through in X86SelectBranch()
1623 if (!X86FastEmitCompare(CmpLHS, CmpRHS, VT, CI->getDebugLoc())) in X86SelectBranch()
1636 finishCondBranch(BI->getParent(), TrueMBB, FalseMBB); in X86SelectBranch()
1639 } else if (TruncInst *TI = dyn_cast<TruncInst>(BI->getCondition())) { in X86SelectBranch()
1643 if (TI->hasOneUse() && TI->getParent() == I->getParent() && in X86SelectBranch()
1644 isTypeLegal(TI->getOperand(0)->getType(), SourceVT)) { in X86SelectBranch()
1654 unsigned OpReg = getRegForValue(TI->getOperand(0)); in X86SelectBranch()
1660 if (FuncInfo.MBB->isLayoutSuccessor(TrueMBB)) { in X86SelectBranch()
1668 finishCondBranch(BI->getParent(), TrueMBB, FalseMBB); in X86SelectBranch()
1672 } else if (foldX86XALUIntrinsic(CC, BI, BI->getCondition())) { in X86SelectBranch()
1675 unsigned TmpReg = getRegForValue(BI->getCondition()); in X86SelectBranch()
1683 finishCondBranch(BI->getParent(), TrueMBB, FalseMBB); in X86SelectBranch()
1687 // Otherwise do a clumsy setcc and re-test it. in X86SelectBranch()
1690 unsigned OpReg = getRegForValue(BI->getCondition()); in X86SelectBranch()
1697 finishCondBranch(BI->getParent(), TrueMBB, FalseMBB); in X86SelectBranch()
1704 if (I->getType()->isIntegerTy(8)) { in X86SelectShift()
1707 switch (I->getOpcode()) { in X86SelectShift()
1713 } else if (I->getType()->isIntegerTy(16)) { in X86SelectShift()
1716 switch (I->getOpcode()) { in X86SelectShift()
1722 } else if (I->getType()->isIntegerTy(32)) { in X86SelectShift()
1725 switch (I->getOpcode()) { in X86SelectShift()
1731 } else if (I->getType()->isIntegerTy(64)) { in X86SelectShift()
1734 switch (I->getOpcode()) { in X86SelectShift()
1745 if (!isTypeLegal(I->getType(), VT)) in X86SelectShift()
1748 unsigned Op0Reg = getRegForValue(I->getOperand(0)); in X86SelectShift()
1751 unsigned Op1Reg = getRegForValue(I->getOperand(1)); in X86SelectShift()
1756 // The shift instruction uses X86::CL. If we defined a super-register in X86SelectShift()
1780 // copied into lowreg, and lowreg is sign-extended or zero-extended in X86SelectDivRem()
1783 // therefore directly sign-extend or zero-extend the dividend into in X86SelectDivRem()
1793 unsigned OpSignExtend; // Opcode for sign-extending lowreg into in X86SelectDivRem()
1796 // zero/sign-extending into lowreg for i8. in X86SelectDivRem()
1832 if (!isTypeLegal(I->getType(), VT)) in X86SelectDivRem()
1842 if (!Subtarget->is64Bit()) in X86SelectDivRem()
1847 switch (I->getOpcode()) { in X86SelectDivRem()
1857 unsigned Op0Reg = getRegForValue(I->getOperand(0)); in X86SelectDivRem()
1860 unsigned Op1Reg = getRegForValue(I->getOperand(1)); in X86SelectDivRem()
1864 // Move op0 into low-order input register. in X86SelectDivRem()
1867 // Zero-extend or sign-extend into high-order input register. in X86SelectDivRem()
1904 // the allocator and/or the backend get enhanced to be more robust in in X86SelectDivRem()
1907 if ((I->getOpcode() == Instruction::SRem || in X86SelectDivRem()
1908 I->getOpcode() == Instruction::URem) && in X86SelectDivRem()
1909 OpEntry.DivRemResultReg == X86::AH && Subtarget->is64Bit()) { in X86SelectDivRem()
1919 // Now reference the 8-bit subreg of the result. in X86SelectDivRem()
1938 if (!Subtarget->hasCMov()) in X86FastEmitCMoveSelect()
1945 const Value *Cond = I->getOperand(0); in X86FastEmitCMoveSelect()
1954 if (CI && (CI->getParent() == I->getParent())) { in X86FastEmitCMoveSelect()
1979 const Value *CmpLHS = CI->getOperand(0); in X86FastEmitCMoveSelect()
1980 const Value *CmpRHS = CI->getOperand(1); in X86FastEmitCMoveSelect()
1984 EVT CmpVT = TLI.getValueType(DL, CmpLHS->getType()); in X86FastEmitCMoveSelect()
1986 if (!X86FastEmitCompare(CmpLHS, CmpRHS, CmpVT, CI->getDebugLoc())) in X86FastEmitCMoveSelect()
2020 // accurate. If we read more than the lsb, we may see non-zero values in X86FastEmitCMoveSelect()
2032 const Value *LHS = I->getOperand(1); in X86FastEmitCMoveSelect()
2033 const Value *RHS = I->getOperand(2); in X86FastEmitCMoveSelect()
2044 unsigned Opc = X86::getCMovFromCond(CC, RC->getSize()); in X86FastEmitCMoveSelect()
2060 const auto *CI = dyn_cast<FCmpInst>(I->getOperand(0)); in X86FastEmitSSESelect()
2061 if (!CI || (CI->getParent() != I->getParent())) in X86FastEmitSSESelect()
2064 if (I->getType() != CI->getOperand(0)->getType() || in X86FastEmitSSESelect()
2065 !((Subtarget->hasSSE1() && RetVT == MVT::f32) || in X86FastEmitSSESelect()
2066 (Subtarget->hasSSE2() && RetVT == MVT::f64))) in X86FastEmitSSESelect()
2069 const Value *CmpLHS = CI->getOperand(0); in X86FastEmitSSESelect()
2070 const Value *CmpRHS = CI->getOperand(1); in X86FastEmitSSESelect()
2078 if (CmpRHSC && CmpRHSC->isNullValue()) in X86FastEmitSSESelect()
2104 const Value *LHS = I->getOperand(1); in X86FastEmitSSESelect()
2105 const Value *RHS = I->getOperand(2); in X86FastEmitSSESelect()
2125 if (Subtarget->hasAVX()) { in X86FastEmitSSESelect()
2161 // These are pseudo CMOV instructions and will be later expanded into control- in X86FastEmitPseudoSelect()
2173 const Value *Cond = I->getOperand(0); in X86FastEmitPseudoSelect()
2180 if (CI && (CI->getParent() == I->getParent())) { in X86FastEmitPseudoSelect()
2182 std::tie(CC, NeedSwap) = getX86ConditionCode(CI->getPredicate()); in X86FastEmitPseudoSelect()
2186 const Value *CmpLHS = CI->getOperand(0); in X86FastEmitPseudoSelect()
2187 const Value *CmpRHS = CI->getOperand(1); in X86FastEmitPseudoSelect()
2192 EVT CmpVT = TLI.getValueType(DL, CmpLHS->getType()); in X86FastEmitPseudoSelect()
2193 if (!X86FastEmitCompare(CmpLHS, CmpRHS, CmpVT, CI->getDebugLoc())) in X86FastEmitPseudoSelect()
2204 const Value *LHS = I->getOperand(1); in X86FastEmitPseudoSelect()
2205 const Value *RHS = I->getOperand(2); in X86FastEmitPseudoSelect()
2226 if (!isTypeLegal(I->getType(), RetVT)) in X86SelectSelect()
2230 if (const auto *CI = dyn_cast<CmpInst>(I->getOperand(0))) { in X86SelectSelect()
2235 case CmpInst::FCMP_FALSE: Opnd = I->getOperand(2); break; in X86SelectSelect()
2236 case CmpInst::FCMP_TRUE: Opnd = I->getOperand(1); break; in X86SelectSelect()
2238 // No need for a select anymore - this is an unconditional move. in X86SelectSelect()
2262 // Fall-back to pseudo conditional move instructions, which will be later in X86SelectSelect()
2263 // converted to control-flow. in X86SelectSelect()
2271 // The target-independent selection algorithm in FastISel already knows how in X86SelectSIToFP()
2274 if (!Subtarget->hasAVX()) in X86SelectSIToFP()
2277 if (!I->getOperand(0)->getType()->isIntegerTy(32)) in X86SelectSIToFP()
2281 unsigned OpReg = getRegForValue(I->getOperand(0)); in X86SelectSIToFP()
2288 if (I->getType()->isDoubleTy()) { in X86SelectSIToFP()
2289 // sitofp int -> double in X86SelectSIToFP()
2292 } else if (I->getType()->isFloatTy()) { in X86SelectSIToFP()
2293 // sitofp int -> float in X86SelectSIToFP()
2312 assert((I->getOpcode() == Instruction::FPExt || in X86SelectFPExtOrFPTrunc()
2313 I->getOpcode() == Instruction::FPTrunc) && in X86SelectFPExtOrFPTrunc()
2316 unsigned OpReg = getRegForValue(I->getOperand(0)); in X86SelectFPExtOrFPTrunc()
2324 if (Subtarget->hasAVX()) in X86SelectFPExtOrFPTrunc()
2332 if (X86ScalarSSEf64 && I->getType()->isDoubleTy() && in X86SelectFPExt()
2333 I->getOperand(0)->getType()->isFloatTy()) { in X86SelectFPExt()
2335 unsigned Opc = Subtarget->hasAVX() ? X86::VCVTSS2SDrr : X86::CVTSS2SDrr; in X86SelectFPExt()
2343 if (X86ScalarSSEf64 && I->getType()->isFloatTy() && in X86SelectFPTrunc()
2344 I->getOperand(0)->getType()->isDoubleTy()) { in X86SelectFPTrunc()
2346 unsigned Opc = Subtarget->hasAVX() ? X86::VCVTSD2SSrr : X86::CVTSD2SSrr; in X86SelectFPTrunc()
2354 EVT SrcVT = TLI.getValueType(DL, I->getOperand(0)->getType()); in X86SelectTrunc()
2355 EVT DstVT = TLI.getValueType(DL, I->getType()); in X86SelectTrunc()
2363 unsigned InputReg = getRegForValue(I->getOperand(0)); in X86SelectTrunc()
2375 if (!Subtarget->is64Bit()) { in X86SelectTrunc()
2376 // If we're on x86-32; we can't extract an i8 from a general register. in X86SelectTrunc()
2399 return Len <= (Subtarget->is64Bit() ? 32 : 16); in IsMemcpySmall()
2409 bool i64Legal = Subtarget->is64Bit(); in TryEmitSmallMemcpy()
2429 Len -= Size; in TryEmitSmallMemcpy()
2439 switch (II->getIntrinsicID()) { in fastLowerIntrinsicCall()
2443 if (Subtarget->useSoftFloat() || !Subtarget->hasF16C()) in fastLowerIntrinsicCall()
2446 const Value *Op = II->getArgOperand(0); in fastLowerIntrinsicCall()
2452 bool IsFloatToHalf = II->getIntrinsicID() == Intrinsic::convert_to_fp16; in fastLowerIntrinsicCall()
2454 if (!Op->getType()->isFloatTy()) in fastLowerIntrinsicCall()
2457 if (!II->getType()->isFloatTy()) in fastLowerIntrinsicCall()
2473 // Move the lower 32-bits of ResultReg to another register of class GR32. in fastLowerIntrinsicCall()
2479 // The result value is in the lower 16-bits of ResultReg. in fastLowerIntrinsicCall()
2483 assert(Op->getType()->isIntegerTy(16) && "Expected a 16-bit integer!"); in fastLowerIntrinsicCall()
2484 // Explicitly sign-extend the input to 32-bit. in fastLowerIntrinsicCall()
2494 // The result value is in the lower 32-bits of ResultReg. in fastLowerIntrinsicCall()
2507 if (MF->getTarget().getMCAsmInfo()->usesWindowsCFI()) in fastLowerIntrinsicCall()
2510 Type *RetTy = II->getCalledFunction()->getReturnType(); in fastLowerIntrinsicCall()
2527 MachineFrameInfo *MFI = MF->getFrameInfo(); in fastLowerIntrinsicCall()
2528 MFI->setFrameAddressIsTaken(true); in fastLowerIntrinsicCall()
2530 const X86RegisterInfo *RegInfo = Subtarget->getRegisterInfo(); in fastLowerIntrinsicCall()
2531 unsigned FrameReg = RegInfo->getPtrSizedFrameRegister(*MF); in fastLowerIntrinsicCall()
2537 // never directly reference the frame register (the TwoAddressInstruction- in fastLowerIntrinsicCall()
2549 unsigned Depth = cast<ConstantInt>(II->getOperand(0))->getZExtValue(); in fastLowerIntrinsicCall()
2550 while (Depth--) { in fastLowerIntrinsicCall()
2563 if (MCI->isVolatile()) in fastLowerIntrinsicCall()
2566 if (isa<ConstantInt>(MCI->getLength())) { in fastLowerIntrinsicCall()
2569 uint64_t Len = cast<ConstantInt>(MCI->getLength())->getZExtValue(); in fastLowerIntrinsicCall()
2572 if (!X86SelectAddress(MCI->getRawDest(), DestAM) || in fastLowerIntrinsicCall()
2573 !X86SelectAddress(MCI->getRawSource(), SrcAM)) in fastLowerIntrinsicCall()
2580 unsigned SizeWidth = Subtarget->is64Bit() ? 64 : 32; in fastLowerIntrinsicCall()
2581 if (!MCI->getLength()->getType()->isIntegerTy(SizeWidth)) in fastLowerIntrinsicCall()
2584 if (MCI->getSourceAddressSpace() > 255 || MCI->getDestAddressSpace() > 255) in fastLowerIntrinsicCall()
2587 return lowerCallTo(II, "memcpy", II->getNumArgOperands() - 2); in fastLowerIntrinsicCall()
2592 if (MSI->isVolatile()) in fastLowerIntrinsicCall()
2595 unsigned SizeWidth = Subtarget->is64Bit() ? 64 : 32; in fastLowerIntrinsicCall()
2596 if (!MSI->getLength()->getType()->isIntegerTy(SizeWidth)) in fastLowerIntrinsicCall()
2599 if (MSI->getDestAddressSpace() > 255) in fastLowerIntrinsicCall()
2602 return lowerCallTo(II, "memset", II->getNumArgOperands() - 2); in fastLowerIntrinsicCall()
2608 const Value *Op1 = II->getArgOperand(0); // The guard's value. in fastLowerIntrinsicCall()
2609 const AllocaInst *Slot = cast<AllocaInst>(II->getArgOperand(1)); in fastLowerIntrinsicCall()
2622 assert(DI->getAddress() && "Null address should be checked earlier!"); in fastLowerIntrinsicCall()
2623 if (!X86SelectAddress(DI->getAddress(), AM)) in fastLowerIntrinsicCall()
2628 assert(DI->getVariable()->isValidLocationForIntrinsic(DbgLoc) && in fastLowerIntrinsicCall()
2629 "Expected inlined-at fields to agree"); in fastLowerIntrinsicCall()
2632 .addMetadata(DI->getVariable()) in fastLowerIntrinsicCall()
2633 .addMetadata(DI->getExpression()); in fastLowerIntrinsicCall()
2641 if (!Subtarget->hasSSE1()) in fastLowerIntrinsicCall()
2644 Type *RetTy = II->getCalledFunction()->getReturnType(); in fastLowerIntrinsicCall()
2657 bool HasAVX = Subtarget->hasAVX(); in fastLowerIntrinsicCall()
2666 const Value *SrcVal = II->getArgOperand(0); in fastLowerIntrinsicCall()
2700 const Function *Callee = II->getCalledFunction(); in fastLowerIntrinsicCall()
2701 auto *Ty = cast<StructType>(Callee->getReturnType()); in fastLowerIntrinsicCall()
2702 Type *RetTy = Ty->getTypeAtIndex(0U); in fastLowerIntrinsicCall()
2703 Type *CondTy = Ty->getTypeAtIndex(1); in fastLowerIntrinsicCall()
2712 const Value *LHS = II->getArgOperand(0); in fastLowerIntrinsicCall()
2713 const Value *RHS = II->getArgOperand(1); in fastLowerIntrinsicCall()
2721 if (isa<ConstantInt>(RHS) && cast<ConstantInt>(RHS)->isOne()) in fastLowerIntrinsicCall()
2725 switch (II->getIntrinsicID()) { in fastLowerIntrinsicCall()
2762 TII.get(Opc[IsDec][VT.SimpleTy-MVT::i8]), ResultReg) in fastLowerIntrinsicCall()
2766 CI->getZExtValue()); in fastLowerIntrinsicCall()
2789 TII.get(TargetOpcode::COPY), Reg[VT.SimpleTy-MVT::i8]) in fastLowerIntrinsicCall()
2791 ResultReg = fastEmitInst_r(MULOpc[VT.SimpleTy-MVT::i8], in fastLowerIntrinsicCall()
2805 ResultReg = fastEmitInst_rr(MULOpc[VT.SimpleTy-MVT::i8], in fastLowerIntrinsicCall()
2826 switch (II->getIntrinsicID()) { in fastLowerIntrinsicCall()
2830 if (!Subtarget->hasSSE1()) in fastLowerIntrinsicCall()
2836 if (!Subtarget->hasSSE2()) in fastLowerIntrinsicCall()
2842 Type *RetTy = II->getCalledFunction()->getReturnType(); in fastLowerIntrinsicCall()
2853 bool HasAVX = Subtarget->hasAVX(); in fastLowerIntrinsicCall()
2862 const Value *Op = II->getArgOperand(0); in fastLowerIntrinsicCall()
2864 const Value *Index = IE->getOperand(2); in fastLowerIntrinsicCall()
2867 unsigned Idx = cast<ConstantInt>(Index)->getZExtValue(); in fastLowerIntrinsicCall()
2870 Op = IE->getOperand(1); in fastLowerIntrinsicCall()
2873 Op = IE->getOperand(0); in fastLowerIntrinsicCall()
2895 if (F->isVarArg()) in fastLowerArguments()
2898 CallingConv::ID CC = F->getCallingConv(); in fastLowerArguments()
2902 if (Subtarget->isCallingConvWin64(CC)) in fastLowerArguments()
2905 if (!Subtarget->is64Bit()) in fastLowerArguments()
2912 for (auto const &Arg : F->args()) { in fastLowerArguments()
2915 if (F->getAttributes().hasAttribute(Idx, Attribute::ByVal) || in fastLowerArguments()
2916 F->getAttributes().hasAttribute(Idx, Attribute::InReg) || in fastLowerArguments()
2917 F->getAttributes().hasAttribute(Idx, Attribute::StructRet) || in fastLowerArguments()
2918 F->getAttributes().hasAttribute(Idx, Attribute::SwiftSelf) || in fastLowerArguments()
2919 F->getAttributes().hasAttribute(Idx, Attribute::SwiftError) || in fastLowerArguments()
2920 F->getAttributes().hasAttribute(Idx, Attribute::Nest)) in fastLowerArguments()
2924 if (ArgTy->isStructTy() || ArgTy->isArrayTy() || ArgTy->isVectorTy()) in fastLowerArguments()
2937 if (!Subtarget->hasSSE1()) in fastLowerArguments()
2963 for (auto const &Arg : F->args()) { in fastLowerArguments()
2971 case MVT::f32: // fall-through in fastLowerArguments()
2974 unsigned DstReg = FuncInfo.MF->addLiveIn(SrcReg, RC); in fastLowerArguments()
2990 if (Subtarget->is64Bit()) in computeBytesPoppedByCalleeForSRet()
2992 if (Subtarget->getTargetTriple().isOSMSVCRT()) in computeBytesPoppedByCalleeForSRet()
2999 if (CS->arg_empty() || !CS->paramHasAttr(1, Attribute::StructRet) || in computeBytesPoppedByCalleeForSRet()
3000 CS->paramHasAttr(1, Attribute::InReg) || Subtarget->isTargetMCU()) in computeBytesPoppedByCalleeForSRet()
3018 bool Is64Bit = Subtarget->is64Bit(); in fastLowerCall()
3019 bool IsWin64 = Subtarget->isCallingConvWin64(CC); in fastLowerCall()
3040 // fastcc with -tailcallopt is intended to provide a guaranteed in fastLowerCall()
3046 // x86-32. Special handling for x86-64 is implemented. in fastLowerCall()
3051 if (CLI.CS && CLI.CS->hasInAllocaArgument()) in fastLowerCall()
3068 if (CI->getBitWidth() < 32) { in fastLowerCall()
3070 Val = ConstantExpr::getSExt(CI, Type::getInt32Ty(CI->getContext())); in fastLowerCall()
3072 Val = ConstantExpr::getZExt(CI, Type::getInt32Ty(CI->getContext())); in fastLowerCall()
3081 if (TI && TI->getType()->isIntegerTy(1) && CLI.CS && in fastLowerCall()
3082 (TI->getParent() == CLI.CS->getInstruction()->getParent()) && in fastLowerCall()
3083 TI->hasOneUse()) { in fastLowerCall()
3084 Value *PrevVal = TI->getOperand(0); in fastLowerCall()
3090 if (!isTypeLegal(PrevVal->getType(), VT)) in fastLowerCall()
3096 if (!isTypeLegal(Val->getType(), VT)) in fastLowerCall()
3110 CCState CCInfo(CC, IsVarArg, *FuncInfo.MF, ArgLocs, CLI.RetTy->getContext()); in fastLowerCall()
3127 const X86RegisterInfo *RegInfo = Subtarget->getRegisterInfo(); in fastLowerCall()
3158 // Handle zero-extension from i1 to i8, which is common. in fastLowerCall()
3207 // FIXME: Indirect doesn't need extending, but fast-isel doesn't fully in fastLowerCall()
3225 AM.Base.Reg = RegInfo->getStackRegister(); in fastLowerCall()
3228 unsigned Alignment = DL.getABITypeAlignment(ArgVal->getType()); in fastLowerCall()
3229 MachineMemOperand *MMO = FuncInfo.MF->getMachineMemOperand( in fastLowerCall()
3253 if (Subtarget->isPICStyleGOT()) { in fastLowerCall()
3254 unsigned Base = getInstrInfo()->getGlobalBaseReg(FuncInfo.MF); in fastLowerCall()
3262 // (prototype-less calls or calls to functions containing ellipsis (...) in in fastLowerCall()
3266 // registers used and is in the range 0 - 8 inclusive. in fastLowerCall()
3274 assert((Subtarget->hasSSE1() || !NumXMMRegs) in fastLowerCall()
3298 // Register-indirect call. in fastLowerCall()
3307 // See if we need any target-specific flags on the GV operand. in fastLowerCall()
3308 unsigned char OpFlags = Subtarget->classifyGlobalFunctionReference(GV); in fastLowerCall()
3320 // Add a register mask operand representing the call-preserved registers. in fastLowerCall()
3325 if (Subtarget->isPICStyleGOT()) in fastLowerCall()
3337 X86::isCalleePop(CC, Subtarget->is64Bit(), IsVarArg, in fastLowerCall()
3348 CLI.RetTy->getContext()); in fastLowerCall()
3358 // If this is x86-64, and we disabled SSE, we can't return FP values in fastLowerCall()
3360 ((Is64Bit || Ins[i].Flags.isInReg()) && !Subtarget->hasSSE1())) { in fastLowerCall()
3403 switch (I->getOpcode()) { in fastSelectInstruction()
3437 case Instruction::IntToPtr: // Deliberate fall-through. in fastSelectInstruction()
3439 EVT SrcVT = TLI.getValueType(DL, I->getOperand(0)->getType()); in fastSelectInstruction()
3440 EVT DstVT = TLI.getValueType(DL, I->getType()); in fastSelectInstruction()
3445 unsigned Reg = getRegForValue(I->getOperand(0)); in fastSelectInstruction()
3452 if (!Subtarget->hasSSE2()) in fastSelectInstruction()
3455 EVT SrcVT = TLI.getValueType(DL, I->getOperand(0)->getType()); in fastSelectInstruction()
3456 EVT DstVT = TLI.getValueType(DL, I->getType()); in fastSelectInstruction()
3462 !(Subtarget->hasAVX() && SrcVT.is256BitVector())) in fastSelectInstruction()
3465 unsigned Reg = getRegForValue(I->getOperand(0)); in fastSelectInstruction()
3483 uint64_t Imm = CI->getZExtValue(); in X86MaterializeInt()
3510 case MVT::i1: VT = MVT::i8; // fall-through in X86MaterializeInt()
3536 if (CFP->isNullValue()) in X86MaterializeFP()
3551 Opc = Subtarget->hasAVX() ? X86::VMOVSSrm : X86::MOVSSrm; in X86MaterializeFP()
3560 Opc = Subtarget->hasAVX() ? X86::VMOVSDrm : X86::MOVSDrm; in X86MaterializeFP()
3573 unsigned Align = DL.getPrefTypeAlignment(CFP->getType()); in X86MaterializeFP()
3576 Align = DL.getTypeAllocSize(CFP->getType()); in X86MaterializeFP()
3579 // x86-32 PIC requires a PIC base register for constant pools. in X86MaterializeFP()
3581 unsigned char OpFlag = Subtarget->classifyLocalReference(nullptr); in X86MaterializeFP()
3583 PICBase = getInstrInfo()->getGlobalBaseReg(FuncInfo.MF); in X86MaterializeFP()
3585 PICBase = getInstrInfo()->getGlobalBaseReg(FuncInfo.MF); in X86MaterializeFP()
3586 else if (Subtarget->is64Bit() && TM.getCodeModel() == CodeModel::Small) in X86MaterializeFP()
3601 MachineMemOperand *MMO = FuncInfo.MF->getMachineMemOperand( in X86MaterializeFP()
3604 MIB->addMemOperand(*FuncInfo.MF, MMO); in X86MaterializeFP()
3639 ? (Subtarget->isTarget64BitILP32() ? X86::LEA64_32r : X86::LEA32r) in X86MaterializeGV()
3650 EVT CEVT = TLI.getValueType(DL, C->getType(), true); in fastMaterializeConstant()
3677 assert(C->isStaticAlloca() && "dynamic alloca in the static alloca map?"); in fastMaterializeAlloca()
3684 ? (Subtarget->isTarget64BitILP32() ? X86::LEA64_32r : X86::LEA32r) in fastMaterializeAlloca()
3695 if (!isTypeLegal(CF->getType(), VT)) in fastMaterializeFloatZero()
3734 const Value *Ptr = LI->getPointerOperand(); in tryToFoldLoadIntoMI()
3741 unsigned Size = DL.getTypeAllocSize(LI->getType()); in tryToFoldLoadIntoMI()
3742 unsigned Alignment = LI->getAlignment(); in tryToFoldLoadIntoMI()
3745 Alignment = DL.getABITypeAlignment(LI->getType()); in tryToFoldLoadIntoMI()
3762 for (MachineInstr::mop_iterator I = Result->operands_begin(), in tryToFoldLoadIntoMI()
3763 E = Result->operands_end(); I != E; ++I, ++OperandNo) { in tryToFoldLoadIntoMI()
3768 unsigned IndexReg = constrainOperandRegClass(Result->getDesc(), in tryToFoldLoadIntoMI()
3775 Result->addMemOperand(*FuncInfo.MF, createMachineMemOperandFor(LI)); in tryToFoldLoadIntoMI()
3776 MI->eraseFromParent(); in tryToFoldLoadIntoMI()