Lines Matching refs:DAG
60 static SDValue getMOVL(SelectionDAG &DAG, DebugLoc dl, EVT VT, SDValue V1,
66 SelectionDAG &DAG,
71 SelectionDAG &DAG,
74 static SDValue ConcatVectors(SDValue Lower, SDValue Upper, SelectionDAG &DAG);
84 SelectionDAG &DAG, in Extract128BitVector() argument
93 EVT ResultVT = EVT::getVectorVT(*DAG.getContext(), in Extract128BitVector()
99 return DAG.getNode(ISD::UNDEF, dl, ResultVT); in Extract128BitVector()
113 SDValue VecIdx = DAG.getConstant(NormalizedIdxVal, MVT::i32); in Extract128BitVector()
115 SDValue Result = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, ResultVT, Vec, in Extract128BitVector()
132 SelectionDAG &DAG, in Insert128BitVector() argument
152 SDValue VecIdx = DAG.getConstant(NormalizedIdxVal, MVT::i32); in Insert128BitVector()
154 Result = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResultVT, Result, Vec, in Insert128BitVector()
163 static SDValue ConcatVectors(SDValue Lower, SDValue Upper, SelectionDAG &DAG) { in ConcatVectors() argument
168 EVT VT = EVT::getVectorVT(*DAG.getContext(), in ConcatVectors()
176 SDValue Vec = Insert128BitVector(DAG.getNode(ISD::UNDEF, dl, VT), Upper, in ConcatVectors()
177 DAG.getConstant( in ConcatVectors()
183 DAG, dl); in ConcatVectors()
186 Vec = Insert128BitVector(Vec, Lower, DAG.getConstant(0, MVT::i32), DAG, dl); in ConcatVectors()
1253 SelectionDAG &DAG) const { in getPICJumpTableRelocBase()
1257 return DAG.getNode(X86ISD::GlobalBaseReg, DebugLoc(), getPointerTy()); in getPICJumpTableRelocBase()
1344 DebugLoc dl, SelectionDAG &DAG) const { in LowerReturn()
1345 MachineFunction &MF = DAG.getMachineFunction(); in LowerReturn()
1350 RVLocs, *DAG.getContext()); in LowerReturn()
1354 MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo(); in LowerReturn()
1364 RetOps.push_back(DAG.getTargetConstant(FuncInfo->getBytesToPopOnReturn(), in LowerReturn()
1395 ValToCopy = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f80, ValToCopy); in LowerReturn()
1406 ValToCopy = DAG.getNode(ISD::BITCAST, dl, MVT::i64, ValToCopy); in LowerReturn()
1407 ValToCopy = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, in LowerReturn()
1412 ValToCopy = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32,ValToCopy); in LowerReturn()
1417 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), ValToCopy, Flag); in LowerReturn()
1426 DAG.getMachineFunction().getFunction()->hasStructRetAttr()) { in LowerReturn()
1427 MachineFunction &MF = DAG.getMachineFunction(); in LowerReturn()
1432 SDValue Val = DAG.getCopyFromReg(Chain, dl, Reg, getPointerTy()); in LowerReturn()
1434 Chain = DAG.getCopyToReg(Chain, dl, X86::RAX, Val, Flag); in LowerReturn()
1447 return DAG.getNode(X86ISD::RET_FLAG, dl, in LowerReturn()
1494 DebugLoc dl, SelectionDAG &DAG, in LowerCallResult() argument
1500 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), in LowerCallResult()
1501 getTargetMachine(), RVLocs, *DAG.getContext()); in LowerCallResult()
1527 Chain = SDValue(DAG.getMachineNode(X86::FpPOP_RETVAL, dl, CopyVT, in LowerCallResult()
1534 Val = DAG.getNode(ISD::FP_ROUND, dl, VA.getValVT(), Val, in LowerCallResult()
1536 DAG.getIntPtrConstant(1)); in LowerCallResult()
1538 Chain = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), in LowerCallResult()
1585 ISD::ArgFlagsTy Flags, SelectionDAG &DAG, in CreateCopyOfByValArgument() argument
1587 SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), MVT::i32); in CreateCopyOfByValArgument()
1589 return DAG.getMemcpy(Chain, dl, Dst, Src, SizeNode, Flags.getByValAlign(), in CreateCopyOfByValArgument()
1622 DebugLoc dl, SelectionDAG &DAG, in LowerMemArgument() argument
1647 return DAG.getFrameIndex(FI, getPointerTy()); in LowerMemArgument()
1651 SDValue FIN = DAG.getFrameIndex(FI, getPointerTy()); in LowerMemArgument()
1652 return DAG.getLoad(ValVT, dl, Chain, FIN, in LowerMemArgument()
1664 SelectionDAG &DAG, in LowerFormalArguments() argument
1667 MachineFunction &MF = DAG.getMachineFunction(); in LowerFormalArguments()
1686 ArgLocs, *DAG.getContext()); in LowerFormalArguments()
1726 ArgValue = DAG.getCopyFromReg(Chain, dl, Reg, RegVT); in LowerFormalArguments()
1732 ArgValue = DAG.getNode(ISD::AssertSext, dl, RegVT, ArgValue, in LowerFormalArguments()
1733 DAG.getValueType(VA.getValVT())); in LowerFormalArguments()
1735 ArgValue = DAG.getNode(ISD::AssertZext, dl, RegVT, ArgValue, in LowerFormalArguments()
1736 DAG.getValueType(VA.getValVT())); in LowerFormalArguments()
1738 ArgValue = DAG.getNode(ISD::BITCAST, dl, VA.getValVT(), ArgValue); in LowerFormalArguments()
1743 ArgValue = DAG.getNode(X86ISD::MOVDQ2Q, dl, VA.getValVT(), in LowerFormalArguments()
1746 ArgValue = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), ArgValue); in LowerFormalArguments()
1750 ArgValue = LowerMemArgument(Chain, CallConv, Ins, dl, DAG, VA, MFI, i); in LowerFormalArguments()
1755 ArgValue = DAG.getLoad(VA.getValVT(), dl, Chain, ArgValue, in LowerFormalArguments()
1771 SDValue Copy = DAG.getCopyToReg(DAG.getEntryNode(), dl, Reg, InVals[0]); in LowerFormalArguments()
1772 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Copy, Chain); in LowerFormalArguments()
1778 StackSize = GetAlignedArgumentStackSize(StackSize, DAG); in LowerFormalArguments()
1852 SDValue RSFIN = DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(), in LowerFormalArguments()
1856 SDValue FIN = DAG.getNode(ISD::ADD, dl, getPointerTy(), RSFIN, in LowerFormalArguments()
1857 DAG.getIntPtrConstant(Offset)); in LowerFormalArguments()
1860 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i64); in LowerFormalArguments()
1862 DAG.getStore(Val.getValue(1), dl, Val, FIN, in LowerFormalArguments()
1876 SDValue ALVal = DAG.getCopyFromReg(DAG.getEntryNode(), dl, AL, MVT::i8); in LowerFormalArguments()
1879 SaveXMMOps.push_back(DAG.getIntPtrConstant( in LowerFormalArguments()
1881 SaveXMMOps.push_back(DAG.getIntPtrConstant( in LowerFormalArguments()
1887 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, MVT::v4f32); in LowerFormalArguments()
1890 MemOps.push_back(DAG.getNode(X86ISD::VASTART_SAVE_XMM_REGS, dl, in LowerFormalArguments()
1896 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, in LowerFormalArguments()
1926 DebugLoc dl, SelectionDAG &DAG, in LowerMemOpCallTo() argument
1930 SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset); in LowerMemOpCallTo()
1931 PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(), StackPtr, PtrOff); in LowerMemOpCallTo()
1933 return CreateCopyOfByValArgument(Arg, PtrOff, Chain, Flags, DAG, dl); in LowerMemOpCallTo()
1935 return DAG.getStore(Chain, dl, Arg, PtrOff, in LowerMemOpCallTo()
1943 X86TargetLowering::EmitTailCallLoadRetAddr(SelectionDAG &DAG, in EmitTailCallLoadRetAddr() argument
1949 OutRetAddr = getReturnAddressFrameIndex(DAG); in EmitTailCallLoadRetAddr()
1952 OutRetAddr = DAG.getLoad(VT, dl, Chain, OutRetAddr, MachinePointerInfo(), in EmitTailCallLoadRetAddr()
1960 EmitTailCallStoreRetAddr(SelectionDAG & DAG, MachineFunction &MF, in EmitTailCallStoreRetAddr() argument
1970 SDValue NewRetAddrFrIdx = DAG.getFrameIndex(NewReturnAddrFI, VT); in EmitTailCallStoreRetAddr()
1971 Chain = DAG.getStore(Chain, dl, RetAddrFrIdx, NewRetAddrFrIdx, in EmitTailCallStoreRetAddr()
1984 DebugLoc dl, SelectionDAG &DAG, in LowerCall() argument
1986 MachineFunction &MF = DAG.getMachineFunction(); in LowerCall()
1996 Outs, OutVals, Ins, DAG); in LowerCall()
2013 ArgLocs, *DAG.getContext()); in LowerCall()
2029 NumBytes = GetAlignedArgumentStackSize(NumBytes, DAG); in LowerCall()
2045 Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(NumBytes, true)); in LowerCall()
2050 Chain = EmitTailCallLoadRetAddr(DAG, RetAddrFrIdx, Chain, isTailCall, in LowerCall()
2071 Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, RegVT, Arg); in LowerCall()
2074 Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, RegVT, Arg); in LowerCall()
2079 Arg = DAG.getNode(ISD::BITCAST, dl, MVT::i64, Arg); in LowerCall()
2080 Arg = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Arg); in LowerCall()
2081 Arg = getMOVL(DAG, dl, MVT::v2i64, DAG.getUNDEF(MVT::v2i64), Arg); in LowerCall()
2083 Arg = DAG.getNode(ISD::ANY_EXTEND, dl, RegVT, Arg); in LowerCall()
2086 Arg = DAG.getNode(ISD::BITCAST, dl, RegVT, Arg); in LowerCall()
2090 SDValue SpillSlot = DAG.CreateStackTemporary(VA.getValVT()); in LowerCall()
2092 Chain = DAG.getStore(Chain, dl, Arg, SpillSlot, in LowerCall()
2118 StackPtr = DAG.getCopyFromReg(Chain, dl, X86StackPtr, getPointerTy()); in LowerCall()
2120 dl, DAG, VA, Flags)); in LowerCall()
2125 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, in LowerCall()
2135 Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first, in LowerCall()
2144 Chain = DAG.getCopyToReg(Chain, dl, X86::EBX, in LowerCall()
2145 DAG.getNode(X86ISD::GlobalBaseReg, in LowerCall()
2162 Callee = LowerGlobalAddress(Callee, DAG); in LowerCall()
2164 Callee = LowerExternalSymbol(Callee, DAG); in LowerCall()
2186 Chain = DAG.getCopyToReg(Chain, dl, X86::AL, in LowerCall()
2187 DAG.getConstant(NumXMMRegs, MVT::i8), InFlag); in LowerCall()
2200 SDValue ArgChain = DAG.getStackArgumentTokenFactor(Chain); in LowerCall()
2219 FIN = DAG.getFrameIndex(FI, getPointerTy()); in LowerCall()
2223 SDValue Source = DAG.getIntPtrConstant(VA.getLocMemOffset()); in LowerCall()
2225 StackPtr = DAG.getCopyFromReg(Chain, dl, X86StackPtr, in LowerCall()
2227 Source = DAG.getNode(ISD::ADD, dl, getPointerTy(), StackPtr, Source); in LowerCall()
2231 Flags, DAG, dl)); in LowerCall()
2235 DAG.getStore(ArgChain, dl, Arg, FIN, in LowerCall()
2243 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, in LowerCall()
2248 Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first, in LowerCall()
2255 Chain = EmitTailCallStoreRetAddr(DAG, MF, Chain, RetAddrFrIdx, Is64Bit, in LowerCall()
2305 Callee = DAG.getTargetGlobalAddress(GV, dl, getPointerTy(), in LowerCall()
2310 Callee = DAG.getNode(X86ISD::WrapperRIP, dl, getPointerTy(), Callee); in LowerCall()
2313 Callee = DAG.getLoad(getPointerTy(), dl, DAG.getEntryNode(), Callee, in LowerCall()
2334 Callee = DAG.getTargetExternalSymbol(S->getSymbol(), getPointerTy(), in LowerCall()
2339 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); in LowerCall()
2343 Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, true), in LowerCall()
2344 DAG.getIntPtrConstant(0, true), InFlag); in LowerCall()
2352 Ops.push_back(DAG.getConstant(FPDiff, MVT::i32)); in LowerCall()
2357 Ops.push_back(DAG.getRegister(RegsToPass[i].first, in LowerCall()
2362 Ops.push_back(DAG.getRegister(X86::EBX, getPointerTy())); in LowerCall()
2366 Ops.push_back(DAG.getRegister(X86::AL, MVT::i8)); in LowerCall()
2378 return DAG.getNode(X86ISD::TC_RETURN, dl, in LowerCall()
2382 Chain = DAG.getNode(X86ISD::CALL, dl, NodeTys, &Ops[0], Ops.size()); in LowerCall()
2399 Chain = DAG.getCALLSEQ_END(Chain, in LowerCall()
2400 DAG.getIntPtrConstant(NumBytes, true), in LowerCall()
2401 DAG.getIntPtrConstant(NumBytesForCalleeToPush, in LowerCall()
2410 Ins, dl, DAG, InVals); in LowerCall()
2449 SelectionDAG& DAG) const { in GetAlignedArgumentStackSize()
2450 MachineFunction &MF = DAG.getMachineFunction(); in GetAlignedArgumentStackSize()
2534 SelectionDAG& DAG) const { in IsEligibleForTailCallOptimization()
2540 const MachineFunction &MF = DAG.getMachineFunction(); in IsEligibleForTailCallOptimization()
2541 const Function *CallerF = DAG.getMachineFunction().getFunction(); in IsEligibleForTailCallOptimization()
2579 CCState CCInfo(CalleeCC, isVarArg, DAG.getMachineFunction(), in IsEligibleForTailCallOptimization()
2580 getTargetMachine(), ArgLocs, *DAG.getContext()); in IsEligibleForTailCallOptimization()
2600 CCState CCInfo(CalleeCC, false, DAG.getMachineFunction(), in IsEligibleForTailCallOptimization()
2601 getTargetMachine(), RVLocs, *DAG.getContext()); in IsEligibleForTailCallOptimization()
2614 CCState CCInfo1(CalleeCC, false, DAG.getMachineFunction(), in IsEligibleForTailCallOptimization()
2615 getTargetMachine(), RVLocs1, *DAG.getContext()); in IsEligibleForTailCallOptimization()
2619 CCState CCInfo2(CallerCC, false, DAG.getMachineFunction(), in IsEligibleForTailCallOptimization()
2620 getTargetMachine(), RVLocs2, *DAG.getContext()); in IsEligibleForTailCallOptimization()
2646 CCState CCInfo(CalleeCC, isVarArg, DAG.getMachineFunction(), in IsEligibleForTailCallOptimization()
2647 getTargetMachine(), ArgLocs, *DAG.getContext()); in IsEligibleForTailCallOptimization()
2656 MachineFunction &MF = DAG.getMachineFunction(); in IsEligibleForTailCallOptimization()
2767 SDValue V1, SelectionDAG &DAG) { in getTargetShuffleNode() argument
2773 return DAG.getNode(Opc, dl, VT, V1); in getTargetShuffleNode()
2780 SDValue V1, unsigned TargetMask, SelectionDAG &DAG) { in getTargetShuffleNode() argument
2786 return DAG.getNode(Opc, dl, VT, V1, DAG.getConstant(TargetMask, MVT::i8)); in getTargetShuffleNode()
2793 SDValue V1, SDValue V2, unsigned TargetMask, SelectionDAG &DAG) { in getTargetShuffleNode() argument
2799 return DAG.getNode(Opc, dl, VT, V1, V2, in getTargetShuffleNode()
2800 DAG.getConstant(TargetMask, MVT::i8)); in getTargetShuffleNode()
2806 SDValue V1, SDValue V2, SelectionDAG &DAG) { in getTargetShuffleNode() argument
2832 return DAG.getNode(Opc, dl, VT, V1, V2); in getTargetShuffleNode()
2837 SDValue X86TargetLowering::getReturnAddressFrameIndex(SelectionDAG &DAG) const { in getReturnAddressFrameIndex()
2838 MachineFunction &MF = DAG.getMachineFunction(); in getReturnAddressFrameIndex()
2850 return DAG.getFrameIndex(ReturnAddrIndex, getPointerTy()); in getReturnAddressFrameIndex()
2911 SDValue &LHS, SDValue &RHS, SelectionDAG &DAG) { in TranslateX86CC() argument
2916 RHS = DAG.getConstant(0, RHS.getValueType()); in TranslateX86CC()
2923 RHS = DAG.getConstant(0, RHS.getValueType()); in TranslateX86CC()
3681 SelectionDAG &DAG) { in CommuteVectorShuffle() argument
3695 return DAG.getVectorShuffle(VT, SVOp->getDebugLoc(), SVOp->getOperand(1), in CommuteVectorShuffle()
3814 static SDValue getZeroVector(EVT VT, bool HasSSE2, SelectionDAG &DAG, in getZeroVector() argument
3823 SDValue Cst = DAG.getTargetConstant(0, MVT::i32); in getZeroVector()
3824 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, Cst, Cst, Cst, Cst); in getZeroVector()
3826 SDValue Cst = DAG.getTargetConstantFP(+0.0, MVT::f32); in getZeroVector()
3827 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4f32, Cst, Cst, Cst, Cst); in getZeroVector()
3833 SDValue Cst = DAG.getTargetConstantFP(+0.0, MVT::f32); in getZeroVector()
3835 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v8f32, Ops, 8); in getZeroVector()
3837 return DAG.getNode(ISD::BITCAST, dl, VT, Vec); in getZeroVector()
3843 static SDValue getOnesVector(EVT VT, SelectionDAG &DAG, DebugLoc dl) { in getOnesVector() argument
3848 SDValue Cst = DAG.getTargetConstant(~0U, MVT::i32); in getOnesVector()
3853 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v8i32, Ops, 8); in getOnesVector()
3855 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, Cst, Cst, Cst, Cst); in getOnesVector()
3856 return DAG.getNode(ISD::BITCAST, dl, VT, Vec); in getOnesVector()
3861 static SDValue NormalizeMask(ShuffleVectorSDNode *SVOp, SelectionDAG &DAG) { in NormalizeMask() argument
3876 return DAG.getVectorShuffle(VT, SVOp->getDebugLoc(), SVOp->getOperand(0), in NormalizeMask()
3883 static SDValue getMOVL(SelectionDAG &DAG, DebugLoc dl, EVT VT, SDValue V1, in getMOVL() argument
3890 return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask[0]); in getMOVL()
3894 static SDValue getUnpackl(SelectionDAG &DAG, DebugLoc dl, EVT VT, SDValue V1, in getUnpackl() argument
3902 return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask[0]); in getUnpackl()
3906 static SDValue getUnpackh(SelectionDAG &DAG, DebugLoc dl, EVT VT, SDValue V1, in getUnpackh() argument
3915 return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask[0]); in getUnpackh()
3919 static SDValue PromoteSplat(ShuffleVectorSDNode *SV, SelectionDAG &DAG) { in PromoteSplat() argument
3930 V1 = getUnpackl(DAG, dl, VT, V1, V1); in PromoteSplat()
3932 V1 = getUnpackh(DAG, dl, VT, V1, V1); in PromoteSplat()
3940 V1 = DAG.getNode(ISD::BITCAST, dl, PVT, V1); in PromoteSplat()
3941 V1 = DAG.getVectorShuffle(PVT, dl, V1, DAG.getUNDEF(PVT), &SplatMask[0]); in PromoteSplat()
3942 return DAG.getNode(ISD::BITCAST, dl, VT, V1); in PromoteSplat()
3951 SelectionDAG &DAG) { in getShuffleVectorZeroOrUndef() argument
3954 ? getZeroVector(VT, HasSSE2, DAG, V2.getDebugLoc()) : DAG.getUNDEF(VT); in getShuffleVectorZeroOrUndef()
3960 return DAG.getVectorShuffle(VT, V2.getDebugLoc(), V1, V2, &MaskVec[0]); in getShuffleVectorZeroOrUndef()
3965 static SDValue getShuffleScalarElt(SDNode *N, int Index, SelectionDAG &DAG, in getShuffleScalarElt() argument
3979 return DAG.getUNDEF(VT.getVectorElementType()); in getShuffleScalarElt()
3983 return getShuffleScalarElt(NewV.getNode(), Index % NumElems, DAG, Depth+1); in getShuffleScalarElt()
4052 return getShuffleScalarElt(V.getOperand(OpNum).getNode(), Index, DAG, in getShuffleScalarElt()
4062 return DAG.getUNDEF(VT.getVectorElementType()); in getShuffleScalarElt()
4065 return getShuffleScalarElt(NewV.getNode(), Index % NumElems, DAG, in getShuffleScalarElt()
4081 : DAG.getUNDEF(VT.getVectorElementType()); in getShuffleScalarElt()
4094 bool ZerosFromLeft, SelectionDAG &DAG) { in getNumOfConsecutiveZeros() argument
4099 SDValue Elt = getShuffleScalarElt(N, Index, DAG, 0); in getNumOfConsecutiveZeros()
4140 static bool isVectorShiftRight(ShuffleVectorSDNode *SVOp, SelectionDAG &DAG, in isVectorShiftRight() argument
4144 false /* check zeros from right */, DAG); in isVectorShiftRight()
4173 static bool isVectorShiftLeft(ShuffleVectorSDNode *SVOp, SelectionDAG &DAG, in isVectorShiftLeft() argument
4177 true /* check zeros from left */, DAG); in isVectorShiftLeft()
4206 static bool isVectorShift(ShuffleVectorSDNode *SVOp, SelectionDAG &DAG, in isVectorShift() argument
4208 if (isVectorShiftLeft(SVOp, DAG, isLeft, ShVal, ShAmt) || in isVectorShift()
4209 isVectorShiftRight(SVOp, DAG, isLeft, ShVal, ShAmt)) in isVectorShift()
4219 SelectionDAG &DAG, in LowerBuildVectorv16i8() argument
4231 V = getZeroVector(MVT::v8i16, true, DAG, dl); in LowerBuildVectorv16i8()
4233 V = DAG.getUNDEF(MVT::v8i16); in LowerBuildVectorv16i8()
4241 LastElt = DAG.getNode(ISD::ZERO_EXTEND, dl, in LowerBuildVectorv16i8()
4245 ThisElt = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, Op.getOperand(i)); in LowerBuildVectorv16i8()
4246 ThisElt = DAG.getNode(ISD::SHL, dl, MVT::i16, in LowerBuildVectorv16i8()
4247 ThisElt, DAG.getConstant(8, MVT::i8)); in LowerBuildVectorv16i8()
4249 ThisElt = DAG.getNode(ISD::OR, dl, MVT::i16, ThisElt, LastElt); in LowerBuildVectorv16i8()
4254 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, V, ThisElt, in LowerBuildVectorv16i8()
4255 DAG.getIntPtrConstant(i/2)); in LowerBuildVectorv16i8()
4259 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, V); in LowerBuildVectorv16i8()
4266 SelectionDAG &DAG, in LowerBuildVectorv8i16() argument
4279 V = getZeroVector(MVT::v8i16, true, DAG, dl); in LowerBuildVectorv8i16()
4281 V = DAG.getUNDEF(MVT::v8i16); in LowerBuildVectorv8i16()
4284 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, in LowerBuildVectorv8i16()
4286 DAG.getIntPtrConstant(i)); in LowerBuildVectorv8i16()
4296 unsigned NumBits, SelectionDAG &DAG, in getVShift() argument
4300 SrcOp = DAG.getNode(ISD::BITCAST, dl, ShVT, SrcOp); in getVShift()
4301 return DAG.getNode(ISD::BITCAST, dl, VT, in getVShift()
4302 DAG.getNode(Opc, dl, ShVT, SrcOp, in getVShift()
4303 DAG.getConstant(NumBits, in getVShift()
4309 SelectionDAG &DAG) const { in LowerAsSplatVectorLoad()
4327 } else if (DAG.isBaseWithConstantOffset(Ptr) && in LowerAsSplatVectorLoad()
4338 MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo(); in LowerAsSplatVectorLoad()
4339 if (DAG.InferPtrAlignment(Ptr) < 16) { in LowerAsSplatVectorLoad()
4358 Ptr = DAG.getNode(ISD::ADD, Ptr.getDebugLoc(), Ptr.getValueType(), in LowerAsSplatVectorLoad()
4359 Ptr,DAG.getConstant(StartOffset, Ptr.getValueType())); in LowerAsSplatVectorLoad()
4364 SDValue V1 = DAG.getLoad(VT, dl, Chain, Ptr, in LowerAsSplatVectorLoad()
4368 V1 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, V1); in LowerAsSplatVectorLoad()
4369 return DAG.getNode(ISD::BITCAST, dl, VT, in LowerAsSplatVectorLoad()
4370 DAG.getVectorShuffle(MVT::v4i32, dl, V1, in LowerAsSplatVectorLoad()
4371 DAG.getUNDEF(MVT::v4i32),&Mask[0])); in LowerAsSplatVectorLoad()
4387 DebugLoc &DL, SelectionDAG &DAG) { in EltsFromConsecutiveLoads() argument
4414 if (!DAG.isConsecutiveLoad(LD, LDBase, EltVT.getSizeInBits()/8, i)) in EltsFromConsecutiveLoads()
4423 if (DAG.InferPtrAlignment(LDBase->getBasePtr()) >= 16) in EltsFromConsecutiveLoads()
4424 return DAG.getLoad(VT, DL, LDBase->getChain(), LDBase->getBasePtr(), in EltsFromConsecutiveLoads()
4427 return DAG.getLoad(VT, DL, LDBase->getChain(), LDBase->getBasePtr(), in EltsFromConsecutiveLoads()
4432 SDVTList Tys = DAG.getVTList(MVT::v2i64, MVT::Other); in EltsFromConsecutiveLoads()
4434 SDValue ResNode = DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, DL, Tys, in EltsFromConsecutiveLoads()
4437 return DAG.getNode(ISD::BITCAST, DL, VT, ResNode); in EltsFromConsecutiveLoads()
4443 X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const { in LowerBUILD_VECTOR()
4462 EVT HVT = EVT::getVectorVT(*DAG.getContext(), ExtVT, NumElems/2); in LowerBUILD_VECTOR()
4465 SDValue Lower = DAG.getNode(ISD::BUILD_VECTOR, dl, HVT, &V[0], NumElems/2); in LowerBUILD_VECTOR()
4467 SDValue Upper = DAG.getNode(ISD::BUILD_VECTOR, dl, HVT, &V[NumElems / 2], in LowerBUILD_VECTOR()
4470 return ConcatVectors(Lower, Upper, DAG); in LowerBUILD_VECTOR()
4487 return getOnesVector(Op.getValueType(), DAG, dl); in LowerBUILD_VECTOR()
4488 return getZeroVector(Op.getValueType(), Subtarget->hasSSE2(), DAG, dl); in LowerBUILD_VECTOR()
4516 return DAG.getUNDEF(VT); in LowerBUILD_VECTOR()
4530 if (DAG.MaskedValueIsZero(Item, APInt::getBitsSet(64, 32, 64))) { in LowerBUILD_VECTOR()
4538 Item = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Item); in LowerBUILD_VECTOR()
4539 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VecVT, Item); in LowerBUILD_VECTOR()
4541 Subtarget->hasSSE2(), DAG); in LowerBUILD_VECTOR()
4550 Item = DAG.getVectorShuffle(VecVT, dl, Item, in LowerBUILD_VECTOR()
4551 DAG.getUNDEF(Item.getValueType()), in LowerBUILD_VECTOR()
4554 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Item); in LowerBUILD_VECTOR()
4564 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item); in LowerBUILD_VECTOR()
4567 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item); in LowerBUILD_VECTOR()
4570 DAG); in LowerBUILD_VECTOR()
4572 Item = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, Item); in LowerBUILD_VECTOR()
4575 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MiddleVT, Item); in LowerBUILD_VECTOR()
4577 Subtarget->hasSSE2(), DAG); in LowerBUILD_VECTOR()
4578 return DAG.getNode(ISD::BITCAST, dl, VT, Item); in LowerBUILD_VECTOR()
4588 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, in LowerBUILD_VECTOR()
4590 NumBits/2, DAG, *this, dl); in LowerBUILD_VECTOR()
4602 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item); in LowerBUILD_VECTOR()
4606 Subtarget->hasSSE2(), DAG); in LowerBUILD_VECTOR()
4610 return DAG.getVectorShuffle(VT, dl, Item, DAG.getUNDEF(VT), &MaskVec[0]); in LowerBUILD_VECTOR()
4624 return LowerAsSplatVectorLoad(Item, VT, dl, DAG); in LowerBUILD_VECTOR()
4639 SDValue V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, in LowerBUILD_VECTOR()
4642 Subtarget->hasSSE2(), DAG); in LowerBUILD_VECTOR()
4649 SDValue V = LowerBuildVectorv16i8(Op, NonZeros,NumNonZero,NumZero, DAG, in LowerBUILD_VECTOR()
4655 SDValue V = LowerBuildVectorv8i16(Op, NonZeros,NumNonZero,NumZero, DAG, in LowerBUILD_VECTOR()
4667 V[i] = getZeroVector(VT, Subtarget->hasSSE2(), DAG, dl); in LowerBUILD_VECTOR()
4669 V[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(i)); in LowerBUILD_VECTOR()
4679 V[i] = getMOVL(DAG, dl, VT, V[i*2+1], V[i*2]); in LowerBUILD_VECTOR()
4682 V[i] = getMOVL(DAG, dl, VT, V[i*2], V[i*2+1]); in LowerBUILD_VECTOR()
4685 V[i] = getUnpackl(DAG, dl, VT, V[i*2], V[i*2+1]); in LowerBUILD_VECTOR()
4697 return DAG.getVectorShuffle(VT, dl, V[0], V[1], &MaskVec[0]); in LowerBUILD_VECTOR()
4706 SDValue LD = EltsFromConsecutiveLoads(VT, V, dl, DAG); in LowerBUILD_VECTOR()
4714 Result = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(0)); in LowerBUILD_VECTOR()
4716 Result = DAG.getUNDEF(VT); in LowerBUILD_VECTOR()
4720 Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Result, in LowerBUILD_VECTOR()
4721 Op.getOperand(i), DAG.getIntPtrConstant(i)); in LowerBUILD_VECTOR()
4731 V[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(i)); in LowerBUILD_VECTOR()
4733 V[i] = DAG.getUNDEF(VT); in LowerBUILD_VECTOR()
4752 V[i] = getUnpackl(DAG, dl, VT, V[i], V[i + EltStride]); in LowerBUILD_VECTOR()
4762 X86TargetLowering::LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const { in LowerCONCAT_VECTORS()
4771 SDValue InVec = DAG.getNode(ISD::BITCAST,dl, MVT::v1i64, Op.getOperand(0)); in LowerCONCAT_VECTORS()
4772 SDValue VecOp = DAG.getNode(X86ISD::MOVQ2DQ, dl, MVT::v2i64, InVec); in LowerCONCAT_VECTORS()
4776 VecOp = DAG.getNode(ISD::BITCAST, dl, ResVT, VecOp); in LowerCONCAT_VECTORS()
4777 VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, ResVT, VecOp, in LowerCONCAT_VECTORS()
4778 InVec.getOperand(0), DAG.getIntPtrConstant(NumElts/2+1)); in LowerCONCAT_VECTORS()
4780 InVec = DAG.getNode(ISD::BITCAST, dl, MVT::v1i64, InVec); in LowerCONCAT_VECTORS()
4781 SDValue VecOp2 = DAG.getNode(X86ISD::MOVQ2DQ, dl, MVT::v2i64, InVec); in LowerCONCAT_VECTORS()
4783 VecOp = DAG.getVectorShuffle(MVT::v2i64, dl, VecOp, VecOp2, Mask); in LowerCONCAT_VECTORS()
4785 return DAG.getNode(ISD::BITCAST, dl, ResVT, VecOp); in LowerCONCAT_VECTORS()
4795 SelectionDAG &DAG) const { in LowerVECTOR_SHUFFLEv8i16()
4866 NewV = DAG.getVectorShuffle(MVT::v2i64, dl, in LowerVECTOR_SHUFFLEv8i16()
4867 DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, V1), in LowerVECTOR_SHUFFLEv8i16()
4868 DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, V2), &MaskV[0]); in LowerVECTOR_SHUFFLEv8i16()
4869 NewV = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, NewV); in LowerVECTOR_SHUFFLEv8i16()
4908 NewV = DAG.getVectorShuffle(MVT::v8i16, dl, NewV, in LowerVECTOR_SHUFFLEv8i16()
4909 DAG.getUNDEF(MVT::v8i16), &MaskVals[0]); in LowerVECTOR_SHUFFLEv8i16()
4913 return getTargetShuffleNode(Opc, dl, MVT::v8i16, V1, TargetMask, DAG); in LowerVECTOR_SHUFFLEv8i16()
4931 pshufbMask.push_back(DAG.getConstant(0x80, MVT::i8)); in LowerVECTOR_SHUFFLEv8i16()
4932 pshufbMask.push_back(DAG.getConstant(0x80, MVT::i8)); in LowerVECTOR_SHUFFLEv8i16()
4935 pshufbMask.push_back(DAG.getConstant(EltIdx, MVT::i8)); in LowerVECTOR_SHUFFLEv8i16()
4936 pshufbMask.push_back(DAG.getConstant(EltIdx+1, MVT::i8)); in LowerVECTOR_SHUFFLEv8i16()
4938 V1 = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, V1); in LowerVECTOR_SHUFFLEv8i16()
4939 V1 = DAG.getNode(X86ISD::PSHUFB, dl, MVT::v16i8, V1, in LowerVECTOR_SHUFFLEv8i16()
4940 DAG.getNode(ISD::BUILD_VECTOR, dl, in LowerVECTOR_SHUFFLEv8i16()
4943 return DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V1); in LowerVECTOR_SHUFFLEv8i16()
4951 pshufbMask.push_back(DAG.getConstant(0x80, MVT::i8)); in LowerVECTOR_SHUFFLEv8i16()
4952 pshufbMask.push_back(DAG.getConstant(0x80, MVT::i8)); in LowerVECTOR_SHUFFLEv8i16()
4955 pshufbMask.push_back(DAG.getConstant(EltIdx - 16, MVT::i8)); in LowerVECTOR_SHUFFLEv8i16()
4956 pshufbMask.push_back(DAG.getConstant(EltIdx - 15, MVT::i8)); in LowerVECTOR_SHUFFLEv8i16()
4958 V2 = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, V2); in LowerVECTOR_SHUFFLEv8i16()
4959 V2 = DAG.getNode(X86ISD::PSHUFB, dl, MVT::v16i8, V2, in LowerVECTOR_SHUFFLEv8i16()
4960 DAG.getNode(ISD::BUILD_VECTOR, dl, in LowerVECTOR_SHUFFLEv8i16()
4962 V1 = DAG.getNode(ISD::OR, dl, MVT::v16i8, V1, V2); in LowerVECTOR_SHUFFLEv8i16()
4963 return DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V1); in LowerVECTOR_SHUFFLEv8i16()
4985 NewV = DAG.getVectorShuffle(MVT::v8i16, dl, NewV, DAG.getUNDEF(MVT::v8i16), in LowerVECTOR_SHUFFLEv8i16()
4992 DAG); in LowerVECTOR_SHUFFLEv8i16()
5013 NewV = DAG.getVectorShuffle(MVT::v8i16, dl, NewV, DAG.getUNDEF(MVT::v8i16), in LowerVECTOR_SHUFFLEv8i16()
5020 DAG); in LowerVECTOR_SHUFFLEv8i16()
5041 ? DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, V1, in LowerVECTOR_SHUFFLEv8i16()
5042 DAG.getIntPtrConstant(EltIdx)) in LowerVECTOR_SHUFFLEv8i16()
5043 : DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, V2, in LowerVECTOR_SHUFFLEv8i16()
5044 DAG.getIntPtrConstant(EltIdx - 8)); in LowerVECTOR_SHUFFLEv8i16()
5045 NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, NewV, ExtOp, in LowerVECTOR_SHUFFLEv8i16()
5046 DAG.getIntPtrConstant(i)); in LowerVECTOR_SHUFFLEv8i16()
5057 SelectionDAG &DAG, in LowerVECTOR_SHUFFLEv16i8() argument
5095 pshufbMask.push_back(DAG.getConstant(0x80, MVT::i8)); in LowerVECTOR_SHUFFLEv16i8()
5098 pshufbMask.push_back(DAG.getConstant(EltIdx, MVT::i8)); in LowerVECTOR_SHUFFLEv16i8()
5104 V1 = DAG.getNode(X86ISD::PSHUFB, dl, MVT::v16i8, V1, in LowerVECTOR_SHUFFLEv16i8()
5105 DAG.getNode(ISD::BUILD_VECTOR, dl, in LowerVECTOR_SHUFFLEv16i8()
5116 pshufbMask.push_back(DAG.getConstant(0x80, MVT::i8)); in LowerVECTOR_SHUFFLEv16i8()
5119 pshufbMask.push_back(DAG.getConstant(EltIdx - 16, MVT::i8)); in LowerVECTOR_SHUFFLEv16i8()
5121 V2 = DAG.getNode(X86ISD::PSHUFB, dl, MVT::v16i8, V2, in LowerVECTOR_SHUFFLEv16i8()
5122 DAG.getNode(ISD::BUILD_VECTOR, dl, in LowerVECTOR_SHUFFLEv16i8()
5124 return DAG.getNode(ISD::OR, dl, MVT::v16i8, V1, V2); in LowerVECTOR_SHUFFLEv16i8()
5130 V1 = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V1); in LowerVECTOR_SHUFFLEv16i8()
5131 V2 = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V2); in LowerVECTOR_SHUFFLEv16i8()
5154 InsElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, Elt1Src, in LowerVECTOR_SHUFFLEv16i8()
5155 DAG.getIntPtrConstant(Elt1 / 2)); in LowerVECTOR_SHUFFLEv16i8()
5156 NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, NewV, InsElt, in LowerVECTOR_SHUFFLEv16i8()
5157 DAG.getIntPtrConstant(i)); in LowerVECTOR_SHUFFLEv16i8()
5165 InsElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, Elt1Src, in LowerVECTOR_SHUFFLEv16i8()
5166 DAG.getIntPtrConstant(Elt1 / 2)); in LowerVECTOR_SHUFFLEv16i8()
5168 InsElt = DAG.getNode(ISD::SHL, dl, MVT::i16, InsElt, in LowerVECTOR_SHUFFLEv16i8()
5169 DAG.getConstant(8, in LowerVECTOR_SHUFFLEv16i8()
5172 InsElt = DAG.getNode(ISD::AND, dl, MVT::i16, InsElt, in LowerVECTOR_SHUFFLEv16i8()
5173 DAG.getConstant(0xFF00, MVT::i16)); in LowerVECTOR_SHUFFLEv16i8()
5180 SDValue InsElt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, in LowerVECTOR_SHUFFLEv16i8()
5181 Elt0Src, DAG.getIntPtrConstant(Elt0 / 2)); in LowerVECTOR_SHUFFLEv16i8()
5183 InsElt0 = DAG.getNode(ISD::SRL, dl, MVT::i16, InsElt0, in LowerVECTOR_SHUFFLEv16i8()
5184 DAG.getConstant(8, in LowerVECTOR_SHUFFLEv16i8()
5187 InsElt0 = DAG.getNode(ISD::AND, dl, MVT::i16, InsElt0, in LowerVECTOR_SHUFFLEv16i8()
5188 DAG.getConstant(0x00FF, MVT::i16)); in LowerVECTOR_SHUFFLEv16i8()
5189 InsElt = Elt1 >= 0 ? DAG.getNode(ISD::OR, dl, MVT::i16, InsElt, InsElt0) in LowerVECTOR_SHUFFLEv16i8()
5192 NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, NewV, InsElt, in LowerVECTOR_SHUFFLEv16i8()
5193 DAG.getIntPtrConstant(i)); in LowerVECTOR_SHUFFLEv16i8()
5195 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, NewV); in LowerVECTOR_SHUFFLEv16i8()
5205 SelectionDAG &DAG, DebugLoc dl) { in RewriteAsNarrowerShuffle() argument
5239 V1 = DAG.getNode(ISD::BITCAST, dl, NewVT, V1); in RewriteAsNarrowerShuffle()
5240 V2 = DAG.getNode(ISD::BITCAST, dl, NewVT, V2); in RewriteAsNarrowerShuffle()
5241 return DAG.getVectorShuffle(NewVT, dl, V1, V2, &MaskVec[0]); in RewriteAsNarrowerShuffle()
5247 SDValue SrcOp, SelectionDAG &DAG, in getVZextMovL() argument
5263 return DAG.getNode(ISD::BITCAST, dl, VT, in getVZextMovL()
5264 DAG.getNode(X86ISD::VZEXT_MOVL, dl, OpVT, in getVZextMovL()
5265 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, in getVZextMovL()
5273 return DAG.getNode(ISD::BITCAST, dl, VT, in getVZextMovL()
5274 DAG.getNode(X86ISD::VZEXT_MOVL, dl, OpVT, in getVZextMovL()
5275 DAG.getNode(ISD::BITCAST, dl, in getVZextMovL()
5282 LowerVECTOR_SHUFFLE_4wide(ShuffleVectorSDNode *SVOp, SelectionDAG &DAG) { in LowerVECTOR_SHUFFLE_4wide() argument
5320 V1 = DAG.getVectorShuffle(VT, dl, V1, V2, &Mask1[0]); in LowerVECTOR_SHUFFLE_4wide()
5334 return DAG.getVectorShuffle(VT, dl, V1, V1, &Mask2[0]); in LowerVECTOR_SHUFFLE_4wide()
5363 V2 = DAG.getVectorShuffle(VT, dl, V1, V2, &Mask1[0]); in LowerVECTOR_SHUFFLE_4wide()
5370 return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask1[0]); in LowerVECTOR_SHUFFLE_4wide()
5380 return DAG.getVectorShuffle(VT, dl, V2, V1, &Mask1[0]); in LowerVECTOR_SHUFFLE_4wide()
5415 SDValue LoShuffle = DAG.getVectorShuffle(VT, dl, V1, V2, &LoMask[0]); in LowerVECTOR_SHUFFLE_4wide()
5416 SDValue HiShuffle = DAG.getVectorShuffle(VT, dl, V1, V2, &HiMask[0]); in LowerVECTOR_SHUFFLE_4wide()
5426 return DAG.getVectorShuffle(VT, dl, LoShuffle, HiShuffle, &MaskOps[0]); in LowerVECTOR_SHUFFLE_4wide()
5464 bool CanXFormVExtractWithShuffleIntoLoad(SDValue V, SelectionDAG &DAG, in CanXFormVExtractWithShuffleIntoLoad() argument
5523 VT.getTypeForEVT(*DAG.getContext())); in CanXFormVExtractWithShuffleIntoLoad()
5533 SDValue getMOVDDup(SDValue &Op, DebugLoc &dl, SDValue V1, SelectionDAG &DAG) { in getMOVDDup() argument
5537 V1 = DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, V1); in getMOVDDup()
5538 return DAG.getNode(ISD::BITCAST, dl, VT, in getMOVDDup()
5540 V1, DAG)); in getMOVDDup()
5544 SDValue getMOVLowToHigh(SDValue &Op, DebugLoc &dl, SelectionDAG &DAG, in getMOVLowToHigh() argument
5553 return getTargetShuffleNode(X86ISD::MOVLHPD, dl, VT, V1, V2, DAG); in getMOVLowToHigh()
5556 return getTargetShuffleNode(X86ISD::MOVLHPS, dl, VT, V1, V2, DAG); in getMOVLowToHigh()
5560 SDValue getMOVHighToLow(SDValue &Op, DebugLoc &dl, SelectionDAG &DAG) { in getMOVHighToLow() argument
5572 return getTargetShuffleNode(X86ISD::MOVHLPS, dl, VT, V1, V2, DAG); in getMOVHighToLow()
5576 SDValue getMOVLP(SDValue &Op, DebugLoc &dl, SelectionDAG &DAG, bool HasSSE2) { in getMOVLP() argument
5606 return getTargetShuffleNode(X86ISD::MOVLPD, dl, VT, V1, V2, DAG); in getMOVLP()
5609 return getTargetShuffleNode(X86ISD::MOVLPS, dl, VT, V1, V2, DAG); in getMOVLP()
5620 return getTargetShuffleNode(X86ISD::MOVSD, dl, VT, V1, V2, DAG); in getMOVLP()
5622 return getTargetShuffleNode(X86ISD::MOVSS, dl, VT, V1, V2, DAG); in getMOVLP()
5629 X86::getShuffleSHUFImmediate(SVOp), DAG); in getMOVLP()
5665 SDValue NormalizeVectorShuffle(SDValue Op, SelectionDAG &DAG, in NormalizeVectorShuffle() argument
5675 return getZeroVector(VT, Subtarget->hasSSE2(), DAG, dl); in NormalizeVectorShuffle()
5684 CanXFormVExtractWithShuffleIntoLoad(Op, DAG, TLI)) in NormalizeVectorShuffle()
5692 return PromoteSplat(SVOp, DAG); in NormalizeVectorShuffle()
5698 SDValue NewOp = RewriteAsNarrowerShuffle(SVOp, DAG, dl); in NormalizeVectorShuffle()
5700 return DAG.getNode(ISD::BITCAST, dl, VT, NewOp); in NormalizeVectorShuffle()
5705 SDValue NewOp = RewriteAsNarrowerShuffle(SVOp, DAG, dl); in NormalizeVectorShuffle()
5709 DAG, Subtarget, dl); in NormalizeVectorShuffle()
5712 SDValue NewOp = RewriteAsNarrowerShuffle(SVOp, DAG, dl); in NormalizeVectorShuffle()
5715 DAG, Subtarget, dl); in NormalizeVectorShuffle()
5722 X86TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const { in LowerVECTOR_SHUFFLE()
5737 MachineFunction &MF = DAG.getMachineFunction(); in LowerVECTOR_SHUFFLE()
5761 SDValue NewOp = NormalizeVectorShuffle(Op, DAG, *this, Subtarget); in LowerVECTOR_SHUFFLE()
5769 return getTargetShuffleNode(getUNPCKLOpcode(VT, getSubtarget()), dl, VT, V1, V1, DAG); in LowerVECTOR_SHUFFLE()
5772 return getTargetShuffleNode(getUNPCKHOpcode(VT), dl, VT, V1, V1, DAG); in LowerVECTOR_SHUFFLE()
5776 return getMOVDDup(Op, dl, V1, DAG); in LowerVECTOR_SHUFFLE()
5779 return getMOVHighToLow(Op, dl, DAG); in LowerVECTOR_SHUFFLE()
5784 return getTargetShuffleNode(getUNPCKHOpcode(VT), dl, VT, V1, V1, DAG); in LowerVECTOR_SHUFFLE()
5791 return getTargetShuffleNode(X86ISD::MOVLHPS, dl, VT, V1, V1, DAG); in LowerVECTOR_SHUFFLE()
5796 return getTargetShuffleNode(X86ISD::PSHUFD, dl, VT, V1, TargetMask, DAG); in LowerVECTOR_SHUFFLE()
5800 TargetMask, DAG); in LowerVECTOR_SHUFFLE()
5804 TargetMask, DAG); in LowerVECTOR_SHUFFLE()
5812 isVectorShift(SVOp, DAG, isLeft, ShVal, ShAmt); in LowerVECTOR_SHUFFLE()
5818 return getVShift(isLeft, VT, ShVal, ShAmt, DAG, *this, dl); in LowerVECTOR_SHUFFLE()
5825 return getVZextMovL(VT, VT, V2, DAG, Subtarget, dl); in LowerVECTOR_SHUFFLE()
5828 return getTargetShuffleNode(X86ISD::MOVSD, dl, VT, V1, V2, DAG); in LowerVECTOR_SHUFFLE()
5831 return getTargetShuffleNode(X86ISD::MOVSS, dl, VT, V1, V2, DAG); in LowerVECTOR_SHUFFLE()
5837 return getMOVLowToHigh(Op, dl, DAG, HasSSE2); in LowerVECTOR_SHUFFLE()
5840 return getMOVHighToLow(Op, dl, DAG); in LowerVECTOR_SHUFFLE()
5843 return getTargetShuffleNode(X86ISD::MOVSHDUP, dl, VT, V1, DAG); in LowerVECTOR_SHUFFLE()
5846 return getTargetShuffleNode(X86ISD::MOVSLDUP, dl, VT, V1, DAG); in LowerVECTOR_SHUFFLE()
5849 return getMOVLP(Op, dl, DAG, HasSSE2); in LowerVECTOR_SHUFFLE()
5853 return CommuteVectorShuffle(SVOp, DAG); in LowerVECTOR_SHUFFLE()
5859 return getVShift(isLeft, VT, ShVal, ShAmt, DAG, *this, dl); in LowerVECTOR_SHUFFLE()
5870 Op = CommuteVectorShuffle(SVOp, DAG); in LowerVECTOR_SHUFFLE()
5886 return getMOVL(DAG, dl, VT, V2, V1); in LowerVECTOR_SHUFFLE()
5891 dl, VT, V1, V2, DAG); in LowerVECTOR_SHUFFLE()
5894 return getTargetShuffleNode(getUNPCKHOpcode(VT), dl, VT, V1, V2, DAG); in LowerVECTOR_SHUFFLE()
5900 SDValue NewMask = NormalizeMask(SVOp, DAG); in LowerVECTOR_SHUFFLE()
5914 SDValue NewOp = CommuteVectorShuffle(SVOp, DAG); in LowerVECTOR_SHUFFLE()
5919 dl, VT, V2, V1, DAG); in LowerVECTOR_SHUFFLE()
5922 return getTargetShuffleNode(getUNPCKHOpcode(VT), dl, VT, V2, V1, DAG); in LowerVECTOR_SHUFFLE()
5927 return CommuteVectorShuffle(SVOp, DAG); in LowerVECTOR_SHUFFLE()
5938 DAG); in LowerVECTOR_SHUFFLE()
5945 return getTargetShuffleNode(Opcode, dl, VT, V1, V1, DAG); in LowerVECTOR_SHUFFLE()
5948 return getTargetShuffleNode(X86ISD::PUNPCKLQDQ, dl, VT, V1, V1, DAG); in LowerVECTOR_SHUFFLE()
5954 DAG); in LowerVECTOR_SHUFFLE()
5959 DAG); in LowerVECTOR_SHUFFLE()
5965 TargetMask, DAG); in LowerVECTOR_SHUFFLE()
5968 TargetMask, DAG); in LowerVECTOR_SHUFFLE()
5974 dl, VT, V1, V1, DAG); in LowerVECTOR_SHUFFLE()
5977 return getTargetShuffleNode(getUNPCKHOpcode(VT), dl, VT, V1, V1, DAG); in LowerVECTOR_SHUFFLE()
5981 SDValue NewOp = LowerVECTOR_SHUFFLEv8i16(Op, DAG); in LowerVECTOR_SHUFFLE()
5987 SDValue NewOp = LowerVECTOR_SHUFFLEv16i8(SVOp, DAG, *this); in LowerVECTOR_SHUFFLE()
5994 return LowerVECTOR_SHUFFLE_4wide(SVOp, DAG); in LowerVECTOR_SHUFFLE()
6001 SelectionDAG &DAG) const { in LowerEXTRACT_VECTOR_ELT_SSE4()
6005 SDValue Extract = DAG.getNode(X86ISD::PEXTRB, dl, MVT::i32, in LowerEXTRACT_VECTOR_ELT_SSE4()
6007 SDValue Assert = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Extract, in LowerEXTRACT_VECTOR_ELT_SSE4()
6008 DAG.getValueType(VT)); in LowerEXTRACT_VECTOR_ELT_SSE4()
6009 return DAG.getNode(ISD::TRUNCATE, dl, VT, Assert); in LowerEXTRACT_VECTOR_ELT_SSE4()
6014 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, in LowerEXTRACT_VECTOR_ELT_SSE4()
6015 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, in LowerEXTRACT_VECTOR_ELT_SSE4()
6016 DAG.getNode(ISD::BITCAST, dl, in LowerEXTRACT_VECTOR_ELT_SSE4()
6020 SDValue Extract = DAG.getNode(X86ISD::PEXTRW, dl, MVT::i32, in LowerEXTRACT_VECTOR_ELT_SSE4()
6022 SDValue Assert = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Extract, in LowerEXTRACT_VECTOR_ELT_SSE4()
6023 DAG.getValueType(VT)); in LowerEXTRACT_VECTOR_ELT_SSE4()
6024 return DAG.getNode(ISD::TRUNCATE, dl, VT, Assert); in LowerEXTRACT_VECTOR_ELT_SSE4()
6040 SDValue Extract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, in LowerEXTRACT_VECTOR_ELT_SSE4()
6041 DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, in LowerEXTRACT_VECTOR_ELT_SSE4()
6044 return DAG.getNode(ISD::BITCAST, dl, MVT::f32, Extract); in LowerEXTRACT_VECTOR_ELT_SSE4()
6056 SelectionDAG &DAG) const { in LowerEXTRACT_VECTOR_ELT()
6078 Vec = Extract128BitVector(Vec, Idx, DAG, dl); in LowerEXTRACT_VECTOR_ELT()
6083 ScaledIdx = DAG.getNode(ISD::SUB, dl, Idx.getValueType(), Idx, in LowerEXTRACT_VECTOR_ELT()
6084 DAG.getConstant(ExtractNumElems, in LowerEXTRACT_VECTOR_ELT()
6086 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, Op.getValueType(), Vec, in LowerEXTRACT_VECTOR_ELT()
6093 SDValue Res = LowerEXTRACT_VECTOR_ELT_SSE4(Op, DAG); in LowerEXTRACT_VECTOR_ELT()
6105 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, in LowerEXTRACT_VECTOR_ELT()
6106 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, in LowerEXTRACT_VECTOR_ELT()
6107 DAG.getNode(ISD::BITCAST, dl, in LowerEXTRACT_VECTOR_ELT()
6112 SDValue Extract = DAG.getNode(X86ISD::PEXTRW, dl, EltVT, in LowerEXTRACT_VECTOR_ELT()
6114 SDValue Assert = DAG.getNode(ISD::AssertZext, dl, EltVT, Extract, in LowerEXTRACT_VECTOR_ELT()
6115 DAG.getValueType(VT)); in LowerEXTRACT_VECTOR_ELT()
6116 return DAG.getNode(ISD::TRUNCATE, dl, VT, Assert); in LowerEXTRACT_VECTOR_ELT()
6125 SDValue Vec = DAG.getVectorShuffle(VVT, dl, Op.getOperand(0), in LowerEXTRACT_VECTOR_ELT()
6126 DAG.getUNDEF(VVT), Mask); in LowerEXTRACT_VECTOR_ELT()
6127 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Vec, in LowerEXTRACT_VECTOR_ELT()
6128 DAG.getIntPtrConstant(0)); in LowerEXTRACT_VECTOR_ELT()
6142 SDValue Vec = DAG.getVectorShuffle(VVT, dl, Op.getOperand(0), in LowerEXTRACT_VECTOR_ELT()
6143 DAG.getUNDEF(VVT), Mask); in LowerEXTRACT_VECTOR_ELT()
6144 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Vec, in LowerEXTRACT_VECTOR_ELT()
6145 DAG.getIntPtrConstant(0)); in LowerEXTRACT_VECTOR_ELT()
6153 SelectionDAG &DAG) const { in LowerINSERT_VECTOR_ELT_SSE4()
6175 N1 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, N1); in LowerINSERT_VECTOR_ELT_SSE4()
6177 N2 = DAG.getIntPtrConstant(cast<ConstantSDNode>(N2)->getZExtValue()); in LowerINSERT_VECTOR_ELT_SSE4()
6178 return DAG.getNode(Opc, dl, VT, N0, N1, N2); in LowerINSERT_VECTOR_ELT_SSE4()
6188 N2 = DAG.getIntPtrConstant(cast<ConstantSDNode>(N2)->getZExtValue() << 4); in LowerINSERT_VECTOR_ELT_SSE4()
6190 N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32, N1); in LowerINSERT_VECTOR_ELT_SSE4()
6191 return DAG.getNode(X86ISD::INSERTPS, dl, VT, N0, N1, N2); in LowerINSERT_VECTOR_ELT_SSE4()
6200 X86TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const { in LowerINSERT_VECTOR_ELT()
6220 SDValue SubN0 = Extract128BitVector(N0, N2, DAG, dl); in LowerINSERT_VECTOR_ELT()
6225 ScaledN2 = DAG.getNode(ISD::SUB, dl, N2.getValueType(), N2, in LowerINSERT_VECTOR_ELT()
6226 DAG.getConstant(NumElems / in LowerINSERT_VECTOR_ELT()
6229 Op = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, SubN0.getValueType(), SubN0, in LowerINSERT_VECTOR_ELT()
6234 return Insert128BitVector(N0, Op, N2, DAG, dl); in LowerINSERT_VECTOR_ELT()
6238 return LowerINSERT_VECTOR_ELT_SSE4(Op, DAG); in LowerINSERT_VECTOR_ELT()
6247 N1 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, N1); in LowerINSERT_VECTOR_ELT()
6249 N2 = DAG.getIntPtrConstant(cast<ConstantSDNode>(N2)->getZExtValue()); in LowerINSERT_VECTOR_ELT()
6250 return DAG.getNode(X86ISD::PINSRW, dl, VT, N0, N1, N2); in LowerINSERT_VECTOR_ELT()
6256 X86TargetLowering::LowerSCALAR_TO_VECTOR(SDValue Op, SelectionDAG &DAG) const { in LowerSCALAR_TO_VECTOR()
6257 LLVMContext *Context = DAG.getContext(); in LowerSCALAR_TO_VECTOR()
6269 Op = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT128, Op.getOperand(0)); in LowerSCALAR_TO_VECTOR()
6272 return Insert128BitVector(DAG.getNode(ISD::UNDEF, dl, OpVT), Op, in LowerSCALAR_TO_VECTOR()
6273 DAG.getConstant(0, MVT::i32), in LowerSCALAR_TO_VECTOR()
6274 DAG, dl); in LowerSCALAR_TO_VECTOR()
6279 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v1i64, Op.getOperand(0)); in LowerSCALAR_TO_VECTOR()
6281 SDValue AnyExt = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, Op.getOperand(0)); in LowerSCALAR_TO_VECTOR()
6284 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), in LowerSCALAR_TO_VECTOR()
6285 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32,AnyExt)); in LowerSCALAR_TO_VECTOR()
6292 X86TargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const { in LowerEXTRACT_SUBVECTOR()
6300 return Extract128BitVector(Vec, Idx, DAG, dl); in LowerEXTRACT_SUBVECTOR()
6310 X86TargetLowering::LowerINSERT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const { in LowerINSERT_SUBVECTOR()
6319 return Insert128BitVector(Vec, SubVec, Idx, DAG, dl); in LowerINSERT_SUBVECTOR()
6332 X86TargetLowering::LowerConstantPool(SDValue Op, SelectionDAG &DAG) const { in LowerConstantPool()
6349 SDValue Result = DAG.getTargetConstantPool(CP->getConstVal(), getPointerTy(), in LowerConstantPool()
6353 Result = DAG.getNode(WrapperKind, DL, getPointerTy(), Result); in LowerConstantPool()
6356 Result = DAG.getNode(ISD::ADD, DL, getPointerTy(), in LowerConstantPool()
6357 DAG.getNode(X86ISD::GlobalBaseReg, in LowerConstantPool()
6365 SDValue X86TargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const { in LowerJumpTable()
6382 SDValue Result = DAG.getTargetJumpTable(JT->getIndex(), getPointerTy(), in LowerJumpTable()
6385 Result = DAG.getNode(WrapperKind, DL, getPointerTy(), Result); in LowerJumpTable()
6389 Result = DAG.getNode(ISD::ADD, DL, getPointerTy(), in LowerJumpTable()
6390 DAG.getNode(X86ISD::GlobalBaseReg, in LowerJumpTable()
6398 X86TargetLowering::LowerExternalSymbol(SDValue Op, SelectionDAG &DAG) const { in LowerExternalSymbol()
6415 SDValue Result = DAG.getTargetExternalSymbol(Sym, getPointerTy(), OpFlag); in LowerExternalSymbol()
6418 Result = DAG.getNode(WrapperKind, DL, getPointerTy(), Result); in LowerExternalSymbol()
6424 Result = DAG.getNode(ISD::ADD, DL, getPointerTy(), in LowerExternalSymbol()
6425 DAG.getNode(X86ISD::GlobalBaseReg, in LowerExternalSymbol()
6434 X86TargetLowering::LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const { in LowerBlockAddress()
6441 SDValue Result = DAG.getBlockAddress(BA, getPointerTy(), in LowerBlockAddress()
6446 Result = DAG.getNode(X86ISD::WrapperRIP, dl, getPointerTy(), Result); in LowerBlockAddress()
6448 Result = DAG.getNode(X86ISD::Wrapper, dl, getPointerTy(), Result); in LowerBlockAddress()
6452 Result = DAG.getNode(ISD::ADD, dl, getPointerTy(), in LowerBlockAddress()
6453 DAG.getNode(X86ISD::GlobalBaseReg, dl, getPointerTy()), in LowerBlockAddress()
6463 SelectionDAG &DAG) const { in LowerGlobalAddress()
6473 Result = DAG.getTargetGlobalAddress(GV, dl, getPointerTy(), Offset); in LowerGlobalAddress()
6476 Result = DAG.getTargetGlobalAddress(GV, dl, getPointerTy(), 0, OpFlags); in LowerGlobalAddress()
6481 Result = DAG.getNode(X86ISD::WrapperRIP, dl, getPointerTy(), Result); in LowerGlobalAddress()
6483 Result = DAG.getNode(X86ISD::Wrapper, dl, getPointerTy(), Result); in LowerGlobalAddress()
6487 Result = DAG.getNode(ISD::ADD, dl, getPointerTy(), in LowerGlobalAddress()
6488 DAG.getNode(X86ISD::GlobalBaseReg, dl, getPointerTy()), in LowerGlobalAddress()
6495 Result = DAG.getLoad(getPointerTy(), dl, DAG.getEntryNode(), Result, in LowerGlobalAddress()
6501 Result = DAG.getNode(ISD::ADD, dl, getPointerTy(), Result, in LowerGlobalAddress()
6502 DAG.getConstant(Offset, getPointerTy())); in LowerGlobalAddress()
6508 X86TargetLowering::LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const { in LowerGlobalAddress()
6511 return LowerGlobalAddress(GV, Op.getDebugLoc(), Offset, DAG); in LowerGlobalAddress()
6515 GetTLSADDR(SelectionDAG &DAG, SDValue Chain, GlobalAddressSDNode *GA, in GetTLSADDR() argument
6518 MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo(); in GetTLSADDR()
6519 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); in GetTLSADDR()
6521 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl, in GetTLSADDR()
6527 Chain = DAG.getNode(X86ISD::TLSADDR, dl, NodeTys, Ops, 3); in GetTLSADDR()
6530 Chain = DAG.getNode(X86ISD::TLSADDR, dl, NodeTys, Ops, 2); in GetTLSADDR()
6537 return DAG.getCopyFromReg(Chain, dl, ReturnReg, PtrVT, Flag); in GetTLSADDR()
6542 LowerToTLSGeneralDynamicModel32(GlobalAddressSDNode *GA, SelectionDAG &DAG, in LowerToTLSGeneralDynamicModel32() argument
6546 SDValue Chain = DAG.getCopyToReg(DAG.getEntryNode(), dl, X86::EBX, in LowerToTLSGeneralDynamicModel32()
6547 DAG.getNode(X86ISD::GlobalBaseReg, in LowerToTLSGeneralDynamicModel32()
6551 return GetTLSADDR(DAG, Chain, GA, &InFlag, PtrVT, X86::EAX, X86II::MO_TLSGD); in LowerToTLSGeneralDynamicModel32()
6556 LowerToTLSGeneralDynamicModel64(GlobalAddressSDNode *GA, SelectionDAG &DAG, in LowerToTLSGeneralDynamicModel64() argument
6558 return GetTLSADDR(DAG, DAG.getEntryNode(), GA, NULL, PtrVT, in LowerToTLSGeneralDynamicModel64()
6564 static SDValue LowerToTLSExecModel(GlobalAddressSDNode *GA, SelectionDAG &DAG, in LowerToTLSExecModel() argument
6570 Value *Ptr = Constant::getNullValue(Type::getInt8PtrTy(*DAG.getContext(), in LowerToTLSExecModel()
6573 SDValue ThreadPointer = DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), in LowerToTLSExecModel()
6574 DAG.getIntPtrConstant(0), in LowerToTLSExecModel()
6594 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl, in LowerToTLSExecModel()
6597 SDValue Offset = DAG.getNode(WrapperKind, dl, PtrVT, TGA); in LowerToTLSExecModel()
6600 Offset = DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Offset, in LowerToTLSExecModel()
6605 return DAG.getNode(ISD::ADD, dl, PtrVT, ThreadPointer, Offset); in LowerToTLSExecModel()
6609 X86TargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const { in LowerGlobalTLSAddress()
6630 return LowerToTLSGeneralDynamicModel64(GA, DAG, getPointerTy()); in LowerGlobalTLSAddress()
6631 return LowerToTLSGeneralDynamicModel32(GA, DAG, getPointerTy()); in LowerGlobalTLSAddress()
6635 return LowerToTLSExecModel(GA, DAG, getPointerTy(), model, in LowerGlobalTLSAddress()
6653 SDValue Result = DAG.getTargetGlobalAddress(GA->getGlobal(), DL, in LowerGlobalTLSAddress()
6656 SDValue Offset = DAG.getNode(WrapperKind, DL, getPointerTy(), Result); in LowerGlobalTLSAddress()
6660 Offset = DAG.getNode(ISD::ADD, DL, getPointerTy(), in LowerGlobalTLSAddress()
6661 DAG.getNode(X86ISD::GlobalBaseReg, in LowerGlobalTLSAddress()
6667 SDValue Chain = DAG.getEntryNode(); in LowerGlobalTLSAddress()
6668 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); in LowerGlobalTLSAddress()
6670 Chain = DAG.getNode(X86ISD::TLSCALL, DL, NodeTys, Args, 2); in LowerGlobalTLSAddress()
6673 MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo(); in LowerGlobalTLSAddress()
6679 return DAG.getCopyFromReg(Chain, DL, Reg, getPointerTy()); in LowerGlobalTLSAddress()
6692 SDValue X86TargetLowering::LowerShiftParts(SDValue Op, SelectionDAG &DAG) const { in LowerShiftParts()
6701 SDValue Tmp1 = isSRA ? DAG.getNode(ISD::SRA, dl, VT, ShOpHi, in LowerShiftParts()
6702 DAG.getConstant(VTBits - 1, MVT::i8)) in LowerShiftParts()
6703 : DAG.getConstant(0, VT); in LowerShiftParts()
6707 Tmp2 = DAG.getNode(X86ISD::SHLD, dl, VT, ShOpHi, ShOpLo, ShAmt); in LowerShiftParts()
6708 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt); in LowerShiftParts()
6710 Tmp2 = DAG.getNode(X86ISD::SHRD, dl, VT, ShOpLo, ShOpHi, ShAmt); in LowerShiftParts()
6711 Tmp3 = DAG.getNode(isSRA ? ISD::SRA : ISD::SRL, dl, VT, ShOpHi, ShAmt); in LowerShiftParts()
6714 SDValue AndNode = DAG.getNode(ISD::AND, dl, MVT::i8, ShAmt, in LowerShiftParts()
6715 DAG.getConstant(VTBits, MVT::i8)); in LowerShiftParts()
6716 SDValue Cond = DAG.getNode(X86ISD::CMP, dl, MVT::i32, in LowerShiftParts()
6717 AndNode, DAG.getConstant(0, MVT::i8)); in LowerShiftParts()
6720 SDValue CC = DAG.getConstant(X86::COND_NE, MVT::i8); in LowerShiftParts()
6725 Hi = DAG.getNode(X86ISD::CMOV, dl, VT, Ops0, 4); in LowerShiftParts()
6726 Lo = DAG.getNode(X86ISD::CMOV, dl, VT, Ops1, 4); in LowerShiftParts()
6728 Lo = DAG.getNode(X86ISD::CMOV, dl, VT, Ops0, 4); in LowerShiftParts()
6729 Hi = DAG.getNode(X86ISD::CMOV, dl, VT, Ops1, 4); in LowerShiftParts()
6733 return DAG.getMergeValues(Ops, 2, dl); in LowerShiftParts()
6737 SelectionDAG &DAG) const { in LowerSINT_TO_FP()
6757 MachineFunction &MF = DAG.getMachineFunction(); in LowerSINT_TO_FP()
6759 SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); in LowerSINT_TO_FP()
6760 SDValue Chain = DAG.getStore(DAG.getEntryNode(), dl, Op.getOperand(0), in LowerSINT_TO_FP()
6764 return BuildFILD(Op, SrcVT, Chain, StackSlot, DAG); in LowerSINT_TO_FP()
6769 SelectionDAG &DAG) const { in BuildFILD()
6775 Tys = DAG.getVTList(MVT::f64, MVT::Other, MVT::Glue); in BuildFILD()
6777 Tys = DAG.getVTList(Op.getValueType(), MVT::Other); in BuildFILD()
6786 DAG.getMachineFunction() in BuildFILD()
6793 SDValue Ops[] = { Chain, StackSlot, DAG.getValueType(SrcVT) }; in BuildFILD()
6794 SDValue Result = DAG.getMemIntrinsicNode(useSSE ? X86ISD::FILD_FLAG : in BuildFILD()
6806 MachineFunction &MF = DAG.getMachineFunction(); in BuildFILD()
6809 SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); in BuildFILD()
6810 Tys = DAG.getVTList(MVT::Other); in BuildFILD()
6812 Chain, Result, StackSlot, DAG.getValueType(Op.getValueType()), InFlag in BuildFILD()
6815 DAG.getMachineFunction() in BuildFILD()
6819 Chain = DAG.getMemIntrinsicNode(X86ISD::FST, DL, Tys, in BuildFILD()
6822 Result = DAG.getLoad(Op.getValueType(), DL, Chain, StackSlot, in BuildFILD()
6832 SelectionDAG &DAG) const { in LowerUINT_TO_FP_i64()
6868 LLVMContext *Context = DAG.getContext(); in LowerUINT_TO_FP_i64()
6877 SDValue CPIdx0 = DAG.getConstantPool(C0, getPointerTy(), 16); in LowerUINT_TO_FP_i64()
6885 SDValue CPIdx1 = DAG.getConstantPool(C1, getPointerTy(), 16); in LowerUINT_TO_FP_i64()
6887 SDValue XR1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, in LowerUINT_TO_FP_i64()
6888 DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, in LowerUINT_TO_FP_i64()
6890 DAG.getIntPtrConstant(1))); in LowerUINT_TO_FP_i64()
6891 SDValue XR2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, in LowerUINT_TO_FP_i64()
6892 DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, in LowerUINT_TO_FP_i64()
6894 DAG.getIntPtrConstant(0))); in LowerUINT_TO_FP_i64()
6895 SDValue Unpck1 = getUnpackl(DAG, dl, MVT::v4i32, XR1, XR2); in LowerUINT_TO_FP_i64()
6896 SDValue CLod0 = DAG.getLoad(MVT::v4i32, dl, DAG.getEntryNode(), CPIdx0, in LowerUINT_TO_FP_i64()
6899 SDValue Unpck2 = getUnpackl(DAG, dl, MVT::v4i32, Unpck1, CLod0); in LowerUINT_TO_FP_i64()
6900 SDValue XR2F = DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, Unpck2); in LowerUINT_TO_FP_i64()
6901 SDValue CLod1 = DAG.getLoad(MVT::v2f64, dl, CLod0.getValue(1), CPIdx1, in LowerUINT_TO_FP_i64()
6904 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::v2f64, XR2F, CLod1); in LowerUINT_TO_FP_i64()
6908 SDValue Shuf = DAG.getVectorShuffle(MVT::v2f64, dl, Sub, in LowerUINT_TO_FP_i64()
6909 DAG.getUNDEF(MVT::v2f64), ShufMask); in LowerUINT_TO_FP_i64()
6910 SDValue Add = DAG.getNode(ISD::FADD, dl, MVT::v2f64, Shuf, Sub); in LowerUINT_TO_FP_i64()
6911 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Add, in LowerUINT_TO_FP_i64()
6912 DAG.getIntPtrConstant(0)); in LowerUINT_TO_FP_i64()
6917 SelectionDAG &DAG) const { in LowerUINT_TO_FP_i32()
6920 SDValue Bias = DAG.getConstantFP(BitsToDouble(0x4330000000000000ULL), in LowerUINT_TO_FP_i32()
6924 SDValue Load = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, in LowerUINT_TO_FP_i32()
6925 DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, in LowerUINT_TO_FP_i32()
6927 DAG.getIntPtrConstant(0))); in LowerUINT_TO_FP_i32()
6929 Load = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, in LowerUINT_TO_FP_i32()
6930 DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, Load), in LowerUINT_TO_FP_i32()
6931 DAG.getIntPtrConstant(0)); in LowerUINT_TO_FP_i32()
6934 SDValue Or = DAG.getNode(ISD::OR, dl, MVT::v2i64, in LowerUINT_TO_FP_i32()
6935 DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, in LowerUINT_TO_FP_i32()
6936 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, in LowerUINT_TO_FP_i32()
6938 DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, in LowerUINT_TO_FP_i32()
6939 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, in LowerUINT_TO_FP_i32()
6941 Or = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, in LowerUINT_TO_FP_i32()
6942 DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, Or), in LowerUINT_TO_FP_i32()
6943 DAG.getIntPtrConstant(0)); in LowerUINT_TO_FP_i32()
6946 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Or, Bias); in LowerUINT_TO_FP_i32()
6952 return DAG.getNode(ISD::FP_ROUND, dl, DestVT, Sub, in LowerUINT_TO_FP_i32()
6953 DAG.getIntPtrConstant(0)); in LowerUINT_TO_FP_i32()
6955 return DAG.getNode(ISD::FP_EXTEND, dl, DestVT, Sub); in LowerUINT_TO_FP_i32()
6963 SelectionDAG &DAG) const { in LowerUINT_TO_FP()
6970 if (DAG.SignBitIsZero(N0)) in LowerUINT_TO_FP()
6971 return DAG.getNode(ISD::SINT_TO_FP, dl, Op.getValueType(), N0); in LowerUINT_TO_FP()
6976 return LowerUINT_TO_FP_i64(Op, DAG); in LowerUINT_TO_FP()
6978 return LowerUINT_TO_FP_i32(Op, DAG); in LowerUINT_TO_FP()
6981 SDValue StackSlot = DAG.CreateStackTemporary(MVT::i64); in LowerUINT_TO_FP()
6983 SDValue WordOff = DAG.getConstant(4, getPointerTy()); in LowerUINT_TO_FP()
6984 SDValue OffsetSlot = DAG.getNode(ISD::ADD, dl, in LowerUINT_TO_FP()
6986 SDValue Store1 = DAG.getStore(DAG.getEntryNode(), dl, Op.getOperand(0), in LowerUINT_TO_FP()
6989 SDValue Store2 = DAG.getStore(Store1, dl, DAG.getConstant(0, MVT::i32), in LowerUINT_TO_FP()
6992 SDValue Fild = BuildFILD(Op, MVT::i64, Store2, StackSlot, DAG); in LowerUINT_TO_FP()
6997 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Op.getOperand(0), in LowerUINT_TO_FP()
7007 DAG.getMachineFunction() in LowerUINT_TO_FP()
7011 SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other); in LowerUINT_TO_FP()
7012 SDValue Ops[] = { Store, StackSlot, DAG.getValueType(MVT::i64) }; in LowerUINT_TO_FP()
7013 SDValue Fild = DAG.getMemIntrinsicNode(X86ISD::FILD, dl, Tys, Ops, 3, in LowerUINT_TO_FP()
7019 SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(MVT::i64), in LowerUINT_TO_FP()
7020 Op.getOperand(0), DAG.getConstant(0, MVT::i64), in LowerUINT_TO_FP()
7024 SDValue FudgePtr = DAG.getConstantPool( in LowerUINT_TO_FP()
7025 ConstantInt::get(*DAG.getContext(), FF.zext(64)), in LowerUINT_TO_FP()
7029 SDValue Zero = DAG.getIntPtrConstant(0); in LowerUINT_TO_FP()
7030 SDValue Four = DAG.getIntPtrConstant(4); in LowerUINT_TO_FP()
7031 SDValue Offset = DAG.getNode(ISD::SELECT, dl, Zero.getValueType(), SignSet, in LowerUINT_TO_FP()
7033 FudgePtr = DAG.getNode(ISD::ADD, dl, getPointerTy(), FudgePtr, Offset); in LowerUINT_TO_FP()
7037 SDValue Fudge = DAG.getExtLoad(ISD::EXTLOAD, dl, MVT::f80, DAG.getEntryNode(), in LowerUINT_TO_FP()
7041 SDValue Add = DAG.getNode(ISD::FADD, dl, MVT::f80, Fild, Fudge); in LowerUINT_TO_FP()
7042 return DAG.getNode(ISD::FP_ROUND, dl, DstVT, Add, DAG.getIntPtrConstant(0)); in LowerUINT_TO_FP()
7046 FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG, bool IsSigned) const { in FP_TO_INTHelper() argument
7071 MachineFunction &MF = DAG.getMachineFunction(); in FP_TO_INTHelper()
7074 SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); in FP_TO_INTHelper()
7086 SDValue Chain = DAG.getEntryNode(); in FP_TO_INTHelper()
7091 Chain = DAG.getStore(Chain, DL, Value, StackSlot, in FP_TO_INTHelper()
7094 SDVTList Tys = DAG.getVTList(Op.getOperand(0).getValueType(), MVT::Other); in FP_TO_INTHelper()
7096 Chain, StackSlot, DAG.getValueType(TheVT) in FP_TO_INTHelper()
7102 Value = DAG.getMemIntrinsicNode(X86ISD::FLD, DL, Tys, Ops, 3, in FP_TO_INTHelper()
7106 StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); in FP_TO_INTHelper()
7115 SDValue FIST = DAG.getMemIntrinsicNode(Opc, DL, DAG.getVTList(MVT::Other), in FP_TO_INTHelper()
7122 SelectionDAG &DAG) const { in LowerFP_TO_SINT()
7126 std::pair<SDValue,SDValue> Vals = FP_TO_INTHelper(Op, DAG, true); in LowerFP_TO_SINT()
7132 return DAG.getLoad(Op.getValueType(), Op.getDebugLoc(), in LowerFP_TO_SINT()
7137 SelectionDAG &DAG) const { in LowerFP_TO_UINT()
7138 std::pair<SDValue,SDValue> Vals = FP_TO_INTHelper(Op, DAG, false); in LowerFP_TO_UINT()
7143 return DAG.getLoad(Op.getValueType(), Op.getDebugLoc(), in LowerFP_TO_UINT()
7148 SelectionDAG &DAG) const { in LowerFABS()
7149 LLVMContext *Context = DAG.getContext(); in LowerFABS()
7168 SDValue CPIdx = DAG.getConstantPool(C, getPointerTy(), 16); in LowerFABS()
7169 SDValue Mask = DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx, in LowerFABS()
7172 return DAG.getNode(X86ISD::FAND, dl, VT, Op.getOperand(0), Mask); in LowerFABS()
7175 SDValue X86TargetLowering::LowerFNEG(SDValue Op, SelectionDAG &DAG) const { in LowerFNEG()
7176 LLVMContext *Context = DAG.getContext(); in LowerFNEG()
7195 SDValue CPIdx = DAG.getConstantPool(C, getPointerTy(), 16); in LowerFNEG()
7196 SDValue Mask = DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx, in LowerFNEG()
7200 return DAG.getNode(ISD::BITCAST, dl, VT, in LowerFNEG()
7201 DAG.getNode(ISD::XOR, dl, MVT::v2i64, in LowerFNEG()
7202 DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, in LowerFNEG()
7204 DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, Mask))); in LowerFNEG()
7206 return DAG.getNode(X86ISD::FXOR, dl, VT, Op.getOperand(0), Mask); in LowerFNEG()
7210 SDValue X86TargetLowering::LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const { in LowerFCOPYSIGN()
7211 LLVMContext *Context = DAG.getContext(); in LowerFCOPYSIGN()
7220 Op1 = DAG.getNode(ISD::FP_EXTEND, dl, VT, Op1); in LowerFCOPYSIGN()
7225 Op1 = DAG.getNode(ISD::FP_ROUND, dl, VT, Op1, DAG.getIntPtrConstant(1)); in LowerFCOPYSIGN()
7244 SDValue CPIdx = DAG.getConstantPool(C, getPointerTy(), 16); in LowerFCOPYSIGN()
7245 SDValue Mask1 = DAG.getLoad(SrcVT, dl, DAG.getEntryNode(), CPIdx, in LowerFCOPYSIGN()
7248 SDValue SignBit = DAG.getNode(X86ISD::FAND, dl, SrcVT, Op1, Mask1); in LowerFCOPYSIGN()
7253 SignBit = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2f64, SignBit); in LowerFCOPYSIGN()
7254 SignBit = DAG.getNode(X86ISD::FSRL, dl, MVT::v2f64, SignBit, in LowerFCOPYSIGN()
7255 DAG.getConstant(32, MVT::i32)); in LowerFCOPYSIGN()
7256 SignBit = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, SignBit); in LowerFCOPYSIGN()
7257 SignBit = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f32, SignBit, in LowerFCOPYSIGN()
7258 DAG.getIntPtrConstant(0)); in LowerFCOPYSIGN()
7273 CPIdx = DAG.getConstantPool(C, getPointerTy(), 16); in LowerFCOPYSIGN()
7274 SDValue Mask2 = DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx, in LowerFCOPYSIGN()
7277 SDValue Val = DAG.getNode(X86ISD::FAND, dl, VT, Op0, Mask2); in LowerFCOPYSIGN()
7280 return DAG.getNode(X86ISD::FOR, dl, VT, Val, SignBit); in LowerFCOPYSIGN()
7283 SDValue X86TargetLowering::LowerFGETSIGN(SDValue Op, SelectionDAG &DAG) const { in LowerFGETSIGN()
7289 SDValue xFGETSIGN = DAG.getNode(X86ISD::FGETSIGNx86, dl, VT, N0, in LowerFGETSIGN()
7290 DAG.getConstant(1, VT)); in LowerFGETSIGN()
7291 return DAG.getNode(ISD::AND, dl, VT, xFGETSIGN, DAG.getConstant(1, VT)); in LowerFGETSIGN()
7297 SelectionDAG &DAG) const { in EmitTest()
7322 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op, in EmitTest()
7323 DAG.getConstant(0, Op.getValueType())); in EmitTest()
7427 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op, in EmitTest()
7428 DAG.getConstant(0, Op.getValueType())); in EmitTest()
7430 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32); in EmitTest()
7435 SDValue New = DAG.getNode(Opcode, dl, VTs, &Ops[0], NumOperands); in EmitTest()
7436 DAG.ReplaceAllUsesWith(Op, New); in EmitTest()
7443 SelectionDAG &DAG) const { in EmitCmp()
7446 return EmitTest(Op0, X86CC, DAG); in EmitCmp()
7449 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op0, Op1); in EmitCmp()
7455 DebugLoc dl, SelectionDAG &DAG) const { in LowerToBT()
7475 DAG.ComputeMaskedBits(Op0, Mask, Zeros, Ones); in LowerToBT()
7499 LHS = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, LHS); in LowerToBT()
7504 RHS = DAG.getNode(ISD::ANY_EXTEND, dl, LHS.getValueType(), RHS); in LowerToBT()
7506 SDValue BT = DAG.getNode(X86ISD::BT, dl, MVT::i32, LHS, RHS); in LowerToBT()
7508 return DAG.getNode(X86ISD::SETCC, dl, MVT::i8, in LowerToBT()
7509 DAG.getConstant(Cond, MVT::i8), BT); in LowerToBT()
7515 SDValue X86TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const { in LowerSETCC()
7530 SDValue NewSetCC = LowerToBT(Op0, CC, dl, DAG); in LowerSETCC()
7551 return DAG.getNode(X86ISD::SETCC, dl, MVT::i8, in LowerSETCC()
7552 DAG.getConstant(CCode, MVT::i8), Op0.getOperand(1)); in LowerSETCC()
7557 unsigned X86CC = TranslateX86CC(CC, isFP, Op0, Op1, DAG); in LowerSETCC()
7561 SDValue EFLAGS = EmitCmp(Op0, Op1, X86CC, DAG); in LowerSETCC()
7562 return DAG.getNode(X86ISD::SETCC, dl, MVT::i8, in LowerSETCC()
7563 DAG.getConstant(X86CC, MVT::i8), EFLAGS); in LowerSETCC()
7566 SDValue X86TargetLowering::LowerVSETCC(SDValue Op, SelectionDAG &DAG) const { in LowerVSETCC()
7611 UNORD = DAG.getNode(Opc, dl, VT, Op0, Op1, DAG.getConstant(3, MVT::i8)); in LowerVSETCC()
7612 EQ = DAG.getNode(Opc, dl, VT, Op0, Op1, DAG.getConstant(0, MVT::i8)); in LowerVSETCC()
7613 return DAG.getNode(ISD::OR, dl, VT, UNORD, EQ); in LowerVSETCC()
7617 ORD = DAG.getNode(Opc, dl, VT, Op0, Op1, DAG.getConstant(7, MVT::i8)); in LowerVSETCC()
7618 NEQ = DAG.getNode(Opc, dl, VT, Op0, Op1, DAG.getConstant(4, MVT::i8)); in LowerVSETCC()
7619 return DAG.getNode(ISD::AND, dl, VT, ORD, NEQ); in LowerVSETCC()
7624 return DAG.getNode(Opc, dl, VT, Op0, Op1, DAG.getConstant(SSECC, MVT::i8)); in LowerVSETCC()
7661 SDValue SignBit = DAG.getConstant(APInt::getSignBit(EltVT.getSizeInBits()), in LowerVSETCC()
7664 SDValue SignVec = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &SignBits[0], in LowerVSETCC()
7666 Op0 = DAG.getNode(ISD::XOR, dl, VT, Op0, SignVec); in LowerVSETCC()
7667 Op1 = DAG.getNode(ISD::XOR, dl, VT, Op1, SignVec); in LowerVSETCC()
7670 SDValue Result = DAG.getNode(Opc, dl, VT, Op0, Op1); in LowerVSETCC()
7674 Result = DAG.getNOT(dl, Result, VT); in LowerVSETCC()
7714 SDValue X86TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const { in LowerSELECT()
7723 SDValue NewCond = LowerSETCC(Cond, DAG); in LowerSELECT()
7744 Cmp = DAG.getNode(X86ISD::CMP, DL, MVT::i32, in LowerSELECT()
7745 CmpOp0, DAG.getConstant(1, CmpOp0.getValueType())); in LowerSELECT()
7748 DAG.getNode(X86ISD::SETCC_CARRY, DL, Op.getValueType(), in LowerSELECT()
7749 DAG.getConstant(X86::COND_B, MVT::i8), Cmp); in LowerSELECT()
7752 Res = DAG.getNOT(DL, Res, Res.getValueType()); in LowerSELECT()
7756 Res = DAG.getNode(ISD::OR, DL, Res.getValueType(), Res, Y); in LowerSELECT()
7799 SDValue NewSetCC = LowerToBT(Cond, ISD::SETNE, DL, DAG); in LowerSELECT()
7809 CC = DAG.getConstant(X86::COND_NE, MVT::i8); in LowerSELECT()
7810 Cond = EmitTest(Cond, X86::COND_NE, DAG); in LowerSELECT()
7822 SDValue Res = DAG.getNode(X86ISD::SETCC_CARRY, DL, Op.getValueType(), in LowerSELECT()
7823 DAG.getConstant(X86::COND_B, MVT::i8), Cond); in LowerSELECT()
7825 return DAG.getNOT(DL, Res, Res.getValueType()); in LowerSELECT()
7832 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Glue); in LowerSELECT()
7834 return DAG.getNode(X86ISD::CMOV, DL, VTs, Ops, array_lengthof(Ops)); in LowerSELECT()
7863 SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const { in LowerBRCOND()
7872 SDValue NewCond = LowerSETCC(Cond, DAG); in LowerBRCOND()
7882 Cond = LowerXALUO(Cond, DAG); in LowerBRCOND()
7928 Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(), in LowerBRCOND()
7946 CC = DAG.getConstant(CCode, MVT::i8); in LowerBRCOND()
7954 DAG.UpdateNodeOperands(User, User->getOperand(0), Dest); in LowerBRCOND()
7959 Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(), in LowerBRCOND()
7964 CC = DAG.getConstant(CCode, MVT::i8); in LowerBRCOND()
7977 CC = DAG.getConstant(CCode, MVT::i8); in LowerBRCOND()
7991 SDValue NewSetCC = LowerToBT(Cond, ISD::SETNE, dl, DAG); in LowerBRCOND()
8001 CC = DAG.getConstant(X86::COND_NE, MVT::i8); in LowerBRCOND()
8002 Cond = EmitTest(Cond, X86::COND_NE, DAG); in LowerBRCOND()
8004 return DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(), in LowerBRCOND()
8016 SelectionDAG &DAG) const { in LowerDYNAMIC_STACKALLOC()
8032 Chain = DAG.getCopyToReg(Chain, dl, Reg, Size, Flag); in LowerDYNAMIC_STACKALLOC()
8035 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); in LowerDYNAMIC_STACKALLOC()
8037 Chain = DAG.getNode(X86ISD::WIN_ALLOCA, dl, NodeTys, Chain, Flag); in LowerDYNAMIC_STACKALLOC()
8040 Chain = DAG.getCopyFromReg(Chain, dl, X86StackPtr, SPTy).getValue(1); in LowerDYNAMIC_STACKALLOC()
8043 return DAG.getMergeValues(Ops1, 2, dl); in LowerDYNAMIC_STACKALLOC()
8046 SDValue X86TargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const { in LowerVASTART()
8047 MachineFunction &MF = DAG.getMachineFunction(); in LowerVASTART()
8056 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), in LowerVASTART()
8058 return DAG.getStore(Op.getOperand(0), DL, FR, Op.getOperand(1), in LowerVASTART()
8070 SDValue Store = DAG.getStore(Op.getOperand(0), DL, in LowerVASTART()
8071 DAG.getConstant(FuncInfo->getVarArgsGPOffset(), in LowerVASTART()
8077 FIN = DAG.getNode(ISD::ADD, DL, getPointerTy(), in LowerVASTART()
8078 FIN, DAG.getIntPtrConstant(4)); in LowerVASTART()
8079 Store = DAG.getStore(Op.getOperand(0), DL, in LowerVASTART()
8080 DAG.getConstant(FuncInfo->getVarArgsFPOffset(), in LowerVASTART()
8086 FIN = DAG.getNode(ISD::ADD, DL, getPointerTy(), in LowerVASTART()
8087 FIN, DAG.getIntPtrConstant(4)); in LowerVASTART()
8088 SDValue OVFIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), in LowerVASTART()
8090 Store = DAG.getStore(Op.getOperand(0), DL, OVFIN, FIN, in LowerVASTART()
8096 FIN = DAG.getNode(ISD::ADD, DL, getPointerTy(), in LowerVASTART()
8097 FIN, DAG.getIntPtrConstant(8)); in LowerVASTART()
8098 SDValue RSFIN = DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(), in LowerVASTART()
8100 Store = DAG.getStore(Op.getOperand(0), DL, RSFIN, FIN, in LowerVASTART()
8103 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, in LowerVASTART()
8107 SDValue X86TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const { in LowerVAARG()
8121 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); in LowerVAARG()
8141 !(DAG.getMachineFunction() in LowerVAARG()
8151 InstOps.push_back(DAG.getConstant(ArgSize, MVT::i32)); in LowerVAARG()
8152 InstOps.push_back(DAG.getConstant(ArgMode, MVT::i8)); in LowerVAARG()
8153 InstOps.push_back(DAG.getConstant(Align, MVT::i32)); in LowerVAARG()
8154 SDVTList VTs = DAG.getVTList(getPointerTy(), MVT::Other); in LowerVAARG()
8155 SDValue VAARG = DAG.getMemIntrinsicNode(X86ISD::VAARG_64, dl, in LowerVAARG()
8166 return DAG.getLoad(ArgVT, dl, in LowerVAARG()
8173 SDValue X86TargetLowering::LowerVACOPY(SDValue Op, SelectionDAG &DAG) const { in LowerVACOPY()
8183 return DAG.getMemcpy(Chain, DL, DstPtr, SrcPtr, in LowerVACOPY()
8184 DAG.getIntPtrConstant(24), 8, /*isVolatile*/false, in LowerVACOPY()
8190 X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const { in LowerINTRINSIC_WO_CHAIN()
8288 unsigned X86CC = TranslateX86CC(CC, true, LHS, RHS, DAG); in LowerINTRINSIC_WO_CHAIN()
8290 SDValue Cond = DAG.getNode(Opc, dl, MVT::i32, LHS, RHS); in LowerINTRINSIC_WO_CHAIN()
8291 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, in LowerINTRINSIC_WO_CHAIN()
8292 DAG.getConstant(X86CC, MVT::i8), Cond); in LowerINTRINSIC_WO_CHAIN()
8293 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC); in LowerINTRINSIC_WO_CHAIN()
8355 SDValue Test = DAG.getNode(TestOpc, dl, MVT::i32, LHS, RHS); in LowerINTRINSIC_WO_CHAIN()
8356 SDValue CC = DAG.getConstant(X86CC, MVT::i8); in LowerINTRINSIC_WO_CHAIN()
8357 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, CC, Test); in LowerINTRINSIC_WO_CHAIN()
8358 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC); in LowerINTRINSIC_WO_CHAIN()
8448 ShOps[1] = DAG.getConstant(0, MVT::i32); in LowerINTRINSIC_WO_CHAIN()
8450 ShOps[2] = DAG.getUNDEF(MVT::i32); in LowerINTRINSIC_WO_CHAIN()
8451 ShOps[3] = DAG.getUNDEF(MVT::i32); in LowerINTRINSIC_WO_CHAIN()
8452 ShAmt = DAG.getNode(ISD::BUILD_VECTOR, dl, ShAmtVT, &ShOps[0], 4); in LowerINTRINSIC_WO_CHAIN()
8454 ShAmt = DAG.getNode(ISD::BUILD_VECTOR, dl, ShAmtVT, &ShOps[0], 2); in LowerINTRINSIC_WO_CHAIN()
8459 ShAmt = DAG.getNode(ISD::BITCAST, dl, VT, ShAmt); in LowerINTRINSIC_WO_CHAIN()
8460 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, in LowerINTRINSIC_WO_CHAIN()
8461 DAG.getConstant(NewIntNo, MVT::i32), in LowerINTRINSIC_WO_CHAIN()
8468 SelectionDAG &DAG) const { in LowerRETURNADDR()
8469 MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo(); in LowerRETURNADDR()
8476 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG); in LowerRETURNADDR()
8478 DAG.getConstant(TD->getPointerSize(), in LowerRETURNADDR()
8480 return DAG.getLoad(getPointerTy(), dl, DAG.getEntryNode(), in LowerRETURNADDR()
8481 DAG.getNode(ISD::ADD, dl, getPointerTy(), in LowerRETURNADDR()
8487 SDValue RetAddrFI = getReturnAddressFrameIndex(DAG); in LowerRETURNADDR()
8488 return DAG.getLoad(getPointerTy(), dl, DAG.getEntryNode(), in LowerRETURNADDR()
8492 SDValue X86TargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const { in LowerFRAMEADDR()
8493 MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo(); in LowerFRAMEADDR()
8500 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, VT); in LowerFRAMEADDR()
8502 FrameAddr = DAG.getLoad(VT, dl, DAG.getEntryNode(), FrameAddr, in LowerFRAMEADDR()
8509 SelectionDAG &DAG) const { in LowerFRAME_TO_ARGS_OFFSET()
8510 return DAG.getIntPtrConstant(2*TD->getPointerSize()); in LowerFRAME_TO_ARGS_OFFSET()
8513 SDValue X86TargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const { in LowerEH_RETURN()
8514 MachineFunction &MF = DAG.getMachineFunction(); in LowerEH_RETURN()
8520 SDValue Frame = DAG.getCopyFromReg(DAG.getEntryNode(), dl, in LowerEH_RETURN()
8525 SDValue StoreAddr = DAG.getNode(ISD::ADD, dl, getPointerTy(), Frame, in LowerEH_RETURN()
8526 DAG.getIntPtrConstant(TD->getPointerSize())); in LowerEH_RETURN()
8527 StoreAddr = DAG.getNode(ISD::ADD, dl, getPointerTy(), StoreAddr, Offset); in LowerEH_RETURN()
8528 Chain = DAG.getStore(Chain, dl, Handler, StoreAddr, MachinePointerInfo(), in LowerEH_RETURN()
8530 Chain = DAG.getCopyToReg(Chain, dl, StoreAddrReg, StoreAddr); in LowerEH_RETURN()
8533 return DAG.getNode(X86ISD::EH_RETURN, dl, in LowerEH_RETURN()
8535 Chain, DAG.getRegister(StoreAddrReg, getPointerTy())); in LowerEH_RETURN()
8539 SelectionDAG &DAG) const { in LowerTRAMPOLINE()
8563 OutChains[0] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, MVT::i16), in LowerTRAMPOLINE()
8567 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, in LowerTRAMPOLINE()
8568 DAG.getConstant(2, MVT::i64)); in LowerTRAMPOLINE()
8569 OutChains[1] = DAG.getStore(Root, dl, FPtr, Addr, in LowerTRAMPOLINE()
8576 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, in LowerTRAMPOLINE()
8577 DAG.getConstant(10, MVT::i64)); in LowerTRAMPOLINE()
8578 OutChains[2] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, MVT::i16), in LowerTRAMPOLINE()
8582 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, in LowerTRAMPOLINE()
8583 DAG.getConstant(12, MVT::i64)); in LowerTRAMPOLINE()
8584 OutChains[3] = DAG.getStore(Root, dl, Nest, Addr, in LowerTRAMPOLINE()
8590 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, in LowerTRAMPOLINE()
8591 DAG.getConstant(20, MVT::i64)); in LowerTRAMPOLINE()
8592 OutChains[4] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, MVT::i16), in LowerTRAMPOLINE()
8597 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, in LowerTRAMPOLINE()
8598 DAG.getConstant(22, MVT::i64)); in LowerTRAMPOLINE()
8599 OutChains[5] = DAG.getStore(Root, dl, DAG.getConstant(ModRM, MVT::i8), Addr, in LowerTRAMPOLINE()
8604 { Trmp, DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains, 6) }; in LowerTRAMPOLINE()
8605 return DAG.getMergeValues(Ops, 2, dl); in LowerTRAMPOLINE()
8654 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp, in LowerTRAMPOLINE()
8655 DAG.getConstant(10, MVT::i32)); in LowerTRAMPOLINE()
8656 Disp = DAG.getNode(ISD::SUB, dl, MVT::i32, FPtr, Addr); in LowerTRAMPOLINE()
8661 OutChains[0] = DAG.getStore(Root, dl, in LowerTRAMPOLINE()
8662 DAG.getConstant(MOV32ri|N86Reg, MVT::i8), in LowerTRAMPOLINE()
8666 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp, in LowerTRAMPOLINE()
8667 DAG.getConstant(1, MVT::i32)); in LowerTRAMPOLINE()
8668 OutChains[1] = DAG.getStore(Root, dl, Nest, Addr, in LowerTRAMPOLINE()
8673 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp, in LowerTRAMPOLINE()
8674 DAG.getConstant(5, MVT::i32)); in LowerTRAMPOLINE()
8675 OutChains[2] = DAG.getStore(Root, dl, DAG.getConstant(JMP, MVT::i8), Addr, in LowerTRAMPOLINE()
8679 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp, in LowerTRAMPOLINE()
8680 DAG.getConstant(6, MVT::i32)); in LowerTRAMPOLINE()
8681 OutChains[3] = DAG.getStore(Root, dl, Disp, Addr, in LowerTRAMPOLINE()
8686 { Trmp, DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains, 4) }; in LowerTRAMPOLINE()
8687 return DAG.getMergeValues(Ops, 2, dl); in LowerTRAMPOLINE()
8692 SelectionDAG &DAG) const { in LowerFLT_ROUNDS_()
8712 MachineFunction &MF = DAG.getMachineFunction(); in LowerFLT_ROUNDS_()
8721 SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); in LowerFLT_ROUNDS_()
8728 SDValue Ops[] = { DAG.getEntryNode(), StackSlot }; in LowerFLT_ROUNDS_()
8729 SDValue Chain = DAG.getMemIntrinsicNode(X86ISD::FNSTCW16m, DL, in LowerFLT_ROUNDS_()
8730 DAG.getVTList(MVT::Other), in LowerFLT_ROUNDS_()
8734 SDValue CWD = DAG.getLoad(MVT::i16, DL, Chain, StackSlot, in LowerFLT_ROUNDS_()
8739 DAG.getNode(ISD::SRL, DL, MVT::i16, in LowerFLT_ROUNDS_()
8740 DAG.getNode(ISD::AND, DL, MVT::i16, in LowerFLT_ROUNDS_()
8741 CWD, DAG.getConstant(0x800, MVT::i16)), in LowerFLT_ROUNDS_()
8742 DAG.getConstant(11, MVT::i8)); in LowerFLT_ROUNDS_()
8744 DAG.getNode(ISD::SRL, DL, MVT::i16, in LowerFLT_ROUNDS_()
8745 DAG.getNode(ISD::AND, DL, MVT::i16, in LowerFLT_ROUNDS_()
8746 CWD, DAG.getConstant(0x400, MVT::i16)), in LowerFLT_ROUNDS_()
8747 DAG.getConstant(9, MVT::i8)); in LowerFLT_ROUNDS_()
8750 DAG.getNode(ISD::AND, DL, MVT::i16, in LowerFLT_ROUNDS_()
8751 DAG.getNode(ISD::ADD, DL, MVT::i16, in LowerFLT_ROUNDS_()
8752 DAG.getNode(ISD::OR, DL, MVT::i16, CWD1, CWD2), in LowerFLT_ROUNDS_()
8753 DAG.getConstant(1, MVT::i16)), in LowerFLT_ROUNDS_()
8754 DAG.getConstant(3, MVT::i16)); in LowerFLT_ROUNDS_()
8757 return DAG.getNode((VT.getSizeInBits() < 16 ? in LowerFLT_ROUNDS_()
8761 SDValue X86TargetLowering::LowerCTLZ(SDValue Op, SelectionDAG &DAG) const { in LowerCTLZ()
8771 Op = DAG.getNode(ISD::ZERO_EXTEND, dl, OpVT, Op); in LowerCTLZ()
8775 SDVTList VTs = DAG.getVTList(OpVT, MVT::i32); in LowerCTLZ()
8776 Op = DAG.getNode(X86ISD::BSR, dl, VTs, Op); in LowerCTLZ()
8781 DAG.getConstant(NumBits+NumBits-1, OpVT), in LowerCTLZ()
8782 DAG.getConstant(X86::COND_E, MVT::i8), in LowerCTLZ()
8785 Op = DAG.getNode(X86ISD::CMOV, dl, OpVT, Ops, array_lengthof(Ops)); in LowerCTLZ()
8788 Op = DAG.getNode(ISD::XOR, dl, OpVT, Op, DAG.getConstant(NumBits-1, OpVT)); in LowerCTLZ()
8791 Op = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Op); in LowerCTLZ()
8795 SDValue X86TargetLowering::LowerCTTZ(SDValue Op, SelectionDAG &DAG) const { in LowerCTTZ()
8804 Op = DAG.getNode(ISD::ZERO_EXTEND, dl, OpVT, Op); in LowerCTTZ()
8808 SDVTList VTs = DAG.getVTList(OpVT, MVT::i32); in LowerCTTZ()
8809 Op = DAG.getNode(X86ISD::BSF, dl, VTs, Op); in LowerCTTZ()
8814 DAG.getConstant(NumBits, OpVT), in LowerCTTZ()
8815 DAG.getConstant(X86::COND_E, MVT::i8), in LowerCTTZ()
8818 Op = DAG.getNode(X86ISD::CMOV, dl, OpVT, Ops, array_lengthof(Ops)); in LowerCTTZ()
8821 Op = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Op); in LowerCTTZ()
8825 SDValue X86TargetLowering::LowerMUL_V2I64(SDValue Op, SelectionDAG &DAG) const { in LowerMUL_V2I64()
8843 SDValue Ahi = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, in LowerMUL_V2I64()
8844 DAG.getConstant(Intrinsic::x86_sse2_psrli_q, MVT::i32), in LowerMUL_V2I64()
8845 A, DAG.getConstant(32, MVT::i32)); in LowerMUL_V2I64()
8846 SDValue Bhi = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, in LowerMUL_V2I64()
8847 DAG.getConstant(Intrinsic::x86_sse2_psrli_q, MVT::i32), in LowerMUL_V2I64()
8848 B, DAG.getConstant(32, MVT::i32)); in LowerMUL_V2I64()
8849 SDValue AloBlo = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, in LowerMUL_V2I64()
8850 DAG.getConstant(Intrinsic::x86_sse2_pmulu_dq, MVT::i32), in LowerMUL_V2I64()
8852 SDValue AloBhi = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, in LowerMUL_V2I64()
8853 DAG.getConstant(Intrinsic::x86_sse2_pmulu_dq, MVT::i32), in LowerMUL_V2I64()
8855 SDValue AhiBlo = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, in LowerMUL_V2I64()
8856 DAG.getConstant(Intrinsic::x86_sse2_pmulu_dq, MVT::i32), in LowerMUL_V2I64()
8858 AloBhi = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, in LowerMUL_V2I64()
8859 DAG.getConstant(Intrinsic::x86_sse2_pslli_q, MVT::i32), in LowerMUL_V2I64()
8860 AloBhi, DAG.getConstant(32, MVT::i32)); in LowerMUL_V2I64()
8861 AhiBlo = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, in LowerMUL_V2I64()
8862 DAG.getConstant(Intrinsic::x86_sse2_pslli_q, MVT::i32), in LowerMUL_V2I64()
8863 AhiBlo, DAG.getConstant(32, MVT::i32)); in LowerMUL_V2I64()
8864 SDValue Res = DAG.getNode(ISD::ADD, dl, VT, AloBlo, AloBhi); in LowerMUL_V2I64()
8865 Res = DAG.getNode(ISD::ADD, dl, VT, Res, AhiBlo); in LowerMUL_V2I64()
8869 SDValue X86TargetLowering::LowerShift(SDValue Op, SelectionDAG &DAG) const { in LowerShift()
8876 LLVMContext *Context = DAG.getContext(); in LowerShift()
8888 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, in LowerShift()
8889 DAG.getConstant(Intrinsic::x86_sse2_pslli_q, MVT::i32), in LowerShift()
8890 R, DAG.getConstant(ShiftAmt, MVT::i32)); in LowerShift()
8893 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, in LowerShift()
8894 DAG.getConstant(Intrinsic::x86_sse2_pslli_d, MVT::i32), in LowerShift()
8895 R, DAG.getConstant(ShiftAmt, MVT::i32)); in LowerShift()
8898 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, in LowerShift()
8899 DAG.getConstant(Intrinsic::x86_sse2_pslli_w, MVT::i32), in LowerShift()
8900 R, DAG.getConstant(ShiftAmt, MVT::i32)); in LowerShift()
8903 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, in LowerShift()
8904 DAG.getConstant(Intrinsic::x86_sse2_psrli_q, MVT::i32), in LowerShift()
8905 R, DAG.getConstant(ShiftAmt, MVT::i32)); in LowerShift()
8908 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, in LowerShift()
8909 DAG.getConstant(Intrinsic::x86_sse2_psrli_d, MVT::i32), in LowerShift()
8910 R, DAG.getConstant(ShiftAmt, MVT::i32)); in LowerShift()
8913 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, in LowerShift()
8914 DAG.getConstant(Intrinsic::x86_sse2_psrli_w, MVT::i32), in LowerShift()
8915 R, DAG.getConstant(ShiftAmt, MVT::i32)); in LowerShift()
8918 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, in LowerShift()
8919 DAG.getConstant(Intrinsic::x86_sse2_psrai_d, MVT::i32), in LowerShift()
8920 R, DAG.getConstant(ShiftAmt, MVT::i32)); in LowerShift()
8923 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, in LowerShift()
8924 DAG.getConstant(Intrinsic::x86_sse2_psrai_w, MVT::i32), in LowerShift()
8925 R, DAG.getConstant(ShiftAmt, MVT::i32)); in LowerShift()
8934 Op = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, in LowerShift()
8935 DAG.getConstant(Intrinsic::x86_sse2_pslli_d, MVT::i32), in LowerShift()
8936 Op.getOperand(1), DAG.getConstant(23, MVT::i32)); in LowerShift()
8942 SDValue CPIdx = DAG.getConstantPool(C, getPointerTy(), 16); in LowerShift()
8943 SDValue Addend = DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx, in LowerShift()
8947 Op = DAG.getNode(ISD::ADD, dl, VT, Op, Addend); in LowerShift()
8948 Op = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, Op); in LowerShift()
8949 Op = DAG.getNode(ISD::FP_TO_SINT, dl, VT, Op); in LowerShift()
8950 return DAG.getNode(ISD::MUL, dl, VT, Op, R); in LowerShift()
8954 Op = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, in LowerShift()
8955 DAG.getConstant(Intrinsic::x86_sse2_pslli_w, MVT::i32), in LowerShift()
8956 Op.getOperand(1), DAG.getConstant(5, MVT::i32)); in LowerShift()
8964 SDValue CPIdx = DAG.getConstantPool(C, getPointerTy(), 16); in LowerShift()
8965 SDValue M = DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx, in LowerShift()
8970 M = DAG.getNode(ISD::AND, dl, VT, R, M); in LowerShift()
8971 M = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, in LowerShift()
8972 DAG.getConstant(Intrinsic::x86_sse2_pslli_w, MVT::i32), M, in LowerShift()
8973 DAG.getConstant(4, MVT::i32)); in LowerShift()
8974 R = DAG.getNode(X86ISD::PBLENDVB, dl, VT, R, M, Op); in LowerShift()
8976 Op = DAG.getNode(ISD::ADD, dl, VT, Op, Op); in LowerShift()
8979 CPIdx = DAG.getConstantPool(C, getPointerTy(), 16); in LowerShift()
8980 M = DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx, in LowerShift()
8985 M = DAG.getNode(ISD::AND, dl, VT, R, M); in LowerShift()
8986 M = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, in LowerShift()
8987 DAG.getConstant(Intrinsic::x86_sse2_pslli_w, MVT::i32), M, in LowerShift()
8988 DAG.getConstant(2, MVT::i32)); in LowerShift()
8989 R = DAG.getNode(X86ISD::PBLENDVB, dl, VT, R, M, Op); in LowerShift()
8991 Op = DAG.getNode(ISD::ADD, dl, VT, Op, Op); in LowerShift()
8994 R = DAG.getNode(X86ISD::PBLENDVB, dl, VT, in LowerShift()
8995 R, DAG.getNode(ISD::ADD, dl, VT, R, R), Op); in LowerShift()
9001 SDValue X86TargetLowering::LowerXALUO(SDValue Op, SelectionDAG &DAG) const { in LowerXALUO()
9051 SDVTList VTs = DAG.getVTList(N->getValueType(0), N->getValueType(0), in LowerXALUO()
9053 SDValue Sum = DAG.getNode(X86ISD::UMUL, DL, VTs, LHS, RHS); in LowerXALUO()
9056 DAG.getNode(X86ISD::SETCC, DL, MVT::i8, in LowerXALUO()
9057 DAG.getConstant(X86::COND_O, MVT::i32), in LowerXALUO()
9060 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), SetCC); in LowerXALUO()
9066 SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::i32); in LowerXALUO()
9067 SDValue Sum = DAG.getNode(BaseOp, DL, VTs, LHS, RHS); in LowerXALUO()
9070 DAG.getNode(X86ISD::SETCC, DL, N->getValueType(1), in LowerXALUO()
9071 DAG.getConstant(Cond, MVT::i32), in LowerXALUO()
9074 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), SetCC); in LowerXALUO()
9078 SDValue X86TargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op, SelectionDAG &DAG) const{ in LowerSIGN_EXTEND_INREG()
9087 SDValue ShAmt = DAG.getConstant(BitsDiff, MVT::i32); in LowerSIGN_EXTEND_INREG()
9111 SDValue Tmp1 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, in LowerSIGN_EXTEND_INREG()
9112 DAG.getConstant(SHLIntrinsicsID, MVT::i32), in LowerSIGN_EXTEND_INREG()
9119 Tmp1 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, in LowerSIGN_EXTEND_INREG()
9120 DAG.getConstant(SRAIntrinsicsID, MVT::i32), in LowerSIGN_EXTEND_INREG()
9130 SDValue X86TargetLowering::LowerMEMBARRIER(SDValue Op, SelectionDAG &DAG) const{ in LowerMEMBARRIER()
9137 SDValue Zero = DAG.getConstant(0, MVT::i32); in LowerMEMBARRIER()
9139 DAG.getRegister(X86::ESP, MVT::i32), // Base in LowerMEMBARRIER()
9140 DAG.getTargetConstant(1, MVT::i8), // Scale in LowerMEMBARRIER()
9141 DAG.getRegister(0, MVT::i32), // Index in LowerMEMBARRIER()
9142 DAG.getTargetConstant(0, MVT::i32), // Disp in LowerMEMBARRIER()
9143 DAG.getRegister(0, MVT::i32), // Segment. in LowerMEMBARRIER()
9148 DAG.getMachineNode(X86::OR32mrLocked, dl, MVT::Other, Ops, in LowerMEMBARRIER()
9155 return DAG.getNode(X86ISD::MEMBARRIER, dl, MVT::Other, Op.getOperand(0)); in LowerMEMBARRIER()
9164 return DAG.getNode(X86ISD::SFENCE, dl, MVT::Other, Op.getOperand(0)); in LowerMEMBARRIER()
9168 return DAG.getNode(X86ISD::LFENCE, dl, MVT::Other, Op.getOperand(0)); in LowerMEMBARRIER()
9172 return DAG.getNode(X86ISD::MFENCE, dl, MVT::Other, Op.getOperand(0)); in LowerMEMBARRIER()
9175 SDValue X86TargetLowering::LowerCMP_SWAP(SDValue Op, SelectionDAG &DAG) const { in LowerCMP_SWAP()
9191 SDValue cpIn = DAG.getCopyToReg(Op.getOperand(0), DL, Reg, in LowerCMP_SWAP()
9196 DAG.getTargetConstant(size, MVT::i8), in LowerCMP_SWAP()
9198 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue); in LowerCMP_SWAP()
9200 SDValue Result = DAG.getMemIntrinsicNode(X86ISD::LCMPXCHG_DAG, DL, Tys, in LowerCMP_SWAP()
9203 DAG.getCopyFromReg(Result.getValue(0), DL, Reg, T, Result.getValue(1)); in LowerCMP_SWAP()
9208 SelectionDAG &DAG) const { in LowerREADCYCLECOUNTER()
9210 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue); in LowerREADCYCLECOUNTER()
9213 SDValue rd = DAG.getNode(X86ISD::RDTSC_DAG, dl, Tys, &TheChain, 1); in LowerREADCYCLECOUNTER()
9214 SDValue rax = DAG.getCopyFromReg(rd, dl, X86::RAX, MVT::i64, rd.getValue(1)); in LowerREADCYCLECOUNTER()
9215 SDValue rdx = DAG.getCopyFromReg(rax.getValue(1), dl, X86::RDX, MVT::i64, in LowerREADCYCLECOUNTER()
9217 SDValue Tmp = DAG.getNode(ISD::SHL, dl, MVT::i64, rdx, in LowerREADCYCLECOUNTER()
9218 DAG.getConstant(32, MVT::i8)); in LowerREADCYCLECOUNTER()
9220 DAG.getNode(ISD::OR, dl, MVT::i64, rax, Tmp), in LowerREADCYCLECOUNTER()
9223 return DAG.getMergeValues(Ops, 2, dl); in LowerREADCYCLECOUNTER()
9227 SelectionDAG &DAG) const { in LowerBITCAST()
9247 SDValue X86TargetLowering::LowerLOAD_SUB(SDValue Op, SelectionDAG &DAG) const { in LowerLOAD_SUB()
9251 SDValue negOp = DAG.getNode(ISD::SUB, dl, T, in LowerLOAD_SUB()
9252 DAG.getConstant(0, T), Node->getOperand(2)); in LowerLOAD_SUB()
9253 return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, dl, in LowerLOAD_SUB()
9261 static SDValue LowerADDC_ADDE_SUBC_SUBE(SDValue Op, SelectionDAG &DAG) { in LowerADDC_ADDE_SUBC_SUBE() argument
9265 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT)) in LowerADDC_ADDE_SUBC_SUBE()
9268 SDVTList VTs = DAG.getVTList(VT, MVT::i32); in LowerADDC_ADDE_SUBC_SUBE()
9281 return DAG.getNode(Opc, Op->getDebugLoc(), VTs, Op.getOperand(0), in LowerADDC_ADDE_SUBC_SUBE()
9283 return DAG.getNode(Opc, Op->getDebugLoc(), VTs, Op.getOperand(0), in LowerADDC_ADDE_SUBC_SUBE()
9289 SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { in LowerOperation()
9292 case ISD::SIGN_EXTEND_INREG: return LowerSIGN_EXTEND_INREG(Op,DAG); in LowerOperation()
9293 case ISD::MEMBARRIER: return LowerMEMBARRIER(Op,DAG); in LowerOperation()
9294 case ISD::ATOMIC_CMP_SWAP: return LowerCMP_SWAP(Op,DAG); in LowerOperation()
9295 case ISD::ATOMIC_LOAD_SUB: return LowerLOAD_SUB(Op,DAG); in LowerOperation()
9296 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG); in LowerOperation()
9297 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG); in LowerOperation()
9298 case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG); in LowerOperation()
9299 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG); in LowerOperation()
9300 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG); in LowerOperation()
9301 case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op, DAG); in LowerOperation()
9302 case ISD::INSERT_SUBVECTOR: return LowerINSERT_SUBVECTOR(Op, DAG); in LowerOperation()
9303 case ISD::SCALAR_TO_VECTOR: return LowerSCALAR_TO_VECTOR(Op, DAG); in LowerOperation()
9304 case ISD::ConstantPool: return LowerConstantPool(Op, DAG); in LowerOperation()
9305 case ISD::GlobalAddress: return LowerGlobalAddress(Op, DAG); in LowerOperation()
9306 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG); in LowerOperation()
9307 case ISD::ExternalSymbol: return LowerExternalSymbol(Op, DAG); in LowerOperation()
9308 case ISD::BlockAddress: return LowerBlockAddress(Op, DAG); in LowerOperation()
9311 case ISD::SRL_PARTS: return LowerShiftParts(Op, DAG); in LowerOperation()
9312 case ISD::SINT_TO_FP: return LowerSINT_TO_FP(Op, DAG); in LowerOperation()
9313 case ISD::UINT_TO_FP: return LowerUINT_TO_FP(Op, DAG); in LowerOperation()
9314 case ISD::FP_TO_SINT: return LowerFP_TO_SINT(Op, DAG); in LowerOperation()
9315 case ISD::FP_TO_UINT: return LowerFP_TO_UINT(Op, DAG); in LowerOperation()
9316 case ISD::FABS: return LowerFABS(Op, DAG); in LowerOperation()
9317 case ISD::FNEG: return LowerFNEG(Op, DAG); in LowerOperation()
9318 case ISD::FCOPYSIGN: return LowerFCOPYSIGN(Op, DAG); in LowerOperation()
9319 case ISD::FGETSIGN: return LowerFGETSIGN(Op, DAG); in LowerOperation()
9320 case ISD::SETCC: return LowerSETCC(Op, DAG); in LowerOperation()
9321 case ISD::VSETCC: return LowerVSETCC(Op, DAG); in LowerOperation()
9322 case ISD::SELECT: return LowerSELECT(Op, DAG); in LowerOperation()
9323 case ISD::BRCOND: return LowerBRCOND(Op, DAG); in LowerOperation()
9324 case ISD::JumpTable: return LowerJumpTable(Op, DAG); in LowerOperation()
9325 case ISD::VASTART: return LowerVASTART(Op, DAG); in LowerOperation()
9326 case ISD::VAARG: return LowerVAARG(Op, DAG); in LowerOperation()
9327 case ISD::VACOPY: return LowerVACOPY(Op, DAG); in LowerOperation()
9328 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG); in LowerOperation()
9329 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG); in LowerOperation()
9330 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG); in LowerOperation()
9332 return LowerFRAME_TO_ARGS_OFFSET(Op, DAG); in LowerOperation()
9333 case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG); in LowerOperation()
9334 case ISD::EH_RETURN: return LowerEH_RETURN(Op, DAG); in LowerOperation()
9335 case ISD::TRAMPOLINE: return LowerTRAMPOLINE(Op, DAG); in LowerOperation()
9336 case ISD::FLT_ROUNDS_: return LowerFLT_ROUNDS_(Op, DAG); in LowerOperation()
9337 case ISD::CTLZ: return LowerCTLZ(Op, DAG); in LowerOperation()
9338 case ISD::CTTZ: return LowerCTTZ(Op, DAG); in LowerOperation()
9339 case ISD::MUL: return LowerMUL_V2I64(Op, DAG); in LowerOperation()
9342 case ISD::SHL: return LowerShift(Op, DAG); in LowerOperation()
9348 case ISD::UMULO: return LowerXALUO(Op, DAG); in LowerOperation()
9349 case ISD::READCYCLECOUNTER: return LowerREADCYCLECOUNTER(Op, DAG); in LowerOperation()
9350 case ISD::BITCAST: return LowerBITCAST(Op, DAG); in LowerOperation()
9354 case ISD::SUBE: return LowerADDC_ADDE_SUBC_SUBE(Op, DAG); in LowerOperation()
9360 SelectionDAG &DAG, unsigned NewOp) const { in ReplaceATOMIC_BINARY_64() argument
9367 SDValue In2L = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, in ReplaceATOMIC_BINARY_64()
9368 Node->getOperand(2), DAG.getIntPtrConstant(0)); in ReplaceATOMIC_BINARY_64()
9369 SDValue In2H = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, in ReplaceATOMIC_BINARY_64()
9370 Node->getOperand(2), DAG.getIntPtrConstant(1)); in ReplaceATOMIC_BINARY_64()
9372 SDVTList Tys = DAG.getVTList(MVT::i32, MVT::i32, MVT::Other); in ReplaceATOMIC_BINARY_64()
9374 DAG.getMemIntrinsicNode(NewOp, dl, Tys, Ops, 4, MVT::i64, in ReplaceATOMIC_BINARY_64()
9377 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, OpsF, 2)); in ReplaceATOMIC_BINARY_64()
9385 SelectionDAG &DAG) const { in ReplaceNodeResults()
9400 FP_TO_INTHelper(SDValue(N, 0), DAG, true); in ReplaceNodeResults()
9405 Results.push_back(DAG.getLoad(VT, dl, FIST, StackSlot, in ReplaceNodeResults()
9411 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue); in ReplaceNodeResults()
9413 SDValue rd = DAG.getNode(X86ISD::RDTSC_DAG, dl, Tys, &TheChain, 1); in ReplaceNodeResults()
9414 SDValue eax = DAG.getCopyFromReg(rd, dl, X86::EAX, MVT::i32, in ReplaceNodeResults()
9416 SDValue edx = DAG.getCopyFromReg(eax.getValue(1), dl, X86::EDX, MVT::i32, in ReplaceNodeResults()
9420 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Ops, 2)); in ReplaceNodeResults()
9428 cpInL = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(2), in ReplaceNodeResults()
9429 DAG.getConstant(0, MVT::i32)); in ReplaceNodeResults()
9430 cpInH = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(2), in ReplaceNodeResults()
9431 DAG.getConstant(1, MVT::i32)); in ReplaceNodeResults()
9432 cpInL = DAG.getCopyToReg(N->getOperand(0), dl, X86::EAX, cpInL, SDValue()); in ReplaceNodeResults()
9433 cpInH = DAG.getCopyToReg(cpInL.getValue(0), dl, X86::EDX, cpInH, in ReplaceNodeResults()
9436 swapInL = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(3), in ReplaceNodeResults()
9437 DAG.getConstant(0, MVT::i32)); in ReplaceNodeResults()
9438 swapInH = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(3), in ReplaceNodeResults()
9439 DAG.getConstant(1, MVT::i32)); in ReplaceNodeResults()
9440 swapInL = DAG.getCopyToReg(cpInH.getValue(0), dl, X86::EBX, swapInL, in ReplaceNodeResults()
9442 swapInH = DAG.getCopyToReg(swapInL.getValue(0), dl, X86::ECX, swapInH, in ReplaceNodeResults()
9447 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue); in ReplaceNodeResults()
9449 SDValue Result = DAG.getMemIntrinsicNode(X86ISD::LCMPXCHG8_DAG, dl, Tys, in ReplaceNodeResults()
9451 SDValue cpOutL = DAG.getCopyFromReg(Result.getValue(0), dl, X86::EAX, in ReplaceNodeResults()
9453 SDValue cpOutH = DAG.getCopyFromReg(cpOutL.getValue(1), dl, X86::EDX, in ReplaceNodeResults()
9456 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, OpsF, 2)); in ReplaceNodeResults()
9461 ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMADD64_DAG); in ReplaceNodeResults()
9464 ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMAND64_DAG); in ReplaceNodeResults()
9467 ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMNAND64_DAG); in ReplaceNodeResults()
9470 ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMOR64_DAG); in ReplaceNodeResults()
9473 ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMSUB64_DAG); in ReplaceNodeResults()
9476 ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMXOR64_DAG); in ReplaceNodeResults()
9479 ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMSWAP64_DAG); in ReplaceNodeResults()
11045 const SelectionDAG &DAG, in computeMaskedBitsForTargetNode() argument
11109 static SDValue PerformShuffleCombine(SDNode *N, SelectionDAG &DAG, in PerformShuffleCombine() argument
11118 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in PerformShuffleCombine()
11124 Elts.push_back(getShuffleScalarElt(N, i, DAG, 0)); in PerformShuffleCombine()
11126 return EltsFromConsecutiveLoads(VT, Elts, dl, DAG); in PerformShuffleCombine()
11132 static SDValue PerformEXTRACT_VECTOR_ELTCombine(SDNode *N, SelectionDAG &DAG, in PerformEXTRACT_VECTOR_ELTCombine() argument
11180 SDValue StackPtr = DAG.CreateStackTemporary(InputVector.getValueType()); in PerformEXTRACT_VECTOR_ELTCombine()
11181 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, InputVector, StackPtr, in PerformEXTRACT_VECTOR_ELTCombine()
11194 SDValue OffsetVal = DAG.getConstant(Offset, TLI.getPointerTy()); in PerformEXTRACT_VECTOR_ELTCombine()
11196 SDValue ScalarAddr = DAG.getNode(ISD::ADD, dl, TLI.getPointerTy(), in PerformEXTRACT_VECTOR_ELTCombine()
11200 SDValue LoadScalar = DAG.getLoad(Extract->getValueType(0), dl, Ch, in PerformEXTRACT_VECTOR_ELTCombine()
11205 DAG.ReplaceAllUsesOfValueWith(SDValue(Extract, 0), LoadScalar); in PerformEXTRACT_VECTOR_ELTCombine()
11213 static SDValue PerformSELECTCombine(SDNode *N, SelectionDAG &DAG, in PerformSELECTCombine() argument
11232 if (DAG.isEqualTo(LHS, Cond.getOperand(0)) && in PerformSELECTCombine()
11233 DAG.isEqualTo(RHS, Cond.getOperand(1))) { in PerformSELECTCombine()
11240 if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS)) { in PerformSELECTCombine()
11242 !(DAG.isKnownNeverZero(LHS) || DAG.isKnownNeverZero(RHS))) in PerformSELECTCombine()
11252 !DAG.isKnownNeverZero(LHS) && !DAG.isKnownNeverZero(RHS)) in PerformSELECTCombine()
11270 !DAG.isKnownNeverZero(LHS) && !DAG.isKnownNeverZero(LHS)) in PerformSELECTCombine()
11278 if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS)) { in PerformSELECTCombine()
11280 !(DAG.isKnownNeverZero(LHS) || DAG.isKnownNeverZero(RHS))) in PerformSELECTCombine()
11297 } else if (DAG.isEqualTo(LHS, Cond.getOperand(1)) && in PerformSELECTCombine()
11298 DAG.isEqualTo(RHS, Cond.getOperand(0))) { in PerformSELECTCombine()
11306 !(DAG.isKnownNeverZero(LHS) || DAG.isKnownNeverZero(RHS))) { in PerformSELECTCombine()
11307 if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS)) in PerformSELECTCombine()
11316 (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS))) in PerformSELECTCombine()
11332 if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS)) in PerformSELECTCombine()
11341 !DAG.isKnownNeverZero(LHS) && !DAG.isKnownNeverZero(RHS)) { in PerformSELECTCombine()
11342 if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS)) in PerformSELECTCombine()
11361 return DAG.getNode(Opcode, DL, N->getValueType(0), LHS, RHS); in PerformSELECTCombine()
11369 if (DAG.getTargetLoweringInfo().isTypeLegal(LHS.getValueType())) { in PerformSELECTCombine()
11387 Cond = DAG.getNode(ISD::XOR, DL, Cond.getValueType(), Cond, in PerformSELECTCombine()
11388 DAG.getConstant(1, Cond.getValueType())); in PerformSELECTCombine()
11391 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, LHS.getValueType(), Cond); in PerformSELECTCombine()
11394 return DAG.getNode(ISD::SHL, DL, LHS.getValueType(), Cond, in PerformSELECTCombine()
11395 DAG.getConstant(ShAmt, MVT::i8)); in PerformSELECTCombine()
11401 Cond = DAG.getNode(ISD::XOR, DL, Cond.getValueType(), Cond, in PerformSELECTCombine()
11402 DAG.getConstant(1, Cond.getValueType())); in PerformSELECTCombine()
11405 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, in PerformSELECTCombine()
11407 return DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond, in PerformSELECTCombine()
11436 Cond = DAG.getNode(ISD::XOR, DL, Cond.getValueType(), Cond, in PerformSELECTCombine()
11437 DAG.getConstant(1, Cond.getValueType())); in PerformSELECTCombine()
11440 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, FalseC->getValueType(0), in PerformSELECTCombine()
11444 Cond = DAG.getNode(ISD::MUL, DL, Cond.getValueType(), Cond, in PerformSELECTCombine()
11445 DAG.getConstant(Diff, Cond.getValueType())); in PerformSELECTCombine()
11449 Cond = DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond, in PerformSELECTCombine()
11461 static SDValue PerformCMOVCombine(SDNode *N, SelectionDAG &DAG, in PerformCMOVCombine() argument
11479 if (DAG.isKnownNeverZero(Cond.getOperand(0))) in PerformCMOVCombine()
11500 Cond = DAG.getNode(X86ISD::SETCC, DL, MVT::i8, in PerformCMOVCombine()
11501 DAG.getConstant(CC, MVT::i8), Cond); in PerformCMOVCombine()
11504 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, TrueC->getValueType(0), Cond); in PerformCMOVCombine()
11507 Cond = DAG.getNode(ISD::SHL, DL, Cond.getValueType(), Cond, in PerformCMOVCombine()
11508 DAG.getConstant(ShAmt, MVT::i8)); in PerformCMOVCombine()
11517 Cond = DAG.getNode(X86ISD::SETCC, DL, MVT::i8, in PerformCMOVCombine()
11518 DAG.getConstant(CC, MVT::i8), Cond); in PerformCMOVCombine()
11521 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, in PerformCMOVCombine()
11523 Cond = DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond, in PerformCMOVCombine()
11555 Cond = DAG.getNode(X86ISD::SETCC, DL, MVT::i8, in PerformCMOVCombine()
11556 DAG.getConstant(CC, MVT::i8), Cond); in PerformCMOVCombine()
11558 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, FalseC->getValueType(0), in PerformCMOVCombine()
11562 Cond = DAG.getNode(ISD::MUL, DL, Cond.getValueType(), Cond, in PerformCMOVCombine()
11563 DAG.getConstant(Diff, Cond.getValueType())); in PerformCMOVCombine()
11567 Cond = DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond, in PerformCMOVCombine()
11583 static SDValue PerformMulCombine(SDNode *N, SelectionDAG &DAG, in PerformMulCombine() argument
11624 NewMul = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), in PerformMulCombine()
11625 DAG.getConstant(Log2_64(MulAmt1), MVT::i8)); in PerformMulCombine()
11627 NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0), in PerformMulCombine()
11628 DAG.getConstant(MulAmt1, VT)); in PerformMulCombine()
11631 NewMul = DAG.getNode(ISD::SHL, DL, VT, NewMul, in PerformMulCombine()
11632 DAG.getConstant(Log2_64(MulAmt2), MVT::i8)); in PerformMulCombine()
11634 NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, NewMul, in PerformMulCombine()
11635 DAG.getConstant(MulAmt2, VT)); in PerformMulCombine()
11643 static SDValue PerformSHLCombine(SDNode *N, SelectionDAG &DAG) { in PerformSHLCombine() argument
11662 return DAG.getNode(ISD::AND, N->getDebugLoc(), VT, in PerformSHLCombine()
11663 N00, DAG.getConstant(Mask, VT)); in PerformSHLCombine()
11672 static SDValue PerformShiftCombine(SDNode* N, SelectionDAG &DAG, in PerformShiftCombine() argument
11677 return PerformSHLCombine(N, DAG); in PerformShiftCombine()
11729 BaseShAmt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, ShAmtOp, in PerformShiftCombine()
11730 DAG.getIntPtrConstant(0)); in PerformShiftCombine()
11736 BaseShAmt = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, BaseShAmt); in PerformShiftCombine()
11738 BaseShAmt = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, BaseShAmt); in PerformShiftCombine()
11748 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, in PerformShiftCombine()
11749 DAG.getConstant(Intrinsic::x86_sse2_pslli_q, MVT::i32), in PerformShiftCombine()
11752 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, in PerformShiftCombine()
11753 DAG.getConstant(Intrinsic::x86_sse2_pslli_d, MVT::i32), in PerformShiftCombine()
11756 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, in PerformShiftCombine()
11757 DAG.getConstant(Intrinsic::x86_sse2_pslli_w, MVT::i32), in PerformShiftCombine()
11762 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, in PerformShiftCombine()
11763 DAG.getConstant(Intrinsic::x86_sse2_psrai_d, MVT::i32), in PerformShiftCombine()
11766 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, in PerformShiftCombine()
11767 DAG.getConstant(Intrinsic::x86_sse2_psrai_w, MVT::i32), in PerformShiftCombine()
11772 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, in PerformShiftCombine()
11773 DAG.getConstant(Intrinsic::x86_sse2_psrli_q, MVT::i32), in PerformShiftCombine()
11776 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, in PerformShiftCombine()
11777 DAG.getConstant(Intrinsic::x86_sse2_psrli_d, MVT::i32), in PerformShiftCombine()
11780 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, in PerformShiftCombine()
11781 DAG.getConstant(Intrinsic::x86_sse2_psrli_w, MVT::i32), in PerformShiftCombine()
11792 static SDValue CMPEQCombine(SDNode *N, SelectionDAG &DAG, in CMPEQCombine() argument
11852 SDValue OnesOrZeroesF = DAG.getNode(NTOperator, DL, MVT::f32, CMP00, CMP01, in CMPEQCombine()
11853 DAG.getConstant(x86cc, MVT::i8)); in CMPEQCombine()
11854 SDValue OnesOrZeroesI = DAG.getNode(ISD::BITCAST, DL, MVT::i32, in CMPEQCombine()
11856 SDValue ANDed = DAG.getNode(ISD::AND, DL, MVT::i32, OnesOrZeroesI, in CMPEQCombine()
11857 DAG.getConstant(1, MVT::i32)); in CMPEQCombine()
11858 SDValue OneBitOfTruth = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, ANDed); in CMPEQCombine()
11867 static SDValue PerformAndCombine(SDNode *N, SelectionDAG &DAG, in PerformAndCombine() argument
11873 SDValue R = CMPEQCombine(N, DAG, DCI, Subtarget); in PerformAndCombine()
11892 return DAG.getNode(X86ISD::ANDNP, DL, VT, N0.getOperand(0), N1); in PerformAndCombine()
11897 return DAG.getNode(X86ISD::ANDNP, DL, VT, N1.getOperand(0), N0); in PerformAndCombine()
11902 static SDValue PerformOrCombine(SDNode *N, SelectionDAG &DAG, in PerformOrCombine() argument
11908 SDValue R = CMPEQCombine(N, DAG, DCI, Subtarget); in PerformOrCombine()
11988 SDValue Sign = DAG.getNode(Opc, DL, MaskVT, X, Mask.getOperand(1)); in PerformOrCombine()
11989 return DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Sign); in PerformOrCombine()
11996 X = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, X); in PerformOrCombine()
11997 Y = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, Y); in PerformOrCombine()
11998 Mask = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, Mask); in PerformOrCombine()
11999 Mask = DAG.getNode(X86ISD::PBLENDVB, DL, MVT::v16i8, X, Y, Mask); in PerformOrCombine()
12000 return DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Mask); in PerformOrCombine()
12042 return DAG.getNode(Opc, DL, VT, in PerformOrCombine()
12044 DAG.getNode(ISD::TRUNCATE, DL, in PerformOrCombine()
12051 return DAG.getNode(Opc, DL, VT, in PerformOrCombine()
12053 DAG.getNode(ISD::TRUNCATE, DL, in PerformOrCombine()
12061 static SDValue PerformSTORECombine(SDNode *N, SelectionDAG &DAG, in PerformSTORECombine() argument
12074 const Function *F = DAG.getMachineFunction().getFunction(); in PerformSTORECombine()
12120 SDValue NewLd = DAG.getLoad(LdVT, LdDL, Ld->getChain(), Ld->getBasePtr(), in PerformSTORECombine()
12126 NewChain = DAG.getNode(ISD::TokenFactor, LdDL, MVT::Other, &Ops[0], in PerformSTORECombine()
12129 return DAG.getStore(NewChain, StDL, NewLd, St->getBasePtr(), in PerformSTORECombine()
12137 SDValue HiAddr = DAG.getNode(ISD::ADD, LdDL, MVT::i32, LoAddr, in PerformSTORECombine()
12138 DAG.getConstant(4, MVT::i32)); in PerformSTORECombine()
12140 SDValue LoLd = DAG.getLoad(MVT::i32, LdDL, Ld->getChain(), LoAddr, in PerformSTORECombine()
12144 SDValue HiLd = DAG.getLoad(MVT::i32, LdDL, Ld->getChain(), HiAddr, in PerformSTORECombine()
12153 NewChain = DAG.getNode(ISD::TokenFactor, LdDL, MVT::Other, &Ops[0], in PerformSTORECombine()
12158 HiAddr = DAG.getNode(ISD::ADD, StDL, MVT::i32, LoAddr, in PerformSTORECombine()
12159 DAG.getConstant(4, MVT::i32)); in PerformSTORECombine()
12161 SDValue LoSt = DAG.getStore(NewChain, StDL, LoLd, LoAddr, in PerformSTORECombine()
12165 SDValue HiSt = DAG.getStore(NewChain, StDL, HiLd, HiAddr, in PerformSTORECombine()
12170 return DAG.getNode(ISD::TokenFactor, StDL, MVT::Other, LoSt, HiSt); in PerformSTORECombine()
12177 static SDValue PerformFORCombine(SDNode *N, SelectionDAG &DAG) { in PerformFORCombine() argument
12191 static SDValue PerformFANDCombine(SDNode *N, SelectionDAG &DAG) { in PerformFANDCombine() argument
12204 SelectionDAG &DAG, in PerformBTCombine() argument
12212 TargetLowering::TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(), in PerformBTCombine()
12214 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in PerformBTCombine()
12222 static SDValue PerformVZEXT_MOVLCombine(SDNode *N, SelectionDAG &DAG) { in PerformVZEXT_MOVLCombine() argument
12230 return DAG.getNode(ISD::BITCAST, N->getDebugLoc(), VT, Op); in PerformVZEXT_MOVLCombine()
12235 static SDValue PerformZExtCombine(SDNode *N, SelectionDAG &DAG) { in PerformZExtCombine() argument
12252 return DAG.getNode(ISD::AND, dl, VT, in PerformZExtCombine()
12253 DAG.getNode(X86ISD::SETCC_CARRY, dl, VT, in PerformZExtCombine()
12255 DAG.getConstant(1, VT)); in PerformZExtCombine()
12262 static SDValue PerformSETCCCombine(SDNode *N, SelectionDAG &DAG) { in PerformSETCCCombine() argument
12271 return DAG.getNode(ISD::AND, DL, MVT::i8, in PerformSETCCCombine()
12272 DAG.getNode(X86ISD::SETCC_CARRY, DL, MVT::i8, in PerformSETCCCombine()
12273 DAG.getConstant(X86CC, MVT::i8), EFLAG), in PerformSETCCCombine()
12274 DAG.getConstant(1, MVT::i8)); in PerformSETCCCombine()
12279 static SDValue PerformSINT_TO_FPCombine(SDNode *N, SelectionDAG &DAG, in PerformSINT_TO_FPCombine() argument
12290 !DAG.getTargetLoweringInfo().isTypeLegal(VT)) { in PerformSINT_TO_FPCombine()
12292 Ld->getChain(), Op0, DAG); in PerformSINT_TO_FPCombine()
12293 DAG.ReplaceAllUsesOfValueWith(Op0.getValue(1), FILDChain.getValue(1)); in PerformSINT_TO_FPCombine()
12301 static SDValue PerformADCCombine(SDNode *N, SelectionDAG &DAG, in PerformADCCombine() argument
12313 SDValue CarryOut = DAG.getConstant(0, N->getValueType(1)); in PerformADCCombine()
12314 SDValue Res1 = DAG.getNode(ISD::AND, DL, VT, in PerformADCCombine()
12315 DAG.getNode(X86ISD::SETCC_CARRY, DL, VT, in PerformADCCombine()
12316 DAG.getConstant(X86::COND_B,MVT::i8), in PerformADCCombine()
12318 DAG.getConstant(1, VT)); in PerformADCCombine()
12329 static SDValue OptimizeConditonalInDecrement(SDNode *N, SelectionDAG &DAG) { in OptimizeConditonalInDecrement() argument
12352 SDValue NewCmp = DAG.getNode(X86ISD::CMP, DL, MVT::i32, CmpOp0, in OptimizeConditonalInDecrement()
12353 DAG.getConstant(1, CmpOp0.getValueType())); in OptimizeConditonalInDecrement()
12357 return DAG.getNode(N->getOpcode() == ISD::SUB ? X86ISD::ADC : X86ISD::SBB, in OptimizeConditonalInDecrement()
12359 DAG.getConstant(-1ULL, OtherVal.getValueType()), NewCmp); in OptimizeConditonalInDecrement()
12360 return DAG.getNode(N->getOpcode() == ISD::SUB ? X86ISD::SBB : X86ISD::ADC, in OptimizeConditonalInDecrement()
12362 DAG.getConstant(0, OtherVal.getValueType()), NewCmp); in OptimizeConditonalInDecrement()
12367 SelectionDAG &DAG = DCI.DAG; in PerformDAGCombine() local
12371 return PerformEXTRACT_VECTOR_ELTCombine(N, DAG, *this); in PerformDAGCombine()
12372 case ISD::SELECT: return PerformSELECTCombine(N, DAG, Subtarget); in PerformDAGCombine()
12373 case X86ISD::CMOV: return PerformCMOVCombine(N, DAG, DCI); in PerformDAGCombine()
12375 case ISD::SUB: return OptimizeConditonalInDecrement(N, DAG); in PerformDAGCombine()
12376 case X86ISD::ADC: return PerformADCCombine(N, DAG, DCI); in PerformDAGCombine()
12377 case ISD::MUL: return PerformMulCombine(N, DAG, DCI); in PerformDAGCombine()
12380 case ISD::SRL: return PerformShiftCombine(N, DAG, Subtarget); in PerformDAGCombine()
12381 case ISD::AND: return PerformAndCombine(N, DAG, DCI, Subtarget); in PerformDAGCombine()
12382 case ISD::OR: return PerformOrCombine(N, DAG, DCI, Subtarget); in PerformDAGCombine()
12383 case ISD::STORE: return PerformSTORECombine(N, DAG, Subtarget); in PerformDAGCombine()
12384 case ISD::SINT_TO_FP: return PerformSINT_TO_FPCombine(N, DAG, this); in PerformDAGCombine()
12386 case X86ISD::FOR: return PerformFORCombine(N, DAG); in PerformDAGCombine()
12387 case X86ISD::FAND: return PerformFANDCombine(N, DAG); in PerformDAGCombine()
12388 case X86ISD::BT: return PerformBTCombine(N, DAG, DCI); in PerformDAGCombine()
12389 case X86ISD::VZEXT_MOVL: return PerformVZEXT_MOVLCombine(N, DAG); in PerformDAGCombine()
12390 case ISD::ZERO_EXTEND: return PerformZExtCombine(N, DAG); in PerformDAGCombine()
12391 case X86ISD::SETCC: return PerformSETCCCombine(N, DAG); in PerformDAGCombine()
12418 case ISD::VECTOR_SHUFFLE: return PerformShuffleCombine(N, DAG, DCI); in PerformDAGCombine()
12817 SelectionDAG &DAG) const { in LowerAsmOperandForConstraint()
12829 Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType()); in LowerAsmOperandForConstraint()
12837 Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType()); in LowerAsmOperandForConstraint()
12845 Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType()); in LowerAsmOperandForConstraint()
12853 Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType()); in LowerAsmOperandForConstraint()
12861 if (ConstantInt::isValueValidForType(Type::getInt32Ty(*DAG.getContext()), in LowerAsmOperandForConstraint()
12864 Result = DAG.getTargetConstant(C->getSExtValue(), MVT::i64); in LowerAsmOperandForConstraint()
12875 if (ConstantInt::isValueValidForType(Type::getInt32Ty(*DAG.getContext()), in LowerAsmOperandForConstraint()
12877 Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType()); in LowerAsmOperandForConstraint()
12889 Result = DAG.getTargetConstant(CST->getSExtValue(), MVT::i64); in LowerAsmOperandForConstraint()
12934 Result = DAG.getTargetGlobalAddress(GV, Op.getDebugLoc(), in LowerAsmOperandForConstraint()
12944 return TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG); in LowerAsmOperandForConstraint()