Lines Matching +full:robust +full:- +full:predicates
1 //===- X86ISelDAGToDAG.cpp - A DAG pattern matching inst selector for X86 -===//
8 //===----------------------------------------------------------------------===//
13 //===----------------------------------------------------------------------===//
40 #define DEBUG_TYPE "x86-isel"
44 //===----------------------------------------------------------------------===//
46 //===----------------------------------------------------------------------===//
77 MCSym(nullptr), JT(-1), Align(0), SymbolFlags(X86II::MO_NO_FLAG) {} in X86ISelAddressMode()
81 MCSym != nullptr || JT != -1 || BlockAddr != nullptr; in hasSymbolicDisplacement()
89 /// Return true if this addressing mode is already RIP-relative.
94 return RegNode->getReg() == X86::RIP; in isRIPRelative()
108 Base_Reg.getNode()->dump(); in dump()
115 IndexReg.getNode()->dump(); in dump()
121 GV->dump(); in dump()
126 CP->dump(); in dump()
147 //===--------------------------------------------------------------------===//
148 /// ISel - X86-specific code to select X86 machine instructions for
169 return "X86 DAG->DAG Instruction Selection"; in getPassName()
186 return isInt<8>(cast<ConstantSDNode>(N)->getSExtValue()); in immSext8()
189 // True if the 64-bit immediate fits in a 32-bit sign-extended field.
191 uint64_t v = cast<ConstantSDNode>(N)->getZExtValue(); in i64immSExt32()
249 ? CurDAG->getTargetFrameIndex( in getAddressOperands()
251 TLI->getPointerTy(CurDAG->getDataLayout())) in getAddressOperands()
255 // These are 32-bit even in 64-bit mode since RIP-relative offset in getAddressOperands()
256 // is 32-bit. in getAddressOperands()
258 Disp = CurDAG->getTargetGlobalAddress(AM.GV, SDLoc(), in getAddressOperands()
262 Disp = CurDAG->getTargetConstantPool(AM.CP, MVT::i32, in getAddressOperands()
265 assert(!AM.Disp && "Non-zero displacement is ignored with ES."); in getAddressOperands()
266 Disp = CurDAG->getTargetExternalSymbol(AM.ES, MVT::i32, AM.SymbolFlags); in getAddressOperands()
268 assert(!AM.Disp && "Non-zero displacement is ignored with MCSym."); in getAddressOperands()
270 Disp = CurDAG->getMCSymbol(AM.MCSym, MVT::i32); in getAddressOperands()
271 } else if (AM.JT != -1) { in getAddressOperands()
272 assert(!AM.Disp && "Non-zero displacement is ignored with JT."); in getAddressOperands()
273 Disp = CurDAG->getTargetJumpTable(AM.JT, MVT::i32, AM.SymbolFlags); in getAddressOperands()
275 Disp = CurDAG->getTargetBlockAddress(AM.BlockAddr, MVT::i32, AM.Disp, in getAddressOperands()
278 Disp = CurDAG->getTargetConstant(AM.Disp, DL, MVT::i32); in getAddressOperands()
283 Segment = CurDAG->getRegister(0, MVT::i32); in getAddressOperands()
302 for (SDNode::use_iterator UI = N->use_begin(), in shouldAvoidImmediateInstFormsForSize()
303 UE = N->use_end(); (UI != UE) && (UseCount < 2); ++UI) { in shouldAvoidImmediateInstFormsForSize()
309 if (User->isMachineOpcode()) { in shouldAvoidImmediateInstFormsForSize()
315 if (User->getOpcode() == ISD::STORE && in shouldAvoidImmediateInstFormsForSize()
316 User->getOperand(1).getNode() == N) { in shouldAvoidImmediateInstFormsForSize()
327 if (User->getNumOperands() != 2) in shouldAvoidImmediateInstFormsForSize()
334 if (User->getOpcode() == X86ISD::ADD || in shouldAvoidImmediateInstFormsForSize()
335 User->getOpcode() == ISD::ADD || in shouldAvoidImmediateInstFormsForSize()
336 User->getOpcode() == X86ISD::SUB || in shouldAvoidImmediateInstFormsForSize()
337 User->getOpcode() == ISD::SUB) { in shouldAvoidImmediateInstFormsForSize()
340 SDValue OtherOp = User->getOperand(0); in shouldAvoidImmediateInstFormsForSize()
342 OtherOp = User->getOperand(1); in shouldAvoidImmediateInstFormsForSize()
346 if (OtherOp->getOpcode() == ISD::CopyFromReg && in shouldAvoidImmediateInstFormsForSize()
348 OtherOp->getOperand(1).getNode()))) in shouldAvoidImmediateInstFormsForSize()
349 if ((RegNode->getReg() == X86::ESP) || in shouldAvoidImmediateInstFormsForSize()
350 (RegNode->getReg() == X86::RSP)) in shouldAvoidImmediateInstFormsForSize()
364 return CurDAG->getTargetConstant(Imm, DL, MVT::i8); in getI8Imm()
369 return CurDAG->getTargetConstant(Imm, DL, MVT::i32); in getI32Imm()
377 /// Return a reference to the TargetMachine, casted to the target-specific
383 /// Return a reference to the TargetInstrInfo, casted to the target-specific
386 return Subtarget->getInstrInfo(); in getInstrInfo()
389 /// \brief Address-mode matching performs shift-of-and to and-of-shift
411 switch (U->getOpcode()) { in IsProfitableToFold()
424 SDValue Op1 = U->getOperand(1); in IsProfitableToFold()
426 // If the other operand is a 8-bit immediate we should fold the immediate in IsProfitableToFold()
437 if (Imm->getAPIntValue().isSignedIntN(8)) in IsProfitableToFold()
449 // FIXME: This is probably also true for non-TLS addresses. in IsProfitableToFold()
479 CurDAG->getNode(ISD::TokenFactor, SDLoc(Load), MVT::Other, Ops); in moveBelowOrigChain()
483 Ops.append(OrigChain->op_begin() + 1, OrigChain->op_end()); in moveBelowOrigChain()
484 CurDAG->UpdateNodeOperands(OrigChain.getNode(), Ops); in moveBelowOrigChain()
485 CurDAG->UpdateNodeOperands(Load.getNode(), Call.getOperand(0), in moveBelowOrigChain()
490 Ops.append(Call->op_begin() + 1, Call->op_end()); in moveBelowOrigChain()
491 CurDAG->UpdateNodeOperands(Call.getNode(), Ops); in moveBelowOrigChain()
508 LD->isVolatile() || in isCalleeLoad()
509 LD->getAddressingMode() != ISD::UNINDEXED || in isCalleeLoad()
510 LD->getExtensionType() != ISD::NON_EXTLOAD) in isCalleeLoad()
525 cast<MemSDNode>(Chain.getNode())->writeMem()) in isCalleeLoad()
537 // OptFor[Min]Size are used in pattern predicates that isel is matching. in PreprocessISelDAG()
538 OptForSize = MF->getFunction()->optForSize(); in PreprocessISelDAG()
539 OptForMinSize = MF->getFunction()->optForMinSize(); in PreprocessISelDAG()
542 for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(), in PreprocessISelDAG()
543 E = CurDAG->allnodes_end(); I != E; ) { in PreprocessISelDAG()
549 ((N->getOpcode() == X86ISD::CALL && !Subtarget->callRegIndirect()) || in PreprocessISelDAG()
550 (N->getOpcode() == X86ISD::TC_RETURN && in PreprocessISelDAG()
552 (Subtarget->is64Bit() || in PreprocessISelDAG()
563 /// / \-- in PreprocessISelDAG()
573 bool HasCallSeq = N->getOpcode() == X86ISD::CALL; in PreprocessISelDAG()
574 SDValue Chain = N->getOperand(0); in PreprocessISelDAG()
575 SDValue Load = N->getOperand(1); in PreprocessISelDAG()
590 // FIXME: This should only happen when not compiled with -O0. in PreprocessISelDAG()
591 if (N->getOpcode() != ISD::FP_ROUND && N->getOpcode() != ISD::FP_EXTEND) in PreprocessISelDAG()
594 MVT SrcVT = N->getOperand(0).getSimpleValueType(); in PreprocessISelDAG()
595 MVT DstVT = N->getSimpleValueType(0); in PreprocessISelDAG()
605 bool SrcIsSSE = X86Lowering->isScalarFPTypeInSSEReg(SrcVT); in PreprocessISelDAG()
606 bool DstIsSSE = X86Lowering->isScalarFPTypeInSSEReg(DstVT); in PreprocessISelDAG()
612 if (N->getOpcode() == ISD::FP_EXTEND) in PreprocessISelDAG()
614 // If this is a value-preserving FPStack truncation, it is a noop. in PreprocessISelDAG()
615 if (N->getConstantOperandVal(1)) in PreprocessISelDAG()
619 // Here we could have an FP stack truncation or an FPStack <-> SSE convert. in PreprocessISelDAG()
623 if (N->getOpcode() == ISD::FP_ROUND) in PreprocessISelDAG()
628 SDValue MemTmp = CurDAG->CreateStackTemporary(MemVT); in PreprocessISelDAG()
632 SDValue Store = CurDAG->getTruncStore(CurDAG->getEntryNode(), dl, in PreprocessISelDAG()
633 N->getOperand(0), in PreprocessISelDAG()
636 SDValue Result = CurDAG->getExtLoad(ISD::EXTLOAD, dl, DstVT, Store, MemTmp, in PreprocessISelDAG()
644 --I; in PreprocessISelDAG()
645 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Result); in PreprocessISelDAG()
650 CurDAG->DeleteNode(N); in PreprocessISelDAG()
657 if (Subtarget->isTargetCygMing()) { in emitSpecialCodeForMain()
659 auto &DL = CurDAG->getDataLayout(); in emitSpecialCodeForMain()
662 CLI.setChain(CurDAG->getRoot()) in emitSpecialCodeForMain()
663 .setCallee(CallingConv::C, Type::getVoidTy(*CurDAG->getContext()), in emitSpecialCodeForMain()
664 CurDAG->getExternalSymbol("__main", TLI->getPointerTy(DL)), in emitSpecialCodeForMain()
666 const TargetLowering &TLI = CurDAG->getTargetLoweringInfo(); in emitSpecialCodeForMain()
668 CurDAG->setRoot(Result.second); in emitSpecialCodeForMain()
674 if (const Function *Fn = MF->getFunction()) in EmitFunctionEntryCode()
675 if (Fn->hasExternalLinkage() && Fn->getName() == "main") in EmitFunctionEntryCode()
680 // On 64-bit platforms, we can run into an issue where a frame index in isDispSafeForFrameIndex()
683 // displacement fits into a 31-bit integer (which is only slightly more in isDispSafeForFrameIndex()
685 // a 32-bit integer), a 31-bit disp should always be safe. in isDispSafeForFrameIndex()
696 if (Subtarget->is64Bit()) { in foldOffsetIntoAddress()
712 SDValue Address = N->getOperand(1); in matchLoadInAddress()
714 // load gs:0 -> GS segment register. in matchLoadInAddress()
715 // load fs:0 -> FS segment register. in matchLoadInAddress()
718 // gs:0 (or fs:0 on X86-64) contains its own address. in matchLoadInAddress()
721 if (C->getSExtValue() == 0 && AM.Segment.getNode() == nullptr && in matchLoadInAddress()
722 Subtarget->isTargetGlibc()) in matchLoadInAddress()
723 switch (N->getPointerInfo().getAddrSpace()) { in matchLoadInAddress()
725 AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16); in matchLoadInAddress()
728 AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16); in matchLoadInAddress()
749 // Handle X86-64 rip-relative addresses. We check this before checking direct in matchWrapper()
750 // folding because RIP is preferable to non-RIP accesses. in matchWrapper()
751 if (Subtarget->is64Bit() && N.getOpcode() == X86ISD::WrapperRIP && in matchWrapper()
752 // Under X86-64 non-small code model, GV (and friends) are 64-bits, so in matchWrapper()
761 AM.GV = G->getGlobal(); in matchWrapper()
762 AM.SymbolFlags = G->getTargetFlags(); in matchWrapper()
763 if (foldOffsetIntoAddress(G->getOffset(), AM)) { in matchWrapper()
769 AM.CP = CP->getConstVal(); in matchWrapper()
770 AM.Align = CP->getAlignment(); in matchWrapper()
771 AM.SymbolFlags = CP->getTargetFlags(); in matchWrapper()
772 if (foldOffsetIntoAddress(CP->getOffset(), AM)) { in matchWrapper()
777 AM.ES = S->getSymbol(); in matchWrapper()
778 AM.SymbolFlags = S->getTargetFlags(); in matchWrapper()
780 AM.MCSym = S->getMCSymbol(); in matchWrapper()
782 AM.JT = J->getIndex(); in matchWrapper()
783 AM.SymbolFlags = J->getTargetFlags(); in matchWrapper()
786 AM.BlockAddr = BA->getBlockAddress(); in matchWrapper()
787 AM.SymbolFlags = BA->getTargetFlags(); in matchWrapper()
788 if (foldOffsetIntoAddress(BA->getOffset(), AM)) { in matchWrapper()
796 AM.setBaseReg(CurDAG->getRegister(X86::RIP, MVT::i64)); in matchWrapper()
801 // X86-32 always and X86-64 when in -mcmodel=small mode. In 64-bit in matchWrapper()
802 // mode, this only applies to a non-RIP-relative computation. in matchWrapper()
803 if (!Subtarget->is64Bit() || in matchWrapper()
806 "RIP-relative addressing already handled"); in matchWrapper()
808 AM.GV = G->getGlobal(); in matchWrapper()
809 AM.Disp += G->getOffset(); in matchWrapper()
810 AM.SymbolFlags = G->getTargetFlags(); in matchWrapper()
812 AM.CP = CP->getConstVal(); in matchWrapper()
813 AM.Align = CP->getAlignment(); in matchWrapper()
814 AM.Disp += CP->getOffset(); in matchWrapper()
815 AM.SymbolFlags = CP->getTargetFlags(); in matchWrapper()
817 AM.ES = S->getSymbol(); in matchWrapper()
818 AM.SymbolFlags = S->getTargetFlags(); in matchWrapper()
820 AM.MCSym = S->getMCSymbol(); in matchWrapper()
822 AM.JT = J->getIndex(); in matchWrapper()
823 AM.SymbolFlags = J->getTargetFlags(); in matchWrapper()
825 AM.BlockAddr = BA->getBlockAddress(); in matchWrapper()
826 AM.Disp += BA->getOffset(); in matchWrapper()
827 AM.SymbolFlags = BA->getTargetFlags(); in matchWrapper()
842 // Post-processing: Convert lea(,%reg,2) to lea(%reg,%reg), which has in matchAddress()
843 // a smaller encoding and avoids a scaled-index. in matchAddress()
851 // Post-processing: Convert foo to foo(%rip), even in non-PIC mode, in matchAddress()
855 Subtarget->is64Bit() && in matchAddress()
862 AM.Base_Reg = CurDAG->getRegister(X86::RIP, MVT::i64); in matchAddress()
907 if (N.getNode()->getNodeId() == -1 || in insertDAGNode()
908 N.getNode()->getNodeId() > Pos.getNode()->getNodeId()) { in insertDAGNode()
909 DAG.RepositionNode(Pos.getNode()->getIterator(), N.getNode()); in insertDAGNode()
910 N.getNode()->setNodeId(Pos.getNode()->getNodeId()); in insertDAGNode()
914 // Transform "(X >> (8-C1)) & (0xff << C1)" to "((X >> 8) & 0xff) << C1" if
915 // safe. This allows us to convert the shift and and into an h-register
927 int ScaleLog = 8 - Shift.getConstantOperandVal(1); in foldMaskAndShiftToExtract()
942 // a valid topological ordering as nothing is going to go back and re-sort in foldMaskAndShiftToExtract()
944 // essentially a pre-flattened and pre-sorted sequence of nodes. There is no in foldMaskAndShiftToExtract()
987 // a valid topological ordering as nothing is going to go back and re-sort in foldMaskedShiftToScaledMask()
989 // essentially a pre-flattened and pre-sorted sequence of nodes. There is no in foldMaskedShiftToScaledMask()
1053 MaskLZ -= (64 - X.getSimpleValueType().getSizeInBits()) + ShiftAmt; in foldMaskAndShiftToScale()
1063 unsigned ExtendBits = X.getSimpleValueType().getSizeInBits() - in foldMaskAndShiftToScale()
1065 // Assume that we'll replace the any-extend with a zero-extend, and in foldMaskAndShiftToScale()
1068 MaskLZ = ExtendBits > MaskLZ ? 0 : MaskLZ - ExtendBits; in foldMaskAndShiftToScale()
1094 // a valid topological ordering as nothing is going to go back and re-sort in foldMaskAndShiftToScale()
1096 // essentially a pre-flattened and pre-sorted sequence of nodes. There is no in foldMaskAndShiftToScale()
1122 // RIP relative addressing: %rip + 32-bit displacement! in matchAddressRecursively()
1127 if (!(AM.ES || AM.MCSym) && AM.JT != -1) in matchAddressRecursively()
1131 if (!foldOffsetIntoAddress(Cst->getSExtValue(), AM)) in matchAddressRecursively()
1142 AM.MCSym = ESNode->getMCSymbol(); in matchAddressRecursively()
1148 uint64_t Val = cast<ConstantSDNode>(N)->getSExtValue(); in matchAddressRecursively()
1168 (!Subtarget->is64Bit() || isDispSafeForFrameIndex(AM.Disp))) { in matchAddressRecursively()
1170 AM.Base_FrameIndex = cast<FrameIndexSDNode>(N)->getIndex(); in matchAddressRecursively()
1180 *CN = dyn_cast<ConstantSDNode>(N.getNode()->getOperand(1))) { in matchAddressRecursively()
1181 unsigned Val = CN->getZExtValue(); in matchAddressRecursively()
1184 // the base doesn't end up getting used, a post-processing step in matchAddressRecursively()
1188 SDValue ShVal = N.getNode()->getOperand(0); in matchAddressRecursively()
1193 if (CurDAG->isBaseWithConstantOffset(ShVal)) { in matchAddressRecursively()
1194 AM.IndexReg = ShVal.getNode()->getOperand(0); in matchAddressRecursively()
1196 cast<ConstantSDNode>(ShVal.getNode()->getOperand(1)); in matchAddressRecursively()
1197 uint64_t Disp = (uint64_t)AddVal->getSExtValue() << Val; in matchAddressRecursively()
1216 // We only handle up to 64-bit values here as those are what matter for in matchAddressRecursively()
1220 // The mask used for the transform is expected to be post-shift, but we in matchAddressRecursively()
1242 // X*[3,5,9] -> X+X*[2,4,8] in matchAddressRecursively()
1247 *CN = dyn_cast<ConstantSDNode>(N.getNode()->getOperand(1))) in matchAddressRecursively()
1248 if (CN->getZExtValue() == 3 || CN->getZExtValue() == 5 || in matchAddressRecursively()
1249 CN->getZExtValue() == 9) { in matchAddressRecursively()
1250 AM.Scale = unsigned(CN->getZExtValue())-1; in matchAddressRecursively()
1252 SDValue MulVal = N.getNode()->getOperand(0); in matchAddressRecursively()
1258 if (MulVal.getNode()->getOpcode() == ISD::ADD && MulVal.hasOneUse() && in matchAddressRecursively()
1259 isa<ConstantSDNode>(MulVal.getNode()->getOperand(1))) { in matchAddressRecursively()
1260 Reg = MulVal.getNode()->getOperand(0); in matchAddressRecursively()
1262 cast<ConstantSDNode>(MulVal.getNode()->getOperand(1)); in matchAddressRecursively()
1263 uint64_t Disp = AddVal->getSExtValue() * CN->getZExtValue(); in matchAddressRecursively()
1265 Reg = N.getNode()->getOperand(0); in matchAddressRecursively()
1267 Reg = N.getNode()->getOperand(0); in matchAddressRecursively()
1277 // Given A-B, if A can be completely folded into the address and in matchAddressRecursively()
1278 // the index field with the index field unused, use -B as the index. in matchAddressRecursively()
1281 // other uses, since it avoids a two-address sub instruction, however in matchAddressRecursively()
1290 if (matchAddressRecursively(N.getNode()->getOperand(0), AM, Depth+1)) { in matchAddressRecursively()
1301 SDValue RHS = Handle.getValue().getNode()->getOperand(1); in matchAddressRecursively()
1305 if (!RHS.getNode()->hasOneUse() || in matchAddressRecursively()
1306 RHS.getNode()->getOpcode() == ISD::CopyFromReg || in matchAddressRecursively()
1307 RHS.getNode()->getOpcode() == ISD::TRUNCATE || in matchAddressRecursively()
1308 RHS.getNode()->getOpcode() == ISD::ANY_EXTEND || in matchAddressRecursively()
1309 (RHS.getNode()->getOpcode() == ISD::ZERO_EXTEND && in matchAddressRecursively()
1310 RHS.getNode()->getOperand(0).getValueType() == MVT::i32)) in matchAddressRecursively()
1316 !AM.Base_Reg.getNode()->hasOneUse()) || in matchAddressRecursively()
1318 --Cost; in matchAddressRecursively()
1324 --Cost; in matchAddressRecursively()
1332 SDValue Zero = CurDAG->getConstant(0, dl, N.getValueType()); in matchAddressRecursively()
1333 SDValue Neg = CurDAG->getNode(ISD::SUB, dl, N.getValueType(), Zero, RHS); in matchAddressRecursively()
1351 // Example: (or (and x, 1), (shl y, 3)) --> (add (and x, 1), (shl y, 3)) in matchAddressRecursively()
1355 if (CurDAG->haveNoCommonBitsSet(N.getOperand(0), N.getOperand(1)) && in matchAddressRecursively()
1361 // Perform some heroic transforms on an and of a constant-count shift in matchAddressRecursively()
1371 // We only handle up to 64-bit values here as those are what matter for in matchAddressRecursively()
1428 unsigned AddrSpace = Mgs->getPointerInfo().getAddrSpace(); in selectVectorAddr()
1429 // AddrSpace 256 -> GS, 257 -> FS, 258 -> SS. in selectVectorAddr()
1431 AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16); in selectVectorAddr()
1433 AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16); in selectVectorAddr()
1435 AM.Segment = CurDAG->getRegister(X86::SS, MVT::i16); in selectVectorAddr()
1438 Base = Mgs->getBasePtr(); in selectVectorAddr()
1439 Index = Mgs->getIndex(); in selectVectorAddr()
1440 unsigned ScalarSize = Mgs->getValue().getValueType().getScalarSizeInBits(); in selectVectorAddr()
1445 assert(cast<ConstantSDNode>(Base)->isNullValue() && in selectVectorAddr()
1448 Base = CurDAG->getRegister(0, MVT::i32); in selectVectorAddr()
1453 Segment = CurDAG->getRegister(0, MVT::i32); in selectVectorAddr()
1454 Disp = CurDAG->getTargetConstant(0, DL, MVT::i32); in selectVectorAddr()
1473 Parent->getOpcode() != ISD::INTRINSIC_W_CHAIN && // unaligned loads, fixme in selectAddr()
1474 Parent->getOpcode() != ISD::INTRINSIC_VOID && // nontemporal stores in selectAddr()
1475 Parent->getOpcode() != X86ISD::TLSCALL && // Fixme in selectAddr()
1476 Parent->getOpcode() != X86ISD::EH_SJLJ_SETJMP && // setjmp in selectAddr()
1477 Parent->getOpcode() != X86ISD::EH_SJLJ_LONGJMP) { // longjmp in selectAddr()
1479 cast<MemSDNode>(Parent)->getPointerInfo().getAddrSpace(); in selectAddr()
1480 // AddrSpace 256 -> GS, 257 -> FS, 258 -> SS. in selectAddr()
1482 AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16); in selectAddr()
1484 AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16); in selectAddr()
1486 AM.Segment = CurDAG->getRegister(X86::SS, MVT::i16); in selectAddr()
1495 AM.Base_Reg = CurDAG->getRegister(0, VT); in selectAddr()
1499 AM.IndexReg = CurDAG->getRegister(0, VT); in selectAddr()
1524 if (!selectAddr(LD, LD->getBasePtr(), Base, Scale, Index, Disp, Segment)) in selectScalarSSELoad()
1532 if (N.getOpcode() == X86ISD::VZEXT_MOVL && N.getNode()->hasOneUse() && in selectScalarSSELoad()
1535 N.getOperand(0).getNode()->hasOneUse() && in selectScalarSSELoad()
1542 if (!selectAddr(LD, LD->getBasePtr(), Base, Scale, Index, Disp, Segment)) in selectScalarSSELoad()
1553 uint64_t ImmVal = CN->getZExtValue(); in selectMOV64Imm32()
1557 Imm = CurDAG->getTargetConstant(ImmVal, SDLoc(N), MVT::i64); in selectMOV64Imm32()
1564 assert(N->getOpcode() == X86ISD::Wrapper && in selectMOV64Imm32()
1568 if (N->getOpcode() != ISD::TargetConstantPool && in selectMOV64Imm32()
1569 N->getOpcode() != ISD::TargetJumpTable && in selectMOV64Imm32()
1570 N->getOpcode() != ISD::TargetGlobalAddress && in selectMOV64Imm32()
1571 N->getOpcode() != ISD::TargetExternalSymbol && in selectMOV64Imm32()
1572 N->getOpcode() != ISD::MCSymbol && in selectMOV64Imm32()
1573 N->getOpcode() != ISD::TargetBlockAddress) in selectMOV64Imm32()
1590 if (RN && RN->getReg() == 0) in selectLEA64_32Addr()
1591 Base = CurDAG->getRegister(0, MVT::i64); in selectLEA64_32Addr()
1594 Base = SDValue(CurDAG->getMachineNode( in selectLEA64_32Addr()
1596 CurDAG->getTargetConstant(0, DL, MVT::i64), in selectLEA64_32Addr()
1598 CurDAG->getTargetConstant(X86::sub_32bit, DL, MVT::i32)), in selectLEA64_32Addr()
1603 if (RN && RN->getReg() == 0) in selectLEA64_32Addr()
1604 Index = CurDAG->getRegister(0, MVT::i64); in selectLEA64_32Addr()
1607 "Expect to be extending 32-bit registers for use in LEA"); in selectLEA64_32Addr()
1608 Index = SDValue(CurDAG->getMachineNode( in selectLEA64_32Addr()
1610 CurDAG->getTargetConstant(0, DL, MVT::i64), in selectLEA64_32Addr()
1612 CurDAG->getTargetConstant(X86::sub_32bit, DL, in selectLEA64_32Addr()
1635 SDValue T = CurDAG->getRegister(0, MVT::i32); in selectLEAAddr()
1647 AM.Base_Reg = CurDAG->getRegister(0, VT); in selectLEAAddr()
1654 AM.IndexReg = CurDAG->getRegister(0, VT); in selectLEAAddr()
1664 // its three-address nature. Tweak the cost function again when we can run in selectLEAAddr()
1667 // For X86-64, always use LEA to materialize RIP-relative addresses. in selectLEAAddr()
1668 if (Subtarget->is64Bit()) in selectLEAAddr()
1693 AM.GV = GA->getGlobal(); in selectTLSADDRAddr()
1694 AM.Disp += GA->getOffset(); in selectTLSADDRAddr()
1695 AM.Base_Reg = CurDAG->getRegister(0, N.getValueType()); in selectTLSADDRAddr()
1696 AM.SymbolFlags = GA->getTargetFlags(); in selectTLSADDRAddr()
1700 AM.IndexReg = CurDAG->getRegister(X86::EBX, MVT::i32); in selectTLSADDRAddr()
1702 AM.IndexReg = CurDAG->getRegister(0, MVT::i64); in selectTLSADDRAddr()
1727 unsigned GlobalBaseReg = getInstrInfo()->getGlobalBaseReg(MF); in getGlobalBaseReg()
1728 auto &DL = MF->getDataLayout(); in getGlobalBaseReg()
1729 return CurDAG->getRegister(GlobalBaseReg, TLI->getPointerTy(DL)).getNode(); in getGlobalBaseReg()
1736 for (SDNode::use_iterator UI = N->use_begin(), in hasNoSignedComparisonUses()
1737 UE = N->use_end(); UI != UE; ++UI) { in hasNoSignedComparisonUses()
1739 if (UI->getOpcode() != ISD::CopyToReg) in hasNoSignedComparisonUses()
1742 if (cast<RegisterSDNode>(UI->getOperand(1))->getReg() != in hasNoSignedComparisonUses()
1746 for (SDNode::use_iterator FlagUI = UI->use_begin(), in hasNoSignedComparisonUses()
1747 FlagUE = UI->use_end(); FlagUI != FlagUE; ++FlagUI) { in hasNoSignedComparisonUses()
1751 if (!FlagUI->isMachineOpcode()) return false; in hasNoSignedComparisonUses()
1753 switch (FlagUI->getMachineOpcode()) { in hasNoSignedComparisonUses()
1807 if (!StoredVal.getNode()->hasNUsesOfValue(1, 0)) return false; in isLoadIncOrDecStore()
1809 // is the store non-extending and non-indexed? in isLoadIncOrDecStore()
1810 if (!ISD::isNormalStore(StoreNode) || StoreNode->isNonTemporal()) in isLoadIncOrDecStore()
1813 SDValue Load = StoredVal->getOperand(0); in isLoadIncOrDecStore()
1814 // Is the stored value a non-extending and non-indexed load? in isLoadIncOrDecStore()
1820 EVT LdVT = LoadNode->getMemoryVT(); in isLoadIncOrDecStore()
1830 if (LoadNode->getBasePtr() != StoreNode->getBasePtr() || in isLoadIncOrDecStore()
1831 LoadNode->getOffset() != StoreNode->getOffset()) in isLoadIncOrDecStore()
1836 SDValue Chain = StoreNode->getChain(); in isLoadIncOrDecStore()
1841 InputChain = LoadNode->getChain(); in isLoadIncOrDecStore()
1855 int LoadId = LoadNode->getNodeId(); in isLoadIncOrDecStore()
1856 for (SDNode::use_iterator UI = Op.getNode()->use_begin(), in isLoadIncOrDecStore()
1857 UE = UI->use_end(); UI != UE; ++UI) { in isLoadIncOrDecStore()
1860 if (UI->getNodeId() > LoadId) in isLoadIncOrDecStore()
1870 InputChain = CurDAG->getNode(ISD::TokenFactor, SDLoc(Chain), in isLoadIncOrDecStore()
1879 /// Get the appropriate X86 opcode for an in-memory increment or decrement.
1900 SDValue Chain = Node->getOperand(0); in tryGather()
1901 SDValue VSrc = Node->getOperand(2); in tryGather()
1902 SDValue Base = Node->getOperand(3); in tryGather()
1903 SDValue VIdx = Node->getOperand(4); in tryGather()
1904 SDValue VMask = Node->getOperand(5); in tryGather()
1905 ConstantSDNode *Scale = dyn_cast<ConstantSDNode>(Node->getOperand(6)); in tryGather()
1909 SDVTList VTs = CurDAG->getVTList(VSrc.getValueType(), VSrc.getValueType(), in tryGather()
1915 SDValue Disp = CurDAG->getTargetConstant(0, DL, MVT::i32); in tryGather()
1916 SDValue Segment = CurDAG->getRegister(0, MVT::i32); in tryGather()
1917 const SDValue Ops[] = { VSrc, Base, getI8Imm(Scale->getSExtValue(), DL), VIdx, in tryGather()
1919 SDNode *ResNode = CurDAG->getMachineNode(Opc, DL, VTs, Ops); in tryGather()
1926 CurDAG->RemoveDeadNode(Node); in tryGather()
1931 MVT NVT = Node->getSimpleValueType(0); in Select()
1933 unsigned Opcode = Node->getOpcode(); in Select()
1936 DEBUG(dbgs() << "Selecting: "; Node->dump(CurDAG); dbgs() << '\n'); in Select()
1938 if (Node->isMachineOpcode()) { in Select()
1939 DEBUG(dbgs() << "== "; Node->dump(CurDAG); dbgs() << '\n'); in Select()
1940 Node->setNodeId(-1); in Select()
1947 if (Subtarget->isTargetNaCl()) in Select()
1951 if (Subtarget->isTarget64BitILP32()) { in Select()
1952 // Converts a 32-bit register to a 64-bit, zero-extended version of in Select()
1953 // it. This is needed because x86-64 can do many things, but jmp %r32 in Select()
1955 const SDValue &Target = Node->getOperand(1); in Select()
1957 SDValue ZextTarget = CurDAG->getZExtOrTrunc(Target, dl, EVT(MVT::i64)); in Select()
1958 SDValue Brind = CurDAG->getNode(ISD::BRIND, dl, MVT::Other, in Select()
1959 Node->getOperand(0), ZextTarget); in Select()
1968 unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue(); in Select()
1987 if (!Subtarget->hasAVX2()) in Select()
2022 SDValue VSelect = CurDAG->getNode( in Select()
2023 ISD::VSELECT, SDLoc(Node), Node->getValueType(0), Node->getOperand(0), in Select()
2024 Node->getOperand(1), Node->getOperand(2)); in Select()
2036 SDValue N0 = Node->getOperand(0); in Select()
2037 SDValue N1 = Node->getOperand(1); in Select()
2039 if (N0->getOpcode() != ISD::SHL || !N0->hasOneUse()) in Select()
2047 ConstantSDNode *ShlCst = dyn_cast<ConstantSDNode>(N0->getOperand(1)); in Select()
2051 int64_t Val = Cst->getSExtValue(); in Select()
2052 uint64_t ShlVal = ShlCst->getZExtValue(); in Select()
2056 uint64_t RemovedBitsMask = (1ULL << ShlVal) - 1; in Select()
2105 SDValue NewCst = CurDAG->getTargetConstant(Val >> ShlVal, dl, CstVT); in Select()
2106 SDNode *New = CurDAG->getMachineNode(Op, dl, NVT, N0->getOperand(0),NewCst); in Select()
2108 CurDAG->SelectNodeTo(Node, AddOp, NVT, SDValue(New, 0), in Select()
2111 CurDAG->SelectNodeTo(Node, ShlOp, NVT, SDValue(New, 0), in Select()
2117 SDValue N0 = Node->getOperand(0); in Select()
2118 SDValue N1 = Node->getOperand(1); in Select()
2122 SDValue InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, X86::AL, in Select()
2125 SDVTList VTs = CurDAG->getVTList(NVT, MVT::i32); in Select()
2127 SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops); in Select()
2134 SDValue N0 = Node->getOperand(0); in Select()
2135 SDValue N1 = Node->getOperand(1); in Select()
2146 SDValue InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, LoReg, in Select()
2149 SDVTList VTs = CurDAG->getVTList(NVT, NVT, MVT::i32); in Select()
2151 SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops); in Select()
2159 SDValue N0 = Node->getOperand(0); in Select()
2160 SDValue N1 = Node->getOperand(1); in Select()
2163 bool hasBMI2 = Subtarget->hasBMI2(); in Select()
2220 SDValue InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, SrcReg, in Select()
2230 SDVTList VTs = CurDAG->getVTList(NVT, NVT, MVT::Other, MVT::Glue); in Select()
2231 CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops); in Select()
2237 SDVTList VTs = CurDAG->getVTList(MVT::Other, MVT::Glue); in Select()
2238 CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops); in Select()
2245 // Record the mem-refs in Select()
2248 MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); in Select()
2249 MemOp[0] = LoadNode->getMemOperand(); in Select()
2250 CNode->setMemRefs(MemOp, MemOp + 1); in Select()
2255 SDVTList VTs = CurDAG->getVTList(NVT, NVT, MVT::Glue); in Select()
2256 SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops); in Select()
2261 SDVTList VTs = CurDAG->getVTList(MVT::Glue); in Select()
2262 SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops); in Select()
2268 if (HiReg == X86::AH && Subtarget->is64Bit() && in Select()
2270 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, in Select()
2277 CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl, MVT::i8, Result)); in Select()
2280 Result = SDValue(CurDAG->getMachineNode(X86::SHR16ri, dl, MVT::i16, in Select()
2282 CurDAG->getTargetConstant(8, dl, MVT::i8)), in Select()
2286 CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl, MVT::i8, Result)); in Select()
2292 ResLo = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, LoReg, NVT, in Select()
2297 DEBUG(dbgs() << "=> "; ResLo.getNode()->dump(CurDAG); dbgs() << '\n'); in Select()
2303 ResHi = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, HiReg, NVT, in Select()
2308 DEBUG(dbgs() << "=> "; ResHi.getNode()->dump(CurDAG); dbgs() << '\n'); in Select()
2318 SDValue N0 = Node->getOperand(0); in Select()
2319 SDValue N1 = Node->getOperand(1); in Select()
2366 bool signBitIsZero = CurDAG->SignBitIsZero(N0); in Select()
2376 SDValue(CurDAG->getMachineNode(X86::MOVZX32rm8, dl, MVT::i32, in Select()
2382 SDValue(CurDAG->getMachineNode(X86::MOVZX32rr8, dl, MVT::i32, N0),0); in Select()
2383 Chain = CurDAG->getEntryNode(); in Select()
2385 Chain = CurDAG->getCopyToReg(Chain, dl, X86::EAX, Move, SDValue()); in Select()
2389 CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, in Select()
2394 SDValue(CurDAG->getMachineNode(SExtOpcode, dl, MVT::Glue, InFlag),0); in Select()
2397 SDValue ClrNode = SDValue(CurDAG->getMachineNode(X86::MOV32r0, dl, NVT), 0); in Select()
2401 SDValue(CurDAG->getMachineNode( in Select()
2403 CurDAG->getTargetConstant(X86::sub_16bit, dl, in Select()
2411 SDValue(CurDAG->getMachineNode( in Select()
2413 CurDAG->getTargetConstant(0, dl, MVT::i64), ClrNode, in Select()
2414 CurDAG->getTargetConstant(X86::sub_32bit, dl, in Select()
2422 InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, ClrReg, in Select()
2431 CurDAG->getMachineNode(MOpc, dl, MVT::Other, MVT::Glue, Ops); in Select()
2437 SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Glue, N1, InFlag), 0); in Select()
2445 // the allocator and/or the backend get enhanced to be more robust in in Select()
2448 SDValue AHCopy = CurDAG->getRegister(X86::AH, MVT::i8); in Select()
2452 SDNode *RNode = CurDAG->getMachineNode(AHExtOpcode, dl, MVT::i32, in Select()
2459 if (Node->getValueType(1) == MVT::i64) { in Select()
2463 "Unexpected i64 sext of h-register"); in Select()
2465 SDValue(CurDAG->getMachineNode( in Select()
2467 CurDAG->getTargetConstant(0, dl, MVT::i64), Result, in Select()
2468 CurDAG->getTargetConstant(X86::sub_32bit, dl, in Select()
2474 CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl, MVT::i8, Result); in Select()
2477 DEBUG(dbgs() << "=> "; Result.getNode()->dump(CurDAG); dbgs() << '\n'); in Select()
2481 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, in Select()
2485 DEBUG(dbgs() << "=> "; Result.getNode()->dump(CurDAG); dbgs() << '\n'); in Select()
2489 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, in Select()
2493 DEBUG(dbgs() << "=> "; Result.getNode()->dump(CurDAG); dbgs() << '\n'); in Select()
2501 if (Opcode == X86ISD::SUB && Node->hasAnyUseOfValue(0)) in Select()
2504 SDValue N0 = Node->getOperand(0); in Select()
2505 SDValue N1 = Node->getOperand(1); in Select()
2514 if ((N0.getNode()->getOpcode() == ISD::AND || in Select()
2515 (N0.getResNo() == 0 && N0.getNode()->getOpcode() == X86ISD::AND)) && in Select()
2516 N0.getNode()->hasOneUse() && in Select()
2519 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N0.getNode()->getOperand(1)); in Select()
2523 if ((C->getZExtValue() & ~UINT64_C(0xff)) == 0 && in Select()
2524 (!(C->getZExtValue() & 0x80) || in Select()
2526 SDValue Imm = CurDAG->getTargetConstant(C->getZExtValue(), dl, MVT::i8); in Select()
2527 SDValue Reg = N0.getNode()->getOperand(0); in Select()
2529 // On x86-32, only the ABCD registers have 8-bit subregisters. in Select()
2530 if (!Subtarget->is64Bit()) { in Select()
2537 SDValue RC = CurDAG->getTargetConstant(TRC->getID(), dl, MVT::i32); in Select()
2538 Reg = SDValue(CurDAG->getMachineNode(X86::COPY_TO_REGCLASS, dl, in Select()
2542 // Extract the l-register. in Select()
2543 SDValue Subreg = CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl, in Select()
2547 SDNode *NewNode = CurDAG->getMachineNode(X86::TEST8ri, dl, MVT::i32, in Select()
2557 if ((C->getZExtValue() & ~UINT64_C(0xff00)) == 0 && in Select()
2558 (!(C->getZExtValue() & 0x8000) || in Select()
2561 SDValue ShiftedImm = CurDAG->getTargetConstant(C->getZExtValue() >> 8, in Select()
2563 SDValue Reg = N0.getNode()->getOperand(0); in Select()
2573 SDValue RC = CurDAG->getTargetConstant(TRC->getID(), dl, MVT::i32); in Select()
2574 Reg = SDValue(CurDAG->getMachineNode(X86::COPY_TO_REGCLASS, dl, in Select()
2577 // Extract the h-register. in Select()
2578 SDValue Subreg = CurDAG->getTargetExtractSubreg(X86::sub_8bit_hi, dl, in Select()
2584 SDNode *NewNode = CurDAG->getMachineNode(X86::TEST8ri_NOREX, dl, in Select()
2594 if ((C->getZExtValue() & ~UINT64_C(0xffff)) == 0 && in Select()
2596 (!(C->getZExtValue() & 0x8000) || in Select()
2598 SDValue Imm = CurDAG->getTargetConstant(C->getZExtValue(), dl, in Select()
2600 SDValue Reg = N0.getNode()->getOperand(0); in Select()
2602 // Extract the 16-bit subregister. in Select()
2603 SDValue Subreg = CurDAG->getTargetExtractSubreg(X86::sub_16bit, dl, in Select()
2607 SDNode *NewNode = CurDAG->getMachineNode(X86::TEST16ri, dl, MVT::i32, in Select()
2617 if ((C->getZExtValue() & ~UINT64_C(0xffffffff)) == 0 && in Select()
2619 (!(C->getZExtValue() & 0x80000000) || in Select()
2621 SDValue Imm = CurDAG->getTargetConstant(C->getZExtValue(), dl, in Select()
2623 SDValue Reg = N0.getNode()->getOperand(0); in Select()
2625 // Extract the 32-bit subregister. in Select()
2626 SDValue Subreg = CurDAG->getTargetExtractSubreg(X86::sub_32bit, dl, in Select()
2630 SDNode *NewNode = CurDAG->getMachineNode(X86::TEST32ri, dl, MVT::i32, in Select()
2652 // [(store (add (loadi64 addr:$dst), -1), addr:$dst), in Select()
2656 // [(store (add (loadi64 addr:$dst), -1), addr:$dst), in Select()
2660 SDValue StoredVal = StoreNode->getOperand(1); in Select()
2661 unsigned Opc = StoredVal->getOpcode(); in Select()
2670 if (!selectAddr(LoadNode, LoadNode->getBasePtr(), in Select()
2674 MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(2); in Select()
2675 MemOp[0] = StoreNode->getMemOperand(); in Select()
2676 MemOp[1] = LoadNode->getMemOperand(); in Select()
2678 EVT LdVT = LoadNode->getMemoryVT(); in Select()
2680 MachineSDNode *Result = CurDAG->getMachineNode(newOpc, in Select()
2683 Result->setMemRefs(MemOp, MemOp + 2); in Select()
2687 CurDAG->RemoveDeadNode(Node); in Select()
2723 /// This pass converts a legalized DAG into a X86-specific DAG,