Lines Matching refs:VT
212 for (MVT VT : MVT::vector_valuetypes()) { in AArch64TargetLowering() local
213 setOperationAction(ISD::ROTL, VT, Expand); in AArch64TargetLowering()
214 setOperationAction(ISD::ROTR, VT, Expand); in AArch64TargetLowering()
227 for (MVT VT : MVT::vector_valuetypes()) { in AArch64TargetLowering() local
228 setOperationAction(ISD::SDIVREM, VT, Expand); in AArch64TargetLowering()
229 setOperationAction(ISD::UDIVREM, VT, Expand); in AArch64TargetLowering()
419 for (MVT VT : MVT::fp_valuetypes()) { in AArch64TargetLowering() local
420 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f16, Expand); in AArch64TargetLowering()
421 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f32, Expand); in AArch64TargetLowering()
422 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f64, Expand); in AArch64TargetLowering()
423 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f80, Expand); in AArch64TargetLowering()
425 for (MVT VT : MVT::integer_valuetypes()) in AArch64TargetLowering() local
426 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Expand); in AArch64TargetLowering()
602 for (MVT VT : MVT::vector_valuetypes()) { in AArch64TargetLowering() local
603 setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand); in AArch64TargetLowering()
605 setOperationAction(ISD::MULHS, VT, Expand); in AArch64TargetLowering()
606 setOperationAction(ISD::SMUL_LOHI, VT, Expand); in AArch64TargetLowering()
607 setOperationAction(ISD::MULHU, VT, Expand); in AArch64TargetLowering()
608 setOperationAction(ISD::UMUL_LOHI, VT, Expand); in AArch64TargetLowering()
610 setOperationAction(ISD::BSWAP, VT, Expand); in AArch64TargetLowering()
613 setTruncStoreAction(VT, InnerVT, Expand); in AArch64TargetLowering()
614 setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand); in AArch64TargetLowering()
615 setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand); in AArch64TargetLowering()
616 setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand); in AArch64TargetLowering()
634 void AArch64TargetLowering::addTypeForNEON(MVT VT, MVT PromotedBitwiseVT) { in addTypeForNEON() argument
635 if (VT == MVT::v2f32 || VT == MVT::v4f16) { in addTypeForNEON()
636 setOperationAction(ISD::LOAD, VT, Promote); in addTypeForNEON()
637 AddPromotedToType(ISD::LOAD, VT, MVT::v2i32); in addTypeForNEON()
639 setOperationAction(ISD::STORE, VT, Promote); in addTypeForNEON()
640 AddPromotedToType(ISD::STORE, VT, MVT::v2i32); in addTypeForNEON()
641 } else if (VT == MVT::v2f64 || VT == MVT::v4f32 || VT == MVT::v8f16) { in addTypeForNEON()
642 setOperationAction(ISD::LOAD, VT, Promote); in addTypeForNEON()
643 AddPromotedToType(ISD::LOAD, VT, MVT::v2i64); in addTypeForNEON()
645 setOperationAction(ISD::STORE, VT, Promote); in addTypeForNEON()
646 AddPromotedToType(ISD::STORE, VT, MVT::v2i64); in addTypeForNEON()
650 if (VT == MVT::v2f32 || VT == MVT::v4f32 || VT == MVT::v2f64) { in addTypeForNEON()
651 setOperationAction(ISD::FSIN, VT, Expand); in addTypeForNEON()
652 setOperationAction(ISD::FCOS, VT, Expand); in addTypeForNEON()
653 setOperationAction(ISD::FPOWI, VT, Expand); in addTypeForNEON()
654 setOperationAction(ISD::FPOW, VT, Expand); in addTypeForNEON()
655 setOperationAction(ISD::FLOG, VT, Expand); in addTypeForNEON()
656 setOperationAction(ISD::FLOG2, VT, Expand); in addTypeForNEON()
657 setOperationAction(ISD::FLOG10, VT, Expand); in addTypeForNEON()
658 setOperationAction(ISD::FEXP, VT, Expand); in addTypeForNEON()
659 setOperationAction(ISD::FEXP2, VT, Expand); in addTypeForNEON()
662 setOperationAction(ISD::FCOPYSIGN, VT, Custom); in addTypeForNEON()
665 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); in addTypeForNEON()
666 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom); in addTypeForNEON()
667 setOperationAction(ISD::BUILD_VECTOR, VT, Custom); in addTypeForNEON()
668 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); in addTypeForNEON()
669 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom); in addTypeForNEON()
670 setOperationAction(ISD::SRA, VT, Custom); in addTypeForNEON()
671 setOperationAction(ISD::SRL, VT, Custom); in addTypeForNEON()
672 setOperationAction(ISD::SHL, VT, Custom); in addTypeForNEON()
673 setOperationAction(ISD::AND, VT, Custom); in addTypeForNEON()
674 setOperationAction(ISD::OR, VT, Custom); in addTypeForNEON()
675 setOperationAction(ISD::SETCC, VT, Custom); in addTypeForNEON()
676 setOperationAction(ISD::CONCAT_VECTORS, VT, Legal); in addTypeForNEON()
678 setOperationAction(ISD::SELECT, VT, Expand); in addTypeForNEON()
679 setOperationAction(ISD::SELECT_CC, VT, Expand); in addTypeForNEON()
680 setOperationAction(ISD::VSELECT, VT, Expand); in addTypeForNEON()
682 setLoadExtAction(ISD::EXTLOAD, InnerVT, VT, Expand); in addTypeForNEON()
685 if (VT != MVT::v8i8 && VT != MVT::v16i8) in addTypeForNEON()
686 setOperationAction(ISD::CTPOP, VT, Expand); in addTypeForNEON()
688 setOperationAction(ISD::UDIV, VT, Expand); in addTypeForNEON()
689 setOperationAction(ISD::SDIV, VT, Expand); in addTypeForNEON()
690 setOperationAction(ISD::UREM, VT, Expand); in addTypeForNEON()
691 setOperationAction(ISD::SREM, VT, Expand); in addTypeForNEON()
692 setOperationAction(ISD::FREM, VT, Expand); in addTypeForNEON()
694 setOperationAction(ISD::FP_TO_SINT, VT, Custom); in addTypeForNEON()
695 setOperationAction(ISD::FP_TO_UINT, VT, Custom); in addTypeForNEON()
698 if (!VT.isFloatingPoint() && VT != MVT::v2i64 && VT != MVT::v1i64) in addTypeForNEON()
700 setOperationAction(Opcode, VT, Legal); in addTypeForNEON()
703 if (VT.isFloatingPoint() && VT.getVectorElementType() != MVT::f16) in addTypeForNEON()
706 setOperationAction(Opcode, VT, Legal); in addTypeForNEON()
711 setIndexedLoadAction(im, VT, Legal); in addTypeForNEON()
712 setIndexedStoreAction(im, VT, Legal); in addTypeForNEON()
717 void AArch64TargetLowering::addDRTypeForNEON(MVT VT) { in addDRTypeForNEON() argument
718 addRegisterClass(VT, &AArch64::FPR64RegClass); in addDRTypeForNEON()
719 addTypeForNEON(VT, MVT::v2i32); in addDRTypeForNEON()
722 void AArch64TargetLowering::addQRTypeForNEON(MVT VT) { in addQRTypeForNEON() argument
723 addRegisterClass(VT, &AArch64::FPR128RegClass); in addQRTypeForNEON()
724 addTypeForNEON(VT, MVT::v4i32); in addQRTypeForNEON()
728 EVT VT) const { in getSetCCResultType()
729 if (!VT.isVector()) in getSetCCResultType()
731 return VT.changeVectorElementTypeToInteger(); in getSetCCResultType()
759 EVT VT = cast<MemIntrinsicSDNode>(Op)->getMemoryVT(); in computeKnownBitsForTargetNode() local
760 unsigned MemBits = VT.getScalarType().getSizeInBits(); in computeKnownBitsForTargetNode()
779 MVT VT = Op.getOperand(1).getValueType().getSimpleVT(); in computeKnownBitsForTargetNode() local
781 if (VT == MVT::v8i8 || VT == MVT::v16i8) { in computeKnownBitsForTargetNode()
785 } else if (VT == MVT::v4i16 || VT == MVT::v8i16) { in computeKnownBitsForTargetNode()
802 bool AArch64TargetLowering::allowsMisalignedMemoryAccesses(EVT VT, in allowsMisalignedMemoryAccesses() argument
811 *Fast = !Subtarget->isMisaligned128StoreSlow() || VT.getStoreSize() != 16 || in allowsMisalignedMemoryAccesses()
822 VT == MVT::v2i64; in allowsMisalignedMemoryAccesses()
1207 EVT VT = LHS.getValueType(); in emitComparison() local
1209 if (VT.isFloatingPoint()) { in emitComparison()
1210 assert(VT != MVT::f128); in emitComparison()
1211 if (VT == MVT::f16) { in emitComparison()
1214 VT = MVT::f32; in emitComparison()
1216 return DAG.getNode(AArch64ISD::FCMP, dl, VT, LHS, RHS); in emitComparison()
1249 return DAG.getNode(Opcode, dl, DAG.getVTList(VT, MVT_CC), LHS, RHS) in emitComparison()
1509 EVT VT = RHS.getValueType(); in getAArch64Cmp() local
1518 if ((VT == MVT::i32 && C != 0x80000000 && in getAArch64Cmp()
1520 (VT == MVT::i64 && C != 0x80000000ULL && in getAArch64Cmp()
1523 C = (VT == MVT::i32) ? (uint32_t)(C - 1) : C - 1; in getAArch64Cmp()
1524 RHS = DAG.getConstant(C, dl, VT); in getAArch64Cmp()
1529 if ((VT == MVT::i32 && C != 0 && in getAArch64Cmp()
1531 (VT == MVT::i64 && C != 0ULL && isLegalArithImmed(C - 1ULL))) { in getAArch64Cmp()
1533 C = (VT == MVT::i32) ? (uint32_t)(C - 1) : C - 1; in getAArch64Cmp()
1534 RHS = DAG.getConstant(C, dl, VT); in getAArch64Cmp()
1539 if ((VT == MVT::i32 && C != INT32_MAX && in getAArch64Cmp()
1541 (VT == MVT::i64 && C != INT64_MAX && in getAArch64Cmp()
1544 C = (VT == MVT::i32) ? (uint32_t)(C + 1) : C + 1; in getAArch64Cmp()
1545 RHS = DAG.getConstant(C, dl, VT); in getAArch64Cmp()
1550 if ((VT == MVT::i32 && C != UINT32_MAX && in getAArch64Cmp()
1552 (VT == MVT::i64 && C != UINT64_MAX && in getAArch64Cmp()
1555 C = (VT == MVT::i32) ? (uint32_t)(C + 1) : C + 1; in getAArch64Cmp()
1556 RHS = DAG.getConstant(C, dl, VT); in getAArch64Cmp()
1796 EVT VT = Op.getValueType(); in LowerADDC_ADDE_SUBC_SUBE() local
1799 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT)) in LowerADDC_ADDE_SUBC_SUBE()
1802 SDVTList VTs = DAG.getVTList(VT, MVT::i32); in LowerADDC_ADDE_SUBC_SUBE()
1921 EVT VT = Op.getValueType(); in LowerVectorFP_TO_INT() local
1933 if (VT.getSizeInBits() < InVT.getSizeInBits()) { in LowerVectorFP_TO_INT()
1938 return DAG.getNode(ISD::TRUNCATE, dl, VT, Cv); in LowerVectorFP_TO_INT()
1941 if (VT.getSizeInBits() > InVT.getSizeInBits()) { in LowerVectorFP_TO_INT()
1944 MVT::getVectorVT(MVT::getFloatingPointVT(VT.getScalarSizeInBits()), in LowerVectorFP_TO_INT()
1945 VT.getVectorNumElements()); in LowerVectorFP_TO_INT()
1947 return DAG.getNode(Op.getOpcode(), dl, VT, Ext); in LowerVectorFP_TO_INT()
1986 EVT VT = Op.getValueType(); in LowerVectorINT_TO_FP() local
1991 if (VT.getSizeInBits() < InVT.getSizeInBits()) { in LowerVectorINT_TO_FP()
1996 return DAG.getNode(ISD::FP_ROUND, dl, VT, In, DAG.getIntPtrConstant(0, dl)); in LowerVectorINT_TO_FP()
1999 if (VT.getSizeInBits() > InVT.getSizeInBits()) { in LowerVectorINT_TO_FP()
2002 EVT CastVT = VT.changeVectorElementTypeToInteger(); in LowerVectorINT_TO_FP()
2004 return DAG.getNode(Op.getOpcode(), dl, VT, In); in LowerVectorINT_TO_FP()
2125 EVT VT = N->getValueType(0); in isExtendedBUILD_VECTOR() local
2132 unsigned EltSize = VT.getVectorElementType().getSizeInBits(); in isExtendedBUILD_VECTOR()
2157 EVT VT = N->getValueType(0); in skipExtensionForVectorMULL() local
2159 unsigned EltSize = VT.getVectorElementType().getSizeInBits() / 2; in skipExtensionForVectorMULL()
2160 unsigned NumElts = VT.getVectorNumElements(); in skipExtensionForVectorMULL()
2214 EVT VT = Op.getValueType(); in LowerMUL() local
2215 assert(VT.is128BitVector() && VT.isInteger() && in LowerMUL()
2247 if (VT == MVT::v2i64) in LowerMUL()
2265 return DAG.getNode(NewOpc, DL, VT, Op0, Op1); in LowerMUL()
2273 return DAG.getNode(N0->getOpcode(), DL, VT, in LowerMUL()
2274 DAG.getNode(NewOpc, DL, VT, in LowerMUL()
2276 DAG.getNode(NewOpc, DL, VT, in LowerMUL()
2459 MVT ValVT = Ins[i].VT; in LowerFormalArguments()
2544 assert(RegVT == Ins[i].VT && "incorrect register location selected"); in LowerFormalArguments()
2962 MVT ArgVT = Outs[i].VT; in LowerCall()
2979 MVT ValVT = Outs[i].VT; in LowerCall()
3085 if (realArgIdx == 0 && Flags.isReturned() && Outs[0].VT == MVT::i64) { in LowerCall()
3088 assert(!Ins.empty() && Ins[0].VT == MVT::i64 && in LowerCall()
3745 EVT VT = Op.getValueType(); in LowerFCOPYSIGN() local
3752 if (SrcVT.bitsLT(VT)) in LowerFCOPYSIGN()
3753 In2 = DAG.getNode(ISD::FP_EXTEND, DL, VT, In2); in LowerFCOPYSIGN()
3754 else if (SrcVT.bitsGT(VT)) in LowerFCOPYSIGN()
3755 In2 = DAG.getNode(ISD::FP_ROUND, DL, VT, In2, DAG.getIntPtrConstant(0, DL)); in LowerFCOPYSIGN()
3761 if (VT == MVT::f32 || VT == MVT::v2f32 || VT == MVT::v4f32) { in LowerFCOPYSIGN()
3763 VecVT = (VT == MVT::v2f32 ? MVT::v2i32 : MVT::v4i32); in LowerFCOPYSIGN()
3766 if (!VT.isVector()) { in LowerFCOPYSIGN()
3775 } else if (VT == MVT::f64 || VT == MVT::v2f64) { in LowerFCOPYSIGN()
3784 if (!VT.isVector()) { in LowerFCOPYSIGN()
3801 if (VT == MVT::f64 || VT == MVT::v2f64) { in LowerFCOPYSIGN()
3810 if (VT == MVT::f32) in LowerFCOPYSIGN()
3811 return DAG.getTargetExtractSubreg(AArch64::ssub, DL, VT, Sel); in LowerFCOPYSIGN()
3812 else if (VT == MVT::f64) in LowerFCOPYSIGN()
3813 return DAG.getTargetExtractSubreg(AArch64::dsub, DL, VT, Sel); in LowerFCOPYSIGN()
3815 return DAG.getNode(ISD::BITCAST, DL, VT, Sel); in LowerFCOPYSIGN()
3836 EVT VT = Op.getValueType(); in LowerCTPOP() local
3838 if (VT == MVT::i32) in LowerCTPOP()
3847 if (VT == MVT::i64) in LowerCTPOP()
3863 EVT VT = Op.getValueType(); in LowerSETCC() local
3864 SDValue TVal = DAG.getConstant(1, dl, VT); in LowerSETCC()
3865 SDValue FVal = DAG.getConstant(0, dl, VT); in LowerSETCC()
3888 return DAG.getNode(AArch64ISD::CSEL, dl, VT, FVal, TVal, CCVal, Cmp); in LowerSETCC()
3907 return DAG.getNode(AArch64ISD::CSEL, dl, VT, FVal, TVal, CC1Val, Cmp); in LowerSETCC()
3917 DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, FVal, CC1Val, Cmp); in LowerSETCC()
3920 return DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, CS1, CC2Val, Cmp); in LowerSETCC()
4037 EVT VT = TVal.getValueType(); in LowerSELECT_CC() local
4038 return DAG.getNode(Opcode, dl, VT, TVal, FVal, CCVal, Cmp); in LowerSELECT_CC()
4044 EVT VT = TVal.getValueType(); in LowerSELECT_CC() local
4052 SDValue CS1 = DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, FVal, CC1Val, Cmp); in LowerSELECT_CC()
4058 return DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, CS1, CC2Val, Cmp); in LowerSELECT_CC()
4318 EVT VT = Op.getValueType(); in LowerVAARG() local
4337 Type *ArgTy = VT.getTypeForEVT(*DAG.getContext()); in LowerVAARG()
4344 if (VT.isInteger() && !VT.isVector()) in LowerVAARG()
4347 if (VT.isFloatingPoint() && !VT.isVector() && VT != MVT::f64) { in LowerVAARG()
4365 SDValue NarrowFP = DAG.getNode(ISD::FP_ROUND, DL, VT, WideFP.getValue(0), in LowerVAARG()
4372 return DAG.getLoad(VT, DL, APStore, VAList, MachinePointerInfo(), false, in LowerVAARG()
4381 EVT VT = Op.getValueType(); in LowerFRAMEADDR() local
4385 DAG.getCopyFromReg(DAG.getEntryNode(), DL, AArch64::FP, VT); in LowerFRAMEADDR()
4387 FrameAddr = DAG.getLoad(VT, DL, DAG.getEntryNode(), FrameAddr, in LowerFRAMEADDR()
4394 unsigned AArch64TargetLowering::getRegisterByName(const char* RegName, EVT VT, in getRegisterByName() argument
4411 EVT VT = Op.getValueType(); in LowerRETURNADDR() local
4417 return DAG.getLoad(VT, DL, DAG.getEntryNode(), in LowerRETURNADDR()
4418 DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset), in LowerRETURNADDR()
4424 return DAG.getCopyFromReg(DAG.getEntryNode(), DL, Reg, VT); in LowerRETURNADDR()
4432 EVT VT = Op.getValueType(); in LowerShiftRightParts() local
4433 unsigned VTBits = VT.getSizeInBits(); in LowerShiftRightParts()
4444 SDValue HiBitsForLo = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, RevShAmt); in LowerShiftRightParts()
4452 DAG.getNode(AArch64ISD::CSEL, dl, VT, DAG.getConstant(0, dl, MVT::i64), in LowerShiftRightParts()
4458 SDValue LoBitsForLo = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, ShAmt); in LowerShiftRightParts()
4460 DAG.getNode(ISD::OR, dl, VT, LoBitsForLo, HiBitsForLo); in LowerShiftRightParts()
4465 SDValue LoForBigShift = DAG.getNode(Opc, dl, VT, ShOpHi, ExtraShAmt); in LowerShiftRightParts()
4466 SDValue Lo = DAG.getNode(AArch64ISD::CSEL, dl, VT, LoForBigShift, in LowerShiftRightParts()
4471 SDValue HiForNormalShift = DAG.getNode(Opc, dl, VT, ShOpHi, ShAmt); in LowerShiftRightParts()
4474 ? DAG.getNode(Opc, dl, VT, ShOpHi, in LowerShiftRightParts()
4476 : DAG.getConstant(0, dl, VT); in LowerShiftRightParts()
4477 SDValue Hi = DAG.getNode(AArch64ISD::CSEL, dl, VT, HiForBigShift, in LowerShiftRightParts()
4490 EVT VT = Op.getValueType(); in LowerShiftLeftParts() local
4491 unsigned VTBits = VT.getSizeInBits(); in LowerShiftLeftParts()
4500 SDValue LoBitsForHi = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, RevShAmt); in LowerShiftLeftParts()
4508 DAG.getNode(AArch64ISD::CSEL, dl, VT, DAG.getConstant(0, dl, MVT::i64), in LowerShiftLeftParts()
4513 SDValue HiBitsForHi = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, ShAmt); in LowerShiftLeftParts()
4515 DAG.getNode(ISD::OR, dl, VT, LoBitsForHi, HiBitsForHi); in LowerShiftLeftParts()
4517 SDValue HiForBigShift = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ExtraShAmt); in LowerShiftLeftParts()
4522 SDValue Hi = DAG.getNode(AArch64ISD::CSEL, dl, VT, HiForBigShift, in LowerShiftLeftParts()
4527 SDValue LoForBigShift = DAG.getConstant(0, dl, VT); in LowerShiftLeftParts()
4528 SDValue LoForNormalShift = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt); in LowerShiftLeftParts()
4529 SDValue Lo = DAG.getNode(AArch64ISD::CSEL, dl, VT, LoForBigShift, in LowerShiftLeftParts()
4542 bool AArch64TargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT) const { in isFPImmLegal()
4545 if (Imm.isPosZero() && (VT == MVT::f64 || VT == MVT::f32)) in isFPImmLegal()
4548 if (VT == MVT::f64) in isFPImmLegal()
4550 else if (VT == MVT::f32) in isFPImmLegal()
4567 EVT VT = Operand.getValueType(); in getEstimate() local
4571 RecipOp = ((VT.isVector()) ? "vec-": "") + RecipOp; in getEstimate()
4572 RecipOp += (VT.getScalarType() == MVT::f64) ? "d": "f"; in getEstimate()
4579 return DCI.DAG.getNode(Opcode, SDLoc(Operand), VT, Operand); in getEstimate()
4696 const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const { in getRegForInlineAsmConstraint()
4700 if (VT.getSizeInBits() == 64) in getRegForInlineAsmConstraint()
4704 if (VT.getSizeInBits() == 32) in getRegForInlineAsmConstraint()
4706 if (VT.getSizeInBits() == 64) in getRegForInlineAsmConstraint()
4708 if (VT.getSizeInBits() == 128) in getRegForInlineAsmConstraint()
4714 if (VT.getSizeInBits() == 128) in getRegForInlineAsmConstraint()
4725 Res = TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT); in getRegForInlineAsmConstraint()
4738 if (VT != MVT::Other && VT.getSizeInBits() == 64) { in getRegForInlineAsmConstraint()
4896 EVT VT = V64Reg.getValueType(); in WidenVector() local
4897 unsigned NarrowSize = VT.getVectorNumElements(); in WidenVector()
4898 MVT EltTy = VT.getVectorElementType().getSimpleVT(); in WidenVector()
4916 EVT VT = V128Reg.getValueType(); in NarrowVector() local
4917 unsigned WideSize = VT.getVectorNumElements(); in NarrowVector()
4918 MVT EltTy = VT.getVectorElementType().getSimpleVT(); in NarrowVector()
4931 EVT VT = Op.getValueType(); in ReconstructShuffle() local
4932 unsigned NumElts = VT.getVectorNumElements(); in ReconstructShuffle()
4988 EVT SmallestEltTy = VT.getVectorElementType(); in ReconstructShuffle()
4996 VT.getVectorElementType().getSizeInBits() / SmallestEltTy.getSizeInBits(); in ReconstructShuffle()
4997 NumElts = VT.getSizeInBits() / SmallestEltTy.getSizeInBits(); in ReconstructShuffle()
5006 if (SrcVT.getSizeInBits() == VT.getSizeInBits()) in ReconstructShuffle()
5012 unsigned NumSrcElts = VT.getSizeInBits() / EltVT.getSizeInBits(); in ReconstructShuffle()
5015 if (SrcVT.getSizeInBits() < VT.getSizeInBits()) { in ReconstructShuffle()
5016 assert(2 * SrcVT.getSizeInBits() == VT.getSizeInBits()); in ReconstructShuffle()
5025 assert(SrcVT.getSizeInBits() == 2 * VT.getSizeInBits()); in ReconstructShuffle()
5082 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) { in ReconstructShuffle()
5095 VT.getVectorElementType().getSizeInBits()); in ReconstructShuffle()
5118 return DAG.getNode(ISD::BITCAST, dl, VT, Shuffle); in ReconstructShuffle()
5123 static bool isSingletonEXTMask(ArrayRef<int> M, EVT VT, unsigned &Imm) { in isSingletonEXTMask() argument
5124 unsigned NumElts = VT.getVectorNumElements(); in isSingletonEXTMask()
5154 static bool isEXTMask(ArrayRef<int> M, EVT VT, bool &ReverseEXT, in isEXTMask() argument
5161 unsigned NumElts = VT.getVectorNumElements(); in isEXTMask()
5196 static bool isREVMask(ArrayRef<int> M, EVT VT, unsigned BlockSize) { in isREVMask() argument
5200 unsigned EltSz = VT.getVectorElementType().getSizeInBits(); in isREVMask()
5204 unsigned NumElts = VT.getVectorNumElements(); in isREVMask()
5223 static bool isZIPMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) { in isZIPMask() argument
5224 unsigned NumElts = VT.getVectorNumElements(); in isZIPMask()
5237 static bool isUZPMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) { in isUZPMask() argument
5238 unsigned NumElts = VT.getVectorNumElements(); in isUZPMask()
5250 static bool isTRNMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) { in isTRNMask() argument
5251 unsigned NumElts = VT.getVectorNumElements(); in isTRNMask()
5264 static bool isZIP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) { in isZIP_v_undef_Mask() argument
5265 unsigned NumElts = VT.getVectorNumElements(); in isZIP_v_undef_Mask()
5281 static bool isUZP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) { in isUZP_v_undef_Mask() argument
5282 unsigned Half = VT.getVectorNumElements() / 2; in isUZP_v_undef_Mask()
5300 static bool isTRN_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) { in isTRN_v_undef_Mask() argument
5301 unsigned NumElts = VT.getVectorNumElements(); in isTRN_v_undef_Mask()
5350 static bool isConcatMask(ArrayRef<int> Mask, EVT VT, bool SplitLHS) { in isConcatMask() argument
5351 if (VT.getSizeInBits() != 128) in isConcatMask()
5354 unsigned NumElts = VT.getVectorNumElements(); in isConcatMask()
5372 EVT VT = Op.getValueType(); in tryFormConcatFromShuffle() local
5377 if (VT.getVectorElementType() != V0.getValueType().getVectorElementType() || in tryFormConcatFromShuffle()
5378 VT.getVectorElementType() != V1.getValueType().getVectorElementType()) in tryFormConcatFromShuffle()
5383 if (!isConcatMask(Mask, VT, SplitV0)) in tryFormConcatFromShuffle()
5386 EVT CastVT = EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(), in tryFormConcatFromShuffle()
5387 VT.getVectorNumElements() / 2); in tryFormConcatFromShuffle()
5396 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, V0, V1); in tryFormConcatFromShuffle()
5436 EVT VT = OpLHS.getValueType(); in GeneratePerfectShuffle() local
5443 if (VT.getVectorElementType() == MVT::i32 || in GeneratePerfectShuffle()
5444 VT.getVectorElementType() == MVT::f32) in GeneratePerfectShuffle()
5445 return DAG.getNode(AArch64ISD::REV64, dl, VT, OpLHS); in GeneratePerfectShuffle()
5447 if (VT.getVectorElementType() == MVT::i16 || in GeneratePerfectShuffle()
5448 VT.getVectorElementType() == MVT::f16) in GeneratePerfectShuffle()
5449 return DAG.getNode(AArch64ISD::REV32, dl, VT, OpLHS); in GeneratePerfectShuffle()
5451 assert(VT.getVectorElementType() == MVT::i8); in GeneratePerfectShuffle()
5452 return DAG.getNode(AArch64ISD::REV16, dl, VT, OpLHS); in GeneratePerfectShuffle()
5457 EVT EltTy = VT.getVectorElementType(); in GeneratePerfectShuffle()
5470 if (VT.getSizeInBits() == 64) in GeneratePerfectShuffle()
5473 return DAG.getNode(Opcode, dl, VT, OpLHS, Lane); in GeneratePerfectShuffle()
5479 return DAG.getNode(AArch64ISD::EXT, dl, VT, OpLHS, OpRHS, in GeneratePerfectShuffle()
5483 return DAG.getNode(AArch64ISD::UZP1, dl, DAG.getVTList(VT, VT), OpLHS, in GeneratePerfectShuffle()
5486 return DAG.getNode(AArch64ISD::UZP2, dl, DAG.getVTList(VT, VT), OpLHS, in GeneratePerfectShuffle()
5489 return DAG.getNode(AArch64ISD::ZIP1, dl, DAG.getVTList(VT, VT), OpLHS, in GeneratePerfectShuffle()
5492 return DAG.getNode(AArch64ISD::ZIP2, dl, DAG.getVTList(VT, VT), OpLHS, in GeneratePerfectShuffle()
5495 return DAG.getNode(AArch64ISD::TRN1, dl, DAG.getVTList(VT, VT), OpLHS, in GeneratePerfectShuffle()
5498 return DAG.getNode(AArch64ISD::TRN2, dl, DAG.getVTList(VT, VT), OpLHS, in GeneratePerfectShuffle()
5581 EVT VT = Op.getValueType(); in LowerVECTOR_SHUFFLE() local
5607 return DAG.getNode(AArch64ISD::DUP, dl, VT, V1.getOperand(Lane)); in LowerVECTOR_SHUFFLE()
5619 unsigned Idx = Lane >= (int)VT.getVectorNumElements() / 2; in LowerVECTOR_SHUFFLE()
5620 Lane -= Idx * VT.getVectorNumElements() / 2; in LowerVECTOR_SHUFFLE()
5622 } else if (VT.getSizeInBits() == 64) in LowerVECTOR_SHUFFLE()
5625 return DAG.getNode(Opcode, dl, VT, V1, DAG.getConstant(Lane, dl, MVT::i64)); in LowerVECTOR_SHUFFLE()
5628 if (isREVMask(ShuffleMask, VT, 64)) in LowerVECTOR_SHUFFLE()
5630 if (isREVMask(ShuffleMask, VT, 32)) in LowerVECTOR_SHUFFLE()
5632 if (isREVMask(ShuffleMask, VT, 16)) in LowerVECTOR_SHUFFLE()
5637 if (isEXTMask(ShuffleMask, VT, ReverseEXT, Imm)) { in LowerVECTOR_SHUFFLE()
5643 } else if (V2->isUndef() && isSingletonEXTMask(ShuffleMask, VT, Imm)) { in LowerVECTOR_SHUFFLE()
5650 if (isZIPMask(ShuffleMask, VT, WhichResult)) { in LowerVECTOR_SHUFFLE()
5654 if (isUZPMask(ShuffleMask, VT, WhichResult)) { in LowerVECTOR_SHUFFLE()
5658 if (isTRNMask(ShuffleMask, VT, WhichResult)) { in LowerVECTOR_SHUFFLE()
5663 if (isZIP_v_undef_Mask(ShuffleMask, VT, WhichResult)) { in LowerVECTOR_SHUFFLE()
5667 if (isUZP_v_undef_Mask(ShuffleMask, VT, WhichResult)) { in LowerVECTOR_SHUFFLE()
5671 if (isTRN_v_undef_Mask(ShuffleMask, VT, WhichResult)) { in LowerVECTOR_SHUFFLE()
5690 SrcLane -= VT.getVectorNumElements(); in LowerVECTOR_SHUFFLE()
5694 EVT ScalarVT = VT.getVectorElementType(); in LowerVECTOR_SHUFFLE()
5700 ISD::INSERT_VECTOR_ELT, dl, VT, DstVec, in LowerVECTOR_SHUFFLE()
5707 unsigned NumElts = VT.getVectorNumElements(); in LowerVECTOR_SHUFFLE()
5732 EVT VT = BVN->getValueType(0); in resolveBuildVector() local
5737 unsigned NumSplats = VT.getSizeInBits() / SplatBitSize; in resolveBuildVector()
5742 CnstBits |= SplatBits.zextOrTrunc(VT.getSizeInBits()); in resolveBuildVector()
5743 UndefBits |= (SplatBits ^ SplatUndef).zextOrTrunc(VT.getSizeInBits()); in resolveBuildVector()
5758 EVT VT = Op.getValueType(); in LowerVectorAND() local
5763 APInt CnstBits(VT.getSizeInBits(), 0); in LowerVectorAND()
5764 APInt UndefBits(VT.getSizeInBits(), 0); in LowerVectorAND()
5780 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerVectorAND()
5784 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); in LowerVectorAND()
5789 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerVectorAND()
5793 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); in LowerVectorAND()
5798 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerVectorAND()
5802 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); in LowerVectorAND()
5807 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerVectorAND()
5811 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); in LowerVectorAND()
5816 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16; in LowerVectorAND()
5820 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); in LowerVectorAND()
5825 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16; in LowerVectorAND()
5829 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); in LowerVectorAND()
5856 EVT VT = Bvec->getValueType(0); in isAllConstantBuildVector() local
5857 unsigned NumElts = VT.getVectorNumElements(); in isAllConstantBuildVector()
5884 EVT VT = N->getValueType(0); in tryLowerToSLI() local
5886 if (!VT.isVector()) in tryLowerToSLI()
5918 unsigned ElemSizeInBits = VT.getVectorElementType().getSizeInBits(); in tryLowerToSLI()
5931 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, in tryLowerToSLI()
5956 EVT VT = Op.getValueType(); in LowerVectorOR() local
5966 APInt CnstBits(VT.getSizeInBits(), 0); in LowerVectorOR()
5967 APInt UndefBits(VT.getSizeInBits(), 0); in LowerVectorOR()
5980 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerVectorOR()
5984 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); in LowerVectorOR()
5989 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerVectorOR()
5993 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); in LowerVectorOR()
5998 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerVectorOR()
6002 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); in LowerVectorOR()
6007 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerVectorOR()
6011 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); in LowerVectorOR()
6016 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16; in LowerVectorOR()
6020 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); in LowerVectorOR()
6025 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16; in LowerVectorOR()
6029 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); in LowerVectorOR()
6051 EVT VT = Op.getValueType(); in NormalizeBuildVector() local
6052 EVT EltTy= VT.getVectorElementType(); in NormalizeBuildVector()
6066 return DAG.getBuildVector(VT, dl, Ops); in NormalizeBuildVector()
6072 EVT VT = Op.getValueType(); in LowerBUILD_VECTOR() local
6076 APInt CnstBits(VT.getSizeInBits(), 0); in LowerBUILD_VECTOR()
6077 APInt UndefBits(VT.getSizeInBits(), 0); in LowerBUILD_VECTOR()
6092 if (VT.isInteger() && (CnstVal == 0 || CnstVal == ~0ULL)) in LowerBUILD_VECTOR()
6098 if (VT.getSizeInBits() == 128) { in LowerBUILD_VECTOR()
6101 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); in LowerBUILD_VECTOR()
6107 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); in LowerBUILD_VECTOR()
6112 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerBUILD_VECTOR()
6116 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); in LowerBUILD_VECTOR()
6121 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerBUILD_VECTOR()
6125 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); in LowerBUILD_VECTOR()
6130 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerBUILD_VECTOR()
6134 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); in LowerBUILD_VECTOR()
6139 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerBUILD_VECTOR()
6143 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); in LowerBUILD_VECTOR()
6148 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16; in LowerBUILD_VECTOR()
6152 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); in LowerBUILD_VECTOR()
6157 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16; in LowerBUILD_VECTOR()
6161 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); in LowerBUILD_VECTOR()
6166 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerBUILD_VECTOR()
6170 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); in LowerBUILD_VECTOR()
6175 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerBUILD_VECTOR()
6179 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); in LowerBUILD_VECTOR()
6184 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v16i8 : MVT::v8i8; in LowerBUILD_VECTOR()
6187 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); in LowerBUILD_VECTOR()
6193 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4f32 : MVT::v2f32; in LowerBUILD_VECTOR()
6196 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); in LowerBUILD_VECTOR()
6200 VT.getSizeInBits() == 128) { in LowerBUILD_VECTOR()
6204 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); in LowerBUILD_VECTOR()
6211 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerBUILD_VECTOR()
6215 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); in LowerBUILD_VECTOR()
6220 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerBUILD_VECTOR()
6224 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); in LowerBUILD_VECTOR()
6229 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerBUILD_VECTOR()
6233 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); in LowerBUILD_VECTOR()
6238 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerBUILD_VECTOR()
6242 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); in LowerBUILD_VECTOR()
6247 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16; in LowerBUILD_VECTOR()
6251 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); in LowerBUILD_VECTOR()
6256 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16; in LowerBUILD_VECTOR()
6260 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); in LowerBUILD_VECTOR()
6265 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerBUILD_VECTOR()
6269 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); in LowerBUILD_VECTOR()
6274 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerBUILD_VECTOR()
6278 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); in LowerBUILD_VECTOR()
6301 unsigned NumElts = VT.getVectorNumElements(); in LowerBUILD_VECTOR()
6333 return DAG.getUNDEF(VT); in LowerBUILD_VECTOR()
6336 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value); in LowerBUILD_VECTOR()
6343 Value.getValueType() != VT) in LowerBUILD_VECTOR()
6344 return DAG.getNode(AArch64ISD::DUP, dl, VT, Value); in LowerBUILD_VECTOR()
6354 unsigned Opcode = getDUPLANEOp(VT.getVectorElementType()); in LowerBUILD_VECTOR()
6355 return DAG.getNode(Opcode, dl, VT, Value, Lane); in LowerBUILD_VECTOR()
6358 if (VT.getVectorElementType().isFloatingPoint()) { in LowerBUILD_VECTOR()
6360 EVT EltTy = VT.getVectorElementType(); in LowerBUILD_VECTOR()
6370 return DAG.getNode(ISD::BITCAST, dl, VT, Val); in LowerBUILD_VECTOR()
6379 SDValue Val = DAG.getNode(AArch64ISD::DUP, dl, VT, ConstantValue); in LowerBUILD_VECTOR()
6387 Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Val, V, LaneIdx); in LowerBUILD_VECTOR()
6412 SDValue Vec = DAG.getUNDEF(VT); in LowerBUILD_VECTOR()
6414 unsigned ElemSize = VT.getVectorElementType().getSizeInBits(); in LowerBUILD_VECTOR()
6426 DAG.getMachineNode(TargetOpcode::INSERT_SUBREG, dl, VT, Vec, Op0, in LowerBUILD_VECTOR()
6436 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Vec, V, LaneIdx); in LowerBUILD_VECTOR()
6450 EVT VT = Op.getOperand(0).getValueType(); in LowerINSERT_VECTOR_ELT() local
6452 if (!CI || CI->getZExtValue() >= VT.getVectorNumElements()) in LowerINSERT_VECTOR_ELT()
6457 if (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32 || in LowerINSERT_VECTOR_ELT()
6458 VT == MVT::v2i64 || VT == MVT::v4f32 || VT == MVT::v2f64 || in LowerINSERT_VECTOR_ELT()
6459 VT == MVT::v8f16) in LowerINSERT_VECTOR_ELT()
6462 if (VT != MVT::v8i8 && VT != MVT::v4i16 && VT != MVT::v2i32 && in LowerINSERT_VECTOR_ELT()
6463 VT != MVT::v1i64 && VT != MVT::v2f32 && VT != MVT::v4f16) in LowerINSERT_VECTOR_ELT()
6484 EVT VT = Op.getOperand(0).getValueType(); in LowerEXTRACT_VECTOR_ELT() local
6486 if (!CI || CI->getZExtValue() >= VT.getVectorNumElements()) in LowerEXTRACT_VECTOR_ELT()
6491 if (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32 || in LowerEXTRACT_VECTOR_ELT()
6492 VT == MVT::v2i64 || VT == MVT::v4f32 || VT == MVT::v2f64 || in LowerEXTRACT_VECTOR_ELT()
6493 VT == MVT::v8f16) in LowerEXTRACT_VECTOR_ELT()
6496 if (VT != MVT::v8i8 && VT != MVT::v4i16 && VT != MVT::v2i32 && in LowerEXTRACT_VECTOR_ELT()
6497 VT != MVT::v1i64 && VT != MVT::v2f32 && VT != MVT::v4f16) in LowerEXTRACT_VECTOR_ELT()
6517 EVT VT = Op.getOperand(0).getValueType(); in LowerEXTRACT_SUBVECTOR() local
6520 if (!VT.isVector()) in LowerEXTRACT_SUBVECTOR()
6536 if (Size == 64 && Val * VT.getVectorElementType().getSizeInBits() == 64) in LowerEXTRACT_SUBVECTOR()
6543 EVT VT) const { in isShuffleMaskLegal()
6544 if (VT.getVectorNumElements() == 4 && in isShuffleMaskLegal()
6545 (VT.is128BitVector() || VT.is64BitVector())) { in isShuffleMaskLegal()
6568 return (ShuffleVectorSDNode::isSplatMask(&M[0], VT) || isREVMask(M, VT, 64) || in isShuffleMaskLegal()
6569 isREVMask(M, VT, 32) || isREVMask(M, VT, 16) || in isShuffleMaskLegal()
6570 isEXTMask(M, VT, DummyBool, DummyUnsigned) || in isShuffleMaskLegal()
6572 isTRNMask(M, VT, DummyUnsigned) || isUZPMask(M, VT, DummyUnsigned) || in isShuffleMaskLegal()
6573 isZIPMask(M, VT, DummyUnsigned) || in isShuffleMaskLegal()
6574 isTRN_v_undef_Mask(M, VT, DummyUnsigned) || in isShuffleMaskLegal()
6575 isUZP_v_undef_Mask(M, VT, DummyUnsigned) || in isShuffleMaskLegal()
6576 isZIP_v_undef_Mask(M, VT, DummyUnsigned) || in isShuffleMaskLegal()
6577 isINSMask(M, VT.getVectorNumElements(), DummyBool, DummyInt) || in isShuffleMaskLegal()
6578 isConcatMask(M, VT, VT.getSizeInBits() == 128)); in isShuffleMaskLegal()
6604 static bool isVShiftLImm(SDValue Op, EVT VT, bool isLong, int64_t &Cnt) { in isVShiftLImm() argument
6605 assert(VT.isVector() && "vector shift count is not a vector type"); in isVShiftLImm()
6606 int64_t ElementBits = VT.getVectorElementType().getSizeInBits(); in isVShiftLImm()
6615 static bool isVShiftRImm(SDValue Op, EVT VT, bool isNarrow, int64_t &Cnt) { in isVShiftRImm() argument
6616 assert(VT.isVector() && "vector shift count is not a vector type"); in isVShiftRImm()
6617 int64_t ElementBits = VT.getVectorElementType().getSizeInBits(); in isVShiftRImm()
6625 EVT VT = Op.getValueType(); in LowerVectorSRA_SRL_SHL() local
6631 unsigned EltSize = VT.getVectorElementType().getSizeInBits(); in LowerVectorSRA_SRL_SHL()
6638 if (isVShiftLImm(Op.getOperand(1), VT, false, Cnt) && Cnt < EltSize) in LowerVectorSRA_SRL_SHL()
6639 return DAG.getNode(AArch64ISD::VSHL, DL, VT, Op.getOperand(0), in LowerVectorSRA_SRL_SHL()
6641 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, in LowerVectorSRA_SRL_SHL()
6648 if (isVShiftRImm(Op.getOperand(1), VT, false, Cnt) && Cnt < EltSize) { in LowerVectorSRA_SRL_SHL()
6651 return DAG.getNode(Opc, DL, VT, Op.getOperand(0), in LowerVectorSRA_SRL_SHL()
6661 SDValue NegShift = DAG.getNode(AArch64ISD::NEG, DL, VT, Op.getOperand(1)); in LowerVectorSRA_SRL_SHL()
6663 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, in LowerVectorSRA_SRL_SHL()
6673 AArch64CC::CondCode CC, bool NoNans, EVT VT, in EmitVectorComparison() argument
6676 assert(VT.getSizeInBits() == SrcVT.getSizeInBits() && in EmitVectorComparison()
6680 APInt CnstBits(VT.getSizeInBits(), 0); in EmitVectorComparison()
6681 APInt UndefBits(VT.getSizeInBits(), 0); in EmitVectorComparison()
6692 Fcmeq = DAG.getNode(AArch64ISD::FCMEQz, dl, VT, LHS); in EmitVectorComparison()
6694 Fcmeq = DAG.getNode(AArch64ISD::FCMEQ, dl, VT, LHS, RHS); in EmitVectorComparison()
6695 return DAG.getNode(AArch64ISD::NOT, dl, VT, Fcmeq); in EmitVectorComparison()
6699 return DAG.getNode(AArch64ISD::FCMEQz, dl, VT, LHS); in EmitVectorComparison()
6700 return DAG.getNode(AArch64ISD::FCMEQ, dl, VT, LHS, RHS); in EmitVectorComparison()
6703 return DAG.getNode(AArch64ISD::FCMGEz, dl, VT, LHS); in EmitVectorComparison()
6704 return DAG.getNode(AArch64ISD::FCMGE, dl, VT, LHS, RHS); in EmitVectorComparison()
6707 return DAG.getNode(AArch64ISD::FCMGTz, dl, VT, LHS); in EmitVectorComparison()
6708 return DAG.getNode(AArch64ISD::FCMGT, dl, VT, LHS, RHS); in EmitVectorComparison()
6711 return DAG.getNode(AArch64ISD::FCMLEz, dl, VT, LHS); in EmitVectorComparison()
6712 return DAG.getNode(AArch64ISD::FCMGE, dl, VT, RHS, LHS); in EmitVectorComparison()
6720 return DAG.getNode(AArch64ISD::FCMLTz, dl, VT, LHS); in EmitVectorComparison()
6721 return DAG.getNode(AArch64ISD::FCMGT, dl, VT, RHS, LHS); in EmitVectorComparison()
6731 Cmeq = DAG.getNode(AArch64ISD::CMEQz, dl, VT, LHS); in EmitVectorComparison()
6733 Cmeq = DAG.getNode(AArch64ISD::CMEQ, dl, VT, LHS, RHS); in EmitVectorComparison()
6734 return DAG.getNode(AArch64ISD::NOT, dl, VT, Cmeq); in EmitVectorComparison()
6738 return DAG.getNode(AArch64ISD::CMEQz, dl, VT, LHS); in EmitVectorComparison()
6739 return DAG.getNode(AArch64ISD::CMEQ, dl, VT, LHS, RHS); in EmitVectorComparison()
6742 return DAG.getNode(AArch64ISD::CMGEz, dl, VT, LHS); in EmitVectorComparison()
6743 return DAG.getNode(AArch64ISD::CMGE, dl, VT, LHS, RHS); in EmitVectorComparison()
6746 return DAG.getNode(AArch64ISD::CMGTz, dl, VT, LHS); in EmitVectorComparison()
6747 return DAG.getNode(AArch64ISD::CMGT, dl, VT, LHS, RHS); in EmitVectorComparison()
6750 return DAG.getNode(AArch64ISD::CMLEz, dl, VT, LHS); in EmitVectorComparison()
6751 return DAG.getNode(AArch64ISD::CMGE, dl, VT, RHS, LHS); in EmitVectorComparison()
6753 return DAG.getNode(AArch64ISD::CMHS, dl, VT, RHS, LHS); in EmitVectorComparison()
6755 return DAG.getNode(AArch64ISD::CMHI, dl, VT, RHS, LHS); in EmitVectorComparison()
6758 return DAG.getNode(AArch64ISD::CMLTz, dl, VT, LHS); in EmitVectorComparison()
6759 return DAG.getNode(AArch64ISD::CMGT, dl, VT, RHS, LHS); in EmitVectorComparison()
6761 return DAG.getNode(AArch64ISD::CMHI, dl, VT, LHS, RHS); in EmitVectorComparison()
6763 return DAG.getNode(AArch64ISD::CMHS, dl, VT, LHS, RHS); in EmitVectorComparison()
6969 EVT VT = getValueType(DL, User->getOperand(0)->getType()); in isProfitableToHoist() local
6971 return !(isFMAFasterThanFMulAndFAdd(VT) && in isProfitableToHoist()
6972 isOperationLegalOrCustom(ISD::FMA, VT) && in isProfitableToHoist()
7358 bool AArch64TargetLowering::isFMAFasterThanFMulAndFAdd(EVT VT) const { in isFMAFasterThanFMulAndFAdd()
7359 VT = VT.getScalarType(); in isFMAFasterThanFMulAndFAdd()
7361 if (!VT.isSimple()) in isFMAFasterThanFMulAndFAdd()
7364 switch (VT.getSimpleVT().SimpleTy) { in isFMAFasterThanFMulAndFAdd()
7388 EVT VT = N->getValueType(0); in isDesirableToCommuteWithShift() local
7391 if (N->getOpcode() == ISD::AND && (VT == MVT::i32 || VT == MVT::i64) && in isDesirableToCommuteWithShift()
7431 EVT VT = N->getValueType(0); in foldVectorXorShiftIntoCmp() local
7432 if (!Subtarget->hasNEON() || !VT.isVector()) in foldVectorXorShiftIntoCmp()
7449 return DAG.getNode(AArch64ISD::CMGEz, SDLoc(N), VT, Shift.getOperand(0)); in foldVectorXorShiftIntoCmp()
7454 EVT VT = N->getValueType(0); in performIntegerAbsCombine() local
7462 if (VT.isInteger() && N->getOpcode() == ISD::XOR && in performIntegerAbsCombine()
7466 if (Y1C->getAPIntValue() == VT.getSizeInBits() - 1) { in performIntegerAbsCombine()
7467 SDValue Neg = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), in performIntegerAbsCombine()
7471 DAG.getNode(AArch64ISD::SUBS, DL, DAG.getVTList(VT, MVT::i32), in performIntegerAbsCombine()
7472 N0.getOperand(0), DAG.getConstant(0, DL, VT)); in performIntegerAbsCombine()
7473 return DAG.getNode(AArch64ISD::CSEL, DL, VT, N0.getOperand(0), Neg, in performIntegerAbsCombine()
7501 EVT VT = N->getValueType(0); in BuildSDIVPow2() local
7502 if ((VT != MVT::i32 && VT != MVT::i64) || in BuildSDIVPow2()
7509 SDValue Zero = DAG.getConstant(0, DL, VT); in BuildSDIVPow2()
7510 SDValue Pow2MinusOne = DAG.getConstant((1ULL << Lg2) - 1, DL, VT); in BuildSDIVPow2()
7515 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Pow2MinusOne); in BuildSDIVPow2()
7516 SDValue CSel = DAG.getNode(AArch64ISD::CSEL, DL, VT, Add, N0, CCVal, Cmp); in BuildSDIVPow2()
7526 DAG.getNode(ISD::SRA, DL, VT, CSel, DAG.getConstant(Lg2, DL, MVT::i64)); in BuildSDIVPow2()
7535 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), SRA); in BuildSDIVPow2()
7551 EVT VT = N->getValueType(0); in performMulCombine() local
7558 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), in performMulCombine()
7560 return DAG.getNode(ISD::ADD, DL, VT, ShiftedVal, in performMulCombine()
7567 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), in performMulCombine()
7569 return DAG.getNode(ISD::SUB, DL, VT, ShiftedVal, in performMulCombine()
7577 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), in performMulCombine()
7579 return DAG.getNode(ISD::SUB, DL, VT, N->getOperand(0), in performMulCombine()
7586 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), in performMulCombine()
7589 DAG.getNode(ISD::ADD, DL, VT, ShiftedVal, N->getOperand(0)); in performMulCombine()
7590 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Add); in performMulCombine()
7610 EVT VT = N->getValueType(0); in performVectorCompareAndMaskUnaryOpCombine() local
7611 if (!VT.isVector() || N->getOperand(0)->getOpcode() != ISD::AND || in performVectorCompareAndMaskUnaryOpCombine()
7613 VT.getSizeInBits() != N->getOperand(0)->getValueType(0).getSizeInBits()) in performVectorCompareAndMaskUnaryOpCombine()
7631 SDValue SourceConst = DAG.getNode(N->getOpcode(), DL, VT, SDValue(BV, 0)); in performVectorCompareAndMaskUnaryOpCombine()
7636 SDValue Res = DAG.getNode(ISD::BITCAST, DL, VT, NewAnd); in performVectorCompareAndMaskUnaryOpCombine()
7650 EVT VT = N->getValueType(0); in performIntToFpCombine() local
7651 if (VT != MVT::f32 && VT != MVT::f64) in performIntToFpCombine()
7655 if (VT.getSizeInBits() != N->getOperand(0).getValueType().getSizeInBits()) in performIntToFpCombine()
7666 SDValue Load = DAG.getLoad(VT, SDLoc(N), LN0->getChain(), LN0->getBasePtr(), in performIntToFpCombine()
7677 return DAG.getNode(Opcode, SDLoc(N), VT, Load); in performIntToFpCombine()
7847 EVT VT = N->getValueType(0); in tryCombineToEXTR() local
7851 if (VT != MVT::i32 && VT != MVT::i64) in tryCombineToEXTR()
7871 if (ShiftLHS + ShiftRHS != VT.getSizeInBits()) in tryCombineToEXTR()
7879 return DAG.getNode(AArch64ISD::EXTR, DL, VT, LHS, RHS, in tryCombineToEXTR()
7885 EVT VT = N->getValueType(0); in tryCombineToBSL() local
7889 if (!VT.isVector()) in tryCombineToBSL()
7902 unsigned Bits = VT.getVectorElementType().getSizeInBits(); in tryCombineToBSL()
7912 for (unsigned k = 0; k < VT.getVectorNumElements(); ++k) { in tryCombineToBSL()
7923 return DAG.getNode(AArch64ISD::BSL, DL, VT, SDValue(BVN0, 0), in tryCombineToBSL()
7934 EVT VT = N->getValueType(0); in performORCombine() local
7936 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT)) in performORCombine()
7951 EVT VT = N->getValueType(0); in performSRLCombine() local
7952 if (VT != MVT::i32 && VT != MVT::i64) in performSRLCombine()
7965 if (VT == MVT::i32 && ShiftAmt == 16 && in performSRLCombine()
7967 return DAG.getNode(ISD::ROTR, DL, VT, N0, N1); in performSRLCombine()
7968 if (VT == MVT::i64 && ShiftAmt == 32 && in performSRLCombine()
7970 return DAG.getNode(ISD::ROTR, DL, VT, N0, N1); in performSRLCombine()
7992 EVT VT = N->getValueType(0); in performBitcastCombine() local
7993 if (!VT.isVector()) in performBitcastCombine()
7995 if (VT.getSimpleVT().getSizeInBits() != 64) in performBitcastCombine()
8022 if (SVT.getVectorNumElements() != VT.getVectorNumElements() * 2) in performBitcastCombine()
8030 unsigned NumElements = VT.getVectorNumElements(); in performBitcastCombine()
8033 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Source, HalfIdx); in performBitcastCombine()
8036 return SDValue(DAG.getMachineNode(TargetOpcode::EXTRACT_SUBREG, dl, VT, in performBitcastCombine()
8046 EVT VT = N->getValueType(0); in performConcatVectorsCombine() local
8069 N00VT.getScalarSizeInBits() == 4 * VT.getScalarSizeInBits()) { in performConcatVectorsCombine()
8074 return DAG.getNode(ISD::TRUNCATE, dl, VT, in performConcatVectorsCombine()
8090 if (N0 == N1 && VT.getVectorNumElements() == 2) { in performConcatVectorsCombine()
8091 assert(VT.getVectorElementType().getSizeInBits() == 64); in performConcatVectorsCombine()
8092 return DAG.getNode(AArch64ISD::DUPLANE64, dl, VT, WidenVector(N0, DAG), in performConcatVectorsCombine()
8117 return DAG.getNode(ISD::BITCAST, dl, VT, in performConcatVectorsCombine()
8351 EVT VT = Op->getValueType(0); in performSetccAddFolding() local
8352 LHS = DAG.getNode(ISD::ADD, dl, VT, RHS, DAG.getConstant(1, dl, VT)); in performSetccAddFolding()
8353 return DAG.getNode(AArch64ISD::CSEL, dl, VT, RHS, LHS, CCVal, Cmp); in performSetccAddFolding()
8373 MVT VT = N->getSimpleValueType(0); in performAddSubLongCombine() local
8374 if (!VT.is128BitVector()) { in performAddSubLongCombine()
8397 RHS = DAG.getNode(ExtType, SDLoc(N), VT, RHS); in performAddSubLongCombine()
8403 LHS = DAG.getNode(ExtType, SDLoc(N), VT, LHS); in performAddSubLongCombine()
8406 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, LHS, RHS); in performAddSubLongCombine()
8692 EVT VT = StVal.getValueType(); in replaceSplatVectorStore() local
8696 if (VT.isFloatingPoint()) in replaceSplatVectorStore()
8704 unsigned NumVecElts = VT.getVectorNumElements(); in replaceSplatVectorStore()
8767 EVT VT = StVal.getValueType(); in split16BStores() local
8771 if (!VT.isVector() || VT.getVectorNumElements() < 2 || VT == MVT::v2i64) in split16BStores()
8779 if (VT.getSizeInBits() != 128 || S->getAlignment() >= 16 || in split16BStores()
8790 unsigned NumElts = VT.getVectorNumElements() / 2; in split16BStores()
8793 EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(), NumElts); in split16BStores()
8818 EVT VT = N->getValueType(0); in performPostLD1Combine() local
8829 if (MemVT != VT.getVectorElementType()) in performPostLD1Combine()
8865 unsigned NumBytes = VT.getScalarSizeInBits() / 8; in performPostLD1Combine()
8887 EVT Tys[3] = { VT, MVT::i64, MVT::Other }; in performPostLD1Combine()
10010 EVT VT; in getPreIndexedAddressParts() local
10013 VT = LD->getMemoryVT(); in getPreIndexedAddressParts()
10016 VT = ST->getMemoryVT(); in getPreIndexedAddressParts()
10031 EVT VT; in getPostIndexedAddressParts() local
10034 VT = LD->getMemoryVT(); in getPostIndexedAddressParts()
10037 VT = ST->getMemoryVT(); in getPostIndexedAddressParts()
10159 AArch64TargetLowering::getPreferredVectorAction(EVT VT) const { in getPreferredVectorAction()
10160 MVT SVT = VT.getSimpleVT(); in getPreferredVectorAction()
10167 return TargetLoweringBase::getPreferredVectorAction(VT); in getPreferredVectorAction()
10367 bool AArch64TargetLowering::isIntDivCheap(EVT VT, AttributeSet Attr) const { in isIntDivCheap() argument
10377 return OptSize && !VT.isVector(); in isIntDivCheap()