Lines Matching refs:MVT
184 template<MVT::SimpleValueType VT>
189 template<MVT::SimpleValueType VT>
194 template <MVT::SimpleValueType VT>
219 Imm = CurDAG->getTargetConstant(MulImm, SDLoc(N), MVT::i32); in SelectCntImm()
327 bool SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm, SDValue &Shift);
329 bool SelectSVELogicalImm(SDValue N, MVT VT, SDValue &Imm);
335 bool SelectSVEArithImm(SDValue N, MVT VT, SDValue &Imm);
378 SDValue RC = CurDAG->getTargetConstant(TRC->getID(), dl, MVT::i64); in SelectInlineAsmMemoryOperand()
415 Val = CurDAG->getTargetConstant(Immed, dl, MVT::i32); in SelectArithImmed()
416 Shift = CurDAG->getTargetConstant(ShVal, dl, MVT::i32); in SelectArithImmed()
441 if (N.getValueType() == MVT::i32) in SelectNegArithImmed()
449 return SelectArithImmed(CurDAG->getConstant(Immed, SDLoc(N), MVT::i32), Val, in SelectNegArithImmed()
537 Shift = CurDAG->getTargetConstant(ShVal, SDLoc(N), MVT::i32); in SelectShiftedRegister()
556 if (!IsLoadStore && SrcVT == MVT::i8) in getExtendTypeForNode()
558 else if (!IsLoadStore && SrcVT == MVT::i16) in getExtendTypeForNode()
560 else if (SrcVT == MVT::i32) in getExtendTypeForNode()
562 assert(SrcVT != MVT::i64 && "extend from 64-bits?"); in getExtendTypeForNode()
568 if (!IsLoadStore && SrcVT == MVT::i8) in getExtendTypeForNode()
570 else if (!IsLoadStore && SrcVT == MVT::i16) in getExtendTypeForNode()
572 else if (SrcVT == MVT::i32) in getExtendTypeForNode()
574 assert(SrcVT != MVT::i64 && "extend from 64-bits?"); in getExtendTypeForNode()
655 SDValue LaneIdxVal = CurDAG->getTargetConstant(LaneIdx, dl, MVT::i64); in tryMLAV64LaneV128()
664 case MVT::v4i16: in tryMLAV64LaneV128()
667 case MVT::v8i16: in tryMLAV64LaneV128()
670 case MVT::v2i32: in tryMLAV64LaneV128()
673 case MVT::v4i32: in tryMLAV64LaneV128()
692 SDValue LaneIdxVal = CurDAG->getTargetConstant(LaneIdx, dl, MVT::i64); in tryMULLV64LaneV128()
702 case MVT::v4i32: in tryMULLV64LaneV128()
705 case MVT::v2i64: in tryMULLV64LaneV128()
713 case MVT::v4i32: in tryMULLV64LaneV128()
716 case MVT::v2i64: in tryMULLV64LaneV128()
732 if (N.getValueType() == MVT::i32) in narrowIfNeeded()
736 SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32, dl, MVT::i32); in narrowIfNeeded()
738 dl, MVT::i32, N, SubReg); in narrowIfNeeded()
752 Imm = CurDAG->getTargetConstant(RDVLImm, SDLoc(N), MVT::i32); in SelectRDVLImm()
801 MVT::i32); in SelectArithExtendedRegister()
839 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64); in SelectAddrModeIndexedBitWidth()
859 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64); in SelectAddrModeIndexedBitWidth()
874 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64); in SelectAddrModeIndexedBitWidth()
885 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64); in SelectAddrModeIndexedBitWidth()
900 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64); in SelectAddrModeIndexed()
927 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64); in SelectAddrModeIndexed()
943 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64); in SelectAddrModeIndexed()
971 OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i64); in SelectAddrModeUnscaled()
980 SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32, dl, MVT::i32); in Widen()
982 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, MVT::i64), 0); in Widen()
984 TargetOpcode::INSERT_SUBREG, dl, MVT::i64, ImpDef, N, SubReg); in Widen()
1007 MVT::i32); in SelectExtendedSHL()
1010 SignExtend = CurDAG->getTargetConstant(0, dl, MVT::i32); in SelectExtendedSHL()
1053 DoShift = CurDAG->getTargetConstant(true, dl, MVT::i32); in SelectAddrModeWRO()
1061 DoShift = CurDAG->getTargetConstant(true, dl, MVT::i32); in SelectAddrModeWRO()
1066 DoShift = CurDAG->getTargetConstant(false, dl, MVT::i32); in SelectAddrModeWRO()
1076 MVT::i32); in SelectAddrModeWRO()
1088 MVT::i32); in SelectAddrModeWRO()
1153 CurDAG->getMachineNode(AArch64::MOVi64imm, DL, MVT::i64, Ops); in SelectAddrModeXRO()
1156 N = CurDAG->getNode(ISD::ADD, DL, MVT::i64, LHS, MOVIV); in SelectAddrModeXRO()
1166 DoShift = CurDAG->getTargetConstant(true, DL, MVT::i32); in SelectAddrModeXRO()
1174 DoShift = CurDAG->getTargetConstant(true, DL, MVT::i32); in SelectAddrModeXRO()
1181 SignExtend = CurDAG->getTargetConstant(false, DL, MVT::i32); in SelectAddrModeXRO()
1182 DoShift = CurDAG->getTargetConstant(false, DL, MVT::i32); in SelectAddrModeXRO()
1231 CurDAG->getTargetConstant(RegClassIDs[Regs.size() - 2], DL, MVT::i32)); in createTuple()
1236 Ops.push_back(CurDAG->getTargetConstant(SubRegs[i], DL, MVT::i32)); in createTuple()
1240 CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL, MVT::Untyped, Ops); in createTuple()
1281 if (VT == MVT::i64) in tryIndexedLoad()
1283 else if (VT == MVT::i32) { in tryIndexedLoad()
1293 DstVT = MVT::i32; in tryIndexedLoad()
1295 } else if (VT == MVT::i16) { in tryIndexedLoad()
1297 if (DstVT == MVT::i64) in tryIndexedLoad()
1303 InsertTo64 = DstVT == MVT::i64; in tryIndexedLoad()
1306 DstVT = MVT::i32; in tryIndexedLoad()
1308 } else if (VT == MVT::i8) { in tryIndexedLoad()
1310 if (DstVT == MVT::i64) in tryIndexedLoad()
1316 InsertTo64 = DstVT == MVT::i64; in tryIndexedLoad()
1319 DstVT = MVT::i32; in tryIndexedLoad()
1321 } else if (VT == MVT::f16) { in tryIndexedLoad()
1323 } else if (VT == MVT::bf16) { in tryIndexedLoad()
1325 } else if (VT == MVT::f32) { in tryIndexedLoad()
1327 } else if (VT == MVT::f64 || VT.is64BitVector()) { in tryIndexedLoad()
1338 SDValue Offset = CurDAG->getTargetConstant(OffsetVal, dl, MVT::i64); in tryIndexedLoad()
1340 SDNode *Res = CurDAG->getMachineNode(Opcode, dl, MVT::i64, DstVT, in tryIndexedLoad()
1341 MVT::Other, Ops); in tryIndexedLoad()
1345 SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32, dl, MVT::i32); in tryIndexedLoad()
1348 AArch64::SUBREG_TO_REG, dl, MVT::i64, in tryIndexedLoad()
1349 CurDAG->getTargetConstant(0, dl, MVT::i64), LoadedVal, in tryIndexedLoad()
1370 const EVT ResTys[] = {MVT::Untyped, MVT::Other}; in SelectLoad()
1397 const EVT ResTys[] = {MVT::i64, // Type of the write back register in SelectPostLoad()
1398 MVT::Untyped, MVT::Other}; in SelectPostLoad()
1456 CurDAG->getTargetConstant(0, DL, MVT::i64), Scale); in SelectPredicatedLoad()
1462 const EVT ResTys[] = {MVT::Untyped, MVT::Other}; in SelectPredicatedLoad()
1510 CurDAG->getTargetConstant(0, dl, MVT::i64), Scale); in SelectPredicatedStore()
1531 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64); in SelectAddrModeFrameIndexSVE()
1542 const EVT ResTys[] = {MVT::i64, // Type of the write back register in SelectPostStore()
1543 MVT::Other}; // Type for the Chain in SelectPostStore()
1571 MVT EltTy = VT.getVectorElementType().getSimpleVT(); in operator ()()
1572 MVT WideTy = MVT::getVectorVT(EltTy, 2 * NarrowSize); in operator ()()
1587 MVT EltTy = VT.getVectorElementType().getSimpleVT(); in NarrowVector()
1588 MVT NarrowTy = MVT::getVectorVT(EltTy, WideSize / 2); in NarrowVector()
1609 const EVT ResTys[] = {MVT::Untyped, MVT::Other}; in SelectLoadLane()
1614 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64), in SelectLoadLane()
1648 const EVT ResTys[] = {MVT::i64, // Type of the write back register in SelectPostLoadLane()
1649 RegSeq->getValueType(0), MVT::Other}; in SelectPostLoadLane()
1656 MVT::i64), // Lane Number in SelectPostLoadLane()
1706 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64), in SelectStoreLane()
1708 SDNode *St = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops); in SelectStoreLane()
1732 const EVT ResTys[] = {MVT::i64, // Type of the write back register in SelectPostStoreLane()
1733 MVT::Other}; in SelectPostStoreLane()
1738 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64), in SelectPostStoreLane()
1764 assert((VT == MVT::i32 || VT == MVT::i64) && in isBitfieldExtractOpFromAnd()
1792 if (VT == MVT::i64 && Op0->getOpcode() == ISD::ANY_EXTEND && in isBitfieldExtractOpFromAnd()
1799 } else if (VT == MVT::i32 && Op0->getOpcode() == ISD::TRUNCATE && in isBitfieldExtractOpFromAnd()
1829 MSB = SrlImm + (VT == MVT::i32 ? countTrailingOnes<uint32_t>(AndImm) in isBitfieldExtractOpFromAnd()
1839 Opc = VT == MVT::i32 ? AArch64::UBFMWri : AArch64::UBFMXri; in isBitfieldExtractOpFromAnd()
1850 assert((VT == MVT::i32 || VT == MVT::i64) && in isBitfieldExtractOpFromSExtInReg()
1869 Opc = (VT == MVT::i32) ? AArch64::SBFMWri : AArch64::SBFMXri; in isBitfieldExtractOpFromSExtInReg()
1909 if (N->getValueType(0) == MVT::i32) in isSeveralBitsExtractOpFromShr()
1933 assert((VT == MVT::i32 || VT == MVT::i64) && in isBitfieldExtractOpFromShr()
1945 } else if (VT == MVT::i32 && N->getOpcode() == ISD::SRL && in isBitfieldExtractOpFromShr()
1954 assert(VT == MVT::i64 && "the promoted type should be i64"); in isBitfieldExtractOpFromShr()
1982 if (VT == MVT::i32) in isBitfieldExtractOpFromShr()
1994 if (VT != MVT::i64 || NarrowVT != MVT::i32) in tryBitfieldExtractOpFromSExt()
2022 if ((VT != MVT::v2f64 || NarrowVT != MVT::v2f32) && in tryHighFPExt()
2023 (VT != MVT::v4f32 || NarrowVT != MVT::v4f16)) in tryHighFPExt()
2037 auto Opcode = VT == MVT::v2f64 ? AArch64::FCVTLv4i32 : AArch64::FCVTLv8i16; in tryHighFPExt()
2046 if (N->getValueType(0) != MVT::i32 && N->getValueType(0) != MVT::i64) in isBitfieldExtractOp()
2094 if ((Opc == AArch64::SBFMXri || Opc == AArch64::UBFMXri) && VT == MVT::i32) { in tryBitfieldExtractOp()
2095 SDValue Ops64[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, MVT::i64), in tryBitfieldExtractOp()
2096 CurDAG->getTargetConstant(Imms, dl, MVT::i64)}; in tryBitfieldExtractOp()
2098 SDNode *BFM = CurDAG->getMachineNode(Opc, dl, MVT::i64, Ops64); in tryBitfieldExtractOp()
2099 SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32, dl, MVT::i32); in tryBitfieldExtractOp()
2101 MVT::i32, SDValue(BFM, 0), SubReg)); in tryBitfieldExtractOp()
2117 assert((VT == MVT::i32 || VT == MVT::i64) && in isBitfieldDstMask()
2431 assert(VT == MVT::i32 || VT == MVT::i64); in isShiftedMask()
2432 if (VT == MVT::i32) in isShiftedMask()
2443 if (VT != MVT::i32 && VT != MVT::i64) in tryBitfieldInsertOpFromOrAndImm()
2514 unsigned MOVIOpc = VT == MVT::i32 ? AArch64::MOVi32imm : AArch64::MOVi64imm; in tryBitfieldInsertOpFromOrAndImm()
2522 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri; in tryBitfieldInsertOpFromOrAndImm()
2532 if (VT != MVT::i32 && VT != MVT::i64) in tryBitfieldInsertOpFromOr()
2577 if ((BFXOpc != AArch64::UBFMXri && VT == MVT::i64) || in tryBitfieldInsertOpFromOr()
2578 (BFXOpc != AArch64::UBFMWri && VT == MVT::i32)) in tryBitfieldInsertOpFromOr()
2602 assert((VT == MVT::i32 || VT == MVT::i64) && "unexpected OR operand"); in tryBitfieldInsertOpFromOr()
2633 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri; in tryBitfieldInsertOpFromOr()
2666 unsigned ShiftOpc = (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri; in tryBitfieldInsertOpFromOr()
2679 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri; in tryBitfieldInsertOpFromOr()
2714 if (VT != MVT::i32 && VT != MVT::i64) in tryBitfieldInsertInZeroOp()
2731 unsigned Opc = (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri; in tryBitfieldInsertInZeroOp()
2744 Opc = (VT == MVT::i32) ? AArch64::RORVWr : AArch64::RORVXr; in tryShiftAmountMod()
2747 Opc = (VT == MVT::i32) ? AArch64::LSLVWr : AArch64::LSLVXr; in tryShiftAmountMod()
2750 Opc = (VT == MVT::i32) ? AArch64::LSRVWr : AArch64::LSRVXr; in tryShiftAmountMod()
2753 Opc = (VT == MVT::i32) ? AArch64::ASRVWr : AArch64::ASRVXr; in tryShiftAmountMod()
2761 if (VT == MVT::i32) { in tryShiftAmountMod()
2764 } else if (VT == MVT::i64) { in tryShiftAmountMod()
2796 if (SubVT == MVT::i32) { in tryShiftAmountMod()
2800 assert(SubVT == MVT::i64); in tryShiftAmountMod()
2827 if (VT == MVT::i32) in tryShiftAmountMod()
2829 else if (VT == MVT::i64 && NewShiftAmt->getValueType(0) == MVT::i32) { in tryShiftAmountMod()
2830 SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32, DL, MVT::i32); in tryShiftAmountMod()
2833 CurDAG->getTargetConstant(0, DL, MVT::i64), NewShiftAmt, SubReg); in tryShiftAmountMod()
2882 FixedPos = CurDAG->getTargetConstant(FBits, SDLoc(N), MVT::i32); in SelectCVTFixedPosOperand()
2929 AArch64::MRS, DL, N->getSimpleValueType(0), MVT::Other, in tryReadRegister()
2930 CurDAG->getTargetConstant(Reg, DL, MVT::i32), in tryReadRegister()
2946 AArch64::MRS, DL, N->getSimpleValueType(0), MVT::Other, in tryReadRegister()
2947 CurDAG->getTargetConstant(Reg, DL, MVT::i32), in tryReadRegister()
2954 AArch64::ADR, DL, N->getSimpleValueType(0), MVT::Other, in tryReadRegister()
2955 CurDAG->getTargetConstant(0, DL, MVT::i32), in tryReadRegister()
2975 N, CurDAG->getMachineNode(AArch64::MSR, DL, MVT::Other, in tryWriteRegister()
2976 CurDAG->getTargetConstant(Reg, DL, MVT::i32), in tryWriteRegister()
3001 State, DL, MVT::Other, in tryWriteRegister()
3002 CurDAG->getTargetConstant(Reg, DL, MVT::i32), in tryWriteRegister()
3003 CurDAG->getTargetConstant(Immed, DL, MVT::i16), in tryWriteRegister()
3019 AArch64::MSR, DL, MVT::Other, in tryWriteRegister()
3020 CurDAG->getTargetConstant(Reg, DL, MVT::i32), in tryWriteRegister()
3036 if (MemTy == MVT::i8) in SelectCMP_SWAP()
3038 else if (MemTy == MVT::i16) in SelectCMP_SWAP()
3040 else if (MemTy == MVT::i32) in SelectCMP_SWAP()
3042 else if (MemTy == MVT::i64) in SelectCMP_SWAP()
3047 MVT RegTy = MemTy == MVT::i64 ? MVT::i64 : MVT::i32; in SelectCMP_SWAP()
3052 CurDAG->getVTList(RegTy, MVT::i32, MVT::Other), Ops); in SelectCMP_SWAP()
3090 bool AArch64DAGToDAGISel::SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm, SDValue &Shift) { in SelectSVEAddSubImm()
3096 case MVT::i8: in SelectSVEAddSubImm()
3098 Shift = CurDAG->getTargetConstant(0, DL, MVT::i32); in SelectSVEAddSubImm()
3099 Imm = CurDAG->getTargetConstant(ImmVal, DL, MVT::i32); in SelectSVEAddSubImm()
3103 case MVT::i16: in SelectSVEAddSubImm()
3104 case MVT::i32: in SelectSVEAddSubImm()
3105 case MVT::i64: in SelectSVEAddSubImm()
3107 Shift = CurDAG->getTargetConstant(0, DL, MVT::i32); in SelectSVEAddSubImm()
3108 Imm = CurDAG->getTargetConstant(ImmVal, DL, MVT::i32); in SelectSVEAddSubImm()
3111 Shift = CurDAG->getTargetConstant(8, DL, MVT::i32); in SelectSVEAddSubImm()
3112 Imm = CurDAG->getTargetConstant(ImmVal >> 8, DL, MVT::i32); in SelectSVEAddSubImm()
3129 Imm = CurDAG->getTargetConstant(ImmVal, DL, MVT::i32); in SelectSVESignedArithImm()
3136 bool AArch64DAGToDAGISel::SelectSVEArithImm(SDValue N, MVT VT, SDValue &Imm) { in SelectSVEArithImm()
3141 case MVT::i8: in SelectSVEArithImm()
3144 case MVT::i16: in SelectSVEArithImm()
3147 case MVT::i32: in SelectSVEArithImm()
3150 case MVT::i64: in SelectSVEArithImm()
3157 Imm = CurDAG->getTargetConstant(ImmVal, SDLoc(N), MVT::i32); in SelectSVEArithImm()
3164 bool AArch64DAGToDAGISel::SelectSVELogicalImm(SDValue N, MVT VT, SDValue &Imm) { in SelectSVELogicalImm()
3171 case MVT::i8: in SelectSVELogicalImm()
3177 case MVT::i16: in SelectSVELogicalImm()
3182 case MVT::i32: in SelectSVELogicalImm()
3186 case MVT::i64: in SelectSVELogicalImm()
3194 Imm = CurDAG->getTargetConstant(encoding, DL, MVT::i64); in SelectSVELogicalImm()
3223 Imm = CurDAG->getTargetConstant(ImmVal, SDLoc(N), MVT::i32); in SelectSVEShiftImm()
3253 AArch64::TAGPstack, DL, MVT::i64, in trySelectStackSlotTagP()
3254 {FiOp, CurDAG->getTargetConstant(0, DL, MVT::i64), N->getOperand(2), in trySelectStackSlotTagP()
3255 CurDAG->getTargetConstant(TagOffset, DL, MVT::i64)}); in trySelectStackSlotTagP()
3271 SDNode *N1 = CurDAG->getMachineNode(AArch64::SUBP, DL, MVT::i64, in SelectTagP()
3273 SDNode *N2 = CurDAG->getMachineNode(AArch64::ADDXrr, DL, MVT::i64, in SelectTagP()
3276 AArch64::ADDG, DL, MVT::i64, in SelectTagP()
3277 {SDValue(N2, 0), CurDAG->getTargetConstant(0, DL, MVT::i64), in SelectTagP()
3278 CurDAG->getTargetConstant(TagOffset, DL, MVT::i64)}); in SelectTagP()
3295 auto SubReg = DAG->getTargetConstant(AArch64::dsub, DL, MVT::i32); in extractSubReg()
3299 auto SubReg = DAG->getTargetConstant(AArch64::zsub, DL, MVT::i32); in extractSubReg()
3303 auto RC = DAG->getTargetConstant(AArch64::ZPRRegClassID, DL, MVT::i64); in extractSubReg()
3321 auto SubReg = DAG->getTargetConstant(AArch64::dsub, DL, MVT::i32); in insertSubReg()
3327 auto SubReg = DAG->getTargetConstant(AArch64::zsub, DL, MVT::i32); in insertSubReg()
3333 auto RC = DAG->getTargetConstant(AArch64::ZPRRegClassID, DL, MVT::i64); in insertSubReg()
3463 if (VT == MVT::i32) { in Select()
3465 CurDAG->getEntryNode(), SDLoc(Node), AArch64::WZR, MVT::i32); in Select()
3468 } else if (VT == MVT::i64) { in Select()
3470 CurDAG->getEntryNode(), SDLoc(Node), AArch64::XZR, MVT::i64); in Select()
3486 SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, DL, MVT::i32), in Select()
3487 CurDAG->getTargetConstant(Shifter, DL, MVT::i32) }; in Select()
3488 CurDAG->SelectNodeTo(Node, AArch64::ADDXri, MVT::i64, Ops); in Select()
3504 SDNode *Ld = CurDAG->getMachineNode(Op, DL, MVT::i64, MVT::i64, in Select()
3505 MVT::Other, MemAddr, Chain); in Select()
3527 SDNode *St = CurDAG->getMachineNode(Op, DL, MVT::i32, MVT::Other, Ops); in Select()
3537 if (VT == MVT::v8i8) { in Select()
3540 } else if (VT == MVT::v16i8) { in Select()
3543 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
3546 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
3549 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
3552 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
3555 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
3558 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
3564 if (VT == MVT::v8i8) { in Select()
3567 } else if (VT == MVT::v16i8) { in Select()
3570 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
3573 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
3576 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
3579 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
3582 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
3585 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
3591 if (VT == MVT::v8i8) { in Select()
3594 } else if (VT == MVT::v16i8) { in Select()
3597 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
3600 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
3603 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
3606 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
3609 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
3612 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
3618 if (VT == MVT::v8i8) { in Select()
3621 } else if (VT == MVT::v16i8) { in Select()
3624 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
3627 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
3630 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
3633 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
3636 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
3639 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
3645 if (VT == MVT::v8i8) { in Select()
3648 } else if (VT == MVT::v16i8) { in Select()
3651 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
3654 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
3657 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
3660 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
3663 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
3666 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
3672 if (VT == MVT::v8i8) { in Select()
3675 } else if (VT == MVT::v16i8) { in Select()
3678 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
3681 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
3684 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
3687 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
3690 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
3693 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
3699 if (VT == MVT::v8i8) { in Select()
3702 } else if (VT == MVT::v16i8) { in Select()
3705 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
3708 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
3711 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
3714 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
3717 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
3720 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
3726 if (VT == MVT::v8i8) { in Select()
3729 } else if (VT == MVT::v16i8) { in Select()
3732 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
3735 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
3738 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
3741 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
3744 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
3747 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
3753 if (VT == MVT::v8i8) { in Select()
3756 } else if (VT == MVT::v16i8) { in Select()
3759 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
3762 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
3765 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
3768 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
3771 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
3774 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
3780 if (VT == MVT::v16i8 || VT == MVT::v8i8) { in Select()
3783 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
3784 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) { in Select()
3787 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 || in Select()
3788 VT == MVT::v2f32) { in Select()
3791 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 || in Select()
3792 VT == MVT::v1f64) { in Select()
3798 if (VT == MVT::v16i8 || VT == MVT::v8i8) { in Select()
3801 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
3802 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) { in Select()
3805 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 || in Select()
3806 VT == MVT::v2f32) { in Select()
3809 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 || in Select()
3810 VT == MVT::v1f64) { in Select()
3816 if (VT == MVT::v16i8 || VT == MVT::v8i8) { in Select()
3819 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
3820 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) { in Select()
3823 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 || in Select()
3824 VT == MVT::v2f32) { in Select()
3827 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 || in Select()
3828 VT == MVT::v1f64) { in Select()
3845 VT == MVT::v8i8 ? AArch64::TBLv8i8Two : AArch64::TBLv16i8Two, in Select()
3849 SelectTable(Node, 3, VT == MVT::v8i8 ? AArch64::TBLv8i8Three in Select()
3854 SelectTable(Node, 4, VT == MVT::v8i8 ? AArch64::TBLv8i8Four in Select()
3860 VT == MVT::v8i8 ? AArch64::TBXv8i8Two : AArch64::TBXv16i8Two, in Select()
3864 SelectTable(Node, 3, VT == MVT::v8i8 ? AArch64::TBXv8i8Three in Select()
3869 SelectTable(Node, 4, VT == MVT::v8i8 ? AArch64::TBXv8i8Four in Select()
3889 if (VT == MVT::v8i8) { in Select()
3892 } else if (VT == MVT::v16i8) { in Select()
3895 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
3896 VT == MVT::v4bf16) { in Select()
3899 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || in Select()
3900 VT == MVT::v8bf16) { in Select()
3903 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
3906 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
3909 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
3912 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
3919 if (VT == MVT::v8i8) { in Select()
3922 } else if (VT == MVT::v16i8) { in Select()
3925 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
3926 VT == MVT::v4bf16) { in Select()
3929 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || in Select()
3930 VT == MVT::v8bf16) { in Select()
3933 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
3936 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
3939 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
3942 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
3949 if (VT == MVT::v8i8) { in Select()
3952 } else if (VT == MVT::v16i8) { in Select()
3955 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
3956 VT == MVT::v4bf16) { in Select()
3959 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || in Select()
3960 VT == MVT::v8bf16) { in Select()
3963 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
3966 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
3969 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
3972 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
3979 if (VT == MVT::v8i8) { in Select()
3982 } else if (VT == MVT::v16i8) { in Select()
3985 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
3986 VT == MVT::v4bf16) { in Select()
3989 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || in Select()
3990 VT == MVT::v8bf16) { in Select()
3993 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
3996 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
3999 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
4002 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
4009 if (VT == MVT::v8i8) { in Select()
4012 } else if (VT == MVT::v16i8) { in Select()
4015 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
4016 VT == MVT::v4bf16) { in Select()
4019 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || in Select()
4020 VT == MVT::v8bf16) { in Select()
4023 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
4026 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
4029 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
4032 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
4039 if (VT == MVT::v8i8) { in Select()
4042 } else if (VT == MVT::v16i8) { in Select()
4045 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
4046 VT == MVT::v4bf16) { in Select()
4049 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || in Select()
4050 VT == MVT::v8bf16) { in Select()
4053 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
4056 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
4059 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
4062 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
4069 if (VT == MVT::v16i8 || VT == MVT::v8i8) { in Select()
4072 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
4073 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) { in Select()
4076 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 || in Select()
4077 VT == MVT::v2f32) { in Select()
4080 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 || in Select()
4081 VT == MVT::v1f64) { in Select()
4088 if (VT == MVT::v16i8 || VT == MVT::v8i8) { in Select()
4091 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
4092 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) { in Select()
4095 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 || in Select()
4096 VT == MVT::v2f32) { in Select()
4099 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 || in Select()
4100 VT == MVT::v1f64) { in Select()
4107 if (VT == MVT::v16i8 || VT == MVT::v8i8) { in Select()
4110 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
4111 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) { in Select()
4114 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 || in Select()
4115 VT == MVT::v2f32) { in Select()
4118 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 || in Select()
4119 VT == MVT::v1f64) { in Select()
4126 if (VT == MVT::nxv16i8) { in Select()
4129 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 || in Select()
4130 (VT == MVT::nxv8bf16 && Subtarget->hasBF16())) { in Select()
4133 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) { in Select()
4136 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) { in Select()
4143 if (VT == MVT::nxv16i8) { in Select()
4146 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 || in Select()
4147 (VT == MVT::nxv8bf16 && Subtarget->hasBF16())) { in Select()
4150 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) { in Select()
4153 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) { in Select()
4160 if (VT == MVT::nxv16i8) { in Select()
4163 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 || in Select()
4164 (VT == MVT::nxv8bf16 && Subtarget->hasBF16())) { in Select()
4167 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) { in Select()
4170 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) { in Select()
4180 if (VT == MVT::v8i8) { in Select()
4183 } else if (VT == MVT::v16i8) { in Select()
4186 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
4189 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
4192 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
4195 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
4198 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
4201 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
4208 if (VT == MVT::v8i8) { in Select()
4211 } else if (VT == MVT::v16i8) { in Select()
4214 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
4217 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
4220 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
4223 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
4226 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
4229 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
4236 if (VT == MVT::v8i8) { in Select()
4239 } else if (VT == MVT::v16i8) { in Select()
4242 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
4245 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
4248 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
4251 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
4254 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
4257 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
4264 if (VT == MVT::v8i8) { in Select()
4267 } else if (VT == MVT::v16i8) { in Select()
4270 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
4273 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
4276 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
4279 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
4282 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
4285 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
4292 if (VT == MVT::v8i8) { in Select()
4295 } else if (VT == MVT::v16i8) { in Select()
4298 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
4301 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
4304 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
4307 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
4310 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
4313 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
4320 if (VT == MVT::v8i8) { in Select()
4323 } else if (VT == MVT::v16i8) { in Select()
4326 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
4329 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
4332 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
4335 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
4338 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
4341 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
4348 if (VT == MVT::v8i8) { in Select()
4351 } else if (VT == MVT::v16i8) { in Select()
4354 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
4357 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
4360 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
4363 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
4366 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
4369 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
4376 if (VT == MVT::v8i8) { in Select()
4379 } else if (VT == MVT::v16i8) { in Select()
4382 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
4385 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
4388 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
4391 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
4394 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
4397 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
4404 if (VT == MVT::v8i8) { in Select()
4407 } else if (VT == MVT::v16i8) { in Select()
4410 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
4413 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
4416 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
4419 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
4422 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
4425 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
4432 if (VT == MVT::v8i8) { in Select()
4435 } else if (VT == MVT::v16i8) { in Select()
4438 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
4441 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
4444 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
4447 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
4450 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
4453 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
4460 if (VT == MVT::v16i8 || VT == MVT::v8i8) { in Select()
4463 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
4464 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) { in Select()
4467 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 || in Select()
4468 VT == MVT::v2f32) { in Select()
4471 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 || in Select()
4472 VT == MVT::v1f64) { in Select()
4479 if (VT == MVT::v16i8 || VT == MVT::v8i8) { in Select()
4482 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
4483 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) { in Select()
4486 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 || in Select()
4487 VT == MVT::v2f32) { in Select()
4490 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 || in Select()
4491 VT == MVT::v1f64) { in Select()
4498 if (VT == MVT::v16i8 || VT == MVT::v8i8) { in Select()
4501 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
4502 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) { in Select()
4505 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 || in Select()
4506 VT == MVT::v2f32) { in Select()
4509 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 || in Select()
4510 VT == MVT::v1f64) { in Select()
4517 if (VT == MVT::v16i8 || VT == MVT::v8i8) { in Select()
4520 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
4521 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) { in Select()
4524 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 || in Select()
4525 VT == MVT::v2f32) { in Select()
4528 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 || in Select()
4529 VT == MVT::v1f64) { in Select()
4537 if (VT == MVT::v8i8) { in Select()
4540 } else if (VT == MVT::v16i8) { in Select()
4543 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
4546 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
4549 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
4552 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
4555 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
4558 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
4566 if (VT == MVT::v8i8) { in Select()
4569 } else if (VT == MVT::v16i8) { in Select()
4572 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
4575 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
4578 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
4581 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
4584 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
4587 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
4595 if (VT == MVT::v8i8) { in Select()
4598 } else if (VT == MVT::v16i8) { in Select()
4601 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
4604 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
4607 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
4610 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
4613 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
4616 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
4624 if (VT == MVT::v8i8) { in Select()
4627 } else if (VT == MVT::v16i8) { in Select()
4630 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
4633 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
4636 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
4639 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
4642 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
4645 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
4653 if (VT == MVT::v8i8) { in Select()
4656 } else if (VT == MVT::v16i8) { in Select()
4659 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
4662 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16 ) { in Select()
4665 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
4668 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
4671 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
4674 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
4682 if (VT == MVT::v8i8) { in Select()
4685 } else if (VT == MVT::v16i8) { in Select()
4688 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
4691 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
4694 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
4697 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
4700 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
4703 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
4711 if (VT == MVT::v16i8 || VT == MVT::v8i8) { in Select()
4714 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
4715 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) { in Select()
4718 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 || in Select()
4719 VT == MVT::v2f32) { in Select()
4722 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 || in Select()
4723 VT == MVT::v1f64) { in Select()
4731 if (VT == MVT::v16i8 || VT == MVT::v8i8) { in Select()
4734 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
4735 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) { in Select()
4738 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 || in Select()
4739 VT == MVT::v2f32) { in Select()
4742 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 || in Select()
4743 VT == MVT::v1f64) { in Select()
4751 if (VT == MVT::v16i8 || VT == MVT::v8i8) { in Select()
4754 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
4755 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) { in Select()
4758 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 || in Select()
4759 VT == MVT::v2f32) { in Select()
4762 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 || in Select()
4763 VT == MVT::v1f64) { in Select()
4770 if (VT == MVT::nxv16i8) { in Select()
4773 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 || in Select()
4774 (VT == MVT::nxv8bf16 && Subtarget->hasBF16())) { in Select()
4777 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) { in Select()
4780 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) { in Select()
4787 if (VT == MVT::nxv16i8) { in Select()
4790 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 || in Select()
4791 (VT == MVT::nxv8bf16 && Subtarget->hasBF16())) { in Select()
4794 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) { in Select()
4797 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) { in Select()
4804 if (VT == MVT::nxv16i8) { in Select()
4807 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 || in Select()
4808 (VT == MVT::nxv8bf16 && Subtarget->hasBF16())) { in Select()
4811 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) { in Select()
4814 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) { in Select()
4843 if (!PredVT.isScalableVector() || PredVT.getVectorElementType() != MVT::i1) in getPackedVectorTypeFromPredicateType()
4846 if (PredVT != MVT::nxv16i1 && PredVT != MVT::nxv8i1 && in getPackedVectorTypeFromPredicateType()
4847 PredVT != MVT::nxv4i1 && PredVT != MVT::nxv2i1) in getPackedVectorTypeFromPredicateType()
4936 OffImm = CurDAG->getTargetConstant(Offset, SDLoc(N), MVT::i64); in SelectAddrModeIndexedSVE()