Lines Matching refs:OpNode
1354 multiclass avx512_cmp_scalar<X86VectorVTInfo _, SDNode OpNode, SDNode OpNodeRnd>{
1361 (OpNode (_.VT _.RC:$src1),
1369 (OpNode (_.VT _.RC:$src1),
1409 [(set _.KRC:$dst, (OpNode _.FRC:$src1,
1418 [(set _.KRC:$dst, (OpNode _.FRC:$src1,
1432 multiclass avx512_icmp_packed<bits<8> opc, string OpcodeStr, SDNode OpNode,
1437 [(set _.KRC:$dst, (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2)))],
1442 [(set _.KRC:$dst, (OpNode (_.VT _.RC:$src1),
1450 (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2))))],
1457 (OpNode (_.VT _.RC:$src1),
1463 multiclass avx512_icmp_packed_rmb<bits<8> opc, string OpcodeStr, SDNode OpNode,
1465 avx512_icmp_packed<opc, OpcodeStr, OpNode, _> {
1470 [(set _.KRC:$dst, (OpNode (_.VT _.RC:$src1),
1480 (OpNode (_.VT _.RC:$src1),
1486 multiclass avx512_icmp_packed_vl<bits<8> opc, string OpcodeStr, SDNode OpNode,
1489 defm Z : avx512_icmp_packed<opc, OpcodeStr, OpNode, VTInfo.info512>,
1493 defm Z256 : avx512_icmp_packed<opc, OpcodeStr, OpNode, VTInfo.info256>,
1495 defm Z128 : avx512_icmp_packed<opc, OpcodeStr, OpNode, VTInfo.info128>,
1501 SDNode OpNode, AVX512VLVectorVTInfo VTInfo,
1504 defm Z : avx512_icmp_packed_rmb<opc, OpcodeStr, OpNode, VTInfo.info512>,
1508 defm Z256 : avx512_icmp_packed_rmb<opc, OpcodeStr, OpNode, VTInfo.info256>,
1510 defm Z128 : avx512_icmp_packed_rmb<opc, OpcodeStr, OpNode, VTInfo.info128>,
1557 multiclass avx512_icmp_cc<bits<8> opc, string Suffix, SDNode OpNode,
1563 [(set _.KRC:$dst, (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2),
1570 [(set _.KRC:$dst, (OpNode (_.VT _.RC:$src1),
1581 (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2),
1591 (OpNode (_.VT _.RC:$src1),
1627 multiclass avx512_icmp_cc_rmb<bits<8> opc, string Suffix, SDNode OpNode,
1629 avx512_icmp_cc<opc, Suffix, OpNode, _> {
1636 [(set _.KRC:$dst, (OpNode (_.VT _.RC:$src1),
1647 (OpNode (_.VT _.RC:$src1),
1671 multiclass avx512_icmp_cc_vl<bits<8> opc, string Suffix, SDNode OpNode,
1674 defm Z : avx512_icmp_cc<opc, Suffix, OpNode, VTInfo.info512>, EVEX_V512;
1677 defm Z256 : avx512_icmp_cc<opc, Suffix, OpNode, VTInfo.info256>, EVEX_V256;
1678 defm Z128 : avx512_icmp_cc<opc, Suffix, OpNode, VTInfo.info128>, EVEX_V128;
1682 multiclass avx512_icmp_cc_rmb_vl<bits<8> opc, string Suffix, SDNode OpNode,
1685 defm Z : avx512_icmp_cc_rmb<opc, Suffix, OpNode, VTInfo.info512>,
1689 defm Z256 : avx512_icmp_cc_rmb<opc, Suffix, OpNode, VTInfo.info256>,
1691 defm Z128 : avx512_icmp_cc_rmb<opc, Suffix, OpNode, VTInfo.info128>,
1826 multiclass avx512_scalar_fpclass<bits<8> opc, string OpcodeStr, SDNode OpNode,
1832 [(set _.KRC:$dst,(OpNode (_.VT _.RC:$src1),
1839 (OpNode (_.VT _.RC:$src1),
1847 (OpNode (_.VT (bitconvert (_.LdFrag addr:$src1))),
1854 (OpNode (_.VT (bitconvert (_.LdFrag addr:$src1))),
1863 multiclass avx512_vector_fpclass<bits<8> opc, string OpcodeStr, SDNode OpNode,
1868 [(set _.KRC:$dst,(OpNode (_.VT _.RC:$src1),
1875 (OpNode (_.VT _.RC:$src1),
1881 [(set _.KRC:$dst,(OpNode
1888 [(set _.KRC:$dst, (or _.KRCWM:$mask, (OpNode
1896 [(set _.KRC:$dst,(OpNode
1905 [(set _.KRC:$dst,(or _.KRCWM:$mask, (OpNode
1913 AVX512VLVectorVTInfo _, bits<8> opc, SDNode OpNode, Predicate prd,
1916 defm Z : avx512_vector_fpclass<opc, OpcodeStr, OpNode, _.info512, "{z}",
1920 defm Z128 : avx512_vector_fpclass<opc, OpcodeStr, OpNode, _.info128, "{x}",
1922 defm Z256 : avx512_vector_fpclass<opc, OpcodeStr, OpNode, _.info256, "{y}",
2197 RegisterClass KRC, SDPatternOperator OpNode,
2202 [(set KRC:$dst, (OpNode KRC:$src))]>;
2206 SDPatternOperator OpNode> {
2207 defm B : avx512_mask_unop<opc, !strconcat(OpcodeStr, "b"), VK8, OpNode,
2209 defm W : avx512_mask_unop<opc, !strconcat(OpcodeStr, "w"), VK16, OpNode,
2211 defm D : avx512_mask_unop<opc, !strconcat(OpcodeStr, "d"), VK32, OpNode,
2213 defm Q : avx512_mask_unop<opc, !strconcat(OpcodeStr, "q"), VK64, OpNode,
2253 RegisterClass KRC, SDPatternOperator OpNode,
2259 [(set KRC:$dst, (OpNode KRC:$src1, KRC:$src2))]>;
2263 SDPatternOperator OpNode, bit IsCommutable,
2265 defm B : avx512_mask_binop<opc, !strconcat(OpcodeStr, "b"), VK8, OpNode,
2267 defm W : avx512_mask_binop<opc, !strconcat(OpcodeStr, "w"), VK16, OpNode,
2269 defm D : avx512_mask_binop<opc, !strconcat(OpcodeStr, "d"), VK32, OpNode,
2271 defm Q : avx512_mask_binop<opc, !strconcat(OpcodeStr, "q"), VK64, OpNode,
2300 multiclass avx512_binop_pat<SDPatternOperator OpNode, Instruction Inst> {
2304 def : Pat<(OpNode VK8:$src1, VK8:$src2),
2310 def : Pat<(OpNode VK1:$src1, VK1:$src2),
2314 def : Pat<(OpNode VK2:$src1, VK2:$src2),
2318 def : Pat<(OpNode VK4:$src1, VK4:$src2),
2379 SDNode OpNode, Predicate prd> {
2383 [(set EFLAGS, (OpNode KRC:$src1, KRC:$src2))]>;
2386 multiclass avx512_mask_testop_w<bits<8> opc, string OpcodeStr, SDNode OpNode,
2388 defm B : avx512_mask_testop<opc, OpcodeStr#"b", VK8, OpNode, HasDQI>,
2390 defm W : avx512_mask_testop<opc, OpcodeStr#"w", VK16, OpNode, prdW>,
2392 defm Q : avx512_mask_testop<opc, OpcodeStr#"q", VK64, OpNode, HasBWI>,
2394 defm D : avx512_mask_testop<opc, OpcodeStr#"d", VK32, OpNode, HasBWI>,
2403 SDNode OpNode> {
2408 [(set KRC:$dst, (OpNode KRC:$src, (i8 imm:$imm)))]>;
2412 SDNode OpNode> {
2413 defm W : avx512_mask_shiftop<opc1, !strconcat(OpcodeStr, "w"), VK16, OpNode>,
2416 defm B : avx512_mask_shiftop<opc1, !strconcat(OpcodeStr, "b"), VK8, OpNode>,
2419 defm Q : avx512_mask_shiftop<opc2, !strconcat(OpcodeStr, "q"), VK64, OpNode>,
2421 defm D : avx512_mask_shiftop<opc2, !strconcat(OpcodeStr, "d"), VK32, OpNode>,
2991 multiclass avx512_move_scalar <string asm, SDNode OpNode,
2996 (_.VT (OpNode (_.VT _.RC:$src1),
3004 (_.VT (OpNode (_.VT _.RC:$src1),
3011 [(set _.RC:$dst, (_.VT (OpNode _.RC:$src1,
3403 multiclass avx512_binop_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
3409 (_.VT (OpNode _.RC:$src1, _.RC:$src2)),
3416 (_.VT (OpNode _.RC:$src1,
3422 multiclass avx512_binop_rmb<bits<8> opc, string OpcodeStr, SDNode OpNode,
3425 avx512_binop_rm<opc, OpcodeStr, OpNode, _, itins, IsCommutable> {
3430 (_.VT (OpNode _.RC:$src1,
3437 multiclass avx512_binop_rm_vl<bits<8> opc, string OpcodeStr, SDNode OpNode,
3441 defm Z : avx512_binop_rm<opc, OpcodeStr, OpNode, VTInfo.info512, itins,
3445 defm Z256 : avx512_binop_rm<opc, OpcodeStr, OpNode, VTInfo.info256, itins,
3447 defm Z128 : avx512_binop_rm<opc, OpcodeStr, OpNode, VTInfo.info128, itins,
3452 multiclass avx512_binop_rmb_vl<bits<8> opc, string OpcodeStr, SDNode OpNode,
3456 defm Z : avx512_binop_rmb<opc, OpcodeStr, OpNode, VTInfo.info512, itins,
3460 defm Z256 : avx512_binop_rmb<opc, OpcodeStr, OpNode, VTInfo.info256, itins,
3462 defm Z128 : avx512_binop_rmb<opc, OpcodeStr, OpNode, VTInfo.info128, itins,
3467 multiclass avx512_binop_rm_vl_q<bits<8> opc, string OpcodeStr, SDNode OpNode,
3470 defm NAME : avx512_binop_rmb_vl<opc, OpcodeStr, OpNode, avx512vl_i64_info,
3475 multiclass avx512_binop_rm_vl_d<bits<8> opc, string OpcodeStr, SDNode OpNode,
3478 defm NAME : avx512_binop_rmb_vl<opc, OpcodeStr, OpNode, avx512vl_i32_info,
3482 multiclass avx512_binop_rm_vl_w<bits<8> opc, string OpcodeStr, SDNode OpNode,
3485 defm NAME : avx512_binop_rm_vl<opc, OpcodeStr, OpNode, avx512vl_i16_info,
3489 multiclass avx512_binop_rm_vl_b<bits<8> opc, string OpcodeStr, SDNode OpNode,
3492 defm NAME : avx512_binop_rm_vl<opc, OpcodeStr, OpNode, avx512vl_i8_info,
3497 SDNode OpNode, OpndItins itins, Predicate prd,
3499 defm Q : avx512_binop_rm_vl_q<opc_q, OpcodeStr#"q", OpNode, itins, prd,
3502 defm D : avx512_binop_rm_vl_d<opc_d, OpcodeStr#"d", OpNode, itins, prd,
3507 SDNode OpNode, OpndItins itins, Predicate prd,
3509 defm W : avx512_binop_rm_vl_w<opc_w, OpcodeStr#"w", OpNode, itins, prd,
3512 defm B : avx512_binop_rm_vl_b<opc_b, OpcodeStr#"b", OpNode, itins, prd,
3518 string OpcodeStr, SDNode OpNode,
3520 defm NAME : avx512_binop_rm_vl_dq<opc_d, opc_q, OpcodeStr, OpNode,
3522 avx512_binop_rm_vl_bw<opc_b, opc_w, OpcodeStr, OpNode,
3527 SDNode OpNode,X86VectorVTInfo _Src,
3533 (_Dst.VT (OpNode
3541 (_Dst.VT (OpNode (_Src.VT _Src.RC:$src1),
3551 (_Dst.VT (OpNode (_Src.VT _Src.RC:$src1), (bitconvert
3587 SDNode OpNode, Predicate prd, bit IsCommutable = 0> {
3589 defm NAME#Z : avx512_binop_rm2<opc, OpcodeStr, itins, OpNode,
3594 defm NAME#Z256 : avx512_binop_rm2<opc, OpcodeStr, itins, OpNode,
3598 defm NAME#Z128 : avx512_binop_rm2<opc, OpcodeStr, itins, OpNode,
3615 multiclass avx512_packs_rmb<bits<8> opc, string OpcodeStr, SDNode OpNode,
3622 (_Dst.VT (OpNode (_Src.VT _Src.RC:$src1), (bitconvert
3629 SDNode OpNode,X86VectorVTInfo _Src,
3634 (_Dst.VT (OpNode
3641 (_Dst.VT (OpNode (_Src.VT _Src.RC:$src1),
3647 SDNode OpNode> {
3649 defm NAME#Z : avx512_packs_rm<opc, OpcodeStr, OpNode, v16i32_info,
3651 avx512_packs_rmb<opc, OpcodeStr, OpNode, v16i32_info,
3654 defm NAME#Z256 : avx512_packs_rm<opc, OpcodeStr, OpNode, v8i32x_info,
3656 avx512_packs_rmb<opc, OpcodeStr, OpNode, v8i32x_info,
3658 defm NAME#Z128 : avx512_packs_rm<opc, OpcodeStr, OpNode, v4i32x_info,
3660 avx512_packs_rmb<opc, OpcodeStr, OpNode, v4i32x_info,
3665 SDNode OpNode> {
3667 defm NAME#Z : avx512_packs_rm<opc, OpcodeStr, OpNode, v32i16_info,
3670 defm NAME#Z256 : avx512_packs_rm<opc, OpcodeStr, OpNode, v16i16x_info,
3672 defm NAME#Z128 : avx512_packs_rm<opc, OpcodeStr, OpNode, v8i16x_info,
3678 SDNode OpNode, AVX512VLVectorVTInfo _Src,
3681 defm NAME#Z : avx512_packs_rm<opc, OpcodeStr, OpNode, _Src.info512,
3684 defm NAME#Z256 : avx512_packs_rm<opc, OpcodeStr, OpNode, _Src.info256,
3686 defm NAME#Z128 : avx512_packs_rm<opc, OpcodeStr, OpNode, _Src.info128,
3745 SDNode OpNode, SDNode VecNode, OpndItins itins,
3767 [(set _.FRC:$dst, (OpNode _.FRC:$src1, _.FRC:$src2))],
3772 [(set _.FRC:$dst, (OpNode _.FRC:$src1,
3797 multiclass avx512_binop_s_round<bits<8> opc, string OpcodeStr, SDNode OpNode,
3800 defm SSZ : avx512_fp_scalar<opc, OpcodeStr#"ss", f32x_info, OpNode, VecNode,
3805 defm SDZ : avx512_fp_scalar<opc, OpcodeStr#"sd", f64x_info, OpNode, VecNode,
3812 multiclass avx512_binop_s_sae<bits<8> opc, string OpcodeStr, SDNode OpNode,
3815 defm SSZ : avx512_fp_scalar<opc, OpcodeStr#"ss", f32x_info, OpNode, VecNode,
3820 defm SDZ : avx512_fp_scalar<opc, OpcodeStr#"sd", f64x_info, OpNode, VecNode,
3836 X86VectorVTInfo _, SDNode OpNode, OpndItins itins> {
3841 [(set _.FRC:$dst, (OpNode _.FRC:$src1, _.FRC:$src2))],
3846 [(set _.FRC:$dst, (OpNode _.FRC:$src1,
3866 multiclass avx512_fp_packed<bits<8> opc, string OpcodeStr, SDNode OpNode,
3871 (_.VT (OpNode _.RC:$src1, _.RC:$src2))>, EVEX_4V;
3875 (OpNode _.RC:$src1, (_.LdFrag addr:$src2))>, EVEX_4V;
3880 (OpNode _.RC:$src1, (_.VT (X86VBroadcast
3904 multiclass avx512_fp_binop_p<bits<8> opc, string OpcodeStr, SDNode OpNode,
3907 defm PSZ : avx512_fp_packed<opc, OpcodeStr, OpNode, v16f32_info,
3910 defm PDZ : avx512_fp_packed<opc, OpcodeStr, OpNode, v8f64_info,
3917 defm PSZ128 : avx512_fp_packed<opc, OpcodeStr, OpNode, v4f32x_info,
3920 defm PSZ256 : avx512_fp_packed<opc, OpcodeStr, OpNode, v8f32x_info,
3923 defm PDZ128 : avx512_fp_packed<opc, OpcodeStr, OpNode, v2f64x_info,
3926 defm PDZ256 : avx512_fp_packed<opc, OpcodeStr, OpNode, v4f64x_info,
3967 multiclass avx512_fp_scalef_p<bits<8> opc, string OpcodeStr, SDNode OpNode,
3972 (_.VT (OpNode _.RC:$src1, _.RC:$src2, (i32 FROUND_CURRENT)))>, EVEX_4V;
3976 (OpNode _.RC:$src1, (_.LdFrag addr:$src2), (i32 FROUND_CURRENT))>, EVEX_4V;
3981 (OpNode _.RC:$src1, (_.VT (X86VBroadcast
3986 multiclass avx512_fp_scalef_scalar<bits<8> opc, string OpcodeStr, SDNode OpNode,
3991 (_.VT (OpNode _.RC:$src1, _.RC:$src2, (i32 FROUND_CURRENT)))>;
3995 (OpNode _.RC:$src1,
4000 multiclass avx512_fp_scalef_all<bits<8> opc, bits<8> opcScaler, string OpcodeStr, SDNode OpNode, SD…
4001 defm PSZ : avx512_fp_scalef_p<opc, OpcodeStr, OpNode, v16f32_info>,
4002 avx512_fp_round_packed<opc, OpcodeStr, OpNode, v16f32_info>,
4004 defm PDZ : avx512_fp_scalef_p<opc, OpcodeStr, OpNode, v8f64_info>,
4005 avx512_fp_round_packed<opc, OpcodeStr, OpNode, v8f64_info>,
4016 defm PSZ128 : avx512_fp_scalef_p<opc, OpcodeStr, OpNode, v4f32x_info>,
4018 defm PSZ256 : avx512_fp_scalef_p<opc, OpcodeStr, OpNode, v8f32x_info>,
4020 defm PDZ128 : avx512_fp_scalef_p<opc, OpcodeStr, OpNode, v2f64x_info>,
4022 defm PDZ256 : avx512_fp_scalef_p<opc, OpcodeStr, OpNode, v4f64x_info>,
4032 multiclass avx512_vptest<bits<8> opc, string OpcodeStr, SDNode OpNode,
4038 (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2))>,
4043 (OpNode (_.VT _.RC:$src1),
4049 multiclass avx512_vptest_mb<bits<8> opc, string OpcodeStr, SDNode OpNode,
4055 (OpNode (_.VT _.RC:$src1), (_.VT (X86VBroadcast
4061 multiclass avx512_vptest_lowering<SDNode OpNode, X86VectorVTInfo ExtendInfo,
4063 def : Pat<(_.KVT (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2))),
4073 multiclass avx512_vptest_dq_sizes<bits<8> opc, string OpcodeStr, SDNode OpNode,
4076 defm Z : avx512_vptest<opc, OpcodeStr, OpNode, _.info512>,
4077 avx512_vptest_mb<opc, OpcodeStr, OpNode, _.info512>, EVEX_V512;
4080 defm Z256 : avx512_vptest<opc, OpcodeStr, OpNode, _.info256>,
4081 avx512_vptest_mb<opc, OpcodeStr, OpNode, _.info256>, EVEX_V256;
4082 defm Z128 : avx512_vptest<opc, OpcodeStr, OpNode, _.info128>,
4083 avx512_vptest_mb<opc, OpcodeStr, OpNode, _.info128>, EVEX_V128;
4086 defm Z256_Alt : avx512_vptest_lowering< OpNode, _.info512, _.info256, Suffix>;
4087 defm Z128_Alt : avx512_vptest_lowering< OpNode, _.info512, _.info128, Suffix>;
4091 multiclass avx512_vptest_dq<bits<8> opc, string OpcodeStr, SDNode OpNode> {
4092 defm D : avx512_vptest_dq_sizes<opc, OpcodeStr#"d", OpNode,
4094 defm Q : avx512_vptest_dq_sizes<opc, OpcodeStr#"q", OpNode,
4099 SDNode OpNode> {
4101 defm WZ: avx512_vptest<opc, OpcodeStr#"w", OpNode, v32i16_info>,
4103 defm BZ: avx512_vptest<opc, OpcodeStr#"b", OpNode, v64i8_info>,
4108 defm WZ256: avx512_vptest<opc, OpcodeStr#"w", OpNode, v16i16x_info>,
4110 defm WZ128: avx512_vptest<opc, OpcodeStr#"w", OpNode, v8i16x_info>,
4112 defm BZ256: avx512_vptest<opc, OpcodeStr#"b", OpNode, v32i8x_info>,
4114 defm BZ128: avx512_vptest<opc, OpcodeStr#"b", OpNode, v16i8x_info>,
4119 defm BZ256_Alt : avx512_vptest_lowering< OpNode, v64i8_info, v32i8x_info, "B">;
4120 defm BZ128_Alt : avx512_vptest_lowering< OpNode, v64i8_info, v16i8x_info, "B">;
4121 defm WZ256_Alt : avx512_vptest_lowering< OpNode, v32i16_info, v16i16x_info, "W">;
4122 defm WZ128_Alt : avx512_vptest_lowering< OpNode, v32i16_info, v8i16x_info, "W">;
4128 SDNode OpNode> :
4129 avx512_vptest_wb <opc_wb, OpcodeStr, OpNode>,
4130 avx512_vptest_dq<opc_dq, OpcodeStr, OpNode>;
4140 string OpcodeStr, SDNode OpNode, X86VectorVTInfo _> {
4144 (_.VT (OpNode _.RC:$src1, (i8 imm:$src2))),
4149 (_.VT (OpNode (_.VT (bitconvert (_.LdFrag addr:$src1))),
4155 string OpcodeStr, SDNode OpNode, X86VectorVTInfo _> {
4159 (_.VT (OpNode (X86VBroadcast (_.ScalarLdFrag addr:$src1)), (i8 imm:$src2))),
4163 multiclass avx512_shift_rrm<bits<8> opc, string OpcodeStr, SDNode OpNode,
4169 (_.VT (OpNode _.RC:$src1, (SrcVT VR128X:$src2))),
4174 (_.VT (OpNode _.RC:$src1, (bc_frag (loadv2i64 addr:$src2)))),
4179 multiclass avx512_shift_sizes<bits<8> opc, string OpcodeStr, SDNode OpNode,
4183 defm Z : avx512_shift_rrm<opc, OpcodeStr, OpNode, SrcVT, bc_frag,
4187 defm Z256 : avx512_shift_rrm<opc, OpcodeStr, OpNode, SrcVT, bc_frag,
4190 defm Z128 : avx512_shift_rrm<opc, OpcodeStr, OpNode, SrcVT, bc_frag,
4197 string OpcodeStr, SDNode OpNode> {
4198 defm D : avx512_shift_sizes<opcd, OpcodeStr#"d", OpNode, v4i32, bc_v4i32,
4200 defm Q : avx512_shift_sizes<opcq, OpcodeStr#"q", OpNode, v2i64, bc_v2i64,
4202 defm W : avx512_shift_sizes<opcw, OpcodeStr#"w", OpNode, v8i16, bc_v8i16,
4207 string OpcodeStr, SDNode OpNode,
4210 defm Z: avx512_shift_rmi<opc, ImmFormR, ImmFormM, OpcodeStr, OpNode,
4212 avx512_shift_rmbi<opc, ImmFormM, OpcodeStr, OpNode,
4215 defm Z256: avx512_shift_rmi<opc, ImmFormR, ImmFormM, OpcodeStr, OpNode,
4217 avx512_shift_rmbi<opc, ImmFormM, OpcodeStr, OpNode,
4219 defm Z128: avx512_shift_rmi<opc, ImmFormR, ImmFormM, OpcodeStr, OpNode,
4221 avx512_shift_rmbi<opc, ImmFormM, OpcodeStr, OpNode,
4228 string OpcodeStr, SDNode OpNode> {
4230 defm WZ: avx512_shift_rmi<opcw, ImmFormR, ImmFormM, OpcodeStr, OpNode,
4233 defm WZ256: avx512_shift_rmi<opcw, ImmFormR, ImmFormM, OpcodeStr, OpNode,
4235 defm WZ128: avx512_shift_rmi<opcw, ImmFormR, ImmFormM, OpcodeStr, OpNode,
4242 string OpcodeStr, SDNode OpNode> {
4243 defm D: avx512_shift_rmi_sizes<opcd, ImmFormR, ImmFormM, OpcodeStr#"d", OpNode,
4245 defm Q: avx512_shift_rmi_sizes<opcq, ImmFormR, ImmFormM, OpcodeStr#"q", OpNode,
4268 multiclass avx512_var_shift<bits<8> opc, string OpcodeStr, SDNode OpNode,
4273 (_.VT (OpNode _.RC:$src1, (_.VT _.RC:$src2))),
4278 (_.VT (OpNode _.RC:$src1,
4284 multiclass avx512_var_shift_mb<bits<8> opc, string OpcodeStr, SDNode OpNode,
4290 (_.VT (OpNode _.RC:$src1, (_.VT (X86VBroadcast
4295 multiclass avx512_var_shift_sizes<bits<8> opc, string OpcodeStr, SDNode OpNode,
4298 defm Z : avx512_var_shift<opc, OpcodeStr, OpNode, _.info512>,
4299 avx512_var_shift_mb<opc, OpcodeStr, OpNode, _.info512>, EVEX_V512;
4302 defm Z256 : avx512_var_shift<opc, OpcodeStr, OpNode, _.info256>,
4303 avx512_var_shift_mb<opc, OpcodeStr, OpNode, _.info256>, EVEX_V256;
4304 defm Z128 : avx512_var_shift<opc, OpcodeStr, OpNode, _.info128>,
4305 avx512_var_shift_mb<opc, OpcodeStr, OpNode, _.info128>, EVEX_V128;
4310 SDNode OpNode> {
4311 defm D : avx512_var_shift_sizes<opc, OpcodeStr#"d", OpNode,
4313 defm Q : avx512_var_shift_sizes<opc, OpcodeStr#"q", OpNode,
4318 multiclass avx512_var_shift_w_lowering<AVX512VLVectorVTInfo _, SDNode OpNode> {
4320 def : Pat<(_.info256.VT (OpNode (_.info256.VT _.info256.RC:$src1),
4328 def : Pat<(_.info128.VT (OpNode (_.info128.VT _.info128.RC:$src1),
4339 SDNode OpNode> {
4341 defm WZ: avx512_var_shift<opc, OpcodeStr, OpNode, v32i16_info>,
4345 defm WZ256: avx512_var_shift<opc, OpcodeStr, OpNode, v16i16x_info>,
4347 defm WZ128: avx512_var_shift<opc, OpcodeStr, OpNode, v8i16x_info>,
4372 multiclass avx512_vperm_dq_sizes<bits<8> opc, string OpcodeStr, SDNode OpNode,
4375 defm Z : avx512_var_shift<opc, OpcodeStr, OpNode, _.info512>,
4376 avx512_var_shift_mb<opc, OpcodeStr, OpNode, _.info512>, EVEX_V512;
4379 defm Z256 : avx512_var_shift<opc, OpcodeStr, OpNode, _.info256>,
4380 avx512_var_shift_mb<opc, OpcodeStr, OpNode, _.info256>, EVEX_V256;
4384 string OpcodeStr, SDNode OpNode,
4387 defm Z: avx512_shift_rmi<opc, ImmFormR, ImmFormM, OpcodeStr, OpNode,
4389 avx512_shift_rmbi<opc, ImmFormM, OpcodeStr, OpNode,
4392 defm Z256: avx512_shift_rmi<opc, ImmFormR, ImmFormM, OpcodeStr, OpNode,
4394 avx512_shift_rmbi<opc, ImmFormM, OpcodeStr, OpNode,
4399 Predicate prd, SDNode OpNode,
4402 defm Z: avx512_var_shift<opc, OpcodeStr, OpNode, _.info512>,
4405 defm Z256: avx512_var_shift<opc, OpcodeStr, OpNode, _.info256>,
4407 defm Z128: avx512_var_shift<opc, OpcodeStr, OpNode, _.info128>,
4436 multiclass avx512_permil_vec<bits<8> OpcVar, string OpcodeStr, SDNode OpNode,
4441 (_.VT (OpNode _.RC:$src1,
4447 (_.VT (OpNode
4455 (_.VT (OpNode
4501 multiclass avx512_pshufb_sizes<bits<8> opc, string OpcodeStr, SDNode OpNode> {
4503 defm Z: avx512_var_shift<opc, OpcodeStr, OpNode, v64i8_info>, EVEX_V512;
4506 defm Z256: avx512_var_shift<opc, OpcodeStr, OpNode, v32i8x_info>, EVEX_V256;
4507 defm Z128: avx512_var_shift<opc, OpcodeStr, OpNode, v16i8x_info>, EVEX_V128;
4543 multiclass avx512_mov_hilo_packed<bits<8> opc, string OpcodeStr, SDNode OpNode,
4550 (OpNode _.RC:$src1,
4651 multiclass avx512_fma3p_213_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
4656 (_.VT (OpNode _.RC:$src1, _.RC:$src2, _.RC:$src3))>,
4662 (_.VT (OpNode _.RC:$src1, _.RC:$src2, (_.LdFrag addr:$src3)))>,
4669 (OpNode _.RC:$src1,
4674 multiclass avx512_fma3_213_round<bits<8> opc, string OpcodeStr, SDNode OpNode,
4679 (_.VT ( OpNode _.RC:$src1, _.RC:$src2, _.RC:$src3, (i32 imm:$rc)))>,
4684 multiclass avx512_fma3p_213_common<bits<8> opc, string OpcodeStr, SDNode OpNode,
4687 defm Z : avx512_fma3p_213_rm<opc, OpcodeStr, OpNode, _.info512>,
4692 defm Z256 : avx512_fma3p_213_rm<opc, OpcodeStr, OpNode, _.info256>,
4694 defm Z128 : avx512_fma3p_213_rm<opc, OpcodeStr, OpNode, _.info128>,
4699 multiclass avx512_fma3p_213_f<bits<8> opc, string OpcodeStr, SDNode OpNode,
4701 defm PS : avx512_fma3p_213_common<opc, OpcodeStr#"ps", OpNode, OpNodeRnd,
4703 defm PD : avx512_fma3p_213_common<opc, OpcodeStr#"pd", OpNode, OpNodeRnd,
4716 multiclass avx512_fma3p_231_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
4721 (_.VT (OpNode _.RC:$src2, _.RC:$src3, _.RC:$src1))>,
4727 (_.VT (OpNode _.RC:$src2, (_.LdFrag addr:$src3), _.RC:$src1))>,
4734 (_.VT (OpNode _.RC:$src2,
4739 multiclass avx512_fma3_231_round<bits<8> opc, string OpcodeStr, SDNode OpNode,
4744 (_.VT ( OpNode _.RC:$src2, _.RC:$src3, _.RC:$src1, (i32 imm:$rc)))>,
4749 multiclass avx512_fma3p_231_common<bits<8> opc, string OpcodeStr, SDNode OpNode,
4752 defm Z : avx512_fma3p_231_rm<opc, OpcodeStr, OpNode, _.info512>,
4757 defm Z256 : avx512_fma3p_231_rm<opc, OpcodeStr, OpNode, _.info256>,
4759 defm Z128 : avx512_fma3p_231_rm<opc, OpcodeStr, OpNode, _.info128>,
4764 multiclass avx512_fma3p_231_f<bits<8> opc, string OpcodeStr, SDNode OpNode,
4766 defm PS : avx512_fma3p_231_common<opc, OpcodeStr#"ps", OpNode, OpNodeRnd,
4768 defm PD : avx512_fma3p_231_common<opc, OpcodeStr#"pd", OpNode, OpNodeRnd,
4780 multiclass avx512_fma3p_132_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
4785 (_.VT (OpNode _.RC:$src1, _.RC:$src2, _.RC:$src3))>,
4791 (_.VT (OpNode _.RC:$src1, (_.LdFrag addr:$src2), _.RC:$src3))>,
4798 (_.VT (OpNode _.RC:$src1,
4803 multiclass avx512_fma3_132_round<bits<8> opc, string OpcodeStr, SDNode OpNode,
4808 (_.VT ( OpNode _.RC:$src1, _.RC:$src2, _.RC:$src3, (i32 imm:$rc)))>,
4813 multiclass avx512_fma3p_132_common<bits<8> opc, string OpcodeStr, SDNode OpNode,
4816 defm Z : avx512_fma3p_132_rm<opc, OpcodeStr, OpNode, _.info512>,
4821 defm Z256 : avx512_fma3p_132_rm<opc, OpcodeStr, OpNode, _.info256>,
4823 defm Z128 : avx512_fma3p_132_rm<opc, OpcodeStr, OpNode, _.info128>,
4828 multiclass avx512_fma3p_132_f<bits<8> opc, string OpcodeStr, SDNode OpNode,
4830 defm PS : avx512_fma3p_132_common<opc, OpcodeStr#"ps", OpNode, OpNodeRnd,
4832 defm PD : avx512_fma3p_132_common<opc, OpcodeStr#"pd", OpNode, OpNodeRnd,
4877 string OpcodeStr, SDNode OpNode, SDNode OpNodeRnd, X86VectorVTInfo _ ,
4886 (set _.FRC:$dst, (_.EltVT (OpNode _.FRC:$src2, _.FRC:$src1,
4888 (set _.FRC:$dst, (_.EltVT (OpNode _.FRC:$src2, _.FRC:$src1,
4898 (set _.FRC:$dst, (_.EltVT (OpNode _.FRC:$src2, _.FRC:$src3,
4900 (set _.FRC:$dst, (_.EltVT (OpNode _.FRC:$src2,
4910 (set _.FRC:$dst, (_.EltVT (OpNode _.FRC:$src1, _.FRC:$src3,
4912 (set _.FRC:$dst, (_.EltVT (OpNode _.FRC:$src1,
4917 string OpcodeStr, SDNode OpNode, SDNode OpNodeRnd>{
4919 defm NAME : avx512_fma3s_all<opc213, opc231, opc132, OpcodeStr, OpNode,
4922 defm NAME : avx512_fma3s_all<opc213, opc231, opc132, OpcodeStr, OpNode,
4937 multiclass avx512_pmadd52_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
4942 (_.VT (OpNode _.RC:$src1, _.RC:$src2, _.RC:$src3))>,
4948 (_.VT (OpNode _.RC:$src1, _.RC:$src2, (_.LdFrag addr:$src3)))>,
4955 (OpNode _.RC:$src1,
4961 multiclass avx512_pmadd52_common<bits<8> opc, string OpcodeStr, SDNode OpNode,
4964 defm Z : avx512_pmadd52_rm<opc, OpcodeStr, OpNode, _.info512>,
4968 defm Z256 : avx512_pmadd52_rm<opc, OpcodeStr, OpNode, _.info256>,
4970 defm Z128 : avx512_pmadd52_rm<opc, OpcodeStr, OpNode, _.info128>,
4984 multiclass avx512_vcvtsi<bits<8> opc, SDNode OpNode, RegisterClass SrcRC,
5003 (OpNode (DstVT.VT DstVT.RC:$src1),
5011 (OpNode (DstVT.VT DstVT.RC:$src1),
5017 multiclass avx512_vcvtsi_round<bits<8> opc, SDNode OpNode, RegisterClass SrcRC,
5024 (OpNode (DstVT.VT DstVT.RC:$src1),
5029 multiclass avx512_vcvtsi_common<bits<8> opc, SDNode OpNode, RegisterClass SrcRC,
5032 defm NAME : avx512_vcvtsi_round<opc, OpNode, SrcRC, DstVT, asm>,
5033 avx512_vcvtsi<opc, OpNode, SrcRC, DstVT, x86memop, ld_frag, asm>,
5105 X86VectorVTInfo DstVT, SDNode OpNode, string asm> {
5109 [(set DstVT.RC:$dst, (OpNode (SrcVT.VT SrcVT.RC:$src),(i32 FROUND_CURRENT)))]>,
5113 [(set DstVT.RC:$dst, (OpNode (SrcVT.VT SrcVT.RC:$src),(i32 imm:$rc)))]>,
5117 [(set DstVT.RC:$dst, (OpNode
5184 X86VectorVTInfo _DstRC, SDNode OpNode,
5189 [(set _DstRC.RC:$dst, (OpNode _SrcRC.FRC:$src))]>, EVEX;
5195 [(set _DstRC.RC:$dst, (OpNode (_SrcRC.ScalarLdFrag addr:$src)))]>,
5259 X86VectorVTInfo _Src, SDNode OpNode> {
5263 (_.VT (OpNode (_.VT _.RC:$src1),
5269 (_.VT (OpNode (_.VT _.RC:$src1),
5298 multiclass avx512_cvt_fp_scalar_sd2ss<bits<8> opc, string OpcodeStr, SDNode OpNode,
5302 defm Z : avx512_cvt_fp_scalar<opc, OpcodeStr, _dst, _src, OpNode>,
5309 multiclass avx512_cvt_fp_scalar_ss2sd<bits<8> opc, string OpcodeStr, SDNode OpNode,
5313 defm Z : avx512_cvt_fp_scalar<opc, OpcodeStr, _dst, _src, OpNode>,
5350 X86VectorVTInfo _Src, SDNode OpNode,
5356 (_.VT (OpNode (_Src.VT _Src.RC:$src)))>, EVEX;
5360 (_.VT (OpNode (_Src.VT
5366 (_.VT (OpNode (_Src.VT
5435 multiclass avx512_cvtdq2pd<bits<8> opc, string OpcodeStr, SDNode OpNode,
5439 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8f64_info, v8i32x_info, OpNode>,
5445 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4f64x_info, v4i32x_info, OpNode>,
5451 multiclass avx512_cvtdq2ps<bits<8> opc, string OpcodeStr, SDNode OpNode,
5454 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v16f32_info, v16i32_info, OpNode>,
5459 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4f32x_info, v4i32x_info, OpNode>,
5461 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v8f32x_info, v8i32x_info, OpNode>,
5468 SDNode OpNode, SDNode OpNodeRnd> {
5470 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v16i32_info, v16f32_info, OpNode>,
5475 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v4f32x_info, OpNode>,
5477 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v8i32x_info, v8f32x_info, OpNode>,
5484 SDNode OpNode, SDNode OpNodeRnd> {
5486 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v16i32_info, v16f32_info, OpNode>,
5491 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v4f32x_info, OpNode>,
5493 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v8i32x_info, v8f32x_info, OpNode>,
5500 SDNode OpNode, SDNode OpNodeRnd> {
5502 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i32x_info, v8f64_info, OpNode>,
5511 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v2f64x_info, OpNode,
5513 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v4f64x_info, OpNode,
5520 SDNode OpNode, SDNode OpNodeRnd> {
5522 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i32x_info, v8f64_info, OpNode>,
5531 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v2f64x_info, OpNode,
5533 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v4f64x_info, OpNode,
5540 SDNode OpNode, SDNode OpNodeRnd> {
5542 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i64_info, v8f64_info, OpNode>,
5547 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2i64x_info, v2f64x_info, OpNode>,
5549 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i64x_info, v4f64x_info, OpNode>,
5556 SDNode OpNode, SDNode OpNodeRnd> {
5558 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i64_info, v8f64_info, OpNode>,
5563 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2i64x_info, v2f64x_info, OpNode>,
5565 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i64x_info, v4f64x_info, OpNode>,
5572 SDNode OpNode, SDNode OpNodeRnd> {
5574 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8f64_info, v8i64_info, OpNode>,
5579 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2f64x_info, v2i64x_info, OpNode>,
5581 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4f64x_info, v4i64x_info, OpNode>,
5588 SDNode OpNode, SDNode OpNodeRnd> {
5590 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i64_info, v8f32x_info, OpNode>,
5597 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2i64x_info, v4f32x_info, OpNode,
5599 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i64x_info, v4f32x_info, OpNode>,
5606 SDNode OpNode, SDNode OpNodeRnd> {
5608 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i64_info, v8f32x_info, OpNode>,
5615 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2i64x_info, v4f32x_info, OpNode,
5617 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i64x_info, v4f32x_info, OpNode>,
5624 SDNode OpNode, SDNode OpNodeRnd> {
5626 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8f32x_info, v8i64_info, OpNode>,
5635 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4f32x_info, v2i64x_info, OpNode,
5637 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4f32x_info, v4i64x_info, OpNode,
5837 multiclass avx512_ord_cmp_sae<bits<8> opc, X86VectorVTInfo _, SDNode OpNode,
5841 [(set EFLAGS, (OpNode (_.VT _.RC:$src1), _.RC:$src2,
5891 multiclass avx512_fp14_s<bits<8> opc, string OpcodeStr, SDNode OpNode,
5897 (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2))>, EVEX_4V;
5901 (OpNode (_.VT _.RC:$src1),
5916 multiclass avx512_fp14_p<bits<8> opc, string OpcodeStr, SDNode OpNode,
5920 (_.FloatVT (OpNode _.RC:$src))>, EVEX, T8PD;
5923 (OpNode (_.FloatVT
5928 (OpNode (_.FloatVT
5933 multiclass avx512_fp14_p_vl_all<bits<8> opc, string OpcodeStr, SDNode OpNode> {
5934 defm PSZ : avx512_fp14_p<opc, !strconcat(OpcodeStr, "ps"), OpNode, v16f32_info>,
5936 defm PDZ : avx512_fp14_p<opc, !strconcat(OpcodeStr, "pd"), OpNode, v8f64_info>,
5942 OpNode, v4f32x_info>,
5945 OpNode, v8f32x_info>,
5948 OpNode, v2f64x_info>,
5951 OpNode, v4f64x_info>,
5961 SDNode OpNode> {
5966 (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2),
5972 (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2),
5978 (OpNode (_.VT _.RC:$src1),
5983 multiclass avx512_eri_s<bits<8> opc, string OpcodeStr, SDNode OpNode> {
5984 defm SS : avx512_fp28_s<opc, OpcodeStr#"ss", f32x_info, OpNode>,
5986 defm SD : avx512_fp28_s<opc, OpcodeStr#"sd", f64x_info, OpNode>,
5999 SDNode OpNode> {
6003 (OpNode (_.VT _.RC:$src), (i32 FROUND_CURRENT))>;
6007 (OpNode (_.FloatVT
6014 (OpNode (_.FloatVT
6019 SDNode OpNode> {
6023 (OpNode (_.VT _.RC:$src), (i32 FROUND_NO_EXC))>, EVEX_B;
6026 multiclass avx512_eri<bits<8> opc, string OpcodeStr, SDNode OpNode> {
6027 defm PS : avx512_fp28_p<opc, OpcodeStr#"ps", v16f32_info, OpNode>,
6028 avx512_fp28_p_round<opc, OpcodeStr#"ps", v16f32_info, OpNode>,
6030 defm PD : avx512_fp28_p<opc, OpcodeStr#"pd", v8f64_info, OpNode>,
6031 avx512_fp28_p_round<opc, OpcodeStr#"pd", v8f64_info, OpNode>,
6036 SDNode OpNode> {
6039 defm PSZ128 : avx512_fp28_p<opc, OpcodeStr#"ps", v4f32x_info, OpNode>,
6041 defm PSZ256 : avx512_fp28_p<opc, OpcodeStr#"ps", v8f32x_info, OpNode>,
6043 defm PDZ128 : avx512_fp28_p<opc, OpcodeStr#"pd", v2f64x_info, OpNode>,
6045 defm PDZ256 : avx512_fp28_p<opc, OpcodeStr#"pd", v4f64x_info, OpNode>,
6067 SDNode OpNode, X86VectorVTInfo _>{
6070 (_.FloatVT (OpNode _.RC:$src))>, EVEX;
6073 (OpNode (_.FloatVT
6079 (OpNode (_.FloatVT
6085 SDNode OpNode> {
6086 defm PSZ : avx512_sqrt_packed<opc, !strconcat(OpcodeStr, "ps"), OpNode,
6089 defm PDZ : avx512_sqrt_packed<opc, !strconcat(OpcodeStr, "pd"), OpNode,
6095 OpNode, v4f32x_info>,
6098 OpNode, v8f32x_info>,
6101 OpNode, v2f64x_info>,
6104 OpNode, v4f64x_info>,
6118 string SUFF, SDNode OpNode, SDNode OpNodeRnd> {
6153 def : Pat<(_.EltVT (OpNode _.FRC:$src)),
6157 def : Pat<(_.EltVT (OpNode (load addr:$src))),
6256 multiclass avx512_trunc_common<bits<8> opc, string OpcodeStr, SDNode OpNode,
6262 (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src1)))>,
6267 (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src1))),
6273 (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src1))),
6279 (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src1))),
6329 multiclass avx512_trunc<bits<8> opc, string OpcodeStr, SDNode OpNode,
6337 defm Z128: avx512_trunc_common<opc, OpcodeStr, OpNode, VTSrcInfo.info128,
6342 defm Z256: avx512_trunc_common<opc, OpcodeStr, OpNode, VTSrcInfo.info256,
6348 defm Z: avx512_trunc_common<opc, OpcodeStr, OpNode, VTSrcInfo.info512,
6354 multiclass avx512_trunc_sat<bits<8> opc, string OpcodeStr, SDNode OpNode,
6361 defm Z128: avx512_trunc_common<opc, OpcodeStr, OpNode, VTSrcInfo.info128,
6366 defm Z256: avx512_trunc_common<opc, OpcodeStr, OpNode, VTSrcInfo.info256,
6372 defm Z: avx512_trunc_common<opc, OpcodeStr, OpNode, VTSrcInfo.info512,
6378 multiclass avx512_trunc_qb<bits<8> opc, string OpcodeStr, SDNode OpNode> {
6379 defm NAME: avx512_trunc<opc, OpcodeStr, OpNode, avx512vl_i64_info,
6383 multiclass avx512_trunc_sat_qb<bits<8> opc, string sat, SDNode OpNode> {
6384 defm NAME: avx512_trunc_sat<opc, "vpmov"##sat##"qb", OpNode, avx512vl_i64_info,
6389 multiclass avx512_trunc_qw<bits<8> opc, string OpcodeStr, SDNode OpNode> {
6390 defm NAME: avx512_trunc<opc, OpcodeStr, OpNode, avx512vl_i64_info,
6394 multiclass avx512_trunc_sat_qw<bits<8> opc, string sat, SDNode OpNode> {
6395 defm NAME: avx512_trunc_sat<opc, "vpmov"##sat##"qw", OpNode, avx512vl_i64_info,
6400 multiclass avx512_trunc_qd<bits<8> opc, string OpcodeStr, SDNode OpNode> {
6401 defm NAME: avx512_trunc<opc, OpcodeStr, OpNode, avx512vl_i64_info,
6405 multiclass avx512_trunc_sat_qd<bits<8> opc, string sat, SDNode OpNode> {
6406 defm NAME: avx512_trunc_sat<opc, "vpmov"##sat##"qd", OpNode, avx512vl_i64_info,
6411 multiclass avx512_trunc_db<bits<8> opc, string OpcodeStr, SDNode OpNode> {
6412 defm NAME: avx512_trunc<opc, OpcodeStr, OpNode, avx512vl_i32_info,
6416 multiclass avx512_trunc_sat_db<bits<8> opc, string sat, SDNode OpNode> {
6417 defm NAME: avx512_trunc_sat<opc, "vpmov"##sat##"db", OpNode, avx512vl_i32_info,
6422 multiclass avx512_trunc_dw<bits<8> opc, string OpcodeStr, SDNode OpNode> {
6423 defm NAME: avx512_trunc<opc, OpcodeStr, OpNode, avx512vl_i32_info,
6427 multiclass avx512_trunc_sat_dw<bits<8> opc, string sat, SDNode OpNode> {
6428 defm NAME: avx512_trunc_sat<opc, "vpmov"##sat##"dw", OpNode, avx512vl_i32_info,
6433 multiclass avx512_trunc_wb<bits<8> opc, string OpcodeStr, SDNode OpNode> {
6434 defm NAME: avx512_trunc<opc, OpcodeStr, OpNode, avx512vl_i16_info,
6438 multiclass avx512_trunc_sat_wb<bits<8> opc, string sat, SDNode OpNode> {
6439 defm NAME: avx512_trunc_sat<opc, "vpmov"##sat##"wb", OpNode, avx512vl_i16_info,
6487 X86MemOperand x86memop, PatFrag LdFrag, SDPatternOperator OpNode>{
6490 (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src)))>,
6500 SDPatternOperator OpNode,
6504 v16i8x_info, i64mem, LdFrag, OpNode>,
6508 v16i8x_info, i128mem, LdFrag, OpNode>,
6513 v32i8x_info, i256mem, LdFrag, OpNode>,
6519 SDPatternOperator OpNode,
6523 v16i8x_info, i32mem, LdFrag, OpNode>,
6527 v16i8x_info, i64mem, LdFrag, OpNode>,
6532 v16i8x_info, i128mem, LdFrag, OpNode>,
6538 SDPatternOperator OpNode,
6542 v16i8x_info, i16mem, LdFrag, OpNode>,
6546 v16i8x_info, i32mem, LdFrag, OpNode>,
6551 v16i8x_info, i64mem, LdFrag, OpNode>,
6557 SDPatternOperator OpNode,
6561 v8i16x_info, i64mem, LdFrag, OpNode>,
6565 v8i16x_info, i128mem, LdFrag, OpNode>,
6570 v16i16x_info, i256mem, LdFrag, OpNode>,
6576 SDPatternOperator OpNode,
6580 v8i16x_info, i32mem, LdFrag, OpNode>,
6584 v8i16x_info, i64mem, LdFrag, OpNode>,
6589 v8i16x_info, i128mem, LdFrag, OpNode>,
6595 SDPatternOperator OpNode,
6600 v4i32x_info, i64mem, LdFrag, OpNode>,
6604 v4i32x_info, i128mem, LdFrag, OpNode>,
6609 v8i32x_info, i256mem, LdFrag, OpNode>,
7002 multiclass avx512_unary_fp_packed_imm<bits<8> opc, string OpcodeStr, SDNode OpNode,
7007 (OpNode (_.VT _.RC:$src1),
7013 (OpNode (_.VT (bitconvert (_.LdFrag addr:$src1))),
7020 (OpNode (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src1))),
7027 SDNode OpNode, X86VectorVTInfo _>{
7032 (OpNode (_.VT _.RC:$src1),
7038 AVX512VLVectorVTInfo _, bits<8> opc, SDNode OpNode, Predicate prd>{
7040 defm Z : avx512_unary_fp_packed_imm<opc, OpcodeStr, OpNode, _.info512>,
7041 avx512_unary_fp_sae_packed_imm<opc, OpcodeStr, OpNode, _.info512>,
7045 defm Z128 : avx512_unary_fp_packed_imm<opc, OpcodeStr, OpNode, _.info128>,
7047 defm Z256 : avx512_unary_fp_packed_imm<opc, OpcodeStr, OpNode, _.info256>,
7056 multiclass avx512_fp_packed_imm<bits<8> opc, string OpcodeStr, SDNode OpNode,
7061 (OpNode (_.VT _.RC:$src1),
7068 (OpNode (_.VT _.RC:$src1),
7076 (OpNode (_.VT _.RC:$src1),
7084 multiclass avx512_3Op_rm_imm8<bits<8> opc, string OpcodeStr, SDNode OpNode,
7090 (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src1),
7096 (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src1),
7105 multiclass avx512_3Op_imm8<bits<8> opc, string OpcodeStr, SDNode OpNode,
7107 avx512_3Op_rm_imm8<opc, OpcodeStr, OpNode, _, _>{
7113 (OpNode (_.VT _.RC:$src1),
7121 multiclass avx512_fp_scalar_imm<bits<8> opc, string OpcodeStr, SDNode OpNode,
7127 (OpNode (_.VT _.RC:$src1),
7134 (OpNode (_.VT _.RC:$src1),
7150 SDNode OpNode, X86VectorVTInfo _>{
7155 (OpNode (_.VT _.RC:$src1),
7162 SDNode OpNode, X86VectorVTInfo _> {
7167 (OpNode (_.VT _.RC:$src1),
7174 AVX512VLVectorVTInfo _, bits<8> opc, SDNode OpNode, Predicate prd>{
7176 defm Z : avx512_fp_packed_imm<opc, OpcodeStr, OpNode, _.info512>,
7177 avx512_fp_sae_packed_imm<opc, OpcodeStr, OpNode, _.info512>,
7182 defm Z128 : avx512_fp_packed_imm<opc, OpcodeStr, OpNode, _.info128>,
7184 defm Z256 : avx512_fp_packed_imm<opc, OpcodeStr, OpNode, _.info256>,
7189 multiclass avx512_common_3Op_rm_imm8<bits<8> opc, SDNode OpNode, string OpStr,
7192 defm Z : avx512_3Op_rm_imm8<opc, OpStr, OpNode, DestInfo.info512,
7196 defm Z128 : avx512_3Op_rm_imm8<opc, OpStr, OpNode, DestInfo.info128,
7198 defm Z256 : avx512_3Op_rm_imm8<opc, OpStr, OpNode, DestInfo.info256,
7204 bits<8> opc, SDNode OpNode>{
7206 defm Z : avx512_3Op_imm8<opc, OpcodeStr, OpNode, _.info512>, EVEX_V512;
7209 defm Z128 : avx512_3Op_imm8<opc, OpcodeStr, OpNode, _.info128>, EVEX_V128;
7210 defm Z256 : avx512_3Op_imm8<opc, OpcodeStr, OpNode, _.info256>, EVEX_V256;
7215 X86VectorVTInfo _, bits<8> opc, SDNode OpNode, Predicate prd>{
7217 defm Z128 : avx512_fp_scalar_imm<opc, OpcodeStr, OpNode, _>,
7218 avx512_fp_sae_scalar_imm<opc, OpcodeStr, OpNode, _>;
7223 bits<8> opcPs, bits<8> opcPd, SDNode OpNode, Predicate prd>{
7225 opcPs, OpNode, prd>, EVEX_CD8<32, CD8VF>;
7227 opcPd, OpNode, prd>, EVEX_CD8<64, CD8VF>, VEX_W;
7268 bits<8> opc, SDNode OpNode = X86Shuf128>{
7270 defm Z : avx512_3Op_imm8<opc, OpcodeStr, OpNode, _.info512>, EVEX_V512;
7274 defm Z256 : avx512_3Op_imm8<opc, OpcodeStr, OpNode, _.info256>, EVEX_V256;
7345 multiclass avx512_unary_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
7350 (_.VT (OpNode _.RC:$src1))>, EVEX, AVX5128IBase;
7355 (_.VT (OpNode (bitconvert (_.LdFrag addr:$src1))))>,
7359 multiclass avx512_unary_rmb<bits<8> opc, string OpcodeStr, SDNode OpNode,
7361 avx512_unary_rm<opc, OpcodeStr, OpNode, _> {
7366 (_.VT (OpNode (X86VBroadcast
7371 multiclass avx512_unary_rm_vl<bits<8> opc, string OpcodeStr, SDNode OpNode,
7374 defm Z : avx512_unary_rm<opc, OpcodeStr, OpNode, VTInfo.info512>, EVEX_V512;
7377 defm Z256 : avx512_unary_rm<opc, OpcodeStr, OpNode, VTInfo.info256>,
7379 defm Z128 : avx512_unary_rm<opc, OpcodeStr, OpNode, VTInfo.info128>,
7384 multiclass avx512_unary_rmb_vl<bits<8> opc, string OpcodeStr, SDNode OpNode,
7387 defm Z : avx512_unary_rmb<opc, OpcodeStr, OpNode, VTInfo.info512>,
7391 defm Z256 : avx512_unary_rmb<opc, OpcodeStr, OpNode, VTInfo.info256>,
7393 defm Z128 : avx512_unary_rmb<opc, OpcodeStr, OpNode, VTInfo.info128>,
7399 SDNode OpNode, Predicate prd> {
7400 defm Q : avx512_unary_rmb_vl<opc_q, OpcodeStr#"q", OpNode, avx512vl_i64_info,
7402 defm D : avx512_unary_rmb_vl<opc_d, OpcodeStr#"d", OpNode, avx512vl_i32_info,
7407 SDNode OpNode, Predicate prd> {
7408 defm W : avx512_unary_rm_vl<opc_w, OpcodeStr#"w", OpNode, avx512vl_i16_info, prd>;
7409 defm B : avx512_unary_rm_vl<opc_b, OpcodeStr#"b", OpNode, avx512vl_i8_info, prd>;
7414 string OpcodeStr, SDNode OpNode> {
7415 defm NAME : avx512_unary_rm_vl_dq<opc_d, opc_q, OpcodeStr, OpNode,
7417 avx512_unary_rm_vl_bw<opc_b, opc_w, OpcodeStr, OpNode,
7443 multiclass avx512_replicate<bits<8> opc, string OpcodeStr, SDNode OpNode>{
7444 defm NAME: avx512_unary_rm_vl<opc, OpcodeStr, OpNode, avx512vl_f32_info,
7455 multiclass avx512_movddup_128<bits<8> opc, string OpcodeStr, SDNode OpNode,
7459 (_.VT (OpNode (_.VT _.RC:$src)))>, EVEX;
7462 (_.VT (OpNode (_.VT (scalar_to_vector
7467 multiclass avx512_movddup_common<bits<8> opc, string OpcodeStr, SDNode OpNode,
7470 defm Z : avx512_unary_rm<opc, OpcodeStr, OpNode, VTInfo.info512>, EVEX_V512;
7473 defm Z256 : avx512_unary_rm<opc, OpcodeStr, OpNode, VTInfo.info256>,
7475 defm Z128 : avx512_movddup_128<opc, OpcodeStr, OpNode, VTInfo.info128>,
7480 multiclass avx512_movddup<bits<8> opc, string OpcodeStr, SDNode OpNode>{
7481 defm NAME: avx512_movddup_common<opc, OpcodeStr, OpNode,
7520 multiclass avx512_extract_elt_bw_m<bits<8> opc, string OpcodeStr, SDNode OpNode,
7525 [(store (_.EltVT (trunc (assertzext (OpNode (_.VT _.RC:$src1),
7587 multiclass avx512_insert_elt_m<bits<8> opc, string OpcodeStr, SDNode OpNode,
7593 (_.VT (OpNode _.RC:$src1, (LdFrag addr:$src2), imm:$src3)))]>,
7597 multiclass avx512_insert_elt_bw<bits<8> opc, string OpcodeStr, SDNode OpNode,
7604 (OpNode _.RC:$src1, GR32orGR64:$src2, imm:$src3))]>, EVEX_4V;
7606 defm NAME : avx512_insert_elt_m<opc, OpcodeStr, OpNode, _, LdFrag>;
7647 multiclass avx512_shift_packed<bits<8> opc, SDNode OpNode, Format MRMr,
7652 [(set _.RC:$dst,(_.VT (OpNode _.RC:$src1, (i8 imm:$src2))))]>;
7656 [(set _.RC:$dst,(_.VT (OpNode
7661 multiclass avx512_shift_packed_all<bits<8> opc, SDNode OpNode, Format MRMr,
7664 defm Z512 : avx512_shift_packed<opc, OpNode, MRMr, MRMm,
7667 defm Z256 : avx512_shift_packed<opc, OpNode, MRMr, MRMm,
7669 defm Z128 : avx512_shift_packed<opc, OpNode, MRMr, MRMm,
7679 multiclass avx512_psadbw_packed<bits<8> opc, SDNode OpNode,
7686 (OpNode (_src.VT _src.RC:$src1),
7692 (OpNode (_src.VT _src.RC:$src1),
7697 multiclass avx512_psadbw_packed_all<bits<8> opc, SDNode OpNode,
7700 defm Z512 : avx512_psadbw_packed<opc, OpNode, OpcodeStr, v8i64_info,
7703 defm Z256 : avx512_psadbw_packed<opc, OpNode, OpcodeStr, v4i64x_info,
7705 defm Z128 : avx512_psadbw_packed<opc, OpNode, OpcodeStr, v2i64x_info,
7713 multiclass avx512_ternlog<bits<8> opc, string OpcodeStr, SDNode OpNode,
7719 (OpNode (_.VT _.RC:$src1),
7726 (OpNode (_.VT _.RC:$src1),
7735 (OpNode (_.VT _.RC:$src1),
7759 multiclass avx512_fixupimm_packed<bits<8> opc, string OpcodeStr, SDNode OpNode,
7765 (OpNode (_.VT _.RC:$src1),
7773 (OpNode (_.VT _.RC:$src1),
7782 (OpNode (_.VT _.RC:$src1),
7791 SDNode OpNode, X86VectorVTInfo _>{
7797 (OpNode (_.VT _.RC:$src1),
7805 multiclass avx512_fixupimm_scalar<bits<8> opc, string OpcodeStr, SDNode OpNode,
7811 (OpNode (_.VT _.RC:$src1),
7821 (OpNode (_.VT _.RC:$src1),
7829 (OpNode (_.VT _.RC:$src1),