Lines Matching refs:opc
827 multiclass avx512_broadcast_rm<bits<8> opc, string OpcodeStr,
830 defm r : AVX512_maskable<opc, MRMSrcReg, DestInfo, (outs DestInfo.RC:$dst),
835 defm m : AVX512_maskable<opc, MRMSrcMem, DestInfo, (outs DestInfo.RC:$dst),
842 multiclass avx512_fp_broadcast_vl<bits<8> opc, string OpcodeStr,
844 defm Z : avx512_broadcast_rm<opc, OpcodeStr, _.info512, _.info128>,
848 defm Z256 : avx512_broadcast_rm<opc, OpcodeStr, _.info256, _.info128>,
916 multiclass avx512_int_broadcast_reg<bits<8> opc, X86VectorVTInfo _,
918 defm r : AVX512_maskable_in_asm<opc, MRMSrcReg, _, (outs _.RC:$dst),
923 multiclass avx512_int_broadcast_reg_vl<bits<8> opc, AVX512VLVectorVTInfo _,
926 defm Z : avx512_int_broadcast_reg<opc, _.info512, SrcRC>, EVEX_V512;
928 defm Z256 : avx512_int_broadcast_reg<opc, _.info256, SrcRC>, EVEX_V256;
929 defm Z128 : avx512_int_broadcast_reg<opc, _.info128, SrcRC>, EVEX_V128;
974 multiclass avx512_int_broadcast_rm_vl<bits<8> opc, string OpcodeStr,
977 defm Z : avx512_broadcast_rm<opc, OpcodeStr, _.info512, _.info128>,
984 defm Z256 : avx512_broadcast_rm<opc, OpcodeStr, _.info256, _.info128>,
987 defm Z128 : avx512_broadcast_rm<opc, OpcodeStr, _.info128, _.info128>,
1001 multiclass avx512_subvec_broadcast_rm<bits<8> opc, string OpcodeStr,
1004 def rm : AVX5128I<opc, MRMSrcMem, (outs _Dst.RC:$dst), (ins _Src.MemOp:$src),
1009 def rmk : AVX5128I<opc, MRMSrcMem, (outs _Dst.RC:$dst), (ins _Dst.KRCWM:$mask,
1014 def rmkz : AVX5128I<opc, MRMSrcMem, (outs _Dst.RC:$dst), (ins _Dst.KRCWM:$mask,
1066 multiclass avx512_broadcast_32x2<bits<8> opc, string OpcodeStr,
1070 defm r : AVX512_maskable<opc, MRMSrcReg, _Dst, (outs _Dst.RC:$dst),
1075 defm m : AVX512_maskable<opc, MRMSrcMem, _Dst, (outs _Dst.RC:$dst),
1082 multiclass avx512_common_broadcast_32x2<bits<8> opc, string OpcodeStr,
1085 defm Z : avx512_broadcast_32x2<opc, OpcodeStr, _.info512, _.info128>,
1088 defm Z256 : avx512_broadcast_32x2<opc, OpcodeStr, _.info256, _.info128>,
1092 multiclass avx512_common_broadcast_i32x2<bits<8> opc, string OpcodeStr,
1094 avx512_common_broadcast_32x2<opc, OpcodeStr, _> {
1097 defm Z128 : avx512_broadcast_32x2<opc, OpcodeStr, _.info128, _.info128,
1132 multiclass avx512_mask_broadcastm<bits<8> opc, string OpcodeStr,
1134 def rr : AVX512XS8I<opc, MRMSrcReg, (outs _.RC:$dst), (ins KRC:$src),
1139 multiclass avx512_mask_broadcast<bits<8> opc, string OpcodeStr,
1142 defm Z : avx512_mask_broadcastm<opc, OpcodeStr, VTInfo.info512, KRC>, EVEX_V512;
1144 defm Z256 : avx512_mask_broadcastm<opc, OpcodeStr, VTInfo.info256, KRC>, EVEX_V256;
1145 defm Z128 : avx512_mask_broadcastm<opc, OpcodeStr, VTInfo.info128, KRC>, EVEX_V128;
1156 multiclass avx512_perm_i<bits<8> opc, string OpcodeStr,
1159 defm rr: AVX512_maskable_3src_cast<opc, MRMSrcReg, _, IdxVT, (outs _.RC:$dst),
1166 defm rm: AVX512_maskable_3src_cast<opc, MRMSrcMem, _, IdxVT, (outs _.RC:$dst),
1174 multiclass avx512_perm_i_mb<bits<8> opc, string OpcodeStr,
1177 defm rmb: AVX512_maskable_3src_cast<opc, MRMSrcMem, _, IdxVT, (outs _.RC:$dst),
1186 multiclass avx512_perm_i_sizes<bits<8> opc, string OpcodeStr,
1189 defm NAME: avx512_perm_i<opc, OpcodeStr, VTInfo.info512,
1191 avx512_perm_i_mb<opc, OpcodeStr, VTInfo.info512,
1194 defm NAME#128: avx512_perm_i<opc, OpcodeStr, VTInfo.info128,
1196 avx512_perm_i_mb<opc, OpcodeStr, VTInfo.info128,
1198 defm NAME#256: avx512_perm_i<opc, OpcodeStr, VTInfo.info256,
1200 avx512_perm_i_mb<opc, OpcodeStr, VTInfo.info256,
1205 multiclass avx512_perm_i_sizes_w<bits<8> opc, string OpcodeStr,
1209 defm NAME: avx512_perm_i<opc, OpcodeStr, VTInfo.info512,
1212 defm NAME#128: avx512_perm_i<opc, OpcodeStr, VTInfo.info128,
1214 defm NAME#256: avx512_perm_i<opc, OpcodeStr, VTInfo.info256,
1231 multiclass avx512_perm_t<bits<8> opc, string OpcodeStr,
1234 defm rr: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),
1241 defm rm: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
1249 multiclass avx512_perm_t_mb<bits<8> opc, string OpcodeStr,
1252 defm rmb: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
1261 multiclass avx512_perm_t_sizes<bits<8> opc, string OpcodeStr,
1264 defm NAME: avx512_perm_t<opc, OpcodeStr, VTInfo.info512,
1266 avx512_perm_t_mb<opc, OpcodeStr, VTInfo.info512,
1269 defm NAME#128: avx512_perm_t<opc, OpcodeStr, VTInfo.info128,
1271 avx512_perm_t_mb<opc, OpcodeStr, VTInfo.info128,
1273 defm NAME#256: avx512_perm_t<opc, OpcodeStr, VTInfo.info256,
1275 avx512_perm_t_mb<opc, OpcodeStr, VTInfo.info256,
1280 multiclass avx512_perm_t_sizes_w<bits<8> opc, string OpcodeStr,
1284 defm NAME: avx512_perm_t<opc, OpcodeStr, VTInfo.info512,
1287 defm NAME#128: avx512_perm_t<opc, OpcodeStr, VTInfo.info128,
1289 defm NAME#256: avx512_perm_t<opc, OpcodeStr, VTInfo.info256,
1308 multiclass avx512_blendmask<bits<8> opc, string OpcodeStr, X86VectorVTInfo _> {
1310 def rr : AVX5128I<opc, MRMSrcReg, (outs _.RC:$dst),
1315 def rrk : AVX5128I<opc, MRMSrcReg, (outs _.RC:$dst),
1321 def rrkz : AVX5128I<opc, MRMSrcReg, (outs _.RC:$dst),
1327 def rm : AVX5128I<opc, MRMSrcMem, (outs _.RC:$dst),
1332 def rmk : AVX5128I<opc, MRMSrcMem, (outs _.RC:$dst),
1339 def rmkz : AVX5128I<opc, MRMSrcMem, (outs _.RC:$dst),
1347 multiclass avx512_blendmask_rmb<bits<8> opc, string OpcodeStr, X86VectorVTInfo _> {
1349 def rmbk : AVX5128I<opc, MRMSrcMem, (outs _.RC:$dst),
1358 def rmb : AVX5128I<opc, MRMSrcMem, (outs _.RC:$dst),
1367 multiclass blendmask_dq <bits<8> opc, string OpcodeStr,
1369 defm Z : avx512_blendmask <opc, OpcodeStr, VTInfo.info512>,
1370 avx512_blendmask_rmb <opc, OpcodeStr, VTInfo.info512>, EVEX_V512;
1373 defm Z256 : avx512_blendmask<opc, OpcodeStr, VTInfo.info256>,
1374 avx512_blendmask_rmb <opc, OpcodeStr, VTInfo.info256>, EVEX_V256;
1375 defm Z128 : avx512_blendmask<opc, OpcodeStr, VTInfo.info128>,
1376 avx512_blendmask_rmb <opc, OpcodeStr, VTInfo.info128>, EVEX_V128;
1380 multiclass blendmask_bw <bits<8> opc, string OpcodeStr,
1383 defm Z : avx512_blendmask <opc, OpcodeStr, VTInfo.info512>, EVEX_V512;
1386 defm Z256 : avx512_blendmask <opc, OpcodeStr, VTInfo.info256>, EVEX_V256;
1387 defm Z128 : avx512_blendmask <opc, OpcodeStr, VTInfo.info128>, EVEX_V128;
1501 multiclass avx512_icmp_packed<bits<8> opc, string OpcodeStr, SDNode OpNode,
1503 def rr : AVX512BI<opc, MRMSrcReg,
1509 def rm : AVX512BI<opc, MRMSrcMem,
1515 def rrk : AVX512BI<opc, MRMSrcReg,
1523 def rmk : AVX512BI<opc, MRMSrcMem,
1534 multiclass avx512_icmp_packed_rmb<bits<8> opc, string OpcodeStr, SDNode OpNode,
1536 avx512_icmp_packed<opc, OpcodeStr, OpNode, _> {
1538 def rmb : AVX512BI<opc, MRMSrcMem,
1545 def rmbk : AVX512BI<opc, MRMSrcMem,
1559 multiclass avx512_icmp_packed_vl<bits<8> opc, string OpcodeStr, SDNode OpNode,
1562 defm Z : avx512_icmp_packed<opc, OpcodeStr, OpNode, VTInfo.info512>,
1566 defm Z256 : avx512_icmp_packed<opc, OpcodeStr, OpNode, VTInfo.info256>,
1568 defm Z128 : avx512_icmp_packed<opc, OpcodeStr, OpNode, VTInfo.info128>,
1573 multiclass avx512_icmp_packed_rmb_vl<bits<8> opc, string OpcodeStr,
1577 defm Z : avx512_icmp_packed_rmb<opc, OpcodeStr, OpNode, VTInfo.info512>,
1581 defm Z256 : avx512_icmp_packed_rmb<opc, OpcodeStr, OpNode, VTInfo.info256>,
1583 defm Z128 : avx512_icmp_packed_rmb<opc, OpcodeStr, OpNode, VTInfo.info128>,
1630 multiclass avx512_icmp_cc<bits<8> opc, string Suffix, SDNode OpNode,
1632 def rri : AVX512AIi8<opc, MRMSrcReg,
1640 def rmi : AVX512AIi8<opc, MRMSrcMem,
1648 def rrik : AVX512AIi8<opc, MRMSrcReg,
1659 def rmik : AVX512AIi8<opc, MRMSrcMem,
1673 def rri_alt : AVX512AIi8<opc, MRMSrcReg,
1679 def rmi_alt : AVX512AIi8<opc, MRMSrcMem,
1684 def rrik_alt : AVX512AIi8<opc, MRMSrcReg,
1692 def rmik_alt : AVX512AIi8<opc, MRMSrcMem,
1702 multiclass avx512_icmp_cc_rmb<bits<8> opc, string Suffix, SDNode OpNode,
1704 avx512_icmp_cc<opc, Suffix, OpNode, _> {
1705 def rmib : AVX512AIi8<opc, MRMSrcMem,
1715 def rmibk : AVX512AIi8<opc, MRMSrcMem,
1729 def rmib_alt : AVX512AIi8<opc, MRMSrcMem,
1736 def rmibk_alt : AVX512AIi8<opc, MRMSrcMem,
1746 multiclass avx512_icmp_cc_vl<bits<8> opc, string Suffix, SDNode OpNode,
1749 defm Z : avx512_icmp_cc<opc, Suffix, OpNode, VTInfo.info512>, EVEX_V512;
1752 defm Z256 : avx512_icmp_cc<opc, Suffix, OpNode, VTInfo.info256>, EVEX_V256;
1753 defm Z128 : avx512_icmp_cc<opc, Suffix, OpNode, VTInfo.info128>, EVEX_V128;
1757 multiclass avx512_icmp_cc_rmb_vl<bits<8> opc, string Suffix, SDNode OpNode,
1760 defm Z : avx512_icmp_cc_rmb<opc, Suffix, OpNode, VTInfo.info512>,
1764 defm Z256 : avx512_icmp_cc_rmb<opc, Suffix, OpNode, VTInfo.info256>,
1766 defm Z128 : avx512_icmp_cc_rmb<opc, Suffix, OpNode, VTInfo.info128>,
1903 multiclass avx512_scalar_fpclass<bits<8> opc, string OpcodeStr, SDNode OpNode,
1906 def rr : AVX512<opc, MRMSrcReg, (outs _.KRC:$dst),//_.KRC:$dst),
1911 def rrk : AVX512<opc, MRMSrcReg, (outs _.KRC:$dst),
1919 def rm : AVX512<opc, MRMSrcMem, (outs _.KRC:$dst),
1926 def rmk : AVX512<opc, MRMSrcMem, (outs _.KRC:$dst),
1940 multiclass avx512_vector_fpclass<bits<8> opc, string OpcodeStr, SDNode OpNode,
1942 def rr : AVX512<opc, MRMSrcReg, (outs _.KRC:$dst),
1947 def rrk : AVX512<opc, MRMSrcReg, (outs _.KRC:$dst),
1955 def rm : AVX512<opc, MRMSrcMem, (outs _.KRC:$dst),
1962 def rmk : AVX512<opc, MRMSrcMem, (outs _.KRC:$dst),
1969 def rmb : AVX512<opc, MRMSrcMem, (outs _.KRC:$dst),
1978 def rmbk : AVX512<opc, MRMSrcMem, (outs _.KRC:$dst),
1992 AVX512VLVectorVTInfo _, bits<8> opc, SDNode OpNode, Predicate prd,
1995 defm Z : avx512_vector_fpclass<opc, OpcodeStr, OpNode, _.info512, "{z}",
1999 defm Z128 : avx512_vector_fpclass<opc, OpcodeStr, OpNode, _.info128, "{x}",
2001 defm Z256 : avx512_vector_fpclass<opc, OpcodeStr, OpNode, _.info256, "{y}",
2221 multiclass avx512_mask_unop<bits<8> opc, string OpcodeStr,
2225 def rr : I<opc, MRMSrcReg, (outs KRC:$dst), (ins KRC:$src),
2230 multiclass avx512_mask_unop_all<bits<8> opc, string OpcodeStr,
2232 defm B : avx512_mask_unop<opc, !strconcat(OpcodeStr, "b"), VK8, OpNode,
2234 defm W : avx512_mask_unop<opc, !strconcat(OpcodeStr, "w"), VK16, OpNode,
2236 defm D : avx512_mask_unop<opc, !strconcat(OpcodeStr, "d"), VK32, OpNode,
2238 defm Q : avx512_mask_unop<opc, !strconcat(OpcodeStr, "q"), VK64, OpNode,
2277 multiclass avx512_mask_binop<bits<8> opc, string OpcodeStr,
2281 def rr : I<opc, MRMSrcReg, (outs KRC:$dst), (ins KRC:$src1, KRC:$src2),
2287 multiclass avx512_mask_binop_all<bits<8> opc, string OpcodeStr,
2290 defm B : avx512_mask_binop<opc, !strconcat(OpcodeStr, "b"), VK8, OpNode,
2292 defm W : avx512_mask_binop<opc, !strconcat(OpcodeStr, "w"), VK16, OpNode,
2294 defm D : avx512_mask_binop<opc, !strconcat(OpcodeStr, "d"), VK32, OpNode,
2296 defm Q : avx512_mask_binop<opc, !strconcat(OpcodeStr, "q"), VK64, OpNode,
2402 multiclass avx512_mask_testop<bits<8> opc, string OpcodeStr, RegisterClass KRC,
2405 def rr : I<opc, MRMSrcReg, (outs), (ins KRC:$src1, KRC:$src2),
2410 multiclass avx512_mask_testop_w<bits<8> opc, string OpcodeStr, SDNode OpNode,
2412 defm B : avx512_mask_testop<opc, OpcodeStr#"b", VK8, OpNode, HasDQI>,
2414 defm W : avx512_mask_testop<opc, OpcodeStr#"w", VK16, OpNode, prdW>,
2416 defm Q : avx512_mask_testop<opc, OpcodeStr#"q", VK64, OpNode, HasBWI>,
2418 defm D : avx512_mask_testop<opc, OpcodeStr#"d", VK32, OpNode, HasBWI>,
2426 multiclass avx512_mask_shiftop<bits<8> opc, string OpcodeStr, RegisterClass KRC,
2429 def ri : Ii8<opc, MRMSrcReg, (outs KRC:$dst), (ins KRC:$src, u8imm:$imm),
2563 multiclass avx512_load<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
2567 def rr : AVX512PI<opc, MRMSrcReg, (outs _.RC:$dst), (ins _.RC:$src),
2570 def rrkz : AVX512PI<opc, MRMSrcReg, (outs _.RC:$dst),
2578 def rm : AVX512PI<opc, MRMSrcMem, (outs _.RC:$dst), (ins _.MemOp:$src),
2584 def rrk : AVX512PI<opc, MRMSrcReg, (outs _.RC:$dst),
2593 def rmk : AVX512PI<opc, MRMSrcMem, (outs _.RC:$dst),
2603 def rmkz : AVX512PI<opc, MRMSrcMem, (outs _.RC:$dst),
2622 multiclass avx512_alignedload_vl<bits<8> opc, string OpcodeStr,
2627 defm Z : avx512_load<opc, OpcodeStr, _.info512, _.info512.AlignedLdFrag,
2631 defm Z256 : avx512_load<opc, OpcodeStr, _.info256, _.info256.AlignedLdFrag,
2633 defm Z128 : avx512_load<opc, OpcodeStr, _.info128, _.info128.AlignedLdFrag,
2638 multiclass avx512_load_vl<bits<8> opc, string OpcodeStr,
2643 defm Z : avx512_load<opc, OpcodeStr, _.info512, _.info512.LdFrag,
2647 defm Z256 : avx512_load<opc, OpcodeStr, _.info256, _.info256.LdFrag,
2649 defm Z128 : avx512_load<opc, OpcodeStr, _.info128, _.info128.LdFrag,
2654 multiclass avx512_store<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
2657 def rr_REV : AVX512PI<opc, MRMDestReg, (outs _.RC:$dst), (ins _.RC:$src),
2660 def rrk_REV : AVX512PI<opc, MRMDestReg, (outs _.RC:$dst),
2665 def rrkz_REV : AVX512PI<opc, MRMDestReg, (outs _.RC:$dst),
2672 def mr : AVX512PI<opc, MRMDestMem, (outs), (ins _.MemOp:$dst, _.RC:$src),
2675 def mrk : AVX512PI<opc, MRMDestMem, (outs),
2687 multiclass avx512_store_vl< bits<8> opc, string OpcodeStr,
2690 defm Z : avx512_store<opc, OpcodeStr, _.info512, store,
2694 defm Z256 : avx512_store<opc, OpcodeStr, _.info256, store,
2696 defm Z128 : avx512_store<opc, OpcodeStr, _.info128, store,
2701 multiclass avx512_alignedstore_vl<bits<8> opc, string OpcodeStr,
2704 defm Z : avx512_store<opc, OpcodeStr, _.info512, alignedstore512,
2708 defm Z256 : avx512_store<opc, OpcodeStr, _.info256, alignedstore256,
2710 defm Z128 : avx512_store<opc, OpcodeStr, _.info128, alignedstore,
3243 multiclass avx512_movnt<bits<8> opc, string OpcodeStr, PatFrag st_frag,
3248 def mr : AVX512PI<opc, MRMDestMem, (outs), (ins memop:$dst, RC:$src),
3253 multiclass avx512_movnt_vl<bits<8> opc, string OpcodeStr, PatFrag st_frag,
3258 defm Z : avx512_movnt<opc, OpcodeStr, st_frag,
3264 defm Z256 : avx512_movnt<opc, OpcodeStr, st_frag,
3269 defm Z128 : avx512_movnt<opc, OpcodeStr, st_frag,
3291 multiclass avx512_binop_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
3294 defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
3302 defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
3311 multiclass avx512_binop_rmb<bits<8> opc, string OpcodeStr, SDNode OpNode,
3314 avx512_binop_rm<opc, OpcodeStr, OpNode, _, itins, IsCommutable> {
3316 defm rmb : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
3327 multiclass avx512_binop_rm_vl<bits<8> opc, string OpcodeStr, SDNode OpNode,
3331 defm Z : avx512_binop_rm<opc, OpcodeStr, OpNode, VTInfo.info512, itins,
3335 defm Z256 : avx512_binop_rm<opc, OpcodeStr, OpNode, VTInfo.info256, itins,
3337 defm Z128 : avx512_binop_rm<opc, OpcodeStr, OpNode, VTInfo.info128, itins,
3342 multiclass avx512_binop_rmb_vl<bits<8> opc, string OpcodeStr, SDNode OpNode,
3346 defm Z : avx512_binop_rmb<opc, OpcodeStr, OpNode, VTInfo.info512, itins,
3350 defm Z256 : avx512_binop_rmb<opc, OpcodeStr, OpNode, VTInfo.info256, itins,
3352 defm Z128 : avx512_binop_rmb<opc, OpcodeStr, OpNode, VTInfo.info128, itins,
3357 multiclass avx512_binop_rm_vl_q<bits<8> opc, string OpcodeStr, SDNode OpNode,
3360 defm NAME : avx512_binop_rmb_vl<opc, OpcodeStr, OpNode, avx512vl_i64_info,
3365 multiclass avx512_binop_rm_vl_d<bits<8> opc, string OpcodeStr, SDNode OpNode,
3368 defm NAME : avx512_binop_rmb_vl<opc, OpcodeStr, OpNode, avx512vl_i32_info,
3372 multiclass avx512_binop_rm_vl_w<bits<8> opc, string OpcodeStr, SDNode OpNode,
3375 defm NAME : avx512_binop_rm_vl<opc, OpcodeStr, OpNode, avx512vl_i16_info,
3379 multiclass avx512_binop_rm_vl_b<bits<8> opc, string OpcodeStr, SDNode OpNode,
3382 defm NAME : avx512_binop_rm_vl<opc, OpcodeStr, OpNode, avx512vl_i8_info,
3416 multiclass avx512_binop_rm2<bits<8> opc, string OpcodeStr, OpndItins itins,
3419 defm rr : AVX512_maskable<opc, MRMSrcReg, _Dst, (outs _Dst.RC:$dst),
3428 defm rm : AVX512_maskable<opc, MRMSrcMem, _Dst, (outs _Dst.RC:$dst),
3436 defm rmb : AVX512_maskable<opc, MRMSrcMem, _Dst, (outs _Dst.RC:$dst),
3476 multiclass avx512_binop_all<bits<8> opc, string OpcodeStr, OpndItins itins,
3479 defm NAME#Z : avx512_binop_rm2<opc, OpcodeStr, itins, OpNode,
3483 defm NAME#Z256 : avx512_binop_rm2<opc, OpcodeStr, itins, OpNode,
3486 defm NAME#Z128 : avx512_binop_rm2<opc, OpcodeStr, itins, OpNode,
3497 multiclass avx512_packs_rmb<bits<8> opc, string OpcodeStr, SDNode OpNode,
3500 defm rmb : AVX512_maskable<opc, MRMSrcMem, _Dst, (outs _Dst.RC:$dst),
3512 multiclass avx512_packs_rm<bits<8> opc, string OpcodeStr,
3515 defm rr : AVX512_maskable<opc, MRMSrcReg, _Dst, (outs _Dst.RC:$dst),
3523 defm rm : AVX512_maskable<opc, MRMSrcMem, _Dst, (outs _Dst.RC:$dst),
3532 multiclass avx512_packs_all_i32_i16<bits<8> opc, string OpcodeStr,
3534 defm NAME#Z : avx512_packs_rm<opc, OpcodeStr, OpNode, v16i32_info,
3536 avx512_packs_rmb<opc, OpcodeStr, OpNode, v16i32_info,
3539 defm NAME#Z256 : avx512_packs_rm<opc, OpcodeStr, OpNode, v8i32x_info,
3541 avx512_packs_rmb<opc, OpcodeStr, OpNode, v8i32x_info,
3543 defm NAME#Z128 : avx512_packs_rm<opc, OpcodeStr, OpNode, v4i32x_info,
3545 avx512_packs_rmb<opc, OpcodeStr, OpNode, v4i32x_info,
3549 multiclass avx512_packs_all_i16_i8<bits<8> opc, string OpcodeStr,
3551 defm NAME#Z : avx512_packs_rm<opc, OpcodeStr, OpNode, v32i16_info,
3554 defm NAME#Z256 : avx512_packs_rm<opc, OpcodeStr, OpNode, v16i16x_info,
3556 defm NAME#Z128 : avx512_packs_rm<opc, OpcodeStr, OpNode, v8i16x_info,
3561 multiclass avx512_vpmadd<bits<8> opc, string OpcodeStr,
3564 defm NAME#Z : avx512_packs_rm<opc, OpcodeStr, OpNode, _Src.info512,
3567 defm NAME#Z256 : avx512_packs_rm<opc, OpcodeStr, OpNode, _Src.info256,
3569 defm NAME#Z128 : avx512_packs_rm<opc, OpcodeStr, OpNode, _Src.info128,
3629 multiclass avx512_fp_scalar<bits<8> opc, string OpcodeStr,X86VectorVTInfo _,
3633 defm rr_Int : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
3640 defm rm_Int : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),
3649 def rr : I< opc, MRMSrcReg, (outs _.FRC:$dst),
3654 def rm : I< opc, MRMSrcMem, (outs _.FRC:$dst),
3662 multiclass avx512_fp_scalar_round<bits<8> opc, string OpcodeStr,X86VectorVTInfo _,
3665 defm rrb : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
3672 multiclass avx512_fp_scalar_sae<bits<8> opc, string OpcodeStr,X86VectorVTInfo _,
3675 defm rrb : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
3682 multiclass avx512_binop_s_round<bits<8> opc, string OpcodeStr, SDNode OpNode,
3685 defm SSZ : avx512_fp_scalar<opc, OpcodeStr#"ss", f32x_info, OpNode, VecNode,
3687 avx512_fp_scalar_round<opc, OpcodeStr#"ss", f32x_info, VecNode,
3690 defm SDZ : avx512_fp_scalar<opc, OpcodeStr#"sd", f64x_info, OpNode, VecNode,
3692 avx512_fp_scalar_round<opc, OpcodeStr#"sd", f64x_info, VecNode,
3697 multiclass avx512_binop_s_sae<bits<8> opc, string OpcodeStr, SDNode OpNode,
3700 defm SSZ : avx512_fp_scalar<opc, OpcodeStr#"ss", f32x_info, OpNode, VecNode,
3702 avx512_fp_scalar_sae<opc, OpcodeStr#"ss", f32x_info, VecNode,
3705 defm SDZ : avx512_fp_scalar<opc, OpcodeStr#"sd", f64x_info, OpNode, VecNode,
3707 avx512_fp_scalar_sae<opc, OpcodeStr#"sd", f64x_info, VecNode,
3718 multiclass avx512_fp_packed<bits<8> opc, string OpcodeStr, SDNode OpNode,
3720 defm rr: AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
3725 defm rm: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
3729 defm rmb: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
3739 multiclass avx512_fp_round_packed<bits<8> opc, string OpcodeStr, SDNode OpNodeRnd,
3741 defm rb: AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
3749 multiclass avx512_fp_sae_packed<bits<8> opc, string OpcodeStr, SDNode OpNodeRnd,
3751 defm rb: AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
3758 multiclass avx512_fp_binop_p<bits<8> opc, string OpcodeStr, SDNode OpNode,
3760 defm PSZ : avx512_fp_packed<opc, OpcodeStr, OpNode, v16f32_info,
3763 defm PDZ : avx512_fp_packed<opc, OpcodeStr, OpNode, v8f64_info,
3769 defm PSZ128 : avx512_fp_packed<opc, OpcodeStr, OpNode, v4f32x_info,
3772 defm PSZ256 : avx512_fp_packed<opc, OpcodeStr, OpNode, v8f32x_info,
3775 defm PDZ128 : avx512_fp_packed<opc, OpcodeStr, OpNode, v2f64x_info,
3778 defm PDZ256 : avx512_fp_packed<opc, OpcodeStr, OpNode, v4f64x_info,
3784 multiclass avx512_fp_binop_p_round<bits<8> opc, string OpcodeStr, SDNode OpNodeRnd> {
3785 defm PSZ : avx512_fp_round_packed<opc, OpcodeStr, OpNodeRnd, v16f32_info>,
3787 defm PDZ : avx512_fp_round_packed<opc, OpcodeStr, OpNodeRnd, v8f64_info>,
3791 multiclass avx512_fp_binop_p_sae<bits<8> opc, string OpcodeStr, SDNode OpNodeRnd> {
3792 defm PSZ : avx512_fp_sae_packed<opc, OpcodeStr, OpNodeRnd, v16f32_info>,
3794 defm PDZ : avx512_fp_sae_packed<opc, OpcodeStr, OpNodeRnd, v8f64_info>,
3817 multiclass avx512_fp_scalef_p<bits<8> opc, string OpcodeStr, SDNode OpNode,
3819 defm rr: AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
3824 defm rm: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
3828 defm rmb: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
3838 multiclass avx512_fp_scalef_scalar<bits<8> opc, string OpcodeStr, SDNode OpNode,
3840 defm rr: AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
3845 defm rm: AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),
3852 multiclass avx512_fp_scalef_all<bits<8> opc, bits<8> opcScaler, string OpcodeStr, SDNode OpNode> {
3853 defm PSZ : avx512_fp_scalef_p<opc, OpcodeStr, OpNode, v16f32_info>,
3854 avx512_fp_round_packed<opc, OpcodeStr, OpNode, v16f32_info>,
3856 defm PDZ : avx512_fp_scalef_p<opc, OpcodeStr, OpNode, v8f64_info>,
3857 avx512_fp_round_packed<opc, OpcodeStr, OpNode, v8f64_info>,
3868 defm PSZ128 : avx512_fp_scalef_p<opc, OpcodeStr, OpNode, v4f32x_info>,
3870 defm PSZ256 : avx512_fp_scalef_p<opc, OpcodeStr, OpNode, v8f32x_info>,
3872 defm PDZ128 : avx512_fp_scalef_p<opc, OpcodeStr, OpNode, v2f64x_info>,
3874 defm PDZ256 : avx512_fp_scalef_p<opc, OpcodeStr, OpNode, v4f64x_info>,
3884 multiclass avx512_vptest<bits<8> opc, string OpcodeStr, SDNode OpNode,
3886 defm rr : AVX512_maskable_cmp<opc, MRMSrcReg, _, (outs _.KRC:$dst),
3892 defm rm : AVX512_maskable_cmp<opc, MRMSrcMem, _, (outs _.KRC:$dst),
3901 multiclass avx512_vptest_mb<bits<8> opc, string OpcodeStr, SDNode OpNode,
3904 defm rmb : AVX512_maskable_cmp<opc, MRMSrcMem, _, (outs _.KRC:$dst),
3912 multiclass avx512_vptest_dq_sizes<bits<8> opc, string OpcodeStr, SDNode OpNode,
3915 defm Z : avx512_vptest<opc, OpcodeStr, OpNode, _.info512>,
3916 avx512_vptest_mb<opc, OpcodeStr, OpNode, _.info512>, EVEX_V512;
3919 defm Z256 : avx512_vptest<opc, OpcodeStr, OpNode, _.info256>,
3920 avx512_vptest_mb<opc, OpcodeStr, OpNode, _.info256>, EVEX_V256;
3921 defm Z128 : avx512_vptest<opc, OpcodeStr, OpNode, _.info128>,
3922 avx512_vptest_mb<opc, OpcodeStr, OpNode, _.info128>, EVEX_V128;
3926 multiclass avx512_vptest_dq<bits<8> opc, string OpcodeStr, SDNode OpNode> {
3927 defm D : avx512_vptest_dq_sizes<opc, OpcodeStr#"d", OpNode,
3929 defm Q : avx512_vptest_dq_sizes<opc, OpcodeStr#"q", OpNode,
3933 multiclass avx512_vptest_wb<bits<8> opc, string OpcodeStr,
3936 defm WZ: avx512_vptest<opc, OpcodeStr#"w", OpNode, v32i16_info>,
3938 defm BZ: avx512_vptest<opc, OpcodeStr#"b", OpNode, v64i8_info>,
3943 defm WZ256: avx512_vptest<opc, OpcodeStr#"w", OpNode, v16i16x_info>,
3945 defm WZ128: avx512_vptest<opc, OpcodeStr#"w", OpNode, v8i16x_info>,
3947 defm BZ256: avx512_vptest<opc, OpcodeStr#"b", OpNode, v32i8x_info>,
3949 defm BZ128: avx512_vptest<opc, OpcodeStr#"b", OpNode, v16i8x_info>,
3973 multiclass avx512_shift_rmi<bits<8> opc, Format ImmFormR, Format ImmFormM,
3975 defm ri : AVX512_maskable<opc, ImmFormR, _, (outs _.RC:$dst),
3981 defm mi : AVX512_maskable<opc, ImmFormM, _, (outs _.RC:$dst),
3989 multiclass avx512_shift_rmbi<bits<8> opc, Format ImmFormM,
3992 defm mbi : AVX512_maskable<opc, ImmFormM, _, (outs _.RC:$dst),
3999 multiclass avx512_shift_rrm<bits<8> opc, string OpcodeStr, SDNode OpNode,
4002 defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
4007 defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
4015 multiclass avx512_shift_sizes<bits<8> opc, string OpcodeStr, SDNode OpNode,
4019 defm Z : avx512_shift_rrm<opc, OpcodeStr, OpNode, SrcVT, bc_frag,
4023 defm Z256 : avx512_shift_rrm<opc, OpcodeStr, OpNode, SrcVT, bc_frag,
4026 defm Z128 : avx512_shift_rrm<opc, OpcodeStr, OpNode, SrcVT, bc_frag,
4042 multiclass avx512_shift_rmi_sizes<bits<8> opc, Format ImmFormR, Format ImmFormM,
4046 defm Z: avx512_shift_rmi<opc, ImmFormR, ImmFormM, OpcodeStr, OpNode,
4048 avx512_shift_rmbi<opc, ImmFormM, OpcodeStr, OpNode,
4051 defm Z256: avx512_shift_rmi<opc, ImmFormR, ImmFormM, OpcodeStr, OpNode,
4053 avx512_shift_rmbi<opc, ImmFormM, OpcodeStr, OpNode,
4055 defm Z128: avx512_shift_rmi<opc, ImmFormR, ImmFormM, OpcodeStr, OpNode,
4057 avx512_shift_rmbi<opc, ImmFormM, OpcodeStr, OpNode,
4104 multiclass avx512_var_shift<bits<8> opc, string OpcodeStr, SDNode OpNode,
4106 defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
4112 defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
4121 multiclass avx512_var_shift_mb<bits<8> opc, string OpcodeStr, SDNode OpNode,
4124 defm rmb : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
4133 multiclass avx512_var_shift_sizes<bits<8> opc, string OpcodeStr, SDNode OpNode,
4136 defm Z : avx512_var_shift<opc, OpcodeStr, OpNode, _.info512>,
4137 avx512_var_shift_mb<opc, OpcodeStr, OpNode, _.info512>, EVEX_V512;
4140 defm Z256 : avx512_var_shift<opc, OpcodeStr, OpNode, _.info256>,
4141 avx512_var_shift_mb<opc, OpcodeStr, OpNode, _.info256>, EVEX_V256;
4142 defm Z128 : avx512_var_shift<opc, OpcodeStr, OpNode, _.info128>,
4143 avx512_var_shift_mb<opc, OpcodeStr, OpNode, _.info128>, EVEX_V128;
4147 multiclass avx512_var_shift_types<bits<8> opc, string OpcodeStr,
4149 defm D : avx512_var_shift_sizes<opc, OpcodeStr#"d", OpNode,
4151 defm Q : avx512_var_shift_sizes<opc, OpcodeStr#"q", OpNode,
4155 multiclass avx512_var_shift_w<bits<8> opc, string OpcodeStr,
4158 defm WZ: avx512_var_shift<opc, OpcodeStr, OpNode, v32i16_info>,
4162 defm WZ256: avx512_var_shift<opc, OpcodeStr, OpNode, v16i16x_info>,
4164 defm WZ128: avx512_var_shift<opc, OpcodeStr, OpNode, v8i16x_info>,
4181 multiclass avx512_vperm_dq_sizes<bits<8> opc, string OpcodeStr, SDNode OpNode,
4184 defm Z : avx512_var_shift<opc, OpcodeStr, OpNode, _.info512>,
4185 avx512_var_shift_mb<opc, OpcodeStr, OpNode, _.info512>, EVEX_V512;
4188 defm Z256 : avx512_var_shift<opc, OpcodeStr, OpNode, _.info256>,
4189 avx512_var_shift_mb<opc, OpcodeStr, OpNode, _.info256>, EVEX_V256;
4192 multiclass avx512_vpermi_dq_sizes<bits<8> opc, Format ImmFormR, Format ImmFormM,
4196 defm Z: avx512_shift_rmi<opc, ImmFormR, ImmFormM, OpcodeStr, OpNode,
4198 avx512_shift_rmbi<opc, ImmFormM, OpcodeStr, OpNode,
4201 defm Z256: avx512_shift_rmi<opc, ImmFormR, ImmFormM, OpcodeStr, OpNode,
4203 avx512_shift_rmbi<opc, ImmFormM, OpcodeStr, OpNode,
4304 multiclass avx512_pshufb_sizes<bits<8> opc, string OpcodeStr, SDNode OpNode> {
4306 defm Z: avx512_var_shift<opc, OpcodeStr, OpNode, v64i8_info>, EVEX_V512;
4309 defm Z256: avx512_var_shift<opc, OpcodeStr, OpNode, v32i8x_info>, EVEX_V256;
4310 defm Z128: avx512_var_shift<opc, OpcodeStr, OpNode, v16i8x_info>, EVEX_V128;
4346 multiclass avx512_mov_hilo_packed<bits<8> opc, string OpcodeStr, SDNode OpNode,
4349 def rm : AVX512<opc, MRMSrcMem, (outs _.RC:$dst),
4456 multiclass avx512_fma3p_213_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
4458 defm r: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),
4465 defm m: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
4471 defm mb: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
4481 multiclass avx512_fma3_213_round<bits<8> opc, string OpcodeStr, SDNode OpNode,
4483 defm rb: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),
4491 multiclass avx512_fma3p_213_common<bits<8> opc, string OpcodeStr, SDNode OpNode,
4494 defm Z : avx512_fma3p_213_rm<opc, OpcodeStr, OpNode, _.info512>,
4495 avx512_fma3_213_round<opc, OpcodeStr, OpNodeRnd, _.info512>,
4499 defm Z256 : avx512_fma3p_213_rm<opc, OpcodeStr, OpNode, _.info256>,
4501 defm Z128 : avx512_fma3p_213_rm<opc, OpcodeStr, OpNode, _.info128>,
4506 multiclass avx512_fma3p_213_f<bits<8> opc, string OpcodeStr, SDNode OpNode,
4508 defm PS : avx512_fma3p_213_common<opc, OpcodeStr#"ps", OpNode, OpNodeRnd,
4510 defm PD : avx512_fma3p_213_common<opc, OpcodeStr#"pd", OpNode, OpNodeRnd,
4523 multiclass avx512_fma3p_231_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
4525 defm r: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),
4532 defm m: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
4538 defm mb: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
4548 multiclass avx512_fma3_231_round<bits<8> opc, string OpcodeStr, SDNode OpNode,
4550 defm rb: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),
4558 multiclass avx512_fma3p_231_common<bits<8> opc, string OpcodeStr, SDNode OpNode,
4561 defm Z : avx512_fma3p_231_rm<opc, OpcodeStr, OpNode, _.info512>,
4562 avx512_fma3_231_round<opc, OpcodeStr, OpNodeRnd, _.info512>,
4566 defm Z256 : avx512_fma3p_231_rm<opc, OpcodeStr, OpNode, _.info256>,
4568 defm Z128 : avx512_fma3p_231_rm<opc, OpcodeStr, OpNode, _.info128>,
4573 multiclass avx512_fma3p_231_f<bits<8> opc, string OpcodeStr, SDNode OpNode,
4575 defm PS : avx512_fma3p_231_common<opc, OpcodeStr#"ps", OpNode, OpNodeRnd,
4577 defm PD : avx512_fma3p_231_common<opc, OpcodeStr#"pd", OpNode, OpNodeRnd,
4589 multiclass avx512_fma3p_132_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
4591 defm r: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),
4598 defm m: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
4604 defm mb: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
4614 multiclass avx512_fma3_132_round<bits<8> opc, string OpcodeStr, SDNode OpNode,
4616 defm rb: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),
4624 multiclass avx512_fma3p_132_common<bits<8> opc, string OpcodeStr, SDNode OpNode,
4627 defm Z : avx512_fma3p_132_rm<opc, OpcodeStr, OpNode, _.info512>,
4628 avx512_fma3_132_round<opc, OpcodeStr, OpNodeRnd, _.info512>,
4632 defm Z256 : avx512_fma3p_132_rm<opc, OpcodeStr, OpNode, _.info256>,
4634 defm Z128 : avx512_fma3p_132_rm<opc, OpcodeStr, OpNode, _.info128>,
4639 multiclass avx512_fma3p_132_f<bits<8> opc, string OpcodeStr, SDNode OpNode,
4641 defm PS : avx512_fma3p_132_common<opc, OpcodeStr#"ps", OpNode, OpNodeRnd,
4643 defm PD : avx512_fma3p_132_common<opc, OpcodeStr#"pd", OpNode, OpNodeRnd,
4656 multiclass avx512_fma3s_common<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
4659 defm r_Int: AVX512_maskable_3src_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
4664 defm m_Int: AVX512_maskable_3src_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),
4668 defm rb_Int: AVX512_maskable_3src_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
4674 def r : AVX512FMA3<opc, MRMSrcReg, (outs _.FRC:$dst),
4680 def m : AVX512FMA3<opc, MRMSrcMem, (outs _.FRC:$dst),
4750 multiclass avx512_vcvtsi<bits<8> opc, SDNode OpNode, RegisterClass SrcRC,
4754 def rr : SI<opc, MRMSrcReg, (outs DstVT.FRC:$dst),
4759 def rm : SI<opc, MRMSrcMem, (outs DstVT.FRC:$dst),
4765 def rr_Int : SI<opc, MRMSrcReg, (outs DstVT.RC:$dst),
4773 def rm_Int : SI<opc, MRMSrcMem, (outs DstVT.RC:$dst),
4783 multiclass avx512_vcvtsi_round<bits<8> opc, SDNode OpNode, RegisterClass SrcRC,
4785 def rrb_Int : SI<opc, MRMSrcReg, (outs DstVT.RC:$dst),
4795 multiclass avx512_vcvtsi_common<bits<8> opc, SDNode OpNode, RegisterClass SrcRC,
4798 defm NAME : avx512_vcvtsi_round<opc, OpNode, SrcRC, DstVT, asm>,
4799 avx512_vcvtsi<opc, OpNode, SrcRC, DstVT, x86memop, ld_frag, asm>,
4870 multiclass avx512_cvt_s_int_round<bits<8> opc, RegisterClass SrcRC,
4874 def rr : SI<opc, MRMSrcReg, (outs DstRC:$dst), (ins SrcRC:$src),
4877 def rb : SI<opc, MRMSrcReg, (outs DstRC:$dst), (ins SrcRC:$src, AVX512RC:$rc),
4881 def rm : SI<opc, MRMSrcMem, (outs DstRC:$dst), (ins memop:$src),
4938 multiclass avx512_cvt_s_all<bits<8> opc, string asm, X86VectorVTInfo _SrcRC,
4942 def rr : SI<opc, MRMSrcReg, (outs _DstRC.RC:$dst), (ins _SrcRC.FRC:$src),
4945 def rb : SI<opc, MRMSrcReg, (outs _DstRC.RC:$dst), (ins _SrcRC.FRC:$src),
4948 def rm : SI<opc, MRMSrcMem, (outs _DstRC.RC:$dst), (ins _SrcRC.MemOp:$src),
4954 def rr_Int : SI<opc, MRMSrcReg, (outs _DstRC.RC:$dst), (ins _SrcRC.RC:$src),
4958 def rb_Int : SI<opc, MRMSrcReg, (outs _DstRC.RC:$dst), (ins _SrcRC.RC:$src),
4964 def rm_Int : SI<opc, MRMSrcMem, (outs _DstRC.RC:$dst),
5013 multiclass avx512_cvt_fp_scalar<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
5015 defm rr : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
5021 defm rm : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),
5031 multiclass avx512_cvt_fp_sae_scalar<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
5033 defm rrb : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
5043 multiclass avx512_cvt_fp_rc_scalar<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
5045 defm rrb : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
5053 multiclass avx512_cvt_fp_scalar_sd2ss<bits<8> opc, string OpcodeStr, SDNode OpNode,
5057 defm Z : avx512_cvt_fp_scalar<opc, OpcodeStr, _dst, _src, OpNode>,
5058 avx512_cvt_fp_rc_scalar<opc, OpcodeStr, _dst, _src,
5064 multiclass avx512_cvt_fp_scalar_ss2sd<bits<8> opc, string OpcodeStr, SDNode OpNode,
5068 defm Z : avx512_cvt_fp_scalar<opc, OpcodeStr, _dst, _src, OpNode>,
5069 avx512_cvt_fp_sae_scalar<opc, OpcodeStr, _dst, _src, OpNodeRnd>,
5104 multiclass avx512_vcvt_fp<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
5109 defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
5113 defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
5118 defm rmb : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
5126 multiclass avx512_vcvt_fp_sae<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
5128 defm rrb : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
5137 multiclass avx512_vcvt_fp_rc<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
5139 defm rrb : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
5147 multiclass avx512_cvtps2pd<bits<8> opc, string OpcodeStr> {
5149 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8f64_info, v8f32x_info, fextend>,
5150 avx512_vcvt_fp_sae<opc, OpcodeStr, v8f64_info, v8f32x_info,
5154 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2f64x_info, v4f32x_info,
5156 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4f64x_info, v4f32x_info, fextend>,
5162 multiclass avx512_cvtpd2ps<bits<8> opc, string OpcodeStr> {
5164 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8f32x_info, v8f64_info, fround>,
5165 avx512_vcvt_fp_rc<opc, OpcodeStr, v8f32x_info, v8f64_info,
5169 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4f32x_info, v2f64x_info,
5171 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4f32x_info, v4f64x_info, fround,
5190 multiclass avx512_cvtdq2pd<bits<8> opc, string OpcodeStr, SDNode OpNode,
5194 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8f64_info, v8i32x_info, OpNode>,
5198 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2f64x_info, v4i32x_info,
5200 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4f64x_info, v4i32x_info, OpNode>,
5206 multiclass avx512_cvtdq2ps<bits<8> opc, string OpcodeStr, SDNode OpNode,
5209 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v16f32_info, v16i32_info, OpNode>,
5210 avx512_vcvt_fp_rc<opc, OpcodeStr, v16f32_info, v16i32_info,
5214 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4f32x_info, v4i32x_info, OpNode>,
5216 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v8f32x_info, v8i32x_info, OpNode>,
5222 multiclass avx512_cvttps2dq<bits<8> opc, string OpcodeStr,
5225 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v16i32_info, v16f32_info, OpNode>,
5226 avx512_vcvt_fp_sae<opc, OpcodeStr, v16i32_info, v16f32_info,
5230 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v4f32x_info, OpNode>,
5232 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v8i32x_info, v8f32x_info, OpNode>,
5238 multiclass avx512_cvtps2dq<bits<8> opc, string OpcodeStr,
5241 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v16i32_info, v16f32_info, OpNode>,
5242 avx512_vcvt_fp_rc<opc, OpcodeStr, v16i32_info, v16f32_info,
5246 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v4f32x_info, OpNode>,
5248 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v8i32x_info, v8f32x_info, OpNode>,
5254 multiclass avx512_cvttpd2dq<bits<8> opc, string OpcodeStr,
5257 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i32x_info, v8f64_info, OpNode>,
5258 avx512_vcvt_fp_sae<opc, OpcodeStr, v8i32x_info, v8f64_info,
5266 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v2f64x_info, OpNode,
5268 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v4f64x_info, OpNode,
5274 multiclass avx512_cvtpd2dq<bits<8> opc, string OpcodeStr,
5277 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i32x_info, v8f64_info, OpNode>,
5278 avx512_vcvt_fp_rc<opc, OpcodeStr, v8i32x_info, v8f64_info,
5286 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v2f64x_info, OpNode,
5288 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v4f64x_info, OpNode,
5294 multiclass avx512_cvtpd2qq<bits<8> opc, string OpcodeStr,
5297 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i64_info, v8f64_info, OpNode>,
5298 avx512_vcvt_fp_rc<opc, OpcodeStr, v8i64_info, v8f64_info,
5302 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2i64x_info, v2f64x_info, OpNode>,
5304 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i64x_info, v4f64x_info, OpNode>,
5310 multiclass avx512_cvttpd2qq<bits<8> opc, string OpcodeStr,
5313 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i64_info, v8f64_info, OpNode>,
5314 avx512_vcvt_fp_sae<opc, OpcodeStr, v8i64_info, v8f64_info,
5318 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2i64x_info, v2f64x_info, OpNode>,
5320 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i64x_info, v4f64x_info, OpNode>,
5326 multiclass avx512_cvtqq2pd<bits<8> opc, string OpcodeStr,
5329 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8f64_info, v8i64_info, OpNode>,
5330 avx512_vcvt_fp_rc<opc, OpcodeStr, v8f64_info, v8i64_info,
5334 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2f64x_info, v2i64x_info, OpNode>,
5336 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4f64x_info, v4i64x_info, OpNode>,
5342 multiclass avx512_cvtps2qq<bits<8> opc, string OpcodeStr,
5345 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i64_info, v8f32x_info, OpNode>,
5346 avx512_vcvt_fp_rc<opc, OpcodeStr, v8i64_info, v8f32x_info,
5352 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2i64x_info, v4f32x_info, OpNode,
5354 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i64x_info, v4f32x_info, OpNode>,
5360 multiclass avx512_cvttps2qq<bits<8> opc, string OpcodeStr,
5363 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i64_info, v8f32x_info, OpNode>,
5364 avx512_vcvt_fp_sae<opc, OpcodeStr, v8i64_info, v8f32x_info,
5370 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2i64x_info, v4f32x_info, OpNode,
5372 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i64x_info, v4f32x_info, OpNode>,
5378 multiclass avx512_cvtqq2ps<bits<8> opc, string OpcodeStr,
5381 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8f32x_info, v8i64_info, OpNode>,
5382 avx512_vcvt_fp_rc<opc, OpcodeStr, v8f32x_info, v8i64_info,
5390 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4f32x_info, v2i64x_info, OpNode,
5392 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4f32x_info, v4i64x_info, OpNode,
5589 multiclass avx512_ord_cmp_sae<bits<8> opc, X86VectorVTInfo _, SDNode OpNode,
5591 def rb: AVX512<opc, MRMSrcReg, (outs), (ins _.RC:$src1, _.RC:$src2),
5643 multiclass avx512_fp14_s<bits<8> opc, string OpcodeStr, SDNode OpNode,
5646 defm rr : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
5651 defm rm : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),
5670 multiclass avx512_fp14_p<bits<8> opc, string OpcodeStr, SDNode OpNode,
5672 defm r: AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
5676 defm m: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
5680 defm mb: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
5689 multiclass avx512_fp14_p_vl_all<bits<8> opc, string OpcodeStr, SDNode OpNode> {
5690 defm PSZ : avx512_fp14_p<opc, !strconcat(OpcodeStr, "ps"), OpNode, v16f32_info>,
5692 defm PDZ : avx512_fp14_p<opc, !strconcat(OpcodeStr, "pd"), OpNode, v8f64_info>,
5697 defm PSZ128 : avx512_fp14_p<opc, !strconcat(OpcodeStr, "ps"),
5700 defm PSZ256 : avx512_fp14_p<opc, !strconcat(OpcodeStr, "ps"),
5703 defm PDZ128 : avx512_fp14_p<opc, !strconcat(OpcodeStr, "pd"),
5706 defm PDZ256 : avx512_fp14_p<opc, !strconcat(OpcodeStr, "pd"),
5730 multiclass avx512_fp28_s<bits<8> opc, string OpcodeStr,X86VectorVTInfo _,
5733 defm r : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
5739 defm rb : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
5745 defm m : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),
5753 multiclass avx512_eri_s<bits<8> opc, string OpcodeStr, SDNode OpNode> {
5754 defm SS : avx512_fp28_s<opc, OpcodeStr#"ss", f32x_info, OpNode>,
5756 defm SD : avx512_fp28_s<opc, OpcodeStr#"sd", f64x_info, OpNode>,
5768 multiclass avx512_fp28_p<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
5771 defm r : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
5775 defm m : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
5781 defm mb : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
5788 multiclass avx512_fp28_p_round<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
5790 defm rb : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
5796 multiclass avx512_eri<bits<8> opc, string OpcodeStr, SDNode OpNode> {
5797 defm PS : avx512_fp28_p<opc, OpcodeStr#"ps", v16f32_info, OpNode>,
5798 avx512_fp28_p_round<opc, OpcodeStr#"ps", v16f32_info, OpNode>,
5800 defm PD : avx512_fp28_p<opc, OpcodeStr#"pd", v8f64_info, OpNode>,
5801 avx512_fp28_p_round<opc, OpcodeStr#"pd", v8f64_info, OpNode>,
5805 multiclass avx512_fp_unaryop_packed<bits<8> opc, string OpcodeStr,
5809 defm PSZ128 : avx512_fp28_p<opc, OpcodeStr#"ps", v4f32x_info, OpNode>,
5811 defm PSZ256 : avx512_fp28_p<opc, OpcodeStr#"ps", v8f32x_info, OpNode>,
5813 defm PDZ128 : avx512_fp28_p<opc, OpcodeStr#"pd", v2f64x_info, OpNode>,
5815 defm PDZ256 : avx512_fp28_p<opc, OpcodeStr#"pd", v4f64x_info, OpNode>,
5828 multiclass avx512_sqrt_packed_round<bits<8> opc, string OpcodeStr,
5830 defm rb: AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
5836 multiclass avx512_sqrt_packed<bits<8> opc, string OpcodeStr,
5838 defm r: AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
5842 defm m: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
5847 defm mb: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
5856 multiclass avx512_sqrt_packed_all<bits<8> opc, string OpcodeStr,
5858 defm PSZ : avx512_sqrt_packed<opc, !strconcat(OpcodeStr, "ps"), OpNode,
5861 defm PDZ : avx512_sqrt_packed<opc, !strconcat(OpcodeStr, "pd"), OpNode,
5866 defm PSZ128 : avx512_sqrt_packed<opc, !strconcat(OpcodeStr, "ps"),
5869 defm PSZ256 : avx512_sqrt_packed<opc, !strconcat(OpcodeStr, "ps"),
5872 defm PDZ128 : avx512_sqrt_packed<opc, !strconcat(OpcodeStr, "pd"),
5875 defm PDZ256 : avx512_sqrt_packed<opc, !strconcat(OpcodeStr, "pd"),
5881 multiclass avx512_sqrt_packed_all_round<bits<8> opc, string OpcodeStr,
5883 defm PSZ : avx512_sqrt_packed_round<opc, !strconcat(OpcodeStr, "ps"), OpNodeRnd,
5885 defm PDZ : avx512_sqrt_packed_round<opc, !strconcat(OpcodeStr, "pd"), OpNodeRnd,
5889 multiclass avx512_sqrt_scalar<bits<8> opc, string OpcodeStr,X86VectorVTInfo _,
5892 defm r_Int : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
5899 defm m_Int : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),
5907 defm rb_Int : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
5916 def r : I<opc, MRMSrcReg, (outs _.FRC:$dst),
5921 def m : I<opc, MRMSrcMem, (outs _.FRC:$dst),
5935 multiclass avx512_sqrt_scalar_all<bits<8> opc, string OpcodeStr> {
5936 defm SSZ : avx512_sqrt_scalar<opc, OpcodeStr#"ss", f32x_info, "SS", fsqrt,
5938 defm SDZ : avx512_sqrt_scalar<opc, OpcodeStr#"sd", f64x_info, "SD", fsqrt,
5961 avx512_rndscale_scalar<bits<8> opc, string OpcodeStr, X86VectorVTInfo _> {
5964 defm r : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
5970 defm rb : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
5977 defm m : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),
6029 multiclass avx512_trunc_common<bits<8> opc, string OpcodeStr, SDNode OpNode,
6033 defm rr : AVX512_maskable<opc, MRMDestReg, DestInfo, (outs DestInfo.RC:$dst),
6059 def mr : AVX512XS8I<opc, MRMDestMem, (outs),
6064 def mrk : AVX512XS8I<opc, MRMDestMem, (outs),
6102 multiclass avx512_trunc<bits<8> opc, string OpcodeStr, SDNode OpNode,
6110 defm Z128: avx512_trunc_common<opc, OpcodeStr, OpNode, VTSrcInfo.info128,
6115 defm Z256: avx512_trunc_common<opc, OpcodeStr, OpNode, VTSrcInfo.info256,
6121 defm Z: avx512_trunc_common<opc, OpcodeStr, OpNode, VTSrcInfo.info512,
6127 multiclass avx512_trunc_sat<bits<8> opc, string OpcodeStr, SDNode OpNode,
6134 defm Z128: avx512_trunc_common<opc, OpcodeStr, OpNode, VTSrcInfo.info128,
6139 defm Z256: avx512_trunc_common<opc, OpcodeStr, OpNode, VTSrcInfo.info256,
6145 defm Z: avx512_trunc_common<opc, OpcodeStr, OpNode, VTSrcInfo.info512,
6151 multiclass avx512_trunc_qb<bits<8> opc, string OpcodeStr, SDNode OpNode> {
6152 defm NAME: avx512_trunc<opc, OpcodeStr, OpNode, avx512vl_i64_info,
6156 multiclass avx512_trunc_sat_qb<bits<8> opc, string sat, SDNode OpNode> {
6157 defm NAME: avx512_trunc_sat<opc, "vpmov"##sat##"qb", OpNode, avx512vl_i64_info,
6162 multiclass avx512_trunc_qw<bits<8> opc, string OpcodeStr, SDNode OpNode> {
6163 defm NAME: avx512_trunc<opc, OpcodeStr, OpNode, avx512vl_i64_info,
6167 multiclass avx512_trunc_sat_qw<bits<8> opc, string sat, SDNode OpNode> {
6168 defm NAME: avx512_trunc_sat<opc, "vpmov"##sat##"qw", OpNode, avx512vl_i64_info,
6173 multiclass avx512_trunc_qd<bits<8> opc, string OpcodeStr, SDNode OpNode> {
6174 defm NAME: avx512_trunc<opc, OpcodeStr, OpNode, avx512vl_i64_info,
6178 multiclass avx512_trunc_sat_qd<bits<8> opc, string sat, SDNode OpNode> {
6179 defm NAME: avx512_trunc_sat<opc, "vpmov"##sat##"qd", OpNode, avx512vl_i64_info,
6184 multiclass avx512_trunc_db<bits<8> opc, string OpcodeStr, SDNode OpNode> {
6185 defm NAME: avx512_trunc<opc, OpcodeStr, OpNode, avx512vl_i32_info,
6189 multiclass avx512_trunc_sat_db<bits<8> opc, string sat, SDNode OpNode> {
6190 defm NAME: avx512_trunc_sat<opc, "vpmov"##sat##"db", OpNode, avx512vl_i32_info,
6195 multiclass avx512_trunc_dw<bits<8> opc, string OpcodeStr, SDNode OpNode> {
6196 defm NAME: avx512_trunc<opc, OpcodeStr, OpNode, avx512vl_i32_info,
6200 multiclass avx512_trunc_sat_dw<bits<8> opc, string sat, SDNode OpNode> {
6201 defm NAME: avx512_trunc_sat<opc, "vpmov"##sat##"dw", OpNode, avx512vl_i32_info,
6206 multiclass avx512_trunc_wb<bits<8> opc, string OpcodeStr, SDNode OpNode> {
6207 defm NAME: avx512_trunc<opc, OpcodeStr, OpNode, avx512vl_i16_info,
6211 multiclass avx512_trunc_sat_wb<bits<8> opc, string sat, SDNode OpNode> {
6212 defm NAME: avx512_trunc_sat<opc, "vpmov"##sat##"wb", OpNode, avx512vl_i16_info,
6258 multiclass avx512_extend_common<bits<8> opc, string OpcodeStr,
6262 defm rr : AVX512_maskable<opc, MRMSrcReg, DestInfo, (outs DestInfo.RC:$dst),
6268 defm rm : AVX512_maskable<opc, MRMSrcMem, DestInfo, (outs DestInfo.RC:$dst),
6275 multiclass avx512_extend_BW<bits<8> opc, string OpcodeStr, SDNode OpNode,
6278 defm Z128: avx512_extend_common<opc, OpcodeStr, v8i16x_info,
6282 defm Z256: avx512_extend_common<opc, OpcodeStr, v16i16x_info,
6287 defm Z : avx512_extend_common<opc, OpcodeStr, v32i16_info,
6293 multiclass avx512_extend_BD<bits<8> opc, string OpcodeStr, SDNode OpNode,
6296 defm Z128: avx512_extend_common<opc, OpcodeStr, v4i32x_info,
6300 defm Z256: avx512_extend_common<opc, OpcodeStr, v8i32x_info,
6305 defm Z : avx512_extend_common<opc, OpcodeStr, v16i32_info,
6311 multiclass avx512_extend_BQ<bits<8> opc, string OpcodeStr, SDNode OpNode,
6314 defm Z128: avx512_extend_common<opc, OpcodeStr, v2i64x_info,
6318 defm Z256: avx512_extend_common<opc, OpcodeStr, v4i64x_info,
6323 defm Z : avx512_extend_common<opc, OpcodeStr, v8i64_info,
6329 multiclass avx512_extend_WD<bits<8> opc, string OpcodeStr, SDNode OpNode,
6332 defm Z128: avx512_extend_common<opc, OpcodeStr, v4i32x_info,
6336 defm Z256: avx512_extend_common<opc, OpcodeStr, v8i32x_info,
6341 defm Z : avx512_extend_common<opc, OpcodeStr, v16i32_info,
6347 multiclass avx512_extend_WQ<bits<8> opc, string OpcodeStr, SDNode OpNode,
6350 defm Z128: avx512_extend_common<opc, OpcodeStr, v2i64x_info,
6354 defm Z256: avx512_extend_common<opc, OpcodeStr, v4i64x_info,
6359 defm Z : avx512_extend_common<opc, OpcodeStr, v8i64_info,
6365 multiclass avx512_extend_DQ<bits<8> opc, string OpcodeStr, SDNode OpNode,
6369 defm Z128: avx512_extend_common<opc, OpcodeStr, v2i64x_info,
6373 defm Z256: avx512_extend_common<opc, OpcodeStr, v4i64x_info,
6378 defm Z : avx512_extend_common<opc, OpcodeStr, v8i64_info,
6402 multiclass avx512_gather<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
6406 def rm : AVX5128I<opc, MRMSrcMem, (outs _.RC:$dst, _.KRCWM:$mask_wb),
6459 multiclass avx512_scatter<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
6464 def mr : AVX5128I<opc, MRMDestMem, (outs _.KRCWM:$mask_wb),
6516 multiclass avx512_gather_scatter_prefetch<bits<8> opc, Format F, string OpcodeStr,
6519 def m : AVX5128I<opc, F, (outs), (ins KRC:$mask, memop:$src),
6598 multiclass cvt_by_vec_width<bits<8> opc, X86VectorVTInfo Vec, string OpcodeStr > {
6599 def rr : AVX512XS8I<opc, MRMSrcReg, (outs Vec.RC:$dst), (ins Vec.KRC:$src),
6604 multiclass cvt_mask_by_elt_width<bits<8> opc, AVX512VLVectorVTInfo VTInfo,
6607 defm Z : cvt_by_vec_width<opc, VTInfo.info512, OpcodeStr>, EVEX_V512;
6610 defm Z256 : cvt_by_vec_width<opc, VTInfo.info256, OpcodeStr>, EVEX_V256;
6611 defm Z128 : cvt_by_vec_width<opc, VTInfo.info128, OpcodeStr>, EVEX_V128;
6628 multiclass convert_vector_to_mask_common<bits<8> opc, X86VectorVTInfo _, string OpcodeStr > {
6629 def rr : AVX512XS8I<opc, MRMSrcReg, (outs _.KRC:$dst), (ins _.RC:$src),
6634 multiclass avx512_convert_vector_to_mask<bits<8> opc, string OpcodeStr,
6637 defm Z : convert_vector_to_mask_common <opc, VTInfo.info512, OpcodeStr>,
6641 defm Z256 : convert_vector_to_mask_common<opc, VTInfo.info256, OpcodeStr>,
6643 defm Z128 : convert_vector_to_mask_common<opc, VTInfo.info128, OpcodeStr>,
6661 multiclass compress_by_vec_width<bits<8> opc, X86VectorVTInfo _,
6663 defm rr : AVX512_maskable<opc, MRMDestReg, _, (outs _.RC:$dst),
6668 def mr : AVX5128I<opc, MRMDestMem, (outs),
6673 def mrk : AVX5128I<opc, MRMDestMem, (outs),
6683 multiclass compress_by_elt_width<bits<8> opc, string OpcodeStr,
6685 defm Z : compress_by_vec_width<opc, VTInfo.info512, OpcodeStr>, EVEX_V512;
6688 defm Z256 : compress_by_vec_width<opc, VTInfo.info256, OpcodeStr>, EVEX_V256;
6689 defm Z128 : compress_by_vec_width<opc, VTInfo.info128, OpcodeStr>, EVEX_V128;
6703 multiclass expand_by_vec_width<bits<8> opc, X86VectorVTInfo _,
6705 defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
6710 defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
6717 multiclass expand_by_elt_width<bits<8> opc, string OpcodeStr,
6719 defm Z : expand_by_vec_width<opc, VTInfo.info512, OpcodeStr>, EVEX_V512;
6722 defm Z256 : expand_by_vec_width<opc, VTInfo.info256, OpcodeStr>, EVEX_V256;
6723 defm Z128 : expand_by_vec_width<opc, VTInfo.info128, OpcodeStr>, EVEX_V128;
6740 multiclass avx512_unary_fp_packed_imm<bits<8> opc, string OpcodeStr, SDNode OpNode,
6742 defm rri : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
6749 defm rmi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
6755 defm rmbi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
6766 multiclass avx512_unary_fp_sae_packed_imm<bits<8> opc, string OpcodeStr,
6768 defm rrib : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
6778 AVX512VLVectorVTInfo _, bits<8> opc, SDNode OpNode, Predicate prd>{
6780 defm Z : avx512_unary_fp_packed_imm<opc, OpcodeStr, OpNode, _.info512>,
6781 avx512_unary_fp_sae_packed_imm<opc, OpcodeStr, OpNode, _.info512>,
6785 defm Z128 : avx512_unary_fp_packed_imm<opc, OpcodeStr, OpNode, _.info128>,
6787 defm Z256 : avx512_unary_fp_packed_imm<opc, OpcodeStr, OpNode, _.info256>,
6796 multiclass avx512_fp_packed_imm<bits<8> opc, string OpcodeStr, SDNode OpNode,
6798 defm rri : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
6806 defm rmi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
6813 defm rmbi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
6826 multiclass avx512_3Op_rm_imm8<bits<8> opc, string OpcodeStr, SDNode OpNode,
6829 defm rri : AVX512_maskable<opc, MRMSrcReg, DestInfo, (outs DestInfo.RC:$dst),
6836 defm rmi : AVX512_maskable<opc, MRMSrcMem, DestInfo, (outs DestInfo.RC:$dst),
6848 multiclass avx512_3Op_imm8<bits<8> opc, string OpcodeStr, SDNode OpNode,
6850 avx512_3Op_rm_imm8<opc, OpcodeStr, OpNode, _, _>{
6853 defm rmbi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
6865 multiclass avx512_fp_scalar_imm<bits<8> opc, string OpcodeStr, SDNode OpNode,
6868 defm rri : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
6876 defm rmi : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),
6886 defm rmi_alt :AVX512_maskable_in_asm<opc, MRMSrcMem, _, (outs _.FRC:$dst),
6895 multiclass avx512_fp_sae_packed_imm<bits<8> opc, string OpcodeStr,
6897 defm rrib : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
6907 multiclass avx512_fp_sae_scalar_imm<bits<8> opc, string OpcodeStr,
6909 defm NAME#rrib : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
6920 AVX512VLVectorVTInfo _, bits<8> opc, SDNode OpNode, Predicate prd>{
6922 defm Z : avx512_fp_packed_imm<opc, OpcodeStr, OpNode, _.info512>,
6923 avx512_fp_sae_packed_imm<opc, OpcodeStr, OpNode, _.info512>,
6928 defm Z128 : avx512_fp_packed_imm<opc, OpcodeStr, OpNode, _.info128>,
6930 defm Z256 : avx512_fp_packed_imm<opc, OpcodeStr, OpNode, _.info256>,
6935 multiclass avx512_common_3Op_rm_imm8<bits<8> opc, SDNode OpNode, string OpStr,
6938 defm Z : avx512_3Op_rm_imm8<opc, OpStr, OpNode, DestInfo.info512,
6942 defm Z128 : avx512_3Op_rm_imm8<opc, OpStr, OpNode, DestInfo.info128,
6944 defm Z256 : avx512_3Op_rm_imm8<opc, OpStr, OpNode, DestInfo.info256,
6950 bits<8> opc, SDNode OpNode>{
6952 defm Z : avx512_3Op_imm8<opc, OpcodeStr, OpNode, _.info512>, EVEX_V512;
6955 defm Z128 : avx512_3Op_imm8<opc, OpcodeStr, OpNode, _.info128>, EVEX_V128;
6956 defm Z256 : avx512_3Op_imm8<opc, OpcodeStr, OpNode, _.info256>, EVEX_V256;
6961 X86VectorVTInfo _, bits<8> opc, SDNode OpNode, Predicate prd>{
6963 defm Z128 : avx512_fp_scalar_imm<opc, OpcodeStr, OpNode, _>,
6964 avx512_fp_sae_scalar_imm<opc, OpcodeStr, OpNode, _>;
7027 bits<8> opc, SDNode OpNode = X86Shuf128>{
7029 defm Z : avx512_3Op_imm8<opc, OpcodeStr, OpNode, _.info512>, EVEX_V512;
7033 defm Z256 : avx512_3Op_imm8<opc, OpcodeStr, OpNode, _.info256>, EVEX_V256;
7109 multiclass avx512_unary_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
7111 defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
7117 defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
7124 multiclass avx512_unary_rmb<bits<8> opc, string OpcodeStr, SDNode OpNode,
7126 avx512_unary_rm<opc, OpcodeStr, OpNode, _> {
7128 defm rmb : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
7137 multiclass avx512_unary_rm_vl<bits<8> opc, string OpcodeStr, SDNode OpNode,
7140 defm Z : avx512_unary_rm<opc, OpcodeStr, OpNode, VTInfo.info512>, EVEX_V512;
7143 defm Z256 : avx512_unary_rm<opc, OpcodeStr, OpNode, VTInfo.info256>,
7145 defm Z128 : avx512_unary_rm<opc, OpcodeStr, OpNode, VTInfo.info128>,
7150 multiclass avx512_unary_rmb_vl<bits<8> opc, string OpcodeStr, SDNode OpNode,
7153 defm Z : avx512_unary_rmb<opc, OpcodeStr, OpNode, VTInfo.info512>,
7157 defm Z256 : avx512_unary_rmb<opc, OpcodeStr, OpNode, VTInfo.info256>,
7159 defm Z128 : avx512_unary_rmb<opc, OpcodeStr, OpNode, VTInfo.info128>,
7198 multiclass avx512_ctlz<bits<8> opc, string OpcodeStr, Predicate prd>{
7200 defm NAME : avx512_unary_rm_vl_dq<opc, opc, OpcodeStr, ctlz, prd>;
7202 defm NAME#_UNDEF : avx512_unary_rm_vl_dq<opc, opc, OpcodeStr,
7212 multiclass avx512_replicate<bits<8> opc, string OpcodeStr, SDNode OpNode>{
7213 defm NAME: avx512_unary_rm_vl<opc, OpcodeStr, OpNode, avx512vl_f32_info,
7216 defm NAME#_I: avx512_unary_rm_vl<opc, OpcodeStr, OpNode, avx512vl_i32_info,
7227 multiclass avx512_movddup_128<bits<8> opc, string OpcodeStr, SDNode OpNode,
7229 defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
7233 defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
7240 multiclass avx512_movddup_common<bits<8> opc, string OpcodeStr, SDNode OpNode,
7243 defm Z : avx512_unary_rm<opc, OpcodeStr, OpNode, VTInfo.info512>, EVEX_V512;
7246 defm Z256 : avx512_unary_rm<opc, OpcodeStr, OpNode, VTInfo.info256>,
7248 defm Z128 : avx512_movddup_128<opc, OpcodeStr, OpNode, VTInfo.info128>,
7253 multiclass avx512_movddup<bits<8> opc, string OpcodeStr, SDNode OpNode>{
7254 defm NAME: avx512_movddup_common<opc, OpcodeStr, OpNode,
7257 defm NAME#_I: avx512_movddup_common<opc, OpcodeStr, OpNode,
7296 multiclass avx512_extract_elt_bw_m<bits<8> opc, string OpcodeStr, SDNode OpNode,
7299 def mr : AVX512Ii8<opc, MRMDestMem, (outs),
7364 multiclass avx512_insert_elt_m<bits<8> opc, string OpcodeStr, SDNode OpNode,
7366 def rm : AVX512Ii8<opc, MRMSrcMem, (outs _.RC:$dst),
7374 multiclass avx512_insert_elt_bw<bits<8> opc, string OpcodeStr, SDNode OpNode,
7377 def rr : AVX512Ii8<opc, MRMSrcReg, (outs _.RC:$dst),
7383 defm NAME : avx512_insert_elt_m<opc, OpcodeStr, OpNode, _, LdFrag>;
7387 multiclass avx512_insert_elt_dq<bits<8> opc, string OpcodeStr,
7390 def rr : AVX512Ii8<opc, MRMSrcReg, (outs _.RC:$dst),
7397 defm NAME : avx512_insert_elt_m<opc, OpcodeStr, insertelt, _,
7429 multiclass avx512_shift_packed<bits<8> opc, SDNode OpNode, Format MRMr,
7431 def rr : AVX512<opc, MRMr,
7436 def rm : AVX512<opc, MRMm,
7443 multiclass avx512_shift_packed_all<bits<8> opc, SDNode OpNode, Format MRMr,
7446 defm Z512 : avx512_shift_packed<opc, OpNode, MRMr, MRMm,
7449 defm Z256 : avx512_shift_packed<opc, OpNode, MRMr, MRMm,
7451 defm Z128 : avx512_shift_packed<opc, OpNode, MRMr, MRMm,
7461 multiclass avx512_psadbw_packed<bits<8> opc, SDNode OpNode,
7464 def rr : AVX512BI<opc, MRMSrcReg,
7471 def rm : AVX512BI<opc, MRMSrcMem,
7480 multiclass avx512_psadbw_packed_all<bits<8> opc, SDNode OpNode,
7483 defm Z512 : avx512_psadbw_packed<opc, OpNode, OpcodeStr, v8i64_info,
7486 defm Z256 : avx512_psadbw_packed<opc, OpNode, OpcodeStr, v4i64x_info,
7488 defm Z128 : avx512_psadbw_packed<opc, OpNode, OpcodeStr, v2i64x_info,
7496 multiclass avx512_ternlog<bits<8> opc, string OpcodeStr, SDNode OpNode,
7499 defm rri : AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),
7507 defm rmi : AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
7515 defm rmbi : AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),