1/*===- TableGen'erated file -------------------------------------*- C++ -*-===*\ 2|* *| 3|* "Fast" Instruction Selector for the X86 target *| 4|* *| 5|* Automatically generated file, do not edit! *| 6|* *| 7\*===----------------------------------------------------------------------===*/ 8 9 10// FastEmit Immediate Predicate functions. 11static bool Predicate_i16immSExt8(int64_t Imm) { 12 return isInt<8>(Imm); 13} 14static bool Predicate_i32immSExt8(int64_t Imm) { 15 return isInt<8>(Imm); 16} 17static bool Predicate_i64immSExt8(int64_t Imm) { 18 return isInt<8>(Imm); 19} 20static bool Predicate_i64immSExt32(int64_t Imm) { 21 return isInt<32>(Imm); 22} 23static bool Predicate_AndMask64(int64_t Imm) { 24 25 return isMask_64(Imm) && !isUInt<32>(Imm); 26 27} 28static bool Predicate_BTRMask64(int64_t Imm) { 29 30 return !isUInt<32>(Imm) && !isInt<32>(Imm) && isPowerOf2_64(~Imm); 31 32} 33static bool Predicate_BTCBTSMask64(int64_t Imm) { 34 35 return !isInt<32>(Imm) && isPowerOf2_64(Imm); 36 37} 38 39 40// FastEmit functions for ISD::ABS. 41 42unsigned fastEmit_ISD_ABS_MVT_v16i8_r(MVT RetVT, unsigned Op0) { 43 if (RetVT.SimpleTy != MVT::v16i8) 44 return 0; 45 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 46 return fastEmitInst_r(X86::VPABSBZ128rr, &X86::VR128XRegClass, Op0); 47 } 48 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) { 49 return fastEmitInst_r(X86::PABSBrr, &X86::VR128RegClass, Op0); 50 } 51 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 52 return fastEmitInst_r(X86::VPABSBrr, &X86::VR128RegClass, Op0); 53 } 54 return 0; 55} 56 57unsigned fastEmit_ISD_ABS_MVT_v32i8_r(MVT RetVT, unsigned Op0) { 58 if (RetVT.SimpleTy != MVT::v32i8) 59 return 0; 60 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 61 return fastEmitInst_r(X86::VPABSBZ256rr, &X86::VR256XRegClass, Op0); 62 } 63 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 64 return fastEmitInst_r(X86::VPABSBYrr, &X86::VR256RegClass, Op0); 65 } 66 return 0; 67} 68 69unsigned fastEmit_ISD_ABS_MVT_v64i8_r(MVT RetVT, unsigned Op0) { 70 if (RetVT.SimpleTy != MVT::v64i8) 71 return 0; 72 if ((Subtarget->hasBWI())) { 73 return fastEmitInst_r(X86::VPABSBZrr, &X86::VR512RegClass, Op0); 74 } 75 return 0; 76} 77 78unsigned fastEmit_ISD_ABS_MVT_v8i16_r(MVT RetVT, unsigned Op0) { 79 if (RetVT.SimpleTy != MVT::v8i16) 80 return 0; 81 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 82 return fastEmitInst_r(X86::VPABSWZ128rr, &X86::VR128XRegClass, Op0); 83 } 84 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) { 85 return fastEmitInst_r(X86::PABSWrr, &X86::VR128RegClass, Op0); 86 } 87 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 88 return fastEmitInst_r(X86::VPABSWrr, &X86::VR128RegClass, Op0); 89 } 90 return 0; 91} 92 93unsigned fastEmit_ISD_ABS_MVT_v16i16_r(MVT RetVT, unsigned Op0) { 94 if (RetVT.SimpleTy != MVT::v16i16) 95 return 0; 96 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 97 return fastEmitInst_r(X86::VPABSWZ256rr, &X86::VR256XRegClass, Op0); 98 } 99 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 100 return fastEmitInst_r(X86::VPABSWYrr, &X86::VR256RegClass, Op0); 101 } 102 return 0; 103} 104 105unsigned fastEmit_ISD_ABS_MVT_v32i16_r(MVT RetVT, unsigned Op0) { 106 if (RetVT.SimpleTy != MVT::v32i16) 107 return 0; 108 if ((Subtarget->hasBWI())) { 109 return fastEmitInst_r(X86::VPABSWZrr, &X86::VR512RegClass, Op0); 110 } 111 return 0; 112} 113 114unsigned fastEmit_ISD_ABS_MVT_v4i32_r(MVT RetVT, unsigned Op0) { 115 if (RetVT.SimpleTy != MVT::v4i32) 116 return 0; 117 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 118 return fastEmitInst_r(X86::VPABSDZ128rr, &X86::VR128XRegClass, Op0); 119 } 120 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) { 121 return fastEmitInst_r(X86::PABSDrr, &X86::VR128RegClass, Op0); 122 } 123 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 124 return fastEmitInst_r(X86::VPABSDrr, &X86::VR128RegClass, Op0); 125 } 126 return 0; 127} 128 129unsigned fastEmit_ISD_ABS_MVT_v8i32_r(MVT RetVT, unsigned Op0) { 130 if (RetVT.SimpleTy != MVT::v8i32) 131 return 0; 132 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 133 return fastEmitInst_r(X86::VPABSDZ256rr, &X86::VR256XRegClass, Op0); 134 } 135 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 136 return fastEmitInst_r(X86::VPABSDYrr, &X86::VR256RegClass, Op0); 137 } 138 return 0; 139} 140 141unsigned fastEmit_ISD_ABS_MVT_v16i32_r(MVT RetVT, unsigned Op0) { 142 if (RetVT.SimpleTy != MVT::v16i32) 143 return 0; 144 if ((Subtarget->hasAVX512())) { 145 return fastEmitInst_r(X86::VPABSDZrr, &X86::VR512RegClass, Op0); 146 } 147 return 0; 148} 149 150unsigned fastEmit_ISD_ABS_MVT_v2i64_r(MVT RetVT, unsigned Op0) { 151 if (RetVT.SimpleTy != MVT::v2i64) 152 return 0; 153 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 154 return fastEmitInst_r(X86::VPABSQZ128rr, &X86::VR128XRegClass, Op0); 155 } 156 return 0; 157} 158 159unsigned fastEmit_ISD_ABS_MVT_v4i64_r(MVT RetVT, unsigned Op0) { 160 if (RetVT.SimpleTy != MVT::v4i64) 161 return 0; 162 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 163 return fastEmitInst_r(X86::VPABSQZ256rr, &X86::VR256XRegClass, Op0); 164 } 165 return 0; 166} 167 168unsigned fastEmit_ISD_ABS_MVT_v8i64_r(MVT RetVT, unsigned Op0) { 169 if (RetVT.SimpleTy != MVT::v8i64) 170 return 0; 171 if ((Subtarget->hasAVX512())) { 172 return fastEmitInst_r(X86::VPABSQZrr, &X86::VR512RegClass, Op0); 173 } 174 return 0; 175} 176 177unsigned fastEmit_ISD_ABS_r(MVT VT, MVT RetVT, unsigned Op0) { 178 switch (VT.SimpleTy) { 179 case MVT::v16i8: return fastEmit_ISD_ABS_MVT_v16i8_r(RetVT, Op0); 180 case MVT::v32i8: return fastEmit_ISD_ABS_MVT_v32i8_r(RetVT, Op0); 181 case MVT::v64i8: return fastEmit_ISD_ABS_MVT_v64i8_r(RetVT, Op0); 182 case MVT::v8i16: return fastEmit_ISD_ABS_MVT_v8i16_r(RetVT, Op0); 183 case MVT::v16i16: return fastEmit_ISD_ABS_MVT_v16i16_r(RetVT, Op0); 184 case MVT::v32i16: return fastEmit_ISD_ABS_MVT_v32i16_r(RetVT, Op0); 185 case MVT::v4i32: return fastEmit_ISD_ABS_MVT_v4i32_r(RetVT, Op0); 186 case MVT::v8i32: return fastEmit_ISD_ABS_MVT_v8i32_r(RetVT, Op0); 187 case MVT::v16i32: return fastEmit_ISD_ABS_MVT_v16i32_r(RetVT, Op0); 188 case MVT::v2i64: return fastEmit_ISD_ABS_MVT_v2i64_r(RetVT, Op0); 189 case MVT::v4i64: return fastEmit_ISD_ABS_MVT_v4i64_r(RetVT, Op0); 190 case MVT::v8i64: return fastEmit_ISD_ABS_MVT_v8i64_r(RetVT, Op0); 191 default: return 0; 192 } 193} 194 195// FastEmit functions for ISD::ANY_EXTEND. 196 197unsigned fastEmit_ISD_ANY_EXTEND_MVT_i8_r(MVT RetVT, unsigned Op0) { 198 if (RetVT.SimpleTy != MVT::i32) 199 return 0; 200 return fastEmitInst_r(X86::MOVZX32rr8, &X86::GR32RegClass, Op0); 201} 202 203unsigned fastEmit_ISD_ANY_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0) { 204 switch (VT.SimpleTy) { 205 case MVT::i8: return fastEmit_ISD_ANY_EXTEND_MVT_i8_r(RetVT, Op0); 206 default: return 0; 207 } 208} 209 210// FastEmit functions for ISD::BITCAST. 211 212unsigned fastEmit_ISD_BITCAST_MVT_i32_r(MVT RetVT, unsigned Op0) { 213 if (RetVT.SimpleTy != MVT::f32) 214 return 0; 215 if ((Subtarget->hasAVX512())) { 216 return fastEmitInst_r(X86::VMOVDI2SSZrr, &X86::FR32XRegClass, Op0); 217 } 218 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 219 return fastEmitInst_r(X86::MOVDI2SSrr, &X86::FR32RegClass, Op0); 220 } 221 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 222 return fastEmitInst_r(X86::VMOVDI2SSrr, &X86::FR32RegClass, Op0); 223 } 224 return 0; 225} 226 227unsigned fastEmit_ISD_BITCAST_MVT_i64_MVT_f64_r(unsigned Op0) { 228 if ((Subtarget->hasAVX512())) { 229 return fastEmitInst_r(X86::VMOV64toSDZrr, &X86::FR64XRegClass, Op0); 230 } 231 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 232 return fastEmitInst_r(X86::MOV64toSDrr, &X86::FR64RegClass, Op0); 233 } 234 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 235 return fastEmitInst_r(X86::VMOV64toSDrr, &X86::FR64RegClass, Op0); 236 } 237 return 0; 238} 239 240unsigned fastEmit_ISD_BITCAST_MVT_i64_MVT_x86mmx_r(unsigned Op0) { 241 if ((Subtarget->hasMMX()) && (Subtarget->is64Bit())) { 242 return fastEmitInst_r(X86::MMX_MOVD64to64rr, &X86::VR64RegClass, Op0); 243 } 244 return 0; 245} 246 247unsigned fastEmit_ISD_BITCAST_MVT_i64_r(MVT RetVT, unsigned Op0) { 248switch (RetVT.SimpleTy) { 249 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_i64_MVT_f64_r(Op0); 250 case MVT::x86mmx: return fastEmit_ISD_BITCAST_MVT_i64_MVT_x86mmx_r(Op0); 251 default: return 0; 252} 253} 254 255unsigned fastEmit_ISD_BITCAST_MVT_f32_r(MVT RetVT, unsigned Op0) { 256 if (RetVT.SimpleTy != MVT::i32) 257 return 0; 258 if ((Subtarget->hasAVX512())) { 259 return fastEmitInst_r(X86::VMOVSS2DIZrr, &X86::GR32RegClass, Op0); 260 } 261 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 262 return fastEmitInst_r(X86::MOVSS2DIrr, &X86::GR32RegClass, Op0); 263 } 264 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 265 return fastEmitInst_r(X86::VMOVSS2DIrr, &X86::GR32RegClass, Op0); 266 } 267 return 0; 268} 269 270unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_i64_r(unsigned Op0) { 271 if ((Subtarget->hasAVX512())) { 272 return fastEmitInst_r(X86::VMOVSDto64Zrr, &X86::GR64RegClass, Op0); 273 } 274 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 275 return fastEmitInst_r(X86::MOVSDto64rr, &X86::GR64RegClass, Op0); 276 } 277 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 278 return fastEmitInst_r(X86::VMOVSDto64rr, &X86::GR64RegClass, Op0); 279 } 280 return 0; 281} 282 283unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_x86mmx_r(unsigned Op0) { 284 if ((Subtarget->hasMMX()) && (Subtarget->hasSSE2())) { 285 return fastEmitInst_r(X86::MMX_MOVFR642Qrr, &X86::VR64RegClass, Op0); 286 } 287 return 0; 288} 289 290unsigned fastEmit_ISD_BITCAST_MVT_f64_r(MVT RetVT, unsigned Op0) { 291switch (RetVT.SimpleTy) { 292 case MVT::i64: return fastEmit_ISD_BITCAST_MVT_f64_MVT_i64_r(Op0); 293 case MVT::x86mmx: return fastEmit_ISD_BITCAST_MVT_f64_MVT_x86mmx_r(Op0); 294 default: return 0; 295} 296} 297 298unsigned fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_i64_r(unsigned Op0) { 299 if ((Subtarget->hasMMX()) && (Subtarget->is64Bit())) { 300 return fastEmitInst_r(X86::MMX_MOVD64from64rr, &X86::GR64RegClass, Op0); 301 } 302 return 0; 303} 304 305unsigned fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_f64_r(unsigned Op0) { 306 if ((Subtarget->hasMMX()) && (Subtarget->hasSSE2())) { 307 return fastEmitInst_r(X86::MMX_MOVQ2FR64rr, &X86::FR64RegClass, Op0); 308 } 309 return 0; 310} 311 312unsigned fastEmit_ISD_BITCAST_MVT_x86mmx_r(MVT RetVT, unsigned Op0) { 313switch (RetVT.SimpleTy) { 314 case MVT::i64: return fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_i64_r(Op0); 315 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_f64_r(Op0); 316 default: return 0; 317} 318} 319 320unsigned fastEmit_ISD_BITCAST_r(MVT VT, MVT RetVT, unsigned Op0) { 321 switch (VT.SimpleTy) { 322 case MVT::i32: return fastEmit_ISD_BITCAST_MVT_i32_r(RetVT, Op0); 323 case MVT::i64: return fastEmit_ISD_BITCAST_MVT_i64_r(RetVT, Op0); 324 case MVT::f32: return fastEmit_ISD_BITCAST_MVT_f32_r(RetVT, Op0); 325 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_f64_r(RetVT, Op0); 326 case MVT::x86mmx: return fastEmit_ISD_BITCAST_MVT_x86mmx_r(RetVT, Op0); 327 default: return 0; 328 } 329} 330 331// FastEmit functions for ISD::BRIND. 332 333unsigned fastEmit_ISD_BRIND_MVT_i16_r(MVT RetVT, unsigned Op0) { 334 if (RetVT.SimpleTy != MVT::isVoid) 335 return 0; 336 if ((!Subtarget->is64Bit())) { 337 return fastEmitInst_r(X86::JMP16r, &X86::GR16RegClass, Op0); 338 } 339 return 0; 340} 341 342unsigned fastEmit_ISD_BRIND_MVT_i32_r(MVT RetVT, unsigned Op0) { 343 if (RetVT.SimpleTy != MVT::isVoid) 344 return 0; 345 if ((!Subtarget->is64Bit())) { 346 return fastEmitInst_r(X86::JMP32r, &X86::GR32RegClass, Op0); 347 } 348 return 0; 349} 350 351unsigned fastEmit_ISD_BRIND_MVT_i64_r(MVT RetVT, unsigned Op0) { 352 if (RetVT.SimpleTy != MVT::isVoid) 353 return 0; 354 if ((Subtarget->is64Bit())) { 355 return fastEmitInst_r(X86::JMP64r, &X86::GR64RegClass, Op0); 356 } 357 return 0; 358} 359 360unsigned fastEmit_ISD_BRIND_r(MVT VT, MVT RetVT, unsigned Op0) { 361 switch (VT.SimpleTy) { 362 case MVT::i16: return fastEmit_ISD_BRIND_MVT_i16_r(RetVT, Op0); 363 case MVT::i32: return fastEmit_ISD_BRIND_MVT_i32_r(RetVT, Op0); 364 case MVT::i64: return fastEmit_ISD_BRIND_MVT_i64_r(RetVT, Op0); 365 default: return 0; 366 } 367} 368 369// FastEmit functions for ISD::BSWAP. 370 371unsigned fastEmit_ISD_BSWAP_MVT_i32_r(MVT RetVT, unsigned Op0) { 372 if (RetVT.SimpleTy != MVT::i32) 373 return 0; 374 return fastEmitInst_r(X86::BSWAP32r, &X86::GR32RegClass, Op0); 375} 376 377unsigned fastEmit_ISD_BSWAP_MVT_i64_r(MVT RetVT, unsigned Op0) { 378 if (RetVT.SimpleTy != MVT::i64) 379 return 0; 380 return fastEmitInst_r(X86::BSWAP64r, &X86::GR64RegClass, Op0); 381} 382 383unsigned fastEmit_ISD_BSWAP_r(MVT VT, MVT RetVT, unsigned Op0) { 384 switch (VT.SimpleTy) { 385 case MVT::i32: return fastEmit_ISD_BSWAP_MVT_i32_r(RetVT, Op0); 386 case MVT::i64: return fastEmit_ISD_BSWAP_MVT_i64_r(RetVT, Op0); 387 default: return 0; 388 } 389} 390 391// FastEmit functions for ISD::CTLZ. 392 393unsigned fastEmit_ISD_CTLZ_MVT_i16_r(MVT RetVT, unsigned Op0) { 394 if (RetVT.SimpleTy != MVT::i16) 395 return 0; 396 if ((Subtarget->hasLZCNT())) { 397 return fastEmitInst_r(X86::LZCNT16rr, &X86::GR16RegClass, Op0); 398 } 399 return 0; 400} 401 402unsigned fastEmit_ISD_CTLZ_MVT_i32_r(MVT RetVT, unsigned Op0) { 403 if (RetVT.SimpleTy != MVT::i32) 404 return 0; 405 if ((Subtarget->hasLZCNT())) { 406 return fastEmitInst_r(X86::LZCNT32rr, &X86::GR32RegClass, Op0); 407 } 408 return 0; 409} 410 411unsigned fastEmit_ISD_CTLZ_MVT_i64_r(MVT RetVT, unsigned Op0) { 412 if (RetVT.SimpleTy != MVT::i64) 413 return 0; 414 if ((Subtarget->hasLZCNT())) { 415 return fastEmitInst_r(X86::LZCNT64rr, &X86::GR64RegClass, Op0); 416 } 417 return 0; 418} 419 420unsigned fastEmit_ISD_CTLZ_MVT_v4i32_r(MVT RetVT, unsigned Op0) { 421 if (RetVT.SimpleTy != MVT::v4i32) 422 return 0; 423 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) { 424 return fastEmitInst_r(X86::VPLZCNTDZ128rr, &X86::VR128XRegClass, Op0); 425 } 426 return 0; 427} 428 429unsigned fastEmit_ISD_CTLZ_MVT_v8i32_r(MVT RetVT, unsigned Op0) { 430 if (RetVT.SimpleTy != MVT::v8i32) 431 return 0; 432 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) { 433 return fastEmitInst_r(X86::VPLZCNTDZ256rr, &X86::VR256XRegClass, Op0); 434 } 435 return 0; 436} 437 438unsigned fastEmit_ISD_CTLZ_MVT_v16i32_r(MVT RetVT, unsigned Op0) { 439 if (RetVT.SimpleTy != MVT::v16i32) 440 return 0; 441 if ((Subtarget->hasCDI())) { 442 return fastEmitInst_r(X86::VPLZCNTDZrr, &X86::VR512RegClass, Op0); 443 } 444 return 0; 445} 446 447unsigned fastEmit_ISD_CTLZ_MVT_v2i64_r(MVT RetVT, unsigned Op0) { 448 if (RetVT.SimpleTy != MVT::v2i64) 449 return 0; 450 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) { 451 return fastEmitInst_r(X86::VPLZCNTQZ128rr, &X86::VR128XRegClass, Op0); 452 } 453 return 0; 454} 455 456unsigned fastEmit_ISD_CTLZ_MVT_v4i64_r(MVT RetVT, unsigned Op0) { 457 if (RetVT.SimpleTy != MVT::v4i64) 458 return 0; 459 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) { 460 return fastEmitInst_r(X86::VPLZCNTQZ256rr, &X86::VR256XRegClass, Op0); 461 } 462 return 0; 463} 464 465unsigned fastEmit_ISD_CTLZ_MVT_v8i64_r(MVT RetVT, unsigned Op0) { 466 if (RetVT.SimpleTy != MVT::v8i64) 467 return 0; 468 if ((Subtarget->hasCDI())) { 469 return fastEmitInst_r(X86::VPLZCNTQZrr, &X86::VR512RegClass, Op0); 470 } 471 return 0; 472} 473 474unsigned fastEmit_ISD_CTLZ_r(MVT VT, MVT RetVT, unsigned Op0) { 475 switch (VT.SimpleTy) { 476 case MVT::i16: return fastEmit_ISD_CTLZ_MVT_i16_r(RetVT, Op0); 477 case MVT::i32: return fastEmit_ISD_CTLZ_MVT_i32_r(RetVT, Op0); 478 case MVT::i64: return fastEmit_ISD_CTLZ_MVT_i64_r(RetVT, Op0); 479 case MVT::v4i32: return fastEmit_ISD_CTLZ_MVT_v4i32_r(RetVT, Op0); 480 case MVT::v8i32: return fastEmit_ISD_CTLZ_MVT_v8i32_r(RetVT, Op0); 481 case MVT::v16i32: return fastEmit_ISD_CTLZ_MVT_v16i32_r(RetVT, Op0); 482 case MVT::v2i64: return fastEmit_ISD_CTLZ_MVT_v2i64_r(RetVT, Op0); 483 case MVT::v4i64: return fastEmit_ISD_CTLZ_MVT_v4i64_r(RetVT, Op0); 484 case MVT::v8i64: return fastEmit_ISD_CTLZ_MVT_v8i64_r(RetVT, Op0); 485 default: return 0; 486 } 487} 488 489// FastEmit functions for ISD::CTPOP. 490 491unsigned fastEmit_ISD_CTPOP_MVT_i16_r(MVT RetVT, unsigned Op0) { 492 if (RetVT.SimpleTy != MVT::i16) 493 return 0; 494 if ((Subtarget->hasPOPCNT())) { 495 return fastEmitInst_r(X86::POPCNT16rr, &X86::GR16RegClass, Op0); 496 } 497 return 0; 498} 499 500unsigned fastEmit_ISD_CTPOP_MVT_i32_r(MVT RetVT, unsigned Op0) { 501 if (RetVT.SimpleTy != MVT::i32) 502 return 0; 503 if ((Subtarget->hasPOPCNT())) { 504 return fastEmitInst_r(X86::POPCNT32rr, &X86::GR32RegClass, Op0); 505 } 506 return 0; 507} 508 509unsigned fastEmit_ISD_CTPOP_MVT_i64_r(MVT RetVT, unsigned Op0) { 510 if (RetVT.SimpleTy != MVT::i64) 511 return 0; 512 if ((Subtarget->hasPOPCNT())) { 513 return fastEmitInst_r(X86::POPCNT64rr, &X86::GR64RegClass, Op0); 514 } 515 return 0; 516} 517 518unsigned fastEmit_ISD_CTPOP_MVT_v16i8_r(MVT RetVT, unsigned Op0) { 519 if (RetVT.SimpleTy != MVT::v16i8) 520 return 0; 521 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) { 522 return fastEmitInst_r(X86::VPOPCNTBZ128rr, &X86::VR128XRegClass, Op0); 523 } 524 return 0; 525} 526 527unsigned fastEmit_ISD_CTPOP_MVT_v32i8_r(MVT RetVT, unsigned Op0) { 528 if (RetVT.SimpleTy != MVT::v32i8) 529 return 0; 530 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) { 531 return fastEmitInst_r(X86::VPOPCNTBZ256rr, &X86::VR256XRegClass, Op0); 532 } 533 return 0; 534} 535 536unsigned fastEmit_ISD_CTPOP_MVT_v64i8_r(MVT RetVT, unsigned Op0) { 537 if (RetVT.SimpleTy != MVT::v64i8) 538 return 0; 539 if ((Subtarget->hasBITALG())) { 540 return fastEmitInst_r(X86::VPOPCNTBZrr, &X86::VR512RegClass, Op0); 541 } 542 return 0; 543} 544 545unsigned fastEmit_ISD_CTPOP_MVT_v8i16_r(MVT RetVT, unsigned Op0) { 546 if (RetVT.SimpleTy != MVT::v8i16) 547 return 0; 548 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) { 549 return fastEmitInst_r(X86::VPOPCNTWZ128rr, &X86::VR128XRegClass, Op0); 550 } 551 return 0; 552} 553 554unsigned fastEmit_ISD_CTPOP_MVT_v16i16_r(MVT RetVT, unsigned Op0) { 555 if (RetVT.SimpleTy != MVT::v16i16) 556 return 0; 557 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) { 558 return fastEmitInst_r(X86::VPOPCNTWZ256rr, &X86::VR256XRegClass, Op0); 559 } 560 return 0; 561} 562 563unsigned fastEmit_ISD_CTPOP_MVT_v32i16_r(MVT RetVT, unsigned Op0) { 564 if (RetVT.SimpleTy != MVT::v32i16) 565 return 0; 566 if ((Subtarget->hasBITALG())) { 567 return fastEmitInst_r(X86::VPOPCNTWZrr, &X86::VR512RegClass, Op0); 568 } 569 return 0; 570} 571 572unsigned fastEmit_ISD_CTPOP_MVT_v4i32_r(MVT RetVT, unsigned Op0) { 573 if (RetVT.SimpleTy != MVT::v4i32) 574 return 0; 575 if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) { 576 return fastEmitInst_r(X86::VPOPCNTDZ128rr, &X86::VR128XRegClass, Op0); 577 } 578 return 0; 579} 580 581unsigned fastEmit_ISD_CTPOP_MVT_v8i32_r(MVT RetVT, unsigned Op0) { 582 if (RetVT.SimpleTy != MVT::v8i32) 583 return 0; 584 if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) { 585 return fastEmitInst_r(X86::VPOPCNTDZ256rr, &X86::VR256XRegClass, Op0); 586 } 587 return 0; 588} 589 590unsigned fastEmit_ISD_CTPOP_MVT_v16i32_r(MVT RetVT, unsigned Op0) { 591 if (RetVT.SimpleTy != MVT::v16i32) 592 return 0; 593 if ((Subtarget->hasVPOPCNTDQ())) { 594 return fastEmitInst_r(X86::VPOPCNTDZrr, &X86::VR512RegClass, Op0); 595 } 596 return 0; 597} 598 599unsigned fastEmit_ISD_CTPOP_MVT_v2i64_r(MVT RetVT, unsigned Op0) { 600 if (RetVT.SimpleTy != MVT::v2i64) 601 return 0; 602 if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) { 603 return fastEmitInst_r(X86::VPOPCNTQZ128rr, &X86::VR128XRegClass, Op0); 604 } 605 return 0; 606} 607 608unsigned fastEmit_ISD_CTPOP_MVT_v4i64_r(MVT RetVT, unsigned Op0) { 609 if (RetVT.SimpleTy != MVT::v4i64) 610 return 0; 611 if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) { 612 return fastEmitInst_r(X86::VPOPCNTQZ256rr, &X86::VR256XRegClass, Op0); 613 } 614 return 0; 615} 616 617unsigned fastEmit_ISD_CTPOP_MVT_v8i64_r(MVT RetVT, unsigned Op0) { 618 if (RetVT.SimpleTy != MVT::v8i64) 619 return 0; 620 if ((Subtarget->hasVPOPCNTDQ())) { 621 return fastEmitInst_r(X86::VPOPCNTQZrr, &X86::VR512RegClass, Op0); 622 } 623 return 0; 624} 625 626unsigned fastEmit_ISD_CTPOP_r(MVT VT, MVT RetVT, unsigned Op0) { 627 switch (VT.SimpleTy) { 628 case MVT::i16: return fastEmit_ISD_CTPOP_MVT_i16_r(RetVT, Op0); 629 case MVT::i32: return fastEmit_ISD_CTPOP_MVT_i32_r(RetVT, Op0); 630 case MVT::i64: return fastEmit_ISD_CTPOP_MVT_i64_r(RetVT, Op0); 631 case MVT::v16i8: return fastEmit_ISD_CTPOP_MVT_v16i8_r(RetVT, Op0); 632 case MVT::v32i8: return fastEmit_ISD_CTPOP_MVT_v32i8_r(RetVT, Op0); 633 case MVT::v64i8: return fastEmit_ISD_CTPOP_MVT_v64i8_r(RetVT, Op0); 634 case MVT::v8i16: return fastEmit_ISD_CTPOP_MVT_v8i16_r(RetVT, Op0); 635 case MVT::v16i16: return fastEmit_ISD_CTPOP_MVT_v16i16_r(RetVT, Op0); 636 case MVT::v32i16: return fastEmit_ISD_CTPOP_MVT_v32i16_r(RetVT, Op0); 637 case MVT::v4i32: return fastEmit_ISD_CTPOP_MVT_v4i32_r(RetVT, Op0); 638 case MVT::v8i32: return fastEmit_ISD_CTPOP_MVT_v8i32_r(RetVT, Op0); 639 case MVT::v16i32: return fastEmit_ISD_CTPOP_MVT_v16i32_r(RetVT, Op0); 640 case MVT::v2i64: return fastEmit_ISD_CTPOP_MVT_v2i64_r(RetVT, Op0); 641 case MVT::v4i64: return fastEmit_ISD_CTPOP_MVT_v4i64_r(RetVT, Op0); 642 case MVT::v8i64: return fastEmit_ISD_CTPOP_MVT_v8i64_r(RetVT, Op0); 643 default: return 0; 644 } 645} 646 647// FastEmit functions for ISD::CTTZ. 648 649unsigned fastEmit_ISD_CTTZ_MVT_i16_r(MVT RetVT, unsigned Op0) { 650 if (RetVT.SimpleTy != MVT::i16) 651 return 0; 652 if ((Subtarget->hasBMI())) { 653 return fastEmitInst_r(X86::TZCNT16rr, &X86::GR16RegClass, Op0); 654 } 655 return 0; 656} 657 658unsigned fastEmit_ISD_CTTZ_MVT_i32_r(MVT RetVT, unsigned Op0) { 659 if (RetVT.SimpleTy != MVT::i32) 660 return 0; 661 if ((Subtarget->hasBMI())) { 662 return fastEmitInst_r(X86::TZCNT32rr, &X86::GR32RegClass, Op0); 663 } 664 return 0; 665} 666 667unsigned fastEmit_ISD_CTTZ_MVT_i64_r(MVT RetVT, unsigned Op0) { 668 if (RetVT.SimpleTy != MVT::i64) 669 return 0; 670 if ((Subtarget->hasBMI())) { 671 return fastEmitInst_r(X86::TZCNT64rr, &X86::GR64RegClass, Op0); 672 } 673 return 0; 674} 675 676unsigned fastEmit_ISD_CTTZ_r(MVT VT, MVT RetVT, unsigned Op0) { 677 switch (VT.SimpleTy) { 678 case MVT::i16: return fastEmit_ISD_CTTZ_MVT_i16_r(RetVT, Op0); 679 case MVT::i32: return fastEmit_ISD_CTTZ_MVT_i32_r(RetVT, Op0); 680 case MVT::i64: return fastEmit_ISD_CTTZ_MVT_i64_r(RetVT, Op0); 681 default: return 0; 682 } 683} 684 685// FastEmit functions for ISD::CTTZ_ZERO_UNDEF. 686 687unsigned fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i16_r(MVT RetVT, unsigned Op0) { 688 if (RetVT.SimpleTy != MVT::i16) 689 return 0; 690 return fastEmitInst_r(X86::BSF16rr, &X86::GR16RegClass, Op0); 691} 692 693unsigned fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i32_r(MVT RetVT, unsigned Op0) { 694 if (RetVT.SimpleTy != MVT::i32) 695 return 0; 696 return fastEmitInst_r(X86::BSF32rr, &X86::GR32RegClass, Op0); 697} 698 699unsigned fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i64_r(MVT RetVT, unsigned Op0) { 700 if (RetVT.SimpleTy != MVT::i64) 701 return 0; 702 return fastEmitInst_r(X86::BSF64rr, &X86::GR64RegClass, Op0); 703} 704 705unsigned fastEmit_ISD_CTTZ_ZERO_UNDEF_r(MVT VT, MVT RetVT, unsigned Op0) { 706 switch (VT.SimpleTy) { 707 case MVT::i16: return fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i16_r(RetVT, Op0); 708 case MVT::i32: return fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i32_r(RetVT, Op0); 709 case MVT::i64: return fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i64_r(RetVT, Op0); 710 default: return 0; 711 } 712} 713 714// FastEmit functions for ISD::FABS. 715 716unsigned fastEmit_ISD_FABS_MVT_f32_r(MVT RetVT, unsigned Op0) { 717 if (RetVT.SimpleTy != MVT::f32) 718 return 0; 719 if ((!Subtarget->hasSSE1())) { 720 return fastEmitInst_r(X86::ABS_Fp32, &X86::RFP32RegClass, Op0); 721 } 722 return 0; 723} 724 725unsigned fastEmit_ISD_FABS_MVT_f64_r(MVT RetVT, unsigned Op0) { 726 if (RetVT.SimpleTy != MVT::f64) 727 return 0; 728 if ((!Subtarget->hasSSE2())) { 729 return fastEmitInst_r(X86::ABS_Fp64, &X86::RFP64RegClass, Op0); 730 } 731 return 0; 732} 733 734unsigned fastEmit_ISD_FABS_MVT_f80_r(MVT RetVT, unsigned Op0) { 735 if (RetVT.SimpleTy != MVT::f80) 736 return 0; 737 if ((Subtarget->hasX87())) { 738 return fastEmitInst_r(X86::ABS_Fp80, &X86::RFP80RegClass, Op0); 739 } 740 return 0; 741} 742 743unsigned fastEmit_ISD_FABS_r(MVT VT, MVT RetVT, unsigned Op0) { 744 switch (VT.SimpleTy) { 745 case MVT::f32: return fastEmit_ISD_FABS_MVT_f32_r(RetVT, Op0); 746 case MVT::f64: return fastEmit_ISD_FABS_MVT_f64_r(RetVT, Op0); 747 case MVT::f80: return fastEmit_ISD_FABS_MVT_f80_r(RetVT, Op0); 748 default: return 0; 749 } 750} 751 752// FastEmit functions for ISD::FNEG. 753 754unsigned fastEmit_ISD_FNEG_MVT_f32_r(MVT RetVT, unsigned Op0) { 755 if (RetVT.SimpleTy != MVT::f32) 756 return 0; 757 if ((!Subtarget->hasSSE1())) { 758 return fastEmitInst_r(X86::CHS_Fp32, &X86::RFP32RegClass, Op0); 759 } 760 return 0; 761} 762 763unsigned fastEmit_ISD_FNEG_MVT_f64_r(MVT RetVT, unsigned Op0) { 764 if (RetVT.SimpleTy != MVT::f64) 765 return 0; 766 if ((!Subtarget->hasSSE2())) { 767 return fastEmitInst_r(X86::CHS_Fp64, &X86::RFP64RegClass, Op0); 768 } 769 return 0; 770} 771 772unsigned fastEmit_ISD_FNEG_MVT_f80_r(MVT RetVT, unsigned Op0) { 773 if (RetVT.SimpleTy != MVT::f80) 774 return 0; 775 if ((Subtarget->hasX87())) { 776 return fastEmitInst_r(X86::CHS_Fp80, &X86::RFP80RegClass, Op0); 777 } 778 return 0; 779} 780 781unsigned fastEmit_ISD_FNEG_r(MVT VT, MVT RetVT, unsigned Op0) { 782 switch (VT.SimpleTy) { 783 case MVT::f32: return fastEmit_ISD_FNEG_MVT_f32_r(RetVT, Op0); 784 case MVT::f64: return fastEmit_ISD_FNEG_MVT_f64_r(RetVT, Op0); 785 case MVT::f80: return fastEmit_ISD_FNEG_MVT_f80_r(RetVT, Op0); 786 default: return 0; 787 } 788} 789 790// FastEmit functions for ISD::FP_EXTEND. 791 792unsigned fastEmit_ISD_FP_EXTEND_MVT_f32_r(MVT RetVT, unsigned Op0) { 793 if (RetVT.SimpleTy != MVT::f64) 794 return 0; 795 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 796 return fastEmitInst_r(X86::CVTSS2SDrr, &X86::FR64RegClass, Op0); 797 } 798 return 0; 799} 800 801unsigned fastEmit_ISD_FP_EXTEND_MVT_v8f16_MVT_v8f32_r(unsigned Op0) { 802 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 803 return fastEmitInst_r(X86::VCVTPH2PSXZ256rr, &X86::VR256XRegClass, Op0); 804 } 805 return 0; 806} 807 808unsigned fastEmit_ISD_FP_EXTEND_MVT_v8f16_MVT_v8f64_r(unsigned Op0) { 809 if ((Subtarget->hasFP16())) { 810 return fastEmitInst_r(X86::VCVTPH2PDZrr, &X86::VR512RegClass, Op0); 811 } 812 return 0; 813} 814 815unsigned fastEmit_ISD_FP_EXTEND_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 816switch (RetVT.SimpleTy) { 817 case MVT::v8f32: return fastEmit_ISD_FP_EXTEND_MVT_v8f16_MVT_v8f32_r(Op0); 818 case MVT::v8f64: return fastEmit_ISD_FP_EXTEND_MVT_v8f16_MVT_v8f64_r(Op0); 819 default: return 0; 820} 821} 822 823unsigned fastEmit_ISD_FP_EXTEND_MVT_v16f16_r(MVT RetVT, unsigned Op0) { 824 if (RetVT.SimpleTy != MVT::v16f32) 825 return 0; 826 if ((Subtarget->hasFP16())) { 827 return fastEmitInst_r(X86::VCVTPH2PSXZrr, &X86::VR512RegClass, Op0); 828 } 829 return 0; 830} 831 832unsigned fastEmit_ISD_FP_EXTEND_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 833 if (RetVT.SimpleTy != MVT::v4f64) 834 return 0; 835 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 836 return fastEmitInst_r(X86::VCVTPS2PDZ256rr, &X86::VR256XRegClass, Op0); 837 } 838 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 839 return fastEmitInst_r(X86::VCVTPS2PDYrr, &X86::VR256RegClass, Op0); 840 } 841 return 0; 842} 843 844unsigned fastEmit_ISD_FP_EXTEND_MVT_v8f32_r(MVT RetVT, unsigned Op0) { 845 if (RetVT.SimpleTy != MVT::v8f64) 846 return 0; 847 if ((Subtarget->hasAVX512())) { 848 return fastEmitInst_r(X86::VCVTPS2PDZrr, &X86::VR512RegClass, Op0); 849 } 850 return 0; 851} 852 853unsigned fastEmit_ISD_FP_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0) { 854 switch (VT.SimpleTy) { 855 case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f32_r(RetVT, Op0); 856 case MVT::v8f16: return fastEmit_ISD_FP_EXTEND_MVT_v8f16_r(RetVT, Op0); 857 case MVT::v16f16: return fastEmit_ISD_FP_EXTEND_MVT_v16f16_r(RetVT, Op0); 858 case MVT::v4f32: return fastEmit_ISD_FP_EXTEND_MVT_v4f32_r(RetVT, Op0); 859 case MVT::v8f32: return fastEmit_ISD_FP_EXTEND_MVT_v8f32_r(RetVT, Op0); 860 default: return 0; 861 } 862} 863 864// FastEmit functions for ISD::FP_ROUND. 865 866unsigned fastEmit_ISD_FP_ROUND_MVT_f64_r(MVT RetVT, unsigned Op0) { 867 if (RetVT.SimpleTy != MVT::f32) 868 return 0; 869 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 870 return fastEmitInst_r(X86::CVTSD2SSrr, &X86::FR32RegClass, Op0); 871 } 872 return 0; 873} 874 875unsigned fastEmit_ISD_FP_ROUND_r(MVT VT, MVT RetVT, unsigned Op0) { 876 switch (VT.SimpleTy) { 877 case MVT::f64: return fastEmit_ISD_FP_ROUND_MVT_f64_r(RetVT, Op0); 878 default: return 0; 879 } 880} 881 882// FastEmit functions for ISD::FP_TO_SINT. 883 884unsigned fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i32_r(unsigned Op0) { 885 if ((Subtarget->hasFP16())) { 886 return fastEmitInst_r(X86::VCVTTSH2SIZrr, &X86::GR32RegClass, Op0); 887 } 888 return 0; 889} 890 891unsigned fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i64_r(unsigned Op0) { 892 if ((Subtarget->hasFP16())) { 893 return fastEmitInst_r(X86::VCVTTSH2SI64Zrr, &X86::GR64RegClass, Op0); 894 } 895 return 0; 896} 897 898unsigned fastEmit_ISD_FP_TO_SINT_MVT_f16_r(MVT RetVT, unsigned Op0) { 899switch (RetVT.SimpleTy) { 900 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i32_r(Op0); 901 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i64_r(Op0); 902 default: return 0; 903} 904} 905 906unsigned fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(unsigned Op0) { 907 if ((Subtarget->hasAVX512())) { 908 return fastEmitInst_r(X86::VCVTTSS2SIZrr, &X86::GR32RegClass, Op0); 909 } 910 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 911 return fastEmitInst_r(X86::CVTTSS2SIrr, &X86::GR32RegClass, Op0); 912 } 913 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 914 return fastEmitInst_r(X86::VCVTTSS2SIrr, &X86::GR32RegClass, Op0); 915 } 916 return 0; 917} 918 919unsigned fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(unsigned Op0) { 920 if ((Subtarget->hasAVX512())) { 921 return fastEmitInst_r(X86::VCVTTSS2SI64Zrr, &X86::GR64RegClass, Op0); 922 } 923 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 924 return fastEmitInst_r(X86::CVTTSS2SI64rr, &X86::GR64RegClass, Op0); 925 } 926 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 927 return fastEmitInst_r(X86::VCVTTSS2SI64rr, &X86::GR64RegClass, Op0); 928 } 929 return 0; 930} 931 932unsigned fastEmit_ISD_FP_TO_SINT_MVT_f32_r(MVT RetVT, unsigned Op0) { 933switch (RetVT.SimpleTy) { 934 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0); 935 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0); 936 default: return 0; 937} 938} 939 940unsigned fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(unsigned Op0) { 941 if ((Subtarget->hasAVX512())) { 942 return fastEmitInst_r(X86::VCVTTSD2SIZrr, &X86::GR32RegClass, Op0); 943 } 944 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 945 return fastEmitInst_r(X86::CVTTSD2SIrr, &X86::GR32RegClass, Op0); 946 } 947 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 948 return fastEmitInst_r(X86::VCVTTSD2SIrr, &X86::GR32RegClass, Op0); 949 } 950 return 0; 951} 952 953unsigned fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(unsigned Op0) { 954 if ((Subtarget->hasAVX512())) { 955 return fastEmitInst_r(X86::VCVTTSD2SI64Zrr, &X86::GR64RegClass, Op0); 956 } 957 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 958 return fastEmitInst_r(X86::CVTTSD2SI64rr, &X86::GR64RegClass, Op0); 959 } 960 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 961 return fastEmitInst_r(X86::VCVTTSD2SI64rr, &X86::GR64RegClass, Op0); 962 } 963 return 0; 964} 965 966unsigned fastEmit_ISD_FP_TO_SINT_MVT_f64_r(MVT RetVT, unsigned Op0) { 967switch (RetVT.SimpleTy) { 968 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0); 969 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0); 970 default: return 0; 971} 972} 973 974unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f64_r(MVT RetVT, unsigned Op0) { 975 if (RetVT.SimpleTy != MVT::v4i32) 976 return 0; 977 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 978 return fastEmitInst_r(X86::VCVTTPD2DQYrr, &X86::VR128RegClass, Op0); 979 } 980 return 0; 981} 982 983unsigned fastEmit_ISD_FP_TO_SINT_r(MVT VT, MVT RetVT, unsigned Op0) { 984 switch (VT.SimpleTy) { 985 case MVT::f16: return fastEmit_ISD_FP_TO_SINT_MVT_f16_r(RetVT, Op0); 986 case MVT::f32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_r(RetVT, Op0); 987 case MVT::f64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_r(RetVT, Op0); 988 case MVT::v4f64: return fastEmit_ISD_FP_TO_SINT_MVT_v4f64_r(RetVT, Op0); 989 default: return 0; 990 } 991} 992 993// FastEmit functions for ISD::FP_TO_UINT. 994 995unsigned fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i32_r(unsigned Op0) { 996 if ((Subtarget->hasFP16())) { 997 return fastEmitInst_r(X86::VCVTTSH2USIZrr, &X86::GR32RegClass, Op0); 998 } 999 return 0; 1000} 1001 1002unsigned fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i64_r(unsigned Op0) { 1003 if ((Subtarget->hasFP16())) { 1004 return fastEmitInst_r(X86::VCVTTSH2USI64Zrr, &X86::GR64RegClass, Op0); 1005 } 1006 return 0; 1007} 1008 1009unsigned fastEmit_ISD_FP_TO_UINT_MVT_f16_r(MVT RetVT, unsigned Op0) { 1010switch (RetVT.SimpleTy) { 1011 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i32_r(Op0); 1012 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i64_r(Op0); 1013 default: return 0; 1014} 1015} 1016 1017unsigned fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(unsigned Op0) { 1018 if ((Subtarget->hasAVX512())) { 1019 return fastEmitInst_r(X86::VCVTTSS2USIZrr, &X86::GR32RegClass, Op0); 1020 } 1021 return 0; 1022} 1023 1024unsigned fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(unsigned Op0) { 1025 if ((Subtarget->hasAVX512())) { 1026 return fastEmitInst_r(X86::VCVTTSS2USI64Zrr, &X86::GR64RegClass, Op0); 1027 } 1028 return 0; 1029} 1030 1031unsigned fastEmit_ISD_FP_TO_UINT_MVT_f32_r(MVT RetVT, unsigned Op0) { 1032switch (RetVT.SimpleTy) { 1033 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0); 1034 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0); 1035 default: return 0; 1036} 1037} 1038 1039unsigned fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(unsigned Op0) { 1040 if ((Subtarget->hasAVX512())) { 1041 return fastEmitInst_r(X86::VCVTTSD2USIZrr, &X86::GR32RegClass, Op0); 1042 } 1043 return 0; 1044} 1045 1046unsigned fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(unsigned Op0) { 1047 if ((Subtarget->hasAVX512())) { 1048 return fastEmitInst_r(X86::VCVTTSD2USI64Zrr, &X86::GR64RegClass, Op0); 1049 } 1050 return 0; 1051} 1052 1053unsigned fastEmit_ISD_FP_TO_UINT_MVT_f64_r(MVT RetVT, unsigned Op0) { 1054switch (RetVT.SimpleTy) { 1055 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0); 1056 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0); 1057 default: return 0; 1058} 1059} 1060 1061unsigned fastEmit_ISD_FP_TO_UINT_r(MVT VT, MVT RetVT, unsigned Op0) { 1062 switch (VT.SimpleTy) { 1063 case MVT::f16: return fastEmit_ISD_FP_TO_UINT_MVT_f16_r(RetVT, Op0); 1064 case MVT::f32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_r(RetVT, Op0); 1065 case MVT::f64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_r(RetVT, Op0); 1066 default: return 0; 1067 } 1068} 1069 1070// FastEmit functions for ISD::FSQRT. 1071 1072unsigned fastEmit_ISD_FSQRT_MVT_f32_r(MVT RetVT, unsigned Op0) { 1073 if (RetVT.SimpleTy != MVT::f32) 1074 return 0; 1075 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 1076 return fastEmitInst_r(X86::SQRTSSr, &X86::FR32RegClass, Op0); 1077 } 1078 if ((!Subtarget->hasSSE1())) { 1079 return fastEmitInst_r(X86::SQRT_Fp32, &X86::RFP32RegClass, Op0); 1080 } 1081 return 0; 1082} 1083 1084unsigned fastEmit_ISD_FSQRT_MVT_f64_r(MVT RetVT, unsigned Op0) { 1085 if (RetVT.SimpleTy != MVT::f64) 1086 return 0; 1087 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 1088 return fastEmitInst_r(X86::SQRTSDr, &X86::FR64RegClass, Op0); 1089 } 1090 if ((!Subtarget->hasSSE2())) { 1091 return fastEmitInst_r(X86::SQRT_Fp64, &X86::RFP64RegClass, Op0); 1092 } 1093 return 0; 1094} 1095 1096unsigned fastEmit_ISD_FSQRT_MVT_f80_r(MVT RetVT, unsigned Op0) { 1097 if (RetVT.SimpleTy != MVT::f80) 1098 return 0; 1099 if ((Subtarget->hasX87())) { 1100 return fastEmitInst_r(X86::SQRT_Fp80, &X86::RFP80RegClass, Op0); 1101 } 1102 return 0; 1103} 1104 1105unsigned fastEmit_ISD_FSQRT_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 1106 if (RetVT.SimpleTy != MVT::v8f16) 1107 return 0; 1108 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 1109 return fastEmitInst_r(X86::VSQRTPHZ128r, &X86::VR128XRegClass, Op0); 1110 } 1111 return 0; 1112} 1113 1114unsigned fastEmit_ISD_FSQRT_MVT_v16f16_r(MVT RetVT, unsigned Op0) { 1115 if (RetVT.SimpleTy != MVT::v16f16) 1116 return 0; 1117 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 1118 return fastEmitInst_r(X86::VSQRTPHZ256r, &X86::VR256XRegClass, Op0); 1119 } 1120 return 0; 1121} 1122 1123unsigned fastEmit_ISD_FSQRT_MVT_v32f16_r(MVT RetVT, unsigned Op0) { 1124 if (RetVT.SimpleTy != MVT::v32f16) 1125 return 0; 1126 if ((Subtarget->hasFP16())) { 1127 return fastEmitInst_r(X86::VSQRTPHZr, &X86::VR512RegClass, Op0); 1128 } 1129 return 0; 1130} 1131 1132unsigned fastEmit_ISD_FSQRT_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 1133 if (RetVT.SimpleTy != MVT::v4f32) 1134 return 0; 1135 if ((Subtarget->hasVLX())) { 1136 return fastEmitInst_r(X86::VSQRTPSZ128r, &X86::VR128XRegClass, Op0); 1137 } 1138 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 1139 return fastEmitInst_r(X86::SQRTPSr, &X86::VR128RegClass, Op0); 1140 } 1141 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 1142 return fastEmitInst_r(X86::VSQRTPSr, &X86::VR128RegClass, Op0); 1143 } 1144 return 0; 1145} 1146 1147unsigned fastEmit_ISD_FSQRT_MVT_v8f32_r(MVT RetVT, unsigned Op0) { 1148 if (RetVT.SimpleTy != MVT::v8f32) 1149 return 0; 1150 if ((Subtarget->hasVLX())) { 1151 return fastEmitInst_r(X86::VSQRTPSZ256r, &X86::VR256XRegClass, Op0); 1152 } 1153 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 1154 return fastEmitInst_r(X86::VSQRTPSYr, &X86::VR256RegClass, Op0); 1155 } 1156 return 0; 1157} 1158 1159unsigned fastEmit_ISD_FSQRT_MVT_v16f32_r(MVT RetVT, unsigned Op0) { 1160 if (RetVT.SimpleTy != MVT::v16f32) 1161 return 0; 1162 if ((Subtarget->hasAVX512())) { 1163 return fastEmitInst_r(X86::VSQRTPSZr, &X86::VR512RegClass, Op0); 1164 } 1165 return 0; 1166} 1167 1168unsigned fastEmit_ISD_FSQRT_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 1169 if (RetVT.SimpleTy != MVT::v2f64) 1170 return 0; 1171 if ((Subtarget->hasVLX())) { 1172 return fastEmitInst_r(X86::VSQRTPDZ128r, &X86::VR128XRegClass, Op0); 1173 } 1174 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 1175 return fastEmitInst_r(X86::SQRTPDr, &X86::VR128RegClass, Op0); 1176 } 1177 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 1178 return fastEmitInst_r(X86::VSQRTPDr, &X86::VR128RegClass, Op0); 1179 } 1180 return 0; 1181} 1182 1183unsigned fastEmit_ISD_FSQRT_MVT_v4f64_r(MVT RetVT, unsigned Op0) { 1184 if (RetVT.SimpleTy != MVT::v4f64) 1185 return 0; 1186 if ((Subtarget->hasVLX())) { 1187 return fastEmitInst_r(X86::VSQRTPDZ256r, &X86::VR256XRegClass, Op0); 1188 } 1189 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 1190 return fastEmitInst_r(X86::VSQRTPDYr, &X86::VR256RegClass, Op0); 1191 } 1192 return 0; 1193} 1194 1195unsigned fastEmit_ISD_FSQRT_MVT_v8f64_r(MVT RetVT, unsigned Op0) { 1196 if (RetVT.SimpleTy != MVT::v8f64) 1197 return 0; 1198 if ((Subtarget->hasAVX512())) { 1199 return fastEmitInst_r(X86::VSQRTPDZr, &X86::VR512RegClass, Op0); 1200 } 1201 return 0; 1202} 1203 1204unsigned fastEmit_ISD_FSQRT_r(MVT VT, MVT RetVT, unsigned Op0) { 1205 switch (VT.SimpleTy) { 1206 case MVT::f32: return fastEmit_ISD_FSQRT_MVT_f32_r(RetVT, Op0); 1207 case MVT::f64: return fastEmit_ISD_FSQRT_MVT_f64_r(RetVT, Op0); 1208 case MVT::f80: return fastEmit_ISD_FSQRT_MVT_f80_r(RetVT, Op0); 1209 case MVT::v8f16: return fastEmit_ISD_FSQRT_MVT_v8f16_r(RetVT, Op0); 1210 case MVT::v16f16: return fastEmit_ISD_FSQRT_MVT_v16f16_r(RetVT, Op0); 1211 case MVT::v32f16: return fastEmit_ISD_FSQRT_MVT_v32f16_r(RetVT, Op0); 1212 case MVT::v4f32: return fastEmit_ISD_FSQRT_MVT_v4f32_r(RetVT, Op0); 1213 case MVT::v8f32: return fastEmit_ISD_FSQRT_MVT_v8f32_r(RetVT, Op0); 1214 case MVT::v16f32: return fastEmit_ISD_FSQRT_MVT_v16f32_r(RetVT, Op0); 1215 case MVT::v2f64: return fastEmit_ISD_FSQRT_MVT_v2f64_r(RetVT, Op0); 1216 case MVT::v4f64: return fastEmit_ISD_FSQRT_MVT_v4f64_r(RetVT, Op0); 1217 case MVT::v8f64: return fastEmit_ISD_FSQRT_MVT_v8f64_r(RetVT, Op0); 1218 default: return 0; 1219 } 1220} 1221 1222// FastEmit functions for ISD::LLRINT. 1223 1224unsigned fastEmit_ISD_LLRINT_MVT_f32_r(MVT RetVT, unsigned Op0) { 1225 if (RetVT.SimpleTy != MVT::i64) 1226 return 0; 1227 if ((Subtarget->hasAVX512())) { 1228 return fastEmitInst_r(X86::VCVTSS2SI64Zrr, &X86::GR64RegClass, Op0); 1229 } 1230 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 1231 return fastEmitInst_r(X86::CVTSS2SI64rr, &X86::GR64RegClass, Op0); 1232 } 1233 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 1234 return fastEmitInst_r(X86::VCVTSS2SI64rr, &X86::GR64RegClass, Op0); 1235 } 1236 return 0; 1237} 1238 1239unsigned fastEmit_ISD_LLRINT_MVT_f64_r(MVT RetVT, unsigned Op0) { 1240 if (RetVT.SimpleTy != MVT::i64) 1241 return 0; 1242 if ((Subtarget->hasAVX512())) { 1243 return fastEmitInst_r(X86::VCVTSD2SI64Zrr, &X86::GR64RegClass, Op0); 1244 } 1245 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 1246 return fastEmitInst_r(X86::CVTSD2SI64rr, &X86::GR64RegClass, Op0); 1247 } 1248 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 1249 return fastEmitInst_r(X86::VCVTSD2SI64rr, &X86::GR64RegClass, Op0); 1250 } 1251 return 0; 1252} 1253 1254unsigned fastEmit_ISD_LLRINT_r(MVT VT, MVT RetVT, unsigned Op0) { 1255 switch (VT.SimpleTy) { 1256 case MVT::f32: return fastEmit_ISD_LLRINT_MVT_f32_r(RetVT, Op0); 1257 case MVT::f64: return fastEmit_ISD_LLRINT_MVT_f64_r(RetVT, Op0); 1258 default: return 0; 1259 } 1260} 1261 1262// FastEmit functions for ISD::LRINT. 1263 1264unsigned fastEmit_ISD_LRINT_MVT_f32_MVT_i32_r(unsigned Op0) { 1265 if ((Subtarget->hasAVX512())) { 1266 return fastEmitInst_r(X86::VCVTSS2SIZrr, &X86::GR32RegClass, Op0); 1267 } 1268 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 1269 return fastEmitInst_r(X86::CVTSS2SIrr, &X86::GR32RegClass, Op0); 1270 } 1271 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 1272 return fastEmitInst_r(X86::VCVTSS2SIrr, &X86::GR32RegClass, Op0); 1273 } 1274 return 0; 1275} 1276 1277unsigned fastEmit_ISD_LRINT_MVT_f32_MVT_i64_r(unsigned Op0) { 1278 if ((Subtarget->hasAVX512())) { 1279 return fastEmitInst_r(X86::VCVTSS2SI64Zrr, &X86::GR64RegClass, Op0); 1280 } 1281 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 1282 return fastEmitInst_r(X86::CVTSS2SI64rr, &X86::GR64RegClass, Op0); 1283 } 1284 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 1285 return fastEmitInst_r(X86::VCVTSS2SI64rr, &X86::GR64RegClass, Op0); 1286 } 1287 return 0; 1288} 1289 1290unsigned fastEmit_ISD_LRINT_MVT_f32_r(MVT RetVT, unsigned Op0) { 1291switch (RetVT.SimpleTy) { 1292 case MVT::i32: return fastEmit_ISD_LRINT_MVT_f32_MVT_i32_r(Op0); 1293 case MVT::i64: return fastEmit_ISD_LRINT_MVT_f32_MVT_i64_r(Op0); 1294 default: return 0; 1295} 1296} 1297 1298unsigned fastEmit_ISD_LRINT_MVT_f64_MVT_i32_r(unsigned Op0) { 1299 if ((Subtarget->hasAVX512())) { 1300 return fastEmitInst_r(X86::VCVTSD2SIZrr, &X86::GR32RegClass, Op0); 1301 } 1302 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 1303 return fastEmitInst_r(X86::CVTSD2SIrr, &X86::GR32RegClass, Op0); 1304 } 1305 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 1306 return fastEmitInst_r(X86::VCVTSD2SIrr, &X86::GR32RegClass, Op0); 1307 } 1308 return 0; 1309} 1310 1311unsigned fastEmit_ISD_LRINT_MVT_f64_MVT_i64_r(unsigned Op0) { 1312 if ((Subtarget->hasAVX512())) { 1313 return fastEmitInst_r(X86::VCVTSD2SI64Zrr, &X86::GR64RegClass, Op0); 1314 } 1315 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 1316 return fastEmitInst_r(X86::CVTSD2SI64rr, &X86::GR64RegClass, Op0); 1317 } 1318 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 1319 return fastEmitInst_r(X86::VCVTSD2SI64rr, &X86::GR64RegClass, Op0); 1320 } 1321 return 0; 1322} 1323 1324unsigned fastEmit_ISD_LRINT_MVT_f64_r(MVT RetVT, unsigned Op0) { 1325switch (RetVT.SimpleTy) { 1326 case MVT::i32: return fastEmit_ISD_LRINT_MVT_f64_MVT_i32_r(Op0); 1327 case MVT::i64: return fastEmit_ISD_LRINT_MVT_f64_MVT_i64_r(Op0); 1328 default: return 0; 1329} 1330} 1331 1332unsigned fastEmit_ISD_LRINT_r(MVT VT, MVT RetVT, unsigned Op0) { 1333 switch (VT.SimpleTy) { 1334 case MVT::f32: return fastEmit_ISD_LRINT_MVT_f32_r(RetVT, Op0); 1335 case MVT::f64: return fastEmit_ISD_LRINT_MVT_f64_r(RetVT, Op0); 1336 default: return 0; 1337 } 1338} 1339 1340// FastEmit functions for ISD::SCALAR_TO_VECTOR. 1341 1342unsigned fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_r(MVT RetVT, unsigned Op0) { 1343 if (RetVT.SimpleTy != MVT::v4i32) 1344 return 0; 1345 if ((Subtarget->hasAVX512())) { 1346 return fastEmitInst_r(X86::VMOVDI2PDIZrr, &X86::VR128XRegClass, Op0); 1347 } 1348 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 1349 return fastEmitInst_r(X86::MOVDI2PDIrr, &X86::VR128RegClass, Op0); 1350 } 1351 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 1352 return fastEmitInst_r(X86::VMOVDI2PDIrr, &X86::VR128RegClass, Op0); 1353 } 1354 return 0; 1355} 1356 1357unsigned fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i64_r(MVT RetVT, unsigned Op0) { 1358 if (RetVT.SimpleTy != MVT::v2i64) 1359 return 0; 1360 if ((Subtarget->hasAVX512())) { 1361 return fastEmitInst_r(X86::VMOV64toPQIZrr, &X86::VR128XRegClass, Op0); 1362 } 1363 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 1364 return fastEmitInst_r(X86::MOV64toPQIrr, &X86::VR128RegClass, Op0); 1365 } 1366 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 1367 return fastEmitInst_r(X86::VMOV64toPQIrr, &X86::VR128RegClass, Op0); 1368 } 1369 return 0; 1370} 1371 1372unsigned fastEmit_ISD_SCALAR_TO_VECTOR_r(MVT VT, MVT RetVT, unsigned Op0) { 1373 switch (VT.SimpleTy) { 1374 case MVT::i32: return fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_r(RetVT, Op0); 1375 case MVT::i64: return fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i64_r(RetVT, Op0); 1376 default: return 0; 1377 } 1378} 1379 1380// FastEmit functions for ISD::SIGN_EXTEND. 1381 1382unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i32_r(unsigned Op0) { 1383 return fastEmitInst_r(X86::MOVSX32rr8, &X86::GR32RegClass, Op0); 1384} 1385 1386unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i64_r(unsigned Op0) { 1387 return fastEmitInst_r(X86::MOVSX64rr8, &X86::GR64RegClass, Op0); 1388} 1389 1390unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i8_r(MVT RetVT, unsigned Op0) { 1391switch (RetVT.SimpleTy) { 1392 case MVT::i32: return fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i32_r(Op0); 1393 case MVT::i64: return fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i64_r(Op0); 1394 default: return 0; 1395} 1396} 1397 1398unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i32_r(unsigned Op0) { 1399 return fastEmitInst_r(X86::MOVSX32rr16, &X86::GR32RegClass, Op0); 1400} 1401 1402unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i64_r(unsigned Op0) { 1403 return fastEmitInst_r(X86::MOVSX64rr16, &X86::GR64RegClass, Op0); 1404} 1405 1406unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i16_r(MVT RetVT, unsigned Op0) { 1407switch (RetVT.SimpleTy) { 1408 case MVT::i32: return fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i32_r(Op0); 1409 case MVT::i64: return fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i64_r(Op0); 1410 default: return 0; 1411} 1412} 1413 1414unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i32_r(MVT RetVT, unsigned Op0) { 1415 if (RetVT.SimpleTy != MVT::i64) 1416 return 0; 1417 if ((Subtarget->is64Bit())) { 1418 return fastEmitInst_r(X86::MOVSX64rr32, &X86::GR64RegClass, Op0); 1419 } 1420 return 0; 1421} 1422 1423unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v2i1_r(MVT RetVT, unsigned Op0) { 1424 if (RetVT.SimpleTy != MVT::v2i64) 1425 return 0; 1426 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 1427 return fastEmitInst_r(X86::VPMOVM2QZ128rr, &X86::VR128XRegClass, Op0); 1428 } 1429 return 0; 1430} 1431 1432unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i32_r(unsigned Op0) { 1433 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 1434 return fastEmitInst_r(X86::VPMOVM2DZ128rr, &X86::VR128XRegClass, Op0); 1435 } 1436 return 0; 1437} 1438 1439unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i64_r(unsigned Op0) { 1440 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 1441 return fastEmitInst_r(X86::VPMOVM2QZ256rr, &X86::VR256XRegClass, Op0); 1442 } 1443 return 0; 1444} 1445 1446unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_r(MVT RetVT, unsigned Op0) { 1447switch (RetVT.SimpleTy) { 1448 case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i32_r(Op0); 1449 case MVT::v4i64: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i64_r(Op0); 1450 default: return 0; 1451} 1452} 1453 1454unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i16_r(unsigned Op0) { 1455 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 1456 return fastEmitInst_r(X86::VPMOVM2WZ128rr, &X86::VR128XRegClass, Op0); 1457 } 1458 return 0; 1459} 1460 1461unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i32_r(unsigned Op0) { 1462 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 1463 return fastEmitInst_r(X86::VPMOVM2DZ256rr, &X86::VR256XRegClass, Op0); 1464 } 1465 return 0; 1466} 1467 1468unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i64_r(unsigned Op0) { 1469 if ((Subtarget->hasDQI())) { 1470 return fastEmitInst_r(X86::VPMOVM2QZrr, &X86::VR512RegClass, Op0); 1471 } 1472 return 0; 1473} 1474 1475unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_r(MVT RetVT, unsigned Op0) { 1476switch (RetVT.SimpleTy) { 1477 case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i16_r(Op0); 1478 case MVT::v8i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i32_r(Op0); 1479 case MVT::v8i64: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i64_r(Op0); 1480 default: return 0; 1481} 1482} 1483 1484unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i8_r(unsigned Op0) { 1485 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 1486 return fastEmitInst_r(X86::VPMOVM2BZ128rr, &X86::VR128XRegClass, Op0); 1487 } 1488 return 0; 1489} 1490 1491unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i16_r(unsigned Op0) { 1492 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 1493 return fastEmitInst_r(X86::VPMOVM2WZ256rr, &X86::VR256XRegClass, Op0); 1494 } 1495 return 0; 1496} 1497 1498unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i32_r(unsigned Op0) { 1499 if ((Subtarget->hasDQI())) { 1500 return fastEmitInst_r(X86::VPMOVM2DZrr, &X86::VR512RegClass, Op0); 1501 } 1502 return 0; 1503} 1504 1505unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_r(MVT RetVT, unsigned Op0) { 1506switch (RetVT.SimpleTy) { 1507 case MVT::v16i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i8_r(Op0); 1508 case MVT::v16i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i16_r(Op0); 1509 case MVT::v16i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i32_r(Op0); 1510 default: return 0; 1511} 1512} 1513 1514unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i8_r(unsigned Op0) { 1515 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 1516 return fastEmitInst_r(X86::VPMOVM2BZ256rr, &X86::VR256XRegClass, Op0); 1517 } 1518 return 0; 1519} 1520 1521unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i16_r(unsigned Op0) { 1522 if ((Subtarget->hasBWI())) { 1523 return fastEmitInst_r(X86::VPMOVM2WZrr, &X86::VR512RegClass, Op0); 1524 } 1525 return 0; 1526} 1527 1528unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_r(MVT RetVT, unsigned Op0) { 1529switch (RetVT.SimpleTy) { 1530 case MVT::v32i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i8_r(Op0); 1531 case MVT::v32i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i16_r(Op0); 1532 default: return 0; 1533} 1534} 1535 1536unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v64i1_r(MVT RetVT, unsigned Op0) { 1537 if (RetVT.SimpleTy != MVT::v64i8) 1538 return 0; 1539 if ((Subtarget->hasBWI())) { 1540 return fastEmitInst_r(X86::VPMOVM2BZrr, &X86::VR512RegClass, Op0); 1541 } 1542 return 0; 1543} 1544 1545unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_MVT_v16i16_r(unsigned Op0) { 1546 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 1547 return fastEmitInst_r(X86::VPMOVSXBWYrr, &X86::VR256RegClass, Op0); 1548 } 1549 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 1550 return fastEmitInst_r(X86::VPMOVSXBWZ256rr, &X86::VR256XRegClass, Op0); 1551 } 1552 return 0; 1553} 1554 1555unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_MVT_v16i32_r(unsigned Op0) { 1556 if ((Subtarget->hasAVX512())) { 1557 return fastEmitInst_r(X86::VPMOVSXBDZrr, &X86::VR512RegClass, Op0); 1558 } 1559 return 0; 1560} 1561 1562unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_r(MVT RetVT, unsigned Op0) { 1563switch (RetVT.SimpleTy) { 1564 case MVT::v16i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_MVT_v16i16_r(Op0); 1565 case MVT::v16i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_MVT_v16i32_r(Op0); 1566 default: return 0; 1567} 1568} 1569 1570unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v32i8_r(MVT RetVT, unsigned Op0) { 1571 if (RetVT.SimpleTy != MVT::v32i16) 1572 return 0; 1573 if ((Subtarget->hasBWI())) { 1574 return fastEmitInst_r(X86::VPMOVSXBWZrr, &X86::VR512RegClass, Op0); 1575 } 1576 return 0; 1577} 1578 1579unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_MVT_v8i32_r(unsigned Op0) { 1580 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 1581 return fastEmitInst_r(X86::VPMOVSXWDYrr, &X86::VR256RegClass, Op0); 1582 } 1583 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 1584 return fastEmitInst_r(X86::VPMOVSXWDZ256rr, &X86::VR256XRegClass, Op0); 1585 } 1586 return 0; 1587} 1588 1589unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_MVT_v8i64_r(unsigned Op0) { 1590 if ((Subtarget->hasAVX512())) { 1591 return fastEmitInst_r(X86::VPMOVSXWQZrr, &X86::VR512RegClass, Op0); 1592 } 1593 return 0; 1594} 1595 1596unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_r(MVT RetVT, unsigned Op0) { 1597switch (RetVT.SimpleTy) { 1598 case MVT::v8i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_MVT_v8i32_r(Op0); 1599 case MVT::v8i64: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_MVT_v8i64_r(Op0); 1600 default: return 0; 1601} 1602} 1603 1604unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i16_r(MVT RetVT, unsigned Op0) { 1605 if (RetVT.SimpleTy != MVT::v16i32) 1606 return 0; 1607 if ((Subtarget->hasAVX512())) { 1608 return fastEmitInst_r(X86::VPMOVSXWDZrr, &X86::VR512RegClass, Op0); 1609 } 1610 return 0; 1611} 1612 1613unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v4i32_r(MVT RetVT, unsigned Op0) { 1614 if (RetVT.SimpleTy != MVT::v4i64) 1615 return 0; 1616 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 1617 return fastEmitInst_r(X86::VPMOVSXDQYrr, &X86::VR256RegClass, Op0); 1618 } 1619 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 1620 return fastEmitInst_r(X86::VPMOVSXDQZ256rr, &X86::VR256XRegClass, Op0); 1621 } 1622 return 0; 1623} 1624 1625unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i32_r(MVT RetVT, unsigned Op0) { 1626 if (RetVT.SimpleTy != MVT::v8i64) 1627 return 0; 1628 if ((Subtarget->hasAVX512())) { 1629 return fastEmitInst_r(X86::VPMOVSXDQZrr, &X86::VR512RegClass, Op0); 1630 } 1631 return 0; 1632} 1633 1634unsigned fastEmit_ISD_SIGN_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0) { 1635 switch (VT.SimpleTy) { 1636 case MVT::i8: return fastEmit_ISD_SIGN_EXTEND_MVT_i8_r(RetVT, Op0); 1637 case MVT::i16: return fastEmit_ISD_SIGN_EXTEND_MVT_i16_r(RetVT, Op0); 1638 case MVT::i32: return fastEmit_ISD_SIGN_EXTEND_MVT_i32_r(RetVT, Op0); 1639 case MVT::v2i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v2i1_r(RetVT, Op0); 1640 case MVT::v4i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_r(RetVT, Op0); 1641 case MVT::v8i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_r(RetVT, Op0); 1642 case MVT::v16i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_r(RetVT, Op0); 1643 case MVT::v32i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_r(RetVT, Op0); 1644 case MVT::v64i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v64i1_r(RetVT, Op0); 1645 case MVT::v16i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_r(RetVT, Op0); 1646 case MVT::v32i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i8_r(RetVT, Op0); 1647 case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_r(RetVT, Op0); 1648 case MVT::v16i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i16_r(RetVT, Op0); 1649 case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i32_r(RetVT, Op0); 1650 case MVT::v8i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i32_r(RetVT, Op0); 1651 default: return 0; 1652 } 1653} 1654 1655// FastEmit functions for ISD::SIGN_EXTEND_VECTOR_INREG. 1656 1657unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(unsigned Op0) { 1658 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 1659 return fastEmitInst_r(X86::PMOVSXBWrr, &X86::VR128RegClass, Op0); 1660 } 1661 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 1662 return fastEmitInst_r(X86::VPMOVSXBWrr, &X86::VR128RegClass, Op0); 1663 } 1664 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 1665 return fastEmitInst_r(X86::VPMOVSXBWZ128rr, &X86::VR128XRegClass, Op0); 1666 } 1667 return 0; 1668} 1669 1670unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(unsigned Op0) { 1671 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 1672 return fastEmitInst_r(X86::PMOVSXBDrr, &X86::VR128RegClass, Op0); 1673 } 1674 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 1675 return fastEmitInst_r(X86::VPMOVSXBDrr, &X86::VR128RegClass, Op0); 1676 } 1677 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 1678 return fastEmitInst_r(X86::VPMOVSXBDZ128rr, &X86::VR128XRegClass, Op0); 1679 } 1680 return 0; 1681} 1682 1683unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i32_r(unsigned Op0) { 1684 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 1685 return fastEmitInst_r(X86::VPMOVSXBDYrr, &X86::VR256RegClass, Op0); 1686 } 1687 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 1688 return fastEmitInst_r(X86::VPMOVSXBDZ256rr, &X86::VR256XRegClass, Op0); 1689 } 1690 return 0; 1691} 1692 1693unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(unsigned Op0) { 1694 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 1695 return fastEmitInst_r(X86::PMOVSXBQrr, &X86::VR128RegClass, Op0); 1696 } 1697 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 1698 return fastEmitInst_r(X86::VPMOVSXBQrr, &X86::VR128RegClass, Op0); 1699 } 1700 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 1701 return fastEmitInst_r(X86::VPMOVSXBQZ128rr, &X86::VR128XRegClass, Op0); 1702 } 1703 return 0; 1704} 1705 1706unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i64_r(unsigned Op0) { 1707 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 1708 return fastEmitInst_r(X86::VPMOVSXBQYrr, &X86::VR256RegClass, Op0); 1709 } 1710 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 1711 return fastEmitInst_r(X86::VPMOVSXBQZ256rr, &X86::VR256XRegClass, Op0); 1712 } 1713 return 0; 1714} 1715 1716unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i64_r(unsigned Op0) { 1717 if ((Subtarget->hasAVX512())) { 1718 return fastEmitInst_r(X86::VPMOVSXBQZrr, &X86::VR512RegClass, Op0); 1719 } 1720 return 0; 1721} 1722 1723unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_r(MVT RetVT, unsigned Op0) { 1724switch (RetVT.SimpleTy) { 1725 case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(Op0); 1726 case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(Op0); 1727 case MVT::v8i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i32_r(Op0); 1728 case MVT::v2i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(Op0); 1729 case MVT::v4i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i64_r(Op0); 1730 case MVT::v8i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i64_r(Op0); 1731 default: return 0; 1732} 1733} 1734 1735unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(unsigned Op0) { 1736 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 1737 return fastEmitInst_r(X86::PMOVSXWDrr, &X86::VR128RegClass, Op0); 1738 } 1739 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 1740 return fastEmitInst_r(X86::VPMOVSXWDrr, &X86::VR128RegClass, Op0); 1741 } 1742 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 1743 return fastEmitInst_r(X86::VPMOVSXWDZ128rr, &X86::VR128XRegClass, Op0); 1744 } 1745 return 0; 1746} 1747 1748unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(unsigned Op0) { 1749 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 1750 return fastEmitInst_r(X86::PMOVSXWQrr, &X86::VR128RegClass, Op0); 1751 } 1752 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 1753 return fastEmitInst_r(X86::VPMOVSXWQrr, &X86::VR128RegClass, Op0); 1754 } 1755 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 1756 return fastEmitInst_r(X86::VPMOVSXWQZ128rr, &X86::VR128XRegClass, Op0); 1757 } 1758 return 0; 1759} 1760 1761unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i64_r(unsigned Op0) { 1762 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 1763 return fastEmitInst_r(X86::VPMOVSXWQYrr, &X86::VR256RegClass, Op0); 1764 } 1765 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 1766 return fastEmitInst_r(X86::VPMOVSXWQZ256rr, &X86::VR256XRegClass, Op0); 1767 } 1768 return 0; 1769} 1770 1771unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_r(MVT RetVT, unsigned Op0) { 1772switch (RetVT.SimpleTy) { 1773 case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(Op0); 1774 case MVT::v2i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(Op0); 1775 case MVT::v4i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i64_r(Op0); 1776 default: return 0; 1777} 1778} 1779 1780unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v4i32_r(MVT RetVT, unsigned Op0) { 1781 if (RetVT.SimpleTy != MVT::v2i64) 1782 return 0; 1783 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 1784 return fastEmitInst_r(X86::PMOVSXDQrr, &X86::VR128RegClass, Op0); 1785 } 1786 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 1787 return fastEmitInst_r(X86::VPMOVSXDQrr, &X86::VR128RegClass, Op0); 1788 } 1789 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 1790 return fastEmitInst_r(X86::VPMOVSXDQZ128rr, &X86::VR128XRegClass, Op0); 1791 } 1792 return 0; 1793} 1794 1795unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_r(MVT VT, MVT RetVT, unsigned Op0) { 1796 switch (VT.SimpleTy) { 1797 case MVT::v16i8: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_r(RetVT, Op0); 1798 case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_r(RetVT, Op0); 1799 case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v4i32_r(RetVT, Op0); 1800 default: return 0; 1801 } 1802} 1803 1804// FastEmit functions for ISD::SINT_TO_FP. 1805 1806unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(unsigned Op0) { 1807 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 1808 return fastEmitInst_r(X86::CVTSI2SSrr, &X86::FR32RegClass, Op0); 1809 } 1810 return 0; 1811} 1812 1813unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(unsigned Op0) { 1814 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 1815 return fastEmitInst_r(X86::CVTSI2SDrr, &X86::FR64RegClass, Op0); 1816 } 1817 return 0; 1818} 1819 1820unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_r(MVT RetVT, unsigned Op0) { 1821switch (RetVT.SimpleTy) { 1822 case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0); 1823 case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0); 1824 default: return 0; 1825} 1826} 1827 1828unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(unsigned Op0) { 1829 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 1830 return fastEmitInst_r(X86::CVTSI642SSrr, &X86::FR32RegClass, Op0); 1831 } 1832 return 0; 1833} 1834 1835unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(unsigned Op0) { 1836 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 1837 return fastEmitInst_r(X86::CVTSI642SDrr, &X86::FR64RegClass, Op0); 1838 } 1839 return 0; 1840} 1841 1842unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_r(MVT RetVT, unsigned Op0) { 1843switch (RetVT.SimpleTy) { 1844 case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0); 1845 case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0); 1846 default: return 0; 1847} 1848} 1849 1850unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(MVT RetVT, unsigned Op0) { 1851 if (RetVT.SimpleTy != MVT::v8f16) 1852 return 0; 1853 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 1854 return fastEmitInst_r(X86::VCVTW2PHZ128rr, &X86::VR128XRegClass, Op0); 1855 } 1856 return 0; 1857} 1858 1859unsigned fastEmit_ISD_SINT_TO_FP_MVT_v16i16_r(MVT RetVT, unsigned Op0) { 1860 if (RetVT.SimpleTy != MVT::v16f16) 1861 return 0; 1862 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 1863 return fastEmitInst_r(X86::VCVTW2PHZ256rr, &X86::VR256XRegClass, Op0); 1864 } 1865 return 0; 1866} 1867 1868unsigned fastEmit_ISD_SINT_TO_FP_MVT_v32i16_r(MVT RetVT, unsigned Op0) { 1869 if (RetVT.SimpleTy != MVT::v32f16) 1870 return 0; 1871 if ((Subtarget->hasFP16())) { 1872 return fastEmitInst_r(X86::VCVTW2PHZrr, &X86::VR512RegClass, Op0); 1873 } 1874 return 0; 1875} 1876 1877unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(unsigned Op0) { 1878 if ((Subtarget->hasVLX())) { 1879 return fastEmitInst_r(X86::VCVTDQ2PSZ128rr, &X86::VR128XRegClass, Op0); 1880 } 1881 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 1882 return fastEmitInst_r(X86::CVTDQ2PSrr, &X86::VR128RegClass, Op0); 1883 } 1884 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 1885 return fastEmitInst_r(X86::VCVTDQ2PSrr, &X86::VR128RegClass, Op0); 1886 } 1887 return 0; 1888} 1889 1890unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(unsigned Op0) { 1891 if ((Subtarget->hasVLX())) { 1892 return fastEmitInst_r(X86::VCVTDQ2PDZ256rr, &X86::VR256XRegClass, Op0); 1893 } 1894 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 1895 return fastEmitInst_r(X86::VCVTDQ2PDYrr, &X86::VR256RegClass, Op0); 1896 } 1897 return 0; 1898} 1899 1900unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0) { 1901switch (RetVT.SimpleTy) { 1902 case MVT::v4f32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0); 1903 case MVT::v4f64: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0); 1904 default: return 0; 1905} 1906} 1907 1908unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f16_r(unsigned Op0) { 1909 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 1910 return fastEmitInst_r(X86::VCVTDQ2PHZ256rr, &X86::VR128XRegClass, Op0); 1911 } 1912 return 0; 1913} 1914 1915unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(unsigned Op0) { 1916 if ((Subtarget->hasVLX())) { 1917 return fastEmitInst_r(X86::VCVTDQ2PSZ256rr, &X86::VR256XRegClass, Op0); 1918 } 1919 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 1920 return fastEmitInst_r(X86::VCVTDQ2PSYrr, &X86::VR256RegClass, Op0); 1921 } 1922 return 0; 1923} 1924 1925unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(unsigned Op0) { 1926 if ((Subtarget->hasAVX512())) { 1927 return fastEmitInst_r(X86::VCVTDQ2PDZrr, &X86::VR512RegClass, Op0); 1928 } 1929 return 0; 1930} 1931 1932unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i32_r(MVT RetVT, unsigned Op0) { 1933switch (RetVT.SimpleTy) { 1934 case MVT::v8f16: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Op0); 1935 case MVT::v8f32: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0); 1936 case MVT::v8f64: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0); 1937 default: return 0; 1938} 1939} 1940 1941unsigned fastEmit_ISD_SINT_TO_FP_MVT_v16i32_MVT_v16f16_r(unsigned Op0) { 1942 if ((Subtarget->hasFP16())) { 1943 return fastEmitInst_r(X86::VCVTDQ2PHZrr, &X86::VR256XRegClass, Op0); 1944 } 1945 return 0; 1946} 1947 1948unsigned fastEmit_ISD_SINT_TO_FP_MVT_v16i32_MVT_v16f32_r(unsigned Op0) { 1949 if ((Subtarget->hasAVX512())) { 1950 return fastEmitInst_r(X86::VCVTDQ2PSZrr, &X86::VR512RegClass, Op0); 1951 } 1952 return 0; 1953} 1954 1955unsigned fastEmit_ISD_SINT_TO_FP_MVT_v16i32_r(MVT RetVT, unsigned Op0) { 1956switch (RetVT.SimpleTy) { 1957 case MVT::v16f16: return fastEmit_ISD_SINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Op0); 1958 case MVT::v16f32: return fastEmit_ISD_SINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Op0); 1959 default: return 0; 1960} 1961} 1962 1963unsigned fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, unsigned Op0) { 1964 if (RetVT.SimpleTy != MVT::v2f64) 1965 return 0; 1966 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 1967 return fastEmitInst_r(X86::VCVTQQ2PDZ128rr, &X86::VR128XRegClass, Op0); 1968 } 1969 return 0; 1970} 1971 1972unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(unsigned Op0) { 1973 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 1974 return fastEmitInst_r(X86::VCVTQQ2PSZ256rr, &X86::VR128XRegClass, Op0); 1975 } 1976 return 0; 1977} 1978 1979unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(unsigned Op0) { 1980 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 1981 return fastEmitInst_r(X86::VCVTQQ2PDZ256rr, &X86::VR256XRegClass, Op0); 1982 } 1983 return 0; 1984} 1985 1986unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i64_r(MVT RetVT, unsigned Op0) { 1987switch (RetVT.SimpleTy) { 1988 case MVT::v4f32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0); 1989 case MVT::v4f64: return fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0); 1990 default: return 0; 1991} 1992} 1993 1994unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f16_r(unsigned Op0) { 1995 if ((Subtarget->hasFP16())) { 1996 return fastEmitInst_r(X86::VCVTQQ2PHZrr, &X86::VR128XRegClass, Op0); 1997 } 1998 return 0; 1999} 2000 2001unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(unsigned Op0) { 2002 if ((Subtarget->hasDQI())) { 2003 return fastEmitInst_r(X86::VCVTQQ2PSZrr, &X86::VR256XRegClass, Op0); 2004 } 2005 return 0; 2006} 2007 2008unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(unsigned Op0) { 2009 if ((Subtarget->hasDQI())) { 2010 return fastEmitInst_r(X86::VCVTQQ2PDZrr, &X86::VR512RegClass, Op0); 2011 } 2012 return 0; 2013} 2014 2015unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i64_r(MVT RetVT, unsigned Op0) { 2016switch (RetVT.SimpleTy) { 2017 case MVT::v8f16: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Op0); 2018 case MVT::v8f32: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0); 2019 case MVT::v8f64: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0); 2020 default: return 0; 2021} 2022} 2023 2024unsigned fastEmit_ISD_SINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0) { 2025 switch (VT.SimpleTy) { 2026 case MVT::i32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_r(RetVT, Op0); 2027 case MVT::i64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_r(RetVT, Op0); 2028 case MVT::v8i16: return fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(RetVT, Op0); 2029 case MVT::v16i16: return fastEmit_ISD_SINT_TO_FP_MVT_v16i16_r(RetVT, Op0); 2030 case MVT::v32i16: return fastEmit_ISD_SINT_TO_FP_MVT_v32i16_r(RetVT, Op0); 2031 case MVT::v4i32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0); 2032 case MVT::v8i32: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_r(RetVT, Op0); 2033 case MVT::v16i32: return fastEmit_ISD_SINT_TO_FP_MVT_v16i32_r(RetVT, Op0); 2034 case MVT::v2i64: return fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0); 2035 case MVT::v4i64: return fastEmit_ISD_SINT_TO_FP_MVT_v4i64_r(RetVT, Op0); 2036 case MVT::v8i64: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_r(RetVT, Op0); 2037 default: return 0; 2038 } 2039} 2040 2041// FastEmit functions for ISD::STRICT_FP_EXTEND. 2042 2043unsigned fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(MVT RetVT, unsigned Op0) { 2044 if (RetVT.SimpleTy != MVT::f64) 2045 return 0; 2046 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 2047 return fastEmitInst_r(X86::CVTSS2SDrr, &X86::FR64RegClass, Op0); 2048 } 2049 return 0; 2050} 2051 2052unsigned fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f16_MVT_v8f32_r(unsigned Op0) { 2053 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 2054 return fastEmitInst_r(X86::VCVTPH2PSXZ256rr, &X86::VR256XRegClass, Op0); 2055 } 2056 return 0; 2057} 2058 2059unsigned fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f16_MVT_v8f64_r(unsigned Op0) { 2060 if ((Subtarget->hasFP16())) { 2061 return fastEmitInst_r(X86::VCVTPH2PDZrr, &X86::VR512RegClass, Op0); 2062 } 2063 return 0; 2064} 2065 2066unsigned fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 2067switch (RetVT.SimpleTy) { 2068 case MVT::v8f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f16_MVT_v8f32_r(Op0); 2069 case MVT::v8f64: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f16_MVT_v8f64_r(Op0); 2070 default: return 0; 2071} 2072} 2073 2074unsigned fastEmit_ISD_STRICT_FP_EXTEND_MVT_v16f16_r(MVT RetVT, unsigned Op0) { 2075 if (RetVT.SimpleTy != MVT::v16f32) 2076 return 0; 2077 if ((Subtarget->hasFP16())) { 2078 return fastEmitInst_r(X86::VCVTPH2PSXZrr, &X86::VR512RegClass, Op0); 2079 } 2080 return 0; 2081} 2082 2083unsigned fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 2084 if (RetVT.SimpleTy != MVT::v4f64) 2085 return 0; 2086 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 2087 return fastEmitInst_r(X86::VCVTPS2PDZ256rr, &X86::VR256XRegClass, Op0); 2088 } 2089 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 2090 return fastEmitInst_r(X86::VCVTPS2PDYrr, &X86::VR256RegClass, Op0); 2091 } 2092 return 0; 2093} 2094 2095unsigned fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f32_r(MVT RetVT, unsigned Op0) { 2096 if (RetVT.SimpleTy != MVT::v8f64) 2097 return 0; 2098 if ((Subtarget->hasAVX512())) { 2099 return fastEmitInst_r(X86::VCVTPS2PDZrr, &X86::VR512RegClass, Op0); 2100 } 2101 return 0; 2102} 2103 2104unsigned fastEmit_ISD_STRICT_FP_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0) { 2105 switch (VT.SimpleTy) { 2106 case MVT::f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(RetVT, Op0); 2107 case MVT::v8f16: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f16_r(RetVT, Op0); 2108 case MVT::v16f16: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v16f16_r(RetVT, Op0); 2109 case MVT::v4f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4f32_r(RetVT, Op0); 2110 case MVT::v8f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f32_r(RetVT, Op0); 2111 default: return 0; 2112 } 2113} 2114 2115// FastEmit functions for ISD::STRICT_FP_ROUND. 2116 2117unsigned fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(MVT RetVT, unsigned Op0) { 2118 if (RetVT.SimpleTy != MVT::f32) 2119 return 0; 2120 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 2121 return fastEmitInst_r(X86::CVTSD2SSrr, &X86::FR32RegClass, Op0); 2122 } 2123 return 0; 2124} 2125 2126unsigned fastEmit_ISD_STRICT_FP_ROUND_r(MVT VT, MVT RetVT, unsigned Op0) { 2127 switch (VT.SimpleTy) { 2128 case MVT::f64: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(RetVT, Op0); 2129 default: return 0; 2130 } 2131} 2132 2133// FastEmit functions for ISD::STRICT_FP_TO_SINT. 2134 2135unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i32_r(unsigned Op0) { 2136 if ((Subtarget->hasFP16())) { 2137 return fastEmitInst_r(X86::VCVTTSH2SIZrr, &X86::GR32RegClass, Op0); 2138 } 2139 return 0; 2140} 2141 2142unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i64_r(unsigned Op0) { 2143 if ((Subtarget->hasFP16())) { 2144 return fastEmitInst_r(X86::VCVTTSH2SI64Zrr, &X86::GR64RegClass, Op0); 2145 } 2146 return 0; 2147} 2148 2149unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_r(MVT RetVT, unsigned Op0) { 2150switch (RetVT.SimpleTy) { 2151 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i32_r(Op0); 2152 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i64_r(Op0); 2153 default: return 0; 2154} 2155} 2156 2157unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i32_r(unsigned Op0) { 2158 if ((Subtarget->hasAVX512())) { 2159 return fastEmitInst_r(X86::VCVTTSS2SIZrr, &X86::GR32RegClass, Op0); 2160 } 2161 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 2162 return fastEmitInst_r(X86::CVTTSS2SIrr, &X86::GR32RegClass, Op0); 2163 } 2164 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 2165 return fastEmitInst_r(X86::VCVTTSS2SIrr, &X86::GR32RegClass, Op0); 2166 } 2167 return 0; 2168} 2169 2170unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i64_r(unsigned Op0) { 2171 if ((Subtarget->hasAVX512())) { 2172 return fastEmitInst_r(X86::VCVTTSS2SI64Zrr, &X86::GR64RegClass, Op0); 2173 } 2174 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 2175 return fastEmitInst_r(X86::CVTTSS2SI64rr, &X86::GR64RegClass, Op0); 2176 } 2177 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 2178 return fastEmitInst_r(X86::VCVTTSS2SI64rr, &X86::GR64RegClass, Op0); 2179 } 2180 return 0; 2181} 2182 2183unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_r(MVT RetVT, unsigned Op0) { 2184switch (RetVT.SimpleTy) { 2185 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0); 2186 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0); 2187 default: return 0; 2188} 2189} 2190 2191unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i32_r(unsigned Op0) { 2192 if ((Subtarget->hasAVX512())) { 2193 return fastEmitInst_r(X86::VCVTTSD2SIZrr, &X86::GR32RegClass, Op0); 2194 } 2195 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 2196 return fastEmitInst_r(X86::CVTTSD2SIrr, &X86::GR32RegClass, Op0); 2197 } 2198 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 2199 return fastEmitInst_r(X86::VCVTTSD2SIrr, &X86::GR32RegClass, Op0); 2200 } 2201 return 0; 2202} 2203 2204unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i64_r(unsigned Op0) { 2205 if ((Subtarget->hasAVX512())) { 2206 return fastEmitInst_r(X86::VCVTTSD2SI64Zrr, &X86::GR64RegClass, Op0); 2207 } 2208 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 2209 return fastEmitInst_r(X86::CVTTSD2SI64rr, &X86::GR64RegClass, Op0); 2210 } 2211 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 2212 return fastEmitInst_r(X86::VCVTTSD2SI64rr, &X86::GR64RegClass, Op0); 2213 } 2214 return 0; 2215} 2216 2217unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_r(MVT RetVT, unsigned Op0) { 2218switch (RetVT.SimpleTy) { 2219 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0); 2220 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0); 2221 default: return 0; 2222} 2223} 2224 2225unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f64_r(MVT RetVT, unsigned Op0) { 2226 if (RetVT.SimpleTy != MVT::v4i32) 2227 return 0; 2228 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 2229 return fastEmitInst_r(X86::VCVTTPD2DQYrr, &X86::VR128RegClass, Op0); 2230 } 2231 return 0; 2232} 2233 2234unsigned fastEmit_ISD_STRICT_FP_TO_SINT_r(MVT VT, MVT RetVT, unsigned Op0) { 2235 switch (VT.SimpleTy) { 2236 case MVT::f16: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_r(RetVT, Op0); 2237 case MVT::f32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_r(RetVT, Op0); 2238 case MVT::f64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_r(RetVT, Op0); 2239 case MVT::v4f64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f64_r(RetVT, Op0); 2240 default: return 0; 2241 } 2242} 2243 2244// FastEmit functions for ISD::STRICT_FP_TO_UINT. 2245 2246unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i32_r(unsigned Op0) { 2247 if ((Subtarget->hasFP16())) { 2248 return fastEmitInst_r(X86::VCVTTSH2USIZrr, &X86::GR32RegClass, Op0); 2249 } 2250 return 0; 2251} 2252 2253unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i64_r(unsigned Op0) { 2254 if ((Subtarget->hasFP16())) { 2255 return fastEmitInst_r(X86::VCVTTSH2USI64Zrr, &X86::GR64RegClass, Op0); 2256 } 2257 return 0; 2258} 2259 2260unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_r(MVT RetVT, unsigned Op0) { 2261switch (RetVT.SimpleTy) { 2262 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i32_r(Op0); 2263 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i64_r(Op0); 2264 default: return 0; 2265} 2266} 2267 2268unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i32_r(unsigned Op0) { 2269 if ((Subtarget->hasAVX512())) { 2270 return fastEmitInst_r(X86::VCVTTSS2USIZrr, &X86::GR32RegClass, Op0); 2271 } 2272 return 0; 2273} 2274 2275unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i64_r(unsigned Op0) { 2276 if ((Subtarget->hasAVX512())) { 2277 return fastEmitInst_r(X86::VCVTTSS2USI64Zrr, &X86::GR64RegClass, Op0); 2278 } 2279 return 0; 2280} 2281 2282unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_r(MVT RetVT, unsigned Op0) { 2283switch (RetVT.SimpleTy) { 2284 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0); 2285 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0); 2286 default: return 0; 2287} 2288} 2289 2290unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i32_r(unsigned Op0) { 2291 if ((Subtarget->hasAVX512())) { 2292 return fastEmitInst_r(X86::VCVTTSD2USIZrr, &X86::GR32RegClass, Op0); 2293 } 2294 return 0; 2295} 2296 2297unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i64_r(unsigned Op0) { 2298 if ((Subtarget->hasAVX512())) { 2299 return fastEmitInst_r(X86::VCVTTSD2USI64Zrr, &X86::GR64RegClass, Op0); 2300 } 2301 return 0; 2302} 2303 2304unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_r(MVT RetVT, unsigned Op0) { 2305switch (RetVT.SimpleTy) { 2306 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0); 2307 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0); 2308 default: return 0; 2309} 2310} 2311 2312unsigned fastEmit_ISD_STRICT_FP_TO_UINT_r(MVT VT, MVT RetVT, unsigned Op0) { 2313 switch (VT.SimpleTy) { 2314 case MVT::f16: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_r(RetVT, Op0); 2315 case MVT::f32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_r(RetVT, Op0); 2316 case MVT::f64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_r(RetVT, Op0); 2317 default: return 0; 2318 } 2319} 2320 2321// FastEmit functions for ISD::STRICT_FSQRT. 2322 2323unsigned fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(MVT RetVT, unsigned Op0) { 2324 if (RetVT.SimpleTy != MVT::f32) 2325 return 0; 2326 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 2327 return fastEmitInst_r(X86::SQRTSSr, &X86::FR32RegClass, Op0); 2328 } 2329 if ((!Subtarget->hasSSE1())) { 2330 return fastEmitInst_r(X86::SQRT_Fp32, &X86::RFP32RegClass, Op0); 2331 } 2332 return 0; 2333} 2334 2335unsigned fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(MVT RetVT, unsigned Op0) { 2336 if (RetVT.SimpleTy != MVT::f64) 2337 return 0; 2338 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 2339 return fastEmitInst_r(X86::SQRTSDr, &X86::FR64RegClass, Op0); 2340 } 2341 if ((!Subtarget->hasSSE2())) { 2342 return fastEmitInst_r(X86::SQRT_Fp64, &X86::RFP64RegClass, Op0); 2343 } 2344 return 0; 2345} 2346 2347unsigned fastEmit_ISD_STRICT_FSQRT_MVT_f80_r(MVT RetVT, unsigned Op0) { 2348 if (RetVT.SimpleTy != MVT::f80) 2349 return 0; 2350 if ((Subtarget->hasX87())) { 2351 return fastEmitInst_r(X86::SQRT_Fp80, &X86::RFP80RegClass, Op0); 2352 } 2353 return 0; 2354} 2355 2356unsigned fastEmit_ISD_STRICT_FSQRT_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 2357 if (RetVT.SimpleTy != MVT::v8f16) 2358 return 0; 2359 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 2360 return fastEmitInst_r(X86::VSQRTPHZ128r, &X86::VR128XRegClass, Op0); 2361 } 2362 return 0; 2363} 2364 2365unsigned fastEmit_ISD_STRICT_FSQRT_MVT_v16f16_r(MVT RetVT, unsigned Op0) { 2366 if (RetVT.SimpleTy != MVT::v16f16) 2367 return 0; 2368 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 2369 return fastEmitInst_r(X86::VSQRTPHZ256r, &X86::VR256XRegClass, Op0); 2370 } 2371 return 0; 2372} 2373 2374unsigned fastEmit_ISD_STRICT_FSQRT_MVT_v32f16_r(MVT RetVT, unsigned Op0) { 2375 if (RetVT.SimpleTy != MVT::v32f16) 2376 return 0; 2377 if ((Subtarget->hasFP16())) { 2378 return fastEmitInst_r(X86::VSQRTPHZr, &X86::VR512RegClass, Op0); 2379 } 2380 return 0; 2381} 2382 2383unsigned fastEmit_ISD_STRICT_FSQRT_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 2384 if (RetVT.SimpleTy != MVT::v4f32) 2385 return 0; 2386 if ((Subtarget->hasVLX())) { 2387 return fastEmitInst_r(X86::VSQRTPSZ128r, &X86::VR128XRegClass, Op0); 2388 } 2389 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 2390 return fastEmitInst_r(X86::SQRTPSr, &X86::VR128RegClass, Op0); 2391 } 2392 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 2393 return fastEmitInst_r(X86::VSQRTPSr, &X86::VR128RegClass, Op0); 2394 } 2395 return 0; 2396} 2397 2398unsigned fastEmit_ISD_STRICT_FSQRT_MVT_v8f32_r(MVT RetVT, unsigned Op0) { 2399 if (RetVT.SimpleTy != MVT::v8f32) 2400 return 0; 2401 if ((Subtarget->hasVLX())) { 2402 return fastEmitInst_r(X86::VSQRTPSZ256r, &X86::VR256XRegClass, Op0); 2403 } 2404 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 2405 return fastEmitInst_r(X86::VSQRTPSYr, &X86::VR256RegClass, Op0); 2406 } 2407 return 0; 2408} 2409 2410unsigned fastEmit_ISD_STRICT_FSQRT_MVT_v16f32_r(MVT RetVT, unsigned Op0) { 2411 if (RetVT.SimpleTy != MVT::v16f32) 2412 return 0; 2413 if ((Subtarget->hasAVX512())) { 2414 return fastEmitInst_r(X86::VSQRTPSZr, &X86::VR512RegClass, Op0); 2415 } 2416 return 0; 2417} 2418 2419unsigned fastEmit_ISD_STRICT_FSQRT_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 2420 if (RetVT.SimpleTy != MVT::v2f64) 2421 return 0; 2422 if ((Subtarget->hasVLX())) { 2423 return fastEmitInst_r(X86::VSQRTPDZ128r, &X86::VR128XRegClass, Op0); 2424 } 2425 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 2426 return fastEmitInst_r(X86::SQRTPDr, &X86::VR128RegClass, Op0); 2427 } 2428 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 2429 return fastEmitInst_r(X86::VSQRTPDr, &X86::VR128RegClass, Op0); 2430 } 2431 return 0; 2432} 2433 2434unsigned fastEmit_ISD_STRICT_FSQRT_MVT_v4f64_r(MVT RetVT, unsigned Op0) { 2435 if (RetVT.SimpleTy != MVT::v4f64) 2436 return 0; 2437 if ((Subtarget->hasVLX())) { 2438 return fastEmitInst_r(X86::VSQRTPDZ256r, &X86::VR256XRegClass, Op0); 2439 } 2440 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 2441 return fastEmitInst_r(X86::VSQRTPDYr, &X86::VR256RegClass, Op0); 2442 } 2443 return 0; 2444} 2445 2446unsigned fastEmit_ISD_STRICT_FSQRT_MVT_v8f64_r(MVT RetVT, unsigned Op0) { 2447 if (RetVT.SimpleTy != MVT::v8f64) 2448 return 0; 2449 if ((Subtarget->hasAVX512())) { 2450 return fastEmitInst_r(X86::VSQRTPDZr, &X86::VR512RegClass, Op0); 2451 } 2452 return 0; 2453} 2454 2455unsigned fastEmit_ISD_STRICT_FSQRT_r(MVT VT, MVT RetVT, unsigned Op0) { 2456 switch (VT.SimpleTy) { 2457 case MVT::f32: return fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(RetVT, Op0); 2458 case MVT::f64: return fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(RetVT, Op0); 2459 case MVT::f80: return fastEmit_ISD_STRICT_FSQRT_MVT_f80_r(RetVT, Op0); 2460 case MVT::v8f16: return fastEmit_ISD_STRICT_FSQRT_MVT_v8f16_r(RetVT, Op0); 2461 case MVT::v16f16: return fastEmit_ISD_STRICT_FSQRT_MVT_v16f16_r(RetVT, Op0); 2462 case MVT::v32f16: return fastEmit_ISD_STRICT_FSQRT_MVT_v32f16_r(RetVT, Op0); 2463 case MVT::v4f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v4f32_r(RetVT, Op0); 2464 case MVT::v8f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v8f32_r(RetVT, Op0); 2465 case MVT::v16f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v16f32_r(RetVT, Op0); 2466 case MVT::v2f64: return fastEmit_ISD_STRICT_FSQRT_MVT_v2f64_r(RetVT, Op0); 2467 case MVT::v4f64: return fastEmit_ISD_STRICT_FSQRT_MVT_v4f64_r(RetVT, Op0); 2468 case MVT::v8f64: return fastEmit_ISD_STRICT_FSQRT_MVT_v8f64_r(RetVT, Op0); 2469 default: return 0; 2470 } 2471} 2472 2473// FastEmit functions for ISD::STRICT_SINT_TO_FP. 2474 2475unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f32_r(unsigned Op0) { 2476 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 2477 return fastEmitInst_r(X86::CVTSI2SSrr, &X86::FR32RegClass, Op0); 2478 } 2479 return 0; 2480} 2481 2482unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f64_r(unsigned Op0) { 2483 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 2484 return fastEmitInst_r(X86::CVTSI2SDrr, &X86::FR64RegClass, Op0); 2485 } 2486 return 0; 2487} 2488 2489unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_r(MVT RetVT, unsigned Op0) { 2490switch (RetVT.SimpleTy) { 2491 case MVT::f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0); 2492 case MVT::f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0); 2493 default: return 0; 2494} 2495} 2496 2497unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f32_r(unsigned Op0) { 2498 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 2499 return fastEmitInst_r(X86::CVTSI642SSrr, &X86::FR32RegClass, Op0); 2500 } 2501 return 0; 2502} 2503 2504unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f64_r(unsigned Op0) { 2505 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 2506 return fastEmitInst_r(X86::CVTSI642SDrr, &X86::FR64RegClass, Op0); 2507 } 2508 return 0; 2509} 2510 2511unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_r(MVT RetVT, unsigned Op0) { 2512switch (RetVT.SimpleTy) { 2513 case MVT::f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0); 2514 case MVT::f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0); 2515 default: return 0; 2516} 2517} 2518 2519unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i16_r(MVT RetVT, unsigned Op0) { 2520 if (RetVT.SimpleTy != MVT::v8f16) 2521 return 0; 2522 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 2523 return fastEmitInst_r(X86::VCVTW2PHZ128rr, &X86::VR128XRegClass, Op0); 2524 } 2525 return 0; 2526} 2527 2528unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i16_r(MVT RetVT, unsigned Op0) { 2529 if (RetVT.SimpleTy != MVT::v16f16) 2530 return 0; 2531 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 2532 return fastEmitInst_r(X86::VCVTW2PHZ256rr, &X86::VR256XRegClass, Op0); 2533 } 2534 return 0; 2535} 2536 2537unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v32i16_r(MVT RetVT, unsigned Op0) { 2538 if (RetVT.SimpleTy != MVT::v32f16) 2539 return 0; 2540 if ((Subtarget->hasFP16())) { 2541 return fastEmitInst_r(X86::VCVTW2PHZrr, &X86::VR512RegClass, Op0); 2542 } 2543 return 0; 2544} 2545 2546unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(unsigned Op0) { 2547 if ((Subtarget->hasVLX())) { 2548 return fastEmitInst_r(X86::VCVTDQ2PSZ128rr, &X86::VR128XRegClass, Op0); 2549 } 2550 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 2551 return fastEmitInst_r(X86::CVTDQ2PSrr, &X86::VR128RegClass, Op0); 2552 } 2553 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 2554 return fastEmitInst_r(X86::VCVTDQ2PSrr, &X86::VR128RegClass, Op0); 2555 } 2556 return 0; 2557} 2558 2559unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(unsigned Op0) { 2560 if ((Subtarget->hasVLX())) { 2561 return fastEmitInst_r(X86::VCVTDQ2PDZ256rr, &X86::VR256XRegClass, Op0); 2562 } 2563 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 2564 return fastEmitInst_r(X86::VCVTDQ2PDYrr, &X86::VR256RegClass, Op0); 2565 } 2566 return 0; 2567} 2568 2569unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0) { 2570switch (RetVT.SimpleTy) { 2571 case MVT::v4f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0); 2572 case MVT::v4f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0); 2573 default: return 0; 2574} 2575} 2576 2577unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f16_r(unsigned Op0) { 2578 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 2579 return fastEmitInst_r(X86::VCVTDQ2PHZ256rr, &X86::VR128XRegClass, Op0); 2580 } 2581 return 0; 2582} 2583 2584unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(unsigned Op0) { 2585 if ((Subtarget->hasVLX())) { 2586 return fastEmitInst_r(X86::VCVTDQ2PSZ256rr, &X86::VR256XRegClass, Op0); 2587 } 2588 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 2589 return fastEmitInst_r(X86::VCVTDQ2PSYrr, &X86::VR256RegClass, Op0); 2590 } 2591 return 0; 2592} 2593 2594unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(unsigned Op0) { 2595 if ((Subtarget->hasAVX512())) { 2596 return fastEmitInst_r(X86::VCVTDQ2PDZrr, &X86::VR512RegClass, Op0); 2597 } 2598 return 0; 2599} 2600 2601unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_r(MVT RetVT, unsigned Op0) { 2602switch (RetVT.SimpleTy) { 2603 case MVT::v8f16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Op0); 2604 case MVT::v8f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0); 2605 case MVT::v8f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0); 2606 default: return 0; 2607} 2608} 2609 2610unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_MVT_v16f16_r(unsigned Op0) { 2611 if ((Subtarget->hasFP16())) { 2612 return fastEmitInst_r(X86::VCVTDQ2PHZrr, &X86::VR256XRegClass, Op0); 2613 } 2614 return 0; 2615} 2616 2617unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_MVT_v16f32_r(unsigned Op0) { 2618 if ((Subtarget->hasAVX512())) { 2619 return fastEmitInst_r(X86::VCVTDQ2PSZrr, &X86::VR512RegClass, Op0); 2620 } 2621 return 0; 2622} 2623 2624unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_r(MVT RetVT, unsigned Op0) { 2625switch (RetVT.SimpleTy) { 2626 case MVT::v16f16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Op0); 2627 case MVT::v16f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Op0); 2628 default: return 0; 2629} 2630} 2631 2632unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, unsigned Op0) { 2633 if (RetVT.SimpleTy != MVT::v2f64) 2634 return 0; 2635 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 2636 return fastEmitInst_r(X86::VCVTQQ2PDZ128rr, &X86::VR128XRegClass, Op0); 2637 } 2638 return 0; 2639} 2640 2641unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(unsigned Op0) { 2642 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 2643 return fastEmitInst_r(X86::VCVTQQ2PSZ256rr, &X86::VR128XRegClass, Op0); 2644 } 2645 return 0; 2646} 2647 2648unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(unsigned Op0) { 2649 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 2650 return fastEmitInst_r(X86::VCVTQQ2PDZ256rr, &X86::VR256XRegClass, Op0); 2651 } 2652 return 0; 2653} 2654 2655unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_r(MVT RetVT, unsigned Op0) { 2656switch (RetVT.SimpleTy) { 2657 case MVT::v4f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0); 2658 case MVT::v4f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0); 2659 default: return 0; 2660} 2661} 2662 2663unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f16_r(unsigned Op0) { 2664 if ((Subtarget->hasFP16())) { 2665 return fastEmitInst_r(X86::VCVTQQ2PHZrr, &X86::VR128XRegClass, Op0); 2666 } 2667 return 0; 2668} 2669 2670unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(unsigned Op0) { 2671 if ((Subtarget->hasDQI())) { 2672 return fastEmitInst_r(X86::VCVTQQ2PSZrr, &X86::VR256XRegClass, Op0); 2673 } 2674 return 0; 2675} 2676 2677unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(unsigned Op0) { 2678 if ((Subtarget->hasDQI())) { 2679 return fastEmitInst_r(X86::VCVTQQ2PDZrr, &X86::VR512RegClass, Op0); 2680 } 2681 return 0; 2682} 2683 2684unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_r(MVT RetVT, unsigned Op0) { 2685switch (RetVT.SimpleTy) { 2686 case MVT::v8f16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Op0); 2687 case MVT::v8f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0); 2688 case MVT::v8f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0); 2689 default: return 0; 2690} 2691} 2692 2693unsigned fastEmit_ISD_STRICT_SINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0) { 2694 switch (VT.SimpleTy) { 2695 case MVT::i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_r(RetVT, Op0); 2696 case MVT::i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_r(RetVT, Op0); 2697 case MVT::v8i16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i16_r(RetVT, Op0); 2698 case MVT::v16i16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i16_r(RetVT, Op0); 2699 case MVT::v32i16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v32i16_r(RetVT, Op0); 2700 case MVT::v4i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0); 2701 case MVT::v8i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_r(RetVT, Op0); 2702 case MVT::v16i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_r(RetVT, Op0); 2703 case MVT::v2i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0); 2704 case MVT::v4i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_r(RetVT, Op0); 2705 case MVT::v8i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_r(RetVT, Op0); 2706 default: return 0; 2707 } 2708} 2709 2710// FastEmit functions for ISD::STRICT_UINT_TO_FP. 2711 2712unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i16_r(MVT RetVT, unsigned Op0) { 2713 if (RetVT.SimpleTy != MVT::v8f16) 2714 return 0; 2715 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 2716 return fastEmitInst_r(X86::VCVTUW2PHZ128rr, &X86::VR128XRegClass, Op0); 2717 } 2718 return 0; 2719} 2720 2721unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i16_r(MVT RetVT, unsigned Op0) { 2722 if (RetVT.SimpleTy != MVT::v16f16) 2723 return 0; 2724 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 2725 return fastEmitInst_r(X86::VCVTUW2PHZ256rr, &X86::VR256XRegClass, Op0); 2726 } 2727 return 0; 2728} 2729 2730unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v32i16_r(MVT RetVT, unsigned Op0) { 2731 if (RetVT.SimpleTy != MVT::v32f16) 2732 return 0; 2733 if ((Subtarget->hasFP16())) { 2734 return fastEmitInst_r(X86::VCVTUW2PHZrr, &X86::VR512RegClass, Op0); 2735 } 2736 return 0; 2737} 2738 2739unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(unsigned Op0) { 2740 if ((Subtarget->hasVLX())) { 2741 return fastEmitInst_r(X86::VCVTUDQ2PSZ128rr, &X86::VR128XRegClass, Op0); 2742 } 2743 return 0; 2744} 2745 2746unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(unsigned Op0) { 2747 if ((Subtarget->hasVLX())) { 2748 return fastEmitInst_r(X86::VCVTUDQ2PDZ256rr, &X86::VR256XRegClass, Op0); 2749 } 2750 return 0; 2751} 2752 2753unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0) { 2754switch (RetVT.SimpleTy) { 2755 case MVT::v4f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0); 2756 case MVT::v4f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0); 2757 default: return 0; 2758} 2759} 2760 2761unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f16_r(unsigned Op0) { 2762 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 2763 return fastEmitInst_r(X86::VCVTUDQ2PHZ256rr, &X86::VR128XRegClass, Op0); 2764 } 2765 return 0; 2766} 2767 2768unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(unsigned Op0) { 2769 if ((Subtarget->hasVLX())) { 2770 return fastEmitInst_r(X86::VCVTUDQ2PSZ256rr, &X86::VR256XRegClass, Op0); 2771 } 2772 return 0; 2773} 2774 2775unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(unsigned Op0) { 2776 if ((Subtarget->hasAVX512())) { 2777 return fastEmitInst_r(X86::VCVTUDQ2PDZrr, &X86::VR512RegClass, Op0); 2778 } 2779 return 0; 2780} 2781 2782unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_r(MVT RetVT, unsigned Op0) { 2783switch (RetVT.SimpleTy) { 2784 case MVT::v8f16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Op0); 2785 case MVT::v8f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0); 2786 case MVT::v8f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0); 2787 default: return 0; 2788} 2789} 2790 2791unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_MVT_v16f16_r(unsigned Op0) { 2792 if ((Subtarget->hasFP16())) { 2793 return fastEmitInst_r(X86::VCVTUDQ2PHZrr, &X86::VR256XRegClass, Op0); 2794 } 2795 return 0; 2796} 2797 2798unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_MVT_v16f32_r(unsigned Op0) { 2799 if ((Subtarget->hasAVX512())) { 2800 return fastEmitInst_r(X86::VCVTUDQ2PSZrr, &X86::VR512RegClass, Op0); 2801 } 2802 return 0; 2803} 2804 2805unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_r(MVT RetVT, unsigned Op0) { 2806switch (RetVT.SimpleTy) { 2807 case MVT::v16f16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Op0); 2808 case MVT::v16f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Op0); 2809 default: return 0; 2810} 2811} 2812 2813unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, unsigned Op0) { 2814 if (RetVT.SimpleTy != MVT::v2f64) 2815 return 0; 2816 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 2817 return fastEmitInst_r(X86::VCVTUQQ2PDZ128rr, &X86::VR128XRegClass, Op0); 2818 } 2819 return 0; 2820} 2821 2822unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(unsigned Op0) { 2823 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 2824 return fastEmitInst_r(X86::VCVTUQQ2PSZ256rr, &X86::VR128XRegClass, Op0); 2825 } 2826 return 0; 2827} 2828 2829unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(unsigned Op0) { 2830 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 2831 return fastEmitInst_r(X86::VCVTUQQ2PDZ256rr, &X86::VR256XRegClass, Op0); 2832 } 2833 return 0; 2834} 2835 2836unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_r(MVT RetVT, unsigned Op0) { 2837switch (RetVT.SimpleTy) { 2838 case MVT::v4f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0); 2839 case MVT::v4f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0); 2840 default: return 0; 2841} 2842} 2843 2844unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f16_r(unsigned Op0) { 2845 if ((Subtarget->hasFP16())) { 2846 return fastEmitInst_r(X86::VCVTUQQ2PHZrr, &X86::VR128XRegClass, Op0); 2847 } 2848 return 0; 2849} 2850 2851unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(unsigned Op0) { 2852 if ((Subtarget->hasDQI())) { 2853 return fastEmitInst_r(X86::VCVTUQQ2PSZrr, &X86::VR256XRegClass, Op0); 2854 } 2855 return 0; 2856} 2857 2858unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(unsigned Op0) { 2859 if ((Subtarget->hasDQI())) { 2860 return fastEmitInst_r(X86::VCVTUQQ2PDZrr, &X86::VR512RegClass, Op0); 2861 } 2862 return 0; 2863} 2864 2865unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_r(MVT RetVT, unsigned Op0) { 2866switch (RetVT.SimpleTy) { 2867 case MVT::v8f16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Op0); 2868 case MVT::v8f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0); 2869 case MVT::v8f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0); 2870 default: return 0; 2871} 2872} 2873 2874unsigned fastEmit_ISD_STRICT_UINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0) { 2875 switch (VT.SimpleTy) { 2876 case MVT::v8i16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i16_r(RetVT, Op0); 2877 case MVT::v16i16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i16_r(RetVT, Op0); 2878 case MVT::v32i16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v32i16_r(RetVT, Op0); 2879 case MVT::v4i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0); 2880 case MVT::v8i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_r(RetVT, Op0); 2881 case MVT::v16i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_r(RetVT, Op0); 2882 case MVT::v2i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0); 2883 case MVT::v4i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_r(RetVT, Op0); 2884 case MVT::v8i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_r(RetVT, Op0); 2885 default: return 0; 2886 } 2887} 2888 2889// FastEmit functions for ISD::TRUNCATE. 2890 2891unsigned fastEmit_ISD_TRUNCATE_MVT_i16_r(MVT RetVT, unsigned Op0) { 2892 if (RetVT.SimpleTy != MVT::i8) 2893 return 0; 2894 if ((Subtarget->is64Bit())) { 2895 return fastEmitInst_extractsubreg(RetVT, Op0, X86::sub_8bit); 2896 } 2897 return 0; 2898} 2899 2900unsigned fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i8_r(unsigned Op0) { 2901 if ((Subtarget->is64Bit())) { 2902 return fastEmitInst_extractsubreg(MVT::i8, Op0, X86::sub_8bit); 2903 } 2904 return 0; 2905} 2906 2907unsigned fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i16_r(unsigned Op0) { 2908 return fastEmitInst_extractsubreg(MVT::i16, Op0, X86::sub_16bit); 2909} 2910 2911unsigned fastEmit_ISD_TRUNCATE_MVT_i32_r(MVT RetVT, unsigned Op0) { 2912switch (RetVT.SimpleTy) { 2913 case MVT::i8: return fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i8_r(Op0); 2914 case MVT::i16: return fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i16_r(Op0); 2915 default: return 0; 2916} 2917} 2918 2919unsigned fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i8_r(unsigned Op0) { 2920 return fastEmitInst_extractsubreg(MVT::i8, Op0, X86::sub_8bit); 2921} 2922 2923unsigned fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i16_r(unsigned Op0) { 2924 return fastEmitInst_extractsubreg(MVT::i16, Op0, X86::sub_16bit); 2925} 2926 2927unsigned fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i32_r(unsigned Op0) { 2928 return fastEmitInst_extractsubreg(MVT::i32, Op0, X86::sub_32bit); 2929} 2930 2931unsigned fastEmit_ISD_TRUNCATE_MVT_i64_r(MVT RetVT, unsigned Op0) { 2932switch (RetVT.SimpleTy) { 2933 case MVT::i8: return fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i8_r(Op0); 2934 case MVT::i16: return fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i16_r(Op0); 2935 case MVT::i32: return fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i32_r(Op0); 2936 default: return 0; 2937} 2938} 2939 2940unsigned fastEmit_ISD_TRUNCATE_MVT_v16i16_r(MVT RetVT, unsigned Op0) { 2941 if (RetVT.SimpleTy != MVT::v16i8) 2942 return 0; 2943 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 2944 return fastEmitInst_r(X86::VPMOVWBZ256rr, &X86::VR128XRegClass, Op0); 2945 } 2946 return 0; 2947} 2948 2949unsigned fastEmit_ISD_TRUNCATE_MVT_v32i16_r(MVT RetVT, unsigned Op0) { 2950 if (RetVT.SimpleTy != MVT::v32i8) 2951 return 0; 2952 if ((Subtarget->hasBWI())) { 2953 return fastEmitInst_r(X86::VPMOVWBZrr, &X86::VR256XRegClass, Op0); 2954 } 2955 return 0; 2956} 2957 2958unsigned fastEmit_ISD_TRUNCATE_MVT_v8i32_r(MVT RetVT, unsigned Op0) { 2959 if (RetVT.SimpleTy != MVT::v8i16) 2960 return 0; 2961 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 2962 return fastEmitInst_r(X86::VPMOVDWZ256rr, &X86::VR128XRegClass, Op0); 2963 } 2964 return 0; 2965} 2966 2967unsigned fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i8_r(unsigned Op0) { 2968 if ((Subtarget->hasAVX512())) { 2969 return fastEmitInst_r(X86::VPMOVDBZrr, &X86::VR128XRegClass, Op0); 2970 } 2971 return 0; 2972} 2973 2974unsigned fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i16_r(unsigned Op0) { 2975 if ((Subtarget->hasAVX512())) { 2976 return fastEmitInst_r(X86::VPMOVDWZrr, &X86::VR256XRegClass, Op0); 2977 } 2978 return 0; 2979} 2980 2981unsigned fastEmit_ISD_TRUNCATE_MVT_v16i32_r(MVT RetVT, unsigned Op0) { 2982switch (RetVT.SimpleTy) { 2983 case MVT::v16i8: return fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i8_r(Op0); 2984 case MVT::v16i16: return fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i16_r(Op0); 2985 default: return 0; 2986} 2987} 2988 2989unsigned fastEmit_ISD_TRUNCATE_MVT_v4i64_r(MVT RetVT, unsigned Op0) { 2990 if (RetVT.SimpleTy != MVT::v4i32) 2991 return 0; 2992 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 2993 return fastEmitInst_r(X86::VPMOVQDZ256rr, &X86::VR128XRegClass, Op0); 2994 } 2995 return 0; 2996} 2997 2998unsigned fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i16_r(unsigned Op0) { 2999 if ((Subtarget->hasAVX512())) { 3000 return fastEmitInst_r(X86::VPMOVQWZrr, &X86::VR128XRegClass, Op0); 3001 } 3002 return 0; 3003} 3004 3005unsigned fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i32_r(unsigned Op0) { 3006 if ((Subtarget->hasAVX512())) { 3007 return fastEmitInst_r(X86::VPMOVQDZrr, &X86::VR256XRegClass, Op0); 3008 } 3009 return 0; 3010} 3011 3012unsigned fastEmit_ISD_TRUNCATE_MVT_v8i64_r(MVT RetVT, unsigned Op0) { 3013switch (RetVT.SimpleTy) { 3014 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i16_r(Op0); 3015 case MVT::v8i32: return fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i32_r(Op0); 3016 default: return 0; 3017} 3018} 3019 3020unsigned fastEmit_ISD_TRUNCATE_r(MVT VT, MVT RetVT, unsigned Op0) { 3021 switch (VT.SimpleTy) { 3022 case MVT::i16: return fastEmit_ISD_TRUNCATE_MVT_i16_r(RetVT, Op0); 3023 case MVT::i32: return fastEmit_ISD_TRUNCATE_MVT_i32_r(RetVT, Op0); 3024 case MVT::i64: return fastEmit_ISD_TRUNCATE_MVT_i64_r(RetVT, Op0); 3025 case MVT::v16i16: return fastEmit_ISD_TRUNCATE_MVT_v16i16_r(RetVT, Op0); 3026 case MVT::v32i16: return fastEmit_ISD_TRUNCATE_MVT_v32i16_r(RetVT, Op0); 3027 case MVT::v8i32: return fastEmit_ISD_TRUNCATE_MVT_v8i32_r(RetVT, Op0); 3028 case MVT::v16i32: return fastEmit_ISD_TRUNCATE_MVT_v16i32_r(RetVT, Op0); 3029 case MVT::v4i64: return fastEmit_ISD_TRUNCATE_MVT_v4i64_r(RetVT, Op0); 3030 case MVT::v8i64: return fastEmit_ISD_TRUNCATE_MVT_v8i64_r(RetVT, Op0); 3031 default: return 0; 3032 } 3033} 3034 3035// FastEmit functions for ISD::UINT_TO_FP. 3036 3037unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(MVT RetVT, unsigned Op0) { 3038 if (RetVT.SimpleTy != MVT::v8f16) 3039 return 0; 3040 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 3041 return fastEmitInst_r(X86::VCVTUW2PHZ128rr, &X86::VR128XRegClass, Op0); 3042 } 3043 return 0; 3044} 3045 3046unsigned fastEmit_ISD_UINT_TO_FP_MVT_v16i16_r(MVT RetVT, unsigned Op0) { 3047 if (RetVT.SimpleTy != MVT::v16f16) 3048 return 0; 3049 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 3050 return fastEmitInst_r(X86::VCVTUW2PHZ256rr, &X86::VR256XRegClass, Op0); 3051 } 3052 return 0; 3053} 3054 3055unsigned fastEmit_ISD_UINT_TO_FP_MVT_v32i16_r(MVT RetVT, unsigned Op0) { 3056 if (RetVT.SimpleTy != MVT::v32f16) 3057 return 0; 3058 if ((Subtarget->hasFP16())) { 3059 return fastEmitInst_r(X86::VCVTUW2PHZrr, &X86::VR512RegClass, Op0); 3060 } 3061 return 0; 3062} 3063 3064unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(unsigned Op0) { 3065 if ((Subtarget->hasVLX())) { 3066 return fastEmitInst_r(X86::VCVTUDQ2PSZ128rr, &X86::VR128XRegClass, Op0); 3067 } 3068 return 0; 3069} 3070 3071unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(unsigned Op0) { 3072 if ((Subtarget->hasVLX())) { 3073 return fastEmitInst_r(X86::VCVTUDQ2PDZ256rr, &X86::VR256XRegClass, Op0); 3074 } 3075 return 0; 3076} 3077 3078unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0) { 3079switch (RetVT.SimpleTy) { 3080 case MVT::v4f32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0); 3081 case MVT::v4f64: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0); 3082 default: return 0; 3083} 3084} 3085 3086unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f16_r(unsigned Op0) { 3087 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 3088 return fastEmitInst_r(X86::VCVTUDQ2PHZ256rr, &X86::VR128XRegClass, Op0); 3089 } 3090 return 0; 3091} 3092 3093unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(unsigned Op0) { 3094 if ((Subtarget->hasVLX())) { 3095 return fastEmitInst_r(X86::VCVTUDQ2PSZ256rr, &X86::VR256XRegClass, Op0); 3096 } 3097 return 0; 3098} 3099 3100unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(unsigned Op0) { 3101 if ((Subtarget->hasAVX512())) { 3102 return fastEmitInst_r(X86::VCVTUDQ2PDZrr, &X86::VR512RegClass, Op0); 3103 } 3104 return 0; 3105} 3106 3107unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i32_r(MVT RetVT, unsigned Op0) { 3108switch (RetVT.SimpleTy) { 3109 case MVT::v8f16: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f16_r(Op0); 3110 case MVT::v8f32: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0); 3111 case MVT::v8f64: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0); 3112 default: return 0; 3113} 3114} 3115 3116unsigned fastEmit_ISD_UINT_TO_FP_MVT_v16i32_MVT_v16f16_r(unsigned Op0) { 3117 if ((Subtarget->hasFP16())) { 3118 return fastEmitInst_r(X86::VCVTUDQ2PHZrr, &X86::VR256XRegClass, Op0); 3119 } 3120 return 0; 3121} 3122 3123unsigned fastEmit_ISD_UINT_TO_FP_MVT_v16i32_MVT_v16f32_r(unsigned Op0) { 3124 if ((Subtarget->hasAVX512())) { 3125 return fastEmitInst_r(X86::VCVTUDQ2PSZrr, &X86::VR512RegClass, Op0); 3126 } 3127 return 0; 3128} 3129 3130unsigned fastEmit_ISD_UINT_TO_FP_MVT_v16i32_r(MVT RetVT, unsigned Op0) { 3131switch (RetVT.SimpleTy) { 3132 case MVT::v16f16: return fastEmit_ISD_UINT_TO_FP_MVT_v16i32_MVT_v16f16_r(Op0); 3133 case MVT::v16f32: return fastEmit_ISD_UINT_TO_FP_MVT_v16i32_MVT_v16f32_r(Op0); 3134 default: return 0; 3135} 3136} 3137 3138unsigned fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, unsigned Op0) { 3139 if (RetVT.SimpleTy != MVT::v2f64) 3140 return 0; 3141 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 3142 return fastEmitInst_r(X86::VCVTUQQ2PDZ128rr, &X86::VR128XRegClass, Op0); 3143 } 3144 return 0; 3145} 3146 3147unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(unsigned Op0) { 3148 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 3149 return fastEmitInst_r(X86::VCVTUQQ2PSZ256rr, &X86::VR128XRegClass, Op0); 3150 } 3151 return 0; 3152} 3153 3154unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(unsigned Op0) { 3155 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 3156 return fastEmitInst_r(X86::VCVTUQQ2PDZ256rr, &X86::VR256XRegClass, Op0); 3157 } 3158 return 0; 3159} 3160 3161unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i64_r(MVT RetVT, unsigned Op0) { 3162switch (RetVT.SimpleTy) { 3163 case MVT::v4f32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0); 3164 case MVT::v4f64: return fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0); 3165 default: return 0; 3166} 3167} 3168 3169unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f16_r(unsigned Op0) { 3170 if ((Subtarget->hasFP16())) { 3171 return fastEmitInst_r(X86::VCVTUQQ2PHZrr, &X86::VR128XRegClass, Op0); 3172 } 3173 return 0; 3174} 3175 3176unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(unsigned Op0) { 3177 if ((Subtarget->hasDQI())) { 3178 return fastEmitInst_r(X86::VCVTUQQ2PSZrr, &X86::VR256XRegClass, Op0); 3179 } 3180 return 0; 3181} 3182 3183unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(unsigned Op0) { 3184 if ((Subtarget->hasDQI())) { 3185 return fastEmitInst_r(X86::VCVTUQQ2PDZrr, &X86::VR512RegClass, Op0); 3186 } 3187 return 0; 3188} 3189 3190unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i64_r(MVT RetVT, unsigned Op0) { 3191switch (RetVT.SimpleTy) { 3192 case MVT::v8f16: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f16_r(Op0); 3193 case MVT::v8f32: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0); 3194 case MVT::v8f64: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0); 3195 default: return 0; 3196} 3197} 3198 3199unsigned fastEmit_ISD_UINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0) { 3200 switch (VT.SimpleTy) { 3201 case MVT::v8i16: return fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(RetVT, Op0); 3202 case MVT::v16i16: return fastEmit_ISD_UINT_TO_FP_MVT_v16i16_r(RetVT, Op0); 3203 case MVT::v32i16: return fastEmit_ISD_UINT_TO_FP_MVT_v32i16_r(RetVT, Op0); 3204 case MVT::v4i32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0); 3205 case MVT::v8i32: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_r(RetVT, Op0); 3206 case MVT::v16i32: return fastEmit_ISD_UINT_TO_FP_MVT_v16i32_r(RetVT, Op0); 3207 case MVT::v2i64: return fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0); 3208 case MVT::v4i64: return fastEmit_ISD_UINT_TO_FP_MVT_v4i64_r(RetVT, Op0); 3209 case MVT::v8i64: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_r(RetVT, Op0); 3210 default: return 0; 3211 } 3212} 3213 3214// FastEmit functions for ISD::ZERO_EXTEND. 3215 3216unsigned fastEmit_ISD_ZERO_EXTEND_MVT_i8_r(MVT RetVT, unsigned Op0) { 3217 if (RetVT.SimpleTy != MVT::i32) 3218 return 0; 3219 return fastEmitInst_r(X86::MOVZX32rr8, &X86::GR32RegClass, Op0); 3220} 3221 3222unsigned fastEmit_ISD_ZERO_EXTEND_MVT_i16_r(MVT RetVT, unsigned Op0) { 3223 if (RetVT.SimpleTy != MVT::i32) 3224 return 0; 3225 return fastEmitInst_r(X86::MOVZX32rr16, &X86::GR32RegClass, Op0); 3226} 3227 3228unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_MVT_v16i16_r(unsigned Op0) { 3229 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 3230 return fastEmitInst_r(X86::VPMOVZXBWYrr, &X86::VR256RegClass, Op0); 3231 } 3232 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 3233 return fastEmitInst_r(X86::VPMOVZXBWZ256rr, &X86::VR256XRegClass, Op0); 3234 } 3235 return 0; 3236} 3237 3238unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_MVT_v16i32_r(unsigned Op0) { 3239 if ((Subtarget->hasAVX512())) { 3240 return fastEmitInst_r(X86::VPMOVZXBDZrr, &X86::VR512RegClass, Op0); 3241 } 3242 return 0; 3243} 3244 3245unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_r(MVT RetVT, unsigned Op0) { 3246switch (RetVT.SimpleTy) { 3247 case MVT::v16i16: return fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_MVT_v16i16_r(Op0); 3248 case MVT::v16i32: return fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_MVT_v16i32_r(Op0); 3249 default: return 0; 3250} 3251} 3252 3253unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v32i8_r(MVT RetVT, unsigned Op0) { 3254 if (RetVT.SimpleTy != MVT::v32i16) 3255 return 0; 3256 if ((Subtarget->hasBWI())) { 3257 return fastEmitInst_r(X86::VPMOVZXBWZrr, &X86::VR512RegClass, Op0); 3258 } 3259 return 0; 3260} 3261 3262unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_MVT_v8i32_r(unsigned Op0) { 3263 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 3264 return fastEmitInst_r(X86::VPMOVZXWDYrr, &X86::VR256RegClass, Op0); 3265 } 3266 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 3267 return fastEmitInst_r(X86::VPMOVZXWDZ256rr, &X86::VR256XRegClass, Op0); 3268 } 3269 return 0; 3270} 3271 3272unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_MVT_v8i64_r(unsigned Op0) { 3273 if ((Subtarget->hasAVX512())) { 3274 return fastEmitInst_r(X86::VPMOVZXWQZrr, &X86::VR512RegClass, Op0); 3275 } 3276 return 0; 3277} 3278 3279unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_r(MVT RetVT, unsigned Op0) { 3280switch (RetVT.SimpleTy) { 3281 case MVT::v8i32: return fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_MVT_v8i32_r(Op0); 3282 case MVT::v8i64: return fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_MVT_v8i64_r(Op0); 3283 default: return 0; 3284} 3285} 3286 3287unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v16i16_r(MVT RetVT, unsigned Op0) { 3288 if (RetVT.SimpleTy != MVT::v16i32) 3289 return 0; 3290 if ((Subtarget->hasAVX512())) { 3291 return fastEmitInst_r(X86::VPMOVZXWDZrr, &X86::VR512RegClass, Op0); 3292 } 3293 return 0; 3294} 3295 3296unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v4i32_r(MVT RetVT, unsigned Op0) { 3297 if (RetVT.SimpleTy != MVT::v4i64) 3298 return 0; 3299 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 3300 return fastEmitInst_r(X86::VPMOVZXDQYrr, &X86::VR256RegClass, Op0); 3301 } 3302 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 3303 return fastEmitInst_r(X86::VPMOVZXDQZ256rr, &X86::VR256XRegClass, Op0); 3304 } 3305 return 0; 3306} 3307 3308unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v8i32_r(MVT RetVT, unsigned Op0) { 3309 if (RetVT.SimpleTy != MVT::v8i64) 3310 return 0; 3311 if ((Subtarget->hasAVX512())) { 3312 return fastEmitInst_r(X86::VPMOVZXDQZrr, &X86::VR512RegClass, Op0); 3313 } 3314 return 0; 3315} 3316 3317unsigned fastEmit_ISD_ZERO_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0) { 3318 switch (VT.SimpleTy) { 3319 case MVT::i8: return fastEmit_ISD_ZERO_EXTEND_MVT_i8_r(RetVT, Op0); 3320 case MVT::i16: return fastEmit_ISD_ZERO_EXTEND_MVT_i16_r(RetVT, Op0); 3321 case MVT::v16i8: return fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_r(RetVT, Op0); 3322 case MVT::v32i8: return fastEmit_ISD_ZERO_EXTEND_MVT_v32i8_r(RetVT, Op0); 3323 case MVT::v8i16: return fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_r(RetVT, Op0); 3324 case MVT::v16i16: return fastEmit_ISD_ZERO_EXTEND_MVT_v16i16_r(RetVT, Op0); 3325 case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_MVT_v4i32_r(RetVT, Op0); 3326 case MVT::v8i32: return fastEmit_ISD_ZERO_EXTEND_MVT_v8i32_r(RetVT, Op0); 3327 default: return 0; 3328 } 3329} 3330 3331// FastEmit functions for ISD::ZERO_EXTEND_VECTOR_INREG. 3332 3333unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(unsigned Op0) { 3334 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 3335 return fastEmitInst_r(X86::PMOVZXBWrr, &X86::VR128RegClass, Op0); 3336 } 3337 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 3338 return fastEmitInst_r(X86::VPMOVZXBWrr, &X86::VR128RegClass, Op0); 3339 } 3340 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 3341 return fastEmitInst_r(X86::VPMOVZXBWZ128rr, &X86::VR128XRegClass, Op0); 3342 } 3343 return 0; 3344} 3345 3346unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(unsigned Op0) { 3347 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 3348 return fastEmitInst_r(X86::PMOVZXBDrr, &X86::VR128RegClass, Op0); 3349 } 3350 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 3351 return fastEmitInst_r(X86::VPMOVZXBDrr, &X86::VR128RegClass, Op0); 3352 } 3353 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 3354 return fastEmitInst_r(X86::VPMOVZXBDZ128rr, &X86::VR128XRegClass, Op0); 3355 } 3356 return 0; 3357} 3358 3359unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i32_r(unsigned Op0) { 3360 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 3361 return fastEmitInst_r(X86::VPMOVZXBDYrr, &X86::VR256RegClass, Op0); 3362 } 3363 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 3364 return fastEmitInst_r(X86::VPMOVZXBDZ256rr, &X86::VR256XRegClass, Op0); 3365 } 3366 return 0; 3367} 3368 3369unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(unsigned Op0) { 3370 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 3371 return fastEmitInst_r(X86::PMOVZXBQrr, &X86::VR128RegClass, Op0); 3372 } 3373 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 3374 return fastEmitInst_r(X86::VPMOVZXBQrr, &X86::VR128RegClass, Op0); 3375 } 3376 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 3377 return fastEmitInst_r(X86::VPMOVZXBQZ128rr, &X86::VR128XRegClass, Op0); 3378 } 3379 return 0; 3380} 3381 3382unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i64_r(unsigned Op0) { 3383 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 3384 return fastEmitInst_r(X86::VPMOVZXBQYrr, &X86::VR256RegClass, Op0); 3385 } 3386 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 3387 return fastEmitInst_r(X86::VPMOVZXBQZ256rr, &X86::VR256XRegClass, Op0); 3388 } 3389 return 0; 3390} 3391 3392unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i64_r(unsigned Op0) { 3393 if ((Subtarget->hasAVX512())) { 3394 return fastEmitInst_r(X86::VPMOVZXBQZrr, &X86::VR512RegClass, Op0); 3395 } 3396 return 0; 3397} 3398 3399unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_r(MVT RetVT, unsigned Op0) { 3400switch (RetVT.SimpleTy) { 3401 case MVT::v8i16: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(Op0); 3402 case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(Op0); 3403 case MVT::v8i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i32_r(Op0); 3404 case MVT::v2i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(Op0); 3405 case MVT::v4i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i64_r(Op0); 3406 case MVT::v8i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i64_r(Op0); 3407 default: return 0; 3408} 3409} 3410 3411unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(unsigned Op0) { 3412 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 3413 return fastEmitInst_r(X86::PMOVZXWDrr, &X86::VR128RegClass, Op0); 3414 } 3415 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 3416 return fastEmitInst_r(X86::VPMOVZXWDrr, &X86::VR128RegClass, Op0); 3417 } 3418 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 3419 return fastEmitInst_r(X86::VPMOVZXWDZ128rr, &X86::VR128XRegClass, Op0); 3420 } 3421 return 0; 3422} 3423 3424unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(unsigned Op0) { 3425 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 3426 return fastEmitInst_r(X86::PMOVZXWQrr, &X86::VR128RegClass, Op0); 3427 } 3428 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 3429 return fastEmitInst_r(X86::VPMOVZXWQrr, &X86::VR128RegClass, Op0); 3430 } 3431 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 3432 return fastEmitInst_r(X86::VPMOVZXWQZ128rr, &X86::VR128XRegClass, Op0); 3433 } 3434 return 0; 3435} 3436 3437unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i64_r(unsigned Op0) { 3438 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 3439 return fastEmitInst_r(X86::VPMOVZXWQYrr, &X86::VR256RegClass, Op0); 3440 } 3441 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 3442 return fastEmitInst_r(X86::VPMOVZXWQZ256rr, &X86::VR256XRegClass, Op0); 3443 } 3444 return 0; 3445} 3446 3447unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_r(MVT RetVT, unsigned Op0) { 3448switch (RetVT.SimpleTy) { 3449 case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(Op0); 3450 case MVT::v2i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(Op0); 3451 case MVT::v4i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i64_r(Op0); 3452 default: return 0; 3453} 3454} 3455 3456unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v4i32_r(MVT RetVT, unsigned Op0) { 3457 if (RetVT.SimpleTy != MVT::v2i64) 3458 return 0; 3459 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 3460 return fastEmitInst_r(X86::PMOVZXDQrr, &X86::VR128RegClass, Op0); 3461 } 3462 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 3463 return fastEmitInst_r(X86::VPMOVZXDQrr, &X86::VR128RegClass, Op0); 3464 } 3465 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 3466 return fastEmitInst_r(X86::VPMOVZXDQZ128rr, &X86::VR128XRegClass, Op0); 3467 } 3468 return 0; 3469} 3470 3471unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_r(MVT VT, MVT RetVT, unsigned Op0) { 3472 switch (VT.SimpleTy) { 3473 case MVT::v16i8: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_r(RetVT, Op0); 3474 case MVT::v8i16: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_r(RetVT, Op0); 3475 case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v4i32_r(RetVT, Op0); 3476 default: return 0; 3477 } 3478} 3479 3480// FastEmit functions for X86ISD::CALL. 3481 3482unsigned fastEmit_X86ISD_CALL_MVT_i16_r(MVT RetVT, unsigned Op0) { 3483 if (RetVT.SimpleTy != MVT::isVoid) 3484 return 0; 3485 if ((!Subtarget->is64Bit())) { 3486 return fastEmitInst_r(X86::CALL16r, &X86::GR16RegClass, Op0); 3487 } 3488 return 0; 3489} 3490 3491unsigned fastEmit_X86ISD_CALL_MVT_i32_r(MVT RetVT, unsigned Op0) { 3492 if (RetVT.SimpleTy != MVT::isVoid) 3493 return 0; 3494 if ((!Subtarget->is64Bit()) && (Subtarget->useIndirectThunkCalls())) { 3495 return fastEmitInst_r(X86::INDIRECT_THUNK_CALL32, &X86::GR32RegClass, Op0); 3496 } 3497 if ((!Subtarget->is64Bit()) && (!Subtarget->useIndirectThunkCalls())) { 3498 return fastEmitInst_r(X86::CALL32r, &X86::GR32RegClass, Op0); 3499 } 3500 return 0; 3501} 3502 3503unsigned fastEmit_X86ISD_CALL_MVT_i64_r(MVT RetVT, unsigned Op0) { 3504 if (RetVT.SimpleTy != MVT::isVoid) 3505 return 0; 3506 if ((Subtarget->is64Bit()) && (Subtarget->useIndirectThunkCalls())) { 3507 return fastEmitInst_r(X86::INDIRECT_THUNK_CALL64, &X86::GR64RegClass, Op0); 3508 } 3509 if ((Subtarget->is64Bit()) && (!Subtarget->useIndirectThunkCalls())) { 3510 return fastEmitInst_r(X86::CALL64r, &X86::GR64RegClass, Op0); 3511 } 3512 return 0; 3513} 3514 3515unsigned fastEmit_X86ISD_CALL_r(MVT VT, MVT RetVT, unsigned Op0) { 3516 switch (VT.SimpleTy) { 3517 case MVT::i16: return fastEmit_X86ISD_CALL_MVT_i16_r(RetVT, Op0); 3518 case MVT::i32: return fastEmit_X86ISD_CALL_MVT_i32_r(RetVT, Op0); 3519 case MVT::i64: return fastEmit_X86ISD_CALL_MVT_i64_r(RetVT, Op0); 3520 default: return 0; 3521 } 3522} 3523 3524// FastEmit functions for X86ISD::CONFLICT. 3525 3526unsigned fastEmit_X86ISD_CONFLICT_MVT_v4i32_r(MVT RetVT, unsigned Op0) { 3527 if (RetVT.SimpleTy != MVT::v4i32) 3528 return 0; 3529 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) { 3530 return fastEmitInst_r(X86::VPCONFLICTDZ128rr, &X86::VR128XRegClass, Op0); 3531 } 3532 return 0; 3533} 3534 3535unsigned fastEmit_X86ISD_CONFLICT_MVT_v8i32_r(MVT RetVT, unsigned Op0) { 3536 if (RetVT.SimpleTy != MVT::v8i32) 3537 return 0; 3538 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) { 3539 return fastEmitInst_r(X86::VPCONFLICTDZ256rr, &X86::VR256XRegClass, Op0); 3540 } 3541 return 0; 3542} 3543 3544unsigned fastEmit_X86ISD_CONFLICT_MVT_v16i32_r(MVT RetVT, unsigned Op0) { 3545 if (RetVT.SimpleTy != MVT::v16i32) 3546 return 0; 3547 if ((Subtarget->hasCDI())) { 3548 return fastEmitInst_r(X86::VPCONFLICTDZrr, &X86::VR512RegClass, Op0); 3549 } 3550 return 0; 3551} 3552 3553unsigned fastEmit_X86ISD_CONFLICT_MVT_v2i64_r(MVT RetVT, unsigned Op0) { 3554 if (RetVT.SimpleTy != MVT::v2i64) 3555 return 0; 3556 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) { 3557 return fastEmitInst_r(X86::VPCONFLICTQZ128rr, &X86::VR128XRegClass, Op0); 3558 } 3559 return 0; 3560} 3561 3562unsigned fastEmit_X86ISD_CONFLICT_MVT_v4i64_r(MVT RetVT, unsigned Op0) { 3563 if (RetVT.SimpleTy != MVT::v4i64) 3564 return 0; 3565 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) { 3566 return fastEmitInst_r(X86::VPCONFLICTQZ256rr, &X86::VR256XRegClass, Op0); 3567 } 3568 return 0; 3569} 3570 3571unsigned fastEmit_X86ISD_CONFLICT_MVT_v8i64_r(MVT RetVT, unsigned Op0) { 3572 if (RetVT.SimpleTy != MVT::v8i64) 3573 return 0; 3574 if ((Subtarget->hasCDI())) { 3575 return fastEmitInst_r(X86::VPCONFLICTQZrr, &X86::VR512RegClass, Op0); 3576 } 3577 return 0; 3578} 3579 3580unsigned fastEmit_X86ISD_CONFLICT_r(MVT VT, MVT RetVT, unsigned Op0) { 3581 switch (VT.SimpleTy) { 3582 case MVT::v4i32: return fastEmit_X86ISD_CONFLICT_MVT_v4i32_r(RetVT, Op0); 3583 case MVT::v8i32: return fastEmit_X86ISD_CONFLICT_MVT_v8i32_r(RetVT, Op0); 3584 case MVT::v16i32: return fastEmit_X86ISD_CONFLICT_MVT_v16i32_r(RetVT, Op0); 3585 case MVT::v2i64: return fastEmit_X86ISD_CONFLICT_MVT_v2i64_r(RetVT, Op0); 3586 case MVT::v4i64: return fastEmit_X86ISD_CONFLICT_MVT_v4i64_r(RetVT, Op0); 3587 case MVT::v8i64: return fastEmit_X86ISD_CONFLICT_MVT_v8i64_r(RetVT, Op0); 3588 default: return 0; 3589 } 3590} 3591 3592// FastEmit functions for X86ISD::CVTNEPS2BF16. 3593 3594unsigned fastEmit_X86ISD_CVTNEPS2BF16_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 3595 if (RetVT.SimpleTy != MVT::v8bf16) 3596 return 0; 3597 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) { 3598 return fastEmitInst_r(X86::VCVTNEPS2BF16Z128rr, &X86::VR128XRegClass, Op0); 3599 } 3600 return 0; 3601} 3602 3603unsigned fastEmit_X86ISD_CVTNEPS2BF16_MVT_v8f32_r(MVT RetVT, unsigned Op0) { 3604 if (RetVT.SimpleTy != MVT::v8bf16) 3605 return 0; 3606 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) { 3607 return fastEmitInst_r(X86::VCVTNEPS2BF16Z256rr, &X86::VR128XRegClass, Op0); 3608 } 3609 return 0; 3610} 3611 3612unsigned fastEmit_X86ISD_CVTNEPS2BF16_MVT_v16f32_r(MVT RetVT, unsigned Op0) { 3613 if (RetVT.SimpleTy != MVT::v16bf16) 3614 return 0; 3615 if ((Subtarget->hasBF16())) { 3616 return fastEmitInst_r(X86::VCVTNEPS2BF16Zrr, &X86::VR256XRegClass, Op0); 3617 } 3618 return 0; 3619} 3620 3621unsigned fastEmit_X86ISD_CVTNEPS2BF16_r(MVT VT, MVT RetVT, unsigned Op0) { 3622 switch (VT.SimpleTy) { 3623 case MVT::v4f32: return fastEmit_X86ISD_CVTNEPS2BF16_MVT_v4f32_r(RetVT, Op0); 3624 case MVT::v8f32: return fastEmit_X86ISD_CVTNEPS2BF16_MVT_v8f32_r(RetVT, Op0); 3625 case MVT::v16f32: return fastEmit_X86ISD_CVTNEPS2BF16_MVT_v16f32_r(RetVT, Op0); 3626 default: return 0; 3627 } 3628} 3629 3630// FastEmit functions for X86ISD::CVTP2SI. 3631 3632unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v8i16_r(unsigned Op0) { 3633 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 3634 return fastEmitInst_r(X86::VCVTPH2WZ128rr, &X86::VR128XRegClass, Op0); 3635 } 3636 return 0; 3637} 3638 3639unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v4i32_r(unsigned Op0) { 3640 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 3641 return fastEmitInst_r(X86::VCVTPH2DQZ128rr, &X86::VR128XRegClass, Op0); 3642 } 3643 return 0; 3644} 3645 3646unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v8i32_r(unsigned Op0) { 3647 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 3648 return fastEmitInst_r(X86::VCVTPH2DQZ256rr, &X86::VR256XRegClass, Op0); 3649 } 3650 return 0; 3651} 3652 3653unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v2i64_r(unsigned Op0) { 3654 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 3655 return fastEmitInst_r(X86::VCVTPH2QQZ128rr, &X86::VR128XRegClass, Op0); 3656 } 3657 return 0; 3658} 3659 3660unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v4i64_r(unsigned Op0) { 3661 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 3662 return fastEmitInst_r(X86::VCVTPH2QQZ256rr, &X86::VR256XRegClass, Op0); 3663 } 3664 return 0; 3665} 3666 3667unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v8i64_r(unsigned Op0) { 3668 if ((Subtarget->hasFP16())) { 3669 return fastEmitInst_r(X86::VCVTPH2QQZrr, &X86::VR512RegClass, Op0); 3670 } 3671 return 0; 3672} 3673 3674unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 3675switch (RetVT.SimpleTy) { 3676 case MVT::v8i16: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v8i16_r(Op0); 3677 case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v4i32_r(Op0); 3678 case MVT::v8i32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v8i32_r(Op0); 3679 case MVT::v2i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v2i64_r(Op0); 3680 case MVT::v4i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v4i64_r(Op0); 3681 case MVT::v8i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_MVT_v8i64_r(Op0); 3682 default: return 0; 3683} 3684} 3685 3686unsigned fastEmit_X86ISD_CVTP2SI_MVT_v16f16_MVT_v16i16_r(unsigned Op0) { 3687 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 3688 return fastEmitInst_r(X86::VCVTPH2WZ256rr, &X86::VR256XRegClass, Op0); 3689 } 3690 return 0; 3691} 3692 3693unsigned fastEmit_X86ISD_CVTP2SI_MVT_v16f16_MVT_v16i32_r(unsigned Op0) { 3694 if ((Subtarget->hasFP16())) { 3695 return fastEmitInst_r(X86::VCVTPH2DQZrr, &X86::VR512RegClass, Op0); 3696 } 3697 return 0; 3698} 3699 3700unsigned fastEmit_X86ISD_CVTP2SI_MVT_v16f16_r(MVT RetVT, unsigned Op0) { 3701switch (RetVT.SimpleTy) { 3702 case MVT::v16i16: return fastEmit_X86ISD_CVTP2SI_MVT_v16f16_MVT_v16i16_r(Op0); 3703 case MVT::v16i32: return fastEmit_X86ISD_CVTP2SI_MVT_v16f16_MVT_v16i32_r(Op0); 3704 default: return 0; 3705} 3706} 3707 3708unsigned fastEmit_X86ISD_CVTP2SI_MVT_v32f16_r(MVT RetVT, unsigned Op0) { 3709 if (RetVT.SimpleTy != MVT::v32i16) 3710 return 0; 3711 if ((Subtarget->hasFP16())) { 3712 return fastEmitInst_r(X86::VCVTPH2WZrr, &X86::VR512RegClass, Op0); 3713 } 3714 return 0; 3715} 3716 3717unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i32_r(unsigned Op0) { 3718 if ((Subtarget->hasVLX())) { 3719 return fastEmitInst_r(X86::VCVTPS2DQZ128rr, &X86::VR128XRegClass, Op0); 3720 } 3721 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 3722 return fastEmitInst_r(X86::CVTPS2DQrr, &X86::VR128RegClass, Op0); 3723 } 3724 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 3725 return fastEmitInst_r(X86::VCVTPS2DQrr, &X86::VR128RegClass, Op0); 3726 } 3727 return 0; 3728} 3729 3730unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v2i64_r(unsigned Op0) { 3731 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 3732 return fastEmitInst_r(X86::VCVTPS2QQZ128rr, &X86::VR128XRegClass, Op0); 3733 } 3734 return 0; 3735} 3736 3737unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i64_r(unsigned Op0) { 3738 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 3739 return fastEmitInst_r(X86::VCVTPS2QQZ256rr, &X86::VR256XRegClass, Op0); 3740 } 3741 return 0; 3742} 3743 3744unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 3745switch (RetVT.SimpleTy) { 3746 case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i32_r(Op0); 3747 case MVT::v2i64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v2i64_r(Op0); 3748 case MVT::v4i64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i64_r(Op0); 3749 default: return 0; 3750} 3751} 3752 3753unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i32_r(unsigned Op0) { 3754 if ((Subtarget->hasVLX())) { 3755 return fastEmitInst_r(X86::VCVTPS2DQZ256rr, &X86::VR256XRegClass, Op0); 3756 } 3757 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 3758 return fastEmitInst_r(X86::VCVTPS2DQYrr, &X86::VR256RegClass, Op0); 3759 } 3760 return 0; 3761} 3762 3763unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i64_r(unsigned Op0) { 3764 if ((Subtarget->hasDQI())) { 3765 return fastEmitInst_r(X86::VCVTPS2QQZrr, &X86::VR512RegClass, Op0); 3766 } 3767 return 0; 3768} 3769 3770unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f32_r(MVT RetVT, unsigned Op0) { 3771switch (RetVT.SimpleTy) { 3772 case MVT::v8i32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i32_r(Op0); 3773 case MVT::v8i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i64_r(Op0); 3774 default: return 0; 3775} 3776} 3777 3778unsigned fastEmit_X86ISD_CVTP2SI_MVT_v16f32_r(MVT RetVT, unsigned Op0) { 3779 if (RetVT.SimpleTy != MVT::v16i32) 3780 return 0; 3781 if ((Subtarget->hasAVX512())) { 3782 return fastEmitInst_r(X86::VCVTPS2DQZrr, &X86::VR512RegClass, Op0); 3783 } 3784 return 0; 3785} 3786 3787unsigned fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v4i32_r(unsigned Op0) { 3788 if ((Subtarget->hasVLX())) { 3789 return fastEmitInst_r(X86::VCVTPD2DQZ128rr, &X86::VR128XRegClass, Op0); 3790 } 3791 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 3792 return fastEmitInst_r(X86::CVTPD2DQrr, &X86::VR128RegClass, Op0); 3793 } 3794 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 3795 return fastEmitInst_r(X86::VCVTPD2DQrr, &X86::VR128RegClass, Op0); 3796 } 3797 return 0; 3798} 3799 3800unsigned fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v2i64_r(unsigned Op0) { 3801 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 3802 return fastEmitInst_r(X86::VCVTPD2QQZ128rr, &X86::VR128XRegClass, Op0); 3803 } 3804 return 0; 3805} 3806 3807unsigned fastEmit_X86ISD_CVTP2SI_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 3808switch (RetVT.SimpleTy) { 3809 case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v4i32_r(Op0); 3810 case MVT::v2i64: return fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v2i64_r(Op0); 3811 default: return 0; 3812} 3813} 3814 3815unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i32_r(unsigned Op0) { 3816 if ((Subtarget->hasVLX())) { 3817 return fastEmitInst_r(X86::VCVTPD2DQZ256rr, &X86::VR128XRegClass, Op0); 3818 } 3819 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 3820 return fastEmitInst_r(X86::VCVTPD2DQYrr, &X86::VR128RegClass, Op0); 3821 } 3822 return 0; 3823} 3824 3825unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i64_r(unsigned Op0) { 3826 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 3827 return fastEmitInst_r(X86::VCVTPD2QQZ256rr, &X86::VR256XRegClass, Op0); 3828 } 3829 return 0; 3830} 3831 3832unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f64_r(MVT RetVT, unsigned Op0) { 3833switch (RetVT.SimpleTy) { 3834 case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i32_r(Op0); 3835 case MVT::v4i64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i64_r(Op0); 3836 default: return 0; 3837} 3838} 3839 3840unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i32_r(unsigned Op0) { 3841 if ((Subtarget->hasAVX512())) { 3842 return fastEmitInst_r(X86::VCVTPD2DQZrr, &X86::VR256XRegClass, Op0); 3843 } 3844 return 0; 3845} 3846 3847unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i64_r(unsigned Op0) { 3848 if ((Subtarget->hasDQI())) { 3849 return fastEmitInst_r(X86::VCVTPD2QQZrr, &X86::VR512RegClass, Op0); 3850 } 3851 return 0; 3852} 3853 3854unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f64_r(MVT RetVT, unsigned Op0) { 3855switch (RetVT.SimpleTy) { 3856 case MVT::v8i32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i32_r(Op0); 3857 case MVT::v8i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i64_r(Op0); 3858 default: return 0; 3859} 3860} 3861 3862unsigned fastEmit_X86ISD_CVTP2SI_r(MVT VT, MVT RetVT, unsigned Op0) { 3863 switch (VT.SimpleTy) { 3864 case MVT::v8f16: return fastEmit_X86ISD_CVTP2SI_MVT_v8f16_r(RetVT, Op0); 3865 case MVT::v16f16: return fastEmit_X86ISD_CVTP2SI_MVT_v16f16_r(RetVT, Op0); 3866 case MVT::v32f16: return fastEmit_X86ISD_CVTP2SI_MVT_v32f16_r(RetVT, Op0); 3867 case MVT::v4f32: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_r(RetVT, Op0); 3868 case MVT::v8f32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f32_r(RetVT, Op0); 3869 case MVT::v16f32: return fastEmit_X86ISD_CVTP2SI_MVT_v16f32_r(RetVT, Op0); 3870 case MVT::v2f64: return fastEmit_X86ISD_CVTP2SI_MVT_v2f64_r(RetVT, Op0); 3871 case MVT::v4f64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f64_r(RetVT, Op0); 3872 case MVT::v8f64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f64_r(RetVT, Op0); 3873 default: return 0; 3874 } 3875} 3876 3877// FastEmit functions for X86ISD::CVTP2UI. 3878 3879unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v8i16_r(unsigned Op0) { 3880 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 3881 return fastEmitInst_r(X86::VCVTPH2UWZ128rr, &X86::VR128XRegClass, Op0); 3882 } 3883 return 0; 3884} 3885 3886unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v4i32_r(unsigned Op0) { 3887 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 3888 return fastEmitInst_r(X86::VCVTPH2UDQZ128rr, &X86::VR128XRegClass, Op0); 3889 } 3890 return 0; 3891} 3892 3893unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v8i32_r(unsigned Op0) { 3894 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 3895 return fastEmitInst_r(X86::VCVTPH2UDQZ256rr, &X86::VR256XRegClass, Op0); 3896 } 3897 return 0; 3898} 3899 3900unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v2i64_r(unsigned Op0) { 3901 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 3902 return fastEmitInst_r(X86::VCVTPH2UQQZ128rr, &X86::VR128XRegClass, Op0); 3903 } 3904 return 0; 3905} 3906 3907unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v4i64_r(unsigned Op0) { 3908 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 3909 return fastEmitInst_r(X86::VCVTPH2UQQZ256rr, &X86::VR256XRegClass, Op0); 3910 } 3911 return 0; 3912} 3913 3914unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v8i64_r(unsigned Op0) { 3915 if ((Subtarget->hasFP16())) { 3916 return fastEmitInst_r(X86::VCVTPH2UQQZrr, &X86::VR512RegClass, Op0); 3917 } 3918 return 0; 3919} 3920 3921unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 3922switch (RetVT.SimpleTy) { 3923 case MVT::v8i16: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v8i16_r(Op0); 3924 case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v4i32_r(Op0); 3925 case MVT::v8i32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v8i32_r(Op0); 3926 case MVT::v2i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v2i64_r(Op0); 3927 case MVT::v4i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v4i64_r(Op0); 3928 case MVT::v8i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_MVT_v8i64_r(Op0); 3929 default: return 0; 3930} 3931} 3932 3933unsigned fastEmit_X86ISD_CVTP2UI_MVT_v16f16_MVT_v16i16_r(unsigned Op0) { 3934 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 3935 return fastEmitInst_r(X86::VCVTPH2UWZ256rr, &X86::VR256XRegClass, Op0); 3936 } 3937 return 0; 3938} 3939 3940unsigned fastEmit_X86ISD_CVTP2UI_MVT_v16f16_MVT_v16i32_r(unsigned Op0) { 3941 if ((Subtarget->hasFP16())) { 3942 return fastEmitInst_r(X86::VCVTPH2UDQZrr, &X86::VR512RegClass, Op0); 3943 } 3944 return 0; 3945} 3946 3947unsigned fastEmit_X86ISD_CVTP2UI_MVT_v16f16_r(MVT RetVT, unsigned Op0) { 3948switch (RetVT.SimpleTy) { 3949 case MVT::v16i16: return fastEmit_X86ISD_CVTP2UI_MVT_v16f16_MVT_v16i16_r(Op0); 3950 case MVT::v16i32: return fastEmit_X86ISD_CVTP2UI_MVT_v16f16_MVT_v16i32_r(Op0); 3951 default: return 0; 3952} 3953} 3954 3955unsigned fastEmit_X86ISD_CVTP2UI_MVT_v32f16_r(MVT RetVT, unsigned Op0) { 3956 if (RetVT.SimpleTy != MVT::v32i16) 3957 return 0; 3958 if ((Subtarget->hasFP16())) { 3959 return fastEmitInst_r(X86::VCVTPH2UWZrr, &X86::VR512RegClass, Op0); 3960 } 3961 return 0; 3962} 3963 3964unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i32_r(unsigned Op0) { 3965 if ((Subtarget->hasVLX())) { 3966 return fastEmitInst_r(X86::VCVTPS2UDQZ128rr, &X86::VR128XRegClass, Op0); 3967 } 3968 return 0; 3969} 3970 3971unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v2i64_r(unsigned Op0) { 3972 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 3973 return fastEmitInst_r(X86::VCVTPS2UQQZ128rr, &X86::VR128XRegClass, Op0); 3974 } 3975 return 0; 3976} 3977 3978unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i64_r(unsigned Op0) { 3979 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 3980 return fastEmitInst_r(X86::VCVTPS2UQQZ256rr, &X86::VR256XRegClass, Op0); 3981 } 3982 return 0; 3983} 3984 3985unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 3986switch (RetVT.SimpleTy) { 3987 case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i32_r(Op0); 3988 case MVT::v2i64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v2i64_r(Op0); 3989 case MVT::v4i64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i64_r(Op0); 3990 default: return 0; 3991} 3992} 3993 3994unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i32_r(unsigned Op0) { 3995 if ((Subtarget->hasVLX())) { 3996 return fastEmitInst_r(X86::VCVTPS2UDQZ256rr, &X86::VR256XRegClass, Op0); 3997 } 3998 return 0; 3999} 4000 4001unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i64_r(unsigned Op0) { 4002 if ((Subtarget->hasDQI())) { 4003 return fastEmitInst_r(X86::VCVTPS2UQQZrr, &X86::VR512RegClass, Op0); 4004 } 4005 return 0; 4006} 4007 4008unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f32_r(MVT RetVT, unsigned Op0) { 4009switch (RetVT.SimpleTy) { 4010 case MVT::v8i32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i32_r(Op0); 4011 case MVT::v8i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i64_r(Op0); 4012 default: return 0; 4013} 4014} 4015 4016unsigned fastEmit_X86ISD_CVTP2UI_MVT_v16f32_r(MVT RetVT, unsigned Op0) { 4017 if (RetVT.SimpleTy != MVT::v16i32) 4018 return 0; 4019 if ((Subtarget->hasAVX512())) { 4020 return fastEmitInst_r(X86::VCVTPS2UDQZrr, &X86::VR512RegClass, Op0); 4021 } 4022 return 0; 4023} 4024 4025unsigned fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v4i32_r(unsigned Op0) { 4026 if ((Subtarget->hasVLX())) { 4027 return fastEmitInst_r(X86::VCVTPD2UDQZ128rr, &X86::VR128XRegClass, Op0); 4028 } 4029 return 0; 4030} 4031 4032unsigned fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v2i64_r(unsigned Op0) { 4033 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 4034 return fastEmitInst_r(X86::VCVTPD2UQQZ128rr, &X86::VR128XRegClass, Op0); 4035 } 4036 return 0; 4037} 4038 4039unsigned fastEmit_X86ISD_CVTP2UI_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 4040switch (RetVT.SimpleTy) { 4041 case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v4i32_r(Op0); 4042 case MVT::v2i64: return fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v2i64_r(Op0); 4043 default: return 0; 4044} 4045} 4046 4047unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i32_r(unsigned Op0) { 4048 if ((Subtarget->hasVLX())) { 4049 return fastEmitInst_r(X86::VCVTPD2UDQZ256rr, &X86::VR128XRegClass, Op0); 4050 } 4051 return 0; 4052} 4053 4054unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i64_r(unsigned Op0) { 4055 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 4056 return fastEmitInst_r(X86::VCVTPD2UQQZ256rr, &X86::VR256XRegClass, Op0); 4057 } 4058 return 0; 4059} 4060 4061unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f64_r(MVT RetVT, unsigned Op0) { 4062switch (RetVT.SimpleTy) { 4063 case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i32_r(Op0); 4064 case MVT::v4i64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i64_r(Op0); 4065 default: return 0; 4066} 4067} 4068 4069unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i32_r(unsigned Op0) { 4070 if ((Subtarget->hasAVX512())) { 4071 return fastEmitInst_r(X86::VCVTPD2UDQZrr, &X86::VR256XRegClass, Op0); 4072 } 4073 return 0; 4074} 4075 4076unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i64_r(unsigned Op0) { 4077 if ((Subtarget->hasDQI())) { 4078 return fastEmitInst_r(X86::VCVTPD2UQQZrr, &X86::VR512RegClass, Op0); 4079 } 4080 return 0; 4081} 4082 4083unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f64_r(MVT RetVT, unsigned Op0) { 4084switch (RetVT.SimpleTy) { 4085 case MVT::v8i32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i32_r(Op0); 4086 case MVT::v8i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i64_r(Op0); 4087 default: return 0; 4088} 4089} 4090 4091unsigned fastEmit_X86ISD_CVTP2UI_r(MVT VT, MVT RetVT, unsigned Op0) { 4092 switch (VT.SimpleTy) { 4093 case MVT::v8f16: return fastEmit_X86ISD_CVTP2UI_MVT_v8f16_r(RetVT, Op0); 4094 case MVT::v16f16: return fastEmit_X86ISD_CVTP2UI_MVT_v16f16_r(RetVT, Op0); 4095 case MVT::v32f16: return fastEmit_X86ISD_CVTP2UI_MVT_v32f16_r(RetVT, Op0); 4096 case MVT::v4f32: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_r(RetVT, Op0); 4097 case MVT::v8f32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f32_r(RetVT, Op0); 4098 case MVT::v16f32: return fastEmit_X86ISD_CVTP2UI_MVT_v16f32_r(RetVT, Op0); 4099 case MVT::v2f64: return fastEmit_X86ISD_CVTP2UI_MVT_v2f64_r(RetVT, Op0); 4100 case MVT::v4f64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f64_r(RetVT, Op0); 4101 case MVT::v8f64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f64_r(RetVT, Op0); 4102 default: return 0; 4103 } 4104} 4105 4106// FastEmit functions for X86ISD::CVTPH2PS. 4107 4108unsigned fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v4f32_r(unsigned Op0) { 4109 if ((Subtarget->hasVLX())) { 4110 return fastEmitInst_r(X86::VCVTPH2PSZ128rr, &X86::VR128XRegClass, Op0); 4111 } 4112 if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) { 4113 return fastEmitInst_r(X86::VCVTPH2PSrr, &X86::VR128RegClass, Op0); 4114 } 4115 return 0; 4116} 4117 4118unsigned fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v8f32_r(unsigned Op0) { 4119 if ((Subtarget->hasVLX())) { 4120 return fastEmitInst_r(X86::VCVTPH2PSZ256rr, &X86::VR256XRegClass, Op0); 4121 } 4122 if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) { 4123 return fastEmitInst_r(X86::VCVTPH2PSYrr, &X86::VR256RegClass, Op0); 4124 } 4125 return 0; 4126} 4127 4128unsigned fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_r(MVT RetVT, unsigned Op0) { 4129switch (RetVT.SimpleTy) { 4130 case MVT::v4f32: return fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v4f32_r(Op0); 4131 case MVT::v8f32: return fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v8f32_r(Op0); 4132 default: return 0; 4133} 4134} 4135 4136unsigned fastEmit_X86ISD_CVTPH2PS_MVT_v16i16_r(MVT RetVT, unsigned Op0) { 4137 if (RetVT.SimpleTy != MVT::v16f32) 4138 return 0; 4139 if ((Subtarget->hasAVX512())) { 4140 return fastEmitInst_r(X86::VCVTPH2PSZrr, &X86::VR512RegClass, Op0); 4141 } 4142 return 0; 4143} 4144 4145unsigned fastEmit_X86ISD_CVTPH2PS_r(MVT VT, MVT RetVT, unsigned Op0) { 4146 switch (VT.SimpleTy) { 4147 case MVT::v8i16: return fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_r(RetVT, Op0); 4148 case MVT::v16i16: return fastEmit_X86ISD_CVTPH2PS_MVT_v16i16_r(RetVT, Op0); 4149 default: return 0; 4150 } 4151} 4152 4153// FastEmit functions for X86ISD::CVTPH2PS_SAE. 4154 4155unsigned fastEmit_X86ISD_CVTPH2PS_SAE_MVT_v16i16_r(MVT RetVT, unsigned Op0) { 4156 if (RetVT.SimpleTy != MVT::v16f32) 4157 return 0; 4158 if ((Subtarget->hasAVX512())) { 4159 return fastEmitInst_r(X86::VCVTPH2PSZrrb, &X86::VR512RegClass, Op0); 4160 } 4161 return 0; 4162} 4163 4164unsigned fastEmit_X86ISD_CVTPH2PS_SAE_r(MVT VT, MVT RetVT, unsigned Op0) { 4165 switch (VT.SimpleTy) { 4166 case MVT::v16i16: return fastEmit_X86ISD_CVTPH2PS_SAE_MVT_v16i16_r(RetVT, Op0); 4167 default: return 0; 4168 } 4169} 4170 4171// FastEmit functions for X86ISD::CVTS2SI. 4172 4173unsigned fastEmit_X86ISD_CVTS2SI_MVT_v8f16_MVT_i32_r(unsigned Op0) { 4174 if ((Subtarget->hasFP16())) { 4175 return fastEmitInst_r(X86::VCVTSH2SIZrr_Int, &X86::GR32RegClass, Op0); 4176 } 4177 return 0; 4178} 4179 4180unsigned fastEmit_X86ISD_CVTS2SI_MVT_v8f16_MVT_i64_r(unsigned Op0) { 4181 if ((Subtarget->hasFP16())) { 4182 return fastEmitInst_r(X86::VCVTSH2SI64Zrr_Int, &X86::GR64RegClass, Op0); 4183 } 4184 return 0; 4185} 4186 4187unsigned fastEmit_X86ISD_CVTS2SI_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 4188switch (RetVT.SimpleTy) { 4189 case MVT::i32: return fastEmit_X86ISD_CVTS2SI_MVT_v8f16_MVT_i32_r(Op0); 4190 case MVT::i64: return fastEmit_X86ISD_CVTS2SI_MVT_v8f16_MVT_i64_r(Op0); 4191 default: return 0; 4192} 4193} 4194 4195unsigned fastEmit_X86ISD_CVTS2SI_MVT_v4f32_MVT_i32_r(unsigned Op0) { 4196 if ((Subtarget->hasAVX512())) { 4197 return fastEmitInst_r(X86::VCVTSS2SIZrr_Int, &X86::GR32RegClass, Op0); 4198 } 4199 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 4200 return fastEmitInst_r(X86::CVTSS2SIrr_Int, &X86::GR32RegClass, Op0); 4201 } 4202 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 4203 return fastEmitInst_r(X86::VCVTSS2SIrr_Int, &X86::GR32RegClass, Op0); 4204 } 4205 return 0; 4206} 4207 4208unsigned fastEmit_X86ISD_CVTS2SI_MVT_v4f32_MVT_i64_r(unsigned Op0) { 4209 if ((Subtarget->hasAVX512())) { 4210 return fastEmitInst_r(X86::VCVTSS2SI64Zrr_Int, &X86::GR64RegClass, Op0); 4211 } 4212 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 4213 return fastEmitInst_r(X86::CVTSS2SI64rr_Int, &X86::GR64RegClass, Op0); 4214 } 4215 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 4216 return fastEmitInst_r(X86::VCVTSS2SI64rr_Int, &X86::GR64RegClass, Op0); 4217 } 4218 return 0; 4219} 4220 4221unsigned fastEmit_X86ISD_CVTS2SI_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 4222switch (RetVT.SimpleTy) { 4223 case MVT::i32: return fastEmit_X86ISD_CVTS2SI_MVT_v4f32_MVT_i32_r(Op0); 4224 case MVT::i64: return fastEmit_X86ISD_CVTS2SI_MVT_v4f32_MVT_i64_r(Op0); 4225 default: return 0; 4226} 4227} 4228 4229unsigned fastEmit_X86ISD_CVTS2SI_MVT_v2f64_MVT_i32_r(unsigned Op0) { 4230 if ((Subtarget->hasAVX512())) { 4231 return fastEmitInst_r(X86::VCVTSD2SIZrr_Int, &X86::GR32RegClass, Op0); 4232 } 4233 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 4234 return fastEmitInst_r(X86::CVTSD2SIrr_Int, &X86::GR32RegClass, Op0); 4235 } 4236 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 4237 return fastEmitInst_r(X86::VCVTSD2SIrr_Int, &X86::GR32RegClass, Op0); 4238 } 4239 return 0; 4240} 4241 4242unsigned fastEmit_X86ISD_CVTS2SI_MVT_v2f64_MVT_i64_r(unsigned Op0) { 4243 if ((Subtarget->hasAVX512())) { 4244 return fastEmitInst_r(X86::VCVTSD2SI64Zrr_Int, &X86::GR64RegClass, Op0); 4245 } 4246 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 4247 return fastEmitInst_r(X86::CVTSD2SI64rr_Int, &X86::GR64RegClass, Op0); 4248 } 4249 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 4250 return fastEmitInst_r(X86::VCVTSD2SI64rr_Int, &X86::GR64RegClass, Op0); 4251 } 4252 return 0; 4253} 4254 4255unsigned fastEmit_X86ISD_CVTS2SI_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 4256switch (RetVT.SimpleTy) { 4257 case MVT::i32: return fastEmit_X86ISD_CVTS2SI_MVT_v2f64_MVT_i32_r(Op0); 4258 case MVT::i64: return fastEmit_X86ISD_CVTS2SI_MVT_v2f64_MVT_i64_r(Op0); 4259 default: return 0; 4260} 4261} 4262 4263unsigned fastEmit_X86ISD_CVTS2SI_r(MVT VT, MVT RetVT, unsigned Op0) { 4264 switch (VT.SimpleTy) { 4265 case MVT::v8f16: return fastEmit_X86ISD_CVTS2SI_MVT_v8f16_r(RetVT, Op0); 4266 case MVT::v4f32: return fastEmit_X86ISD_CVTS2SI_MVT_v4f32_r(RetVT, Op0); 4267 case MVT::v2f64: return fastEmit_X86ISD_CVTS2SI_MVT_v2f64_r(RetVT, Op0); 4268 default: return 0; 4269 } 4270} 4271 4272// FastEmit functions for X86ISD::CVTS2UI. 4273 4274unsigned fastEmit_X86ISD_CVTS2UI_MVT_v8f16_MVT_i32_r(unsigned Op0) { 4275 if ((Subtarget->hasFP16())) { 4276 return fastEmitInst_r(X86::VCVTSH2USIZrr_Int, &X86::GR32RegClass, Op0); 4277 } 4278 return 0; 4279} 4280 4281unsigned fastEmit_X86ISD_CVTS2UI_MVT_v8f16_MVT_i64_r(unsigned Op0) { 4282 if ((Subtarget->hasFP16())) { 4283 return fastEmitInst_r(X86::VCVTSH2USI64Zrr_Int, &X86::GR64RegClass, Op0); 4284 } 4285 return 0; 4286} 4287 4288unsigned fastEmit_X86ISD_CVTS2UI_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 4289switch (RetVT.SimpleTy) { 4290 case MVT::i32: return fastEmit_X86ISD_CVTS2UI_MVT_v8f16_MVT_i32_r(Op0); 4291 case MVT::i64: return fastEmit_X86ISD_CVTS2UI_MVT_v8f16_MVT_i64_r(Op0); 4292 default: return 0; 4293} 4294} 4295 4296unsigned fastEmit_X86ISD_CVTS2UI_MVT_v4f32_MVT_i32_r(unsigned Op0) { 4297 if ((Subtarget->hasAVX512())) { 4298 return fastEmitInst_r(X86::VCVTSS2USIZrr_Int, &X86::GR32RegClass, Op0); 4299 } 4300 return 0; 4301} 4302 4303unsigned fastEmit_X86ISD_CVTS2UI_MVT_v4f32_MVT_i64_r(unsigned Op0) { 4304 if ((Subtarget->hasAVX512())) { 4305 return fastEmitInst_r(X86::VCVTSS2USI64Zrr_Int, &X86::GR64RegClass, Op0); 4306 } 4307 return 0; 4308} 4309 4310unsigned fastEmit_X86ISD_CVTS2UI_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 4311switch (RetVT.SimpleTy) { 4312 case MVT::i32: return fastEmit_X86ISD_CVTS2UI_MVT_v4f32_MVT_i32_r(Op0); 4313 case MVT::i64: return fastEmit_X86ISD_CVTS2UI_MVT_v4f32_MVT_i64_r(Op0); 4314 default: return 0; 4315} 4316} 4317 4318unsigned fastEmit_X86ISD_CVTS2UI_MVT_v2f64_MVT_i32_r(unsigned Op0) { 4319 if ((Subtarget->hasAVX512())) { 4320 return fastEmitInst_r(X86::VCVTSD2USIZrr_Int, &X86::GR32RegClass, Op0); 4321 } 4322 return 0; 4323} 4324 4325unsigned fastEmit_X86ISD_CVTS2UI_MVT_v2f64_MVT_i64_r(unsigned Op0) { 4326 if ((Subtarget->hasAVX512())) { 4327 return fastEmitInst_r(X86::VCVTSD2USI64Zrr_Int, &X86::GR64RegClass, Op0); 4328 } 4329 return 0; 4330} 4331 4332unsigned fastEmit_X86ISD_CVTS2UI_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 4333switch (RetVT.SimpleTy) { 4334 case MVT::i32: return fastEmit_X86ISD_CVTS2UI_MVT_v2f64_MVT_i32_r(Op0); 4335 case MVT::i64: return fastEmit_X86ISD_CVTS2UI_MVT_v2f64_MVT_i64_r(Op0); 4336 default: return 0; 4337} 4338} 4339 4340unsigned fastEmit_X86ISD_CVTS2UI_r(MVT VT, MVT RetVT, unsigned Op0) { 4341 switch (VT.SimpleTy) { 4342 case MVT::v8f16: return fastEmit_X86ISD_CVTS2UI_MVT_v8f16_r(RetVT, Op0); 4343 case MVT::v4f32: return fastEmit_X86ISD_CVTS2UI_MVT_v4f32_r(RetVT, Op0); 4344 case MVT::v2f64: return fastEmit_X86ISD_CVTS2UI_MVT_v2f64_r(RetVT, Op0); 4345 default: return 0; 4346 } 4347} 4348 4349// FastEmit functions for X86ISD::CVTSI2P. 4350 4351unsigned fastEmit_X86ISD_CVTSI2P_MVT_v4i32_MVT_v8f16_r(unsigned Op0) { 4352 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 4353 return fastEmitInst_r(X86::VCVTDQ2PHZ128rr, &X86::VR128XRegClass, Op0); 4354 } 4355 return 0; 4356} 4357 4358unsigned fastEmit_X86ISD_CVTSI2P_MVT_v4i32_MVT_v2f64_r(unsigned Op0) { 4359 if ((Subtarget->hasVLX())) { 4360 return fastEmitInst_r(X86::VCVTDQ2PDZ128rr, &X86::VR128XRegClass, Op0); 4361 } 4362 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 4363 return fastEmitInst_r(X86::CVTDQ2PDrr, &X86::VR128RegClass, Op0); 4364 } 4365 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 4366 return fastEmitInst_r(X86::VCVTDQ2PDrr, &X86::VR128RegClass, Op0); 4367 } 4368 return 0; 4369} 4370 4371unsigned fastEmit_X86ISD_CVTSI2P_MVT_v4i32_r(MVT RetVT, unsigned Op0) { 4372switch (RetVT.SimpleTy) { 4373 case MVT::v8f16: return fastEmit_X86ISD_CVTSI2P_MVT_v4i32_MVT_v8f16_r(Op0); 4374 case MVT::v2f64: return fastEmit_X86ISD_CVTSI2P_MVT_v4i32_MVT_v2f64_r(Op0); 4375 default: return 0; 4376} 4377} 4378 4379unsigned fastEmit_X86ISD_CVTSI2P_MVT_v2i64_MVT_v8f16_r(unsigned Op0) { 4380 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 4381 return fastEmitInst_r(X86::VCVTQQ2PHZ128rr, &X86::VR128XRegClass, Op0); 4382 } 4383 return 0; 4384} 4385 4386unsigned fastEmit_X86ISD_CVTSI2P_MVT_v2i64_MVT_v4f32_r(unsigned Op0) { 4387 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 4388 return fastEmitInst_r(X86::VCVTQQ2PSZ128rr, &X86::VR128XRegClass, Op0); 4389 } 4390 return 0; 4391} 4392 4393unsigned fastEmit_X86ISD_CVTSI2P_MVT_v2i64_r(MVT RetVT, unsigned Op0) { 4394switch (RetVT.SimpleTy) { 4395 case MVT::v8f16: return fastEmit_X86ISD_CVTSI2P_MVT_v2i64_MVT_v8f16_r(Op0); 4396 case MVT::v4f32: return fastEmit_X86ISD_CVTSI2P_MVT_v2i64_MVT_v4f32_r(Op0); 4397 default: return 0; 4398} 4399} 4400 4401unsigned fastEmit_X86ISD_CVTSI2P_MVT_v4i64_r(MVT RetVT, unsigned Op0) { 4402 if (RetVT.SimpleTy != MVT::v8f16) 4403 return 0; 4404 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 4405 return fastEmitInst_r(X86::VCVTQQ2PHZ256rr, &X86::VR128XRegClass, Op0); 4406 } 4407 return 0; 4408} 4409 4410unsigned fastEmit_X86ISD_CVTSI2P_r(MVT VT, MVT RetVT, unsigned Op0) { 4411 switch (VT.SimpleTy) { 4412 case MVT::v4i32: return fastEmit_X86ISD_CVTSI2P_MVT_v4i32_r(RetVT, Op0); 4413 case MVT::v2i64: return fastEmit_X86ISD_CVTSI2P_MVT_v2i64_r(RetVT, Op0); 4414 case MVT::v4i64: return fastEmit_X86ISD_CVTSI2P_MVT_v4i64_r(RetVT, Op0); 4415 default: return 0; 4416 } 4417} 4418 4419// FastEmit functions for X86ISD::CVTTP2SI. 4420 4421unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v8i16_r(unsigned Op0) { 4422 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 4423 return fastEmitInst_r(X86::VCVTTPH2WZ128rr, &X86::VR128XRegClass, Op0); 4424 } 4425 return 0; 4426} 4427 4428unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v4i32_r(unsigned Op0) { 4429 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 4430 return fastEmitInst_r(X86::VCVTTPH2DQZ128rr, &X86::VR128XRegClass, Op0); 4431 } 4432 return 0; 4433} 4434 4435unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v8i32_r(unsigned Op0) { 4436 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 4437 return fastEmitInst_r(X86::VCVTTPH2DQZ256rr, &X86::VR256XRegClass, Op0); 4438 } 4439 return 0; 4440} 4441 4442unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v2i64_r(unsigned Op0) { 4443 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 4444 return fastEmitInst_r(X86::VCVTTPH2QQZ128rr, &X86::VR128XRegClass, Op0); 4445 } 4446 return 0; 4447} 4448 4449unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v4i64_r(unsigned Op0) { 4450 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 4451 return fastEmitInst_r(X86::VCVTTPH2QQZ256rr, &X86::VR256XRegClass, Op0); 4452 } 4453 return 0; 4454} 4455 4456unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v8i64_r(unsigned Op0) { 4457 if ((Subtarget->hasFP16())) { 4458 return fastEmitInst_r(X86::VCVTTPH2QQZrr, &X86::VR512RegClass, Op0); 4459 } 4460 return 0; 4461} 4462 4463unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 4464switch (RetVT.SimpleTy) { 4465 case MVT::v8i16: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v8i16_r(Op0); 4466 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v4i32_r(Op0); 4467 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v8i32_r(Op0); 4468 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v2i64_r(Op0); 4469 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v4i64_r(Op0); 4470 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_MVT_v8i64_r(Op0); 4471 default: return 0; 4472} 4473} 4474 4475unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v16f16_MVT_v16i16_r(unsigned Op0) { 4476 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 4477 return fastEmitInst_r(X86::VCVTTPH2WZ256rr, &X86::VR256XRegClass, Op0); 4478 } 4479 return 0; 4480} 4481 4482unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v16f16_MVT_v16i32_r(unsigned Op0) { 4483 if ((Subtarget->hasFP16())) { 4484 return fastEmitInst_r(X86::VCVTTPH2DQZrr, &X86::VR512RegClass, Op0); 4485 } 4486 return 0; 4487} 4488 4489unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v16f16_r(MVT RetVT, unsigned Op0) { 4490switch (RetVT.SimpleTy) { 4491 case MVT::v16i16: return fastEmit_X86ISD_CVTTP2SI_MVT_v16f16_MVT_v16i16_r(Op0); 4492 case MVT::v16i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v16f16_MVT_v16i32_r(Op0); 4493 default: return 0; 4494} 4495} 4496 4497unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v32f16_r(MVT RetVT, unsigned Op0) { 4498 if (RetVT.SimpleTy != MVT::v32i16) 4499 return 0; 4500 if ((Subtarget->hasFP16())) { 4501 return fastEmitInst_r(X86::VCVTTPH2WZrr, &X86::VR512RegClass, Op0); 4502 } 4503 return 0; 4504} 4505 4506unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i32_r(unsigned Op0) { 4507 if ((Subtarget->hasVLX())) { 4508 return fastEmitInst_r(X86::VCVTTPS2DQZ128rr, &X86::VR128XRegClass, Op0); 4509 } 4510 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 4511 return fastEmitInst_r(X86::CVTTPS2DQrr, &X86::VR128RegClass, Op0); 4512 } 4513 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 4514 return fastEmitInst_r(X86::VCVTTPS2DQrr, &X86::VR128RegClass, Op0); 4515 } 4516 return 0; 4517} 4518 4519unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v2i64_r(unsigned Op0) { 4520 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 4521 return fastEmitInst_r(X86::VCVTTPS2QQZ128rr, &X86::VR128XRegClass, Op0); 4522 } 4523 return 0; 4524} 4525 4526unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i64_r(unsigned Op0) { 4527 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 4528 return fastEmitInst_r(X86::VCVTTPS2QQZ256rr, &X86::VR256XRegClass, Op0); 4529 } 4530 return 0; 4531} 4532 4533unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 4534switch (RetVT.SimpleTy) { 4535 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i32_r(Op0); 4536 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v2i64_r(Op0); 4537 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i64_r(Op0); 4538 default: return 0; 4539} 4540} 4541 4542unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i32_r(unsigned Op0) { 4543 if ((Subtarget->hasVLX())) { 4544 return fastEmitInst_r(X86::VCVTTPS2DQZ256rr, &X86::VR256XRegClass, Op0); 4545 } 4546 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 4547 return fastEmitInst_r(X86::VCVTTPS2DQYrr, &X86::VR256RegClass, Op0); 4548 } 4549 return 0; 4550} 4551 4552unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i64_r(unsigned Op0) { 4553 if ((Subtarget->hasDQI())) { 4554 return fastEmitInst_r(X86::VCVTTPS2QQZrr, &X86::VR512RegClass, Op0); 4555 } 4556 return 0; 4557} 4558 4559unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_r(MVT RetVT, unsigned Op0) { 4560switch (RetVT.SimpleTy) { 4561 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i32_r(Op0); 4562 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i64_r(Op0); 4563 default: return 0; 4564} 4565} 4566 4567unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v16f32_r(MVT RetVT, unsigned Op0) { 4568 if (RetVT.SimpleTy != MVT::v16i32) 4569 return 0; 4570 if ((Subtarget->hasAVX512())) { 4571 return fastEmitInst_r(X86::VCVTTPS2DQZrr, &X86::VR512RegClass, Op0); 4572 } 4573 return 0; 4574} 4575 4576unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v4i32_r(unsigned Op0) { 4577 if ((Subtarget->hasVLX())) { 4578 return fastEmitInst_r(X86::VCVTTPD2DQZ128rr, &X86::VR128XRegClass, Op0); 4579 } 4580 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 4581 return fastEmitInst_r(X86::CVTTPD2DQrr, &X86::VR128RegClass, Op0); 4582 } 4583 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 4584 return fastEmitInst_r(X86::VCVTTPD2DQrr, &X86::VR128RegClass, Op0); 4585 } 4586 return 0; 4587} 4588 4589unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v2i64_r(unsigned Op0) { 4590 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 4591 return fastEmitInst_r(X86::VCVTTPD2QQZ128rr, &X86::VR128XRegClass, Op0); 4592 } 4593 return 0; 4594} 4595 4596unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 4597switch (RetVT.SimpleTy) { 4598 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v4i32_r(Op0); 4599 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v2i64_r(Op0); 4600 default: return 0; 4601} 4602} 4603 4604unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i32_r(unsigned Op0) { 4605 if ((Subtarget->hasVLX())) { 4606 return fastEmitInst_r(X86::VCVTTPD2DQZ256rr, &X86::VR128XRegClass, Op0); 4607 } 4608 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 4609 return fastEmitInst_r(X86::VCVTTPD2DQYrr, &X86::VR128RegClass, Op0); 4610 } 4611 return 0; 4612} 4613 4614unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i64_r(unsigned Op0) { 4615 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 4616 return fastEmitInst_r(X86::VCVTTPD2QQZ256rr, &X86::VR256XRegClass, Op0); 4617 } 4618 return 0; 4619} 4620 4621unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_r(MVT RetVT, unsigned Op0) { 4622switch (RetVT.SimpleTy) { 4623 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i32_r(Op0); 4624 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i64_r(Op0); 4625 default: return 0; 4626} 4627} 4628 4629unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i32_r(unsigned Op0) { 4630 if ((Subtarget->hasAVX512())) { 4631 return fastEmitInst_r(X86::VCVTTPD2DQZrr, &X86::VR256XRegClass, Op0); 4632 } 4633 return 0; 4634} 4635 4636unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i64_r(unsigned Op0) { 4637 if ((Subtarget->hasDQI())) { 4638 return fastEmitInst_r(X86::VCVTTPD2QQZrr, &X86::VR512RegClass, Op0); 4639 } 4640 return 0; 4641} 4642 4643unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_r(MVT RetVT, unsigned Op0) { 4644switch (RetVT.SimpleTy) { 4645 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i32_r(Op0); 4646 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i64_r(Op0); 4647 default: return 0; 4648} 4649} 4650 4651unsigned fastEmit_X86ISD_CVTTP2SI_r(MVT VT, MVT RetVT, unsigned Op0) { 4652 switch (VT.SimpleTy) { 4653 case MVT::v8f16: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f16_r(RetVT, Op0); 4654 case MVT::v16f16: return fastEmit_X86ISD_CVTTP2SI_MVT_v16f16_r(RetVT, Op0); 4655 case MVT::v32f16: return fastEmit_X86ISD_CVTTP2SI_MVT_v32f16_r(RetVT, Op0); 4656 case MVT::v4f32: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_r(RetVT, Op0); 4657 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_r(RetVT, Op0); 4658 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2SI_MVT_v16f32_r(RetVT, Op0); 4659 case MVT::v2f64: return fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_r(RetVT, Op0); 4660 case MVT::v4f64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_r(RetVT, Op0); 4661 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_r(RetVT, Op0); 4662 default: return 0; 4663 } 4664} 4665 4666// FastEmit functions for X86ISD::CVTTP2SI_SAE. 4667 4668unsigned fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 4669 if (RetVT.SimpleTy != MVT::v8i64) 4670 return 0; 4671 if ((Subtarget->hasFP16())) { 4672 return fastEmitInst_r(X86::VCVTTPH2QQZrrb, &X86::VR512RegClass, Op0); 4673 } 4674 return 0; 4675} 4676 4677unsigned fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v16f16_r(MVT RetVT, unsigned Op0) { 4678 if (RetVT.SimpleTy != MVT::v16i32) 4679 return 0; 4680 if ((Subtarget->hasFP16())) { 4681 return fastEmitInst_r(X86::VCVTTPH2DQZrrb, &X86::VR512RegClass, Op0); 4682 } 4683 return 0; 4684} 4685 4686unsigned fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v32f16_r(MVT RetVT, unsigned Op0) { 4687 if (RetVT.SimpleTy != MVT::v32i16) 4688 return 0; 4689 if ((Subtarget->hasFP16())) { 4690 return fastEmitInst_r(X86::VCVTTPH2WZrrb, &X86::VR512RegClass, Op0); 4691 } 4692 return 0; 4693} 4694 4695unsigned fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f32_r(MVT RetVT, unsigned Op0) { 4696 if (RetVT.SimpleTy != MVT::v8i64) 4697 return 0; 4698 if ((Subtarget->hasDQI())) { 4699 return fastEmitInst_r(X86::VCVTTPS2QQZrrb, &X86::VR512RegClass, Op0); 4700 } 4701 return 0; 4702} 4703 4704unsigned fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v16f32_r(MVT RetVT, unsigned Op0) { 4705 if (RetVT.SimpleTy != MVT::v16i32) 4706 return 0; 4707 if ((Subtarget->hasAVX512())) { 4708 return fastEmitInst_r(X86::VCVTTPS2DQZrrb, &X86::VR512RegClass, Op0); 4709 } 4710 return 0; 4711} 4712 4713unsigned fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_MVT_v8i32_r(unsigned Op0) { 4714 if ((Subtarget->hasAVX512())) { 4715 return fastEmitInst_r(X86::VCVTTPD2DQZrrb, &X86::VR256XRegClass, Op0); 4716 } 4717 return 0; 4718} 4719 4720unsigned fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_MVT_v8i64_r(unsigned Op0) { 4721 if ((Subtarget->hasDQI())) { 4722 return fastEmitInst_r(X86::VCVTTPD2QQZrrb, &X86::VR512RegClass, Op0); 4723 } 4724 return 0; 4725} 4726 4727unsigned fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_r(MVT RetVT, unsigned Op0) { 4728switch (RetVT.SimpleTy) { 4729 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_MVT_v8i32_r(Op0); 4730 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_MVT_v8i64_r(Op0); 4731 default: return 0; 4732} 4733} 4734 4735unsigned fastEmit_X86ISD_CVTTP2SI_SAE_r(MVT VT, MVT RetVT, unsigned Op0) { 4736 switch (VT.SimpleTy) { 4737 case MVT::v8f16: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f16_r(RetVT, Op0); 4738 case MVT::v16f16: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v16f16_r(RetVT, Op0); 4739 case MVT::v32f16: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v32f16_r(RetVT, Op0); 4740 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f32_r(RetVT, Op0); 4741 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v16f32_r(RetVT, Op0); 4742 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_r(RetVT, Op0); 4743 default: return 0; 4744 } 4745} 4746 4747// FastEmit functions for X86ISD::CVTTP2UI. 4748 4749unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v8i16_r(unsigned Op0) { 4750 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 4751 return fastEmitInst_r(X86::VCVTTPH2UWZ128rr, &X86::VR128XRegClass, Op0); 4752 } 4753 return 0; 4754} 4755 4756unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v4i32_r(unsigned Op0) { 4757 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 4758 return fastEmitInst_r(X86::VCVTTPH2UDQZ128rr, &X86::VR128XRegClass, Op0); 4759 } 4760 return 0; 4761} 4762 4763unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v8i32_r(unsigned Op0) { 4764 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 4765 return fastEmitInst_r(X86::VCVTTPH2UDQZ256rr, &X86::VR256XRegClass, Op0); 4766 } 4767 return 0; 4768} 4769 4770unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v2i64_r(unsigned Op0) { 4771 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 4772 return fastEmitInst_r(X86::VCVTTPH2UQQZ128rr, &X86::VR128XRegClass, Op0); 4773 } 4774 return 0; 4775} 4776 4777unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v4i64_r(unsigned Op0) { 4778 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 4779 return fastEmitInst_r(X86::VCVTTPH2UQQZ256rr, &X86::VR256XRegClass, Op0); 4780 } 4781 return 0; 4782} 4783 4784unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v8i64_r(unsigned Op0) { 4785 if ((Subtarget->hasFP16())) { 4786 return fastEmitInst_r(X86::VCVTTPH2UQQZrr, &X86::VR512RegClass, Op0); 4787 } 4788 return 0; 4789} 4790 4791unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 4792switch (RetVT.SimpleTy) { 4793 case MVT::v8i16: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v8i16_r(Op0); 4794 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v4i32_r(Op0); 4795 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v8i32_r(Op0); 4796 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v2i64_r(Op0); 4797 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v4i64_r(Op0); 4798 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_MVT_v8i64_r(Op0); 4799 default: return 0; 4800} 4801} 4802 4803unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v16f16_MVT_v16i16_r(unsigned Op0) { 4804 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 4805 return fastEmitInst_r(X86::VCVTTPH2UWZ256rr, &X86::VR256XRegClass, Op0); 4806 } 4807 return 0; 4808} 4809 4810unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v16f16_MVT_v16i32_r(unsigned Op0) { 4811 if ((Subtarget->hasFP16())) { 4812 return fastEmitInst_r(X86::VCVTTPH2UDQZrr, &X86::VR512RegClass, Op0); 4813 } 4814 return 0; 4815} 4816 4817unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v16f16_r(MVT RetVT, unsigned Op0) { 4818switch (RetVT.SimpleTy) { 4819 case MVT::v16i16: return fastEmit_X86ISD_CVTTP2UI_MVT_v16f16_MVT_v16i16_r(Op0); 4820 case MVT::v16i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v16f16_MVT_v16i32_r(Op0); 4821 default: return 0; 4822} 4823} 4824 4825unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v32f16_r(MVT RetVT, unsigned Op0) { 4826 if (RetVT.SimpleTy != MVT::v32i16) 4827 return 0; 4828 if ((Subtarget->hasFP16())) { 4829 return fastEmitInst_r(X86::VCVTTPH2UWZrr, &X86::VR512RegClass, Op0); 4830 } 4831 return 0; 4832} 4833 4834unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i32_r(unsigned Op0) { 4835 if ((Subtarget->hasVLX())) { 4836 return fastEmitInst_r(X86::VCVTTPS2UDQZ128rr, &X86::VR128XRegClass, Op0); 4837 } 4838 return 0; 4839} 4840 4841unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v2i64_r(unsigned Op0) { 4842 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 4843 return fastEmitInst_r(X86::VCVTTPS2UQQZ128rr, &X86::VR128XRegClass, Op0); 4844 } 4845 return 0; 4846} 4847 4848unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i64_r(unsigned Op0) { 4849 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 4850 return fastEmitInst_r(X86::VCVTTPS2UQQZ256rr, &X86::VR256XRegClass, Op0); 4851 } 4852 return 0; 4853} 4854 4855unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 4856switch (RetVT.SimpleTy) { 4857 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i32_r(Op0); 4858 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v2i64_r(Op0); 4859 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i64_r(Op0); 4860 default: return 0; 4861} 4862} 4863 4864unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i32_r(unsigned Op0) { 4865 if ((Subtarget->hasVLX())) { 4866 return fastEmitInst_r(X86::VCVTTPS2UDQZ256rr, &X86::VR256XRegClass, Op0); 4867 } 4868 return 0; 4869} 4870 4871unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i64_r(unsigned Op0) { 4872 if ((Subtarget->hasDQI())) { 4873 return fastEmitInst_r(X86::VCVTTPS2UQQZrr, &X86::VR512RegClass, Op0); 4874 } 4875 return 0; 4876} 4877 4878unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_r(MVT RetVT, unsigned Op0) { 4879switch (RetVT.SimpleTy) { 4880 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i32_r(Op0); 4881 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i64_r(Op0); 4882 default: return 0; 4883} 4884} 4885 4886unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v16f32_r(MVT RetVT, unsigned Op0) { 4887 if (RetVT.SimpleTy != MVT::v16i32) 4888 return 0; 4889 if ((Subtarget->hasAVX512())) { 4890 return fastEmitInst_r(X86::VCVTTPS2UDQZrr, &X86::VR512RegClass, Op0); 4891 } 4892 return 0; 4893} 4894 4895unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v4i32_r(unsigned Op0) { 4896 if ((Subtarget->hasVLX())) { 4897 return fastEmitInst_r(X86::VCVTTPD2UDQZ128rr, &X86::VR128XRegClass, Op0); 4898 } 4899 return 0; 4900} 4901 4902unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v2i64_r(unsigned Op0) { 4903 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 4904 return fastEmitInst_r(X86::VCVTTPD2UQQZ128rr, &X86::VR128XRegClass, Op0); 4905 } 4906 return 0; 4907} 4908 4909unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 4910switch (RetVT.SimpleTy) { 4911 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v4i32_r(Op0); 4912 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v2i64_r(Op0); 4913 default: return 0; 4914} 4915} 4916 4917unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i32_r(unsigned Op0) { 4918 if ((Subtarget->hasVLX())) { 4919 return fastEmitInst_r(X86::VCVTTPD2UDQZ256rr, &X86::VR128XRegClass, Op0); 4920 } 4921 return 0; 4922} 4923 4924unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i64_r(unsigned Op0) { 4925 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 4926 return fastEmitInst_r(X86::VCVTTPD2UQQZ256rr, &X86::VR256XRegClass, Op0); 4927 } 4928 return 0; 4929} 4930 4931unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_r(MVT RetVT, unsigned Op0) { 4932switch (RetVT.SimpleTy) { 4933 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i32_r(Op0); 4934 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i64_r(Op0); 4935 default: return 0; 4936} 4937} 4938 4939unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i32_r(unsigned Op0) { 4940 if ((Subtarget->hasAVX512())) { 4941 return fastEmitInst_r(X86::VCVTTPD2UDQZrr, &X86::VR256XRegClass, Op0); 4942 } 4943 return 0; 4944} 4945 4946unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i64_r(unsigned Op0) { 4947 if ((Subtarget->hasDQI())) { 4948 return fastEmitInst_r(X86::VCVTTPD2UQQZrr, &X86::VR512RegClass, Op0); 4949 } 4950 return 0; 4951} 4952 4953unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_r(MVT RetVT, unsigned Op0) { 4954switch (RetVT.SimpleTy) { 4955 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i32_r(Op0); 4956 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i64_r(Op0); 4957 default: return 0; 4958} 4959} 4960 4961unsigned fastEmit_X86ISD_CVTTP2UI_r(MVT VT, MVT RetVT, unsigned Op0) { 4962 switch (VT.SimpleTy) { 4963 case MVT::v8f16: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f16_r(RetVT, Op0); 4964 case MVT::v16f16: return fastEmit_X86ISD_CVTTP2UI_MVT_v16f16_r(RetVT, Op0); 4965 case MVT::v32f16: return fastEmit_X86ISD_CVTTP2UI_MVT_v32f16_r(RetVT, Op0); 4966 case MVT::v4f32: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_r(RetVT, Op0); 4967 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_r(RetVT, Op0); 4968 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2UI_MVT_v16f32_r(RetVT, Op0); 4969 case MVT::v2f64: return fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_r(RetVT, Op0); 4970 case MVT::v4f64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_r(RetVT, Op0); 4971 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_r(RetVT, Op0); 4972 default: return 0; 4973 } 4974} 4975 4976// FastEmit functions for X86ISD::CVTTP2UI_SAE. 4977 4978unsigned fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 4979 if (RetVT.SimpleTy != MVT::v8i64) 4980 return 0; 4981 if ((Subtarget->hasFP16())) { 4982 return fastEmitInst_r(X86::VCVTTPH2UQQZrrb, &X86::VR512RegClass, Op0); 4983 } 4984 return 0; 4985} 4986 4987unsigned fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v16f16_r(MVT RetVT, unsigned Op0) { 4988 if (RetVT.SimpleTy != MVT::v16i32) 4989 return 0; 4990 if ((Subtarget->hasFP16())) { 4991 return fastEmitInst_r(X86::VCVTTPH2UDQZrrb, &X86::VR512RegClass, Op0); 4992 } 4993 return 0; 4994} 4995 4996unsigned fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v32f16_r(MVT RetVT, unsigned Op0) { 4997 if (RetVT.SimpleTy != MVT::v32i16) 4998 return 0; 4999 if ((Subtarget->hasFP16())) { 5000 return fastEmitInst_r(X86::VCVTTPH2UWZrrb, &X86::VR512RegClass, Op0); 5001 } 5002 return 0; 5003} 5004 5005unsigned fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f32_r(MVT RetVT, unsigned Op0) { 5006 if (RetVT.SimpleTy != MVT::v8i64) 5007 return 0; 5008 if ((Subtarget->hasDQI())) { 5009 return fastEmitInst_r(X86::VCVTTPS2UQQZrrb, &X86::VR512RegClass, Op0); 5010 } 5011 return 0; 5012} 5013 5014unsigned fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v16f32_r(MVT RetVT, unsigned Op0) { 5015 if (RetVT.SimpleTy != MVT::v16i32) 5016 return 0; 5017 if ((Subtarget->hasAVX512())) { 5018 return fastEmitInst_r(X86::VCVTTPS2UDQZrrb, &X86::VR512RegClass, Op0); 5019 } 5020 return 0; 5021} 5022 5023unsigned fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_MVT_v8i32_r(unsigned Op0) { 5024 if ((Subtarget->hasAVX512())) { 5025 return fastEmitInst_r(X86::VCVTTPD2UDQZrrb, &X86::VR256XRegClass, Op0); 5026 } 5027 return 0; 5028} 5029 5030unsigned fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_MVT_v8i64_r(unsigned Op0) { 5031 if ((Subtarget->hasDQI())) { 5032 return fastEmitInst_r(X86::VCVTTPD2UQQZrrb, &X86::VR512RegClass, Op0); 5033 } 5034 return 0; 5035} 5036 5037unsigned fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_r(MVT RetVT, unsigned Op0) { 5038switch (RetVT.SimpleTy) { 5039 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_MVT_v8i32_r(Op0); 5040 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_MVT_v8i64_r(Op0); 5041 default: return 0; 5042} 5043} 5044 5045unsigned fastEmit_X86ISD_CVTTP2UI_SAE_r(MVT VT, MVT RetVT, unsigned Op0) { 5046 switch (VT.SimpleTy) { 5047 case MVT::v8f16: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f16_r(RetVT, Op0); 5048 case MVT::v16f16: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v16f16_r(RetVT, Op0); 5049 case MVT::v32f16: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v32f16_r(RetVT, Op0); 5050 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f32_r(RetVT, Op0); 5051 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v16f32_r(RetVT, Op0); 5052 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_r(RetVT, Op0); 5053 default: return 0; 5054 } 5055} 5056 5057// FastEmit functions for X86ISD::CVTTS2SI. 5058 5059unsigned fastEmit_X86ISD_CVTTS2SI_MVT_v8f16_MVT_i32_r(unsigned Op0) { 5060 if ((Subtarget->hasFP16())) { 5061 return fastEmitInst_r(X86::VCVTTSH2SIZrr_Int, &X86::GR32RegClass, Op0); 5062 } 5063 return 0; 5064} 5065 5066unsigned fastEmit_X86ISD_CVTTS2SI_MVT_v8f16_MVT_i64_r(unsigned Op0) { 5067 if ((Subtarget->hasFP16())) { 5068 return fastEmitInst_r(X86::VCVTTSH2SI64Zrr_Int, &X86::GR64RegClass, Op0); 5069 } 5070 return 0; 5071} 5072 5073unsigned fastEmit_X86ISD_CVTTS2SI_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 5074switch (RetVT.SimpleTy) { 5075 case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_MVT_v8f16_MVT_i32_r(Op0); 5076 case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_MVT_v8f16_MVT_i64_r(Op0); 5077 default: return 0; 5078} 5079} 5080 5081unsigned fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_MVT_i32_r(unsigned Op0) { 5082 if ((Subtarget->hasAVX512())) { 5083 return fastEmitInst_r(X86::VCVTTSS2SIZrr_Int, &X86::GR32RegClass, Op0); 5084 } 5085 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 5086 return fastEmitInst_r(X86::CVTTSS2SIrr_Int, &X86::GR32RegClass, Op0); 5087 } 5088 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 5089 return fastEmitInst_r(X86::VCVTTSS2SIrr_Int, &X86::GR32RegClass, Op0); 5090 } 5091 return 0; 5092} 5093 5094unsigned fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_MVT_i64_r(unsigned Op0) { 5095 if ((Subtarget->hasAVX512())) { 5096 return fastEmitInst_r(X86::VCVTTSS2SI64Zrr_Int, &X86::GR64RegClass, Op0); 5097 } 5098 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 5099 return fastEmitInst_r(X86::CVTTSS2SI64rr_Int, &X86::GR64RegClass, Op0); 5100 } 5101 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 5102 return fastEmitInst_r(X86::VCVTTSS2SI64rr_Int, &X86::GR64RegClass, Op0); 5103 } 5104 return 0; 5105} 5106 5107unsigned fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 5108switch (RetVT.SimpleTy) { 5109 case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_MVT_i32_r(Op0); 5110 case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_MVT_i64_r(Op0); 5111 default: return 0; 5112} 5113} 5114 5115unsigned fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_MVT_i32_r(unsigned Op0) { 5116 if ((Subtarget->hasAVX512())) { 5117 return fastEmitInst_r(X86::VCVTTSD2SIZrr_Int, &X86::GR32RegClass, Op0); 5118 } 5119 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 5120 return fastEmitInst_r(X86::CVTTSD2SIrr_Int, &X86::GR32RegClass, Op0); 5121 } 5122 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 5123 return fastEmitInst_r(X86::VCVTTSD2SIrr_Int, &X86::GR32RegClass, Op0); 5124 } 5125 return 0; 5126} 5127 5128unsigned fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_MVT_i64_r(unsigned Op0) { 5129 if ((Subtarget->hasAVX512())) { 5130 return fastEmitInst_r(X86::VCVTTSD2SI64Zrr_Int, &X86::GR64RegClass, Op0); 5131 } 5132 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 5133 return fastEmitInst_r(X86::CVTTSD2SI64rr_Int, &X86::GR64RegClass, Op0); 5134 } 5135 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 5136 return fastEmitInst_r(X86::VCVTTSD2SI64rr_Int, &X86::GR64RegClass, Op0); 5137 } 5138 return 0; 5139} 5140 5141unsigned fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 5142switch (RetVT.SimpleTy) { 5143 case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_MVT_i32_r(Op0); 5144 case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_MVT_i64_r(Op0); 5145 default: return 0; 5146} 5147} 5148 5149unsigned fastEmit_X86ISD_CVTTS2SI_r(MVT VT, MVT RetVT, unsigned Op0) { 5150 switch (VT.SimpleTy) { 5151 case MVT::v8f16: return fastEmit_X86ISD_CVTTS2SI_MVT_v8f16_r(RetVT, Op0); 5152 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_r(RetVT, Op0); 5153 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_r(RetVT, Op0); 5154 default: return 0; 5155 } 5156} 5157 5158// FastEmit functions for X86ISD::CVTTS2SI_SAE. 5159 5160unsigned fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v8f16_MVT_i32_r(unsigned Op0) { 5161 if ((Subtarget->hasFP16())) { 5162 return fastEmitInst_r(X86::VCVTTSH2SIZrrb_Int, &X86::GR32RegClass, Op0); 5163 } 5164 return 0; 5165} 5166 5167unsigned fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v8f16_MVT_i64_r(unsigned Op0) { 5168 if ((Subtarget->hasFP16())) { 5169 return fastEmitInst_r(X86::VCVTTSH2SI64Zrrb_Int, &X86::GR64RegClass, Op0); 5170 } 5171 return 0; 5172} 5173 5174unsigned fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 5175switch (RetVT.SimpleTy) { 5176 case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v8f16_MVT_i32_r(Op0); 5177 case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v8f16_MVT_i64_r(Op0); 5178 default: return 0; 5179} 5180} 5181 5182unsigned fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_MVT_i32_r(unsigned Op0) { 5183 if ((Subtarget->hasAVX512())) { 5184 return fastEmitInst_r(X86::VCVTTSS2SIZrrb_Int, &X86::GR32RegClass, Op0); 5185 } 5186 return 0; 5187} 5188 5189unsigned fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_MVT_i64_r(unsigned Op0) { 5190 if ((Subtarget->hasAVX512())) { 5191 return fastEmitInst_r(X86::VCVTTSS2SI64Zrrb_Int, &X86::GR64RegClass, Op0); 5192 } 5193 return 0; 5194} 5195 5196unsigned fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 5197switch (RetVT.SimpleTy) { 5198 case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_MVT_i32_r(Op0); 5199 case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_MVT_i64_r(Op0); 5200 default: return 0; 5201} 5202} 5203 5204unsigned fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_MVT_i32_r(unsigned Op0) { 5205 if ((Subtarget->hasAVX512())) { 5206 return fastEmitInst_r(X86::VCVTTSD2SIZrrb_Int, &X86::GR32RegClass, Op0); 5207 } 5208 return 0; 5209} 5210 5211unsigned fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_MVT_i64_r(unsigned Op0) { 5212 if ((Subtarget->hasAVX512())) { 5213 return fastEmitInst_r(X86::VCVTTSD2SI64Zrrb_Int, &X86::GR64RegClass, Op0); 5214 } 5215 return 0; 5216} 5217 5218unsigned fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 5219switch (RetVT.SimpleTy) { 5220 case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_MVT_i32_r(Op0); 5221 case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_MVT_i64_r(Op0); 5222 default: return 0; 5223} 5224} 5225 5226unsigned fastEmit_X86ISD_CVTTS2SI_SAE_r(MVT VT, MVT RetVT, unsigned Op0) { 5227 switch (VT.SimpleTy) { 5228 case MVT::v8f16: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v8f16_r(RetVT, Op0); 5229 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_r(RetVT, Op0); 5230 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_r(RetVT, Op0); 5231 default: return 0; 5232 } 5233} 5234 5235// FastEmit functions for X86ISD::CVTTS2UI. 5236 5237unsigned fastEmit_X86ISD_CVTTS2UI_MVT_v8f16_MVT_i32_r(unsigned Op0) { 5238 if ((Subtarget->hasFP16())) { 5239 return fastEmitInst_r(X86::VCVTTSH2USIZrr_Int, &X86::GR32RegClass, Op0); 5240 } 5241 return 0; 5242} 5243 5244unsigned fastEmit_X86ISD_CVTTS2UI_MVT_v8f16_MVT_i64_r(unsigned Op0) { 5245 if ((Subtarget->hasFP16())) { 5246 return fastEmitInst_r(X86::VCVTTSH2USI64Zrr_Int, &X86::GR64RegClass, Op0); 5247 } 5248 return 0; 5249} 5250 5251unsigned fastEmit_X86ISD_CVTTS2UI_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 5252switch (RetVT.SimpleTy) { 5253 case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_MVT_v8f16_MVT_i32_r(Op0); 5254 case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_MVT_v8f16_MVT_i64_r(Op0); 5255 default: return 0; 5256} 5257} 5258 5259unsigned fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_MVT_i32_r(unsigned Op0) { 5260 if ((Subtarget->hasAVX512())) { 5261 return fastEmitInst_r(X86::VCVTTSS2USIZrr_Int, &X86::GR32RegClass, Op0); 5262 } 5263 return 0; 5264} 5265 5266unsigned fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_MVT_i64_r(unsigned Op0) { 5267 if ((Subtarget->hasAVX512())) { 5268 return fastEmitInst_r(X86::VCVTTSS2USI64Zrr_Int, &X86::GR64RegClass, Op0); 5269 } 5270 return 0; 5271} 5272 5273unsigned fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 5274switch (RetVT.SimpleTy) { 5275 case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_MVT_i32_r(Op0); 5276 case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_MVT_i64_r(Op0); 5277 default: return 0; 5278} 5279} 5280 5281unsigned fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_MVT_i32_r(unsigned Op0) { 5282 if ((Subtarget->hasAVX512())) { 5283 return fastEmitInst_r(X86::VCVTTSD2USIZrr_Int, &X86::GR32RegClass, Op0); 5284 } 5285 return 0; 5286} 5287 5288unsigned fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_MVT_i64_r(unsigned Op0) { 5289 if ((Subtarget->hasAVX512())) { 5290 return fastEmitInst_r(X86::VCVTTSD2USI64Zrr_Int, &X86::GR64RegClass, Op0); 5291 } 5292 return 0; 5293} 5294 5295unsigned fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 5296switch (RetVT.SimpleTy) { 5297 case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_MVT_i32_r(Op0); 5298 case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_MVT_i64_r(Op0); 5299 default: return 0; 5300} 5301} 5302 5303unsigned fastEmit_X86ISD_CVTTS2UI_r(MVT VT, MVT RetVT, unsigned Op0) { 5304 switch (VT.SimpleTy) { 5305 case MVT::v8f16: return fastEmit_X86ISD_CVTTS2UI_MVT_v8f16_r(RetVT, Op0); 5306 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_r(RetVT, Op0); 5307 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_r(RetVT, Op0); 5308 default: return 0; 5309 } 5310} 5311 5312// FastEmit functions for X86ISD::CVTTS2UI_SAE. 5313 5314unsigned fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v8f16_MVT_i32_r(unsigned Op0) { 5315 if ((Subtarget->hasFP16())) { 5316 return fastEmitInst_r(X86::VCVTTSH2USIZrrb_Int, &X86::GR32RegClass, Op0); 5317 } 5318 return 0; 5319} 5320 5321unsigned fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v8f16_MVT_i64_r(unsigned Op0) { 5322 if ((Subtarget->hasFP16())) { 5323 return fastEmitInst_r(X86::VCVTTSH2USI64Zrrb_Int, &X86::GR64RegClass, Op0); 5324 } 5325 return 0; 5326} 5327 5328unsigned fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 5329switch (RetVT.SimpleTy) { 5330 case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v8f16_MVT_i32_r(Op0); 5331 case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v8f16_MVT_i64_r(Op0); 5332 default: return 0; 5333} 5334} 5335 5336unsigned fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_MVT_i32_r(unsigned Op0) { 5337 if ((Subtarget->hasAVX512())) { 5338 return fastEmitInst_r(X86::VCVTTSS2USIZrrb_Int, &X86::GR32RegClass, Op0); 5339 } 5340 return 0; 5341} 5342 5343unsigned fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_MVT_i64_r(unsigned Op0) { 5344 if ((Subtarget->hasAVX512())) { 5345 return fastEmitInst_r(X86::VCVTTSS2USI64Zrrb_Int, &X86::GR64RegClass, Op0); 5346 } 5347 return 0; 5348} 5349 5350unsigned fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 5351switch (RetVT.SimpleTy) { 5352 case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_MVT_i32_r(Op0); 5353 case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_MVT_i64_r(Op0); 5354 default: return 0; 5355} 5356} 5357 5358unsigned fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_MVT_i32_r(unsigned Op0) { 5359 if ((Subtarget->hasAVX512())) { 5360 return fastEmitInst_r(X86::VCVTTSD2USIZrrb_Int, &X86::GR32RegClass, Op0); 5361 } 5362 return 0; 5363} 5364 5365unsigned fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_MVT_i64_r(unsigned Op0) { 5366 if ((Subtarget->hasAVX512())) { 5367 return fastEmitInst_r(X86::VCVTTSD2USI64Zrrb_Int, &X86::GR64RegClass, Op0); 5368 } 5369 return 0; 5370} 5371 5372unsigned fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 5373switch (RetVT.SimpleTy) { 5374 case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_MVT_i32_r(Op0); 5375 case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_MVT_i64_r(Op0); 5376 default: return 0; 5377} 5378} 5379 5380unsigned fastEmit_X86ISD_CVTTS2UI_SAE_r(MVT VT, MVT RetVT, unsigned Op0) { 5381 switch (VT.SimpleTy) { 5382 case MVT::v8f16: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v8f16_r(RetVT, Op0); 5383 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_r(RetVT, Op0); 5384 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_r(RetVT, Op0); 5385 default: return 0; 5386 } 5387} 5388 5389// FastEmit functions for X86ISD::CVTUI2P. 5390 5391unsigned fastEmit_X86ISD_CVTUI2P_MVT_v4i32_MVT_v8f16_r(unsigned Op0) { 5392 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 5393 return fastEmitInst_r(X86::VCVTUDQ2PHZ128rr, &X86::VR128XRegClass, Op0); 5394 } 5395 return 0; 5396} 5397 5398unsigned fastEmit_X86ISD_CVTUI2P_MVT_v4i32_MVT_v2f64_r(unsigned Op0) { 5399 if ((Subtarget->hasVLX())) { 5400 return fastEmitInst_r(X86::VCVTUDQ2PDZ128rr, &X86::VR128XRegClass, Op0); 5401 } 5402 return 0; 5403} 5404 5405unsigned fastEmit_X86ISD_CVTUI2P_MVT_v4i32_r(MVT RetVT, unsigned Op0) { 5406switch (RetVT.SimpleTy) { 5407 case MVT::v8f16: return fastEmit_X86ISD_CVTUI2P_MVT_v4i32_MVT_v8f16_r(Op0); 5408 case MVT::v2f64: return fastEmit_X86ISD_CVTUI2P_MVT_v4i32_MVT_v2f64_r(Op0); 5409 default: return 0; 5410} 5411} 5412 5413unsigned fastEmit_X86ISD_CVTUI2P_MVT_v2i64_MVT_v8f16_r(unsigned Op0) { 5414 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 5415 return fastEmitInst_r(X86::VCVTUQQ2PHZ128rr, &X86::VR128XRegClass, Op0); 5416 } 5417 return 0; 5418} 5419 5420unsigned fastEmit_X86ISD_CVTUI2P_MVT_v2i64_MVT_v4f32_r(unsigned Op0) { 5421 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 5422 return fastEmitInst_r(X86::VCVTUQQ2PSZ128rr, &X86::VR128XRegClass, Op0); 5423 } 5424 return 0; 5425} 5426 5427unsigned fastEmit_X86ISD_CVTUI2P_MVT_v2i64_r(MVT RetVT, unsigned Op0) { 5428switch (RetVT.SimpleTy) { 5429 case MVT::v8f16: return fastEmit_X86ISD_CVTUI2P_MVT_v2i64_MVT_v8f16_r(Op0); 5430 case MVT::v4f32: return fastEmit_X86ISD_CVTUI2P_MVT_v2i64_MVT_v4f32_r(Op0); 5431 default: return 0; 5432} 5433} 5434 5435unsigned fastEmit_X86ISD_CVTUI2P_MVT_v4i64_r(MVT RetVT, unsigned Op0) { 5436 if (RetVT.SimpleTy != MVT::v8f16) 5437 return 0; 5438 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 5439 return fastEmitInst_r(X86::VCVTUQQ2PHZ256rr, &X86::VR128XRegClass, Op0); 5440 } 5441 return 0; 5442} 5443 5444unsigned fastEmit_X86ISD_CVTUI2P_r(MVT VT, MVT RetVT, unsigned Op0) { 5445 switch (VT.SimpleTy) { 5446 case MVT::v4i32: return fastEmit_X86ISD_CVTUI2P_MVT_v4i32_r(RetVT, Op0); 5447 case MVT::v2i64: return fastEmit_X86ISD_CVTUI2P_MVT_v2i64_r(RetVT, Op0); 5448 case MVT::v4i64: return fastEmit_X86ISD_CVTUI2P_MVT_v4i64_r(RetVT, Op0); 5449 default: return 0; 5450 } 5451} 5452 5453// FastEmit functions for X86ISD::DYN_ALLOCA. 5454 5455unsigned fastEmit_X86ISD_DYN_ALLOCA_MVT_i32_r(MVT RetVT, unsigned Op0) { 5456 if (RetVT.SimpleTy != MVT::isVoid) 5457 return 0; 5458 if ((!Subtarget->isTarget64BitLP64())) { 5459 return fastEmitInst_r(X86::DYN_ALLOCA_32, &X86::GR32RegClass, Op0); 5460 } 5461 return 0; 5462} 5463 5464unsigned fastEmit_X86ISD_DYN_ALLOCA_MVT_i64_r(MVT RetVT, unsigned Op0) { 5465 if (RetVT.SimpleTy != MVT::isVoid) 5466 return 0; 5467 if ((Subtarget->is64Bit())) { 5468 return fastEmitInst_r(X86::DYN_ALLOCA_64, &X86::GR64RegClass, Op0); 5469 } 5470 return 0; 5471} 5472 5473unsigned fastEmit_X86ISD_DYN_ALLOCA_r(MVT VT, MVT RetVT, unsigned Op0) { 5474 switch (VT.SimpleTy) { 5475 case MVT::i32: return fastEmit_X86ISD_DYN_ALLOCA_MVT_i32_r(RetVT, Op0); 5476 case MVT::i64: return fastEmit_X86ISD_DYN_ALLOCA_MVT_i64_r(RetVT, Op0); 5477 default: return 0; 5478 } 5479} 5480 5481// FastEmit functions for X86ISD::EH_RETURN. 5482 5483unsigned fastEmit_X86ISD_EH_RETURN_MVT_i32_r(MVT RetVT, unsigned Op0) { 5484 if (RetVT.SimpleTy != MVT::isVoid) 5485 return 0; 5486 return fastEmitInst_r(X86::EH_RETURN, &X86::GR32RegClass, Op0); 5487} 5488 5489unsigned fastEmit_X86ISD_EH_RETURN_MVT_i64_r(MVT RetVT, unsigned Op0) { 5490 if (RetVT.SimpleTy != MVT::isVoid) 5491 return 0; 5492 return fastEmitInst_r(X86::EH_RETURN64, &X86::GR64RegClass, Op0); 5493} 5494 5495unsigned fastEmit_X86ISD_EH_RETURN_r(MVT VT, MVT RetVT, unsigned Op0) { 5496 switch (VT.SimpleTy) { 5497 case MVT::i32: return fastEmit_X86ISD_EH_RETURN_MVT_i32_r(RetVT, Op0); 5498 case MVT::i64: return fastEmit_X86ISD_EH_RETURN_MVT_i64_r(RetVT, Op0); 5499 default: return 0; 5500 } 5501} 5502 5503// FastEmit functions for X86ISD::EXP2. 5504 5505unsigned fastEmit_X86ISD_EXP2_MVT_v16f32_r(MVT RetVT, unsigned Op0) { 5506 if (RetVT.SimpleTy != MVT::v16f32) 5507 return 0; 5508 if ((Subtarget->hasERI())) { 5509 return fastEmitInst_r(X86::VEXP2PSZr, &X86::VR512RegClass, Op0); 5510 } 5511 return 0; 5512} 5513 5514unsigned fastEmit_X86ISD_EXP2_MVT_v8f64_r(MVT RetVT, unsigned Op0) { 5515 if (RetVT.SimpleTy != MVT::v8f64) 5516 return 0; 5517 if ((Subtarget->hasERI())) { 5518 return fastEmitInst_r(X86::VEXP2PDZr, &X86::VR512RegClass, Op0); 5519 } 5520 return 0; 5521} 5522 5523unsigned fastEmit_X86ISD_EXP2_r(MVT VT, MVT RetVT, unsigned Op0) { 5524 switch (VT.SimpleTy) { 5525 case MVT::v16f32: return fastEmit_X86ISD_EXP2_MVT_v16f32_r(RetVT, Op0); 5526 case MVT::v8f64: return fastEmit_X86ISD_EXP2_MVT_v8f64_r(RetVT, Op0); 5527 default: return 0; 5528 } 5529} 5530 5531// FastEmit functions for X86ISD::EXP2_SAE. 5532 5533unsigned fastEmit_X86ISD_EXP2_SAE_MVT_v16f32_r(MVT RetVT, unsigned Op0) { 5534 if (RetVT.SimpleTy != MVT::v16f32) 5535 return 0; 5536 if ((Subtarget->hasERI())) { 5537 return fastEmitInst_r(X86::VEXP2PSZrb, &X86::VR512RegClass, Op0); 5538 } 5539 return 0; 5540} 5541 5542unsigned fastEmit_X86ISD_EXP2_SAE_MVT_v8f64_r(MVT RetVT, unsigned Op0) { 5543 if (RetVT.SimpleTy != MVT::v8f64) 5544 return 0; 5545 if ((Subtarget->hasERI())) { 5546 return fastEmitInst_r(X86::VEXP2PDZrb, &X86::VR512RegClass, Op0); 5547 } 5548 return 0; 5549} 5550 5551unsigned fastEmit_X86ISD_EXP2_SAE_r(MVT VT, MVT RetVT, unsigned Op0) { 5552 switch (VT.SimpleTy) { 5553 case MVT::v16f32: return fastEmit_X86ISD_EXP2_SAE_MVT_v16f32_r(RetVT, Op0); 5554 case MVT::v8f64: return fastEmit_X86ISD_EXP2_SAE_MVT_v8f64_r(RetVT, Op0); 5555 default: return 0; 5556 } 5557} 5558 5559// FastEmit functions for X86ISD::FGETEXP. 5560 5561unsigned fastEmit_X86ISD_FGETEXP_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 5562 if (RetVT.SimpleTy != MVT::v8f16) 5563 return 0; 5564 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 5565 return fastEmitInst_r(X86::VGETEXPPHZ128r, &X86::VR128XRegClass, Op0); 5566 } 5567 return 0; 5568} 5569 5570unsigned fastEmit_X86ISD_FGETEXP_MVT_v16f16_r(MVT RetVT, unsigned Op0) { 5571 if (RetVT.SimpleTy != MVT::v16f16) 5572 return 0; 5573 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 5574 return fastEmitInst_r(X86::VGETEXPPHZ256r, &X86::VR256XRegClass, Op0); 5575 } 5576 return 0; 5577} 5578 5579unsigned fastEmit_X86ISD_FGETEXP_MVT_v32f16_r(MVT RetVT, unsigned Op0) { 5580 if (RetVT.SimpleTy != MVT::v32f16) 5581 return 0; 5582 if ((Subtarget->hasFP16())) { 5583 return fastEmitInst_r(X86::VGETEXPPHZr, &X86::VR512RegClass, Op0); 5584 } 5585 return 0; 5586} 5587 5588unsigned fastEmit_X86ISD_FGETEXP_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 5589 if (RetVT.SimpleTy != MVT::v4f32) 5590 return 0; 5591 if ((Subtarget->hasVLX())) { 5592 return fastEmitInst_r(X86::VGETEXPPSZ128r, &X86::VR128XRegClass, Op0); 5593 } 5594 return 0; 5595} 5596 5597unsigned fastEmit_X86ISD_FGETEXP_MVT_v8f32_r(MVT RetVT, unsigned Op0) { 5598 if (RetVT.SimpleTy != MVT::v8f32) 5599 return 0; 5600 if ((Subtarget->hasVLX())) { 5601 return fastEmitInst_r(X86::VGETEXPPSZ256r, &X86::VR256XRegClass, Op0); 5602 } 5603 return 0; 5604} 5605 5606unsigned fastEmit_X86ISD_FGETEXP_MVT_v16f32_r(MVT RetVT, unsigned Op0) { 5607 if (RetVT.SimpleTy != MVT::v16f32) 5608 return 0; 5609 if ((Subtarget->hasAVX512())) { 5610 return fastEmitInst_r(X86::VGETEXPPSZr, &X86::VR512RegClass, Op0); 5611 } 5612 return 0; 5613} 5614 5615unsigned fastEmit_X86ISD_FGETEXP_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 5616 if (RetVT.SimpleTy != MVT::v2f64) 5617 return 0; 5618 if ((Subtarget->hasVLX())) { 5619 return fastEmitInst_r(X86::VGETEXPPDZ128r, &X86::VR128XRegClass, Op0); 5620 } 5621 return 0; 5622} 5623 5624unsigned fastEmit_X86ISD_FGETEXP_MVT_v4f64_r(MVT RetVT, unsigned Op0) { 5625 if (RetVT.SimpleTy != MVT::v4f64) 5626 return 0; 5627 if ((Subtarget->hasVLX())) { 5628 return fastEmitInst_r(X86::VGETEXPPDZ256r, &X86::VR256XRegClass, Op0); 5629 } 5630 return 0; 5631} 5632 5633unsigned fastEmit_X86ISD_FGETEXP_MVT_v8f64_r(MVT RetVT, unsigned Op0) { 5634 if (RetVT.SimpleTy != MVT::v8f64) 5635 return 0; 5636 if ((Subtarget->hasAVX512())) { 5637 return fastEmitInst_r(X86::VGETEXPPDZr, &X86::VR512RegClass, Op0); 5638 } 5639 return 0; 5640} 5641 5642unsigned fastEmit_X86ISD_FGETEXP_r(MVT VT, MVT RetVT, unsigned Op0) { 5643 switch (VT.SimpleTy) { 5644 case MVT::v8f16: return fastEmit_X86ISD_FGETEXP_MVT_v8f16_r(RetVT, Op0); 5645 case MVT::v16f16: return fastEmit_X86ISD_FGETEXP_MVT_v16f16_r(RetVT, Op0); 5646 case MVT::v32f16: return fastEmit_X86ISD_FGETEXP_MVT_v32f16_r(RetVT, Op0); 5647 case MVT::v4f32: return fastEmit_X86ISD_FGETEXP_MVT_v4f32_r(RetVT, Op0); 5648 case MVT::v8f32: return fastEmit_X86ISD_FGETEXP_MVT_v8f32_r(RetVT, Op0); 5649 case MVT::v16f32: return fastEmit_X86ISD_FGETEXP_MVT_v16f32_r(RetVT, Op0); 5650 case MVT::v2f64: return fastEmit_X86ISD_FGETEXP_MVT_v2f64_r(RetVT, Op0); 5651 case MVT::v4f64: return fastEmit_X86ISD_FGETEXP_MVT_v4f64_r(RetVT, Op0); 5652 case MVT::v8f64: return fastEmit_X86ISD_FGETEXP_MVT_v8f64_r(RetVT, Op0); 5653 default: return 0; 5654 } 5655} 5656 5657// FastEmit functions for X86ISD::FGETEXP_SAE. 5658 5659unsigned fastEmit_X86ISD_FGETEXP_SAE_MVT_v32f16_r(MVT RetVT, unsigned Op0) { 5660 if (RetVT.SimpleTy != MVT::v32f16) 5661 return 0; 5662 if ((Subtarget->hasFP16())) { 5663 return fastEmitInst_r(X86::VGETEXPPHZrb, &X86::VR512RegClass, Op0); 5664 } 5665 return 0; 5666} 5667 5668unsigned fastEmit_X86ISD_FGETEXP_SAE_MVT_v16f32_r(MVT RetVT, unsigned Op0) { 5669 if (RetVT.SimpleTy != MVT::v16f32) 5670 return 0; 5671 if ((Subtarget->hasAVX512())) { 5672 return fastEmitInst_r(X86::VGETEXPPSZrb, &X86::VR512RegClass, Op0); 5673 } 5674 return 0; 5675} 5676 5677unsigned fastEmit_X86ISD_FGETEXP_SAE_MVT_v8f64_r(MVT RetVT, unsigned Op0) { 5678 if (RetVT.SimpleTy != MVT::v8f64) 5679 return 0; 5680 if ((Subtarget->hasAVX512())) { 5681 return fastEmitInst_r(X86::VGETEXPPDZrb, &X86::VR512RegClass, Op0); 5682 } 5683 return 0; 5684} 5685 5686unsigned fastEmit_X86ISD_FGETEXP_SAE_r(MVT VT, MVT RetVT, unsigned Op0) { 5687 switch (VT.SimpleTy) { 5688 case MVT::v32f16: return fastEmit_X86ISD_FGETEXP_SAE_MVT_v32f16_r(RetVT, Op0); 5689 case MVT::v16f32: return fastEmit_X86ISD_FGETEXP_SAE_MVT_v16f32_r(RetVT, Op0); 5690 case MVT::v8f64: return fastEmit_X86ISD_FGETEXP_SAE_MVT_v8f64_r(RetVT, Op0); 5691 default: return 0; 5692 } 5693} 5694 5695// FastEmit functions for X86ISD::FRCP. 5696 5697unsigned fastEmit_X86ISD_FRCP_MVT_f32_r(MVT RetVT, unsigned Op0) { 5698 if (RetVT.SimpleTy != MVT::f32) 5699 return 0; 5700 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 5701 return fastEmitInst_r(X86::RCPSSr, &X86::FR32RegClass, Op0); 5702 } 5703 return 0; 5704} 5705 5706unsigned fastEmit_X86ISD_FRCP_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 5707 if (RetVT.SimpleTy != MVT::v4f32) 5708 return 0; 5709 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 5710 return fastEmitInst_r(X86::RCPPSr, &X86::VR128RegClass, Op0); 5711 } 5712 if ((Subtarget->hasAVX())) { 5713 return fastEmitInst_r(X86::VRCPPSr, &X86::VR128RegClass, Op0); 5714 } 5715 return 0; 5716} 5717 5718unsigned fastEmit_X86ISD_FRCP_MVT_v8f32_r(MVT RetVT, unsigned Op0) { 5719 if (RetVT.SimpleTy != MVT::v8f32) 5720 return 0; 5721 if ((Subtarget->hasAVX())) { 5722 return fastEmitInst_r(X86::VRCPPSYr, &X86::VR256RegClass, Op0); 5723 } 5724 return 0; 5725} 5726 5727unsigned fastEmit_X86ISD_FRCP_r(MVT VT, MVT RetVT, unsigned Op0) { 5728 switch (VT.SimpleTy) { 5729 case MVT::f32: return fastEmit_X86ISD_FRCP_MVT_f32_r(RetVT, Op0); 5730 case MVT::v4f32: return fastEmit_X86ISD_FRCP_MVT_v4f32_r(RetVT, Op0); 5731 case MVT::v8f32: return fastEmit_X86ISD_FRCP_MVT_v8f32_r(RetVT, Op0); 5732 default: return 0; 5733 } 5734} 5735 5736// FastEmit functions for X86ISD::FRSQRT. 5737 5738unsigned fastEmit_X86ISD_FRSQRT_MVT_f32_r(MVT RetVT, unsigned Op0) { 5739 if (RetVT.SimpleTy != MVT::f32) 5740 return 0; 5741 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 5742 return fastEmitInst_r(X86::RSQRTSSr, &X86::FR32RegClass, Op0); 5743 } 5744 return 0; 5745} 5746 5747unsigned fastEmit_X86ISD_FRSQRT_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 5748 if (RetVT.SimpleTy != MVT::v4f32) 5749 return 0; 5750 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 5751 return fastEmitInst_r(X86::RSQRTPSr, &X86::VR128RegClass, Op0); 5752 } 5753 if ((Subtarget->hasAVX())) { 5754 return fastEmitInst_r(X86::VRSQRTPSr, &X86::VR128RegClass, Op0); 5755 } 5756 return 0; 5757} 5758 5759unsigned fastEmit_X86ISD_FRSQRT_MVT_v8f32_r(MVT RetVT, unsigned Op0) { 5760 if (RetVT.SimpleTy != MVT::v8f32) 5761 return 0; 5762 if ((Subtarget->hasAVX())) { 5763 return fastEmitInst_r(X86::VRSQRTPSYr, &X86::VR256RegClass, Op0); 5764 } 5765 return 0; 5766} 5767 5768unsigned fastEmit_X86ISD_FRSQRT_r(MVT VT, MVT RetVT, unsigned Op0) { 5769 switch (VT.SimpleTy) { 5770 case MVT::f32: return fastEmit_X86ISD_FRSQRT_MVT_f32_r(RetVT, Op0); 5771 case MVT::v4f32: return fastEmit_X86ISD_FRSQRT_MVT_v4f32_r(RetVT, Op0); 5772 case MVT::v8f32: return fastEmit_X86ISD_FRSQRT_MVT_v8f32_r(RetVT, Op0); 5773 default: return 0; 5774 } 5775} 5776 5777// FastEmit functions for X86ISD::MMX_MOVD2W. 5778 5779unsigned fastEmit_X86ISD_MMX_MOVD2W_MVT_x86mmx_r(MVT RetVT, unsigned Op0) { 5780 if (RetVT.SimpleTy != MVT::i32) 5781 return 0; 5782 if ((Subtarget->hasMMX())) { 5783 return fastEmitInst_r(X86::MMX_MOVD64grr, &X86::GR32RegClass, Op0); 5784 } 5785 return 0; 5786} 5787 5788unsigned fastEmit_X86ISD_MMX_MOVD2W_r(MVT VT, MVT RetVT, unsigned Op0) { 5789 switch (VT.SimpleTy) { 5790 case MVT::x86mmx: return fastEmit_X86ISD_MMX_MOVD2W_MVT_x86mmx_r(RetVT, Op0); 5791 default: return 0; 5792 } 5793} 5794 5795// FastEmit functions for X86ISD::MMX_MOVW2D. 5796 5797unsigned fastEmit_X86ISD_MMX_MOVW2D_MVT_i32_r(MVT RetVT, unsigned Op0) { 5798 if (RetVT.SimpleTy != MVT::x86mmx) 5799 return 0; 5800 if ((Subtarget->hasMMX())) { 5801 return fastEmitInst_r(X86::MMX_MOVD64rr, &X86::VR64RegClass, Op0); 5802 } 5803 return 0; 5804} 5805 5806unsigned fastEmit_X86ISD_MMX_MOVW2D_r(MVT VT, MVT RetVT, unsigned Op0) { 5807 switch (VT.SimpleTy) { 5808 case MVT::i32: return fastEmit_X86ISD_MMX_MOVW2D_MVT_i32_r(RetVT, Op0); 5809 default: return 0; 5810 } 5811} 5812 5813// FastEmit functions for X86ISD::MOVDDUP. 5814 5815unsigned fastEmit_X86ISD_MOVDDUP_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 5816 if (RetVT.SimpleTy != MVT::v2f64) 5817 return 0; 5818 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) { 5819 return fastEmitInst_r(X86::MOVDDUPrr, &X86::VR128RegClass, Op0); 5820 } 5821 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 5822 return fastEmitInst_r(X86::VMOVDDUPrr, &X86::VR128RegClass, Op0); 5823 } 5824 return 0; 5825} 5826 5827unsigned fastEmit_X86ISD_MOVDDUP_MVT_v4f64_r(MVT RetVT, unsigned Op0) { 5828 if (RetVT.SimpleTy != MVT::v4f64) 5829 return 0; 5830 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 5831 return fastEmitInst_r(X86::VMOVDDUPZ256rr, &X86::VR256XRegClass, Op0); 5832 } 5833 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 5834 return fastEmitInst_r(X86::VMOVDDUPYrr, &X86::VR256RegClass, Op0); 5835 } 5836 return 0; 5837} 5838 5839unsigned fastEmit_X86ISD_MOVDDUP_MVT_v8f64_r(MVT RetVT, unsigned Op0) { 5840 if (RetVT.SimpleTy != MVT::v8f64) 5841 return 0; 5842 if ((Subtarget->hasAVX512())) { 5843 return fastEmitInst_r(X86::VMOVDDUPZrr, &X86::VR512RegClass, Op0); 5844 } 5845 return 0; 5846} 5847 5848unsigned fastEmit_X86ISD_MOVDDUP_r(MVT VT, MVT RetVT, unsigned Op0) { 5849 switch (VT.SimpleTy) { 5850 case MVT::v2f64: return fastEmit_X86ISD_MOVDDUP_MVT_v2f64_r(RetVT, Op0); 5851 case MVT::v4f64: return fastEmit_X86ISD_MOVDDUP_MVT_v4f64_r(RetVT, Op0); 5852 case MVT::v8f64: return fastEmit_X86ISD_MOVDDUP_MVT_v8f64_r(RetVT, Op0); 5853 default: return 0; 5854 } 5855} 5856 5857// FastEmit functions for X86ISD::MOVDQ2Q. 5858 5859unsigned fastEmit_X86ISD_MOVDQ2Q_MVT_v2i64_r(MVT RetVT, unsigned Op0) { 5860 if (RetVT.SimpleTy != MVT::x86mmx) 5861 return 0; 5862 if ((Subtarget->hasMMX()) && (Subtarget->hasSSE2())) { 5863 return fastEmitInst_r(X86::MMX_MOVDQ2Qrr, &X86::VR64RegClass, Op0); 5864 } 5865 return 0; 5866} 5867 5868unsigned fastEmit_X86ISD_MOVDQ2Q_r(MVT VT, MVT RetVT, unsigned Op0) { 5869 switch (VT.SimpleTy) { 5870 case MVT::v2i64: return fastEmit_X86ISD_MOVDQ2Q_MVT_v2i64_r(RetVT, Op0); 5871 default: return 0; 5872 } 5873} 5874 5875// FastEmit functions for X86ISD::MOVMSK. 5876 5877unsigned fastEmit_X86ISD_MOVMSK_MVT_v16i8_r(MVT RetVT, unsigned Op0) { 5878 if (RetVT.SimpleTy != MVT::i32) 5879 return 0; 5880 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 5881 return fastEmitInst_r(X86::PMOVMSKBrr, &X86::GR32RegClass, Op0); 5882 } 5883 if ((Subtarget->hasAVX())) { 5884 return fastEmitInst_r(X86::VPMOVMSKBrr, &X86::GR32RegClass, Op0); 5885 } 5886 return 0; 5887} 5888 5889unsigned fastEmit_X86ISD_MOVMSK_MVT_v32i8_r(MVT RetVT, unsigned Op0) { 5890 if (RetVT.SimpleTy != MVT::i32) 5891 return 0; 5892 if ((Subtarget->hasAVX2())) { 5893 return fastEmitInst_r(X86::VPMOVMSKBYrr, &X86::GR32RegClass, Op0); 5894 } 5895 return 0; 5896} 5897 5898unsigned fastEmit_X86ISD_MOVMSK_MVT_v4i32_r(MVT RetVT, unsigned Op0) { 5899 if (RetVT.SimpleTy != MVT::i32) 5900 return 0; 5901 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 5902 return fastEmitInst_r(X86::MOVMSKPSrr, &X86::GR32RegClass, Op0); 5903 } 5904 if ((Subtarget->hasAVX())) { 5905 return fastEmitInst_r(X86::VMOVMSKPSrr, &X86::GR32RegClass, Op0); 5906 } 5907 return 0; 5908} 5909 5910unsigned fastEmit_X86ISD_MOVMSK_MVT_v8i32_r(MVT RetVT, unsigned Op0) { 5911 if (RetVT.SimpleTy != MVT::i32) 5912 return 0; 5913 if ((Subtarget->hasAVX())) { 5914 return fastEmitInst_r(X86::VMOVMSKPSYrr, &X86::GR32RegClass, Op0); 5915 } 5916 return 0; 5917} 5918 5919unsigned fastEmit_X86ISD_MOVMSK_MVT_v2i64_r(MVT RetVT, unsigned Op0) { 5920 if (RetVT.SimpleTy != MVT::i32) 5921 return 0; 5922 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 5923 return fastEmitInst_r(X86::MOVMSKPDrr, &X86::GR32RegClass, Op0); 5924 } 5925 if ((Subtarget->hasAVX())) { 5926 return fastEmitInst_r(X86::VMOVMSKPDrr, &X86::GR32RegClass, Op0); 5927 } 5928 return 0; 5929} 5930 5931unsigned fastEmit_X86ISD_MOVMSK_MVT_v4i64_r(MVT RetVT, unsigned Op0) { 5932 if (RetVT.SimpleTy != MVT::i32) 5933 return 0; 5934 if ((Subtarget->hasAVX())) { 5935 return fastEmitInst_r(X86::VMOVMSKPDYrr, &X86::GR32RegClass, Op0); 5936 } 5937 return 0; 5938} 5939 5940unsigned fastEmit_X86ISD_MOVMSK_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 5941 if (RetVT.SimpleTy != MVT::i32) 5942 return 0; 5943 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 5944 return fastEmitInst_r(X86::MOVMSKPSrr, &X86::GR32RegClass, Op0); 5945 } 5946 if ((Subtarget->hasAVX())) { 5947 return fastEmitInst_r(X86::VMOVMSKPSrr, &X86::GR32RegClass, Op0); 5948 } 5949 return 0; 5950} 5951 5952unsigned fastEmit_X86ISD_MOVMSK_MVT_v8f32_r(MVT RetVT, unsigned Op0) { 5953 if (RetVT.SimpleTy != MVT::i32) 5954 return 0; 5955 if ((Subtarget->hasAVX())) { 5956 return fastEmitInst_r(X86::VMOVMSKPSYrr, &X86::GR32RegClass, Op0); 5957 } 5958 return 0; 5959} 5960 5961unsigned fastEmit_X86ISD_MOVMSK_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 5962 if (RetVT.SimpleTy != MVT::i32) 5963 return 0; 5964 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 5965 return fastEmitInst_r(X86::MOVMSKPDrr, &X86::GR32RegClass, Op0); 5966 } 5967 if ((Subtarget->hasAVX())) { 5968 return fastEmitInst_r(X86::VMOVMSKPDrr, &X86::GR32RegClass, Op0); 5969 } 5970 return 0; 5971} 5972 5973unsigned fastEmit_X86ISD_MOVMSK_MVT_v4f64_r(MVT RetVT, unsigned Op0) { 5974 if (RetVT.SimpleTy != MVT::i32) 5975 return 0; 5976 if ((Subtarget->hasAVX())) { 5977 return fastEmitInst_r(X86::VMOVMSKPDYrr, &X86::GR32RegClass, Op0); 5978 } 5979 return 0; 5980} 5981 5982unsigned fastEmit_X86ISD_MOVMSK_r(MVT VT, MVT RetVT, unsigned Op0) { 5983 switch (VT.SimpleTy) { 5984 case MVT::v16i8: return fastEmit_X86ISD_MOVMSK_MVT_v16i8_r(RetVT, Op0); 5985 case MVT::v32i8: return fastEmit_X86ISD_MOVMSK_MVT_v32i8_r(RetVT, Op0); 5986 case MVT::v4i32: return fastEmit_X86ISD_MOVMSK_MVT_v4i32_r(RetVT, Op0); 5987 case MVT::v8i32: return fastEmit_X86ISD_MOVMSK_MVT_v8i32_r(RetVT, Op0); 5988 case MVT::v2i64: return fastEmit_X86ISD_MOVMSK_MVT_v2i64_r(RetVT, Op0); 5989 case MVT::v4i64: return fastEmit_X86ISD_MOVMSK_MVT_v4i64_r(RetVT, Op0); 5990 case MVT::v4f32: return fastEmit_X86ISD_MOVMSK_MVT_v4f32_r(RetVT, Op0); 5991 case MVT::v8f32: return fastEmit_X86ISD_MOVMSK_MVT_v8f32_r(RetVT, Op0); 5992 case MVT::v2f64: return fastEmit_X86ISD_MOVMSK_MVT_v2f64_r(RetVT, Op0); 5993 case MVT::v4f64: return fastEmit_X86ISD_MOVMSK_MVT_v4f64_r(RetVT, Op0); 5994 default: return 0; 5995 } 5996} 5997 5998// FastEmit functions for X86ISD::MOVQ2DQ. 5999 6000unsigned fastEmit_X86ISD_MOVQ2DQ_MVT_x86mmx_r(MVT RetVT, unsigned Op0) { 6001 if (RetVT.SimpleTy != MVT::v2i64) 6002 return 0; 6003 if ((Subtarget->hasMMX()) && (Subtarget->hasSSE2())) { 6004 return fastEmitInst_r(X86::MMX_MOVQ2DQrr, &X86::VR128RegClass, Op0); 6005 } 6006 return 0; 6007} 6008 6009unsigned fastEmit_X86ISD_MOVQ2DQ_r(MVT VT, MVT RetVT, unsigned Op0) { 6010 switch (VT.SimpleTy) { 6011 case MVT::x86mmx: return fastEmit_X86ISD_MOVQ2DQ_MVT_x86mmx_r(RetVT, Op0); 6012 default: return 0; 6013 } 6014} 6015 6016// FastEmit functions for X86ISD::MOVSHDUP. 6017 6018unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v4i32_r(MVT RetVT, unsigned Op0) { 6019 if (RetVT.SimpleTy != MVT::v4i32) 6020 return 0; 6021 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) { 6022 return fastEmitInst_r(X86::MOVSHDUPrr, &X86::VR128RegClass, Op0); 6023 } 6024 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 6025 return fastEmitInst_r(X86::VMOVSHDUPrr, &X86::VR128RegClass, Op0); 6026 } 6027 return 0; 6028} 6029 6030unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v8i32_r(MVT RetVT, unsigned Op0) { 6031 if (RetVT.SimpleTy != MVT::v8i32) 6032 return 0; 6033 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 6034 return fastEmitInst_r(X86::VMOVSHDUPYrr, &X86::VR256RegClass, Op0); 6035 } 6036 return 0; 6037} 6038 6039unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 6040 if (RetVT.SimpleTy != MVT::v4f32) 6041 return 0; 6042 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 6043 return fastEmitInst_r(X86::VMOVSHDUPZ128rr, &X86::VR128XRegClass, Op0); 6044 } 6045 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) { 6046 return fastEmitInst_r(X86::MOVSHDUPrr, &X86::VR128RegClass, Op0); 6047 } 6048 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 6049 return fastEmitInst_r(X86::VMOVSHDUPrr, &X86::VR128RegClass, Op0); 6050 } 6051 return 0; 6052} 6053 6054unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v8f32_r(MVT RetVT, unsigned Op0) { 6055 if (RetVT.SimpleTy != MVT::v8f32) 6056 return 0; 6057 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 6058 return fastEmitInst_r(X86::VMOVSHDUPZ256rr, &X86::VR256XRegClass, Op0); 6059 } 6060 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 6061 return fastEmitInst_r(X86::VMOVSHDUPYrr, &X86::VR256RegClass, Op0); 6062 } 6063 return 0; 6064} 6065 6066unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v16f32_r(MVT RetVT, unsigned Op0) { 6067 if (RetVT.SimpleTy != MVT::v16f32) 6068 return 0; 6069 if ((Subtarget->hasAVX512())) { 6070 return fastEmitInst_r(X86::VMOVSHDUPZrr, &X86::VR512RegClass, Op0); 6071 } 6072 return 0; 6073} 6074 6075unsigned fastEmit_X86ISD_MOVSHDUP_r(MVT VT, MVT RetVT, unsigned Op0) { 6076 switch (VT.SimpleTy) { 6077 case MVT::v4i32: return fastEmit_X86ISD_MOVSHDUP_MVT_v4i32_r(RetVT, Op0); 6078 case MVT::v8i32: return fastEmit_X86ISD_MOVSHDUP_MVT_v8i32_r(RetVT, Op0); 6079 case MVT::v4f32: return fastEmit_X86ISD_MOVSHDUP_MVT_v4f32_r(RetVT, Op0); 6080 case MVT::v8f32: return fastEmit_X86ISD_MOVSHDUP_MVT_v8f32_r(RetVT, Op0); 6081 case MVT::v16f32: return fastEmit_X86ISD_MOVSHDUP_MVT_v16f32_r(RetVT, Op0); 6082 default: return 0; 6083 } 6084} 6085 6086// FastEmit functions for X86ISD::MOVSLDUP. 6087 6088unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v4i32_r(MVT RetVT, unsigned Op0) { 6089 if (RetVT.SimpleTy != MVT::v4i32) 6090 return 0; 6091 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) { 6092 return fastEmitInst_r(X86::MOVSLDUPrr, &X86::VR128RegClass, Op0); 6093 } 6094 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 6095 return fastEmitInst_r(X86::VMOVSLDUPrr, &X86::VR128RegClass, Op0); 6096 } 6097 return 0; 6098} 6099 6100unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v8i32_r(MVT RetVT, unsigned Op0) { 6101 if (RetVT.SimpleTy != MVT::v8i32) 6102 return 0; 6103 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 6104 return fastEmitInst_r(X86::VMOVSLDUPYrr, &X86::VR256RegClass, Op0); 6105 } 6106 return 0; 6107} 6108 6109unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 6110 if (RetVT.SimpleTy != MVT::v4f32) 6111 return 0; 6112 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 6113 return fastEmitInst_r(X86::VMOVSLDUPZ128rr, &X86::VR128XRegClass, Op0); 6114 } 6115 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) { 6116 return fastEmitInst_r(X86::MOVSLDUPrr, &X86::VR128RegClass, Op0); 6117 } 6118 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 6119 return fastEmitInst_r(X86::VMOVSLDUPrr, &X86::VR128RegClass, Op0); 6120 } 6121 return 0; 6122} 6123 6124unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v8f32_r(MVT RetVT, unsigned Op0) { 6125 if (RetVT.SimpleTy != MVT::v8f32) 6126 return 0; 6127 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 6128 return fastEmitInst_r(X86::VMOVSLDUPZ256rr, &X86::VR256XRegClass, Op0); 6129 } 6130 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 6131 return fastEmitInst_r(X86::VMOVSLDUPYrr, &X86::VR256RegClass, Op0); 6132 } 6133 return 0; 6134} 6135 6136unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v16f32_r(MVT RetVT, unsigned Op0) { 6137 if (RetVT.SimpleTy != MVT::v16f32) 6138 return 0; 6139 if ((Subtarget->hasAVX512())) { 6140 return fastEmitInst_r(X86::VMOVSLDUPZrr, &X86::VR512RegClass, Op0); 6141 } 6142 return 0; 6143} 6144 6145unsigned fastEmit_X86ISD_MOVSLDUP_r(MVT VT, MVT RetVT, unsigned Op0) { 6146 switch (VT.SimpleTy) { 6147 case MVT::v4i32: return fastEmit_X86ISD_MOVSLDUP_MVT_v4i32_r(RetVT, Op0); 6148 case MVT::v8i32: return fastEmit_X86ISD_MOVSLDUP_MVT_v8i32_r(RetVT, Op0); 6149 case MVT::v4f32: return fastEmit_X86ISD_MOVSLDUP_MVT_v4f32_r(RetVT, Op0); 6150 case MVT::v8f32: return fastEmit_X86ISD_MOVSLDUP_MVT_v8f32_r(RetVT, Op0); 6151 case MVT::v16f32: return fastEmit_X86ISD_MOVSLDUP_MVT_v16f32_r(RetVT, Op0); 6152 default: return 0; 6153 } 6154} 6155 6156// FastEmit functions for X86ISD::NT_BRIND. 6157 6158unsigned fastEmit_X86ISD_NT_BRIND_MVT_i16_r(MVT RetVT, unsigned Op0) { 6159 if (RetVT.SimpleTy != MVT::isVoid) 6160 return 0; 6161 if ((!Subtarget->is64Bit())) { 6162 return fastEmitInst_r(X86::JMP16r_NT, &X86::GR16RegClass, Op0); 6163 } 6164 return 0; 6165} 6166 6167unsigned fastEmit_X86ISD_NT_BRIND_MVT_i32_r(MVT RetVT, unsigned Op0) { 6168 if (RetVT.SimpleTy != MVT::isVoid) 6169 return 0; 6170 if ((!Subtarget->is64Bit())) { 6171 return fastEmitInst_r(X86::JMP32r_NT, &X86::GR32RegClass, Op0); 6172 } 6173 return 0; 6174} 6175 6176unsigned fastEmit_X86ISD_NT_BRIND_MVT_i64_r(MVT RetVT, unsigned Op0) { 6177 if (RetVT.SimpleTy != MVT::isVoid) 6178 return 0; 6179 if ((Subtarget->is64Bit())) { 6180 return fastEmitInst_r(X86::JMP64r_NT, &X86::GR64RegClass, Op0); 6181 } 6182 return 0; 6183} 6184 6185unsigned fastEmit_X86ISD_NT_BRIND_r(MVT VT, MVT RetVT, unsigned Op0) { 6186 switch (VT.SimpleTy) { 6187 case MVT::i16: return fastEmit_X86ISD_NT_BRIND_MVT_i16_r(RetVT, Op0); 6188 case MVT::i32: return fastEmit_X86ISD_NT_BRIND_MVT_i32_r(RetVT, Op0); 6189 case MVT::i64: return fastEmit_X86ISD_NT_BRIND_MVT_i64_r(RetVT, Op0); 6190 default: return 0; 6191 } 6192} 6193 6194// FastEmit functions for X86ISD::NT_CALL. 6195 6196unsigned fastEmit_X86ISD_NT_CALL_MVT_i16_r(MVT RetVT, unsigned Op0) { 6197 if (RetVT.SimpleTy != MVT::isVoid) 6198 return 0; 6199 if ((!Subtarget->is64Bit())) { 6200 return fastEmitInst_r(X86::CALL16r_NT, &X86::GR16RegClass, Op0); 6201 } 6202 return 0; 6203} 6204 6205unsigned fastEmit_X86ISD_NT_CALL_MVT_i32_r(MVT RetVT, unsigned Op0) { 6206 if (RetVT.SimpleTy != MVT::isVoid) 6207 return 0; 6208 if ((!Subtarget->is64Bit())) { 6209 return fastEmitInst_r(X86::CALL32r_NT, &X86::GR32RegClass, Op0); 6210 } 6211 return 0; 6212} 6213 6214unsigned fastEmit_X86ISD_NT_CALL_MVT_i64_r(MVT RetVT, unsigned Op0) { 6215 if (RetVT.SimpleTy != MVT::isVoid) 6216 return 0; 6217 if ((Subtarget->is64Bit())) { 6218 return fastEmitInst_r(X86::CALL64r_NT, &X86::GR64RegClass, Op0); 6219 } 6220 return 0; 6221} 6222 6223unsigned fastEmit_X86ISD_NT_CALL_r(MVT VT, MVT RetVT, unsigned Op0) { 6224 switch (VT.SimpleTy) { 6225 case MVT::i16: return fastEmit_X86ISD_NT_CALL_MVT_i16_r(RetVT, Op0); 6226 case MVT::i32: return fastEmit_X86ISD_NT_CALL_MVT_i32_r(RetVT, Op0); 6227 case MVT::i64: return fastEmit_X86ISD_NT_CALL_MVT_i64_r(RetVT, Op0); 6228 default: return 0; 6229 } 6230} 6231 6232// FastEmit functions for X86ISD::PHMINPOS. 6233 6234unsigned fastEmit_X86ISD_PHMINPOS_MVT_v8i16_r(MVT RetVT, unsigned Op0) { 6235 if (RetVT.SimpleTy != MVT::v8i16) 6236 return 0; 6237 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 6238 return fastEmitInst_r(X86::PHMINPOSUWrr, &X86::VR128RegClass, Op0); 6239 } 6240 if ((Subtarget->hasAVX())) { 6241 return fastEmitInst_r(X86::VPHMINPOSUWrr, &X86::VR128RegClass, Op0); 6242 } 6243 return 0; 6244} 6245 6246unsigned fastEmit_X86ISD_PHMINPOS_r(MVT VT, MVT RetVT, unsigned Op0) { 6247 switch (VT.SimpleTy) { 6248 case MVT::v8i16: return fastEmit_X86ISD_PHMINPOS_MVT_v8i16_r(RetVT, Op0); 6249 default: return 0; 6250 } 6251} 6252 6253// FastEmit functions for X86ISD::PROBED_ALLOCA. 6254 6255unsigned fastEmit_X86ISD_PROBED_ALLOCA_MVT_i32_r(MVT RetVT, unsigned Op0) { 6256 if (RetVT.SimpleTy != MVT::i32) 6257 return 0; 6258 if ((!Subtarget->isTarget64BitLP64())) { 6259 return fastEmitInst_r(X86::PROBED_ALLOCA_32, &X86::GR32RegClass, Op0); 6260 } 6261 return 0; 6262} 6263 6264unsigned fastEmit_X86ISD_PROBED_ALLOCA_MVT_i64_r(MVT RetVT, unsigned Op0) { 6265 if (RetVT.SimpleTy != MVT::i64) 6266 return 0; 6267 if ((Subtarget->is64Bit())) { 6268 return fastEmitInst_r(X86::PROBED_ALLOCA_64, &X86::GR64RegClass, Op0); 6269 } 6270 return 0; 6271} 6272 6273unsigned fastEmit_X86ISD_PROBED_ALLOCA_r(MVT VT, MVT RetVT, unsigned Op0) { 6274 switch (VT.SimpleTy) { 6275 case MVT::i32: return fastEmit_X86ISD_PROBED_ALLOCA_MVT_i32_r(RetVT, Op0); 6276 case MVT::i64: return fastEmit_X86ISD_PROBED_ALLOCA_MVT_i64_r(RetVT, Op0); 6277 default: return 0; 6278 } 6279} 6280 6281// FastEmit functions for X86ISD::RCP14. 6282 6283unsigned fastEmit_X86ISD_RCP14_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 6284 if (RetVT.SimpleTy != MVT::v8f16) 6285 return 0; 6286 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 6287 return fastEmitInst_r(X86::VRCPPHZ128r, &X86::VR128XRegClass, Op0); 6288 } 6289 return 0; 6290} 6291 6292unsigned fastEmit_X86ISD_RCP14_MVT_v16f16_r(MVT RetVT, unsigned Op0) { 6293 if (RetVT.SimpleTy != MVT::v16f16) 6294 return 0; 6295 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 6296 return fastEmitInst_r(X86::VRCPPHZ256r, &X86::VR256XRegClass, Op0); 6297 } 6298 return 0; 6299} 6300 6301unsigned fastEmit_X86ISD_RCP14_MVT_v32f16_r(MVT RetVT, unsigned Op0) { 6302 if (RetVT.SimpleTy != MVT::v32f16) 6303 return 0; 6304 if ((Subtarget->hasFP16())) { 6305 return fastEmitInst_r(X86::VRCPPHZr, &X86::VR512RegClass, Op0); 6306 } 6307 return 0; 6308} 6309 6310unsigned fastEmit_X86ISD_RCP14_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 6311 if (RetVT.SimpleTy != MVT::v4f32) 6312 return 0; 6313 if ((Subtarget->hasVLX())) { 6314 return fastEmitInst_r(X86::VRCP14PSZ128r, &X86::VR128XRegClass, Op0); 6315 } 6316 return 0; 6317} 6318 6319unsigned fastEmit_X86ISD_RCP14_MVT_v8f32_r(MVT RetVT, unsigned Op0) { 6320 if (RetVT.SimpleTy != MVT::v8f32) 6321 return 0; 6322 if ((Subtarget->hasVLX())) { 6323 return fastEmitInst_r(X86::VRCP14PSZ256r, &X86::VR256XRegClass, Op0); 6324 } 6325 return 0; 6326} 6327 6328unsigned fastEmit_X86ISD_RCP14_MVT_v16f32_r(MVT RetVT, unsigned Op0) { 6329 if (RetVT.SimpleTy != MVT::v16f32) 6330 return 0; 6331 if ((Subtarget->hasAVX512())) { 6332 return fastEmitInst_r(X86::VRCP14PSZr, &X86::VR512RegClass, Op0); 6333 } 6334 return 0; 6335} 6336 6337unsigned fastEmit_X86ISD_RCP14_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 6338 if (RetVT.SimpleTy != MVT::v2f64) 6339 return 0; 6340 if ((Subtarget->hasVLX())) { 6341 return fastEmitInst_r(X86::VRCP14PDZ128r, &X86::VR128XRegClass, Op0); 6342 } 6343 return 0; 6344} 6345 6346unsigned fastEmit_X86ISD_RCP14_MVT_v4f64_r(MVT RetVT, unsigned Op0) { 6347 if (RetVT.SimpleTy != MVT::v4f64) 6348 return 0; 6349 if ((Subtarget->hasVLX())) { 6350 return fastEmitInst_r(X86::VRCP14PDZ256r, &X86::VR256XRegClass, Op0); 6351 } 6352 return 0; 6353} 6354 6355unsigned fastEmit_X86ISD_RCP14_MVT_v8f64_r(MVT RetVT, unsigned Op0) { 6356 if (RetVT.SimpleTy != MVT::v8f64) 6357 return 0; 6358 if ((Subtarget->hasAVX512())) { 6359 return fastEmitInst_r(X86::VRCP14PDZr, &X86::VR512RegClass, Op0); 6360 } 6361 return 0; 6362} 6363 6364unsigned fastEmit_X86ISD_RCP14_r(MVT VT, MVT RetVT, unsigned Op0) { 6365 switch (VT.SimpleTy) { 6366 case MVT::v8f16: return fastEmit_X86ISD_RCP14_MVT_v8f16_r(RetVT, Op0); 6367 case MVT::v16f16: return fastEmit_X86ISD_RCP14_MVT_v16f16_r(RetVT, Op0); 6368 case MVT::v32f16: return fastEmit_X86ISD_RCP14_MVT_v32f16_r(RetVT, Op0); 6369 case MVT::v4f32: return fastEmit_X86ISD_RCP14_MVT_v4f32_r(RetVT, Op0); 6370 case MVT::v8f32: return fastEmit_X86ISD_RCP14_MVT_v8f32_r(RetVT, Op0); 6371 case MVT::v16f32: return fastEmit_X86ISD_RCP14_MVT_v16f32_r(RetVT, Op0); 6372 case MVT::v2f64: return fastEmit_X86ISD_RCP14_MVT_v2f64_r(RetVT, Op0); 6373 case MVT::v4f64: return fastEmit_X86ISD_RCP14_MVT_v4f64_r(RetVT, Op0); 6374 case MVT::v8f64: return fastEmit_X86ISD_RCP14_MVT_v8f64_r(RetVT, Op0); 6375 default: return 0; 6376 } 6377} 6378 6379// FastEmit functions for X86ISD::RCP28. 6380 6381unsigned fastEmit_X86ISD_RCP28_MVT_v16f32_r(MVT RetVT, unsigned Op0) { 6382 if (RetVT.SimpleTy != MVT::v16f32) 6383 return 0; 6384 if ((Subtarget->hasERI())) { 6385 return fastEmitInst_r(X86::VRCP28PSZr, &X86::VR512RegClass, Op0); 6386 } 6387 return 0; 6388} 6389 6390unsigned fastEmit_X86ISD_RCP28_MVT_v8f64_r(MVT RetVT, unsigned Op0) { 6391 if (RetVT.SimpleTy != MVT::v8f64) 6392 return 0; 6393 if ((Subtarget->hasERI())) { 6394 return fastEmitInst_r(X86::VRCP28PDZr, &X86::VR512RegClass, Op0); 6395 } 6396 return 0; 6397} 6398 6399unsigned fastEmit_X86ISD_RCP28_r(MVT VT, MVT RetVT, unsigned Op0) { 6400 switch (VT.SimpleTy) { 6401 case MVT::v16f32: return fastEmit_X86ISD_RCP28_MVT_v16f32_r(RetVT, Op0); 6402 case MVT::v8f64: return fastEmit_X86ISD_RCP28_MVT_v8f64_r(RetVT, Op0); 6403 default: return 0; 6404 } 6405} 6406 6407// FastEmit functions for X86ISD::RCP28_SAE. 6408 6409unsigned fastEmit_X86ISD_RCP28_SAE_MVT_v16f32_r(MVT RetVT, unsigned Op0) { 6410 if (RetVT.SimpleTy != MVT::v16f32) 6411 return 0; 6412 if ((Subtarget->hasERI())) { 6413 return fastEmitInst_r(X86::VRCP28PSZrb, &X86::VR512RegClass, Op0); 6414 } 6415 return 0; 6416} 6417 6418unsigned fastEmit_X86ISD_RCP28_SAE_MVT_v8f64_r(MVT RetVT, unsigned Op0) { 6419 if (RetVT.SimpleTy != MVT::v8f64) 6420 return 0; 6421 if ((Subtarget->hasERI())) { 6422 return fastEmitInst_r(X86::VRCP28PDZrb, &X86::VR512RegClass, Op0); 6423 } 6424 return 0; 6425} 6426 6427unsigned fastEmit_X86ISD_RCP28_SAE_r(MVT VT, MVT RetVT, unsigned Op0) { 6428 switch (VT.SimpleTy) { 6429 case MVT::v16f32: return fastEmit_X86ISD_RCP28_SAE_MVT_v16f32_r(RetVT, Op0); 6430 case MVT::v8f64: return fastEmit_X86ISD_RCP28_SAE_MVT_v8f64_r(RetVT, Op0); 6431 default: return 0; 6432 } 6433} 6434 6435// FastEmit functions for X86ISD::RSQRT14. 6436 6437unsigned fastEmit_X86ISD_RSQRT14_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 6438 if (RetVT.SimpleTy != MVT::v8f16) 6439 return 0; 6440 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 6441 return fastEmitInst_r(X86::VRSQRTPHZ128r, &X86::VR128XRegClass, Op0); 6442 } 6443 return 0; 6444} 6445 6446unsigned fastEmit_X86ISD_RSQRT14_MVT_v16f16_r(MVT RetVT, unsigned Op0) { 6447 if (RetVT.SimpleTy != MVT::v16f16) 6448 return 0; 6449 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 6450 return fastEmitInst_r(X86::VRSQRTPHZ256r, &X86::VR256XRegClass, Op0); 6451 } 6452 return 0; 6453} 6454 6455unsigned fastEmit_X86ISD_RSQRT14_MVT_v32f16_r(MVT RetVT, unsigned Op0) { 6456 if (RetVT.SimpleTy != MVT::v32f16) 6457 return 0; 6458 if ((Subtarget->hasFP16())) { 6459 return fastEmitInst_r(X86::VRSQRTPHZr, &X86::VR512RegClass, Op0); 6460 } 6461 return 0; 6462} 6463 6464unsigned fastEmit_X86ISD_RSQRT14_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 6465 if (RetVT.SimpleTy != MVT::v4f32) 6466 return 0; 6467 if ((Subtarget->hasVLX())) { 6468 return fastEmitInst_r(X86::VRSQRT14PSZ128r, &X86::VR128XRegClass, Op0); 6469 } 6470 return 0; 6471} 6472 6473unsigned fastEmit_X86ISD_RSQRT14_MVT_v8f32_r(MVT RetVT, unsigned Op0) { 6474 if (RetVT.SimpleTy != MVT::v8f32) 6475 return 0; 6476 if ((Subtarget->hasVLX())) { 6477 return fastEmitInst_r(X86::VRSQRT14PSZ256r, &X86::VR256XRegClass, Op0); 6478 } 6479 return 0; 6480} 6481 6482unsigned fastEmit_X86ISD_RSQRT14_MVT_v16f32_r(MVT RetVT, unsigned Op0) { 6483 if (RetVT.SimpleTy != MVT::v16f32) 6484 return 0; 6485 if ((Subtarget->hasAVX512())) { 6486 return fastEmitInst_r(X86::VRSQRT14PSZr, &X86::VR512RegClass, Op0); 6487 } 6488 return 0; 6489} 6490 6491unsigned fastEmit_X86ISD_RSQRT14_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 6492 if (RetVT.SimpleTy != MVT::v2f64) 6493 return 0; 6494 if ((Subtarget->hasVLX())) { 6495 return fastEmitInst_r(X86::VRSQRT14PDZ128r, &X86::VR128XRegClass, Op0); 6496 } 6497 return 0; 6498} 6499 6500unsigned fastEmit_X86ISD_RSQRT14_MVT_v4f64_r(MVT RetVT, unsigned Op0) { 6501 if (RetVT.SimpleTy != MVT::v4f64) 6502 return 0; 6503 if ((Subtarget->hasVLX())) { 6504 return fastEmitInst_r(X86::VRSQRT14PDZ256r, &X86::VR256XRegClass, Op0); 6505 } 6506 return 0; 6507} 6508 6509unsigned fastEmit_X86ISD_RSQRT14_MVT_v8f64_r(MVT RetVT, unsigned Op0) { 6510 if (RetVT.SimpleTy != MVT::v8f64) 6511 return 0; 6512 if ((Subtarget->hasAVX512())) { 6513 return fastEmitInst_r(X86::VRSQRT14PDZr, &X86::VR512RegClass, Op0); 6514 } 6515 return 0; 6516} 6517 6518unsigned fastEmit_X86ISD_RSQRT14_r(MVT VT, MVT RetVT, unsigned Op0) { 6519 switch (VT.SimpleTy) { 6520 case MVT::v8f16: return fastEmit_X86ISD_RSQRT14_MVT_v8f16_r(RetVT, Op0); 6521 case MVT::v16f16: return fastEmit_X86ISD_RSQRT14_MVT_v16f16_r(RetVT, Op0); 6522 case MVT::v32f16: return fastEmit_X86ISD_RSQRT14_MVT_v32f16_r(RetVT, Op0); 6523 case MVT::v4f32: return fastEmit_X86ISD_RSQRT14_MVT_v4f32_r(RetVT, Op0); 6524 case MVT::v8f32: return fastEmit_X86ISD_RSQRT14_MVT_v8f32_r(RetVT, Op0); 6525 case MVT::v16f32: return fastEmit_X86ISD_RSQRT14_MVT_v16f32_r(RetVT, Op0); 6526 case MVT::v2f64: return fastEmit_X86ISD_RSQRT14_MVT_v2f64_r(RetVT, Op0); 6527 case MVT::v4f64: return fastEmit_X86ISD_RSQRT14_MVT_v4f64_r(RetVT, Op0); 6528 case MVT::v8f64: return fastEmit_X86ISD_RSQRT14_MVT_v8f64_r(RetVT, Op0); 6529 default: return 0; 6530 } 6531} 6532 6533// FastEmit functions for X86ISD::RSQRT28. 6534 6535unsigned fastEmit_X86ISD_RSQRT28_MVT_v16f32_r(MVT RetVT, unsigned Op0) { 6536 if (RetVT.SimpleTy != MVT::v16f32) 6537 return 0; 6538 if ((Subtarget->hasERI())) { 6539 return fastEmitInst_r(X86::VRSQRT28PSZr, &X86::VR512RegClass, Op0); 6540 } 6541 return 0; 6542} 6543 6544unsigned fastEmit_X86ISD_RSQRT28_MVT_v8f64_r(MVT RetVT, unsigned Op0) { 6545 if (RetVT.SimpleTy != MVT::v8f64) 6546 return 0; 6547 if ((Subtarget->hasERI())) { 6548 return fastEmitInst_r(X86::VRSQRT28PDZr, &X86::VR512RegClass, Op0); 6549 } 6550 return 0; 6551} 6552 6553unsigned fastEmit_X86ISD_RSQRT28_r(MVT VT, MVT RetVT, unsigned Op0) { 6554 switch (VT.SimpleTy) { 6555 case MVT::v16f32: return fastEmit_X86ISD_RSQRT28_MVT_v16f32_r(RetVT, Op0); 6556 case MVT::v8f64: return fastEmit_X86ISD_RSQRT28_MVT_v8f64_r(RetVT, Op0); 6557 default: return 0; 6558 } 6559} 6560 6561// FastEmit functions for X86ISD::RSQRT28_SAE. 6562 6563unsigned fastEmit_X86ISD_RSQRT28_SAE_MVT_v16f32_r(MVT RetVT, unsigned Op0) { 6564 if (RetVT.SimpleTy != MVT::v16f32) 6565 return 0; 6566 if ((Subtarget->hasERI())) { 6567 return fastEmitInst_r(X86::VRSQRT28PSZrb, &X86::VR512RegClass, Op0); 6568 } 6569 return 0; 6570} 6571 6572unsigned fastEmit_X86ISD_RSQRT28_SAE_MVT_v8f64_r(MVT RetVT, unsigned Op0) { 6573 if (RetVT.SimpleTy != MVT::v8f64) 6574 return 0; 6575 if ((Subtarget->hasERI())) { 6576 return fastEmitInst_r(X86::VRSQRT28PDZrb, &X86::VR512RegClass, Op0); 6577 } 6578 return 0; 6579} 6580 6581unsigned fastEmit_X86ISD_RSQRT28_SAE_r(MVT VT, MVT RetVT, unsigned Op0) { 6582 switch (VT.SimpleTy) { 6583 case MVT::v16f32: return fastEmit_X86ISD_RSQRT28_SAE_MVT_v16f32_r(RetVT, Op0); 6584 case MVT::v8f64: return fastEmit_X86ISD_RSQRT28_SAE_MVT_v8f64_r(RetVT, Op0); 6585 default: return 0; 6586 } 6587} 6588 6589// FastEmit functions for X86ISD::SEG_ALLOCA. 6590 6591unsigned fastEmit_X86ISD_SEG_ALLOCA_MVT_i32_r(MVT RetVT, unsigned Op0) { 6592 if (RetVT.SimpleTy != MVT::i32) 6593 return 0; 6594 if ((!Subtarget->isTarget64BitLP64())) { 6595 return fastEmitInst_r(X86::SEG_ALLOCA_32, &X86::GR32RegClass, Op0); 6596 } 6597 return 0; 6598} 6599 6600unsigned fastEmit_X86ISD_SEG_ALLOCA_MVT_i64_r(MVT RetVT, unsigned Op0) { 6601 if (RetVT.SimpleTy != MVT::i64) 6602 return 0; 6603 if ((Subtarget->is64Bit())) { 6604 return fastEmitInst_r(X86::SEG_ALLOCA_64, &X86::GR64RegClass, Op0); 6605 } 6606 return 0; 6607} 6608 6609unsigned fastEmit_X86ISD_SEG_ALLOCA_r(MVT VT, MVT RetVT, unsigned Op0) { 6610 switch (VT.SimpleTy) { 6611 case MVT::i32: return fastEmit_X86ISD_SEG_ALLOCA_MVT_i32_r(RetVT, Op0); 6612 case MVT::i64: return fastEmit_X86ISD_SEG_ALLOCA_MVT_i64_r(RetVT, Op0); 6613 default: return 0; 6614 } 6615} 6616 6617// FastEmit functions for X86ISD::STRICT_CVTPH2PS. 6618 6619unsigned fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_MVT_v4f32_r(unsigned Op0) { 6620 if ((Subtarget->hasVLX())) { 6621 return fastEmitInst_r(X86::VCVTPH2PSZ128rr, &X86::VR128XRegClass, Op0); 6622 } 6623 if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) { 6624 return fastEmitInst_r(X86::VCVTPH2PSrr, &X86::VR128RegClass, Op0); 6625 } 6626 return 0; 6627} 6628 6629unsigned fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_MVT_v8f32_r(unsigned Op0) { 6630 if ((Subtarget->hasVLX())) { 6631 return fastEmitInst_r(X86::VCVTPH2PSZ256rr, &X86::VR256XRegClass, Op0); 6632 } 6633 if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) { 6634 return fastEmitInst_r(X86::VCVTPH2PSYrr, &X86::VR256RegClass, Op0); 6635 } 6636 return 0; 6637} 6638 6639unsigned fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_r(MVT RetVT, unsigned Op0) { 6640switch (RetVT.SimpleTy) { 6641 case MVT::v4f32: return fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_MVT_v4f32_r(Op0); 6642 case MVT::v8f32: return fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_MVT_v8f32_r(Op0); 6643 default: return 0; 6644} 6645} 6646 6647unsigned fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v16i16_r(MVT RetVT, unsigned Op0) { 6648 if (RetVT.SimpleTy != MVT::v16f32) 6649 return 0; 6650 if ((Subtarget->hasAVX512())) { 6651 return fastEmitInst_r(X86::VCVTPH2PSZrr, &X86::VR512RegClass, Op0); 6652 } 6653 return 0; 6654} 6655 6656unsigned fastEmit_X86ISD_STRICT_CVTPH2PS_r(MVT VT, MVT RetVT, unsigned Op0) { 6657 switch (VT.SimpleTy) { 6658 case MVT::v8i16: return fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_r(RetVT, Op0); 6659 case MVT::v16i16: return fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v16i16_r(RetVT, Op0); 6660 default: return 0; 6661 } 6662} 6663 6664// FastEmit functions for X86ISD::STRICT_CVTSI2P. 6665 6666unsigned fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_MVT_v8f16_r(unsigned Op0) { 6667 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 6668 return fastEmitInst_r(X86::VCVTDQ2PHZ128rr, &X86::VR128XRegClass, Op0); 6669 } 6670 return 0; 6671} 6672 6673unsigned fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_MVT_v2f64_r(unsigned Op0) { 6674 if ((Subtarget->hasVLX())) { 6675 return fastEmitInst_r(X86::VCVTDQ2PDZ128rr, &X86::VR128XRegClass, Op0); 6676 } 6677 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 6678 return fastEmitInst_r(X86::CVTDQ2PDrr, &X86::VR128RegClass, Op0); 6679 } 6680 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 6681 return fastEmitInst_r(X86::VCVTDQ2PDrr, &X86::VR128RegClass, Op0); 6682 } 6683 return 0; 6684} 6685 6686unsigned fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_r(MVT RetVT, unsigned Op0) { 6687switch (RetVT.SimpleTy) { 6688 case MVT::v8f16: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_MVT_v8f16_r(Op0); 6689 case MVT::v2f64: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_MVT_v2f64_r(Op0); 6690 default: return 0; 6691} 6692} 6693 6694unsigned fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_MVT_v8f16_r(unsigned Op0) { 6695 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 6696 return fastEmitInst_r(X86::VCVTQQ2PHZ128rr, &X86::VR128XRegClass, Op0); 6697 } 6698 return 0; 6699} 6700 6701unsigned fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_MVT_v4f32_r(unsigned Op0) { 6702 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 6703 return fastEmitInst_r(X86::VCVTQQ2PSZ128rr, &X86::VR128XRegClass, Op0); 6704 } 6705 return 0; 6706} 6707 6708unsigned fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_r(MVT RetVT, unsigned Op0) { 6709switch (RetVT.SimpleTy) { 6710 case MVT::v8f16: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_MVT_v8f16_r(Op0); 6711 case MVT::v4f32: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_MVT_v4f32_r(Op0); 6712 default: return 0; 6713} 6714} 6715 6716unsigned fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i64_r(MVT RetVT, unsigned Op0) { 6717 if (RetVT.SimpleTy != MVT::v8f16) 6718 return 0; 6719 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 6720 return fastEmitInst_r(X86::VCVTQQ2PHZ256rr, &X86::VR128XRegClass, Op0); 6721 } 6722 return 0; 6723} 6724 6725unsigned fastEmit_X86ISD_STRICT_CVTSI2P_r(MVT VT, MVT RetVT, unsigned Op0) { 6726 switch (VT.SimpleTy) { 6727 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_r(RetVT, Op0); 6728 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_r(RetVT, Op0); 6729 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i64_r(RetVT, Op0); 6730 default: return 0; 6731 } 6732} 6733 6734// FastEmit functions for X86ISD::STRICT_CVTTP2SI. 6735 6736unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v8i16_r(unsigned Op0) { 6737 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 6738 return fastEmitInst_r(X86::VCVTTPH2WZ128rr, &X86::VR128XRegClass, Op0); 6739 } 6740 return 0; 6741} 6742 6743unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v4i32_r(unsigned Op0) { 6744 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 6745 return fastEmitInst_r(X86::VCVTTPH2DQZ128rr, &X86::VR128XRegClass, Op0); 6746 } 6747 return 0; 6748} 6749 6750unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v8i32_r(unsigned Op0) { 6751 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 6752 return fastEmitInst_r(X86::VCVTTPH2DQZ256rr, &X86::VR256XRegClass, Op0); 6753 } 6754 return 0; 6755} 6756 6757unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v2i64_r(unsigned Op0) { 6758 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 6759 return fastEmitInst_r(X86::VCVTTPH2QQZ128rr, &X86::VR128XRegClass, Op0); 6760 } 6761 return 0; 6762} 6763 6764unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v4i64_r(unsigned Op0) { 6765 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 6766 return fastEmitInst_r(X86::VCVTTPH2QQZ256rr, &X86::VR256XRegClass, Op0); 6767 } 6768 return 0; 6769} 6770 6771unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v8i64_r(unsigned Op0) { 6772 if ((Subtarget->hasFP16())) { 6773 return fastEmitInst_r(X86::VCVTTPH2QQZrr, &X86::VR512RegClass, Op0); 6774 } 6775 return 0; 6776} 6777 6778unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 6779switch (RetVT.SimpleTy) { 6780 case MVT::v8i16: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v8i16_r(Op0); 6781 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v4i32_r(Op0); 6782 case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v8i32_r(Op0); 6783 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v2i64_r(Op0); 6784 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v4i64_r(Op0); 6785 case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_MVT_v8i64_r(Op0); 6786 default: return 0; 6787} 6788} 6789 6790unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f16_MVT_v16i16_r(unsigned Op0) { 6791 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 6792 return fastEmitInst_r(X86::VCVTTPH2WZ256rr, &X86::VR256XRegClass, Op0); 6793 } 6794 return 0; 6795} 6796 6797unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f16_MVT_v16i32_r(unsigned Op0) { 6798 if ((Subtarget->hasFP16())) { 6799 return fastEmitInst_r(X86::VCVTTPH2DQZrr, &X86::VR512RegClass, Op0); 6800 } 6801 return 0; 6802} 6803 6804unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f16_r(MVT RetVT, unsigned Op0) { 6805switch (RetVT.SimpleTy) { 6806 case MVT::v16i16: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f16_MVT_v16i16_r(Op0); 6807 case MVT::v16i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f16_MVT_v16i32_r(Op0); 6808 default: return 0; 6809} 6810} 6811 6812unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v32f16_r(MVT RetVT, unsigned Op0) { 6813 if (RetVT.SimpleTy != MVT::v32i16) 6814 return 0; 6815 if ((Subtarget->hasFP16())) { 6816 return fastEmitInst_r(X86::VCVTTPH2WZrr, &X86::VR512RegClass, Op0); 6817 } 6818 return 0; 6819} 6820 6821unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v4i32_r(unsigned Op0) { 6822 if ((Subtarget->hasVLX())) { 6823 return fastEmitInst_r(X86::VCVTTPS2DQZ128rr, &X86::VR128XRegClass, Op0); 6824 } 6825 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 6826 return fastEmitInst_r(X86::CVTTPS2DQrr, &X86::VR128RegClass, Op0); 6827 } 6828 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 6829 return fastEmitInst_r(X86::VCVTTPS2DQrr, &X86::VR128RegClass, Op0); 6830 } 6831 return 0; 6832} 6833 6834unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v2i64_r(unsigned Op0) { 6835 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 6836 return fastEmitInst_r(X86::VCVTTPS2QQZ128rr, &X86::VR128XRegClass, Op0); 6837 } 6838 return 0; 6839} 6840 6841unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v4i64_r(unsigned Op0) { 6842 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 6843 return fastEmitInst_r(X86::VCVTTPS2QQZ256rr, &X86::VR256XRegClass, Op0); 6844 } 6845 return 0; 6846} 6847 6848unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 6849switch (RetVT.SimpleTy) { 6850 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v4i32_r(Op0); 6851 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v2i64_r(Op0); 6852 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v4i64_r(Op0); 6853 default: return 0; 6854} 6855} 6856 6857unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_MVT_v8i32_r(unsigned Op0) { 6858 if ((Subtarget->hasVLX())) { 6859 return fastEmitInst_r(X86::VCVTTPS2DQZ256rr, &X86::VR256XRegClass, Op0); 6860 } 6861 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 6862 return fastEmitInst_r(X86::VCVTTPS2DQYrr, &X86::VR256RegClass, Op0); 6863 } 6864 return 0; 6865} 6866 6867unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_MVT_v8i64_r(unsigned Op0) { 6868 if ((Subtarget->hasDQI())) { 6869 return fastEmitInst_r(X86::VCVTTPS2QQZrr, &X86::VR512RegClass, Op0); 6870 } 6871 return 0; 6872} 6873 6874unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_r(MVT RetVT, unsigned Op0) { 6875switch (RetVT.SimpleTy) { 6876 case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_MVT_v8i32_r(Op0); 6877 case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_MVT_v8i64_r(Op0); 6878 default: return 0; 6879} 6880} 6881 6882unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f32_r(MVT RetVT, unsigned Op0) { 6883 if (RetVT.SimpleTy != MVT::v16i32) 6884 return 0; 6885 if ((Subtarget->hasAVX512())) { 6886 return fastEmitInst_r(X86::VCVTTPS2DQZrr, &X86::VR512RegClass, Op0); 6887 } 6888 return 0; 6889} 6890 6891unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_MVT_v4i32_r(unsigned Op0) { 6892 if ((Subtarget->hasVLX())) { 6893 return fastEmitInst_r(X86::VCVTTPD2DQZ128rr, &X86::VR128XRegClass, Op0); 6894 } 6895 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 6896 return fastEmitInst_r(X86::CVTTPD2DQrr, &X86::VR128RegClass, Op0); 6897 } 6898 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 6899 return fastEmitInst_r(X86::VCVTTPD2DQrr, &X86::VR128RegClass, Op0); 6900 } 6901 return 0; 6902} 6903 6904unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_MVT_v2i64_r(unsigned Op0) { 6905 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 6906 return fastEmitInst_r(X86::VCVTTPD2QQZ128rr, &X86::VR128XRegClass, Op0); 6907 } 6908 return 0; 6909} 6910 6911unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 6912switch (RetVT.SimpleTy) { 6913 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_MVT_v4i32_r(Op0); 6914 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_MVT_v2i64_r(Op0); 6915 default: return 0; 6916} 6917} 6918 6919unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_MVT_v4i32_r(unsigned Op0) { 6920 if ((Subtarget->hasVLX())) { 6921 return fastEmitInst_r(X86::VCVTTPD2DQZ256rr, &X86::VR128XRegClass, Op0); 6922 } 6923 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 6924 return fastEmitInst_r(X86::VCVTTPD2DQYrr, &X86::VR128RegClass, Op0); 6925 } 6926 return 0; 6927} 6928 6929unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_MVT_v4i64_r(unsigned Op0) { 6930 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 6931 return fastEmitInst_r(X86::VCVTTPD2QQZ256rr, &X86::VR256XRegClass, Op0); 6932 } 6933 return 0; 6934} 6935 6936unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_r(MVT RetVT, unsigned Op0) { 6937switch (RetVT.SimpleTy) { 6938 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_MVT_v4i32_r(Op0); 6939 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_MVT_v4i64_r(Op0); 6940 default: return 0; 6941} 6942} 6943 6944unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_MVT_v8i32_r(unsigned Op0) { 6945 if ((Subtarget->hasAVX512())) { 6946 return fastEmitInst_r(X86::VCVTTPD2DQZrr, &X86::VR256XRegClass, Op0); 6947 } 6948 return 0; 6949} 6950 6951unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_MVT_v8i64_r(unsigned Op0) { 6952 if ((Subtarget->hasDQI())) { 6953 return fastEmitInst_r(X86::VCVTTPD2QQZrr, &X86::VR512RegClass, Op0); 6954 } 6955 return 0; 6956} 6957 6958unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_r(MVT RetVT, unsigned Op0) { 6959switch (RetVT.SimpleTy) { 6960 case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_MVT_v8i32_r(Op0); 6961 case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_MVT_v8i64_r(Op0); 6962 default: return 0; 6963} 6964} 6965 6966unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_r(MVT VT, MVT RetVT, unsigned Op0) { 6967 switch (VT.SimpleTy) { 6968 case MVT::v8f16: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f16_r(RetVT, Op0); 6969 case MVT::v16f16: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f16_r(RetVT, Op0); 6970 case MVT::v32f16: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v32f16_r(RetVT, Op0); 6971 case MVT::v4f32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_r(RetVT, Op0); 6972 case MVT::v8f32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_r(RetVT, Op0); 6973 case MVT::v16f32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f32_r(RetVT, Op0); 6974 case MVT::v2f64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_r(RetVT, Op0); 6975 case MVT::v4f64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_r(RetVT, Op0); 6976 case MVT::v8f64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_r(RetVT, Op0); 6977 default: return 0; 6978 } 6979} 6980 6981// FastEmit functions for X86ISD::STRICT_CVTTP2UI. 6982 6983unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v8i16_r(unsigned Op0) { 6984 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 6985 return fastEmitInst_r(X86::VCVTTPH2UWZ128rr, &X86::VR128XRegClass, Op0); 6986 } 6987 return 0; 6988} 6989 6990unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v4i32_r(unsigned Op0) { 6991 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 6992 return fastEmitInst_r(X86::VCVTTPH2UDQZ128rr, &X86::VR128XRegClass, Op0); 6993 } 6994 return 0; 6995} 6996 6997unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v8i32_r(unsigned Op0) { 6998 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 6999 return fastEmitInst_r(X86::VCVTTPH2UDQZ256rr, &X86::VR256XRegClass, Op0); 7000 } 7001 return 0; 7002} 7003 7004unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v2i64_r(unsigned Op0) { 7005 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 7006 return fastEmitInst_r(X86::VCVTTPH2UQQZ128rr, &X86::VR128XRegClass, Op0); 7007 } 7008 return 0; 7009} 7010 7011unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v4i64_r(unsigned Op0) { 7012 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 7013 return fastEmitInst_r(X86::VCVTTPH2UQQZ256rr, &X86::VR256XRegClass, Op0); 7014 } 7015 return 0; 7016} 7017 7018unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v8i64_r(unsigned Op0) { 7019 if ((Subtarget->hasFP16())) { 7020 return fastEmitInst_r(X86::VCVTTPH2UQQZrr, &X86::VR512RegClass, Op0); 7021 } 7022 return 0; 7023} 7024 7025unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 7026switch (RetVT.SimpleTy) { 7027 case MVT::v8i16: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v8i16_r(Op0); 7028 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v4i32_r(Op0); 7029 case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v8i32_r(Op0); 7030 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v2i64_r(Op0); 7031 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v4i64_r(Op0); 7032 case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_MVT_v8i64_r(Op0); 7033 default: return 0; 7034} 7035} 7036 7037unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f16_MVT_v16i16_r(unsigned Op0) { 7038 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 7039 return fastEmitInst_r(X86::VCVTTPH2UWZ256rr, &X86::VR256XRegClass, Op0); 7040 } 7041 return 0; 7042} 7043 7044unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f16_MVT_v16i32_r(unsigned Op0) { 7045 if ((Subtarget->hasFP16())) { 7046 return fastEmitInst_r(X86::VCVTTPH2UDQZrr, &X86::VR512RegClass, Op0); 7047 } 7048 return 0; 7049} 7050 7051unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f16_r(MVT RetVT, unsigned Op0) { 7052switch (RetVT.SimpleTy) { 7053 case MVT::v16i16: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f16_MVT_v16i16_r(Op0); 7054 case MVT::v16i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f16_MVT_v16i32_r(Op0); 7055 default: return 0; 7056} 7057} 7058 7059unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v32f16_r(MVT RetVT, unsigned Op0) { 7060 if (RetVT.SimpleTy != MVT::v32i16) 7061 return 0; 7062 if ((Subtarget->hasFP16())) { 7063 return fastEmitInst_r(X86::VCVTTPH2UWZrr, &X86::VR512RegClass, Op0); 7064 } 7065 return 0; 7066} 7067 7068unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v4i32_r(unsigned Op0) { 7069 if ((Subtarget->hasVLX())) { 7070 return fastEmitInst_r(X86::VCVTTPS2UDQZ128rr, &X86::VR128XRegClass, Op0); 7071 } 7072 return 0; 7073} 7074 7075unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v2i64_r(unsigned Op0) { 7076 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 7077 return fastEmitInst_r(X86::VCVTTPS2UQQZ128rr, &X86::VR128XRegClass, Op0); 7078 } 7079 return 0; 7080} 7081 7082unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v4i64_r(unsigned Op0) { 7083 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 7084 return fastEmitInst_r(X86::VCVTTPS2UQQZ256rr, &X86::VR256XRegClass, Op0); 7085 } 7086 return 0; 7087} 7088 7089unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 7090switch (RetVT.SimpleTy) { 7091 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v4i32_r(Op0); 7092 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v2i64_r(Op0); 7093 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v4i64_r(Op0); 7094 default: return 0; 7095} 7096} 7097 7098unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_MVT_v8i32_r(unsigned Op0) { 7099 if ((Subtarget->hasVLX())) { 7100 return fastEmitInst_r(X86::VCVTTPS2UDQZ256rr, &X86::VR256XRegClass, Op0); 7101 } 7102 return 0; 7103} 7104 7105unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_MVT_v8i64_r(unsigned Op0) { 7106 if ((Subtarget->hasDQI())) { 7107 return fastEmitInst_r(X86::VCVTTPS2UQQZrr, &X86::VR512RegClass, Op0); 7108 } 7109 return 0; 7110} 7111 7112unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_r(MVT RetVT, unsigned Op0) { 7113switch (RetVT.SimpleTy) { 7114 case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_MVT_v8i32_r(Op0); 7115 case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_MVT_v8i64_r(Op0); 7116 default: return 0; 7117} 7118} 7119 7120unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f32_r(MVT RetVT, unsigned Op0) { 7121 if (RetVT.SimpleTy != MVT::v16i32) 7122 return 0; 7123 if ((Subtarget->hasAVX512())) { 7124 return fastEmitInst_r(X86::VCVTTPS2UDQZrr, &X86::VR512RegClass, Op0); 7125 } 7126 return 0; 7127} 7128 7129unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_MVT_v4i32_r(unsigned Op0) { 7130 if ((Subtarget->hasVLX())) { 7131 return fastEmitInst_r(X86::VCVTTPD2UDQZ128rr, &X86::VR128XRegClass, Op0); 7132 } 7133 return 0; 7134} 7135 7136unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_MVT_v2i64_r(unsigned Op0) { 7137 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 7138 return fastEmitInst_r(X86::VCVTTPD2UQQZ128rr, &X86::VR128XRegClass, Op0); 7139 } 7140 return 0; 7141} 7142 7143unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 7144switch (RetVT.SimpleTy) { 7145 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_MVT_v4i32_r(Op0); 7146 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_MVT_v2i64_r(Op0); 7147 default: return 0; 7148} 7149} 7150 7151unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_MVT_v4i32_r(unsigned Op0) { 7152 if ((Subtarget->hasVLX())) { 7153 return fastEmitInst_r(X86::VCVTTPD2UDQZ256rr, &X86::VR128XRegClass, Op0); 7154 } 7155 return 0; 7156} 7157 7158unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_MVT_v4i64_r(unsigned Op0) { 7159 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 7160 return fastEmitInst_r(X86::VCVTTPD2UQQZ256rr, &X86::VR256XRegClass, Op0); 7161 } 7162 return 0; 7163} 7164 7165unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_r(MVT RetVT, unsigned Op0) { 7166switch (RetVT.SimpleTy) { 7167 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_MVT_v4i32_r(Op0); 7168 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_MVT_v4i64_r(Op0); 7169 default: return 0; 7170} 7171} 7172 7173unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_MVT_v8i32_r(unsigned Op0) { 7174 if ((Subtarget->hasAVX512())) { 7175 return fastEmitInst_r(X86::VCVTTPD2UDQZrr, &X86::VR256XRegClass, Op0); 7176 } 7177 return 0; 7178} 7179 7180unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_MVT_v8i64_r(unsigned Op0) { 7181 if ((Subtarget->hasDQI())) { 7182 return fastEmitInst_r(X86::VCVTTPD2UQQZrr, &X86::VR512RegClass, Op0); 7183 } 7184 return 0; 7185} 7186 7187unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_r(MVT RetVT, unsigned Op0) { 7188switch (RetVT.SimpleTy) { 7189 case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_MVT_v8i32_r(Op0); 7190 case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_MVT_v8i64_r(Op0); 7191 default: return 0; 7192} 7193} 7194 7195unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_r(MVT VT, MVT RetVT, unsigned Op0) { 7196 switch (VT.SimpleTy) { 7197 case MVT::v8f16: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f16_r(RetVT, Op0); 7198 case MVT::v16f16: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f16_r(RetVT, Op0); 7199 case MVT::v32f16: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v32f16_r(RetVT, Op0); 7200 case MVT::v4f32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_r(RetVT, Op0); 7201 case MVT::v8f32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_r(RetVT, Op0); 7202 case MVT::v16f32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f32_r(RetVT, Op0); 7203 case MVT::v2f64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_r(RetVT, Op0); 7204 case MVT::v4f64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_r(RetVT, Op0); 7205 case MVT::v8f64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_r(RetVT, Op0); 7206 default: return 0; 7207 } 7208} 7209 7210// FastEmit functions for X86ISD::STRICT_CVTUI2P. 7211 7212unsigned fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_MVT_v8f16_r(unsigned Op0) { 7213 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 7214 return fastEmitInst_r(X86::VCVTUDQ2PHZ128rr, &X86::VR128XRegClass, Op0); 7215 } 7216 return 0; 7217} 7218 7219unsigned fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_MVT_v2f64_r(unsigned Op0) { 7220 if ((Subtarget->hasVLX())) { 7221 return fastEmitInst_r(X86::VCVTUDQ2PDZ128rr, &X86::VR128XRegClass, Op0); 7222 } 7223 return 0; 7224} 7225 7226unsigned fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_r(MVT RetVT, unsigned Op0) { 7227switch (RetVT.SimpleTy) { 7228 case MVT::v8f16: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_MVT_v8f16_r(Op0); 7229 case MVT::v2f64: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_MVT_v2f64_r(Op0); 7230 default: return 0; 7231} 7232} 7233 7234unsigned fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_MVT_v8f16_r(unsigned Op0) { 7235 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 7236 return fastEmitInst_r(X86::VCVTUQQ2PHZ128rr, &X86::VR128XRegClass, Op0); 7237 } 7238 return 0; 7239} 7240 7241unsigned fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_MVT_v4f32_r(unsigned Op0) { 7242 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 7243 return fastEmitInst_r(X86::VCVTUQQ2PSZ128rr, &X86::VR128XRegClass, Op0); 7244 } 7245 return 0; 7246} 7247 7248unsigned fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_r(MVT RetVT, unsigned Op0) { 7249switch (RetVT.SimpleTy) { 7250 case MVT::v8f16: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_MVT_v8f16_r(Op0); 7251 case MVT::v4f32: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_MVT_v4f32_r(Op0); 7252 default: return 0; 7253} 7254} 7255 7256unsigned fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i64_r(MVT RetVT, unsigned Op0) { 7257 if (RetVT.SimpleTy != MVT::v8f16) 7258 return 0; 7259 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 7260 return fastEmitInst_r(X86::VCVTUQQ2PHZ256rr, &X86::VR128XRegClass, Op0); 7261 } 7262 return 0; 7263} 7264 7265unsigned fastEmit_X86ISD_STRICT_CVTUI2P_r(MVT VT, MVT RetVT, unsigned Op0) { 7266 switch (VT.SimpleTy) { 7267 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_r(RetVT, Op0); 7268 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_r(RetVT, Op0); 7269 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i64_r(RetVT, Op0); 7270 default: return 0; 7271 } 7272} 7273 7274// FastEmit functions for X86ISD::STRICT_VFPEXT. 7275 7276unsigned fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_MVT_v4f32_r(unsigned Op0) { 7277 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 7278 return fastEmitInst_r(X86::VCVTPH2PSXZ128rr, &X86::VR128XRegClass, Op0); 7279 } 7280 return 0; 7281} 7282 7283unsigned fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_MVT_v2f64_r(unsigned Op0) { 7284 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 7285 return fastEmitInst_r(X86::VCVTPH2PDZ128rr, &X86::VR128XRegClass, Op0); 7286 } 7287 return 0; 7288} 7289 7290unsigned fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_MVT_v4f64_r(unsigned Op0) { 7291 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 7292 return fastEmitInst_r(X86::VCVTPH2PDZ256rr, &X86::VR256XRegClass, Op0); 7293 } 7294 return 0; 7295} 7296 7297unsigned fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 7298switch (RetVT.SimpleTy) { 7299 case MVT::v4f32: return fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_MVT_v4f32_r(Op0); 7300 case MVT::v2f64: return fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_MVT_v2f64_r(Op0); 7301 case MVT::v4f64: return fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_MVT_v4f64_r(Op0); 7302 default: return 0; 7303} 7304} 7305 7306unsigned fastEmit_X86ISD_STRICT_VFPEXT_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 7307 if (RetVT.SimpleTy != MVT::v2f64) 7308 return 0; 7309 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 7310 return fastEmitInst_r(X86::VCVTPS2PDZ128rr, &X86::VR128XRegClass, Op0); 7311 } 7312 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 7313 return fastEmitInst_r(X86::CVTPS2PDrr, &X86::VR128RegClass, Op0); 7314 } 7315 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 7316 return fastEmitInst_r(X86::VCVTPS2PDrr, &X86::VR128RegClass, Op0); 7317 } 7318 return 0; 7319} 7320 7321unsigned fastEmit_X86ISD_STRICT_VFPEXT_r(MVT VT, MVT RetVT, unsigned Op0) { 7322 switch (VT.SimpleTy) { 7323 case MVT::v8f16: return fastEmit_X86ISD_STRICT_VFPEXT_MVT_v8f16_r(RetVT, Op0); 7324 case MVT::v4f32: return fastEmit_X86ISD_STRICT_VFPEXT_MVT_v4f32_r(RetVT, Op0); 7325 default: return 0; 7326 } 7327} 7328 7329// FastEmit functions for X86ISD::STRICT_VFPROUND. 7330 7331unsigned fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 7332 if (RetVT.SimpleTy != MVT::v8f16) 7333 return 0; 7334 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 7335 return fastEmitInst_r(X86::VCVTPS2PHXZ128rr, &X86::VR128XRegClass, Op0); 7336 } 7337 return 0; 7338} 7339 7340unsigned fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f32_r(MVT RetVT, unsigned Op0) { 7341 if (RetVT.SimpleTy != MVT::v8f16) 7342 return 0; 7343 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 7344 return fastEmitInst_r(X86::VCVTPS2PHXZ256rr, &X86::VR128XRegClass, Op0); 7345 } 7346 return 0; 7347} 7348 7349unsigned fastEmit_X86ISD_STRICT_VFPROUND_MVT_v16f32_r(MVT RetVT, unsigned Op0) { 7350 if (RetVT.SimpleTy != MVT::v16f16) 7351 return 0; 7352 if ((Subtarget->hasFP16())) { 7353 return fastEmitInst_r(X86::VCVTPS2PHXZrr, &X86::VR256XRegClass, Op0); 7354 } 7355 return 0; 7356} 7357 7358unsigned fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_MVT_v8f16_r(unsigned Op0) { 7359 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 7360 return fastEmitInst_r(X86::VCVTPD2PHZ128rr, &X86::VR128XRegClass, Op0); 7361 } 7362 return 0; 7363} 7364 7365unsigned fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_MVT_v4f32_r(unsigned Op0) { 7366 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 7367 return fastEmitInst_r(X86::VCVTPD2PSZ128rr, &X86::VR128XRegClass, Op0); 7368 } 7369 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 7370 return fastEmitInst_r(X86::CVTPD2PSrr, &X86::VR128RegClass, Op0); 7371 } 7372 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 7373 return fastEmitInst_r(X86::VCVTPD2PSrr, &X86::VR128RegClass, Op0); 7374 } 7375 return 0; 7376} 7377 7378unsigned fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 7379switch (RetVT.SimpleTy) { 7380 case MVT::v8f16: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_MVT_v8f16_r(Op0); 7381 case MVT::v4f32: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_MVT_v4f32_r(Op0); 7382 default: return 0; 7383} 7384} 7385 7386unsigned fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_MVT_v8f16_r(unsigned Op0) { 7387 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 7388 return fastEmitInst_r(X86::VCVTPD2PHZ256rr, &X86::VR128XRegClass, Op0); 7389 } 7390 return 0; 7391} 7392 7393unsigned fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_MVT_v4f32_r(unsigned Op0) { 7394 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 7395 return fastEmitInst_r(X86::VCVTPD2PSZ256rr, &X86::VR128XRegClass, Op0); 7396 } 7397 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 7398 return fastEmitInst_r(X86::VCVTPD2PSYrr, &X86::VR128RegClass, Op0); 7399 } 7400 return 0; 7401} 7402 7403unsigned fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_r(MVT RetVT, unsigned Op0) { 7404switch (RetVT.SimpleTy) { 7405 case MVT::v8f16: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_MVT_v8f16_r(Op0); 7406 case MVT::v4f32: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_MVT_v4f32_r(Op0); 7407 default: return 0; 7408} 7409} 7410 7411unsigned fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_MVT_v8f16_r(unsigned Op0) { 7412 if ((Subtarget->hasFP16())) { 7413 return fastEmitInst_r(X86::VCVTPD2PHZrr, &X86::VR128XRegClass, Op0); 7414 } 7415 return 0; 7416} 7417 7418unsigned fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_MVT_v8f32_r(unsigned Op0) { 7419 if ((Subtarget->hasAVX512())) { 7420 return fastEmitInst_r(X86::VCVTPD2PSZrr, &X86::VR256XRegClass, Op0); 7421 } 7422 return 0; 7423} 7424 7425unsigned fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_r(MVT RetVT, unsigned Op0) { 7426switch (RetVT.SimpleTy) { 7427 case MVT::v8f16: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_MVT_v8f16_r(Op0); 7428 case MVT::v8f32: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_MVT_v8f32_r(Op0); 7429 default: return 0; 7430} 7431} 7432 7433unsigned fastEmit_X86ISD_STRICT_VFPROUND_r(MVT VT, MVT RetVT, unsigned Op0) { 7434 switch (VT.SimpleTy) { 7435 case MVT::v4f32: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f32_r(RetVT, Op0); 7436 case MVT::v8f32: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f32_r(RetVT, Op0); 7437 case MVT::v16f32: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v16f32_r(RetVT, Op0); 7438 case MVT::v2f64: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_r(RetVT, Op0); 7439 case MVT::v4f64: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_r(RetVT, Op0); 7440 case MVT::v8f64: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_r(RetVT, Op0); 7441 default: return 0; 7442 } 7443} 7444 7445// FastEmit functions for X86ISD::VBROADCAST. 7446 7447unsigned fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v4i32_r(unsigned Op0) { 7448 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 7449 return fastEmitInst_r(X86::VPBROADCASTDrZ128rr, &X86::VR128XRegClass, Op0); 7450 } 7451 return 0; 7452} 7453 7454unsigned fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v8i32_r(unsigned Op0) { 7455 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 7456 return fastEmitInst_r(X86::VPBROADCASTDrZ256rr, &X86::VR256XRegClass, Op0); 7457 } 7458 return 0; 7459} 7460 7461unsigned fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v16i32_r(unsigned Op0) { 7462 if ((Subtarget->hasAVX512())) { 7463 return fastEmitInst_r(X86::VPBROADCASTDrZrr, &X86::VR512RegClass, Op0); 7464 } 7465 return 0; 7466} 7467 7468unsigned fastEmit_X86ISD_VBROADCAST_MVT_i32_r(MVT RetVT, unsigned Op0) { 7469switch (RetVT.SimpleTy) { 7470 case MVT::v4i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v4i32_r(Op0); 7471 case MVT::v8i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v8i32_r(Op0); 7472 case MVT::v16i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v16i32_r(Op0); 7473 default: return 0; 7474} 7475} 7476 7477unsigned fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v2i64_r(unsigned Op0) { 7478 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 7479 return fastEmitInst_r(X86::VPBROADCASTQrZ128rr, &X86::VR128XRegClass, Op0); 7480 } 7481 return 0; 7482} 7483 7484unsigned fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v4i64_r(unsigned Op0) { 7485 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 7486 return fastEmitInst_r(X86::VPBROADCASTQrZ256rr, &X86::VR256XRegClass, Op0); 7487 } 7488 return 0; 7489} 7490 7491unsigned fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v8i64_r(unsigned Op0) { 7492 if ((Subtarget->hasAVX512())) { 7493 return fastEmitInst_r(X86::VPBROADCASTQrZrr, &X86::VR512RegClass, Op0); 7494 } 7495 return 0; 7496} 7497 7498unsigned fastEmit_X86ISD_VBROADCAST_MVT_i64_r(MVT RetVT, unsigned Op0) { 7499switch (RetVT.SimpleTy) { 7500 case MVT::v2i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v2i64_r(Op0); 7501 case MVT::v4i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v4i64_r(Op0); 7502 case MVT::v8i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v8i64_r(Op0); 7503 default: return 0; 7504} 7505} 7506 7507unsigned fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v16i8_r(unsigned Op0) { 7508 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 7509 return fastEmitInst_r(X86::VPBROADCASTBZ128rr, &X86::VR128XRegClass, Op0); 7510 } 7511 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 7512 return fastEmitInst_r(X86::VPBROADCASTBrr, &X86::VR128RegClass, Op0); 7513 } 7514 return 0; 7515} 7516 7517unsigned fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v32i8_r(unsigned Op0) { 7518 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 7519 return fastEmitInst_r(X86::VPBROADCASTBZ256rr, &X86::VR256XRegClass, Op0); 7520 } 7521 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 7522 return fastEmitInst_r(X86::VPBROADCASTBYrr, &X86::VR256RegClass, Op0); 7523 } 7524 return 0; 7525} 7526 7527unsigned fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v64i8_r(unsigned Op0) { 7528 if ((Subtarget->hasBWI())) { 7529 return fastEmitInst_r(X86::VPBROADCASTBZrr, &X86::VR512RegClass, Op0); 7530 } 7531 return 0; 7532} 7533 7534unsigned fastEmit_X86ISD_VBROADCAST_MVT_v16i8_r(MVT RetVT, unsigned Op0) { 7535switch (RetVT.SimpleTy) { 7536 case MVT::v16i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v16i8_r(Op0); 7537 case MVT::v32i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v32i8_r(Op0); 7538 case MVT::v64i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v64i8_r(Op0); 7539 default: return 0; 7540} 7541} 7542 7543unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v8i16_r(unsigned Op0) { 7544 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 7545 return fastEmitInst_r(X86::VPBROADCASTWZ128rr, &X86::VR128XRegClass, Op0); 7546 } 7547 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 7548 return fastEmitInst_r(X86::VPBROADCASTWrr, &X86::VR128RegClass, Op0); 7549 } 7550 return 0; 7551} 7552 7553unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v16i16_r(unsigned Op0) { 7554 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 7555 return fastEmitInst_r(X86::VPBROADCASTWZ256rr, &X86::VR256XRegClass, Op0); 7556 } 7557 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 7558 return fastEmitInst_r(X86::VPBROADCASTWYrr, &X86::VR256RegClass, Op0); 7559 } 7560 return 0; 7561} 7562 7563unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v32i16_r(unsigned Op0) { 7564 if ((Subtarget->hasBWI())) { 7565 return fastEmitInst_r(X86::VPBROADCASTWZrr, &X86::VR512RegClass, Op0); 7566 } 7567 return 0; 7568} 7569 7570unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8i16_r(MVT RetVT, unsigned Op0) { 7571switch (RetVT.SimpleTy) { 7572 case MVT::v8i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v8i16_r(Op0); 7573 case MVT::v16i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v16i16_r(Op0); 7574 case MVT::v32i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v32i16_r(Op0); 7575 default: return 0; 7576} 7577} 7578 7579unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v4i32_r(unsigned Op0) { 7580 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 7581 return fastEmitInst_r(X86::VPBROADCASTDZ128rr, &X86::VR128XRegClass, Op0); 7582 } 7583 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 7584 return fastEmitInst_r(X86::VPBROADCASTDrr, &X86::VR128RegClass, Op0); 7585 } 7586 return 0; 7587} 7588 7589unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v8i32_r(unsigned Op0) { 7590 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 7591 return fastEmitInst_r(X86::VPBROADCASTDZ256rr, &X86::VR256XRegClass, Op0); 7592 } 7593 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 7594 return fastEmitInst_r(X86::VPBROADCASTDYrr, &X86::VR256RegClass, Op0); 7595 } 7596 return 0; 7597} 7598 7599unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v16i32_r(unsigned Op0) { 7600 if ((Subtarget->hasAVX512())) { 7601 return fastEmitInst_r(X86::VPBROADCASTDZrr, &X86::VR512RegClass, Op0); 7602 } 7603 return 0; 7604} 7605 7606unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4i32_r(MVT RetVT, unsigned Op0) { 7607switch (RetVT.SimpleTy) { 7608 case MVT::v4i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v4i32_r(Op0); 7609 case MVT::v8i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v8i32_r(Op0); 7610 case MVT::v16i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v16i32_r(Op0); 7611 default: return 0; 7612} 7613} 7614 7615unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v2i64_r(unsigned Op0) { 7616 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 7617 return fastEmitInst_r(X86::VPBROADCASTQZ128rr, &X86::VR128XRegClass, Op0); 7618 } 7619 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 7620 return fastEmitInst_r(X86::VPBROADCASTQrr, &X86::VR128RegClass, Op0); 7621 } 7622 return 0; 7623} 7624 7625unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v4i64_r(unsigned Op0) { 7626 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 7627 return fastEmitInst_r(X86::VPBROADCASTQZ256rr, &X86::VR256XRegClass, Op0); 7628 } 7629 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 7630 return fastEmitInst_r(X86::VPBROADCASTQYrr, &X86::VR256RegClass, Op0); 7631 } 7632 return 0; 7633} 7634 7635unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v8i64_r(unsigned Op0) { 7636 if ((Subtarget->hasAVX512())) { 7637 return fastEmitInst_r(X86::VPBROADCASTQZrr, &X86::VR512RegClass, Op0); 7638 } 7639 return 0; 7640} 7641 7642unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2i64_r(MVT RetVT, unsigned Op0) { 7643switch (RetVT.SimpleTy) { 7644 case MVT::v2i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v2i64_r(Op0); 7645 case MVT::v4i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v4i64_r(Op0); 7646 case MVT::v8i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v8i64_r(Op0); 7647 default: return 0; 7648} 7649} 7650 7651unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8f16_MVT_v8f16_r(unsigned Op0) { 7652 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 7653 return fastEmitInst_r(X86::VPBROADCASTWZ128rr, &X86::VR128XRegClass, Op0); 7654 } 7655 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 7656 return fastEmitInst_r(X86::VPBROADCASTWrr, &X86::VR128RegClass, Op0); 7657 } 7658 return 0; 7659} 7660 7661unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8f16_MVT_v16f16_r(unsigned Op0) { 7662 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 7663 return fastEmitInst_r(X86::VPBROADCASTWZ256rr, &X86::VR256XRegClass, Op0); 7664 } 7665 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 7666 return fastEmitInst_r(X86::VPBROADCASTWYrr, &X86::VR256RegClass, Op0); 7667 } 7668 return 0; 7669} 7670 7671unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8f16_MVT_v32f16_r(unsigned Op0) { 7672 if ((Subtarget->hasBWI())) { 7673 return fastEmitInst_r(X86::VPBROADCASTWZrr, &X86::VR512RegClass, Op0); 7674 } 7675 return 0; 7676} 7677 7678unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 7679switch (RetVT.SimpleTy) { 7680 case MVT::v8f16: return fastEmit_X86ISD_VBROADCAST_MVT_v8f16_MVT_v8f16_r(Op0); 7681 case MVT::v16f16: return fastEmit_X86ISD_VBROADCAST_MVT_v8f16_MVT_v16f16_r(Op0); 7682 case MVT::v32f16: return fastEmit_X86ISD_VBROADCAST_MVT_v8f16_MVT_v32f16_r(Op0); 7683 default: return 0; 7684} 7685} 7686 7687unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v4f32_r(unsigned Op0) { 7688 if ((Subtarget->hasVLX())) { 7689 return fastEmitInst_r(X86::VBROADCASTSSZ128rr, &X86::VR128XRegClass, Op0); 7690 } 7691 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 7692 return fastEmitInst_r(X86::VBROADCASTSSrr, &X86::VR128RegClass, Op0); 7693 } 7694 return 0; 7695} 7696 7697unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v8f32_r(unsigned Op0) { 7698 if ((Subtarget->hasVLX())) { 7699 return fastEmitInst_r(X86::VBROADCASTSSZ256rr, &X86::VR256XRegClass, Op0); 7700 } 7701 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 7702 return fastEmitInst_r(X86::VBROADCASTSSYrr, &X86::VR256RegClass, Op0); 7703 } 7704 return 0; 7705} 7706 7707unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v16f32_r(unsigned Op0) { 7708 if ((Subtarget->hasAVX512())) { 7709 return fastEmitInst_r(X86::VBROADCASTSSZrr, &X86::VR512RegClass, Op0); 7710 } 7711 return 0; 7712} 7713 7714unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 7715switch (RetVT.SimpleTy) { 7716 case MVT::v4f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v4f32_r(Op0); 7717 case MVT::v8f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v8f32_r(Op0); 7718 case MVT::v16f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v16f32_r(Op0); 7719 default: return 0; 7720} 7721} 7722 7723unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v2f64_r(unsigned Op0) { 7724 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 7725 return fastEmitInst_r(X86::VMOVDDUPrr, &X86::VR128RegClass, Op0); 7726 } 7727 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 7728 return fastEmitInst_r(X86::VMOVDDUPZ128rr, &X86::VR128XRegClass, Op0); 7729 } 7730 return 0; 7731} 7732 7733unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v4f64_r(unsigned Op0) { 7734 if ((Subtarget->hasVLX())) { 7735 return fastEmitInst_r(X86::VBROADCASTSDZ256rr, &X86::VR256XRegClass, Op0); 7736 } 7737 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 7738 return fastEmitInst_r(X86::VBROADCASTSDYrr, &X86::VR256RegClass, Op0); 7739 } 7740 return 0; 7741} 7742 7743unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v8f64_r(unsigned Op0) { 7744 if ((Subtarget->hasAVX512())) { 7745 return fastEmitInst_r(X86::VBROADCASTSDZrr, &X86::VR512RegClass, Op0); 7746 } 7747 return 0; 7748} 7749 7750unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 7751switch (RetVT.SimpleTy) { 7752 case MVT::v2f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v2f64_r(Op0); 7753 case MVT::v4f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v4f64_r(Op0); 7754 case MVT::v8f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v8f64_r(Op0); 7755 default: return 0; 7756} 7757} 7758 7759unsigned fastEmit_X86ISD_VBROADCAST_r(MVT VT, MVT RetVT, unsigned Op0) { 7760 switch (VT.SimpleTy) { 7761 case MVT::i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_r(RetVT, Op0); 7762 case MVT::i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_r(RetVT, Op0); 7763 case MVT::v16i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_r(RetVT, Op0); 7764 case MVT::v8i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_r(RetVT, Op0); 7765 case MVT::v4i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_r(RetVT, Op0); 7766 case MVT::v2i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_r(RetVT, Op0); 7767 case MVT::v8f16: return fastEmit_X86ISD_VBROADCAST_MVT_v8f16_r(RetVT, Op0); 7768 case MVT::v4f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_r(RetVT, Op0); 7769 case MVT::v2f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_r(RetVT, Op0); 7770 default: return 0; 7771 } 7772} 7773 7774// FastEmit functions for X86ISD::VBROADCASTM. 7775 7776unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v2i64_r(unsigned Op0) { 7777 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) { 7778 return fastEmitInst_r(X86::VPBROADCASTMB2QZ128rr, &X86::VR128XRegClass, Op0); 7779 } 7780 return 0; 7781} 7782 7783unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v4i64_r(unsigned Op0) { 7784 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) { 7785 return fastEmitInst_r(X86::VPBROADCASTMB2QZ256rr, &X86::VR256XRegClass, Op0); 7786 } 7787 return 0; 7788} 7789 7790unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v8i64_r(unsigned Op0) { 7791 if ((Subtarget->hasCDI())) { 7792 return fastEmitInst_r(X86::VPBROADCASTMB2QZrr, &X86::VR512RegClass, Op0); 7793 } 7794 return 0; 7795} 7796 7797unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_r(MVT RetVT, unsigned Op0) { 7798switch (RetVT.SimpleTy) { 7799 case MVT::v2i64: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v2i64_r(Op0); 7800 case MVT::v4i64: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v4i64_r(Op0); 7801 case MVT::v8i64: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v8i64_r(Op0); 7802 default: return 0; 7803} 7804} 7805 7806unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v4i32_r(unsigned Op0) { 7807 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) { 7808 return fastEmitInst_r(X86::VPBROADCASTMW2DZ128rr, &X86::VR128XRegClass, Op0); 7809 } 7810 return 0; 7811} 7812 7813unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v8i32_r(unsigned Op0) { 7814 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) { 7815 return fastEmitInst_r(X86::VPBROADCASTMW2DZ256rr, &X86::VR256XRegClass, Op0); 7816 } 7817 return 0; 7818} 7819 7820unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v16i32_r(unsigned Op0) { 7821 if ((Subtarget->hasCDI())) { 7822 return fastEmitInst_r(X86::VPBROADCASTMW2DZrr, &X86::VR512RegClass, Op0); 7823 } 7824 return 0; 7825} 7826 7827unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_r(MVT RetVT, unsigned Op0) { 7828switch (RetVT.SimpleTy) { 7829 case MVT::v4i32: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v4i32_r(Op0); 7830 case MVT::v8i32: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v8i32_r(Op0); 7831 case MVT::v16i32: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v16i32_r(Op0); 7832 default: return 0; 7833} 7834} 7835 7836unsigned fastEmit_X86ISD_VBROADCASTM_r(MVT VT, MVT RetVT, unsigned Op0) { 7837 switch (VT.SimpleTy) { 7838 case MVT::v8i1: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_r(RetVT, Op0); 7839 case MVT::v16i1: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_r(RetVT, Op0); 7840 default: return 0; 7841 } 7842} 7843 7844// FastEmit functions for X86ISD::VFPEXT. 7845 7846unsigned fastEmit_X86ISD_VFPEXT_MVT_v8f16_MVT_v4f32_r(unsigned Op0) { 7847 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 7848 return fastEmitInst_r(X86::VCVTPH2PSXZ128rr, &X86::VR128XRegClass, Op0); 7849 } 7850 return 0; 7851} 7852 7853unsigned fastEmit_X86ISD_VFPEXT_MVT_v8f16_MVT_v2f64_r(unsigned Op0) { 7854 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 7855 return fastEmitInst_r(X86::VCVTPH2PDZ128rr, &X86::VR128XRegClass, Op0); 7856 } 7857 return 0; 7858} 7859 7860unsigned fastEmit_X86ISD_VFPEXT_MVT_v8f16_MVT_v4f64_r(unsigned Op0) { 7861 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 7862 return fastEmitInst_r(X86::VCVTPH2PDZ256rr, &X86::VR256XRegClass, Op0); 7863 } 7864 return 0; 7865} 7866 7867unsigned fastEmit_X86ISD_VFPEXT_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 7868switch (RetVT.SimpleTy) { 7869 case MVT::v4f32: return fastEmit_X86ISD_VFPEXT_MVT_v8f16_MVT_v4f32_r(Op0); 7870 case MVT::v2f64: return fastEmit_X86ISD_VFPEXT_MVT_v8f16_MVT_v2f64_r(Op0); 7871 case MVT::v4f64: return fastEmit_X86ISD_VFPEXT_MVT_v8f16_MVT_v4f64_r(Op0); 7872 default: return 0; 7873} 7874} 7875 7876unsigned fastEmit_X86ISD_VFPEXT_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 7877 if (RetVT.SimpleTy != MVT::v2f64) 7878 return 0; 7879 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 7880 return fastEmitInst_r(X86::VCVTPS2PDZ128rr, &X86::VR128XRegClass, Op0); 7881 } 7882 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 7883 return fastEmitInst_r(X86::CVTPS2PDrr, &X86::VR128RegClass, Op0); 7884 } 7885 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 7886 return fastEmitInst_r(X86::VCVTPS2PDrr, &X86::VR128RegClass, Op0); 7887 } 7888 return 0; 7889} 7890 7891unsigned fastEmit_X86ISD_VFPEXT_r(MVT VT, MVT RetVT, unsigned Op0) { 7892 switch (VT.SimpleTy) { 7893 case MVT::v8f16: return fastEmit_X86ISD_VFPEXT_MVT_v8f16_r(RetVT, Op0); 7894 case MVT::v4f32: return fastEmit_X86ISD_VFPEXT_MVT_v4f32_r(RetVT, Op0); 7895 default: return 0; 7896 } 7897} 7898 7899// FastEmit functions for X86ISD::VFPEXT_SAE. 7900 7901unsigned fastEmit_X86ISD_VFPEXT_SAE_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 7902 if (RetVT.SimpleTy != MVT::v8f64) 7903 return 0; 7904 if ((Subtarget->hasFP16())) { 7905 return fastEmitInst_r(X86::VCVTPH2PDZrrb, &X86::VR512RegClass, Op0); 7906 } 7907 return 0; 7908} 7909 7910unsigned fastEmit_X86ISD_VFPEXT_SAE_MVT_v16f16_r(MVT RetVT, unsigned Op0) { 7911 if (RetVT.SimpleTy != MVT::v16f32) 7912 return 0; 7913 if ((Subtarget->hasFP16())) { 7914 return fastEmitInst_r(X86::VCVTPH2PSXZrrb, &X86::VR512RegClass, Op0); 7915 } 7916 return 0; 7917} 7918 7919unsigned fastEmit_X86ISD_VFPEXT_SAE_MVT_v8f32_r(MVT RetVT, unsigned Op0) { 7920 if (RetVT.SimpleTy != MVT::v8f64) 7921 return 0; 7922 if ((Subtarget->hasAVX512())) { 7923 return fastEmitInst_r(X86::VCVTPS2PDZrrb, &X86::VR512RegClass, Op0); 7924 } 7925 return 0; 7926} 7927 7928unsigned fastEmit_X86ISD_VFPEXT_SAE_r(MVT VT, MVT RetVT, unsigned Op0) { 7929 switch (VT.SimpleTy) { 7930 case MVT::v8f16: return fastEmit_X86ISD_VFPEXT_SAE_MVT_v8f16_r(RetVT, Op0); 7931 case MVT::v16f16: return fastEmit_X86ISD_VFPEXT_SAE_MVT_v16f16_r(RetVT, Op0); 7932 case MVT::v8f32: return fastEmit_X86ISD_VFPEXT_SAE_MVT_v8f32_r(RetVT, Op0); 7933 default: return 0; 7934 } 7935} 7936 7937// FastEmit functions for X86ISD::VFPROUND. 7938 7939unsigned fastEmit_X86ISD_VFPROUND_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 7940 if (RetVT.SimpleTy != MVT::v8f16) 7941 return 0; 7942 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 7943 return fastEmitInst_r(X86::VCVTPS2PHXZ128rr, &X86::VR128XRegClass, Op0); 7944 } 7945 return 0; 7946} 7947 7948unsigned fastEmit_X86ISD_VFPROUND_MVT_v8f32_r(MVT RetVT, unsigned Op0) { 7949 if (RetVT.SimpleTy != MVT::v8f16) 7950 return 0; 7951 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 7952 return fastEmitInst_r(X86::VCVTPS2PHXZ256rr, &X86::VR128XRegClass, Op0); 7953 } 7954 return 0; 7955} 7956 7957unsigned fastEmit_X86ISD_VFPROUND_MVT_v16f32_r(MVT RetVT, unsigned Op0) { 7958 if (RetVT.SimpleTy != MVT::v16f16) 7959 return 0; 7960 if ((Subtarget->hasFP16())) { 7961 return fastEmitInst_r(X86::VCVTPS2PHXZrr, &X86::VR256XRegClass, Op0); 7962 } 7963 return 0; 7964} 7965 7966unsigned fastEmit_X86ISD_VFPROUND_MVT_v2f64_MVT_v8f16_r(unsigned Op0) { 7967 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 7968 return fastEmitInst_r(X86::VCVTPD2PHZ128rr, &X86::VR128XRegClass, Op0); 7969 } 7970 return 0; 7971} 7972 7973unsigned fastEmit_X86ISD_VFPROUND_MVT_v2f64_MVT_v4f32_r(unsigned Op0) { 7974 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 7975 return fastEmitInst_r(X86::VCVTPD2PSZ128rr, &X86::VR128XRegClass, Op0); 7976 } 7977 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 7978 return fastEmitInst_r(X86::CVTPD2PSrr, &X86::VR128RegClass, Op0); 7979 } 7980 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 7981 return fastEmitInst_r(X86::VCVTPD2PSrr, &X86::VR128RegClass, Op0); 7982 } 7983 return 0; 7984} 7985 7986unsigned fastEmit_X86ISD_VFPROUND_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 7987switch (RetVT.SimpleTy) { 7988 case MVT::v8f16: return fastEmit_X86ISD_VFPROUND_MVT_v2f64_MVT_v8f16_r(Op0); 7989 case MVT::v4f32: return fastEmit_X86ISD_VFPROUND_MVT_v2f64_MVT_v4f32_r(Op0); 7990 default: return 0; 7991} 7992} 7993 7994unsigned fastEmit_X86ISD_VFPROUND_MVT_v4f64_MVT_v8f16_r(unsigned Op0) { 7995 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 7996 return fastEmitInst_r(X86::VCVTPD2PHZ256rr, &X86::VR128XRegClass, Op0); 7997 } 7998 return 0; 7999} 8000 8001unsigned fastEmit_X86ISD_VFPROUND_MVT_v4f64_MVT_v4f32_r(unsigned Op0) { 8002 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8003 return fastEmitInst_r(X86::VCVTPD2PSZ256rr, &X86::VR128XRegClass, Op0); 8004 } 8005 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 8006 return fastEmitInst_r(X86::VCVTPD2PSYrr, &X86::VR128RegClass, Op0); 8007 } 8008 return 0; 8009} 8010 8011unsigned fastEmit_X86ISD_VFPROUND_MVT_v4f64_r(MVT RetVT, unsigned Op0) { 8012switch (RetVT.SimpleTy) { 8013 case MVT::v8f16: return fastEmit_X86ISD_VFPROUND_MVT_v4f64_MVT_v8f16_r(Op0); 8014 case MVT::v4f32: return fastEmit_X86ISD_VFPROUND_MVT_v4f64_MVT_v4f32_r(Op0); 8015 default: return 0; 8016} 8017} 8018 8019unsigned fastEmit_X86ISD_VFPROUND_MVT_v8f64_MVT_v8f16_r(unsigned Op0) { 8020 if ((Subtarget->hasFP16())) { 8021 return fastEmitInst_r(X86::VCVTPD2PHZrr, &X86::VR128XRegClass, Op0); 8022 } 8023 return 0; 8024} 8025 8026unsigned fastEmit_X86ISD_VFPROUND_MVT_v8f64_MVT_v8f32_r(unsigned Op0) { 8027 if ((Subtarget->hasAVX512())) { 8028 return fastEmitInst_r(X86::VCVTPD2PSZrr, &X86::VR256XRegClass, Op0); 8029 } 8030 return 0; 8031} 8032 8033unsigned fastEmit_X86ISD_VFPROUND_MVT_v8f64_r(MVT RetVT, unsigned Op0) { 8034switch (RetVT.SimpleTy) { 8035 case MVT::v8f16: return fastEmit_X86ISD_VFPROUND_MVT_v8f64_MVT_v8f16_r(Op0); 8036 case MVT::v8f32: return fastEmit_X86ISD_VFPROUND_MVT_v8f64_MVT_v8f32_r(Op0); 8037 default: return 0; 8038} 8039} 8040 8041unsigned fastEmit_X86ISD_VFPROUND_r(MVT VT, MVT RetVT, unsigned Op0) { 8042 switch (VT.SimpleTy) { 8043 case MVT::v4f32: return fastEmit_X86ISD_VFPROUND_MVT_v4f32_r(RetVT, Op0); 8044 case MVT::v8f32: return fastEmit_X86ISD_VFPROUND_MVT_v8f32_r(RetVT, Op0); 8045 case MVT::v16f32: return fastEmit_X86ISD_VFPROUND_MVT_v16f32_r(RetVT, Op0); 8046 case MVT::v2f64: return fastEmit_X86ISD_VFPROUND_MVT_v2f64_r(RetVT, Op0); 8047 case MVT::v4f64: return fastEmit_X86ISD_VFPROUND_MVT_v4f64_r(RetVT, Op0); 8048 case MVT::v8f64: return fastEmit_X86ISD_VFPROUND_MVT_v8f64_r(RetVT, Op0); 8049 default: return 0; 8050 } 8051} 8052 8053// FastEmit functions for X86ISD::VTRUNC. 8054 8055unsigned fastEmit_X86ISD_VTRUNC_MVT_v8i16_r(MVT RetVT, unsigned Op0) { 8056 if (RetVT.SimpleTy != MVT::v16i8) 8057 return 0; 8058 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 8059 return fastEmitInst_r(X86::VPMOVWBZ128rr, &X86::VR128XRegClass, Op0); 8060 } 8061 return 0; 8062} 8063 8064unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v16i8_r(unsigned Op0) { 8065 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8066 return fastEmitInst_r(X86::VPMOVDBZ128rr, &X86::VR128XRegClass, Op0); 8067 } 8068 return 0; 8069} 8070 8071unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v8i16_r(unsigned Op0) { 8072 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8073 return fastEmitInst_r(X86::VPMOVDWZ128rr, &X86::VR128XRegClass, Op0); 8074 } 8075 return 0; 8076} 8077 8078unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i32_r(MVT RetVT, unsigned Op0) { 8079switch (RetVT.SimpleTy) { 8080 case MVT::v16i8: return fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v16i8_r(Op0); 8081 case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v8i16_r(Op0); 8082 default: return 0; 8083} 8084} 8085 8086unsigned fastEmit_X86ISD_VTRUNC_MVT_v8i32_r(MVT RetVT, unsigned Op0) { 8087 if (RetVT.SimpleTy != MVT::v16i8) 8088 return 0; 8089 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8090 return fastEmitInst_r(X86::VPMOVDBZ256rr, &X86::VR128XRegClass, Op0); 8091 } 8092 return 0; 8093} 8094 8095unsigned fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v16i8_r(unsigned Op0) { 8096 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8097 return fastEmitInst_r(X86::VPMOVQBZ128rr, &X86::VR128XRegClass, Op0); 8098 } 8099 return 0; 8100} 8101 8102unsigned fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v8i16_r(unsigned Op0) { 8103 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8104 return fastEmitInst_r(X86::VPMOVQWZ128rr, &X86::VR128XRegClass, Op0); 8105 } 8106 return 0; 8107} 8108 8109unsigned fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v4i32_r(unsigned Op0) { 8110 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8111 return fastEmitInst_r(X86::VPMOVQDZ128rr, &X86::VR128XRegClass, Op0); 8112 } 8113 return 0; 8114} 8115 8116unsigned fastEmit_X86ISD_VTRUNC_MVT_v2i64_r(MVT RetVT, unsigned Op0) { 8117switch (RetVT.SimpleTy) { 8118 case MVT::v16i8: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v16i8_r(Op0); 8119 case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v8i16_r(Op0); 8120 case MVT::v4i32: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v4i32_r(Op0); 8121 default: return 0; 8122} 8123} 8124 8125unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v16i8_r(unsigned Op0) { 8126 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8127 return fastEmitInst_r(X86::VPMOVQBZ256rr, &X86::VR128XRegClass, Op0); 8128 } 8129 return 0; 8130} 8131 8132unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v8i16_r(unsigned Op0) { 8133 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8134 return fastEmitInst_r(X86::VPMOVQWZ256rr, &X86::VR128XRegClass, Op0); 8135 } 8136 return 0; 8137} 8138 8139unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i64_r(MVT RetVT, unsigned Op0) { 8140switch (RetVT.SimpleTy) { 8141 case MVT::v16i8: return fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v16i8_r(Op0); 8142 case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v8i16_r(Op0); 8143 default: return 0; 8144} 8145} 8146 8147unsigned fastEmit_X86ISD_VTRUNC_MVT_v8i64_r(MVT RetVT, unsigned Op0) { 8148 if (RetVT.SimpleTy != MVT::v16i8) 8149 return 0; 8150 if ((Subtarget->hasAVX512())) { 8151 return fastEmitInst_r(X86::VPMOVQBZrr, &X86::VR128XRegClass, Op0); 8152 } 8153 return 0; 8154} 8155 8156unsigned fastEmit_X86ISD_VTRUNC_r(MVT VT, MVT RetVT, unsigned Op0) { 8157 switch (VT.SimpleTy) { 8158 case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v8i16_r(RetVT, Op0); 8159 case MVT::v4i32: return fastEmit_X86ISD_VTRUNC_MVT_v4i32_r(RetVT, Op0); 8160 case MVT::v8i32: return fastEmit_X86ISD_VTRUNC_MVT_v8i32_r(RetVT, Op0); 8161 case MVT::v2i64: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_r(RetVT, Op0); 8162 case MVT::v4i64: return fastEmit_X86ISD_VTRUNC_MVT_v4i64_r(RetVT, Op0); 8163 case MVT::v8i64: return fastEmit_X86ISD_VTRUNC_MVT_v8i64_r(RetVT, Op0); 8164 default: return 0; 8165 } 8166} 8167 8168// FastEmit functions for X86ISD::VTRUNCS. 8169 8170unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i16_r(MVT RetVT, unsigned Op0) { 8171 if (RetVT.SimpleTy != MVT::v16i8) 8172 return 0; 8173 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 8174 return fastEmitInst_r(X86::VPMOVSWBZ128rr, &X86::VR128XRegClass, Op0); 8175 } 8176 return 0; 8177} 8178 8179unsigned fastEmit_X86ISD_VTRUNCS_MVT_v16i16_r(MVT RetVT, unsigned Op0) { 8180 if (RetVT.SimpleTy != MVT::v16i8) 8181 return 0; 8182 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 8183 return fastEmitInst_r(X86::VPMOVSWBZ256rr, &X86::VR128XRegClass, Op0); 8184 } 8185 return 0; 8186} 8187 8188unsigned fastEmit_X86ISD_VTRUNCS_MVT_v32i16_r(MVT RetVT, unsigned Op0) { 8189 if (RetVT.SimpleTy != MVT::v32i8) 8190 return 0; 8191 if ((Subtarget->hasBWI())) { 8192 return fastEmitInst_r(X86::VPMOVSWBZrr, &X86::VR256XRegClass, Op0); 8193 } 8194 return 0; 8195} 8196 8197unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v16i8_r(unsigned Op0) { 8198 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8199 return fastEmitInst_r(X86::VPMOVSDBZ128rr, &X86::VR128XRegClass, Op0); 8200 } 8201 return 0; 8202} 8203 8204unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v8i16_r(unsigned Op0) { 8205 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8206 return fastEmitInst_r(X86::VPMOVSDWZ128rr, &X86::VR128XRegClass, Op0); 8207 } 8208 return 0; 8209} 8210 8211unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i32_r(MVT RetVT, unsigned Op0) { 8212switch (RetVT.SimpleTy) { 8213 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v16i8_r(Op0); 8214 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v8i16_r(Op0); 8215 default: return 0; 8216} 8217} 8218 8219unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v16i8_r(unsigned Op0) { 8220 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8221 return fastEmitInst_r(X86::VPMOVSDBZ256rr, &X86::VR128XRegClass, Op0); 8222 } 8223 return 0; 8224} 8225 8226unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v8i16_r(unsigned Op0) { 8227 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8228 return fastEmitInst_r(X86::VPMOVSDWZ256rr, &X86::VR128XRegClass, Op0); 8229 } 8230 return 0; 8231} 8232 8233unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i32_r(MVT RetVT, unsigned Op0) { 8234switch (RetVT.SimpleTy) { 8235 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v16i8_r(Op0); 8236 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v8i16_r(Op0); 8237 default: return 0; 8238} 8239} 8240 8241unsigned fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i8_r(unsigned Op0) { 8242 if ((Subtarget->hasAVX512())) { 8243 return fastEmitInst_r(X86::VPMOVSDBZrr, &X86::VR128XRegClass, Op0); 8244 } 8245 return 0; 8246} 8247 8248unsigned fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i16_r(unsigned Op0) { 8249 if ((Subtarget->hasAVX512())) { 8250 return fastEmitInst_r(X86::VPMOVSDWZrr, &X86::VR256XRegClass, Op0); 8251 } 8252 return 0; 8253} 8254 8255unsigned fastEmit_X86ISD_VTRUNCS_MVT_v16i32_r(MVT RetVT, unsigned Op0) { 8256switch (RetVT.SimpleTy) { 8257 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i8_r(Op0); 8258 case MVT::v16i16: return fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i16_r(Op0); 8259 default: return 0; 8260} 8261} 8262 8263unsigned fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v16i8_r(unsigned Op0) { 8264 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8265 return fastEmitInst_r(X86::VPMOVSQBZ128rr, &X86::VR128XRegClass, Op0); 8266 } 8267 return 0; 8268} 8269 8270unsigned fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v8i16_r(unsigned Op0) { 8271 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8272 return fastEmitInst_r(X86::VPMOVSQWZ128rr, &X86::VR128XRegClass, Op0); 8273 } 8274 return 0; 8275} 8276 8277unsigned fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v4i32_r(unsigned Op0) { 8278 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8279 return fastEmitInst_r(X86::VPMOVSQDZ128rr, &X86::VR128XRegClass, Op0); 8280 } 8281 return 0; 8282} 8283 8284unsigned fastEmit_X86ISD_VTRUNCS_MVT_v2i64_r(MVT RetVT, unsigned Op0) { 8285switch (RetVT.SimpleTy) { 8286 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v16i8_r(Op0); 8287 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v8i16_r(Op0); 8288 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v4i32_r(Op0); 8289 default: return 0; 8290} 8291} 8292 8293unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v16i8_r(unsigned Op0) { 8294 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8295 return fastEmitInst_r(X86::VPMOVSQBZ256rr, &X86::VR128XRegClass, Op0); 8296 } 8297 return 0; 8298} 8299 8300unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v8i16_r(unsigned Op0) { 8301 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8302 return fastEmitInst_r(X86::VPMOVSQWZ256rr, &X86::VR128XRegClass, Op0); 8303 } 8304 return 0; 8305} 8306 8307unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v4i32_r(unsigned Op0) { 8308 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8309 return fastEmitInst_r(X86::VPMOVSQDZ256rr, &X86::VR128XRegClass, Op0); 8310 } 8311 return 0; 8312} 8313 8314unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i64_r(MVT RetVT, unsigned Op0) { 8315switch (RetVT.SimpleTy) { 8316 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v16i8_r(Op0); 8317 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v8i16_r(Op0); 8318 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v4i32_r(Op0); 8319 default: return 0; 8320} 8321} 8322 8323unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v16i8_r(unsigned Op0) { 8324 if ((Subtarget->hasAVX512())) { 8325 return fastEmitInst_r(X86::VPMOVSQBZrr, &X86::VR128XRegClass, Op0); 8326 } 8327 return 0; 8328} 8329 8330unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i16_r(unsigned Op0) { 8331 if ((Subtarget->hasAVX512())) { 8332 return fastEmitInst_r(X86::VPMOVSQWZrr, &X86::VR128XRegClass, Op0); 8333 } 8334 return 0; 8335} 8336 8337unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i32_r(unsigned Op0) { 8338 if ((Subtarget->hasAVX512())) { 8339 return fastEmitInst_r(X86::VPMOVSQDZrr, &X86::VR256XRegClass, Op0); 8340 } 8341 return 0; 8342} 8343 8344unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i64_r(MVT RetVT, unsigned Op0) { 8345switch (RetVT.SimpleTy) { 8346 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v16i8_r(Op0); 8347 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i16_r(Op0); 8348 case MVT::v8i32: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i32_r(Op0); 8349 default: return 0; 8350} 8351} 8352 8353unsigned fastEmit_X86ISD_VTRUNCS_r(MVT VT, MVT RetVT, unsigned Op0) { 8354 switch (VT.SimpleTy) { 8355 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v8i16_r(RetVT, Op0); 8356 case MVT::v16i16: return fastEmit_X86ISD_VTRUNCS_MVT_v16i16_r(RetVT, Op0); 8357 case MVT::v32i16: return fastEmit_X86ISD_VTRUNCS_MVT_v32i16_r(RetVT, Op0); 8358 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCS_MVT_v4i32_r(RetVT, Op0); 8359 case MVT::v8i32: return fastEmit_X86ISD_VTRUNCS_MVT_v8i32_r(RetVT, Op0); 8360 case MVT::v16i32: return fastEmit_X86ISD_VTRUNCS_MVT_v16i32_r(RetVT, Op0); 8361 case MVT::v2i64: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_r(RetVT, Op0); 8362 case MVT::v4i64: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_r(RetVT, Op0); 8363 case MVT::v8i64: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_r(RetVT, Op0); 8364 default: return 0; 8365 } 8366} 8367 8368// FastEmit functions for X86ISD::VTRUNCUS. 8369 8370unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i16_r(MVT RetVT, unsigned Op0) { 8371 if (RetVT.SimpleTy != MVT::v16i8) 8372 return 0; 8373 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 8374 return fastEmitInst_r(X86::VPMOVUSWBZ128rr, &X86::VR128XRegClass, Op0); 8375 } 8376 return 0; 8377} 8378 8379unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v16i16_r(MVT RetVT, unsigned Op0) { 8380 if (RetVT.SimpleTy != MVT::v16i8) 8381 return 0; 8382 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 8383 return fastEmitInst_r(X86::VPMOVUSWBZ256rr, &X86::VR128XRegClass, Op0); 8384 } 8385 return 0; 8386} 8387 8388unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v32i16_r(MVT RetVT, unsigned Op0) { 8389 if (RetVT.SimpleTy != MVT::v32i8) 8390 return 0; 8391 if ((Subtarget->hasBWI())) { 8392 return fastEmitInst_r(X86::VPMOVUSWBZrr, &X86::VR256XRegClass, Op0); 8393 } 8394 return 0; 8395} 8396 8397unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v16i8_r(unsigned Op0) { 8398 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8399 return fastEmitInst_r(X86::VPMOVUSDBZ128rr, &X86::VR128XRegClass, Op0); 8400 } 8401 return 0; 8402} 8403 8404unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v8i16_r(unsigned Op0) { 8405 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8406 return fastEmitInst_r(X86::VPMOVUSDWZ128rr, &X86::VR128XRegClass, Op0); 8407 } 8408 return 0; 8409} 8410 8411unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_r(MVT RetVT, unsigned Op0) { 8412switch (RetVT.SimpleTy) { 8413 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v16i8_r(Op0); 8414 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v8i16_r(Op0); 8415 default: return 0; 8416} 8417} 8418 8419unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v16i8_r(unsigned Op0) { 8420 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8421 return fastEmitInst_r(X86::VPMOVUSDBZ256rr, &X86::VR128XRegClass, Op0); 8422 } 8423 return 0; 8424} 8425 8426unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v8i16_r(unsigned Op0) { 8427 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8428 return fastEmitInst_r(X86::VPMOVUSDWZ256rr, &X86::VR128XRegClass, Op0); 8429 } 8430 return 0; 8431} 8432 8433unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_r(MVT RetVT, unsigned Op0) { 8434switch (RetVT.SimpleTy) { 8435 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v16i8_r(Op0); 8436 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v8i16_r(Op0); 8437 default: return 0; 8438} 8439} 8440 8441unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i8_r(unsigned Op0) { 8442 if ((Subtarget->hasAVX512())) { 8443 return fastEmitInst_r(X86::VPMOVUSDBZrr, &X86::VR128XRegClass, Op0); 8444 } 8445 return 0; 8446} 8447 8448unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i16_r(unsigned Op0) { 8449 if ((Subtarget->hasAVX512())) { 8450 return fastEmitInst_r(X86::VPMOVUSDWZrr, &X86::VR256XRegClass, Op0); 8451 } 8452 return 0; 8453} 8454 8455unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_r(MVT RetVT, unsigned Op0) { 8456switch (RetVT.SimpleTy) { 8457 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i8_r(Op0); 8458 case MVT::v16i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i16_r(Op0); 8459 default: return 0; 8460} 8461} 8462 8463unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v16i8_r(unsigned Op0) { 8464 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8465 return fastEmitInst_r(X86::VPMOVUSQBZ128rr, &X86::VR128XRegClass, Op0); 8466 } 8467 return 0; 8468} 8469 8470unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v8i16_r(unsigned Op0) { 8471 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8472 return fastEmitInst_r(X86::VPMOVUSQWZ128rr, &X86::VR128XRegClass, Op0); 8473 } 8474 return 0; 8475} 8476 8477unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v4i32_r(unsigned Op0) { 8478 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8479 return fastEmitInst_r(X86::VPMOVUSQDZ128rr, &X86::VR128XRegClass, Op0); 8480 } 8481 return 0; 8482} 8483 8484unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_r(MVT RetVT, unsigned Op0) { 8485switch (RetVT.SimpleTy) { 8486 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v16i8_r(Op0); 8487 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v8i16_r(Op0); 8488 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v4i32_r(Op0); 8489 default: return 0; 8490} 8491} 8492 8493unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v16i8_r(unsigned Op0) { 8494 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8495 return fastEmitInst_r(X86::VPMOVUSQBZ256rr, &X86::VR128XRegClass, Op0); 8496 } 8497 return 0; 8498} 8499 8500unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v8i16_r(unsigned Op0) { 8501 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8502 return fastEmitInst_r(X86::VPMOVUSQWZ256rr, &X86::VR128XRegClass, Op0); 8503 } 8504 return 0; 8505} 8506 8507unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v4i32_r(unsigned Op0) { 8508 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8509 return fastEmitInst_r(X86::VPMOVUSQDZ256rr, &X86::VR128XRegClass, Op0); 8510 } 8511 return 0; 8512} 8513 8514unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_r(MVT RetVT, unsigned Op0) { 8515switch (RetVT.SimpleTy) { 8516 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v16i8_r(Op0); 8517 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v8i16_r(Op0); 8518 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v4i32_r(Op0); 8519 default: return 0; 8520} 8521} 8522 8523unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v16i8_r(unsigned Op0) { 8524 if ((Subtarget->hasAVX512())) { 8525 return fastEmitInst_r(X86::VPMOVUSQBZrr, &X86::VR128XRegClass, Op0); 8526 } 8527 return 0; 8528} 8529 8530unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i16_r(unsigned Op0) { 8531 if ((Subtarget->hasAVX512())) { 8532 return fastEmitInst_r(X86::VPMOVUSQWZrr, &X86::VR128XRegClass, Op0); 8533 } 8534 return 0; 8535} 8536 8537unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i32_r(unsigned Op0) { 8538 if ((Subtarget->hasAVX512())) { 8539 return fastEmitInst_r(X86::VPMOVUSQDZrr, &X86::VR256XRegClass, Op0); 8540 } 8541 return 0; 8542} 8543 8544unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_r(MVT RetVT, unsigned Op0) { 8545switch (RetVT.SimpleTy) { 8546 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v16i8_r(Op0); 8547 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i16_r(Op0); 8548 case MVT::v8i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i32_r(Op0); 8549 default: return 0; 8550} 8551} 8552 8553unsigned fastEmit_X86ISD_VTRUNCUS_r(MVT VT, MVT RetVT, unsigned Op0) { 8554 switch (VT.SimpleTy) { 8555 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i16_r(RetVT, Op0); 8556 case MVT::v16i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i16_r(RetVT, Op0); 8557 case MVT::v32i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v32i16_r(RetVT, Op0); 8558 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_r(RetVT, Op0); 8559 case MVT::v8i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_r(RetVT, Op0); 8560 case MVT::v16i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_r(RetVT, Op0); 8561 case MVT::v2i64: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_r(RetVT, Op0); 8562 case MVT::v4i64: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_r(RetVT, Op0); 8563 case MVT::v8i64: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_r(RetVT, Op0); 8564 default: return 0; 8565 } 8566} 8567 8568// FastEmit functions for X86ISD::VZEXT_MOVL. 8569 8570unsigned fastEmit_X86ISD_VZEXT_MOVL_MVT_v2i64_r(MVT RetVT, unsigned Op0) { 8571 if (RetVT.SimpleTy != MVT::v2i64) 8572 return 0; 8573 if ((Subtarget->hasAVX512())) { 8574 return fastEmitInst_r(X86::VMOVZPQILo2PQIZrr, &X86::VR128XRegClass, Op0); 8575 } 8576 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 8577 return fastEmitInst_r(X86::MOVZPQILo2PQIrr, &X86::VR128RegClass, Op0); 8578 } 8579 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 8580 return fastEmitInst_r(X86::VMOVZPQILo2PQIrr, &X86::VR128RegClass, Op0); 8581 } 8582 return 0; 8583} 8584 8585unsigned fastEmit_X86ISD_VZEXT_MOVL_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 8586 if (RetVT.SimpleTy != MVT::v2f64) 8587 return 0; 8588 if ((Subtarget->hasAVX512())) { 8589 return fastEmitInst_r(X86::VMOVZPQILo2PQIZrr, &X86::VR128XRegClass, Op0); 8590 } 8591 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 8592 return fastEmitInst_r(X86::MOVZPQILo2PQIrr, &X86::VR128RegClass, Op0); 8593 } 8594 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 8595 return fastEmitInst_r(X86::VMOVZPQILo2PQIrr, &X86::VR128RegClass, Op0); 8596 } 8597 return 0; 8598} 8599 8600unsigned fastEmit_X86ISD_VZEXT_MOVL_r(MVT VT, MVT RetVT, unsigned Op0) { 8601 switch (VT.SimpleTy) { 8602 case MVT::v2i64: return fastEmit_X86ISD_VZEXT_MOVL_MVT_v2i64_r(RetVT, Op0); 8603 case MVT::v2f64: return fastEmit_X86ISD_VZEXT_MOVL_MVT_v2f64_r(RetVT, Op0); 8604 default: return 0; 8605 } 8606} 8607 8608// Top-level FastEmit function. 8609 8610unsigned fastEmit_r(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0) override { 8611 switch (Opcode) { 8612 case ISD::ABS: return fastEmit_ISD_ABS_r(VT, RetVT, Op0); 8613 case ISD::ANY_EXTEND: return fastEmit_ISD_ANY_EXTEND_r(VT, RetVT, Op0); 8614 case ISD::BITCAST: return fastEmit_ISD_BITCAST_r(VT, RetVT, Op0); 8615 case ISD::BRIND: return fastEmit_ISD_BRIND_r(VT, RetVT, Op0); 8616 case ISD::BSWAP: return fastEmit_ISD_BSWAP_r(VT, RetVT, Op0); 8617 case ISD::CTLZ: return fastEmit_ISD_CTLZ_r(VT, RetVT, Op0); 8618 case ISD::CTPOP: return fastEmit_ISD_CTPOP_r(VT, RetVT, Op0); 8619 case ISD::CTTZ: return fastEmit_ISD_CTTZ_r(VT, RetVT, Op0); 8620 case ISD::CTTZ_ZERO_UNDEF: return fastEmit_ISD_CTTZ_ZERO_UNDEF_r(VT, RetVT, Op0); 8621 case ISD::FABS: return fastEmit_ISD_FABS_r(VT, RetVT, Op0); 8622 case ISD::FNEG: return fastEmit_ISD_FNEG_r(VT, RetVT, Op0); 8623 case ISD::FP_EXTEND: return fastEmit_ISD_FP_EXTEND_r(VT, RetVT, Op0); 8624 case ISD::FP_ROUND: return fastEmit_ISD_FP_ROUND_r(VT, RetVT, Op0); 8625 case ISD::FP_TO_SINT: return fastEmit_ISD_FP_TO_SINT_r(VT, RetVT, Op0); 8626 case ISD::FP_TO_UINT: return fastEmit_ISD_FP_TO_UINT_r(VT, RetVT, Op0); 8627 case ISD::FSQRT: return fastEmit_ISD_FSQRT_r(VT, RetVT, Op0); 8628 case ISD::LLRINT: return fastEmit_ISD_LLRINT_r(VT, RetVT, Op0); 8629 case ISD::LRINT: return fastEmit_ISD_LRINT_r(VT, RetVT, Op0); 8630 case ISD::SCALAR_TO_VECTOR: return fastEmit_ISD_SCALAR_TO_VECTOR_r(VT, RetVT, Op0); 8631 case ISD::SIGN_EXTEND: return fastEmit_ISD_SIGN_EXTEND_r(VT, RetVT, Op0); 8632 case ISD::SIGN_EXTEND_VECTOR_INREG: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_r(VT, RetVT, Op0); 8633 case ISD::SINT_TO_FP: return fastEmit_ISD_SINT_TO_FP_r(VT, RetVT, Op0); 8634 case ISD::STRICT_FP_EXTEND: return fastEmit_ISD_STRICT_FP_EXTEND_r(VT, RetVT, Op0); 8635 case ISD::STRICT_FP_ROUND: return fastEmit_ISD_STRICT_FP_ROUND_r(VT, RetVT, Op0); 8636 case ISD::STRICT_FP_TO_SINT: return fastEmit_ISD_STRICT_FP_TO_SINT_r(VT, RetVT, Op0); 8637 case ISD::STRICT_FP_TO_UINT: return fastEmit_ISD_STRICT_FP_TO_UINT_r(VT, RetVT, Op0); 8638 case ISD::STRICT_FSQRT: return fastEmit_ISD_STRICT_FSQRT_r(VT, RetVT, Op0); 8639 case ISD::STRICT_SINT_TO_FP: return fastEmit_ISD_STRICT_SINT_TO_FP_r(VT, RetVT, Op0); 8640 case ISD::STRICT_UINT_TO_FP: return fastEmit_ISD_STRICT_UINT_TO_FP_r(VT, RetVT, Op0); 8641 case ISD::TRUNCATE: return fastEmit_ISD_TRUNCATE_r(VT, RetVT, Op0); 8642 case ISD::UINT_TO_FP: return fastEmit_ISD_UINT_TO_FP_r(VT, RetVT, Op0); 8643 case ISD::ZERO_EXTEND: return fastEmit_ISD_ZERO_EXTEND_r(VT, RetVT, Op0); 8644 case ISD::ZERO_EXTEND_VECTOR_INREG: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_r(VT, RetVT, Op0); 8645 case X86ISD::CALL: return fastEmit_X86ISD_CALL_r(VT, RetVT, Op0); 8646 case X86ISD::CONFLICT: return fastEmit_X86ISD_CONFLICT_r(VT, RetVT, Op0); 8647 case X86ISD::CVTNEPS2BF16: return fastEmit_X86ISD_CVTNEPS2BF16_r(VT, RetVT, Op0); 8648 case X86ISD::CVTP2SI: return fastEmit_X86ISD_CVTP2SI_r(VT, RetVT, Op0); 8649 case X86ISD::CVTP2UI: return fastEmit_X86ISD_CVTP2UI_r(VT, RetVT, Op0); 8650 case X86ISD::CVTPH2PS: return fastEmit_X86ISD_CVTPH2PS_r(VT, RetVT, Op0); 8651 case X86ISD::CVTPH2PS_SAE: return fastEmit_X86ISD_CVTPH2PS_SAE_r(VT, RetVT, Op0); 8652 case X86ISD::CVTS2SI: return fastEmit_X86ISD_CVTS2SI_r(VT, RetVT, Op0); 8653 case X86ISD::CVTS2UI: return fastEmit_X86ISD_CVTS2UI_r(VT, RetVT, Op0); 8654 case X86ISD::CVTSI2P: return fastEmit_X86ISD_CVTSI2P_r(VT, RetVT, Op0); 8655 case X86ISD::CVTTP2SI: return fastEmit_X86ISD_CVTTP2SI_r(VT, RetVT, Op0); 8656 case X86ISD::CVTTP2SI_SAE: return fastEmit_X86ISD_CVTTP2SI_SAE_r(VT, RetVT, Op0); 8657 case X86ISD::CVTTP2UI: return fastEmit_X86ISD_CVTTP2UI_r(VT, RetVT, Op0); 8658 case X86ISD::CVTTP2UI_SAE: return fastEmit_X86ISD_CVTTP2UI_SAE_r(VT, RetVT, Op0); 8659 case X86ISD::CVTTS2SI: return fastEmit_X86ISD_CVTTS2SI_r(VT, RetVT, Op0); 8660 case X86ISD::CVTTS2SI_SAE: return fastEmit_X86ISD_CVTTS2SI_SAE_r(VT, RetVT, Op0); 8661 case X86ISD::CVTTS2UI: return fastEmit_X86ISD_CVTTS2UI_r(VT, RetVT, Op0); 8662 case X86ISD::CVTTS2UI_SAE: return fastEmit_X86ISD_CVTTS2UI_SAE_r(VT, RetVT, Op0); 8663 case X86ISD::CVTUI2P: return fastEmit_X86ISD_CVTUI2P_r(VT, RetVT, Op0); 8664 case X86ISD::DYN_ALLOCA: return fastEmit_X86ISD_DYN_ALLOCA_r(VT, RetVT, Op0); 8665 case X86ISD::EH_RETURN: return fastEmit_X86ISD_EH_RETURN_r(VT, RetVT, Op0); 8666 case X86ISD::EXP2: return fastEmit_X86ISD_EXP2_r(VT, RetVT, Op0); 8667 case X86ISD::EXP2_SAE: return fastEmit_X86ISD_EXP2_SAE_r(VT, RetVT, Op0); 8668 case X86ISD::FGETEXP: return fastEmit_X86ISD_FGETEXP_r(VT, RetVT, Op0); 8669 case X86ISD::FGETEXP_SAE: return fastEmit_X86ISD_FGETEXP_SAE_r(VT, RetVT, Op0); 8670 case X86ISD::FRCP: return fastEmit_X86ISD_FRCP_r(VT, RetVT, Op0); 8671 case X86ISD::FRSQRT: return fastEmit_X86ISD_FRSQRT_r(VT, RetVT, Op0); 8672 case X86ISD::MMX_MOVD2W: return fastEmit_X86ISD_MMX_MOVD2W_r(VT, RetVT, Op0); 8673 case X86ISD::MMX_MOVW2D: return fastEmit_X86ISD_MMX_MOVW2D_r(VT, RetVT, Op0); 8674 case X86ISD::MOVDDUP: return fastEmit_X86ISD_MOVDDUP_r(VT, RetVT, Op0); 8675 case X86ISD::MOVDQ2Q: return fastEmit_X86ISD_MOVDQ2Q_r(VT, RetVT, Op0); 8676 case X86ISD::MOVMSK: return fastEmit_X86ISD_MOVMSK_r(VT, RetVT, Op0); 8677 case X86ISD::MOVQ2DQ: return fastEmit_X86ISD_MOVQ2DQ_r(VT, RetVT, Op0); 8678 case X86ISD::MOVSHDUP: return fastEmit_X86ISD_MOVSHDUP_r(VT, RetVT, Op0); 8679 case X86ISD::MOVSLDUP: return fastEmit_X86ISD_MOVSLDUP_r(VT, RetVT, Op0); 8680 case X86ISD::NT_BRIND: return fastEmit_X86ISD_NT_BRIND_r(VT, RetVT, Op0); 8681 case X86ISD::NT_CALL: return fastEmit_X86ISD_NT_CALL_r(VT, RetVT, Op0); 8682 case X86ISD::PHMINPOS: return fastEmit_X86ISD_PHMINPOS_r(VT, RetVT, Op0); 8683 case X86ISD::PROBED_ALLOCA: return fastEmit_X86ISD_PROBED_ALLOCA_r(VT, RetVT, Op0); 8684 case X86ISD::RCP14: return fastEmit_X86ISD_RCP14_r(VT, RetVT, Op0); 8685 case X86ISD::RCP28: return fastEmit_X86ISD_RCP28_r(VT, RetVT, Op0); 8686 case X86ISD::RCP28_SAE: return fastEmit_X86ISD_RCP28_SAE_r(VT, RetVT, Op0); 8687 case X86ISD::RSQRT14: return fastEmit_X86ISD_RSQRT14_r(VT, RetVT, Op0); 8688 case X86ISD::RSQRT28: return fastEmit_X86ISD_RSQRT28_r(VT, RetVT, Op0); 8689 case X86ISD::RSQRT28_SAE: return fastEmit_X86ISD_RSQRT28_SAE_r(VT, RetVT, Op0); 8690 case X86ISD::SEG_ALLOCA: return fastEmit_X86ISD_SEG_ALLOCA_r(VT, RetVT, Op0); 8691 case X86ISD::STRICT_CVTPH2PS: return fastEmit_X86ISD_STRICT_CVTPH2PS_r(VT, RetVT, Op0); 8692 case X86ISD::STRICT_CVTSI2P: return fastEmit_X86ISD_STRICT_CVTSI2P_r(VT, RetVT, Op0); 8693 case X86ISD::STRICT_CVTTP2SI: return fastEmit_X86ISD_STRICT_CVTTP2SI_r(VT, RetVT, Op0); 8694 case X86ISD::STRICT_CVTTP2UI: return fastEmit_X86ISD_STRICT_CVTTP2UI_r(VT, RetVT, Op0); 8695 case X86ISD::STRICT_CVTUI2P: return fastEmit_X86ISD_STRICT_CVTUI2P_r(VT, RetVT, Op0); 8696 case X86ISD::STRICT_VFPEXT: return fastEmit_X86ISD_STRICT_VFPEXT_r(VT, RetVT, Op0); 8697 case X86ISD::STRICT_VFPROUND: return fastEmit_X86ISD_STRICT_VFPROUND_r(VT, RetVT, Op0); 8698 case X86ISD::VBROADCAST: return fastEmit_X86ISD_VBROADCAST_r(VT, RetVT, Op0); 8699 case X86ISD::VBROADCASTM: return fastEmit_X86ISD_VBROADCASTM_r(VT, RetVT, Op0); 8700 case X86ISD::VFPEXT: return fastEmit_X86ISD_VFPEXT_r(VT, RetVT, Op0); 8701 case X86ISD::VFPEXT_SAE: return fastEmit_X86ISD_VFPEXT_SAE_r(VT, RetVT, Op0); 8702 case X86ISD::VFPROUND: return fastEmit_X86ISD_VFPROUND_r(VT, RetVT, Op0); 8703 case X86ISD::VTRUNC: return fastEmit_X86ISD_VTRUNC_r(VT, RetVT, Op0); 8704 case X86ISD::VTRUNCS: return fastEmit_X86ISD_VTRUNCS_r(VT, RetVT, Op0); 8705 case X86ISD::VTRUNCUS: return fastEmit_X86ISD_VTRUNCUS_r(VT, RetVT, Op0); 8706 case X86ISD::VZEXT_MOVL: return fastEmit_X86ISD_VZEXT_MOVL_r(VT, RetVT, Op0); 8707 default: return 0; 8708 } 8709} 8710 8711// FastEmit functions for ISD::ADD. 8712 8713unsigned fastEmit_ISD_ADD_MVT_i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8714 if (RetVT.SimpleTy != MVT::i8) 8715 return 0; 8716 return fastEmitInst_rr(X86::ADD8rr, &X86::GR8RegClass, Op0, Op1); 8717} 8718 8719unsigned fastEmit_ISD_ADD_MVT_i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8720 if (RetVT.SimpleTy != MVT::i16) 8721 return 0; 8722 return fastEmitInst_rr(X86::ADD16rr, &X86::GR16RegClass, Op0, Op1); 8723} 8724 8725unsigned fastEmit_ISD_ADD_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8726 if (RetVT.SimpleTy != MVT::i32) 8727 return 0; 8728 return fastEmitInst_rr(X86::ADD32rr, &X86::GR32RegClass, Op0, Op1); 8729} 8730 8731unsigned fastEmit_ISD_ADD_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8732 if (RetVT.SimpleTy != MVT::i64) 8733 return 0; 8734 return fastEmitInst_rr(X86::ADD64rr, &X86::GR64RegClass, Op0, Op1); 8735} 8736 8737unsigned fastEmit_ISD_ADD_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8738 if (RetVT.SimpleTy != MVT::v16i8) 8739 return 0; 8740 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 8741 return fastEmitInst_rr(X86::VPADDBZ128rr, &X86::VR128XRegClass, Op0, Op1); 8742 } 8743 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 8744 return fastEmitInst_rr(X86::PADDBrr, &X86::VR128RegClass, Op0, Op1); 8745 } 8746 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 8747 return fastEmitInst_rr(X86::VPADDBrr, &X86::VR128RegClass, Op0, Op1); 8748 } 8749 return 0; 8750} 8751 8752unsigned fastEmit_ISD_ADD_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8753 if (RetVT.SimpleTy != MVT::v32i8) 8754 return 0; 8755 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 8756 return fastEmitInst_rr(X86::VPADDBZ256rr, &X86::VR256XRegClass, Op0, Op1); 8757 } 8758 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 8759 return fastEmitInst_rr(X86::VPADDBYrr, &X86::VR256RegClass, Op0, Op1); 8760 } 8761 return 0; 8762} 8763 8764unsigned fastEmit_ISD_ADD_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8765 if (RetVT.SimpleTy != MVT::v64i8) 8766 return 0; 8767 if ((Subtarget->hasBWI())) { 8768 return fastEmitInst_rr(X86::VPADDBZrr, &X86::VR512RegClass, Op0, Op1); 8769 } 8770 return 0; 8771} 8772 8773unsigned fastEmit_ISD_ADD_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8774 if (RetVT.SimpleTy != MVT::v8i16) 8775 return 0; 8776 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 8777 return fastEmitInst_rr(X86::VPADDWZ128rr, &X86::VR128XRegClass, Op0, Op1); 8778 } 8779 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 8780 return fastEmitInst_rr(X86::PADDWrr, &X86::VR128RegClass, Op0, Op1); 8781 } 8782 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 8783 return fastEmitInst_rr(X86::VPADDWrr, &X86::VR128RegClass, Op0, Op1); 8784 } 8785 return 0; 8786} 8787 8788unsigned fastEmit_ISD_ADD_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8789 if (RetVT.SimpleTy != MVT::v16i16) 8790 return 0; 8791 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 8792 return fastEmitInst_rr(X86::VPADDWZ256rr, &X86::VR256XRegClass, Op0, Op1); 8793 } 8794 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 8795 return fastEmitInst_rr(X86::VPADDWYrr, &X86::VR256RegClass, Op0, Op1); 8796 } 8797 return 0; 8798} 8799 8800unsigned fastEmit_ISD_ADD_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8801 if (RetVT.SimpleTy != MVT::v32i16) 8802 return 0; 8803 if ((Subtarget->hasBWI())) { 8804 return fastEmitInst_rr(X86::VPADDWZrr, &X86::VR512RegClass, Op0, Op1); 8805 } 8806 return 0; 8807} 8808 8809unsigned fastEmit_ISD_ADD_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8810 if (RetVT.SimpleTy != MVT::v4i32) 8811 return 0; 8812 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8813 return fastEmitInst_rr(X86::VPADDDZ128rr, &X86::VR128XRegClass, Op0, Op1); 8814 } 8815 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 8816 return fastEmitInst_rr(X86::PADDDrr, &X86::VR128RegClass, Op0, Op1); 8817 } 8818 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 8819 return fastEmitInst_rr(X86::VPADDDrr, &X86::VR128RegClass, Op0, Op1); 8820 } 8821 return 0; 8822} 8823 8824unsigned fastEmit_ISD_ADD_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8825 if (RetVT.SimpleTy != MVT::v8i32) 8826 return 0; 8827 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8828 return fastEmitInst_rr(X86::VPADDDZ256rr, &X86::VR256XRegClass, Op0, Op1); 8829 } 8830 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 8831 return fastEmitInst_rr(X86::VPADDDYrr, &X86::VR256RegClass, Op0, Op1); 8832 } 8833 return 0; 8834} 8835 8836unsigned fastEmit_ISD_ADD_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8837 if (RetVT.SimpleTy != MVT::v16i32) 8838 return 0; 8839 if ((Subtarget->hasAVX512())) { 8840 return fastEmitInst_rr(X86::VPADDDZrr, &X86::VR512RegClass, Op0, Op1); 8841 } 8842 return 0; 8843} 8844 8845unsigned fastEmit_ISD_ADD_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8846 if (RetVT.SimpleTy != MVT::v2i64) 8847 return 0; 8848 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8849 return fastEmitInst_rr(X86::VPADDQZ128rr, &X86::VR128XRegClass, Op0, Op1); 8850 } 8851 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 8852 return fastEmitInst_rr(X86::PADDQrr, &X86::VR128RegClass, Op0, Op1); 8853 } 8854 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 8855 return fastEmitInst_rr(X86::VPADDQrr, &X86::VR128RegClass, Op0, Op1); 8856 } 8857 return 0; 8858} 8859 8860unsigned fastEmit_ISD_ADD_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8861 if (RetVT.SimpleTy != MVT::v4i64) 8862 return 0; 8863 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8864 return fastEmitInst_rr(X86::VPADDQZ256rr, &X86::VR256XRegClass, Op0, Op1); 8865 } 8866 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 8867 return fastEmitInst_rr(X86::VPADDQYrr, &X86::VR256RegClass, Op0, Op1); 8868 } 8869 return 0; 8870} 8871 8872unsigned fastEmit_ISD_ADD_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8873 if (RetVT.SimpleTy != MVT::v8i64) 8874 return 0; 8875 if ((Subtarget->hasAVX512())) { 8876 return fastEmitInst_rr(X86::VPADDQZrr, &X86::VR512RegClass, Op0, Op1); 8877 } 8878 return 0; 8879} 8880 8881unsigned fastEmit_ISD_ADD_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 8882 switch (VT.SimpleTy) { 8883 case MVT::i8: return fastEmit_ISD_ADD_MVT_i8_rr(RetVT, Op0, Op1); 8884 case MVT::i16: return fastEmit_ISD_ADD_MVT_i16_rr(RetVT, Op0, Op1); 8885 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_rr(RetVT, Op0, Op1); 8886 case MVT::i64: return fastEmit_ISD_ADD_MVT_i64_rr(RetVT, Op0, Op1); 8887 case MVT::v16i8: return fastEmit_ISD_ADD_MVT_v16i8_rr(RetVT, Op0, Op1); 8888 case MVT::v32i8: return fastEmit_ISD_ADD_MVT_v32i8_rr(RetVT, Op0, Op1); 8889 case MVT::v64i8: return fastEmit_ISD_ADD_MVT_v64i8_rr(RetVT, Op0, Op1); 8890 case MVT::v8i16: return fastEmit_ISD_ADD_MVT_v8i16_rr(RetVT, Op0, Op1); 8891 case MVT::v16i16: return fastEmit_ISD_ADD_MVT_v16i16_rr(RetVT, Op0, Op1); 8892 case MVT::v32i16: return fastEmit_ISD_ADD_MVT_v32i16_rr(RetVT, Op0, Op1); 8893 case MVT::v4i32: return fastEmit_ISD_ADD_MVT_v4i32_rr(RetVT, Op0, Op1); 8894 case MVT::v8i32: return fastEmit_ISD_ADD_MVT_v8i32_rr(RetVT, Op0, Op1); 8895 case MVT::v16i32: return fastEmit_ISD_ADD_MVT_v16i32_rr(RetVT, Op0, Op1); 8896 case MVT::v2i64: return fastEmit_ISD_ADD_MVT_v2i64_rr(RetVT, Op0, Op1); 8897 case MVT::v4i64: return fastEmit_ISD_ADD_MVT_v4i64_rr(RetVT, Op0, Op1); 8898 case MVT::v8i64: return fastEmit_ISD_ADD_MVT_v8i64_rr(RetVT, Op0, Op1); 8899 default: return 0; 8900 } 8901} 8902 8903// FastEmit functions for ISD::AND. 8904 8905unsigned fastEmit_ISD_AND_MVT_i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8906 if (RetVT.SimpleTy != MVT::i8) 8907 return 0; 8908 return fastEmitInst_rr(X86::AND8rr, &X86::GR8RegClass, Op0, Op1); 8909} 8910 8911unsigned fastEmit_ISD_AND_MVT_i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8912 if (RetVT.SimpleTy != MVT::i16) 8913 return 0; 8914 return fastEmitInst_rr(X86::AND16rr, &X86::GR16RegClass, Op0, Op1); 8915} 8916 8917unsigned fastEmit_ISD_AND_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8918 if (RetVT.SimpleTy != MVT::i32) 8919 return 0; 8920 return fastEmitInst_rr(X86::AND32rr, &X86::GR32RegClass, Op0, Op1); 8921} 8922 8923unsigned fastEmit_ISD_AND_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8924 if (RetVT.SimpleTy != MVT::i64) 8925 return 0; 8926 return fastEmitInst_rr(X86::AND64rr, &X86::GR64RegClass, Op0, Op1); 8927} 8928 8929unsigned fastEmit_ISD_AND_MVT_v8i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8930 if (RetVT.SimpleTy != MVT::v8i1) 8931 return 0; 8932 if ((Subtarget->hasDQI())) { 8933 return fastEmitInst_rr(X86::KANDBrr, &X86::VK8RegClass, Op0, Op1); 8934 } 8935 return 0; 8936} 8937 8938unsigned fastEmit_ISD_AND_MVT_v16i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8939 if (RetVT.SimpleTy != MVT::v16i1) 8940 return 0; 8941 if ((Subtarget->hasAVX512())) { 8942 return fastEmitInst_rr(X86::KANDWrr, &X86::VK16RegClass, Op0, Op1); 8943 } 8944 return 0; 8945} 8946 8947unsigned fastEmit_ISD_AND_MVT_v32i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8948 if (RetVT.SimpleTy != MVT::v32i1) 8949 return 0; 8950 if ((Subtarget->hasBWI())) { 8951 return fastEmitInst_rr(X86::KANDDrr, &X86::VK32RegClass, Op0, Op1); 8952 } 8953 return 0; 8954} 8955 8956unsigned fastEmit_ISD_AND_MVT_v64i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8957 if (RetVT.SimpleTy != MVT::v64i1) 8958 return 0; 8959 if ((Subtarget->hasBWI())) { 8960 return fastEmitInst_rr(X86::KANDQrr, &X86::VK64RegClass, Op0, Op1); 8961 } 8962 return 0; 8963} 8964 8965unsigned fastEmit_ISD_AND_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8966 if (RetVT.SimpleTy != MVT::v16i8) 8967 return 0; 8968 if ((Subtarget->hasVLX())) { 8969 return fastEmitInst_rr(X86::VPANDQZ128rr, &X86::VR128XRegClass, Op0, Op1); 8970 } 8971 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 8972 return fastEmitInst_rr(X86::PANDrr, &X86::VR128RegClass, Op0, Op1); 8973 } 8974 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 8975 return fastEmitInst_rr(X86::VPANDrr, &X86::VR128RegClass, Op0, Op1); 8976 } 8977 return 0; 8978} 8979 8980unsigned fastEmit_ISD_AND_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8981 if (RetVT.SimpleTy != MVT::v32i8) 8982 return 0; 8983 if ((Subtarget->hasVLX())) { 8984 return fastEmitInst_rr(X86::VPANDQZ256rr, &X86::VR256XRegClass, Op0, Op1); 8985 } 8986 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) { 8987 return fastEmitInst_rr(X86::VANDPSYrr, &X86::VR256RegClass, Op0, Op1); 8988 } 8989 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 8990 return fastEmitInst_rr(X86::VPANDYrr, &X86::VR256RegClass, Op0, Op1); 8991 } 8992 return 0; 8993} 8994 8995unsigned fastEmit_ISD_AND_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8996 if (RetVT.SimpleTy != MVT::v64i8) 8997 return 0; 8998 if ((Subtarget->hasAVX512())) { 8999 return fastEmitInst_rr(X86::VPANDQZrr, &X86::VR512RegClass, Op0, Op1); 9000 } 9001 return 0; 9002} 9003 9004unsigned fastEmit_ISD_AND_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9005 if (RetVT.SimpleTy != MVT::v8i16) 9006 return 0; 9007 if ((Subtarget->hasVLX())) { 9008 return fastEmitInst_rr(X86::VPANDQZ128rr, &X86::VR128XRegClass, Op0, Op1); 9009 } 9010 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 9011 return fastEmitInst_rr(X86::PANDrr, &X86::VR128RegClass, Op0, Op1); 9012 } 9013 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 9014 return fastEmitInst_rr(X86::VPANDrr, &X86::VR128RegClass, Op0, Op1); 9015 } 9016 return 0; 9017} 9018 9019unsigned fastEmit_ISD_AND_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9020 if (RetVT.SimpleTy != MVT::v16i16) 9021 return 0; 9022 if ((Subtarget->hasVLX())) { 9023 return fastEmitInst_rr(X86::VPANDQZ256rr, &X86::VR256XRegClass, Op0, Op1); 9024 } 9025 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) { 9026 return fastEmitInst_rr(X86::VANDPSYrr, &X86::VR256RegClass, Op0, Op1); 9027 } 9028 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 9029 return fastEmitInst_rr(X86::VPANDYrr, &X86::VR256RegClass, Op0, Op1); 9030 } 9031 return 0; 9032} 9033 9034unsigned fastEmit_ISD_AND_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9035 if (RetVT.SimpleTy != MVT::v32i16) 9036 return 0; 9037 if ((Subtarget->hasAVX512())) { 9038 return fastEmitInst_rr(X86::VPANDQZrr, &X86::VR512RegClass, Op0, Op1); 9039 } 9040 return 0; 9041} 9042 9043unsigned fastEmit_ISD_AND_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9044 if (RetVT.SimpleTy != MVT::v4i32) 9045 return 0; 9046 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 9047 return fastEmitInst_rr(X86::PANDrr, &X86::VR128RegClass, Op0, Op1); 9048 } 9049 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 9050 return fastEmitInst_rr(X86::VPANDrr, &X86::VR128RegClass, Op0, Op1); 9051 } 9052 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 9053 return fastEmitInst_rr(X86::VPANDDZ128rr, &X86::VR128XRegClass, Op0, Op1); 9054 } 9055 return 0; 9056} 9057 9058unsigned fastEmit_ISD_AND_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9059 if (RetVT.SimpleTy != MVT::v8i32) 9060 return 0; 9061 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) { 9062 return fastEmitInst_rr(X86::VANDPSYrr, &X86::VR256RegClass, Op0, Op1); 9063 } 9064 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 9065 return fastEmitInst_rr(X86::VPANDYrr, &X86::VR256RegClass, Op0, Op1); 9066 } 9067 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 9068 return fastEmitInst_rr(X86::VPANDDZ256rr, &X86::VR256XRegClass, Op0, Op1); 9069 } 9070 return 0; 9071} 9072 9073unsigned fastEmit_ISD_AND_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9074 if (RetVT.SimpleTy != MVT::v16i32) 9075 return 0; 9076 if ((Subtarget->hasAVX512())) { 9077 return fastEmitInst_rr(X86::VPANDDZrr, &X86::VR512RegClass, Op0, Op1); 9078 } 9079 return 0; 9080} 9081 9082unsigned fastEmit_ISD_AND_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9083 if (RetVT.SimpleTy != MVT::v2i64) 9084 return 0; 9085 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 9086 return fastEmitInst_rr(X86::VPANDQZ128rr, &X86::VR128XRegClass, Op0, Op1); 9087 } 9088 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 9089 return fastEmitInst_rr(X86::PANDrr, &X86::VR128RegClass, Op0, Op1); 9090 } 9091 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 9092 return fastEmitInst_rr(X86::VPANDrr, &X86::VR128RegClass, Op0, Op1); 9093 } 9094 return 0; 9095} 9096 9097unsigned fastEmit_ISD_AND_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9098 if (RetVT.SimpleTy != MVT::v4i64) 9099 return 0; 9100 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) { 9101 return fastEmitInst_rr(X86::VANDPSYrr, &X86::VR256RegClass, Op0, Op1); 9102 } 9103 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 9104 return fastEmitInst_rr(X86::VPANDQZ256rr, &X86::VR256XRegClass, Op0, Op1); 9105 } 9106 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 9107 return fastEmitInst_rr(X86::VPANDYrr, &X86::VR256RegClass, Op0, Op1); 9108 } 9109 return 0; 9110} 9111 9112unsigned fastEmit_ISD_AND_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9113 if (RetVT.SimpleTy != MVT::v8i64) 9114 return 0; 9115 if ((Subtarget->hasAVX512())) { 9116 return fastEmitInst_rr(X86::VPANDQZrr, &X86::VR512RegClass, Op0, Op1); 9117 } 9118 return 0; 9119} 9120 9121unsigned fastEmit_ISD_AND_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 9122 switch (VT.SimpleTy) { 9123 case MVT::i8: return fastEmit_ISD_AND_MVT_i8_rr(RetVT, Op0, Op1); 9124 case MVT::i16: return fastEmit_ISD_AND_MVT_i16_rr(RetVT, Op0, Op1); 9125 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_rr(RetVT, Op0, Op1); 9126 case MVT::i64: return fastEmit_ISD_AND_MVT_i64_rr(RetVT, Op0, Op1); 9127 case MVT::v8i1: return fastEmit_ISD_AND_MVT_v8i1_rr(RetVT, Op0, Op1); 9128 case MVT::v16i1: return fastEmit_ISD_AND_MVT_v16i1_rr(RetVT, Op0, Op1); 9129 case MVT::v32i1: return fastEmit_ISD_AND_MVT_v32i1_rr(RetVT, Op0, Op1); 9130 case MVT::v64i1: return fastEmit_ISD_AND_MVT_v64i1_rr(RetVT, Op0, Op1); 9131 case MVT::v16i8: return fastEmit_ISD_AND_MVT_v16i8_rr(RetVT, Op0, Op1); 9132 case MVT::v32i8: return fastEmit_ISD_AND_MVT_v32i8_rr(RetVT, Op0, Op1); 9133 case MVT::v64i8: return fastEmit_ISD_AND_MVT_v64i8_rr(RetVT, Op0, Op1); 9134 case MVT::v8i16: return fastEmit_ISD_AND_MVT_v8i16_rr(RetVT, Op0, Op1); 9135 case MVT::v16i16: return fastEmit_ISD_AND_MVT_v16i16_rr(RetVT, Op0, Op1); 9136 case MVT::v32i16: return fastEmit_ISD_AND_MVT_v32i16_rr(RetVT, Op0, Op1); 9137 case MVT::v4i32: return fastEmit_ISD_AND_MVT_v4i32_rr(RetVT, Op0, Op1); 9138 case MVT::v8i32: return fastEmit_ISD_AND_MVT_v8i32_rr(RetVT, Op0, Op1); 9139 case MVT::v16i32: return fastEmit_ISD_AND_MVT_v16i32_rr(RetVT, Op0, Op1); 9140 case MVT::v2i64: return fastEmit_ISD_AND_MVT_v2i64_rr(RetVT, Op0, Op1); 9141 case MVT::v4i64: return fastEmit_ISD_AND_MVT_v4i64_rr(RetVT, Op0, Op1); 9142 case MVT::v8i64: return fastEmit_ISD_AND_MVT_v8i64_rr(RetVT, Op0, Op1); 9143 default: return 0; 9144 } 9145} 9146 9147// FastEmit functions for ISD::AVGCEILU. 9148 9149unsigned fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9150 if (RetVT.SimpleTy != MVT::v16i8) 9151 return 0; 9152 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 9153 return fastEmitInst_rr(X86::VPAVGBZ128rr, &X86::VR128XRegClass, Op0, Op1); 9154 } 9155 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 9156 return fastEmitInst_rr(X86::PAVGBrr, &X86::VR128RegClass, Op0, Op1); 9157 } 9158 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 9159 return fastEmitInst_rr(X86::VPAVGBrr, &X86::VR128RegClass, Op0, Op1); 9160 } 9161 return 0; 9162} 9163 9164unsigned fastEmit_ISD_AVGCEILU_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9165 if (RetVT.SimpleTy != MVT::v32i8) 9166 return 0; 9167 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 9168 return fastEmitInst_rr(X86::VPAVGBZ256rr, &X86::VR256XRegClass, Op0, Op1); 9169 } 9170 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 9171 return fastEmitInst_rr(X86::VPAVGBYrr, &X86::VR256RegClass, Op0, Op1); 9172 } 9173 return 0; 9174} 9175 9176unsigned fastEmit_ISD_AVGCEILU_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9177 if (RetVT.SimpleTy != MVT::v64i8) 9178 return 0; 9179 if ((Subtarget->hasBWI())) { 9180 return fastEmitInst_rr(X86::VPAVGBZrr, &X86::VR512RegClass, Op0, Op1); 9181 } 9182 return 0; 9183} 9184 9185unsigned fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9186 if (RetVT.SimpleTy != MVT::v8i16) 9187 return 0; 9188 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 9189 return fastEmitInst_rr(X86::VPAVGWZ128rr, &X86::VR128XRegClass, Op0, Op1); 9190 } 9191 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 9192 return fastEmitInst_rr(X86::PAVGWrr, &X86::VR128RegClass, Op0, Op1); 9193 } 9194 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 9195 return fastEmitInst_rr(X86::VPAVGWrr, &X86::VR128RegClass, Op0, Op1); 9196 } 9197 return 0; 9198} 9199 9200unsigned fastEmit_ISD_AVGCEILU_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9201 if (RetVT.SimpleTy != MVT::v16i16) 9202 return 0; 9203 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 9204 return fastEmitInst_rr(X86::VPAVGWZ256rr, &X86::VR256XRegClass, Op0, Op1); 9205 } 9206 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 9207 return fastEmitInst_rr(X86::VPAVGWYrr, &X86::VR256RegClass, Op0, Op1); 9208 } 9209 return 0; 9210} 9211 9212unsigned fastEmit_ISD_AVGCEILU_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9213 if (RetVT.SimpleTy != MVT::v32i16) 9214 return 0; 9215 if ((Subtarget->hasBWI())) { 9216 return fastEmitInst_rr(X86::VPAVGWZrr, &X86::VR512RegClass, Op0, Op1); 9217 } 9218 return 0; 9219} 9220 9221unsigned fastEmit_ISD_AVGCEILU_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 9222 switch (VT.SimpleTy) { 9223 case MVT::v16i8: return fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(RetVT, Op0, Op1); 9224 case MVT::v32i8: return fastEmit_ISD_AVGCEILU_MVT_v32i8_rr(RetVT, Op0, Op1); 9225 case MVT::v64i8: return fastEmit_ISD_AVGCEILU_MVT_v64i8_rr(RetVT, Op0, Op1); 9226 case MVT::v8i16: return fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(RetVT, Op0, Op1); 9227 case MVT::v16i16: return fastEmit_ISD_AVGCEILU_MVT_v16i16_rr(RetVT, Op0, Op1); 9228 case MVT::v32i16: return fastEmit_ISD_AVGCEILU_MVT_v32i16_rr(RetVT, Op0, Op1); 9229 default: return 0; 9230 } 9231} 9232 9233// FastEmit functions for ISD::FADD. 9234 9235unsigned fastEmit_ISD_FADD_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9236 if (RetVT.SimpleTy != MVT::f16) 9237 return 0; 9238 if ((Subtarget->hasFP16())) { 9239 return fastEmitInst_rr(X86::VADDSHZrr, &X86::FR16XRegClass, Op0, Op1); 9240 } 9241 return 0; 9242} 9243 9244unsigned fastEmit_ISD_FADD_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9245 if (RetVT.SimpleTy != MVT::f32) 9246 return 0; 9247 if ((Subtarget->hasAVX512())) { 9248 return fastEmitInst_rr(X86::VADDSSZrr, &X86::FR32XRegClass, Op0, Op1); 9249 } 9250 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 9251 return fastEmitInst_rr(X86::ADDSSrr, &X86::FR32RegClass, Op0, Op1); 9252 } 9253 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 9254 return fastEmitInst_rr(X86::VADDSSrr, &X86::FR32RegClass, Op0, Op1); 9255 } 9256 if ((!Subtarget->hasSSE1())) { 9257 return fastEmitInst_rr(X86::ADD_Fp32, &X86::RFP32RegClass, Op0, Op1); 9258 } 9259 return 0; 9260} 9261 9262unsigned fastEmit_ISD_FADD_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9263 if (RetVT.SimpleTy != MVT::f64) 9264 return 0; 9265 if ((Subtarget->hasAVX512())) { 9266 return fastEmitInst_rr(X86::VADDSDZrr, &X86::FR64XRegClass, Op0, Op1); 9267 } 9268 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 9269 return fastEmitInst_rr(X86::ADDSDrr, &X86::FR64RegClass, Op0, Op1); 9270 } 9271 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 9272 return fastEmitInst_rr(X86::VADDSDrr, &X86::FR64RegClass, Op0, Op1); 9273 } 9274 if ((!Subtarget->hasSSE2())) { 9275 return fastEmitInst_rr(X86::ADD_Fp64, &X86::RFP64RegClass, Op0, Op1); 9276 } 9277 return 0; 9278} 9279 9280unsigned fastEmit_ISD_FADD_MVT_f80_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9281 if (RetVT.SimpleTy != MVT::f80) 9282 return 0; 9283 if ((Subtarget->hasX87())) { 9284 return fastEmitInst_rr(X86::ADD_Fp80, &X86::RFP80RegClass, Op0, Op1); 9285 } 9286 return 0; 9287} 9288 9289unsigned fastEmit_ISD_FADD_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9290 if (RetVT.SimpleTy != MVT::v8f16) 9291 return 0; 9292 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 9293 return fastEmitInst_rr(X86::VADDPHZ128rr, &X86::VR128XRegClass, Op0, Op1); 9294 } 9295 return 0; 9296} 9297 9298unsigned fastEmit_ISD_FADD_MVT_v16f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9299 if (RetVT.SimpleTy != MVT::v16f16) 9300 return 0; 9301 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 9302 return fastEmitInst_rr(X86::VADDPHZ256rr, &X86::VR256XRegClass, Op0, Op1); 9303 } 9304 return 0; 9305} 9306 9307unsigned fastEmit_ISD_FADD_MVT_v32f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9308 if (RetVT.SimpleTy != MVT::v32f16) 9309 return 0; 9310 if ((Subtarget->hasFP16())) { 9311 return fastEmitInst_rr(X86::VADDPHZrr, &X86::VR512RegClass, Op0, Op1); 9312 } 9313 return 0; 9314} 9315 9316unsigned fastEmit_ISD_FADD_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9317 if (RetVT.SimpleTy != MVT::v4f32) 9318 return 0; 9319 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 9320 return fastEmitInst_rr(X86::VADDPSZ128rr, &X86::VR128XRegClass, Op0, Op1); 9321 } 9322 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 9323 return fastEmitInst_rr(X86::ADDPSrr, &X86::VR128RegClass, Op0, Op1); 9324 } 9325 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 9326 return fastEmitInst_rr(X86::VADDPSrr, &X86::VR128RegClass, Op0, Op1); 9327 } 9328 return 0; 9329} 9330 9331unsigned fastEmit_ISD_FADD_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9332 if (RetVT.SimpleTy != MVT::v8f32) 9333 return 0; 9334 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 9335 return fastEmitInst_rr(X86::VADDPSZ256rr, &X86::VR256XRegClass, Op0, Op1); 9336 } 9337 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 9338 return fastEmitInst_rr(X86::VADDPSYrr, &X86::VR256RegClass, Op0, Op1); 9339 } 9340 return 0; 9341} 9342 9343unsigned fastEmit_ISD_FADD_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9344 if (RetVT.SimpleTy != MVT::v16f32) 9345 return 0; 9346 if ((Subtarget->hasAVX512())) { 9347 return fastEmitInst_rr(X86::VADDPSZrr, &X86::VR512RegClass, Op0, Op1); 9348 } 9349 return 0; 9350} 9351 9352unsigned fastEmit_ISD_FADD_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9353 if (RetVT.SimpleTy != MVT::v2f64) 9354 return 0; 9355 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 9356 return fastEmitInst_rr(X86::VADDPDZ128rr, &X86::VR128XRegClass, Op0, Op1); 9357 } 9358 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 9359 return fastEmitInst_rr(X86::ADDPDrr, &X86::VR128RegClass, Op0, Op1); 9360 } 9361 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 9362 return fastEmitInst_rr(X86::VADDPDrr, &X86::VR128RegClass, Op0, Op1); 9363 } 9364 return 0; 9365} 9366 9367unsigned fastEmit_ISD_FADD_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9368 if (RetVT.SimpleTy != MVT::v4f64) 9369 return 0; 9370 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 9371 return fastEmitInst_rr(X86::VADDPDZ256rr, &X86::VR256XRegClass, Op0, Op1); 9372 } 9373 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 9374 return fastEmitInst_rr(X86::VADDPDYrr, &X86::VR256RegClass, Op0, Op1); 9375 } 9376 return 0; 9377} 9378 9379unsigned fastEmit_ISD_FADD_MVT_v8f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9380 if (RetVT.SimpleTy != MVT::v8f64) 9381 return 0; 9382 if ((Subtarget->hasAVX512())) { 9383 return fastEmitInst_rr(X86::VADDPDZrr, &X86::VR512RegClass, Op0, Op1); 9384 } 9385 return 0; 9386} 9387 9388unsigned fastEmit_ISD_FADD_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 9389 switch (VT.SimpleTy) { 9390 case MVT::f16: return fastEmit_ISD_FADD_MVT_f16_rr(RetVT, Op0, Op1); 9391 case MVT::f32: return fastEmit_ISD_FADD_MVT_f32_rr(RetVT, Op0, Op1); 9392 case MVT::f64: return fastEmit_ISD_FADD_MVT_f64_rr(RetVT, Op0, Op1); 9393 case MVT::f80: return fastEmit_ISD_FADD_MVT_f80_rr(RetVT, Op0, Op1); 9394 case MVT::v8f16: return fastEmit_ISD_FADD_MVT_v8f16_rr(RetVT, Op0, Op1); 9395 case MVT::v16f16: return fastEmit_ISD_FADD_MVT_v16f16_rr(RetVT, Op0, Op1); 9396 case MVT::v32f16: return fastEmit_ISD_FADD_MVT_v32f16_rr(RetVT, Op0, Op1); 9397 case MVT::v4f32: return fastEmit_ISD_FADD_MVT_v4f32_rr(RetVT, Op0, Op1); 9398 case MVT::v8f32: return fastEmit_ISD_FADD_MVT_v8f32_rr(RetVT, Op0, Op1); 9399 case MVT::v16f32: return fastEmit_ISD_FADD_MVT_v16f32_rr(RetVT, Op0, Op1); 9400 case MVT::v2f64: return fastEmit_ISD_FADD_MVT_v2f64_rr(RetVT, Op0, Op1); 9401 case MVT::v4f64: return fastEmit_ISD_FADD_MVT_v4f64_rr(RetVT, Op0, Op1); 9402 case MVT::v8f64: return fastEmit_ISD_FADD_MVT_v8f64_rr(RetVT, Op0, Op1); 9403 default: return 0; 9404 } 9405} 9406 9407// FastEmit functions for ISD::FDIV. 9408 9409unsigned fastEmit_ISD_FDIV_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9410 if (RetVT.SimpleTy != MVT::f16) 9411 return 0; 9412 if ((Subtarget->hasFP16())) { 9413 return fastEmitInst_rr(X86::VDIVSHZrr, &X86::FR16XRegClass, Op0, Op1); 9414 } 9415 return 0; 9416} 9417 9418unsigned fastEmit_ISD_FDIV_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9419 if (RetVT.SimpleTy != MVT::f32) 9420 return 0; 9421 if ((Subtarget->hasAVX512())) { 9422 return fastEmitInst_rr(X86::VDIVSSZrr, &X86::FR32XRegClass, Op0, Op1); 9423 } 9424 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 9425 return fastEmitInst_rr(X86::DIVSSrr, &X86::FR32RegClass, Op0, Op1); 9426 } 9427 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 9428 return fastEmitInst_rr(X86::VDIVSSrr, &X86::FR32RegClass, Op0, Op1); 9429 } 9430 if ((!Subtarget->hasSSE1())) { 9431 return fastEmitInst_rr(X86::DIV_Fp32, &X86::RFP32RegClass, Op0, Op1); 9432 } 9433 return 0; 9434} 9435 9436unsigned fastEmit_ISD_FDIV_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9437 if (RetVT.SimpleTy != MVT::f64) 9438 return 0; 9439 if ((Subtarget->hasAVX512())) { 9440 return fastEmitInst_rr(X86::VDIVSDZrr, &X86::FR64XRegClass, Op0, Op1); 9441 } 9442 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 9443 return fastEmitInst_rr(X86::DIVSDrr, &X86::FR64RegClass, Op0, Op1); 9444 } 9445 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 9446 return fastEmitInst_rr(X86::VDIVSDrr, &X86::FR64RegClass, Op0, Op1); 9447 } 9448 if ((!Subtarget->hasSSE2())) { 9449 return fastEmitInst_rr(X86::DIV_Fp64, &X86::RFP64RegClass, Op0, Op1); 9450 } 9451 return 0; 9452} 9453 9454unsigned fastEmit_ISD_FDIV_MVT_f80_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9455 if (RetVT.SimpleTy != MVT::f80) 9456 return 0; 9457 if ((Subtarget->hasX87())) { 9458 return fastEmitInst_rr(X86::DIV_Fp80, &X86::RFP80RegClass, Op0, Op1); 9459 } 9460 return 0; 9461} 9462 9463unsigned fastEmit_ISD_FDIV_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9464 if (RetVT.SimpleTy != MVT::v8f16) 9465 return 0; 9466 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 9467 return fastEmitInst_rr(X86::VDIVPHZ128rr, &X86::VR128XRegClass, Op0, Op1); 9468 } 9469 return 0; 9470} 9471 9472unsigned fastEmit_ISD_FDIV_MVT_v16f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9473 if (RetVT.SimpleTy != MVT::v16f16) 9474 return 0; 9475 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 9476 return fastEmitInst_rr(X86::VDIVPHZ256rr, &X86::VR256XRegClass, Op0, Op1); 9477 } 9478 return 0; 9479} 9480 9481unsigned fastEmit_ISD_FDIV_MVT_v32f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9482 if (RetVT.SimpleTy != MVT::v32f16) 9483 return 0; 9484 if ((Subtarget->hasFP16())) { 9485 return fastEmitInst_rr(X86::VDIVPHZrr, &X86::VR512RegClass, Op0, Op1); 9486 } 9487 return 0; 9488} 9489 9490unsigned fastEmit_ISD_FDIV_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9491 if (RetVT.SimpleTy != MVT::v4f32) 9492 return 0; 9493 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 9494 return fastEmitInst_rr(X86::VDIVPSZ128rr, &X86::VR128XRegClass, Op0, Op1); 9495 } 9496 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 9497 return fastEmitInst_rr(X86::DIVPSrr, &X86::VR128RegClass, Op0, Op1); 9498 } 9499 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 9500 return fastEmitInst_rr(X86::VDIVPSrr, &X86::VR128RegClass, Op0, Op1); 9501 } 9502 return 0; 9503} 9504 9505unsigned fastEmit_ISD_FDIV_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9506 if (RetVT.SimpleTy != MVT::v8f32) 9507 return 0; 9508 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 9509 return fastEmitInst_rr(X86::VDIVPSZ256rr, &X86::VR256XRegClass, Op0, Op1); 9510 } 9511 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 9512 return fastEmitInst_rr(X86::VDIVPSYrr, &X86::VR256RegClass, Op0, Op1); 9513 } 9514 return 0; 9515} 9516 9517unsigned fastEmit_ISD_FDIV_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9518 if (RetVT.SimpleTy != MVT::v16f32) 9519 return 0; 9520 if ((Subtarget->hasAVX512())) { 9521 return fastEmitInst_rr(X86::VDIVPSZrr, &X86::VR512RegClass, Op0, Op1); 9522 } 9523 return 0; 9524} 9525 9526unsigned fastEmit_ISD_FDIV_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9527 if (RetVT.SimpleTy != MVT::v2f64) 9528 return 0; 9529 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 9530 return fastEmitInst_rr(X86::VDIVPDZ128rr, &X86::VR128XRegClass, Op0, Op1); 9531 } 9532 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 9533 return fastEmitInst_rr(X86::DIVPDrr, &X86::VR128RegClass, Op0, Op1); 9534 } 9535 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 9536 return fastEmitInst_rr(X86::VDIVPDrr, &X86::VR128RegClass, Op0, Op1); 9537 } 9538 return 0; 9539} 9540 9541unsigned fastEmit_ISD_FDIV_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9542 if (RetVT.SimpleTy != MVT::v4f64) 9543 return 0; 9544 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 9545 return fastEmitInst_rr(X86::VDIVPDZ256rr, &X86::VR256XRegClass, Op0, Op1); 9546 } 9547 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 9548 return fastEmitInst_rr(X86::VDIVPDYrr, &X86::VR256RegClass, Op0, Op1); 9549 } 9550 return 0; 9551} 9552 9553unsigned fastEmit_ISD_FDIV_MVT_v8f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9554 if (RetVT.SimpleTy != MVT::v8f64) 9555 return 0; 9556 if ((Subtarget->hasAVX512())) { 9557 return fastEmitInst_rr(X86::VDIVPDZrr, &X86::VR512RegClass, Op0, Op1); 9558 } 9559 return 0; 9560} 9561 9562unsigned fastEmit_ISD_FDIV_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 9563 switch (VT.SimpleTy) { 9564 case MVT::f16: return fastEmit_ISD_FDIV_MVT_f16_rr(RetVT, Op0, Op1); 9565 case MVT::f32: return fastEmit_ISD_FDIV_MVT_f32_rr(RetVT, Op0, Op1); 9566 case MVT::f64: return fastEmit_ISD_FDIV_MVT_f64_rr(RetVT, Op0, Op1); 9567 case MVT::f80: return fastEmit_ISD_FDIV_MVT_f80_rr(RetVT, Op0, Op1); 9568 case MVT::v8f16: return fastEmit_ISD_FDIV_MVT_v8f16_rr(RetVT, Op0, Op1); 9569 case MVT::v16f16: return fastEmit_ISD_FDIV_MVT_v16f16_rr(RetVT, Op0, Op1); 9570 case MVT::v32f16: return fastEmit_ISD_FDIV_MVT_v32f16_rr(RetVT, Op0, Op1); 9571 case MVT::v4f32: return fastEmit_ISD_FDIV_MVT_v4f32_rr(RetVT, Op0, Op1); 9572 case MVT::v8f32: return fastEmit_ISD_FDIV_MVT_v8f32_rr(RetVT, Op0, Op1); 9573 case MVT::v16f32: return fastEmit_ISD_FDIV_MVT_v16f32_rr(RetVT, Op0, Op1); 9574 case MVT::v2f64: return fastEmit_ISD_FDIV_MVT_v2f64_rr(RetVT, Op0, Op1); 9575 case MVT::v4f64: return fastEmit_ISD_FDIV_MVT_v4f64_rr(RetVT, Op0, Op1); 9576 case MVT::v8f64: return fastEmit_ISD_FDIV_MVT_v8f64_rr(RetVT, Op0, Op1); 9577 default: return 0; 9578 } 9579} 9580 9581// FastEmit functions for ISD::FMUL. 9582 9583unsigned fastEmit_ISD_FMUL_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9584 if (RetVT.SimpleTy != MVT::f16) 9585 return 0; 9586 if ((Subtarget->hasFP16())) { 9587 return fastEmitInst_rr(X86::VMULSHZrr, &X86::FR16XRegClass, Op0, Op1); 9588 } 9589 return 0; 9590} 9591 9592unsigned fastEmit_ISD_FMUL_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9593 if (RetVT.SimpleTy != MVT::f32) 9594 return 0; 9595 if ((Subtarget->hasAVX512())) { 9596 return fastEmitInst_rr(X86::VMULSSZrr, &X86::FR32XRegClass, Op0, Op1); 9597 } 9598 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 9599 return fastEmitInst_rr(X86::MULSSrr, &X86::FR32RegClass, Op0, Op1); 9600 } 9601 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 9602 return fastEmitInst_rr(X86::VMULSSrr, &X86::FR32RegClass, Op0, Op1); 9603 } 9604 if ((!Subtarget->hasSSE1())) { 9605 return fastEmitInst_rr(X86::MUL_Fp32, &X86::RFP32RegClass, Op0, Op1); 9606 } 9607 return 0; 9608} 9609 9610unsigned fastEmit_ISD_FMUL_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9611 if (RetVT.SimpleTy != MVT::f64) 9612 return 0; 9613 if ((Subtarget->hasAVX512())) { 9614 return fastEmitInst_rr(X86::VMULSDZrr, &X86::FR64XRegClass, Op0, Op1); 9615 } 9616 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 9617 return fastEmitInst_rr(X86::MULSDrr, &X86::FR64RegClass, Op0, Op1); 9618 } 9619 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 9620 return fastEmitInst_rr(X86::VMULSDrr, &X86::FR64RegClass, Op0, Op1); 9621 } 9622 if ((!Subtarget->hasSSE2())) { 9623 return fastEmitInst_rr(X86::MUL_Fp64, &X86::RFP64RegClass, Op0, Op1); 9624 } 9625 return 0; 9626} 9627 9628unsigned fastEmit_ISD_FMUL_MVT_f80_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9629 if (RetVT.SimpleTy != MVT::f80) 9630 return 0; 9631 if ((Subtarget->hasX87())) { 9632 return fastEmitInst_rr(X86::MUL_Fp80, &X86::RFP80RegClass, Op0, Op1); 9633 } 9634 return 0; 9635} 9636 9637unsigned fastEmit_ISD_FMUL_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9638 if (RetVT.SimpleTy != MVT::v8f16) 9639 return 0; 9640 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 9641 return fastEmitInst_rr(X86::VMULPHZ128rr, &X86::VR128XRegClass, Op0, Op1); 9642 } 9643 return 0; 9644} 9645 9646unsigned fastEmit_ISD_FMUL_MVT_v16f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9647 if (RetVT.SimpleTy != MVT::v16f16) 9648 return 0; 9649 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 9650 return fastEmitInst_rr(X86::VMULPHZ256rr, &X86::VR256XRegClass, Op0, Op1); 9651 } 9652 return 0; 9653} 9654 9655unsigned fastEmit_ISD_FMUL_MVT_v32f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9656 if (RetVT.SimpleTy != MVT::v32f16) 9657 return 0; 9658 if ((Subtarget->hasFP16())) { 9659 return fastEmitInst_rr(X86::VMULPHZrr, &X86::VR512RegClass, Op0, Op1); 9660 } 9661 return 0; 9662} 9663 9664unsigned fastEmit_ISD_FMUL_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9665 if (RetVT.SimpleTy != MVT::v4f32) 9666 return 0; 9667 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 9668 return fastEmitInst_rr(X86::VMULPSZ128rr, &X86::VR128XRegClass, Op0, Op1); 9669 } 9670 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 9671 return fastEmitInst_rr(X86::MULPSrr, &X86::VR128RegClass, Op0, Op1); 9672 } 9673 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 9674 return fastEmitInst_rr(X86::VMULPSrr, &X86::VR128RegClass, Op0, Op1); 9675 } 9676 return 0; 9677} 9678 9679unsigned fastEmit_ISD_FMUL_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9680 if (RetVT.SimpleTy != MVT::v8f32) 9681 return 0; 9682 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 9683 return fastEmitInst_rr(X86::VMULPSZ256rr, &X86::VR256XRegClass, Op0, Op1); 9684 } 9685 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 9686 return fastEmitInst_rr(X86::VMULPSYrr, &X86::VR256RegClass, Op0, Op1); 9687 } 9688 return 0; 9689} 9690 9691unsigned fastEmit_ISD_FMUL_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9692 if (RetVT.SimpleTy != MVT::v16f32) 9693 return 0; 9694 if ((Subtarget->hasAVX512())) { 9695 return fastEmitInst_rr(X86::VMULPSZrr, &X86::VR512RegClass, Op0, Op1); 9696 } 9697 return 0; 9698} 9699 9700unsigned fastEmit_ISD_FMUL_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9701 if (RetVT.SimpleTy != MVT::v2f64) 9702 return 0; 9703 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 9704 return fastEmitInst_rr(X86::VMULPDZ128rr, &X86::VR128XRegClass, Op0, Op1); 9705 } 9706 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 9707 return fastEmitInst_rr(X86::MULPDrr, &X86::VR128RegClass, Op0, Op1); 9708 } 9709 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 9710 return fastEmitInst_rr(X86::VMULPDrr, &X86::VR128RegClass, Op0, Op1); 9711 } 9712 return 0; 9713} 9714 9715unsigned fastEmit_ISD_FMUL_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9716 if (RetVT.SimpleTy != MVT::v4f64) 9717 return 0; 9718 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 9719 return fastEmitInst_rr(X86::VMULPDZ256rr, &X86::VR256XRegClass, Op0, Op1); 9720 } 9721 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 9722 return fastEmitInst_rr(X86::VMULPDYrr, &X86::VR256RegClass, Op0, Op1); 9723 } 9724 return 0; 9725} 9726 9727unsigned fastEmit_ISD_FMUL_MVT_v8f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9728 if (RetVT.SimpleTy != MVT::v8f64) 9729 return 0; 9730 if ((Subtarget->hasAVX512())) { 9731 return fastEmitInst_rr(X86::VMULPDZrr, &X86::VR512RegClass, Op0, Op1); 9732 } 9733 return 0; 9734} 9735 9736unsigned fastEmit_ISD_FMUL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 9737 switch (VT.SimpleTy) { 9738 case MVT::f16: return fastEmit_ISD_FMUL_MVT_f16_rr(RetVT, Op0, Op1); 9739 case MVT::f32: return fastEmit_ISD_FMUL_MVT_f32_rr(RetVT, Op0, Op1); 9740 case MVT::f64: return fastEmit_ISD_FMUL_MVT_f64_rr(RetVT, Op0, Op1); 9741 case MVT::f80: return fastEmit_ISD_FMUL_MVT_f80_rr(RetVT, Op0, Op1); 9742 case MVT::v8f16: return fastEmit_ISD_FMUL_MVT_v8f16_rr(RetVT, Op0, Op1); 9743 case MVT::v16f16: return fastEmit_ISD_FMUL_MVT_v16f16_rr(RetVT, Op0, Op1); 9744 case MVT::v32f16: return fastEmit_ISD_FMUL_MVT_v32f16_rr(RetVT, Op0, Op1); 9745 case MVT::v4f32: return fastEmit_ISD_FMUL_MVT_v4f32_rr(RetVT, Op0, Op1); 9746 case MVT::v8f32: return fastEmit_ISD_FMUL_MVT_v8f32_rr(RetVT, Op0, Op1); 9747 case MVT::v16f32: return fastEmit_ISD_FMUL_MVT_v16f32_rr(RetVT, Op0, Op1); 9748 case MVT::v2f64: return fastEmit_ISD_FMUL_MVT_v2f64_rr(RetVT, Op0, Op1); 9749 case MVT::v4f64: return fastEmit_ISD_FMUL_MVT_v4f64_rr(RetVT, Op0, Op1); 9750 case MVT::v8f64: return fastEmit_ISD_FMUL_MVT_v8f64_rr(RetVT, Op0, Op1); 9751 default: return 0; 9752 } 9753} 9754 9755// FastEmit functions for ISD::FSUB. 9756 9757unsigned fastEmit_ISD_FSUB_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9758 if (RetVT.SimpleTy != MVT::f16) 9759 return 0; 9760 if ((Subtarget->hasFP16())) { 9761 return fastEmitInst_rr(X86::VSUBSHZrr, &X86::FR16XRegClass, Op0, Op1); 9762 } 9763 return 0; 9764} 9765 9766unsigned fastEmit_ISD_FSUB_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9767 if (RetVT.SimpleTy != MVT::f32) 9768 return 0; 9769 if ((Subtarget->hasAVX512())) { 9770 return fastEmitInst_rr(X86::VSUBSSZrr, &X86::FR32XRegClass, Op0, Op1); 9771 } 9772 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 9773 return fastEmitInst_rr(X86::SUBSSrr, &X86::FR32RegClass, Op0, Op1); 9774 } 9775 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 9776 return fastEmitInst_rr(X86::VSUBSSrr, &X86::FR32RegClass, Op0, Op1); 9777 } 9778 if ((!Subtarget->hasSSE1())) { 9779 return fastEmitInst_rr(X86::SUB_Fp32, &X86::RFP32RegClass, Op0, Op1); 9780 } 9781 return 0; 9782} 9783 9784unsigned fastEmit_ISD_FSUB_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9785 if (RetVT.SimpleTy != MVT::f64) 9786 return 0; 9787 if ((Subtarget->hasAVX512())) { 9788 return fastEmitInst_rr(X86::VSUBSDZrr, &X86::FR64XRegClass, Op0, Op1); 9789 } 9790 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 9791 return fastEmitInst_rr(X86::SUBSDrr, &X86::FR64RegClass, Op0, Op1); 9792 } 9793 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 9794 return fastEmitInst_rr(X86::VSUBSDrr, &X86::FR64RegClass, Op0, Op1); 9795 } 9796 if ((!Subtarget->hasSSE2())) { 9797 return fastEmitInst_rr(X86::SUB_Fp64, &X86::RFP64RegClass, Op0, Op1); 9798 } 9799 return 0; 9800} 9801 9802unsigned fastEmit_ISD_FSUB_MVT_f80_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9803 if (RetVT.SimpleTy != MVT::f80) 9804 return 0; 9805 if ((Subtarget->hasX87())) { 9806 return fastEmitInst_rr(X86::SUB_Fp80, &X86::RFP80RegClass, Op0, Op1); 9807 } 9808 return 0; 9809} 9810 9811unsigned fastEmit_ISD_FSUB_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9812 if (RetVT.SimpleTy != MVT::v8f16) 9813 return 0; 9814 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 9815 return fastEmitInst_rr(X86::VSUBPHZ128rr, &X86::VR128XRegClass, Op0, Op1); 9816 } 9817 return 0; 9818} 9819 9820unsigned fastEmit_ISD_FSUB_MVT_v16f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9821 if (RetVT.SimpleTy != MVT::v16f16) 9822 return 0; 9823 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 9824 return fastEmitInst_rr(X86::VSUBPHZ256rr, &X86::VR256XRegClass, Op0, Op1); 9825 } 9826 return 0; 9827} 9828 9829unsigned fastEmit_ISD_FSUB_MVT_v32f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9830 if (RetVT.SimpleTy != MVT::v32f16) 9831 return 0; 9832 if ((Subtarget->hasFP16())) { 9833 return fastEmitInst_rr(X86::VSUBPHZrr, &X86::VR512RegClass, Op0, Op1); 9834 } 9835 return 0; 9836} 9837 9838unsigned fastEmit_ISD_FSUB_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9839 if (RetVT.SimpleTy != MVT::v4f32) 9840 return 0; 9841 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 9842 return fastEmitInst_rr(X86::VSUBPSZ128rr, &X86::VR128XRegClass, Op0, Op1); 9843 } 9844 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 9845 return fastEmitInst_rr(X86::SUBPSrr, &X86::VR128RegClass, Op0, Op1); 9846 } 9847 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 9848 return fastEmitInst_rr(X86::VSUBPSrr, &X86::VR128RegClass, Op0, Op1); 9849 } 9850 return 0; 9851} 9852 9853unsigned fastEmit_ISD_FSUB_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9854 if (RetVT.SimpleTy != MVT::v8f32) 9855 return 0; 9856 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 9857 return fastEmitInst_rr(X86::VSUBPSZ256rr, &X86::VR256XRegClass, Op0, Op1); 9858 } 9859 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 9860 return fastEmitInst_rr(X86::VSUBPSYrr, &X86::VR256RegClass, Op0, Op1); 9861 } 9862 return 0; 9863} 9864 9865unsigned fastEmit_ISD_FSUB_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9866 if (RetVT.SimpleTy != MVT::v16f32) 9867 return 0; 9868 if ((Subtarget->hasAVX512())) { 9869 return fastEmitInst_rr(X86::VSUBPSZrr, &X86::VR512RegClass, Op0, Op1); 9870 } 9871 return 0; 9872} 9873 9874unsigned fastEmit_ISD_FSUB_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9875 if (RetVT.SimpleTy != MVT::v2f64) 9876 return 0; 9877 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 9878 return fastEmitInst_rr(X86::VSUBPDZ128rr, &X86::VR128XRegClass, Op0, Op1); 9879 } 9880 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 9881 return fastEmitInst_rr(X86::SUBPDrr, &X86::VR128RegClass, Op0, Op1); 9882 } 9883 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 9884 return fastEmitInst_rr(X86::VSUBPDrr, &X86::VR128RegClass, Op0, Op1); 9885 } 9886 return 0; 9887} 9888 9889unsigned fastEmit_ISD_FSUB_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9890 if (RetVT.SimpleTy != MVT::v4f64) 9891 return 0; 9892 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 9893 return fastEmitInst_rr(X86::VSUBPDZ256rr, &X86::VR256XRegClass, Op0, Op1); 9894 } 9895 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 9896 return fastEmitInst_rr(X86::VSUBPDYrr, &X86::VR256RegClass, Op0, Op1); 9897 } 9898 return 0; 9899} 9900 9901unsigned fastEmit_ISD_FSUB_MVT_v8f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9902 if (RetVT.SimpleTy != MVT::v8f64) 9903 return 0; 9904 if ((Subtarget->hasAVX512())) { 9905 return fastEmitInst_rr(X86::VSUBPDZrr, &X86::VR512RegClass, Op0, Op1); 9906 } 9907 return 0; 9908} 9909 9910unsigned fastEmit_ISD_FSUB_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 9911 switch (VT.SimpleTy) { 9912 case MVT::f16: return fastEmit_ISD_FSUB_MVT_f16_rr(RetVT, Op0, Op1); 9913 case MVT::f32: return fastEmit_ISD_FSUB_MVT_f32_rr(RetVT, Op0, Op1); 9914 case MVT::f64: return fastEmit_ISD_FSUB_MVT_f64_rr(RetVT, Op0, Op1); 9915 case MVT::f80: return fastEmit_ISD_FSUB_MVT_f80_rr(RetVT, Op0, Op1); 9916 case MVT::v8f16: return fastEmit_ISD_FSUB_MVT_v8f16_rr(RetVT, Op0, Op1); 9917 case MVT::v16f16: return fastEmit_ISD_FSUB_MVT_v16f16_rr(RetVT, Op0, Op1); 9918 case MVT::v32f16: return fastEmit_ISD_FSUB_MVT_v32f16_rr(RetVT, Op0, Op1); 9919 case MVT::v4f32: return fastEmit_ISD_FSUB_MVT_v4f32_rr(RetVT, Op0, Op1); 9920 case MVT::v8f32: return fastEmit_ISD_FSUB_MVT_v8f32_rr(RetVT, Op0, Op1); 9921 case MVT::v16f32: return fastEmit_ISD_FSUB_MVT_v16f32_rr(RetVT, Op0, Op1); 9922 case MVT::v2f64: return fastEmit_ISD_FSUB_MVT_v2f64_rr(RetVT, Op0, Op1); 9923 case MVT::v4f64: return fastEmit_ISD_FSUB_MVT_v4f64_rr(RetVT, Op0, Op1); 9924 case MVT::v8f64: return fastEmit_ISD_FSUB_MVT_v8f64_rr(RetVT, Op0, Op1); 9925 default: return 0; 9926 } 9927} 9928 9929// FastEmit functions for ISD::MUL. 9930 9931unsigned fastEmit_ISD_MUL_MVT_i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9932 if (RetVT.SimpleTy != MVT::i8) 9933 return 0; 9934 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(TargetOpcode::COPY), X86::AL).addReg(Op0); 9935 return fastEmitInst_r(X86::MUL8r, &X86::GR8RegClass, Op1); 9936} 9937 9938unsigned fastEmit_ISD_MUL_MVT_i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9939 if (RetVT.SimpleTy != MVT::i16) 9940 return 0; 9941 return fastEmitInst_rr(X86::IMUL16rr, &X86::GR16RegClass, Op0, Op1); 9942} 9943 9944unsigned fastEmit_ISD_MUL_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9945 if (RetVT.SimpleTy != MVT::i32) 9946 return 0; 9947 return fastEmitInst_rr(X86::IMUL32rr, &X86::GR32RegClass, Op0, Op1); 9948} 9949 9950unsigned fastEmit_ISD_MUL_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9951 if (RetVT.SimpleTy != MVT::i64) 9952 return 0; 9953 return fastEmitInst_rr(X86::IMUL64rr, &X86::GR64RegClass, Op0, Op1); 9954} 9955 9956unsigned fastEmit_ISD_MUL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9957 if (RetVT.SimpleTy != MVT::v8i16) 9958 return 0; 9959 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 9960 return fastEmitInst_rr(X86::VPMULLWZ128rr, &X86::VR128XRegClass, Op0, Op1); 9961 } 9962 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 9963 return fastEmitInst_rr(X86::PMULLWrr, &X86::VR128RegClass, Op0, Op1); 9964 } 9965 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 9966 return fastEmitInst_rr(X86::VPMULLWrr, &X86::VR128RegClass, Op0, Op1); 9967 } 9968 return 0; 9969} 9970 9971unsigned fastEmit_ISD_MUL_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9972 if (RetVT.SimpleTy != MVT::v16i16) 9973 return 0; 9974 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 9975 return fastEmitInst_rr(X86::VPMULLWZ256rr, &X86::VR256XRegClass, Op0, Op1); 9976 } 9977 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 9978 return fastEmitInst_rr(X86::VPMULLWYrr, &X86::VR256RegClass, Op0, Op1); 9979 } 9980 return 0; 9981} 9982 9983unsigned fastEmit_ISD_MUL_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9984 if (RetVT.SimpleTy != MVT::v32i16) 9985 return 0; 9986 if ((Subtarget->hasBWI())) { 9987 return fastEmitInst_rr(X86::VPMULLWZrr, &X86::VR512RegClass, Op0, Op1); 9988 } 9989 return 0; 9990} 9991 9992unsigned fastEmit_ISD_MUL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9993 if (RetVT.SimpleTy != MVT::v4i32) 9994 return 0; 9995 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 9996 return fastEmitInst_rr(X86::VPMULLDZ128rr, &X86::VR128XRegClass, Op0, Op1); 9997 } 9998 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 9999 return fastEmitInst_rr(X86::PMULLDrr, &X86::VR128RegClass, Op0, Op1); 10000 } 10001 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 10002 return fastEmitInst_rr(X86::VPMULLDrr, &X86::VR128RegClass, Op0, Op1); 10003 } 10004 return 0; 10005} 10006 10007unsigned fastEmit_ISD_MUL_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10008 if (RetVT.SimpleTy != MVT::v8i32) 10009 return 0; 10010 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 10011 return fastEmitInst_rr(X86::VPMULLDZ256rr, &X86::VR256XRegClass, Op0, Op1); 10012 } 10013 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 10014 return fastEmitInst_rr(X86::VPMULLDYrr, &X86::VR256RegClass, Op0, Op1); 10015 } 10016 return 0; 10017} 10018 10019unsigned fastEmit_ISD_MUL_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10020 if (RetVT.SimpleTy != MVT::v16i32) 10021 return 0; 10022 if ((Subtarget->hasAVX512())) { 10023 return fastEmitInst_rr(X86::VPMULLDZrr, &X86::VR512RegClass, Op0, Op1); 10024 } 10025 return 0; 10026} 10027 10028unsigned fastEmit_ISD_MUL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10029 if (RetVT.SimpleTy != MVT::v2i64) 10030 return 0; 10031 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 10032 return fastEmitInst_rr(X86::VPMULLQZ128rr, &X86::VR128XRegClass, Op0, Op1); 10033 } 10034 return 0; 10035} 10036 10037unsigned fastEmit_ISD_MUL_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10038 if (RetVT.SimpleTy != MVT::v4i64) 10039 return 0; 10040 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 10041 return fastEmitInst_rr(X86::VPMULLQZ256rr, &X86::VR256XRegClass, Op0, Op1); 10042 } 10043 return 0; 10044} 10045 10046unsigned fastEmit_ISD_MUL_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10047 if (RetVT.SimpleTy != MVT::v8i64) 10048 return 0; 10049 if ((Subtarget->hasDQI())) { 10050 return fastEmitInst_rr(X86::VPMULLQZrr, &X86::VR512RegClass, Op0, Op1); 10051 } 10052 return 0; 10053} 10054 10055unsigned fastEmit_ISD_MUL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 10056 switch (VT.SimpleTy) { 10057 case MVT::i8: return fastEmit_ISD_MUL_MVT_i8_rr(RetVT, Op0, Op1); 10058 case MVT::i16: return fastEmit_ISD_MUL_MVT_i16_rr(RetVT, Op0, Op1); 10059 case MVT::i32: return fastEmit_ISD_MUL_MVT_i32_rr(RetVT, Op0, Op1); 10060 case MVT::i64: return fastEmit_ISD_MUL_MVT_i64_rr(RetVT, Op0, Op1); 10061 case MVT::v8i16: return fastEmit_ISD_MUL_MVT_v8i16_rr(RetVT, Op0, Op1); 10062 case MVT::v16i16: return fastEmit_ISD_MUL_MVT_v16i16_rr(RetVT, Op0, Op1); 10063 case MVT::v32i16: return fastEmit_ISD_MUL_MVT_v32i16_rr(RetVT, Op0, Op1); 10064 case MVT::v4i32: return fastEmit_ISD_MUL_MVT_v4i32_rr(RetVT, Op0, Op1); 10065 case MVT::v8i32: return fastEmit_ISD_MUL_MVT_v8i32_rr(RetVT, Op0, Op1); 10066 case MVT::v16i32: return fastEmit_ISD_MUL_MVT_v16i32_rr(RetVT, Op0, Op1); 10067 case MVT::v2i64: return fastEmit_ISD_MUL_MVT_v2i64_rr(RetVT, Op0, Op1); 10068 case MVT::v4i64: return fastEmit_ISD_MUL_MVT_v4i64_rr(RetVT, Op0, Op1); 10069 case MVT::v8i64: return fastEmit_ISD_MUL_MVT_v8i64_rr(RetVT, Op0, Op1); 10070 default: return 0; 10071 } 10072} 10073 10074// FastEmit functions for ISD::MULHS. 10075 10076unsigned fastEmit_ISD_MULHS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10077 if (RetVT.SimpleTy != MVT::v8i16) 10078 return 0; 10079 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 10080 return fastEmitInst_rr(X86::VPMULHWZ128rr, &X86::VR128XRegClass, Op0, Op1); 10081 } 10082 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 10083 return fastEmitInst_rr(X86::PMULHWrr, &X86::VR128RegClass, Op0, Op1); 10084 } 10085 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 10086 return fastEmitInst_rr(X86::VPMULHWrr, &X86::VR128RegClass, Op0, Op1); 10087 } 10088 return 0; 10089} 10090 10091unsigned fastEmit_ISD_MULHS_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10092 if (RetVT.SimpleTy != MVT::v16i16) 10093 return 0; 10094 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 10095 return fastEmitInst_rr(X86::VPMULHWZ256rr, &X86::VR256XRegClass, Op0, Op1); 10096 } 10097 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 10098 return fastEmitInst_rr(X86::VPMULHWYrr, &X86::VR256RegClass, Op0, Op1); 10099 } 10100 return 0; 10101} 10102 10103unsigned fastEmit_ISD_MULHS_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10104 if (RetVT.SimpleTy != MVT::v32i16) 10105 return 0; 10106 if ((Subtarget->hasBWI())) { 10107 return fastEmitInst_rr(X86::VPMULHWZrr, &X86::VR512RegClass, Op0, Op1); 10108 } 10109 return 0; 10110} 10111 10112unsigned fastEmit_ISD_MULHS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 10113 switch (VT.SimpleTy) { 10114 case MVT::v8i16: return fastEmit_ISD_MULHS_MVT_v8i16_rr(RetVT, Op0, Op1); 10115 case MVT::v16i16: return fastEmit_ISD_MULHS_MVT_v16i16_rr(RetVT, Op0, Op1); 10116 case MVT::v32i16: return fastEmit_ISD_MULHS_MVT_v32i16_rr(RetVT, Op0, Op1); 10117 default: return 0; 10118 } 10119} 10120 10121// FastEmit functions for ISD::MULHU. 10122 10123unsigned fastEmit_ISD_MULHU_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10124 if (RetVT.SimpleTy != MVT::v8i16) 10125 return 0; 10126 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 10127 return fastEmitInst_rr(X86::VPMULHUWZ128rr, &X86::VR128XRegClass, Op0, Op1); 10128 } 10129 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 10130 return fastEmitInst_rr(X86::PMULHUWrr, &X86::VR128RegClass, Op0, Op1); 10131 } 10132 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 10133 return fastEmitInst_rr(X86::VPMULHUWrr, &X86::VR128RegClass, Op0, Op1); 10134 } 10135 return 0; 10136} 10137 10138unsigned fastEmit_ISD_MULHU_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10139 if (RetVT.SimpleTy != MVT::v16i16) 10140 return 0; 10141 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 10142 return fastEmitInst_rr(X86::VPMULHUWZ256rr, &X86::VR256XRegClass, Op0, Op1); 10143 } 10144 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 10145 return fastEmitInst_rr(X86::VPMULHUWYrr, &X86::VR256RegClass, Op0, Op1); 10146 } 10147 return 0; 10148} 10149 10150unsigned fastEmit_ISD_MULHU_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10151 if (RetVT.SimpleTy != MVT::v32i16) 10152 return 0; 10153 if ((Subtarget->hasBWI())) { 10154 return fastEmitInst_rr(X86::VPMULHUWZrr, &X86::VR512RegClass, Op0, Op1); 10155 } 10156 return 0; 10157} 10158 10159unsigned fastEmit_ISD_MULHU_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 10160 switch (VT.SimpleTy) { 10161 case MVT::v8i16: return fastEmit_ISD_MULHU_MVT_v8i16_rr(RetVT, Op0, Op1); 10162 case MVT::v16i16: return fastEmit_ISD_MULHU_MVT_v16i16_rr(RetVT, Op0, Op1); 10163 case MVT::v32i16: return fastEmit_ISD_MULHU_MVT_v32i16_rr(RetVT, Op0, Op1); 10164 default: return 0; 10165 } 10166} 10167 10168// FastEmit functions for ISD::OR. 10169 10170unsigned fastEmit_ISD_OR_MVT_i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10171 if (RetVT.SimpleTy != MVT::i8) 10172 return 0; 10173 return fastEmitInst_rr(X86::OR8rr, &X86::GR8RegClass, Op0, Op1); 10174} 10175 10176unsigned fastEmit_ISD_OR_MVT_i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10177 if (RetVT.SimpleTy != MVT::i16) 10178 return 0; 10179 return fastEmitInst_rr(X86::OR16rr, &X86::GR16RegClass, Op0, Op1); 10180} 10181 10182unsigned fastEmit_ISD_OR_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10183 if (RetVT.SimpleTy != MVT::i32) 10184 return 0; 10185 return fastEmitInst_rr(X86::OR32rr, &X86::GR32RegClass, Op0, Op1); 10186} 10187 10188unsigned fastEmit_ISD_OR_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10189 if (RetVT.SimpleTy != MVT::i64) 10190 return 0; 10191 return fastEmitInst_rr(X86::OR64rr, &X86::GR64RegClass, Op0, Op1); 10192} 10193 10194unsigned fastEmit_ISD_OR_MVT_v8i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10195 if (RetVT.SimpleTy != MVT::v8i1) 10196 return 0; 10197 if ((Subtarget->hasDQI())) { 10198 return fastEmitInst_rr(X86::KORBrr, &X86::VK8RegClass, Op0, Op1); 10199 } 10200 return 0; 10201} 10202 10203unsigned fastEmit_ISD_OR_MVT_v16i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10204 if (RetVT.SimpleTy != MVT::v16i1) 10205 return 0; 10206 if ((Subtarget->hasAVX512())) { 10207 return fastEmitInst_rr(X86::KORWrr, &X86::VK16RegClass, Op0, Op1); 10208 } 10209 return 0; 10210} 10211 10212unsigned fastEmit_ISD_OR_MVT_v32i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10213 if (RetVT.SimpleTy != MVT::v32i1) 10214 return 0; 10215 if ((Subtarget->hasBWI())) { 10216 return fastEmitInst_rr(X86::KORDrr, &X86::VK32RegClass, Op0, Op1); 10217 } 10218 return 0; 10219} 10220 10221unsigned fastEmit_ISD_OR_MVT_v64i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10222 if (RetVT.SimpleTy != MVT::v64i1) 10223 return 0; 10224 if ((Subtarget->hasBWI())) { 10225 return fastEmitInst_rr(X86::KORQrr, &X86::VK64RegClass, Op0, Op1); 10226 } 10227 return 0; 10228} 10229 10230unsigned fastEmit_ISD_OR_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10231 if (RetVT.SimpleTy != MVT::v16i8) 10232 return 0; 10233 if ((Subtarget->hasVLX())) { 10234 return fastEmitInst_rr(X86::VPORQZ128rr, &X86::VR128XRegClass, Op0, Op1); 10235 } 10236 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 10237 return fastEmitInst_rr(X86::PORrr, &X86::VR128RegClass, Op0, Op1); 10238 } 10239 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 10240 return fastEmitInst_rr(X86::VPORrr, &X86::VR128RegClass, Op0, Op1); 10241 } 10242 return 0; 10243} 10244 10245unsigned fastEmit_ISD_OR_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10246 if (RetVT.SimpleTy != MVT::v32i8) 10247 return 0; 10248 if ((Subtarget->hasVLX())) { 10249 return fastEmitInst_rr(X86::VPORQZ256rr, &X86::VR256XRegClass, Op0, Op1); 10250 } 10251 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) { 10252 return fastEmitInst_rr(X86::VORPSYrr, &X86::VR256RegClass, Op0, Op1); 10253 } 10254 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 10255 return fastEmitInst_rr(X86::VPORYrr, &X86::VR256RegClass, Op0, Op1); 10256 } 10257 return 0; 10258} 10259 10260unsigned fastEmit_ISD_OR_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10261 if (RetVT.SimpleTy != MVT::v64i8) 10262 return 0; 10263 if ((Subtarget->hasAVX512())) { 10264 return fastEmitInst_rr(X86::VPORQZrr, &X86::VR512RegClass, Op0, Op1); 10265 } 10266 return 0; 10267} 10268 10269unsigned fastEmit_ISD_OR_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10270 if (RetVT.SimpleTy != MVT::v8i16) 10271 return 0; 10272 if ((Subtarget->hasVLX())) { 10273 return fastEmitInst_rr(X86::VPORQZ128rr, &X86::VR128XRegClass, Op0, Op1); 10274 } 10275 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 10276 return fastEmitInst_rr(X86::PORrr, &X86::VR128RegClass, Op0, Op1); 10277 } 10278 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 10279 return fastEmitInst_rr(X86::VPORrr, &X86::VR128RegClass, Op0, Op1); 10280 } 10281 return 0; 10282} 10283 10284unsigned fastEmit_ISD_OR_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10285 if (RetVT.SimpleTy != MVT::v16i16) 10286 return 0; 10287 if ((Subtarget->hasVLX())) { 10288 return fastEmitInst_rr(X86::VPORQZ256rr, &X86::VR256XRegClass, Op0, Op1); 10289 } 10290 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) { 10291 return fastEmitInst_rr(X86::VORPSYrr, &X86::VR256RegClass, Op0, Op1); 10292 } 10293 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 10294 return fastEmitInst_rr(X86::VPORYrr, &X86::VR256RegClass, Op0, Op1); 10295 } 10296 return 0; 10297} 10298 10299unsigned fastEmit_ISD_OR_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10300 if (RetVT.SimpleTy != MVT::v32i16) 10301 return 0; 10302 if ((Subtarget->hasAVX512())) { 10303 return fastEmitInst_rr(X86::VPORQZrr, &X86::VR512RegClass, Op0, Op1); 10304 } 10305 return 0; 10306} 10307 10308unsigned fastEmit_ISD_OR_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10309 if (RetVT.SimpleTy != MVT::v4i32) 10310 return 0; 10311 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 10312 return fastEmitInst_rr(X86::PORrr, &X86::VR128RegClass, Op0, Op1); 10313 } 10314 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 10315 return fastEmitInst_rr(X86::VPORrr, &X86::VR128RegClass, Op0, Op1); 10316 } 10317 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 10318 return fastEmitInst_rr(X86::VPORDZ128rr, &X86::VR128XRegClass, Op0, Op1); 10319 } 10320 return 0; 10321} 10322 10323unsigned fastEmit_ISD_OR_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10324 if (RetVT.SimpleTy != MVT::v8i32) 10325 return 0; 10326 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) { 10327 return fastEmitInst_rr(X86::VORPSYrr, &X86::VR256RegClass, Op0, Op1); 10328 } 10329 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 10330 return fastEmitInst_rr(X86::VPORYrr, &X86::VR256RegClass, Op0, Op1); 10331 } 10332 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 10333 return fastEmitInst_rr(X86::VPORDZ256rr, &X86::VR256XRegClass, Op0, Op1); 10334 } 10335 return 0; 10336} 10337 10338unsigned fastEmit_ISD_OR_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10339 if (RetVT.SimpleTy != MVT::v16i32) 10340 return 0; 10341 if ((Subtarget->hasAVX512())) { 10342 return fastEmitInst_rr(X86::VPORDZrr, &X86::VR512RegClass, Op0, Op1); 10343 } 10344 return 0; 10345} 10346 10347unsigned fastEmit_ISD_OR_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10348 if (RetVT.SimpleTy != MVT::v2i64) 10349 return 0; 10350 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 10351 return fastEmitInst_rr(X86::VPORQZ128rr, &X86::VR128XRegClass, Op0, Op1); 10352 } 10353 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 10354 return fastEmitInst_rr(X86::PORrr, &X86::VR128RegClass, Op0, Op1); 10355 } 10356 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 10357 return fastEmitInst_rr(X86::VPORrr, &X86::VR128RegClass, Op0, Op1); 10358 } 10359 return 0; 10360} 10361 10362unsigned fastEmit_ISD_OR_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10363 if (RetVT.SimpleTy != MVT::v4i64) 10364 return 0; 10365 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) { 10366 return fastEmitInst_rr(X86::VORPSYrr, &X86::VR256RegClass, Op0, Op1); 10367 } 10368 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 10369 return fastEmitInst_rr(X86::VPORQZ256rr, &X86::VR256XRegClass, Op0, Op1); 10370 } 10371 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 10372 return fastEmitInst_rr(X86::VPORYrr, &X86::VR256RegClass, Op0, Op1); 10373 } 10374 return 0; 10375} 10376 10377unsigned fastEmit_ISD_OR_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10378 if (RetVT.SimpleTy != MVT::v8i64) 10379 return 0; 10380 if ((Subtarget->hasAVX512())) { 10381 return fastEmitInst_rr(X86::VPORQZrr, &X86::VR512RegClass, Op0, Op1); 10382 } 10383 return 0; 10384} 10385 10386unsigned fastEmit_ISD_OR_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 10387 switch (VT.SimpleTy) { 10388 case MVT::i8: return fastEmit_ISD_OR_MVT_i8_rr(RetVT, Op0, Op1); 10389 case MVT::i16: return fastEmit_ISD_OR_MVT_i16_rr(RetVT, Op0, Op1); 10390 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_rr(RetVT, Op0, Op1); 10391 case MVT::i64: return fastEmit_ISD_OR_MVT_i64_rr(RetVT, Op0, Op1); 10392 case MVT::v8i1: return fastEmit_ISD_OR_MVT_v8i1_rr(RetVT, Op0, Op1); 10393 case MVT::v16i1: return fastEmit_ISD_OR_MVT_v16i1_rr(RetVT, Op0, Op1); 10394 case MVT::v32i1: return fastEmit_ISD_OR_MVT_v32i1_rr(RetVT, Op0, Op1); 10395 case MVT::v64i1: return fastEmit_ISD_OR_MVT_v64i1_rr(RetVT, Op0, Op1); 10396 case MVT::v16i8: return fastEmit_ISD_OR_MVT_v16i8_rr(RetVT, Op0, Op1); 10397 case MVT::v32i8: return fastEmit_ISD_OR_MVT_v32i8_rr(RetVT, Op0, Op1); 10398 case MVT::v64i8: return fastEmit_ISD_OR_MVT_v64i8_rr(RetVT, Op0, Op1); 10399 case MVT::v8i16: return fastEmit_ISD_OR_MVT_v8i16_rr(RetVT, Op0, Op1); 10400 case MVT::v16i16: return fastEmit_ISD_OR_MVT_v16i16_rr(RetVT, Op0, Op1); 10401 case MVT::v32i16: return fastEmit_ISD_OR_MVT_v32i16_rr(RetVT, Op0, Op1); 10402 case MVT::v4i32: return fastEmit_ISD_OR_MVT_v4i32_rr(RetVT, Op0, Op1); 10403 case MVT::v8i32: return fastEmit_ISD_OR_MVT_v8i32_rr(RetVT, Op0, Op1); 10404 case MVT::v16i32: return fastEmit_ISD_OR_MVT_v16i32_rr(RetVT, Op0, Op1); 10405 case MVT::v2i64: return fastEmit_ISD_OR_MVT_v2i64_rr(RetVT, Op0, Op1); 10406 case MVT::v4i64: return fastEmit_ISD_OR_MVT_v4i64_rr(RetVT, Op0, Op1); 10407 case MVT::v8i64: return fastEmit_ISD_OR_MVT_v8i64_rr(RetVT, Op0, Op1); 10408 default: return 0; 10409 } 10410} 10411 10412// FastEmit functions for ISD::ROTL. 10413 10414unsigned fastEmit_ISD_ROTL_MVT_i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10415 if (RetVT.SimpleTy != MVT::i8) 10416 return 0; 10417 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(TargetOpcode::COPY), X86::CL).addReg(Op1); 10418 return fastEmitInst_r(X86::ROL8rCL, &X86::GR8RegClass, Op0); 10419} 10420 10421unsigned fastEmit_ISD_ROTL_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10422 if (RetVT.SimpleTy != MVT::v16i8) 10423 return 0; 10424 if ((Subtarget->hasXOP())) { 10425 return fastEmitInst_rr(X86::VPROTBrr, &X86::VR128RegClass, Op0, Op1); 10426 } 10427 return 0; 10428} 10429 10430unsigned fastEmit_ISD_ROTL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10431 if (RetVT.SimpleTy != MVT::v8i16) 10432 return 0; 10433 if ((Subtarget->hasXOP())) { 10434 return fastEmitInst_rr(X86::VPROTWrr, &X86::VR128RegClass, Op0, Op1); 10435 } 10436 return 0; 10437} 10438 10439unsigned fastEmit_ISD_ROTL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10440 if (RetVT.SimpleTy != MVT::v4i32) 10441 return 0; 10442 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 10443 return fastEmitInst_rr(X86::VPROLVDZ128rr, &X86::VR128XRegClass, Op0, Op1); 10444 } 10445 if ((Subtarget->hasXOP())) { 10446 return fastEmitInst_rr(X86::VPROTDrr, &X86::VR128RegClass, Op0, Op1); 10447 } 10448 return 0; 10449} 10450 10451unsigned fastEmit_ISD_ROTL_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10452 if (RetVT.SimpleTy != MVT::v8i32) 10453 return 0; 10454 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 10455 return fastEmitInst_rr(X86::VPROLVDZ256rr, &X86::VR256XRegClass, Op0, Op1); 10456 } 10457 return 0; 10458} 10459 10460unsigned fastEmit_ISD_ROTL_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10461 if (RetVT.SimpleTy != MVT::v16i32) 10462 return 0; 10463 if ((Subtarget->hasAVX512())) { 10464 return fastEmitInst_rr(X86::VPROLVDZrr, &X86::VR512RegClass, Op0, Op1); 10465 } 10466 return 0; 10467} 10468 10469unsigned fastEmit_ISD_ROTL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10470 if (RetVT.SimpleTy != MVT::v2i64) 10471 return 0; 10472 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 10473 return fastEmitInst_rr(X86::VPROLVQZ128rr, &X86::VR128XRegClass, Op0, Op1); 10474 } 10475 if ((Subtarget->hasXOP())) { 10476 return fastEmitInst_rr(X86::VPROTQrr, &X86::VR128RegClass, Op0, Op1); 10477 } 10478 return 0; 10479} 10480 10481unsigned fastEmit_ISD_ROTL_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10482 if (RetVT.SimpleTy != MVT::v4i64) 10483 return 0; 10484 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 10485 return fastEmitInst_rr(X86::VPROLVQZ256rr, &X86::VR256XRegClass, Op0, Op1); 10486 } 10487 return 0; 10488} 10489 10490unsigned fastEmit_ISD_ROTL_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10491 if (RetVT.SimpleTy != MVT::v8i64) 10492 return 0; 10493 if ((Subtarget->hasAVX512())) { 10494 return fastEmitInst_rr(X86::VPROLVQZrr, &X86::VR512RegClass, Op0, Op1); 10495 } 10496 return 0; 10497} 10498 10499unsigned fastEmit_ISD_ROTL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 10500 switch (VT.SimpleTy) { 10501 case MVT::i8: return fastEmit_ISD_ROTL_MVT_i8_rr(RetVT, Op0, Op1); 10502 case MVT::v16i8: return fastEmit_ISD_ROTL_MVT_v16i8_rr(RetVT, Op0, Op1); 10503 case MVT::v8i16: return fastEmit_ISD_ROTL_MVT_v8i16_rr(RetVT, Op0, Op1); 10504 case MVT::v4i32: return fastEmit_ISD_ROTL_MVT_v4i32_rr(RetVT, Op0, Op1); 10505 case MVT::v8i32: return fastEmit_ISD_ROTL_MVT_v8i32_rr(RetVT, Op0, Op1); 10506 case MVT::v16i32: return fastEmit_ISD_ROTL_MVT_v16i32_rr(RetVT, Op0, Op1); 10507 case MVT::v2i64: return fastEmit_ISD_ROTL_MVT_v2i64_rr(RetVT, Op0, Op1); 10508 case MVT::v4i64: return fastEmit_ISD_ROTL_MVT_v4i64_rr(RetVT, Op0, Op1); 10509 case MVT::v8i64: return fastEmit_ISD_ROTL_MVT_v8i64_rr(RetVT, Op0, Op1); 10510 default: return 0; 10511 } 10512} 10513 10514// FastEmit functions for ISD::ROTR. 10515 10516unsigned fastEmit_ISD_ROTR_MVT_i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10517 if (RetVT.SimpleTy != MVT::i8) 10518 return 0; 10519 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(TargetOpcode::COPY), X86::CL).addReg(Op1); 10520 return fastEmitInst_r(X86::ROR8rCL, &X86::GR8RegClass, Op0); 10521} 10522 10523unsigned fastEmit_ISD_ROTR_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10524 if (RetVT.SimpleTy != MVT::v4i32) 10525 return 0; 10526 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 10527 return fastEmitInst_rr(X86::VPRORVDZ128rr, &X86::VR128XRegClass, Op0, Op1); 10528 } 10529 return 0; 10530} 10531 10532unsigned fastEmit_ISD_ROTR_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10533 if (RetVT.SimpleTy != MVT::v8i32) 10534 return 0; 10535 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 10536 return fastEmitInst_rr(X86::VPRORVDZ256rr, &X86::VR256XRegClass, Op0, Op1); 10537 } 10538 return 0; 10539} 10540 10541unsigned fastEmit_ISD_ROTR_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10542 if (RetVT.SimpleTy != MVT::v16i32) 10543 return 0; 10544 if ((Subtarget->hasAVX512())) { 10545 return fastEmitInst_rr(X86::VPRORVDZrr, &X86::VR512RegClass, Op0, Op1); 10546 } 10547 return 0; 10548} 10549 10550unsigned fastEmit_ISD_ROTR_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10551 if (RetVT.SimpleTy != MVT::v2i64) 10552 return 0; 10553 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 10554 return fastEmitInst_rr(X86::VPRORVQZ128rr, &X86::VR128XRegClass, Op0, Op1); 10555 } 10556 return 0; 10557} 10558 10559unsigned fastEmit_ISD_ROTR_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10560 if (RetVT.SimpleTy != MVT::v4i64) 10561 return 0; 10562 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 10563 return fastEmitInst_rr(X86::VPRORVQZ256rr, &X86::VR256XRegClass, Op0, Op1); 10564 } 10565 return 0; 10566} 10567 10568unsigned fastEmit_ISD_ROTR_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10569 if (RetVT.SimpleTy != MVT::v8i64) 10570 return 0; 10571 if ((Subtarget->hasAVX512())) { 10572 return fastEmitInst_rr(X86::VPRORVQZrr, &X86::VR512RegClass, Op0, Op1); 10573 } 10574 return 0; 10575} 10576 10577unsigned fastEmit_ISD_ROTR_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 10578 switch (VT.SimpleTy) { 10579 case MVT::i8: return fastEmit_ISD_ROTR_MVT_i8_rr(RetVT, Op0, Op1); 10580 case MVT::v4i32: return fastEmit_ISD_ROTR_MVT_v4i32_rr(RetVT, Op0, Op1); 10581 case MVT::v8i32: return fastEmit_ISD_ROTR_MVT_v8i32_rr(RetVT, Op0, Op1); 10582 case MVT::v16i32: return fastEmit_ISD_ROTR_MVT_v16i32_rr(RetVT, Op0, Op1); 10583 case MVT::v2i64: return fastEmit_ISD_ROTR_MVT_v2i64_rr(RetVT, Op0, Op1); 10584 case MVT::v4i64: return fastEmit_ISD_ROTR_MVT_v4i64_rr(RetVT, Op0, Op1); 10585 case MVT::v8i64: return fastEmit_ISD_ROTR_MVT_v8i64_rr(RetVT, Op0, Op1); 10586 default: return 0; 10587 } 10588} 10589 10590// FastEmit functions for ISD::SADDSAT. 10591 10592unsigned fastEmit_ISD_SADDSAT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10593 if (RetVT.SimpleTy != MVT::v16i8) 10594 return 0; 10595 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 10596 return fastEmitInst_rr(X86::VPADDSBZ128rr, &X86::VR128XRegClass, Op0, Op1); 10597 } 10598 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 10599 return fastEmitInst_rr(X86::PADDSBrr, &X86::VR128RegClass, Op0, Op1); 10600 } 10601 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 10602 return fastEmitInst_rr(X86::VPADDSBrr, &X86::VR128RegClass, Op0, Op1); 10603 } 10604 return 0; 10605} 10606 10607unsigned fastEmit_ISD_SADDSAT_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10608 if (RetVT.SimpleTy != MVT::v32i8) 10609 return 0; 10610 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 10611 return fastEmitInst_rr(X86::VPADDSBZ256rr, &X86::VR256XRegClass, Op0, Op1); 10612 } 10613 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 10614 return fastEmitInst_rr(X86::VPADDSBYrr, &X86::VR256RegClass, Op0, Op1); 10615 } 10616 return 0; 10617} 10618 10619unsigned fastEmit_ISD_SADDSAT_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10620 if (RetVT.SimpleTy != MVT::v64i8) 10621 return 0; 10622 if ((Subtarget->hasBWI())) { 10623 return fastEmitInst_rr(X86::VPADDSBZrr, &X86::VR512RegClass, Op0, Op1); 10624 } 10625 return 0; 10626} 10627 10628unsigned fastEmit_ISD_SADDSAT_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10629 if (RetVT.SimpleTy != MVT::v8i16) 10630 return 0; 10631 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 10632 return fastEmitInst_rr(X86::VPADDSWZ128rr, &X86::VR128XRegClass, Op0, Op1); 10633 } 10634 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 10635 return fastEmitInst_rr(X86::PADDSWrr, &X86::VR128RegClass, Op0, Op1); 10636 } 10637 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 10638 return fastEmitInst_rr(X86::VPADDSWrr, &X86::VR128RegClass, Op0, Op1); 10639 } 10640 return 0; 10641} 10642 10643unsigned fastEmit_ISD_SADDSAT_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10644 if (RetVT.SimpleTy != MVT::v16i16) 10645 return 0; 10646 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 10647 return fastEmitInst_rr(X86::VPADDSWZ256rr, &X86::VR256XRegClass, Op0, Op1); 10648 } 10649 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 10650 return fastEmitInst_rr(X86::VPADDSWYrr, &X86::VR256RegClass, Op0, Op1); 10651 } 10652 return 0; 10653} 10654 10655unsigned fastEmit_ISD_SADDSAT_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10656 if (RetVT.SimpleTy != MVT::v32i16) 10657 return 0; 10658 if ((Subtarget->hasBWI())) { 10659 return fastEmitInst_rr(X86::VPADDSWZrr, &X86::VR512RegClass, Op0, Op1); 10660 } 10661 return 0; 10662} 10663 10664unsigned fastEmit_ISD_SADDSAT_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 10665 switch (VT.SimpleTy) { 10666 case MVT::v16i8: return fastEmit_ISD_SADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1); 10667 case MVT::v32i8: return fastEmit_ISD_SADDSAT_MVT_v32i8_rr(RetVT, Op0, Op1); 10668 case MVT::v64i8: return fastEmit_ISD_SADDSAT_MVT_v64i8_rr(RetVT, Op0, Op1); 10669 case MVT::v8i16: return fastEmit_ISD_SADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1); 10670 case MVT::v16i16: return fastEmit_ISD_SADDSAT_MVT_v16i16_rr(RetVT, Op0, Op1); 10671 case MVT::v32i16: return fastEmit_ISD_SADDSAT_MVT_v32i16_rr(RetVT, Op0, Op1); 10672 default: return 0; 10673 } 10674} 10675 10676// FastEmit functions for ISD::SHL. 10677 10678unsigned fastEmit_ISD_SHL_MVT_i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10679 if (RetVT.SimpleTy != MVT::i8) 10680 return 0; 10681 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(TargetOpcode::COPY), X86::CL).addReg(Op1); 10682 return fastEmitInst_r(X86::SHL8rCL, &X86::GR8RegClass, Op0); 10683} 10684 10685unsigned fastEmit_ISD_SHL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 10686 switch (VT.SimpleTy) { 10687 case MVT::i8: return fastEmit_ISD_SHL_MVT_i8_rr(RetVT, Op0, Op1); 10688 default: return 0; 10689 } 10690} 10691 10692// FastEmit functions for ISD::SMAX. 10693 10694unsigned fastEmit_ISD_SMAX_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10695 if (RetVT.SimpleTy != MVT::v16i8) 10696 return 0; 10697 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 10698 return fastEmitInst_rr(X86::VPMAXSBZ128rr, &X86::VR128XRegClass, Op0, Op1); 10699 } 10700 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 10701 return fastEmitInst_rr(X86::PMAXSBrr, &X86::VR128RegClass, Op0, Op1); 10702 } 10703 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 10704 return fastEmitInst_rr(X86::VPMAXSBrr, &X86::VR128RegClass, Op0, Op1); 10705 } 10706 return 0; 10707} 10708 10709unsigned fastEmit_ISD_SMAX_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10710 if (RetVT.SimpleTy != MVT::v32i8) 10711 return 0; 10712 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 10713 return fastEmitInst_rr(X86::VPMAXSBZ256rr, &X86::VR256XRegClass, Op0, Op1); 10714 } 10715 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 10716 return fastEmitInst_rr(X86::VPMAXSBYrr, &X86::VR256RegClass, Op0, Op1); 10717 } 10718 return 0; 10719} 10720 10721unsigned fastEmit_ISD_SMAX_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10722 if (RetVT.SimpleTy != MVT::v64i8) 10723 return 0; 10724 if ((Subtarget->hasBWI())) { 10725 return fastEmitInst_rr(X86::VPMAXSBZrr, &X86::VR512RegClass, Op0, Op1); 10726 } 10727 return 0; 10728} 10729 10730unsigned fastEmit_ISD_SMAX_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10731 if (RetVT.SimpleTy != MVT::v8i16) 10732 return 0; 10733 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 10734 return fastEmitInst_rr(X86::VPMAXSWZ128rr, &X86::VR128XRegClass, Op0, Op1); 10735 } 10736 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 10737 return fastEmitInst_rr(X86::PMAXSWrr, &X86::VR128RegClass, Op0, Op1); 10738 } 10739 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 10740 return fastEmitInst_rr(X86::VPMAXSWrr, &X86::VR128RegClass, Op0, Op1); 10741 } 10742 return 0; 10743} 10744 10745unsigned fastEmit_ISD_SMAX_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10746 if (RetVT.SimpleTy != MVT::v16i16) 10747 return 0; 10748 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 10749 return fastEmitInst_rr(X86::VPMAXSWZ256rr, &X86::VR256XRegClass, Op0, Op1); 10750 } 10751 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 10752 return fastEmitInst_rr(X86::VPMAXSWYrr, &X86::VR256RegClass, Op0, Op1); 10753 } 10754 return 0; 10755} 10756 10757unsigned fastEmit_ISD_SMAX_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10758 if (RetVT.SimpleTy != MVT::v32i16) 10759 return 0; 10760 if ((Subtarget->hasBWI())) { 10761 return fastEmitInst_rr(X86::VPMAXSWZrr, &X86::VR512RegClass, Op0, Op1); 10762 } 10763 return 0; 10764} 10765 10766unsigned fastEmit_ISD_SMAX_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10767 if (RetVT.SimpleTy != MVT::v4i32) 10768 return 0; 10769 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 10770 return fastEmitInst_rr(X86::VPMAXSDZ128rr, &X86::VR128XRegClass, Op0, Op1); 10771 } 10772 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 10773 return fastEmitInst_rr(X86::PMAXSDrr, &X86::VR128RegClass, Op0, Op1); 10774 } 10775 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 10776 return fastEmitInst_rr(X86::VPMAXSDrr, &X86::VR128RegClass, Op0, Op1); 10777 } 10778 return 0; 10779} 10780 10781unsigned fastEmit_ISD_SMAX_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10782 if (RetVT.SimpleTy != MVT::v8i32) 10783 return 0; 10784 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 10785 return fastEmitInst_rr(X86::VPMAXSDZ256rr, &X86::VR256XRegClass, Op0, Op1); 10786 } 10787 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 10788 return fastEmitInst_rr(X86::VPMAXSDYrr, &X86::VR256RegClass, Op0, Op1); 10789 } 10790 return 0; 10791} 10792 10793unsigned fastEmit_ISD_SMAX_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10794 if (RetVT.SimpleTy != MVT::v16i32) 10795 return 0; 10796 if ((Subtarget->hasAVX512())) { 10797 return fastEmitInst_rr(X86::VPMAXSDZrr, &X86::VR512RegClass, Op0, Op1); 10798 } 10799 return 0; 10800} 10801 10802unsigned fastEmit_ISD_SMAX_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10803 if (RetVT.SimpleTy != MVT::v2i64) 10804 return 0; 10805 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 10806 return fastEmitInst_rr(X86::VPMAXSQZ128rr, &X86::VR128XRegClass, Op0, Op1); 10807 } 10808 return 0; 10809} 10810 10811unsigned fastEmit_ISD_SMAX_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10812 if (RetVT.SimpleTy != MVT::v4i64) 10813 return 0; 10814 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 10815 return fastEmitInst_rr(X86::VPMAXSQZ256rr, &X86::VR256XRegClass, Op0, Op1); 10816 } 10817 return 0; 10818} 10819 10820unsigned fastEmit_ISD_SMAX_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10821 if (RetVT.SimpleTy != MVT::v8i64) 10822 return 0; 10823 if ((Subtarget->hasAVX512())) { 10824 return fastEmitInst_rr(X86::VPMAXSQZrr, &X86::VR512RegClass, Op0, Op1); 10825 } 10826 return 0; 10827} 10828 10829unsigned fastEmit_ISD_SMAX_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 10830 switch (VT.SimpleTy) { 10831 case MVT::v16i8: return fastEmit_ISD_SMAX_MVT_v16i8_rr(RetVT, Op0, Op1); 10832 case MVT::v32i8: return fastEmit_ISD_SMAX_MVT_v32i8_rr(RetVT, Op0, Op1); 10833 case MVT::v64i8: return fastEmit_ISD_SMAX_MVT_v64i8_rr(RetVT, Op0, Op1); 10834 case MVT::v8i16: return fastEmit_ISD_SMAX_MVT_v8i16_rr(RetVT, Op0, Op1); 10835 case MVT::v16i16: return fastEmit_ISD_SMAX_MVT_v16i16_rr(RetVT, Op0, Op1); 10836 case MVT::v32i16: return fastEmit_ISD_SMAX_MVT_v32i16_rr(RetVT, Op0, Op1); 10837 case MVT::v4i32: return fastEmit_ISD_SMAX_MVT_v4i32_rr(RetVT, Op0, Op1); 10838 case MVT::v8i32: return fastEmit_ISD_SMAX_MVT_v8i32_rr(RetVT, Op0, Op1); 10839 case MVT::v16i32: return fastEmit_ISD_SMAX_MVT_v16i32_rr(RetVT, Op0, Op1); 10840 case MVT::v2i64: return fastEmit_ISD_SMAX_MVT_v2i64_rr(RetVT, Op0, Op1); 10841 case MVT::v4i64: return fastEmit_ISD_SMAX_MVT_v4i64_rr(RetVT, Op0, Op1); 10842 case MVT::v8i64: return fastEmit_ISD_SMAX_MVT_v8i64_rr(RetVT, Op0, Op1); 10843 default: return 0; 10844 } 10845} 10846 10847// FastEmit functions for ISD::SMIN. 10848 10849unsigned fastEmit_ISD_SMIN_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10850 if (RetVT.SimpleTy != MVT::v16i8) 10851 return 0; 10852 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 10853 return fastEmitInst_rr(X86::VPMINSBZ128rr, &X86::VR128XRegClass, Op0, Op1); 10854 } 10855 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 10856 return fastEmitInst_rr(X86::PMINSBrr, &X86::VR128RegClass, Op0, Op1); 10857 } 10858 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 10859 return fastEmitInst_rr(X86::VPMINSBrr, &X86::VR128RegClass, Op0, Op1); 10860 } 10861 return 0; 10862} 10863 10864unsigned fastEmit_ISD_SMIN_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10865 if (RetVT.SimpleTy != MVT::v32i8) 10866 return 0; 10867 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 10868 return fastEmitInst_rr(X86::VPMINSBZ256rr, &X86::VR256XRegClass, Op0, Op1); 10869 } 10870 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 10871 return fastEmitInst_rr(X86::VPMINSBYrr, &X86::VR256RegClass, Op0, Op1); 10872 } 10873 return 0; 10874} 10875 10876unsigned fastEmit_ISD_SMIN_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10877 if (RetVT.SimpleTy != MVT::v64i8) 10878 return 0; 10879 if ((Subtarget->hasBWI())) { 10880 return fastEmitInst_rr(X86::VPMINSBZrr, &X86::VR512RegClass, Op0, Op1); 10881 } 10882 return 0; 10883} 10884 10885unsigned fastEmit_ISD_SMIN_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10886 if (RetVT.SimpleTy != MVT::v8i16) 10887 return 0; 10888 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 10889 return fastEmitInst_rr(X86::VPMINSWZ128rr, &X86::VR128XRegClass, Op0, Op1); 10890 } 10891 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 10892 return fastEmitInst_rr(X86::PMINSWrr, &X86::VR128RegClass, Op0, Op1); 10893 } 10894 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 10895 return fastEmitInst_rr(X86::VPMINSWrr, &X86::VR128RegClass, Op0, Op1); 10896 } 10897 return 0; 10898} 10899 10900unsigned fastEmit_ISD_SMIN_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10901 if (RetVT.SimpleTy != MVT::v16i16) 10902 return 0; 10903 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 10904 return fastEmitInst_rr(X86::VPMINSWZ256rr, &X86::VR256XRegClass, Op0, Op1); 10905 } 10906 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 10907 return fastEmitInst_rr(X86::VPMINSWYrr, &X86::VR256RegClass, Op0, Op1); 10908 } 10909 return 0; 10910} 10911 10912unsigned fastEmit_ISD_SMIN_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10913 if (RetVT.SimpleTy != MVT::v32i16) 10914 return 0; 10915 if ((Subtarget->hasBWI())) { 10916 return fastEmitInst_rr(X86::VPMINSWZrr, &X86::VR512RegClass, Op0, Op1); 10917 } 10918 return 0; 10919} 10920 10921unsigned fastEmit_ISD_SMIN_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10922 if (RetVT.SimpleTy != MVT::v4i32) 10923 return 0; 10924 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 10925 return fastEmitInst_rr(X86::VPMINSDZ128rr, &X86::VR128XRegClass, Op0, Op1); 10926 } 10927 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 10928 return fastEmitInst_rr(X86::PMINSDrr, &X86::VR128RegClass, Op0, Op1); 10929 } 10930 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 10931 return fastEmitInst_rr(X86::VPMINSDrr, &X86::VR128RegClass, Op0, Op1); 10932 } 10933 return 0; 10934} 10935 10936unsigned fastEmit_ISD_SMIN_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10937 if (RetVT.SimpleTy != MVT::v8i32) 10938 return 0; 10939 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 10940 return fastEmitInst_rr(X86::VPMINSDZ256rr, &X86::VR256XRegClass, Op0, Op1); 10941 } 10942 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 10943 return fastEmitInst_rr(X86::VPMINSDYrr, &X86::VR256RegClass, Op0, Op1); 10944 } 10945 return 0; 10946} 10947 10948unsigned fastEmit_ISD_SMIN_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10949 if (RetVT.SimpleTy != MVT::v16i32) 10950 return 0; 10951 if ((Subtarget->hasAVX512())) { 10952 return fastEmitInst_rr(X86::VPMINSDZrr, &X86::VR512RegClass, Op0, Op1); 10953 } 10954 return 0; 10955} 10956 10957unsigned fastEmit_ISD_SMIN_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10958 if (RetVT.SimpleTy != MVT::v2i64) 10959 return 0; 10960 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 10961 return fastEmitInst_rr(X86::VPMINSQZ128rr, &X86::VR128XRegClass, Op0, Op1); 10962 } 10963 return 0; 10964} 10965 10966unsigned fastEmit_ISD_SMIN_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10967 if (RetVT.SimpleTy != MVT::v4i64) 10968 return 0; 10969 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 10970 return fastEmitInst_rr(X86::VPMINSQZ256rr, &X86::VR256XRegClass, Op0, Op1); 10971 } 10972 return 0; 10973} 10974 10975unsigned fastEmit_ISD_SMIN_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10976 if (RetVT.SimpleTy != MVT::v8i64) 10977 return 0; 10978 if ((Subtarget->hasAVX512())) { 10979 return fastEmitInst_rr(X86::VPMINSQZrr, &X86::VR512RegClass, Op0, Op1); 10980 } 10981 return 0; 10982} 10983 10984unsigned fastEmit_ISD_SMIN_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 10985 switch (VT.SimpleTy) { 10986 case MVT::v16i8: return fastEmit_ISD_SMIN_MVT_v16i8_rr(RetVT, Op0, Op1); 10987 case MVT::v32i8: return fastEmit_ISD_SMIN_MVT_v32i8_rr(RetVT, Op0, Op1); 10988 case MVT::v64i8: return fastEmit_ISD_SMIN_MVT_v64i8_rr(RetVT, Op0, Op1); 10989 case MVT::v8i16: return fastEmit_ISD_SMIN_MVT_v8i16_rr(RetVT, Op0, Op1); 10990 case MVT::v16i16: return fastEmit_ISD_SMIN_MVT_v16i16_rr(RetVT, Op0, Op1); 10991 case MVT::v32i16: return fastEmit_ISD_SMIN_MVT_v32i16_rr(RetVT, Op0, Op1); 10992 case MVT::v4i32: return fastEmit_ISD_SMIN_MVT_v4i32_rr(RetVT, Op0, Op1); 10993 case MVT::v8i32: return fastEmit_ISD_SMIN_MVT_v8i32_rr(RetVT, Op0, Op1); 10994 case MVT::v16i32: return fastEmit_ISD_SMIN_MVT_v16i32_rr(RetVT, Op0, Op1); 10995 case MVT::v2i64: return fastEmit_ISD_SMIN_MVT_v2i64_rr(RetVT, Op0, Op1); 10996 case MVT::v4i64: return fastEmit_ISD_SMIN_MVT_v4i64_rr(RetVT, Op0, Op1); 10997 case MVT::v8i64: return fastEmit_ISD_SMIN_MVT_v8i64_rr(RetVT, Op0, Op1); 10998 default: return 0; 10999 } 11000} 11001 11002// FastEmit functions for ISD::SRA. 11003 11004unsigned fastEmit_ISD_SRA_MVT_i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11005 if (RetVT.SimpleTy != MVT::i8) 11006 return 0; 11007 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(TargetOpcode::COPY), X86::CL).addReg(Op1); 11008 return fastEmitInst_r(X86::SAR8rCL, &X86::GR8RegClass, Op0); 11009} 11010 11011unsigned fastEmit_ISD_SRA_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 11012 switch (VT.SimpleTy) { 11013 case MVT::i8: return fastEmit_ISD_SRA_MVT_i8_rr(RetVT, Op0, Op1); 11014 default: return 0; 11015 } 11016} 11017 11018// FastEmit functions for ISD::SRL. 11019 11020unsigned fastEmit_ISD_SRL_MVT_i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11021 if (RetVT.SimpleTy != MVT::i8) 11022 return 0; 11023 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(TargetOpcode::COPY), X86::CL).addReg(Op1); 11024 return fastEmitInst_r(X86::SHR8rCL, &X86::GR8RegClass, Op0); 11025} 11026 11027unsigned fastEmit_ISD_SRL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 11028 switch (VT.SimpleTy) { 11029 case MVT::i8: return fastEmit_ISD_SRL_MVT_i8_rr(RetVT, Op0, Op1); 11030 default: return 0; 11031 } 11032} 11033 11034// FastEmit functions for ISD::SSUBSAT. 11035 11036unsigned fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11037 if (RetVT.SimpleTy != MVT::v16i8) 11038 return 0; 11039 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 11040 return fastEmitInst_rr(X86::VPSUBSBZ128rr, &X86::VR128XRegClass, Op0, Op1); 11041 } 11042 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 11043 return fastEmitInst_rr(X86::PSUBSBrr, &X86::VR128RegClass, Op0, Op1); 11044 } 11045 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 11046 return fastEmitInst_rr(X86::VPSUBSBrr, &X86::VR128RegClass, Op0, Op1); 11047 } 11048 return 0; 11049} 11050 11051unsigned fastEmit_ISD_SSUBSAT_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11052 if (RetVT.SimpleTy != MVT::v32i8) 11053 return 0; 11054 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 11055 return fastEmitInst_rr(X86::VPSUBSBZ256rr, &X86::VR256XRegClass, Op0, Op1); 11056 } 11057 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 11058 return fastEmitInst_rr(X86::VPSUBSBYrr, &X86::VR256RegClass, Op0, Op1); 11059 } 11060 return 0; 11061} 11062 11063unsigned fastEmit_ISD_SSUBSAT_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11064 if (RetVT.SimpleTy != MVT::v64i8) 11065 return 0; 11066 if ((Subtarget->hasBWI())) { 11067 return fastEmitInst_rr(X86::VPSUBSBZrr, &X86::VR512RegClass, Op0, Op1); 11068 } 11069 return 0; 11070} 11071 11072unsigned fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11073 if (RetVT.SimpleTy != MVT::v8i16) 11074 return 0; 11075 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 11076 return fastEmitInst_rr(X86::VPSUBSWZ128rr, &X86::VR128XRegClass, Op0, Op1); 11077 } 11078 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 11079 return fastEmitInst_rr(X86::PSUBSWrr, &X86::VR128RegClass, Op0, Op1); 11080 } 11081 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 11082 return fastEmitInst_rr(X86::VPSUBSWrr, &X86::VR128RegClass, Op0, Op1); 11083 } 11084 return 0; 11085} 11086 11087unsigned fastEmit_ISD_SSUBSAT_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11088 if (RetVT.SimpleTy != MVT::v16i16) 11089 return 0; 11090 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 11091 return fastEmitInst_rr(X86::VPSUBSWZ256rr, &X86::VR256XRegClass, Op0, Op1); 11092 } 11093 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 11094 return fastEmitInst_rr(X86::VPSUBSWYrr, &X86::VR256RegClass, Op0, Op1); 11095 } 11096 return 0; 11097} 11098 11099unsigned fastEmit_ISD_SSUBSAT_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11100 if (RetVT.SimpleTy != MVT::v32i16) 11101 return 0; 11102 if ((Subtarget->hasBWI())) { 11103 return fastEmitInst_rr(X86::VPSUBSWZrr, &X86::VR512RegClass, Op0, Op1); 11104 } 11105 return 0; 11106} 11107 11108unsigned fastEmit_ISD_SSUBSAT_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 11109 switch (VT.SimpleTy) { 11110 case MVT::v16i8: return fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1); 11111 case MVT::v32i8: return fastEmit_ISD_SSUBSAT_MVT_v32i8_rr(RetVT, Op0, Op1); 11112 case MVT::v64i8: return fastEmit_ISD_SSUBSAT_MVT_v64i8_rr(RetVT, Op0, Op1); 11113 case MVT::v8i16: return fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1); 11114 case MVT::v16i16: return fastEmit_ISD_SSUBSAT_MVT_v16i16_rr(RetVT, Op0, Op1); 11115 case MVT::v32i16: return fastEmit_ISD_SSUBSAT_MVT_v32i16_rr(RetVT, Op0, Op1); 11116 default: return 0; 11117 } 11118} 11119 11120// FastEmit functions for ISD::STRICT_FADD. 11121 11122unsigned fastEmit_ISD_STRICT_FADD_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11123 if (RetVT.SimpleTy != MVT::f16) 11124 return 0; 11125 if ((Subtarget->hasFP16())) { 11126 return fastEmitInst_rr(X86::VADDSHZrr, &X86::FR16XRegClass, Op0, Op1); 11127 } 11128 return 0; 11129} 11130 11131unsigned fastEmit_ISD_STRICT_FADD_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11132 if (RetVT.SimpleTy != MVT::f32) 11133 return 0; 11134 if ((Subtarget->hasAVX512())) { 11135 return fastEmitInst_rr(X86::VADDSSZrr, &X86::FR32XRegClass, Op0, Op1); 11136 } 11137 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 11138 return fastEmitInst_rr(X86::ADDSSrr, &X86::FR32RegClass, Op0, Op1); 11139 } 11140 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 11141 return fastEmitInst_rr(X86::VADDSSrr, &X86::FR32RegClass, Op0, Op1); 11142 } 11143 if ((!Subtarget->hasSSE1())) { 11144 return fastEmitInst_rr(X86::ADD_Fp32, &X86::RFP32RegClass, Op0, Op1); 11145 } 11146 return 0; 11147} 11148 11149unsigned fastEmit_ISD_STRICT_FADD_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11150 if (RetVT.SimpleTy != MVT::f64) 11151 return 0; 11152 if ((Subtarget->hasAVX512())) { 11153 return fastEmitInst_rr(X86::VADDSDZrr, &X86::FR64XRegClass, Op0, Op1); 11154 } 11155 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 11156 return fastEmitInst_rr(X86::ADDSDrr, &X86::FR64RegClass, Op0, Op1); 11157 } 11158 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 11159 return fastEmitInst_rr(X86::VADDSDrr, &X86::FR64RegClass, Op0, Op1); 11160 } 11161 if ((!Subtarget->hasSSE2())) { 11162 return fastEmitInst_rr(X86::ADD_Fp64, &X86::RFP64RegClass, Op0, Op1); 11163 } 11164 return 0; 11165} 11166 11167unsigned fastEmit_ISD_STRICT_FADD_MVT_f80_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11168 if (RetVT.SimpleTy != MVT::f80) 11169 return 0; 11170 if ((Subtarget->hasX87())) { 11171 return fastEmitInst_rr(X86::ADD_Fp80, &X86::RFP80RegClass, Op0, Op1); 11172 } 11173 return 0; 11174} 11175 11176unsigned fastEmit_ISD_STRICT_FADD_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11177 if (RetVT.SimpleTy != MVT::v8f16) 11178 return 0; 11179 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 11180 return fastEmitInst_rr(X86::VADDPHZ128rr, &X86::VR128XRegClass, Op0, Op1); 11181 } 11182 return 0; 11183} 11184 11185unsigned fastEmit_ISD_STRICT_FADD_MVT_v16f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11186 if (RetVT.SimpleTy != MVT::v16f16) 11187 return 0; 11188 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 11189 return fastEmitInst_rr(X86::VADDPHZ256rr, &X86::VR256XRegClass, Op0, Op1); 11190 } 11191 return 0; 11192} 11193 11194unsigned fastEmit_ISD_STRICT_FADD_MVT_v32f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11195 if (RetVT.SimpleTy != MVT::v32f16) 11196 return 0; 11197 if ((Subtarget->hasFP16())) { 11198 return fastEmitInst_rr(X86::VADDPHZrr, &X86::VR512RegClass, Op0, Op1); 11199 } 11200 return 0; 11201} 11202 11203unsigned fastEmit_ISD_STRICT_FADD_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11204 if (RetVT.SimpleTy != MVT::v4f32) 11205 return 0; 11206 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 11207 return fastEmitInst_rr(X86::VADDPSZ128rr, &X86::VR128XRegClass, Op0, Op1); 11208 } 11209 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 11210 return fastEmitInst_rr(X86::ADDPSrr, &X86::VR128RegClass, Op0, Op1); 11211 } 11212 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 11213 return fastEmitInst_rr(X86::VADDPSrr, &X86::VR128RegClass, Op0, Op1); 11214 } 11215 return 0; 11216} 11217 11218unsigned fastEmit_ISD_STRICT_FADD_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11219 if (RetVT.SimpleTy != MVT::v8f32) 11220 return 0; 11221 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 11222 return fastEmitInst_rr(X86::VADDPSZ256rr, &X86::VR256XRegClass, Op0, Op1); 11223 } 11224 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 11225 return fastEmitInst_rr(X86::VADDPSYrr, &X86::VR256RegClass, Op0, Op1); 11226 } 11227 return 0; 11228} 11229 11230unsigned fastEmit_ISD_STRICT_FADD_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11231 if (RetVT.SimpleTy != MVT::v16f32) 11232 return 0; 11233 if ((Subtarget->hasAVX512())) { 11234 return fastEmitInst_rr(X86::VADDPSZrr, &X86::VR512RegClass, Op0, Op1); 11235 } 11236 return 0; 11237} 11238 11239unsigned fastEmit_ISD_STRICT_FADD_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11240 if (RetVT.SimpleTy != MVT::v2f64) 11241 return 0; 11242 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 11243 return fastEmitInst_rr(X86::VADDPDZ128rr, &X86::VR128XRegClass, Op0, Op1); 11244 } 11245 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 11246 return fastEmitInst_rr(X86::ADDPDrr, &X86::VR128RegClass, Op0, Op1); 11247 } 11248 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 11249 return fastEmitInst_rr(X86::VADDPDrr, &X86::VR128RegClass, Op0, Op1); 11250 } 11251 return 0; 11252} 11253 11254unsigned fastEmit_ISD_STRICT_FADD_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11255 if (RetVT.SimpleTy != MVT::v4f64) 11256 return 0; 11257 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 11258 return fastEmitInst_rr(X86::VADDPDZ256rr, &X86::VR256XRegClass, Op0, Op1); 11259 } 11260 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 11261 return fastEmitInst_rr(X86::VADDPDYrr, &X86::VR256RegClass, Op0, Op1); 11262 } 11263 return 0; 11264} 11265 11266unsigned fastEmit_ISD_STRICT_FADD_MVT_v8f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11267 if (RetVT.SimpleTy != MVT::v8f64) 11268 return 0; 11269 if ((Subtarget->hasAVX512())) { 11270 return fastEmitInst_rr(X86::VADDPDZrr, &X86::VR512RegClass, Op0, Op1); 11271 } 11272 return 0; 11273} 11274 11275unsigned fastEmit_ISD_STRICT_FADD_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 11276 switch (VT.SimpleTy) { 11277 case MVT::f16: return fastEmit_ISD_STRICT_FADD_MVT_f16_rr(RetVT, Op0, Op1); 11278 case MVT::f32: return fastEmit_ISD_STRICT_FADD_MVT_f32_rr(RetVT, Op0, Op1); 11279 case MVT::f64: return fastEmit_ISD_STRICT_FADD_MVT_f64_rr(RetVT, Op0, Op1); 11280 case MVT::f80: return fastEmit_ISD_STRICT_FADD_MVT_f80_rr(RetVT, Op0, Op1); 11281 case MVT::v8f16: return fastEmit_ISD_STRICT_FADD_MVT_v8f16_rr(RetVT, Op0, Op1); 11282 case MVT::v16f16: return fastEmit_ISD_STRICT_FADD_MVT_v16f16_rr(RetVT, Op0, Op1); 11283 case MVT::v32f16: return fastEmit_ISD_STRICT_FADD_MVT_v32f16_rr(RetVT, Op0, Op1); 11284 case MVT::v4f32: return fastEmit_ISD_STRICT_FADD_MVT_v4f32_rr(RetVT, Op0, Op1); 11285 case MVT::v8f32: return fastEmit_ISD_STRICT_FADD_MVT_v8f32_rr(RetVT, Op0, Op1); 11286 case MVT::v16f32: return fastEmit_ISD_STRICT_FADD_MVT_v16f32_rr(RetVT, Op0, Op1); 11287 case MVT::v2f64: return fastEmit_ISD_STRICT_FADD_MVT_v2f64_rr(RetVT, Op0, Op1); 11288 case MVT::v4f64: return fastEmit_ISD_STRICT_FADD_MVT_v4f64_rr(RetVT, Op0, Op1); 11289 case MVT::v8f64: return fastEmit_ISD_STRICT_FADD_MVT_v8f64_rr(RetVT, Op0, Op1); 11290 default: return 0; 11291 } 11292} 11293 11294// FastEmit functions for ISD::STRICT_FDIV. 11295 11296unsigned fastEmit_ISD_STRICT_FDIV_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11297 if (RetVT.SimpleTy != MVT::f16) 11298 return 0; 11299 if ((Subtarget->hasFP16())) { 11300 return fastEmitInst_rr(X86::VDIVSHZrr, &X86::FR16XRegClass, Op0, Op1); 11301 } 11302 return 0; 11303} 11304 11305unsigned fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11306 if (RetVT.SimpleTy != MVT::f32) 11307 return 0; 11308 if ((Subtarget->hasAVX512())) { 11309 return fastEmitInst_rr(X86::VDIVSSZrr, &X86::FR32XRegClass, Op0, Op1); 11310 } 11311 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 11312 return fastEmitInst_rr(X86::DIVSSrr, &X86::FR32RegClass, Op0, Op1); 11313 } 11314 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 11315 return fastEmitInst_rr(X86::VDIVSSrr, &X86::FR32RegClass, Op0, Op1); 11316 } 11317 if ((!Subtarget->hasSSE1())) { 11318 return fastEmitInst_rr(X86::DIV_Fp32, &X86::RFP32RegClass, Op0, Op1); 11319 } 11320 return 0; 11321} 11322 11323unsigned fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11324 if (RetVT.SimpleTy != MVT::f64) 11325 return 0; 11326 if ((Subtarget->hasAVX512())) { 11327 return fastEmitInst_rr(X86::VDIVSDZrr, &X86::FR64XRegClass, Op0, Op1); 11328 } 11329 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 11330 return fastEmitInst_rr(X86::DIVSDrr, &X86::FR64RegClass, Op0, Op1); 11331 } 11332 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 11333 return fastEmitInst_rr(X86::VDIVSDrr, &X86::FR64RegClass, Op0, Op1); 11334 } 11335 if ((!Subtarget->hasSSE2())) { 11336 return fastEmitInst_rr(X86::DIV_Fp64, &X86::RFP64RegClass, Op0, Op1); 11337 } 11338 return 0; 11339} 11340 11341unsigned fastEmit_ISD_STRICT_FDIV_MVT_f80_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11342 if (RetVT.SimpleTy != MVT::f80) 11343 return 0; 11344 if ((Subtarget->hasX87())) { 11345 return fastEmitInst_rr(X86::DIV_Fp80, &X86::RFP80RegClass, Op0, Op1); 11346 } 11347 return 0; 11348} 11349 11350unsigned fastEmit_ISD_STRICT_FDIV_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11351 if (RetVT.SimpleTy != MVT::v8f16) 11352 return 0; 11353 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 11354 return fastEmitInst_rr(X86::VDIVPHZ128rr, &X86::VR128XRegClass, Op0, Op1); 11355 } 11356 return 0; 11357} 11358 11359unsigned fastEmit_ISD_STRICT_FDIV_MVT_v16f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11360 if (RetVT.SimpleTy != MVT::v16f16) 11361 return 0; 11362 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 11363 return fastEmitInst_rr(X86::VDIVPHZ256rr, &X86::VR256XRegClass, Op0, Op1); 11364 } 11365 return 0; 11366} 11367 11368unsigned fastEmit_ISD_STRICT_FDIV_MVT_v32f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11369 if (RetVT.SimpleTy != MVT::v32f16) 11370 return 0; 11371 if ((Subtarget->hasFP16())) { 11372 return fastEmitInst_rr(X86::VDIVPHZrr, &X86::VR512RegClass, Op0, Op1); 11373 } 11374 return 0; 11375} 11376 11377unsigned fastEmit_ISD_STRICT_FDIV_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11378 if (RetVT.SimpleTy != MVT::v4f32) 11379 return 0; 11380 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 11381 return fastEmitInst_rr(X86::VDIVPSZ128rr, &X86::VR128XRegClass, Op0, Op1); 11382 } 11383 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 11384 return fastEmitInst_rr(X86::DIVPSrr, &X86::VR128RegClass, Op0, Op1); 11385 } 11386 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 11387 return fastEmitInst_rr(X86::VDIVPSrr, &X86::VR128RegClass, Op0, Op1); 11388 } 11389 return 0; 11390} 11391 11392unsigned fastEmit_ISD_STRICT_FDIV_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11393 if (RetVT.SimpleTy != MVT::v8f32) 11394 return 0; 11395 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 11396 return fastEmitInst_rr(X86::VDIVPSZ256rr, &X86::VR256XRegClass, Op0, Op1); 11397 } 11398 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 11399 return fastEmitInst_rr(X86::VDIVPSYrr, &X86::VR256RegClass, Op0, Op1); 11400 } 11401 return 0; 11402} 11403 11404unsigned fastEmit_ISD_STRICT_FDIV_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11405 if (RetVT.SimpleTy != MVT::v16f32) 11406 return 0; 11407 if ((Subtarget->hasAVX512())) { 11408 return fastEmitInst_rr(X86::VDIVPSZrr, &X86::VR512RegClass, Op0, Op1); 11409 } 11410 return 0; 11411} 11412 11413unsigned fastEmit_ISD_STRICT_FDIV_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11414 if (RetVT.SimpleTy != MVT::v2f64) 11415 return 0; 11416 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 11417 return fastEmitInst_rr(X86::VDIVPDZ128rr, &X86::VR128XRegClass, Op0, Op1); 11418 } 11419 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 11420 return fastEmitInst_rr(X86::DIVPDrr, &X86::VR128RegClass, Op0, Op1); 11421 } 11422 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 11423 return fastEmitInst_rr(X86::VDIVPDrr, &X86::VR128RegClass, Op0, Op1); 11424 } 11425 return 0; 11426} 11427 11428unsigned fastEmit_ISD_STRICT_FDIV_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11429 if (RetVT.SimpleTy != MVT::v4f64) 11430 return 0; 11431 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 11432 return fastEmitInst_rr(X86::VDIVPDZ256rr, &X86::VR256XRegClass, Op0, Op1); 11433 } 11434 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 11435 return fastEmitInst_rr(X86::VDIVPDYrr, &X86::VR256RegClass, Op0, Op1); 11436 } 11437 return 0; 11438} 11439 11440unsigned fastEmit_ISD_STRICT_FDIV_MVT_v8f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11441 if (RetVT.SimpleTy != MVT::v8f64) 11442 return 0; 11443 if ((Subtarget->hasAVX512())) { 11444 return fastEmitInst_rr(X86::VDIVPDZrr, &X86::VR512RegClass, Op0, Op1); 11445 } 11446 return 0; 11447} 11448 11449unsigned fastEmit_ISD_STRICT_FDIV_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 11450 switch (VT.SimpleTy) { 11451 case MVT::f16: return fastEmit_ISD_STRICT_FDIV_MVT_f16_rr(RetVT, Op0, Op1); 11452 case MVT::f32: return fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(RetVT, Op0, Op1); 11453 case MVT::f64: return fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(RetVT, Op0, Op1); 11454 case MVT::f80: return fastEmit_ISD_STRICT_FDIV_MVT_f80_rr(RetVT, Op0, Op1); 11455 case MVT::v8f16: return fastEmit_ISD_STRICT_FDIV_MVT_v8f16_rr(RetVT, Op0, Op1); 11456 case MVT::v16f16: return fastEmit_ISD_STRICT_FDIV_MVT_v16f16_rr(RetVT, Op0, Op1); 11457 case MVT::v32f16: return fastEmit_ISD_STRICT_FDIV_MVT_v32f16_rr(RetVT, Op0, Op1); 11458 case MVT::v4f32: return fastEmit_ISD_STRICT_FDIV_MVT_v4f32_rr(RetVT, Op0, Op1); 11459 case MVT::v8f32: return fastEmit_ISD_STRICT_FDIV_MVT_v8f32_rr(RetVT, Op0, Op1); 11460 case MVT::v16f32: return fastEmit_ISD_STRICT_FDIV_MVT_v16f32_rr(RetVT, Op0, Op1); 11461 case MVT::v2f64: return fastEmit_ISD_STRICT_FDIV_MVT_v2f64_rr(RetVT, Op0, Op1); 11462 case MVT::v4f64: return fastEmit_ISD_STRICT_FDIV_MVT_v4f64_rr(RetVT, Op0, Op1); 11463 case MVT::v8f64: return fastEmit_ISD_STRICT_FDIV_MVT_v8f64_rr(RetVT, Op0, Op1); 11464 default: return 0; 11465 } 11466} 11467 11468// FastEmit functions for ISD::STRICT_FMUL. 11469 11470unsigned fastEmit_ISD_STRICT_FMUL_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11471 if (RetVT.SimpleTy != MVT::f16) 11472 return 0; 11473 if ((Subtarget->hasFP16())) { 11474 return fastEmitInst_rr(X86::VMULSHZrr, &X86::FR16XRegClass, Op0, Op1); 11475 } 11476 return 0; 11477} 11478 11479unsigned fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11480 if (RetVT.SimpleTy != MVT::f32) 11481 return 0; 11482 if ((Subtarget->hasAVX512())) { 11483 return fastEmitInst_rr(X86::VMULSSZrr, &X86::FR32XRegClass, Op0, Op1); 11484 } 11485 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 11486 return fastEmitInst_rr(X86::MULSSrr, &X86::FR32RegClass, Op0, Op1); 11487 } 11488 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 11489 return fastEmitInst_rr(X86::VMULSSrr, &X86::FR32RegClass, Op0, Op1); 11490 } 11491 if ((!Subtarget->hasSSE1())) { 11492 return fastEmitInst_rr(X86::MUL_Fp32, &X86::RFP32RegClass, Op0, Op1); 11493 } 11494 return 0; 11495} 11496 11497unsigned fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11498 if (RetVT.SimpleTy != MVT::f64) 11499 return 0; 11500 if ((Subtarget->hasAVX512())) { 11501 return fastEmitInst_rr(X86::VMULSDZrr, &X86::FR64XRegClass, Op0, Op1); 11502 } 11503 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 11504 return fastEmitInst_rr(X86::MULSDrr, &X86::FR64RegClass, Op0, Op1); 11505 } 11506 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 11507 return fastEmitInst_rr(X86::VMULSDrr, &X86::FR64RegClass, Op0, Op1); 11508 } 11509 if ((!Subtarget->hasSSE2())) { 11510 return fastEmitInst_rr(X86::MUL_Fp64, &X86::RFP64RegClass, Op0, Op1); 11511 } 11512 return 0; 11513} 11514 11515unsigned fastEmit_ISD_STRICT_FMUL_MVT_f80_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11516 if (RetVT.SimpleTy != MVT::f80) 11517 return 0; 11518 if ((Subtarget->hasX87())) { 11519 return fastEmitInst_rr(X86::MUL_Fp80, &X86::RFP80RegClass, Op0, Op1); 11520 } 11521 return 0; 11522} 11523 11524unsigned fastEmit_ISD_STRICT_FMUL_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11525 if (RetVT.SimpleTy != MVT::v8f16) 11526 return 0; 11527 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 11528 return fastEmitInst_rr(X86::VMULPHZ128rr, &X86::VR128XRegClass, Op0, Op1); 11529 } 11530 return 0; 11531} 11532 11533unsigned fastEmit_ISD_STRICT_FMUL_MVT_v16f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11534 if (RetVT.SimpleTy != MVT::v16f16) 11535 return 0; 11536 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 11537 return fastEmitInst_rr(X86::VMULPHZ256rr, &X86::VR256XRegClass, Op0, Op1); 11538 } 11539 return 0; 11540} 11541 11542unsigned fastEmit_ISD_STRICT_FMUL_MVT_v32f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11543 if (RetVT.SimpleTy != MVT::v32f16) 11544 return 0; 11545 if ((Subtarget->hasFP16())) { 11546 return fastEmitInst_rr(X86::VMULPHZrr, &X86::VR512RegClass, Op0, Op1); 11547 } 11548 return 0; 11549} 11550 11551unsigned fastEmit_ISD_STRICT_FMUL_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11552 if (RetVT.SimpleTy != MVT::v4f32) 11553 return 0; 11554 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 11555 return fastEmitInst_rr(X86::VMULPSZ128rr, &X86::VR128XRegClass, Op0, Op1); 11556 } 11557 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 11558 return fastEmitInst_rr(X86::MULPSrr, &X86::VR128RegClass, Op0, Op1); 11559 } 11560 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 11561 return fastEmitInst_rr(X86::VMULPSrr, &X86::VR128RegClass, Op0, Op1); 11562 } 11563 return 0; 11564} 11565 11566unsigned fastEmit_ISD_STRICT_FMUL_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11567 if (RetVT.SimpleTy != MVT::v8f32) 11568 return 0; 11569 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 11570 return fastEmitInst_rr(X86::VMULPSZ256rr, &X86::VR256XRegClass, Op0, Op1); 11571 } 11572 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 11573 return fastEmitInst_rr(X86::VMULPSYrr, &X86::VR256RegClass, Op0, Op1); 11574 } 11575 return 0; 11576} 11577 11578unsigned fastEmit_ISD_STRICT_FMUL_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11579 if (RetVT.SimpleTy != MVT::v16f32) 11580 return 0; 11581 if ((Subtarget->hasAVX512())) { 11582 return fastEmitInst_rr(X86::VMULPSZrr, &X86::VR512RegClass, Op0, Op1); 11583 } 11584 return 0; 11585} 11586 11587unsigned fastEmit_ISD_STRICT_FMUL_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11588 if (RetVT.SimpleTy != MVT::v2f64) 11589 return 0; 11590 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 11591 return fastEmitInst_rr(X86::VMULPDZ128rr, &X86::VR128XRegClass, Op0, Op1); 11592 } 11593 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 11594 return fastEmitInst_rr(X86::MULPDrr, &X86::VR128RegClass, Op0, Op1); 11595 } 11596 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 11597 return fastEmitInst_rr(X86::VMULPDrr, &X86::VR128RegClass, Op0, Op1); 11598 } 11599 return 0; 11600} 11601 11602unsigned fastEmit_ISD_STRICT_FMUL_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11603 if (RetVT.SimpleTy != MVT::v4f64) 11604 return 0; 11605 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 11606 return fastEmitInst_rr(X86::VMULPDZ256rr, &X86::VR256XRegClass, Op0, Op1); 11607 } 11608 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 11609 return fastEmitInst_rr(X86::VMULPDYrr, &X86::VR256RegClass, Op0, Op1); 11610 } 11611 return 0; 11612} 11613 11614unsigned fastEmit_ISD_STRICT_FMUL_MVT_v8f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11615 if (RetVT.SimpleTy != MVT::v8f64) 11616 return 0; 11617 if ((Subtarget->hasAVX512())) { 11618 return fastEmitInst_rr(X86::VMULPDZrr, &X86::VR512RegClass, Op0, Op1); 11619 } 11620 return 0; 11621} 11622 11623unsigned fastEmit_ISD_STRICT_FMUL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 11624 switch (VT.SimpleTy) { 11625 case MVT::f16: return fastEmit_ISD_STRICT_FMUL_MVT_f16_rr(RetVT, Op0, Op1); 11626 case MVT::f32: return fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(RetVT, Op0, Op1); 11627 case MVT::f64: return fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(RetVT, Op0, Op1); 11628 case MVT::f80: return fastEmit_ISD_STRICT_FMUL_MVT_f80_rr(RetVT, Op0, Op1); 11629 case MVT::v8f16: return fastEmit_ISD_STRICT_FMUL_MVT_v8f16_rr(RetVT, Op0, Op1); 11630 case MVT::v16f16: return fastEmit_ISD_STRICT_FMUL_MVT_v16f16_rr(RetVT, Op0, Op1); 11631 case MVT::v32f16: return fastEmit_ISD_STRICT_FMUL_MVT_v32f16_rr(RetVT, Op0, Op1); 11632 case MVT::v4f32: return fastEmit_ISD_STRICT_FMUL_MVT_v4f32_rr(RetVT, Op0, Op1); 11633 case MVT::v8f32: return fastEmit_ISD_STRICT_FMUL_MVT_v8f32_rr(RetVT, Op0, Op1); 11634 case MVT::v16f32: return fastEmit_ISD_STRICT_FMUL_MVT_v16f32_rr(RetVT, Op0, Op1); 11635 case MVT::v2f64: return fastEmit_ISD_STRICT_FMUL_MVT_v2f64_rr(RetVT, Op0, Op1); 11636 case MVT::v4f64: return fastEmit_ISD_STRICT_FMUL_MVT_v4f64_rr(RetVT, Op0, Op1); 11637 case MVT::v8f64: return fastEmit_ISD_STRICT_FMUL_MVT_v8f64_rr(RetVT, Op0, Op1); 11638 default: return 0; 11639 } 11640} 11641 11642// FastEmit functions for ISD::STRICT_FSUB. 11643 11644unsigned fastEmit_ISD_STRICT_FSUB_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11645 if (RetVT.SimpleTy != MVT::f16) 11646 return 0; 11647 if ((Subtarget->hasFP16())) { 11648 return fastEmitInst_rr(X86::VSUBSHZrr, &X86::FR16XRegClass, Op0, Op1); 11649 } 11650 return 0; 11651} 11652 11653unsigned fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11654 if (RetVT.SimpleTy != MVT::f32) 11655 return 0; 11656 if ((Subtarget->hasAVX512())) { 11657 return fastEmitInst_rr(X86::VSUBSSZrr, &X86::FR32XRegClass, Op0, Op1); 11658 } 11659 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 11660 return fastEmitInst_rr(X86::SUBSSrr, &X86::FR32RegClass, Op0, Op1); 11661 } 11662 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 11663 return fastEmitInst_rr(X86::VSUBSSrr, &X86::FR32RegClass, Op0, Op1); 11664 } 11665 if ((!Subtarget->hasSSE1())) { 11666 return fastEmitInst_rr(X86::SUB_Fp32, &X86::RFP32RegClass, Op0, Op1); 11667 } 11668 return 0; 11669} 11670 11671unsigned fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11672 if (RetVT.SimpleTy != MVT::f64) 11673 return 0; 11674 if ((Subtarget->hasAVX512())) { 11675 return fastEmitInst_rr(X86::VSUBSDZrr, &X86::FR64XRegClass, Op0, Op1); 11676 } 11677 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 11678 return fastEmitInst_rr(X86::SUBSDrr, &X86::FR64RegClass, Op0, Op1); 11679 } 11680 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 11681 return fastEmitInst_rr(X86::VSUBSDrr, &X86::FR64RegClass, Op0, Op1); 11682 } 11683 if ((!Subtarget->hasSSE2())) { 11684 return fastEmitInst_rr(X86::SUB_Fp64, &X86::RFP64RegClass, Op0, Op1); 11685 } 11686 return 0; 11687} 11688 11689unsigned fastEmit_ISD_STRICT_FSUB_MVT_f80_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11690 if (RetVT.SimpleTy != MVT::f80) 11691 return 0; 11692 if ((Subtarget->hasX87())) { 11693 return fastEmitInst_rr(X86::SUB_Fp80, &X86::RFP80RegClass, Op0, Op1); 11694 } 11695 return 0; 11696} 11697 11698unsigned fastEmit_ISD_STRICT_FSUB_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11699 if (RetVT.SimpleTy != MVT::v8f16) 11700 return 0; 11701 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 11702 return fastEmitInst_rr(X86::VSUBPHZ128rr, &X86::VR128XRegClass, Op0, Op1); 11703 } 11704 return 0; 11705} 11706 11707unsigned fastEmit_ISD_STRICT_FSUB_MVT_v16f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11708 if (RetVT.SimpleTy != MVT::v16f16) 11709 return 0; 11710 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 11711 return fastEmitInst_rr(X86::VSUBPHZ256rr, &X86::VR256XRegClass, Op0, Op1); 11712 } 11713 return 0; 11714} 11715 11716unsigned fastEmit_ISD_STRICT_FSUB_MVT_v32f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11717 if (RetVT.SimpleTy != MVT::v32f16) 11718 return 0; 11719 if ((Subtarget->hasFP16())) { 11720 return fastEmitInst_rr(X86::VSUBPHZrr, &X86::VR512RegClass, Op0, Op1); 11721 } 11722 return 0; 11723} 11724 11725unsigned fastEmit_ISD_STRICT_FSUB_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11726 if (RetVT.SimpleTy != MVT::v4f32) 11727 return 0; 11728 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 11729 return fastEmitInst_rr(X86::VSUBPSZ128rr, &X86::VR128XRegClass, Op0, Op1); 11730 } 11731 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 11732 return fastEmitInst_rr(X86::SUBPSrr, &X86::VR128RegClass, Op0, Op1); 11733 } 11734 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 11735 return fastEmitInst_rr(X86::VSUBPSrr, &X86::VR128RegClass, Op0, Op1); 11736 } 11737 return 0; 11738} 11739 11740unsigned fastEmit_ISD_STRICT_FSUB_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11741 if (RetVT.SimpleTy != MVT::v8f32) 11742 return 0; 11743 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 11744 return fastEmitInst_rr(X86::VSUBPSZ256rr, &X86::VR256XRegClass, Op0, Op1); 11745 } 11746 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 11747 return fastEmitInst_rr(X86::VSUBPSYrr, &X86::VR256RegClass, Op0, Op1); 11748 } 11749 return 0; 11750} 11751 11752unsigned fastEmit_ISD_STRICT_FSUB_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11753 if (RetVT.SimpleTy != MVT::v16f32) 11754 return 0; 11755 if ((Subtarget->hasAVX512())) { 11756 return fastEmitInst_rr(X86::VSUBPSZrr, &X86::VR512RegClass, Op0, Op1); 11757 } 11758 return 0; 11759} 11760 11761unsigned fastEmit_ISD_STRICT_FSUB_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11762 if (RetVT.SimpleTy != MVT::v2f64) 11763 return 0; 11764 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 11765 return fastEmitInst_rr(X86::VSUBPDZ128rr, &X86::VR128XRegClass, Op0, Op1); 11766 } 11767 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 11768 return fastEmitInst_rr(X86::SUBPDrr, &X86::VR128RegClass, Op0, Op1); 11769 } 11770 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 11771 return fastEmitInst_rr(X86::VSUBPDrr, &X86::VR128RegClass, Op0, Op1); 11772 } 11773 return 0; 11774} 11775 11776unsigned fastEmit_ISD_STRICT_FSUB_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11777 if (RetVT.SimpleTy != MVT::v4f64) 11778 return 0; 11779 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 11780 return fastEmitInst_rr(X86::VSUBPDZ256rr, &X86::VR256XRegClass, Op0, Op1); 11781 } 11782 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 11783 return fastEmitInst_rr(X86::VSUBPDYrr, &X86::VR256RegClass, Op0, Op1); 11784 } 11785 return 0; 11786} 11787 11788unsigned fastEmit_ISD_STRICT_FSUB_MVT_v8f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11789 if (RetVT.SimpleTy != MVT::v8f64) 11790 return 0; 11791 if ((Subtarget->hasAVX512())) { 11792 return fastEmitInst_rr(X86::VSUBPDZrr, &X86::VR512RegClass, Op0, Op1); 11793 } 11794 return 0; 11795} 11796 11797unsigned fastEmit_ISD_STRICT_FSUB_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 11798 switch (VT.SimpleTy) { 11799 case MVT::f16: return fastEmit_ISD_STRICT_FSUB_MVT_f16_rr(RetVT, Op0, Op1); 11800 case MVT::f32: return fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(RetVT, Op0, Op1); 11801 case MVT::f64: return fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(RetVT, Op0, Op1); 11802 case MVT::f80: return fastEmit_ISD_STRICT_FSUB_MVT_f80_rr(RetVT, Op0, Op1); 11803 case MVT::v8f16: return fastEmit_ISD_STRICT_FSUB_MVT_v8f16_rr(RetVT, Op0, Op1); 11804 case MVT::v16f16: return fastEmit_ISD_STRICT_FSUB_MVT_v16f16_rr(RetVT, Op0, Op1); 11805 case MVT::v32f16: return fastEmit_ISD_STRICT_FSUB_MVT_v32f16_rr(RetVT, Op0, Op1); 11806 case MVT::v4f32: return fastEmit_ISD_STRICT_FSUB_MVT_v4f32_rr(RetVT, Op0, Op1); 11807 case MVT::v8f32: return fastEmit_ISD_STRICT_FSUB_MVT_v8f32_rr(RetVT, Op0, Op1); 11808 case MVT::v16f32: return fastEmit_ISD_STRICT_FSUB_MVT_v16f32_rr(RetVT, Op0, Op1); 11809 case MVT::v2f64: return fastEmit_ISD_STRICT_FSUB_MVT_v2f64_rr(RetVT, Op0, Op1); 11810 case MVT::v4f64: return fastEmit_ISD_STRICT_FSUB_MVT_v4f64_rr(RetVT, Op0, Op1); 11811 case MVT::v8f64: return fastEmit_ISD_STRICT_FSUB_MVT_v8f64_rr(RetVT, Op0, Op1); 11812 default: return 0; 11813 } 11814} 11815 11816// FastEmit functions for ISD::SUB. 11817 11818unsigned fastEmit_ISD_SUB_MVT_i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11819 if (RetVT.SimpleTy != MVT::i8) 11820 return 0; 11821 return fastEmitInst_rr(X86::SUB8rr, &X86::GR8RegClass, Op0, Op1); 11822} 11823 11824unsigned fastEmit_ISD_SUB_MVT_i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11825 if (RetVT.SimpleTy != MVT::i16) 11826 return 0; 11827 return fastEmitInst_rr(X86::SUB16rr, &X86::GR16RegClass, Op0, Op1); 11828} 11829 11830unsigned fastEmit_ISD_SUB_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11831 if (RetVT.SimpleTy != MVT::i32) 11832 return 0; 11833 return fastEmitInst_rr(X86::SUB32rr, &X86::GR32RegClass, Op0, Op1); 11834} 11835 11836unsigned fastEmit_ISD_SUB_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11837 if (RetVT.SimpleTy != MVT::i64) 11838 return 0; 11839 return fastEmitInst_rr(X86::SUB64rr, &X86::GR64RegClass, Op0, Op1); 11840} 11841 11842unsigned fastEmit_ISD_SUB_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11843 if (RetVT.SimpleTy != MVT::v16i8) 11844 return 0; 11845 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 11846 return fastEmitInst_rr(X86::VPSUBBZ128rr, &X86::VR128XRegClass, Op0, Op1); 11847 } 11848 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 11849 return fastEmitInst_rr(X86::PSUBBrr, &X86::VR128RegClass, Op0, Op1); 11850 } 11851 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 11852 return fastEmitInst_rr(X86::VPSUBBrr, &X86::VR128RegClass, Op0, Op1); 11853 } 11854 return 0; 11855} 11856 11857unsigned fastEmit_ISD_SUB_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11858 if (RetVT.SimpleTy != MVT::v32i8) 11859 return 0; 11860 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 11861 return fastEmitInst_rr(X86::VPSUBBZ256rr, &X86::VR256XRegClass, Op0, Op1); 11862 } 11863 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 11864 return fastEmitInst_rr(X86::VPSUBBYrr, &X86::VR256RegClass, Op0, Op1); 11865 } 11866 return 0; 11867} 11868 11869unsigned fastEmit_ISD_SUB_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11870 if (RetVT.SimpleTy != MVT::v64i8) 11871 return 0; 11872 if ((Subtarget->hasBWI())) { 11873 return fastEmitInst_rr(X86::VPSUBBZrr, &X86::VR512RegClass, Op0, Op1); 11874 } 11875 return 0; 11876} 11877 11878unsigned fastEmit_ISD_SUB_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11879 if (RetVT.SimpleTy != MVT::v8i16) 11880 return 0; 11881 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 11882 return fastEmitInst_rr(X86::VPSUBWZ128rr, &X86::VR128XRegClass, Op0, Op1); 11883 } 11884 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 11885 return fastEmitInst_rr(X86::PSUBWrr, &X86::VR128RegClass, Op0, Op1); 11886 } 11887 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 11888 return fastEmitInst_rr(X86::VPSUBWrr, &X86::VR128RegClass, Op0, Op1); 11889 } 11890 return 0; 11891} 11892 11893unsigned fastEmit_ISD_SUB_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11894 if (RetVT.SimpleTy != MVT::v16i16) 11895 return 0; 11896 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 11897 return fastEmitInst_rr(X86::VPSUBWZ256rr, &X86::VR256XRegClass, Op0, Op1); 11898 } 11899 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 11900 return fastEmitInst_rr(X86::VPSUBWYrr, &X86::VR256RegClass, Op0, Op1); 11901 } 11902 return 0; 11903} 11904 11905unsigned fastEmit_ISD_SUB_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11906 if (RetVT.SimpleTy != MVT::v32i16) 11907 return 0; 11908 if ((Subtarget->hasBWI())) { 11909 return fastEmitInst_rr(X86::VPSUBWZrr, &X86::VR512RegClass, Op0, Op1); 11910 } 11911 return 0; 11912} 11913 11914unsigned fastEmit_ISD_SUB_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11915 if (RetVT.SimpleTy != MVT::v4i32) 11916 return 0; 11917 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 11918 return fastEmitInst_rr(X86::VPSUBDZ128rr, &X86::VR128XRegClass, Op0, Op1); 11919 } 11920 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 11921 return fastEmitInst_rr(X86::PSUBDrr, &X86::VR128RegClass, Op0, Op1); 11922 } 11923 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 11924 return fastEmitInst_rr(X86::VPSUBDrr, &X86::VR128RegClass, Op0, Op1); 11925 } 11926 return 0; 11927} 11928 11929unsigned fastEmit_ISD_SUB_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11930 if (RetVT.SimpleTy != MVT::v8i32) 11931 return 0; 11932 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 11933 return fastEmitInst_rr(X86::VPSUBDZ256rr, &X86::VR256XRegClass, Op0, Op1); 11934 } 11935 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 11936 return fastEmitInst_rr(X86::VPSUBDYrr, &X86::VR256RegClass, Op0, Op1); 11937 } 11938 return 0; 11939} 11940 11941unsigned fastEmit_ISD_SUB_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11942 if (RetVT.SimpleTy != MVT::v16i32) 11943 return 0; 11944 if ((Subtarget->hasAVX512())) { 11945 return fastEmitInst_rr(X86::VPSUBDZrr, &X86::VR512RegClass, Op0, Op1); 11946 } 11947 return 0; 11948} 11949 11950unsigned fastEmit_ISD_SUB_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11951 if (RetVT.SimpleTy != MVT::v2i64) 11952 return 0; 11953 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 11954 return fastEmitInst_rr(X86::VPSUBQZ128rr, &X86::VR128XRegClass, Op0, Op1); 11955 } 11956 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 11957 return fastEmitInst_rr(X86::PSUBQrr, &X86::VR128RegClass, Op0, Op1); 11958 } 11959 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 11960 return fastEmitInst_rr(X86::VPSUBQrr, &X86::VR128RegClass, Op0, Op1); 11961 } 11962 return 0; 11963} 11964 11965unsigned fastEmit_ISD_SUB_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11966 if (RetVT.SimpleTy != MVT::v4i64) 11967 return 0; 11968 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 11969 return fastEmitInst_rr(X86::VPSUBQZ256rr, &X86::VR256XRegClass, Op0, Op1); 11970 } 11971 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 11972 return fastEmitInst_rr(X86::VPSUBQYrr, &X86::VR256RegClass, Op0, Op1); 11973 } 11974 return 0; 11975} 11976 11977unsigned fastEmit_ISD_SUB_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11978 if (RetVT.SimpleTy != MVT::v8i64) 11979 return 0; 11980 if ((Subtarget->hasAVX512())) { 11981 return fastEmitInst_rr(X86::VPSUBQZrr, &X86::VR512RegClass, Op0, Op1); 11982 } 11983 return 0; 11984} 11985 11986unsigned fastEmit_ISD_SUB_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 11987 switch (VT.SimpleTy) { 11988 case MVT::i8: return fastEmit_ISD_SUB_MVT_i8_rr(RetVT, Op0, Op1); 11989 case MVT::i16: return fastEmit_ISD_SUB_MVT_i16_rr(RetVT, Op0, Op1); 11990 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_rr(RetVT, Op0, Op1); 11991 case MVT::i64: return fastEmit_ISD_SUB_MVT_i64_rr(RetVT, Op0, Op1); 11992 case MVT::v16i8: return fastEmit_ISD_SUB_MVT_v16i8_rr(RetVT, Op0, Op1); 11993 case MVT::v32i8: return fastEmit_ISD_SUB_MVT_v32i8_rr(RetVT, Op0, Op1); 11994 case MVT::v64i8: return fastEmit_ISD_SUB_MVT_v64i8_rr(RetVT, Op0, Op1); 11995 case MVT::v8i16: return fastEmit_ISD_SUB_MVT_v8i16_rr(RetVT, Op0, Op1); 11996 case MVT::v16i16: return fastEmit_ISD_SUB_MVT_v16i16_rr(RetVT, Op0, Op1); 11997 case MVT::v32i16: return fastEmit_ISD_SUB_MVT_v32i16_rr(RetVT, Op0, Op1); 11998 case MVT::v4i32: return fastEmit_ISD_SUB_MVT_v4i32_rr(RetVT, Op0, Op1); 11999 case MVT::v8i32: return fastEmit_ISD_SUB_MVT_v8i32_rr(RetVT, Op0, Op1); 12000 case MVT::v16i32: return fastEmit_ISD_SUB_MVT_v16i32_rr(RetVT, Op0, Op1); 12001 case MVT::v2i64: return fastEmit_ISD_SUB_MVT_v2i64_rr(RetVT, Op0, Op1); 12002 case MVT::v4i64: return fastEmit_ISD_SUB_MVT_v4i64_rr(RetVT, Op0, Op1); 12003 case MVT::v8i64: return fastEmit_ISD_SUB_MVT_v8i64_rr(RetVT, Op0, Op1); 12004 default: return 0; 12005 } 12006} 12007 12008// FastEmit functions for ISD::UADDSAT. 12009 12010unsigned fastEmit_ISD_UADDSAT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12011 if (RetVT.SimpleTy != MVT::v16i8) 12012 return 0; 12013 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 12014 return fastEmitInst_rr(X86::VPADDUSBZ128rr, &X86::VR128XRegClass, Op0, Op1); 12015 } 12016 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 12017 return fastEmitInst_rr(X86::PADDUSBrr, &X86::VR128RegClass, Op0, Op1); 12018 } 12019 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 12020 return fastEmitInst_rr(X86::VPADDUSBrr, &X86::VR128RegClass, Op0, Op1); 12021 } 12022 return 0; 12023} 12024 12025unsigned fastEmit_ISD_UADDSAT_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12026 if (RetVT.SimpleTy != MVT::v32i8) 12027 return 0; 12028 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 12029 return fastEmitInst_rr(X86::VPADDUSBZ256rr, &X86::VR256XRegClass, Op0, Op1); 12030 } 12031 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 12032 return fastEmitInst_rr(X86::VPADDUSBYrr, &X86::VR256RegClass, Op0, Op1); 12033 } 12034 return 0; 12035} 12036 12037unsigned fastEmit_ISD_UADDSAT_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12038 if (RetVT.SimpleTy != MVT::v64i8) 12039 return 0; 12040 if ((Subtarget->hasBWI())) { 12041 return fastEmitInst_rr(X86::VPADDUSBZrr, &X86::VR512RegClass, Op0, Op1); 12042 } 12043 return 0; 12044} 12045 12046unsigned fastEmit_ISD_UADDSAT_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12047 if (RetVT.SimpleTy != MVT::v8i16) 12048 return 0; 12049 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 12050 return fastEmitInst_rr(X86::VPADDUSWZ128rr, &X86::VR128XRegClass, Op0, Op1); 12051 } 12052 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 12053 return fastEmitInst_rr(X86::PADDUSWrr, &X86::VR128RegClass, Op0, Op1); 12054 } 12055 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 12056 return fastEmitInst_rr(X86::VPADDUSWrr, &X86::VR128RegClass, Op0, Op1); 12057 } 12058 return 0; 12059} 12060 12061unsigned fastEmit_ISD_UADDSAT_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12062 if (RetVT.SimpleTy != MVT::v16i16) 12063 return 0; 12064 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 12065 return fastEmitInst_rr(X86::VPADDUSWZ256rr, &X86::VR256XRegClass, Op0, Op1); 12066 } 12067 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 12068 return fastEmitInst_rr(X86::VPADDUSWYrr, &X86::VR256RegClass, Op0, Op1); 12069 } 12070 return 0; 12071} 12072 12073unsigned fastEmit_ISD_UADDSAT_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12074 if (RetVT.SimpleTy != MVT::v32i16) 12075 return 0; 12076 if ((Subtarget->hasBWI())) { 12077 return fastEmitInst_rr(X86::VPADDUSWZrr, &X86::VR512RegClass, Op0, Op1); 12078 } 12079 return 0; 12080} 12081 12082unsigned fastEmit_ISD_UADDSAT_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 12083 switch (VT.SimpleTy) { 12084 case MVT::v16i8: return fastEmit_ISD_UADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1); 12085 case MVT::v32i8: return fastEmit_ISD_UADDSAT_MVT_v32i8_rr(RetVT, Op0, Op1); 12086 case MVT::v64i8: return fastEmit_ISD_UADDSAT_MVT_v64i8_rr(RetVT, Op0, Op1); 12087 case MVT::v8i16: return fastEmit_ISD_UADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1); 12088 case MVT::v16i16: return fastEmit_ISD_UADDSAT_MVT_v16i16_rr(RetVT, Op0, Op1); 12089 case MVT::v32i16: return fastEmit_ISD_UADDSAT_MVT_v32i16_rr(RetVT, Op0, Op1); 12090 default: return 0; 12091 } 12092} 12093 12094// FastEmit functions for ISD::UMAX. 12095 12096unsigned fastEmit_ISD_UMAX_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12097 if (RetVT.SimpleTy != MVT::v16i8) 12098 return 0; 12099 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 12100 return fastEmitInst_rr(X86::VPMAXUBZ128rr, &X86::VR128XRegClass, Op0, Op1); 12101 } 12102 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 12103 return fastEmitInst_rr(X86::PMAXUBrr, &X86::VR128RegClass, Op0, Op1); 12104 } 12105 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 12106 return fastEmitInst_rr(X86::VPMAXUBrr, &X86::VR128RegClass, Op0, Op1); 12107 } 12108 return 0; 12109} 12110 12111unsigned fastEmit_ISD_UMAX_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12112 if (RetVT.SimpleTy != MVT::v32i8) 12113 return 0; 12114 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 12115 return fastEmitInst_rr(X86::VPMAXUBZ256rr, &X86::VR256XRegClass, Op0, Op1); 12116 } 12117 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 12118 return fastEmitInst_rr(X86::VPMAXUBYrr, &X86::VR256RegClass, Op0, Op1); 12119 } 12120 return 0; 12121} 12122 12123unsigned fastEmit_ISD_UMAX_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12124 if (RetVT.SimpleTy != MVT::v64i8) 12125 return 0; 12126 if ((Subtarget->hasBWI())) { 12127 return fastEmitInst_rr(X86::VPMAXUBZrr, &X86::VR512RegClass, Op0, Op1); 12128 } 12129 return 0; 12130} 12131 12132unsigned fastEmit_ISD_UMAX_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12133 if (RetVT.SimpleTy != MVT::v8i16) 12134 return 0; 12135 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 12136 return fastEmitInst_rr(X86::VPMAXUWZ128rr, &X86::VR128XRegClass, Op0, Op1); 12137 } 12138 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 12139 return fastEmitInst_rr(X86::PMAXUWrr, &X86::VR128RegClass, Op0, Op1); 12140 } 12141 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 12142 return fastEmitInst_rr(X86::VPMAXUWrr, &X86::VR128RegClass, Op0, Op1); 12143 } 12144 return 0; 12145} 12146 12147unsigned fastEmit_ISD_UMAX_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12148 if (RetVT.SimpleTy != MVT::v16i16) 12149 return 0; 12150 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 12151 return fastEmitInst_rr(X86::VPMAXUWZ256rr, &X86::VR256XRegClass, Op0, Op1); 12152 } 12153 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 12154 return fastEmitInst_rr(X86::VPMAXUWYrr, &X86::VR256RegClass, Op0, Op1); 12155 } 12156 return 0; 12157} 12158 12159unsigned fastEmit_ISD_UMAX_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12160 if (RetVT.SimpleTy != MVT::v32i16) 12161 return 0; 12162 if ((Subtarget->hasBWI())) { 12163 return fastEmitInst_rr(X86::VPMAXUWZrr, &X86::VR512RegClass, Op0, Op1); 12164 } 12165 return 0; 12166} 12167 12168unsigned fastEmit_ISD_UMAX_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12169 if (RetVT.SimpleTy != MVT::v4i32) 12170 return 0; 12171 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 12172 return fastEmitInst_rr(X86::VPMAXUDZ128rr, &X86::VR128XRegClass, Op0, Op1); 12173 } 12174 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 12175 return fastEmitInst_rr(X86::PMAXUDrr, &X86::VR128RegClass, Op0, Op1); 12176 } 12177 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 12178 return fastEmitInst_rr(X86::VPMAXUDrr, &X86::VR128RegClass, Op0, Op1); 12179 } 12180 return 0; 12181} 12182 12183unsigned fastEmit_ISD_UMAX_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12184 if (RetVT.SimpleTy != MVT::v8i32) 12185 return 0; 12186 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 12187 return fastEmitInst_rr(X86::VPMAXUDZ256rr, &X86::VR256XRegClass, Op0, Op1); 12188 } 12189 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 12190 return fastEmitInst_rr(X86::VPMAXUDYrr, &X86::VR256RegClass, Op0, Op1); 12191 } 12192 return 0; 12193} 12194 12195unsigned fastEmit_ISD_UMAX_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12196 if (RetVT.SimpleTy != MVT::v16i32) 12197 return 0; 12198 if ((Subtarget->hasAVX512())) { 12199 return fastEmitInst_rr(X86::VPMAXUDZrr, &X86::VR512RegClass, Op0, Op1); 12200 } 12201 return 0; 12202} 12203 12204unsigned fastEmit_ISD_UMAX_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12205 if (RetVT.SimpleTy != MVT::v2i64) 12206 return 0; 12207 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 12208 return fastEmitInst_rr(X86::VPMAXUQZ128rr, &X86::VR128XRegClass, Op0, Op1); 12209 } 12210 return 0; 12211} 12212 12213unsigned fastEmit_ISD_UMAX_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12214 if (RetVT.SimpleTy != MVT::v4i64) 12215 return 0; 12216 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 12217 return fastEmitInst_rr(X86::VPMAXUQZ256rr, &X86::VR256XRegClass, Op0, Op1); 12218 } 12219 return 0; 12220} 12221 12222unsigned fastEmit_ISD_UMAX_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12223 if (RetVT.SimpleTy != MVT::v8i64) 12224 return 0; 12225 if ((Subtarget->hasAVX512())) { 12226 return fastEmitInst_rr(X86::VPMAXUQZrr, &X86::VR512RegClass, Op0, Op1); 12227 } 12228 return 0; 12229} 12230 12231unsigned fastEmit_ISD_UMAX_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 12232 switch (VT.SimpleTy) { 12233 case MVT::v16i8: return fastEmit_ISD_UMAX_MVT_v16i8_rr(RetVT, Op0, Op1); 12234 case MVT::v32i8: return fastEmit_ISD_UMAX_MVT_v32i8_rr(RetVT, Op0, Op1); 12235 case MVT::v64i8: return fastEmit_ISD_UMAX_MVT_v64i8_rr(RetVT, Op0, Op1); 12236 case MVT::v8i16: return fastEmit_ISD_UMAX_MVT_v8i16_rr(RetVT, Op0, Op1); 12237 case MVT::v16i16: return fastEmit_ISD_UMAX_MVT_v16i16_rr(RetVT, Op0, Op1); 12238 case MVT::v32i16: return fastEmit_ISD_UMAX_MVT_v32i16_rr(RetVT, Op0, Op1); 12239 case MVT::v4i32: return fastEmit_ISD_UMAX_MVT_v4i32_rr(RetVT, Op0, Op1); 12240 case MVT::v8i32: return fastEmit_ISD_UMAX_MVT_v8i32_rr(RetVT, Op0, Op1); 12241 case MVT::v16i32: return fastEmit_ISD_UMAX_MVT_v16i32_rr(RetVT, Op0, Op1); 12242 case MVT::v2i64: return fastEmit_ISD_UMAX_MVT_v2i64_rr(RetVT, Op0, Op1); 12243 case MVT::v4i64: return fastEmit_ISD_UMAX_MVT_v4i64_rr(RetVT, Op0, Op1); 12244 case MVT::v8i64: return fastEmit_ISD_UMAX_MVT_v8i64_rr(RetVT, Op0, Op1); 12245 default: return 0; 12246 } 12247} 12248 12249// FastEmit functions for ISD::UMIN. 12250 12251unsigned fastEmit_ISD_UMIN_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12252 if (RetVT.SimpleTy != MVT::v16i8) 12253 return 0; 12254 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 12255 return fastEmitInst_rr(X86::VPMINUBZ128rr, &X86::VR128XRegClass, Op0, Op1); 12256 } 12257 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 12258 return fastEmitInst_rr(X86::PMINUBrr, &X86::VR128RegClass, Op0, Op1); 12259 } 12260 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 12261 return fastEmitInst_rr(X86::VPMINUBrr, &X86::VR128RegClass, Op0, Op1); 12262 } 12263 return 0; 12264} 12265 12266unsigned fastEmit_ISD_UMIN_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12267 if (RetVT.SimpleTy != MVT::v32i8) 12268 return 0; 12269 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 12270 return fastEmitInst_rr(X86::VPMINUBZ256rr, &X86::VR256XRegClass, Op0, Op1); 12271 } 12272 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 12273 return fastEmitInst_rr(X86::VPMINUBYrr, &X86::VR256RegClass, Op0, Op1); 12274 } 12275 return 0; 12276} 12277 12278unsigned fastEmit_ISD_UMIN_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12279 if (RetVT.SimpleTy != MVT::v64i8) 12280 return 0; 12281 if ((Subtarget->hasBWI())) { 12282 return fastEmitInst_rr(X86::VPMINUBZrr, &X86::VR512RegClass, Op0, Op1); 12283 } 12284 return 0; 12285} 12286 12287unsigned fastEmit_ISD_UMIN_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12288 if (RetVT.SimpleTy != MVT::v8i16) 12289 return 0; 12290 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 12291 return fastEmitInst_rr(X86::VPMINUWZ128rr, &X86::VR128XRegClass, Op0, Op1); 12292 } 12293 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 12294 return fastEmitInst_rr(X86::PMINUWrr, &X86::VR128RegClass, Op0, Op1); 12295 } 12296 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 12297 return fastEmitInst_rr(X86::VPMINUWrr, &X86::VR128RegClass, Op0, Op1); 12298 } 12299 return 0; 12300} 12301 12302unsigned fastEmit_ISD_UMIN_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12303 if (RetVT.SimpleTy != MVT::v16i16) 12304 return 0; 12305 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 12306 return fastEmitInst_rr(X86::VPMINUWZ256rr, &X86::VR256XRegClass, Op0, Op1); 12307 } 12308 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 12309 return fastEmitInst_rr(X86::VPMINUWYrr, &X86::VR256RegClass, Op0, Op1); 12310 } 12311 return 0; 12312} 12313 12314unsigned fastEmit_ISD_UMIN_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12315 if (RetVT.SimpleTy != MVT::v32i16) 12316 return 0; 12317 if ((Subtarget->hasBWI())) { 12318 return fastEmitInst_rr(X86::VPMINUWZrr, &X86::VR512RegClass, Op0, Op1); 12319 } 12320 return 0; 12321} 12322 12323unsigned fastEmit_ISD_UMIN_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12324 if (RetVT.SimpleTy != MVT::v4i32) 12325 return 0; 12326 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 12327 return fastEmitInst_rr(X86::VPMINUDZ128rr, &X86::VR128XRegClass, Op0, Op1); 12328 } 12329 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 12330 return fastEmitInst_rr(X86::PMINUDrr, &X86::VR128RegClass, Op0, Op1); 12331 } 12332 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 12333 return fastEmitInst_rr(X86::VPMINUDrr, &X86::VR128RegClass, Op0, Op1); 12334 } 12335 return 0; 12336} 12337 12338unsigned fastEmit_ISD_UMIN_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12339 if (RetVT.SimpleTy != MVT::v8i32) 12340 return 0; 12341 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 12342 return fastEmitInst_rr(X86::VPMINUDZ256rr, &X86::VR256XRegClass, Op0, Op1); 12343 } 12344 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 12345 return fastEmitInst_rr(X86::VPMINUDYrr, &X86::VR256RegClass, Op0, Op1); 12346 } 12347 return 0; 12348} 12349 12350unsigned fastEmit_ISD_UMIN_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12351 if (RetVT.SimpleTy != MVT::v16i32) 12352 return 0; 12353 if ((Subtarget->hasAVX512())) { 12354 return fastEmitInst_rr(X86::VPMINUDZrr, &X86::VR512RegClass, Op0, Op1); 12355 } 12356 return 0; 12357} 12358 12359unsigned fastEmit_ISD_UMIN_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12360 if (RetVT.SimpleTy != MVT::v2i64) 12361 return 0; 12362 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 12363 return fastEmitInst_rr(X86::VPMINUQZ128rr, &X86::VR128XRegClass, Op0, Op1); 12364 } 12365 return 0; 12366} 12367 12368unsigned fastEmit_ISD_UMIN_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12369 if (RetVT.SimpleTy != MVT::v4i64) 12370 return 0; 12371 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 12372 return fastEmitInst_rr(X86::VPMINUQZ256rr, &X86::VR256XRegClass, Op0, Op1); 12373 } 12374 return 0; 12375} 12376 12377unsigned fastEmit_ISD_UMIN_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12378 if (RetVT.SimpleTy != MVT::v8i64) 12379 return 0; 12380 if ((Subtarget->hasAVX512())) { 12381 return fastEmitInst_rr(X86::VPMINUQZrr, &X86::VR512RegClass, Op0, Op1); 12382 } 12383 return 0; 12384} 12385 12386unsigned fastEmit_ISD_UMIN_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 12387 switch (VT.SimpleTy) { 12388 case MVT::v16i8: return fastEmit_ISD_UMIN_MVT_v16i8_rr(RetVT, Op0, Op1); 12389 case MVT::v32i8: return fastEmit_ISD_UMIN_MVT_v32i8_rr(RetVT, Op0, Op1); 12390 case MVT::v64i8: return fastEmit_ISD_UMIN_MVT_v64i8_rr(RetVT, Op0, Op1); 12391 case MVT::v8i16: return fastEmit_ISD_UMIN_MVT_v8i16_rr(RetVT, Op0, Op1); 12392 case MVT::v16i16: return fastEmit_ISD_UMIN_MVT_v16i16_rr(RetVT, Op0, Op1); 12393 case MVT::v32i16: return fastEmit_ISD_UMIN_MVT_v32i16_rr(RetVT, Op0, Op1); 12394 case MVT::v4i32: return fastEmit_ISD_UMIN_MVT_v4i32_rr(RetVT, Op0, Op1); 12395 case MVT::v8i32: return fastEmit_ISD_UMIN_MVT_v8i32_rr(RetVT, Op0, Op1); 12396 case MVT::v16i32: return fastEmit_ISD_UMIN_MVT_v16i32_rr(RetVT, Op0, Op1); 12397 case MVT::v2i64: return fastEmit_ISD_UMIN_MVT_v2i64_rr(RetVT, Op0, Op1); 12398 case MVT::v4i64: return fastEmit_ISD_UMIN_MVT_v4i64_rr(RetVT, Op0, Op1); 12399 case MVT::v8i64: return fastEmit_ISD_UMIN_MVT_v8i64_rr(RetVT, Op0, Op1); 12400 default: return 0; 12401 } 12402} 12403 12404// FastEmit functions for ISD::USUBSAT. 12405 12406unsigned fastEmit_ISD_USUBSAT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12407 if (RetVT.SimpleTy != MVT::v16i8) 12408 return 0; 12409 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 12410 return fastEmitInst_rr(X86::VPSUBUSBZ128rr, &X86::VR128XRegClass, Op0, Op1); 12411 } 12412 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 12413 return fastEmitInst_rr(X86::PSUBUSBrr, &X86::VR128RegClass, Op0, Op1); 12414 } 12415 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 12416 return fastEmitInst_rr(X86::VPSUBUSBrr, &X86::VR128RegClass, Op0, Op1); 12417 } 12418 return 0; 12419} 12420 12421unsigned fastEmit_ISD_USUBSAT_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12422 if (RetVT.SimpleTy != MVT::v32i8) 12423 return 0; 12424 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 12425 return fastEmitInst_rr(X86::VPSUBUSBZ256rr, &X86::VR256XRegClass, Op0, Op1); 12426 } 12427 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 12428 return fastEmitInst_rr(X86::VPSUBUSBYrr, &X86::VR256RegClass, Op0, Op1); 12429 } 12430 return 0; 12431} 12432 12433unsigned fastEmit_ISD_USUBSAT_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12434 if (RetVT.SimpleTy != MVT::v64i8) 12435 return 0; 12436 if ((Subtarget->hasBWI())) { 12437 return fastEmitInst_rr(X86::VPSUBUSBZrr, &X86::VR512RegClass, Op0, Op1); 12438 } 12439 return 0; 12440} 12441 12442unsigned fastEmit_ISD_USUBSAT_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12443 if (RetVT.SimpleTy != MVT::v8i16) 12444 return 0; 12445 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 12446 return fastEmitInst_rr(X86::VPSUBUSWZ128rr, &X86::VR128XRegClass, Op0, Op1); 12447 } 12448 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 12449 return fastEmitInst_rr(X86::PSUBUSWrr, &X86::VR128RegClass, Op0, Op1); 12450 } 12451 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 12452 return fastEmitInst_rr(X86::VPSUBUSWrr, &X86::VR128RegClass, Op0, Op1); 12453 } 12454 return 0; 12455} 12456 12457unsigned fastEmit_ISD_USUBSAT_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12458 if (RetVT.SimpleTy != MVT::v16i16) 12459 return 0; 12460 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 12461 return fastEmitInst_rr(X86::VPSUBUSWZ256rr, &X86::VR256XRegClass, Op0, Op1); 12462 } 12463 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 12464 return fastEmitInst_rr(X86::VPSUBUSWYrr, &X86::VR256RegClass, Op0, Op1); 12465 } 12466 return 0; 12467} 12468 12469unsigned fastEmit_ISD_USUBSAT_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12470 if (RetVT.SimpleTy != MVT::v32i16) 12471 return 0; 12472 if ((Subtarget->hasBWI())) { 12473 return fastEmitInst_rr(X86::VPSUBUSWZrr, &X86::VR512RegClass, Op0, Op1); 12474 } 12475 return 0; 12476} 12477 12478unsigned fastEmit_ISD_USUBSAT_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 12479 switch (VT.SimpleTy) { 12480 case MVT::v16i8: return fastEmit_ISD_USUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1); 12481 case MVT::v32i8: return fastEmit_ISD_USUBSAT_MVT_v32i8_rr(RetVT, Op0, Op1); 12482 case MVT::v64i8: return fastEmit_ISD_USUBSAT_MVT_v64i8_rr(RetVT, Op0, Op1); 12483 case MVT::v8i16: return fastEmit_ISD_USUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1); 12484 case MVT::v16i16: return fastEmit_ISD_USUBSAT_MVT_v16i16_rr(RetVT, Op0, Op1); 12485 case MVT::v32i16: return fastEmit_ISD_USUBSAT_MVT_v32i16_rr(RetVT, Op0, Op1); 12486 default: return 0; 12487 } 12488} 12489 12490// FastEmit functions for ISD::XOR. 12491 12492unsigned fastEmit_ISD_XOR_MVT_i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12493 if (RetVT.SimpleTy != MVT::i8) 12494 return 0; 12495 return fastEmitInst_rr(X86::XOR8rr, &X86::GR8RegClass, Op0, Op1); 12496} 12497 12498unsigned fastEmit_ISD_XOR_MVT_i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12499 if (RetVT.SimpleTy != MVT::i16) 12500 return 0; 12501 return fastEmitInst_rr(X86::XOR16rr, &X86::GR16RegClass, Op0, Op1); 12502} 12503 12504unsigned fastEmit_ISD_XOR_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12505 if (RetVT.SimpleTy != MVT::i32) 12506 return 0; 12507 return fastEmitInst_rr(X86::XOR32rr, &X86::GR32RegClass, Op0, Op1); 12508} 12509 12510unsigned fastEmit_ISD_XOR_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12511 if (RetVT.SimpleTy != MVT::i64) 12512 return 0; 12513 return fastEmitInst_rr(X86::XOR64rr, &X86::GR64RegClass, Op0, Op1); 12514} 12515 12516unsigned fastEmit_ISD_XOR_MVT_v8i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12517 if (RetVT.SimpleTy != MVT::v8i1) 12518 return 0; 12519 if ((Subtarget->hasDQI())) { 12520 return fastEmitInst_rr(X86::KXORBrr, &X86::VK8RegClass, Op0, Op1); 12521 } 12522 return 0; 12523} 12524 12525unsigned fastEmit_ISD_XOR_MVT_v16i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12526 if (RetVT.SimpleTy != MVT::v16i1) 12527 return 0; 12528 if ((Subtarget->hasAVX512())) { 12529 return fastEmitInst_rr(X86::KXORWrr, &X86::VK16RegClass, Op0, Op1); 12530 } 12531 return 0; 12532} 12533 12534unsigned fastEmit_ISD_XOR_MVT_v32i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12535 if (RetVT.SimpleTy != MVT::v32i1) 12536 return 0; 12537 if ((Subtarget->hasBWI())) { 12538 return fastEmitInst_rr(X86::KXORDrr, &X86::VK32RegClass, Op0, Op1); 12539 } 12540 return 0; 12541} 12542 12543unsigned fastEmit_ISD_XOR_MVT_v64i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12544 if (RetVT.SimpleTy != MVT::v64i1) 12545 return 0; 12546 if ((Subtarget->hasBWI())) { 12547 return fastEmitInst_rr(X86::KXORQrr, &X86::VK64RegClass, Op0, Op1); 12548 } 12549 return 0; 12550} 12551 12552unsigned fastEmit_ISD_XOR_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12553 if (RetVT.SimpleTy != MVT::v16i8) 12554 return 0; 12555 if ((Subtarget->hasVLX())) { 12556 return fastEmitInst_rr(X86::VPXORQZ128rr, &X86::VR128XRegClass, Op0, Op1); 12557 } 12558 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 12559 return fastEmitInst_rr(X86::PXORrr, &X86::VR128RegClass, Op0, Op1); 12560 } 12561 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 12562 return fastEmitInst_rr(X86::VPXORrr, &X86::VR128RegClass, Op0, Op1); 12563 } 12564 return 0; 12565} 12566 12567unsigned fastEmit_ISD_XOR_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12568 if (RetVT.SimpleTy != MVT::v32i8) 12569 return 0; 12570 if ((Subtarget->hasVLX())) { 12571 return fastEmitInst_rr(X86::VPXORQZ256rr, &X86::VR256XRegClass, Op0, Op1); 12572 } 12573 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) { 12574 return fastEmitInst_rr(X86::VXORPSYrr, &X86::VR256RegClass, Op0, Op1); 12575 } 12576 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 12577 return fastEmitInst_rr(X86::VPXORYrr, &X86::VR256RegClass, Op0, Op1); 12578 } 12579 return 0; 12580} 12581 12582unsigned fastEmit_ISD_XOR_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12583 if (RetVT.SimpleTy != MVT::v64i8) 12584 return 0; 12585 if ((Subtarget->hasAVX512())) { 12586 return fastEmitInst_rr(X86::VPXORQZrr, &X86::VR512RegClass, Op0, Op1); 12587 } 12588 return 0; 12589} 12590 12591unsigned fastEmit_ISD_XOR_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12592 if (RetVT.SimpleTy != MVT::v8i16) 12593 return 0; 12594 if ((Subtarget->hasVLX())) { 12595 return fastEmitInst_rr(X86::VPXORQZ128rr, &X86::VR128XRegClass, Op0, Op1); 12596 } 12597 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 12598 return fastEmitInst_rr(X86::PXORrr, &X86::VR128RegClass, Op0, Op1); 12599 } 12600 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 12601 return fastEmitInst_rr(X86::VPXORrr, &X86::VR128RegClass, Op0, Op1); 12602 } 12603 return 0; 12604} 12605 12606unsigned fastEmit_ISD_XOR_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12607 if (RetVT.SimpleTy != MVT::v16i16) 12608 return 0; 12609 if ((Subtarget->hasVLX())) { 12610 return fastEmitInst_rr(X86::VPXORQZ256rr, &X86::VR256XRegClass, Op0, Op1); 12611 } 12612 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) { 12613 return fastEmitInst_rr(X86::VXORPSYrr, &X86::VR256RegClass, Op0, Op1); 12614 } 12615 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 12616 return fastEmitInst_rr(X86::VPXORYrr, &X86::VR256RegClass, Op0, Op1); 12617 } 12618 return 0; 12619} 12620 12621unsigned fastEmit_ISD_XOR_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12622 if (RetVT.SimpleTy != MVT::v32i16) 12623 return 0; 12624 if ((Subtarget->hasAVX512())) { 12625 return fastEmitInst_rr(X86::VPXORQZrr, &X86::VR512RegClass, Op0, Op1); 12626 } 12627 return 0; 12628} 12629 12630unsigned fastEmit_ISD_XOR_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12631 if (RetVT.SimpleTy != MVT::v4i32) 12632 return 0; 12633 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 12634 return fastEmitInst_rr(X86::PXORrr, &X86::VR128RegClass, Op0, Op1); 12635 } 12636 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 12637 return fastEmitInst_rr(X86::VPXORrr, &X86::VR128RegClass, Op0, Op1); 12638 } 12639 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 12640 return fastEmitInst_rr(X86::VPXORDZ128rr, &X86::VR128XRegClass, Op0, Op1); 12641 } 12642 return 0; 12643} 12644 12645unsigned fastEmit_ISD_XOR_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12646 if (RetVT.SimpleTy != MVT::v8i32) 12647 return 0; 12648 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) { 12649 return fastEmitInst_rr(X86::VXORPSYrr, &X86::VR256RegClass, Op0, Op1); 12650 } 12651 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 12652 return fastEmitInst_rr(X86::VPXORYrr, &X86::VR256RegClass, Op0, Op1); 12653 } 12654 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 12655 return fastEmitInst_rr(X86::VPXORDZ256rr, &X86::VR256XRegClass, Op0, Op1); 12656 } 12657 return 0; 12658} 12659 12660unsigned fastEmit_ISD_XOR_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12661 if (RetVT.SimpleTy != MVT::v16i32) 12662 return 0; 12663 if ((Subtarget->hasAVX512())) { 12664 return fastEmitInst_rr(X86::VPXORDZrr, &X86::VR512RegClass, Op0, Op1); 12665 } 12666 return 0; 12667} 12668 12669unsigned fastEmit_ISD_XOR_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12670 if (RetVT.SimpleTy != MVT::v2i64) 12671 return 0; 12672 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 12673 return fastEmitInst_rr(X86::VPXORQZ128rr, &X86::VR128XRegClass, Op0, Op1); 12674 } 12675 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 12676 return fastEmitInst_rr(X86::PXORrr, &X86::VR128RegClass, Op0, Op1); 12677 } 12678 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 12679 return fastEmitInst_rr(X86::VPXORrr, &X86::VR128RegClass, Op0, Op1); 12680 } 12681 return 0; 12682} 12683 12684unsigned fastEmit_ISD_XOR_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12685 if (RetVT.SimpleTy != MVT::v4i64) 12686 return 0; 12687 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) { 12688 return fastEmitInst_rr(X86::VXORPSYrr, &X86::VR256RegClass, Op0, Op1); 12689 } 12690 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 12691 return fastEmitInst_rr(X86::VPXORQZ256rr, &X86::VR256XRegClass, Op0, Op1); 12692 } 12693 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 12694 return fastEmitInst_rr(X86::VPXORYrr, &X86::VR256RegClass, Op0, Op1); 12695 } 12696 return 0; 12697} 12698 12699unsigned fastEmit_ISD_XOR_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12700 if (RetVT.SimpleTy != MVT::v8i64) 12701 return 0; 12702 if ((Subtarget->hasAVX512())) { 12703 return fastEmitInst_rr(X86::VPXORQZrr, &X86::VR512RegClass, Op0, Op1); 12704 } 12705 return 0; 12706} 12707 12708unsigned fastEmit_ISD_XOR_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 12709 switch (VT.SimpleTy) { 12710 case MVT::i8: return fastEmit_ISD_XOR_MVT_i8_rr(RetVT, Op0, Op1); 12711 case MVT::i16: return fastEmit_ISD_XOR_MVT_i16_rr(RetVT, Op0, Op1); 12712 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_rr(RetVT, Op0, Op1); 12713 case MVT::i64: return fastEmit_ISD_XOR_MVT_i64_rr(RetVT, Op0, Op1); 12714 case MVT::v8i1: return fastEmit_ISD_XOR_MVT_v8i1_rr(RetVT, Op0, Op1); 12715 case MVT::v16i1: return fastEmit_ISD_XOR_MVT_v16i1_rr(RetVT, Op0, Op1); 12716 case MVT::v32i1: return fastEmit_ISD_XOR_MVT_v32i1_rr(RetVT, Op0, Op1); 12717 case MVT::v64i1: return fastEmit_ISD_XOR_MVT_v64i1_rr(RetVT, Op0, Op1); 12718 case MVT::v16i8: return fastEmit_ISD_XOR_MVT_v16i8_rr(RetVT, Op0, Op1); 12719 case MVT::v32i8: return fastEmit_ISD_XOR_MVT_v32i8_rr(RetVT, Op0, Op1); 12720 case MVT::v64i8: return fastEmit_ISD_XOR_MVT_v64i8_rr(RetVT, Op0, Op1); 12721 case MVT::v8i16: return fastEmit_ISD_XOR_MVT_v8i16_rr(RetVT, Op0, Op1); 12722 case MVT::v16i16: return fastEmit_ISD_XOR_MVT_v16i16_rr(RetVT, Op0, Op1); 12723 case MVT::v32i16: return fastEmit_ISD_XOR_MVT_v32i16_rr(RetVT, Op0, Op1); 12724 case MVT::v4i32: return fastEmit_ISD_XOR_MVT_v4i32_rr(RetVT, Op0, Op1); 12725 case MVT::v8i32: return fastEmit_ISD_XOR_MVT_v8i32_rr(RetVT, Op0, Op1); 12726 case MVT::v16i32: return fastEmit_ISD_XOR_MVT_v16i32_rr(RetVT, Op0, Op1); 12727 case MVT::v2i64: return fastEmit_ISD_XOR_MVT_v2i64_rr(RetVT, Op0, Op1); 12728 case MVT::v4i64: return fastEmit_ISD_XOR_MVT_v4i64_rr(RetVT, Op0, Op1); 12729 case MVT::v8i64: return fastEmit_ISD_XOR_MVT_v8i64_rr(RetVT, Op0, Op1); 12730 default: return 0; 12731 } 12732} 12733 12734// FastEmit functions for X86ISD::ADDSUB. 12735 12736unsigned fastEmit_X86ISD_ADDSUB_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12737 if (RetVT.SimpleTy != MVT::v4f32) 12738 return 0; 12739 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) { 12740 return fastEmitInst_rr(X86::ADDSUBPSrr, &X86::VR128RegClass, Op0, Op1); 12741 } 12742 if ((Subtarget->hasAVX())) { 12743 return fastEmitInst_rr(X86::VADDSUBPSrr, &X86::VR128RegClass, Op0, Op1); 12744 } 12745 return 0; 12746} 12747 12748unsigned fastEmit_X86ISD_ADDSUB_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12749 if (RetVT.SimpleTy != MVT::v8f32) 12750 return 0; 12751 if ((Subtarget->hasAVX())) { 12752 return fastEmitInst_rr(X86::VADDSUBPSYrr, &X86::VR256RegClass, Op0, Op1); 12753 } 12754 return 0; 12755} 12756 12757unsigned fastEmit_X86ISD_ADDSUB_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12758 if (RetVT.SimpleTy != MVT::v2f64) 12759 return 0; 12760 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) { 12761 return fastEmitInst_rr(X86::ADDSUBPDrr, &X86::VR128RegClass, Op0, Op1); 12762 } 12763 if ((Subtarget->hasAVX())) { 12764 return fastEmitInst_rr(X86::VADDSUBPDrr, &X86::VR128RegClass, Op0, Op1); 12765 } 12766 return 0; 12767} 12768 12769unsigned fastEmit_X86ISD_ADDSUB_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12770 if (RetVT.SimpleTy != MVT::v4f64) 12771 return 0; 12772 if ((Subtarget->hasAVX())) { 12773 return fastEmitInst_rr(X86::VADDSUBPDYrr, &X86::VR256RegClass, Op0, Op1); 12774 } 12775 return 0; 12776} 12777 12778unsigned fastEmit_X86ISD_ADDSUB_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 12779 switch (VT.SimpleTy) { 12780 case MVT::v4f32: return fastEmit_X86ISD_ADDSUB_MVT_v4f32_rr(RetVT, Op0, Op1); 12781 case MVT::v8f32: return fastEmit_X86ISD_ADDSUB_MVT_v8f32_rr(RetVT, Op0, Op1); 12782 case MVT::v2f64: return fastEmit_X86ISD_ADDSUB_MVT_v2f64_rr(RetVT, Op0, Op1); 12783 case MVT::v4f64: return fastEmit_X86ISD_ADDSUB_MVT_v4f64_rr(RetVT, Op0, Op1); 12784 default: return 0; 12785 } 12786} 12787 12788// FastEmit functions for X86ISD::ANDNP. 12789 12790unsigned fastEmit_X86ISD_ANDNP_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12791 if (RetVT.SimpleTy != MVT::v16i8) 12792 return 0; 12793 if ((Subtarget->hasVLX())) { 12794 return fastEmitInst_rr(X86::VPANDNQZ128rr, &X86::VR128XRegClass, Op0, Op1); 12795 } 12796 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 12797 return fastEmitInst_rr(X86::PANDNrr, &X86::VR128RegClass, Op0, Op1); 12798 } 12799 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 12800 return fastEmitInst_rr(X86::VPANDNrr, &X86::VR128RegClass, Op0, Op1); 12801 } 12802 return 0; 12803} 12804 12805unsigned fastEmit_X86ISD_ANDNP_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12806 if (RetVT.SimpleTy != MVT::v32i8) 12807 return 0; 12808 if ((Subtarget->hasVLX())) { 12809 return fastEmitInst_rr(X86::VPANDNQZ256rr, &X86::VR256XRegClass, Op0, Op1); 12810 } 12811 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) { 12812 return fastEmitInst_rr(X86::VANDNPSYrr, &X86::VR256RegClass, Op0, Op1); 12813 } 12814 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 12815 return fastEmitInst_rr(X86::VPANDNYrr, &X86::VR256RegClass, Op0, Op1); 12816 } 12817 return 0; 12818} 12819 12820unsigned fastEmit_X86ISD_ANDNP_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12821 if (RetVT.SimpleTy != MVT::v64i8) 12822 return 0; 12823 if ((Subtarget->hasAVX512())) { 12824 return fastEmitInst_rr(X86::VPANDNQZrr, &X86::VR512RegClass, Op0, Op1); 12825 } 12826 return 0; 12827} 12828 12829unsigned fastEmit_X86ISD_ANDNP_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12830 if (RetVT.SimpleTy != MVT::v8i16) 12831 return 0; 12832 if ((Subtarget->hasVLX())) { 12833 return fastEmitInst_rr(X86::VPANDNQZ128rr, &X86::VR128XRegClass, Op0, Op1); 12834 } 12835 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 12836 return fastEmitInst_rr(X86::PANDNrr, &X86::VR128RegClass, Op0, Op1); 12837 } 12838 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 12839 return fastEmitInst_rr(X86::VPANDNrr, &X86::VR128RegClass, Op0, Op1); 12840 } 12841 return 0; 12842} 12843 12844unsigned fastEmit_X86ISD_ANDNP_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12845 if (RetVT.SimpleTy != MVT::v16i16) 12846 return 0; 12847 if ((Subtarget->hasVLX())) { 12848 return fastEmitInst_rr(X86::VPANDNQZ256rr, &X86::VR256XRegClass, Op0, Op1); 12849 } 12850 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) { 12851 return fastEmitInst_rr(X86::VANDNPSYrr, &X86::VR256RegClass, Op0, Op1); 12852 } 12853 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 12854 return fastEmitInst_rr(X86::VPANDNYrr, &X86::VR256RegClass, Op0, Op1); 12855 } 12856 return 0; 12857} 12858 12859unsigned fastEmit_X86ISD_ANDNP_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12860 if (RetVT.SimpleTy != MVT::v32i16) 12861 return 0; 12862 if ((Subtarget->hasAVX512())) { 12863 return fastEmitInst_rr(X86::VPANDNQZrr, &X86::VR512RegClass, Op0, Op1); 12864 } 12865 return 0; 12866} 12867 12868unsigned fastEmit_X86ISD_ANDNP_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12869 if (RetVT.SimpleTy != MVT::v4i32) 12870 return 0; 12871 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 12872 return fastEmitInst_rr(X86::PANDNrr, &X86::VR128RegClass, Op0, Op1); 12873 } 12874 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 12875 return fastEmitInst_rr(X86::VPANDNrr, &X86::VR128RegClass, Op0, Op1); 12876 } 12877 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 12878 return fastEmitInst_rr(X86::VPANDNDZ128rr, &X86::VR128XRegClass, Op0, Op1); 12879 } 12880 return 0; 12881} 12882 12883unsigned fastEmit_X86ISD_ANDNP_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12884 if (RetVT.SimpleTy != MVT::v8i32) 12885 return 0; 12886 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) { 12887 return fastEmitInst_rr(X86::VANDNPSYrr, &X86::VR256RegClass, Op0, Op1); 12888 } 12889 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 12890 return fastEmitInst_rr(X86::VPANDNYrr, &X86::VR256RegClass, Op0, Op1); 12891 } 12892 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 12893 return fastEmitInst_rr(X86::VPANDNDZ256rr, &X86::VR256XRegClass, Op0, Op1); 12894 } 12895 return 0; 12896} 12897 12898unsigned fastEmit_X86ISD_ANDNP_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12899 if (RetVT.SimpleTy != MVT::v16i32) 12900 return 0; 12901 if ((Subtarget->hasAVX512())) { 12902 return fastEmitInst_rr(X86::VPANDNDZrr, &X86::VR512RegClass, Op0, Op1); 12903 } 12904 return 0; 12905} 12906 12907unsigned fastEmit_X86ISD_ANDNP_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12908 if (RetVT.SimpleTy != MVT::v2i64) 12909 return 0; 12910 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 12911 return fastEmitInst_rr(X86::VPANDNQZ128rr, &X86::VR128XRegClass, Op0, Op1); 12912 } 12913 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 12914 return fastEmitInst_rr(X86::PANDNrr, &X86::VR128RegClass, Op0, Op1); 12915 } 12916 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 12917 return fastEmitInst_rr(X86::VPANDNrr, &X86::VR128RegClass, Op0, Op1); 12918 } 12919 return 0; 12920} 12921 12922unsigned fastEmit_X86ISD_ANDNP_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12923 if (RetVT.SimpleTy != MVT::v4i64) 12924 return 0; 12925 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) { 12926 return fastEmitInst_rr(X86::VANDNPSYrr, &X86::VR256RegClass, Op0, Op1); 12927 } 12928 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 12929 return fastEmitInst_rr(X86::VPANDNQZ256rr, &X86::VR256XRegClass, Op0, Op1); 12930 } 12931 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 12932 return fastEmitInst_rr(X86::VPANDNYrr, &X86::VR256RegClass, Op0, Op1); 12933 } 12934 return 0; 12935} 12936 12937unsigned fastEmit_X86ISD_ANDNP_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12938 if (RetVT.SimpleTy != MVT::v8i64) 12939 return 0; 12940 if ((Subtarget->hasAVX512())) { 12941 return fastEmitInst_rr(X86::VPANDNQZrr, &X86::VR512RegClass, Op0, Op1); 12942 } 12943 return 0; 12944} 12945 12946unsigned fastEmit_X86ISD_ANDNP_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 12947 switch (VT.SimpleTy) { 12948 case MVT::v16i8: return fastEmit_X86ISD_ANDNP_MVT_v16i8_rr(RetVT, Op0, Op1); 12949 case MVT::v32i8: return fastEmit_X86ISD_ANDNP_MVT_v32i8_rr(RetVT, Op0, Op1); 12950 case MVT::v64i8: return fastEmit_X86ISD_ANDNP_MVT_v64i8_rr(RetVT, Op0, Op1); 12951 case MVT::v8i16: return fastEmit_X86ISD_ANDNP_MVT_v8i16_rr(RetVT, Op0, Op1); 12952 case MVT::v16i16: return fastEmit_X86ISD_ANDNP_MVT_v16i16_rr(RetVT, Op0, Op1); 12953 case MVT::v32i16: return fastEmit_X86ISD_ANDNP_MVT_v32i16_rr(RetVT, Op0, Op1); 12954 case MVT::v4i32: return fastEmit_X86ISD_ANDNP_MVT_v4i32_rr(RetVT, Op0, Op1); 12955 case MVT::v8i32: return fastEmit_X86ISD_ANDNP_MVT_v8i32_rr(RetVT, Op0, Op1); 12956 case MVT::v16i32: return fastEmit_X86ISD_ANDNP_MVT_v16i32_rr(RetVT, Op0, Op1); 12957 case MVT::v2i64: return fastEmit_X86ISD_ANDNP_MVT_v2i64_rr(RetVT, Op0, Op1); 12958 case MVT::v4i64: return fastEmit_X86ISD_ANDNP_MVT_v4i64_rr(RetVT, Op0, Op1); 12959 case MVT::v8i64: return fastEmit_X86ISD_ANDNP_MVT_v8i64_rr(RetVT, Op0, Op1); 12960 default: return 0; 12961 } 12962} 12963 12964// FastEmit functions for X86ISD::BEXTR. 12965 12966unsigned fastEmit_X86ISD_BEXTR_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12967 if (RetVT.SimpleTy != MVT::i32) 12968 return 0; 12969 if ((Subtarget->hasBMI())) { 12970 return fastEmitInst_rr(X86::BEXTR32rr, &X86::GR32RegClass, Op0, Op1); 12971 } 12972 return 0; 12973} 12974 12975unsigned fastEmit_X86ISD_BEXTR_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12976 if (RetVT.SimpleTy != MVT::i64) 12977 return 0; 12978 if ((Subtarget->hasBMI())) { 12979 return fastEmitInst_rr(X86::BEXTR64rr, &X86::GR64RegClass, Op0, Op1); 12980 } 12981 return 0; 12982} 12983 12984unsigned fastEmit_X86ISD_BEXTR_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 12985 switch (VT.SimpleTy) { 12986 case MVT::i32: return fastEmit_X86ISD_BEXTR_MVT_i32_rr(RetVT, Op0, Op1); 12987 case MVT::i64: return fastEmit_X86ISD_BEXTR_MVT_i64_rr(RetVT, Op0, Op1); 12988 default: return 0; 12989 } 12990} 12991 12992// FastEmit functions for X86ISD::BT. 12993 12994unsigned fastEmit_X86ISD_BT_MVT_i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12995 if (RetVT.SimpleTy != MVT::i32) 12996 return 0; 12997 return fastEmitInst_rr(X86::BT16rr, &X86::GR16RegClass, Op0, Op1); 12998} 12999 13000unsigned fastEmit_X86ISD_BT_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13001 if (RetVT.SimpleTy != MVT::i32) 13002 return 0; 13003 return fastEmitInst_rr(X86::BT32rr, &X86::GR32RegClass, Op0, Op1); 13004} 13005 13006unsigned fastEmit_X86ISD_BT_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13007 if (RetVT.SimpleTy != MVT::i32) 13008 return 0; 13009 return fastEmitInst_rr(X86::BT64rr, &X86::GR64RegClass, Op0, Op1); 13010} 13011 13012unsigned fastEmit_X86ISD_BT_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 13013 switch (VT.SimpleTy) { 13014 case MVT::i16: return fastEmit_X86ISD_BT_MVT_i16_rr(RetVT, Op0, Op1); 13015 case MVT::i32: return fastEmit_X86ISD_BT_MVT_i32_rr(RetVT, Op0, Op1); 13016 case MVT::i64: return fastEmit_X86ISD_BT_MVT_i64_rr(RetVT, Op0, Op1); 13017 default: return 0; 13018 } 13019} 13020 13021// FastEmit functions for X86ISD::BZHI. 13022 13023unsigned fastEmit_X86ISD_BZHI_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13024 if (RetVT.SimpleTy != MVT::i32) 13025 return 0; 13026 if ((Subtarget->hasBMI2())) { 13027 return fastEmitInst_rr(X86::BZHI32rr, &X86::GR32RegClass, Op0, Op1); 13028 } 13029 return 0; 13030} 13031 13032unsigned fastEmit_X86ISD_BZHI_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13033 if (RetVT.SimpleTy != MVT::i64) 13034 return 0; 13035 if ((Subtarget->hasBMI2())) { 13036 return fastEmitInst_rr(X86::BZHI64rr, &X86::GR64RegClass, Op0, Op1); 13037 } 13038 return 0; 13039} 13040 13041unsigned fastEmit_X86ISD_BZHI_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 13042 switch (VT.SimpleTy) { 13043 case MVT::i32: return fastEmit_X86ISD_BZHI_MVT_i32_rr(RetVT, Op0, Op1); 13044 case MVT::i64: return fastEmit_X86ISD_BZHI_MVT_i64_rr(RetVT, Op0, Op1); 13045 default: return 0; 13046 } 13047} 13048 13049// FastEmit functions for X86ISD::CMP. 13050 13051unsigned fastEmit_X86ISD_CMP_MVT_i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13052 if (RetVT.SimpleTy != MVT::i32) 13053 return 0; 13054 return fastEmitInst_rr(X86::CMP8rr, &X86::GR8RegClass, Op0, Op1); 13055} 13056 13057unsigned fastEmit_X86ISD_CMP_MVT_i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13058 if (RetVT.SimpleTy != MVT::i32) 13059 return 0; 13060 return fastEmitInst_rr(X86::CMP16rr, &X86::GR16RegClass, Op0, Op1); 13061} 13062 13063unsigned fastEmit_X86ISD_CMP_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13064 if (RetVT.SimpleTy != MVT::i32) 13065 return 0; 13066 return fastEmitInst_rr(X86::CMP32rr, &X86::GR32RegClass, Op0, Op1); 13067} 13068 13069unsigned fastEmit_X86ISD_CMP_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13070 if (RetVT.SimpleTy != MVT::i32) 13071 return 0; 13072 return fastEmitInst_rr(X86::CMP64rr, &X86::GR64RegClass, Op0, Op1); 13073} 13074 13075unsigned fastEmit_X86ISD_CMP_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 13076 switch (VT.SimpleTy) { 13077 case MVT::i8: return fastEmit_X86ISD_CMP_MVT_i8_rr(RetVT, Op0, Op1); 13078 case MVT::i16: return fastEmit_X86ISD_CMP_MVT_i16_rr(RetVT, Op0, Op1); 13079 case MVT::i32: return fastEmit_X86ISD_CMP_MVT_i32_rr(RetVT, Op0, Op1); 13080 case MVT::i64: return fastEmit_X86ISD_CMP_MVT_i64_rr(RetVT, Op0, Op1); 13081 default: return 0; 13082 } 13083} 13084 13085// FastEmit functions for X86ISD::COMI. 13086 13087unsigned fastEmit_X86ISD_COMI_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13088 if (RetVT.SimpleTy != MVT::i32) 13089 return 0; 13090 if ((Subtarget->hasFP16())) { 13091 return fastEmitInst_rr(X86::VCOMISHZrr_Int, &X86::VR128XRegClass, Op0, Op1); 13092 } 13093 return 0; 13094} 13095 13096unsigned fastEmit_X86ISD_COMI_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13097 if (RetVT.SimpleTy != MVT::i32) 13098 return 0; 13099 if ((Subtarget->hasAVX512())) { 13100 return fastEmitInst_rr(X86::VCOMISSZrr_Int, &X86::VR128XRegClass, Op0, Op1); 13101 } 13102 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 13103 return fastEmitInst_rr(X86::COMISSrr_Int, &X86::VR128RegClass, Op0, Op1); 13104 } 13105 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 13106 return fastEmitInst_rr(X86::VCOMISSrr_Int, &X86::VR128RegClass, Op0, Op1); 13107 } 13108 return 0; 13109} 13110 13111unsigned fastEmit_X86ISD_COMI_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13112 if (RetVT.SimpleTy != MVT::i32) 13113 return 0; 13114 if ((Subtarget->hasAVX512())) { 13115 return fastEmitInst_rr(X86::VCOMISDZrr_Int, &X86::VR128XRegClass, Op0, Op1); 13116 } 13117 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 13118 return fastEmitInst_rr(X86::COMISDrr_Int, &X86::VR128RegClass, Op0, Op1); 13119 } 13120 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 13121 return fastEmitInst_rr(X86::VCOMISDrr_Int, &X86::VR128RegClass, Op0, Op1); 13122 } 13123 return 0; 13124} 13125 13126unsigned fastEmit_X86ISD_COMI_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 13127 switch (VT.SimpleTy) { 13128 case MVT::v8f16: return fastEmit_X86ISD_COMI_MVT_v8f16_rr(RetVT, Op0, Op1); 13129 case MVT::v4f32: return fastEmit_X86ISD_COMI_MVT_v4f32_rr(RetVT, Op0, Op1); 13130 case MVT::v2f64: return fastEmit_X86ISD_COMI_MVT_v2f64_rr(RetVT, Op0, Op1); 13131 default: return 0; 13132 } 13133} 13134 13135// FastEmit functions for X86ISD::CVTNE2PS2BF16. 13136 13137unsigned fastEmit_X86ISD_CVTNE2PS2BF16_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13138 if (RetVT.SimpleTy != MVT::v8bf16) 13139 return 0; 13140 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) { 13141 return fastEmitInst_rr(X86::VCVTNE2PS2BF16Z128rr, &X86::VR128XRegClass, Op0, Op1); 13142 } 13143 return 0; 13144} 13145 13146unsigned fastEmit_X86ISD_CVTNE2PS2BF16_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13147 if (RetVT.SimpleTy != MVT::v16bf16) 13148 return 0; 13149 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) { 13150 return fastEmitInst_rr(X86::VCVTNE2PS2BF16Z256rr, &X86::VR256XRegClass, Op0, Op1); 13151 } 13152 return 0; 13153} 13154 13155unsigned fastEmit_X86ISD_CVTNE2PS2BF16_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13156 if (RetVT.SimpleTy != MVT::v32bf16) 13157 return 0; 13158 if ((Subtarget->hasBF16())) { 13159 return fastEmitInst_rr(X86::VCVTNE2PS2BF16Zrr, &X86::VR512RegClass, Op0, Op1); 13160 } 13161 return 0; 13162} 13163 13164unsigned fastEmit_X86ISD_CVTNE2PS2BF16_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 13165 switch (VT.SimpleTy) { 13166 case MVT::v4f32: return fastEmit_X86ISD_CVTNE2PS2BF16_MVT_v4f32_rr(RetVT, Op0, Op1); 13167 case MVT::v8f32: return fastEmit_X86ISD_CVTNE2PS2BF16_MVT_v8f32_rr(RetVT, Op0, Op1); 13168 case MVT::v16f32: return fastEmit_X86ISD_CVTNE2PS2BF16_MVT_v16f32_rr(RetVT, Op0, Op1); 13169 default: return 0; 13170 } 13171} 13172 13173// FastEmit functions for X86ISD::FADDS. 13174 13175unsigned fastEmit_X86ISD_FADDS_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13176 if (RetVT.SimpleTy != MVT::v8f16) 13177 return 0; 13178 if ((Subtarget->hasFP16())) { 13179 return fastEmitInst_rr(X86::VADDSHZrr_Int, &X86::VR128XRegClass, Op0, Op1); 13180 } 13181 return 0; 13182} 13183 13184unsigned fastEmit_X86ISD_FADDS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13185 if (RetVT.SimpleTy != MVT::v4f32) 13186 return 0; 13187 if ((Subtarget->hasAVX512())) { 13188 return fastEmitInst_rr(X86::VADDSSZrr_Int, &X86::VR128XRegClass, Op0, Op1); 13189 } 13190 return 0; 13191} 13192 13193unsigned fastEmit_X86ISD_FADDS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13194 if (RetVT.SimpleTy != MVT::v2f64) 13195 return 0; 13196 if ((Subtarget->hasAVX512())) { 13197 return fastEmitInst_rr(X86::VADDSDZrr_Int, &X86::VR128XRegClass, Op0, Op1); 13198 } 13199 return 0; 13200} 13201 13202unsigned fastEmit_X86ISD_FADDS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 13203 switch (VT.SimpleTy) { 13204 case MVT::v8f16: return fastEmit_X86ISD_FADDS_MVT_v8f16_rr(RetVT, Op0, Op1); 13205 case MVT::v4f32: return fastEmit_X86ISD_FADDS_MVT_v4f32_rr(RetVT, Op0, Op1); 13206 case MVT::v2f64: return fastEmit_X86ISD_FADDS_MVT_v2f64_rr(RetVT, Op0, Op1); 13207 default: return 0; 13208 } 13209} 13210 13211// FastEmit functions for X86ISD::FAND. 13212 13213unsigned fastEmit_X86ISD_FAND_MVT_f128_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13214 if (RetVT.SimpleTy != MVT::f128) 13215 return 0; 13216 if ((Subtarget->hasVLX())) { 13217 return fastEmitInst_rr(X86::VANDPSZ128rr, &X86::VR128XRegClass, Op0, Op1); 13218 } 13219 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 13220 return fastEmitInst_rr(X86::VANDPSrr, &X86::VR128RegClass, Op0, Op1); 13221 } 13222 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 13223 return fastEmitInst_rr(X86::ANDPSrr, &X86::VR128RegClass, Op0, Op1); 13224 } 13225 return 0; 13226} 13227 13228unsigned fastEmit_X86ISD_FAND_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13229 if (RetVT.SimpleTy != MVT::v4f32) 13230 return 0; 13231 return fastEmitInst_rr(X86::ANDPSrr, &X86::VR128RegClass, Op0, Op1); 13232} 13233 13234unsigned fastEmit_X86ISD_FAND_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 13235 switch (VT.SimpleTy) { 13236 case MVT::f128: return fastEmit_X86ISD_FAND_MVT_f128_rr(RetVT, Op0, Op1); 13237 case MVT::v4f32: return fastEmit_X86ISD_FAND_MVT_v4f32_rr(RetVT, Op0, Op1); 13238 default: return 0; 13239 } 13240} 13241 13242// FastEmit functions for X86ISD::FANDN. 13243 13244unsigned fastEmit_X86ISD_FANDN_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13245 if (RetVT.SimpleTy != MVT::v4f32) 13246 return 0; 13247 return fastEmitInst_rr(X86::ANDNPSrr, &X86::VR128RegClass, Op0, Op1); 13248} 13249 13250unsigned fastEmit_X86ISD_FANDN_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 13251 switch (VT.SimpleTy) { 13252 case MVT::v4f32: return fastEmit_X86ISD_FANDN_MVT_v4f32_rr(RetVT, Op0, Op1); 13253 default: return 0; 13254 } 13255} 13256 13257// FastEmit functions for X86ISD::FCMP. 13258 13259unsigned fastEmit_X86ISD_FCMP_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13260 if (RetVT.SimpleTy != MVT::i32) 13261 return 0; 13262 if ((Subtarget->hasFP16())) { 13263 return fastEmitInst_rr(X86::VUCOMISHZrr, &X86::FR16XRegClass, Op0, Op1); 13264 } 13265 return 0; 13266} 13267 13268unsigned fastEmit_X86ISD_FCMP_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13269 if (RetVT.SimpleTy != MVT::i32) 13270 return 0; 13271 if ((Subtarget->hasAVX512())) { 13272 return fastEmitInst_rr(X86::VUCOMISSZrr, &X86::FR32XRegClass, Op0, Op1); 13273 } 13274 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 13275 return fastEmitInst_rr(X86::UCOMISSrr, &X86::FR32RegClass, Op0, Op1); 13276 } 13277 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 13278 return fastEmitInst_rr(X86::VUCOMISSrr, &X86::FR32RegClass, Op0, Op1); 13279 } 13280 if ((!Subtarget->hasSSE1()) && (Subtarget->canUseCMOV())) { 13281 return fastEmitInst_rr(X86::UCOM_FpIr32, &X86::RFP32RegClass, Op0, Op1); 13282 } 13283 return 0; 13284} 13285 13286unsigned fastEmit_X86ISD_FCMP_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13287 if (RetVT.SimpleTy != MVT::i32) 13288 return 0; 13289 if ((Subtarget->hasAVX512())) { 13290 return fastEmitInst_rr(X86::VUCOMISDZrr, &X86::FR64XRegClass, Op0, Op1); 13291 } 13292 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 13293 return fastEmitInst_rr(X86::UCOMISDrr, &X86::FR64RegClass, Op0, Op1); 13294 } 13295 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 13296 return fastEmitInst_rr(X86::VUCOMISDrr, &X86::FR64RegClass, Op0, Op1); 13297 } 13298 if ((!Subtarget->hasSSE2()) && (Subtarget->canUseCMOV())) { 13299 return fastEmitInst_rr(X86::UCOM_FpIr64, &X86::RFP64RegClass, Op0, Op1); 13300 } 13301 return 0; 13302} 13303 13304unsigned fastEmit_X86ISD_FCMP_MVT_f80_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13305 if (RetVT.SimpleTy != MVT::i32) 13306 return 0; 13307 if ((Subtarget->canUseCMOV())) { 13308 return fastEmitInst_rr(X86::UCOM_FpIr80, &X86::RFP80RegClass, Op0, Op1); 13309 } 13310 return 0; 13311} 13312 13313unsigned fastEmit_X86ISD_FCMP_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 13314 switch (VT.SimpleTy) { 13315 case MVT::f16: return fastEmit_X86ISD_FCMP_MVT_f16_rr(RetVT, Op0, Op1); 13316 case MVT::f32: return fastEmit_X86ISD_FCMP_MVT_f32_rr(RetVT, Op0, Op1); 13317 case MVT::f64: return fastEmit_X86ISD_FCMP_MVT_f64_rr(RetVT, Op0, Op1); 13318 case MVT::f80: return fastEmit_X86ISD_FCMP_MVT_f80_rr(RetVT, Op0, Op1); 13319 default: return 0; 13320 } 13321} 13322 13323// FastEmit functions for X86ISD::FDIVS. 13324 13325unsigned fastEmit_X86ISD_FDIVS_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13326 if (RetVT.SimpleTy != MVT::v8f16) 13327 return 0; 13328 if ((Subtarget->hasFP16())) { 13329 return fastEmitInst_rr(X86::VDIVSHZrr_Int, &X86::VR128XRegClass, Op0, Op1); 13330 } 13331 return 0; 13332} 13333 13334unsigned fastEmit_X86ISD_FDIVS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13335 if (RetVT.SimpleTy != MVT::v4f32) 13336 return 0; 13337 if ((Subtarget->hasAVX512())) { 13338 return fastEmitInst_rr(X86::VDIVSSZrr_Int, &X86::VR128XRegClass, Op0, Op1); 13339 } 13340 return 0; 13341} 13342 13343unsigned fastEmit_X86ISD_FDIVS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13344 if (RetVT.SimpleTy != MVT::v2f64) 13345 return 0; 13346 if ((Subtarget->hasAVX512())) { 13347 return fastEmitInst_rr(X86::VDIVSDZrr_Int, &X86::VR128XRegClass, Op0, Op1); 13348 } 13349 return 0; 13350} 13351 13352unsigned fastEmit_X86ISD_FDIVS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 13353 switch (VT.SimpleTy) { 13354 case MVT::v8f16: return fastEmit_X86ISD_FDIVS_MVT_v8f16_rr(RetVT, Op0, Op1); 13355 case MVT::v4f32: return fastEmit_X86ISD_FDIVS_MVT_v4f32_rr(RetVT, Op0, Op1); 13356 case MVT::v2f64: return fastEmit_X86ISD_FDIVS_MVT_v2f64_rr(RetVT, Op0, Op1); 13357 default: return 0; 13358 } 13359} 13360 13361// FastEmit functions for X86ISD::FGETEXPS. 13362 13363unsigned fastEmit_X86ISD_FGETEXPS_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13364 if (RetVT.SimpleTy != MVT::v8f16) 13365 return 0; 13366 if ((Subtarget->hasFP16())) { 13367 return fastEmitInst_rr(X86::VGETEXPSHZr, &X86::VR128XRegClass, Op0, Op1); 13368 } 13369 return 0; 13370} 13371 13372unsigned fastEmit_X86ISD_FGETEXPS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13373 if (RetVT.SimpleTy != MVT::v4f32) 13374 return 0; 13375 if ((Subtarget->hasAVX512())) { 13376 return fastEmitInst_rr(X86::VGETEXPSSZr, &X86::VR128XRegClass, Op0, Op1); 13377 } 13378 return 0; 13379} 13380 13381unsigned fastEmit_X86ISD_FGETEXPS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13382 if (RetVT.SimpleTy != MVT::v2f64) 13383 return 0; 13384 if ((Subtarget->hasAVX512())) { 13385 return fastEmitInst_rr(X86::VGETEXPSDZr, &X86::VR128XRegClass, Op0, Op1); 13386 } 13387 return 0; 13388} 13389 13390unsigned fastEmit_X86ISD_FGETEXPS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 13391 switch (VT.SimpleTy) { 13392 case MVT::v8f16: return fastEmit_X86ISD_FGETEXPS_MVT_v8f16_rr(RetVT, Op0, Op1); 13393 case MVT::v4f32: return fastEmit_X86ISD_FGETEXPS_MVT_v4f32_rr(RetVT, Op0, Op1); 13394 case MVT::v2f64: return fastEmit_X86ISD_FGETEXPS_MVT_v2f64_rr(RetVT, Op0, Op1); 13395 default: return 0; 13396 } 13397} 13398 13399// FastEmit functions for X86ISD::FGETEXPS_SAE. 13400 13401unsigned fastEmit_X86ISD_FGETEXPS_SAE_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13402 if (RetVT.SimpleTy != MVT::v8f16) 13403 return 0; 13404 if ((Subtarget->hasFP16())) { 13405 return fastEmitInst_rr(X86::VGETEXPSHZrb, &X86::VR128XRegClass, Op0, Op1); 13406 } 13407 return 0; 13408} 13409 13410unsigned fastEmit_X86ISD_FGETEXPS_SAE_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13411 if (RetVT.SimpleTy != MVT::v4f32) 13412 return 0; 13413 if ((Subtarget->hasAVX512())) { 13414 return fastEmitInst_rr(X86::VGETEXPSSZrb, &X86::VR128XRegClass, Op0, Op1); 13415 } 13416 return 0; 13417} 13418 13419unsigned fastEmit_X86ISD_FGETEXPS_SAE_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13420 if (RetVT.SimpleTy != MVT::v2f64) 13421 return 0; 13422 if ((Subtarget->hasAVX512())) { 13423 return fastEmitInst_rr(X86::VGETEXPSDZrb, &X86::VR128XRegClass, Op0, Op1); 13424 } 13425 return 0; 13426} 13427 13428unsigned fastEmit_X86ISD_FGETEXPS_SAE_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 13429 switch (VT.SimpleTy) { 13430 case MVT::v8f16: return fastEmit_X86ISD_FGETEXPS_SAE_MVT_v8f16_rr(RetVT, Op0, Op1); 13431 case MVT::v4f32: return fastEmit_X86ISD_FGETEXPS_SAE_MVT_v4f32_rr(RetVT, Op0, Op1); 13432 case MVT::v2f64: return fastEmit_X86ISD_FGETEXPS_SAE_MVT_v2f64_rr(RetVT, Op0, Op1); 13433 default: return 0; 13434 } 13435} 13436 13437// FastEmit functions for X86ISD::FHADD. 13438 13439unsigned fastEmit_X86ISD_FHADD_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13440 if (RetVT.SimpleTy != MVT::v4f32) 13441 return 0; 13442 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) { 13443 return fastEmitInst_rr(X86::HADDPSrr, &X86::VR128RegClass, Op0, Op1); 13444 } 13445 if ((Subtarget->hasAVX())) { 13446 return fastEmitInst_rr(X86::VHADDPSrr, &X86::VR128RegClass, Op0, Op1); 13447 } 13448 return 0; 13449} 13450 13451unsigned fastEmit_X86ISD_FHADD_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13452 if (RetVT.SimpleTy != MVT::v8f32) 13453 return 0; 13454 if ((Subtarget->hasAVX())) { 13455 return fastEmitInst_rr(X86::VHADDPSYrr, &X86::VR256RegClass, Op0, Op1); 13456 } 13457 return 0; 13458} 13459 13460unsigned fastEmit_X86ISD_FHADD_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13461 if (RetVT.SimpleTy != MVT::v2f64) 13462 return 0; 13463 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) { 13464 return fastEmitInst_rr(X86::HADDPDrr, &X86::VR128RegClass, Op0, Op1); 13465 } 13466 if ((Subtarget->hasAVX())) { 13467 return fastEmitInst_rr(X86::VHADDPDrr, &X86::VR128RegClass, Op0, Op1); 13468 } 13469 return 0; 13470} 13471 13472unsigned fastEmit_X86ISD_FHADD_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13473 if (RetVT.SimpleTy != MVT::v4f64) 13474 return 0; 13475 if ((Subtarget->hasAVX())) { 13476 return fastEmitInst_rr(X86::VHADDPDYrr, &X86::VR256RegClass, Op0, Op1); 13477 } 13478 return 0; 13479} 13480 13481unsigned fastEmit_X86ISD_FHADD_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 13482 switch (VT.SimpleTy) { 13483 case MVT::v4f32: return fastEmit_X86ISD_FHADD_MVT_v4f32_rr(RetVT, Op0, Op1); 13484 case MVT::v8f32: return fastEmit_X86ISD_FHADD_MVT_v8f32_rr(RetVT, Op0, Op1); 13485 case MVT::v2f64: return fastEmit_X86ISD_FHADD_MVT_v2f64_rr(RetVT, Op0, Op1); 13486 case MVT::v4f64: return fastEmit_X86ISD_FHADD_MVT_v4f64_rr(RetVT, Op0, Op1); 13487 default: return 0; 13488 } 13489} 13490 13491// FastEmit functions for X86ISD::FHSUB. 13492 13493unsigned fastEmit_X86ISD_FHSUB_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13494 if (RetVT.SimpleTy != MVT::v4f32) 13495 return 0; 13496 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) { 13497 return fastEmitInst_rr(X86::HSUBPSrr, &X86::VR128RegClass, Op0, Op1); 13498 } 13499 if ((Subtarget->hasAVX())) { 13500 return fastEmitInst_rr(X86::VHSUBPSrr, &X86::VR128RegClass, Op0, Op1); 13501 } 13502 return 0; 13503} 13504 13505unsigned fastEmit_X86ISD_FHSUB_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13506 if (RetVT.SimpleTy != MVT::v8f32) 13507 return 0; 13508 if ((Subtarget->hasAVX())) { 13509 return fastEmitInst_rr(X86::VHSUBPSYrr, &X86::VR256RegClass, Op0, Op1); 13510 } 13511 return 0; 13512} 13513 13514unsigned fastEmit_X86ISD_FHSUB_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13515 if (RetVT.SimpleTy != MVT::v2f64) 13516 return 0; 13517 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) { 13518 return fastEmitInst_rr(X86::HSUBPDrr, &X86::VR128RegClass, Op0, Op1); 13519 } 13520 if ((Subtarget->hasAVX())) { 13521 return fastEmitInst_rr(X86::VHSUBPDrr, &X86::VR128RegClass, Op0, Op1); 13522 } 13523 return 0; 13524} 13525 13526unsigned fastEmit_X86ISD_FHSUB_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13527 if (RetVT.SimpleTy != MVT::v4f64) 13528 return 0; 13529 if ((Subtarget->hasAVX())) { 13530 return fastEmitInst_rr(X86::VHSUBPDYrr, &X86::VR256RegClass, Op0, Op1); 13531 } 13532 return 0; 13533} 13534 13535unsigned fastEmit_X86ISD_FHSUB_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 13536 switch (VT.SimpleTy) { 13537 case MVT::v4f32: return fastEmit_X86ISD_FHSUB_MVT_v4f32_rr(RetVT, Op0, Op1); 13538 case MVT::v8f32: return fastEmit_X86ISD_FHSUB_MVT_v8f32_rr(RetVT, Op0, Op1); 13539 case MVT::v2f64: return fastEmit_X86ISD_FHSUB_MVT_v2f64_rr(RetVT, Op0, Op1); 13540 case MVT::v4f64: return fastEmit_X86ISD_FHSUB_MVT_v4f64_rr(RetVT, Op0, Op1); 13541 default: return 0; 13542 } 13543} 13544 13545// FastEmit functions for X86ISD::FMAX. 13546 13547unsigned fastEmit_X86ISD_FMAX_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13548 if (RetVT.SimpleTy != MVT::f16) 13549 return 0; 13550 if ((Subtarget->hasFP16())) { 13551 return fastEmitInst_rr(X86::VMAXSHZrr, &X86::FR16XRegClass, Op0, Op1); 13552 } 13553 return 0; 13554} 13555 13556unsigned fastEmit_X86ISD_FMAX_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13557 if (RetVT.SimpleTy != MVT::f32) 13558 return 0; 13559 if ((Subtarget->hasAVX512())) { 13560 return fastEmitInst_rr(X86::VMAXSSZrr, &X86::FR32XRegClass, Op0, Op1); 13561 } 13562 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 13563 return fastEmitInst_rr(X86::MAXSSrr, &X86::FR32RegClass, Op0, Op1); 13564 } 13565 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 13566 return fastEmitInst_rr(X86::VMAXSSrr, &X86::FR32RegClass, Op0, Op1); 13567 } 13568 return 0; 13569} 13570 13571unsigned fastEmit_X86ISD_FMAX_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13572 if (RetVT.SimpleTy != MVT::f64) 13573 return 0; 13574 if ((Subtarget->hasAVX512())) { 13575 return fastEmitInst_rr(X86::VMAXSDZrr, &X86::FR64XRegClass, Op0, Op1); 13576 } 13577 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 13578 return fastEmitInst_rr(X86::MAXSDrr, &X86::FR64RegClass, Op0, Op1); 13579 } 13580 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 13581 return fastEmitInst_rr(X86::VMAXSDrr, &X86::FR64RegClass, Op0, Op1); 13582 } 13583 return 0; 13584} 13585 13586unsigned fastEmit_X86ISD_FMAX_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13587 if (RetVT.SimpleTy != MVT::v8f16) 13588 return 0; 13589 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 13590 return fastEmitInst_rr(X86::VMAXPHZ128rr, &X86::VR128XRegClass, Op0, Op1); 13591 } 13592 return 0; 13593} 13594 13595unsigned fastEmit_X86ISD_FMAX_MVT_v16f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13596 if (RetVT.SimpleTy != MVT::v16f16) 13597 return 0; 13598 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 13599 return fastEmitInst_rr(X86::VMAXPHZ256rr, &X86::VR256XRegClass, Op0, Op1); 13600 } 13601 return 0; 13602} 13603 13604unsigned fastEmit_X86ISD_FMAX_MVT_v32f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13605 if (RetVT.SimpleTy != MVT::v32f16) 13606 return 0; 13607 if ((Subtarget->hasFP16())) { 13608 return fastEmitInst_rr(X86::VMAXPHZrr, &X86::VR512RegClass, Op0, Op1); 13609 } 13610 return 0; 13611} 13612 13613unsigned fastEmit_X86ISD_FMAX_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13614 if (RetVT.SimpleTy != MVT::v4f32) 13615 return 0; 13616 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 13617 return fastEmitInst_rr(X86::VMAXPSZ128rr, &X86::VR128XRegClass, Op0, Op1); 13618 } 13619 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 13620 return fastEmitInst_rr(X86::MAXPSrr, &X86::VR128RegClass, Op0, Op1); 13621 } 13622 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 13623 return fastEmitInst_rr(X86::VMAXPSrr, &X86::VR128RegClass, Op0, Op1); 13624 } 13625 return 0; 13626} 13627 13628unsigned fastEmit_X86ISD_FMAX_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13629 if (RetVT.SimpleTy != MVT::v8f32) 13630 return 0; 13631 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 13632 return fastEmitInst_rr(X86::VMAXPSZ256rr, &X86::VR256XRegClass, Op0, Op1); 13633 } 13634 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 13635 return fastEmitInst_rr(X86::VMAXPSYrr, &X86::VR256RegClass, Op0, Op1); 13636 } 13637 return 0; 13638} 13639 13640unsigned fastEmit_X86ISD_FMAX_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13641 if (RetVT.SimpleTy != MVT::v16f32) 13642 return 0; 13643 if ((Subtarget->hasAVX512())) { 13644 return fastEmitInst_rr(X86::VMAXPSZrr, &X86::VR512RegClass, Op0, Op1); 13645 } 13646 return 0; 13647} 13648 13649unsigned fastEmit_X86ISD_FMAX_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13650 if (RetVT.SimpleTy != MVT::v2f64) 13651 return 0; 13652 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 13653 return fastEmitInst_rr(X86::VMAXPDZ128rr, &X86::VR128XRegClass, Op0, Op1); 13654 } 13655 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 13656 return fastEmitInst_rr(X86::MAXPDrr, &X86::VR128RegClass, Op0, Op1); 13657 } 13658 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 13659 return fastEmitInst_rr(X86::VMAXPDrr, &X86::VR128RegClass, Op0, Op1); 13660 } 13661 return 0; 13662} 13663 13664unsigned fastEmit_X86ISD_FMAX_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13665 if (RetVT.SimpleTy != MVT::v4f64) 13666 return 0; 13667 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 13668 return fastEmitInst_rr(X86::VMAXPDZ256rr, &X86::VR256XRegClass, Op0, Op1); 13669 } 13670 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 13671 return fastEmitInst_rr(X86::VMAXPDYrr, &X86::VR256RegClass, Op0, Op1); 13672 } 13673 return 0; 13674} 13675 13676unsigned fastEmit_X86ISD_FMAX_MVT_v8f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13677 if (RetVT.SimpleTy != MVT::v8f64) 13678 return 0; 13679 if ((Subtarget->hasAVX512())) { 13680 return fastEmitInst_rr(X86::VMAXPDZrr, &X86::VR512RegClass, Op0, Op1); 13681 } 13682 return 0; 13683} 13684 13685unsigned fastEmit_X86ISD_FMAX_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 13686 switch (VT.SimpleTy) { 13687 case MVT::f16: return fastEmit_X86ISD_FMAX_MVT_f16_rr(RetVT, Op0, Op1); 13688 case MVT::f32: return fastEmit_X86ISD_FMAX_MVT_f32_rr(RetVT, Op0, Op1); 13689 case MVT::f64: return fastEmit_X86ISD_FMAX_MVT_f64_rr(RetVT, Op0, Op1); 13690 case MVT::v8f16: return fastEmit_X86ISD_FMAX_MVT_v8f16_rr(RetVT, Op0, Op1); 13691 case MVT::v16f16: return fastEmit_X86ISD_FMAX_MVT_v16f16_rr(RetVT, Op0, Op1); 13692 case MVT::v32f16: return fastEmit_X86ISD_FMAX_MVT_v32f16_rr(RetVT, Op0, Op1); 13693 case MVT::v4f32: return fastEmit_X86ISD_FMAX_MVT_v4f32_rr(RetVT, Op0, Op1); 13694 case MVT::v8f32: return fastEmit_X86ISD_FMAX_MVT_v8f32_rr(RetVT, Op0, Op1); 13695 case MVT::v16f32: return fastEmit_X86ISD_FMAX_MVT_v16f32_rr(RetVT, Op0, Op1); 13696 case MVT::v2f64: return fastEmit_X86ISD_FMAX_MVT_v2f64_rr(RetVT, Op0, Op1); 13697 case MVT::v4f64: return fastEmit_X86ISD_FMAX_MVT_v4f64_rr(RetVT, Op0, Op1); 13698 case MVT::v8f64: return fastEmit_X86ISD_FMAX_MVT_v8f64_rr(RetVT, Op0, Op1); 13699 default: return 0; 13700 } 13701} 13702 13703// FastEmit functions for X86ISD::FMAXC. 13704 13705unsigned fastEmit_X86ISD_FMAXC_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13706 if (RetVT.SimpleTy != MVT::f16) 13707 return 0; 13708 if ((Subtarget->hasAVX512())) { 13709 return fastEmitInst_rr(X86::VMAXCSHZrr, &X86::FR16XRegClass, Op0, Op1); 13710 } 13711 return 0; 13712} 13713 13714unsigned fastEmit_X86ISD_FMAXC_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13715 if (RetVT.SimpleTy != MVT::f32) 13716 return 0; 13717 if ((Subtarget->hasAVX512())) { 13718 return fastEmitInst_rr(X86::VMAXCSSZrr, &X86::FR32XRegClass, Op0, Op1); 13719 } 13720 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 13721 return fastEmitInst_rr(X86::MAXCSSrr, &X86::FR32RegClass, Op0, Op1); 13722 } 13723 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 13724 return fastEmitInst_rr(X86::VMAXCSSrr, &X86::FR32RegClass, Op0, Op1); 13725 } 13726 return 0; 13727} 13728 13729unsigned fastEmit_X86ISD_FMAXC_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13730 if (RetVT.SimpleTy != MVT::f64) 13731 return 0; 13732 if ((Subtarget->hasAVX512())) { 13733 return fastEmitInst_rr(X86::VMAXCSDZrr, &X86::FR64XRegClass, Op0, Op1); 13734 } 13735 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 13736 return fastEmitInst_rr(X86::MAXCSDrr, &X86::FR64RegClass, Op0, Op1); 13737 } 13738 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 13739 return fastEmitInst_rr(X86::VMAXCSDrr, &X86::FR64RegClass, Op0, Op1); 13740 } 13741 return 0; 13742} 13743 13744unsigned fastEmit_X86ISD_FMAXC_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13745 if (RetVT.SimpleTy != MVT::v8f16) 13746 return 0; 13747 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 13748 return fastEmitInst_rr(X86::VMAXCPHZ128rr, &X86::VR128XRegClass, Op0, Op1); 13749 } 13750 return 0; 13751} 13752 13753unsigned fastEmit_X86ISD_FMAXC_MVT_v16f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13754 if (RetVT.SimpleTy != MVT::v16f16) 13755 return 0; 13756 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 13757 return fastEmitInst_rr(X86::VMAXCPHZ256rr, &X86::VR256XRegClass, Op0, Op1); 13758 } 13759 return 0; 13760} 13761 13762unsigned fastEmit_X86ISD_FMAXC_MVT_v32f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13763 if (RetVT.SimpleTy != MVT::v32f16) 13764 return 0; 13765 if ((Subtarget->hasFP16())) { 13766 return fastEmitInst_rr(X86::VMAXCPHZrr, &X86::VR512RegClass, Op0, Op1); 13767 } 13768 return 0; 13769} 13770 13771unsigned fastEmit_X86ISD_FMAXC_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13772 if (RetVT.SimpleTy != MVT::v4f32) 13773 return 0; 13774 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 13775 return fastEmitInst_rr(X86::VMAXCPSZ128rr, &X86::VR128XRegClass, Op0, Op1); 13776 } 13777 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 13778 return fastEmitInst_rr(X86::MAXCPSrr, &X86::VR128RegClass, Op0, Op1); 13779 } 13780 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 13781 return fastEmitInst_rr(X86::VMAXCPSrr, &X86::VR128RegClass, Op0, Op1); 13782 } 13783 return 0; 13784} 13785 13786unsigned fastEmit_X86ISD_FMAXC_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13787 if (RetVT.SimpleTy != MVT::v8f32) 13788 return 0; 13789 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 13790 return fastEmitInst_rr(X86::VMAXCPSZ256rr, &X86::VR256XRegClass, Op0, Op1); 13791 } 13792 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 13793 return fastEmitInst_rr(X86::VMAXCPSYrr, &X86::VR256RegClass, Op0, Op1); 13794 } 13795 return 0; 13796} 13797 13798unsigned fastEmit_X86ISD_FMAXC_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13799 if (RetVT.SimpleTy != MVT::v16f32) 13800 return 0; 13801 if ((Subtarget->hasAVX512())) { 13802 return fastEmitInst_rr(X86::VMAXCPSZrr, &X86::VR512RegClass, Op0, Op1); 13803 } 13804 return 0; 13805} 13806 13807unsigned fastEmit_X86ISD_FMAXC_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13808 if (RetVT.SimpleTy != MVT::v2f64) 13809 return 0; 13810 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 13811 return fastEmitInst_rr(X86::VMAXCPDZ128rr, &X86::VR128XRegClass, Op0, Op1); 13812 } 13813 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 13814 return fastEmitInst_rr(X86::MAXCPDrr, &X86::VR128RegClass, Op0, Op1); 13815 } 13816 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 13817 return fastEmitInst_rr(X86::VMAXCPDrr, &X86::VR128RegClass, Op0, Op1); 13818 } 13819 return 0; 13820} 13821 13822unsigned fastEmit_X86ISD_FMAXC_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13823 if (RetVT.SimpleTy != MVT::v4f64) 13824 return 0; 13825 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 13826 return fastEmitInst_rr(X86::VMAXCPDZ256rr, &X86::VR256XRegClass, Op0, Op1); 13827 } 13828 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 13829 return fastEmitInst_rr(X86::VMAXCPDYrr, &X86::VR256RegClass, Op0, Op1); 13830 } 13831 return 0; 13832} 13833 13834unsigned fastEmit_X86ISD_FMAXC_MVT_v8f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13835 if (RetVT.SimpleTy != MVT::v8f64) 13836 return 0; 13837 if ((Subtarget->hasAVX512())) { 13838 return fastEmitInst_rr(X86::VMAXCPDZrr, &X86::VR512RegClass, Op0, Op1); 13839 } 13840 return 0; 13841} 13842 13843unsigned fastEmit_X86ISD_FMAXC_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 13844 switch (VT.SimpleTy) { 13845 case MVT::f16: return fastEmit_X86ISD_FMAXC_MVT_f16_rr(RetVT, Op0, Op1); 13846 case MVT::f32: return fastEmit_X86ISD_FMAXC_MVT_f32_rr(RetVT, Op0, Op1); 13847 case MVT::f64: return fastEmit_X86ISD_FMAXC_MVT_f64_rr(RetVT, Op0, Op1); 13848 case MVT::v8f16: return fastEmit_X86ISD_FMAXC_MVT_v8f16_rr(RetVT, Op0, Op1); 13849 case MVT::v16f16: return fastEmit_X86ISD_FMAXC_MVT_v16f16_rr(RetVT, Op0, Op1); 13850 case MVT::v32f16: return fastEmit_X86ISD_FMAXC_MVT_v32f16_rr(RetVT, Op0, Op1); 13851 case MVT::v4f32: return fastEmit_X86ISD_FMAXC_MVT_v4f32_rr(RetVT, Op0, Op1); 13852 case MVT::v8f32: return fastEmit_X86ISD_FMAXC_MVT_v8f32_rr(RetVT, Op0, Op1); 13853 case MVT::v16f32: return fastEmit_X86ISD_FMAXC_MVT_v16f32_rr(RetVT, Op0, Op1); 13854 case MVT::v2f64: return fastEmit_X86ISD_FMAXC_MVT_v2f64_rr(RetVT, Op0, Op1); 13855 case MVT::v4f64: return fastEmit_X86ISD_FMAXC_MVT_v4f64_rr(RetVT, Op0, Op1); 13856 case MVT::v8f64: return fastEmit_X86ISD_FMAXC_MVT_v8f64_rr(RetVT, Op0, Op1); 13857 default: return 0; 13858 } 13859} 13860 13861// FastEmit functions for X86ISD::FMAXS. 13862 13863unsigned fastEmit_X86ISD_FMAXS_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13864 if (RetVT.SimpleTy != MVT::v8f16) 13865 return 0; 13866 if ((Subtarget->hasFP16())) { 13867 return fastEmitInst_rr(X86::VMAXSHZrr_Int, &X86::VR128XRegClass, Op0, Op1); 13868 } 13869 return 0; 13870} 13871 13872unsigned fastEmit_X86ISD_FMAXS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13873 if (RetVT.SimpleTy != MVT::v4f32) 13874 return 0; 13875 if ((Subtarget->hasAVX512())) { 13876 return fastEmitInst_rr(X86::VMAXSSZrr_Int, &X86::VR128XRegClass, Op0, Op1); 13877 } 13878 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 13879 return fastEmitInst_rr(X86::MAXSSrr_Int, &X86::VR128RegClass, Op0, Op1); 13880 } 13881 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 13882 return fastEmitInst_rr(X86::VMAXSSrr_Int, &X86::VR128RegClass, Op0, Op1); 13883 } 13884 return 0; 13885} 13886 13887unsigned fastEmit_X86ISD_FMAXS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13888 if (RetVT.SimpleTy != MVT::v2f64) 13889 return 0; 13890 if ((Subtarget->hasAVX512())) { 13891 return fastEmitInst_rr(X86::VMAXSDZrr_Int, &X86::VR128XRegClass, Op0, Op1); 13892 } 13893 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 13894 return fastEmitInst_rr(X86::MAXSDrr_Int, &X86::VR128RegClass, Op0, Op1); 13895 } 13896 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 13897 return fastEmitInst_rr(X86::VMAXSDrr_Int, &X86::VR128RegClass, Op0, Op1); 13898 } 13899 return 0; 13900} 13901 13902unsigned fastEmit_X86ISD_FMAXS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 13903 switch (VT.SimpleTy) { 13904 case MVT::v8f16: return fastEmit_X86ISD_FMAXS_MVT_v8f16_rr(RetVT, Op0, Op1); 13905 case MVT::v4f32: return fastEmit_X86ISD_FMAXS_MVT_v4f32_rr(RetVT, Op0, Op1); 13906 case MVT::v2f64: return fastEmit_X86ISD_FMAXS_MVT_v2f64_rr(RetVT, Op0, Op1); 13907 default: return 0; 13908 } 13909} 13910 13911// FastEmit functions for X86ISD::FMAXS_SAE. 13912 13913unsigned fastEmit_X86ISD_FMAXS_SAE_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13914 if (RetVT.SimpleTy != MVT::v8f16) 13915 return 0; 13916 if ((Subtarget->hasFP16())) { 13917 return fastEmitInst_rr(X86::VMAXSHZrrb_Int, &X86::VR128XRegClass, Op0, Op1); 13918 } 13919 return 0; 13920} 13921 13922unsigned fastEmit_X86ISD_FMAXS_SAE_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13923 if (RetVT.SimpleTy != MVT::v4f32) 13924 return 0; 13925 if ((Subtarget->hasAVX512())) { 13926 return fastEmitInst_rr(X86::VMAXSSZrrb_Int, &X86::VR128XRegClass, Op0, Op1); 13927 } 13928 return 0; 13929} 13930 13931unsigned fastEmit_X86ISD_FMAXS_SAE_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13932 if (RetVT.SimpleTy != MVT::v2f64) 13933 return 0; 13934 if ((Subtarget->hasAVX512())) { 13935 return fastEmitInst_rr(X86::VMAXSDZrrb_Int, &X86::VR128XRegClass, Op0, Op1); 13936 } 13937 return 0; 13938} 13939 13940unsigned fastEmit_X86ISD_FMAXS_SAE_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 13941 switch (VT.SimpleTy) { 13942 case MVT::v8f16: return fastEmit_X86ISD_FMAXS_SAE_MVT_v8f16_rr(RetVT, Op0, Op1); 13943 case MVT::v4f32: return fastEmit_X86ISD_FMAXS_SAE_MVT_v4f32_rr(RetVT, Op0, Op1); 13944 case MVT::v2f64: return fastEmit_X86ISD_FMAXS_SAE_MVT_v2f64_rr(RetVT, Op0, Op1); 13945 default: return 0; 13946 } 13947} 13948 13949// FastEmit functions for X86ISD::FMAX_SAE. 13950 13951unsigned fastEmit_X86ISD_FMAX_SAE_MVT_v32f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13952 if (RetVT.SimpleTy != MVT::v32f16) 13953 return 0; 13954 if ((Subtarget->hasFP16())) { 13955 return fastEmitInst_rr(X86::VMAXPHZrrb, &X86::VR512RegClass, Op0, Op1); 13956 } 13957 return 0; 13958} 13959 13960unsigned fastEmit_X86ISD_FMAX_SAE_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13961 if (RetVT.SimpleTy != MVT::v16f32) 13962 return 0; 13963 if ((Subtarget->hasAVX512())) { 13964 return fastEmitInst_rr(X86::VMAXPSZrrb, &X86::VR512RegClass, Op0, Op1); 13965 } 13966 return 0; 13967} 13968 13969unsigned fastEmit_X86ISD_FMAX_SAE_MVT_v8f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13970 if (RetVT.SimpleTy != MVT::v8f64) 13971 return 0; 13972 if ((Subtarget->hasAVX512())) { 13973 return fastEmitInst_rr(X86::VMAXPDZrrb, &X86::VR512RegClass, Op0, Op1); 13974 } 13975 return 0; 13976} 13977 13978unsigned fastEmit_X86ISD_FMAX_SAE_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 13979 switch (VT.SimpleTy) { 13980 case MVT::v32f16: return fastEmit_X86ISD_FMAX_SAE_MVT_v32f16_rr(RetVT, Op0, Op1); 13981 case MVT::v16f32: return fastEmit_X86ISD_FMAX_SAE_MVT_v16f32_rr(RetVT, Op0, Op1); 13982 case MVT::v8f64: return fastEmit_X86ISD_FMAX_SAE_MVT_v8f64_rr(RetVT, Op0, Op1); 13983 default: return 0; 13984 } 13985} 13986 13987// FastEmit functions for X86ISD::FMIN. 13988 13989unsigned fastEmit_X86ISD_FMIN_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13990 if (RetVT.SimpleTy != MVT::f16) 13991 return 0; 13992 if ((Subtarget->hasFP16())) { 13993 return fastEmitInst_rr(X86::VMINSHZrr, &X86::FR16XRegClass, Op0, Op1); 13994 } 13995 return 0; 13996} 13997 13998unsigned fastEmit_X86ISD_FMIN_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13999 if (RetVT.SimpleTy != MVT::f32) 14000 return 0; 14001 if ((Subtarget->hasAVX512())) { 14002 return fastEmitInst_rr(X86::VMINSSZrr, &X86::FR32XRegClass, Op0, Op1); 14003 } 14004 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 14005 return fastEmitInst_rr(X86::MINSSrr, &X86::FR32RegClass, Op0, Op1); 14006 } 14007 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 14008 return fastEmitInst_rr(X86::VMINSSrr, &X86::FR32RegClass, Op0, Op1); 14009 } 14010 return 0; 14011} 14012 14013unsigned fastEmit_X86ISD_FMIN_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14014 if (RetVT.SimpleTy != MVT::f64) 14015 return 0; 14016 if ((Subtarget->hasAVX512())) { 14017 return fastEmitInst_rr(X86::VMINSDZrr, &X86::FR64XRegClass, Op0, Op1); 14018 } 14019 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 14020 return fastEmitInst_rr(X86::MINSDrr, &X86::FR64RegClass, Op0, Op1); 14021 } 14022 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 14023 return fastEmitInst_rr(X86::VMINSDrr, &X86::FR64RegClass, Op0, Op1); 14024 } 14025 return 0; 14026} 14027 14028unsigned fastEmit_X86ISD_FMIN_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14029 if (RetVT.SimpleTy != MVT::v8f16) 14030 return 0; 14031 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 14032 return fastEmitInst_rr(X86::VMINPHZ128rr, &X86::VR128XRegClass, Op0, Op1); 14033 } 14034 return 0; 14035} 14036 14037unsigned fastEmit_X86ISD_FMIN_MVT_v16f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14038 if (RetVT.SimpleTy != MVT::v16f16) 14039 return 0; 14040 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 14041 return fastEmitInst_rr(X86::VMINPHZ256rr, &X86::VR256XRegClass, Op0, Op1); 14042 } 14043 return 0; 14044} 14045 14046unsigned fastEmit_X86ISD_FMIN_MVT_v32f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14047 if (RetVT.SimpleTy != MVT::v32f16) 14048 return 0; 14049 if ((Subtarget->hasFP16())) { 14050 return fastEmitInst_rr(X86::VMINPHZrr, &X86::VR512RegClass, Op0, Op1); 14051 } 14052 return 0; 14053} 14054 14055unsigned fastEmit_X86ISD_FMIN_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14056 if (RetVT.SimpleTy != MVT::v4f32) 14057 return 0; 14058 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 14059 return fastEmitInst_rr(X86::VMINPSZ128rr, &X86::VR128XRegClass, Op0, Op1); 14060 } 14061 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 14062 return fastEmitInst_rr(X86::MINPSrr, &X86::VR128RegClass, Op0, Op1); 14063 } 14064 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 14065 return fastEmitInst_rr(X86::VMINPSrr, &X86::VR128RegClass, Op0, Op1); 14066 } 14067 return 0; 14068} 14069 14070unsigned fastEmit_X86ISD_FMIN_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14071 if (RetVT.SimpleTy != MVT::v8f32) 14072 return 0; 14073 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 14074 return fastEmitInst_rr(X86::VMINPSZ256rr, &X86::VR256XRegClass, Op0, Op1); 14075 } 14076 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 14077 return fastEmitInst_rr(X86::VMINPSYrr, &X86::VR256RegClass, Op0, Op1); 14078 } 14079 return 0; 14080} 14081 14082unsigned fastEmit_X86ISD_FMIN_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14083 if (RetVT.SimpleTy != MVT::v16f32) 14084 return 0; 14085 if ((Subtarget->hasAVX512())) { 14086 return fastEmitInst_rr(X86::VMINPSZrr, &X86::VR512RegClass, Op0, Op1); 14087 } 14088 return 0; 14089} 14090 14091unsigned fastEmit_X86ISD_FMIN_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14092 if (RetVT.SimpleTy != MVT::v2f64) 14093 return 0; 14094 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 14095 return fastEmitInst_rr(X86::VMINPDZ128rr, &X86::VR128XRegClass, Op0, Op1); 14096 } 14097 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 14098 return fastEmitInst_rr(X86::MINPDrr, &X86::VR128RegClass, Op0, Op1); 14099 } 14100 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 14101 return fastEmitInst_rr(X86::VMINPDrr, &X86::VR128RegClass, Op0, Op1); 14102 } 14103 return 0; 14104} 14105 14106unsigned fastEmit_X86ISD_FMIN_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14107 if (RetVT.SimpleTy != MVT::v4f64) 14108 return 0; 14109 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 14110 return fastEmitInst_rr(X86::VMINPDZ256rr, &X86::VR256XRegClass, Op0, Op1); 14111 } 14112 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 14113 return fastEmitInst_rr(X86::VMINPDYrr, &X86::VR256RegClass, Op0, Op1); 14114 } 14115 return 0; 14116} 14117 14118unsigned fastEmit_X86ISD_FMIN_MVT_v8f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14119 if (RetVT.SimpleTy != MVT::v8f64) 14120 return 0; 14121 if ((Subtarget->hasAVX512())) { 14122 return fastEmitInst_rr(X86::VMINPDZrr, &X86::VR512RegClass, Op0, Op1); 14123 } 14124 return 0; 14125} 14126 14127unsigned fastEmit_X86ISD_FMIN_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 14128 switch (VT.SimpleTy) { 14129 case MVT::f16: return fastEmit_X86ISD_FMIN_MVT_f16_rr(RetVT, Op0, Op1); 14130 case MVT::f32: return fastEmit_X86ISD_FMIN_MVT_f32_rr(RetVT, Op0, Op1); 14131 case MVT::f64: return fastEmit_X86ISD_FMIN_MVT_f64_rr(RetVT, Op0, Op1); 14132 case MVT::v8f16: return fastEmit_X86ISD_FMIN_MVT_v8f16_rr(RetVT, Op0, Op1); 14133 case MVT::v16f16: return fastEmit_X86ISD_FMIN_MVT_v16f16_rr(RetVT, Op0, Op1); 14134 case MVT::v32f16: return fastEmit_X86ISD_FMIN_MVT_v32f16_rr(RetVT, Op0, Op1); 14135 case MVT::v4f32: return fastEmit_X86ISD_FMIN_MVT_v4f32_rr(RetVT, Op0, Op1); 14136 case MVT::v8f32: return fastEmit_X86ISD_FMIN_MVT_v8f32_rr(RetVT, Op0, Op1); 14137 case MVT::v16f32: return fastEmit_X86ISD_FMIN_MVT_v16f32_rr(RetVT, Op0, Op1); 14138 case MVT::v2f64: return fastEmit_X86ISD_FMIN_MVT_v2f64_rr(RetVT, Op0, Op1); 14139 case MVT::v4f64: return fastEmit_X86ISD_FMIN_MVT_v4f64_rr(RetVT, Op0, Op1); 14140 case MVT::v8f64: return fastEmit_X86ISD_FMIN_MVT_v8f64_rr(RetVT, Op0, Op1); 14141 default: return 0; 14142 } 14143} 14144 14145// FastEmit functions for X86ISD::FMINC. 14146 14147unsigned fastEmit_X86ISD_FMINC_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14148 if (RetVT.SimpleTy != MVT::f16) 14149 return 0; 14150 if ((Subtarget->hasAVX512())) { 14151 return fastEmitInst_rr(X86::VMINCSHZrr, &X86::FR16XRegClass, Op0, Op1); 14152 } 14153 return 0; 14154} 14155 14156unsigned fastEmit_X86ISD_FMINC_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14157 if (RetVT.SimpleTy != MVT::f32) 14158 return 0; 14159 if ((Subtarget->hasAVX512())) { 14160 return fastEmitInst_rr(X86::VMINCSSZrr, &X86::FR32XRegClass, Op0, Op1); 14161 } 14162 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 14163 return fastEmitInst_rr(X86::MINCSSrr, &X86::FR32RegClass, Op0, Op1); 14164 } 14165 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 14166 return fastEmitInst_rr(X86::VMINCSSrr, &X86::FR32RegClass, Op0, Op1); 14167 } 14168 return 0; 14169} 14170 14171unsigned fastEmit_X86ISD_FMINC_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14172 if (RetVT.SimpleTy != MVT::f64) 14173 return 0; 14174 if ((Subtarget->hasAVX512())) { 14175 return fastEmitInst_rr(X86::VMINCSDZrr, &X86::FR64XRegClass, Op0, Op1); 14176 } 14177 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 14178 return fastEmitInst_rr(X86::MINCSDrr, &X86::FR64RegClass, Op0, Op1); 14179 } 14180 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 14181 return fastEmitInst_rr(X86::VMINCSDrr, &X86::FR64RegClass, Op0, Op1); 14182 } 14183 return 0; 14184} 14185 14186unsigned fastEmit_X86ISD_FMINC_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14187 if (RetVT.SimpleTy != MVT::v8f16) 14188 return 0; 14189 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 14190 return fastEmitInst_rr(X86::VMINCPHZ128rr, &X86::VR128XRegClass, Op0, Op1); 14191 } 14192 return 0; 14193} 14194 14195unsigned fastEmit_X86ISD_FMINC_MVT_v16f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14196 if (RetVT.SimpleTy != MVT::v16f16) 14197 return 0; 14198 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 14199 return fastEmitInst_rr(X86::VMINCPHZ256rr, &X86::VR256XRegClass, Op0, Op1); 14200 } 14201 return 0; 14202} 14203 14204unsigned fastEmit_X86ISD_FMINC_MVT_v32f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14205 if (RetVT.SimpleTy != MVT::v32f16) 14206 return 0; 14207 if ((Subtarget->hasFP16())) { 14208 return fastEmitInst_rr(X86::VMINCPHZrr, &X86::VR512RegClass, Op0, Op1); 14209 } 14210 return 0; 14211} 14212 14213unsigned fastEmit_X86ISD_FMINC_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14214 if (RetVT.SimpleTy != MVT::v4f32) 14215 return 0; 14216 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 14217 return fastEmitInst_rr(X86::VMINCPSZ128rr, &X86::VR128XRegClass, Op0, Op1); 14218 } 14219 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 14220 return fastEmitInst_rr(X86::MINCPSrr, &X86::VR128RegClass, Op0, Op1); 14221 } 14222 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 14223 return fastEmitInst_rr(X86::VMINCPSrr, &X86::VR128RegClass, Op0, Op1); 14224 } 14225 return 0; 14226} 14227 14228unsigned fastEmit_X86ISD_FMINC_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14229 if (RetVT.SimpleTy != MVT::v8f32) 14230 return 0; 14231 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 14232 return fastEmitInst_rr(X86::VMINCPSZ256rr, &X86::VR256XRegClass, Op0, Op1); 14233 } 14234 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 14235 return fastEmitInst_rr(X86::VMINCPSYrr, &X86::VR256RegClass, Op0, Op1); 14236 } 14237 return 0; 14238} 14239 14240unsigned fastEmit_X86ISD_FMINC_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14241 if (RetVT.SimpleTy != MVT::v16f32) 14242 return 0; 14243 if ((Subtarget->hasAVX512())) { 14244 return fastEmitInst_rr(X86::VMINCPSZrr, &X86::VR512RegClass, Op0, Op1); 14245 } 14246 return 0; 14247} 14248 14249unsigned fastEmit_X86ISD_FMINC_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14250 if (RetVT.SimpleTy != MVT::v2f64) 14251 return 0; 14252 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 14253 return fastEmitInst_rr(X86::VMINCPDZ128rr, &X86::VR128XRegClass, Op0, Op1); 14254 } 14255 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 14256 return fastEmitInst_rr(X86::MINCPDrr, &X86::VR128RegClass, Op0, Op1); 14257 } 14258 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 14259 return fastEmitInst_rr(X86::VMINCPDrr, &X86::VR128RegClass, Op0, Op1); 14260 } 14261 return 0; 14262} 14263 14264unsigned fastEmit_X86ISD_FMINC_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14265 if (RetVT.SimpleTy != MVT::v4f64) 14266 return 0; 14267 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 14268 return fastEmitInst_rr(X86::VMINCPDZ256rr, &X86::VR256XRegClass, Op0, Op1); 14269 } 14270 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 14271 return fastEmitInst_rr(X86::VMINCPDYrr, &X86::VR256RegClass, Op0, Op1); 14272 } 14273 return 0; 14274} 14275 14276unsigned fastEmit_X86ISD_FMINC_MVT_v8f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14277 if (RetVT.SimpleTy != MVT::v8f64) 14278 return 0; 14279 if ((Subtarget->hasAVX512())) { 14280 return fastEmitInst_rr(X86::VMINCPDZrr, &X86::VR512RegClass, Op0, Op1); 14281 } 14282 return 0; 14283} 14284 14285unsigned fastEmit_X86ISD_FMINC_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 14286 switch (VT.SimpleTy) { 14287 case MVT::f16: return fastEmit_X86ISD_FMINC_MVT_f16_rr(RetVT, Op0, Op1); 14288 case MVT::f32: return fastEmit_X86ISD_FMINC_MVT_f32_rr(RetVT, Op0, Op1); 14289 case MVT::f64: return fastEmit_X86ISD_FMINC_MVT_f64_rr(RetVT, Op0, Op1); 14290 case MVT::v8f16: return fastEmit_X86ISD_FMINC_MVT_v8f16_rr(RetVT, Op0, Op1); 14291 case MVT::v16f16: return fastEmit_X86ISD_FMINC_MVT_v16f16_rr(RetVT, Op0, Op1); 14292 case MVT::v32f16: return fastEmit_X86ISD_FMINC_MVT_v32f16_rr(RetVT, Op0, Op1); 14293 case MVT::v4f32: return fastEmit_X86ISD_FMINC_MVT_v4f32_rr(RetVT, Op0, Op1); 14294 case MVT::v8f32: return fastEmit_X86ISD_FMINC_MVT_v8f32_rr(RetVT, Op0, Op1); 14295 case MVT::v16f32: return fastEmit_X86ISD_FMINC_MVT_v16f32_rr(RetVT, Op0, Op1); 14296 case MVT::v2f64: return fastEmit_X86ISD_FMINC_MVT_v2f64_rr(RetVT, Op0, Op1); 14297 case MVT::v4f64: return fastEmit_X86ISD_FMINC_MVT_v4f64_rr(RetVT, Op0, Op1); 14298 case MVT::v8f64: return fastEmit_X86ISD_FMINC_MVT_v8f64_rr(RetVT, Op0, Op1); 14299 default: return 0; 14300 } 14301} 14302 14303// FastEmit functions for X86ISD::FMINS. 14304 14305unsigned fastEmit_X86ISD_FMINS_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14306 if (RetVT.SimpleTy != MVT::v8f16) 14307 return 0; 14308 if ((Subtarget->hasFP16())) { 14309 return fastEmitInst_rr(X86::VMINSHZrr_Int, &X86::VR128XRegClass, Op0, Op1); 14310 } 14311 return 0; 14312} 14313 14314unsigned fastEmit_X86ISD_FMINS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14315 if (RetVT.SimpleTy != MVT::v4f32) 14316 return 0; 14317 if ((Subtarget->hasAVX512())) { 14318 return fastEmitInst_rr(X86::VMINSSZrr_Int, &X86::VR128XRegClass, Op0, Op1); 14319 } 14320 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 14321 return fastEmitInst_rr(X86::MINSSrr_Int, &X86::VR128RegClass, Op0, Op1); 14322 } 14323 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 14324 return fastEmitInst_rr(X86::VMINSSrr_Int, &X86::VR128RegClass, Op0, Op1); 14325 } 14326 return 0; 14327} 14328 14329unsigned fastEmit_X86ISD_FMINS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14330 if (RetVT.SimpleTy != MVT::v2f64) 14331 return 0; 14332 if ((Subtarget->hasAVX512())) { 14333 return fastEmitInst_rr(X86::VMINSDZrr_Int, &X86::VR128XRegClass, Op0, Op1); 14334 } 14335 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 14336 return fastEmitInst_rr(X86::MINSDrr_Int, &X86::VR128RegClass, Op0, Op1); 14337 } 14338 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 14339 return fastEmitInst_rr(X86::VMINSDrr_Int, &X86::VR128RegClass, Op0, Op1); 14340 } 14341 return 0; 14342} 14343 14344unsigned fastEmit_X86ISD_FMINS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 14345 switch (VT.SimpleTy) { 14346 case MVT::v8f16: return fastEmit_X86ISD_FMINS_MVT_v8f16_rr(RetVT, Op0, Op1); 14347 case MVT::v4f32: return fastEmit_X86ISD_FMINS_MVT_v4f32_rr(RetVT, Op0, Op1); 14348 case MVT::v2f64: return fastEmit_X86ISD_FMINS_MVT_v2f64_rr(RetVT, Op0, Op1); 14349 default: return 0; 14350 } 14351} 14352 14353// FastEmit functions for X86ISD::FMINS_SAE. 14354 14355unsigned fastEmit_X86ISD_FMINS_SAE_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14356 if (RetVT.SimpleTy != MVT::v8f16) 14357 return 0; 14358 if ((Subtarget->hasFP16())) { 14359 return fastEmitInst_rr(X86::VMINSHZrrb_Int, &X86::VR128XRegClass, Op0, Op1); 14360 } 14361 return 0; 14362} 14363 14364unsigned fastEmit_X86ISD_FMINS_SAE_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14365 if (RetVT.SimpleTy != MVT::v4f32) 14366 return 0; 14367 if ((Subtarget->hasAVX512())) { 14368 return fastEmitInst_rr(X86::VMINSSZrrb_Int, &X86::VR128XRegClass, Op0, Op1); 14369 } 14370 return 0; 14371} 14372 14373unsigned fastEmit_X86ISD_FMINS_SAE_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14374 if (RetVT.SimpleTy != MVT::v2f64) 14375 return 0; 14376 if ((Subtarget->hasAVX512())) { 14377 return fastEmitInst_rr(X86::VMINSDZrrb_Int, &X86::VR128XRegClass, Op0, Op1); 14378 } 14379 return 0; 14380} 14381 14382unsigned fastEmit_X86ISD_FMINS_SAE_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 14383 switch (VT.SimpleTy) { 14384 case MVT::v8f16: return fastEmit_X86ISD_FMINS_SAE_MVT_v8f16_rr(RetVT, Op0, Op1); 14385 case MVT::v4f32: return fastEmit_X86ISD_FMINS_SAE_MVT_v4f32_rr(RetVT, Op0, Op1); 14386 case MVT::v2f64: return fastEmit_X86ISD_FMINS_SAE_MVT_v2f64_rr(RetVT, Op0, Op1); 14387 default: return 0; 14388 } 14389} 14390 14391// FastEmit functions for X86ISD::FMIN_SAE. 14392 14393unsigned fastEmit_X86ISD_FMIN_SAE_MVT_v32f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14394 if (RetVT.SimpleTy != MVT::v32f16) 14395 return 0; 14396 if ((Subtarget->hasFP16())) { 14397 return fastEmitInst_rr(X86::VMINPHZrrb, &X86::VR512RegClass, Op0, Op1); 14398 } 14399 return 0; 14400} 14401 14402unsigned fastEmit_X86ISD_FMIN_SAE_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14403 if (RetVT.SimpleTy != MVT::v16f32) 14404 return 0; 14405 if ((Subtarget->hasAVX512())) { 14406 return fastEmitInst_rr(X86::VMINPSZrrb, &X86::VR512RegClass, Op0, Op1); 14407 } 14408 return 0; 14409} 14410 14411unsigned fastEmit_X86ISD_FMIN_SAE_MVT_v8f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14412 if (RetVT.SimpleTy != MVT::v8f64) 14413 return 0; 14414 if ((Subtarget->hasAVX512())) { 14415 return fastEmitInst_rr(X86::VMINPDZrrb, &X86::VR512RegClass, Op0, Op1); 14416 } 14417 return 0; 14418} 14419 14420unsigned fastEmit_X86ISD_FMIN_SAE_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 14421 switch (VT.SimpleTy) { 14422 case MVT::v32f16: return fastEmit_X86ISD_FMIN_SAE_MVT_v32f16_rr(RetVT, Op0, Op1); 14423 case MVT::v16f32: return fastEmit_X86ISD_FMIN_SAE_MVT_v16f32_rr(RetVT, Op0, Op1); 14424 case MVT::v8f64: return fastEmit_X86ISD_FMIN_SAE_MVT_v8f64_rr(RetVT, Op0, Op1); 14425 default: return 0; 14426 } 14427} 14428 14429// FastEmit functions for X86ISD::FMULS. 14430 14431unsigned fastEmit_X86ISD_FMULS_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14432 if (RetVT.SimpleTy != MVT::v8f16) 14433 return 0; 14434 if ((Subtarget->hasFP16())) { 14435 return fastEmitInst_rr(X86::VMULSHZrr_Int, &X86::VR128XRegClass, Op0, Op1); 14436 } 14437 return 0; 14438} 14439 14440unsigned fastEmit_X86ISD_FMULS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14441 if (RetVT.SimpleTy != MVT::v4f32) 14442 return 0; 14443 if ((Subtarget->hasAVX512())) { 14444 return fastEmitInst_rr(X86::VMULSSZrr_Int, &X86::VR128XRegClass, Op0, Op1); 14445 } 14446 return 0; 14447} 14448 14449unsigned fastEmit_X86ISD_FMULS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14450 if (RetVT.SimpleTy != MVT::v2f64) 14451 return 0; 14452 if ((Subtarget->hasAVX512())) { 14453 return fastEmitInst_rr(X86::VMULSDZrr_Int, &X86::VR128XRegClass, Op0, Op1); 14454 } 14455 return 0; 14456} 14457 14458unsigned fastEmit_X86ISD_FMULS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 14459 switch (VT.SimpleTy) { 14460 case MVT::v8f16: return fastEmit_X86ISD_FMULS_MVT_v8f16_rr(RetVT, Op0, Op1); 14461 case MVT::v4f32: return fastEmit_X86ISD_FMULS_MVT_v4f32_rr(RetVT, Op0, Op1); 14462 case MVT::v2f64: return fastEmit_X86ISD_FMULS_MVT_v2f64_rr(RetVT, Op0, Op1); 14463 default: return 0; 14464 } 14465} 14466 14467// FastEmit functions for X86ISD::FOR. 14468 14469unsigned fastEmit_X86ISD_FOR_MVT_f128_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14470 if (RetVT.SimpleTy != MVT::f128) 14471 return 0; 14472 if ((Subtarget->hasVLX())) { 14473 return fastEmitInst_rr(X86::VORPSZ128rr, &X86::VR128XRegClass, Op0, Op1); 14474 } 14475 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 14476 return fastEmitInst_rr(X86::VORPSrr, &X86::VR128RegClass, Op0, Op1); 14477 } 14478 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 14479 return fastEmitInst_rr(X86::ORPSrr, &X86::VR128RegClass, Op0, Op1); 14480 } 14481 return 0; 14482} 14483 14484unsigned fastEmit_X86ISD_FOR_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14485 if (RetVT.SimpleTy != MVT::v4f32) 14486 return 0; 14487 return fastEmitInst_rr(X86::ORPSrr, &X86::VR128RegClass, Op0, Op1); 14488} 14489 14490unsigned fastEmit_X86ISD_FOR_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 14491 switch (VT.SimpleTy) { 14492 case MVT::f128: return fastEmit_X86ISD_FOR_MVT_f128_rr(RetVT, Op0, Op1); 14493 case MVT::v4f32: return fastEmit_X86ISD_FOR_MVT_v4f32_rr(RetVT, Op0, Op1); 14494 default: return 0; 14495 } 14496} 14497 14498// FastEmit functions for X86ISD::FP80_ADD. 14499 14500unsigned fastEmit_X86ISD_FP80_ADD_MVT_f80_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14501 if (RetVT.SimpleTy != MVT::f80) 14502 return 0; 14503 return fastEmitInst_rr(X86::FP80_ADDr, &X86::RFP80RegClass, Op0, Op1); 14504} 14505 14506unsigned fastEmit_X86ISD_FP80_ADD_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 14507 switch (VT.SimpleTy) { 14508 case MVT::f80: return fastEmit_X86ISD_FP80_ADD_MVT_f80_rr(RetVT, Op0, Op1); 14509 default: return 0; 14510 } 14511} 14512 14513// FastEmit functions for X86ISD::FSQRTS. 14514 14515unsigned fastEmit_X86ISD_FSQRTS_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14516 if (RetVT.SimpleTy != MVT::v8f16) 14517 return 0; 14518 if ((Subtarget->hasFP16())) { 14519 return fastEmitInst_rr(X86::VSQRTSHZr_Int, &X86::VR128XRegClass, Op0, Op1); 14520 } 14521 return 0; 14522} 14523 14524unsigned fastEmit_X86ISD_FSQRTS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14525 if (RetVT.SimpleTy != MVT::v4f32) 14526 return 0; 14527 if ((Subtarget->hasAVX512())) { 14528 return fastEmitInst_rr(X86::VSQRTSSZr_Int, &X86::VR128XRegClass, Op0, Op1); 14529 } 14530 return 0; 14531} 14532 14533unsigned fastEmit_X86ISD_FSQRTS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14534 if (RetVT.SimpleTy != MVT::v2f64) 14535 return 0; 14536 if ((Subtarget->hasAVX512())) { 14537 return fastEmitInst_rr(X86::VSQRTSDZr_Int, &X86::VR128XRegClass, Op0, Op1); 14538 } 14539 return 0; 14540} 14541 14542unsigned fastEmit_X86ISD_FSQRTS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 14543 switch (VT.SimpleTy) { 14544 case MVT::v8f16: return fastEmit_X86ISD_FSQRTS_MVT_v8f16_rr(RetVT, Op0, Op1); 14545 case MVT::v4f32: return fastEmit_X86ISD_FSQRTS_MVT_v4f32_rr(RetVT, Op0, Op1); 14546 case MVT::v2f64: return fastEmit_X86ISD_FSQRTS_MVT_v2f64_rr(RetVT, Op0, Op1); 14547 default: return 0; 14548 } 14549} 14550 14551// FastEmit functions for X86ISD::FSUBS. 14552 14553unsigned fastEmit_X86ISD_FSUBS_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14554 if (RetVT.SimpleTy != MVT::v8f16) 14555 return 0; 14556 if ((Subtarget->hasFP16())) { 14557 return fastEmitInst_rr(X86::VSUBSHZrr_Int, &X86::VR128XRegClass, Op0, Op1); 14558 } 14559 return 0; 14560} 14561 14562unsigned fastEmit_X86ISD_FSUBS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14563 if (RetVT.SimpleTy != MVT::v4f32) 14564 return 0; 14565 if ((Subtarget->hasAVX512())) { 14566 return fastEmitInst_rr(X86::VSUBSSZrr_Int, &X86::VR128XRegClass, Op0, Op1); 14567 } 14568 return 0; 14569} 14570 14571unsigned fastEmit_X86ISD_FSUBS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14572 if (RetVT.SimpleTy != MVT::v2f64) 14573 return 0; 14574 if ((Subtarget->hasAVX512())) { 14575 return fastEmitInst_rr(X86::VSUBSDZrr_Int, &X86::VR128XRegClass, Op0, Op1); 14576 } 14577 return 0; 14578} 14579 14580unsigned fastEmit_X86ISD_FSUBS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 14581 switch (VT.SimpleTy) { 14582 case MVT::v8f16: return fastEmit_X86ISD_FSUBS_MVT_v8f16_rr(RetVT, Op0, Op1); 14583 case MVT::v4f32: return fastEmit_X86ISD_FSUBS_MVT_v4f32_rr(RetVT, Op0, Op1); 14584 case MVT::v2f64: return fastEmit_X86ISD_FSUBS_MVT_v2f64_rr(RetVT, Op0, Op1); 14585 default: return 0; 14586 } 14587} 14588 14589// FastEmit functions for X86ISD::FXOR. 14590 14591unsigned fastEmit_X86ISD_FXOR_MVT_f128_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14592 if (RetVT.SimpleTy != MVT::f128) 14593 return 0; 14594 if ((Subtarget->hasVLX())) { 14595 return fastEmitInst_rr(X86::VXORPSZ128rr, &X86::VR128XRegClass, Op0, Op1); 14596 } 14597 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 14598 return fastEmitInst_rr(X86::VXORPSrr, &X86::VR128RegClass, Op0, Op1); 14599 } 14600 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 14601 return fastEmitInst_rr(X86::XORPSrr, &X86::VR128RegClass, Op0, Op1); 14602 } 14603 return 0; 14604} 14605 14606unsigned fastEmit_X86ISD_FXOR_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14607 if (RetVT.SimpleTy != MVT::v4f32) 14608 return 0; 14609 return fastEmitInst_rr(X86::XORPSrr, &X86::VR128RegClass, Op0, Op1); 14610} 14611 14612unsigned fastEmit_X86ISD_FXOR_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 14613 switch (VT.SimpleTy) { 14614 case MVT::f128: return fastEmit_X86ISD_FXOR_MVT_f128_rr(RetVT, Op0, Op1); 14615 case MVT::v4f32: return fastEmit_X86ISD_FXOR_MVT_v4f32_rr(RetVT, Op0, Op1); 14616 default: return 0; 14617 } 14618} 14619 14620// FastEmit functions for X86ISD::GF2P8MULB. 14621 14622unsigned fastEmit_X86ISD_GF2P8MULB_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14623 if (RetVT.SimpleTy != MVT::v16i8) 14624 return 0; 14625 if ((Subtarget->hasGFNI()) && (Subtarget->hasVLX())) { 14626 return fastEmitInst_rr(X86::VGF2P8MULBZ128rr, &X86::VR128XRegClass, Op0, Op1); 14627 } 14628 if ((Subtarget->hasAVX()) && (Subtarget->hasGFNI()) && (!Subtarget->hasVLX())) { 14629 return fastEmitInst_rr(X86::VGF2P8MULBrr, &X86::VR128RegClass, Op0, Op1); 14630 } 14631 if ((Subtarget->hasGFNI()) && (Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 14632 return fastEmitInst_rr(X86::GF2P8MULBrr, &X86::VR128RegClass, Op0, Op1); 14633 } 14634 return 0; 14635} 14636 14637unsigned fastEmit_X86ISD_GF2P8MULB_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14638 if (RetVT.SimpleTy != MVT::v32i8) 14639 return 0; 14640 if ((Subtarget->hasGFNI()) && (Subtarget->hasVLX())) { 14641 return fastEmitInst_rr(X86::VGF2P8MULBZ256rr, &X86::VR256XRegClass, Op0, Op1); 14642 } 14643 if ((Subtarget->hasAVX()) && (Subtarget->hasGFNI()) && (!Subtarget->hasVLX())) { 14644 return fastEmitInst_rr(X86::VGF2P8MULBYrr, &X86::VR256RegClass, Op0, Op1); 14645 } 14646 return 0; 14647} 14648 14649unsigned fastEmit_X86ISD_GF2P8MULB_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14650 if (RetVT.SimpleTy != MVT::v64i8) 14651 return 0; 14652 if ((Subtarget->hasAVX512()) && (Subtarget->hasGFNI())) { 14653 return fastEmitInst_rr(X86::VGF2P8MULBZrr, &X86::VR512RegClass, Op0, Op1); 14654 } 14655 return 0; 14656} 14657 14658unsigned fastEmit_X86ISD_GF2P8MULB_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 14659 switch (VT.SimpleTy) { 14660 case MVT::v16i8: return fastEmit_X86ISD_GF2P8MULB_MVT_v16i8_rr(RetVT, Op0, Op1); 14661 case MVT::v32i8: return fastEmit_X86ISD_GF2P8MULB_MVT_v32i8_rr(RetVT, Op0, Op1); 14662 case MVT::v64i8: return fastEmit_X86ISD_GF2P8MULB_MVT_v64i8_rr(RetVT, Op0, Op1); 14663 default: return 0; 14664 } 14665} 14666 14667// FastEmit functions for X86ISD::HADD. 14668 14669unsigned fastEmit_X86ISD_HADD_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14670 if (RetVT.SimpleTy != MVT::v8i16) 14671 return 0; 14672 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) { 14673 return fastEmitInst_rr(X86::PHADDWrr, &X86::VR128RegClass, Op0, Op1); 14674 } 14675 if ((Subtarget->hasAVX())) { 14676 return fastEmitInst_rr(X86::VPHADDWrr, &X86::VR128RegClass, Op0, Op1); 14677 } 14678 return 0; 14679} 14680 14681unsigned fastEmit_X86ISD_HADD_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14682 if (RetVT.SimpleTy != MVT::v16i16) 14683 return 0; 14684 if ((Subtarget->hasAVX2())) { 14685 return fastEmitInst_rr(X86::VPHADDWYrr, &X86::VR256RegClass, Op0, Op1); 14686 } 14687 return 0; 14688} 14689 14690unsigned fastEmit_X86ISD_HADD_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14691 if (RetVT.SimpleTy != MVT::v4i32) 14692 return 0; 14693 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) { 14694 return fastEmitInst_rr(X86::PHADDDrr, &X86::VR128RegClass, Op0, Op1); 14695 } 14696 if ((Subtarget->hasAVX())) { 14697 return fastEmitInst_rr(X86::VPHADDDrr, &X86::VR128RegClass, Op0, Op1); 14698 } 14699 return 0; 14700} 14701 14702unsigned fastEmit_X86ISD_HADD_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14703 if (RetVT.SimpleTy != MVT::v8i32) 14704 return 0; 14705 if ((Subtarget->hasAVX2())) { 14706 return fastEmitInst_rr(X86::VPHADDDYrr, &X86::VR256RegClass, Op0, Op1); 14707 } 14708 return 0; 14709} 14710 14711unsigned fastEmit_X86ISD_HADD_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 14712 switch (VT.SimpleTy) { 14713 case MVT::v8i16: return fastEmit_X86ISD_HADD_MVT_v8i16_rr(RetVT, Op0, Op1); 14714 case MVT::v16i16: return fastEmit_X86ISD_HADD_MVT_v16i16_rr(RetVT, Op0, Op1); 14715 case MVT::v4i32: return fastEmit_X86ISD_HADD_MVT_v4i32_rr(RetVT, Op0, Op1); 14716 case MVT::v8i32: return fastEmit_X86ISD_HADD_MVT_v8i32_rr(RetVT, Op0, Op1); 14717 default: return 0; 14718 } 14719} 14720 14721// FastEmit functions for X86ISD::HSUB. 14722 14723unsigned fastEmit_X86ISD_HSUB_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14724 if (RetVT.SimpleTy != MVT::v8i16) 14725 return 0; 14726 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) { 14727 return fastEmitInst_rr(X86::PHSUBWrr, &X86::VR128RegClass, Op0, Op1); 14728 } 14729 if ((Subtarget->hasAVX())) { 14730 return fastEmitInst_rr(X86::VPHSUBWrr, &X86::VR128RegClass, Op0, Op1); 14731 } 14732 return 0; 14733} 14734 14735unsigned fastEmit_X86ISD_HSUB_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14736 if (RetVT.SimpleTy != MVT::v16i16) 14737 return 0; 14738 if ((Subtarget->hasAVX2())) { 14739 return fastEmitInst_rr(X86::VPHSUBWYrr, &X86::VR256RegClass, Op0, Op1); 14740 } 14741 return 0; 14742} 14743 14744unsigned fastEmit_X86ISD_HSUB_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14745 if (RetVT.SimpleTy != MVT::v4i32) 14746 return 0; 14747 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) { 14748 return fastEmitInst_rr(X86::PHSUBDrr, &X86::VR128RegClass, Op0, Op1); 14749 } 14750 if ((Subtarget->hasAVX())) { 14751 return fastEmitInst_rr(X86::VPHSUBDrr, &X86::VR128RegClass, Op0, Op1); 14752 } 14753 return 0; 14754} 14755 14756unsigned fastEmit_X86ISD_HSUB_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14757 if (RetVT.SimpleTy != MVT::v8i32) 14758 return 0; 14759 if ((Subtarget->hasAVX2())) { 14760 return fastEmitInst_rr(X86::VPHSUBDYrr, &X86::VR256RegClass, Op0, Op1); 14761 } 14762 return 0; 14763} 14764 14765unsigned fastEmit_X86ISD_HSUB_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 14766 switch (VT.SimpleTy) { 14767 case MVT::v8i16: return fastEmit_X86ISD_HSUB_MVT_v8i16_rr(RetVT, Op0, Op1); 14768 case MVT::v16i16: return fastEmit_X86ISD_HSUB_MVT_v16i16_rr(RetVT, Op0, Op1); 14769 case MVT::v4i32: return fastEmit_X86ISD_HSUB_MVT_v4i32_rr(RetVT, Op0, Op1); 14770 case MVT::v8i32: return fastEmit_X86ISD_HSUB_MVT_v8i32_rr(RetVT, Op0, Op1); 14771 default: return 0; 14772 } 14773} 14774 14775// FastEmit functions for X86ISD::KADD. 14776 14777unsigned fastEmit_X86ISD_KADD_MVT_v8i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14778 if (RetVT.SimpleTy != MVT::v8i1) 14779 return 0; 14780 if ((Subtarget->hasDQI())) { 14781 return fastEmitInst_rr(X86::KADDBrr, &X86::VK8RegClass, Op0, Op1); 14782 } 14783 return 0; 14784} 14785 14786unsigned fastEmit_X86ISD_KADD_MVT_v16i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14787 if (RetVT.SimpleTy != MVT::v16i1) 14788 return 0; 14789 if ((Subtarget->hasDQI())) { 14790 return fastEmitInst_rr(X86::KADDWrr, &X86::VK16RegClass, Op0, Op1); 14791 } 14792 return 0; 14793} 14794 14795unsigned fastEmit_X86ISD_KADD_MVT_v32i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14796 if (RetVT.SimpleTy != MVT::v32i1) 14797 return 0; 14798 if ((Subtarget->hasBWI())) { 14799 return fastEmitInst_rr(X86::KADDDrr, &X86::VK32RegClass, Op0, Op1); 14800 } 14801 return 0; 14802} 14803 14804unsigned fastEmit_X86ISD_KADD_MVT_v64i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14805 if (RetVT.SimpleTy != MVT::v64i1) 14806 return 0; 14807 if ((Subtarget->hasBWI())) { 14808 return fastEmitInst_rr(X86::KADDQrr, &X86::VK64RegClass, Op0, Op1); 14809 } 14810 return 0; 14811} 14812 14813unsigned fastEmit_X86ISD_KADD_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 14814 switch (VT.SimpleTy) { 14815 case MVT::v8i1: return fastEmit_X86ISD_KADD_MVT_v8i1_rr(RetVT, Op0, Op1); 14816 case MVT::v16i1: return fastEmit_X86ISD_KADD_MVT_v16i1_rr(RetVT, Op0, Op1); 14817 case MVT::v32i1: return fastEmit_X86ISD_KADD_MVT_v32i1_rr(RetVT, Op0, Op1); 14818 case MVT::v64i1: return fastEmit_X86ISD_KADD_MVT_v64i1_rr(RetVT, Op0, Op1); 14819 default: return 0; 14820 } 14821} 14822 14823// FastEmit functions for X86ISD::KORTEST. 14824 14825unsigned fastEmit_X86ISD_KORTEST_MVT_v8i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14826 if (RetVT.SimpleTy != MVT::i32) 14827 return 0; 14828 if ((Subtarget->hasDQI())) { 14829 return fastEmitInst_rr(X86::KORTESTBrr, &X86::VK8RegClass, Op0, Op1); 14830 } 14831 return 0; 14832} 14833 14834unsigned fastEmit_X86ISD_KORTEST_MVT_v16i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14835 if (RetVT.SimpleTy != MVT::i32) 14836 return 0; 14837 if ((Subtarget->hasAVX512())) { 14838 return fastEmitInst_rr(X86::KORTESTWrr, &X86::VK16RegClass, Op0, Op1); 14839 } 14840 return 0; 14841} 14842 14843unsigned fastEmit_X86ISD_KORTEST_MVT_v32i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14844 if (RetVT.SimpleTy != MVT::i32) 14845 return 0; 14846 if ((Subtarget->hasBWI())) { 14847 return fastEmitInst_rr(X86::KORTESTDrr, &X86::VK32RegClass, Op0, Op1); 14848 } 14849 return 0; 14850} 14851 14852unsigned fastEmit_X86ISD_KORTEST_MVT_v64i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14853 if (RetVT.SimpleTy != MVT::i32) 14854 return 0; 14855 if ((Subtarget->hasBWI())) { 14856 return fastEmitInst_rr(X86::KORTESTQrr, &X86::VK64RegClass, Op0, Op1); 14857 } 14858 return 0; 14859} 14860 14861unsigned fastEmit_X86ISD_KORTEST_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 14862 switch (VT.SimpleTy) { 14863 case MVT::v8i1: return fastEmit_X86ISD_KORTEST_MVT_v8i1_rr(RetVT, Op0, Op1); 14864 case MVT::v16i1: return fastEmit_X86ISD_KORTEST_MVT_v16i1_rr(RetVT, Op0, Op1); 14865 case MVT::v32i1: return fastEmit_X86ISD_KORTEST_MVT_v32i1_rr(RetVT, Op0, Op1); 14866 case MVT::v64i1: return fastEmit_X86ISD_KORTEST_MVT_v64i1_rr(RetVT, Op0, Op1); 14867 default: return 0; 14868 } 14869} 14870 14871// FastEmit functions for X86ISD::KTEST. 14872 14873unsigned fastEmit_X86ISD_KTEST_MVT_v8i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14874 if (RetVT.SimpleTy != MVT::i32) 14875 return 0; 14876 if ((Subtarget->hasDQI())) { 14877 return fastEmitInst_rr(X86::KTESTBrr, &X86::VK8RegClass, Op0, Op1); 14878 } 14879 return 0; 14880} 14881 14882unsigned fastEmit_X86ISD_KTEST_MVT_v16i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14883 if (RetVT.SimpleTy != MVT::i32) 14884 return 0; 14885 if ((Subtarget->hasDQI())) { 14886 return fastEmitInst_rr(X86::KTESTWrr, &X86::VK16RegClass, Op0, Op1); 14887 } 14888 return 0; 14889} 14890 14891unsigned fastEmit_X86ISD_KTEST_MVT_v32i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14892 if (RetVT.SimpleTy != MVT::i32) 14893 return 0; 14894 if ((Subtarget->hasBWI())) { 14895 return fastEmitInst_rr(X86::KTESTDrr, &X86::VK32RegClass, Op0, Op1); 14896 } 14897 return 0; 14898} 14899 14900unsigned fastEmit_X86ISD_KTEST_MVT_v64i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14901 if (RetVT.SimpleTy != MVT::i32) 14902 return 0; 14903 if ((Subtarget->hasBWI())) { 14904 return fastEmitInst_rr(X86::KTESTQrr, &X86::VK64RegClass, Op0, Op1); 14905 } 14906 return 0; 14907} 14908 14909unsigned fastEmit_X86ISD_KTEST_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 14910 switch (VT.SimpleTy) { 14911 case MVT::v8i1: return fastEmit_X86ISD_KTEST_MVT_v8i1_rr(RetVT, Op0, Op1); 14912 case MVT::v16i1: return fastEmit_X86ISD_KTEST_MVT_v16i1_rr(RetVT, Op0, Op1); 14913 case MVT::v32i1: return fastEmit_X86ISD_KTEST_MVT_v32i1_rr(RetVT, Op0, Op1); 14914 case MVT::v64i1: return fastEmit_X86ISD_KTEST_MVT_v64i1_rr(RetVT, Op0, Op1); 14915 default: return 0; 14916 } 14917} 14918 14919// FastEmit functions for X86ISD::MOVHLPS. 14920 14921unsigned fastEmit_X86ISD_MOVHLPS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14922 if (RetVT.SimpleTy != MVT::v4f32) 14923 return 0; 14924 if ((Subtarget->hasAVX512())) { 14925 return fastEmitInst_rr(X86::VMOVHLPSZrr, &X86::VR128XRegClass, Op0, Op1); 14926 } 14927 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 14928 return fastEmitInst_rr(X86::MOVHLPSrr, &X86::VR128RegClass, Op0, Op1); 14929 } 14930 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 14931 return fastEmitInst_rr(X86::VMOVHLPSrr, &X86::VR128RegClass, Op0, Op1); 14932 } 14933 return 0; 14934} 14935 14936unsigned fastEmit_X86ISD_MOVHLPS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 14937 switch (VT.SimpleTy) { 14938 case MVT::v4f32: return fastEmit_X86ISD_MOVHLPS_MVT_v4f32_rr(RetVT, Op0, Op1); 14939 default: return 0; 14940 } 14941} 14942 14943// FastEmit functions for X86ISD::MOVLHPS. 14944 14945unsigned fastEmit_X86ISD_MOVLHPS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14946 if (RetVT.SimpleTy != MVT::v4f32) 14947 return 0; 14948 if ((Subtarget->hasAVX512())) { 14949 return fastEmitInst_rr(X86::VMOVLHPSZrr, &X86::VR128XRegClass, Op0, Op1); 14950 } 14951 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 14952 return fastEmitInst_rr(X86::MOVLHPSrr, &X86::VR128RegClass, Op0, Op1); 14953 } 14954 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 14955 return fastEmitInst_rr(X86::VMOVLHPSrr, &X86::VR128RegClass, Op0, Op1); 14956 } 14957 return 0; 14958} 14959 14960unsigned fastEmit_X86ISD_MOVLHPS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 14961 switch (VT.SimpleTy) { 14962 case MVT::v4f32: return fastEmit_X86ISD_MOVLHPS_MVT_v4f32_rr(RetVT, Op0, Op1); 14963 default: return 0; 14964 } 14965} 14966 14967// FastEmit functions for X86ISD::MOVSD. 14968 14969unsigned fastEmit_X86ISD_MOVSD_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14970 if (RetVT.SimpleTy != MVT::v2f64) 14971 return 0; 14972 if ((Subtarget->hasAVX512()) && (shouldOptForSize(MF))) { 14973 return fastEmitInst_rr(X86::VMOVSDZrr, &X86::VR128XRegClass, Op0, Op1); 14974 } 14975 if ((shouldOptForSize(MF) || !Subtarget->hasSSE41()) && (Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 14976 return fastEmitInst_rr(X86::MOVSDrr, &X86::VR128RegClass, Op0, Op1); 14977 } 14978 if ((shouldOptForSize(MF)) && (Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 14979 return fastEmitInst_rr(X86::VMOVSDrr, &X86::VR128RegClass, Op0, Op1); 14980 } 14981 return 0; 14982} 14983 14984unsigned fastEmit_X86ISD_MOVSD_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 14985 switch (VT.SimpleTy) { 14986 case MVT::v2f64: return fastEmit_X86ISD_MOVSD_MVT_v2f64_rr(RetVT, Op0, Op1); 14987 default: return 0; 14988 } 14989} 14990 14991// FastEmit functions for X86ISD::MOVSH. 14992 14993unsigned fastEmit_X86ISD_MOVSH_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 14994 if (RetVT.SimpleTy != MVT::v8f16) 14995 return 0; 14996 if ((Subtarget->hasFP16())) { 14997 return fastEmitInst_rr(X86::VMOVSHZrr, &X86::VR128XRegClass, Op0, Op1); 14998 } 14999 return 0; 15000} 15001 15002unsigned fastEmit_X86ISD_MOVSH_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 15003 switch (VT.SimpleTy) { 15004 case MVT::v8f16: return fastEmit_X86ISD_MOVSH_MVT_v8f16_rr(RetVT, Op0, Op1); 15005 default: return 0; 15006 } 15007} 15008 15009// FastEmit functions for X86ISD::MOVSS. 15010 15011unsigned fastEmit_X86ISD_MOVSS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15012 if (RetVT.SimpleTy != MVT::v4f32) 15013 return 0; 15014 if ((Subtarget->hasAVX512()) && (shouldOptForSize(MF))) { 15015 return fastEmitInst_rr(X86::VMOVSSZrr, &X86::VR128XRegClass, Op0, Op1); 15016 } 15017 if ((shouldOptForSize(MF) || !Subtarget->hasSSE41()) && (Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 15018 return fastEmitInst_rr(X86::MOVSSrr, &X86::VR128RegClass, Op0, Op1); 15019 } 15020 if ((shouldOptForSize(MF)) && (Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 15021 return fastEmitInst_rr(X86::VMOVSSrr, &X86::VR128RegClass, Op0, Op1); 15022 } 15023 return 0; 15024} 15025 15026unsigned fastEmit_X86ISD_MOVSS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 15027 switch (VT.SimpleTy) { 15028 case MVT::v4f32: return fastEmit_X86ISD_MOVSS_MVT_v4f32_rr(RetVT, Op0, Op1); 15029 default: return 0; 15030 } 15031} 15032 15033// FastEmit functions for X86ISD::MULHRS. 15034 15035unsigned fastEmit_X86ISD_MULHRS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15036 if (RetVT.SimpleTy != MVT::v8i16) 15037 return 0; 15038 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 15039 return fastEmitInst_rr(X86::VPMULHRSWZ128rr, &X86::VR128XRegClass, Op0, Op1); 15040 } 15041 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) { 15042 return fastEmitInst_rr(X86::PMULHRSWrr, &X86::VR128RegClass, Op0, Op1); 15043 } 15044 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 15045 return fastEmitInst_rr(X86::VPMULHRSWrr, &X86::VR128RegClass, Op0, Op1); 15046 } 15047 return 0; 15048} 15049 15050unsigned fastEmit_X86ISD_MULHRS_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15051 if (RetVT.SimpleTy != MVT::v16i16) 15052 return 0; 15053 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 15054 return fastEmitInst_rr(X86::VPMULHRSWZ256rr, &X86::VR256XRegClass, Op0, Op1); 15055 } 15056 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 15057 return fastEmitInst_rr(X86::VPMULHRSWYrr, &X86::VR256RegClass, Op0, Op1); 15058 } 15059 return 0; 15060} 15061 15062unsigned fastEmit_X86ISD_MULHRS_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15063 if (RetVT.SimpleTy != MVT::v32i16) 15064 return 0; 15065 if ((Subtarget->hasBWI())) { 15066 return fastEmitInst_rr(X86::VPMULHRSWZrr, &X86::VR512RegClass, Op0, Op1); 15067 } 15068 return 0; 15069} 15070 15071unsigned fastEmit_X86ISD_MULHRS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 15072 switch (VT.SimpleTy) { 15073 case MVT::v8i16: return fastEmit_X86ISD_MULHRS_MVT_v8i16_rr(RetVT, Op0, Op1); 15074 case MVT::v16i16: return fastEmit_X86ISD_MULHRS_MVT_v16i16_rr(RetVT, Op0, Op1); 15075 case MVT::v32i16: return fastEmit_X86ISD_MULHRS_MVT_v32i16_rr(RetVT, Op0, Op1); 15076 default: return 0; 15077 } 15078} 15079 15080// FastEmit functions for X86ISD::MULTISHIFT. 15081 15082unsigned fastEmit_X86ISD_MULTISHIFT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15083 if (RetVT.SimpleTy != MVT::v16i8) 15084 return 0; 15085 if ((Subtarget->hasVBMI()) && (Subtarget->hasVLX())) { 15086 return fastEmitInst_rr(X86::VPMULTISHIFTQBZ128rr, &X86::VR128XRegClass, Op0, Op1); 15087 } 15088 return 0; 15089} 15090 15091unsigned fastEmit_X86ISD_MULTISHIFT_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15092 if (RetVT.SimpleTy != MVT::v32i8) 15093 return 0; 15094 if ((Subtarget->hasVBMI()) && (Subtarget->hasVLX())) { 15095 return fastEmitInst_rr(X86::VPMULTISHIFTQBZ256rr, &X86::VR256XRegClass, Op0, Op1); 15096 } 15097 return 0; 15098} 15099 15100unsigned fastEmit_X86ISD_MULTISHIFT_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15101 if (RetVT.SimpleTy != MVT::v64i8) 15102 return 0; 15103 if ((Subtarget->hasVBMI())) { 15104 return fastEmitInst_rr(X86::VPMULTISHIFTQBZrr, &X86::VR512RegClass, Op0, Op1); 15105 } 15106 return 0; 15107} 15108 15109unsigned fastEmit_X86ISD_MULTISHIFT_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 15110 switch (VT.SimpleTy) { 15111 case MVT::v16i8: return fastEmit_X86ISD_MULTISHIFT_MVT_v16i8_rr(RetVT, Op0, Op1); 15112 case MVT::v32i8: return fastEmit_X86ISD_MULTISHIFT_MVT_v32i8_rr(RetVT, Op0, Op1); 15113 case MVT::v64i8: return fastEmit_X86ISD_MULTISHIFT_MVT_v64i8_rr(RetVT, Op0, Op1); 15114 default: return 0; 15115 } 15116} 15117 15118// FastEmit functions for X86ISD::PACKSS. 15119 15120unsigned fastEmit_X86ISD_PACKSS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15121 if (RetVT.SimpleTy != MVT::v16i8) 15122 return 0; 15123 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 15124 return fastEmitInst_rr(X86::VPACKSSWBZ128rr, &X86::VR128XRegClass, Op0, Op1); 15125 } 15126 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 15127 return fastEmitInst_rr(X86::PACKSSWBrr, &X86::VR128RegClass, Op0, Op1); 15128 } 15129 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 15130 return fastEmitInst_rr(X86::VPACKSSWBrr, &X86::VR128RegClass, Op0, Op1); 15131 } 15132 return 0; 15133} 15134 15135unsigned fastEmit_X86ISD_PACKSS_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15136 if (RetVT.SimpleTy != MVT::v32i8) 15137 return 0; 15138 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 15139 return fastEmitInst_rr(X86::VPACKSSWBZ256rr, &X86::VR256XRegClass, Op0, Op1); 15140 } 15141 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 15142 return fastEmitInst_rr(X86::VPACKSSWBYrr, &X86::VR256RegClass, Op0, Op1); 15143 } 15144 return 0; 15145} 15146 15147unsigned fastEmit_X86ISD_PACKSS_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15148 if (RetVT.SimpleTy != MVT::v64i8) 15149 return 0; 15150 if ((Subtarget->hasBWI())) { 15151 return fastEmitInst_rr(X86::VPACKSSWBZrr, &X86::VR512RegClass, Op0, Op1); 15152 } 15153 return 0; 15154} 15155 15156unsigned fastEmit_X86ISD_PACKSS_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15157 if (RetVT.SimpleTy != MVT::v8i16) 15158 return 0; 15159 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 15160 return fastEmitInst_rr(X86::VPACKSSDWZ128rr, &X86::VR128XRegClass, Op0, Op1); 15161 } 15162 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 15163 return fastEmitInst_rr(X86::PACKSSDWrr, &X86::VR128RegClass, Op0, Op1); 15164 } 15165 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 15166 return fastEmitInst_rr(X86::VPACKSSDWrr, &X86::VR128RegClass, Op0, Op1); 15167 } 15168 return 0; 15169} 15170 15171unsigned fastEmit_X86ISD_PACKSS_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15172 if (RetVT.SimpleTy != MVT::v16i16) 15173 return 0; 15174 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 15175 return fastEmitInst_rr(X86::VPACKSSDWZ256rr, &X86::VR256XRegClass, Op0, Op1); 15176 } 15177 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 15178 return fastEmitInst_rr(X86::VPACKSSDWYrr, &X86::VR256RegClass, Op0, Op1); 15179 } 15180 return 0; 15181} 15182 15183unsigned fastEmit_X86ISD_PACKSS_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15184 if (RetVT.SimpleTy != MVT::v32i16) 15185 return 0; 15186 if ((Subtarget->hasBWI())) { 15187 return fastEmitInst_rr(X86::VPACKSSDWZrr, &X86::VR512RegClass, Op0, Op1); 15188 } 15189 return 0; 15190} 15191 15192unsigned fastEmit_X86ISD_PACKSS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 15193 switch (VT.SimpleTy) { 15194 case MVT::v8i16: return fastEmit_X86ISD_PACKSS_MVT_v8i16_rr(RetVT, Op0, Op1); 15195 case MVT::v16i16: return fastEmit_X86ISD_PACKSS_MVT_v16i16_rr(RetVT, Op0, Op1); 15196 case MVT::v32i16: return fastEmit_X86ISD_PACKSS_MVT_v32i16_rr(RetVT, Op0, Op1); 15197 case MVT::v4i32: return fastEmit_X86ISD_PACKSS_MVT_v4i32_rr(RetVT, Op0, Op1); 15198 case MVT::v8i32: return fastEmit_X86ISD_PACKSS_MVT_v8i32_rr(RetVT, Op0, Op1); 15199 case MVT::v16i32: return fastEmit_X86ISD_PACKSS_MVT_v16i32_rr(RetVT, Op0, Op1); 15200 default: return 0; 15201 } 15202} 15203 15204// FastEmit functions for X86ISD::PACKUS. 15205 15206unsigned fastEmit_X86ISD_PACKUS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15207 if (RetVT.SimpleTy != MVT::v16i8) 15208 return 0; 15209 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 15210 return fastEmitInst_rr(X86::VPACKUSWBZ128rr, &X86::VR128XRegClass, Op0, Op1); 15211 } 15212 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 15213 return fastEmitInst_rr(X86::PACKUSWBrr, &X86::VR128RegClass, Op0, Op1); 15214 } 15215 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 15216 return fastEmitInst_rr(X86::VPACKUSWBrr, &X86::VR128RegClass, Op0, Op1); 15217 } 15218 return 0; 15219} 15220 15221unsigned fastEmit_X86ISD_PACKUS_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15222 if (RetVT.SimpleTy != MVT::v32i8) 15223 return 0; 15224 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 15225 return fastEmitInst_rr(X86::VPACKUSWBZ256rr, &X86::VR256XRegClass, Op0, Op1); 15226 } 15227 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 15228 return fastEmitInst_rr(X86::VPACKUSWBYrr, &X86::VR256RegClass, Op0, Op1); 15229 } 15230 return 0; 15231} 15232 15233unsigned fastEmit_X86ISD_PACKUS_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15234 if (RetVT.SimpleTy != MVT::v64i8) 15235 return 0; 15236 if ((Subtarget->hasBWI())) { 15237 return fastEmitInst_rr(X86::VPACKUSWBZrr, &X86::VR512RegClass, Op0, Op1); 15238 } 15239 return 0; 15240} 15241 15242unsigned fastEmit_X86ISD_PACKUS_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15243 if (RetVT.SimpleTy != MVT::v8i16) 15244 return 0; 15245 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 15246 return fastEmitInst_rr(X86::VPACKUSDWZ128rr, &X86::VR128XRegClass, Op0, Op1); 15247 } 15248 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 15249 return fastEmitInst_rr(X86::PACKUSDWrr, &X86::VR128RegClass, Op0, Op1); 15250 } 15251 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 15252 return fastEmitInst_rr(X86::VPACKUSDWrr, &X86::VR128RegClass, Op0, Op1); 15253 } 15254 return 0; 15255} 15256 15257unsigned fastEmit_X86ISD_PACKUS_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15258 if (RetVT.SimpleTy != MVT::v16i16) 15259 return 0; 15260 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 15261 return fastEmitInst_rr(X86::VPACKUSDWZ256rr, &X86::VR256XRegClass, Op0, Op1); 15262 } 15263 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 15264 return fastEmitInst_rr(X86::VPACKUSDWYrr, &X86::VR256RegClass, Op0, Op1); 15265 } 15266 return 0; 15267} 15268 15269unsigned fastEmit_X86ISD_PACKUS_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15270 if (RetVT.SimpleTy != MVT::v32i16) 15271 return 0; 15272 if ((Subtarget->hasBWI())) { 15273 return fastEmitInst_rr(X86::VPACKUSDWZrr, &X86::VR512RegClass, Op0, Op1); 15274 } 15275 return 0; 15276} 15277 15278unsigned fastEmit_X86ISD_PACKUS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 15279 switch (VT.SimpleTy) { 15280 case MVT::v8i16: return fastEmit_X86ISD_PACKUS_MVT_v8i16_rr(RetVT, Op0, Op1); 15281 case MVT::v16i16: return fastEmit_X86ISD_PACKUS_MVT_v16i16_rr(RetVT, Op0, Op1); 15282 case MVT::v32i16: return fastEmit_X86ISD_PACKUS_MVT_v32i16_rr(RetVT, Op0, Op1); 15283 case MVT::v4i32: return fastEmit_X86ISD_PACKUS_MVT_v4i32_rr(RetVT, Op0, Op1); 15284 case MVT::v8i32: return fastEmit_X86ISD_PACKUS_MVT_v8i32_rr(RetVT, Op0, Op1); 15285 case MVT::v16i32: return fastEmit_X86ISD_PACKUS_MVT_v16i32_rr(RetVT, Op0, Op1); 15286 default: return 0; 15287 } 15288} 15289 15290// FastEmit functions for X86ISD::PCMPEQ. 15291 15292unsigned fastEmit_X86ISD_PCMPEQ_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15293 if (RetVT.SimpleTy != MVT::v16i8) 15294 return 0; 15295 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 15296 return fastEmitInst_rr(X86::PCMPEQBrr, &X86::VR128RegClass, Op0, Op1); 15297 } 15298 if ((Subtarget->hasAVX()) && (true)) { 15299 return fastEmitInst_rr(X86::VPCMPEQBrr, &X86::VR128RegClass, Op0, Op1); 15300 } 15301 return 0; 15302} 15303 15304unsigned fastEmit_X86ISD_PCMPEQ_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15305 if (RetVT.SimpleTy != MVT::v32i8) 15306 return 0; 15307 if ((Subtarget->hasAVX2()) && (true)) { 15308 return fastEmitInst_rr(X86::VPCMPEQBYrr, &X86::VR256RegClass, Op0, Op1); 15309 } 15310 return 0; 15311} 15312 15313unsigned fastEmit_X86ISD_PCMPEQ_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15314 if (RetVT.SimpleTy != MVT::v8i16) 15315 return 0; 15316 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 15317 return fastEmitInst_rr(X86::PCMPEQWrr, &X86::VR128RegClass, Op0, Op1); 15318 } 15319 if ((Subtarget->hasAVX()) && (true)) { 15320 return fastEmitInst_rr(X86::VPCMPEQWrr, &X86::VR128RegClass, Op0, Op1); 15321 } 15322 return 0; 15323} 15324 15325unsigned fastEmit_X86ISD_PCMPEQ_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15326 if (RetVT.SimpleTy != MVT::v16i16) 15327 return 0; 15328 if ((Subtarget->hasAVX2()) && (true)) { 15329 return fastEmitInst_rr(X86::VPCMPEQWYrr, &X86::VR256RegClass, Op0, Op1); 15330 } 15331 return 0; 15332} 15333 15334unsigned fastEmit_X86ISD_PCMPEQ_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15335 if (RetVT.SimpleTy != MVT::v4i32) 15336 return 0; 15337 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 15338 return fastEmitInst_rr(X86::PCMPEQDrr, &X86::VR128RegClass, Op0, Op1); 15339 } 15340 if ((Subtarget->hasAVX()) && (true)) { 15341 return fastEmitInst_rr(X86::VPCMPEQDrr, &X86::VR128RegClass, Op0, Op1); 15342 } 15343 return 0; 15344} 15345 15346unsigned fastEmit_X86ISD_PCMPEQ_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15347 if (RetVT.SimpleTy != MVT::v8i32) 15348 return 0; 15349 if ((Subtarget->hasAVX2()) && (true)) { 15350 return fastEmitInst_rr(X86::VPCMPEQDYrr, &X86::VR256RegClass, Op0, Op1); 15351 } 15352 return 0; 15353} 15354 15355unsigned fastEmit_X86ISD_PCMPEQ_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15356 if (RetVT.SimpleTy != MVT::v2i64) 15357 return 0; 15358 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 15359 return fastEmitInst_rr(X86::PCMPEQQrr, &X86::VR128RegClass, Op0, Op1); 15360 } 15361 if ((Subtarget->hasAVX())) { 15362 return fastEmitInst_rr(X86::VPCMPEQQrr, &X86::VR128RegClass, Op0, Op1); 15363 } 15364 return 0; 15365} 15366 15367unsigned fastEmit_X86ISD_PCMPEQ_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15368 if (RetVT.SimpleTy != MVT::v4i64) 15369 return 0; 15370 if ((Subtarget->hasAVX2())) { 15371 return fastEmitInst_rr(X86::VPCMPEQQYrr, &X86::VR256RegClass, Op0, Op1); 15372 } 15373 return 0; 15374} 15375 15376unsigned fastEmit_X86ISD_PCMPEQ_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 15377 switch (VT.SimpleTy) { 15378 case MVT::v16i8: return fastEmit_X86ISD_PCMPEQ_MVT_v16i8_rr(RetVT, Op0, Op1); 15379 case MVT::v32i8: return fastEmit_X86ISD_PCMPEQ_MVT_v32i8_rr(RetVT, Op0, Op1); 15380 case MVT::v8i16: return fastEmit_X86ISD_PCMPEQ_MVT_v8i16_rr(RetVT, Op0, Op1); 15381 case MVT::v16i16: return fastEmit_X86ISD_PCMPEQ_MVT_v16i16_rr(RetVT, Op0, Op1); 15382 case MVT::v4i32: return fastEmit_X86ISD_PCMPEQ_MVT_v4i32_rr(RetVT, Op0, Op1); 15383 case MVT::v8i32: return fastEmit_X86ISD_PCMPEQ_MVT_v8i32_rr(RetVT, Op0, Op1); 15384 case MVT::v2i64: return fastEmit_X86ISD_PCMPEQ_MVT_v2i64_rr(RetVT, Op0, Op1); 15385 case MVT::v4i64: return fastEmit_X86ISD_PCMPEQ_MVT_v4i64_rr(RetVT, Op0, Op1); 15386 default: return 0; 15387 } 15388} 15389 15390// FastEmit functions for X86ISD::PCMPGT. 15391 15392unsigned fastEmit_X86ISD_PCMPGT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15393 if (RetVT.SimpleTy != MVT::v16i8) 15394 return 0; 15395 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 15396 return fastEmitInst_rr(X86::PCMPGTBrr, &X86::VR128RegClass, Op0, Op1); 15397 } 15398 if ((Subtarget->hasAVX()) && (true)) { 15399 return fastEmitInst_rr(X86::VPCMPGTBrr, &X86::VR128RegClass, Op0, Op1); 15400 } 15401 return 0; 15402} 15403 15404unsigned fastEmit_X86ISD_PCMPGT_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15405 if (RetVT.SimpleTy != MVT::v32i8) 15406 return 0; 15407 if ((Subtarget->hasAVX2()) && (true)) { 15408 return fastEmitInst_rr(X86::VPCMPGTBYrr, &X86::VR256RegClass, Op0, Op1); 15409 } 15410 return 0; 15411} 15412 15413unsigned fastEmit_X86ISD_PCMPGT_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15414 if (RetVT.SimpleTy != MVT::v8i16) 15415 return 0; 15416 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 15417 return fastEmitInst_rr(X86::PCMPGTWrr, &X86::VR128RegClass, Op0, Op1); 15418 } 15419 if ((Subtarget->hasAVX()) && (true)) { 15420 return fastEmitInst_rr(X86::VPCMPGTWrr, &X86::VR128RegClass, Op0, Op1); 15421 } 15422 return 0; 15423} 15424 15425unsigned fastEmit_X86ISD_PCMPGT_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15426 if (RetVT.SimpleTy != MVT::v16i16) 15427 return 0; 15428 if ((Subtarget->hasAVX2()) && (true)) { 15429 return fastEmitInst_rr(X86::VPCMPGTWYrr, &X86::VR256RegClass, Op0, Op1); 15430 } 15431 return 0; 15432} 15433 15434unsigned fastEmit_X86ISD_PCMPGT_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15435 if (RetVT.SimpleTy != MVT::v4i32) 15436 return 0; 15437 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 15438 return fastEmitInst_rr(X86::PCMPGTDrr, &X86::VR128RegClass, Op0, Op1); 15439 } 15440 if ((Subtarget->hasAVX()) && (true)) { 15441 return fastEmitInst_rr(X86::VPCMPGTDrr, &X86::VR128RegClass, Op0, Op1); 15442 } 15443 return 0; 15444} 15445 15446unsigned fastEmit_X86ISD_PCMPGT_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15447 if (RetVT.SimpleTy != MVT::v8i32) 15448 return 0; 15449 if ((Subtarget->hasAVX2()) && (true)) { 15450 return fastEmitInst_rr(X86::VPCMPGTDYrr, &X86::VR256RegClass, Op0, Op1); 15451 } 15452 return 0; 15453} 15454 15455unsigned fastEmit_X86ISD_PCMPGT_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15456 if (RetVT.SimpleTy != MVT::v2i64) 15457 return 0; 15458 if ((Subtarget->hasSSE42() && !Subtarget->hasAVX())) { 15459 return fastEmitInst_rr(X86::PCMPGTQrr, &X86::VR128RegClass, Op0, Op1); 15460 } 15461 if ((Subtarget->hasAVX())) { 15462 return fastEmitInst_rr(X86::VPCMPGTQrr, &X86::VR128RegClass, Op0, Op1); 15463 } 15464 return 0; 15465} 15466 15467unsigned fastEmit_X86ISD_PCMPGT_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15468 if (RetVT.SimpleTy != MVT::v4i64) 15469 return 0; 15470 if ((Subtarget->hasAVX2())) { 15471 return fastEmitInst_rr(X86::VPCMPGTQYrr, &X86::VR256RegClass, Op0, Op1); 15472 } 15473 return 0; 15474} 15475 15476unsigned fastEmit_X86ISD_PCMPGT_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 15477 switch (VT.SimpleTy) { 15478 case MVT::v16i8: return fastEmit_X86ISD_PCMPGT_MVT_v16i8_rr(RetVT, Op0, Op1); 15479 case MVT::v32i8: return fastEmit_X86ISD_PCMPGT_MVT_v32i8_rr(RetVT, Op0, Op1); 15480 case MVT::v8i16: return fastEmit_X86ISD_PCMPGT_MVT_v8i16_rr(RetVT, Op0, Op1); 15481 case MVT::v16i16: return fastEmit_X86ISD_PCMPGT_MVT_v16i16_rr(RetVT, Op0, Op1); 15482 case MVT::v4i32: return fastEmit_X86ISD_PCMPGT_MVT_v4i32_rr(RetVT, Op0, Op1); 15483 case MVT::v8i32: return fastEmit_X86ISD_PCMPGT_MVT_v8i32_rr(RetVT, Op0, Op1); 15484 case MVT::v2i64: return fastEmit_X86ISD_PCMPGT_MVT_v2i64_rr(RetVT, Op0, Op1); 15485 case MVT::v4i64: return fastEmit_X86ISD_PCMPGT_MVT_v4i64_rr(RetVT, Op0, Op1); 15486 default: return 0; 15487 } 15488} 15489 15490// FastEmit functions for X86ISD::PDEP. 15491 15492unsigned fastEmit_X86ISD_PDEP_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15493 if (RetVT.SimpleTy != MVT::i32) 15494 return 0; 15495 if ((Subtarget->hasBMI2())) { 15496 return fastEmitInst_rr(X86::PDEP32rr, &X86::GR32RegClass, Op0, Op1); 15497 } 15498 return 0; 15499} 15500 15501unsigned fastEmit_X86ISD_PDEP_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15502 if (RetVT.SimpleTy != MVT::i64) 15503 return 0; 15504 if ((Subtarget->hasBMI2())) { 15505 return fastEmitInst_rr(X86::PDEP64rr, &X86::GR64RegClass, Op0, Op1); 15506 } 15507 return 0; 15508} 15509 15510unsigned fastEmit_X86ISD_PDEP_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 15511 switch (VT.SimpleTy) { 15512 case MVT::i32: return fastEmit_X86ISD_PDEP_MVT_i32_rr(RetVT, Op0, Op1); 15513 case MVT::i64: return fastEmit_X86ISD_PDEP_MVT_i64_rr(RetVT, Op0, Op1); 15514 default: return 0; 15515 } 15516} 15517 15518// FastEmit functions for X86ISD::PEXT. 15519 15520unsigned fastEmit_X86ISD_PEXT_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15521 if (RetVT.SimpleTy != MVT::i32) 15522 return 0; 15523 if ((Subtarget->hasBMI2())) { 15524 return fastEmitInst_rr(X86::PEXT32rr, &X86::GR32RegClass, Op0, Op1); 15525 } 15526 return 0; 15527} 15528 15529unsigned fastEmit_X86ISD_PEXT_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15530 if (RetVT.SimpleTy != MVT::i64) 15531 return 0; 15532 if ((Subtarget->hasBMI2())) { 15533 return fastEmitInst_rr(X86::PEXT64rr, &X86::GR64RegClass, Op0, Op1); 15534 } 15535 return 0; 15536} 15537 15538unsigned fastEmit_X86ISD_PEXT_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 15539 switch (VT.SimpleTy) { 15540 case MVT::i32: return fastEmit_X86ISD_PEXT_MVT_i32_rr(RetVT, Op0, Op1); 15541 case MVT::i64: return fastEmit_X86ISD_PEXT_MVT_i64_rr(RetVT, Op0, Op1); 15542 default: return 0; 15543 } 15544} 15545 15546// FastEmit functions for X86ISD::PMULDQ. 15547 15548unsigned fastEmit_X86ISD_PMULDQ_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15549 if (RetVT.SimpleTy != MVT::v2i64) 15550 return 0; 15551 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 15552 return fastEmitInst_rr(X86::VPMULDQZ128rr, &X86::VR128XRegClass, Op0, Op1); 15553 } 15554 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 15555 return fastEmitInst_rr(X86::PMULDQrr, &X86::VR128RegClass, Op0, Op1); 15556 } 15557 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 15558 return fastEmitInst_rr(X86::VPMULDQrr, &X86::VR128RegClass, Op0, Op1); 15559 } 15560 return 0; 15561} 15562 15563unsigned fastEmit_X86ISD_PMULDQ_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15564 if (RetVT.SimpleTy != MVT::v4i64) 15565 return 0; 15566 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 15567 return fastEmitInst_rr(X86::VPMULDQZ256rr, &X86::VR256XRegClass, Op0, Op1); 15568 } 15569 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 15570 return fastEmitInst_rr(X86::VPMULDQYrr, &X86::VR256RegClass, Op0, Op1); 15571 } 15572 return 0; 15573} 15574 15575unsigned fastEmit_X86ISD_PMULDQ_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15576 if (RetVT.SimpleTy != MVT::v8i64) 15577 return 0; 15578 if ((Subtarget->hasAVX512())) { 15579 return fastEmitInst_rr(X86::VPMULDQZrr, &X86::VR512RegClass, Op0, Op1); 15580 } 15581 return 0; 15582} 15583 15584unsigned fastEmit_X86ISD_PMULDQ_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 15585 switch (VT.SimpleTy) { 15586 case MVT::v2i64: return fastEmit_X86ISD_PMULDQ_MVT_v2i64_rr(RetVT, Op0, Op1); 15587 case MVT::v4i64: return fastEmit_X86ISD_PMULDQ_MVT_v4i64_rr(RetVT, Op0, Op1); 15588 case MVT::v8i64: return fastEmit_X86ISD_PMULDQ_MVT_v8i64_rr(RetVT, Op0, Op1); 15589 default: return 0; 15590 } 15591} 15592 15593// FastEmit functions for X86ISD::PMULUDQ. 15594 15595unsigned fastEmit_X86ISD_PMULUDQ_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15596 if (RetVT.SimpleTy != MVT::v2i64) 15597 return 0; 15598 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 15599 return fastEmitInst_rr(X86::VPMULUDQZ128rr, &X86::VR128XRegClass, Op0, Op1); 15600 } 15601 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 15602 return fastEmitInst_rr(X86::PMULUDQrr, &X86::VR128RegClass, Op0, Op1); 15603 } 15604 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 15605 return fastEmitInst_rr(X86::VPMULUDQrr, &X86::VR128RegClass, Op0, Op1); 15606 } 15607 return 0; 15608} 15609 15610unsigned fastEmit_X86ISD_PMULUDQ_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15611 if (RetVT.SimpleTy != MVT::v4i64) 15612 return 0; 15613 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 15614 return fastEmitInst_rr(X86::VPMULUDQZ256rr, &X86::VR256XRegClass, Op0, Op1); 15615 } 15616 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 15617 return fastEmitInst_rr(X86::VPMULUDQYrr, &X86::VR256RegClass, Op0, Op1); 15618 } 15619 return 0; 15620} 15621 15622unsigned fastEmit_X86ISD_PMULUDQ_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15623 if (RetVT.SimpleTy != MVT::v8i64) 15624 return 0; 15625 if ((Subtarget->hasAVX512())) { 15626 return fastEmitInst_rr(X86::VPMULUDQZrr, &X86::VR512RegClass, Op0, Op1); 15627 } 15628 return 0; 15629} 15630 15631unsigned fastEmit_X86ISD_PMULUDQ_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 15632 switch (VT.SimpleTy) { 15633 case MVT::v2i64: return fastEmit_X86ISD_PMULUDQ_MVT_v2i64_rr(RetVT, Op0, Op1); 15634 case MVT::v4i64: return fastEmit_X86ISD_PMULUDQ_MVT_v4i64_rr(RetVT, Op0, Op1); 15635 case MVT::v8i64: return fastEmit_X86ISD_PMULUDQ_MVT_v8i64_rr(RetVT, Op0, Op1); 15636 default: return 0; 15637 } 15638} 15639 15640// FastEmit functions for X86ISD::PSADBW. 15641 15642unsigned fastEmit_X86ISD_PSADBW_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15643 if (RetVT.SimpleTy != MVT::v2i64) 15644 return 0; 15645 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 15646 return fastEmitInst_rr(X86::VPSADBWZ128rr, &X86::VR128XRegClass, Op0, Op1); 15647 } 15648 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 15649 return fastEmitInst_rr(X86::PSADBWrr, &X86::VR128RegClass, Op0, Op1); 15650 } 15651 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 15652 return fastEmitInst_rr(X86::VPSADBWrr, &X86::VR128RegClass, Op0, Op1); 15653 } 15654 return 0; 15655} 15656 15657unsigned fastEmit_X86ISD_PSADBW_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15658 if (RetVT.SimpleTy != MVT::v4i64) 15659 return 0; 15660 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 15661 return fastEmitInst_rr(X86::VPSADBWZ256rr, &X86::VR256XRegClass, Op0, Op1); 15662 } 15663 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 15664 return fastEmitInst_rr(X86::VPSADBWYrr, &X86::VR256RegClass, Op0, Op1); 15665 } 15666 return 0; 15667} 15668 15669unsigned fastEmit_X86ISD_PSADBW_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15670 if (RetVT.SimpleTy != MVT::v8i64) 15671 return 0; 15672 if ((Subtarget->hasBWI())) { 15673 return fastEmitInst_rr(X86::VPSADBWZrr, &X86::VR512RegClass, Op0, Op1); 15674 } 15675 return 0; 15676} 15677 15678unsigned fastEmit_X86ISD_PSADBW_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 15679 switch (VT.SimpleTy) { 15680 case MVT::v16i8: return fastEmit_X86ISD_PSADBW_MVT_v16i8_rr(RetVT, Op0, Op1); 15681 case MVT::v32i8: return fastEmit_X86ISD_PSADBW_MVT_v32i8_rr(RetVT, Op0, Op1); 15682 case MVT::v64i8: return fastEmit_X86ISD_PSADBW_MVT_v64i8_rr(RetVT, Op0, Op1); 15683 default: return 0; 15684 } 15685} 15686 15687// FastEmit functions for X86ISD::PSHUFB. 15688 15689unsigned fastEmit_X86ISD_PSHUFB_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15690 if (RetVT.SimpleTy != MVT::v16i8) 15691 return 0; 15692 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 15693 return fastEmitInst_rr(X86::VPSHUFBZ128rr, &X86::VR128XRegClass, Op0, Op1); 15694 } 15695 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) { 15696 return fastEmitInst_rr(X86::PSHUFBrr, &X86::VR128RegClass, Op0, Op1); 15697 } 15698 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 15699 return fastEmitInst_rr(X86::VPSHUFBrr, &X86::VR128RegClass, Op0, Op1); 15700 } 15701 return 0; 15702} 15703 15704unsigned fastEmit_X86ISD_PSHUFB_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15705 if (RetVT.SimpleTy != MVT::v32i8) 15706 return 0; 15707 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 15708 return fastEmitInst_rr(X86::VPSHUFBZ256rr, &X86::VR256XRegClass, Op0, Op1); 15709 } 15710 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 15711 return fastEmitInst_rr(X86::VPSHUFBYrr, &X86::VR256RegClass, Op0, Op1); 15712 } 15713 return 0; 15714} 15715 15716unsigned fastEmit_X86ISD_PSHUFB_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15717 if (RetVT.SimpleTy != MVT::v64i8) 15718 return 0; 15719 if ((Subtarget->hasBWI())) { 15720 return fastEmitInst_rr(X86::VPSHUFBZrr, &X86::VR512RegClass, Op0, Op1); 15721 } 15722 return 0; 15723} 15724 15725unsigned fastEmit_X86ISD_PSHUFB_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 15726 switch (VT.SimpleTy) { 15727 case MVT::v16i8: return fastEmit_X86ISD_PSHUFB_MVT_v16i8_rr(RetVT, Op0, Op1); 15728 case MVT::v32i8: return fastEmit_X86ISD_PSHUFB_MVT_v32i8_rr(RetVT, Op0, Op1); 15729 case MVT::v64i8: return fastEmit_X86ISD_PSHUFB_MVT_v64i8_rr(RetVT, Op0, Op1); 15730 default: return 0; 15731 } 15732} 15733 15734// FastEmit functions for X86ISD::PTEST. 15735 15736unsigned fastEmit_X86ISD_PTEST_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15737 if (RetVT.SimpleTy != MVT::i32) 15738 return 0; 15739 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 15740 return fastEmitInst_rr(X86::PTESTrr, &X86::VR128RegClass, Op0, Op1); 15741 } 15742 if ((Subtarget->hasAVX())) { 15743 return fastEmitInst_rr(X86::VPTESTrr, &X86::VR128RegClass, Op0, Op1); 15744 } 15745 return 0; 15746} 15747 15748unsigned fastEmit_X86ISD_PTEST_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15749 if (RetVT.SimpleTy != MVT::i32) 15750 return 0; 15751 if ((Subtarget->hasAVX())) { 15752 return fastEmitInst_rr(X86::VPTESTYrr, &X86::VR256RegClass, Op0, Op1); 15753 } 15754 return 0; 15755} 15756 15757unsigned fastEmit_X86ISD_PTEST_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 15758 switch (VT.SimpleTy) { 15759 case MVT::v2i64: return fastEmit_X86ISD_PTEST_MVT_v2i64_rr(RetVT, Op0, Op1); 15760 case MVT::v4i64: return fastEmit_X86ISD_PTEST_MVT_v4i64_rr(RetVT, Op0, Op1); 15761 default: return 0; 15762 } 15763} 15764 15765// FastEmit functions for X86ISD::RCP14S. 15766 15767unsigned fastEmit_X86ISD_RCP14S_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15768 if (RetVT.SimpleTy != MVT::v8f16) 15769 return 0; 15770 if ((Subtarget->hasFP16())) { 15771 return fastEmitInst_rr(X86::VRCPSHZrr, &X86::VR128XRegClass, Op0, Op1); 15772 } 15773 return 0; 15774} 15775 15776unsigned fastEmit_X86ISD_RCP14S_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15777 if (RetVT.SimpleTy != MVT::v4f32) 15778 return 0; 15779 if ((Subtarget->hasAVX512())) { 15780 return fastEmitInst_rr(X86::VRCP14SSZrr, &X86::VR128XRegClass, Op0, Op1); 15781 } 15782 return 0; 15783} 15784 15785unsigned fastEmit_X86ISD_RCP14S_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15786 if (RetVT.SimpleTy != MVT::v2f64) 15787 return 0; 15788 if ((Subtarget->hasAVX512())) { 15789 return fastEmitInst_rr(X86::VRCP14SDZrr, &X86::VR128XRegClass, Op0, Op1); 15790 } 15791 return 0; 15792} 15793 15794unsigned fastEmit_X86ISD_RCP14S_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 15795 switch (VT.SimpleTy) { 15796 case MVT::v8f16: return fastEmit_X86ISD_RCP14S_MVT_v8f16_rr(RetVT, Op0, Op1); 15797 case MVT::v4f32: return fastEmit_X86ISD_RCP14S_MVT_v4f32_rr(RetVT, Op0, Op1); 15798 case MVT::v2f64: return fastEmit_X86ISD_RCP14S_MVT_v2f64_rr(RetVT, Op0, Op1); 15799 default: return 0; 15800 } 15801} 15802 15803// FastEmit functions for X86ISD::RCP28S. 15804 15805unsigned fastEmit_X86ISD_RCP28S_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15806 if (RetVT.SimpleTy != MVT::v4f32) 15807 return 0; 15808 if ((Subtarget->hasERI())) { 15809 return fastEmitInst_rr(X86::VRCP28SSZr, &X86::VR128XRegClass, Op0, Op1); 15810 } 15811 return 0; 15812} 15813 15814unsigned fastEmit_X86ISD_RCP28S_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15815 if (RetVT.SimpleTy != MVT::v2f64) 15816 return 0; 15817 if ((Subtarget->hasERI())) { 15818 return fastEmitInst_rr(X86::VRCP28SDZr, &X86::VR128XRegClass, Op0, Op1); 15819 } 15820 return 0; 15821} 15822 15823unsigned fastEmit_X86ISD_RCP28S_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 15824 switch (VT.SimpleTy) { 15825 case MVT::v4f32: return fastEmit_X86ISD_RCP28S_MVT_v4f32_rr(RetVT, Op0, Op1); 15826 case MVT::v2f64: return fastEmit_X86ISD_RCP28S_MVT_v2f64_rr(RetVT, Op0, Op1); 15827 default: return 0; 15828 } 15829} 15830 15831// FastEmit functions for X86ISD::RCP28S_SAE. 15832 15833unsigned fastEmit_X86ISD_RCP28S_SAE_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15834 if (RetVT.SimpleTy != MVT::v4f32) 15835 return 0; 15836 if ((Subtarget->hasERI())) { 15837 return fastEmitInst_rr(X86::VRCP28SSZrb, &X86::VR128XRegClass, Op0, Op1); 15838 } 15839 return 0; 15840} 15841 15842unsigned fastEmit_X86ISD_RCP28S_SAE_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15843 if (RetVT.SimpleTy != MVT::v2f64) 15844 return 0; 15845 if ((Subtarget->hasERI())) { 15846 return fastEmitInst_rr(X86::VRCP28SDZrb, &X86::VR128XRegClass, Op0, Op1); 15847 } 15848 return 0; 15849} 15850 15851unsigned fastEmit_X86ISD_RCP28S_SAE_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 15852 switch (VT.SimpleTy) { 15853 case MVT::v4f32: return fastEmit_X86ISD_RCP28S_SAE_MVT_v4f32_rr(RetVT, Op0, Op1); 15854 case MVT::v2f64: return fastEmit_X86ISD_RCP28S_SAE_MVT_v2f64_rr(RetVT, Op0, Op1); 15855 default: return 0; 15856 } 15857} 15858 15859// FastEmit functions for X86ISD::RSQRT14S. 15860 15861unsigned fastEmit_X86ISD_RSQRT14S_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15862 if (RetVT.SimpleTy != MVT::v8f16) 15863 return 0; 15864 if ((Subtarget->hasFP16())) { 15865 return fastEmitInst_rr(X86::VRSQRTSHZrr, &X86::VR128XRegClass, Op0, Op1); 15866 } 15867 return 0; 15868} 15869 15870unsigned fastEmit_X86ISD_RSQRT14S_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15871 if (RetVT.SimpleTy != MVT::v4f32) 15872 return 0; 15873 if ((Subtarget->hasAVX512())) { 15874 return fastEmitInst_rr(X86::VRSQRT14SSZrr, &X86::VR128XRegClass, Op0, Op1); 15875 } 15876 return 0; 15877} 15878 15879unsigned fastEmit_X86ISD_RSQRT14S_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15880 if (RetVT.SimpleTy != MVT::v2f64) 15881 return 0; 15882 if ((Subtarget->hasAVX512())) { 15883 return fastEmitInst_rr(X86::VRSQRT14SDZrr, &X86::VR128XRegClass, Op0, Op1); 15884 } 15885 return 0; 15886} 15887 15888unsigned fastEmit_X86ISD_RSQRT14S_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 15889 switch (VT.SimpleTy) { 15890 case MVT::v8f16: return fastEmit_X86ISD_RSQRT14S_MVT_v8f16_rr(RetVT, Op0, Op1); 15891 case MVT::v4f32: return fastEmit_X86ISD_RSQRT14S_MVT_v4f32_rr(RetVT, Op0, Op1); 15892 case MVT::v2f64: return fastEmit_X86ISD_RSQRT14S_MVT_v2f64_rr(RetVT, Op0, Op1); 15893 default: return 0; 15894 } 15895} 15896 15897// FastEmit functions for X86ISD::RSQRT28S. 15898 15899unsigned fastEmit_X86ISD_RSQRT28S_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15900 if (RetVT.SimpleTy != MVT::v4f32) 15901 return 0; 15902 if ((Subtarget->hasERI())) { 15903 return fastEmitInst_rr(X86::VRSQRT28SSZr, &X86::VR128XRegClass, Op0, Op1); 15904 } 15905 return 0; 15906} 15907 15908unsigned fastEmit_X86ISD_RSQRT28S_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15909 if (RetVT.SimpleTy != MVT::v2f64) 15910 return 0; 15911 if ((Subtarget->hasERI())) { 15912 return fastEmitInst_rr(X86::VRSQRT28SDZr, &X86::VR128XRegClass, Op0, Op1); 15913 } 15914 return 0; 15915} 15916 15917unsigned fastEmit_X86ISD_RSQRT28S_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 15918 switch (VT.SimpleTy) { 15919 case MVT::v4f32: return fastEmit_X86ISD_RSQRT28S_MVT_v4f32_rr(RetVT, Op0, Op1); 15920 case MVT::v2f64: return fastEmit_X86ISD_RSQRT28S_MVT_v2f64_rr(RetVT, Op0, Op1); 15921 default: return 0; 15922 } 15923} 15924 15925// FastEmit functions for X86ISD::RSQRT28S_SAE. 15926 15927unsigned fastEmit_X86ISD_RSQRT28S_SAE_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15928 if (RetVT.SimpleTy != MVT::v4f32) 15929 return 0; 15930 if ((Subtarget->hasERI())) { 15931 return fastEmitInst_rr(X86::VRSQRT28SSZrb, &X86::VR128XRegClass, Op0, Op1); 15932 } 15933 return 0; 15934} 15935 15936unsigned fastEmit_X86ISD_RSQRT28S_SAE_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15937 if (RetVT.SimpleTy != MVT::v2f64) 15938 return 0; 15939 if ((Subtarget->hasERI())) { 15940 return fastEmitInst_rr(X86::VRSQRT28SDZrb, &X86::VR128XRegClass, Op0, Op1); 15941 } 15942 return 0; 15943} 15944 15945unsigned fastEmit_X86ISD_RSQRT28S_SAE_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 15946 switch (VT.SimpleTy) { 15947 case MVT::v4f32: return fastEmit_X86ISD_RSQRT28S_SAE_MVT_v4f32_rr(RetVT, Op0, Op1); 15948 case MVT::v2f64: return fastEmit_X86ISD_RSQRT28S_SAE_MVT_v2f64_rr(RetVT, Op0, Op1); 15949 default: return 0; 15950 } 15951} 15952 15953// FastEmit functions for X86ISD::SCALEF. 15954 15955unsigned fastEmit_X86ISD_SCALEF_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15956 if (RetVT.SimpleTy != MVT::v8f16) 15957 return 0; 15958 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 15959 return fastEmitInst_rr(X86::VSCALEFPHZ128rr, &X86::VR128XRegClass, Op0, Op1); 15960 } 15961 return 0; 15962} 15963 15964unsigned fastEmit_X86ISD_SCALEF_MVT_v16f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15965 if (RetVT.SimpleTy != MVT::v16f16) 15966 return 0; 15967 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 15968 return fastEmitInst_rr(X86::VSCALEFPHZ256rr, &X86::VR256XRegClass, Op0, Op1); 15969 } 15970 return 0; 15971} 15972 15973unsigned fastEmit_X86ISD_SCALEF_MVT_v32f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15974 if (RetVT.SimpleTy != MVT::v32f16) 15975 return 0; 15976 if ((Subtarget->hasFP16())) { 15977 return fastEmitInst_rr(X86::VSCALEFPHZrr, &X86::VR512RegClass, Op0, Op1); 15978 } 15979 return 0; 15980} 15981 15982unsigned fastEmit_X86ISD_SCALEF_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15983 if (RetVT.SimpleTy != MVT::v4f32) 15984 return 0; 15985 if ((Subtarget->hasVLX())) { 15986 return fastEmitInst_rr(X86::VSCALEFPSZ128rr, &X86::VR128XRegClass, Op0, Op1); 15987 } 15988 return 0; 15989} 15990 15991unsigned fastEmit_X86ISD_SCALEF_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 15992 if (RetVT.SimpleTy != MVT::v8f32) 15993 return 0; 15994 if ((Subtarget->hasVLX())) { 15995 return fastEmitInst_rr(X86::VSCALEFPSZ256rr, &X86::VR256XRegClass, Op0, Op1); 15996 } 15997 return 0; 15998} 15999 16000unsigned fastEmit_X86ISD_SCALEF_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16001 if (RetVT.SimpleTy != MVT::v16f32) 16002 return 0; 16003 if ((Subtarget->hasAVX512())) { 16004 return fastEmitInst_rr(X86::VSCALEFPSZrr, &X86::VR512RegClass, Op0, Op1); 16005 } 16006 return 0; 16007} 16008 16009unsigned fastEmit_X86ISD_SCALEF_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16010 if (RetVT.SimpleTy != MVT::v2f64) 16011 return 0; 16012 if ((Subtarget->hasVLX())) { 16013 return fastEmitInst_rr(X86::VSCALEFPDZ128rr, &X86::VR128XRegClass, Op0, Op1); 16014 } 16015 return 0; 16016} 16017 16018unsigned fastEmit_X86ISD_SCALEF_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16019 if (RetVT.SimpleTy != MVT::v4f64) 16020 return 0; 16021 if ((Subtarget->hasVLX())) { 16022 return fastEmitInst_rr(X86::VSCALEFPDZ256rr, &X86::VR256XRegClass, Op0, Op1); 16023 } 16024 return 0; 16025} 16026 16027unsigned fastEmit_X86ISD_SCALEF_MVT_v8f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16028 if (RetVT.SimpleTy != MVT::v8f64) 16029 return 0; 16030 if ((Subtarget->hasAVX512())) { 16031 return fastEmitInst_rr(X86::VSCALEFPDZrr, &X86::VR512RegClass, Op0, Op1); 16032 } 16033 return 0; 16034} 16035 16036unsigned fastEmit_X86ISD_SCALEF_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 16037 switch (VT.SimpleTy) { 16038 case MVT::v8f16: return fastEmit_X86ISD_SCALEF_MVT_v8f16_rr(RetVT, Op0, Op1); 16039 case MVT::v16f16: return fastEmit_X86ISD_SCALEF_MVT_v16f16_rr(RetVT, Op0, Op1); 16040 case MVT::v32f16: return fastEmit_X86ISD_SCALEF_MVT_v32f16_rr(RetVT, Op0, Op1); 16041 case MVT::v4f32: return fastEmit_X86ISD_SCALEF_MVT_v4f32_rr(RetVT, Op0, Op1); 16042 case MVT::v8f32: return fastEmit_X86ISD_SCALEF_MVT_v8f32_rr(RetVT, Op0, Op1); 16043 case MVT::v16f32: return fastEmit_X86ISD_SCALEF_MVT_v16f32_rr(RetVT, Op0, Op1); 16044 case MVT::v2f64: return fastEmit_X86ISD_SCALEF_MVT_v2f64_rr(RetVT, Op0, Op1); 16045 case MVT::v4f64: return fastEmit_X86ISD_SCALEF_MVT_v4f64_rr(RetVT, Op0, Op1); 16046 case MVT::v8f64: return fastEmit_X86ISD_SCALEF_MVT_v8f64_rr(RetVT, Op0, Op1); 16047 default: return 0; 16048 } 16049} 16050 16051// FastEmit functions for X86ISD::SCALEFS. 16052 16053unsigned fastEmit_X86ISD_SCALEFS_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16054 if (RetVT.SimpleTy != MVT::v8f16) 16055 return 0; 16056 if ((Subtarget->hasFP16())) { 16057 return fastEmitInst_rr(X86::VSCALEFSHZrr, &X86::VR128XRegClass, Op0, Op1); 16058 } 16059 return 0; 16060} 16061 16062unsigned fastEmit_X86ISD_SCALEFS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16063 if (RetVT.SimpleTy != MVT::v4f32) 16064 return 0; 16065 if ((Subtarget->hasAVX512())) { 16066 return fastEmitInst_rr(X86::VSCALEFSSZrr, &X86::VR128XRegClass, Op0, Op1); 16067 } 16068 return 0; 16069} 16070 16071unsigned fastEmit_X86ISD_SCALEFS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16072 if (RetVT.SimpleTy != MVT::v2f64) 16073 return 0; 16074 if ((Subtarget->hasAVX512())) { 16075 return fastEmitInst_rr(X86::VSCALEFSDZrr, &X86::VR128XRegClass, Op0, Op1); 16076 } 16077 return 0; 16078} 16079 16080unsigned fastEmit_X86ISD_SCALEFS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 16081 switch (VT.SimpleTy) { 16082 case MVT::v8f16: return fastEmit_X86ISD_SCALEFS_MVT_v8f16_rr(RetVT, Op0, Op1); 16083 case MVT::v4f32: return fastEmit_X86ISD_SCALEFS_MVT_v4f32_rr(RetVT, Op0, Op1); 16084 case MVT::v2f64: return fastEmit_X86ISD_SCALEFS_MVT_v2f64_rr(RetVT, Op0, Op1); 16085 default: return 0; 16086 } 16087} 16088 16089// FastEmit functions for X86ISD::STRICT_FCMP. 16090 16091unsigned fastEmit_X86ISD_STRICT_FCMP_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16092 if (RetVT.SimpleTy != MVT::i32) 16093 return 0; 16094 if ((Subtarget->hasFP16())) { 16095 return fastEmitInst_rr(X86::VUCOMISHZrr, &X86::FR16XRegClass, Op0, Op1); 16096 } 16097 return 0; 16098} 16099 16100unsigned fastEmit_X86ISD_STRICT_FCMP_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16101 if (RetVT.SimpleTy != MVT::i32) 16102 return 0; 16103 if ((Subtarget->hasAVX512())) { 16104 return fastEmitInst_rr(X86::VUCOMISSZrr, &X86::FR32XRegClass, Op0, Op1); 16105 } 16106 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 16107 return fastEmitInst_rr(X86::UCOMISSrr, &X86::FR32RegClass, Op0, Op1); 16108 } 16109 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 16110 return fastEmitInst_rr(X86::VUCOMISSrr, &X86::FR32RegClass, Op0, Op1); 16111 } 16112 if ((!Subtarget->hasSSE1()) && (Subtarget->canUseCMOV())) { 16113 return fastEmitInst_rr(X86::UCOM_FpIr32, &X86::RFP32RegClass, Op0, Op1); 16114 } 16115 return 0; 16116} 16117 16118unsigned fastEmit_X86ISD_STRICT_FCMP_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16119 if (RetVT.SimpleTy != MVT::i32) 16120 return 0; 16121 if ((Subtarget->hasAVX512())) { 16122 return fastEmitInst_rr(X86::VUCOMISDZrr, &X86::FR64XRegClass, Op0, Op1); 16123 } 16124 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 16125 return fastEmitInst_rr(X86::UCOMISDrr, &X86::FR64RegClass, Op0, Op1); 16126 } 16127 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 16128 return fastEmitInst_rr(X86::VUCOMISDrr, &X86::FR64RegClass, Op0, Op1); 16129 } 16130 if ((!Subtarget->hasSSE2()) && (Subtarget->canUseCMOV())) { 16131 return fastEmitInst_rr(X86::UCOM_FpIr64, &X86::RFP64RegClass, Op0, Op1); 16132 } 16133 return 0; 16134} 16135 16136unsigned fastEmit_X86ISD_STRICT_FCMP_MVT_f80_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16137 if (RetVT.SimpleTy != MVT::i32) 16138 return 0; 16139 if ((Subtarget->canUseCMOV())) { 16140 return fastEmitInst_rr(X86::UCOM_FpIr80, &X86::RFP80RegClass, Op0, Op1); 16141 } 16142 return 0; 16143} 16144 16145unsigned fastEmit_X86ISD_STRICT_FCMP_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 16146 switch (VT.SimpleTy) { 16147 case MVT::f16: return fastEmit_X86ISD_STRICT_FCMP_MVT_f16_rr(RetVT, Op0, Op1); 16148 case MVT::f32: return fastEmit_X86ISD_STRICT_FCMP_MVT_f32_rr(RetVT, Op0, Op1); 16149 case MVT::f64: return fastEmit_X86ISD_STRICT_FCMP_MVT_f64_rr(RetVT, Op0, Op1); 16150 case MVT::f80: return fastEmit_X86ISD_STRICT_FCMP_MVT_f80_rr(RetVT, Op0, Op1); 16151 default: return 0; 16152 } 16153} 16154 16155// FastEmit functions for X86ISD::STRICT_FCMPS. 16156 16157unsigned fastEmit_X86ISD_STRICT_FCMPS_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16158 if (RetVT.SimpleTy != MVT::i32) 16159 return 0; 16160 if ((Subtarget->hasFP16())) { 16161 return fastEmitInst_rr(X86::VCOMISHZrr, &X86::FR16XRegClass, Op0, Op1); 16162 } 16163 return 0; 16164} 16165 16166unsigned fastEmit_X86ISD_STRICT_FCMPS_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16167 if (RetVT.SimpleTy != MVT::i32) 16168 return 0; 16169 if ((Subtarget->hasAVX512())) { 16170 return fastEmitInst_rr(X86::VCOMISSZrr, &X86::FR32XRegClass, Op0, Op1); 16171 } 16172 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 16173 return fastEmitInst_rr(X86::COMISSrr, &X86::FR32RegClass, Op0, Op1); 16174 } 16175 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 16176 return fastEmitInst_rr(X86::VCOMISSrr, &X86::FR32RegClass, Op0, Op1); 16177 } 16178 if ((!Subtarget->hasSSE1()) && (Subtarget->canUseCMOV())) { 16179 return fastEmitInst_rr(X86::COM_FpIr32, &X86::RFP32RegClass, Op0, Op1); 16180 } 16181 return 0; 16182} 16183 16184unsigned fastEmit_X86ISD_STRICT_FCMPS_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16185 if (RetVT.SimpleTy != MVT::i32) 16186 return 0; 16187 if ((Subtarget->hasAVX512())) { 16188 return fastEmitInst_rr(X86::VCOMISDZrr, &X86::FR64XRegClass, Op0, Op1); 16189 } 16190 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 16191 return fastEmitInst_rr(X86::COMISDrr, &X86::FR64RegClass, Op0, Op1); 16192 } 16193 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 16194 return fastEmitInst_rr(X86::VCOMISDrr, &X86::FR64RegClass, Op0, Op1); 16195 } 16196 if ((!Subtarget->hasSSE2()) && (Subtarget->canUseCMOV())) { 16197 return fastEmitInst_rr(X86::COM_FpIr64, &X86::RFP64RegClass, Op0, Op1); 16198 } 16199 return 0; 16200} 16201 16202unsigned fastEmit_X86ISD_STRICT_FCMPS_MVT_f80_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16203 if (RetVT.SimpleTy != MVT::i32) 16204 return 0; 16205 if ((Subtarget->canUseCMOV())) { 16206 return fastEmitInst_rr(X86::COM_FpIr80, &X86::RFP80RegClass, Op0, Op1); 16207 } 16208 return 0; 16209} 16210 16211unsigned fastEmit_X86ISD_STRICT_FCMPS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 16212 switch (VT.SimpleTy) { 16213 case MVT::f16: return fastEmit_X86ISD_STRICT_FCMPS_MVT_f16_rr(RetVT, Op0, Op1); 16214 case MVT::f32: return fastEmit_X86ISD_STRICT_FCMPS_MVT_f32_rr(RetVT, Op0, Op1); 16215 case MVT::f64: return fastEmit_X86ISD_STRICT_FCMPS_MVT_f64_rr(RetVT, Op0, Op1); 16216 case MVT::f80: return fastEmit_X86ISD_STRICT_FCMPS_MVT_f80_rr(RetVT, Op0, Op1); 16217 default: return 0; 16218 } 16219} 16220 16221// FastEmit functions for X86ISD::STRICT_FP80_ADD. 16222 16223unsigned fastEmit_X86ISD_STRICT_FP80_ADD_MVT_f80_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16224 if (RetVT.SimpleTy != MVT::f80) 16225 return 0; 16226 return fastEmitInst_rr(X86::FP80_ADDr, &X86::RFP80RegClass, Op0, Op1); 16227} 16228 16229unsigned fastEmit_X86ISD_STRICT_FP80_ADD_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 16230 switch (VT.SimpleTy) { 16231 case MVT::f80: return fastEmit_X86ISD_STRICT_FP80_ADD_MVT_f80_rr(RetVT, Op0, Op1); 16232 default: return 0; 16233 } 16234} 16235 16236// FastEmit functions for X86ISD::TESTP. 16237 16238unsigned fastEmit_X86ISD_TESTP_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16239 if (RetVT.SimpleTy != MVT::i32) 16240 return 0; 16241 if ((Subtarget->hasAVX())) { 16242 return fastEmitInst_rr(X86::VTESTPSrr, &X86::VR128RegClass, Op0, Op1); 16243 } 16244 return 0; 16245} 16246 16247unsigned fastEmit_X86ISD_TESTP_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16248 if (RetVT.SimpleTy != MVT::i32) 16249 return 0; 16250 if ((Subtarget->hasAVX())) { 16251 return fastEmitInst_rr(X86::VTESTPSYrr, &X86::VR256RegClass, Op0, Op1); 16252 } 16253 return 0; 16254} 16255 16256unsigned fastEmit_X86ISD_TESTP_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16257 if (RetVT.SimpleTy != MVT::i32) 16258 return 0; 16259 if ((Subtarget->hasAVX())) { 16260 return fastEmitInst_rr(X86::VTESTPDrr, &X86::VR128RegClass, Op0, Op1); 16261 } 16262 return 0; 16263} 16264 16265unsigned fastEmit_X86ISD_TESTP_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16266 if (RetVT.SimpleTy != MVT::i32) 16267 return 0; 16268 if ((Subtarget->hasAVX())) { 16269 return fastEmitInst_rr(X86::VTESTPDYrr, &X86::VR256RegClass, Op0, Op1); 16270 } 16271 return 0; 16272} 16273 16274unsigned fastEmit_X86ISD_TESTP_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 16275 switch (VT.SimpleTy) { 16276 case MVT::v4f32: return fastEmit_X86ISD_TESTP_MVT_v4f32_rr(RetVT, Op0, Op1); 16277 case MVT::v8f32: return fastEmit_X86ISD_TESTP_MVT_v8f32_rr(RetVT, Op0, Op1); 16278 case MVT::v2f64: return fastEmit_X86ISD_TESTP_MVT_v2f64_rr(RetVT, Op0, Op1); 16279 case MVT::v4f64: return fastEmit_X86ISD_TESTP_MVT_v4f64_rr(RetVT, Op0, Op1); 16280 default: return 0; 16281 } 16282} 16283 16284// FastEmit functions for X86ISD::UCOMI. 16285 16286unsigned fastEmit_X86ISD_UCOMI_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16287 if (RetVT.SimpleTy != MVT::i32) 16288 return 0; 16289 if ((Subtarget->hasFP16())) { 16290 return fastEmitInst_rr(X86::VUCOMISHZrr_Int, &X86::VR128XRegClass, Op0, Op1); 16291 } 16292 return 0; 16293} 16294 16295unsigned fastEmit_X86ISD_UCOMI_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16296 if (RetVT.SimpleTy != MVT::i32) 16297 return 0; 16298 if ((Subtarget->hasAVX512())) { 16299 return fastEmitInst_rr(X86::VUCOMISSZrr_Int, &X86::VR128XRegClass, Op0, Op1); 16300 } 16301 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 16302 return fastEmitInst_rr(X86::UCOMISSrr_Int, &X86::VR128RegClass, Op0, Op1); 16303 } 16304 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 16305 return fastEmitInst_rr(X86::VUCOMISSrr_Int, &X86::VR128RegClass, Op0, Op1); 16306 } 16307 return 0; 16308} 16309 16310unsigned fastEmit_X86ISD_UCOMI_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16311 if (RetVT.SimpleTy != MVT::i32) 16312 return 0; 16313 if ((Subtarget->hasAVX512())) { 16314 return fastEmitInst_rr(X86::VUCOMISDZrr_Int, &X86::VR128XRegClass, Op0, Op1); 16315 } 16316 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 16317 return fastEmitInst_rr(X86::UCOMISDrr_Int, &X86::VR128RegClass, Op0, Op1); 16318 } 16319 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 16320 return fastEmitInst_rr(X86::VUCOMISDrr_Int, &X86::VR128RegClass, Op0, Op1); 16321 } 16322 return 0; 16323} 16324 16325unsigned fastEmit_X86ISD_UCOMI_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 16326 switch (VT.SimpleTy) { 16327 case MVT::v8f16: return fastEmit_X86ISD_UCOMI_MVT_v8f16_rr(RetVT, Op0, Op1); 16328 case MVT::v4f32: return fastEmit_X86ISD_UCOMI_MVT_v4f32_rr(RetVT, Op0, Op1); 16329 case MVT::v2f64: return fastEmit_X86ISD_UCOMI_MVT_v2f64_rr(RetVT, Op0, Op1); 16330 default: return 0; 16331 } 16332} 16333 16334// FastEmit functions for X86ISD::UNPCKH. 16335 16336unsigned fastEmit_X86ISD_UNPCKH_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16337 if (RetVT.SimpleTy != MVT::v16i8) 16338 return 0; 16339 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 16340 return fastEmitInst_rr(X86::VPUNPCKHBWZ128rr, &X86::VR128XRegClass, Op0, Op1); 16341 } 16342 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 16343 return fastEmitInst_rr(X86::PUNPCKHBWrr, &X86::VR128RegClass, Op0, Op1); 16344 } 16345 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 16346 return fastEmitInst_rr(X86::VPUNPCKHBWrr, &X86::VR128RegClass, Op0, Op1); 16347 } 16348 return 0; 16349} 16350 16351unsigned fastEmit_X86ISD_UNPCKH_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16352 if (RetVT.SimpleTy != MVT::v32i8) 16353 return 0; 16354 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 16355 return fastEmitInst_rr(X86::VPUNPCKHBWZ256rr, &X86::VR256XRegClass, Op0, Op1); 16356 } 16357 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 16358 return fastEmitInst_rr(X86::VPUNPCKHBWYrr, &X86::VR256RegClass, Op0, Op1); 16359 } 16360 return 0; 16361} 16362 16363unsigned fastEmit_X86ISD_UNPCKH_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16364 if (RetVT.SimpleTy != MVT::v64i8) 16365 return 0; 16366 if ((Subtarget->hasBWI())) { 16367 return fastEmitInst_rr(X86::VPUNPCKHBWZrr, &X86::VR512RegClass, Op0, Op1); 16368 } 16369 return 0; 16370} 16371 16372unsigned fastEmit_X86ISD_UNPCKH_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16373 if (RetVT.SimpleTy != MVT::v8i16) 16374 return 0; 16375 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 16376 return fastEmitInst_rr(X86::VPUNPCKHWDZ128rr, &X86::VR128XRegClass, Op0, Op1); 16377 } 16378 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 16379 return fastEmitInst_rr(X86::PUNPCKHWDrr, &X86::VR128RegClass, Op0, Op1); 16380 } 16381 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 16382 return fastEmitInst_rr(X86::VPUNPCKHWDrr, &X86::VR128RegClass, Op0, Op1); 16383 } 16384 return 0; 16385} 16386 16387unsigned fastEmit_X86ISD_UNPCKH_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16388 if (RetVT.SimpleTy != MVT::v16i16) 16389 return 0; 16390 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 16391 return fastEmitInst_rr(X86::VPUNPCKHWDZ256rr, &X86::VR256XRegClass, Op0, Op1); 16392 } 16393 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 16394 return fastEmitInst_rr(X86::VPUNPCKHWDYrr, &X86::VR256RegClass, Op0, Op1); 16395 } 16396 return 0; 16397} 16398 16399unsigned fastEmit_X86ISD_UNPCKH_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16400 if (RetVT.SimpleTy != MVT::v32i16) 16401 return 0; 16402 if ((Subtarget->hasBWI())) { 16403 return fastEmitInst_rr(X86::VPUNPCKHWDZrr, &X86::VR512RegClass, Op0, Op1); 16404 } 16405 return 0; 16406} 16407 16408unsigned fastEmit_X86ISD_UNPCKH_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16409 if (RetVT.SimpleTy != MVT::v4i32) 16410 return 0; 16411 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 16412 return fastEmitInst_rr(X86::VPUNPCKHDQZ128rr, &X86::VR128XRegClass, Op0, Op1); 16413 } 16414 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 16415 return fastEmitInst_rr(X86::PUNPCKHDQrr, &X86::VR128RegClass, Op0, Op1); 16416 } 16417 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 16418 return fastEmitInst_rr(X86::VPUNPCKHDQrr, &X86::VR128RegClass, Op0, Op1); 16419 } 16420 return 0; 16421} 16422 16423unsigned fastEmit_X86ISD_UNPCKH_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16424 if (RetVT.SimpleTy != MVT::v8i32) 16425 return 0; 16426 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) { 16427 return fastEmitInst_rr(X86::VUNPCKHPSYrr, &X86::VR256RegClass, Op0, Op1); 16428 } 16429 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 16430 return fastEmitInst_rr(X86::VPUNPCKHDQZ256rr, &X86::VR256XRegClass, Op0, Op1); 16431 } 16432 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 16433 return fastEmitInst_rr(X86::VPUNPCKHDQYrr, &X86::VR256RegClass, Op0, Op1); 16434 } 16435 return 0; 16436} 16437 16438unsigned fastEmit_X86ISD_UNPCKH_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16439 if (RetVT.SimpleTy != MVT::v16i32) 16440 return 0; 16441 if ((Subtarget->hasAVX512())) { 16442 return fastEmitInst_rr(X86::VPUNPCKHDQZrr, &X86::VR512RegClass, Op0, Op1); 16443 } 16444 return 0; 16445} 16446 16447unsigned fastEmit_X86ISD_UNPCKH_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16448 if (RetVT.SimpleTy != MVT::v2i64) 16449 return 0; 16450 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 16451 return fastEmitInst_rr(X86::VPUNPCKHQDQZ128rr, &X86::VR128XRegClass, Op0, Op1); 16452 } 16453 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 16454 return fastEmitInst_rr(X86::PUNPCKHQDQrr, &X86::VR128RegClass, Op0, Op1); 16455 } 16456 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 16457 return fastEmitInst_rr(X86::VPUNPCKHQDQrr, &X86::VR128RegClass, Op0, Op1); 16458 } 16459 return 0; 16460} 16461 16462unsigned fastEmit_X86ISD_UNPCKH_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16463 if (RetVT.SimpleTy != MVT::v4i64) 16464 return 0; 16465 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) { 16466 return fastEmitInst_rr(X86::VUNPCKHPDYrr, &X86::VR256RegClass, Op0, Op1); 16467 } 16468 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 16469 return fastEmitInst_rr(X86::VPUNPCKHQDQZ256rr, &X86::VR256XRegClass, Op0, Op1); 16470 } 16471 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 16472 return fastEmitInst_rr(X86::VPUNPCKHQDQYrr, &X86::VR256RegClass, Op0, Op1); 16473 } 16474 return 0; 16475} 16476 16477unsigned fastEmit_X86ISD_UNPCKH_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16478 if (RetVT.SimpleTy != MVT::v8i64) 16479 return 0; 16480 if ((Subtarget->hasAVX512())) { 16481 return fastEmitInst_rr(X86::VPUNPCKHQDQZrr, &X86::VR512RegClass, Op0, Op1); 16482 } 16483 return 0; 16484} 16485 16486unsigned fastEmit_X86ISD_UNPCKH_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16487 if (RetVT.SimpleTy != MVT::v4f32) 16488 return 0; 16489 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 16490 return fastEmitInst_rr(X86::VUNPCKHPSZ128rr, &X86::VR128XRegClass, Op0, Op1); 16491 } 16492 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 16493 return fastEmitInst_rr(X86::UNPCKHPSrr, &X86::VR128RegClass, Op0, Op1); 16494 } 16495 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 16496 return fastEmitInst_rr(X86::VUNPCKHPSrr, &X86::VR128RegClass, Op0, Op1); 16497 } 16498 return 0; 16499} 16500 16501unsigned fastEmit_X86ISD_UNPCKH_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16502 if (RetVT.SimpleTy != MVT::v8f32) 16503 return 0; 16504 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 16505 return fastEmitInst_rr(X86::VUNPCKHPSZ256rr, &X86::VR256XRegClass, Op0, Op1); 16506 } 16507 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 16508 return fastEmitInst_rr(X86::VUNPCKHPSYrr, &X86::VR256RegClass, Op0, Op1); 16509 } 16510 return 0; 16511} 16512 16513unsigned fastEmit_X86ISD_UNPCKH_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16514 if (RetVT.SimpleTy != MVT::v16f32) 16515 return 0; 16516 if ((Subtarget->hasAVX512())) { 16517 return fastEmitInst_rr(X86::VUNPCKHPSZrr, &X86::VR512RegClass, Op0, Op1); 16518 } 16519 return 0; 16520} 16521 16522unsigned fastEmit_X86ISD_UNPCKH_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16523 if (RetVT.SimpleTy != MVT::v2f64) 16524 return 0; 16525 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 16526 return fastEmitInst_rr(X86::VUNPCKHPDZ128rr, &X86::VR128XRegClass, Op0, Op1); 16527 } 16528 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 16529 return fastEmitInst_rr(X86::UNPCKHPDrr, &X86::VR128RegClass, Op0, Op1); 16530 } 16531 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 16532 return fastEmitInst_rr(X86::VUNPCKHPDrr, &X86::VR128RegClass, Op0, Op1); 16533 } 16534 return 0; 16535} 16536 16537unsigned fastEmit_X86ISD_UNPCKH_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16538 if (RetVT.SimpleTy != MVT::v4f64) 16539 return 0; 16540 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 16541 return fastEmitInst_rr(X86::VUNPCKHPDZ256rr, &X86::VR256XRegClass, Op0, Op1); 16542 } 16543 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 16544 return fastEmitInst_rr(X86::VUNPCKHPDYrr, &X86::VR256RegClass, Op0, Op1); 16545 } 16546 return 0; 16547} 16548 16549unsigned fastEmit_X86ISD_UNPCKH_MVT_v8f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16550 if (RetVT.SimpleTy != MVT::v8f64) 16551 return 0; 16552 if ((Subtarget->hasAVX512())) { 16553 return fastEmitInst_rr(X86::VUNPCKHPDZrr, &X86::VR512RegClass, Op0, Op1); 16554 } 16555 return 0; 16556} 16557 16558unsigned fastEmit_X86ISD_UNPCKH_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 16559 switch (VT.SimpleTy) { 16560 case MVT::v16i8: return fastEmit_X86ISD_UNPCKH_MVT_v16i8_rr(RetVT, Op0, Op1); 16561 case MVT::v32i8: return fastEmit_X86ISD_UNPCKH_MVT_v32i8_rr(RetVT, Op0, Op1); 16562 case MVT::v64i8: return fastEmit_X86ISD_UNPCKH_MVT_v64i8_rr(RetVT, Op0, Op1); 16563 case MVT::v8i16: return fastEmit_X86ISD_UNPCKH_MVT_v8i16_rr(RetVT, Op0, Op1); 16564 case MVT::v16i16: return fastEmit_X86ISD_UNPCKH_MVT_v16i16_rr(RetVT, Op0, Op1); 16565 case MVT::v32i16: return fastEmit_X86ISD_UNPCKH_MVT_v32i16_rr(RetVT, Op0, Op1); 16566 case MVT::v4i32: return fastEmit_X86ISD_UNPCKH_MVT_v4i32_rr(RetVT, Op0, Op1); 16567 case MVT::v8i32: return fastEmit_X86ISD_UNPCKH_MVT_v8i32_rr(RetVT, Op0, Op1); 16568 case MVT::v16i32: return fastEmit_X86ISD_UNPCKH_MVT_v16i32_rr(RetVT, Op0, Op1); 16569 case MVT::v2i64: return fastEmit_X86ISD_UNPCKH_MVT_v2i64_rr(RetVT, Op0, Op1); 16570 case MVT::v4i64: return fastEmit_X86ISD_UNPCKH_MVT_v4i64_rr(RetVT, Op0, Op1); 16571 case MVT::v8i64: return fastEmit_X86ISD_UNPCKH_MVT_v8i64_rr(RetVT, Op0, Op1); 16572 case MVT::v4f32: return fastEmit_X86ISD_UNPCKH_MVT_v4f32_rr(RetVT, Op0, Op1); 16573 case MVT::v8f32: return fastEmit_X86ISD_UNPCKH_MVT_v8f32_rr(RetVT, Op0, Op1); 16574 case MVT::v16f32: return fastEmit_X86ISD_UNPCKH_MVT_v16f32_rr(RetVT, Op0, Op1); 16575 case MVT::v2f64: return fastEmit_X86ISD_UNPCKH_MVT_v2f64_rr(RetVT, Op0, Op1); 16576 case MVT::v4f64: return fastEmit_X86ISD_UNPCKH_MVT_v4f64_rr(RetVT, Op0, Op1); 16577 case MVT::v8f64: return fastEmit_X86ISD_UNPCKH_MVT_v8f64_rr(RetVT, Op0, Op1); 16578 default: return 0; 16579 } 16580} 16581 16582// FastEmit functions for X86ISD::UNPCKL. 16583 16584unsigned fastEmit_X86ISD_UNPCKL_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16585 if (RetVT.SimpleTy != MVT::v16i8) 16586 return 0; 16587 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 16588 return fastEmitInst_rr(X86::VPUNPCKLBWZ128rr, &X86::VR128XRegClass, Op0, Op1); 16589 } 16590 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 16591 return fastEmitInst_rr(X86::PUNPCKLBWrr, &X86::VR128RegClass, Op0, Op1); 16592 } 16593 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 16594 return fastEmitInst_rr(X86::VPUNPCKLBWrr, &X86::VR128RegClass, Op0, Op1); 16595 } 16596 return 0; 16597} 16598 16599unsigned fastEmit_X86ISD_UNPCKL_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16600 if (RetVT.SimpleTy != MVT::v32i8) 16601 return 0; 16602 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 16603 return fastEmitInst_rr(X86::VPUNPCKLBWZ256rr, &X86::VR256XRegClass, Op0, Op1); 16604 } 16605 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 16606 return fastEmitInst_rr(X86::VPUNPCKLBWYrr, &X86::VR256RegClass, Op0, Op1); 16607 } 16608 return 0; 16609} 16610 16611unsigned fastEmit_X86ISD_UNPCKL_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16612 if (RetVT.SimpleTy != MVT::v64i8) 16613 return 0; 16614 if ((Subtarget->hasBWI())) { 16615 return fastEmitInst_rr(X86::VPUNPCKLBWZrr, &X86::VR512RegClass, Op0, Op1); 16616 } 16617 return 0; 16618} 16619 16620unsigned fastEmit_X86ISD_UNPCKL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16621 if (RetVT.SimpleTy != MVT::v8i16) 16622 return 0; 16623 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 16624 return fastEmitInst_rr(X86::VPUNPCKLWDZ128rr, &X86::VR128XRegClass, Op0, Op1); 16625 } 16626 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 16627 return fastEmitInst_rr(X86::PUNPCKLWDrr, &X86::VR128RegClass, Op0, Op1); 16628 } 16629 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 16630 return fastEmitInst_rr(X86::VPUNPCKLWDrr, &X86::VR128RegClass, Op0, Op1); 16631 } 16632 return 0; 16633} 16634 16635unsigned fastEmit_X86ISD_UNPCKL_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16636 if (RetVT.SimpleTy != MVT::v16i16) 16637 return 0; 16638 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 16639 return fastEmitInst_rr(X86::VPUNPCKLWDZ256rr, &X86::VR256XRegClass, Op0, Op1); 16640 } 16641 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 16642 return fastEmitInst_rr(X86::VPUNPCKLWDYrr, &X86::VR256RegClass, Op0, Op1); 16643 } 16644 return 0; 16645} 16646 16647unsigned fastEmit_X86ISD_UNPCKL_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16648 if (RetVT.SimpleTy != MVT::v32i16) 16649 return 0; 16650 if ((Subtarget->hasBWI())) { 16651 return fastEmitInst_rr(X86::VPUNPCKLWDZrr, &X86::VR512RegClass, Op0, Op1); 16652 } 16653 return 0; 16654} 16655 16656unsigned fastEmit_X86ISD_UNPCKL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16657 if (RetVT.SimpleTy != MVT::v4i32) 16658 return 0; 16659 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 16660 return fastEmitInst_rr(X86::VPUNPCKLDQZ128rr, &X86::VR128XRegClass, Op0, Op1); 16661 } 16662 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 16663 return fastEmitInst_rr(X86::PUNPCKLDQrr, &X86::VR128RegClass, Op0, Op1); 16664 } 16665 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 16666 return fastEmitInst_rr(X86::VPUNPCKLDQrr, &X86::VR128RegClass, Op0, Op1); 16667 } 16668 return 0; 16669} 16670 16671unsigned fastEmit_X86ISD_UNPCKL_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16672 if (RetVT.SimpleTy != MVT::v8i32) 16673 return 0; 16674 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) { 16675 return fastEmitInst_rr(X86::VUNPCKLPSYrr, &X86::VR256RegClass, Op0, Op1); 16676 } 16677 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 16678 return fastEmitInst_rr(X86::VPUNPCKLDQZ256rr, &X86::VR256XRegClass, Op0, Op1); 16679 } 16680 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 16681 return fastEmitInst_rr(X86::VPUNPCKLDQYrr, &X86::VR256RegClass, Op0, Op1); 16682 } 16683 return 0; 16684} 16685 16686unsigned fastEmit_X86ISD_UNPCKL_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16687 if (RetVT.SimpleTy != MVT::v16i32) 16688 return 0; 16689 if ((Subtarget->hasAVX512())) { 16690 return fastEmitInst_rr(X86::VPUNPCKLDQZrr, &X86::VR512RegClass, Op0, Op1); 16691 } 16692 return 0; 16693} 16694 16695unsigned fastEmit_X86ISD_UNPCKL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16696 if (RetVT.SimpleTy != MVT::v2i64) 16697 return 0; 16698 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 16699 return fastEmitInst_rr(X86::VPUNPCKLQDQZ128rr, &X86::VR128XRegClass, Op0, Op1); 16700 } 16701 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 16702 return fastEmitInst_rr(X86::PUNPCKLQDQrr, &X86::VR128RegClass, Op0, Op1); 16703 } 16704 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 16705 return fastEmitInst_rr(X86::VPUNPCKLQDQrr, &X86::VR128RegClass, Op0, Op1); 16706 } 16707 return 0; 16708} 16709 16710unsigned fastEmit_X86ISD_UNPCKL_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16711 if (RetVT.SimpleTy != MVT::v4i64) 16712 return 0; 16713 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) { 16714 return fastEmitInst_rr(X86::VUNPCKLPDYrr, &X86::VR256RegClass, Op0, Op1); 16715 } 16716 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 16717 return fastEmitInst_rr(X86::VPUNPCKLQDQZ256rr, &X86::VR256XRegClass, Op0, Op1); 16718 } 16719 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 16720 return fastEmitInst_rr(X86::VPUNPCKLQDQYrr, &X86::VR256RegClass, Op0, Op1); 16721 } 16722 return 0; 16723} 16724 16725unsigned fastEmit_X86ISD_UNPCKL_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16726 if (RetVT.SimpleTy != MVT::v8i64) 16727 return 0; 16728 if ((Subtarget->hasAVX512())) { 16729 return fastEmitInst_rr(X86::VPUNPCKLQDQZrr, &X86::VR512RegClass, Op0, Op1); 16730 } 16731 return 0; 16732} 16733 16734unsigned fastEmit_X86ISD_UNPCKL_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16735 if (RetVT.SimpleTy != MVT::v4f32) 16736 return 0; 16737 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 16738 return fastEmitInst_rr(X86::VUNPCKLPSZ128rr, &X86::VR128XRegClass, Op0, Op1); 16739 } 16740 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 16741 return fastEmitInst_rr(X86::UNPCKLPSrr, &X86::VR128RegClass, Op0, Op1); 16742 } 16743 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 16744 return fastEmitInst_rr(X86::VUNPCKLPSrr, &X86::VR128RegClass, Op0, Op1); 16745 } 16746 return 0; 16747} 16748 16749unsigned fastEmit_X86ISD_UNPCKL_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16750 if (RetVT.SimpleTy != MVT::v8f32) 16751 return 0; 16752 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 16753 return fastEmitInst_rr(X86::VUNPCKLPSZ256rr, &X86::VR256XRegClass, Op0, Op1); 16754 } 16755 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 16756 return fastEmitInst_rr(X86::VUNPCKLPSYrr, &X86::VR256RegClass, Op0, Op1); 16757 } 16758 return 0; 16759} 16760 16761unsigned fastEmit_X86ISD_UNPCKL_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16762 if (RetVT.SimpleTy != MVT::v16f32) 16763 return 0; 16764 if ((Subtarget->hasAVX512())) { 16765 return fastEmitInst_rr(X86::VUNPCKLPSZrr, &X86::VR512RegClass, Op0, Op1); 16766 } 16767 return 0; 16768} 16769 16770unsigned fastEmit_X86ISD_UNPCKL_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16771 if (RetVT.SimpleTy != MVT::v2f64) 16772 return 0; 16773 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 16774 return fastEmitInst_rr(X86::VUNPCKLPDZ128rr, &X86::VR128XRegClass, Op0, Op1); 16775 } 16776 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 16777 return fastEmitInst_rr(X86::UNPCKLPDrr, &X86::VR128RegClass, Op0, Op1); 16778 } 16779 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 16780 return fastEmitInst_rr(X86::VUNPCKLPDrr, &X86::VR128RegClass, Op0, Op1); 16781 } 16782 return 0; 16783} 16784 16785unsigned fastEmit_X86ISD_UNPCKL_MVT_v4f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16786 if (RetVT.SimpleTy != MVT::v4f64) 16787 return 0; 16788 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 16789 return fastEmitInst_rr(X86::VUNPCKLPDZ256rr, &X86::VR256XRegClass, Op0, Op1); 16790 } 16791 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 16792 return fastEmitInst_rr(X86::VUNPCKLPDYrr, &X86::VR256RegClass, Op0, Op1); 16793 } 16794 return 0; 16795} 16796 16797unsigned fastEmit_X86ISD_UNPCKL_MVT_v8f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16798 if (RetVT.SimpleTy != MVT::v8f64) 16799 return 0; 16800 if ((Subtarget->hasAVX512())) { 16801 return fastEmitInst_rr(X86::VUNPCKLPDZrr, &X86::VR512RegClass, Op0, Op1); 16802 } 16803 return 0; 16804} 16805 16806unsigned fastEmit_X86ISD_UNPCKL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 16807 switch (VT.SimpleTy) { 16808 case MVT::v16i8: return fastEmit_X86ISD_UNPCKL_MVT_v16i8_rr(RetVT, Op0, Op1); 16809 case MVT::v32i8: return fastEmit_X86ISD_UNPCKL_MVT_v32i8_rr(RetVT, Op0, Op1); 16810 case MVT::v64i8: return fastEmit_X86ISD_UNPCKL_MVT_v64i8_rr(RetVT, Op0, Op1); 16811 case MVT::v8i16: return fastEmit_X86ISD_UNPCKL_MVT_v8i16_rr(RetVT, Op0, Op1); 16812 case MVT::v16i16: return fastEmit_X86ISD_UNPCKL_MVT_v16i16_rr(RetVT, Op0, Op1); 16813 case MVT::v32i16: return fastEmit_X86ISD_UNPCKL_MVT_v32i16_rr(RetVT, Op0, Op1); 16814 case MVT::v4i32: return fastEmit_X86ISD_UNPCKL_MVT_v4i32_rr(RetVT, Op0, Op1); 16815 case MVT::v8i32: return fastEmit_X86ISD_UNPCKL_MVT_v8i32_rr(RetVT, Op0, Op1); 16816 case MVT::v16i32: return fastEmit_X86ISD_UNPCKL_MVT_v16i32_rr(RetVT, Op0, Op1); 16817 case MVT::v2i64: return fastEmit_X86ISD_UNPCKL_MVT_v2i64_rr(RetVT, Op0, Op1); 16818 case MVT::v4i64: return fastEmit_X86ISD_UNPCKL_MVT_v4i64_rr(RetVT, Op0, Op1); 16819 case MVT::v8i64: return fastEmit_X86ISD_UNPCKL_MVT_v8i64_rr(RetVT, Op0, Op1); 16820 case MVT::v4f32: return fastEmit_X86ISD_UNPCKL_MVT_v4f32_rr(RetVT, Op0, Op1); 16821 case MVT::v8f32: return fastEmit_X86ISD_UNPCKL_MVT_v8f32_rr(RetVT, Op0, Op1); 16822 case MVT::v16f32: return fastEmit_X86ISD_UNPCKL_MVT_v16f32_rr(RetVT, Op0, Op1); 16823 case MVT::v2f64: return fastEmit_X86ISD_UNPCKL_MVT_v2f64_rr(RetVT, Op0, Op1); 16824 case MVT::v4f64: return fastEmit_X86ISD_UNPCKL_MVT_v4f64_rr(RetVT, Op0, Op1); 16825 case MVT::v8f64: return fastEmit_X86ISD_UNPCKL_MVT_v8f64_rr(RetVT, Op0, Op1); 16826 default: return 0; 16827 } 16828} 16829 16830// FastEmit functions for X86ISD::VFCMULC. 16831 16832unsigned fastEmit_X86ISD_VFCMULC_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16833 if (RetVT.SimpleTy != MVT::v4f32) 16834 return 0; 16835 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 16836 return fastEmitInst_rr(X86::VFCMULCPHZ128rr, &X86::VR128XRegClass, Op0, Op1); 16837 } 16838 return 0; 16839} 16840 16841unsigned fastEmit_X86ISD_VFCMULC_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16842 if (RetVT.SimpleTy != MVT::v8f32) 16843 return 0; 16844 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 16845 return fastEmitInst_rr(X86::VFCMULCPHZ256rr, &X86::VR256XRegClass, Op0, Op1); 16846 } 16847 return 0; 16848} 16849 16850unsigned fastEmit_X86ISD_VFCMULC_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16851 if (RetVT.SimpleTy != MVT::v16f32) 16852 return 0; 16853 if ((Subtarget->hasFP16())) { 16854 return fastEmitInst_rr(X86::VFCMULCPHZrr, &X86::VR512RegClass, Op0, Op1); 16855 } 16856 return 0; 16857} 16858 16859unsigned fastEmit_X86ISD_VFCMULC_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 16860 switch (VT.SimpleTy) { 16861 case MVT::v4f32: return fastEmit_X86ISD_VFCMULC_MVT_v4f32_rr(RetVT, Op0, Op1); 16862 case MVT::v8f32: return fastEmit_X86ISD_VFCMULC_MVT_v8f32_rr(RetVT, Op0, Op1); 16863 case MVT::v16f32: return fastEmit_X86ISD_VFCMULC_MVT_v16f32_rr(RetVT, Op0, Op1); 16864 default: return 0; 16865 } 16866} 16867 16868// FastEmit functions for X86ISD::VFCMULCSH. 16869 16870unsigned fastEmit_X86ISD_VFCMULCSH_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16871 if (RetVT.SimpleTy != MVT::v4f32) 16872 return 0; 16873 if ((Subtarget->hasFP16())) { 16874 return fastEmitInst_rr(X86::VFCMULCSHZrr, &X86::VR128XRegClass, Op0, Op1); 16875 } 16876 return 0; 16877} 16878 16879unsigned fastEmit_X86ISD_VFCMULCSH_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 16880 switch (VT.SimpleTy) { 16881 case MVT::v4f32: return fastEmit_X86ISD_VFCMULCSH_MVT_v4f32_rr(RetVT, Op0, Op1); 16882 default: return 0; 16883 } 16884} 16885 16886// FastEmit functions for X86ISD::VFMULC. 16887 16888unsigned fastEmit_X86ISD_VFMULC_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16889 if (RetVT.SimpleTy != MVT::v4f32) 16890 return 0; 16891 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 16892 return fastEmitInst_rr(X86::VFMULCPHZ128rr, &X86::VR128XRegClass, Op0, Op1); 16893 } 16894 return 0; 16895} 16896 16897unsigned fastEmit_X86ISD_VFMULC_MVT_v8f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16898 if (RetVT.SimpleTy != MVT::v8f32) 16899 return 0; 16900 if ((Subtarget->hasFP16()) && (Subtarget->hasVLX())) { 16901 return fastEmitInst_rr(X86::VFMULCPHZ256rr, &X86::VR256XRegClass, Op0, Op1); 16902 } 16903 return 0; 16904} 16905 16906unsigned fastEmit_X86ISD_VFMULC_MVT_v16f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16907 if (RetVT.SimpleTy != MVT::v16f32) 16908 return 0; 16909 if ((Subtarget->hasFP16())) { 16910 return fastEmitInst_rr(X86::VFMULCPHZrr, &X86::VR512RegClass, Op0, Op1); 16911 } 16912 return 0; 16913} 16914 16915unsigned fastEmit_X86ISD_VFMULC_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 16916 switch (VT.SimpleTy) { 16917 case MVT::v4f32: return fastEmit_X86ISD_VFMULC_MVT_v4f32_rr(RetVT, Op0, Op1); 16918 case MVT::v8f32: return fastEmit_X86ISD_VFMULC_MVT_v8f32_rr(RetVT, Op0, Op1); 16919 case MVT::v16f32: return fastEmit_X86ISD_VFMULC_MVT_v16f32_rr(RetVT, Op0, Op1); 16920 default: return 0; 16921 } 16922} 16923 16924// FastEmit functions for X86ISD::VFMULCSH. 16925 16926unsigned fastEmit_X86ISD_VFMULCSH_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16927 if (RetVT.SimpleTy != MVT::v4f32) 16928 return 0; 16929 if ((Subtarget->hasFP16())) { 16930 return fastEmitInst_rr(X86::VFMULCSHZrr, &X86::VR128XRegClass, Op0, Op1); 16931 } 16932 return 0; 16933} 16934 16935unsigned fastEmit_X86ISD_VFMULCSH_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 16936 switch (VT.SimpleTy) { 16937 case MVT::v4f32: return fastEmit_X86ISD_VFMULCSH_MVT_v4f32_rr(RetVT, Op0, Op1); 16938 default: return 0; 16939 } 16940} 16941 16942// FastEmit functions for X86ISD::VP2INTERSECT. 16943 16944unsigned fastEmit_X86ISD_VP2INTERSECT_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16945 if (RetVT.SimpleTy != MVT::Untyped) 16946 return 0; 16947 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX()) && (Subtarget->hasVP2INTERSECT())) { 16948 return fastEmitInst_rr(X86::VP2INTERSECTDZ128rr, &X86::VK4PAIRRegClass, Op0, Op1); 16949 } 16950 return 0; 16951} 16952 16953unsigned fastEmit_X86ISD_VP2INTERSECT_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16954 if (RetVT.SimpleTy != MVT::Untyped) 16955 return 0; 16956 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX()) && (Subtarget->hasVP2INTERSECT())) { 16957 return fastEmitInst_rr(X86::VP2INTERSECTDZ256rr, &X86::VK8PAIRRegClass, Op0, Op1); 16958 } 16959 return 0; 16960} 16961 16962unsigned fastEmit_X86ISD_VP2INTERSECT_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16963 if (RetVT.SimpleTy != MVT::Untyped) 16964 return 0; 16965 if ((Subtarget->hasAVX512()) && (Subtarget->hasVP2INTERSECT())) { 16966 return fastEmitInst_rr(X86::VP2INTERSECTDZrr, &X86::VK16PAIRRegClass, Op0, Op1); 16967 } 16968 return 0; 16969} 16970 16971unsigned fastEmit_X86ISD_VP2INTERSECT_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16972 if (RetVT.SimpleTy != MVT::Untyped) 16973 return 0; 16974 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX()) && (Subtarget->hasVP2INTERSECT())) { 16975 return fastEmitInst_rr(X86::VP2INTERSECTQZ128rr, &X86::VK2PAIRRegClass, Op0, Op1); 16976 } 16977 return 0; 16978} 16979 16980unsigned fastEmit_X86ISD_VP2INTERSECT_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16981 if (RetVT.SimpleTy != MVT::Untyped) 16982 return 0; 16983 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX()) && (Subtarget->hasVP2INTERSECT())) { 16984 return fastEmitInst_rr(X86::VP2INTERSECTQZ256rr, &X86::VK4PAIRRegClass, Op0, Op1); 16985 } 16986 return 0; 16987} 16988 16989unsigned fastEmit_X86ISD_VP2INTERSECT_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 16990 if (RetVT.SimpleTy != MVT::Untyped) 16991 return 0; 16992 if ((Subtarget->hasAVX512()) && (Subtarget->hasVP2INTERSECT())) { 16993 return fastEmitInst_rr(X86::VP2INTERSECTQZrr, &X86::VK8PAIRRegClass, Op0, Op1); 16994 } 16995 return 0; 16996} 16997 16998unsigned fastEmit_X86ISD_VP2INTERSECT_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 16999 switch (VT.SimpleTy) { 17000 case MVT::v4i32: return fastEmit_X86ISD_VP2INTERSECT_MVT_v4i32_rr(RetVT, Op0, Op1); 17001 case MVT::v8i32: return fastEmit_X86ISD_VP2INTERSECT_MVT_v8i32_rr(RetVT, Op0, Op1); 17002 case MVT::v16i32: return fastEmit_X86ISD_VP2INTERSECT_MVT_v16i32_rr(RetVT, Op0, Op1); 17003 case MVT::v2i64: return fastEmit_X86ISD_VP2INTERSECT_MVT_v2i64_rr(RetVT, Op0, Op1); 17004 case MVT::v4i64: return fastEmit_X86ISD_VP2INTERSECT_MVT_v4i64_rr(RetVT, Op0, Op1); 17005 case MVT::v8i64: return fastEmit_X86ISD_VP2INTERSECT_MVT_v8i64_rr(RetVT, Op0, Op1); 17006 default: return 0; 17007 } 17008} 17009 17010// FastEmit functions for X86ISD::VPERMV. 17011 17012unsigned fastEmit_X86ISD_VPERMV_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17013 if (RetVT.SimpleTy != MVT::v16i8) 17014 return 0; 17015 if ((Subtarget->hasVBMI()) && (Subtarget->hasVLX())) { 17016 return fastEmitInst_rr(X86::VPERMBZ128rr, &X86::VR128XRegClass, Op0, Op1); 17017 } 17018 return 0; 17019} 17020 17021unsigned fastEmit_X86ISD_VPERMV_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17022 if (RetVT.SimpleTy != MVT::v32i8) 17023 return 0; 17024 if ((Subtarget->hasVBMI()) && (Subtarget->hasVLX())) { 17025 return fastEmitInst_rr(X86::VPERMBZ256rr, &X86::VR256XRegClass, Op0, Op1); 17026 } 17027 return 0; 17028} 17029 17030unsigned fastEmit_X86ISD_VPERMV_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17031 if (RetVT.SimpleTy != MVT::v64i8) 17032 return 0; 17033 if ((Subtarget->hasVBMI())) { 17034 return fastEmitInst_rr(X86::VPERMBZrr, &X86::VR512RegClass, Op0, Op1); 17035 } 17036 return 0; 17037} 17038 17039unsigned fastEmit_X86ISD_VPERMV_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17040 if (RetVT.SimpleTy != MVT::v8i16) 17041 return 0; 17042 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 17043 return fastEmitInst_rr(X86::VPERMWZ128rr, &X86::VR128XRegClass, Op0, Op1); 17044 } 17045 return 0; 17046} 17047 17048unsigned fastEmit_X86ISD_VPERMV_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17049 if (RetVT.SimpleTy != MVT::v16i16) 17050 return 0; 17051 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 17052 return fastEmitInst_rr(X86::VPERMWZ256rr, &X86::VR256XRegClass, Op0, Op1); 17053 } 17054 return 0; 17055} 17056 17057unsigned fastEmit_X86ISD_VPERMV_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17058 if (RetVT.SimpleTy != MVT::v32i16) 17059 return 0; 17060 if ((Subtarget->hasBWI())) { 17061 return fastEmitInst_rr(X86::VPERMWZrr, &X86::VR512RegClass, Op0, Op1); 17062 } 17063 return 0; 17064} 17065 17066unsigned fastEmit_X86ISD_VPERMV_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17067 if (RetVT.SimpleTy != MVT::v8i32) 17068 return 0; 17069 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 17070 return fastEmitInst_rr(X86::VPERMDZ256rr, &X86::VR256XRegClass, Op0, Op1); 17071 } 17072 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 17073 return fastEmitInst_rr(X86::VPERMDYrr, &X86::VR256RegClass, Op0, Op1); 17074 } 17075 return 0; 17076} 17077 17078unsigned fastEmit_X86ISD_VPERMV_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17079 if (RetVT.SimpleTy != MVT::v16i32) 17080 return 0; 17081 if ((Subtarget->hasAVX512())) { 17082 return fastEmitInst_rr(X86::VPERMDZrr, &X86::VR512RegClass, Op0, Op1); 17083 } 17084 return 0; 17085} 17086 17087unsigned fastEmit_X86ISD_VPERMV_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17088 if (RetVT.SimpleTy != MVT::v4i64) 17089 return 0; 17090 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 17091 return fastEmitInst_rr(X86::VPERMQZ256rr, &X86::VR256XRegClass, Op0, Op1); 17092 } 17093 return 0; 17094} 17095 17096unsigned fastEmit_X86ISD_VPERMV_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17097 if (RetVT.SimpleTy != MVT::v8i64) 17098 return 0; 17099 if ((Subtarget->hasAVX512())) { 17100 return fastEmitInst_rr(X86::VPERMQZrr, &X86::VR512RegClass, Op0, Op1); 17101 } 17102 return 0; 17103} 17104 17105unsigned fastEmit_X86ISD_VPERMV_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 17106 switch (VT.SimpleTy) { 17107 case MVT::v16i8: return fastEmit_X86ISD_VPERMV_MVT_v16i8_rr(RetVT, Op0, Op1); 17108 case MVT::v32i8: return fastEmit_X86ISD_VPERMV_MVT_v32i8_rr(RetVT, Op0, Op1); 17109 case MVT::v64i8: return fastEmit_X86ISD_VPERMV_MVT_v64i8_rr(RetVT, Op0, Op1); 17110 case MVT::v8i16: return fastEmit_X86ISD_VPERMV_MVT_v8i16_rr(RetVT, Op0, Op1); 17111 case MVT::v16i16: return fastEmit_X86ISD_VPERMV_MVT_v16i16_rr(RetVT, Op0, Op1); 17112 case MVT::v32i16: return fastEmit_X86ISD_VPERMV_MVT_v32i16_rr(RetVT, Op0, Op1); 17113 case MVT::v8i32: return fastEmit_X86ISD_VPERMV_MVT_v8i32_rr(RetVT, Op0, Op1); 17114 case MVT::v16i32: return fastEmit_X86ISD_VPERMV_MVT_v16i32_rr(RetVT, Op0, Op1); 17115 case MVT::v4i64: return fastEmit_X86ISD_VPERMV_MVT_v4i64_rr(RetVT, Op0, Op1); 17116 case MVT::v8i64: return fastEmit_X86ISD_VPERMV_MVT_v8i64_rr(RetVT, Op0, Op1); 17117 default: return 0; 17118 } 17119} 17120 17121// FastEmit functions for X86ISD::VPMADDUBSW. 17122 17123unsigned fastEmit_X86ISD_VPMADDUBSW_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17124 if (RetVT.SimpleTy != MVT::v8i16) 17125 return 0; 17126 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 17127 return fastEmitInst_rr(X86::VPMADDUBSWZ128rr, &X86::VR128XRegClass, Op0, Op1); 17128 } 17129 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) { 17130 return fastEmitInst_rr(X86::PMADDUBSWrr, &X86::VR128RegClass, Op0, Op1); 17131 } 17132 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 17133 return fastEmitInst_rr(X86::VPMADDUBSWrr, &X86::VR128RegClass, Op0, Op1); 17134 } 17135 return 0; 17136} 17137 17138unsigned fastEmit_X86ISD_VPMADDUBSW_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17139 if (RetVT.SimpleTy != MVT::v16i16) 17140 return 0; 17141 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 17142 return fastEmitInst_rr(X86::VPMADDUBSWZ256rr, &X86::VR256XRegClass, Op0, Op1); 17143 } 17144 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 17145 return fastEmitInst_rr(X86::VPMADDUBSWYrr, &X86::VR256RegClass, Op0, Op1); 17146 } 17147 return 0; 17148} 17149 17150unsigned fastEmit_X86ISD_VPMADDUBSW_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17151 if (RetVT.SimpleTy != MVT::v32i16) 17152 return 0; 17153 if ((Subtarget->hasBWI())) { 17154 return fastEmitInst_rr(X86::VPMADDUBSWZrr, &X86::VR512RegClass, Op0, Op1); 17155 } 17156 return 0; 17157} 17158 17159unsigned fastEmit_X86ISD_VPMADDUBSW_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 17160 switch (VT.SimpleTy) { 17161 case MVT::v16i8: return fastEmit_X86ISD_VPMADDUBSW_MVT_v16i8_rr(RetVT, Op0, Op1); 17162 case MVT::v32i8: return fastEmit_X86ISD_VPMADDUBSW_MVT_v32i8_rr(RetVT, Op0, Op1); 17163 case MVT::v64i8: return fastEmit_X86ISD_VPMADDUBSW_MVT_v64i8_rr(RetVT, Op0, Op1); 17164 default: return 0; 17165 } 17166} 17167 17168// FastEmit functions for X86ISD::VPMADDWD. 17169 17170unsigned fastEmit_X86ISD_VPMADDWD_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17171 if (RetVT.SimpleTy != MVT::v4i32) 17172 return 0; 17173 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 17174 return fastEmitInst_rr(X86::VPMADDWDZ128rr, &X86::VR128XRegClass, Op0, Op1); 17175 } 17176 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 17177 return fastEmitInst_rr(X86::PMADDWDrr, &X86::VR128RegClass, Op0, Op1); 17178 } 17179 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 17180 return fastEmitInst_rr(X86::VPMADDWDrr, &X86::VR128RegClass, Op0, Op1); 17181 } 17182 return 0; 17183} 17184 17185unsigned fastEmit_X86ISD_VPMADDWD_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17186 if (RetVT.SimpleTy != MVT::v8i32) 17187 return 0; 17188 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 17189 return fastEmitInst_rr(X86::VPMADDWDZ256rr, &X86::VR256XRegClass, Op0, Op1); 17190 } 17191 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 17192 return fastEmitInst_rr(X86::VPMADDWDYrr, &X86::VR256RegClass, Op0, Op1); 17193 } 17194 return 0; 17195} 17196 17197unsigned fastEmit_X86ISD_VPMADDWD_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17198 if (RetVT.SimpleTy != MVT::v16i32) 17199 return 0; 17200 if ((Subtarget->hasBWI())) { 17201 return fastEmitInst_rr(X86::VPMADDWDZrr, &X86::VR512RegClass, Op0, Op1); 17202 } 17203 return 0; 17204} 17205 17206unsigned fastEmit_X86ISD_VPMADDWD_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 17207 switch (VT.SimpleTy) { 17208 case MVT::v8i16: return fastEmit_X86ISD_VPMADDWD_MVT_v8i16_rr(RetVT, Op0, Op1); 17209 case MVT::v16i16: return fastEmit_X86ISD_VPMADDWD_MVT_v16i16_rr(RetVT, Op0, Op1); 17210 case MVT::v32i16: return fastEmit_X86ISD_VPMADDWD_MVT_v32i16_rr(RetVT, Op0, Op1); 17211 default: return 0; 17212 } 17213} 17214 17215// FastEmit functions for X86ISD::VPSHA. 17216 17217unsigned fastEmit_X86ISD_VPSHA_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17218 if (RetVT.SimpleTy != MVT::v16i8) 17219 return 0; 17220 if ((Subtarget->hasXOP())) { 17221 return fastEmitInst_rr(X86::VPSHABrr, &X86::VR128RegClass, Op0, Op1); 17222 } 17223 return 0; 17224} 17225 17226unsigned fastEmit_X86ISD_VPSHA_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17227 if (RetVT.SimpleTy != MVT::v8i16) 17228 return 0; 17229 if ((Subtarget->hasXOP())) { 17230 return fastEmitInst_rr(X86::VPSHAWrr, &X86::VR128RegClass, Op0, Op1); 17231 } 17232 return 0; 17233} 17234 17235unsigned fastEmit_X86ISD_VPSHA_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17236 if (RetVT.SimpleTy != MVT::v4i32) 17237 return 0; 17238 if ((Subtarget->hasXOP())) { 17239 return fastEmitInst_rr(X86::VPSHADrr, &X86::VR128RegClass, Op0, Op1); 17240 } 17241 return 0; 17242} 17243 17244unsigned fastEmit_X86ISD_VPSHA_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17245 if (RetVT.SimpleTy != MVT::v2i64) 17246 return 0; 17247 if ((Subtarget->hasXOP())) { 17248 return fastEmitInst_rr(X86::VPSHAQrr, &X86::VR128RegClass, Op0, Op1); 17249 } 17250 return 0; 17251} 17252 17253unsigned fastEmit_X86ISD_VPSHA_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 17254 switch (VT.SimpleTy) { 17255 case MVT::v16i8: return fastEmit_X86ISD_VPSHA_MVT_v16i8_rr(RetVT, Op0, Op1); 17256 case MVT::v8i16: return fastEmit_X86ISD_VPSHA_MVT_v8i16_rr(RetVT, Op0, Op1); 17257 case MVT::v4i32: return fastEmit_X86ISD_VPSHA_MVT_v4i32_rr(RetVT, Op0, Op1); 17258 case MVT::v2i64: return fastEmit_X86ISD_VPSHA_MVT_v2i64_rr(RetVT, Op0, Op1); 17259 default: return 0; 17260 } 17261} 17262 17263// FastEmit functions for X86ISD::VPSHL. 17264 17265unsigned fastEmit_X86ISD_VPSHL_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17266 if (RetVT.SimpleTy != MVT::v16i8) 17267 return 0; 17268 if ((Subtarget->hasXOP())) { 17269 return fastEmitInst_rr(X86::VPSHLBrr, &X86::VR128RegClass, Op0, Op1); 17270 } 17271 return 0; 17272} 17273 17274unsigned fastEmit_X86ISD_VPSHL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17275 if (RetVT.SimpleTy != MVT::v8i16) 17276 return 0; 17277 if ((Subtarget->hasXOP())) { 17278 return fastEmitInst_rr(X86::VPSHLWrr, &X86::VR128RegClass, Op0, Op1); 17279 } 17280 return 0; 17281} 17282 17283unsigned fastEmit_X86ISD_VPSHL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17284 if (RetVT.SimpleTy != MVT::v4i32) 17285 return 0; 17286 if ((Subtarget->hasXOP())) { 17287 return fastEmitInst_rr(X86::VPSHLDrr, &X86::VR128RegClass, Op0, Op1); 17288 } 17289 return 0; 17290} 17291 17292unsigned fastEmit_X86ISD_VPSHL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17293 if (RetVT.SimpleTy != MVT::v2i64) 17294 return 0; 17295 if ((Subtarget->hasXOP())) { 17296 return fastEmitInst_rr(X86::VPSHLQrr, &X86::VR128RegClass, Op0, Op1); 17297 } 17298 return 0; 17299} 17300 17301unsigned fastEmit_X86ISD_VPSHL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 17302 switch (VT.SimpleTy) { 17303 case MVT::v16i8: return fastEmit_X86ISD_VPSHL_MVT_v16i8_rr(RetVT, Op0, Op1); 17304 case MVT::v8i16: return fastEmit_X86ISD_VPSHL_MVT_v8i16_rr(RetVT, Op0, Op1); 17305 case MVT::v4i32: return fastEmit_X86ISD_VPSHL_MVT_v4i32_rr(RetVT, Op0, Op1); 17306 case MVT::v2i64: return fastEmit_X86ISD_VPSHL_MVT_v2i64_rr(RetVT, Op0, Op1); 17307 default: return 0; 17308 } 17309} 17310 17311// FastEmit functions for X86ISD::VPSHUFBITQMB. 17312 17313unsigned fastEmit_X86ISD_VPSHUFBITQMB_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17314 if (RetVT.SimpleTy != MVT::v16i1) 17315 return 0; 17316 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) { 17317 return fastEmitInst_rr(X86::VPSHUFBITQMBZ128rr, &X86::VK16RegClass, Op0, Op1); 17318 } 17319 return 0; 17320} 17321 17322unsigned fastEmit_X86ISD_VPSHUFBITQMB_MVT_v32i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17323 if (RetVT.SimpleTy != MVT::v32i1) 17324 return 0; 17325 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) { 17326 return fastEmitInst_rr(X86::VPSHUFBITQMBZ256rr, &X86::VK32RegClass, Op0, Op1); 17327 } 17328 return 0; 17329} 17330 17331unsigned fastEmit_X86ISD_VPSHUFBITQMB_MVT_v64i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17332 if (RetVT.SimpleTy != MVT::v64i1) 17333 return 0; 17334 if ((Subtarget->hasBITALG())) { 17335 return fastEmitInst_rr(X86::VPSHUFBITQMBZrr, &X86::VK64RegClass, Op0, Op1); 17336 } 17337 return 0; 17338} 17339 17340unsigned fastEmit_X86ISD_VPSHUFBITQMB_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 17341 switch (VT.SimpleTy) { 17342 case MVT::v16i8: return fastEmit_X86ISD_VPSHUFBITQMB_MVT_v16i8_rr(RetVT, Op0, Op1); 17343 case MVT::v32i8: return fastEmit_X86ISD_VPSHUFBITQMB_MVT_v32i8_rr(RetVT, Op0, Op1); 17344 case MVT::v64i8: return fastEmit_X86ISD_VPSHUFBITQMB_MVT_v64i8_rr(RetVT, Op0, Op1); 17345 default: return 0; 17346 } 17347} 17348 17349// FastEmit functions for X86ISD::VSHL. 17350 17351unsigned fastEmit_X86ISD_VSHL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17352 if (RetVT.SimpleTy != MVT::v8i16) 17353 return 0; 17354 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 17355 return fastEmitInst_rr(X86::VPSLLWZ128rr, &X86::VR128XRegClass, Op0, Op1); 17356 } 17357 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 17358 return fastEmitInst_rr(X86::PSLLWrr, &X86::VR128RegClass, Op0, Op1); 17359 } 17360 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 17361 return fastEmitInst_rr(X86::VPSLLWrr, &X86::VR128RegClass, Op0, Op1); 17362 } 17363 return 0; 17364} 17365 17366unsigned fastEmit_X86ISD_VSHL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17367 if (RetVT.SimpleTy != MVT::v4i32) 17368 return 0; 17369 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 17370 return fastEmitInst_rr(X86::VPSLLDZ128rr, &X86::VR128XRegClass, Op0, Op1); 17371 } 17372 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 17373 return fastEmitInst_rr(X86::PSLLDrr, &X86::VR128RegClass, Op0, Op1); 17374 } 17375 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 17376 return fastEmitInst_rr(X86::VPSLLDrr, &X86::VR128RegClass, Op0, Op1); 17377 } 17378 return 0; 17379} 17380 17381unsigned fastEmit_X86ISD_VSHL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17382 if (RetVT.SimpleTy != MVT::v2i64) 17383 return 0; 17384 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 17385 return fastEmitInst_rr(X86::VPSLLQZ128rr, &X86::VR128XRegClass, Op0, Op1); 17386 } 17387 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 17388 return fastEmitInst_rr(X86::PSLLQrr, &X86::VR128RegClass, Op0, Op1); 17389 } 17390 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 17391 return fastEmitInst_rr(X86::VPSLLQrr, &X86::VR128RegClass, Op0, Op1); 17392 } 17393 return 0; 17394} 17395 17396unsigned fastEmit_X86ISD_VSHL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 17397 switch (VT.SimpleTy) { 17398 case MVT::v8i16: return fastEmit_X86ISD_VSHL_MVT_v8i16_rr(RetVT, Op0, Op1); 17399 case MVT::v4i32: return fastEmit_X86ISD_VSHL_MVT_v4i32_rr(RetVT, Op0, Op1); 17400 case MVT::v2i64: return fastEmit_X86ISD_VSHL_MVT_v2i64_rr(RetVT, Op0, Op1); 17401 default: return 0; 17402 } 17403} 17404 17405// FastEmit functions for X86ISD::VSHLV. 17406 17407unsigned fastEmit_X86ISD_VSHLV_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17408 if (RetVT.SimpleTy != MVT::v8i16) 17409 return 0; 17410 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 17411 return fastEmitInst_rr(X86::VPSLLVWZ128rr, &X86::VR128XRegClass, Op0, Op1); 17412 } 17413 return 0; 17414} 17415 17416unsigned fastEmit_X86ISD_VSHLV_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17417 if (RetVT.SimpleTy != MVT::v16i16) 17418 return 0; 17419 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 17420 return fastEmitInst_rr(X86::VPSLLVWZ256rr, &X86::VR256XRegClass, Op0, Op1); 17421 } 17422 return 0; 17423} 17424 17425unsigned fastEmit_X86ISD_VSHLV_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17426 if (RetVT.SimpleTy != MVT::v32i16) 17427 return 0; 17428 if ((Subtarget->hasBWI())) { 17429 return fastEmitInst_rr(X86::VPSLLVWZrr, &X86::VR512RegClass, Op0, Op1); 17430 } 17431 return 0; 17432} 17433 17434unsigned fastEmit_X86ISD_VSHLV_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17435 if (RetVT.SimpleTy != MVT::v4i32) 17436 return 0; 17437 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 17438 return fastEmitInst_rr(X86::VPSLLVDZ128rr, &X86::VR128XRegClass, Op0, Op1); 17439 } 17440 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 17441 return fastEmitInst_rr(X86::VPSLLVDrr, &X86::VR128RegClass, Op0, Op1); 17442 } 17443 return 0; 17444} 17445 17446unsigned fastEmit_X86ISD_VSHLV_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17447 if (RetVT.SimpleTy != MVT::v8i32) 17448 return 0; 17449 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 17450 return fastEmitInst_rr(X86::VPSLLVDZ256rr, &X86::VR256XRegClass, Op0, Op1); 17451 } 17452 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 17453 return fastEmitInst_rr(X86::VPSLLVDYrr, &X86::VR256RegClass, Op0, Op1); 17454 } 17455 return 0; 17456} 17457 17458unsigned fastEmit_X86ISD_VSHLV_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17459 if (RetVT.SimpleTy != MVT::v16i32) 17460 return 0; 17461 if ((Subtarget->hasAVX512())) { 17462 return fastEmitInst_rr(X86::VPSLLVDZrr, &X86::VR512RegClass, Op0, Op1); 17463 } 17464 return 0; 17465} 17466 17467unsigned fastEmit_X86ISD_VSHLV_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17468 if (RetVT.SimpleTy != MVT::v2i64) 17469 return 0; 17470 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 17471 return fastEmitInst_rr(X86::VPSLLVQZ128rr, &X86::VR128XRegClass, Op0, Op1); 17472 } 17473 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 17474 return fastEmitInst_rr(X86::VPSLLVQrr, &X86::VR128RegClass, Op0, Op1); 17475 } 17476 return 0; 17477} 17478 17479unsigned fastEmit_X86ISD_VSHLV_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17480 if (RetVT.SimpleTy != MVT::v4i64) 17481 return 0; 17482 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 17483 return fastEmitInst_rr(X86::VPSLLVQZ256rr, &X86::VR256XRegClass, Op0, Op1); 17484 } 17485 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 17486 return fastEmitInst_rr(X86::VPSLLVQYrr, &X86::VR256RegClass, Op0, Op1); 17487 } 17488 return 0; 17489} 17490 17491unsigned fastEmit_X86ISD_VSHLV_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17492 if (RetVT.SimpleTy != MVT::v8i64) 17493 return 0; 17494 if ((Subtarget->hasAVX512())) { 17495 return fastEmitInst_rr(X86::VPSLLVQZrr, &X86::VR512RegClass, Op0, Op1); 17496 } 17497 return 0; 17498} 17499 17500unsigned fastEmit_X86ISD_VSHLV_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 17501 switch (VT.SimpleTy) { 17502 case MVT::v8i16: return fastEmit_X86ISD_VSHLV_MVT_v8i16_rr(RetVT, Op0, Op1); 17503 case MVT::v16i16: return fastEmit_X86ISD_VSHLV_MVT_v16i16_rr(RetVT, Op0, Op1); 17504 case MVT::v32i16: return fastEmit_X86ISD_VSHLV_MVT_v32i16_rr(RetVT, Op0, Op1); 17505 case MVT::v4i32: return fastEmit_X86ISD_VSHLV_MVT_v4i32_rr(RetVT, Op0, Op1); 17506 case MVT::v8i32: return fastEmit_X86ISD_VSHLV_MVT_v8i32_rr(RetVT, Op0, Op1); 17507 case MVT::v16i32: return fastEmit_X86ISD_VSHLV_MVT_v16i32_rr(RetVT, Op0, Op1); 17508 case MVT::v2i64: return fastEmit_X86ISD_VSHLV_MVT_v2i64_rr(RetVT, Op0, Op1); 17509 case MVT::v4i64: return fastEmit_X86ISD_VSHLV_MVT_v4i64_rr(RetVT, Op0, Op1); 17510 case MVT::v8i64: return fastEmit_X86ISD_VSHLV_MVT_v8i64_rr(RetVT, Op0, Op1); 17511 default: return 0; 17512 } 17513} 17514 17515// FastEmit functions for X86ISD::VSRA. 17516 17517unsigned fastEmit_X86ISD_VSRA_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17518 if (RetVT.SimpleTy != MVT::v8i16) 17519 return 0; 17520 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 17521 return fastEmitInst_rr(X86::VPSRAWZ128rr, &X86::VR128XRegClass, Op0, Op1); 17522 } 17523 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 17524 return fastEmitInst_rr(X86::PSRAWrr, &X86::VR128RegClass, Op0, Op1); 17525 } 17526 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 17527 return fastEmitInst_rr(X86::VPSRAWrr, &X86::VR128RegClass, Op0, Op1); 17528 } 17529 return 0; 17530} 17531 17532unsigned fastEmit_X86ISD_VSRA_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17533 if (RetVT.SimpleTy != MVT::v4i32) 17534 return 0; 17535 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 17536 return fastEmitInst_rr(X86::VPSRADZ128rr, &X86::VR128XRegClass, Op0, Op1); 17537 } 17538 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 17539 return fastEmitInst_rr(X86::PSRADrr, &X86::VR128RegClass, Op0, Op1); 17540 } 17541 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 17542 return fastEmitInst_rr(X86::VPSRADrr, &X86::VR128RegClass, Op0, Op1); 17543 } 17544 return 0; 17545} 17546 17547unsigned fastEmit_X86ISD_VSRA_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17548 if (RetVT.SimpleTy != MVT::v2i64) 17549 return 0; 17550 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 17551 return fastEmitInst_rr(X86::VPSRAQZ128rr, &X86::VR128XRegClass, Op0, Op1); 17552 } 17553 return 0; 17554} 17555 17556unsigned fastEmit_X86ISD_VSRA_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 17557 switch (VT.SimpleTy) { 17558 case MVT::v8i16: return fastEmit_X86ISD_VSRA_MVT_v8i16_rr(RetVT, Op0, Op1); 17559 case MVT::v4i32: return fastEmit_X86ISD_VSRA_MVT_v4i32_rr(RetVT, Op0, Op1); 17560 case MVT::v2i64: return fastEmit_X86ISD_VSRA_MVT_v2i64_rr(RetVT, Op0, Op1); 17561 default: return 0; 17562 } 17563} 17564 17565// FastEmit functions for X86ISD::VSRAV. 17566 17567unsigned fastEmit_X86ISD_VSRAV_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17568 if (RetVT.SimpleTy != MVT::v8i16) 17569 return 0; 17570 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 17571 return fastEmitInst_rr(X86::VPSRAVWZ128rr, &X86::VR128XRegClass, Op0, Op1); 17572 } 17573 return 0; 17574} 17575 17576unsigned fastEmit_X86ISD_VSRAV_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17577 if (RetVT.SimpleTy != MVT::v16i16) 17578 return 0; 17579 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 17580 return fastEmitInst_rr(X86::VPSRAVWZ256rr, &X86::VR256XRegClass, Op0, Op1); 17581 } 17582 return 0; 17583} 17584 17585unsigned fastEmit_X86ISD_VSRAV_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17586 if (RetVT.SimpleTy != MVT::v32i16) 17587 return 0; 17588 if ((Subtarget->hasBWI())) { 17589 return fastEmitInst_rr(X86::VPSRAVWZrr, &X86::VR512RegClass, Op0, Op1); 17590 } 17591 return 0; 17592} 17593 17594unsigned fastEmit_X86ISD_VSRAV_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17595 if (RetVT.SimpleTy != MVT::v4i32) 17596 return 0; 17597 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 17598 return fastEmitInst_rr(X86::VPSRAVDZ128rr, &X86::VR128XRegClass, Op0, Op1); 17599 } 17600 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 17601 return fastEmitInst_rr(X86::VPSRAVDrr, &X86::VR128RegClass, Op0, Op1); 17602 } 17603 return 0; 17604} 17605 17606unsigned fastEmit_X86ISD_VSRAV_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17607 if (RetVT.SimpleTy != MVT::v8i32) 17608 return 0; 17609 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 17610 return fastEmitInst_rr(X86::VPSRAVDZ256rr, &X86::VR256XRegClass, Op0, Op1); 17611 } 17612 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 17613 return fastEmitInst_rr(X86::VPSRAVDYrr, &X86::VR256RegClass, Op0, Op1); 17614 } 17615 return 0; 17616} 17617 17618unsigned fastEmit_X86ISD_VSRAV_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17619 if (RetVT.SimpleTy != MVT::v16i32) 17620 return 0; 17621 if ((Subtarget->hasAVX512())) { 17622 return fastEmitInst_rr(X86::VPSRAVDZrr, &X86::VR512RegClass, Op0, Op1); 17623 } 17624 return 0; 17625} 17626 17627unsigned fastEmit_X86ISD_VSRAV_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17628 if (RetVT.SimpleTy != MVT::v2i64) 17629 return 0; 17630 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 17631 return fastEmitInst_rr(X86::VPSRAVQZ128rr, &X86::VR128XRegClass, Op0, Op1); 17632 } 17633 return 0; 17634} 17635 17636unsigned fastEmit_X86ISD_VSRAV_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17637 if (RetVT.SimpleTy != MVT::v4i64) 17638 return 0; 17639 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 17640 return fastEmitInst_rr(X86::VPSRAVQZ256rr, &X86::VR256XRegClass, Op0, Op1); 17641 } 17642 return 0; 17643} 17644 17645unsigned fastEmit_X86ISD_VSRAV_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17646 if (RetVT.SimpleTy != MVT::v8i64) 17647 return 0; 17648 if ((Subtarget->hasAVX512())) { 17649 return fastEmitInst_rr(X86::VPSRAVQZrr, &X86::VR512RegClass, Op0, Op1); 17650 } 17651 return 0; 17652} 17653 17654unsigned fastEmit_X86ISD_VSRAV_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 17655 switch (VT.SimpleTy) { 17656 case MVT::v8i16: return fastEmit_X86ISD_VSRAV_MVT_v8i16_rr(RetVT, Op0, Op1); 17657 case MVT::v16i16: return fastEmit_X86ISD_VSRAV_MVT_v16i16_rr(RetVT, Op0, Op1); 17658 case MVT::v32i16: return fastEmit_X86ISD_VSRAV_MVT_v32i16_rr(RetVT, Op0, Op1); 17659 case MVT::v4i32: return fastEmit_X86ISD_VSRAV_MVT_v4i32_rr(RetVT, Op0, Op1); 17660 case MVT::v8i32: return fastEmit_X86ISD_VSRAV_MVT_v8i32_rr(RetVT, Op0, Op1); 17661 case MVT::v16i32: return fastEmit_X86ISD_VSRAV_MVT_v16i32_rr(RetVT, Op0, Op1); 17662 case MVT::v2i64: return fastEmit_X86ISD_VSRAV_MVT_v2i64_rr(RetVT, Op0, Op1); 17663 case MVT::v4i64: return fastEmit_X86ISD_VSRAV_MVT_v4i64_rr(RetVT, Op0, Op1); 17664 case MVT::v8i64: return fastEmit_X86ISD_VSRAV_MVT_v8i64_rr(RetVT, Op0, Op1); 17665 default: return 0; 17666 } 17667} 17668 17669// FastEmit functions for X86ISD::VSRL. 17670 17671unsigned fastEmit_X86ISD_VSRL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17672 if (RetVT.SimpleTy != MVT::v8i16) 17673 return 0; 17674 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 17675 return fastEmitInst_rr(X86::VPSRLWZ128rr, &X86::VR128XRegClass, Op0, Op1); 17676 } 17677 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 17678 return fastEmitInst_rr(X86::PSRLWrr, &X86::VR128RegClass, Op0, Op1); 17679 } 17680 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 17681 return fastEmitInst_rr(X86::VPSRLWrr, &X86::VR128RegClass, Op0, Op1); 17682 } 17683 return 0; 17684} 17685 17686unsigned fastEmit_X86ISD_VSRL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17687 if (RetVT.SimpleTy != MVT::v4i32) 17688 return 0; 17689 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 17690 return fastEmitInst_rr(X86::VPSRLDZ128rr, &X86::VR128XRegClass, Op0, Op1); 17691 } 17692 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 17693 return fastEmitInst_rr(X86::PSRLDrr, &X86::VR128RegClass, Op0, Op1); 17694 } 17695 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 17696 return fastEmitInst_rr(X86::VPSRLDrr, &X86::VR128RegClass, Op0, Op1); 17697 } 17698 return 0; 17699} 17700 17701unsigned fastEmit_X86ISD_VSRL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17702 if (RetVT.SimpleTy != MVT::v2i64) 17703 return 0; 17704 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 17705 return fastEmitInst_rr(X86::VPSRLQZ128rr, &X86::VR128XRegClass, Op0, Op1); 17706 } 17707 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 17708 return fastEmitInst_rr(X86::PSRLQrr, &X86::VR128RegClass, Op0, Op1); 17709 } 17710 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 17711 return fastEmitInst_rr(X86::VPSRLQrr, &X86::VR128RegClass, Op0, Op1); 17712 } 17713 return 0; 17714} 17715 17716unsigned fastEmit_X86ISD_VSRL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 17717 switch (VT.SimpleTy) { 17718 case MVT::v8i16: return fastEmit_X86ISD_VSRL_MVT_v8i16_rr(RetVT, Op0, Op1); 17719 case MVT::v4i32: return fastEmit_X86ISD_VSRL_MVT_v4i32_rr(RetVT, Op0, Op1); 17720 case MVT::v2i64: return fastEmit_X86ISD_VSRL_MVT_v2i64_rr(RetVT, Op0, Op1); 17721 default: return 0; 17722 } 17723} 17724 17725// FastEmit functions for X86ISD::VSRLV. 17726 17727unsigned fastEmit_X86ISD_VSRLV_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17728 if (RetVT.SimpleTy != MVT::v8i16) 17729 return 0; 17730 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 17731 return fastEmitInst_rr(X86::VPSRLVWZ128rr, &X86::VR128XRegClass, Op0, Op1); 17732 } 17733 return 0; 17734} 17735 17736unsigned fastEmit_X86ISD_VSRLV_MVT_v16i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17737 if (RetVT.SimpleTy != MVT::v16i16) 17738 return 0; 17739 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 17740 return fastEmitInst_rr(X86::VPSRLVWZ256rr, &X86::VR256XRegClass, Op0, Op1); 17741 } 17742 return 0; 17743} 17744 17745unsigned fastEmit_X86ISD_VSRLV_MVT_v32i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17746 if (RetVT.SimpleTy != MVT::v32i16) 17747 return 0; 17748 if ((Subtarget->hasBWI())) { 17749 return fastEmitInst_rr(X86::VPSRLVWZrr, &X86::VR512RegClass, Op0, Op1); 17750 } 17751 return 0; 17752} 17753 17754unsigned fastEmit_X86ISD_VSRLV_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17755 if (RetVT.SimpleTy != MVT::v4i32) 17756 return 0; 17757 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 17758 return fastEmitInst_rr(X86::VPSRLVDZ128rr, &X86::VR128XRegClass, Op0, Op1); 17759 } 17760 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 17761 return fastEmitInst_rr(X86::VPSRLVDrr, &X86::VR128RegClass, Op0, Op1); 17762 } 17763 return 0; 17764} 17765 17766unsigned fastEmit_X86ISD_VSRLV_MVT_v8i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17767 if (RetVT.SimpleTy != MVT::v8i32) 17768 return 0; 17769 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 17770 return fastEmitInst_rr(X86::VPSRLVDZ256rr, &X86::VR256XRegClass, Op0, Op1); 17771 } 17772 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 17773 return fastEmitInst_rr(X86::VPSRLVDYrr, &X86::VR256RegClass, Op0, Op1); 17774 } 17775 return 0; 17776} 17777 17778unsigned fastEmit_X86ISD_VSRLV_MVT_v16i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17779 if (RetVT.SimpleTy != MVT::v16i32) 17780 return 0; 17781 if ((Subtarget->hasAVX512())) { 17782 return fastEmitInst_rr(X86::VPSRLVDZrr, &X86::VR512RegClass, Op0, Op1); 17783 } 17784 return 0; 17785} 17786 17787unsigned fastEmit_X86ISD_VSRLV_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17788 if (RetVT.SimpleTy != MVT::v2i64) 17789 return 0; 17790 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 17791 return fastEmitInst_rr(X86::VPSRLVQZ128rr, &X86::VR128XRegClass, Op0, Op1); 17792 } 17793 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 17794 return fastEmitInst_rr(X86::VPSRLVQrr, &X86::VR128RegClass, Op0, Op1); 17795 } 17796 return 0; 17797} 17798 17799unsigned fastEmit_X86ISD_VSRLV_MVT_v4i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17800 if (RetVT.SimpleTy != MVT::v4i64) 17801 return 0; 17802 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 17803 return fastEmitInst_rr(X86::VPSRLVQZ256rr, &X86::VR256XRegClass, Op0, Op1); 17804 } 17805 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 17806 return fastEmitInst_rr(X86::VPSRLVQYrr, &X86::VR256RegClass, Op0, Op1); 17807 } 17808 return 0; 17809} 17810 17811unsigned fastEmit_X86ISD_VSRLV_MVT_v8i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 17812 if (RetVT.SimpleTy != MVT::v8i64) 17813 return 0; 17814 if ((Subtarget->hasAVX512())) { 17815 return fastEmitInst_rr(X86::VPSRLVQZrr, &X86::VR512RegClass, Op0, Op1); 17816 } 17817 return 0; 17818} 17819 17820unsigned fastEmit_X86ISD_VSRLV_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 17821 switch (VT.SimpleTy) { 17822 case MVT::v8i16: return fastEmit_X86ISD_VSRLV_MVT_v8i16_rr(RetVT, Op0, Op1); 17823 case MVT::v16i16: return fastEmit_X86ISD_VSRLV_MVT_v16i16_rr(RetVT, Op0, Op1); 17824 case MVT::v32i16: return fastEmit_X86ISD_VSRLV_MVT_v32i16_rr(RetVT, Op0, Op1); 17825 case MVT::v4i32: return fastEmit_X86ISD_VSRLV_MVT_v4i32_rr(RetVT, Op0, Op1); 17826 case MVT::v8i32: return fastEmit_X86ISD_VSRLV_MVT_v8i32_rr(RetVT, Op0, Op1); 17827 case MVT::v16i32: return fastEmit_X86ISD_VSRLV_MVT_v16i32_rr(RetVT, Op0, Op1); 17828 case MVT::v2i64: return fastEmit_X86ISD_VSRLV_MVT_v2i64_rr(RetVT, Op0, Op1); 17829 case MVT::v4i64: return fastEmit_X86ISD_VSRLV_MVT_v4i64_rr(RetVT, Op0, Op1); 17830 case MVT::v8i64: return fastEmit_X86ISD_VSRLV_MVT_v8i64_rr(RetVT, Op0, Op1); 17831 default: return 0; 17832 } 17833} 17834 17835// Top-level FastEmit function. 17836 17837unsigned fastEmit_rr(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, unsigned Op1) override { 17838 switch (Opcode) { 17839 case ISD::ADD: return fastEmit_ISD_ADD_rr(VT, RetVT, Op0, Op1); 17840 case ISD::AND: return fastEmit_ISD_AND_rr(VT, RetVT, Op0, Op1); 17841 case ISD::AVGCEILU: return fastEmit_ISD_AVGCEILU_rr(VT, RetVT, Op0, Op1); 17842 case ISD::FADD: return fastEmit_ISD_FADD_rr(VT, RetVT, Op0, Op1); 17843 case ISD::FDIV: return fastEmit_ISD_FDIV_rr(VT, RetVT, Op0, Op1); 17844 case ISD::FMUL: return fastEmit_ISD_FMUL_rr(VT, RetVT, Op0, Op1); 17845 case ISD::FSUB: return fastEmit_ISD_FSUB_rr(VT, RetVT, Op0, Op1); 17846 case ISD::MUL: return fastEmit_ISD_MUL_rr(VT, RetVT, Op0, Op1); 17847 case ISD::MULHS: return fastEmit_ISD_MULHS_rr(VT, RetVT, Op0, Op1); 17848 case ISD::MULHU: return fastEmit_ISD_MULHU_rr(VT, RetVT, Op0, Op1); 17849 case ISD::OR: return fastEmit_ISD_OR_rr(VT, RetVT, Op0, Op1); 17850 case ISD::ROTL: return fastEmit_ISD_ROTL_rr(VT, RetVT, Op0, Op1); 17851 case ISD::ROTR: return fastEmit_ISD_ROTR_rr(VT, RetVT, Op0, Op1); 17852 case ISD::SADDSAT: return fastEmit_ISD_SADDSAT_rr(VT, RetVT, Op0, Op1); 17853 case ISD::SHL: return fastEmit_ISD_SHL_rr(VT, RetVT, Op0, Op1); 17854 case ISD::SMAX: return fastEmit_ISD_SMAX_rr(VT, RetVT, Op0, Op1); 17855 case ISD::SMIN: return fastEmit_ISD_SMIN_rr(VT, RetVT, Op0, Op1); 17856 case ISD::SRA: return fastEmit_ISD_SRA_rr(VT, RetVT, Op0, Op1); 17857 case ISD::SRL: return fastEmit_ISD_SRL_rr(VT, RetVT, Op0, Op1); 17858 case ISD::SSUBSAT: return fastEmit_ISD_SSUBSAT_rr(VT, RetVT, Op0, Op1); 17859 case ISD::STRICT_FADD: return fastEmit_ISD_STRICT_FADD_rr(VT, RetVT, Op0, Op1); 17860 case ISD::STRICT_FDIV: return fastEmit_ISD_STRICT_FDIV_rr(VT, RetVT, Op0, Op1); 17861 case ISD::STRICT_FMUL: return fastEmit_ISD_STRICT_FMUL_rr(VT, RetVT, Op0, Op1); 17862 case ISD::STRICT_FSUB: return fastEmit_ISD_STRICT_FSUB_rr(VT, RetVT, Op0, Op1); 17863 case ISD::SUB: return fastEmit_ISD_SUB_rr(VT, RetVT, Op0, Op1); 17864 case ISD::UADDSAT: return fastEmit_ISD_UADDSAT_rr(VT, RetVT, Op0, Op1); 17865 case ISD::UMAX: return fastEmit_ISD_UMAX_rr(VT, RetVT, Op0, Op1); 17866 case ISD::UMIN: return fastEmit_ISD_UMIN_rr(VT, RetVT, Op0, Op1); 17867 case ISD::USUBSAT: return fastEmit_ISD_USUBSAT_rr(VT, RetVT, Op0, Op1); 17868 case ISD::XOR: return fastEmit_ISD_XOR_rr(VT, RetVT, Op0, Op1); 17869 case X86ISD::ADDSUB: return fastEmit_X86ISD_ADDSUB_rr(VT, RetVT, Op0, Op1); 17870 case X86ISD::ANDNP: return fastEmit_X86ISD_ANDNP_rr(VT, RetVT, Op0, Op1); 17871 case X86ISD::BEXTR: return fastEmit_X86ISD_BEXTR_rr(VT, RetVT, Op0, Op1); 17872 case X86ISD::BT: return fastEmit_X86ISD_BT_rr(VT, RetVT, Op0, Op1); 17873 case X86ISD::BZHI: return fastEmit_X86ISD_BZHI_rr(VT, RetVT, Op0, Op1); 17874 case X86ISD::CMP: return fastEmit_X86ISD_CMP_rr(VT, RetVT, Op0, Op1); 17875 case X86ISD::COMI: return fastEmit_X86ISD_COMI_rr(VT, RetVT, Op0, Op1); 17876 case X86ISD::CVTNE2PS2BF16: return fastEmit_X86ISD_CVTNE2PS2BF16_rr(VT, RetVT, Op0, Op1); 17877 case X86ISD::FADDS: return fastEmit_X86ISD_FADDS_rr(VT, RetVT, Op0, Op1); 17878 case X86ISD::FAND: return fastEmit_X86ISD_FAND_rr(VT, RetVT, Op0, Op1); 17879 case X86ISD::FANDN: return fastEmit_X86ISD_FANDN_rr(VT, RetVT, Op0, Op1); 17880 case X86ISD::FCMP: return fastEmit_X86ISD_FCMP_rr(VT, RetVT, Op0, Op1); 17881 case X86ISD::FDIVS: return fastEmit_X86ISD_FDIVS_rr(VT, RetVT, Op0, Op1); 17882 case X86ISD::FGETEXPS: return fastEmit_X86ISD_FGETEXPS_rr(VT, RetVT, Op0, Op1); 17883 case X86ISD::FGETEXPS_SAE: return fastEmit_X86ISD_FGETEXPS_SAE_rr(VT, RetVT, Op0, Op1); 17884 case X86ISD::FHADD: return fastEmit_X86ISD_FHADD_rr(VT, RetVT, Op0, Op1); 17885 case X86ISD::FHSUB: return fastEmit_X86ISD_FHSUB_rr(VT, RetVT, Op0, Op1); 17886 case X86ISD::FMAX: return fastEmit_X86ISD_FMAX_rr(VT, RetVT, Op0, Op1); 17887 case X86ISD::FMAXC: return fastEmit_X86ISD_FMAXC_rr(VT, RetVT, Op0, Op1); 17888 case X86ISD::FMAXS: return fastEmit_X86ISD_FMAXS_rr(VT, RetVT, Op0, Op1); 17889 case X86ISD::FMAXS_SAE: return fastEmit_X86ISD_FMAXS_SAE_rr(VT, RetVT, Op0, Op1); 17890 case X86ISD::FMAX_SAE: return fastEmit_X86ISD_FMAX_SAE_rr(VT, RetVT, Op0, Op1); 17891 case X86ISD::FMIN: return fastEmit_X86ISD_FMIN_rr(VT, RetVT, Op0, Op1); 17892 case X86ISD::FMINC: return fastEmit_X86ISD_FMINC_rr(VT, RetVT, Op0, Op1); 17893 case X86ISD::FMINS: return fastEmit_X86ISD_FMINS_rr(VT, RetVT, Op0, Op1); 17894 case X86ISD::FMINS_SAE: return fastEmit_X86ISD_FMINS_SAE_rr(VT, RetVT, Op0, Op1); 17895 case X86ISD::FMIN_SAE: return fastEmit_X86ISD_FMIN_SAE_rr(VT, RetVT, Op0, Op1); 17896 case X86ISD::FMULS: return fastEmit_X86ISD_FMULS_rr(VT, RetVT, Op0, Op1); 17897 case X86ISD::FOR: return fastEmit_X86ISD_FOR_rr(VT, RetVT, Op0, Op1); 17898 case X86ISD::FP80_ADD: return fastEmit_X86ISD_FP80_ADD_rr(VT, RetVT, Op0, Op1); 17899 case X86ISD::FSQRTS: return fastEmit_X86ISD_FSQRTS_rr(VT, RetVT, Op0, Op1); 17900 case X86ISD::FSUBS: return fastEmit_X86ISD_FSUBS_rr(VT, RetVT, Op0, Op1); 17901 case X86ISD::FXOR: return fastEmit_X86ISD_FXOR_rr(VT, RetVT, Op0, Op1); 17902 case X86ISD::GF2P8MULB: return fastEmit_X86ISD_GF2P8MULB_rr(VT, RetVT, Op0, Op1); 17903 case X86ISD::HADD: return fastEmit_X86ISD_HADD_rr(VT, RetVT, Op0, Op1); 17904 case X86ISD::HSUB: return fastEmit_X86ISD_HSUB_rr(VT, RetVT, Op0, Op1); 17905 case X86ISD::KADD: return fastEmit_X86ISD_KADD_rr(VT, RetVT, Op0, Op1); 17906 case X86ISD::KORTEST: return fastEmit_X86ISD_KORTEST_rr(VT, RetVT, Op0, Op1); 17907 case X86ISD::KTEST: return fastEmit_X86ISD_KTEST_rr(VT, RetVT, Op0, Op1); 17908 case X86ISD::MOVHLPS: return fastEmit_X86ISD_MOVHLPS_rr(VT, RetVT, Op0, Op1); 17909 case X86ISD::MOVLHPS: return fastEmit_X86ISD_MOVLHPS_rr(VT, RetVT, Op0, Op1); 17910 case X86ISD::MOVSD: return fastEmit_X86ISD_MOVSD_rr(VT, RetVT, Op0, Op1); 17911 case X86ISD::MOVSH: return fastEmit_X86ISD_MOVSH_rr(VT, RetVT, Op0, Op1); 17912 case X86ISD::MOVSS: return fastEmit_X86ISD_MOVSS_rr(VT, RetVT, Op0, Op1); 17913 case X86ISD::MULHRS: return fastEmit_X86ISD_MULHRS_rr(VT, RetVT, Op0, Op1); 17914 case X86ISD::MULTISHIFT: return fastEmit_X86ISD_MULTISHIFT_rr(VT, RetVT, Op0, Op1); 17915 case X86ISD::PACKSS: return fastEmit_X86ISD_PACKSS_rr(VT, RetVT, Op0, Op1); 17916 case X86ISD::PACKUS: return fastEmit_X86ISD_PACKUS_rr(VT, RetVT, Op0, Op1); 17917 case X86ISD::PCMPEQ: return fastEmit_X86ISD_PCMPEQ_rr(VT, RetVT, Op0, Op1); 17918 case X86ISD::PCMPGT: return fastEmit_X86ISD_PCMPGT_rr(VT, RetVT, Op0, Op1); 17919 case X86ISD::PDEP: return fastEmit_X86ISD_PDEP_rr(VT, RetVT, Op0, Op1); 17920 case X86ISD::PEXT: return fastEmit_X86ISD_PEXT_rr(VT, RetVT, Op0, Op1); 17921 case X86ISD::PMULDQ: return fastEmit_X86ISD_PMULDQ_rr(VT, RetVT, Op0, Op1); 17922 case X86ISD::PMULUDQ: return fastEmit_X86ISD_PMULUDQ_rr(VT, RetVT, Op0, Op1); 17923 case X86ISD::PSADBW: return fastEmit_X86ISD_PSADBW_rr(VT, RetVT, Op0, Op1); 17924 case X86ISD::PSHUFB: return fastEmit_X86ISD_PSHUFB_rr(VT, RetVT, Op0, Op1); 17925 case X86ISD::PTEST: return fastEmit_X86ISD_PTEST_rr(VT, RetVT, Op0, Op1); 17926 case X86ISD::RCP14S: return fastEmit_X86ISD_RCP14S_rr(VT, RetVT, Op0, Op1); 17927 case X86ISD::RCP28S: return fastEmit_X86ISD_RCP28S_rr(VT, RetVT, Op0, Op1); 17928 case X86ISD::RCP28S_SAE: return fastEmit_X86ISD_RCP28S_SAE_rr(VT, RetVT, Op0, Op1); 17929 case X86ISD::RSQRT14S: return fastEmit_X86ISD_RSQRT14S_rr(VT, RetVT, Op0, Op1); 17930 case X86ISD::RSQRT28S: return fastEmit_X86ISD_RSQRT28S_rr(VT, RetVT, Op0, Op1); 17931 case X86ISD::RSQRT28S_SAE: return fastEmit_X86ISD_RSQRT28S_SAE_rr(VT, RetVT, Op0, Op1); 17932 case X86ISD::SCALEF: return fastEmit_X86ISD_SCALEF_rr(VT, RetVT, Op0, Op1); 17933 case X86ISD::SCALEFS: return fastEmit_X86ISD_SCALEFS_rr(VT, RetVT, Op0, Op1); 17934 case X86ISD::STRICT_FCMP: return fastEmit_X86ISD_STRICT_FCMP_rr(VT, RetVT, Op0, Op1); 17935 case X86ISD::STRICT_FCMPS: return fastEmit_X86ISD_STRICT_FCMPS_rr(VT, RetVT, Op0, Op1); 17936 case X86ISD::STRICT_FP80_ADD: return fastEmit_X86ISD_STRICT_FP80_ADD_rr(VT, RetVT, Op0, Op1); 17937 case X86ISD::TESTP: return fastEmit_X86ISD_TESTP_rr(VT, RetVT, Op0, Op1); 17938 case X86ISD::UCOMI: return fastEmit_X86ISD_UCOMI_rr(VT, RetVT, Op0, Op1); 17939 case X86ISD::UNPCKH: return fastEmit_X86ISD_UNPCKH_rr(VT, RetVT, Op0, Op1); 17940 case X86ISD::UNPCKL: return fastEmit_X86ISD_UNPCKL_rr(VT, RetVT, Op0, Op1); 17941 case X86ISD::VFCMULC: return fastEmit_X86ISD_VFCMULC_rr(VT, RetVT, Op0, Op1); 17942 case X86ISD::VFCMULCSH: return fastEmit_X86ISD_VFCMULCSH_rr(VT, RetVT, Op0, Op1); 17943 case X86ISD::VFMULC: return fastEmit_X86ISD_VFMULC_rr(VT, RetVT, Op0, Op1); 17944 case X86ISD::VFMULCSH: return fastEmit_X86ISD_VFMULCSH_rr(VT, RetVT, Op0, Op1); 17945 case X86ISD::VP2INTERSECT: return fastEmit_X86ISD_VP2INTERSECT_rr(VT, RetVT, Op0, Op1); 17946 case X86ISD::VPERMV: return fastEmit_X86ISD_VPERMV_rr(VT, RetVT, Op0, Op1); 17947 case X86ISD::VPMADDUBSW: return fastEmit_X86ISD_VPMADDUBSW_rr(VT, RetVT, Op0, Op1); 17948 case X86ISD::VPMADDWD: return fastEmit_X86ISD_VPMADDWD_rr(VT, RetVT, Op0, Op1); 17949 case X86ISD::VPSHA: return fastEmit_X86ISD_VPSHA_rr(VT, RetVT, Op0, Op1); 17950 case X86ISD::VPSHL: return fastEmit_X86ISD_VPSHL_rr(VT, RetVT, Op0, Op1); 17951 case X86ISD::VPSHUFBITQMB: return fastEmit_X86ISD_VPSHUFBITQMB_rr(VT, RetVT, Op0, Op1); 17952 case X86ISD::VSHL: return fastEmit_X86ISD_VSHL_rr(VT, RetVT, Op0, Op1); 17953 case X86ISD::VSHLV: return fastEmit_X86ISD_VSHLV_rr(VT, RetVT, Op0, Op1); 17954 case X86ISD::VSRA: return fastEmit_X86ISD_VSRA_rr(VT, RetVT, Op0, Op1); 17955 case X86ISD::VSRAV: return fastEmit_X86ISD_VSRAV_rr(VT, RetVT, Op0, Op1); 17956 case X86ISD::VSRL: return fastEmit_X86ISD_VSRL_rr(VT, RetVT, Op0, Op1); 17957 case X86ISD::VSRLV: return fastEmit_X86ISD_VSRLV_rr(VT, RetVT, Op0, Op1); 17958 default: return 0; 17959 } 17960} 17961 17962// FastEmit functions for ISD::ADD. 17963 17964unsigned fastEmit_ISD_ADD_MVT_i8_ri(MVT RetVT, unsigned Op0, uint64_t imm1) { 17965 if (RetVT.SimpleTy != MVT::i8) 17966 return 0; 17967 return fastEmitInst_ri(X86::ADD8ri, &X86::GR8RegClass, Op0, imm1); 17968} 17969 17970unsigned fastEmit_ISD_ADD_MVT_i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) { 17971 if (RetVT.SimpleTy != MVT::i16) 17972 return 0; 17973 return fastEmitInst_ri(X86::ADD16ri, &X86::GR16RegClass, Op0, imm1); 17974} 17975 17976unsigned fastEmit_ISD_ADD_MVT_i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) { 17977 if (RetVT.SimpleTy != MVT::i32) 17978 return 0; 17979 return fastEmitInst_ri(X86::ADD32ri, &X86::GR32RegClass, Op0, imm1); 17980} 17981 17982unsigned fastEmit_ISD_ADD_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 17983 switch (VT.SimpleTy) { 17984 case MVT::i8: return fastEmit_ISD_ADD_MVT_i8_ri(RetVT, Op0, imm1); 17985 case MVT::i16: return fastEmit_ISD_ADD_MVT_i16_ri(RetVT, Op0, imm1); 17986 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri(RetVT, Op0, imm1); 17987 default: return 0; 17988 } 17989} 17990 17991// FastEmit functions for ISD::AND. 17992 17993unsigned fastEmit_ISD_AND_MVT_i8_ri(MVT RetVT, unsigned Op0, uint64_t imm1) { 17994 if (RetVT.SimpleTy != MVT::i8) 17995 return 0; 17996 return fastEmitInst_ri(X86::AND8ri, &X86::GR8RegClass, Op0, imm1); 17997} 17998 17999unsigned fastEmit_ISD_AND_MVT_i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) { 18000 if (RetVT.SimpleTy != MVT::i16) 18001 return 0; 18002 return fastEmitInst_ri(X86::AND16ri, &X86::GR16RegClass, Op0, imm1); 18003} 18004 18005unsigned fastEmit_ISD_AND_MVT_i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) { 18006 if (RetVT.SimpleTy != MVT::i32) 18007 return 0; 18008 return fastEmitInst_ri(X86::AND32ri, &X86::GR32RegClass, Op0, imm1); 18009} 18010 18011unsigned fastEmit_ISD_AND_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 18012 switch (VT.SimpleTy) { 18013 case MVT::i8: return fastEmit_ISD_AND_MVT_i8_ri(RetVT, Op0, imm1); 18014 case MVT::i16: return fastEmit_ISD_AND_MVT_i16_ri(RetVT, Op0, imm1); 18015 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_ri(RetVT, Op0, imm1); 18016 default: return 0; 18017 } 18018} 18019 18020// FastEmit functions for ISD::EXTRACT_VECTOR_ELT. 18021 18022unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) { 18023 if (RetVT.SimpleTy != MVT::i32) 18024 return 0; 18025 if ((Subtarget->hasDQI())) { 18026 return fastEmitInst_ri(X86::VPEXTRDZrr, &X86::GR32RegClass, Op0, imm1); 18027 } 18028 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 18029 return fastEmitInst_ri(X86::PEXTRDrr, &X86::GR32RegClass, Op0, imm1); 18030 } 18031 if ((Subtarget->hasAVX()) && (!Subtarget->hasDQI())) { 18032 return fastEmitInst_ri(X86::VPEXTRDrr, &X86::GR32RegClass, Op0, imm1); 18033 } 18034 return 0; 18035} 18036 18037unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri(MVT RetVT, unsigned Op0, uint64_t imm1) { 18038 if (RetVT.SimpleTy != MVT::i64) 18039 return 0; 18040 if ((Subtarget->hasDQI())) { 18041 return fastEmitInst_ri(X86::VPEXTRQZrr, &X86::GR64RegClass, Op0, imm1); 18042 } 18043 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 18044 return fastEmitInst_ri(X86::PEXTRQrr, &X86::GR64RegClass, Op0, imm1); 18045 } 18046 if ((Subtarget->hasAVX()) && (!Subtarget->hasDQI())) { 18047 return fastEmitInst_ri(X86::VPEXTRQrr, &X86::GR64RegClass, Op0, imm1); 18048 } 18049 return 0; 18050} 18051 18052unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 18053 switch (VT.SimpleTy) { 18054 case MVT::v4i32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri(RetVT, Op0, imm1); 18055 case MVT::v2i64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri(RetVT, Op0, imm1); 18056 default: return 0; 18057 } 18058} 18059 18060// FastEmit functions for ISD::MUL. 18061 18062unsigned fastEmit_ISD_MUL_MVT_i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) { 18063 if (RetVT.SimpleTy != MVT::i16) 18064 return 0; 18065 return fastEmitInst_ri(X86::IMUL16rri, &X86::GR16RegClass, Op0, imm1); 18066} 18067 18068unsigned fastEmit_ISD_MUL_MVT_i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) { 18069 if (RetVT.SimpleTy != MVT::i32) 18070 return 0; 18071 return fastEmitInst_ri(X86::IMUL32rri, &X86::GR32RegClass, Op0, imm1); 18072} 18073 18074unsigned fastEmit_ISD_MUL_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 18075 switch (VT.SimpleTy) { 18076 case MVT::i16: return fastEmit_ISD_MUL_MVT_i16_ri(RetVT, Op0, imm1); 18077 case MVT::i32: return fastEmit_ISD_MUL_MVT_i32_ri(RetVT, Op0, imm1); 18078 default: return 0; 18079 } 18080} 18081 18082// FastEmit functions for ISD::OR. 18083 18084unsigned fastEmit_ISD_OR_MVT_i8_ri(MVT RetVT, unsigned Op0, uint64_t imm1) { 18085 if (RetVT.SimpleTy != MVT::i8) 18086 return 0; 18087 return fastEmitInst_ri(X86::OR8ri, &X86::GR8RegClass, Op0, imm1); 18088} 18089 18090unsigned fastEmit_ISD_OR_MVT_i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) { 18091 if (RetVT.SimpleTy != MVT::i16) 18092 return 0; 18093 return fastEmitInst_ri(X86::OR16ri, &X86::GR16RegClass, Op0, imm1); 18094} 18095 18096unsigned fastEmit_ISD_OR_MVT_i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) { 18097 if (RetVT.SimpleTy != MVT::i32) 18098 return 0; 18099 return fastEmitInst_ri(X86::OR32ri, &X86::GR32RegClass, Op0, imm1); 18100} 18101 18102unsigned fastEmit_ISD_OR_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 18103 switch (VT.SimpleTy) { 18104 case MVT::i8: return fastEmit_ISD_OR_MVT_i8_ri(RetVT, Op0, imm1); 18105 case MVT::i16: return fastEmit_ISD_OR_MVT_i16_ri(RetVT, Op0, imm1); 18106 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_ri(RetVT, Op0, imm1); 18107 default: return 0; 18108 } 18109} 18110 18111// FastEmit functions for ISD::ROTL. 18112 18113unsigned fastEmit_ISD_ROTL_MVT_i8_ri(MVT RetVT, unsigned Op0, uint64_t imm1) { 18114 if (RetVT.SimpleTy != MVT::i8) 18115 return 0; 18116 return fastEmitInst_ri(X86::ROL8ri, &X86::GR8RegClass, Op0, imm1); 18117} 18118 18119unsigned fastEmit_ISD_ROTL_MVT_i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) { 18120 if (RetVT.SimpleTy != MVT::i16) 18121 return 0; 18122 return fastEmitInst_ri(X86::ROL16ri, &X86::GR16RegClass, Op0, imm1); 18123} 18124 18125unsigned fastEmit_ISD_ROTL_MVT_i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) { 18126 if (RetVT.SimpleTy != MVT::i32) 18127 return 0; 18128 if ((Subtarget->hasFastSHLDRotate())) { 18129 return fastEmitInst_ri(X86::SHLDROT32ri, &X86::GR32RegClass, Op0, imm1); 18130 } 18131 return fastEmitInst_ri(X86::ROL32ri, &X86::GR32RegClass, Op0, imm1); 18132} 18133 18134unsigned fastEmit_ISD_ROTL_MVT_i64_ri(MVT RetVT, unsigned Op0, uint64_t imm1) { 18135 if (RetVT.SimpleTy != MVT::i64) 18136 return 0; 18137 if ((Subtarget->hasFastSHLDRotate())) { 18138 return fastEmitInst_ri(X86::SHLDROT64ri, &X86::GR64RegClass, Op0, imm1); 18139 } 18140 return fastEmitInst_ri(X86::ROL64ri, &X86::GR64RegClass, Op0, imm1); 18141} 18142 18143unsigned fastEmit_ISD_ROTL_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 18144 switch (VT.SimpleTy) { 18145 case MVT::i8: return fastEmit_ISD_ROTL_MVT_i8_ri(RetVT, Op0, imm1); 18146 case MVT::i16: return fastEmit_ISD_ROTL_MVT_i16_ri(RetVT, Op0, imm1); 18147 case MVT::i32: return fastEmit_ISD_ROTL_MVT_i32_ri(RetVT, Op0, imm1); 18148 case MVT::i64: return fastEmit_ISD_ROTL_MVT_i64_ri(RetVT, Op0, imm1); 18149 default: return 0; 18150 } 18151} 18152 18153// FastEmit functions for ISD::ROTR. 18154 18155unsigned fastEmit_ISD_ROTR_MVT_i8_ri(MVT RetVT, unsigned Op0, uint64_t imm1) { 18156 if (RetVT.SimpleTy != MVT::i8) 18157 return 0; 18158 return fastEmitInst_ri(X86::ROR8ri, &X86::GR8RegClass, Op0, imm1); 18159} 18160 18161unsigned fastEmit_ISD_ROTR_MVT_i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) { 18162 if (RetVT.SimpleTy != MVT::i16) 18163 return 0; 18164 return fastEmitInst_ri(X86::ROR16ri, &X86::GR16RegClass, Op0, imm1); 18165} 18166 18167unsigned fastEmit_ISD_ROTR_MVT_i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) { 18168 if (RetVT.SimpleTy != MVT::i32) 18169 return 0; 18170 if ((Subtarget->hasBMI2())) { 18171 return fastEmitInst_ri(X86::RORX32ri, &X86::GR32RegClass, Op0, imm1); 18172 } 18173 if ((Subtarget->hasFastSHLDRotate())) { 18174 return fastEmitInst_ri(X86::SHRDROT32ri, &X86::GR32RegClass, Op0, imm1); 18175 } 18176 return fastEmitInst_ri(X86::ROR32ri, &X86::GR32RegClass, Op0, imm1); 18177} 18178 18179unsigned fastEmit_ISD_ROTR_MVT_i64_ri(MVT RetVT, unsigned Op0, uint64_t imm1) { 18180 if (RetVT.SimpleTy != MVT::i64) 18181 return 0; 18182 if ((Subtarget->hasBMI2())) { 18183 return fastEmitInst_ri(X86::RORX64ri, &X86::GR64RegClass, Op0, imm1); 18184 } 18185 if ((Subtarget->hasFastSHLDRotate())) { 18186 return fastEmitInst_ri(X86::SHRDROT64ri, &X86::GR64RegClass, Op0, imm1); 18187 } 18188 return fastEmitInst_ri(X86::ROR64ri, &X86::GR64RegClass, Op0, imm1); 18189} 18190 18191unsigned fastEmit_ISD_ROTR_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 18192 switch (VT.SimpleTy) { 18193 case MVT::i8: return fastEmit_ISD_ROTR_MVT_i8_ri(RetVT, Op0, imm1); 18194 case MVT::i16: return fastEmit_ISD_ROTR_MVT_i16_ri(RetVT, Op0, imm1); 18195 case MVT::i32: return fastEmit_ISD_ROTR_MVT_i32_ri(RetVT, Op0, imm1); 18196 case MVT::i64: return fastEmit_ISD_ROTR_MVT_i64_ri(RetVT, Op0, imm1); 18197 default: return 0; 18198 } 18199} 18200 18201// FastEmit functions for ISD::SHL. 18202 18203unsigned fastEmit_ISD_SHL_MVT_i8_ri(MVT RetVT, unsigned Op0, uint64_t imm1) { 18204 if (RetVT.SimpleTy != MVT::i8) 18205 return 0; 18206 return fastEmitInst_ri(X86::SHL8ri, &X86::GR8RegClass, Op0, imm1); 18207} 18208 18209unsigned fastEmit_ISD_SHL_MVT_i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) { 18210 if (RetVT.SimpleTy != MVT::i16) 18211 return 0; 18212 return fastEmitInst_ri(X86::SHL16ri, &X86::GR16RegClass, Op0, imm1); 18213} 18214 18215unsigned fastEmit_ISD_SHL_MVT_i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) { 18216 if (RetVT.SimpleTy != MVT::i32) 18217 return 0; 18218 return fastEmitInst_ri(X86::SHL32ri, &X86::GR32RegClass, Op0, imm1); 18219} 18220 18221unsigned fastEmit_ISD_SHL_MVT_i64_ri(MVT RetVT, unsigned Op0, uint64_t imm1) { 18222 if (RetVT.SimpleTy != MVT::i64) 18223 return 0; 18224 return fastEmitInst_ri(X86::SHL64ri, &X86::GR64RegClass, Op0, imm1); 18225} 18226 18227unsigned fastEmit_ISD_SHL_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 18228 switch (VT.SimpleTy) { 18229 case MVT::i8: return fastEmit_ISD_SHL_MVT_i8_ri(RetVT, Op0, imm1); 18230 case MVT::i16: return fastEmit_ISD_SHL_MVT_i16_ri(RetVT, Op0, imm1); 18231 case MVT::i32: return fastEmit_ISD_SHL_MVT_i32_ri(RetVT, Op0, imm1); 18232 case MVT::i64: return fastEmit_ISD_SHL_MVT_i64_ri(RetVT, Op0, imm1); 18233 default: return 0; 18234 } 18235} 18236 18237// FastEmit functions for ISD::SRA. 18238 18239unsigned fastEmit_ISD_SRA_MVT_i8_ri(MVT RetVT, unsigned Op0, uint64_t imm1) { 18240 if (RetVT.SimpleTy != MVT::i8) 18241 return 0; 18242 return fastEmitInst_ri(X86::SAR8ri, &X86::GR8RegClass, Op0, imm1); 18243} 18244 18245unsigned fastEmit_ISD_SRA_MVT_i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) { 18246 if (RetVT.SimpleTy != MVT::i16) 18247 return 0; 18248 return fastEmitInst_ri(X86::SAR16ri, &X86::GR16RegClass, Op0, imm1); 18249} 18250 18251unsigned fastEmit_ISD_SRA_MVT_i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) { 18252 if (RetVT.SimpleTy != MVT::i32) 18253 return 0; 18254 return fastEmitInst_ri(X86::SAR32ri, &X86::GR32RegClass, Op0, imm1); 18255} 18256 18257unsigned fastEmit_ISD_SRA_MVT_i64_ri(MVT RetVT, unsigned Op0, uint64_t imm1) { 18258 if (RetVT.SimpleTy != MVT::i64) 18259 return 0; 18260 return fastEmitInst_ri(X86::SAR64ri, &X86::GR64RegClass, Op0, imm1); 18261} 18262 18263unsigned fastEmit_ISD_SRA_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 18264 switch (VT.SimpleTy) { 18265 case MVT::i8: return fastEmit_ISD_SRA_MVT_i8_ri(RetVT, Op0, imm1); 18266 case MVT::i16: return fastEmit_ISD_SRA_MVT_i16_ri(RetVT, Op0, imm1); 18267 case MVT::i32: return fastEmit_ISD_SRA_MVT_i32_ri(RetVT, Op0, imm1); 18268 case MVT::i64: return fastEmit_ISD_SRA_MVT_i64_ri(RetVT, Op0, imm1); 18269 default: return 0; 18270 } 18271} 18272 18273// FastEmit functions for ISD::SRL. 18274 18275unsigned fastEmit_ISD_SRL_MVT_i8_ri(MVT RetVT, unsigned Op0, uint64_t imm1) { 18276 if (RetVT.SimpleTy != MVT::i8) 18277 return 0; 18278 return fastEmitInst_ri(X86::SHR8ri, &X86::GR8RegClass, Op0, imm1); 18279} 18280 18281unsigned fastEmit_ISD_SRL_MVT_i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) { 18282 if (RetVT.SimpleTy != MVT::i16) 18283 return 0; 18284 return fastEmitInst_ri(X86::SHR16ri, &X86::GR16RegClass, Op0, imm1); 18285} 18286 18287unsigned fastEmit_ISD_SRL_MVT_i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) { 18288 if (RetVT.SimpleTy != MVT::i32) 18289 return 0; 18290 return fastEmitInst_ri(X86::SHR32ri, &X86::GR32RegClass, Op0, imm1); 18291} 18292 18293unsigned fastEmit_ISD_SRL_MVT_i64_ri(MVT RetVT, unsigned Op0, uint64_t imm1) { 18294 if (RetVT.SimpleTy != MVT::i64) 18295 return 0; 18296 return fastEmitInst_ri(X86::SHR64ri, &X86::GR64RegClass, Op0, imm1); 18297} 18298 18299unsigned fastEmit_ISD_SRL_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 18300 switch (VT.SimpleTy) { 18301 case MVT::i8: return fastEmit_ISD_SRL_MVT_i8_ri(RetVT, Op0, imm1); 18302 case MVT::i16: return fastEmit_ISD_SRL_MVT_i16_ri(RetVT, Op0, imm1); 18303 case MVT::i32: return fastEmit_ISD_SRL_MVT_i32_ri(RetVT, Op0, imm1); 18304 case MVT::i64: return fastEmit_ISD_SRL_MVT_i64_ri(RetVT, Op0, imm1); 18305 default: return 0; 18306 } 18307} 18308 18309// FastEmit functions for ISD::SUB. 18310 18311unsigned fastEmit_ISD_SUB_MVT_i8_ri(MVT RetVT, unsigned Op0, uint64_t imm1) { 18312 if (RetVT.SimpleTy != MVT::i8) 18313 return 0; 18314 return fastEmitInst_ri(X86::SUB8ri, &X86::GR8RegClass, Op0, imm1); 18315} 18316 18317unsigned fastEmit_ISD_SUB_MVT_i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) { 18318 if (RetVT.SimpleTy != MVT::i16) 18319 return 0; 18320 return fastEmitInst_ri(X86::SUB16ri, &X86::GR16RegClass, Op0, imm1); 18321} 18322 18323unsigned fastEmit_ISD_SUB_MVT_i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) { 18324 if (RetVT.SimpleTy != MVT::i32) 18325 return 0; 18326 return fastEmitInst_ri(X86::SUB32ri, &X86::GR32RegClass, Op0, imm1); 18327} 18328 18329unsigned fastEmit_ISD_SUB_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 18330 switch (VT.SimpleTy) { 18331 case MVT::i8: return fastEmit_ISD_SUB_MVT_i8_ri(RetVT, Op0, imm1); 18332 case MVT::i16: return fastEmit_ISD_SUB_MVT_i16_ri(RetVT, Op0, imm1); 18333 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_ri(RetVT, Op0, imm1); 18334 default: return 0; 18335 } 18336} 18337 18338// FastEmit functions for ISD::XOR. 18339 18340unsigned fastEmit_ISD_XOR_MVT_i8_ri(MVT RetVT, unsigned Op0, uint64_t imm1) { 18341 if (RetVT.SimpleTy != MVT::i8) 18342 return 0; 18343 return fastEmitInst_ri(X86::XOR8ri, &X86::GR8RegClass, Op0, imm1); 18344} 18345 18346unsigned fastEmit_ISD_XOR_MVT_i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) { 18347 if (RetVT.SimpleTy != MVT::i16) 18348 return 0; 18349 return fastEmitInst_ri(X86::XOR16ri, &X86::GR16RegClass, Op0, imm1); 18350} 18351 18352unsigned fastEmit_ISD_XOR_MVT_i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) { 18353 if (RetVT.SimpleTy != MVT::i32) 18354 return 0; 18355 return fastEmitInst_ri(X86::XOR32ri, &X86::GR32RegClass, Op0, imm1); 18356} 18357 18358unsigned fastEmit_ISD_XOR_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 18359 switch (VT.SimpleTy) { 18360 case MVT::i8: return fastEmit_ISD_XOR_MVT_i8_ri(RetVT, Op0, imm1); 18361 case MVT::i16: return fastEmit_ISD_XOR_MVT_i16_ri(RetVT, Op0, imm1); 18362 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_ri(RetVT, Op0, imm1); 18363 default: return 0; 18364 } 18365} 18366 18367// FastEmit functions for X86ISD::BT. 18368 18369unsigned fastEmit_X86ISD_BT_MVT_i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) { 18370 if (RetVT.SimpleTy != MVT::i32) 18371 return 0; 18372 return fastEmitInst_ri(X86::BT16ri8, &X86::GR16RegClass, Op0, imm1); 18373} 18374 18375unsigned fastEmit_X86ISD_BT_MVT_i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) { 18376 if (RetVT.SimpleTy != MVT::i32) 18377 return 0; 18378 return fastEmitInst_ri(X86::BT32ri8, &X86::GR32RegClass, Op0, imm1); 18379} 18380 18381unsigned fastEmit_X86ISD_BT_MVT_i64_ri(MVT RetVT, unsigned Op0, uint64_t imm1) { 18382 if (RetVT.SimpleTy != MVT::i32) 18383 return 0; 18384 return fastEmitInst_ri(X86::BT64ri8, &X86::GR64RegClass, Op0, imm1); 18385} 18386 18387unsigned fastEmit_X86ISD_BT_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 18388 switch (VT.SimpleTy) { 18389 case MVT::i16: return fastEmit_X86ISD_BT_MVT_i16_ri(RetVT, Op0, imm1); 18390 case MVT::i32: return fastEmit_X86ISD_BT_MVT_i32_ri(RetVT, Op0, imm1); 18391 case MVT::i64: return fastEmit_X86ISD_BT_MVT_i64_ri(RetVT, Op0, imm1); 18392 default: return 0; 18393 } 18394} 18395 18396// Top-level FastEmit function. 18397 18398unsigned fastEmit_ri(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) override { 18399 if (VT == MVT::i16 && Predicate_i16immSExt8(imm1)) 18400 if (unsigned Reg = fastEmit_ri_Predicate_i16immSExt8(VT, RetVT, Opcode, Op0, imm1)) 18401 return Reg; 18402 18403 if (VT == MVT::i32 && Predicate_i32immSExt8(imm1)) 18404 if (unsigned Reg = fastEmit_ri_Predicate_i32immSExt8(VT, RetVT, Opcode, Op0, imm1)) 18405 return Reg; 18406 18407 if (VT == MVT::i64 && Predicate_i64immSExt8(imm1)) 18408 if (unsigned Reg = fastEmit_ri_Predicate_i64immSExt8(VT, RetVT, Opcode, Op0, imm1)) 18409 return Reg; 18410 18411 if (VT == MVT::i64 && Predicate_i64immSExt32(imm1)) 18412 if (unsigned Reg = fastEmit_ri_Predicate_i64immSExt32(VT, RetVT, Opcode, Op0, imm1)) 18413 return Reg; 18414 18415 switch (Opcode) { 18416 case ISD::ADD: return fastEmit_ISD_ADD_ri(VT, RetVT, Op0, imm1); 18417 case ISD::AND: return fastEmit_ISD_AND_ri(VT, RetVT, Op0, imm1); 18418 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri(VT, RetVT, Op0, imm1); 18419 case ISD::MUL: return fastEmit_ISD_MUL_ri(VT, RetVT, Op0, imm1); 18420 case ISD::OR: return fastEmit_ISD_OR_ri(VT, RetVT, Op0, imm1); 18421 case ISD::ROTL: return fastEmit_ISD_ROTL_ri(VT, RetVT, Op0, imm1); 18422 case ISD::ROTR: return fastEmit_ISD_ROTR_ri(VT, RetVT, Op0, imm1); 18423 case ISD::SHL: return fastEmit_ISD_SHL_ri(VT, RetVT, Op0, imm1); 18424 case ISD::SRA: return fastEmit_ISD_SRA_ri(VT, RetVT, Op0, imm1); 18425 case ISD::SRL: return fastEmit_ISD_SRL_ri(VT, RetVT, Op0, imm1); 18426 case ISD::SUB: return fastEmit_ISD_SUB_ri(VT, RetVT, Op0, imm1); 18427 case ISD::XOR: return fastEmit_ISD_XOR_ri(VT, RetVT, Op0, imm1); 18428 case X86ISD::BT: return fastEmit_X86ISD_BT_ri(VT, RetVT, Op0, imm1); 18429 default: return 0; 18430 } 18431} 18432 18433// FastEmit functions for ISD::ADD. 18434 18435unsigned fastEmit_ISD_ADD_MVT_i16_ri_Predicate_i16immSExt8(MVT RetVT, unsigned Op0, uint64_t imm1) { 18436 if (RetVT.SimpleTy != MVT::i16) 18437 return 0; 18438 return fastEmitInst_ri(X86::ADD16ri8, &X86::GR16RegClass, Op0, imm1); 18439} 18440 18441unsigned fastEmit_ISD_ADD_ri_Predicate_i16immSExt8(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 18442 switch (VT.SimpleTy) { 18443 case MVT::i16: return fastEmit_ISD_ADD_MVT_i16_ri_Predicate_i16immSExt8(RetVT, Op0, imm1); 18444 default: return 0; 18445 } 18446} 18447 18448// FastEmit functions for ISD::AND. 18449 18450unsigned fastEmit_ISD_AND_MVT_i16_ri_Predicate_i16immSExt8(MVT RetVT, unsigned Op0, uint64_t imm1) { 18451 if (RetVT.SimpleTy != MVT::i16) 18452 return 0; 18453 return fastEmitInst_ri(X86::AND16ri8, &X86::GR16RegClass, Op0, imm1); 18454} 18455 18456unsigned fastEmit_ISD_AND_ri_Predicate_i16immSExt8(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 18457 switch (VT.SimpleTy) { 18458 case MVT::i16: return fastEmit_ISD_AND_MVT_i16_ri_Predicate_i16immSExt8(RetVT, Op0, imm1); 18459 default: return 0; 18460 } 18461} 18462 18463// FastEmit functions for ISD::MUL. 18464 18465unsigned fastEmit_ISD_MUL_MVT_i16_ri_Predicate_i16immSExt8(MVT RetVT, unsigned Op0, uint64_t imm1) { 18466 if (RetVT.SimpleTy != MVT::i16) 18467 return 0; 18468 return fastEmitInst_ri(X86::IMUL16rri8, &X86::GR16RegClass, Op0, imm1); 18469} 18470 18471unsigned fastEmit_ISD_MUL_ri_Predicate_i16immSExt8(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 18472 switch (VT.SimpleTy) { 18473 case MVT::i16: return fastEmit_ISD_MUL_MVT_i16_ri_Predicate_i16immSExt8(RetVT, Op0, imm1); 18474 default: return 0; 18475 } 18476} 18477 18478// FastEmit functions for ISD::OR. 18479 18480unsigned fastEmit_ISD_OR_MVT_i16_ri_Predicate_i16immSExt8(MVT RetVT, unsigned Op0, uint64_t imm1) { 18481 if (RetVT.SimpleTy != MVT::i16) 18482 return 0; 18483 return fastEmitInst_ri(X86::OR16ri8, &X86::GR16RegClass, Op0, imm1); 18484} 18485 18486unsigned fastEmit_ISD_OR_ri_Predicate_i16immSExt8(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 18487 switch (VT.SimpleTy) { 18488 case MVT::i16: return fastEmit_ISD_OR_MVT_i16_ri_Predicate_i16immSExt8(RetVT, Op0, imm1); 18489 default: return 0; 18490 } 18491} 18492 18493// FastEmit functions for ISD::SUB. 18494 18495unsigned fastEmit_ISD_SUB_MVT_i16_ri_Predicate_i16immSExt8(MVT RetVT, unsigned Op0, uint64_t imm1) { 18496 if (RetVT.SimpleTy != MVT::i16) 18497 return 0; 18498 return fastEmitInst_ri(X86::SUB16ri8, &X86::GR16RegClass, Op0, imm1); 18499} 18500 18501unsigned fastEmit_ISD_SUB_ri_Predicate_i16immSExt8(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 18502 switch (VT.SimpleTy) { 18503 case MVT::i16: return fastEmit_ISD_SUB_MVT_i16_ri_Predicate_i16immSExt8(RetVT, Op0, imm1); 18504 default: return 0; 18505 } 18506} 18507 18508// FastEmit functions for ISD::XOR. 18509 18510unsigned fastEmit_ISD_XOR_MVT_i16_ri_Predicate_i16immSExt8(MVT RetVT, unsigned Op0, uint64_t imm1) { 18511 if (RetVT.SimpleTy != MVT::i16) 18512 return 0; 18513 return fastEmitInst_ri(X86::XOR16ri8, &X86::GR16RegClass, Op0, imm1); 18514} 18515 18516unsigned fastEmit_ISD_XOR_ri_Predicate_i16immSExt8(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 18517 switch (VT.SimpleTy) { 18518 case MVT::i16: return fastEmit_ISD_XOR_MVT_i16_ri_Predicate_i16immSExt8(RetVT, Op0, imm1); 18519 default: return 0; 18520 } 18521} 18522 18523// Top-level FastEmit function. 18524 18525unsigned fastEmit_ri_Predicate_i16immSExt8(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) { 18526 switch (Opcode) { 18527 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_i16immSExt8(VT, RetVT, Op0, imm1); 18528 case ISD::AND: return fastEmit_ISD_AND_ri_Predicate_i16immSExt8(VT, RetVT, Op0, imm1); 18529 case ISD::MUL: return fastEmit_ISD_MUL_ri_Predicate_i16immSExt8(VT, RetVT, Op0, imm1); 18530 case ISD::OR: return fastEmit_ISD_OR_ri_Predicate_i16immSExt8(VT, RetVT, Op0, imm1); 18531 case ISD::SUB: return fastEmit_ISD_SUB_ri_Predicate_i16immSExt8(VT, RetVT, Op0, imm1); 18532 case ISD::XOR: return fastEmit_ISD_XOR_ri_Predicate_i16immSExt8(VT, RetVT, Op0, imm1); 18533 default: return 0; 18534 } 18535} 18536 18537// FastEmit functions for ISD::ADD. 18538 18539unsigned fastEmit_ISD_ADD_MVT_i32_ri_Predicate_i32immSExt8(MVT RetVT, unsigned Op0, uint64_t imm1) { 18540 if (RetVT.SimpleTy != MVT::i32) 18541 return 0; 18542 return fastEmitInst_ri(X86::ADD32ri8, &X86::GR32RegClass, Op0, imm1); 18543} 18544 18545unsigned fastEmit_ISD_ADD_ri_Predicate_i32immSExt8(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 18546 switch (VT.SimpleTy) { 18547 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri_Predicate_i32immSExt8(RetVT, Op0, imm1); 18548 default: return 0; 18549 } 18550} 18551 18552// FastEmit functions for ISD::AND. 18553 18554unsigned fastEmit_ISD_AND_MVT_i32_ri_Predicate_i32immSExt8(MVT RetVT, unsigned Op0, uint64_t imm1) { 18555 if (RetVT.SimpleTy != MVT::i32) 18556 return 0; 18557 return fastEmitInst_ri(X86::AND32ri8, &X86::GR32RegClass, Op0, imm1); 18558} 18559 18560unsigned fastEmit_ISD_AND_ri_Predicate_i32immSExt8(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 18561 switch (VT.SimpleTy) { 18562 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_ri_Predicate_i32immSExt8(RetVT, Op0, imm1); 18563 default: return 0; 18564 } 18565} 18566 18567// FastEmit functions for ISD::MUL. 18568 18569unsigned fastEmit_ISD_MUL_MVT_i32_ri_Predicate_i32immSExt8(MVT RetVT, unsigned Op0, uint64_t imm1) { 18570 if (RetVT.SimpleTy != MVT::i32) 18571 return 0; 18572 return fastEmitInst_ri(X86::IMUL32rri8, &X86::GR32RegClass, Op0, imm1); 18573} 18574 18575unsigned fastEmit_ISD_MUL_ri_Predicate_i32immSExt8(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 18576 switch (VT.SimpleTy) { 18577 case MVT::i32: return fastEmit_ISD_MUL_MVT_i32_ri_Predicate_i32immSExt8(RetVT, Op0, imm1); 18578 default: return 0; 18579 } 18580} 18581 18582// FastEmit functions for ISD::OR. 18583 18584unsigned fastEmit_ISD_OR_MVT_i32_ri_Predicate_i32immSExt8(MVT RetVT, unsigned Op0, uint64_t imm1) { 18585 if (RetVT.SimpleTy != MVT::i32) 18586 return 0; 18587 return fastEmitInst_ri(X86::OR32ri8, &X86::GR32RegClass, Op0, imm1); 18588} 18589 18590unsigned fastEmit_ISD_OR_ri_Predicate_i32immSExt8(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 18591 switch (VT.SimpleTy) { 18592 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_ri_Predicate_i32immSExt8(RetVT, Op0, imm1); 18593 default: return 0; 18594 } 18595} 18596 18597// FastEmit functions for ISD::SUB. 18598 18599unsigned fastEmit_ISD_SUB_MVT_i32_ri_Predicate_i32immSExt8(MVT RetVT, unsigned Op0, uint64_t imm1) { 18600 if (RetVT.SimpleTy != MVT::i32) 18601 return 0; 18602 return fastEmitInst_ri(X86::SUB32ri8, &X86::GR32RegClass, Op0, imm1); 18603} 18604 18605unsigned fastEmit_ISD_SUB_ri_Predicate_i32immSExt8(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 18606 switch (VT.SimpleTy) { 18607 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_ri_Predicate_i32immSExt8(RetVT, Op0, imm1); 18608 default: return 0; 18609 } 18610} 18611 18612// FastEmit functions for ISD::XOR. 18613 18614unsigned fastEmit_ISD_XOR_MVT_i32_ri_Predicate_i32immSExt8(MVT RetVT, unsigned Op0, uint64_t imm1) { 18615 if (RetVT.SimpleTy != MVT::i32) 18616 return 0; 18617 return fastEmitInst_ri(X86::XOR32ri8, &X86::GR32RegClass, Op0, imm1); 18618} 18619 18620unsigned fastEmit_ISD_XOR_ri_Predicate_i32immSExt8(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 18621 switch (VT.SimpleTy) { 18622 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_ri_Predicate_i32immSExt8(RetVT, Op0, imm1); 18623 default: return 0; 18624 } 18625} 18626 18627// Top-level FastEmit function. 18628 18629unsigned fastEmit_ri_Predicate_i32immSExt8(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) { 18630 switch (Opcode) { 18631 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_i32immSExt8(VT, RetVT, Op0, imm1); 18632 case ISD::AND: return fastEmit_ISD_AND_ri_Predicate_i32immSExt8(VT, RetVT, Op0, imm1); 18633 case ISD::MUL: return fastEmit_ISD_MUL_ri_Predicate_i32immSExt8(VT, RetVT, Op0, imm1); 18634 case ISD::OR: return fastEmit_ISD_OR_ri_Predicate_i32immSExt8(VT, RetVT, Op0, imm1); 18635 case ISD::SUB: return fastEmit_ISD_SUB_ri_Predicate_i32immSExt8(VT, RetVT, Op0, imm1); 18636 case ISD::XOR: return fastEmit_ISD_XOR_ri_Predicate_i32immSExt8(VT, RetVT, Op0, imm1); 18637 default: return 0; 18638 } 18639} 18640 18641// FastEmit functions for ISD::ADD. 18642 18643unsigned fastEmit_ISD_ADD_MVT_i64_ri_Predicate_i64immSExt8(MVT RetVT, unsigned Op0, uint64_t imm1) { 18644 if (RetVT.SimpleTy != MVT::i64) 18645 return 0; 18646 return fastEmitInst_ri(X86::ADD64ri8, &X86::GR64RegClass, Op0, imm1); 18647} 18648 18649unsigned fastEmit_ISD_ADD_ri_Predicate_i64immSExt8(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 18650 switch (VT.SimpleTy) { 18651 case MVT::i64: return fastEmit_ISD_ADD_MVT_i64_ri_Predicate_i64immSExt8(RetVT, Op0, imm1); 18652 default: return 0; 18653 } 18654} 18655 18656// FastEmit functions for ISD::AND. 18657 18658unsigned fastEmit_ISD_AND_MVT_i64_ri_Predicate_i64immSExt8(MVT RetVT, unsigned Op0, uint64_t imm1) { 18659 if (RetVT.SimpleTy != MVT::i64) 18660 return 0; 18661 return fastEmitInst_ri(X86::AND64ri8, &X86::GR64RegClass, Op0, imm1); 18662} 18663 18664unsigned fastEmit_ISD_AND_ri_Predicate_i64immSExt8(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 18665 switch (VT.SimpleTy) { 18666 case MVT::i64: return fastEmit_ISD_AND_MVT_i64_ri_Predicate_i64immSExt8(RetVT, Op0, imm1); 18667 default: return 0; 18668 } 18669} 18670 18671// FastEmit functions for ISD::MUL. 18672 18673unsigned fastEmit_ISD_MUL_MVT_i64_ri_Predicate_i64immSExt8(MVT RetVT, unsigned Op0, uint64_t imm1) { 18674 if (RetVT.SimpleTy != MVT::i64) 18675 return 0; 18676 return fastEmitInst_ri(X86::IMUL64rri8, &X86::GR64RegClass, Op0, imm1); 18677} 18678 18679unsigned fastEmit_ISD_MUL_ri_Predicate_i64immSExt8(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 18680 switch (VT.SimpleTy) { 18681 case MVT::i64: return fastEmit_ISD_MUL_MVT_i64_ri_Predicate_i64immSExt8(RetVT, Op0, imm1); 18682 default: return 0; 18683 } 18684} 18685 18686// FastEmit functions for ISD::OR. 18687 18688unsigned fastEmit_ISD_OR_MVT_i64_ri_Predicate_i64immSExt8(MVT RetVT, unsigned Op0, uint64_t imm1) { 18689 if (RetVT.SimpleTy != MVT::i64) 18690 return 0; 18691 return fastEmitInst_ri(X86::OR64ri8, &X86::GR64RegClass, Op0, imm1); 18692} 18693 18694unsigned fastEmit_ISD_OR_ri_Predicate_i64immSExt8(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 18695 switch (VT.SimpleTy) { 18696 case MVT::i64: return fastEmit_ISD_OR_MVT_i64_ri_Predicate_i64immSExt8(RetVT, Op0, imm1); 18697 default: return 0; 18698 } 18699} 18700 18701// FastEmit functions for ISD::SUB. 18702 18703unsigned fastEmit_ISD_SUB_MVT_i64_ri_Predicate_i64immSExt8(MVT RetVT, unsigned Op0, uint64_t imm1) { 18704 if (RetVT.SimpleTy != MVT::i64) 18705 return 0; 18706 return fastEmitInst_ri(X86::SUB64ri8, &X86::GR64RegClass, Op0, imm1); 18707} 18708 18709unsigned fastEmit_ISD_SUB_ri_Predicate_i64immSExt8(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 18710 switch (VT.SimpleTy) { 18711 case MVT::i64: return fastEmit_ISD_SUB_MVT_i64_ri_Predicate_i64immSExt8(RetVT, Op0, imm1); 18712 default: return 0; 18713 } 18714} 18715 18716// FastEmit functions for ISD::XOR. 18717 18718unsigned fastEmit_ISD_XOR_MVT_i64_ri_Predicate_i64immSExt8(MVT RetVT, unsigned Op0, uint64_t imm1) { 18719 if (RetVT.SimpleTy != MVT::i64) 18720 return 0; 18721 return fastEmitInst_ri(X86::XOR64ri8, &X86::GR64RegClass, Op0, imm1); 18722} 18723 18724unsigned fastEmit_ISD_XOR_ri_Predicate_i64immSExt8(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 18725 switch (VT.SimpleTy) { 18726 case MVT::i64: return fastEmit_ISD_XOR_MVT_i64_ri_Predicate_i64immSExt8(RetVT, Op0, imm1); 18727 default: return 0; 18728 } 18729} 18730 18731// Top-level FastEmit function. 18732 18733unsigned fastEmit_ri_Predicate_i64immSExt8(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) { 18734 switch (Opcode) { 18735 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_i64immSExt8(VT, RetVT, Op0, imm1); 18736 case ISD::AND: return fastEmit_ISD_AND_ri_Predicate_i64immSExt8(VT, RetVT, Op0, imm1); 18737 case ISD::MUL: return fastEmit_ISD_MUL_ri_Predicate_i64immSExt8(VT, RetVT, Op0, imm1); 18738 case ISD::OR: return fastEmit_ISD_OR_ri_Predicate_i64immSExt8(VT, RetVT, Op0, imm1); 18739 case ISD::SUB: return fastEmit_ISD_SUB_ri_Predicate_i64immSExt8(VT, RetVT, Op0, imm1); 18740 case ISD::XOR: return fastEmit_ISD_XOR_ri_Predicate_i64immSExt8(VT, RetVT, Op0, imm1); 18741 default: return 0; 18742 } 18743} 18744 18745// FastEmit functions for ISD::ADD. 18746 18747unsigned fastEmit_ISD_ADD_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, unsigned Op0, uint64_t imm1) { 18748 if (RetVT.SimpleTy != MVT::i64) 18749 return 0; 18750 return fastEmitInst_ri(X86::ADD64ri32, &X86::GR64RegClass, Op0, imm1); 18751} 18752 18753unsigned fastEmit_ISD_ADD_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 18754 switch (VT.SimpleTy) { 18755 case MVT::i64: return fastEmit_ISD_ADD_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, imm1); 18756 default: return 0; 18757 } 18758} 18759 18760// FastEmit functions for ISD::AND. 18761 18762unsigned fastEmit_ISD_AND_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, unsigned Op0, uint64_t imm1) { 18763 if (RetVT.SimpleTy != MVT::i64) 18764 return 0; 18765 return fastEmitInst_ri(X86::AND64ri32, &X86::GR64RegClass, Op0, imm1); 18766} 18767 18768unsigned fastEmit_ISD_AND_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 18769 switch (VT.SimpleTy) { 18770 case MVT::i64: return fastEmit_ISD_AND_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, imm1); 18771 default: return 0; 18772 } 18773} 18774 18775// FastEmit functions for ISD::MUL. 18776 18777unsigned fastEmit_ISD_MUL_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, unsigned Op0, uint64_t imm1) { 18778 if (RetVT.SimpleTy != MVT::i64) 18779 return 0; 18780 return fastEmitInst_ri(X86::IMUL64rri32, &X86::GR64RegClass, Op0, imm1); 18781} 18782 18783unsigned fastEmit_ISD_MUL_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 18784 switch (VT.SimpleTy) { 18785 case MVT::i64: return fastEmit_ISD_MUL_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, imm1); 18786 default: return 0; 18787 } 18788} 18789 18790// FastEmit functions for ISD::OR. 18791 18792unsigned fastEmit_ISD_OR_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, unsigned Op0, uint64_t imm1) { 18793 if (RetVT.SimpleTy != MVT::i64) 18794 return 0; 18795 return fastEmitInst_ri(X86::OR64ri32, &X86::GR64RegClass, Op0, imm1); 18796} 18797 18798unsigned fastEmit_ISD_OR_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 18799 switch (VT.SimpleTy) { 18800 case MVT::i64: return fastEmit_ISD_OR_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, imm1); 18801 default: return 0; 18802 } 18803} 18804 18805// FastEmit functions for ISD::SUB. 18806 18807unsigned fastEmit_ISD_SUB_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, unsigned Op0, uint64_t imm1) { 18808 if (RetVT.SimpleTy != MVT::i64) 18809 return 0; 18810 return fastEmitInst_ri(X86::SUB64ri32, &X86::GR64RegClass, Op0, imm1); 18811} 18812 18813unsigned fastEmit_ISD_SUB_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 18814 switch (VT.SimpleTy) { 18815 case MVT::i64: return fastEmit_ISD_SUB_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, imm1); 18816 default: return 0; 18817 } 18818} 18819 18820// FastEmit functions for ISD::XOR. 18821 18822unsigned fastEmit_ISD_XOR_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, unsigned Op0, uint64_t imm1) { 18823 if (RetVT.SimpleTy != MVT::i64) 18824 return 0; 18825 return fastEmitInst_ri(X86::XOR64ri32, &X86::GR64RegClass, Op0, imm1); 18826} 18827 18828unsigned fastEmit_ISD_XOR_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 18829 switch (VT.SimpleTy) { 18830 case MVT::i64: return fastEmit_ISD_XOR_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, imm1); 18831 default: return 0; 18832 } 18833} 18834 18835// Top-level FastEmit function. 18836 18837unsigned fastEmit_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) { 18838 switch (Opcode) { 18839 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_i64immSExt32(VT, RetVT, Op0, imm1); 18840 case ISD::AND: return fastEmit_ISD_AND_ri_Predicate_i64immSExt32(VT, RetVT, Op0, imm1); 18841 case ISD::MUL: return fastEmit_ISD_MUL_ri_Predicate_i64immSExt32(VT, RetVT, Op0, imm1); 18842 case ISD::OR: return fastEmit_ISD_OR_ri_Predicate_i64immSExt32(VT, RetVT, Op0, imm1); 18843 case ISD::SUB: return fastEmit_ISD_SUB_ri_Predicate_i64immSExt32(VT, RetVT, Op0, imm1); 18844 case ISD::XOR: return fastEmit_ISD_XOR_ri_Predicate_i64immSExt32(VT, RetVT, Op0, imm1); 18845 default: return 0; 18846 } 18847} 18848 18849// FastEmit functions for ISD::Constant. 18850 18851unsigned fastEmit_ISD_Constant_MVT_i8_i(MVT RetVT, uint64_t imm0) { 18852 if (RetVT.SimpleTy != MVT::i8) 18853 return 0; 18854 return fastEmitInst_i(X86::MOV8ri, &X86::GR8RegClass, imm0); 18855} 18856 18857unsigned fastEmit_ISD_Constant_MVT_i16_i(MVT RetVT, uint64_t imm0) { 18858 if (RetVT.SimpleTy != MVT::i16) 18859 return 0; 18860 return fastEmitInst_i(X86::MOV16ri, &X86::GR16RegClass, imm0); 18861} 18862 18863unsigned fastEmit_ISD_Constant_MVT_i32_i(MVT RetVT, uint64_t imm0) { 18864 if (RetVT.SimpleTy != MVT::i32) 18865 return 0; 18866 return fastEmitInst_i(X86::MOV32ri, &X86::GR32RegClass, imm0); 18867} 18868 18869unsigned fastEmit_ISD_Constant_MVT_i64_i(MVT RetVT, uint64_t imm0) { 18870 if (RetVT.SimpleTy != MVT::i64) 18871 return 0; 18872 return fastEmitInst_i(X86::MOV64ri, &X86::GR64RegClass, imm0); 18873} 18874 18875unsigned fastEmit_ISD_Constant_i(MVT VT, MVT RetVT, uint64_t imm0) { 18876 switch (VT.SimpleTy) { 18877 case MVT::i8: return fastEmit_ISD_Constant_MVT_i8_i(RetVT, imm0); 18878 case MVT::i16: return fastEmit_ISD_Constant_MVT_i16_i(RetVT, imm0); 18879 case MVT::i32: return fastEmit_ISD_Constant_MVT_i32_i(RetVT, imm0); 18880 case MVT::i64: return fastEmit_ISD_Constant_MVT_i64_i(RetVT, imm0); 18881 default: return 0; 18882 } 18883} 18884 18885// Top-level FastEmit function. 18886 18887unsigned fastEmit_i(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) override { 18888 switch (Opcode) { 18889 case ISD::Constant: return fastEmit_ISD_Constant_i(VT, RetVT, imm0); 18890 default: return 0; 18891 } 18892} 18893 18894