1/*===- TableGen'erated file -------------------------------------*- C++ -*-===*\ 2|* *| 3|* "Fast" Instruction Selector for the AArch64 target *| 4|* *| 5|* Automatically generated file, do not edit! *| 6|* *| 7\*===----------------------------------------------------------------------===*/ 8 9 10// FastEmit Immediate Predicate functions. 11static bool Predicate_imm0_31(int64_t Imm) { 12 13 return ((uint64_t)Imm) < 32; 14 15} 16static bool Predicate_imm0_63(int64_t Imm) { 17 18 return ((uint64_t)Imm) < 64; 19 20} 21static bool Predicate_imm32_0_31(int64_t Imm) { 22 23 return ((uint64_t)Imm) < 32; 24 25} 26static bool Predicate_tbz_imm0_31_diag(int64_t Imm) { 27 28 return (((uint32_t)Imm) < 32); 29 30} 31static bool Predicate_tbz_imm32_63(int64_t Imm) { 32 33 return (((uint32_t)Imm) > 31) && (((uint32_t)Imm) < 64); 34 35} 36static bool Predicate_VectorIndexD(int64_t Imm) { 37 return ((uint64_t)Imm) < 2; 38} 39static bool Predicate_VectorIndexS(int64_t Imm) { 40 return ((uint64_t)Imm) < 4; 41} 42static bool Predicate_VectorIndexH(int64_t Imm) { 43 return ((uint64_t)Imm) < 8; 44} 45static bool Predicate_VectorIndexB(int64_t Imm) { 46 return ((uint64_t)Imm) < 16; 47} 48static bool Predicate_VectorIndex0(int64_t Imm) { 49 return ((uint64_t)Imm) == 0; 50} 51static bool Predicate_imm0_255(int64_t Imm) { 52 53 return ((uint32_t)Imm) < 256; 54 55} 56static bool Predicate_vecshiftL64(int64_t Imm) { 57 58 return (((uint32_t)Imm) < 64); 59 60} 61static bool Predicate_vecshiftL32(int64_t Imm) { 62 63 return (((uint32_t)Imm) < 32); 64 65} 66static bool Predicate_vecshiftR64(int64_t Imm) { 67 68 return (((uint32_t)Imm) > 0) && (((uint32_t)Imm) < 65); 69 70} 71static bool Predicate_vecshiftL8(int64_t Imm) { 72 73 return (((uint32_t)Imm) < 8); 74 75} 76static bool Predicate_vecshiftL16(int64_t Imm) { 77 78 return (((uint32_t)Imm) < 16); 79 80} 81static bool Predicate_vecshiftR8(int64_t Imm) { 82 83 return (((uint32_t)Imm) > 0) && (((uint32_t)Imm) < 9); 84 85} 86static bool Predicate_vecshiftR16(int64_t Imm) { 87 88 return (((uint32_t)Imm) > 0) && (((uint32_t)Imm) < 17); 89 90} 91static bool Predicate_vecshiftR32(int64_t Imm) { 92 93 return (((uint32_t)Imm) > 0) && (((uint32_t)Imm) < 33); 94 95} 96static bool Predicate_simm8_32b(int64_t Imm) { 97 return Imm >= -128 && Imm < 128; 98} 99static bool Predicate_simm8_64b(int64_t Imm) { 100 return Imm >= -128 && Imm < 128; 101} 102static bool Predicate_uimm8_32b(int64_t Imm) { 103 return Imm >= 0 && Imm < 256; 104} 105static bool Predicate_uimm8_64b(int64_t Imm) { 106 return Imm >= 0 && Imm < 256; 107} 108static bool Predicate_simm6_32b(int64_t Imm) { 109 return Imm >= -32 && Imm < 32; 110} 111 112 113// FastEmit functions for AArch64ISD::THREAD_POINTER. 114 115unsigned fastEmit_AArch64ISD_THREAD_POINTER_MVT_i64_(MVT RetVT) { 116 if (RetVT.SimpleTy != MVT::i64) 117 return 0; 118 return fastEmitInst_(AArch64::MOVbaseTLS, &AArch64::GPR64RegClass); 119} 120 121unsigned fastEmit_AArch64ISD_THREAD_POINTER_(MVT VT, MVT RetVT) { 122 switch (VT.SimpleTy) { 123 case MVT::i64: return fastEmit_AArch64ISD_THREAD_POINTER_MVT_i64_(RetVT); 124 default: return 0; 125 } 126} 127 128// Top-level FastEmit function. 129 130unsigned fastEmit_(MVT VT, MVT RetVT, unsigned Opcode) override { 131 switch (Opcode) { 132 case AArch64ISD::THREAD_POINTER: return fastEmit_AArch64ISD_THREAD_POINTER_(VT, RetVT); 133 default: return 0; 134 } 135} 136 137// FastEmit functions for AArch64ISD::CALL. 138 139unsigned fastEmit_AArch64ISD_CALL_MVT_i64_r(MVT RetVT, unsigned Op0) { 140 if (RetVT.SimpleTy != MVT::isVoid) 141 return 0; 142 if (( MF->getSubtarget<AArch64Subtarget>().hardenSlsBlr() )) { 143 return fastEmitInst_r(AArch64::BLRNoIP, &AArch64::GPR64noipRegClass, Op0); 144 } 145 if (( !MF->getSubtarget<AArch64Subtarget>().hardenSlsBlr() )) { 146 return fastEmitInst_r(AArch64::BLR, &AArch64::GPR64RegClass, Op0); 147 } 148 return 0; 149} 150 151unsigned fastEmit_AArch64ISD_CALL_r(MVT VT, MVT RetVT, unsigned Op0) { 152 switch (VT.SimpleTy) { 153 case MVT::i64: return fastEmit_AArch64ISD_CALL_MVT_i64_r(RetVT, Op0); 154 default: return 0; 155 } 156} 157 158// FastEmit functions for AArch64ISD::CMEQz. 159 160unsigned fastEmit_AArch64ISD_CMEQz_MVT_v8i8_r(MVT RetVT, unsigned Op0) { 161 if (RetVT.SimpleTy != MVT::v8i8) 162 return 0; 163 if ((Subtarget->hasNEON())) { 164 return fastEmitInst_r(AArch64::CMEQv8i8rz, &AArch64::FPR64RegClass, Op0); 165 } 166 return 0; 167} 168 169unsigned fastEmit_AArch64ISD_CMEQz_MVT_v16i8_r(MVT RetVT, unsigned Op0) { 170 if (RetVT.SimpleTy != MVT::v16i8) 171 return 0; 172 if ((Subtarget->hasNEON())) { 173 return fastEmitInst_r(AArch64::CMEQv16i8rz, &AArch64::FPR128RegClass, Op0); 174 } 175 return 0; 176} 177 178unsigned fastEmit_AArch64ISD_CMEQz_MVT_v4i16_r(MVT RetVT, unsigned Op0) { 179 if (RetVT.SimpleTy != MVT::v4i16) 180 return 0; 181 if ((Subtarget->hasNEON())) { 182 return fastEmitInst_r(AArch64::CMEQv4i16rz, &AArch64::FPR64RegClass, Op0); 183 } 184 return 0; 185} 186 187unsigned fastEmit_AArch64ISD_CMEQz_MVT_v8i16_r(MVT RetVT, unsigned Op0) { 188 if (RetVT.SimpleTy != MVT::v8i16) 189 return 0; 190 if ((Subtarget->hasNEON())) { 191 return fastEmitInst_r(AArch64::CMEQv8i16rz, &AArch64::FPR128RegClass, Op0); 192 } 193 return 0; 194} 195 196unsigned fastEmit_AArch64ISD_CMEQz_MVT_v2i32_r(MVT RetVT, unsigned Op0) { 197 if (RetVT.SimpleTy != MVT::v2i32) 198 return 0; 199 if ((Subtarget->hasNEON())) { 200 return fastEmitInst_r(AArch64::CMEQv2i32rz, &AArch64::FPR64RegClass, Op0); 201 } 202 return 0; 203} 204 205unsigned fastEmit_AArch64ISD_CMEQz_MVT_v4i32_r(MVT RetVT, unsigned Op0) { 206 if (RetVT.SimpleTy != MVT::v4i32) 207 return 0; 208 if ((Subtarget->hasNEON())) { 209 return fastEmitInst_r(AArch64::CMEQv4i32rz, &AArch64::FPR128RegClass, Op0); 210 } 211 return 0; 212} 213 214unsigned fastEmit_AArch64ISD_CMEQz_MVT_v1i64_r(MVT RetVT, unsigned Op0) { 215 if (RetVT.SimpleTy != MVT::v1i64) 216 return 0; 217 if ((Subtarget->hasNEON())) { 218 return fastEmitInst_r(AArch64::CMEQv1i64rz, &AArch64::FPR64RegClass, Op0); 219 } 220 return 0; 221} 222 223unsigned fastEmit_AArch64ISD_CMEQz_MVT_v2i64_r(MVT RetVT, unsigned Op0) { 224 if (RetVT.SimpleTy != MVT::v2i64) 225 return 0; 226 if ((Subtarget->hasNEON())) { 227 return fastEmitInst_r(AArch64::CMEQv2i64rz, &AArch64::FPR128RegClass, Op0); 228 } 229 return 0; 230} 231 232unsigned fastEmit_AArch64ISD_CMEQz_r(MVT VT, MVT RetVT, unsigned Op0) { 233 switch (VT.SimpleTy) { 234 case MVT::v8i8: return fastEmit_AArch64ISD_CMEQz_MVT_v8i8_r(RetVT, Op0); 235 case MVT::v16i8: return fastEmit_AArch64ISD_CMEQz_MVT_v16i8_r(RetVT, Op0); 236 case MVT::v4i16: return fastEmit_AArch64ISD_CMEQz_MVT_v4i16_r(RetVT, Op0); 237 case MVT::v8i16: return fastEmit_AArch64ISD_CMEQz_MVT_v8i16_r(RetVT, Op0); 238 case MVT::v2i32: return fastEmit_AArch64ISD_CMEQz_MVT_v2i32_r(RetVT, Op0); 239 case MVT::v4i32: return fastEmit_AArch64ISD_CMEQz_MVT_v4i32_r(RetVT, Op0); 240 case MVT::v1i64: return fastEmit_AArch64ISD_CMEQz_MVT_v1i64_r(RetVT, Op0); 241 case MVT::v2i64: return fastEmit_AArch64ISD_CMEQz_MVT_v2i64_r(RetVT, Op0); 242 default: return 0; 243 } 244} 245 246// FastEmit functions for AArch64ISD::CMGEz. 247 248unsigned fastEmit_AArch64ISD_CMGEz_MVT_v8i8_r(MVT RetVT, unsigned Op0) { 249 if (RetVT.SimpleTy != MVT::v8i8) 250 return 0; 251 if ((Subtarget->hasNEON())) { 252 return fastEmitInst_r(AArch64::CMGEv8i8rz, &AArch64::FPR64RegClass, Op0); 253 } 254 return 0; 255} 256 257unsigned fastEmit_AArch64ISD_CMGEz_MVT_v16i8_r(MVT RetVT, unsigned Op0) { 258 if (RetVT.SimpleTy != MVT::v16i8) 259 return 0; 260 if ((Subtarget->hasNEON())) { 261 return fastEmitInst_r(AArch64::CMGEv16i8rz, &AArch64::FPR128RegClass, Op0); 262 } 263 return 0; 264} 265 266unsigned fastEmit_AArch64ISD_CMGEz_MVT_v4i16_r(MVT RetVT, unsigned Op0) { 267 if (RetVT.SimpleTy != MVT::v4i16) 268 return 0; 269 if ((Subtarget->hasNEON())) { 270 return fastEmitInst_r(AArch64::CMGEv4i16rz, &AArch64::FPR64RegClass, Op0); 271 } 272 return 0; 273} 274 275unsigned fastEmit_AArch64ISD_CMGEz_MVT_v8i16_r(MVT RetVT, unsigned Op0) { 276 if (RetVT.SimpleTy != MVT::v8i16) 277 return 0; 278 if ((Subtarget->hasNEON())) { 279 return fastEmitInst_r(AArch64::CMGEv8i16rz, &AArch64::FPR128RegClass, Op0); 280 } 281 return 0; 282} 283 284unsigned fastEmit_AArch64ISD_CMGEz_MVT_v2i32_r(MVT RetVT, unsigned Op0) { 285 if (RetVT.SimpleTy != MVT::v2i32) 286 return 0; 287 if ((Subtarget->hasNEON())) { 288 return fastEmitInst_r(AArch64::CMGEv2i32rz, &AArch64::FPR64RegClass, Op0); 289 } 290 return 0; 291} 292 293unsigned fastEmit_AArch64ISD_CMGEz_MVT_v4i32_r(MVT RetVT, unsigned Op0) { 294 if (RetVT.SimpleTy != MVT::v4i32) 295 return 0; 296 if ((Subtarget->hasNEON())) { 297 return fastEmitInst_r(AArch64::CMGEv4i32rz, &AArch64::FPR128RegClass, Op0); 298 } 299 return 0; 300} 301 302unsigned fastEmit_AArch64ISD_CMGEz_MVT_v1i64_r(MVT RetVT, unsigned Op0) { 303 if (RetVT.SimpleTy != MVT::v1i64) 304 return 0; 305 if ((Subtarget->hasNEON())) { 306 return fastEmitInst_r(AArch64::CMGEv1i64rz, &AArch64::FPR64RegClass, Op0); 307 } 308 return 0; 309} 310 311unsigned fastEmit_AArch64ISD_CMGEz_MVT_v2i64_r(MVT RetVT, unsigned Op0) { 312 if (RetVT.SimpleTy != MVT::v2i64) 313 return 0; 314 if ((Subtarget->hasNEON())) { 315 return fastEmitInst_r(AArch64::CMGEv2i64rz, &AArch64::FPR128RegClass, Op0); 316 } 317 return 0; 318} 319 320unsigned fastEmit_AArch64ISD_CMGEz_r(MVT VT, MVT RetVT, unsigned Op0) { 321 switch (VT.SimpleTy) { 322 case MVT::v8i8: return fastEmit_AArch64ISD_CMGEz_MVT_v8i8_r(RetVT, Op0); 323 case MVT::v16i8: return fastEmit_AArch64ISD_CMGEz_MVT_v16i8_r(RetVT, Op0); 324 case MVT::v4i16: return fastEmit_AArch64ISD_CMGEz_MVT_v4i16_r(RetVT, Op0); 325 case MVT::v8i16: return fastEmit_AArch64ISD_CMGEz_MVT_v8i16_r(RetVT, Op0); 326 case MVT::v2i32: return fastEmit_AArch64ISD_CMGEz_MVT_v2i32_r(RetVT, Op0); 327 case MVT::v4i32: return fastEmit_AArch64ISD_CMGEz_MVT_v4i32_r(RetVT, Op0); 328 case MVT::v1i64: return fastEmit_AArch64ISD_CMGEz_MVT_v1i64_r(RetVT, Op0); 329 case MVT::v2i64: return fastEmit_AArch64ISD_CMGEz_MVT_v2i64_r(RetVT, Op0); 330 default: return 0; 331 } 332} 333 334// FastEmit functions for AArch64ISD::CMGTz. 335 336unsigned fastEmit_AArch64ISD_CMGTz_MVT_v8i8_r(MVT RetVT, unsigned Op0) { 337 if (RetVT.SimpleTy != MVT::v8i8) 338 return 0; 339 if ((Subtarget->hasNEON())) { 340 return fastEmitInst_r(AArch64::CMGTv8i8rz, &AArch64::FPR64RegClass, Op0); 341 } 342 return 0; 343} 344 345unsigned fastEmit_AArch64ISD_CMGTz_MVT_v16i8_r(MVT RetVT, unsigned Op0) { 346 if (RetVT.SimpleTy != MVT::v16i8) 347 return 0; 348 if ((Subtarget->hasNEON())) { 349 return fastEmitInst_r(AArch64::CMGTv16i8rz, &AArch64::FPR128RegClass, Op0); 350 } 351 return 0; 352} 353 354unsigned fastEmit_AArch64ISD_CMGTz_MVT_v4i16_r(MVT RetVT, unsigned Op0) { 355 if (RetVT.SimpleTy != MVT::v4i16) 356 return 0; 357 if ((Subtarget->hasNEON())) { 358 return fastEmitInst_r(AArch64::CMGTv4i16rz, &AArch64::FPR64RegClass, Op0); 359 } 360 return 0; 361} 362 363unsigned fastEmit_AArch64ISD_CMGTz_MVT_v8i16_r(MVT RetVT, unsigned Op0) { 364 if (RetVT.SimpleTy != MVT::v8i16) 365 return 0; 366 if ((Subtarget->hasNEON())) { 367 return fastEmitInst_r(AArch64::CMGTv8i16rz, &AArch64::FPR128RegClass, Op0); 368 } 369 return 0; 370} 371 372unsigned fastEmit_AArch64ISD_CMGTz_MVT_v2i32_r(MVT RetVT, unsigned Op0) { 373 if (RetVT.SimpleTy != MVT::v2i32) 374 return 0; 375 if ((Subtarget->hasNEON())) { 376 return fastEmitInst_r(AArch64::CMGTv2i32rz, &AArch64::FPR64RegClass, Op0); 377 } 378 return 0; 379} 380 381unsigned fastEmit_AArch64ISD_CMGTz_MVT_v4i32_r(MVT RetVT, unsigned Op0) { 382 if (RetVT.SimpleTy != MVT::v4i32) 383 return 0; 384 if ((Subtarget->hasNEON())) { 385 return fastEmitInst_r(AArch64::CMGTv4i32rz, &AArch64::FPR128RegClass, Op0); 386 } 387 return 0; 388} 389 390unsigned fastEmit_AArch64ISD_CMGTz_MVT_v1i64_r(MVT RetVT, unsigned Op0) { 391 if (RetVT.SimpleTy != MVT::v1i64) 392 return 0; 393 if ((Subtarget->hasNEON())) { 394 return fastEmitInst_r(AArch64::CMGTv1i64rz, &AArch64::FPR64RegClass, Op0); 395 } 396 return 0; 397} 398 399unsigned fastEmit_AArch64ISD_CMGTz_MVT_v2i64_r(MVT RetVT, unsigned Op0) { 400 if (RetVT.SimpleTy != MVT::v2i64) 401 return 0; 402 if ((Subtarget->hasNEON())) { 403 return fastEmitInst_r(AArch64::CMGTv2i64rz, &AArch64::FPR128RegClass, Op0); 404 } 405 return 0; 406} 407 408unsigned fastEmit_AArch64ISD_CMGTz_r(MVT VT, MVT RetVT, unsigned Op0) { 409 switch (VT.SimpleTy) { 410 case MVT::v8i8: return fastEmit_AArch64ISD_CMGTz_MVT_v8i8_r(RetVT, Op0); 411 case MVT::v16i8: return fastEmit_AArch64ISD_CMGTz_MVT_v16i8_r(RetVT, Op0); 412 case MVT::v4i16: return fastEmit_AArch64ISD_CMGTz_MVT_v4i16_r(RetVT, Op0); 413 case MVT::v8i16: return fastEmit_AArch64ISD_CMGTz_MVT_v8i16_r(RetVT, Op0); 414 case MVT::v2i32: return fastEmit_AArch64ISD_CMGTz_MVT_v2i32_r(RetVT, Op0); 415 case MVT::v4i32: return fastEmit_AArch64ISD_CMGTz_MVT_v4i32_r(RetVT, Op0); 416 case MVT::v1i64: return fastEmit_AArch64ISD_CMGTz_MVT_v1i64_r(RetVT, Op0); 417 case MVT::v2i64: return fastEmit_AArch64ISD_CMGTz_MVT_v2i64_r(RetVT, Op0); 418 default: return 0; 419 } 420} 421 422// FastEmit functions for AArch64ISD::CMLEz. 423 424unsigned fastEmit_AArch64ISD_CMLEz_MVT_v8i8_r(MVT RetVT, unsigned Op0) { 425 if (RetVT.SimpleTy != MVT::v8i8) 426 return 0; 427 if ((Subtarget->hasNEON())) { 428 return fastEmitInst_r(AArch64::CMLEv8i8rz, &AArch64::FPR64RegClass, Op0); 429 } 430 return 0; 431} 432 433unsigned fastEmit_AArch64ISD_CMLEz_MVT_v16i8_r(MVT RetVT, unsigned Op0) { 434 if (RetVT.SimpleTy != MVT::v16i8) 435 return 0; 436 if ((Subtarget->hasNEON())) { 437 return fastEmitInst_r(AArch64::CMLEv16i8rz, &AArch64::FPR128RegClass, Op0); 438 } 439 return 0; 440} 441 442unsigned fastEmit_AArch64ISD_CMLEz_MVT_v4i16_r(MVT RetVT, unsigned Op0) { 443 if (RetVT.SimpleTy != MVT::v4i16) 444 return 0; 445 if ((Subtarget->hasNEON())) { 446 return fastEmitInst_r(AArch64::CMLEv4i16rz, &AArch64::FPR64RegClass, Op0); 447 } 448 return 0; 449} 450 451unsigned fastEmit_AArch64ISD_CMLEz_MVT_v8i16_r(MVT RetVT, unsigned Op0) { 452 if (RetVT.SimpleTy != MVT::v8i16) 453 return 0; 454 if ((Subtarget->hasNEON())) { 455 return fastEmitInst_r(AArch64::CMLEv8i16rz, &AArch64::FPR128RegClass, Op0); 456 } 457 return 0; 458} 459 460unsigned fastEmit_AArch64ISD_CMLEz_MVT_v2i32_r(MVT RetVT, unsigned Op0) { 461 if (RetVT.SimpleTy != MVT::v2i32) 462 return 0; 463 if ((Subtarget->hasNEON())) { 464 return fastEmitInst_r(AArch64::CMLEv2i32rz, &AArch64::FPR64RegClass, Op0); 465 } 466 return 0; 467} 468 469unsigned fastEmit_AArch64ISD_CMLEz_MVT_v4i32_r(MVT RetVT, unsigned Op0) { 470 if (RetVT.SimpleTy != MVT::v4i32) 471 return 0; 472 if ((Subtarget->hasNEON())) { 473 return fastEmitInst_r(AArch64::CMLEv4i32rz, &AArch64::FPR128RegClass, Op0); 474 } 475 return 0; 476} 477 478unsigned fastEmit_AArch64ISD_CMLEz_MVT_v1i64_r(MVT RetVT, unsigned Op0) { 479 if (RetVT.SimpleTy != MVT::v1i64) 480 return 0; 481 if ((Subtarget->hasNEON())) { 482 return fastEmitInst_r(AArch64::CMLEv1i64rz, &AArch64::FPR64RegClass, Op0); 483 } 484 return 0; 485} 486 487unsigned fastEmit_AArch64ISD_CMLEz_MVT_v2i64_r(MVT RetVT, unsigned Op0) { 488 if (RetVT.SimpleTy != MVT::v2i64) 489 return 0; 490 if ((Subtarget->hasNEON())) { 491 return fastEmitInst_r(AArch64::CMLEv2i64rz, &AArch64::FPR128RegClass, Op0); 492 } 493 return 0; 494} 495 496unsigned fastEmit_AArch64ISD_CMLEz_r(MVT VT, MVT RetVT, unsigned Op0) { 497 switch (VT.SimpleTy) { 498 case MVT::v8i8: return fastEmit_AArch64ISD_CMLEz_MVT_v8i8_r(RetVT, Op0); 499 case MVT::v16i8: return fastEmit_AArch64ISD_CMLEz_MVT_v16i8_r(RetVT, Op0); 500 case MVT::v4i16: return fastEmit_AArch64ISD_CMLEz_MVT_v4i16_r(RetVT, Op0); 501 case MVT::v8i16: return fastEmit_AArch64ISD_CMLEz_MVT_v8i16_r(RetVT, Op0); 502 case MVT::v2i32: return fastEmit_AArch64ISD_CMLEz_MVT_v2i32_r(RetVT, Op0); 503 case MVT::v4i32: return fastEmit_AArch64ISD_CMLEz_MVT_v4i32_r(RetVT, Op0); 504 case MVT::v1i64: return fastEmit_AArch64ISD_CMLEz_MVT_v1i64_r(RetVT, Op0); 505 case MVT::v2i64: return fastEmit_AArch64ISD_CMLEz_MVT_v2i64_r(RetVT, Op0); 506 default: return 0; 507 } 508} 509 510// FastEmit functions for AArch64ISD::CMLTz. 511 512unsigned fastEmit_AArch64ISD_CMLTz_MVT_v8i8_r(MVT RetVT, unsigned Op0) { 513 if (RetVT.SimpleTy != MVT::v8i8) 514 return 0; 515 if ((Subtarget->hasNEON())) { 516 return fastEmitInst_r(AArch64::CMLTv8i8rz, &AArch64::FPR64RegClass, Op0); 517 } 518 return 0; 519} 520 521unsigned fastEmit_AArch64ISD_CMLTz_MVT_v16i8_r(MVT RetVT, unsigned Op0) { 522 if (RetVT.SimpleTy != MVT::v16i8) 523 return 0; 524 if ((Subtarget->hasNEON())) { 525 return fastEmitInst_r(AArch64::CMLTv16i8rz, &AArch64::FPR128RegClass, Op0); 526 } 527 return 0; 528} 529 530unsigned fastEmit_AArch64ISD_CMLTz_MVT_v4i16_r(MVT RetVT, unsigned Op0) { 531 if (RetVT.SimpleTy != MVT::v4i16) 532 return 0; 533 if ((Subtarget->hasNEON())) { 534 return fastEmitInst_r(AArch64::CMLTv4i16rz, &AArch64::FPR64RegClass, Op0); 535 } 536 return 0; 537} 538 539unsigned fastEmit_AArch64ISD_CMLTz_MVT_v8i16_r(MVT RetVT, unsigned Op0) { 540 if (RetVT.SimpleTy != MVT::v8i16) 541 return 0; 542 if ((Subtarget->hasNEON())) { 543 return fastEmitInst_r(AArch64::CMLTv8i16rz, &AArch64::FPR128RegClass, Op0); 544 } 545 return 0; 546} 547 548unsigned fastEmit_AArch64ISD_CMLTz_MVT_v2i32_r(MVT RetVT, unsigned Op0) { 549 if (RetVT.SimpleTy != MVT::v2i32) 550 return 0; 551 if ((Subtarget->hasNEON())) { 552 return fastEmitInst_r(AArch64::CMLTv2i32rz, &AArch64::FPR64RegClass, Op0); 553 } 554 return 0; 555} 556 557unsigned fastEmit_AArch64ISD_CMLTz_MVT_v4i32_r(MVT RetVT, unsigned Op0) { 558 if (RetVT.SimpleTy != MVT::v4i32) 559 return 0; 560 if ((Subtarget->hasNEON())) { 561 return fastEmitInst_r(AArch64::CMLTv4i32rz, &AArch64::FPR128RegClass, Op0); 562 } 563 return 0; 564} 565 566unsigned fastEmit_AArch64ISD_CMLTz_MVT_v1i64_r(MVT RetVT, unsigned Op0) { 567 if (RetVT.SimpleTy != MVT::v1i64) 568 return 0; 569 if ((Subtarget->hasNEON())) { 570 return fastEmitInst_r(AArch64::CMLTv1i64rz, &AArch64::FPR64RegClass, Op0); 571 } 572 return 0; 573} 574 575unsigned fastEmit_AArch64ISD_CMLTz_MVT_v2i64_r(MVT RetVT, unsigned Op0) { 576 if (RetVT.SimpleTy != MVT::v2i64) 577 return 0; 578 if ((Subtarget->hasNEON())) { 579 return fastEmitInst_r(AArch64::CMLTv2i64rz, &AArch64::FPR128RegClass, Op0); 580 } 581 return 0; 582} 583 584unsigned fastEmit_AArch64ISD_CMLTz_r(MVT VT, MVT RetVT, unsigned Op0) { 585 switch (VT.SimpleTy) { 586 case MVT::v8i8: return fastEmit_AArch64ISD_CMLTz_MVT_v8i8_r(RetVT, Op0); 587 case MVT::v16i8: return fastEmit_AArch64ISD_CMLTz_MVT_v16i8_r(RetVT, Op0); 588 case MVT::v4i16: return fastEmit_AArch64ISD_CMLTz_MVT_v4i16_r(RetVT, Op0); 589 case MVT::v8i16: return fastEmit_AArch64ISD_CMLTz_MVT_v8i16_r(RetVT, Op0); 590 case MVT::v2i32: return fastEmit_AArch64ISD_CMLTz_MVT_v2i32_r(RetVT, Op0); 591 case MVT::v4i32: return fastEmit_AArch64ISD_CMLTz_MVT_v4i32_r(RetVT, Op0); 592 case MVT::v1i64: return fastEmit_AArch64ISD_CMLTz_MVT_v1i64_r(RetVT, Op0); 593 case MVT::v2i64: return fastEmit_AArch64ISD_CMLTz_MVT_v2i64_r(RetVT, Op0); 594 default: return 0; 595 } 596} 597 598// FastEmit functions for AArch64ISD::DUP. 599 600unsigned fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i8_r(unsigned Op0) { 601 if ((Subtarget->hasNEON())) { 602 return fastEmitInst_r(AArch64::DUPv8i8gpr, &AArch64::FPR64RegClass, Op0); 603 } 604 return 0; 605} 606 607unsigned fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v16i8_r(unsigned Op0) { 608 if ((Subtarget->hasNEON())) { 609 return fastEmitInst_r(AArch64::DUPv16i8gpr, &AArch64::FPR128RegClass, Op0); 610 } 611 return 0; 612} 613 614unsigned fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i16_r(unsigned Op0) { 615 if ((Subtarget->hasNEON())) { 616 return fastEmitInst_r(AArch64::DUPv4i16gpr, &AArch64::FPR64RegClass, Op0); 617 } 618 return 0; 619} 620 621unsigned fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i16_r(unsigned Op0) { 622 if ((Subtarget->hasNEON())) { 623 return fastEmitInst_r(AArch64::DUPv8i16gpr, &AArch64::FPR128RegClass, Op0); 624 } 625 return 0; 626} 627 628unsigned fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v2i32_r(unsigned Op0) { 629 if ((Subtarget->hasNEON())) { 630 return fastEmitInst_r(AArch64::DUPv2i32gpr, &AArch64::FPR64RegClass, Op0); 631 } 632 return 0; 633} 634 635unsigned fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i32_r(unsigned Op0) { 636 if ((Subtarget->hasNEON())) { 637 return fastEmitInst_r(AArch64::DUPv4i32gpr, &AArch64::FPR128RegClass, Op0); 638 } 639 return 0; 640} 641 642unsigned fastEmit_AArch64ISD_DUP_MVT_i32_r(MVT RetVT, unsigned Op0) { 643switch (RetVT.SimpleTy) { 644 case MVT::v8i8: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i8_r(Op0); 645 case MVT::v16i8: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v16i8_r(Op0); 646 case MVT::v4i16: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i16_r(Op0); 647 case MVT::v8i16: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i16_r(Op0); 648 case MVT::v2i32: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v2i32_r(Op0); 649 case MVT::v4i32: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i32_r(Op0); 650 default: return 0; 651} 652} 653 654unsigned fastEmit_AArch64ISD_DUP_MVT_i64_r(MVT RetVT, unsigned Op0) { 655 if (RetVT.SimpleTy != MVT::v2i64) 656 return 0; 657 if ((Subtarget->hasNEON())) { 658 return fastEmitInst_r(AArch64::DUPv2i64gpr, &AArch64::FPR128RegClass, Op0); 659 } 660 return 0; 661} 662 663unsigned fastEmit_AArch64ISD_DUP_r(MVT VT, MVT RetVT, unsigned Op0) { 664 switch (VT.SimpleTy) { 665 case MVT::i32: return fastEmit_AArch64ISD_DUP_MVT_i32_r(RetVT, Op0); 666 case MVT::i64: return fastEmit_AArch64ISD_DUP_MVT_i64_r(RetVT, Op0); 667 default: return 0; 668 } 669} 670 671// FastEmit functions for AArch64ISD::FCMEQz. 672 673unsigned fastEmit_AArch64ISD_FCMEQz_MVT_v4f16_r(MVT RetVT, unsigned Op0) { 674 if (RetVT.SimpleTy != MVT::v4i16) 675 return 0; 676 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 677 return fastEmitInst_r(AArch64::FCMEQv4i16rz, &AArch64::FPR64RegClass, Op0); 678 } 679 return 0; 680} 681 682unsigned fastEmit_AArch64ISD_FCMEQz_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 683 if (RetVT.SimpleTy != MVT::v8i16) 684 return 0; 685 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 686 return fastEmitInst_r(AArch64::FCMEQv8i16rz, &AArch64::FPR128RegClass, Op0); 687 } 688 return 0; 689} 690 691unsigned fastEmit_AArch64ISD_FCMEQz_MVT_v2f32_r(MVT RetVT, unsigned Op0) { 692 if (RetVT.SimpleTy != MVT::v2i32) 693 return 0; 694 if ((Subtarget->hasNEON())) { 695 return fastEmitInst_r(AArch64::FCMEQv2i32rz, &AArch64::FPR64RegClass, Op0); 696 } 697 return 0; 698} 699 700unsigned fastEmit_AArch64ISD_FCMEQz_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 701 if (RetVT.SimpleTy != MVT::v4i32) 702 return 0; 703 if ((Subtarget->hasNEON())) { 704 return fastEmitInst_r(AArch64::FCMEQv4i32rz, &AArch64::FPR128RegClass, Op0); 705 } 706 return 0; 707} 708 709unsigned fastEmit_AArch64ISD_FCMEQz_MVT_v1f64_r(MVT RetVT, unsigned Op0) { 710 if (RetVT.SimpleTy != MVT::v1i64) 711 return 0; 712 if ((Subtarget->hasNEON())) { 713 return fastEmitInst_r(AArch64::FCMEQv1i64rz, &AArch64::FPR64RegClass, Op0); 714 } 715 return 0; 716} 717 718unsigned fastEmit_AArch64ISD_FCMEQz_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 719 if (RetVT.SimpleTy != MVT::v2i64) 720 return 0; 721 if ((Subtarget->hasNEON())) { 722 return fastEmitInst_r(AArch64::FCMEQv2i64rz, &AArch64::FPR128RegClass, Op0); 723 } 724 return 0; 725} 726 727unsigned fastEmit_AArch64ISD_FCMEQz_r(MVT VT, MVT RetVT, unsigned Op0) { 728 switch (VT.SimpleTy) { 729 case MVT::v4f16: return fastEmit_AArch64ISD_FCMEQz_MVT_v4f16_r(RetVT, Op0); 730 case MVT::v8f16: return fastEmit_AArch64ISD_FCMEQz_MVT_v8f16_r(RetVT, Op0); 731 case MVT::v2f32: return fastEmit_AArch64ISD_FCMEQz_MVT_v2f32_r(RetVT, Op0); 732 case MVT::v4f32: return fastEmit_AArch64ISD_FCMEQz_MVT_v4f32_r(RetVT, Op0); 733 case MVT::v1f64: return fastEmit_AArch64ISD_FCMEQz_MVT_v1f64_r(RetVT, Op0); 734 case MVT::v2f64: return fastEmit_AArch64ISD_FCMEQz_MVT_v2f64_r(RetVT, Op0); 735 default: return 0; 736 } 737} 738 739// FastEmit functions for AArch64ISD::FCMGEz. 740 741unsigned fastEmit_AArch64ISD_FCMGEz_MVT_v4f16_r(MVT RetVT, unsigned Op0) { 742 if (RetVT.SimpleTy != MVT::v4i16) 743 return 0; 744 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 745 return fastEmitInst_r(AArch64::FCMGEv4i16rz, &AArch64::FPR64RegClass, Op0); 746 } 747 return 0; 748} 749 750unsigned fastEmit_AArch64ISD_FCMGEz_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 751 if (RetVT.SimpleTy != MVT::v8i16) 752 return 0; 753 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 754 return fastEmitInst_r(AArch64::FCMGEv8i16rz, &AArch64::FPR128RegClass, Op0); 755 } 756 return 0; 757} 758 759unsigned fastEmit_AArch64ISD_FCMGEz_MVT_v2f32_r(MVT RetVT, unsigned Op0) { 760 if (RetVT.SimpleTy != MVT::v2i32) 761 return 0; 762 if ((Subtarget->hasNEON())) { 763 return fastEmitInst_r(AArch64::FCMGEv2i32rz, &AArch64::FPR64RegClass, Op0); 764 } 765 return 0; 766} 767 768unsigned fastEmit_AArch64ISD_FCMGEz_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 769 if (RetVT.SimpleTy != MVT::v4i32) 770 return 0; 771 if ((Subtarget->hasNEON())) { 772 return fastEmitInst_r(AArch64::FCMGEv4i32rz, &AArch64::FPR128RegClass, Op0); 773 } 774 return 0; 775} 776 777unsigned fastEmit_AArch64ISD_FCMGEz_MVT_v1f64_r(MVT RetVT, unsigned Op0) { 778 if (RetVT.SimpleTy != MVT::v1i64) 779 return 0; 780 if ((Subtarget->hasNEON())) { 781 return fastEmitInst_r(AArch64::FCMGEv1i64rz, &AArch64::FPR64RegClass, Op0); 782 } 783 return 0; 784} 785 786unsigned fastEmit_AArch64ISD_FCMGEz_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 787 if (RetVT.SimpleTy != MVT::v2i64) 788 return 0; 789 if ((Subtarget->hasNEON())) { 790 return fastEmitInst_r(AArch64::FCMGEv2i64rz, &AArch64::FPR128RegClass, Op0); 791 } 792 return 0; 793} 794 795unsigned fastEmit_AArch64ISD_FCMGEz_r(MVT VT, MVT RetVT, unsigned Op0) { 796 switch (VT.SimpleTy) { 797 case MVT::v4f16: return fastEmit_AArch64ISD_FCMGEz_MVT_v4f16_r(RetVT, Op0); 798 case MVT::v8f16: return fastEmit_AArch64ISD_FCMGEz_MVT_v8f16_r(RetVT, Op0); 799 case MVT::v2f32: return fastEmit_AArch64ISD_FCMGEz_MVT_v2f32_r(RetVT, Op0); 800 case MVT::v4f32: return fastEmit_AArch64ISD_FCMGEz_MVT_v4f32_r(RetVT, Op0); 801 case MVT::v1f64: return fastEmit_AArch64ISD_FCMGEz_MVT_v1f64_r(RetVT, Op0); 802 case MVT::v2f64: return fastEmit_AArch64ISD_FCMGEz_MVT_v2f64_r(RetVT, Op0); 803 default: return 0; 804 } 805} 806 807// FastEmit functions for AArch64ISD::FCMGTz. 808 809unsigned fastEmit_AArch64ISD_FCMGTz_MVT_v4f16_r(MVT RetVT, unsigned Op0) { 810 if (RetVT.SimpleTy != MVT::v4i16) 811 return 0; 812 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 813 return fastEmitInst_r(AArch64::FCMGTv4i16rz, &AArch64::FPR64RegClass, Op0); 814 } 815 return 0; 816} 817 818unsigned fastEmit_AArch64ISD_FCMGTz_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 819 if (RetVT.SimpleTy != MVT::v8i16) 820 return 0; 821 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 822 return fastEmitInst_r(AArch64::FCMGTv8i16rz, &AArch64::FPR128RegClass, Op0); 823 } 824 return 0; 825} 826 827unsigned fastEmit_AArch64ISD_FCMGTz_MVT_v2f32_r(MVT RetVT, unsigned Op0) { 828 if (RetVT.SimpleTy != MVT::v2i32) 829 return 0; 830 if ((Subtarget->hasNEON())) { 831 return fastEmitInst_r(AArch64::FCMGTv2i32rz, &AArch64::FPR64RegClass, Op0); 832 } 833 return 0; 834} 835 836unsigned fastEmit_AArch64ISD_FCMGTz_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 837 if (RetVT.SimpleTy != MVT::v4i32) 838 return 0; 839 if ((Subtarget->hasNEON())) { 840 return fastEmitInst_r(AArch64::FCMGTv4i32rz, &AArch64::FPR128RegClass, Op0); 841 } 842 return 0; 843} 844 845unsigned fastEmit_AArch64ISD_FCMGTz_MVT_v1f64_r(MVT RetVT, unsigned Op0) { 846 if (RetVT.SimpleTy != MVT::v1i64) 847 return 0; 848 if ((Subtarget->hasNEON())) { 849 return fastEmitInst_r(AArch64::FCMGTv1i64rz, &AArch64::FPR64RegClass, Op0); 850 } 851 return 0; 852} 853 854unsigned fastEmit_AArch64ISD_FCMGTz_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 855 if (RetVT.SimpleTy != MVT::v2i64) 856 return 0; 857 if ((Subtarget->hasNEON())) { 858 return fastEmitInst_r(AArch64::FCMGTv2i64rz, &AArch64::FPR128RegClass, Op0); 859 } 860 return 0; 861} 862 863unsigned fastEmit_AArch64ISD_FCMGTz_r(MVT VT, MVT RetVT, unsigned Op0) { 864 switch (VT.SimpleTy) { 865 case MVT::v4f16: return fastEmit_AArch64ISD_FCMGTz_MVT_v4f16_r(RetVT, Op0); 866 case MVT::v8f16: return fastEmit_AArch64ISD_FCMGTz_MVT_v8f16_r(RetVT, Op0); 867 case MVT::v2f32: return fastEmit_AArch64ISD_FCMGTz_MVT_v2f32_r(RetVT, Op0); 868 case MVT::v4f32: return fastEmit_AArch64ISD_FCMGTz_MVT_v4f32_r(RetVT, Op0); 869 case MVT::v1f64: return fastEmit_AArch64ISD_FCMGTz_MVT_v1f64_r(RetVT, Op0); 870 case MVT::v2f64: return fastEmit_AArch64ISD_FCMGTz_MVT_v2f64_r(RetVT, Op0); 871 default: return 0; 872 } 873} 874 875// FastEmit functions for AArch64ISD::FCMLEz. 876 877unsigned fastEmit_AArch64ISD_FCMLEz_MVT_v4f16_r(MVT RetVT, unsigned Op0) { 878 if (RetVT.SimpleTy != MVT::v4i16) 879 return 0; 880 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 881 return fastEmitInst_r(AArch64::FCMLEv4i16rz, &AArch64::FPR64RegClass, Op0); 882 } 883 return 0; 884} 885 886unsigned fastEmit_AArch64ISD_FCMLEz_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 887 if (RetVT.SimpleTy != MVT::v8i16) 888 return 0; 889 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 890 return fastEmitInst_r(AArch64::FCMLEv8i16rz, &AArch64::FPR128RegClass, Op0); 891 } 892 return 0; 893} 894 895unsigned fastEmit_AArch64ISD_FCMLEz_MVT_v2f32_r(MVT RetVT, unsigned Op0) { 896 if (RetVT.SimpleTy != MVT::v2i32) 897 return 0; 898 if ((Subtarget->hasNEON())) { 899 return fastEmitInst_r(AArch64::FCMLEv2i32rz, &AArch64::FPR64RegClass, Op0); 900 } 901 return 0; 902} 903 904unsigned fastEmit_AArch64ISD_FCMLEz_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 905 if (RetVT.SimpleTy != MVT::v4i32) 906 return 0; 907 if ((Subtarget->hasNEON())) { 908 return fastEmitInst_r(AArch64::FCMLEv4i32rz, &AArch64::FPR128RegClass, Op0); 909 } 910 return 0; 911} 912 913unsigned fastEmit_AArch64ISD_FCMLEz_MVT_v1f64_r(MVT RetVT, unsigned Op0) { 914 if (RetVT.SimpleTy != MVT::v1i64) 915 return 0; 916 if ((Subtarget->hasNEON())) { 917 return fastEmitInst_r(AArch64::FCMLEv1i64rz, &AArch64::FPR64RegClass, Op0); 918 } 919 return 0; 920} 921 922unsigned fastEmit_AArch64ISD_FCMLEz_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 923 if (RetVT.SimpleTy != MVT::v2i64) 924 return 0; 925 if ((Subtarget->hasNEON())) { 926 return fastEmitInst_r(AArch64::FCMLEv2i64rz, &AArch64::FPR128RegClass, Op0); 927 } 928 return 0; 929} 930 931unsigned fastEmit_AArch64ISD_FCMLEz_r(MVT VT, MVT RetVT, unsigned Op0) { 932 switch (VT.SimpleTy) { 933 case MVT::v4f16: return fastEmit_AArch64ISD_FCMLEz_MVT_v4f16_r(RetVT, Op0); 934 case MVT::v8f16: return fastEmit_AArch64ISD_FCMLEz_MVT_v8f16_r(RetVT, Op0); 935 case MVT::v2f32: return fastEmit_AArch64ISD_FCMLEz_MVT_v2f32_r(RetVT, Op0); 936 case MVT::v4f32: return fastEmit_AArch64ISD_FCMLEz_MVT_v4f32_r(RetVT, Op0); 937 case MVT::v1f64: return fastEmit_AArch64ISD_FCMLEz_MVT_v1f64_r(RetVT, Op0); 938 case MVT::v2f64: return fastEmit_AArch64ISD_FCMLEz_MVT_v2f64_r(RetVT, Op0); 939 default: return 0; 940 } 941} 942 943// FastEmit functions for AArch64ISD::FCMLTz. 944 945unsigned fastEmit_AArch64ISD_FCMLTz_MVT_v4f16_r(MVT RetVT, unsigned Op0) { 946 if (RetVT.SimpleTy != MVT::v4i16) 947 return 0; 948 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 949 return fastEmitInst_r(AArch64::FCMLTv4i16rz, &AArch64::FPR64RegClass, Op0); 950 } 951 return 0; 952} 953 954unsigned fastEmit_AArch64ISD_FCMLTz_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 955 if (RetVT.SimpleTy != MVT::v8i16) 956 return 0; 957 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 958 return fastEmitInst_r(AArch64::FCMLTv8i16rz, &AArch64::FPR128RegClass, Op0); 959 } 960 return 0; 961} 962 963unsigned fastEmit_AArch64ISD_FCMLTz_MVT_v2f32_r(MVT RetVT, unsigned Op0) { 964 if (RetVT.SimpleTy != MVT::v2i32) 965 return 0; 966 if ((Subtarget->hasNEON())) { 967 return fastEmitInst_r(AArch64::FCMLTv2i32rz, &AArch64::FPR64RegClass, Op0); 968 } 969 return 0; 970} 971 972unsigned fastEmit_AArch64ISD_FCMLTz_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 973 if (RetVT.SimpleTy != MVT::v4i32) 974 return 0; 975 if ((Subtarget->hasNEON())) { 976 return fastEmitInst_r(AArch64::FCMLTv4i32rz, &AArch64::FPR128RegClass, Op0); 977 } 978 return 0; 979} 980 981unsigned fastEmit_AArch64ISD_FCMLTz_MVT_v1f64_r(MVT RetVT, unsigned Op0) { 982 if (RetVT.SimpleTy != MVT::v1i64) 983 return 0; 984 if ((Subtarget->hasNEON())) { 985 return fastEmitInst_r(AArch64::FCMLTv1i64rz, &AArch64::FPR64RegClass, Op0); 986 } 987 return 0; 988} 989 990unsigned fastEmit_AArch64ISD_FCMLTz_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 991 if (RetVT.SimpleTy != MVT::v2i64) 992 return 0; 993 if ((Subtarget->hasNEON())) { 994 return fastEmitInst_r(AArch64::FCMLTv2i64rz, &AArch64::FPR128RegClass, Op0); 995 } 996 return 0; 997} 998 999unsigned fastEmit_AArch64ISD_FCMLTz_r(MVT VT, MVT RetVT, unsigned Op0) { 1000 switch (VT.SimpleTy) { 1001 case MVT::v4f16: return fastEmit_AArch64ISD_FCMLTz_MVT_v4f16_r(RetVT, Op0); 1002 case MVT::v8f16: return fastEmit_AArch64ISD_FCMLTz_MVT_v8f16_r(RetVT, Op0); 1003 case MVT::v2f32: return fastEmit_AArch64ISD_FCMLTz_MVT_v2f32_r(RetVT, Op0); 1004 case MVT::v4f32: return fastEmit_AArch64ISD_FCMLTz_MVT_v4f32_r(RetVT, Op0); 1005 case MVT::v1f64: return fastEmit_AArch64ISD_FCMLTz_MVT_v1f64_r(RetVT, Op0); 1006 case MVT::v2f64: return fastEmit_AArch64ISD_FCMLTz_MVT_v2f64_r(RetVT, Op0); 1007 default: return 0; 1008 } 1009} 1010 1011// FastEmit functions for AArch64ISD::FRECPE. 1012 1013unsigned fastEmit_AArch64ISD_FRECPE_MVT_f32_r(MVT RetVT, unsigned Op0) { 1014 if (RetVT.SimpleTy != MVT::f32) 1015 return 0; 1016 return fastEmitInst_r(AArch64::FRECPEv1i32, &AArch64::FPR32RegClass, Op0); 1017} 1018 1019unsigned fastEmit_AArch64ISD_FRECPE_MVT_f64_r(MVT RetVT, unsigned Op0) { 1020 if (RetVT.SimpleTy != MVT::f64) 1021 return 0; 1022 return fastEmitInst_r(AArch64::FRECPEv1i64, &AArch64::FPR64RegClass, Op0); 1023} 1024 1025unsigned fastEmit_AArch64ISD_FRECPE_MVT_v2f32_r(MVT RetVT, unsigned Op0) { 1026 if (RetVT.SimpleTy != MVT::v2f32) 1027 return 0; 1028 return fastEmitInst_r(AArch64::FRECPEv2f32, &AArch64::FPR64RegClass, Op0); 1029} 1030 1031unsigned fastEmit_AArch64ISD_FRECPE_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 1032 if (RetVT.SimpleTy != MVT::v4f32) 1033 return 0; 1034 return fastEmitInst_r(AArch64::FRECPEv4f32, &AArch64::FPR128RegClass, Op0); 1035} 1036 1037unsigned fastEmit_AArch64ISD_FRECPE_MVT_v1f64_r(MVT RetVT, unsigned Op0) { 1038 if (RetVT.SimpleTy != MVT::v1f64) 1039 return 0; 1040 return fastEmitInst_r(AArch64::FRECPEv1i64, &AArch64::FPR64RegClass, Op0); 1041} 1042 1043unsigned fastEmit_AArch64ISD_FRECPE_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 1044 if (RetVT.SimpleTy != MVT::v2f64) 1045 return 0; 1046 return fastEmitInst_r(AArch64::FRECPEv2f64, &AArch64::FPR128RegClass, Op0); 1047} 1048 1049unsigned fastEmit_AArch64ISD_FRECPE_MVT_nxv8f16_r(MVT RetVT, unsigned Op0) { 1050 if (RetVT.SimpleTy != MVT::nxv8f16) 1051 return 0; 1052 if ((Subtarget->hasSVEorSME())) { 1053 return fastEmitInst_r(AArch64::FRECPE_ZZ_H, &AArch64::ZPRRegClass, Op0); 1054 } 1055 return 0; 1056} 1057 1058unsigned fastEmit_AArch64ISD_FRECPE_MVT_nxv4f32_r(MVT RetVT, unsigned Op0) { 1059 if (RetVT.SimpleTy != MVT::nxv4f32) 1060 return 0; 1061 if ((Subtarget->hasSVEorSME())) { 1062 return fastEmitInst_r(AArch64::FRECPE_ZZ_S, &AArch64::ZPRRegClass, Op0); 1063 } 1064 return 0; 1065} 1066 1067unsigned fastEmit_AArch64ISD_FRECPE_MVT_nxv2f64_r(MVT RetVT, unsigned Op0) { 1068 if (RetVT.SimpleTy != MVT::nxv2f64) 1069 return 0; 1070 if ((Subtarget->hasSVEorSME())) { 1071 return fastEmitInst_r(AArch64::FRECPE_ZZ_D, &AArch64::ZPRRegClass, Op0); 1072 } 1073 return 0; 1074} 1075 1076unsigned fastEmit_AArch64ISD_FRECPE_r(MVT VT, MVT RetVT, unsigned Op0) { 1077 switch (VT.SimpleTy) { 1078 case MVT::f32: return fastEmit_AArch64ISD_FRECPE_MVT_f32_r(RetVT, Op0); 1079 case MVT::f64: return fastEmit_AArch64ISD_FRECPE_MVT_f64_r(RetVT, Op0); 1080 case MVT::v2f32: return fastEmit_AArch64ISD_FRECPE_MVT_v2f32_r(RetVT, Op0); 1081 case MVT::v4f32: return fastEmit_AArch64ISD_FRECPE_MVT_v4f32_r(RetVT, Op0); 1082 case MVT::v1f64: return fastEmit_AArch64ISD_FRECPE_MVT_v1f64_r(RetVT, Op0); 1083 case MVT::v2f64: return fastEmit_AArch64ISD_FRECPE_MVT_v2f64_r(RetVT, Op0); 1084 case MVT::nxv8f16: return fastEmit_AArch64ISD_FRECPE_MVT_nxv8f16_r(RetVT, Op0); 1085 case MVT::nxv4f32: return fastEmit_AArch64ISD_FRECPE_MVT_nxv4f32_r(RetVT, Op0); 1086 case MVT::nxv2f64: return fastEmit_AArch64ISD_FRECPE_MVT_nxv2f64_r(RetVT, Op0); 1087 default: return 0; 1088 } 1089} 1090 1091// FastEmit functions for AArch64ISD::FRSQRTE. 1092 1093unsigned fastEmit_AArch64ISD_FRSQRTE_MVT_f32_r(MVT RetVT, unsigned Op0) { 1094 if (RetVT.SimpleTy != MVT::f32) 1095 return 0; 1096 return fastEmitInst_r(AArch64::FRSQRTEv1i32, &AArch64::FPR32RegClass, Op0); 1097} 1098 1099unsigned fastEmit_AArch64ISD_FRSQRTE_MVT_f64_r(MVT RetVT, unsigned Op0) { 1100 if (RetVT.SimpleTy != MVT::f64) 1101 return 0; 1102 return fastEmitInst_r(AArch64::FRSQRTEv1i64, &AArch64::FPR64RegClass, Op0); 1103} 1104 1105unsigned fastEmit_AArch64ISD_FRSQRTE_MVT_v2f32_r(MVT RetVT, unsigned Op0) { 1106 if (RetVT.SimpleTy != MVT::v2f32) 1107 return 0; 1108 return fastEmitInst_r(AArch64::FRSQRTEv2f32, &AArch64::FPR64RegClass, Op0); 1109} 1110 1111unsigned fastEmit_AArch64ISD_FRSQRTE_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 1112 if (RetVT.SimpleTy != MVT::v4f32) 1113 return 0; 1114 return fastEmitInst_r(AArch64::FRSQRTEv4f32, &AArch64::FPR128RegClass, Op0); 1115} 1116 1117unsigned fastEmit_AArch64ISD_FRSQRTE_MVT_v1f64_r(MVT RetVT, unsigned Op0) { 1118 if (RetVT.SimpleTy != MVT::v1f64) 1119 return 0; 1120 return fastEmitInst_r(AArch64::FRSQRTEv1i64, &AArch64::FPR64RegClass, Op0); 1121} 1122 1123unsigned fastEmit_AArch64ISD_FRSQRTE_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 1124 if (RetVT.SimpleTy != MVT::v2f64) 1125 return 0; 1126 return fastEmitInst_r(AArch64::FRSQRTEv2f64, &AArch64::FPR128RegClass, Op0); 1127} 1128 1129unsigned fastEmit_AArch64ISD_FRSQRTE_MVT_nxv8f16_r(MVT RetVT, unsigned Op0) { 1130 if (RetVT.SimpleTy != MVT::nxv8f16) 1131 return 0; 1132 if ((Subtarget->hasSVEorSME())) { 1133 return fastEmitInst_r(AArch64::FRSQRTE_ZZ_H, &AArch64::ZPRRegClass, Op0); 1134 } 1135 return 0; 1136} 1137 1138unsigned fastEmit_AArch64ISD_FRSQRTE_MVT_nxv4f32_r(MVT RetVT, unsigned Op0) { 1139 if (RetVT.SimpleTy != MVT::nxv4f32) 1140 return 0; 1141 if ((Subtarget->hasSVEorSME())) { 1142 return fastEmitInst_r(AArch64::FRSQRTE_ZZ_S, &AArch64::ZPRRegClass, Op0); 1143 } 1144 return 0; 1145} 1146 1147unsigned fastEmit_AArch64ISD_FRSQRTE_MVT_nxv2f64_r(MVT RetVT, unsigned Op0) { 1148 if (RetVT.SimpleTy != MVT::nxv2f64) 1149 return 0; 1150 if ((Subtarget->hasSVEorSME())) { 1151 return fastEmitInst_r(AArch64::FRSQRTE_ZZ_D, &AArch64::ZPRRegClass, Op0); 1152 } 1153 return 0; 1154} 1155 1156unsigned fastEmit_AArch64ISD_FRSQRTE_r(MVT VT, MVT RetVT, unsigned Op0) { 1157 switch (VT.SimpleTy) { 1158 case MVT::f32: return fastEmit_AArch64ISD_FRSQRTE_MVT_f32_r(RetVT, Op0); 1159 case MVT::f64: return fastEmit_AArch64ISD_FRSQRTE_MVT_f64_r(RetVT, Op0); 1160 case MVT::v2f32: return fastEmit_AArch64ISD_FRSQRTE_MVT_v2f32_r(RetVT, Op0); 1161 case MVT::v4f32: return fastEmit_AArch64ISD_FRSQRTE_MVT_v4f32_r(RetVT, Op0); 1162 case MVT::v1f64: return fastEmit_AArch64ISD_FRSQRTE_MVT_v1f64_r(RetVT, Op0); 1163 case MVT::v2f64: return fastEmit_AArch64ISD_FRSQRTE_MVT_v2f64_r(RetVT, Op0); 1164 case MVT::nxv8f16: return fastEmit_AArch64ISD_FRSQRTE_MVT_nxv8f16_r(RetVT, Op0); 1165 case MVT::nxv4f32: return fastEmit_AArch64ISD_FRSQRTE_MVT_nxv4f32_r(RetVT, Op0); 1166 case MVT::nxv2f64: return fastEmit_AArch64ISD_FRSQRTE_MVT_nxv2f64_r(RetVT, Op0); 1167 default: return 0; 1168 } 1169} 1170 1171// FastEmit functions for AArch64ISD::OBSCURE_COPY. 1172 1173unsigned fastEmit_AArch64ISD_OBSCURE_COPY_MVT_i64_r(MVT RetVT, unsigned Op0) { 1174 if (RetVT.SimpleTy != MVT::i64) 1175 return 0; 1176 if ((Subtarget->hasSME())) { 1177 return fastEmitInst_r(AArch64::OBSCURE_COPY, &AArch64::GPR64RegClass, Op0); 1178 } 1179 return 0; 1180} 1181 1182unsigned fastEmit_AArch64ISD_OBSCURE_COPY_r(MVT VT, MVT RetVT, unsigned Op0) { 1183 switch (VT.SimpleTy) { 1184 case MVT::i64: return fastEmit_AArch64ISD_OBSCURE_COPY_MVT_i64_r(RetVT, Op0); 1185 default: return 0; 1186 } 1187} 1188 1189// FastEmit functions for AArch64ISD::REV16. 1190 1191unsigned fastEmit_AArch64ISD_REV16_MVT_v8i8_r(MVT RetVT, unsigned Op0) { 1192 if (RetVT.SimpleTy != MVT::v8i8) 1193 return 0; 1194 if ((Subtarget->hasNEON())) { 1195 return fastEmitInst_r(AArch64::REV16v8i8, &AArch64::FPR64RegClass, Op0); 1196 } 1197 return 0; 1198} 1199 1200unsigned fastEmit_AArch64ISD_REV16_MVT_v16i8_r(MVT RetVT, unsigned Op0) { 1201 if (RetVT.SimpleTy != MVT::v16i8) 1202 return 0; 1203 if ((Subtarget->hasNEON())) { 1204 return fastEmitInst_r(AArch64::REV16v16i8, &AArch64::FPR128RegClass, Op0); 1205 } 1206 return 0; 1207} 1208 1209unsigned fastEmit_AArch64ISD_REV16_r(MVT VT, MVT RetVT, unsigned Op0) { 1210 switch (VT.SimpleTy) { 1211 case MVT::v8i8: return fastEmit_AArch64ISD_REV16_MVT_v8i8_r(RetVT, Op0); 1212 case MVT::v16i8: return fastEmit_AArch64ISD_REV16_MVT_v16i8_r(RetVT, Op0); 1213 default: return 0; 1214 } 1215} 1216 1217// FastEmit functions for AArch64ISD::REV32. 1218 1219unsigned fastEmit_AArch64ISD_REV32_MVT_v8i8_r(MVT RetVT, unsigned Op0) { 1220 if (RetVT.SimpleTy != MVT::v8i8) 1221 return 0; 1222 if ((Subtarget->hasNEON())) { 1223 return fastEmitInst_r(AArch64::REV32v8i8, &AArch64::FPR64RegClass, Op0); 1224 } 1225 return 0; 1226} 1227 1228unsigned fastEmit_AArch64ISD_REV32_MVT_v16i8_r(MVT RetVT, unsigned Op0) { 1229 if (RetVT.SimpleTy != MVT::v16i8) 1230 return 0; 1231 if ((Subtarget->hasNEON())) { 1232 return fastEmitInst_r(AArch64::REV32v16i8, &AArch64::FPR128RegClass, Op0); 1233 } 1234 return 0; 1235} 1236 1237unsigned fastEmit_AArch64ISD_REV32_MVT_v4i16_r(MVT RetVT, unsigned Op0) { 1238 if (RetVT.SimpleTy != MVT::v4i16) 1239 return 0; 1240 if ((Subtarget->hasNEON())) { 1241 return fastEmitInst_r(AArch64::REV32v4i16, &AArch64::FPR64RegClass, Op0); 1242 } 1243 return 0; 1244} 1245 1246unsigned fastEmit_AArch64ISD_REV32_MVT_v8i16_r(MVT RetVT, unsigned Op0) { 1247 if (RetVT.SimpleTy != MVT::v8i16) 1248 return 0; 1249 if ((Subtarget->hasNEON())) { 1250 return fastEmitInst_r(AArch64::REV32v8i16, &AArch64::FPR128RegClass, Op0); 1251 } 1252 return 0; 1253} 1254 1255unsigned fastEmit_AArch64ISD_REV32_MVT_v4f16_r(MVT RetVT, unsigned Op0) { 1256 if (RetVT.SimpleTy != MVT::v4f16) 1257 return 0; 1258 return fastEmitInst_r(AArch64::REV32v4i16, &AArch64::FPR64RegClass, Op0); 1259} 1260 1261unsigned fastEmit_AArch64ISD_REV32_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 1262 if (RetVT.SimpleTy != MVT::v8f16) 1263 return 0; 1264 return fastEmitInst_r(AArch64::REV32v8i16, &AArch64::FPR128RegClass, Op0); 1265} 1266 1267unsigned fastEmit_AArch64ISD_REV32_MVT_v4bf16_r(MVT RetVT, unsigned Op0) { 1268 if (RetVT.SimpleTy != MVT::v4bf16) 1269 return 0; 1270 return fastEmitInst_r(AArch64::REV32v4i16, &AArch64::FPR64RegClass, Op0); 1271} 1272 1273unsigned fastEmit_AArch64ISD_REV32_MVT_v8bf16_r(MVT RetVT, unsigned Op0) { 1274 if (RetVT.SimpleTy != MVT::v8bf16) 1275 return 0; 1276 return fastEmitInst_r(AArch64::REV32v8i16, &AArch64::FPR128RegClass, Op0); 1277} 1278 1279unsigned fastEmit_AArch64ISD_REV32_r(MVT VT, MVT RetVT, unsigned Op0) { 1280 switch (VT.SimpleTy) { 1281 case MVT::v8i8: return fastEmit_AArch64ISD_REV32_MVT_v8i8_r(RetVT, Op0); 1282 case MVT::v16i8: return fastEmit_AArch64ISD_REV32_MVT_v16i8_r(RetVT, Op0); 1283 case MVT::v4i16: return fastEmit_AArch64ISD_REV32_MVT_v4i16_r(RetVT, Op0); 1284 case MVT::v8i16: return fastEmit_AArch64ISD_REV32_MVT_v8i16_r(RetVT, Op0); 1285 case MVT::v4f16: return fastEmit_AArch64ISD_REV32_MVT_v4f16_r(RetVT, Op0); 1286 case MVT::v8f16: return fastEmit_AArch64ISD_REV32_MVT_v8f16_r(RetVT, Op0); 1287 case MVT::v4bf16: return fastEmit_AArch64ISD_REV32_MVT_v4bf16_r(RetVT, Op0); 1288 case MVT::v8bf16: return fastEmit_AArch64ISD_REV32_MVT_v8bf16_r(RetVT, Op0); 1289 default: return 0; 1290 } 1291} 1292 1293// FastEmit functions for AArch64ISD::REV64. 1294 1295unsigned fastEmit_AArch64ISD_REV64_MVT_v8i8_r(MVT RetVT, unsigned Op0) { 1296 if (RetVT.SimpleTy != MVT::v8i8) 1297 return 0; 1298 if ((Subtarget->hasNEON())) { 1299 return fastEmitInst_r(AArch64::REV64v8i8, &AArch64::FPR64RegClass, Op0); 1300 } 1301 return 0; 1302} 1303 1304unsigned fastEmit_AArch64ISD_REV64_MVT_v16i8_r(MVT RetVT, unsigned Op0) { 1305 if (RetVT.SimpleTy != MVT::v16i8) 1306 return 0; 1307 if ((Subtarget->hasNEON())) { 1308 return fastEmitInst_r(AArch64::REV64v16i8, &AArch64::FPR128RegClass, Op0); 1309 } 1310 return 0; 1311} 1312 1313unsigned fastEmit_AArch64ISD_REV64_MVT_v4i16_r(MVT RetVT, unsigned Op0) { 1314 if (RetVT.SimpleTy != MVT::v4i16) 1315 return 0; 1316 if ((Subtarget->hasNEON())) { 1317 return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0); 1318 } 1319 return 0; 1320} 1321 1322unsigned fastEmit_AArch64ISD_REV64_MVT_v8i16_r(MVT RetVT, unsigned Op0) { 1323 if (RetVT.SimpleTy != MVT::v8i16) 1324 return 0; 1325 if ((Subtarget->hasNEON())) { 1326 return fastEmitInst_r(AArch64::REV64v8i16, &AArch64::FPR128RegClass, Op0); 1327 } 1328 return 0; 1329} 1330 1331unsigned fastEmit_AArch64ISD_REV64_MVT_v2i32_r(MVT RetVT, unsigned Op0) { 1332 if (RetVT.SimpleTy != MVT::v2i32) 1333 return 0; 1334 if ((Subtarget->hasNEON())) { 1335 return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0); 1336 } 1337 return 0; 1338} 1339 1340unsigned fastEmit_AArch64ISD_REV64_MVT_v4i32_r(MVT RetVT, unsigned Op0) { 1341 if (RetVT.SimpleTy != MVT::v4i32) 1342 return 0; 1343 if ((Subtarget->hasNEON())) { 1344 return fastEmitInst_r(AArch64::REV64v4i32, &AArch64::FPR128RegClass, Op0); 1345 } 1346 return 0; 1347} 1348 1349unsigned fastEmit_AArch64ISD_REV64_MVT_v4f16_r(MVT RetVT, unsigned Op0) { 1350 if (RetVT.SimpleTy != MVT::v4f16) 1351 return 0; 1352 return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0); 1353} 1354 1355unsigned fastEmit_AArch64ISD_REV64_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 1356 if (RetVT.SimpleTy != MVT::v8f16) 1357 return 0; 1358 return fastEmitInst_r(AArch64::REV64v8i16, &AArch64::FPR128RegClass, Op0); 1359} 1360 1361unsigned fastEmit_AArch64ISD_REV64_MVT_v4bf16_r(MVT RetVT, unsigned Op0) { 1362 if (RetVT.SimpleTy != MVT::v4bf16) 1363 return 0; 1364 return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0); 1365} 1366 1367unsigned fastEmit_AArch64ISD_REV64_MVT_v8bf16_r(MVT RetVT, unsigned Op0) { 1368 if (RetVT.SimpleTy != MVT::v8bf16) 1369 return 0; 1370 return fastEmitInst_r(AArch64::REV64v8i16, &AArch64::FPR128RegClass, Op0); 1371} 1372 1373unsigned fastEmit_AArch64ISD_REV64_MVT_v2f32_r(MVT RetVT, unsigned Op0) { 1374 if (RetVT.SimpleTy != MVT::v2f32) 1375 return 0; 1376 return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0); 1377} 1378 1379unsigned fastEmit_AArch64ISD_REV64_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 1380 if (RetVT.SimpleTy != MVT::v4f32) 1381 return 0; 1382 return fastEmitInst_r(AArch64::REV64v4i32, &AArch64::FPR128RegClass, Op0); 1383} 1384 1385unsigned fastEmit_AArch64ISD_REV64_r(MVT VT, MVT RetVT, unsigned Op0) { 1386 switch (VT.SimpleTy) { 1387 case MVT::v8i8: return fastEmit_AArch64ISD_REV64_MVT_v8i8_r(RetVT, Op0); 1388 case MVT::v16i8: return fastEmit_AArch64ISD_REV64_MVT_v16i8_r(RetVT, Op0); 1389 case MVT::v4i16: return fastEmit_AArch64ISD_REV64_MVT_v4i16_r(RetVT, Op0); 1390 case MVT::v8i16: return fastEmit_AArch64ISD_REV64_MVT_v8i16_r(RetVT, Op0); 1391 case MVT::v2i32: return fastEmit_AArch64ISD_REV64_MVT_v2i32_r(RetVT, Op0); 1392 case MVT::v4i32: return fastEmit_AArch64ISD_REV64_MVT_v4i32_r(RetVT, Op0); 1393 case MVT::v4f16: return fastEmit_AArch64ISD_REV64_MVT_v4f16_r(RetVT, Op0); 1394 case MVT::v8f16: return fastEmit_AArch64ISD_REV64_MVT_v8f16_r(RetVT, Op0); 1395 case MVT::v4bf16: return fastEmit_AArch64ISD_REV64_MVT_v4bf16_r(RetVT, Op0); 1396 case MVT::v8bf16: return fastEmit_AArch64ISD_REV64_MVT_v8bf16_r(RetVT, Op0); 1397 case MVT::v2f32: return fastEmit_AArch64ISD_REV64_MVT_v2f32_r(RetVT, Op0); 1398 case MVT::v4f32: return fastEmit_AArch64ISD_REV64_MVT_v4f32_r(RetVT, Op0); 1399 default: return 0; 1400 } 1401} 1402 1403// FastEmit functions for AArch64ISD::SADDLP. 1404 1405unsigned fastEmit_AArch64ISD_SADDLP_MVT_v8i8_r(MVT RetVT, unsigned Op0) { 1406 if (RetVT.SimpleTy != MVT::v4i16) 1407 return 0; 1408 if ((Subtarget->hasNEON())) { 1409 return fastEmitInst_r(AArch64::SADDLPv8i8_v4i16, &AArch64::FPR64RegClass, Op0); 1410 } 1411 return 0; 1412} 1413 1414unsigned fastEmit_AArch64ISD_SADDLP_MVT_v16i8_r(MVT RetVT, unsigned Op0) { 1415 if (RetVT.SimpleTy != MVT::v8i16) 1416 return 0; 1417 if ((Subtarget->hasNEON())) { 1418 return fastEmitInst_r(AArch64::SADDLPv16i8_v8i16, &AArch64::FPR128RegClass, Op0); 1419 } 1420 return 0; 1421} 1422 1423unsigned fastEmit_AArch64ISD_SADDLP_MVT_v4i16_r(MVT RetVT, unsigned Op0) { 1424 if (RetVT.SimpleTy != MVT::v2i32) 1425 return 0; 1426 if ((Subtarget->hasNEON())) { 1427 return fastEmitInst_r(AArch64::SADDLPv4i16_v2i32, &AArch64::FPR64RegClass, Op0); 1428 } 1429 return 0; 1430} 1431 1432unsigned fastEmit_AArch64ISD_SADDLP_MVT_v8i16_r(MVT RetVT, unsigned Op0) { 1433 if (RetVT.SimpleTy != MVT::v4i32) 1434 return 0; 1435 if ((Subtarget->hasNEON())) { 1436 return fastEmitInst_r(AArch64::SADDLPv8i16_v4i32, &AArch64::FPR128RegClass, Op0); 1437 } 1438 return 0; 1439} 1440 1441unsigned fastEmit_AArch64ISD_SADDLP_MVT_v2i32_r(MVT RetVT, unsigned Op0) { 1442 if (RetVT.SimpleTy != MVT::v1i64) 1443 return 0; 1444 if ((Subtarget->hasNEON())) { 1445 return fastEmitInst_r(AArch64::SADDLPv2i32_v1i64, &AArch64::FPR64RegClass, Op0); 1446 } 1447 return 0; 1448} 1449 1450unsigned fastEmit_AArch64ISD_SADDLP_MVT_v4i32_r(MVT RetVT, unsigned Op0) { 1451 if (RetVT.SimpleTy != MVT::v2i64) 1452 return 0; 1453 if ((Subtarget->hasNEON())) { 1454 return fastEmitInst_r(AArch64::SADDLPv4i32_v2i64, &AArch64::FPR128RegClass, Op0); 1455 } 1456 return 0; 1457} 1458 1459unsigned fastEmit_AArch64ISD_SADDLP_r(MVT VT, MVT RetVT, unsigned Op0) { 1460 switch (VT.SimpleTy) { 1461 case MVT::v8i8: return fastEmit_AArch64ISD_SADDLP_MVT_v8i8_r(RetVT, Op0); 1462 case MVT::v16i8: return fastEmit_AArch64ISD_SADDLP_MVT_v16i8_r(RetVT, Op0); 1463 case MVT::v4i16: return fastEmit_AArch64ISD_SADDLP_MVT_v4i16_r(RetVT, Op0); 1464 case MVT::v8i16: return fastEmit_AArch64ISD_SADDLP_MVT_v8i16_r(RetVT, Op0); 1465 case MVT::v2i32: return fastEmit_AArch64ISD_SADDLP_MVT_v2i32_r(RetVT, Op0); 1466 case MVT::v4i32: return fastEmit_AArch64ISD_SADDLP_MVT_v4i32_r(RetVT, Op0); 1467 default: return 0; 1468 } 1469} 1470 1471// FastEmit functions for AArch64ISD::SITOF. 1472 1473unsigned fastEmit_AArch64ISD_SITOF_MVT_f16_r(MVT RetVT, unsigned Op0) { 1474 if (RetVT.SimpleTy != MVT::f16) 1475 return 0; 1476 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 1477 return fastEmitInst_r(AArch64::SCVTFv1i16, &AArch64::FPR16RegClass, Op0); 1478 } 1479 return 0; 1480} 1481 1482unsigned fastEmit_AArch64ISD_SITOF_MVT_f32_r(MVT RetVT, unsigned Op0) { 1483 if (RetVT.SimpleTy != MVT::f32) 1484 return 0; 1485 if ((Subtarget->hasNEON())) { 1486 return fastEmitInst_r(AArch64::SCVTFv1i32, &AArch64::FPR32RegClass, Op0); 1487 } 1488 return 0; 1489} 1490 1491unsigned fastEmit_AArch64ISD_SITOF_MVT_f64_r(MVT RetVT, unsigned Op0) { 1492 if (RetVT.SimpleTy != MVT::f64) 1493 return 0; 1494 if ((Subtarget->hasNEON())) { 1495 return fastEmitInst_r(AArch64::SCVTFv1i64, &AArch64::FPR64RegClass, Op0); 1496 } 1497 return 0; 1498} 1499 1500unsigned fastEmit_AArch64ISD_SITOF_r(MVT VT, MVT RetVT, unsigned Op0) { 1501 switch (VT.SimpleTy) { 1502 case MVT::f16: return fastEmit_AArch64ISD_SITOF_MVT_f16_r(RetVT, Op0); 1503 case MVT::f32: return fastEmit_AArch64ISD_SITOF_MVT_f32_r(RetVT, Op0); 1504 case MVT::f64: return fastEmit_AArch64ISD_SITOF_MVT_f64_r(RetVT, Op0); 1505 default: return 0; 1506 } 1507} 1508 1509// FastEmit functions for AArch64ISD::SUNPKHI. 1510 1511unsigned fastEmit_AArch64ISD_SUNPKHI_MVT_nxv16i8_r(MVT RetVT, unsigned Op0) { 1512 if (RetVT.SimpleTy != MVT::nxv8i16) 1513 return 0; 1514 if ((Subtarget->hasSVEorSME())) { 1515 return fastEmitInst_r(AArch64::SUNPKHI_ZZ_H, &AArch64::ZPRRegClass, Op0); 1516 } 1517 return 0; 1518} 1519 1520unsigned fastEmit_AArch64ISD_SUNPKHI_MVT_nxv8i16_r(MVT RetVT, unsigned Op0) { 1521 if (RetVT.SimpleTy != MVT::nxv4i32) 1522 return 0; 1523 if ((Subtarget->hasSVEorSME())) { 1524 return fastEmitInst_r(AArch64::SUNPKHI_ZZ_S, &AArch64::ZPRRegClass, Op0); 1525 } 1526 return 0; 1527} 1528 1529unsigned fastEmit_AArch64ISD_SUNPKHI_MVT_nxv4i32_r(MVT RetVT, unsigned Op0) { 1530 if (RetVT.SimpleTy != MVT::nxv2i64) 1531 return 0; 1532 if ((Subtarget->hasSVEorSME())) { 1533 return fastEmitInst_r(AArch64::SUNPKHI_ZZ_D, &AArch64::ZPRRegClass, Op0); 1534 } 1535 return 0; 1536} 1537 1538unsigned fastEmit_AArch64ISD_SUNPKHI_r(MVT VT, MVT RetVT, unsigned Op0) { 1539 switch (VT.SimpleTy) { 1540 case MVT::nxv16i8: return fastEmit_AArch64ISD_SUNPKHI_MVT_nxv16i8_r(RetVT, Op0); 1541 case MVT::nxv8i16: return fastEmit_AArch64ISD_SUNPKHI_MVT_nxv8i16_r(RetVT, Op0); 1542 case MVT::nxv4i32: return fastEmit_AArch64ISD_SUNPKHI_MVT_nxv4i32_r(RetVT, Op0); 1543 default: return 0; 1544 } 1545} 1546 1547// FastEmit functions for AArch64ISD::SUNPKLO. 1548 1549unsigned fastEmit_AArch64ISD_SUNPKLO_MVT_nxv16i8_r(MVT RetVT, unsigned Op0) { 1550 if (RetVT.SimpleTy != MVT::nxv8i16) 1551 return 0; 1552 if ((Subtarget->hasSVEorSME())) { 1553 return fastEmitInst_r(AArch64::SUNPKLO_ZZ_H, &AArch64::ZPRRegClass, Op0); 1554 } 1555 return 0; 1556} 1557 1558unsigned fastEmit_AArch64ISD_SUNPKLO_MVT_nxv8i16_r(MVT RetVT, unsigned Op0) { 1559 if (RetVT.SimpleTy != MVT::nxv4i32) 1560 return 0; 1561 if ((Subtarget->hasSVEorSME())) { 1562 return fastEmitInst_r(AArch64::SUNPKLO_ZZ_S, &AArch64::ZPRRegClass, Op0); 1563 } 1564 return 0; 1565} 1566 1567unsigned fastEmit_AArch64ISD_SUNPKLO_MVT_nxv4i32_r(MVT RetVT, unsigned Op0) { 1568 if (RetVT.SimpleTy != MVT::nxv2i64) 1569 return 0; 1570 if ((Subtarget->hasSVEorSME())) { 1571 return fastEmitInst_r(AArch64::SUNPKLO_ZZ_D, &AArch64::ZPRRegClass, Op0); 1572 } 1573 return 0; 1574} 1575 1576unsigned fastEmit_AArch64ISD_SUNPKLO_r(MVT VT, MVT RetVT, unsigned Op0) { 1577 switch (VT.SimpleTy) { 1578 case MVT::nxv16i8: return fastEmit_AArch64ISD_SUNPKLO_MVT_nxv16i8_r(RetVT, Op0); 1579 case MVT::nxv8i16: return fastEmit_AArch64ISD_SUNPKLO_MVT_nxv8i16_r(RetVT, Op0); 1580 case MVT::nxv4i32: return fastEmit_AArch64ISD_SUNPKLO_MVT_nxv4i32_r(RetVT, Op0); 1581 default: return 0; 1582 } 1583} 1584 1585// FastEmit functions for AArch64ISD::UADDLP. 1586 1587unsigned fastEmit_AArch64ISD_UADDLP_MVT_v8i8_r(MVT RetVT, unsigned Op0) { 1588 if (RetVT.SimpleTy != MVT::v4i16) 1589 return 0; 1590 if ((Subtarget->hasNEON())) { 1591 return fastEmitInst_r(AArch64::UADDLPv8i8_v4i16, &AArch64::FPR64RegClass, Op0); 1592 } 1593 return 0; 1594} 1595 1596unsigned fastEmit_AArch64ISD_UADDLP_MVT_v16i8_r(MVT RetVT, unsigned Op0) { 1597 if (RetVT.SimpleTy != MVT::v8i16) 1598 return 0; 1599 if ((Subtarget->hasNEON())) { 1600 return fastEmitInst_r(AArch64::UADDLPv16i8_v8i16, &AArch64::FPR128RegClass, Op0); 1601 } 1602 return 0; 1603} 1604 1605unsigned fastEmit_AArch64ISD_UADDLP_MVT_v4i16_r(MVT RetVT, unsigned Op0) { 1606 if (RetVT.SimpleTy != MVT::v2i32) 1607 return 0; 1608 if ((Subtarget->hasNEON())) { 1609 return fastEmitInst_r(AArch64::UADDLPv4i16_v2i32, &AArch64::FPR64RegClass, Op0); 1610 } 1611 return 0; 1612} 1613 1614unsigned fastEmit_AArch64ISD_UADDLP_MVT_v8i16_r(MVT RetVT, unsigned Op0) { 1615 if (RetVT.SimpleTy != MVT::v4i32) 1616 return 0; 1617 if ((Subtarget->hasNEON())) { 1618 return fastEmitInst_r(AArch64::UADDLPv8i16_v4i32, &AArch64::FPR128RegClass, Op0); 1619 } 1620 return 0; 1621} 1622 1623unsigned fastEmit_AArch64ISD_UADDLP_MVT_v2i32_r(MVT RetVT, unsigned Op0) { 1624 if (RetVT.SimpleTy != MVT::v1i64) 1625 return 0; 1626 if ((Subtarget->hasNEON())) { 1627 return fastEmitInst_r(AArch64::UADDLPv2i32_v1i64, &AArch64::FPR64RegClass, Op0); 1628 } 1629 return 0; 1630} 1631 1632unsigned fastEmit_AArch64ISD_UADDLP_MVT_v4i32_r(MVT RetVT, unsigned Op0) { 1633 if (RetVT.SimpleTy != MVT::v2i64) 1634 return 0; 1635 if ((Subtarget->hasNEON())) { 1636 return fastEmitInst_r(AArch64::UADDLPv4i32_v2i64, &AArch64::FPR128RegClass, Op0); 1637 } 1638 return 0; 1639} 1640 1641unsigned fastEmit_AArch64ISD_UADDLP_r(MVT VT, MVT RetVT, unsigned Op0) { 1642 switch (VT.SimpleTy) { 1643 case MVT::v8i8: return fastEmit_AArch64ISD_UADDLP_MVT_v8i8_r(RetVT, Op0); 1644 case MVT::v16i8: return fastEmit_AArch64ISD_UADDLP_MVT_v16i8_r(RetVT, Op0); 1645 case MVT::v4i16: return fastEmit_AArch64ISD_UADDLP_MVT_v4i16_r(RetVT, Op0); 1646 case MVT::v8i16: return fastEmit_AArch64ISD_UADDLP_MVT_v8i16_r(RetVT, Op0); 1647 case MVT::v2i32: return fastEmit_AArch64ISD_UADDLP_MVT_v2i32_r(RetVT, Op0); 1648 case MVT::v4i32: return fastEmit_AArch64ISD_UADDLP_MVT_v4i32_r(RetVT, Op0); 1649 default: return 0; 1650 } 1651} 1652 1653// FastEmit functions for AArch64ISD::UITOF. 1654 1655unsigned fastEmit_AArch64ISD_UITOF_MVT_f16_r(MVT RetVT, unsigned Op0) { 1656 if (RetVT.SimpleTy != MVT::f16) 1657 return 0; 1658 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 1659 return fastEmitInst_r(AArch64::UCVTFv1i16, &AArch64::FPR16RegClass, Op0); 1660 } 1661 return 0; 1662} 1663 1664unsigned fastEmit_AArch64ISD_UITOF_MVT_f32_r(MVT RetVT, unsigned Op0) { 1665 if (RetVT.SimpleTy != MVT::f32) 1666 return 0; 1667 if ((Subtarget->hasNEON())) { 1668 return fastEmitInst_r(AArch64::UCVTFv1i32, &AArch64::FPR32RegClass, Op0); 1669 } 1670 return 0; 1671} 1672 1673unsigned fastEmit_AArch64ISD_UITOF_MVT_f64_r(MVT RetVT, unsigned Op0) { 1674 if (RetVT.SimpleTy != MVT::f64) 1675 return 0; 1676 if ((Subtarget->hasNEON())) { 1677 return fastEmitInst_r(AArch64::UCVTFv1i64, &AArch64::FPR64RegClass, Op0); 1678 } 1679 return 0; 1680} 1681 1682unsigned fastEmit_AArch64ISD_UITOF_r(MVT VT, MVT RetVT, unsigned Op0) { 1683 switch (VT.SimpleTy) { 1684 case MVT::f16: return fastEmit_AArch64ISD_UITOF_MVT_f16_r(RetVT, Op0); 1685 case MVT::f32: return fastEmit_AArch64ISD_UITOF_MVT_f32_r(RetVT, Op0); 1686 case MVT::f64: return fastEmit_AArch64ISD_UITOF_MVT_f64_r(RetVT, Op0); 1687 default: return 0; 1688 } 1689} 1690 1691// FastEmit functions for AArch64ISD::UUNPKHI. 1692 1693unsigned fastEmit_AArch64ISD_UUNPKHI_MVT_nxv16i8_r(MVT RetVT, unsigned Op0) { 1694 if (RetVT.SimpleTy != MVT::nxv8i16) 1695 return 0; 1696 if ((Subtarget->hasSVEorSME())) { 1697 return fastEmitInst_r(AArch64::UUNPKHI_ZZ_H, &AArch64::ZPRRegClass, Op0); 1698 } 1699 return 0; 1700} 1701 1702unsigned fastEmit_AArch64ISD_UUNPKHI_MVT_nxv8i16_r(MVT RetVT, unsigned Op0) { 1703 if (RetVT.SimpleTy != MVT::nxv4i32) 1704 return 0; 1705 if ((Subtarget->hasSVEorSME())) { 1706 return fastEmitInst_r(AArch64::UUNPKHI_ZZ_S, &AArch64::ZPRRegClass, Op0); 1707 } 1708 return 0; 1709} 1710 1711unsigned fastEmit_AArch64ISD_UUNPKHI_MVT_nxv4i32_r(MVT RetVT, unsigned Op0) { 1712 if (RetVT.SimpleTy != MVT::nxv2i64) 1713 return 0; 1714 if ((Subtarget->hasSVEorSME())) { 1715 return fastEmitInst_r(AArch64::UUNPKHI_ZZ_D, &AArch64::ZPRRegClass, Op0); 1716 } 1717 return 0; 1718} 1719 1720unsigned fastEmit_AArch64ISD_UUNPKHI_r(MVT VT, MVT RetVT, unsigned Op0) { 1721 switch (VT.SimpleTy) { 1722 case MVT::nxv16i8: return fastEmit_AArch64ISD_UUNPKHI_MVT_nxv16i8_r(RetVT, Op0); 1723 case MVT::nxv8i16: return fastEmit_AArch64ISD_UUNPKHI_MVT_nxv8i16_r(RetVT, Op0); 1724 case MVT::nxv4i32: return fastEmit_AArch64ISD_UUNPKHI_MVT_nxv4i32_r(RetVT, Op0); 1725 default: return 0; 1726 } 1727} 1728 1729// FastEmit functions for AArch64ISD::UUNPKLO. 1730 1731unsigned fastEmit_AArch64ISD_UUNPKLO_MVT_nxv16i8_r(MVT RetVT, unsigned Op0) { 1732 if (RetVT.SimpleTy != MVT::nxv8i16) 1733 return 0; 1734 if ((Subtarget->hasSVEorSME())) { 1735 return fastEmitInst_r(AArch64::UUNPKLO_ZZ_H, &AArch64::ZPRRegClass, Op0); 1736 } 1737 return 0; 1738} 1739 1740unsigned fastEmit_AArch64ISD_UUNPKLO_MVT_nxv8i16_r(MVT RetVT, unsigned Op0) { 1741 if (RetVT.SimpleTy != MVT::nxv4i32) 1742 return 0; 1743 if ((Subtarget->hasSVEorSME())) { 1744 return fastEmitInst_r(AArch64::UUNPKLO_ZZ_S, &AArch64::ZPRRegClass, Op0); 1745 } 1746 return 0; 1747} 1748 1749unsigned fastEmit_AArch64ISD_UUNPKLO_MVT_nxv4i32_r(MVT RetVT, unsigned Op0) { 1750 if (RetVT.SimpleTy != MVT::nxv2i64) 1751 return 0; 1752 if ((Subtarget->hasSVEorSME())) { 1753 return fastEmitInst_r(AArch64::UUNPKLO_ZZ_D, &AArch64::ZPRRegClass, Op0); 1754 } 1755 return 0; 1756} 1757 1758unsigned fastEmit_AArch64ISD_UUNPKLO_r(MVT VT, MVT RetVT, unsigned Op0) { 1759 switch (VT.SimpleTy) { 1760 case MVT::nxv16i8: return fastEmit_AArch64ISD_UUNPKLO_MVT_nxv16i8_r(RetVT, Op0); 1761 case MVT::nxv8i16: return fastEmit_AArch64ISD_UUNPKLO_MVT_nxv8i16_r(RetVT, Op0); 1762 case MVT::nxv4i32: return fastEmit_AArch64ISD_UUNPKLO_MVT_nxv4i32_r(RetVT, Op0); 1763 default: return 0; 1764 } 1765} 1766 1767// FastEmit functions for ISD::ABS. 1768 1769unsigned fastEmit_ISD_ABS_MVT_i32_r(MVT RetVT, unsigned Op0) { 1770 if (RetVT.SimpleTy != MVT::i32) 1771 return 0; 1772 if ((Subtarget->hasCSSC())) { 1773 return fastEmitInst_r(AArch64::ABSWr, &AArch64::GPR32RegClass, Op0); 1774 } 1775 return 0; 1776} 1777 1778unsigned fastEmit_ISD_ABS_MVT_i64_r(MVT RetVT, unsigned Op0) { 1779 if (RetVT.SimpleTy != MVT::i64) 1780 return 0; 1781 if ((!Subtarget->hasCSSC())) { 1782 return fastEmitInst_r(AArch64::ABSv1i64, &AArch64::FPR64RegClass, Op0); 1783 } 1784 if ((Subtarget->hasCSSC())) { 1785 return fastEmitInst_r(AArch64::ABSXr, &AArch64::GPR64RegClass, Op0); 1786 } 1787 return 0; 1788} 1789 1790unsigned fastEmit_ISD_ABS_MVT_v8i8_r(MVT RetVT, unsigned Op0) { 1791 if (RetVT.SimpleTy != MVT::v8i8) 1792 return 0; 1793 if ((Subtarget->hasNEON())) { 1794 return fastEmitInst_r(AArch64::ABSv8i8, &AArch64::FPR64RegClass, Op0); 1795 } 1796 return 0; 1797} 1798 1799unsigned fastEmit_ISD_ABS_MVT_v16i8_r(MVT RetVT, unsigned Op0) { 1800 if (RetVT.SimpleTy != MVT::v16i8) 1801 return 0; 1802 if ((Subtarget->hasNEON())) { 1803 return fastEmitInst_r(AArch64::ABSv16i8, &AArch64::FPR128RegClass, Op0); 1804 } 1805 return 0; 1806} 1807 1808unsigned fastEmit_ISD_ABS_MVT_v4i16_r(MVT RetVT, unsigned Op0) { 1809 if (RetVT.SimpleTy != MVT::v4i16) 1810 return 0; 1811 if ((Subtarget->hasNEON())) { 1812 return fastEmitInst_r(AArch64::ABSv4i16, &AArch64::FPR64RegClass, Op0); 1813 } 1814 return 0; 1815} 1816 1817unsigned fastEmit_ISD_ABS_MVT_v8i16_r(MVT RetVT, unsigned Op0) { 1818 if (RetVT.SimpleTy != MVT::v8i16) 1819 return 0; 1820 if ((Subtarget->hasNEON())) { 1821 return fastEmitInst_r(AArch64::ABSv8i16, &AArch64::FPR128RegClass, Op0); 1822 } 1823 return 0; 1824} 1825 1826unsigned fastEmit_ISD_ABS_MVT_v2i32_r(MVT RetVT, unsigned Op0) { 1827 if (RetVT.SimpleTy != MVT::v2i32) 1828 return 0; 1829 if ((Subtarget->hasNEON())) { 1830 return fastEmitInst_r(AArch64::ABSv2i32, &AArch64::FPR64RegClass, Op0); 1831 } 1832 return 0; 1833} 1834 1835unsigned fastEmit_ISD_ABS_MVT_v4i32_r(MVT RetVT, unsigned Op0) { 1836 if (RetVT.SimpleTy != MVT::v4i32) 1837 return 0; 1838 if ((Subtarget->hasNEON())) { 1839 return fastEmitInst_r(AArch64::ABSv4i32, &AArch64::FPR128RegClass, Op0); 1840 } 1841 return 0; 1842} 1843 1844unsigned fastEmit_ISD_ABS_MVT_v1i64_r(MVT RetVT, unsigned Op0) { 1845 if (RetVT.SimpleTy != MVT::v1i64) 1846 return 0; 1847 if ((Subtarget->hasNEON())) { 1848 return fastEmitInst_r(AArch64::ABSv1i64, &AArch64::FPR64RegClass, Op0); 1849 } 1850 return 0; 1851} 1852 1853unsigned fastEmit_ISD_ABS_MVT_v2i64_r(MVT RetVT, unsigned Op0) { 1854 if (RetVT.SimpleTy != MVT::v2i64) 1855 return 0; 1856 if ((Subtarget->hasNEON())) { 1857 return fastEmitInst_r(AArch64::ABSv2i64, &AArch64::FPR128RegClass, Op0); 1858 } 1859 return 0; 1860} 1861 1862unsigned fastEmit_ISD_ABS_r(MVT VT, MVT RetVT, unsigned Op0) { 1863 switch (VT.SimpleTy) { 1864 case MVT::i32: return fastEmit_ISD_ABS_MVT_i32_r(RetVT, Op0); 1865 case MVT::i64: return fastEmit_ISD_ABS_MVT_i64_r(RetVT, Op0); 1866 case MVT::v8i8: return fastEmit_ISD_ABS_MVT_v8i8_r(RetVT, Op0); 1867 case MVT::v16i8: return fastEmit_ISD_ABS_MVT_v16i8_r(RetVT, Op0); 1868 case MVT::v4i16: return fastEmit_ISD_ABS_MVT_v4i16_r(RetVT, Op0); 1869 case MVT::v8i16: return fastEmit_ISD_ABS_MVT_v8i16_r(RetVT, Op0); 1870 case MVT::v2i32: return fastEmit_ISD_ABS_MVT_v2i32_r(RetVT, Op0); 1871 case MVT::v4i32: return fastEmit_ISD_ABS_MVT_v4i32_r(RetVT, Op0); 1872 case MVT::v1i64: return fastEmit_ISD_ABS_MVT_v1i64_r(RetVT, Op0); 1873 case MVT::v2i64: return fastEmit_ISD_ABS_MVT_v2i64_r(RetVT, Op0); 1874 default: return 0; 1875 } 1876} 1877 1878// FastEmit functions for ISD::BITCAST. 1879 1880unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_v8i8_r(unsigned Op0) { 1881 if ((!Subtarget->isLittleEndian())) { 1882 return fastEmitInst_r(AArch64::REV64v8i8, &AArch64::FPR64RegClass, Op0); 1883 } 1884 return 0; 1885} 1886 1887unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_v4i16_r(unsigned Op0) { 1888 if ((!Subtarget->isLittleEndian())) { 1889 return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0); 1890 } 1891 return 0; 1892} 1893 1894unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_v2i32_r(unsigned Op0) { 1895 if ((!Subtarget->isLittleEndian())) { 1896 return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0); 1897 } 1898 return 0; 1899} 1900 1901unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_v4f16_r(unsigned Op0) { 1902 if ((!Subtarget->isLittleEndian())) { 1903 return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0); 1904 } 1905 return 0; 1906} 1907 1908unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_v4bf16_r(unsigned Op0) { 1909 if ((!Subtarget->isLittleEndian())) { 1910 return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0); 1911 } 1912 return 0; 1913} 1914 1915unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_v2f32_r(unsigned Op0) { 1916 if ((!Subtarget->isLittleEndian())) { 1917 return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0); 1918 } 1919 return 0; 1920} 1921 1922unsigned fastEmit_ISD_BITCAST_MVT_f64_r(MVT RetVT, unsigned Op0) { 1923switch (RetVT.SimpleTy) { 1924 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v8i8_r(Op0); 1925 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4i16_r(Op0); 1926 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v2i32_r(Op0); 1927 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4f16_r(Op0); 1928 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4bf16_r(Op0); 1929 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v2f32_r(Op0); 1930 default: return 0; 1931} 1932} 1933 1934unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_f64_r(unsigned Op0) { 1935 if ((!Subtarget->isLittleEndian())) { 1936 return fastEmitInst_r(AArch64::REV64v8i8, &AArch64::FPR64RegClass, Op0); 1937 } 1938 return 0; 1939} 1940 1941unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4i16_r(unsigned Op0) { 1942 if ((!Subtarget->isLittleEndian())) { 1943 return fastEmitInst_r(AArch64::REV16v8i8, &AArch64::FPR64RegClass, Op0); 1944 } 1945 return 0; 1946} 1947 1948unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2i32_r(unsigned Op0) { 1949 if ((!Subtarget->isLittleEndian())) { 1950 return fastEmitInst_r(AArch64::REV32v8i8, &AArch64::FPR64RegClass, Op0); 1951 } 1952 return 0; 1953} 1954 1955unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1i64_r(unsigned Op0) { 1956 if ((!Subtarget->isLittleEndian())) { 1957 return fastEmitInst_r(AArch64::REV64v8i8, &AArch64::FPR64RegClass, Op0); 1958 } 1959 return 0; 1960} 1961 1962unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4f16_r(unsigned Op0) { 1963 if ((!Subtarget->isLittleEndian())) { 1964 return fastEmitInst_r(AArch64::REV16v8i8, &AArch64::FPR64RegClass, Op0); 1965 } 1966 return 0; 1967} 1968 1969unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4bf16_r(unsigned Op0) { 1970 if ((!Subtarget->isLittleEndian())) { 1971 return fastEmitInst_r(AArch64::REV16v8i8, &AArch64::FPR64RegClass, Op0); 1972 } 1973 return 0; 1974} 1975 1976unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2f32_r(unsigned Op0) { 1977 if ((!Subtarget->isLittleEndian())) { 1978 return fastEmitInst_r(AArch64::REV32v8i8, &AArch64::FPR64RegClass, Op0); 1979 } 1980 return 0; 1981} 1982 1983unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1f64_r(unsigned Op0) { 1984 if ((!Subtarget->isLittleEndian())) { 1985 return fastEmitInst_r(AArch64::REV64v8i8, &AArch64::FPR64RegClass, Op0); 1986 } 1987 return 0; 1988} 1989 1990unsigned fastEmit_ISD_BITCAST_MVT_v8i8_r(MVT RetVT, unsigned Op0) { 1991switch (RetVT.SimpleTy) { 1992 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_f64_r(Op0); 1993 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4i16_r(Op0); 1994 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2i32_r(Op0); 1995 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1i64_r(Op0); 1996 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4f16_r(Op0); 1997 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4bf16_r(Op0); 1998 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2f32_r(Op0); 1999 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1f64_r(Op0); 2000 default: return 0; 2001} 2002} 2003 2004unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8i16_r(unsigned Op0) { 2005 if ((!Subtarget->isLittleEndian())) { 2006 return fastEmitInst_r(AArch64::REV16v16i8, &AArch64::FPR128RegClass, Op0); 2007 } 2008 return 0; 2009} 2010 2011unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4i32_r(unsigned Op0) { 2012 if ((!Subtarget->isLittleEndian())) { 2013 return fastEmitInst_r(AArch64::REV32v16i8, &AArch64::FPR128RegClass, Op0); 2014 } 2015 return 0; 2016} 2017 2018unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2i64_r(unsigned Op0) { 2019 if ((!Subtarget->isLittleEndian())) { 2020 return fastEmitInst_r(AArch64::REV64v16i8, &AArch64::FPR128RegClass, Op0); 2021 } 2022 return 0; 2023} 2024 2025unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8f16_r(unsigned Op0) { 2026 if ((!Subtarget->isLittleEndian())) { 2027 return fastEmitInst_r(AArch64::REV16v16i8, &AArch64::FPR128RegClass, Op0); 2028 } 2029 return 0; 2030} 2031 2032unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8bf16_r(unsigned Op0) { 2033 if ((!Subtarget->isLittleEndian())) { 2034 return fastEmitInst_r(AArch64::REV16v16i8, &AArch64::FPR128RegClass, Op0); 2035 } 2036 return 0; 2037} 2038 2039unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4f32_r(unsigned Op0) { 2040 if ((!Subtarget->isLittleEndian())) { 2041 return fastEmitInst_r(AArch64::REV32v16i8, &AArch64::FPR128RegClass, Op0); 2042 } 2043 return 0; 2044} 2045 2046unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2f64_r(unsigned Op0) { 2047 if ((!Subtarget->isLittleEndian())) { 2048 return fastEmitInst_r(AArch64::REV64v16i8, &AArch64::FPR128RegClass, Op0); 2049 } 2050 return 0; 2051} 2052 2053unsigned fastEmit_ISD_BITCAST_MVT_v16i8_r(MVT RetVT, unsigned Op0) { 2054switch (RetVT.SimpleTy) { 2055 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8i16_r(Op0); 2056 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4i32_r(Op0); 2057 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2i64_r(Op0); 2058 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8f16_r(Op0); 2059 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8bf16_r(Op0); 2060 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4f32_r(Op0); 2061 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2f64_r(Op0); 2062 default: return 0; 2063} 2064} 2065 2066unsigned fastEmit_ISD_BITCAST_MVT_v4i16_MVT_f64_r(unsigned Op0) { 2067 if ((!Subtarget->isLittleEndian())) { 2068 return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0); 2069 } 2070 return 0; 2071} 2072 2073unsigned fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v8i8_r(unsigned Op0) { 2074 if ((!Subtarget->isLittleEndian())) { 2075 return fastEmitInst_r(AArch64::REV16v8i8, &AArch64::FPR64RegClass, Op0); 2076 } 2077 return 0; 2078} 2079 2080unsigned fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2i32_r(unsigned Op0) { 2081 if ((!Subtarget->isLittleEndian())) { 2082 return fastEmitInst_r(AArch64::REV32v4i16, &AArch64::FPR64RegClass, Op0); 2083 } 2084 return 0; 2085} 2086 2087unsigned fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1i64_r(unsigned Op0) { 2088 if ((!Subtarget->isLittleEndian())) { 2089 return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0); 2090 } 2091 return 0; 2092} 2093 2094unsigned fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2f32_r(unsigned Op0) { 2095 if ((!Subtarget->isLittleEndian())) { 2096 return fastEmitInst_r(AArch64::REV32v4i16, &AArch64::FPR64RegClass, Op0); 2097 } 2098 return 0; 2099} 2100 2101unsigned fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1f64_r(unsigned Op0) { 2102 if ((!Subtarget->isLittleEndian())) { 2103 return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0); 2104 } 2105 return 0; 2106} 2107 2108unsigned fastEmit_ISD_BITCAST_MVT_v4i16_r(MVT RetVT, unsigned Op0) { 2109switch (RetVT.SimpleTy) { 2110 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_f64_r(Op0); 2111 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v8i8_r(Op0); 2112 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2i32_r(Op0); 2113 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1i64_r(Op0); 2114 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2f32_r(Op0); 2115 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1f64_r(Op0); 2116 default: return 0; 2117} 2118} 2119 2120unsigned fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v16i8_r(unsigned Op0) { 2121 if ((!Subtarget->isLittleEndian())) { 2122 return fastEmitInst_r(AArch64::REV16v16i8, &AArch64::FPR128RegClass, Op0); 2123 } 2124 return 0; 2125} 2126 2127unsigned fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4i32_r(unsigned Op0) { 2128 if ((!Subtarget->isLittleEndian())) { 2129 return fastEmitInst_r(AArch64::REV32v8i16, &AArch64::FPR128RegClass, Op0); 2130 } 2131 return 0; 2132} 2133 2134unsigned fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2i64_r(unsigned Op0) { 2135 if ((!Subtarget->isLittleEndian())) { 2136 return fastEmitInst_r(AArch64::REV64v8i16, &AArch64::FPR128RegClass, Op0); 2137 } 2138 return 0; 2139} 2140 2141unsigned fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4f32_r(unsigned Op0) { 2142 if ((!Subtarget->isLittleEndian())) { 2143 return fastEmitInst_r(AArch64::REV32v8i16, &AArch64::FPR128RegClass, Op0); 2144 } 2145 return 0; 2146} 2147 2148unsigned fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2f64_r(unsigned Op0) { 2149 if ((!Subtarget->isLittleEndian())) { 2150 return fastEmitInst_r(AArch64::REV64v8i16, &AArch64::FPR128RegClass, Op0); 2151 } 2152 return 0; 2153} 2154 2155unsigned fastEmit_ISD_BITCAST_MVT_v8i16_r(MVT RetVT, unsigned Op0) { 2156switch (RetVT.SimpleTy) { 2157 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v16i8_r(Op0); 2158 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4i32_r(Op0); 2159 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2i64_r(Op0); 2160 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4f32_r(Op0); 2161 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2f64_r(Op0); 2162 default: return 0; 2163} 2164} 2165 2166unsigned fastEmit_ISD_BITCAST_MVT_v2i32_MVT_f64_r(unsigned Op0) { 2167 if ((!Subtarget->isLittleEndian())) { 2168 return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0); 2169 } 2170 return 0; 2171} 2172 2173unsigned fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v8i8_r(unsigned Op0) { 2174 if ((!Subtarget->isLittleEndian())) { 2175 return fastEmitInst_r(AArch64::REV32v8i8, &AArch64::FPR64RegClass, Op0); 2176 } 2177 return 0; 2178} 2179 2180unsigned fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4i16_r(unsigned Op0) { 2181 if ((!Subtarget->isLittleEndian())) { 2182 return fastEmitInst_r(AArch64::REV32v4i16, &AArch64::FPR64RegClass, Op0); 2183 } 2184 return 0; 2185} 2186 2187unsigned fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1i64_r(unsigned Op0) { 2188 if ((!Subtarget->isLittleEndian())) { 2189 return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0); 2190 } 2191 return 0; 2192} 2193 2194unsigned fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4f16_r(unsigned Op0) { 2195 if ((!Subtarget->isLittleEndian())) { 2196 return fastEmitInst_r(AArch64::REV32v4i16, &AArch64::FPR64RegClass, Op0); 2197 } 2198 return 0; 2199} 2200 2201unsigned fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4bf16_r(unsigned Op0) { 2202 if ((!Subtarget->isLittleEndian())) { 2203 return fastEmitInst_r(AArch64::REV32v4i16, &AArch64::FPR64RegClass, Op0); 2204 } 2205 return 0; 2206} 2207 2208unsigned fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1f64_r(unsigned Op0) { 2209 if ((!Subtarget->isLittleEndian())) { 2210 return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0); 2211 } 2212 return 0; 2213} 2214 2215unsigned fastEmit_ISD_BITCAST_MVT_v2i32_r(MVT RetVT, unsigned Op0) { 2216switch (RetVT.SimpleTy) { 2217 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_f64_r(Op0); 2218 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v8i8_r(Op0); 2219 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4i16_r(Op0); 2220 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1i64_r(Op0); 2221 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4f16_r(Op0); 2222 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4bf16_r(Op0); 2223 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1f64_r(Op0); 2224 default: return 0; 2225} 2226} 2227 2228unsigned fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v16i8_r(unsigned Op0) { 2229 if ((!Subtarget->isLittleEndian())) { 2230 return fastEmitInst_r(AArch64::REV32v16i8, &AArch64::FPR128RegClass, Op0); 2231 } 2232 return 0; 2233} 2234 2235unsigned fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8i16_r(unsigned Op0) { 2236 if ((!Subtarget->isLittleEndian())) { 2237 return fastEmitInst_r(AArch64::REV32v8i16, &AArch64::FPR128RegClass, Op0); 2238 } 2239 return 0; 2240} 2241 2242unsigned fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2i64_r(unsigned Op0) { 2243 if ((!Subtarget->isLittleEndian())) { 2244 return fastEmitInst_r(AArch64::REV64v4i32, &AArch64::FPR128RegClass, Op0); 2245 } 2246 return 0; 2247} 2248 2249unsigned fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8f16_r(unsigned Op0) { 2250 if ((!Subtarget->isLittleEndian())) { 2251 return fastEmitInst_r(AArch64::REV32v8i16, &AArch64::FPR128RegClass, Op0); 2252 } 2253 return 0; 2254} 2255 2256unsigned fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8bf16_r(unsigned Op0) { 2257 if ((!Subtarget->isLittleEndian())) { 2258 return fastEmitInst_r(AArch64::REV32v8i16, &AArch64::FPR128RegClass, Op0); 2259 } 2260 return 0; 2261} 2262 2263unsigned fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2f64_r(unsigned Op0) { 2264 if ((!Subtarget->isLittleEndian())) { 2265 return fastEmitInst_r(AArch64::REV64v4i32, &AArch64::FPR128RegClass, Op0); 2266 } 2267 return 0; 2268} 2269 2270unsigned fastEmit_ISD_BITCAST_MVT_v4i32_r(MVT RetVT, unsigned Op0) { 2271switch (RetVT.SimpleTy) { 2272 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v16i8_r(Op0); 2273 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8i16_r(Op0); 2274 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2i64_r(Op0); 2275 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8f16_r(Op0); 2276 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8bf16_r(Op0); 2277 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2f64_r(Op0); 2278 default: return 0; 2279} 2280} 2281 2282unsigned fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v8i8_r(unsigned Op0) { 2283 if ((!Subtarget->isLittleEndian())) { 2284 return fastEmitInst_r(AArch64::REV64v8i8, &AArch64::FPR64RegClass, Op0); 2285 } 2286 return 0; 2287} 2288 2289unsigned fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4i16_r(unsigned Op0) { 2290 if ((!Subtarget->isLittleEndian())) { 2291 return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0); 2292 } 2293 return 0; 2294} 2295 2296unsigned fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2i32_r(unsigned Op0) { 2297 if ((!Subtarget->isLittleEndian())) { 2298 return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0); 2299 } 2300 return 0; 2301} 2302 2303unsigned fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4f16_r(unsigned Op0) { 2304 if ((!Subtarget->isLittleEndian())) { 2305 return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0); 2306 } 2307 return 0; 2308} 2309 2310unsigned fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4bf16_r(unsigned Op0) { 2311 if ((!Subtarget->isLittleEndian())) { 2312 return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0); 2313 } 2314 return 0; 2315} 2316 2317unsigned fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2f32_r(unsigned Op0) { 2318 if ((!Subtarget->isLittleEndian())) { 2319 return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0); 2320 } 2321 return 0; 2322} 2323 2324unsigned fastEmit_ISD_BITCAST_MVT_v1i64_r(MVT RetVT, unsigned Op0) { 2325switch (RetVT.SimpleTy) { 2326 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v8i8_r(Op0); 2327 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4i16_r(Op0); 2328 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2i32_r(Op0); 2329 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4f16_r(Op0); 2330 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4bf16_r(Op0); 2331 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2f32_r(Op0); 2332 default: return 0; 2333} 2334} 2335 2336unsigned fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v16i8_r(unsigned Op0) { 2337 if ((!Subtarget->isLittleEndian())) { 2338 return fastEmitInst_r(AArch64::REV64v16i8, &AArch64::FPR128RegClass, Op0); 2339 } 2340 return 0; 2341} 2342 2343unsigned fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8i16_r(unsigned Op0) { 2344 if ((!Subtarget->isLittleEndian())) { 2345 return fastEmitInst_r(AArch64::REV64v8i16, &AArch64::FPR128RegClass, Op0); 2346 } 2347 return 0; 2348} 2349 2350unsigned fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4i32_r(unsigned Op0) { 2351 if ((!Subtarget->isLittleEndian())) { 2352 return fastEmitInst_r(AArch64::REV64v4i32, &AArch64::FPR128RegClass, Op0); 2353 } 2354 return 0; 2355} 2356 2357unsigned fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8f16_r(unsigned Op0) { 2358 if ((!Subtarget->isLittleEndian())) { 2359 return fastEmitInst_r(AArch64::REV64v8i16, &AArch64::FPR128RegClass, Op0); 2360 } 2361 return 0; 2362} 2363 2364unsigned fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8bf16_r(unsigned Op0) { 2365 if ((!Subtarget->isLittleEndian())) { 2366 return fastEmitInst_r(AArch64::REV64v8i16, &AArch64::FPR128RegClass, Op0); 2367 } 2368 return 0; 2369} 2370 2371unsigned fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4f32_r(unsigned Op0) { 2372 if ((!Subtarget->isLittleEndian())) { 2373 return fastEmitInst_r(AArch64::REV64v4i32, &AArch64::FPR128RegClass, Op0); 2374 } 2375 return 0; 2376} 2377 2378unsigned fastEmit_ISD_BITCAST_MVT_v2i64_r(MVT RetVT, unsigned Op0) { 2379switch (RetVT.SimpleTy) { 2380 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v16i8_r(Op0); 2381 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8i16_r(Op0); 2382 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4i32_r(Op0); 2383 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8f16_r(Op0); 2384 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8bf16_r(Op0); 2385 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4f32_r(Op0); 2386 default: return 0; 2387} 2388} 2389 2390unsigned fastEmit_ISD_BITCAST_MVT_v4f16_MVT_f64_r(unsigned Op0) { 2391 if ((!Subtarget->isLittleEndian())) { 2392 return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0); 2393 } 2394 return 0; 2395} 2396 2397unsigned fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v8i8_r(unsigned Op0) { 2398 if ((!Subtarget->isLittleEndian())) { 2399 return fastEmitInst_r(AArch64::REV16v8i8, &AArch64::FPR64RegClass, Op0); 2400 } 2401 return 0; 2402} 2403 2404unsigned fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2i32_r(unsigned Op0) { 2405 if ((!Subtarget->isLittleEndian())) { 2406 return fastEmitInst_r(AArch64::REV32v4i16, &AArch64::FPR64RegClass, Op0); 2407 } 2408 return 0; 2409} 2410 2411unsigned fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1i64_r(unsigned Op0) { 2412 if ((!Subtarget->isLittleEndian())) { 2413 return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0); 2414 } 2415 return 0; 2416} 2417 2418unsigned fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2f32_r(unsigned Op0) { 2419 if ((!Subtarget->isLittleEndian())) { 2420 return fastEmitInst_r(AArch64::REV32v4i16, &AArch64::FPR64RegClass, Op0); 2421 } 2422 return 0; 2423} 2424 2425unsigned fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1f64_r(unsigned Op0) { 2426 if ((!Subtarget->isLittleEndian())) { 2427 return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0); 2428 } 2429 return 0; 2430} 2431 2432unsigned fastEmit_ISD_BITCAST_MVT_v4f16_r(MVT RetVT, unsigned Op0) { 2433switch (RetVT.SimpleTy) { 2434 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_f64_r(Op0); 2435 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v8i8_r(Op0); 2436 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2i32_r(Op0); 2437 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1i64_r(Op0); 2438 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2f32_r(Op0); 2439 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1f64_r(Op0); 2440 default: return 0; 2441} 2442} 2443 2444unsigned fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v16i8_r(unsigned Op0) { 2445 if ((!Subtarget->isLittleEndian())) { 2446 return fastEmitInst_r(AArch64::REV16v16i8, &AArch64::FPR128RegClass, Op0); 2447 } 2448 return 0; 2449} 2450 2451unsigned fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4i32_r(unsigned Op0) { 2452 if ((!Subtarget->isLittleEndian())) { 2453 return fastEmitInst_r(AArch64::REV32v8i16, &AArch64::FPR128RegClass, Op0); 2454 } 2455 return 0; 2456} 2457 2458unsigned fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2i64_r(unsigned Op0) { 2459 if ((!Subtarget->isLittleEndian())) { 2460 return fastEmitInst_r(AArch64::REV64v8i16, &AArch64::FPR128RegClass, Op0); 2461 } 2462 return 0; 2463} 2464 2465unsigned fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4f32_r(unsigned Op0) { 2466 if ((!Subtarget->isLittleEndian())) { 2467 return fastEmitInst_r(AArch64::REV32v8i16, &AArch64::FPR128RegClass, Op0); 2468 } 2469 return 0; 2470} 2471 2472unsigned fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2f64_r(unsigned Op0) { 2473 if ((!Subtarget->isLittleEndian())) { 2474 return fastEmitInst_r(AArch64::REV64v8i16, &AArch64::FPR128RegClass, Op0); 2475 } 2476 return 0; 2477} 2478 2479unsigned fastEmit_ISD_BITCAST_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 2480switch (RetVT.SimpleTy) { 2481 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v16i8_r(Op0); 2482 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4i32_r(Op0); 2483 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2i64_r(Op0); 2484 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4f32_r(Op0); 2485 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2f64_r(Op0); 2486 default: return 0; 2487} 2488} 2489 2490unsigned fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_f64_r(unsigned Op0) { 2491 if ((!Subtarget->isLittleEndian())) { 2492 return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0); 2493 } 2494 return 0; 2495} 2496 2497unsigned fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v8i8_r(unsigned Op0) { 2498 if ((!Subtarget->isLittleEndian())) { 2499 return fastEmitInst_r(AArch64::REV16v8i8, &AArch64::FPR64RegClass, Op0); 2500 } 2501 return 0; 2502} 2503 2504unsigned fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2i32_r(unsigned Op0) { 2505 if ((!Subtarget->isLittleEndian())) { 2506 return fastEmitInst_r(AArch64::REV32v4i16, &AArch64::FPR64RegClass, Op0); 2507 } 2508 return 0; 2509} 2510 2511unsigned fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1i64_r(unsigned Op0) { 2512 if ((!Subtarget->isLittleEndian())) { 2513 return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0); 2514 } 2515 return 0; 2516} 2517 2518unsigned fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2f32_r(unsigned Op0) { 2519 if ((!Subtarget->isLittleEndian())) { 2520 return fastEmitInst_r(AArch64::REV32v4i16, &AArch64::FPR64RegClass, Op0); 2521 } 2522 return 0; 2523} 2524 2525unsigned fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1f64_r(unsigned Op0) { 2526 if ((!Subtarget->isLittleEndian())) { 2527 return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0); 2528 } 2529 return 0; 2530} 2531 2532unsigned fastEmit_ISD_BITCAST_MVT_v4bf16_r(MVT RetVT, unsigned Op0) { 2533switch (RetVT.SimpleTy) { 2534 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_f64_r(Op0); 2535 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v8i8_r(Op0); 2536 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2i32_r(Op0); 2537 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1i64_r(Op0); 2538 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2f32_r(Op0); 2539 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1f64_r(Op0); 2540 default: return 0; 2541} 2542} 2543 2544unsigned fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v16i8_r(unsigned Op0) { 2545 if ((!Subtarget->isLittleEndian())) { 2546 return fastEmitInst_r(AArch64::REV16v16i8, &AArch64::FPR128RegClass, Op0); 2547 } 2548 return 0; 2549} 2550 2551unsigned fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4i32_r(unsigned Op0) { 2552 if ((!Subtarget->isLittleEndian())) { 2553 return fastEmitInst_r(AArch64::REV32v8i16, &AArch64::FPR128RegClass, Op0); 2554 } 2555 return 0; 2556} 2557 2558unsigned fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2i64_r(unsigned Op0) { 2559 if ((!Subtarget->isLittleEndian())) { 2560 return fastEmitInst_r(AArch64::REV64v8i16, &AArch64::FPR128RegClass, Op0); 2561 } 2562 return 0; 2563} 2564 2565unsigned fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4f32_r(unsigned Op0) { 2566 if ((!Subtarget->isLittleEndian())) { 2567 return fastEmitInst_r(AArch64::REV32v8i16, &AArch64::FPR128RegClass, Op0); 2568 } 2569 return 0; 2570} 2571 2572unsigned fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2f64_r(unsigned Op0) { 2573 if ((!Subtarget->isLittleEndian())) { 2574 return fastEmitInst_r(AArch64::REV64v8i16, &AArch64::FPR128RegClass, Op0); 2575 } 2576 return 0; 2577} 2578 2579unsigned fastEmit_ISD_BITCAST_MVT_v8bf16_r(MVT RetVT, unsigned Op0) { 2580switch (RetVT.SimpleTy) { 2581 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v16i8_r(Op0); 2582 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4i32_r(Op0); 2583 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2i64_r(Op0); 2584 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4f32_r(Op0); 2585 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2f64_r(Op0); 2586 default: return 0; 2587} 2588} 2589 2590unsigned fastEmit_ISD_BITCAST_MVT_v2f32_MVT_f64_r(unsigned Op0) { 2591 if ((!Subtarget->isLittleEndian())) { 2592 return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0); 2593 } 2594 return 0; 2595} 2596 2597unsigned fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v8i8_r(unsigned Op0) { 2598 if ((!Subtarget->isLittleEndian())) { 2599 return fastEmitInst_r(AArch64::REV32v8i8, &AArch64::FPR64RegClass, Op0); 2600 } 2601 return 0; 2602} 2603 2604unsigned fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4i16_r(unsigned Op0) { 2605 if ((!Subtarget->isLittleEndian())) { 2606 return fastEmitInst_r(AArch64::REV32v4i16, &AArch64::FPR64RegClass, Op0); 2607 } 2608 return 0; 2609} 2610 2611unsigned fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1i64_r(unsigned Op0) { 2612 if ((!Subtarget->isLittleEndian())) { 2613 return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0); 2614 } 2615 return 0; 2616} 2617 2618unsigned fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4f16_r(unsigned Op0) { 2619 if ((!Subtarget->isLittleEndian())) { 2620 return fastEmitInst_r(AArch64::REV32v4i16, &AArch64::FPR64RegClass, Op0); 2621 } 2622 return 0; 2623} 2624 2625unsigned fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4bf16_r(unsigned Op0) { 2626 if ((!Subtarget->isLittleEndian())) { 2627 return fastEmitInst_r(AArch64::REV32v4i16, &AArch64::FPR64RegClass, Op0); 2628 } 2629 return 0; 2630} 2631 2632unsigned fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1f64_r(unsigned Op0) { 2633 if ((!Subtarget->isLittleEndian())) { 2634 return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0); 2635 } 2636 return 0; 2637} 2638 2639unsigned fastEmit_ISD_BITCAST_MVT_v2f32_r(MVT RetVT, unsigned Op0) { 2640switch (RetVT.SimpleTy) { 2641 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_f64_r(Op0); 2642 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v8i8_r(Op0); 2643 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4i16_r(Op0); 2644 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1i64_r(Op0); 2645 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4f16_r(Op0); 2646 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4bf16_r(Op0); 2647 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1f64_r(Op0); 2648 default: return 0; 2649} 2650} 2651 2652unsigned fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v16i8_r(unsigned Op0) { 2653 if ((!Subtarget->isLittleEndian())) { 2654 return fastEmitInst_r(AArch64::REV32v16i8, &AArch64::FPR128RegClass, Op0); 2655 } 2656 return 0; 2657} 2658 2659unsigned fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8i16_r(unsigned Op0) { 2660 if ((!Subtarget->isLittleEndian())) { 2661 return fastEmitInst_r(AArch64::REV32v8i16, &AArch64::FPR128RegClass, Op0); 2662 } 2663 return 0; 2664} 2665 2666unsigned fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2i64_r(unsigned Op0) { 2667 if ((!Subtarget->isLittleEndian())) { 2668 return fastEmitInst_r(AArch64::REV64v4i32, &AArch64::FPR128RegClass, Op0); 2669 } 2670 return 0; 2671} 2672 2673unsigned fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8f16_r(unsigned Op0) { 2674 if ((!Subtarget->isLittleEndian())) { 2675 return fastEmitInst_r(AArch64::REV32v8i16, &AArch64::FPR128RegClass, Op0); 2676 } 2677 return 0; 2678} 2679 2680unsigned fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8bf16_r(unsigned Op0) { 2681 if ((!Subtarget->isLittleEndian())) { 2682 return fastEmitInst_r(AArch64::REV32v8i16, &AArch64::FPR128RegClass, Op0); 2683 } 2684 return 0; 2685} 2686 2687unsigned fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2f64_r(unsigned Op0) { 2688 if ((!Subtarget->isLittleEndian())) { 2689 return fastEmitInst_r(AArch64::REV64v4i32, &AArch64::FPR128RegClass, Op0); 2690 } 2691 return 0; 2692} 2693 2694unsigned fastEmit_ISD_BITCAST_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 2695switch (RetVT.SimpleTy) { 2696 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v16i8_r(Op0); 2697 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8i16_r(Op0); 2698 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2i64_r(Op0); 2699 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8f16_r(Op0); 2700 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8bf16_r(Op0); 2701 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2f64_r(Op0); 2702 default: return 0; 2703} 2704} 2705 2706unsigned fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v8i8_r(unsigned Op0) { 2707 if ((!Subtarget->isLittleEndian())) { 2708 return fastEmitInst_r(AArch64::REV64v8i8, &AArch64::FPR64RegClass, Op0); 2709 } 2710 return 0; 2711} 2712 2713unsigned fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4i16_r(unsigned Op0) { 2714 if ((!Subtarget->isLittleEndian())) { 2715 return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0); 2716 } 2717 return 0; 2718} 2719 2720unsigned fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2i32_r(unsigned Op0) { 2721 if ((!Subtarget->isLittleEndian())) { 2722 return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0); 2723 } 2724 return 0; 2725} 2726 2727unsigned fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4f16_r(unsigned Op0) { 2728 if ((!Subtarget->isLittleEndian())) { 2729 return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0); 2730 } 2731 return 0; 2732} 2733 2734unsigned fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4bf16_r(unsigned Op0) { 2735 if ((!Subtarget->isLittleEndian())) { 2736 return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0); 2737 } 2738 return 0; 2739} 2740 2741unsigned fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2f32_r(unsigned Op0) { 2742 if ((!Subtarget->isLittleEndian())) { 2743 return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0); 2744 } 2745 return 0; 2746} 2747 2748unsigned fastEmit_ISD_BITCAST_MVT_v1f64_r(MVT RetVT, unsigned Op0) { 2749switch (RetVT.SimpleTy) { 2750 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v8i8_r(Op0); 2751 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4i16_r(Op0); 2752 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2i32_r(Op0); 2753 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4f16_r(Op0); 2754 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4bf16_r(Op0); 2755 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2f32_r(Op0); 2756 default: return 0; 2757} 2758} 2759 2760unsigned fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v16i8_r(unsigned Op0) { 2761 if ((!Subtarget->isLittleEndian())) { 2762 return fastEmitInst_r(AArch64::REV64v16i8, &AArch64::FPR128RegClass, Op0); 2763 } 2764 return 0; 2765} 2766 2767unsigned fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8i16_r(unsigned Op0) { 2768 if ((!Subtarget->isLittleEndian())) { 2769 return fastEmitInst_r(AArch64::REV64v8i16, &AArch64::FPR128RegClass, Op0); 2770 } 2771 return 0; 2772} 2773 2774unsigned fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4i32_r(unsigned Op0) { 2775 if ((!Subtarget->isLittleEndian())) { 2776 return fastEmitInst_r(AArch64::REV64v4i32, &AArch64::FPR128RegClass, Op0); 2777 } 2778 return 0; 2779} 2780 2781unsigned fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8f16_r(unsigned Op0) { 2782 if ((!Subtarget->isLittleEndian())) { 2783 return fastEmitInst_r(AArch64::REV64v8i16, &AArch64::FPR128RegClass, Op0); 2784 } 2785 return 0; 2786} 2787 2788unsigned fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8bf16_r(unsigned Op0) { 2789 if ((!Subtarget->isLittleEndian())) { 2790 return fastEmitInst_r(AArch64::REV64v8i16, &AArch64::FPR128RegClass, Op0); 2791 } 2792 return 0; 2793} 2794 2795unsigned fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4f32_r(unsigned Op0) { 2796 if ((!Subtarget->isLittleEndian())) { 2797 return fastEmitInst_r(AArch64::REV64v4i32, &AArch64::FPR128RegClass, Op0); 2798 } 2799 return 0; 2800} 2801 2802unsigned fastEmit_ISD_BITCAST_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 2803switch (RetVT.SimpleTy) { 2804 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v16i8_r(Op0); 2805 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8i16_r(Op0); 2806 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4i32_r(Op0); 2807 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8f16_r(Op0); 2808 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8bf16_r(Op0); 2809 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4f32_r(Op0); 2810 default: return 0; 2811} 2812} 2813 2814unsigned fastEmit_ISD_BITCAST_r(MVT VT, MVT RetVT, unsigned Op0) { 2815 switch (VT.SimpleTy) { 2816 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_f64_r(RetVT, Op0); 2817 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v8i8_r(RetVT, Op0); 2818 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v16i8_r(RetVT, Op0); 2819 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v4i16_r(RetVT, Op0); 2820 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v8i16_r(RetVT, Op0); 2821 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v2i32_r(RetVT, Op0); 2822 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v4i32_r(RetVT, Op0); 2823 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v1i64_r(RetVT, Op0); 2824 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v2i64_r(RetVT, Op0); 2825 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v4f16_r(RetVT, Op0); 2826 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v8f16_r(RetVT, Op0); 2827 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v4bf16_r(RetVT, Op0); 2828 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v8bf16_r(RetVT, Op0); 2829 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v2f32_r(RetVT, Op0); 2830 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v4f32_r(RetVT, Op0); 2831 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v1f64_r(RetVT, Op0); 2832 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v2f64_r(RetVT, Op0); 2833 default: return 0; 2834 } 2835} 2836 2837// FastEmit functions for ISD::BITREVERSE. 2838 2839unsigned fastEmit_ISD_BITREVERSE_MVT_i32_r(MVT RetVT, unsigned Op0) { 2840 if (RetVT.SimpleTy != MVT::i32) 2841 return 0; 2842 return fastEmitInst_r(AArch64::RBITWr, &AArch64::GPR32RegClass, Op0); 2843} 2844 2845unsigned fastEmit_ISD_BITREVERSE_MVT_i64_r(MVT RetVT, unsigned Op0) { 2846 if (RetVT.SimpleTy != MVT::i64) 2847 return 0; 2848 return fastEmitInst_r(AArch64::RBITXr, &AArch64::GPR64RegClass, Op0); 2849} 2850 2851unsigned fastEmit_ISD_BITREVERSE_MVT_v8i8_r(MVT RetVT, unsigned Op0) { 2852 if (RetVT.SimpleTy != MVT::v8i8) 2853 return 0; 2854 if ((Subtarget->hasNEON())) { 2855 return fastEmitInst_r(AArch64::RBITv8i8, &AArch64::FPR64RegClass, Op0); 2856 } 2857 return 0; 2858} 2859 2860unsigned fastEmit_ISD_BITREVERSE_MVT_v16i8_r(MVT RetVT, unsigned Op0) { 2861 if (RetVT.SimpleTy != MVT::v16i8) 2862 return 0; 2863 if ((Subtarget->hasNEON())) { 2864 return fastEmitInst_r(AArch64::RBITv16i8, &AArch64::FPR128RegClass, Op0); 2865 } 2866 return 0; 2867} 2868 2869unsigned fastEmit_ISD_BITREVERSE_r(MVT VT, MVT RetVT, unsigned Op0) { 2870 switch (VT.SimpleTy) { 2871 case MVT::i32: return fastEmit_ISD_BITREVERSE_MVT_i32_r(RetVT, Op0); 2872 case MVT::i64: return fastEmit_ISD_BITREVERSE_MVT_i64_r(RetVT, Op0); 2873 case MVT::v8i8: return fastEmit_ISD_BITREVERSE_MVT_v8i8_r(RetVT, Op0); 2874 case MVT::v16i8: return fastEmit_ISD_BITREVERSE_MVT_v16i8_r(RetVT, Op0); 2875 default: return 0; 2876 } 2877} 2878 2879// FastEmit functions for ISD::BRIND. 2880 2881unsigned fastEmit_ISD_BRIND_MVT_i64_r(MVT RetVT, unsigned Op0) { 2882 if (RetVT.SimpleTy != MVT::isVoid) 2883 return 0; 2884 return fastEmitInst_r(AArch64::BR, &AArch64::GPR64RegClass, Op0); 2885} 2886 2887unsigned fastEmit_ISD_BRIND_r(MVT VT, MVT RetVT, unsigned Op0) { 2888 switch (VT.SimpleTy) { 2889 case MVT::i64: return fastEmit_ISD_BRIND_MVT_i64_r(RetVT, Op0); 2890 default: return 0; 2891 } 2892} 2893 2894// FastEmit functions for ISD::BSWAP. 2895 2896unsigned fastEmit_ISD_BSWAP_MVT_i32_r(MVT RetVT, unsigned Op0) { 2897 if (RetVT.SimpleTy != MVT::i32) 2898 return 0; 2899 return fastEmitInst_r(AArch64::REVWr, &AArch64::GPR32RegClass, Op0); 2900} 2901 2902unsigned fastEmit_ISD_BSWAP_MVT_i64_r(MVT RetVT, unsigned Op0) { 2903 if (RetVT.SimpleTy != MVT::i64) 2904 return 0; 2905 return fastEmitInst_r(AArch64::REVXr, &AArch64::GPR64RegClass, Op0); 2906} 2907 2908unsigned fastEmit_ISD_BSWAP_r(MVT VT, MVT RetVT, unsigned Op0) { 2909 switch (VT.SimpleTy) { 2910 case MVT::i32: return fastEmit_ISD_BSWAP_MVT_i32_r(RetVT, Op0); 2911 case MVT::i64: return fastEmit_ISD_BSWAP_MVT_i64_r(RetVT, Op0); 2912 default: return 0; 2913 } 2914} 2915 2916// FastEmit functions for ISD::CTLZ. 2917 2918unsigned fastEmit_ISD_CTLZ_MVT_i32_r(MVT RetVT, unsigned Op0) { 2919 if (RetVT.SimpleTy != MVT::i32) 2920 return 0; 2921 return fastEmitInst_r(AArch64::CLZWr, &AArch64::GPR32RegClass, Op0); 2922} 2923 2924unsigned fastEmit_ISD_CTLZ_MVT_i64_r(MVT RetVT, unsigned Op0) { 2925 if (RetVT.SimpleTy != MVT::i64) 2926 return 0; 2927 return fastEmitInst_r(AArch64::CLZXr, &AArch64::GPR64RegClass, Op0); 2928} 2929 2930unsigned fastEmit_ISD_CTLZ_MVT_v8i8_r(MVT RetVT, unsigned Op0) { 2931 if (RetVT.SimpleTy != MVT::v8i8) 2932 return 0; 2933 if ((Subtarget->hasNEON())) { 2934 return fastEmitInst_r(AArch64::CLZv8i8, &AArch64::FPR64RegClass, Op0); 2935 } 2936 return 0; 2937} 2938 2939unsigned fastEmit_ISD_CTLZ_MVT_v16i8_r(MVT RetVT, unsigned Op0) { 2940 if (RetVT.SimpleTy != MVT::v16i8) 2941 return 0; 2942 if ((Subtarget->hasNEON())) { 2943 return fastEmitInst_r(AArch64::CLZv16i8, &AArch64::FPR128RegClass, Op0); 2944 } 2945 return 0; 2946} 2947 2948unsigned fastEmit_ISD_CTLZ_MVT_v4i16_r(MVT RetVT, unsigned Op0) { 2949 if (RetVT.SimpleTy != MVT::v4i16) 2950 return 0; 2951 if ((Subtarget->hasNEON())) { 2952 return fastEmitInst_r(AArch64::CLZv4i16, &AArch64::FPR64RegClass, Op0); 2953 } 2954 return 0; 2955} 2956 2957unsigned fastEmit_ISD_CTLZ_MVT_v8i16_r(MVT RetVT, unsigned Op0) { 2958 if (RetVT.SimpleTy != MVT::v8i16) 2959 return 0; 2960 if ((Subtarget->hasNEON())) { 2961 return fastEmitInst_r(AArch64::CLZv8i16, &AArch64::FPR128RegClass, Op0); 2962 } 2963 return 0; 2964} 2965 2966unsigned fastEmit_ISD_CTLZ_MVT_v2i32_r(MVT RetVT, unsigned Op0) { 2967 if (RetVT.SimpleTy != MVT::v2i32) 2968 return 0; 2969 if ((Subtarget->hasNEON())) { 2970 return fastEmitInst_r(AArch64::CLZv2i32, &AArch64::FPR64RegClass, Op0); 2971 } 2972 return 0; 2973} 2974 2975unsigned fastEmit_ISD_CTLZ_MVT_v4i32_r(MVT RetVT, unsigned Op0) { 2976 if (RetVT.SimpleTy != MVT::v4i32) 2977 return 0; 2978 if ((Subtarget->hasNEON())) { 2979 return fastEmitInst_r(AArch64::CLZv4i32, &AArch64::FPR128RegClass, Op0); 2980 } 2981 return 0; 2982} 2983 2984unsigned fastEmit_ISD_CTLZ_r(MVT VT, MVT RetVT, unsigned Op0) { 2985 switch (VT.SimpleTy) { 2986 case MVT::i32: return fastEmit_ISD_CTLZ_MVT_i32_r(RetVT, Op0); 2987 case MVT::i64: return fastEmit_ISD_CTLZ_MVT_i64_r(RetVT, Op0); 2988 case MVT::v8i8: return fastEmit_ISD_CTLZ_MVT_v8i8_r(RetVT, Op0); 2989 case MVT::v16i8: return fastEmit_ISD_CTLZ_MVT_v16i8_r(RetVT, Op0); 2990 case MVT::v4i16: return fastEmit_ISD_CTLZ_MVT_v4i16_r(RetVT, Op0); 2991 case MVT::v8i16: return fastEmit_ISD_CTLZ_MVT_v8i16_r(RetVT, Op0); 2992 case MVT::v2i32: return fastEmit_ISD_CTLZ_MVT_v2i32_r(RetVT, Op0); 2993 case MVT::v4i32: return fastEmit_ISD_CTLZ_MVT_v4i32_r(RetVT, Op0); 2994 default: return 0; 2995 } 2996} 2997 2998// FastEmit functions for ISD::CTPOP. 2999 3000unsigned fastEmit_ISD_CTPOP_MVT_i32_r(MVT RetVT, unsigned Op0) { 3001 if (RetVT.SimpleTy != MVT::i32) 3002 return 0; 3003 if ((Subtarget->hasCSSC())) { 3004 return fastEmitInst_r(AArch64::CNTWr, &AArch64::GPR32RegClass, Op0); 3005 } 3006 return 0; 3007} 3008 3009unsigned fastEmit_ISD_CTPOP_MVT_i64_r(MVT RetVT, unsigned Op0) { 3010 if (RetVT.SimpleTy != MVT::i64) 3011 return 0; 3012 if ((Subtarget->hasCSSC())) { 3013 return fastEmitInst_r(AArch64::CNTXr, &AArch64::GPR64RegClass, Op0); 3014 } 3015 return 0; 3016} 3017 3018unsigned fastEmit_ISD_CTPOP_MVT_v8i8_r(MVT RetVT, unsigned Op0) { 3019 if (RetVT.SimpleTy != MVT::v8i8) 3020 return 0; 3021 if ((Subtarget->hasNEON())) { 3022 return fastEmitInst_r(AArch64::CNTv8i8, &AArch64::FPR64RegClass, Op0); 3023 } 3024 return 0; 3025} 3026 3027unsigned fastEmit_ISD_CTPOP_MVT_v16i8_r(MVT RetVT, unsigned Op0) { 3028 if (RetVT.SimpleTy != MVT::v16i8) 3029 return 0; 3030 if ((Subtarget->hasNEON())) { 3031 return fastEmitInst_r(AArch64::CNTv16i8, &AArch64::FPR128RegClass, Op0); 3032 } 3033 return 0; 3034} 3035 3036unsigned fastEmit_ISD_CTPOP_r(MVT VT, MVT RetVT, unsigned Op0) { 3037 switch (VT.SimpleTy) { 3038 case MVT::i32: return fastEmit_ISD_CTPOP_MVT_i32_r(RetVT, Op0); 3039 case MVT::i64: return fastEmit_ISD_CTPOP_MVT_i64_r(RetVT, Op0); 3040 case MVT::v8i8: return fastEmit_ISD_CTPOP_MVT_v8i8_r(RetVT, Op0); 3041 case MVT::v16i8: return fastEmit_ISD_CTPOP_MVT_v16i8_r(RetVT, Op0); 3042 default: return 0; 3043 } 3044} 3045 3046// FastEmit functions for ISD::CTTZ. 3047 3048unsigned fastEmit_ISD_CTTZ_MVT_i32_r(MVT RetVT, unsigned Op0) { 3049 if (RetVT.SimpleTy != MVT::i32) 3050 return 0; 3051 if ((Subtarget->hasCSSC())) { 3052 return fastEmitInst_r(AArch64::CTZWr, &AArch64::GPR32RegClass, Op0); 3053 } 3054 return 0; 3055} 3056 3057unsigned fastEmit_ISD_CTTZ_MVT_i64_r(MVT RetVT, unsigned Op0) { 3058 if (RetVT.SimpleTy != MVT::i64) 3059 return 0; 3060 if ((Subtarget->hasCSSC())) { 3061 return fastEmitInst_r(AArch64::CTZXr, &AArch64::GPR64RegClass, Op0); 3062 } 3063 return 0; 3064} 3065 3066unsigned fastEmit_ISD_CTTZ_r(MVT VT, MVT RetVT, unsigned Op0) { 3067 switch (VT.SimpleTy) { 3068 case MVT::i32: return fastEmit_ISD_CTTZ_MVT_i32_r(RetVT, Op0); 3069 case MVT::i64: return fastEmit_ISD_CTTZ_MVT_i64_r(RetVT, Op0); 3070 default: return 0; 3071 } 3072} 3073 3074// FastEmit functions for ISD::FABS. 3075 3076unsigned fastEmit_ISD_FABS_MVT_f16_r(MVT RetVT, unsigned Op0) { 3077 if (RetVT.SimpleTy != MVT::f16) 3078 return 0; 3079 if ((Subtarget->hasFullFP16())) { 3080 return fastEmitInst_r(AArch64::FABSHr, &AArch64::FPR16RegClass, Op0); 3081 } 3082 return 0; 3083} 3084 3085unsigned fastEmit_ISD_FABS_MVT_f32_r(MVT RetVT, unsigned Op0) { 3086 if (RetVT.SimpleTy != MVT::f32) 3087 return 0; 3088 if ((Subtarget->hasFPARMv8())) { 3089 return fastEmitInst_r(AArch64::FABSSr, &AArch64::FPR32RegClass, Op0); 3090 } 3091 return 0; 3092} 3093 3094unsigned fastEmit_ISD_FABS_MVT_f64_r(MVT RetVT, unsigned Op0) { 3095 if (RetVT.SimpleTy != MVT::f64) 3096 return 0; 3097 if ((Subtarget->hasFPARMv8())) { 3098 return fastEmitInst_r(AArch64::FABSDr, &AArch64::FPR64RegClass, Op0); 3099 } 3100 return 0; 3101} 3102 3103unsigned fastEmit_ISD_FABS_MVT_v4f16_r(MVT RetVT, unsigned Op0) { 3104 if (RetVT.SimpleTy != MVT::v4f16) 3105 return 0; 3106 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 3107 return fastEmitInst_r(AArch64::FABSv4f16, &AArch64::FPR64RegClass, Op0); 3108 } 3109 return 0; 3110} 3111 3112unsigned fastEmit_ISD_FABS_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 3113 if (RetVT.SimpleTy != MVT::v8f16) 3114 return 0; 3115 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 3116 return fastEmitInst_r(AArch64::FABSv8f16, &AArch64::FPR128RegClass, Op0); 3117 } 3118 return 0; 3119} 3120 3121unsigned fastEmit_ISD_FABS_MVT_v2f32_r(MVT RetVT, unsigned Op0) { 3122 if (RetVT.SimpleTy != MVT::v2f32) 3123 return 0; 3124 if ((Subtarget->hasNEON())) { 3125 return fastEmitInst_r(AArch64::FABSv2f32, &AArch64::FPR64RegClass, Op0); 3126 } 3127 return 0; 3128} 3129 3130unsigned fastEmit_ISD_FABS_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 3131 if (RetVT.SimpleTy != MVT::v4f32) 3132 return 0; 3133 if ((Subtarget->hasNEON())) { 3134 return fastEmitInst_r(AArch64::FABSv4f32, &AArch64::FPR128RegClass, Op0); 3135 } 3136 return 0; 3137} 3138 3139unsigned fastEmit_ISD_FABS_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 3140 if (RetVT.SimpleTy != MVT::v2f64) 3141 return 0; 3142 if ((Subtarget->hasNEON())) { 3143 return fastEmitInst_r(AArch64::FABSv2f64, &AArch64::FPR128RegClass, Op0); 3144 } 3145 return 0; 3146} 3147 3148unsigned fastEmit_ISD_FABS_r(MVT VT, MVT RetVT, unsigned Op0) { 3149 switch (VT.SimpleTy) { 3150 case MVT::f16: return fastEmit_ISD_FABS_MVT_f16_r(RetVT, Op0); 3151 case MVT::f32: return fastEmit_ISD_FABS_MVT_f32_r(RetVT, Op0); 3152 case MVT::f64: return fastEmit_ISD_FABS_MVT_f64_r(RetVT, Op0); 3153 case MVT::v4f16: return fastEmit_ISD_FABS_MVT_v4f16_r(RetVT, Op0); 3154 case MVT::v8f16: return fastEmit_ISD_FABS_MVT_v8f16_r(RetVT, Op0); 3155 case MVT::v2f32: return fastEmit_ISD_FABS_MVT_v2f32_r(RetVT, Op0); 3156 case MVT::v4f32: return fastEmit_ISD_FABS_MVT_v4f32_r(RetVT, Op0); 3157 case MVT::v2f64: return fastEmit_ISD_FABS_MVT_v2f64_r(RetVT, Op0); 3158 default: return 0; 3159 } 3160} 3161 3162// FastEmit functions for ISD::FCEIL. 3163 3164unsigned fastEmit_ISD_FCEIL_MVT_f16_r(MVT RetVT, unsigned Op0) { 3165 if (RetVT.SimpleTy != MVT::f16) 3166 return 0; 3167 if ((Subtarget->hasFullFP16())) { 3168 return fastEmitInst_r(AArch64::FRINTPHr, &AArch64::FPR16RegClass, Op0); 3169 } 3170 return 0; 3171} 3172 3173unsigned fastEmit_ISD_FCEIL_MVT_f32_r(MVT RetVT, unsigned Op0) { 3174 if (RetVT.SimpleTy != MVT::f32) 3175 return 0; 3176 if ((Subtarget->hasFPARMv8())) { 3177 return fastEmitInst_r(AArch64::FRINTPSr, &AArch64::FPR32RegClass, Op0); 3178 } 3179 return 0; 3180} 3181 3182unsigned fastEmit_ISD_FCEIL_MVT_f64_r(MVT RetVT, unsigned Op0) { 3183 if (RetVT.SimpleTy != MVT::f64) 3184 return 0; 3185 if ((Subtarget->hasFPARMv8())) { 3186 return fastEmitInst_r(AArch64::FRINTPDr, &AArch64::FPR64RegClass, Op0); 3187 } 3188 return 0; 3189} 3190 3191unsigned fastEmit_ISD_FCEIL_MVT_v4f16_r(MVT RetVT, unsigned Op0) { 3192 if (RetVT.SimpleTy != MVT::v4f16) 3193 return 0; 3194 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 3195 return fastEmitInst_r(AArch64::FRINTPv4f16, &AArch64::FPR64RegClass, Op0); 3196 } 3197 return 0; 3198} 3199 3200unsigned fastEmit_ISD_FCEIL_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 3201 if (RetVT.SimpleTy != MVT::v8f16) 3202 return 0; 3203 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 3204 return fastEmitInst_r(AArch64::FRINTPv8f16, &AArch64::FPR128RegClass, Op0); 3205 } 3206 return 0; 3207} 3208 3209unsigned fastEmit_ISD_FCEIL_MVT_v2f32_r(MVT RetVT, unsigned Op0) { 3210 if (RetVT.SimpleTy != MVT::v2f32) 3211 return 0; 3212 if ((Subtarget->hasNEON())) { 3213 return fastEmitInst_r(AArch64::FRINTPv2f32, &AArch64::FPR64RegClass, Op0); 3214 } 3215 return 0; 3216} 3217 3218unsigned fastEmit_ISD_FCEIL_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 3219 if (RetVT.SimpleTy != MVT::v4f32) 3220 return 0; 3221 if ((Subtarget->hasNEON())) { 3222 return fastEmitInst_r(AArch64::FRINTPv4f32, &AArch64::FPR128RegClass, Op0); 3223 } 3224 return 0; 3225} 3226 3227unsigned fastEmit_ISD_FCEIL_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 3228 if (RetVT.SimpleTy != MVT::v2f64) 3229 return 0; 3230 if ((Subtarget->hasNEON())) { 3231 return fastEmitInst_r(AArch64::FRINTPv2f64, &AArch64::FPR128RegClass, Op0); 3232 } 3233 return 0; 3234} 3235 3236unsigned fastEmit_ISD_FCEIL_r(MVT VT, MVT RetVT, unsigned Op0) { 3237 switch (VT.SimpleTy) { 3238 case MVT::f16: return fastEmit_ISD_FCEIL_MVT_f16_r(RetVT, Op0); 3239 case MVT::f32: return fastEmit_ISD_FCEIL_MVT_f32_r(RetVT, Op0); 3240 case MVT::f64: return fastEmit_ISD_FCEIL_MVT_f64_r(RetVT, Op0); 3241 case MVT::v4f16: return fastEmit_ISD_FCEIL_MVT_v4f16_r(RetVT, Op0); 3242 case MVT::v8f16: return fastEmit_ISD_FCEIL_MVT_v8f16_r(RetVT, Op0); 3243 case MVT::v2f32: return fastEmit_ISD_FCEIL_MVT_v2f32_r(RetVT, Op0); 3244 case MVT::v4f32: return fastEmit_ISD_FCEIL_MVT_v4f32_r(RetVT, Op0); 3245 case MVT::v2f64: return fastEmit_ISD_FCEIL_MVT_v2f64_r(RetVT, Op0); 3246 default: return 0; 3247 } 3248} 3249 3250// FastEmit functions for ISD::FFLOOR. 3251 3252unsigned fastEmit_ISD_FFLOOR_MVT_f16_r(MVT RetVT, unsigned Op0) { 3253 if (RetVT.SimpleTy != MVT::f16) 3254 return 0; 3255 if ((Subtarget->hasFullFP16())) { 3256 return fastEmitInst_r(AArch64::FRINTMHr, &AArch64::FPR16RegClass, Op0); 3257 } 3258 return 0; 3259} 3260 3261unsigned fastEmit_ISD_FFLOOR_MVT_f32_r(MVT RetVT, unsigned Op0) { 3262 if (RetVT.SimpleTy != MVT::f32) 3263 return 0; 3264 if ((Subtarget->hasFPARMv8())) { 3265 return fastEmitInst_r(AArch64::FRINTMSr, &AArch64::FPR32RegClass, Op0); 3266 } 3267 return 0; 3268} 3269 3270unsigned fastEmit_ISD_FFLOOR_MVT_f64_r(MVT RetVT, unsigned Op0) { 3271 if (RetVT.SimpleTy != MVT::f64) 3272 return 0; 3273 if ((Subtarget->hasFPARMv8())) { 3274 return fastEmitInst_r(AArch64::FRINTMDr, &AArch64::FPR64RegClass, Op0); 3275 } 3276 return 0; 3277} 3278 3279unsigned fastEmit_ISD_FFLOOR_MVT_v4f16_r(MVT RetVT, unsigned Op0) { 3280 if (RetVT.SimpleTy != MVT::v4f16) 3281 return 0; 3282 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 3283 return fastEmitInst_r(AArch64::FRINTMv4f16, &AArch64::FPR64RegClass, Op0); 3284 } 3285 return 0; 3286} 3287 3288unsigned fastEmit_ISD_FFLOOR_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 3289 if (RetVT.SimpleTy != MVT::v8f16) 3290 return 0; 3291 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 3292 return fastEmitInst_r(AArch64::FRINTMv8f16, &AArch64::FPR128RegClass, Op0); 3293 } 3294 return 0; 3295} 3296 3297unsigned fastEmit_ISD_FFLOOR_MVT_v2f32_r(MVT RetVT, unsigned Op0) { 3298 if (RetVT.SimpleTy != MVT::v2f32) 3299 return 0; 3300 if ((Subtarget->hasNEON())) { 3301 return fastEmitInst_r(AArch64::FRINTMv2f32, &AArch64::FPR64RegClass, Op0); 3302 } 3303 return 0; 3304} 3305 3306unsigned fastEmit_ISD_FFLOOR_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 3307 if (RetVT.SimpleTy != MVT::v4f32) 3308 return 0; 3309 if ((Subtarget->hasNEON())) { 3310 return fastEmitInst_r(AArch64::FRINTMv4f32, &AArch64::FPR128RegClass, Op0); 3311 } 3312 return 0; 3313} 3314 3315unsigned fastEmit_ISD_FFLOOR_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 3316 if (RetVT.SimpleTy != MVT::v2f64) 3317 return 0; 3318 if ((Subtarget->hasNEON())) { 3319 return fastEmitInst_r(AArch64::FRINTMv2f64, &AArch64::FPR128RegClass, Op0); 3320 } 3321 return 0; 3322} 3323 3324unsigned fastEmit_ISD_FFLOOR_r(MVT VT, MVT RetVT, unsigned Op0) { 3325 switch (VT.SimpleTy) { 3326 case MVT::f16: return fastEmit_ISD_FFLOOR_MVT_f16_r(RetVT, Op0); 3327 case MVT::f32: return fastEmit_ISD_FFLOOR_MVT_f32_r(RetVT, Op0); 3328 case MVT::f64: return fastEmit_ISD_FFLOOR_MVT_f64_r(RetVT, Op0); 3329 case MVT::v4f16: return fastEmit_ISD_FFLOOR_MVT_v4f16_r(RetVT, Op0); 3330 case MVT::v8f16: return fastEmit_ISD_FFLOOR_MVT_v8f16_r(RetVT, Op0); 3331 case MVT::v2f32: return fastEmit_ISD_FFLOOR_MVT_v2f32_r(RetVT, Op0); 3332 case MVT::v4f32: return fastEmit_ISD_FFLOOR_MVT_v4f32_r(RetVT, Op0); 3333 case MVT::v2f64: return fastEmit_ISD_FFLOOR_MVT_v2f64_r(RetVT, Op0); 3334 default: return 0; 3335 } 3336} 3337 3338// FastEmit functions for ISD::FNEARBYINT. 3339 3340unsigned fastEmit_ISD_FNEARBYINT_MVT_f16_r(MVT RetVT, unsigned Op0) { 3341 if (RetVT.SimpleTy != MVT::f16) 3342 return 0; 3343 if ((Subtarget->hasFullFP16())) { 3344 return fastEmitInst_r(AArch64::FRINTIHr, &AArch64::FPR16RegClass, Op0); 3345 } 3346 return 0; 3347} 3348 3349unsigned fastEmit_ISD_FNEARBYINT_MVT_f32_r(MVT RetVT, unsigned Op0) { 3350 if (RetVT.SimpleTy != MVT::f32) 3351 return 0; 3352 if ((Subtarget->hasFPARMv8())) { 3353 return fastEmitInst_r(AArch64::FRINTISr, &AArch64::FPR32RegClass, Op0); 3354 } 3355 return 0; 3356} 3357 3358unsigned fastEmit_ISD_FNEARBYINT_MVT_f64_r(MVT RetVT, unsigned Op0) { 3359 if (RetVT.SimpleTy != MVT::f64) 3360 return 0; 3361 if ((Subtarget->hasFPARMv8())) { 3362 return fastEmitInst_r(AArch64::FRINTIDr, &AArch64::FPR64RegClass, Op0); 3363 } 3364 return 0; 3365} 3366 3367unsigned fastEmit_ISD_FNEARBYINT_MVT_v4f16_r(MVT RetVT, unsigned Op0) { 3368 if (RetVT.SimpleTy != MVT::v4f16) 3369 return 0; 3370 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 3371 return fastEmitInst_r(AArch64::FRINTIv4f16, &AArch64::FPR64RegClass, Op0); 3372 } 3373 return 0; 3374} 3375 3376unsigned fastEmit_ISD_FNEARBYINT_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 3377 if (RetVT.SimpleTy != MVT::v8f16) 3378 return 0; 3379 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 3380 return fastEmitInst_r(AArch64::FRINTIv8f16, &AArch64::FPR128RegClass, Op0); 3381 } 3382 return 0; 3383} 3384 3385unsigned fastEmit_ISD_FNEARBYINT_MVT_v2f32_r(MVT RetVT, unsigned Op0) { 3386 if (RetVT.SimpleTy != MVT::v2f32) 3387 return 0; 3388 if ((Subtarget->hasNEON())) { 3389 return fastEmitInst_r(AArch64::FRINTIv2f32, &AArch64::FPR64RegClass, Op0); 3390 } 3391 return 0; 3392} 3393 3394unsigned fastEmit_ISD_FNEARBYINT_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 3395 if (RetVT.SimpleTy != MVT::v4f32) 3396 return 0; 3397 if ((Subtarget->hasNEON())) { 3398 return fastEmitInst_r(AArch64::FRINTIv4f32, &AArch64::FPR128RegClass, Op0); 3399 } 3400 return 0; 3401} 3402 3403unsigned fastEmit_ISD_FNEARBYINT_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 3404 if (RetVT.SimpleTy != MVT::v2f64) 3405 return 0; 3406 if ((Subtarget->hasNEON())) { 3407 return fastEmitInst_r(AArch64::FRINTIv2f64, &AArch64::FPR128RegClass, Op0); 3408 } 3409 return 0; 3410} 3411 3412unsigned fastEmit_ISD_FNEARBYINT_r(MVT VT, MVT RetVT, unsigned Op0) { 3413 switch (VT.SimpleTy) { 3414 case MVT::f16: return fastEmit_ISD_FNEARBYINT_MVT_f16_r(RetVT, Op0); 3415 case MVT::f32: return fastEmit_ISD_FNEARBYINT_MVT_f32_r(RetVT, Op0); 3416 case MVT::f64: return fastEmit_ISD_FNEARBYINT_MVT_f64_r(RetVT, Op0); 3417 case MVT::v4f16: return fastEmit_ISD_FNEARBYINT_MVT_v4f16_r(RetVT, Op0); 3418 case MVT::v8f16: return fastEmit_ISD_FNEARBYINT_MVT_v8f16_r(RetVT, Op0); 3419 case MVT::v2f32: return fastEmit_ISD_FNEARBYINT_MVT_v2f32_r(RetVT, Op0); 3420 case MVT::v4f32: return fastEmit_ISD_FNEARBYINT_MVT_v4f32_r(RetVT, Op0); 3421 case MVT::v2f64: return fastEmit_ISD_FNEARBYINT_MVT_v2f64_r(RetVT, Op0); 3422 default: return 0; 3423 } 3424} 3425 3426// FastEmit functions for ISD::FNEG. 3427 3428unsigned fastEmit_ISD_FNEG_MVT_f16_r(MVT RetVT, unsigned Op0) { 3429 if (RetVT.SimpleTy != MVT::f16) 3430 return 0; 3431 if ((Subtarget->hasFullFP16())) { 3432 return fastEmitInst_r(AArch64::FNEGHr, &AArch64::FPR16RegClass, Op0); 3433 } 3434 return 0; 3435} 3436 3437unsigned fastEmit_ISD_FNEG_MVT_f32_r(MVT RetVT, unsigned Op0) { 3438 if (RetVT.SimpleTy != MVT::f32) 3439 return 0; 3440 if ((Subtarget->hasFPARMv8())) { 3441 return fastEmitInst_r(AArch64::FNEGSr, &AArch64::FPR32RegClass, Op0); 3442 } 3443 return 0; 3444} 3445 3446unsigned fastEmit_ISD_FNEG_MVT_f64_r(MVT RetVT, unsigned Op0) { 3447 if (RetVT.SimpleTy != MVT::f64) 3448 return 0; 3449 if ((Subtarget->hasFPARMv8())) { 3450 return fastEmitInst_r(AArch64::FNEGDr, &AArch64::FPR64RegClass, Op0); 3451 } 3452 return 0; 3453} 3454 3455unsigned fastEmit_ISD_FNEG_MVT_v4f16_r(MVT RetVT, unsigned Op0) { 3456 if (RetVT.SimpleTy != MVT::v4f16) 3457 return 0; 3458 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 3459 return fastEmitInst_r(AArch64::FNEGv4f16, &AArch64::FPR64RegClass, Op0); 3460 } 3461 return 0; 3462} 3463 3464unsigned fastEmit_ISD_FNEG_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 3465 if (RetVT.SimpleTy != MVT::v8f16) 3466 return 0; 3467 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 3468 return fastEmitInst_r(AArch64::FNEGv8f16, &AArch64::FPR128RegClass, Op0); 3469 } 3470 return 0; 3471} 3472 3473unsigned fastEmit_ISD_FNEG_MVT_v2f32_r(MVT RetVT, unsigned Op0) { 3474 if (RetVT.SimpleTy != MVT::v2f32) 3475 return 0; 3476 if ((Subtarget->hasNEON())) { 3477 return fastEmitInst_r(AArch64::FNEGv2f32, &AArch64::FPR64RegClass, Op0); 3478 } 3479 return 0; 3480} 3481 3482unsigned fastEmit_ISD_FNEG_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 3483 if (RetVT.SimpleTy != MVT::v4f32) 3484 return 0; 3485 if ((Subtarget->hasNEON())) { 3486 return fastEmitInst_r(AArch64::FNEGv4f32, &AArch64::FPR128RegClass, Op0); 3487 } 3488 return 0; 3489} 3490 3491unsigned fastEmit_ISD_FNEG_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 3492 if (RetVT.SimpleTy != MVT::v2f64) 3493 return 0; 3494 if ((Subtarget->hasNEON())) { 3495 return fastEmitInst_r(AArch64::FNEGv2f64, &AArch64::FPR128RegClass, Op0); 3496 } 3497 return 0; 3498} 3499 3500unsigned fastEmit_ISD_FNEG_r(MVT VT, MVT RetVT, unsigned Op0) { 3501 switch (VT.SimpleTy) { 3502 case MVT::f16: return fastEmit_ISD_FNEG_MVT_f16_r(RetVT, Op0); 3503 case MVT::f32: return fastEmit_ISD_FNEG_MVT_f32_r(RetVT, Op0); 3504 case MVT::f64: return fastEmit_ISD_FNEG_MVT_f64_r(RetVT, Op0); 3505 case MVT::v4f16: return fastEmit_ISD_FNEG_MVT_v4f16_r(RetVT, Op0); 3506 case MVT::v8f16: return fastEmit_ISD_FNEG_MVT_v8f16_r(RetVT, Op0); 3507 case MVT::v2f32: return fastEmit_ISD_FNEG_MVT_v2f32_r(RetVT, Op0); 3508 case MVT::v4f32: return fastEmit_ISD_FNEG_MVT_v4f32_r(RetVT, Op0); 3509 case MVT::v2f64: return fastEmit_ISD_FNEG_MVT_v2f64_r(RetVT, Op0); 3510 default: return 0; 3511 } 3512} 3513 3514// FastEmit functions for ISD::FP_EXTEND. 3515 3516unsigned fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f32_r(unsigned Op0) { 3517 if ((Subtarget->hasFPARMv8())) { 3518 return fastEmitInst_r(AArch64::FCVTSHr, &AArch64::FPR32RegClass, Op0); 3519 } 3520 return 0; 3521} 3522 3523unsigned fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f64_r(unsigned Op0) { 3524 if ((Subtarget->hasFPARMv8())) { 3525 return fastEmitInst_r(AArch64::FCVTDHr, &AArch64::FPR64RegClass, Op0); 3526 } 3527 return 0; 3528} 3529 3530unsigned fastEmit_ISD_FP_EXTEND_MVT_f16_r(MVT RetVT, unsigned Op0) { 3531switch (RetVT.SimpleTy) { 3532 case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f32_r(Op0); 3533 case MVT::f64: return fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f64_r(Op0); 3534 default: return 0; 3535} 3536} 3537 3538unsigned fastEmit_ISD_FP_EXTEND_MVT_f32_r(MVT RetVT, unsigned Op0) { 3539 if (RetVT.SimpleTy != MVT::f64) 3540 return 0; 3541 if ((Subtarget->hasFPARMv8())) { 3542 return fastEmitInst_r(AArch64::FCVTDSr, &AArch64::FPR64RegClass, Op0); 3543 } 3544 return 0; 3545} 3546 3547unsigned fastEmit_ISD_FP_EXTEND_MVT_v4f16_r(MVT RetVT, unsigned Op0) { 3548 if (RetVT.SimpleTy != MVT::v4f32) 3549 return 0; 3550 return fastEmitInst_r(AArch64::FCVTLv4i16, &AArch64::FPR128RegClass, Op0); 3551} 3552 3553unsigned fastEmit_ISD_FP_EXTEND_MVT_v2f32_r(MVT RetVT, unsigned Op0) { 3554 if (RetVT.SimpleTy != MVT::v2f64) 3555 return 0; 3556 return fastEmitInst_r(AArch64::FCVTLv2i32, &AArch64::FPR128RegClass, Op0); 3557} 3558 3559unsigned fastEmit_ISD_FP_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0) { 3560 switch (VT.SimpleTy) { 3561 case MVT::f16: return fastEmit_ISD_FP_EXTEND_MVT_f16_r(RetVT, Op0); 3562 case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f32_r(RetVT, Op0); 3563 case MVT::v4f16: return fastEmit_ISD_FP_EXTEND_MVT_v4f16_r(RetVT, Op0); 3564 case MVT::v2f32: return fastEmit_ISD_FP_EXTEND_MVT_v2f32_r(RetVT, Op0); 3565 default: return 0; 3566 } 3567} 3568 3569// FastEmit functions for ISD::FP_ROUND. 3570 3571unsigned fastEmit_ISD_FP_ROUND_MVT_f32_r(MVT RetVT, unsigned Op0) { 3572 if (RetVT.SimpleTy != MVT::f16) 3573 return 0; 3574 if ((Subtarget->hasFPARMv8())) { 3575 return fastEmitInst_r(AArch64::FCVTHSr, &AArch64::FPR16RegClass, Op0); 3576 } 3577 return 0; 3578} 3579 3580unsigned fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f16_r(unsigned Op0) { 3581 if ((Subtarget->hasFPARMv8())) { 3582 return fastEmitInst_r(AArch64::FCVTHDr, &AArch64::FPR16RegClass, Op0); 3583 } 3584 return 0; 3585} 3586 3587unsigned fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f32_r(unsigned Op0) { 3588 if ((Subtarget->hasFPARMv8())) { 3589 return fastEmitInst_r(AArch64::FCVTSDr, &AArch64::FPR32RegClass, Op0); 3590 } 3591 return 0; 3592} 3593 3594unsigned fastEmit_ISD_FP_ROUND_MVT_f64_r(MVT RetVT, unsigned Op0) { 3595switch (RetVT.SimpleTy) { 3596 case MVT::f16: return fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f16_r(Op0); 3597 case MVT::f32: return fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f32_r(Op0); 3598 default: return 0; 3599} 3600} 3601 3602unsigned fastEmit_ISD_FP_ROUND_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 3603 if (RetVT.SimpleTy != MVT::v4f16) 3604 return 0; 3605 return fastEmitInst_r(AArch64::FCVTNv4i16, &AArch64::FPR64RegClass, Op0); 3606} 3607 3608unsigned fastEmit_ISD_FP_ROUND_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 3609 if (RetVT.SimpleTy != MVT::v2f32) 3610 return 0; 3611 return fastEmitInst_r(AArch64::FCVTNv2i32, &AArch64::FPR64RegClass, Op0); 3612} 3613 3614unsigned fastEmit_ISD_FP_ROUND_r(MVT VT, MVT RetVT, unsigned Op0) { 3615 switch (VT.SimpleTy) { 3616 case MVT::f32: return fastEmit_ISD_FP_ROUND_MVT_f32_r(RetVT, Op0); 3617 case MVT::f64: return fastEmit_ISD_FP_ROUND_MVT_f64_r(RetVT, Op0); 3618 case MVT::v4f32: return fastEmit_ISD_FP_ROUND_MVT_v4f32_r(RetVT, Op0); 3619 case MVT::v2f64: return fastEmit_ISD_FP_ROUND_MVT_v2f64_r(RetVT, Op0); 3620 default: return 0; 3621 } 3622} 3623 3624// FastEmit functions for ISD::FP_TO_SINT. 3625 3626unsigned fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i32_r(unsigned Op0) { 3627 if ((Subtarget->hasFullFP16())) { 3628 return fastEmitInst_r(AArch64::FCVTZSUWHr, &AArch64::GPR32RegClass, Op0); 3629 } 3630 return 0; 3631} 3632 3633unsigned fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i64_r(unsigned Op0) { 3634 if ((Subtarget->hasFullFP16())) { 3635 return fastEmitInst_r(AArch64::FCVTZSUXHr, &AArch64::GPR64RegClass, Op0); 3636 } 3637 return 0; 3638} 3639 3640unsigned fastEmit_ISD_FP_TO_SINT_MVT_f16_r(MVT RetVT, unsigned Op0) { 3641switch (RetVT.SimpleTy) { 3642 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i32_r(Op0); 3643 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i64_r(Op0); 3644 default: return 0; 3645} 3646} 3647 3648unsigned fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(unsigned Op0) { 3649 if ((Subtarget->hasFPARMv8())) { 3650 return fastEmitInst_r(AArch64::FCVTZSUWSr, &AArch64::GPR32RegClass, Op0); 3651 } 3652 return 0; 3653} 3654 3655unsigned fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(unsigned Op0) { 3656 if ((Subtarget->hasFPARMv8())) { 3657 return fastEmitInst_r(AArch64::FCVTZSUXSr, &AArch64::GPR64RegClass, Op0); 3658 } 3659 return 0; 3660} 3661 3662unsigned fastEmit_ISD_FP_TO_SINT_MVT_f32_r(MVT RetVT, unsigned Op0) { 3663switch (RetVT.SimpleTy) { 3664 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0); 3665 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0); 3666 default: return 0; 3667} 3668} 3669 3670unsigned fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(unsigned Op0) { 3671 if ((Subtarget->hasFPARMv8())) { 3672 return fastEmitInst_r(AArch64::FCVTZSUWDr, &AArch64::GPR32RegClass, Op0); 3673 } 3674 return 0; 3675} 3676 3677unsigned fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(unsigned Op0) { 3678 if ((Subtarget->hasFPARMv8())) { 3679 return fastEmitInst_r(AArch64::FCVTZSUXDr, &AArch64::GPR64RegClass, Op0); 3680 } 3681 return 0; 3682} 3683 3684unsigned fastEmit_ISD_FP_TO_SINT_MVT_f64_r(MVT RetVT, unsigned Op0) { 3685switch (RetVT.SimpleTy) { 3686 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0); 3687 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0); 3688 default: return 0; 3689} 3690} 3691 3692unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f16_r(MVT RetVT, unsigned Op0) { 3693 if (RetVT.SimpleTy != MVT::v4i16) 3694 return 0; 3695 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 3696 return fastEmitInst_r(AArch64::FCVTZSv4f16, &AArch64::FPR64RegClass, Op0); 3697 } 3698 return 0; 3699} 3700 3701unsigned fastEmit_ISD_FP_TO_SINT_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 3702 if (RetVT.SimpleTy != MVT::v8i16) 3703 return 0; 3704 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 3705 return fastEmitInst_r(AArch64::FCVTZSv8f16, &AArch64::FPR128RegClass, Op0); 3706 } 3707 return 0; 3708} 3709 3710unsigned fastEmit_ISD_FP_TO_SINT_MVT_v2f32_r(MVT RetVT, unsigned Op0) { 3711 if (RetVT.SimpleTy != MVT::v2i32) 3712 return 0; 3713 if ((Subtarget->hasNEON())) { 3714 return fastEmitInst_r(AArch64::FCVTZSv2f32, &AArch64::FPR64RegClass, Op0); 3715 } 3716 return 0; 3717} 3718 3719unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 3720 if (RetVT.SimpleTy != MVT::v4i32) 3721 return 0; 3722 if ((Subtarget->hasNEON())) { 3723 return fastEmitInst_r(AArch64::FCVTZSv4f32, &AArch64::FPR128RegClass, Op0); 3724 } 3725 return 0; 3726} 3727 3728unsigned fastEmit_ISD_FP_TO_SINT_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 3729 if (RetVT.SimpleTy != MVT::v2i64) 3730 return 0; 3731 if ((Subtarget->hasNEON())) { 3732 return fastEmitInst_r(AArch64::FCVTZSv2f64, &AArch64::FPR128RegClass, Op0); 3733 } 3734 return 0; 3735} 3736 3737unsigned fastEmit_ISD_FP_TO_SINT_r(MVT VT, MVT RetVT, unsigned Op0) { 3738 switch (VT.SimpleTy) { 3739 case MVT::f16: return fastEmit_ISD_FP_TO_SINT_MVT_f16_r(RetVT, Op0); 3740 case MVT::f32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_r(RetVT, Op0); 3741 case MVT::f64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_r(RetVT, Op0); 3742 case MVT::v4f16: return fastEmit_ISD_FP_TO_SINT_MVT_v4f16_r(RetVT, Op0); 3743 case MVT::v8f16: return fastEmit_ISD_FP_TO_SINT_MVT_v8f16_r(RetVT, Op0); 3744 case MVT::v2f32: return fastEmit_ISD_FP_TO_SINT_MVT_v2f32_r(RetVT, Op0); 3745 case MVT::v4f32: return fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(RetVT, Op0); 3746 case MVT::v2f64: return fastEmit_ISD_FP_TO_SINT_MVT_v2f64_r(RetVT, Op0); 3747 default: return 0; 3748 } 3749} 3750 3751// FastEmit functions for ISD::FP_TO_UINT. 3752 3753unsigned fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i32_r(unsigned Op0) { 3754 if ((Subtarget->hasFullFP16())) { 3755 return fastEmitInst_r(AArch64::FCVTZUUWHr, &AArch64::GPR32RegClass, Op0); 3756 } 3757 return 0; 3758} 3759 3760unsigned fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i64_r(unsigned Op0) { 3761 if ((Subtarget->hasFullFP16())) { 3762 return fastEmitInst_r(AArch64::FCVTZUUXHr, &AArch64::GPR64RegClass, Op0); 3763 } 3764 return 0; 3765} 3766 3767unsigned fastEmit_ISD_FP_TO_UINT_MVT_f16_r(MVT RetVT, unsigned Op0) { 3768switch (RetVT.SimpleTy) { 3769 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i32_r(Op0); 3770 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i64_r(Op0); 3771 default: return 0; 3772} 3773} 3774 3775unsigned fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(unsigned Op0) { 3776 if ((Subtarget->hasFPARMv8())) { 3777 return fastEmitInst_r(AArch64::FCVTZUUWSr, &AArch64::GPR32RegClass, Op0); 3778 } 3779 return 0; 3780} 3781 3782unsigned fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(unsigned Op0) { 3783 if ((Subtarget->hasFPARMv8())) { 3784 return fastEmitInst_r(AArch64::FCVTZUUXSr, &AArch64::GPR64RegClass, Op0); 3785 } 3786 return 0; 3787} 3788 3789unsigned fastEmit_ISD_FP_TO_UINT_MVT_f32_r(MVT RetVT, unsigned Op0) { 3790switch (RetVT.SimpleTy) { 3791 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0); 3792 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0); 3793 default: return 0; 3794} 3795} 3796 3797unsigned fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(unsigned Op0) { 3798 if ((Subtarget->hasFPARMv8())) { 3799 return fastEmitInst_r(AArch64::FCVTZUUWDr, &AArch64::GPR32RegClass, Op0); 3800 } 3801 return 0; 3802} 3803 3804unsigned fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(unsigned Op0) { 3805 if ((Subtarget->hasFPARMv8())) { 3806 return fastEmitInst_r(AArch64::FCVTZUUXDr, &AArch64::GPR64RegClass, Op0); 3807 } 3808 return 0; 3809} 3810 3811unsigned fastEmit_ISD_FP_TO_UINT_MVT_f64_r(MVT RetVT, unsigned Op0) { 3812switch (RetVT.SimpleTy) { 3813 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0); 3814 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0); 3815 default: return 0; 3816} 3817} 3818 3819unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f16_r(MVT RetVT, unsigned Op0) { 3820 if (RetVT.SimpleTy != MVT::v4i16) 3821 return 0; 3822 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 3823 return fastEmitInst_r(AArch64::FCVTZUv4f16, &AArch64::FPR64RegClass, Op0); 3824 } 3825 return 0; 3826} 3827 3828unsigned fastEmit_ISD_FP_TO_UINT_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 3829 if (RetVT.SimpleTy != MVT::v8i16) 3830 return 0; 3831 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 3832 return fastEmitInst_r(AArch64::FCVTZUv8f16, &AArch64::FPR128RegClass, Op0); 3833 } 3834 return 0; 3835} 3836 3837unsigned fastEmit_ISD_FP_TO_UINT_MVT_v2f32_r(MVT RetVT, unsigned Op0) { 3838 if (RetVT.SimpleTy != MVT::v2i32) 3839 return 0; 3840 if ((Subtarget->hasNEON())) { 3841 return fastEmitInst_r(AArch64::FCVTZUv2f32, &AArch64::FPR64RegClass, Op0); 3842 } 3843 return 0; 3844} 3845 3846unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 3847 if (RetVT.SimpleTy != MVT::v4i32) 3848 return 0; 3849 if ((Subtarget->hasNEON())) { 3850 return fastEmitInst_r(AArch64::FCVTZUv4f32, &AArch64::FPR128RegClass, Op0); 3851 } 3852 return 0; 3853} 3854 3855unsigned fastEmit_ISD_FP_TO_UINT_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 3856 if (RetVT.SimpleTy != MVT::v2i64) 3857 return 0; 3858 if ((Subtarget->hasNEON())) { 3859 return fastEmitInst_r(AArch64::FCVTZUv2f64, &AArch64::FPR128RegClass, Op0); 3860 } 3861 return 0; 3862} 3863 3864unsigned fastEmit_ISD_FP_TO_UINT_r(MVT VT, MVT RetVT, unsigned Op0) { 3865 switch (VT.SimpleTy) { 3866 case MVT::f16: return fastEmit_ISD_FP_TO_UINT_MVT_f16_r(RetVT, Op0); 3867 case MVT::f32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_r(RetVT, Op0); 3868 case MVT::f64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_r(RetVT, Op0); 3869 case MVT::v4f16: return fastEmit_ISD_FP_TO_UINT_MVT_v4f16_r(RetVT, Op0); 3870 case MVT::v8f16: return fastEmit_ISD_FP_TO_UINT_MVT_v8f16_r(RetVT, Op0); 3871 case MVT::v2f32: return fastEmit_ISD_FP_TO_UINT_MVT_v2f32_r(RetVT, Op0); 3872 case MVT::v4f32: return fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(RetVT, Op0); 3873 case MVT::v2f64: return fastEmit_ISD_FP_TO_UINT_MVT_v2f64_r(RetVT, Op0); 3874 default: return 0; 3875 } 3876} 3877 3878// FastEmit functions for ISD::FRINT. 3879 3880unsigned fastEmit_ISD_FRINT_MVT_f16_r(MVT RetVT, unsigned Op0) { 3881 if (RetVT.SimpleTy != MVT::f16) 3882 return 0; 3883 if ((Subtarget->hasFullFP16())) { 3884 return fastEmitInst_r(AArch64::FRINTXHr, &AArch64::FPR16RegClass, Op0); 3885 } 3886 return 0; 3887} 3888 3889unsigned fastEmit_ISD_FRINT_MVT_f32_r(MVT RetVT, unsigned Op0) { 3890 if (RetVT.SimpleTy != MVT::f32) 3891 return 0; 3892 if ((Subtarget->hasFPARMv8())) { 3893 return fastEmitInst_r(AArch64::FRINTXSr, &AArch64::FPR32RegClass, Op0); 3894 } 3895 return 0; 3896} 3897 3898unsigned fastEmit_ISD_FRINT_MVT_f64_r(MVT RetVT, unsigned Op0) { 3899 if (RetVT.SimpleTy != MVT::f64) 3900 return 0; 3901 if ((Subtarget->hasFPARMv8())) { 3902 return fastEmitInst_r(AArch64::FRINTXDr, &AArch64::FPR64RegClass, Op0); 3903 } 3904 return 0; 3905} 3906 3907unsigned fastEmit_ISD_FRINT_MVT_v4f16_r(MVT RetVT, unsigned Op0) { 3908 if (RetVT.SimpleTy != MVT::v4f16) 3909 return 0; 3910 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 3911 return fastEmitInst_r(AArch64::FRINTXv4f16, &AArch64::FPR64RegClass, Op0); 3912 } 3913 return 0; 3914} 3915 3916unsigned fastEmit_ISD_FRINT_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 3917 if (RetVT.SimpleTy != MVT::v8f16) 3918 return 0; 3919 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 3920 return fastEmitInst_r(AArch64::FRINTXv8f16, &AArch64::FPR128RegClass, Op0); 3921 } 3922 return 0; 3923} 3924 3925unsigned fastEmit_ISD_FRINT_MVT_v2f32_r(MVT RetVT, unsigned Op0) { 3926 if (RetVT.SimpleTy != MVT::v2f32) 3927 return 0; 3928 if ((Subtarget->hasNEON())) { 3929 return fastEmitInst_r(AArch64::FRINTXv2f32, &AArch64::FPR64RegClass, Op0); 3930 } 3931 return 0; 3932} 3933 3934unsigned fastEmit_ISD_FRINT_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 3935 if (RetVT.SimpleTy != MVT::v4f32) 3936 return 0; 3937 if ((Subtarget->hasNEON())) { 3938 return fastEmitInst_r(AArch64::FRINTXv4f32, &AArch64::FPR128RegClass, Op0); 3939 } 3940 return 0; 3941} 3942 3943unsigned fastEmit_ISD_FRINT_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 3944 if (RetVT.SimpleTy != MVT::v2f64) 3945 return 0; 3946 if ((Subtarget->hasNEON())) { 3947 return fastEmitInst_r(AArch64::FRINTXv2f64, &AArch64::FPR128RegClass, Op0); 3948 } 3949 return 0; 3950} 3951 3952unsigned fastEmit_ISD_FRINT_r(MVT VT, MVT RetVT, unsigned Op0) { 3953 switch (VT.SimpleTy) { 3954 case MVT::f16: return fastEmit_ISD_FRINT_MVT_f16_r(RetVT, Op0); 3955 case MVT::f32: return fastEmit_ISD_FRINT_MVT_f32_r(RetVT, Op0); 3956 case MVT::f64: return fastEmit_ISD_FRINT_MVT_f64_r(RetVT, Op0); 3957 case MVT::v4f16: return fastEmit_ISD_FRINT_MVT_v4f16_r(RetVT, Op0); 3958 case MVT::v8f16: return fastEmit_ISD_FRINT_MVT_v8f16_r(RetVT, Op0); 3959 case MVT::v2f32: return fastEmit_ISD_FRINT_MVT_v2f32_r(RetVT, Op0); 3960 case MVT::v4f32: return fastEmit_ISD_FRINT_MVT_v4f32_r(RetVT, Op0); 3961 case MVT::v2f64: return fastEmit_ISD_FRINT_MVT_v2f64_r(RetVT, Op0); 3962 default: return 0; 3963 } 3964} 3965 3966// FastEmit functions for ISD::FROUND. 3967 3968unsigned fastEmit_ISD_FROUND_MVT_f16_r(MVT RetVT, unsigned Op0) { 3969 if (RetVT.SimpleTy != MVT::f16) 3970 return 0; 3971 if ((Subtarget->hasFullFP16())) { 3972 return fastEmitInst_r(AArch64::FRINTAHr, &AArch64::FPR16RegClass, Op0); 3973 } 3974 return 0; 3975} 3976 3977unsigned fastEmit_ISD_FROUND_MVT_f32_r(MVT RetVT, unsigned Op0) { 3978 if (RetVT.SimpleTy != MVT::f32) 3979 return 0; 3980 if ((Subtarget->hasFPARMv8())) { 3981 return fastEmitInst_r(AArch64::FRINTASr, &AArch64::FPR32RegClass, Op0); 3982 } 3983 return 0; 3984} 3985 3986unsigned fastEmit_ISD_FROUND_MVT_f64_r(MVT RetVT, unsigned Op0) { 3987 if (RetVT.SimpleTy != MVT::f64) 3988 return 0; 3989 if ((Subtarget->hasFPARMv8())) { 3990 return fastEmitInst_r(AArch64::FRINTADr, &AArch64::FPR64RegClass, Op0); 3991 } 3992 return 0; 3993} 3994 3995unsigned fastEmit_ISD_FROUND_MVT_v4f16_r(MVT RetVT, unsigned Op0) { 3996 if (RetVT.SimpleTy != MVT::v4f16) 3997 return 0; 3998 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 3999 return fastEmitInst_r(AArch64::FRINTAv4f16, &AArch64::FPR64RegClass, Op0); 4000 } 4001 return 0; 4002} 4003 4004unsigned fastEmit_ISD_FROUND_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 4005 if (RetVT.SimpleTy != MVT::v8f16) 4006 return 0; 4007 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 4008 return fastEmitInst_r(AArch64::FRINTAv8f16, &AArch64::FPR128RegClass, Op0); 4009 } 4010 return 0; 4011} 4012 4013unsigned fastEmit_ISD_FROUND_MVT_v2f32_r(MVT RetVT, unsigned Op0) { 4014 if (RetVT.SimpleTy != MVT::v2f32) 4015 return 0; 4016 if ((Subtarget->hasNEON())) { 4017 return fastEmitInst_r(AArch64::FRINTAv2f32, &AArch64::FPR64RegClass, Op0); 4018 } 4019 return 0; 4020} 4021 4022unsigned fastEmit_ISD_FROUND_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 4023 if (RetVT.SimpleTy != MVT::v4f32) 4024 return 0; 4025 if ((Subtarget->hasNEON())) { 4026 return fastEmitInst_r(AArch64::FRINTAv4f32, &AArch64::FPR128RegClass, Op0); 4027 } 4028 return 0; 4029} 4030 4031unsigned fastEmit_ISD_FROUND_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 4032 if (RetVT.SimpleTy != MVT::v2f64) 4033 return 0; 4034 if ((Subtarget->hasNEON())) { 4035 return fastEmitInst_r(AArch64::FRINTAv2f64, &AArch64::FPR128RegClass, Op0); 4036 } 4037 return 0; 4038} 4039 4040unsigned fastEmit_ISD_FROUND_r(MVT VT, MVT RetVT, unsigned Op0) { 4041 switch (VT.SimpleTy) { 4042 case MVT::f16: return fastEmit_ISD_FROUND_MVT_f16_r(RetVT, Op0); 4043 case MVT::f32: return fastEmit_ISD_FROUND_MVT_f32_r(RetVT, Op0); 4044 case MVT::f64: return fastEmit_ISD_FROUND_MVT_f64_r(RetVT, Op0); 4045 case MVT::v4f16: return fastEmit_ISD_FROUND_MVT_v4f16_r(RetVT, Op0); 4046 case MVT::v8f16: return fastEmit_ISD_FROUND_MVT_v8f16_r(RetVT, Op0); 4047 case MVT::v2f32: return fastEmit_ISD_FROUND_MVT_v2f32_r(RetVT, Op0); 4048 case MVT::v4f32: return fastEmit_ISD_FROUND_MVT_v4f32_r(RetVT, Op0); 4049 case MVT::v2f64: return fastEmit_ISD_FROUND_MVT_v2f64_r(RetVT, Op0); 4050 default: return 0; 4051 } 4052} 4053 4054// FastEmit functions for ISD::FROUNDEVEN. 4055 4056unsigned fastEmit_ISD_FROUNDEVEN_MVT_f16_r(MVT RetVT, unsigned Op0) { 4057 if (RetVT.SimpleTy != MVT::f16) 4058 return 0; 4059 if ((Subtarget->hasFullFP16())) { 4060 return fastEmitInst_r(AArch64::FRINTNHr, &AArch64::FPR16RegClass, Op0); 4061 } 4062 return 0; 4063} 4064 4065unsigned fastEmit_ISD_FROUNDEVEN_MVT_f32_r(MVT RetVT, unsigned Op0) { 4066 if (RetVT.SimpleTy != MVT::f32) 4067 return 0; 4068 if ((Subtarget->hasFPARMv8())) { 4069 return fastEmitInst_r(AArch64::FRINTNSr, &AArch64::FPR32RegClass, Op0); 4070 } 4071 return 0; 4072} 4073 4074unsigned fastEmit_ISD_FROUNDEVEN_MVT_f64_r(MVT RetVT, unsigned Op0) { 4075 if (RetVT.SimpleTy != MVT::f64) 4076 return 0; 4077 if ((Subtarget->hasFPARMv8())) { 4078 return fastEmitInst_r(AArch64::FRINTNDr, &AArch64::FPR64RegClass, Op0); 4079 } 4080 return 0; 4081} 4082 4083unsigned fastEmit_ISD_FROUNDEVEN_MVT_v4f16_r(MVT RetVT, unsigned Op0) { 4084 if (RetVT.SimpleTy != MVT::v4f16) 4085 return 0; 4086 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 4087 return fastEmitInst_r(AArch64::FRINTNv4f16, &AArch64::FPR64RegClass, Op0); 4088 } 4089 return 0; 4090} 4091 4092unsigned fastEmit_ISD_FROUNDEVEN_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 4093 if (RetVT.SimpleTy != MVT::v8f16) 4094 return 0; 4095 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 4096 return fastEmitInst_r(AArch64::FRINTNv8f16, &AArch64::FPR128RegClass, Op0); 4097 } 4098 return 0; 4099} 4100 4101unsigned fastEmit_ISD_FROUNDEVEN_MVT_v2f32_r(MVT RetVT, unsigned Op0) { 4102 if (RetVT.SimpleTy != MVT::v2f32) 4103 return 0; 4104 if ((Subtarget->hasNEON())) { 4105 return fastEmitInst_r(AArch64::FRINTNv2f32, &AArch64::FPR64RegClass, Op0); 4106 } 4107 return 0; 4108} 4109 4110unsigned fastEmit_ISD_FROUNDEVEN_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 4111 if (RetVT.SimpleTy != MVT::v4f32) 4112 return 0; 4113 if ((Subtarget->hasNEON())) { 4114 return fastEmitInst_r(AArch64::FRINTNv4f32, &AArch64::FPR128RegClass, Op0); 4115 } 4116 return 0; 4117} 4118 4119unsigned fastEmit_ISD_FROUNDEVEN_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 4120 if (RetVT.SimpleTy != MVT::v2f64) 4121 return 0; 4122 if ((Subtarget->hasNEON())) { 4123 return fastEmitInst_r(AArch64::FRINTNv2f64, &AArch64::FPR128RegClass, Op0); 4124 } 4125 return 0; 4126} 4127 4128unsigned fastEmit_ISD_FROUNDEVEN_r(MVT VT, MVT RetVT, unsigned Op0) { 4129 switch (VT.SimpleTy) { 4130 case MVT::f16: return fastEmit_ISD_FROUNDEVEN_MVT_f16_r(RetVT, Op0); 4131 case MVT::f32: return fastEmit_ISD_FROUNDEVEN_MVT_f32_r(RetVT, Op0); 4132 case MVT::f64: return fastEmit_ISD_FROUNDEVEN_MVT_f64_r(RetVT, Op0); 4133 case MVT::v4f16: return fastEmit_ISD_FROUNDEVEN_MVT_v4f16_r(RetVT, Op0); 4134 case MVT::v8f16: return fastEmit_ISD_FROUNDEVEN_MVT_v8f16_r(RetVT, Op0); 4135 case MVT::v2f32: return fastEmit_ISD_FROUNDEVEN_MVT_v2f32_r(RetVT, Op0); 4136 case MVT::v4f32: return fastEmit_ISD_FROUNDEVEN_MVT_v4f32_r(RetVT, Op0); 4137 case MVT::v2f64: return fastEmit_ISD_FROUNDEVEN_MVT_v2f64_r(RetVT, Op0); 4138 default: return 0; 4139 } 4140} 4141 4142// FastEmit functions for ISD::FSQRT. 4143 4144unsigned fastEmit_ISD_FSQRT_MVT_f16_r(MVT RetVT, unsigned Op0) { 4145 if (RetVT.SimpleTy != MVT::f16) 4146 return 0; 4147 if ((Subtarget->hasFullFP16())) { 4148 return fastEmitInst_r(AArch64::FSQRTHr, &AArch64::FPR16RegClass, Op0); 4149 } 4150 return 0; 4151} 4152 4153unsigned fastEmit_ISD_FSQRT_MVT_f32_r(MVT RetVT, unsigned Op0) { 4154 if (RetVT.SimpleTy != MVT::f32) 4155 return 0; 4156 if ((Subtarget->hasFPARMv8())) { 4157 return fastEmitInst_r(AArch64::FSQRTSr, &AArch64::FPR32RegClass, Op0); 4158 } 4159 return 0; 4160} 4161 4162unsigned fastEmit_ISD_FSQRT_MVT_f64_r(MVT RetVT, unsigned Op0) { 4163 if (RetVT.SimpleTy != MVT::f64) 4164 return 0; 4165 if ((Subtarget->hasFPARMv8())) { 4166 return fastEmitInst_r(AArch64::FSQRTDr, &AArch64::FPR64RegClass, Op0); 4167 } 4168 return 0; 4169} 4170 4171unsigned fastEmit_ISD_FSQRT_MVT_v4f16_r(MVT RetVT, unsigned Op0) { 4172 if (RetVT.SimpleTy != MVT::v4f16) 4173 return 0; 4174 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 4175 return fastEmitInst_r(AArch64::FSQRTv4f16, &AArch64::FPR64RegClass, Op0); 4176 } 4177 return 0; 4178} 4179 4180unsigned fastEmit_ISD_FSQRT_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 4181 if (RetVT.SimpleTy != MVT::v8f16) 4182 return 0; 4183 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 4184 return fastEmitInst_r(AArch64::FSQRTv8f16, &AArch64::FPR128RegClass, Op0); 4185 } 4186 return 0; 4187} 4188 4189unsigned fastEmit_ISD_FSQRT_MVT_v2f32_r(MVT RetVT, unsigned Op0) { 4190 if (RetVT.SimpleTy != MVT::v2f32) 4191 return 0; 4192 if ((Subtarget->hasNEON())) { 4193 return fastEmitInst_r(AArch64::FSQRTv2f32, &AArch64::FPR64RegClass, Op0); 4194 } 4195 return 0; 4196} 4197 4198unsigned fastEmit_ISD_FSQRT_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 4199 if (RetVT.SimpleTy != MVT::v4f32) 4200 return 0; 4201 if ((Subtarget->hasNEON())) { 4202 return fastEmitInst_r(AArch64::FSQRTv4f32, &AArch64::FPR128RegClass, Op0); 4203 } 4204 return 0; 4205} 4206 4207unsigned fastEmit_ISD_FSQRT_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 4208 if (RetVT.SimpleTy != MVT::v2f64) 4209 return 0; 4210 if ((Subtarget->hasNEON())) { 4211 return fastEmitInst_r(AArch64::FSQRTv2f64, &AArch64::FPR128RegClass, Op0); 4212 } 4213 return 0; 4214} 4215 4216unsigned fastEmit_ISD_FSQRT_r(MVT VT, MVT RetVT, unsigned Op0) { 4217 switch (VT.SimpleTy) { 4218 case MVT::f16: return fastEmit_ISD_FSQRT_MVT_f16_r(RetVT, Op0); 4219 case MVT::f32: return fastEmit_ISD_FSQRT_MVT_f32_r(RetVT, Op0); 4220 case MVT::f64: return fastEmit_ISD_FSQRT_MVT_f64_r(RetVT, Op0); 4221 case MVT::v4f16: return fastEmit_ISD_FSQRT_MVT_v4f16_r(RetVT, Op0); 4222 case MVT::v8f16: return fastEmit_ISD_FSQRT_MVT_v8f16_r(RetVT, Op0); 4223 case MVT::v2f32: return fastEmit_ISD_FSQRT_MVT_v2f32_r(RetVT, Op0); 4224 case MVT::v4f32: return fastEmit_ISD_FSQRT_MVT_v4f32_r(RetVT, Op0); 4225 case MVT::v2f64: return fastEmit_ISD_FSQRT_MVT_v2f64_r(RetVT, Op0); 4226 default: return 0; 4227 } 4228} 4229 4230// FastEmit functions for ISD::FTRUNC. 4231 4232unsigned fastEmit_ISD_FTRUNC_MVT_f16_r(MVT RetVT, unsigned Op0) { 4233 if (RetVT.SimpleTy != MVT::f16) 4234 return 0; 4235 if ((Subtarget->hasFullFP16())) { 4236 return fastEmitInst_r(AArch64::FRINTZHr, &AArch64::FPR16RegClass, Op0); 4237 } 4238 return 0; 4239} 4240 4241unsigned fastEmit_ISD_FTRUNC_MVT_f32_r(MVT RetVT, unsigned Op0) { 4242 if (RetVT.SimpleTy != MVT::f32) 4243 return 0; 4244 if ((Subtarget->hasFPARMv8())) { 4245 return fastEmitInst_r(AArch64::FRINTZSr, &AArch64::FPR32RegClass, Op0); 4246 } 4247 return 0; 4248} 4249 4250unsigned fastEmit_ISD_FTRUNC_MVT_f64_r(MVT RetVT, unsigned Op0) { 4251 if (RetVT.SimpleTy != MVT::f64) 4252 return 0; 4253 if ((Subtarget->hasFPARMv8())) { 4254 return fastEmitInst_r(AArch64::FRINTZDr, &AArch64::FPR64RegClass, Op0); 4255 } 4256 return 0; 4257} 4258 4259unsigned fastEmit_ISD_FTRUNC_MVT_v4f16_r(MVT RetVT, unsigned Op0) { 4260 if (RetVT.SimpleTy != MVT::v4f16) 4261 return 0; 4262 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 4263 return fastEmitInst_r(AArch64::FRINTZv4f16, &AArch64::FPR64RegClass, Op0); 4264 } 4265 return 0; 4266} 4267 4268unsigned fastEmit_ISD_FTRUNC_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 4269 if (RetVT.SimpleTy != MVT::v8f16) 4270 return 0; 4271 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 4272 return fastEmitInst_r(AArch64::FRINTZv8f16, &AArch64::FPR128RegClass, Op0); 4273 } 4274 return 0; 4275} 4276 4277unsigned fastEmit_ISD_FTRUNC_MVT_v2f32_r(MVT RetVT, unsigned Op0) { 4278 if (RetVT.SimpleTy != MVT::v2f32) 4279 return 0; 4280 if ((Subtarget->hasNEON())) { 4281 return fastEmitInst_r(AArch64::FRINTZv2f32, &AArch64::FPR64RegClass, Op0); 4282 } 4283 return 0; 4284} 4285 4286unsigned fastEmit_ISD_FTRUNC_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 4287 if (RetVT.SimpleTy != MVT::v4f32) 4288 return 0; 4289 if ((Subtarget->hasNEON())) { 4290 return fastEmitInst_r(AArch64::FRINTZv4f32, &AArch64::FPR128RegClass, Op0); 4291 } 4292 return 0; 4293} 4294 4295unsigned fastEmit_ISD_FTRUNC_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 4296 if (RetVT.SimpleTy != MVT::v2f64) 4297 return 0; 4298 if ((Subtarget->hasNEON())) { 4299 return fastEmitInst_r(AArch64::FRINTZv2f64, &AArch64::FPR128RegClass, Op0); 4300 } 4301 return 0; 4302} 4303 4304unsigned fastEmit_ISD_FTRUNC_r(MVT VT, MVT RetVT, unsigned Op0) { 4305 switch (VT.SimpleTy) { 4306 case MVT::f16: return fastEmit_ISD_FTRUNC_MVT_f16_r(RetVT, Op0); 4307 case MVT::f32: return fastEmit_ISD_FTRUNC_MVT_f32_r(RetVT, Op0); 4308 case MVT::f64: return fastEmit_ISD_FTRUNC_MVT_f64_r(RetVT, Op0); 4309 case MVT::v4f16: return fastEmit_ISD_FTRUNC_MVT_v4f16_r(RetVT, Op0); 4310 case MVT::v8f16: return fastEmit_ISD_FTRUNC_MVT_v8f16_r(RetVT, Op0); 4311 case MVT::v2f32: return fastEmit_ISD_FTRUNC_MVT_v2f32_r(RetVT, Op0); 4312 case MVT::v4f32: return fastEmit_ISD_FTRUNC_MVT_v4f32_r(RetVT, Op0); 4313 case MVT::v2f64: return fastEmit_ISD_FTRUNC_MVT_v2f64_r(RetVT, Op0); 4314 default: return 0; 4315 } 4316} 4317 4318// FastEmit functions for ISD::LLROUND. 4319 4320unsigned fastEmit_ISD_LLROUND_MVT_f16_r(MVT RetVT, unsigned Op0) { 4321 if (RetVT.SimpleTy != MVT::i64) 4322 return 0; 4323 if ((Subtarget->hasFullFP16())) { 4324 return fastEmitInst_r(AArch64::FCVTASUXHr, &AArch64::GPR64RegClass, Op0); 4325 } 4326 return 0; 4327} 4328 4329unsigned fastEmit_ISD_LLROUND_MVT_f32_r(MVT RetVT, unsigned Op0) { 4330 if (RetVT.SimpleTy != MVT::i64) 4331 return 0; 4332 return fastEmitInst_r(AArch64::FCVTASUXSr, &AArch64::GPR64RegClass, Op0); 4333} 4334 4335unsigned fastEmit_ISD_LLROUND_MVT_f64_r(MVT RetVT, unsigned Op0) { 4336 if (RetVT.SimpleTy != MVT::i64) 4337 return 0; 4338 return fastEmitInst_r(AArch64::FCVTASUXDr, &AArch64::GPR64RegClass, Op0); 4339} 4340 4341unsigned fastEmit_ISD_LLROUND_r(MVT VT, MVT RetVT, unsigned Op0) { 4342 switch (VT.SimpleTy) { 4343 case MVT::f16: return fastEmit_ISD_LLROUND_MVT_f16_r(RetVT, Op0); 4344 case MVT::f32: return fastEmit_ISD_LLROUND_MVT_f32_r(RetVT, Op0); 4345 case MVT::f64: return fastEmit_ISD_LLROUND_MVT_f64_r(RetVT, Op0); 4346 default: return 0; 4347 } 4348} 4349 4350// FastEmit functions for ISD::LROUND. 4351 4352unsigned fastEmit_ISD_LROUND_MVT_f16_MVT_i32_r(unsigned Op0) { 4353 if ((Subtarget->hasFullFP16())) { 4354 return fastEmitInst_r(AArch64::FCVTASUWHr, &AArch64::GPR32RegClass, Op0); 4355 } 4356 return 0; 4357} 4358 4359unsigned fastEmit_ISD_LROUND_MVT_f16_MVT_i64_r(unsigned Op0) { 4360 if ((Subtarget->hasFullFP16())) { 4361 return fastEmitInst_r(AArch64::FCVTASUXHr, &AArch64::GPR64RegClass, Op0); 4362 } 4363 return 0; 4364} 4365 4366unsigned fastEmit_ISD_LROUND_MVT_f16_r(MVT RetVT, unsigned Op0) { 4367switch (RetVT.SimpleTy) { 4368 case MVT::i32: return fastEmit_ISD_LROUND_MVT_f16_MVT_i32_r(Op0); 4369 case MVT::i64: return fastEmit_ISD_LROUND_MVT_f16_MVT_i64_r(Op0); 4370 default: return 0; 4371} 4372} 4373 4374unsigned fastEmit_ISD_LROUND_MVT_f32_MVT_i32_r(unsigned Op0) { 4375 return fastEmitInst_r(AArch64::FCVTASUWSr, &AArch64::GPR32RegClass, Op0); 4376} 4377 4378unsigned fastEmit_ISD_LROUND_MVT_f32_MVT_i64_r(unsigned Op0) { 4379 return fastEmitInst_r(AArch64::FCVTASUXSr, &AArch64::GPR64RegClass, Op0); 4380} 4381 4382unsigned fastEmit_ISD_LROUND_MVT_f32_r(MVT RetVT, unsigned Op0) { 4383switch (RetVT.SimpleTy) { 4384 case MVT::i32: return fastEmit_ISD_LROUND_MVT_f32_MVT_i32_r(Op0); 4385 case MVT::i64: return fastEmit_ISD_LROUND_MVT_f32_MVT_i64_r(Op0); 4386 default: return 0; 4387} 4388} 4389 4390unsigned fastEmit_ISD_LROUND_MVT_f64_MVT_i32_r(unsigned Op0) { 4391 return fastEmitInst_r(AArch64::FCVTASUWDr, &AArch64::GPR32RegClass, Op0); 4392} 4393 4394unsigned fastEmit_ISD_LROUND_MVT_f64_MVT_i64_r(unsigned Op0) { 4395 return fastEmitInst_r(AArch64::FCVTASUXDr, &AArch64::GPR64RegClass, Op0); 4396} 4397 4398unsigned fastEmit_ISD_LROUND_MVT_f64_r(MVT RetVT, unsigned Op0) { 4399switch (RetVT.SimpleTy) { 4400 case MVT::i32: return fastEmit_ISD_LROUND_MVT_f64_MVT_i32_r(Op0); 4401 case MVT::i64: return fastEmit_ISD_LROUND_MVT_f64_MVT_i64_r(Op0); 4402 default: return 0; 4403} 4404} 4405 4406unsigned fastEmit_ISD_LROUND_r(MVT VT, MVT RetVT, unsigned Op0) { 4407 switch (VT.SimpleTy) { 4408 case MVT::f16: return fastEmit_ISD_LROUND_MVT_f16_r(RetVT, Op0); 4409 case MVT::f32: return fastEmit_ISD_LROUND_MVT_f32_r(RetVT, Op0); 4410 case MVT::f64: return fastEmit_ISD_LROUND_MVT_f64_r(RetVT, Op0); 4411 default: return 0; 4412 } 4413} 4414 4415// FastEmit functions for ISD::SINT_TO_FP. 4416 4417unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f16_r(unsigned Op0) { 4418 if ((Subtarget->hasFullFP16())) { 4419 return fastEmitInst_r(AArch64::SCVTFUWHri, &AArch64::FPR16RegClass, Op0); 4420 } 4421 return 0; 4422} 4423 4424unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(unsigned Op0) { 4425 if ((Subtarget->hasFPARMv8())) { 4426 return fastEmitInst_r(AArch64::SCVTFUWSri, &AArch64::FPR32RegClass, Op0); 4427 } 4428 return 0; 4429} 4430 4431unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(unsigned Op0) { 4432 if ((Subtarget->hasFPARMv8())) { 4433 return fastEmitInst_r(AArch64::SCVTFUWDri, &AArch64::FPR64RegClass, Op0); 4434 } 4435 return 0; 4436} 4437 4438unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_r(MVT RetVT, unsigned Op0) { 4439switch (RetVT.SimpleTy) { 4440 case MVT::f16: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f16_r(Op0); 4441 case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0); 4442 case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0); 4443 default: return 0; 4444} 4445} 4446 4447unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f16_r(unsigned Op0) { 4448 if ((Subtarget->hasFullFP16())) { 4449 return fastEmitInst_r(AArch64::SCVTFUXHri, &AArch64::FPR16RegClass, Op0); 4450 } 4451 return 0; 4452} 4453 4454unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(unsigned Op0) { 4455 if ((Subtarget->hasFPARMv8())) { 4456 return fastEmitInst_r(AArch64::SCVTFUXSri, &AArch64::FPR32RegClass, Op0); 4457 } 4458 return 0; 4459} 4460 4461unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(unsigned Op0) { 4462 if ((Subtarget->hasFPARMv8())) { 4463 return fastEmitInst_r(AArch64::SCVTFUXDri, &AArch64::FPR64RegClass, Op0); 4464 } 4465 return 0; 4466} 4467 4468unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_r(MVT RetVT, unsigned Op0) { 4469switch (RetVT.SimpleTy) { 4470 case MVT::f16: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f16_r(Op0); 4471 case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0); 4472 case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0); 4473 default: return 0; 4474} 4475} 4476 4477unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i16_r(MVT RetVT, unsigned Op0) { 4478 if (RetVT.SimpleTy != MVT::v4f16) 4479 return 0; 4480 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 4481 return fastEmitInst_r(AArch64::SCVTFv4f16, &AArch64::FPR64RegClass, Op0); 4482 } 4483 return 0; 4484} 4485 4486unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(MVT RetVT, unsigned Op0) { 4487 if (RetVT.SimpleTy != MVT::v8f16) 4488 return 0; 4489 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 4490 return fastEmitInst_r(AArch64::SCVTFv8f16, &AArch64::FPR128RegClass, Op0); 4491 } 4492 return 0; 4493} 4494 4495unsigned fastEmit_ISD_SINT_TO_FP_MVT_v2i32_r(MVT RetVT, unsigned Op0) { 4496 if (RetVT.SimpleTy != MVT::v2f32) 4497 return 0; 4498 if ((Subtarget->hasNEON())) { 4499 return fastEmitInst_r(AArch64::SCVTFv2f32, &AArch64::FPR64RegClass, Op0); 4500 } 4501 return 0; 4502} 4503 4504unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0) { 4505 if (RetVT.SimpleTy != MVT::v4f32) 4506 return 0; 4507 if ((Subtarget->hasNEON())) { 4508 return fastEmitInst_r(AArch64::SCVTFv4f32, &AArch64::FPR128RegClass, Op0); 4509 } 4510 return 0; 4511} 4512 4513unsigned fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, unsigned Op0) { 4514 if (RetVT.SimpleTy != MVT::v2f64) 4515 return 0; 4516 if ((Subtarget->hasNEON())) { 4517 return fastEmitInst_r(AArch64::SCVTFv2f64, &AArch64::FPR128RegClass, Op0); 4518 } 4519 return 0; 4520} 4521 4522unsigned fastEmit_ISD_SINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0) { 4523 switch (VT.SimpleTy) { 4524 case MVT::i32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_r(RetVT, Op0); 4525 case MVT::i64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_r(RetVT, Op0); 4526 case MVT::v4i16: return fastEmit_ISD_SINT_TO_FP_MVT_v4i16_r(RetVT, Op0); 4527 case MVT::v8i16: return fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(RetVT, Op0); 4528 case MVT::v2i32: return fastEmit_ISD_SINT_TO_FP_MVT_v2i32_r(RetVT, Op0); 4529 case MVT::v4i32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0); 4530 case MVT::v2i64: return fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0); 4531 default: return 0; 4532 } 4533} 4534 4535// FastEmit functions for ISD::SPLAT_VECTOR. 4536 4537unsigned fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv16i8_r(unsigned Op0) { 4538 if ((Subtarget->hasSVEorSME())) { 4539 return fastEmitInst_r(AArch64::DUP_ZR_B, &AArch64::ZPRRegClass, Op0); 4540 } 4541 return 0; 4542} 4543 4544unsigned fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv8i16_r(unsigned Op0) { 4545 if ((Subtarget->hasSVEorSME())) { 4546 return fastEmitInst_r(AArch64::DUP_ZR_H, &AArch64::ZPRRegClass, Op0); 4547 } 4548 return 0; 4549} 4550 4551unsigned fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv4i32_r(unsigned Op0) { 4552 if ((Subtarget->hasSVEorSME())) { 4553 return fastEmitInst_r(AArch64::DUP_ZR_S, &AArch64::ZPRRegClass, Op0); 4554 } 4555 return 0; 4556} 4557 4558unsigned fastEmit_ISD_SPLAT_VECTOR_MVT_i32_r(MVT RetVT, unsigned Op0) { 4559switch (RetVT.SimpleTy) { 4560 case MVT::nxv16i8: return fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv16i8_r(Op0); 4561 case MVT::nxv8i16: return fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv8i16_r(Op0); 4562 case MVT::nxv4i32: return fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv4i32_r(Op0); 4563 default: return 0; 4564} 4565} 4566 4567unsigned fastEmit_ISD_SPLAT_VECTOR_MVT_i64_r(MVT RetVT, unsigned Op0) { 4568 if (RetVT.SimpleTy != MVT::nxv2i64) 4569 return 0; 4570 if ((Subtarget->hasSVEorSME())) { 4571 return fastEmitInst_r(AArch64::DUP_ZR_D, &AArch64::ZPRRegClass, Op0); 4572 } 4573 return 0; 4574} 4575 4576unsigned fastEmit_ISD_SPLAT_VECTOR_r(MVT VT, MVT RetVT, unsigned Op0) { 4577 switch (VT.SimpleTy) { 4578 case MVT::i32: return fastEmit_ISD_SPLAT_VECTOR_MVT_i32_r(RetVT, Op0); 4579 case MVT::i64: return fastEmit_ISD_SPLAT_VECTOR_MVT_i64_r(RetVT, Op0); 4580 default: return 0; 4581 } 4582} 4583 4584// FastEmit functions for ISD::STRICT_FCEIL. 4585 4586unsigned fastEmit_ISD_STRICT_FCEIL_MVT_f16_r(MVT RetVT, unsigned Op0) { 4587 if (RetVT.SimpleTy != MVT::f16) 4588 return 0; 4589 if ((Subtarget->hasFullFP16())) { 4590 return fastEmitInst_r(AArch64::FRINTPHr, &AArch64::FPR16RegClass, Op0); 4591 } 4592 return 0; 4593} 4594 4595unsigned fastEmit_ISD_STRICT_FCEIL_MVT_f32_r(MVT RetVT, unsigned Op0) { 4596 if (RetVT.SimpleTy != MVT::f32) 4597 return 0; 4598 if ((Subtarget->hasFPARMv8())) { 4599 return fastEmitInst_r(AArch64::FRINTPSr, &AArch64::FPR32RegClass, Op0); 4600 } 4601 return 0; 4602} 4603 4604unsigned fastEmit_ISD_STRICT_FCEIL_MVT_f64_r(MVT RetVT, unsigned Op0) { 4605 if (RetVT.SimpleTy != MVT::f64) 4606 return 0; 4607 if ((Subtarget->hasFPARMv8())) { 4608 return fastEmitInst_r(AArch64::FRINTPDr, &AArch64::FPR64RegClass, Op0); 4609 } 4610 return 0; 4611} 4612 4613unsigned fastEmit_ISD_STRICT_FCEIL_MVT_v4f16_r(MVT RetVT, unsigned Op0) { 4614 if (RetVT.SimpleTy != MVT::v4f16) 4615 return 0; 4616 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 4617 return fastEmitInst_r(AArch64::FRINTPv4f16, &AArch64::FPR64RegClass, Op0); 4618 } 4619 return 0; 4620} 4621 4622unsigned fastEmit_ISD_STRICT_FCEIL_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 4623 if (RetVT.SimpleTy != MVT::v8f16) 4624 return 0; 4625 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 4626 return fastEmitInst_r(AArch64::FRINTPv8f16, &AArch64::FPR128RegClass, Op0); 4627 } 4628 return 0; 4629} 4630 4631unsigned fastEmit_ISD_STRICT_FCEIL_MVT_v2f32_r(MVT RetVT, unsigned Op0) { 4632 if (RetVT.SimpleTy != MVT::v2f32) 4633 return 0; 4634 if ((Subtarget->hasNEON())) { 4635 return fastEmitInst_r(AArch64::FRINTPv2f32, &AArch64::FPR64RegClass, Op0); 4636 } 4637 return 0; 4638} 4639 4640unsigned fastEmit_ISD_STRICT_FCEIL_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 4641 if (RetVT.SimpleTy != MVT::v4f32) 4642 return 0; 4643 if ((Subtarget->hasNEON())) { 4644 return fastEmitInst_r(AArch64::FRINTPv4f32, &AArch64::FPR128RegClass, Op0); 4645 } 4646 return 0; 4647} 4648 4649unsigned fastEmit_ISD_STRICT_FCEIL_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 4650 if (RetVT.SimpleTy != MVT::v2f64) 4651 return 0; 4652 if ((Subtarget->hasNEON())) { 4653 return fastEmitInst_r(AArch64::FRINTPv2f64, &AArch64::FPR128RegClass, Op0); 4654 } 4655 return 0; 4656} 4657 4658unsigned fastEmit_ISD_STRICT_FCEIL_r(MVT VT, MVT RetVT, unsigned Op0) { 4659 switch (VT.SimpleTy) { 4660 case MVT::f16: return fastEmit_ISD_STRICT_FCEIL_MVT_f16_r(RetVT, Op0); 4661 case MVT::f32: return fastEmit_ISD_STRICT_FCEIL_MVT_f32_r(RetVT, Op0); 4662 case MVT::f64: return fastEmit_ISD_STRICT_FCEIL_MVT_f64_r(RetVT, Op0); 4663 case MVT::v4f16: return fastEmit_ISD_STRICT_FCEIL_MVT_v4f16_r(RetVT, Op0); 4664 case MVT::v8f16: return fastEmit_ISD_STRICT_FCEIL_MVT_v8f16_r(RetVT, Op0); 4665 case MVT::v2f32: return fastEmit_ISD_STRICT_FCEIL_MVT_v2f32_r(RetVT, Op0); 4666 case MVT::v4f32: return fastEmit_ISD_STRICT_FCEIL_MVT_v4f32_r(RetVT, Op0); 4667 case MVT::v2f64: return fastEmit_ISD_STRICT_FCEIL_MVT_v2f64_r(RetVT, Op0); 4668 default: return 0; 4669 } 4670} 4671 4672// FastEmit functions for ISD::STRICT_FFLOOR. 4673 4674unsigned fastEmit_ISD_STRICT_FFLOOR_MVT_f16_r(MVT RetVT, unsigned Op0) { 4675 if (RetVT.SimpleTy != MVT::f16) 4676 return 0; 4677 if ((Subtarget->hasFullFP16())) { 4678 return fastEmitInst_r(AArch64::FRINTMHr, &AArch64::FPR16RegClass, Op0); 4679 } 4680 return 0; 4681} 4682 4683unsigned fastEmit_ISD_STRICT_FFLOOR_MVT_f32_r(MVT RetVT, unsigned Op0) { 4684 if (RetVT.SimpleTy != MVT::f32) 4685 return 0; 4686 if ((Subtarget->hasFPARMv8())) { 4687 return fastEmitInst_r(AArch64::FRINTMSr, &AArch64::FPR32RegClass, Op0); 4688 } 4689 return 0; 4690} 4691 4692unsigned fastEmit_ISD_STRICT_FFLOOR_MVT_f64_r(MVT RetVT, unsigned Op0) { 4693 if (RetVT.SimpleTy != MVT::f64) 4694 return 0; 4695 if ((Subtarget->hasFPARMv8())) { 4696 return fastEmitInst_r(AArch64::FRINTMDr, &AArch64::FPR64RegClass, Op0); 4697 } 4698 return 0; 4699} 4700 4701unsigned fastEmit_ISD_STRICT_FFLOOR_MVT_v4f16_r(MVT RetVT, unsigned Op0) { 4702 if (RetVT.SimpleTy != MVT::v4f16) 4703 return 0; 4704 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 4705 return fastEmitInst_r(AArch64::FRINTMv4f16, &AArch64::FPR64RegClass, Op0); 4706 } 4707 return 0; 4708} 4709 4710unsigned fastEmit_ISD_STRICT_FFLOOR_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 4711 if (RetVT.SimpleTy != MVT::v8f16) 4712 return 0; 4713 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 4714 return fastEmitInst_r(AArch64::FRINTMv8f16, &AArch64::FPR128RegClass, Op0); 4715 } 4716 return 0; 4717} 4718 4719unsigned fastEmit_ISD_STRICT_FFLOOR_MVT_v2f32_r(MVT RetVT, unsigned Op0) { 4720 if (RetVT.SimpleTy != MVT::v2f32) 4721 return 0; 4722 if ((Subtarget->hasNEON())) { 4723 return fastEmitInst_r(AArch64::FRINTMv2f32, &AArch64::FPR64RegClass, Op0); 4724 } 4725 return 0; 4726} 4727 4728unsigned fastEmit_ISD_STRICT_FFLOOR_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 4729 if (RetVT.SimpleTy != MVT::v4f32) 4730 return 0; 4731 if ((Subtarget->hasNEON())) { 4732 return fastEmitInst_r(AArch64::FRINTMv4f32, &AArch64::FPR128RegClass, Op0); 4733 } 4734 return 0; 4735} 4736 4737unsigned fastEmit_ISD_STRICT_FFLOOR_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 4738 if (RetVT.SimpleTy != MVT::v2f64) 4739 return 0; 4740 if ((Subtarget->hasNEON())) { 4741 return fastEmitInst_r(AArch64::FRINTMv2f64, &AArch64::FPR128RegClass, Op0); 4742 } 4743 return 0; 4744} 4745 4746unsigned fastEmit_ISD_STRICT_FFLOOR_r(MVT VT, MVT RetVT, unsigned Op0) { 4747 switch (VT.SimpleTy) { 4748 case MVT::f16: return fastEmit_ISD_STRICT_FFLOOR_MVT_f16_r(RetVT, Op0); 4749 case MVT::f32: return fastEmit_ISD_STRICT_FFLOOR_MVT_f32_r(RetVT, Op0); 4750 case MVT::f64: return fastEmit_ISD_STRICT_FFLOOR_MVT_f64_r(RetVT, Op0); 4751 case MVT::v4f16: return fastEmit_ISD_STRICT_FFLOOR_MVT_v4f16_r(RetVT, Op0); 4752 case MVT::v8f16: return fastEmit_ISD_STRICT_FFLOOR_MVT_v8f16_r(RetVT, Op0); 4753 case MVT::v2f32: return fastEmit_ISD_STRICT_FFLOOR_MVT_v2f32_r(RetVT, Op0); 4754 case MVT::v4f32: return fastEmit_ISD_STRICT_FFLOOR_MVT_v4f32_r(RetVT, Op0); 4755 case MVT::v2f64: return fastEmit_ISD_STRICT_FFLOOR_MVT_v2f64_r(RetVT, Op0); 4756 default: return 0; 4757 } 4758} 4759 4760// FastEmit functions for ISD::STRICT_FNEARBYINT. 4761 4762unsigned fastEmit_ISD_STRICT_FNEARBYINT_MVT_f16_r(MVT RetVT, unsigned Op0) { 4763 if (RetVT.SimpleTy != MVT::f16) 4764 return 0; 4765 if ((Subtarget->hasFullFP16())) { 4766 return fastEmitInst_r(AArch64::FRINTIHr, &AArch64::FPR16RegClass, Op0); 4767 } 4768 return 0; 4769} 4770 4771unsigned fastEmit_ISD_STRICT_FNEARBYINT_MVT_f32_r(MVT RetVT, unsigned Op0) { 4772 if (RetVT.SimpleTy != MVT::f32) 4773 return 0; 4774 if ((Subtarget->hasFPARMv8())) { 4775 return fastEmitInst_r(AArch64::FRINTISr, &AArch64::FPR32RegClass, Op0); 4776 } 4777 return 0; 4778} 4779 4780unsigned fastEmit_ISD_STRICT_FNEARBYINT_MVT_f64_r(MVT RetVT, unsigned Op0) { 4781 if (RetVT.SimpleTy != MVT::f64) 4782 return 0; 4783 if ((Subtarget->hasFPARMv8())) { 4784 return fastEmitInst_r(AArch64::FRINTIDr, &AArch64::FPR64RegClass, Op0); 4785 } 4786 return 0; 4787} 4788 4789unsigned fastEmit_ISD_STRICT_FNEARBYINT_MVT_v4f16_r(MVT RetVT, unsigned Op0) { 4790 if (RetVT.SimpleTy != MVT::v4f16) 4791 return 0; 4792 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 4793 return fastEmitInst_r(AArch64::FRINTIv4f16, &AArch64::FPR64RegClass, Op0); 4794 } 4795 return 0; 4796} 4797 4798unsigned fastEmit_ISD_STRICT_FNEARBYINT_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 4799 if (RetVT.SimpleTy != MVT::v8f16) 4800 return 0; 4801 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 4802 return fastEmitInst_r(AArch64::FRINTIv8f16, &AArch64::FPR128RegClass, Op0); 4803 } 4804 return 0; 4805} 4806 4807unsigned fastEmit_ISD_STRICT_FNEARBYINT_MVT_v2f32_r(MVT RetVT, unsigned Op0) { 4808 if (RetVT.SimpleTy != MVT::v2f32) 4809 return 0; 4810 if ((Subtarget->hasNEON())) { 4811 return fastEmitInst_r(AArch64::FRINTIv2f32, &AArch64::FPR64RegClass, Op0); 4812 } 4813 return 0; 4814} 4815 4816unsigned fastEmit_ISD_STRICT_FNEARBYINT_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 4817 if (RetVT.SimpleTy != MVT::v4f32) 4818 return 0; 4819 if ((Subtarget->hasNEON())) { 4820 return fastEmitInst_r(AArch64::FRINTIv4f32, &AArch64::FPR128RegClass, Op0); 4821 } 4822 return 0; 4823} 4824 4825unsigned fastEmit_ISD_STRICT_FNEARBYINT_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 4826 if (RetVT.SimpleTy != MVT::v2f64) 4827 return 0; 4828 if ((Subtarget->hasNEON())) { 4829 return fastEmitInst_r(AArch64::FRINTIv2f64, &AArch64::FPR128RegClass, Op0); 4830 } 4831 return 0; 4832} 4833 4834unsigned fastEmit_ISD_STRICT_FNEARBYINT_r(MVT VT, MVT RetVT, unsigned Op0) { 4835 switch (VT.SimpleTy) { 4836 case MVT::f16: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_f16_r(RetVT, Op0); 4837 case MVT::f32: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_f32_r(RetVT, Op0); 4838 case MVT::f64: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_f64_r(RetVT, Op0); 4839 case MVT::v4f16: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v4f16_r(RetVT, Op0); 4840 case MVT::v8f16: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v8f16_r(RetVT, Op0); 4841 case MVT::v2f32: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v2f32_r(RetVT, Op0); 4842 case MVT::v4f32: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v4f32_r(RetVT, Op0); 4843 case MVT::v2f64: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v2f64_r(RetVT, Op0); 4844 default: return 0; 4845 } 4846} 4847 4848// FastEmit functions for ISD::STRICT_FP_EXTEND. 4849 4850unsigned fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_MVT_f32_r(unsigned Op0) { 4851 if ((Subtarget->hasFPARMv8())) { 4852 return fastEmitInst_r(AArch64::FCVTSHr, &AArch64::FPR32RegClass, Op0); 4853 } 4854 return 0; 4855} 4856 4857unsigned fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_MVT_f64_r(unsigned Op0) { 4858 if ((Subtarget->hasFPARMv8())) { 4859 return fastEmitInst_r(AArch64::FCVTDHr, &AArch64::FPR64RegClass, Op0); 4860 } 4861 return 0; 4862} 4863 4864unsigned fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_r(MVT RetVT, unsigned Op0) { 4865switch (RetVT.SimpleTy) { 4866 case MVT::f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_MVT_f32_r(Op0); 4867 case MVT::f64: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_MVT_f64_r(Op0); 4868 default: return 0; 4869} 4870} 4871 4872unsigned fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(MVT RetVT, unsigned Op0) { 4873 if (RetVT.SimpleTy != MVT::f64) 4874 return 0; 4875 if ((Subtarget->hasFPARMv8())) { 4876 return fastEmitInst_r(AArch64::FCVTDSr, &AArch64::FPR64RegClass, Op0); 4877 } 4878 return 0; 4879} 4880 4881unsigned fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4f16_r(MVT RetVT, unsigned Op0) { 4882 if (RetVT.SimpleTy != MVT::v4f32) 4883 return 0; 4884 return fastEmitInst_r(AArch64::FCVTLv4i16, &AArch64::FPR128RegClass, Op0); 4885} 4886 4887unsigned fastEmit_ISD_STRICT_FP_EXTEND_MVT_v2f32_r(MVT RetVT, unsigned Op0) { 4888 if (RetVT.SimpleTy != MVT::v2f64) 4889 return 0; 4890 return fastEmitInst_r(AArch64::FCVTLv2i32, &AArch64::FPR128RegClass, Op0); 4891} 4892 4893unsigned fastEmit_ISD_STRICT_FP_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0) { 4894 switch (VT.SimpleTy) { 4895 case MVT::f16: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_r(RetVT, Op0); 4896 case MVT::f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(RetVT, Op0); 4897 case MVT::v4f16: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4f16_r(RetVT, Op0); 4898 case MVT::v2f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v2f32_r(RetVT, Op0); 4899 default: return 0; 4900 } 4901} 4902 4903// FastEmit functions for ISD::STRICT_FP_ROUND. 4904 4905unsigned fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_r(MVT RetVT, unsigned Op0) { 4906 if (RetVT.SimpleTy != MVT::f16) 4907 return 0; 4908 if ((Subtarget->hasFPARMv8())) { 4909 return fastEmitInst_r(AArch64::FCVTHSr, &AArch64::FPR16RegClass, Op0); 4910 } 4911 return 0; 4912} 4913 4914unsigned fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f16_r(unsigned Op0) { 4915 if ((Subtarget->hasFPARMv8())) { 4916 return fastEmitInst_r(AArch64::FCVTHDr, &AArch64::FPR16RegClass, Op0); 4917 } 4918 return 0; 4919} 4920 4921unsigned fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f32_r(unsigned Op0) { 4922 if ((Subtarget->hasFPARMv8())) { 4923 return fastEmitInst_r(AArch64::FCVTSDr, &AArch64::FPR32RegClass, Op0); 4924 } 4925 return 0; 4926} 4927 4928unsigned fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(MVT RetVT, unsigned Op0) { 4929switch (RetVT.SimpleTy) { 4930 case MVT::f16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f16_r(Op0); 4931 case MVT::f32: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f32_r(Op0); 4932 default: return 0; 4933} 4934} 4935 4936unsigned fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 4937 if (RetVT.SimpleTy != MVT::v4f16) 4938 return 0; 4939 return fastEmitInst_r(AArch64::FCVTNv4i16, &AArch64::FPR64RegClass, Op0); 4940} 4941 4942unsigned fastEmit_ISD_STRICT_FP_ROUND_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 4943 if (RetVT.SimpleTy != MVT::v2f32) 4944 return 0; 4945 return fastEmitInst_r(AArch64::FCVTNv2i32, &AArch64::FPR64RegClass, Op0); 4946} 4947 4948unsigned fastEmit_ISD_STRICT_FP_ROUND_r(MVT VT, MVT RetVT, unsigned Op0) { 4949 switch (VT.SimpleTy) { 4950 case MVT::f32: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_r(RetVT, Op0); 4951 case MVT::f64: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(RetVT, Op0); 4952 case MVT::v4f32: return fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_r(RetVT, Op0); 4953 case MVT::v2f64: return fastEmit_ISD_STRICT_FP_ROUND_MVT_v2f64_r(RetVT, Op0); 4954 default: return 0; 4955 } 4956} 4957 4958// FastEmit functions for ISD::STRICT_FP_TO_SINT. 4959 4960unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i32_r(unsigned Op0) { 4961 if ((Subtarget->hasFullFP16())) { 4962 return fastEmitInst_r(AArch64::FCVTZSUWHr, &AArch64::GPR32RegClass, Op0); 4963 } 4964 return 0; 4965} 4966 4967unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i64_r(unsigned Op0) { 4968 if ((Subtarget->hasFullFP16())) { 4969 return fastEmitInst_r(AArch64::FCVTZSUXHr, &AArch64::GPR64RegClass, Op0); 4970 } 4971 return 0; 4972} 4973 4974unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_r(MVT RetVT, unsigned Op0) { 4975switch (RetVT.SimpleTy) { 4976 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i32_r(Op0); 4977 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i64_r(Op0); 4978 default: return 0; 4979} 4980} 4981 4982unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i32_r(unsigned Op0) { 4983 if ((Subtarget->hasFPARMv8())) { 4984 return fastEmitInst_r(AArch64::FCVTZSUWSr, &AArch64::GPR32RegClass, Op0); 4985 } 4986 return 0; 4987} 4988 4989unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i64_r(unsigned Op0) { 4990 if ((Subtarget->hasFPARMv8())) { 4991 return fastEmitInst_r(AArch64::FCVTZSUXSr, &AArch64::GPR64RegClass, Op0); 4992 } 4993 return 0; 4994} 4995 4996unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_r(MVT RetVT, unsigned Op0) { 4997switch (RetVT.SimpleTy) { 4998 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0); 4999 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0); 5000 default: return 0; 5001} 5002} 5003 5004unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i32_r(unsigned Op0) { 5005 if ((Subtarget->hasFPARMv8())) { 5006 return fastEmitInst_r(AArch64::FCVTZSUWDr, &AArch64::GPR32RegClass, Op0); 5007 } 5008 return 0; 5009} 5010 5011unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i64_r(unsigned Op0) { 5012 if ((Subtarget->hasFPARMv8())) { 5013 return fastEmitInst_r(AArch64::FCVTZSUXDr, &AArch64::GPR64RegClass, Op0); 5014 } 5015 return 0; 5016} 5017 5018unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_r(MVT RetVT, unsigned Op0) { 5019switch (RetVT.SimpleTy) { 5020 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0); 5021 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0); 5022 default: return 0; 5023} 5024} 5025 5026unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f16_r(MVT RetVT, unsigned Op0) { 5027 if (RetVT.SimpleTy != MVT::v4i16) 5028 return 0; 5029 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 5030 return fastEmitInst_r(AArch64::FCVTZSv4f16, &AArch64::FPR64RegClass, Op0); 5031 } 5032 return 0; 5033} 5034 5035unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 5036 if (RetVT.SimpleTy != MVT::v8i16) 5037 return 0; 5038 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 5039 return fastEmitInst_r(AArch64::FCVTZSv8f16, &AArch64::FPR128RegClass, Op0); 5040 } 5041 return 0; 5042} 5043 5044unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v2f32_r(MVT RetVT, unsigned Op0) { 5045 if (RetVT.SimpleTy != MVT::v2i32) 5046 return 0; 5047 if ((Subtarget->hasNEON())) { 5048 return fastEmitInst_r(AArch64::FCVTZSv2f32, &AArch64::FPR64RegClass, Op0); 5049 } 5050 return 0; 5051} 5052 5053unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 5054 if (RetVT.SimpleTy != MVT::v4i32) 5055 return 0; 5056 if ((Subtarget->hasNEON())) { 5057 return fastEmitInst_r(AArch64::FCVTZSv4f32, &AArch64::FPR128RegClass, Op0); 5058 } 5059 return 0; 5060} 5061 5062unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 5063 if (RetVT.SimpleTy != MVT::v2i64) 5064 return 0; 5065 if ((Subtarget->hasNEON())) { 5066 return fastEmitInst_r(AArch64::FCVTZSv2f64, &AArch64::FPR128RegClass, Op0); 5067 } 5068 return 0; 5069} 5070 5071unsigned fastEmit_ISD_STRICT_FP_TO_SINT_r(MVT VT, MVT RetVT, unsigned Op0) { 5072 switch (VT.SimpleTy) { 5073 case MVT::f16: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_r(RetVT, Op0); 5074 case MVT::f32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_r(RetVT, Op0); 5075 case MVT::f64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_r(RetVT, Op0); 5076 case MVT::v4f16: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f16_r(RetVT, Op0); 5077 case MVT::v8f16: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v8f16_r(RetVT, Op0); 5078 case MVT::v2f32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v2f32_r(RetVT, Op0); 5079 case MVT::v4f32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f32_r(RetVT, Op0); 5080 case MVT::v2f64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v2f64_r(RetVT, Op0); 5081 default: return 0; 5082 } 5083} 5084 5085// FastEmit functions for ISD::STRICT_FP_TO_UINT. 5086 5087unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i32_r(unsigned Op0) { 5088 if ((Subtarget->hasFullFP16())) { 5089 return fastEmitInst_r(AArch64::FCVTZUUWHr, &AArch64::GPR32RegClass, Op0); 5090 } 5091 return 0; 5092} 5093 5094unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i64_r(unsigned Op0) { 5095 if ((Subtarget->hasFullFP16())) { 5096 return fastEmitInst_r(AArch64::FCVTZUUXHr, &AArch64::GPR64RegClass, Op0); 5097 } 5098 return 0; 5099} 5100 5101unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_r(MVT RetVT, unsigned Op0) { 5102switch (RetVT.SimpleTy) { 5103 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i32_r(Op0); 5104 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i64_r(Op0); 5105 default: return 0; 5106} 5107} 5108 5109unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i32_r(unsigned Op0) { 5110 if ((Subtarget->hasFPARMv8())) { 5111 return fastEmitInst_r(AArch64::FCVTZUUWSr, &AArch64::GPR32RegClass, Op0); 5112 } 5113 return 0; 5114} 5115 5116unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i64_r(unsigned Op0) { 5117 if ((Subtarget->hasFPARMv8())) { 5118 return fastEmitInst_r(AArch64::FCVTZUUXSr, &AArch64::GPR64RegClass, Op0); 5119 } 5120 return 0; 5121} 5122 5123unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_r(MVT RetVT, unsigned Op0) { 5124switch (RetVT.SimpleTy) { 5125 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0); 5126 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0); 5127 default: return 0; 5128} 5129} 5130 5131unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i32_r(unsigned Op0) { 5132 if ((Subtarget->hasFPARMv8())) { 5133 return fastEmitInst_r(AArch64::FCVTZUUWDr, &AArch64::GPR32RegClass, Op0); 5134 } 5135 return 0; 5136} 5137 5138unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i64_r(unsigned Op0) { 5139 if ((Subtarget->hasFPARMv8())) { 5140 return fastEmitInst_r(AArch64::FCVTZUUXDr, &AArch64::GPR64RegClass, Op0); 5141 } 5142 return 0; 5143} 5144 5145unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_r(MVT RetVT, unsigned Op0) { 5146switch (RetVT.SimpleTy) { 5147 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0); 5148 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0); 5149 default: return 0; 5150} 5151} 5152 5153unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v4f16_r(MVT RetVT, unsigned Op0) { 5154 if (RetVT.SimpleTy != MVT::v4i16) 5155 return 0; 5156 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 5157 return fastEmitInst_r(AArch64::FCVTZUv4f16, &AArch64::FPR64RegClass, Op0); 5158 } 5159 return 0; 5160} 5161 5162unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 5163 if (RetVT.SimpleTy != MVT::v8i16) 5164 return 0; 5165 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 5166 return fastEmitInst_r(AArch64::FCVTZUv8f16, &AArch64::FPR128RegClass, Op0); 5167 } 5168 return 0; 5169} 5170 5171unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v2f32_r(MVT RetVT, unsigned Op0) { 5172 if (RetVT.SimpleTy != MVT::v2i32) 5173 return 0; 5174 if ((Subtarget->hasNEON())) { 5175 return fastEmitInst_r(AArch64::FCVTZUv2f32, &AArch64::FPR64RegClass, Op0); 5176 } 5177 return 0; 5178} 5179 5180unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 5181 if (RetVT.SimpleTy != MVT::v4i32) 5182 return 0; 5183 if ((Subtarget->hasNEON())) { 5184 return fastEmitInst_r(AArch64::FCVTZUv4f32, &AArch64::FPR128RegClass, Op0); 5185 } 5186 return 0; 5187} 5188 5189unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 5190 if (RetVT.SimpleTy != MVT::v2i64) 5191 return 0; 5192 if ((Subtarget->hasNEON())) { 5193 return fastEmitInst_r(AArch64::FCVTZUv2f64, &AArch64::FPR128RegClass, Op0); 5194 } 5195 return 0; 5196} 5197 5198unsigned fastEmit_ISD_STRICT_FP_TO_UINT_r(MVT VT, MVT RetVT, unsigned Op0) { 5199 switch (VT.SimpleTy) { 5200 case MVT::f16: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_r(RetVT, Op0); 5201 case MVT::f32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_r(RetVT, Op0); 5202 case MVT::f64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_r(RetVT, Op0); 5203 case MVT::v4f16: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v4f16_r(RetVT, Op0); 5204 case MVT::v8f16: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v8f16_r(RetVT, Op0); 5205 case MVT::v2f32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v2f32_r(RetVT, Op0); 5206 case MVT::v4f32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v4f32_r(RetVT, Op0); 5207 case MVT::v2f64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v2f64_r(RetVT, Op0); 5208 default: return 0; 5209 } 5210} 5211 5212// FastEmit functions for ISD::STRICT_FRINT. 5213 5214unsigned fastEmit_ISD_STRICT_FRINT_MVT_f16_r(MVT RetVT, unsigned Op0) { 5215 if (RetVT.SimpleTy != MVT::f16) 5216 return 0; 5217 if ((Subtarget->hasFullFP16())) { 5218 return fastEmitInst_r(AArch64::FRINTXHr, &AArch64::FPR16RegClass, Op0); 5219 } 5220 return 0; 5221} 5222 5223unsigned fastEmit_ISD_STRICT_FRINT_MVT_f32_r(MVT RetVT, unsigned Op0) { 5224 if (RetVT.SimpleTy != MVT::f32) 5225 return 0; 5226 if ((Subtarget->hasFPARMv8())) { 5227 return fastEmitInst_r(AArch64::FRINTXSr, &AArch64::FPR32RegClass, Op0); 5228 } 5229 return 0; 5230} 5231 5232unsigned fastEmit_ISD_STRICT_FRINT_MVT_f64_r(MVT RetVT, unsigned Op0) { 5233 if (RetVT.SimpleTy != MVT::f64) 5234 return 0; 5235 if ((Subtarget->hasFPARMv8())) { 5236 return fastEmitInst_r(AArch64::FRINTXDr, &AArch64::FPR64RegClass, Op0); 5237 } 5238 return 0; 5239} 5240 5241unsigned fastEmit_ISD_STRICT_FRINT_MVT_v4f16_r(MVT RetVT, unsigned Op0) { 5242 if (RetVT.SimpleTy != MVT::v4f16) 5243 return 0; 5244 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 5245 return fastEmitInst_r(AArch64::FRINTXv4f16, &AArch64::FPR64RegClass, Op0); 5246 } 5247 return 0; 5248} 5249 5250unsigned fastEmit_ISD_STRICT_FRINT_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 5251 if (RetVT.SimpleTy != MVT::v8f16) 5252 return 0; 5253 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 5254 return fastEmitInst_r(AArch64::FRINTXv8f16, &AArch64::FPR128RegClass, Op0); 5255 } 5256 return 0; 5257} 5258 5259unsigned fastEmit_ISD_STRICT_FRINT_MVT_v2f32_r(MVT RetVT, unsigned Op0) { 5260 if (RetVT.SimpleTy != MVT::v2f32) 5261 return 0; 5262 if ((Subtarget->hasNEON())) { 5263 return fastEmitInst_r(AArch64::FRINTXv2f32, &AArch64::FPR64RegClass, Op0); 5264 } 5265 return 0; 5266} 5267 5268unsigned fastEmit_ISD_STRICT_FRINT_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 5269 if (RetVT.SimpleTy != MVT::v4f32) 5270 return 0; 5271 if ((Subtarget->hasNEON())) { 5272 return fastEmitInst_r(AArch64::FRINTXv4f32, &AArch64::FPR128RegClass, Op0); 5273 } 5274 return 0; 5275} 5276 5277unsigned fastEmit_ISD_STRICT_FRINT_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 5278 if (RetVT.SimpleTy != MVT::v2f64) 5279 return 0; 5280 if ((Subtarget->hasNEON())) { 5281 return fastEmitInst_r(AArch64::FRINTXv2f64, &AArch64::FPR128RegClass, Op0); 5282 } 5283 return 0; 5284} 5285 5286unsigned fastEmit_ISD_STRICT_FRINT_r(MVT VT, MVT RetVT, unsigned Op0) { 5287 switch (VT.SimpleTy) { 5288 case MVT::f16: return fastEmit_ISD_STRICT_FRINT_MVT_f16_r(RetVT, Op0); 5289 case MVT::f32: return fastEmit_ISD_STRICT_FRINT_MVT_f32_r(RetVT, Op0); 5290 case MVT::f64: return fastEmit_ISD_STRICT_FRINT_MVT_f64_r(RetVT, Op0); 5291 case MVT::v4f16: return fastEmit_ISD_STRICT_FRINT_MVT_v4f16_r(RetVT, Op0); 5292 case MVT::v8f16: return fastEmit_ISD_STRICT_FRINT_MVT_v8f16_r(RetVT, Op0); 5293 case MVT::v2f32: return fastEmit_ISD_STRICT_FRINT_MVT_v2f32_r(RetVT, Op0); 5294 case MVT::v4f32: return fastEmit_ISD_STRICT_FRINT_MVT_v4f32_r(RetVT, Op0); 5295 case MVT::v2f64: return fastEmit_ISD_STRICT_FRINT_MVT_v2f64_r(RetVT, Op0); 5296 default: return 0; 5297 } 5298} 5299 5300// FastEmit functions for ISD::STRICT_FROUND. 5301 5302unsigned fastEmit_ISD_STRICT_FROUND_MVT_f16_r(MVT RetVT, unsigned Op0) { 5303 if (RetVT.SimpleTy != MVT::f16) 5304 return 0; 5305 if ((Subtarget->hasFullFP16())) { 5306 return fastEmitInst_r(AArch64::FRINTAHr, &AArch64::FPR16RegClass, Op0); 5307 } 5308 return 0; 5309} 5310 5311unsigned fastEmit_ISD_STRICT_FROUND_MVT_f32_r(MVT RetVT, unsigned Op0) { 5312 if (RetVT.SimpleTy != MVT::f32) 5313 return 0; 5314 if ((Subtarget->hasFPARMv8())) { 5315 return fastEmitInst_r(AArch64::FRINTASr, &AArch64::FPR32RegClass, Op0); 5316 } 5317 return 0; 5318} 5319 5320unsigned fastEmit_ISD_STRICT_FROUND_MVT_f64_r(MVT RetVT, unsigned Op0) { 5321 if (RetVT.SimpleTy != MVT::f64) 5322 return 0; 5323 if ((Subtarget->hasFPARMv8())) { 5324 return fastEmitInst_r(AArch64::FRINTADr, &AArch64::FPR64RegClass, Op0); 5325 } 5326 return 0; 5327} 5328 5329unsigned fastEmit_ISD_STRICT_FROUND_MVT_v4f16_r(MVT RetVT, unsigned Op0) { 5330 if (RetVT.SimpleTy != MVT::v4f16) 5331 return 0; 5332 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 5333 return fastEmitInst_r(AArch64::FRINTAv4f16, &AArch64::FPR64RegClass, Op0); 5334 } 5335 return 0; 5336} 5337 5338unsigned fastEmit_ISD_STRICT_FROUND_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 5339 if (RetVT.SimpleTy != MVT::v8f16) 5340 return 0; 5341 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 5342 return fastEmitInst_r(AArch64::FRINTAv8f16, &AArch64::FPR128RegClass, Op0); 5343 } 5344 return 0; 5345} 5346 5347unsigned fastEmit_ISD_STRICT_FROUND_MVT_v2f32_r(MVT RetVT, unsigned Op0) { 5348 if (RetVT.SimpleTy != MVT::v2f32) 5349 return 0; 5350 if ((Subtarget->hasNEON())) { 5351 return fastEmitInst_r(AArch64::FRINTAv2f32, &AArch64::FPR64RegClass, Op0); 5352 } 5353 return 0; 5354} 5355 5356unsigned fastEmit_ISD_STRICT_FROUND_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 5357 if (RetVT.SimpleTy != MVT::v4f32) 5358 return 0; 5359 if ((Subtarget->hasNEON())) { 5360 return fastEmitInst_r(AArch64::FRINTAv4f32, &AArch64::FPR128RegClass, Op0); 5361 } 5362 return 0; 5363} 5364 5365unsigned fastEmit_ISD_STRICT_FROUND_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 5366 if (RetVT.SimpleTy != MVT::v2f64) 5367 return 0; 5368 if ((Subtarget->hasNEON())) { 5369 return fastEmitInst_r(AArch64::FRINTAv2f64, &AArch64::FPR128RegClass, Op0); 5370 } 5371 return 0; 5372} 5373 5374unsigned fastEmit_ISD_STRICT_FROUND_r(MVT VT, MVT RetVT, unsigned Op0) { 5375 switch (VT.SimpleTy) { 5376 case MVT::f16: return fastEmit_ISD_STRICT_FROUND_MVT_f16_r(RetVT, Op0); 5377 case MVT::f32: return fastEmit_ISD_STRICT_FROUND_MVT_f32_r(RetVT, Op0); 5378 case MVT::f64: return fastEmit_ISD_STRICT_FROUND_MVT_f64_r(RetVT, Op0); 5379 case MVT::v4f16: return fastEmit_ISD_STRICT_FROUND_MVT_v4f16_r(RetVT, Op0); 5380 case MVT::v8f16: return fastEmit_ISD_STRICT_FROUND_MVT_v8f16_r(RetVT, Op0); 5381 case MVT::v2f32: return fastEmit_ISD_STRICT_FROUND_MVT_v2f32_r(RetVT, Op0); 5382 case MVT::v4f32: return fastEmit_ISD_STRICT_FROUND_MVT_v4f32_r(RetVT, Op0); 5383 case MVT::v2f64: return fastEmit_ISD_STRICT_FROUND_MVT_v2f64_r(RetVT, Op0); 5384 default: return 0; 5385 } 5386} 5387 5388// FastEmit functions for ISD::STRICT_FROUNDEVEN. 5389 5390unsigned fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f16_r(MVT RetVT, unsigned Op0) { 5391 if (RetVT.SimpleTy != MVT::f16) 5392 return 0; 5393 if ((Subtarget->hasFullFP16())) { 5394 return fastEmitInst_r(AArch64::FRINTNHr, &AArch64::FPR16RegClass, Op0); 5395 } 5396 return 0; 5397} 5398 5399unsigned fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f32_r(MVT RetVT, unsigned Op0) { 5400 if (RetVT.SimpleTy != MVT::f32) 5401 return 0; 5402 if ((Subtarget->hasFPARMv8())) { 5403 return fastEmitInst_r(AArch64::FRINTNSr, &AArch64::FPR32RegClass, Op0); 5404 } 5405 return 0; 5406} 5407 5408unsigned fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f64_r(MVT RetVT, unsigned Op0) { 5409 if (RetVT.SimpleTy != MVT::f64) 5410 return 0; 5411 if ((Subtarget->hasFPARMv8())) { 5412 return fastEmitInst_r(AArch64::FRINTNDr, &AArch64::FPR64RegClass, Op0); 5413 } 5414 return 0; 5415} 5416 5417unsigned fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f16_r(MVT RetVT, unsigned Op0) { 5418 if (RetVT.SimpleTy != MVT::v4f16) 5419 return 0; 5420 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 5421 return fastEmitInst_r(AArch64::FRINTNv4f16, &AArch64::FPR64RegClass, Op0); 5422 } 5423 return 0; 5424} 5425 5426unsigned fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 5427 if (RetVT.SimpleTy != MVT::v8f16) 5428 return 0; 5429 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 5430 return fastEmitInst_r(AArch64::FRINTNv8f16, &AArch64::FPR128RegClass, Op0); 5431 } 5432 return 0; 5433} 5434 5435unsigned fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f32_r(MVT RetVT, unsigned Op0) { 5436 if (RetVT.SimpleTy != MVT::v2f32) 5437 return 0; 5438 if ((Subtarget->hasNEON())) { 5439 return fastEmitInst_r(AArch64::FRINTNv2f32, &AArch64::FPR64RegClass, Op0); 5440 } 5441 return 0; 5442} 5443 5444unsigned fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 5445 if (RetVT.SimpleTy != MVT::v4f32) 5446 return 0; 5447 if ((Subtarget->hasNEON())) { 5448 return fastEmitInst_r(AArch64::FRINTNv4f32, &AArch64::FPR128RegClass, Op0); 5449 } 5450 return 0; 5451} 5452 5453unsigned fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 5454 if (RetVT.SimpleTy != MVT::v2f64) 5455 return 0; 5456 if ((Subtarget->hasNEON())) { 5457 return fastEmitInst_r(AArch64::FRINTNv2f64, &AArch64::FPR128RegClass, Op0); 5458 } 5459 return 0; 5460} 5461 5462unsigned fastEmit_ISD_STRICT_FROUNDEVEN_r(MVT VT, MVT RetVT, unsigned Op0) { 5463 switch (VT.SimpleTy) { 5464 case MVT::f16: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f16_r(RetVT, Op0); 5465 case MVT::f32: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f32_r(RetVT, Op0); 5466 case MVT::f64: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f64_r(RetVT, Op0); 5467 case MVT::v4f16: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f16_r(RetVT, Op0); 5468 case MVT::v8f16: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v8f16_r(RetVT, Op0); 5469 case MVT::v2f32: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f32_r(RetVT, Op0); 5470 case MVT::v4f32: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f32_r(RetVT, Op0); 5471 case MVT::v2f64: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f64_r(RetVT, Op0); 5472 default: return 0; 5473 } 5474} 5475 5476// FastEmit functions for ISD::STRICT_FSQRT. 5477 5478unsigned fastEmit_ISD_STRICT_FSQRT_MVT_f16_r(MVT RetVT, unsigned Op0) { 5479 if (RetVT.SimpleTy != MVT::f16) 5480 return 0; 5481 if ((Subtarget->hasFullFP16())) { 5482 return fastEmitInst_r(AArch64::FSQRTHr, &AArch64::FPR16RegClass, Op0); 5483 } 5484 return 0; 5485} 5486 5487unsigned fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(MVT RetVT, unsigned Op0) { 5488 if (RetVT.SimpleTy != MVT::f32) 5489 return 0; 5490 if ((Subtarget->hasFPARMv8())) { 5491 return fastEmitInst_r(AArch64::FSQRTSr, &AArch64::FPR32RegClass, Op0); 5492 } 5493 return 0; 5494} 5495 5496unsigned fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(MVT RetVT, unsigned Op0) { 5497 if (RetVT.SimpleTy != MVT::f64) 5498 return 0; 5499 if ((Subtarget->hasFPARMv8())) { 5500 return fastEmitInst_r(AArch64::FSQRTDr, &AArch64::FPR64RegClass, Op0); 5501 } 5502 return 0; 5503} 5504 5505unsigned fastEmit_ISD_STRICT_FSQRT_MVT_v4f16_r(MVT RetVT, unsigned Op0) { 5506 if (RetVT.SimpleTy != MVT::v4f16) 5507 return 0; 5508 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 5509 return fastEmitInst_r(AArch64::FSQRTv4f16, &AArch64::FPR64RegClass, Op0); 5510 } 5511 return 0; 5512} 5513 5514unsigned fastEmit_ISD_STRICT_FSQRT_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 5515 if (RetVT.SimpleTy != MVT::v8f16) 5516 return 0; 5517 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 5518 return fastEmitInst_r(AArch64::FSQRTv8f16, &AArch64::FPR128RegClass, Op0); 5519 } 5520 return 0; 5521} 5522 5523unsigned fastEmit_ISD_STRICT_FSQRT_MVT_v2f32_r(MVT RetVT, unsigned Op0) { 5524 if (RetVT.SimpleTy != MVT::v2f32) 5525 return 0; 5526 if ((Subtarget->hasNEON())) { 5527 return fastEmitInst_r(AArch64::FSQRTv2f32, &AArch64::FPR64RegClass, Op0); 5528 } 5529 return 0; 5530} 5531 5532unsigned fastEmit_ISD_STRICT_FSQRT_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 5533 if (RetVT.SimpleTy != MVT::v4f32) 5534 return 0; 5535 if ((Subtarget->hasNEON())) { 5536 return fastEmitInst_r(AArch64::FSQRTv4f32, &AArch64::FPR128RegClass, Op0); 5537 } 5538 return 0; 5539} 5540 5541unsigned fastEmit_ISD_STRICT_FSQRT_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 5542 if (RetVT.SimpleTy != MVT::v2f64) 5543 return 0; 5544 if ((Subtarget->hasNEON())) { 5545 return fastEmitInst_r(AArch64::FSQRTv2f64, &AArch64::FPR128RegClass, Op0); 5546 } 5547 return 0; 5548} 5549 5550unsigned fastEmit_ISD_STRICT_FSQRT_r(MVT VT, MVT RetVT, unsigned Op0) { 5551 switch (VT.SimpleTy) { 5552 case MVT::f16: return fastEmit_ISD_STRICT_FSQRT_MVT_f16_r(RetVT, Op0); 5553 case MVT::f32: return fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(RetVT, Op0); 5554 case MVT::f64: return fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(RetVT, Op0); 5555 case MVT::v4f16: return fastEmit_ISD_STRICT_FSQRT_MVT_v4f16_r(RetVT, Op0); 5556 case MVT::v8f16: return fastEmit_ISD_STRICT_FSQRT_MVT_v8f16_r(RetVT, Op0); 5557 case MVT::v2f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v2f32_r(RetVT, Op0); 5558 case MVT::v4f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v4f32_r(RetVT, Op0); 5559 case MVT::v2f64: return fastEmit_ISD_STRICT_FSQRT_MVT_v2f64_r(RetVT, Op0); 5560 default: return 0; 5561 } 5562} 5563 5564// FastEmit functions for ISD::STRICT_FTRUNC. 5565 5566unsigned fastEmit_ISD_STRICT_FTRUNC_MVT_f16_r(MVT RetVT, unsigned Op0) { 5567 if (RetVT.SimpleTy != MVT::f16) 5568 return 0; 5569 if ((Subtarget->hasFullFP16())) { 5570 return fastEmitInst_r(AArch64::FRINTZHr, &AArch64::FPR16RegClass, Op0); 5571 } 5572 return 0; 5573} 5574 5575unsigned fastEmit_ISD_STRICT_FTRUNC_MVT_f32_r(MVT RetVT, unsigned Op0) { 5576 if (RetVT.SimpleTy != MVT::f32) 5577 return 0; 5578 if ((Subtarget->hasFPARMv8())) { 5579 return fastEmitInst_r(AArch64::FRINTZSr, &AArch64::FPR32RegClass, Op0); 5580 } 5581 return 0; 5582} 5583 5584unsigned fastEmit_ISD_STRICT_FTRUNC_MVT_f64_r(MVT RetVT, unsigned Op0) { 5585 if (RetVT.SimpleTy != MVT::f64) 5586 return 0; 5587 if ((Subtarget->hasFPARMv8())) { 5588 return fastEmitInst_r(AArch64::FRINTZDr, &AArch64::FPR64RegClass, Op0); 5589 } 5590 return 0; 5591} 5592 5593unsigned fastEmit_ISD_STRICT_FTRUNC_MVT_v4f16_r(MVT RetVT, unsigned Op0) { 5594 if (RetVT.SimpleTy != MVT::v4f16) 5595 return 0; 5596 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 5597 return fastEmitInst_r(AArch64::FRINTZv4f16, &AArch64::FPR64RegClass, Op0); 5598 } 5599 return 0; 5600} 5601 5602unsigned fastEmit_ISD_STRICT_FTRUNC_MVT_v8f16_r(MVT RetVT, unsigned Op0) { 5603 if (RetVT.SimpleTy != MVT::v8f16) 5604 return 0; 5605 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 5606 return fastEmitInst_r(AArch64::FRINTZv8f16, &AArch64::FPR128RegClass, Op0); 5607 } 5608 return 0; 5609} 5610 5611unsigned fastEmit_ISD_STRICT_FTRUNC_MVT_v2f32_r(MVT RetVT, unsigned Op0) { 5612 if (RetVT.SimpleTy != MVT::v2f32) 5613 return 0; 5614 if ((Subtarget->hasNEON())) { 5615 return fastEmitInst_r(AArch64::FRINTZv2f32, &AArch64::FPR64RegClass, Op0); 5616 } 5617 return 0; 5618} 5619 5620unsigned fastEmit_ISD_STRICT_FTRUNC_MVT_v4f32_r(MVT RetVT, unsigned Op0) { 5621 if (RetVT.SimpleTy != MVT::v4f32) 5622 return 0; 5623 if ((Subtarget->hasNEON())) { 5624 return fastEmitInst_r(AArch64::FRINTZv4f32, &AArch64::FPR128RegClass, Op0); 5625 } 5626 return 0; 5627} 5628 5629unsigned fastEmit_ISD_STRICT_FTRUNC_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 5630 if (RetVT.SimpleTy != MVT::v2f64) 5631 return 0; 5632 if ((Subtarget->hasNEON())) { 5633 return fastEmitInst_r(AArch64::FRINTZv2f64, &AArch64::FPR128RegClass, Op0); 5634 } 5635 return 0; 5636} 5637 5638unsigned fastEmit_ISD_STRICT_FTRUNC_r(MVT VT, MVT RetVT, unsigned Op0) { 5639 switch (VT.SimpleTy) { 5640 case MVT::f16: return fastEmit_ISD_STRICT_FTRUNC_MVT_f16_r(RetVT, Op0); 5641 case MVT::f32: return fastEmit_ISD_STRICT_FTRUNC_MVT_f32_r(RetVT, Op0); 5642 case MVT::f64: return fastEmit_ISD_STRICT_FTRUNC_MVT_f64_r(RetVT, Op0); 5643 case MVT::v4f16: return fastEmit_ISD_STRICT_FTRUNC_MVT_v4f16_r(RetVT, Op0); 5644 case MVT::v8f16: return fastEmit_ISD_STRICT_FTRUNC_MVT_v8f16_r(RetVT, Op0); 5645 case MVT::v2f32: return fastEmit_ISD_STRICT_FTRUNC_MVT_v2f32_r(RetVT, Op0); 5646 case MVT::v4f32: return fastEmit_ISD_STRICT_FTRUNC_MVT_v4f32_r(RetVT, Op0); 5647 case MVT::v2f64: return fastEmit_ISD_STRICT_FTRUNC_MVT_v2f64_r(RetVT, Op0); 5648 default: return 0; 5649 } 5650} 5651 5652// FastEmit functions for ISD::STRICT_LLROUND. 5653 5654unsigned fastEmit_ISD_STRICT_LLROUND_MVT_f16_r(MVT RetVT, unsigned Op0) { 5655 if (RetVT.SimpleTy != MVT::i64) 5656 return 0; 5657 if ((Subtarget->hasFullFP16())) { 5658 return fastEmitInst_r(AArch64::FCVTASUXHr, &AArch64::GPR64RegClass, Op0); 5659 } 5660 return 0; 5661} 5662 5663unsigned fastEmit_ISD_STRICT_LLROUND_MVT_f32_r(MVT RetVT, unsigned Op0) { 5664 if (RetVT.SimpleTy != MVT::i64) 5665 return 0; 5666 return fastEmitInst_r(AArch64::FCVTASUXSr, &AArch64::GPR64RegClass, Op0); 5667} 5668 5669unsigned fastEmit_ISD_STRICT_LLROUND_MVT_f64_r(MVT RetVT, unsigned Op0) { 5670 if (RetVT.SimpleTy != MVT::i64) 5671 return 0; 5672 return fastEmitInst_r(AArch64::FCVTASUXDr, &AArch64::GPR64RegClass, Op0); 5673} 5674 5675unsigned fastEmit_ISD_STRICT_LLROUND_r(MVT VT, MVT RetVT, unsigned Op0) { 5676 switch (VT.SimpleTy) { 5677 case MVT::f16: return fastEmit_ISD_STRICT_LLROUND_MVT_f16_r(RetVT, Op0); 5678 case MVT::f32: return fastEmit_ISD_STRICT_LLROUND_MVT_f32_r(RetVT, Op0); 5679 case MVT::f64: return fastEmit_ISD_STRICT_LLROUND_MVT_f64_r(RetVT, Op0); 5680 default: return 0; 5681 } 5682} 5683 5684// FastEmit functions for ISD::STRICT_LROUND. 5685 5686unsigned fastEmit_ISD_STRICT_LROUND_MVT_f16_MVT_i32_r(unsigned Op0) { 5687 if ((Subtarget->hasFullFP16())) { 5688 return fastEmitInst_r(AArch64::FCVTASUWHr, &AArch64::GPR32RegClass, Op0); 5689 } 5690 return 0; 5691} 5692 5693unsigned fastEmit_ISD_STRICT_LROUND_MVT_f16_MVT_i64_r(unsigned Op0) { 5694 if ((Subtarget->hasFullFP16())) { 5695 return fastEmitInst_r(AArch64::FCVTASUXHr, &AArch64::GPR64RegClass, Op0); 5696 } 5697 return 0; 5698} 5699 5700unsigned fastEmit_ISD_STRICT_LROUND_MVT_f16_r(MVT RetVT, unsigned Op0) { 5701switch (RetVT.SimpleTy) { 5702 case MVT::i32: return fastEmit_ISD_STRICT_LROUND_MVT_f16_MVT_i32_r(Op0); 5703 case MVT::i64: return fastEmit_ISD_STRICT_LROUND_MVT_f16_MVT_i64_r(Op0); 5704 default: return 0; 5705} 5706} 5707 5708unsigned fastEmit_ISD_STRICT_LROUND_MVT_f32_MVT_i32_r(unsigned Op0) { 5709 return fastEmitInst_r(AArch64::FCVTASUWSr, &AArch64::GPR32RegClass, Op0); 5710} 5711 5712unsigned fastEmit_ISD_STRICT_LROUND_MVT_f32_MVT_i64_r(unsigned Op0) { 5713 return fastEmitInst_r(AArch64::FCVTASUXSr, &AArch64::GPR64RegClass, Op0); 5714} 5715 5716unsigned fastEmit_ISD_STRICT_LROUND_MVT_f32_r(MVT RetVT, unsigned Op0) { 5717switch (RetVT.SimpleTy) { 5718 case MVT::i32: return fastEmit_ISD_STRICT_LROUND_MVT_f32_MVT_i32_r(Op0); 5719 case MVT::i64: return fastEmit_ISD_STRICT_LROUND_MVT_f32_MVT_i64_r(Op0); 5720 default: return 0; 5721} 5722} 5723 5724unsigned fastEmit_ISD_STRICT_LROUND_MVT_f64_MVT_i32_r(unsigned Op0) { 5725 return fastEmitInst_r(AArch64::FCVTASUWDr, &AArch64::GPR32RegClass, Op0); 5726} 5727 5728unsigned fastEmit_ISD_STRICT_LROUND_MVT_f64_MVT_i64_r(unsigned Op0) { 5729 return fastEmitInst_r(AArch64::FCVTASUXDr, &AArch64::GPR64RegClass, Op0); 5730} 5731 5732unsigned fastEmit_ISD_STRICT_LROUND_MVT_f64_r(MVT RetVT, unsigned Op0) { 5733switch (RetVT.SimpleTy) { 5734 case MVT::i32: return fastEmit_ISD_STRICT_LROUND_MVT_f64_MVT_i32_r(Op0); 5735 case MVT::i64: return fastEmit_ISD_STRICT_LROUND_MVT_f64_MVT_i64_r(Op0); 5736 default: return 0; 5737} 5738} 5739 5740unsigned fastEmit_ISD_STRICT_LROUND_r(MVT VT, MVT RetVT, unsigned Op0) { 5741 switch (VT.SimpleTy) { 5742 case MVT::f16: return fastEmit_ISD_STRICT_LROUND_MVT_f16_r(RetVT, Op0); 5743 case MVT::f32: return fastEmit_ISD_STRICT_LROUND_MVT_f32_r(RetVT, Op0); 5744 case MVT::f64: return fastEmit_ISD_STRICT_LROUND_MVT_f64_r(RetVT, Op0); 5745 default: return 0; 5746 } 5747} 5748 5749// FastEmit functions for ISD::STRICT_SINT_TO_FP. 5750 5751unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f16_r(unsigned Op0) { 5752 if ((Subtarget->hasFullFP16())) { 5753 return fastEmitInst_r(AArch64::SCVTFUWHri, &AArch64::FPR16RegClass, Op0); 5754 } 5755 return 0; 5756} 5757 5758unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f32_r(unsigned Op0) { 5759 if ((Subtarget->hasFPARMv8())) { 5760 return fastEmitInst_r(AArch64::SCVTFUWSri, &AArch64::FPR32RegClass, Op0); 5761 } 5762 return 0; 5763} 5764 5765unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f64_r(unsigned Op0) { 5766 if ((Subtarget->hasFPARMv8())) { 5767 return fastEmitInst_r(AArch64::SCVTFUWDri, &AArch64::FPR64RegClass, Op0); 5768 } 5769 return 0; 5770} 5771 5772unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_r(MVT RetVT, unsigned Op0) { 5773switch (RetVT.SimpleTy) { 5774 case MVT::f16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f16_r(Op0); 5775 case MVT::f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0); 5776 case MVT::f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0); 5777 default: return 0; 5778} 5779} 5780 5781unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f16_r(unsigned Op0) { 5782 if ((Subtarget->hasFullFP16())) { 5783 return fastEmitInst_r(AArch64::SCVTFUXHri, &AArch64::FPR16RegClass, Op0); 5784 } 5785 return 0; 5786} 5787 5788unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f32_r(unsigned Op0) { 5789 if ((Subtarget->hasFPARMv8())) { 5790 return fastEmitInst_r(AArch64::SCVTFUXSri, &AArch64::FPR32RegClass, Op0); 5791 } 5792 return 0; 5793} 5794 5795unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f64_r(unsigned Op0) { 5796 if ((Subtarget->hasFPARMv8())) { 5797 return fastEmitInst_r(AArch64::SCVTFUXDri, &AArch64::FPR64RegClass, Op0); 5798 } 5799 return 0; 5800} 5801 5802unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_r(MVT RetVT, unsigned Op0) { 5803switch (RetVT.SimpleTy) { 5804 case MVT::f16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f16_r(Op0); 5805 case MVT::f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0); 5806 case MVT::f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0); 5807 default: return 0; 5808} 5809} 5810 5811unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i16_r(MVT RetVT, unsigned Op0) { 5812 if (RetVT.SimpleTy != MVT::v4f16) 5813 return 0; 5814 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 5815 return fastEmitInst_r(AArch64::SCVTFv4f16, &AArch64::FPR64RegClass, Op0); 5816 } 5817 return 0; 5818} 5819 5820unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i16_r(MVT RetVT, unsigned Op0) { 5821 if (RetVT.SimpleTy != MVT::v8f16) 5822 return 0; 5823 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 5824 return fastEmitInst_r(AArch64::SCVTFv8f16, &AArch64::FPR128RegClass, Op0); 5825 } 5826 return 0; 5827} 5828 5829unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i32_r(MVT RetVT, unsigned Op0) { 5830 if (RetVT.SimpleTy != MVT::v2f32) 5831 return 0; 5832 if ((Subtarget->hasNEON())) { 5833 return fastEmitInst_r(AArch64::SCVTFv2f32, &AArch64::FPR64RegClass, Op0); 5834 } 5835 return 0; 5836} 5837 5838unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0) { 5839 if (RetVT.SimpleTy != MVT::v4f32) 5840 return 0; 5841 if ((Subtarget->hasNEON())) { 5842 return fastEmitInst_r(AArch64::SCVTFv4f32, &AArch64::FPR128RegClass, Op0); 5843 } 5844 return 0; 5845} 5846 5847unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, unsigned Op0) { 5848 if (RetVT.SimpleTy != MVT::v2f64) 5849 return 0; 5850 if ((Subtarget->hasNEON())) { 5851 return fastEmitInst_r(AArch64::SCVTFv2f64, &AArch64::FPR128RegClass, Op0); 5852 } 5853 return 0; 5854} 5855 5856unsigned fastEmit_ISD_STRICT_SINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0) { 5857 switch (VT.SimpleTy) { 5858 case MVT::i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_r(RetVT, Op0); 5859 case MVT::i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_r(RetVT, Op0); 5860 case MVT::v4i16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i16_r(RetVT, Op0); 5861 case MVT::v8i16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i16_r(RetVT, Op0); 5862 case MVT::v2i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i32_r(RetVT, Op0); 5863 case MVT::v4i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0); 5864 case MVT::v2i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0); 5865 default: return 0; 5866 } 5867} 5868 5869// FastEmit functions for ISD::STRICT_UINT_TO_FP. 5870 5871unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f16_r(unsigned Op0) { 5872 if ((Subtarget->hasFullFP16())) { 5873 return fastEmitInst_r(AArch64::UCVTFUWHri, &AArch64::FPR16RegClass, Op0); 5874 } 5875 return 0; 5876} 5877 5878unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f32_r(unsigned Op0) { 5879 if ((Subtarget->hasFPARMv8())) { 5880 return fastEmitInst_r(AArch64::UCVTFUWSri, &AArch64::FPR32RegClass, Op0); 5881 } 5882 return 0; 5883} 5884 5885unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f64_r(unsigned Op0) { 5886 if ((Subtarget->hasFPARMv8())) { 5887 return fastEmitInst_r(AArch64::UCVTFUWDri, &AArch64::FPR64RegClass, Op0); 5888 } 5889 return 0; 5890} 5891 5892unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_r(MVT RetVT, unsigned Op0) { 5893switch (RetVT.SimpleTy) { 5894 case MVT::f16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f16_r(Op0); 5895 case MVT::f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f32_r(Op0); 5896 case MVT::f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f64_r(Op0); 5897 default: return 0; 5898} 5899} 5900 5901unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f16_r(unsigned Op0) { 5902 if ((Subtarget->hasFullFP16())) { 5903 return fastEmitInst_r(AArch64::UCVTFUXHri, &AArch64::FPR16RegClass, Op0); 5904 } 5905 return 0; 5906} 5907 5908unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f32_r(unsigned Op0) { 5909 if ((Subtarget->hasFPARMv8())) { 5910 return fastEmitInst_r(AArch64::UCVTFUXSri, &AArch64::FPR32RegClass, Op0); 5911 } 5912 return 0; 5913} 5914 5915unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f64_r(unsigned Op0) { 5916 if ((Subtarget->hasFPARMv8())) { 5917 return fastEmitInst_r(AArch64::UCVTFUXDri, &AArch64::FPR64RegClass, Op0); 5918 } 5919 return 0; 5920} 5921 5922unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_r(MVT RetVT, unsigned Op0) { 5923switch (RetVT.SimpleTy) { 5924 case MVT::f16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f16_r(Op0); 5925 case MVT::f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f32_r(Op0); 5926 case MVT::f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f64_r(Op0); 5927 default: return 0; 5928} 5929} 5930 5931unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i16_r(MVT RetVT, unsigned Op0) { 5932 if (RetVT.SimpleTy != MVT::v4f16) 5933 return 0; 5934 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 5935 return fastEmitInst_r(AArch64::UCVTFv4f16, &AArch64::FPR64RegClass, Op0); 5936 } 5937 return 0; 5938} 5939 5940unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i16_r(MVT RetVT, unsigned Op0) { 5941 if (RetVT.SimpleTy != MVT::v8f16) 5942 return 0; 5943 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 5944 return fastEmitInst_r(AArch64::UCVTFv8f16, &AArch64::FPR128RegClass, Op0); 5945 } 5946 return 0; 5947} 5948 5949unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i32_r(MVT RetVT, unsigned Op0) { 5950 if (RetVT.SimpleTy != MVT::v2f32) 5951 return 0; 5952 if ((Subtarget->hasNEON())) { 5953 return fastEmitInst_r(AArch64::UCVTFv2f32, &AArch64::FPR64RegClass, Op0); 5954 } 5955 return 0; 5956} 5957 5958unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0) { 5959 if (RetVT.SimpleTy != MVT::v4f32) 5960 return 0; 5961 if ((Subtarget->hasNEON())) { 5962 return fastEmitInst_r(AArch64::UCVTFv4f32, &AArch64::FPR128RegClass, Op0); 5963 } 5964 return 0; 5965} 5966 5967unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, unsigned Op0) { 5968 if (RetVT.SimpleTy != MVT::v2f64) 5969 return 0; 5970 if ((Subtarget->hasNEON())) { 5971 return fastEmitInst_r(AArch64::UCVTFv2f64, &AArch64::FPR128RegClass, Op0); 5972 } 5973 return 0; 5974} 5975 5976unsigned fastEmit_ISD_STRICT_UINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0) { 5977 switch (VT.SimpleTy) { 5978 case MVT::i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_r(RetVT, Op0); 5979 case MVT::i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_r(RetVT, Op0); 5980 case MVT::v4i16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i16_r(RetVT, Op0); 5981 case MVT::v8i16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i16_r(RetVT, Op0); 5982 case MVT::v2i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i32_r(RetVT, Op0); 5983 case MVT::v4i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0); 5984 case MVT::v2i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0); 5985 default: return 0; 5986 } 5987} 5988 5989// FastEmit functions for ISD::TRUNCATE. 5990 5991unsigned fastEmit_ISD_TRUNCATE_MVT_i64_r(MVT RetVT, unsigned Op0) { 5992 if (RetVT.SimpleTy != MVT::i32) 5993 return 0; 5994 return fastEmitInst_extractsubreg(RetVT, Op0, AArch64::sub_32); 5995} 5996 5997unsigned fastEmit_ISD_TRUNCATE_MVT_v8i16_r(MVT RetVT, unsigned Op0) { 5998 if (RetVT.SimpleTy != MVT::v8i8) 5999 return 0; 6000 if ((Subtarget->hasNEON())) { 6001 return fastEmitInst_r(AArch64::XTNv8i8, &AArch64::FPR64RegClass, Op0); 6002 } 6003 return 0; 6004} 6005 6006unsigned fastEmit_ISD_TRUNCATE_MVT_v4i32_r(MVT RetVT, unsigned Op0) { 6007 if (RetVT.SimpleTy != MVT::v4i16) 6008 return 0; 6009 if ((Subtarget->hasNEON())) { 6010 return fastEmitInst_r(AArch64::XTNv4i16, &AArch64::FPR64RegClass, Op0); 6011 } 6012 return 0; 6013} 6014 6015unsigned fastEmit_ISD_TRUNCATE_MVT_v2i64_r(MVT RetVT, unsigned Op0) { 6016 if (RetVT.SimpleTy != MVT::v2i32) 6017 return 0; 6018 if ((Subtarget->hasNEON())) { 6019 return fastEmitInst_r(AArch64::XTNv2i32, &AArch64::FPR64RegClass, Op0); 6020 } 6021 return 0; 6022} 6023 6024unsigned fastEmit_ISD_TRUNCATE_r(MVT VT, MVT RetVT, unsigned Op0) { 6025 switch (VT.SimpleTy) { 6026 case MVT::i64: return fastEmit_ISD_TRUNCATE_MVT_i64_r(RetVT, Op0); 6027 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_MVT_v8i16_r(RetVT, Op0); 6028 case MVT::v4i32: return fastEmit_ISD_TRUNCATE_MVT_v4i32_r(RetVT, Op0); 6029 case MVT::v2i64: return fastEmit_ISD_TRUNCATE_MVT_v2i64_r(RetVT, Op0); 6030 default: return 0; 6031 } 6032} 6033 6034// FastEmit functions for ISD::UINT_TO_FP. 6035 6036unsigned fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f16_r(unsigned Op0) { 6037 if ((Subtarget->hasFullFP16())) { 6038 return fastEmitInst_r(AArch64::UCVTFUWHri, &AArch64::FPR16RegClass, Op0); 6039 } 6040 return 0; 6041} 6042 6043unsigned fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f32_r(unsigned Op0) { 6044 if ((Subtarget->hasFPARMv8())) { 6045 return fastEmitInst_r(AArch64::UCVTFUWSri, &AArch64::FPR32RegClass, Op0); 6046 } 6047 return 0; 6048} 6049 6050unsigned fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f64_r(unsigned Op0) { 6051 if ((Subtarget->hasFPARMv8())) { 6052 return fastEmitInst_r(AArch64::UCVTFUWDri, &AArch64::FPR64RegClass, Op0); 6053 } 6054 return 0; 6055} 6056 6057unsigned fastEmit_ISD_UINT_TO_FP_MVT_i32_r(MVT RetVT, unsigned Op0) { 6058switch (RetVT.SimpleTy) { 6059 case MVT::f16: return fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f16_r(Op0); 6060 case MVT::f32: return fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f32_r(Op0); 6061 case MVT::f64: return fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f64_r(Op0); 6062 default: return 0; 6063} 6064} 6065 6066unsigned fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f16_r(unsigned Op0) { 6067 if ((Subtarget->hasFullFP16())) { 6068 return fastEmitInst_r(AArch64::UCVTFUXHri, &AArch64::FPR16RegClass, Op0); 6069 } 6070 return 0; 6071} 6072 6073unsigned fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f32_r(unsigned Op0) { 6074 if ((Subtarget->hasFPARMv8())) { 6075 return fastEmitInst_r(AArch64::UCVTFUXSri, &AArch64::FPR32RegClass, Op0); 6076 } 6077 return 0; 6078} 6079 6080unsigned fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f64_r(unsigned Op0) { 6081 if ((Subtarget->hasFPARMv8())) { 6082 return fastEmitInst_r(AArch64::UCVTFUXDri, &AArch64::FPR64RegClass, Op0); 6083 } 6084 return 0; 6085} 6086 6087unsigned fastEmit_ISD_UINT_TO_FP_MVT_i64_r(MVT RetVT, unsigned Op0) { 6088switch (RetVT.SimpleTy) { 6089 case MVT::f16: return fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f16_r(Op0); 6090 case MVT::f32: return fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f32_r(Op0); 6091 case MVT::f64: return fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f64_r(Op0); 6092 default: return 0; 6093} 6094} 6095 6096unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i16_r(MVT RetVT, unsigned Op0) { 6097 if (RetVT.SimpleTy != MVT::v4f16) 6098 return 0; 6099 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 6100 return fastEmitInst_r(AArch64::UCVTFv4f16, &AArch64::FPR64RegClass, Op0); 6101 } 6102 return 0; 6103} 6104 6105unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(MVT RetVT, unsigned Op0) { 6106 if (RetVT.SimpleTy != MVT::v8f16) 6107 return 0; 6108 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 6109 return fastEmitInst_r(AArch64::UCVTFv8f16, &AArch64::FPR128RegClass, Op0); 6110 } 6111 return 0; 6112} 6113 6114unsigned fastEmit_ISD_UINT_TO_FP_MVT_v2i32_r(MVT RetVT, unsigned Op0) { 6115 if (RetVT.SimpleTy != MVT::v2f32) 6116 return 0; 6117 if ((Subtarget->hasNEON())) { 6118 return fastEmitInst_r(AArch64::UCVTFv2f32, &AArch64::FPR64RegClass, Op0); 6119 } 6120 return 0; 6121} 6122 6123unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0) { 6124 if (RetVT.SimpleTy != MVT::v4f32) 6125 return 0; 6126 if ((Subtarget->hasNEON())) { 6127 return fastEmitInst_r(AArch64::UCVTFv4f32, &AArch64::FPR128RegClass, Op0); 6128 } 6129 return 0; 6130} 6131 6132unsigned fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, unsigned Op0) { 6133 if (RetVT.SimpleTy != MVT::v2f64) 6134 return 0; 6135 if ((Subtarget->hasNEON())) { 6136 return fastEmitInst_r(AArch64::UCVTFv2f64, &AArch64::FPR128RegClass, Op0); 6137 } 6138 return 0; 6139} 6140 6141unsigned fastEmit_ISD_UINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0) { 6142 switch (VT.SimpleTy) { 6143 case MVT::i32: return fastEmit_ISD_UINT_TO_FP_MVT_i32_r(RetVT, Op0); 6144 case MVT::i64: return fastEmit_ISD_UINT_TO_FP_MVT_i64_r(RetVT, Op0); 6145 case MVT::v4i16: return fastEmit_ISD_UINT_TO_FP_MVT_v4i16_r(RetVT, Op0); 6146 case MVT::v8i16: return fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(RetVT, Op0); 6147 case MVT::v2i32: return fastEmit_ISD_UINT_TO_FP_MVT_v2i32_r(RetVT, Op0); 6148 case MVT::v4i32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0); 6149 case MVT::v2i64: return fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0); 6150 default: return 0; 6151 } 6152} 6153 6154// FastEmit functions for ISD::VECREDUCE_FADD. 6155 6156unsigned fastEmit_ISD_VECREDUCE_FADD_MVT_v2f32_r(MVT RetVT, unsigned Op0) { 6157 if (RetVT.SimpleTy != MVT::f32) 6158 return 0; 6159 return fastEmitInst_r(AArch64::FADDPv2i32p, &AArch64::FPR32RegClass, Op0); 6160} 6161 6162unsigned fastEmit_ISD_VECREDUCE_FADD_MVT_v2f64_r(MVT RetVT, unsigned Op0) { 6163 if (RetVT.SimpleTy != MVT::f64) 6164 return 0; 6165 return fastEmitInst_r(AArch64::FADDPv2i64p, &AArch64::FPR64RegClass, Op0); 6166} 6167 6168unsigned fastEmit_ISD_VECREDUCE_FADD_r(MVT VT, MVT RetVT, unsigned Op0) { 6169 switch (VT.SimpleTy) { 6170 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FADD_MVT_v2f32_r(RetVT, Op0); 6171 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FADD_MVT_v2f64_r(RetVT, Op0); 6172 default: return 0; 6173 } 6174} 6175 6176// FastEmit functions for ISD::VECTOR_REVERSE. 6177 6178unsigned fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2i1_r(MVT RetVT, unsigned Op0) { 6179 if (RetVT.SimpleTy != MVT::nxv2i1) 6180 return 0; 6181 if ((Subtarget->hasSVEorSME())) { 6182 return fastEmitInst_r(AArch64::REV_PP_D, &AArch64::PPRRegClass, Op0); 6183 } 6184 return 0; 6185} 6186 6187unsigned fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4i1_r(MVT RetVT, unsigned Op0) { 6188 if (RetVT.SimpleTy != MVT::nxv4i1) 6189 return 0; 6190 if ((Subtarget->hasSVEorSME())) { 6191 return fastEmitInst_r(AArch64::REV_PP_S, &AArch64::PPRRegClass, Op0); 6192 } 6193 return 0; 6194} 6195 6196unsigned fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8i1_r(MVT RetVT, unsigned Op0) { 6197 if (RetVT.SimpleTy != MVT::nxv8i1) 6198 return 0; 6199 if ((Subtarget->hasSVEorSME())) { 6200 return fastEmitInst_r(AArch64::REV_PP_H, &AArch64::PPRRegClass, Op0); 6201 } 6202 return 0; 6203} 6204 6205unsigned fastEmit_ISD_VECTOR_REVERSE_MVT_nxv16i1_r(MVT RetVT, unsigned Op0) { 6206 if (RetVT.SimpleTy != MVT::nxv16i1) 6207 return 0; 6208 if ((Subtarget->hasSVEorSME())) { 6209 return fastEmitInst_r(AArch64::REV_PP_B, &AArch64::PPRRegClass, Op0); 6210 } 6211 return 0; 6212} 6213 6214unsigned fastEmit_ISD_VECTOR_REVERSE_MVT_nxv16i8_r(MVT RetVT, unsigned Op0) { 6215 if (RetVT.SimpleTy != MVT::nxv16i8) 6216 return 0; 6217 if ((Subtarget->hasSVEorSME())) { 6218 return fastEmitInst_r(AArch64::REV_ZZ_B, &AArch64::ZPRRegClass, Op0); 6219 } 6220 return 0; 6221} 6222 6223unsigned fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8i16_r(MVT RetVT, unsigned Op0) { 6224 if (RetVT.SimpleTy != MVT::nxv8i16) 6225 return 0; 6226 if ((Subtarget->hasSVEorSME())) { 6227 return fastEmitInst_r(AArch64::REV_ZZ_H, &AArch64::ZPRRegClass, Op0); 6228 } 6229 return 0; 6230} 6231 6232unsigned fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4i32_r(MVT RetVT, unsigned Op0) { 6233 if (RetVT.SimpleTy != MVT::nxv4i32) 6234 return 0; 6235 if ((Subtarget->hasSVEorSME())) { 6236 return fastEmitInst_r(AArch64::REV_ZZ_S, &AArch64::ZPRRegClass, Op0); 6237 } 6238 return 0; 6239} 6240 6241unsigned fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2i64_r(MVT RetVT, unsigned Op0) { 6242 if (RetVT.SimpleTy != MVT::nxv2i64) 6243 return 0; 6244 if ((Subtarget->hasSVEorSME())) { 6245 return fastEmitInst_r(AArch64::REV_ZZ_D, &AArch64::ZPRRegClass, Op0); 6246 } 6247 return 0; 6248} 6249 6250unsigned fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f16_r(MVT RetVT, unsigned Op0) { 6251 if (RetVT.SimpleTy != MVT::nxv2f16) 6252 return 0; 6253 if ((Subtarget->hasSVEorSME())) { 6254 return fastEmitInst_r(AArch64::REV_ZZ_D, &AArch64::ZPRRegClass, Op0); 6255 } 6256 return 0; 6257} 6258 6259unsigned fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4f16_r(MVT RetVT, unsigned Op0) { 6260 if (RetVT.SimpleTy != MVT::nxv4f16) 6261 return 0; 6262 if ((Subtarget->hasSVEorSME())) { 6263 return fastEmitInst_r(AArch64::REV_ZZ_S, &AArch64::ZPRRegClass, Op0); 6264 } 6265 return 0; 6266} 6267 6268unsigned fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8f16_r(MVT RetVT, unsigned Op0) { 6269 if (RetVT.SimpleTy != MVT::nxv8f16) 6270 return 0; 6271 if ((Subtarget->hasSVEorSME())) { 6272 return fastEmitInst_r(AArch64::REV_ZZ_H, &AArch64::ZPRRegClass, Op0); 6273 } 6274 return 0; 6275} 6276 6277unsigned fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2bf16_r(MVT RetVT, unsigned Op0) { 6278 if (RetVT.SimpleTy != MVT::nxv2bf16) 6279 return 0; 6280 if ((Subtarget->hasSVEorSME())) { 6281 return fastEmitInst_r(AArch64::REV_ZZ_D, &AArch64::ZPRRegClass, Op0); 6282 } 6283 return 0; 6284} 6285 6286unsigned fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4bf16_r(MVT RetVT, unsigned Op0) { 6287 if (RetVT.SimpleTy != MVT::nxv4bf16) 6288 return 0; 6289 if ((Subtarget->hasSVEorSME())) { 6290 return fastEmitInst_r(AArch64::REV_ZZ_S, &AArch64::ZPRRegClass, Op0); 6291 } 6292 return 0; 6293} 6294 6295unsigned fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8bf16_r(MVT RetVT, unsigned Op0) { 6296 if (RetVT.SimpleTy != MVT::nxv8bf16) 6297 return 0; 6298 if ((Subtarget->hasSVEorSME())) { 6299 return fastEmitInst_r(AArch64::REV_ZZ_H, &AArch64::ZPRRegClass, Op0); 6300 } 6301 return 0; 6302} 6303 6304unsigned fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f32_r(MVT RetVT, unsigned Op0) { 6305 if (RetVT.SimpleTy != MVT::nxv2f32) 6306 return 0; 6307 if ((Subtarget->hasSVEorSME())) { 6308 return fastEmitInst_r(AArch64::REV_ZZ_D, &AArch64::ZPRRegClass, Op0); 6309 } 6310 return 0; 6311} 6312 6313unsigned fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4f32_r(MVT RetVT, unsigned Op0) { 6314 if (RetVT.SimpleTy != MVT::nxv4f32) 6315 return 0; 6316 if ((Subtarget->hasSVEorSME())) { 6317 return fastEmitInst_r(AArch64::REV_ZZ_S, &AArch64::ZPRRegClass, Op0); 6318 } 6319 return 0; 6320} 6321 6322unsigned fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f64_r(MVT RetVT, unsigned Op0) { 6323 if (RetVT.SimpleTy != MVT::nxv2f64) 6324 return 0; 6325 if ((Subtarget->hasSVEorSME())) { 6326 return fastEmitInst_r(AArch64::REV_ZZ_D, &AArch64::ZPRRegClass, Op0); 6327 } 6328 return 0; 6329} 6330 6331unsigned fastEmit_ISD_VECTOR_REVERSE_r(MVT VT, MVT RetVT, unsigned Op0) { 6332 switch (VT.SimpleTy) { 6333 case MVT::nxv2i1: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2i1_r(RetVT, Op0); 6334 case MVT::nxv4i1: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4i1_r(RetVT, Op0); 6335 case MVT::nxv8i1: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8i1_r(RetVT, Op0); 6336 case MVT::nxv16i1: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv16i1_r(RetVT, Op0); 6337 case MVT::nxv16i8: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv16i8_r(RetVT, Op0); 6338 case MVT::nxv8i16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8i16_r(RetVT, Op0); 6339 case MVT::nxv4i32: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4i32_r(RetVT, Op0); 6340 case MVT::nxv2i64: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2i64_r(RetVT, Op0); 6341 case MVT::nxv2f16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f16_r(RetVT, Op0); 6342 case MVT::nxv4f16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4f16_r(RetVT, Op0); 6343 case MVT::nxv8f16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8f16_r(RetVT, Op0); 6344 case MVT::nxv2bf16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2bf16_r(RetVT, Op0); 6345 case MVT::nxv4bf16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4bf16_r(RetVT, Op0); 6346 case MVT::nxv8bf16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8bf16_r(RetVT, Op0); 6347 case MVT::nxv2f32: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f32_r(RetVT, Op0); 6348 case MVT::nxv4f32: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4f32_r(RetVT, Op0); 6349 case MVT::nxv2f64: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f64_r(RetVT, Op0); 6350 default: return 0; 6351 } 6352} 6353 6354// Top-level FastEmit function. 6355 6356unsigned fastEmit_r(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0) override { 6357 switch (Opcode) { 6358 case AArch64ISD::CALL: return fastEmit_AArch64ISD_CALL_r(VT, RetVT, Op0); 6359 case AArch64ISD::CMEQz: return fastEmit_AArch64ISD_CMEQz_r(VT, RetVT, Op0); 6360 case AArch64ISD::CMGEz: return fastEmit_AArch64ISD_CMGEz_r(VT, RetVT, Op0); 6361 case AArch64ISD::CMGTz: return fastEmit_AArch64ISD_CMGTz_r(VT, RetVT, Op0); 6362 case AArch64ISD::CMLEz: return fastEmit_AArch64ISD_CMLEz_r(VT, RetVT, Op0); 6363 case AArch64ISD::CMLTz: return fastEmit_AArch64ISD_CMLTz_r(VT, RetVT, Op0); 6364 case AArch64ISD::DUP: return fastEmit_AArch64ISD_DUP_r(VT, RetVT, Op0); 6365 case AArch64ISD::FCMEQz: return fastEmit_AArch64ISD_FCMEQz_r(VT, RetVT, Op0); 6366 case AArch64ISD::FCMGEz: return fastEmit_AArch64ISD_FCMGEz_r(VT, RetVT, Op0); 6367 case AArch64ISD::FCMGTz: return fastEmit_AArch64ISD_FCMGTz_r(VT, RetVT, Op0); 6368 case AArch64ISD::FCMLEz: return fastEmit_AArch64ISD_FCMLEz_r(VT, RetVT, Op0); 6369 case AArch64ISD::FCMLTz: return fastEmit_AArch64ISD_FCMLTz_r(VT, RetVT, Op0); 6370 case AArch64ISD::FRECPE: return fastEmit_AArch64ISD_FRECPE_r(VT, RetVT, Op0); 6371 case AArch64ISD::FRSQRTE: return fastEmit_AArch64ISD_FRSQRTE_r(VT, RetVT, Op0); 6372 case AArch64ISD::OBSCURE_COPY: return fastEmit_AArch64ISD_OBSCURE_COPY_r(VT, RetVT, Op0); 6373 case AArch64ISD::REV16: return fastEmit_AArch64ISD_REV16_r(VT, RetVT, Op0); 6374 case AArch64ISD::REV32: return fastEmit_AArch64ISD_REV32_r(VT, RetVT, Op0); 6375 case AArch64ISD::REV64: return fastEmit_AArch64ISD_REV64_r(VT, RetVT, Op0); 6376 case AArch64ISD::SADDLP: return fastEmit_AArch64ISD_SADDLP_r(VT, RetVT, Op0); 6377 case AArch64ISD::SITOF: return fastEmit_AArch64ISD_SITOF_r(VT, RetVT, Op0); 6378 case AArch64ISD::SUNPKHI: return fastEmit_AArch64ISD_SUNPKHI_r(VT, RetVT, Op0); 6379 case AArch64ISD::SUNPKLO: return fastEmit_AArch64ISD_SUNPKLO_r(VT, RetVT, Op0); 6380 case AArch64ISD::UADDLP: return fastEmit_AArch64ISD_UADDLP_r(VT, RetVT, Op0); 6381 case AArch64ISD::UITOF: return fastEmit_AArch64ISD_UITOF_r(VT, RetVT, Op0); 6382 case AArch64ISD::UUNPKHI: return fastEmit_AArch64ISD_UUNPKHI_r(VT, RetVT, Op0); 6383 case AArch64ISD::UUNPKLO: return fastEmit_AArch64ISD_UUNPKLO_r(VT, RetVT, Op0); 6384 case ISD::ABS: return fastEmit_ISD_ABS_r(VT, RetVT, Op0); 6385 case ISD::BITCAST: return fastEmit_ISD_BITCAST_r(VT, RetVT, Op0); 6386 case ISD::BITREVERSE: return fastEmit_ISD_BITREVERSE_r(VT, RetVT, Op0); 6387 case ISD::BRIND: return fastEmit_ISD_BRIND_r(VT, RetVT, Op0); 6388 case ISD::BSWAP: return fastEmit_ISD_BSWAP_r(VT, RetVT, Op0); 6389 case ISD::CTLZ: return fastEmit_ISD_CTLZ_r(VT, RetVT, Op0); 6390 case ISD::CTPOP: return fastEmit_ISD_CTPOP_r(VT, RetVT, Op0); 6391 case ISD::CTTZ: return fastEmit_ISD_CTTZ_r(VT, RetVT, Op0); 6392 case ISD::FABS: return fastEmit_ISD_FABS_r(VT, RetVT, Op0); 6393 case ISD::FCEIL: return fastEmit_ISD_FCEIL_r(VT, RetVT, Op0); 6394 case ISD::FFLOOR: return fastEmit_ISD_FFLOOR_r(VT, RetVT, Op0); 6395 case ISD::FNEARBYINT: return fastEmit_ISD_FNEARBYINT_r(VT, RetVT, Op0); 6396 case ISD::FNEG: return fastEmit_ISD_FNEG_r(VT, RetVT, Op0); 6397 case ISD::FP_EXTEND: return fastEmit_ISD_FP_EXTEND_r(VT, RetVT, Op0); 6398 case ISD::FP_ROUND: return fastEmit_ISD_FP_ROUND_r(VT, RetVT, Op0); 6399 case ISD::FP_TO_SINT: return fastEmit_ISD_FP_TO_SINT_r(VT, RetVT, Op0); 6400 case ISD::FP_TO_UINT: return fastEmit_ISD_FP_TO_UINT_r(VT, RetVT, Op0); 6401 case ISD::FRINT: return fastEmit_ISD_FRINT_r(VT, RetVT, Op0); 6402 case ISD::FROUND: return fastEmit_ISD_FROUND_r(VT, RetVT, Op0); 6403 case ISD::FROUNDEVEN: return fastEmit_ISD_FROUNDEVEN_r(VT, RetVT, Op0); 6404 case ISD::FSQRT: return fastEmit_ISD_FSQRT_r(VT, RetVT, Op0); 6405 case ISD::FTRUNC: return fastEmit_ISD_FTRUNC_r(VT, RetVT, Op0); 6406 case ISD::LLROUND: return fastEmit_ISD_LLROUND_r(VT, RetVT, Op0); 6407 case ISD::LROUND: return fastEmit_ISD_LROUND_r(VT, RetVT, Op0); 6408 case ISD::SINT_TO_FP: return fastEmit_ISD_SINT_TO_FP_r(VT, RetVT, Op0); 6409 case ISD::SPLAT_VECTOR: return fastEmit_ISD_SPLAT_VECTOR_r(VT, RetVT, Op0); 6410 case ISD::STRICT_FCEIL: return fastEmit_ISD_STRICT_FCEIL_r(VT, RetVT, Op0); 6411 case ISD::STRICT_FFLOOR: return fastEmit_ISD_STRICT_FFLOOR_r(VT, RetVT, Op0); 6412 case ISD::STRICT_FNEARBYINT: return fastEmit_ISD_STRICT_FNEARBYINT_r(VT, RetVT, Op0); 6413 case ISD::STRICT_FP_EXTEND: return fastEmit_ISD_STRICT_FP_EXTEND_r(VT, RetVT, Op0); 6414 case ISD::STRICT_FP_ROUND: return fastEmit_ISD_STRICT_FP_ROUND_r(VT, RetVT, Op0); 6415 case ISD::STRICT_FP_TO_SINT: return fastEmit_ISD_STRICT_FP_TO_SINT_r(VT, RetVT, Op0); 6416 case ISD::STRICT_FP_TO_UINT: return fastEmit_ISD_STRICT_FP_TO_UINT_r(VT, RetVT, Op0); 6417 case ISD::STRICT_FRINT: return fastEmit_ISD_STRICT_FRINT_r(VT, RetVT, Op0); 6418 case ISD::STRICT_FROUND: return fastEmit_ISD_STRICT_FROUND_r(VT, RetVT, Op0); 6419 case ISD::STRICT_FROUNDEVEN: return fastEmit_ISD_STRICT_FROUNDEVEN_r(VT, RetVT, Op0); 6420 case ISD::STRICT_FSQRT: return fastEmit_ISD_STRICT_FSQRT_r(VT, RetVT, Op0); 6421 case ISD::STRICT_FTRUNC: return fastEmit_ISD_STRICT_FTRUNC_r(VT, RetVT, Op0); 6422 case ISD::STRICT_LLROUND: return fastEmit_ISD_STRICT_LLROUND_r(VT, RetVT, Op0); 6423 case ISD::STRICT_LROUND: return fastEmit_ISD_STRICT_LROUND_r(VT, RetVT, Op0); 6424 case ISD::STRICT_SINT_TO_FP: return fastEmit_ISD_STRICT_SINT_TO_FP_r(VT, RetVT, Op0); 6425 case ISD::STRICT_UINT_TO_FP: return fastEmit_ISD_STRICT_UINT_TO_FP_r(VT, RetVT, Op0); 6426 case ISD::TRUNCATE: return fastEmit_ISD_TRUNCATE_r(VT, RetVT, Op0); 6427 case ISD::UINT_TO_FP: return fastEmit_ISD_UINT_TO_FP_r(VT, RetVT, Op0); 6428 case ISD::VECREDUCE_FADD: return fastEmit_ISD_VECREDUCE_FADD_r(VT, RetVT, Op0); 6429 case ISD::VECTOR_REVERSE: return fastEmit_ISD_VECTOR_REVERSE_r(VT, RetVT, Op0); 6430 default: return 0; 6431 } 6432} 6433 6434// FastEmit functions for AArch64ISD::ADDP. 6435 6436unsigned fastEmit_AArch64ISD_ADDP_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6437 if (RetVT.SimpleTy != MVT::v8i8) 6438 return 0; 6439 if ((Subtarget->hasNEON())) { 6440 return fastEmitInst_rr(AArch64::ADDPv8i8, &AArch64::FPR64RegClass, Op0, Op1); 6441 } 6442 return 0; 6443} 6444 6445unsigned fastEmit_AArch64ISD_ADDP_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6446 if (RetVT.SimpleTy != MVT::v16i8) 6447 return 0; 6448 if ((Subtarget->hasNEON())) { 6449 return fastEmitInst_rr(AArch64::ADDPv16i8, &AArch64::FPR128RegClass, Op0, Op1); 6450 } 6451 return 0; 6452} 6453 6454unsigned fastEmit_AArch64ISD_ADDP_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6455 if (RetVT.SimpleTy != MVT::v4i16) 6456 return 0; 6457 if ((Subtarget->hasNEON())) { 6458 return fastEmitInst_rr(AArch64::ADDPv4i16, &AArch64::FPR64RegClass, Op0, Op1); 6459 } 6460 return 0; 6461} 6462 6463unsigned fastEmit_AArch64ISD_ADDP_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6464 if (RetVT.SimpleTy != MVT::v8i16) 6465 return 0; 6466 if ((Subtarget->hasNEON())) { 6467 return fastEmitInst_rr(AArch64::ADDPv8i16, &AArch64::FPR128RegClass, Op0, Op1); 6468 } 6469 return 0; 6470} 6471 6472unsigned fastEmit_AArch64ISD_ADDP_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6473 if (RetVT.SimpleTy != MVT::v2i32) 6474 return 0; 6475 if ((Subtarget->hasNEON())) { 6476 return fastEmitInst_rr(AArch64::ADDPv2i32, &AArch64::FPR64RegClass, Op0, Op1); 6477 } 6478 return 0; 6479} 6480 6481unsigned fastEmit_AArch64ISD_ADDP_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6482 if (RetVT.SimpleTy != MVT::v4i32) 6483 return 0; 6484 if ((Subtarget->hasNEON())) { 6485 return fastEmitInst_rr(AArch64::ADDPv4i32, &AArch64::FPR128RegClass, Op0, Op1); 6486 } 6487 return 0; 6488} 6489 6490unsigned fastEmit_AArch64ISD_ADDP_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6491 if (RetVT.SimpleTy != MVT::v2i64) 6492 return 0; 6493 if ((Subtarget->hasNEON())) { 6494 return fastEmitInst_rr(AArch64::ADDPv2i64, &AArch64::FPR128RegClass, Op0, Op1); 6495 } 6496 return 0; 6497} 6498 6499unsigned fastEmit_AArch64ISD_ADDP_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6500 if (RetVT.SimpleTy != MVT::v4f16) 6501 return 0; 6502 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 6503 return fastEmitInst_rr(AArch64::FADDPv4f16, &AArch64::FPR64RegClass, Op0, Op1); 6504 } 6505 return 0; 6506} 6507 6508unsigned fastEmit_AArch64ISD_ADDP_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6509 if (RetVT.SimpleTy != MVT::v8f16) 6510 return 0; 6511 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 6512 return fastEmitInst_rr(AArch64::FADDPv8f16, &AArch64::FPR128RegClass, Op0, Op1); 6513 } 6514 return 0; 6515} 6516 6517unsigned fastEmit_AArch64ISD_ADDP_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6518 if (RetVT.SimpleTy != MVT::v2f32) 6519 return 0; 6520 if ((Subtarget->hasNEON())) { 6521 return fastEmitInst_rr(AArch64::FADDPv2f32, &AArch64::FPR64RegClass, Op0, Op1); 6522 } 6523 return 0; 6524} 6525 6526unsigned fastEmit_AArch64ISD_ADDP_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6527 if (RetVT.SimpleTy != MVT::v4f32) 6528 return 0; 6529 if ((Subtarget->hasNEON())) { 6530 return fastEmitInst_rr(AArch64::FADDPv4f32, &AArch64::FPR128RegClass, Op0, Op1); 6531 } 6532 return 0; 6533} 6534 6535unsigned fastEmit_AArch64ISD_ADDP_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6536 if (RetVT.SimpleTy != MVT::v2f64) 6537 return 0; 6538 if ((Subtarget->hasNEON())) { 6539 return fastEmitInst_rr(AArch64::FADDPv2f64, &AArch64::FPR128RegClass, Op0, Op1); 6540 } 6541 return 0; 6542} 6543 6544unsigned fastEmit_AArch64ISD_ADDP_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 6545 switch (VT.SimpleTy) { 6546 case MVT::v8i8: return fastEmit_AArch64ISD_ADDP_MVT_v8i8_rr(RetVT, Op0, Op1); 6547 case MVT::v16i8: return fastEmit_AArch64ISD_ADDP_MVT_v16i8_rr(RetVT, Op0, Op1); 6548 case MVT::v4i16: return fastEmit_AArch64ISD_ADDP_MVT_v4i16_rr(RetVT, Op0, Op1); 6549 case MVT::v8i16: return fastEmit_AArch64ISD_ADDP_MVT_v8i16_rr(RetVT, Op0, Op1); 6550 case MVT::v2i32: return fastEmit_AArch64ISD_ADDP_MVT_v2i32_rr(RetVT, Op0, Op1); 6551 case MVT::v4i32: return fastEmit_AArch64ISD_ADDP_MVT_v4i32_rr(RetVT, Op0, Op1); 6552 case MVT::v2i64: return fastEmit_AArch64ISD_ADDP_MVT_v2i64_rr(RetVT, Op0, Op1); 6553 case MVT::v4f16: return fastEmit_AArch64ISD_ADDP_MVT_v4f16_rr(RetVT, Op0, Op1); 6554 case MVT::v8f16: return fastEmit_AArch64ISD_ADDP_MVT_v8f16_rr(RetVT, Op0, Op1); 6555 case MVT::v2f32: return fastEmit_AArch64ISD_ADDP_MVT_v2f32_rr(RetVT, Op0, Op1); 6556 case MVT::v4f32: return fastEmit_AArch64ISD_ADDP_MVT_v4f32_rr(RetVT, Op0, Op1); 6557 case MVT::v2f64: return fastEmit_AArch64ISD_ADDP_MVT_v2f64_rr(RetVT, Op0, Op1); 6558 default: return 0; 6559 } 6560} 6561 6562// FastEmit functions for AArch64ISD::BIC. 6563 6564unsigned fastEmit_AArch64ISD_BIC_MVT_nxv16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6565 if (RetVT.SimpleTy != MVT::nxv16i8) 6566 return 0; 6567 if ((Subtarget->hasSVEorSME())) { 6568 return fastEmitInst_rr(AArch64::BIC_ZZZ, &AArch64::ZPRRegClass, Op0, Op1); 6569 } 6570 return 0; 6571} 6572 6573unsigned fastEmit_AArch64ISD_BIC_MVT_nxv8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6574 if (RetVT.SimpleTy != MVT::nxv8i16) 6575 return 0; 6576 if ((Subtarget->hasSVEorSME())) { 6577 return fastEmitInst_rr(AArch64::BIC_ZZZ, &AArch64::ZPRRegClass, Op0, Op1); 6578 } 6579 return 0; 6580} 6581 6582unsigned fastEmit_AArch64ISD_BIC_MVT_nxv4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6583 if (RetVT.SimpleTy != MVT::nxv4i32) 6584 return 0; 6585 if ((Subtarget->hasSVEorSME())) { 6586 return fastEmitInst_rr(AArch64::BIC_ZZZ, &AArch64::ZPRRegClass, Op0, Op1); 6587 } 6588 return 0; 6589} 6590 6591unsigned fastEmit_AArch64ISD_BIC_MVT_nxv2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6592 if (RetVT.SimpleTy != MVT::nxv2i64) 6593 return 0; 6594 if ((Subtarget->hasSVEorSME())) { 6595 return fastEmitInst_rr(AArch64::BIC_ZZZ, &AArch64::ZPRRegClass, Op0, Op1); 6596 } 6597 return 0; 6598} 6599 6600unsigned fastEmit_AArch64ISD_BIC_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 6601 switch (VT.SimpleTy) { 6602 case MVT::nxv16i8: return fastEmit_AArch64ISD_BIC_MVT_nxv16i8_rr(RetVT, Op0, Op1); 6603 case MVT::nxv8i16: return fastEmit_AArch64ISD_BIC_MVT_nxv8i16_rr(RetVT, Op0, Op1); 6604 case MVT::nxv4i32: return fastEmit_AArch64ISD_BIC_MVT_nxv4i32_rr(RetVT, Op0, Op1); 6605 case MVT::nxv2i64: return fastEmit_AArch64ISD_BIC_MVT_nxv2i64_rr(RetVT, Op0, Op1); 6606 default: return 0; 6607 } 6608} 6609 6610// FastEmit functions for AArch64ISD::CMEQ. 6611 6612unsigned fastEmit_AArch64ISD_CMEQ_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6613 if (RetVT.SimpleTy != MVT::v8i8) 6614 return 0; 6615 if ((Subtarget->hasNEON())) { 6616 return fastEmitInst_rr(AArch64::CMEQv8i8, &AArch64::FPR64RegClass, Op0, Op1); 6617 } 6618 return 0; 6619} 6620 6621unsigned fastEmit_AArch64ISD_CMEQ_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6622 if (RetVT.SimpleTy != MVT::v16i8) 6623 return 0; 6624 if ((Subtarget->hasNEON())) { 6625 return fastEmitInst_rr(AArch64::CMEQv16i8, &AArch64::FPR128RegClass, Op0, Op1); 6626 } 6627 return 0; 6628} 6629 6630unsigned fastEmit_AArch64ISD_CMEQ_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6631 if (RetVT.SimpleTy != MVT::v4i16) 6632 return 0; 6633 if ((Subtarget->hasNEON())) { 6634 return fastEmitInst_rr(AArch64::CMEQv4i16, &AArch64::FPR64RegClass, Op0, Op1); 6635 } 6636 return 0; 6637} 6638 6639unsigned fastEmit_AArch64ISD_CMEQ_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6640 if (RetVT.SimpleTy != MVT::v8i16) 6641 return 0; 6642 if ((Subtarget->hasNEON())) { 6643 return fastEmitInst_rr(AArch64::CMEQv8i16, &AArch64::FPR128RegClass, Op0, Op1); 6644 } 6645 return 0; 6646} 6647 6648unsigned fastEmit_AArch64ISD_CMEQ_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6649 if (RetVT.SimpleTy != MVT::v2i32) 6650 return 0; 6651 if ((Subtarget->hasNEON())) { 6652 return fastEmitInst_rr(AArch64::CMEQv2i32, &AArch64::FPR64RegClass, Op0, Op1); 6653 } 6654 return 0; 6655} 6656 6657unsigned fastEmit_AArch64ISD_CMEQ_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6658 if (RetVT.SimpleTy != MVT::v4i32) 6659 return 0; 6660 if ((Subtarget->hasNEON())) { 6661 return fastEmitInst_rr(AArch64::CMEQv4i32, &AArch64::FPR128RegClass, Op0, Op1); 6662 } 6663 return 0; 6664} 6665 6666unsigned fastEmit_AArch64ISD_CMEQ_MVT_v1i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6667 if (RetVT.SimpleTy != MVT::v1i64) 6668 return 0; 6669 if ((Subtarget->hasNEON())) { 6670 return fastEmitInst_rr(AArch64::CMEQv1i64, &AArch64::FPR64RegClass, Op0, Op1); 6671 } 6672 return 0; 6673} 6674 6675unsigned fastEmit_AArch64ISD_CMEQ_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6676 if (RetVT.SimpleTy != MVT::v2i64) 6677 return 0; 6678 if ((Subtarget->hasNEON())) { 6679 return fastEmitInst_rr(AArch64::CMEQv2i64, &AArch64::FPR128RegClass, Op0, Op1); 6680 } 6681 return 0; 6682} 6683 6684unsigned fastEmit_AArch64ISD_CMEQ_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 6685 switch (VT.SimpleTy) { 6686 case MVT::v8i8: return fastEmit_AArch64ISD_CMEQ_MVT_v8i8_rr(RetVT, Op0, Op1); 6687 case MVT::v16i8: return fastEmit_AArch64ISD_CMEQ_MVT_v16i8_rr(RetVT, Op0, Op1); 6688 case MVT::v4i16: return fastEmit_AArch64ISD_CMEQ_MVT_v4i16_rr(RetVT, Op0, Op1); 6689 case MVT::v8i16: return fastEmit_AArch64ISD_CMEQ_MVT_v8i16_rr(RetVT, Op0, Op1); 6690 case MVT::v2i32: return fastEmit_AArch64ISD_CMEQ_MVT_v2i32_rr(RetVT, Op0, Op1); 6691 case MVT::v4i32: return fastEmit_AArch64ISD_CMEQ_MVT_v4i32_rr(RetVT, Op0, Op1); 6692 case MVT::v1i64: return fastEmit_AArch64ISD_CMEQ_MVT_v1i64_rr(RetVT, Op0, Op1); 6693 case MVT::v2i64: return fastEmit_AArch64ISD_CMEQ_MVT_v2i64_rr(RetVT, Op0, Op1); 6694 default: return 0; 6695 } 6696} 6697 6698// FastEmit functions for AArch64ISD::CMGE. 6699 6700unsigned fastEmit_AArch64ISD_CMGE_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6701 if (RetVT.SimpleTy != MVT::v8i8) 6702 return 0; 6703 if ((Subtarget->hasNEON())) { 6704 return fastEmitInst_rr(AArch64::CMGEv8i8, &AArch64::FPR64RegClass, Op0, Op1); 6705 } 6706 return 0; 6707} 6708 6709unsigned fastEmit_AArch64ISD_CMGE_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6710 if (RetVT.SimpleTy != MVT::v16i8) 6711 return 0; 6712 if ((Subtarget->hasNEON())) { 6713 return fastEmitInst_rr(AArch64::CMGEv16i8, &AArch64::FPR128RegClass, Op0, Op1); 6714 } 6715 return 0; 6716} 6717 6718unsigned fastEmit_AArch64ISD_CMGE_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6719 if (RetVT.SimpleTy != MVT::v4i16) 6720 return 0; 6721 if ((Subtarget->hasNEON())) { 6722 return fastEmitInst_rr(AArch64::CMGEv4i16, &AArch64::FPR64RegClass, Op0, Op1); 6723 } 6724 return 0; 6725} 6726 6727unsigned fastEmit_AArch64ISD_CMGE_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6728 if (RetVT.SimpleTy != MVT::v8i16) 6729 return 0; 6730 if ((Subtarget->hasNEON())) { 6731 return fastEmitInst_rr(AArch64::CMGEv8i16, &AArch64::FPR128RegClass, Op0, Op1); 6732 } 6733 return 0; 6734} 6735 6736unsigned fastEmit_AArch64ISD_CMGE_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6737 if (RetVT.SimpleTy != MVT::v2i32) 6738 return 0; 6739 if ((Subtarget->hasNEON())) { 6740 return fastEmitInst_rr(AArch64::CMGEv2i32, &AArch64::FPR64RegClass, Op0, Op1); 6741 } 6742 return 0; 6743} 6744 6745unsigned fastEmit_AArch64ISD_CMGE_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6746 if (RetVT.SimpleTy != MVT::v4i32) 6747 return 0; 6748 if ((Subtarget->hasNEON())) { 6749 return fastEmitInst_rr(AArch64::CMGEv4i32, &AArch64::FPR128RegClass, Op0, Op1); 6750 } 6751 return 0; 6752} 6753 6754unsigned fastEmit_AArch64ISD_CMGE_MVT_v1i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6755 if (RetVT.SimpleTy != MVT::v1i64) 6756 return 0; 6757 if ((Subtarget->hasNEON())) { 6758 return fastEmitInst_rr(AArch64::CMGEv1i64, &AArch64::FPR64RegClass, Op0, Op1); 6759 } 6760 return 0; 6761} 6762 6763unsigned fastEmit_AArch64ISD_CMGE_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6764 if (RetVT.SimpleTy != MVT::v2i64) 6765 return 0; 6766 if ((Subtarget->hasNEON())) { 6767 return fastEmitInst_rr(AArch64::CMGEv2i64, &AArch64::FPR128RegClass, Op0, Op1); 6768 } 6769 return 0; 6770} 6771 6772unsigned fastEmit_AArch64ISD_CMGE_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 6773 switch (VT.SimpleTy) { 6774 case MVT::v8i8: return fastEmit_AArch64ISD_CMGE_MVT_v8i8_rr(RetVT, Op0, Op1); 6775 case MVT::v16i8: return fastEmit_AArch64ISD_CMGE_MVT_v16i8_rr(RetVT, Op0, Op1); 6776 case MVT::v4i16: return fastEmit_AArch64ISD_CMGE_MVT_v4i16_rr(RetVT, Op0, Op1); 6777 case MVT::v8i16: return fastEmit_AArch64ISD_CMGE_MVT_v8i16_rr(RetVT, Op0, Op1); 6778 case MVT::v2i32: return fastEmit_AArch64ISD_CMGE_MVT_v2i32_rr(RetVT, Op0, Op1); 6779 case MVT::v4i32: return fastEmit_AArch64ISD_CMGE_MVT_v4i32_rr(RetVT, Op0, Op1); 6780 case MVT::v1i64: return fastEmit_AArch64ISD_CMGE_MVT_v1i64_rr(RetVT, Op0, Op1); 6781 case MVT::v2i64: return fastEmit_AArch64ISD_CMGE_MVT_v2i64_rr(RetVT, Op0, Op1); 6782 default: return 0; 6783 } 6784} 6785 6786// FastEmit functions for AArch64ISD::CMGT. 6787 6788unsigned fastEmit_AArch64ISD_CMGT_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6789 if (RetVT.SimpleTy != MVT::v8i8) 6790 return 0; 6791 if ((Subtarget->hasNEON())) { 6792 return fastEmitInst_rr(AArch64::CMGTv8i8, &AArch64::FPR64RegClass, Op0, Op1); 6793 } 6794 return 0; 6795} 6796 6797unsigned fastEmit_AArch64ISD_CMGT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6798 if (RetVT.SimpleTy != MVT::v16i8) 6799 return 0; 6800 if ((Subtarget->hasNEON())) { 6801 return fastEmitInst_rr(AArch64::CMGTv16i8, &AArch64::FPR128RegClass, Op0, Op1); 6802 } 6803 return 0; 6804} 6805 6806unsigned fastEmit_AArch64ISD_CMGT_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6807 if (RetVT.SimpleTy != MVT::v4i16) 6808 return 0; 6809 if ((Subtarget->hasNEON())) { 6810 return fastEmitInst_rr(AArch64::CMGTv4i16, &AArch64::FPR64RegClass, Op0, Op1); 6811 } 6812 return 0; 6813} 6814 6815unsigned fastEmit_AArch64ISD_CMGT_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6816 if (RetVT.SimpleTy != MVT::v8i16) 6817 return 0; 6818 if ((Subtarget->hasNEON())) { 6819 return fastEmitInst_rr(AArch64::CMGTv8i16, &AArch64::FPR128RegClass, Op0, Op1); 6820 } 6821 return 0; 6822} 6823 6824unsigned fastEmit_AArch64ISD_CMGT_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6825 if (RetVT.SimpleTy != MVT::v2i32) 6826 return 0; 6827 if ((Subtarget->hasNEON())) { 6828 return fastEmitInst_rr(AArch64::CMGTv2i32, &AArch64::FPR64RegClass, Op0, Op1); 6829 } 6830 return 0; 6831} 6832 6833unsigned fastEmit_AArch64ISD_CMGT_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6834 if (RetVT.SimpleTy != MVT::v4i32) 6835 return 0; 6836 if ((Subtarget->hasNEON())) { 6837 return fastEmitInst_rr(AArch64::CMGTv4i32, &AArch64::FPR128RegClass, Op0, Op1); 6838 } 6839 return 0; 6840} 6841 6842unsigned fastEmit_AArch64ISD_CMGT_MVT_v1i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6843 if (RetVT.SimpleTy != MVT::v1i64) 6844 return 0; 6845 if ((Subtarget->hasNEON())) { 6846 return fastEmitInst_rr(AArch64::CMGTv1i64, &AArch64::FPR64RegClass, Op0, Op1); 6847 } 6848 return 0; 6849} 6850 6851unsigned fastEmit_AArch64ISD_CMGT_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6852 if (RetVT.SimpleTy != MVT::v2i64) 6853 return 0; 6854 if ((Subtarget->hasNEON())) { 6855 return fastEmitInst_rr(AArch64::CMGTv2i64, &AArch64::FPR128RegClass, Op0, Op1); 6856 } 6857 return 0; 6858} 6859 6860unsigned fastEmit_AArch64ISD_CMGT_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 6861 switch (VT.SimpleTy) { 6862 case MVT::v8i8: return fastEmit_AArch64ISD_CMGT_MVT_v8i8_rr(RetVT, Op0, Op1); 6863 case MVT::v16i8: return fastEmit_AArch64ISD_CMGT_MVT_v16i8_rr(RetVT, Op0, Op1); 6864 case MVT::v4i16: return fastEmit_AArch64ISD_CMGT_MVT_v4i16_rr(RetVT, Op0, Op1); 6865 case MVT::v8i16: return fastEmit_AArch64ISD_CMGT_MVT_v8i16_rr(RetVT, Op0, Op1); 6866 case MVT::v2i32: return fastEmit_AArch64ISD_CMGT_MVT_v2i32_rr(RetVT, Op0, Op1); 6867 case MVT::v4i32: return fastEmit_AArch64ISD_CMGT_MVT_v4i32_rr(RetVT, Op0, Op1); 6868 case MVT::v1i64: return fastEmit_AArch64ISD_CMGT_MVT_v1i64_rr(RetVT, Op0, Op1); 6869 case MVT::v2i64: return fastEmit_AArch64ISD_CMGT_MVT_v2i64_rr(RetVT, Op0, Op1); 6870 default: return 0; 6871 } 6872} 6873 6874// FastEmit functions for AArch64ISD::CMHI. 6875 6876unsigned fastEmit_AArch64ISD_CMHI_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6877 if (RetVT.SimpleTy != MVT::v8i8) 6878 return 0; 6879 if ((Subtarget->hasNEON())) { 6880 return fastEmitInst_rr(AArch64::CMHIv8i8, &AArch64::FPR64RegClass, Op0, Op1); 6881 } 6882 return 0; 6883} 6884 6885unsigned fastEmit_AArch64ISD_CMHI_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6886 if (RetVT.SimpleTy != MVT::v16i8) 6887 return 0; 6888 if ((Subtarget->hasNEON())) { 6889 return fastEmitInst_rr(AArch64::CMHIv16i8, &AArch64::FPR128RegClass, Op0, Op1); 6890 } 6891 return 0; 6892} 6893 6894unsigned fastEmit_AArch64ISD_CMHI_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6895 if (RetVT.SimpleTy != MVT::v4i16) 6896 return 0; 6897 if ((Subtarget->hasNEON())) { 6898 return fastEmitInst_rr(AArch64::CMHIv4i16, &AArch64::FPR64RegClass, Op0, Op1); 6899 } 6900 return 0; 6901} 6902 6903unsigned fastEmit_AArch64ISD_CMHI_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6904 if (RetVT.SimpleTy != MVT::v8i16) 6905 return 0; 6906 if ((Subtarget->hasNEON())) { 6907 return fastEmitInst_rr(AArch64::CMHIv8i16, &AArch64::FPR128RegClass, Op0, Op1); 6908 } 6909 return 0; 6910} 6911 6912unsigned fastEmit_AArch64ISD_CMHI_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6913 if (RetVT.SimpleTy != MVT::v2i32) 6914 return 0; 6915 if ((Subtarget->hasNEON())) { 6916 return fastEmitInst_rr(AArch64::CMHIv2i32, &AArch64::FPR64RegClass, Op0, Op1); 6917 } 6918 return 0; 6919} 6920 6921unsigned fastEmit_AArch64ISD_CMHI_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6922 if (RetVT.SimpleTy != MVT::v4i32) 6923 return 0; 6924 if ((Subtarget->hasNEON())) { 6925 return fastEmitInst_rr(AArch64::CMHIv4i32, &AArch64::FPR128RegClass, Op0, Op1); 6926 } 6927 return 0; 6928} 6929 6930unsigned fastEmit_AArch64ISD_CMHI_MVT_v1i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6931 if (RetVT.SimpleTy != MVT::v1i64) 6932 return 0; 6933 if ((Subtarget->hasNEON())) { 6934 return fastEmitInst_rr(AArch64::CMHIv1i64, &AArch64::FPR64RegClass, Op0, Op1); 6935 } 6936 return 0; 6937} 6938 6939unsigned fastEmit_AArch64ISD_CMHI_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6940 if (RetVT.SimpleTy != MVT::v2i64) 6941 return 0; 6942 if ((Subtarget->hasNEON())) { 6943 return fastEmitInst_rr(AArch64::CMHIv2i64, &AArch64::FPR128RegClass, Op0, Op1); 6944 } 6945 return 0; 6946} 6947 6948unsigned fastEmit_AArch64ISD_CMHI_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 6949 switch (VT.SimpleTy) { 6950 case MVT::v8i8: return fastEmit_AArch64ISD_CMHI_MVT_v8i8_rr(RetVT, Op0, Op1); 6951 case MVT::v16i8: return fastEmit_AArch64ISD_CMHI_MVT_v16i8_rr(RetVT, Op0, Op1); 6952 case MVT::v4i16: return fastEmit_AArch64ISD_CMHI_MVT_v4i16_rr(RetVT, Op0, Op1); 6953 case MVT::v8i16: return fastEmit_AArch64ISD_CMHI_MVT_v8i16_rr(RetVT, Op0, Op1); 6954 case MVT::v2i32: return fastEmit_AArch64ISD_CMHI_MVT_v2i32_rr(RetVT, Op0, Op1); 6955 case MVT::v4i32: return fastEmit_AArch64ISD_CMHI_MVT_v4i32_rr(RetVT, Op0, Op1); 6956 case MVT::v1i64: return fastEmit_AArch64ISD_CMHI_MVT_v1i64_rr(RetVT, Op0, Op1); 6957 case MVT::v2i64: return fastEmit_AArch64ISD_CMHI_MVT_v2i64_rr(RetVT, Op0, Op1); 6958 default: return 0; 6959 } 6960} 6961 6962// FastEmit functions for AArch64ISD::CMHS. 6963 6964unsigned fastEmit_AArch64ISD_CMHS_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6965 if (RetVT.SimpleTy != MVT::v8i8) 6966 return 0; 6967 if ((Subtarget->hasNEON())) { 6968 return fastEmitInst_rr(AArch64::CMHSv8i8, &AArch64::FPR64RegClass, Op0, Op1); 6969 } 6970 return 0; 6971} 6972 6973unsigned fastEmit_AArch64ISD_CMHS_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6974 if (RetVT.SimpleTy != MVT::v16i8) 6975 return 0; 6976 if ((Subtarget->hasNEON())) { 6977 return fastEmitInst_rr(AArch64::CMHSv16i8, &AArch64::FPR128RegClass, Op0, Op1); 6978 } 6979 return 0; 6980} 6981 6982unsigned fastEmit_AArch64ISD_CMHS_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6983 if (RetVT.SimpleTy != MVT::v4i16) 6984 return 0; 6985 if ((Subtarget->hasNEON())) { 6986 return fastEmitInst_rr(AArch64::CMHSv4i16, &AArch64::FPR64RegClass, Op0, Op1); 6987 } 6988 return 0; 6989} 6990 6991unsigned fastEmit_AArch64ISD_CMHS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 6992 if (RetVT.SimpleTy != MVT::v8i16) 6993 return 0; 6994 if ((Subtarget->hasNEON())) { 6995 return fastEmitInst_rr(AArch64::CMHSv8i16, &AArch64::FPR128RegClass, Op0, Op1); 6996 } 6997 return 0; 6998} 6999 7000unsigned fastEmit_AArch64ISD_CMHS_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7001 if (RetVT.SimpleTy != MVT::v2i32) 7002 return 0; 7003 if ((Subtarget->hasNEON())) { 7004 return fastEmitInst_rr(AArch64::CMHSv2i32, &AArch64::FPR64RegClass, Op0, Op1); 7005 } 7006 return 0; 7007} 7008 7009unsigned fastEmit_AArch64ISD_CMHS_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7010 if (RetVT.SimpleTy != MVT::v4i32) 7011 return 0; 7012 if ((Subtarget->hasNEON())) { 7013 return fastEmitInst_rr(AArch64::CMHSv4i32, &AArch64::FPR128RegClass, Op0, Op1); 7014 } 7015 return 0; 7016} 7017 7018unsigned fastEmit_AArch64ISD_CMHS_MVT_v1i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7019 if (RetVT.SimpleTy != MVT::v1i64) 7020 return 0; 7021 if ((Subtarget->hasNEON())) { 7022 return fastEmitInst_rr(AArch64::CMHSv1i64, &AArch64::FPR64RegClass, Op0, Op1); 7023 } 7024 return 0; 7025} 7026 7027unsigned fastEmit_AArch64ISD_CMHS_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7028 if (RetVT.SimpleTy != MVT::v2i64) 7029 return 0; 7030 if ((Subtarget->hasNEON())) { 7031 return fastEmitInst_rr(AArch64::CMHSv2i64, &AArch64::FPR128RegClass, Op0, Op1); 7032 } 7033 return 0; 7034} 7035 7036unsigned fastEmit_AArch64ISD_CMHS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 7037 switch (VT.SimpleTy) { 7038 case MVT::v8i8: return fastEmit_AArch64ISD_CMHS_MVT_v8i8_rr(RetVT, Op0, Op1); 7039 case MVT::v16i8: return fastEmit_AArch64ISD_CMHS_MVT_v16i8_rr(RetVT, Op0, Op1); 7040 case MVT::v4i16: return fastEmit_AArch64ISD_CMHS_MVT_v4i16_rr(RetVT, Op0, Op1); 7041 case MVT::v8i16: return fastEmit_AArch64ISD_CMHS_MVT_v8i16_rr(RetVT, Op0, Op1); 7042 case MVT::v2i32: return fastEmit_AArch64ISD_CMHS_MVT_v2i32_rr(RetVT, Op0, Op1); 7043 case MVT::v4i32: return fastEmit_AArch64ISD_CMHS_MVT_v4i32_rr(RetVT, Op0, Op1); 7044 case MVT::v1i64: return fastEmit_AArch64ISD_CMHS_MVT_v1i64_rr(RetVT, Op0, Op1); 7045 case MVT::v2i64: return fastEmit_AArch64ISD_CMHS_MVT_v2i64_rr(RetVT, Op0, Op1); 7046 default: return 0; 7047 } 7048} 7049 7050// FastEmit functions for AArch64ISD::FCMEQ. 7051 7052unsigned fastEmit_AArch64ISD_FCMEQ_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7053 if (RetVT.SimpleTy != MVT::i32) 7054 return 0; 7055 if ((Subtarget->hasNEON())) { 7056 return fastEmitInst_rr(AArch64::FCMEQ32, &AArch64::FPR32RegClass, Op0, Op1); 7057 } 7058 return 0; 7059} 7060 7061unsigned fastEmit_AArch64ISD_FCMEQ_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7062 if (RetVT.SimpleTy != MVT::i64) 7063 return 0; 7064 if ((Subtarget->hasNEON())) { 7065 return fastEmitInst_rr(AArch64::FCMEQ64, &AArch64::FPR64RegClass, Op0, Op1); 7066 } 7067 return 0; 7068} 7069 7070unsigned fastEmit_AArch64ISD_FCMEQ_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7071 if (RetVT.SimpleTy != MVT::v4i16) 7072 return 0; 7073 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 7074 return fastEmitInst_rr(AArch64::FCMEQv4f16, &AArch64::FPR64RegClass, Op0, Op1); 7075 } 7076 return 0; 7077} 7078 7079unsigned fastEmit_AArch64ISD_FCMEQ_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7080 if (RetVT.SimpleTy != MVT::v8i16) 7081 return 0; 7082 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 7083 return fastEmitInst_rr(AArch64::FCMEQv8f16, &AArch64::FPR128RegClass, Op0, Op1); 7084 } 7085 return 0; 7086} 7087 7088unsigned fastEmit_AArch64ISD_FCMEQ_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7089 if (RetVT.SimpleTy != MVT::v2i32) 7090 return 0; 7091 if ((Subtarget->hasNEON())) { 7092 return fastEmitInst_rr(AArch64::FCMEQv2f32, &AArch64::FPR64RegClass, Op0, Op1); 7093 } 7094 return 0; 7095} 7096 7097unsigned fastEmit_AArch64ISD_FCMEQ_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7098 if (RetVT.SimpleTy != MVT::v4i32) 7099 return 0; 7100 if ((Subtarget->hasNEON())) { 7101 return fastEmitInst_rr(AArch64::FCMEQv4f32, &AArch64::FPR128RegClass, Op0, Op1); 7102 } 7103 return 0; 7104} 7105 7106unsigned fastEmit_AArch64ISD_FCMEQ_MVT_v1f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7107 if (RetVT.SimpleTy != MVT::v1i64) 7108 return 0; 7109 if ((Subtarget->hasNEON())) { 7110 return fastEmitInst_rr(AArch64::FCMEQ64, &AArch64::FPR64RegClass, Op0, Op1); 7111 } 7112 return 0; 7113} 7114 7115unsigned fastEmit_AArch64ISD_FCMEQ_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7116 if (RetVT.SimpleTy != MVT::v2i64) 7117 return 0; 7118 if ((Subtarget->hasNEON())) { 7119 return fastEmitInst_rr(AArch64::FCMEQv2f64, &AArch64::FPR128RegClass, Op0, Op1); 7120 } 7121 return 0; 7122} 7123 7124unsigned fastEmit_AArch64ISD_FCMEQ_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 7125 switch (VT.SimpleTy) { 7126 case MVT::f32: return fastEmit_AArch64ISD_FCMEQ_MVT_f32_rr(RetVT, Op0, Op1); 7127 case MVT::f64: return fastEmit_AArch64ISD_FCMEQ_MVT_f64_rr(RetVT, Op0, Op1); 7128 case MVT::v4f16: return fastEmit_AArch64ISD_FCMEQ_MVT_v4f16_rr(RetVT, Op0, Op1); 7129 case MVT::v8f16: return fastEmit_AArch64ISD_FCMEQ_MVT_v8f16_rr(RetVT, Op0, Op1); 7130 case MVT::v2f32: return fastEmit_AArch64ISD_FCMEQ_MVT_v2f32_rr(RetVT, Op0, Op1); 7131 case MVT::v4f32: return fastEmit_AArch64ISD_FCMEQ_MVT_v4f32_rr(RetVT, Op0, Op1); 7132 case MVT::v1f64: return fastEmit_AArch64ISD_FCMEQ_MVT_v1f64_rr(RetVT, Op0, Op1); 7133 case MVT::v2f64: return fastEmit_AArch64ISD_FCMEQ_MVT_v2f64_rr(RetVT, Op0, Op1); 7134 default: return 0; 7135 } 7136} 7137 7138// FastEmit functions for AArch64ISD::FCMGE. 7139 7140unsigned fastEmit_AArch64ISD_FCMGE_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7141 if (RetVT.SimpleTy != MVT::i32) 7142 return 0; 7143 if ((Subtarget->hasNEON())) { 7144 return fastEmitInst_rr(AArch64::FCMGE32, &AArch64::FPR32RegClass, Op0, Op1); 7145 } 7146 return 0; 7147} 7148 7149unsigned fastEmit_AArch64ISD_FCMGE_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7150 if (RetVT.SimpleTy != MVT::i64) 7151 return 0; 7152 if ((Subtarget->hasNEON())) { 7153 return fastEmitInst_rr(AArch64::FCMGE64, &AArch64::FPR64RegClass, Op0, Op1); 7154 } 7155 return 0; 7156} 7157 7158unsigned fastEmit_AArch64ISD_FCMGE_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7159 if (RetVT.SimpleTy != MVT::v4i16) 7160 return 0; 7161 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 7162 return fastEmitInst_rr(AArch64::FCMGEv4f16, &AArch64::FPR64RegClass, Op0, Op1); 7163 } 7164 return 0; 7165} 7166 7167unsigned fastEmit_AArch64ISD_FCMGE_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7168 if (RetVT.SimpleTy != MVT::v8i16) 7169 return 0; 7170 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 7171 return fastEmitInst_rr(AArch64::FCMGEv8f16, &AArch64::FPR128RegClass, Op0, Op1); 7172 } 7173 return 0; 7174} 7175 7176unsigned fastEmit_AArch64ISD_FCMGE_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7177 if (RetVT.SimpleTy != MVT::v2i32) 7178 return 0; 7179 if ((Subtarget->hasNEON())) { 7180 return fastEmitInst_rr(AArch64::FCMGEv2f32, &AArch64::FPR64RegClass, Op0, Op1); 7181 } 7182 return 0; 7183} 7184 7185unsigned fastEmit_AArch64ISD_FCMGE_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7186 if (RetVT.SimpleTy != MVT::v4i32) 7187 return 0; 7188 if ((Subtarget->hasNEON())) { 7189 return fastEmitInst_rr(AArch64::FCMGEv4f32, &AArch64::FPR128RegClass, Op0, Op1); 7190 } 7191 return 0; 7192} 7193 7194unsigned fastEmit_AArch64ISD_FCMGE_MVT_v1f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7195 if (RetVT.SimpleTy != MVT::v1i64) 7196 return 0; 7197 if ((Subtarget->hasNEON())) { 7198 return fastEmitInst_rr(AArch64::FCMGE64, &AArch64::FPR64RegClass, Op0, Op1); 7199 } 7200 return 0; 7201} 7202 7203unsigned fastEmit_AArch64ISD_FCMGE_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7204 if (RetVT.SimpleTy != MVT::v2i64) 7205 return 0; 7206 if ((Subtarget->hasNEON())) { 7207 return fastEmitInst_rr(AArch64::FCMGEv2f64, &AArch64::FPR128RegClass, Op0, Op1); 7208 } 7209 return 0; 7210} 7211 7212unsigned fastEmit_AArch64ISD_FCMGE_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 7213 switch (VT.SimpleTy) { 7214 case MVT::f32: return fastEmit_AArch64ISD_FCMGE_MVT_f32_rr(RetVT, Op0, Op1); 7215 case MVT::f64: return fastEmit_AArch64ISD_FCMGE_MVT_f64_rr(RetVT, Op0, Op1); 7216 case MVT::v4f16: return fastEmit_AArch64ISD_FCMGE_MVT_v4f16_rr(RetVT, Op0, Op1); 7217 case MVT::v8f16: return fastEmit_AArch64ISD_FCMGE_MVT_v8f16_rr(RetVT, Op0, Op1); 7218 case MVT::v2f32: return fastEmit_AArch64ISD_FCMGE_MVT_v2f32_rr(RetVT, Op0, Op1); 7219 case MVT::v4f32: return fastEmit_AArch64ISD_FCMGE_MVT_v4f32_rr(RetVT, Op0, Op1); 7220 case MVT::v1f64: return fastEmit_AArch64ISD_FCMGE_MVT_v1f64_rr(RetVT, Op0, Op1); 7221 case MVT::v2f64: return fastEmit_AArch64ISD_FCMGE_MVT_v2f64_rr(RetVT, Op0, Op1); 7222 default: return 0; 7223 } 7224} 7225 7226// FastEmit functions for AArch64ISD::FCMGT. 7227 7228unsigned fastEmit_AArch64ISD_FCMGT_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7229 if (RetVT.SimpleTy != MVT::i32) 7230 return 0; 7231 if ((Subtarget->hasNEON())) { 7232 return fastEmitInst_rr(AArch64::FCMGT32, &AArch64::FPR32RegClass, Op0, Op1); 7233 } 7234 return 0; 7235} 7236 7237unsigned fastEmit_AArch64ISD_FCMGT_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7238 if (RetVT.SimpleTy != MVT::i64) 7239 return 0; 7240 if ((Subtarget->hasNEON())) { 7241 return fastEmitInst_rr(AArch64::FCMGT64, &AArch64::FPR64RegClass, Op0, Op1); 7242 } 7243 return 0; 7244} 7245 7246unsigned fastEmit_AArch64ISD_FCMGT_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7247 if (RetVT.SimpleTy != MVT::v4i16) 7248 return 0; 7249 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 7250 return fastEmitInst_rr(AArch64::FCMGTv4f16, &AArch64::FPR64RegClass, Op0, Op1); 7251 } 7252 return 0; 7253} 7254 7255unsigned fastEmit_AArch64ISD_FCMGT_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7256 if (RetVT.SimpleTy != MVT::v8i16) 7257 return 0; 7258 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 7259 return fastEmitInst_rr(AArch64::FCMGTv8f16, &AArch64::FPR128RegClass, Op0, Op1); 7260 } 7261 return 0; 7262} 7263 7264unsigned fastEmit_AArch64ISD_FCMGT_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7265 if (RetVT.SimpleTy != MVT::v2i32) 7266 return 0; 7267 if ((Subtarget->hasNEON())) { 7268 return fastEmitInst_rr(AArch64::FCMGTv2f32, &AArch64::FPR64RegClass, Op0, Op1); 7269 } 7270 return 0; 7271} 7272 7273unsigned fastEmit_AArch64ISD_FCMGT_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7274 if (RetVT.SimpleTy != MVT::v4i32) 7275 return 0; 7276 if ((Subtarget->hasNEON())) { 7277 return fastEmitInst_rr(AArch64::FCMGTv4f32, &AArch64::FPR128RegClass, Op0, Op1); 7278 } 7279 return 0; 7280} 7281 7282unsigned fastEmit_AArch64ISD_FCMGT_MVT_v1f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7283 if (RetVT.SimpleTy != MVT::v1i64) 7284 return 0; 7285 if ((Subtarget->hasNEON())) { 7286 return fastEmitInst_rr(AArch64::FCMGT64, &AArch64::FPR64RegClass, Op0, Op1); 7287 } 7288 return 0; 7289} 7290 7291unsigned fastEmit_AArch64ISD_FCMGT_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7292 if (RetVT.SimpleTy != MVT::v2i64) 7293 return 0; 7294 if ((Subtarget->hasNEON())) { 7295 return fastEmitInst_rr(AArch64::FCMGTv2f64, &AArch64::FPR128RegClass, Op0, Op1); 7296 } 7297 return 0; 7298} 7299 7300unsigned fastEmit_AArch64ISD_FCMGT_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 7301 switch (VT.SimpleTy) { 7302 case MVT::f32: return fastEmit_AArch64ISD_FCMGT_MVT_f32_rr(RetVT, Op0, Op1); 7303 case MVT::f64: return fastEmit_AArch64ISD_FCMGT_MVT_f64_rr(RetVT, Op0, Op1); 7304 case MVT::v4f16: return fastEmit_AArch64ISD_FCMGT_MVT_v4f16_rr(RetVT, Op0, Op1); 7305 case MVT::v8f16: return fastEmit_AArch64ISD_FCMGT_MVT_v8f16_rr(RetVT, Op0, Op1); 7306 case MVT::v2f32: return fastEmit_AArch64ISD_FCMGT_MVT_v2f32_rr(RetVT, Op0, Op1); 7307 case MVT::v4f32: return fastEmit_AArch64ISD_FCMGT_MVT_v4f32_rr(RetVT, Op0, Op1); 7308 case MVT::v1f64: return fastEmit_AArch64ISD_FCMGT_MVT_v1f64_rr(RetVT, Op0, Op1); 7309 case MVT::v2f64: return fastEmit_AArch64ISD_FCMGT_MVT_v2f64_rr(RetVT, Op0, Op1); 7310 default: return 0; 7311 } 7312} 7313 7314// FastEmit functions for AArch64ISD::FCMP. 7315 7316unsigned fastEmit_AArch64ISD_FCMP_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7317 if (RetVT.SimpleTy != MVT::isVoid) 7318 return 0; 7319 if ((Subtarget->hasFullFP16())) { 7320 return fastEmitInst_rr(AArch64::FCMPHrr, &AArch64::FPR16RegClass, Op0, Op1); 7321 } 7322 return 0; 7323} 7324 7325unsigned fastEmit_AArch64ISD_FCMP_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7326 if (RetVT.SimpleTy != MVT::isVoid) 7327 return 0; 7328 if ((Subtarget->hasFPARMv8())) { 7329 return fastEmitInst_rr(AArch64::FCMPSrr, &AArch64::FPR32RegClass, Op0, Op1); 7330 } 7331 return 0; 7332} 7333 7334unsigned fastEmit_AArch64ISD_FCMP_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7335 if (RetVT.SimpleTy != MVT::isVoid) 7336 return 0; 7337 if ((Subtarget->hasFPARMv8())) { 7338 return fastEmitInst_rr(AArch64::FCMPDrr, &AArch64::FPR64RegClass, Op0, Op1); 7339 } 7340 return 0; 7341} 7342 7343unsigned fastEmit_AArch64ISD_FCMP_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 7344 switch (VT.SimpleTy) { 7345 case MVT::f16: return fastEmit_AArch64ISD_FCMP_MVT_f16_rr(RetVT, Op0, Op1); 7346 case MVT::f32: return fastEmit_AArch64ISD_FCMP_MVT_f32_rr(RetVT, Op0, Op1); 7347 case MVT::f64: return fastEmit_AArch64ISD_FCMP_MVT_f64_rr(RetVT, Op0, Op1); 7348 default: return 0; 7349 } 7350} 7351 7352// FastEmit functions for AArch64ISD::FRECPS. 7353 7354unsigned fastEmit_AArch64ISD_FRECPS_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7355 if (RetVT.SimpleTy != MVT::f32) 7356 return 0; 7357 return fastEmitInst_rr(AArch64::FRECPS32, &AArch64::FPR32RegClass, Op0, Op1); 7358} 7359 7360unsigned fastEmit_AArch64ISD_FRECPS_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7361 if (RetVT.SimpleTy != MVT::f64) 7362 return 0; 7363 return fastEmitInst_rr(AArch64::FRECPS64, &AArch64::FPR64RegClass, Op0, Op1); 7364} 7365 7366unsigned fastEmit_AArch64ISD_FRECPS_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7367 if (RetVT.SimpleTy != MVT::v2f32) 7368 return 0; 7369 return fastEmitInst_rr(AArch64::FRECPSv2f32, &AArch64::FPR64RegClass, Op0, Op1); 7370} 7371 7372unsigned fastEmit_AArch64ISD_FRECPS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7373 if (RetVT.SimpleTy != MVT::v4f32) 7374 return 0; 7375 return fastEmitInst_rr(AArch64::FRECPSv4f32, &AArch64::FPR128RegClass, Op0, Op1); 7376} 7377 7378unsigned fastEmit_AArch64ISD_FRECPS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7379 if (RetVT.SimpleTy != MVT::v2f64) 7380 return 0; 7381 return fastEmitInst_rr(AArch64::FRECPSv2f64, &AArch64::FPR128RegClass, Op0, Op1); 7382} 7383 7384unsigned fastEmit_AArch64ISD_FRECPS_MVT_nxv8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7385 if (RetVT.SimpleTy != MVT::nxv8f16) 7386 return 0; 7387 if ((Subtarget->hasSVEorSME())) { 7388 return fastEmitInst_rr(AArch64::FRECPS_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op1); 7389 } 7390 return 0; 7391} 7392 7393unsigned fastEmit_AArch64ISD_FRECPS_MVT_nxv4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7394 if (RetVT.SimpleTy != MVT::nxv4f32) 7395 return 0; 7396 if ((Subtarget->hasSVEorSME())) { 7397 return fastEmitInst_rr(AArch64::FRECPS_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op1); 7398 } 7399 return 0; 7400} 7401 7402unsigned fastEmit_AArch64ISD_FRECPS_MVT_nxv2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7403 if (RetVT.SimpleTy != MVT::nxv2f64) 7404 return 0; 7405 if ((Subtarget->hasSVEorSME())) { 7406 return fastEmitInst_rr(AArch64::FRECPS_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op1); 7407 } 7408 return 0; 7409} 7410 7411unsigned fastEmit_AArch64ISD_FRECPS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 7412 switch (VT.SimpleTy) { 7413 case MVT::f32: return fastEmit_AArch64ISD_FRECPS_MVT_f32_rr(RetVT, Op0, Op1); 7414 case MVT::f64: return fastEmit_AArch64ISD_FRECPS_MVT_f64_rr(RetVT, Op0, Op1); 7415 case MVT::v2f32: return fastEmit_AArch64ISD_FRECPS_MVT_v2f32_rr(RetVT, Op0, Op1); 7416 case MVT::v4f32: return fastEmit_AArch64ISD_FRECPS_MVT_v4f32_rr(RetVT, Op0, Op1); 7417 case MVT::v2f64: return fastEmit_AArch64ISD_FRECPS_MVT_v2f64_rr(RetVT, Op0, Op1); 7418 case MVT::nxv8f16: return fastEmit_AArch64ISD_FRECPS_MVT_nxv8f16_rr(RetVT, Op0, Op1); 7419 case MVT::nxv4f32: return fastEmit_AArch64ISD_FRECPS_MVT_nxv4f32_rr(RetVT, Op0, Op1); 7420 case MVT::nxv2f64: return fastEmit_AArch64ISD_FRECPS_MVT_nxv2f64_rr(RetVT, Op0, Op1); 7421 default: return 0; 7422 } 7423} 7424 7425// FastEmit functions for AArch64ISD::FRSQRTS. 7426 7427unsigned fastEmit_AArch64ISD_FRSQRTS_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7428 if (RetVT.SimpleTy != MVT::f32) 7429 return 0; 7430 return fastEmitInst_rr(AArch64::FRSQRTS32, &AArch64::FPR32RegClass, Op0, Op1); 7431} 7432 7433unsigned fastEmit_AArch64ISD_FRSQRTS_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7434 if (RetVT.SimpleTy != MVT::f64) 7435 return 0; 7436 return fastEmitInst_rr(AArch64::FRSQRTS64, &AArch64::FPR64RegClass, Op0, Op1); 7437} 7438 7439unsigned fastEmit_AArch64ISD_FRSQRTS_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7440 if (RetVT.SimpleTy != MVT::v2f32) 7441 return 0; 7442 return fastEmitInst_rr(AArch64::FRSQRTSv2f32, &AArch64::FPR64RegClass, Op0, Op1); 7443} 7444 7445unsigned fastEmit_AArch64ISD_FRSQRTS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7446 if (RetVT.SimpleTy != MVT::v4f32) 7447 return 0; 7448 return fastEmitInst_rr(AArch64::FRSQRTSv4f32, &AArch64::FPR128RegClass, Op0, Op1); 7449} 7450 7451unsigned fastEmit_AArch64ISD_FRSQRTS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7452 if (RetVT.SimpleTy != MVT::v2f64) 7453 return 0; 7454 return fastEmitInst_rr(AArch64::FRSQRTSv2f64, &AArch64::FPR128RegClass, Op0, Op1); 7455} 7456 7457unsigned fastEmit_AArch64ISD_FRSQRTS_MVT_nxv8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7458 if (RetVT.SimpleTy != MVT::nxv8f16) 7459 return 0; 7460 if ((Subtarget->hasSVEorSME())) { 7461 return fastEmitInst_rr(AArch64::FRSQRTS_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op1); 7462 } 7463 return 0; 7464} 7465 7466unsigned fastEmit_AArch64ISD_FRSQRTS_MVT_nxv4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7467 if (RetVT.SimpleTy != MVT::nxv4f32) 7468 return 0; 7469 if ((Subtarget->hasSVEorSME())) { 7470 return fastEmitInst_rr(AArch64::FRSQRTS_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op1); 7471 } 7472 return 0; 7473} 7474 7475unsigned fastEmit_AArch64ISD_FRSQRTS_MVT_nxv2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7476 if (RetVT.SimpleTy != MVT::nxv2f64) 7477 return 0; 7478 if ((Subtarget->hasSVEorSME())) { 7479 return fastEmitInst_rr(AArch64::FRSQRTS_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op1); 7480 } 7481 return 0; 7482} 7483 7484unsigned fastEmit_AArch64ISD_FRSQRTS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 7485 switch (VT.SimpleTy) { 7486 case MVT::f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_f32_rr(RetVT, Op0, Op1); 7487 case MVT::f64: return fastEmit_AArch64ISD_FRSQRTS_MVT_f64_rr(RetVT, Op0, Op1); 7488 case MVT::v2f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_v2f32_rr(RetVT, Op0, Op1); 7489 case MVT::v4f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_v4f32_rr(RetVT, Op0, Op1); 7490 case MVT::v2f64: return fastEmit_AArch64ISD_FRSQRTS_MVT_v2f64_rr(RetVT, Op0, Op1); 7491 case MVT::nxv8f16: return fastEmit_AArch64ISD_FRSQRTS_MVT_nxv8f16_rr(RetVT, Op0, Op1); 7492 case MVT::nxv4f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_nxv4f32_rr(RetVT, Op0, Op1); 7493 case MVT::nxv2f64: return fastEmit_AArch64ISD_FRSQRTS_MVT_nxv2f64_rr(RetVT, Op0, Op1); 7494 default: return 0; 7495 } 7496} 7497 7498// FastEmit functions for AArch64ISD::PMULL. 7499 7500unsigned fastEmit_AArch64ISD_PMULL_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7501 if (RetVT.SimpleTy != MVT::v8i16) 7502 return 0; 7503 if ((Subtarget->hasNEON())) { 7504 return fastEmitInst_rr(AArch64::PMULLv8i8, &AArch64::FPR128RegClass, Op0, Op1); 7505 } 7506 return 0; 7507} 7508 7509unsigned fastEmit_AArch64ISD_PMULL_MVT_v1i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7510 if (RetVT.SimpleTy != MVT::v16i8) 7511 return 0; 7512 if ((Subtarget->hasAES())) { 7513 return fastEmitInst_rr(AArch64::PMULLv1i64, &AArch64::FPR128RegClass, Op0, Op1); 7514 } 7515 return 0; 7516} 7517 7518unsigned fastEmit_AArch64ISD_PMULL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 7519 switch (VT.SimpleTy) { 7520 case MVT::v8i8: return fastEmit_AArch64ISD_PMULL_MVT_v8i8_rr(RetVT, Op0, Op1); 7521 case MVT::v1i64: return fastEmit_AArch64ISD_PMULL_MVT_v1i64_rr(RetVT, Op0, Op1); 7522 default: return 0; 7523 } 7524} 7525 7526// FastEmit functions for AArch64ISD::PTEST. 7527 7528unsigned fastEmit_AArch64ISD_PTEST_MVT_nxv16i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7529 if (RetVT.SimpleTy != MVT::isVoid) 7530 return 0; 7531 if ((Subtarget->hasSVEorSME())) { 7532 return fastEmitInst_rr(AArch64::PTEST_PP, &AArch64::PPRRegClass, Op0, Op1); 7533 } 7534 return 0; 7535} 7536 7537unsigned fastEmit_AArch64ISD_PTEST_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 7538 switch (VT.SimpleTy) { 7539 case MVT::nxv16i1: return fastEmit_AArch64ISD_PTEST_MVT_nxv16i1_rr(RetVT, Op0, Op1); 7540 default: return 0; 7541 } 7542} 7543 7544// FastEmit functions for AArch64ISD::PTEST_ANY. 7545 7546unsigned fastEmit_AArch64ISD_PTEST_ANY_MVT_nxv16i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7547 if (RetVT.SimpleTy != MVT::isVoid) 7548 return 0; 7549 if ((Subtarget->hasSVEorSME())) { 7550 return fastEmitInst_rr(AArch64::PTEST_PP_ANY, &AArch64::PPRRegClass, Op0, Op1); 7551 } 7552 return 0; 7553} 7554 7555unsigned fastEmit_AArch64ISD_PTEST_ANY_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 7556 switch (VT.SimpleTy) { 7557 case MVT::nxv16i1: return fastEmit_AArch64ISD_PTEST_ANY_MVT_nxv16i1_rr(RetVT, Op0, Op1); 7558 default: return 0; 7559 } 7560} 7561 7562// FastEmit functions for AArch64ISD::SMULL. 7563 7564unsigned fastEmit_AArch64ISD_SMULL_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7565 if (RetVT.SimpleTy != MVT::v8i16) 7566 return 0; 7567 if ((Subtarget->hasNEON())) { 7568 return fastEmitInst_rr(AArch64::SMULLv8i8_v8i16, &AArch64::FPR128RegClass, Op0, Op1); 7569 } 7570 return 0; 7571} 7572 7573unsigned fastEmit_AArch64ISD_SMULL_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7574 if (RetVT.SimpleTy != MVT::v4i32) 7575 return 0; 7576 if ((Subtarget->hasNEON())) { 7577 return fastEmitInst_rr(AArch64::SMULLv4i16_v4i32, &AArch64::FPR128RegClass, Op0, Op1); 7578 } 7579 return 0; 7580} 7581 7582unsigned fastEmit_AArch64ISD_SMULL_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7583 if (RetVT.SimpleTy != MVT::v2i64) 7584 return 0; 7585 if ((Subtarget->hasNEON())) { 7586 return fastEmitInst_rr(AArch64::SMULLv2i32_v2i64, &AArch64::FPR128RegClass, Op0, Op1); 7587 } 7588 return 0; 7589} 7590 7591unsigned fastEmit_AArch64ISD_SMULL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 7592 switch (VT.SimpleTy) { 7593 case MVT::v8i8: return fastEmit_AArch64ISD_SMULL_MVT_v8i8_rr(RetVT, Op0, Op1); 7594 case MVT::v4i16: return fastEmit_AArch64ISD_SMULL_MVT_v4i16_rr(RetVT, Op0, Op1); 7595 case MVT::v2i32: return fastEmit_AArch64ISD_SMULL_MVT_v2i32_rr(RetVT, Op0, Op1); 7596 default: return 0; 7597 } 7598} 7599 7600// FastEmit functions for AArch64ISD::STRICT_FCMP. 7601 7602unsigned fastEmit_AArch64ISD_STRICT_FCMP_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7603 if (RetVT.SimpleTy != MVT::isVoid) 7604 return 0; 7605 if ((Subtarget->hasFullFP16())) { 7606 return fastEmitInst_rr(AArch64::FCMPHrr, &AArch64::FPR16RegClass, Op0, Op1); 7607 } 7608 return 0; 7609} 7610 7611unsigned fastEmit_AArch64ISD_STRICT_FCMP_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7612 if (RetVT.SimpleTy != MVT::isVoid) 7613 return 0; 7614 if ((Subtarget->hasFPARMv8())) { 7615 return fastEmitInst_rr(AArch64::FCMPSrr, &AArch64::FPR32RegClass, Op0, Op1); 7616 } 7617 return 0; 7618} 7619 7620unsigned fastEmit_AArch64ISD_STRICT_FCMP_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7621 if (RetVT.SimpleTy != MVT::isVoid) 7622 return 0; 7623 if ((Subtarget->hasFPARMv8())) { 7624 return fastEmitInst_rr(AArch64::FCMPDrr, &AArch64::FPR64RegClass, Op0, Op1); 7625 } 7626 return 0; 7627} 7628 7629unsigned fastEmit_AArch64ISD_STRICT_FCMP_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 7630 switch (VT.SimpleTy) { 7631 case MVT::f16: return fastEmit_AArch64ISD_STRICT_FCMP_MVT_f16_rr(RetVT, Op0, Op1); 7632 case MVT::f32: return fastEmit_AArch64ISD_STRICT_FCMP_MVT_f32_rr(RetVT, Op0, Op1); 7633 case MVT::f64: return fastEmit_AArch64ISD_STRICT_FCMP_MVT_f64_rr(RetVT, Op0, Op1); 7634 default: return 0; 7635 } 7636} 7637 7638// FastEmit functions for AArch64ISD::STRICT_FCMPE. 7639 7640unsigned fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7641 if (RetVT.SimpleTy != MVT::isVoid) 7642 return 0; 7643 if ((Subtarget->hasFullFP16())) { 7644 return fastEmitInst_rr(AArch64::FCMPEHrr, &AArch64::FPR16RegClass, Op0, Op1); 7645 } 7646 return 0; 7647} 7648 7649unsigned fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7650 if (RetVT.SimpleTy != MVT::isVoid) 7651 return 0; 7652 if ((Subtarget->hasFPARMv8())) { 7653 return fastEmitInst_rr(AArch64::FCMPESrr, &AArch64::FPR32RegClass, Op0, Op1); 7654 } 7655 return 0; 7656} 7657 7658unsigned fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7659 if (RetVT.SimpleTy != MVT::isVoid) 7660 return 0; 7661 if ((Subtarget->hasFPARMv8())) { 7662 return fastEmitInst_rr(AArch64::FCMPEDrr, &AArch64::FPR64RegClass, Op0, Op1); 7663 } 7664 return 0; 7665} 7666 7667unsigned fastEmit_AArch64ISD_STRICT_FCMPE_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 7668 switch (VT.SimpleTy) { 7669 case MVT::f16: return fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f16_rr(RetVT, Op0, Op1); 7670 case MVT::f32: return fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f32_rr(RetVT, Op0, Op1); 7671 case MVT::f64: return fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f64_rr(RetVT, Op0, Op1); 7672 default: return 0; 7673 } 7674} 7675 7676// FastEmit functions for AArch64ISD::TBL. 7677 7678unsigned fastEmit_AArch64ISD_TBL_MVT_nxv16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7679 if (RetVT.SimpleTy != MVT::nxv16i8) 7680 return 0; 7681 if ((Subtarget->hasSVEorSME())) { 7682 return fastEmitInst_rr(AArch64::TBL_ZZZ_B, &AArch64::ZPRRegClass, Op0, Op1); 7683 } 7684 return 0; 7685} 7686 7687unsigned fastEmit_AArch64ISD_TBL_MVT_nxv8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7688 if (RetVT.SimpleTy != MVT::nxv8i16) 7689 return 0; 7690 if ((Subtarget->hasSVEorSME())) { 7691 return fastEmitInst_rr(AArch64::TBL_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op1); 7692 } 7693 return 0; 7694} 7695 7696unsigned fastEmit_AArch64ISD_TBL_MVT_nxv4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7697 if (RetVT.SimpleTy != MVT::nxv4i32) 7698 return 0; 7699 if ((Subtarget->hasSVEorSME())) { 7700 return fastEmitInst_rr(AArch64::TBL_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op1); 7701 } 7702 return 0; 7703} 7704 7705unsigned fastEmit_AArch64ISD_TBL_MVT_nxv2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7706 if (RetVT.SimpleTy != MVT::nxv2i64) 7707 return 0; 7708 if ((Subtarget->hasSVEorSME())) { 7709 return fastEmitInst_rr(AArch64::TBL_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op1); 7710 } 7711 return 0; 7712} 7713 7714unsigned fastEmit_AArch64ISD_TBL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 7715 switch (VT.SimpleTy) { 7716 case MVT::nxv16i8: return fastEmit_AArch64ISD_TBL_MVT_nxv16i8_rr(RetVT, Op0, Op1); 7717 case MVT::nxv8i16: return fastEmit_AArch64ISD_TBL_MVT_nxv8i16_rr(RetVT, Op0, Op1); 7718 case MVT::nxv4i32: return fastEmit_AArch64ISD_TBL_MVT_nxv4i32_rr(RetVT, Op0, Op1); 7719 case MVT::nxv2i64: return fastEmit_AArch64ISD_TBL_MVT_nxv2i64_rr(RetVT, Op0, Op1); 7720 default: return 0; 7721 } 7722} 7723 7724// FastEmit functions for AArch64ISD::TRN1. 7725 7726unsigned fastEmit_AArch64ISD_TRN1_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7727 if (RetVT.SimpleTy != MVT::v8i8) 7728 return 0; 7729 if ((Subtarget->hasNEON())) { 7730 return fastEmitInst_rr(AArch64::TRN1v8i8, &AArch64::FPR64RegClass, Op0, Op1); 7731 } 7732 return 0; 7733} 7734 7735unsigned fastEmit_AArch64ISD_TRN1_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7736 if (RetVT.SimpleTy != MVT::v16i8) 7737 return 0; 7738 if ((Subtarget->hasNEON())) { 7739 return fastEmitInst_rr(AArch64::TRN1v16i8, &AArch64::FPR128RegClass, Op0, Op1); 7740 } 7741 return 0; 7742} 7743 7744unsigned fastEmit_AArch64ISD_TRN1_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7745 if (RetVT.SimpleTy != MVT::v4i16) 7746 return 0; 7747 if ((Subtarget->hasNEON())) { 7748 return fastEmitInst_rr(AArch64::TRN1v4i16, &AArch64::FPR64RegClass, Op0, Op1); 7749 } 7750 return 0; 7751} 7752 7753unsigned fastEmit_AArch64ISD_TRN1_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7754 if (RetVT.SimpleTy != MVT::v8i16) 7755 return 0; 7756 if ((Subtarget->hasNEON())) { 7757 return fastEmitInst_rr(AArch64::TRN1v8i16, &AArch64::FPR128RegClass, Op0, Op1); 7758 } 7759 return 0; 7760} 7761 7762unsigned fastEmit_AArch64ISD_TRN1_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7763 if (RetVT.SimpleTy != MVT::v2i32) 7764 return 0; 7765 if ((Subtarget->hasNEON())) { 7766 return fastEmitInst_rr(AArch64::TRN1v2i32, &AArch64::FPR64RegClass, Op0, Op1); 7767 } 7768 return 0; 7769} 7770 7771unsigned fastEmit_AArch64ISD_TRN1_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7772 if (RetVT.SimpleTy != MVT::v4i32) 7773 return 0; 7774 if ((Subtarget->hasNEON())) { 7775 return fastEmitInst_rr(AArch64::TRN1v4i32, &AArch64::FPR128RegClass, Op0, Op1); 7776 } 7777 return 0; 7778} 7779 7780unsigned fastEmit_AArch64ISD_TRN1_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7781 if (RetVT.SimpleTy != MVT::v2i64) 7782 return 0; 7783 if ((Subtarget->hasNEON())) { 7784 return fastEmitInst_rr(AArch64::TRN1v2i64, &AArch64::FPR128RegClass, Op0, Op1); 7785 } 7786 return 0; 7787} 7788 7789unsigned fastEmit_AArch64ISD_TRN1_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7790 if (RetVT.SimpleTy != MVT::v4f16) 7791 return 0; 7792 if ((Subtarget->hasNEON())) { 7793 return fastEmitInst_rr(AArch64::TRN1v4i16, &AArch64::FPR64RegClass, Op0, Op1); 7794 } 7795 return 0; 7796} 7797 7798unsigned fastEmit_AArch64ISD_TRN1_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7799 if (RetVT.SimpleTy != MVT::v8f16) 7800 return 0; 7801 if ((Subtarget->hasNEON())) { 7802 return fastEmitInst_rr(AArch64::TRN1v8i16, &AArch64::FPR128RegClass, Op0, Op1); 7803 } 7804 return 0; 7805} 7806 7807unsigned fastEmit_AArch64ISD_TRN1_MVT_v4bf16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7808 if (RetVT.SimpleTy != MVT::v4bf16) 7809 return 0; 7810 if ((Subtarget->hasNEON())) { 7811 return fastEmitInst_rr(AArch64::TRN1v4i16, &AArch64::FPR64RegClass, Op0, Op1); 7812 } 7813 return 0; 7814} 7815 7816unsigned fastEmit_AArch64ISD_TRN1_MVT_v8bf16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7817 if (RetVT.SimpleTy != MVT::v8bf16) 7818 return 0; 7819 if ((Subtarget->hasNEON())) { 7820 return fastEmitInst_rr(AArch64::TRN1v8i16, &AArch64::FPR128RegClass, Op0, Op1); 7821 } 7822 return 0; 7823} 7824 7825unsigned fastEmit_AArch64ISD_TRN1_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7826 if (RetVT.SimpleTy != MVT::v2f32) 7827 return 0; 7828 if ((Subtarget->hasNEON())) { 7829 return fastEmitInst_rr(AArch64::TRN1v2i32, &AArch64::FPR64RegClass, Op0, Op1); 7830 } 7831 return 0; 7832} 7833 7834unsigned fastEmit_AArch64ISD_TRN1_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7835 if (RetVT.SimpleTy != MVT::v4f32) 7836 return 0; 7837 if ((Subtarget->hasNEON())) { 7838 return fastEmitInst_rr(AArch64::TRN1v4i32, &AArch64::FPR128RegClass, Op0, Op1); 7839 } 7840 return 0; 7841} 7842 7843unsigned fastEmit_AArch64ISD_TRN1_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7844 if (RetVT.SimpleTy != MVT::v2f64) 7845 return 0; 7846 if ((Subtarget->hasNEON())) { 7847 return fastEmitInst_rr(AArch64::TRN1v2i64, &AArch64::FPR128RegClass, Op0, Op1); 7848 } 7849 return 0; 7850} 7851 7852unsigned fastEmit_AArch64ISD_TRN1_MVT_nxv2i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7853 if (RetVT.SimpleTy != MVT::nxv2i1) 7854 return 0; 7855 if ((Subtarget->hasSVEorSME())) { 7856 return fastEmitInst_rr(AArch64::TRN1_PPP_D, &AArch64::PPRRegClass, Op0, Op1); 7857 } 7858 return 0; 7859} 7860 7861unsigned fastEmit_AArch64ISD_TRN1_MVT_nxv4i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7862 if (RetVT.SimpleTy != MVT::nxv4i1) 7863 return 0; 7864 if ((Subtarget->hasSVEorSME())) { 7865 return fastEmitInst_rr(AArch64::TRN1_PPP_S, &AArch64::PPRRegClass, Op0, Op1); 7866 } 7867 return 0; 7868} 7869 7870unsigned fastEmit_AArch64ISD_TRN1_MVT_nxv8i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7871 if (RetVT.SimpleTy != MVT::nxv8i1) 7872 return 0; 7873 if ((Subtarget->hasSVEorSME())) { 7874 return fastEmitInst_rr(AArch64::TRN1_PPP_H, &AArch64::PPRRegClass, Op0, Op1); 7875 } 7876 return 0; 7877} 7878 7879unsigned fastEmit_AArch64ISD_TRN1_MVT_nxv16i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7880 if (RetVT.SimpleTy != MVT::nxv16i1) 7881 return 0; 7882 if ((Subtarget->hasSVEorSME())) { 7883 return fastEmitInst_rr(AArch64::TRN1_PPP_B, &AArch64::PPRRegClass, Op0, Op1); 7884 } 7885 return 0; 7886} 7887 7888unsigned fastEmit_AArch64ISD_TRN1_MVT_nxv16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7889 if (RetVT.SimpleTy != MVT::nxv16i8) 7890 return 0; 7891 if ((Subtarget->hasSVEorSME())) { 7892 return fastEmitInst_rr(AArch64::TRN1_ZZZ_B, &AArch64::ZPRRegClass, Op0, Op1); 7893 } 7894 return 0; 7895} 7896 7897unsigned fastEmit_AArch64ISD_TRN1_MVT_nxv8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7898 if (RetVT.SimpleTy != MVT::nxv8i16) 7899 return 0; 7900 if ((Subtarget->hasSVEorSME())) { 7901 return fastEmitInst_rr(AArch64::TRN1_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op1); 7902 } 7903 return 0; 7904} 7905 7906unsigned fastEmit_AArch64ISD_TRN1_MVT_nxv4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7907 if (RetVT.SimpleTy != MVT::nxv4i32) 7908 return 0; 7909 if ((Subtarget->hasSVEorSME())) { 7910 return fastEmitInst_rr(AArch64::TRN1_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op1); 7911 } 7912 return 0; 7913} 7914 7915unsigned fastEmit_AArch64ISD_TRN1_MVT_nxv2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7916 if (RetVT.SimpleTy != MVT::nxv2i64) 7917 return 0; 7918 if ((Subtarget->hasSVEorSME())) { 7919 return fastEmitInst_rr(AArch64::TRN1_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op1); 7920 } 7921 return 0; 7922} 7923 7924unsigned fastEmit_AArch64ISD_TRN1_MVT_nxv2f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7925 if (RetVT.SimpleTy != MVT::nxv2f16) 7926 return 0; 7927 if ((Subtarget->hasSVEorSME())) { 7928 return fastEmitInst_rr(AArch64::TRN1_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op1); 7929 } 7930 return 0; 7931} 7932 7933unsigned fastEmit_AArch64ISD_TRN1_MVT_nxv4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7934 if (RetVT.SimpleTy != MVT::nxv4f16) 7935 return 0; 7936 if ((Subtarget->hasSVEorSME())) { 7937 return fastEmitInst_rr(AArch64::TRN1_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op1); 7938 } 7939 return 0; 7940} 7941 7942unsigned fastEmit_AArch64ISD_TRN1_MVT_nxv8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7943 if (RetVT.SimpleTy != MVT::nxv8f16) 7944 return 0; 7945 if ((Subtarget->hasSVEorSME())) { 7946 return fastEmitInst_rr(AArch64::TRN1_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op1); 7947 } 7948 return 0; 7949} 7950 7951unsigned fastEmit_AArch64ISD_TRN1_MVT_nxv8bf16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7952 if (RetVT.SimpleTy != MVT::nxv8bf16) 7953 return 0; 7954 if ((Subtarget->hasSVEorSME())) { 7955 return fastEmitInst_rr(AArch64::TRN1_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op1); 7956 } 7957 return 0; 7958} 7959 7960unsigned fastEmit_AArch64ISD_TRN1_MVT_nxv2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7961 if (RetVT.SimpleTy != MVT::nxv2f32) 7962 return 0; 7963 if ((Subtarget->hasSVEorSME())) { 7964 return fastEmitInst_rr(AArch64::TRN1_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op1); 7965 } 7966 return 0; 7967} 7968 7969unsigned fastEmit_AArch64ISD_TRN1_MVT_nxv4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7970 if (RetVT.SimpleTy != MVT::nxv4f32) 7971 return 0; 7972 if ((Subtarget->hasSVEorSME())) { 7973 return fastEmitInst_rr(AArch64::TRN1_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op1); 7974 } 7975 return 0; 7976} 7977 7978unsigned fastEmit_AArch64ISD_TRN1_MVT_nxv2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 7979 if (RetVT.SimpleTy != MVT::nxv2f64) 7980 return 0; 7981 if ((Subtarget->hasSVEorSME())) { 7982 return fastEmitInst_rr(AArch64::TRN1_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op1); 7983 } 7984 return 0; 7985} 7986 7987unsigned fastEmit_AArch64ISD_TRN1_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 7988 switch (VT.SimpleTy) { 7989 case MVT::v8i8: return fastEmit_AArch64ISD_TRN1_MVT_v8i8_rr(RetVT, Op0, Op1); 7990 case MVT::v16i8: return fastEmit_AArch64ISD_TRN1_MVT_v16i8_rr(RetVT, Op0, Op1); 7991 case MVT::v4i16: return fastEmit_AArch64ISD_TRN1_MVT_v4i16_rr(RetVT, Op0, Op1); 7992 case MVT::v8i16: return fastEmit_AArch64ISD_TRN1_MVT_v8i16_rr(RetVT, Op0, Op1); 7993 case MVT::v2i32: return fastEmit_AArch64ISD_TRN1_MVT_v2i32_rr(RetVT, Op0, Op1); 7994 case MVT::v4i32: return fastEmit_AArch64ISD_TRN1_MVT_v4i32_rr(RetVT, Op0, Op1); 7995 case MVT::v2i64: return fastEmit_AArch64ISD_TRN1_MVT_v2i64_rr(RetVT, Op0, Op1); 7996 case MVT::v4f16: return fastEmit_AArch64ISD_TRN1_MVT_v4f16_rr(RetVT, Op0, Op1); 7997 case MVT::v8f16: return fastEmit_AArch64ISD_TRN1_MVT_v8f16_rr(RetVT, Op0, Op1); 7998 case MVT::v4bf16: return fastEmit_AArch64ISD_TRN1_MVT_v4bf16_rr(RetVT, Op0, Op1); 7999 case MVT::v8bf16: return fastEmit_AArch64ISD_TRN1_MVT_v8bf16_rr(RetVT, Op0, Op1); 8000 case MVT::v2f32: return fastEmit_AArch64ISD_TRN1_MVT_v2f32_rr(RetVT, Op0, Op1); 8001 case MVT::v4f32: return fastEmit_AArch64ISD_TRN1_MVT_v4f32_rr(RetVT, Op0, Op1); 8002 case MVT::v2f64: return fastEmit_AArch64ISD_TRN1_MVT_v2f64_rr(RetVT, Op0, Op1); 8003 case MVT::nxv2i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv2i1_rr(RetVT, Op0, Op1); 8004 case MVT::nxv4i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv4i1_rr(RetVT, Op0, Op1); 8005 case MVT::nxv8i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv8i1_rr(RetVT, Op0, Op1); 8006 case MVT::nxv16i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv16i1_rr(RetVT, Op0, Op1); 8007 case MVT::nxv16i8: return fastEmit_AArch64ISD_TRN1_MVT_nxv16i8_rr(RetVT, Op0, Op1); 8008 case MVT::nxv8i16: return fastEmit_AArch64ISD_TRN1_MVT_nxv8i16_rr(RetVT, Op0, Op1); 8009 case MVT::nxv4i32: return fastEmit_AArch64ISD_TRN1_MVT_nxv4i32_rr(RetVT, Op0, Op1); 8010 case MVT::nxv2i64: return fastEmit_AArch64ISD_TRN1_MVT_nxv2i64_rr(RetVT, Op0, Op1); 8011 case MVT::nxv2f16: return fastEmit_AArch64ISD_TRN1_MVT_nxv2f16_rr(RetVT, Op0, Op1); 8012 case MVT::nxv4f16: return fastEmit_AArch64ISD_TRN1_MVT_nxv4f16_rr(RetVT, Op0, Op1); 8013 case MVT::nxv8f16: return fastEmit_AArch64ISD_TRN1_MVT_nxv8f16_rr(RetVT, Op0, Op1); 8014 case MVT::nxv8bf16: return fastEmit_AArch64ISD_TRN1_MVT_nxv8bf16_rr(RetVT, Op0, Op1); 8015 case MVT::nxv2f32: return fastEmit_AArch64ISD_TRN1_MVT_nxv2f32_rr(RetVT, Op0, Op1); 8016 case MVT::nxv4f32: return fastEmit_AArch64ISD_TRN1_MVT_nxv4f32_rr(RetVT, Op0, Op1); 8017 case MVT::nxv2f64: return fastEmit_AArch64ISD_TRN1_MVT_nxv2f64_rr(RetVT, Op0, Op1); 8018 default: return 0; 8019 } 8020} 8021 8022// FastEmit functions for AArch64ISD::TRN2. 8023 8024unsigned fastEmit_AArch64ISD_TRN2_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8025 if (RetVT.SimpleTy != MVT::v8i8) 8026 return 0; 8027 if ((Subtarget->hasNEON())) { 8028 return fastEmitInst_rr(AArch64::TRN2v8i8, &AArch64::FPR64RegClass, Op0, Op1); 8029 } 8030 return 0; 8031} 8032 8033unsigned fastEmit_AArch64ISD_TRN2_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8034 if (RetVT.SimpleTy != MVT::v16i8) 8035 return 0; 8036 if ((Subtarget->hasNEON())) { 8037 return fastEmitInst_rr(AArch64::TRN2v16i8, &AArch64::FPR128RegClass, Op0, Op1); 8038 } 8039 return 0; 8040} 8041 8042unsigned fastEmit_AArch64ISD_TRN2_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8043 if (RetVT.SimpleTy != MVT::v4i16) 8044 return 0; 8045 if ((Subtarget->hasNEON())) { 8046 return fastEmitInst_rr(AArch64::TRN2v4i16, &AArch64::FPR64RegClass, Op0, Op1); 8047 } 8048 return 0; 8049} 8050 8051unsigned fastEmit_AArch64ISD_TRN2_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8052 if (RetVT.SimpleTy != MVT::v8i16) 8053 return 0; 8054 if ((Subtarget->hasNEON())) { 8055 return fastEmitInst_rr(AArch64::TRN2v8i16, &AArch64::FPR128RegClass, Op0, Op1); 8056 } 8057 return 0; 8058} 8059 8060unsigned fastEmit_AArch64ISD_TRN2_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8061 if (RetVT.SimpleTy != MVT::v2i32) 8062 return 0; 8063 if ((Subtarget->hasNEON())) { 8064 return fastEmitInst_rr(AArch64::TRN2v2i32, &AArch64::FPR64RegClass, Op0, Op1); 8065 } 8066 return 0; 8067} 8068 8069unsigned fastEmit_AArch64ISD_TRN2_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8070 if (RetVT.SimpleTy != MVT::v4i32) 8071 return 0; 8072 if ((Subtarget->hasNEON())) { 8073 return fastEmitInst_rr(AArch64::TRN2v4i32, &AArch64::FPR128RegClass, Op0, Op1); 8074 } 8075 return 0; 8076} 8077 8078unsigned fastEmit_AArch64ISD_TRN2_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8079 if (RetVT.SimpleTy != MVT::v2i64) 8080 return 0; 8081 if ((Subtarget->hasNEON())) { 8082 return fastEmitInst_rr(AArch64::TRN2v2i64, &AArch64::FPR128RegClass, Op0, Op1); 8083 } 8084 return 0; 8085} 8086 8087unsigned fastEmit_AArch64ISD_TRN2_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8088 if (RetVT.SimpleTy != MVT::v4f16) 8089 return 0; 8090 if ((Subtarget->hasNEON())) { 8091 return fastEmitInst_rr(AArch64::TRN2v4i16, &AArch64::FPR64RegClass, Op0, Op1); 8092 } 8093 return 0; 8094} 8095 8096unsigned fastEmit_AArch64ISD_TRN2_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8097 if (RetVT.SimpleTy != MVT::v8f16) 8098 return 0; 8099 if ((Subtarget->hasNEON())) { 8100 return fastEmitInst_rr(AArch64::TRN2v8i16, &AArch64::FPR128RegClass, Op0, Op1); 8101 } 8102 return 0; 8103} 8104 8105unsigned fastEmit_AArch64ISD_TRN2_MVT_v4bf16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8106 if (RetVT.SimpleTy != MVT::v4bf16) 8107 return 0; 8108 if ((Subtarget->hasNEON())) { 8109 return fastEmitInst_rr(AArch64::TRN2v4i16, &AArch64::FPR64RegClass, Op0, Op1); 8110 } 8111 return 0; 8112} 8113 8114unsigned fastEmit_AArch64ISD_TRN2_MVT_v8bf16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8115 if (RetVT.SimpleTy != MVT::v8bf16) 8116 return 0; 8117 if ((Subtarget->hasNEON())) { 8118 return fastEmitInst_rr(AArch64::TRN2v8i16, &AArch64::FPR128RegClass, Op0, Op1); 8119 } 8120 return 0; 8121} 8122 8123unsigned fastEmit_AArch64ISD_TRN2_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8124 if (RetVT.SimpleTy != MVT::v2f32) 8125 return 0; 8126 if ((Subtarget->hasNEON())) { 8127 return fastEmitInst_rr(AArch64::TRN2v2i32, &AArch64::FPR64RegClass, Op0, Op1); 8128 } 8129 return 0; 8130} 8131 8132unsigned fastEmit_AArch64ISD_TRN2_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8133 if (RetVT.SimpleTy != MVT::v4f32) 8134 return 0; 8135 if ((Subtarget->hasNEON())) { 8136 return fastEmitInst_rr(AArch64::TRN2v4i32, &AArch64::FPR128RegClass, Op0, Op1); 8137 } 8138 return 0; 8139} 8140 8141unsigned fastEmit_AArch64ISD_TRN2_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8142 if (RetVT.SimpleTy != MVT::v2f64) 8143 return 0; 8144 if ((Subtarget->hasNEON())) { 8145 return fastEmitInst_rr(AArch64::TRN2v2i64, &AArch64::FPR128RegClass, Op0, Op1); 8146 } 8147 return 0; 8148} 8149 8150unsigned fastEmit_AArch64ISD_TRN2_MVT_nxv2i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8151 if (RetVT.SimpleTy != MVT::nxv2i1) 8152 return 0; 8153 if ((Subtarget->hasSVEorSME())) { 8154 return fastEmitInst_rr(AArch64::TRN2_PPP_D, &AArch64::PPRRegClass, Op0, Op1); 8155 } 8156 return 0; 8157} 8158 8159unsigned fastEmit_AArch64ISD_TRN2_MVT_nxv4i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8160 if (RetVT.SimpleTy != MVT::nxv4i1) 8161 return 0; 8162 if ((Subtarget->hasSVEorSME())) { 8163 return fastEmitInst_rr(AArch64::TRN2_PPP_S, &AArch64::PPRRegClass, Op0, Op1); 8164 } 8165 return 0; 8166} 8167 8168unsigned fastEmit_AArch64ISD_TRN2_MVT_nxv8i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8169 if (RetVT.SimpleTy != MVT::nxv8i1) 8170 return 0; 8171 if ((Subtarget->hasSVEorSME())) { 8172 return fastEmitInst_rr(AArch64::TRN2_PPP_H, &AArch64::PPRRegClass, Op0, Op1); 8173 } 8174 return 0; 8175} 8176 8177unsigned fastEmit_AArch64ISD_TRN2_MVT_nxv16i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8178 if (RetVT.SimpleTy != MVT::nxv16i1) 8179 return 0; 8180 if ((Subtarget->hasSVEorSME())) { 8181 return fastEmitInst_rr(AArch64::TRN2_PPP_B, &AArch64::PPRRegClass, Op0, Op1); 8182 } 8183 return 0; 8184} 8185 8186unsigned fastEmit_AArch64ISD_TRN2_MVT_nxv16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8187 if (RetVT.SimpleTy != MVT::nxv16i8) 8188 return 0; 8189 if ((Subtarget->hasSVEorSME())) { 8190 return fastEmitInst_rr(AArch64::TRN2_ZZZ_B, &AArch64::ZPRRegClass, Op0, Op1); 8191 } 8192 return 0; 8193} 8194 8195unsigned fastEmit_AArch64ISD_TRN2_MVT_nxv8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8196 if (RetVT.SimpleTy != MVT::nxv8i16) 8197 return 0; 8198 if ((Subtarget->hasSVEorSME())) { 8199 return fastEmitInst_rr(AArch64::TRN2_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op1); 8200 } 8201 return 0; 8202} 8203 8204unsigned fastEmit_AArch64ISD_TRN2_MVT_nxv4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8205 if (RetVT.SimpleTy != MVT::nxv4i32) 8206 return 0; 8207 if ((Subtarget->hasSVEorSME())) { 8208 return fastEmitInst_rr(AArch64::TRN2_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op1); 8209 } 8210 return 0; 8211} 8212 8213unsigned fastEmit_AArch64ISD_TRN2_MVT_nxv2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8214 if (RetVT.SimpleTy != MVT::nxv2i64) 8215 return 0; 8216 if ((Subtarget->hasSVEorSME())) { 8217 return fastEmitInst_rr(AArch64::TRN2_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op1); 8218 } 8219 return 0; 8220} 8221 8222unsigned fastEmit_AArch64ISD_TRN2_MVT_nxv2f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8223 if (RetVT.SimpleTy != MVT::nxv2f16) 8224 return 0; 8225 if ((Subtarget->hasSVEorSME())) { 8226 return fastEmitInst_rr(AArch64::TRN2_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op1); 8227 } 8228 return 0; 8229} 8230 8231unsigned fastEmit_AArch64ISD_TRN2_MVT_nxv4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8232 if (RetVT.SimpleTy != MVT::nxv4f16) 8233 return 0; 8234 if ((Subtarget->hasSVEorSME())) { 8235 return fastEmitInst_rr(AArch64::TRN2_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op1); 8236 } 8237 return 0; 8238} 8239 8240unsigned fastEmit_AArch64ISD_TRN2_MVT_nxv8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8241 if (RetVT.SimpleTy != MVT::nxv8f16) 8242 return 0; 8243 if ((Subtarget->hasSVEorSME())) { 8244 return fastEmitInst_rr(AArch64::TRN2_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op1); 8245 } 8246 return 0; 8247} 8248 8249unsigned fastEmit_AArch64ISD_TRN2_MVT_nxv8bf16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8250 if (RetVT.SimpleTy != MVT::nxv8bf16) 8251 return 0; 8252 if ((Subtarget->hasSVEorSME())) { 8253 return fastEmitInst_rr(AArch64::TRN2_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op1); 8254 } 8255 return 0; 8256} 8257 8258unsigned fastEmit_AArch64ISD_TRN2_MVT_nxv2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8259 if (RetVT.SimpleTy != MVT::nxv2f32) 8260 return 0; 8261 if ((Subtarget->hasSVEorSME())) { 8262 return fastEmitInst_rr(AArch64::TRN2_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op1); 8263 } 8264 return 0; 8265} 8266 8267unsigned fastEmit_AArch64ISD_TRN2_MVT_nxv4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8268 if (RetVT.SimpleTy != MVT::nxv4f32) 8269 return 0; 8270 if ((Subtarget->hasSVEorSME())) { 8271 return fastEmitInst_rr(AArch64::TRN2_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op1); 8272 } 8273 return 0; 8274} 8275 8276unsigned fastEmit_AArch64ISD_TRN2_MVT_nxv2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8277 if (RetVT.SimpleTy != MVT::nxv2f64) 8278 return 0; 8279 if ((Subtarget->hasSVEorSME())) { 8280 return fastEmitInst_rr(AArch64::TRN2_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op1); 8281 } 8282 return 0; 8283} 8284 8285unsigned fastEmit_AArch64ISD_TRN2_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 8286 switch (VT.SimpleTy) { 8287 case MVT::v8i8: return fastEmit_AArch64ISD_TRN2_MVT_v8i8_rr(RetVT, Op0, Op1); 8288 case MVT::v16i8: return fastEmit_AArch64ISD_TRN2_MVT_v16i8_rr(RetVT, Op0, Op1); 8289 case MVT::v4i16: return fastEmit_AArch64ISD_TRN2_MVT_v4i16_rr(RetVT, Op0, Op1); 8290 case MVT::v8i16: return fastEmit_AArch64ISD_TRN2_MVT_v8i16_rr(RetVT, Op0, Op1); 8291 case MVT::v2i32: return fastEmit_AArch64ISD_TRN2_MVT_v2i32_rr(RetVT, Op0, Op1); 8292 case MVT::v4i32: return fastEmit_AArch64ISD_TRN2_MVT_v4i32_rr(RetVT, Op0, Op1); 8293 case MVT::v2i64: return fastEmit_AArch64ISD_TRN2_MVT_v2i64_rr(RetVT, Op0, Op1); 8294 case MVT::v4f16: return fastEmit_AArch64ISD_TRN2_MVT_v4f16_rr(RetVT, Op0, Op1); 8295 case MVT::v8f16: return fastEmit_AArch64ISD_TRN2_MVT_v8f16_rr(RetVT, Op0, Op1); 8296 case MVT::v4bf16: return fastEmit_AArch64ISD_TRN2_MVT_v4bf16_rr(RetVT, Op0, Op1); 8297 case MVT::v8bf16: return fastEmit_AArch64ISD_TRN2_MVT_v8bf16_rr(RetVT, Op0, Op1); 8298 case MVT::v2f32: return fastEmit_AArch64ISD_TRN2_MVT_v2f32_rr(RetVT, Op0, Op1); 8299 case MVT::v4f32: return fastEmit_AArch64ISD_TRN2_MVT_v4f32_rr(RetVT, Op0, Op1); 8300 case MVT::v2f64: return fastEmit_AArch64ISD_TRN2_MVT_v2f64_rr(RetVT, Op0, Op1); 8301 case MVT::nxv2i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv2i1_rr(RetVT, Op0, Op1); 8302 case MVT::nxv4i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv4i1_rr(RetVT, Op0, Op1); 8303 case MVT::nxv8i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv8i1_rr(RetVT, Op0, Op1); 8304 case MVT::nxv16i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv16i1_rr(RetVT, Op0, Op1); 8305 case MVT::nxv16i8: return fastEmit_AArch64ISD_TRN2_MVT_nxv16i8_rr(RetVT, Op0, Op1); 8306 case MVT::nxv8i16: return fastEmit_AArch64ISD_TRN2_MVT_nxv8i16_rr(RetVT, Op0, Op1); 8307 case MVT::nxv4i32: return fastEmit_AArch64ISD_TRN2_MVT_nxv4i32_rr(RetVT, Op0, Op1); 8308 case MVT::nxv2i64: return fastEmit_AArch64ISD_TRN2_MVT_nxv2i64_rr(RetVT, Op0, Op1); 8309 case MVT::nxv2f16: return fastEmit_AArch64ISD_TRN2_MVT_nxv2f16_rr(RetVT, Op0, Op1); 8310 case MVT::nxv4f16: return fastEmit_AArch64ISD_TRN2_MVT_nxv4f16_rr(RetVT, Op0, Op1); 8311 case MVT::nxv8f16: return fastEmit_AArch64ISD_TRN2_MVT_nxv8f16_rr(RetVT, Op0, Op1); 8312 case MVT::nxv8bf16: return fastEmit_AArch64ISD_TRN2_MVT_nxv8bf16_rr(RetVT, Op0, Op1); 8313 case MVT::nxv2f32: return fastEmit_AArch64ISD_TRN2_MVT_nxv2f32_rr(RetVT, Op0, Op1); 8314 case MVT::nxv4f32: return fastEmit_AArch64ISD_TRN2_MVT_nxv4f32_rr(RetVT, Op0, Op1); 8315 case MVT::nxv2f64: return fastEmit_AArch64ISD_TRN2_MVT_nxv2f64_rr(RetVT, Op0, Op1); 8316 default: return 0; 8317 } 8318} 8319 8320// FastEmit functions for AArch64ISD::UMULL. 8321 8322unsigned fastEmit_AArch64ISD_UMULL_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8323 if (RetVT.SimpleTy != MVT::v8i16) 8324 return 0; 8325 if ((Subtarget->hasNEON())) { 8326 return fastEmitInst_rr(AArch64::UMULLv8i8_v8i16, &AArch64::FPR128RegClass, Op0, Op1); 8327 } 8328 return 0; 8329} 8330 8331unsigned fastEmit_AArch64ISD_UMULL_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8332 if (RetVT.SimpleTy != MVT::v4i32) 8333 return 0; 8334 if ((Subtarget->hasNEON())) { 8335 return fastEmitInst_rr(AArch64::UMULLv4i16_v4i32, &AArch64::FPR128RegClass, Op0, Op1); 8336 } 8337 return 0; 8338} 8339 8340unsigned fastEmit_AArch64ISD_UMULL_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8341 if (RetVT.SimpleTy != MVT::v2i64) 8342 return 0; 8343 if ((Subtarget->hasNEON())) { 8344 return fastEmitInst_rr(AArch64::UMULLv2i32_v2i64, &AArch64::FPR128RegClass, Op0, Op1); 8345 } 8346 return 0; 8347} 8348 8349unsigned fastEmit_AArch64ISD_UMULL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 8350 switch (VT.SimpleTy) { 8351 case MVT::v8i8: return fastEmit_AArch64ISD_UMULL_MVT_v8i8_rr(RetVT, Op0, Op1); 8352 case MVT::v4i16: return fastEmit_AArch64ISD_UMULL_MVT_v4i16_rr(RetVT, Op0, Op1); 8353 case MVT::v2i32: return fastEmit_AArch64ISD_UMULL_MVT_v2i32_rr(RetVT, Op0, Op1); 8354 default: return 0; 8355 } 8356} 8357 8358// FastEmit functions for AArch64ISD::UZP1. 8359 8360unsigned fastEmit_AArch64ISD_UZP1_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8361 if (RetVT.SimpleTy != MVT::v8i8) 8362 return 0; 8363 if ((Subtarget->hasNEON())) { 8364 return fastEmitInst_rr(AArch64::UZP1v8i8, &AArch64::FPR64RegClass, Op0, Op1); 8365 } 8366 return 0; 8367} 8368 8369unsigned fastEmit_AArch64ISD_UZP1_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8370 if (RetVT.SimpleTy != MVT::v16i8) 8371 return 0; 8372 if ((Subtarget->hasNEON())) { 8373 return fastEmitInst_rr(AArch64::UZP1v16i8, &AArch64::FPR128RegClass, Op0, Op1); 8374 } 8375 return 0; 8376} 8377 8378unsigned fastEmit_AArch64ISD_UZP1_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8379 if (RetVT.SimpleTy != MVT::v4i16) 8380 return 0; 8381 if ((Subtarget->hasNEON())) { 8382 return fastEmitInst_rr(AArch64::UZP1v4i16, &AArch64::FPR64RegClass, Op0, Op1); 8383 } 8384 return 0; 8385} 8386 8387unsigned fastEmit_AArch64ISD_UZP1_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8388 if (RetVT.SimpleTy != MVT::v8i16) 8389 return 0; 8390 if ((Subtarget->hasNEON())) { 8391 return fastEmitInst_rr(AArch64::UZP1v8i16, &AArch64::FPR128RegClass, Op0, Op1); 8392 } 8393 return 0; 8394} 8395 8396unsigned fastEmit_AArch64ISD_UZP1_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8397 if (RetVT.SimpleTy != MVT::v2i32) 8398 return 0; 8399 if ((Subtarget->hasNEON())) { 8400 return fastEmitInst_rr(AArch64::UZP1v2i32, &AArch64::FPR64RegClass, Op0, Op1); 8401 } 8402 return 0; 8403} 8404 8405unsigned fastEmit_AArch64ISD_UZP1_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8406 if (RetVT.SimpleTy != MVT::v4i32) 8407 return 0; 8408 if ((Subtarget->hasNEON())) { 8409 return fastEmitInst_rr(AArch64::UZP1v4i32, &AArch64::FPR128RegClass, Op0, Op1); 8410 } 8411 return 0; 8412} 8413 8414unsigned fastEmit_AArch64ISD_UZP1_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8415 if (RetVT.SimpleTy != MVT::v2i64) 8416 return 0; 8417 if ((Subtarget->hasNEON())) { 8418 return fastEmitInst_rr(AArch64::UZP1v2i64, &AArch64::FPR128RegClass, Op0, Op1); 8419 } 8420 return 0; 8421} 8422 8423unsigned fastEmit_AArch64ISD_UZP1_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8424 if (RetVT.SimpleTy != MVT::v4f16) 8425 return 0; 8426 if ((Subtarget->hasNEON())) { 8427 return fastEmitInst_rr(AArch64::UZP1v4i16, &AArch64::FPR64RegClass, Op0, Op1); 8428 } 8429 return 0; 8430} 8431 8432unsigned fastEmit_AArch64ISD_UZP1_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8433 if (RetVT.SimpleTy != MVT::v8f16) 8434 return 0; 8435 if ((Subtarget->hasNEON())) { 8436 return fastEmitInst_rr(AArch64::UZP1v8i16, &AArch64::FPR128RegClass, Op0, Op1); 8437 } 8438 return 0; 8439} 8440 8441unsigned fastEmit_AArch64ISD_UZP1_MVT_v4bf16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8442 if (RetVT.SimpleTy != MVT::v4bf16) 8443 return 0; 8444 if ((Subtarget->hasNEON())) { 8445 return fastEmitInst_rr(AArch64::UZP1v4i16, &AArch64::FPR64RegClass, Op0, Op1); 8446 } 8447 return 0; 8448} 8449 8450unsigned fastEmit_AArch64ISD_UZP1_MVT_v8bf16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8451 if (RetVT.SimpleTy != MVT::v8bf16) 8452 return 0; 8453 if ((Subtarget->hasNEON())) { 8454 return fastEmitInst_rr(AArch64::UZP1v8i16, &AArch64::FPR128RegClass, Op0, Op1); 8455 } 8456 return 0; 8457} 8458 8459unsigned fastEmit_AArch64ISD_UZP1_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8460 if (RetVT.SimpleTy != MVT::v2f32) 8461 return 0; 8462 if ((Subtarget->hasNEON())) { 8463 return fastEmitInst_rr(AArch64::UZP1v2i32, &AArch64::FPR64RegClass, Op0, Op1); 8464 } 8465 return 0; 8466} 8467 8468unsigned fastEmit_AArch64ISD_UZP1_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8469 if (RetVT.SimpleTy != MVT::v4f32) 8470 return 0; 8471 if ((Subtarget->hasNEON())) { 8472 return fastEmitInst_rr(AArch64::UZP1v4i32, &AArch64::FPR128RegClass, Op0, Op1); 8473 } 8474 return 0; 8475} 8476 8477unsigned fastEmit_AArch64ISD_UZP1_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8478 if (RetVT.SimpleTy != MVT::v2f64) 8479 return 0; 8480 if ((Subtarget->hasNEON())) { 8481 return fastEmitInst_rr(AArch64::UZP1v2i64, &AArch64::FPR128RegClass, Op0, Op1); 8482 } 8483 return 0; 8484} 8485 8486unsigned fastEmit_AArch64ISD_UZP1_MVT_nxv2i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8487 if (RetVT.SimpleTy != MVT::nxv2i1) 8488 return 0; 8489 if ((Subtarget->hasSVEorSME())) { 8490 return fastEmitInst_rr(AArch64::UZP1_PPP_D, &AArch64::PPRRegClass, Op0, Op1); 8491 } 8492 return 0; 8493} 8494 8495unsigned fastEmit_AArch64ISD_UZP1_MVT_nxv4i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8496 if (RetVT.SimpleTy != MVT::nxv4i1) 8497 return 0; 8498 if ((Subtarget->hasSVEorSME())) { 8499 return fastEmitInst_rr(AArch64::UZP1_PPP_S, &AArch64::PPRRegClass, Op0, Op1); 8500 } 8501 return 0; 8502} 8503 8504unsigned fastEmit_AArch64ISD_UZP1_MVT_nxv8i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8505 if (RetVT.SimpleTy != MVT::nxv8i1) 8506 return 0; 8507 if ((Subtarget->hasSVEorSME())) { 8508 return fastEmitInst_rr(AArch64::UZP1_PPP_H, &AArch64::PPRRegClass, Op0, Op1); 8509 } 8510 return 0; 8511} 8512 8513unsigned fastEmit_AArch64ISD_UZP1_MVT_nxv16i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8514 if (RetVT.SimpleTy != MVT::nxv16i1) 8515 return 0; 8516 if ((Subtarget->hasSVEorSME())) { 8517 return fastEmitInst_rr(AArch64::UZP1_PPP_B, &AArch64::PPRRegClass, Op0, Op1); 8518 } 8519 return 0; 8520} 8521 8522unsigned fastEmit_AArch64ISD_UZP1_MVT_nxv16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8523 if (RetVT.SimpleTy != MVT::nxv16i8) 8524 return 0; 8525 if ((Subtarget->hasSVEorSME())) { 8526 return fastEmitInst_rr(AArch64::UZP1_ZZZ_B, &AArch64::ZPRRegClass, Op0, Op1); 8527 } 8528 return 0; 8529} 8530 8531unsigned fastEmit_AArch64ISD_UZP1_MVT_nxv8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8532 if (RetVT.SimpleTy != MVT::nxv8i16) 8533 return 0; 8534 if ((Subtarget->hasSVEorSME())) { 8535 return fastEmitInst_rr(AArch64::UZP1_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op1); 8536 } 8537 return 0; 8538} 8539 8540unsigned fastEmit_AArch64ISD_UZP1_MVT_nxv4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8541 if (RetVT.SimpleTy != MVT::nxv4i32) 8542 return 0; 8543 if ((Subtarget->hasSVEorSME())) { 8544 return fastEmitInst_rr(AArch64::UZP1_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op1); 8545 } 8546 return 0; 8547} 8548 8549unsigned fastEmit_AArch64ISD_UZP1_MVT_nxv2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8550 if (RetVT.SimpleTy != MVT::nxv2i64) 8551 return 0; 8552 if ((Subtarget->hasSVEorSME())) { 8553 return fastEmitInst_rr(AArch64::UZP1_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op1); 8554 } 8555 return 0; 8556} 8557 8558unsigned fastEmit_AArch64ISD_UZP1_MVT_nxv2f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8559 if (RetVT.SimpleTy != MVT::nxv2f16) 8560 return 0; 8561 if ((Subtarget->hasSVEorSME())) { 8562 return fastEmitInst_rr(AArch64::UZP1_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op1); 8563 } 8564 return 0; 8565} 8566 8567unsigned fastEmit_AArch64ISD_UZP1_MVT_nxv4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8568 if (RetVT.SimpleTy != MVT::nxv4f16) 8569 return 0; 8570 if ((Subtarget->hasSVEorSME())) { 8571 return fastEmitInst_rr(AArch64::UZP1_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op1); 8572 } 8573 return 0; 8574} 8575 8576unsigned fastEmit_AArch64ISD_UZP1_MVT_nxv8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8577 if (RetVT.SimpleTy != MVT::nxv8f16) 8578 return 0; 8579 if ((Subtarget->hasSVEorSME())) { 8580 return fastEmitInst_rr(AArch64::UZP1_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op1); 8581 } 8582 return 0; 8583} 8584 8585unsigned fastEmit_AArch64ISD_UZP1_MVT_nxv8bf16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8586 if (RetVT.SimpleTy != MVT::nxv8bf16) 8587 return 0; 8588 if ((Subtarget->hasSVEorSME())) { 8589 return fastEmitInst_rr(AArch64::UZP1_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op1); 8590 } 8591 return 0; 8592} 8593 8594unsigned fastEmit_AArch64ISD_UZP1_MVT_nxv2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8595 if (RetVT.SimpleTy != MVT::nxv2f32) 8596 return 0; 8597 if ((Subtarget->hasSVEorSME())) { 8598 return fastEmitInst_rr(AArch64::UZP1_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op1); 8599 } 8600 return 0; 8601} 8602 8603unsigned fastEmit_AArch64ISD_UZP1_MVT_nxv4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8604 if (RetVT.SimpleTy != MVT::nxv4f32) 8605 return 0; 8606 if ((Subtarget->hasSVEorSME())) { 8607 return fastEmitInst_rr(AArch64::UZP1_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op1); 8608 } 8609 return 0; 8610} 8611 8612unsigned fastEmit_AArch64ISD_UZP1_MVT_nxv2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8613 if (RetVT.SimpleTy != MVT::nxv2f64) 8614 return 0; 8615 if ((Subtarget->hasSVEorSME())) { 8616 return fastEmitInst_rr(AArch64::UZP1_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op1); 8617 } 8618 return 0; 8619} 8620 8621unsigned fastEmit_AArch64ISD_UZP1_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 8622 switch (VT.SimpleTy) { 8623 case MVT::v8i8: return fastEmit_AArch64ISD_UZP1_MVT_v8i8_rr(RetVT, Op0, Op1); 8624 case MVT::v16i8: return fastEmit_AArch64ISD_UZP1_MVT_v16i8_rr(RetVT, Op0, Op1); 8625 case MVT::v4i16: return fastEmit_AArch64ISD_UZP1_MVT_v4i16_rr(RetVT, Op0, Op1); 8626 case MVT::v8i16: return fastEmit_AArch64ISD_UZP1_MVT_v8i16_rr(RetVT, Op0, Op1); 8627 case MVT::v2i32: return fastEmit_AArch64ISD_UZP1_MVT_v2i32_rr(RetVT, Op0, Op1); 8628 case MVT::v4i32: return fastEmit_AArch64ISD_UZP1_MVT_v4i32_rr(RetVT, Op0, Op1); 8629 case MVT::v2i64: return fastEmit_AArch64ISD_UZP1_MVT_v2i64_rr(RetVT, Op0, Op1); 8630 case MVT::v4f16: return fastEmit_AArch64ISD_UZP1_MVT_v4f16_rr(RetVT, Op0, Op1); 8631 case MVT::v8f16: return fastEmit_AArch64ISD_UZP1_MVT_v8f16_rr(RetVT, Op0, Op1); 8632 case MVT::v4bf16: return fastEmit_AArch64ISD_UZP1_MVT_v4bf16_rr(RetVT, Op0, Op1); 8633 case MVT::v8bf16: return fastEmit_AArch64ISD_UZP1_MVT_v8bf16_rr(RetVT, Op0, Op1); 8634 case MVT::v2f32: return fastEmit_AArch64ISD_UZP1_MVT_v2f32_rr(RetVT, Op0, Op1); 8635 case MVT::v4f32: return fastEmit_AArch64ISD_UZP1_MVT_v4f32_rr(RetVT, Op0, Op1); 8636 case MVT::v2f64: return fastEmit_AArch64ISD_UZP1_MVT_v2f64_rr(RetVT, Op0, Op1); 8637 case MVT::nxv2i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv2i1_rr(RetVT, Op0, Op1); 8638 case MVT::nxv4i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv4i1_rr(RetVT, Op0, Op1); 8639 case MVT::nxv8i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv8i1_rr(RetVT, Op0, Op1); 8640 case MVT::nxv16i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv16i1_rr(RetVT, Op0, Op1); 8641 case MVT::nxv16i8: return fastEmit_AArch64ISD_UZP1_MVT_nxv16i8_rr(RetVT, Op0, Op1); 8642 case MVT::nxv8i16: return fastEmit_AArch64ISD_UZP1_MVT_nxv8i16_rr(RetVT, Op0, Op1); 8643 case MVT::nxv4i32: return fastEmit_AArch64ISD_UZP1_MVT_nxv4i32_rr(RetVT, Op0, Op1); 8644 case MVT::nxv2i64: return fastEmit_AArch64ISD_UZP1_MVT_nxv2i64_rr(RetVT, Op0, Op1); 8645 case MVT::nxv2f16: return fastEmit_AArch64ISD_UZP1_MVT_nxv2f16_rr(RetVT, Op0, Op1); 8646 case MVT::nxv4f16: return fastEmit_AArch64ISD_UZP1_MVT_nxv4f16_rr(RetVT, Op0, Op1); 8647 case MVT::nxv8f16: return fastEmit_AArch64ISD_UZP1_MVT_nxv8f16_rr(RetVT, Op0, Op1); 8648 case MVT::nxv8bf16: return fastEmit_AArch64ISD_UZP1_MVT_nxv8bf16_rr(RetVT, Op0, Op1); 8649 case MVT::nxv2f32: return fastEmit_AArch64ISD_UZP1_MVT_nxv2f32_rr(RetVT, Op0, Op1); 8650 case MVT::nxv4f32: return fastEmit_AArch64ISD_UZP1_MVT_nxv4f32_rr(RetVT, Op0, Op1); 8651 case MVT::nxv2f64: return fastEmit_AArch64ISD_UZP1_MVT_nxv2f64_rr(RetVT, Op0, Op1); 8652 default: return 0; 8653 } 8654} 8655 8656// FastEmit functions for AArch64ISD::UZP2. 8657 8658unsigned fastEmit_AArch64ISD_UZP2_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8659 if (RetVT.SimpleTy != MVT::v8i8) 8660 return 0; 8661 if ((Subtarget->hasNEON())) { 8662 return fastEmitInst_rr(AArch64::UZP2v8i8, &AArch64::FPR64RegClass, Op0, Op1); 8663 } 8664 return 0; 8665} 8666 8667unsigned fastEmit_AArch64ISD_UZP2_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8668 if (RetVT.SimpleTy != MVT::v16i8) 8669 return 0; 8670 if ((Subtarget->hasNEON())) { 8671 return fastEmitInst_rr(AArch64::UZP2v16i8, &AArch64::FPR128RegClass, Op0, Op1); 8672 } 8673 return 0; 8674} 8675 8676unsigned fastEmit_AArch64ISD_UZP2_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8677 if (RetVT.SimpleTy != MVT::v4i16) 8678 return 0; 8679 if ((Subtarget->hasNEON())) { 8680 return fastEmitInst_rr(AArch64::UZP2v4i16, &AArch64::FPR64RegClass, Op0, Op1); 8681 } 8682 return 0; 8683} 8684 8685unsigned fastEmit_AArch64ISD_UZP2_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8686 if (RetVT.SimpleTy != MVT::v8i16) 8687 return 0; 8688 if ((Subtarget->hasNEON())) { 8689 return fastEmitInst_rr(AArch64::UZP2v8i16, &AArch64::FPR128RegClass, Op0, Op1); 8690 } 8691 return 0; 8692} 8693 8694unsigned fastEmit_AArch64ISD_UZP2_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8695 if (RetVT.SimpleTy != MVT::v2i32) 8696 return 0; 8697 if ((Subtarget->hasNEON())) { 8698 return fastEmitInst_rr(AArch64::UZP2v2i32, &AArch64::FPR64RegClass, Op0, Op1); 8699 } 8700 return 0; 8701} 8702 8703unsigned fastEmit_AArch64ISD_UZP2_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8704 if (RetVT.SimpleTy != MVT::v4i32) 8705 return 0; 8706 if ((Subtarget->hasNEON())) { 8707 return fastEmitInst_rr(AArch64::UZP2v4i32, &AArch64::FPR128RegClass, Op0, Op1); 8708 } 8709 return 0; 8710} 8711 8712unsigned fastEmit_AArch64ISD_UZP2_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8713 if (RetVT.SimpleTy != MVT::v2i64) 8714 return 0; 8715 if ((Subtarget->hasNEON())) { 8716 return fastEmitInst_rr(AArch64::UZP2v2i64, &AArch64::FPR128RegClass, Op0, Op1); 8717 } 8718 return 0; 8719} 8720 8721unsigned fastEmit_AArch64ISD_UZP2_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8722 if (RetVT.SimpleTy != MVT::v4f16) 8723 return 0; 8724 if ((Subtarget->hasNEON())) { 8725 return fastEmitInst_rr(AArch64::UZP2v4i16, &AArch64::FPR64RegClass, Op0, Op1); 8726 } 8727 return 0; 8728} 8729 8730unsigned fastEmit_AArch64ISD_UZP2_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8731 if (RetVT.SimpleTy != MVT::v8f16) 8732 return 0; 8733 if ((Subtarget->hasNEON())) { 8734 return fastEmitInst_rr(AArch64::UZP2v8i16, &AArch64::FPR128RegClass, Op0, Op1); 8735 } 8736 return 0; 8737} 8738 8739unsigned fastEmit_AArch64ISD_UZP2_MVT_v4bf16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8740 if (RetVT.SimpleTy != MVT::v4bf16) 8741 return 0; 8742 if ((Subtarget->hasNEON())) { 8743 return fastEmitInst_rr(AArch64::UZP2v4i16, &AArch64::FPR64RegClass, Op0, Op1); 8744 } 8745 return 0; 8746} 8747 8748unsigned fastEmit_AArch64ISD_UZP2_MVT_v8bf16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8749 if (RetVT.SimpleTy != MVT::v8bf16) 8750 return 0; 8751 if ((Subtarget->hasNEON())) { 8752 return fastEmitInst_rr(AArch64::UZP2v8i16, &AArch64::FPR128RegClass, Op0, Op1); 8753 } 8754 return 0; 8755} 8756 8757unsigned fastEmit_AArch64ISD_UZP2_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8758 if (RetVT.SimpleTy != MVT::v2f32) 8759 return 0; 8760 if ((Subtarget->hasNEON())) { 8761 return fastEmitInst_rr(AArch64::UZP2v2i32, &AArch64::FPR64RegClass, Op0, Op1); 8762 } 8763 return 0; 8764} 8765 8766unsigned fastEmit_AArch64ISD_UZP2_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8767 if (RetVT.SimpleTy != MVT::v4f32) 8768 return 0; 8769 if ((Subtarget->hasNEON())) { 8770 return fastEmitInst_rr(AArch64::UZP2v4i32, &AArch64::FPR128RegClass, Op0, Op1); 8771 } 8772 return 0; 8773} 8774 8775unsigned fastEmit_AArch64ISD_UZP2_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8776 if (RetVT.SimpleTy != MVT::v2f64) 8777 return 0; 8778 if ((Subtarget->hasNEON())) { 8779 return fastEmitInst_rr(AArch64::UZP2v2i64, &AArch64::FPR128RegClass, Op0, Op1); 8780 } 8781 return 0; 8782} 8783 8784unsigned fastEmit_AArch64ISD_UZP2_MVT_nxv2i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8785 if (RetVT.SimpleTy != MVT::nxv2i1) 8786 return 0; 8787 if ((Subtarget->hasSVEorSME())) { 8788 return fastEmitInst_rr(AArch64::UZP2_PPP_D, &AArch64::PPRRegClass, Op0, Op1); 8789 } 8790 return 0; 8791} 8792 8793unsigned fastEmit_AArch64ISD_UZP2_MVT_nxv4i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8794 if (RetVT.SimpleTy != MVT::nxv4i1) 8795 return 0; 8796 if ((Subtarget->hasSVEorSME())) { 8797 return fastEmitInst_rr(AArch64::UZP2_PPP_S, &AArch64::PPRRegClass, Op0, Op1); 8798 } 8799 return 0; 8800} 8801 8802unsigned fastEmit_AArch64ISD_UZP2_MVT_nxv8i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8803 if (RetVT.SimpleTy != MVT::nxv8i1) 8804 return 0; 8805 if ((Subtarget->hasSVEorSME())) { 8806 return fastEmitInst_rr(AArch64::UZP2_PPP_H, &AArch64::PPRRegClass, Op0, Op1); 8807 } 8808 return 0; 8809} 8810 8811unsigned fastEmit_AArch64ISD_UZP2_MVT_nxv16i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8812 if (RetVT.SimpleTy != MVT::nxv16i1) 8813 return 0; 8814 if ((Subtarget->hasSVEorSME())) { 8815 return fastEmitInst_rr(AArch64::UZP2_PPP_B, &AArch64::PPRRegClass, Op0, Op1); 8816 } 8817 return 0; 8818} 8819 8820unsigned fastEmit_AArch64ISD_UZP2_MVT_nxv16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8821 if (RetVT.SimpleTy != MVT::nxv16i8) 8822 return 0; 8823 if ((Subtarget->hasSVEorSME())) { 8824 return fastEmitInst_rr(AArch64::UZP2_ZZZ_B, &AArch64::ZPRRegClass, Op0, Op1); 8825 } 8826 return 0; 8827} 8828 8829unsigned fastEmit_AArch64ISD_UZP2_MVT_nxv8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8830 if (RetVT.SimpleTy != MVT::nxv8i16) 8831 return 0; 8832 if ((Subtarget->hasSVEorSME())) { 8833 return fastEmitInst_rr(AArch64::UZP2_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op1); 8834 } 8835 return 0; 8836} 8837 8838unsigned fastEmit_AArch64ISD_UZP2_MVT_nxv4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8839 if (RetVT.SimpleTy != MVT::nxv4i32) 8840 return 0; 8841 if ((Subtarget->hasSVEorSME())) { 8842 return fastEmitInst_rr(AArch64::UZP2_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op1); 8843 } 8844 return 0; 8845} 8846 8847unsigned fastEmit_AArch64ISD_UZP2_MVT_nxv2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8848 if (RetVT.SimpleTy != MVT::nxv2i64) 8849 return 0; 8850 if ((Subtarget->hasSVEorSME())) { 8851 return fastEmitInst_rr(AArch64::UZP2_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op1); 8852 } 8853 return 0; 8854} 8855 8856unsigned fastEmit_AArch64ISD_UZP2_MVT_nxv2f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8857 if (RetVT.SimpleTy != MVT::nxv2f16) 8858 return 0; 8859 if ((Subtarget->hasSVEorSME())) { 8860 return fastEmitInst_rr(AArch64::UZP2_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op1); 8861 } 8862 return 0; 8863} 8864 8865unsigned fastEmit_AArch64ISD_UZP2_MVT_nxv4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8866 if (RetVT.SimpleTy != MVT::nxv4f16) 8867 return 0; 8868 if ((Subtarget->hasSVEorSME())) { 8869 return fastEmitInst_rr(AArch64::UZP2_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op1); 8870 } 8871 return 0; 8872} 8873 8874unsigned fastEmit_AArch64ISD_UZP2_MVT_nxv8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8875 if (RetVT.SimpleTy != MVT::nxv8f16) 8876 return 0; 8877 if ((Subtarget->hasSVEorSME())) { 8878 return fastEmitInst_rr(AArch64::UZP2_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op1); 8879 } 8880 return 0; 8881} 8882 8883unsigned fastEmit_AArch64ISD_UZP2_MVT_nxv8bf16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8884 if (RetVT.SimpleTy != MVT::nxv8bf16) 8885 return 0; 8886 if ((Subtarget->hasSVEorSME())) { 8887 return fastEmitInst_rr(AArch64::UZP2_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op1); 8888 } 8889 return 0; 8890} 8891 8892unsigned fastEmit_AArch64ISD_UZP2_MVT_nxv2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8893 if (RetVT.SimpleTy != MVT::nxv2f32) 8894 return 0; 8895 if ((Subtarget->hasSVEorSME())) { 8896 return fastEmitInst_rr(AArch64::UZP2_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op1); 8897 } 8898 return 0; 8899} 8900 8901unsigned fastEmit_AArch64ISD_UZP2_MVT_nxv4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8902 if (RetVT.SimpleTy != MVT::nxv4f32) 8903 return 0; 8904 if ((Subtarget->hasSVEorSME())) { 8905 return fastEmitInst_rr(AArch64::UZP2_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op1); 8906 } 8907 return 0; 8908} 8909 8910unsigned fastEmit_AArch64ISD_UZP2_MVT_nxv2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8911 if (RetVT.SimpleTy != MVT::nxv2f64) 8912 return 0; 8913 if ((Subtarget->hasSVEorSME())) { 8914 return fastEmitInst_rr(AArch64::UZP2_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op1); 8915 } 8916 return 0; 8917} 8918 8919unsigned fastEmit_AArch64ISD_UZP2_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 8920 switch (VT.SimpleTy) { 8921 case MVT::v8i8: return fastEmit_AArch64ISD_UZP2_MVT_v8i8_rr(RetVT, Op0, Op1); 8922 case MVT::v16i8: return fastEmit_AArch64ISD_UZP2_MVT_v16i8_rr(RetVT, Op0, Op1); 8923 case MVT::v4i16: return fastEmit_AArch64ISD_UZP2_MVT_v4i16_rr(RetVT, Op0, Op1); 8924 case MVT::v8i16: return fastEmit_AArch64ISD_UZP2_MVT_v8i16_rr(RetVT, Op0, Op1); 8925 case MVT::v2i32: return fastEmit_AArch64ISD_UZP2_MVT_v2i32_rr(RetVT, Op0, Op1); 8926 case MVT::v4i32: return fastEmit_AArch64ISD_UZP2_MVT_v4i32_rr(RetVT, Op0, Op1); 8927 case MVT::v2i64: return fastEmit_AArch64ISD_UZP2_MVT_v2i64_rr(RetVT, Op0, Op1); 8928 case MVT::v4f16: return fastEmit_AArch64ISD_UZP2_MVT_v4f16_rr(RetVT, Op0, Op1); 8929 case MVT::v8f16: return fastEmit_AArch64ISD_UZP2_MVT_v8f16_rr(RetVT, Op0, Op1); 8930 case MVT::v4bf16: return fastEmit_AArch64ISD_UZP2_MVT_v4bf16_rr(RetVT, Op0, Op1); 8931 case MVT::v8bf16: return fastEmit_AArch64ISD_UZP2_MVT_v8bf16_rr(RetVT, Op0, Op1); 8932 case MVT::v2f32: return fastEmit_AArch64ISD_UZP2_MVT_v2f32_rr(RetVT, Op0, Op1); 8933 case MVT::v4f32: return fastEmit_AArch64ISD_UZP2_MVT_v4f32_rr(RetVT, Op0, Op1); 8934 case MVT::v2f64: return fastEmit_AArch64ISD_UZP2_MVT_v2f64_rr(RetVT, Op0, Op1); 8935 case MVT::nxv2i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv2i1_rr(RetVT, Op0, Op1); 8936 case MVT::nxv4i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv4i1_rr(RetVT, Op0, Op1); 8937 case MVT::nxv8i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv8i1_rr(RetVT, Op0, Op1); 8938 case MVT::nxv16i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv16i1_rr(RetVT, Op0, Op1); 8939 case MVT::nxv16i8: return fastEmit_AArch64ISD_UZP2_MVT_nxv16i8_rr(RetVT, Op0, Op1); 8940 case MVT::nxv8i16: return fastEmit_AArch64ISD_UZP2_MVT_nxv8i16_rr(RetVT, Op0, Op1); 8941 case MVT::nxv4i32: return fastEmit_AArch64ISD_UZP2_MVT_nxv4i32_rr(RetVT, Op0, Op1); 8942 case MVT::nxv2i64: return fastEmit_AArch64ISD_UZP2_MVT_nxv2i64_rr(RetVT, Op0, Op1); 8943 case MVT::nxv2f16: return fastEmit_AArch64ISD_UZP2_MVT_nxv2f16_rr(RetVT, Op0, Op1); 8944 case MVT::nxv4f16: return fastEmit_AArch64ISD_UZP2_MVT_nxv4f16_rr(RetVT, Op0, Op1); 8945 case MVT::nxv8f16: return fastEmit_AArch64ISD_UZP2_MVT_nxv8f16_rr(RetVT, Op0, Op1); 8946 case MVT::nxv8bf16: return fastEmit_AArch64ISD_UZP2_MVT_nxv8bf16_rr(RetVT, Op0, Op1); 8947 case MVT::nxv2f32: return fastEmit_AArch64ISD_UZP2_MVT_nxv2f32_rr(RetVT, Op0, Op1); 8948 case MVT::nxv4f32: return fastEmit_AArch64ISD_UZP2_MVT_nxv4f32_rr(RetVT, Op0, Op1); 8949 case MVT::nxv2f64: return fastEmit_AArch64ISD_UZP2_MVT_nxv2f64_rr(RetVT, Op0, Op1); 8950 default: return 0; 8951 } 8952} 8953 8954// FastEmit functions for AArch64ISD::ZIP1. 8955 8956unsigned fastEmit_AArch64ISD_ZIP1_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8957 if (RetVT.SimpleTy != MVT::v8i8) 8958 return 0; 8959 if ((Subtarget->hasNEON())) { 8960 return fastEmitInst_rr(AArch64::ZIP1v8i8, &AArch64::FPR64RegClass, Op0, Op1); 8961 } 8962 return 0; 8963} 8964 8965unsigned fastEmit_AArch64ISD_ZIP1_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8966 if (RetVT.SimpleTy != MVT::v16i8) 8967 return 0; 8968 if ((Subtarget->hasNEON())) { 8969 return fastEmitInst_rr(AArch64::ZIP1v16i8, &AArch64::FPR128RegClass, Op0, Op1); 8970 } 8971 return 0; 8972} 8973 8974unsigned fastEmit_AArch64ISD_ZIP1_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8975 if (RetVT.SimpleTy != MVT::v4i16) 8976 return 0; 8977 if ((Subtarget->hasNEON())) { 8978 return fastEmitInst_rr(AArch64::ZIP1v4i16, &AArch64::FPR64RegClass, Op0, Op1); 8979 } 8980 return 0; 8981} 8982 8983unsigned fastEmit_AArch64ISD_ZIP1_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8984 if (RetVT.SimpleTy != MVT::v8i16) 8985 return 0; 8986 if ((Subtarget->hasNEON())) { 8987 return fastEmitInst_rr(AArch64::ZIP1v8i16, &AArch64::FPR128RegClass, Op0, Op1); 8988 } 8989 return 0; 8990} 8991 8992unsigned fastEmit_AArch64ISD_ZIP1_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 8993 if (RetVT.SimpleTy != MVT::v2i32) 8994 return 0; 8995 if ((Subtarget->hasNEON())) { 8996 return fastEmitInst_rr(AArch64::ZIP1v2i32, &AArch64::FPR64RegClass, Op0, Op1); 8997 } 8998 return 0; 8999} 9000 9001unsigned fastEmit_AArch64ISD_ZIP1_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9002 if (RetVT.SimpleTy != MVT::v4i32) 9003 return 0; 9004 if ((Subtarget->hasNEON())) { 9005 return fastEmitInst_rr(AArch64::ZIP1v4i32, &AArch64::FPR128RegClass, Op0, Op1); 9006 } 9007 return 0; 9008} 9009 9010unsigned fastEmit_AArch64ISD_ZIP1_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9011 if (RetVT.SimpleTy != MVT::v2i64) 9012 return 0; 9013 if ((Subtarget->hasNEON())) { 9014 return fastEmitInst_rr(AArch64::ZIP1v2i64, &AArch64::FPR128RegClass, Op0, Op1); 9015 } 9016 return 0; 9017} 9018 9019unsigned fastEmit_AArch64ISD_ZIP1_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9020 if (RetVT.SimpleTy != MVT::v4f16) 9021 return 0; 9022 if ((Subtarget->hasNEON())) { 9023 return fastEmitInst_rr(AArch64::ZIP1v4i16, &AArch64::FPR64RegClass, Op0, Op1); 9024 } 9025 return 0; 9026} 9027 9028unsigned fastEmit_AArch64ISD_ZIP1_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9029 if (RetVT.SimpleTy != MVT::v8f16) 9030 return 0; 9031 if ((Subtarget->hasNEON())) { 9032 return fastEmitInst_rr(AArch64::ZIP1v8i16, &AArch64::FPR128RegClass, Op0, Op1); 9033 } 9034 return 0; 9035} 9036 9037unsigned fastEmit_AArch64ISD_ZIP1_MVT_v4bf16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9038 if (RetVT.SimpleTy != MVT::v4bf16) 9039 return 0; 9040 if ((Subtarget->hasNEON())) { 9041 return fastEmitInst_rr(AArch64::ZIP1v4i16, &AArch64::FPR64RegClass, Op0, Op1); 9042 } 9043 return 0; 9044} 9045 9046unsigned fastEmit_AArch64ISD_ZIP1_MVT_v8bf16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9047 if (RetVT.SimpleTy != MVT::v8bf16) 9048 return 0; 9049 if ((Subtarget->hasNEON())) { 9050 return fastEmitInst_rr(AArch64::ZIP1v8i16, &AArch64::FPR128RegClass, Op0, Op1); 9051 } 9052 return 0; 9053} 9054 9055unsigned fastEmit_AArch64ISD_ZIP1_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9056 if (RetVT.SimpleTy != MVT::v2f32) 9057 return 0; 9058 if ((Subtarget->hasNEON())) { 9059 return fastEmitInst_rr(AArch64::ZIP1v2i32, &AArch64::FPR64RegClass, Op0, Op1); 9060 } 9061 return 0; 9062} 9063 9064unsigned fastEmit_AArch64ISD_ZIP1_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9065 if (RetVT.SimpleTy != MVT::v4f32) 9066 return 0; 9067 if ((Subtarget->hasNEON())) { 9068 return fastEmitInst_rr(AArch64::ZIP1v4i32, &AArch64::FPR128RegClass, Op0, Op1); 9069 } 9070 return 0; 9071} 9072 9073unsigned fastEmit_AArch64ISD_ZIP1_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9074 if (RetVT.SimpleTy != MVT::v2f64) 9075 return 0; 9076 if ((Subtarget->hasNEON())) { 9077 return fastEmitInst_rr(AArch64::ZIP1v2i64, &AArch64::FPR128RegClass, Op0, Op1); 9078 } 9079 return 0; 9080} 9081 9082unsigned fastEmit_AArch64ISD_ZIP1_MVT_nxv2i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9083 if (RetVT.SimpleTy != MVT::nxv2i1) 9084 return 0; 9085 if ((Subtarget->hasSVEorSME())) { 9086 return fastEmitInst_rr(AArch64::ZIP1_PPP_D, &AArch64::PPRRegClass, Op0, Op1); 9087 } 9088 return 0; 9089} 9090 9091unsigned fastEmit_AArch64ISD_ZIP1_MVT_nxv4i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9092 if (RetVT.SimpleTy != MVT::nxv4i1) 9093 return 0; 9094 if ((Subtarget->hasSVEorSME())) { 9095 return fastEmitInst_rr(AArch64::ZIP1_PPP_S, &AArch64::PPRRegClass, Op0, Op1); 9096 } 9097 return 0; 9098} 9099 9100unsigned fastEmit_AArch64ISD_ZIP1_MVT_nxv8i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9101 if (RetVT.SimpleTy != MVT::nxv8i1) 9102 return 0; 9103 if ((Subtarget->hasSVEorSME())) { 9104 return fastEmitInst_rr(AArch64::ZIP1_PPP_H, &AArch64::PPRRegClass, Op0, Op1); 9105 } 9106 return 0; 9107} 9108 9109unsigned fastEmit_AArch64ISD_ZIP1_MVT_nxv16i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9110 if (RetVT.SimpleTy != MVT::nxv16i1) 9111 return 0; 9112 if ((Subtarget->hasSVEorSME())) { 9113 return fastEmitInst_rr(AArch64::ZIP1_PPP_B, &AArch64::PPRRegClass, Op0, Op1); 9114 } 9115 return 0; 9116} 9117 9118unsigned fastEmit_AArch64ISD_ZIP1_MVT_nxv16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9119 if (RetVT.SimpleTy != MVT::nxv16i8) 9120 return 0; 9121 if ((Subtarget->hasSVEorSME())) { 9122 return fastEmitInst_rr(AArch64::ZIP1_ZZZ_B, &AArch64::ZPRRegClass, Op0, Op1); 9123 } 9124 return 0; 9125} 9126 9127unsigned fastEmit_AArch64ISD_ZIP1_MVT_nxv8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9128 if (RetVT.SimpleTy != MVT::nxv8i16) 9129 return 0; 9130 if ((Subtarget->hasSVEorSME())) { 9131 return fastEmitInst_rr(AArch64::ZIP1_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op1); 9132 } 9133 return 0; 9134} 9135 9136unsigned fastEmit_AArch64ISD_ZIP1_MVT_nxv4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9137 if (RetVT.SimpleTy != MVT::nxv4i32) 9138 return 0; 9139 if ((Subtarget->hasSVEorSME())) { 9140 return fastEmitInst_rr(AArch64::ZIP1_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op1); 9141 } 9142 return 0; 9143} 9144 9145unsigned fastEmit_AArch64ISD_ZIP1_MVT_nxv2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9146 if (RetVT.SimpleTy != MVT::nxv2i64) 9147 return 0; 9148 if ((Subtarget->hasSVEorSME())) { 9149 return fastEmitInst_rr(AArch64::ZIP1_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op1); 9150 } 9151 return 0; 9152} 9153 9154unsigned fastEmit_AArch64ISD_ZIP1_MVT_nxv2f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9155 if (RetVT.SimpleTy != MVT::nxv2f16) 9156 return 0; 9157 if ((Subtarget->hasSVEorSME())) { 9158 return fastEmitInst_rr(AArch64::ZIP1_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op1); 9159 } 9160 return 0; 9161} 9162 9163unsigned fastEmit_AArch64ISD_ZIP1_MVT_nxv4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9164 if (RetVT.SimpleTy != MVT::nxv4f16) 9165 return 0; 9166 if ((Subtarget->hasSVEorSME())) { 9167 return fastEmitInst_rr(AArch64::ZIP1_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op1); 9168 } 9169 return 0; 9170} 9171 9172unsigned fastEmit_AArch64ISD_ZIP1_MVT_nxv8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9173 if (RetVT.SimpleTy != MVT::nxv8f16) 9174 return 0; 9175 if ((Subtarget->hasSVEorSME())) { 9176 return fastEmitInst_rr(AArch64::ZIP1_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op1); 9177 } 9178 return 0; 9179} 9180 9181unsigned fastEmit_AArch64ISD_ZIP1_MVT_nxv8bf16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9182 if (RetVT.SimpleTy != MVT::nxv8bf16) 9183 return 0; 9184 if ((Subtarget->hasSVEorSME())) { 9185 return fastEmitInst_rr(AArch64::ZIP1_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op1); 9186 } 9187 return 0; 9188} 9189 9190unsigned fastEmit_AArch64ISD_ZIP1_MVT_nxv2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9191 if (RetVT.SimpleTy != MVT::nxv2f32) 9192 return 0; 9193 if ((Subtarget->hasSVEorSME())) { 9194 return fastEmitInst_rr(AArch64::ZIP1_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op1); 9195 } 9196 return 0; 9197} 9198 9199unsigned fastEmit_AArch64ISD_ZIP1_MVT_nxv4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9200 if (RetVT.SimpleTy != MVT::nxv4f32) 9201 return 0; 9202 if ((Subtarget->hasSVEorSME())) { 9203 return fastEmitInst_rr(AArch64::ZIP1_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op1); 9204 } 9205 return 0; 9206} 9207 9208unsigned fastEmit_AArch64ISD_ZIP1_MVT_nxv2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9209 if (RetVT.SimpleTy != MVT::nxv2f64) 9210 return 0; 9211 if ((Subtarget->hasSVEorSME())) { 9212 return fastEmitInst_rr(AArch64::ZIP1_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op1); 9213 } 9214 return 0; 9215} 9216 9217unsigned fastEmit_AArch64ISD_ZIP1_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 9218 switch (VT.SimpleTy) { 9219 case MVT::v8i8: return fastEmit_AArch64ISD_ZIP1_MVT_v8i8_rr(RetVT, Op0, Op1); 9220 case MVT::v16i8: return fastEmit_AArch64ISD_ZIP1_MVT_v16i8_rr(RetVT, Op0, Op1); 9221 case MVT::v4i16: return fastEmit_AArch64ISD_ZIP1_MVT_v4i16_rr(RetVT, Op0, Op1); 9222 case MVT::v8i16: return fastEmit_AArch64ISD_ZIP1_MVT_v8i16_rr(RetVT, Op0, Op1); 9223 case MVT::v2i32: return fastEmit_AArch64ISD_ZIP1_MVT_v2i32_rr(RetVT, Op0, Op1); 9224 case MVT::v4i32: return fastEmit_AArch64ISD_ZIP1_MVT_v4i32_rr(RetVT, Op0, Op1); 9225 case MVT::v2i64: return fastEmit_AArch64ISD_ZIP1_MVT_v2i64_rr(RetVT, Op0, Op1); 9226 case MVT::v4f16: return fastEmit_AArch64ISD_ZIP1_MVT_v4f16_rr(RetVT, Op0, Op1); 9227 case MVT::v8f16: return fastEmit_AArch64ISD_ZIP1_MVT_v8f16_rr(RetVT, Op0, Op1); 9228 case MVT::v4bf16: return fastEmit_AArch64ISD_ZIP1_MVT_v4bf16_rr(RetVT, Op0, Op1); 9229 case MVT::v8bf16: return fastEmit_AArch64ISD_ZIP1_MVT_v8bf16_rr(RetVT, Op0, Op1); 9230 case MVT::v2f32: return fastEmit_AArch64ISD_ZIP1_MVT_v2f32_rr(RetVT, Op0, Op1); 9231 case MVT::v4f32: return fastEmit_AArch64ISD_ZIP1_MVT_v4f32_rr(RetVT, Op0, Op1); 9232 case MVT::v2f64: return fastEmit_AArch64ISD_ZIP1_MVT_v2f64_rr(RetVT, Op0, Op1); 9233 case MVT::nxv2i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2i1_rr(RetVT, Op0, Op1); 9234 case MVT::nxv4i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4i1_rr(RetVT, Op0, Op1); 9235 case MVT::nxv8i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8i1_rr(RetVT, Op0, Op1); 9236 case MVT::nxv16i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv16i1_rr(RetVT, Op0, Op1); 9237 case MVT::nxv16i8: return fastEmit_AArch64ISD_ZIP1_MVT_nxv16i8_rr(RetVT, Op0, Op1); 9238 case MVT::nxv8i16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8i16_rr(RetVT, Op0, Op1); 9239 case MVT::nxv4i32: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4i32_rr(RetVT, Op0, Op1); 9240 case MVT::nxv2i64: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2i64_rr(RetVT, Op0, Op1); 9241 case MVT::nxv2f16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2f16_rr(RetVT, Op0, Op1); 9242 case MVT::nxv4f16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4f16_rr(RetVT, Op0, Op1); 9243 case MVT::nxv8f16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8f16_rr(RetVT, Op0, Op1); 9244 case MVT::nxv8bf16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8bf16_rr(RetVT, Op0, Op1); 9245 case MVT::nxv2f32: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2f32_rr(RetVT, Op0, Op1); 9246 case MVT::nxv4f32: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4f32_rr(RetVT, Op0, Op1); 9247 case MVT::nxv2f64: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2f64_rr(RetVT, Op0, Op1); 9248 default: return 0; 9249 } 9250} 9251 9252// FastEmit functions for AArch64ISD::ZIP2. 9253 9254unsigned fastEmit_AArch64ISD_ZIP2_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9255 if (RetVT.SimpleTy != MVT::v8i8) 9256 return 0; 9257 if ((Subtarget->hasNEON())) { 9258 return fastEmitInst_rr(AArch64::ZIP2v8i8, &AArch64::FPR64RegClass, Op0, Op1); 9259 } 9260 return 0; 9261} 9262 9263unsigned fastEmit_AArch64ISD_ZIP2_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9264 if (RetVT.SimpleTy != MVT::v16i8) 9265 return 0; 9266 if ((Subtarget->hasNEON())) { 9267 return fastEmitInst_rr(AArch64::ZIP2v16i8, &AArch64::FPR128RegClass, Op0, Op1); 9268 } 9269 return 0; 9270} 9271 9272unsigned fastEmit_AArch64ISD_ZIP2_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9273 if (RetVT.SimpleTy != MVT::v4i16) 9274 return 0; 9275 if ((Subtarget->hasNEON())) { 9276 return fastEmitInst_rr(AArch64::ZIP2v4i16, &AArch64::FPR64RegClass, Op0, Op1); 9277 } 9278 return 0; 9279} 9280 9281unsigned fastEmit_AArch64ISD_ZIP2_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9282 if (RetVT.SimpleTy != MVT::v8i16) 9283 return 0; 9284 if ((Subtarget->hasNEON())) { 9285 return fastEmitInst_rr(AArch64::ZIP2v8i16, &AArch64::FPR128RegClass, Op0, Op1); 9286 } 9287 return 0; 9288} 9289 9290unsigned fastEmit_AArch64ISD_ZIP2_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9291 if (RetVT.SimpleTy != MVT::v2i32) 9292 return 0; 9293 if ((Subtarget->hasNEON())) { 9294 return fastEmitInst_rr(AArch64::ZIP2v2i32, &AArch64::FPR64RegClass, Op0, Op1); 9295 } 9296 return 0; 9297} 9298 9299unsigned fastEmit_AArch64ISD_ZIP2_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9300 if (RetVT.SimpleTy != MVT::v4i32) 9301 return 0; 9302 if ((Subtarget->hasNEON())) { 9303 return fastEmitInst_rr(AArch64::ZIP2v4i32, &AArch64::FPR128RegClass, Op0, Op1); 9304 } 9305 return 0; 9306} 9307 9308unsigned fastEmit_AArch64ISD_ZIP2_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9309 if (RetVT.SimpleTy != MVT::v2i64) 9310 return 0; 9311 if ((Subtarget->hasNEON())) { 9312 return fastEmitInst_rr(AArch64::ZIP2v2i64, &AArch64::FPR128RegClass, Op0, Op1); 9313 } 9314 return 0; 9315} 9316 9317unsigned fastEmit_AArch64ISD_ZIP2_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9318 if (RetVT.SimpleTy != MVT::v4f16) 9319 return 0; 9320 if ((Subtarget->hasNEON())) { 9321 return fastEmitInst_rr(AArch64::ZIP2v4i16, &AArch64::FPR64RegClass, Op0, Op1); 9322 } 9323 return 0; 9324} 9325 9326unsigned fastEmit_AArch64ISD_ZIP2_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9327 if (RetVT.SimpleTy != MVT::v8f16) 9328 return 0; 9329 if ((Subtarget->hasNEON())) { 9330 return fastEmitInst_rr(AArch64::ZIP2v8i16, &AArch64::FPR128RegClass, Op0, Op1); 9331 } 9332 return 0; 9333} 9334 9335unsigned fastEmit_AArch64ISD_ZIP2_MVT_v4bf16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9336 if (RetVT.SimpleTy != MVT::v4bf16) 9337 return 0; 9338 if ((Subtarget->hasNEON())) { 9339 return fastEmitInst_rr(AArch64::ZIP2v4i16, &AArch64::FPR64RegClass, Op0, Op1); 9340 } 9341 return 0; 9342} 9343 9344unsigned fastEmit_AArch64ISD_ZIP2_MVT_v8bf16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9345 if (RetVT.SimpleTy != MVT::v8bf16) 9346 return 0; 9347 if ((Subtarget->hasNEON())) { 9348 return fastEmitInst_rr(AArch64::ZIP2v8i16, &AArch64::FPR128RegClass, Op0, Op1); 9349 } 9350 return 0; 9351} 9352 9353unsigned fastEmit_AArch64ISD_ZIP2_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9354 if (RetVT.SimpleTy != MVT::v2f32) 9355 return 0; 9356 if ((Subtarget->hasNEON())) { 9357 return fastEmitInst_rr(AArch64::ZIP2v2i32, &AArch64::FPR64RegClass, Op0, Op1); 9358 } 9359 return 0; 9360} 9361 9362unsigned fastEmit_AArch64ISD_ZIP2_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9363 if (RetVT.SimpleTy != MVT::v4f32) 9364 return 0; 9365 if ((Subtarget->hasNEON())) { 9366 return fastEmitInst_rr(AArch64::ZIP2v4i32, &AArch64::FPR128RegClass, Op0, Op1); 9367 } 9368 return 0; 9369} 9370 9371unsigned fastEmit_AArch64ISD_ZIP2_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9372 if (RetVT.SimpleTy != MVT::v2f64) 9373 return 0; 9374 if ((Subtarget->hasNEON())) { 9375 return fastEmitInst_rr(AArch64::ZIP2v2i64, &AArch64::FPR128RegClass, Op0, Op1); 9376 } 9377 return 0; 9378} 9379 9380unsigned fastEmit_AArch64ISD_ZIP2_MVT_nxv2i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9381 if (RetVT.SimpleTy != MVT::nxv2i1) 9382 return 0; 9383 if ((Subtarget->hasSVEorSME())) { 9384 return fastEmitInst_rr(AArch64::ZIP2_PPP_D, &AArch64::PPRRegClass, Op0, Op1); 9385 } 9386 return 0; 9387} 9388 9389unsigned fastEmit_AArch64ISD_ZIP2_MVT_nxv4i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9390 if (RetVT.SimpleTy != MVT::nxv4i1) 9391 return 0; 9392 if ((Subtarget->hasSVEorSME())) { 9393 return fastEmitInst_rr(AArch64::ZIP2_PPP_S, &AArch64::PPRRegClass, Op0, Op1); 9394 } 9395 return 0; 9396} 9397 9398unsigned fastEmit_AArch64ISD_ZIP2_MVT_nxv8i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9399 if (RetVT.SimpleTy != MVT::nxv8i1) 9400 return 0; 9401 if ((Subtarget->hasSVEorSME())) { 9402 return fastEmitInst_rr(AArch64::ZIP2_PPP_H, &AArch64::PPRRegClass, Op0, Op1); 9403 } 9404 return 0; 9405} 9406 9407unsigned fastEmit_AArch64ISD_ZIP2_MVT_nxv16i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9408 if (RetVT.SimpleTy != MVT::nxv16i1) 9409 return 0; 9410 if ((Subtarget->hasSVEorSME())) { 9411 return fastEmitInst_rr(AArch64::ZIP2_PPP_B, &AArch64::PPRRegClass, Op0, Op1); 9412 } 9413 return 0; 9414} 9415 9416unsigned fastEmit_AArch64ISD_ZIP2_MVT_nxv16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9417 if (RetVT.SimpleTy != MVT::nxv16i8) 9418 return 0; 9419 if ((Subtarget->hasSVEorSME())) { 9420 return fastEmitInst_rr(AArch64::ZIP2_ZZZ_B, &AArch64::ZPRRegClass, Op0, Op1); 9421 } 9422 return 0; 9423} 9424 9425unsigned fastEmit_AArch64ISD_ZIP2_MVT_nxv8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9426 if (RetVT.SimpleTy != MVT::nxv8i16) 9427 return 0; 9428 if ((Subtarget->hasSVEorSME())) { 9429 return fastEmitInst_rr(AArch64::ZIP2_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op1); 9430 } 9431 return 0; 9432} 9433 9434unsigned fastEmit_AArch64ISD_ZIP2_MVT_nxv4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9435 if (RetVT.SimpleTy != MVT::nxv4i32) 9436 return 0; 9437 if ((Subtarget->hasSVEorSME())) { 9438 return fastEmitInst_rr(AArch64::ZIP2_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op1); 9439 } 9440 return 0; 9441} 9442 9443unsigned fastEmit_AArch64ISD_ZIP2_MVT_nxv2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9444 if (RetVT.SimpleTy != MVT::nxv2i64) 9445 return 0; 9446 if ((Subtarget->hasSVEorSME())) { 9447 return fastEmitInst_rr(AArch64::ZIP2_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op1); 9448 } 9449 return 0; 9450} 9451 9452unsigned fastEmit_AArch64ISD_ZIP2_MVT_nxv2f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9453 if (RetVT.SimpleTy != MVT::nxv2f16) 9454 return 0; 9455 if ((Subtarget->hasSVEorSME())) { 9456 return fastEmitInst_rr(AArch64::ZIP2_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op1); 9457 } 9458 return 0; 9459} 9460 9461unsigned fastEmit_AArch64ISD_ZIP2_MVT_nxv4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9462 if (RetVT.SimpleTy != MVT::nxv4f16) 9463 return 0; 9464 if ((Subtarget->hasSVEorSME())) { 9465 return fastEmitInst_rr(AArch64::ZIP2_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op1); 9466 } 9467 return 0; 9468} 9469 9470unsigned fastEmit_AArch64ISD_ZIP2_MVT_nxv8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9471 if (RetVT.SimpleTy != MVT::nxv8f16) 9472 return 0; 9473 if ((Subtarget->hasSVEorSME())) { 9474 return fastEmitInst_rr(AArch64::ZIP2_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op1); 9475 } 9476 return 0; 9477} 9478 9479unsigned fastEmit_AArch64ISD_ZIP2_MVT_nxv8bf16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9480 if (RetVT.SimpleTy != MVT::nxv8bf16) 9481 return 0; 9482 if ((Subtarget->hasSVEorSME())) { 9483 return fastEmitInst_rr(AArch64::ZIP2_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op1); 9484 } 9485 return 0; 9486} 9487 9488unsigned fastEmit_AArch64ISD_ZIP2_MVT_nxv2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9489 if (RetVT.SimpleTy != MVT::nxv2f32) 9490 return 0; 9491 if ((Subtarget->hasSVEorSME())) { 9492 return fastEmitInst_rr(AArch64::ZIP2_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op1); 9493 } 9494 return 0; 9495} 9496 9497unsigned fastEmit_AArch64ISD_ZIP2_MVT_nxv4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9498 if (RetVT.SimpleTy != MVT::nxv4f32) 9499 return 0; 9500 if ((Subtarget->hasSVEorSME())) { 9501 return fastEmitInst_rr(AArch64::ZIP2_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op1); 9502 } 9503 return 0; 9504} 9505 9506unsigned fastEmit_AArch64ISD_ZIP2_MVT_nxv2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9507 if (RetVT.SimpleTy != MVT::nxv2f64) 9508 return 0; 9509 if ((Subtarget->hasSVEorSME())) { 9510 return fastEmitInst_rr(AArch64::ZIP2_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op1); 9511 } 9512 return 0; 9513} 9514 9515unsigned fastEmit_AArch64ISD_ZIP2_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 9516 switch (VT.SimpleTy) { 9517 case MVT::v8i8: return fastEmit_AArch64ISD_ZIP2_MVT_v8i8_rr(RetVT, Op0, Op1); 9518 case MVT::v16i8: return fastEmit_AArch64ISD_ZIP2_MVT_v16i8_rr(RetVT, Op0, Op1); 9519 case MVT::v4i16: return fastEmit_AArch64ISD_ZIP2_MVT_v4i16_rr(RetVT, Op0, Op1); 9520 case MVT::v8i16: return fastEmit_AArch64ISD_ZIP2_MVT_v8i16_rr(RetVT, Op0, Op1); 9521 case MVT::v2i32: return fastEmit_AArch64ISD_ZIP2_MVT_v2i32_rr(RetVT, Op0, Op1); 9522 case MVT::v4i32: return fastEmit_AArch64ISD_ZIP2_MVT_v4i32_rr(RetVT, Op0, Op1); 9523 case MVT::v2i64: return fastEmit_AArch64ISD_ZIP2_MVT_v2i64_rr(RetVT, Op0, Op1); 9524 case MVT::v4f16: return fastEmit_AArch64ISD_ZIP2_MVT_v4f16_rr(RetVT, Op0, Op1); 9525 case MVT::v8f16: return fastEmit_AArch64ISD_ZIP2_MVT_v8f16_rr(RetVT, Op0, Op1); 9526 case MVT::v4bf16: return fastEmit_AArch64ISD_ZIP2_MVT_v4bf16_rr(RetVT, Op0, Op1); 9527 case MVT::v8bf16: return fastEmit_AArch64ISD_ZIP2_MVT_v8bf16_rr(RetVT, Op0, Op1); 9528 case MVT::v2f32: return fastEmit_AArch64ISD_ZIP2_MVT_v2f32_rr(RetVT, Op0, Op1); 9529 case MVT::v4f32: return fastEmit_AArch64ISD_ZIP2_MVT_v4f32_rr(RetVT, Op0, Op1); 9530 case MVT::v2f64: return fastEmit_AArch64ISD_ZIP2_MVT_v2f64_rr(RetVT, Op0, Op1); 9531 case MVT::nxv2i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2i1_rr(RetVT, Op0, Op1); 9532 case MVT::nxv4i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4i1_rr(RetVT, Op0, Op1); 9533 case MVT::nxv8i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8i1_rr(RetVT, Op0, Op1); 9534 case MVT::nxv16i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv16i1_rr(RetVT, Op0, Op1); 9535 case MVT::nxv16i8: return fastEmit_AArch64ISD_ZIP2_MVT_nxv16i8_rr(RetVT, Op0, Op1); 9536 case MVT::nxv8i16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8i16_rr(RetVT, Op0, Op1); 9537 case MVT::nxv4i32: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4i32_rr(RetVT, Op0, Op1); 9538 case MVT::nxv2i64: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2i64_rr(RetVT, Op0, Op1); 9539 case MVT::nxv2f16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2f16_rr(RetVT, Op0, Op1); 9540 case MVT::nxv4f16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4f16_rr(RetVT, Op0, Op1); 9541 case MVT::nxv8f16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8f16_rr(RetVT, Op0, Op1); 9542 case MVT::nxv8bf16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8bf16_rr(RetVT, Op0, Op1); 9543 case MVT::nxv2f32: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2f32_rr(RetVT, Op0, Op1); 9544 case MVT::nxv4f32: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4f32_rr(RetVT, Op0, Op1); 9545 case MVT::nxv2f64: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2f64_rr(RetVT, Op0, Op1); 9546 default: return 0; 9547 } 9548} 9549 9550// FastEmit functions for ISD::ABDS. 9551 9552unsigned fastEmit_ISD_ABDS_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9553 if (RetVT.SimpleTy != MVT::v8i8) 9554 return 0; 9555 if ((Subtarget->hasNEON())) { 9556 return fastEmitInst_rr(AArch64::SABDv8i8, &AArch64::FPR64RegClass, Op0, Op1); 9557 } 9558 return 0; 9559} 9560 9561unsigned fastEmit_ISD_ABDS_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9562 if (RetVT.SimpleTy != MVT::v16i8) 9563 return 0; 9564 if ((Subtarget->hasNEON())) { 9565 return fastEmitInst_rr(AArch64::SABDv16i8, &AArch64::FPR128RegClass, Op0, Op1); 9566 } 9567 return 0; 9568} 9569 9570unsigned fastEmit_ISD_ABDS_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9571 if (RetVT.SimpleTy != MVT::v4i16) 9572 return 0; 9573 if ((Subtarget->hasNEON())) { 9574 return fastEmitInst_rr(AArch64::SABDv4i16, &AArch64::FPR64RegClass, Op0, Op1); 9575 } 9576 return 0; 9577} 9578 9579unsigned fastEmit_ISD_ABDS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9580 if (RetVT.SimpleTy != MVT::v8i16) 9581 return 0; 9582 if ((Subtarget->hasNEON())) { 9583 return fastEmitInst_rr(AArch64::SABDv8i16, &AArch64::FPR128RegClass, Op0, Op1); 9584 } 9585 return 0; 9586} 9587 9588unsigned fastEmit_ISD_ABDS_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9589 if (RetVT.SimpleTy != MVT::v2i32) 9590 return 0; 9591 if ((Subtarget->hasNEON())) { 9592 return fastEmitInst_rr(AArch64::SABDv2i32, &AArch64::FPR64RegClass, Op0, Op1); 9593 } 9594 return 0; 9595} 9596 9597unsigned fastEmit_ISD_ABDS_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9598 if (RetVT.SimpleTy != MVT::v4i32) 9599 return 0; 9600 if ((Subtarget->hasNEON())) { 9601 return fastEmitInst_rr(AArch64::SABDv4i32, &AArch64::FPR128RegClass, Op0, Op1); 9602 } 9603 return 0; 9604} 9605 9606unsigned fastEmit_ISD_ABDS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 9607 switch (VT.SimpleTy) { 9608 case MVT::v8i8: return fastEmit_ISD_ABDS_MVT_v8i8_rr(RetVT, Op0, Op1); 9609 case MVT::v16i8: return fastEmit_ISD_ABDS_MVT_v16i8_rr(RetVT, Op0, Op1); 9610 case MVT::v4i16: return fastEmit_ISD_ABDS_MVT_v4i16_rr(RetVT, Op0, Op1); 9611 case MVT::v8i16: return fastEmit_ISD_ABDS_MVT_v8i16_rr(RetVT, Op0, Op1); 9612 case MVT::v2i32: return fastEmit_ISD_ABDS_MVT_v2i32_rr(RetVT, Op0, Op1); 9613 case MVT::v4i32: return fastEmit_ISD_ABDS_MVT_v4i32_rr(RetVT, Op0, Op1); 9614 default: return 0; 9615 } 9616} 9617 9618// FastEmit functions for ISD::ABDU. 9619 9620unsigned fastEmit_ISD_ABDU_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9621 if (RetVT.SimpleTy != MVT::v8i8) 9622 return 0; 9623 if ((Subtarget->hasNEON())) { 9624 return fastEmitInst_rr(AArch64::UABDv8i8, &AArch64::FPR64RegClass, Op0, Op1); 9625 } 9626 return 0; 9627} 9628 9629unsigned fastEmit_ISD_ABDU_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9630 if (RetVT.SimpleTy != MVT::v16i8) 9631 return 0; 9632 if ((Subtarget->hasNEON())) { 9633 return fastEmitInst_rr(AArch64::UABDv16i8, &AArch64::FPR128RegClass, Op0, Op1); 9634 } 9635 return 0; 9636} 9637 9638unsigned fastEmit_ISD_ABDU_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9639 if (RetVT.SimpleTy != MVT::v4i16) 9640 return 0; 9641 if ((Subtarget->hasNEON())) { 9642 return fastEmitInst_rr(AArch64::UABDv4i16, &AArch64::FPR64RegClass, Op0, Op1); 9643 } 9644 return 0; 9645} 9646 9647unsigned fastEmit_ISD_ABDU_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9648 if (RetVT.SimpleTy != MVT::v8i16) 9649 return 0; 9650 if ((Subtarget->hasNEON())) { 9651 return fastEmitInst_rr(AArch64::UABDv8i16, &AArch64::FPR128RegClass, Op0, Op1); 9652 } 9653 return 0; 9654} 9655 9656unsigned fastEmit_ISD_ABDU_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9657 if (RetVT.SimpleTy != MVT::v2i32) 9658 return 0; 9659 if ((Subtarget->hasNEON())) { 9660 return fastEmitInst_rr(AArch64::UABDv2i32, &AArch64::FPR64RegClass, Op0, Op1); 9661 } 9662 return 0; 9663} 9664 9665unsigned fastEmit_ISD_ABDU_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9666 if (RetVT.SimpleTy != MVT::v4i32) 9667 return 0; 9668 if ((Subtarget->hasNEON())) { 9669 return fastEmitInst_rr(AArch64::UABDv4i32, &AArch64::FPR128RegClass, Op0, Op1); 9670 } 9671 return 0; 9672} 9673 9674unsigned fastEmit_ISD_ABDU_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 9675 switch (VT.SimpleTy) { 9676 case MVT::v8i8: return fastEmit_ISD_ABDU_MVT_v8i8_rr(RetVT, Op0, Op1); 9677 case MVT::v16i8: return fastEmit_ISD_ABDU_MVT_v16i8_rr(RetVT, Op0, Op1); 9678 case MVT::v4i16: return fastEmit_ISD_ABDU_MVT_v4i16_rr(RetVT, Op0, Op1); 9679 case MVT::v8i16: return fastEmit_ISD_ABDU_MVT_v8i16_rr(RetVT, Op0, Op1); 9680 case MVT::v2i32: return fastEmit_ISD_ABDU_MVT_v2i32_rr(RetVT, Op0, Op1); 9681 case MVT::v4i32: return fastEmit_ISD_ABDU_MVT_v4i32_rr(RetVT, Op0, Op1); 9682 default: return 0; 9683 } 9684} 9685 9686// FastEmit functions for ISD::ADD. 9687 9688unsigned fastEmit_ISD_ADD_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9689 if (RetVT.SimpleTy != MVT::i32) 9690 return 0; 9691 return fastEmitInst_rr(AArch64::ADDWrr, &AArch64::GPR32RegClass, Op0, Op1); 9692} 9693 9694unsigned fastEmit_ISD_ADD_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9695 if (RetVT.SimpleTy != MVT::i64) 9696 return 0; 9697 return fastEmitInst_rr(AArch64::ADDXrr, &AArch64::GPR64RegClass, Op0, Op1); 9698} 9699 9700unsigned fastEmit_ISD_ADD_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9701 if (RetVT.SimpleTy != MVT::v8i8) 9702 return 0; 9703 if ((Subtarget->hasNEON())) { 9704 return fastEmitInst_rr(AArch64::ADDv8i8, &AArch64::FPR64RegClass, Op0, Op1); 9705 } 9706 return 0; 9707} 9708 9709unsigned fastEmit_ISD_ADD_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9710 if (RetVT.SimpleTy != MVT::v16i8) 9711 return 0; 9712 if ((Subtarget->hasNEON())) { 9713 return fastEmitInst_rr(AArch64::ADDv16i8, &AArch64::FPR128RegClass, Op0, Op1); 9714 } 9715 return 0; 9716} 9717 9718unsigned fastEmit_ISD_ADD_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9719 if (RetVT.SimpleTy != MVT::v4i16) 9720 return 0; 9721 if ((Subtarget->hasNEON())) { 9722 return fastEmitInst_rr(AArch64::ADDv4i16, &AArch64::FPR64RegClass, Op0, Op1); 9723 } 9724 return 0; 9725} 9726 9727unsigned fastEmit_ISD_ADD_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9728 if (RetVT.SimpleTy != MVT::v8i16) 9729 return 0; 9730 if ((Subtarget->hasNEON())) { 9731 return fastEmitInst_rr(AArch64::ADDv8i16, &AArch64::FPR128RegClass, Op0, Op1); 9732 } 9733 return 0; 9734} 9735 9736unsigned fastEmit_ISD_ADD_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9737 if (RetVT.SimpleTy != MVT::v2i32) 9738 return 0; 9739 if ((Subtarget->hasNEON())) { 9740 return fastEmitInst_rr(AArch64::ADDv2i32, &AArch64::FPR64RegClass, Op0, Op1); 9741 } 9742 return 0; 9743} 9744 9745unsigned fastEmit_ISD_ADD_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9746 if (RetVT.SimpleTy != MVT::v4i32) 9747 return 0; 9748 if ((Subtarget->hasNEON())) { 9749 return fastEmitInst_rr(AArch64::ADDv4i32, &AArch64::FPR128RegClass, Op0, Op1); 9750 } 9751 return 0; 9752} 9753 9754unsigned fastEmit_ISD_ADD_MVT_v1i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9755 if (RetVT.SimpleTy != MVT::v1i64) 9756 return 0; 9757 if ((Subtarget->hasNEON())) { 9758 return fastEmitInst_rr(AArch64::ADDv1i64, &AArch64::FPR64RegClass, Op0, Op1); 9759 } 9760 return 0; 9761} 9762 9763unsigned fastEmit_ISD_ADD_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9764 if (RetVT.SimpleTy != MVT::v2i64) 9765 return 0; 9766 if ((Subtarget->hasNEON())) { 9767 return fastEmitInst_rr(AArch64::ADDv2i64, &AArch64::FPR128RegClass, Op0, Op1); 9768 } 9769 return 0; 9770} 9771 9772unsigned fastEmit_ISD_ADD_MVT_nxv16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9773 if (RetVT.SimpleTy != MVT::nxv16i8) 9774 return 0; 9775 if ((Subtarget->hasSVEorSME())) { 9776 return fastEmitInst_rr(AArch64::ADD_ZZZ_B, &AArch64::ZPRRegClass, Op0, Op1); 9777 } 9778 return 0; 9779} 9780 9781unsigned fastEmit_ISD_ADD_MVT_nxv8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9782 if (RetVT.SimpleTy != MVT::nxv8i16) 9783 return 0; 9784 if ((Subtarget->hasSVEorSME())) { 9785 return fastEmitInst_rr(AArch64::ADD_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op1); 9786 } 9787 return 0; 9788} 9789 9790unsigned fastEmit_ISD_ADD_MVT_nxv4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9791 if (RetVT.SimpleTy != MVT::nxv4i32) 9792 return 0; 9793 if ((Subtarget->hasSVEorSME())) { 9794 return fastEmitInst_rr(AArch64::ADD_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op1); 9795 } 9796 return 0; 9797} 9798 9799unsigned fastEmit_ISD_ADD_MVT_nxv2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9800 if (RetVT.SimpleTy != MVT::nxv2i64) 9801 return 0; 9802 if ((Subtarget->hasSVEorSME())) { 9803 return fastEmitInst_rr(AArch64::ADD_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op1); 9804 } 9805 return 0; 9806} 9807 9808unsigned fastEmit_ISD_ADD_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 9809 switch (VT.SimpleTy) { 9810 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_rr(RetVT, Op0, Op1); 9811 case MVT::i64: return fastEmit_ISD_ADD_MVT_i64_rr(RetVT, Op0, Op1); 9812 case MVT::v8i8: return fastEmit_ISD_ADD_MVT_v8i8_rr(RetVT, Op0, Op1); 9813 case MVT::v16i8: return fastEmit_ISD_ADD_MVT_v16i8_rr(RetVT, Op0, Op1); 9814 case MVT::v4i16: return fastEmit_ISD_ADD_MVT_v4i16_rr(RetVT, Op0, Op1); 9815 case MVT::v8i16: return fastEmit_ISD_ADD_MVT_v8i16_rr(RetVT, Op0, Op1); 9816 case MVT::v2i32: return fastEmit_ISD_ADD_MVT_v2i32_rr(RetVT, Op0, Op1); 9817 case MVT::v4i32: return fastEmit_ISD_ADD_MVT_v4i32_rr(RetVT, Op0, Op1); 9818 case MVT::v1i64: return fastEmit_ISD_ADD_MVT_v1i64_rr(RetVT, Op0, Op1); 9819 case MVT::v2i64: return fastEmit_ISD_ADD_MVT_v2i64_rr(RetVT, Op0, Op1); 9820 case MVT::nxv16i8: return fastEmit_ISD_ADD_MVT_nxv16i8_rr(RetVT, Op0, Op1); 9821 case MVT::nxv8i16: return fastEmit_ISD_ADD_MVT_nxv8i16_rr(RetVT, Op0, Op1); 9822 case MVT::nxv4i32: return fastEmit_ISD_ADD_MVT_nxv4i32_rr(RetVT, Op0, Op1); 9823 case MVT::nxv2i64: return fastEmit_ISD_ADD_MVT_nxv2i64_rr(RetVT, Op0, Op1); 9824 default: return 0; 9825 } 9826} 9827 9828// FastEmit functions for ISD::AND. 9829 9830unsigned fastEmit_ISD_AND_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9831 if (RetVT.SimpleTy != MVT::i32) 9832 return 0; 9833 return fastEmitInst_rr(AArch64::ANDWrr, &AArch64::GPR32RegClass, Op0, Op1); 9834} 9835 9836unsigned fastEmit_ISD_AND_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9837 if (RetVT.SimpleTy != MVT::i64) 9838 return 0; 9839 return fastEmitInst_rr(AArch64::ANDXrr, &AArch64::GPR64RegClass, Op0, Op1); 9840} 9841 9842unsigned fastEmit_ISD_AND_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9843 if (RetVT.SimpleTy != MVT::v8i8) 9844 return 0; 9845 if ((Subtarget->hasNEON())) { 9846 return fastEmitInst_rr(AArch64::ANDv8i8, &AArch64::FPR64RegClass, Op0, Op1); 9847 } 9848 return 0; 9849} 9850 9851unsigned fastEmit_ISD_AND_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9852 if (RetVT.SimpleTy != MVT::v16i8) 9853 return 0; 9854 if ((Subtarget->hasNEON())) { 9855 return fastEmitInst_rr(AArch64::ANDv16i8, &AArch64::FPR128RegClass, Op0, Op1); 9856 } 9857 return 0; 9858} 9859 9860unsigned fastEmit_ISD_AND_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9861 if (RetVT.SimpleTy != MVT::v4i16) 9862 return 0; 9863 if ((Subtarget->hasNEON())) { 9864 return fastEmitInst_rr(AArch64::ANDv8i8, &AArch64::FPR64RegClass, Op0, Op1); 9865 } 9866 return 0; 9867} 9868 9869unsigned fastEmit_ISD_AND_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9870 if (RetVT.SimpleTy != MVT::v8i16) 9871 return 0; 9872 if ((Subtarget->hasNEON())) { 9873 return fastEmitInst_rr(AArch64::ANDv16i8, &AArch64::FPR128RegClass, Op0, Op1); 9874 } 9875 return 0; 9876} 9877 9878unsigned fastEmit_ISD_AND_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9879 if (RetVT.SimpleTy != MVT::v2i32) 9880 return 0; 9881 if ((Subtarget->hasNEON())) { 9882 return fastEmitInst_rr(AArch64::ANDv8i8, &AArch64::FPR64RegClass, Op0, Op1); 9883 } 9884 return 0; 9885} 9886 9887unsigned fastEmit_ISD_AND_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9888 if (RetVT.SimpleTy != MVT::v4i32) 9889 return 0; 9890 if ((Subtarget->hasNEON())) { 9891 return fastEmitInst_rr(AArch64::ANDv16i8, &AArch64::FPR128RegClass, Op0, Op1); 9892 } 9893 return 0; 9894} 9895 9896unsigned fastEmit_ISD_AND_MVT_v1i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9897 if (RetVT.SimpleTy != MVT::v1i64) 9898 return 0; 9899 if ((Subtarget->hasNEON())) { 9900 return fastEmitInst_rr(AArch64::ANDv8i8, &AArch64::FPR64RegClass, Op0, Op1); 9901 } 9902 return 0; 9903} 9904 9905unsigned fastEmit_ISD_AND_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9906 if (RetVT.SimpleTy != MVT::v2i64) 9907 return 0; 9908 if ((Subtarget->hasNEON())) { 9909 return fastEmitInst_rr(AArch64::ANDv16i8, &AArch64::FPR128RegClass, Op0, Op1); 9910 } 9911 return 0; 9912} 9913 9914unsigned fastEmit_ISD_AND_MVT_nxv16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9915 if (RetVT.SimpleTy != MVT::nxv16i8) 9916 return 0; 9917 if ((Subtarget->hasSVEorSME())) { 9918 return fastEmitInst_rr(AArch64::AND_ZZZ, &AArch64::ZPRRegClass, Op0, Op1); 9919 } 9920 return 0; 9921} 9922 9923unsigned fastEmit_ISD_AND_MVT_nxv8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9924 if (RetVT.SimpleTy != MVT::nxv8i16) 9925 return 0; 9926 if ((Subtarget->hasSVEorSME())) { 9927 return fastEmitInst_rr(AArch64::AND_ZZZ, &AArch64::ZPRRegClass, Op0, Op1); 9928 } 9929 return 0; 9930} 9931 9932unsigned fastEmit_ISD_AND_MVT_nxv4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9933 if (RetVT.SimpleTy != MVT::nxv4i32) 9934 return 0; 9935 if ((Subtarget->hasSVEorSME())) { 9936 return fastEmitInst_rr(AArch64::AND_ZZZ, &AArch64::ZPRRegClass, Op0, Op1); 9937 } 9938 return 0; 9939} 9940 9941unsigned fastEmit_ISD_AND_MVT_nxv2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9942 if (RetVT.SimpleTy != MVT::nxv2i64) 9943 return 0; 9944 if ((Subtarget->hasSVEorSME())) { 9945 return fastEmitInst_rr(AArch64::AND_ZZZ, &AArch64::ZPRRegClass, Op0, Op1); 9946 } 9947 return 0; 9948} 9949 9950unsigned fastEmit_ISD_AND_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 9951 switch (VT.SimpleTy) { 9952 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_rr(RetVT, Op0, Op1); 9953 case MVT::i64: return fastEmit_ISD_AND_MVT_i64_rr(RetVT, Op0, Op1); 9954 case MVT::v8i8: return fastEmit_ISD_AND_MVT_v8i8_rr(RetVT, Op0, Op1); 9955 case MVT::v16i8: return fastEmit_ISD_AND_MVT_v16i8_rr(RetVT, Op0, Op1); 9956 case MVT::v4i16: return fastEmit_ISD_AND_MVT_v4i16_rr(RetVT, Op0, Op1); 9957 case MVT::v8i16: return fastEmit_ISD_AND_MVT_v8i16_rr(RetVT, Op0, Op1); 9958 case MVT::v2i32: return fastEmit_ISD_AND_MVT_v2i32_rr(RetVT, Op0, Op1); 9959 case MVT::v4i32: return fastEmit_ISD_AND_MVT_v4i32_rr(RetVT, Op0, Op1); 9960 case MVT::v1i64: return fastEmit_ISD_AND_MVT_v1i64_rr(RetVT, Op0, Op1); 9961 case MVT::v2i64: return fastEmit_ISD_AND_MVT_v2i64_rr(RetVT, Op0, Op1); 9962 case MVT::nxv16i8: return fastEmit_ISD_AND_MVT_nxv16i8_rr(RetVT, Op0, Op1); 9963 case MVT::nxv8i16: return fastEmit_ISD_AND_MVT_nxv8i16_rr(RetVT, Op0, Op1); 9964 case MVT::nxv4i32: return fastEmit_ISD_AND_MVT_nxv4i32_rr(RetVT, Op0, Op1); 9965 case MVT::nxv2i64: return fastEmit_ISD_AND_MVT_nxv2i64_rr(RetVT, Op0, Op1); 9966 default: return 0; 9967 } 9968} 9969 9970// FastEmit functions for ISD::AVGCEILS. 9971 9972unsigned fastEmit_ISD_AVGCEILS_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9973 if (RetVT.SimpleTy != MVT::v8i8) 9974 return 0; 9975 if ((Subtarget->hasNEON())) { 9976 return fastEmitInst_rr(AArch64::SRHADDv8i8, &AArch64::FPR64RegClass, Op0, Op1); 9977 } 9978 return 0; 9979} 9980 9981unsigned fastEmit_ISD_AVGCEILS_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9982 if (RetVT.SimpleTy != MVT::v16i8) 9983 return 0; 9984 if ((Subtarget->hasNEON())) { 9985 return fastEmitInst_rr(AArch64::SRHADDv16i8, &AArch64::FPR128RegClass, Op0, Op1); 9986 } 9987 return 0; 9988} 9989 9990unsigned fastEmit_ISD_AVGCEILS_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 9991 if (RetVT.SimpleTy != MVT::v4i16) 9992 return 0; 9993 if ((Subtarget->hasNEON())) { 9994 return fastEmitInst_rr(AArch64::SRHADDv4i16, &AArch64::FPR64RegClass, Op0, Op1); 9995 } 9996 return 0; 9997} 9998 9999unsigned fastEmit_ISD_AVGCEILS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10000 if (RetVT.SimpleTy != MVT::v8i16) 10001 return 0; 10002 if ((Subtarget->hasNEON())) { 10003 return fastEmitInst_rr(AArch64::SRHADDv8i16, &AArch64::FPR128RegClass, Op0, Op1); 10004 } 10005 return 0; 10006} 10007 10008unsigned fastEmit_ISD_AVGCEILS_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10009 if (RetVT.SimpleTy != MVT::v2i32) 10010 return 0; 10011 if ((Subtarget->hasNEON())) { 10012 return fastEmitInst_rr(AArch64::SRHADDv2i32, &AArch64::FPR64RegClass, Op0, Op1); 10013 } 10014 return 0; 10015} 10016 10017unsigned fastEmit_ISD_AVGCEILS_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10018 if (RetVT.SimpleTy != MVT::v4i32) 10019 return 0; 10020 if ((Subtarget->hasNEON())) { 10021 return fastEmitInst_rr(AArch64::SRHADDv4i32, &AArch64::FPR128RegClass, Op0, Op1); 10022 } 10023 return 0; 10024} 10025 10026unsigned fastEmit_ISD_AVGCEILS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 10027 switch (VT.SimpleTy) { 10028 case MVT::v8i8: return fastEmit_ISD_AVGCEILS_MVT_v8i8_rr(RetVT, Op0, Op1); 10029 case MVT::v16i8: return fastEmit_ISD_AVGCEILS_MVT_v16i8_rr(RetVT, Op0, Op1); 10030 case MVT::v4i16: return fastEmit_ISD_AVGCEILS_MVT_v4i16_rr(RetVT, Op0, Op1); 10031 case MVT::v8i16: return fastEmit_ISD_AVGCEILS_MVT_v8i16_rr(RetVT, Op0, Op1); 10032 case MVT::v2i32: return fastEmit_ISD_AVGCEILS_MVT_v2i32_rr(RetVT, Op0, Op1); 10033 case MVT::v4i32: return fastEmit_ISD_AVGCEILS_MVT_v4i32_rr(RetVT, Op0, Op1); 10034 default: return 0; 10035 } 10036} 10037 10038// FastEmit functions for ISD::AVGCEILU. 10039 10040unsigned fastEmit_ISD_AVGCEILU_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10041 if (RetVT.SimpleTy != MVT::v8i8) 10042 return 0; 10043 if ((Subtarget->hasNEON())) { 10044 return fastEmitInst_rr(AArch64::URHADDv8i8, &AArch64::FPR64RegClass, Op0, Op1); 10045 } 10046 return 0; 10047} 10048 10049unsigned fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10050 if (RetVT.SimpleTy != MVT::v16i8) 10051 return 0; 10052 if ((Subtarget->hasNEON())) { 10053 return fastEmitInst_rr(AArch64::URHADDv16i8, &AArch64::FPR128RegClass, Op0, Op1); 10054 } 10055 return 0; 10056} 10057 10058unsigned fastEmit_ISD_AVGCEILU_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10059 if (RetVT.SimpleTy != MVT::v4i16) 10060 return 0; 10061 if ((Subtarget->hasNEON())) { 10062 return fastEmitInst_rr(AArch64::URHADDv4i16, &AArch64::FPR64RegClass, Op0, Op1); 10063 } 10064 return 0; 10065} 10066 10067unsigned fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10068 if (RetVT.SimpleTy != MVT::v8i16) 10069 return 0; 10070 if ((Subtarget->hasNEON())) { 10071 return fastEmitInst_rr(AArch64::URHADDv8i16, &AArch64::FPR128RegClass, Op0, Op1); 10072 } 10073 return 0; 10074} 10075 10076unsigned fastEmit_ISD_AVGCEILU_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10077 if (RetVT.SimpleTy != MVT::v2i32) 10078 return 0; 10079 if ((Subtarget->hasNEON())) { 10080 return fastEmitInst_rr(AArch64::URHADDv2i32, &AArch64::FPR64RegClass, Op0, Op1); 10081 } 10082 return 0; 10083} 10084 10085unsigned fastEmit_ISD_AVGCEILU_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10086 if (RetVT.SimpleTy != MVT::v4i32) 10087 return 0; 10088 if ((Subtarget->hasNEON())) { 10089 return fastEmitInst_rr(AArch64::URHADDv4i32, &AArch64::FPR128RegClass, Op0, Op1); 10090 } 10091 return 0; 10092} 10093 10094unsigned fastEmit_ISD_AVGCEILU_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 10095 switch (VT.SimpleTy) { 10096 case MVT::v8i8: return fastEmit_ISD_AVGCEILU_MVT_v8i8_rr(RetVT, Op0, Op1); 10097 case MVT::v16i8: return fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(RetVT, Op0, Op1); 10098 case MVT::v4i16: return fastEmit_ISD_AVGCEILU_MVT_v4i16_rr(RetVT, Op0, Op1); 10099 case MVT::v8i16: return fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(RetVT, Op0, Op1); 10100 case MVT::v2i32: return fastEmit_ISD_AVGCEILU_MVT_v2i32_rr(RetVT, Op0, Op1); 10101 case MVT::v4i32: return fastEmit_ISD_AVGCEILU_MVT_v4i32_rr(RetVT, Op0, Op1); 10102 default: return 0; 10103 } 10104} 10105 10106// FastEmit functions for ISD::AVGFLOORS. 10107 10108unsigned fastEmit_ISD_AVGFLOORS_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10109 if (RetVT.SimpleTy != MVT::v8i8) 10110 return 0; 10111 if ((Subtarget->hasNEON())) { 10112 return fastEmitInst_rr(AArch64::SHADDv8i8, &AArch64::FPR64RegClass, Op0, Op1); 10113 } 10114 return 0; 10115} 10116 10117unsigned fastEmit_ISD_AVGFLOORS_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10118 if (RetVT.SimpleTy != MVT::v16i8) 10119 return 0; 10120 if ((Subtarget->hasNEON())) { 10121 return fastEmitInst_rr(AArch64::SHADDv16i8, &AArch64::FPR128RegClass, Op0, Op1); 10122 } 10123 return 0; 10124} 10125 10126unsigned fastEmit_ISD_AVGFLOORS_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10127 if (RetVT.SimpleTy != MVT::v4i16) 10128 return 0; 10129 if ((Subtarget->hasNEON())) { 10130 return fastEmitInst_rr(AArch64::SHADDv4i16, &AArch64::FPR64RegClass, Op0, Op1); 10131 } 10132 return 0; 10133} 10134 10135unsigned fastEmit_ISD_AVGFLOORS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10136 if (RetVT.SimpleTy != MVT::v8i16) 10137 return 0; 10138 if ((Subtarget->hasNEON())) { 10139 return fastEmitInst_rr(AArch64::SHADDv8i16, &AArch64::FPR128RegClass, Op0, Op1); 10140 } 10141 return 0; 10142} 10143 10144unsigned fastEmit_ISD_AVGFLOORS_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10145 if (RetVT.SimpleTy != MVT::v2i32) 10146 return 0; 10147 if ((Subtarget->hasNEON())) { 10148 return fastEmitInst_rr(AArch64::SHADDv2i32, &AArch64::FPR64RegClass, Op0, Op1); 10149 } 10150 return 0; 10151} 10152 10153unsigned fastEmit_ISD_AVGFLOORS_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10154 if (RetVT.SimpleTy != MVT::v4i32) 10155 return 0; 10156 if ((Subtarget->hasNEON())) { 10157 return fastEmitInst_rr(AArch64::SHADDv4i32, &AArch64::FPR128RegClass, Op0, Op1); 10158 } 10159 return 0; 10160} 10161 10162unsigned fastEmit_ISD_AVGFLOORS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 10163 switch (VT.SimpleTy) { 10164 case MVT::v8i8: return fastEmit_ISD_AVGFLOORS_MVT_v8i8_rr(RetVT, Op0, Op1); 10165 case MVT::v16i8: return fastEmit_ISD_AVGFLOORS_MVT_v16i8_rr(RetVT, Op0, Op1); 10166 case MVT::v4i16: return fastEmit_ISD_AVGFLOORS_MVT_v4i16_rr(RetVT, Op0, Op1); 10167 case MVT::v8i16: return fastEmit_ISD_AVGFLOORS_MVT_v8i16_rr(RetVT, Op0, Op1); 10168 case MVT::v2i32: return fastEmit_ISD_AVGFLOORS_MVT_v2i32_rr(RetVT, Op0, Op1); 10169 case MVT::v4i32: return fastEmit_ISD_AVGFLOORS_MVT_v4i32_rr(RetVT, Op0, Op1); 10170 default: return 0; 10171 } 10172} 10173 10174// FastEmit functions for ISD::AVGFLOORU. 10175 10176unsigned fastEmit_ISD_AVGFLOORU_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10177 if (RetVT.SimpleTy != MVT::v8i8) 10178 return 0; 10179 if ((Subtarget->hasNEON())) { 10180 return fastEmitInst_rr(AArch64::UHADDv8i8, &AArch64::FPR64RegClass, Op0, Op1); 10181 } 10182 return 0; 10183} 10184 10185unsigned fastEmit_ISD_AVGFLOORU_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10186 if (RetVT.SimpleTy != MVT::v16i8) 10187 return 0; 10188 if ((Subtarget->hasNEON())) { 10189 return fastEmitInst_rr(AArch64::UHADDv16i8, &AArch64::FPR128RegClass, Op0, Op1); 10190 } 10191 return 0; 10192} 10193 10194unsigned fastEmit_ISD_AVGFLOORU_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10195 if (RetVT.SimpleTy != MVT::v4i16) 10196 return 0; 10197 if ((Subtarget->hasNEON())) { 10198 return fastEmitInst_rr(AArch64::UHADDv4i16, &AArch64::FPR64RegClass, Op0, Op1); 10199 } 10200 return 0; 10201} 10202 10203unsigned fastEmit_ISD_AVGFLOORU_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10204 if (RetVT.SimpleTy != MVT::v8i16) 10205 return 0; 10206 if ((Subtarget->hasNEON())) { 10207 return fastEmitInst_rr(AArch64::UHADDv8i16, &AArch64::FPR128RegClass, Op0, Op1); 10208 } 10209 return 0; 10210} 10211 10212unsigned fastEmit_ISD_AVGFLOORU_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10213 if (RetVT.SimpleTy != MVT::v2i32) 10214 return 0; 10215 if ((Subtarget->hasNEON())) { 10216 return fastEmitInst_rr(AArch64::UHADDv2i32, &AArch64::FPR64RegClass, Op0, Op1); 10217 } 10218 return 0; 10219} 10220 10221unsigned fastEmit_ISD_AVGFLOORU_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10222 if (RetVT.SimpleTy != MVT::v4i32) 10223 return 0; 10224 if ((Subtarget->hasNEON())) { 10225 return fastEmitInst_rr(AArch64::UHADDv4i32, &AArch64::FPR128RegClass, Op0, Op1); 10226 } 10227 return 0; 10228} 10229 10230unsigned fastEmit_ISD_AVGFLOORU_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 10231 switch (VT.SimpleTy) { 10232 case MVT::v8i8: return fastEmit_ISD_AVGFLOORU_MVT_v8i8_rr(RetVT, Op0, Op1); 10233 case MVT::v16i8: return fastEmit_ISD_AVGFLOORU_MVT_v16i8_rr(RetVT, Op0, Op1); 10234 case MVT::v4i16: return fastEmit_ISD_AVGFLOORU_MVT_v4i16_rr(RetVT, Op0, Op1); 10235 case MVT::v8i16: return fastEmit_ISD_AVGFLOORU_MVT_v8i16_rr(RetVT, Op0, Op1); 10236 case MVT::v2i32: return fastEmit_ISD_AVGFLOORU_MVT_v2i32_rr(RetVT, Op0, Op1); 10237 case MVT::v4i32: return fastEmit_ISD_AVGFLOORU_MVT_v4i32_rr(RetVT, Op0, Op1); 10238 default: return 0; 10239 } 10240} 10241 10242// FastEmit functions for ISD::CONCAT_VECTORS. 10243 10244unsigned fastEmit_ISD_CONCAT_VECTORS_MVT_nxv1i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10245 if (RetVT.SimpleTy != MVT::nxv2i1) 10246 return 0; 10247 if ((Subtarget->hasSVEorSME())) { 10248 return fastEmitInst_rr(AArch64::UZP1_PPP_D, &AArch64::PPRRegClass, Op0, Op1); 10249 } 10250 return 0; 10251} 10252 10253unsigned fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10254 if (RetVT.SimpleTy != MVT::nxv4i1) 10255 return 0; 10256 if ((Subtarget->hasSVEorSME())) { 10257 return fastEmitInst_rr(AArch64::UZP1_PPP_S, &AArch64::PPRRegClass, Op0, Op1); 10258 } 10259 return 0; 10260} 10261 10262unsigned fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10263 if (RetVT.SimpleTy != MVT::nxv8i1) 10264 return 0; 10265 if ((Subtarget->hasSVEorSME())) { 10266 return fastEmitInst_rr(AArch64::UZP1_PPP_H, &AArch64::PPRRegClass, Op0, Op1); 10267 } 10268 return 0; 10269} 10270 10271unsigned fastEmit_ISD_CONCAT_VECTORS_MVT_nxv8i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10272 if (RetVT.SimpleTy != MVT::nxv16i1) 10273 return 0; 10274 if ((Subtarget->hasSVEorSME())) { 10275 return fastEmitInst_rr(AArch64::UZP1_PPP_B, &AArch64::PPRRegClass, Op0, Op1); 10276 } 10277 return 0; 10278} 10279 10280unsigned fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10281 if (RetVT.SimpleTy != MVT::nxv4f16) 10282 return 0; 10283 if ((Subtarget->hasSVEorSME())) { 10284 return fastEmitInst_rr(AArch64::UZP1_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op1); 10285 } 10286 return 0; 10287} 10288 10289unsigned fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10290 if (RetVT.SimpleTy != MVT::nxv8f16) 10291 return 0; 10292 if ((Subtarget->hasSVEorSME())) { 10293 return fastEmitInst_rr(AArch64::UZP1_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op1); 10294 } 10295 return 0; 10296} 10297 10298unsigned fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2bf16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10299 if (RetVT.SimpleTy != MVT::nxv4bf16) 10300 return 0; 10301 if ((Subtarget->hasSVEorSME())) { 10302 return fastEmitInst_rr(AArch64::UZP1_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op1); 10303 } 10304 return 0; 10305} 10306 10307unsigned fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4bf16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10308 if (RetVT.SimpleTy != MVT::nxv8bf16) 10309 return 0; 10310 if ((Subtarget->hasSVEorSME())) { 10311 return fastEmitInst_rr(AArch64::UZP1_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op1); 10312 } 10313 return 0; 10314} 10315 10316unsigned fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10317 if (RetVT.SimpleTy != MVT::nxv4f32) 10318 return 0; 10319 if ((Subtarget->hasSVEorSME())) { 10320 return fastEmitInst_rr(AArch64::UZP1_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op1); 10321 } 10322 return 0; 10323} 10324 10325unsigned fastEmit_ISD_CONCAT_VECTORS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 10326 switch (VT.SimpleTy) { 10327 case MVT::nxv1i1: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv1i1_rr(RetVT, Op0, Op1); 10328 case MVT::nxv2i1: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2i1_rr(RetVT, Op0, Op1); 10329 case MVT::nxv4i1: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4i1_rr(RetVT, Op0, Op1); 10330 case MVT::nxv8i1: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv8i1_rr(RetVT, Op0, Op1); 10331 case MVT::nxv2f16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f16_rr(RetVT, Op0, Op1); 10332 case MVT::nxv4f16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4f16_rr(RetVT, Op0, Op1); 10333 case MVT::nxv2bf16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2bf16_rr(RetVT, Op0, Op1); 10334 case MVT::nxv4bf16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4bf16_rr(RetVT, Op0, Op1); 10335 case MVT::nxv2f32: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f32_rr(RetVT, Op0, Op1); 10336 default: return 0; 10337 } 10338} 10339 10340// FastEmit functions for ISD::FADD. 10341 10342unsigned fastEmit_ISD_FADD_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10343 if (RetVT.SimpleTy != MVT::f16) 10344 return 0; 10345 if ((Subtarget->hasFullFP16())) { 10346 return fastEmitInst_rr(AArch64::FADDHrr, &AArch64::FPR16RegClass, Op0, Op1); 10347 } 10348 return 0; 10349} 10350 10351unsigned fastEmit_ISD_FADD_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10352 if (RetVT.SimpleTy != MVT::f32) 10353 return 0; 10354 if ((Subtarget->hasFPARMv8())) { 10355 return fastEmitInst_rr(AArch64::FADDSrr, &AArch64::FPR32RegClass, Op0, Op1); 10356 } 10357 return 0; 10358} 10359 10360unsigned fastEmit_ISD_FADD_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10361 if (RetVT.SimpleTy != MVT::f64) 10362 return 0; 10363 if ((Subtarget->hasFPARMv8())) { 10364 return fastEmitInst_rr(AArch64::FADDDrr, &AArch64::FPR64RegClass, Op0, Op1); 10365 } 10366 return 0; 10367} 10368 10369unsigned fastEmit_ISD_FADD_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10370 if (RetVT.SimpleTy != MVT::v4f16) 10371 return 0; 10372 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 10373 return fastEmitInst_rr(AArch64::FADDv4f16, &AArch64::FPR64RegClass, Op0, Op1); 10374 } 10375 return 0; 10376} 10377 10378unsigned fastEmit_ISD_FADD_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10379 if (RetVT.SimpleTy != MVT::v8f16) 10380 return 0; 10381 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 10382 return fastEmitInst_rr(AArch64::FADDv8f16, &AArch64::FPR128RegClass, Op0, Op1); 10383 } 10384 return 0; 10385} 10386 10387unsigned fastEmit_ISD_FADD_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10388 if (RetVT.SimpleTy != MVT::v2f32) 10389 return 0; 10390 if ((Subtarget->hasNEON())) { 10391 return fastEmitInst_rr(AArch64::FADDv2f32, &AArch64::FPR64RegClass, Op0, Op1); 10392 } 10393 return 0; 10394} 10395 10396unsigned fastEmit_ISD_FADD_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10397 if (RetVT.SimpleTy != MVT::v4f32) 10398 return 0; 10399 if ((Subtarget->hasNEON())) { 10400 return fastEmitInst_rr(AArch64::FADDv4f32, &AArch64::FPR128RegClass, Op0, Op1); 10401 } 10402 return 0; 10403} 10404 10405unsigned fastEmit_ISD_FADD_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10406 if (RetVT.SimpleTy != MVT::v2f64) 10407 return 0; 10408 if ((Subtarget->hasNEON())) { 10409 return fastEmitInst_rr(AArch64::FADDv2f64, &AArch64::FPR128RegClass, Op0, Op1); 10410 } 10411 return 0; 10412} 10413 10414unsigned fastEmit_ISD_FADD_MVT_nxv8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10415 if (RetVT.SimpleTy != MVT::nxv8f16) 10416 return 0; 10417 if ((Subtarget->hasSVEorSME())) { 10418 return fastEmitInst_rr(AArch64::FADD_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op1); 10419 } 10420 return 0; 10421} 10422 10423unsigned fastEmit_ISD_FADD_MVT_nxv4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10424 if (RetVT.SimpleTy != MVT::nxv4f32) 10425 return 0; 10426 if ((Subtarget->hasSVEorSME())) { 10427 return fastEmitInst_rr(AArch64::FADD_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op1); 10428 } 10429 return 0; 10430} 10431 10432unsigned fastEmit_ISD_FADD_MVT_nxv2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10433 if (RetVT.SimpleTy != MVT::nxv2f64) 10434 return 0; 10435 if ((Subtarget->hasSVEorSME())) { 10436 return fastEmitInst_rr(AArch64::FADD_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op1); 10437 } 10438 return 0; 10439} 10440 10441unsigned fastEmit_ISD_FADD_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 10442 switch (VT.SimpleTy) { 10443 case MVT::f16: return fastEmit_ISD_FADD_MVT_f16_rr(RetVT, Op0, Op1); 10444 case MVT::f32: return fastEmit_ISD_FADD_MVT_f32_rr(RetVT, Op0, Op1); 10445 case MVT::f64: return fastEmit_ISD_FADD_MVT_f64_rr(RetVT, Op0, Op1); 10446 case MVT::v4f16: return fastEmit_ISD_FADD_MVT_v4f16_rr(RetVT, Op0, Op1); 10447 case MVT::v8f16: return fastEmit_ISD_FADD_MVT_v8f16_rr(RetVT, Op0, Op1); 10448 case MVT::v2f32: return fastEmit_ISD_FADD_MVT_v2f32_rr(RetVT, Op0, Op1); 10449 case MVT::v4f32: return fastEmit_ISD_FADD_MVT_v4f32_rr(RetVT, Op0, Op1); 10450 case MVT::v2f64: return fastEmit_ISD_FADD_MVT_v2f64_rr(RetVT, Op0, Op1); 10451 case MVT::nxv8f16: return fastEmit_ISD_FADD_MVT_nxv8f16_rr(RetVT, Op0, Op1); 10452 case MVT::nxv4f32: return fastEmit_ISD_FADD_MVT_nxv4f32_rr(RetVT, Op0, Op1); 10453 case MVT::nxv2f64: return fastEmit_ISD_FADD_MVT_nxv2f64_rr(RetVT, Op0, Op1); 10454 default: return 0; 10455 } 10456} 10457 10458// FastEmit functions for ISD::FDIV. 10459 10460unsigned fastEmit_ISD_FDIV_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10461 if (RetVT.SimpleTy != MVT::f16) 10462 return 0; 10463 if ((Subtarget->hasFullFP16())) { 10464 return fastEmitInst_rr(AArch64::FDIVHrr, &AArch64::FPR16RegClass, Op0, Op1); 10465 } 10466 return 0; 10467} 10468 10469unsigned fastEmit_ISD_FDIV_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10470 if (RetVT.SimpleTy != MVT::f32) 10471 return 0; 10472 if ((Subtarget->hasFPARMv8())) { 10473 return fastEmitInst_rr(AArch64::FDIVSrr, &AArch64::FPR32RegClass, Op0, Op1); 10474 } 10475 return 0; 10476} 10477 10478unsigned fastEmit_ISD_FDIV_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10479 if (RetVT.SimpleTy != MVT::f64) 10480 return 0; 10481 if ((Subtarget->hasFPARMv8())) { 10482 return fastEmitInst_rr(AArch64::FDIVDrr, &AArch64::FPR64RegClass, Op0, Op1); 10483 } 10484 return 0; 10485} 10486 10487unsigned fastEmit_ISD_FDIV_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10488 if (RetVT.SimpleTy != MVT::v4f16) 10489 return 0; 10490 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 10491 return fastEmitInst_rr(AArch64::FDIVv4f16, &AArch64::FPR64RegClass, Op0, Op1); 10492 } 10493 return 0; 10494} 10495 10496unsigned fastEmit_ISD_FDIV_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10497 if (RetVT.SimpleTy != MVT::v8f16) 10498 return 0; 10499 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 10500 return fastEmitInst_rr(AArch64::FDIVv8f16, &AArch64::FPR128RegClass, Op0, Op1); 10501 } 10502 return 0; 10503} 10504 10505unsigned fastEmit_ISD_FDIV_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10506 if (RetVT.SimpleTy != MVT::v2f32) 10507 return 0; 10508 if ((Subtarget->hasNEON())) { 10509 return fastEmitInst_rr(AArch64::FDIVv2f32, &AArch64::FPR64RegClass, Op0, Op1); 10510 } 10511 return 0; 10512} 10513 10514unsigned fastEmit_ISD_FDIV_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10515 if (RetVT.SimpleTy != MVT::v4f32) 10516 return 0; 10517 if ((Subtarget->hasNEON())) { 10518 return fastEmitInst_rr(AArch64::FDIVv4f32, &AArch64::FPR128RegClass, Op0, Op1); 10519 } 10520 return 0; 10521} 10522 10523unsigned fastEmit_ISD_FDIV_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10524 if (RetVT.SimpleTy != MVT::v2f64) 10525 return 0; 10526 if ((Subtarget->hasNEON())) { 10527 return fastEmitInst_rr(AArch64::FDIVv2f64, &AArch64::FPR128RegClass, Op0, Op1); 10528 } 10529 return 0; 10530} 10531 10532unsigned fastEmit_ISD_FDIV_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 10533 switch (VT.SimpleTy) { 10534 case MVT::f16: return fastEmit_ISD_FDIV_MVT_f16_rr(RetVT, Op0, Op1); 10535 case MVT::f32: return fastEmit_ISD_FDIV_MVT_f32_rr(RetVT, Op0, Op1); 10536 case MVT::f64: return fastEmit_ISD_FDIV_MVT_f64_rr(RetVT, Op0, Op1); 10537 case MVT::v4f16: return fastEmit_ISD_FDIV_MVT_v4f16_rr(RetVT, Op0, Op1); 10538 case MVT::v8f16: return fastEmit_ISD_FDIV_MVT_v8f16_rr(RetVT, Op0, Op1); 10539 case MVT::v2f32: return fastEmit_ISD_FDIV_MVT_v2f32_rr(RetVT, Op0, Op1); 10540 case MVT::v4f32: return fastEmit_ISD_FDIV_MVT_v4f32_rr(RetVT, Op0, Op1); 10541 case MVT::v2f64: return fastEmit_ISD_FDIV_MVT_v2f64_rr(RetVT, Op0, Op1); 10542 default: return 0; 10543 } 10544} 10545 10546// FastEmit functions for ISD::FMAXIMUM. 10547 10548unsigned fastEmit_ISD_FMAXIMUM_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10549 if (RetVT.SimpleTy != MVT::f16) 10550 return 0; 10551 if ((Subtarget->hasFullFP16())) { 10552 return fastEmitInst_rr(AArch64::FMAXHrr, &AArch64::FPR16RegClass, Op0, Op1); 10553 } 10554 return 0; 10555} 10556 10557unsigned fastEmit_ISD_FMAXIMUM_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10558 if (RetVT.SimpleTy != MVT::f32) 10559 return 0; 10560 if ((Subtarget->hasFPARMv8())) { 10561 return fastEmitInst_rr(AArch64::FMAXSrr, &AArch64::FPR32RegClass, Op0, Op1); 10562 } 10563 return 0; 10564} 10565 10566unsigned fastEmit_ISD_FMAXIMUM_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10567 if (RetVT.SimpleTy != MVT::f64) 10568 return 0; 10569 if ((Subtarget->hasFPARMv8())) { 10570 return fastEmitInst_rr(AArch64::FMAXDrr, &AArch64::FPR64RegClass, Op0, Op1); 10571 } 10572 return 0; 10573} 10574 10575unsigned fastEmit_ISD_FMAXIMUM_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10576 if (RetVT.SimpleTy != MVT::v4f16) 10577 return 0; 10578 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 10579 return fastEmitInst_rr(AArch64::FMAXv4f16, &AArch64::FPR64RegClass, Op0, Op1); 10580 } 10581 return 0; 10582} 10583 10584unsigned fastEmit_ISD_FMAXIMUM_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10585 if (RetVT.SimpleTy != MVT::v8f16) 10586 return 0; 10587 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 10588 return fastEmitInst_rr(AArch64::FMAXv8f16, &AArch64::FPR128RegClass, Op0, Op1); 10589 } 10590 return 0; 10591} 10592 10593unsigned fastEmit_ISD_FMAXIMUM_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10594 if (RetVT.SimpleTy != MVT::v2f32) 10595 return 0; 10596 if ((Subtarget->hasNEON())) { 10597 return fastEmitInst_rr(AArch64::FMAXv2f32, &AArch64::FPR64RegClass, Op0, Op1); 10598 } 10599 return 0; 10600} 10601 10602unsigned fastEmit_ISD_FMAXIMUM_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10603 if (RetVT.SimpleTy != MVT::v4f32) 10604 return 0; 10605 if ((Subtarget->hasNEON())) { 10606 return fastEmitInst_rr(AArch64::FMAXv4f32, &AArch64::FPR128RegClass, Op0, Op1); 10607 } 10608 return 0; 10609} 10610 10611unsigned fastEmit_ISD_FMAXIMUM_MVT_v1f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10612 if (RetVT.SimpleTy != MVT::v1f64) 10613 return 0; 10614 return fastEmitInst_rr(AArch64::FMAXDrr, &AArch64::FPR64RegClass, Op0, Op1); 10615} 10616 10617unsigned fastEmit_ISD_FMAXIMUM_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10618 if (RetVT.SimpleTy != MVT::v2f64) 10619 return 0; 10620 if ((Subtarget->hasNEON())) { 10621 return fastEmitInst_rr(AArch64::FMAXv2f64, &AArch64::FPR128RegClass, Op0, Op1); 10622 } 10623 return 0; 10624} 10625 10626unsigned fastEmit_ISD_FMAXIMUM_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 10627 switch (VT.SimpleTy) { 10628 case MVT::f16: return fastEmit_ISD_FMAXIMUM_MVT_f16_rr(RetVT, Op0, Op1); 10629 case MVT::f32: return fastEmit_ISD_FMAXIMUM_MVT_f32_rr(RetVT, Op0, Op1); 10630 case MVT::f64: return fastEmit_ISD_FMAXIMUM_MVT_f64_rr(RetVT, Op0, Op1); 10631 case MVT::v4f16: return fastEmit_ISD_FMAXIMUM_MVT_v4f16_rr(RetVT, Op0, Op1); 10632 case MVT::v8f16: return fastEmit_ISD_FMAXIMUM_MVT_v8f16_rr(RetVT, Op0, Op1); 10633 case MVT::v2f32: return fastEmit_ISD_FMAXIMUM_MVT_v2f32_rr(RetVT, Op0, Op1); 10634 case MVT::v4f32: return fastEmit_ISD_FMAXIMUM_MVT_v4f32_rr(RetVT, Op0, Op1); 10635 case MVT::v1f64: return fastEmit_ISD_FMAXIMUM_MVT_v1f64_rr(RetVT, Op0, Op1); 10636 case MVT::v2f64: return fastEmit_ISD_FMAXIMUM_MVT_v2f64_rr(RetVT, Op0, Op1); 10637 default: return 0; 10638 } 10639} 10640 10641// FastEmit functions for ISD::FMAXNUM. 10642 10643unsigned fastEmit_ISD_FMAXNUM_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10644 if (RetVT.SimpleTy != MVT::f16) 10645 return 0; 10646 if ((Subtarget->hasFullFP16())) { 10647 return fastEmitInst_rr(AArch64::FMAXNMHrr, &AArch64::FPR16RegClass, Op0, Op1); 10648 } 10649 return 0; 10650} 10651 10652unsigned fastEmit_ISD_FMAXNUM_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10653 if (RetVT.SimpleTy != MVT::f32) 10654 return 0; 10655 if ((Subtarget->hasFPARMv8())) { 10656 return fastEmitInst_rr(AArch64::FMAXNMSrr, &AArch64::FPR32RegClass, Op0, Op1); 10657 } 10658 return 0; 10659} 10660 10661unsigned fastEmit_ISD_FMAXNUM_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10662 if (RetVT.SimpleTy != MVT::f64) 10663 return 0; 10664 if ((Subtarget->hasFPARMv8())) { 10665 return fastEmitInst_rr(AArch64::FMAXNMDrr, &AArch64::FPR64RegClass, Op0, Op1); 10666 } 10667 return 0; 10668} 10669 10670unsigned fastEmit_ISD_FMAXNUM_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10671 if (RetVT.SimpleTy != MVT::v4f16) 10672 return 0; 10673 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 10674 return fastEmitInst_rr(AArch64::FMAXNMv4f16, &AArch64::FPR64RegClass, Op0, Op1); 10675 } 10676 return 0; 10677} 10678 10679unsigned fastEmit_ISD_FMAXNUM_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10680 if (RetVT.SimpleTy != MVT::v8f16) 10681 return 0; 10682 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 10683 return fastEmitInst_rr(AArch64::FMAXNMv8f16, &AArch64::FPR128RegClass, Op0, Op1); 10684 } 10685 return 0; 10686} 10687 10688unsigned fastEmit_ISD_FMAXNUM_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10689 if (RetVT.SimpleTy != MVT::v2f32) 10690 return 0; 10691 if ((Subtarget->hasNEON())) { 10692 return fastEmitInst_rr(AArch64::FMAXNMv2f32, &AArch64::FPR64RegClass, Op0, Op1); 10693 } 10694 return 0; 10695} 10696 10697unsigned fastEmit_ISD_FMAXNUM_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10698 if (RetVT.SimpleTy != MVT::v4f32) 10699 return 0; 10700 if ((Subtarget->hasNEON())) { 10701 return fastEmitInst_rr(AArch64::FMAXNMv4f32, &AArch64::FPR128RegClass, Op0, Op1); 10702 } 10703 return 0; 10704} 10705 10706unsigned fastEmit_ISD_FMAXNUM_MVT_v1f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10707 if (RetVT.SimpleTy != MVT::v1f64) 10708 return 0; 10709 return fastEmitInst_rr(AArch64::FMAXNMDrr, &AArch64::FPR64RegClass, Op0, Op1); 10710} 10711 10712unsigned fastEmit_ISD_FMAXNUM_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10713 if (RetVT.SimpleTy != MVT::v2f64) 10714 return 0; 10715 if ((Subtarget->hasNEON())) { 10716 return fastEmitInst_rr(AArch64::FMAXNMv2f64, &AArch64::FPR128RegClass, Op0, Op1); 10717 } 10718 return 0; 10719} 10720 10721unsigned fastEmit_ISD_FMAXNUM_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 10722 switch (VT.SimpleTy) { 10723 case MVT::f16: return fastEmit_ISD_FMAXNUM_MVT_f16_rr(RetVT, Op0, Op1); 10724 case MVT::f32: return fastEmit_ISD_FMAXNUM_MVT_f32_rr(RetVT, Op0, Op1); 10725 case MVT::f64: return fastEmit_ISD_FMAXNUM_MVT_f64_rr(RetVT, Op0, Op1); 10726 case MVT::v4f16: return fastEmit_ISD_FMAXNUM_MVT_v4f16_rr(RetVT, Op0, Op1); 10727 case MVT::v8f16: return fastEmit_ISD_FMAXNUM_MVT_v8f16_rr(RetVT, Op0, Op1); 10728 case MVT::v2f32: return fastEmit_ISD_FMAXNUM_MVT_v2f32_rr(RetVT, Op0, Op1); 10729 case MVT::v4f32: return fastEmit_ISD_FMAXNUM_MVT_v4f32_rr(RetVT, Op0, Op1); 10730 case MVT::v1f64: return fastEmit_ISD_FMAXNUM_MVT_v1f64_rr(RetVT, Op0, Op1); 10731 case MVT::v2f64: return fastEmit_ISD_FMAXNUM_MVT_v2f64_rr(RetVT, Op0, Op1); 10732 default: return 0; 10733 } 10734} 10735 10736// FastEmit functions for ISD::FMINIMUM. 10737 10738unsigned fastEmit_ISD_FMINIMUM_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10739 if (RetVT.SimpleTy != MVT::f16) 10740 return 0; 10741 if ((Subtarget->hasFullFP16())) { 10742 return fastEmitInst_rr(AArch64::FMINHrr, &AArch64::FPR16RegClass, Op0, Op1); 10743 } 10744 return 0; 10745} 10746 10747unsigned fastEmit_ISD_FMINIMUM_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10748 if (RetVT.SimpleTy != MVT::f32) 10749 return 0; 10750 if ((Subtarget->hasFPARMv8())) { 10751 return fastEmitInst_rr(AArch64::FMINSrr, &AArch64::FPR32RegClass, Op0, Op1); 10752 } 10753 return 0; 10754} 10755 10756unsigned fastEmit_ISD_FMINIMUM_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10757 if (RetVT.SimpleTy != MVT::f64) 10758 return 0; 10759 if ((Subtarget->hasFPARMv8())) { 10760 return fastEmitInst_rr(AArch64::FMINDrr, &AArch64::FPR64RegClass, Op0, Op1); 10761 } 10762 return 0; 10763} 10764 10765unsigned fastEmit_ISD_FMINIMUM_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10766 if (RetVT.SimpleTy != MVT::v4f16) 10767 return 0; 10768 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 10769 return fastEmitInst_rr(AArch64::FMINv4f16, &AArch64::FPR64RegClass, Op0, Op1); 10770 } 10771 return 0; 10772} 10773 10774unsigned fastEmit_ISD_FMINIMUM_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10775 if (RetVT.SimpleTy != MVT::v8f16) 10776 return 0; 10777 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 10778 return fastEmitInst_rr(AArch64::FMINv8f16, &AArch64::FPR128RegClass, Op0, Op1); 10779 } 10780 return 0; 10781} 10782 10783unsigned fastEmit_ISD_FMINIMUM_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10784 if (RetVT.SimpleTy != MVT::v2f32) 10785 return 0; 10786 if ((Subtarget->hasNEON())) { 10787 return fastEmitInst_rr(AArch64::FMINv2f32, &AArch64::FPR64RegClass, Op0, Op1); 10788 } 10789 return 0; 10790} 10791 10792unsigned fastEmit_ISD_FMINIMUM_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10793 if (RetVT.SimpleTy != MVT::v4f32) 10794 return 0; 10795 if ((Subtarget->hasNEON())) { 10796 return fastEmitInst_rr(AArch64::FMINv4f32, &AArch64::FPR128RegClass, Op0, Op1); 10797 } 10798 return 0; 10799} 10800 10801unsigned fastEmit_ISD_FMINIMUM_MVT_v1f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10802 if (RetVT.SimpleTy != MVT::v1f64) 10803 return 0; 10804 return fastEmitInst_rr(AArch64::FMINDrr, &AArch64::FPR64RegClass, Op0, Op1); 10805} 10806 10807unsigned fastEmit_ISD_FMINIMUM_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10808 if (RetVT.SimpleTy != MVT::v2f64) 10809 return 0; 10810 if ((Subtarget->hasNEON())) { 10811 return fastEmitInst_rr(AArch64::FMINv2f64, &AArch64::FPR128RegClass, Op0, Op1); 10812 } 10813 return 0; 10814} 10815 10816unsigned fastEmit_ISD_FMINIMUM_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 10817 switch (VT.SimpleTy) { 10818 case MVT::f16: return fastEmit_ISD_FMINIMUM_MVT_f16_rr(RetVT, Op0, Op1); 10819 case MVT::f32: return fastEmit_ISD_FMINIMUM_MVT_f32_rr(RetVT, Op0, Op1); 10820 case MVT::f64: return fastEmit_ISD_FMINIMUM_MVT_f64_rr(RetVT, Op0, Op1); 10821 case MVT::v4f16: return fastEmit_ISD_FMINIMUM_MVT_v4f16_rr(RetVT, Op0, Op1); 10822 case MVT::v8f16: return fastEmit_ISD_FMINIMUM_MVT_v8f16_rr(RetVT, Op0, Op1); 10823 case MVT::v2f32: return fastEmit_ISD_FMINIMUM_MVT_v2f32_rr(RetVT, Op0, Op1); 10824 case MVT::v4f32: return fastEmit_ISD_FMINIMUM_MVT_v4f32_rr(RetVT, Op0, Op1); 10825 case MVT::v1f64: return fastEmit_ISD_FMINIMUM_MVT_v1f64_rr(RetVT, Op0, Op1); 10826 case MVT::v2f64: return fastEmit_ISD_FMINIMUM_MVT_v2f64_rr(RetVT, Op0, Op1); 10827 default: return 0; 10828 } 10829} 10830 10831// FastEmit functions for ISD::FMINNUM. 10832 10833unsigned fastEmit_ISD_FMINNUM_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10834 if (RetVT.SimpleTy != MVT::f16) 10835 return 0; 10836 if ((Subtarget->hasFullFP16())) { 10837 return fastEmitInst_rr(AArch64::FMINNMHrr, &AArch64::FPR16RegClass, Op0, Op1); 10838 } 10839 return 0; 10840} 10841 10842unsigned fastEmit_ISD_FMINNUM_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10843 if (RetVT.SimpleTy != MVT::f32) 10844 return 0; 10845 if ((Subtarget->hasFPARMv8())) { 10846 return fastEmitInst_rr(AArch64::FMINNMSrr, &AArch64::FPR32RegClass, Op0, Op1); 10847 } 10848 return 0; 10849} 10850 10851unsigned fastEmit_ISD_FMINNUM_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10852 if (RetVT.SimpleTy != MVT::f64) 10853 return 0; 10854 if ((Subtarget->hasFPARMv8())) { 10855 return fastEmitInst_rr(AArch64::FMINNMDrr, &AArch64::FPR64RegClass, Op0, Op1); 10856 } 10857 return 0; 10858} 10859 10860unsigned fastEmit_ISD_FMINNUM_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10861 if (RetVT.SimpleTy != MVT::v4f16) 10862 return 0; 10863 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 10864 return fastEmitInst_rr(AArch64::FMINNMv4f16, &AArch64::FPR64RegClass, Op0, Op1); 10865 } 10866 return 0; 10867} 10868 10869unsigned fastEmit_ISD_FMINNUM_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10870 if (RetVT.SimpleTy != MVT::v8f16) 10871 return 0; 10872 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 10873 return fastEmitInst_rr(AArch64::FMINNMv8f16, &AArch64::FPR128RegClass, Op0, Op1); 10874 } 10875 return 0; 10876} 10877 10878unsigned fastEmit_ISD_FMINNUM_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10879 if (RetVT.SimpleTy != MVT::v2f32) 10880 return 0; 10881 if ((Subtarget->hasNEON())) { 10882 return fastEmitInst_rr(AArch64::FMINNMv2f32, &AArch64::FPR64RegClass, Op0, Op1); 10883 } 10884 return 0; 10885} 10886 10887unsigned fastEmit_ISD_FMINNUM_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10888 if (RetVT.SimpleTy != MVT::v4f32) 10889 return 0; 10890 if ((Subtarget->hasNEON())) { 10891 return fastEmitInst_rr(AArch64::FMINNMv4f32, &AArch64::FPR128RegClass, Op0, Op1); 10892 } 10893 return 0; 10894} 10895 10896unsigned fastEmit_ISD_FMINNUM_MVT_v1f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10897 if (RetVT.SimpleTy != MVT::v1f64) 10898 return 0; 10899 return fastEmitInst_rr(AArch64::FMINNMDrr, &AArch64::FPR64RegClass, Op0, Op1); 10900} 10901 10902unsigned fastEmit_ISD_FMINNUM_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10903 if (RetVT.SimpleTy != MVT::v2f64) 10904 return 0; 10905 if ((Subtarget->hasNEON())) { 10906 return fastEmitInst_rr(AArch64::FMINNMv2f64, &AArch64::FPR128RegClass, Op0, Op1); 10907 } 10908 return 0; 10909} 10910 10911unsigned fastEmit_ISD_FMINNUM_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 10912 switch (VT.SimpleTy) { 10913 case MVT::f16: return fastEmit_ISD_FMINNUM_MVT_f16_rr(RetVT, Op0, Op1); 10914 case MVT::f32: return fastEmit_ISD_FMINNUM_MVT_f32_rr(RetVT, Op0, Op1); 10915 case MVT::f64: return fastEmit_ISD_FMINNUM_MVT_f64_rr(RetVT, Op0, Op1); 10916 case MVT::v4f16: return fastEmit_ISD_FMINNUM_MVT_v4f16_rr(RetVT, Op0, Op1); 10917 case MVT::v8f16: return fastEmit_ISD_FMINNUM_MVT_v8f16_rr(RetVT, Op0, Op1); 10918 case MVT::v2f32: return fastEmit_ISD_FMINNUM_MVT_v2f32_rr(RetVT, Op0, Op1); 10919 case MVT::v4f32: return fastEmit_ISD_FMINNUM_MVT_v4f32_rr(RetVT, Op0, Op1); 10920 case MVT::v1f64: return fastEmit_ISD_FMINNUM_MVT_v1f64_rr(RetVT, Op0, Op1); 10921 case MVT::v2f64: return fastEmit_ISD_FMINNUM_MVT_v2f64_rr(RetVT, Op0, Op1); 10922 default: return 0; 10923 } 10924} 10925 10926// FastEmit functions for ISD::FMUL. 10927 10928unsigned fastEmit_ISD_FMUL_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10929 if (RetVT.SimpleTy != MVT::f16) 10930 return 0; 10931 if ((Subtarget->hasFullFP16())) { 10932 return fastEmitInst_rr(AArch64::FMULHrr, &AArch64::FPR16RegClass, Op0, Op1); 10933 } 10934 return 0; 10935} 10936 10937unsigned fastEmit_ISD_FMUL_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10938 if (RetVT.SimpleTy != MVT::f32) 10939 return 0; 10940 if ((Subtarget->hasFPARMv8())) { 10941 return fastEmitInst_rr(AArch64::FMULSrr, &AArch64::FPR32RegClass, Op0, Op1); 10942 } 10943 return 0; 10944} 10945 10946unsigned fastEmit_ISD_FMUL_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10947 if (RetVT.SimpleTy != MVT::f64) 10948 return 0; 10949 if ((Subtarget->hasFPARMv8())) { 10950 return fastEmitInst_rr(AArch64::FMULDrr, &AArch64::FPR64RegClass, Op0, Op1); 10951 } 10952 return 0; 10953} 10954 10955unsigned fastEmit_ISD_FMUL_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10956 if (RetVT.SimpleTy != MVT::v4f16) 10957 return 0; 10958 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 10959 return fastEmitInst_rr(AArch64::FMULv4f16, &AArch64::FPR64RegClass, Op0, Op1); 10960 } 10961 return 0; 10962} 10963 10964unsigned fastEmit_ISD_FMUL_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10965 if (RetVT.SimpleTy != MVT::v8f16) 10966 return 0; 10967 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 10968 return fastEmitInst_rr(AArch64::FMULv8f16, &AArch64::FPR128RegClass, Op0, Op1); 10969 } 10970 return 0; 10971} 10972 10973unsigned fastEmit_ISD_FMUL_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10974 if (RetVT.SimpleTy != MVT::v2f32) 10975 return 0; 10976 if ((Subtarget->hasNEON())) { 10977 return fastEmitInst_rr(AArch64::FMULv2f32, &AArch64::FPR64RegClass, Op0, Op1); 10978 } 10979 return 0; 10980} 10981 10982unsigned fastEmit_ISD_FMUL_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10983 if (RetVT.SimpleTy != MVT::v4f32) 10984 return 0; 10985 if ((Subtarget->hasNEON())) { 10986 return fastEmitInst_rr(AArch64::FMULv4f32, &AArch64::FPR128RegClass, Op0, Op1); 10987 } 10988 return 0; 10989} 10990 10991unsigned fastEmit_ISD_FMUL_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 10992 if (RetVT.SimpleTy != MVT::v2f64) 10993 return 0; 10994 if ((Subtarget->hasNEON())) { 10995 return fastEmitInst_rr(AArch64::FMULv2f64, &AArch64::FPR128RegClass, Op0, Op1); 10996 } 10997 return 0; 10998} 10999 11000unsigned fastEmit_ISD_FMUL_MVT_nxv8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11001 if (RetVT.SimpleTy != MVT::nxv8f16) 11002 return 0; 11003 if ((Subtarget->hasSVEorSME())) { 11004 return fastEmitInst_rr(AArch64::FMUL_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op1); 11005 } 11006 return 0; 11007} 11008 11009unsigned fastEmit_ISD_FMUL_MVT_nxv4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11010 if (RetVT.SimpleTy != MVT::nxv4f32) 11011 return 0; 11012 if ((Subtarget->hasSVEorSME())) { 11013 return fastEmitInst_rr(AArch64::FMUL_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op1); 11014 } 11015 return 0; 11016} 11017 11018unsigned fastEmit_ISD_FMUL_MVT_nxv2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11019 if (RetVT.SimpleTy != MVT::nxv2f64) 11020 return 0; 11021 if ((Subtarget->hasSVEorSME())) { 11022 return fastEmitInst_rr(AArch64::FMUL_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op1); 11023 } 11024 return 0; 11025} 11026 11027unsigned fastEmit_ISD_FMUL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 11028 switch (VT.SimpleTy) { 11029 case MVT::f16: return fastEmit_ISD_FMUL_MVT_f16_rr(RetVT, Op0, Op1); 11030 case MVT::f32: return fastEmit_ISD_FMUL_MVT_f32_rr(RetVT, Op0, Op1); 11031 case MVT::f64: return fastEmit_ISD_FMUL_MVT_f64_rr(RetVT, Op0, Op1); 11032 case MVT::v4f16: return fastEmit_ISD_FMUL_MVT_v4f16_rr(RetVT, Op0, Op1); 11033 case MVT::v8f16: return fastEmit_ISD_FMUL_MVT_v8f16_rr(RetVT, Op0, Op1); 11034 case MVT::v2f32: return fastEmit_ISD_FMUL_MVT_v2f32_rr(RetVT, Op0, Op1); 11035 case MVT::v4f32: return fastEmit_ISD_FMUL_MVT_v4f32_rr(RetVT, Op0, Op1); 11036 case MVT::v2f64: return fastEmit_ISD_FMUL_MVT_v2f64_rr(RetVT, Op0, Op1); 11037 case MVT::nxv8f16: return fastEmit_ISD_FMUL_MVT_nxv8f16_rr(RetVT, Op0, Op1); 11038 case MVT::nxv4f32: return fastEmit_ISD_FMUL_MVT_nxv4f32_rr(RetVT, Op0, Op1); 11039 case MVT::nxv2f64: return fastEmit_ISD_FMUL_MVT_nxv2f64_rr(RetVT, Op0, Op1); 11040 default: return 0; 11041 } 11042} 11043 11044// FastEmit functions for ISD::FSUB. 11045 11046unsigned fastEmit_ISD_FSUB_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11047 if (RetVT.SimpleTy != MVT::f16) 11048 return 0; 11049 if ((Subtarget->hasFullFP16())) { 11050 return fastEmitInst_rr(AArch64::FSUBHrr, &AArch64::FPR16RegClass, Op0, Op1); 11051 } 11052 return 0; 11053} 11054 11055unsigned fastEmit_ISD_FSUB_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11056 if (RetVT.SimpleTy != MVT::f32) 11057 return 0; 11058 if ((Subtarget->hasFPARMv8())) { 11059 return fastEmitInst_rr(AArch64::FSUBSrr, &AArch64::FPR32RegClass, Op0, Op1); 11060 } 11061 return 0; 11062} 11063 11064unsigned fastEmit_ISD_FSUB_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11065 if (RetVT.SimpleTy != MVT::f64) 11066 return 0; 11067 if ((Subtarget->hasFPARMv8())) { 11068 return fastEmitInst_rr(AArch64::FSUBDrr, &AArch64::FPR64RegClass, Op0, Op1); 11069 } 11070 return 0; 11071} 11072 11073unsigned fastEmit_ISD_FSUB_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11074 if (RetVT.SimpleTy != MVT::v4f16) 11075 return 0; 11076 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 11077 return fastEmitInst_rr(AArch64::FSUBv4f16, &AArch64::FPR64RegClass, Op0, Op1); 11078 } 11079 return 0; 11080} 11081 11082unsigned fastEmit_ISD_FSUB_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11083 if (RetVT.SimpleTy != MVT::v8f16) 11084 return 0; 11085 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 11086 return fastEmitInst_rr(AArch64::FSUBv8f16, &AArch64::FPR128RegClass, Op0, Op1); 11087 } 11088 return 0; 11089} 11090 11091unsigned fastEmit_ISD_FSUB_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11092 if (RetVT.SimpleTy != MVT::v2f32) 11093 return 0; 11094 if ((Subtarget->hasNEON())) { 11095 return fastEmitInst_rr(AArch64::FSUBv2f32, &AArch64::FPR64RegClass, Op0, Op1); 11096 } 11097 return 0; 11098} 11099 11100unsigned fastEmit_ISD_FSUB_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11101 if (RetVT.SimpleTy != MVT::v4f32) 11102 return 0; 11103 if ((Subtarget->hasNEON())) { 11104 return fastEmitInst_rr(AArch64::FSUBv4f32, &AArch64::FPR128RegClass, Op0, Op1); 11105 } 11106 return 0; 11107} 11108 11109unsigned fastEmit_ISD_FSUB_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11110 if (RetVT.SimpleTy != MVT::v2f64) 11111 return 0; 11112 if ((Subtarget->hasNEON())) { 11113 return fastEmitInst_rr(AArch64::FSUBv2f64, &AArch64::FPR128RegClass, Op0, Op1); 11114 } 11115 return 0; 11116} 11117 11118unsigned fastEmit_ISD_FSUB_MVT_nxv8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11119 if (RetVT.SimpleTy != MVT::nxv8f16) 11120 return 0; 11121 if ((Subtarget->hasSVEorSME())) { 11122 return fastEmitInst_rr(AArch64::FSUB_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op1); 11123 } 11124 return 0; 11125} 11126 11127unsigned fastEmit_ISD_FSUB_MVT_nxv4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11128 if (RetVT.SimpleTy != MVT::nxv4f32) 11129 return 0; 11130 if ((Subtarget->hasSVEorSME())) { 11131 return fastEmitInst_rr(AArch64::FSUB_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op1); 11132 } 11133 return 0; 11134} 11135 11136unsigned fastEmit_ISD_FSUB_MVT_nxv2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11137 if (RetVT.SimpleTy != MVT::nxv2f64) 11138 return 0; 11139 if ((Subtarget->hasSVEorSME())) { 11140 return fastEmitInst_rr(AArch64::FSUB_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op1); 11141 } 11142 return 0; 11143} 11144 11145unsigned fastEmit_ISD_FSUB_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 11146 switch (VT.SimpleTy) { 11147 case MVT::f16: return fastEmit_ISD_FSUB_MVT_f16_rr(RetVT, Op0, Op1); 11148 case MVT::f32: return fastEmit_ISD_FSUB_MVT_f32_rr(RetVT, Op0, Op1); 11149 case MVT::f64: return fastEmit_ISD_FSUB_MVT_f64_rr(RetVT, Op0, Op1); 11150 case MVT::v4f16: return fastEmit_ISD_FSUB_MVT_v4f16_rr(RetVT, Op0, Op1); 11151 case MVT::v8f16: return fastEmit_ISD_FSUB_MVT_v8f16_rr(RetVT, Op0, Op1); 11152 case MVT::v2f32: return fastEmit_ISD_FSUB_MVT_v2f32_rr(RetVT, Op0, Op1); 11153 case MVT::v4f32: return fastEmit_ISD_FSUB_MVT_v4f32_rr(RetVT, Op0, Op1); 11154 case MVT::v2f64: return fastEmit_ISD_FSUB_MVT_v2f64_rr(RetVT, Op0, Op1); 11155 case MVT::nxv8f16: return fastEmit_ISD_FSUB_MVT_nxv8f16_rr(RetVT, Op0, Op1); 11156 case MVT::nxv4f32: return fastEmit_ISD_FSUB_MVT_nxv4f32_rr(RetVT, Op0, Op1); 11157 case MVT::nxv2f64: return fastEmit_ISD_FSUB_MVT_nxv2f64_rr(RetVT, Op0, Op1); 11158 default: return 0; 11159 } 11160} 11161 11162// FastEmit functions for ISD::MUL. 11163 11164unsigned fastEmit_ISD_MUL_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11165 if (RetVT.SimpleTy != MVT::v8i8) 11166 return 0; 11167 if ((Subtarget->hasNEON())) { 11168 return fastEmitInst_rr(AArch64::MULv8i8, &AArch64::FPR64RegClass, Op0, Op1); 11169 } 11170 return 0; 11171} 11172 11173unsigned fastEmit_ISD_MUL_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11174 if (RetVT.SimpleTy != MVT::v16i8) 11175 return 0; 11176 if ((Subtarget->hasNEON())) { 11177 return fastEmitInst_rr(AArch64::MULv16i8, &AArch64::FPR128RegClass, Op0, Op1); 11178 } 11179 return 0; 11180} 11181 11182unsigned fastEmit_ISD_MUL_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11183 if (RetVT.SimpleTy != MVT::v4i16) 11184 return 0; 11185 if ((Subtarget->hasNEON())) { 11186 return fastEmitInst_rr(AArch64::MULv4i16, &AArch64::FPR64RegClass, Op0, Op1); 11187 } 11188 return 0; 11189} 11190 11191unsigned fastEmit_ISD_MUL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11192 if (RetVT.SimpleTy != MVT::v8i16) 11193 return 0; 11194 if ((Subtarget->hasNEON())) { 11195 return fastEmitInst_rr(AArch64::MULv8i16, &AArch64::FPR128RegClass, Op0, Op1); 11196 } 11197 return 0; 11198} 11199 11200unsigned fastEmit_ISD_MUL_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11201 if (RetVT.SimpleTy != MVT::v2i32) 11202 return 0; 11203 if ((Subtarget->hasNEON())) { 11204 return fastEmitInst_rr(AArch64::MULv2i32, &AArch64::FPR64RegClass, Op0, Op1); 11205 } 11206 return 0; 11207} 11208 11209unsigned fastEmit_ISD_MUL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11210 if (RetVT.SimpleTy != MVT::v4i32) 11211 return 0; 11212 if ((Subtarget->hasNEON())) { 11213 return fastEmitInst_rr(AArch64::MULv4i32, &AArch64::FPR128RegClass, Op0, Op1); 11214 } 11215 return 0; 11216} 11217 11218unsigned fastEmit_ISD_MUL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 11219 switch (VT.SimpleTy) { 11220 case MVT::v8i8: return fastEmit_ISD_MUL_MVT_v8i8_rr(RetVT, Op0, Op1); 11221 case MVT::v16i8: return fastEmit_ISD_MUL_MVT_v16i8_rr(RetVT, Op0, Op1); 11222 case MVT::v4i16: return fastEmit_ISD_MUL_MVT_v4i16_rr(RetVT, Op0, Op1); 11223 case MVT::v8i16: return fastEmit_ISD_MUL_MVT_v8i16_rr(RetVT, Op0, Op1); 11224 case MVT::v2i32: return fastEmit_ISD_MUL_MVT_v2i32_rr(RetVT, Op0, Op1); 11225 case MVT::v4i32: return fastEmit_ISD_MUL_MVT_v4i32_rr(RetVT, Op0, Op1); 11226 default: return 0; 11227 } 11228} 11229 11230// FastEmit functions for ISD::MULHS. 11231 11232unsigned fastEmit_ISD_MULHS_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11233 if (RetVT.SimpleTy != MVT::i64) 11234 return 0; 11235 return fastEmitInst_rr(AArch64::SMULHrr, &AArch64::GPR64RegClass, Op0, Op1); 11236} 11237 11238unsigned fastEmit_ISD_MULHS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 11239 switch (VT.SimpleTy) { 11240 case MVT::i64: return fastEmit_ISD_MULHS_MVT_i64_rr(RetVT, Op0, Op1); 11241 default: return 0; 11242 } 11243} 11244 11245// FastEmit functions for ISD::MULHU. 11246 11247unsigned fastEmit_ISD_MULHU_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11248 if (RetVT.SimpleTy != MVT::i64) 11249 return 0; 11250 return fastEmitInst_rr(AArch64::UMULHrr, &AArch64::GPR64RegClass, Op0, Op1); 11251} 11252 11253unsigned fastEmit_ISD_MULHU_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 11254 switch (VT.SimpleTy) { 11255 case MVT::i64: return fastEmit_ISD_MULHU_MVT_i64_rr(RetVT, Op0, Op1); 11256 default: return 0; 11257 } 11258} 11259 11260// FastEmit functions for ISD::OR. 11261 11262unsigned fastEmit_ISD_OR_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11263 if (RetVT.SimpleTy != MVT::i32) 11264 return 0; 11265 return fastEmitInst_rr(AArch64::ORRWrr, &AArch64::GPR32RegClass, Op0, Op1); 11266} 11267 11268unsigned fastEmit_ISD_OR_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11269 if (RetVT.SimpleTy != MVT::i64) 11270 return 0; 11271 return fastEmitInst_rr(AArch64::ORRXrr, &AArch64::GPR64RegClass, Op0, Op1); 11272} 11273 11274unsigned fastEmit_ISD_OR_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11275 if (RetVT.SimpleTy != MVT::v8i8) 11276 return 0; 11277 if ((Subtarget->hasNEON())) { 11278 return fastEmitInst_rr(AArch64::ORRv8i8, &AArch64::FPR64RegClass, Op0, Op1); 11279 } 11280 return 0; 11281} 11282 11283unsigned fastEmit_ISD_OR_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11284 if (RetVT.SimpleTy != MVT::v16i8) 11285 return 0; 11286 if ((Subtarget->hasNEON())) { 11287 return fastEmitInst_rr(AArch64::ORRv16i8, &AArch64::FPR128RegClass, Op0, Op1); 11288 } 11289 return 0; 11290} 11291 11292unsigned fastEmit_ISD_OR_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11293 if (RetVT.SimpleTy != MVT::v4i16) 11294 return 0; 11295 if ((Subtarget->hasNEON())) { 11296 return fastEmitInst_rr(AArch64::ORRv8i8, &AArch64::FPR64RegClass, Op0, Op1); 11297 } 11298 return 0; 11299} 11300 11301unsigned fastEmit_ISD_OR_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11302 if (RetVT.SimpleTy != MVT::v8i16) 11303 return 0; 11304 if ((Subtarget->hasNEON())) { 11305 return fastEmitInst_rr(AArch64::ORRv16i8, &AArch64::FPR128RegClass, Op0, Op1); 11306 } 11307 return 0; 11308} 11309 11310unsigned fastEmit_ISD_OR_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11311 if (RetVT.SimpleTy != MVT::v2i32) 11312 return 0; 11313 if ((Subtarget->hasNEON())) { 11314 return fastEmitInst_rr(AArch64::ORRv8i8, &AArch64::FPR64RegClass, Op0, Op1); 11315 } 11316 return 0; 11317} 11318 11319unsigned fastEmit_ISD_OR_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11320 if (RetVT.SimpleTy != MVT::v4i32) 11321 return 0; 11322 if ((Subtarget->hasNEON())) { 11323 return fastEmitInst_rr(AArch64::ORRv16i8, &AArch64::FPR128RegClass, Op0, Op1); 11324 } 11325 return 0; 11326} 11327 11328unsigned fastEmit_ISD_OR_MVT_v1i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11329 if (RetVT.SimpleTy != MVT::v1i64) 11330 return 0; 11331 if ((Subtarget->hasNEON())) { 11332 return fastEmitInst_rr(AArch64::ORRv8i8, &AArch64::FPR64RegClass, Op0, Op1); 11333 } 11334 return 0; 11335} 11336 11337unsigned fastEmit_ISD_OR_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11338 if (RetVT.SimpleTy != MVT::v2i64) 11339 return 0; 11340 if ((Subtarget->hasNEON())) { 11341 return fastEmitInst_rr(AArch64::ORRv16i8, &AArch64::FPR128RegClass, Op0, Op1); 11342 } 11343 return 0; 11344} 11345 11346unsigned fastEmit_ISD_OR_MVT_nxv16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11347 if (RetVT.SimpleTy != MVT::nxv16i8) 11348 return 0; 11349 if ((Subtarget->hasSVEorSME())) { 11350 return fastEmitInst_rr(AArch64::ORR_ZZZ, &AArch64::ZPRRegClass, Op0, Op1); 11351 } 11352 return 0; 11353} 11354 11355unsigned fastEmit_ISD_OR_MVT_nxv8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11356 if (RetVT.SimpleTy != MVT::nxv8i16) 11357 return 0; 11358 if ((Subtarget->hasSVEorSME())) { 11359 return fastEmitInst_rr(AArch64::ORR_ZZZ, &AArch64::ZPRRegClass, Op0, Op1); 11360 } 11361 return 0; 11362} 11363 11364unsigned fastEmit_ISD_OR_MVT_nxv4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11365 if (RetVT.SimpleTy != MVT::nxv4i32) 11366 return 0; 11367 if ((Subtarget->hasSVEorSME())) { 11368 return fastEmitInst_rr(AArch64::ORR_ZZZ, &AArch64::ZPRRegClass, Op0, Op1); 11369 } 11370 return 0; 11371} 11372 11373unsigned fastEmit_ISD_OR_MVT_nxv2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11374 if (RetVT.SimpleTy != MVT::nxv2i64) 11375 return 0; 11376 if ((Subtarget->hasSVEorSME())) { 11377 return fastEmitInst_rr(AArch64::ORR_ZZZ, &AArch64::ZPRRegClass, Op0, Op1); 11378 } 11379 return 0; 11380} 11381 11382unsigned fastEmit_ISD_OR_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 11383 switch (VT.SimpleTy) { 11384 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_rr(RetVT, Op0, Op1); 11385 case MVT::i64: return fastEmit_ISD_OR_MVT_i64_rr(RetVT, Op0, Op1); 11386 case MVT::v8i8: return fastEmit_ISD_OR_MVT_v8i8_rr(RetVT, Op0, Op1); 11387 case MVT::v16i8: return fastEmit_ISD_OR_MVT_v16i8_rr(RetVT, Op0, Op1); 11388 case MVT::v4i16: return fastEmit_ISD_OR_MVT_v4i16_rr(RetVT, Op0, Op1); 11389 case MVT::v8i16: return fastEmit_ISD_OR_MVT_v8i16_rr(RetVT, Op0, Op1); 11390 case MVT::v2i32: return fastEmit_ISD_OR_MVT_v2i32_rr(RetVT, Op0, Op1); 11391 case MVT::v4i32: return fastEmit_ISD_OR_MVT_v4i32_rr(RetVT, Op0, Op1); 11392 case MVT::v1i64: return fastEmit_ISD_OR_MVT_v1i64_rr(RetVT, Op0, Op1); 11393 case MVT::v2i64: return fastEmit_ISD_OR_MVT_v2i64_rr(RetVT, Op0, Op1); 11394 case MVT::nxv16i8: return fastEmit_ISD_OR_MVT_nxv16i8_rr(RetVT, Op0, Op1); 11395 case MVT::nxv8i16: return fastEmit_ISD_OR_MVT_nxv8i16_rr(RetVT, Op0, Op1); 11396 case MVT::nxv4i32: return fastEmit_ISD_OR_MVT_nxv4i32_rr(RetVT, Op0, Op1); 11397 case MVT::nxv2i64: return fastEmit_ISD_OR_MVT_nxv2i64_rr(RetVT, Op0, Op1); 11398 default: return 0; 11399 } 11400} 11401 11402// FastEmit functions for ISD::ROTR. 11403 11404unsigned fastEmit_ISD_ROTR_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11405 if (RetVT.SimpleTy != MVT::i64) 11406 return 0; 11407 return fastEmitInst_rr(AArch64::RORVXr, &AArch64::GPR64RegClass, Op0, Op1); 11408} 11409 11410unsigned fastEmit_ISD_ROTR_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 11411 switch (VT.SimpleTy) { 11412 case MVT::i64: return fastEmit_ISD_ROTR_MVT_i64_rr(RetVT, Op0, Op1); 11413 default: return 0; 11414 } 11415} 11416 11417// FastEmit functions for ISD::SADDSAT. 11418 11419unsigned fastEmit_ISD_SADDSAT_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11420 if (RetVT.SimpleTy != MVT::v8i8) 11421 return 0; 11422 if ((Subtarget->hasNEON())) { 11423 return fastEmitInst_rr(AArch64::SQADDv8i8, &AArch64::FPR64RegClass, Op0, Op1); 11424 } 11425 return 0; 11426} 11427 11428unsigned fastEmit_ISD_SADDSAT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11429 if (RetVT.SimpleTy != MVT::v16i8) 11430 return 0; 11431 if ((Subtarget->hasNEON())) { 11432 return fastEmitInst_rr(AArch64::SQADDv16i8, &AArch64::FPR128RegClass, Op0, Op1); 11433 } 11434 return 0; 11435} 11436 11437unsigned fastEmit_ISD_SADDSAT_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11438 if (RetVT.SimpleTy != MVT::v4i16) 11439 return 0; 11440 if ((Subtarget->hasNEON())) { 11441 return fastEmitInst_rr(AArch64::SQADDv4i16, &AArch64::FPR64RegClass, Op0, Op1); 11442 } 11443 return 0; 11444} 11445 11446unsigned fastEmit_ISD_SADDSAT_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11447 if (RetVT.SimpleTy != MVT::v8i16) 11448 return 0; 11449 if ((Subtarget->hasNEON())) { 11450 return fastEmitInst_rr(AArch64::SQADDv8i16, &AArch64::FPR128RegClass, Op0, Op1); 11451 } 11452 return 0; 11453} 11454 11455unsigned fastEmit_ISD_SADDSAT_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11456 if (RetVT.SimpleTy != MVT::v2i32) 11457 return 0; 11458 if ((Subtarget->hasNEON())) { 11459 return fastEmitInst_rr(AArch64::SQADDv2i32, &AArch64::FPR64RegClass, Op0, Op1); 11460 } 11461 return 0; 11462} 11463 11464unsigned fastEmit_ISD_SADDSAT_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11465 if (RetVT.SimpleTy != MVT::v4i32) 11466 return 0; 11467 if ((Subtarget->hasNEON())) { 11468 return fastEmitInst_rr(AArch64::SQADDv4i32, &AArch64::FPR128RegClass, Op0, Op1); 11469 } 11470 return 0; 11471} 11472 11473unsigned fastEmit_ISD_SADDSAT_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11474 if (RetVT.SimpleTy != MVT::v2i64) 11475 return 0; 11476 if ((Subtarget->hasNEON())) { 11477 return fastEmitInst_rr(AArch64::SQADDv2i64, &AArch64::FPR128RegClass, Op0, Op1); 11478 } 11479 return 0; 11480} 11481 11482unsigned fastEmit_ISD_SADDSAT_MVT_nxv16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11483 if (RetVT.SimpleTy != MVT::nxv16i8) 11484 return 0; 11485 if ((Subtarget->hasSVEorSME())) { 11486 return fastEmitInst_rr(AArch64::SQADD_ZZZ_B, &AArch64::ZPRRegClass, Op0, Op1); 11487 } 11488 return 0; 11489} 11490 11491unsigned fastEmit_ISD_SADDSAT_MVT_nxv8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11492 if (RetVT.SimpleTy != MVT::nxv8i16) 11493 return 0; 11494 if ((Subtarget->hasSVEorSME())) { 11495 return fastEmitInst_rr(AArch64::SQADD_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op1); 11496 } 11497 return 0; 11498} 11499 11500unsigned fastEmit_ISD_SADDSAT_MVT_nxv4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11501 if (RetVT.SimpleTy != MVT::nxv4i32) 11502 return 0; 11503 if ((Subtarget->hasSVEorSME())) { 11504 return fastEmitInst_rr(AArch64::SQADD_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op1); 11505 } 11506 return 0; 11507} 11508 11509unsigned fastEmit_ISD_SADDSAT_MVT_nxv2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11510 if (RetVT.SimpleTy != MVT::nxv2i64) 11511 return 0; 11512 if ((Subtarget->hasSVEorSME())) { 11513 return fastEmitInst_rr(AArch64::SQADD_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op1); 11514 } 11515 return 0; 11516} 11517 11518unsigned fastEmit_ISD_SADDSAT_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 11519 switch (VT.SimpleTy) { 11520 case MVT::v8i8: return fastEmit_ISD_SADDSAT_MVT_v8i8_rr(RetVT, Op0, Op1); 11521 case MVT::v16i8: return fastEmit_ISD_SADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1); 11522 case MVT::v4i16: return fastEmit_ISD_SADDSAT_MVT_v4i16_rr(RetVT, Op0, Op1); 11523 case MVT::v8i16: return fastEmit_ISD_SADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1); 11524 case MVT::v2i32: return fastEmit_ISD_SADDSAT_MVT_v2i32_rr(RetVT, Op0, Op1); 11525 case MVT::v4i32: return fastEmit_ISD_SADDSAT_MVT_v4i32_rr(RetVT, Op0, Op1); 11526 case MVT::v2i64: return fastEmit_ISD_SADDSAT_MVT_v2i64_rr(RetVT, Op0, Op1); 11527 case MVT::nxv16i8: return fastEmit_ISD_SADDSAT_MVT_nxv16i8_rr(RetVT, Op0, Op1); 11528 case MVT::nxv8i16: return fastEmit_ISD_SADDSAT_MVT_nxv8i16_rr(RetVT, Op0, Op1); 11529 case MVT::nxv4i32: return fastEmit_ISD_SADDSAT_MVT_nxv4i32_rr(RetVT, Op0, Op1); 11530 case MVT::nxv2i64: return fastEmit_ISD_SADDSAT_MVT_nxv2i64_rr(RetVT, Op0, Op1); 11531 default: return 0; 11532 } 11533} 11534 11535// FastEmit functions for ISD::SDIV. 11536 11537unsigned fastEmit_ISD_SDIV_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11538 if (RetVT.SimpleTy != MVT::i32) 11539 return 0; 11540 return fastEmitInst_rr(AArch64::SDIVWr, &AArch64::GPR32RegClass, Op0, Op1); 11541} 11542 11543unsigned fastEmit_ISD_SDIV_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11544 if (RetVT.SimpleTy != MVT::i64) 11545 return 0; 11546 return fastEmitInst_rr(AArch64::SDIVXr, &AArch64::GPR64RegClass, Op0, Op1); 11547} 11548 11549unsigned fastEmit_ISD_SDIV_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 11550 switch (VT.SimpleTy) { 11551 case MVT::i32: return fastEmit_ISD_SDIV_MVT_i32_rr(RetVT, Op0, Op1); 11552 case MVT::i64: return fastEmit_ISD_SDIV_MVT_i64_rr(RetVT, Op0, Op1); 11553 default: return 0; 11554 } 11555} 11556 11557// FastEmit functions for ISD::SHL. 11558 11559unsigned fastEmit_ISD_SHL_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11560 if (RetVT.SimpleTy != MVT::i64) 11561 return 0; 11562 return fastEmitInst_rr(AArch64::LSLVXr, &AArch64::GPR64RegClass, Op0, Op1); 11563} 11564 11565unsigned fastEmit_ISD_SHL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 11566 switch (VT.SimpleTy) { 11567 case MVT::i64: return fastEmit_ISD_SHL_MVT_i64_rr(RetVT, Op0, Op1); 11568 default: return 0; 11569 } 11570} 11571 11572// FastEmit functions for ISD::SMAX. 11573 11574unsigned fastEmit_ISD_SMAX_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11575 if (RetVT.SimpleTy != MVT::i32) 11576 return 0; 11577 if ((Subtarget->hasCSSC())) { 11578 return fastEmitInst_rr(AArch64::SMAXWrr, &AArch64::GPR32RegClass, Op0, Op1); 11579 } 11580 return 0; 11581} 11582 11583unsigned fastEmit_ISD_SMAX_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11584 if (RetVT.SimpleTy != MVT::i64) 11585 return 0; 11586 if ((Subtarget->hasCSSC())) { 11587 return fastEmitInst_rr(AArch64::SMAXXrr, &AArch64::GPR64RegClass, Op0, Op1); 11588 } 11589 return 0; 11590} 11591 11592unsigned fastEmit_ISD_SMAX_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11593 if (RetVT.SimpleTy != MVT::v8i8) 11594 return 0; 11595 if ((Subtarget->hasNEON())) { 11596 return fastEmitInst_rr(AArch64::SMAXv8i8, &AArch64::FPR64RegClass, Op0, Op1); 11597 } 11598 return 0; 11599} 11600 11601unsigned fastEmit_ISD_SMAX_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11602 if (RetVT.SimpleTy != MVT::v16i8) 11603 return 0; 11604 if ((Subtarget->hasNEON())) { 11605 return fastEmitInst_rr(AArch64::SMAXv16i8, &AArch64::FPR128RegClass, Op0, Op1); 11606 } 11607 return 0; 11608} 11609 11610unsigned fastEmit_ISD_SMAX_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11611 if (RetVT.SimpleTy != MVT::v4i16) 11612 return 0; 11613 if ((Subtarget->hasNEON())) { 11614 return fastEmitInst_rr(AArch64::SMAXv4i16, &AArch64::FPR64RegClass, Op0, Op1); 11615 } 11616 return 0; 11617} 11618 11619unsigned fastEmit_ISD_SMAX_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11620 if (RetVT.SimpleTy != MVT::v8i16) 11621 return 0; 11622 if ((Subtarget->hasNEON())) { 11623 return fastEmitInst_rr(AArch64::SMAXv8i16, &AArch64::FPR128RegClass, Op0, Op1); 11624 } 11625 return 0; 11626} 11627 11628unsigned fastEmit_ISD_SMAX_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11629 if (RetVT.SimpleTy != MVT::v2i32) 11630 return 0; 11631 if ((Subtarget->hasNEON())) { 11632 return fastEmitInst_rr(AArch64::SMAXv2i32, &AArch64::FPR64RegClass, Op0, Op1); 11633 } 11634 return 0; 11635} 11636 11637unsigned fastEmit_ISD_SMAX_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11638 if (RetVT.SimpleTy != MVT::v4i32) 11639 return 0; 11640 if ((Subtarget->hasNEON())) { 11641 return fastEmitInst_rr(AArch64::SMAXv4i32, &AArch64::FPR128RegClass, Op0, Op1); 11642 } 11643 return 0; 11644} 11645 11646unsigned fastEmit_ISD_SMAX_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 11647 switch (VT.SimpleTy) { 11648 case MVT::i32: return fastEmit_ISD_SMAX_MVT_i32_rr(RetVT, Op0, Op1); 11649 case MVT::i64: return fastEmit_ISD_SMAX_MVT_i64_rr(RetVT, Op0, Op1); 11650 case MVT::v8i8: return fastEmit_ISD_SMAX_MVT_v8i8_rr(RetVT, Op0, Op1); 11651 case MVT::v16i8: return fastEmit_ISD_SMAX_MVT_v16i8_rr(RetVT, Op0, Op1); 11652 case MVT::v4i16: return fastEmit_ISD_SMAX_MVT_v4i16_rr(RetVT, Op0, Op1); 11653 case MVT::v8i16: return fastEmit_ISD_SMAX_MVT_v8i16_rr(RetVT, Op0, Op1); 11654 case MVT::v2i32: return fastEmit_ISD_SMAX_MVT_v2i32_rr(RetVT, Op0, Op1); 11655 case MVT::v4i32: return fastEmit_ISD_SMAX_MVT_v4i32_rr(RetVT, Op0, Op1); 11656 default: return 0; 11657 } 11658} 11659 11660// FastEmit functions for ISD::SMIN. 11661 11662unsigned fastEmit_ISD_SMIN_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11663 if (RetVT.SimpleTy != MVT::i32) 11664 return 0; 11665 if ((Subtarget->hasCSSC())) { 11666 return fastEmitInst_rr(AArch64::SMINWrr, &AArch64::GPR32RegClass, Op0, Op1); 11667 } 11668 return 0; 11669} 11670 11671unsigned fastEmit_ISD_SMIN_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11672 if (RetVT.SimpleTy != MVT::i64) 11673 return 0; 11674 if ((Subtarget->hasCSSC())) { 11675 return fastEmitInst_rr(AArch64::SMINXrr, &AArch64::GPR64RegClass, Op0, Op1); 11676 } 11677 return 0; 11678} 11679 11680unsigned fastEmit_ISD_SMIN_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11681 if (RetVT.SimpleTy != MVT::v8i8) 11682 return 0; 11683 if ((Subtarget->hasNEON())) { 11684 return fastEmitInst_rr(AArch64::SMINv8i8, &AArch64::FPR64RegClass, Op0, Op1); 11685 } 11686 return 0; 11687} 11688 11689unsigned fastEmit_ISD_SMIN_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11690 if (RetVT.SimpleTy != MVT::v16i8) 11691 return 0; 11692 if ((Subtarget->hasNEON())) { 11693 return fastEmitInst_rr(AArch64::SMINv16i8, &AArch64::FPR128RegClass, Op0, Op1); 11694 } 11695 return 0; 11696} 11697 11698unsigned fastEmit_ISD_SMIN_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11699 if (RetVT.SimpleTy != MVT::v4i16) 11700 return 0; 11701 if ((Subtarget->hasNEON())) { 11702 return fastEmitInst_rr(AArch64::SMINv4i16, &AArch64::FPR64RegClass, Op0, Op1); 11703 } 11704 return 0; 11705} 11706 11707unsigned fastEmit_ISD_SMIN_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11708 if (RetVT.SimpleTy != MVT::v8i16) 11709 return 0; 11710 if ((Subtarget->hasNEON())) { 11711 return fastEmitInst_rr(AArch64::SMINv8i16, &AArch64::FPR128RegClass, Op0, Op1); 11712 } 11713 return 0; 11714} 11715 11716unsigned fastEmit_ISD_SMIN_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11717 if (RetVT.SimpleTy != MVT::v2i32) 11718 return 0; 11719 if ((Subtarget->hasNEON())) { 11720 return fastEmitInst_rr(AArch64::SMINv2i32, &AArch64::FPR64RegClass, Op0, Op1); 11721 } 11722 return 0; 11723} 11724 11725unsigned fastEmit_ISD_SMIN_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11726 if (RetVT.SimpleTy != MVT::v4i32) 11727 return 0; 11728 if ((Subtarget->hasNEON())) { 11729 return fastEmitInst_rr(AArch64::SMINv4i32, &AArch64::FPR128RegClass, Op0, Op1); 11730 } 11731 return 0; 11732} 11733 11734unsigned fastEmit_ISD_SMIN_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 11735 switch (VT.SimpleTy) { 11736 case MVT::i32: return fastEmit_ISD_SMIN_MVT_i32_rr(RetVT, Op0, Op1); 11737 case MVT::i64: return fastEmit_ISD_SMIN_MVT_i64_rr(RetVT, Op0, Op1); 11738 case MVT::v8i8: return fastEmit_ISD_SMIN_MVT_v8i8_rr(RetVT, Op0, Op1); 11739 case MVT::v16i8: return fastEmit_ISD_SMIN_MVT_v16i8_rr(RetVT, Op0, Op1); 11740 case MVT::v4i16: return fastEmit_ISD_SMIN_MVT_v4i16_rr(RetVT, Op0, Op1); 11741 case MVT::v8i16: return fastEmit_ISD_SMIN_MVT_v8i16_rr(RetVT, Op0, Op1); 11742 case MVT::v2i32: return fastEmit_ISD_SMIN_MVT_v2i32_rr(RetVT, Op0, Op1); 11743 case MVT::v4i32: return fastEmit_ISD_SMIN_MVT_v4i32_rr(RetVT, Op0, Op1); 11744 default: return 0; 11745 } 11746} 11747 11748// FastEmit functions for ISD::SRA. 11749 11750unsigned fastEmit_ISD_SRA_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11751 if (RetVT.SimpleTy != MVT::i64) 11752 return 0; 11753 return fastEmitInst_rr(AArch64::ASRVXr, &AArch64::GPR64RegClass, Op0, Op1); 11754} 11755 11756unsigned fastEmit_ISD_SRA_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 11757 switch (VT.SimpleTy) { 11758 case MVT::i64: return fastEmit_ISD_SRA_MVT_i64_rr(RetVT, Op0, Op1); 11759 default: return 0; 11760 } 11761} 11762 11763// FastEmit functions for ISD::SRL. 11764 11765unsigned fastEmit_ISD_SRL_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11766 if (RetVT.SimpleTy != MVT::i64) 11767 return 0; 11768 return fastEmitInst_rr(AArch64::LSRVXr, &AArch64::GPR64RegClass, Op0, Op1); 11769} 11770 11771unsigned fastEmit_ISD_SRL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 11772 switch (VT.SimpleTy) { 11773 case MVT::i64: return fastEmit_ISD_SRL_MVT_i64_rr(RetVT, Op0, Op1); 11774 default: return 0; 11775 } 11776} 11777 11778// FastEmit functions for ISD::SSUBSAT. 11779 11780unsigned fastEmit_ISD_SSUBSAT_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11781 if (RetVT.SimpleTy != MVT::v8i8) 11782 return 0; 11783 if ((Subtarget->hasNEON())) { 11784 return fastEmitInst_rr(AArch64::SQSUBv8i8, &AArch64::FPR64RegClass, Op0, Op1); 11785 } 11786 return 0; 11787} 11788 11789unsigned fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11790 if (RetVT.SimpleTy != MVT::v16i8) 11791 return 0; 11792 if ((Subtarget->hasNEON())) { 11793 return fastEmitInst_rr(AArch64::SQSUBv16i8, &AArch64::FPR128RegClass, Op0, Op1); 11794 } 11795 return 0; 11796} 11797 11798unsigned fastEmit_ISD_SSUBSAT_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11799 if (RetVT.SimpleTy != MVT::v4i16) 11800 return 0; 11801 if ((Subtarget->hasNEON())) { 11802 return fastEmitInst_rr(AArch64::SQSUBv4i16, &AArch64::FPR64RegClass, Op0, Op1); 11803 } 11804 return 0; 11805} 11806 11807unsigned fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11808 if (RetVT.SimpleTy != MVT::v8i16) 11809 return 0; 11810 if ((Subtarget->hasNEON())) { 11811 return fastEmitInst_rr(AArch64::SQSUBv8i16, &AArch64::FPR128RegClass, Op0, Op1); 11812 } 11813 return 0; 11814} 11815 11816unsigned fastEmit_ISD_SSUBSAT_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11817 if (RetVT.SimpleTy != MVT::v2i32) 11818 return 0; 11819 if ((Subtarget->hasNEON())) { 11820 return fastEmitInst_rr(AArch64::SQSUBv2i32, &AArch64::FPR64RegClass, Op0, Op1); 11821 } 11822 return 0; 11823} 11824 11825unsigned fastEmit_ISD_SSUBSAT_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11826 if (RetVT.SimpleTy != MVT::v4i32) 11827 return 0; 11828 if ((Subtarget->hasNEON())) { 11829 return fastEmitInst_rr(AArch64::SQSUBv4i32, &AArch64::FPR128RegClass, Op0, Op1); 11830 } 11831 return 0; 11832} 11833 11834unsigned fastEmit_ISD_SSUBSAT_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11835 if (RetVT.SimpleTy != MVT::v2i64) 11836 return 0; 11837 if ((Subtarget->hasNEON())) { 11838 return fastEmitInst_rr(AArch64::SQSUBv2i64, &AArch64::FPR128RegClass, Op0, Op1); 11839 } 11840 return 0; 11841} 11842 11843unsigned fastEmit_ISD_SSUBSAT_MVT_nxv16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11844 if (RetVT.SimpleTy != MVT::nxv16i8) 11845 return 0; 11846 if ((Subtarget->hasSVEorSME())) { 11847 return fastEmitInst_rr(AArch64::SQSUB_ZZZ_B, &AArch64::ZPRRegClass, Op0, Op1); 11848 } 11849 return 0; 11850} 11851 11852unsigned fastEmit_ISD_SSUBSAT_MVT_nxv8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11853 if (RetVT.SimpleTy != MVT::nxv8i16) 11854 return 0; 11855 if ((Subtarget->hasSVEorSME())) { 11856 return fastEmitInst_rr(AArch64::SQSUB_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op1); 11857 } 11858 return 0; 11859} 11860 11861unsigned fastEmit_ISD_SSUBSAT_MVT_nxv4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11862 if (RetVT.SimpleTy != MVT::nxv4i32) 11863 return 0; 11864 if ((Subtarget->hasSVEorSME())) { 11865 return fastEmitInst_rr(AArch64::SQSUB_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op1); 11866 } 11867 return 0; 11868} 11869 11870unsigned fastEmit_ISD_SSUBSAT_MVT_nxv2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11871 if (RetVT.SimpleTy != MVT::nxv2i64) 11872 return 0; 11873 if ((Subtarget->hasSVEorSME())) { 11874 return fastEmitInst_rr(AArch64::SQSUB_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op1); 11875 } 11876 return 0; 11877} 11878 11879unsigned fastEmit_ISD_SSUBSAT_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 11880 switch (VT.SimpleTy) { 11881 case MVT::v8i8: return fastEmit_ISD_SSUBSAT_MVT_v8i8_rr(RetVT, Op0, Op1); 11882 case MVT::v16i8: return fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1); 11883 case MVT::v4i16: return fastEmit_ISD_SSUBSAT_MVT_v4i16_rr(RetVT, Op0, Op1); 11884 case MVT::v8i16: return fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1); 11885 case MVT::v2i32: return fastEmit_ISD_SSUBSAT_MVT_v2i32_rr(RetVT, Op0, Op1); 11886 case MVT::v4i32: return fastEmit_ISD_SSUBSAT_MVT_v4i32_rr(RetVT, Op0, Op1); 11887 case MVT::v2i64: return fastEmit_ISD_SSUBSAT_MVT_v2i64_rr(RetVT, Op0, Op1); 11888 case MVT::nxv16i8: return fastEmit_ISD_SSUBSAT_MVT_nxv16i8_rr(RetVT, Op0, Op1); 11889 case MVT::nxv8i16: return fastEmit_ISD_SSUBSAT_MVT_nxv8i16_rr(RetVT, Op0, Op1); 11890 case MVT::nxv4i32: return fastEmit_ISD_SSUBSAT_MVT_nxv4i32_rr(RetVT, Op0, Op1); 11891 case MVT::nxv2i64: return fastEmit_ISD_SSUBSAT_MVT_nxv2i64_rr(RetVT, Op0, Op1); 11892 default: return 0; 11893 } 11894} 11895 11896// FastEmit functions for ISD::STRICT_FADD. 11897 11898unsigned fastEmit_ISD_STRICT_FADD_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11899 if (RetVT.SimpleTy != MVT::f16) 11900 return 0; 11901 if ((Subtarget->hasFullFP16())) { 11902 return fastEmitInst_rr(AArch64::FADDHrr, &AArch64::FPR16RegClass, Op0, Op1); 11903 } 11904 return 0; 11905} 11906 11907unsigned fastEmit_ISD_STRICT_FADD_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11908 if (RetVT.SimpleTy != MVT::f32) 11909 return 0; 11910 if ((Subtarget->hasFPARMv8())) { 11911 return fastEmitInst_rr(AArch64::FADDSrr, &AArch64::FPR32RegClass, Op0, Op1); 11912 } 11913 return 0; 11914} 11915 11916unsigned fastEmit_ISD_STRICT_FADD_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11917 if (RetVT.SimpleTy != MVT::f64) 11918 return 0; 11919 if ((Subtarget->hasFPARMv8())) { 11920 return fastEmitInst_rr(AArch64::FADDDrr, &AArch64::FPR64RegClass, Op0, Op1); 11921 } 11922 return 0; 11923} 11924 11925unsigned fastEmit_ISD_STRICT_FADD_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11926 if (RetVT.SimpleTy != MVT::v4f16) 11927 return 0; 11928 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 11929 return fastEmitInst_rr(AArch64::FADDv4f16, &AArch64::FPR64RegClass, Op0, Op1); 11930 } 11931 return 0; 11932} 11933 11934unsigned fastEmit_ISD_STRICT_FADD_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11935 if (RetVT.SimpleTy != MVT::v8f16) 11936 return 0; 11937 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 11938 return fastEmitInst_rr(AArch64::FADDv8f16, &AArch64::FPR128RegClass, Op0, Op1); 11939 } 11940 return 0; 11941} 11942 11943unsigned fastEmit_ISD_STRICT_FADD_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11944 if (RetVT.SimpleTy != MVT::v2f32) 11945 return 0; 11946 if ((Subtarget->hasNEON())) { 11947 return fastEmitInst_rr(AArch64::FADDv2f32, &AArch64::FPR64RegClass, Op0, Op1); 11948 } 11949 return 0; 11950} 11951 11952unsigned fastEmit_ISD_STRICT_FADD_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11953 if (RetVT.SimpleTy != MVT::v4f32) 11954 return 0; 11955 if ((Subtarget->hasNEON())) { 11956 return fastEmitInst_rr(AArch64::FADDv4f32, &AArch64::FPR128RegClass, Op0, Op1); 11957 } 11958 return 0; 11959} 11960 11961unsigned fastEmit_ISD_STRICT_FADD_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11962 if (RetVT.SimpleTy != MVT::v2f64) 11963 return 0; 11964 if ((Subtarget->hasNEON())) { 11965 return fastEmitInst_rr(AArch64::FADDv2f64, &AArch64::FPR128RegClass, Op0, Op1); 11966 } 11967 return 0; 11968} 11969 11970unsigned fastEmit_ISD_STRICT_FADD_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 11971 switch (VT.SimpleTy) { 11972 case MVT::f16: return fastEmit_ISD_STRICT_FADD_MVT_f16_rr(RetVT, Op0, Op1); 11973 case MVT::f32: return fastEmit_ISD_STRICT_FADD_MVT_f32_rr(RetVT, Op0, Op1); 11974 case MVT::f64: return fastEmit_ISD_STRICT_FADD_MVT_f64_rr(RetVT, Op0, Op1); 11975 case MVT::v4f16: return fastEmit_ISD_STRICT_FADD_MVT_v4f16_rr(RetVT, Op0, Op1); 11976 case MVT::v8f16: return fastEmit_ISD_STRICT_FADD_MVT_v8f16_rr(RetVT, Op0, Op1); 11977 case MVT::v2f32: return fastEmit_ISD_STRICT_FADD_MVT_v2f32_rr(RetVT, Op0, Op1); 11978 case MVT::v4f32: return fastEmit_ISD_STRICT_FADD_MVT_v4f32_rr(RetVT, Op0, Op1); 11979 case MVT::v2f64: return fastEmit_ISD_STRICT_FADD_MVT_v2f64_rr(RetVT, Op0, Op1); 11980 default: return 0; 11981 } 11982} 11983 11984// FastEmit functions for ISD::STRICT_FDIV. 11985 11986unsigned fastEmit_ISD_STRICT_FDIV_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11987 if (RetVT.SimpleTy != MVT::f16) 11988 return 0; 11989 if ((Subtarget->hasFullFP16())) { 11990 return fastEmitInst_rr(AArch64::FDIVHrr, &AArch64::FPR16RegClass, Op0, Op1); 11991 } 11992 return 0; 11993} 11994 11995unsigned fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 11996 if (RetVT.SimpleTy != MVT::f32) 11997 return 0; 11998 if ((Subtarget->hasFPARMv8())) { 11999 return fastEmitInst_rr(AArch64::FDIVSrr, &AArch64::FPR32RegClass, Op0, Op1); 12000 } 12001 return 0; 12002} 12003 12004unsigned fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12005 if (RetVT.SimpleTy != MVT::f64) 12006 return 0; 12007 if ((Subtarget->hasFPARMv8())) { 12008 return fastEmitInst_rr(AArch64::FDIVDrr, &AArch64::FPR64RegClass, Op0, Op1); 12009 } 12010 return 0; 12011} 12012 12013unsigned fastEmit_ISD_STRICT_FDIV_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12014 if (RetVT.SimpleTy != MVT::v4f16) 12015 return 0; 12016 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 12017 return fastEmitInst_rr(AArch64::FDIVv4f16, &AArch64::FPR64RegClass, Op0, Op1); 12018 } 12019 return 0; 12020} 12021 12022unsigned fastEmit_ISD_STRICT_FDIV_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12023 if (RetVT.SimpleTy != MVT::v8f16) 12024 return 0; 12025 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 12026 return fastEmitInst_rr(AArch64::FDIVv8f16, &AArch64::FPR128RegClass, Op0, Op1); 12027 } 12028 return 0; 12029} 12030 12031unsigned fastEmit_ISD_STRICT_FDIV_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12032 if (RetVT.SimpleTy != MVT::v2f32) 12033 return 0; 12034 if ((Subtarget->hasNEON())) { 12035 return fastEmitInst_rr(AArch64::FDIVv2f32, &AArch64::FPR64RegClass, Op0, Op1); 12036 } 12037 return 0; 12038} 12039 12040unsigned fastEmit_ISD_STRICT_FDIV_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12041 if (RetVT.SimpleTy != MVT::v4f32) 12042 return 0; 12043 if ((Subtarget->hasNEON())) { 12044 return fastEmitInst_rr(AArch64::FDIVv4f32, &AArch64::FPR128RegClass, Op0, Op1); 12045 } 12046 return 0; 12047} 12048 12049unsigned fastEmit_ISD_STRICT_FDIV_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12050 if (RetVT.SimpleTy != MVT::v2f64) 12051 return 0; 12052 if ((Subtarget->hasNEON())) { 12053 return fastEmitInst_rr(AArch64::FDIVv2f64, &AArch64::FPR128RegClass, Op0, Op1); 12054 } 12055 return 0; 12056} 12057 12058unsigned fastEmit_ISD_STRICT_FDIV_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 12059 switch (VT.SimpleTy) { 12060 case MVT::f16: return fastEmit_ISD_STRICT_FDIV_MVT_f16_rr(RetVT, Op0, Op1); 12061 case MVT::f32: return fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(RetVT, Op0, Op1); 12062 case MVT::f64: return fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(RetVT, Op0, Op1); 12063 case MVT::v4f16: return fastEmit_ISD_STRICT_FDIV_MVT_v4f16_rr(RetVT, Op0, Op1); 12064 case MVT::v8f16: return fastEmit_ISD_STRICT_FDIV_MVT_v8f16_rr(RetVT, Op0, Op1); 12065 case MVT::v2f32: return fastEmit_ISD_STRICT_FDIV_MVT_v2f32_rr(RetVT, Op0, Op1); 12066 case MVT::v4f32: return fastEmit_ISD_STRICT_FDIV_MVT_v4f32_rr(RetVT, Op0, Op1); 12067 case MVT::v2f64: return fastEmit_ISD_STRICT_FDIV_MVT_v2f64_rr(RetVT, Op0, Op1); 12068 default: return 0; 12069 } 12070} 12071 12072// FastEmit functions for ISD::STRICT_FMAXIMUM. 12073 12074unsigned fastEmit_ISD_STRICT_FMAXIMUM_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12075 if (RetVT.SimpleTy != MVT::f16) 12076 return 0; 12077 if ((Subtarget->hasFullFP16())) { 12078 return fastEmitInst_rr(AArch64::FMAXHrr, &AArch64::FPR16RegClass, Op0, Op1); 12079 } 12080 return 0; 12081} 12082 12083unsigned fastEmit_ISD_STRICT_FMAXIMUM_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12084 if (RetVT.SimpleTy != MVT::f32) 12085 return 0; 12086 if ((Subtarget->hasFPARMv8())) { 12087 return fastEmitInst_rr(AArch64::FMAXSrr, &AArch64::FPR32RegClass, Op0, Op1); 12088 } 12089 return 0; 12090} 12091 12092unsigned fastEmit_ISD_STRICT_FMAXIMUM_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12093 if (RetVT.SimpleTy != MVT::f64) 12094 return 0; 12095 if ((Subtarget->hasFPARMv8())) { 12096 return fastEmitInst_rr(AArch64::FMAXDrr, &AArch64::FPR64RegClass, Op0, Op1); 12097 } 12098 return 0; 12099} 12100 12101unsigned fastEmit_ISD_STRICT_FMAXIMUM_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12102 if (RetVT.SimpleTy != MVT::v4f16) 12103 return 0; 12104 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 12105 return fastEmitInst_rr(AArch64::FMAXv4f16, &AArch64::FPR64RegClass, Op0, Op1); 12106 } 12107 return 0; 12108} 12109 12110unsigned fastEmit_ISD_STRICT_FMAXIMUM_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12111 if (RetVT.SimpleTy != MVT::v8f16) 12112 return 0; 12113 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 12114 return fastEmitInst_rr(AArch64::FMAXv8f16, &AArch64::FPR128RegClass, Op0, Op1); 12115 } 12116 return 0; 12117} 12118 12119unsigned fastEmit_ISD_STRICT_FMAXIMUM_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12120 if (RetVT.SimpleTy != MVT::v2f32) 12121 return 0; 12122 if ((Subtarget->hasNEON())) { 12123 return fastEmitInst_rr(AArch64::FMAXv2f32, &AArch64::FPR64RegClass, Op0, Op1); 12124 } 12125 return 0; 12126} 12127 12128unsigned fastEmit_ISD_STRICT_FMAXIMUM_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12129 if (RetVT.SimpleTy != MVT::v4f32) 12130 return 0; 12131 if ((Subtarget->hasNEON())) { 12132 return fastEmitInst_rr(AArch64::FMAXv4f32, &AArch64::FPR128RegClass, Op0, Op1); 12133 } 12134 return 0; 12135} 12136 12137unsigned fastEmit_ISD_STRICT_FMAXIMUM_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12138 if (RetVT.SimpleTy != MVT::v2f64) 12139 return 0; 12140 if ((Subtarget->hasNEON())) { 12141 return fastEmitInst_rr(AArch64::FMAXv2f64, &AArch64::FPR128RegClass, Op0, Op1); 12142 } 12143 return 0; 12144} 12145 12146unsigned fastEmit_ISD_STRICT_FMAXIMUM_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 12147 switch (VT.SimpleTy) { 12148 case MVT::f16: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_f16_rr(RetVT, Op0, Op1); 12149 case MVT::f32: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_f32_rr(RetVT, Op0, Op1); 12150 case MVT::f64: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_f64_rr(RetVT, Op0, Op1); 12151 case MVT::v4f16: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v4f16_rr(RetVT, Op0, Op1); 12152 case MVT::v8f16: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v8f16_rr(RetVT, Op0, Op1); 12153 case MVT::v2f32: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v2f32_rr(RetVT, Op0, Op1); 12154 case MVT::v4f32: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v4f32_rr(RetVT, Op0, Op1); 12155 case MVT::v2f64: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v2f64_rr(RetVT, Op0, Op1); 12156 default: return 0; 12157 } 12158} 12159 12160// FastEmit functions for ISD::STRICT_FMAXNUM. 12161 12162unsigned fastEmit_ISD_STRICT_FMAXNUM_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12163 if (RetVT.SimpleTy != MVT::f16) 12164 return 0; 12165 if ((Subtarget->hasFullFP16())) { 12166 return fastEmitInst_rr(AArch64::FMAXNMHrr, &AArch64::FPR16RegClass, Op0, Op1); 12167 } 12168 return 0; 12169} 12170 12171unsigned fastEmit_ISD_STRICT_FMAXNUM_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12172 if (RetVT.SimpleTy != MVT::f32) 12173 return 0; 12174 if ((Subtarget->hasFPARMv8())) { 12175 return fastEmitInst_rr(AArch64::FMAXNMSrr, &AArch64::FPR32RegClass, Op0, Op1); 12176 } 12177 return 0; 12178} 12179 12180unsigned fastEmit_ISD_STRICT_FMAXNUM_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12181 if (RetVT.SimpleTy != MVT::f64) 12182 return 0; 12183 if ((Subtarget->hasFPARMv8())) { 12184 return fastEmitInst_rr(AArch64::FMAXNMDrr, &AArch64::FPR64RegClass, Op0, Op1); 12185 } 12186 return 0; 12187} 12188 12189unsigned fastEmit_ISD_STRICT_FMAXNUM_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12190 if (RetVT.SimpleTy != MVT::v4f16) 12191 return 0; 12192 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 12193 return fastEmitInst_rr(AArch64::FMAXNMv4f16, &AArch64::FPR64RegClass, Op0, Op1); 12194 } 12195 return 0; 12196} 12197 12198unsigned fastEmit_ISD_STRICT_FMAXNUM_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12199 if (RetVT.SimpleTy != MVT::v8f16) 12200 return 0; 12201 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 12202 return fastEmitInst_rr(AArch64::FMAXNMv8f16, &AArch64::FPR128RegClass, Op0, Op1); 12203 } 12204 return 0; 12205} 12206 12207unsigned fastEmit_ISD_STRICT_FMAXNUM_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12208 if (RetVT.SimpleTy != MVT::v2f32) 12209 return 0; 12210 if ((Subtarget->hasNEON())) { 12211 return fastEmitInst_rr(AArch64::FMAXNMv2f32, &AArch64::FPR64RegClass, Op0, Op1); 12212 } 12213 return 0; 12214} 12215 12216unsigned fastEmit_ISD_STRICT_FMAXNUM_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12217 if (RetVT.SimpleTy != MVT::v4f32) 12218 return 0; 12219 if ((Subtarget->hasNEON())) { 12220 return fastEmitInst_rr(AArch64::FMAXNMv4f32, &AArch64::FPR128RegClass, Op0, Op1); 12221 } 12222 return 0; 12223} 12224 12225unsigned fastEmit_ISD_STRICT_FMAXNUM_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12226 if (RetVT.SimpleTy != MVT::v2f64) 12227 return 0; 12228 if ((Subtarget->hasNEON())) { 12229 return fastEmitInst_rr(AArch64::FMAXNMv2f64, &AArch64::FPR128RegClass, Op0, Op1); 12230 } 12231 return 0; 12232} 12233 12234unsigned fastEmit_ISD_STRICT_FMAXNUM_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 12235 switch (VT.SimpleTy) { 12236 case MVT::f16: return fastEmit_ISD_STRICT_FMAXNUM_MVT_f16_rr(RetVT, Op0, Op1); 12237 case MVT::f32: return fastEmit_ISD_STRICT_FMAXNUM_MVT_f32_rr(RetVT, Op0, Op1); 12238 case MVT::f64: return fastEmit_ISD_STRICT_FMAXNUM_MVT_f64_rr(RetVT, Op0, Op1); 12239 case MVT::v4f16: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v4f16_rr(RetVT, Op0, Op1); 12240 case MVT::v8f16: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v8f16_rr(RetVT, Op0, Op1); 12241 case MVT::v2f32: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v2f32_rr(RetVT, Op0, Op1); 12242 case MVT::v4f32: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v4f32_rr(RetVT, Op0, Op1); 12243 case MVT::v2f64: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v2f64_rr(RetVT, Op0, Op1); 12244 default: return 0; 12245 } 12246} 12247 12248// FastEmit functions for ISD::STRICT_FMINIMUM. 12249 12250unsigned fastEmit_ISD_STRICT_FMINIMUM_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12251 if (RetVT.SimpleTy != MVT::f16) 12252 return 0; 12253 if ((Subtarget->hasFullFP16())) { 12254 return fastEmitInst_rr(AArch64::FMINHrr, &AArch64::FPR16RegClass, Op0, Op1); 12255 } 12256 return 0; 12257} 12258 12259unsigned fastEmit_ISD_STRICT_FMINIMUM_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12260 if (RetVT.SimpleTy != MVT::f32) 12261 return 0; 12262 if ((Subtarget->hasFPARMv8())) { 12263 return fastEmitInst_rr(AArch64::FMINSrr, &AArch64::FPR32RegClass, Op0, Op1); 12264 } 12265 return 0; 12266} 12267 12268unsigned fastEmit_ISD_STRICT_FMINIMUM_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12269 if (RetVT.SimpleTy != MVT::f64) 12270 return 0; 12271 if ((Subtarget->hasFPARMv8())) { 12272 return fastEmitInst_rr(AArch64::FMINDrr, &AArch64::FPR64RegClass, Op0, Op1); 12273 } 12274 return 0; 12275} 12276 12277unsigned fastEmit_ISD_STRICT_FMINIMUM_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12278 if (RetVT.SimpleTy != MVT::v4f16) 12279 return 0; 12280 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 12281 return fastEmitInst_rr(AArch64::FMINv4f16, &AArch64::FPR64RegClass, Op0, Op1); 12282 } 12283 return 0; 12284} 12285 12286unsigned fastEmit_ISD_STRICT_FMINIMUM_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12287 if (RetVT.SimpleTy != MVT::v8f16) 12288 return 0; 12289 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 12290 return fastEmitInst_rr(AArch64::FMINv8f16, &AArch64::FPR128RegClass, Op0, Op1); 12291 } 12292 return 0; 12293} 12294 12295unsigned fastEmit_ISD_STRICT_FMINIMUM_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12296 if (RetVT.SimpleTy != MVT::v2f32) 12297 return 0; 12298 if ((Subtarget->hasNEON())) { 12299 return fastEmitInst_rr(AArch64::FMINv2f32, &AArch64::FPR64RegClass, Op0, Op1); 12300 } 12301 return 0; 12302} 12303 12304unsigned fastEmit_ISD_STRICT_FMINIMUM_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12305 if (RetVT.SimpleTy != MVT::v4f32) 12306 return 0; 12307 if ((Subtarget->hasNEON())) { 12308 return fastEmitInst_rr(AArch64::FMINv4f32, &AArch64::FPR128RegClass, Op0, Op1); 12309 } 12310 return 0; 12311} 12312 12313unsigned fastEmit_ISD_STRICT_FMINIMUM_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12314 if (RetVT.SimpleTy != MVT::v2f64) 12315 return 0; 12316 if ((Subtarget->hasNEON())) { 12317 return fastEmitInst_rr(AArch64::FMINv2f64, &AArch64::FPR128RegClass, Op0, Op1); 12318 } 12319 return 0; 12320} 12321 12322unsigned fastEmit_ISD_STRICT_FMINIMUM_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 12323 switch (VT.SimpleTy) { 12324 case MVT::f16: return fastEmit_ISD_STRICT_FMINIMUM_MVT_f16_rr(RetVT, Op0, Op1); 12325 case MVT::f32: return fastEmit_ISD_STRICT_FMINIMUM_MVT_f32_rr(RetVT, Op0, Op1); 12326 case MVT::f64: return fastEmit_ISD_STRICT_FMINIMUM_MVT_f64_rr(RetVT, Op0, Op1); 12327 case MVT::v4f16: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v4f16_rr(RetVT, Op0, Op1); 12328 case MVT::v8f16: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v8f16_rr(RetVT, Op0, Op1); 12329 case MVT::v2f32: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v2f32_rr(RetVT, Op0, Op1); 12330 case MVT::v4f32: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v4f32_rr(RetVT, Op0, Op1); 12331 case MVT::v2f64: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v2f64_rr(RetVT, Op0, Op1); 12332 default: return 0; 12333 } 12334} 12335 12336// FastEmit functions for ISD::STRICT_FMINNUM. 12337 12338unsigned fastEmit_ISD_STRICT_FMINNUM_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12339 if (RetVT.SimpleTy != MVT::f16) 12340 return 0; 12341 if ((Subtarget->hasFullFP16())) { 12342 return fastEmitInst_rr(AArch64::FMINNMHrr, &AArch64::FPR16RegClass, Op0, Op1); 12343 } 12344 return 0; 12345} 12346 12347unsigned fastEmit_ISD_STRICT_FMINNUM_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12348 if (RetVT.SimpleTy != MVT::f32) 12349 return 0; 12350 if ((Subtarget->hasFPARMv8())) { 12351 return fastEmitInst_rr(AArch64::FMINNMSrr, &AArch64::FPR32RegClass, Op0, Op1); 12352 } 12353 return 0; 12354} 12355 12356unsigned fastEmit_ISD_STRICT_FMINNUM_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12357 if (RetVT.SimpleTy != MVT::f64) 12358 return 0; 12359 if ((Subtarget->hasFPARMv8())) { 12360 return fastEmitInst_rr(AArch64::FMINNMDrr, &AArch64::FPR64RegClass, Op0, Op1); 12361 } 12362 return 0; 12363} 12364 12365unsigned fastEmit_ISD_STRICT_FMINNUM_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12366 if (RetVT.SimpleTy != MVT::v4f16) 12367 return 0; 12368 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 12369 return fastEmitInst_rr(AArch64::FMINNMv4f16, &AArch64::FPR64RegClass, Op0, Op1); 12370 } 12371 return 0; 12372} 12373 12374unsigned fastEmit_ISD_STRICT_FMINNUM_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12375 if (RetVT.SimpleTy != MVT::v8f16) 12376 return 0; 12377 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 12378 return fastEmitInst_rr(AArch64::FMINNMv8f16, &AArch64::FPR128RegClass, Op0, Op1); 12379 } 12380 return 0; 12381} 12382 12383unsigned fastEmit_ISD_STRICT_FMINNUM_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12384 if (RetVT.SimpleTy != MVT::v2f32) 12385 return 0; 12386 if ((Subtarget->hasNEON())) { 12387 return fastEmitInst_rr(AArch64::FMINNMv2f32, &AArch64::FPR64RegClass, Op0, Op1); 12388 } 12389 return 0; 12390} 12391 12392unsigned fastEmit_ISD_STRICT_FMINNUM_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12393 if (RetVT.SimpleTy != MVT::v4f32) 12394 return 0; 12395 if ((Subtarget->hasNEON())) { 12396 return fastEmitInst_rr(AArch64::FMINNMv4f32, &AArch64::FPR128RegClass, Op0, Op1); 12397 } 12398 return 0; 12399} 12400 12401unsigned fastEmit_ISD_STRICT_FMINNUM_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12402 if (RetVT.SimpleTy != MVT::v2f64) 12403 return 0; 12404 if ((Subtarget->hasNEON())) { 12405 return fastEmitInst_rr(AArch64::FMINNMv2f64, &AArch64::FPR128RegClass, Op0, Op1); 12406 } 12407 return 0; 12408} 12409 12410unsigned fastEmit_ISD_STRICT_FMINNUM_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 12411 switch (VT.SimpleTy) { 12412 case MVT::f16: return fastEmit_ISD_STRICT_FMINNUM_MVT_f16_rr(RetVT, Op0, Op1); 12413 case MVT::f32: return fastEmit_ISD_STRICT_FMINNUM_MVT_f32_rr(RetVT, Op0, Op1); 12414 case MVT::f64: return fastEmit_ISD_STRICT_FMINNUM_MVT_f64_rr(RetVT, Op0, Op1); 12415 case MVT::v4f16: return fastEmit_ISD_STRICT_FMINNUM_MVT_v4f16_rr(RetVT, Op0, Op1); 12416 case MVT::v8f16: return fastEmit_ISD_STRICT_FMINNUM_MVT_v8f16_rr(RetVT, Op0, Op1); 12417 case MVT::v2f32: return fastEmit_ISD_STRICT_FMINNUM_MVT_v2f32_rr(RetVT, Op0, Op1); 12418 case MVT::v4f32: return fastEmit_ISD_STRICT_FMINNUM_MVT_v4f32_rr(RetVT, Op0, Op1); 12419 case MVT::v2f64: return fastEmit_ISD_STRICT_FMINNUM_MVT_v2f64_rr(RetVT, Op0, Op1); 12420 default: return 0; 12421 } 12422} 12423 12424// FastEmit functions for ISD::STRICT_FMUL. 12425 12426unsigned fastEmit_ISD_STRICT_FMUL_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12427 if (RetVT.SimpleTy != MVT::f16) 12428 return 0; 12429 if ((Subtarget->hasFullFP16())) { 12430 return fastEmitInst_rr(AArch64::FMULHrr, &AArch64::FPR16RegClass, Op0, Op1); 12431 } 12432 return 0; 12433} 12434 12435unsigned fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12436 if (RetVT.SimpleTy != MVT::f32) 12437 return 0; 12438 if ((Subtarget->hasFPARMv8())) { 12439 return fastEmitInst_rr(AArch64::FMULSrr, &AArch64::FPR32RegClass, Op0, Op1); 12440 } 12441 return 0; 12442} 12443 12444unsigned fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12445 if (RetVT.SimpleTy != MVT::f64) 12446 return 0; 12447 if ((Subtarget->hasFPARMv8())) { 12448 return fastEmitInst_rr(AArch64::FMULDrr, &AArch64::FPR64RegClass, Op0, Op1); 12449 } 12450 return 0; 12451} 12452 12453unsigned fastEmit_ISD_STRICT_FMUL_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12454 if (RetVT.SimpleTy != MVT::v4f16) 12455 return 0; 12456 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 12457 return fastEmitInst_rr(AArch64::FMULv4f16, &AArch64::FPR64RegClass, Op0, Op1); 12458 } 12459 return 0; 12460} 12461 12462unsigned fastEmit_ISD_STRICT_FMUL_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12463 if (RetVT.SimpleTy != MVT::v8f16) 12464 return 0; 12465 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 12466 return fastEmitInst_rr(AArch64::FMULv8f16, &AArch64::FPR128RegClass, Op0, Op1); 12467 } 12468 return 0; 12469} 12470 12471unsigned fastEmit_ISD_STRICT_FMUL_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12472 if (RetVT.SimpleTy != MVT::v2f32) 12473 return 0; 12474 if ((Subtarget->hasNEON())) { 12475 return fastEmitInst_rr(AArch64::FMULv2f32, &AArch64::FPR64RegClass, Op0, Op1); 12476 } 12477 return 0; 12478} 12479 12480unsigned fastEmit_ISD_STRICT_FMUL_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12481 if (RetVT.SimpleTy != MVT::v4f32) 12482 return 0; 12483 if ((Subtarget->hasNEON())) { 12484 return fastEmitInst_rr(AArch64::FMULv4f32, &AArch64::FPR128RegClass, Op0, Op1); 12485 } 12486 return 0; 12487} 12488 12489unsigned fastEmit_ISD_STRICT_FMUL_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12490 if (RetVT.SimpleTy != MVT::v2f64) 12491 return 0; 12492 if ((Subtarget->hasNEON())) { 12493 return fastEmitInst_rr(AArch64::FMULv2f64, &AArch64::FPR128RegClass, Op0, Op1); 12494 } 12495 return 0; 12496} 12497 12498unsigned fastEmit_ISD_STRICT_FMUL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 12499 switch (VT.SimpleTy) { 12500 case MVT::f16: return fastEmit_ISD_STRICT_FMUL_MVT_f16_rr(RetVT, Op0, Op1); 12501 case MVT::f32: return fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(RetVT, Op0, Op1); 12502 case MVT::f64: return fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(RetVT, Op0, Op1); 12503 case MVT::v4f16: return fastEmit_ISD_STRICT_FMUL_MVT_v4f16_rr(RetVT, Op0, Op1); 12504 case MVT::v8f16: return fastEmit_ISD_STRICT_FMUL_MVT_v8f16_rr(RetVT, Op0, Op1); 12505 case MVT::v2f32: return fastEmit_ISD_STRICT_FMUL_MVT_v2f32_rr(RetVT, Op0, Op1); 12506 case MVT::v4f32: return fastEmit_ISD_STRICT_FMUL_MVT_v4f32_rr(RetVT, Op0, Op1); 12507 case MVT::v2f64: return fastEmit_ISD_STRICT_FMUL_MVT_v2f64_rr(RetVT, Op0, Op1); 12508 default: return 0; 12509 } 12510} 12511 12512// FastEmit functions for ISD::STRICT_FSUB. 12513 12514unsigned fastEmit_ISD_STRICT_FSUB_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12515 if (RetVT.SimpleTy != MVT::f16) 12516 return 0; 12517 if ((Subtarget->hasFullFP16())) { 12518 return fastEmitInst_rr(AArch64::FSUBHrr, &AArch64::FPR16RegClass, Op0, Op1); 12519 } 12520 return 0; 12521} 12522 12523unsigned fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12524 if (RetVT.SimpleTy != MVT::f32) 12525 return 0; 12526 if ((Subtarget->hasFPARMv8())) { 12527 return fastEmitInst_rr(AArch64::FSUBSrr, &AArch64::FPR32RegClass, Op0, Op1); 12528 } 12529 return 0; 12530} 12531 12532unsigned fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12533 if (RetVT.SimpleTy != MVT::f64) 12534 return 0; 12535 if ((Subtarget->hasFPARMv8())) { 12536 return fastEmitInst_rr(AArch64::FSUBDrr, &AArch64::FPR64RegClass, Op0, Op1); 12537 } 12538 return 0; 12539} 12540 12541unsigned fastEmit_ISD_STRICT_FSUB_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12542 if (RetVT.SimpleTy != MVT::v4f16) 12543 return 0; 12544 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 12545 return fastEmitInst_rr(AArch64::FSUBv4f16, &AArch64::FPR64RegClass, Op0, Op1); 12546 } 12547 return 0; 12548} 12549 12550unsigned fastEmit_ISD_STRICT_FSUB_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12551 if (RetVT.SimpleTy != MVT::v8f16) 12552 return 0; 12553 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 12554 return fastEmitInst_rr(AArch64::FSUBv8f16, &AArch64::FPR128RegClass, Op0, Op1); 12555 } 12556 return 0; 12557} 12558 12559unsigned fastEmit_ISD_STRICT_FSUB_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12560 if (RetVT.SimpleTy != MVT::v2f32) 12561 return 0; 12562 if ((Subtarget->hasNEON())) { 12563 return fastEmitInst_rr(AArch64::FSUBv2f32, &AArch64::FPR64RegClass, Op0, Op1); 12564 } 12565 return 0; 12566} 12567 12568unsigned fastEmit_ISD_STRICT_FSUB_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12569 if (RetVT.SimpleTy != MVT::v4f32) 12570 return 0; 12571 if ((Subtarget->hasNEON())) { 12572 return fastEmitInst_rr(AArch64::FSUBv4f32, &AArch64::FPR128RegClass, Op0, Op1); 12573 } 12574 return 0; 12575} 12576 12577unsigned fastEmit_ISD_STRICT_FSUB_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12578 if (RetVT.SimpleTy != MVT::v2f64) 12579 return 0; 12580 if ((Subtarget->hasNEON())) { 12581 return fastEmitInst_rr(AArch64::FSUBv2f64, &AArch64::FPR128RegClass, Op0, Op1); 12582 } 12583 return 0; 12584} 12585 12586unsigned fastEmit_ISD_STRICT_FSUB_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 12587 switch (VT.SimpleTy) { 12588 case MVT::f16: return fastEmit_ISD_STRICT_FSUB_MVT_f16_rr(RetVT, Op0, Op1); 12589 case MVT::f32: return fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(RetVT, Op0, Op1); 12590 case MVT::f64: return fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(RetVT, Op0, Op1); 12591 case MVT::v4f16: return fastEmit_ISD_STRICT_FSUB_MVT_v4f16_rr(RetVT, Op0, Op1); 12592 case MVT::v8f16: return fastEmit_ISD_STRICT_FSUB_MVT_v8f16_rr(RetVT, Op0, Op1); 12593 case MVT::v2f32: return fastEmit_ISD_STRICT_FSUB_MVT_v2f32_rr(RetVT, Op0, Op1); 12594 case MVT::v4f32: return fastEmit_ISD_STRICT_FSUB_MVT_v4f32_rr(RetVT, Op0, Op1); 12595 case MVT::v2f64: return fastEmit_ISD_STRICT_FSUB_MVT_v2f64_rr(RetVT, Op0, Op1); 12596 default: return 0; 12597 } 12598} 12599 12600// FastEmit functions for ISD::SUB. 12601 12602unsigned fastEmit_ISD_SUB_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12603 if (RetVT.SimpleTy != MVT::i32) 12604 return 0; 12605 return fastEmitInst_rr(AArch64::SUBSWrr, &AArch64::GPR32RegClass, Op0, Op1); 12606} 12607 12608unsigned fastEmit_ISD_SUB_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12609 if (RetVT.SimpleTy != MVT::i64) 12610 return 0; 12611 return fastEmitInst_rr(AArch64::SUBSXrr, &AArch64::GPR64RegClass, Op0, Op1); 12612} 12613 12614unsigned fastEmit_ISD_SUB_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12615 if (RetVT.SimpleTy != MVT::v8i8) 12616 return 0; 12617 if ((Subtarget->hasNEON())) { 12618 return fastEmitInst_rr(AArch64::SUBv8i8, &AArch64::FPR64RegClass, Op0, Op1); 12619 } 12620 return 0; 12621} 12622 12623unsigned fastEmit_ISD_SUB_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12624 if (RetVT.SimpleTy != MVT::v16i8) 12625 return 0; 12626 if ((Subtarget->hasNEON())) { 12627 return fastEmitInst_rr(AArch64::SUBv16i8, &AArch64::FPR128RegClass, Op0, Op1); 12628 } 12629 return 0; 12630} 12631 12632unsigned fastEmit_ISD_SUB_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12633 if (RetVT.SimpleTy != MVT::v4i16) 12634 return 0; 12635 if ((Subtarget->hasNEON())) { 12636 return fastEmitInst_rr(AArch64::SUBv4i16, &AArch64::FPR64RegClass, Op0, Op1); 12637 } 12638 return 0; 12639} 12640 12641unsigned fastEmit_ISD_SUB_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12642 if (RetVT.SimpleTy != MVT::v8i16) 12643 return 0; 12644 if ((Subtarget->hasNEON())) { 12645 return fastEmitInst_rr(AArch64::SUBv8i16, &AArch64::FPR128RegClass, Op0, Op1); 12646 } 12647 return 0; 12648} 12649 12650unsigned fastEmit_ISD_SUB_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12651 if (RetVT.SimpleTy != MVT::v2i32) 12652 return 0; 12653 if ((Subtarget->hasNEON())) { 12654 return fastEmitInst_rr(AArch64::SUBv2i32, &AArch64::FPR64RegClass, Op0, Op1); 12655 } 12656 return 0; 12657} 12658 12659unsigned fastEmit_ISD_SUB_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12660 if (RetVT.SimpleTy != MVT::v4i32) 12661 return 0; 12662 if ((Subtarget->hasNEON())) { 12663 return fastEmitInst_rr(AArch64::SUBv4i32, &AArch64::FPR128RegClass, Op0, Op1); 12664 } 12665 return 0; 12666} 12667 12668unsigned fastEmit_ISD_SUB_MVT_v1i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12669 if (RetVT.SimpleTy != MVT::v1i64) 12670 return 0; 12671 if ((Subtarget->hasNEON())) { 12672 return fastEmitInst_rr(AArch64::SUBv1i64, &AArch64::FPR64RegClass, Op0, Op1); 12673 } 12674 return 0; 12675} 12676 12677unsigned fastEmit_ISD_SUB_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12678 if (RetVT.SimpleTy != MVT::v2i64) 12679 return 0; 12680 if ((Subtarget->hasNEON())) { 12681 return fastEmitInst_rr(AArch64::SUBv2i64, &AArch64::FPR128RegClass, Op0, Op1); 12682 } 12683 return 0; 12684} 12685 12686unsigned fastEmit_ISD_SUB_MVT_nxv16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12687 if (RetVT.SimpleTy != MVT::nxv16i8) 12688 return 0; 12689 if ((Subtarget->hasSVEorSME())) { 12690 return fastEmitInst_rr(AArch64::SUB_ZZZ_B, &AArch64::ZPRRegClass, Op0, Op1); 12691 } 12692 return 0; 12693} 12694 12695unsigned fastEmit_ISD_SUB_MVT_nxv8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12696 if (RetVT.SimpleTy != MVT::nxv8i16) 12697 return 0; 12698 if ((Subtarget->hasSVEorSME())) { 12699 return fastEmitInst_rr(AArch64::SUB_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op1); 12700 } 12701 return 0; 12702} 12703 12704unsigned fastEmit_ISD_SUB_MVT_nxv4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12705 if (RetVT.SimpleTy != MVT::nxv4i32) 12706 return 0; 12707 if ((Subtarget->hasSVEorSME())) { 12708 return fastEmitInst_rr(AArch64::SUB_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op1); 12709 } 12710 return 0; 12711} 12712 12713unsigned fastEmit_ISD_SUB_MVT_nxv2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12714 if (RetVT.SimpleTy != MVT::nxv2i64) 12715 return 0; 12716 if ((Subtarget->hasSVEorSME())) { 12717 return fastEmitInst_rr(AArch64::SUB_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op1); 12718 } 12719 return 0; 12720} 12721 12722unsigned fastEmit_ISD_SUB_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 12723 switch (VT.SimpleTy) { 12724 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_rr(RetVT, Op0, Op1); 12725 case MVT::i64: return fastEmit_ISD_SUB_MVT_i64_rr(RetVT, Op0, Op1); 12726 case MVT::v8i8: return fastEmit_ISD_SUB_MVT_v8i8_rr(RetVT, Op0, Op1); 12727 case MVT::v16i8: return fastEmit_ISD_SUB_MVT_v16i8_rr(RetVT, Op0, Op1); 12728 case MVT::v4i16: return fastEmit_ISD_SUB_MVT_v4i16_rr(RetVT, Op0, Op1); 12729 case MVT::v8i16: return fastEmit_ISD_SUB_MVT_v8i16_rr(RetVT, Op0, Op1); 12730 case MVT::v2i32: return fastEmit_ISD_SUB_MVT_v2i32_rr(RetVT, Op0, Op1); 12731 case MVT::v4i32: return fastEmit_ISD_SUB_MVT_v4i32_rr(RetVT, Op0, Op1); 12732 case MVT::v1i64: return fastEmit_ISD_SUB_MVT_v1i64_rr(RetVT, Op0, Op1); 12733 case MVT::v2i64: return fastEmit_ISD_SUB_MVT_v2i64_rr(RetVT, Op0, Op1); 12734 case MVT::nxv16i8: return fastEmit_ISD_SUB_MVT_nxv16i8_rr(RetVT, Op0, Op1); 12735 case MVT::nxv8i16: return fastEmit_ISD_SUB_MVT_nxv8i16_rr(RetVT, Op0, Op1); 12736 case MVT::nxv4i32: return fastEmit_ISD_SUB_MVT_nxv4i32_rr(RetVT, Op0, Op1); 12737 case MVT::nxv2i64: return fastEmit_ISD_SUB_MVT_nxv2i64_rr(RetVT, Op0, Op1); 12738 default: return 0; 12739 } 12740} 12741 12742// FastEmit functions for ISD::UADDSAT. 12743 12744unsigned fastEmit_ISD_UADDSAT_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12745 if (RetVT.SimpleTy != MVT::v8i8) 12746 return 0; 12747 if ((Subtarget->hasNEON())) { 12748 return fastEmitInst_rr(AArch64::UQADDv8i8, &AArch64::FPR64RegClass, Op0, Op1); 12749 } 12750 return 0; 12751} 12752 12753unsigned fastEmit_ISD_UADDSAT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12754 if (RetVT.SimpleTy != MVT::v16i8) 12755 return 0; 12756 if ((Subtarget->hasNEON())) { 12757 return fastEmitInst_rr(AArch64::UQADDv16i8, &AArch64::FPR128RegClass, Op0, Op1); 12758 } 12759 return 0; 12760} 12761 12762unsigned fastEmit_ISD_UADDSAT_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12763 if (RetVT.SimpleTy != MVT::v4i16) 12764 return 0; 12765 if ((Subtarget->hasNEON())) { 12766 return fastEmitInst_rr(AArch64::UQADDv4i16, &AArch64::FPR64RegClass, Op0, Op1); 12767 } 12768 return 0; 12769} 12770 12771unsigned fastEmit_ISD_UADDSAT_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12772 if (RetVT.SimpleTy != MVT::v8i16) 12773 return 0; 12774 if ((Subtarget->hasNEON())) { 12775 return fastEmitInst_rr(AArch64::UQADDv8i16, &AArch64::FPR128RegClass, Op0, Op1); 12776 } 12777 return 0; 12778} 12779 12780unsigned fastEmit_ISD_UADDSAT_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12781 if (RetVT.SimpleTy != MVT::v2i32) 12782 return 0; 12783 if ((Subtarget->hasNEON())) { 12784 return fastEmitInst_rr(AArch64::UQADDv2i32, &AArch64::FPR64RegClass, Op0, Op1); 12785 } 12786 return 0; 12787} 12788 12789unsigned fastEmit_ISD_UADDSAT_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12790 if (RetVT.SimpleTy != MVT::v4i32) 12791 return 0; 12792 if ((Subtarget->hasNEON())) { 12793 return fastEmitInst_rr(AArch64::UQADDv4i32, &AArch64::FPR128RegClass, Op0, Op1); 12794 } 12795 return 0; 12796} 12797 12798unsigned fastEmit_ISD_UADDSAT_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12799 if (RetVT.SimpleTy != MVT::v2i64) 12800 return 0; 12801 if ((Subtarget->hasNEON())) { 12802 return fastEmitInst_rr(AArch64::UQADDv2i64, &AArch64::FPR128RegClass, Op0, Op1); 12803 } 12804 return 0; 12805} 12806 12807unsigned fastEmit_ISD_UADDSAT_MVT_nxv16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12808 if (RetVT.SimpleTy != MVT::nxv16i8) 12809 return 0; 12810 if ((Subtarget->hasSVEorSME())) { 12811 return fastEmitInst_rr(AArch64::UQADD_ZZZ_B, &AArch64::ZPRRegClass, Op0, Op1); 12812 } 12813 return 0; 12814} 12815 12816unsigned fastEmit_ISD_UADDSAT_MVT_nxv8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12817 if (RetVT.SimpleTy != MVT::nxv8i16) 12818 return 0; 12819 if ((Subtarget->hasSVEorSME())) { 12820 return fastEmitInst_rr(AArch64::UQADD_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op1); 12821 } 12822 return 0; 12823} 12824 12825unsigned fastEmit_ISD_UADDSAT_MVT_nxv4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12826 if (RetVT.SimpleTy != MVT::nxv4i32) 12827 return 0; 12828 if ((Subtarget->hasSVEorSME())) { 12829 return fastEmitInst_rr(AArch64::UQADD_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op1); 12830 } 12831 return 0; 12832} 12833 12834unsigned fastEmit_ISD_UADDSAT_MVT_nxv2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12835 if (RetVT.SimpleTy != MVT::nxv2i64) 12836 return 0; 12837 if ((Subtarget->hasSVEorSME())) { 12838 return fastEmitInst_rr(AArch64::UQADD_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op1); 12839 } 12840 return 0; 12841} 12842 12843unsigned fastEmit_ISD_UADDSAT_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 12844 switch (VT.SimpleTy) { 12845 case MVT::v8i8: return fastEmit_ISD_UADDSAT_MVT_v8i8_rr(RetVT, Op0, Op1); 12846 case MVT::v16i8: return fastEmit_ISD_UADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1); 12847 case MVT::v4i16: return fastEmit_ISD_UADDSAT_MVT_v4i16_rr(RetVT, Op0, Op1); 12848 case MVT::v8i16: return fastEmit_ISD_UADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1); 12849 case MVT::v2i32: return fastEmit_ISD_UADDSAT_MVT_v2i32_rr(RetVT, Op0, Op1); 12850 case MVT::v4i32: return fastEmit_ISD_UADDSAT_MVT_v4i32_rr(RetVT, Op0, Op1); 12851 case MVT::v2i64: return fastEmit_ISD_UADDSAT_MVT_v2i64_rr(RetVT, Op0, Op1); 12852 case MVT::nxv16i8: return fastEmit_ISD_UADDSAT_MVT_nxv16i8_rr(RetVT, Op0, Op1); 12853 case MVT::nxv8i16: return fastEmit_ISD_UADDSAT_MVT_nxv8i16_rr(RetVT, Op0, Op1); 12854 case MVT::nxv4i32: return fastEmit_ISD_UADDSAT_MVT_nxv4i32_rr(RetVT, Op0, Op1); 12855 case MVT::nxv2i64: return fastEmit_ISD_UADDSAT_MVT_nxv2i64_rr(RetVT, Op0, Op1); 12856 default: return 0; 12857 } 12858} 12859 12860// FastEmit functions for ISD::UDIV. 12861 12862unsigned fastEmit_ISD_UDIV_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12863 if (RetVT.SimpleTy != MVT::i32) 12864 return 0; 12865 return fastEmitInst_rr(AArch64::UDIVWr, &AArch64::GPR32RegClass, Op0, Op1); 12866} 12867 12868unsigned fastEmit_ISD_UDIV_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12869 if (RetVT.SimpleTy != MVT::i64) 12870 return 0; 12871 return fastEmitInst_rr(AArch64::UDIVXr, &AArch64::GPR64RegClass, Op0, Op1); 12872} 12873 12874unsigned fastEmit_ISD_UDIV_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 12875 switch (VT.SimpleTy) { 12876 case MVT::i32: return fastEmit_ISD_UDIV_MVT_i32_rr(RetVT, Op0, Op1); 12877 case MVT::i64: return fastEmit_ISD_UDIV_MVT_i64_rr(RetVT, Op0, Op1); 12878 default: return 0; 12879 } 12880} 12881 12882// FastEmit functions for ISD::UMAX. 12883 12884unsigned fastEmit_ISD_UMAX_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12885 if (RetVT.SimpleTy != MVT::i32) 12886 return 0; 12887 if ((Subtarget->hasCSSC())) { 12888 return fastEmitInst_rr(AArch64::UMAXWrr, &AArch64::GPR32RegClass, Op0, Op1); 12889 } 12890 return 0; 12891} 12892 12893unsigned fastEmit_ISD_UMAX_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12894 if (RetVT.SimpleTy != MVT::i64) 12895 return 0; 12896 if ((Subtarget->hasCSSC())) { 12897 return fastEmitInst_rr(AArch64::UMAXXrr, &AArch64::GPR64RegClass, Op0, Op1); 12898 } 12899 return 0; 12900} 12901 12902unsigned fastEmit_ISD_UMAX_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12903 if (RetVT.SimpleTy != MVT::v8i8) 12904 return 0; 12905 if ((Subtarget->hasNEON())) { 12906 return fastEmitInst_rr(AArch64::UMAXv8i8, &AArch64::FPR64RegClass, Op0, Op1); 12907 } 12908 return 0; 12909} 12910 12911unsigned fastEmit_ISD_UMAX_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12912 if (RetVT.SimpleTy != MVT::v16i8) 12913 return 0; 12914 if ((Subtarget->hasNEON())) { 12915 return fastEmitInst_rr(AArch64::UMAXv16i8, &AArch64::FPR128RegClass, Op0, Op1); 12916 } 12917 return 0; 12918} 12919 12920unsigned fastEmit_ISD_UMAX_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12921 if (RetVT.SimpleTy != MVT::v4i16) 12922 return 0; 12923 if ((Subtarget->hasNEON())) { 12924 return fastEmitInst_rr(AArch64::UMAXv4i16, &AArch64::FPR64RegClass, Op0, Op1); 12925 } 12926 return 0; 12927} 12928 12929unsigned fastEmit_ISD_UMAX_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12930 if (RetVT.SimpleTy != MVT::v8i16) 12931 return 0; 12932 if ((Subtarget->hasNEON())) { 12933 return fastEmitInst_rr(AArch64::UMAXv8i16, &AArch64::FPR128RegClass, Op0, Op1); 12934 } 12935 return 0; 12936} 12937 12938unsigned fastEmit_ISD_UMAX_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12939 if (RetVT.SimpleTy != MVT::v2i32) 12940 return 0; 12941 if ((Subtarget->hasNEON())) { 12942 return fastEmitInst_rr(AArch64::UMAXv2i32, &AArch64::FPR64RegClass, Op0, Op1); 12943 } 12944 return 0; 12945} 12946 12947unsigned fastEmit_ISD_UMAX_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12948 if (RetVT.SimpleTy != MVT::v4i32) 12949 return 0; 12950 if ((Subtarget->hasNEON())) { 12951 return fastEmitInst_rr(AArch64::UMAXv4i32, &AArch64::FPR128RegClass, Op0, Op1); 12952 } 12953 return 0; 12954} 12955 12956unsigned fastEmit_ISD_UMAX_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 12957 switch (VT.SimpleTy) { 12958 case MVT::i32: return fastEmit_ISD_UMAX_MVT_i32_rr(RetVT, Op0, Op1); 12959 case MVT::i64: return fastEmit_ISD_UMAX_MVT_i64_rr(RetVT, Op0, Op1); 12960 case MVT::v8i8: return fastEmit_ISD_UMAX_MVT_v8i8_rr(RetVT, Op0, Op1); 12961 case MVT::v16i8: return fastEmit_ISD_UMAX_MVT_v16i8_rr(RetVT, Op0, Op1); 12962 case MVT::v4i16: return fastEmit_ISD_UMAX_MVT_v4i16_rr(RetVT, Op0, Op1); 12963 case MVT::v8i16: return fastEmit_ISD_UMAX_MVT_v8i16_rr(RetVT, Op0, Op1); 12964 case MVT::v2i32: return fastEmit_ISD_UMAX_MVT_v2i32_rr(RetVT, Op0, Op1); 12965 case MVT::v4i32: return fastEmit_ISD_UMAX_MVT_v4i32_rr(RetVT, Op0, Op1); 12966 default: return 0; 12967 } 12968} 12969 12970// FastEmit functions for ISD::UMIN. 12971 12972unsigned fastEmit_ISD_UMIN_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12973 if (RetVT.SimpleTy != MVT::i32) 12974 return 0; 12975 if ((Subtarget->hasCSSC())) { 12976 return fastEmitInst_rr(AArch64::UMINWrr, &AArch64::GPR32RegClass, Op0, Op1); 12977 } 12978 return 0; 12979} 12980 12981unsigned fastEmit_ISD_UMIN_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12982 if (RetVT.SimpleTy != MVT::i64) 12983 return 0; 12984 if ((Subtarget->hasCSSC())) { 12985 return fastEmitInst_rr(AArch64::UMINXrr, &AArch64::GPR64RegClass, Op0, Op1); 12986 } 12987 return 0; 12988} 12989 12990unsigned fastEmit_ISD_UMIN_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 12991 if (RetVT.SimpleTy != MVT::v8i8) 12992 return 0; 12993 if ((Subtarget->hasNEON())) { 12994 return fastEmitInst_rr(AArch64::UMINv8i8, &AArch64::FPR64RegClass, Op0, Op1); 12995 } 12996 return 0; 12997} 12998 12999unsigned fastEmit_ISD_UMIN_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13000 if (RetVT.SimpleTy != MVT::v16i8) 13001 return 0; 13002 if ((Subtarget->hasNEON())) { 13003 return fastEmitInst_rr(AArch64::UMINv16i8, &AArch64::FPR128RegClass, Op0, Op1); 13004 } 13005 return 0; 13006} 13007 13008unsigned fastEmit_ISD_UMIN_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13009 if (RetVT.SimpleTy != MVT::v4i16) 13010 return 0; 13011 if ((Subtarget->hasNEON())) { 13012 return fastEmitInst_rr(AArch64::UMINv4i16, &AArch64::FPR64RegClass, Op0, Op1); 13013 } 13014 return 0; 13015} 13016 13017unsigned fastEmit_ISD_UMIN_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13018 if (RetVT.SimpleTy != MVT::v8i16) 13019 return 0; 13020 if ((Subtarget->hasNEON())) { 13021 return fastEmitInst_rr(AArch64::UMINv8i16, &AArch64::FPR128RegClass, Op0, Op1); 13022 } 13023 return 0; 13024} 13025 13026unsigned fastEmit_ISD_UMIN_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13027 if (RetVT.SimpleTy != MVT::v2i32) 13028 return 0; 13029 if ((Subtarget->hasNEON())) { 13030 return fastEmitInst_rr(AArch64::UMINv2i32, &AArch64::FPR64RegClass, Op0, Op1); 13031 } 13032 return 0; 13033} 13034 13035unsigned fastEmit_ISD_UMIN_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13036 if (RetVT.SimpleTy != MVT::v4i32) 13037 return 0; 13038 if ((Subtarget->hasNEON())) { 13039 return fastEmitInst_rr(AArch64::UMINv4i32, &AArch64::FPR128RegClass, Op0, Op1); 13040 } 13041 return 0; 13042} 13043 13044unsigned fastEmit_ISD_UMIN_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 13045 switch (VT.SimpleTy) { 13046 case MVT::i32: return fastEmit_ISD_UMIN_MVT_i32_rr(RetVT, Op0, Op1); 13047 case MVT::i64: return fastEmit_ISD_UMIN_MVT_i64_rr(RetVT, Op0, Op1); 13048 case MVT::v8i8: return fastEmit_ISD_UMIN_MVT_v8i8_rr(RetVT, Op0, Op1); 13049 case MVT::v16i8: return fastEmit_ISD_UMIN_MVT_v16i8_rr(RetVT, Op0, Op1); 13050 case MVT::v4i16: return fastEmit_ISD_UMIN_MVT_v4i16_rr(RetVT, Op0, Op1); 13051 case MVT::v8i16: return fastEmit_ISD_UMIN_MVT_v8i16_rr(RetVT, Op0, Op1); 13052 case MVT::v2i32: return fastEmit_ISD_UMIN_MVT_v2i32_rr(RetVT, Op0, Op1); 13053 case MVT::v4i32: return fastEmit_ISD_UMIN_MVT_v4i32_rr(RetVT, Op0, Op1); 13054 default: return 0; 13055 } 13056} 13057 13058// FastEmit functions for ISD::USUBSAT. 13059 13060unsigned fastEmit_ISD_USUBSAT_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13061 if (RetVT.SimpleTy != MVT::v8i8) 13062 return 0; 13063 if ((Subtarget->hasNEON())) { 13064 return fastEmitInst_rr(AArch64::UQSUBv8i8, &AArch64::FPR64RegClass, Op0, Op1); 13065 } 13066 return 0; 13067} 13068 13069unsigned fastEmit_ISD_USUBSAT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13070 if (RetVT.SimpleTy != MVT::v16i8) 13071 return 0; 13072 if ((Subtarget->hasNEON())) { 13073 return fastEmitInst_rr(AArch64::UQSUBv16i8, &AArch64::FPR128RegClass, Op0, Op1); 13074 } 13075 return 0; 13076} 13077 13078unsigned fastEmit_ISD_USUBSAT_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13079 if (RetVT.SimpleTy != MVT::v4i16) 13080 return 0; 13081 if ((Subtarget->hasNEON())) { 13082 return fastEmitInst_rr(AArch64::UQSUBv4i16, &AArch64::FPR64RegClass, Op0, Op1); 13083 } 13084 return 0; 13085} 13086 13087unsigned fastEmit_ISD_USUBSAT_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13088 if (RetVT.SimpleTy != MVT::v8i16) 13089 return 0; 13090 if ((Subtarget->hasNEON())) { 13091 return fastEmitInst_rr(AArch64::UQSUBv8i16, &AArch64::FPR128RegClass, Op0, Op1); 13092 } 13093 return 0; 13094} 13095 13096unsigned fastEmit_ISD_USUBSAT_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13097 if (RetVT.SimpleTy != MVT::v2i32) 13098 return 0; 13099 if ((Subtarget->hasNEON())) { 13100 return fastEmitInst_rr(AArch64::UQSUBv2i32, &AArch64::FPR64RegClass, Op0, Op1); 13101 } 13102 return 0; 13103} 13104 13105unsigned fastEmit_ISD_USUBSAT_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13106 if (RetVT.SimpleTy != MVT::v4i32) 13107 return 0; 13108 if ((Subtarget->hasNEON())) { 13109 return fastEmitInst_rr(AArch64::UQSUBv4i32, &AArch64::FPR128RegClass, Op0, Op1); 13110 } 13111 return 0; 13112} 13113 13114unsigned fastEmit_ISD_USUBSAT_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13115 if (RetVT.SimpleTy != MVT::v2i64) 13116 return 0; 13117 if ((Subtarget->hasNEON())) { 13118 return fastEmitInst_rr(AArch64::UQSUBv2i64, &AArch64::FPR128RegClass, Op0, Op1); 13119 } 13120 return 0; 13121} 13122 13123unsigned fastEmit_ISD_USUBSAT_MVT_nxv16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13124 if (RetVT.SimpleTy != MVT::nxv16i8) 13125 return 0; 13126 if ((Subtarget->hasSVEorSME())) { 13127 return fastEmitInst_rr(AArch64::UQSUB_ZZZ_B, &AArch64::ZPRRegClass, Op0, Op1); 13128 } 13129 return 0; 13130} 13131 13132unsigned fastEmit_ISD_USUBSAT_MVT_nxv8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13133 if (RetVT.SimpleTy != MVT::nxv8i16) 13134 return 0; 13135 if ((Subtarget->hasSVEorSME())) { 13136 return fastEmitInst_rr(AArch64::UQSUB_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op1); 13137 } 13138 return 0; 13139} 13140 13141unsigned fastEmit_ISD_USUBSAT_MVT_nxv4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13142 if (RetVT.SimpleTy != MVT::nxv4i32) 13143 return 0; 13144 if ((Subtarget->hasSVEorSME())) { 13145 return fastEmitInst_rr(AArch64::UQSUB_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op1); 13146 } 13147 return 0; 13148} 13149 13150unsigned fastEmit_ISD_USUBSAT_MVT_nxv2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13151 if (RetVT.SimpleTy != MVT::nxv2i64) 13152 return 0; 13153 if ((Subtarget->hasSVEorSME())) { 13154 return fastEmitInst_rr(AArch64::UQSUB_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op1); 13155 } 13156 return 0; 13157} 13158 13159unsigned fastEmit_ISD_USUBSAT_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 13160 switch (VT.SimpleTy) { 13161 case MVT::v8i8: return fastEmit_ISD_USUBSAT_MVT_v8i8_rr(RetVT, Op0, Op1); 13162 case MVT::v16i8: return fastEmit_ISD_USUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1); 13163 case MVT::v4i16: return fastEmit_ISD_USUBSAT_MVT_v4i16_rr(RetVT, Op0, Op1); 13164 case MVT::v8i16: return fastEmit_ISD_USUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1); 13165 case MVT::v2i32: return fastEmit_ISD_USUBSAT_MVT_v2i32_rr(RetVT, Op0, Op1); 13166 case MVT::v4i32: return fastEmit_ISD_USUBSAT_MVT_v4i32_rr(RetVT, Op0, Op1); 13167 case MVT::v2i64: return fastEmit_ISD_USUBSAT_MVT_v2i64_rr(RetVT, Op0, Op1); 13168 case MVT::nxv16i8: return fastEmit_ISD_USUBSAT_MVT_nxv16i8_rr(RetVT, Op0, Op1); 13169 case MVT::nxv8i16: return fastEmit_ISD_USUBSAT_MVT_nxv8i16_rr(RetVT, Op0, Op1); 13170 case MVT::nxv4i32: return fastEmit_ISD_USUBSAT_MVT_nxv4i32_rr(RetVT, Op0, Op1); 13171 case MVT::nxv2i64: return fastEmit_ISD_USUBSAT_MVT_nxv2i64_rr(RetVT, Op0, Op1); 13172 default: return 0; 13173 } 13174} 13175 13176// FastEmit functions for ISD::XOR. 13177 13178unsigned fastEmit_ISD_XOR_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13179 if (RetVT.SimpleTy != MVT::i32) 13180 return 0; 13181 return fastEmitInst_rr(AArch64::EORWrr, &AArch64::GPR32RegClass, Op0, Op1); 13182} 13183 13184unsigned fastEmit_ISD_XOR_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13185 if (RetVT.SimpleTy != MVT::i64) 13186 return 0; 13187 return fastEmitInst_rr(AArch64::EORXrr, &AArch64::GPR64RegClass, Op0, Op1); 13188} 13189 13190unsigned fastEmit_ISD_XOR_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13191 if (RetVT.SimpleTy != MVT::v8i8) 13192 return 0; 13193 if ((Subtarget->hasNEON())) { 13194 return fastEmitInst_rr(AArch64::EORv8i8, &AArch64::FPR64RegClass, Op0, Op1); 13195 } 13196 return 0; 13197} 13198 13199unsigned fastEmit_ISD_XOR_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13200 if (RetVT.SimpleTy != MVT::v16i8) 13201 return 0; 13202 if ((Subtarget->hasNEON())) { 13203 return fastEmitInst_rr(AArch64::EORv16i8, &AArch64::FPR128RegClass, Op0, Op1); 13204 } 13205 return 0; 13206} 13207 13208unsigned fastEmit_ISD_XOR_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13209 if (RetVT.SimpleTy != MVT::v4i16) 13210 return 0; 13211 if ((Subtarget->hasNEON())) { 13212 return fastEmitInst_rr(AArch64::EORv8i8, &AArch64::FPR64RegClass, Op0, Op1); 13213 } 13214 return 0; 13215} 13216 13217unsigned fastEmit_ISD_XOR_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13218 if (RetVT.SimpleTy != MVT::v8i16) 13219 return 0; 13220 if ((Subtarget->hasNEON())) { 13221 return fastEmitInst_rr(AArch64::EORv16i8, &AArch64::FPR128RegClass, Op0, Op1); 13222 } 13223 return 0; 13224} 13225 13226unsigned fastEmit_ISD_XOR_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13227 if (RetVT.SimpleTy != MVT::v2i32) 13228 return 0; 13229 if ((Subtarget->hasNEON())) { 13230 return fastEmitInst_rr(AArch64::EORv8i8, &AArch64::FPR64RegClass, Op0, Op1); 13231 } 13232 return 0; 13233} 13234 13235unsigned fastEmit_ISD_XOR_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13236 if (RetVT.SimpleTy != MVT::v4i32) 13237 return 0; 13238 if ((Subtarget->hasNEON())) { 13239 return fastEmitInst_rr(AArch64::EORv16i8, &AArch64::FPR128RegClass, Op0, Op1); 13240 } 13241 return 0; 13242} 13243 13244unsigned fastEmit_ISD_XOR_MVT_v1i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13245 if (RetVT.SimpleTy != MVT::v1i64) 13246 return 0; 13247 if ((Subtarget->hasNEON())) { 13248 return fastEmitInst_rr(AArch64::EORv8i8, &AArch64::FPR64RegClass, Op0, Op1); 13249 } 13250 return 0; 13251} 13252 13253unsigned fastEmit_ISD_XOR_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13254 if (RetVT.SimpleTy != MVT::v2i64) 13255 return 0; 13256 if ((Subtarget->hasNEON())) { 13257 return fastEmitInst_rr(AArch64::EORv16i8, &AArch64::FPR128RegClass, Op0, Op1); 13258 } 13259 return 0; 13260} 13261 13262unsigned fastEmit_ISD_XOR_MVT_nxv16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13263 if (RetVT.SimpleTy != MVT::nxv16i8) 13264 return 0; 13265 if ((Subtarget->hasSVEorSME())) { 13266 return fastEmitInst_rr(AArch64::EOR_ZZZ, &AArch64::ZPRRegClass, Op0, Op1); 13267 } 13268 return 0; 13269} 13270 13271unsigned fastEmit_ISD_XOR_MVT_nxv8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13272 if (RetVT.SimpleTy != MVT::nxv8i16) 13273 return 0; 13274 if ((Subtarget->hasSVEorSME())) { 13275 return fastEmitInst_rr(AArch64::EOR_ZZZ, &AArch64::ZPRRegClass, Op0, Op1); 13276 } 13277 return 0; 13278} 13279 13280unsigned fastEmit_ISD_XOR_MVT_nxv4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13281 if (RetVT.SimpleTy != MVT::nxv4i32) 13282 return 0; 13283 if ((Subtarget->hasSVEorSME())) { 13284 return fastEmitInst_rr(AArch64::EOR_ZZZ, &AArch64::ZPRRegClass, Op0, Op1); 13285 } 13286 return 0; 13287} 13288 13289unsigned fastEmit_ISD_XOR_MVT_nxv2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) { 13290 if (RetVT.SimpleTy != MVT::nxv2i64) 13291 return 0; 13292 if ((Subtarget->hasSVEorSME())) { 13293 return fastEmitInst_rr(AArch64::EOR_ZZZ, &AArch64::ZPRRegClass, Op0, Op1); 13294 } 13295 return 0; 13296} 13297 13298unsigned fastEmit_ISD_XOR_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) { 13299 switch (VT.SimpleTy) { 13300 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_rr(RetVT, Op0, Op1); 13301 case MVT::i64: return fastEmit_ISD_XOR_MVT_i64_rr(RetVT, Op0, Op1); 13302 case MVT::v8i8: return fastEmit_ISD_XOR_MVT_v8i8_rr(RetVT, Op0, Op1); 13303 case MVT::v16i8: return fastEmit_ISD_XOR_MVT_v16i8_rr(RetVT, Op0, Op1); 13304 case MVT::v4i16: return fastEmit_ISD_XOR_MVT_v4i16_rr(RetVT, Op0, Op1); 13305 case MVT::v8i16: return fastEmit_ISD_XOR_MVT_v8i16_rr(RetVT, Op0, Op1); 13306 case MVT::v2i32: return fastEmit_ISD_XOR_MVT_v2i32_rr(RetVT, Op0, Op1); 13307 case MVT::v4i32: return fastEmit_ISD_XOR_MVT_v4i32_rr(RetVT, Op0, Op1); 13308 case MVT::v1i64: return fastEmit_ISD_XOR_MVT_v1i64_rr(RetVT, Op0, Op1); 13309 case MVT::v2i64: return fastEmit_ISD_XOR_MVT_v2i64_rr(RetVT, Op0, Op1); 13310 case MVT::nxv16i8: return fastEmit_ISD_XOR_MVT_nxv16i8_rr(RetVT, Op0, Op1); 13311 case MVT::nxv8i16: return fastEmit_ISD_XOR_MVT_nxv8i16_rr(RetVT, Op0, Op1); 13312 case MVT::nxv4i32: return fastEmit_ISD_XOR_MVT_nxv4i32_rr(RetVT, Op0, Op1); 13313 case MVT::nxv2i64: return fastEmit_ISD_XOR_MVT_nxv2i64_rr(RetVT, Op0, Op1); 13314 default: return 0; 13315 } 13316} 13317 13318// Top-level FastEmit function. 13319 13320unsigned fastEmit_rr(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, unsigned Op1) override { 13321 switch (Opcode) { 13322 case AArch64ISD::ADDP: return fastEmit_AArch64ISD_ADDP_rr(VT, RetVT, Op0, Op1); 13323 case AArch64ISD::BIC: return fastEmit_AArch64ISD_BIC_rr(VT, RetVT, Op0, Op1); 13324 case AArch64ISD::CMEQ: return fastEmit_AArch64ISD_CMEQ_rr(VT, RetVT, Op0, Op1); 13325 case AArch64ISD::CMGE: return fastEmit_AArch64ISD_CMGE_rr(VT, RetVT, Op0, Op1); 13326 case AArch64ISD::CMGT: return fastEmit_AArch64ISD_CMGT_rr(VT, RetVT, Op0, Op1); 13327 case AArch64ISD::CMHI: return fastEmit_AArch64ISD_CMHI_rr(VT, RetVT, Op0, Op1); 13328 case AArch64ISD::CMHS: return fastEmit_AArch64ISD_CMHS_rr(VT, RetVT, Op0, Op1); 13329 case AArch64ISD::FCMEQ: return fastEmit_AArch64ISD_FCMEQ_rr(VT, RetVT, Op0, Op1); 13330 case AArch64ISD::FCMGE: return fastEmit_AArch64ISD_FCMGE_rr(VT, RetVT, Op0, Op1); 13331 case AArch64ISD::FCMGT: return fastEmit_AArch64ISD_FCMGT_rr(VT, RetVT, Op0, Op1); 13332 case AArch64ISD::FCMP: return fastEmit_AArch64ISD_FCMP_rr(VT, RetVT, Op0, Op1); 13333 case AArch64ISD::FRECPS: return fastEmit_AArch64ISD_FRECPS_rr(VT, RetVT, Op0, Op1); 13334 case AArch64ISD::FRSQRTS: return fastEmit_AArch64ISD_FRSQRTS_rr(VT, RetVT, Op0, Op1); 13335 case AArch64ISD::PMULL: return fastEmit_AArch64ISD_PMULL_rr(VT, RetVT, Op0, Op1); 13336 case AArch64ISD::PTEST: return fastEmit_AArch64ISD_PTEST_rr(VT, RetVT, Op0, Op1); 13337 case AArch64ISD::PTEST_ANY: return fastEmit_AArch64ISD_PTEST_ANY_rr(VT, RetVT, Op0, Op1); 13338 case AArch64ISD::SMULL: return fastEmit_AArch64ISD_SMULL_rr(VT, RetVT, Op0, Op1); 13339 case AArch64ISD::STRICT_FCMP: return fastEmit_AArch64ISD_STRICT_FCMP_rr(VT, RetVT, Op0, Op1); 13340 case AArch64ISD::STRICT_FCMPE: return fastEmit_AArch64ISD_STRICT_FCMPE_rr(VT, RetVT, Op0, Op1); 13341 case AArch64ISD::TBL: return fastEmit_AArch64ISD_TBL_rr(VT, RetVT, Op0, Op1); 13342 case AArch64ISD::TRN1: return fastEmit_AArch64ISD_TRN1_rr(VT, RetVT, Op0, Op1); 13343 case AArch64ISD::TRN2: return fastEmit_AArch64ISD_TRN2_rr(VT, RetVT, Op0, Op1); 13344 case AArch64ISD::UMULL: return fastEmit_AArch64ISD_UMULL_rr(VT, RetVT, Op0, Op1); 13345 case AArch64ISD::UZP1: return fastEmit_AArch64ISD_UZP1_rr(VT, RetVT, Op0, Op1); 13346 case AArch64ISD::UZP2: return fastEmit_AArch64ISD_UZP2_rr(VT, RetVT, Op0, Op1); 13347 case AArch64ISD::ZIP1: return fastEmit_AArch64ISD_ZIP1_rr(VT, RetVT, Op0, Op1); 13348 case AArch64ISD::ZIP2: return fastEmit_AArch64ISD_ZIP2_rr(VT, RetVT, Op0, Op1); 13349 case ISD::ABDS: return fastEmit_ISD_ABDS_rr(VT, RetVT, Op0, Op1); 13350 case ISD::ABDU: return fastEmit_ISD_ABDU_rr(VT, RetVT, Op0, Op1); 13351 case ISD::ADD: return fastEmit_ISD_ADD_rr(VT, RetVT, Op0, Op1); 13352 case ISD::AND: return fastEmit_ISD_AND_rr(VT, RetVT, Op0, Op1); 13353 case ISD::AVGCEILS: return fastEmit_ISD_AVGCEILS_rr(VT, RetVT, Op0, Op1); 13354 case ISD::AVGCEILU: return fastEmit_ISD_AVGCEILU_rr(VT, RetVT, Op0, Op1); 13355 case ISD::AVGFLOORS: return fastEmit_ISD_AVGFLOORS_rr(VT, RetVT, Op0, Op1); 13356 case ISD::AVGFLOORU: return fastEmit_ISD_AVGFLOORU_rr(VT, RetVT, Op0, Op1); 13357 case ISD::CONCAT_VECTORS: return fastEmit_ISD_CONCAT_VECTORS_rr(VT, RetVT, Op0, Op1); 13358 case ISD::FADD: return fastEmit_ISD_FADD_rr(VT, RetVT, Op0, Op1); 13359 case ISD::FDIV: return fastEmit_ISD_FDIV_rr(VT, RetVT, Op0, Op1); 13360 case ISD::FMAXIMUM: return fastEmit_ISD_FMAXIMUM_rr(VT, RetVT, Op0, Op1); 13361 case ISD::FMAXNUM: return fastEmit_ISD_FMAXNUM_rr(VT, RetVT, Op0, Op1); 13362 case ISD::FMINIMUM: return fastEmit_ISD_FMINIMUM_rr(VT, RetVT, Op0, Op1); 13363 case ISD::FMINNUM: return fastEmit_ISD_FMINNUM_rr(VT, RetVT, Op0, Op1); 13364 case ISD::FMUL: return fastEmit_ISD_FMUL_rr(VT, RetVT, Op0, Op1); 13365 case ISD::FSUB: return fastEmit_ISD_FSUB_rr(VT, RetVT, Op0, Op1); 13366 case ISD::MUL: return fastEmit_ISD_MUL_rr(VT, RetVT, Op0, Op1); 13367 case ISD::MULHS: return fastEmit_ISD_MULHS_rr(VT, RetVT, Op0, Op1); 13368 case ISD::MULHU: return fastEmit_ISD_MULHU_rr(VT, RetVT, Op0, Op1); 13369 case ISD::OR: return fastEmit_ISD_OR_rr(VT, RetVT, Op0, Op1); 13370 case ISD::ROTR: return fastEmit_ISD_ROTR_rr(VT, RetVT, Op0, Op1); 13371 case ISD::SADDSAT: return fastEmit_ISD_SADDSAT_rr(VT, RetVT, Op0, Op1); 13372 case ISD::SDIV: return fastEmit_ISD_SDIV_rr(VT, RetVT, Op0, Op1); 13373 case ISD::SHL: return fastEmit_ISD_SHL_rr(VT, RetVT, Op0, Op1); 13374 case ISD::SMAX: return fastEmit_ISD_SMAX_rr(VT, RetVT, Op0, Op1); 13375 case ISD::SMIN: return fastEmit_ISD_SMIN_rr(VT, RetVT, Op0, Op1); 13376 case ISD::SRA: return fastEmit_ISD_SRA_rr(VT, RetVT, Op0, Op1); 13377 case ISD::SRL: return fastEmit_ISD_SRL_rr(VT, RetVT, Op0, Op1); 13378 case ISD::SSUBSAT: return fastEmit_ISD_SSUBSAT_rr(VT, RetVT, Op0, Op1); 13379 case ISD::STRICT_FADD: return fastEmit_ISD_STRICT_FADD_rr(VT, RetVT, Op0, Op1); 13380 case ISD::STRICT_FDIV: return fastEmit_ISD_STRICT_FDIV_rr(VT, RetVT, Op0, Op1); 13381 case ISD::STRICT_FMAXIMUM: return fastEmit_ISD_STRICT_FMAXIMUM_rr(VT, RetVT, Op0, Op1); 13382 case ISD::STRICT_FMAXNUM: return fastEmit_ISD_STRICT_FMAXNUM_rr(VT, RetVT, Op0, Op1); 13383 case ISD::STRICT_FMINIMUM: return fastEmit_ISD_STRICT_FMINIMUM_rr(VT, RetVT, Op0, Op1); 13384 case ISD::STRICT_FMINNUM: return fastEmit_ISD_STRICT_FMINNUM_rr(VT, RetVT, Op0, Op1); 13385 case ISD::STRICT_FMUL: return fastEmit_ISD_STRICT_FMUL_rr(VT, RetVT, Op0, Op1); 13386 case ISD::STRICT_FSUB: return fastEmit_ISD_STRICT_FSUB_rr(VT, RetVT, Op0, Op1); 13387 case ISD::SUB: return fastEmit_ISD_SUB_rr(VT, RetVT, Op0, Op1); 13388 case ISD::UADDSAT: return fastEmit_ISD_UADDSAT_rr(VT, RetVT, Op0, Op1); 13389 case ISD::UDIV: return fastEmit_ISD_UDIV_rr(VT, RetVT, Op0, Op1); 13390 case ISD::UMAX: return fastEmit_ISD_UMAX_rr(VT, RetVT, Op0, Op1); 13391 case ISD::UMIN: return fastEmit_ISD_UMIN_rr(VT, RetVT, Op0, Op1); 13392 case ISD::USUBSAT: return fastEmit_ISD_USUBSAT_rr(VT, RetVT, Op0, Op1); 13393 case ISD::XOR: return fastEmit_ISD_XOR_rr(VT, RetVT, Op0, Op1); 13394 default: return 0; 13395 } 13396} 13397 13398// FastEmit functions for AArch64ISD::DUPLANE64. 13399 13400unsigned fastEmit_AArch64ISD_DUPLANE64_MVT_v2i64_ri_Predicate_VectorIndexD(MVT RetVT, unsigned Op0, uint64_t imm1) { 13401 if (RetVT.SimpleTy != MVT::v2i64) 13402 return 0; 13403 if ((Subtarget->hasNEON())) { 13404 return fastEmitInst_ri(AArch64::DUPv2i64lane, &AArch64::FPR128RegClass, Op0, imm1); 13405 } 13406 return 0; 13407} 13408 13409unsigned fastEmit_AArch64ISD_DUPLANE64_MVT_v2f64_ri_Predicate_VectorIndexD(MVT RetVT, unsigned Op0, uint64_t imm1) { 13410 if (RetVT.SimpleTy != MVT::v2f64) 13411 return 0; 13412 return fastEmitInst_ri(AArch64::DUPv2i64lane, &AArch64::FPR128RegClass, Op0, imm1); 13413} 13414 13415unsigned fastEmit_AArch64ISD_DUPLANE64_ri_Predicate_VectorIndexD(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 13416 switch (VT.SimpleTy) { 13417 case MVT::v2i64: return fastEmit_AArch64ISD_DUPLANE64_MVT_v2i64_ri_Predicate_VectorIndexD(RetVT, Op0, imm1); 13418 case MVT::v2f64: return fastEmit_AArch64ISD_DUPLANE64_MVT_v2f64_ri_Predicate_VectorIndexD(RetVT, Op0, imm1); 13419 default: return 0; 13420 } 13421} 13422 13423// FastEmit functions for ISD::EXTRACT_VECTOR_ELT. 13424 13425unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndexD(MVT RetVT, unsigned Op0, uint64_t imm1) { 13426 if (RetVT.SimpleTy != MVT::i64) 13427 return 0; 13428 if ((Subtarget->hasNEON())) { 13429 return fastEmitInst_ri(AArch64::UMOVvi64, &AArch64::GPR64RegClass, Op0, imm1); 13430 } 13431 return 0; 13432} 13433 13434unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2f64_ri_Predicate_VectorIndexD(MVT RetVT, unsigned Op0, uint64_t imm1) { 13435 if (RetVT.SimpleTy != MVT::f64) 13436 return 0; 13437 return fastEmitInst_ri(AArch64::DUPi64, &AArch64::FPR64RegClass, Op0, imm1); 13438} 13439 13440unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexD(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 13441 switch (VT.SimpleTy) { 13442 case MVT::v2i64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndexD(RetVT, Op0, imm1); 13443 case MVT::v2f64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2f64_ri_Predicate_VectorIndexD(RetVT, Op0, imm1); 13444 default: return 0; 13445 } 13446} 13447 13448// Top-level FastEmit function. 13449 13450unsigned fastEmit_ri_Predicate_VectorIndexD(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) { 13451 switch (Opcode) { 13452 case AArch64ISD::DUPLANE64: return fastEmit_AArch64ISD_DUPLANE64_ri_Predicate_VectorIndexD(VT, RetVT, Op0, imm1); 13453 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexD(VT, RetVT, Op0, imm1); 13454 default: return 0; 13455 } 13456} 13457 13458// FastEmit functions for AArch64ISD::DUPLANE32. 13459 13460unsigned fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v2i32_ri_Predicate_VectorIndexS(unsigned Op0, uint64_t imm1) { 13461 if ((Subtarget->hasNEON())) { 13462 return fastEmitInst_ri(AArch64::DUPv2i32lane, &AArch64::FPR64RegClass, Op0, imm1); 13463 } 13464 return 0; 13465} 13466 13467unsigned fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v4i32_ri_Predicate_VectorIndexS(unsigned Op0, uint64_t imm1) { 13468 if ((Subtarget->hasNEON())) { 13469 return fastEmitInst_ri(AArch64::DUPv4i32lane, &AArch64::FPR128RegClass, Op0, imm1); 13470 } 13471 return 0; 13472} 13473 13474unsigned fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_ri_Predicate_VectorIndexS(MVT RetVT, unsigned Op0, uint64_t imm1) { 13475switch (RetVT.SimpleTy) { 13476 case MVT::v2i32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v2i32_ri_Predicate_VectorIndexS(Op0, imm1); 13477 case MVT::v4i32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v4i32_ri_Predicate_VectorIndexS(Op0, imm1); 13478 default: return 0; 13479} 13480} 13481 13482unsigned fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v2f32_ri_Predicate_VectorIndexS(unsigned Op0, uint64_t imm1) { 13483 return fastEmitInst_ri(AArch64::DUPv2i32lane, &AArch64::FPR64RegClass, Op0, imm1); 13484} 13485 13486unsigned fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v4f32_ri_Predicate_VectorIndexS(unsigned Op0, uint64_t imm1) { 13487 return fastEmitInst_ri(AArch64::DUPv4i32lane, &AArch64::FPR128RegClass, Op0, imm1); 13488} 13489 13490unsigned fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_ri_Predicate_VectorIndexS(MVT RetVT, unsigned Op0, uint64_t imm1) { 13491switch (RetVT.SimpleTy) { 13492 case MVT::v2f32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v2f32_ri_Predicate_VectorIndexS(Op0, imm1); 13493 case MVT::v4f32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v4f32_ri_Predicate_VectorIndexS(Op0, imm1); 13494 default: return 0; 13495} 13496} 13497 13498unsigned fastEmit_AArch64ISD_DUPLANE32_ri_Predicate_VectorIndexS(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 13499 switch (VT.SimpleTy) { 13500 case MVT::v4i32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_ri_Predicate_VectorIndexS(RetVT, Op0, imm1); 13501 case MVT::v4f32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_ri_Predicate_VectorIndexS(RetVT, Op0, imm1); 13502 default: return 0; 13503 } 13504} 13505 13506// FastEmit functions for ISD::EXTRACT_VECTOR_ELT. 13507 13508unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndexS(MVT RetVT, unsigned Op0, uint64_t imm1) { 13509 if (RetVT.SimpleTy != MVT::i32) 13510 return 0; 13511 if ((Subtarget->hasNEON())) { 13512 return fastEmitInst_ri(AArch64::UMOVvi32, &AArch64::GPR32RegClass, Op0, imm1); 13513 } 13514 return 0; 13515} 13516 13517unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4f32_ri_Predicate_VectorIndexS(MVT RetVT, unsigned Op0, uint64_t imm1) { 13518 if (RetVT.SimpleTy != MVT::f32) 13519 return 0; 13520 return fastEmitInst_ri(AArch64::DUPi32, &AArch64::FPR32RegClass, Op0, imm1); 13521} 13522 13523unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexS(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 13524 switch (VT.SimpleTy) { 13525 case MVT::v4i32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndexS(RetVT, Op0, imm1); 13526 case MVT::v4f32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4f32_ri_Predicate_VectorIndexS(RetVT, Op0, imm1); 13527 default: return 0; 13528 } 13529} 13530 13531// Top-level FastEmit function. 13532 13533unsigned fastEmit_ri_Predicate_VectorIndexS(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) { 13534 switch (Opcode) { 13535 case AArch64ISD::DUPLANE32: return fastEmit_AArch64ISD_DUPLANE32_ri_Predicate_VectorIndexS(VT, RetVT, Op0, imm1); 13536 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexS(VT, RetVT, Op0, imm1); 13537 default: return 0; 13538 } 13539} 13540 13541// FastEmit functions for AArch64ISD::DUPLANE16. 13542 13543unsigned fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v4i16_ri_Predicate_VectorIndexH(unsigned Op0, uint64_t imm1) { 13544 if ((Subtarget->hasNEON())) { 13545 return fastEmitInst_ri(AArch64::DUPv4i16lane, &AArch64::FPR64RegClass, Op0, imm1); 13546 } 13547 return 0; 13548} 13549 13550unsigned fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v8i16_ri_Predicate_VectorIndexH(unsigned Op0, uint64_t imm1) { 13551 if ((Subtarget->hasNEON())) { 13552 return fastEmitInst_ri(AArch64::DUPv8i16lane, &AArch64::FPR128RegClass, Op0, imm1); 13553 } 13554 return 0; 13555} 13556 13557unsigned fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_ri_Predicate_VectorIndexH(MVT RetVT, unsigned Op0, uint64_t imm1) { 13558switch (RetVT.SimpleTy) { 13559 case MVT::v4i16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v4i16_ri_Predicate_VectorIndexH(Op0, imm1); 13560 case MVT::v8i16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v8i16_ri_Predicate_VectorIndexH(Op0, imm1); 13561 default: return 0; 13562} 13563} 13564 13565unsigned fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v4f16_ri_Predicate_VectorIndexH(unsigned Op0, uint64_t imm1) { 13566 return fastEmitInst_ri(AArch64::DUPv4i16lane, &AArch64::FPR64RegClass, Op0, imm1); 13567} 13568 13569unsigned fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v8f16_ri_Predicate_VectorIndexH(unsigned Op0, uint64_t imm1) { 13570 return fastEmitInst_ri(AArch64::DUPv8i16lane, &AArch64::FPR128RegClass, Op0, imm1); 13571} 13572 13573unsigned fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_ri_Predicate_VectorIndexH(MVT RetVT, unsigned Op0, uint64_t imm1) { 13574switch (RetVT.SimpleTy) { 13575 case MVT::v4f16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v4f16_ri_Predicate_VectorIndexH(Op0, imm1); 13576 case MVT::v8f16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v8f16_ri_Predicate_VectorIndexH(Op0, imm1); 13577 default: return 0; 13578} 13579} 13580 13581unsigned fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v4bf16_ri_Predicate_VectorIndexH(unsigned Op0, uint64_t imm1) { 13582 return fastEmitInst_ri(AArch64::DUPv4i16lane, &AArch64::FPR64RegClass, Op0, imm1); 13583} 13584 13585unsigned fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v8bf16_ri_Predicate_VectorIndexH(unsigned Op0, uint64_t imm1) { 13586 return fastEmitInst_ri(AArch64::DUPv8i16lane, &AArch64::FPR128RegClass, Op0, imm1); 13587} 13588 13589unsigned fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_ri_Predicate_VectorIndexH(MVT RetVT, unsigned Op0, uint64_t imm1) { 13590switch (RetVT.SimpleTy) { 13591 case MVT::v4bf16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v4bf16_ri_Predicate_VectorIndexH(Op0, imm1); 13592 case MVT::v8bf16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v8bf16_ri_Predicate_VectorIndexH(Op0, imm1); 13593 default: return 0; 13594} 13595} 13596 13597unsigned fastEmit_AArch64ISD_DUPLANE16_ri_Predicate_VectorIndexH(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 13598 switch (VT.SimpleTy) { 13599 case MVT::v8i16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1); 13600 case MVT::v8f16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1); 13601 case MVT::v8bf16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1); 13602 default: return 0; 13603 } 13604} 13605 13606// FastEmit functions for ISD::EXTRACT_VECTOR_ELT. 13607 13608unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndexH(MVT RetVT, unsigned Op0, uint64_t imm1) { 13609 if (RetVT.SimpleTy != MVT::i32) 13610 return 0; 13611 if ((Subtarget->hasNEON())) { 13612 return fastEmitInst_ri(AArch64::UMOVvi16, &AArch64::GPR32RegClass, Op0, imm1); 13613 } 13614 return 0; 13615} 13616 13617unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8f16_ri_Predicate_VectorIndexH(MVT RetVT, unsigned Op0, uint64_t imm1) { 13618 if (RetVT.SimpleTy != MVT::f16) 13619 return 0; 13620 return fastEmitInst_ri(AArch64::DUPi16, &AArch64::FPR16RegClass, Op0, imm1); 13621} 13622 13623unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8bf16_ri_Predicate_VectorIndexH(MVT RetVT, unsigned Op0, uint64_t imm1) { 13624 if (RetVT.SimpleTy != MVT::bf16) 13625 return 0; 13626 return fastEmitInst_ri(AArch64::DUPi16, &AArch64::FPR16RegClass, Op0, imm1); 13627} 13628 13629unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexH(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 13630 switch (VT.SimpleTy) { 13631 case MVT::v8i16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1); 13632 case MVT::v8f16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8f16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1); 13633 case MVT::v8bf16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8bf16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1); 13634 default: return 0; 13635 } 13636} 13637 13638// Top-level FastEmit function. 13639 13640unsigned fastEmit_ri_Predicate_VectorIndexH(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) { 13641 switch (Opcode) { 13642 case AArch64ISD::DUPLANE16: return fastEmit_AArch64ISD_DUPLANE16_ri_Predicate_VectorIndexH(VT, RetVT, Op0, imm1); 13643 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexH(VT, RetVT, Op0, imm1); 13644 default: return 0; 13645 } 13646} 13647 13648// FastEmit functions for AArch64ISD::DUPLANE8. 13649 13650unsigned fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v8i8_ri_Predicate_VectorIndexB(unsigned Op0, uint64_t imm1) { 13651 if ((Subtarget->hasNEON())) { 13652 return fastEmitInst_ri(AArch64::DUPv8i8lane, &AArch64::FPR64RegClass, Op0, imm1); 13653 } 13654 return 0; 13655} 13656 13657unsigned fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v16i8_ri_Predicate_VectorIndexB(unsigned Op0, uint64_t imm1) { 13658 if ((Subtarget->hasNEON())) { 13659 return fastEmitInst_ri(AArch64::DUPv16i8lane, &AArch64::FPR128RegClass, Op0, imm1); 13660 } 13661 return 0; 13662} 13663 13664unsigned fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_ri_Predicate_VectorIndexB(MVT RetVT, unsigned Op0, uint64_t imm1) { 13665switch (RetVT.SimpleTy) { 13666 case MVT::v8i8: return fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v8i8_ri_Predicate_VectorIndexB(Op0, imm1); 13667 case MVT::v16i8: return fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v16i8_ri_Predicate_VectorIndexB(Op0, imm1); 13668 default: return 0; 13669} 13670} 13671 13672unsigned fastEmit_AArch64ISD_DUPLANE8_ri_Predicate_VectorIndexB(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 13673 switch (VT.SimpleTy) { 13674 case MVT::v16i8: return fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_ri_Predicate_VectorIndexB(RetVT, Op0, imm1); 13675 default: return 0; 13676 } 13677} 13678 13679// FastEmit functions for ISD::EXTRACT_VECTOR_ELT. 13680 13681unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndexB(MVT RetVT, unsigned Op0, uint64_t imm1) { 13682 if (RetVT.SimpleTy != MVT::i32) 13683 return 0; 13684 if ((Subtarget->hasNEON())) { 13685 return fastEmitInst_ri(AArch64::UMOVvi8, &AArch64::GPR32RegClass, Op0, imm1); 13686 } 13687 return 0; 13688} 13689 13690unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexB(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 13691 switch (VT.SimpleTy) { 13692 case MVT::v16i8: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndexB(RetVT, Op0, imm1); 13693 default: return 0; 13694 } 13695} 13696 13697// Top-level FastEmit function. 13698 13699unsigned fastEmit_ri_Predicate_VectorIndexB(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) { 13700 switch (Opcode) { 13701 case AArch64ISD::DUPLANE8: return fastEmit_AArch64ISD_DUPLANE8_ri_Predicate_VectorIndexB(VT, RetVT, Op0, imm1); 13702 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexB(VT, RetVT, Op0, imm1); 13703 default: return 0; 13704 } 13705} 13706 13707// FastEmit functions for ISD::EXTRACT_VECTOR_ELT. 13708 13709unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndex0(MVT RetVT, unsigned Op0, uint64_t imm1) { 13710 if (RetVT.SimpleTy != MVT::i32) 13711 return 0; 13712 if ((Subtarget->hasNEON() || Subtarget->hasSME())) { 13713 return fastEmitInst_ri(AArch64::UMOVvi8_idx0, &AArch64::GPR32RegClass, Op0, imm1); 13714 } 13715 return 0; 13716} 13717 13718unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndex0(MVT RetVT, unsigned Op0, uint64_t imm1) { 13719 if (RetVT.SimpleTy != MVT::i32) 13720 return 0; 13721 if ((Subtarget->hasNEON() || Subtarget->hasSME())) { 13722 return fastEmitInst_ri(AArch64::UMOVvi16_idx0, &AArch64::GPR32RegClass, Op0, imm1); 13723 } 13724 return 0; 13725} 13726 13727unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndex0(MVT RetVT, unsigned Op0, uint64_t imm1) { 13728 if (RetVT.SimpleTy != MVT::i32) 13729 return 0; 13730 if ((Subtarget->hasNEON() || Subtarget->hasSME())) { 13731 return fastEmitInst_ri(AArch64::UMOVvi32_idx0, &AArch64::GPR32RegClass, Op0, imm1); 13732 } 13733 return 0; 13734} 13735 13736unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndex0(MVT RetVT, unsigned Op0, uint64_t imm1) { 13737 if (RetVT.SimpleTy != MVT::i64) 13738 return 0; 13739 if ((Subtarget->hasNEON() || Subtarget->hasSME())) { 13740 return fastEmitInst_ri(AArch64::UMOVvi64_idx0, &AArch64::GPR64RegClass, Op0, imm1); 13741 } 13742 return 0; 13743} 13744 13745unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndex0(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 13746 switch (VT.SimpleTy) { 13747 case MVT::v16i8: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndex0(RetVT, Op0, imm1); 13748 case MVT::v8i16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndex0(RetVT, Op0, imm1); 13749 case MVT::v4i32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndex0(RetVT, Op0, imm1); 13750 case MVT::v2i64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndex0(RetVT, Op0, imm1); 13751 default: return 0; 13752 } 13753} 13754 13755// Top-level FastEmit function. 13756 13757unsigned fastEmit_ri_Predicate_VectorIndex0(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) { 13758 switch (Opcode) { 13759 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndex0(VT, RetVT, Op0, imm1); 13760 default: return 0; 13761 } 13762} 13763 13764// FastEmit functions for AArch64ISD::SQSHLU_I. 13765 13766unsigned fastEmit_AArch64ISD_SQSHLU_I_MVT_i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, uint64_t imm1) { 13767 if (RetVT.SimpleTy != MVT::i64) 13768 return 0; 13769 if ((Subtarget->hasNEON())) { 13770 return fastEmitInst_ri(AArch64::SQSHLUd, &AArch64::FPR64RegClass, Op0, imm1); 13771 } 13772 return 0; 13773} 13774 13775unsigned fastEmit_AArch64ISD_SQSHLU_I_MVT_v1i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, uint64_t imm1) { 13776 if (RetVT.SimpleTy != MVT::v1i64) 13777 return 0; 13778 if ((Subtarget->hasNEON())) { 13779 return fastEmitInst_ri(AArch64::SQSHLUd, &AArch64::FPR64RegClass, Op0, imm1); 13780 } 13781 return 0; 13782} 13783 13784unsigned fastEmit_AArch64ISD_SQSHLU_I_MVT_v2i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, uint64_t imm1) { 13785 if (RetVT.SimpleTy != MVT::v2i64) 13786 return 0; 13787 if ((Subtarget->hasNEON())) { 13788 return fastEmitInst_ri(AArch64::SQSHLUv2i64_shift, &AArch64::FPR128RegClass, Op0, imm1); 13789 } 13790 return 0; 13791} 13792 13793unsigned fastEmit_AArch64ISD_SQSHLU_I_ri_Predicate_vecshiftL64(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 13794 switch (VT.SimpleTy) { 13795 case MVT::i64: return fastEmit_AArch64ISD_SQSHLU_I_MVT_i64_ri_Predicate_vecshiftL64(RetVT, Op0, imm1); 13796 case MVT::v1i64: return fastEmit_AArch64ISD_SQSHLU_I_MVT_v1i64_ri_Predicate_vecshiftL64(RetVT, Op0, imm1); 13797 case MVT::v2i64: return fastEmit_AArch64ISD_SQSHLU_I_MVT_v2i64_ri_Predicate_vecshiftL64(RetVT, Op0, imm1); 13798 default: return 0; 13799 } 13800} 13801 13802// FastEmit functions for AArch64ISD::SQSHL_I. 13803 13804unsigned fastEmit_AArch64ISD_SQSHL_I_MVT_i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, uint64_t imm1) { 13805 if (RetVT.SimpleTy != MVT::i64) 13806 return 0; 13807 if ((Subtarget->hasNEON())) { 13808 return fastEmitInst_ri(AArch64::SQSHLd, &AArch64::FPR64RegClass, Op0, imm1); 13809 } 13810 return 0; 13811} 13812 13813unsigned fastEmit_AArch64ISD_SQSHL_I_MVT_v1i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, uint64_t imm1) { 13814 if (RetVT.SimpleTy != MVT::v1i64) 13815 return 0; 13816 if ((Subtarget->hasNEON())) { 13817 return fastEmitInst_ri(AArch64::SQSHLd, &AArch64::FPR64RegClass, Op0, imm1); 13818 } 13819 return 0; 13820} 13821 13822unsigned fastEmit_AArch64ISD_SQSHL_I_MVT_v2i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, uint64_t imm1) { 13823 if (RetVT.SimpleTy != MVT::v2i64) 13824 return 0; 13825 if ((Subtarget->hasNEON())) { 13826 return fastEmitInst_ri(AArch64::SQSHLv2i64_shift, &AArch64::FPR128RegClass, Op0, imm1); 13827 } 13828 return 0; 13829} 13830 13831unsigned fastEmit_AArch64ISD_SQSHL_I_ri_Predicate_vecshiftL64(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 13832 switch (VT.SimpleTy) { 13833 case MVT::i64: return fastEmit_AArch64ISD_SQSHL_I_MVT_i64_ri_Predicate_vecshiftL64(RetVT, Op0, imm1); 13834 case MVT::v1i64: return fastEmit_AArch64ISD_SQSHL_I_MVT_v1i64_ri_Predicate_vecshiftL64(RetVT, Op0, imm1); 13835 case MVT::v2i64: return fastEmit_AArch64ISD_SQSHL_I_MVT_v2i64_ri_Predicate_vecshiftL64(RetVT, Op0, imm1); 13836 default: return 0; 13837 } 13838} 13839 13840// FastEmit functions for AArch64ISD::UQSHL_I. 13841 13842unsigned fastEmit_AArch64ISD_UQSHL_I_MVT_i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, uint64_t imm1) { 13843 if (RetVT.SimpleTy != MVT::i64) 13844 return 0; 13845 if ((Subtarget->hasNEON())) { 13846 return fastEmitInst_ri(AArch64::UQSHLd, &AArch64::FPR64RegClass, Op0, imm1); 13847 } 13848 return 0; 13849} 13850 13851unsigned fastEmit_AArch64ISD_UQSHL_I_MVT_v1i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, uint64_t imm1) { 13852 if (RetVT.SimpleTy != MVT::v1i64) 13853 return 0; 13854 if ((Subtarget->hasNEON())) { 13855 return fastEmitInst_ri(AArch64::UQSHLd, &AArch64::FPR64RegClass, Op0, imm1); 13856 } 13857 return 0; 13858} 13859 13860unsigned fastEmit_AArch64ISD_UQSHL_I_MVT_v2i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, uint64_t imm1) { 13861 if (RetVT.SimpleTy != MVT::v2i64) 13862 return 0; 13863 if ((Subtarget->hasNEON())) { 13864 return fastEmitInst_ri(AArch64::UQSHLv2i64_shift, &AArch64::FPR128RegClass, Op0, imm1); 13865 } 13866 return 0; 13867} 13868 13869unsigned fastEmit_AArch64ISD_UQSHL_I_ri_Predicate_vecshiftL64(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 13870 switch (VT.SimpleTy) { 13871 case MVT::i64: return fastEmit_AArch64ISD_UQSHL_I_MVT_i64_ri_Predicate_vecshiftL64(RetVT, Op0, imm1); 13872 case MVT::v1i64: return fastEmit_AArch64ISD_UQSHL_I_MVT_v1i64_ri_Predicate_vecshiftL64(RetVT, Op0, imm1); 13873 case MVT::v2i64: return fastEmit_AArch64ISD_UQSHL_I_MVT_v2i64_ri_Predicate_vecshiftL64(RetVT, Op0, imm1); 13874 default: return 0; 13875 } 13876} 13877 13878// FastEmit functions for AArch64ISD::VSHL. 13879 13880unsigned fastEmit_AArch64ISD_VSHL_MVT_i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, uint64_t imm1) { 13881 if (RetVT.SimpleTy != MVT::i64) 13882 return 0; 13883 if ((Subtarget->hasNEON())) { 13884 return fastEmitInst_ri(AArch64::SHLd, &AArch64::FPR64RegClass, Op0, imm1); 13885 } 13886 return 0; 13887} 13888 13889unsigned fastEmit_AArch64ISD_VSHL_MVT_v1i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, uint64_t imm1) { 13890 if (RetVT.SimpleTy != MVT::v1i64) 13891 return 0; 13892 if ((Subtarget->hasNEON())) { 13893 return fastEmitInst_ri(AArch64::SHLd, &AArch64::FPR64RegClass, Op0, imm1); 13894 } 13895 return 0; 13896} 13897 13898unsigned fastEmit_AArch64ISD_VSHL_MVT_v2i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, uint64_t imm1) { 13899 if (RetVT.SimpleTy != MVT::v2i64) 13900 return 0; 13901 if ((Subtarget->hasNEON())) { 13902 return fastEmitInst_ri(AArch64::SHLv2i64_shift, &AArch64::FPR128RegClass, Op0, imm1); 13903 } 13904 return 0; 13905} 13906 13907unsigned fastEmit_AArch64ISD_VSHL_ri_Predicate_vecshiftL64(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 13908 switch (VT.SimpleTy) { 13909 case MVT::i64: return fastEmit_AArch64ISD_VSHL_MVT_i64_ri_Predicate_vecshiftL64(RetVT, Op0, imm1); 13910 case MVT::v1i64: return fastEmit_AArch64ISD_VSHL_MVT_v1i64_ri_Predicate_vecshiftL64(RetVT, Op0, imm1); 13911 case MVT::v2i64: return fastEmit_AArch64ISD_VSHL_MVT_v2i64_ri_Predicate_vecshiftL64(RetVT, Op0, imm1); 13912 default: return 0; 13913 } 13914} 13915 13916// Top-level FastEmit function. 13917 13918unsigned fastEmit_ri_Predicate_vecshiftL64(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) { 13919 switch (Opcode) { 13920 case AArch64ISD::SQSHLU_I: return fastEmit_AArch64ISD_SQSHLU_I_ri_Predicate_vecshiftL64(VT, RetVT, Op0, imm1); 13921 case AArch64ISD::SQSHL_I: return fastEmit_AArch64ISD_SQSHL_I_ri_Predicate_vecshiftL64(VT, RetVT, Op0, imm1); 13922 case AArch64ISD::UQSHL_I: return fastEmit_AArch64ISD_UQSHL_I_ri_Predicate_vecshiftL64(VT, RetVT, Op0, imm1); 13923 case AArch64ISD::VSHL: return fastEmit_AArch64ISD_VSHL_ri_Predicate_vecshiftL64(VT, RetVT, Op0, imm1); 13924 default: return 0; 13925 } 13926} 13927 13928// FastEmit functions for AArch64ISD::SQSHLU_I. 13929 13930unsigned fastEmit_AArch64ISD_SQSHLU_I_MVT_i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, uint64_t imm1) { 13931 if (RetVT.SimpleTy != MVT::i32) 13932 return 0; 13933 if ((Subtarget->hasNEON())) { 13934 return fastEmitInst_ri(AArch64::SQSHLUs, &AArch64::FPR32RegClass, Op0, imm1); 13935 } 13936 return 0; 13937} 13938 13939unsigned fastEmit_AArch64ISD_SQSHLU_I_MVT_v2i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, uint64_t imm1) { 13940 if (RetVT.SimpleTy != MVT::v2i32) 13941 return 0; 13942 if ((Subtarget->hasNEON())) { 13943 return fastEmitInst_ri(AArch64::SQSHLUv2i32_shift, &AArch64::FPR64RegClass, Op0, imm1); 13944 } 13945 return 0; 13946} 13947 13948unsigned fastEmit_AArch64ISD_SQSHLU_I_MVT_v4i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, uint64_t imm1) { 13949 if (RetVT.SimpleTy != MVT::v4i32) 13950 return 0; 13951 if ((Subtarget->hasNEON())) { 13952 return fastEmitInst_ri(AArch64::SQSHLUv4i32_shift, &AArch64::FPR128RegClass, Op0, imm1); 13953 } 13954 return 0; 13955} 13956 13957unsigned fastEmit_AArch64ISD_SQSHLU_I_ri_Predicate_vecshiftL32(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 13958 switch (VT.SimpleTy) { 13959 case MVT::i32: return fastEmit_AArch64ISD_SQSHLU_I_MVT_i32_ri_Predicate_vecshiftL32(RetVT, Op0, imm1); 13960 case MVT::v2i32: return fastEmit_AArch64ISD_SQSHLU_I_MVT_v2i32_ri_Predicate_vecshiftL32(RetVT, Op0, imm1); 13961 case MVT::v4i32: return fastEmit_AArch64ISD_SQSHLU_I_MVT_v4i32_ri_Predicate_vecshiftL32(RetVT, Op0, imm1); 13962 default: return 0; 13963 } 13964} 13965 13966// FastEmit functions for AArch64ISD::SQSHL_I. 13967 13968unsigned fastEmit_AArch64ISD_SQSHL_I_MVT_i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, uint64_t imm1) { 13969 if (RetVT.SimpleTy != MVT::i32) 13970 return 0; 13971 if ((Subtarget->hasNEON())) { 13972 return fastEmitInst_ri(AArch64::SQSHLs, &AArch64::FPR32RegClass, Op0, imm1); 13973 } 13974 return 0; 13975} 13976 13977unsigned fastEmit_AArch64ISD_SQSHL_I_MVT_v2i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, uint64_t imm1) { 13978 if (RetVT.SimpleTy != MVT::v2i32) 13979 return 0; 13980 if ((Subtarget->hasNEON())) { 13981 return fastEmitInst_ri(AArch64::SQSHLv2i32_shift, &AArch64::FPR64RegClass, Op0, imm1); 13982 } 13983 return 0; 13984} 13985 13986unsigned fastEmit_AArch64ISD_SQSHL_I_MVT_v4i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, uint64_t imm1) { 13987 if (RetVT.SimpleTy != MVT::v4i32) 13988 return 0; 13989 if ((Subtarget->hasNEON())) { 13990 return fastEmitInst_ri(AArch64::SQSHLv4i32_shift, &AArch64::FPR128RegClass, Op0, imm1); 13991 } 13992 return 0; 13993} 13994 13995unsigned fastEmit_AArch64ISD_SQSHL_I_ri_Predicate_vecshiftL32(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 13996 switch (VT.SimpleTy) { 13997 case MVT::i32: return fastEmit_AArch64ISD_SQSHL_I_MVT_i32_ri_Predicate_vecshiftL32(RetVT, Op0, imm1); 13998 case MVT::v2i32: return fastEmit_AArch64ISD_SQSHL_I_MVT_v2i32_ri_Predicate_vecshiftL32(RetVT, Op0, imm1); 13999 case MVT::v4i32: return fastEmit_AArch64ISD_SQSHL_I_MVT_v4i32_ri_Predicate_vecshiftL32(RetVT, Op0, imm1); 14000 default: return 0; 14001 } 14002} 14003 14004// FastEmit functions for AArch64ISD::UQSHL_I. 14005 14006unsigned fastEmit_AArch64ISD_UQSHL_I_MVT_i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, uint64_t imm1) { 14007 if (RetVT.SimpleTy != MVT::i32) 14008 return 0; 14009 if ((Subtarget->hasNEON())) { 14010 return fastEmitInst_ri(AArch64::UQSHLs, &AArch64::FPR32RegClass, Op0, imm1); 14011 } 14012 return 0; 14013} 14014 14015unsigned fastEmit_AArch64ISD_UQSHL_I_MVT_v2i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, uint64_t imm1) { 14016 if (RetVT.SimpleTy != MVT::v2i32) 14017 return 0; 14018 if ((Subtarget->hasNEON())) { 14019 return fastEmitInst_ri(AArch64::UQSHLv2i32_shift, &AArch64::FPR64RegClass, Op0, imm1); 14020 } 14021 return 0; 14022} 14023 14024unsigned fastEmit_AArch64ISD_UQSHL_I_MVT_v4i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, uint64_t imm1) { 14025 if (RetVT.SimpleTy != MVT::v4i32) 14026 return 0; 14027 if ((Subtarget->hasNEON())) { 14028 return fastEmitInst_ri(AArch64::UQSHLv4i32_shift, &AArch64::FPR128RegClass, Op0, imm1); 14029 } 14030 return 0; 14031} 14032 14033unsigned fastEmit_AArch64ISD_UQSHL_I_ri_Predicate_vecshiftL32(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 14034 switch (VT.SimpleTy) { 14035 case MVT::i32: return fastEmit_AArch64ISD_UQSHL_I_MVT_i32_ri_Predicate_vecshiftL32(RetVT, Op0, imm1); 14036 case MVT::v2i32: return fastEmit_AArch64ISD_UQSHL_I_MVT_v2i32_ri_Predicate_vecshiftL32(RetVT, Op0, imm1); 14037 case MVT::v4i32: return fastEmit_AArch64ISD_UQSHL_I_MVT_v4i32_ri_Predicate_vecshiftL32(RetVT, Op0, imm1); 14038 default: return 0; 14039 } 14040} 14041 14042// FastEmit functions for AArch64ISD::VSHL. 14043 14044unsigned fastEmit_AArch64ISD_VSHL_MVT_v2i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, uint64_t imm1) { 14045 if (RetVT.SimpleTy != MVT::v2i32) 14046 return 0; 14047 if ((Subtarget->hasNEON())) { 14048 return fastEmitInst_ri(AArch64::SHLv2i32_shift, &AArch64::FPR64RegClass, Op0, imm1); 14049 } 14050 return 0; 14051} 14052 14053unsigned fastEmit_AArch64ISD_VSHL_MVT_v4i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, uint64_t imm1) { 14054 if (RetVT.SimpleTy != MVT::v4i32) 14055 return 0; 14056 if ((Subtarget->hasNEON())) { 14057 return fastEmitInst_ri(AArch64::SHLv4i32_shift, &AArch64::FPR128RegClass, Op0, imm1); 14058 } 14059 return 0; 14060} 14061 14062unsigned fastEmit_AArch64ISD_VSHL_ri_Predicate_vecshiftL32(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 14063 switch (VT.SimpleTy) { 14064 case MVT::v2i32: return fastEmit_AArch64ISD_VSHL_MVT_v2i32_ri_Predicate_vecshiftL32(RetVT, Op0, imm1); 14065 case MVT::v4i32: return fastEmit_AArch64ISD_VSHL_MVT_v4i32_ri_Predicate_vecshiftL32(RetVT, Op0, imm1); 14066 default: return 0; 14067 } 14068} 14069 14070// Top-level FastEmit function. 14071 14072unsigned fastEmit_ri_Predicate_vecshiftL32(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) { 14073 switch (Opcode) { 14074 case AArch64ISD::SQSHLU_I: return fastEmit_AArch64ISD_SQSHLU_I_ri_Predicate_vecshiftL32(VT, RetVT, Op0, imm1); 14075 case AArch64ISD::SQSHL_I: return fastEmit_AArch64ISD_SQSHL_I_ri_Predicate_vecshiftL32(VT, RetVT, Op0, imm1); 14076 case AArch64ISD::UQSHL_I: return fastEmit_AArch64ISD_UQSHL_I_ri_Predicate_vecshiftL32(VT, RetVT, Op0, imm1); 14077 case AArch64ISD::VSHL: return fastEmit_AArch64ISD_VSHL_ri_Predicate_vecshiftL32(VT, RetVT, Op0, imm1); 14078 default: return 0; 14079 } 14080} 14081 14082// FastEmit functions for AArch64ISD::SRSHR_I. 14083 14084unsigned fastEmit_AArch64ISD_SRSHR_I_MVT_i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, uint64_t imm1) { 14085 if (RetVT.SimpleTy != MVT::i64) 14086 return 0; 14087 if ((Subtarget->hasNEON())) { 14088 return fastEmitInst_ri(AArch64::SRSHRd, &AArch64::FPR64RegClass, Op0, imm1); 14089 } 14090 return 0; 14091} 14092 14093unsigned fastEmit_AArch64ISD_SRSHR_I_MVT_v1i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, uint64_t imm1) { 14094 if (RetVT.SimpleTy != MVT::v1i64) 14095 return 0; 14096 if ((Subtarget->hasNEON())) { 14097 return fastEmitInst_ri(AArch64::SRSHRd, &AArch64::FPR64RegClass, Op0, imm1); 14098 } 14099 return 0; 14100} 14101 14102unsigned fastEmit_AArch64ISD_SRSHR_I_MVT_v2i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, uint64_t imm1) { 14103 if (RetVT.SimpleTy != MVT::v2i64) 14104 return 0; 14105 if ((Subtarget->hasNEON())) { 14106 return fastEmitInst_ri(AArch64::SRSHRv2i64_shift, &AArch64::FPR128RegClass, Op0, imm1); 14107 } 14108 return 0; 14109} 14110 14111unsigned fastEmit_AArch64ISD_SRSHR_I_ri_Predicate_vecshiftR64(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 14112 switch (VT.SimpleTy) { 14113 case MVT::i64: return fastEmit_AArch64ISD_SRSHR_I_MVT_i64_ri_Predicate_vecshiftR64(RetVT, Op0, imm1); 14114 case MVT::v1i64: return fastEmit_AArch64ISD_SRSHR_I_MVT_v1i64_ri_Predicate_vecshiftR64(RetVT, Op0, imm1); 14115 case MVT::v2i64: return fastEmit_AArch64ISD_SRSHR_I_MVT_v2i64_ri_Predicate_vecshiftR64(RetVT, Op0, imm1); 14116 default: return 0; 14117 } 14118} 14119 14120// FastEmit functions for AArch64ISD::URSHR_I. 14121 14122unsigned fastEmit_AArch64ISD_URSHR_I_MVT_i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, uint64_t imm1) { 14123 if (RetVT.SimpleTy != MVT::i64) 14124 return 0; 14125 if ((Subtarget->hasNEON())) { 14126 return fastEmitInst_ri(AArch64::URSHRd, &AArch64::FPR64RegClass, Op0, imm1); 14127 } 14128 return 0; 14129} 14130 14131unsigned fastEmit_AArch64ISD_URSHR_I_MVT_v1i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, uint64_t imm1) { 14132 if (RetVT.SimpleTy != MVT::v1i64) 14133 return 0; 14134 if ((Subtarget->hasNEON())) { 14135 return fastEmitInst_ri(AArch64::URSHRd, &AArch64::FPR64RegClass, Op0, imm1); 14136 } 14137 return 0; 14138} 14139 14140unsigned fastEmit_AArch64ISD_URSHR_I_MVT_v2i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, uint64_t imm1) { 14141 if (RetVT.SimpleTy != MVT::v2i64) 14142 return 0; 14143 if ((Subtarget->hasNEON())) { 14144 return fastEmitInst_ri(AArch64::URSHRv2i64_shift, &AArch64::FPR128RegClass, Op0, imm1); 14145 } 14146 return 0; 14147} 14148 14149unsigned fastEmit_AArch64ISD_URSHR_I_ri_Predicate_vecshiftR64(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 14150 switch (VT.SimpleTy) { 14151 case MVT::i64: return fastEmit_AArch64ISD_URSHR_I_MVT_i64_ri_Predicate_vecshiftR64(RetVT, Op0, imm1); 14152 case MVT::v1i64: return fastEmit_AArch64ISD_URSHR_I_MVT_v1i64_ri_Predicate_vecshiftR64(RetVT, Op0, imm1); 14153 case MVT::v2i64: return fastEmit_AArch64ISD_URSHR_I_MVT_v2i64_ri_Predicate_vecshiftR64(RetVT, Op0, imm1); 14154 default: return 0; 14155 } 14156} 14157 14158// FastEmit functions for AArch64ISD::VASHR. 14159 14160unsigned fastEmit_AArch64ISD_VASHR_MVT_i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, uint64_t imm1) { 14161 if (RetVT.SimpleTy != MVT::i64) 14162 return 0; 14163 if ((Subtarget->hasNEON())) { 14164 return fastEmitInst_ri(AArch64::SSHRd, &AArch64::FPR64RegClass, Op0, imm1); 14165 } 14166 return 0; 14167} 14168 14169unsigned fastEmit_AArch64ISD_VASHR_MVT_v1i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, uint64_t imm1) { 14170 if (RetVT.SimpleTy != MVT::v1i64) 14171 return 0; 14172 if ((Subtarget->hasNEON())) { 14173 return fastEmitInst_ri(AArch64::SSHRd, &AArch64::FPR64RegClass, Op0, imm1); 14174 } 14175 return 0; 14176} 14177 14178unsigned fastEmit_AArch64ISD_VASHR_MVT_v2i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, uint64_t imm1) { 14179 if (RetVT.SimpleTy != MVT::v2i64) 14180 return 0; 14181 if ((Subtarget->hasNEON())) { 14182 return fastEmitInst_ri(AArch64::SSHRv2i64_shift, &AArch64::FPR128RegClass, Op0, imm1); 14183 } 14184 return 0; 14185} 14186 14187unsigned fastEmit_AArch64ISD_VASHR_ri_Predicate_vecshiftR64(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 14188 switch (VT.SimpleTy) { 14189 case MVT::i64: return fastEmit_AArch64ISD_VASHR_MVT_i64_ri_Predicate_vecshiftR64(RetVT, Op0, imm1); 14190 case MVT::v1i64: return fastEmit_AArch64ISD_VASHR_MVT_v1i64_ri_Predicate_vecshiftR64(RetVT, Op0, imm1); 14191 case MVT::v2i64: return fastEmit_AArch64ISD_VASHR_MVT_v2i64_ri_Predicate_vecshiftR64(RetVT, Op0, imm1); 14192 default: return 0; 14193 } 14194} 14195 14196// FastEmit functions for AArch64ISD::VLSHR. 14197 14198unsigned fastEmit_AArch64ISD_VLSHR_MVT_i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, uint64_t imm1) { 14199 if (RetVT.SimpleTy != MVT::i64) 14200 return 0; 14201 if ((Subtarget->hasNEON())) { 14202 return fastEmitInst_ri(AArch64::USHRd, &AArch64::FPR64RegClass, Op0, imm1); 14203 } 14204 return 0; 14205} 14206 14207unsigned fastEmit_AArch64ISD_VLSHR_MVT_v1i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, uint64_t imm1) { 14208 if (RetVT.SimpleTy != MVT::v1i64) 14209 return 0; 14210 if ((Subtarget->hasNEON())) { 14211 return fastEmitInst_ri(AArch64::USHRd, &AArch64::FPR64RegClass, Op0, imm1); 14212 } 14213 return 0; 14214} 14215 14216unsigned fastEmit_AArch64ISD_VLSHR_MVT_v2i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, uint64_t imm1) { 14217 if (RetVT.SimpleTy != MVT::v2i64) 14218 return 0; 14219 if ((Subtarget->hasNEON())) { 14220 return fastEmitInst_ri(AArch64::USHRv2i64_shift, &AArch64::FPR128RegClass, Op0, imm1); 14221 } 14222 return 0; 14223} 14224 14225unsigned fastEmit_AArch64ISD_VLSHR_ri_Predicate_vecshiftR64(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 14226 switch (VT.SimpleTy) { 14227 case MVT::i64: return fastEmit_AArch64ISD_VLSHR_MVT_i64_ri_Predicate_vecshiftR64(RetVT, Op0, imm1); 14228 case MVT::v1i64: return fastEmit_AArch64ISD_VLSHR_MVT_v1i64_ri_Predicate_vecshiftR64(RetVT, Op0, imm1); 14229 case MVT::v2i64: return fastEmit_AArch64ISD_VLSHR_MVT_v2i64_ri_Predicate_vecshiftR64(RetVT, Op0, imm1); 14230 default: return 0; 14231 } 14232} 14233 14234// Top-level FastEmit function. 14235 14236unsigned fastEmit_ri_Predicate_vecshiftR64(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) { 14237 switch (Opcode) { 14238 case AArch64ISD::SRSHR_I: return fastEmit_AArch64ISD_SRSHR_I_ri_Predicate_vecshiftR64(VT, RetVT, Op0, imm1); 14239 case AArch64ISD::URSHR_I: return fastEmit_AArch64ISD_URSHR_I_ri_Predicate_vecshiftR64(VT, RetVT, Op0, imm1); 14240 case AArch64ISD::VASHR: return fastEmit_AArch64ISD_VASHR_ri_Predicate_vecshiftR64(VT, RetVT, Op0, imm1); 14241 case AArch64ISD::VLSHR: return fastEmit_AArch64ISD_VLSHR_ri_Predicate_vecshiftR64(VT, RetVT, Op0, imm1); 14242 default: return 0; 14243 } 14244} 14245 14246// FastEmit functions for AArch64ISD::SQSHLU_I. 14247 14248unsigned fastEmit_AArch64ISD_SQSHLU_I_MVT_v8i8_ri_Predicate_vecshiftL8(MVT RetVT, unsigned Op0, uint64_t imm1) { 14249 if (RetVT.SimpleTy != MVT::v8i8) 14250 return 0; 14251 if ((Subtarget->hasNEON())) { 14252 return fastEmitInst_ri(AArch64::SQSHLUv8i8_shift, &AArch64::FPR64RegClass, Op0, imm1); 14253 } 14254 return 0; 14255} 14256 14257unsigned fastEmit_AArch64ISD_SQSHLU_I_MVT_v16i8_ri_Predicate_vecshiftL8(MVT RetVT, unsigned Op0, uint64_t imm1) { 14258 if (RetVT.SimpleTy != MVT::v16i8) 14259 return 0; 14260 if ((Subtarget->hasNEON())) { 14261 return fastEmitInst_ri(AArch64::SQSHLUv16i8_shift, &AArch64::FPR128RegClass, Op0, imm1); 14262 } 14263 return 0; 14264} 14265 14266unsigned fastEmit_AArch64ISD_SQSHLU_I_ri_Predicate_vecshiftL8(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 14267 switch (VT.SimpleTy) { 14268 case MVT::v8i8: return fastEmit_AArch64ISD_SQSHLU_I_MVT_v8i8_ri_Predicate_vecshiftL8(RetVT, Op0, imm1); 14269 case MVT::v16i8: return fastEmit_AArch64ISD_SQSHLU_I_MVT_v16i8_ri_Predicate_vecshiftL8(RetVT, Op0, imm1); 14270 default: return 0; 14271 } 14272} 14273 14274// FastEmit functions for AArch64ISD::SQSHL_I. 14275 14276unsigned fastEmit_AArch64ISD_SQSHL_I_MVT_v8i8_ri_Predicate_vecshiftL8(MVT RetVT, unsigned Op0, uint64_t imm1) { 14277 if (RetVT.SimpleTy != MVT::v8i8) 14278 return 0; 14279 if ((Subtarget->hasNEON())) { 14280 return fastEmitInst_ri(AArch64::SQSHLv8i8_shift, &AArch64::FPR64RegClass, Op0, imm1); 14281 } 14282 return 0; 14283} 14284 14285unsigned fastEmit_AArch64ISD_SQSHL_I_MVT_v16i8_ri_Predicate_vecshiftL8(MVT RetVT, unsigned Op0, uint64_t imm1) { 14286 if (RetVT.SimpleTy != MVT::v16i8) 14287 return 0; 14288 if ((Subtarget->hasNEON())) { 14289 return fastEmitInst_ri(AArch64::SQSHLv16i8_shift, &AArch64::FPR128RegClass, Op0, imm1); 14290 } 14291 return 0; 14292} 14293 14294unsigned fastEmit_AArch64ISD_SQSHL_I_ri_Predicate_vecshiftL8(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 14295 switch (VT.SimpleTy) { 14296 case MVT::v8i8: return fastEmit_AArch64ISD_SQSHL_I_MVT_v8i8_ri_Predicate_vecshiftL8(RetVT, Op0, imm1); 14297 case MVT::v16i8: return fastEmit_AArch64ISD_SQSHL_I_MVT_v16i8_ri_Predicate_vecshiftL8(RetVT, Op0, imm1); 14298 default: return 0; 14299 } 14300} 14301 14302// FastEmit functions for AArch64ISD::UQSHL_I. 14303 14304unsigned fastEmit_AArch64ISD_UQSHL_I_MVT_v8i8_ri_Predicate_vecshiftL8(MVT RetVT, unsigned Op0, uint64_t imm1) { 14305 if (RetVT.SimpleTy != MVT::v8i8) 14306 return 0; 14307 if ((Subtarget->hasNEON())) { 14308 return fastEmitInst_ri(AArch64::UQSHLv8i8_shift, &AArch64::FPR64RegClass, Op0, imm1); 14309 } 14310 return 0; 14311} 14312 14313unsigned fastEmit_AArch64ISD_UQSHL_I_MVT_v16i8_ri_Predicate_vecshiftL8(MVT RetVT, unsigned Op0, uint64_t imm1) { 14314 if (RetVT.SimpleTy != MVT::v16i8) 14315 return 0; 14316 if ((Subtarget->hasNEON())) { 14317 return fastEmitInst_ri(AArch64::UQSHLv16i8_shift, &AArch64::FPR128RegClass, Op0, imm1); 14318 } 14319 return 0; 14320} 14321 14322unsigned fastEmit_AArch64ISD_UQSHL_I_ri_Predicate_vecshiftL8(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 14323 switch (VT.SimpleTy) { 14324 case MVT::v8i8: return fastEmit_AArch64ISD_UQSHL_I_MVT_v8i8_ri_Predicate_vecshiftL8(RetVT, Op0, imm1); 14325 case MVT::v16i8: return fastEmit_AArch64ISD_UQSHL_I_MVT_v16i8_ri_Predicate_vecshiftL8(RetVT, Op0, imm1); 14326 default: return 0; 14327 } 14328} 14329 14330// FastEmit functions for AArch64ISD::VSHL. 14331 14332unsigned fastEmit_AArch64ISD_VSHL_MVT_v8i8_ri_Predicate_vecshiftL8(MVT RetVT, unsigned Op0, uint64_t imm1) { 14333 if (RetVT.SimpleTy != MVT::v8i8) 14334 return 0; 14335 if ((Subtarget->hasNEON())) { 14336 return fastEmitInst_ri(AArch64::SHLv8i8_shift, &AArch64::FPR64RegClass, Op0, imm1); 14337 } 14338 return 0; 14339} 14340 14341unsigned fastEmit_AArch64ISD_VSHL_MVT_v16i8_ri_Predicate_vecshiftL8(MVT RetVT, unsigned Op0, uint64_t imm1) { 14342 if (RetVT.SimpleTy != MVT::v16i8) 14343 return 0; 14344 if ((Subtarget->hasNEON())) { 14345 return fastEmitInst_ri(AArch64::SHLv16i8_shift, &AArch64::FPR128RegClass, Op0, imm1); 14346 } 14347 return 0; 14348} 14349 14350unsigned fastEmit_AArch64ISD_VSHL_ri_Predicate_vecshiftL8(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 14351 switch (VT.SimpleTy) { 14352 case MVT::v8i8: return fastEmit_AArch64ISD_VSHL_MVT_v8i8_ri_Predicate_vecshiftL8(RetVT, Op0, imm1); 14353 case MVT::v16i8: return fastEmit_AArch64ISD_VSHL_MVT_v16i8_ri_Predicate_vecshiftL8(RetVT, Op0, imm1); 14354 default: return 0; 14355 } 14356} 14357 14358// Top-level FastEmit function. 14359 14360unsigned fastEmit_ri_Predicate_vecshiftL8(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) { 14361 switch (Opcode) { 14362 case AArch64ISD::SQSHLU_I: return fastEmit_AArch64ISD_SQSHLU_I_ri_Predicate_vecshiftL8(VT, RetVT, Op0, imm1); 14363 case AArch64ISD::SQSHL_I: return fastEmit_AArch64ISD_SQSHL_I_ri_Predicate_vecshiftL8(VT, RetVT, Op0, imm1); 14364 case AArch64ISD::UQSHL_I: return fastEmit_AArch64ISD_UQSHL_I_ri_Predicate_vecshiftL8(VT, RetVT, Op0, imm1); 14365 case AArch64ISD::VSHL: return fastEmit_AArch64ISD_VSHL_ri_Predicate_vecshiftL8(VT, RetVT, Op0, imm1); 14366 default: return 0; 14367 } 14368} 14369 14370// FastEmit functions for AArch64ISD::SQSHLU_I. 14371 14372unsigned fastEmit_AArch64ISD_SQSHLU_I_MVT_v4i16_ri_Predicate_vecshiftL16(MVT RetVT, unsigned Op0, uint64_t imm1) { 14373 if (RetVT.SimpleTy != MVT::v4i16) 14374 return 0; 14375 if ((Subtarget->hasNEON())) { 14376 return fastEmitInst_ri(AArch64::SQSHLUv4i16_shift, &AArch64::FPR64RegClass, Op0, imm1); 14377 } 14378 return 0; 14379} 14380 14381unsigned fastEmit_AArch64ISD_SQSHLU_I_MVT_v8i16_ri_Predicate_vecshiftL16(MVT RetVT, unsigned Op0, uint64_t imm1) { 14382 if (RetVT.SimpleTy != MVT::v8i16) 14383 return 0; 14384 if ((Subtarget->hasNEON())) { 14385 return fastEmitInst_ri(AArch64::SQSHLUv8i16_shift, &AArch64::FPR128RegClass, Op0, imm1); 14386 } 14387 return 0; 14388} 14389 14390unsigned fastEmit_AArch64ISD_SQSHLU_I_ri_Predicate_vecshiftL16(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 14391 switch (VT.SimpleTy) { 14392 case MVT::v4i16: return fastEmit_AArch64ISD_SQSHLU_I_MVT_v4i16_ri_Predicate_vecshiftL16(RetVT, Op0, imm1); 14393 case MVT::v8i16: return fastEmit_AArch64ISD_SQSHLU_I_MVT_v8i16_ri_Predicate_vecshiftL16(RetVT, Op0, imm1); 14394 default: return 0; 14395 } 14396} 14397 14398// FastEmit functions for AArch64ISD::SQSHL_I. 14399 14400unsigned fastEmit_AArch64ISD_SQSHL_I_MVT_v4i16_ri_Predicate_vecshiftL16(MVT RetVT, unsigned Op0, uint64_t imm1) { 14401 if (RetVT.SimpleTy != MVT::v4i16) 14402 return 0; 14403 if ((Subtarget->hasNEON())) { 14404 return fastEmitInst_ri(AArch64::SQSHLv4i16_shift, &AArch64::FPR64RegClass, Op0, imm1); 14405 } 14406 return 0; 14407} 14408 14409unsigned fastEmit_AArch64ISD_SQSHL_I_MVT_v8i16_ri_Predicate_vecshiftL16(MVT RetVT, unsigned Op0, uint64_t imm1) { 14410 if (RetVT.SimpleTy != MVT::v8i16) 14411 return 0; 14412 if ((Subtarget->hasNEON())) { 14413 return fastEmitInst_ri(AArch64::SQSHLv8i16_shift, &AArch64::FPR128RegClass, Op0, imm1); 14414 } 14415 return 0; 14416} 14417 14418unsigned fastEmit_AArch64ISD_SQSHL_I_ri_Predicate_vecshiftL16(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 14419 switch (VT.SimpleTy) { 14420 case MVT::v4i16: return fastEmit_AArch64ISD_SQSHL_I_MVT_v4i16_ri_Predicate_vecshiftL16(RetVT, Op0, imm1); 14421 case MVT::v8i16: return fastEmit_AArch64ISD_SQSHL_I_MVT_v8i16_ri_Predicate_vecshiftL16(RetVT, Op0, imm1); 14422 default: return 0; 14423 } 14424} 14425 14426// FastEmit functions for AArch64ISD::UQSHL_I. 14427 14428unsigned fastEmit_AArch64ISD_UQSHL_I_MVT_v4i16_ri_Predicate_vecshiftL16(MVT RetVT, unsigned Op0, uint64_t imm1) { 14429 if (RetVT.SimpleTy != MVT::v4i16) 14430 return 0; 14431 if ((Subtarget->hasNEON())) { 14432 return fastEmitInst_ri(AArch64::UQSHLv4i16_shift, &AArch64::FPR64RegClass, Op0, imm1); 14433 } 14434 return 0; 14435} 14436 14437unsigned fastEmit_AArch64ISD_UQSHL_I_MVT_v8i16_ri_Predicate_vecshiftL16(MVT RetVT, unsigned Op0, uint64_t imm1) { 14438 if (RetVT.SimpleTy != MVT::v8i16) 14439 return 0; 14440 if ((Subtarget->hasNEON())) { 14441 return fastEmitInst_ri(AArch64::UQSHLv8i16_shift, &AArch64::FPR128RegClass, Op0, imm1); 14442 } 14443 return 0; 14444} 14445 14446unsigned fastEmit_AArch64ISD_UQSHL_I_ri_Predicate_vecshiftL16(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 14447 switch (VT.SimpleTy) { 14448 case MVT::v4i16: return fastEmit_AArch64ISD_UQSHL_I_MVT_v4i16_ri_Predicate_vecshiftL16(RetVT, Op0, imm1); 14449 case MVT::v8i16: return fastEmit_AArch64ISD_UQSHL_I_MVT_v8i16_ri_Predicate_vecshiftL16(RetVT, Op0, imm1); 14450 default: return 0; 14451 } 14452} 14453 14454// FastEmit functions for AArch64ISD::VSHL. 14455 14456unsigned fastEmit_AArch64ISD_VSHL_MVT_v4i16_ri_Predicate_vecshiftL16(MVT RetVT, unsigned Op0, uint64_t imm1) { 14457 if (RetVT.SimpleTy != MVT::v4i16) 14458 return 0; 14459 if ((Subtarget->hasNEON())) { 14460 return fastEmitInst_ri(AArch64::SHLv4i16_shift, &AArch64::FPR64RegClass, Op0, imm1); 14461 } 14462 return 0; 14463} 14464 14465unsigned fastEmit_AArch64ISD_VSHL_MVT_v8i16_ri_Predicate_vecshiftL16(MVT RetVT, unsigned Op0, uint64_t imm1) { 14466 if (RetVT.SimpleTy != MVT::v8i16) 14467 return 0; 14468 if ((Subtarget->hasNEON())) { 14469 return fastEmitInst_ri(AArch64::SHLv8i16_shift, &AArch64::FPR128RegClass, Op0, imm1); 14470 } 14471 return 0; 14472} 14473 14474unsigned fastEmit_AArch64ISD_VSHL_ri_Predicate_vecshiftL16(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 14475 switch (VT.SimpleTy) { 14476 case MVT::v4i16: return fastEmit_AArch64ISD_VSHL_MVT_v4i16_ri_Predicate_vecshiftL16(RetVT, Op0, imm1); 14477 case MVT::v8i16: return fastEmit_AArch64ISD_VSHL_MVT_v8i16_ri_Predicate_vecshiftL16(RetVT, Op0, imm1); 14478 default: return 0; 14479 } 14480} 14481 14482// Top-level FastEmit function. 14483 14484unsigned fastEmit_ri_Predicate_vecshiftL16(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) { 14485 switch (Opcode) { 14486 case AArch64ISD::SQSHLU_I: return fastEmit_AArch64ISD_SQSHLU_I_ri_Predicate_vecshiftL16(VT, RetVT, Op0, imm1); 14487 case AArch64ISD::SQSHL_I: return fastEmit_AArch64ISD_SQSHL_I_ri_Predicate_vecshiftL16(VT, RetVT, Op0, imm1); 14488 case AArch64ISD::UQSHL_I: return fastEmit_AArch64ISD_UQSHL_I_ri_Predicate_vecshiftL16(VT, RetVT, Op0, imm1); 14489 case AArch64ISD::VSHL: return fastEmit_AArch64ISD_VSHL_ri_Predicate_vecshiftL16(VT, RetVT, Op0, imm1); 14490 default: return 0; 14491 } 14492} 14493 14494// FastEmit functions for AArch64ISD::SRSHR_I. 14495 14496unsigned fastEmit_AArch64ISD_SRSHR_I_MVT_v8i8_ri_Predicate_vecshiftR8(MVT RetVT, unsigned Op0, uint64_t imm1) { 14497 if (RetVT.SimpleTy != MVT::v8i8) 14498 return 0; 14499 if ((Subtarget->hasNEON())) { 14500 return fastEmitInst_ri(AArch64::SRSHRv8i8_shift, &AArch64::FPR64RegClass, Op0, imm1); 14501 } 14502 return 0; 14503} 14504 14505unsigned fastEmit_AArch64ISD_SRSHR_I_MVT_v16i8_ri_Predicate_vecshiftR8(MVT RetVT, unsigned Op0, uint64_t imm1) { 14506 if (RetVT.SimpleTy != MVT::v16i8) 14507 return 0; 14508 if ((Subtarget->hasNEON())) { 14509 return fastEmitInst_ri(AArch64::SRSHRv16i8_shift, &AArch64::FPR128RegClass, Op0, imm1); 14510 } 14511 return 0; 14512} 14513 14514unsigned fastEmit_AArch64ISD_SRSHR_I_ri_Predicate_vecshiftR8(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 14515 switch (VT.SimpleTy) { 14516 case MVT::v8i8: return fastEmit_AArch64ISD_SRSHR_I_MVT_v8i8_ri_Predicate_vecshiftR8(RetVT, Op0, imm1); 14517 case MVT::v16i8: return fastEmit_AArch64ISD_SRSHR_I_MVT_v16i8_ri_Predicate_vecshiftR8(RetVT, Op0, imm1); 14518 default: return 0; 14519 } 14520} 14521 14522// FastEmit functions for AArch64ISD::URSHR_I. 14523 14524unsigned fastEmit_AArch64ISD_URSHR_I_MVT_v8i8_ri_Predicate_vecshiftR8(MVT RetVT, unsigned Op0, uint64_t imm1) { 14525 if (RetVT.SimpleTy != MVT::v8i8) 14526 return 0; 14527 if ((Subtarget->hasNEON())) { 14528 return fastEmitInst_ri(AArch64::URSHRv8i8_shift, &AArch64::FPR64RegClass, Op0, imm1); 14529 } 14530 return 0; 14531} 14532 14533unsigned fastEmit_AArch64ISD_URSHR_I_MVT_v16i8_ri_Predicate_vecshiftR8(MVT RetVT, unsigned Op0, uint64_t imm1) { 14534 if (RetVT.SimpleTy != MVT::v16i8) 14535 return 0; 14536 if ((Subtarget->hasNEON())) { 14537 return fastEmitInst_ri(AArch64::URSHRv16i8_shift, &AArch64::FPR128RegClass, Op0, imm1); 14538 } 14539 return 0; 14540} 14541 14542unsigned fastEmit_AArch64ISD_URSHR_I_ri_Predicate_vecshiftR8(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 14543 switch (VT.SimpleTy) { 14544 case MVT::v8i8: return fastEmit_AArch64ISD_URSHR_I_MVT_v8i8_ri_Predicate_vecshiftR8(RetVT, Op0, imm1); 14545 case MVT::v16i8: return fastEmit_AArch64ISD_URSHR_I_MVT_v16i8_ri_Predicate_vecshiftR8(RetVT, Op0, imm1); 14546 default: return 0; 14547 } 14548} 14549 14550// FastEmit functions for AArch64ISD::VASHR. 14551 14552unsigned fastEmit_AArch64ISD_VASHR_MVT_v8i8_ri_Predicate_vecshiftR8(MVT RetVT, unsigned Op0, uint64_t imm1) { 14553 if (RetVT.SimpleTy != MVT::v8i8) 14554 return 0; 14555 if ((Subtarget->hasNEON())) { 14556 return fastEmitInst_ri(AArch64::SSHRv8i8_shift, &AArch64::FPR64RegClass, Op0, imm1); 14557 } 14558 return 0; 14559} 14560 14561unsigned fastEmit_AArch64ISD_VASHR_MVT_v16i8_ri_Predicate_vecshiftR8(MVT RetVT, unsigned Op0, uint64_t imm1) { 14562 if (RetVT.SimpleTy != MVT::v16i8) 14563 return 0; 14564 if ((Subtarget->hasNEON())) { 14565 return fastEmitInst_ri(AArch64::SSHRv16i8_shift, &AArch64::FPR128RegClass, Op0, imm1); 14566 } 14567 return 0; 14568} 14569 14570unsigned fastEmit_AArch64ISD_VASHR_ri_Predicate_vecshiftR8(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 14571 switch (VT.SimpleTy) { 14572 case MVT::v8i8: return fastEmit_AArch64ISD_VASHR_MVT_v8i8_ri_Predicate_vecshiftR8(RetVT, Op0, imm1); 14573 case MVT::v16i8: return fastEmit_AArch64ISD_VASHR_MVT_v16i8_ri_Predicate_vecshiftR8(RetVT, Op0, imm1); 14574 default: return 0; 14575 } 14576} 14577 14578// FastEmit functions for AArch64ISD::VLSHR. 14579 14580unsigned fastEmit_AArch64ISD_VLSHR_MVT_v8i8_ri_Predicate_vecshiftR8(MVT RetVT, unsigned Op0, uint64_t imm1) { 14581 if (RetVT.SimpleTy != MVT::v8i8) 14582 return 0; 14583 if ((Subtarget->hasNEON())) { 14584 return fastEmitInst_ri(AArch64::USHRv8i8_shift, &AArch64::FPR64RegClass, Op0, imm1); 14585 } 14586 return 0; 14587} 14588 14589unsigned fastEmit_AArch64ISD_VLSHR_MVT_v16i8_ri_Predicate_vecshiftR8(MVT RetVT, unsigned Op0, uint64_t imm1) { 14590 if (RetVT.SimpleTy != MVT::v16i8) 14591 return 0; 14592 if ((Subtarget->hasNEON())) { 14593 return fastEmitInst_ri(AArch64::USHRv16i8_shift, &AArch64::FPR128RegClass, Op0, imm1); 14594 } 14595 return 0; 14596} 14597 14598unsigned fastEmit_AArch64ISD_VLSHR_ri_Predicate_vecshiftR8(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 14599 switch (VT.SimpleTy) { 14600 case MVT::v8i8: return fastEmit_AArch64ISD_VLSHR_MVT_v8i8_ri_Predicate_vecshiftR8(RetVT, Op0, imm1); 14601 case MVT::v16i8: return fastEmit_AArch64ISD_VLSHR_MVT_v16i8_ri_Predicate_vecshiftR8(RetVT, Op0, imm1); 14602 default: return 0; 14603 } 14604} 14605 14606// Top-level FastEmit function. 14607 14608unsigned fastEmit_ri_Predicate_vecshiftR8(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) { 14609 switch (Opcode) { 14610 case AArch64ISD::SRSHR_I: return fastEmit_AArch64ISD_SRSHR_I_ri_Predicate_vecshiftR8(VT, RetVT, Op0, imm1); 14611 case AArch64ISD::URSHR_I: return fastEmit_AArch64ISD_URSHR_I_ri_Predicate_vecshiftR8(VT, RetVT, Op0, imm1); 14612 case AArch64ISD::VASHR: return fastEmit_AArch64ISD_VASHR_ri_Predicate_vecshiftR8(VT, RetVT, Op0, imm1); 14613 case AArch64ISD::VLSHR: return fastEmit_AArch64ISD_VLSHR_ri_Predicate_vecshiftR8(VT, RetVT, Op0, imm1); 14614 default: return 0; 14615 } 14616} 14617 14618// FastEmit functions for AArch64ISD::SRSHR_I. 14619 14620unsigned fastEmit_AArch64ISD_SRSHR_I_MVT_v4i16_ri_Predicate_vecshiftR16(MVT RetVT, unsigned Op0, uint64_t imm1) { 14621 if (RetVT.SimpleTy != MVT::v4i16) 14622 return 0; 14623 if ((Subtarget->hasNEON())) { 14624 return fastEmitInst_ri(AArch64::SRSHRv4i16_shift, &AArch64::FPR64RegClass, Op0, imm1); 14625 } 14626 return 0; 14627} 14628 14629unsigned fastEmit_AArch64ISD_SRSHR_I_MVT_v8i16_ri_Predicate_vecshiftR16(MVT RetVT, unsigned Op0, uint64_t imm1) { 14630 if (RetVT.SimpleTy != MVT::v8i16) 14631 return 0; 14632 if ((Subtarget->hasNEON())) { 14633 return fastEmitInst_ri(AArch64::SRSHRv8i16_shift, &AArch64::FPR128RegClass, Op0, imm1); 14634 } 14635 return 0; 14636} 14637 14638unsigned fastEmit_AArch64ISD_SRSHR_I_ri_Predicate_vecshiftR16(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 14639 switch (VT.SimpleTy) { 14640 case MVT::v4i16: return fastEmit_AArch64ISD_SRSHR_I_MVT_v4i16_ri_Predicate_vecshiftR16(RetVT, Op0, imm1); 14641 case MVT::v8i16: return fastEmit_AArch64ISD_SRSHR_I_MVT_v8i16_ri_Predicate_vecshiftR16(RetVT, Op0, imm1); 14642 default: return 0; 14643 } 14644} 14645 14646// FastEmit functions for AArch64ISD::URSHR_I. 14647 14648unsigned fastEmit_AArch64ISD_URSHR_I_MVT_v4i16_ri_Predicate_vecshiftR16(MVT RetVT, unsigned Op0, uint64_t imm1) { 14649 if (RetVT.SimpleTy != MVT::v4i16) 14650 return 0; 14651 if ((Subtarget->hasNEON())) { 14652 return fastEmitInst_ri(AArch64::URSHRv4i16_shift, &AArch64::FPR64RegClass, Op0, imm1); 14653 } 14654 return 0; 14655} 14656 14657unsigned fastEmit_AArch64ISD_URSHR_I_MVT_v8i16_ri_Predicate_vecshiftR16(MVT RetVT, unsigned Op0, uint64_t imm1) { 14658 if (RetVT.SimpleTy != MVT::v8i16) 14659 return 0; 14660 if ((Subtarget->hasNEON())) { 14661 return fastEmitInst_ri(AArch64::URSHRv8i16_shift, &AArch64::FPR128RegClass, Op0, imm1); 14662 } 14663 return 0; 14664} 14665 14666unsigned fastEmit_AArch64ISD_URSHR_I_ri_Predicate_vecshiftR16(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 14667 switch (VT.SimpleTy) { 14668 case MVT::v4i16: return fastEmit_AArch64ISD_URSHR_I_MVT_v4i16_ri_Predicate_vecshiftR16(RetVT, Op0, imm1); 14669 case MVT::v8i16: return fastEmit_AArch64ISD_URSHR_I_MVT_v8i16_ri_Predicate_vecshiftR16(RetVT, Op0, imm1); 14670 default: return 0; 14671 } 14672} 14673 14674// FastEmit functions for AArch64ISD::VASHR. 14675 14676unsigned fastEmit_AArch64ISD_VASHR_MVT_v4i16_ri_Predicate_vecshiftR16(MVT RetVT, unsigned Op0, uint64_t imm1) { 14677 if (RetVT.SimpleTy != MVT::v4i16) 14678 return 0; 14679 if ((Subtarget->hasNEON())) { 14680 return fastEmitInst_ri(AArch64::SSHRv4i16_shift, &AArch64::FPR64RegClass, Op0, imm1); 14681 } 14682 return 0; 14683} 14684 14685unsigned fastEmit_AArch64ISD_VASHR_MVT_v8i16_ri_Predicate_vecshiftR16(MVT RetVT, unsigned Op0, uint64_t imm1) { 14686 if (RetVT.SimpleTy != MVT::v8i16) 14687 return 0; 14688 if ((Subtarget->hasNEON())) { 14689 return fastEmitInst_ri(AArch64::SSHRv8i16_shift, &AArch64::FPR128RegClass, Op0, imm1); 14690 } 14691 return 0; 14692} 14693 14694unsigned fastEmit_AArch64ISD_VASHR_ri_Predicate_vecshiftR16(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 14695 switch (VT.SimpleTy) { 14696 case MVT::v4i16: return fastEmit_AArch64ISD_VASHR_MVT_v4i16_ri_Predicate_vecshiftR16(RetVT, Op0, imm1); 14697 case MVT::v8i16: return fastEmit_AArch64ISD_VASHR_MVT_v8i16_ri_Predicate_vecshiftR16(RetVT, Op0, imm1); 14698 default: return 0; 14699 } 14700} 14701 14702// FastEmit functions for AArch64ISD::VLSHR. 14703 14704unsigned fastEmit_AArch64ISD_VLSHR_MVT_v4i16_ri_Predicate_vecshiftR16(MVT RetVT, unsigned Op0, uint64_t imm1) { 14705 if (RetVT.SimpleTy != MVT::v4i16) 14706 return 0; 14707 if ((Subtarget->hasNEON())) { 14708 return fastEmitInst_ri(AArch64::USHRv4i16_shift, &AArch64::FPR64RegClass, Op0, imm1); 14709 } 14710 return 0; 14711} 14712 14713unsigned fastEmit_AArch64ISD_VLSHR_MVT_v8i16_ri_Predicate_vecshiftR16(MVT RetVT, unsigned Op0, uint64_t imm1) { 14714 if (RetVT.SimpleTy != MVT::v8i16) 14715 return 0; 14716 if ((Subtarget->hasNEON())) { 14717 return fastEmitInst_ri(AArch64::USHRv8i16_shift, &AArch64::FPR128RegClass, Op0, imm1); 14718 } 14719 return 0; 14720} 14721 14722unsigned fastEmit_AArch64ISD_VLSHR_ri_Predicate_vecshiftR16(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 14723 switch (VT.SimpleTy) { 14724 case MVT::v4i16: return fastEmit_AArch64ISD_VLSHR_MVT_v4i16_ri_Predicate_vecshiftR16(RetVT, Op0, imm1); 14725 case MVT::v8i16: return fastEmit_AArch64ISD_VLSHR_MVT_v8i16_ri_Predicate_vecshiftR16(RetVT, Op0, imm1); 14726 default: return 0; 14727 } 14728} 14729 14730// Top-level FastEmit function. 14731 14732unsigned fastEmit_ri_Predicate_vecshiftR16(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) { 14733 switch (Opcode) { 14734 case AArch64ISD::SRSHR_I: return fastEmit_AArch64ISD_SRSHR_I_ri_Predicate_vecshiftR16(VT, RetVT, Op0, imm1); 14735 case AArch64ISD::URSHR_I: return fastEmit_AArch64ISD_URSHR_I_ri_Predicate_vecshiftR16(VT, RetVT, Op0, imm1); 14736 case AArch64ISD::VASHR: return fastEmit_AArch64ISD_VASHR_ri_Predicate_vecshiftR16(VT, RetVT, Op0, imm1); 14737 case AArch64ISD::VLSHR: return fastEmit_AArch64ISD_VLSHR_ri_Predicate_vecshiftR16(VT, RetVT, Op0, imm1); 14738 default: return 0; 14739 } 14740} 14741 14742// FastEmit functions for AArch64ISD::SRSHR_I. 14743 14744unsigned fastEmit_AArch64ISD_SRSHR_I_MVT_v2i32_ri_Predicate_vecshiftR32(MVT RetVT, unsigned Op0, uint64_t imm1) { 14745 if (RetVT.SimpleTy != MVT::v2i32) 14746 return 0; 14747 if ((Subtarget->hasNEON())) { 14748 return fastEmitInst_ri(AArch64::SRSHRv2i32_shift, &AArch64::FPR64RegClass, Op0, imm1); 14749 } 14750 return 0; 14751} 14752 14753unsigned fastEmit_AArch64ISD_SRSHR_I_MVT_v4i32_ri_Predicate_vecshiftR32(MVT RetVT, unsigned Op0, uint64_t imm1) { 14754 if (RetVT.SimpleTy != MVT::v4i32) 14755 return 0; 14756 if ((Subtarget->hasNEON())) { 14757 return fastEmitInst_ri(AArch64::SRSHRv4i32_shift, &AArch64::FPR128RegClass, Op0, imm1); 14758 } 14759 return 0; 14760} 14761 14762unsigned fastEmit_AArch64ISD_SRSHR_I_ri_Predicate_vecshiftR32(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 14763 switch (VT.SimpleTy) { 14764 case MVT::v2i32: return fastEmit_AArch64ISD_SRSHR_I_MVT_v2i32_ri_Predicate_vecshiftR32(RetVT, Op0, imm1); 14765 case MVT::v4i32: return fastEmit_AArch64ISD_SRSHR_I_MVT_v4i32_ri_Predicate_vecshiftR32(RetVT, Op0, imm1); 14766 default: return 0; 14767 } 14768} 14769 14770// FastEmit functions for AArch64ISD::URSHR_I. 14771 14772unsigned fastEmit_AArch64ISD_URSHR_I_MVT_v2i32_ri_Predicate_vecshiftR32(MVT RetVT, unsigned Op0, uint64_t imm1) { 14773 if (RetVT.SimpleTy != MVT::v2i32) 14774 return 0; 14775 if ((Subtarget->hasNEON())) { 14776 return fastEmitInst_ri(AArch64::URSHRv2i32_shift, &AArch64::FPR64RegClass, Op0, imm1); 14777 } 14778 return 0; 14779} 14780 14781unsigned fastEmit_AArch64ISD_URSHR_I_MVT_v4i32_ri_Predicate_vecshiftR32(MVT RetVT, unsigned Op0, uint64_t imm1) { 14782 if (RetVT.SimpleTy != MVT::v4i32) 14783 return 0; 14784 if ((Subtarget->hasNEON())) { 14785 return fastEmitInst_ri(AArch64::URSHRv4i32_shift, &AArch64::FPR128RegClass, Op0, imm1); 14786 } 14787 return 0; 14788} 14789 14790unsigned fastEmit_AArch64ISD_URSHR_I_ri_Predicate_vecshiftR32(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 14791 switch (VT.SimpleTy) { 14792 case MVT::v2i32: return fastEmit_AArch64ISD_URSHR_I_MVT_v2i32_ri_Predicate_vecshiftR32(RetVT, Op0, imm1); 14793 case MVT::v4i32: return fastEmit_AArch64ISD_URSHR_I_MVT_v4i32_ri_Predicate_vecshiftR32(RetVT, Op0, imm1); 14794 default: return 0; 14795 } 14796} 14797 14798// FastEmit functions for AArch64ISD::VASHR. 14799 14800unsigned fastEmit_AArch64ISD_VASHR_MVT_v2i32_ri_Predicate_vecshiftR32(MVT RetVT, unsigned Op0, uint64_t imm1) { 14801 if (RetVT.SimpleTy != MVT::v2i32) 14802 return 0; 14803 if ((Subtarget->hasNEON())) { 14804 return fastEmitInst_ri(AArch64::SSHRv2i32_shift, &AArch64::FPR64RegClass, Op0, imm1); 14805 } 14806 return 0; 14807} 14808 14809unsigned fastEmit_AArch64ISD_VASHR_MVT_v4i32_ri_Predicate_vecshiftR32(MVT RetVT, unsigned Op0, uint64_t imm1) { 14810 if (RetVT.SimpleTy != MVT::v4i32) 14811 return 0; 14812 if ((Subtarget->hasNEON())) { 14813 return fastEmitInst_ri(AArch64::SSHRv4i32_shift, &AArch64::FPR128RegClass, Op0, imm1); 14814 } 14815 return 0; 14816} 14817 14818unsigned fastEmit_AArch64ISD_VASHR_ri_Predicate_vecshiftR32(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 14819 switch (VT.SimpleTy) { 14820 case MVT::v2i32: return fastEmit_AArch64ISD_VASHR_MVT_v2i32_ri_Predicate_vecshiftR32(RetVT, Op0, imm1); 14821 case MVT::v4i32: return fastEmit_AArch64ISD_VASHR_MVT_v4i32_ri_Predicate_vecshiftR32(RetVT, Op0, imm1); 14822 default: return 0; 14823 } 14824} 14825 14826// FastEmit functions for AArch64ISD::VLSHR. 14827 14828unsigned fastEmit_AArch64ISD_VLSHR_MVT_v2i32_ri_Predicate_vecshiftR32(MVT RetVT, unsigned Op0, uint64_t imm1) { 14829 if (RetVT.SimpleTy != MVT::v2i32) 14830 return 0; 14831 if ((Subtarget->hasNEON())) { 14832 return fastEmitInst_ri(AArch64::USHRv2i32_shift, &AArch64::FPR64RegClass, Op0, imm1); 14833 } 14834 return 0; 14835} 14836 14837unsigned fastEmit_AArch64ISD_VLSHR_MVT_v4i32_ri_Predicate_vecshiftR32(MVT RetVT, unsigned Op0, uint64_t imm1) { 14838 if (RetVT.SimpleTy != MVT::v4i32) 14839 return 0; 14840 if ((Subtarget->hasNEON())) { 14841 return fastEmitInst_ri(AArch64::USHRv4i32_shift, &AArch64::FPR128RegClass, Op0, imm1); 14842 } 14843 return 0; 14844} 14845 14846unsigned fastEmit_AArch64ISD_VLSHR_ri_Predicate_vecshiftR32(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 14847 switch (VT.SimpleTy) { 14848 case MVT::v2i32: return fastEmit_AArch64ISD_VLSHR_MVT_v2i32_ri_Predicate_vecshiftR32(RetVT, Op0, imm1); 14849 case MVT::v4i32: return fastEmit_AArch64ISD_VLSHR_MVT_v4i32_ri_Predicate_vecshiftR32(RetVT, Op0, imm1); 14850 default: return 0; 14851 } 14852} 14853 14854// Top-level FastEmit function. 14855 14856unsigned fastEmit_ri_Predicate_vecshiftR32(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) { 14857 switch (Opcode) { 14858 case AArch64ISD::SRSHR_I: return fastEmit_AArch64ISD_SRSHR_I_ri_Predicate_vecshiftR32(VT, RetVT, Op0, imm1); 14859 case AArch64ISD::URSHR_I: return fastEmit_AArch64ISD_URSHR_I_ri_Predicate_vecshiftR32(VT, RetVT, Op0, imm1); 14860 case AArch64ISD::VASHR: return fastEmit_AArch64ISD_VASHR_ri_Predicate_vecshiftR32(VT, RetVT, Op0, imm1); 14861 case AArch64ISD::VLSHR: return fastEmit_AArch64ISD_VLSHR_ri_Predicate_vecshiftR32(VT, RetVT, Op0, imm1); 14862 default: return 0; 14863 } 14864} 14865 14866// FastEmit functions for ISD::SMAX. 14867 14868unsigned fastEmit_ISD_SMAX_MVT_i32_ri_Predicate_simm8_32b(MVT RetVT, unsigned Op0, uint64_t imm1) { 14869 if (RetVT.SimpleTy != MVT::i32) 14870 return 0; 14871 if ((Subtarget->hasCSSC())) { 14872 return fastEmitInst_ri(AArch64::SMAXWri, &AArch64::GPR32RegClass, Op0, imm1); 14873 } 14874 return 0; 14875} 14876 14877unsigned fastEmit_ISD_SMAX_ri_Predicate_simm8_32b(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 14878 switch (VT.SimpleTy) { 14879 case MVT::i32: return fastEmit_ISD_SMAX_MVT_i32_ri_Predicate_simm8_32b(RetVT, Op0, imm1); 14880 default: return 0; 14881 } 14882} 14883 14884// FastEmit functions for ISD::SMIN. 14885 14886unsigned fastEmit_ISD_SMIN_MVT_i32_ri_Predicate_simm8_32b(MVT RetVT, unsigned Op0, uint64_t imm1) { 14887 if (RetVT.SimpleTy != MVT::i32) 14888 return 0; 14889 if ((Subtarget->hasCSSC())) { 14890 return fastEmitInst_ri(AArch64::SMINWri, &AArch64::GPR32RegClass, Op0, imm1); 14891 } 14892 return 0; 14893} 14894 14895unsigned fastEmit_ISD_SMIN_ri_Predicate_simm8_32b(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 14896 switch (VT.SimpleTy) { 14897 case MVT::i32: return fastEmit_ISD_SMIN_MVT_i32_ri_Predicate_simm8_32b(RetVT, Op0, imm1); 14898 default: return 0; 14899 } 14900} 14901 14902// Top-level FastEmit function. 14903 14904unsigned fastEmit_ri_Predicate_simm8_32b(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) { 14905 switch (Opcode) { 14906 case ISD::SMAX: return fastEmit_ISD_SMAX_ri_Predicate_simm8_32b(VT, RetVT, Op0, imm1); 14907 case ISD::SMIN: return fastEmit_ISD_SMIN_ri_Predicate_simm8_32b(VT, RetVT, Op0, imm1); 14908 default: return 0; 14909 } 14910} 14911 14912// FastEmit functions for ISD::SMAX. 14913 14914unsigned fastEmit_ISD_SMAX_MVT_i64_ri_Predicate_simm8_64b(MVT RetVT, unsigned Op0, uint64_t imm1) { 14915 if (RetVT.SimpleTy != MVT::i64) 14916 return 0; 14917 if ((Subtarget->hasCSSC())) { 14918 return fastEmitInst_ri(AArch64::SMAXXri, &AArch64::GPR64RegClass, Op0, imm1); 14919 } 14920 return 0; 14921} 14922 14923unsigned fastEmit_ISD_SMAX_ri_Predicate_simm8_64b(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 14924 switch (VT.SimpleTy) { 14925 case MVT::i64: return fastEmit_ISD_SMAX_MVT_i64_ri_Predicate_simm8_64b(RetVT, Op0, imm1); 14926 default: return 0; 14927 } 14928} 14929 14930// FastEmit functions for ISD::SMIN. 14931 14932unsigned fastEmit_ISD_SMIN_MVT_i64_ri_Predicate_simm8_64b(MVT RetVT, unsigned Op0, uint64_t imm1) { 14933 if (RetVT.SimpleTy != MVT::i64) 14934 return 0; 14935 if ((Subtarget->hasCSSC())) { 14936 return fastEmitInst_ri(AArch64::SMINXri, &AArch64::GPR64RegClass, Op0, imm1); 14937 } 14938 return 0; 14939} 14940 14941unsigned fastEmit_ISD_SMIN_ri_Predicate_simm8_64b(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 14942 switch (VT.SimpleTy) { 14943 case MVT::i64: return fastEmit_ISD_SMIN_MVT_i64_ri_Predicate_simm8_64b(RetVT, Op0, imm1); 14944 default: return 0; 14945 } 14946} 14947 14948// Top-level FastEmit function. 14949 14950unsigned fastEmit_ri_Predicate_simm8_64b(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) { 14951 switch (Opcode) { 14952 case ISD::SMAX: return fastEmit_ISD_SMAX_ri_Predicate_simm8_64b(VT, RetVT, Op0, imm1); 14953 case ISD::SMIN: return fastEmit_ISD_SMIN_ri_Predicate_simm8_64b(VT, RetVT, Op0, imm1); 14954 default: return 0; 14955 } 14956} 14957 14958// FastEmit functions for ISD::UMAX. 14959 14960unsigned fastEmit_ISD_UMAX_MVT_i32_ri_Predicate_uimm8_32b(MVT RetVT, unsigned Op0, uint64_t imm1) { 14961 if (RetVT.SimpleTy != MVT::i32) 14962 return 0; 14963 if ((Subtarget->hasCSSC())) { 14964 return fastEmitInst_ri(AArch64::UMAXWri, &AArch64::GPR32RegClass, Op0, imm1); 14965 } 14966 return 0; 14967} 14968 14969unsigned fastEmit_ISD_UMAX_ri_Predicate_uimm8_32b(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 14970 switch (VT.SimpleTy) { 14971 case MVT::i32: return fastEmit_ISD_UMAX_MVT_i32_ri_Predicate_uimm8_32b(RetVT, Op0, imm1); 14972 default: return 0; 14973 } 14974} 14975 14976// FastEmit functions for ISD::UMIN. 14977 14978unsigned fastEmit_ISD_UMIN_MVT_i32_ri_Predicate_uimm8_32b(MVT RetVT, unsigned Op0, uint64_t imm1) { 14979 if (RetVT.SimpleTy != MVT::i32) 14980 return 0; 14981 if ((Subtarget->hasCSSC())) { 14982 return fastEmitInst_ri(AArch64::UMINWri, &AArch64::GPR32RegClass, Op0, imm1); 14983 } 14984 return 0; 14985} 14986 14987unsigned fastEmit_ISD_UMIN_ri_Predicate_uimm8_32b(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 14988 switch (VT.SimpleTy) { 14989 case MVT::i32: return fastEmit_ISD_UMIN_MVT_i32_ri_Predicate_uimm8_32b(RetVT, Op0, imm1); 14990 default: return 0; 14991 } 14992} 14993 14994// Top-level FastEmit function. 14995 14996unsigned fastEmit_ri_Predicate_uimm8_32b(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) { 14997 switch (Opcode) { 14998 case ISD::UMAX: return fastEmit_ISD_UMAX_ri_Predicate_uimm8_32b(VT, RetVT, Op0, imm1); 14999 case ISD::UMIN: return fastEmit_ISD_UMIN_ri_Predicate_uimm8_32b(VT, RetVT, Op0, imm1); 15000 default: return 0; 15001 } 15002} 15003 15004// FastEmit functions for ISD::UMAX. 15005 15006unsigned fastEmit_ISD_UMAX_MVT_i64_ri_Predicate_uimm8_64b(MVT RetVT, unsigned Op0, uint64_t imm1) { 15007 if (RetVT.SimpleTy != MVT::i64) 15008 return 0; 15009 if ((Subtarget->hasCSSC())) { 15010 return fastEmitInst_ri(AArch64::UMAXXri, &AArch64::GPR64RegClass, Op0, imm1); 15011 } 15012 return 0; 15013} 15014 15015unsigned fastEmit_ISD_UMAX_ri_Predicate_uimm8_64b(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 15016 switch (VT.SimpleTy) { 15017 case MVT::i64: return fastEmit_ISD_UMAX_MVT_i64_ri_Predicate_uimm8_64b(RetVT, Op0, imm1); 15018 default: return 0; 15019 } 15020} 15021 15022// FastEmit functions for ISD::UMIN. 15023 15024unsigned fastEmit_ISD_UMIN_MVT_i64_ri_Predicate_uimm8_64b(MVT RetVT, unsigned Op0, uint64_t imm1) { 15025 if (RetVT.SimpleTy != MVT::i64) 15026 return 0; 15027 if ((Subtarget->hasCSSC())) { 15028 return fastEmitInst_ri(AArch64::UMINXri, &AArch64::GPR64RegClass, Op0, imm1); 15029 } 15030 return 0; 15031} 15032 15033unsigned fastEmit_ISD_UMIN_ri_Predicate_uimm8_64b(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) { 15034 switch (VT.SimpleTy) { 15035 case MVT::i64: return fastEmit_ISD_UMIN_MVT_i64_ri_Predicate_uimm8_64b(RetVT, Op0, imm1); 15036 default: return 0; 15037 } 15038} 15039 15040// Top-level FastEmit function. 15041 15042unsigned fastEmit_ri_Predicate_uimm8_64b(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) { 15043 switch (Opcode) { 15044 case ISD::UMAX: return fastEmit_ISD_UMAX_ri_Predicate_uimm8_64b(VT, RetVT, Op0, imm1); 15045 case ISD::UMIN: return fastEmit_ISD_UMIN_ri_Predicate_uimm8_64b(VT, RetVT, Op0, imm1); 15046 default: return 0; 15047 } 15048} 15049 15050// FastEmit functions for AArch64ISD::MRS. 15051 15052unsigned fastEmit_AArch64ISD_MRS_MVT_i32_i(MVT RetVT, uint64_t imm0) { 15053 if (RetVT.SimpleTy != MVT::i64) 15054 return 0; 15055 return fastEmitInst_i(AArch64::MRS, &AArch64::GPR64RegClass, imm0); 15056} 15057 15058unsigned fastEmit_AArch64ISD_MRS_i(MVT VT, MVT RetVT, uint64_t imm0) { 15059 switch (VT.SimpleTy) { 15060 case MVT::i32: return fastEmit_AArch64ISD_MRS_MVT_i32_i(RetVT, imm0); 15061 default: return 0; 15062 } 15063} 15064 15065// FastEmit functions for ISD::Constant. 15066 15067unsigned fastEmit_ISD_Constant_MVT_i32_i(MVT RetVT, uint64_t imm0) { 15068 if (RetVT.SimpleTy != MVT::i32) 15069 return 0; 15070 return fastEmitInst_i(AArch64::MOVi32imm, &AArch64::GPR32RegClass, imm0); 15071} 15072 15073unsigned fastEmit_ISD_Constant_MVT_i64_i(MVT RetVT, uint64_t imm0) { 15074 if (RetVT.SimpleTy != MVT::i64) 15075 return 0; 15076 return fastEmitInst_i(AArch64::MOVi64imm, &AArch64::GPR64RegClass, imm0); 15077} 15078 15079unsigned fastEmit_ISD_Constant_i(MVT VT, MVT RetVT, uint64_t imm0) { 15080 switch (VT.SimpleTy) { 15081 case MVT::i32: return fastEmit_ISD_Constant_MVT_i32_i(RetVT, imm0); 15082 case MVT::i64: return fastEmit_ISD_Constant_MVT_i64_i(RetVT, imm0); 15083 default: return 0; 15084 } 15085} 15086 15087// Top-level FastEmit function. 15088 15089unsigned fastEmit_i(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) override { 15090 if (VT == MVT::i32 && Predicate_imm0_255(imm0)) 15091 if (unsigned Reg = fastEmit_i_Predicate_imm0_255(VT, RetVT, Opcode, imm0)) 15092 return Reg; 15093 15094 if (VT == MVT::i32 && Predicate_simm6_32b(imm0)) 15095 if (unsigned Reg = fastEmit_i_Predicate_simm6_32b(VT, RetVT, Opcode, imm0)) 15096 return Reg; 15097 15098 switch (Opcode) { 15099 case AArch64ISD::MRS: return fastEmit_AArch64ISD_MRS_i(VT, RetVT, imm0); 15100 case ISD::Constant: return fastEmit_ISD_Constant_i(VT, RetVT, imm0); 15101 default: return 0; 15102 } 15103} 15104 15105// FastEmit functions for AArch64ISD::FMOV. 15106 15107unsigned fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f16_i_Predicate_imm0_255(uint64_t imm0) { 15108 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 15109 return fastEmitInst_i(AArch64::FMOVv4f16_ns, &AArch64::FPR64RegClass, imm0); 15110 } 15111 return 0; 15112} 15113 15114unsigned fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v8f16_i_Predicate_imm0_255(uint64_t imm0) { 15115 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) { 15116 return fastEmitInst_i(AArch64::FMOVv8f16_ns, &AArch64::FPR128RegClass, imm0); 15117 } 15118 return 0; 15119} 15120 15121unsigned fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f32_i_Predicate_imm0_255(uint64_t imm0) { 15122 if ((Subtarget->hasNEON())) { 15123 return fastEmitInst_i(AArch64::FMOVv2f32_ns, &AArch64::FPR64RegClass, imm0); 15124 } 15125 return 0; 15126} 15127 15128unsigned fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f32_i_Predicate_imm0_255(uint64_t imm0) { 15129 if ((Subtarget->hasNEON())) { 15130 return fastEmitInst_i(AArch64::FMOVv4f32_ns, &AArch64::FPR128RegClass, imm0); 15131 } 15132 return 0; 15133} 15134 15135unsigned fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f64_i_Predicate_imm0_255(uint64_t imm0) { 15136 if ((Subtarget->hasNEON())) { 15137 return fastEmitInst_i(AArch64::FMOVv2f64_ns, &AArch64::FPR128RegClass, imm0); 15138 } 15139 return 0; 15140} 15141 15142unsigned fastEmit_AArch64ISD_FMOV_MVT_i32_i_Predicate_imm0_255(MVT RetVT, uint64_t imm0) { 15143switch (RetVT.SimpleTy) { 15144 case MVT::v4f16: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f16_i_Predicate_imm0_255(imm0); 15145 case MVT::v8f16: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v8f16_i_Predicate_imm0_255(imm0); 15146 case MVT::v2f32: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f32_i_Predicate_imm0_255(imm0); 15147 case MVT::v4f32: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f32_i_Predicate_imm0_255(imm0); 15148 case MVT::v2f64: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f64_i_Predicate_imm0_255(imm0); 15149 default: return 0; 15150} 15151} 15152 15153unsigned fastEmit_AArch64ISD_FMOV_i_Predicate_imm0_255(MVT VT, MVT RetVT, uint64_t imm0) { 15154 switch (VT.SimpleTy) { 15155 case MVT::i32: return fastEmit_AArch64ISD_FMOV_MVT_i32_i_Predicate_imm0_255(RetVT, imm0); 15156 default: return 0; 15157 } 15158} 15159 15160// FastEmit functions for AArch64ISD::MOVI. 15161 15162unsigned fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v8i8_i_Predicate_imm0_255(uint64_t imm0) { 15163 if ((Subtarget->hasNEON())) { 15164 return fastEmitInst_i(AArch64::MOVIv8b_ns, &AArch64::FPR64RegClass, imm0); 15165 } 15166 return 0; 15167} 15168 15169unsigned fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v16i8_i_Predicate_imm0_255(uint64_t imm0) { 15170 if ((Subtarget->hasNEON())) { 15171 return fastEmitInst_i(AArch64::MOVIv16b_ns, &AArch64::FPR128RegClass, imm0); 15172 } 15173 return 0; 15174} 15175 15176unsigned fastEmit_AArch64ISD_MOVI_MVT_i32_i_Predicate_imm0_255(MVT RetVT, uint64_t imm0) { 15177switch (RetVT.SimpleTy) { 15178 case MVT::v8i8: return fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v8i8_i_Predicate_imm0_255(imm0); 15179 case MVT::v16i8: return fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v16i8_i_Predicate_imm0_255(imm0); 15180 default: return 0; 15181} 15182} 15183 15184unsigned fastEmit_AArch64ISD_MOVI_i_Predicate_imm0_255(MVT VT, MVT RetVT, uint64_t imm0) { 15185 switch (VT.SimpleTy) { 15186 case MVT::i32: return fastEmit_AArch64ISD_MOVI_MVT_i32_i_Predicate_imm0_255(RetVT, imm0); 15187 default: return 0; 15188 } 15189} 15190 15191// FastEmit functions for AArch64ISD::MOVIedit. 15192 15193unsigned fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_f64_i_Predicate_imm0_255(uint64_t imm0) { 15194 return fastEmitInst_i(AArch64::MOVID, &AArch64::FPR64RegClass, imm0); 15195} 15196 15197unsigned fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_v2i64_i_Predicate_imm0_255(uint64_t imm0) { 15198 if ((Subtarget->hasNEON())) { 15199 return fastEmitInst_i(AArch64::MOVIv2d_ns, &AArch64::FPR128RegClass, imm0); 15200 } 15201 return 0; 15202} 15203 15204unsigned fastEmit_AArch64ISD_MOVIedit_MVT_i32_i_Predicate_imm0_255(MVT RetVT, uint64_t imm0) { 15205switch (RetVT.SimpleTy) { 15206 case MVT::f64: return fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_f64_i_Predicate_imm0_255(imm0); 15207 case MVT::v2i64: return fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_v2i64_i_Predicate_imm0_255(imm0); 15208 default: return 0; 15209} 15210} 15211 15212unsigned fastEmit_AArch64ISD_MOVIedit_i_Predicate_imm0_255(MVT VT, MVT RetVT, uint64_t imm0) { 15213 switch (VT.SimpleTy) { 15214 case MVT::i32: return fastEmit_AArch64ISD_MOVIedit_MVT_i32_i_Predicate_imm0_255(RetVT, imm0); 15215 default: return 0; 15216 } 15217} 15218 15219// Top-level FastEmit function. 15220 15221unsigned fastEmit_i_Predicate_imm0_255(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) { 15222 switch (Opcode) { 15223 case AArch64ISD::FMOV: return fastEmit_AArch64ISD_FMOV_i_Predicate_imm0_255(VT, RetVT, imm0); 15224 case AArch64ISD::MOVI: return fastEmit_AArch64ISD_MOVI_i_Predicate_imm0_255(VT, RetVT, imm0); 15225 case AArch64ISD::MOVIedit: return fastEmit_AArch64ISD_MOVIedit_i_Predicate_imm0_255(VT, RetVT, imm0); 15226 default: return 0; 15227 } 15228} 15229 15230// FastEmit functions for AArch64ISD::RDSVL. 15231 15232unsigned fastEmit_AArch64ISD_RDSVL_MVT_i32_i_Predicate_simm6_32b(MVT RetVT, uint64_t imm0) { 15233 if (RetVT.SimpleTy != MVT::i64) 15234 return 0; 15235 if ((Subtarget->hasSME())) { 15236 return fastEmitInst_i(AArch64::RDSVLI_XI, &AArch64::GPR64RegClass, imm0); 15237 } 15238 return 0; 15239} 15240 15241unsigned fastEmit_AArch64ISD_RDSVL_i_Predicate_simm6_32b(MVT VT, MVT RetVT, uint64_t imm0) { 15242 switch (VT.SimpleTy) { 15243 case MVT::i32: return fastEmit_AArch64ISD_RDSVL_MVT_i32_i_Predicate_simm6_32b(RetVT, imm0); 15244 default: return 0; 15245 } 15246} 15247 15248// Top-level FastEmit function. 15249 15250unsigned fastEmit_i_Predicate_simm6_32b(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) { 15251 switch (Opcode) { 15252 case AArch64ISD::RDSVL: return fastEmit_AArch64ISD_RDSVL_i_Predicate_simm6_32b(VT, RetVT, imm0); 15253 default: return 0; 15254 } 15255} 15256 15257