Lines Matching refs:MVT
241 static const MVT IntVTs[] = { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }; in resetOperationActions()
303 addRegisterClass(MVT::i8, &X86::GR8RegClass); in resetOperationActions()
304 addRegisterClass(MVT::i16, &X86::GR16RegClass); in resetOperationActions()
305 addRegisterClass(MVT::i32, &X86::GR32RegClass); in resetOperationActions()
307 addRegisterClass(MVT::i64, &X86::GR64RegClass); in resetOperationActions()
309 setLoadExtAction(ISD::SEXTLOAD, MVT::i1, Promote); in resetOperationActions()
312 setTruncStoreAction(MVT::i64, MVT::i32, Expand); in resetOperationActions()
313 setTruncStoreAction(MVT::i64, MVT::i16, Expand); in resetOperationActions()
314 setTruncStoreAction(MVT::i64, MVT::i8 , Expand); in resetOperationActions()
315 setTruncStoreAction(MVT::i32, MVT::i16, Expand); in resetOperationActions()
316 setTruncStoreAction(MVT::i32, MVT::i8 , Expand); in resetOperationActions()
317 setTruncStoreAction(MVT::i16, MVT::i8, Expand); in resetOperationActions()
320 setCondCodeAction(ISD::SETOEQ, MVT::f32, Expand); in resetOperationActions()
321 setCondCodeAction(ISD::SETOEQ, MVT::f64, Expand); in resetOperationActions()
322 setCondCodeAction(ISD::SETOEQ, MVT::f80, Expand); in resetOperationActions()
323 setCondCodeAction(ISD::SETUNE, MVT::f32, Expand); in resetOperationActions()
324 setCondCodeAction(ISD::SETUNE, MVT::f64, Expand); in resetOperationActions()
325 setCondCodeAction(ISD::SETUNE, MVT::f80, Expand); in resetOperationActions()
329 setOperationAction(ISD::UINT_TO_FP , MVT::i1 , Promote); in resetOperationActions()
330 setOperationAction(ISD::UINT_TO_FP , MVT::i8 , Promote); in resetOperationActions()
331 setOperationAction(ISD::UINT_TO_FP , MVT::i16 , Promote); in resetOperationActions()
334 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Promote); in resetOperationActions()
335 setOperationAction(ISD::UINT_TO_FP , MVT::i64 , Custom); in resetOperationActions()
339 setOperationAction(ISD::UINT_TO_FP , MVT::i64 , Custom); in resetOperationActions()
342 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Custom); in resetOperationActions()
347 setOperationAction(ISD::SINT_TO_FP , MVT::i1 , Promote); in resetOperationActions()
348 setOperationAction(ISD::SINT_TO_FP , MVT::i8 , Promote); in resetOperationActions()
353 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Promote); in resetOperationActions()
355 setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Custom); in resetOperationActions()
357 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Custom); in resetOperationActions()
358 setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Custom); in resetOperationActions()
361 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Promote); in resetOperationActions()
362 setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Promote); in resetOperationActions()
367 setOperationAction(ISD::FP_TO_SINT , MVT::i64 , Custom); in resetOperationActions()
368 setOperationAction(ISD::SINT_TO_FP , MVT::i64 , Custom); in resetOperationActions()
372 setOperationAction(ISD::FP_TO_SINT , MVT::i1 , Promote); in resetOperationActions()
373 setOperationAction(ISD::FP_TO_SINT , MVT::i8 , Promote); in resetOperationActions()
376 setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Promote); in resetOperationActions()
378 setOperationAction(ISD::FP_TO_SINT , MVT::i32 , Custom); in resetOperationActions()
380 setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Custom); in resetOperationActions()
381 setOperationAction(ISD::FP_TO_SINT , MVT::i32 , Custom); in resetOperationActions()
386 setOperationAction(ISD::FP_TO_UINT , MVT::i1 , Promote); in resetOperationActions()
387 setOperationAction(ISD::FP_TO_UINT , MVT::i8 , Promote); in resetOperationActions()
388 setOperationAction(ISD::FP_TO_UINT , MVT::i16 , Promote); in resetOperationActions()
391 setOperationAction(ISD::FP_TO_UINT , MVT::i64 , Expand); in resetOperationActions()
392 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Promote); in resetOperationActions()
399 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Expand); in resetOperationActions()
403 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Custom); in resetOperationActions()
409 setOperationAction(ISD::FP_TO_UINT , MVT::i64 , Custom); in resetOperationActions()
414 setOperationAction(ISD::BITCAST , MVT::f32 , Expand); in resetOperationActions()
415 setOperationAction(ISD::BITCAST , MVT::i32 , Expand); in resetOperationActions()
417 setOperationAction(ISD::BITCAST , MVT::f64 , Expand); in resetOperationActions()
419 setOperationAction(ISD::BITCAST , MVT::i64 , Expand); in resetOperationActions()
434 MVT VT = IntVTs[i]; in resetOperationActions()
449 setOperationAction(ISD::BR_JT , MVT::Other, Expand); in resetOperationActions()
450 setOperationAction(ISD::BRCOND , MVT::Other, Custom); in resetOperationActions()
451 setOperationAction(ISD::BR_CC , MVT::f32, Expand); in resetOperationActions()
452 setOperationAction(ISD::BR_CC , MVT::f64, Expand); in resetOperationActions()
453 setOperationAction(ISD::BR_CC , MVT::f80, Expand); in resetOperationActions()
454 setOperationAction(ISD::BR_CC , MVT::i8, Expand); in resetOperationActions()
455 setOperationAction(ISD::BR_CC , MVT::i16, Expand); in resetOperationActions()
456 setOperationAction(ISD::BR_CC , MVT::i32, Expand); in resetOperationActions()
457 setOperationAction(ISD::BR_CC , MVT::i64, Expand); in resetOperationActions()
458 setOperationAction(ISD::SELECT_CC , MVT::f32, Expand); in resetOperationActions()
459 setOperationAction(ISD::SELECT_CC , MVT::f64, Expand); in resetOperationActions()
460 setOperationAction(ISD::SELECT_CC , MVT::f80, Expand); in resetOperationActions()
461 setOperationAction(ISD::SELECT_CC , MVT::i8, Expand); in resetOperationActions()
462 setOperationAction(ISD::SELECT_CC , MVT::i16, Expand); in resetOperationActions()
463 setOperationAction(ISD::SELECT_CC , MVT::i32, Expand); in resetOperationActions()
464 setOperationAction(ISD::SELECT_CC , MVT::i64, Expand); in resetOperationActions()
466 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Legal); in resetOperationActions()
467 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16 , Legal); in resetOperationActions()
468 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8 , Legal); in resetOperationActions()
469 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1 , Expand); in resetOperationActions()
470 setOperationAction(ISD::FP_ROUND_INREG , MVT::f32 , Expand); in resetOperationActions()
471 setOperationAction(ISD::FREM , MVT::f32 , Expand); in resetOperationActions()
472 setOperationAction(ISD::FREM , MVT::f64 , Expand); in resetOperationActions()
473 setOperationAction(ISD::FREM , MVT::f80 , Expand); in resetOperationActions()
474 setOperationAction(ISD::FLT_ROUNDS_ , MVT::i32 , Custom); in resetOperationActions()
478 setOperationAction(ISD::CTTZ , MVT::i8 , Promote); in resetOperationActions()
479 AddPromotedToType (ISD::CTTZ , MVT::i8 , MVT::i32); in resetOperationActions()
480 setOperationAction(ISD::CTTZ_ZERO_UNDEF , MVT::i8 , Promote); in resetOperationActions()
481 AddPromotedToType (ISD::CTTZ_ZERO_UNDEF , MVT::i8 , MVT::i32); in resetOperationActions()
483 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i16 , Expand); in resetOperationActions()
484 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i32 , Expand); in resetOperationActions()
486 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i64, Expand); in resetOperationActions()
488 setOperationAction(ISD::CTTZ , MVT::i16 , Custom); in resetOperationActions()
489 setOperationAction(ISD::CTTZ , MVT::i32 , Custom); in resetOperationActions()
491 setOperationAction(ISD::CTTZ , MVT::i64 , Custom); in resetOperationActions()
497 setOperationAction(ISD::CTLZ , MVT::i8 , Promote); in resetOperationActions()
498 AddPromotedToType (ISD::CTLZ , MVT::i8 , MVT::i32); in resetOperationActions()
499 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i8 , Promote); in resetOperationActions()
500 AddPromotedToType (ISD::CTLZ_ZERO_UNDEF, MVT::i8 , MVT::i32); in resetOperationActions()
501 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i16 , Expand); in resetOperationActions()
502 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32 , Expand); in resetOperationActions()
504 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i64, Expand); in resetOperationActions()
506 setOperationAction(ISD::CTLZ , MVT::i8 , Custom); in resetOperationActions()
507 setOperationAction(ISD::CTLZ , MVT::i16 , Custom); in resetOperationActions()
508 setOperationAction(ISD::CTLZ , MVT::i32 , Custom); in resetOperationActions()
509 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i8 , Custom); in resetOperationActions()
510 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i16 , Custom); in resetOperationActions()
511 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32 , Custom); in resetOperationActions()
513 setOperationAction(ISD::CTLZ , MVT::i64 , Custom); in resetOperationActions()
514 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i64, Custom); in resetOperationActions()
522 setOperationAction(ISD::FP16_TO_FP32, MVT::f32, Expand); in resetOperationActions()
523 setOperationAction(ISD::FP32_TO_FP16, MVT::i16, Expand); in resetOperationActions()
527 setOperationAction(ISD::CTPOP , MVT::i8 , Promote); in resetOperationActions()
529 setOperationAction(ISD::CTPOP , MVT::i8 , Expand); in resetOperationActions()
530 setOperationAction(ISD::CTPOP , MVT::i16 , Expand); in resetOperationActions()
531 setOperationAction(ISD::CTPOP , MVT::i32 , Expand); in resetOperationActions()
533 setOperationAction(ISD::CTPOP , MVT::i64 , Expand); in resetOperationActions()
536 setOperationAction(ISD::READCYCLECOUNTER , MVT::i64 , Custom); in resetOperationActions()
539 setOperationAction(ISD::BSWAP , MVT::i16 , Expand); in resetOperationActions()
542 setOperationAction(ISD::SELECT , MVT::i1 , Promote); in resetOperationActions()
544 setOperationAction(ISD::SELECT , MVT::i8 , Custom); in resetOperationActions()
545 setOperationAction(ISD::SELECT , MVT::i16 , Custom); in resetOperationActions()
546 setOperationAction(ISD::SELECT , MVT::i32 , Custom); in resetOperationActions()
547 setOperationAction(ISD::SELECT , MVT::f32 , Custom); in resetOperationActions()
548 setOperationAction(ISD::SELECT , MVT::f64 , Custom); in resetOperationActions()
549 setOperationAction(ISD::SELECT , MVT::f80 , Custom); in resetOperationActions()
550 setOperationAction(ISD::SETCC , MVT::i8 , Custom); in resetOperationActions()
551 setOperationAction(ISD::SETCC , MVT::i16 , Custom); in resetOperationActions()
552 setOperationAction(ISD::SETCC , MVT::i32 , Custom); in resetOperationActions()
553 setOperationAction(ISD::SETCC , MVT::f32 , Custom); in resetOperationActions()
554 setOperationAction(ISD::SETCC , MVT::f64 , Custom); in resetOperationActions()
555 setOperationAction(ISD::SETCC , MVT::f80 , Custom); in resetOperationActions()
557 setOperationAction(ISD::SELECT , MVT::i64 , Custom); in resetOperationActions()
558 setOperationAction(ISD::SETCC , MVT::i64 , Custom); in resetOperationActions()
560 setOperationAction(ISD::EH_RETURN , MVT::Other, Custom); in resetOperationActions()
567 setOperationAction(ISD::EH_SJLJ_SETJMP, MVT::i32, Custom); in resetOperationActions()
568 setOperationAction(ISD::EH_SJLJ_LONGJMP, MVT::Other, Custom); in resetOperationActions()
571 setOperationAction(ISD::ConstantPool , MVT::i32 , Custom); in resetOperationActions()
572 setOperationAction(ISD::JumpTable , MVT::i32 , Custom); in resetOperationActions()
573 setOperationAction(ISD::GlobalAddress , MVT::i32 , Custom); in resetOperationActions()
574 setOperationAction(ISD::GlobalTLSAddress, MVT::i32 , Custom); in resetOperationActions()
576 setOperationAction(ISD::GlobalTLSAddress, MVT::i64, Custom); in resetOperationActions()
577 setOperationAction(ISD::ExternalSymbol , MVT::i32 , Custom); in resetOperationActions()
578 setOperationAction(ISD::BlockAddress , MVT::i32 , Custom); in resetOperationActions()
580 setOperationAction(ISD::ConstantPool , MVT::i64 , Custom); in resetOperationActions()
581 setOperationAction(ISD::JumpTable , MVT::i64 , Custom); in resetOperationActions()
582 setOperationAction(ISD::GlobalAddress , MVT::i64 , Custom); in resetOperationActions()
583 setOperationAction(ISD::ExternalSymbol, MVT::i64 , Custom); in resetOperationActions()
584 setOperationAction(ISD::BlockAddress , MVT::i64 , Custom); in resetOperationActions()
587 setOperationAction(ISD::SHL_PARTS , MVT::i32 , Custom); in resetOperationActions()
588 setOperationAction(ISD::SRA_PARTS , MVT::i32 , Custom); in resetOperationActions()
589 setOperationAction(ISD::SRL_PARTS , MVT::i32 , Custom); in resetOperationActions()
591 setOperationAction(ISD::SHL_PARTS , MVT::i64 , Custom); in resetOperationActions()
592 setOperationAction(ISD::SRA_PARTS , MVT::i64 , Custom); in resetOperationActions()
593 setOperationAction(ISD::SRL_PARTS , MVT::i64 , Custom); in resetOperationActions()
597 setOperationAction(ISD::PREFETCH , MVT::Other, Legal); in resetOperationActions()
599 setOperationAction(ISD::ATOMIC_FENCE , MVT::Other, Custom); in resetOperationActions()
603 MVT VT = IntVTs[i]; in resetOperationActions()
610 setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, MVT::i128, Custom); in resetOperationActions()
616 setOperationAction(ISD::EH_LABEL, MVT::Other, Expand); in resetOperationActions()
626 setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i32, Custom); in resetOperationActions()
627 setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i64, Custom); in resetOperationActions()
629 setOperationAction(ISD::INIT_TRAMPOLINE, MVT::Other, Custom); in resetOperationActions()
630 setOperationAction(ISD::ADJUST_TRAMPOLINE, MVT::Other, Custom); in resetOperationActions()
632 setOperationAction(ISD::TRAP, MVT::Other, Legal); in resetOperationActions()
633 setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal); in resetOperationActions()
636 setOperationAction(ISD::VASTART , MVT::Other, Custom); in resetOperationActions()
637 setOperationAction(ISD::VAEND , MVT::Other, Expand); in resetOperationActions()
640 setOperationAction(ISD::VAARG , MVT::Other, Custom); in resetOperationActions()
641 setOperationAction(ISD::VACOPY , MVT::Other, Custom); in resetOperationActions()
644 setOperationAction(ISD::VAARG , MVT::Other, Expand); in resetOperationActions()
645 setOperationAction(ISD::VACOPY , MVT::Other, Expand); in resetOperationActions()
648 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand); in resetOperationActions()
649 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand); in resetOperationActions()
652 MVT::i64 : MVT::i32, Custom); in resetOperationActions()
657 addRegisterClass(MVT::f32, &X86::FR32RegClass); in resetOperationActions()
658 addRegisterClass(MVT::f64, &X86::FR64RegClass); in resetOperationActions()
661 setOperationAction(ISD::FABS , MVT::f64, Custom); in resetOperationActions()
662 setOperationAction(ISD::FABS , MVT::f32, Custom); in resetOperationActions()
665 setOperationAction(ISD::FNEG , MVT::f64, Custom); in resetOperationActions()
666 setOperationAction(ISD::FNEG , MVT::f32, Custom); in resetOperationActions()
669 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom); in resetOperationActions()
670 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom); in resetOperationActions()
673 setOperationAction(ISD::FGETSIGN, MVT::i64, Custom); in resetOperationActions()
674 setOperationAction(ISD::FGETSIGN, MVT::i32, Custom); in resetOperationActions()
677 setOperationAction(ISD::FSIN , MVT::f64, Expand); in resetOperationActions()
678 setOperationAction(ISD::FCOS , MVT::f64, Expand); in resetOperationActions()
679 setOperationAction(ISD::FSINCOS, MVT::f64, Expand); in resetOperationActions()
680 setOperationAction(ISD::FSIN , MVT::f32, Expand); in resetOperationActions()
681 setOperationAction(ISD::FCOS , MVT::f32, Expand); in resetOperationActions()
682 setOperationAction(ISD::FSINCOS, MVT::f32, Expand); in resetOperationActions()
691 addRegisterClass(MVT::f32, &X86::FR32RegClass); in resetOperationActions()
692 addRegisterClass(MVT::f64, &X86::RFP64RegClass); in resetOperationActions()
695 setOperationAction(ISD::FABS , MVT::f32, Custom); in resetOperationActions()
698 setOperationAction(ISD::FNEG , MVT::f32, Custom); in resetOperationActions()
700 setOperationAction(ISD::UNDEF, MVT::f64, Expand); in resetOperationActions()
703 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand); in resetOperationActions()
704 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom); in resetOperationActions()
707 setOperationAction(ISD::FSIN , MVT::f32, Expand); in resetOperationActions()
708 setOperationAction(ISD::FCOS , MVT::f32, Expand); in resetOperationActions()
709 setOperationAction(ISD::FSINCOS, MVT::f32, Expand); in resetOperationActions()
719 setOperationAction(ISD::FSIN , MVT::f64, Expand); in resetOperationActions()
720 setOperationAction(ISD::FCOS , MVT::f64, Expand); in resetOperationActions()
721 setOperationAction(ISD::FSINCOS, MVT::f64, Expand); in resetOperationActions()
726 addRegisterClass(MVT::f64, &X86::RFP64RegClass); in resetOperationActions()
727 addRegisterClass(MVT::f32, &X86::RFP32RegClass); in resetOperationActions()
729 setOperationAction(ISD::UNDEF, MVT::f64, Expand); in resetOperationActions()
730 setOperationAction(ISD::UNDEF, MVT::f32, Expand); in resetOperationActions()
731 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand); in resetOperationActions()
732 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand); in resetOperationActions()
735 setOperationAction(ISD::FSIN , MVT::f64, Expand); in resetOperationActions()
736 setOperationAction(ISD::FSIN , MVT::f32, Expand); in resetOperationActions()
737 setOperationAction(ISD::FCOS , MVT::f64, Expand); in resetOperationActions()
738 setOperationAction(ISD::FCOS , MVT::f32, Expand); in resetOperationActions()
739 setOperationAction(ISD::FSINCOS, MVT::f64, Expand); in resetOperationActions()
740 setOperationAction(ISD::FSINCOS, MVT::f32, Expand); in resetOperationActions()
753 setOperationAction(ISD::FMA, MVT::f64, Expand); in resetOperationActions()
754 setOperationAction(ISD::FMA, MVT::f32, Expand); in resetOperationActions()
758 addRegisterClass(MVT::f80, &X86::RFP80RegClass); in resetOperationActions()
759 setOperationAction(ISD::UNDEF, MVT::f80, Expand); in resetOperationActions()
760 setOperationAction(ISD::FCOPYSIGN, MVT::f80, Expand); in resetOperationActions()
777 setOperationAction(ISD::FSIN , MVT::f80, Expand); in resetOperationActions()
778 setOperationAction(ISD::FCOS , MVT::f80, Expand); in resetOperationActions()
779 setOperationAction(ISD::FSINCOS, MVT::f80, Expand); in resetOperationActions()
782 setOperationAction(ISD::FFLOOR, MVT::f80, Expand); in resetOperationActions()
783 setOperationAction(ISD::FCEIL, MVT::f80, Expand); in resetOperationActions()
784 setOperationAction(ISD::FTRUNC, MVT::f80, Expand); in resetOperationActions()
785 setOperationAction(ISD::FRINT, MVT::f80, Expand); in resetOperationActions()
786 setOperationAction(ISD::FNEARBYINT, MVT::f80, Expand); in resetOperationActions()
787 setOperationAction(ISD::FMA, MVT::f80, Expand); in resetOperationActions()
791 setOperationAction(ISD::FPOW , MVT::f32 , Expand); in resetOperationActions()
792 setOperationAction(ISD::FPOW , MVT::f64 , Expand); in resetOperationActions()
793 setOperationAction(ISD::FPOW , MVT::f80 , Expand); in resetOperationActions()
795 setOperationAction(ISD::FLOG, MVT::f80, Expand); in resetOperationActions()
796 setOperationAction(ISD::FLOG2, MVT::f80, Expand); in resetOperationActions()
797 setOperationAction(ISD::FLOG10, MVT::f80, Expand); in resetOperationActions()
798 setOperationAction(ISD::FEXP, MVT::f80, Expand); in resetOperationActions()
799 setOperationAction(ISD::FEXP2, MVT::f80, Expand); in resetOperationActions()
804 for (int i = MVT::FIRST_VECTOR_VALUETYPE; in resetOperationActions()
805 i <= MVT::LAST_VECTOR_VALUETYPE; ++i) { in resetOperationActions()
806 MVT VT = (MVT::SimpleValueType)i; in resetOperationActions()
875 for (int InnerVT = MVT::FIRST_VECTOR_VALUETYPE; in resetOperationActions()
876 InnerVT <= MVT::LAST_VECTOR_VALUETYPE; ++InnerVT) in resetOperationActions()
878 (MVT::SimpleValueType)InnerVT, Expand); in resetOperationActions()
887 addRegisterClass(MVT::x86mmx, &X86::VR64RegClass); in resetOperationActions()
893 setOperationAction(ISD::MULHS, MVT::v8i8, Expand); in resetOperationActions()
894 setOperationAction(ISD::MULHS, MVT::v4i16, Expand); in resetOperationActions()
895 setOperationAction(ISD::MULHS, MVT::v2i32, Expand); in resetOperationActions()
896 setOperationAction(ISD::MULHS, MVT::v1i64, Expand); in resetOperationActions()
897 setOperationAction(ISD::AND, MVT::v8i8, Expand); in resetOperationActions()
898 setOperationAction(ISD::AND, MVT::v4i16, Expand); in resetOperationActions()
899 setOperationAction(ISD::AND, MVT::v2i32, Expand); in resetOperationActions()
900 setOperationAction(ISD::AND, MVT::v1i64, Expand); in resetOperationActions()
901 setOperationAction(ISD::OR, MVT::v8i8, Expand); in resetOperationActions()
902 setOperationAction(ISD::OR, MVT::v4i16, Expand); in resetOperationActions()
903 setOperationAction(ISD::OR, MVT::v2i32, Expand); in resetOperationActions()
904 setOperationAction(ISD::OR, MVT::v1i64, Expand); in resetOperationActions()
905 setOperationAction(ISD::XOR, MVT::v8i8, Expand); in resetOperationActions()
906 setOperationAction(ISD::XOR, MVT::v4i16, Expand); in resetOperationActions()
907 setOperationAction(ISD::XOR, MVT::v2i32, Expand); in resetOperationActions()
908 setOperationAction(ISD::XOR, MVT::v1i64, Expand); in resetOperationActions()
909 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8i8, Expand); in resetOperationActions()
910 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4i16, Expand); in resetOperationActions()
911 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v2i32, Expand); in resetOperationActions()
912 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v1i64, Expand); in resetOperationActions()
913 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v1i64, Expand); in resetOperationActions()
914 setOperationAction(ISD::SELECT, MVT::v8i8, Expand); in resetOperationActions()
915 setOperationAction(ISD::SELECT, MVT::v4i16, Expand); in resetOperationActions()
916 setOperationAction(ISD::SELECT, MVT::v2i32, Expand); in resetOperationActions()
917 setOperationAction(ISD::SELECT, MVT::v1i64, Expand); in resetOperationActions()
918 setOperationAction(ISD::BITCAST, MVT::v8i8, Expand); in resetOperationActions()
919 setOperationAction(ISD::BITCAST, MVT::v4i16, Expand); in resetOperationActions()
920 setOperationAction(ISD::BITCAST, MVT::v2i32, Expand); in resetOperationActions()
921 setOperationAction(ISD::BITCAST, MVT::v1i64, Expand); in resetOperationActions()
924 addRegisterClass(MVT::v4f32, &X86::VR128RegClass); in resetOperationActions()
926 setOperationAction(ISD::FADD, MVT::v4f32, Legal); in resetOperationActions()
927 setOperationAction(ISD::FSUB, MVT::v4f32, Legal); in resetOperationActions()
928 setOperationAction(ISD::FMUL, MVT::v4f32, Legal); in resetOperationActions()
929 setOperationAction(ISD::FDIV, MVT::v4f32, Legal); in resetOperationActions()
930 setOperationAction(ISD::FSQRT, MVT::v4f32, Legal); in resetOperationActions()
931 setOperationAction(ISD::FNEG, MVT::v4f32, Custom); in resetOperationActions()
932 setOperationAction(ISD::FABS, MVT::v4f32, Custom); in resetOperationActions()
933 setOperationAction(ISD::LOAD, MVT::v4f32, Legal); in resetOperationActions()
934 setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom); in resetOperationActions()
935 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4f32, Custom); in resetOperationActions()
936 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom); in resetOperationActions()
937 setOperationAction(ISD::SELECT, MVT::v4f32, Custom); in resetOperationActions()
941 addRegisterClass(MVT::v2f64, &X86::VR128RegClass); in resetOperationActions()
945 addRegisterClass(MVT::v16i8, &X86::VR128RegClass); in resetOperationActions()
946 addRegisterClass(MVT::v8i16, &X86::VR128RegClass); in resetOperationActions()
947 addRegisterClass(MVT::v4i32, &X86::VR128RegClass); in resetOperationActions()
948 addRegisterClass(MVT::v2i64, &X86::VR128RegClass); in resetOperationActions()
950 setOperationAction(ISD::ADD, MVT::v16i8, Legal); in resetOperationActions()
951 setOperationAction(ISD::ADD, MVT::v8i16, Legal); in resetOperationActions()
952 setOperationAction(ISD::ADD, MVT::v4i32, Legal); in resetOperationActions()
953 setOperationAction(ISD::ADD, MVT::v2i64, Legal); in resetOperationActions()
954 setOperationAction(ISD::MUL, MVT::v4i32, Custom); in resetOperationActions()
955 setOperationAction(ISD::MUL, MVT::v2i64, Custom); in resetOperationActions()
956 setOperationAction(ISD::UMUL_LOHI, MVT::v4i32, Custom); in resetOperationActions()
957 setOperationAction(ISD::SMUL_LOHI, MVT::v4i32, Custom); in resetOperationActions()
958 setOperationAction(ISD::MULHU, MVT::v8i16, Legal); in resetOperationActions()
959 setOperationAction(ISD::MULHS, MVT::v8i16, Legal); in resetOperationActions()
960 setOperationAction(ISD::SUB, MVT::v16i8, Legal); in resetOperationActions()
961 setOperationAction(ISD::SUB, MVT::v8i16, Legal); in resetOperationActions()
962 setOperationAction(ISD::SUB, MVT::v4i32, Legal); in resetOperationActions()
963 setOperationAction(ISD::SUB, MVT::v2i64, Legal); in resetOperationActions()
964 setOperationAction(ISD::MUL, MVT::v8i16, Legal); in resetOperationActions()
965 setOperationAction(ISD::FADD, MVT::v2f64, Legal); in resetOperationActions()
966 setOperationAction(ISD::FSUB, MVT::v2f64, Legal); in resetOperationActions()
967 setOperationAction(ISD::FMUL, MVT::v2f64, Legal); in resetOperationActions()
968 setOperationAction(ISD::FDIV, MVT::v2f64, Legal); in resetOperationActions()
969 setOperationAction(ISD::FSQRT, MVT::v2f64, Legal); in resetOperationActions()
970 setOperationAction(ISD::FNEG, MVT::v2f64, Custom); in resetOperationActions()
971 setOperationAction(ISD::FABS, MVT::v2f64, Custom); in resetOperationActions()
973 setOperationAction(ISD::SETCC, MVT::v2i64, Custom); in resetOperationActions()
974 setOperationAction(ISD::SETCC, MVT::v16i8, Custom); in resetOperationActions()
975 setOperationAction(ISD::SETCC, MVT::v8i16, Custom); in resetOperationActions()
976 setOperationAction(ISD::SETCC, MVT::v4i32, Custom); in resetOperationActions()
978 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v16i8, Custom); in resetOperationActions()
979 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8i16, Custom); in resetOperationActions()
980 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom); in resetOperationActions()
981 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom); in resetOperationActions()
982 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom); in resetOperationActions()
985 for (int i = MVT::v16i8; i != MVT::v2i64; ++i) { in resetOperationActions()
986 MVT VT = (MVT::SimpleValueType)i; in resetOperationActions()
998 setOperationAction(ISD::BUILD_VECTOR, MVT::v2f64, Custom); in resetOperationActions()
999 setOperationAction(ISD::BUILD_VECTOR, MVT::v2i64, Custom); in resetOperationActions()
1000 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2f64, Custom); in resetOperationActions()
1001 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2i64, Custom); in resetOperationActions()
1002 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2f64, Custom); in resetOperationActions()
1003 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f64, Custom); in resetOperationActions()
1006 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i64, Custom); in resetOperationActions()
1007 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i64, Custom); in resetOperationActions()
1011 for (int i = MVT::v16i8; i != MVT::v2i64; ++i) { in resetOperationActions()
1012 MVT VT = (MVT::SimpleValueType)i; in resetOperationActions()
1019 AddPromotedToType (ISD::AND, VT, MVT::v2i64); in resetOperationActions()
1021 AddPromotedToType (ISD::OR, VT, MVT::v2i64); in resetOperationActions()
1023 AddPromotedToType (ISD::XOR, VT, MVT::v2i64); in resetOperationActions()
1025 AddPromotedToType (ISD::LOAD, VT, MVT::v2i64); in resetOperationActions()
1027 AddPromotedToType (ISD::SELECT, VT, MVT::v2i64); in resetOperationActions()
1030 setTruncStoreAction(MVT::f64, MVT::f32, Expand); in resetOperationActions()
1033 setOperationAction(ISD::LOAD, MVT::v2f64, Legal); in resetOperationActions()
1034 setOperationAction(ISD::LOAD, MVT::v2i64, Legal); in resetOperationActions()
1035 setOperationAction(ISD::SELECT, MVT::v2f64, Custom); in resetOperationActions()
1036 setOperationAction(ISD::SELECT, MVT::v2i64, Custom); in resetOperationActions()
1038 setOperationAction(ISD::FP_TO_SINT, MVT::v4i32, Legal); in resetOperationActions()
1039 setOperationAction(ISD::SINT_TO_FP, MVT::v4i32, Legal); in resetOperationActions()
1041 setOperationAction(ISD::UINT_TO_FP, MVT::v4i8, Custom); in resetOperationActions()
1042 setOperationAction(ISD::UINT_TO_FP, MVT::v4i16, Custom); in resetOperationActions()
1046 setOperationAction(ISD::UINT_TO_FP, MVT::v2f32, Custom); in resetOperationActions()
1048 setOperationAction(ISD::FP_EXTEND, MVT::v2f32, Custom); in resetOperationActions()
1049 setOperationAction(ISD::FP_ROUND, MVT::v2f32, Custom); in resetOperationActions()
1051 setLoadExtAction(ISD::EXTLOAD, MVT::v2f32, Legal); in resetOperationActions()
1053 setOperationAction(ISD::BITCAST, MVT::v2i32, Custom); in resetOperationActions()
1054 setOperationAction(ISD::BITCAST, MVT::v4i16, Custom); in resetOperationActions()
1055 setOperationAction(ISD::BITCAST, MVT::v8i8, Custom); in resetOperationActions()
1059 setOperationAction(ISD::FFLOOR, MVT::f32, Legal); in resetOperationActions()
1060 setOperationAction(ISD::FCEIL, MVT::f32, Legal); in resetOperationActions()
1061 setOperationAction(ISD::FTRUNC, MVT::f32, Legal); in resetOperationActions()
1062 setOperationAction(ISD::FRINT, MVT::f32, Legal); in resetOperationActions()
1063 setOperationAction(ISD::FNEARBYINT, MVT::f32, Legal); in resetOperationActions()
1064 setOperationAction(ISD::FFLOOR, MVT::f64, Legal); in resetOperationActions()
1065 setOperationAction(ISD::FCEIL, MVT::f64, Legal); in resetOperationActions()
1066 setOperationAction(ISD::FTRUNC, MVT::f64, Legal); in resetOperationActions()
1067 setOperationAction(ISD::FRINT, MVT::f64, Legal); in resetOperationActions()
1068 setOperationAction(ISD::FNEARBYINT, MVT::f64, Legal); in resetOperationActions()
1070 setOperationAction(ISD::FFLOOR, MVT::v4f32, Legal); in resetOperationActions()
1071 setOperationAction(ISD::FCEIL, MVT::v4f32, Legal); in resetOperationActions()
1072 setOperationAction(ISD::FTRUNC, MVT::v4f32, Legal); in resetOperationActions()
1073 setOperationAction(ISD::FRINT, MVT::v4f32, Legal); in resetOperationActions()
1074 setOperationAction(ISD::FNEARBYINT, MVT::v4f32, Legal); in resetOperationActions()
1075 setOperationAction(ISD::FFLOOR, MVT::v2f64, Legal); in resetOperationActions()
1076 setOperationAction(ISD::FCEIL, MVT::v2f64, Legal); in resetOperationActions()
1077 setOperationAction(ISD::FTRUNC, MVT::v2f64, Legal); in resetOperationActions()
1078 setOperationAction(ISD::FRINT, MVT::v2f64, Legal); in resetOperationActions()
1079 setOperationAction(ISD::FNEARBYINT, MVT::v2f64, Legal); in resetOperationActions()
1082 setOperationAction(ISD::MUL, MVT::v4i32, Legal); in resetOperationActions()
1084 setOperationAction(ISD::VSELECT, MVT::v2f64, Custom); in resetOperationActions()
1085 setOperationAction(ISD::VSELECT, MVT::v2i64, Custom); in resetOperationActions()
1086 setOperationAction(ISD::VSELECT, MVT::v4i32, Custom); in resetOperationActions()
1087 setOperationAction(ISD::VSELECT, MVT::v4f32, Custom); in resetOperationActions()
1088 setOperationAction(ISD::VSELECT, MVT::v8i16, Custom); in resetOperationActions()
1091 setOperationAction(ISD::VSELECT, MVT::v16i8, Legal); in resetOperationActions()
1097 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16i8, Custom); in resetOperationActions()
1098 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom); in resetOperationActions()
1099 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom); in resetOperationActions()
1100 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom); in resetOperationActions()
1102 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v16i8, Custom); in resetOperationActions()
1103 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v8i16, Custom); in resetOperationActions()
1104 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4i32, Custom); in resetOperationActions()
1105 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom); in resetOperationActions()
1110 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i64, Custom); in resetOperationActions()
1111 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i64, Custom); in resetOperationActions()
1116 setOperationAction(ISD::SRL, MVT::v8i16, Custom); in resetOperationActions()
1117 setOperationAction(ISD::SRL, MVT::v16i8, Custom); in resetOperationActions()
1119 setOperationAction(ISD::SHL, MVT::v8i16, Custom); in resetOperationActions()
1120 setOperationAction(ISD::SHL, MVT::v16i8, Custom); in resetOperationActions()
1122 setOperationAction(ISD::SRA, MVT::v8i16, Custom); in resetOperationActions()
1123 setOperationAction(ISD::SRA, MVT::v16i8, Custom); in resetOperationActions()
1127 setOperationAction(ISD::SRL, MVT::v2i64, Custom); in resetOperationActions()
1128 setOperationAction(ISD::SRL, MVT::v4i32, Custom); in resetOperationActions()
1130 setOperationAction(ISD::SHL, MVT::v2i64, Custom); in resetOperationActions()
1131 setOperationAction(ISD::SHL, MVT::v4i32, Custom); in resetOperationActions()
1133 setOperationAction(ISD::SRA, MVT::v4i32, Custom); in resetOperationActions()
1137 addRegisterClass(MVT::v32i8, &X86::VR256RegClass); in resetOperationActions()
1138 addRegisterClass(MVT::v16i16, &X86::VR256RegClass); in resetOperationActions()
1139 addRegisterClass(MVT::v8i32, &X86::VR256RegClass); in resetOperationActions()
1140 addRegisterClass(MVT::v8f32, &X86::VR256RegClass); in resetOperationActions()
1141 addRegisterClass(MVT::v4i64, &X86::VR256RegClass); in resetOperationActions()
1142 addRegisterClass(MVT::v4f64, &X86::VR256RegClass); in resetOperationActions()
1144 setOperationAction(ISD::LOAD, MVT::v8f32, Legal); in resetOperationActions()
1145 setOperationAction(ISD::LOAD, MVT::v4f64, Legal); in resetOperationActions()
1146 setOperationAction(ISD::LOAD, MVT::v4i64, Legal); in resetOperationActions()
1148 setOperationAction(ISD::FADD, MVT::v8f32, Legal); in resetOperationActions()
1149 setOperationAction(ISD::FSUB, MVT::v8f32, Legal); in resetOperationActions()
1150 setOperationAction(ISD::FMUL, MVT::v8f32, Legal); in resetOperationActions()
1151 setOperationAction(ISD::FDIV, MVT::v8f32, Legal); in resetOperationActions()
1152 setOperationAction(ISD::FSQRT, MVT::v8f32, Legal); in resetOperationActions()
1153 setOperationAction(ISD::FFLOOR, MVT::v8f32, Legal); in resetOperationActions()
1154 setOperationAction(ISD::FCEIL, MVT::v8f32, Legal); in resetOperationActions()
1155 setOperationAction(ISD::FTRUNC, MVT::v8f32, Legal); in resetOperationActions()
1156 setOperationAction(ISD::FRINT, MVT::v8f32, Legal); in resetOperationActions()
1157 setOperationAction(ISD::FNEARBYINT, MVT::v8f32, Legal); in resetOperationActions()
1158 setOperationAction(ISD::FNEG, MVT::v8f32, Custom); in resetOperationActions()
1159 setOperationAction(ISD::FABS, MVT::v8f32, Custom); in resetOperationActions()
1161 setOperationAction(ISD::FADD, MVT::v4f64, Legal); in resetOperationActions()
1162 setOperationAction(ISD::FSUB, MVT::v4f64, Legal); in resetOperationActions()
1163 setOperationAction(ISD::FMUL, MVT::v4f64, Legal); in resetOperationActions()
1164 setOperationAction(ISD::FDIV, MVT::v4f64, Legal); in resetOperationActions()
1165 setOperationAction(ISD::FSQRT, MVT::v4f64, Legal); in resetOperationActions()
1166 setOperationAction(ISD::FFLOOR, MVT::v4f64, Legal); in resetOperationActions()
1167 setOperationAction(ISD::FCEIL, MVT::v4f64, Legal); in resetOperationActions()
1168 setOperationAction(ISD::FTRUNC, MVT::v4f64, Legal); in resetOperationActions()
1169 setOperationAction(ISD::FRINT, MVT::v4f64, Legal); in resetOperationActions()
1170 setOperationAction(ISD::FNEARBYINT, MVT::v4f64, Legal); in resetOperationActions()
1171 setOperationAction(ISD::FNEG, MVT::v4f64, Custom); in resetOperationActions()
1172 setOperationAction(ISD::FABS, MVT::v4f64, Custom); in resetOperationActions()
1176 setOperationAction(ISD::FP_TO_SINT, MVT::v8i16, Promote); in resetOperationActions()
1177 setOperationAction(ISD::FP_TO_UINT, MVT::v8i16, Promote); in resetOperationActions()
1178 setOperationAction(ISD::FP_TO_SINT, MVT::v8i32, Legal); in resetOperationActions()
1180 setOperationAction(ISD::SINT_TO_FP, MVT::v8i16, Promote); in resetOperationActions()
1181 setOperationAction(ISD::SINT_TO_FP, MVT::v8i32, Legal); in resetOperationActions()
1182 setOperationAction(ISD::FP_ROUND, MVT::v4f32, Legal); in resetOperationActions()
1184 setOperationAction(ISD::UINT_TO_FP, MVT::v8i8, Custom); in resetOperationActions()
1185 setOperationAction(ISD::UINT_TO_FP, MVT::v8i16, Custom); in resetOperationActions()
1187 setLoadExtAction(ISD::EXTLOAD, MVT::v4f32, Legal); in resetOperationActions()
1189 setOperationAction(ISD::SRL, MVT::v16i16, Custom); in resetOperationActions()
1190 setOperationAction(ISD::SRL, MVT::v32i8, Custom); in resetOperationActions()
1192 setOperationAction(ISD::SHL, MVT::v16i16, Custom); in resetOperationActions()
1193 setOperationAction(ISD::SHL, MVT::v32i8, Custom); in resetOperationActions()
1195 setOperationAction(ISD::SRA, MVT::v16i16, Custom); in resetOperationActions()
1196 setOperationAction(ISD::SRA, MVT::v32i8, Custom); in resetOperationActions()
1198 setOperationAction(ISD::SETCC, MVT::v32i8, Custom); in resetOperationActions()
1199 setOperationAction(ISD::SETCC, MVT::v16i16, Custom); in resetOperationActions()
1200 setOperationAction(ISD::SETCC, MVT::v8i32, Custom); in resetOperationActions()
1201 setOperationAction(ISD::SETCC, MVT::v4i64, Custom); in resetOperationActions()
1203 setOperationAction(ISD::SELECT, MVT::v4f64, Custom); in resetOperationActions()
1204 setOperationAction(ISD::SELECT, MVT::v4i64, Custom); in resetOperationActions()
1205 setOperationAction(ISD::SELECT, MVT::v8f32, Custom); in resetOperationActions()
1207 setOperationAction(ISD::VSELECT, MVT::v4f64, Custom); in resetOperationActions()
1208 setOperationAction(ISD::VSELECT, MVT::v4i64, Custom); in resetOperationActions()
1209 setOperationAction(ISD::VSELECT, MVT::v8i32, Custom); in resetOperationActions()
1210 setOperationAction(ISD::VSELECT, MVT::v8f32, Custom); in resetOperationActions()
1212 setOperationAction(ISD::SIGN_EXTEND, MVT::v4i64, Custom); in resetOperationActions()
1213 setOperationAction(ISD::SIGN_EXTEND, MVT::v8i32, Custom); in resetOperationActions()
1214 setOperationAction(ISD::SIGN_EXTEND, MVT::v16i16, Custom); in resetOperationActions()
1215 setOperationAction(ISD::ZERO_EXTEND, MVT::v4i64, Custom); in resetOperationActions()
1216 setOperationAction(ISD::ZERO_EXTEND, MVT::v8i32, Custom); in resetOperationActions()
1217 setOperationAction(ISD::ZERO_EXTEND, MVT::v16i16, Custom); in resetOperationActions()
1218 setOperationAction(ISD::ANY_EXTEND, MVT::v4i64, Custom); in resetOperationActions()
1219 setOperationAction(ISD::ANY_EXTEND, MVT::v8i32, Custom); in resetOperationActions()
1220 setOperationAction(ISD::ANY_EXTEND, MVT::v16i16, Custom); in resetOperationActions()
1221 setOperationAction(ISD::TRUNCATE, MVT::v16i8, Custom); in resetOperationActions()
1222 setOperationAction(ISD::TRUNCATE, MVT::v8i16, Custom); in resetOperationActions()
1223 setOperationAction(ISD::TRUNCATE, MVT::v4i32, Custom); in resetOperationActions()
1226 setOperationAction(ISD::FMA, MVT::v8f32, Legal); in resetOperationActions()
1227 setOperationAction(ISD::FMA, MVT::v4f64, Legal); in resetOperationActions()
1228 setOperationAction(ISD::FMA, MVT::v4f32, Legal); in resetOperationActions()
1229 setOperationAction(ISD::FMA, MVT::v2f64, Legal); in resetOperationActions()
1230 setOperationAction(ISD::FMA, MVT::f32, Legal); in resetOperationActions()
1231 setOperationAction(ISD::FMA, MVT::f64, Legal); in resetOperationActions()
1235 setOperationAction(ISD::ADD, MVT::v4i64, Legal); in resetOperationActions()
1236 setOperationAction(ISD::ADD, MVT::v8i32, Legal); in resetOperationActions()
1237 setOperationAction(ISD::ADD, MVT::v16i16, Legal); in resetOperationActions()
1238 setOperationAction(ISD::ADD, MVT::v32i8, Legal); in resetOperationActions()
1240 setOperationAction(ISD::SUB, MVT::v4i64, Legal); in resetOperationActions()
1241 setOperationAction(ISD::SUB, MVT::v8i32, Legal); in resetOperationActions()
1242 setOperationAction(ISD::SUB, MVT::v16i16, Legal); in resetOperationActions()
1243 setOperationAction(ISD::SUB, MVT::v32i8, Legal); in resetOperationActions()
1245 setOperationAction(ISD::MUL, MVT::v4i64, Custom); in resetOperationActions()
1246 setOperationAction(ISD::MUL, MVT::v8i32, Legal); in resetOperationActions()
1247 setOperationAction(ISD::MUL, MVT::v16i16, Legal); in resetOperationActions()
1250 setOperationAction(ISD::UMUL_LOHI, MVT::v8i32, Custom); in resetOperationActions()
1251 setOperationAction(ISD::SMUL_LOHI, MVT::v8i32, Custom); in resetOperationActions()
1252 setOperationAction(ISD::MULHU, MVT::v16i16, Legal); in resetOperationActions()
1253 setOperationAction(ISD::MULHS, MVT::v16i16, Legal); in resetOperationActions()
1255 setOperationAction(ISD::VSELECT, MVT::v16i16, Custom); in resetOperationActions()
1256 setOperationAction(ISD::VSELECT, MVT::v32i8, Legal); in resetOperationActions()
1258 setOperationAction(ISD::ADD, MVT::v4i64, Custom); in resetOperationActions()
1259 setOperationAction(ISD::ADD, MVT::v8i32, Custom); in resetOperationActions()
1260 setOperationAction(ISD::ADD, MVT::v16i16, Custom); in resetOperationActions()
1261 setOperationAction(ISD::ADD, MVT::v32i8, Custom); in resetOperationActions()
1263 setOperationAction(ISD::SUB, MVT::v4i64, Custom); in resetOperationActions()
1264 setOperationAction(ISD::SUB, MVT::v8i32, Custom); in resetOperationActions()
1265 setOperationAction(ISD::SUB, MVT::v16i16, Custom); in resetOperationActions()
1266 setOperationAction(ISD::SUB, MVT::v32i8, Custom); in resetOperationActions()
1268 setOperationAction(ISD::MUL, MVT::v4i64, Custom); in resetOperationActions()
1269 setOperationAction(ISD::MUL, MVT::v8i32, Custom); in resetOperationActions()
1270 setOperationAction(ISD::MUL, MVT::v16i16, Custom); in resetOperationActions()
1276 setOperationAction(ISD::SRL, MVT::v4i64, Custom); in resetOperationActions()
1277 setOperationAction(ISD::SRL, MVT::v8i32, Custom); in resetOperationActions()
1279 setOperationAction(ISD::SHL, MVT::v4i64, Custom); in resetOperationActions()
1280 setOperationAction(ISD::SHL, MVT::v8i32, Custom); in resetOperationActions()
1282 setOperationAction(ISD::SRA, MVT::v8i32, Custom); in resetOperationActions()
1285 for (int i = MVT::FIRST_VECTOR_VALUETYPE; in resetOperationActions()
1286 i <= MVT::LAST_VECTOR_VALUETYPE; ++i) { in resetOperationActions()
1287 MVT VT = (MVT::SimpleValueType)i; in resetOperationActions()
1308 for (int i = MVT::v32i8; i != MVT::v4i64; ++i) { in resetOperationActions()
1309 MVT VT = (MVT::SimpleValueType)i; in resetOperationActions()
1316 AddPromotedToType (ISD::AND, VT, MVT::v4i64); in resetOperationActions()
1318 AddPromotedToType (ISD::OR, VT, MVT::v4i64); in resetOperationActions()
1320 AddPromotedToType (ISD::XOR, VT, MVT::v4i64); in resetOperationActions()
1322 AddPromotedToType (ISD::LOAD, VT, MVT::v4i64); in resetOperationActions()
1324 AddPromotedToType (ISD::SELECT, VT, MVT::v4i64); in resetOperationActions()
1329 addRegisterClass(MVT::v16i32, &X86::VR512RegClass); in resetOperationActions()
1330 addRegisterClass(MVT::v16f32, &X86::VR512RegClass); in resetOperationActions()
1331 addRegisterClass(MVT::v8i64, &X86::VR512RegClass); in resetOperationActions()
1332 addRegisterClass(MVT::v8f64, &X86::VR512RegClass); in resetOperationActions()
1334 addRegisterClass(MVT::i1, &X86::VK1RegClass); in resetOperationActions()
1335 addRegisterClass(MVT::v8i1, &X86::VK8RegClass); in resetOperationActions()
1336 addRegisterClass(MVT::v16i1, &X86::VK16RegClass); in resetOperationActions()
1338 setOperationAction(ISD::BR_CC, MVT::i1, Expand); in resetOperationActions()
1339 setOperationAction(ISD::SETCC, MVT::i1, Custom); in resetOperationActions()
1340 setOperationAction(ISD::XOR, MVT::i1, Legal); in resetOperationActions()
1341 setOperationAction(ISD::OR, MVT::i1, Legal); in resetOperationActions()
1342 setOperationAction(ISD::AND, MVT::i1, Legal); in resetOperationActions()
1343 setLoadExtAction(ISD::EXTLOAD, MVT::v8f32, Legal); in resetOperationActions()
1344 setOperationAction(ISD::LOAD, MVT::v16f32, Legal); in resetOperationActions()
1345 setOperationAction(ISD::LOAD, MVT::v8f64, Legal); in resetOperationActions()
1346 setOperationAction(ISD::LOAD, MVT::v8i64, Legal); in resetOperationActions()
1347 setOperationAction(ISD::LOAD, MVT::v16i32, Legal); in resetOperationActions()
1348 setOperationAction(ISD::LOAD, MVT::v16i1, Legal); in resetOperationActions()
1350 setOperationAction(ISD::FADD, MVT::v16f32, Legal); in resetOperationActions()
1351 setOperationAction(ISD::FSUB, MVT::v16f32, Legal); in resetOperationActions()
1352 setOperationAction(ISD::FMUL, MVT::v16f32, Legal); in resetOperationActions()
1353 setOperationAction(ISD::FDIV, MVT::v16f32, Legal); in resetOperationActions()
1354 setOperationAction(ISD::FSQRT, MVT::v16f32, Legal); in resetOperationActions()
1355 setOperationAction(ISD::FNEG, MVT::v16f32, Custom); in resetOperationActions()
1357 setOperationAction(ISD::FADD, MVT::v8f64, Legal); in resetOperationActions()
1358 setOperationAction(ISD::FSUB, MVT::v8f64, Legal); in resetOperationActions()
1359 setOperationAction(ISD::FMUL, MVT::v8f64, Legal); in resetOperationActions()
1360 setOperationAction(ISD::FDIV, MVT::v8f64, Legal); in resetOperationActions()
1361 setOperationAction(ISD::FSQRT, MVT::v8f64, Legal); in resetOperationActions()
1362 setOperationAction(ISD::FNEG, MVT::v8f64, Custom); in resetOperationActions()
1363 setOperationAction(ISD::FMA, MVT::v8f64, Legal); in resetOperationActions()
1364 setOperationAction(ISD::FMA, MVT::v16f32, Legal); in resetOperationActions()
1366 setOperationAction(ISD::FP_TO_SINT, MVT::i32, Legal); in resetOperationActions()
1367 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Legal); in resetOperationActions()
1368 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Legal); in resetOperationActions()
1369 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Legal); in resetOperationActions()
1371 setOperationAction(ISD::FP_TO_UINT, MVT::i64, Legal); in resetOperationActions()
1372 setOperationAction(ISD::FP_TO_SINT, MVT::i64, Legal); in resetOperationActions()
1373 setOperationAction(ISD::SINT_TO_FP, MVT::i64, Legal); in resetOperationActions()
1374 setOperationAction(ISD::UINT_TO_FP, MVT::i64, Legal); in resetOperationActions()
1376 setOperationAction(ISD::FP_TO_SINT, MVT::v16i32, Legal); in resetOperationActions()
1377 setOperationAction(ISD::FP_TO_UINT, MVT::v16i32, Legal); in resetOperationActions()
1378 setOperationAction(ISD::FP_TO_UINT, MVT::v8i32, Legal); in resetOperationActions()
1379 setOperationAction(ISD::FP_TO_UINT, MVT::v4i32, Legal); in resetOperationActions()
1380 setOperationAction(ISD::SINT_TO_FP, MVT::v16i32, Legal); in resetOperationActions()
1381 setOperationAction(ISD::UINT_TO_FP, MVT::v16i32, Legal); in resetOperationActions()
1382 setOperationAction(ISD::UINT_TO_FP, MVT::v8i32, Legal); in resetOperationActions()
1383 setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Legal); in resetOperationActions()
1384 setOperationAction(ISD::FP_ROUND, MVT::v8f32, Legal); in resetOperationActions()
1385 setOperationAction(ISD::FP_EXTEND, MVT::v8f32, Legal); in resetOperationActions()
1387 setOperationAction(ISD::TRUNCATE, MVT::i1, Custom); in resetOperationActions()
1388 setOperationAction(ISD::TRUNCATE, MVT::v16i8, Custom); in resetOperationActions()
1389 setOperationAction(ISD::TRUNCATE, MVT::v8i32, Custom); in resetOperationActions()
1390 setOperationAction(ISD::TRUNCATE, MVT::v8i1, Custom); in resetOperationActions()
1391 setOperationAction(ISD::TRUNCATE, MVT::v16i1, Custom); in resetOperationActions()
1392 setOperationAction(ISD::TRUNCATE, MVT::v16i16, Custom); in resetOperationActions()
1393 setOperationAction(ISD::ZERO_EXTEND, MVT::v16i32, Custom); in resetOperationActions()
1394 setOperationAction(ISD::ZERO_EXTEND, MVT::v8i64, Custom); in resetOperationActions()
1395 setOperationAction(ISD::SIGN_EXTEND, MVT::v16i32, Custom); in resetOperationActions()
1396 setOperationAction(ISD::SIGN_EXTEND, MVT::v8i64, Custom); in resetOperationActions()
1397 setOperationAction(ISD::SIGN_EXTEND, MVT::v16i8, Custom); in resetOperationActions()
1398 setOperationAction(ISD::SIGN_EXTEND, MVT::v8i16, Custom); in resetOperationActions()
1399 setOperationAction(ISD::SIGN_EXTEND, MVT::v16i16, Custom); in resetOperationActions()
1401 setOperationAction(ISD::CONCAT_VECTORS, MVT::v8f64, Custom); in resetOperationActions()
1402 setOperationAction(ISD::CONCAT_VECTORS, MVT::v8i64, Custom); in resetOperationActions()
1403 setOperationAction(ISD::CONCAT_VECTORS, MVT::v16f32, Custom); in resetOperationActions()
1404 setOperationAction(ISD::CONCAT_VECTORS, MVT::v16i32, Custom); in resetOperationActions()
1405 setOperationAction(ISD::CONCAT_VECTORS, MVT::v8i1, Custom); in resetOperationActions()
1406 setOperationAction(ISD::CONCAT_VECTORS, MVT::v16i1, Legal); in resetOperationActions()
1408 setOperationAction(ISD::SETCC, MVT::v16i1, Custom); in resetOperationActions()
1409 setOperationAction(ISD::SETCC, MVT::v8i1, Custom); in resetOperationActions()
1411 setOperationAction(ISD::MUL, MVT::v8i64, Custom); in resetOperationActions()
1413 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v8i1, Custom); in resetOperationActions()
1414 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v16i1, Custom); in resetOperationActions()
1415 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16i1, Custom); in resetOperationActions()
1416 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i1, Custom); in resetOperationActions()
1417 setOperationAction(ISD::BUILD_VECTOR, MVT::v8i1, Custom); in resetOperationActions()
1418 setOperationAction(ISD::BUILD_VECTOR, MVT::v16i1, Custom); in resetOperationActions()
1419 setOperationAction(ISD::SELECT, MVT::v8f64, Custom); in resetOperationActions()
1420 setOperationAction(ISD::SELECT, MVT::v8i64, Custom); in resetOperationActions()
1421 setOperationAction(ISD::SELECT, MVT::v16f32, Custom); in resetOperationActions()
1423 setOperationAction(ISD::ADD, MVT::v8i64, Legal); in resetOperationActions()
1424 setOperationAction(ISD::ADD, MVT::v16i32, Legal); in resetOperationActions()
1426 setOperationAction(ISD::SUB, MVT::v8i64, Legal); in resetOperationActions()
1427 setOperationAction(ISD::SUB, MVT::v16i32, Legal); in resetOperationActions()
1429 setOperationAction(ISD::MUL, MVT::v16i32, Legal); in resetOperationActions()
1431 setOperationAction(ISD::SRL, MVT::v8i64, Custom); in resetOperationActions()
1432 setOperationAction(ISD::SRL, MVT::v16i32, Custom); in resetOperationActions()
1434 setOperationAction(ISD::SHL, MVT::v8i64, Custom); in resetOperationActions()
1435 setOperationAction(ISD::SHL, MVT::v16i32, Custom); in resetOperationActions()
1437 setOperationAction(ISD::SRA, MVT::v8i64, Custom); in resetOperationActions()
1438 setOperationAction(ISD::SRA, MVT::v16i32, Custom); in resetOperationActions()
1440 setOperationAction(ISD::AND, MVT::v8i64, Legal); in resetOperationActions()
1441 setOperationAction(ISD::OR, MVT::v8i64, Legal); in resetOperationActions()
1442 setOperationAction(ISD::XOR, MVT::v8i64, Legal); in resetOperationActions()
1443 setOperationAction(ISD::AND, MVT::v16i32, Legal); in resetOperationActions()
1444 setOperationAction(ISD::OR, MVT::v16i32, Legal); in resetOperationActions()
1445 setOperationAction(ISD::XOR, MVT::v16i32, Legal); in resetOperationActions()
1448 setOperationAction(ISD::CTLZ, MVT::v8i64, Legal); in resetOperationActions()
1449 setOperationAction(ISD::CTLZ, MVT::v16i32, Legal); in resetOperationActions()
1453 for (int i = MVT::FIRST_VECTOR_VALUETYPE; in resetOperationActions()
1454 i <= MVT::LAST_VECTOR_VALUETYPE; ++i) { in resetOperationActions()
1455 MVT VT = (MVT::SimpleValueType)i; in resetOperationActions()
1463 if (VT.getVectorElementType() == MVT::i1) in resetOperationActions()
1480 for (int i = MVT::v32i8; i != MVT::v8i64; ++i) { in resetOperationActions()
1481 MVT VT = (MVT::SimpleValueType)i; in resetOperationActions()
1488 AddPromotedToType (ISD::SELECT, VT, MVT::v8i64); in resetOperationActions()
1494 for (int VT = MVT::FIRST_VECTOR_VALUETYPE; in resetOperationActions()
1495 VT != MVT::LAST_VECTOR_VALUETYPE; VT++) { in resetOperationActions()
1496 setOperationAction(ISD::SIGN_EXTEND_INREG, (MVT::SimpleValueType)VT, in resetOperationActions()
1501 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); in resetOperationActions()
1502 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom); in resetOperationActions()
1503 setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom); in resetOperationActions()
1505 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i64, Custom); in resetOperationActions()
1515 MVT VT = IntVTs[i]; in resetOperationActions()
1525 setOperationAction(ISD::SMULO, MVT::i8, Expand); in resetOperationActions()
1526 setOperationAction(ISD::UMULO, MVT::i8, Expand); in resetOperationActions()
1543 setOperationAction(ISD::FSINCOS, MVT::f64, Custom); in resetOperationActions()
1544 setOperationAction(ISD::FSINCOS, MVT::f32, Custom); in resetOperationActions()
1549 setOperationAction(ISD::SDIV, MVT::i128, Custom); in resetOperationActions()
1550 setOperationAction(ISD::UDIV, MVT::i128, Custom); in resetOperationActions()
1551 setOperationAction(ISD::SREM, MVT::i128, Custom); in resetOperationActions()
1552 setOperationAction(ISD::UREM, MVT::i128, Custom); in resetOperationActions()
1553 setOperationAction(ISD::SDIVREM, MVT::i128, Custom); in resetOperationActions()
1554 setOperationAction(ISD::UDIVREM, MVT::i128, Custom); in resetOperationActions()
1609 VT.getVectorElementType().getSimpleVT() != MVT::i1) in getPreferredVectorAction()
1617 return Subtarget->hasAVX512() ? MVT::i1: MVT::i8; in getSetCCResultType()
1621 case 8: return MVT::v8i1; in getSetCCResultType()
1622 case 16: return MVT::v16i1; in getSetCCResultType()
1699 return MVT::v8i32; in getOptimalMemOpType()
1701 return MVT::v8f32; in getOptimalMemOpType()
1704 return MVT::v4i32; in getOptimalMemOpType()
1706 return MVT::v4f32; in getOptimalMemOpType()
1712 return MVT::f64; in getOptimalMemOpType()
1716 return MVT::i64; in getOptimalMemOpType()
1717 return MVT::i32; in getOptimalMemOpType()
1720 bool X86TargetLowering::isSafeMemOpType(MVT VT) const { in isSafeMemOpType()
1721 if (VT == MVT::f32) in isSafeMemOpType()
1723 else if (VT == MVT::f64) in isSafeMemOpType()
1790 X86TargetLowering::findRepresentativeClass(MVT VT) const{ in findRepresentativeClass()
1796 case MVT::i8: case MVT::i16: case MVT::i32: case MVT::i64: in findRepresentativeClass()
1801 case MVT::x86mmx: in findRepresentativeClass()
1804 case MVT::f32: case MVT::f64: in findRepresentativeClass()
1805 case MVT::v16i8: case MVT::v8i16: case MVT::v4i32: case MVT::v2i64: in findRepresentativeClass()
1806 case MVT::v4f32: case MVT::v2f64: in findRepresentativeClass()
1807 case MVT::v32i8: case MVT::v8i32: case MVT::v4i64: case MVT::v8f32: in findRepresentativeClass()
1808 case MVT::v4f64: in findRepresentativeClass()
1883 MVT::i16)); in LowerReturn()
1907 if ((ValVT == MVT::f32 || ValVT == MVT::f64 || in LowerReturn()
1915 if (ValVT == MVT::f64 && in LowerReturn()
1926 ValToCopy = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f80, ValToCopy); in LowerReturn()
1935 if (ValVT == MVT::x86mmx) { in LowerReturn()
1937 ValToCopy = DAG.getNode(ISD::BITCAST, dl, MVT::i64, ValToCopy); in LowerReturn()
1938 ValToCopy = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, in LowerReturn()
1943 ValToCopy = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32,ValToCopy); in LowerReturn()
1983 return DAG.getNode(X86ISD::RET_FLAG, dl, MVT::Other, RetOps); in LowerReturn()
1997 if (Copy->getOperand(Copy->getNumOperands()-1).getValueType() == MVT::Glue) in isUsedByReturnOnly()
2018 MVT
2019 X86TargetLowering::getTypeForExtArgOrReturn(MVT VT, in getTypeForExtArgOrReturn()
2021 MVT ReturnMVT; in getTypeForExtArgOrReturn()
2023 if (Subtarget->is64Bit() && VT == MVT::i1 && ExtendKind == ISD::ZERO_EXTEND) in getTypeForExtArgOrReturn()
2024 ReturnMVT = MVT::i8; in getTypeForExtArgOrReturn()
2026 ReturnMVT = MVT::i32; in getTypeForExtArgOrReturn()
2028 MVT MinVT = getRegisterType(ReturnMVT); in getTypeForExtArgOrReturn()
2055 if ((CopyVT == MVT::f32 || CopyVT == MVT::f64) && in LowerCallResult()
2070 if (isScalarFPTypeInSSEReg(VA.getValVT())) CopyVT = MVT::f80; in LowerCallResult()
2073 MVT::Other, MVT::Glue, Ops), 1); in LowerCallResult()
2147 SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), MVT::i32); in CreateCopyOfByValArgument()
2277 if (RegVT == MVT::i32) in LowerFormalArguments()
2279 else if (Is64Bit && RegVT == MVT::i64) in LowerFormalArguments()
2281 else if (RegVT == MVT::f32) in LowerFormalArguments()
2283 else if (RegVT == MVT::f64) in LowerFormalArguments()
2291 else if (RegVT == MVT::x86mmx) in LowerFormalArguments()
2293 else if (RegVT == MVT::i1) in LowerFormalArguments()
2295 else if (RegVT == MVT::v8i1) in LowerFormalArguments()
2297 else if (RegVT == MVT::v16i1) in LowerFormalArguments()
2347 MVT PtrTy = getPointerTy(); in LowerFormalArguments()
2352 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Copy, Chain); in LowerFormalArguments()
2448 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i64); in LowerFormalArguments()
2464 SDValue ALVal = DAG.getCopyFromReg(DAG.getEntryNode(), dl, AL, MVT::i8); in LowerFormalArguments()
2475 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, MVT::v4f32); in LowerFormalArguments()
2479 MVT::Other, SaveXMMOps)); in LowerFormalArguments()
2483 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps); in LowerFormalArguments()
2704 Arg = DAG.getNode(ISD::BITCAST, dl, MVT::i64, Arg); in LowerCall()
2705 Arg = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Arg); in LowerCall()
2706 Arg = getMOVL(DAG, dl, MVT::v2i64, DAG.getUNDEF(MVT::v2i64), Arg); in LowerCall()
2751 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains); in LowerCall()
2797 DAG.getConstant(NumXMMRegs, MVT::i8))); in LowerCall()
2853 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains2); in LowerCall()
2952 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); in LowerCall()
2966 Ops.push_back(DAG.getConstant(FPDiff, MVT::i32)); in LowerCall()
3410 return DAG.getNode(Opc, dl, VT, V1, DAG.getConstant(TargetMask, MVT::i8)); in getTargetShuffleNode()
3423 DAG.getConstant(TargetMask, MVT::i8)); in getTargetShuffleNode()
3692 static bool isPSHUFDMask(ArrayRef<int> Mask, MVT VT) { in isPSHUFDMask()
3693 if (VT == MVT::v4f32 || VT == MVT::v4i32 ) in isPSHUFDMask()
3695 if (VT == MVT::v2f64 || VT == MVT::v2i64) in isPSHUFDMask()
3702 static bool isPSHUFHWMask(ArrayRef<int> Mask, MVT VT, bool HasInt256) { in isPSHUFHWMask()
3703 if (VT != MVT::v8i16 && (!HasInt256 || VT != MVT::v16i16)) in isPSHUFHWMask()
3715 if (VT == MVT::v16i16) { in isPSHUFHWMask()
3731 static bool isPSHUFLWMask(ArrayRef<int> Mask, MVT VT, bool HasInt256) { in isPSHUFLWMask()
3732 if (VT != MVT::v8i16 && (!HasInt256 || VT != MVT::v16i16)) in isPSHUFLWMask()
3744 if (VT == MVT::v16i16) { in isPSHUFLWMask()
3760 static bool isPALIGNRMask(ArrayRef<int> Mask, MVT VT, in isPALIGNRMask()
3850 static bool isSHUFPMask(ArrayRef<int> Mask, MVT VT, bool Commuted = false) { in isSHUFPMask()
3909 static bool isMOVHLPSMask(ArrayRef<int> Mask, MVT VT) { in isMOVHLPSMask()
3928 static bool isMOVHLPS_v_undef_Mask(ArrayRef<int> Mask, MVT VT) { in isMOVHLPS_v_undef_Mask()
3945 static bool isMOVLPMask(ArrayRef<int> Mask, MVT VT) { in isMOVLPMask()
3967 static bool isMOVLHPSMask(ArrayRef<int> Mask, MVT VT) { in isMOVLHPSMask()
3990 static bool isINSERTPSMask(ArrayRef<int> Mask, MVT VT) { in isINSERTPSMask()
3992 if (!VT.is128BitVector() || (VT != MVT::v4f32 && VT != MVT::v4i32)) in isINSERTPSMask()
4024 MVT VT = SVOp->getSimpleValueType(0); in Compact8x32ShuffleNode()
4027 if (VT != MVT::v8i32 && VT != MVT::v8f32) in Compact8x32ShuffleNode()
4067 static bool isUNPCKLMask(ArrayRef<int> Mask, MVT VT, in isUNPCKLMask()
4115 static bool isUNPCKHMask(ArrayRef<int> Mask, MVT VT, in isUNPCKHMask()
4163 static bool isUNPCKL_v_undef_Mask(ArrayRef<int> Mask, MVT VT, bool HasInt256) { in isUNPCKL_v_undef_Mask()
4206 static bool isUNPCKH_v_undef_Mask(ArrayRef<int> Mask, MVT VT, bool HasInt256) { in isUNPCKH_v_undef_Mask()
4239 static bool isINSERT64x4Mask(ArrayRef<int> Mask, MVT VT, unsigned int *Imm) { in isINSERT64x4Mask()
4287 static bool isVPERM2X128Mask(ArrayRef<int> Mask, MVT VT, bool HasFp256) { in isVPERM2X128Mask()
4319 MVT VT = SVOp->getSimpleValueType(0); in getShuffleVPERM2X128Immediate()
4341 static bool isPermImmMask(ArrayRef<int> Mask, MVT VT, unsigned& Imm8) { in isPermImmMask()
4385 static bool isVPERMILPMask(ArrayRef<int> Mask, MVT VT) { in isVPERMILPMask()
4417 static bool isCommutedMOVLMask(ArrayRef<int> Mask, MVT VT, in isCommutedMOVLMask()
4441 static bool isMOVSHDUPMask(ArrayRef<int> Mask, MVT VT, in isMOVSHDUPMask()
4465 static bool isMOVSLDUPMask(ArrayRef<int> Mask, MVT VT, in isMOVSLDUPMask()
4489 static bool isMOVDDUPYMask(ArrayRef<int> Mask, MVT VT, bool HasFp256) { in isMOVDDUPYMask()
4509 static bool isMOVDDUPMask(ArrayRef<int> Mask, MVT VT) { in isMOVDDUPMask()
4535 MVT VT = N->getSimpleValueType(0); in isVEXTRACTIndex()
4553 MVT VT = N->getSimpleValueType(0); in isVINSERTIndex()
4580 MVT VT = N->getSimpleValueType(0); in getShuffleSHUFImmediate()
4610 MVT VT = N->getSimpleValueType(0); in getShufflePSHUFHWImmediate()
4612 assert((VT == MVT::v8i16 || VT == MVT::v16i16) && in getShufflePSHUFHWImmediate()
4634 MVT VT = N->getSimpleValueType(0); in getShufflePSHUFLWImmediate()
4636 assert((VT == MVT::v8i16 || VT == MVT::v16i16) && in getShufflePSHUFLWImmediate()
4658 MVT VT = SVOp->getSimpleValueType(0); in getShufflePALIGNRImmediate()
4688 MVT VecVT = N->getOperand(0).getSimpleValueType(); in getExtractVEXTRACTImmediate()
4689 MVT ElVT = VecVT.getVectorElementType(); in getExtractVEXTRACTImmediate()
4703 MVT VecVT = N->getSimpleValueType(0); in getInsertVINSERTImmediate()
4704 MVT ElVT = VecVT.getVectorElementType(); in getInsertVINSERTImmediate()
4758 MVT VT = SVOp->getSimpleValueType(0); in CommuteVectorShuffle()
4780 static bool ShouldXformToMOVHLPS(ArrayRef<int> Mask, MVT VT) { in ShouldXformToMOVHLPS()
4837 ArrayRef<int> Mask, MVT VT) { in ShouldXformToMOVLP()
4900 SDValue Cst = DAG.getTargetConstant(0, MVT::i32); in getZeroVector()
4901 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, Cst, Cst, Cst, Cst); in getZeroVector()
4903 SDValue Cst = DAG.getTargetConstantFP(+0.0, MVT::f32); in getZeroVector()
4904 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4f32, Cst, Cst, Cst, Cst); in getZeroVector()
4908 SDValue Cst = DAG.getTargetConstant(0, MVT::i32); in getZeroVector()
4910 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v8i32, Ops); in getZeroVector()
4914 SDValue Cst = DAG.getTargetConstantFP(+0.0, MVT::f32); in getZeroVector()
4916 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v8f32, Ops); in getZeroVector()
4919 SDValue Cst = DAG.getTargetConstant(0, MVT::i32); in getZeroVector()
4922 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v16i32, Ops); in getZeroVector()
4923 } else if (VT.getScalarType() == MVT::i1) { in getZeroVector()
4925 SDValue Cst = DAG.getTargetConstant(0, MVT::i1); in getZeroVector()
4938 static SDValue getOnesVector(MVT VT, bool HasInt256, SelectionDAG &DAG, in getOnesVector()
4942 SDValue Cst = DAG.getTargetConstant(~0U, MVT::i32); in getOnesVector()
4947 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v8i32, Ops); in getOnesVector()
4949 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, Cst, Cst, Cst, Cst); in getOnesVector()
4950 Vec = Concat128BitVectors(Vec, Vec, MVT::v8i32, 8, DAG, dl); in getOnesVector()
4953 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, Cst, Cst, Cst, Cst); in getOnesVector()
4983 static SDValue getUnpackl(SelectionDAG &DAG, SDLoc dl, MVT VT, SDValue V1, in getUnpackl()
4995 static SDValue getUnpackh(SelectionDAG &DAG, SDLoc dl, MVT VT, SDValue V1, in getUnpackh()
5011 MVT VT = V.getSimpleValueType(); in PromoteSplati8i16()
5029 MVT VT = V.getSimpleValueType(); in getLegalSplat()
5033 V = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, V); in getLegalSplat()
5035 V = DAG.getVectorShuffle(MVT::v4f32, dl, V, DAG.getUNDEF(MVT::v4f32), in getLegalSplat()
5044 V = DAG.getNode(ISD::BITCAST, dl, MVT::v8f32, V); in getLegalSplat()
5045 V = DAG.getVectorShuffle(MVT::v8f32, dl, V, DAG.getUNDEF(MVT::v8f32), in getLegalSplat()
5055 MVT SrcVT = SV->getSimpleValueType(0); in PromoteSplat()
5078 MVT EltVT = SrcVT.getVectorElementType(); in PromoteSplat()
5079 if (EltVT == MVT::i8 || EltVT == MVT::i16) in PromoteSplat()
5100 MVT VT = V2.getSimpleValueType(); in getShuffleVectorZeroOrUndef()
5114 static bool getTargetShuffleMask(SDNode *N, MVT VT, in getTargetShuffleMask()
5218 MVT ShufVT = V.getSimpleValueType(); in getShuffleScalarElt()
5415 V = getZeroVector(MVT::v8i16, Subtarget, DAG, dl); in LowerBuildVectorv16i8()
5417 V = DAG.getUNDEF(MVT::v8i16); in LowerBuildVectorv16i8()
5426 MVT::i16, Op.getOperand(i-1)); in LowerBuildVectorv16i8()
5429 ThisElt = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, Op.getOperand(i)); in LowerBuildVectorv16i8()
5430 ThisElt = DAG.getNode(ISD::SHL, dl, MVT::i16, in LowerBuildVectorv16i8()
5431 ThisElt, DAG.getConstant(8, MVT::i8)); in LowerBuildVectorv16i8()
5433 ThisElt = DAG.getNode(ISD::OR, dl, MVT::i16, ThisElt, LastElt); in LowerBuildVectorv16i8()
5438 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, V, ThisElt, in LowerBuildVectorv16i8()
5443 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, V); in LowerBuildVectorv16i8()
5464 V = getZeroVector(MVT::v8i16, Subtarget, DAG, dl); in LowerBuildVectorv8i16()
5466 V = DAG.getUNDEF(MVT::v8i16); in LowerBuildVectorv8i16()
5470 MVT::v8i16, V, Op.getOperand(i), in LowerBuildVectorv8i16()
5498 MVT VVT = V.getSimpleValueType(); in LowerBuildVectorv4x32()
5499 if (!Subtarget->hasSSE41() || (VVT != MVT::v4f32 && VVT != MVT::v4i32)) in LowerBuildVectorv4x32()
5557 EVT ShVT = MVT::v2i64; in getVShift()
5567 LowerAsSplatVectorLoad(SDValue SrcOp, MVT VT, SDLoc dl, SelectionDAG &DAG) { in LowerAsSplatVectorLoad()
5577 if (PVT != MVT::i32 && PVT != MVT::f32) in LowerAsSplatVectorLoad()
5705 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, in EltsFromConsecutiveLoads()
5716 DAG.getTargetLoweringInfo().isTypeLegal(MVT::v2i64)) { in EltsFromConsecutiveLoads()
5717 SDVTList Tys = DAG.getVTList(MVT::v2i64, MVT::Other); in EltsFromConsecutiveLoads()
5720 DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, DL, Tys, Ops, MVT::i64, in EltsFromConsecutiveLoads()
5730 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, in EltsFromConsecutiveLoads()
5754 MVT VT = Op.getSimpleValueType(); in LowerVectorBroadcast()
5907 MVT ShuffleVecVT = ShuffleVec.getSimpleValueType(); in getUnderlyingExtractedFromVec()
5920 MVT VT = Op.getSimpleValueType(); in buildFromShuffleMostly()
5995 MVT VT = Op.getSimpleValueType(); in LowerBUILD_VECTORvXi1()
5996 assert((VT.getVectorElementType() == MVT::i1) && (VT.getSizeInBits() <= 16) && in LowerBUILD_VECTORvXi1()
6001 SDValue Cst = DAG.getTargetConstant(0, MVT::i1); in LowerBUILD_VECTORvXi1()
6007 SDValue Cst = DAG.getTargetConstant(1, MVT::i1); in LowerBUILD_VECTORvXi1()
6037 SDValue FullMask = DAG.getNode(ISD::BITCAST, dl, MVT::v16i1, in LowerBUILD_VECTORvXi1()
6038 DAG.getConstant(Immediate, MVT::i16)); in LowerBUILD_VECTORvXi1()
6047 MVT::getIntegerVT(VT.getSizeInBits())); in LowerBUILD_VECTORvXi1()
6058 MVT SelectVT = (VT == MVT::v16i1)? MVT::i16 : MVT::i8; in LowerBUILD_VECTORvXi1()
6241 assert((VT == MVT::v8f32 || VT == MVT::v4f64 || VT == MVT::v4f32 || in matchAddSub()
6242 VT == MVT::v2f64) && "build_vector with an invalid type found!"); in matchAddSub()
6361 if ((Subtarget->hasSSE3() && (VT == MVT::v4f32 || VT == MVT::v2f64)) || in PerformBUILD_VECTORCombine()
6362 (Subtarget->hasAVX() && (VT == MVT::v8f32 || VT == MVT::v4f64))) { in PerformBUILD_VECTORCombine()
6387 if ((VT == MVT::v4f32 || VT == MVT::v2f64) && Subtarget->hasSSE3()) { in PerformBUILD_VECTORCombine()
6394 } else if ((VT == MVT::v4i32 || VT == MVT::v8i16) && Subtarget->hasSSSE3()) { in PerformBUILD_VECTORCombine()
6406 if ((VT == MVT::v8f32 || VT == MVT::v4f64)) { in PerformBUILD_VECTORCombine()
6425 } else if (VT == MVT::v8i32 || VT == MVT::v16i16) { in PerformBUILD_VECTORCombine()
6468 if ((VT == MVT::v8f32 || VT == MVT::v4f64 || VT == MVT::v8i32 || in PerformBUILD_VECTORCombine()
6469 VT == MVT::v16i16) && Subtarget->hasAVX()) { in PerformBUILD_VECTORCombine()
6502 MVT VT = Op.getSimpleValueType(); in LowerBUILD_VECTOR()
6503 MVT ExtVT = VT.getVectorElementType(); in LowerBUILD_VECTOR()
6507 if (VT.getScalarType() == MVT::i1 && Subtarget->hasAVX512()) in LowerBUILD_VECTOR()
6514 if (VT == MVT::v4i32 || VT == MVT::v8i32 || VT == MVT::v16i32) in LowerBUILD_VECTOR()
6524 if (VT == MVT::v4i32 || (VT == MVT::v8i32 && Subtarget->hasInt256())) in LowerBUILD_VECTOR()
6572 if (ExtVT == MVT::i64 && !Subtarget->is64Bit() && in LowerBUILD_VECTOR()
6576 assert(VT == MVT::v2i64 && "Expected an SSE value type!"); in LowerBUILD_VECTOR()
6577 EVT VecVT = MVT::v4i32; in LowerBUILD_VECTOR()
6582 Item = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Item); in LowerBUILD_VECTOR()
6608 if (ExtVT == MVT::i32 || ExtVT == MVT::f32 || ExtVT == MVT::f64 || in LowerBUILD_VECTOR()
6609 (ExtVT == MVT::i64 && Subtarget->is64Bit())) { in LowerBUILD_VECTOR()
6621 if (ExtVT == MVT::i16 || ExtVT == MVT::i8) { in LowerBUILD_VECTOR()
6622 Item = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, Item); in LowerBUILD_VECTOR()
6623 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, Item); in LowerBUILD_VECTOR()
6625 SDValue ZeroVec = getZeroVector(MVT::v8i32, Subtarget, DAG, dl); in LowerBUILD_VECTOR()
6850 MVT ResVT = Op.getSimpleValueType(); in LowerAVXCONCAT_VECTORS()
6862 MVT HalfVT = MVT::getVectorVT(ResVT.getScalarType(), in LowerAVXCONCAT_VECTORS()
6873 MVT LLVM_ATTRIBUTE_UNUSED VT = Op.getSimpleValueType(); in LowerCONCAT_VECTORS()
6947 return DAG.getConstant(Imm, MVT::i8); in getV4X86ShuffleImm8ForMask()
6961 assert(Op.getSimpleValueType() == MVT::v2f64 && "Bad shuffle type!"); in lowerV2F64VectorShuffle()
6962 assert(V1.getSimpleValueType() == MVT::v2f64 && "Bad operand type!"); in lowerV2F64VectorShuffle()
6963 assert(V2.getSimpleValueType() == MVT::v2f64 && "Bad operand type!"); in lowerV2F64VectorShuffle()
6972 return DAG.getNode(X86ISD::SHUFP, SDLoc(Op), MVT::v2f64, V1, V1, in lowerV2F64VectorShuffle()
6973 DAG.getConstant(SHUFPDMask, MVT::i8)); in lowerV2F64VectorShuffle()
6979 return DAG.getNode(X86ISD::SHUFP, SDLoc(Op), MVT::v2f64, V1, V2, in lowerV2F64VectorShuffle()
6980 DAG.getConstant(SHUFPDMask, MVT::i8)); in lowerV2F64VectorShuffle()
6993 assert(Op.getSimpleValueType() == MVT::v2i64 && "Bad shuffle type!"); in lowerV2I64VectorShuffle()
6994 assert(V1.getSimpleValueType() == MVT::v2i64 && "Bad operand type!"); in lowerV2I64VectorShuffle()
6995 assert(V2.getSimpleValueType() == MVT::v2i64 && "Bad operand type!"); in lowerV2I64VectorShuffle()
7004 V1 = DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, V1); in lowerV2I64VectorShuffle()
7009 ISD::BITCAST, DL, MVT::v2i64, in lowerV2I64VectorShuffle()
7010 DAG.getNode(X86ISD::PSHUFD, SDLoc(Op), MVT::v4i32, V1, in lowerV2I64VectorShuffle()
7018 V1 = DAG.getNode(ISD::BITCAST, DL, MVT::v2f64, V1); in lowerV2I64VectorShuffle()
7019 V2 = DAG.getNode(ISD::BITCAST, DL, MVT::v2f64, V2); in lowerV2I64VectorShuffle()
7020 return DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, in lowerV2I64VectorShuffle()
7021 DAG.getVectorShuffle(MVT::v2f64, DL, V1, V2, Mask)); in lowerV2I64VectorShuffle()
7033 assert(Op.getSimpleValueType() == MVT::v4f32 && "Bad shuffle type!"); in lowerV4F32VectorShuffle()
7034 assert(V1.getSimpleValueType() == MVT::v4f32 && "Bad operand type!"); in lowerV4F32VectorShuffle()
7035 assert(V2.getSimpleValueType() == MVT::v4f32 && "Bad operand type!"); in lowerV4F32VectorShuffle()
7049 return DAG.getNode(X86ISD::SHUFP, DL, MVT::v4f32, V1, V1, in lowerV4F32VectorShuffle()
7072 V2 = DAG.getNode(X86ISD::SHUFP, DL, MVT::v4f32, V2, V1, in lowerV4F32VectorShuffle()
7103 V1 = DAG.getNode(X86ISD::SHUFP, DL, MVT::v4f32, V1, V2, in lowerV4F32VectorShuffle()
7115 return DAG.getNode(X86ISD::SHUFP, DL, MVT::v4f32, LowV, HighV, in lowerV4F32VectorShuffle()
7127 assert(Op.getSimpleValueType() == MVT::v4i32 && "Bad shuffle type!"); in lowerV4I32VectorShuffle()
7128 assert(V1.getSimpleValueType() == MVT::v4i32 && "Bad operand type!"); in lowerV4I32VectorShuffle()
7129 assert(V2.getSimpleValueType() == MVT::v4i32 && "Bad operand type!"); in lowerV4I32VectorShuffle()
7137 return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, V1, in lowerV4I32VectorShuffle()
7145 return DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, in lowerV4I32VectorShuffle()
7147 MVT::v4f32, DL, in lowerV4I32VectorShuffle()
7148 DAG.getNode(ISD::BITCAST, DL, MVT::v4f32, V1), in lowerV4I32VectorShuffle()
7149 DAG.getNode(ISD::BITCAST, DL, MVT::v4f32, V2), Mask)); in lowerV4I32VectorShuffle()
7166 assert(V.getSimpleValueType() == MVT::v8i16 && "Bad input type!"); in lowerV8I16SingleInputVectorShuffle()
7215 V = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, in lowerV8I16SingleInputVectorShuffle()
7216 DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, in lowerV8I16SingleInputVectorShuffle()
7217 DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, V), in lowerV8I16SingleInputVectorShuffle()
7229 return DAG.getVectorShuffle(MVT::v8i16, DL, V, DAG.getUNDEF(MVT::v8i16), in lowerV8I16SingleInputVectorShuffle()
7398 V = DAG.getNode(X86ISD::PSHUFLW, DL, MVT::v8i16, V, in lowerV8I16SingleInputVectorShuffle()
7401 V = DAG.getNode(X86ISD::PSHUFHW, DL, MVT::v8i16, V, in lowerV8I16SingleInputVectorShuffle()
7404 V = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, in lowerV8I16SingleInputVectorShuffle()
7405 DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, in lowerV8I16SingleInputVectorShuffle()
7406 DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, V), in lowerV8I16SingleInputVectorShuffle()
7420 V = DAG.getNode(X86ISD::PSHUFLW, DL, MVT::v8i16, V, in lowerV8I16SingleInputVectorShuffle()
7428 V = DAG.getNode(X86ISD::PSHUFHW, DL, MVT::v8i16, V, in lowerV8I16SingleInputVectorShuffle()
7485 assert(V1.getSimpleValueType() == MVT::v8i16 && "Bad input type!"); in lowerV8I16BasicBlendVectorShuffle()
7486 assert(V2.getSimpleValueType() == MVT::v8i16 && "Bad input type!"); in lowerV8I16BasicBlendVectorShuffle()
7562 return DAG.getVectorShuffle(MVT::v8i16, DL, V, DAG.getUNDEF(MVT::v8i16), in lowerV8I16BasicBlendVectorShuffle()
7580 MVT::v8i16, DL, DAG.getNode(MergeFromLo ? X86ISD::UNPCKL : X86ISD::UNPCKH, in lowerV8I16BasicBlendVectorShuffle()
7581 DL, MVT::v8i16, V1, V2), in lowerV8I16BasicBlendVectorShuffle()
7582 DAG.getUNDEF(MVT::v8i16), Mask); in lowerV8I16BasicBlendVectorShuffle()
7601 assert(Op.getSimpleValueType() == MVT::v8i16 && "Bad shuffle type!"); in lowerV8I16VectorShuffle()
7602 assert(V1.getSimpleValueType() == MVT::v8i16 && "Bad operand type!"); in lowerV8I16VectorShuffle()
7603 assert(V2.getSimpleValueType() == MVT::v8i16 && "Bad operand type!"); in lowerV8I16VectorShuffle()
7644 SDValue Evens = DAG.getVectorShuffle(MVT::v8i16, DL, V1, V2, EMask); in lowerV8I16VectorShuffle()
7645 SDValue Odds = DAG.getVectorShuffle(MVT::v8i16, DL, V1, V2, OMask); in lowerV8I16VectorShuffle()
7647 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v8i16, Evens, Odds); in lowerV8I16VectorShuffle()
7658 SDValue LoV = DAG.getVectorShuffle(MVT::v8i16, DL, V1, V2, LoBlendMask); in lowerV8I16VectorShuffle()
7659 SDValue HiV = DAG.getVectorShuffle(MVT::v8i16, DL, V1, V2, HiBlendMask); in lowerV8I16VectorShuffle()
7660 LoV = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, LoV); in lowerV8I16VectorShuffle()
7661 HiV = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, HiV); in lowerV8I16VectorShuffle()
7663 return DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, in lowerV8I16VectorShuffle()
7664 DAG.getNode(X86ISD::UNPCKL, DL, MVT::v2i64, LoV, HiV)); in lowerV8I16VectorShuffle()
7678 assert(Op.getSimpleValueType() == MVT::v16i8 && "Bad shuffle type!"); in lowerV16I8VectorShuffle()
7679 assert(V1.getSimpleValueType() == MVT::v16i8 && "Bad operand type!"); in lowerV16I8VectorShuffle()
7680 assert(V2.getSimpleValueType() == MVT::v16i8 && "Bad operand type!"); in lowerV16I8VectorShuffle()
7758 ISD::BITCAST, DL, MVT::v16i8, in lowerV16I8VectorShuffle()
7759 DAG.getVectorShuffle(MVT::v8i16, DL, in lowerV16I8VectorShuffle()
7760 DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V1), in lowerV16I8VectorShuffle()
7761 DAG.getUNDEF(MVT::v8i16), PreDupI16Shuffle)); in lowerV16I8VectorShuffle()
7765 MVT::v16i8, V1, V1); in lowerV16I8VectorShuffle()
7774 ISD::BITCAST, DL, MVT::v16i8, in lowerV16I8VectorShuffle()
7775 DAG.getVectorShuffle(MVT::v8i16, DL, in lowerV16I8VectorShuffle()
7776 DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V1), in lowerV16I8VectorShuffle()
7777 DAG.getUNDEF(MVT::v8i16), PostDupI16Shuffle)); in lowerV16I8VectorShuffle()
7800 SDValue Evens = DAG.getVectorShuffle(MVT::v16i8, DL, V1, V2, EMask); in lowerV16I8VectorShuffle()
7801 SDValue Odds = DAG.getVectorShuffle(MVT::v16i8, DL, V1, V2, OMask); in lowerV16I8VectorShuffle()
7803 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v16i8, Evens, Odds); in lowerV16I8VectorShuffle()
7826 SDValue Zero = getZeroVector(MVT::v8i16, Subtarget, DAG, DL); in lowerV16I8VectorShuffle()
7839 V1 = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V); in lowerV16I8VectorShuffle()
7840 V1 = DAG.getNode(ISD::AND, DL, MVT::v8i16, V1, in lowerV16I8VectorShuffle()
7841 DAG.getConstant(0x00FF, MVT::v8i16)); in lowerV16I8VectorShuffle()
7844 V2 = DAG.getUNDEF(MVT::v8i16); in lowerV16I8VectorShuffle()
7856 V1 = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, in lowerV16I8VectorShuffle()
7857 DAG.getNode(X86ISD::UNPCKL, DL, MVT::v16i8, V, Zero)); in lowerV16I8VectorShuffle()
7858 V2 = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, in lowerV16I8VectorShuffle()
7859 DAG.getNode(X86ISD::UNPCKH, DL, MVT::v16i8, V, Zero)); in lowerV16I8VectorShuffle()
7862 SDValue BlendedLo = DAG.getVectorShuffle(MVT::v8i16, DL, V1, V2, LoBlendMask); in lowerV16I8VectorShuffle()
7863 SDValue BlendedHi = DAG.getVectorShuffle(MVT::v8i16, DL, V1, V2, HiBlendMask); in lowerV16I8VectorShuffle()
7870 SDValue LoV = DAG.getVectorShuffle(MVT::v8i16, DL, V1Lo, V2Lo, LoMask); in lowerV16I8VectorShuffle()
7871 SDValue HiV = DAG.getVectorShuffle(MVT::v8i16, DL, V1Hi, V2Hi, HiMask); in lowerV16I8VectorShuffle()
7873 return DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, LoV, HiV); in lowerV16I8VectorShuffle()
7881 MVT VT, const X86Subtarget *Subtarget, in lower128BitVectorShuffle()
7884 case MVT::v2i64: in lower128BitVectorShuffle()
7886 case MVT::v2f64: in lower128BitVectorShuffle()
7888 case MVT::v4i32: in lower128BitVectorShuffle()
7890 case MVT::v4f32: in lower128BitVectorShuffle()
7892 case MVT::v8i16: in lower128BitVectorShuffle()
7894 case MVT::v16i8: in lower128BitVectorShuffle()
7924 MVT VT = Op.getSimpleValueType(); in lowerVectorShuffle()
7963 MVT NewVT = in lowerVectorShuffle()
7964 MVT::getVectorVT(MVT::getIntegerVT(VT.getScalarSizeInBits() * 2), in lowerVectorShuffle()
8015 static bool isBlendMask(ArrayRef<int> MaskVals, MVT VT, bool hasSSE41, in isBlendMask()
8017 MVT EltVT = VT.getVectorElementType(); in isBlendMask()
8023 if (!hasSSE41 || EltVT == MVT::i8) in isBlendMask()
8025 if (!hasInt256 && VT == MVT::v16i16) in isBlendMask()
8064 MVT VT = SVOp->getSimpleValueType(0); in LowerVECTOR_SHUFFLEtoBlend()
8065 MVT EltVT = VT.getVectorElementType(); in LowerVECTOR_SHUFFLEtoBlend()
8077 MVT BlendVT = VT; in LowerVECTOR_SHUFFLEtoBlend()
8078 if (EltVT == MVT::i64 || (EltVT == MVT::i32 && !Subtarget->hasInt256())) { in LowerVECTOR_SHUFFLEtoBlend()
8079 BlendVT = MVT::getVectorVT(MVT::getFloatingPointVT(EltVT.getSizeInBits()), in LowerVECTOR_SHUFFLEtoBlend()
8086 DAG.getConstant(MaskValue, MVT::i32)); in LowerVECTOR_SHUFFLEtoBlend()
8092 static bool ShuffleCrosses128bitLane(MVT VT, unsigned InputIdx, in ShuffleCrosses128bitLane()
8105 MVT VT = V1.getSimpleValueType(); in getPSHUFB()
8108 MVT EltVT = VT.getVectorElementType(); in getPSHUFB()
8129 PshufbMask.push_back(DAG.getConstant(InputByteIdx, MVT::i8)); in getPSHUFB()
8135 MVT ShufVT = MVT::getVectorVT(MVT::i8, PshufbMask.size()); in getPSHUFB()
8225 NewV = DAG.getVectorShuffle(MVT::v2i64, dl, in LowerVECTOR_SHUFFLEv8i16()
8226 DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, V1), in LowerVECTOR_SHUFFLEv8i16()
8227 DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, V2), &MaskV[0]); in LowerVECTOR_SHUFFLEv8i16()
8228 NewV = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, NewV); in LowerVECTOR_SHUFFLEv8i16()
8267 NewV = DAG.getVectorShuffle(MVT::v8i16, dl, NewV, in LowerVECTOR_SHUFFLEv8i16()
8268 DAG.getUNDEF(MVT::v8i16), &MaskVals[0]); in LowerVECTOR_SHUFFLEv8i16()
8273 return getTargetShuffleNode(Opc, dl, MVT::v8i16, V1, TargetMask, DAG); in LowerVECTOR_SHUFFLEv8i16()
8295 return DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V1); in LowerVECTOR_SHUFFLEv8i16()
8301 V1 = DAG.getNode(ISD::OR, dl, MVT::v16i8, V1, V2); in LowerVECTOR_SHUFFLEv8i16()
8302 return DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V1); in LowerVECTOR_SHUFFLEv8i16()
8319 NewV = DAG.getVectorShuffle(MVT::v8i16, dl, NewV, DAG.getUNDEF(MVT::v8i16), in LowerVECTOR_SHUFFLEv8i16()
8324 NewV = getTargetShuffleNode(X86ISD::PSHUFLW, dl, MVT::v8i16, in LowerVECTOR_SHUFFLEv8i16()
8343 NewV = DAG.getVectorShuffle(MVT::v8i16, dl, NewV, DAG.getUNDEF(MVT::v8i16), in LowerVECTOR_SHUFFLEv8i16()
8348 NewV = getTargetShuffleNode(X86ISD::PSHUFHW, dl, MVT::v8i16, in LowerVECTOR_SHUFFLEv8i16()
8372 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, V1, in LowerVECTOR_SHUFFLEv8i16()
8374 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, V2, in LowerVECTOR_SHUFFLEv8i16()
8376 NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, NewV, ExtOp, in LowerVECTOR_SHUFFLEv8i16()
8437 pshufbMask.push_back(DAG.getConstant(EltIdx, MVT::i8)); in LowerVECTOR_SHUFFLEv16i8()
8439 V1 = DAG.getNode(X86ISD::PSHUFB, dl, MVT::v16i8, V1, in LowerVECTOR_SHUFFLEv16i8()
8441 MVT::v16i8, pshufbMask)); in LowerVECTOR_SHUFFLEv16i8()
8455 pshufbMask.push_back(DAG.getConstant(EltIdx, MVT::i8)); in LowerVECTOR_SHUFFLEv16i8()
8457 V2 = DAG.getNode(X86ISD::PSHUFB, dl, MVT::v16i8, V2, in LowerVECTOR_SHUFFLEv16i8()
8459 MVT::v16i8, pshufbMask)); in LowerVECTOR_SHUFFLEv16i8()
8460 return DAG.getNode(ISD::OR, dl, MVT::v16i8, V1, V2); in LowerVECTOR_SHUFFLEv16i8()
8466 V1 = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V1); in LowerVECTOR_SHUFFLEv16i8()
8467 V2 = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V2); in LowerVECTOR_SHUFFLEv16i8()
8488 InsElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, Elt1Src, in LowerVECTOR_SHUFFLEv16i8()
8490 NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, NewV, InsElt, in LowerVECTOR_SHUFFLEv16i8()
8499 InsElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, Elt1Src, in LowerVECTOR_SHUFFLEv16i8()
8502 InsElt = DAG.getNode(ISD::SHL, dl, MVT::i16, InsElt, in LowerVECTOR_SHUFFLEv16i8()
8506 InsElt = DAG.getNode(ISD::AND, dl, MVT::i16, InsElt, in LowerVECTOR_SHUFFLEv16i8()
8507 DAG.getConstant(0xFF00, MVT::i16)); in LowerVECTOR_SHUFFLEv16i8()
8514 SDValue InsElt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, in LowerVECTOR_SHUFFLEv16i8()
8517 InsElt0 = DAG.getNode(ISD::SRL, dl, MVT::i16, InsElt0, in LowerVECTOR_SHUFFLEv16i8()
8521 InsElt0 = DAG.getNode(ISD::AND, dl, MVT::i16, InsElt0, in LowerVECTOR_SHUFFLEv16i8()
8522 DAG.getConstant(0x00FF, MVT::i16)); in LowerVECTOR_SHUFFLEv16i8()
8523 InsElt = Elt1 >= 0 ? DAG.getNode(ISD::OR, dl, MVT::i16, InsElt, InsElt0) in LowerVECTOR_SHUFFLEv16i8()
8526 NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, NewV, InsElt, in LowerVECTOR_SHUFFLEv16i8()
8529 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, NewV); in LowerVECTOR_SHUFFLEv16i8()
8537 MVT VT = SVOp->getSimpleValueType(0); in LowerVECTOR_SHUFFLEv32i8()
8551 if (VT != MVT::v32i8 || !Subtarget->hasInt256() || in LowerVECTOR_SHUFFLEv32i8()
8570 MVT VT = SVOp->getSimpleValueType(0); in RewriteAsNarrowerShuffle()
8573 MVT NewVT; in RewriteAsNarrowerShuffle()
8577 case MVT::v2i64: in RewriteAsNarrowerShuffle()
8578 case MVT::v2f64: in RewriteAsNarrowerShuffle()
8580 case MVT::v4f32: NewVT = MVT::v2f64; Scale = 2; break; in RewriteAsNarrowerShuffle()
8581 case MVT::v4i32: NewVT = MVT::v2i64; Scale = 2; break; in RewriteAsNarrowerShuffle()
8582 case MVT::v8i16: NewVT = MVT::v4i32; Scale = 2; break; in RewriteAsNarrowerShuffle()
8583 case MVT::v16i8: NewVT = MVT::v4i32; Scale = 4; break; in RewriteAsNarrowerShuffle()
8584 case MVT::v16i16: NewVT = MVT::v8i32; Scale = 2; break; in RewriteAsNarrowerShuffle()
8585 case MVT::v32i8: NewVT = MVT::v8i32; Scale = 4; break; in RewriteAsNarrowerShuffle()
8610 static SDValue getVZextMovL(MVT VT, MVT OpVT, in getVZextMovL()
8613 if (VT == MVT::v2f64 || VT == MVT::v4f32) { in getVZextMovL()
8620 MVT ExtVT = (OpVT == MVT::v2f64) ? MVT::i64 : MVT::i32; in getVZextMovL()
8621 if ((ExtVT != MVT::i64 || Subtarget->is64Bit()) && in getVZextMovL()
8626 OpVT = (OpVT == MVT::v2f64) ? MVT::v2i64 : MVT::v4i32; in getVZextMovL()
8652 MVT VT = SVOp->getSimpleValueType(0); in LowerVECTOR_SHUFFLE_256()
8658 MVT EltVT = VT.getVectorElementType(); in LowerVECTOR_SHUFFLE_256()
8659 MVT NVT = MVT::getVectorVT(EltVT, NumLaneElems); in LowerVECTOR_SHUFFLE_256()
8763 MVT VT = SVOp->getSimpleValueType(0); in LowerVECTOR_SHUFFLE_128v4()
8914 MVT VT = Op.getSimpleValueType(); in getMOVDDup()
8917 V1 = DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, V1); in getMOVDDup()
8919 getTargetShuffleNode(X86ISD::MOVDDUP, dl, MVT::v2f64, in getMOVDDup()
8928 MVT VT = Op.getSimpleValueType(); in getMOVLowToHigh()
8930 assert(VT != MVT::v2i64 && "unsupported shuffle type"); in getMOVLowToHigh()
8932 if (HasSSE2 && VT == MVT::v2f64) in getMOVLowToHigh()
8937 getTargetShuffleNode(X86ISD::MOVLHPS, dl, MVT::v4f32, in getMOVLowToHigh()
8938 DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, V1), in getMOVLowToHigh()
8939 DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, V2), DAG)); in getMOVLowToHigh()
8946 MVT VT = Op.getSimpleValueType(); in getMOVHighToLow()
8948 assert((VT == MVT::v4i32 || VT == MVT::v4f32) && in getMOVHighToLow()
8962 MVT VT = Op.getSimpleValueType(); in getMOVLP()
9008 assert(VT != MVT::v4i32 && "unsupported shuffle type"); in getMOVLP()
9018 MVT VT = Load->getSimpleValueType(0); in NarrowVectorLoadToElement()
9019 MVT EVT = VT.getVectorElementType(); in NarrowVectorLoadToElement()
9043 MVT VT = SVOp->getSimpleValueType(0); in getINSERTPS()
9044 MVT EVT = VT.getVectorElementType(); in getINSERTPS()
9048 assert((VT == MVT::v4f32 || VT == MVT::v4i32) && in getINSERTPS()
9084 if (EVT == MVT::f32) { in getINSERTPS()
9096 DAG.getConstant(DestIndex, MVT::i32)); in getINSERTPS()
9112 MVT VT = Op.getSimpleValueType(); in LowerVectorIntExtend()
9127 VT.getVectorElementType() == MVT::i64) in LowerVectorIntExtend()
9152 MVT NeVT = MVT::getIntegerVT(NBits); in LowerVectorIntExtend()
9153 MVT NVT = MVT::getVectorVT(NeVT, NumElems >> Shift); in LowerVectorIntExtend()
9173 MVT FullVT = V.getSimpleValueType(); in LowerVectorIntExtend()
9174 MVT V1VT = V1.getSimpleValueType(); in LowerVectorIntExtend()
9180 MVT SubVecVT = MVT::getVectorVT(FullVT.getVectorElementType(), in LowerVectorIntExtend()
9196 MVT VT = Op.getSimpleValueType(); in NormalizeVectorShuffle()
9219 if (VT == MVT::v8i16 || VT == MVT::v16i8 || VT == MVT::v16i16 || in NormalizeVectorShuffle()
9220 VT == MVT::v32i8) { in NormalizeVectorShuffle()
9229 MVT NewVT = NewOp.getSimpleValueType(); in NormalizeVectorShuffle()
9238 MVT NewVT = NewOp.getSimpleValueType(); in NormalizeVectorShuffle()
9253 MVT VT = Op.getSimpleValueType(); in LowerVECTOR_SHUFFLE()
9322 (VT == MVT::v2f64 || VT == MVT::v2i64)) in LowerVECTOR_SHUFFLE()
9329 if (isMOVDDUPMask(M, VT) && ((VT == MVT::v4f32 || VT == MVT::v2i64))) in LowerVECTOR_SHUFFLE()
9334 if (HasSSE2 && (VT == MVT::v4f32 || VT == MVT::v4i32)) in LowerVECTOR_SHUFFLE()
9337 if (HasFp256 && (VT == MVT::v4f32 || VT == MVT::v2f64)) in LowerVECTOR_SHUFFLE()
9358 MVT EltVT = VT.getVectorElementType(); in LowerVECTOR_SHUFFLE()
9367 if (HasSSE2 && (VT == MVT::v2i64 || VT == MVT::v2f64)) in LowerVECTOR_SHUFFLE()
9370 if (VT == MVT::v4i32 || VT == MVT::v4f32) in LowerVECTOR_SHUFFLE()
9397 MVT EltVT = VT.getVectorElementType(); in LowerVECTOR_SHUFFLE()
9473 if (VT == MVT::v2f64 || VT == MVT::v2i64) in LowerVECTOR_SHUFFLE()
9512 if ((HasInt256 && VT == MVT::v8i32) || VT == MVT::v16i32) in LowerVECTOR_SHUFFLE()
9538 MVT MaskEltVT = MVT::getIntegerVT(VT.getVectorElementType().getSizeInBits()); in LowerVECTOR_SHUFFLE()
9539 MVT MaskVectorVT = MVT::getVectorVT(MaskEltVT, NumElems); in LowerVECTOR_SHUFFLE()
9561 if (VT == MVT::v8i16) { in LowerVECTOR_SHUFFLE()
9567 if (VT == MVT::v16i16 && Subtarget->hasInt256()) { in LowerVECTOR_SHUFFLE()
9573 if (VT == MVT::v16i8) { in LowerVECTOR_SHUFFLE()
9579 if (VT == MVT::v32i8) { in LowerVECTOR_SHUFFLE()
9641 MVT VT = Op.getSimpleValueType(); in LowerVSELECTtoBlend()
9642 MVT EltVT = VT.getVectorElementType(); in LowerVSELECTtoBlend()
9649 if (!Subtarget->hasSSE41() || EltVT == MVT::i8) in LowerVSELECTtoBlend()
9651 if (!Subtarget->hasInt256() && VT == MVT::v16i16) in LowerVSELECTtoBlend()
9664 MVT BlendVT = VT; in LowerVSELECTtoBlend()
9665 if (EltVT == MVT::i64 || (EltVT == MVT::i32 && !Subtarget->hasInt256())) { in LowerVSELECTtoBlend()
9666 BlendVT = MVT::getVectorVT(MVT::getFloatingPointVT(EltVT.getSizeInBits()), in LowerVSELECTtoBlend()
9673 DAG.getConstant(MaskValue, MVT::i32)); in LowerVSELECTtoBlend()
9685 MVT VT = Op.getSimpleValueType(); in LowerVSELECT()
9689 case MVT::v8i16: in LowerVSELECT()
9690 case MVT::v16i16: in LowerVSELECT()
9701 MVT VT = Op.getSimpleValueType(); in LowerEXTRACT_VECTOR_ELT_SSE4()
9708 SDValue Extract = DAG.getNode(X86ISD::PEXTRB, dl, MVT::i32, in LowerEXTRACT_VECTOR_ELT_SSE4()
9710 SDValue Assert = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Extract, in LowerEXTRACT_VECTOR_ELT_SSE4()
9719 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, in LowerEXTRACT_VECTOR_ELT_SSE4()
9720 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, in LowerEXTRACT_VECTOR_ELT_SSE4()
9722 MVT::v4i32, in LowerEXTRACT_VECTOR_ELT_SSE4()
9725 SDValue Extract = DAG.getNode(X86ISD::PEXTRW, dl, MVT::i32, in LowerEXTRACT_VECTOR_ELT_SSE4()
9727 SDValue Assert = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Extract, in LowerEXTRACT_VECTOR_ELT_SSE4()
9732 if (VT == MVT::f32) { in LowerEXTRACT_VECTOR_ELT_SSE4()
9745 User->getValueType(0) != MVT::i32)) in LowerEXTRACT_VECTOR_ELT_SSE4()
9747 SDValue Extract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, in LowerEXTRACT_VECTOR_ELT_SSE4()
9748 DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, in LowerEXTRACT_VECTOR_ELT_SSE4()
9751 return DAG.getNode(ISD::BITCAST, dl, MVT::f32, Extract); in LowerEXTRACT_VECTOR_ELT_SSE4()
9754 if (VT == MVT::i32 || VT == MVT::i64) { in LowerEXTRACT_VECTOR_ELT_SSE4()
9768 MVT VecVT = Vec.getSimpleValueType(); in ExtractBitFromMaskVector()
9770 MVT EltVT = Op.getSimpleValueType(); in ExtractBitFromMaskVector()
9772 assert((EltVT == MVT::i1) && "Unexpected operands in ExtractBitFromMaskVector"); in ExtractBitFromMaskVector()
9777 MVT ExtVT = (VecVT == MVT::v8i1 ? MVT::v8i64 : MVT::v16i32); in ExtractBitFromMaskVector()
9788 DAG.getConstant(MaxSift - IdxVal, MVT::i8)); in ExtractBitFromMaskVector()
9790 DAG.getConstant(MaxSift, MVT::i8)); in ExtractBitFromMaskVector()
9791 return DAG.getNode(X86ISD::VEXTRACT, dl, MVT::i1, Vec, in ExtractBitFromMaskVector()
9800 MVT VecVT = Vec.getSimpleValueType(); in LowerEXTRACT_VECTOR_ELT()
9803 if (Op.getSimpleValueType() == MVT::i1) in LowerEXTRACT_VECTOR_ELT()
9811 MVT MaskEltVT = in LowerEXTRACT_VECTOR_ELT()
9812 MVT::getIntegerVT(VecVT.getVectorElementType().getSizeInBits()); in LowerEXTRACT_VECTOR_ELT()
9813 MVT MaskVT = MVT::getVectorVT(MaskEltVT, VecVT.getSizeInBits() / in LowerEXTRACT_VECTOR_ELT()
9834 MVT EltVT = VecVT.getVectorElementType(); in LowerEXTRACT_VECTOR_ELT()
9842 DAG.getConstant(IdxVal, MVT::i32)); in LowerEXTRACT_VECTOR_ELT()
9853 MVT VT = Op.getSimpleValueType(); in LowerEXTRACT_VECTOR_ELT()
9859 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, in LowerEXTRACT_VECTOR_ELT()
9860 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, in LowerEXTRACT_VECTOR_ELT()
9862 MVT::v4i32, Vec), in LowerEXTRACT_VECTOR_ELT()
9865 MVT EltVT = MVT::i32; in LowerEXTRACT_VECTOR_ELT()
9880 MVT VVT = Op.getOperand(0).getSimpleValueType(); in LowerEXTRACT_VECTOR_ELT()
9899 MVT VVT = Op.getOperand(0).getSimpleValueType(); in LowerEXTRACT_VECTOR_ELT()
9910 MVT VT = Op.getSimpleValueType(); in LowerINSERT_VECTOR_ELT_SSE4()
9911 MVT EltVT = VT.getVectorElementType(); in LowerINSERT_VECTOR_ELT_SSE4()
9924 if (VT == MVT::v8i16) in LowerINSERT_VECTOR_ELT_SSE4()
9926 else if (VT == MVT::v16i8) in LowerINSERT_VECTOR_ELT_SSE4()
9933 if (N1.getValueType() != MVT::i32) in LowerINSERT_VECTOR_ELT_SSE4()
9934 N1 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, N1); in LowerINSERT_VECTOR_ELT_SSE4()
9935 if (N2.getValueType() != MVT::i32) in LowerINSERT_VECTOR_ELT_SSE4()
9940 if (EltVT == MVT::f32 && isa<ConstantSDNode>(N2)) { in LowerINSERT_VECTOR_ELT_SSE4()
9951 N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32, N1); in LowerINSERT_VECTOR_ELT_SSE4()
9955 if ((EltVT == MVT::i32 || EltVT == MVT::i64) && isa<ConstantSDNode>(N2)) { in LowerINSERT_VECTOR_ELT_SSE4()
9970 MVT VecVT = Vec.getSimpleValueType(); in InsertBitToMaskVector()
9975 MVT ExtVecVT = (VecVT == MVT::v8i1 ? MVT::v8i64 : MVT::v16i32); in InsertBitToMaskVector()
9976 MVT ExtEltVT = (VecVT == MVT::v8i1 ? MVT::i64 : MVT::i32); in InsertBitToMaskVector()
9987 DAG.getConstant(IdxVal, MVT::i8)); in InsertBitToMaskVector()
9991 DAG.getConstant(MaxSift, MVT::i8)); in InsertBitToMaskVector()
9993 DAG.getConstant(MaxSift - IdxVal, MVT::i8)); in InsertBitToMaskVector()
9998 MVT VT = Op.getSimpleValueType(); in LowerINSERT_VECTOR_ELT()
9999 MVT EltVT = VT.getVectorElementType(); in LowerINSERT_VECTOR_ELT()
10001 if (EltVT == MVT::i1) in LowerINSERT_VECTOR_ELT()
10024 DAG.getConstant(IdxIn128, MVT::i32)); in LowerINSERT_VECTOR_ELT()
10033 if (EltVT == MVT::i8) in LowerINSERT_VECTOR_ELT()
10039 if (N1.getValueType() != MVT::i32) in LowerINSERT_VECTOR_ELT()
10040 N1 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, N1); in LowerINSERT_VECTOR_ELT()
10041 if (N2.getValueType() != MVT::i32) in LowerINSERT_VECTOR_ELT()
10050 MVT OpVT = Op.getSimpleValueType(); in LowerSCALAR_TO_VECTOR()
10057 MVT VT128 = MVT::getVectorVT(OpVT.getVectorElementType(), in LowerSCALAR_TO_VECTOR()
10066 if (OpVT == MVT::v1i64 && in LowerSCALAR_TO_VECTOR()
10067 Op.getOperand(0).getValueType() == MVT::i64) in LowerSCALAR_TO_VECTOR()
10068 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v1i64, Op.getOperand(0)); in LowerSCALAR_TO_VECTOR()
10070 SDValue AnyExt = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, Op.getOperand(0)); in LowerSCALAR_TO_VECTOR()
10073 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32,AnyExt)); in LowerSCALAR_TO_VECTOR()
10085 MVT ResVT = Op.getSimpleValueType(); in LowerEXTRACT_SUBVECTOR()
10086 MVT InVT = In.getSimpleValueType(); in LowerEXTRACT_SUBVECTOR()
10335 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); in GetTLSADDR()
10532 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); in LowerGlobalTLSAddress()
10587 IDX, MachinePointerInfo(), MVT::i32, in LowerGlobalTLSAddress()
10619 MVT VT = Op.getSimpleValueType(); in LowerShiftParts()
10629 SDValue SafeShAmt = DAG.getNode(ISD::AND, dl, MVT::i8, ShAmt, in LowerShiftParts()
10630 DAG.getConstant(VTBits - 1, MVT::i8)); in LowerShiftParts()
10632 DAG.getConstant(VTBits - 1, MVT::i8)) in LowerShiftParts()
10647 SDValue AndNode = DAG.getNode(ISD::AND, dl, MVT::i8, ShAmt, in LowerShiftParts()
10648 DAG.getConstant(VTBits, MVT::i8)); in LowerShiftParts()
10649 SDValue Cond = DAG.getNode(X86ISD::CMP, dl, MVT::i32, in LowerShiftParts()
10650 AndNode, DAG.getConstant(0, MVT::i8)); in LowerShiftParts()
10653 SDValue CC = DAG.getConstant(X86::COND_NE, MVT::i8); in LowerShiftParts()
10671 MVT SrcVT = Op.getOperand(0).getSimpleValueType(); in LowerSINT_TO_FP()
10676 assert(SrcVT <= MVT::i64 && SrcVT >= MVT::i16 && in LowerSINT_TO_FP()
10681 if (SrcVT == MVT::i32 && isScalarFPTypeInSSEReg(Op.getValueType())) in LowerSINT_TO_FP()
10683 if (SrcVT == MVT::i64 && isScalarFPTypeInSSEReg(Op.getValueType()) && in LowerSINT_TO_FP()
10708 Tys = DAG.getVTList(MVT::f64, MVT::Other, MVT::Glue); in BuildFILD()
10710 Tys = DAG.getVTList(Op.getValueType(), MVT::Other); in BuildFILD()
10742 Tys = DAG.getVTList(MVT::Other); in BuildFILD()
10796 SDValue XR1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, in LowerUINT_TO_FP_i64()
10798 SDValue CLod0 = DAG.getLoad(MVT::v4i32, dl, DAG.getEntryNode(), CPIdx0, in LowerUINT_TO_FP_i64()
10801 SDValue Unpck1 = getUnpackl(DAG, dl, MVT::v4i32, in LowerUINT_TO_FP_i64()
10802 DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, XR1), in LowerUINT_TO_FP_i64()
10805 SDValue CLod1 = DAG.getLoad(MVT::v2f64, dl, CLod0.getValue(1), CPIdx1, in LowerUINT_TO_FP_i64()
10808 SDValue XR2F = DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, Unpck1); in LowerUINT_TO_FP_i64()
10809 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::v2f64, XR2F, CLod1); in LowerUINT_TO_FP_i64()
10814 Result = DAG.getNode(X86ISD::FHADD, dl, MVT::v2f64, Sub, Sub); in LowerUINT_TO_FP_i64()
10816 SDValue S2F = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Sub); in LowerUINT_TO_FP_i64()
10817 SDValue Shuffle = getTargetShuffleNode(X86ISD::PSHUFD, dl, MVT::v4i32, in LowerUINT_TO_FP_i64()
10819 Result = DAG.getNode(ISD::FADD, dl, MVT::v2f64, in LowerUINT_TO_FP_i64()
10820 DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, Shuffle), in LowerUINT_TO_FP_i64()
10824 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Result, in LowerUINT_TO_FP_i64()
10834 MVT::f64); in LowerUINT_TO_FP_i32()
10837 SDValue Load = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, in LowerUINT_TO_FP_i32()
10843 Load = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, in LowerUINT_TO_FP_i32()
10844 DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, Load), in LowerUINT_TO_FP_i32()
10848 SDValue Or = DAG.getNode(ISD::OR, dl, MVT::v2i64, in LowerUINT_TO_FP_i32()
10849 DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, in LowerUINT_TO_FP_i32()
10851 MVT::v2f64, Load)), in LowerUINT_TO_FP_i32()
10852 DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, in LowerUINT_TO_FP_i32()
10854 MVT::v2f64, Bias))); in LowerUINT_TO_FP_i32()
10855 Or = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, in LowerUINT_TO_FP_i32()
10856 DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, Or), in LowerUINT_TO_FP_i32()
10860 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Or, Bias); in LowerUINT_TO_FP_i32()
10865 if (DestVT.bitsLT(MVT::f64)) in LowerUINT_TO_FP_i32()
10868 if (DestVT.bitsGT(MVT::f64)) in LowerUINT_TO_FP_i32()
10878 MVT SVT = N0.getSimpleValueType(); in lowerUINT_TO_FP_vec()
10881 assert((SVT == MVT::v4i8 || SVT == MVT::v4i16 || in lowerUINT_TO_FP_vec()
10882 SVT == MVT::v8i8 || SVT == MVT::v8i16) && in lowerUINT_TO_FP_vec()
10885 MVT NVT = MVT::getVectorVT(MVT::i32, SVT.getVectorNumElements()); in lowerUINT_TO_FP_vec()
10904 MVT SrcVT = N0.getSimpleValueType(); in LowerUINT_TO_FP()
10905 MVT DstVT = Op.getSimpleValueType(); in LowerUINT_TO_FP()
10906 if (SrcVT == MVT::i64 && DstVT == MVT::f64 && X86ScalarSSEf64) in LowerUINT_TO_FP()
10908 if (SrcVT == MVT::i32 && X86ScalarSSEf64) in LowerUINT_TO_FP()
10910 if (Subtarget->is64Bit() && SrcVT == MVT::i64 && DstVT == MVT::f32) in LowerUINT_TO_FP()
10914 SDValue StackSlot = DAG.CreateStackTemporary(MVT::i64); in LowerUINT_TO_FP()
10915 if (SrcVT == MVT::i32) { in LowerUINT_TO_FP()
10922 SDValue Store2 = DAG.getStore(Store1, dl, DAG.getConstant(0, MVT::i32), in LowerUINT_TO_FP()
10925 SDValue Fild = BuildFILD(Op, MVT::i64, Store2, StackSlot, DAG); in LowerUINT_TO_FP()
10929 assert(SrcVT == MVT::i64 && "Unexpected type in UINT_TO_FP"); in LowerUINT_TO_FP()
10944 SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other); in LowerUINT_TO_FP()
10945 SDValue Ops[] = { Store, StackSlot, DAG.getValueType(MVT::i64) }; in LowerUINT_TO_FP()
10947 MVT::i64, MMO); in LowerUINT_TO_FP()
10953 getSetCCResultType(*DAG.getContext(), MVT::i64), in LowerUINT_TO_FP()
10954 Op.getOperand(0), DAG.getConstant(0, MVT::i64), in LowerUINT_TO_FP()
10971 SDValue Fudge = DAG.getExtLoad(ISD::EXTLOAD, dl, MVT::f80, DAG.getEntryNode(), in LowerUINT_TO_FP()
10973 MVT::f32, false, false, 4); in LowerUINT_TO_FP()
10975 SDValue Add = DAG.getNode(ISD::FADD, dl, MVT::f80, Fild, Fudge); in LowerUINT_TO_FP()
10987 assert(DstTy == MVT::i32 && "Unexpected FP_TO_UINT"); in FP_TO_INTHelper()
10988 DstTy = MVT::i64; in FP_TO_INTHelper()
10991 assert(DstTy.getSimpleVT() <= MVT::i64 && in FP_TO_INTHelper()
10992 DstTy.getSimpleVT() >= MVT::i16 && in FP_TO_INTHelper()
10996 if (DstTy == MVT::i32 && in FP_TO_INTHelper()
11000 DstTy == MVT::i64 && in FP_TO_INTHelper()
11017 case MVT::i16: Opc = X86ISD::FP_TO_INT16_IN_MEM; break; in FP_TO_INTHelper()
11018 case MVT::i32: Opc = X86ISD::FP_TO_INT32_IN_MEM; break; in FP_TO_INTHelper()
11019 case MVT::i64: Opc = X86ISD::FP_TO_INT64_IN_MEM; break; in FP_TO_INTHelper()
11028 assert(DstTy == MVT::i64 && "Invalid FP_TO_SINT to lower!"); in FP_TO_INTHelper()
11032 SDVTList Tys = DAG.getVTList(Op.getOperand(0).getValueType(), MVT::Other); in FP_TO_INTHelper()
11053 SDValue FIST = DAG.getMemIntrinsicNode(Opc, DL, DAG.getVTList(MVT::Other), in FP_TO_INTHelper()
11058 DAG.getVTList(MVT::Other, MVT::Glue), in FP_TO_INTHelper()
11061 MVT::i32, ftol.getValue(1)); in FP_TO_INTHelper()
11063 MVT::i32, eax.getValue(2)); in FP_TO_INTHelper()
11066 ? DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Ops) in FP_TO_INTHelper()
11074 MVT VT = Op->getSimpleValueType(0); in LowerAVXExtend()
11076 MVT InVT = In.getSimpleValueType(); in LowerAVXExtend()
11092 if (((VT != MVT::v16i16) || (InVT != MVT::v16i8)) && in LowerAVXExtend()
11093 ((VT != MVT::v8i32) || (InVT != MVT::v8i16)) && in LowerAVXExtend()
11094 ((VT != MVT::v4i64) || (InVT != MVT::v4i32))) in LowerAVXExtend()
11106 MVT HVT = MVT::getVectorVT(VT.getVectorElementType(), in LowerAVXExtend()
11117 MVT VT = Op->getSimpleValueType(0); in LowerZERO_EXTEND_AVX512()
11119 MVT InVT = In.getSimpleValueType(); in LowerZERO_EXTEND_AVX512()
11125 if (VT.is512BitVector() && InVT.getVectorElementType() != MVT::i1) in LowerZERO_EXTEND_AVX512()
11128 EVT ExtVT = (NumElts == 8)? MVT::v8i64 : MVT::v16i32; in LowerZERO_EXTEND_AVX512()
11131 assert(InVT.getVectorElementType() == MVT::i1); in LowerZERO_EXTEND_AVX512()
11160 MVT VT = Op.getSimpleValueType(); in LowerZERO_EXTEND()
11162 MVT SVT = In.getSimpleValueType(); in LowerZERO_EXTEND()
11164 if (VT.is512BitVector() || SVT.getVectorElementType() == MVT::i1) in LowerZERO_EXTEND()
11180 MVT VT = Op.getSimpleValueType(); in LowerTRUNCATE()
11182 MVT InVT = In.getSimpleValueType(); in LowerTRUNCATE()
11184 if (VT == MVT::i1) { in LowerTRUNCATE()
11187 if (InVT == MVT::i32) in LowerTRUNCATE()
11190 In = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::i32, In); in LowerTRUNCATE()
11192 In = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, In); in LowerTRUNCATE()
11198 if (InVT.is512BitVector() || VT.getVectorElementType() == MVT::i1) { in LowerTRUNCATE()
11202 assert(VT.getVectorElementType() == MVT::i1 && "Unexpected vector type"); in LowerTRUNCATE()
11206 MVT ExtVT = (NumElts == 16)? MVT::v16i32 : MVT::v8i64; in LowerTRUNCATE()
11223 if ((VT == MVT::v4i32) && (InVT == MVT::v4i64)) { in LowerTRUNCATE()
11227 In = DAG.getNode(ISD::BITCAST, DL, MVT::v8i32, In); in LowerTRUNCATE()
11228 In = DAG.getVectorShuffle(MVT::v8i32, DL, In, DAG.getUNDEF(MVT::v8i32), in LowerTRUNCATE()
11234 SDValue OpLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In, in LowerTRUNCATE()
11236 SDValue OpHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In, in LowerTRUNCATE()
11238 OpLo = DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, OpLo); in LowerTRUNCATE()
11239 OpHi = DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, OpHi); in LowerTRUNCATE()
11244 if ((VT == MVT::v8i16) && (InVT == MVT::v8i32)) { in LowerTRUNCATE()
11247 In = DAG.getNode(ISD::BITCAST, DL, MVT::v32i8, In); in LowerTRUNCATE()
11251 pshufbMask.push_back(DAG.getConstant(0x0, MVT::i8)); in LowerTRUNCATE()
11252 pshufbMask.push_back(DAG.getConstant(0x1, MVT::i8)); in LowerTRUNCATE()
11253 pshufbMask.push_back(DAG.getConstant(0x4, MVT::i8)); in LowerTRUNCATE()
11254 pshufbMask.push_back(DAG.getConstant(0x5, MVT::i8)); in LowerTRUNCATE()
11255 pshufbMask.push_back(DAG.getConstant(0x8, MVT::i8)); in LowerTRUNCATE()
11256 pshufbMask.push_back(DAG.getConstant(0x9, MVT::i8)); in LowerTRUNCATE()
11257 pshufbMask.push_back(DAG.getConstant(0xc, MVT::i8)); in LowerTRUNCATE()
11258 pshufbMask.push_back(DAG.getConstant(0xd, MVT::i8)); in LowerTRUNCATE()
11260 pshufbMask.push_back(DAG.getConstant(0x80, MVT::i8)); in LowerTRUNCATE()
11262 SDValue BV = DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v32i8, pshufbMask); in LowerTRUNCATE()
11263 In = DAG.getNode(X86ISD::PSHUFB, DL, MVT::v32i8, In, BV); in LowerTRUNCATE()
11264 In = DAG.getNode(ISD::BITCAST, DL, MVT::v4i64, In); in LowerTRUNCATE()
11267 In = DAG.getVectorShuffle(MVT::v4i64, DL, In, DAG.getUNDEF(MVT::v4i64), in LowerTRUNCATE()
11269 In = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In, in LowerTRUNCATE()
11274 SDValue OpLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i32, In, in LowerTRUNCATE()
11277 SDValue OpHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i32, In, in LowerTRUNCATE()
11280 OpLo = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, OpLo); in LowerTRUNCATE()
11281 OpHi = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, OpHi); in LowerTRUNCATE()
11287 SDValue Undef = DAG.getUNDEF(MVT::v16i8); in LowerTRUNCATE()
11288 OpLo = DAG.getVectorShuffle(MVT::v16i8, DL, OpLo, Undef, ShufMask1); in LowerTRUNCATE()
11289 OpHi = DAG.getVectorShuffle(MVT::v16i8, DL, OpHi, Undef, ShufMask1); in LowerTRUNCATE()
11291 OpLo = DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, OpLo); in LowerTRUNCATE()
11292 OpHi = DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, OpHi); in LowerTRUNCATE()
11296 SDValue res = DAG.getVectorShuffle(MVT::v4i32, DL, OpLo, OpHi, ShufMask2); in LowerTRUNCATE()
11297 return DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, res); in LowerTRUNCATE()
11307 MVT NVT = MVT::getVectorVT(VT.getVectorElementType(), NumElems * 2); in LowerTRUNCATE()
11359 MVT VT = Op.getSimpleValueType(); in LowerFP_EXTEND()
11361 MVT SVT = In.getSimpleValueType(); in LowerFP_EXTEND()
11363 assert(SVT == MVT::v2f32 && "Only customize MVT::v2f32 type legalization!"); in LowerFP_EXTEND()
11366 DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4f32, in LowerFP_EXTEND()
11373 MVT VT = Op.getSimpleValueType(); in LowerFABS()
11374 MVT EltVT = VT; in LowerFABS()
11375 unsigned NumElts = VT == MVT::f64 ? 2 : 4; in LowerFABS()
11381 if (EltVT == MVT::f64) in LowerFABS()
11395 MVT ANDVT = VT.is128BitVector() ? MVT::v2i64 : MVT::v4i64; in LowerFABS()
11408 MVT VT = Op.getSimpleValueType(); in LowerFNEG()
11409 MVT EltVT = VT; in LowerFNEG()
11410 unsigned NumElts = VT == MVT::f64 ? 2 : 4; in LowerFNEG()
11416 if (EltVT == MVT::f64) in LowerFNEG()
11430 MVT XORVT = MVT::getVectorVT(MVT::i64, VT.getSizeInBits()/64); in LowerFNEG()
11447 MVT VT = Op.getSimpleValueType(); in LowerFCOPYSIGN()
11448 MVT SrcVT = Op1.getSimpleValueType(); in LowerFCOPYSIGN()
11466 if (SrcVT == MVT::f64) { in LowerFCOPYSIGN()
11487 SignBit = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2f64, SignBit); in LowerFCOPYSIGN()
11488 SignBit = DAG.getNode(X86ISD::FSRL, dl, MVT::v2f64, SignBit, in LowerFCOPYSIGN()
11489 DAG.getConstant(32, MVT::i32)); in LowerFCOPYSIGN()
11490 SignBit = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, SignBit); in LowerFCOPYSIGN()
11491 SignBit = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f32, SignBit, in LowerFCOPYSIGN()
11497 if (VT == MVT::f64) { in LowerFCOPYSIGN()
11524 MVT VT = Op.getSimpleValueType(); in LowerFGETSIGN()
11550 EVT VT = MVT::Other; in LowerVectorAllZeroTest()
11606 EVT TestVT = VT.is128BitVector() ? MVT::v2i64 : MVT::v4i64; in LowerVectorAllZeroTest()
11621 return DAG.getNode(X86ISD::PTEST, DL, MVT::i32, in LowerVectorAllZeroTest()
11648 if (Op.getValueType() == MVT::i1) in EmitTest()
11650 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op, in EmitTest()
11694 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op, in EmitTest()
11868 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op, in EmitTest()
11871 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32); in EmitTest()
11889 if (Op0.getValueType() == MVT::i1) in EmitCmp()
11893 if ((Op0.getValueType() == MVT::i8 || Op0.getValueType() == MVT::i16 || in EmitCmp()
11894 Op0.getValueType() == MVT::i32 || Op0.getValueType() == MVT::i64)) { in EmitCmp()
11899 if (Op0.getValueType() != MVT::i32 && Op0.getValueType() != MVT::i64 && in EmitCmp()
11905 Op0 = DAG.getNode(ExtendOp, dl, MVT::i32, Op0); in EmitCmp()
11906 Op1 = DAG.getNode(ExtendOp, dl, MVT::i32, Op1); in EmitCmp()
11909 SDVTList VTs = DAG.getVTList(Op0.getValueType(), MVT::i32); in EmitCmp()
11914 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op0, Op1); in EmitCmp()
11933 SDValue TruncFPSW = DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, Cmp); in ConvertCmpIfNecessary()
11934 SDValue FNStSW = DAG.getNode(X86ISD::FNSTSW16r, dl, MVT::i16, TruncFPSW); in ConvertCmpIfNecessary()
11935 SDValue Srl = DAG.getNode(ISD::SRL, dl, MVT::i16, FNStSW, in ConvertCmpIfNecessary()
11936 DAG.getConstant(8, MVT::i8)); in ConvertCmpIfNecessary()
11937 SDValue TruncSrl = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Srl); in ConvertCmpIfNecessary()
11938 return DAG.getNode(X86ISD::SAHF, dl, MVT::i32, TruncSrl); in ConvertCmpIfNecessary()
11999 if (LHS.getValueType() == MVT::i8 || in LowerToBT()
12000 LHS.getValueType() == MVT::i16) in LowerToBT()
12001 LHS = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, LHS); in LowerToBT()
12008 SDValue BT = DAG.getNode(X86ISD::BT, dl, MVT::i32, LHS, RHS); in LowerToBT()
12010 return DAG.getNode(X86ISD::SETCC, dl, MVT::i8, in LowerToBT()
12011 DAG.getConstant(Cond, MVT::i8), BT); in LowerToBT()
12065 MVT VT = Op.getSimpleValueType(); in Lower256IntVSETCC()
12085 MVT EltVT = VT.getVectorElementType(); in Lower256IntVSETCC()
12086 MVT NewVT = MVT::getVectorVT(EltVT, NumElems/2); in Lower256IntVSETCC()
12097 MVT VT = Op.getSimpleValueType(); in LowerIntVSETCC_AVX512()
12101 Op.getValueType().getScalarType() == MVT::i1 && in LowerIntVSETCC_AVX512()
12129 DAG.getConstant(SSECC, MVT::i8)); in LowerIntVSETCC_AVX512()
12141 MVT VT = Op1.getSimpleValueType(); in ChangeVSETULTtoVSETULE()
12142 MVT EVT = VT.getVectorElementType(); in ChangeVSETULTtoVSETULE()
12167 MVT VT = Op.getSimpleValueType(); in LowerVSETCC()
12174 MVT EltVT = Op0.getSimpleValueType().getVectorElementType(); in LowerVSETCC()
12175 assert(EltVT == MVT::f32 || EltVT == MVT::f64); in LowerVSETCC()
12180 if (Subtarget->hasAVX512() && VT.getVectorElementType() == MVT::i1) { in LowerVSETCC()
12196 DAG.getConstant(CC0, MVT::i8)); in LowerVSETCC()
12198 DAG.getConstant(CC1, MVT::i8)); in LowerVSETCC()
12203 DAG.getConstant(SSECC, MVT::i8)); in LowerVSETCC()
12210 bool MaskResult = (VT.getVectorElementType() == MVT::i1); in LowerVSETCC()
12253 MVT VET = VT.getVectorElementType(); in LowerVSETCC()
12255 (Subtarget->hasSSE41() && (VET >= MVT::i8 && VET <= MVT::i32)) in LowerVSETCC()
12256 || (Subtarget->hasSSE2() && (VET == MVT::i8)); in LowerVSETCC()
12268 bool hasSubus = Subtarget->hasSSE2() && (VET == MVT::i8 || VET == MVT::i16); in LowerVSETCC()
12307 if (VT == MVT::v2i64) { in LowerVSETCC()
12312 Op0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Op0); in LowerVSETCC()
12313 Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Op1); in LowerVSETCC()
12320 SB = DAG.getConstant(0x80000000U, MVT::v4i32); in LowerVSETCC()
12322 SDValue Sign = DAG.getConstant(0x80000000U, MVT::i32); in LowerVSETCC()
12323 SDValue Zero = DAG.getConstant(0x00000000U, MVT::i32); in LowerVSETCC()
12324 SB = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, in LowerVSETCC()
12327 Op0 = DAG.getNode(ISD::XOR, dl, MVT::v4i32, Op0, SB); in LowerVSETCC()
12328 Op1 = DAG.getNode(ISD::XOR, dl, MVT::v4i32, Op1, SB); in LowerVSETCC()
12331 SDValue GT = DAG.getNode(X86ISD::PCMPGT, dl, MVT::v4i32, Op0, Op1); in LowerVSETCC()
12332 SDValue EQ = DAG.getNode(X86ISD::PCMPEQ, dl, MVT::v4i32, Op0, Op1); in LowerVSETCC()
12337 SDValue EQHi = DAG.getVectorShuffle(MVT::v4i32, dl, EQ, EQ, MaskHi); in LowerVSETCC()
12338 SDValue GTLo = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskLo); in LowerVSETCC()
12339 SDValue GTHi = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskHi); in LowerVSETCC()
12341 SDValue Result = DAG.getNode(ISD::AND, dl, MVT::v4i32, EQHi, GTLo); in LowerVSETCC()
12342 Result = DAG.getNode(ISD::OR, dl, MVT::v4i32, Result, GTHi); in LowerVSETCC()
12345 Result = DAG.getNOT(dl, Result, MVT::v4i32); in LowerVSETCC()
12356 Op0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Op0); in LowerVSETCC()
12357 Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Op1); in LowerVSETCC()
12360 SDValue Result = DAG.getNode(Opc, dl, MVT::v4i32, Op0, Op1); in LowerVSETCC()
12364 SDValue Shuf = DAG.getVectorShuffle(MVT::v4i32, dl, Result, Result, Mask); in LowerVSETCC()
12365 Result = DAG.getNode(ISD::AND, dl, MVT::v4i32, Result, Shuf); in LowerVSETCC()
12368 Result = DAG.getNOT(dl, Result, MVT::v4i32); in LowerVSETCC()
12401 MVT VT = Op.getSimpleValueType(); in LowerSETCC()
12405 assert(((!Subtarget->hasAVX512() && VT == MVT::i8) || (VT == MVT::i1)) in LowerSETCC()
12442 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, in LowerSETCC()
12443 DAG.getConstant(CCode, MVT::i8), in LowerSETCC()
12445 if (VT == MVT::i1) in LowerSETCC()
12446 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, SetCC); in LowerSETCC()
12450 if ((Op0.getValueType() == MVT::i1) && (Op1.getOpcode() == ISD::Constant) && in LowerSETCC()
12455 return DAG.getSetCC(dl, VT, Op0, DAG.getConstant(0, MVT::i1), NewCC); in LowerSETCC()
12465 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, in LowerSETCC()
12466 DAG.getConstant(X86CC, MVT::i8), EFLAGS); in LowerSETCC()
12467 if (VT == MVT::i1) in LowerSETCC()
12468 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, SetCC); in LowerSETCC()
12521 ((Subtarget->hasSSE2() && (VT == MVT::f32 || VT == MVT::f64)) || in LowerSELECT()
12522 (Subtarget->hasSSE1() && VT == MVT::f32)) && in LowerSELECT()
12530 SDValue Cmp = DAG.getNode(X86ISD::FSETCC, DL, MVT::i1, CondOp0, CondOp1, in LowerSELECT()
12531 DAG.getConstant(SSECC, MVT::i8)); in LowerSELECT()
12535 DAG.getConstant(SSECC, MVT::i8)); in LowerSELECT()
12570 SDVTList VTs = DAG.getVTList(CmpOp0.getValueType(), MVT::i32); in LowerSELECT()
12575 DAG.getConstant(X86::COND_B, MVT::i8), in LowerSELECT()
12580 Cmp = DAG.getNode(X86ISD::CMP, DL, MVT::i32, in LowerSELECT()
12586 DAG.getConstant(X86::COND_B, MVT::i8), Cmp); in LowerSELECT()
12615 MVT VT = Op.getSimpleValueType(); in LowerSELECT()
12630 Cond.getOperand(0).getValueType() != MVT::i8)) { in LowerSELECT()
12647 MVT::i32); in LowerSELECT()
12649 VTs = DAG.getVTList(LHS.getValueType(), MVT::i32); in LowerSELECT()
12658 CC = DAG.getConstant(X86Cond, MVT::i8); in LowerSELECT()
12680 CC = DAG.getConstant(X86::COND_NE, MVT::i8); in LowerSELECT()
12695 DAG.getConstant(X86::COND_B, MVT::i8), Cond); in LowerSELECT()
12705 if (Op.getValueType() == MVT::i8 && in LowerSELECT()
12711 SDVTList VTs = DAG.getVTList(T1.getValueType(), MVT::Glue); in LowerSELECT()
12719 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Glue); in LowerSELECT()
12725 MVT VT = Op->getSimpleValueType(0); in LowerSIGN_EXTEND_AVX512()
12727 MVT InVT = In.getSimpleValueType(); in LowerSIGN_EXTEND_AVX512()
12734 if (VT.is512BitVector() && InVT.getVectorElementType() != MVT::i1) in LowerSIGN_EXTEND_AVX512()
12738 assert (InVT.getVectorElementType() == MVT::i1 && "Unexpected vector type"); in LowerSIGN_EXTEND_AVX512()
12740 MVT ExtVT = (NumElts == 8) ? MVT::v8i64 : MVT::v16i32; in LowerSIGN_EXTEND_AVX512()
12757 MVT VT = Op->getSimpleValueType(0); in LowerSIGN_EXTEND()
12759 MVT InVT = In.getSimpleValueType(); in LowerSIGN_EXTEND()
12762 if (VT.is512BitVector() || InVT.getVectorElementType() == MVT::i1) in LowerSIGN_EXTEND()
12765 if ((VT != MVT::v4i64 || InVT != MVT::v4i32) && in LowerSIGN_EXTEND()
12766 (VT != MVT::v8i32 || InVT != MVT::v8i16) && in LowerSIGN_EXTEND()
12767 (VT != MVT::v16i16 || InVT != MVT::v16i8)) in LowerSIGN_EXTEND()
12797 MVT HalfVT = MVT::getVectorVT(VT.getScalarType(), in LowerSIGN_EXTEND()
12908 Cond.getOperand(0).getValueType() != MVT::i8)) { in LowerBRCOND()
12942 MVT::i32); in LowerBRCOND()
12944 VTs = DAG.getVTList(LHS.getValueType(), MVT::i32); in LowerBRCOND()
12953 CC = DAG.getConstant(X86Cond, MVT::i8); in LowerBRCOND()
12984 CC = DAG.getConstant(CCode, MVT::i8); in LowerBRCOND()
13002 CC = DAG.getConstant(CCode, MVT::i8); in LowerBRCOND()
13015 CC = DAG.getConstant(CCode, MVT::i8); in LowerBRCOND()
13038 SDValue Cmp = DAG.getNode(X86ISD::CMP, dl, MVT::i32, in LowerBRCOND()
13041 CC = DAG.getConstant(X86::COND_NE, MVT::i8); in LowerBRCOND()
13044 CC = DAG.getConstant(X86::COND_P, MVT::i8); in LowerBRCOND()
13068 SDValue Cmp = DAG.getNode(X86ISD::CMP, dl, MVT::i32, in LowerBRCOND()
13071 CC = DAG.getConstant(X86::COND_NE, MVT::i8); in LowerBRCOND()
13074 CC = DAG.getConstant(X86::COND_NP, MVT::i8); in LowerBRCOND()
13102 CC = DAG.getConstant(X86Cond, MVT::i8); in LowerBRCOND()
13169 EVT SPTy = Is64Bit ? MVT::i64 : MVT::i32; in LowerDYNAMIC_STACKALLOC()
13187 getRegClassFor(Subtarget->is64Bit() ? MVT::i64:MVT::i32); in LowerDYNAMIC_STACKALLOC()
13200 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); in LowerDYNAMIC_STACKALLOC()
13247 MVT::i32), in LowerVASTART()
13256 MVT::i32), in LowerVASTART()
13278 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps); in LowerVASTART()
13302 if (ArgVT == MVT::f80) { in LowerVAARG()
13327 InstOps.push_back(DAG.getConstant(ArgSize, MVT::i32)); in LowerVAARG()
13328 InstOps.push_back(DAG.getConstant(ArgMode, MVT::i8)); in LowerVAARG()
13329 InstOps.push_back(DAG.getConstant(Align, MVT::i32)); in LowerVAARG()
13330 SDVTList VTs = DAG.getVTList(getPointerTy(), MVT::Other); in LowerVAARG()
13332 VTs, InstOps, MVT::i64, in LowerVAARG()
13367 static SDValue getTargetVShiftByConstNode(unsigned Opc, SDLoc dl, MVT VT, in getTargetVShiftByConstNode()
13370 MVT ElementType = VT.getVectorElementType(); in getTargetVShiftByConstNode()
13438 return DAG.getNode(Opc, dl, VT, SrcOp, DAG.getConstant(ShiftAmt, MVT::i8)); in getTargetVShiftByConstNode()
13443 static SDValue getTargetVShiftNode(unsigned Opc, SDLoc dl, MVT VT, in getTargetVShiftNode()
13446 assert(ShAmt.getValueType() == MVT::i32 && "ShAmt is not i32"); in getTargetVShiftNode()
13465 ShOps[1] = DAG.getConstant(0, MVT::i32); in getTargetVShiftNode()
13466 ShOps[2] = ShOps[3] = DAG.getUNDEF(MVT::i32); in getTargetVShiftNode()
13467 ShAmt = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, ShOps); in getTargetVShiftNode()
13471 MVT EltVT = VT.getVectorElementType(); in getTargetVShiftNode()
13472 EVT ShVT = MVT::getVectorVT(EltVT, 128/EltVT.getSizeInBits()); in getTargetVShiftNode()
13578 SDValue Cond = DAG.getNode(Opc, dl, MVT::i32, LHS, RHS); in LowerINTRINSIC_WO_CHAIN()
13579 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, in LowerINTRINSIC_WO_CHAIN()
13580 DAG.getConstant(X86CC, MVT::i8), Cond); in LowerINTRINSIC_WO_CHAIN()
13581 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC); in LowerINTRINSIC_WO_CHAIN()
13915 SDValue Test = DAG.getNode(TestOpc, dl, MVT::i32, LHS, RHS); in LowerINTRINSIC_WO_CHAIN()
13916 SDValue CC = DAG.getConstant(X86CC, MVT::i8); in LowerINTRINSIC_WO_CHAIN()
13917 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, CC, Test); in LowerINTRINSIC_WO_CHAIN()
13918 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC); in LowerINTRINSIC_WO_CHAIN()
13923 SDValue LHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i1, Op.getOperand(1)); in LowerINTRINSIC_WO_CHAIN()
13924 SDValue RHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i1, Op.getOperand(2)); in LowerINTRINSIC_WO_CHAIN()
13925 SDValue CC = DAG.getConstant(X86CC, MVT::i8); in LowerINTRINSIC_WO_CHAIN()
13926 SDValue Test = DAG.getNode(X86ISD::KORTEST, dl, MVT::i32, LHS, RHS); in LowerINTRINSIC_WO_CHAIN()
13927 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i1, CC, Test); in LowerINTRINSIC_WO_CHAIN()
13928 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC); in LowerINTRINSIC_WO_CHAIN()
14081 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32); in LowerINTRINSIC_WO_CHAIN()
14083 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, in LowerINTRINSIC_WO_CHAIN()
14084 DAG.getConstant(X86CC, MVT::i8), in LowerINTRINSIC_WO_CHAIN()
14086 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC); in LowerINTRINSIC_WO_CHAIN()
14098 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32); in LowerINTRINSIC_WO_CHAIN()
14203 SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), MVT::i8); in getGatherNode()
14204 EVT MaskVT = MVT::getVectorVT(MVT::i1, in getGatherNode()
14212 SDVTList VTs = DAG.getVTList(Op.getValueType(), MaskVT, MVT::Other); in getGatherNode()
14213 SDValue Disp = DAG.getTargetConstant(0, MVT::i32); in getGatherNode()
14214 SDValue Segment = DAG.getRegister(0, MVT::i32); in getGatherNode()
14229 SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), MVT::i8); in getScatterNode()
14230 SDValue Disp = DAG.getTargetConstant(0, MVT::i32); in getScatterNode()
14231 SDValue Segment = DAG.getRegister(0, MVT::i32); in getScatterNode()
14232 EVT MaskVT = MVT::getVectorVT(MVT::i1, in getScatterNode()
14240 SDVTList VTs = DAG.getVTList(MaskVT, MVT::Other); in getScatterNode()
14252 SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), MVT::i8); in getPrefetchNode()
14253 SDValue Disp = DAG.getTargetConstant(0, MVT::i32); in getPrefetchNode()
14254 SDValue Segment = DAG.getRegister(0, MVT::i32); in getPrefetchNode()
14256 MVT::getVectorVT(MVT::i1, Index.getSimpleValueType().getVectorNumElements()); in getPrefetchNode()
14265 SDNode *Res = DAG.getMachineNode(Opc, dl, MVT::Other, Ops); in getPrefetchNode()
14275 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue); in getReadPerformanceCounter()
14287 LO = DAG.getCopyFromReg(rd, DL, X86::RAX, MVT::i64, rd.getValue(1)); in getReadPerformanceCounter()
14288 HI = DAG.getCopyFromReg(LO.getValue(1), DL, X86::RDX, MVT::i64, in getReadPerformanceCounter()
14291 LO = DAG.getCopyFromReg(rd, DL, X86::EAX, MVT::i32, rd.getValue(1)); in getReadPerformanceCounter()
14292 HI = DAG.getCopyFromReg(LO.getValue(1), DL, X86::EDX, MVT::i32, in getReadPerformanceCounter()
14300 SDValue Tmp = DAG.getNode(ISD::SHL, DL, MVT::i64, HI, in getReadPerformanceCounter()
14301 DAG.getConstant(32, MVT::i8)); in getReadPerformanceCounter()
14302 Results.push_back(DAG.getNode(ISD::OR, DL, MVT::i64, LO, Tmp)); in getReadPerformanceCounter()
14309 SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Ops); in getReadPerformanceCounter()
14320 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue); in getReadTimeStampCounter()
14328 LO = DAG.getCopyFromReg(rd, DL, X86::RAX, MVT::i64, rd.getValue(1)); in getReadTimeStampCounter()
14329 HI = DAG.getCopyFromReg(LO.getValue(1), DL, X86::RDX, MVT::i64, in getReadTimeStampCounter()
14332 LO = DAG.getCopyFromReg(rd, DL, X86::EAX, MVT::i32, rd.getValue(1)); in getReadTimeStampCounter()
14333 HI = DAG.getCopyFromReg(LO.getValue(1), DL, X86::EDX, MVT::i32, in getReadTimeStampCounter()
14343 SDValue ecx = DAG.getCopyFromReg(Chain, DL, X86::ECX, MVT::i32, in getReadTimeStampCounter()
14354 SDValue Tmp = DAG.getNode(ISD::SHL, DL, MVT::i64, HI, in getReadTimeStampCounter()
14355 DAG.getConstant(32, MVT::i8)); in getReadTimeStampCounter()
14356 Results.push_back(DAG.getNode(ISD::OR, DL, MVT::i64, LO, Tmp)); in getReadTimeStampCounter()
14363 SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Ops); in getReadTimeStampCounter()
14491 SDVTList VTs = DAG.getVTList(Op->getValueType(0), MVT::Glue, MVT::Other); in LowerINTRINSIC_W_CHAIN()
14498 DAG.getConstant(X86::COND_B, MVT::i32), in LowerINTRINSIC_W_CHAIN()
14501 DAG.getVTList(Op->getValueType(1), MVT::Glue), in LowerINTRINSIC_W_CHAIN()
14557 SDVTList VTs = DAG.getVTList(Op->getValueType(0), MVT::Other); in LowerINTRINSIC_W_CHAIN()
14559 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, in LowerINTRINSIC_W_CHAIN()
14560 DAG.getConstant(X86::COND_NE, MVT::i8), in LowerINTRINSIC_W_CHAIN()
14609 assert(((FrameReg == X86::RBP && VT == MVT::i64) || in LowerFRAMEADDR()
14610 (FrameReg == X86::EBP && VT == MVT::i32)) && in LowerFRAMEADDR()
14650 assert(((FrameReg == X86::RBP && PtrVT == MVT::i64) || in LowerEH_RETURN()
14651 (FrameReg == X86::EBP && PtrVT == MVT::i32)) && in LowerEH_RETURN()
14654 unsigned StoreAddrReg = (PtrVT == MVT::i64) ? X86::RCX : X86::ECX; in LowerEH_RETURN()
14663 return DAG.getNode(X86ISD::EH_RETURN, dl, MVT::Other, Chain, in LowerEH_RETURN()
14671 DAG.getVTList(MVT::i32, MVT::Other), in lowerEH_SJLJ_SETJMP()
14678 return DAG.getNode(X86ISD::EH_SJLJ_LONGJMP, DL, MVT::Other, in lowerEH_SJLJ_LONGJMP()
14712 OutChains[0] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, MVT::i16), in LowerINIT_TRAMPOLINE()
14716 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, in LowerINIT_TRAMPOLINE()
14717 DAG.getConstant(2, MVT::i64)); in LowerINIT_TRAMPOLINE()
14725 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, in LowerINIT_TRAMPOLINE()
14726 DAG.getConstant(10, MVT::i64)); in LowerINIT_TRAMPOLINE()
14727 OutChains[2] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, MVT::i16), in LowerINIT_TRAMPOLINE()
14731 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, in LowerINIT_TRAMPOLINE()
14732 DAG.getConstant(12, MVT::i64)); in LowerINIT_TRAMPOLINE()
14739 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, in LowerINIT_TRAMPOLINE()
14740 DAG.getConstant(20, MVT::i64)); in LowerINIT_TRAMPOLINE()
14741 OutChains[4] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, MVT::i16), in LowerINIT_TRAMPOLINE()
14746 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, in LowerINIT_TRAMPOLINE()
14747 DAG.getConstant(22, MVT::i64)); in LowerINIT_TRAMPOLINE()
14748 OutChains[5] = DAG.getStore(Root, dl, DAG.getConstant(ModRM, MVT::i8), Addr, in LowerINIT_TRAMPOLINE()
14752 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains); in LowerINIT_TRAMPOLINE()
14801 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp, in LowerINIT_TRAMPOLINE()
14802 DAG.getConstant(10, MVT::i32)); in LowerINIT_TRAMPOLINE()
14803 Disp = DAG.getNode(ISD::SUB, dl, MVT::i32, FPtr, Addr); in LowerINIT_TRAMPOLINE()
14809 DAG.getConstant(MOV32ri|N86Reg, MVT::i8), in LowerINIT_TRAMPOLINE()
14813 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp, in LowerINIT_TRAMPOLINE()
14814 DAG.getConstant(1, MVT::i32)); in LowerINIT_TRAMPOLINE()
14820 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp, in LowerINIT_TRAMPOLINE()
14821 DAG.getConstant(5, MVT::i32)); in LowerINIT_TRAMPOLINE()
14822 OutChains[2] = DAG.getStore(Root, dl, DAG.getConstant(JMP, MVT::i8), Addr, in LowerINIT_TRAMPOLINE()
14826 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp, in LowerINIT_TRAMPOLINE()
14827 DAG.getConstant(6, MVT::i32)); in LowerINIT_TRAMPOLINE()
14832 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains); in LowerINIT_TRAMPOLINE()
14861 MVT VT = Op.getSimpleValueType(); in LowerFLT_ROUNDS_()
14874 DAG.getVTList(MVT::Other), in LowerFLT_ROUNDS_()
14875 Ops, MVT::i16, MMO); in LowerFLT_ROUNDS_()
14878 SDValue CWD = DAG.getLoad(MVT::i16, DL, Chain, StackSlot, in LowerFLT_ROUNDS_()
14883 DAG.getNode(ISD::SRL, DL, MVT::i16, in LowerFLT_ROUNDS_()
14884 DAG.getNode(ISD::AND, DL, MVT::i16, in LowerFLT_ROUNDS_()
14885 CWD, DAG.getConstant(0x800, MVT::i16)), in LowerFLT_ROUNDS_()
14886 DAG.getConstant(11, MVT::i8)); in LowerFLT_ROUNDS_()
14888 DAG.getNode(ISD::SRL, DL, MVT::i16, in LowerFLT_ROUNDS_()
14889 DAG.getNode(ISD::AND, DL, MVT::i16, in LowerFLT_ROUNDS_()
14890 CWD, DAG.getConstant(0x400, MVT::i16)), in LowerFLT_ROUNDS_()
14891 DAG.getConstant(9, MVT::i8)); in LowerFLT_ROUNDS_()
14894 DAG.getNode(ISD::AND, DL, MVT::i16, in LowerFLT_ROUNDS_()
14895 DAG.getNode(ISD::ADD, DL, MVT::i16, in LowerFLT_ROUNDS_()
14896 DAG.getNode(ISD::OR, DL, MVT::i16, CWD1, CWD2), in LowerFLT_ROUNDS_()
14897 DAG.getConstant(1, MVT::i16)), in LowerFLT_ROUNDS_()
14898 DAG.getConstant(3, MVT::i16)); in LowerFLT_ROUNDS_()
14905 MVT VT = Op.getSimpleValueType(); in LowerCTLZ()
14911 if (VT == MVT::i8) { in LowerCTLZ()
14913 OpVT = MVT::i32; in LowerCTLZ()
14918 SDVTList VTs = DAG.getVTList(OpVT, MVT::i32); in LowerCTLZ()
14925 DAG.getConstant(X86::COND_E, MVT::i8), in LowerCTLZ()
14933 if (VT == MVT::i8) in LowerCTLZ()
14934 Op = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Op); in LowerCTLZ()
14939 MVT VT = Op.getSimpleValueType(); in LowerCTLZ_ZERO_UNDEF()
14945 if (VT == MVT::i8) { in LowerCTLZ_ZERO_UNDEF()
14947 OpVT = MVT::i32; in LowerCTLZ_ZERO_UNDEF()
14952 SDVTList VTs = DAG.getVTList(OpVT, MVT::i32); in LowerCTLZ_ZERO_UNDEF()
14958 if (VT == MVT::i8) in LowerCTLZ_ZERO_UNDEF()
14959 Op = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Op); in LowerCTLZ_ZERO_UNDEF()
14964 MVT VT = Op.getSimpleValueType(); in LowerCTTZ()
14970 SDVTList VTs = DAG.getVTList(VT, MVT::i32); in LowerCTTZ()
14977 DAG.getConstant(X86::COND_E, MVT::i8), in LowerCTTZ()
14986 MVT VT = Op.getSimpleValueType(); in Lower256IntArith()
15004 MVT EltVT = VT.getVectorElementType(); in Lower256IntArith()
15005 MVT NewVT = MVT::getVectorVT(EltVT, NumElems/2); in Lower256IntArith()
15029 MVT VT = Op.getSimpleValueType(); in LowerMUL()
15039 if (VT == MVT::v4i32) { in LowerMUL()
15049 SDValue Evens = DAG.getNode(X86ISD::PMULUDQ, dl, MVT::v2i64, A, B); in LowerMUL()
15051 SDValue Odds = DAG.getNode(X86ISD::PMULUDQ, dl, MVT::v2i64, Aodds, Bodds); in LowerMUL()
15062 assert((VT == MVT::v2i64 || VT == MVT::v4i64 || VT == MVT::v8i64) && in LowerMUL()
15080 EVT MulVT = (VT == MVT::v2i64) ? MVT::v4i32 : in LowerMUL()
15081 (VT == MVT::v4i64) ? MVT::v8i32 : MVT::v16i32; in LowerMUL()
15142 static_cast<EVT>(MVT::v2i64).getTypeForEVT(*DAG.getContext()), in LowerWin64_i128OP()
15156 assert((VT == MVT::v4i32 && Subtarget->hasSSE2()) || in LowerMUL_LOHI()
15157 (VT == MVT::v8i32 && Subtarget->hasInt256())); in LowerMUL_LOHI()
15166 MVT MulVT = VT == MVT::v4i32 ? MVT::v2i64 : MVT::v4i64; in LowerMUL_LOHI()
15177 if (VT == MVT::v8i32) { in LowerMUL_LOHI()
15208 MVT VT = Op.getSimpleValueType(); in LowerScalarImmediateShift()
15218 if (VT == MVT::v2i64 || VT == MVT::v4i32 || VT == MVT::v8i16 || in LowerScalarImmediateShift()
15220 (VT == MVT::v4i64 || VT == MVT::v8i32 || VT == MVT::v16i16)) || in LowerScalarImmediateShift()
15222 (VT == MVT::v8i64 || VT == MVT::v16i32))) { in LowerScalarImmediateShift()
15229 if (Op.getOpcode() == ISD::SRA && VT != MVT::v2i64 && VT != MVT::v4i64) in LowerScalarImmediateShift()
15234 if (VT == MVT::v16i8) { in LowerScalarImmediateShift()
15238 MVT::v8i16, R, ShiftAmt, in LowerScalarImmediateShift()
15244 MVT::i8)); in LowerScalarImmediateShift()
15251 MVT::v8i16, R, ShiftAmt, in LowerScalarImmediateShift()
15257 MVT::i8)); in LowerScalarImmediateShift()
15271 MVT::i8)); in LowerScalarImmediateShift()
15280 if (Subtarget->hasInt256() && VT == MVT::v32i8) { in LowerScalarImmediateShift()
15284 MVT::v16i16, R, ShiftAmt, in LowerScalarImmediateShift()
15290 MVT::i8)); in LowerScalarImmediateShift()
15297 MVT::v16i16, R, ShiftAmt, in LowerScalarImmediateShift()
15303 MVT::i8)); in LowerScalarImmediateShift()
15317 MVT::i8)); in LowerScalarImmediateShift()
15330 (VT == MVT::v2i64 || (Subtarget->hasInt256() && VT == MVT::v4i64)) && in LowerScalarImmediateShift()
15379 MVT VT = Op.getSimpleValueType(); in LowerScalarVariableShift()
15384 if ((VT == MVT::v2i64 && Op.getOpcode() != ISD::SRA) || in LowerScalarVariableShift()
15385 VT == MVT::v4i32 || VT == MVT::v8i16 || in LowerScalarVariableShift()
15387 ((VT == MVT::v4i64 && Op.getOpcode() != ISD::SRA) || in LowerScalarVariableShift()
15388 VT == MVT::v8i32 || VT == MVT::v16i16)) || in LowerScalarVariableShift()
15389 (Subtarget->hasAVX512() && (VT == MVT::v8i64 || VT == MVT::v16i32))) { in LowerScalarVariableShift()
15440 if (EltVT.bitsGT(MVT::i32)) in LowerScalarVariableShift()
15441 BaseShAmt = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, BaseShAmt); in LowerScalarVariableShift()
15442 else if (EltVT.bitsLT(MVT::i32)) in LowerScalarVariableShift()
15443 BaseShAmt = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, BaseShAmt); in LowerScalarVariableShift()
15451 case MVT::v2i64: in LowerScalarVariableShift()
15452 case MVT::v4i32: in LowerScalarVariableShift()
15453 case MVT::v8i16: in LowerScalarVariableShift()
15454 case MVT::v4i64: in LowerScalarVariableShift()
15455 case MVT::v8i32: in LowerScalarVariableShift()
15456 case MVT::v16i16: in LowerScalarVariableShift()
15457 case MVT::v16i32: in LowerScalarVariableShift()
15458 case MVT::v8i64: in LowerScalarVariableShift()
15464 case MVT::v4i32: in LowerScalarVariableShift()
15465 case MVT::v8i16: in LowerScalarVariableShift()
15466 case MVT::v8i32: in LowerScalarVariableShift()
15467 case MVT::v16i16: in LowerScalarVariableShift()
15468 case MVT::v16i32: in LowerScalarVariableShift()
15469 case MVT::v8i64: in LowerScalarVariableShift()
15475 case MVT::v2i64: in LowerScalarVariableShift()
15476 case MVT::v4i32: in LowerScalarVariableShift()
15477 case MVT::v8i16: in LowerScalarVariableShift()
15478 case MVT::v4i64: in LowerScalarVariableShift()
15479 case MVT::v8i32: in LowerScalarVariableShift()
15480 case MVT::v16i16: in LowerScalarVariableShift()
15481 case MVT::v16i32: in LowerScalarVariableShift()
15482 case MVT::v8i64: in LowerScalarVariableShift()
15491 (VT == MVT::v2i64 || (Subtarget->hasInt256() && VT == MVT::v4i64) || in LowerScalarVariableShift()
15492 (Subtarget->hasAVX512() && VT == MVT::v8i64)) && in LowerScalarVariableShift()
15523 MVT VT = Op.getSimpleValueType(); in LowerShift()
15540 if (Subtarget->hasAVX512() && (VT == MVT::v16i32 || VT == MVT::v8i64)) in LowerShift()
15545 (VT == MVT::v2i64 || VT == MVT::v4i32 || in LowerShift()
15546 VT == MVT::v4i64 || VT == MVT::v8i32)) in LowerShift()
15549 (VT == MVT::v2i64 || VT == MVT::v4i32 || in LowerShift()
15550 VT == MVT::v4i64 || VT == MVT::v8i32)) in LowerShift()
15552 if (Op.getOpcode() == ISD::SRA && (VT == MVT::v4i32 || VT == MVT::v8i32)) in LowerShift()
15560 (VT == MVT::v8i16 || VT == MVT::v4i32 || in LowerShift()
15561 (Subtarget->hasInt256() && VT == MVT::v16i16)) && in LowerShift()
15590 if (VT == MVT::v4i32 && Op->getOpcode() == ISD::SHL) { in LowerShift()
15594 Op = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, Op); in LowerShift()
15611 if ((VT == MVT::v8i16 || VT == MVT::v4i32) && in LowerShift()
15618 SDValue Amt2 = (VT == MVT::v4i32) ? Amt->getOperand(1) : in LowerShift()
15623 if (VT == MVT::v4i32) { in LowerShift()
15655 EVT CastVT = MVT::v4i32; in LowerShift()
15663 CastVT = MVT::v2i64; in LowerShift()
15672 if (VT == MVT::v16i8 && Op->getOpcode() == ISD::SHL) { in LowerShift()
15689 M = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, MVT::v8i16, M, 4, DAG); in LowerShift()
15700 M = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, MVT::v8i16, M, 2, DAG); in LowerShift()
15718 if (Subtarget->hasInt256() && VT == MVT::v8i16) { in LowerShift()
15719 MVT NewVT = VT == MVT::v8i16 ? MVT::v8i32 : MVT::v16i16; in LowerShift()
15731 MVT EltVT = VT.getVectorElementType(); in LowerShift()
15732 EVT NewVT = MVT::getVectorVT(EltVT, NumElems/2); in LowerShift()
15819 MVT::i32); in LowerXALUO()
15823 DAG.getNode(X86ISD::SETCC, DL, MVT::i8, in LowerXALUO()
15824 DAG.getConstant(X86::COND_O, MVT::i32), in LowerXALUO()
15832 SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::i32); in LowerXALUO()
15837 DAG.getConstant(Cond, MVT::i32), in LowerXALUO()
15847 MVT VT = Op.getSimpleValueType(); in LowerSIGN_EXTEND_INREG()
15857 case MVT::v8i32: in LowerSIGN_EXTEND_INREG()
15858 case MVT::v16i16: in LowerSIGN_EXTEND_INREG()
15870 MVT EltVT = VT.getVectorElementType(); in LowerSIGN_EXTEND_INREG()
15871 EVT NewVT = MVT::getVectorVT(EltVT, NumElems/2); in LowerSIGN_EXTEND_INREG()
15885 case MVT::v4i32: in LowerSIGN_EXTEND_INREG()
15886 case MVT::v8i16: { in LowerSIGN_EXTEND_INREG()
15899 if (ExtraEltVT == MVT::i8 || ExtraEltVT == MVT::i16 || in LowerSIGN_EXTEND_INREG()
15900 ExtraEltVT == MVT::i32) { in LowerSIGN_EXTEND_INREG()
15934 return DAG.getNode(X86ISD::MFENCE, dl, MVT::Other, Op.getOperand(0)); in LowerATOMIC_FENCE()
15937 SDValue Zero = DAG.getConstant(0, MVT::i32); in LowerATOMIC_FENCE()
15939 DAG.getRegister(X86::ESP, MVT::i32), // Base in LowerATOMIC_FENCE()
15940 DAG.getTargetConstant(1, MVT::i8), // Scale in LowerATOMIC_FENCE()
15941 DAG.getRegister(0, MVT::i32), // Index in LowerATOMIC_FENCE()
15942 DAG.getTargetConstant(0, MVT::i32), // Disp in LowerATOMIC_FENCE()
15943 DAG.getRegister(0, MVT::i32), // Segment. in LowerATOMIC_FENCE()
15947 SDNode *Res = DAG.getMachineNode(X86::OR32mrLocked, dl, MVT::Other, Ops); in LowerATOMIC_FENCE()
15952 return DAG.getNode(X86ISD::MEMBARRIER, dl, MVT::Other, Op.getOperand(0)); in LowerATOMIC_FENCE()
15957 MVT T = Op.getSimpleValueType(); in LowerCMP_SWAP()
15963 case MVT::i8: Reg = X86::AL; size = 1; break; in LowerCMP_SWAP()
15964 case MVT::i16: Reg = X86::AX; size = 2; break; in LowerCMP_SWAP()
15965 case MVT::i32: Reg = X86::EAX; size = 4; break; in LowerCMP_SWAP()
15966 case MVT::i64: in LowerCMP_SWAP()
15976 DAG.getTargetConstant(size, MVT::i8), in LowerCMP_SWAP()
15978 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue); in LowerCMP_SWAP()
15986 MVT::i32, cpOut.getValue(2)); in LowerCMP_SWAP()
15988 DAG.getConstant(X86::COND_E, MVT::i8), EFLAGS); in LowerCMP_SWAP()
15998 MVT SrcVT = Op.getOperand(0).getSimpleValueType(); in LowerBITCAST()
15999 MVT DstVT = Op.getSimpleValueType(); in LowerBITCAST()
16001 if (SrcVT == MVT::v2i32 || SrcVT == MVT::v4i16 || SrcVT == MVT::v8i8) { in LowerBITCAST()
16003 if (DstVT != MVT::f64) in LowerBITCAST()
16026 SDValue ToV2F64 = DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, BV); in LowerBITCAST()
16027 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, ToV2F64, in LowerBITCAST()
16033 assert((DstVT == MVT::i64 || in LowerBITCAST()
16037 if (SrcVT==MVT::i64 && DstVT.isVector()) in LowerBITCAST()
16039 if (DstVT==MVT::i64 && SrcVT.isVector()) in LowerBITCAST()
16095 SDVTList VTs = DAG.getVTList(VT, MVT::i32); in LowerADDC_ADDE_SUBC_SUBE()
16135 bool isF64 = ArgVT == MVT::f64; in LowerFSINCOS()
16266 SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other); in ReplaceATOMIC_LOAD()
16329 if (N->getOperand(0).getValueType() != MVT::v2i32 || in ReplaceNodeResults()
16330 N->getValueType(0) != MVT::v2f32) in ReplaceNodeResults()
16332 SDValue ZExtIn = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v2i64, in ReplaceNodeResults()
16335 MVT::f64); in ReplaceNodeResults()
16336 SDValue VBias = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v2f64, Bias, Bias); in ReplaceNodeResults()
16337 SDValue Or = DAG.getNode(ISD::OR, dl, MVT::v2i64, ZExtIn, in ReplaceNodeResults()
16338 DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, VBias)); in ReplaceNodeResults()
16339 Or = DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, Or); in ReplaceNodeResults()
16340 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::v2f64, Or, VBias); in ReplaceNodeResults()
16341 Results.push_back(DAG.getNode(X86ISD::VFPROUND, dl, MVT::v4f32, Sub)); in ReplaceNodeResults()
16347 SDValue V = DAG.getNode(X86ISD::VFPROUND, dl, MVT::v4f32, N->getOperand(0)); in ReplaceNodeResults()
16372 assert((T == MVT::i64 || T == MVT::i128) && "can only expand cmpxchg pair"); in ReplaceNodeResults()
16373 bool Regs64bit = T == MVT::i128; in ReplaceNodeResults()
16374 EVT HalfT = Regs64bit ? MVT::i64 : MVT::i32; in ReplaceNodeResults()
16400 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue); in ReplaceNodeResults()
16414 MVT::i32, cpOutH.getValue(2)); in ReplaceNodeResults()
16416 DAG.getNode(X86ISD::SETCC, dl, MVT::i8, in ReplaceNodeResults()
16417 DAG.getConstant(X86::COND_E, MVT::i8), EFLAGS); in ReplaceNodeResults()
16448 if (SrcVT != MVT::f64 || in ReplaceNodeResults()
16449 (DstVT != MVT::v2i32 && DstVT != MVT::v4i16 && DstVT != MVT::v8i8)) in ReplaceNodeResults()
16456 MVT::v2f64, N->getOperand(0)); in ReplaceNodeResults()
16762 return VT1 == MVT::i32 && VT2 == MVT::i64 && Subtarget->is64Bit(); in isZExtFree()
16779 case MVT::i8: in isZExtFree()
16780 case MVT::i16: in isZExtFree()
16781 case MVT::i32: in isZExtFree()
16800 case MVT::f32: in isFMAFasterThanFMulAndFAdd()
16801 case MVT::f64: in isFMAFasterThanFMulAndFAdd()
16812 return !(VT1 == MVT::i32 && VT2 == MVT::i16); in isNarrowingProfitable()
16825 MVT SVT = VT.getSimpleVT(); in isShuffleMaskLegal()
16869 MVT SVT = VT.getSimpleVT(); in isVectorClearMaskLegal()
17078 const TargetRegisterClass *AddrRegClass = getRegClassFor(MVT::i64); in EmitVAARG64WithCustomInserter()
17079 const TargetRegisterClass *OffsetRegClass = getRegClassFor(MVT::i32); in EmitVAARG64WithCustomInserter()
17514 getRegClassFor(Is64Bit ? MVT::i64:MVT::i32); in EmitLoweredSegAlloca()
17734 assert(RC->hasType(MVT::i32) && "Invalid destination!"); in emitEHSjLjSetJmp()
17740 MVT PVT = getPointerTy(); in emitEHSjLjSetJmp()
17741 assert((PVT == MVT::i64 || PVT == MVT::i32) && in emitEHSjLjSetJmp()
17784 PtrStoreOpc = (PVT == MVT::i64) ? X86::MOV64mr : X86::MOV32mr; in emitEHSjLjSetJmp()
17804 PtrStoreOpc = (PVT == MVT::i64) ? X86::MOV64mi32 : X86::MOV32mi; in emitEHSjLjSetJmp()
17860 MVT PVT = getPointerTy(); in emitEHSjLjLongJmp()
17861 assert((PVT == MVT::i64 || PVT == MVT::i32) && in emitEHSjLjLongJmp()
17865 (PVT == MVT::i64) ? &X86::GR64RegClass : &X86::GR32RegClass; in emitEHSjLjLongJmp()
17870 unsigned FP = (PVT == MVT::i64) ? X86::RBP : X86::EBP; in emitEHSjLjLongJmp()
17878 unsigned PtrLoadOpc = (PVT == MVT::i64) ? X86::MOV64rm : X86::MOV32rm; in emitEHSjLjLongJmp()
17879 unsigned IJmpOpc = (PVT == MVT::i64) ? X86::JMP64r : X86::JMP32r; in emitEHSjLjLongJmp()
18370 SDVTList Tys = DAG.getVTList(MVT::v4i64, MVT::Other); in PerformShuffleCombine256()
18384 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, in PerformShuffleCombine256()
18500 if (V.getValueType() != MVT::v16i8 && V.getValueType() != MVT::v8i16) in combineRedundantDWordShuffle()
18633 V = DAG.getNode(V.getOpcode(), DL, MVT::v8i16, V.getOperand(0), in combineRedundantHalfShuffle()
18650 MVT VT = N.getSimpleValueType(); in PerformTargetShuffleCombine()
18675 assert(VT == MVT::v8i16); in PerformTargetShuffleCombine()
18689 V = DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, V); in PerformTargetShuffleCombine()
18691 V = DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, V, in PerformTargetShuffleCombine()
18694 return DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V); in PerformTargetShuffleCombine()
18729 V = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, D.getOperand(0)); in PerformTargetShuffleCombine()
18733 DL, MVT::v8i16, V, V); in PerformTargetShuffleCombine()
18772 ((Subtarget->hasSSE3() && (VT == MVT::v4f32 || VT == MVT::v2f64)) || in PerformShuffleCombine()
18773 (Subtarget->hasAVX() && (VT == MVT::v8f32 || VT == MVT::v4f64))) && in PerformShuffleCombine()
19006 InputVector.getNode()->getOperand(0).getValueType() == MVT::x86mmx && in PerformEXTRACT_VECTOR_ELTCombine()
19007 InputVector.hasOneUse() && N->getValueType(0) == MVT::i32) in PerformEXTRACT_VECTOR_ELTCombine()
19014 if (InputVector.getValueType() != MVT::v4i32) in PerformEXTRACT_VECTOR_ELTCombine()
19031 if (Extract->getValueType(0) != MVT::i32) in PerformEXTRACT_VECTOR_ELTCombine()
19099 case MVT::v32i8: in matchIntegerMINMAX()
19100 case MVT::v16i16: in matchIntegerMINMAX()
19101 case MVT::v8i32: in matchIntegerMINMAX()
19107 case MVT::v16i8: in matchIntegerMINMAX()
19108 case MVT::v8i16: in matchIntegerMINMAX()
19109 case MVT::v4i32: in matchIntegerMINMAX()
19116 (Subtarget->hasSSE2() && VT == MVT::v16i8); in matchIntegerMINMAX()
19118 (Subtarget->hasSSE2() && VT == MVT::v8i16); in matchIntegerMINMAX()
19178 MVT VT = N->getSimpleValueType(0); in TransformVSELECTtoBlendVECTOR_SHUFFLE()
19179 MVT EltVT = VT.getVectorElementType(); in TransformVSELECTtoBlendVECTOR_SHUFFLE()
19185 if (!Subtarget->hasSSE41() || EltVT == MVT::i8) in TransformVSELECTtoBlendVECTOR_SHUFFLE()
19187 if (!Subtarget->hasInt256() && VT == MVT::v16i16) in TransformVSELECTtoBlendVECTOR_SHUFFLE()
19227 VT != MVT::f80 && TLI.isTypeLegal(VT) && in PerformSELECTCombine()
19229 (Subtarget->hasSSE1() && VT.getScalarType() == MVT::f32))) { in PerformSELECTCombine()
19368 CondVT.getVectorElementType() == MVT::i1) { in PerformSELECTCombine()
19375 (OpVT.getVectorElementType() == MVT::i8 || in PerformSELECTCombine()
19376 OpVT.getVectorElementType() == MVT::i16)) { in PerformSELECTCombine()
19413 DAG.getConstant(ShAmt, MVT::i8)); in PerformSELECTCombine()
19431 if (N->getValueType(0) == MVT::i32 || N->getValueType(0) == MVT::i64) { in PerformSELECTCombine()
19433 if (N->getValueType(0) == MVT::i32) Diff = (unsigned)Diff; in PerformSELECTCombine()
19513 ((Subtarget->hasSSE2() && (VT == MVT::v16i8 || VT == MVT::v8i16)) || in PerformSELECTCombine()
19514 (Subtarget->hasAVX2() && (VT == MVT::v32i8 || VT == MVT::v16i16)))) { in PerformSELECTCombine()
19646 if (VT == MVT::v4i32 || VT == MVT::v4f32 || in PerformSELECTCombine()
19647 (Subtarget->hasSSE2() && (VT == MVT::v2i64 || VT == MVT::v2f64))) { in PerformSELECTCombine()
19671 if (VT == MVT::v4i32 || VT == MVT::v4f32) in PerformSELECTCombine()
19676 if (Subtarget->hasSSE2() && (VT == MVT::v4i32 || VT == MVT::v4f32)) { in PerformSELECTCombine()
19707 EVT NVT = (VT == MVT::v4i32) ? MVT::v2i64 : MVT::v2f64; in PerformSELECTCombine()
19728 (TLI.isOperationLegalOrCustom(ISD::VSELECT, VT) && VT != MVT::v16i16 && in PerformSELECTCombine()
19729 VT != MVT::v8i16)) { in PerformSELECTCombine()
19939 (FalseOp.getValueType() != MVT::f80 || hasFPCMov(CC))) { in PerformCMOVCombine()
19941 DAG.getConstant(CC, MVT::i8), Flags }; in PerformCMOVCombine()
19962 Cond = DAG.getNode(X86ISD::SETCC, DL, MVT::i8, in PerformCMOVCombine()
19963 DAG.getConstant(CC, MVT::i8), Cond); in PerformCMOVCombine()
19970 DAG.getConstant(ShAmt, MVT::i8)); in PerformCMOVCombine()
19979 Cond = DAG.getNode(X86ISD::SETCC, DL, MVT::i8, in PerformCMOVCombine()
19980 DAG.getConstant(CC, MVT::i8), Cond); in PerformCMOVCombine()
19995 if (N->getValueType(0) == MVT::i32 || N->getValueType(0) == MVT::i64) { in PerformCMOVCombine()
19997 if (N->getValueType(0) == MVT::i32) Diff = (unsigned)Diff; in PerformCMOVCombine()
20017 Cond = DAG.getNode(X86ISD::SETCC, DL, MVT::i8, in PerformCMOVCombine()
20018 DAG.getConstant(CC, MVT::i8), Cond); in PerformCMOVCombine()
20071 DAG.getConstant(CC, MVT::i8), Cond }; in PerformCMOVCombine()
20191 if (VT != MVT::i64) in PerformMulCombine()
20227 DAG.getConstant(Log2_64(MulAmt1), MVT::i8)); in PerformMulCombine()
20234 DAG.getConstant(Log2_64(MulAmt2), MVT::i8)); in PerformMulCombine()
20294 if (VT != MVT::v2i64 && VT != MVT::v4i32 && VT != MVT::v8i16 && in performShiftToAllZeros()
20296 (VT != MVT::v4i64 && VT != MVT::v8i32 && VT != MVT::v16i16))) in performShiftToAllZeros()
20360 if (VT == MVT::f32 || VT == MVT::f64) { in CMPEQCombine()
20395 SDValue FSetCC = DAG.getNode(X86ISD::FSETCC, DL, MVT::i1, CMP00, in CMPEQCombine()
20396 CMP01, DAG.getConstant(x86cc, MVT::i8)); in CMPEQCombine()
20397 if (N->getValueType(0) != MVT::i1) in CMPEQCombine()
20404 DAG.getConstant(x86cc, MVT::i8)); in CMPEQCombine()
20406 bool is64BitFP = (CMP00.getValueType() == MVT::f64); in CMPEQCombine()
20407 MVT IntVT = is64BitFP ? MVT::i64 : MVT::i32; in CMPEQCombine()
20415 SDValue Vector64 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64, in CMPEQCombine()
20417 SDValue Vector32 = DAG.getNode(ISD::BITCAST, DL, MVT::v4f32, in CMPEQCombine()
20419 OnesOrZeroesF = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, in CMPEQCombine()
20421 IntVT = MVT::i32; in CMPEQCombine()
20427 SDValue OneBitOfTruth = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, ANDed); in CMPEQCombine()
20562 if (VT == MVT::i32 || VT == MVT::i64) { in PerformAndCombine()
20591 if (VT != MVT::v2i64 && VT != MVT::v4i64) in PerformAndCombine()
20628 if (VT == MVT::v2i64 || VT == MVT::v4i64) { in PerformOrCombine()
20630 (VT == MVT::v4i64 && !Subtarget->hasInt256())) in PerformOrCombine()
20694 EVT BlendVT = (VT == MVT::v4i64) ? MVT::v32i8 : MVT::v16i8; in PerformOrCombine()
20704 if (VT != MVT::i16 && VT != MVT::i32 && VT != MVT::i64) in PerformOrCombine()
20728 if (ShAmt0.getValueType() != MVT::i8) in PerformOrCombine()
20731 if (ShAmt1.getValueType() != MVT::i8) in PerformOrCombine()
20759 MVT::i8, ShAmt0)); in PerformOrCombine()
20768 MVT::i8, ShAmt0)); in PerformOrCombine()
20797 SDValue Neg = DAG.getNode(X86ISD::SUB, DL, DAG.getVTList(VT, MVT::i32), in performIntegerAbsCombine()
20801 DAG.getConstant(X86::COND_GE, MVT::i8), in performIntegerAbsCombine()
20803 return DAG.getNode(X86ISD::CMOV, DL, DAG.getVTList(VT, MVT::Glue), Ops); in performIntegerAbsCombine()
20859 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, in PerformLOADCombine()
20894 MVT SclrLoadTy = MVT::i8; in PerformLOADCombine()
20895 for (unsigned tp = MVT::FIRST_INTEGER_VALUETYPE; in PerformLOADCombine()
20896 tp < MVT::LAST_INTEGER_VALUETYPE; ++tp) { in PerformLOADCombine()
20897 MVT Tp = (MVT::SimpleValueType)tp; in PerformLOADCombine()
20904 if (TLI.isTypeLegal(MVT::f64) && SclrLoadTy.getSizeInBits() < 64 && in PerformLOADCombine()
20906 SclrLoadTy = MVT::f64; in PerformLOADCombine()
20960 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains); in PerformLOADCombine()
21055 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Ch0, Ch1); in PerformSTORECombine()
21100 MVT StoreType = MVT::i8; in PerformSTORECombine()
21101 for (unsigned tp = MVT::FIRST_INTEGER_VALUETYPE; in PerformSTORECombine()
21102 tp < MVT::LAST_INTEGER_VALUETYPE; ++tp) { in PerformSTORECombine()
21103 MVT Tp = (MVT::SimpleValueType)tp; in PerformSTORECombine()
21109 if (TLI.isTypeLegal(MVT::f64) && StoreType.getSizeInBits() < 64 && in PerformSTORECombine()
21111 StoreType = MVT::f64; in PerformSTORECombine()
21135 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains); in PerformSTORECombine()
21153 (VT == MVT::i64 && F64IsLegal && !Subtarget->is64Bit())) && in PerformSTORECombine()
21193 EVT LdVT = Subtarget->is64Bit() ? MVT::i64 : MVT::f64; in PerformSTORECombine()
21201 NewChain = DAG.getNode(ISD::TokenFactor, LdDL, MVT::Other, Ops); in PerformSTORECombine()
21211 SDValue HiAddr = DAG.getNode(ISD::ADD, LdDL, MVT::i32, LoAddr, in PerformSTORECombine()
21212 DAG.getConstant(4, MVT::i32)); in PerformSTORECombine()
21214 SDValue LoLd = DAG.getLoad(MVT::i32, LdDL, Ld->getChain(), LoAddr, in PerformSTORECombine()
21218 SDValue HiLd = DAG.getLoad(MVT::i32, LdDL, Ld->getChain(), HiAddr, in PerformSTORECombine()
21228 NewChain = DAG.getNode(ISD::TokenFactor, LdDL, MVT::Other, Ops); in PerformSTORECombine()
21232 HiAddr = DAG.getNode(ISD::ADD, StDL, MVT::i32, LoAddr, in PerformSTORECombine()
21233 DAG.getConstant(4, MVT::i32)); in PerformSTORECombine()
21244 return DAG.getNode(ISD::TokenFactor, StDL, MVT::Other, LoSt, HiSt); in PerformSTORECombine()
21279 MVT VT = LHS.getSimpleValueType(); in isHorizontalBinOp()
21383 if (((Subtarget->hasSSE3() && (VT == MVT::v4f32 || VT == MVT::v2f64)) || in PerformFADDCombine()
21384 (Subtarget->hasFp256() && (VT == MVT::v8f32 || VT == MVT::v4f64))) && in PerformFADDCombine()
21398 if (((Subtarget->hasSSE3() && (VT == MVT::v4f32 || VT == MVT::v2f64)) || in PerformFSUBCombine()
21399 (Subtarget->hasFp256() && (VT == MVT::v8f32 || VT == MVT::v4f64))) && in PerformFSUBCombine()
21516 if (VT == MVT::v4i64 && (N0.getOpcode() == ISD::ANY_EXTEND || in PerformSIGN_EXTEND_INREGCombine()
21526 if (N00.getValueType() == MVT::v4i32 && ExtraVT.getSizeInBits() < 128) { in PerformSIGN_EXTEND_INREGCombine()
21527 SDValue Tmp = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, MVT::v4i32, in PerformSIGN_EXTEND_INREGCombine()
21529 return DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i64, Tmp); in PerformSIGN_EXTEND_INREGCombine()
21564 if ((ScalarVT != MVT::f32 && ScalarVT != MVT::f64) || in PerformFMACombine()
21667 if (VT.getScalarType() == MVT::i1) { in PerformISDSETCCCombine()
21669 (LHS.getOperand(0).getValueType().getScalarType() == MVT::i1); in PerformISDSETCCCombine()
21674 (RHS.getOperand(0).getValueType().getScalarType() == MVT::i1); in PerformISDSETCCCombine()
21700 MVT VT = N->getOperand(1)->getSimpleValueType(0); in PerformINSERTPSCombine()
21701 assert((VT == MVT::v4f32 || VT == MVT::v4i32) && in PerformINSERTPSCombine()
21728 MVT VT) { in MaterializeSETB()
21729 if (VT == MVT::i8) in MaterializeSETB()
21731 DAG.getNode(X86ISD::SETCC_CARRY, DL, MVT::i8, in MaterializeSETB()
21732 DAG.getConstant(X86::COND_B, MVT::i8), EFLAGS), in MaterializeSETB()
21734 assert (VT == MVT::i1 && "Unexpected type for SECCC node"); in MaterializeSETB()
21735 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, in MaterializeSETB()
21736 DAG.getNode(X86ISD::SETCC_CARRY, DL, MVT::i8, in MaterializeSETB()
21737 DAG.getConstant(X86::COND_B, MVT::i8), EFLAGS)); in MaterializeSETB()
21776 SDValue Cond = DAG.getConstant(CC, MVT::i8); in PerformSETCCCombine()
21798 SDValue Cond = DAG.getConstant(CC, MVT::i8); in PerformBrCondCombine()
21812 if (InVT == MVT::v8i8 || InVT == MVT::v4i8) { in PerformSINT_TO_FPCombine()
21814 MVT DstVT = InVT == MVT::v4i8 ? MVT::v4i32 : MVT::v8i32; in PerformSINT_TO_FPCombine()
21827 VT == MVT::i64) { in PerformSINT_TO_FPCombine()
21853 DAG.getConstant(X86::COND_B,MVT::i8), in PerformADCCombine()
21889 SDValue NewCmp = DAG.getNode(X86ISD::CMP, DL, MVT::i32, CmpOp0, in OptimizeConditionalInDecrement()
21910 if (((Subtarget->hasSSSE3() && (VT == MVT::v8i16 || VT == MVT::v4i32)) || in PerformAddCombine()
21911 (Subtarget->hasInt256() && (VT == MVT::v16i16 || VT == MVT::v8i32))) && in PerformAddCombine()
21943 if (((Subtarget->hasSSSE3() && (VT == MVT::v8i16 || VT == MVT::v4i32)) || in PerformSubCombine()
21944 (Subtarget->hasInt256() && (VT == MVT::v16i16 || VT == MVT::v8i32))) && in PerformSubCombine()
22042 if (VT != MVT::i16) in isTypeDesirableForOp()
22069 if (VT != MVT::i16) in IsDesirableToPromoteOp()
22128 PVT = MVT::i32; in IsDesirableToPromoteOp()
22468 Result = DAG.getTargetConstant(C->getSExtValue(), MVT::i64); in LowerAsmOperandForConstraint()
22493 Result = DAG.getTargetConstant(CST->getSExtValue(), MVT::i64); in LowerAsmOperandForConstraint()
22553 MVT VT) const { in getRegForInlineAsmConstraint()
22565 if (VT == MVT::i32 || VT == MVT::f32) in getRegForInlineAsmConstraint()
22567 if (VT == MVT::i16) in getRegForInlineAsmConstraint()
22569 if (VT == MVT::i8 || VT == MVT::i1) in getRegForInlineAsmConstraint()
22571 if (VT == MVT::i64 || VT == MVT::f64) in getRegForInlineAsmConstraint()
22577 if (VT == MVT::i32 || VT == MVT::f32) in getRegForInlineAsmConstraint()
22579 if (VT == MVT::i16) in getRegForInlineAsmConstraint()
22581 if (VT == MVT::i8 || VT == MVT::i1) in getRegForInlineAsmConstraint()
22583 if (VT == MVT::i64) in getRegForInlineAsmConstraint()
22588 if (VT == MVT::i8 || VT == MVT::i1) in getRegForInlineAsmConstraint()
22590 if (VT == MVT::i16) in getRegForInlineAsmConstraint()
22592 if (VT == MVT::i32 || VT == MVT::f32 || !Subtarget->is64Bit()) in getRegForInlineAsmConstraint()
22596 if (VT == MVT::i8 || VT == MVT::i1) in getRegForInlineAsmConstraint()
22598 if (VT == MVT::i16) in getRegForInlineAsmConstraint()
22600 if (VT == MVT::i32 || !Subtarget->is64Bit()) in getRegForInlineAsmConstraint()
22606 if (VT == MVT::f32 && !isScalarFPTypeInSSEReg(VT)) in getRegForInlineAsmConstraint()
22608 if (VT == MVT::f64 && !isScalarFPTypeInSSEReg(VT)) in getRegForInlineAsmConstraint()
22623 case MVT::f32: in getRegForInlineAsmConstraint()
22624 case MVT::i32: in getRegForInlineAsmConstraint()
22626 case MVT::f64: in getRegForInlineAsmConstraint()
22627 case MVT::i64: in getRegForInlineAsmConstraint()
22630 case MVT::v16i8: in getRegForInlineAsmConstraint()
22631 case MVT::v8i16: in getRegForInlineAsmConstraint()
22632 case MVT::v4i32: in getRegForInlineAsmConstraint()
22633 case MVT::v2i64: in getRegForInlineAsmConstraint()
22634 case MVT::v4f32: in getRegForInlineAsmConstraint()
22635 case MVT::v2f64: in getRegForInlineAsmConstraint()
22638 case MVT::v32i8: in getRegForInlineAsmConstraint()
22639 case MVT::v16i16: in getRegForInlineAsmConstraint()
22640 case MVT::v8i32: in getRegForInlineAsmConstraint()
22641 case MVT::v4i64: in getRegForInlineAsmConstraint()
22642 case MVT::v8f32: in getRegForInlineAsmConstraint()
22643 case MVT::v4f64: in getRegForInlineAsmConstraint()
22645 case MVT::v8f64: in getRegForInlineAsmConstraint()
22646 case MVT::v16f32: in getRegForInlineAsmConstraint()
22647 case MVT::v16i32: in getRegForInlineAsmConstraint()
22648 case MVT::v8i64: in getRegForInlineAsmConstraint()
22710 if (VT == MVT::i8 || VT == MVT::i1) { in getRegForInlineAsmConstraint()
22723 } else if (VT == MVT::i32 || VT == MVT::f32) { in getRegForInlineAsmConstraint()
22740 } else if (VT == MVT::i64 || VT == MVT::f64) { in getRegForInlineAsmConstraint()
22772 if (VT == MVT::f32 || VT == MVT::i32) in getRegForInlineAsmConstraint()
22774 else if (VT == MVT::f64 || VT == MVT::i64) in getRegForInlineAsmConstraint()