Lines Matching refs:MVT
88 static const MVT IntVTs[] = { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }; in X86TargetLowering()
150 addRegisterClass(MVT::i8, &X86::GR8RegClass); in X86TargetLowering()
151 addRegisterClass(MVT::i16, &X86::GR16RegClass); in X86TargetLowering()
152 addRegisterClass(MVT::i32, &X86::GR32RegClass); in X86TargetLowering()
154 addRegisterClass(MVT::i64, &X86::GR64RegClass); in X86TargetLowering()
156 for (MVT VT : MVT::integer_valuetypes()) in X86TargetLowering()
157 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote); in X86TargetLowering()
160 setTruncStoreAction(MVT::i64, MVT::i32, Expand); in X86TargetLowering()
161 setTruncStoreAction(MVT::i64, MVT::i16, Expand); in X86TargetLowering()
162 setTruncStoreAction(MVT::i64, MVT::i8 , Expand); in X86TargetLowering()
163 setTruncStoreAction(MVT::i32, MVT::i16, Expand); in X86TargetLowering()
164 setTruncStoreAction(MVT::i32, MVT::i8 , Expand); in X86TargetLowering()
165 setTruncStoreAction(MVT::i16, MVT::i8, Expand); in X86TargetLowering()
167 setTruncStoreAction(MVT::f64, MVT::f32, Expand); in X86TargetLowering()
170 setCondCodeAction(ISD::SETOEQ, MVT::f32, Expand); in X86TargetLowering()
171 setCondCodeAction(ISD::SETOEQ, MVT::f64, Expand); in X86TargetLowering()
172 setCondCodeAction(ISD::SETOEQ, MVT::f80, Expand); in X86TargetLowering()
173 setCondCodeAction(ISD::SETUNE, MVT::f32, Expand); in X86TargetLowering()
174 setCondCodeAction(ISD::SETUNE, MVT::f64, Expand); in X86TargetLowering()
175 setCondCodeAction(ISD::SETUNE, MVT::f80, Expand); in X86TargetLowering()
179 setOperationAction(ISD::UINT_TO_FP , MVT::i1 , Promote); in X86TargetLowering()
180 setOperationAction(ISD::UINT_TO_FP , MVT::i8 , Promote); in X86TargetLowering()
181 setOperationAction(ISD::UINT_TO_FP , MVT::i16 , Promote); in X86TargetLowering()
184 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Promote); in X86TargetLowering()
185 setOperationAction(ISD::UINT_TO_FP , MVT::i64 , Custom); in X86TargetLowering()
189 setOperationAction(ISD::UINT_TO_FP , MVT::i64 , Custom); in X86TargetLowering()
192 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Custom); in X86TargetLowering()
197 setOperationAction(ISD::SINT_TO_FP , MVT::i1 , Promote); in X86TargetLowering()
198 setOperationAction(ISD::SINT_TO_FP , MVT::i8 , Promote); in X86TargetLowering()
203 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Promote); in X86TargetLowering()
205 setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Custom); in X86TargetLowering()
207 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Custom); in X86TargetLowering()
208 setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Custom); in X86TargetLowering()
211 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Promote); in X86TargetLowering()
212 setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Promote); in X86TargetLowering()
217 setOperationAction(ISD::FP_TO_SINT , MVT::i64 , Custom); in X86TargetLowering()
218 setOperationAction(ISD::SINT_TO_FP , MVT::i64 , Custom); in X86TargetLowering()
222 setOperationAction(ISD::FP_TO_SINT , MVT::i1 , Promote); in X86TargetLowering()
223 setOperationAction(ISD::FP_TO_SINT , MVT::i8 , Promote); in X86TargetLowering()
226 setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Promote); in X86TargetLowering()
228 setOperationAction(ISD::FP_TO_SINT , MVT::i32 , Custom); in X86TargetLowering()
230 setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Custom); in X86TargetLowering()
231 setOperationAction(ISD::FP_TO_SINT , MVT::i32 , Custom); in X86TargetLowering()
236 setOperationAction(ISD::FP_TO_UINT , MVT::i1 , Promote); in X86TargetLowering()
237 setOperationAction(ISD::FP_TO_UINT , MVT::i8 , Promote); in X86TargetLowering()
238 setOperationAction(ISD::FP_TO_UINT , MVT::i16 , Promote); in X86TargetLowering()
241 setOperationAction(ISD::FP_TO_UINT , MVT::i64 , Expand); in X86TargetLowering()
242 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Promote); in X86TargetLowering()
249 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Expand); in X86TargetLowering()
253 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Custom); in X86TargetLowering()
259 setOperationAction(ISD::FP_TO_UINT , MVT::i64 , Custom); in X86TargetLowering()
264 setOperationAction(ISD::BITCAST , MVT::f32 , Expand); in X86TargetLowering()
265 setOperationAction(ISD::BITCAST , MVT::i32 , Expand); in X86TargetLowering()
267 setOperationAction(ISD::BITCAST , MVT::f64 , Expand); in X86TargetLowering()
269 setOperationAction(ISD::BITCAST , MVT::i64 , Expand); in X86TargetLowering()
284 MVT VT = IntVTs[i]; in X86TargetLowering()
299 setOperationAction(ISD::BR_JT , MVT::Other, Expand); in X86TargetLowering()
300 setOperationAction(ISD::BRCOND , MVT::Other, Custom); in X86TargetLowering()
301 setOperationAction(ISD::BR_CC , MVT::f32, Expand); in X86TargetLowering()
302 setOperationAction(ISD::BR_CC , MVT::f64, Expand); in X86TargetLowering()
303 setOperationAction(ISD::BR_CC , MVT::f80, Expand); in X86TargetLowering()
304 setOperationAction(ISD::BR_CC , MVT::i8, Expand); in X86TargetLowering()
305 setOperationAction(ISD::BR_CC , MVT::i16, Expand); in X86TargetLowering()
306 setOperationAction(ISD::BR_CC , MVT::i32, Expand); in X86TargetLowering()
307 setOperationAction(ISD::BR_CC , MVT::i64, Expand); in X86TargetLowering()
308 setOperationAction(ISD::SELECT_CC , MVT::f32, Expand); in X86TargetLowering()
309 setOperationAction(ISD::SELECT_CC , MVT::f64, Expand); in X86TargetLowering()
310 setOperationAction(ISD::SELECT_CC , MVT::f80, Expand); in X86TargetLowering()
311 setOperationAction(ISD::SELECT_CC , MVT::i8, Expand); in X86TargetLowering()
312 setOperationAction(ISD::SELECT_CC , MVT::i16, Expand); in X86TargetLowering()
313 setOperationAction(ISD::SELECT_CC , MVT::i32, Expand); in X86TargetLowering()
314 setOperationAction(ISD::SELECT_CC , MVT::i64, Expand); in X86TargetLowering()
316 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Legal); in X86TargetLowering()
317 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16 , Legal); in X86TargetLowering()
318 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8 , Legal); in X86TargetLowering()
319 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1 , Expand); in X86TargetLowering()
320 setOperationAction(ISD::FP_ROUND_INREG , MVT::f32 , Expand); in X86TargetLowering()
321 setOperationAction(ISD::FREM , MVT::f32 , Expand); in X86TargetLowering()
322 setOperationAction(ISD::FREM , MVT::f64 , Expand); in X86TargetLowering()
323 setOperationAction(ISD::FREM , MVT::f80 , Expand); in X86TargetLowering()
324 setOperationAction(ISD::FLT_ROUNDS_ , MVT::i32 , Custom); in X86TargetLowering()
328 setOperationAction(ISD::CTTZ , MVT::i8 , Promote); in X86TargetLowering()
329 AddPromotedToType (ISD::CTTZ , MVT::i8 , MVT::i32); in X86TargetLowering()
330 setOperationAction(ISD::CTTZ_ZERO_UNDEF , MVT::i8 , Promote); in X86TargetLowering()
331 AddPromotedToType (ISD::CTTZ_ZERO_UNDEF , MVT::i8 , MVT::i32); in X86TargetLowering()
333 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i16 , Expand); in X86TargetLowering()
334 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i32 , Expand); in X86TargetLowering()
336 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i64, Expand); in X86TargetLowering()
338 setOperationAction(ISD::CTTZ , MVT::i16 , Custom); in X86TargetLowering()
339 setOperationAction(ISD::CTTZ , MVT::i32 , Custom); in X86TargetLowering()
341 setOperationAction(ISD::CTTZ , MVT::i64 , Custom); in X86TargetLowering()
347 setOperationAction(ISD::CTLZ , MVT::i8 , Promote); in X86TargetLowering()
348 AddPromotedToType (ISD::CTLZ , MVT::i8 , MVT::i32); in X86TargetLowering()
349 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i8 , Promote); in X86TargetLowering()
350 AddPromotedToType (ISD::CTLZ_ZERO_UNDEF, MVT::i8 , MVT::i32); in X86TargetLowering()
351 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i16 , Expand); in X86TargetLowering()
352 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32 , Expand); in X86TargetLowering()
354 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i64, Expand); in X86TargetLowering()
356 setOperationAction(ISD::CTLZ , MVT::i8 , Custom); in X86TargetLowering()
357 setOperationAction(ISD::CTLZ , MVT::i16 , Custom); in X86TargetLowering()
358 setOperationAction(ISD::CTLZ , MVT::i32 , Custom); in X86TargetLowering()
359 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i8 , Custom); in X86TargetLowering()
360 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i16 , Custom); in X86TargetLowering()
361 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32 , Custom); in X86TargetLowering()
363 setOperationAction(ISD::CTLZ , MVT::i64 , Custom); in X86TargetLowering()
364 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i64, Custom); in X86TargetLowering()
372 setOperationAction(ISD::FP16_TO_FP, MVT::f32, Expand); in X86TargetLowering()
373 setOperationAction(ISD::FP_TO_FP16, MVT::f32, Expand); in X86TargetLowering()
377 setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand); in X86TargetLowering()
378 setOperationAction(ISD::FP16_TO_FP, MVT::f80, Expand); in X86TargetLowering()
379 setOperationAction(ISD::FP_TO_FP16, MVT::f64, Expand); in X86TargetLowering()
380 setOperationAction(ISD::FP_TO_FP16, MVT::f80, Expand); in X86TargetLowering()
382 setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand); in X86TargetLowering()
383 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand); in X86TargetLowering()
384 setLoadExtAction(ISD::EXTLOAD, MVT::f80, MVT::f16, Expand); in X86TargetLowering()
385 setTruncStoreAction(MVT::f32, MVT::f16, Expand); in X86TargetLowering()
386 setTruncStoreAction(MVT::f64, MVT::f16, Expand); in X86TargetLowering()
387 setTruncStoreAction(MVT::f80, MVT::f16, Expand); in X86TargetLowering()
390 setOperationAction(ISD::CTPOP , MVT::i8 , Promote); in X86TargetLowering()
392 setOperationAction(ISD::CTPOP , MVT::i8 , Expand); in X86TargetLowering()
393 setOperationAction(ISD::CTPOP , MVT::i16 , Expand); in X86TargetLowering()
394 setOperationAction(ISD::CTPOP , MVT::i32 , Expand); in X86TargetLowering()
396 setOperationAction(ISD::CTPOP , MVT::i64 , Expand); in X86TargetLowering()
399 setOperationAction(ISD::READCYCLECOUNTER , MVT::i64 , Custom); in X86TargetLowering()
402 setOperationAction(ISD::BSWAP , MVT::i16 , Expand); in X86TargetLowering()
405 setOperationAction(ISD::SELECT , MVT::i1 , Promote); in X86TargetLowering()
407 setOperationAction(ISD::SELECT , MVT::i8 , Custom); in X86TargetLowering()
408 setOperationAction(ISD::SELECT , MVT::i16 , Custom); in X86TargetLowering()
409 setOperationAction(ISD::SELECT , MVT::i32 , Custom); in X86TargetLowering()
410 setOperationAction(ISD::SELECT , MVT::f32 , Custom); in X86TargetLowering()
411 setOperationAction(ISD::SELECT , MVT::f64 , Custom); in X86TargetLowering()
412 setOperationAction(ISD::SELECT , MVT::f80 , Custom); in X86TargetLowering()
413 setOperationAction(ISD::SETCC , MVT::i8 , Custom); in X86TargetLowering()
414 setOperationAction(ISD::SETCC , MVT::i16 , Custom); in X86TargetLowering()
415 setOperationAction(ISD::SETCC , MVT::i32 , Custom); in X86TargetLowering()
416 setOperationAction(ISD::SETCC , MVT::f32 , Custom); in X86TargetLowering()
417 setOperationAction(ISD::SETCC , MVT::f64 , Custom); in X86TargetLowering()
418 setOperationAction(ISD::SETCC , MVT::f80 , Custom); in X86TargetLowering()
420 setOperationAction(ISD::SELECT , MVT::i64 , Custom); in X86TargetLowering()
421 setOperationAction(ISD::SETCC , MVT::i64 , Custom); in X86TargetLowering()
423 setOperationAction(ISD::EH_RETURN , MVT::Other, Custom); in X86TargetLowering()
430 setOperationAction(ISD::EH_SJLJ_SETJMP, MVT::i32, Custom); in X86TargetLowering()
431 setOperationAction(ISD::EH_SJLJ_LONGJMP, MVT::Other, Custom); in X86TargetLowering()
434 setOperationAction(ISD::ConstantPool , MVT::i32 , Custom); in X86TargetLowering()
435 setOperationAction(ISD::JumpTable , MVT::i32 , Custom); in X86TargetLowering()
436 setOperationAction(ISD::GlobalAddress , MVT::i32 , Custom); in X86TargetLowering()
437 setOperationAction(ISD::GlobalTLSAddress, MVT::i32 , Custom); in X86TargetLowering()
439 setOperationAction(ISD::GlobalTLSAddress, MVT::i64, Custom); in X86TargetLowering()
440 setOperationAction(ISD::ExternalSymbol , MVT::i32 , Custom); in X86TargetLowering()
441 setOperationAction(ISD::BlockAddress , MVT::i32 , Custom); in X86TargetLowering()
443 setOperationAction(ISD::ConstantPool , MVT::i64 , Custom); in X86TargetLowering()
444 setOperationAction(ISD::JumpTable , MVT::i64 , Custom); in X86TargetLowering()
445 setOperationAction(ISD::GlobalAddress , MVT::i64 , Custom); in X86TargetLowering()
446 setOperationAction(ISD::ExternalSymbol, MVT::i64 , Custom); in X86TargetLowering()
447 setOperationAction(ISD::BlockAddress , MVT::i64 , Custom); in X86TargetLowering()
450 setOperationAction(ISD::SHL_PARTS , MVT::i32 , Custom); in X86TargetLowering()
451 setOperationAction(ISD::SRA_PARTS , MVT::i32 , Custom); in X86TargetLowering()
452 setOperationAction(ISD::SRL_PARTS , MVT::i32 , Custom); in X86TargetLowering()
454 setOperationAction(ISD::SHL_PARTS , MVT::i64 , Custom); in X86TargetLowering()
455 setOperationAction(ISD::SRA_PARTS , MVT::i64 , Custom); in X86TargetLowering()
456 setOperationAction(ISD::SRL_PARTS , MVT::i64 , Custom); in X86TargetLowering()
460 setOperationAction(ISD::PREFETCH , MVT::Other, Legal); in X86TargetLowering()
462 setOperationAction(ISD::ATOMIC_FENCE , MVT::Other, Custom); in X86TargetLowering()
466 MVT VT = IntVTs[i]; in X86TargetLowering()
473 setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, MVT::i128, Custom); in X86TargetLowering()
479 setOperationAction(ISD::EH_LABEL, MVT::Other, Expand); in X86TargetLowering()
489 setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i32, Custom); in X86TargetLowering()
490 setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i64, Custom); in X86TargetLowering()
492 setOperationAction(ISD::INIT_TRAMPOLINE, MVT::Other, Custom); in X86TargetLowering()
493 setOperationAction(ISD::ADJUST_TRAMPOLINE, MVT::Other, Custom); in X86TargetLowering()
495 setOperationAction(ISD::TRAP, MVT::Other, Legal); in X86TargetLowering()
496 setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal); in X86TargetLowering()
499 setOperationAction(ISD::VASTART , MVT::Other, Custom); in X86TargetLowering()
500 setOperationAction(ISD::VAEND , MVT::Other, Expand); in X86TargetLowering()
503 setOperationAction(ISD::VAARG , MVT::Other, Custom); in X86TargetLowering()
504 setOperationAction(ISD::VACOPY , MVT::Other, Custom); in X86TargetLowering()
507 setOperationAction(ISD::VAARG , MVT::Other, Expand); in X86TargetLowering()
508 setOperationAction(ISD::VACOPY , MVT::Other, Expand); in X86TargetLowering()
511 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand); in X86TargetLowering()
512 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand); in X86TargetLowering()
519 addRegisterClass(MVT::f32, &X86::FR32RegClass); in X86TargetLowering()
520 addRegisterClass(MVT::f64, &X86::FR64RegClass); in X86TargetLowering()
523 setOperationAction(ISD::FABS , MVT::f64, Custom); in X86TargetLowering()
524 setOperationAction(ISD::FABS , MVT::f32, Custom); in X86TargetLowering()
527 setOperationAction(ISD::FNEG , MVT::f64, Custom); in X86TargetLowering()
528 setOperationAction(ISD::FNEG , MVT::f32, Custom); in X86TargetLowering()
531 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom); in X86TargetLowering()
532 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom); in X86TargetLowering()
535 setOperationAction(ISD::FGETSIGN, MVT::i64, Custom); in X86TargetLowering()
536 setOperationAction(ISD::FGETSIGN, MVT::i32, Custom); in X86TargetLowering()
539 setOperationAction(ISD::FSIN , MVT::f64, Expand); in X86TargetLowering()
540 setOperationAction(ISD::FCOS , MVT::f64, Expand); in X86TargetLowering()
541 setOperationAction(ISD::FSINCOS, MVT::f64, Expand); in X86TargetLowering()
542 setOperationAction(ISD::FSIN , MVT::f32, Expand); in X86TargetLowering()
543 setOperationAction(ISD::FCOS , MVT::f32, Expand); in X86TargetLowering()
544 setOperationAction(ISD::FSINCOS, MVT::f32, Expand); in X86TargetLowering()
553 addRegisterClass(MVT::f32, &X86::FR32RegClass); in X86TargetLowering()
554 addRegisterClass(MVT::f64, &X86::RFP64RegClass); in X86TargetLowering()
557 setOperationAction(ISD::FABS , MVT::f32, Custom); in X86TargetLowering()
560 setOperationAction(ISD::FNEG , MVT::f32, Custom); in X86TargetLowering()
562 setOperationAction(ISD::UNDEF, MVT::f64, Expand); in X86TargetLowering()
565 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand); in X86TargetLowering()
566 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom); in X86TargetLowering()
569 setOperationAction(ISD::FSIN , MVT::f32, Expand); in X86TargetLowering()
570 setOperationAction(ISD::FCOS , MVT::f32, Expand); in X86TargetLowering()
571 setOperationAction(ISD::FSINCOS, MVT::f32, Expand); in X86TargetLowering()
581 setOperationAction(ISD::FSIN , MVT::f64, Expand); in X86TargetLowering()
582 setOperationAction(ISD::FCOS , MVT::f64, Expand); in X86TargetLowering()
583 setOperationAction(ISD::FSINCOS, MVT::f64, Expand); in X86TargetLowering()
588 addRegisterClass(MVT::f64, &X86::RFP64RegClass); in X86TargetLowering()
589 addRegisterClass(MVT::f32, &X86::RFP32RegClass); in X86TargetLowering()
591 setOperationAction(ISD::UNDEF, MVT::f64, Expand); in X86TargetLowering()
592 setOperationAction(ISD::UNDEF, MVT::f32, Expand); in X86TargetLowering()
593 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand); in X86TargetLowering()
594 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand); in X86TargetLowering()
597 setOperationAction(ISD::FSIN , MVT::f64, Expand); in X86TargetLowering()
598 setOperationAction(ISD::FSIN , MVT::f32, Expand); in X86TargetLowering()
599 setOperationAction(ISD::FCOS , MVT::f64, Expand); in X86TargetLowering()
600 setOperationAction(ISD::FCOS , MVT::f32, Expand); in X86TargetLowering()
601 setOperationAction(ISD::FSINCOS, MVT::f64, Expand); in X86TargetLowering()
602 setOperationAction(ISD::FSINCOS, MVT::f32, Expand); in X86TargetLowering()
615 setOperationAction(ISD::FMA, MVT::f64, Expand); in X86TargetLowering()
616 setOperationAction(ISD::FMA, MVT::f32, Expand); in X86TargetLowering()
620 addRegisterClass(MVT::f80, &X86::RFP80RegClass); in X86TargetLowering()
621 setOperationAction(ISD::UNDEF, MVT::f80, Expand); in X86TargetLowering()
622 setOperationAction(ISD::FCOPYSIGN, MVT::f80, Expand); in X86TargetLowering()
639 setOperationAction(ISD::FSIN , MVT::f80, Expand); in X86TargetLowering()
640 setOperationAction(ISD::FCOS , MVT::f80, Expand); in X86TargetLowering()
641 setOperationAction(ISD::FSINCOS, MVT::f80, Expand); in X86TargetLowering()
644 setOperationAction(ISD::FFLOOR, MVT::f80, Expand); in X86TargetLowering()
645 setOperationAction(ISD::FCEIL, MVT::f80, Expand); in X86TargetLowering()
646 setOperationAction(ISD::FTRUNC, MVT::f80, Expand); in X86TargetLowering()
647 setOperationAction(ISD::FRINT, MVT::f80, Expand); in X86TargetLowering()
648 setOperationAction(ISD::FNEARBYINT, MVT::f80, Expand); in X86TargetLowering()
649 setOperationAction(ISD::FMA, MVT::f80, Expand); in X86TargetLowering()
653 setOperationAction(ISD::FPOW , MVT::f32 , Expand); in X86TargetLowering()
654 setOperationAction(ISD::FPOW , MVT::f64 , Expand); in X86TargetLowering()
655 setOperationAction(ISD::FPOW , MVT::f80 , Expand); in X86TargetLowering()
657 setOperationAction(ISD::FLOG, MVT::f80, Expand); in X86TargetLowering()
658 setOperationAction(ISD::FLOG2, MVT::f80, Expand); in X86TargetLowering()
659 setOperationAction(ISD::FLOG10, MVT::f80, Expand); in X86TargetLowering()
660 setOperationAction(ISD::FEXP, MVT::f80, Expand); in X86TargetLowering()
661 setOperationAction(ISD::FEXP2, MVT::f80, Expand); in X86TargetLowering()
662 setOperationAction(ISD::FMINNUM, MVT::f80, Expand); in X86TargetLowering()
663 setOperationAction(ISD::FMAXNUM, MVT::f80, Expand); in X86TargetLowering()
668 for (MVT VT : MVT::vector_valuetypes()) { in X86TargetLowering()
737 for (MVT InnerVT : MVT::vector_valuetypes()) { in X86TargetLowering()
747 if (VT.getVectorElementType() == MVT::i1) in X86TargetLowering()
752 if (VT.getVectorElementType() == MVT::f16) in X86TargetLowering()
760 addRegisterClass(MVT::x86mmx, &X86::VR64RegClass); in X86TargetLowering()
766 for (MVT MMXTy : {MVT::v8i8, MVT::v4i16, MVT::v2i32, MVT::v1i64}) { in X86TargetLowering()
775 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v1i64, Expand); in X86TargetLowering()
778 addRegisterClass(MVT::v4f32, &X86::VR128RegClass); in X86TargetLowering()
780 setOperationAction(ISD::FADD, MVT::v4f32, Legal); in X86TargetLowering()
781 setOperationAction(ISD::FSUB, MVT::v4f32, Legal); in X86TargetLowering()
782 setOperationAction(ISD::FMUL, MVT::v4f32, Legal); in X86TargetLowering()
783 setOperationAction(ISD::FDIV, MVT::v4f32, Legal); in X86TargetLowering()
784 setOperationAction(ISD::FSQRT, MVT::v4f32, Legal); in X86TargetLowering()
785 setOperationAction(ISD::FNEG, MVT::v4f32, Custom); in X86TargetLowering()
786 setOperationAction(ISD::FABS, MVT::v4f32, Custom); in X86TargetLowering()
787 setOperationAction(ISD::LOAD, MVT::v4f32, Legal); in X86TargetLowering()
788 setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom); in X86TargetLowering()
789 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4f32, Custom); in X86TargetLowering()
790 setOperationAction(ISD::VSELECT, MVT::v4f32, Custom); in X86TargetLowering()
791 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom); in X86TargetLowering()
792 setOperationAction(ISD::SELECT, MVT::v4f32, Custom); in X86TargetLowering()
793 setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Custom); in X86TargetLowering()
797 addRegisterClass(MVT::v2f64, &X86::VR128RegClass); in X86TargetLowering()
801 addRegisterClass(MVT::v16i8, &X86::VR128RegClass); in X86TargetLowering()
802 addRegisterClass(MVT::v8i16, &X86::VR128RegClass); in X86TargetLowering()
803 addRegisterClass(MVT::v4i32, &X86::VR128RegClass); in X86TargetLowering()
804 addRegisterClass(MVT::v2i64, &X86::VR128RegClass); in X86TargetLowering()
806 setOperationAction(ISD::ADD, MVT::v16i8, Legal); in X86TargetLowering()
807 setOperationAction(ISD::ADD, MVT::v8i16, Legal); in X86TargetLowering()
808 setOperationAction(ISD::ADD, MVT::v4i32, Legal); in X86TargetLowering()
809 setOperationAction(ISD::ADD, MVT::v2i64, Legal); in X86TargetLowering()
810 setOperationAction(ISD::MUL, MVT::v4i32, Custom); in X86TargetLowering()
811 setOperationAction(ISD::MUL, MVT::v2i64, Custom); in X86TargetLowering()
812 setOperationAction(ISD::UMUL_LOHI, MVT::v4i32, Custom); in X86TargetLowering()
813 setOperationAction(ISD::SMUL_LOHI, MVT::v4i32, Custom); in X86TargetLowering()
814 setOperationAction(ISD::MULHU, MVT::v8i16, Legal); in X86TargetLowering()
815 setOperationAction(ISD::MULHS, MVT::v8i16, Legal); in X86TargetLowering()
816 setOperationAction(ISD::SUB, MVT::v16i8, Legal); in X86TargetLowering()
817 setOperationAction(ISD::SUB, MVT::v8i16, Legal); in X86TargetLowering()
818 setOperationAction(ISD::SUB, MVT::v4i32, Legal); in X86TargetLowering()
819 setOperationAction(ISD::SUB, MVT::v2i64, Legal); in X86TargetLowering()
820 setOperationAction(ISD::MUL, MVT::v8i16, Legal); in X86TargetLowering()
821 setOperationAction(ISD::FADD, MVT::v2f64, Legal); in X86TargetLowering()
822 setOperationAction(ISD::FSUB, MVT::v2f64, Legal); in X86TargetLowering()
823 setOperationAction(ISD::FMUL, MVT::v2f64, Legal); in X86TargetLowering()
824 setOperationAction(ISD::FDIV, MVT::v2f64, Legal); in X86TargetLowering()
825 setOperationAction(ISD::FSQRT, MVT::v2f64, Legal); in X86TargetLowering()
826 setOperationAction(ISD::FNEG, MVT::v2f64, Custom); in X86TargetLowering()
827 setOperationAction(ISD::FABS, MVT::v2f64, Custom); in X86TargetLowering()
829 setOperationAction(ISD::SETCC, MVT::v2i64, Custom); in X86TargetLowering()
830 setOperationAction(ISD::SETCC, MVT::v16i8, Custom); in X86TargetLowering()
831 setOperationAction(ISD::SETCC, MVT::v8i16, Custom); in X86TargetLowering()
832 setOperationAction(ISD::SETCC, MVT::v4i32, Custom); in X86TargetLowering()
834 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v16i8, Custom); in X86TargetLowering()
835 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8i16, Custom); in X86TargetLowering()
836 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom); in X86TargetLowering()
837 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom); in X86TargetLowering()
838 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom); in X86TargetLowering()
844 setOperationAction(ISD::CTPOP, MVT::v4i32, Custom); in X86TargetLowering()
845 setOperationAction(ISD::CTPOP, MVT::v2i64, Custom); in X86TargetLowering()
849 for (int i = MVT::v16i8; i != MVT::v2i64; ++i) { in X86TargetLowering()
850 MVT VT = (MVT::SimpleValueType)i; in X86TargetLowering()
867 for (MVT VT : MVT::integer_vector_valuetypes()) { in X86TargetLowering()
868 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v4i8, Custom); in X86TargetLowering()
869 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v4i16, Custom); in X86TargetLowering()
870 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v8i8, Custom); in X86TargetLowering()
871 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2i8, Custom); in X86TargetLowering()
872 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2i16, Custom); in X86TargetLowering()
873 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2i32, Custom); in X86TargetLowering()
874 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v4i8, Custom); in X86TargetLowering()
875 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v4i16, Custom); in X86TargetLowering()
876 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v8i8, Custom); in X86TargetLowering()
879 setOperationAction(ISD::BUILD_VECTOR, MVT::v2f64, Custom); in X86TargetLowering()
880 setOperationAction(ISD::BUILD_VECTOR, MVT::v2i64, Custom); in X86TargetLowering()
881 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2f64, Custom); in X86TargetLowering()
882 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2i64, Custom); in X86TargetLowering()
883 setOperationAction(ISD::VSELECT, MVT::v2f64, Custom); in X86TargetLowering()
884 setOperationAction(ISD::VSELECT, MVT::v2i64, Custom); in X86TargetLowering()
885 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2f64, Custom); in X86TargetLowering()
886 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f64, Custom); in X86TargetLowering()
889 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i64, Custom); in X86TargetLowering()
890 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i64, Custom); in X86TargetLowering()
894 for (int i = MVT::v16i8; i != MVT::v2i64; ++i) { in X86TargetLowering()
895 MVT VT = (MVT::SimpleValueType)i; in X86TargetLowering()
902 AddPromotedToType (ISD::AND, VT, MVT::v2i64); in X86TargetLowering()
904 AddPromotedToType (ISD::OR, VT, MVT::v2i64); in X86TargetLowering()
906 AddPromotedToType (ISD::XOR, VT, MVT::v2i64); in X86TargetLowering()
908 AddPromotedToType (ISD::LOAD, VT, MVT::v2i64); in X86TargetLowering()
910 AddPromotedToType (ISD::SELECT, VT, MVT::v2i64); in X86TargetLowering()
914 setOperationAction(ISD::LOAD, MVT::v2f64, Legal); in X86TargetLowering()
915 setOperationAction(ISD::LOAD, MVT::v2i64, Legal); in X86TargetLowering()
916 setOperationAction(ISD::SELECT, MVT::v2f64, Custom); in X86TargetLowering()
917 setOperationAction(ISD::SELECT, MVT::v2i64, Custom); in X86TargetLowering()
919 setOperationAction(ISD::FP_TO_SINT, MVT::v4i32, Legal); in X86TargetLowering()
920 setOperationAction(ISD::SINT_TO_FP, MVT::v4i32, Legal); in X86TargetLowering()
922 setOperationAction(ISD::UINT_TO_FP, MVT::v4i8, Custom); in X86TargetLowering()
923 setOperationAction(ISD::UINT_TO_FP, MVT::v4i16, Custom); in X86TargetLowering()
927 setOperationAction(ISD::UINT_TO_FP, MVT::v2f32, Custom); in X86TargetLowering()
929 setOperationAction(ISD::FP_EXTEND, MVT::v2f32, Custom); in X86TargetLowering()
930 setOperationAction(ISD::FP_ROUND, MVT::v2f32, Custom); in X86TargetLowering()
932 for (MVT VT : MVT::fp_vector_valuetypes()) in X86TargetLowering()
933 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2f32, Legal); in X86TargetLowering()
935 setOperationAction(ISD::BITCAST, MVT::v2i32, Custom); in X86TargetLowering()
936 setOperationAction(ISD::BITCAST, MVT::v4i16, Custom); in X86TargetLowering()
937 setOperationAction(ISD::BITCAST, MVT::v8i8, Custom); in X86TargetLowering()
941 for (MVT RoundedTy : {MVT::f32, MVT::f64, MVT::v4f32, MVT::v2f64}) { in X86TargetLowering()
950 setOperationAction(ISD::MUL, MVT::v4i32, Legal); in X86TargetLowering()
954 setOperationAction(ISD::VSELECT, MVT::v16i8, Legal); in X86TargetLowering()
958 for (MVT VT : MVT::integer_vector_valuetypes()) { in X86TargetLowering()
959 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v2i8, Custom); in X86TargetLowering()
960 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v2i16, Custom); in X86TargetLowering()
961 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v2i32, Custom); in X86TargetLowering()
965 setLoadExtAction(ISD::SEXTLOAD, MVT::v8i16, MVT::v8i8, Legal); in X86TargetLowering()
966 setLoadExtAction(ISD::SEXTLOAD, MVT::v4i32, MVT::v4i8, Legal); in X86TargetLowering()
967 setLoadExtAction(ISD::SEXTLOAD, MVT::v2i64, MVT::v2i8, Legal); in X86TargetLowering()
968 setLoadExtAction(ISD::SEXTLOAD, MVT::v4i32, MVT::v4i16, Legal); in X86TargetLowering()
969 setLoadExtAction(ISD::SEXTLOAD, MVT::v2i64, MVT::v2i16, Legal); in X86TargetLowering()
970 setLoadExtAction(ISD::SEXTLOAD, MVT::v2i64, MVT::v2i32, Legal); in X86TargetLowering()
972 setLoadExtAction(ISD::ZEXTLOAD, MVT::v8i16, MVT::v8i8, Legal); in X86TargetLowering()
973 setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i32, MVT::v4i8, Legal); in X86TargetLowering()
974 setLoadExtAction(ISD::ZEXTLOAD, MVT::v2i64, MVT::v2i8, Legal); in X86TargetLowering()
975 setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i32, MVT::v4i16, Legal); in X86TargetLowering()
976 setLoadExtAction(ISD::ZEXTLOAD, MVT::v2i64, MVT::v2i16, Legal); in X86TargetLowering()
977 setLoadExtAction(ISD::ZEXTLOAD, MVT::v2i64, MVT::v2i32, Legal); in X86TargetLowering()
983 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16i8, Custom); in X86TargetLowering()
984 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom); in X86TargetLowering()
985 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom); in X86TargetLowering()
986 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom); in X86TargetLowering()
988 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v16i8, Custom); in X86TargetLowering()
989 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v8i16, Custom); in X86TargetLowering()
990 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4i32, Custom); in X86TargetLowering()
991 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom); in X86TargetLowering()
996 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i64, Custom); in X86TargetLowering()
997 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i64, Custom); in X86TargetLowering()
1002 setOperationAction(ISD::SRL, MVT::v8i16, Custom); in X86TargetLowering()
1003 setOperationAction(ISD::SRL, MVT::v16i8, Custom); in X86TargetLowering()
1005 setOperationAction(ISD::SHL, MVT::v8i16, Custom); in X86TargetLowering()
1006 setOperationAction(ISD::SHL, MVT::v16i8, Custom); in X86TargetLowering()
1008 setOperationAction(ISD::SRA, MVT::v8i16, Custom); in X86TargetLowering()
1009 setOperationAction(ISD::SRA, MVT::v16i8, Custom); in X86TargetLowering()
1013 setOperationAction(ISD::SRL, MVT::v2i64, Custom); in X86TargetLowering()
1014 setOperationAction(ISD::SRL, MVT::v4i32, Custom); in X86TargetLowering()
1016 setOperationAction(ISD::SHL, MVT::v2i64, Custom); in X86TargetLowering()
1017 setOperationAction(ISD::SHL, MVT::v4i32, Custom); in X86TargetLowering()
1019 setOperationAction(ISD::SRA, MVT::v4i32, Custom); in X86TargetLowering()
1023 addRegisterClass(MVT::v32i8, &X86::VR256RegClass); in X86TargetLowering()
1024 addRegisterClass(MVT::v16i16, &X86::VR256RegClass); in X86TargetLowering()
1025 addRegisterClass(MVT::v8i32, &X86::VR256RegClass); in X86TargetLowering()
1026 addRegisterClass(MVT::v8f32, &X86::VR256RegClass); in X86TargetLowering()
1027 addRegisterClass(MVT::v4i64, &X86::VR256RegClass); in X86TargetLowering()
1028 addRegisterClass(MVT::v4f64, &X86::VR256RegClass); in X86TargetLowering()
1030 setOperationAction(ISD::LOAD, MVT::v8f32, Legal); in X86TargetLowering()
1031 setOperationAction(ISD::LOAD, MVT::v4f64, Legal); in X86TargetLowering()
1032 setOperationAction(ISD::LOAD, MVT::v4i64, Legal); in X86TargetLowering()
1034 setOperationAction(ISD::FADD, MVT::v8f32, Legal); in X86TargetLowering()
1035 setOperationAction(ISD::FSUB, MVT::v8f32, Legal); in X86TargetLowering()
1036 setOperationAction(ISD::FMUL, MVT::v8f32, Legal); in X86TargetLowering()
1037 setOperationAction(ISD::FDIV, MVT::v8f32, Legal); in X86TargetLowering()
1038 setOperationAction(ISD::FSQRT, MVT::v8f32, Legal); in X86TargetLowering()
1039 setOperationAction(ISD::FFLOOR, MVT::v8f32, Legal); in X86TargetLowering()
1040 setOperationAction(ISD::FCEIL, MVT::v8f32, Legal); in X86TargetLowering()
1041 setOperationAction(ISD::FTRUNC, MVT::v8f32, Legal); in X86TargetLowering()
1042 setOperationAction(ISD::FRINT, MVT::v8f32, Legal); in X86TargetLowering()
1043 setOperationAction(ISD::FNEARBYINT, MVT::v8f32, Legal); in X86TargetLowering()
1044 setOperationAction(ISD::FNEG, MVT::v8f32, Custom); in X86TargetLowering()
1045 setOperationAction(ISD::FABS, MVT::v8f32, Custom); in X86TargetLowering()
1047 setOperationAction(ISD::FADD, MVT::v4f64, Legal); in X86TargetLowering()
1048 setOperationAction(ISD::FSUB, MVT::v4f64, Legal); in X86TargetLowering()
1049 setOperationAction(ISD::FMUL, MVT::v4f64, Legal); in X86TargetLowering()
1050 setOperationAction(ISD::FDIV, MVT::v4f64, Legal); in X86TargetLowering()
1051 setOperationAction(ISD::FSQRT, MVT::v4f64, Legal); in X86TargetLowering()
1052 setOperationAction(ISD::FFLOOR, MVT::v4f64, Legal); in X86TargetLowering()
1053 setOperationAction(ISD::FCEIL, MVT::v4f64, Legal); in X86TargetLowering()
1054 setOperationAction(ISD::FTRUNC, MVT::v4f64, Legal); in X86TargetLowering()
1055 setOperationAction(ISD::FRINT, MVT::v4f64, Legal); in X86TargetLowering()
1056 setOperationAction(ISD::FNEARBYINT, MVT::v4f64, Legal); in X86TargetLowering()
1057 setOperationAction(ISD::FNEG, MVT::v4f64, Custom); in X86TargetLowering()
1058 setOperationAction(ISD::FABS, MVT::v4f64, Custom); in X86TargetLowering()
1062 setOperationAction(ISD::FP_TO_SINT, MVT::v8i16, Promote); in X86TargetLowering()
1063 setOperationAction(ISD::FP_TO_UINT, MVT::v8i16, Promote); in X86TargetLowering()
1064 setOperationAction(ISD::FP_TO_SINT, MVT::v8i32, Legal); in X86TargetLowering()
1066 setOperationAction(ISD::SINT_TO_FP, MVT::v8i16, Promote); in X86TargetLowering()
1067 setOperationAction(ISD::SINT_TO_FP, MVT::v8i32, Legal); in X86TargetLowering()
1068 setOperationAction(ISD::FP_ROUND, MVT::v4f32, Legal); in X86TargetLowering()
1070 setOperationAction(ISD::UINT_TO_FP, MVT::v8i8, Custom); in X86TargetLowering()
1071 setOperationAction(ISD::UINT_TO_FP, MVT::v8i16, Custom); in X86TargetLowering()
1073 for (MVT VT : MVT::fp_vector_valuetypes()) in X86TargetLowering()
1074 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v4f32, Legal); in X86TargetLowering()
1076 setOperationAction(ISD::SRL, MVT::v16i16, Custom); in X86TargetLowering()
1077 setOperationAction(ISD::SRL, MVT::v32i8, Custom); in X86TargetLowering()
1079 setOperationAction(ISD::SHL, MVT::v16i16, Custom); in X86TargetLowering()
1080 setOperationAction(ISD::SHL, MVT::v32i8, Custom); in X86TargetLowering()
1082 setOperationAction(ISD::SRA, MVT::v16i16, Custom); in X86TargetLowering()
1083 setOperationAction(ISD::SRA, MVT::v32i8, Custom); in X86TargetLowering()
1085 setOperationAction(ISD::SETCC, MVT::v32i8, Custom); in X86TargetLowering()
1086 setOperationAction(ISD::SETCC, MVT::v16i16, Custom); in X86TargetLowering()
1087 setOperationAction(ISD::SETCC, MVT::v8i32, Custom); in X86TargetLowering()
1088 setOperationAction(ISD::SETCC, MVT::v4i64, Custom); in X86TargetLowering()
1090 setOperationAction(ISD::SELECT, MVT::v4f64, Custom); in X86TargetLowering()
1091 setOperationAction(ISD::SELECT, MVT::v4i64, Custom); in X86TargetLowering()
1092 setOperationAction(ISD::SELECT, MVT::v8f32, Custom); in X86TargetLowering()
1094 setOperationAction(ISD::SIGN_EXTEND, MVT::v4i64, Custom); in X86TargetLowering()
1095 setOperationAction(ISD::SIGN_EXTEND, MVT::v8i32, Custom); in X86TargetLowering()
1096 setOperationAction(ISD::SIGN_EXTEND, MVT::v16i16, Custom); in X86TargetLowering()
1097 setOperationAction(ISD::ZERO_EXTEND, MVT::v4i64, Custom); in X86TargetLowering()
1098 setOperationAction(ISD::ZERO_EXTEND, MVT::v8i32, Custom); in X86TargetLowering()
1099 setOperationAction(ISD::ZERO_EXTEND, MVT::v16i16, Custom); in X86TargetLowering()
1100 setOperationAction(ISD::ANY_EXTEND, MVT::v4i64, Custom); in X86TargetLowering()
1101 setOperationAction(ISD::ANY_EXTEND, MVT::v8i32, Custom); in X86TargetLowering()
1102 setOperationAction(ISD::ANY_EXTEND, MVT::v16i16, Custom); in X86TargetLowering()
1103 setOperationAction(ISD::TRUNCATE, MVT::v16i8, Custom); in X86TargetLowering()
1104 setOperationAction(ISD::TRUNCATE, MVT::v8i16, Custom); in X86TargetLowering()
1105 setOperationAction(ISD::TRUNCATE, MVT::v4i32, Custom); in X86TargetLowering()
1108 setOperationAction(ISD::FMA, MVT::v8f32, Legal); in X86TargetLowering()
1109 setOperationAction(ISD::FMA, MVT::v4f64, Legal); in X86TargetLowering()
1110 setOperationAction(ISD::FMA, MVT::v4f32, Legal); in X86TargetLowering()
1111 setOperationAction(ISD::FMA, MVT::v2f64, Legal); in X86TargetLowering()
1112 setOperationAction(ISD::FMA, MVT::f32, Legal); in X86TargetLowering()
1113 setOperationAction(ISD::FMA, MVT::f64, Legal); in X86TargetLowering()
1117 setOperationAction(ISD::ADD, MVT::v4i64, Legal); in X86TargetLowering()
1118 setOperationAction(ISD::ADD, MVT::v8i32, Legal); in X86TargetLowering()
1119 setOperationAction(ISD::ADD, MVT::v16i16, Legal); in X86TargetLowering()
1120 setOperationAction(ISD::ADD, MVT::v32i8, Legal); in X86TargetLowering()
1122 setOperationAction(ISD::SUB, MVT::v4i64, Legal); in X86TargetLowering()
1123 setOperationAction(ISD::SUB, MVT::v8i32, Legal); in X86TargetLowering()
1124 setOperationAction(ISD::SUB, MVT::v16i16, Legal); in X86TargetLowering()
1125 setOperationAction(ISD::SUB, MVT::v32i8, Legal); in X86TargetLowering()
1127 setOperationAction(ISD::MUL, MVT::v4i64, Custom); in X86TargetLowering()
1128 setOperationAction(ISD::MUL, MVT::v8i32, Legal); in X86TargetLowering()
1129 setOperationAction(ISD::MUL, MVT::v16i16, Legal); in X86TargetLowering()
1132 setOperationAction(ISD::UMUL_LOHI, MVT::v8i32, Custom); in X86TargetLowering()
1133 setOperationAction(ISD::SMUL_LOHI, MVT::v8i32, Custom); in X86TargetLowering()
1134 setOperationAction(ISD::MULHU, MVT::v16i16, Legal); in X86TargetLowering()
1135 setOperationAction(ISD::MULHS, MVT::v16i16, Legal); in X86TargetLowering()
1139 setOperationAction(ISD::UINT_TO_FP, MVT::v8i32, Custom); in X86TargetLowering()
1146 setOperationAction(ISD::CTPOP, MVT::v4i64, Custom); in X86TargetLowering()
1149 setOperationAction(ISD::CTPOP, MVT::v8i32, Custom); in X86TargetLowering()
1152 setLoadExtAction(ISD::SEXTLOAD, MVT::v16i16, MVT::v16i8, Legal); in X86TargetLowering()
1153 setLoadExtAction(ISD::SEXTLOAD, MVT::v8i32, MVT::v8i8, Legal); in X86TargetLowering()
1154 setLoadExtAction(ISD::SEXTLOAD, MVT::v4i64, MVT::v4i8, Legal); in X86TargetLowering()
1155 setLoadExtAction(ISD::SEXTLOAD, MVT::v8i32, MVT::v8i16, Legal); in X86TargetLowering()
1156 setLoadExtAction(ISD::SEXTLOAD, MVT::v4i64, MVT::v4i16, Legal); in X86TargetLowering()
1157 setLoadExtAction(ISD::SEXTLOAD, MVT::v4i64, MVT::v4i32, Legal); in X86TargetLowering()
1159 setLoadExtAction(ISD::ZEXTLOAD, MVT::v16i16, MVT::v16i8, Legal); in X86TargetLowering()
1160 setLoadExtAction(ISD::ZEXTLOAD, MVT::v8i32, MVT::v8i8, Legal); in X86TargetLowering()
1161 setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i64, MVT::v4i8, Legal); in X86TargetLowering()
1162 setLoadExtAction(ISD::ZEXTLOAD, MVT::v8i32, MVT::v8i16, Legal); in X86TargetLowering()
1163 setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i64, MVT::v4i16, Legal); in X86TargetLowering()
1164 setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i64, MVT::v4i32, Legal); in X86TargetLowering()
1166 setOperationAction(ISD::ADD, MVT::v4i64, Custom); in X86TargetLowering()
1167 setOperationAction(ISD::ADD, MVT::v8i32, Custom); in X86TargetLowering()
1168 setOperationAction(ISD::ADD, MVT::v16i16, Custom); in X86TargetLowering()
1169 setOperationAction(ISD::ADD, MVT::v32i8, Custom); in X86TargetLowering()
1171 setOperationAction(ISD::SUB, MVT::v4i64, Custom); in X86TargetLowering()
1172 setOperationAction(ISD::SUB, MVT::v8i32, Custom); in X86TargetLowering()
1173 setOperationAction(ISD::SUB, MVT::v16i16, Custom); in X86TargetLowering()
1174 setOperationAction(ISD::SUB, MVT::v32i8, Custom); in X86TargetLowering()
1176 setOperationAction(ISD::MUL, MVT::v4i64, Custom); in X86TargetLowering()
1177 setOperationAction(ISD::MUL, MVT::v8i32, Custom); in X86TargetLowering()
1178 setOperationAction(ISD::MUL, MVT::v16i16, Custom); in X86TargetLowering()
1184 setOperationAction(ISD::SRL, MVT::v4i64, Custom); in X86TargetLowering()
1185 setOperationAction(ISD::SRL, MVT::v8i32, Custom); in X86TargetLowering()
1187 setOperationAction(ISD::SHL, MVT::v4i64, Custom); in X86TargetLowering()
1188 setOperationAction(ISD::SHL, MVT::v8i32, Custom); in X86TargetLowering()
1190 setOperationAction(ISD::SRA, MVT::v8i32, Custom); in X86TargetLowering()
1193 for (MVT VT : MVT::vector_valuetypes()) { in X86TargetLowering()
1218 setOperationAction(ISD::VSELECT, MVT::v32i8, Legal); in X86TargetLowering()
1222 for (int i = MVT::v32i8; i != MVT::v4i64; ++i) { in X86TargetLowering()
1223 MVT VT = (MVT::SimpleValueType)i; in X86TargetLowering()
1230 AddPromotedToType (ISD::AND, VT, MVT::v4i64); in X86TargetLowering()
1232 AddPromotedToType (ISD::OR, VT, MVT::v4i64); in X86TargetLowering()
1234 AddPromotedToType (ISD::XOR, VT, MVT::v4i64); in X86TargetLowering()
1236 AddPromotedToType (ISD::LOAD, VT, MVT::v4i64); in X86TargetLowering()
1238 AddPromotedToType (ISD::SELECT, VT, MVT::v4i64); in X86TargetLowering()
1243 addRegisterClass(MVT::v16i32, &X86::VR512RegClass); in X86TargetLowering()
1244 addRegisterClass(MVT::v16f32, &X86::VR512RegClass); in X86TargetLowering()
1245 addRegisterClass(MVT::v8i64, &X86::VR512RegClass); in X86TargetLowering()
1246 addRegisterClass(MVT::v8f64, &X86::VR512RegClass); in X86TargetLowering()
1248 addRegisterClass(MVT::i1, &X86::VK1RegClass); in X86TargetLowering()
1249 addRegisterClass(MVT::v8i1, &X86::VK8RegClass); in X86TargetLowering()
1250 addRegisterClass(MVT::v16i1, &X86::VK16RegClass); in X86TargetLowering()
1252 for (MVT VT : MVT::fp_vector_valuetypes()) in X86TargetLowering()
1253 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v8f32, Legal); in X86TargetLowering()
1255 setOperationAction(ISD::BR_CC, MVT::i1, Expand); in X86TargetLowering()
1256 setOperationAction(ISD::SETCC, MVT::i1, Custom); in X86TargetLowering()
1257 setOperationAction(ISD::XOR, MVT::i1, Legal); in X86TargetLowering()
1258 setOperationAction(ISD::OR, MVT::i1, Legal); in X86TargetLowering()
1259 setOperationAction(ISD::AND, MVT::i1, Legal); in X86TargetLowering()
1260 setOperationAction(ISD::LOAD, MVT::v16f32, Legal); in X86TargetLowering()
1261 setOperationAction(ISD::LOAD, MVT::v8f64, Legal); in X86TargetLowering()
1262 setOperationAction(ISD::LOAD, MVT::v8i64, Legal); in X86TargetLowering()
1263 setOperationAction(ISD::LOAD, MVT::v16i32, Legal); in X86TargetLowering()
1264 setOperationAction(ISD::LOAD, MVT::v16i1, Legal); in X86TargetLowering()
1266 setOperationAction(ISD::FADD, MVT::v16f32, Legal); in X86TargetLowering()
1267 setOperationAction(ISD::FSUB, MVT::v16f32, Legal); in X86TargetLowering()
1268 setOperationAction(ISD::FMUL, MVT::v16f32, Legal); in X86TargetLowering()
1269 setOperationAction(ISD::FDIV, MVT::v16f32, Legal); in X86TargetLowering()
1270 setOperationAction(ISD::FSQRT, MVT::v16f32, Legal); in X86TargetLowering()
1271 setOperationAction(ISD::FNEG, MVT::v16f32, Custom); in X86TargetLowering()
1273 setOperationAction(ISD::FADD, MVT::v8f64, Legal); in X86TargetLowering()
1274 setOperationAction(ISD::FSUB, MVT::v8f64, Legal); in X86TargetLowering()
1275 setOperationAction(ISD::FMUL, MVT::v8f64, Legal); in X86TargetLowering()
1276 setOperationAction(ISD::FDIV, MVT::v8f64, Legal); in X86TargetLowering()
1277 setOperationAction(ISD::FSQRT, MVT::v8f64, Legal); in X86TargetLowering()
1278 setOperationAction(ISD::FNEG, MVT::v8f64, Custom); in X86TargetLowering()
1279 setOperationAction(ISD::FMA, MVT::v8f64, Legal); in X86TargetLowering()
1280 setOperationAction(ISD::FMA, MVT::v16f32, Legal); in X86TargetLowering()
1282 setOperationAction(ISD::FP_TO_SINT, MVT::i32, Legal); in X86TargetLowering()
1283 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Legal); in X86TargetLowering()
1284 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Legal); in X86TargetLowering()
1285 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Legal); in X86TargetLowering()
1287 setOperationAction(ISD::FP_TO_UINT, MVT::i64, Legal); in X86TargetLowering()
1288 setOperationAction(ISD::FP_TO_SINT, MVT::i64, Legal); in X86TargetLowering()
1289 setOperationAction(ISD::SINT_TO_FP, MVT::i64, Legal); in X86TargetLowering()
1290 setOperationAction(ISD::UINT_TO_FP, MVT::i64, Legal); in X86TargetLowering()
1292 setOperationAction(ISD::FP_TO_SINT, MVT::v16i32, Legal); in X86TargetLowering()
1293 setOperationAction(ISD::FP_TO_UINT, MVT::v16i32, Legal); in X86TargetLowering()
1294 setOperationAction(ISD::FP_TO_UINT, MVT::v8i32, Legal); in X86TargetLowering()
1295 setOperationAction(ISD::FP_TO_UINT, MVT::v4i32, Legal); in X86TargetLowering()
1296 setOperationAction(ISD::SINT_TO_FP, MVT::v16i32, Legal); in X86TargetLowering()
1297 setOperationAction(ISD::SINT_TO_FP, MVT::v8i1, Custom); in X86TargetLowering()
1298 setOperationAction(ISD::SINT_TO_FP, MVT::v16i1, Custom); in X86TargetLowering()
1299 setOperationAction(ISD::SINT_TO_FP, MVT::v16i8, Promote); in X86TargetLowering()
1300 setOperationAction(ISD::SINT_TO_FP, MVT::v16i16, Promote); in X86TargetLowering()
1301 setOperationAction(ISD::UINT_TO_FP, MVT::v16i32, Legal); in X86TargetLowering()
1302 setOperationAction(ISD::UINT_TO_FP, MVT::v8i32, Legal); in X86TargetLowering()
1303 setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Legal); in X86TargetLowering()
1304 setOperationAction(ISD::FP_ROUND, MVT::v8f32, Legal); in X86TargetLowering()
1305 setOperationAction(ISD::FP_EXTEND, MVT::v8f32, Legal); in X86TargetLowering()
1307 setOperationAction(ISD::TRUNCATE, MVT::i1, Custom); in X86TargetLowering()
1308 setOperationAction(ISD::TRUNCATE, MVT::v16i8, Custom); in X86TargetLowering()
1309 setOperationAction(ISD::TRUNCATE, MVT::v8i32, Custom); in X86TargetLowering()
1310 setOperationAction(ISD::TRUNCATE, MVT::v8i1, Custom); in X86TargetLowering()
1311 setOperationAction(ISD::TRUNCATE, MVT::v16i1, Custom); in X86TargetLowering()
1312 setOperationAction(ISD::TRUNCATE, MVT::v16i16, Custom); in X86TargetLowering()
1313 setOperationAction(ISD::ZERO_EXTEND, MVT::v16i32, Custom); in X86TargetLowering()
1314 setOperationAction(ISD::ZERO_EXTEND, MVT::v8i64, Custom); in X86TargetLowering()
1315 setOperationAction(ISD::SIGN_EXTEND, MVT::v16i32, Custom); in X86TargetLowering()
1316 setOperationAction(ISD::SIGN_EXTEND, MVT::v8i64, Custom); in X86TargetLowering()
1317 setOperationAction(ISD::SIGN_EXTEND, MVT::v16i8, Custom); in X86TargetLowering()
1318 setOperationAction(ISD::SIGN_EXTEND, MVT::v8i16, Custom); in X86TargetLowering()
1319 setOperationAction(ISD::SIGN_EXTEND, MVT::v16i16, Custom); in X86TargetLowering()
1321 setOperationAction(ISD::FFLOOR, MVT::v16f32, Legal); in X86TargetLowering()
1322 setOperationAction(ISD::FFLOOR, MVT::v8f64, Legal); in X86TargetLowering()
1323 setOperationAction(ISD::FCEIL, MVT::v16f32, Legal); in X86TargetLowering()
1324 setOperationAction(ISD::FCEIL, MVT::v8f64, Legal); in X86TargetLowering()
1325 setOperationAction(ISD::FTRUNC, MVT::v16f32, Legal); in X86TargetLowering()
1326 setOperationAction(ISD::FTRUNC, MVT::v8f64, Legal); in X86TargetLowering()
1327 setOperationAction(ISD::FRINT, MVT::v16f32, Legal); in X86TargetLowering()
1328 setOperationAction(ISD::FRINT, MVT::v8f64, Legal); in X86TargetLowering()
1329 setOperationAction(ISD::FNEARBYINT, MVT::v16f32, Legal); in X86TargetLowering()
1330 setOperationAction(ISD::FNEARBYINT, MVT::v8f64, Legal); in X86TargetLowering()
1332 setOperationAction(ISD::CONCAT_VECTORS, MVT::v8f64, Custom); in X86TargetLowering()
1333 setOperationAction(ISD::CONCAT_VECTORS, MVT::v8i64, Custom); in X86TargetLowering()
1334 setOperationAction(ISD::CONCAT_VECTORS, MVT::v16f32, Custom); in X86TargetLowering()
1335 setOperationAction(ISD::CONCAT_VECTORS, MVT::v16i32, Custom); in X86TargetLowering()
1336 setOperationAction(ISD::CONCAT_VECTORS, MVT::v16i1, Legal); in X86TargetLowering()
1338 setOperationAction(ISD::SETCC, MVT::v16i1, Custom); in X86TargetLowering()
1339 setOperationAction(ISD::SETCC, MVT::v8i1, Custom); in X86TargetLowering()
1341 setOperationAction(ISD::MUL, MVT::v8i64, Custom); in X86TargetLowering()
1343 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v8i1, Custom); in X86TargetLowering()
1344 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v16i1, Custom); in X86TargetLowering()
1345 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16i1, Custom); in X86TargetLowering()
1346 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i1, Custom); in X86TargetLowering()
1347 setOperationAction(ISD::BUILD_VECTOR, MVT::v8i1, Custom); in X86TargetLowering()
1348 setOperationAction(ISD::BUILD_VECTOR, MVT::v16i1, Custom); in X86TargetLowering()
1349 setOperationAction(ISD::SELECT, MVT::v8f64, Custom); in X86TargetLowering()
1350 setOperationAction(ISD::SELECT, MVT::v8i64, Custom); in X86TargetLowering()
1351 setOperationAction(ISD::SELECT, MVT::v16f32, Custom); in X86TargetLowering()
1353 setOperationAction(ISD::ADD, MVT::v8i64, Legal); in X86TargetLowering()
1354 setOperationAction(ISD::ADD, MVT::v16i32, Legal); in X86TargetLowering()
1356 setOperationAction(ISD::SUB, MVT::v8i64, Legal); in X86TargetLowering()
1357 setOperationAction(ISD::SUB, MVT::v16i32, Legal); in X86TargetLowering()
1359 setOperationAction(ISD::MUL, MVT::v16i32, Legal); in X86TargetLowering()
1361 setOperationAction(ISD::SRL, MVT::v8i64, Custom); in X86TargetLowering()
1362 setOperationAction(ISD::SRL, MVT::v16i32, Custom); in X86TargetLowering()
1364 setOperationAction(ISD::SHL, MVT::v8i64, Custom); in X86TargetLowering()
1365 setOperationAction(ISD::SHL, MVT::v16i32, Custom); in X86TargetLowering()
1367 setOperationAction(ISD::SRA, MVT::v8i64, Custom); in X86TargetLowering()
1368 setOperationAction(ISD::SRA, MVT::v16i32, Custom); in X86TargetLowering()
1370 setOperationAction(ISD::AND, MVT::v8i64, Legal); in X86TargetLowering()
1371 setOperationAction(ISD::OR, MVT::v8i64, Legal); in X86TargetLowering()
1372 setOperationAction(ISD::XOR, MVT::v8i64, Legal); in X86TargetLowering()
1373 setOperationAction(ISD::AND, MVT::v16i32, Legal); in X86TargetLowering()
1374 setOperationAction(ISD::OR, MVT::v16i32, Legal); in X86TargetLowering()
1375 setOperationAction(ISD::XOR, MVT::v16i32, Legal); in X86TargetLowering()
1378 setOperationAction(ISD::CTLZ, MVT::v8i64, Legal); in X86TargetLowering()
1379 setOperationAction(ISD::CTLZ, MVT::v16i32, Legal); in X86TargetLowering()
1383 for (MVT VT : MVT::vector_valuetypes()) { in X86TargetLowering()
1390 if (VT.getVectorElementType() == MVT::i1) in X86TargetLowering()
1409 for (int i = MVT::v32i8; i != MVT::v8i64; ++i) { in X86TargetLowering()
1410 MVT VT = (MVT::SimpleValueType)i; in X86TargetLowering()
1417 AddPromotedToType (ISD::SELECT, VT, MVT::v8i64); in X86TargetLowering()
1422 addRegisterClass(MVT::v32i16, &X86::VR512RegClass); in X86TargetLowering()
1423 addRegisterClass(MVT::v64i8, &X86::VR512RegClass); in X86TargetLowering()
1425 addRegisterClass(MVT::v32i1, &X86::VK32RegClass); in X86TargetLowering()
1426 addRegisterClass(MVT::v64i1, &X86::VK64RegClass); in X86TargetLowering()
1428 setOperationAction(ISD::LOAD, MVT::v32i16, Legal); in X86TargetLowering()
1429 setOperationAction(ISD::LOAD, MVT::v64i8, Legal); in X86TargetLowering()
1430 setOperationAction(ISD::SETCC, MVT::v32i1, Custom); in X86TargetLowering()
1431 setOperationAction(ISD::SETCC, MVT::v64i1, Custom); in X86TargetLowering()
1432 setOperationAction(ISD::ADD, MVT::v32i16, Legal); in X86TargetLowering()
1433 setOperationAction(ISD::ADD, MVT::v64i8, Legal); in X86TargetLowering()
1434 setOperationAction(ISD::SUB, MVT::v32i16, Legal); in X86TargetLowering()
1435 setOperationAction(ISD::SUB, MVT::v64i8, Legal); in X86TargetLowering()
1436 setOperationAction(ISD::MUL, MVT::v32i16, Legal); in X86TargetLowering()
1437 setOperationAction(ISD::CONCAT_VECTORS, MVT::v32i1, Custom); in X86TargetLowering()
1438 setOperationAction(ISD::CONCAT_VECTORS, MVT::v64i1, Custom); in X86TargetLowering()
1439 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v32i1, Custom); in X86TargetLowering()
1440 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v64i1, Custom); in X86TargetLowering()
1442 for (int i = MVT::v32i8; i != MVT::v8i64; ++i) { in X86TargetLowering()
1443 const MVT VT = (MVT::SimpleValueType)i; in X86TargetLowering()
1459 addRegisterClass(MVT::v4i1, &X86::VK4RegClass); in X86TargetLowering()
1460 addRegisterClass(MVT::v2i1, &X86::VK2RegClass); in X86TargetLowering()
1462 setOperationAction(ISD::SETCC, MVT::v4i1, Custom); in X86TargetLowering()
1463 setOperationAction(ISD::SETCC, MVT::v2i1, Custom); in X86TargetLowering()
1464 setOperationAction(ISD::CONCAT_VECTORS, MVT::v4i1, Custom); in X86TargetLowering()
1465 setOperationAction(ISD::CONCAT_VECTORS, MVT::v8i1, Custom); in X86TargetLowering()
1466 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v8i1, Custom); in X86TargetLowering()
1467 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v4i1, Custom); in X86TargetLowering()
1469 setOperationAction(ISD::AND, MVT::v8i32, Legal); in X86TargetLowering()
1470 setOperationAction(ISD::OR, MVT::v8i32, Legal); in X86TargetLowering()
1471 setOperationAction(ISD::XOR, MVT::v8i32, Legal); in X86TargetLowering()
1472 setOperationAction(ISD::AND, MVT::v4i32, Legal); in X86TargetLowering()
1473 setOperationAction(ISD::OR, MVT::v4i32, Legal); in X86TargetLowering()
1474 setOperationAction(ISD::XOR, MVT::v4i32, Legal); in X86TargetLowering()
1478 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); in X86TargetLowering()
1479 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom); in X86TargetLowering()
1480 setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom); in X86TargetLowering()
1482 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i64, Custom); in X86TargetLowering()
1492 MVT VT = IntVTs[i]; in X86TargetLowering()
1516 setOperationAction(ISD::FSINCOS, MVT::f64, Custom); in X86TargetLowering()
1517 setOperationAction(ISD::FSINCOS, MVT::f32, Custom); in X86TargetLowering()
1522 setOperationAction(ISD::SDIV, MVT::i128, Custom); in X86TargetLowering()
1523 setOperationAction(ISD::UDIV, MVT::i128, Custom); in X86TargetLowering()
1524 setOperationAction(ISD::SREM, MVT::i128, Custom); in X86TargetLowering()
1525 setOperationAction(ISD::UREM, MVT::i128, Custom); in X86TargetLowering()
1526 setOperationAction(ISD::SDIVREM, MVT::i128, Custom); in X86TargetLowering()
1527 setOperationAction(ISD::UDIVREM, MVT::i128, Custom); in X86TargetLowering()
1591 VT.getVectorElementType().getSimpleVT() != MVT::i1) in getPreferredVectorAction()
1599 return Subtarget->hasAVX512() ? MVT::i1: MVT::i8; in getSetCCResultType()
1605 if (EltVT == MVT::i32 || EltVT == MVT::i64 || in getSetCCResultType()
1606 EltVT == MVT::f32 || EltVT == MVT::f64) in getSetCCResultType()
1608 case 8: return MVT::v8i1; in getSetCCResultType()
1609 case 16: return MVT::v16i1; in getSetCCResultType()
1612 if (EltVT == MVT::i8 || EltVT == MVT::i16) in getSetCCResultType()
1614 case 32: return MVT::v32i1; in getSetCCResultType()
1615 case 64: return MVT::v64i1; in getSetCCResultType()
1621 if (EltVT == MVT::i32 || EltVT == MVT::i64 || in getSetCCResultType()
1622 EltVT == MVT::f32 || EltVT == MVT::f64) in getSetCCResultType()
1624 case 2: return MVT::v2i1; in getSetCCResultType()
1625 case 4: return MVT::v4i1; in getSetCCResultType()
1626 case 8: return MVT::v8i1; in getSetCCResultType()
1629 if (EltVT == MVT::i8 || EltVT == MVT::i16) in getSetCCResultType()
1631 case 8: return MVT::v8i1; in getSetCCResultType()
1632 case 16: return MVT::v16i1; in getSetCCResultType()
1633 case 32: return MVT::v32i1; in getSetCCResultType()
1710 return MVT::v8i32; in getOptimalMemOpType()
1712 return MVT::v8f32; in getOptimalMemOpType()
1715 return MVT::v4i32; in getOptimalMemOpType()
1717 return MVT::v4f32; in getOptimalMemOpType()
1723 return MVT::f64; in getOptimalMemOpType()
1727 return MVT::i64; in getOptimalMemOpType()
1728 return MVT::i32; in getOptimalMemOpType()
1731 bool X86TargetLowering::isSafeMemOpType(MVT VT) const { in isSafeMemOpType()
1732 if (VT == MVT::f32) in isSafeMemOpType()
1734 else if (VT == MVT::f64) in isSafeMemOpType()
1800 MVT VT) const { in findRepresentativeClass()
1806 case MVT::i8: case MVT::i16: case MVT::i32: case MVT::i64: in findRepresentativeClass()
1809 case MVT::x86mmx: in findRepresentativeClass()
1812 case MVT::f32: case MVT::f64: in findRepresentativeClass()
1813 case MVT::v16i8: case MVT::v8i16: case MVT::v4i32: case MVT::v2i64: in findRepresentativeClass()
1814 case MVT::v4f32: case MVT::v2f64: in findRepresentativeClass()
1815 case MVT::v32i8: case MVT::v8i32: case MVT::v4i64: case MVT::v8f32: in findRepresentativeClass()
1816 case MVT::v4f64: in findRepresentativeClass()
1889 MVT::i16)); in LowerReturn()
1913 if ((ValVT == MVT::f32 || ValVT == MVT::f64 || in LowerReturn()
1921 if (ValVT == MVT::f64 && in LowerReturn()
1932 ValToCopy = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f80, ValToCopy); in LowerReturn()
1941 if (ValVT == MVT::x86mmx) { in LowerReturn()
1943 ValToCopy = DAG.getNode(ISD::BITCAST, dl, MVT::i64, ValToCopy); in LowerReturn()
1944 ValToCopy = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, in LowerReturn()
1949 ValToCopy = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32,ValToCopy); in LowerReturn()
1990 return DAG.getNode(X86ISD::RET_FLAG, dl, MVT::Other, RetOps); in LowerReturn()
2004 if (Copy->getOperand(Copy->getNumOperands()-1).getValueType() == MVT::Glue) in isUsedByReturnOnly()
2020 UI->getOperand(UI->getNumOperands()-1).getValueType() != MVT::Glue) in isUsedByReturnOnly()
2035 MVT ReturnMVT; in getTypeForExtArgOrReturn()
2037 if (Subtarget->is64Bit() && VT == MVT::i1 && ExtendKind == ISD::ZERO_EXTEND) in getTypeForExtArgOrReturn()
2038 ReturnMVT = MVT::i8; in getTypeForExtArgOrReturn()
2040 ReturnMVT = MVT::i32; in getTypeForExtArgOrReturn()
2069 if ((CopyVT == MVT::f32 || CopyVT == MVT::f64) && in LowerCallResult()
2078 CopyVT = MVT::f80; in LowerCallResult()
2147 SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), MVT::i32); in CreateCopyOfByValArgument()
2326 if (RegVT == MVT::i32) in LowerFormalArguments()
2328 else if (Is64Bit && RegVT == MVT::i64) in LowerFormalArguments()
2330 else if (RegVT == MVT::f32) in LowerFormalArguments()
2332 else if (RegVT == MVT::f64) in LowerFormalArguments()
2340 else if (RegVT == MVT::x86mmx) in LowerFormalArguments()
2342 else if (RegVT == MVT::i1) in LowerFormalArguments()
2344 else if (RegVT == MVT::v8i1) in LowerFormalArguments()
2346 else if (RegVT == MVT::v16i1) in LowerFormalArguments()
2348 else if (RegVT == MVT::v32i1) in LowerFormalArguments()
2350 else if (RegVT == MVT::v64i1) in LowerFormalArguments()
2400 MVT PtrTy = getPointerTy(); in LowerFormalArguments()
2405 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Copy, Chain); in LowerFormalArguments()
2457 DAG.getCopyFromReg(Chain, dl, GPR, MVT::i64)); in LowerFormalArguments()
2461 ALVal = DAG.getCopyFromReg(Chain, dl, AL, MVT::i8); in LowerFormalArguments()
2465 DAG.getCopyFromReg(Chain, dl, XMMReg, MVT::v4f32)); in LowerFormalArguments()
2517 MVT::Other, SaveXMMOps)); in LowerFormalArguments()
2521 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps); in LowerFormalArguments()
2538 SDValue Val = DAG.getCopyFromReg(Chain, dl, GPR, MVT::i64); in LowerFormalArguments()
2547 MVT VecVT = MVT::Other; in LowerFormalArguments()
2552 VecVT = MVT::v16f32; in LowerFormalArguments()
2554 VecVT = MVT::v8f32; in LowerFormalArguments()
2556 VecVT = MVT::v4f32; in LowerFormalArguments()
2559 SmallVector<MVT, 2> RegParmTypes; in LowerFormalArguments()
2560 MVT IntVT = Is64Bit ? MVT::i64 : MVT::i32; in LowerFormalArguments()
2562 if (VecVT != MVT::Other) in LowerFormalArguments()
2573 Forwards.push_back(ForwardedRegister(ALVReg, X86::AL, MVT::i8)); in LowerFormalArguments()
2611 SDValue StackSlot = DAG.getFrameIndex(UnwindHelpFI, MVT::i64); in LowerFormalArguments()
2613 SDValue Neg2 = DAG.getConstant(-2, MVT::i64); in LowerFormalArguments()
2815 Arg = DAG.getNode(ISD::BITCAST, dl, MVT::i64, Arg); in LowerCall()
2816 Arg = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Arg); in LowerCall()
2817 Arg = getMOVL(DAG, dl, MVT::v2i64, DAG.getUNDEF(MVT::v2i64), Arg); in LowerCall()
2862 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains); in LowerCall()
2908 DAG.getConstant(NumXMMRegs, MVT::i8))); in LowerCall()
2972 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains2); in LowerCall()
3067 Callee->getValueType(0) == MVT::i32) { in LowerCall()
3069 Callee = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i64, Callee); in LowerCall()
3073 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); in LowerCall()
3087 Ops.push_back(DAG.getConstant(FPDiff, MVT::i32)); in LowerCall()
3524 return DAG.getNode(Opc, dl, VT, V1, DAG.getConstant(TargetMask, MVT::i8)); in getTargetShuffleNode()
3826 MVT VT = N->getSimpleValueType(0); in isVEXTRACTIndex()
3844 MVT VT = N->getSimpleValueType(0); in isVINSERTIndex()
3875 MVT VecVT = N->getOperand(0).getSimpleValueType(); in getExtractVEXTRACTImmediate()
3876 MVT ElVT = VecVT.getVectorElementType(); in getExtractVEXTRACTImmediate()
3890 MVT VecVT = N->getSimpleValueType(0); in getInsertVINSERTImmediate()
3891 MVT ElVT = VecVT.getVectorElementType(); in getInsertVINSERTImmediate()
3952 SDValue Cst = DAG.getConstant(0, MVT::i32); in getZeroVector()
3953 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, Cst, Cst, Cst, Cst); in getZeroVector()
3955 SDValue Cst = DAG.getConstantFP(+0.0, MVT::f32); in getZeroVector()
3956 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4f32, Cst, Cst, Cst, Cst); in getZeroVector()
3960 SDValue Cst = DAG.getConstant(0, MVT::i32); in getZeroVector()
3962 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v8i32, Ops); in getZeroVector()
3966 SDValue Cst = DAG.getConstantFP(+0.0, MVT::f32); in getZeroVector()
3968 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v8f32, Ops); in getZeroVector()
3971 SDValue Cst = DAG.getConstant(0, MVT::i32); in getZeroVector()
3974 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v16i32, Ops); in getZeroVector()
3975 } else if (VT.getScalarType() == MVT::i1) { in getZeroVector()
3981 SDValue Cst = DAG.getConstant(0, MVT::i1); in getZeroVector()
4091 MVT ScalarType = ResultVT.getScalarType().getSimpleVT(); in Insert128BitVector()
4097 SDValue Mask = DAG.getConstant(MaskVal, MVT::i8); in Insert128BitVector()
4111 MVT CastVT = Subtarget.hasAVX2() ? MVT::v8i32 : MVT::v8f32; in Insert128BitVector()
4113 SDValue Mask = DAG.getConstant(0x0f, MVT::i8); in Insert128BitVector()
4150 static SDValue getOnesVector(MVT VT, bool HasInt256, SelectionDAG &DAG, in getOnesVector()
4154 SDValue Cst = DAG.getConstant(~0U, MVT::i32); in getOnesVector()
4159 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v8i32, Ops); in getOnesVector()
4161 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, Cst, Cst, Cst, Cst); in getOnesVector()
4162 Vec = Concat128BitVectors(Vec, Vec, MVT::v8i32, 8, DAG, dl); in getOnesVector()
4165 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, Cst, Cst, Cst, Cst); in getOnesVector()
4185 static SDValue getUnpackl(SelectionDAG &DAG, SDLoc dl, MVT VT, SDValue V1, in getUnpackl()
4197 static SDValue getUnpackh(SelectionDAG &DAG, SDLoc dl, MVT VT, SDValue V1, in getUnpackh()
4216 MVT VT = V2.getSimpleValueType(); in getShuffleVectorZeroOrUndef()
4232 static bool getTargetShuffleMask(SDNode *N, MVT VT, in getTargetShuffleMask()
4418 MVT ShufVT = V.getSimpleValueType(); in getShuffleScalarElt()
4477 V = getZeroVector(MVT::v16i8, Subtarget, DAG, dl); in LowerBuildVectorv16i8()
4479 V = DAG.getUNDEF(MVT::v16i8); in LowerBuildVectorv16i8()
4483 MVT::v16i8, V, Op.getOperand(i), in LowerBuildVectorv16i8()
4496 V = getZeroVector(MVT::v8i16, Subtarget, DAG, dl); in LowerBuildVectorv16i8()
4498 V = DAG.getUNDEF(MVT::v8i16); in LowerBuildVectorv16i8()
4507 MVT::i16, Op.getOperand(i-1)); in LowerBuildVectorv16i8()
4510 ThisElt = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, Op.getOperand(i)); in LowerBuildVectorv16i8()
4511 ThisElt = DAG.getNode(ISD::SHL, dl, MVT::i16, in LowerBuildVectorv16i8()
4512 ThisElt, DAG.getConstant(8, MVT::i8)); in LowerBuildVectorv16i8()
4514 ThisElt = DAG.getNode(ISD::OR, dl, MVT::i16, ThisElt, LastElt); in LowerBuildVectorv16i8()
4519 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, V, ThisElt, in LowerBuildVectorv16i8()
4524 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, V); in LowerBuildVectorv16i8()
4545 V = getZeroVector(MVT::v8i16, Subtarget, DAG, dl); in LowerBuildVectorv8i16()
4547 V = DAG.getUNDEF(MVT::v8i16); in LowerBuildVectorv8i16()
4551 MVT::v8i16, V, Op.getOperand(i), in LowerBuildVectorv8i16()
4584 MVT VT = Elt.getOperand(0).getSimpleValueType(); in LowerBuildVectorv4x32()
4595 MVT VT = V1.getSimpleValueType(); in LowerBuildVectorv4x32()
4649 if (V1.getSimpleValueType() != MVT::v4f32) in LowerBuildVectorv4x32()
4650 V1 = DAG.getNode(ISD::BITCAST, SDLoc(V1), MVT::v4f32, V1); in LowerBuildVectorv4x32()
4651 if (V2.getSimpleValueType() != MVT::v4f32) in LowerBuildVectorv4x32()
4652 V2 = DAG.getNode(ISD::BITCAST, SDLoc(V2), MVT::v4f32, V2); in LowerBuildVectorv4x32()
4659 SDValue Result = DAG.getNode(X86ISD::INSERTPS, SDLoc(Op), MVT::v4f32, V1, V2, in LowerBuildVectorv4x32()
4669 MVT ShVT = MVT::v2i64; in getVShift()
4672 MVT ScalarShiftTy = TLI.getScalarShiftAmountTy(SrcOp.getValueType()); in getVShift()
4680 LowerAsSplatVectorLoad(SDValue SrcOp, MVT VT, SDLoc dl, SelectionDAG &DAG) { in LowerAsSplatVectorLoad()
4690 if (PVT != MVT::i32 && PVT != MVT::f32) in LowerAsSplatVectorLoad()
4823 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, in EltsFromConsecutiveLoads()
4838 DAG.getTargetLoweringInfo().isTypeLegal(MVT::v2i64)) { in EltsFromConsecutiveLoads()
4839 SDVTList Tys = DAG.getVTList(MVT::v2i64, MVT::Other); in EltsFromConsecutiveLoads()
4842 DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, DL, Tys, Ops, MVT::i64, in EltsFromConsecutiveLoads()
4852 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, in EltsFromConsecutiveLoads()
4879 MVT VT = Op.getSimpleValueType(); in LowerVectorBroadcast()
5048 MVT ShuffleVecVT = ShuffleVec.getSimpleValueType(); in getUnderlyingExtractedFromVec()
5061 MVT VT = Op.getSimpleValueType(); in buildFromShuffleMostly()
5136 MVT VT = Op.getSimpleValueType(); in LowerBUILD_VECTORvXi1()
5137 assert((VT.getVectorElementType() == MVT::i1) && (VT.getSizeInBits() <= 16) && in LowerBUILD_VECTORvXi1()
5142 SDValue Cst = DAG.getTargetConstant(0, MVT::i1); in LowerBUILD_VECTORvXi1()
5148 SDValue Cst = DAG.getTargetConstant(1, MVT::i1); in LowerBUILD_VECTORvXi1()
5177 SDValue FullMask = DAG.getNode(ISD::BITCAST, dl, MVT::v16i1, in LowerBUILD_VECTORvXi1()
5178 DAG.getConstant(Immediate, MVT::i16)); in LowerBUILD_VECTORvXi1()
5187 MVT::getIntegerVT(VT.getSizeInBits())); in LowerBUILD_VECTORvXi1()
5198 MVT SelectVT = (VT == MVT::v16i1)? MVT::i16 : MVT::i8; in LowerBUILD_VECTORvXi1()
5381 assert((VT == MVT::v8f32 || VT == MVT::v4f64 || VT == MVT::v4f32 || in matchAddSub()
5382 VT == MVT::v2f64) && "build_vector with an invalid type found!"); in matchAddSub()
5473 if ((Subtarget->hasSSE3() && (VT == MVT::v4f32 || VT == MVT::v2f64)) || in PerformBUILD_VECTORCombine()
5474 (Subtarget->hasAVX() && (VT == MVT::v8f32 || VT == MVT::v4f64))) { in PerformBUILD_VECTORCombine()
5499 if ((VT == MVT::v4f32 || VT == MVT::v2f64) && Subtarget->hasSSE3()) { in PerformBUILD_VECTORCombine()
5506 } else if ((VT == MVT::v4i32 || VT == MVT::v8i16) && Subtarget->hasSSSE3()) { in PerformBUILD_VECTORCombine()
5518 if ((VT == MVT::v8f32 || VT == MVT::v4f64)) { in PerformBUILD_VECTORCombine()
5537 } else if (VT == MVT::v8i32 || VT == MVT::v16i16) { in PerformBUILD_VECTORCombine()
5580 if ((VT == MVT::v8f32 || VT == MVT::v4f64 || VT == MVT::v8i32 || in PerformBUILD_VECTORCombine()
5581 VT == MVT::v16i16) && Subtarget->hasAVX()) { in PerformBUILD_VECTORCombine()
5614 MVT VT = Op.getSimpleValueType(); in LowerBUILD_VECTOR()
5615 MVT ExtVT = VT.getVectorElementType(); in LowerBUILD_VECTOR()
5619 if (VT.getScalarType() == MVT::i1 && Subtarget->hasAVX512()) in LowerBUILD_VECTOR()
5626 if (VT == MVT::v4i32 || VT == MVT::v8i32 || VT == MVT::v16i32) in LowerBUILD_VECTOR()
5636 if (VT == MVT::v4i32 || (VT == MVT::v8i32 && Subtarget->hasInt256())) in LowerBUILD_VECTOR()
5683 if (ExtVT == MVT::i64 && !Subtarget->is64Bit() && in LowerBUILD_VECTOR()
5687 assert(VT == MVT::v2i64 && "Expected an SSE value type!"); in LowerBUILD_VECTOR()
5688 EVT VecVT = MVT::v4i32; in LowerBUILD_VECTOR()
5692 Item = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Item); in LowerBUILD_VECTOR()
5708 if (ExtVT == MVT::i32 || ExtVT == MVT::f32 || ExtVT == MVT::f64 || in LowerBUILD_VECTOR()
5709 (ExtVT == MVT::i64 && Subtarget->is64Bit())) { in LowerBUILD_VECTOR()
5724 if (ExtVT == MVT::i16 || ExtVT == MVT::i8) { in LowerBUILD_VECTOR()
5725 Item = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, Item); in LowerBUILD_VECTOR()
5728 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v8i32, Item); in LowerBUILD_VECTOR()
5733 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, Item); in LowerBUILD_VECTOR()
5734 SDValue ZeroVec = getZeroVector(MVT::v8i32, Subtarget, DAG, dl); in LowerBUILD_VECTOR()
5739 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, Item); in LowerBUILD_VECTOR()
5951 MVT ResVT = Op.getSimpleValueType(); in LowerAVXCONCAT_VECTORS()
5963 MVT HalfVT = MVT::getVectorVT(ResVT.getScalarType(), in LowerAVXCONCAT_VECTORS()
5977 MVT ResVT = Op.getSimpleValueType(); in LowerCONCAT_VECTORSvXi1()
5984 MVT HalfVT = MVT::getVectorVT(ResVT.getScalarType(), in LowerCONCAT_VECTORSvXi1()
6008 SDValue ShiftBits = DAG.getConstant(NumElems/2, MVT::i8); in LowerCONCAT_VECTORSvXi1()
6027 MVT VT = Op.getSimpleValueType(); in LowerCONCAT_VECTORS()
6028 if (VT.getVectorElementType() == MVT::i1) in LowerCONCAT_VECTORS()
6088 static bool is128BitLaneCrossingShuffleMask(MVT VT, ArrayRef<int> Mask) { in is128BitLaneCrossingShuffleMask()
6109 is128BitLaneRepeatedShuffleMask(MVT VT, ArrayRef<int> Mask, in is128BitLaneRepeatedShuffleMask()
6189 return DAG.getConstant(Imm, MVT::i8); in getV4X86ShuffleImm8ForMask()
6197 static SDValue lowerVectorShuffleAsBitBlend(SDLoc DL, MVT VT, SDValue V1, in lowerVectorShuffleAsBitBlend()
6201 MVT EltVT = VT.getScalarType(); in lowerVectorShuffleAsBitBlend()
6215 MVT MaskVT = MVT::getVectorVT(MVT::i64, VT.getSizeInBits() / 64); in lowerVectorShuffleAsBitBlend()
6229 static SDValue lowerVectorShuffleAsBlend(SDLoc DL, MVT VT, SDValue V1, in lowerVectorShuffleAsBlend()
6245 case MVT::v2f64: in lowerVectorShuffleAsBlend()
6246 case MVT::v4f32: in lowerVectorShuffleAsBlend()
6247 case MVT::v4f64: in lowerVectorShuffleAsBlend()
6248 case MVT::v8f32: in lowerVectorShuffleAsBlend()
6250 DAG.getConstant(BlendMask, MVT::i8)); in lowerVectorShuffleAsBlend()
6252 case MVT::v4i64: in lowerVectorShuffleAsBlend()
6253 case MVT::v8i32: in lowerVectorShuffleAsBlend()
6256 case MVT::v2i64: in lowerVectorShuffleAsBlend()
6257 case MVT::v4i32: in lowerVectorShuffleAsBlend()
6269 MVT BlendVT = VT.getSizeInBits() > 128 ? MVT::v8i32 : MVT::v4i32; in lowerVectorShuffleAsBlend()
6274 DAG.getConstant(BlendMask, MVT::i8))); in lowerVectorShuffleAsBlend()
6277 case MVT::v8i16: { in lowerVectorShuffleAsBlend()
6287 V1 = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V1); in lowerVectorShuffleAsBlend()
6288 V2 = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V2); in lowerVectorShuffleAsBlend()
6290 DAG.getNode(X86ISD::BLENDI, DL, MVT::v8i16, V1, V2, in lowerVectorShuffleAsBlend()
6291 DAG.getConstant(BlendMask, MVT::i8))); in lowerVectorShuffleAsBlend()
6294 case MVT::v16i16: { in lowerVectorShuffleAsBlend()
6297 if (is128BitLaneRepeatedShuffleMask(MVT::v16i16, Mask, RepeatedMask)) { in lowerVectorShuffleAsBlend()
6304 return DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1, V2, in lowerVectorShuffleAsBlend()
6305 DAG.getConstant(BlendMask, MVT::i8)); in lowerVectorShuffleAsBlend()
6309 case MVT::v16i8: in lowerVectorShuffleAsBlend()
6310 case MVT::v32i8: { in lowerVectorShuffleAsBlend()
6319 MVT BlendVT = MVT::getVectorVT(MVT::i8, VT.getSizeInBits() / 8); in lowerVectorShuffleAsBlend()
6336 Mask[i] < 0 ? DAG.getUNDEF(MVT::i8) in lowerVectorShuffleAsBlend()
6337 : DAG.getConstant(Mask[i] < Size ? -1 : 0, MVT::i8)); in lowerVectorShuffleAsBlend()
6358 static SDValue lowerVectorShuffleAsBlendAndPermute(SDLoc DL, MVT VT, SDValue V1, in lowerVectorShuffleAsBlendAndPermute()
6392 static SDValue lowerVectorShuffleAsDecomposedShuffleBlend(SDLoc DL, MVT VT, in lowerVectorShuffleAsDecomposedShuffleBlend()
6442 static SDValue lowerVectorShuffleAsByteRotate(SDLoc DL, MVT VT, SDValue V1, in lowerVectorShuffleAsByteRotate()
6525 MVT AlignVT = MVT::getVectorVT(MVT::i8, 16 * NumLanes); in lowerVectorShuffleAsByteRotate()
6531 DAG.getConstant(Rotation * Scale, MVT::i8))); in lowerVectorShuffleAsByteRotate()
6544 Lo = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Lo); in lowerVectorShuffleAsByteRotate()
6545 Hi = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Hi); in lowerVectorShuffleAsByteRotate()
6547 SDValue LoShift = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v2i64, Lo, in lowerVectorShuffleAsByteRotate()
6548 DAG.getConstant(LoByteShift, MVT::i8)); in lowerVectorShuffleAsByteRotate()
6549 SDValue HiShift = DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v2i64, Hi, in lowerVectorShuffleAsByteRotate()
6550 DAG.getConstant(HiByteShift, MVT::i8)); in lowerVectorShuffleAsByteRotate()
6552 DAG.getNode(ISD::OR, DL, MVT::v2i64, LoShift, HiShift)); in lowerVectorShuffleAsByteRotate()
6603 static SDValue lowerVectorShuffleAsBitMask(SDLoc DL, MVT VT, SDValue V1, in lowerVectorShuffleAsBitMask()
6606 MVT EltVT = VT.getScalarType(); in lowerVectorShuffleAsBitMask()
6608 MVT IntEltVT = MVT::getIntegerVT(NumEltBits); in lowerVectorShuffleAsBitMask()
6663 static SDValue lowerVectorShuffleAsShift(SDLoc DL, MVT VT, SDValue V1, in lowerVectorShuffleAsShift()
6701 MVT ShiftSVT = MVT::getIntegerVT(VT.getScalarSizeInBits() * Scale); in lowerVectorShuffleAsShift()
6702 MVT ShiftVT = MVT::getVectorVT(ShiftSVT, Size / Scale); in lowerVectorShuffleAsShift()
6707 V = DAG.getNode(OpCode, DL, ShiftVT, V, DAG.getConstant(ShiftAmt, MVT::i8)); in lowerVectorShuffleAsShift()
6735 SDLoc DL, MVT VT, int Scale, bool AnyExt, SDValue InputV, in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
6747 MVT ExtVT = MVT::getVectorVT(MVT::getIntegerVT(EltBits * Scale), in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
6759 DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
6760 DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, InputV), in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
6765 InputV = DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
6766 DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, InputV), in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
6771 DAG.getNode(X86ISD::PSHUFHW, DL, MVT::v8i16, in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
6772 DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, InputV), in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
6784 DAG.getConstant((i % Scale == 0) ? i / Scale : 0x80, MVT::i8); in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
6785 InputV = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, InputV); in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
6787 DAG.getNode(X86ISD::PSHUFB, DL, MVT::v16i8, InputV, in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
6789 MVT::v16i8, PSHUFBMask))); in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
6794 MVT InputVT = MVT::getVectorVT(MVT::getIntegerVT(EltBits), NumElements); in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
6819 SDLoc DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask, in lowerVectorShuffleAsZeroOrAnyExtend()
6901 V = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, V); in lowerVectorShuffleAsZeroOrAnyExtend()
6902 V = DAG.getNode(X86ISD::VZEXT_MOVL, DL, MVT::v2i64, V); in lowerVectorShuffleAsZeroOrAnyExtend()
6915 MVT VT = V.getSimpleValueType(); in getScalarValueForVectorElement()
6916 MVT EltVT = VT.getVectorElementType(); in getScalarValueForVectorElement()
6921 MVT NewVT = V.getSimpleValueType(); in getScalarValueForVectorElement()
6948 SDLoc DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask, in lowerVectorShuffleAsElementInsertion()
6951 MVT ExtVT = VT; in lowerVectorShuffleAsElementInsertion()
6952 MVT EltVT = VT.getVectorElementType(); in lowerVectorShuffleAsElementInsertion()
6973 if (EltVT == MVT::i8 || EltVT == MVT::i16) { in lowerVectorShuffleAsElementInsertion()
6980 ExtVT = MVT::v4i32; in lowerVectorShuffleAsElementInsertion()
6981 V2S = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, V2S); in lowerVectorShuffleAsElementInsertion()
6984 } else if (Mask[V2Index] != (int)Mask.size() || EltVT == MVT::i8 || in lowerVectorShuffleAsElementInsertion()
6985 EltVT == MVT::i16) { in lowerVectorShuffleAsElementInsertion()
7009 assert((EltVT == MVT::f32 || EltVT == MVT::f64) && in lowerVectorShuffleAsElementInsertion()
7011 return DAG.getNode(EltVT == MVT::f32 ? X86ISD::MOVSS : X86ISD::MOVSD, DL, in lowerVectorShuffleAsElementInsertion()
7033 V2 = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, V2); in lowerVectorShuffleAsElementInsertion()
7035 X86ISD::VSHLDQ, DL, MVT::v2i64, V2, in lowerVectorShuffleAsElementInsertion()
7038 DAG.getTargetLoweringInfo().getScalarShiftAmountTy(MVT::v2i64))); in lowerVectorShuffleAsElementInsertion()
7050 static SDValue lowerVectorShuffleAsBroadcast(SDLoc DL, MVT VT, SDValue V, in lowerVectorShuffleAsBroadcast()
7131 assert(Op.getSimpleValueType() == MVT::v4f32 && "Bad shuffle type!"); in lowerVectorShuffleAsInsertPS()
7132 assert(V1.getSimpleValueType() == MVT::v4f32 && "Bad operand type!"); in lowerVectorShuffleAsInsertPS()
7133 assert(V2.getSimpleValueType() == MVT::v4f32 && "Bad operand type!"); in lowerVectorShuffleAsInsertPS()
7189 V1 = DAG.getUNDEF(MVT::v4f32); in lowerVectorShuffleAsInsertPS()
7196 return DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32, V1, V2, in lowerVectorShuffleAsInsertPS()
7197 DAG.getConstant(InsertPSMask, MVT::i8)); in lowerVectorShuffleAsInsertPS()
7209 static SDValue lowerVectorShuffleAsUnpack(SDLoc DL, MVT VT, SDValue V1, in lowerVectorShuffleAsUnpack()
7228 auto TryUnpack = [&](MVT UnpackVT, int Scale) { in lowerVectorShuffleAsUnpack()
7278 MVT UnpackVT = MVT::getVectorVT(MVT::getIntegerVT(ScalarSize), NumElements); in lowerVectorShuffleAsUnpack()
7326 assert(Op.getSimpleValueType() == MVT::v2f64 && "Bad shuffle type!"); in lowerV2F64VectorShuffle()
7327 assert(V1.getSimpleValueType() == MVT::v2f64 && "Bad operand type!"); in lowerV2F64VectorShuffle()
7328 assert(V2.getSimpleValueType() == MVT::v2f64 && "Bad operand type!"); in lowerV2F64VectorShuffle()
7337 return DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v2f64, V1); in lowerV2F64VectorShuffle()
7346 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v2f64, V1, in lowerV2F64VectorShuffle()
7347 DAG.getConstant(SHUFPDMask, MVT::i8)); in lowerV2F64VectorShuffle()
7350 return DAG.getNode(X86ISD::SHUFP, SDLoc(Op), MVT::v2f64, V1, V1, in lowerV2F64VectorShuffle()
7351 DAG.getConstant(SHUFPDMask, MVT::i8)); in lowerV2F64VectorShuffle()
7359 DL, MVT::v2f64, V1, V2, Mask, Subtarget, DAG)) in lowerV2F64VectorShuffle()
7366 DL, MVT::v2f64, V2, V1, InverseMask, Subtarget, DAG)) in lowerV2F64VectorShuffle()
7379 DL, MVT::v2f64, V2, in lowerV2F64VectorShuffle()
7380 DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64, V1S)); in lowerV2F64VectorShuffle()
7383 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v2f64, V1, V2, Mask, in lowerV2F64VectorShuffle()
7389 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v2f64, V1, V2); in lowerV2F64VectorShuffle()
7391 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v2f64, V1, V2); in lowerV2F64VectorShuffle()
7394 return DAG.getNode(X86ISD::SHUFP, SDLoc(Op), MVT::v2f64, V1, V2, in lowerV2F64VectorShuffle()
7395 DAG.getConstant(SHUFPDMask, MVT::i8)); in lowerV2F64VectorShuffle()
7408 assert(Op.getSimpleValueType() == MVT::v2i64 && "Bad shuffle type!"); in lowerV2I64VectorShuffle()
7409 assert(V1.getSimpleValueType() == MVT::v2i64 && "Bad operand type!"); in lowerV2I64VectorShuffle()
7410 assert(V2.getSimpleValueType() == MVT::v2i64 && "Bad operand type!"); in lowerV2I64VectorShuffle()
7417 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(DL, MVT::v2i64, V1, in lowerV2I64VectorShuffle()
7424 V1 = DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, V1); in lowerV2I64VectorShuffle()
7429 ISD::BITCAST, DL, MVT::v2i64, in lowerV2I64VectorShuffle()
7430 DAG.getNode(X86ISD::PSHUFD, SDLoc(Op), MVT::v4i32, V1, in lowerV2I64VectorShuffle()
7450 return DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, in lowerV2I64VectorShuffle()
7451 DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, in lowerV2I64VectorShuffle()
7459 lowerVectorShuffleAsShift(DL, MVT::v2i64, V1, V2, Mask, DAG)) in lowerV2I64VectorShuffle()
7465 DL, MVT::v2i64, V1, V2, Mask, Subtarget, DAG)) in lowerV2I64VectorShuffle()
7471 DL, MVT::v2i64, V2, V1, InverseMask, Subtarget, DAG)) in lowerV2I64VectorShuffle()
7478 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v2i64, V1, V2, Mask, in lowerV2I64VectorShuffle()
7484 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v2i64, V1, V2); in lowerV2I64VectorShuffle()
7486 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v2i64, V1, V2); in lowerV2I64VectorShuffle()
7492 DL, MVT::v2i64, V1, V2, Mask, Subtarget, DAG)) in lowerV2I64VectorShuffle()
7498 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v2i64, V1, V2, in lowerV2I64VectorShuffle()
7505 V1 = DAG.getNode(ISD::BITCAST, DL, MVT::v2f64, V1); in lowerV2I64VectorShuffle()
7506 V2 = DAG.getNode(ISD::BITCAST, DL, MVT::v2f64, V2); in lowerV2I64VectorShuffle()
7507 return DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, in lowerV2I64VectorShuffle()
7508 DAG.getVectorShuffle(MVT::v2f64, DL, V1, V2, Mask)); in lowerV2I64VectorShuffle()
7534 static SDValue lowerVectorShuffleWithSHUFPS(SDLoc DL, MVT VT, in lowerVectorShuffleWithSHUFPS()
7628 assert(Op.getSimpleValueType() == MVT::v4f32 && "Bad shuffle type!"); in lowerV4F32VectorShuffle()
7629 assert(V1.getSimpleValueType() == MVT::v4f32 && "Bad operand type!"); in lowerV4F32VectorShuffle()
7630 assert(V2.getSimpleValueType() == MVT::v4f32 && "Bad operand type!"); in lowerV4F32VectorShuffle()
7640 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(DL, MVT::v4f32, V1, in lowerV4F32VectorShuffle()
7647 return DAG.getNode(X86ISD::MOVSLDUP, DL, MVT::v4f32, V1); in lowerV4F32VectorShuffle()
7649 return DAG.getNode(X86ISD::MOVSHDUP, DL, MVT::v4f32, V1); in lowerV4F32VectorShuffle()
7655 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v4f32, V1, in lowerV4F32VectorShuffle()
7661 return DAG.getNode(X86ISD::SHUFP, DL, MVT::v4f32, V1, V1, in lowerV4F32VectorShuffle()
7671 if (SDValue V = lowerVectorShuffleAsElementInsertion(DL, MVT::v4f32, V1, V2, in lowerV4F32VectorShuffle()
7676 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v4f32, V1, V2, Mask, in lowerV4F32VectorShuffle()
7686 DL, MVT::v4f32, V1, V2, Mask, DAG)) in lowerV4F32VectorShuffle()
7692 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v4f32, V1, V2); in lowerV4F32VectorShuffle()
7694 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v4f32, V1, V2); in lowerV4F32VectorShuffle()
7696 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v4f32, V2, V1); in lowerV4F32VectorShuffle()
7698 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v4f32, V2, V1); in lowerV4F32VectorShuffle()
7701 return lowerVectorShuffleWithSHUFPS(DL, MVT::v4f32, Mask, V1, V2, DAG); in lowerV4F32VectorShuffle()
7712 assert(Op.getSimpleValueType() == MVT::v4i32 && "Bad shuffle type!"); in lowerV4I32VectorShuffle()
7713 assert(V1.getSimpleValueType() == MVT::v4i32 && "Bad operand type!"); in lowerV4I32VectorShuffle()
7714 assert(V2.getSimpleValueType() == MVT::v4i32 && "Bad operand type!"); in lowerV4I32VectorShuffle()
7722 if (SDValue ZExt = lowerVectorShuffleAsZeroOrAnyExtend(DL, MVT::v4i32, V1, V2, in lowerV4I32VectorShuffle()
7731 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(DL, MVT::v4i32, V1, in lowerV4I32VectorShuffle()
7747 return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, V1, in lowerV4I32VectorShuffle()
7753 lowerVectorShuffleAsShift(DL, MVT::v4i32, V1, V2, Mask, DAG)) in lowerV4I32VectorShuffle()
7758 if (SDValue V = lowerVectorShuffleAsElementInsertion(DL, MVT::v4i32, V1, V2, in lowerV4I32VectorShuffle()
7766 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v4i32, V1, V2, Mask, in lowerV4I32VectorShuffle()
7771 lowerVectorShuffleAsBitMask(DL, MVT::v4i32, V1, V2, Mask, DAG)) in lowerV4I32VectorShuffle()
7776 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v4i32, V1, V2); in lowerV4I32VectorShuffle()
7778 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v4i32, V1, V2); in lowerV4I32VectorShuffle()
7780 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v4i32, V2, V1); in lowerV4I32VectorShuffle()
7782 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v4i32, V2, V1); in lowerV4I32VectorShuffle()
7788 DL, MVT::v4i32, V1, V2, Mask, Subtarget, DAG)) in lowerV4I32VectorShuffle()
7794 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v4i32, V1, V2, in lowerV4I32VectorShuffle()
7799 lowerVectorShuffleAsUnpack(DL, MVT::v4i32, V1, V2, Mask, DAG)) in lowerV4I32VectorShuffle()
7807 return DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, in lowerV4I32VectorShuffle()
7809 MVT::v4f32, DL, in lowerV4I32VectorShuffle()
7810 DAG.getNode(ISD::BITCAST, DL, MVT::v4f32, V1), in lowerV4I32VectorShuffle()
7811 DAG.getNode(ISD::BITCAST, DL, MVT::v4f32, V2), Mask)); in lowerV4I32VectorShuffle()
7831 SDLoc DL, MVT VT, SDValue V, MutableArrayRef<int> Mask, in lowerV8I16GeneralSingleInputVectorShuffle()
7833 assert(VT.getScalarType() == MVT::i16 && "Bad input type!"); in lowerV8I16GeneralSingleInputVectorShuffle()
7834 MVT PSHUFDVT = MVT::getVectorVT(MVT::i32, VT.getVectorNumElements() / 2); in lowerV8I16GeneralSingleInputVectorShuffle()
7964 MVT::v8i16, V, in lowerV8I16GeneralSingleInputVectorShuffle()
8264 static SDValue lowerVectorShuffleAsPSHUFB(SDLoc DL, MVT VT, SDValue V1, in lowerVectorShuffleAsPSHUFB()
8278 V1Mask[i] = V2Mask[i] = DAG.getUNDEF(MVT::i8); in lowerVectorShuffleAsPSHUFB()
8288 V1Mask[i] = DAG.getConstant(V1Idx, MVT::i8); in lowerVectorShuffleAsPSHUFB()
8289 V2Mask[i] = DAG.getConstant(V2Idx, MVT::i8); in lowerVectorShuffleAsPSHUFB()
8296 V1 = DAG.getNode(X86ISD::PSHUFB, DL, MVT::v16i8, in lowerVectorShuffleAsPSHUFB()
8297 DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, V1), in lowerVectorShuffleAsPSHUFB()
8298 DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v16i8, V1Mask)); in lowerVectorShuffleAsPSHUFB()
8300 V2 = DAG.getNode(X86ISD::PSHUFB, DL, MVT::v16i8, in lowerVectorShuffleAsPSHUFB()
8301 DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, V2), in lowerVectorShuffleAsPSHUFB()
8302 DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v16i8, V2Mask)); in lowerVectorShuffleAsPSHUFB()
8307 V = DAG.getNode(ISD::OR, DL, MVT::v16i8, V1, V2); in lowerVectorShuffleAsPSHUFB()
8331 assert(Op.getSimpleValueType() == MVT::v8i16 && "Bad shuffle type!"); in lowerV8I16VectorShuffle()
8332 assert(V1.getSimpleValueType() == MVT::v8i16 && "Bad operand type!"); in lowerV8I16VectorShuffle()
8333 assert(V2.getSimpleValueType() == MVT::v8i16 && "Bad operand type!"); in lowerV8I16VectorShuffle()
8345 DL, MVT::v8i16, V1, V2, OrigMask, Subtarget, DAG)) in lowerV8I16VectorShuffle()
8356 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(DL, MVT::v8i16, V1, in lowerV8I16VectorShuffle()
8362 lowerVectorShuffleAsShift(DL, MVT::v8i16, V1, V1, Mask, DAG)) in lowerV8I16VectorShuffle()
8367 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v8i16, V1, V1); in lowerV8I16VectorShuffle()
8369 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v8i16, V1, V1); in lowerV8I16VectorShuffle()
8372 if (SDValue Rotate = lowerVectorShuffleAsByteRotate(DL, MVT::v8i16, V1, V1, in lowerV8I16VectorShuffle()
8376 return lowerV8I16GeneralSingleInputVectorShuffle(DL, MVT::v8i16, V1, Mask, in lowerV8I16VectorShuffle()
8386 lowerVectorShuffleAsShift(DL, MVT::v8i16, V1, V2, Mask, DAG)) in lowerV8I16VectorShuffle()
8391 if (SDValue V = lowerVectorShuffleAsElementInsertion(DL, MVT::v8i16, V1, V2, in lowerV8I16VectorShuffle()
8399 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v8i16, V1, V2, Mask, in lowerV8I16VectorShuffle()
8404 lowerVectorShuffleAsBitMask(DL, MVT::v8i16, V1, V2, Mask, DAG)) in lowerV8I16VectorShuffle()
8409 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v8i16, V1, V2); in lowerV8I16VectorShuffle()
8411 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v8i16, V1, V2); in lowerV8I16VectorShuffle()
8415 DL, MVT::v8i16, V1, V2, Mask, Subtarget, DAG)) in lowerV8I16VectorShuffle()
8419 lowerVectorShuffleAsBitBlend(DL, MVT::v8i16, V1, V2, Mask, DAG)) in lowerV8I16VectorShuffle()
8423 lowerVectorShuffleAsUnpack(DL, MVT::v8i16, V1, V2, Mask, DAG)) in lowerV8I16VectorShuffle()
8430 return lowerVectorShuffleAsPSHUFB(DL, MVT::v8i16, V1, V2, Mask, DAG, in lowerV8I16VectorShuffle()
8436 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v8i16, V1, V2, in lowerV8I16VectorShuffle()
8519 assert(Op.getSimpleValueType() == MVT::v16i8 && "Bad shuffle type!"); in lowerV16I8VectorShuffle()
8520 assert(V1.getSimpleValueType() == MVT::v16i8 && "Bad operand type!"); in lowerV16I8VectorShuffle()
8521 assert(V2.getSimpleValueType() == MVT::v16i8 && "Bad operand type!"); in lowerV16I8VectorShuffle()
8528 lowerVectorShuffleAsShift(DL, MVT::v16i8, V1, V2, Mask, DAG)) in lowerV16I8VectorShuffle()
8533 DL, MVT::v16i8, V1, V2, Mask, Subtarget, DAG)) in lowerV16I8VectorShuffle()
8538 DL, MVT::v16i8, V1, V2, Mask, Subtarget, DAG)) in lowerV16I8VectorShuffle()
8547 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(DL, MVT::v16i8, V1, in lowerV16I8VectorShuffle()
8614 ISD::BITCAST, DL, MVT::v16i8, in lowerV16I8VectorShuffle()
8615 DAG.getVectorShuffle(MVT::v8i16, DL, in lowerV16I8VectorShuffle()
8616 DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V1), in lowerV16I8VectorShuffle()
8617 DAG.getUNDEF(MVT::v8i16), PreDupI16Shuffle)); in lowerV16I8VectorShuffle()
8621 MVT::v16i8, V1, V1); in lowerV16I8VectorShuffle()
8635 ISD::BITCAST, DL, MVT::v16i8, in lowerV16I8VectorShuffle()
8636 DAG.getVectorShuffle(MVT::v8i16, DL, in lowerV16I8VectorShuffle()
8637 DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V1), in lowerV16I8VectorShuffle()
8638 DAG.getUNDEF(MVT::v8i16), PostDupI16Shuffle)); in lowerV16I8VectorShuffle()
8649 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v16i8, V1, V2); in lowerV16I8VectorShuffle()
8654 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v16i8, V1, V2); in lowerV16I8VectorShuffle()
8673 SDValue PSHUFB = lowerVectorShuffleAsPSHUFB(DL, MVT::v16i8, V1, V2, Mask, in lowerV16I8VectorShuffle()
8681 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v16i8, V1, V2, in lowerV16I8VectorShuffle()
8694 lowerVectorShuffleAsUnpack(DL, MVT::v16i8, V1, V2, Mask, DAG)) in lowerV16I8VectorShuffle()
8703 if (SDValue V = lowerVectorShuffleAsElementInsertion(DL, MVT::v16i8, V1, V2, in lowerV16I8VectorShuffle()
8708 lowerVectorShuffleAsBitBlend(DL, MVT::v16i8, V1, V2, Mask, DAG)) in lowerV16I8VectorShuffle()
8729 MVT MaskVTs[] = { MVT::v8i16, MVT::v4i32, MVT::v2i64 }; in lowerV16I8VectorShuffle()
8731 DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, in lowerV16I8VectorShuffle()
8733 V1 = DAG.getNode(ISD::AND, DL, MVT::v16i8, V1, ByteClearMask); in lowerV16I8VectorShuffle()
8735 V2 = DAG.getNode(ISD::AND, DL, MVT::v16i8, V2, ByteClearMask); in lowerV16I8VectorShuffle()
8738 V1 = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V1); in lowerV16I8VectorShuffle()
8739 V2 = IsSingleInput ? V1 : DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V2); in lowerV16I8VectorShuffle()
8740 SDValue Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, V1, V2); in lowerV16I8VectorShuffle()
8742 Result = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, Result); in lowerV16I8VectorShuffle()
8743 Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, Result, Result); in lowerV16I8VectorShuffle()
8751 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v16i8, V1, V2, in lowerV16I8VectorShuffle()
8765 SDValue Zero = getZeroVector(MVT::v8i16, Subtarget, DAG, DL); in lowerV16I8VectorShuffle()
8776 VLoHalf = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V); in lowerV16I8VectorShuffle()
8777 VLoHalf = DAG.getNode(ISD::AND, DL, MVT::v8i16, VLoHalf, in lowerV16I8VectorShuffle()
8778 DAG.getConstant(0x00FF, MVT::v8i16)); in lowerV16I8VectorShuffle()
8781 VHiHalf = DAG.getUNDEF(MVT::v8i16); in lowerV16I8VectorShuffle()
8793 VLoHalf = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, in lowerV16I8VectorShuffle()
8794 DAG.getNode(X86ISD::UNPCKL, DL, MVT::v16i8, V, Zero)); in lowerV16I8VectorShuffle()
8795 VHiHalf = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, in lowerV16I8VectorShuffle()
8796 DAG.getNode(X86ISD::UNPCKH, DL, MVT::v16i8, V, Zero)); in lowerV16I8VectorShuffle()
8799 SDValue LoV = DAG.getVectorShuffle(MVT::v8i16, DL, VLoHalf, VHiHalf, LoBlendMask); in lowerV16I8VectorShuffle()
8800 SDValue HiV = DAG.getVectorShuffle(MVT::v8i16, DL, VLoHalf, VHiHalf, HiBlendMask); in lowerV16I8VectorShuffle()
8802 return DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, LoV, HiV); in lowerV16I8VectorShuffle()
8810 MVT VT, const X86Subtarget *Subtarget, in lower128BitVectorShuffle()
8813 case MVT::v2i64: in lower128BitVectorShuffle()
8815 case MVT::v2f64: in lower128BitVectorShuffle()
8817 case MVT::v4i32: in lower128BitVectorShuffle()
8819 case MVT::v4f32: in lower128BitVectorShuffle()
8821 case MVT::v8i16: in lower128BitVectorShuffle()
8823 case MVT::v16i8: in lower128BitVectorShuffle()
8891 static SDValue splitAndLowerVectorShuffle(SDLoc DL, MVT VT, SDValue V1, in splitAndLowerVectorShuffle()
8904 MVT ScalarVT = VT.getScalarType(); in splitAndLowerVectorShuffle()
8905 MVT SplitVT = MVT::getVectorVT(ScalarVT, NumElements / 2); in splitAndLowerVectorShuffle()
8913 MVT OrigVT = V.getSimpleValueType(); in splitAndLowerVectorShuffle()
8916 MVT OrigScalarVT = OrigVT.getScalarType(); in splitAndLowerVectorShuffle()
8917 MVT OrigSplitVT = MVT::getVectorVT(OrigScalarVT, OrigNumElements / 2); in splitAndLowerVectorShuffle()
9022 static SDValue lowerVectorShuffleAsSplitOrBlend(SDLoc DL, MVT VT, SDValue V1, in lowerVectorShuffleAsSplitOrBlend()
9081 static SDValue lowerVectorShuffleAsLanePermuteAndBlend(SDLoc DL, MVT VT, in lowerVectorShuffleAsLanePermuteAndBlend()
9115 V1, DAG.getConstant(PERMMask, MVT::i8)); in lowerVectorShuffleAsLanePermuteAndBlend()
9126 static SDValue lowerV2X128VectorShuffle(SDLoc DL, MVT VT, SDValue V1, in lowerV2X128VectorShuffle()
9149 MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), in lowerV2X128VectorShuffle()
9203 DAG.getConstant(PermMask, MVT::i8)); in lowerV2X128VectorShuffle()
9219 SDLoc DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask, in lowerVectorShuffleByMerging128BitLanes()
9260 MVT LaneVT = MVT::getVectorVT(VT.isFloatingPoint() ? MVT::f64 : MVT::i64, in lowerVectorShuffleByMerging128BitLanes()
9312 assert(V1.getSimpleValueType() == MVT::v4f64 && "Bad operand type!"); in lowerV4F64VectorShuffle()
9313 assert(V2.getSimpleValueType() == MVT::v4f64 && "Bad operand type!"); in lowerV4F64VectorShuffle()
9320 return lowerV2X128VectorShuffle(DL, MVT::v4f64, V1, V2, Mask, Subtarget, in lowerV4F64VectorShuffle()
9325 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(DL, MVT::v4f64, V1, in lowerV4F64VectorShuffle()
9331 return DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v4f64, V1); in lowerV4F64VectorShuffle()
9333 if (!is128BitLaneCrossingShuffleMask(MVT::v4f64, Mask)) { in lowerV4F64VectorShuffle()
9338 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v4f64, V1, in lowerV4F64VectorShuffle()
9339 DAG.getConstant(VPERMILPMask, MVT::i8)); in lowerV4F64VectorShuffle()
9344 return DAG.getNode(X86ISD::VPERMI, DL, MVT::v4f64, V1, in lowerV4F64VectorShuffle()
9348 return lowerVectorShuffleAsLanePermuteAndBlend(DL, MVT::v4f64, V1, V2, Mask, in lowerV4F64VectorShuffle()
9355 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v4f64, V1, V2); in lowerV4F64VectorShuffle()
9357 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v4f64, V1, V2); in lowerV4F64VectorShuffle()
9359 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v4f64, V2, V1); in lowerV4F64VectorShuffle()
9361 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v4f64, V2, V1); in lowerV4F64VectorShuffle()
9363 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v4f64, V1, V2, Mask, in lowerV4F64VectorShuffle()
9374 return DAG.getNode(X86ISD::SHUFP, DL, MVT::v4f64, V1, V2, in lowerV4F64VectorShuffle()
9375 DAG.getConstant(SHUFPDMask, MVT::i8)); in lowerV4F64VectorShuffle()
9383 return DAG.getNode(X86ISD::SHUFP, DL, MVT::v4f64, V2, V1, in lowerV4F64VectorShuffle()
9384 DAG.getConstant(SHUFPDMask, MVT::i8)); in lowerV4F64VectorShuffle()
9394 DL, MVT::v4f64, V1, V2, Mask, Subtarget, DAG)) in lowerV4F64VectorShuffle()
9400 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v4f64, V1, V2, in lowerV4F64VectorShuffle()
9404 return lowerVectorShuffleAsSplitOrBlend(DL, MVT::v4f64, V1, V2, Mask, DAG); in lowerV4F64VectorShuffle()
9415 assert(V1.getSimpleValueType() == MVT::v4i64 && "Bad operand type!"); in lowerV4I64VectorShuffle()
9416 assert(V2.getSimpleValueType() == MVT::v4i64 && "Bad operand type!"); in lowerV4I64VectorShuffle()
9424 return lowerV2X128VectorShuffle(DL, MVT::v4i64, V1, V2, Mask, Subtarget, in lowerV4I64VectorShuffle()
9427 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v4i64, V1, V2, Mask, in lowerV4I64VectorShuffle()
9432 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(DL, MVT::v4i64, V1, in lowerV4I64VectorShuffle()
9439 if (is128BitLaneRepeatedShuffleMask(MVT::v4i64, Mask, RepeatedMask)) { in lowerV4I64VectorShuffle()
9448 ISD::BITCAST, DL, MVT::v4i64, in lowerV4I64VectorShuffle()
9449 DAG.getNode(X86ISD::PSHUFD, DL, MVT::v8i32, in lowerV4I64VectorShuffle()
9450 DAG.getNode(ISD::BITCAST, DL, MVT::v8i32, V1), in lowerV4I64VectorShuffle()
9458 return DAG.getNode(X86ISD::VPERMI, DL, MVT::v4i64, V1, in lowerV4I64VectorShuffle()
9463 lowerVectorShuffleAsShift(DL, MVT::v4i64, V1, V2, Mask, DAG)) in lowerV4I64VectorShuffle()
9468 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v4i64, V1, V2); in lowerV4I64VectorShuffle()
9470 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v4i64, V1, V2); in lowerV4I64VectorShuffle()
9472 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v4i64, V2, V1); in lowerV4I64VectorShuffle()
9474 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v4i64, V2, V1); in lowerV4I64VectorShuffle()
9483 DL, MVT::v4i64, V1, V2, Mask, Subtarget, DAG)) in lowerV4I64VectorShuffle()
9487 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v4i64, V1, V2, in lowerV4I64VectorShuffle()
9499 assert(V1.getSimpleValueType() == MVT::v8f32 && "Bad operand type!"); in lowerV8F32VectorShuffle()
9500 assert(V2.getSimpleValueType() == MVT::v8f32 && "Bad operand type!"); in lowerV8F32VectorShuffle()
9505 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v8f32, V1, V2, Mask, in lowerV8F32VectorShuffle()
9510 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(DL, MVT::v8f32, V1, in lowerV8F32VectorShuffle()
9517 if (is128BitLaneRepeatedShuffleMask(MVT::v8f32, Mask, RepeatedMask)) { in lowerV8F32VectorShuffle()
9523 return DAG.getNode(X86ISD::MOVSLDUP, DL, MVT::v8f32, V1); in lowerV8F32VectorShuffle()
9525 return DAG.getNode(X86ISD::MOVSHDUP, DL, MVT::v8f32, V1); in lowerV8F32VectorShuffle()
9528 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v8f32, V1, in lowerV8F32VectorShuffle()
9533 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v8f32, V1, V2); in lowerV8F32VectorShuffle()
9535 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v8f32, V1, V2); in lowerV8F32VectorShuffle()
9537 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v8f32, V2, V1); in lowerV8F32VectorShuffle()
9539 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v8f32, V2, V1); in lowerV8F32VectorShuffle()
9547 return lowerVectorShuffleWithSHUFPS(DL, MVT::v8f32, RepeatedMask, V1, V2, DAG); in lowerV8F32VectorShuffle()
9555 VPermMask[i] = Mask[i] < 0 ? DAG.getUNDEF(MVT::i32) in lowerV8F32VectorShuffle()
9556 : DAG.getConstant(Mask[i], MVT::i32); in lowerV8F32VectorShuffle()
9557 if (!is128BitLaneCrossingShuffleMask(MVT::v8f32, Mask)) in lowerV8F32VectorShuffle()
9559 X86ISD::VPERMILPV, DL, MVT::v8f32, V1, in lowerV8F32VectorShuffle()
9560 DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v8i32, VPermMask)); in lowerV8F32VectorShuffle()
9563 return DAG.getNode(X86ISD::VPERMV, DL, MVT::v8f32, in lowerV8F32VectorShuffle()
9564 DAG.getNode(ISD::BITCAST, DL, MVT::v8f32, in lowerV8F32VectorShuffle()
9566 MVT::v8i32, VPermMask)), in lowerV8F32VectorShuffle()
9570 return lowerVectorShuffleAsLanePermuteAndBlend(DL, MVT::v8f32, V1, V2, Mask, in lowerV8F32VectorShuffle()
9577 DL, MVT::v8f32, V1, V2, Mask, Subtarget, DAG)) in lowerV8F32VectorShuffle()
9583 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v8f32, V1, V2, in lowerV8F32VectorShuffle()
9587 return lowerVectorShuffleAsSplitOrBlend(DL, MVT::v8f32, V1, V2, Mask, DAG); in lowerV8F32VectorShuffle()
9598 assert(V1.getSimpleValueType() == MVT::v8i32 && "Bad operand type!"); in lowerV8I32VectorShuffle()
9599 assert(V2.getSimpleValueType() == MVT::v8i32 && "Bad operand type!"); in lowerV8I32VectorShuffle()
9608 if (SDValue ZExt = lowerVectorShuffleAsZeroOrAnyExtend(DL, MVT::v8i32, V1, V2, in lowerV8I32VectorShuffle()
9612 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v8i32, V1, V2, Mask, in lowerV8I32VectorShuffle()
9617 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(DL, MVT::v8i32, V1, in lowerV8I32VectorShuffle()
9625 if (is128BitLaneRepeatedShuffleMask(MVT::v8i32, Mask, RepeatedMask)) { in lowerV8I32VectorShuffle()
9628 return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v8i32, V1, in lowerV8I32VectorShuffle()
9633 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v8i32, V1, V2); in lowerV8I32VectorShuffle()
9635 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v8i32, V1, V2); in lowerV8I32VectorShuffle()
9637 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v8i32, V2, V1); in lowerV8I32VectorShuffle()
9639 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v8i32, V2, V1); in lowerV8I32VectorShuffle()
9644 lowerVectorShuffleAsShift(DL, MVT::v8i32, V1, V2, Mask, DAG)) in lowerV8I32VectorShuffle()
9648 DL, MVT::v8i32, V1, V2, Mask, Subtarget, DAG)) in lowerV8I32VectorShuffle()
9656 VPermMask[i] = Mask[i] < 0 ? DAG.getUNDEF(MVT::i32) in lowerV8I32VectorShuffle()
9657 : DAG.getConstant(Mask[i], MVT::i32); in lowerV8I32VectorShuffle()
9659 X86ISD::VPERMV, DL, MVT::v8i32, in lowerV8I32VectorShuffle()
9660 DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v8i32, VPermMask), V1); in lowerV8I32VectorShuffle()
9666 DL, MVT::v8i32, V1, V2, Mask, Subtarget, DAG)) in lowerV8I32VectorShuffle()
9670 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v8i32, V1, V2, in lowerV8I32VectorShuffle()
9682 assert(V1.getSimpleValueType() == MVT::v16i16 && "Bad operand type!"); in lowerV16I16VectorShuffle()
9683 assert(V2.getSimpleValueType() == MVT::v16i16 && "Bad operand type!"); in lowerV16I16VectorShuffle()
9692 if (SDValue ZExt = lowerVectorShuffleAsZeroOrAnyExtend(DL, MVT::v16i16, V1, V2, in lowerV16I16VectorShuffle()
9697 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(DL, MVT::v16i16, V1, in lowerV16I16VectorShuffle()
9701 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v16i16, V1, V2, Mask, in lowerV16I16VectorShuffle()
9711 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v16i16, V1, V2); in lowerV16I16VectorShuffle()
9717 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v16i16, V1, V2); in lowerV16I16VectorShuffle()
9721 lowerVectorShuffleAsShift(DL, MVT::v16i16, V1, V2, Mask, DAG)) in lowerV16I16VectorShuffle()
9726 DL, MVT::v16i16, V1, V2, Mask, Subtarget, DAG)) in lowerV16I16VectorShuffle()
9732 if (is128BitLaneCrossingShuffleMask(MVT::v16i16, Mask)) in lowerV16I16VectorShuffle()
9733 return lowerVectorShuffleAsLanePermuteAndBlend(DL, MVT::v16i16, V1, V2, in lowerV16I16VectorShuffle()
9737 if (is128BitLaneRepeatedShuffleMask(MVT::v16i16, Mask, RepeatedMask)) { in lowerV16I16VectorShuffle()
9742 DL, MVT::v16i16, V1, RepeatedMask, Subtarget, DAG); in lowerV16I16VectorShuffle()
9748 PSHUFBMask[2 * i] = PSHUFBMask[2 * i + 1] = DAG.getUNDEF(MVT::i8); in lowerV16I16VectorShuffle()
9754 PSHUFBMask[2 * i] = DAG.getConstant(2 * M, MVT::i8); in lowerV16I16VectorShuffle()
9755 PSHUFBMask[2 * i + 1] = DAG.getConstant(2 * M + 1, MVT::i8); in lowerV16I16VectorShuffle()
9758 ISD::BITCAST, DL, MVT::v16i16, in lowerV16I16VectorShuffle()
9760 X86ISD::PSHUFB, DL, MVT::v32i8, in lowerV16I16VectorShuffle()
9761 DAG.getNode(ISD::BITCAST, DL, MVT::v32i8, V1), in lowerV16I16VectorShuffle()
9762 DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v32i8, PSHUFBMask))); in lowerV16I16VectorShuffle()
9768 DL, MVT::v16i16, V1, V2, Mask, Subtarget, DAG)) in lowerV16I16VectorShuffle()
9772 return lowerVectorShuffleAsSplitOrBlend(DL, MVT::v16i16, V1, V2, Mask, DAG); in lowerV16I16VectorShuffle()
9783 assert(V1.getSimpleValueType() == MVT::v32i8 && "Bad operand type!"); in lowerV32I8VectorShuffle()
9784 assert(V2.getSimpleValueType() == MVT::v32i8 && "Bad operand type!"); in lowerV32I8VectorShuffle()
9793 if (SDValue ZExt = lowerVectorShuffleAsZeroOrAnyExtend(DL, MVT::v32i8, V1, V2, in lowerV32I8VectorShuffle()
9798 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(DL, MVT::v32i8, V1, in lowerV32I8VectorShuffle()
9802 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v32i8, V1, V2, Mask, in lowerV32I8VectorShuffle()
9815 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v32i8, V1, V2); in lowerV32I8VectorShuffle()
9822 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v32i8, V1, V2); in lowerV32I8VectorShuffle()
9826 lowerVectorShuffleAsShift(DL, MVT::v32i8, V1, V2, Mask, DAG)) in lowerV32I8VectorShuffle()
9831 DL, MVT::v32i8, V1, V2, Mask, Subtarget, DAG)) in lowerV32I8VectorShuffle()
9837 if (is128BitLaneCrossingShuffleMask(MVT::v32i8, Mask)) in lowerV32I8VectorShuffle()
9838 return lowerVectorShuffleAsLanePermuteAndBlend(DL, MVT::v32i8, V1, V2, in lowerV32I8VectorShuffle()
9845 ? DAG.getUNDEF(MVT::i8) in lowerV32I8VectorShuffle()
9846 : DAG.getConstant(Mask[i] < 16 ? Mask[i] : Mask[i] - 16, MVT::i8); in lowerV32I8VectorShuffle()
9849 X86ISD::PSHUFB, DL, MVT::v32i8, V1, in lowerV32I8VectorShuffle()
9850 DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v32i8, PSHUFBMask)); in lowerV32I8VectorShuffle()
9856 DL, MVT::v32i8, V1, V2, Mask, Subtarget, DAG)) in lowerV32I8VectorShuffle()
9860 return lowerVectorShuffleAsSplitOrBlend(DL, MVT::v32i8, V1, V2, Mask, DAG); in lowerV32I8VectorShuffle()
9869 MVT VT, const X86Subtarget *Subtarget, in lower256BitVectorShuffle()
9899 MVT FpVT = MVT::getVectorVT(MVT::getFloatingPointVT(ElementBits), in lower256BitVectorShuffle()
9908 case MVT::v4f64: in lower256BitVectorShuffle()
9910 case MVT::v4i64: in lower256BitVectorShuffle()
9912 case MVT::v8f32: in lower256BitVectorShuffle()
9914 case MVT::v8i32: in lower256BitVectorShuffle()
9916 case MVT::v16i16: in lower256BitVectorShuffle()
9918 case MVT::v32i8: in lower256BitVectorShuffle()
9931 assert(V1.getSimpleValueType() == MVT::v8f64 && "Bad operand type!"); in lowerV8F64VectorShuffle()
9932 assert(V2.getSimpleValueType() == MVT::v8f64 && "Bad operand type!"); in lowerV8F64VectorShuffle()
9940 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v8f64, V1, V2); in lowerV8F64VectorShuffle()
9942 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v8f64, V1, V2); in lowerV8F64VectorShuffle()
9945 return splitAndLowerVectorShuffle(DL, MVT::v8f64, V1, V2, Mask, DAG); in lowerV8F64VectorShuffle()
9953 assert(V1.getSimpleValueType() == MVT::v16f32 && "Bad operand type!"); in lowerV16F32VectorShuffle()
9954 assert(V2.getSimpleValueType() == MVT::v16f32 && "Bad operand type!"); in lowerV16F32VectorShuffle()
9965 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v16f32, V1, V2); in lowerV16F32VectorShuffle()
9971 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v16f32, V1, V2); in lowerV16F32VectorShuffle()
9974 return splitAndLowerVectorShuffle(DL, MVT::v16f32, V1, V2, Mask, DAG); in lowerV16F32VectorShuffle()
9982 assert(V1.getSimpleValueType() == MVT::v8i64 && "Bad operand type!"); in lowerV8I64VectorShuffle()
9983 assert(V2.getSimpleValueType() == MVT::v8i64 && "Bad operand type!"); in lowerV8I64VectorShuffle()
9991 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v8i64, V1, V2); in lowerV8I64VectorShuffle()
9993 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v8i64, V1, V2); in lowerV8I64VectorShuffle()
9996 return splitAndLowerVectorShuffle(DL, MVT::v8i64, V1, V2, Mask, DAG); in lowerV8I64VectorShuffle()
10004 assert(V1.getSimpleValueType() == MVT::v16i32 && "Bad operand type!"); in lowerV16I32VectorShuffle()
10005 assert(V2.getSimpleValueType() == MVT::v16i32 && "Bad operand type!"); in lowerV16I32VectorShuffle()
10016 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v16i32, V1, V2); in lowerV16I32VectorShuffle()
10022 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v16i32, V1, V2); in lowerV16I32VectorShuffle()
10025 return splitAndLowerVectorShuffle(DL, MVT::v16i32, V1, V2, Mask, DAG); in lowerV16I32VectorShuffle()
10033 assert(V1.getSimpleValueType() == MVT::v32i16 && "Bad operand type!"); in lowerV32I16VectorShuffle()
10034 assert(V2.getSimpleValueType() == MVT::v32i16 && "Bad operand type!"); in lowerV32I16VectorShuffle()
10041 return splitAndLowerVectorShuffle(DL, MVT::v32i16, V1, V2, Mask, DAG); in lowerV32I16VectorShuffle()
10049 assert(V1.getSimpleValueType() == MVT::v64i8 && "Bad operand type!"); in lowerV64I8VectorShuffle()
10050 assert(V2.getSimpleValueType() == MVT::v64i8 && "Bad operand type!"); in lowerV64I8VectorShuffle()
10057 return splitAndLowerVectorShuffle(DL, MVT::v64i8, V1, V2, Mask, DAG); in lowerV64I8VectorShuffle()
10066 MVT VT, const X86Subtarget *Subtarget, in lower512BitVectorShuffle()
10084 case MVT::v8f64: in lower512BitVectorShuffle()
10086 case MVT::v16f32: in lower512BitVectorShuffle()
10088 case MVT::v8i64: in lower512BitVectorShuffle()
10090 case MVT::v16i32: in lower512BitVectorShuffle()
10092 case MVT::v32i16: in lower512BitVectorShuffle()
10096 case MVT::v64i8: in lower512BitVectorShuffle()
10122 MVT VT = Op.getSimpleValueType(); in lowerVectorShuffle()
10166 MVT NewEltVT = VT.isFloatingPoint() in lowerVectorShuffle()
10167 ? MVT::getFloatingPointVT(VT.getScalarSizeInBits() * 2) in lowerVectorShuffle()
10168 : MVT::getIntegerVT(VT.getScalarSizeInBits() * 2); in lowerVectorShuffle()
10169 MVT NewVT = MVT::getVectorVT(NewEltVT, VT.getVectorNumElements() / 2); in lowerVectorShuffle()
10291 MVT VT = Op.getSimpleValueType(); in lowerVSELECTtoVectorShuffle()
10333 case MVT::v32i8: in LowerVSELECT()
10340 case MVT::v8i16: in LowerVSELECT()
10341 case MVT::v16i16: in LowerVSELECT()
10353 MVT VT = Op.getSimpleValueType(); in LowerEXTRACT_VECTOR_ELT_SSE4()
10360 SDValue Extract = DAG.getNode(X86ISD::PEXTRB, dl, MVT::i32, in LowerEXTRACT_VECTOR_ELT_SSE4()
10362 SDValue Assert = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Extract, in LowerEXTRACT_VECTOR_ELT_SSE4()
10371 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, in LowerEXTRACT_VECTOR_ELT_SSE4()
10372 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, in LowerEXTRACT_VECTOR_ELT_SSE4()
10374 MVT::v4i32, in LowerEXTRACT_VECTOR_ELT_SSE4()
10377 SDValue Extract = DAG.getNode(X86ISD::PEXTRW, dl, MVT::i32, in LowerEXTRACT_VECTOR_ELT_SSE4()
10379 SDValue Assert = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Extract, in LowerEXTRACT_VECTOR_ELT_SSE4()
10384 if (VT == MVT::f32) { in LowerEXTRACT_VECTOR_ELT_SSE4()
10397 User->getValueType(0) != MVT::i32)) in LowerEXTRACT_VECTOR_ELT_SSE4()
10399 SDValue Extract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, in LowerEXTRACT_VECTOR_ELT_SSE4()
10400 DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, in LowerEXTRACT_VECTOR_ELT_SSE4()
10403 return DAG.getNode(ISD::BITCAST, dl, MVT::f32, Extract); in LowerEXTRACT_VECTOR_ELT_SSE4()
10406 if (VT == MVT::i32 || VT == MVT::i64) { in LowerEXTRACT_VECTOR_ELT_SSE4()
10420 MVT VecVT = Vec.getSimpleValueType(); in ExtractBitFromMaskVector()
10422 MVT EltVT = Op.getSimpleValueType(); in ExtractBitFromMaskVector()
10424 assert((EltVT == MVT::i1) && "Unexpected operands in ExtractBitFromMaskVector"); in ExtractBitFromMaskVector()
10431 MVT ExtVT = (VecVT == MVT::v8i1 ? MVT::v8i64 : MVT::v16i32); in ExtractBitFromMaskVector()
10441 rc = getRegClassFor(MVT::v16i1); in ExtractBitFromMaskVector()
10444 DAG.getConstant(MaxSift - IdxVal, MVT::i8)); in ExtractBitFromMaskVector()
10446 DAG.getConstant(MaxSift, MVT::i8)); in ExtractBitFromMaskVector()
10447 return DAG.getNode(X86ISD::VEXTRACT, dl, MVT::i1, Vec, in ExtractBitFromMaskVector()
10456 MVT VecVT = Vec.getSimpleValueType(); in LowerEXTRACT_VECTOR_ELT()
10459 if (Op.getSimpleValueType() == MVT::i1) in LowerEXTRACT_VECTOR_ELT()
10467 MVT MaskEltVT = in LowerEXTRACT_VECTOR_ELT()
10468 MVT::getIntegerVT(VecVT.getVectorElementType().getSizeInBits()); in LowerEXTRACT_VECTOR_ELT()
10469 MVT MaskVT = MVT::getVectorVT(MaskEltVT, VecVT.getSizeInBits() / in LowerEXTRACT_VECTOR_ELT()
10490 MVT EltVT = VecVT.getVectorElementType(); in LowerEXTRACT_VECTOR_ELT()
10498 DAG.getConstant(IdxVal, MVT::i32)); in LowerEXTRACT_VECTOR_ELT()
10509 MVT VT = Op.getSimpleValueType(); in LowerEXTRACT_VECTOR_ELT()
10515 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, in LowerEXTRACT_VECTOR_ELT()
10516 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, in LowerEXTRACT_VECTOR_ELT()
10518 MVT::v4i32, Vec), in LowerEXTRACT_VECTOR_ELT()
10521 MVT EltVT = MVT::i32; in LowerEXTRACT_VECTOR_ELT()
10536 MVT VVT = Op.getOperand(0).getSimpleValueType(); in LowerEXTRACT_VECTOR_ELT()
10555 MVT VVT = Op.getOperand(0).getSimpleValueType(); in LowerEXTRACT_VECTOR_ELT()
10573 MVT VecVT = Vec.getSimpleValueType(); in InsertBitToMaskVector()
10578 MVT ExtVecVT = (VecVT == MVT::v8i1 ? MVT::v8i64 : MVT::v16i32); in InsertBitToMaskVector()
10579 MVT ExtEltVT = (VecVT == MVT::v8i1 ? MVT::i64 : MVT::i32); in InsertBitToMaskVector()
10590 DAG.getConstant(IdxVal, MVT::i8)); in InsertBitToMaskVector()
10594 DAG.getConstant(MaxSift, MVT::i8)); in InsertBitToMaskVector()
10596 DAG.getConstant(MaxSift - IdxVal, MVT::i8)); in InsertBitToMaskVector()
10602 MVT VT = Op.getSimpleValueType(); in LowerINSERT_VECTOR_ELT()
10603 MVT EltVT = VT.getVectorElementType(); in LowerINSERT_VECTOR_ELT()
10605 if (EltVT == MVT::i1) in LowerINSERT_VECTOR_ELT()
10626 if ((Subtarget->hasAVX() && (EltVT == MVT::f64 || EltVT == MVT::f32)) || in LowerINSERT_VECTOR_ELT()
10627 (Subtarget->hasAVX2() && EltVT == MVT::i32)) { in LowerINSERT_VECTOR_ELT()
10642 DAG.getConstant(IdxIn128, MVT::i32)); in LowerINSERT_VECTOR_ELT()
10652 if (VT == MVT::v8i16) { in LowerINSERT_VECTOR_ELT()
10655 assert(VT == MVT::v16i8); in LowerINSERT_VECTOR_ELT()
10661 if (N1.getValueType() != MVT::i32) in LowerINSERT_VECTOR_ELT()
10662 N1 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, N1); in LowerINSERT_VECTOR_ELT()
10663 if (N2.getValueType() != MVT::i32) in LowerINSERT_VECTOR_ELT()
10668 if (EltVT == MVT::f32) { in LowerINSERT_VECTOR_ELT()
10689 N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32, N1); in LowerINSERT_VECTOR_ELT()
10694 N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32, N1); in LowerINSERT_VECTOR_ELT()
10698 if (EltVT == MVT::i32 || EltVT == MVT::i64) { in LowerINSERT_VECTOR_ELT()
10704 if (EltVT == MVT::i8) in LowerINSERT_VECTOR_ELT()
10710 if (N1.getValueType() != MVT::i32) in LowerINSERT_VECTOR_ELT()
10711 N1 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, N1); in LowerINSERT_VECTOR_ELT()
10712 if (N2.getValueType() != MVT::i32) in LowerINSERT_VECTOR_ELT()
10721 MVT OpVT = Op.getSimpleValueType(); in LowerSCALAR_TO_VECTOR()
10728 MVT VT128 = MVT::getVectorVT(OpVT.getVectorElementType(), in LowerSCALAR_TO_VECTOR()
10737 if (OpVT == MVT::v1i64 && in LowerSCALAR_TO_VECTOR()
10738 Op.getOperand(0).getValueType() == MVT::i64) in LowerSCALAR_TO_VECTOR()
10739 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v1i64, Op.getOperand(0)); in LowerSCALAR_TO_VECTOR()
10741 SDValue AnyExt = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, Op.getOperand(0)); in LowerSCALAR_TO_VECTOR()
10744 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32,AnyExt)); in LowerSCALAR_TO_VECTOR()
10756 MVT ResVT = Op.getSimpleValueType(); in LowerEXTRACT_SUBVECTOR()
10757 MVT InVT = In.getSimpleValueType(); in LowerEXTRACT_SUBVECTOR()
10790 MVT OpVT = Op.getSimpleValueType(); in LowerINSERT_SUBVECTOR()
10791 MVT SubVecVT = SubVec.getSimpleValueType(); in LowerINSERT_SUBVECTOR()
10819 if (OpVT.getVectorElementType() == MVT::i1) { in LowerINSERT_SUBVECTOR()
10825 SDValue ShiftBits = DAG.getConstant(NumElems/2, MVT::i8); in LowerINSERT_SUBVECTOR()
11057 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); in GetTLSADDR()
11255 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); in LowerGlobalTLSAddress()
11310 IDX, MachinePointerInfo(), MVT::i32, in LowerGlobalTLSAddress()
11342 MVT VT = Op.getSimpleValueType(); in LowerShiftParts()
11352 SDValue SafeShAmt = DAG.getNode(ISD::AND, dl, MVT::i8, ShAmt, in LowerShiftParts()
11353 DAG.getConstant(VTBits - 1, MVT::i8)); in LowerShiftParts()
11355 DAG.getConstant(VTBits - 1, MVT::i8)) in LowerShiftParts()
11370 SDValue AndNode = DAG.getNode(ISD::AND, dl, MVT::i8, ShAmt, in LowerShiftParts()
11371 DAG.getConstant(VTBits, MVT::i8)); in LowerShiftParts()
11372 SDValue Cond = DAG.getNode(X86ISD::CMP, dl, MVT::i32, in LowerShiftParts()
11373 AndNode, DAG.getConstant(0, MVT::i8)); in LowerShiftParts()
11376 SDValue CC = DAG.getConstant(X86::COND_NE, MVT::i8); in LowerShiftParts()
11394 MVT SrcVT = Op.getOperand(0).getSimpleValueType(); in LowerSINT_TO_FP()
11398 if (SrcVT.getVectorElementType() == MVT::i1) { in LowerSINT_TO_FP()
11399 MVT IntegerVT = MVT::getVectorVT(MVT::i32, SrcVT.getVectorNumElements()); in LowerSINT_TO_FP()
11407 assert(SrcVT <= MVT::i64 && SrcVT >= MVT::i16 && in LowerSINT_TO_FP()
11412 if (SrcVT == MVT::i32 && isScalarFPTypeInSSEReg(Op.getValueType())) in LowerSINT_TO_FP()
11414 if (SrcVT == MVT::i64 && isScalarFPTypeInSSEReg(Op.getValueType()) && in LowerSINT_TO_FP()
11438 Tys = DAG.getVTList(MVT::f64, MVT::Other, MVT::Glue); in BuildFILD()
11440 Tys = DAG.getVTList(Op.getValueType(), MVT::Other); in BuildFILD()
11472 Tys = DAG.getVTList(MVT::Other); in BuildFILD()
11526 SDValue XR1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, in LowerUINT_TO_FP_i64()
11528 SDValue CLod0 = DAG.getLoad(MVT::v4i32, dl, DAG.getEntryNode(), CPIdx0, in LowerUINT_TO_FP_i64()
11531 SDValue Unpck1 = getUnpackl(DAG, dl, MVT::v4i32, in LowerUINT_TO_FP_i64()
11532 DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, XR1), in LowerUINT_TO_FP_i64()
11535 SDValue CLod1 = DAG.getLoad(MVT::v2f64, dl, CLod0.getValue(1), CPIdx1, in LowerUINT_TO_FP_i64()
11538 SDValue XR2F = DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, Unpck1); in LowerUINT_TO_FP_i64()
11539 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::v2f64, XR2F, CLod1); in LowerUINT_TO_FP_i64()
11544 Result = DAG.getNode(X86ISD::FHADD, dl, MVT::v2f64, Sub, Sub); in LowerUINT_TO_FP_i64()
11546 SDValue S2F = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Sub); in LowerUINT_TO_FP_i64()
11547 SDValue Shuffle = getTargetShuffleNode(X86ISD::PSHUFD, dl, MVT::v4i32, in LowerUINT_TO_FP_i64()
11549 Result = DAG.getNode(ISD::FADD, dl, MVT::v2f64, in LowerUINT_TO_FP_i64()
11550 DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, Shuffle), in LowerUINT_TO_FP_i64()
11554 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Result, in LowerUINT_TO_FP_i64()
11564 MVT::f64); in LowerUINT_TO_FP_i32()
11567 SDValue Load = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, in LowerUINT_TO_FP_i32()
11573 Load = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, in LowerUINT_TO_FP_i32()
11574 DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, Load), in LowerUINT_TO_FP_i32()
11578 SDValue Or = DAG.getNode(ISD::OR, dl, MVT::v2i64, in LowerUINT_TO_FP_i32()
11579 DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, in LowerUINT_TO_FP_i32()
11581 MVT::v2f64, Load)), in LowerUINT_TO_FP_i32()
11582 DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, in LowerUINT_TO_FP_i32()
11584 MVT::v2f64, Bias))); in LowerUINT_TO_FP_i32()
11585 Or = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, in LowerUINT_TO_FP_i32()
11586 DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, Or), in LowerUINT_TO_FP_i32()
11590 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Or, Bias); in LowerUINT_TO_FP_i32()
11595 if (DestVT.bitsLT(MVT::f64)) in LowerUINT_TO_FP_i32()
11598 if (DestVT.bitsGT(MVT::f64)) in LowerUINT_TO_FP_i32()
11622 bool Is128 = VecIntVT == MVT::v4i32; in lowerUINT_TO_FP_vXi32()
11623 EVT VecFloatVT = Is128 ? MVT::v4f32 : MVT::v8f32; in lowerUINT_TO_FP_vXi32()
11630 assert((VecIntVT == MVT::v4i32 || VecIntVT == MVT::v8i32) && in lowerUINT_TO_FP_vXi32()
11642 SDValue CstLow = DAG.getConstant(0x4b000000, MVT::i32); in lowerUINT_TO_FP_vXi32()
11648 SDValue CstHigh = DAG.getConstant(0x53000000, MVT::i32); in lowerUINT_TO_FP_vXi32()
11655 SDValue CstShift = DAG.getConstant(16, MVT::i32); in lowerUINT_TO_FP_vXi32()
11664 EVT VecI16VT = Is128 ? MVT::v8i16 : MVT::v16i16; in lowerUINT_TO_FP_vXi32()
11672 VecCstLowBitcast, DAG.getConstant(0xaa, MVT::i32)); in lowerUINT_TO_FP_vXi32()
11682 VecCstHighBitcast, DAG.getConstant(0xaa, MVT::i32)); in lowerUINT_TO_FP_vXi32()
11684 SDValue CstMask = DAG.getConstant(0xffff, MVT::i32); in lowerUINT_TO_FP_vXi32()
11697 APFloat(APFloat::IEEEsingle, APInt(32, 0xD3000080)), MVT::f32); in lowerUINT_TO_FP_vXi32()
11715 MVT SVT = N0.getSimpleValueType(); in lowerUINT_TO_FP_vec()
11721 case MVT::v4i8: in lowerUINT_TO_FP_vec()
11722 case MVT::v4i16: in lowerUINT_TO_FP_vec()
11723 case MVT::v8i8: in lowerUINT_TO_FP_vec()
11724 case MVT::v8i16: { in lowerUINT_TO_FP_vec()
11725 MVT NVT = MVT::getVectorVT(MVT::i32, SVT.getVectorNumElements()); in lowerUINT_TO_FP_vec()
11729 case MVT::v4i32: in lowerUINT_TO_FP_vec()
11730 case MVT::v8i32: in lowerUINT_TO_FP_vec()
11750 MVT SrcVT = N0.getSimpleValueType(); in LowerUINT_TO_FP()
11751 MVT DstVT = Op.getSimpleValueType(); in LowerUINT_TO_FP()
11752 if (SrcVT == MVT::i64 && DstVT == MVT::f64 && X86ScalarSSEf64) in LowerUINT_TO_FP()
11754 if (SrcVT == MVT::i32 && X86ScalarSSEf64) in LowerUINT_TO_FP()
11756 if (Subtarget->is64Bit() && SrcVT == MVT::i64 && DstVT == MVT::f32) in LowerUINT_TO_FP()
11760 SDValue StackSlot = DAG.CreateStackTemporary(MVT::i64); in LowerUINT_TO_FP()
11761 if (SrcVT == MVT::i32) { in LowerUINT_TO_FP()
11768 SDValue Store2 = DAG.getStore(Store1, dl, DAG.getConstant(0, MVT::i32), in LowerUINT_TO_FP()
11771 SDValue Fild = BuildFILD(Op, MVT::i64, Store2, StackSlot, DAG); in LowerUINT_TO_FP()
11775 assert(SrcVT == MVT::i64 && "Unexpected type in UINT_TO_FP"); in LowerUINT_TO_FP()
11790 SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other); in LowerUINT_TO_FP()
11791 SDValue Ops[] = { Store, StackSlot, DAG.getValueType(MVT::i64) }; in LowerUINT_TO_FP()
11793 MVT::i64, MMO); in LowerUINT_TO_FP()
11799 getSetCCResultType(*DAG.getContext(), MVT::i64), in LowerUINT_TO_FP()
11800 Op.getOperand(0), DAG.getConstant(0, MVT::i64), in LowerUINT_TO_FP()
11817 SDValue Fudge = DAG.getExtLoad(ISD::EXTLOAD, dl, MVT::f80, DAG.getEntryNode(), in LowerUINT_TO_FP()
11819 MVT::f32, false, false, false, 4); in LowerUINT_TO_FP()
11821 SDValue Add = DAG.getNode(ISD::FADD, dl, MVT::f80, Fild, Fudge); in LowerUINT_TO_FP()
11833 assert(DstTy == MVT::i32 && "Unexpected FP_TO_UINT"); in FP_TO_INTHelper()
11834 DstTy = MVT::i64; in FP_TO_INTHelper()
11837 assert(DstTy.getSimpleVT() <= MVT::i64 && in FP_TO_INTHelper()
11838 DstTy.getSimpleVT() >= MVT::i16 && in FP_TO_INTHelper()
11842 if (DstTy == MVT::i32 && in FP_TO_INTHelper()
11846 DstTy == MVT::i64 && in FP_TO_INTHelper()
11863 case MVT::i16: Opc = X86ISD::FP_TO_INT16_IN_MEM; break; in FP_TO_INTHelper()
11864 case MVT::i32: Opc = X86ISD::FP_TO_INT32_IN_MEM; break; in FP_TO_INTHelper()
11865 case MVT::i64: Opc = X86ISD::FP_TO_INT64_IN_MEM; break; in FP_TO_INTHelper()
11874 assert(DstTy == MVT::i64 && "Invalid FP_TO_SINT to lower!"); in FP_TO_INTHelper()
11878 SDVTList Tys = DAG.getVTList(Op.getOperand(0).getValueType(), MVT::Other); in FP_TO_INTHelper()
11899 SDValue FIST = DAG.getMemIntrinsicNode(Opc, DL, DAG.getVTList(MVT::Other), in FP_TO_INTHelper()
11904 DAG.getVTList(MVT::Other, MVT::Glue), in FP_TO_INTHelper()
11907 MVT::i32, ftol.getValue(1)); in FP_TO_INTHelper()
11909 MVT::i32, eax.getValue(2)); in FP_TO_INTHelper()
11912 ? DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Ops) in FP_TO_INTHelper()
11920 MVT VT = Op->getSimpleValueType(0); in LowerAVXExtend()
11922 MVT InVT = In.getSimpleValueType(); in LowerAVXExtend()
11938 if (((VT != MVT::v16i16) || (InVT != MVT::v16i8)) && in LowerAVXExtend()
11939 ((VT != MVT::v8i32) || (InVT != MVT::v8i16)) && in LowerAVXExtend()
11940 ((VT != MVT::v4i64) || (InVT != MVT::v4i32))) in LowerAVXExtend()
11952 MVT HVT = MVT::getVectorVT(VT.getVectorElementType(), in LowerAVXExtend()
11963 MVT VT = Op->getSimpleValueType(0); in LowerZERO_EXTEND_AVX512()
11965 MVT InVT = In.getSimpleValueType(); in LowerZERO_EXTEND_AVX512()
11971 if (VT.is512BitVector() && InVT.getVectorElementType() != MVT::i1) in LowerZERO_EXTEND_AVX512()
11974 EVT ExtVT = (NumElts == 8)? MVT::v8i64 : MVT::v16i32; in LowerZERO_EXTEND_AVX512()
11977 assert(InVT.getVectorElementType() == MVT::i1); in LowerZERO_EXTEND_AVX512()
12006 MVT VT = Op.getSimpleValueType(); in LowerZERO_EXTEND()
12008 MVT SVT = In.getSimpleValueType(); in LowerZERO_EXTEND()
12010 if (VT.is512BitVector() || SVT.getVectorElementType() == MVT::i1) in LowerZERO_EXTEND()
12026 MVT VT = Op.getSimpleValueType(); in LowerTRUNCATE()
12028 MVT InVT = In.getSimpleValueType(); in LowerTRUNCATE()
12030 if (VT == MVT::i1) { in LowerTRUNCATE()
12035 In = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, In); in LowerTRUNCATE()
12041 if (InVT.is512BitVector() || VT.getVectorElementType() == MVT::i1) { in LowerTRUNCATE()
12045 assert(VT.getVectorElementType() == MVT::i1 && "Unexpected vector type"); in LowerTRUNCATE()
12049 MVT ExtVT = (NumElts == 16)? MVT::v16i32 : MVT::v8i64; in LowerTRUNCATE()
12066 if ((VT == MVT::v4i32) && (InVT == MVT::v4i64)) { in LowerTRUNCATE()
12070 In = DAG.getNode(ISD::BITCAST, DL, MVT::v8i32, In); in LowerTRUNCATE()
12071 In = DAG.getVectorShuffle(MVT::v8i32, DL, In, DAG.getUNDEF(MVT::v8i32), in LowerTRUNCATE()
12077 SDValue OpLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In, in LowerTRUNCATE()
12079 SDValue OpHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In, in LowerTRUNCATE()
12081 OpLo = DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, OpLo); in LowerTRUNCATE()
12082 OpHi = DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, OpHi); in LowerTRUNCATE()
12087 if ((VT == MVT::v8i16) && (InVT == MVT::v8i32)) { in LowerTRUNCATE()
12090 In = DAG.getNode(ISD::BITCAST, DL, MVT::v32i8, In); in LowerTRUNCATE()
12094 pshufbMask.push_back(DAG.getConstant(0x0, MVT::i8)); in LowerTRUNCATE()
12095 pshufbMask.push_back(DAG.getConstant(0x1, MVT::i8)); in LowerTRUNCATE()
12096 pshufbMask.push_back(DAG.getConstant(0x4, MVT::i8)); in LowerTRUNCATE()
12097 pshufbMask.push_back(DAG.getConstant(0x5, MVT::i8)); in LowerTRUNCATE()
12098 pshufbMask.push_back(DAG.getConstant(0x8, MVT::i8)); in LowerTRUNCATE()
12099 pshufbMask.push_back(DAG.getConstant(0x9, MVT::i8)); in LowerTRUNCATE()
12100 pshufbMask.push_back(DAG.getConstant(0xc, MVT::i8)); in LowerTRUNCATE()
12101 pshufbMask.push_back(DAG.getConstant(0xd, MVT::i8)); in LowerTRUNCATE()
12103 pshufbMask.push_back(DAG.getConstant(0x80, MVT::i8)); in LowerTRUNCATE()
12105 SDValue BV = DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v32i8, pshufbMask); in LowerTRUNCATE()
12106 In = DAG.getNode(X86ISD::PSHUFB, DL, MVT::v32i8, In, BV); in LowerTRUNCATE()
12107 In = DAG.getNode(ISD::BITCAST, DL, MVT::v4i64, In); in LowerTRUNCATE()
12110 In = DAG.getVectorShuffle(MVT::v4i64, DL, In, DAG.getUNDEF(MVT::v4i64), in LowerTRUNCATE()
12112 In = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In, in LowerTRUNCATE()
12117 SDValue OpLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i32, In, in LowerTRUNCATE()
12120 SDValue OpHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i32, In, in LowerTRUNCATE()
12123 OpLo = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, OpLo); in LowerTRUNCATE()
12124 OpHi = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, OpHi); in LowerTRUNCATE()
12130 SDValue Undef = DAG.getUNDEF(MVT::v16i8); in LowerTRUNCATE()
12131 OpLo = DAG.getVectorShuffle(MVT::v16i8, DL, OpLo, Undef, ShufMask1); in LowerTRUNCATE()
12132 OpHi = DAG.getVectorShuffle(MVT::v16i8, DL, OpHi, Undef, ShufMask1); in LowerTRUNCATE()
12134 OpLo = DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, OpLo); in LowerTRUNCATE()
12135 OpHi = DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, OpHi); in LowerTRUNCATE()
12139 SDValue res = DAG.getVectorShuffle(MVT::v4i32, DL, OpLo, OpHi, ShufMask2); in LowerTRUNCATE()
12140 return DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, res); in LowerTRUNCATE()
12150 MVT NVT = MVT::getVectorVT(VT.getVectorElementType(), NumElems * 2); in LowerTRUNCATE()
12202 MVT VT = Op.getSimpleValueType(); in LowerFP_EXTEND()
12204 MVT SVT = In.getSimpleValueType(); in LowerFP_EXTEND()
12206 assert(SVT == MVT::v2f32 && "Only customize MVT::v2f32 type legalization!"); in LowerFP_EXTEND()
12209 DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4f32, in LowerFP_EXTEND()
12232 MVT VT = Op.getSimpleValueType(); in LowerFABSorFNEG()
12238 MVT EltVT = VT; in LowerFABSorFNEG()
12239 unsigned NumElts = VT == MVT::f64 ? 2 : 4; in LowerFABSorFNEG()
12265 MVT VecVT = MVT::getVectorVT(MVT::i64, VT.getSizeInBits() / 64); in LowerFABSorFNEG()
12287 MVT VT = Op.getSimpleValueType(); in LowerFCOPYSIGN()
12288 MVT SrcVT = Op1.getSimpleValueType(); in LowerFCOPYSIGN()
12305 VT == MVT::f64 ? APFloat::IEEEdouble : APFloat::IEEEsingle; in LowerFCOPYSIGN()
12309 VT == MVT::f64 ? 2 : 4, in LowerFCOPYSIGN()
12352 MVT VT = Op.getSimpleValueType(); in LowerFGETSIGN()
12377 EVT VT = MVT::Other; in LowerVectorAllZeroTest()
12433 EVT TestVT = VT.is128BitVector() ? MVT::v2i64 : MVT::v4i64; in LowerVectorAllZeroTest()
12448 return DAG.getNode(X86ISD::PTEST, DL, MVT::i32, in LowerVectorAllZeroTest()
12475 if (Op.getValueType() == MVT::i1) { in EmitTest()
12476 SDValue ExtOp = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i8, Op); in EmitTest()
12477 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, ExtOp, in EmitTest()
12478 DAG.getConstant(0, MVT::i8)); in EmitTest()
12521 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op, in EmitTest()
12695 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op, in EmitTest()
12698 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32); in EmitTest()
12714 if (Op0.getValueType() == MVT::i1) in EmitCmp()
12718 if ((Op0.getValueType() == MVT::i8 || Op0.getValueType() == MVT::i16 || in EmitCmp()
12719 Op0.getValueType() == MVT::i32 || Op0.getValueType() == MVT::i64)) { in EmitCmp()
12724 if (Op0.getValueType() != MVT::i32 && Op0.getValueType() != MVT::i64 && in EmitCmp()
12730 Op0 = DAG.getNode(ExtendOp, dl, MVT::i32, Op0); in EmitCmp()
12731 Op1 = DAG.getNode(ExtendOp, dl, MVT::i32, Op1); in EmitCmp()
12734 SDVTList VTs = DAG.getVTList(Op0.getValueType(), MVT::i32); in EmitCmp()
12739 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op0, Op1); in EmitCmp()
12758 SDValue TruncFPSW = DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, Cmp); in ConvertCmpIfNecessary()
12759 SDValue FNStSW = DAG.getNode(X86ISD::FNSTSW16r, dl, MVT::i16, TruncFPSW); in ConvertCmpIfNecessary()
12760 SDValue Srl = DAG.getNode(ISD::SRL, dl, MVT::i16, FNStSW, in ConvertCmpIfNecessary()
12761 DAG.getConstant(8, MVT::i8)); in ConvertCmpIfNecessary()
12762 SDValue TruncSrl = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Srl); in ConvertCmpIfNecessary()
12763 return DAG.getNode(X86ISD::SAHF, dl, MVT::i32, TruncSrl); in ConvertCmpIfNecessary()
12788 if ((Subtarget->hasSSE1() && (VT == MVT::f32 || VT == MVT::v4f32)) || in getRsqrtEstimate()
12789 (Subtarget->hasAVX() && VT == MVT::v8f32)) { in getRsqrtEstimate()
12818 if ((Subtarget->hasSSE1() && (VT == MVT::f32 || VT == MVT::v4f32)) || in getRecipEstimate()
12819 (Subtarget->hasAVX() && VT == MVT::v8f32)) { in getRecipEstimate()
12894 if (LHS.getValueType() == MVT::i8 || in LowerToBT()
12895 LHS.getValueType() == MVT::i16) in LowerToBT()
12896 LHS = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, LHS); in LowerToBT()
12903 SDValue BT = DAG.getNode(X86ISD::BT, dl, MVT::i32, LHS, RHS); in LowerToBT()
12905 return DAG.getNode(X86ISD::SETCC, dl, MVT::i8, in LowerToBT()
12906 DAG.getConstant(Cond, MVT::i8), BT); in LowerToBT()
12960 MVT VT = Op.getSimpleValueType(); in Lower256IntVSETCC()
12980 MVT EltVT = VT.getVectorElementType(); in Lower256IntVSETCC()
12981 MVT NewVT = MVT::getVectorVT(EltVT, NumElems/2); in Lower256IntVSETCC()
12992 MVT VT = Op.getSimpleValueType(); in LowerIntVSETCC_AVX512()
12996 Op.getValueType().getScalarType() == MVT::i1 && in LowerIntVSETCC_AVX512()
13024 DAG.getConstant(SSECC, MVT::i8)); in LowerIntVSETCC_AVX512()
13036 MVT VT = Op1.getSimpleValueType(); in ChangeVSETULTtoVSETULE()
13037 MVT EVT = VT.getVectorElementType(); in ChangeVSETULTtoVSETULE()
13062 MVT VT = Op.getSimpleValueType(); in LowerVSETCC()
13069 MVT EltVT = Op0.getSimpleValueType().getVectorElementType(); in LowerVSETCC()
13070 assert(EltVT == MVT::f32 || EltVT == MVT::f64); in LowerVSETCC()
13075 if (Subtarget->hasAVX512() && VT.getVectorElementType() == MVT::i1) { in LowerVSETCC()
13091 DAG.getConstant(CC0, MVT::i8)); in LowerVSETCC()
13093 DAG.getConstant(CC1, MVT::i8)); in LowerVSETCC()
13098 DAG.getConstant(SSECC, MVT::i8)); in LowerVSETCC()
13105 bool MaskResult = (VT.getVectorElementType() == MVT::i1); in LowerVSETCC()
13149 MVT VET = VT.getVectorElementType(); in LowerVSETCC()
13151 (Subtarget->hasSSE41() && (VET >= MVT::i8 && VET <= MVT::i32)) in LowerVSETCC()
13152 || (Subtarget->hasSSE2() && (VET == MVT::i8)); in LowerVSETCC()
13164 bool hasSubus = Subtarget->hasSSE2() && (VET == MVT::i8 || VET == MVT::i16); in LowerVSETCC()
13203 if (VT == MVT::v2i64) { in LowerVSETCC()
13208 Op0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Op0); in LowerVSETCC()
13209 Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Op1); in LowerVSETCC()
13216 SB = DAG.getConstant(0x80000000U, MVT::v4i32); in LowerVSETCC()
13218 SDValue Sign = DAG.getConstant(0x80000000U, MVT::i32); in LowerVSETCC()
13219 SDValue Zero = DAG.getConstant(0x00000000U, MVT::i32); in LowerVSETCC()
13220 SB = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, in LowerVSETCC()
13223 Op0 = DAG.getNode(ISD::XOR, dl, MVT::v4i32, Op0, SB); in LowerVSETCC()
13224 Op1 = DAG.getNode(ISD::XOR, dl, MVT::v4i32, Op1, SB); in LowerVSETCC()
13227 SDValue GT = DAG.getNode(X86ISD::PCMPGT, dl, MVT::v4i32, Op0, Op1); in LowerVSETCC()
13228 SDValue EQ = DAG.getNode(X86ISD::PCMPEQ, dl, MVT::v4i32, Op0, Op1); in LowerVSETCC()
13233 SDValue EQHi = DAG.getVectorShuffle(MVT::v4i32, dl, EQ, EQ, MaskHi); in LowerVSETCC()
13234 SDValue GTLo = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskLo); in LowerVSETCC()
13235 SDValue GTHi = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskHi); in LowerVSETCC()
13237 SDValue Result = DAG.getNode(ISD::AND, dl, MVT::v4i32, EQHi, GTLo); in LowerVSETCC()
13238 Result = DAG.getNode(ISD::OR, dl, MVT::v4i32, Result, GTHi); in LowerVSETCC()
13241 Result = DAG.getNOT(dl, Result, MVT::v4i32); in LowerVSETCC()
13252 Op0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Op0); in LowerVSETCC()
13253 Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Op1); in LowerVSETCC()
13256 SDValue Result = DAG.getNode(Opc, dl, MVT::v4i32, Op0, Op1); in LowerVSETCC()
13260 SDValue Shuf = DAG.getVectorShuffle(MVT::v4i32, dl, Result, Result, Mask); in LowerVSETCC()
13261 Result = DAG.getNode(ISD::AND, dl, MVT::v4i32, Result, Shuf); in LowerVSETCC()
13264 Result = DAG.getNOT(dl, Result, MVT::v4i32); in LowerVSETCC()
13297 MVT VT = Op.getSimpleValueType(); in LowerSETCC()
13301 assert(((!Subtarget->hasAVX512() && VT == MVT::i8) || (VT == MVT::i1)) in LowerSETCC()
13318 if (VT == MVT::i1) in LowerSETCC()
13319 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, NewSetCC); in LowerSETCC()
13341 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, in LowerSETCC()
13342 DAG.getConstant(CCode, MVT::i8), in LowerSETCC()
13344 if (VT == MVT::i1) in LowerSETCC()
13345 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, SetCC); in LowerSETCC()
13349 if ((Op0.getValueType() == MVT::i1) && (Op1.getOpcode() == ISD::Constant) && in LowerSETCC()
13354 return DAG.getSetCC(dl, VT, Op0, DAG.getConstant(0, MVT::i1), NewCC); in LowerSETCC()
13364 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, in LowerSETCC()
13365 DAG.getConstant(X86CC, MVT::i8), EFLAGS); in LowerSETCC()
13366 if (VT == MVT::i1) in LowerSETCC()
13367 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, SetCC); in LowerSETCC()
13420 ((Subtarget->hasSSE2() && (VT == MVT::f32 || VT == MVT::f64)) || in LowerSELECT()
13421 (Subtarget->hasSSE1() && VT == MVT::f32)) && in LowerSELECT()
13429 SDValue Cmp = DAG.getNode(X86ISD::FSETCC, DL, MVT::i1, CondOp0, CondOp1, in LowerSELECT()
13430 DAG.getConstant(SSECC, MVT::i8)); in LowerSELECT()
13435 DAG.getConstant(SSECC, MVT::i8)); in LowerSELECT()
13456 EVT VecVT = VT == MVT::f32 ? MVT::v4f32 : MVT::v2f64; in LowerSELECT()
13461 EVT VCmpVT = VT == MVT::f32 ? MVT::v4i32 : MVT::v2i64; in LowerSELECT()
13503 SDVTList VTs = DAG.getVTList(CmpOp0.getValueType(), MVT::i32); in LowerSELECT()
13508 DAG.getConstant(X86::COND_B, MVT::i8), in LowerSELECT()
13513 Cmp = DAG.getNode(X86ISD::CMP, DL, MVT::i32, in LowerSELECT()
13519 DAG.getConstant(X86::COND_B, MVT::i8), Cmp); in LowerSELECT()
13548 MVT VT = Op.getSimpleValueType(); in LowerSELECT()
13563 Cond.getOperand(0).getValueType() != MVT::i8)) { in LowerSELECT()
13580 MVT::i32); in LowerSELECT()
13582 VTs = DAG.getVTList(LHS.getValueType(), MVT::i32); in LowerSELECT()
13591 CC = DAG.getConstant(X86Cond, MVT::i8); in LowerSELECT()
13613 CC = DAG.getConstant(X86::COND_NE, MVT::i8); in LowerSELECT()
13628 DAG.getConstant(X86::COND_B, MVT::i8), Cond); in LowerSELECT()
13638 if (Op.getValueType() == MVT::i8 && in LowerSELECT()
13644 SDVTList VTs = DAG.getVTList(T1.getValueType(), MVT::Glue); in LowerSELECT()
13652 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Glue); in LowerSELECT()
13659 MVT VT = Op->getSimpleValueType(0); in LowerSIGN_EXTEND_AVX512()
13661 MVT InVT = In.getSimpleValueType(); in LowerSIGN_EXTEND_AVX512()
13662 MVT VTElt = VT.getVectorElementType(); in LowerSIGN_EXTEND_AVX512()
13663 MVT InVTElt = InVT.getVectorElementType(); in LowerSIGN_EXTEND_AVX512()
13667 if ((InVTElt == MVT::i1) && in LowerSIGN_EXTEND_AVX512()
13686 if (VT.is512BitVector() && InVT.getVectorElementType() != MVT::i1) { in LowerSIGN_EXTEND_AVX512()
13693 assert (InVT.getVectorElementType() == MVT::i1 && "Unexpected vector type"); in LowerSIGN_EXTEND_AVX512()
13695 MVT ExtVT = (NumElts == 8) ? MVT::v8i64 : MVT::v16i32; in LowerSIGN_EXTEND_AVX512()
13712 MVT VT = Op->getSimpleValueType(0); in LowerSIGN_EXTEND()
13714 MVT InVT = In.getSimpleValueType(); in LowerSIGN_EXTEND()
13717 if (VT.is512BitVector() || InVT.getVectorElementType() == MVT::i1) in LowerSIGN_EXTEND()
13720 if ((VT != MVT::v4i64 || InVT != MVT::v4i32) && in LowerSIGN_EXTEND()
13721 (VT != MVT::v8i32 || InVT != MVT::v8i16) && in LowerSIGN_EXTEND()
13722 (VT != MVT::v16i16 || InVT != MVT::v16i8)) in LowerSIGN_EXTEND()
13752 MVT HalfVT = MVT::getVectorVT(VT.getScalarType(), in LowerSIGN_EXTEND()
13770 MVT RegVT = Op.getSimpleValueType(); in LowerExtendedLoad()
13844 MVT SclrLoadTy = MVT::i8; in LowerExtendedLoad()
13845 for (MVT Tp : MVT::integer_valuetypes()) { in LowerExtendedLoad()
13852 if (TLI.isTypeLegal(MVT::f64) && SclrLoadTy.getSizeInBits() < 64 && in LowerExtendedLoad()
13854 SclrLoadTy = MVT::f64; in LowerExtendedLoad()
13909 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains); in LowerExtendedLoad()
14066 Cond.getOperand(0).getValueType() != MVT::i8)) { in LowerBRCOND()
14100 MVT::i32); in LowerBRCOND()
14102 VTs = DAG.getVTList(LHS.getValueType(), MVT::i32); in LowerBRCOND()
14111 CC = DAG.getConstant(X86Cond, MVT::i8); in LowerBRCOND()
14142 CC = DAG.getConstant(CCode, MVT::i8); in LowerBRCOND()
14160 CC = DAG.getConstant(CCode, MVT::i8); in LowerBRCOND()
14173 CC = DAG.getConstant(CCode, MVT::i8); in LowerBRCOND()
14196 SDValue Cmp = DAG.getNode(X86ISD::CMP, dl, MVT::i32, in LowerBRCOND()
14199 CC = DAG.getConstant(X86::COND_NE, MVT::i8); in LowerBRCOND()
14202 CC = DAG.getConstant(X86::COND_P, MVT::i8); in LowerBRCOND()
14226 SDValue Cmp = DAG.getNode(X86ISD::CMP, dl, MVT::i32, in LowerBRCOND()
14229 CC = DAG.getConstant(X86::COND_NE, MVT::i8); in LowerBRCOND()
14232 CC = DAG.getConstant(X86::COND_NP, MVT::i8); in LowerBRCOND()
14260 CC = DAG.getConstant(X86Cond, MVT::i8); in LowerBRCOND()
14358 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); in LowerDYNAMIC_STACKALLOC()
14404 MVT::i32), in LowerVASTART()
14413 MVT::i32), in LowerVASTART()
14435 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps); in LowerVASTART()
14459 if (ArgVT == MVT::f80) { in LowerVAARG()
14479 SDValue InstOps[] = {Chain, SrcPtr, DAG.getConstant(ArgSize, MVT::i32), in LowerVAARG()
14480 DAG.getConstant(ArgMode, MVT::i8), in LowerVAARG()
14481 DAG.getConstant(Align, MVT::i32)}; in LowerVAARG()
14482 SDVTList VTs = DAG.getVTList(getPointerTy(), MVT::Other); in LowerVAARG()
14484 VTs, InstOps, MVT::i64, in LowerVAARG()
14519 static SDValue getTargetVShiftByConstNode(unsigned Opc, SDLoc dl, MVT VT, in getTargetVShiftByConstNode()
14522 MVT ElementType = VT.getVectorElementType(); in getTargetVShiftByConstNode()
14590 return DAG.getNode(Opc, dl, VT, SrcOp, DAG.getConstant(ShiftAmt, MVT::i8)); in getTargetVShiftByConstNode()
14595 static SDValue getTargetVShiftNode(unsigned Opc, SDLoc dl, MVT VT, in getTargetVShiftNode()
14598 MVT SVT = ShAmt.getSimpleValueType(); in getTargetVShiftNode()
14599 assert((SVT == MVT::i32 || SVT == MVT::i64) && "Unexpected value type!"); in getTargetVShiftNode()
14617 ShAmt.getOperand(0).getSimpleValueType() == MVT::i16) { in getTargetVShiftNode()
14620 Op0 = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(Op0), MVT::v8i16, Op0); in getTargetVShiftNode()
14627 if (SVT == MVT::i32) { in getTargetVShiftNode()
14633 MVT BVT = SVT == MVT::i32 ? MVT::v4i32 : MVT::v2i64; in getTargetVShiftNode()
14639 MVT EltVT = VT.getVectorElementType(); in getTargetVShiftNode()
14640 EVT ShVT = MVT::getVectorVT(EltVT, 128/EltVT.getSizeInBits()); in getTargetVShiftNode()
14655 MVT::i1, VT.getVectorNumElements()); in getVectorMaskingNode()
14656 EVT BitcastVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, in getVectorMaskingNode()
14701 SDValue IMask = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, Mask); in getScalarMaskingNode()
14812 EVT MaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, in LowerINTRINSIC_WO_CHAIN()
14815 EVT BitcastVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, in LowerINTRINSIC_WO_CHAIN()
14840 SDValue Cond = DAG.getNode(IntrData->Opc0, dl, MVT::i32, LHS, RHS); in LowerINTRINSIC_WO_CHAIN()
14841 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, in LowerINTRINSIC_WO_CHAIN()
14842 DAG.getConstant(X86CC, MVT::i8), Cond); in LowerINTRINSIC_WO_CHAIN()
14843 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC); in LowerINTRINSIC_WO_CHAIN()
14862 EVT MaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, in LowerINTRINSIC_WO_CHAIN()
14864 EVT BitcastVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, in LowerINTRINSIC_WO_CHAIN()
14877 EVT MaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, in LowerINTRINSIC_WO_CHAIN()
14879 EVT BitcastVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, in LowerINTRINSIC_WO_CHAIN()
14973 SDValue Test = DAG.getNode(TestOpc, dl, MVT::i32, LHS, RHS); in LowerINTRINSIC_WO_CHAIN()
14974 SDValue CC = DAG.getConstant(X86CC, MVT::i8); in LowerINTRINSIC_WO_CHAIN()
14975 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, CC, Test); in LowerINTRINSIC_WO_CHAIN()
14976 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC); in LowerINTRINSIC_WO_CHAIN()
14981 SDValue LHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i1, Op.getOperand(1)); in LowerINTRINSIC_WO_CHAIN()
14982 SDValue RHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i1, Op.getOperand(2)); in LowerINTRINSIC_WO_CHAIN()
14983 SDValue CC = DAG.getConstant(X86CC, MVT::i8); in LowerINTRINSIC_WO_CHAIN()
14984 SDValue Test = DAG.getNode(X86ISD::KORTEST, dl, MVT::i32, LHS, RHS); in LowerINTRINSIC_WO_CHAIN()
14985 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i1, CC, Test); in LowerINTRINSIC_WO_CHAIN()
14986 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC); in LowerINTRINSIC_WO_CHAIN()
15045 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32); in LowerINTRINSIC_WO_CHAIN()
15047 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, in LowerINTRINSIC_WO_CHAIN()
15048 DAG.getConstant(X86CC, MVT::i8), in LowerINTRINSIC_WO_CHAIN()
15050 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC); in LowerINTRINSIC_WO_CHAIN()
15062 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32); in LowerINTRINSIC_WO_CHAIN()
15075 SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), MVT::i8); in getGatherNode()
15076 EVT MaskVT = MVT::getVectorVT(MVT::i1, in getGatherNode()
15084 SDVTList VTs = DAG.getVTList(Op.getValueType(), MaskVT, MVT::Other); in getGatherNode()
15085 SDValue Disp = DAG.getTargetConstant(0, MVT::i32); in getGatherNode()
15086 SDValue Segment = DAG.getRegister(0, MVT::i32); in getGatherNode()
15101 SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), MVT::i8); in getScatterNode()
15102 SDValue Disp = DAG.getTargetConstant(0, MVT::i32); in getScatterNode()
15103 SDValue Segment = DAG.getRegister(0, MVT::i32); in getScatterNode()
15104 EVT MaskVT = MVT::getVectorVT(MVT::i1, in getScatterNode()
15112 SDVTList VTs = DAG.getVTList(MaskVT, MVT::Other); in getScatterNode()
15124 SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), MVT::i8); in getPrefetchNode()
15125 SDValue Disp = DAG.getTargetConstant(0, MVT::i32); in getPrefetchNode()
15126 SDValue Segment = DAG.getRegister(0, MVT::i32); in getPrefetchNode()
15128 MVT::getVectorVT(MVT::i1, Index.getSimpleValueType().getVectorNumElements()); in getPrefetchNode()
15137 SDNode *Res = DAG.getMachineNode(Opc, dl, MVT::Other, Ops); in getPrefetchNode()
15147 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue); in getReadPerformanceCounter()
15159 LO = DAG.getCopyFromReg(rd, DL, X86::RAX, MVT::i64, rd.getValue(1)); in getReadPerformanceCounter()
15160 HI = DAG.getCopyFromReg(LO.getValue(1), DL, X86::RDX, MVT::i64, in getReadPerformanceCounter()
15163 LO = DAG.getCopyFromReg(rd, DL, X86::EAX, MVT::i32, rd.getValue(1)); in getReadPerformanceCounter()
15164 HI = DAG.getCopyFromReg(LO.getValue(1), DL, X86::EDX, MVT::i32, in getReadPerformanceCounter()
15172 SDValue Tmp = DAG.getNode(ISD::SHL, DL, MVT::i64, HI, in getReadPerformanceCounter()
15173 DAG.getConstant(32, MVT::i8)); in getReadPerformanceCounter()
15174 Results.push_back(DAG.getNode(ISD::OR, DL, MVT::i64, LO, Tmp)); in getReadPerformanceCounter()
15181 SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Ops); in getReadPerformanceCounter()
15192 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue); in getReadTimeStampCounter()
15200 LO = DAG.getCopyFromReg(rd, DL, X86::RAX, MVT::i64, rd.getValue(1)); in getReadTimeStampCounter()
15201 HI = DAG.getCopyFromReg(LO.getValue(1), DL, X86::RDX, MVT::i64, in getReadTimeStampCounter()
15204 LO = DAG.getCopyFromReg(rd, DL, X86::EAX, MVT::i32, rd.getValue(1)); in getReadTimeStampCounter()
15205 HI = DAG.getCopyFromReg(LO.getValue(1), DL, X86::EDX, MVT::i32, in getReadTimeStampCounter()
15215 SDValue ecx = DAG.getCopyFromReg(Chain, DL, X86::ECX, MVT::i32, in getReadTimeStampCounter()
15226 SDValue Tmp = DAG.getNode(ISD::SHL, DL, MVT::i64, HI, in getReadTimeStampCounter()
15227 DAG.getConstant(32, MVT::i8)); in getReadTimeStampCounter()
15228 Results.push_back(DAG.getNode(ISD::OR, DL, MVT::i64, LO, Tmp)); in getReadTimeStampCounter()
15235 SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Ops); in getReadTimeStampCounter()
15266 SDVTList VTs = DAG.getVTList(Op->getValueType(0), MVT::Glue, MVT::Other); in LowerINTRINSIC_W_CHAIN()
15273 DAG.getConstant(X86::COND_B, MVT::i32), in LowerINTRINSIC_W_CHAIN()
15276 DAG.getVTList(Op->getValueType(1), MVT::Glue), in LowerINTRINSIC_W_CHAIN()
15330 SDVTList VTs = DAG.getVTList(Op->getValueType(0), MVT::Other); in LowerINTRINSIC_W_CHAIN()
15332 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, in LowerINTRINSIC_W_CHAIN()
15333 DAG.getConstant(X86::COND_NE, MVT::i8), in LowerINTRINSIC_W_CHAIN()
15342 SDVTList CFVTs = DAG.getVTList(Op->getValueType(0), MVT::Other); in LowerINTRINSIC_W_CHAIN()
15343 SDVTList VTs = DAG.getVTList(Op.getOperand(3)->getValueType(0), MVT::Other); in LowerINTRINSIC_W_CHAIN()
15345 DAG.getConstant(-1, MVT::i8)); in LowerINTRINSIC_W_CHAIN()
15351 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, in LowerINTRINSIC_W_CHAIN()
15352 DAG.getConstant(X86::COND_B, MVT::i8), in LowerINTRINSIC_W_CHAIN()
15370 EVT MaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, in LowerINTRINSIC_W_CHAIN()
15372 EVT BitcastVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, in LowerINTRINSIC_W_CHAIN()
15394 EVT MaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, in LowerINTRINSIC_W_CHAIN()
15396 EVT BitcastVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, in LowerINTRINSIC_W_CHAIN()
15469 assert(((FrameReg == X86::RBP && VT == MVT::i64) || in LowerFRAMEADDR()
15470 (FrameReg == X86::EBP && VT == MVT::i32)) && in LowerFRAMEADDR()
15508 assert(((FrameReg == X86::RBP && PtrVT == MVT::i64) || in LowerEH_RETURN()
15509 (FrameReg == X86::EBP && PtrVT == MVT::i32)) && in LowerEH_RETURN()
15512 unsigned StoreAddrReg = (PtrVT == MVT::i64) ? X86::RCX : X86::ECX; in LowerEH_RETURN()
15521 return DAG.getNode(X86ISD::EH_RETURN, dl, MVT::Other, Chain, in LowerEH_RETURN()
15529 DAG.getVTList(MVT::i32, MVT::Other), in lowerEH_SJLJ_SETJMP()
15536 return DAG.getNode(X86ISD::EH_SJLJ_LONGJMP, DL, MVT::Other, in lowerEH_SJLJ_LONGJMP()
15570 OutChains[0] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, MVT::i16), in LowerINIT_TRAMPOLINE()
15574 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, in LowerINIT_TRAMPOLINE()
15575 DAG.getConstant(2, MVT::i64)); in LowerINIT_TRAMPOLINE()
15583 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, in LowerINIT_TRAMPOLINE()
15584 DAG.getConstant(10, MVT::i64)); in LowerINIT_TRAMPOLINE()
15585 OutChains[2] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, MVT::i16), in LowerINIT_TRAMPOLINE()
15589 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, in LowerINIT_TRAMPOLINE()
15590 DAG.getConstant(12, MVT::i64)); in LowerINIT_TRAMPOLINE()
15597 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, in LowerINIT_TRAMPOLINE()
15598 DAG.getConstant(20, MVT::i64)); in LowerINIT_TRAMPOLINE()
15599 OutChains[4] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, MVT::i16), in LowerINIT_TRAMPOLINE()
15604 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, in LowerINIT_TRAMPOLINE()
15605 DAG.getConstant(22, MVT::i64)); in LowerINIT_TRAMPOLINE()
15606 OutChains[5] = DAG.getStore(Root, dl, DAG.getConstant(ModRM, MVT::i8), Addr, in LowerINIT_TRAMPOLINE()
15610 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains); in LowerINIT_TRAMPOLINE()
15659 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp, in LowerINIT_TRAMPOLINE()
15660 DAG.getConstant(10, MVT::i32)); in LowerINIT_TRAMPOLINE()
15661 Disp = DAG.getNode(ISD::SUB, dl, MVT::i32, FPtr, Addr); in LowerINIT_TRAMPOLINE()
15667 DAG.getConstant(MOV32ri|N86Reg, MVT::i8), in LowerINIT_TRAMPOLINE()
15671 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp, in LowerINIT_TRAMPOLINE()
15672 DAG.getConstant(1, MVT::i32)); in LowerINIT_TRAMPOLINE()
15678 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp, in LowerINIT_TRAMPOLINE()
15679 DAG.getConstant(5, MVT::i32)); in LowerINIT_TRAMPOLINE()
15680 OutChains[2] = DAG.getStore(Root, dl, DAG.getConstant(JMP, MVT::i8), Addr, in LowerINIT_TRAMPOLINE()
15684 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp, in LowerINIT_TRAMPOLINE()
15685 DAG.getConstant(6, MVT::i32)); in LowerINIT_TRAMPOLINE()
15690 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains); in LowerINIT_TRAMPOLINE()
15718 MVT VT = Op.getSimpleValueType(); in LowerFLT_ROUNDS_()
15731 DAG.getVTList(MVT::Other), in LowerFLT_ROUNDS_()
15732 Ops, MVT::i16, MMO); in LowerFLT_ROUNDS_()
15735 SDValue CWD = DAG.getLoad(MVT::i16, DL, Chain, StackSlot, in LowerFLT_ROUNDS_()
15740 DAG.getNode(ISD::SRL, DL, MVT::i16, in LowerFLT_ROUNDS_()
15741 DAG.getNode(ISD::AND, DL, MVT::i16, in LowerFLT_ROUNDS_()
15742 CWD, DAG.getConstant(0x800, MVT::i16)), in LowerFLT_ROUNDS_()
15743 DAG.getConstant(11, MVT::i8)); in LowerFLT_ROUNDS_()
15745 DAG.getNode(ISD::SRL, DL, MVT::i16, in LowerFLT_ROUNDS_()
15746 DAG.getNode(ISD::AND, DL, MVT::i16, in LowerFLT_ROUNDS_()
15747 CWD, DAG.getConstant(0x400, MVT::i16)), in LowerFLT_ROUNDS_()
15748 DAG.getConstant(9, MVT::i8)); in LowerFLT_ROUNDS_()
15751 DAG.getNode(ISD::AND, DL, MVT::i16, in LowerFLT_ROUNDS_()
15752 DAG.getNode(ISD::ADD, DL, MVT::i16, in LowerFLT_ROUNDS_()
15753 DAG.getNode(ISD::OR, DL, MVT::i16, CWD1, CWD2), in LowerFLT_ROUNDS_()
15754 DAG.getConstant(1, MVT::i16)), in LowerFLT_ROUNDS_()
15755 DAG.getConstant(3, MVT::i16)); in LowerFLT_ROUNDS_()
15762 MVT VT = Op.getSimpleValueType(); in LowerCTLZ()
15768 if (VT == MVT::i8) { in LowerCTLZ()
15770 OpVT = MVT::i32; in LowerCTLZ()
15775 SDVTList VTs = DAG.getVTList(OpVT, MVT::i32); in LowerCTLZ()
15782 DAG.getConstant(X86::COND_E, MVT::i8), in LowerCTLZ()
15790 if (VT == MVT::i8) in LowerCTLZ()
15791 Op = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Op); in LowerCTLZ()
15796 MVT VT = Op.getSimpleValueType(); in LowerCTLZ_ZERO_UNDEF()
15802 if (VT == MVT::i8) { in LowerCTLZ_ZERO_UNDEF()
15804 OpVT = MVT::i32; in LowerCTLZ_ZERO_UNDEF()
15809 SDVTList VTs = DAG.getVTList(OpVT, MVT::i32); in LowerCTLZ_ZERO_UNDEF()
15815 if (VT == MVT::i8) in LowerCTLZ_ZERO_UNDEF()
15816 Op = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Op); in LowerCTLZ_ZERO_UNDEF()
15821 MVT VT = Op.getSimpleValueType(); in LowerCTTZ()
15827 SDVTList VTs = DAG.getVTList(VT, MVT::i32); in LowerCTTZ()
15834 DAG.getConstant(X86::COND_E, MVT::i8), in LowerCTTZ()
15843 MVT VT = Op.getSimpleValueType(); in Lower256IntArith()
15861 MVT EltVT = VT.getVectorElementType(); in Lower256IntArith()
15862 MVT NewVT = MVT::getVectorVT(EltVT, NumElems/2); in Lower256IntArith()
15886 MVT VT = Op.getSimpleValueType(); in LowerMUL()
15896 if (VT == MVT::v4i32) { in LowerMUL()
15906 SDValue Evens = DAG.getNode(X86ISD::PMULUDQ, dl, MVT::v2i64, A, B); in LowerMUL()
15908 SDValue Odds = DAG.getNode(X86ISD::PMULUDQ, dl, MVT::v2i64, Aodds, Bodds); in LowerMUL()
15919 assert((VT == MVT::v2i64 || VT == MVT::v4i64 || VT == MVT::v8i64) && in LowerMUL()
15937 EVT MulVT = (VT == MVT::v2i64) ? MVT::v4i32 : in LowerMUL()
15938 (VT == MVT::v4i64) ? MVT::v8i32 : MVT::v16i32; in LowerMUL()
15999 static_cast<EVT>(MVT::v2i64).getTypeForEVT(*DAG.getContext()), in LowerWin64_i128OP()
16013 assert((VT == MVT::v4i32 && Subtarget->hasSSE2()) || in LowerMUL_LOHI()
16014 (VT == MVT::v8i32 && Subtarget->hasInt256())); in LowerMUL_LOHI()
16036 MVT MulVT = VT == MVT::v4i32 ? MVT::v2i64 : MVT::v4i64; in LowerMUL_LOHI()
16051 if (VT == MVT::v8i32) { in LowerMUL_LOHI()
16085 MVT VT = Op.getSimpleValueType(); in LowerScalarImmediateShift()
16095 if (VT == MVT::v2i64 || VT == MVT::v4i32 || VT == MVT::v8i16 || in LowerScalarImmediateShift()
16097 (VT == MVT::v4i64 || VT == MVT::v8i32 || VT == MVT::v16i16)) || in LowerScalarImmediateShift()
16099 (VT == MVT::v8i64 || VT == MVT::v16i32))) { in LowerScalarImmediateShift()
16106 if (Op.getOpcode() == ISD::SRA && VT != MVT::v2i64 && VT != MVT::v4i64) in LowerScalarImmediateShift()
16111 if (VT == MVT::v16i8 || (Subtarget->hasInt256() && VT == MVT::v32i8)) { in LowerScalarImmediateShift()
16113 MVT ShiftVT = MVT::getVectorVT(MVT::i16, NumElts / 2); in LowerScalarImmediateShift()
16122 NumElts, DAG.getConstant(uint8_t(-1U << ShiftAmt), MVT::i8)); in LowerScalarImmediateShift()
16133 NumElts, DAG.getConstant(uint8_t(-1U) >> ShiftAmt, MVT::i8)); in LowerScalarImmediateShift()
16147 DAG.getConstant(128 >> ShiftAmt, MVT::i8)); in LowerScalarImmediateShift()
16160 (VT == MVT::v2i64 || (Subtarget->hasInt256() && VT == MVT::v4i64)) && in LowerScalarImmediateShift()
16209 MVT VT = Op.getSimpleValueType(); in LowerScalarVariableShift()
16214 if ((VT == MVT::v2i64 && Op.getOpcode() != ISD::SRA) || in LowerScalarVariableShift()
16215 VT == MVT::v4i32 || VT == MVT::v8i16 || in LowerScalarVariableShift()
16217 ((VT == MVT::v4i64 && Op.getOpcode() != ISD::SRA) || in LowerScalarVariableShift()
16218 VT == MVT::v8i32 || VT == MVT::v16i16)) || in LowerScalarVariableShift()
16219 (Subtarget->hasAVX512() && (VT == MVT::v8i64 || VT == MVT::v16i32))) { in LowerScalarVariableShift()
16257 assert(EltVT.bitsLE(MVT::i64) && "Unexpected element type!"); in LowerScalarVariableShift()
16258 if (EltVT != MVT::i64 && EltVT.bitsGT(MVT::i32)) in LowerScalarVariableShift()
16259 BaseShAmt = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i64, BaseShAmt); in LowerScalarVariableShift()
16260 else if (EltVT.bitsLT(MVT::i32)) in LowerScalarVariableShift()
16261 BaseShAmt = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, BaseShAmt); in LowerScalarVariableShift()
16269 case MVT::v2i64: in LowerScalarVariableShift()
16270 case MVT::v4i32: in LowerScalarVariableShift()
16271 case MVT::v8i16: in LowerScalarVariableShift()
16272 case MVT::v4i64: in LowerScalarVariableShift()
16273 case MVT::v8i32: in LowerScalarVariableShift()
16274 case MVT::v16i16: in LowerScalarVariableShift()
16275 case MVT::v16i32: in LowerScalarVariableShift()
16276 case MVT::v8i64: in LowerScalarVariableShift()
16282 case MVT::v4i32: in LowerScalarVariableShift()
16283 case MVT::v8i16: in LowerScalarVariableShift()
16284 case MVT::v8i32: in LowerScalarVariableShift()
16285 case MVT::v16i16: in LowerScalarVariableShift()
16286 case MVT::v16i32: in LowerScalarVariableShift()
16287 case MVT::v8i64: in LowerScalarVariableShift()
16293 case MVT::v2i64: in LowerScalarVariableShift()
16294 case MVT::v4i32: in LowerScalarVariableShift()
16295 case MVT::v8i16: in LowerScalarVariableShift()
16296 case MVT::v4i64: in LowerScalarVariableShift()
16297 case MVT::v8i32: in LowerScalarVariableShift()
16298 case MVT::v16i16: in LowerScalarVariableShift()
16299 case MVT::v16i32: in LowerScalarVariableShift()
16300 case MVT::v8i64: in LowerScalarVariableShift()
16309 (VT == MVT::v2i64 || (Subtarget->hasInt256() && VT == MVT::v4i64) || in LowerScalarVariableShift()
16310 (Subtarget->hasAVX512() && VT == MVT::v8i64)) && in LowerScalarVariableShift()
16341 MVT VT = Op.getSimpleValueType(); in LowerShift()
16355 if (Subtarget->hasAVX512() && (VT == MVT::v16i32 || VT == MVT::v8i64)) in LowerShift()
16361 (VT == MVT::v2i64 || VT == MVT::v4i32 || in LowerShift()
16362 VT == MVT::v4i64 || VT == MVT::v8i32)) in LowerShift()
16365 (VT == MVT::v2i64 || VT == MVT::v4i32 || in LowerShift()
16366 VT == MVT::v4i64 || VT == MVT::v8i32)) in LowerShift()
16368 if (Op.getOpcode() == ISD::SRA && (VT == MVT::v4i32 || VT == MVT::v8i32)) in LowerShift()
16374 if (VT == MVT::v2i64 && Op.getOpcode() != ISD::SRA) { in LowerShift()
16387 (VT == MVT::v8i16 || VT == MVT::v4i32 || in LowerShift()
16388 (Subtarget->hasInt256() && VT == MVT::v16i16)) && in LowerShift()
16417 if (VT == MVT::v4i32 && Op->getOpcode() == ISD::SHL) { in LowerShift()
16421 Op = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, Op); in LowerShift()
16438 if ((VT == MVT::v8i16 || VT == MVT::v4i32) && in LowerShift()
16445 SDValue Amt2 = (VT == MVT::v4i32) ? Amt->getOperand(1) : in LowerShift()
16450 if (VT == MVT::v4i32) { in LowerShift()
16482 EVT CastVT = MVT::v4i32; in LowerShift()
16490 CastVT = MVT::v2i64; in LowerShift()
16499 if (VT == MVT::v16i8 && Op->getOpcode() == ISD::SHL) { in LowerShift()
16516 M = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, MVT::v8i16, M, 4, DAG); in LowerShift()
16527 M = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, MVT::v8i16, M, 2, DAG); in LowerShift()
16545 if (Subtarget->hasInt256() && VT == MVT::v8i16) { in LowerShift()
16546 MVT NewVT = VT == MVT::v8i16 ? MVT::v8i32 : MVT::v16i16; in LowerShift()
16558 MVT EltVT = VT.getVectorElementType(); in LowerShift()
16559 EVT NewVT = MVT::getVectorVT(EltVT, NumElems/2); in LowerShift()
16638 BaseOp = N->getValueType(0) == MVT::i8 ? X86ISD::SMUL8 : X86ISD::SMUL; in LowerXALUO()
16642 if (N->getValueType(0) == MVT::i8) { in LowerXALUO()
16648 MVT::i32); in LowerXALUO()
16652 DAG.getNode(X86ISD::SETCC, DL, MVT::i8, in LowerXALUO()
16653 DAG.getConstant(X86::COND_O, MVT::i32), in LowerXALUO()
16661 SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::i32); in LowerXALUO()
16666 DAG.getConstant(Cond, MVT::i32), in LowerXALUO()
16815 return DAG.getNode(X86ISD::MFENCE, dl, MVT::Other, Op.getOperand(0)); in LowerATOMIC_FENCE()
16818 SDValue Zero = DAG.getConstant(0, MVT::i32); in LowerATOMIC_FENCE()
16820 DAG.getRegister(X86::ESP, MVT::i32), // Base in LowerATOMIC_FENCE()
16821 DAG.getTargetConstant(1, MVT::i8), // Scale in LowerATOMIC_FENCE()
16822 DAG.getRegister(0, MVT::i32), // Index in LowerATOMIC_FENCE()
16823 DAG.getTargetConstant(0, MVT::i32), // Disp in LowerATOMIC_FENCE()
16824 DAG.getRegister(0, MVT::i32), // Segment. in LowerATOMIC_FENCE()
16828 SDNode *Res = DAG.getMachineNode(X86::OR32mrLocked, dl, MVT::Other, Ops); in LowerATOMIC_FENCE()
16833 return DAG.getNode(X86ISD::MEMBARRIER, dl, MVT::Other, Op.getOperand(0)); in LowerATOMIC_FENCE()
16838 MVT T = Op.getSimpleValueType(); in LowerCMP_SWAP()
16844 case MVT::i8: Reg = X86::AL; size = 1; break; in LowerCMP_SWAP()
16845 case MVT::i16: Reg = X86::AX; size = 2; break; in LowerCMP_SWAP()
16846 case MVT::i32: Reg = X86::EAX; size = 4; break; in LowerCMP_SWAP()
16847 case MVT::i64: in LowerCMP_SWAP()
16857 DAG.getTargetConstant(size, MVT::i8), in LowerCMP_SWAP()
16859 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue); in LowerCMP_SWAP()
16867 MVT::i32, cpOut.getValue(2)); in LowerCMP_SWAP()
16869 DAG.getConstant(X86::COND_E, MVT::i8), EFLAGS); in LowerCMP_SWAP()
16879 MVT SrcVT = Op.getOperand(0).getSimpleValueType(); in LowerBITCAST()
16880 MVT DstVT = Op.getSimpleValueType(); in LowerBITCAST()
16882 if (SrcVT == MVT::v2i32 || SrcVT == MVT::v4i16 || SrcVT == MVT::v8i8) { in LowerBITCAST()
16884 if (DstVT != MVT::f64) in LowerBITCAST()
16905 SDValue ToV2F64 = DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, BV); in LowerBITCAST()
16906 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, ToV2F64, in LowerBITCAST()
16912 assert((DstVT == MVT::i64 || in LowerBITCAST()
16916 if (SrcVT==MVT::i64 && DstVT.isVector()) in LowerBITCAST()
16918 if (DstVT==MVT::i64 && SrcVT.isVector()) in LowerBITCAST()
16959 bool NeedsBitcast = EltVT == MVT::i32; in LowerCTPOP()
16960 MVT BitcastVT = VT.is256BitVector() ? MVT::v4i64 : MVT::v2i64; in LowerCTPOP()
17107 SDVTList VTs = DAG.getVTList(VT, MVT::i32); in LowerADDC_ADDE_SUBC_SUBE()
17147 bool isF64 = ArgVT == MVT::f64; in LowerFSINCOS()
17284 if (VT != MVT::v2f32) in ReplaceNodeResults()
17287 SDValue LHS = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, in ReplaceNodeResults()
17289 SDValue RHS = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, in ReplaceNodeResults()
17291 Results.push_back(DAG.getNode(N->getOpcode(), dl, MVT::v4f32, LHS, RHS)); in ReplaceNodeResults()
17314 if (N->getOperand(0).getValueType() == MVT::f16) in ReplaceNodeResults()
17340 if (N->getOperand(0).getValueType() != MVT::v2i32 || in ReplaceNodeResults()
17341 N->getValueType(0) != MVT::v2f32) in ReplaceNodeResults()
17343 SDValue ZExtIn = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v2i64, in ReplaceNodeResults()
17346 MVT::f64); in ReplaceNodeResults()
17347 SDValue VBias = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v2f64, Bias, Bias); in ReplaceNodeResults()
17348 SDValue Or = DAG.getNode(ISD::OR, dl, MVT::v2i64, ZExtIn, in ReplaceNodeResults()
17349 DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, VBias)); in ReplaceNodeResults()
17350 Or = DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, Or); in ReplaceNodeResults()
17351 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::v2f64, Or, VBias); in ReplaceNodeResults()
17352 Results.push_back(DAG.getNode(X86ISD::VFPROUND, dl, MVT::v4f32, Sub)); in ReplaceNodeResults()
17358 SDValue V = DAG.getNode(X86ISD::VFPROUND, dl, MVT::v4f32, N->getOperand(0)); in ReplaceNodeResults()
17365 assert(N->getValueType(0) == MVT::v2f32 && in ReplaceNodeResults()
17390 assert((T == MVT::i64 || T == MVT::i128) && "can only expand cmpxchg pair"); in ReplaceNodeResults()
17391 bool Regs64bit = T == MVT::i128; in ReplaceNodeResults()
17392 EVT HalfT = Regs64bit ? MVT::i64 : MVT::i32; in ReplaceNodeResults()
17418 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue); in ReplaceNodeResults()
17432 MVT::i32, cpOutH.getValue(2)); in ReplaceNodeResults()
17434 DAG.getNode(X86ISD::SETCC, dl, MVT::i8, in ReplaceNodeResults()
17435 DAG.getConstant(X86::COND_E, MVT::i8), EFLAGS); in ReplaceNodeResults()
17464 if (SrcVT != MVT::f64 || in ReplaceNodeResults()
17465 (DstVT != MVT::v2i32 && DstVT != MVT::v4i16 && DstVT != MVT::v8i8)) in ReplaceNodeResults()
17472 MVT::v2f64, N->getOperand(0)); in ReplaceNodeResults()
17793 return VT1 == MVT::i32 && VT2 == MVT::i64 && Subtarget->is64Bit(); in isZExtFree()
17810 case MVT::i8: in isZExtFree()
17811 case MVT::i16: in isZExtFree()
17812 case MVT::i32: in isZExtFree()
17833 case MVT::f32: in isFMAFasterThanFMulAndFAdd()
17834 case MVT::f64: in isFMAFasterThanFMulAndFAdd()
17845 return !(VT1 == MVT::i32 && VT2 == MVT::i16); in isNarrowingProfitable()
18068 const TargetRegisterClass *AddrRegClass = getRegClassFor(MVT::i64); in EmitVAARG64WithCustomInserter()
18069 const TargetRegisterClass *OffsetRegClass = getRegClassFor(MVT::i32); in EmitVAARG64WithCustomInserter()
18813 assert(RC->hasType(MVT::i32) && "Invalid destination!"); in emitEHSjLjSetJmp()
18819 MVT PVT = getPointerTy(); in emitEHSjLjSetJmp()
18820 assert((PVT == MVT::i64 || PVT == MVT::i32) && in emitEHSjLjSetJmp()
18864 PtrStoreOpc = (PVT == MVT::i64) ? X86::MOV64mr : X86::MOV32mr; in emitEHSjLjSetJmp()
18884 PtrStoreOpc = (PVT == MVT::i64) ? X86::MOV64mi32 : X86::MOV32mi; in emitEHSjLjSetJmp()
18951 MVT PVT = getPointerTy(); in emitEHSjLjLongJmp()
18952 assert((PVT == MVT::i64 || PVT == MVT::i32) && in emitEHSjLjLongJmp()
18956 (PVT == MVT::i64) ? &X86::GR64RegClass : &X86::GR32RegClass; in emitEHSjLjLongJmp()
18960 unsigned FP = (PVT == MVT::i64) ? X86::RBP : X86::EBP; in emitEHSjLjLongJmp()
18968 unsigned PtrLoadOpc = (PVT == MVT::i64) ? X86::MOV64rm : X86::MOV32rm; in emitEHSjLjLongJmp()
18969 unsigned IJmpOpc = (PVT == MVT::i64) ? X86::JMP64r : X86::JMP32r; in emitEHSjLjLongJmp()
19484 SDVTList Tys = DAG.getVTList(MVT::v4i64, MVT::Other); in PerformShuffleCombine256()
19498 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, in PerformShuffleCombine256()
19558 MVT VT = Input.getSimpleValueType(); in combineX86ShuffleChain()
19559 MVT RootVT = Root.getSimpleValueType(); in combineX86ShuffleChain()
19585 MVT ShuffleVT; in combineX86ShuffleChain()
19591 ShuffleVT = MVT::v2f64; in combineX86ShuffleChain()
19596 ShuffleVT = MVT::v4f32; in combineX86ShuffleChain()
19615 MVT ShuffleVT = MVT::v4f32; in combineX86ShuffleChain()
19629 MVT ShuffleVT = MVT::v4f32; in combineX86ShuffleChain()
19655 MVT ShuffleVT; in combineX86ShuffleChain()
19658 ShuffleVT = MVT::v8i16; in combineX86ShuffleChain()
19661 ShuffleVT = MVT::v16i8; in combineX86ShuffleChain()
19690 PSHUFBMask.push_back(DAG.getUNDEF(MVT::i8)); in combineX86ShuffleChain()
19696 PSHUFBMask.push_back(DAG.getConstant(M, MVT::i8)); in combineX86ShuffleChain()
19698 MVT ByteVT = MVT::getVectorVT(MVT::i8, NumBytes); in combineX86ShuffleChain()
19759 MVT VT = Op.getSimpleValueType(); in combineX86ShufflesRecursively()
19864 MVT VT = N.getSimpleValueType(); in getPSHUFShuffleMask()
19956 if (V.getSimpleValueType().getScalarType() != MVT::i8 && in combineRedundantDWordShuffle()
19957 V.getSimpleValueType().getScalarType() != MVT::i16) in combineRedundantDWordShuffle()
20088 V = DAG.getNode(V.getOpcode(), DL, MVT::v8i16, V.getOperand(0), in combineRedundantHalfShuffle()
20106 MVT VT = N.getSimpleValueType(); in PerformTargetShuffleCombine()
20131 assert(VT.getScalarType() == MVT::i16 && "Bad word shuffle type!"); in PerformTargetShuffleCombine()
20144 MVT DVT = MVT::getVectorVT(MVT::i32, VT.getVectorNumElements() / 2); in PerformTargetShuffleCombine()
20250 assert((VT == MVT::v4f32 || VT == MVT::v2f64 || VT == MVT::v8f32 || in combineShuffleToAddSub()
20251 VT == MVT::v4f64) && in combineShuffleToAddSub()
20476 if (N->getValueType(0) != MVT::x86mmx || in PerformBITCASTCombine()
20478 N->getOperand(0)->getValueType(0) != MVT::v2i32) in PerformBITCASTCombine()
20483 if (C && C->getZExtValue() == 0 && V.getOperand(0).getValueType() == MVT::i32) in PerformBITCASTCombine()
20505 N->getValueType(0) == MVT::i32 && in PerformEXTRACT_VECTOR_ELTCombine()
20506 InputVector.getValueType() == MVT::v2i32) { in PerformEXTRACT_VECTOR_ELTCombine()
20510 if (MMXSrc.getValueType() == MVT::x86mmx) in PerformEXTRACT_VECTOR_ELTCombine()
20518 MMXSrc.getValueType() == MVT::i64 && MMXSrcOp.hasOneUse() && in PerformEXTRACT_VECTOR_ELTCombine()
20520 MMXSrcOp.getValueType() == MVT::v1i64 && in PerformEXTRACT_VECTOR_ELTCombine()
20521 MMXSrcOp.getOperand(0).getValueType() == MVT::x86mmx) in PerformEXTRACT_VECTOR_ELTCombine()
20529 if (InputVector.getValueType() != MVT::v4i32) in PerformEXTRACT_VECTOR_ELTCombine()
20546 if (Extract->getValueType(0) != MVT::i32) in PerformEXTRACT_VECTOR_ELTCombine()
20574 if (TLI.isOperationLegal(ISD::SRA, MVT::i64)) { in PerformEXTRACT_VECTOR_ELTCombine()
20575 SDValue Cst = DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, InputVector); in PerformEXTRACT_VECTOR_ELTCombine()
20577 SDValue BottomHalf = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, Cst, in PerformEXTRACT_VECTOR_ELTCombine()
20579 SDValue TopHalf = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, Cst, in PerformEXTRACT_VECTOR_ELTCombine()
20583 DAG.getTargetLoweringInfo().getShiftAmountTy(MVT::i64)); in PerformEXTRACT_VECTOR_ELTCombine()
20584 Vals[0] = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, BottomHalf); in PerformEXTRACT_VECTOR_ELTCombine()
20585 Vals[1] = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, in PerformEXTRACT_VECTOR_ELTCombine()
20586 DAG.getNode(ISD::SRA, dl, MVT::i64, BottomHalf, ShAmt)); in PerformEXTRACT_VECTOR_ELTCombine()
20587 Vals[2] = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, TopHalf); in PerformEXTRACT_VECTOR_ELTCombine()
20588 Vals[3] = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, in PerformEXTRACT_VECTOR_ELTCombine()
20589 DAG.getNode(ISD::SRA, dl, MVT::i64, TopHalf, ShAmt)); in PerformEXTRACT_VECTOR_ELTCombine()
20639 case MVT::v4i64: in matchIntegerMINMAX()
20640 case MVT::v2i64: in matchIntegerMINMAX()
20644 case MVT::v64i8: in matchIntegerMINMAX()
20645 case MVT::v32i16: in matchIntegerMINMAX()
20649 case MVT::v16i32: in matchIntegerMINMAX()
20650 case MVT::v8i64: in matchIntegerMINMAX()
20654 case MVT::v32i8: in matchIntegerMINMAX()
20655 case MVT::v16i16: in matchIntegerMINMAX()
20656 case MVT::v8i32: in matchIntegerMINMAX()
20662 case MVT::v16i8: in matchIntegerMINMAX()
20663 case MVT::v8i16: in matchIntegerMINMAX()
20664 case MVT::v4i32: in matchIntegerMINMAX()
20671 (Subtarget->hasSSE2() && VT == MVT::v16i8); in matchIntegerMINMAX()
20673 (Subtarget->hasSSE2() && VT == MVT::v8i16); in matchIntegerMINMAX()
20746 MVT VT = N->getSimpleValueType(0); in transformVSELECTtoBlendVECTOR_SHUFFLE()
20782 VT != MVT::f80 && (TLI.isTypeLegal(VT) || VT == MVT::v2f32) && in PerformSELECTCombine()
20784 (Subtarget->hasSSE1() && VT.getScalarType() == MVT::f32))) { in PerformSELECTCombine()
20923 CondVT.getVectorElementType() == MVT::i1) { in PerformSELECTCombine()
20931 (OpVT.getVectorElementType() == MVT::i8 || in PerformSELECTCombine()
20932 OpVT.getVectorElementType() == MVT::i16) && in PerformSELECTCombine()
20970 DAG.getConstant(ShAmt, MVT::i8)); in PerformSELECTCombine()
20988 if (N->getValueType(0) == MVT::i32 || N->getValueType(0) == MVT::i64) { in PerformSELECTCombine()
20990 if (N->getValueType(0) == MVT::i32) Diff = (unsigned)Diff; in PerformSELECTCombine()
21070 ((Subtarget->hasSSE2() && (VT == MVT::v16i8 || VT == MVT::v8i16)) || in PerformSELECTCombine()
21071 (Subtarget->hasAVX2() && (VT == MVT::v32i8 || VT == MVT::v16i16)))) { in PerformSELECTCombine()
21247 if (VT.getScalarType() == MVT::i16) in PerformSELECTCombine()
21253 if (VT.getSizeInBits() == 256 && VT.getScalarType() == MVT::i8 && in PerformSELECTCombine()
21514 (FalseOp.getValueType() != MVT::f80 || hasFPCMov(CC))) { in PerformCMOVCombine()
21516 DAG.getConstant(CC, MVT::i8), Flags }; in PerformCMOVCombine()
21537 Cond = DAG.getNode(X86ISD::SETCC, DL, MVT::i8, in PerformCMOVCombine()
21538 DAG.getConstant(CC, MVT::i8), Cond); in PerformCMOVCombine()
21545 DAG.getConstant(ShAmt, MVT::i8)); in PerformCMOVCombine()
21554 Cond = DAG.getNode(X86ISD::SETCC, DL, MVT::i8, in PerformCMOVCombine()
21555 DAG.getConstant(CC, MVT::i8), Cond); in PerformCMOVCombine()
21570 if (N->getValueType(0) == MVT::i32 || N->getValueType(0) == MVT::i64) { in PerformCMOVCombine()
21572 if (N->getValueType(0) == MVT::i32) Diff = (unsigned)Diff; in PerformCMOVCombine()
21592 Cond = DAG.getNode(X86ISD::SETCC, DL, MVT::i8, in PerformCMOVCombine()
21593 DAG.getConstant(CC, MVT::i8), Cond); in PerformCMOVCombine()
21646 DAG.getConstant(CC, MVT::i8), Cond }; in PerformCMOVCombine()
21680 SDValue LOps[] = {FalseOp, TrueOp, DAG.getConstant(CC0, MVT::i8), in PerformCMOVCombine()
21683 SDValue Ops[] = {LCMOV, TrueOp, DAG.getConstant(CC1, MVT::i8), Flags}; in PerformCMOVCombine()
21796 if (VT != MVT::i64 && VT != MVT::i32) in PerformMulCombine()
21832 DAG.getConstant(Log2_64(MulAmt1), MVT::i8)); in PerformMulCombine()
21839 DAG.getConstant(Log2_64(MulAmt2), MVT::i8)); in PerformMulCombine()
21899 if (VT != MVT::v2i64 && VT != MVT::v4i32 && VT != MVT::v8i16 && in performShiftToAllZeros()
21901 (VT != MVT::v4i64 && VT != MVT::v8i32 && VT != MVT::v16i16))) in performShiftToAllZeros()
21965 if (VT == MVT::f32 || VT == MVT::f64) { in CMPEQCombine()
22000 SDValue FSetCC = DAG.getNode(X86ISD::FSETCC, DL, MVT::i1, CMP00, in CMPEQCombine()
22001 CMP01, DAG.getConstant(x86cc, MVT::i8)); in CMPEQCombine()
22002 if (N->getValueType(0) != MVT::i1) in CMPEQCombine()
22009 DAG.getConstant(x86cc, MVT::i8)); in CMPEQCombine()
22011 bool is64BitFP = (CMP00.getValueType() == MVT::f64); in CMPEQCombine()
22012 MVT IntVT = is64BitFP ? MVT::i64 : MVT::i32; in CMPEQCombine()
22020 SDValue Vector64 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64, in CMPEQCombine()
22022 SDValue Vector32 = DAG.getNode(ISD::BITCAST, DL, MVT::v4f32, in CMPEQCombine()
22024 OnesOrZeroesF = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, in CMPEQCombine()
22026 IntVT = MVT::i32; in CMPEQCombine()
22032 SDValue OneBitOfTruth = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, ANDed); in CMPEQCombine()
22263 if (VT == MVT::i32 || VT == MVT::i64) { in PerformAndCombine()
22288 if (VT != MVT::v2i64 && VT != MVT::v4i64) in PerformAndCombine()
22321 if (VT == MVT::v2i64 || VT == MVT::v4i64) { in PerformOrCombine()
22323 (VT == MVT::v4i64 && !Subtarget->hasInt256())) in PerformOrCombine()
22387 EVT BlendVT = (VT == MVT::v4i64) ? MVT::v32i8 : MVT::v16i8; in PerformOrCombine()
22397 if (VT != MVT::i16 && VT != MVT::i32 && VT != MVT::i64) in PerformOrCombine()
22421 if (ShAmt0.getValueType() != MVT::i8) in PerformOrCombine()
22424 if (ShAmt1.getValueType() != MVT::i8) in PerformOrCombine()
22452 MVT::i8, ShAmt0)); in PerformOrCombine()
22461 MVT::i8, ShAmt0)); in PerformOrCombine()
22490 SDValue Neg = DAG.getNode(X86ISD::SUB, DL, DAG.getVTList(VT, MVT::i32), in performIntegerAbsCombine()
22494 DAG.getConstant(X86::COND_GE, MVT::i8), in performIntegerAbsCombine()
22496 return DAG.getNode(X86ISD::CMOV, DL, DAG.getVTList(VT, MVT::Glue), Ops); in performIntegerAbsCombine()
22552 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, in PerformLOADCombine()
22622 assert(Mask.getValueType().getVectorElementType() == MVT::i1); in PerformMLOADCombine()
22625 EVT NewMaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, in PerformMLOADCombine()
22706 assert(Mask.getValueType().getVectorElementType() == MVT::i1); in PerformMSTORECombine()
22709 EVT NewMaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, in PerformMSTORECombine()
22759 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Ch0, Ch1); in PerformSTORECombine()
22804 MVT StoreType = MVT::i8; in PerformSTORECombine()
22805 for (MVT Tp : MVT::integer_valuetypes()) { in PerformSTORECombine()
22811 if (TLI.isTypeLegal(MVT::f64) && StoreType.getSizeInBits() < 64 && in PerformSTORECombine()
22813 StoreType = MVT::f64; in PerformSTORECombine()
22837 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains); in PerformSTORECombine()
22854 (VT == MVT::i64 && F64IsLegal && !Subtarget->is64Bit())) && in PerformSTORECombine()
22894 EVT LdVT = Subtarget->is64Bit() ? MVT::i64 : MVT::f64; in PerformSTORECombine()
22902 NewChain = DAG.getNode(ISD::TokenFactor, LdDL, MVT::Other, Ops); in PerformSTORECombine()
22912 SDValue HiAddr = DAG.getNode(ISD::ADD, LdDL, MVT::i32, LoAddr, in PerformSTORECombine()
22913 DAG.getConstant(4, MVT::i32)); in PerformSTORECombine()
22915 SDValue LoLd = DAG.getLoad(MVT::i32, LdDL, Ld->getChain(), LoAddr, in PerformSTORECombine()
22919 SDValue HiLd = DAG.getLoad(MVT::i32, LdDL, Ld->getChain(), HiAddr, in PerformSTORECombine()
22929 NewChain = DAG.getNode(ISD::TokenFactor, LdDL, MVT::Other, Ops); in PerformSTORECombine()
22933 HiAddr = DAG.getNode(ISD::ADD, StDL, MVT::i32, LoAddr, in PerformSTORECombine()
22934 DAG.getConstant(4, MVT::i32)); in PerformSTORECombine()
22945 return DAG.getNode(ISD::TokenFactor, StDL, MVT::Other, LoSt, HiSt); in PerformSTORECombine()
22980 MVT VT = LHS.getSimpleValueType(); in isHorizontalBinOp()
23084 if (((Subtarget->hasSSE3() && (VT == MVT::v4f32 || VT == MVT::v2f64)) || in PerformFADDCombine()
23085 (Subtarget->hasFp256() && (VT == MVT::v8f32 || VT == MVT::v4f64))) && in PerformFADDCombine()
23099 if (((Subtarget->hasSSE3() && (VT == MVT::v4f32 || VT == MVT::v2f64)) || in PerformFSUBCombine()
23100 (Subtarget->hasFp256() && (VT == MVT::v8f32 || VT == MVT::v4f64))) && in PerformFSUBCombine()
23221 if (VT == MVT::v4i64 && (N0.getOpcode() == ISD::ANY_EXTEND || in PerformSIGN_EXTEND_INREGCombine()
23231 if (N00.getValueType() == MVT::v4i32 && ExtraVT.getSizeInBits() < 128) { in PerformSIGN_EXTEND_INREGCombine()
23232 SDValue Tmp = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, MVT::v4i32, in PerformSIGN_EXTEND_INREGCombine()
23234 return DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i64, Tmp); in PerformSIGN_EXTEND_INREGCombine()
23251 N0.getValueType() == MVT::i8 && VT == MVT::i32) { in PerformSExtCombine()
23253 SDVTList NodeTys = DAG.getVTList(MVT::i8, VT); in PerformSExtCombine()
23285 if ((ScalarVT != MVT::f32 && ScalarVT != MVT::f64) || in PerformFMACombine()
23363 N0.getResNo() == 1 && N0.getValueType() == MVT::i8 && in PerformZExtCombine()
23364 (VT == MVT::i32 || VT == MVT::i64)) { in PerformZExtCombine()
23365 SDVTList NodeTys = DAG.getVTList(MVT::i8, VT); in PerformZExtCombine()
23402 if (VT.getScalarType() == MVT::i1 && in PerformISDSETCCCombine()
23406 (LHS.getOperand(0).getValueType().getScalarType() == MVT::i1); in PerformISDSETCCCombine()
23415 (LHS.getOperand(0).getValueType().getScalarType() == MVT::i1); in PerformISDSETCCCombine()
23441 MVT VT = Load->getSimpleValueType(0); in NarrowVectorLoadToElement()
23442 MVT EVT = VT.getVectorElementType(); in NarrowVectorLoadToElement()
23458 MVT VT = N->getOperand(1)->getSimpleValueType(0); in PerformINSERTPSCombine()
23459 assert((VT == MVT::v4f32 || VT == MVT::v4i32) && in PerformINSERTPSCombine()
23499 if (VT == MVT::v2f64) in PerformBLENDICombine()
23502 SDValue NewMask = DAG.getConstant(1, MVT::i8); in PerformBLENDICombine()
23513 MVT VT) { in MaterializeSETB()
23514 if (VT == MVT::i8) in MaterializeSETB()
23516 DAG.getNode(X86ISD::SETCC_CARRY, DL, MVT::i8, in MaterializeSETB()
23517 DAG.getConstant(X86::COND_B, MVT::i8), EFLAGS), in MaterializeSETB()
23519 assert (VT == MVT::i1 && "Unexpected type for SECCC node"); in MaterializeSETB()
23520 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, in MaterializeSETB()
23521 DAG.getNode(X86ISD::SETCC_CARRY, DL, MVT::i8, in MaterializeSETB()
23522 DAG.getConstant(X86::COND_B, MVT::i8), EFLAGS)); in MaterializeSETB()
23561 SDValue Cond = DAG.getConstant(CC, MVT::i8); in PerformSETCCCombine()
23583 SDValue Cond = DAG.getConstant(CC, MVT::i8); in PerformBrCondCombine()
23650 if (InVT == MVT::v8i8 || InVT == MVT::v4i8) { in PerformSINT_TO_FPCombine()
23652 MVT DstVT = InVT == MVT::v4i8 ? MVT::v4i32 : MVT::v8i32; in PerformSINT_TO_FPCombine()
23664 if (N->getValueType(0) == MVT::f16) in PerformSINT_TO_FPCombine()
23669 !Subtarget->is64Bit() && VT == MVT::i64) { in PerformSINT_TO_FPCombine()
23695 DAG.getConstant(X86::COND_B,MVT::i8), in PerformADCCombine()
23731 SDValue NewCmp = DAG.getNode(X86ISD::CMP, DL, MVT::i32, CmpOp0, in OptimizeConditionalInDecrement()
23752 if (((Subtarget->hasSSSE3() && (VT == MVT::v8i16 || VT == MVT::v4i32)) || in PerformAddCombine()
23753 (Subtarget->hasInt256() && (VT == MVT::v16i16 || VT == MVT::v8i32))) && in PerformAddCombine()
23785 if (((Subtarget->hasSSSE3() && (VT == MVT::v8i16 || VT == MVT::v4i32)) || in PerformSubCombine()
23786 (Subtarget->hasInt256() && (VT == MVT::v16i16 || VT == MVT::v8i32))) && in PerformSubCombine()
23798 MVT VT = N->getSimpleValueType(0); in performVZEXTCombine()
23800 MVT OpVT = Op.getSimpleValueType(); in performVZEXTCombine()
23801 MVT OpEltVT = OpVT.getVectorElementType(); in performVZEXTCombine()
23810 MVT InnerVT = V.getSimpleValueType(); in performVZEXTCombine()
23811 MVT InnerEltVT = InnerVT.getVectorElementType(); in performVZEXTCombine()
23842 MVT OrigVT = OrigV.getSimpleValueType(); in performVZEXTCombine()
23846 OrigVT = MVT::getVectorVT(OrigVT.getVectorElementType(), in performVZEXTCombine()
23944 if (VT != MVT::i16) in isTypeDesirableForOp()
23971 if (VT != MVT::i16) in IsDesirableToPromoteOp()
24030 PVT = MVT::i32; in IsDesirableToPromoteOp()
24391 Result = DAG.getTargetConstant(C->getSExtValue(), MVT::i64); in LowerAsmOperandForConstraint()
24416 Result = DAG.getTargetConstant(CST->getSExtValue(), MVT::i64); in LowerAsmOperandForConstraint()
24477 MVT VT) const { in getRegForInlineAsmConstraint()
24489 if (VT == MVT::i32 || VT == MVT::f32) in getRegForInlineAsmConstraint()
24491 if (VT == MVT::i16) in getRegForInlineAsmConstraint()
24493 if (VT == MVT::i8 || VT == MVT::i1) in getRegForInlineAsmConstraint()
24495 if (VT == MVT::i64 || VT == MVT::f64) in getRegForInlineAsmConstraint()
24501 if (VT == MVT::i32 || VT == MVT::f32) in getRegForInlineAsmConstraint()
24503 if (VT == MVT::i16) in getRegForInlineAsmConstraint()
24505 if (VT == MVT::i8 || VT == MVT::i1) in getRegForInlineAsmConstraint()
24507 if (VT == MVT::i64) in getRegForInlineAsmConstraint()
24512 if (VT == MVT::i8 || VT == MVT::i1) in getRegForInlineAsmConstraint()
24514 if (VT == MVT::i16) in getRegForInlineAsmConstraint()
24516 if (VT == MVT::i32 || VT == MVT::f32 || !Subtarget->is64Bit()) in getRegForInlineAsmConstraint()
24520 if (VT == MVT::i8 || VT == MVT::i1) in getRegForInlineAsmConstraint()
24522 if (VT == MVT::i16) in getRegForInlineAsmConstraint()
24524 if (VT == MVT::i32 || !Subtarget->is64Bit()) in getRegForInlineAsmConstraint()
24530 if (VT == MVT::f32 && !isScalarFPTypeInSSEReg(VT)) in getRegForInlineAsmConstraint()
24532 if (VT == MVT::f64 && !isScalarFPTypeInSSEReg(VT)) in getRegForInlineAsmConstraint()
24547 case MVT::f32: in getRegForInlineAsmConstraint()
24548 case MVT::i32: in getRegForInlineAsmConstraint()
24550 case MVT::f64: in getRegForInlineAsmConstraint()
24551 case MVT::i64: in getRegForInlineAsmConstraint()
24554 case MVT::v16i8: in getRegForInlineAsmConstraint()
24555 case MVT::v8i16: in getRegForInlineAsmConstraint()
24556 case MVT::v4i32: in getRegForInlineAsmConstraint()
24557 case MVT::v2i64: in getRegForInlineAsmConstraint()
24558 case MVT::v4f32: in getRegForInlineAsmConstraint()
24559 case MVT::v2f64: in getRegForInlineAsmConstraint()
24562 case MVT::v32i8: in getRegForInlineAsmConstraint()
24563 case MVT::v16i16: in getRegForInlineAsmConstraint()
24564 case MVT::v8i32: in getRegForInlineAsmConstraint()
24565 case MVT::v4i64: in getRegForInlineAsmConstraint()
24566 case MVT::v8f32: in getRegForInlineAsmConstraint()
24567 case MVT::v4f64: in getRegForInlineAsmConstraint()
24569 case MVT::v8f64: in getRegForInlineAsmConstraint()
24570 case MVT::v16f32: in getRegForInlineAsmConstraint()
24571 case MVT::v16i32: in getRegForInlineAsmConstraint()
24572 case MVT::v8i64: in getRegForInlineAsmConstraint()
24634 if (VT == MVT::i8 || VT == MVT::i1) { in getRegForInlineAsmConstraint()
24647 } else if (VT == MVT::i32 || VT == MVT::f32) { in getRegForInlineAsmConstraint()
24664 } else if (VT == MVT::i64 || VT == MVT::f64) { in getRegForInlineAsmConstraint()
24696 if (VT == MVT::f32 || VT == MVT::i32) in getRegForInlineAsmConstraint()
24698 else if (VT == MVT::f64 || VT == MVT::i64) in getRegForInlineAsmConstraint()