Lines Matching refs:AMDGPU
138 using namespace AMDGPU::SDWA;
334 if (TII->getNamedOperand(*MI, AMDGPU::OpName::src0) == SrcOp) { in getSrcMods()
335 if (auto *Mod = TII->getNamedOperand(*MI, AMDGPU::OpName::src0_modifiers)) { in getSrcMods()
338 } else if (TII->getNamedOperand(*MI, AMDGPU::OpName::src1) == SrcOp) { in getSrcMods()
339 if (auto *Mod = TII->getNamedOperand(*MI, AMDGPU::OpName::src1_modifiers)) { in getSrcMods()
369 MachineOperand *Src = TII->getNamedOperand(MI, AMDGPU::OpName::src0); in convertToSDWA()
370 MachineOperand *SrcSel = TII->getNamedOperand(MI, AMDGPU::OpName::src0_sel); in convertToSDWA()
372 TII->getNamedOperand(MI, AMDGPU::OpName::src0_modifiers); in convertToSDWA()
376 Src = TII->getNamedOperand(MI, AMDGPU::OpName::src1); in convertToSDWA()
377 SrcSel = TII->getNamedOperand(MI, AMDGPU::OpName::src1_sel); in convertToSDWA()
378 SrcMods = TII->getNamedOperand(MI, AMDGPU::OpName::src1_modifiers); in convertToSDWA()
388 MachineOperand *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst); in convertToSDWA()
390 TII->getNamedOperand(MI, AMDGPU::OpName::dst_unused); in convertToSDWA()
393 DstUnused->getImm() == AMDGPU::SDWA::DstUnused::UNUSED_PRESERVE) { in convertToSDWA()
399 TII->getNamedImmOperand(MI, AMDGPU::OpName::dst_sel)); in convertToSDWA()
400 if (DstSel == AMDGPU::SDWA::SdwaSel::WORD_1 && in convertToSDWA()
401 getSrcSel() == AMDGPU::SDWA::SdwaSel::WORD_0) { in convertToSDWA()
403 auto DstIdx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), in convertToSDWA()
404 AMDGPU::OpName::vdst); in convertToSDWA()
417 if ((MI.getOpcode() == AMDGPU::V_FMAC_F16_sdwa || in convertToSDWA()
418 MI.getOpcode() == AMDGPU::V_FMAC_F32_sdwa || in convertToSDWA()
419 MI.getOpcode() == AMDGPU::V_MAC_F16_sdwa || in convertToSDWA()
420 MI.getOpcode() == AMDGPU::V_MAC_F32_sdwa) && in convertToSDWA()
461 if ((MI.getOpcode() == AMDGPU::V_FMAC_F16_sdwa || in convertToSDWA()
462 MI.getOpcode() == AMDGPU::V_FMAC_F32_sdwa || in convertToSDWA()
463 MI.getOpcode() == AMDGPU::V_MAC_F16_sdwa || in convertToSDWA()
464 MI.getOpcode() == AMDGPU::V_MAC_F32_sdwa) && in convertToSDWA()
465 getDstSel() != AMDGPU::SDWA::DWORD) { in convertToSDWA()
470 MachineOperand *Operand = TII->getNamedOperand(MI, AMDGPU::OpName::vdst); in convertToSDWA()
475 MachineOperand *DstSel= TII->getNamedOperand(MI, AMDGPU::OpName::dst_sel); in convertToSDWA()
478 MachineOperand *DstUnused= TII->getNamedOperand(MI, AMDGPU::OpName::dst_unused); in convertToSDWA()
511 MI.tieOperands(AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdst), in convertToSDWA()
549 case AMDGPU::V_LSHRREV_B32_e32: in matchSDWAOperand()
550 case AMDGPU::V_ASHRREV_I32_e32: in matchSDWAOperand()
551 case AMDGPU::V_LSHLREV_B32_e32: in matchSDWAOperand()
552 case AMDGPU::V_LSHRREV_B32_e64: in matchSDWAOperand()
553 case AMDGPU::V_ASHRREV_I32_e64: in matchSDWAOperand()
554 case AMDGPU::V_LSHLREV_B32_e64: { in matchSDWAOperand()
563 MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0); in matchSDWAOperand()
571 MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1); in matchSDWAOperand()
572 MachineOperand *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst); in matchSDWAOperand()
576 if (Opcode == AMDGPU::V_LSHLREV_B32_e32 || in matchSDWAOperand()
577 Opcode == AMDGPU::V_LSHLREV_B32_e64) { in matchSDWAOperand()
583 Opcode != AMDGPU::V_LSHRREV_B32_e32 && in matchSDWAOperand()
584 Opcode != AMDGPU::V_LSHRREV_B32_e64); in matchSDWAOperand()
589 case AMDGPU::V_LSHRREV_B16_e32: in matchSDWAOperand()
590 case AMDGPU::V_ASHRREV_I16_e32: in matchSDWAOperand()
591 case AMDGPU::V_LSHLREV_B16_e32: in matchSDWAOperand()
592 case AMDGPU::V_LSHRREV_B16_e64: in matchSDWAOperand()
593 case AMDGPU::V_ASHRREV_I16_e64: in matchSDWAOperand()
594 case AMDGPU::V_LSHLREV_B16_e64: { in matchSDWAOperand()
603 MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0); in matchSDWAOperand()
608 MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1); in matchSDWAOperand()
609 MachineOperand *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst); in matchSDWAOperand()
614 if (Opcode == AMDGPU::V_LSHLREV_B16_e32 || in matchSDWAOperand()
615 Opcode == AMDGPU::V_LSHLREV_B16_e64) { in matchSDWAOperand()
620 Opcode != AMDGPU::V_LSHRREV_B16_e32 && in matchSDWAOperand()
621 Opcode != AMDGPU::V_LSHRREV_B16_e64); in matchSDWAOperand()
626 case AMDGPU::V_BFE_I32: in matchSDWAOperand()
627 case AMDGPU::V_BFE_U32: { in matchSDWAOperand()
642 MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1); in matchSDWAOperand()
647 MachineOperand *Src2 = TII->getNamedOperand(MI, AMDGPU::OpName::src2); in matchSDWAOperand()
671 MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0); in matchSDWAOperand()
672 MachineOperand *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst); in matchSDWAOperand()
678 Src0, Dst, SrcSel, false, false, Opcode != AMDGPU::V_BFE_U32); in matchSDWAOperand()
681 case AMDGPU::V_AND_B32_e32: in matchSDWAOperand()
682 case AMDGPU::V_AND_B32_e64: { in matchSDWAOperand()
687 MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0); in matchSDWAOperand()
688 MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1); in matchSDWAOperand()
700 MachineOperand *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst); in matchSDWAOperand()
709 case AMDGPU::V_OR_B32_e32: in matchSDWAOperand()
710 case AMDGPU::V_OR_B32_e64: { in matchSDWAOperand()
741 MachineOperand *OrSDWA = TII->getNamedOperand(MI, AMDGPU::OpName::src0); in matchSDWAOperand()
742 MachineOperand *OrOther = TII->getNamedOperand(MI, AMDGPU::OpName::src1); in matchSDWAOperand()
746 OrSDWA = TII->getNamedOperand(MI, AMDGPU::OpName::src1); in matchSDWAOperand()
747 OrOther = TII->getNamedOperand(MI, AMDGPU::OpName::src0); in matchSDWAOperand()
785 TII->getNamedImmOperand(*SDWAInst, AMDGPU::OpName::dst_sel));; in matchSDWAOperand()
787 TII->getNamedImmOperand(*OtherInst, AMDGPU::OpName::dst_sel)); in matchSDWAOperand()
827 TII->getNamedImmOperand(*OtherInst, AMDGPU::OpName::dst_unused)); in matchSDWAOperand()
832 MachineOperand *OrDst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst); in matchSDWAOperand()
882 assert((Opc == AMDGPU::V_ADD_CO_U32_e64 || Opc == AMDGPU::V_SUB_CO_U32_e64) && in pseudoOpConvertToVOP2()
888 Opc = AMDGPU::getVOPe32(Opc); in pseudoOpConvertToVOP2()
890 const MachineOperand *Sdst = TII->getNamedOperand(MI, AMDGPU::OpName::sdst); in pseudoOpConvertToVOP2()
900 int SuccOpc = AMDGPU::getVOPe32(MISucc.getOpcode()); in pseudoOpConvertToVOP2()
902 MachineOperand *CarryIn = TII->getNamedOperand(MISucc, AMDGPU::OpName::src2); in pseudoOpConvertToVOP2()
905 MachineOperand *CarryOut = TII->getNamedOperand(MISucc, AMDGPU::OpName::sdst); in pseudoOpConvertToVOP2()
912 auto Liveness = MBB.computeRegisterLiveness(TRI, AMDGPU::VCC, MI, 25); in pseudoOpConvertToVOP2()
918 if (I->modifiesRegister(AMDGPU::VCC, TRI)) in pseudoOpConvertToVOP2()
925 .add(*TII->getNamedOperand(MI, AMDGPU::OpName::vdst)) in pseudoOpConvertToVOP2()
926 .add(*TII->getNamedOperand(MI, AMDGPU::OpName::src0)) in pseudoOpConvertToVOP2()
927 .add(*TII->getNamedOperand(MI, AMDGPU::OpName::src1)) in pseudoOpConvertToVOP2()
934 .add(*TII->getNamedOperand(MISucc, AMDGPU::OpName::vdst)) in pseudoOpConvertToVOP2()
935 .add(*TII->getNamedOperand(MISucc, AMDGPU::OpName::src0)) in pseudoOpConvertToVOP2()
936 .add(*TII->getNamedOperand(MISucc, AMDGPU::OpName::src1)) in pseudoOpConvertToVOP2()
950 if (AMDGPU::getSDWAOp(Opc) == -1) in isConvertibleToSDWA()
951 Opc = AMDGPU::getVOPe32(Opc); in isConvertibleToSDWA()
953 if (AMDGPU::getSDWAOp(Opc) == -1) in isConvertibleToSDWA()
956 if (!ST.hasSDWAOmod() && TII->hasModifiersSet(MI, AMDGPU::OpName::omod)) in isConvertibleToSDWA()
961 const MachineOperand *SDst = TII->getNamedOperand(MI, AMDGPU::OpName::sdst); in isConvertibleToSDWA()
962 if (SDst && (SDst->getReg() != AMDGPU::VCC && in isConvertibleToSDWA()
963 SDst->getReg() != AMDGPU::VCC_LO)) in isConvertibleToSDWA()
968 (TII->hasModifiersSet(MI, AMDGPU::OpName::clamp) || in isConvertibleToSDWA()
969 TII->hasModifiersSet(MI, AMDGPU::OpName::omod))) in isConvertibleToSDWA()
972 } else if (TII->getNamedOperand(MI, AMDGPU::OpName::sdst) || in isConvertibleToSDWA()
973 !TII->getNamedOperand(MI, AMDGPU::OpName::vdst)) { in isConvertibleToSDWA()
977 if (!ST.hasSDWAMac() && (Opc == AMDGPU::V_FMAC_F16_e32 || in isConvertibleToSDWA()
978 Opc == AMDGPU::V_FMAC_F32_e32 || in isConvertibleToSDWA()
979 Opc == AMDGPU::V_MAC_F16_e32 || in isConvertibleToSDWA()
980 Opc == AMDGPU::V_MAC_F32_e32)) in isConvertibleToSDWA()
988 if (Opc == AMDGPU::V_CNDMASK_B32_e32) in isConvertibleToSDWA()
991 if (MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0)) { in isConvertibleToSDWA()
996 if (MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1)) { in isConvertibleToSDWA()
1015 SDWAOpcode = AMDGPU::getSDWAOp(Opcode); in convertToSDWA()
1017 SDWAOpcode = AMDGPU::getSDWAOp(AMDGPU::getVOPe32(Opcode)); in convertToSDWA()
1029 MachineOperand *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst); in convertToSDWA()
1031 assert(AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::vdst) != -1); in convertToSDWA()
1033 } else if ((Dst = TII->getNamedOperand(MI, AMDGPU::OpName::sdst))) { in convertToSDWA()
1035 AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::sdst) != -1); in convertToSDWA()
1038 assert(AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::sdst) != -1); in convertToSDWA()
1044 MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0); in convertToSDWA()
1047 AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::src0) != -1 && in convertToSDWA()
1048 AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::src0_modifiers) != -1); in convertToSDWA()
1049 if (auto *Mod = TII->getNamedOperand(MI, AMDGPU::OpName::src0_modifiers)) in convertToSDWA()
1056 MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1); in convertToSDWA()
1059 AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::src1) != -1 && in convertToSDWA()
1060 AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::src1_modifiers) != -1); in convertToSDWA()
1061 if (auto *Mod = TII->getNamedOperand(MI, AMDGPU::OpName::src1_modifiers)) in convertToSDWA()
1068 if (SDWAOpcode == AMDGPU::V_FMAC_F16_sdwa || in convertToSDWA()
1069 SDWAOpcode == AMDGPU::V_FMAC_F32_sdwa || in convertToSDWA()
1070 SDWAOpcode == AMDGPU::V_MAC_F16_sdwa || in convertToSDWA()
1071 SDWAOpcode == AMDGPU::V_MAC_F32_sdwa) { in convertToSDWA()
1073 MachineOperand *Src2 = TII->getNamedOperand(MI, AMDGPU::OpName::src2); in convertToSDWA()
1079 assert(AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::clamp) != -1); in convertToSDWA()
1080 MachineOperand *Clamp = TII->getNamedOperand(MI, AMDGPU::OpName::clamp); in convertToSDWA()
1088 if (AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::omod) != -1) { in convertToSDWA()
1089 MachineOperand *OMod = TII->getNamedOperand(MI, AMDGPU::OpName::omod); in convertToSDWA()
1098 if (AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::dst_sel) != -1) { in convertToSDWA()
1099 MachineOperand *DstSel = TII->getNamedOperand(MI, AMDGPU::OpName::dst_sel); in convertToSDWA()
1103 SDWAInst.addImm(AMDGPU::SDWA::SdwaSel::DWORD); in convertToSDWA()
1108 if (AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::dst_unused) != -1) { in convertToSDWA()
1109 MachineOperand *DstUnused = TII->getNamedOperand(MI, AMDGPU::OpName::dst_unused); in convertToSDWA()
1113 SDWAInst.addImm(AMDGPU::SDWA::DstUnused::UNUSED_PAD); in convertToSDWA()
1118 assert(AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::src0_sel) != -1); in convertToSDWA()
1119 MachineOperand *Src0Sel = TII->getNamedOperand(MI, AMDGPU::OpName::src0_sel); in convertToSDWA()
1123 SDWAInst.addImm(AMDGPU::SDWA::SdwaSel::DWORD); in convertToSDWA()
1128 assert(AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::src1_sel) != -1); in convertToSDWA()
1129 MachineOperand *Src1Sel = TII->getNamedOperand(MI, AMDGPU::OpName::src1_sel); in convertToSDWA()
1133 SDWAInst.addImm(AMDGPU::SDWA::SdwaSel::DWORD); in convertToSDWA()
1138 auto DstUnused = TII->getNamedOperand(MI, AMDGPU::OpName::dst_unused); in convertToSDWA()
1140 DstUnused->getImm() == AMDGPU::SDWA::DstUnused::UNUSED_PRESERVE) { in convertToSDWA()
1146 auto PreserveDstIdx = AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::vdst); in convertToSDWA()
1208 Register VGPR = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in legalizeScalarOperands()
1210 TII->get(AMDGPU::V_MOV_B32_e32), VGPR); in legalizeScalarOperands()
1244 (PotentialMI->getOpcode() == AMDGPU::V_ADD_CO_U32_e64 || in runOnMachineFunction()
1245 PotentialMI->getOpcode() == AMDGPU::V_SUB_CO_U32_e64)) in runOnMachineFunction()