Lines Matching refs:AMDGPU
73 namespace AMDGPU { namespace
96 : AMDGPUGenInstrInfo(AMDGPU::ADJCALLSTACKUP, AMDGPU::ADJCALLSTACKDOWN), in SIInstrInfo()
118 int Op0Idx = AMDGPU::getNamedOperandIdx(Opc0, OpName); in nodesHaveSameOperandValue()
119 int Op1Idx = AMDGPU::getNamedOperandIdx(Opc1, OpName); in nodesHaveSameOperandValue()
145 case AMDGPU::V_MOV_B32_e32: in isReallyTriviallyReMaterializable()
146 case AMDGPU::V_MOV_B32_e64: in isReallyTriviallyReMaterializable()
147 case AMDGPU::V_MOV_B64_PSEUDO: in isReallyTriviallyReMaterializable()
148 case AMDGPU::V_ACCVGPR_READ_B32: in isReallyTriviallyReMaterializable()
149 case AMDGPU::V_ACCVGPR_WRITE_B32: in isReallyTriviallyReMaterializable()
183 int Offset0Idx = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset); in areLoadsFromSameBasePtr()
184 int Offset1Idx = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset); in areLoadsFromSameBasePtr()
201 if (AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::sbase) == -1 || in areLoadsFromSameBasePtr()
202 AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::sbase) == -1) in areLoadsFromSameBasePtr()
228 if (!nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::soffset) || in areLoadsFromSameBasePtr()
229 !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::vaddr) || in areLoadsFromSameBasePtr()
230 !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::srsrc)) in areLoadsFromSameBasePtr()
233 int OffIdx0 = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset); in areLoadsFromSameBasePtr()
234 int OffIdx1 = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset); in areLoadsFromSameBasePtr()
262 case AMDGPU::DS_READ2ST64_B32: in isStride64()
263 case AMDGPU::DS_READ2ST64_B64: in isStride64()
264 case AMDGPU::DS_WRITE2ST64_B32: in isStride64()
265 case AMDGPU::DS_WRITE2ST64_B64: in isStride64()
285 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::addr); in getMemOperandsWithOffsetWidth()
286 OffsetOp = getNamedOperand(LdSt, AMDGPU::OpName::offset); in getMemOperandsWithOffsetWidth()
297 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst); in getMemOperandsWithOffsetWidth()
299 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0); in getMemOperandsWithOffsetWidth()
306 getNamedOperand(LdSt, AMDGPU::OpName::offset0); in getMemOperandsWithOffsetWidth()
308 getNamedOperand(LdSt, AMDGPU::OpName::offset1); in getMemOperandsWithOffsetWidth()
323 int Data0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0); in getMemOperandsWithOffsetWidth()
333 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst); in getMemOperandsWithOffsetWidth()
335 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0); in getMemOperandsWithOffsetWidth()
337 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data1); in getMemOperandsWithOffsetWidth()
347 const MachineOperand *SOffset = getNamedOperand(LdSt, AMDGPU::OpName::soffset); in getMemOperandsWithOffsetWidth()
351 const MachineOperand *AddrReg = getNamedOperand(LdSt, AMDGPU::OpName::vaddr); in getMemOperandsWithOffsetWidth()
355 const MachineOperand *RSrc = getNamedOperand(LdSt, AMDGPU::OpName::srsrc); in getMemOperandsWithOffsetWidth()
362 getNamedOperand(LdSt, AMDGPU::OpName::offset); in getMemOperandsWithOffsetWidth()
367 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::srsrc); in getMemOperandsWithOffsetWidth()
372 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::vaddr); in getMemOperandsWithOffsetWidth()
377 getNamedOperand(LdSt, AMDGPU::OpName::offset); in getMemOperandsWithOffsetWidth()
383 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst); in getMemOperandsWithOffsetWidth()
385 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata); in getMemOperandsWithOffsetWidth()
391 int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::srsrc); in getMemOperandsWithOffsetWidth()
393 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr0); in getMemOperandsWithOffsetWidth()
399 BaseOps.push_back(getNamedOperand(LdSt, AMDGPU::OpName::vaddr)); in getMemOperandsWithOffsetWidth()
403 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata); in getMemOperandsWithOffsetWidth()
409 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::sbase); in getMemOperandsWithOffsetWidth()
413 OffsetOp = getNamedOperand(LdSt, AMDGPU::OpName::offset); in getMemOperandsWithOffsetWidth()
416 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::sdst); in getMemOperandsWithOffsetWidth()
423 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::vaddr); in getMemOperandsWithOffsetWidth()
426 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::saddr); in getMemOperandsWithOffsetWidth()
429 Offset = getNamedOperand(LdSt, AMDGPU::OpName::offset)->getImm(); in getMemOperandsWithOffsetWidth()
431 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst); in getMemOperandsWithOffsetWidth()
433 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata); in getMemOperandsWithOffsetWidth()
533 BuildMI(MBB, MI, DL, TII->get(AMDGPU::SI_ILLEGAL_COPY), DestReg) in reportIllegalCopy()
549 assert(AMDGPU::SReg_32RegClass.contains(SrcReg) || in indirectCopyToAGPR()
550 AMDGPU::AGPR_32RegClass.contains(SrcReg)); in indirectCopyToAGPR()
557 if (Def->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32) in indirectCopyToAGPR()
578 BuildMI(MBB, MI, DL, TII.get(AMDGPU::V_ACCVGPR_WRITE_B32), DestReg) in indirectCopyToAGPR()
596 unsigned MaxVGPRs = RI.getRegPressureLimit(&AMDGPU::VGPR_32RegClass, in indirectCopyToAGPR()
602 Register Tmp = RS.scavengeRegister(&AMDGPU::VGPR_32RegClass, 0); in indirectCopyToAGPR()
609 while (RegNo-- && RS.FindUnusedReg(&AMDGPU::VGPR_32RegClass)) { in indirectCopyToAGPR()
610 Register Tmp2 = RS.scavengeRegister(&AMDGPU::VGPR_32RegClass, 0); in indirectCopyToAGPR()
618 unsigned TmpCopyOp = AMDGPU::V_MOV_B32_e32; in indirectCopyToAGPR()
619 if (AMDGPU::AGPR_32RegClass.contains(SrcReg)) { in indirectCopyToAGPR()
620 TmpCopyOp = AMDGPU::V_ACCVGPR_READ_B32; in indirectCopyToAGPR()
622 assert(AMDGPU::SReg_32RegClass.contains(SrcReg)); in indirectCopyToAGPR()
633 = BuildMI(MBB, MI, DL, TII.get(AMDGPU::V_ACCVGPR_WRITE_B32), DestReg) in indirectCopyToAGPR()
652 unsigned Opcode = AMDGPU::S_MOV_B32; in expandSGPRCopy()
656 bool AlignedDest = ((Reg - AMDGPU::SGPR0) % 2) == 0; in expandSGPRCopy()
657 bool AlignedSrc = ((Src - AMDGPU::SGPR0) % 2) == 0; in expandSGPRCopy()
662 Opcode = AMDGPU::S_MOV_B64; in expandSGPRCopy()
701 assert(RI.getSubReg(Super, AMDGPU::lo16) == RegToFix); in copyPhysReg()
706 BuildMI(MBB, MI, DL, get(AMDGPU::BUNDLE)); in copyPhysReg()
713 if (RC == &AMDGPU::VGPR_32RegClass) { in copyPhysReg()
714 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg) || in copyPhysReg()
715 AMDGPU::SReg_32RegClass.contains(SrcReg) || in copyPhysReg()
716 AMDGPU::AGPR_32RegClass.contains(SrcReg)); in copyPhysReg()
717 unsigned Opc = AMDGPU::AGPR_32RegClass.contains(SrcReg) ? in copyPhysReg()
718 AMDGPU::V_ACCVGPR_READ_B32 : AMDGPU::V_MOV_B32_e32; in copyPhysReg()
724 if (RC == &AMDGPU::SReg_32_XM0RegClass || in copyPhysReg()
725 RC == &AMDGPU::SReg_32RegClass) { in copyPhysReg()
726 if (SrcReg == AMDGPU::SCC) { in copyPhysReg()
727 BuildMI(MBB, MI, DL, get(AMDGPU::S_CSELECT_B32), DestReg) in copyPhysReg()
733 if (DestReg == AMDGPU::VCC_LO) { in copyPhysReg()
734 if (AMDGPU::SReg_32RegClass.contains(SrcReg)) { in copyPhysReg()
735 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), AMDGPU::VCC_LO) in copyPhysReg()
739 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg)); in copyPhysReg()
740 BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32)) in copyPhysReg()
748 if (!AMDGPU::SReg_32RegClass.contains(SrcReg)) { in copyPhysReg()
753 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg) in copyPhysReg()
758 if (RC == &AMDGPU::SReg_64RegClass) { in copyPhysReg()
759 if (SrcReg == AMDGPU::SCC) { in copyPhysReg()
760 BuildMI(MBB, MI, DL, get(AMDGPU::S_CSELECT_B64), DestReg) in copyPhysReg()
766 if (DestReg == AMDGPU::VCC) { in copyPhysReg()
767 if (AMDGPU::SReg_64RegClass.contains(SrcReg)) { in copyPhysReg()
768 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), AMDGPU::VCC) in copyPhysReg()
772 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg)); in copyPhysReg()
773 BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32)) in copyPhysReg()
781 if (!AMDGPU::SReg_64RegClass.contains(SrcReg)) { in copyPhysReg()
786 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg) in copyPhysReg()
791 if (DestReg == AMDGPU::SCC) { in copyPhysReg()
794 if (AMDGPU::SReg_64RegClass.contains(SrcReg)) { in copyPhysReg()
799 BuildMI(MBB, MI, DL, get(AMDGPU::S_CMP_LG_U64)) in copyPhysReg()
803 assert(AMDGPU::SReg_32RegClass.contains(SrcReg)); in copyPhysReg()
804 BuildMI(MBB, MI, DL, get(AMDGPU::S_CMP_LG_U32)) in copyPhysReg()
813 if (RC == &AMDGPU::AGPR_32RegClass) { in copyPhysReg()
814 if (AMDGPU::VGPR_32RegClass.contains(SrcReg)) { in copyPhysReg()
815 BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_WRITE_B32), DestReg) in copyPhysReg()
828 assert(AMDGPU::VGPR_LO16RegClass.contains(SrcReg) || in copyPhysReg()
829 AMDGPU::VGPR_HI16RegClass.contains(SrcReg) || in copyPhysReg()
830 AMDGPU::SReg_LO16RegClass.contains(SrcReg) || in copyPhysReg()
831 AMDGPU::AGPR_LO16RegClass.contains(SrcReg)); in copyPhysReg()
833 bool IsSGPRDst = AMDGPU::SReg_LO16RegClass.contains(DestReg); in copyPhysReg()
834 bool IsSGPRSrc = AMDGPU::SReg_LO16RegClass.contains(SrcReg); in copyPhysReg()
835 bool IsAGPRDst = AMDGPU::AGPR_LO16RegClass.contains(DestReg); in copyPhysReg()
836 bool IsAGPRSrc = AMDGPU::AGPR_LO16RegClass.contains(SrcReg); in copyPhysReg()
837 bool DstLow = AMDGPU::VGPR_LO16RegClass.contains(DestReg) || in copyPhysReg()
838 AMDGPU::SReg_LO16RegClass.contains(DestReg) || in copyPhysReg()
839 AMDGPU::AGPR_LO16RegClass.contains(DestReg); in copyPhysReg()
840 bool SrcLow = AMDGPU::VGPR_LO16RegClass.contains(SrcReg) || in copyPhysReg()
841 AMDGPU::SReg_LO16RegClass.contains(SrcReg) || in copyPhysReg()
842 AMDGPU::AGPR_LO16RegClass.contains(SrcReg); in copyPhysReg()
852 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), NewDestReg) in copyPhysReg()
873 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), NewDestReg) in copyPhysReg()
878 auto MIB = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_sdwa), NewDestReg) in copyPhysReg()
882 .addImm(DstLow ? AMDGPU::SDWA::SdwaSel::WORD_0 in copyPhysReg()
883 : AMDGPU::SDWA::SdwaSel::WORD_1) in copyPhysReg()
884 .addImm(AMDGPU::SDWA::DstUnused::UNUSED_PRESERVE) in copyPhysReg()
885 .addImm(SrcLow ? AMDGPU::SDWA::SdwaSel::WORD_0 in copyPhysReg()
886 : AMDGPU::SDWA::SdwaSel::WORD_1) in copyPhysReg()
903 unsigned Opcode = AMDGPU::V_MOV_B32_e32; in copyPhysReg()
906 AMDGPU::V_ACCVGPR_WRITE_B32 : AMDGPU::INSTRUCTION_LIST_END; in copyPhysReg()
908 Opcode = AMDGPU::V_ACCVGPR_READ_B32; in copyPhysReg()
917 if (Opcode == AMDGPU::INSTRUCTION_LIST_END) in copyPhysReg()
935 if (Opcode == AMDGPU::INSTRUCTION_LIST_END) { in copyPhysReg()
957 NewOpc = AMDGPU::getCommuteRev(Opcode); in commuteOpcode()
963 NewOpc = AMDGPU::getCommuteOrig(Opcode); in commuteOpcode()
977 if (RegClass == &AMDGPU::SReg_32RegClass || in materializeImmediate()
978 RegClass == &AMDGPU::SGPR_32RegClass || in materializeImmediate()
979 RegClass == &AMDGPU::SReg_32_XM0RegClass || in materializeImmediate()
980 RegClass == &AMDGPU::SReg_32_XM0_XEXECRegClass) { in materializeImmediate()
981 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg) in materializeImmediate()
986 if (RegClass == &AMDGPU::SReg_64RegClass || in materializeImmediate()
987 RegClass == &AMDGPU::SGPR_64RegClass || in materializeImmediate()
988 RegClass == &AMDGPU::SReg_64_XEXECRegClass) { in materializeImmediate()
989 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg) in materializeImmediate()
994 if (RegClass == &AMDGPU::VGPR_32RegClass) { in materializeImmediate()
995 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DestReg) in materializeImmediate()
999 if (RegClass == &AMDGPU::VReg_64RegClass) { in materializeImmediate()
1000 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO), DestReg) in materializeImmediate()
1006 unsigned Opcode = AMDGPU::V_MOV_B32_e32; in materializeImmediate()
1009 Opcode = AMDGPU::S_MOV_B64; in materializeImmediate()
1012 Opcode = AMDGPU::S_MOV_B32; in materializeImmediate()
1029 return &AMDGPU::VGPR_32RegClass; in getPreferredSelectRegClass()
1040 RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID); in insertVectorSelect()
1041 assert(MRI.getRegClass(DstReg) == &AMDGPU::VGPR_32RegClass && in insertVectorSelect()
1046 BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg) in insertVectorSelect()
1048 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) in insertVectorSelect()
1059 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32 in insertVectorSelect()
1060 : AMDGPU::S_CSELECT_B64), SReg) in insertVectorSelect()
1063 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) in insertVectorSelect()
1073 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32 in insertVectorSelect()
1074 : AMDGPU::S_CSELECT_B64), SReg) in insertVectorSelect()
1077 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) in insertVectorSelect()
1089 BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg) in insertVectorSelect()
1091 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) in insertVectorSelect()
1103 BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg) in insertVectorSelect()
1105 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) in insertVectorSelect()
1116 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32 in insertVectorSelect()
1117 : AMDGPU::S_OR_SAVEEXEC_B64), SReg2) in insertVectorSelect()
1119 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32 in insertVectorSelect()
1120 : AMDGPU::S_CSELECT_B64), SReg) in insertVectorSelect()
1123 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) in insertVectorSelect()
1134 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32 in insertVectorSelect()
1135 : AMDGPU::S_OR_SAVEEXEC_B64), SReg2) in insertVectorSelect()
1137 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32 in insertVectorSelect()
1138 : AMDGPU::S_CSELECT_B64), SReg) in insertVectorSelect()
1141 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) in insertVectorSelect()
1164 BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_EQ_I32_e64), Reg) in insertEQ()
1177 BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_NE_I32_e64), Reg) in insertNE()
1187 return AMDGPU::COPY; in getMovOpcode()
1189 return RI.isSGPRClass(DstRC) ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32; in getMovOpcode()
1191 return AMDGPU::S_MOV_B64; in getMovOpcode()
1193 return AMDGPU::V_MOV_B64_PSEUDO; in getMovOpcode()
1195 return AMDGPU::COPY; in getMovOpcode()
1203 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V1); in getIndirectGPRIDXPseudo()
1205 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V2); in getIndirectGPRIDXPseudo()
1207 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V3); in getIndirectGPRIDXPseudo()
1209 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V4); in getIndirectGPRIDXPseudo()
1211 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V5); in getIndirectGPRIDXPseudo()
1213 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V8); in getIndirectGPRIDXPseudo()
1215 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V16); in getIndirectGPRIDXPseudo()
1217 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V32); in getIndirectGPRIDXPseudo()
1223 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1); in getIndirectGPRIDXPseudo()
1225 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2); in getIndirectGPRIDXPseudo()
1227 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3); in getIndirectGPRIDXPseudo()
1229 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4); in getIndirectGPRIDXPseudo()
1231 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5); in getIndirectGPRIDXPseudo()
1233 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8); in getIndirectGPRIDXPseudo()
1235 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16); in getIndirectGPRIDXPseudo()
1237 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32); in getIndirectGPRIDXPseudo()
1244 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V1; in getIndirectVGPRWriteMovRelPseudoOpc()
1246 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V2; in getIndirectVGPRWriteMovRelPseudoOpc()
1248 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V3; in getIndirectVGPRWriteMovRelPseudoOpc()
1250 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V4; in getIndirectVGPRWriteMovRelPseudoOpc()
1252 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V5; in getIndirectVGPRWriteMovRelPseudoOpc()
1254 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V8; in getIndirectVGPRWriteMovRelPseudoOpc()
1256 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V16; in getIndirectVGPRWriteMovRelPseudoOpc()
1258 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V32; in getIndirectVGPRWriteMovRelPseudoOpc()
1265 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V1; in getIndirectSGPRWriteMovRelPseudo32()
1267 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V2; in getIndirectSGPRWriteMovRelPseudo32()
1269 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V3; in getIndirectSGPRWriteMovRelPseudo32()
1271 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V4; in getIndirectSGPRWriteMovRelPseudo32()
1273 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V5; in getIndirectSGPRWriteMovRelPseudo32()
1275 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V8; in getIndirectSGPRWriteMovRelPseudo32()
1277 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V16; in getIndirectSGPRWriteMovRelPseudo32()
1279 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V32; in getIndirectSGPRWriteMovRelPseudo32()
1286 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V1; in getIndirectSGPRWriteMovRelPseudo64()
1288 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V2; in getIndirectSGPRWriteMovRelPseudo64()
1290 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V4; in getIndirectSGPRWriteMovRelPseudo64()
1292 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V8; in getIndirectSGPRWriteMovRelPseudo64()
1294 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V16; in getIndirectSGPRWriteMovRelPseudo64()
1320 return AMDGPU::SI_SPILL_S32_SAVE; in getSGPRSpillSaveOpcode()
1322 return AMDGPU::SI_SPILL_S64_SAVE; in getSGPRSpillSaveOpcode()
1324 return AMDGPU::SI_SPILL_S96_SAVE; in getSGPRSpillSaveOpcode()
1326 return AMDGPU::SI_SPILL_S128_SAVE; in getSGPRSpillSaveOpcode()
1328 return AMDGPU::SI_SPILL_S160_SAVE; in getSGPRSpillSaveOpcode()
1330 return AMDGPU::SI_SPILL_S192_SAVE; in getSGPRSpillSaveOpcode()
1332 return AMDGPU::SI_SPILL_S256_SAVE; in getSGPRSpillSaveOpcode()
1334 return AMDGPU::SI_SPILL_S512_SAVE; in getSGPRSpillSaveOpcode()
1336 return AMDGPU::SI_SPILL_S1024_SAVE; in getSGPRSpillSaveOpcode()
1345 return AMDGPU::SI_SPILL_V32_SAVE; in getVGPRSpillSaveOpcode()
1347 return AMDGPU::SI_SPILL_V64_SAVE; in getVGPRSpillSaveOpcode()
1349 return AMDGPU::SI_SPILL_V96_SAVE; in getVGPRSpillSaveOpcode()
1351 return AMDGPU::SI_SPILL_V128_SAVE; in getVGPRSpillSaveOpcode()
1353 return AMDGPU::SI_SPILL_V160_SAVE; in getVGPRSpillSaveOpcode()
1355 return AMDGPU::SI_SPILL_V192_SAVE; in getVGPRSpillSaveOpcode()
1357 return AMDGPU::SI_SPILL_V256_SAVE; in getVGPRSpillSaveOpcode()
1359 return AMDGPU::SI_SPILL_V512_SAVE; in getVGPRSpillSaveOpcode()
1361 return AMDGPU::SI_SPILL_V1024_SAVE; in getVGPRSpillSaveOpcode()
1370 return AMDGPU::SI_SPILL_A32_SAVE; in getAGPRSpillSaveOpcode()
1372 return AMDGPU::SI_SPILL_A64_SAVE; in getAGPRSpillSaveOpcode()
1374 return AMDGPU::SI_SPILL_A96_SAVE; in getAGPRSpillSaveOpcode()
1376 return AMDGPU::SI_SPILL_A128_SAVE; in getAGPRSpillSaveOpcode()
1378 return AMDGPU::SI_SPILL_A160_SAVE; in getAGPRSpillSaveOpcode()
1380 return AMDGPU::SI_SPILL_A192_SAVE; in getAGPRSpillSaveOpcode()
1382 return AMDGPU::SI_SPILL_A256_SAVE; in getAGPRSpillSaveOpcode()
1384 return AMDGPU::SI_SPILL_A512_SAVE; in getAGPRSpillSaveOpcode()
1386 return AMDGPU::SI_SPILL_A1024_SAVE; in getAGPRSpillSaveOpcode()
1412 assert(SrcReg != AMDGPU::M0 && "m0 should not be spilled"); in storeRegToStackSlot()
1413 assert(SrcReg != AMDGPU::EXEC_LO && SrcReg != AMDGPU::EXEC_HI && in storeRegToStackSlot()
1414 SrcReg != AMDGPU::EXEC && "exec should not be spilled"); in storeRegToStackSlot()
1424 MRI.constrainRegClass(SrcReg, &AMDGPU::SReg_32_XM0_XEXECRegClass); in storeRegToStackSlot()
1453 return AMDGPU::SI_SPILL_S32_RESTORE; in getSGPRSpillRestoreOpcode()
1455 return AMDGPU::SI_SPILL_S64_RESTORE; in getSGPRSpillRestoreOpcode()
1457 return AMDGPU::SI_SPILL_S96_RESTORE; in getSGPRSpillRestoreOpcode()
1459 return AMDGPU::SI_SPILL_S128_RESTORE; in getSGPRSpillRestoreOpcode()
1461 return AMDGPU::SI_SPILL_S160_RESTORE; in getSGPRSpillRestoreOpcode()
1463 return AMDGPU::SI_SPILL_S192_RESTORE; in getSGPRSpillRestoreOpcode()
1465 return AMDGPU::SI_SPILL_S256_RESTORE; in getSGPRSpillRestoreOpcode()
1467 return AMDGPU::SI_SPILL_S512_RESTORE; in getSGPRSpillRestoreOpcode()
1469 return AMDGPU::SI_SPILL_S1024_RESTORE; in getSGPRSpillRestoreOpcode()
1478 return AMDGPU::SI_SPILL_V32_RESTORE; in getVGPRSpillRestoreOpcode()
1480 return AMDGPU::SI_SPILL_V64_RESTORE; in getVGPRSpillRestoreOpcode()
1482 return AMDGPU::SI_SPILL_V96_RESTORE; in getVGPRSpillRestoreOpcode()
1484 return AMDGPU::SI_SPILL_V128_RESTORE; in getVGPRSpillRestoreOpcode()
1486 return AMDGPU::SI_SPILL_V160_RESTORE; in getVGPRSpillRestoreOpcode()
1488 return AMDGPU::SI_SPILL_V192_RESTORE; in getVGPRSpillRestoreOpcode()
1490 return AMDGPU::SI_SPILL_V256_RESTORE; in getVGPRSpillRestoreOpcode()
1492 return AMDGPU::SI_SPILL_V512_RESTORE; in getVGPRSpillRestoreOpcode()
1494 return AMDGPU::SI_SPILL_V1024_RESTORE; in getVGPRSpillRestoreOpcode()
1503 return AMDGPU::SI_SPILL_A32_RESTORE; in getAGPRSpillRestoreOpcode()
1505 return AMDGPU::SI_SPILL_A64_RESTORE; in getAGPRSpillRestoreOpcode()
1507 return AMDGPU::SI_SPILL_A96_RESTORE; in getAGPRSpillRestoreOpcode()
1509 return AMDGPU::SI_SPILL_A128_RESTORE; in getAGPRSpillRestoreOpcode()
1511 return AMDGPU::SI_SPILL_A160_RESTORE; in getAGPRSpillRestoreOpcode()
1513 return AMDGPU::SI_SPILL_A192_RESTORE; in getAGPRSpillRestoreOpcode()
1515 return AMDGPU::SI_SPILL_A256_RESTORE; in getAGPRSpillRestoreOpcode()
1517 return AMDGPU::SI_SPILL_A512_RESTORE; in getAGPRSpillRestoreOpcode()
1519 return AMDGPU::SI_SPILL_A1024_RESTORE; in getAGPRSpillRestoreOpcode()
1545 assert(DestReg != AMDGPU::M0 && "m0 should not be reloaded into"); in loadRegFromStackSlot()
1546 assert(DestReg != AMDGPU::EXEC_LO && DestReg != AMDGPU::EXEC_HI && in loadRegFromStackSlot()
1547 DestReg != AMDGPU::EXEC && "exec should not be spilled"); in loadRegFromStackSlot()
1554 MRI.constrainRegClass(DestReg, &AMDGPU::SReg_32_XM0_XEXECRegClass); in loadRegFromStackSlot()
1588 BuildMI(MBB, MI, DL, get(AMDGPU::S_NOP)).addImm(Arg - 1); in insertNoops()
1602 BuildMI(MBB, MBB.end(), DebugLoc(), get(AMDGPU::S_ENDPGM)).addImm(0); in insertReturn()
1604 BuildMI(MBB, MBB.end(), DebugLoc(), get(AMDGPU::SI_RETURN_TO_EPILOG)); in insertReturn()
1614 case AMDGPU::S_NOP: in getNumWaitStates()
1624 case AMDGPU::S_MOV_B64_term: in expandPostRAPseudo()
1627 MI.setDesc(get(AMDGPU::S_MOV_B64)); in expandPostRAPseudo()
1630 case AMDGPU::S_MOV_B32_term: in expandPostRAPseudo()
1633 MI.setDesc(get(AMDGPU::S_MOV_B32)); in expandPostRAPseudo()
1636 case AMDGPU::S_XOR_B64_term: in expandPostRAPseudo()
1639 MI.setDesc(get(AMDGPU::S_XOR_B64)); in expandPostRAPseudo()
1642 case AMDGPU::S_XOR_B32_term: in expandPostRAPseudo()
1645 MI.setDesc(get(AMDGPU::S_XOR_B32)); in expandPostRAPseudo()
1647 case AMDGPU::S_OR_B64_term: in expandPostRAPseudo()
1650 MI.setDesc(get(AMDGPU::S_OR_B64)); in expandPostRAPseudo()
1652 case AMDGPU::S_OR_B32_term: in expandPostRAPseudo()
1655 MI.setDesc(get(AMDGPU::S_OR_B32)); in expandPostRAPseudo()
1658 case AMDGPU::S_ANDN2_B64_term: in expandPostRAPseudo()
1661 MI.setDesc(get(AMDGPU::S_ANDN2_B64)); in expandPostRAPseudo()
1664 case AMDGPU::S_ANDN2_B32_term: in expandPostRAPseudo()
1667 MI.setDesc(get(AMDGPU::S_ANDN2_B32)); in expandPostRAPseudo()
1670 case AMDGPU::V_MOV_B64_PSEUDO: { in expandPostRAPseudo()
1672 Register DstLo = RI.getSubReg(Dst, AMDGPU::sub0); in expandPostRAPseudo()
1673 Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1); in expandPostRAPseudo()
1680 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo) in expandPostRAPseudo()
1683 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi) in expandPostRAPseudo()
1688 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo) in expandPostRAPseudo()
1689 .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub0)) in expandPostRAPseudo()
1691 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi) in expandPostRAPseudo()
1692 .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub1)) in expandPostRAPseudo()
1698 case AMDGPU::V_MOV_B64_DPP_PSEUDO: { in expandPostRAPseudo()
1702 case AMDGPU::V_SET_INACTIVE_B32: { in expandPostRAPseudo()
1703 unsigned NotOpc = ST.isWave32() ? AMDGPU::S_NOT_B32 : AMDGPU::S_NOT_B64; in expandPostRAPseudo()
1704 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; in expandPostRAPseudo()
1707 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), MI.getOperand(0).getReg()) in expandPostRAPseudo()
1714 case AMDGPU::V_SET_INACTIVE_B64: { in expandPostRAPseudo()
1715 unsigned NotOpc = ST.isWave32() ? AMDGPU::S_NOT_B32 : AMDGPU::S_NOT_B64; in expandPostRAPseudo()
1716 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; in expandPostRAPseudo()
1719 MachineInstr *Copy = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO), in expandPostRAPseudo()
1728 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V1: in expandPostRAPseudo()
1729 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V2: in expandPostRAPseudo()
1730 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V3: in expandPostRAPseudo()
1731 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V4: in expandPostRAPseudo()
1732 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V5: in expandPostRAPseudo()
1733 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V8: in expandPostRAPseudo()
1734 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V16: in expandPostRAPseudo()
1735 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V32: in expandPostRAPseudo()
1736 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V1: in expandPostRAPseudo()
1737 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V2: in expandPostRAPseudo()
1738 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V3: in expandPostRAPseudo()
1739 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V4: in expandPostRAPseudo()
1740 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V5: in expandPostRAPseudo()
1741 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V8: in expandPostRAPseudo()
1742 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V16: in expandPostRAPseudo()
1743 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V32: in expandPostRAPseudo()
1744 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V1: in expandPostRAPseudo()
1745 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V2: in expandPostRAPseudo()
1746 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V4: in expandPostRAPseudo()
1747 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V8: in expandPostRAPseudo()
1748 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V16: { in expandPostRAPseudo()
1753 Opc = AMDGPU::V_MOVRELD_B32_e32; in expandPostRAPseudo()
1755 Opc = RI.getRegSizeInBits(*EltRC) == 64 ? AMDGPU::S_MOVRELD_B64 in expandPostRAPseudo()
1756 : AMDGPU::S_MOVRELD_B32; in expandPostRAPseudo()
1779 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1: in expandPostRAPseudo()
1780 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2: in expandPostRAPseudo()
1781 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3: in expandPostRAPseudo()
1782 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4: in expandPostRAPseudo()
1783 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5: in expandPostRAPseudo()
1784 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8: in expandPostRAPseudo()
1785 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16: in expandPostRAPseudo()
1786 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32: { in expandPostRAPseudo()
1793 MachineInstr *SetOn = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_ON)) in expandPostRAPseudo()
1795 .addImm(AMDGPU::VGPRIndexMode::DST_ENABLE); in expandPostRAPseudo()
1798 const MCInstrDesc &OpDesc = get(AMDGPU::V_MOV_B32_indirect); in expandPostRAPseudo()
1811 MachineInstr *SetOff = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_OFF)); in expandPostRAPseudo()
1818 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V1: in expandPostRAPseudo()
1819 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V2: in expandPostRAPseudo()
1820 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V3: in expandPostRAPseudo()
1821 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V4: in expandPostRAPseudo()
1822 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V5: in expandPostRAPseudo()
1823 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V8: in expandPostRAPseudo()
1824 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V16: in expandPostRAPseudo()
1825 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V32: { in expandPostRAPseudo()
1833 MachineInstr *SetOn = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_ON)) in expandPostRAPseudo()
1835 .addImm(AMDGPU::VGPRIndexMode::SRC0_ENABLE); in expandPostRAPseudo()
1838 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32)) in expandPostRAPseudo()
1842 .addReg(AMDGPU::M0, RegState::Implicit); in expandPostRAPseudo()
1844 MachineInstr *SetOff = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_OFF)); in expandPostRAPseudo()
1851 case AMDGPU::SI_PC_ADD_REL_OFFSET: { in expandPostRAPseudo()
1854 Register RegLo = RI.getSubReg(Reg, AMDGPU::sub0); in expandPostRAPseudo()
1855 Register RegHi = RI.getSubReg(Reg, AMDGPU::sub1); in expandPostRAPseudo()
1860 Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_GETPC_B64), Reg)); in expandPostRAPseudo()
1864 Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_ADD_U32), RegLo) in expandPostRAPseudo()
1868 MachineInstrBuilder MIB = BuildMI(MF, DL, get(AMDGPU::S_ADDC_U32), RegHi) in expandPostRAPseudo()
1878 case AMDGPU::ENTER_WWM: { in expandPostRAPseudo()
1881 MI.setDesc(get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32 in expandPostRAPseudo()
1882 : AMDGPU::S_OR_SAVEEXEC_B64)); in expandPostRAPseudo()
1885 case AMDGPU::EXIT_WWM: { in expandPostRAPseudo()
1888 MI.setDesc(get(ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64)); in expandPostRAPseudo()
1897 assert (MI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO); in expandMovDPP64()
1908 for (auto Sub : { AMDGPU::sub0, AMDGPU::sub1 }) { in expandMovDPP64()
1909 auto MovDPP = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_dpp)); in expandMovDPP64()
1914 auto Tmp = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in expandMovDPP64()
1943 BuildMI(MBB, MI, DL, get(AMDGPU::REG_SEQUENCE), Dst) in expandMovDPP64()
1945 .addImm(AMDGPU::sub0) in expandMovDPP64()
1947 .addImm(AMDGPU::sub1); in expandMovDPP64()
2013 assert(AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) == in commuteInstructionImpl()
2015 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1) == in commuteInstructionImpl()
2043 swapSourceModifiers(MI, Src0, AMDGPU::OpName::src0_modifiers, in commuteInstructionImpl()
2044 Src1, AMDGPU::OpName::src1_modifiers); in commuteInstructionImpl()
2067 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0); in findCommutedOpIndices()
2071 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1); in findCommutedOpIndices()
2082 assert(BranchOp != AMDGPU::S_SETPC_B64); in isBranchOffsetInRange()
2096 if (MI.getOpcode() == AMDGPU::S_SETPC_B64) { in getBranchDestBlock()
2120 Register PCReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); in insertIndirectBranch()
2126 MachineInstr *GetPC = BuildMI(MBB, I, DL, get(AMDGPU::S_GETPC_B64), PCReg); in insertIndirectBranch()
2130 BuildMI(MBB, I, DL, get(AMDGPU::S_ADD_U32)) in insertIndirectBranch()
2131 .addReg(PCReg, RegState::Define, AMDGPU::sub0) in insertIndirectBranch()
2132 .addReg(PCReg, 0, AMDGPU::sub0) in insertIndirectBranch()
2134 BuildMI(MBB, I, DL, get(AMDGPU::S_ADDC_U32)) in insertIndirectBranch()
2135 .addReg(PCReg, RegState::Define, AMDGPU::sub1) in insertIndirectBranch()
2136 .addReg(PCReg, 0, AMDGPU::sub1) in insertIndirectBranch()
2140 BuildMI(MBB, I, DL, get(AMDGPU::S_SUB_U32)) in insertIndirectBranch()
2141 .addReg(PCReg, RegState::Define, AMDGPU::sub0) in insertIndirectBranch()
2142 .addReg(PCReg, 0, AMDGPU::sub0) in insertIndirectBranch()
2144 BuildMI(MBB, I, DL, get(AMDGPU::S_SUBB_U32)) in insertIndirectBranch()
2145 .addReg(PCReg, RegState::Define, AMDGPU::sub1) in insertIndirectBranch()
2146 .addReg(PCReg, 0, AMDGPU::sub1) in insertIndirectBranch()
2151 BuildMI(&MBB, DL, get(AMDGPU::S_SETPC_B64)) in insertIndirectBranch()
2192 AMDGPU::SReg_64RegClass, in insertIndirectBranch()
2204 return AMDGPU::S_CBRANCH_SCC1; in getBranchOpcode()
2206 return AMDGPU::S_CBRANCH_SCC0; in getBranchOpcode()
2208 return AMDGPU::S_CBRANCH_VCCNZ; in getBranchOpcode()
2210 return AMDGPU::S_CBRANCH_VCCZ; in getBranchOpcode()
2212 return AMDGPU::S_CBRANCH_EXECNZ; in getBranchOpcode()
2214 return AMDGPU::S_CBRANCH_EXECZ; in getBranchOpcode()
2222 case AMDGPU::S_CBRANCH_SCC0: in getBranchPredicate()
2224 case AMDGPU::S_CBRANCH_SCC1: in getBranchPredicate()
2226 case AMDGPU::S_CBRANCH_VCCNZ: in getBranchPredicate()
2228 case AMDGPU::S_CBRANCH_VCCZ: in getBranchPredicate()
2230 case AMDGPU::S_CBRANCH_EXECNZ: in getBranchPredicate()
2232 case AMDGPU::S_CBRANCH_EXECZ: in getBranchPredicate()
2245 if (I->getOpcode() == AMDGPU::S_BRANCH) { in analyzeBranchImpl()
2253 if (I->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) { in analyzeBranchImpl()
2273 if (I->getOpcode() == AMDGPU::S_BRANCH) { in analyzeBranchImpl()
2294 I->getOpcode() != AMDGPU::SI_MASK_BRANCH) { in analyzeBranch()
2296 case AMDGPU::SI_MASK_BRANCH: in analyzeBranch()
2297 case AMDGPU::S_MOV_B64_term: in analyzeBranch()
2298 case AMDGPU::S_XOR_B64_term: in analyzeBranch()
2299 case AMDGPU::S_OR_B64_term: in analyzeBranch()
2300 case AMDGPU::S_ANDN2_B64_term: in analyzeBranch()
2301 case AMDGPU::S_MOV_B32_term: in analyzeBranch()
2302 case AMDGPU::S_XOR_B32_term: in analyzeBranch()
2303 case AMDGPU::S_OR_B32_term: in analyzeBranch()
2304 case AMDGPU::S_ANDN2_B32_term: in analyzeBranch()
2306 case AMDGPU::SI_IF: in analyzeBranch()
2307 case AMDGPU::SI_ELSE: in analyzeBranch()
2308 case AMDGPU::SI_KILL_I1_TERMINATOR: in analyzeBranch()
2309 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR: in analyzeBranch()
2322 if (I->getOpcode() != AMDGPU::SI_MASK_BRANCH) in analyzeBranch()
2360 if (I->getOpcode() == AMDGPU::SI_MASK_BRANCH) { in removeBranch()
2391 BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH)) in insertBranch()
2399 BuildMI(&MBB, DL, get(AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO)) in insertBranch()
2430 BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH)) in insertBranch()
2470 int NumInsts = AMDGPU::getRegBitWidth(RC->getID()) / 32; in canInsertSelect()
2485 int NumInsts = AMDGPU::getRegBitWidth(RC->getID()) / 32; in canInsertSelect()
2516 Select = BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B32), DstReg) in insertSelect()
2521 Select = BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e32), DstReg) in insertSelect()
2532 BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), DstReg) in insertSelect()
2541 AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3, in insertSelect()
2542 AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7, in insertSelect()
2543 AMDGPU::sub8, AMDGPU::sub9, AMDGPU::sub10, AMDGPU::sub11, in insertSelect()
2544 AMDGPU::sub12, AMDGPU::sub13, AMDGPU::sub14, AMDGPU::sub15, in insertSelect()
2548 AMDGPU::sub0_sub1, AMDGPU::sub2_sub3, in insertSelect()
2549 AMDGPU::sub4_sub5, AMDGPU::sub6_sub7, in insertSelect()
2550 AMDGPU::sub8_sub9, AMDGPU::sub10_sub11, in insertSelect()
2551 AMDGPU::sub12_sub13, AMDGPU::sub14_sub15, in insertSelect()
2554 unsigned SelOp = AMDGPU::V_CNDMASK_B32_e32; in insertSelect()
2555 const TargetRegisterClass *EltRC = &AMDGPU::VGPR_32RegClass; in insertSelect()
2563 SelOp = AMDGPU::S_CSELECT_B32; in insertSelect()
2564 EltRC = &AMDGPU::SGPR_32RegClass; in insertSelect()
2566 SelOp = AMDGPU::S_CSELECT_B64; in insertSelect()
2567 EltRC = &AMDGPU::SGPR_64RegClass; in insertSelect()
2574 MBB, I, DL, get(AMDGPU::REG_SEQUENCE), DstReg); in insertSelect()
2586 if (SelOp == AMDGPU::V_CNDMASK_B32_e32) { in insertSelect()
2608 case AMDGPU::V_MOV_B32_e32: in isFoldableCopy()
2609 case AMDGPU::V_MOV_B32_e64: in isFoldableCopy()
2610 case AMDGPU::V_MOV_B64_PSEUDO: { in isFoldableCopy()
2618 case AMDGPU::S_MOV_B32: in isFoldableCopy()
2619 case AMDGPU::S_MOV_B64: in isFoldableCopy()
2620 case AMDGPU::COPY: in isFoldableCopy()
2621 case AMDGPU::V_ACCVGPR_WRITE_B32: in isFoldableCopy()
2622 case AMDGPU::V_ACCVGPR_READ_B32: in isFoldableCopy()
2648 int Src0ModIdx = AMDGPU::getNamedOperandIdx(Opc, in removeModOperands()
2649 AMDGPU::OpName::src0_modifiers); in removeModOperands()
2650 int Src1ModIdx = AMDGPU::getNamedOperandIdx(Opc, in removeModOperands()
2651 AMDGPU::OpName::src1_modifiers); in removeModOperands()
2652 int Src2ModIdx = AMDGPU::getNamedOperandIdx(Opc, in removeModOperands()
2653 AMDGPU::OpName::src2_modifiers); in removeModOperands()
2668 case AMDGPU::S_MOV_B64: in FoldImmediate()
2673 case AMDGPU::V_MOV_B32_e32: in FoldImmediate()
2674 case AMDGPU::S_MOV_B32: in FoldImmediate()
2675 case AMDGPU::V_ACCVGPR_WRITE_B32: in FoldImmediate()
2679 const MachineOperand *ImmOp = getNamedOperand(DefMI, AMDGPU::OpName::src0); in FoldImmediate()
2686 if (Opc == AMDGPU::COPY) { in FoldImmediate()
2690 unsigned NewOpc = isVGPRCopy ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32; in FoldImmediate()
2693 if (UseMI.getOperand(1).getSubReg() == AMDGPU::hi16) in FoldImmediate()
2699 NewOpc = AMDGPU::V_ACCVGPR_WRITE_B32; in FoldImmediate()
2707 UseMI.getOperand(0).getSubReg() != AMDGPU::lo16) in FoldImmediate()
2724 if (Opc == AMDGPU::V_MAD_F32 || Opc == AMDGPU::V_MAC_F32_e64 || in FoldImmediate()
2725 Opc == AMDGPU::V_MAD_F16 || Opc == AMDGPU::V_MAC_F16_e64 || in FoldImmediate()
2726 Opc == AMDGPU::V_FMA_F32 || Opc == AMDGPU::V_FMAC_F32_e64 || in FoldImmediate()
2727 Opc == AMDGPU::V_FMA_F16 || Opc == AMDGPU::V_FMAC_F16_e64) { in FoldImmediate()
2736 MachineOperand *Src0 = getNamedOperand(UseMI, AMDGPU::OpName::src0); in FoldImmediate()
2742 bool IsF32 = Opc == AMDGPU::V_MAD_F32 || Opc == AMDGPU::V_MAC_F32_e64 || in FoldImmediate()
2743 Opc == AMDGPU::V_FMA_F32 || Opc == AMDGPU::V_FMAC_F32_e64; in FoldImmediate()
2744 bool IsFMA = Opc == AMDGPU::V_FMA_F32 || Opc == AMDGPU::V_FMAC_F32_e64 || in FoldImmediate()
2745 Opc == AMDGPU::V_FMA_F16 || Opc == AMDGPU::V_FMAC_F16_e64; in FoldImmediate()
2746 MachineOperand *Src1 = getNamedOperand(UseMI, AMDGPU::OpName::src1); in FoldImmediate()
2747 MachineOperand *Src2 = getNamedOperand(UseMI, AMDGPU::OpName::src2); in FoldImmediate()
2759 IsFMA ? (IsF32 ? AMDGPU::V_FMAMK_F32 : AMDGPU::V_FMAMK_F16) in FoldImmediate()
2760 : (IsF32 ? AMDGPU::V_MADMK_F32 : AMDGPU::V_MADMK_F16); in FoldImmediate()
2773 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod)); in FoldImmediate()
2775 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp)); in FoldImmediate()
2783 if (Opc == AMDGPU::V_MAC_F32_e64 || in FoldImmediate()
2784 Opc == AMDGPU::V_MAC_F16_e64 || in FoldImmediate()
2785 Opc == AMDGPU::V_FMAC_F32_e64 || in FoldImmediate()
2786 Opc == AMDGPU::V_FMAC_F16_e64) in FoldImmediate()
2788 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2)); in FoldImmediate()
2844 IsFMA ? (IsF32 ? AMDGPU::V_FMAAK_F32 : AMDGPU::V_FMAAK_F16) in FoldImmediate()
2845 : (IsF32 ? AMDGPU::V_MADAK_F32 : AMDGPU::V_MADAK_F16); in FoldImmediate()
2856 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod)); in FoldImmediate()
2858 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp)); in FoldImmediate()
2860 if (Opc == AMDGPU::V_MAC_F32_e64 || in FoldImmediate()
2861 Opc == AMDGPU::V_MAC_F16_e64 || in FoldImmediate()
2862 Opc == AMDGPU::V_FMAC_F32_e64 || in FoldImmediate()
2863 Opc == AMDGPU::V_FMAC_F16_e64) in FoldImmediate()
2865 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2)); in FoldImmediate()
2989 if (Def && Def->getOpcode() == AMDGPU::V_MOV_B32_e32 && in getFoldableImm()
2992 return AMDGPU::NoRegister; in getFoldableImm()
3012 bool IsFMA = Opc == AMDGPU::V_FMAC_F32_e32 || Opc == AMDGPU::V_FMAC_F32_e64 || in convertToThreeAddress()
3013 Opc == AMDGPU::V_FMAC_F16_e32 || Opc == AMDGPU::V_FMAC_F16_e64; in convertToThreeAddress()
3018 case AMDGPU::V_MAC_F16_e64: in convertToThreeAddress()
3019 case AMDGPU::V_FMAC_F16_e64: in convertToThreeAddress()
3022 case AMDGPU::V_MAC_F32_e64: in convertToThreeAddress()
3023 case AMDGPU::V_FMAC_F32_e64: in convertToThreeAddress()
3025 case AMDGPU::V_MAC_F16_e32: in convertToThreeAddress()
3026 case AMDGPU::V_FMAC_F16_e32: in convertToThreeAddress()
3029 case AMDGPU::V_MAC_F32_e32: in convertToThreeAddress()
3030 case AMDGPU::V_FMAC_F32_e32: { in convertToThreeAddress()
3031 int Src0Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), in convertToThreeAddress()
3032 AMDGPU::OpName::src0); in convertToThreeAddress()
3044 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst); in convertToThreeAddress()
3045 const MachineOperand *Src0 = getNamedOperand(MI, AMDGPU::OpName::src0); in convertToThreeAddress()
3047 getNamedOperand(MI, AMDGPU::OpName::src0_modifiers); in convertToThreeAddress()
3048 const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1); in convertToThreeAddress()
3050 getNamedOperand(MI, AMDGPU::OpName::src1_modifiers); in convertToThreeAddress()
3051 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2); in convertToThreeAddress()
3052 const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp); in convertToThreeAddress()
3053 const MachineOperand *Omod = getNamedOperand(MI, AMDGPU::OpName::omod); in convertToThreeAddress()
3062 IsFMA ? (IsF16 ? AMDGPU::V_FMAAK_F16 : AMDGPU::V_FMAAK_F32) in convertToThreeAddress()
3063 : (IsF16 ? AMDGPU::V_MADAK_F16 : AMDGPU::V_MADAK_F32); in convertToThreeAddress()
3075 ? (IsF16 ? AMDGPU::V_FMAMK_F16 : AMDGPU::V_FMAMK_F32) in convertToThreeAddress()
3076 : (IsF16 ? AMDGPU::V_MADMK_F16 : AMDGPU::V_MADMK_F32); in convertToThreeAddress()
3091 MI, AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::src0), in convertToThreeAddress()
3104 unsigned NewOpc = IsFMA ? (IsF16 ? AMDGPU::V_FMA_F16 : AMDGPU::V_FMA_F32) in convertToThreeAddress()
3105 : (IsF16 ? AMDGPU::V_MAD_F16 : AMDGPU::V_MAD_F32); in convertToThreeAddress()
3128 case AMDGPU::S_SET_GPR_IDX_ON: in changesVGPRIndexingMode()
3129 case AMDGPU::S_SET_GPR_IDX_MODE: in changesVGPRIndexingMode()
3130 case AMDGPU::S_SET_GPR_IDX_OFF: in changesVGPRIndexingMode()
3158 return MI.modifiesRegister(AMDGPU::EXEC, &RI) || in isSchedulingBoundary()
3159 MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 || in isSchedulingBoundary()
3160 MI.getOpcode() == AMDGPU::S_SETREG_B32 || in isSchedulingBoundary()
3165 return Opcode == AMDGPU::DS_ORDERED_COUNT || in isAlwaysGDS()
3166 Opcode == AMDGPU::DS_GWS_INIT || in isAlwaysGDS()
3167 Opcode == AMDGPU::DS_GWS_SEMA_V || in isAlwaysGDS()
3168 Opcode == AMDGPU::DS_GWS_SEMA_BR || in isAlwaysGDS()
3169 Opcode == AMDGPU::DS_GWS_SEMA_P || in isAlwaysGDS()
3170 Opcode == AMDGPU::DS_GWS_SEMA_RELEASE_ALL || in isAlwaysGDS()
3171 Opcode == AMDGPU::DS_GWS_BARRIER; in isAlwaysGDS()
3180 if (*ImpDef == AMDGPU::MODE) in modifiesModeRegister()
3204 if (Opcode == AMDGPU::S_SENDMSG || Opcode == AMDGPU::S_SENDMSGHALT || in hasUnwantedEffectsWhenEXECEmpty()
3206 Opcode == AMDGPU::DS_ORDERED_COUNT || Opcode == AMDGPU::S_TRAP || in hasUnwantedEffectsWhenEXECEmpty()
3207 Opcode == AMDGPU::DS_GWS_INIT || Opcode == AMDGPU::DS_GWS_BARRIER) in hasUnwantedEffectsWhenEXECEmpty()
3222 if (Opcode == AMDGPU::V_READFIRSTLANE_B32 || in hasUnwantedEffectsWhenEXECEmpty()
3223 Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32) in hasUnwantedEffectsWhenEXECEmpty()
3240 return MI.readsRegister(AMDGPU::EXEC, &RI); in mayReadEXEC()
3251 return !isSALU(MI) || MI.readsRegister(AMDGPU::EXEC, &RI); in mayReadEXEC()
3260 return AMDGPU::isInlinableLiteral32(Imm.getSExtValue(), in isInlineConstant()
3263 return AMDGPU::isInlinableLiteral64(Imm.getSExtValue(), in isInlineConstant()
3267 AMDGPU::isInlinableLiteral16(Imm.getSExtValue(), in isInlineConstant()
3277 OperandType < AMDGPU::OPERAND_SRC_FIRST || in isInlineConstant()
3278 OperandType > AMDGPU::OPERAND_SRC_LAST) in isInlineConstant()
3288 case AMDGPU::OPERAND_REG_IMM_INT32: in isInlineConstant()
3289 case AMDGPU::OPERAND_REG_IMM_FP32: in isInlineConstant()
3290 case AMDGPU::OPERAND_REG_INLINE_C_INT32: in isInlineConstant()
3291 case AMDGPU::OPERAND_REG_INLINE_C_FP32: in isInlineConstant()
3292 case AMDGPU::OPERAND_REG_INLINE_AC_INT32: in isInlineConstant()
3293 case AMDGPU::OPERAND_REG_INLINE_AC_FP32: { in isInlineConstant()
3295 return AMDGPU::isInlinableLiteral32(Trunc, ST.hasInv2PiInlineImm()); in isInlineConstant()
3297 case AMDGPU::OPERAND_REG_IMM_INT64: in isInlineConstant()
3298 case AMDGPU::OPERAND_REG_IMM_FP64: in isInlineConstant()
3299 case AMDGPU::OPERAND_REG_INLINE_C_INT64: in isInlineConstant()
3300 case AMDGPU::OPERAND_REG_INLINE_C_FP64: in isInlineConstant()
3301 return AMDGPU::isInlinableLiteral64(MO.getImm(), in isInlineConstant()
3303 case AMDGPU::OPERAND_REG_IMM_INT16: in isInlineConstant()
3304 case AMDGPU::OPERAND_REG_INLINE_C_INT16: in isInlineConstant()
3305 case AMDGPU::OPERAND_REG_INLINE_AC_INT16: in isInlineConstant()
3316 return AMDGPU::isInlinableIntLiteral(Imm); in isInlineConstant()
3317 case AMDGPU::OPERAND_REG_IMM_V2INT16: in isInlineConstant()
3318 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16: in isInlineConstant()
3319 case AMDGPU::OPERAND_REG_INLINE_AC_V2INT16: in isInlineConstant()
3321 return AMDGPU::isInlinableIntLiteralV216(Imm); in isInlineConstant()
3322 case AMDGPU::OPERAND_REG_IMM_FP16: in isInlineConstant()
3323 case AMDGPU::OPERAND_REG_INLINE_C_FP16: in isInlineConstant()
3324 case AMDGPU::OPERAND_REG_INLINE_AC_FP16: { in isInlineConstant()
3332 AMDGPU::isInlinableLiteral16(Trunc, ST.hasInv2PiInlineImm()); in isInlineConstant()
3337 case AMDGPU::OPERAND_REG_IMM_V2FP16: in isInlineConstant()
3338 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: in isInlineConstant()
3339 case AMDGPU::OPERAND_REG_INLINE_AC_V2FP16: { in isInlineConstant()
3341 return AMDGPU::isInlinableLiteralV216(Trunc, ST.hasInv2PiInlineImm()); in isInlineConstant()
3396 OpNo ==(unsigned)AMDGPU::getNamedOperandIdx(MI.getOpcode(), in isImmOperandLegal()
3397 AMDGPU::OpName::src2)) in isImmOperandLegal()
3405 if (!isVOP3(MI) || !AMDGPU::isSISrcOperand(InstDesc, OpNo)) in isImmOperandLegal()
3412 int Op32 = AMDGPU::getVOPe32(Opcode); in hasVALU32BitEncoding()
3423 return AMDGPU::getNamedOperandIdx(Opcode, in hasModifiers()
3424 AMDGPU::OpName::src0_modifiers) != -1; in hasModifiers()
3434 return hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers) || in hasAnyModifiersSet()
3435 hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers) || in hasAnyModifiersSet()
3436 hasModifiersSet(MI, AMDGPU::OpName::src2_modifiers) || in hasAnyModifiersSet()
3437 hasModifiersSet(MI, AMDGPU::OpName::clamp) || in hasAnyModifiersSet()
3438 hasModifiersSet(MI, AMDGPU::OpName::omod); in hasAnyModifiersSet()
3443 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2); in canShrink()
3455 case AMDGPU::V_ADDC_U32_e64: in canShrink()
3456 case AMDGPU::V_SUBB_U32_e64: in canShrink()
3457 case AMDGPU::V_SUBBREV_U32_e64: { in canShrink()
3459 = getNamedOperand(MI, AMDGPU::OpName::src1); in canShrink()
3465 case AMDGPU::V_MAC_F32_e64: in canShrink()
3466 case AMDGPU::V_MAC_F16_e64: in canShrink()
3467 case AMDGPU::V_FMAC_F32_e64: in canShrink()
3468 case AMDGPU::V_FMAC_F16_e64: in canShrink()
3470 hasModifiersSet(MI, AMDGPU::OpName::src2_modifiers)) in canShrink()
3474 case AMDGPU::V_CNDMASK_B32_e64: in canShrink()
3479 const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1); in canShrink()
3481 hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers))) in canShrink()
3486 if (hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers)) in canShrink()
3494 return !hasModifiersSet(MI, AMDGPU::OpName::omod) && in canShrink()
3495 !hasModifiersSet(MI, AMDGPU::OpName::clamp); in canShrink()
3505 (Use.getReg() == AMDGPU::VCC || Use.getReg() == AMDGPU::VCC_LO)) { in copyFlagsToImplicitVCC()
3522 int Op32DstIdx = AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::vdst); in buildShrunkInst()
3527 assert(((MI.getOperand(0).getReg() == AMDGPU::VCC) || in buildShrunkInst()
3528 (MI.getOperand(0).getReg() == AMDGPU::VCC_LO)) && in buildShrunkInst()
3532 Inst32.add(*getNamedOperand(MI, AMDGPU::OpName::src0)); in buildShrunkInst()
3534 const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1); in buildShrunkInst()
3538 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2); in buildShrunkInst()
3541 int Op32Src2Idx = AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::src2); in buildShrunkInst()
3577 if (MO.getReg() == AMDGPU::SGPR_NULL) in usesConstantBus()
3582 return MO.getReg() == AMDGPU::M0 || in usesConstantBus()
3583 MO.getReg() == AMDGPU::VCC || in usesConstantBus()
3584 MO.getReg() == AMDGPU::VCC_LO; in usesConstantBus()
3586 return AMDGPU::SReg_32RegClass.contains(MO.getReg()) || in usesConstantBus()
3587 AMDGPU::SReg_64RegClass.contains(MO.getReg()); in usesConstantBus()
3598 case AMDGPU::VCC: in findImplicitSGPRRead()
3599 case AMDGPU::VCC_LO: in findImplicitSGPRRead()
3600 case AMDGPU::VCC_HI: in findImplicitSGPRRead()
3601 case AMDGPU::M0: in findImplicitSGPRRead()
3602 case AMDGPU::FLAT_SCR: in findImplicitSGPRRead()
3610 return AMDGPU::NoRegister; in findImplicitSGPRRead()
3616 case AMDGPU::V_READLANE_B32: in shouldReadExec()
3617 case AMDGPU::V_WRITELANE_B32: in shouldReadExec()
3639 return SubReg.getSubReg() != AMDGPU::NoSubRegister && in isSubRegOf()
3652 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0); in verifyInstruction()
3653 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1); in verifyInstruction()
3654 int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2); in verifyInstruction()
3708 case AMDGPU::OPERAND_REG_IMM_INT32: in verifyInstruction()
3709 case AMDGPU::OPERAND_REG_IMM_FP32: in verifyInstruction()
3711 case AMDGPU::OPERAND_REG_INLINE_C_INT32: in verifyInstruction()
3712 case AMDGPU::OPERAND_REG_INLINE_C_FP32: in verifyInstruction()
3713 case AMDGPU::OPERAND_REG_INLINE_C_INT64: in verifyInstruction()
3714 case AMDGPU::OPERAND_REG_INLINE_C_FP64: in verifyInstruction()
3715 case AMDGPU::OPERAND_REG_INLINE_C_INT16: in verifyInstruction()
3716 case AMDGPU::OPERAND_REG_INLINE_C_FP16: in verifyInstruction()
3717 case AMDGPU::OPERAND_REG_INLINE_AC_INT32: in verifyInstruction()
3718 case AMDGPU::OPERAND_REG_INLINE_AC_FP32: in verifyInstruction()
3719 case AMDGPU::OPERAND_REG_INLINE_AC_INT16: in verifyInstruction()
3720 case AMDGPU::OPERAND_REG_INLINE_AC_FP16: { in verifyInstruction()
3729 case AMDGPU::OPERAND_KIMM32: in verifyInstruction()
3747 if (Reg == AMDGPU::NoRegister || Reg.isVirtual()) in verifyInstruction()
3765 int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst); in verifyInstruction()
3792 const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod); in verifyInstruction()
3800 uint16_t BasicOpcode = AMDGPU::getBasicFromSDWAOp(Opcode); in verifyInstruction()
3805 if (!Dst.isReg() || Dst.getReg() != AMDGPU::VCC) { in verifyInstruction()
3811 const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp); in verifyInstruction()
3818 const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod); in verifyInstruction()
3826 const MachineOperand *DstUnused = getNamedOperand(MI, AMDGPU::OpName::dst_unused); in verifyInstruction()
3828 DstUnused->getImm() == AMDGPU::SDWA::UNUSED_PRESERVE) { in verifyInstruction()
3853 const MachineOperand *DMask = getNamedOperand(MI, AMDGPU::OpName::dmask); in verifyInstruction()
3858 const MachineOperand *TFE = getNamedOperand(MI, AMDGPU::OpName::tfe); in verifyInstruction()
3859 const MachineOperand *LWE = getNamedOperand(MI, AMDGPU::OpName::lwe); in verifyInstruction()
3860 const MachineOperand *D16 = getNamedOperand(MI, AMDGPU::OpName::d16); in verifyInstruction()
3871 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdata); in verifyInstruction()
3886 if (Desc.getOpcode() != AMDGPU::V_WRITELANE_B32 in verifyInstruction()
3896 if (AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::imm) != -1) in verifyInstruction()
3923 if (SGPRUsed != AMDGPU::NoRegister) { in verifyInstruction()
3936 Opcode != AMDGPU::V_WRITELANE_B32) { in verifyInstruction()
3955 if (Desc.getOpcode() == AMDGPU::V_WRITELANE_B32) { in verifyInstruction()
3957 Register SGPRUsed = AMDGPU::NoRegister; in verifyInstruction()
3966 if (MO.isReg() && MO.getReg() != AMDGPU::M0) { in verifyInstruction()
3980 if (Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F32 || in verifyInstruction()
3981 Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F64) { in verifyInstruction()
3992 if ((getNamedOperand(MI, AMDGPU::OpName::src0_modifiers)->getImm() & in verifyInstruction()
3994 (getNamedOperand(MI, AMDGPU::OpName::src1_modifiers)->getImm() & in verifyInstruction()
3996 (getNamedOperand(MI, AMDGPU::OpName::src2_modifiers)->getImm() & in verifyInstruction()
4022 auto Op = getNamedOperand(MI, AMDGPU::OpName::simm16); in verifyInstruction()
4044 if (Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e32 || in verifyInstruction()
4045 Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e64 || in verifyInstruction()
4046 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 || in verifyInstruction()
4047 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64) { in verifyInstruction()
4048 const bool IsDst = Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 || in verifyInstruction()
4049 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64; in verifyInstruction()
4063 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst); in verifyInstruction()
4091 if (!MI.hasRegisterImplicitUseOperand(AMDGPU::EXEC)) { in verifyInstruction()
4101 const MachineOperand *Soff = getNamedOperand(MI, AMDGPU::OpName::soff); in verifyInstruction()
4102 if (Soff && Soff->getReg() != AMDGPU::M0) { in verifyInstruction()
4110 const MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset); in verifyInstruction()
4118 const MachineOperand *DimOp = getNamedOperand(MI, AMDGPU::OpName::dim); in verifyInstruction()
4120 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opcode, in verifyInstruction()
4121 AMDGPU::OpName::vaddr0); in verifyInstruction()
4122 int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::srsrc); in verifyInstruction()
4123 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opcode); in verifyInstruction()
4124 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode = in verifyInstruction()
4125 AMDGPU::getMIMGBaseOpcodeInfo(Info->BaseOpcode); in verifyInstruction()
4126 const AMDGPU::MIMGDimInfo *Dim = in verifyInstruction()
4127 AMDGPU::getMIMGDimInfoByEncoding(DimOp->getImm()); in verifyInstruction()
4136 const MachineOperand *R128A16 = getNamedOperand(MI, AMDGPU::OpName::r128); in verifyInstruction()
4139 const MachineOperand *A16 = getNamedOperand(MI, AMDGPU::OpName::a16); in verifyInstruction()
4188 const MachineOperand *DppCt = getNamedOperand(MI, AMDGPU::OpName::dpp_ctrl); in verifyInstruction()
4190 using namespace AMDGPU::DPP; in verifyInstruction()
4228 default: return AMDGPU::INSTRUCTION_LIST_END; in getVALUOp()
4229 case AMDGPU::REG_SEQUENCE: return AMDGPU::REG_SEQUENCE; in getVALUOp()
4230 case AMDGPU::COPY: return AMDGPU::COPY; in getVALUOp()
4231 case AMDGPU::PHI: return AMDGPU::PHI; in getVALUOp()
4232 case AMDGPU::INSERT_SUBREG: return AMDGPU::INSERT_SUBREG; in getVALUOp()
4233 case AMDGPU::WQM: return AMDGPU::WQM; in getVALUOp()
4234 case AMDGPU::SOFT_WQM: return AMDGPU::SOFT_WQM; in getVALUOp()
4235 case AMDGPU::WWM: return AMDGPU::WWM; in getVALUOp()
4236 case AMDGPU::S_MOV_B32: { in getVALUOp()
4240 AMDGPU::COPY : AMDGPU::V_MOV_B32_e32; in getVALUOp()
4242 case AMDGPU::S_ADD_I32: in getVALUOp()
4243 return ST.hasAddNoCarry() ? AMDGPU::V_ADD_U32_e64 : AMDGPU::V_ADD_CO_U32_e32; in getVALUOp()
4244 case AMDGPU::S_ADDC_U32: in getVALUOp()
4245 return AMDGPU::V_ADDC_U32_e32; in getVALUOp()
4246 case AMDGPU::S_SUB_I32: in getVALUOp()
4247 return ST.hasAddNoCarry() ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_SUB_CO_U32_e32; in getVALUOp()
4250 case AMDGPU::S_ADD_U32: in getVALUOp()
4251 return AMDGPU::V_ADD_CO_U32_e32; in getVALUOp()
4252 case AMDGPU::S_SUB_U32: in getVALUOp()
4253 return AMDGPU::V_SUB_CO_U32_e32; in getVALUOp()
4254 case AMDGPU::S_SUBB_U32: return AMDGPU::V_SUBB_U32_e32; in getVALUOp()
4255 case AMDGPU::S_MUL_I32: return AMDGPU::V_MUL_LO_U32; in getVALUOp()
4256 case AMDGPU::S_MUL_HI_U32: return AMDGPU::V_MUL_HI_U32; in getVALUOp()
4257 case AMDGPU::S_MUL_HI_I32: return AMDGPU::V_MUL_HI_I32; in getVALUOp()
4258 case AMDGPU::S_AND_B32: return AMDGPU::V_AND_B32_e64; in getVALUOp()
4259 case AMDGPU::S_OR_B32: return AMDGPU::V_OR_B32_e64; in getVALUOp()
4260 case AMDGPU::S_XOR_B32: return AMDGPU::V_XOR_B32_e64; in getVALUOp()
4261 case AMDGPU::S_XNOR_B32: in getVALUOp()
4262 return ST.hasDLInsts() ? AMDGPU::V_XNOR_B32_e64 : AMDGPU::INSTRUCTION_LIST_END; in getVALUOp()
4263 case AMDGPU::S_MIN_I32: return AMDGPU::V_MIN_I32_e64; in getVALUOp()
4264 case AMDGPU::S_MIN_U32: return AMDGPU::V_MIN_U32_e64; in getVALUOp()
4265 case AMDGPU::S_MAX_I32: return AMDGPU::V_MAX_I32_e64; in getVALUOp()
4266 case AMDGPU::S_MAX_U32: return AMDGPU::V_MAX_U32_e64; in getVALUOp()
4267 case AMDGPU::S_ASHR_I32: return AMDGPU::V_ASHR_I32_e32; in getVALUOp()
4268 case AMDGPU::S_ASHR_I64: return AMDGPU::V_ASHR_I64; in getVALUOp()
4269 case AMDGPU::S_LSHL_B32: return AMDGPU::V_LSHL_B32_e32; in getVALUOp()
4270 case AMDGPU::S_LSHL_B64: return AMDGPU::V_LSHL_B64; in getVALUOp()
4271 case AMDGPU::S_LSHR_B32: return AMDGPU::V_LSHR_B32_e32; in getVALUOp()
4272 case AMDGPU::S_LSHR_B64: return AMDGPU::V_LSHR_B64; in getVALUOp()
4273 case AMDGPU::S_SEXT_I32_I8: return AMDGPU::V_BFE_I32; in getVALUOp()
4274 case AMDGPU::S_SEXT_I32_I16: return AMDGPU::V_BFE_I32; in getVALUOp()
4275 case AMDGPU::S_BFE_U32: return AMDGPU::V_BFE_U32; in getVALUOp()
4276 case AMDGPU::S_BFE_I32: return AMDGPU::V_BFE_I32; in getVALUOp()
4277 case AMDGPU::S_BFM_B32: return AMDGPU::V_BFM_B32_e64; in getVALUOp()
4278 case AMDGPU::S_BREV_B32: return AMDGPU::V_BFREV_B32_e32; in getVALUOp()
4279 case AMDGPU::S_NOT_B32: return AMDGPU::V_NOT_B32_e32; in getVALUOp()
4280 case AMDGPU::S_NOT_B64: return AMDGPU::V_NOT_B32_e32; in getVALUOp()
4281 case AMDGPU::S_CMP_EQ_I32: return AMDGPU::V_CMP_EQ_I32_e32; in getVALUOp()
4282 case AMDGPU::S_CMP_LG_I32: return AMDGPU::V_CMP_NE_I32_e32; in getVALUOp()
4283 case AMDGPU::S_CMP_GT_I32: return AMDGPU::V_CMP_GT_I32_e32; in getVALUOp()
4284 case AMDGPU::S_CMP_GE_I32: return AMDGPU::V_CMP_GE_I32_e32; in getVALUOp()
4285 case AMDGPU::S_CMP_LT_I32: return AMDGPU::V_CMP_LT_I32_e32; in getVALUOp()
4286 case AMDGPU::S_CMP_LE_I32: return AMDGPU::V_CMP_LE_I32_e32; in getVALUOp()
4287 case AMDGPU::S_CMP_EQ_U32: return AMDGPU::V_CMP_EQ_U32_e32; in getVALUOp()
4288 case AMDGPU::S_CMP_LG_U32: return AMDGPU::V_CMP_NE_U32_e32; in getVALUOp()
4289 case AMDGPU::S_CMP_GT_U32: return AMDGPU::V_CMP_GT_U32_e32; in getVALUOp()
4290 case AMDGPU::S_CMP_GE_U32: return AMDGPU::V_CMP_GE_U32_e32; in getVALUOp()
4291 case AMDGPU::S_CMP_LT_U32: return AMDGPU::V_CMP_LT_U32_e32; in getVALUOp()
4292 case AMDGPU::S_CMP_LE_U32: return AMDGPU::V_CMP_LE_U32_e32; in getVALUOp()
4293 case AMDGPU::S_CMP_EQ_U64: return AMDGPU::V_CMP_EQ_U64_e32; in getVALUOp()
4294 case AMDGPU::S_CMP_LG_U64: return AMDGPU::V_CMP_NE_U64_e32; in getVALUOp()
4295 case AMDGPU::S_BCNT1_I32_B32: return AMDGPU::V_BCNT_U32_B32_e64; in getVALUOp()
4296 case AMDGPU::S_FF1_I32_B32: return AMDGPU::V_FFBL_B32_e32; in getVALUOp()
4297 case AMDGPU::S_FLBIT_I32_B32: return AMDGPU::V_FFBH_U32_e32; in getVALUOp()
4298 case AMDGPU::S_FLBIT_I32: return AMDGPU::V_FFBH_I32_e64; in getVALUOp()
4299 case AMDGPU::S_CBRANCH_SCC0: return AMDGPU::S_CBRANCH_VCCZ; in getVALUOp()
4300 case AMDGPU::S_CBRANCH_SCC1: return AMDGPU::S_CBRANCH_VCCNZ; in getVALUOp()
4331 unsigned Opcode = (Size == 64) ? AMDGPU::V_MOV_B64_PSEUDO : AMDGPU::V_MOV_B32_e32; in legalizeOpWithMove()
4333 Opcode = AMDGPU::COPY; in legalizeOpWithMove()
4335 Opcode = (Size == 64) ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32; in legalizeOpWithMove()
4338 if (RI.getCommonSubClass(&AMDGPU::VReg_64RegClass, VRC)) in legalizeOpWithMove()
4339 VRC = &AMDGPU::VReg_64RegClass; in legalizeOpWithMove()
4341 VRC = &AMDGPU::VGPR_32RegClass; in legalizeOpWithMove()
4360 if (SuperReg.getSubReg() == AMDGPU::NoSubRegister) { in buildExtractSubReg()
4389 if (SubIdx == AMDGPU::sub0) in buildExtractSubRegOrImm()
4391 if (SubIdx == AMDGPU::sub1) in buildExtractSubRegOrImm()
4481 } else if (InstDesc.OpInfo[i].OperandType == AMDGPU::OPERAND_KIMM32) { in isOperandLegal()
4484 } else if (isVOP3(MI) && AMDGPU::isSISrcOperand(InstDesc, i) && in isOperandLegal()
4515 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0); in legalizeOperandsVOP2()
4518 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1); in legalizeOperandsVOP2()
4523 bool HasImplicitSGPR = findImplicitSGPRRead(MI) != AMDGPU::NoRegister; in legalizeOperandsVOP2()
4532 if (Opc == AMDGPU::V_WRITELANE_B32) { in legalizeOperandsVOP2()
4535 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); in legalizeOperandsVOP2()
4536 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) in legalizeOperandsVOP2()
4541 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); in legalizeOperandsVOP2()
4543 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) in legalizeOperandsVOP2()
4565 if (Opc == AMDGPU::V_READLANE_B32 && Src1.isReg() && in legalizeOperandsVOP2()
4567 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); in legalizeOperandsVOP2()
4569 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) in legalizeOperandsVOP2()
4627 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0), in legalizeOperandsVOP3()
4628 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1), in legalizeOperandsVOP3()
4629 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2) in legalizeOperandsVOP3()
4632 if (Opc == AMDGPU::V_PERMLANE16_B32 || in legalizeOperandsVOP3()
4633 Opc == AMDGPU::V_PERMLANEX16_B32) { in legalizeOperandsVOP3()
4639 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); in legalizeOperandsVOP3()
4640 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) in legalizeOperandsVOP3()
4645 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); in legalizeOperandsVOP3()
4646 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) in legalizeOperandsVOP3()
4657 if (SGPRReg != AMDGPU::NoRegister) { in legalizeOperandsVOP3()
4727 get(AMDGPU::V_READFIRSTLANE_B32), DstReg) in readlaneVGPRToSGPR()
4734 Register SGPR = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); in readlaneVGPRToSGPR()
4736 get(AMDGPU::V_READFIRSTLANE_B32), SGPR) in readlaneVGPRToSGPR()
4743 get(AMDGPU::REG_SEQUENCE), DstReg); in readlaneVGPRToSGPR()
4758 MachineOperand *SBase = getNamedOperand(MI, AMDGPU::OpName::sbase); in legalizeOperandsSMRD()
4763 MachineOperand *SOff = getNamedOperand(MI, AMDGPU::OpName::soff); in legalizeOperandsSMRD()
4778 MachineOperand *SAddr = getNamedOperand(MI, AMDGPU::OpName::saddr); in legalizeOperandsFLAT()
4804 BuildMI(InsertMBB, I, DL, get(AMDGPU::COPY), DstReg).add(Op); in legalizeGenericOperand()
4814 if (Def->isMoveImmediate() && DstRC != &AMDGPU::VReg_1RegClass) in legalizeGenericOperand()
4824 if (!RI.isSGPRClass(DstRC) && !Copy->readsRegister(AMDGPU::EXEC, &RI) && in legalizeGenericOperand()
4826 Copy->addOperand(MachineOperand::CreateReg(AMDGPU::EXEC, false, true)); in legalizeGenericOperand()
4839 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; in emitLoadSRsrcFromVGPRLoop()
4841 ST.isWave32() ? AMDGPU::S_AND_SAVEEXEC_B32 : AMDGPU::S_AND_SAVEEXEC_B64; in emitLoadSRsrcFromVGPRLoop()
4843 ST.isWave32() ? AMDGPU::S_XOR_B32_term : AMDGPU::S_XOR_B64_term; in emitLoadSRsrcFromVGPRLoop()
4845 ST.isWave32() ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64; in emitLoadSRsrcFromVGPRLoop()
4846 const auto *BoolXExecRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID); in emitLoadSRsrcFromVGPRLoop()
4851 Register CondReg = AMDGPU::NoRegister; in emitLoadSRsrcFromVGPRLoop()
4862 Register CurRegLo = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); in emitLoadSRsrcFromVGPRLoop()
4863 Register CurRegHi = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); in emitLoadSRsrcFromVGPRLoop()
4866 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurRegLo) in emitLoadSRsrcFromVGPRLoop()
4870 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurRegHi) in emitLoadSRsrcFromVGPRLoop()
4877 Register CurReg = MRI.createVirtualRegister(&AMDGPU::SGPR_64RegClass); in emitLoadSRsrcFromVGPRLoop()
4878 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), CurReg) in emitLoadSRsrcFromVGPRLoop()
4880 .addImm(AMDGPU::sub0) in emitLoadSRsrcFromVGPRLoop()
4882 .addImm(AMDGPU::sub1); in emitLoadSRsrcFromVGPRLoop()
4886 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64), NewCondReg) in emitLoadSRsrcFromVGPRLoop()
4894 if (CondReg == AMDGPU::NoRegister) // First. in emitLoadSRsrcFromVGPRLoop()
4909 auto Merge = BuildMI(LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), SRsrc); in emitLoadSRsrcFromVGPRLoop()
4935 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::S_CBRANCH_EXECNZ)).addMBB(&LoopBB); in emitLoadSRsrcFromVGPRLoop()
4958 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; in loadSRsrcFromVGPR()
4959 unsigned MovExecOpc = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64; in loadSRsrcFromVGPR()
4960 const auto *BoolXExecRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID); in loadSRsrcFromVGPR()
5031 TII.buildExtractSubReg(MI, MRI, Rsrc, &AMDGPU::VReg_128RegClass, in extractRsrcPtr()
5032 AMDGPU::sub0_sub1, &AMDGPU::VReg_64RegClass); in extractRsrcPtr()
5035 Register Zero64 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); in extractRsrcPtr()
5036 Register SRsrcFormatLo = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); in extractRsrcPtr()
5037 Register SRsrcFormatHi = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); in extractRsrcPtr()
5038 Register NewSRsrc = MRI.createVirtualRegister(&AMDGPU::SGPR_128RegClass); in extractRsrcPtr()
5042 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B64), Zero64) in extractRsrcPtr()
5046 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatLo) in extractRsrcPtr()
5050 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatHi) in extractRsrcPtr()
5054 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::REG_SEQUENCE), NewSRsrc) in extractRsrcPtr()
5056 .addImm(AMDGPU::sub0_sub1) in extractRsrcPtr()
5058 .addImm(AMDGPU::sub2) in extractRsrcPtr()
5060 .addImm(AMDGPU::sub3); in extractRsrcPtr()
5099 if (MI.getOpcode() == AMDGPU::PHI) { in legalizeOperands()
5119 if (getOpRegClass(MI, 0) == &AMDGPU::VReg_1RegClass) { in legalizeOperands()
5120 VRC = &AMDGPU::VReg_1RegClass; in legalizeOperands()
5154 if (MI.getOpcode() == AMDGPU::REG_SEQUENCE) { in legalizeOperands()
5181 if (MI.getOpcode() == AMDGPU::INSERT_SUBREG) { in legalizeOperands()
5195 if (MI.getOpcode() == AMDGPU::SI_INIT_M0) { in legalizeOperands()
5207 if (isMIMG(MI) || (AMDGPU::isGraphics(MF.getFunction().getCallingConv()) && in legalizeOperands()
5209 MachineOperand *SRsrc = getNamedOperand(MI, AMDGPU::OpName::srsrc); in legalizeOperands()
5213 MachineOperand *SSamp = getNamedOperand(MI, AMDGPU::OpName::ssamp); in legalizeOperands()
5221 if (MI.getOpcode() == AMDGPU::SI_CALL_ISEL) { in legalizeOperands()
5249 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::srsrc); in legalizeOperands()
5276 MachineOperand *VAddr = getNamedOperand(MI, AMDGPU::OpName::vaddr); in legalizeOperands()
5277 if (VAddr && AMDGPU::getIfAddr64Inst(MI.getOpcode()) != -1) { in legalizeOperands()
5280 Register NewVAddrLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in legalizeOperands()
5281 Register NewVAddrHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in legalizeOperands()
5282 Register NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); in legalizeOperands()
5284 const auto *BoolXExecRC = RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID); in legalizeOperands()
5293 BuildMI(MBB, MI, DL, get(AMDGPU::V_ADD_CO_U32_e64), NewVAddrLo) in legalizeOperands()
5295 .addReg(RsrcPtr, 0, AMDGPU::sub0) in legalizeOperands()
5296 .addReg(VAddr->getReg(), 0, AMDGPU::sub0) in legalizeOperands()
5300 BuildMI(MBB, MI, DL, get(AMDGPU::V_ADDC_U32_e64), NewVAddrHi) in legalizeOperands()
5302 .addReg(RsrcPtr, 0, AMDGPU::sub1) in legalizeOperands()
5303 .addReg(VAddr->getReg(), 0, AMDGPU::sub1) in legalizeOperands()
5308 BuildMI(MBB, MI, MI.getDebugLoc(), get(AMDGPU::REG_SEQUENCE), NewVAddr) in legalizeOperands()
5310 .addImm(AMDGPU::sub0) in legalizeOperands()
5312 .addImm(AMDGPU::sub1); in legalizeOperands()
5325 Register NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); in legalizeOperands()
5326 MachineOperand *VData = getNamedOperand(MI, AMDGPU::OpName::vdata); in legalizeOperands()
5327 MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset); in legalizeOperands()
5328 MachineOperand *SOffset = getNamedOperand(MI, AMDGPU::OpName::soffset); in legalizeOperands()
5329 unsigned Addr64Opcode = AMDGPU::getAddr64Inst(MI.getOpcode()); in legalizeOperands()
5333 MachineOperand *VDataIn = getNamedOperand(MI, AMDGPU::OpName::vdata_in); in legalizeOperands()
5348 getNamedOperand(MI, AMDGPU::OpName::glc)) { in legalizeOperands()
5352 getNamedOperand(MI, AMDGPU::OpName::dlc)) { in legalizeOperands()
5356 MIB.addImm(getNamedImmOperand(MI, AMDGPU::OpName::slc)); in legalizeOperands()
5359 getNamedOperand(MI, AMDGPU::OpName::tfe)) { in legalizeOperands()
5363 MIB.addImm(getNamedImmOperand(MI, AMDGPU::OpName::swz)); in legalizeOperands()
5376 .addImm(getNamedImmOperand(MI, AMDGPU::OpName::slc)) in legalizeOperands()
5383 BuildMI(MBB, Addr64, Addr64->getDebugLoc(), get(AMDGPU::REG_SEQUENCE), in legalizeOperands()
5385 .addReg(RsrcPtr, 0, AMDGPU::sub0) in legalizeOperands()
5386 .addImm(AMDGPU::sub0) in legalizeOperands()
5387 .addReg(RsrcPtr, 0, AMDGPU::sub1) in legalizeOperands()
5388 .addImm(AMDGPU::sub1); in legalizeOperands()
5418 case AMDGPU::S_ADD_U64_PSEUDO: in moveToVALU()
5419 case AMDGPU::S_SUB_U64_PSEUDO: in moveToVALU()
5423 case AMDGPU::S_ADD_I32: in moveToVALU()
5424 case AMDGPU::S_SUB_I32: { in moveToVALU()
5436 case AMDGPU::S_AND_B64: in moveToVALU()
5437 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_AND_B32, MDT); in moveToVALU()
5441 case AMDGPU::S_OR_B64: in moveToVALU()
5442 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_OR_B32, MDT); in moveToVALU()
5446 case AMDGPU::S_XOR_B64: in moveToVALU()
5447 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XOR_B32, MDT); in moveToVALU()
5451 case AMDGPU::S_NAND_B64: in moveToVALU()
5452 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NAND_B32, MDT); in moveToVALU()
5456 case AMDGPU::S_NOR_B64: in moveToVALU()
5457 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NOR_B32, MDT); in moveToVALU()
5461 case AMDGPU::S_XNOR_B64: in moveToVALU()
5463 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XNOR_B32, MDT); in moveToVALU()
5469 case AMDGPU::S_ANDN2_B64: in moveToVALU()
5470 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ANDN2_B32, MDT); in moveToVALU()
5474 case AMDGPU::S_ORN2_B64: in moveToVALU()
5475 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ORN2_B32, MDT); in moveToVALU()
5479 case AMDGPU::S_NOT_B64: in moveToVALU()
5480 splitScalar64BitUnaryOp(Worklist, Inst, AMDGPU::S_NOT_B32); in moveToVALU()
5484 case AMDGPU::S_BCNT1_I32_B64: in moveToVALU()
5489 case AMDGPU::S_BFE_I64: in moveToVALU()
5494 case AMDGPU::S_LSHL_B32: in moveToVALU()
5496 NewOpcode = AMDGPU::V_LSHLREV_B32_e64; in moveToVALU()
5500 case AMDGPU::S_ASHR_I32: in moveToVALU()
5502 NewOpcode = AMDGPU::V_ASHRREV_I32_e64; in moveToVALU()
5506 case AMDGPU::S_LSHR_B32: in moveToVALU()
5508 NewOpcode = AMDGPU::V_LSHRREV_B32_e64; in moveToVALU()
5512 case AMDGPU::S_LSHL_B64: in moveToVALU()
5514 NewOpcode = AMDGPU::V_LSHLREV_B64; in moveToVALU()
5518 case AMDGPU::S_ASHR_I64: in moveToVALU()
5520 NewOpcode = AMDGPU::V_ASHRREV_I64; in moveToVALU()
5524 case AMDGPU::S_LSHR_B64: in moveToVALU()
5526 NewOpcode = AMDGPU::V_LSHRREV_B64; in moveToVALU()
5531 case AMDGPU::S_ABS_I32: in moveToVALU()
5536 case AMDGPU::S_CBRANCH_SCC0: in moveToVALU()
5537 case AMDGPU::S_CBRANCH_SCC1: in moveToVALU()
5540 BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(AMDGPU::S_AND_B32), in moveToVALU()
5541 AMDGPU::VCC_LO) in moveToVALU()
5542 .addReg(AMDGPU::EXEC_LO) in moveToVALU()
5543 .addReg(AMDGPU::VCC_LO); in moveToVALU()
5545 BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(AMDGPU::S_AND_B64), in moveToVALU()
5546 AMDGPU::VCC) in moveToVALU()
5547 .addReg(AMDGPU::EXEC) in moveToVALU()
5548 .addReg(AMDGPU::VCC); in moveToVALU()
5551 case AMDGPU::S_BFE_U64: in moveToVALU()
5552 case AMDGPU::S_BFM_B64: in moveToVALU()
5555 case AMDGPU::S_PACK_LL_B32_B16: in moveToVALU()
5556 case AMDGPU::S_PACK_LH_B32_B16: in moveToVALU()
5557 case AMDGPU::S_PACK_HH_B32_B16: in moveToVALU()
5562 case AMDGPU::S_XNOR_B32: in moveToVALU()
5567 case AMDGPU::S_NAND_B32: in moveToVALU()
5568 splitScalarNotBinop(Worklist, Inst, AMDGPU::S_AND_B32); in moveToVALU()
5572 case AMDGPU::S_NOR_B32: in moveToVALU()
5573 splitScalarNotBinop(Worklist, Inst, AMDGPU::S_OR_B32); in moveToVALU()
5577 case AMDGPU::S_ANDN2_B32: in moveToVALU()
5578 splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_AND_B32); in moveToVALU()
5582 case AMDGPU::S_ORN2_B32: in moveToVALU()
5583 splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_OR_B32); in moveToVALU()
5591 case AMDGPU::S_ADD_CO_PSEUDO: in moveToVALU()
5592 case AMDGPU::S_SUB_CO_PSEUDO: { in moveToVALU()
5593 unsigned Opc = (Inst.getOpcode() == AMDGPU::S_ADD_CO_PSEUDO) in moveToVALU()
5594 ? AMDGPU::V_ADDC_U32_e64 in moveToVALU()
5595 : AMDGPU::V_SUBB_U32_e64; in moveToVALU()
5596 const auto *CarryRC = RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID); in moveToVALU()
5601 BuildMI(*MBB, &Inst, Inst.getDebugLoc(), get(AMDGPU::COPY), NewCarryReg) in moveToVALU()
5624 case AMDGPU::S_UADDO_PSEUDO: in moveToVALU()
5625 case AMDGPU::S_USUBO_PSEUDO: { in moveToVALU()
5632 unsigned Opc = (Inst.getOpcode() == AMDGPU::S_UADDO_PSEUDO) in moveToVALU()
5633 ? AMDGPU::V_ADD_CO_U32_e64 in moveToVALU()
5634 : AMDGPU::V_SUB_CO_U32_e64; in moveToVALU()
5655 case AMDGPU::S_CSELECT_B32: in moveToVALU()
5656 case AMDGPU::S_CSELECT_B64: in moveToVALU()
5662 if (NewOpcode == AMDGPU::INSTRUCTION_LIST_END) { in moveToVALU()
5680 if (Op.isReg() && Op.getReg() == AMDGPU::SCC) { in moveToVALU()
5688 if (Opcode == AMDGPU::S_SEXT_I32_I8 || Opcode == AMDGPU::S_SEXT_I32_I16) { in moveToVALU()
5691 unsigned Size = (Opcode == AMDGPU::S_SEXT_I32_I8) ? 8 : 16; in moveToVALU()
5695 } else if (Opcode == AMDGPU::S_BCNT1_I32_B32) { in moveToVALU()
5704 if (Opcode == AMDGPU::S_BFE_I32 || Opcode == AMDGPU::S_BFE_U32) { in moveToVALU()
5720 unsigned NewDstReg = AMDGPU::NoRegister; in moveToVALU()
5749 Inst.setDesc(get(AMDGPU::IMPLICIT_DEF)); in moveToVALU()
5781 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in moveScalarAddSub()
5784 assert(Opc == AMDGPU::S_ADD_I32 || Opc == AMDGPU::S_SUB_I32); in moveScalarAddSub()
5786 unsigned NewOpc = Opc == AMDGPU::S_ADD_I32 ? in moveScalarAddSub()
5787 AMDGPU::V_ADD_U32_e64 : AMDGPU::V_SUB_U32_e64; in moveScalarAddSub()
5789 assert(Inst.getOperand(3).getReg() == AMDGPU::SCC); in moveScalarAddSub()
5824 if (CandI.findRegisterDefOperandIdx(AMDGPU::SCC, false, false, &RI) != in lowerSelect()
5826 if (CandI.isCopy() && CandI.getOperand(0).getReg() == AMDGPU::SCC) { in lowerSelect()
5838 if ((SCCSource != AMDGPU::SCC) && Src0.isImm() && (Src0.getImm() == -1) && in lowerSelect()
5845 ? &AMDGPU::SReg_64_XEXECRegClass in lowerSelect()
5846 : &AMDGPU::SReg_32_XM0_XEXECRegClass; in lowerSelect()
5849 if (SCCSource == AMDGPU::SCC) { in lowerSelect()
5853 unsigned Opcode = (ST.getWavefrontSize() == 64) ? AMDGPU::S_CSELECT_B64 in lowerSelect()
5854 : AMDGPU::S_CSELECT_B32; in lowerSelect()
5859 BuildMI(MBB, MII, DL, get(AMDGPU::COPY), CopySCC).addReg(SCCSource); in lowerSelect()
5862 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in lowerSelect()
5865 BuildMI(MBB, MII, DL, get(AMDGPU::V_CNDMASK_B32_e64), ResultReg) in lowerSelect()
5886 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in lowerScalarAbs()
5887 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in lowerScalarAbs()
5890 AMDGPU::V_SUB_U32_e32 : AMDGPU::V_SUB_CO_U32_e32; in lowerScalarAbs()
5896 BuildMI(MBB, MII, DL, get(AMDGPU::V_MAX_I32_e64), ResultReg) in lowerScalarAbs()
5916 Register NewDest = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in lowerScalarXnor()
5917 legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src0, MRI, DL); in lowerScalarXnor()
5918 legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src1, MRI, DL); in lowerScalarXnor()
5920 BuildMI(MBB, MII, DL, get(AMDGPU::V_XNOR_B32_e64), NewDest) in lowerScalarXnor()
5936 Register Temp = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); in lowerScalarXnor()
5937 Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); in lowerScalarXnor()
5943 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Temp).add(Src0); in lowerScalarXnor()
5944 Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), NewDest) in lowerScalarXnor()
5948 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Temp).add(Src1); in lowerScalarXnor()
5949 Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), NewDest) in lowerScalarXnor()
5953 Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), Temp) in lowerScalarXnor()
5957 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), NewDest).addReg(Temp); in lowerScalarXnor()
5981 Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); in splitScalarNotBinop()
5982 Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); in splitScalarNotBinop()
5988 MachineInstr &Not = *BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), NewDest) in splitScalarNotBinop()
6010 Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); in splitScalarBinOpN2()
6011 Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); in splitScalarBinOpN2()
6013 MachineInstr &Not = *BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Interm) in splitScalarBinOpN2()
6042 &AMDGPU::SGPR_32RegClass; in splitScalar64BitUnaryOp()
6044 const TargetRegisterClass *Src0SubRC = RI.getSubRegClass(Src0RC, AMDGPU::sub0); in splitScalar64BitUnaryOp()
6047 AMDGPU::sub0, Src0SubRC); in splitScalar64BitUnaryOp()
6051 const TargetRegisterClass *NewDestSubRC = RI.getSubRegClass(NewDestRC, AMDGPU::sub0); in splitScalar64BitUnaryOp()
6057 AMDGPU::sub1, Src0SubRC); in splitScalar64BitUnaryOp()
6065 .addImm(AMDGPU::sub0) in splitScalar64BitUnaryOp()
6067 .addImm(AMDGPU::sub1); in splitScalar64BitUnaryOp()
6084 bool IsAdd = (Inst.getOpcode() == AMDGPU::S_ADD_U64_PSEUDO); in splitScalar64BitAddSub()
6088 const auto *CarryRC = RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID); in splitScalar64BitAddSub()
6090 Register FullDestReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); in splitScalar64BitAddSub()
6091 Register DestSub0 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in splitScalar64BitAddSub()
6092 Register DestSub1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in splitScalar64BitAddSub()
6105 const TargetRegisterClass *Src0SubRC = RI.getSubRegClass(Src0RC, AMDGPU::sub0); in splitScalar64BitAddSub()
6106 const TargetRegisterClass *Src1SubRC = RI.getSubRegClass(Src1RC, AMDGPU::sub0); in splitScalar64BitAddSub()
6109 AMDGPU::sub0, Src0SubRC); in splitScalar64BitAddSub()
6111 AMDGPU::sub0, Src1SubRC); in splitScalar64BitAddSub()
6115 AMDGPU::sub1, Src0SubRC); in splitScalar64BitAddSub()
6117 AMDGPU::sub1, Src1SubRC); in splitScalar64BitAddSub()
6119 unsigned LoOpc = IsAdd ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_SUB_CO_U32_e64; in splitScalar64BitAddSub()
6127 unsigned HiOpc = IsAdd ? AMDGPU::V_ADDC_U32_e64 : AMDGPU::V_SUBB_U32_e64; in splitScalar64BitAddSub()
6138 .addImm(AMDGPU::sub0) in splitScalar64BitAddSub()
6140 .addImm(AMDGPU::sub1); in splitScalar64BitAddSub()
6169 &AMDGPU::SGPR_32RegClass; in splitScalar64BitBinaryOp()
6171 const TargetRegisterClass *Src0SubRC = RI.getSubRegClass(Src0RC, AMDGPU::sub0); in splitScalar64BitBinaryOp()
6174 &AMDGPU::SGPR_32RegClass; in splitScalar64BitBinaryOp()
6176 const TargetRegisterClass *Src1SubRC = RI.getSubRegClass(Src1RC, AMDGPU::sub0); in splitScalar64BitBinaryOp()
6179 AMDGPU::sub0, Src0SubRC); in splitScalar64BitBinaryOp()
6181 AMDGPU::sub0, Src1SubRC); in splitScalar64BitBinaryOp()
6183 AMDGPU::sub1, Src0SubRC); in splitScalar64BitBinaryOp()
6185 AMDGPU::sub1, Src1SubRC); in splitScalar64BitBinaryOp()
6189 const TargetRegisterClass *NewDestSubRC = RI.getSubRegClass(NewDestRC, AMDGPU::sub0); in splitScalar64BitBinaryOp()
6204 .addImm(AMDGPU::sub0) in splitScalar64BitBinaryOp()
6206 .addImm(AMDGPU::sub1); in splitScalar64BitBinaryOp()
6232 Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); in splitScalar64BitXnor()
6245 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B64), Interm) in splitScalar64BitXnor()
6250 MachineInstr &Xor = *BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B64), NewDest) in splitScalar64BitXnor()
6270 const MCInstrDesc &InstDesc = get(AMDGPU::V_BCNT_U32_B32_e64); in splitScalar64BitBCNT()
6273 &AMDGPU::SGPR_32RegClass; in splitScalar64BitBCNT()
6275 Register MidReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in splitScalar64BitBCNT()
6276 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in splitScalar64BitBCNT()
6278 const TargetRegisterClass *SrcSubRC = RI.getSubRegClass(SrcRC, AMDGPU::sub0); in splitScalar64BitBCNT()
6281 AMDGPU::sub0, SrcSubRC); in splitScalar64BitBCNT()
6283 AMDGPU::sub1, SrcSubRC); in splitScalar64BitBCNT()
6311 assert(Inst.getOpcode() == AMDGPU::S_BFE_I64 && BitWidth <= 32 && in splitScalar64BitBFE()
6315 Register MidRegLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in splitScalar64BitBFE()
6316 Register MidRegHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in splitScalar64BitBFE()
6317 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); in splitScalar64BitBFE()
6319 BuildMI(MBB, MII, DL, get(AMDGPU::V_BFE_I32), MidRegLo) in splitScalar64BitBFE()
6320 .addReg(Inst.getOperand(1).getReg(), 0, AMDGPU::sub0) in splitScalar64BitBFE()
6324 BuildMI(MBB, MII, DL, get(AMDGPU::V_ASHRREV_I32_e32), MidRegHi) in splitScalar64BitBFE()
6330 .addImm(AMDGPU::sub0) in splitScalar64BitBFE()
6332 .addImm(AMDGPU::sub1); in splitScalar64BitBFE()
6340 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in splitScalar64BitBFE()
6341 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); in splitScalar64BitBFE()
6343 BuildMI(MBB, MII, DL, get(AMDGPU::V_ASHRREV_I32_e64), TmpReg) in splitScalar64BitBFE()
6345 .addReg(Src.getReg(), 0, AMDGPU::sub0); in splitScalar64BitBFE()
6348 .addReg(Src.getReg(), 0, AMDGPU::sub0) in splitScalar64BitBFE()
6349 .addImm(AMDGPU::sub0) in splitScalar64BitBFE()
6351 .addImm(AMDGPU::sub1); in splitScalar64BitBFE()
6368 case AMDGPU::COPY: in addUsersToMoveToVALUWorklist()
6369 case AMDGPU::WQM: in addUsersToMoveToVALUWorklist()
6370 case AMDGPU::SOFT_WQM: in addUsersToMoveToVALUWorklist()
6371 case AMDGPU::WWM: in addUsersToMoveToVALUWorklist()
6372 case AMDGPU::REG_SEQUENCE: in addUsersToMoveToVALUWorklist()
6373 case AMDGPU::PHI: in addUsersToMoveToVALUWorklist()
6374 case AMDGPU::INSERT_SUBREG: in addUsersToMoveToVALUWorklist()
6396 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in movePackToVALU()
6403 case AMDGPU::S_PACK_LL_B32_B16: { in movePackToVALU()
6404 Register ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in movePackToVALU()
6405 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in movePackToVALU()
6409 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg) in movePackToVALU()
6412 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_AND_B32_e64), TmpReg) in movePackToVALU()
6416 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHL_OR_B32), ResultReg) in movePackToVALU()
6422 case AMDGPU::S_PACK_LH_B32_B16: { in movePackToVALU()
6423 Register ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in movePackToVALU()
6424 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg) in movePackToVALU()
6426 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_BFI_B32), ResultReg) in movePackToVALU()
6432 case AMDGPU::S_PACK_HH_B32_B16: { in movePackToVALU()
6433 Register ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in movePackToVALU()
6434 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in movePackToVALU()
6435 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHRREV_B32_e64), TmpReg) in movePackToVALU()
6438 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg) in movePackToVALU()
6440 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_AND_OR_B32), ResultReg) in movePackToVALU()
6461 assert(Op.isReg() && Op.getReg() == AMDGPU::SCC && Op.isDef() && in addSCCDefUsersToVALUWorklist()
6470 if (MI.findRegisterUseOperandIdx(AMDGPU::SCC, false, &RI) != -1) { in addSCCDefUsersToVALUWorklist()
6476 if ((User.getOpcode() == AMDGPU::S_ADD_CO_PSEUDO) || in addSCCDefUsersToVALUWorklist()
6477 (User.getOpcode() == AMDGPU::S_SUB_CO_PSEUDO)) { in addSCCDefUsersToVALUWorklist()
6480 } else if (User.getOpcode() == AMDGPU::V_CNDMASK_B32_e64) { in addSCCDefUsersToVALUWorklist()
6487 if (MI.getOpcode() == AMDGPU::S_CSELECT_B32 || in addSCCDefUsersToVALUWorklist()
6488 MI.getOpcode() == AMDGPU::S_CSELECT_B64) { in addSCCDefUsersToVALUWorklist()
6502 if (MI.findRegisterDefOperandIdx(AMDGPU::SCC, false, false, &RI) != -1) in addSCCDefUsersToVALUWorklist()
6510 SCCDefInst.getDebugLoc(), get(AMDGPU::COPY), AMDGPU::SCC) in addSCCDefUsersToVALUWorklist()
6523 case AMDGPU::COPY: in getDestEquivalentVGPRClass()
6524 case AMDGPU::PHI: in getDestEquivalentVGPRClass()
6525 case AMDGPU::REG_SEQUENCE: in getDestEquivalentVGPRClass()
6526 case AMDGPU::INSERT_SUBREG: in getDestEquivalentVGPRClass()
6527 case AMDGPU::WQM: in getDestEquivalentVGPRClass()
6528 case AMDGPU::SOFT_WQM: in getDestEquivalentVGPRClass()
6529 case AMDGPU::WWM: { in getDestEquivalentVGPRClass()
6536 case AMDGPU::PHI: in getDestEquivalentVGPRClass()
6537 case AMDGPU::REG_SEQUENCE: in getDestEquivalentVGPRClass()
6538 case AMDGPU::INSERT_SUBREG: in getDestEquivalentVGPRClass()
6548 if (RI.hasVGPRs(NewDstRC) || NewDstRC == &AMDGPU::VReg_1RegClass) in getDestEquivalentVGPRClass()
6578 if (SGPRReg != AMDGPU::NoRegister) in findUsedSGPR()
6581 Register UsedSGPRs[3] = { AMDGPU::NoRegister }; in findUsedSGPR()
6620 if (UsedSGPRs[0] != AMDGPU::NoRegister) { in findUsedSGPR()
6625 if (SGPRReg == AMDGPU::NoRegister && UsedSGPRs[1] != AMDGPU::NoRegister) { in findUsedSGPR()
6635 int Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), OperandName); in getNamedOperand()
6649 uint64_t RsrcDataFormat = AMDGPU::RSRC_DATA_FORMAT; in getDefaultRsrcDataFormat()
6666 AMDGPU::RSRC_TID_ENABLE | in getScratchRsrcWords23()
6672 Rsrc23 |= EltSizeValue << AMDGPU::RSRC_ELEMENT_SIZE_SHIFT; in getScratchRsrcWords23()
6677 Rsrc23 |= IndexStride << AMDGPU::RSRC_INDEX_STRIDE_SHIFT; in getScratchRsrcWords23()
6683 Rsrc23 &= ~AMDGPU::RSRC_DATA_FORMAT; in getScratchRsrcWords23()
6701 const MachineOperand *Addr = getNamedOperand(MI, AMDGPU::OpName::vaddr); in isStackAccess()
6703 return AMDGPU::NoRegister; in isStackAccess()
6709 return getNamedOperand(MI, AMDGPU::OpName::vdata)->getReg(); in isStackAccess()
6714 const MachineOperand *Addr = getNamedOperand(MI, AMDGPU::OpName::addr); in isSGPRStackAccess()
6717 return getNamedOperand(MI, AMDGPU::OpName::data)->getReg(); in isSGPRStackAccess()
6723 return AMDGPU::NoRegister; in isLoadFromStackSlot()
6731 return AMDGPU::NoRegister; in isLoadFromStackSlot()
6737 return AMDGPU::NoRegister; in isStoreToStackSlot()
6745 return AMDGPU::NoRegister; in isStoreToStackSlot()
6781 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0); in getInstSizeInBytes()
6788 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1); in getInstSizeInBytes()
6795 int Src2Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2); in getInstSizeInBytes()
6807 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr0); in getInstSizeInBytes()
6811 int RSrcIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::srsrc); in getInstSizeInBytes()
6849 return Branch.getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO; in isNonUniformBranchInstr()
6861 if (Branch->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) { in convertNonUniformIfRegion()
6864 BuildMI(*MF, Branch->getDebugLoc(), get(AMDGPU::SI_IF), DstReg) in convertNonUniformIfRegion()
6868 BuildMI(*MF, Branch->getDebugLoc(), get(AMDGPU::SI_END_CF)) in convertNonUniformIfRegion()
6887 if (Branch->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) { in convertNonUniformLoopRegion()
6909 get(AMDGPU::SI_IF_BREAK), BackEdgeReg) in convertNonUniformLoopRegion()
6913 BuildMI(*(MF), Branch->getDebugLoc(), get(AMDGPU::SI_LOOP)) in convertNonUniformLoopRegion()
6927 {AMDGPU::TI_CONSTDATA_START, "amdgpu-constdata-start"}, in getSerializableTargetIndices()
6928 {AMDGPU::TI_SCRATCH_RSRC_DWORD0, "amdgpu-scratch-rsrc-dword0"}, in getSerializableTargetIndices()
6929 {AMDGPU::TI_SCRATCH_RSRC_DWORD1, "amdgpu-scratch-rsrc-dword1"}, in getSerializableTargetIndices()
6930 {AMDGPU::TI_SCRATCH_RSRC_DWORD2, "amdgpu-scratch-rsrc-dword2"}, in getSerializableTargetIndices()
6931 {AMDGPU::TI_SCRATCH_RSRC_DWORD3, "amdgpu-scratch-rsrc-dword3"}}; in getSerializableTargetIndices()
6971 return !MI.isTerminator() && MI.getOpcode() != AMDGPU::COPY && in isBasicBlockPrologue()
6972 MI.modifiesRegister(AMDGPU::EXEC, &RI); in isBasicBlockPrologue()
6981 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_U32_e64), DestReg); in getAddNoCarry()
6987 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_CO_U32_e64), DestReg) in getAddNoCarry()
6997 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_U32_e32), DestReg); in getAddNoCarry()
7000 Register UnusedCarry = !RS.isRegUsed(AMDGPU::VCC) in getAddNoCarry()
7008 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_CO_U32_e64), DestReg) in getAddNoCarry()
7014 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR: in isKillTerminator()
7015 case AMDGPU::SI_KILL_I1_TERMINATOR: in isKillTerminator()
7024 case AMDGPU::SI_KILL_F32_COND_IMM_PSEUDO: in getKillTerminatorFromPseudo()
7025 return get(AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR); in getKillTerminatorFromPseudo()
7026 case AMDGPU::SI_KILL_I1_PSEUDO: in getKillTerminatorFromPseudo()
7027 return get(AMDGPU::SI_KILL_I1_TERMINATOR); in getKillTerminatorFromPseudo()
7038 if (Op.isReg() && Op.getReg() == AMDGPU::VCC) in fixImplicitOperands()
7039 Op.setReg(AMDGPU::VCC_LO); in fixImplicitOperands()
7048 int Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::sbase); in isBufferSMRD()
7053 return RI.getRegClass(RCID)->hasSubClassEq(&AMDGPU::SGPR_128RegClass); in isBufferSMRD()
7133 case AMDGPU::V_MOVRELS_B32_dpp_gfx10: in isAsmOnlyOpcode()
7134 case AMDGPU::V_MOVRELS_B32_sdwa_gfx10: in isAsmOnlyOpcode()
7135 case AMDGPU::V_MOVRELD_B32_dpp_gfx10: in isAsmOnlyOpcode()
7136 case AMDGPU::V_MOVRELD_B32_sdwa_gfx10: in isAsmOnlyOpcode()
7137 case AMDGPU::V_MOVRELSD_B32_dpp_gfx10: in isAsmOnlyOpcode()
7138 case AMDGPU::V_MOVRELSD_B32_sdwa_gfx10: in isAsmOnlyOpcode()
7139 case AMDGPU::V_MOVRELSD_2_B32_dpp_gfx10: in isAsmOnlyOpcode()
7140 case AMDGPU::V_MOVRELSD_2_B32_sdwa_gfx10: in isAsmOnlyOpcode()
7174 int MCOp = AMDGPU::getMCOpcode(Opcode, Gen); in pseudoToMCOpcode()
7217 case AMDGPU::REG_SEQUENCE: in followSubRegDef()
7221 case AMDGPU::INSERT_SUBREG: in followSubRegDef()
7247 case AMDGPU::COPY: in getVRegSubRegDef()
7248 case AMDGPU::V_MOV_B32_e32: { in getVRegSubRegDef()
7297 if (I->modifiesRegister(AMDGPU::EXEC, TRI)) in execMayBeModifiedBeforeUse()
7354 } else if (TRI->regsOverlap(Reg, AMDGPU::EXEC)) in execMayBeModifiedBeforeAnyUse()
7379 (InsPt->getOpcode() == AMDGPU::SI_IF || in createPHISourceCopy()
7380 InsPt->getOpcode() == AMDGPU::SI_ELSE || in createPHISourceCopy()
7381 InsPt->getOpcode() == AMDGPU::SI_IF_BREAK) && in createPHISourceCopy()
7385 get(ST.isWave32() ? AMDGPU::S_MOV_B32_term in createPHISourceCopy()
7386 : AMDGPU::S_MOV_B64_term), in createPHISourceCopy()
7389 .addReg(AMDGPU::EXEC, RegState::Implicit); in createPHISourceCopy()
7421 if (RC->hasSuperClassEq(&AMDGPU::SReg_32RegClass)) { in foldMemoryOperandImpl()
7422 MRI.constrainRegClass(VirtReg, &AMDGPU::SReg_32_XM0_XEXECRegClass); in foldMemoryOperandImpl()
7424 } else if (RC->hasSuperClassEq(&AMDGPU::SReg_64RegClass)) { in foldMemoryOperandImpl()
7425 MRI.constrainRegClass(VirtReg, &AMDGPU::SReg_64_XEXECRegClass); in foldMemoryOperandImpl()