Lines Matching refs:instr
190 get_wait_states(aco_ptr<Instruction>& instr) in get_wait_states() argument
192 if (instr->opcode == aco_opcode::s_nop) in get_wait_states()
193 return instr->sopp().imm + 1; in get_wait_states()
194 else if (instr->opcode == aco_opcode::p_constaddr) in get_wait_states()
243 aco_ptr<Instruction>& instr = state.old_instructions[pred_idx]; in handle_raw_hazard_internal() local
244 if (!instr) in handle_raw_hazard_internal()
246 if (handle_raw_hazard_instr<Valu, Vintrp, Salu>(instr, reg, &nops_needed, &mask)) in handle_raw_hazard_internal()
324 handle_smem_clause_hazards(Program* program, NOP_ctx_gfx6& ctx, aco_ptr<Instruction>& instr, in handle_smem_clause_hazards() argument
331 if (ctx.smem_write || instr->definitions.empty() || in handle_smem_clause_hazards()
332 instr_info.is_atomic[(unsigned)instr->opcode]) { in handle_smem_clause_hazards()
335 for (Operand op : instr->operands) { in handle_smem_clause_hazards()
343 Definition def = instr->definitions[0]; in handle_smem_clause_hazards()
352 handle_instruction_gfx6(State& state, NOP_ctx_gfx6& ctx, aco_ptr<Instruction>& instr, in handle_instruction_gfx6() argument
358 if (instr->isSMEM()) { in handle_instruction_gfx6()
364 for (unsigned i = 0; i < instr->operands.size(); i++) { in handle_instruction_gfx6()
365 Operand op = instr->operands[i]; in handle_instruction_gfx6()
377 handle_smem_clause_hazards(state.program, ctx, instr, &NOPs); in handle_instruction_gfx6()
378 } else if (instr->isSALU()) { in handle_instruction_gfx6()
379 if (instr->opcode == aco_opcode::s_setreg_b32 || in handle_instruction_gfx6()
380 instr->opcode == aco_opcode::s_setreg_imm32_b32 || in handle_instruction_gfx6()
381 instr->opcode == aco_opcode::s_getreg_b32) { in handle_instruction_gfx6()
386 if (instr->opcode == aco_opcode::s_movrels_b32 || in handle_instruction_gfx6()
387 instr->opcode == aco_opcode::s_movrels_b64 || in handle_instruction_gfx6()
388 instr->opcode == aco_opcode::s_movreld_b32 || in handle_instruction_gfx6()
389 instr->opcode == aco_opcode::s_movreld_b64) { in handle_instruction_gfx6()
394 if (instr->opcode == aco_opcode::s_sendmsg || instr->opcode == aco_opcode::s_ttracedata) in handle_instruction_gfx6()
396 } else if (instr->isDS() && instr->ds().gds) { in handle_instruction_gfx6()
398 } else if (instr->isVALU() || instr->isVINTRP()) { in handle_instruction_gfx6()
399 for (Operand op : instr->operands) { in handle_instruction_gfx6()
406 if (instr->isDPP()) { in handle_instruction_gfx6()
408 handle_valu_then_read_hazard(state, &NOPs, 2, instr->operands[0]); in handle_instruction_gfx6()
411 for (Definition def : instr->definitions) { in handle_instruction_gfx6()
418 if ((instr->opcode == aco_opcode::v_readlane_b32 || in handle_instruction_gfx6()
419 instr->opcode == aco_opcode::v_readlane_b32_e64 || in handle_instruction_gfx6()
420 instr->opcode == aco_opcode::v_writelane_b32 || in handle_instruction_gfx6()
421 instr->opcode == aco_opcode::v_writelane_b32_e64) && in handle_instruction_gfx6()
422 !instr->operands[1].isConstant()) { in handle_instruction_gfx6()
423 handle_valu_then_read_hazard(state, &NOPs, 4, instr->operands[1]); in handle_instruction_gfx6()
432 (instr->opcode == aco_opcode::v_readlane_b32 || /* GFX6 doesn't have v_readlane_b32_e64 */ in handle_instruction_gfx6()
433 instr->opcode == aco_opcode::v_readfirstlane_b32)) { in handle_instruction_gfx6()
434 handle_vintrp_then_read_hazard(state, &NOPs, 1, instr->operands[0]); in handle_instruction_gfx6()
437 if (instr->opcode == aco_opcode::v_div_fmas_f32 || in handle_instruction_gfx6()
438 instr->opcode == aco_opcode::v_div_fmas_f64) in handle_instruction_gfx6()
440 } else if (instr->isVMEM() || instr->isFlatLike()) { in handle_instruction_gfx6()
442 for (Operand op : instr->operands) { in handle_instruction_gfx6()
448 if (!instr->isSALU() && instr->format != Format::SMEM) in handle_instruction_gfx6()
452 bool lds_scratch_global = (instr->isScratch() || instr->isGlobal()) && instr->flatlike().lds; in handle_instruction_gfx6()
453 if (instr->isVINTRP() || lds_scratch_global || in handle_instruction_gfx6()
454 instr->opcode == aco_opcode::ds_read_addtid_b32 || in handle_instruction_gfx6()
455 instr->opcode == aco_opcode::ds_write_addtid_b32 || in handle_instruction_gfx6()
456 instr->opcode == aco_opcode::buffer_store_lds_dword) { in handle_instruction_gfx6()
461 ctx.add_wait_states(NOPs + get_wait_states(instr)); in handle_instruction_gfx6()
474 if ((ctx.smem_clause || ctx.smem_write) && (NOPs || instr->format != Format::SMEM)) { in handle_instruction_gfx6()
484 if (instr->isSMEM()) { in handle_instruction_gfx6()
485 if (instr->definitions.empty() || instr_info.is_atomic[(unsigned)instr->opcode]) { in handle_instruction_gfx6()
491 for (Operand op : instr->operands) { in handle_instruction_gfx6()
497 Definition def = instr->definitions[0]; in handle_instruction_gfx6()
502 } else if (instr->isVALU()) { in handle_instruction_gfx6()
503 for (Definition def : instr->definitions) { in handle_instruction_gfx6()
515 } else if (instr->isSALU() && !instr->definitions.empty()) { in handle_instruction_gfx6()
516 if (!instr->definitions.empty()) { in handle_instruction_gfx6()
518 Definition def = instr->definitions[0]; in handle_instruction_gfx6()
524 } else if (instr->opcode == aco_opcode::s_setreg_b32 || in handle_instruction_gfx6()
525 instr->opcode == aco_opcode::s_setreg_imm32_b32) { in handle_instruction_gfx6()
526 SOPK_instruction& sopk = instr->sopk(); in handle_instruction_gfx6()
535 } else if (instr->isVMEM() || instr->isFlatLike()) { in handle_instruction_gfx6()
537 bool consider_buf = (instr->isMUBUF() || instr->isMTBUF()) && instr->operands.size() == 4 && in handle_instruction_gfx6()
538 instr->operands[3].size() > 2 && instr->operands[2].physReg() >= 128; in handle_instruction_gfx6()
541 bool consider_mimg = instr->isMIMG() && in handle_instruction_gfx6()
542 instr->operands[1].regClass().type() == RegType::vgpr && in handle_instruction_gfx6()
543 instr->operands[1].size() > 2 && instr->operands[0].size() == 4; in handle_instruction_gfx6()
546 instr->isFlatLike() && instr->operands.size() == 3 && instr->operands[2].size() > 2; in handle_instruction_gfx6()
548 PhysReg wrdata = instr->operands[consider_flat ? 2 : 3].physReg(); in handle_instruction_gfx6()
549 unsigned size = instr->operands[consider_flat ? 2 : 3].size(); in handle_instruction_gfx6()
558 check_written_regs(const aco_ptr<Instruction>& instr, const std::bitset<N>& check_regs) in check_written_regs() argument
560 return std::any_of(instr->definitions.begin(), instr->definitions.end(), in check_written_regs()
574 mark_read_regs(const aco_ptr<Instruction>& instr, std::bitset<N>& reg_reads) in mark_read_regs() argument
576 for (const Operand& op : instr->operands) { in mark_read_regs()
586 VALU_writes_sgpr(aco_ptr<Instruction>& instr) in VALU_writes_sgpr() argument
588 if (instr->isVOPC()) in VALU_writes_sgpr()
590 if (instr->isVOP3() && instr->definitions.size() == 2) in VALU_writes_sgpr()
592 if (instr->opcode == aco_opcode::v_readfirstlane_b32 || in VALU_writes_sgpr()
593 instr->opcode == aco_opcode::v_readlane_b32 || in VALU_writes_sgpr()
594 instr->opcode == aco_opcode::v_readlane_b32_e64) in VALU_writes_sgpr()
600 instr_writes_exec(const aco_ptr<Instruction>& instr) in instr_writes_exec() argument
602 return std::any_of(instr->definitions.begin(), instr->definitions.end(), in instr_writes_exec()
608 instr_writes_sgpr(const aco_ptr<Instruction>& instr) in instr_writes_sgpr() argument
610 return std::any_of(instr->definitions.begin(), instr->definitions.end(), in instr_writes_sgpr()
616 instr_is_branch(const aco_ptr<Instruction>& instr) in instr_is_branch() argument
618 return instr->opcode == aco_opcode::s_branch || instr->opcode == aco_opcode::s_cbranch_scc0 || in instr_is_branch()
619 instr->opcode == aco_opcode::s_cbranch_scc1 || in instr_is_branch()
620 instr->opcode == aco_opcode::s_cbranch_vccz || in instr_is_branch()
621 instr->opcode == aco_opcode::s_cbranch_vccnz || in instr_is_branch()
622 instr->opcode == aco_opcode::s_cbranch_execz || in instr_is_branch()
623 instr->opcode == aco_opcode::s_cbranch_execnz || in instr_is_branch()
624 instr->opcode == aco_opcode::s_cbranch_cdbgsys || in instr_is_branch()
625 instr->opcode == aco_opcode::s_cbranch_cdbguser || in instr_is_branch()
626 instr->opcode == aco_opcode::s_cbranch_cdbgsys_or_user || in instr_is_branch()
627 instr->opcode == aco_opcode::s_cbranch_cdbgsys_and_user || in instr_is_branch()
628 instr->opcode == aco_opcode::s_subvector_loop_begin || in instr_is_branch()
629 instr->opcode == aco_opcode::s_subvector_loop_end || in instr_is_branch()
630 instr->opcode == aco_opcode::s_setpc_b64 || instr->opcode == aco_opcode::s_swappc_b64 || in instr_is_branch()
631 instr->opcode == aco_opcode::s_getpc_b64 || instr->opcode == aco_opcode::s_call_b64; in instr_is_branch()
635 handle_instruction_gfx10(State& state, NOP_ctx_gfx10& ctx, aco_ptr<Instruction>& instr, in handle_instruction_gfx10() argument
644 if (instr->isVMEM() || instr->isFlatLike() || instr->isDS()) { in handle_instruction_gfx10()
646 mark_read_regs(instr, ctx.sgprs_read_by_VMEM); in handle_instruction_gfx10()
650 } else if (instr->isSALU() || instr->isSMEM()) { in handle_instruction_gfx10()
651 if (instr->opcode == aco_opcode::s_waitcnt) { in handle_instruction_gfx10()
653 uint16_t imm = instr->sopp().imm; in handle_instruction_gfx10()
657 } else if (instr->opcode == aco_opcode::s_waitcnt_depctr) { in handle_instruction_gfx10()
659 if (instr->sopp().imm == 0xffe3) in handle_instruction_gfx10()
664 if (check_written_regs(instr, ctx.sgprs_read_by_VMEM)) { in handle_instruction_gfx10()
674 } else if (instr->isVALU()) { in handle_instruction_gfx10()
682 if (instr->isVOPC()) { in handle_instruction_gfx10()
684 } else if (ctx.has_VOPC && (instr->opcode == aco_opcode::v_permlane16_b32 || in handle_instruction_gfx10()
685 instr->opcode == aco_opcode::v_permlanex16_b32)) { in handle_instruction_gfx10()
691 v_mov->definitions[0] = Definition(instr->operands[0].physReg(), v1); in handle_instruction_gfx10()
692 v_mov->operands[0] = Operand(instr->operands[0].physReg(), v1); in handle_instruction_gfx10()
694 } else if (instr->isVALU() && instr->opcode != aco_opcode::v_nop) { in handle_instruction_gfx10()
701 if (!instr->isVALU() && instr->reads_exec()) { in handle_instruction_gfx10()
703 } else if (instr->isVALU()) { in handle_instruction_gfx10()
704 if (instr_writes_exec(instr)) { in handle_instruction_gfx10()
713 } else if (instr_writes_sgpr(instr)) { in handle_instruction_gfx10()
717 } else if (instr->opcode == aco_opcode::s_waitcnt_depctr) { in handle_instruction_gfx10()
719 if ((instr->sopp().imm & 0xfffe) == 0xfffe) in handle_instruction_gfx10()
726 if (instr->isSMEM()) { in handle_instruction_gfx10()
728 mark_read_regs(instr, ctx.sgprs_read_by_SMEM); in handle_instruction_gfx10()
729 } else if (VALU_writes_sgpr(instr)) { in handle_instruction_gfx10()
731 if (check_written_regs(instr, ctx.sgprs_read_by_SMEM)) { in handle_instruction_gfx10()
741 } else if (instr->isSALU()) { in handle_instruction_gfx10()
742 if (instr->format != Format::SOPP) { in handle_instruction_gfx10()
747 const SOPP_instruction& sopp = instr->sopp(); in handle_instruction_gfx10()
762 if (instr->isVMEM() || instr->isGlobal() || instr->isScratch()) { in handle_instruction_gfx10()
767 } else if (instr->isDS()) { in handle_instruction_gfx10()
772 } else if (instr_is_branch(instr)) { in handle_instruction_gfx10()
775 } else if (instr->opcode == aco_opcode::s_waitcnt_vscnt) { in handle_instruction_gfx10()
777 const SOPK_instruction& sopk = instr->sopk(); in handle_instruction_gfx10()
796 if (instr->isMIMG() && get_mimg_nsa_dwords(instr.get()) > 1) { in handle_instruction_gfx10()
801 if (instr->isMUBUF() || instr->isMTBUF()) { in handle_instruction_gfx10()
802 uint32_t offset = instr->isMUBUF() ? instr->mubuf().offset : instr->mtbuf().offset; in handle_instruction_gfx10()
811 if (instr->opcode == aco_opcode::v_writelane_b32_e64) { in handle_instruction_gfx10()
815 if (instr->isMIMG() && get_mimg_nsa_dwords(instr.get()) > 0) in handle_instruction_gfx10()
839 for (aco_ptr<Instruction>& instr : state.old_instructions) { in handle_block()
840 Handle(state, ctx, instr, block.instructions); in handle_block()
841 block.instructions.emplace_back(std::move(instr)); in handle_block()