Lines Matching refs:inst
65 schedule_node(backend_instruction *inst, instruction_scheduler *sched);
69 backend_instruction *inst; member in schedule_node
123 switch (inst->opcode) { in set_latency_gen4()
158 switch (inst->opcode) { in set_latency_gen7()
499 virtual int issue_time(backend_instruction *inst) = 0;
501 virtual void count_reads_remaining(backend_instruction *inst) = 0;
503 virtual void update_register_pressure(backend_instruction *inst) = 0;
504 virtual int get_register_pressure_benefit(backend_instruction *inst) = 0;
571 bool is_compressed(fs_inst *inst);
573 int issue_time(backend_instruction *inst);
576 void count_reads_remaining(backend_instruction *inst);
578 void update_register_pressure(backend_instruction *inst);
579 int get_register_pressure_benefit(backend_instruction *inst);
592 is_src_duplicate(fs_inst *inst, int src) in is_src_duplicate() argument
595 if (inst->src[i].equals(inst->src[src])) in is_src_duplicate()
604 fs_inst *inst = (fs_inst *)be; in count_reads_remaining() local
609 for (int i = 0; i < inst->sources; i++) { in count_reads_remaining()
610 if (is_src_duplicate(inst, i)) in count_reads_remaining()
613 if (inst->src[i].file == VGRF) { in count_reads_remaining()
614 reads_remaining[inst->src[i].nr]++; in count_reads_remaining()
615 } else if (inst->src[i].file == FIXED_GRF) { in count_reads_remaining()
616 if (inst->src[i].nr >= hw_reg_count) in count_reads_remaining()
619 for (unsigned j = 0; j < regs_read(inst, i); j++) in count_reads_remaining()
620 hw_reads_remaining[inst->src[i].nr + j]++; in count_reads_remaining()
684 fs_inst *inst = (fs_inst *)be; in update_register_pressure() local
689 if (inst->dst.file == VGRF) { in update_register_pressure()
690 written[inst->dst.nr] = true; in update_register_pressure()
693 for (int i = 0; i < inst->sources; i++) { in update_register_pressure()
694 if (is_src_duplicate(inst, i)) in update_register_pressure()
697 if (inst->src[i].file == VGRF) { in update_register_pressure()
698 reads_remaining[inst->src[i].nr]--; in update_register_pressure()
699 } else if (inst->src[i].file == FIXED_GRF && in update_register_pressure()
700 inst->src[i].nr < hw_reg_count) { in update_register_pressure()
701 for (unsigned off = 0; off < regs_read(inst, i); off++) in update_register_pressure()
702 hw_reads_remaining[inst->src[i].nr + off]--; in update_register_pressure()
710 fs_inst *inst = (fs_inst *)be; in get_register_pressure_benefit() local
713 if (inst->dst.file == VGRF) { in get_register_pressure_benefit()
714 if (!BITSET_TEST(livein[block_idx], inst->dst.nr) && in get_register_pressure_benefit()
715 !written[inst->dst.nr]) in get_register_pressure_benefit()
716 benefit -= v->alloc.sizes[inst->dst.nr]; in get_register_pressure_benefit()
719 for (int i = 0; i < inst->sources; i++) { in get_register_pressure_benefit()
720 if (is_src_duplicate(inst, i)) in get_register_pressure_benefit()
723 if (inst->src[i].file == VGRF && in get_register_pressure_benefit()
724 !BITSET_TEST(liveout[block_idx], inst->src[i].nr) && in get_register_pressure_benefit()
725 reads_remaining[inst->src[i].nr] == 1) in get_register_pressure_benefit()
726 benefit += v->alloc.sizes[inst->src[i].nr]; in get_register_pressure_benefit()
728 if (inst->src[i].file == FIXED_GRF && in get_register_pressure_benefit()
729 inst->src[i].nr < hw_reg_count) { in get_register_pressure_benefit()
730 for (unsigned off = 0; off < regs_read(inst, i); off++) { in get_register_pressure_benefit()
731 int reg = inst->src[i].nr + off; in get_register_pressure_benefit()
749 int issue_time(backend_instruction *inst);
752 void count_reads_remaining(backend_instruction *inst);
754 void update_register_pressure(backend_instruction *inst);
755 int get_register_pressure_benefit(backend_instruction *inst);
786 schedule_node::schedule_node(backend_instruction *inst, in schedule_node() argument
791 this->inst = inst; in schedule_node()
816 foreach_inst_in_block(backend_instruction, inst, block) { in add_insts_from_block()
817 schedule_node *n = new(mem_ctx) schedule_node(inst, this); in add_insts_from_block()
831 n->delay = issue_time(n->inst); in compute_delays()
852 n->unblocked_time + issue_time(n->inst) + n->child_latency[i]); in compute_exits()
862 n->exit = (n->inst->opcode == FS_OPCODE_DISCARD_JUMP ? n : NULL); in compute_exits()
922 is_scheduling_barrier(const backend_instruction *inst) in is_scheduling_barrier() argument
924 return inst->opcode == FS_OPCODE_PLACEHOLDER_HALT || in is_scheduling_barrier()
925 inst->is_control_flow() || in is_scheduling_barrier()
926 inst->has_side_effects(); in is_scheduling_barrier()
943 if (is_scheduling_barrier(prev->inst)) in add_barrier_deps()
952 if (is_scheduling_barrier(next->inst)) in add_barrier_deps()
963 fs_instruction_scheduler::is_compressed(fs_inst *inst) in is_compressed() argument
965 return inst->exec_size == 16; in is_compressed()
991 fs_inst *inst = (fs_inst *)n->inst; in calculate_deps() local
993 if (is_scheduling_barrier(inst)) in calculate_deps()
997 for (int i = 0; i < inst->sources; i++) { in calculate_deps()
998 if (inst->src[i].file == VGRF) { in calculate_deps()
1000 for (unsigned r = 0; r < regs_read(inst, i); r++) in calculate_deps()
1001 add_dep(last_grf_write[inst->src[i].nr + r], n); in calculate_deps()
1003 for (unsigned r = 0; r < regs_read(inst, i); r++) { in calculate_deps()
1004 add_dep(last_grf_write[inst->src[i].nr * 16 + in calculate_deps()
1005 inst->src[i].offset / REG_SIZE + r], n); in calculate_deps()
1008 } else if (inst->src[i].file == FIXED_GRF) { in calculate_deps()
1010 for (unsigned r = 0; r < regs_read(inst, i); r++) in calculate_deps()
1011 add_dep(last_grf_write[inst->src[i].nr + r], n); in calculate_deps()
1015 } else if (inst->src[i].is_accumulator()) { in calculate_deps()
1017 } else if (inst->src[i].file == ARF) { in calculate_deps()
1022 if (inst->base_mrf != -1) { in calculate_deps()
1023 for (int i = 0; i < inst->mlen; i++) { in calculate_deps()
1028 add_dep(last_mrf_write[inst->base_mrf + i], n); in calculate_deps()
1032 if (const unsigned mask = inst->flags_read(v->devinfo)) { in calculate_deps()
1041 if (inst->reads_accumulator_implicitly()) { in calculate_deps()
1046 if (inst->dst.file == VGRF) { in calculate_deps()
1048 for (unsigned r = 0; r < regs_written(inst); r++) { in calculate_deps()
1049 add_dep(last_grf_write[inst->dst.nr + r], n); in calculate_deps()
1050 last_grf_write[inst->dst.nr + r] = n; in calculate_deps()
1053 for (unsigned r = 0; r < regs_written(inst); r++) { in calculate_deps()
1054 add_dep(last_grf_write[inst->dst.nr * 16 + in calculate_deps()
1055 inst->dst.offset / REG_SIZE + r], n); in calculate_deps()
1056 last_grf_write[inst->dst.nr * 16 + in calculate_deps()
1057 inst->dst.offset / REG_SIZE + r] = n; in calculate_deps()
1060 } else if (inst->dst.file == MRF) { in calculate_deps()
1061 int reg = inst->dst.nr & ~BRW_MRF_COMPR4; in calculate_deps()
1065 if (is_compressed(inst)) { in calculate_deps()
1066 if (inst->dst.nr & BRW_MRF_COMPR4) in calculate_deps()
1073 } else if (inst->dst.file == FIXED_GRF) { in calculate_deps()
1075 for (unsigned r = 0; r < regs_written(inst); r++) in calculate_deps()
1076 last_grf_write[inst->dst.nr + r] = n; in calculate_deps()
1080 } else if (inst->dst.is_accumulator()) { in calculate_deps()
1083 } else if (inst->dst.file == ARF && !inst->dst.is_null()) { in calculate_deps()
1087 if (inst->mlen > 0 && inst->base_mrf != -1) { in calculate_deps()
1088 for (int i = 0; i < v->implied_mrf_writes(inst); i++) { in calculate_deps()
1089 add_dep(last_mrf_write[inst->base_mrf + i], n); in calculate_deps()
1090 last_mrf_write[inst->base_mrf + i] = n; in calculate_deps()
1094 if (const unsigned mask = inst->flags_written()) { in calculate_deps()
1105 if (inst->writes_accumulator_implicitly(v->devinfo) && in calculate_deps()
1106 !inst->dst.is_accumulator()) { in calculate_deps()
1120 fs_inst *inst = (fs_inst *)n->inst; in calculate_deps() local
1123 for (int i = 0; i < inst->sources; i++) { in calculate_deps()
1124 if (inst->src[i].file == VGRF) { in calculate_deps()
1126 for (unsigned r = 0; r < regs_read(inst, i); r++) in calculate_deps()
1127 add_dep(n, last_grf_write[inst->src[i].nr + r], 0); in calculate_deps()
1129 for (unsigned r = 0; r < regs_read(inst, i); r++) { in calculate_deps()
1130 add_dep(n, last_grf_write[inst->src[i].nr * 16 + in calculate_deps()
1131 inst->src[i].offset / REG_SIZE + r], 0); in calculate_deps()
1134 } else if (inst->src[i].file == FIXED_GRF) { in calculate_deps()
1136 for (unsigned r = 0; r < regs_read(inst, i); r++) in calculate_deps()
1137 add_dep(n, last_grf_write[inst->src[i].nr + r], 0); in calculate_deps()
1141 } else if (inst->src[i].is_accumulator()) { in calculate_deps()
1143 } else if (inst->src[i].file == ARF) { in calculate_deps()
1148 if (inst->base_mrf != -1) { in calculate_deps()
1149 for (int i = 0; i < inst->mlen; i++) { in calculate_deps()
1154 add_dep(n, last_mrf_write[inst->base_mrf + i], 2); in calculate_deps()
1158 if (const unsigned mask = inst->flags_read(v->devinfo)) { in calculate_deps()
1167 if (inst->reads_accumulator_implicitly()) { in calculate_deps()
1174 if (inst->dst.file == VGRF) { in calculate_deps()
1176 for (unsigned r = 0; r < regs_written(inst); r++) in calculate_deps()
1177 last_grf_write[inst->dst.nr + r] = n; in calculate_deps()
1179 for (unsigned r = 0; r < regs_written(inst); r++) { in calculate_deps()
1180 last_grf_write[inst->dst.nr * 16 + in calculate_deps()
1181 inst->dst.offset / REG_SIZE + r] = n; in calculate_deps()
1184 } else if (inst->dst.file == MRF) { in calculate_deps()
1185 int reg = inst->dst.nr & ~BRW_MRF_COMPR4; in calculate_deps()
1189 if (is_compressed(inst)) { in calculate_deps()
1190 if (inst->dst.nr & BRW_MRF_COMPR4) in calculate_deps()
1197 } else if (inst->dst.file == FIXED_GRF) { in calculate_deps()
1199 for (unsigned r = 0; r < regs_written(inst); r++) in calculate_deps()
1200 last_grf_write[inst->dst.nr + r] = n; in calculate_deps()
1204 } else if (inst->dst.is_accumulator()) { in calculate_deps()
1206 } else if (inst->dst.file == ARF && !inst->dst.is_null()) { in calculate_deps()
1210 if (inst->mlen > 0 && inst->base_mrf != -1) { in calculate_deps()
1211 for (int i = 0; i < v->implied_mrf_writes(inst); i++) { in calculate_deps()
1212 last_mrf_write[inst->base_mrf + i] = n; in calculate_deps()
1216 if (const unsigned mask = inst->flags_written()) { in calculate_deps()
1225 if (inst->writes_accumulator_implicitly(v->devinfo)) { in calculate_deps()
1250 vec4_instruction *inst = (vec4_instruction *)n->inst; in calculate_deps() local
1252 if (is_scheduling_barrier(inst)) in calculate_deps()
1257 if (inst->src[i].file == VGRF) { in calculate_deps()
1258 for (unsigned j = 0; j < regs_read(inst, i); ++j) in calculate_deps()
1259 add_dep(last_grf_write[inst->src[i].nr + j], n); in calculate_deps()
1260 } else if (inst->src[i].file == FIXED_GRF) { in calculate_deps()
1262 } else if (inst->src[i].is_accumulator()) { in calculate_deps()
1265 } else if (inst->src[i].file == ARF) { in calculate_deps()
1270 if (inst->reads_g0_implicitly()) in calculate_deps()
1273 if (!inst->is_send_from_grf()) { in calculate_deps()
1274 for (int i = 0; i < inst->mlen; i++) { in calculate_deps()
1279 add_dep(last_mrf_write[inst->base_mrf + i], n); in calculate_deps()
1283 if (inst->reads_flag()) { in calculate_deps()
1288 if (inst->reads_accumulator_implicitly()) { in calculate_deps()
1294 if (inst->dst.file == VGRF) { in calculate_deps()
1295 for (unsigned j = 0; j < regs_written(inst); ++j) { in calculate_deps()
1296 add_dep(last_grf_write[inst->dst.nr + j], n); in calculate_deps()
1297 last_grf_write[inst->dst.nr + j] = n; in calculate_deps()
1299 } else if (inst->dst.file == MRF) { in calculate_deps()
1300 add_dep(last_mrf_write[inst->dst.nr], n); in calculate_deps()
1301 last_mrf_write[inst->dst.nr] = n; in calculate_deps()
1302 } else if (inst->dst.file == FIXED_GRF) { in calculate_deps()
1304 } else if (inst->dst.is_accumulator()) { in calculate_deps()
1307 } else if (inst->dst.file == ARF && !inst->dst.is_null()) { in calculate_deps()
1311 if (inst->mlen > 0 && !inst->is_send_from_grf()) { in calculate_deps()
1312 for (int i = 0; i < v->implied_mrf_writes(inst); i++) { in calculate_deps()
1313 add_dep(last_mrf_write[inst->base_mrf + i], n); in calculate_deps()
1314 last_mrf_write[inst->base_mrf + i] = n; in calculate_deps()
1318 if (inst->writes_flag()) { in calculate_deps()
1323 if (inst->writes_accumulator_implicitly(v->devinfo) && in calculate_deps()
1324 !inst->dst.is_accumulator()) { in calculate_deps()
1338 vec4_instruction *inst = (vec4_instruction *)n->inst; in calculate_deps() local
1342 if (inst->src[i].file == VGRF) { in calculate_deps()
1343 for (unsigned j = 0; j < regs_read(inst, i); ++j) in calculate_deps()
1344 add_dep(n, last_grf_write[inst->src[i].nr + j]); in calculate_deps()
1345 } else if (inst->src[i].file == FIXED_GRF) { in calculate_deps()
1347 } else if (inst->src[i].is_accumulator()) { in calculate_deps()
1349 } else if (inst->src[i].file == ARF) { in calculate_deps()
1354 if (!inst->is_send_from_grf()) { in calculate_deps()
1355 for (int i = 0; i < inst->mlen; i++) { in calculate_deps()
1360 add_dep(n, last_mrf_write[inst->base_mrf + i], 2); in calculate_deps()
1364 if (inst->reads_flag()) { in calculate_deps()
1368 if (inst->reads_accumulator_implicitly()) { in calculate_deps()
1375 if (inst->dst.file == VGRF) { in calculate_deps()
1376 for (unsigned j = 0; j < regs_written(inst); ++j) in calculate_deps()
1377 last_grf_write[inst->dst.nr + j] = n; in calculate_deps()
1378 } else if (inst->dst.file == MRF) { in calculate_deps()
1379 last_mrf_write[inst->dst.nr] = n; in calculate_deps()
1380 } else if (inst->dst.file == FIXED_GRF) { in calculate_deps()
1382 } else if (inst->dst.is_accumulator()) { in calculate_deps()
1384 } else if (inst->dst.file == ARF && !inst->dst.is_null()) { in calculate_deps()
1388 if (inst->mlen > 0 && !inst->is_send_from_grf()) { in calculate_deps()
1389 for (int i = 0; i < v->implied_mrf_writes(inst); i++) { in calculate_deps()
1390 last_mrf_write[inst->base_mrf + i] = n; in calculate_deps()
1394 if (inst->writes_flag()) { in calculate_deps()
1398 if (inst->writes_accumulator_implicitly(v->devinfo)) { in calculate_deps()
1433 fs_inst *inst = (fs_inst *)n->inst; in choose_instruction_to_schedule() local
1443 int register_pressure_benefit = get_register_pressure_benefit(n->inst); in choose_instruction_to_schedule()
1445 get_register_pressure_benefit(chosen->inst); in choose_instruction_to_schedule()
1479 fs_inst *chosen_inst = (fs_inst *)chosen->inst; in choose_instruction_to_schedule()
1486 if (inst->size_written <= 4 * inst->exec_size && in choose_instruction_to_schedule()
1490 } else if (inst->size_written > chosen_inst->size_written) { in choose_instruction_to_schedule()
1547 fs_instruction_scheduler::issue_time(backend_instruction *inst) in issue_time() argument
1549 const unsigned overhead = v->bank_conflict_cycles((fs_inst *)inst); in issue_time()
1550 if (is_compressed((fs_inst *)inst)) in issue_time()
1557 vec4_instruction_scheduler::issue_time(backend_instruction *inst) in issue_time() argument
1585 chosen->inst->exec_node::remove(); in schedule_instructions()
1586 block->instructions.push_tail(chosen->inst); in schedule_instructions()
1590 reg_pressure -= get_register_pressure_benefit(chosen->inst); in schedule_instructions()
1591 update_register_pressure(chosen->inst); in schedule_instructions()
1605 time += issue_time(chosen->inst); in schedule_instructions()
1609 bs->dump_instruction(chosen->inst); in schedule_instructions()
1627 bs->dump_instruction(child->inst); in schedule_instructions()
1646 if (devinfo->gen < 6 && chosen->inst->is_math()) { in schedule_instructions()
1648 if (n->inst->is_math()) in schedule_instructions()
1696 foreach_inst_in_block(fs_inst, inst, block) in run()
1697 count_reads_remaining(inst); in run()