Lines Matching refs:regs
106 bi_assign_fau_idx_single(bi_registers *regs, in bi_assign_fau_idx_single() argument
126 if (assigned && regs->fau_idx != C) in bi_assign_fau_idx_single()
129 regs->fau_idx = C; in bi_assign_fau_idx_single()
148 if (assigned && regs->fau_idx != f) in bi_assign_fau_idx_single()
151 regs->fau_idx = f; in bi_assign_fau_idx_single()
162 if (assigned && regs->fau_idx != f) in bi_assign_fau_idx_single()
165 regs->fau_idx = f; in bi_assign_fau_idx_single()
177 if (assigned && regs->fau_idx != (8 | rt)) in bi_assign_fau_idx_single()
180 regs->fau_idx = 8 | rt; in bi_assign_fau_idx_single()
195 bi_assign_fau_idx_single(&bundle->regs, clause, bundle->fma, false, true); in bi_assign_fau_idx()
197 bi_assign_fau_idx_single(&bundle->regs, clause, bundle->add, assigned, false); in bi_assign_fau_idx()
203 bi_assign_slot_read(bi_registers *regs, unsigned src) in bi_assign_slot_read() argument
213 if (regs->slot[i] == reg && regs->enabled[i]) in bi_assign_slot_read()
217 if (regs->slot[2] == reg && regs->slot23.slot2 == BIFROST_OP_READ) in bi_assign_slot_read()
223 if (!regs->enabled[i]) { in bi_assign_slot_read()
224 regs->slot[i] = reg; in bi_assign_slot_read()
225 regs->enabled[i] = true; in bi_assign_slot_read()
230 if (!regs->slot23.slot3) { in bi_assign_slot_read()
231 regs->slot[2] = reg; in bi_assign_slot_read()
232 regs->slot23.slot2 = BIFROST_OP_READ; in bi_assign_slot_read()
236 bi_print_slots(regs, stderr); in bi_assign_slot_read()
257 bi_assign_slot_read(&now->regs, now->fma->src[src]); in bi_assign_slots()
262 bi_assign_slot_read(&now->regs, now->add->src[src]); in bi_assign_slots()
269 now->regs.slot[3] = prev->add->dest & ~BIR_INDEX_REGISTER; in bi_assign_slots()
270 now->regs.slot23.slot3 = BIFROST_OP_WRITE; in bi_assign_slots()
276 if (now->regs.slot23.slot3) { in bi_assign_slots()
278 assert(!now->regs.slot23.slot2); in bi_assign_slots()
279 now->regs.slot[2] = r; in bi_assign_slots()
280 now->regs.slot23.slot2 = BIFROST_OP_WRITE; in bi_assign_slots()
282 now->regs.slot[3] = r; in bi_assign_slots()
283 now->regs.slot23.slot3 = BIFROST_OP_WRITE; in bi_assign_slots()
284 now->regs.slot23.slot3_fma = true; in bi_assign_slots()
288 return now->regs; in bi_assign_slots()
309 bi_pack_registers(bi_registers regs) in bi_pack_registers() argument
311 enum bifrost_reg_mode mode = bi_pack_register_mode(regs); in bi_pack_registers()
321 if (regs.first_instruction) { in bi_pack_registers()
333 if (!(regs.slot23.slot2 && regs.slot23.slot3)) in bi_pack_registers()
341 if (regs.enabled[1]) { in bi_pack_registers()
343 assert(regs.slot[1] > regs.slot[0]); in bi_pack_registers()
344 assert(regs.enabled[0]); in bi_pack_registers()
347 if (regs.slot[0] > 31) { in bi_pack_registers()
348 regs.slot[0] = 63 - regs.slot[0]; in bi_pack_registers()
349 regs.slot[1] = 63 - regs.slot[1]; in bi_pack_registers()
352 assert(regs.slot[0] <= 31); in bi_pack_registers()
353 assert(regs.slot[1] <= 63); in bi_pack_registers()
356 s.reg1 = regs.slot[1]; in bi_pack_registers()
357 s.reg0 = regs.slot[0]; in bi_pack_registers()
363 if (regs.enabled[0]) { in bi_pack_registers()
365 s.reg1 |= (regs.slot[0] >> 5); in bi_pack_registers()
368 s.reg0 = (regs.slot[0] & 0b11111); in bi_pack_registers()
377 assert(regs.slot[3] == regs.slot[2] || !(regs.slot23.slot2 && regs.slot23.slot3)); in bi_pack_registers()
379 if (regs.slot23.slot2) in bi_pack_registers()
380 regs.slot[3] = regs.slot[2]; in bi_pack_registers()
382 regs.slot[2] = regs.slot[3]; in bi_pack_registers()
383 } else if (!regs.first_instruction) { in bi_pack_registers()
385 assert(regs.slot[2] != regs.slot[3]); in bi_pack_registers()
388 s.reg2 = regs.slot[2]; in bi_pack_registers()
389 s.reg3 = regs.slot[3]; in bi_pack_registers()
390 s.fau_idx = regs.fau_idx; in bi_pack_registers()
397 bi_pack_fma_special(bi_clause *clause, bi_instruction *ins, bi_registers *regs) in bi_pack_fma_special() argument
401 return pan_pack_fma_cubeface1(clause, ins, regs); in bi_pack_fma_special()
409 bi_pack_fma_ ## name(bi_clause *clause, bi_instruction *ins, bi_registers *regs) \
413 return pan_pack_fma_ ## name ## _i32(clause, ins, regs); \
415 return pan_pack_fma_ ## name ## _v2i16(clause, ins, regs); \
417 return pan_pack_fma_ ## name ## _v4i8(clause, ins, regs); \
432 bi_pack_fma_bitwise(bi_clause *clause, bi_instruction *ins, bi_registers *regs) in BI_PACK_SHIFT()
437 bi_pack_fma_rshift_and(clause, ins, regs) : in BI_PACK_SHIFT()
438 bi_pack_fma_lshift_and(clause, ins, regs); in BI_PACK_SHIFT()
441 bi_pack_fma_rshift_or(clause, ins, regs) : in BI_PACK_SHIFT()
442 bi_pack_fma_lshift_or(clause, ins, regs); in BI_PACK_SHIFT()
445 bi_pack_fma_rshift_xor(clause, ins, regs) : in BI_PACK_SHIFT()
446 bi_pack_fma_lshift_xor(clause, ins, regs); in BI_PACK_SHIFT()
449 return bi_pack_fma_arshift(clause, ins, regs); in BI_PACK_SHIFT()
456 bi_pack_fma(bi_clause *clause, bi_bundle bundle, bi_registers *regs) in bi_pack_fma() argument
459 return pan_pack_fma_nop_i32(clause, NULL, regs); in bi_pack_fma()
482 return pan_pack_fma_fadd_f32(clause, bundle.fma, regs); in bi_pack_fma()
484 return pan_pack_fma_fadd_v2f16(clause, bundle.fma, regs); in bi_pack_fma()
491 return pan_pack_fma_fcmp_f32(clause, bundle.fma, regs); in bi_pack_fma()
493 return pan_pack_fma_fcmp_v2f16(clause, bundle.fma, regs); in bi_pack_fma()
495 return bi_pack_fma_bitwise(clause, bundle.fma, regs); in bi_pack_fma()
499 return pan_pack_fma_s8_to_s32(clause, bundle.fma, regs); in bi_pack_fma()
502 return pan_pack_fma_u8_to_u32(clause, bundle.fma, regs); in bi_pack_fma()
505 return pan_pack_fma_s16_to_s32(clause, bundle.fma, regs); in bi_pack_fma()
508 return pan_pack_fma_u16_to_u32(clause, bundle.fma, regs); in bi_pack_fma()
511 return pan_pack_fma_f16_to_f32(clause, bundle.fma, regs); in bi_pack_fma()
514 return pan_pack_fma_v2f32_to_v2f16(clause, bundle.fma, regs); in bi_pack_fma()
520 return pan_pack_fma_csel_f32(clause, bundle.fma, regs); in bi_pack_fma()
522 return pan_pack_fma_csel_v2f16(clause, bundle.fma, regs); in bi_pack_fma()
524 return pan_pack_fma_csel_i32(clause, bundle.fma, regs); in bi_pack_fma()
526 return pan_pack_fma_csel_v2i16(clause, bundle.fma, regs); in bi_pack_fma()
528 return pan_pack_fma_csel_u32(clause, bundle.fma, regs); in bi_pack_fma()
530 return pan_pack_fma_csel_v2u16(clause, bundle.fma, regs); in bi_pack_fma()
532 return pan_pack_fma_csel_s32(clause, bundle.fma, regs); in bi_pack_fma()
534 return pan_pack_fma_csel_v2s16(clause, bundle.fma, regs); in bi_pack_fma()
540 return pan_pack_fma_fma_rscale_f32(clause, bundle.fma, regs); in bi_pack_fma()
542 return pan_pack_fma_fma_f32(clause, bundle.fma, regs); in bi_pack_fma()
547 return pan_pack_fma_fma_rscale_v2f16(clause, bundle.fma, regs); in bi_pack_fma()
549 return pan_pack_fma_fma_v2f16(clause, bundle.fma, regs); in bi_pack_fma()
555 return pan_pack_fma_frexpe_f32(clause, bundle.fma, regs); in bi_pack_fma()
557 return pan_pack_fma_frexpe_v2f16(clause, bundle.fma, regs); in bi_pack_fma()
563 return pan_pack_fma_iaddc_i32(clause, bundle.fma, regs); in bi_pack_fma()
565 return pan_pack_fma_isubb_i32(clause, bundle.fma, regs); in bi_pack_fma()
567 return pan_pack_fma_mov_i32(clause, bundle.fma, regs); in bi_pack_fma()
570 return pan_pack_fma_mkvec_v2i16(clause, bundle.fma, regs); in bi_pack_fma()
573 return pan_pack_fma_mkvec_v4i8(clause, bundle.fma, regs); in bi_pack_fma()
579 return pan_pack_fma_fround_v2f16(clause, bundle.fma, regs); in bi_pack_fma()
581 return pan_pack_fma_fround_f32(clause, bundle.fma, regs); in bi_pack_fma()
584 return pan_pack_fma_fadd_lscale_f32(clause, bundle.fma, regs); in bi_pack_fma()
586 return pan_pack_fma_imul_i32(clause, bundle.fma, regs); in bi_pack_fma()
588 return bi_pack_fma_special(clause, bundle.fma, regs); in bi_pack_fma()
595 bi_pack_add_branch_cond(bi_instruction *ins, bi_registers *regs) in bi_pack_add_branch_cond() argument
614 .src0 = bi_get_src(ins, regs, 0), in bi_pack_add_branch_cond()
629 pack.src2 = bi_get_src(ins, regs, 2); in bi_pack_add_branch_cond()
636 bi_pack_add_branch_uncond(bi_instruction *ins, bi_registers *regs) in bi_pack_add_branch_uncond() argument
656 pack.src2 = bi_get_src(ins, regs, 2); in bi_pack_add_branch_uncond()
663 bi_pack_add_branch(bi_instruction *ins, bi_registers *regs) in bi_pack_add_branch() argument
666 return bi_pack_add_branch_uncond(ins, regs); in bi_pack_add_branch()
668 return bi_pack_add_branch_cond(ins, regs); in bi_pack_add_branch()
672 bi_pack_add_special(bi_clause *clause, bi_instruction *ins, bi_registers *regs) in bi_pack_add_special() argument
678 return f16 ? pan_pack_add_frcp_f16(clause, ins, regs) : in bi_pack_add_special()
679 pan_pack_add_frcp_f32(clause, ins, regs); in bi_pack_add_special()
681 return f16 ? pan_pack_add_frsq_f16(clause, ins, regs) : in bi_pack_add_special()
682 pan_pack_add_frsq_f32(clause, ins, regs); in bi_pack_add_special()
685 return pan_pack_add_fexp_f32(clause, ins, regs); in bi_pack_add_special()
688 return pan_pack_add_iabs_s32(clause, ins, regs); in bi_pack_add_special()
690 return pan_pack_add_cubeface2(clause, ins, regs); in bi_pack_add_special()
692 return pan_pack_add_cube_ssel(clause, ins, regs); in bi_pack_add_special()
694 return pan_pack_add_cube_tsel(clause, ins, regs); in bi_pack_add_special()
701 bi_pack_add(bi_clause *clause, bi_bundle bundle, bi_registers *regs, gl_shader_stage stage) in bi_pack_add() argument
704 return pan_pack_add_nop_i32(clause, NULL, regs); in bi_pack_add()
730 return pan_pack_add_fadd_f32(clause, bundle.add, regs); in bi_pack_add()
732 return pan_pack_add_fadd_v2f16(clause, bundle.add, regs); in bi_pack_add()
736 return pan_pack_add_atest(clause, bundle.add, regs); in bi_pack_add()
738 return bi_pack_add_branch(bundle.add, regs); in bi_pack_add()
741 return pan_pack_add_fcmp_f32(clause, bundle.add, regs); in bi_pack_add()
743 return pan_pack_add_fcmp_v2f16(clause, bundle.add, regs); in bi_pack_add()
745 return pan_pack_add_icmp_i32(clause, bundle.add, regs); in bi_pack_add()
747 return pan_pack_add_icmp_v2i16(clause, bundle.add, regs); in bi_pack_add()
749 return pan_pack_add_icmp_v4i8(clause, bundle.add, regs); in bi_pack_add()
751 return pan_pack_add_icmp_u32(clause, bundle.add, regs); in bi_pack_add()
753 return pan_pack_add_icmp_v2u16(clause, bundle.add, regs); in bi_pack_add()
755 return pan_pack_add_icmp_v4u8(clause, bundle.add, regs); in bi_pack_add()
757 return pan_pack_add_icmp_s32(clause, bundle.add, regs); in bi_pack_add()
759 return pan_pack_add_icmp_v2s16(clause, bundle.add, regs); in bi_pack_add()
761 return pan_pack_add_icmp_v4s8(clause, bundle.add, regs); in bi_pack_add()
765 return pan_pack_add_blend(clause, bundle.add, regs); in bi_pack_add()
770 return pan_pack_add_v2f16_to_v2s16(clause, bundle.add, regs); in bi_pack_add()
772 return pan_pack_add_v2f16_to_v2u16(clause, bundle.add, regs); in bi_pack_add()
774 return pan_pack_add_f16_to_s32(clause, bundle.add, regs); in bi_pack_add()
776 return pan_pack_add_f16_to_u32(clause, bundle.add, regs); in bi_pack_add()
778 return pan_pack_add_v2s16_to_v2f16(clause, bundle.add, regs); in bi_pack_add()
780 return pan_pack_add_v2u16_to_v2f16(clause, bundle.add, regs); in bi_pack_add()
782 return pan_pack_add_v2s8_to_v2s16(clause, bundle.add, regs); in bi_pack_add()
784 return pan_pack_add_v2u8_to_v2u16(clause, bundle.add, regs); in bi_pack_add()
786 return pan_pack_add_v2s8_to_v2f16(clause, bundle.add, regs); in bi_pack_add()
788 return pan_pack_add_v2u8_to_v2f16(clause, bundle.add, regs); in bi_pack_add()
790 return pan_pack_add_f32_to_s32(clause, bundle.add, regs); in bi_pack_add()
792 return pan_pack_add_f32_to_u32(clause, bundle.add, regs); in bi_pack_add()
794 return pan_pack_add_s8_to_s32(clause, bundle.add, regs); in bi_pack_add()
796 return pan_pack_add_u8_to_u32(clause, bundle.add, regs); in bi_pack_add()
798 return pan_pack_add_s8_to_f32(clause, bundle.add, regs); in bi_pack_add()
800 return pan_pack_add_u8_to_f32(clause, bundle.add, regs); in bi_pack_add()
802 return pan_pack_add_s32_to_f32(clause, bundle.add, regs); in bi_pack_add()
804 return pan_pack_add_u32_to_f32(clause, bundle.add, regs); in bi_pack_add()
806 return pan_pack_add_s16_to_s32(clause, bundle.add, regs); in bi_pack_add()
808 return pan_pack_add_u16_to_u32(clause, bundle.add, regs); in bi_pack_add()
810 return pan_pack_add_s16_to_f32(clause, bundle.add, regs); in bi_pack_add()
812 return pan_pack_add_u16_to_f32(clause, bundle.add, regs); in bi_pack_add()
814 return pan_pack_add_f16_to_f32(clause, bundle.add, regs); in bi_pack_add()
816 return pan_pack_add_v2f32_to_v2f16(clause, bundle.add, regs); in bi_pack_add()
820 return pan_pack_add_discard_f32(clause, bundle.add, regs); in bi_pack_add()
827 return (sz == 8) ? pan_pack_add_iadd_v4s8(clause, bundle.add, regs) : in bi_pack_add()
828 (sz == 16) ? pan_pack_add_iadd_v2s16(clause, bundle.add, regs) : in bi_pack_add()
829 pan_pack_add_iadd_s32(clause, bundle.add, regs); in bi_pack_add()
831 return (sz == 8) ? pan_pack_add_isub_v4s8(clause, bundle.add, regs) : in bi_pack_add()
832 (sz == 16) ? pan_pack_add_isub_v2s16(clause, bundle.add, regs) : in bi_pack_add()
833 pan_pack_add_isub_s32(clause, bundle.add, regs); in bi_pack_add()
836 return pan_pack_add_ld_attr_imm(clause, bundle.add, regs); in bi_pack_add()
841 case 1: return pan_pack_add_load_i32(clause, bundle.add, regs); in bi_pack_add()
842 case 2: return pan_pack_add_load_i64(clause, bundle.add, regs); in bi_pack_add()
843 case 3: return pan_pack_add_load_i96(clause, bundle.add, regs); in bi_pack_add()
844 case 4: return pan_pack_add_load_i128(clause, bundle.add, regs); in bi_pack_add()
850 return pan_pack_add_ld_var_special(clause, bundle.add, regs); in bi_pack_add()
852 return pan_pack_add_ld_var_flat_imm(clause, bundle.add, regs); in bi_pack_add()
854 return pan_pack_add_ld_var_imm(clause, bundle.add, regs); in bi_pack_add()
857 return pan_pack_add_ld_var_flat(clause, bundle.add, regs); in bi_pack_add()
859 return pan_pack_add_ld_var(clause, bundle.add, regs); in bi_pack_add()
862 return pan_pack_add_lea_attr_imm(clause, bundle.add, regs); in bi_pack_add()
864 return pan_pack_add_ld_tile(clause, bundle.add, regs); in bi_pack_add()
868 return pan_pack_add_fmin_f32(clause, bundle.add, regs); in bi_pack_add()
870 return pan_pack_add_fmin_v2f16(clause, bundle.add, regs); in bi_pack_add()
874 return pan_pack_add_fmax_f32(clause, bundle.add, regs); in bi_pack_add()
876 return pan_pack_add_fmax_v2f16(clause, bundle.add, regs); in bi_pack_add()
884 case 1: return pan_pack_add_store_i32(clause, bundle.add, regs); in bi_pack_add()
885 case 2: return pan_pack_add_store_i64(clause, bundle.add, regs); in bi_pack_add()
886 case 3: return pan_pack_add_store_i96(clause, bundle.add, regs); in bi_pack_add()
887 case 4: return pan_pack_add_store_i128(clause, bundle.add, regs); in bi_pack_add()
891 return pan_pack_add_st_cvt(clause, bundle.add, regs); in bi_pack_add()
893 return bi_pack_add_special(clause, bundle.add, regs); in bi_pack_add()
896 return pan_pack_add_flogd_f32(clause, bundle.add, regs); in bi_pack_add()
899 return pan_pack_add_mkvec_v2i16(clause, bundle.add, regs); in bi_pack_add()
901 return pan_pack_add_texc(clause, bundle.add, regs); in bi_pack_add()
908 return pan_pack_add_texs_2d_f16(clause, bundle.add, regs); in bi_pack_add()
910 return pan_pack_add_texs_2d_f32(clause, bundle.add, regs); in bi_pack_add()
914 return pan_pack_add_zs_emit(clause, bundle.add, regs); in bi_pack_add()
929 bi_flip_slots(bi_registers *regs) in bi_flip_slots() argument
931 if (regs->enabled[0] && regs->enabled[1] && regs->slot[1] < regs->slot[0]) { in bi_flip_slots()
932 unsigned temp = regs->slot[0]; in bi_flip_slots()
933 regs->slot[0] = regs->slot[1]; in bi_flip_slots()
934 regs->slot[1] = temp; in bi_flip_slots()
985 bundle.regs.first_instruction = first_bundle; in bi_pack_bundle()
987 bi_flip_slots(&bundle.regs); in bi_pack_bundle()
989 uint64_t reg = bi_pack_registers(bundle.regs); in bi_pack_bundle()
990 uint64_t fma = bi_pack_fma(clause, bundle, &bundle.regs); in bi_pack_bundle()
991 uint64_t add = bi_pack_add(clause, bundle, &bundle.regs, stage); in bi_pack_bundle()