Lines Matching full:bundle
69 * pushed uniform per bundle. Figure out which one we need in the bundle (the
70 * scheduler needs to ensure we only have one type per bundle), validate
192 bi_bundle *bundle) in bi_assign_fau_idx() argument
195 bi_assign_fau_idx_single(&bundle->regs, clause, bundle->fma, false, true); in bi_assign_fau_idx()
197 bi_assign_fau_idx_single(&bundle->regs, clause, bundle->add, assigned, false); in bi_assign_fau_idx()
456 bi_pack_fma(bi_clause *clause, bi_bundle bundle, bi_registers *regs) in bi_pack_fma() argument
458 if (!bundle.fma) in bi_pack_fma()
461 bool f16 = bundle.fma->dest_type == nir_type_float16; in bi_pack_fma()
462 bool f32 = bundle.fma->dest_type == nir_type_float32; in bi_pack_fma()
463 bool u32 = bundle.fma->dest_type == nir_type_uint32 || in bi_pack_fma()
464 bundle.fma->dest_type == nir_type_bool32; in bi_pack_fma()
465 bool u16 = bundle.fma->dest_type == nir_type_uint16; in bi_pack_fma()
466 bool s32 = bundle.fma->dest_type == nir_type_int32; in bi_pack_fma()
467 bool s16 = bundle.fma->dest_type == nir_type_int16; in bi_pack_fma()
469 bool src0_f16 = bundle.fma->src_types[0] == nir_type_float16; in bi_pack_fma()
470 bool src0_f32 = bundle.fma->src_types[0] == nir_type_float32; in bi_pack_fma()
471 bool src0_u16 = bundle.fma->src_types[0] == nir_type_uint16; in bi_pack_fma()
472 bool src0_s16 = bundle.fma->src_types[0] == nir_type_int16; in bi_pack_fma()
473 bool src0_s8 = bundle.fma->src_types[0] == nir_type_int8; in bi_pack_fma()
474 bool src0_u8 = bundle.fma->src_types[0] == nir_type_uint8; in bi_pack_fma()
476 enum bi_cond cond = bundle.fma->cond; in bi_pack_fma()
479 switch (bundle.fma->type) { in bi_pack_fma()
481 if (bundle.fma->dest_type == nir_type_float32) in bi_pack_fma()
482 return pan_pack_fma_fadd_f32(clause, bundle.fma, regs); in bi_pack_fma()
483 else if (bundle.fma->dest_type == nir_type_float16) in bi_pack_fma()
484 return pan_pack_fma_fadd_v2f16(clause, bundle.fma, regs); in bi_pack_fma()
491 return pan_pack_fma_fcmp_f32(clause, bundle.fma, regs); in bi_pack_fma()
493 return pan_pack_fma_fcmp_v2f16(clause, bundle.fma, regs); in bi_pack_fma()
495 return bi_pack_fma_bitwise(clause, bundle.fma, regs); in bi_pack_fma()
499 return pan_pack_fma_s8_to_s32(clause, bundle.fma, regs); in bi_pack_fma()
502 return pan_pack_fma_u8_to_u32(clause, bundle.fma, regs); in bi_pack_fma()
505 return pan_pack_fma_s16_to_s32(clause, bundle.fma, regs); in bi_pack_fma()
508 return pan_pack_fma_u16_to_u32(clause, bundle.fma, regs); in bi_pack_fma()
511 return pan_pack_fma_f16_to_f32(clause, bundle.fma, regs); in bi_pack_fma()
514 return pan_pack_fma_v2f32_to_v2f16(clause, bundle.fma, regs); in bi_pack_fma()
520 return pan_pack_fma_csel_f32(clause, bundle.fma, regs); in bi_pack_fma()
522 return pan_pack_fma_csel_v2f16(clause, bundle.fma, regs); in bi_pack_fma()
524 return pan_pack_fma_csel_i32(clause, bundle.fma, regs); in bi_pack_fma()
526 return pan_pack_fma_csel_v2i16(clause, bundle.fma, regs); in bi_pack_fma()
528 return pan_pack_fma_csel_u32(clause, bundle.fma, regs); in bi_pack_fma()
530 return pan_pack_fma_csel_v2u16(clause, bundle.fma, regs); in bi_pack_fma()
532 return pan_pack_fma_csel_s32(clause, bundle.fma, regs); in bi_pack_fma()
534 return pan_pack_fma_csel_v2s16(clause, bundle.fma, regs); in bi_pack_fma()
538 if (bundle.fma->dest_type == nir_type_float32) { in bi_pack_fma()
539 if (bundle.fma->op.mscale) in bi_pack_fma()
540 return pan_pack_fma_fma_rscale_f32(clause, bundle.fma, regs); in bi_pack_fma()
542 return pan_pack_fma_fma_f32(clause, bundle.fma, regs); in bi_pack_fma()
544 assert(bundle.fma->dest_type == nir_type_float16); in bi_pack_fma()
546 if (bundle.fma->op.mscale) in bi_pack_fma()
547 return pan_pack_fma_fma_rscale_v2f16(clause, bundle.fma, regs); in bi_pack_fma()
549 return pan_pack_fma_fma_v2f16(clause, bundle.fma, regs); in bi_pack_fma()
555 return pan_pack_fma_frexpe_f32(clause, bundle.fma, regs); in bi_pack_fma()
557 return pan_pack_fma_frexpe_v2f16(clause, bundle.fma, regs); in bi_pack_fma()
562 if (bundle.fma->op.imath == BI_IMATH_ADD) in bi_pack_fma()
563 return pan_pack_fma_iaddc_i32(clause, bundle.fma, regs); in bi_pack_fma()
565 return pan_pack_fma_isubb_i32(clause, bundle.fma, regs); in bi_pack_fma()
567 return pan_pack_fma_mov_i32(clause, bundle.fma, regs); in bi_pack_fma()
569 if (nir_alu_type_get_type_size(bundle.fma->src_types[0]) == 16) { in bi_pack_fma()
570 return pan_pack_fma_mkvec_v2i16(clause, bundle.fma, regs); in bi_pack_fma()
572 assert(nir_alu_type_get_type_size(bundle.fma->src_types[0]) == 8); in bi_pack_fma()
573 return pan_pack_fma_mkvec_v4i8(clause, bundle.fma, regs); in bi_pack_fma()
579 return pan_pack_fma_fround_v2f16(clause, bundle.fma, regs); in bi_pack_fma()
581 return pan_pack_fma_fround_f32(clause, bundle.fma, regs); in bi_pack_fma()
584 return pan_pack_fma_fadd_lscale_f32(clause, bundle.fma, regs); in bi_pack_fma()
586 return pan_pack_fma_imul_i32(clause, bundle.fma, regs); in bi_pack_fma()
588 return bi_pack_fma_special(clause, bundle.fma, regs); in bi_pack_fma()
701 bi_pack_add(bi_clause *clause, bi_bundle bundle, bi_registers *regs, gl_shader_stage stage) in bi_pack_add() argument
703 if (!bundle.add) in bi_pack_add()
706 bool f16 = bundle.add->dest_type == nir_type_float16; in bi_pack_add()
707 bool f32 = bundle.add->dest_type == nir_type_float32; in bi_pack_add()
708 bool u32 = bundle.add->dest_type == nir_type_uint32 || in bi_pack_add()
709 bundle.add->dest_type == nir_type_bool32; in bi_pack_add()
710 bool u16 = bundle.add->dest_type == nir_type_uint16; in bi_pack_add()
711 bool s32 = bundle.add->dest_type == nir_type_int32; in bi_pack_add()
712 bool s16 = bundle.add->dest_type == nir_type_int16; in bi_pack_add()
714 bool src0_f16 = bundle.add->src_types[0] == nir_type_float16; in bi_pack_add()
715 bool src0_f32 = bundle.add->src_types[0] == nir_type_float32; in bi_pack_add()
716 bool src0_u32 = bundle.add->src_types[0] == nir_type_uint32; in bi_pack_add()
717 bool src0_u16 = bundle.add->src_types[0] == nir_type_uint16; in bi_pack_add()
718 bool src0_u8 = bundle.add->src_types[0] == nir_type_uint8; in bi_pack_add()
719 bool src0_s32 = bundle.add->src_types[0] == nir_type_int32; in bi_pack_add()
720 bool src0_s16 = bundle.add->src_types[0] == nir_type_int16; in bi_pack_add()
721 bool src0_s8 = bundle.add->src_types[0] == nir_type_int8; in bi_pack_add()
723 unsigned sz = nir_alu_type_get_type_size(bundle.add->dest_type); in bi_pack_add()
724 enum bi_cond cond = bundle.add->cond; in bi_pack_add()
727 switch (bundle.add->type) { in bi_pack_add()
729 if (bundle.add->dest_type == nir_type_float32) in bi_pack_add()
730 return pan_pack_add_fadd_f32(clause, bundle.add, regs); in bi_pack_add()
731 else if (bundle.add->dest_type == nir_type_float16) in bi_pack_add()
732 return pan_pack_add_fadd_v2f16(clause, bundle.add, regs); in bi_pack_add()
736 return pan_pack_add_atest(clause, bundle.add, regs); in bi_pack_add()
738 return bi_pack_add_branch(bundle.add, regs); in bi_pack_add()
741 return pan_pack_add_fcmp_f32(clause, bundle.add, regs); in bi_pack_add()
743 return pan_pack_add_fcmp_v2f16(clause, bundle.add, regs); in bi_pack_add()
745 return pan_pack_add_icmp_i32(clause, bundle.add, regs); in bi_pack_add()
747 return pan_pack_add_icmp_v2i16(clause, bundle.add, regs); in bi_pack_add()
749 return pan_pack_add_icmp_v4i8(clause, bundle.add, regs); in bi_pack_add()
751 return pan_pack_add_icmp_u32(clause, bundle.add, regs); in bi_pack_add()
753 return pan_pack_add_icmp_v2u16(clause, bundle.add, regs); in bi_pack_add()
755 return pan_pack_add_icmp_v4u8(clause, bundle.add, regs); in bi_pack_add()
757 return pan_pack_add_icmp_s32(clause, bundle.add, regs); in bi_pack_add()
759 return pan_pack_add_icmp_v2s16(clause, bundle.add, regs); in bi_pack_add()
761 return pan_pack_add_icmp_v4s8(clause, bundle.add, regs); in bi_pack_add()
765 return pan_pack_add_blend(clause, bundle.add, regs); in bi_pack_add()
770 return pan_pack_add_v2f16_to_v2s16(clause, bundle.add, regs); in bi_pack_add()
772 return pan_pack_add_v2f16_to_v2u16(clause, bundle.add, regs); in bi_pack_add()
774 return pan_pack_add_f16_to_s32(clause, bundle.add, regs); in bi_pack_add()
776 return pan_pack_add_f16_to_u32(clause, bundle.add, regs); in bi_pack_add()
778 return pan_pack_add_v2s16_to_v2f16(clause, bundle.add, regs); in bi_pack_add()
780 return pan_pack_add_v2u16_to_v2f16(clause, bundle.add, regs); in bi_pack_add()
782 return pan_pack_add_v2s8_to_v2s16(clause, bundle.add, regs); in bi_pack_add()
784 return pan_pack_add_v2u8_to_v2u16(clause, bundle.add, regs); in bi_pack_add()
786 return pan_pack_add_v2s8_to_v2f16(clause, bundle.add, regs); in bi_pack_add()
788 return pan_pack_add_v2u8_to_v2f16(clause, bundle.add, regs); in bi_pack_add()
790 return pan_pack_add_f32_to_s32(clause, bundle.add, regs); in bi_pack_add()
792 return pan_pack_add_f32_to_u32(clause, bundle.add, regs); in bi_pack_add()
794 return pan_pack_add_s8_to_s32(clause, bundle.add, regs); in bi_pack_add()
796 return pan_pack_add_u8_to_u32(clause, bundle.add, regs); in bi_pack_add()
798 return pan_pack_add_s8_to_f32(clause, bundle.add, regs); in bi_pack_add()
800 return pan_pack_add_u8_to_f32(clause, bundle.add, regs); in bi_pack_add()
802 return pan_pack_add_s32_to_f32(clause, bundle.add, regs); in bi_pack_add()
804 return pan_pack_add_u32_to_f32(clause, bundle.add, regs); in bi_pack_add()
806 return pan_pack_add_s16_to_s32(clause, bundle.add, regs); in bi_pack_add()
808 return pan_pack_add_u16_to_u32(clause, bundle.add, regs); in bi_pack_add()
810 return pan_pack_add_s16_to_f32(clause, bundle.add, regs); in bi_pack_add()
812 return pan_pack_add_u16_to_f32(clause, bundle.add, regs); in bi_pack_add()
814 return pan_pack_add_f16_to_f32(clause, bundle.add, regs); in bi_pack_add()
816 return pan_pack_add_v2f32_to_v2f16(clause, bundle.add, regs); in bi_pack_add()
820 return pan_pack_add_discard_f32(clause, bundle.add, regs); in bi_pack_add()
826 if (bundle.add->op.imath == BI_IMATH_ADD) { in bi_pack_add()
827 return (sz == 8) ? pan_pack_add_iadd_v4s8(clause, bundle.add, regs) : in bi_pack_add()
828 (sz == 16) ? pan_pack_add_iadd_v2s16(clause, bundle.add, regs) : in bi_pack_add()
829 pan_pack_add_iadd_s32(clause, bundle.add, regs); in bi_pack_add()
831 return (sz == 8) ? pan_pack_add_isub_v4s8(clause, bundle.add, regs) : in bi_pack_add()
832 (sz == 16) ? pan_pack_add_isub_v2s16(clause, bundle.add, regs) : in bi_pack_add()
833 pan_pack_add_isub_s32(clause, bundle.add, regs); in bi_pack_add()
836 return pan_pack_add_ld_attr_imm(clause, bundle.add, regs); in bi_pack_add()
840 switch (bundle.add->vector_channels) { in bi_pack_add()
841 case 1: return pan_pack_add_load_i32(clause, bundle.add, regs); in bi_pack_add()
842 case 2: return pan_pack_add_load_i64(clause, bundle.add, regs); in bi_pack_add()
843 case 3: return pan_pack_add_load_i96(clause, bundle.add, regs); in bi_pack_add()
844 case 4: return pan_pack_add_load_i128(clause, bundle.add, regs); in bi_pack_add()
848 if (bundle.add->src[0] & BIR_INDEX_CONSTANT) { in bi_pack_add()
849 if (bi_get_immediate(bundle.add, 0) >= 20) in bi_pack_add()
850 return pan_pack_add_ld_var_special(clause, bundle.add, regs); in bi_pack_add()
851 else if (bundle.add->load_vary.flat) in bi_pack_add()
852 return pan_pack_add_ld_var_flat_imm(clause, bundle.add, regs); in bi_pack_add()
854 return pan_pack_add_ld_var_imm(clause, bundle.add, regs); in bi_pack_add()
856 if (bundle.add->load_vary.flat) in bi_pack_add()
857 return pan_pack_add_ld_var_flat(clause, bundle.add, regs); in bi_pack_add()
859 return pan_pack_add_ld_var(clause, bundle.add, regs); in bi_pack_add()
862 return pan_pack_add_lea_attr_imm(clause, bundle.add, regs); in bi_pack_add()
864 return pan_pack_add_ld_tile(clause, bundle.add, regs); in bi_pack_add()
866 if (bundle.add->op.minmax == BI_MINMAX_MIN) { in bi_pack_add()
867 if (bundle.add->dest_type == nir_type_float32) in bi_pack_add()
868 return pan_pack_add_fmin_f32(clause, bundle.add, regs); in bi_pack_add()
869 else if (bundle.add->dest_type == nir_type_float16) in bi_pack_add()
870 return pan_pack_add_fmin_v2f16(clause, bundle.add, regs); in bi_pack_add()
873 if (bundle.add->dest_type == nir_type_float32) in bi_pack_add()
874 return pan_pack_add_fmax_f32(clause, bundle.add, regs); in bi_pack_add()
875 else if (bundle.add->dest_type == nir_type_float16) in bi_pack_add()
876 return pan_pack_add_fmax_v2f16(clause, bundle.add, regs); in bi_pack_add()
883 switch (bundle.add->vector_channels) { in bi_pack_add()
884 case 1: return pan_pack_add_store_i32(clause, bundle.add, regs); in bi_pack_add()
885 case 2: return pan_pack_add_store_i64(clause, bundle.add, regs); in bi_pack_add()
886 case 3: return pan_pack_add_store_i96(clause, bundle.add, regs); in bi_pack_add()
887 case 4: return pan_pack_add_store_i128(clause, bundle.add, regs); in bi_pack_add()
891 return pan_pack_add_st_cvt(clause, bundle.add, regs); in bi_pack_add()
893 return bi_pack_add_special(clause, bundle.add, regs); in bi_pack_add()
895 assert(bundle.add->dest_type == nir_type_float32); in bi_pack_add()
896 return pan_pack_add_flogd_f32(clause, bundle.add, regs); in bi_pack_add()
898 assert(nir_alu_type_get_type_size(bundle.add->src_types[0]) == 16); in bi_pack_add()
899 return pan_pack_add_mkvec_v2i16(clause, bundle.add, regs); in bi_pack_add()
901 return pan_pack_add_texc(clause, bundle.add, regs); in bi_pack_add()
908 return pan_pack_add_texs_2d_f16(clause, bundle.add, regs); in bi_pack_add()
910 return pan_pack_add_texs_2d_f32(clause, bundle.add, regs); in bi_pack_add()
914 return pan_pack_add_zs_emit(clause, bundle.add, regs); in bi_pack_add()
943 bi_lower_cubeface2(bi_context *ctx, bi_bundle *bundle) in bi_lower_cubeface2() argument
946 if (!bundle->add || bundle->add->type != BI_SPECIAL_ADD in bi_lower_cubeface2()
947 || bundle->add->op.special != BI_SPECIAL_CUBEFACE2) { in bi_lower_cubeface2()
953 assert(!bundle->fma); in bi_lower_cubeface2()
965 memcpy(&cubeface1.src, bundle->add->src, sizeof(cubeface1.src)); in bi_lower_cubeface2()
969 assert(bundle->add->swizzle[i][0] == 0); in bi_lower_cubeface2()
972 bundle->fma = bi_emit_before(ctx, bundle->add, cubeface1); in bi_lower_cubeface2()
976 bundle->add->src[0] = BIR_INDEX_PASS | BIFROST_SRC_STAGE; in bi_lower_cubeface2()
977 bundle->add->src[1] = bundle->add->src[2] = 0; in bi_lower_cubeface2()
981 bi_pack_bundle(bi_clause *clause, bi_bundle bundle, bi_bundle prev, bool first_bundle, gl_shader_st… in bi_pack_bundle() argument
983 bi_assign_slots(&bundle, &prev); in bi_pack_bundle()
984 bi_assign_fau_idx(clause, &bundle); in bi_pack_bundle()
985 bundle.regs.first_instruction = first_bundle; in bi_pack_bundle()
987 bi_flip_slots(&bundle.regs); in bi_pack_bundle()
989 uint64_t reg = bi_pack_registers(bundle.regs); in bi_pack_bundle()
990 uint64_t fma = bi_pack_fma(clause, bundle, &bundle.regs); in bi_pack_bundle()
991 uint64_t add = bi_pack_add(clause, bundle, &bundle.regs, stage); in bi_pack_bundle()
1158 const bi_bundle *bundle = &clause->bundles[clause->bundle_count - 1]; in bi_collect_blend_ret_addr() local
1159 const bi_instruction *ins = bundle->add; in bi_collect_blend_ret_addr()