37 append_flrp_to_dead_list(struct u_vector *dead_flrp, struct nir_alu_instr *alu)  in append_flrp_to_dead_list()  argument
40    *tail = alu;  in append_flrp_to_dead_list()
48                          struct nir_alu_instr *alu)  in replace_with_strict_ffma()  argument
50    nir_ssa_def *const a = nir_ssa_for_alu_src(bld, alu, 0);  in replace_with_strict_ffma()
51    nir_ssa_def *const b = nir_ssa_for_alu_src(bld, alu, 1);  in replace_with_strict_ffma()
52    nir_ssa_def *const c = nir_ssa_for_alu_src(bld, alu, 2);  in replace_with_strict_ffma()
55    nir_instr_as_alu(neg_a->parent_instr)->exact = alu->exact;  in replace_with_strict_ffma()
58    nir_instr_as_alu(inner_ffma->parent_instr)->exact = alu->exact;  in replace_with_strict_ffma()
61    nir_instr_as_alu(outer_ffma->parent_instr)->exact = alu->exact;  in replace_with_strict_ffma()
63    nir_ssa_def_rewrite_uses(&alu->dest.dest.ssa, outer_ffma);  in replace_with_strict_ffma()
69    append_flrp_to_dead_list(dead_flrp, alu);  in replace_with_strict_ffma()
77                          struct nir_alu_instr *alu)  in replace_with_single_ffma()  argument
79    nir_ssa_def *const a = nir_ssa_for_alu_src(bld, alu, 0);  in replace_with_single_ffma()
80    nir_ssa_def *const b = nir_ssa_for_alu_src(bld, alu, 1);  in replace_with_single_ffma()
81    nir_ssa_def *const c = nir_ssa_for_alu_src(bld, alu, 2);  in replace_with_single_ffma()
84    nir_instr_as_alu(neg_c->parent_instr)->exact = alu->exact;  in replace_with_single_ffma()
88    nir_instr_as_alu(one_minus_c->parent_instr)->exact = alu->exact;  in replace_with_single_ffma()
91    nir_instr_as_alu(b_times_c->parent_instr)->exact = alu->exact;  in replace_with_single_ffma()
94    nir_instr_as_alu(final_ffma->parent_instr)->exact = alu->exact;  in replace_with_single_ffma()
96    nir_ssa_def_rewrite_uses(&alu->dest.dest.ssa, final_ffma);  in replace_with_single_ffma()
102    append_flrp_to_dead_list(dead_flrp, alu);  in replace_with_single_ffma()
110                     struct nir_alu_instr *alu)  in replace_with_strict()  argument
112    nir_ssa_def *const a = nir_ssa_for_alu_src(bld, alu, 0);  in replace_with_strict()
113    nir_ssa_def *const b = nir_ssa_for_alu_src(bld, alu, 1);  in replace_with_strict()
114    nir_ssa_def *const c = nir_ssa_for_alu_src(bld, alu, 2);  in replace_with_strict()
117    nir_instr_as_alu(neg_c->parent_instr)->exact = alu->exact;  in replace_with_strict()
121    nir_instr_as_alu(one_minus_c->parent_instr)->exact = alu->exact;  in replace_with_strict()
124    nir_instr_as_alu(first_product->parent_instr)->exact = alu->exact;  in replace_with_strict()
127    nir_instr_as_alu(second_product->parent_instr)->exact = alu->exact;  in replace_with_strict()
130    nir_instr_as_alu(sum->parent_instr)->exact = alu->exact;  in replace_with_strict()
132    nir_ssa_def_rewrite_uses(&alu->dest.dest.ssa, sum);  in replace_with_strict()
138    append_flrp_to_dead_list(dead_flrp, alu);  in replace_with_strict()
146                   struct nir_alu_instr *alu)  in replace_with_fast()  argument
148    nir_ssa_def *const a = nir_ssa_for_alu_src(bld, alu, 0);  in replace_with_fast()
149    nir_ssa_def *const b = nir_ssa_for_alu_src(bld, alu, 1);  in replace_with_fast()
150    nir_ssa_def *const c = nir_ssa_for_alu_src(bld, alu, 2);  in replace_with_fast()
153    nir_instr_as_alu(neg_a->parent_instr)->exact = alu->exact;  in replace_with_fast()
156    nir_instr_as_alu(b_minus_a->parent_instr)->exact = alu->exact;  in replace_with_fast()
159    nir_instr_as_alu(product->parent_instr)->exact = alu->exact;  in replace_with_fast()
162    nir_instr_as_alu(sum->parent_instr)->exact = alu->exact;  in replace_with_fast()
164    nir_ssa_def_rewrite_uses(&alu->dest.dest.ssa, sum);  in replace_with_fast()
170    append_flrp_to_dead_list(dead_flrp, alu);  in replace_with_fast()
181                                    struct nir_alu_instr *alu, bool subtract_c)  in replace_with_expanded_ffma_and_add()  argument
183    nir_ssa_def *const a = nir_ssa_for_alu_src(bld, alu, 0);  in replace_with_expanded_ffma_and_add()
184    nir_ssa_def *const b = nir_ssa_for_alu_src(bld, alu, 1);  in replace_with_expanded_ffma_and_add()
185    nir_ssa_def *const c = nir_ssa_for_alu_src(bld, alu, 2);  in replace_with_expanded_ffma_and_add()
188    nir_instr_as_alu(b_times_c->parent_instr)->exact = alu->exact;  in replace_with_expanded_ffma_and_add()
194       nir_instr_as_alu(neg_c->parent_instr)->exact = alu->exact;  in replace_with_expanded_ffma_and_add()
201    nir_instr_as_alu(inner_sum->parent_instr)->exact = alu->exact;  in replace_with_expanded_ffma_and_add()
204    nir_instr_as_alu(outer_sum->parent_instr)->exact = alu->exact;  in replace_with_expanded_ffma_and_add()
206    nir_ssa_def_rewrite_uses(&alu->dest.dest.ssa, outer_sum);  in replace_with_expanded_ffma_and_add()
212    append_flrp_to_dead_list(dead_flrp, alu);  in replace_with_expanded_ffma_and_add()
330 get_similar_flrp_stats(nir_alu_instr *alu, struct similar_flrp_stats *st)  in get_similar_flrp_stats()  argument
334    nir_foreach_use(other_use, alu->src[2].src.ssa) {  in get_similar_flrp_stats()
341       if (other_instr == &alu->instr)  in get_similar_flrp_stats()
351       if (!nir_alu_srcs_equal(alu, other_alu, 2, 2))  in get_similar_flrp_stats()
354       if (nir_alu_srcs_equal(alu, other_alu, 0, 0))  in get_similar_flrp_stats()
356       else if (nir_alu_srcs_equal(alu, other_alu, 1, 1))  in get_similar_flrp_stats()
366                          nir_alu_instr *alu,  in convert_flrp_instruction()  argument
370    unsigned bit_size = nir_dest_bit_size(alu->dest.dest);  in convert_flrp_instruction()
381    bld->cursor = nir_before_instr(&alu->instr);  in convert_flrp_instruction()
426    if (alu->exact) {  in convert_flrp_instruction()
428          replace_with_strict_ffma(bld, dead_flrp, alu);  in convert_flrp_instruction()
430          replace_with_strict(bld, dead_flrp, alu);  in convert_flrp_instruction()
445    if (sources_are_constants_with_similar_magnitudes(alu)) {  in convert_flrp_instruction()
446       replace_with_fast(bld, dead_flrp, alu);  in convert_flrp_instruction()
463    if (all_same_constant(alu, 0, &src0_as_constant)) {  in convert_flrp_instruction()
465          replace_with_expanded_ffma_and_add(bld, dead_flrp, alu,  in convert_flrp_instruction()
469          replace_with_expanded_ffma_and_add(bld, dead_flrp, alu,  in convert_flrp_instruction()
495    if ((all_same_constant(alu, 1, &src1_as_constant) &&  in convert_flrp_instruction()
497       replace_with_strict(bld, dead_flrp, alu);  in convert_flrp_instruction()
503          replace_with_strict_ffma(bld, dead_flrp, alu);  in convert_flrp_instruction()
520       get_similar_flrp_stats(alu, &st);  in convert_flrp_instruction()
522          replace_with_strict_ffma(bld, dead_flrp, alu);  in convert_flrp_instruction()
536          replace_with_single_ffma(bld, dead_flrp, alu);  in convert_flrp_instruction()
541          replace_with_strict(bld, dead_flrp, alu);  in convert_flrp_instruction()
564       get_similar_flrp_stats(alu, &st);  in convert_flrp_instruction()
566          replace_with_strict(bld, dead_flrp, alu);  in convert_flrp_instruction()
583    if (alu->src[2].src.ssa->parent_instr->type == nir_instr_type_load_const) {  in convert_flrp_instruction()
584       replace_with_strict(bld, dead_flrp, alu);  in convert_flrp_instruction()
593    replace_with_fast(bld, dead_flrp, alu);  in convert_flrp_instruction()
608             nir_alu_instr *const alu = nir_instr_as_alu(instr);  in lower_flrp_impl()  local
610             if (alu->op == nir_op_flrp &&  in lower_flrp_impl()
611                 (alu->dest.dest.ssa.bit_size & lowering_mask)) {  in lower_flrp_impl()
612                convert_flrp_instruction(&b, dead_flrp, alu, always_precise);  in lower_flrp_impl()