1 /*
2 * Copyright 2023 Alyssa Rosenzweig
3 * Copyright 2021 Intel Corporation
4 * SPDX-License-Identifier: MIT
5 */
6
7 #include "agx_compile.h"
8 #include "agx_tilebuffer.h"
9 #include "nir.h"
10 #include "nir_builder.h"
11
12 static bool
lower_wrapped(nir_builder * b,nir_intrinsic_instr * intr,void * data)13 lower_wrapped(nir_builder *b, nir_intrinsic_instr *intr, void *data)
14 {
15 nir_def *sample_id = data;
16 b->cursor = nir_before_instr(&intr->instr);
17
18 switch (intr->intrinsic) {
19 case nir_intrinsic_load_sample_id: {
20 unsigned size = intr->def.bit_size;
21 nir_def_rewrite_uses(&intr->def, nir_u2uN(b, sample_id, size));
22 nir_instr_remove(&intr->instr);
23 return true;
24 }
25
26 case nir_intrinsic_load_local_pixel_agx:
27 case nir_intrinsic_store_local_pixel_agx:
28 case nir_intrinsic_store_zs_agx:
29 case nir_intrinsic_discard_agx: {
30 /* Fragment I/O inside the loop should only affect one sample. */
31 unsigned mask_index =
32 (intr->intrinsic == nir_intrinsic_store_local_pixel_agx) ? 1 : 0;
33
34 nir_def *mask = intr->src[mask_index].ssa;
35 nir_def *id_mask = nir_ishl(b, nir_imm_intN_t(b, 1, mask->bit_size),
36 nir_u2u32(b, sample_id));
37 nir_src_rewrite(&intr->src[mask_index], nir_iand(b, mask, id_mask));
38 return true;
39 }
40
41 default:
42 return false;
43 }
44 }
45
46 /*
47 * In a monolithic pixel shader, we wrap the fragment shader in a loop over
48 * each sample, and then let optimizations (like loop unrolling) go to town.
49 * This lowering is not compatible with fragment epilogues, which require
50 * something similar at the binary level since the NIR is long gone by then.
51 */
52 static bool
agx_nir_wrap_per_sample_loop(nir_shader * shader,uint8_t nr_samples)53 agx_nir_wrap_per_sample_loop(nir_shader *shader, uint8_t nr_samples)
54 {
55 assert(nr_samples > 1);
56
57 /* Get the original function */
58 nir_function_impl *impl = nir_shader_get_entrypoint(shader);
59
60 nir_cf_list list;
61 nir_cf_extract(&list, nir_before_impl(impl), nir_after_impl(impl));
62
63 /* Create a builder for the wrapped function */
64 nir_builder b = nir_builder_at(nir_after_block(nir_start_block(impl)));
65
66 nir_variable *i =
67 nir_local_variable_create(impl, glsl_uintN_t_type(16), NULL);
68 nir_store_var(&b, i, nir_imm_intN_t(&b, 0, 16), ~0);
69 nir_def *index = NULL;
70
71 /* Create a loop in the wrapped function */
72 nir_loop *loop = nir_push_loop(&b);
73 {
74 index = nir_load_var(&b, i);
75 nir_push_if(&b, nir_uge(&b, index, nir_imm_intN_t(&b, nr_samples, 16)));
76 {
77 nir_jump(&b, nir_jump_break);
78 }
79 nir_pop_if(&b, NULL);
80
81 b.cursor = nir_cf_reinsert(&list, b.cursor);
82 nir_store_var(&b, i, nir_iadd_imm(&b, index, 1), ~0);
83 }
84 nir_pop_loop(&b, loop);
85
86 /* We've mucked about with control flow */
87 nir_metadata_preserve(impl, nir_metadata_none);
88
89 /* Use the loop counter as the sample ID each iteration */
90 nir_shader_intrinsics_pass(shader, lower_wrapped,
91 nir_metadata_block_index | nir_metadata_dominance,
92 index);
93 return true;
94 }
95
96 static bool
lower_sample_mask_write(nir_builder * b,nir_intrinsic_instr * intr,void * data)97 lower_sample_mask_write(nir_builder *b, nir_intrinsic_instr *intr, void *data)
98 {
99 struct agx_msaa_state *state = data;
100 b->cursor = nir_before_instr(&intr->instr);
101
102 if (intr->intrinsic != nir_intrinsic_store_output)
103 return false;
104
105 nir_io_semantics sem = nir_intrinsic_io_semantics(intr);
106 if (sem.location != FRAG_RESULT_SAMPLE_MASK)
107 return false;
108
109 /* Sample mask writes are ignored unless multisampling is used. If it is
110 * used, the Vulkan spec says:
111 *
112 * If sample shading is enabled, bits written to SampleMask
113 * corresponding to samples that are not being shaded by the fragment
114 * shader invocation are ignored.
115 *
116 * That will be satisfied by outputting gl_SampleMask for the whole pixel
117 * and then lowering sample shading after (splitting up discard targets).
118 */
119 if (state->nr_samples != 1) {
120 nir_discard_agx(b, nir_inot(b, nir_u2u16(b, intr->src[0].ssa)));
121 b->shader->info.fs.uses_discard = true;
122 }
123
124 nir_instr_remove(&intr->instr);
125 return true;
126 }
127
128 /*
129 * Apply API sample mask to sample mask inputs, lowering:
130 *
131 * sample_mask_in --> sample_mask_in & api_sample_mask
132 */
133 static bool
lower_sample_mask_read(nir_builder * b,nir_intrinsic_instr * intr,UNUSED void * _)134 lower_sample_mask_read(nir_builder *b, nir_intrinsic_instr *intr,
135 UNUSED void *_)
136 {
137 b->cursor = nir_after_instr(&intr->instr);
138
139 if (intr->intrinsic != nir_intrinsic_load_sample_mask_in)
140 return false;
141
142 nir_def *old = &intr->def;
143 nir_def *lowered = nir_iand(
144 b, old, nir_u2uN(b, nir_load_api_sample_mask_agx(b), old->bit_size));
145
146 nir_def_rewrite_uses_after(old, lowered, lowered->parent_instr);
147 return true;
148 }
149
150 /* glSampleMask(x) --> gl_SampleMask = x */
151 static void
insert_sample_mask_write(nir_shader * s)152 insert_sample_mask_write(nir_shader *s)
153 {
154 nir_builder b;
155 nir_function_impl *impl = nir_shader_get_entrypoint(s);
156 b = nir_builder_at(nir_before_impl(impl));
157
158 /* Kill samples that are NOT covered by the mask */
159 nir_discard_agx(&b, nir_inot(&b, nir_load_api_sample_mask_agx(&b)));
160 s->info.fs.uses_discard = true;
161 }
162
163 /*
164 * Lower a fragment shader into a monolithic pixel shader, with static sample
165 * count, blend state, and tilebuffer formats in the shader key. For dynamic,
166 * epilogs must be used, which have separate lowerings.
167 */
168 bool
agx_nir_lower_monolithic_msaa(nir_shader * shader,struct agx_msaa_state * state)169 agx_nir_lower_monolithic_msaa(nir_shader *shader, struct agx_msaa_state *state)
170 {
171 assert(shader->info.stage == MESA_SHADER_FRAGMENT);
172 assert(state->nr_samples == 1 || state->nr_samples == 2 ||
173 state->nr_samples == 4);
174
175 /* Lower gl_SampleMask writes */
176 if (shader->info.outputs_written & BITFIELD64_BIT(FRAG_RESULT_SAMPLE_MASK)) {
177 nir_shader_intrinsics_pass(
178 shader, lower_sample_mask_write,
179 nir_metadata_block_index | nir_metadata_dominance, state);
180 }
181
182 /* Lower API sample masks */
183 if ((state->nr_samples > 1) && state->api_sample_mask)
184 insert_sample_mask_write(shader);
185
186 /* Additional, sample_mask_in needs to account for the API-level mask */
187 nir_shader_intrinsics_pass(shader, lower_sample_mask_read,
188 nir_metadata_block_index | nir_metadata_dominance,
189 &state->nr_samples);
190
191 agx_nir_lower_sample_mask(shader);
192
193 /* In single sampled programs, interpolateAtSample needs to return the
194 * center pixel. TODO: Generalize for dynamic sample count.
195 */
196 if (state->nr_samples == 1)
197 nir_lower_single_sampled(shader);
198 else if (shader->info.fs.uses_sample_shading)
199 agx_nir_wrap_per_sample_loop(shader, state->nr_samples);
200
201 return true;
202 }
203