1 /*
2 * Copyright © 2014-2017 Broadcom
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include <inttypes.h>
25 #include "util/format/u_format.h"
26 #include "util/u_math.h"
27 #include "util/u_memory.h"
28 #include "util/ralloc.h"
29 #include "util/hash_table.h"
30 #include "util/u_upload_mgr.h"
31 #include "tgsi/tgsi_dump.h"
32 #include "tgsi/tgsi_parse.h"
33 #include "compiler/nir/nir.h"
34 #include "compiler/nir/nir_builder.h"
35 #include "nir/tgsi_to_nir.h"
36 #include "compiler/v3d_compiler.h"
37 #include "v3d_context.h"
38 #include "broadcom/cle/v3d_packet_v33_pack.h"
39
40 static struct v3d_compiled_shader *
41 v3d_get_compiled_shader(struct v3d_context *v3d,
42 struct v3d_key *key, size_t key_size);
43 static void
44 v3d_setup_shared_precompile_key(struct v3d_uncompiled_shader *uncompiled,
45 struct v3d_key *key);
46
47 static gl_varying_slot
v3d_get_slot_for_driver_location(nir_shader * s,uint32_t driver_location)48 v3d_get_slot_for_driver_location(nir_shader *s, uint32_t driver_location)
49 {
50 nir_foreach_shader_out_variable(var, s) {
51 if (var->data.driver_location == driver_location) {
52 return var->data.location;
53 }
54 }
55
56 return -1;
57 }
58
59 /**
60 * Precomputes the TRANSFORM_FEEDBACK_OUTPUT_DATA_SPEC array for the shader.
61 *
62 * A shader can have 16 of these specs, and each one of them can write up to
63 * 16 dwords. Since we allow a total of 64 transform feedback output
64 * components (not 16 vectors), we have to group the writes of multiple
65 * varyings together in a single data spec.
66 */
67 static void
v3d_set_transform_feedback_outputs(struct v3d_uncompiled_shader * so,const struct pipe_stream_output_info * stream_output)68 v3d_set_transform_feedback_outputs(struct v3d_uncompiled_shader *so,
69 const struct pipe_stream_output_info *stream_output)
70 {
71 if (!stream_output->num_outputs)
72 return;
73
74 struct v3d_varying_slot slots[PIPE_MAX_SO_OUTPUTS * 4];
75 int slot_count = 0;
76
77 for (int buffer = 0; buffer < PIPE_MAX_SO_BUFFERS; buffer++) {
78 uint32_t buffer_offset = 0;
79 uint32_t vpm_start = slot_count;
80
81 for (int i = 0; i < stream_output->num_outputs; i++) {
82 const struct pipe_stream_output *output =
83 &stream_output->output[i];
84
85 if (output->output_buffer != buffer)
86 continue;
87
88 /* We assume that the SO outputs appear in increasing
89 * order in the buffer.
90 */
91 assert(output->dst_offset >= buffer_offset);
92
93 /* Pad any undefined slots in the output */
94 for (int j = buffer_offset; j < output->dst_offset; j++) {
95 slots[slot_count] =
96 v3d_slot_from_slot_and_component(VARYING_SLOT_POS, 0);
97 slot_count++;
98 buffer_offset++;
99 }
100
101 /* Set the coordinate shader up to output the
102 * components of this varying.
103 */
104 for (int j = 0; j < output->num_components; j++) {
105 gl_varying_slot slot =
106 v3d_get_slot_for_driver_location(so->base.ir.nir, output->register_index);
107
108 slots[slot_count] =
109 v3d_slot_from_slot_and_component(slot,
110 output->start_component + j);
111 slot_count++;
112 buffer_offset++;
113 }
114 }
115
116 uint32_t vpm_size = slot_count - vpm_start;
117 if (!vpm_size)
118 continue;
119
120 uint32_t vpm_start_offset = vpm_start + 6;
121
122 while (vpm_size) {
123 uint32_t write_size = MIN2(vpm_size, 1 << 4);
124
125 struct V3D33_TRANSFORM_FEEDBACK_OUTPUT_DATA_SPEC unpacked = {
126 /* We need the offset from the coordinate shader's VPM
127 * output block, which has the [X, Y, Z, W, Xs, Ys]
128 * values at the start.
129 */
130 .first_shaded_vertex_value_to_output = vpm_start_offset,
131 .number_of_consecutive_vertex_values_to_output_as_32_bit_values = write_size,
132 .output_buffer_to_write_to = buffer,
133 };
134
135 /* GFXH-1559 */
136 assert(unpacked.first_shaded_vertex_value_to_output != 8 ||
137 so->num_tf_specs != 0);
138
139 assert(so->num_tf_specs != ARRAY_SIZE(so->tf_specs));
140 V3D33_TRANSFORM_FEEDBACK_OUTPUT_DATA_SPEC_pack(NULL,
141 (void *)&so->tf_specs[so->num_tf_specs],
142 &unpacked);
143
144 /* If point size is being written by the shader, then
145 * all the VPM start offsets are shifted up by one.
146 * We won't know that until the variant is compiled,
147 * though.
148 */
149 unpacked.first_shaded_vertex_value_to_output++;
150
151 /* GFXH-1559 */
152 assert(unpacked.first_shaded_vertex_value_to_output != 8 ||
153 so->num_tf_specs != 0);
154
155 V3D33_TRANSFORM_FEEDBACK_OUTPUT_DATA_SPEC_pack(NULL,
156 (void *)&so->tf_specs_psiz[so->num_tf_specs],
157 &unpacked);
158 so->num_tf_specs++;
159 vpm_start_offset += write_size;
160 vpm_size -= write_size;
161 }
162 so->base.stream_output.stride[buffer] =
163 stream_output->stride[buffer];
164 }
165
166 so->num_tf_outputs = slot_count;
167 so->tf_outputs = ralloc_array(so->base.ir.nir, struct v3d_varying_slot,
168 slot_count);
169 memcpy(so->tf_outputs, slots, sizeof(*slots) * slot_count);
170 }
171
172 static int
type_size(const struct glsl_type * type,bool bindless)173 type_size(const struct glsl_type *type, bool bindless)
174 {
175 return glsl_count_attribute_slots(type, false);
176 }
177
178 static void
precompile_all_outputs(nir_shader * s,struct v3d_varying_slot * outputs,uint8_t * num_outputs)179 precompile_all_outputs(nir_shader *s,
180 struct v3d_varying_slot *outputs,
181 uint8_t *num_outputs)
182 {
183 nir_foreach_shader_out_variable(var, s) {
184 const int array_len = MAX2(glsl_get_length(var->type), 1);
185 for (int j = 0; j < array_len; j++) {
186 const int slot = var->data.location + j;
187 const int num_components =
188 glsl_get_components(var->type);
189 for (int i = 0; i < num_components; i++) {
190 const int swiz = var->data.location_frac + i;
191 outputs[(*num_outputs)++] =
192 v3d_slot_from_slot_and_component(slot,
193 swiz);
194 }
195 }
196 }
197 }
198
199 /**
200 * Precompiles a shader variant at shader state creation time if
201 * V3D_DEBUG=precompile is set. Used for shader-db
202 * (https://gitlab.freedesktop.org/mesa/shader-db)
203 */
204 static void
v3d_shader_precompile(struct v3d_context * v3d,struct v3d_uncompiled_shader * so)205 v3d_shader_precompile(struct v3d_context *v3d,
206 struct v3d_uncompiled_shader *so)
207 {
208 nir_shader *s = so->base.ir.nir;
209
210 if (s->info.stage == MESA_SHADER_FRAGMENT) {
211 struct v3d_fs_key key = {
212 .base.shader_state = so,
213 };
214
215 nir_foreach_shader_out_variable(var, s) {
216 if (var->data.location == FRAG_RESULT_COLOR) {
217 key.cbufs |= 1 << 0;
218 } else if (var->data.location >= FRAG_RESULT_DATA0) {
219 key.cbufs |= 1 << (var->data.location -
220 FRAG_RESULT_DATA0);
221 }
222 }
223
224 key.logicop_func = PIPE_LOGICOP_COPY;
225
226 v3d_setup_shared_precompile_key(so, &key.base);
227 v3d_get_compiled_shader(v3d, &key.base, sizeof(key));
228 } else if (s->info.stage == MESA_SHADER_GEOMETRY) {
229 struct v3d_gs_key key = {
230 .base.shader_state = so,
231 .base.is_last_geometry_stage = true,
232 };
233
234 v3d_setup_shared_precompile_key(so, &key.base);
235
236 precompile_all_outputs(s,
237 key.used_outputs,
238 &key.num_used_outputs);
239
240 v3d_get_compiled_shader(v3d, &key.base, sizeof(key));
241
242 /* Compile GS bin shader: only position (XXX: include TF) */
243 key.is_coord = true;
244 key.num_used_outputs = 0;
245 for (int i = 0; i < 4; i++) {
246 key.used_outputs[key.num_used_outputs++] =
247 v3d_slot_from_slot_and_component(VARYING_SLOT_POS,
248 i);
249 }
250 v3d_get_compiled_shader(v3d, &key.base, sizeof(key));
251 } else {
252 assert(s->info.stage == MESA_SHADER_VERTEX);
253 struct v3d_vs_key key = {
254 .base.shader_state = so,
255 /* Emit fixed function outputs */
256 .base.is_last_geometry_stage = true,
257 };
258
259 v3d_setup_shared_precompile_key(so, &key.base);
260
261 precompile_all_outputs(s,
262 key.used_outputs,
263 &key.num_used_outputs);
264
265 v3d_get_compiled_shader(v3d, &key.base, sizeof(key));
266
267 /* Compile VS bin shader: only position (XXX: include TF) */
268 key.is_coord = true;
269 key.num_used_outputs = 0;
270 for (int i = 0; i < 4; i++) {
271 key.used_outputs[key.num_used_outputs++] =
272 v3d_slot_from_slot_and_component(VARYING_SLOT_POS,
273 i);
274 }
275 v3d_get_compiled_shader(v3d, &key.base, sizeof(key));
276 }
277 }
278
279 static void *
v3d_uncompiled_shader_create(struct pipe_context * pctx,enum pipe_shader_ir type,void * ir)280 v3d_uncompiled_shader_create(struct pipe_context *pctx,
281 enum pipe_shader_ir type, void *ir)
282 {
283 struct v3d_context *v3d = v3d_context(pctx);
284 struct v3d_uncompiled_shader *so = CALLOC_STRUCT(v3d_uncompiled_shader);
285 if (!so)
286 return NULL;
287
288 so->program_id = v3d->next_uncompiled_program_id++;
289
290 nir_shader *s;
291
292 if (type == PIPE_SHADER_IR_NIR) {
293 /* The backend takes ownership of the NIR shader on state
294 * creation.
295 */
296 s = ir;
297 } else {
298 assert(type == PIPE_SHADER_IR_TGSI);
299
300 if (unlikely(V3D_DEBUG & V3D_DEBUG_TGSI)) {
301 fprintf(stderr, "prog %d TGSI:\n",
302 so->program_id);
303 tgsi_dump(ir, 0);
304 fprintf(stderr, "\n");
305 }
306 s = tgsi_to_nir(ir, pctx->screen, false);
307 }
308
309 if (s->info.stage != MESA_SHADER_VERTEX &&
310 s->info.stage != MESA_SHADER_GEOMETRY) {
311 NIR_PASS(_, s, nir_lower_io,
312 nir_var_shader_in | nir_var_shader_out,
313 type_size, (nir_lower_io_options)0);
314 }
315
316 NIR_PASS(_, s, nir_lower_regs_to_ssa);
317 NIR_PASS(_, s, nir_normalize_cubemap_coords);
318
319 NIR_PASS(_, s, nir_lower_load_const_to_scalar);
320
321 v3d_optimize_nir(NULL, s);
322
323 NIR_PASS(_, s, nir_remove_dead_variables, nir_var_function_temp, NULL);
324
325 /* Garbage collect dead instructions */
326 nir_sweep(s);
327
328 so->base.type = PIPE_SHADER_IR_NIR;
329 so->base.ir.nir = s;
330
331 if (unlikely(V3D_DEBUG & (V3D_DEBUG_NIR |
332 v3d_debug_flag_for_shader_stage(s->info.stage)))) {
333 fprintf(stderr, "%s prog %d NIR:\n",
334 gl_shader_stage_name(s->info.stage),
335 so->program_id);
336 nir_print_shader(s, stderr);
337 fprintf(stderr, "\n");
338 }
339
340 if (unlikely(V3D_DEBUG & V3D_DEBUG_PRECOMPILE))
341 v3d_shader_precompile(v3d, so);
342
343 return so;
344 }
345
346 static void
v3d_shader_debug_output(const char * message,void * data)347 v3d_shader_debug_output(const char *message, void *data)
348 {
349 struct v3d_context *v3d = data;
350
351 util_debug_message(&v3d->debug, SHADER_INFO, "%s", message);
352 }
353
354 static void *
v3d_shader_state_create(struct pipe_context * pctx,const struct pipe_shader_state * cso)355 v3d_shader_state_create(struct pipe_context *pctx,
356 const struct pipe_shader_state *cso)
357 {
358 struct v3d_uncompiled_shader *so =
359 v3d_uncompiled_shader_create(pctx,
360 cso->type,
361 (cso->type == PIPE_SHADER_IR_TGSI ?
362 (void *)cso->tokens :
363 cso->ir.nir));
364
365 v3d_set_transform_feedback_outputs(so, &cso->stream_output);
366
367 return so;
368 }
369
370 struct v3d_compiled_shader *
v3d_get_compiled_shader(struct v3d_context * v3d,struct v3d_key * key,size_t key_size)371 v3d_get_compiled_shader(struct v3d_context *v3d,
372 struct v3d_key *key,
373 size_t key_size)
374 {
375 struct v3d_uncompiled_shader *shader_state = key->shader_state;
376 nir_shader *s = shader_state->base.ir.nir;
377
378 struct hash_table *ht = v3d->prog.cache[s->info.stage];
379 struct hash_entry *entry = _mesa_hash_table_search(ht, key);
380 if (entry)
381 return entry->data;
382
383 int variant_id =
384 p_atomic_inc_return(&shader_state->compiled_variant_count);
385
386 struct v3d_compiled_shader *shader = NULL;
387
388 #ifdef ENABLE_SHADER_CACHE
389 shader = v3d_disk_cache_retrieve(v3d, key);
390 #endif
391
392 if (!shader) {
393 shader = rzalloc(NULL, struct v3d_compiled_shader);
394
395 int program_id = shader_state->program_id;
396 uint64_t *qpu_insts;
397 uint32_t shader_size;
398
399 qpu_insts = v3d_compile(v3d->screen->compiler, key,
400 &shader->prog_data.base, s,
401 v3d_shader_debug_output,
402 v3d,
403 program_id, variant_id, &shader_size);
404 ralloc_steal(shader, shader->prog_data.base);
405
406 if (shader_size) {
407 u_upload_data(v3d->state_uploader, 0, shader_size, 8,
408 qpu_insts, &shader->offset, &shader->resource);
409 }
410
411 #ifdef ENABLE_SHADER_CACHE
412 v3d_disk_cache_store(v3d, key, shader, qpu_insts, shader_size);
413 #endif
414
415 free(qpu_insts);
416 }
417
418 v3d_set_shader_uniform_dirty_flags(shader);
419
420 if (ht) {
421 struct v3d_key *dup_key;
422 dup_key = ralloc_size(shader, key_size);
423 memcpy(dup_key, key, key_size);
424 _mesa_hash_table_insert(ht, dup_key, shader);
425 }
426
427 if (shader->prog_data.base->spill_size >
428 v3d->prog.spill_size_per_thread) {
429 /* The TIDX register we use for choosing the area to access
430 * for scratch space is: (core << 6) | (qpu << 2) | thread.
431 * Even at minimum threadcount in a particular shader, that
432 * means we still multiply by qpus by 4.
433 */
434 int total_spill_size = (v3d->screen->devinfo.qpu_count * 4 *
435 shader->prog_data.base->spill_size);
436
437 v3d_bo_unreference(&v3d->prog.spill_bo);
438 v3d->prog.spill_bo = v3d_bo_alloc(v3d->screen,
439 total_spill_size, "spill");
440 v3d->prog.spill_size_per_thread =
441 shader->prog_data.base->spill_size;
442 }
443
444 return shader;
445 }
446
447 static void
v3d_free_compiled_shader(struct v3d_compiled_shader * shader)448 v3d_free_compiled_shader(struct v3d_compiled_shader *shader)
449 {
450 pipe_resource_reference(&shader->resource, NULL);
451 ralloc_free(shader);
452 }
453
454 static void
v3d_setup_shared_key(struct v3d_context * v3d,struct v3d_key * key,struct v3d_texture_stateobj * texstate)455 v3d_setup_shared_key(struct v3d_context *v3d, struct v3d_key *key,
456 struct v3d_texture_stateobj *texstate)
457 {
458 const struct v3d_device_info *devinfo = &v3d->screen->devinfo;
459
460 key->num_tex_used = texstate->num_textures;
461 key->num_samplers_used = texstate->num_textures;
462 assert(key->num_tex_used == key->num_samplers_used);
463 for (int i = 0; i < texstate->num_textures; i++) {
464 struct pipe_sampler_view *sampler = texstate->textures[i];
465 struct v3d_sampler_view *v3d_sampler = v3d_sampler_view(sampler);
466 struct pipe_sampler_state *sampler_state =
467 texstate->samplers[i];
468
469 if (!sampler)
470 continue;
471
472 assert(sampler->target == PIPE_BUFFER || sampler_state);
473
474 unsigned compare_mode = sampler_state ?
475 sampler_state->compare_mode :
476 PIPE_TEX_COMPARE_NONE;
477
478 key->sampler[i].return_size =
479 v3d_get_tex_return_size(devinfo,
480 sampler->format,
481 compare_mode);
482
483 /* For 16-bit, we set up the sampler to always return 2
484 * channels (meaning no recompiles for most statechanges),
485 * while for 32 we actually scale the returns with channels.
486 */
487 if (key->sampler[i].return_size == 16) {
488 key->sampler[i].return_channels = 2;
489 } else if (devinfo->ver > 40) {
490 key->sampler[i].return_channels = 4;
491 } else {
492 key->sampler[i].return_channels =
493 v3d_get_tex_return_channels(devinfo,
494 sampler->format);
495 }
496
497 if (key->sampler[i].return_size == 32 && devinfo->ver < 40) {
498 memcpy(key->tex[i].swizzle,
499 v3d_sampler->swizzle,
500 sizeof(v3d_sampler->swizzle));
501 } else {
502 /* For 16-bit returns, we let the sampler state handle
503 * the swizzle.
504 */
505 key->tex[i].swizzle[0] = PIPE_SWIZZLE_X;
506 key->tex[i].swizzle[1] = PIPE_SWIZZLE_Y;
507 key->tex[i].swizzle[2] = PIPE_SWIZZLE_Z;
508 key->tex[i].swizzle[3] = PIPE_SWIZZLE_W;
509 }
510 }
511 }
512
513 static void
v3d_setup_shared_precompile_key(struct v3d_uncompiled_shader * uncompiled,struct v3d_key * key)514 v3d_setup_shared_precompile_key(struct v3d_uncompiled_shader *uncompiled,
515 struct v3d_key *key)
516 {
517 nir_shader *s = uncompiled->base.ir.nir;
518
519 /* Note that below we access they key's texture and sampler fields
520 * using the same index. On OpenGL they are the same (they are
521 * combined)
522 */
523 key->num_tex_used = s->info.num_textures;
524 key->num_samplers_used = s->info.num_textures;
525 for (int i = 0; i < s->info.num_textures; i++) {
526 key->sampler[i].return_size = 16;
527 key->sampler[i].return_channels = 2;
528
529 key->tex[i].swizzle[0] = PIPE_SWIZZLE_X;
530 key->tex[i].swizzle[1] = PIPE_SWIZZLE_Y;
531 key->tex[i].swizzle[2] = PIPE_SWIZZLE_Z;
532 key->tex[i].swizzle[3] = PIPE_SWIZZLE_W;
533 }
534 }
535
536 static void
v3d_update_compiled_fs(struct v3d_context * v3d,uint8_t prim_mode)537 v3d_update_compiled_fs(struct v3d_context *v3d, uint8_t prim_mode)
538 {
539 struct v3d_job *job = v3d->job;
540 struct v3d_fs_key local_key;
541 struct v3d_fs_key *key = &local_key;
542 nir_shader *s = v3d->prog.bind_fs->base.ir.nir;
543
544 if (!(v3d->dirty & (V3D_DIRTY_PRIM_MODE |
545 V3D_DIRTY_BLEND |
546 V3D_DIRTY_FRAMEBUFFER |
547 V3D_DIRTY_ZSA |
548 V3D_DIRTY_RASTERIZER |
549 V3D_DIRTY_SAMPLE_STATE |
550 V3D_DIRTY_FRAGTEX |
551 V3D_DIRTY_UNCOMPILED_FS))) {
552 return;
553 }
554
555 memset(key, 0, sizeof(*key));
556 v3d_setup_shared_key(v3d, &key->base, &v3d->tex[PIPE_SHADER_FRAGMENT]);
557 key->base.shader_state = v3d->prog.bind_fs;
558 key->base.ucp_enables = v3d->rasterizer->base.clip_plane_enable;
559 key->is_points = (prim_mode == PIPE_PRIM_POINTS);
560 key->is_lines = (prim_mode >= PIPE_PRIM_LINES &&
561 prim_mode <= PIPE_PRIM_LINE_STRIP);
562 key->line_smoothing = (key->is_lines &&
563 v3d_line_smoothing_enabled(v3d));
564 key->has_gs = v3d->prog.bind_gs != NULL;
565 if (v3d->blend->base.logicop_enable) {
566 key->logicop_func = v3d->blend->base.logicop_func;
567 } else {
568 key->logicop_func = PIPE_LOGICOP_COPY;
569 }
570 if (job->msaa) {
571 key->msaa = v3d->rasterizer->base.multisample;
572 key->sample_coverage = (v3d->rasterizer->base.multisample &&
573 v3d->sample_mask != (1 << V3D_MAX_SAMPLES) - 1);
574 key->sample_alpha_to_coverage = v3d->blend->base.alpha_to_coverage;
575 key->sample_alpha_to_one = v3d->blend->base.alpha_to_one;
576 }
577
578 key->swap_color_rb = v3d->swap_color_rb;
579
580 for (int i = 0; i < v3d->framebuffer.nr_cbufs; i++) {
581 struct pipe_surface *cbuf = v3d->framebuffer.cbufs[i];
582 if (!cbuf)
583 continue;
584
585 /* gl_FragColor's propagation to however many bound color
586 * buffers there are means that the shader compile needs to
587 * know what buffers are present.
588 */
589 key->cbufs |= 1 << i;
590
591 /* If logic operations are enabled then we might emit color
592 * reads and we need to know the color buffer format and
593 * swizzle for that.
594 */
595 if (key->logicop_func != PIPE_LOGICOP_COPY) {
596 key->color_fmt[i].format = cbuf->format;
597 memcpy(key->color_fmt[i].swizzle,
598 v3d_get_format_swizzle(&v3d->screen->devinfo,
599 cbuf->format),
600 sizeof(key->color_fmt[i].swizzle));
601 }
602
603 const struct util_format_description *desc =
604 util_format_description(cbuf->format);
605
606 if (desc->channel[0].type == UTIL_FORMAT_TYPE_FLOAT &&
607 desc->channel[0].size == 32) {
608 key->f32_color_rb |= 1 << i;
609 }
610
611 if (s->info.fs.untyped_color_outputs) {
612 if (util_format_is_pure_uint(cbuf->format))
613 key->uint_color_rb |= 1 << i;
614 else if (util_format_is_pure_sint(cbuf->format))
615 key->int_color_rb |= 1 << i;
616 }
617 }
618
619 if (key->is_points) {
620 key->point_sprite_mask =
621 v3d->rasterizer->base.sprite_coord_enable;
622 /* this is handled by lower_wpos_pntc */
623 key->point_coord_upper_left = false;
624 }
625
626 struct v3d_compiled_shader *old_fs = v3d->prog.fs;
627 v3d->prog.fs = v3d_get_compiled_shader(v3d, &key->base, sizeof(*key));
628 if (v3d->prog.fs == old_fs)
629 return;
630
631 v3d->dirty |= V3D_DIRTY_COMPILED_FS;
632
633 if (old_fs) {
634 if (v3d->prog.fs->prog_data.fs->flat_shade_flags !=
635 old_fs->prog_data.fs->flat_shade_flags) {
636 v3d->dirty |= V3D_DIRTY_FLAT_SHADE_FLAGS;
637 }
638
639 if (v3d->prog.fs->prog_data.fs->noperspective_flags !=
640 old_fs->prog_data.fs->noperspective_flags) {
641 v3d->dirty |= V3D_DIRTY_NOPERSPECTIVE_FLAGS;
642 }
643
644 if (v3d->prog.fs->prog_data.fs->centroid_flags !=
645 old_fs->prog_data.fs->centroid_flags) {
646 v3d->dirty |= V3D_DIRTY_CENTROID_FLAGS;
647 }
648 }
649
650 if (old_fs && memcmp(v3d->prog.fs->prog_data.fs->input_slots,
651 old_fs->prog_data.fs->input_slots,
652 sizeof(v3d->prog.fs->prog_data.fs->input_slots))) {
653 v3d->dirty |= V3D_DIRTY_FS_INPUTS;
654 }
655 }
656
657 static void
v3d_update_compiled_gs(struct v3d_context * v3d,uint8_t prim_mode)658 v3d_update_compiled_gs(struct v3d_context *v3d, uint8_t prim_mode)
659 {
660 struct v3d_gs_key local_key;
661 struct v3d_gs_key *key = &local_key;
662
663 if (!(v3d->dirty & (V3D_DIRTY_GEOMTEX |
664 V3D_DIRTY_RASTERIZER |
665 V3D_DIRTY_UNCOMPILED_GS |
666 V3D_DIRTY_PRIM_MODE |
667 V3D_DIRTY_FS_INPUTS))) {
668 return;
669 }
670
671 if (!v3d->prog.bind_gs) {
672 v3d->prog.gs = NULL;
673 v3d->prog.gs_bin = NULL;
674 return;
675 }
676
677 memset(key, 0, sizeof(*key));
678 v3d_setup_shared_key(v3d, &key->base, &v3d->tex[PIPE_SHADER_GEOMETRY]);
679 key->base.shader_state = v3d->prog.bind_gs;
680 key->base.ucp_enables = v3d->rasterizer->base.clip_plane_enable;
681 key->base.is_last_geometry_stage = true;
682 key->num_used_outputs = v3d->prog.fs->prog_data.fs->num_inputs;
683 STATIC_ASSERT(sizeof(key->used_outputs) ==
684 sizeof(v3d->prog.fs->prog_data.fs->input_slots));
685 memcpy(key->used_outputs, v3d->prog.fs->prog_data.fs->input_slots,
686 sizeof(key->used_outputs));
687
688 key->per_vertex_point_size =
689 (prim_mode == PIPE_PRIM_POINTS &&
690 v3d->rasterizer->base.point_size_per_vertex);
691
692 struct v3d_compiled_shader *gs =
693 v3d_get_compiled_shader(v3d, &key->base, sizeof(*key));
694 if (gs != v3d->prog.gs) {
695 v3d->prog.gs = gs;
696 v3d->dirty |= V3D_DIRTY_COMPILED_GS;
697 }
698
699 key->is_coord = true;
700
701 /* The last bin-mode shader in the geometry pipeline only outputs
702 * varyings used by transform feedback.
703 */
704 struct v3d_uncompiled_shader *shader_state = key->base.shader_state;
705 memcpy(key->used_outputs, shader_state->tf_outputs,
706 sizeof(*key->used_outputs) * shader_state->num_tf_outputs);
707 if (shader_state->num_tf_outputs < key->num_used_outputs) {
708 uint32_t size = sizeof(*key->used_outputs) *
709 (key->num_used_outputs -
710 shader_state->num_tf_outputs);
711 memset(&key->used_outputs[shader_state->num_tf_outputs],
712 0, size);
713 }
714 key->num_used_outputs = shader_state->num_tf_outputs;
715
716 struct v3d_compiled_shader *old_gs = v3d->prog.gs;
717 struct v3d_compiled_shader *gs_bin =
718 v3d_get_compiled_shader(v3d, &key->base, sizeof(*key));
719 if (gs_bin != old_gs) {
720 v3d->prog.gs_bin = gs_bin;
721 v3d->dirty |= V3D_DIRTY_COMPILED_GS_BIN;
722 }
723
724 if (old_gs && memcmp(v3d->prog.gs->prog_data.gs->input_slots,
725 old_gs->prog_data.gs->input_slots,
726 sizeof(v3d->prog.gs->prog_data.gs->input_slots))) {
727 v3d->dirty |= V3D_DIRTY_GS_INPUTS;
728 }
729 }
730
731 static void
v3d_update_compiled_vs(struct v3d_context * v3d,uint8_t prim_mode)732 v3d_update_compiled_vs(struct v3d_context *v3d, uint8_t prim_mode)
733 {
734 struct v3d_vs_key local_key;
735 struct v3d_vs_key *key = &local_key;
736
737 if (!(v3d->dirty & (V3D_DIRTY_VERTTEX |
738 V3D_DIRTY_VTXSTATE |
739 V3D_DIRTY_UNCOMPILED_VS |
740 (v3d->prog.bind_gs ? 0 : V3D_DIRTY_RASTERIZER) |
741 (v3d->prog.bind_gs ? 0 : V3D_DIRTY_PRIM_MODE) |
742 (v3d->prog.bind_gs ? V3D_DIRTY_GS_INPUTS :
743 V3D_DIRTY_FS_INPUTS)))) {
744 return;
745 }
746
747 memset(key, 0, sizeof(*key));
748 v3d_setup_shared_key(v3d, &key->base, &v3d->tex[PIPE_SHADER_VERTEX]);
749 key->base.shader_state = v3d->prog.bind_vs;
750 key->base.ucp_enables = v3d->rasterizer->base.clip_plane_enable;
751 key->base.is_last_geometry_stage = !v3d->prog.bind_gs;
752
753 if (!v3d->prog.bind_gs) {
754 key->num_used_outputs = v3d->prog.fs->prog_data.fs->num_inputs;
755 STATIC_ASSERT(sizeof(key->used_outputs) ==
756 sizeof(v3d->prog.fs->prog_data.fs->input_slots));
757 memcpy(key->used_outputs, v3d->prog.fs->prog_data.fs->input_slots,
758 sizeof(key->used_outputs));
759 } else {
760 key->num_used_outputs = v3d->prog.gs->prog_data.gs->num_inputs;
761 STATIC_ASSERT(sizeof(key->used_outputs) ==
762 sizeof(v3d->prog.gs->prog_data.gs->input_slots));
763 memcpy(key->used_outputs, v3d->prog.gs->prog_data.gs->input_slots,
764 sizeof(key->used_outputs));
765 }
766
767 key->per_vertex_point_size =
768 (prim_mode == PIPE_PRIM_POINTS &&
769 v3d->rasterizer->base.point_size_per_vertex);
770
771 nir_shader *s = v3d->prog.bind_vs->base.ir.nir;
772 uint64_t inputs_read = s->info.inputs_read;
773 assert(util_bitcount(inputs_read) <= v3d->vtx->num_elements);
774
775 while (inputs_read) {
776 int location = u_bit_scan64(&inputs_read);
777 nir_variable *var =
778 nir_find_variable_with_location(s, nir_var_shader_in, location);
779 assert (var != NULL);
780 int driver_location = var->data.driver_location;
781 switch (v3d->vtx->pipe[driver_location].src_format) {
782 case PIPE_FORMAT_B8G8R8A8_UNORM:
783 case PIPE_FORMAT_B10G10R10A2_UNORM:
784 case PIPE_FORMAT_B10G10R10A2_SNORM:
785 case PIPE_FORMAT_B10G10R10A2_USCALED:
786 case PIPE_FORMAT_B10G10R10A2_SSCALED:
787 key->va_swap_rb_mask |= 1 << location;
788 break;
789 default:
790 break;
791 }
792 }
793
794 struct v3d_compiled_shader *vs =
795 v3d_get_compiled_shader(v3d, &key->base, sizeof(*key));
796 if (vs != v3d->prog.vs) {
797 v3d->prog.vs = vs;
798 v3d->dirty |= V3D_DIRTY_COMPILED_VS;
799 }
800
801 key->is_coord = true;
802
803 /* Coord shaders only output varyings used by transform feedback,
804 * unless they are linked to other shaders in the geometry side
805 * of the pipeline, since in that case any of the output varyings
806 * could be required in later geometry stages to compute
807 * gl_Position or TF outputs.
808 */
809 if (!v3d->prog.bind_gs) {
810 struct v3d_uncompiled_shader *shader_state =
811 key->base.shader_state;
812 memcpy(key->used_outputs, shader_state->tf_outputs,
813 sizeof(*key->used_outputs) *
814 shader_state->num_tf_outputs);
815 if (shader_state->num_tf_outputs < key->num_used_outputs) {
816 uint32_t tail_bytes =
817 sizeof(*key->used_outputs) *
818 (key->num_used_outputs -
819 shader_state->num_tf_outputs);
820 memset(&key->used_outputs[shader_state->num_tf_outputs],
821 0, tail_bytes);
822 }
823 key->num_used_outputs = shader_state->num_tf_outputs;
824 } else {
825 key->num_used_outputs = v3d->prog.gs_bin->prog_data.gs->num_inputs;
826 STATIC_ASSERT(sizeof(key->used_outputs) ==
827 sizeof(v3d->prog.gs_bin->prog_data.gs->input_slots));
828 memcpy(key->used_outputs, v3d->prog.gs_bin->prog_data.gs->input_slots,
829 sizeof(key->used_outputs));
830 }
831
832 struct v3d_compiled_shader *cs =
833 v3d_get_compiled_shader(v3d, &key->base, sizeof(*key));
834 if (cs != v3d->prog.cs) {
835 v3d->prog.cs = cs;
836 v3d->dirty |= V3D_DIRTY_COMPILED_CS;
837 }
838 }
839
840 void
v3d_update_compiled_shaders(struct v3d_context * v3d,uint8_t prim_mode)841 v3d_update_compiled_shaders(struct v3d_context *v3d, uint8_t prim_mode)
842 {
843 v3d_update_compiled_fs(v3d, prim_mode);
844 v3d_update_compiled_gs(v3d, prim_mode);
845 v3d_update_compiled_vs(v3d, prim_mode);
846 }
847
848 void
v3d_update_compiled_cs(struct v3d_context * v3d)849 v3d_update_compiled_cs(struct v3d_context *v3d)
850 {
851 struct v3d_key local_key;
852 struct v3d_key *key = &local_key;
853
854 if (!(v3d->dirty & (V3D_DIRTY_UNCOMPILED_CS |
855 V3D_DIRTY_COMPTEX))) {
856 return;
857 }
858
859 memset(key, 0, sizeof(*key));
860 v3d_setup_shared_key(v3d, key, &v3d->tex[PIPE_SHADER_COMPUTE]);
861 key->shader_state = v3d->prog.bind_compute;
862
863 struct v3d_compiled_shader *cs =
864 v3d_get_compiled_shader(v3d, key, sizeof(*key));
865 if (cs != v3d->prog.compute) {
866 v3d->prog.compute = cs;
867 v3d->dirty |= V3D_DIRTY_COMPILED_CS; /* XXX */
868 }
869 }
870
871 static uint32_t
fs_cache_hash(const void * key)872 fs_cache_hash(const void *key)
873 {
874 return _mesa_hash_data(key, sizeof(struct v3d_fs_key));
875 }
876
877 static uint32_t
gs_cache_hash(const void * key)878 gs_cache_hash(const void *key)
879 {
880 return _mesa_hash_data(key, sizeof(struct v3d_gs_key));
881 }
882
883 static uint32_t
vs_cache_hash(const void * key)884 vs_cache_hash(const void *key)
885 {
886 return _mesa_hash_data(key, sizeof(struct v3d_vs_key));
887 }
888
889 static uint32_t
cs_cache_hash(const void * key)890 cs_cache_hash(const void *key)
891 {
892 return _mesa_hash_data(key, sizeof(struct v3d_key));
893 }
894
895 static bool
fs_cache_compare(const void * key1,const void * key2)896 fs_cache_compare(const void *key1, const void *key2)
897 {
898 return memcmp(key1, key2, sizeof(struct v3d_fs_key)) == 0;
899 }
900
901 static bool
gs_cache_compare(const void * key1,const void * key2)902 gs_cache_compare(const void *key1, const void *key2)
903 {
904 return memcmp(key1, key2, sizeof(struct v3d_gs_key)) == 0;
905 }
906
907 static bool
vs_cache_compare(const void * key1,const void * key2)908 vs_cache_compare(const void *key1, const void *key2)
909 {
910 return memcmp(key1, key2, sizeof(struct v3d_vs_key)) == 0;
911 }
912
913 static bool
cs_cache_compare(const void * key1,const void * key2)914 cs_cache_compare(const void *key1, const void *key2)
915 {
916 return memcmp(key1, key2, sizeof(struct v3d_key)) == 0;
917 }
918
919 static void
v3d_shader_state_delete(struct pipe_context * pctx,void * hwcso)920 v3d_shader_state_delete(struct pipe_context *pctx, void *hwcso)
921 {
922 struct v3d_context *v3d = v3d_context(pctx);
923 struct v3d_uncompiled_shader *so = hwcso;
924 nir_shader *s = so->base.ir.nir;
925
926 hash_table_foreach(v3d->prog.cache[s->info.stage], entry) {
927 const struct v3d_key *key = entry->key;
928 struct v3d_compiled_shader *shader = entry->data;
929
930 if (key->shader_state != so)
931 continue;
932
933 if (v3d->prog.fs == shader)
934 v3d->prog.fs = NULL;
935 if (v3d->prog.vs == shader)
936 v3d->prog.vs = NULL;
937 if (v3d->prog.cs == shader)
938 v3d->prog.cs = NULL;
939 if (v3d->prog.compute == shader)
940 v3d->prog.compute = NULL;
941
942 _mesa_hash_table_remove(v3d->prog.cache[s->info.stage], entry);
943 v3d_free_compiled_shader(shader);
944 }
945
946 ralloc_free(so->base.ir.nir);
947 free(so);
948 }
949
950 static void
v3d_fp_state_bind(struct pipe_context * pctx,void * hwcso)951 v3d_fp_state_bind(struct pipe_context *pctx, void *hwcso)
952 {
953 struct v3d_context *v3d = v3d_context(pctx);
954 v3d->prog.bind_fs = hwcso;
955 v3d->dirty |= V3D_DIRTY_UNCOMPILED_FS;
956 }
957
958 static void
v3d_gp_state_bind(struct pipe_context * pctx,void * hwcso)959 v3d_gp_state_bind(struct pipe_context *pctx, void *hwcso)
960 {
961 struct v3d_context *v3d = v3d_context(pctx);
962 v3d->prog.bind_gs = hwcso;
963 v3d->dirty |= V3D_DIRTY_UNCOMPILED_GS;
964 }
965
966 static void
v3d_vp_state_bind(struct pipe_context * pctx,void * hwcso)967 v3d_vp_state_bind(struct pipe_context *pctx, void *hwcso)
968 {
969 struct v3d_context *v3d = v3d_context(pctx);
970 v3d->prog.bind_vs = hwcso;
971 v3d->dirty |= V3D_DIRTY_UNCOMPILED_VS;
972 }
973
974 static void
v3d_compute_state_bind(struct pipe_context * pctx,void * state)975 v3d_compute_state_bind(struct pipe_context *pctx, void *state)
976 {
977 struct v3d_context *v3d = v3d_context(pctx);
978
979 v3d->prog.bind_compute = state;
980 v3d->dirty |= V3D_DIRTY_UNCOMPILED_CS;
981 }
982
983 static void *
v3d_create_compute_state(struct pipe_context * pctx,const struct pipe_compute_state * cso)984 v3d_create_compute_state(struct pipe_context *pctx,
985 const struct pipe_compute_state *cso)
986 {
987 return v3d_uncompiled_shader_create(pctx, cso->ir_type,
988 (void *)cso->prog);
989 }
990
991 void
v3d_program_init(struct pipe_context * pctx)992 v3d_program_init(struct pipe_context *pctx)
993 {
994 struct v3d_context *v3d = v3d_context(pctx);
995
996 pctx->create_vs_state = v3d_shader_state_create;
997 pctx->delete_vs_state = v3d_shader_state_delete;
998
999 pctx->create_gs_state = v3d_shader_state_create;
1000 pctx->delete_gs_state = v3d_shader_state_delete;
1001
1002 pctx->create_fs_state = v3d_shader_state_create;
1003 pctx->delete_fs_state = v3d_shader_state_delete;
1004
1005 pctx->bind_fs_state = v3d_fp_state_bind;
1006 pctx->bind_gs_state = v3d_gp_state_bind;
1007 pctx->bind_vs_state = v3d_vp_state_bind;
1008
1009 if (v3d->screen->has_csd) {
1010 pctx->create_compute_state = v3d_create_compute_state;
1011 pctx->delete_compute_state = v3d_shader_state_delete;
1012 pctx->bind_compute_state = v3d_compute_state_bind;
1013 }
1014
1015 v3d->prog.cache[MESA_SHADER_VERTEX] =
1016 _mesa_hash_table_create(pctx, vs_cache_hash, vs_cache_compare);
1017 v3d->prog.cache[MESA_SHADER_GEOMETRY] =
1018 _mesa_hash_table_create(pctx, gs_cache_hash, gs_cache_compare);
1019 v3d->prog.cache[MESA_SHADER_FRAGMENT] =
1020 _mesa_hash_table_create(pctx, fs_cache_hash, fs_cache_compare);
1021 v3d->prog.cache[MESA_SHADER_COMPUTE] =
1022 _mesa_hash_table_create(pctx, cs_cache_hash, cs_cache_compare);
1023 }
1024
1025 void
v3d_program_fini(struct pipe_context * pctx)1026 v3d_program_fini(struct pipe_context *pctx)
1027 {
1028 struct v3d_context *v3d = v3d_context(pctx);
1029
1030 for (int i = 0; i < MESA_SHADER_STAGES; i++) {
1031 struct hash_table *cache = v3d->prog.cache[i];
1032 if (!cache)
1033 continue;
1034
1035 hash_table_foreach(cache, entry) {
1036 struct v3d_compiled_shader *shader = entry->data;
1037 v3d_free_compiled_shader(shader);
1038 _mesa_hash_table_remove(cache, entry);
1039 }
1040 }
1041
1042 v3d_bo_unreference(&v3d->prog.spill_bo);
1043 }
1044