1 /*
2 * Copyright (c) 2017-2019 Lima Project
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sub license,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the
12 * next paragraph) shall be included in all copies or substantial portions
13 * of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 *
23 */
24
25 #include "util/u_memory.h"
26 #include "util/ralloc.h"
27 #include "util/u_debug.h"
28
29 #include "tgsi/tgsi_dump.h"
30 #include "compiler/nir/nir.h"
31 #include "compiler/nir/nir_serialize.h"
32 #include "nir/tgsi_to_nir.h"
33 #include "nir_legacy.h"
34
35 #include "pipe/p_state.h"
36
37 #include "lima_screen.h"
38 #include "lima_context.h"
39 #include "lima_job.h"
40 #include "lima_program.h"
41 #include "lima_bo.h"
42 #include "lima_disk_cache.h"
43
44 #include "ir/lima_ir.h"
45
46 static const nir_shader_compiler_options vs_nir_options = {
47 .lower_ffma16 = true,
48 .lower_ffma32 = true,
49 .lower_ffma64 = true,
50 .lower_fpow = true,
51 .lower_ffract = true,
52 .lower_fdiv = true,
53 .lower_fmod = true,
54 .lower_fsqrt = true,
55 .lower_flrp32 = true,
56 .lower_flrp64 = true,
57 /* could be implemented by clamp */
58 .lower_fsat = true,
59 .lower_bitops = true,
60 .lower_sincos = true,
61 .lower_fceil = true,
62 .lower_insert_byte = true,
63 .lower_insert_word = true,
64 .force_indirect_unrolling = nir_var_all,
65 .force_indirect_unrolling_sampler = true,
66 .lower_varying_from_uniform = true,
67 .max_unroll_iterations = 32,
68 };
69
70 static const nir_shader_compiler_options fs_nir_options = {
71 .lower_ffma16 = true,
72 .lower_ffma32 = true,
73 .lower_ffma64 = true,
74 .lower_fpow = true,
75 .lower_fdiv = true,
76 .lower_fmod = true,
77 .lower_flrp32 = true,
78 .lower_flrp64 = true,
79 .lower_fsign = true,
80 .lower_fdot = true,
81 .lower_fdph = true,
82 .lower_insert_byte = true,
83 .lower_insert_word = true,
84 .lower_bitops = true,
85 .lower_vector_cmp = true,
86 .force_indirect_unrolling = (nir_var_shader_out | nir_var_function_temp),
87 .force_indirect_unrolling_sampler = true,
88 .lower_varying_from_uniform = true,
89 .max_unroll_iterations = 32,
90 };
91
92 const void *
lima_program_get_compiler_options(enum pipe_shader_type shader)93 lima_program_get_compiler_options(enum pipe_shader_type shader)
94 {
95 switch (shader) {
96 case PIPE_SHADER_VERTEX:
97 return &vs_nir_options;
98 case PIPE_SHADER_FRAGMENT:
99 return &fs_nir_options;
100 default:
101 return NULL;
102 }
103 }
104
105 static int
type_size(const struct glsl_type * type,bool bindless)106 type_size(const struct glsl_type *type, bool bindless)
107 {
108 return glsl_count_attribute_slots(type, false);
109 }
110
111 void
lima_program_optimize_vs_nir(struct nir_shader * s)112 lima_program_optimize_vs_nir(struct nir_shader *s)
113 {
114 bool progress;
115
116 NIR_PASS_V(s, nir_lower_viewport_transform);
117 NIR_PASS_V(s, nir_lower_point_size, 1.0f, 100.0f);
118 NIR_PASS_V(s, nir_lower_io,
119 nir_var_shader_in | nir_var_shader_out, type_size, 0);
120 NIR_PASS_V(s, nir_lower_load_const_to_scalar);
121 NIR_PASS_V(s, lima_nir_lower_uniform_to_scalar);
122 NIR_PASS_V(s, nir_lower_io_to_scalar,
123 nir_var_shader_in|nir_var_shader_out, NULL, NULL);
124
125 do {
126 progress = false;
127
128 NIR_PASS_V(s, nir_lower_vars_to_ssa);
129 NIR_PASS(progress, s, nir_lower_alu_to_scalar, NULL, NULL);
130 NIR_PASS(progress, s, nir_lower_phis_to_scalar, false);
131 NIR_PASS(progress, s, nir_copy_prop);
132 NIR_PASS(progress, s, nir_opt_remove_phis);
133 NIR_PASS(progress, s, nir_opt_dce);
134 NIR_PASS(progress, s, nir_opt_dead_cf);
135 NIR_PASS(progress, s, nir_opt_cse);
136 NIR_PASS(progress, s, nir_opt_peephole_select, 8, true, true);
137 NIR_PASS(progress, s, nir_opt_algebraic);
138 NIR_PASS(progress, s, lima_nir_lower_ftrunc);
139 NIR_PASS(progress, s, nir_opt_constant_folding);
140 NIR_PASS(progress, s, nir_opt_undef);
141 NIR_PASS(progress, s, nir_lower_undef_to_zero);
142 NIR_PASS(progress, s, nir_opt_loop_unroll);
143 NIR_PASS(progress, s, nir_lower_undef_to_zero);
144 } while (progress);
145
146 NIR_PASS_V(s, nir_lower_int_to_float);
147 /* int_to_float pass generates ftrunc, so lower it */
148 NIR_PASS(progress, s, lima_nir_lower_ftrunc);
149 NIR_PASS_V(s, nir_lower_bool_to_float, true);
150
151 NIR_PASS_V(s, nir_copy_prop);
152 NIR_PASS_V(s, nir_opt_dce);
153 NIR_PASS_V(s, lima_nir_split_loads);
154 NIR_PASS_V(s, nir_convert_from_ssa, true);
155 NIR_PASS_V(s, nir_opt_dce);
156 NIR_PASS_V(s, nir_remove_dead_variables, nir_var_function_temp, NULL);
157 nir_sweep(s);
158 }
159
160 static bool
lima_alu_to_scalar_filter_cb(const nir_instr * instr,const void * data)161 lima_alu_to_scalar_filter_cb(const nir_instr *instr, const void *data)
162 {
163 if (instr->type != nir_instr_type_alu)
164 return false;
165
166 nir_alu_instr *alu = nir_instr_as_alu(instr);
167 switch (alu->op) {
168 case nir_op_frcp:
169 /* nir_op_idiv is lowered to frcp by lower_int_to_floats which
170 * will be run later, so lower idiv here
171 */
172 case nir_op_idiv:
173 case nir_op_frsq:
174 case nir_op_flog2:
175 case nir_op_fexp2:
176 case nir_op_fsqrt:
177 case nir_op_fsin:
178 case nir_op_fcos:
179 return true;
180 default:
181 break;
182 }
183
184 /* nir vec4 fcsel assumes that each component of the condition will be
185 * used to select the same component from the two options, but Utgard PP
186 * has only 1 component condition. If all condition components are not the
187 * same we need to lower it to scalar.
188 */
189 switch (alu->op) {
190 case nir_op_bcsel:
191 case nir_op_fcsel:
192 break;
193 default:
194 return false;
195 }
196
197 int num_components = alu->def.num_components;
198
199 uint8_t swizzle = alu->src[0].swizzle[0];
200
201 for (int i = 1; i < num_components; i++)
202 if (alu->src[0].swizzle[i] != swizzle)
203 return true;
204
205 return false;
206 }
207
208 static bool
lima_vec_to_regs_filter_cb(const nir_instr * instr,unsigned writemask,const void * data)209 lima_vec_to_regs_filter_cb(const nir_instr *instr, unsigned writemask,
210 const void *data)
211 {
212 assert(writemask > 0);
213 if (util_bitcount(writemask) == 1)
214 return true;
215
216 return !lima_alu_to_scalar_filter_cb(instr, data);
217 }
218
219 void
lima_program_optimize_fs_nir(struct nir_shader * s,struct nir_lower_tex_options * tex_options)220 lima_program_optimize_fs_nir(struct nir_shader *s,
221 struct nir_lower_tex_options *tex_options)
222 {
223 bool progress;
224
225 NIR_PASS_V(s, nir_lower_fragcoord_wtrans);
226 NIR_PASS_V(s, nir_lower_io,
227 nir_var_shader_in | nir_var_shader_out, type_size, 0);
228 NIR_PASS_V(s, nir_lower_tex, tex_options);
229 NIR_PASS_V(s, lima_nir_lower_txp);
230
231 do {
232 progress = false;
233 NIR_PASS(progress, s, nir_opt_vectorize, NULL, NULL);
234 } while (progress);
235
236 do {
237 progress = false;
238
239 NIR_PASS_V(s, nir_lower_vars_to_ssa);
240 NIR_PASS(progress, s, nir_lower_alu_to_scalar, lima_alu_to_scalar_filter_cb, NULL);
241 NIR_PASS(progress, s, nir_copy_prop);
242 NIR_PASS(progress, s, nir_opt_remove_phis);
243 NIR_PASS(progress, s, nir_opt_dce);
244 NIR_PASS(progress, s, nir_opt_dead_cf);
245 NIR_PASS(progress, s, nir_opt_cse);
246 NIR_PASS(progress, s, nir_opt_peephole_select, 8, true, true);
247 NIR_PASS(progress, s, nir_opt_algebraic);
248 NIR_PASS(progress, s, nir_opt_constant_folding);
249 NIR_PASS(progress, s, nir_opt_undef);
250 NIR_PASS(progress, s, nir_opt_loop_unroll);
251 NIR_PASS(progress, s, lima_nir_split_load_input);
252 } while (progress);
253
254 NIR_PASS_V(s, nir_lower_int_to_float);
255 NIR_PASS_V(s, nir_lower_bool_to_float, true);
256
257 /* Some ops must be lowered after being converted from int ops,
258 * so re-run nir_opt_algebraic after int lowering. */
259 do {
260 progress = false;
261 NIR_PASS(progress, s, nir_opt_algebraic);
262 } while (progress);
263
264 /* Must be run after optimization loop */
265 NIR_PASS_V(s, lima_nir_scale_trig);
266
267 NIR_PASS_V(s, nir_copy_prop);
268 NIR_PASS_V(s, nir_opt_dce);
269
270 NIR_PASS_V(s, nir_convert_from_ssa, true);
271 NIR_PASS_V(s, nir_remove_dead_variables, nir_var_function_temp, NULL);
272
273 NIR_PASS_V(s, nir_move_vec_src_uses_to_dest, false);
274 NIR_PASS_V(s, nir_lower_vec_to_regs, lima_vec_to_regs_filter_cb, NULL);
275
276 NIR_PASS_V(s, nir_opt_dce); /* clean up any new dead code from vec to movs */
277
278 NIR_PASS_V(s, lima_nir_duplicate_load_uniforms);
279 NIR_PASS_V(s, lima_nir_duplicate_load_inputs);
280 NIR_PASS_V(s, lima_nir_duplicate_load_consts);
281
282 NIR_PASS_V(s, nir_legacy_trivialize, true);
283
284 nir_sweep(s);
285 }
286
287 static bool
lima_fs_compile_shader(struct lima_context * ctx,struct lima_fs_key * key,struct lima_fs_uncompiled_shader * ufs,struct lima_fs_compiled_shader * fs)288 lima_fs_compile_shader(struct lima_context *ctx,
289 struct lima_fs_key *key,
290 struct lima_fs_uncompiled_shader *ufs,
291 struct lima_fs_compiled_shader *fs)
292 {
293 struct lima_screen *screen = lima_screen(ctx->base.screen);
294 nir_shader *nir = nir_shader_clone(fs, ufs->base.ir.nir);
295
296 struct nir_lower_tex_options tex_options = {
297 .swizzle_result = ~0u,
298 .lower_invalid_implicit_lod = true,
299 };
300
301 for (int i = 0; i < ARRAY_SIZE(key->tex); i++) {
302 for (int j = 0; j < 4; j++)
303 tex_options.swizzles[i][j] = key->tex[i].swizzle[j];
304 }
305
306 lima_program_optimize_fs_nir(nir, &tex_options);
307
308 if (lima_debug & LIMA_DEBUG_PP)
309 nir_print_shader(nir, stdout);
310
311 if (!ppir_compile_nir(fs, nir, screen->pp_ra, &ctx->base.debug)) {
312 ralloc_free(nir);
313 return false;
314 }
315
316 fs->state.uses_discard = nir->info.fs.uses_discard;
317 ralloc_free(nir);
318
319 return true;
320 }
321
322 static bool
lima_fs_upload_shader(struct lima_context * ctx,struct lima_fs_compiled_shader * fs)323 lima_fs_upload_shader(struct lima_context *ctx,
324 struct lima_fs_compiled_shader *fs)
325 {
326 struct lima_screen *screen = lima_screen(ctx->base.screen);
327
328 fs->bo = lima_bo_create(screen, fs->state.shader_size, 0);
329 if (!fs->bo) {
330 fprintf(stderr, "lima: create fs shader bo fail\n");
331 return false;
332 }
333
334 memcpy(lima_bo_map(fs->bo), fs->shader, fs->state.shader_size);
335
336 return true;
337 }
338
339 static struct lima_fs_compiled_shader *
lima_get_compiled_fs(struct lima_context * ctx,struct lima_fs_uncompiled_shader * ufs,struct lima_fs_key * key)340 lima_get_compiled_fs(struct lima_context *ctx,
341 struct lima_fs_uncompiled_shader *ufs,
342 struct lima_fs_key *key)
343 {
344 struct lima_screen *screen = lima_screen(ctx->base.screen);
345 struct hash_table *ht;
346 uint32_t key_size;
347
348 ht = ctx->fs_cache;
349 key_size = sizeof(struct lima_fs_key);
350
351 struct hash_entry *entry = _mesa_hash_table_search(ht, key);
352 if (entry)
353 return entry->data;
354
355 /* Not on memory cache, try disk cache */
356 struct lima_fs_compiled_shader *fs =
357 lima_fs_disk_cache_retrieve(screen->disk_cache, key);
358
359 if (!fs) {
360 /* Not on disk cache, compile and insert into disk cache*/
361 fs = rzalloc(NULL, struct lima_fs_compiled_shader);
362 if (!fs)
363 return NULL;
364
365 if (!lima_fs_compile_shader(ctx, key, ufs, fs))
366 goto err;
367
368 lima_fs_disk_cache_store(screen->disk_cache, key, fs);
369 }
370
371 if (!lima_fs_upload_shader(ctx, fs))
372 goto err;
373
374 ralloc_free(fs->shader);
375 fs->shader = NULL;
376
377 /* Insert into memory cache */
378 struct lima_key *dup_key;
379 dup_key = rzalloc_size(fs, key_size);
380 memcpy(dup_key, key, key_size);
381 _mesa_hash_table_insert(ht, dup_key, fs);
382
383 return fs;
384
385 err:
386 ralloc_free(fs);
387 return NULL;
388 }
389
390 static void *
lima_create_fs_state(struct pipe_context * pctx,const struct pipe_shader_state * cso)391 lima_create_fs_state(struct pipe_context *pctx,
392 const struct pipe_shader_state *cso)
393 {
394 struct lima_context *ctx = lima_context(pctx);
395 struct lima_fs_uncompiled_shader *so = rzalloc(NULL, struct lima_fs_uncompiled_shader);
396
397 if (!so)
398 return NULL;
399
400 nir_shader *nir;
401 if (cso->type == PIPE_SHADER_IR_NIR)
402 /* The backend takes ownership of the NIR shader on state
403 * creation. */
404 nir = cso->ir.nir;
405 else {
406 assert(cso->type == PIPE_SHADER_IR_TGSI);
407
408 nir = tgsi_to_nir(cso->tokens, pctx->screen, false);
409 }
410
411 so->base.type = PIPE_SHADER_IR_NIR;
412 so->base.ir.nir = nir;
413
414 /* Serialize the NIR to a binary blob that we can hash for the disk
415 * cache. Drop unnecessary information (like variable names)
416 * so the serialized NIR is smaller, and also to let us detect more
417 * isomorphic shaders when hashing, increasing cache hits.
418 */
419 struct blob blob;
420 blob_init(&blob);
421 nir_serialize(&blob, nir, true);
422 _mesa_sha1_compute(blob.data, blob.size, so->nir_sha1);
423 blob_finish(&blob);
424
425 if (lima_debug & LIMA_DEBUG_PRECOMPILE) {
426 /* Trigger initial compilation with default settings */
427 struct lima_fs_key key;
428 memset(&key, 0, sizeof(key));
429 memcpy(key.nir_sha1, so->nir_sha1, sizeof(so->nir_sha1));
430 for (int i = 0; i < ARRAY_SIZE(key.tex); i++) {
431 for (int j = 0; j < 4; j++)
432 key.tex[i].swizzle[j] = j;
433 }
434 lima_get_compiled_fs(ctx, so, &key);
435 }
436
437 return so;
438 }
439
440 static void
lima_bind_fs_state(struct pipe_context * pctx,void * hwcso)441 lima_bind_fs_state(struct pipe_context *pctx, void *hwcso)
442 {
443 struct lima_context *ctx = lima_context(pctx);
444
445 ctx->uncomp_fs = hwcso;
446 ctx->dirty |= LIMA_CONTEXT_DIRTY_UNCOMPILED_FS;
447 }
448
449 static void
lima_delete_fs_state(struct pipe_context * pctx,void * hwcso)450 lima_delete_fs_state(struct pipe_context *pctx, void *hwcso)
451 {
452 struct lima_context *ctx = lima_context(pctx);
453 struct lima_fs_uncompiled_shader *so = hwcso;
454
455 hash_table_foreach(ctx->fs_cache, entry) {
456 const struct lima_fs_key *key = entry->key;
457 if (!memcmp(key->nir_sha1, so->nir_sha1, sizeof(so->nir_sha1))) {
458 struct lima_fs_compiled_shader *fs = entry->data;
459 _mesa_hash_table_remove(ctx->fs_cache, entry);
460 if (fs->bo)
461 lima_bo_unreference(fs->bo);
462
463 if (fs == ctx->fs)
464 ctx->fs = NULL;
465
466 ralloc_free(fs);
467 }
468 }
469
470 ralloc_free(so->base.ir.nir);
471 ralloc_free(so);
472 }
473
474 static bool
lima_vs_compile_shader(struct lima_context * ctx,struct lima_vs_key * key,struct lima_vs_uncompiled_shader * uvs,struct lima_vs_compiled_shader * vs)475 lima_vs_compile_shader(struct lima_context *ctx,
476 struct lima_vs_key *key,
477 struct lima_vs_uncompiled_shader *uvs,
478 struct lima_vs_compiled_shader *vs)
479 {
480 nir_shader *nir = nir_shader_clone(vs, uvs->base.ir.nir);
481
482 lima_program_optimize_vs_nir(nir);
483
484 if (lima_debug & LIMA_DEBUG_GP)
485 nir_print_shader(nir, stdout);
486
487 if (!gpir_compile_nir(vs, nir, &ctx->base.debug)) {
488 ralloc_free(nir);
489 return false;
490 }
491
492 ralloc_free(nir);
493
494 return true;
495 }
496
497 static bool
lima_vs_upload_shader(struct lima_context * ctx,struct lima_vs_compiled_shader * vs)498 lima_vs_upload_shader(struct lima_context *ctx,
499 struct lima_vs_compiled_shader *vs)
500 {
501 struct lima_screen *screen = lima_screen(ctx->base.screen);
502 vs->bo = lima_bo_create(screen, vs->state.shader_size, 0);
503 if (!vs->bo) {
504 fprintf(stderr, "lima: create vs shader bo fail\n");
505 return false;
506 }
507
508 memcpy(lima_bo_map(vs->bo), vs->shader, vs->state.shader_size);
509
510 return true;
511 }
512
513 static struct lima_vs_compiled_shader *
lima_get_compiled_vs(struct lima_context * ctx,struct lima_vs_uncompiled_shader * uvs,struct lima_vs_key * key)514 lima_get_compiled_vs(struct lima_context *ctx,
515 struct lima_vs_uncompiled_shader *uvs,
516 struct lima_vs_key *key)
517 {
518 struct lima_screen *screen = lima_screen(ctx->base.screen);
519 struct hash_table *ht;
520 uint32_t key_size;
521
522 ht = ctx->vs_cache;
523 key_size = sizeof(struct lima_vs_key);
524
525 struct hash_entry *entry = _mesa_hash_table_search(ht, key);
526 if (entry)
527 return entry->data;
528
529 /* Not on memory cache, try disk cache */
530 struct lima_vs_compiled_shader *vs =
531 lima_vs_disk_cache_retrieve(screen->disk_cache, key);
532
533 if (!vs) {
534 /* Not on disk cache, compile and insert into disk cache */
535 vs = rzalloc(NULL, struct lima_vs_compiled_shader);
536 if (!vs)
537 return NULL;
538 if (!lima_vs_compile_shader(ctx, key, uvs, vs))
539 goto err;
540
541 lima_vs_disk_cache_store(screen->disk_cache, key, vs);
542 }
543
544 if (!lima_vs_upload_shader(ctx, vs))
545 goto err;
546
547 ralloc_free(vs->shader);
548 vs->shader = NULL;
549
550 struct lima_key *dup_key;
551 dup_key = rzalloc_size(vs, key_size);
552 memcpy(dup_key, key, key_size);
553 _mesa_hash_table_insert(ht, dup_key, vs);
554
555 return vs;
556
557 err:
558 ralloc_free(vs);
559 return NULL;
560 }
561
562 bool
lima_update_vs_state(struct lima_context * ctx)563 lima_update_vs_state(struct lima_context *ctx)
564 {
565 if (!(ctx->dirty & LIMA_CONTEXT_DIRTY_UNCOMPILED_VS)) {
566 return true;
567 }
568
569 struct lima_vs_key local_key;
570 struct lima_vs_key *key = &local_key;
571 memset(key, 0, sizeof(*key));
572 memcpy(key->nir_sha1, ctx->uncomp_vs->nir_sha1,
573 sizeof(ctx->uncomp_vs->nir_sha1));
574
575 struct lima_vs_compiled_shader *old_vs = ctx->vs;
576 struct lima_vs_compiled_shader *vs = lima_get_compiled_vs(ctx,
577 ctx->uncomp_vs,
578 key);
579 if (!vs)
580 return false;
581
582 ctx->vs = vs;
583
584 if (ctx->vs != old_vs)
585 ctx->dirty |= LIMA_CONTEXT_DIRTY_COMPILED_VS;
586
587 return true;
588 }
589
590 bool
lima_update_fs_state(struct lima_context * ctx)591 lima_update_fs_state(struct lima_context *ctx)
592 {
593 if (!(ctx->dirty & (LIMA_CONTEXT_DIRTY_UNCOMPILED_FS |
594 LIMA_CONTEXT_DIRTY_TEXTURES))) {
595 return true;
596 }
597
598 struct lima_texture_stateobj *lima_tex = &ctx->tex_stateobj;
599 struct lima_fs_key local_key;
600 struct lima_fs_key *key = &local_key;
601 memset(key, 0, sizeof(*key));
602 memcpy(key->nir_sha1, ctx->uncomp_fs->nir_sha1,
603 sizeof(ctx->uncomp_fs->nir_sha1));
604
605 uint8_t identity[4] = { PIPE_SWIZZLE_X, PIPE_SWIZZLE_Y,
606 PIPE_SWIZZLE_Z, PIPE_SWIZZLE_W };
607 for (int i = 0; i < lima_tex->num_textures; i++) {
608 struct lima_sampler_view *sampler = lima_sampler_view(lima_tex->textures[i]);
609 if (!sampler) {
610 memcpy(key->tex[i].swizzle, identity, 4);
611 continue;
612 }
613 for (int j = 0; j < 4; j++)
614 key->tex[i].swizzle[j] = sampler->swizzle[j];
615 }
616
617 /* Fill rest with identity swizzle */
618 for (int i = lima_tex->num_textures; i < ARRAY_SIZE(key->tex); i++)
619 memcpy(key->tex[i].swizzle, identity, 4);
620
621 struct lima_fs_compiled_shader *old_fs = ctx->fs;
622
623 struct lima_fs_compiled_shader *fs = lima_get_compiled_fs(ctx,
624 ctx->uncomp_fs,
625 key);
626 if (!fs)
627 return false;
628
629 ctx->fs = fs;
630
631 if (ctx->fs != old_fs)
632 ctx->dirty |= LIMA_CONTEXT_DIRTY_COMPILED_FS;
633
634 return true;
635 }
636
637 static void *
lima_create_vs_state(struct pipe_context * pctx,const struct pipe_shader_state * cso)638 lima_create_vs_state(struct pipe_context *pctx,
639 const struct pipe_shader_state *cso)
640 {
641 struct lima_context *ctx = lima_context(pctx);
642 struct lima_vs_uncompiled_shader *so = rzalloc(NULL, struct lima_vs_uncompiled_shader);
643
644 if (!so)
645 return NULL;
646
647 nir_shader *nir;
648 if (cso->type == PIPE_SHADER_IR_NIR)
649 /* The backend takes ownership of the NIR shader on state
650 * creation. */
651 nir = cso->ir.nir;
652 else {
653 assert(cso->type == PIPE_SHADER_IR_TGSI);
654
655 nir = tgsi_to_nir(cso->tokens, pctx->screen, false);
656 }
657
658 so->base.type = PIPE_SHADER_IR_NIR;
659 so->base.ir.nir = nir;
660
661 /* Serialize the NIR to a binary blob that we can hash for the disk
662 * cache. Drop unnecessary information (like variable names)
663 * so the serialized NIR is smaller, and also to let us detect more
664 * isomorphic shaders when hashing, increasing cache hits.
665 */
666 struct blob blob;
667 blob_init(&blob);
668 nir_serialize(&blob, nir, true);
669 _mesa_sha1_compute(blob.data, blob.size, so->nir_sha1);
670 blob_finish(&blob);
671
672 if (lima_debug & LIMA_DEBUG_PRECOMPILE) {
673 /* Trigger initial compilation with default settings */
674 struct lima_vs_key key;
675 memset(&key, 0, sizeof(key));
676 memcpy(key.nir_sha1, so->nir_sha1, sizeof(so->nir_sha1));
677 lima_get_compiled_vs(ctx, so, &key);
678 }
679
680 return so;
681 }
682
683 static void
lima_bind_vs_state(struct pipe_context * pctx,void * hwcso)684 lima_bind_vs_state(struct pipe_context *pctx, void *hwcso)
685 {
686 struct lima_context *ctx = lima_context(pctx);
687
688 ctx->uncomp_vs = hwcso;
689 ctx->dirty |= LIMA_CONTEXT_DIRTY_UNCOMPILED_VS;
690 }
691
692 static void
lima_delete_vs_state(struct pipe_context * pctx,void * hwcso)693 lima_delete_vs_state(struct pipe_context *pctx, void *hwcso)
694 {
695 struct lima_context *ctx = lima_context(pctx);
696 struct lima_vs_uncompiled_shader *so = hwcso;
697
698 hash_table_foreach(ctx->vs_cache, entry) {
699 const struct lima_vs_key *key = entry->key;
700 if (!memcmp(key->nir_sha1, so->nir_sha1, sizeof(so->nir_sha1))) {
701 struct lima_vs_compiled_shader *vs = entry->data;
702 _mesa_hash_table_remove(ctx->vs_cache, entry);
703 if (vs->bo)
704 lima_bo_unreference(vs->bo);
705
706 if (vs == ctx->vs)
707 ctx->vs = NULL;
708
709 ralloc_free(vs);
710 }
711 }
712
713 ralloc_free(so->base.ir.nir);
714 ralloc_free(so);
715 }
716
717 static uint32_t
lima_fs_cache_hash(const void * key)718 lima_fs_cache_hash(const void *key)
719 {
720 return _mesa_hash_data(key, sizeof(struct lima_fs_key));
721 }
722
723 static uint32_t
lima_vs_cache_hash(const void * key)724 lima_vs_cache_hash(const void *key)
725 {
726 return _mesa_hash_data(key, sizeof(struct lima_vs_key));
727 }
728
729 static bool
lima_fs_cache_compare(const void * key1,const void * key2)730 lima_fs_cache_compare(const void *key1, const void *key2)
731 {
732 return memcmp(key1, key2, sizeof(struct lima_fs_key)) == 0;
733 }
734
735 static bool
lima_vs_cache_compare(const void * key1,const void * key2)736 lima_vs_cache_compare(const void *key1, const void *key2)
737 {
738 return memcmp(key1, key2, sizeof(struct lima_vs_key)) == 0;
739 }
740
741 void
lima_program_init(struct lima_context * ctx)742 lima_program_init(struct lima_context *ctx)
743 {
744 ctx->base.create_fs_state = lima_create_fs_state;
745 ctx->base.bind_fs_state = lima_bind_fs_state;
746 ctx->base.delete_fs_state = lima_delete_fs_state;
747
748 ctx->base.create_vs_state = lima_create_vs_state;
749 ctx->base.bind_vs_state = lima_bind_vs_state;
750 ctx->base.delete_vs_state = lima_delete_vs_state;
751
752 ctx->fs_cache = _mesa_hash_table_create(ctx, lima_fs_cache_hash,
753 lima_fs_cache_compare);
754 ctx->vs_cache = _mesa_hash_table_create(ctx, lima_vs_cache_hash,
755 lima_vs_cache_compare);
756 }
757
758 void
lima_program_fini(struct lima_context * ctx)759 lima_program_fini(struct lima_context *ctx)
760 {
761 hash_table_foreach(ctx->vs_cache, entry) {
762 struct lima_vs_compiled_shader *vs = entry->data;
763 if (vs->bo)
764 lima_bo_unreference(vs->bo);
765 ralloc_free(vs);
766 _mesa_hash_table_remove(ctx->vs_cache, entry);
767 }
768
769 hash_table_foreach(ctx->fs_cache, entry) {
770 struct lima_fs_compiled_shader *fs = entry->data;
771 if (fs->bo)
772 lima_bo_unreference(fs->bo);
773 ralloc_free(fs);
774 _mesa_hash_table_remove(ctx->fs_cache, entry);
775 }
776 }
777