1 /*
2 * Copyright © 2023 Valve Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "lp_context.h"
25 #include "lp_texture_handle.h"
26 #include "lp_screen.h"
27
28 #include "gallivm/lp_bld_const.h"
29 #include "gallivm/lp_bld_debug.h"
30 #include "gallivm/lp_bld_nir.h"
31
32 #include "nir.h"
33 #include "nir_builder.h"
34
35 #include "pipe/p_context.h"
36 #include "pipe/p_screen.h"
37 #include "util/mesa-sha1.h"
38
39 static const char *image_function_base_hash = "8ca89d7a4ab5830be6a1ba1140844081235b01164a8fce8316ca6a2f81f1a899";
40 static const char *sample_function_base_hash = "0789b032c4a1ddba086e07496fe2a992b1ee08f78c0884a2923564b1ed52b9cc";
41 static const char *size_function_base_hash = "6d249ab9c1106c68b87ec9fdb5ade28368171d27f221c687f32ae1544231d2fe";
42 static const char *jit_sample_function_base_hash = "21de75bb5dbcfea1f90d03b8b688f19bdb0d96f95681cbe8b26853e1723846e4";
43
44 static void
45 llvmpipe_register_texture(struct llvmpipe_context *ctx, struct lp_static_texture_state *state, bool sampled);
46
47 static void
48 llvmpipe_register_sampler(struct llvmpipe_context *ctx, struct lp_static_sampler_state *state);
49
50 static uint64_t
llvmpipe_create_texture_handle(struct pipe_context * pctx,struct pipe_sampler_view * view,const struct pipe_sampler_state * sampler)51 llvmpipe_create_texture_handle(struct pipe_context *pctx, struct pipe_sampler_view *view, const struct pipe_sampler_state *sampler)
52 {
53 struct llvmpipe_context *ctx = llvmpipe_context(pctx);
54 struct lp_sampler_matrix *matrix = &ctx->sampler_matrix;
55
56 struct lp_texture_handle *handle = calloc(1, sizeof(struct lp_texture_handle));
57
58 if (view) {
59 struct lp_static_texture_state state;
60 lp_sampler_static_texture_state(&state, view);
61
62 /* Trade a bit of performance for potentially less sampler/texture combinations. */
63 state.pot_width = false;
64 state.pot_height = false;
65 state.pot_depth = false;
66
67 llvmpipe_register_texture(ctx, &state, true);
68
69 bool found = false;
70 for (uint32_t i = 0; i < matrix->texture_count; i++) {
71 if (!memcmp(&matrix->textures[i]->state, &state, sizeof(struct lp_static_texture_state))) {
72 handle->functions = matrix->textures[i];
73 found = true;
74 break;
75 }
76 }
77 assert(found);
78 }
79
80 if (sampler) {
81 struct lp_static_sampler_state state;
82 lp_sampler_static_sampler_state(&state, sampler);
83
84 llvmpipe_register_sampler(ctx, &state);
85
86 bool found = false;
87 for (uint32_t i = 0; i < matrix->sampler_count; i++) {
88 if (!memcmp(matrix->samplers + i, &state, sizeof(struct lp_static_sampler_state))) {
89 handle->sampler_index = i;
90 found = true;
91 break;
92 }
93 }
94 assert(found);
95 }
96
97 return (uint64_t)(uintptr_t)handle;
98 }
99
100 static void
llvmpipe_delete_texture_handle(struct pipe_context * pctx,uint64_t handle)101 llvmpipe_delete_texture_handle(struct pipe_context *pctx, uint64_t handle)
102 {
103 free((void *)(uintptr_t)handle);
104 }
105
106 static uint64_t
llvmpipe_create_image_handle(struct pipe_context * pctx,const struct pipe_image_view * view)107 llvmpipe_create_image_handle(struct pipe_context *pctx, const struct pipe_image_view *view)
108 {
109 struct llvmpipe_context *ctx = llvmpipe_context(pctx);
110 struct lp_sampler_matrix *matrix = &ctx->sampler_matrix;
111
112 struct lp_texture_handle *handle = calloc(1, sizeof(struct lp_texture_handle));
113
114 struct lp_static_texture_state state;
115 lp_sampler_static_texture_state_image(&state, view);
116
117 /* Trade a bit of performance for potentially less sampler/texture combinations. */
118 state.pot_width = false;
119 state.pot_height = false;
120 state.pot_depth = false;
121
122 if (view->u.tex.first_layer == view->u.tex.last_layer) {
123 if (state.target == PIPE_TEXTURE_1D_ARRAY)
124 state.target = PIPE_TEXTURE_1D;
125 else if (state.target == PIPE_TEXTURE_2D_ARRAY || (state.target == PIPE_TEXTURE_3D && !state.tiled))
126 state.target = PIPE_TEXTURE_2D;
127 else if (state.target == PIPE_TEXTURE_CUBE_ARRAY)
128 state.target = PIPE_TEXTURE_CUBE;
129 }
130
131 llvmpipe_register_texture(ctx, &state, false);
132
133 bool found = false;
134 for (uint32_t i = 0; i < matrix->texture_count; i++) {
135 if (!memcmp(&matrix->textures[i]->state, &state, sizeof(struct lp_static_texture_state))) {
136 handle->functions = matrix->textures[i];
137 found = true;
138 break;
139 }
140 }
141 assert(found);
142
143 return (uint64_t)(uintptr_t)handle;
144 }
145
146 static void
llvmpipe_delete_image_handle(struct pipe_context * pctx,uint64_t handle)147 llvmpipe_delete_image_handle(struct pipe_context *pctx, uint64_t handle)
148 {
149 free((void *)(uintptr_t)handle);
150 }
151
152 static uint64_t
153 get_sample_function(uint64_t _matrix, uint64_t _texture_functions, uint64_t _sampler_desc, uint32_t sample_key);
154
155 struct sample_function_cache_key {
156 struct lp_texture_functions *texture_functions;
157 uint32_t sampler_index;
158 uint32_t sample_key;
159 };
160
DERIVE_HASH_TABLE(sample_function_cache_key)161 DERIVE_HASH_TABLE(sample_function_cache_key)
162
163 static struct hash_table *
164 acquire_latest_sample_function_cache(struct lp_sampler_matrix *matrix)
165 {
166 uint64_t value = p_atomic_read(&matrix->latest_cache.value);
167 return (struct hash_table *)(uintptr_t)value;
168 }
169
170 static void
replace_sample_function_cache_locked(struct lp_sampler_matrix * matrix,struct hash_table * new_cache)171 replace_sample_function_cache_locked(struct lp_sampler_matrix *matrix, struct hash_table *new_cache)
172 {
173 uint64_t old_value = p_atomic_xchg(&matrix->latest_cache.value, (uint64_t)(uintptr_t)new_cache);
174 /* Like RCU pointers, defer cleanup of old values until we know no readers are left. */
175 struct hash_table *old_cache = (struct hash_table *)(uintptr_t)old_value;
176 util_dynarray_append(&matrix->trash_caches, struct hash_table *, old_cache);
177 }
178
179 void
llvmpipe_init_sampler_matrix(struct llvmpipe_context * ctx)180 llvmpipe_init_sampler_matrix(struct llvmpipe_context *ctx)
181 {
182 ctx->pipe.create_texture_handle = llvmpipe_create_texture_handle;
183 ctx->pipe.delete_texture_handle = llvmpipe_delete_texture_handle;
184 ctx->pipe.create_image_handle = llvmpipe_create_image_handle;
185 ctx->pipe.delete_image_handle = llvmpipe_delete_image_handle;
186
187 struct lp_sampler_matrix *matrix = &ctx->sampler_matrix;
188
189 util_dynarray_init(&matrix->gallivms, NULL);
190
191 matrix->ctx = ctx;
192
193 matrix->compile_function = get_sample_function;
194
195 struct hash_table *initial_cache = sample_function_cache_key_table_create(NULL);
196 p_atomic_set(&matrix->latest_cache.value, (uint64_t)(uintptr_t)initial_cache);
197 util_dynarray_init(&matrix->trash_caches, NULL);
198
199 simple_mtx_init(&matrix->lock, mtx_plain);
200 }
201
202 void
llvmpipe_sampler_matrix_destroy(struct llvmpipe_context * ctx)203 llvmpipe_sampler_matrix_destroy(struct llvmpipe_context *ctx)
204 {
205 struct lp_sampler_matrix *matrix = &ctx->sampler_matrix;
206
207 simple_mtx_destroy(&matrix->lock);
208
209 _mesa_hash_table_destroy(acquire_latest_sample_function_cache(matrix), NULL);
210 util_dynarray_foreach (&matrix->trash_caches, struct hash_table *, trash)
211 _mesa_hash_table_destroy(*trash, NULL);
212 util_dynarray_fini(&matrix->trash_caches);
213
214 free(matrix->samplers);
215
216 for (uint32_t texture_index = 0; texture_index < matrix->texture_count; texture_index++) {
217 struct lp_texture_functions *texture = matrix->textures[texture_index];
218
219 uint32_t sampler_count = texture->sampler_count;
220 if (texture->state.format == PIPE_FORMAT_NONE)
221 sampler_count = MIN2(sampler_count, 1);
222
223 for (uint32_t sampler_index = 0; sampler_index < sampler_count; sampler_index++)
224 free(texture->sample_functions[sampler_index]);
225
226 free(texture->sample_functions);
227 free(texture->fetch_functions);
228 free(texture->image_functions);
229 free(texture);
230 }
231 free(matrix->textures);
232
233 util_dynarray_foreach (&matrix->gallivms, struct gallivm_state *, gallivm)
234 gallivm_destroy(*gallivm);
235
236 util_dynarray_fini(&matrix->gallivms);
237
238 if (matrix->context.ref)
239 lp_context_destroy(&matrix->context);
240 }
241
242 static lp_context_ref *
get_llvm_context(struct llvmpipe_context * ctx)243 get_llvm_context(struct llvmpipe_context *ctx)
244 {
245 struct lp_sampler_matrix *matrix = &ctx->sampler_matrix;
246
247 if (!matrix->context.ref)
248 lp_context_create(&matrix->context);
249
250 return &matrix->context;
251 }
252
253 static void *
compile_function(struct llvmpipe_context * ctx,struct gallivm_state * gallivm,LLVMValueRef function,const char * func_name,bool needs_caching,uint8_t cache_key[SHA1_DIGEST_LENGTH])254 compile_function(struct llvmpipe_context *ctx, struct gallivm_state *gallivm, LLVMValueRef function,
255 const char *func_name,
256 bool needs_caching,
257 uint8_t cache_key[SHA1_DIGEST_LENGTH])
258 {
259 gallivm_verify_function(gallivm, function);
260 gallivm_compile_module(gallivm);
261
262 void *function_ptr = func_to_pointer(gallivm_jit_function(gallivm, function, func_name));
263
264 if (needs_caching)
265 lp_disk_cache_insert_shader(llvmpipe_screen(ctx->pipe.screen), gallivm->cache, cache_key);
266
267 gallivm_free_ir(gallivm);
268
269 util_dynarray_append(&ctx->sampler_matrix.gallivms, struct gallivm_state *, gallivm);
270
271 return function_ptr;
272 }
273
274 static void *
compile_image_function(struct llvmpipe_context * ctx,struct lp_static_texture_state * texture,uint32_t op)275 compile_image_function(struct llvmpipe_context *ctx, struct lp_static_texture_state *texture, uint32_t op)
276 {
277 const struct util_format_description *desc = util_format_description(texture->format);
278 if (desc->colorspace != UTIL_FORMAT_COLORSPACE_ZS && !lp_storage_render_image_format_supported(texture->format))
279 return NULL;
280
281 bool ms = op >= LP_TOTAL_IMAGE_OP_COUNT / 2;
282 if (ms)
283 op -= LP_TOTAL_IMAGE_OP_COUNT / 2;
284
285 struct lp_img_params params = { 0 };
286
287 params.img_op = op;
288 if (op >= LP_IMG_OP_COUNT - 1) {
289 params.img_op = LP_IMG_ATOMIC;
290 params.op = op - (LP_IMG_OP_COUNT - 1);
291 } else if (op != LP_IMG_LOAD && op != LP_IMG_LOAD_SPARSE && op != LP_IMG_STORE) {
292 params.img_op = LP_IMG_ATOMIC_CAS;
293 }
294
295 /* Loads need to support a wider range of formats for input attachments. */
296 if (params.img_op != LP_IMG_LOAD)
297 if (texture->format != PIPE_FORMAT_NONE && !lp_storage_image_format_supported(texture->format))
298 return NULL;
299
300 uint8_t cache_key[SHA1_DIGEST_LENGTH];
301 struct mesa_sha1 hash_ctx;
302 _mesa_sha1_init(&hash_ctx);
303 _mesa_sha1_update(&hash_ctx, image_function_base_hash, strlen(image_function_base_hash));
304 _mesa_sha1_update(&hash_ctx, texture, sizeof(*texture));
305 _mesa_sha1_update(&hash_ctx, &op, sizeof(op));
306 _mesa_sha1_update(&hash_ctx, &ms, sizeof(ms));
307 _mesa_sha1_final(&hash_ctx, cache_key);
308
309 struct lp_cached_code cached = { 0 };
310 lp_disk_cache_find_shader(llvmpipe_screen(ctx->pipe.screen), &cached, cache_key);
311 bool needs_caching = !cached.data_size;
312
313 struct gallivm_state *gallivm = gallivm_create("sample_function", get_llvm_context(ctx), &cached);
314
315 struct lp_image_static_state state = {
316 .image_state = *texture,
317 };
318 struct lp_build_image_soa *image_soa = lp_bld_llvm_image_soa_create(&state, 1);
319
320 struct lp_type type;
321 memset(&type, 0, sizeof type);
322 type.floating = true; /* floating point values */
323 type.sign = true; /* values are signed */
324 type.norm = false; /* values are not limited to [0,1] or [-1,1] */
325 type.width = 32; /* 32-bit float */
326 type.length = MIN2(lp_native_vector_width / 32, 16); /* n*4 elements per vector */
327
328 struct lp_compute_shader_variant cs = { .gallivm = gallivm };
329 lp_jit_init_cs_types(&cs);
330
331 params.type = type;
332 params.target = texture->target;
333 params.resources_type = cs.jit_resources_type;
334 params.format = texture->format;
335
336 LLVMTypeRef function_type = lp_build_image_function_type(gallivm, ¶ms, ms);
337 if (!function_type) {
338 free(image_soa);
339 gallivm_destroy(gallivm);
340 return NULL;
341 }
342
343 LLVMValueRef function = LLVMAddFunction(gallivm->module, "image", function_type);
344
345 uint32_t arg_index = 0;
346
347 gallivm->texture_descriptor = LLVMGetParam(function, arg_index++);
348
349 if (params.img_op != LP_IMG_LOAD && params.img_op != LP_IMG_LOAD_SPARSE)
350 params.exec_mask = LLVMGetParam(function, arg_index++);
351
352 LLVMValueRef coords[3];
353 params.coords = coords;
354 for (uint32_t i = 0; i < 3; i++)
355 coords[i] = LLVMGetParam(function, arg_index++);
356
357 if (ms)
358 params.ms_index = LLVMGetParam(function, arg_index++);
359
360 if (params.img_op != LP_IMG_LOAD && params.img_op != LP_IMG_LOAD_SPARSE)
361 for (uint32_t i = 0; i < 4; i++)
362 params.indata[i] = LLVMGetParam(function, arg_index++);
363
364 if (params.img_op == LP_IMG_ATOMIC_CAS)
365 for (uint32_t i = 0; i < 4; i++)
366 params.indata2[i] = LLVMGetParam(function, arg_index++);
367
368 LLVMBuilderRef old_builder = gallivm->builder;
369 LLVMBasicBlockRef block = LLVMAppendBasicBlockInContext(gallivm->context, function, "entry");
370 gallivm->builder = LLVMCreateBuilderInContext(gallivm->context);
371 LLVMPositionBuilderAtEnd(gallivm->builder, block);
372
373 LLVMValueRef outdata[5] = { 0 };
374 lp_build_img_op_soa(texture, lp_build_image_soa_dynamic_state(image_soa), gallivm, ¶ms, outdata);
375
376 for (uint32_t i = 1; i < 4; i++)
377 if (!outdata[i])
378 outdata[i] = outdata[0];
379
380 if (outdata[4])
381 outdata[4] = LLVMBuildZExt(gallivm->builder, outdata[4], lp_build_int_vec_type(gallivm, lp_int_type(type)), "");
382 else
383 outdata[4] = lp_build_one(gallivm, lp_int_type(type));
384
385 if (params.img_op != LP_IMG_STORE)
386 LLVMBuildAggregateRet(gallivm->builder, outdata, params.img_op == LP_IMG_LOAD_SPARSE ? 5 : 4);
387 else
388 LLVMBuildRetVoid(gallivm->builder);
389
390 LLVMDisposeBuilder(gallivm->builder);
391 gallivm->builder = old_builder;
392
393 free(image_soa);
394
395 return compile_function(ctx, gallivm, function, "image", needs_caching, cache_key);
396 }
397
398 static void *
compile_sample_function(struct llvmpipe_context * ctx,struct lp_static_texture_state * texture,struct lp_static_sampler_state * sampler,uint32_t sample_key)399 compile_sample_function(struct llvmpipe_context *ctx, struct lp_static_texture_state *texture,
400 struct lp_static_sampler_state *sampler, uint32_t sample_key)
401 {
402 enum lp_sampler_lod_control lod_control = (sample_key & LP_SAMPLER_LOD_CONTROL_MASK) >> LP_SAMPLER_LOD_CONTROL_SHIFT;
403
404 bool supported = true;
405 if (texture->format != PIPE_FORMAT_NONE) {
406 enum lp_sampler_op_type op_type = (sample_key & LP_SAMPLER_OP_TYPE_MASK) >> LP_SAMPLER_OP_TYPE_SHIFT;
407 if (op_type != LP_SAMPLER_OP_LODQ)
408 if ((sampler->compare_mode == PIPE_TEX_COMPARE_NONE) == !!(sample_key & LP_SAMPLER_SHADOW))
409 supported = false;
410
411 /* Skip integer formats which would cause a type mismatch in the compare function. */
412 const struct util_format_description *desc = util_format_description(texture->format);
413 struct lp_type texel_type = {
414 .floating = true,
415 .width = 32,
416 .length = 1,
417 };
418 texel_type = lp_build_texel_type(texel_type, desc);
419 if ((sample_key & LP_SAMPLER_SHADOW) && !texel_type.floating)
420 supported = false;
421
422 if (texture_dims(texture->target) != 2 && op_type == LP_SAMPLER_OP_GATHER)
423 supported = false;
424
425 if (op_type != LP_SAMPLER_OP_FETCH) {
426 if (!sampler->normalized_coords) {
427 if (texture->target != PIPE_TEXTURE_1D && texture->target != PIPE_TEXTURE_2D &&
428 texture->target != PIPE_TEXTURE_1D_ARRAY && texture->target != PIPE_TEXTURE_2D_ARRAY)
429 supported = false;
430
431 if (!texture->level_zero_only)
432 supported = false;
433 }
434 }
435
436 if (util_format_is_pure_integer(texture->format) &&
437 (sampler->min_img_filter == PIPE_TEX_FILTER_LINEAR ||
438 sampler->min_mip_filter == PIPE_TEX_MIPFILTER_LINEAR ||
439 sampler->mag_img_filter == PIPE_TEX_FILTER_LINEAR))
440 supported = false;
441
442 if (sampler->aniso) {
443 if (util_format_is_pure_integer(texture->format))
444 supported = false;
445 }
446
447 if (util_format_get_num_planes(texture->format) > 1)
448 return NULL;
449
450 uint32_t bind = op_type == LP_SAMPLER_OP_FETCH ? PIPE_BIND_CONSTANT_BUFFER : PIPE_BIND_SAMPLER_VIEW;
451 if (!ctx->pipe.screen->is_format_supported(ctx->pipe.screen, texture->format, texture->target, 0, 0, bind))
452 supported = false;
453 }
454
455 uint8_t cache_key[SHA1_DIGEST_LENGTH];
456 struct mesa_sha1 hash_ctx;
457 _mesa_sha1_init(&hash_ctx);
458 _mesa_sha1_update(&hash_ctx, sample_function_base_hash, strlen(sample_function_base_hash));
459 _mesa_sha1_update(&hash_ctx, texture, sizeof(*texture));
460 _mesa_sha1_update(&hash_ctx, sampler, sizeof(*sampler));
461 _mesa_sha1_update(&hash_ctx, &sample_key, sizeof(sample_key));
462 _mesa_sha1_final(&hash_ctx, cache_key);
463
464 struct lp_cached_code cached = { 0 };
465 lp_disk_cache_find_shader(llvmpipe_screen(ctx->pipe.screen), &cached, cache_key);
466 bool needs_caching = !cached.data_size;
467
468 struct gallivm_state *gallivm = gallivm_create("sample_function", get_llvm_context(ctx), &cached);
469
470 struct lp_sampler_static_state state = {
471 .texture_state = *texture,
472 .sampler_state = *sampler,
473 };
474 struct lp_build_sampler_soa *sampler_soa = lp_llvm_sampler_soa_create(&state, 1);
475
476 struct lp_type type;
477 memset(&type, 0, sizeof type);
478 type.floating = true; /* floating point values */
479 type.sign = true; /* values are signed */
480 type.norm = false; /* values are not limited to [0,1] or [-1,1] */
481 type.width = 32; /* 32-bit float */
482 type.length = MIN2(lp_native_vector_width / 32, 16); /* n*4 elements per vector */
483
484 struct lp_compute_shader_variant cs = { .gallivm = gallivm };
485 lp_jit_init_cs_types(&cs);
486
487 LLVMTypeRef function_type = lp_build_sample_function_type(gallivm, sample_key);
488 LLVMValueRef function = LLVMAddFunction(gallivm->module, "sample", function_type);
489
490 uint32_t arg_index = 0;
491
492 gallivm->texture_descriptor = LLVMGetParam(function, arg_index++);
493 gallivm->sampler_descriptor = LLVMGetParam(function, arg_index++);
494
495 LLVMValueRef coords[5];
496 for (unsigned i = 0; i < 4; i++)
497 coords[i] = LLVMGetParam(function, arg_index++);
498
499 if (sample_key & LP_SAMPLER_SHADOW)
500 coords[4] = LLVMGetParam(function, arg_index++);
501 else
502 coords[4] = lp_build_undef(gallivm, type);
503
504 LLVMValueRef ms_index = NULL;
505 if (sample_key & LP_SAMPLER_FETCH_MS)
506 ms_index = LLVMGetParam(function, arg_index++);
507
508 LLVMValueRef offsets[3] = { 0 };
509 if (sample_key & LP_SAMPLER_OFFSETS)
510 for (unsigned i = 0; i < 3; i++)
511 offsets[i] = LLVMGetParam(function, arg_index++);
512
513 LLVMValueRef lod = NULL;
514 if (lod_control == LP_SAMPLER_LOD_BIAS || lod_control == LP_SAMPLER_LOD_EXPLICIT)
515 lod = LLVMGetParam(function, arg_index++);
516
517 LLVMBuilderRef old_builder = gallivm->builder;
518 LLVMBasicBlockRef block = LLVMAppendBasicBlockInContext(gallivm->context, function, "entry");
519 gallivm->builder = LLVMCreateBuilderInContext(gallivm->context);
520 LLVMPositionBuilderAtEnd(gallivm->builder, block);
521
522 LLVMValueRef texel_out[5] = { 0 };
523 if (supported) {
524 lp_build_sample_soa_code(gallivm, texture, sampler, lp_build_sampler_soa_dynamic_state(sampler_soa),
525 type, sample_key, 0, 0, cs.jit_resources_type, NULL, cs.jit_cs_thread_data_type,
526 NULL, coords, offsets, NULL, lod, ms_index, texel_out);
527 } else {
528 lp_build_sample_nop(gallivm, lp_build_texel_type(type, util_format_description(texture->format)), coords, texel_out);
529 }
530
531 if (texel_out[4])
532 texel_out[4] = LLVMBuildZExt(gallivm->builder, texel_out[4], lp_build_int_vec_type(gallivm, lp_int_type(type)), "");
533 else
534 texel_out[4] = lp_build_one(gallivm, lp_int_type(type));
535
536 LLVMBuildAggregateRet(gallivm->builder, texel_out, 5);
537
538 LLVMDisposeBuilder(gallivm->builder);
539 gallivm->builder = old_builder;
540
541 free(sampler_soa);
542
543 return compile_function(ctx, gallivm, function, "sample", needs_caching, cache_key);
544 }
545
546 static uint64_t
get_sample_function(uint64_t _matrix,uint64_t _texture_functions,uint64_t _sampler_desc,uint32_t sample_key)547 get_sample_function(uint64_t _matrix, uint64_t _texture_functions, uint64_t _sampler_desc, uint32_t sample_key)
548 {
549 struct lp_sampler_matrix *matrix = (void *)(uintptr_t)_matrix;
550 struct lp_descriptor *sampler_desc = (void *)(uintptr_t)_sampler_desc;
551 uint32_t sampler_index = sampler_desc->texture.sampler_index;
552
553 struct lp_texture_functions *texture_functions = (void *)(uintptr_t)_texture_functions;
554 struct sample_function_cache_key key = {
555 .texture_functions = texture_functions,
556 .sampler_index = sampler_index,
557 .sample_key = sample_key,
558 };
559
560 void *result;
561 {
562 struct hash_entry *entry = _mesa_hash_table_search(acquire_latest_sample_function_cache(matrix), &key);
563 result = entry ? entry->data : NULL;
564 }
565
566 if (!result) {
567 simple_mtx_lock(&matrix->lock);
568 /* Check once more in case the cache got modified between the first check and acquiring the lock. */
569 struct hash_table *current_cache = acquire_latest_sample_function_cache(matrix);
570 struct hash_entry *entry = _mesa_hash_table_search(current_cache, &key);
571 result = entry ? entry->data : NULL;
572 if (!result) {
573 result = compile_sample_function(matrix->ctx, &texture_functions->state, matrix->samplers + sampler_index, sample_key);
574 struct sample_function_cache_key *allocated_key = malloc(sizeof(struct sample_function_cache_key));
575 *allocated_key = key;
576 /* RCU style update: swap in an updated copy of the cache.
577 / Old caches are kept as trash to be safely deleted later. */
578 struct hash_table *new_cache = _mesa_hash_table_clone(current_cache, NULL);
579 _mesa_hash_table_insert(new_cache, allocated_key, result);
580 replace_sample_function_cache_locked(matrix, new_cache);
581 }
582 simple_mtx_unlock(&matrix->lock);
583 }
584
585 return (uint64_t)(uintptr_t)result;
586 }
587
588 static LLVMTypeRef
lp_build_compile_function_type(struct gallivm_state * gallivm)589 lp_build_compile_function_type(struct gallivm_state *gallivm)
590 {
591 LLVMTypeRef param_types[4] = {
592 LLVMInt64TypeInContext(gallivm->context),
593 LLVMInt64TypeInContext(gallivm->context),
594 LLVMInt64TypeInContext(gallivm->context),
595 LLVMInt32TypeInContext(gallivm->context),
596 };
597 LLVMTypeRef ret_type = LLVMInt64TypeInContext(gallivm->context);
598
599 return LLVMFunctionType(ret_type, param_types, ARRAY_SIZE(param_types), false);
600 }
601
602 static void *
compile_jit_sample_function(struct llvmpipe_context * ctx,uint32_t sample_key)603 compile_jit_sample_function(struct llvmpipe_context *ctx, uint32_t sample_key)
604 {
605 uint8_t cache_key[SHA1_DIGEST_LENGTH];
606 struct mesa_sha1 hash_ctx;
607 _mesa_sha1_init(&hash_ctx);
608 _mesa_sha1_update(&hash_ctx, jit_sample_function_base_hash, strlen(jit_sample_function_base_hash));
609 _mesa_sha1_update(&hash_ctx, &sample_key, sizeof(sample_key));
610 _mesa_sha1_final(&hash_ctx, cache_key);
611
612 struct lp_cached_code cached = { 0 };
613 lp_disk_cache_find_shader(llvmpipe_screen(ctx->pipe.screen), &cached, cache_key);
614 bool needs_caching = !cached.data_size;
615
616 struct gallivm_state *gallivm = gallivm_create("jit_sample_function", get_llvm_context(ctx), &cached);
617
618 struct lp_type type;
619 memset(&type, 0, sizeof type);
620 type.floating = true; /* floating point values */
621 type.sign = true; /* values are signed */
622 type.norm = false; /* values are not limited to [0,1] or [-1,1] */
623 type.width = 32; /* 32-bit float */
624 type.length = MIN2(lp_native_vector_width / 32, 16); /* n*4 elements per vector */
625
626 struct lp_compute_shader_variant cs = { .gallivm = gallivm };
627 lp_jit_init_cs_types(&cs);
628
629 LLVMTypeRef function_type = lp_build_sample_function_type(gallivm, sample_key);
630 LLVMValueRef function = LLVMAddFunction(gallivm->module, "sample", function_type);
631
632 uint32_t arg_index = 0;
633 LLVMValueRef texture_descriptor = LLVMGetParam(function, arg_index++);
634 LLVMValueRef sampler_descriptor = LLVMGetParam(function, arg_index++);
635
636 LLVMBuilderRef old_builder = gallivm->builder;
637 LLVMBasicBlockRef block = LLVMAppendBasicBlockInContext(gallivm->context, function, "entry");
638 gallivm->builder = LLVMCreateBuilderInContext(gallivm->context);
639 LLVMBuilderRef builder = gallivm->builder;
640 LLVMPositionBuilderAtEnd(gallivm->builder, block);
641
642 LLVMValueRef functions_offset =
643 lp_build_const_int64(gallivm, offsetof(struct lp_descriptor, functions));
644 LLVMValueRef functions_ptr =
645 LLVMBuildAdd(builder, texture_descriptor, functions_offset, "");
646
647 LLVMTypeRef functions_ptr_type = LLVMInt64TypeInContext(gallivm->context);
648 LLVMTypeRef functions_ptr_ptr_type = LLVMPointerType(functions_ptr_type, 0);
649
650 functions_ptr = LLVMBuildIntToPtr(builder, functions_ptr, functions_ptr_ptr_type, "");
651 /* struct lp_texture_functions * */
652 functions_ptr = LLVMBuildLoad2(builder, functions_ptr_type, functions_ptr, "");
653
654 LLVMValueRef matrix_offset =
655 lp_build_const_int64(gallivm, offsetof(struct lp_texture_functions, matrix));
656 LLVMValueRef matrix_ptr = LLVMBuildAdd(builder, functions_ptr, matrix_offset, "");
657
658 matrix_ptr = LLVMBuildIntToPtr(builder, matrix_ptr, functions_ptr_ptr_type, "");
659 /* struct lp_sampler_matrix * */
660 matrix_ptr = LLVMBuildLoad2(builder, functions_ptr_type, matrix_ptr, "");
661
662 LLVMTypeRef compile_function_type = lp_build_compile_function_type(gallivm);
663 LLVMTypeRef compile_function_ptr_type = LLVMPointerType(compile_function_type, 0);
664 LLVMTypeRef compile_function_ptr_ptr_type = LLVMPointerType(compile_function_ptr_type, 0);
665
666 LLVMValueRef compile_function_offset =
667 lp_build_const_int64(gallivm, offsetof(struct lp_sampler_matrix, compile_function));
668 LLVMValueRef compile_function_ptr =
669 LLVMBuildAdd(builder, matrix_ptr, compile_function_offset, "");
670
671 compile_function_ptr =
672 LLVMBuildIntToPtr(builder, compile_function_ptr, compile_function_ptr_ptr_type, "");
673 /* struct lp_texture_functions * */
674 compile_function_ptr =
675 LLVMBuildLoad2(builder, compile_function_ptr_type, compile_function_ptr, "");
676
677 LLVMValueRef compile_args[4] = {
678 matrix_ptr, functions_ptr, sampler_descriptor, lp_build_const_int32(gallivm, sample_key)
679 };
680
681 LLVMValueRef sample_function =
682 LLVMBuildCall2(builder, compile_function_type, compile_function_ptr,
683 compile_args, ARRAY_SIZE(compile_args), "");
684
685 sample_function = LLVMBuildIntToPtr(builder, sample_function, LLVMPointerType(function_type, 0), "");
686
687 LLVMValueRef args[LP_MAX_TEX_FUNC_ARGS];
688 uint32_t num_args = 0;
689
690 LLVMValueRef arg = LLVMGetFirstParam(function);
691 while (true) {
692 args[num_args++] = arg;
693 if (arg == LLVMGetLastParam(function))
694 break;
695
696 arg = LLVMGetNextParam(arg);
697 }
698
699 LLVMValueRef result = LLVMBuildCall2(builder, function_type, sample_function, args, num_args, "");
700 LLVMBuildRet(gallivm->builder, result);
701
702 LLVMDisposeBuilder(gallivm->builder);
703 gallivm->builder = old_builder;
704
705 return compile_function(ctx, gallivm, function, "sample", needs_caching, cache_key);
706 }
707
708 static void *
compile_size_function(struct llvmpipe_context * ctx,struct lp_static_texture_state * texture,bool samples)709 compile_size_function(struct llvmpipe_context *ctx, struct lp_static_texture_state *texture, bool samples)
710 {
711 uint8_t cache_key[SHA1_DIGEST_LENGTH];
712 struct mesa_sha1 hash_ctx;
713 _mesa_sha1_init(&hash_ctx);
714 _mesa_sha1_update(&hash_ctx, size_function_base_hash, strlen(size_function_base_hash));
715 _mesa_sha1_update(&hash_ctx, texture, sizeof(*texture));
716 _mesa_sha1_update(&hash_ctx, &samples, sizeof(samples));
717 _mesa_sha1_final(&hash_ctx, cache_key);
718
719 struct lp_cached_code cached = { 0 };
720 lp_disk_cache_find_shader(llvmpipe_screen(ctx->pipe.screen), &cached, cache_key);
721 bool needs_caching = !cached.data_size;
722
723 struct gallivm_state *gallivm = gallivm_create("sample_function", get_llvm_context(ctx), &cached);
724
725 struct lp_sampler_static_state state = {
726 .texture_state = *texture,
727 };
728 struct lp_build_sampler_soa *sampler_soa = lp_llvm_sampler_soa_create(&state, 1);
729
730 struct lp_type type;
731 memset(&type, 0, sizeof type);
732 type.floating = true; /* floating point values */
733 type.sign = true; /* values are signed */
734 type.norm = false; /* values are not limited to [0,1] or [-1,1] */
735 type.width = 32; /* 32-bit float */
736 type.length = MIN2(lp_native_vector_width / 32, 16); /* n*4 elements per vector */
737
738 struct lp_compute_shader_variant cs = { .gallivm = gallivm };
739 lp_jit_init_cs_types(&cs);
740
741 struct lp_sampler_size_query_params params = {
742 .int_type = lp_int_type(type),
743 .target = texture->target,
744 .resources_type = cs.jit_resources_type,
745 .is_sviewinfo = true,
746 .samples_only = samples,
747 .ms = samples,
748 };
749
750 if (params.target == PIPE_TEXTURE_1D)
751 params.target = PIPE_TEXTURE_1D_ARRAY;
752 else if (params.target == PIPE_TEXTURE_2D)
753 params.target = PIPE_TEXTURE_2D_ARRAY;
754 else if (params.target == PIPE_TEXTURE_CUBE)
755 params.target = PIPE_TEXTURE_CUBE_ARRAY;
756
757 LLVMTypeRef function_type = lp_build_size_function_type(gallivm, ¶ms);
758 LLVMValueRef function = LLVMAddFunction(gallivm->module, "size", function_type);
759
760 uint32_t arg_index = 0;
761
762 gallivm->texture_descriptor = LLVMGetParam(function, arg_index++);
763
764 if (!samples)
765 params.explicit_lod = LLVMGetParam(function, arg_index++);
766
767 LLVMBuilderRef old_builder = gallivm->builder;
768 LLVMBasicBlockRef block = LLVMAppendBasicBlockInContext(gallivm->context, function, "entry");
769 gallivm->builder = LLVMCreateBuilderInContext(gallivm->context);
770 LLVMPositionBuilderAtEnd(gallivm->builder, block);
771
772 LLVMValueRef out_sizes[4] = { 0 };
773 params.sizes_out = out_sizes;
774 lp_build_size_query_soa(gallivm, texture, lp_build_sampler_soa_dynamic_state(sampler_soa), ¶ms);
775
776 for (uint32_t i = 0; i < 4; i++)
777 if (!out_sizes[i])
778 out_sizes[i] = lp_build_const_int_vec(gallivm, params.int_type, 0);
779
780 LLVMBuildAggregateRet(gallivm->builder, out_sizes, 4);
781
782 LLVMDisposeBuilder(gallivm->builder);
783 gallivm->builder = old_builder;
784
785 free(sampler_soa);
786
787 return compile_function(ctx, gallivm, function, "size", needs_caching, cache_key);
788 }
789
790 static void
compile_sample_functions(struct llvmpipe_context * ctx,struct lp_static_texture_state * texture,struct lp_static_sampler_state * sampler,void *** dst)791 compile_sample_functions(struct llvmpipe_context *ctx, struct lp_static_texture_state *texture,
792 struct lp_static_sampler_state *sampler, void ***dst)
793 {
794 void **functions;
795 if (*dst) {
796 functions = *dst;
797 } else {
798 functions = calloc(LP_SAMPLE_KEY_COUNT, sizeof(void *));
799 *dst = functions;
800 }
801
802 bool has_sampler = !!sampler;
803
804 struct lp_static_sampler_state dummy_sampler = { 0 };
805 if (!sampler)
806 sampler = &dummy_sampler;
807
808 struct lp_sampler_matrix *matrix = &ctx->sampler_matrix;
809 for (uint32_t sample_key = 0; sample_key < LP_SAMPLE_KEY_COUNT; sample_key++) {
810 if (!BITSET_TEST(matrix->sample_keys, sample_key))
811 continue;
812
813 enum lp_sampler_op_type op_type = (sample_key & LP_SAMPLER_OP_TYPE_MASK) >> LP_SAMPLER_OP_TYPE_SHIFT;
814 if (has_sampler && op_type == LP_SAMPLER_OP_FETCH)
815 continue;
816
817 if (!functions[sample_key]) {
818 if (has_sampler)
819 functions[sample_key] = matrix->jit_sample_functions[sample_key];
820 else
821 functions[sample_key] = compile_sample_function(ctx, texture, sampler, sample_key);
822 }
823 }
824 }
825
826 static void
llvmpipe_register_texture(struct llvmpipe_context * ctx,struct lp_static_texture_state * state,bool sampled)827 llvmpipe_register_texture(struct llvmpipe_context *ctx, struct lp_static_texture_state *state, bool sampled)
828 {
829 struct lp_sampler_matrix *matrix = &ctx->sampler_matrix;
830
831 bool packed = true;
832 uint32_t dst_index = matrix->texture_count;
833 for (uint32_t i = 0; i < matrix->texture_count; i++) {
834 if (memcmp(&matrix->textures[i]->state, state, sizeof(struct lp_static_texture_state)))
835 continue;
836
837 bool has_functions = sampled ? matrix->textures[i]->sampled : matrix->textures[i]->storage;
838 if (has_functions)
839 return;
840
841 packed = false;
842 dst_index = i;
843 break;
844 }
845
846 struct lp_texture_functions *entry;
847 if (packed) {
848 matrix->texture_count++;
849 matrix->textures = realloc(matrix->textures, matrix->texture_count * sizeof(struct lp_texture_functions *));
850
851 entry = calloc(1, sizeof(struct lp_texture_functions));
852 matrix->textures[dst_index] = entry;
853
854 entry->state = *state;
855 entry->image_functions = calloc(LP_TOTAL_IMAGE_OP_COUNT, sizeof(void **));
856 entry->matrix = matrix;
857 } else {
858 entry = matrix->textures[dst_index];
859 }
860
861 if (sampled)
862 entry->sampled = true;
863 else
864 entry->storage = true;
865
866 simple_mtx_lock(&matrix->lock);
867
868 if (entry->sampled) {
869 if (entry->sample_functions) {
870 entry->sample_functions = realloc(entry->sample_functions, matrix->sampler_count * sizeof(void **));
871 memset(entry->sample_functions + entry->sampler_count, 0, (matrix->sampler_count - entry->sampler_count) * sizeof(void **));
872 } else {
873 entry->sample_functions = calloc(matrix->sampler_count, sizeof(void **));
874 }
875 entry->sampler_count = matrix->sampler_count;
876
877 if (state->format == PIPE_FORMAT_NONE) {
878 if (matrix->sampler_count)
879 compile_sample_functions(ctx, state, NULL, entry->sample_functions);
880 for (uint32_t i = 1; i < matrix->sampler_count; i++)
881 entry->sample_functions[i] = entry->sample_functions[0];
882 } else {
883 for (uint32_t i = 0; i < matrix->sampler_count; i++)
884 compile_sample_functions(ctx, state, matrix->samplers + i, entry->sample_functions + i);
885 }
886
887 compile_sample_functions(ctx, state, NULL, &entry->fetch_functions);
888
889 if (!entry->size_function)
890 entry->size_function = compile_size_function(ctx, state, false);
891
892 if (!entry->samples_function)
893 entry->samples_function = compile_size_function(ctx, state, true);
894 }
895
896 if (entry->storage) {
897 uint32_t image_op;
898 BITSET_FOREACH_SET (image_op, matrix->image_ops, LP_TOTAL_IMAGE_OP_COUNT)
899 if (!entry->image_functions[image_op])
900 entry->image_functions[image_op] = compile_image_function(ctx, state, image_op);
901 }
902
903 simple_mtx_unlock(&matrix->lock);
904 }
905
906 static void
llvmpipe_register_sampler(struct llvmpipe_context * ctx,struct lp_static_sampler_state * state)907 llvmpipe_register_sampler(struct llvmpipe_context *ctx, struct lp_static_sampler_state *state)
908 {
909 struct lp_sampler_matrix *matrix = &ctx->sampler_matrix;
910 for (uint32_t i = 0; i < matrix->sampler_count; i++)
911 if (!memcmp(matrix->samplers + i, state, sizeof(struct lp_static_sampler_state)))
912 return;
913
914 matrix->sampler_count++;
915 matrix->samplers = realloc(matrix->samplers, matrix->sampler_count * sizeof(struct lp_static_sampler_state));
916
917 matrix->samplers[matrix->sampler_count - 1] = *state;
918
919 simple_mtx_lock(&matrix->lock);
920
921 for (uint32_t i = 0; i < matrix->texture_count; i++) {
922 struct lp_texture_functions *texture = matrix->textures[i];
923 if (!texture->sampled)
924 continue;
925
926 texture->sampler_count = matrix->sampler_count;
927 texture->sample_functions = realloc(texture->sample_functions, matrix->sampler_count * sizeof(void **));
928
929 void ***dst = texture->sample_functions + (matrix->sampler_count - 1);
930
931 if (texture->state.format == PIPE_FORMAT_NONE) {
932 if (matrix->sampler_count == 1) {
933 *dst = NULL;
934 compile_sample_functions(ctx, &texture->state, NULL, dst);
935 } else {
936 *dst = texture->sample_functions[0];
937 }
938
939 continue;
940 }
941
942 *dst = NULL;
943 compile_sample_functions(ctx, &texture->state, state, dst);
944 }
945
946 simple_mtx_unlock(&matrix->lock);
947 }
948
949 static void
register_sample_key(struct llvmpipe_context * ctx,uint32_t sample_key)950 register_sample_key(struct llvmpipe_context *ctx, uint32_t sample_key)
951 {
952 struct lp_sampler_matrix *matrix = &ctx->sampler_matrix;
953 if (BITSET_TEST(matrix->sample_keys, sample_key))
954 return;
955
956 BITSET_SET(matrix->sample_keys, sample_key);
957
958 simple_mtx_lock(&matrix->lock);
959
960 matrix->jit_sample_functions[sample_key] = compile_jit_sample_function(ctx, sample_key);
961
962 for (uint32_t texture_index = 0; texture_index < matrix->texture_count; texture_index++) {
963 struct lp_texture_functions *texture = matrix->textures[texture_index];
964 if (!texture->sampled)
965 continue;
966
967 enum lp_sampler_op_type op_type = (sample_key & LP_SAMPLER_OP_TYPE_MASK) >> LP_SAMPLER_OP_TYPE_SHIFT;
968 if (op_type == LP_SAMPLER_OP_FETCH) {
969 struct lp_static_sampler_state dummy_sampler = { 0 };
970 texture->fetch_functions[sample_key] = compile_sample_function(ctx, &texture->state, &dummy_sampler, sample_key);
971 continue;
972 }
973
974 if (texture->state.format == PIPE_FORMAT_NONE) {
975 if (matrix->sampler_count) {
976 struct lp_static_sampler_state dummy_sampler = { 0 };
977 texture->sample_functions[0][sample_key] = compile_sample_function(ctx, &texture->state, &dummy_sampler, sample_key);
978 }
979 continue;
980 }
981
982 for (uint32_t sampler_index = 0; sampler_index < matrix->sampler_count; sampler_index++)
983 texture->sample_functions[sampler_index][sample_key] = matrix->jit_sample_functions[sample_key];
984 }
985
986 simple_mtx_unlock(&matrix->lock);
987 }
988
989 static void
register_image_op(struct llvmpipe_context * ctx,uint32_t op)990 register_image_op(struct llvmpipe_context *ctx, uint32_t op)
991 {
992 struct lp_sampler_matrix *matrix = &ctx->sampler_matrix;
993 if (BITSET_TEST(matrix->image_ops, op))
994 return;
995
996 BITSET_SET(matrix->image_ops, op);
997
998 simple_mtx_lock(&matrix->lock);
999
1000 for (uint32_t texture_index = 0; texture_index < matrix->texture_count; texture_index++) {
1001 struct lp_texture_functions *texture = matrix->textures[texture_index];
1002 if (texture->storage)
1003 texture->image_functions[op] = compile_image_function(ctx, &texture->state, op);
1004 }
1005
1006 simple_mtx_unlock(&matrix->lock);
1007 }
1008
1009 static bool
register_instr(nir_builder * b,nir_instr * instr,void * data)1010 register_instr(nir_builder *b, nir_instr *instr, void *data)
1011 {
1012 struct llvmpipe_context *ctx = data;
1013
1014 if (instr->type == nir_instr_type_tex) {
1015 nir_tex_instr *tex = nir_instr_as_tex(instr);
1016 uint32_t sample_key = lp_build_nir_sample_key(b->shader->info.stage, tex);
1017
1018 register_sample_key(ctx, sample_key);
1019 } else if (instr->type == nir_instr_type_intrinsic) {
1020 nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
1021
1022 struct lp_img_params params;
1023 lp_img_op_from_intrinsic(¶ms, intrin);
1024
1025 if (params.img_op == -1)
1026 return false;
1027
1028 uint32_t op = params.img_op;
1029 if (op == LP_IMG_ATOMIC_CAS)
1030 op--;
1031 else if (op == LP_IMG_ATOMIC)
1032 op = params.op + (LP_IMG_OP_COUNT - 1);
1033
1034 if (nir_intrinsic_image_dim(intrin) == GLSL_SAMPLER_DIM_MS ||
1035 nir_intrinsic_image_dim(intrin) == GLSL_SAMPLER_DIM_SUBPASS_MS)
1036 op += LP_TOTAL_IMAGE_OP_COUNT / 2;
1037
1038 register_image_op(ctx, op);
1039 }
1040
1041 return false;
1042 }
1043
1044 void
llvmpipe_register_shader(struct pipe_context * ctx,const struct pipe_shader_state * shader)1045 llvmpipe_register_shader(struct pipe_context *ctx, const struct pipe_shader_state *shader)
1046 {
1047 if (shader->type == PIPE_SHADER_IR_NIR)
1048 nir_shader_instructions_pass(shader->ir.nir, register_instr, nir_metadata_all, ctx);
1049 }
1050
1051 void
llvmpipe_clear_sample_functions_cache(struct llvmpipe_context * ctx,struct pipe_fence_handle ** fence)1052 llvmpipe_clear_sample_functions_cache(struct llvmpipe_context *ctx, struct pipe_fence_handle **fence)
1053 {
1054 if (!fence)
1055 return;
1056
1057 struct lp_sampler_matrix *matrix = &ctx->sampler_matrix;
1058
1059 /* If the cache is empty, there is nothing to do. */
1060 if (!_mesa_hash_table_num_entries(acquire_latest_sample_function_cache(matrix)))
1061 return;
1062
1063 ctx->pipe.screen->fence_finish(ctx->pipe.screen, NULL, *fence, OS_TIMEOUT_INFINITE);
1064
1065 /* All work is finished, it's safe to move cache entries into the table. */
1066 hash_table_foreach_remove(acquire_latest_sample_function_cache(matrix), entry) {
1067 struct sample_function_cache_key *key = (void *)entry->key;
1068 key->texture_functions->sample_functions[key->sampler_index][key->sample_key] = entry->data;
1069 free(key);
1070 }
1071
1072 util_dynarray_foreach (&matrix->trash_caches, struct hash_table *, trash)
1073 _mesa_hash_table_destroy(*trash, NULL);
1074 util_dynarray_clear(&matrix->trash_caches);
1075 }
1076