1 /*
2 * Copyright © 2023 Valve Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "lp_context.h"
25 #include "lp_texture_handle.h"
26 #include "lp_screen.h"
27
28 #include "gallivm/lp_bld_const.h"
29 #include "gallivm/lp_bld_debug.h"
30 #include "gallivm/lp_bld_nir.h"
31
32 #include "nir.h"
33 #include "nir_builder.h"
34
35 #include "pipe/p_context.h"
36 #include "pipe/p_screen.h"
37 #include "util/mesa-sha1.h"
38
39 static const char *image_function_base_hash = "8ca89d7a4ab5830be6a1ba1140844081235b01164a8fce8316ca6a2f81f1a899";
40 static const char *sample_function_base_hash = "0789b032c4a1ddba086e07496fe2a992b1ee08f78c0884a2923564b1ed52b9cc";
41 static const char *size_function_base_hash = "6d249ab9c1106c68b87ec9fdb5ade28368171d27f221c687f32ae1544231d2fe";
42 static const char *jit_sample_function_base_hash = "21de75bb5dbcfea1f90d03b8b688f19bdb0d96f95681cbe8b26853e1723846e4";
43
44 static void
45 llvmpipe_register_texture(struct llvmpipe_context *ctx, struct lp_static_texture_state *state, bool sampled);
46
47 static void
48 llvmpipe_register_sampler(struct llvmpipe_context *ctx, struct lp_static_sampler_state *state);
49
50 static uint64_t
llvmpipe_create_texture_handle(struct pipe_context * pctx,struct pipe_sampler_view * view,const struct pipe_sampler_state * sampler)51 llvmpipe_create_texture_handle(struct pipe_context *pctx, struct pipe_sampler_view *view, const struct pipe_sampler_state *sampler)
52 {
53 struct llvmpipe_context *ctx = llvmpipe_context(pctx);
54 struct lp_sampler_matrix *matrix = &ctx->sampler_matrix;
55
56 struct lp_texture_handle *handle = calloc(1, sizeof(struct lp_texture_handle));
57
58 if (view) {
59 struct lp_static_texture_state state;
60 lp_sampler_static_texture_state(&state, view);
61
62 /* Trade a bit of performance for potentially less sampler/texture combinations. */
63 state.pot_width = false;
64 state.pot_height = false;
65 state.pot_depth = false;
66
67 llvmpipe_register_texture(ctx, &state, true);
68
69 bool found = false;
70 for (uint32_t i = 0; i < matrix->texture_count; i++) {
71 if (!memcmp(&matrix->textures[i]->state, &state, sizeof(struct lp_static_texture_state))) {
72 handle->functions = matrix->textures[i];
73 found = true;
74 break;
75 }
76 }
77 assert(found);
78 }
79
80 if (sampler) {
81 struct lp_static_sampler_state state;
82 lp_sampler_static_sampler_state(&state, sampler);
83
84 llvmpipe_register_sampler(ctx, &state);
85
86 bool found = false;
87 for (uint32_t i = 0; i < matrix->sampler_count; i++) {
88 if (!memcmp(matrix->samplers + i, &state, sizeof(struct lp_static_sampler_state))) {
89 handle->sampler_index = i;
90 found = true;
91 break;
92 }
93 }
94 assert(found);
95 }
96
97 return (uint64_t)(uintptr_t)handle;
98 }
99
100 static void
llvmpipe_delete_texture_handle(struct pipe_context * pctx,uint64_t handle)101 llvmpipe_delete_texture_handle(struct pipe_context *pctx, uint64_t handle)
102 {
103 free((void *)(uintptr_t)handle);
104 }
105
106 static uint64_t
llvmpipe_create_image_handle(struct pipe_context * pctx,const struct pipe_image_view * view)107 llvmpipe_create_image_handle(struct pipe_context *pctx, const struct pipe_image_view *view)
108 {
109 struct llvmpipe_context *ctx = llvmpipe_context(pctx);
110 struct lp_sampler_matrix *matrix = &ctx->sampler_matrix;
111
112 struct lp_texture_handle *handle = calloc(1, sizeof(struct lp_texture_handle));
113
114 struct lp_static_texture_state state;
115 lp_sampler_static_texture_state_image(&state, view);
116
117 /* Trade a bit of performance for potentially less sampler/texture combinations. */
118 state.pot_width = false;
119 state.pot_height = false;
120 state.pot_depth = false;
121
122 if (view->u.tex.first_layer == view->u.tex.last_layer) {
123 if (state.target == PIPE_TEXTURE_1D_ARRAY)
124 state.target = PIPE_TEXTURE_1D;
125 else if (state.target == PIPE_TEXTURE_2D_ARRAY || (state.target == PIPE_TEXTURE_3D && !state.tiled))
126 state.target = PIPE_TEXTURE_2D;
127 else if (state.target == PIPE_TEXTURE_CUBE_ARRAY)
128 state.target = PIPE_TEXTURE_CUBE;
129 }
130
131 llvmpipe_register_texture(ctx, &state, false);
132
133 bool found = false;
134 for (uint32_t i = 0; i < matrix->texture_count; i++) {
135 if (!memcmp(&matrix->textures[i]->state, &state, sizeof(struct lp_static_texture_state))) {
136 handle->functions = matrix->textures[i];
137 found = true;
138 break;
139 }
140 }
141 assert(found);
142
143 return (uint64_t)(uintptr_t)handle;
144 }
145
146 static void
llvmpipe_delete_image_handle(struct pipe_context * pctx,uint64_t handle)147 llvmpipe_delete_image_handle(struct pipe_context *pctx, uint64_t handle)
148 {
149 free((void *)(uintptr_t)handle);
150 }
151
152 static uint64_t
153 get_sample_function(uint64_t _matrix, uint64_t _texture_functions, uint64_t _sampler_desc, uint32_t sample_key);
154
155 struct sample_function_cache_key {
156 struct lp_texture_functions *texture_functions;
157 uint32_t sampler_index;
158 uint32_t sample_key;
159 };
160
DERIVE_HASH_TABLE(sample_function_cache_key)161 DERIVE_HASH_TABLE(sample_function_cache_key)
162
163 static struct hash_table *
164 acquire_latest_sample_function_cache(struct lp_sampler_matrix *matrix)
165 {
166 uint64_t value = p_atomic_read(&matrix->latest_cache.value);
167 return (struct hash_table *)(uintptr_t)value;
168 }
169
170 static void
replace_sample_function_cache_locked(struct lp_sampler_matrix * matrix,struct hash_table * new_cache)171 replace_sample_function_cache_locked(struct lp_sampler_matrix *matrix, struct hash_table *new_cache)
172 {
173 uint64_t old_value = p_atomic_xchg(&matrix->latest_cache.value, (uint64_t)(uintptr_t)new_cache);
174 /* Like RCU pointers, defer cleanup of old values until we know no readers are left. */
175 struct hash_table *old_cache = (struct hash_table *)(uintptr_t)old_value;
176 util_dynarray_append(&matrix->trash_caches, struct hash_table *, old_cache);
177 }
178
179 void
llvmpipe_init_sampler_matrix(struct llvmpipe_context * ctx)180 llvmpipe_init_sampler_matrix(struct llvmpipe_context *ctx)
181 {
182 ctx->pipe.create_texture_handle = llvmpipe_create_texture_handle;
183 ctx->pipe.delete_texture_handle = llvmpipe_delete_texture_handle;
184 ctx->pipe.create_image_handle = llvmpipe_create_image_handle;
185 ctx->pipe.delete_image_handle = llvmpipe_delete_image_handle;
186
187 struct lp_sampler_matrix *matrix = &ctx->sampler_matrix;
188
189 util_dynarray_init(&matrix->gallivms, NULL);
190
191 matrix->ctx = ctx;
192
193 matrix->compile_function = get_sample_function;
194
195 struct hash_table *initial_cache = sample_function_cache_key_table_create(NULL);
196 p_atomic_set(&matrix->latest_cache.value, (uint64_t)(uintptr_t)initial_cache);
197 util_dynarray_init(&matrix->trash_caches, NULL);
198
199 simple_mtx_init(&matrix->lock, mtx_plain);
200 }
201
202 void
llvmpipe_sampler_matrix_destroy(struct llvmpipe_context * ctx)203 llvmpipe_sampler_matrix_destroy(struct llvmpipe_context *ctx)
204 {
205 struct lp_sampler_matrix *matrix = &ctx->sampler_matrix;
206
207 simple_mtx_destroy(&matrix->lock);
208
209 _mesa_hash_table_destroy(acquire_latest_sample_function_cache(matrix), NULL);
210 util_dynarray_foreach (&matrix->trash_caches, struct hash_table *, trash)
211 _mesa_hash_table_destroy(*trash, NULL);
212 util_dynarray_fini(&matrix->trash_caches);
213
214 free(matrix->samplers);
215
216 for (uint32_t texture_index = 0; texture_index < matrix->texture_count; texture_index++) {
217 struct lp_texture_functions *texture = matrix->textures[texture_index];
218
219 uint32_t sampler_count = texture->sampler_count;
220 if (texture->state.format == PIPE_FORMAT_NONE)
221 sampler_count = MIN2(sampler_count, 1);
222
223 for (uint32_t sampler_index = 0; sampler_index < sampler_count; sampler_index++)
224 free(texture->sample_functions[sampler_index]);
225
226 free(texture->sample_functions);
227 free(texture->fetch_functions);
228 free(texture->image_functions);
229 free(texture);
230 }
231 free(matrix->textures);
232
233 util_dynarray_foreach (&matrix->gallivms, struct gallivm_state *, gallivm)
234 gallivm_destroy(*gallivm);
235
236 util_dynarray_fini(&matrix->gallivms);
237
238 if (matrix->context.ref)
239 lp_context_destroy(&matrix->context);
240 }
241
242 static lp_context_ref *
get_llvm_context(struct llvmpipe_context * ctx)243 get_llvm_context(struct llvmpipe_context *ctx)
244 {
245 struct lp_sampler_matrix *matrix = &ctx->sampler_matrix;
246
247 if (!matrix->context.ref)
248 lp_context_create(&matrix->context);
249
250 return &matrix->context;
251 }
252
253 static void *
compile_function(struct llvmpipe_context * ctx,struct gallivm_state * gallivm,LLVMValueRef function,const char * func_name,bool needs_caching,uint8_t cache_key[SHA1_DIGEST_LENGTH])254 compile_function(struct llvmpipe_context *ctx, struct gallivm_state *gallivm, LLVMValueRef function,
255 const char *func_name,
256 bool needs_caching,
257 uint8_t cache_key[SHA1_DIGEST_LENGTH])
258 {
259 gallivm_verify_function(gallivm, function);
260 gallivm_compile_module(gallivm);
261
262 void *function_ptr = func_to_pointer(gallivm_jit_function(gallivm, function, func_name));
263
264 if (needs_caching)
265 lp_disk_cache_insert_shader(llvmpipe_screen(ctx->pipe.screen), gallivm->cache, cache_key);
266
267 gallivm_free_ir(gallivm);
268
269 util_dynarray_append(&ctx->sampler_matrix.gallivms, struct gallivm_state *, gallivm);
270
271 return function_ptr;
272 }
273
274 static void *
compile_image_function(struct llvmpipe_context * ctx,struct lp_static_texture_state * texture,uint32_t op)275 compile_image_function(struct llvmpipe_context *ctx, struct lp_static_texture_state *texture, uint32_t op)
276 {
277 const struct util_format_description *desc = util_format_description(texture->format);
278 if (desc->colorspace != UTIL_FORMAT_COLORSPACE_ZS && !lp_storage_render_image_format_supported(texture->format))
279 return NULL;
280
281 bool ms = op >= LP_TOTAL_IMAGE_OP_COUNT / 2;
282 if (ms)
283 op -= LP_TOTAL_IMAGE_OP_COUNT / 2;
284
285 struct lp_img_params params = { 0 };
286
287 params.img_op = op;
288 if (op >= LP_IMG_OP_COUNT - 1) {
289 params.img_op = LP_IMG_ATOMIC;
290 params.op = op - (LP_IMG_OP_COUNT - 1);
291 } else if (op != LP_IMG_LOAD && op != LP_IMG_LOAD_SPARSE && op != LP_IMG_STORE) {
292 params.img_op = LP_IMG_ATOMIC_CAS;
293 }
294
295 /* Loads need to support a wider range of formats for input attachments. */
296 if (params.img_op != LP_IMG_LOAD)
297 if (texture->format != PIPE_FORMAT_NONE && !lp_storage_image_format_supported(texture->format))
298 return NULL;
299
300 uint8_t cache_key[SHA1_DIGEST_LENGTH];
301 struct mesa_sha1 hash_ctx;
302 _mesa_sha1_init(&hash_ctx);
303 _mesa_sha1_update(&hash_ctx, image_function_base_hash, strlen(image_function_base_hash));
304 _mesa_sha1_update(&hash_ctx, texture, sizeof(*texture));
305 _mesa_sha1_update(&hash_ctx, &op, sizeof(op));
306 _mesa_sha1_update(&hash_ctx, &ms, sizeof(ms));
307 _mesa_sha1_final(&hash_ctx, cache_key);
308
309 struct lp_cached_code cached = { 0 };
310 lp_disk_cache_find_shader(llvmpipe_screen(ctx->pipe.screen), &cached, cache_key);
311 bool needs_caching = !cached.data_size;
312
313 struct gallivm_state *gallivm = gallivm_create("sample_function", get_llvm_context(ctx), &cached);
314
315 struct lp_image_static_state state = {
316 .image_state = *texture,
317 };
318 struct lp_build_image_soa *image_soa = lp_bld_llvm_image_soa_create(&state, 1);
319
320 struct lp_type type;
321 memset(&type, 0, sizeof type);
322 type.floating = true; /* floating point values */
323 type.sign = true; /* values are signed */
324 type.norm = false; /* values are not limited to [0,1] or [-1,1] */
325 type.width = 32; /* 32-bit float */
326 type.length = MIN2(lp_native_vector_width / 32, 16); /* n*4 elements per vector */
327
328 struct lp_compute_shader_variant cs = { .gallivm = gallivm };
329 lp_jit_init_cs_types(&cs);
330
331 params.type = type;
332 params.target = texture->target;
333 params.resources_type = cs.jit_resources_type;
334 params.format = texture->format;
335
336 LLVMTypeRef function_type = lp_build_image_function_type(gallivm, ¶ms, ms);
337 if (!function_type) {
338 free(image_soa);
339 gallivm_destroy(gallivm);
340 return NULL;
341 }
342
343 LLVMValueRef function = LLVMAddFunction(gallivm->module, "image", function_type);
344
345 uint32_t arg_index = 0;
346
347 gallivm->texture_descriptor = LLVMGetParam(function, arg_index++);
348
349 if (params.img_op != LP_IMG_LOAD && params.img_op != LP_IMG_LOAD_SPARSE)
350 params.exec_mask = LLVMGetParam(function, arg_index++);
351
352 LLVMValueRef coords[3];
353 params.coords = coords;
354 for (uint32_t i = 0; i < 3; i++)
355 coords[i] = LLVMGetParam(function, arg_index++);
356
357 if (ms)
358 params.ms_index = LLVMGetParam(function, arg_index++);
359
360 if (params.img_op != LP_IMG_LOAD && params.img_op != LP_IMG_LOAD_SPARSE)
361 for (uint32_t i = 0; i < 4; i++)
362 params.indata[i] = LLVMGetParam(function, arg_index++);
363
364 if (params.img_op == LP_IMG_ATOMIC_CAS)
365 for (uint32_t i = 0; i < 4; i++)
366 params.indata2[i] = LLVMGetParam(function, arg_index++);
367
368 LLVMBuilderRef old_builder = gallivm->builder;
369 LLVMBasicBlockRef block = LLVMAppendBasicBlockInContext(gallivm->context, function, "entry");
370 gallivm->builder = LLVMCreateBuilderInContext(gallivm->context);
371 LLVMPositionBuilderAtEnd(gallivm->builder, block);
372
373 LLVMValueRef outdata[5] = { 0 };
374 lp_build_img_op_soa(texture, lp_build_image_soa_dynamic_state(image_soa), gallivm, ¶ms, outdata);
375
376 for (uint32_t i = 1; i < 4; i++)
377 if (!outdata[i])
378 outdata[i] = outdata[0];
379
380 if (outdata[4])
381 outdata[4] = LLVMBuildZExt(gallivm->builder, outdata[4], lp_build_int_vec_type(gallivm, lp_int_type(type)), "");
382 else
383 outdata[4] = lp_build_one(gallivm, lp_int_type(type));
384
385 if (params.img_op != LP_IMG_STORE)
386 LLVMBuildAggregateRet(gallivm->builder, outdata, params.img_op == LP_IMG_LOAD_SPARSE ? 5 : 4);
387 else
388 LLVMBuildRetVoid(gallivm->builder);
389
390 LLVMDisposeBuilder(gallivm->builder);
391 gallivm->builder = old_builder;
392
393 free(image_soa);
394
395 return compile_function(ctx, gallivm, function, "image", needs_caching, cache_key);
396 }
397
398 static void *
compile_sample_function(struct llvmpipe_context * ctx,struct lp_static_texture_state * texture,struct lp_static_sampler_state * sampler,uint32_t sample_key)399 compile_sample_function(struct llvmpipe_context *ctx, struct lp_static_texture_state *texture,
400 struct lp_static_sampler_state *sampler, uint32_t sample_key)
401 {
402 enum lp_sampler_lod_control lod_control = (sample_key & LP_SAMPLER_LOD_CONTROL_MASK) >> LP_SAMPLER_LOD_CONTROL_SHIFT;
403
404 bool supported = true;
405 if (texture->format != PIPE_FORMAT_NONE) {
406 enum lp_sampler_op_type op_type = (sample_key & LP_SAMPLER_OP_TYPE_MASK) >> LP_SAMPLER_OP_TYPE_SHIFT;
407 if (op_type != LP_SAMPLER_OP_LODQ)
408 if ((sampler->compare_mode == PIPE_TEX_COMPARE_NONE) == !!(sample_key & LP_SAMPLER_SHADOW))
409 supported = false;
410
411 /* Skip integer formats which would cause a type mismatch in the compare function. */
412 const struct util_format_description *desc = util_format_description(texture->format);
413 struct lp_type texel_type = {
414 .floating = true,
415 .width = 32,
416 .length = 1,
417 };
418 texel_type = lp_build_texel_type(texel_type, desc);
419 if ((sample_key & LP_SAMPLER_SHADOW) && !texel_type.floating)
420 supported = false;
421
422 if (texture_dims(texture->target) != 2 && op_type == LP_SAMPLER_OP_GATHER)
423 supported = false;
424
425 if (op_type != LP_SAMPLER_OP_FETCH) {
426 if (!sampler->normalized_coords) {
427 if (texture->target != PIPE_TEXTURE_1D && texture->target != PIPE_TEXTURE_2D &&
428 texture->target != PIPE_TEXTURE_1D_ARRAY && texture->target != PIPE_TEXTURE_2D_ARRAY)
429 supported = false;
430
431 if (!texture->level_zero_only)
432 supported = false;
433 }
434 }
435
436 if (util_format_is_pure_integer(texture->format) &&
437 (sampler->min_img_filter == PIPE_TEX_FILTER_LINEAR ||
438 sampler->min_mip_filter == PIPE_TEX_MIPFILTER_LINEAR ||
439 sampler->mag_img_filter == PIPE_TEX_FILTER_LINEAR))
440 supported = false;
441
442 if (sampler->aniso) {
443 if (texture_dims(texture->target) != 2)
444 supported = false;
445
446 if (util_format_is_pure_integer(texture->format))
447 supported = false;
448 }
449
450 if (util_format_get_num_planes(texture->format) > 1)
451 return NULL;
452
453 uint32_t bind = op_type == LP_SAMPLER_OP_FETCH ? PIPE_BIND_CONSTANT_BUFFER : PIPE_BIND_SAMPLER_VIEW;
454 if (!ctx->pipe.screen->is_format_supported(ctx->pipe.screen, texture->format, texture->target, 0, 0, bind))
455 supported = false;
456 }
457
458 uint8_t cache_key[SHA1_DIGEST_LENGTH];
459 struct mesa_sha1 hash_ctx;
460 _mesa_sha1_init(&hash_ctx);
461 _mesa_sha1_update(&hash_ctx, sample_function_base_hash, strlen(sample_function_base_hash));
462 _mesa_sha1_update(&hash_ctx, texture, sizeof(*texture));
463 _mesa_sha1_update(&hash_ctx, sampler, sizeof(*sampler));
464 _mesa_sha1_update(&hash_ctx, &sample_key, sizeof(sample_key));
465 _mesa_sha1_final(&hash_ctx, cache_key);
466
467 struct lp_cached_code cached = { 0 };
468 lp_disk_cache_find_shader(llvmpipe_screen(ctx->pipe.screen), &cached, cache_key);
469 bool needs_caching = !cached.data_size;
470
471 struct gallivm_state *gallivm = gallivm_create("sample_function", get_llvm_context(ctx), &cached);
472
473 struct lp_sampler_static_state state = {
474 .texture_state = *texture,
475 .sampler_state = *sampler,
476 };
477 struct lp_build_sampler_soa *sampler_soa = lp_llvm_sampler_soa_create(&state, 1);
478
479 struct lp_type type;
480 memset(&type, 0, sizeof type);
481 type.floating = true; /* floating point values */
482 type.sign = true; /* values are signed */
483 type.norm = false; /* values are not limited to [0,1] or [-1,1] */
484 type.width = 32; /* 32-bit float */
485 type.length = MIN2(lp_native_vector_width / 32, 16); /* n*4 elements per vector */
486
487 struct lp_compute_shader_variant cs = { .gallivm = gallivm };
488 lp_jit_init_cs_types(&cs);
489
490 LLVMTypeRef function_type = lp_build_sample_function_type(gallivm, sample_key);
491 LLVMValueRef function = LLVMAddFunction(gallivm->module, "sample", function_type);
492
493 uint32_t arg_index = 0;
494
495 gallivm->texture_descriptor = LLVMGetParam(function, arg_index++);
496 gallivm->sampler_descriptor = LLVMGetParam(function, arg_index++);
497
498 LLVMValueRef coords[5];
499 for (unsigned i = 0; i < 4; i++)
500 coords[i] = LLVMGetParam(function, arg_index++);
501
502 if (sample_key & LP_SAMPLER_SHADOW)
503 coords[4] = LLVMGetParam(function, arg_index++);
504 else
505 coords[4] = lp_build_undef(gallivm, type);
506
507 LLVMValueRef ms_index = NULL;
508 if (sample_key & LP_SAMPLER_FETCH_MS)
509 ms_index = LLVMGetParam(function, arg_index++);
510
511 LLVMValueRef offsets[3] = { 0 };
512 if (sample_key & LP_SAMPLER_OFFSETS)
513 for (unsigned i = 0; i < 3; i++)
514 offsets[i] = LLVMGetParam(function, arg_index++);
515
516 LLVMValueRef lod = NULL;
517 if (lod_control == LP_SAMPLER_LOD_BIAS || lod_control == LP_SAMPLER_LOD_EXPLICIT)
518 lod = LLVMGetParam(function, arg_index++);
519
520 LLVMBuilderRef old_builder = gallivm->builder;
521 LLVMBasicBlockRef block = LLVMAppendBasicBlockInContext(gallivm->context, function, "entry");
522 gallivm->builder = LLVMCreateBuilderInContext(gallivm->context);
523 LLVMPositionBuilderAtEnd(gallivm->builder, block);
524
525 LLVMValueRef texel_out[5] = { 0 };
526 if (supported) {
527 lp_build_sample_soa_code(gallivm, texture, sampler, lp_build_sampler_soa_dynamic_state(sampler_soa),
528 type, sample_key, 0, 0, cs.jit_resources_type, NULL, cs.jit_cs_thread_data_type,
529 NULL, coords, offsets, NULL, lod, ms_index, texel_out);
530 } else {
531 lp_build_sample_nop(gallivm, lp_build_texel_type(type, util_format_description(texture->format)), coords, texel_out);
532 }
533
534 if (texel_out[4])
535 texel_out[4] = LLVMBuildZExt(gallivm->builder, texel_out[4], lp_build_int_vec_type(gallivm, lp_int_type(type)), "");
536 else
537 texel_out[4] = lp_build_one(gallivm, lp_int_type(type));
538
539 LLVMBuildAggregateRet(gallivm->builder, texel_out, 5);
540
541 LLVMDisposeBuilder(gallivm->builder);
542 gallivm->builder = old_builder;
543
544 free(sampler_soa);
545
546 return compile_function(ctx, gallivm, function, "sample", needs_caching, cache_key);
547 }
548
549 static uint64_t
get_sample_function(uint64_t _matrix,uint64_t _texture_functions,uint64_t _sampler_desc,uint32_t sample_key)550 get_sample_function(uint64_t _matrix, uint64_t _texture_functions, uint64_t _sampler_desc, uint32_t sample_key)
551 {
552 struct lp_sampler_matrix *matrix = (void *)(uintptr_t)_matrix;
553 struct lp_descriptor *sampler_desc = (void *)(uintptr_t)_sampler_desc;
554 uint32_t sampler_index = sampler_desc->texture.sampler_index;
555
556 struct lp_texture_functions *texture_functions = (void *)(uintptr_t)_texture_functions;
557 struct sample_function_cache_key key = {
558 .texture_functions = texture_functions,
559 .sampler_index = sampler_index,
560 .sample_key = sample_key,
561 };
562
563 void *result;
564 {
565 struct hash_entry *entry = _mesa_hash_table_search(acquire_latest_sample_function_cache(matrix), &key);
566 result = entry ? entry->data : NULL;
567 }
568
569 if (!result) {
570 simple_mtx_lock(&matrix->lock);
571 /* Check once more in case the cache got modified between the first check and acquiring the lock. */
572 struct hash_table *current_cache = acquire_latest_sample_function_cache(matrix);
573 struct hash_entry *entry = _mesa_hash_table_search(current_cache, &key);
574 result = entry ? entry->data : NULL;
575 if (!result) {
576 result = compile_sample_function(matrix->ctx, &texture_functions->state, matrix->samplers + sampler_index, sample_key);
577 struct sample_function_cache_key *allocated_key = malloc(sizeof(struct sample_function_cache_key));
578 *allocated_key = key;
579 /* RCU style update: swap in an updated copy of the cache.
580 / Old caches are kept as trash to be safely deleted later. */
581 struct hash_table *new_cache = _mesa_hash_table_clone(current_cache, NULL);
582 _mesa_hash_table_insert(new_cache, allocated_key, result);
583 replace_sample_function_cache_locked(matrix, new_cache);
584 }
585 simple_mtx_unlock(&matrix->lock);
586 }
587
588 return (uint64_t)(uintptr_t)result;
589 }
590
591 static LLVMTypeRef
lp_build_compile_function_type(struct gallivm_state * gallivm)592 lp_build_compile_function_type(struct gallivm_state *gallivm)
593 {
594 LLVMTypeRef param_types[4] = {
595 LLVMInt64TypeInContext(gallivm->context),
596 LLVMInt64TypeInContext(gallivm->context),
597 LLVMInt64TypeInContext(gallivm->context),
598 LLVMInt32TypeInContext(gallivm->context),
599 };
600 LLVMTypeRef ret_type = LLVMInt64TypeInContext(gallivm->context);
601
602 return LLVMFunctionType(ret_type, param_types, ARRAY_SIZE(param_types), false);
603 }
604
605 static void *
compile_jit_sample_function(struct llvmpipe_context * ctx,uint32_t sample_key)606 compile_jit_sample_function(struct llvmpipe_context *ctx, uint32_t sample_key)
607 {
608 uint8_t cache_key[SHA1_DIGEST_LENGTH];
609 struct mesa_sha1 hash_ctx;
610 _mesa_sha1_init(&hash_ctx);
611 _mesa_sha1_update(&hash_ctx, jit_sample_function_base_hash, strlen(jit_sample_function_base_hash));
612 _mesa_sha1_update(&hash_ctx, &sample_key, sizeof(sample_key));
613 _mesa_sha1_final(&hash_ctx, cache_key);
614
615 struct lp_cached_code cached = { 0 };
616 lp_disk_cache_find_shader(llvmpipe_screen(ctx->pipe.screen), &cached, cache_key);
617 bool needs_caching = !cached.data_size;
618
619 struct gallivm_state *gallivm = gallivm_create("jit_sample_function", get_llvm_context(ctx), &cached);
620
621 struct lp_type type;
622 memset(&type, 0, sizeof type);
623 type.floating = true; /* floating point values */
624 type.sign = true; /* values are signed */
625 type.norm = false; /* values are not limited to [0,1] or [-1,1] */
626 type.width = 32; /* 32-bit float */
627 type.length = MIN2(lp_native_vector_width / 32, 16); /* n*4 elements per vector */
628
629 struct lp_compute_shader_variant cs = { .gallivm = gallivm };
630 lp_jit_init_cs_types(&cs);
631
632 LLVMTypeRef function_type = lp_build_sample_function_type(gallivm, sample_key);
633 LLVMValueRef function = LLVMAddFunction(gallivm->module, "sample", function_type);
634
635 uint32_t arg_index = 0;
636 LLVMValueRef texture_descriptor = LLVMGetParam(function, arg_index++);
637 LLVMValueRef sampler_descriptor = LLVMGetParam(function, arg_index++);
638
639 LLVMBuilderRef old_builder = gallivm->builder;
640 LLVMBasicBlockRef block = LLVMAppendBasicBlockInContext(gallivm->context, function, "entry");
641 gallivm->builder = LLVMCreateBuilderInContext(gallivm->context);
642 LLVMBuilderRef builder = gallivm->builder;
643 LLVMPositionBuilderAtEnd(gallivm->builder, block);
644
645 LLVMValueRef functions_offset =
646 lp_build_const_int64(gallivm, offsetof(struct lp_descriptor, functions));
647 LLVMValueRef functions_ptr =
648 LLVMBuildAdd(builder, texture_descriptor, functions_offset, "");
649
650 LLVMTypeRef functions_ptr_type = LLVMInt64TypeInContext(gallivm->context);
651 LLVMTypeRef functions_ptr_ptr_type = LLVMPointerType(functions_ptr_type, 0);
652
653 functions_ptr = LLVMBuildIntToPtr(builder, functions_ptr, functions_ptr_ptr_type, "");
654 /* struct lp_texture_functions * */
655 functions_ptr = LLVMBuildLoad2(builder, functions_ptr_type, functions_ptr, "");
656
657 LLVMValueRef matrix_offset =
658 lp_build_const_int64(gallivm, offsetof(struct lp_texture_functions, matrix));
659 LLVMValueRef matrix_ptr = LLVMBuildAdd(builder, functions_ptr, matrix_offset, "");
660
661 matrix_ptr = LLVMBuildIntToPtr(builder, matrix_ptr, functions_ptr_ptr_type, "");
662 /* struct lp_sampler_matrix * */
663 matrix_ptr = LLVMBuildLoad2(builder, functions_ptr_type, matrix_ptr, "");
664
665 LLVMTypeRef compile_function_type = lp_build_compile_function_type(gallivm);
666 LLVMTypeRef compile_function_ptr_type = LLVMPointerType(compile_function_type, 0);
667 LLVMTypeRef compile_function_ptr_ptr_type = LLVMPointerType(compile_function_ptr_type, 0);
668
669 LLVMValueRef compile_function_offset =
670 lp_build_const_int64(gallivm, offsetof(struct lp_sampler_matrix, compile_function));
671 LLVMValueRef compile_function_ptr =
672 LLVMBuildAdd(builder, matrix_ptr, compile_function_offset, "");
673
674 compile_function_ptr =
675 LLVMBuildIntToPtr(builder, compile_function_ptr, compile_function_ptr_ptr_type, "");
676 /* struct lp_texture_functions * */
677 compile_function_ptr =
678 LLVMBuildLoad2(builder, compile_function_ptr_type, compile_function_ptr, "");
679
680 LLVMValueRef compile_args[4] = {
681 matrix_ptr, functions_ptr, sampler_descriptor, lp_build_const_int32(gallivm, sample_key)
682 };
683
684 LLVMValueRef sample_function =
685 LLVMBuildCall2(builder, compile_function_type, compile_function_ptr,
686 compile_args, ARRAY_SIZE(compile_args), "");
687
688 sample_function = LLVMBuildIntToPtr(builder, sample_function, LLVMPointerType(function_type, 0), "");
689
690 LLVMValueRef args[LP_MAX_TEX_FUNC_ARGS];
691 uint32_t num_args = 0;
692
693 LLVMValueRef arg = LLVMGetFirstParam(function);
694 while (true) {
695 args[num_args++] = arg;
696 if (arg == LLVMGetLastParam(function))
697 break;
698
699 arg = LLVMGetNextParam(arg);
700 }
701
702 LLVMValueRef result = LLVMBuildCall2(builder, function_type, sample_function, args, num_args, "");
703 LLVMBuildRet(gallivm->builder, result);
704
705 LLVMDisposeBuilder(gallivm->builder);
706 gallivm->builder = old_builder;
707
708 return compile_function(ctx, gallivm, function, "sample", needs_caching, cache_key);
709 }
710
711 static void *
compile_size_function(struct llvmpipe_context * ctx,struct lp_static_texture_state * texture,bool samples)712 compile_size_function(struct llvmpipe_context *ctx, struct lp_static_texture_state *texture, bool samples)
713 {
714 uint8_t cache_key[SHA1_DIGEST_LENGTH];
715 struct mesa_sha1 hash_ctx;
716 _mesa_sha1_init(&hash_ctx);
717 _mesa_sha1_update(&hash_ctx, size_function_base_hash, strlen(size_function_base_hash));
718 _mesa_sha1_update(&hash_ctx, texture, sizeof(*texture));
719 _mesa_sha1_update(&hash_ctx, &samples, sizeof(samples));
720 _mesa_sha1_final(&hash_ctx, cache_key);
721
722 struct lp_cached_code cached = { 0 };
723 lp_disk_cache_find_shader(llvmpipe_screen(ctx->pipe.screen), &cached, cache_key);
724 bool needs_caching = !cached.data_size;
725
726 struct gallivm_state *gallivm = gallivm_create("sample_function", get_llvm_context(ctx), &cached);
727
728 struct lp_sampler_static_state state = {
729 .texture_state = *texture,
730 };
731 struct lp_build_sampler_soa *sampler_soa = lp_llvm_sampler_soa_create(&state, 1);
732
733 struct lp_type type;
734 memset(&type, 0, sizeof type);
735 type.floating = true; /* floating point values */
736 type.sign = true; /* values are signed */
737 type.norm = false; /* values are not limited to [0,1] or [-1,1] */
738 type.width = 32; /* 32-bit float */
739 type.length = MIN2(lp_native_vector_width / 32, 16); /* n*4 elements per vector */
740
741 struct lp_compute_shader_variant cs = { .gallivm = gallivm };
742 lp_jit_init_cs_types(&cs);
743
744 struct lp_sampler_size_query_params params = {
745 .int_type = lp_int_type(type),
746 .target = texture->target,
747 .resources_type = cs.jit_resources_type,
748 .is_sviewinfo = true,
749 .samples_only = samples,
750 .ms = samples,
751 };
752
753 if (params.target == PIPE_TEXTURE_1D)
754 params.target = PIPE_TEXTURE_1D_ARRAY;
755 else if (params.target == PIPE_TEXTURE_2D)
756 params.target = PIPE_TEXTURE_2D_ARRAY;
757 else if (params.target == PIPE_TEXTURE_CUBE)
758 params.target = PIPE_TEXTURE_CUBE_ARRAY;
759
760 LLVMTypeRef function_type = lp_build_size_function_type(gallivm, ¶ms);
761 LLVMValueRef function = LLVMAddFunction(gallivm->module, "size", function_type);
762
763 uint32_t arg_index = 0;
764
765 gallivm->texture_descriptor = LLVMGetParam(function, arg_index++);
766
767 if (!samples)
768 params.explicit_lod = LLVMGetParam(function, arg_index++);
769
770 LLVMBuilderRef old_builder = gallivm->builder;
771 LLVMBasicBlockRef block = LLVMAppendBasicBlockInContext(gallivm->context, function, "entry");
772 gallivm->builder = LLVMCreateBuilderInContext(gallivm->context);
773 LLVMPositionBuilderAtEnd(gallivm->builder, block);
774
775 LLVMValueRef out_sizes[4] = { 0 };
776 params.sizes_out = out_sizes;
777 lp_build_size_query_soa(gallivm, texture, lp_build_sampler_soa_dynamic_state(sampler_soa), ¶ms);
778
779 for (uint32_t i = 0; i < 4; i++)
780 if (!out_sizes[i])
781 out_sizes[i] = lp_build_const_int_vec(gallivm, params.int_type, 0);
782
783 LLVMBuildAggregateRet(gallivm->builder, out_sizes, 4);
784
785 LLVMDisposeBuilder(gallivm->builder);
786 gallivm->builder = old_builder;
787
788 free(sampler_soa);
789
790 return compile_function(ctx, gallivm, function, "size", needs_caching, cache_key);
791 }
792
793 static void
compile_sample_functions(struct llvmpipe_context * ctx,struct lp_static_texture_state * texture,struct lp_static_sampler_state * sampler,void *** dst)794 compile_sample_functions(struct llvmpipe_context *ctx, struct lp_static_texture_state *texture,
795 struct lp_static_sampler_state *sampler, void ***dst)
796 {
797 void **functions;
798 if (*dst) {
799 functions = *dst;
800 } else {
801 functions = calloc(LP_SAMPLE_KEY_COUNT, sizeof(void *));
802 *dst = functions;
803 }
804
805 bool has_sampler = !!sampler;
806
807 struct lp_static_sampler_state dummy_sampler = { 0 };
808 if (!sampler)
809 sampler = &dummy_sampler;
810
811 struct lp_sampler_matrix *matrix = &ctx->sampler_matrix;
812 for (uint32_t sample_key = 0; sample_key < LP_SAMPLE_KEY_COUNT; sample_key++) {
813 if (!BITSET_TEST(matrix->sample_keys, sample_key))
814 continue;
815
816 enum lp_sampler_op_type op_type = (sample_key & LP_SAMPLER_OP_TYPE_MASK) >> LP_SAMPLER_OP_TYPE_SHIFT;
817 if (has_sampler && op_type == LP_SAMPLER_OP_FETCH)
818 continue;
819
820 if (!functions[sample_key]) {
821 if (has_sampler)
822 functions[sample_key] = matrix->jit_sample_functions[sample_key];
823 else
824 functions[sample_key] = compile_sample_function(ctx, texture, sampler, sample_key);
825 }
826 }
827 }
828
829 static void
llvmpipe_register_texture(struct llvmpipe_context * ctx,struct lp_static_texture_state * state,bool sampled)830 llvmpipe_register_texture(struct llvmpipe_context *ctx, struct lp_static_texture_state *state, bool sampled)
831 {
832 struct lp_sampler_matrix *matrix = &ctx->sampler_matrix;
833
834 bool packed = true;
835 uint32_t dst_index = matrix->texture_count;
836 for (uint32_t i = 0; i < matrix->texture_count; i++) {
837 if (memcmp(&matrix->textures[i]->state, state, sizeof(struct lp_static_texture_state)))
838 continue;
839
840 bool has_functions = sampled ? matrix->textures[i]->sampled : matrix->textures[i]->storage;
841 if (has_functions)
842 return;
843
844 packed = false;
845 dst_index = i;
846 break;
847 }
848
849 struct lp_texture_functions *entry;
850 if (packed) {
851 matrix->texture_count++;
852 matrix->textures = realloc(matrix->textures, matrix->texture_count * sizeof(struct lp_texture_functions *));
853
854 entry = calloc(1, sizeof(struct lp_texture_functions));
855 matrix->textures[dst_index] = entry;
856
857 entry->state = *state;
858 entry->image_functions = calloc(LP_TOTAL_IMAGE_OP_COUNT, sizeof(void **));
859 entry->matrix = matrix;
860 } else {
861 entry = matrix->textures[dst_index];
862 }
863
864 if (sampled)
865 entry->sampled = true;
866 else
867 entry->storage = true;
868
869 simple_mtx_lock(&matrix->lock);
870
871 if (entry->sampled) {
872 if (entry->sample_functions) {
873 entry->sample_functions = realloc(entry->sample_functions, matrix->sampler_count * sizeof(void **));
874 memset(entry->sample_functions + entry->sampler_count, 0, (matrix->sampler_count - entry->sampler_count) * sizeof(void **));
875 } else {
876 entry->sample_functions = calloc(matrix->sampler_count, sizeof(void **));
877 }
878 entry->sampler_count = matrix->sampler_count;
879
880 if (state->format == PIPE_FORMAT_NONE) {
881 if (matrix->sampler_count)
882 compile_sample_functions(ctx, state, NULL, entry->sample_functions);
883 for (uint32_t i = 1; i < matrix->sampler_count; i++)
884 entry->sample_functions[i] = entry->sample_functions[0];
885 } else {
886 for (uint32_t i = 0; i < matrix->sampler_count; i++)
887 compile_sample_functions(ctx, state, matrix->samplers + i, entry->sample_functions + i);
888 }
889
890 compile_sample_functions(ctx, state, NULL, &entry->fetch_functions);
891
892 if (!entry->size_function)
893 entry->size_function = compile_size_function(ctx, state, false);
894
895 if (!entry->samples_function)
896 entry->samples_function = compile_size_function(ctx, state, true);
897 }
898
899 if (entry->storage) {
900 uint32_t image_op;
901 BITSET_FOREACH_SET (image_op, matrix->image_ops, LP_TOTAL_IMAGE_OP_COUNT)
902 if (!entry->image_functions[image_op])
903 entry->image_functions[image_op] = compile_image_function(ctx, state, image_op);
904 }
905
906 simple_mtx_unlock(&matrix->lock);
907 }
908
909 static void
llvmpipe_register_sampler(struct llvmpipe_context * ctx,struct lp_static_sampler_state * state)910 llvmpipe_register_sampler(struct llvmpipe_context *ctx, struct lp_static_sampler_state *state)
911 {
912 struct lp_sampler_matrix *matrix = &ctx->sampler_matrix;
913 for (uint32_t i = 0; i < matrix->sampler_count; i++)
914 if (!memcmp(matrix->samplers + i, state, sizeof(struct lp_static_sampler_state)))
915 return;
916
917 matrix->sampler_count++;
918 matrix->samplers = realloc(matrix->samplers, matrix->sampler_count * sizeof(struct lp_static_sampler_state));
919
920 matrix->samplers[matrix->sampler_count - 1] = *state;
921
922 simple_mtx_lock(&matrix->lock);
923
924 for (uint32_t i = 0; i < matrix->texture_count; i++) {
925 struct lp_texture_functions *texture = matrix->textures[i];
926 if (!texture->sampled)
927 continue;
928
929 texture->sampler_count = matrix->sampler_count;
930 texture->sample_functions = realloc(texture->sample_functions, matrix->sampler_count * sizeof(void **));
931
932 void ***dst = texture->sample_functions + (matrix->sampler_count - 1);
933
934 if (texture->state.format == PIPE_FORMAT_NONE) {
935 if (matrix->sampler_count == 1) {
936 *dst = NULL;
937 compile_sample_functions(ctx, &texture->state, NULL, dst);
938 } else {
939 *dst = texture->sample_functions[0];
940 }
941
942 continue;
943 }
944
945 *dst = NULL;
946 compile_sample_functions(ctx, &texture->state, state, dst);
947 }
948
949 simple_mtx_unlock(&matrix->lock);
950 }
951
952 static void
register_sample_key(struct llvmpipe_context * ctx,uint32_t sample_key)953 register_sample_key(struct llvmpipe_context *ctx, uint32_t sample_key)
954 {
955 struct lp_sampler_matrix *matrix = &ctx->sampler_matrix;
956 if (BITSET_TEST(matrix->sample_keys, sample_key))
957 return;
958
959 BITSET_SET(matrix->sample_keys, sample_key);
960
961 simple_mtx_lock(&matrix->lock);
962
963 matrix->jit_sample_functions[sample_key] = compile_jit_sample_function(ctx, sample_key);
964
965 for (uint32_t texture_index = 0; texture_index < matrix->texture_count; texture_index++) {
966 struct lp_texture_functions *texture = matrix->textures[texture_index];
967 if (!texture->sampled)
968 continue;
969
970 enum lp_sampler_op_type op_type = (sample_key & LP_SAMPLER_OP_TYPE_MASK) >> LP_SAMPLER_OP_TYPE_SHIFT;
971 if (op_type == LP_SAMPLER_OP_FETCH) {
972 struct lp_static_sampler_state dummy_sampler = { 0 };
973 texture->fetch_functions[sample_key] = compile_sample_function(ctx, &texture->state, &dummy_sampler, sample_key);
974 continue;
975 }
976
977 if (texture->state.format == PIPE_FORMAT_NONE) {
978 if (matrix->sampler_count) {
979 struct lp_static_sampler_state dummy_sampler = { 0 };
980 texture->sample_functions[0][sample_key] = compile_sample_function(ctx, &texture->state, &dummy_sampler, sample_key);
981 }
982 continue;
983 }
984
985 for (uint32_t sampler_index = 0; sampler_index < matrix->sampler_count; sampler_index++)
986 texture->sample_functions[sampler_index][sample_key] = matrix->jit_sample_functions[sample_key];
987 }
988
989 simple_mtx_unlock(&matrix->lock);
990 }
991
992 static void
register_image_op(struct llvmpipe_context * ctx,uint32_t op)993 register_image_op(struct llvmpipe_context *ctx, uint32_t op)
994 {
995 struct lp_sampler_matrix *matrix = &ctx->sampler_matrix;
996 if (BITSET_TEST(matrix->image_ops, op))
997 return;
998
999 BITSET_SET(matrix->image_ops, op);
1000
1001 simple_mtx_lock(&matrix->lock);
1002
1003 for (uint32_t texture_index = 0; texture_index < matrix->texture_count; texture_index++) {
1004 struct lp_texture_functions *texture = matrix->textures[texture_index];
1005 if (texture->storage)
1006 texture->image_functions[op] = compile_image_function(ctx, &texture->state, op);
1007 }
1008
1009 simple_mtx_unlock(&matrix->lock);
1010 }
1011
1012 static bool
register_instr(nir_builder * b,nir_instr * instr,void * data)1013 register_instr(nir_builder *b, nir_instr *instr, void *data)
1014 {
1015 struct llvmpipe_context *ctx = data;
1016
1017 if (instr->type == nir_instr_type_tex) {
1018 nir_tex_instr *tex = nir_instr_as_tex(instr);
1019 uint32_t sample_key = lp_build_nir_sample_key(b->shader->info.stage, tex);
1020
1021 register_sample_key(ctx, sample_key);
1022 } else if (instr->type == nir_instr_type_intrinsic) {
1023 nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
1024
1025 struct lp_img_params params;
1026 lp_img_op_from_intrinsic(¶ms, intrin);
1027
1028 if (params.img_op == -1)
1029 return false;
1030
1031 uint32_t op = params.img_op;
1032 if (op == LP_IMG_ATOMIC_CAS)
1033 op--;
1034 else if (op == LP_IMG_ATOMIC)
1035 op = params.op + (LP_IMG_OP_COUNT - 1);
1036
1037 if (nir_intrinsic_image_dim(intrin) == GLSL_SAMPLER_DIM_MS ||
1038 nir_intrinsic_image_dim(intrin) == GLSL_SAMPLER_DIM_SUBPASS_MS)
1039 op += LP_TOTAL_IMAGE_OP_COUNT / 2;
1040
1041 register_image_op(ctx, op);
1042 }
1043
1044 return false;
1045 }
1046
1047 void
llvmpipe_register_shader(struct pipe_context * ctx,const struct pipe_shader_state * shader)1048 llvmpipe_register_shader(struct pipe_context *ctx, const struct pipe_shader_state *shader)
1049 {
1050 if (shader->type == PIPE_SHADER_IR_NIR)
1051 nir_shader_instructions_pass(shader->ir.nir, register_instr, nir_metadata_all, ctx);
1052 }
1053
1054 void
llvmpipe_clear_sample_functions_cache(struct llvmpipe_context * ctx,struct pipe_fence_handle ** fence)1055 llvmpipe_clear_sample_functions_cache(struct llvmpipe_context *ctx, struct pipe_fence_handle **fence)
1056 {
1057 if (!fence)
1058 return;
1059
1060 struct lp_sampler_matrix *matrix = &ctx->sampler_matrix;
1061
1062 /* If the cache is empty, there is nothing to do. */
1063 if (!_mesa_hash_table_num_entries(acquire_latest_sample_function_cache(matrix)))
1064 return;
1065
1066 ctx->pipe.screen->fence_finish(ctx->pipe.screen, NULL, *fence, OS_TIMEOUT_INFINITE);
1067
1068 /* All work is finished, it's safe to move cache entries into the table. */
1069 hash_table_foreach_remove(acquire_latest_sample_function_cache(matrix), entry) {
1070 struct sample_function_cache_key *key = (void *)entry->key;
1071 key->texture_functions->sample_functions[key->sampler_index][key->sample_key] = entry->data;
1072 free(key);
1073 }
1074
1075 util_dynarray_foreach (&matrix->trash_caches, struct hash_table *, trash)
1076 _mesa_hash_table_destroy(*trash, NULL);
1077 util_dynarray_clear(&matrix->trash_caches);
1078 }
1079