1 /**************************************************************************
2 *
3 * Copyright 2009 VMware, Inc.
4 * All rights reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
21 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27
28 #include "gallivm/lp_bld_sample.h"
29 #include "gallivm/lp_bld_limits.h"
30 #include "gallivm/lp_bld_tgsi.h"
31 #include "gallivm/lp_bld_type.h"
32 #include "gallivm/lp_bld_init.h"
33 #include "gallivm/lp_bld_const.h"
34 #include "gallivm/lp_bld_sample.h"
35 #include "gallivm/lp_bld_jit_types.h"
36 #include "gallivm/lp_bld_jit_sample.h"
37 #include "gallivm/lp_bld_flow.h"
38
39 struct lp_bld_sampler_dynamic_state
40 {
41 struct lp_sampler_dynamic_state base;
42
43 const struct lp_sampler_static_state *static_state;
44 };
45
46 struct lp_bld_llvm_sampler_soa
47 {
48 struct lp_build_sampler_soa base;
49
50 struct lp_bld_sampler_dynamic_state dynamic_state;
51 unsigned nr_samplers;
52 };
53
54
55 struct lp_bld_image_dynamic_state
56 {
57 struct lp_sampler_dynamic_state base;
58
59 const struct lp_image_static_state *static_state;
60 };
61
62 struct lp_bld_llvm_image_soa
63 {
64 struct lp_build_image_soa base;
65
66 struct lp_bld_image_dynamic_state dynamic_state;
67 unsigned nr_images;
68 };
69
70 static LLVMValueRef
load_texture_functions_ptr(struct gallivm_state * gallivm,LLVMValueRef descriptor,uint32_t offset1,uint32_t offset2)71 load_texture_functions_ptr(struct gallivm_state *gallivm, LLVMValueRef descriptor,
72 uint32_t offset1, uint32_t offset2)
73 {
74 LLVMBuilderRef builder = gallivm->builder;
75
76 LLVMValueRef texture_base_offset = lp_build_const_int64(gallivm, offset1);
77 LLVMValueRef texture_base_ptr = LLVMBuildAdd(builder, descriptor, texture_base_offset, "");
78
79 LLVMTypeRef texture_base_type = LLVMInt64TypeInContext(gallivm->context);
80 LLVMTypeRef texture_base_ptr_type = LLVMPointerType(texture_base_type, 0);
81
82 texture_base_ptr = LLVMBuildIntToPtr(builder, texture_base_ptr, texture_base_ptr_type, "");
83 /* struct lp_texture_functions * */
84 LLVMValueRef texture_base = LLVMBuildLoad2(builder, texture_base_type, texture_base_ptr, "");
85
86 LLVMValueRef functions_offset = lp_build_const_int64(gallivm, offset2);
87 return LLVMBuildAdd(builder, texture_base, functions_offset, "");
88 }
89
90 static LLVMValueRef
widen_to_simd_width(struct gallivm_state * gallivm,LLVMValueRef value)91 widen_to_simd_width(struct gallivm_state *gallivm, LLVMValueRef value)
92 {
93 LLVMBuilderRef builder = gallivm->builder;
94 LLVMTypeRef type = LLVMTypeOf(value);
95
96 if (LLVMGetTypeKind(type) == LLVMVectorTypeKind) {
97 LLVMTypeRef element_type = LLVMGetElementType(type);
98 uint32_t element_count = LLVMGetVectorSize(type);
99 LLVMValueRef elements[8] = { 0 };
100 for (uint32_t i = 0; i < lp_native_vector_width / 32; i++) {
101 if (i < element_count)
102 elements[i] = LLVMBuildExtractElement(builder, value, lp_build_const_int32(gallivm, i), "");
103 else
104 elements[i] = LLVMConstNull(element_type);
105 }
106
107 LLVMTypeRef result_type = LLVMVectorType(element_type, lp_native_vector_width / 32);
108 LLVMValueRef result = LLVMGetUndef(result_type);
109 for (unsigned i = 0; i < lp_native_vector_width / 32; i++)
110 result = LLVMBuildInsertElement(builder, result, elements[i], lp_build_const_int32(gallivm, i), "");
111
112 return result;
113 }
114
115 return value;
116 }
117
118 static LLVMValueRef
truncate_to_type_width(struct gallivm_state * gallivm,LLVMValueRef value,struct lp_type target_type)119 truncate_to_type_width(struct gallivm_state *gallivm, LLVMValueRef value, struct lp_type target_type)
120 {
121 LLVMBuilderRef builder = gallivm->builder;
122 LLVMTypeRef type = LLVMTypeOf(value);
123
124 if (LLVMGetTypeKind(type) == LLVMVectorTypeKind) {
125 LLVMTypeRef element_type = LLVMGetElementType(type);
126
127 LLVMValueRef elements[8];
128 for (uint32_t i = 0; i < target_type.length; i++)
129 elements[i] = LLVMBuildExtractElement(builder, value, lp_build_const_int32(gallivm, i), "");
130
131 LLVMTypeRef result_type = LLVMVectorType(element_type, target_type.length);
132 LLVMValueRef result = LLVMGetUndef(result_type);
133 for (unsigned i = 0; i < target_type.length; i++)
134 result = LLVMBuildInsertElement(builder, result, elements[i], lp_build_const_int32(gallivm, i), "");
135
136 return result;
137 }
138
139 return value;
140 }
141
142 /**
143 * Fetch filtered values from texture.
144 * The 'texel' parameter returns four vectors corresponding to R, G, B, A.
145 */
146 static void
lp_bld_llvm_sampler_soa_emit_fetch_texel(const struct lp_build_sampler_soa * base,struct gallivm_state * gallivm,const struct lp_sampler_params * params)147 lp_bld_llvm_sampler_soa_emit_fetch_texel(const struct lp_build_sampler_soa *base,
148 struct gallivm_state *gallivm,
149 const struct lp_sampler_params *params)
150 {
151 struct lp_bld_llvm_sampler_soa *sampler = (struct lp_bld_llvm_sampler_soa *)base;
152 LLVMBuilderRef builder = gallivm->builder;
153
154 if (params->texture_resource) {
155 LLVMTypeRef out_data_type = lp_build_vec_type(gallivm, params->type);
156 LLVMTypeRef out_residency_type = lp_build_vec_type(gallivm, lp_int_type(params->type));
157
158 LLVMValueRef out_data[5];
159 for (uint32_t i = 0; i < ARRAY_SIZE(out_data) - 1; i++) {
160 out_data[i] = lp_build_alloca(gallivm, out_data_type, "");
161 }
162 out_data[ARRAY_SIZE(out_data) - 1] = lp_build_alloca(gallivm, out_residency_type, "");
163
164 struct lp_type uint_type = lp_uint_type(params->type);
165 LLVMValueRef uint_zero = lp_build_const_int_vec(gallivm, uint_type, 0);
166
167 LLVMValueRef bitmask = LLVMBuildICmp(builder, LLVMIntNE, params->exec_mask, uint_zero, "exec_bitvec");
168
169 LLVMTypeRef bitmask_type = LLVMIntTypeInContext(gallivm->context, uint_type.length);
170 bitmask = LLVMBuildBitCast(builder, bitmask, bitmask_type, "exec_bitmask");
171
172 LLVMValueRef any_active = LLVMBuildICmp(builder, LLVMIntNE, bitmask, LLVMConstInt(bitmask_type, 0, false), "any_active");
173
174 struct lp_build_if_state if_state;
175 lp_build_if(&if_state, gallivm, any_active);
176
177 LLVMValueRef consts = lp_jit_resources_constants(gallivm, params->resources_type, params->resources_ptr);
178
179 LLVMValueRef texture_descriptor = lp_llvm_descriptor_base(gallivm, consts, params->texture_resource, LP_MAX_TGSI_CONST_BUFFERS);
180
181 enum lp_sampler_op_type op_type = (params->sample_key & LP_SAMPLER_OP_TYPE_MASK) >> LP_SAMPLER_OP_TYPE_SHIFT;
182 uint32_t functions_offset = op_type == LP_SAMPLER_OP_FETCH ? offsetof(struct lp_texture_functions, fetch_functions)
183 : offsetof(struct lp_texture_functions, sample_functions);
184
185 LLVMValueRef texture_base_ptr = load_texture_functions_ptr(
186 gallivm, texture_descriptor, offsetof(struct lp_descriptor, functions), functions_offset);
187
188 LLVMTypeRef texture_function_type = lp_build_sample_function_type(gallivm, params->sample_key);
189 LLVMTypeRef texture_function_ptr_type = LLVMPointerType(texture_function_type, 0);
190 LLVMTypeRef texture_functions_type = LLVMPointerType(texture_function_ptr_type, 0);
191 LLVMTypeRef texture_base_type = LLVMPointerType(texture_functions_type, 0);
192 LLVMTypeRef texture_base_ptr_type = LLVMPointerType(texture_base_type, 0);
193
194 texture_base_ptr = LLVMBuildIntToPtr(builder, texture_base_ptr, texture_base_ptr_type, "");
195 LLVMValueRef texture_base = LLVMBuildLoad2(builder, texture_base_type, texture_base_ptr, "");
196
197 LLVMValueRef texture_functions;
198 LLVMValueRef sampler_desc_ptr;
199 if (op_type == LP_SAMPLER_OP_FETCH) {
200 texture_functions = texture_base;
201 sampler_desc_ptr = LLVMGetUndef(LLVMInt64TypeInContext(gallivm->context));
202 } else {
203 sampler_desc_ptr = lp_llvm_descriptor_base(gallivm, consts, params->sampler_resource, LP_MAX_TGSI_CONST_BUFFERS);
204
205 LLVMValueRef sampler_index_offset = lp_build_const_int64(gallivm, offsetof(struct lp_descriptor, texture.sampler_index));
206 LLVMValueRef sampler_index_ptr = LLVMBuildAdd(builder, sampler_desc_ptr, sampler_index_offset, "");
207
208 LLVMTypeRef sampler_index_type = LLVMInt32TypeInContext(gallivm->context);
209 LLVMTypeRef sampler_index_ptr_type = LLVMPointerType(sampler_index_type, 0);
210
211 sampler_index_ptr = LLVMBuildIntToPtr(builder, sampler_index_ptr, sampler_index_ptr_type, "");
212 LLVMValueRef sampler_index = LLVMBuildLoad2(builder, sampler_index_type, sampler_index_ptr, "");
213
214 LLVMValueRef texture_functions_ptr = LLVMBuildGEP2(builder, texture_functions_type, texture_base, &sampler_index, 1, "");
215 texture_functions = LLVMBuildLoad2(builder, texture_functions_type, texture_functions_ptr, "");
216 }
217
218 LLVMValueRef sample_key = lp_build_const_int32(gallivm, params->sample_key);
219 LLVMValueRef texture_function_ptr = LLVMBuildGEP2(builder, texture_function_ptr_type, texture_functions, &sample_key, 1, "");
220 LLVMValueRef texture_function = LLVMBuildLoad2(builder, texture_function_ptr_type, texture_function_ptr, "");
221
222 LLVMValueRef args[LP_MAX_TEX_FUNC_ARGS];
223 uint32_t num_args = 0;
224
225 args[num_args++] = texture_descriptor;
226 args[num_args++] = sampler_desc_ptr;
227
228 LLVMTypeRef coord_type;
229 if (op_type == LP_SAMPLER_OP_FETCH)
230 coord_type = lp_build_int_vec_type(gallivm, params->type);
231 else
232 coord_type = lp_build_vec_type(gallivm, params->type);
233
234 for (uint32_t i = 0; i < 4; i++) {
235 if (LLVMIsUndef(params->coords[i]))
236 args[num_args++] = LLVMGetUndef(coord_type);
237 else
238 args[num_args++] = params->coords[i];
239 }
240
241 if (params->sample_key & LP_SAMPLER_SHADOW)
242 args[num_args++] = params->coords[4];
243
244 if (params->sample_key & LP_SAMPLER_FETCH_MS)
245 args[num_args++] = params->ms_index;
246
247 if (params->sample_key & LP_SAMPLER_OFFSETS) {
248 for (uint32_t i = 0; i < 3; i++) {
249 if (params->offsets[i])
250 args[num_args++] = params->offsets[i];
251 else
252 args[num_args++] = LLVMGetUndef(lp_build_int_vec_type(gallivm, params->type));
253 }
254 }
255
256 enum lp_sampler_lod_control lod_control = (params->sample_key & LP_SAMPLER_LOD_CONTROL_MASK) >> LP_SAMPLER_LOD_CONTROL_SHIFT;
257 if (lod_control == LP_SAMPLER_LOD_BIAS || lod_control == LP_SAMPLER_LOD_EXPLICIT)
258 args[num_args++] = params->lod;
259
260 if (params->type.length != lp_native_vector_width / 32)
261 for (uint32_t i = 0; i < num_args; i++)
262 args[i] = widen_to_simd_width(gallivm, args[i]);
263
264 LLVMValueRef result = LLVMBuildCall2(builder, texture_function_type, texture_function, args, num_args, "");
265
266 for (unsigned i = 0; i < ARRAY_SIZE(out_data); i++) {
267 params->texel[i] = LLVMBuildExtractValue(builder, result, i, "");
268
269 if (params->type.length != lp_native_vector_width / 32)
270 params->texel[i] = truncate_to_type_width(gallivm, params->texel[i], params->type);
271
272 LLVMBuildStore(builder, params->texel[i], out_data[i]);
273 }
274
275 lp_build_endif(&if_state);
276
277 for (unsigned i = 0; i < ARRAY_SIZE(out_data) - 1; i++)
278 params->texel[i] = LLVMBuildLoad2(builder, out_data_type, out_data[i], "");
279 params->texel[ARRAY_SIZE(out_data) - 1] =
280 LLVMBuildLoad2(builder, out_residency_type, out_data[ARRAY_SIZE(out_data) - 1], "");
281
282 return;
283 }
284
285 const unsigned texture_index = params->texture_index;
286 const unsigned sampler_index = params->sampler_index;
287
288 assert(sampler_index < PIPE_MAX_SAMPLERS);
289 assert(texture_index < PIPE_MAX_SHADER_SAMPLER_VIEWS);
290 #if 0
291 if (LP_PERF & PERF_NO_TEX) {
292 lp_build_sample_nop(gallivm, params->type, params->coords, params->texel);
293 return;
294 }
295 #endif
296
297 if (params->texture_index_offset) {
298 LLVMValueRef unit =
299 LLVMBuildAdd(builder, params->texture_index_offset,
300 lp_build_const_int32(gallivm, texture_index), "");
301
302 struct lp_build_sample_array_switch switch_info;
303 memset(&switch_info, 0, sizeof(switch_info));
304 lp_build_sample_array_init_soa(&switch_info, gallivm, params, unit,
305 0, sampler->nr_samplers);
306 // build the switch cases
307 for (unsigned i = 0; i < sampler->nr_samplers; i++) {
308 lp_build_sample_array_case_soa(&switch_info, i,
309 &sampler->dynamic_state.static_state[i].texture_state,
310 &sampler->dynamic_state.static_state[i].sampler_state,
311 &sampler->dynamic_state.base);
312 }
313 lp_build_sample_array_fini_soa(&switch_info);
314 } else {
315 lp_build_sample_soa(&sampler->dynamic_state.static_state[texture_index].texture_state,
316 &sampler->dynamic_state.static_state[sampler_index].sampler_state,
317 &sampler->dynamic_state.base,
318 gallivm, params);
319 }
320 }
321
322
323 /**
324 * Fetch the texture size.
325 */
326 static void
lp_bld_llvm_sampler_soa_emit_size_query(const struct lp_build_sampler_soa * base,struct gallivm_state * gallivm,const struct lp_sampler_size_query_params * params)327 lp_bld_llvm_sampler_soa_emit_size_query(const struct lp_build_sampler_soa *base,
328 struct gallivm_state *gallivm,
329 const struct lp_sampler_size_query_params *params)
330 {
331 LLVMBuilderRef builder = gallivm->builder;
332
333 if (params->resource) {
334 LLVMTypeRef out_data_type = lp_build_vec_type(gallivm, params->int_type);
335
336 LLVMValueRef out_data[4];
337 for (uint32_t i = 0; i < 4; i++) {
338 out_data[i] = lp_build_alloca(gallivm, out_data_type, "");
339 }
340
341 struct lp_type uint_type = lp_uint_type(params->int_type);
342 LLVMValueRef uint_zero = lp_build_const_int_vec(gallivm, uint_type, 0);
343
344 LLVMValueRef bitmask = LLVMBuildICmp(builder, LLVMIntNE, params->exec_mask, uint_zero, "exec_bitvec");
345
346 LLVMTypeRef bitmask_type = LLVMIntTypeInContext(gallivm->context, uint_type.length);
347 bitmask = LLVMBuildBitCast(builder, bitmask, bitmask_type, "exec_bitmask");
348
349 LLVMValueRef any_active = LLVMBuildICmp(builder, LLVMIntNE, bitmask, LLVMConstInt(bitmask_type, 0, false), "any_active");
350
351 struct lp_build_if_state if_state;
352 lp_build_if(&if_state, gallivm, any_active);
353
354 LLVMValueRef consts = lp_jit_resources_constants(gallivm, params->resources_type, params->resources_ptr);
355
356 LLVMValueRef texture_descriptor = lp_llvm_descriptor_base(gallivm, consts, params->resource, LP_MAX_TGSI_CONST_BUFFERS);
357
358 uint32_t functions_offset = params->samples_only ? offsetof(struct lp_texture_functions, samples_function)
359 : offsetof(struct lp_texture_functions, size_function);
360
361 LLVMValueRef texture_base_ptr = load_texture_functions_ptr(
362 gallivm, texture_descriptor, offsetof(struct lp_descriptor, functions), functions_offset);
363
364 LLVMTypeRef texture_function_type = lp_build_size_function_type(gallivm, params);
365 LLVMTypeRef texture_function_ptr_type = LLVMPointerType(texture_function_type, 0);
366 LLVMTypeRef texture_function_ptr_ptr_type = LLVMPointerType(texture_function_ptr_type, 0);
367
368 texture_base_ptr = LLVMBuildIntToPtr(builder, texture_base_ptr, texture_function_ptr_ptr_type, "");
369 LLVMValueRef texture_function = LLVMBuildLoad2(builder, texture_function_ptr_type, texture_base_ptr, "");
370
371 LLVMValueRef args[LP_MAX_TEX_FUNC_ARGS];
372 uint32_t num_args = 0;
373
374 args[num_args++] = texture_descriptor;
375
376 if (!params->samples_only)
377 args[num_args++] = params->explicit_lod;
378
379 if (params->int_type.length != lp_native_vector_width / 32)
380 for (uint32_t i = 0; i < num_args; i++)
381 args[i] = widen_to_simd_width(gallivm, args[i]);
382
383 LLVMValueRef result = LLVMBuildCall2(builder, texture_function_type, texture_function, args, num_args, "");
384
385 for (unsigned i = 0; i < 4; i++) {
386 params->sizes_out[i] = LLVMBuildExtractValue(gallivm->builder, result, i, "");
387
388 if (params->int_type.length != lp_native_vector_width / 32)
389 params->sizes_out[i] = truncate_to_type_width(gallivm, params->sizes_out[i], params->int_type);
390
391 LLVMBuildStore(builder, params->sizes_out[i], out_data[i]);
392 }
393
394 lp_build_endif(&if_state);
395
396 for (unsigned i = 0; i < 4; i++)
397 params->sizes_out[i] = LLVMBuildLoad2(gallivm->builder, out_data_type, out_data[i], "");
398
399 return;
400 }
401
402 struct lp_bld_llvm_sampler_soa *sampler = (struct lp_bld_llvm_sampler_soa *)base;
403
404 assert(params->texture_unit < PIPE_MAX_SHADER_SAMPLER_VIEWS);
405
406 lp_build_size_query_soa(gallivm,
407 &sampler->dynamic_state.static_state[params->texture_unit].texture_state,
408 &sampler->dynamic_state.base,
409 params);
410 }
411
412
413 struct lp_build_sampler_soa *
lp_bld_llvm_sampler_soa_create(const struct lp_sampler_static_state * static_state,unsigned nr_samplers)414 lp_bld_llvm_sampler_soa_create(const struct lp_sampler_static_state *static_state,
415 unsigned nr_samplers)
416 {
417 assert(static_state);
418
419 struct lp_bld_llvm_sampler_soa *sampler = CALLOC_STRUCT(lp_bld_llvm_sampler_soa);
420 if (!sampler)
421 return NULL;
422
423 sampler->base.emit_tex_sample = lp_bld_llvm_sampler_soa_emit_fetch_texel;
424 sampler->base.emit_size_query = lp_bld_llvm_sampler_soa_emit_size_query;
425
426 lp_build_jit_fill_sampler_dynamic_state(&sampler->dynamic_state.base);
427
428 sampler->dynamic_state.static_state = static_state;
429
430 sampler->nr_samplers = nr_samplers;
431 return &sampler->base;
432 }
433
434
435 static void
lp_bld_llvm_image_soa_emit_op(const struct lp_build_image_soa * base,struct gallivm_state * gallivm,const struct lp_img_params * params)436 lp_bld_llvm_image_soa_emit_op(const struct lp_build_image_soa *base,
437 struct gallivm_state *gallivm,
438 const struct lp_img_params *params)
439 {
440 LLVMBuilderRef builder = gallivm->builder;
441
442 if (params->resource) {
443 const struct util_format_description *desc = util_format_description(params->format);
444 struct lp_type texel_type = lp_build_texel_type(params->type, desc);
445 LLVMTypeRef out_data_type = lp_build_vec_type(gallivm, texel_type);
446 LLVMTypeRef out_residency_type = lp_build_vec_type(gallivm, lp_int_type(texel_type));
447
448 LLVMValueRef out_data[5];
449 for (uint32_t i = 0; i < ARRAY_SIZE(out_data) - 1; i++) {
450 out_data[i] = lp_build_alloca(gallivm, out_data_type, "");
451 }
452 out_data[ARRAY_SIZE(out_data) - 1] = lp_build_alloca(gallivm, out_residency_type, "");
453
454 struct lp_type uint_type = lp_uint_type(params->type);
455 LLVMValueRef uint_zero = lp_build_const_int_vec(gallivm, uint_type, 0);
456
457 LLVMValueRef bitmask = LLVMBuildICmp(builder, LLVMIntNE, params->exec_mask, uint_zero, "exec_bitvec");
458
459 LLVMTypeRef bitmask_type = LLVMIntTypeInContext(gallivm->context, uint_type.length);
460 bitmask = LLVMBuildBitCast(builder, bitmask, bitmask_type, "exec_bitmask");
461
462 LLVMValueRef any_active = LLVMBuildICmp(builder, LLVMIntNE, bitmask, LLVMConstInt(bitmask_type, 0, false), "any_active");
463
464 LLVMValueRef binding_index = LLVMBuildExtractValue(builder, params->resource, 1, "");
465 LLVMValueRef inbounds = LLVMBuildICmp(builder, LLVMIntSGE, binding_index, lp_build_const_int32(gallivm, 0), "inbounds");
466
467 struct lp_build_if_state if_state;
468 lp_build_if(&if_state, gallivm, LLVMBuildAnd(builder, any_active, inbounds, ""));
469
470 LLVMValueRef consts = lp_jit_resources_constants(gallivm, params->resources_type, params->resources_ptr);
471
472 LLVMValueRef image_descriptor = lp_llvm_descriptor_base(gallivm, consts, params->resource, LP_MAX_TGSI_CONST_BUFFERS);
473
474 LLVMValueRef image_base_ptr = load_texture_functions_ptr(
475 gallivm, image_descriptor, offsetof(struct lp_descriptor, functions),
476 offsetof(struct lp_texture_functions, image_functions));
477
478 LLVMTypeRef image_function_type = lp_build_image_function_type(gallivm, params, params->ms_index);
479 LLVMTypeRef image_function_ptr_type = LLVMPointerType(image_function_type, 0);
480 LLVMTypeRef image_functions_type = LLVMPointerType(image_function_ptr_type, 0);
481 LLVMTypeRef image_base_type = LLVMPointerType(image_functions_type, 0);
482
483 image_base_ptr = LLVMBuildIntToPtr(builder, image_base_ptr, image_base_type, "");
484 LLVMValueRef image_functions = LLVMBuildLoad2(builder, image_functions_type, image_base_ptr, "");
485
486 uint32_t op = params->img_op;
487 if (op == LP_IMG_ATOMIC_CAS)
488 op--;
489 else if (op == LP_IMG_ATOMIC)
490 op = params->op + (LP_IMG_OP_COUNT - 1);
491
492 if (params->ms_index)
493 op += LP_TOTAL_IMAGE_OP_COUNT / 2;
494
495 LLVMValueRef function_index = lp_build_const_int32(gallivm, op);
496
497 LLVMValueRef image_function_ptr = LLVMBuildGEP2(builder, image_function_ptr_type, image_functions, &function_index, 1, "");
498 LLVMValueRef image_function = LLVMBuildLoad2(builder, image_function_ptr_type, image_function_ptr, "");
499
500 LLVMValueRef args[LP_MAX_TEX_FUNC_ARGS] = { 0 };
501 uint32_t num_args = 0;
502
503 args[num_args++] = image_descriptor;
504
505 if (params->img_op != LP_IMG_LOAD && params->img_op != LP_IMG_LOAD_SPARSE)
506 args[num_args++] = params->exec_mask;
507
508 for (uint32_t i = 0; i < 3; i++)
509 args[num_args++] = params->coords[i];
510
511 if (params->ms_index)
512 args[num_args++] = params->ms_index;
513
514 if (params->img_op != LP_IMG_LOAD && params->img_op != LP_IMG_LOAD_SPARSE)
515 for (uint32_t i = 0; i < 4; i++)
516 args[num_args++] = params->indata[i];
517
518 if (params->img_op == LP_IMG_ATOMIC_CAS)
519 for (uint32_t i = 0; i < 4; i++)
520 args[num_args++] = params->indata2[i];
521
522 assert(num_args == LLVMCountParamTypes(image_function_type));
523
524 LLVMTypeRef param_types[LP_MAX_TEX_FUNC_ARGS];
525 LLVMGetParamTypes(image_function_type, param_types);
526 for (uint32_t i = 0; i < num_args; i++)
527 if (!args[i])
528 args[i] = LLVMGetUndef(param_types[i]);
529
530 if (params->type.length != lp_native_vector_width / 32)
531 for (uint32_t i = 0; i < num_args; i++)
532 args[i] = widen_to_simd_width(gallivm, args[i]);
533
534 LLVMValueRef result = LLVMBuildCall2(builder, image_function_type, image_function, args, num_args, "");
535
536 if (params->img_op != LP_IMG_STORE) {
537 uint32_t channel_count = params->img_op == LP_IMG_LOAD_SPARSE ? 5 : 4;
538 for (unsigned i = 0; i < channel_count; i++) {
539 LLVMValueRef channel = LLVMBuildExtractValue(builder, result, i, "");
540 if (params->type.length != lp_native_vector_width / 32)
541 channel = truncate_to_type_width(gallivm, channel, params->type);
542
543 LLVMBuildStore(builder, channel, out_data[i]);
544 }
545 }
546
547 lp_build_endif(&if_state);
548
549 if (params->img_op != LP_IMG_STORE) {
550 for (unsigned i = 0; i < ARRAY_SIZE(out_data) - 1; i++) {
551 params->outdata[i] = LLVMBuildLoad2(builder, out_data_type, out_data[i], "");
552 }
553 params->outdata[ARRAY_SIZE(out_data) - 1] =
554 LLVMBuildLoad2(builder, out_residency_type, out_data[ARRAY_SIZE(out_data) - 1], "");
555 }
556
557 return;
558 }
559
560 struct lp_bld_llvm_image_soa *image = (struct lp_bld_llvm_image_soa *)base;
561 const unsigned image_index = params->image_index;
562 assert(image_index < PIPE_MAX_SHADER_IMAGES);
563
564 if (params->image_index_offset) {
565 struct lp_build_img_op_array_switch switch_info;
566 memset(&switch_info, 0, sizeof(switch_info));
567 LLVMValueRef unit = LLVMBuildAdd(builder,
568 params->image_index_offset,
569 lp_build_const_int32(gallivm,
570 image_index), "");
571
572 lp_build_image_op_switch_soa(&switch_info, gallivm, params,
573 unit, 0, image->nr_images);
574
575 for (unsigned i = 0; i < image->nr_images; i++) {
576 lp_build_image_op_array_case(&switch_info, i,
577 &image->dynamic_state.static_state[i].image_state,
578 &image->dynamic_state.base);
579 }
580 lp_build_image_op_array_fini_soa(&switch_info);
581 } else {
582 lp_build_img_op_soa(&image->dynamic_state.static_state[image_index].image_state,
583 &image->dynamic_state.base,
584 gallivm, params, params->outdata);
585 }
586 }
587
588
589 /**
590 * Fetch the texture size.
591 */
592 static void
lp_bld_llvm_image_soa_emit_size_query(const struct lp_build_image_soa * base,struct gallivm_state * gallivm,const struct lp_sampler_size_query_params * params)593 lp_bld_llvm_image_soa_emit_size_query(const struct lp_build_image_soa *base,
594 struct gallivm_state *gallivm,
595 const struct lp_sampler_size_query_params *params)
596 {
597 struct lp_bld_llvm_image_soa *image = (struct lp_bld_llvm_image_soa *)base;
598
599 if (params->resource) {
600 LLVMValueRef old_texture = gallivm->texture_descriptor;
601
602 LLVMValueRef consts = lp_jit_resources_constants(gallivm, params->resources_type, params->resources_ptr);
603 gallivm->texture_descriptor = lp_llvm_descriptor_base(gallivm, consts, params->resource, LP_MAX_TGSI_CONST_BUFFERS);
604
605 enum pipe_format format = params->format;
606 if (format == PIPE_FORMAT_NONE)
607 format = PIPE_FORMAT_R8G8B8A8_UNORM;
608
609 struct lp_static_texture_state state = {
610 .format = format,
611 .res_format = format,
612 .target = params->target,
613 .level_zero_only = params->ms,
614 };
615
616 lp_build_size_query_soa(gallivm, &state, &image->dynamic_state.base, params);
617
618 gallivm->texture_descriptor = old_texture;
619
620 return;
621 }
622
623 assert(params->texture_unit < PIPE_MAX_SHADER_IMAGES);
624
625 lp_build_size_query_soa(gallivm,
626 &image->dynamic_state.static_state[params->texture_unit].image_state,
627 &image->dynamic_state.base,
628 params);
629 }
630
631
632 struct lp_build_image_soa *
lp_bld_llvm_image_soa_create(const struct lp_image_static_state * static_state,unsigned nr_images)633 lp_bld_llvm_image_soa_create(const struct lp_image_static_state *static_state,
634 unsigned nr_images)
635 {
636 struct lp_bld_llvm_image_soa *image = CALLOC_STRUCT(lp_bld_llvm_image_soa);
637 if (!image)
638 return NULL;
639
640 image->base.emit_op = lp_bld_llvm_image_soa_emit_op;
641 image->base.emit_size_query = lp_bld_llvm_image_soa_emit_size_query;
642
643 lp_build_jit_fill_image_dynamic_state(&image->dynamic_state.base);
644 image->dynamic_state.static_state = static_state;
645
646 image->nr_images = nr_images;
647 return &image->base;
648 }
649
650 struct lp_sampler_dynamic_state *
lp_build_sampler_soa_dynamic_state(struct lp_build_sampler_soa * _sampler)651 lp_build_sampler_soa_dynamic_state(struct lp_build_sampler_soa *_sampler)
652 {
653 struct lp_bld_llvm_sampler_soa *sampler = (struct lp_bld_llvm_sampler_soa *)_sampler;
654 return &sampler->dynamic_state.base;
655 }
656
657 struct lp_sampler_dynamic_state *
lp_build_image_soa_dynamic_state(struct lp_build_image_soa * _image)658 lp_build_image_soa_dynamic_state(struct lp_build_image_soa *_image)
659 {
660 struct lp_bld_llvm_image_soa *image = (struct lp_bld_llvm_image_soa *)_image;
661 return &image->dynamic_state.base;
662 }
663