1 /**************************************************************************
2 *
3 * Copyright 2009 VMware, Inc.
4 * All rights reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
21 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27
28 #include "gallivm/lp_bld_sample.h"
29 #include "gallivm/lp_bld_limits.h"
30 #include "gallivm/lp_bld_tgsi.h"
31 #include "gallivm/lp_bld_type.h"
32 #include "gallivm/lp_bld_init.h"
33 #include "gallivm/lp_bld_const.h"
34 #include "gallivm/lp_bld_sample.h"
35 #include "gallivm/lp_bld_jit_types.h"
36 #include "gallivm/lp_bld_jit_sample.h"
37 #include "gallivm/lp_bld_flow.h"
38
39 struct lp_bld_sampler_dynamic_state
40 {
41 struct lp_sampler_dynamic_state base;
42
43 const struct lp_sampler_static_state *static_state;
44 };
45
46 struct lp_bld_llvm_sampler_soa
47 {
48 struct lp_build_sampler_soa base;
49
50 struct lp_bld_sampler_dynamic_state dynamic_state;
51 unsigned nr_samplers;
52 };
53
54
55 struct lp_bld_image_dynamic_state
56 {
57 struct lp_sampler_dynamic_state base;
58
59 const struct lp_image_static_state *static_state;
60 };
61
62 struct lp_bld_llvm_image_soa
63 {
64 struct lp_build_image_soa base;
65
66 struct lp_bld_image_dynamic_state dynamic_state;
67 unsigned nr_images;
68 };
69
70 static LLVMValueRef
load_texture_functions_ptr(struct gallivm_state * gallivm,LLVMValueRef descriptor,uint32_t offset1,uint32_t offset2)71 load_texture_functions_ptr(struct gallivm_state *gallivm, LLVMValueRef descriptor,
72 uint32_t offset1, uint32_t offset2)
73 {
74 LLVMBuilderRef builder = gallivm->builder;
75
76 LLVMValueRef texture_base_offset = lp_build_const_int64(gallivm, offset1);
77 LLVMValueRef texture_base_ptr = LLVMBuildAdd(builder, descriptor, texture_base_offset, "");
78
79 LLVMTypeRef texture_base_type = LLVMInt64TypeInContext(gallivm->context);
80 LLVMTypeRef texture_base_ptr_type = LLVMPointerType(texture_base_type, 0);
81
82 texture_base_ptr = LLVMBuildIntToPtr(builder, texture_base_ptr, texture_base_ptr_type, "");
83 /* struct lp_texture_functions * */
84 LLVMValueRef texture_base = LLVMBuildLoad2(builder, texture_base_type, texture_base_ptr, "");
85
86 LLVMValueRef functions_offset = lp_build_const_int64(gallivm, offset2);
87 return LLVMBuildAdd(builder, texture_base, functions_offset, "");
88 }
89
90 static LLVMValueRef
widen_to_simd_width(struct gallivm_state * gallivm,LLVMValueRef value)91 widen_to_simd_width(struct gallivm_state *gallivm, LLVMValueRef value)
92 {
93 LLVMBuilderRef builder = gallivm->builder;
94 LLVMTypeRef type = LLVMTypeOf(value);
95
96 if (LLVMGetTypeKind(type) == LLVMVectorTypeKind) {
97 LLVMTypeRef element_type = LLVMGetElementType(type);
98 uint32_t element_count = LLVMGetVectorSize(type);
99 LLVMValueRef elements[8] = { 0 };
100 for (uint32_t i = 0; i < lp_native_vector_width / 32; i++) {
101 if (i < element_count)
102 elements[i] = LLVMBuildExtractElement(builder, value, lp_build_const_int32(gallivm, i), "");
103 else
104 elements[i] = LLVMConstNull(element_type);
105 }
106
107 LLVMTypeRef result_type = LLVMVectorType(element_type, lp_native_vector_width / 32);
108 LLVMValueRef result = LLVMGetUndef(result_type);
109 for (unsigned i = 0; i < lp_native_vector_width / 32; i++)
110 result = LLVMBuildInsertElement(builder, result, elements[i], lp_build_const_int32(gallivm, i), "");
111
112 return result;
113 }
114
115 return value;
116 }
117
118 static LLVMValueRef
truncate_to_type_width(struct gallivm_state * gallivm,LLVMValueRef value,struct lp_type target_type)119 truncate_to_type_width(struct gallivm_state *gallivm, LLVMValueRef value, struct lp_type target_type)
120 {
121 LLVMBuilderRef builder = gallivm->builder;
122 LLVMTypeRef type = LLVMTypeOf(value);
123
124 if (LLVMGetTypeKind(type) == LLVMVectorTypeKind) {
125 LLVMTypeRef element_type = LLVMGetElementType(type);
126
127 LLVMValueRef elements[8];
128 for (uint32_t i = 0; i < target_type.length; i++)
129 elements[i] = LLVMBuildExtractElement(builder, value, lp_build_const_int32(gallivm, i), "");
130
131 LLVMTypeRef result_type = LLVMVectorType(element_type, target_type.length);
132 LLVMValueRef result = LLVMGetUndef(result_type);
133 for (unsigned i = 0; i < target_type.length; i++)
134 result = LLVMBuildInsertElement(builder, result, elements[i], lp_build_const_int32(gallivm, i), "");
135
136 return result;
137 }
138
139 return value;
140 }
141
142 /**
143 * Fetch filtered values from texture.
144 * The 'texel' parameter returns four vectors corresponding to R, G, B, A.
145 */
146 static void
lp_bld_llvm_sampler_soa_emit_fetch_texel(const struct lp_build_sampler_soa * base,struct gallivm_state * gallivm,const struct lp_sampler_params * params)147 lp_bld_llvm_sampler_soa_emit_fetch_texel(const struct lp_build_sampler_soa *base,
148 struct gallivm_state *gallivm,
149 const struct lp_sampler_params *params)
150 {
151 struct lp_bld_llvm_sampler_soa *sampler = (struct lp_bld_llvm_sampler_soa *)base;
152 LLVMBuilderRef builder = gallivm->builder;
153
154 if (params->texture_resource) {
155 LLVMTypeRef out_data_type = lp_build_vec_type(gallivm, params->type);
156
157 LLVMValueRef out_data[4];
158 for (uint32_t i = 0; i < 4; i++) {
159 out_data[i] = lp_build_alloca(gallivm, out_data_type, "");
160 LLVMBuildStore(builder, lp_build_const_vec(gallivm, params->type, 0), out_data[i]);
161 }
162
163 struct lp_type uint_type = lp_uint_type(params->type);
164 LLVMValueRef uint_zero = lp_build_const_int_vec(gallivm, uint_type, 0);
165
166 LLVMValueRef bitmask = LLVMBuildICmp(builder, LLVMIntNE, params->exec_mask, uint_zero, "exec_bitvec");
167
168 LLVMTypeRef bitmask_type = LLVMIntTypeInContext(gallivm->context, uint_type.length);
169 bitmask = LLVMBuildBitCast(builder, bitmask, bitmask_type, "exec_bitmask");
170
171 LLVMValueRef any_active = LLVMBuildICmp(builder, LLVMIntNE, bitmask, LLVMConstInt(bitmask_type, 0, false), "any_active");
172
173 struct lp_build_if_state if_state;
174 lp_build_if(&if_state, gallivm, any_active);
175
176 LLVMValueRef consts = lp_jit_resources_constants(gallivm, params->resources_type, params->resources_ptr);
177
178 LLVMValueRef texture_descriptor = lp_llvm_descriptor_base(gallivm, consts, params->texture_resource, LP_MAX_TGSI_CONST_BUFFERS);
179
180 enum lp_sampler_op_type op_type = (params->sample_key & LP_SAMPLER_OP_TYPE_MASK) >> LP_SAMPLER_OP_TYPE_SHIFT;
181 uint32_t functions_offset = op_type == LP_SAMPLER_OP_FETCH ? offsetof(struct lp_texture_functions, fetch_functions)
182 : offsetof(struct lp_texture_functions, sample_functions);
183
184 LLVMValueRef texture_base_ptr = load_texture_functions_ptr(
185 gallivm, texture_descriptor, offsetof(struct lp_descriptor, functions), functions_offset);
186
187 LLVMTypeRef texture_function_type = lp_build_sample_function_type(gallivm, params->sample_key);
188 LLVMTypeRef texture_function_ptr_type = LLVMPointerType(texture_function_type, 0);
189 LLVMTypeRef texture_functions_type = LLVMPointerType(texture_function_ptr_type, 0);
190 LLVMTypeRef texture_base_type = LLVMPointerType(texture_functions_type, 0);
191 LLVMTypeRef texture_base_ptr_type = LLVMPointerType(texture_base_type, 0);
192
193 texture_base_ptr = LLVMBuildIntToPtr(builder, texture_base_ptr, texture_base_ptr_type, "");
194 LLVMValueRef texture_base = LLVMBuildLoad2(builder, texture_base_type, texture_base_ptr, "");
195
196 LLVMValueRef texture_functions;
197 LLVMValueRef sampler_desc_ptr;
198 if (op_type == LP_SAMPLER_OP_FETCH) {
199 texture_functions = texture_base;
200 sampler_desc_ptr = LLVMGetUndef(LLVMInt64TypeInContext(gallivm->context));
201 } else {
202 sampler_desc_ptr = lp_llvm_descriptor_base(gallivm, consts, params->sampler_resource, LP_MAX_TGSI_CONST_BUFFERS);
203
204 LLVMValueRef sampler_index_offset = lp_build_const_int64(gallivm, offsetof(struct lp_descriptor, texture.sampler_index));
205 LLVMValueRef sampler_index_ptr = LLVMBuildAdd(builder, sampler_desc_ptr, sampler_index_offset, "");
206
207 LLVMTypeRef sampler_index_type = LLVMInt32TypeInContext(gallivm->context);
208 LLVMTypeRef sampler_index_ptr_type = LLVMPointerType(sampler_index_type, 0);
209
210 sampler_index_ptr = LLVMBuildIntToPtr(builder, sampler_index_ptr, sampler_index_ptr_type, "");
211 LLVMValueRef sampler_index = LLVMBuildLoad2(builder, sampler_index_type, sampler_index_ptr, "");
212
213 LLVMValueRef texture_functions_ptr = LLVMBuildGEP2(builder, texture_functions_type, texture_base, &sampler_index, 1, "");
214 texture_functions = LLVMBuildLoad2(builder, texture_functions_type, texture_functions_ptr, "");
215 }
216
217 LLVMValueRef sample_key = lp_build_const_int32(gallivm, params->sample_key);
218 LLVMValueRef texture_function_ptr = LLVMBuildGEP2(builder, texture_function_ptr_type, texture_functions, &sample_key, 1, "");
219 LLVMValueRef texture_function = LLVMBuildLoad2(builder, texture_function_ptr_type, texture_function_ptr, "");
220
221 LLVMValueRef args[LP_MAX_TEX_FUNC_ARGS];
222 uint32_t num_args = 0;
223
224 args[num_args++] = texture_descriptor;
225 args[num_args++] = sampler_desc_ptr;
226
227 args[num_args++] = params->aniso_filter_table;
228
229 LLVMTypeRef coord_type;
230 if (op_type == LP_SAMPLER_OP_FETCH)
231 coord_type = lp_build_int_vec_type(gallivm, params->type);
232 else
233 coord_type = lp_build_vec_type(gallivm, params->type);
234
235 for (uint32_t i = 0; i < 4; i++) {
236 if (LLVMIsUndef(params->coords[i]))
237 args[num_args++] = LLVMGetUndef(coord_type);
238 else
239 args[num_args++] = params->coords[i];
240 }
241
242 if (params->sample_key & LP_SAMPLER_SHADOW)
243 args[num_args++] = params->coords[4];
244
245 if (params->sample_key & LP_SAMPLER_FETCH_MS)
246 args[num_args++] = params->ms_index;
247
248 if (params->sample_key & LP_SAMPLER_OFFSETS) {
249 for (uint32_t i = 0; i < 3; i++) {
250 if (params->offsets[i])
251 args[num_args++] = params->offsets[i];
252 else
253 args[num_args++] = LLVMGetUndef(lp_build_int_vec_type(gallivm, params->type));
254 }
255 }
256
257 enum lp_sampler_lod_control lod_control = (params->sample_key & LP_SAMPLER_LOD_CONTROL_MASK) >> LP_SAMPLER_LOD_CONTROL_SHIFT;
258 if (lod_control == LP_SAMPLER_LOD_BIAS || lod_control == LP_SAMPLER_LOD_EXPLICIT)
259 args[num_args++] = params->lod;
260
261 if (params->type.length != lp_native_vector_width / 32)
262 for (uint32_t i = 0; i < num_args; i++)
263 args[i] = widen_to_simd_width(gallivm, args[i]);
264
265 LLVMValueRef result = LLVMBuildCall2(builder, texture_function_type, texture_function, args, num_args, "");
266
267 for (unsigned i = 0; i < 4; i++) {
268 params->texel[i] = LLVMBuildExtractValue(gallivm->builder, result, i, "");
269
270 if (params->type.length != lp_native_vector_width / 32)
271 params->texel[i] = truncate_to_type_width(gallivm, params->texel[i], params->type);
272
273 LLVMBuildStore(builder, params->texel[i], out_data[i]);
274 }
275
276 lp_build_endif(&if_state);
277
278 for (unsigned i = 0; i < 4; i++)
279 params->texel[i] = LLVMBuildLoad2(gallivm->builder, out_data_type, out_data[i], "");
280
281 return;
282 }
283
284 const unsigned texture_index = params->texture_index;
285 const unsigned sampler_index = params->sampler_index;
286
287 assert(sampler_index < PIPE_MAX_SAMPLERS);
288 assert(texture_index < PIPE_MAX_SHADER_SAMPLER_VIEWS);
289 #if 0
290 if (LP_PERF & PERF_NO_TEX) {
291 lp_build_sample_nop(gallivm, params->type, params->coords, params->texel);
292 return;
293 }
294 #endif
295
296 if (params->texture_index_offset) {
297 LLVMValueRef unit =
298 LLVMBuildAdd(gallivm->builder, params->texture_index_offset,
299 lp_build_const_int32(gallivm, texture_index), "");
300
301 struct lp_build_sample_array_switch switch_info;
302 memset(&switch_info, 0, sizeof(switch_info));
303 lp_build_sample_array_init_soa(&switch_info, gallivm, params, unit,
304 0, sampler->nr_samplers);
305 // build the switch cases
306 for (unsigned i = 0; i < sampler->nr_samplers; i++) {
307 lp_build_sample_array_case_soa(&switch_info, i,
308 &sampler->dynamic_state.static_state[i].texture_state,
309 &sampler->dynamic_state.static_state[i].sampler_state,
310 &sampler->dynamic_state.base);
311 }
312 lp_build_sample_array_fini_soa(&switch_info);
313 } else {
314 lp_build_sample_soa(&sampler->dynamic_state.static_state[texture_index].texture_state,
315 &sampler->dynamic_state.static_state[sampler_index].sampler_state,
316 &sampler->dynamic_state.base,
317 gallivm, params);
318 }
319 }
320
321
322 /**
323 * Fetch the texture size.
324 */
325 static void
lp_bld_llvm_sampler_soa_emit_size_query(const struct lp_build_sampler_soa * base,struct gallivm_state * gallivm,const struct lp_sampler_size_query_params * params)326 lp_bld_llvm_sampler_soa_emit_size_query(const struct lp_build_sampler_soa *base,
327 struct gallivm_state *gallivm,
328 const struct lp_sampler_size_query_params *params)
329 {
330 LLVMBuilderRef builder = gallivm->builder;
331
332 if (params->resource) {
333 LLVMTypeRef out_data_type = lp_build_vec_type(gallivm, params->int_type);
334
335 LLVMValueRef out_data[4];
336 for (uint32_t i = 0; i < 4; i++) {
337 out_data[i] = lp_build_alloca(gallivm, out_data_type, "");
338 LLVMBuildStore(builder, lp_build_const_vec(gallivm, params->int_type, 0), out_data[i]);
339 }
340
341 struct lp_type uint_type = lp_uint_type(params->int_type);
342 LLVMValueRef uint_zero = lp_build_const_int_vec(gallivm, uint_type, 0);
343
344 LLVMValueRef bitmask = LLVMBuildICmp(builder, LLVMIntNE, params->exec_mask, uint_zero, "exec_bitvec");
345
346 LLVMTypeRef bitmask_type = LLVMIntTypeInContext(gallivm->context, uint_type.length);
347 bitmask = LLVMBuildBitCast(builder, bitmask, bitmask_type, "exec_bitmask");
348
349 LLVMValueRef any_active = LLVMBuildICmp(builder, LLVMIntNE, bitmask, LLVMConstInt(bitmask_type, 0, false), "any_active");
350
351 struct lp_build_if_state if_state;
352 lp_build_if(&if_state, gallivm, any_active);
353
354 LLVMValueRef consts = lp_jit_resources_constants(gallivm, params->resources_type, params->resources_ptr);
355
356 LLVMValueRef texture_descriptor = lp_llvm_descriptor_base(gallivm, consts, params->resource, LP_MAX_TGSI_CONST_BUFFERS);
357
358 uint32_t functions_offset = params->samples_only ? offsetof(struct lp_texture_functions, samples_function)
359 : offsetof(struct lp_texture_functions, size_function);
360
361 LLVMValueRef texture_base_ptr = load_texture_functions_ptr(
362 gallivm, texture_descriptor, offsetof(struct lp_descriptor, functions), functions_offset);
363
364 LLVMTypeRef texture_function_type = lp_build_size_function_type(gallivm, params);
365 LLVMTypeRef texture_function_ptr_type = LLVMPointerType(texture_function_type, 0);
366 LLVMTypeRef texture_function_ptr_ptr_type = LLVMPointerType(texture_function_ptr_type, 0);
367
368 texture_base_ptr = LLVMBuildIntToPtr(builder, texture_base_ptr, texture_function_ptr_ptr_type, "");
369 LLVMValueRef texture_function = LLVMBuildLoad2(builder, texture_function_ptr_type, texture_base_ptr, "");
370
371 LLVMValueRef args[LP_MAX_TEX_FUNC_ARGS];
372 uint32_t num_args = 0;
373
374 args[num_args++] = texture_descriptor;
375
376 if (!params->samples_only)
377 args[num_args++] = params->explicit_lod;
378
379 if (params->int_type.length != lp_native_vector_width / 32)
380 for (uint32_t i = 0; i < num_args; i++)
381 args[i] = widen_to_simd_width(gallivm, args[i]);
382
383 LLVMValueRef result = LLVMBuildCall2(builder, texture_function_type, texture_function, args, num_args, "");
384
385 for (unsigned i = 0; i < 4; i++) {
386 params->sizes_out[i] = LLVMBuildExtractValue(gallivm->builder, result, i, "");
387
388 if (params->int_type.length != lp_native_vector_width / 32)
389 params->sizes_out[i] = truncate_to_type_width(gallivm, params->sizes_out[i], params->int_type);
390
391 LLVMBuildStore(builder, params->sizes_out[i], out_data[i]);
392 }
393
394 lp_build_endif(&if_state);
395
396 for (unsigned i = 0; i < 4; i++)
397 params->sizes_out[i] = LLVMBuildLoad2(gallivm->builder, out_data_type, out_data[i], "");
398
399 return;
400 }
401
402 struct lp_bld_llvm_sampler_soa *sampler = (struct lp_bld_llvm_sampler_soa *)base;
403
404 assert(params->texture_unit < PIPE_MAX_SHADER_SAMPLER_VIEWS);
405
406 lp_build_size_query_soa(gallivm,
407 &sampler->dynamic_state.static_state[params->texture_unit].texture_state,
408 &sampler->dynamic_state.base,
409 params);
410 }
411
412
413 struct lp_build_sampler_soa *
lp_bld_llvm_sampler_soa_create(const struct lp_sampler_static_state * static_state,unsigned nr_samplers)414 lp_bld_llvm_sampler_soa_create(const struct lp_sampler_static_state *static_state,
415 unsigned nr_samplers)
416 {
417 assert(static_state);
418
419 struct lp_bld_llvm_sampler_soa *sampler = CALLOC_STRUCT(lp_bld_llvm_sampler_soa);
420 if (!sampler)
421 return NULL;
422
423 sampler->base.emit_tex_sample = lp_bld_llvm_sampler_soa_emit_fetch_texel;
424 sampler->base.emit_size_query = lp_bld_llvm_sampler_soa_emit_size_query;
425
426 lp_build_jit_fill_sampler_dynamic_state(&sampler->dynamic_state.base);
427
428 sampler->dynamic_state.static_state = static_state;
429
430 sampler->nr_samplers = nr_samplers;
431 return &sampler->base;
432 }
433
434
435 static void
lp_bld_llvm_image_soa_emit_op(const struct lp_build_image_soa * base,struct gallivm_state * gallivm,const struct lp_img_params * params)436 lp_bld_llvm_image_soa_emit_op(const struct lp_build_image_soa *base,
437 struct gallivm_state *gallivm,
438 const struct lp_img_params *params)
439 {
440 LLVMBuilderRef builder = gallivm->builder;
441
442 if (params->resource) {
443 const struct util_format_description *desc = util_format_description(params->format);
444 LLVMTypeRef out_data_type = lp_build_vec_type(gallivm, lp_build_texel_type(params->type, desc));
445
446 LLVMValueRef out_data[4];
447 for (uint32_t i = 0; i < 4; i++) {
448 out_data[i] = lp_build_alloca(gallivm, out_data_type, "");
449 LLVMBuildStore(builder, lp_build_const_vec(gallivm, lp_build_texel_type(params->type, desc), 0), out_data[i]);
450 }
451
452 struct lp_type uint_type = lp_uint_type(params->type);
453 LLVMValueRef uint_zero = lp_build_const_int_vec(gallivm, uint_type, 0);
454
455 LLVMValueRef bitmask = LLVMBuildICmp(builder, LLVMIntNE, params->exec_mask, uint_zero, "exec_bitvec");
456
457 LLVMTypeRef bitmask_type = LLVMIntTypeInContext(gallivm->context, uint_type.length);
458 bitmask = LLVMBuildBitCast(builder, bitmask, bitmask_type, "exec_bitmask");
459
460 LLVMValueRef any_active = LLVMBuildICmp(builder, LLVMIntNE, bitmask, LLVMConstInt(bitmask_type, 0, false), "any_active");
461
462 LLVMValueRef binding_index = LLVMBuildExtractValue(builder, params->resource, 1, "");
463 LLVMValueRef inbounds = LLVMBuildICmp(builder, LLVMIntSGE, binding_index, lp_build_const_int32(gallivm, 0), "inbounds");
464
465 struct lp_build_if_state if_state;
466 lp_build_if(&if_state, gallivm, LLVMBuildAnd(builder, any_active, inbounds, ""));
467
468 LLVMValueRef consts = lp_jit_resources_constants(gallivm, params->resources_type, params->resources_ptr);
469
470 LLVMValueRef image_descriptor = lp_llvm_descriptor_base(gallivm, consts, params->resource, LP_MAX_TGSI_CONST_BUFFERS);
471
472 LLVMValueRef image_base_ptr = load_texture_functions_ptr(
473 gallivm, image_descriptor, offsetof(struct lp_descriptor, functions),
474 offsetof(struct lp_texture_functions, image_functions));
475
476 LLVMTypeRef image_function_type = lp_build_image_function_type(gallivm, params, params->ms_index);
477 LLVMTypeRef image_function_ptr_type = LLVMPointerType(image_function_type, 0);
478 LLVMTypeRef image_functions_type = LLVMPointerType(image_function_ptr_type, 0);
479 LLVMTypeRef image_base_type = LLVMPointerType(image_functions_type, 0);
480
481 image_base_ptr = LLVMBuildIntToPtr(builder, image_base_ptr, image_base_type, "");
482 LLVMValueRef image_functions = LLVMBuildLoad2(builder, image_functions_type, image_base_ptr, "");
483
484 uint32_t op = params->img_op;
485 if (op == LP_IMG_ATOMIC_CAS)
486 op--;
487 else if (op == LP_IMG_ATOMIC)
488 op = params->op + (LP_IMG_OP_COUNT - 1);
489
490 if (params->ms_index)
491 op += LP_TOTAL_IMAGE_OP_COUNT / 2;
492
493 LLVMValueRef function_index = lp_build_const_int32(gallivm, op);
494
495 LLVMValueRef image_function_ptr = LLVMBuildGEP2(builder, image_function_ptr_type, image_functions, &function_index, 1, "");
496 LLVMValueRef image_function = LLVMBuildLoad2(builder, image_function_ptr_type, image_function_ptr, "");
497
498 LLVMValueRef args[LP_MAX_TEX_FUNC_ARGS] = { 0 };
499 uint32_t num_args = 0;
500
501 args[num_args++] = image_descriptor;
502
503 if (params->img_op != LP_IMG_LOAD)
504 args[num_args++] = params->exec_mask;
505
506 for (uint32_t i = 0; i < 3; i++)
507 args[num_args++] = params->coords[i];
508
509 if (params->ms_index)
510 args[num_args++] = params->ms_index;
511
512 if (params->img_op != LP_IMG_LOAD)
513 for (uint32_t i = 0; i < 4; i++)
514 args[num_args++] = params->indata[i];
515
516 if (params->img_op == LP_IMG_ATOMIC_CAS)
517 for (uint32_t i = 0; i < 4; i++)
518 args[num_args++] = params->indata2[i];
519
520 assert(num_args == LLVMCountParamTypes(image_function_type));
521
522 LLVMTypeRef param_types[LP_MAX_TEX_FUNC_ARGS];
523 LLVMGetParamTypes(image_function_type, param_types);
524 for (uint32_t i = 0; i < num_args; i++)
525 if (!args[i])
526 args[i] = LLVMGetUndef(param_types[i]);
527
528 if (params->type.length != lp_native_vector_width / 32)
529 for (uint32_t i = 0; i < num_args; i++)
530 args[i] = widen_to_simd_width(gallivm, args[i]);
531
532 LLVMValueRef result = LLVMBuildCall2(builder, image_function_type, image_function, args, num_args, "");
533
534 if (params->img_op != LP_IMG_STORE) {
535 for (unsigned i = 0; i < 4; i++) {
536 LLVMValueRef channel = LLVMBuildExtractValue(gallivm->builder, result, i, "");
537 if (params->type.length != lp_native_vector_width / 32)
538 channel = truncate_to_type_width(gallivm, channel, params->type);
539
540 LLVMBuildStore(builder, channel, out_data[i]);
541 }
542 }
543
544 lp_build_endif(&if_state);
545
546 if (params->img_op != LP_IMG_STORE) {
547 for (unsigned i = 0; i < 4; i++) {
548 params->outdata[i] = LLVMBuildLoad2(gallivm->builder, out_data_type, out_data[i], "");
549 }
550 }
551
552 return;
553 }
554
555 struct lp_bld_llvm_image_soa *image = (struct lp_bld_llvm_image_soa *)base;
556 const unsigned image_index = params->image_index;
557 assert(image_index < PIPE_MAX_SHADER_IMAGES);
558
559 if (params->image_index_offset) {
560 struct lp_build_img_op_array_switch switch_info;
561 memset(&switch_info, 0, sizeof(switch_info));
562 LLVMValueRef unit = LLVMBuildAdd(gallivm->builder,
563 params->image_index_offset,
564 lp_build_const_int32(gallivm,
565 image_index), "");
566
567 lp_build_image_op_switch_soa(&switch_info, gallivm, params,
568 unit, 0, image->nr_images);
569
570 for (unsigned i = 0; i < image->nr_images; i++) {
571 lp_build_image_op_array_case(&switch_info, i,
572 &image->dynamic_state.static_state[i].image_state,
573 &image->dynamic_state.base);
574 }
575 lp_build_image_op_array_fini_soa(&switch_info);
576 } else {
577 lp_build_img_op_soa(&image->dynamic_state.static_state[image_index].image_state,
578 &image->dynamic_state.base,
579 gallivm, params, params->outdata);
580 }
581 }
582
583
584 /**
585 * Fetch the texture size.
586 */
587 static void
lp_bld_llvm_image_soa_emit_size_query(const struct lp_build_image_soa * base,struct gallivm_state * gallivm,const struct lp_sampler_size_query_params * params)588 lp_bld_llvm_image_soa_emit_size_query(const struct lp_build_image_soa *base,
589 struct gallivm_state *gallivm,
590 const struct lp_sampler_size_query_params *params)
591 {
592 struct lp_bld_llvm_image_soa *image = (struct lp_bld_llvm_image_soa *)base;
593
594 if (params->resource) {
595 LLVMValueRef old_texture = gallivm->texture_descriptor;
596
597 LLVMValueRef consts = lp_jit_resources_constants(gallivm, params->resources_type, params->resources_ptr);
598 gallivm->texture_descriptor = lp_llvm_descriptor_base(gallivm, consts, params->resource, LP_MAX_TGSI_CONST_BUFFERS);
599
600 enum pipe_format format = params->format;
601 if (format == PIPE_FORMAT_NONE)
602 format = PIPE_FORMAT_R8G8B8A8_UNORM;
603
604 struct lp_static_texture_state state = {
605 .format = format,
606 .res_format = format,
607 .target = params->target,
608 .level_zero_only = params->ms,
609 };
610
611 lp_build_size_query_soa(gallivm, &state, &image->dynamic_state.base, params);
612
613 gallivm->texture_descriptor = old_texture;
614
615 return;
616 }
617
618 assert(params->texture_unit < PIPE_MAX_SHADER_IMAGES);
619
620 lp_build_size_query_soa(gallivm,
621 &image->dynamic_state.static_state[params->texture_unit].image_state,
622 &image->dynamic_state.base,
623 params);
624 }
625
626
627 struct lp_build_image_soa *
lp_bld_llvm_image_soa_create(const struct lp_image_static_state * static_state,unsigned nr_images)628 lp_bld_llvm_image_soa_create(const struct lp_image_static_state *static_state,
629 unsigned nr_images)
630 {
631 struct lp_bld_llvm_image_soa *image = CALLOC_STRUCT(lp_bld_llvm_image_soa);
632 if (!image)
633 return NULL;
634
635 image->base.emit_op = lp_bld_llvm_image_soa_emit_op;
636 image->base.emit_size_query = lp_bld_llvm_image_soa_emit_size_query;
637
638 lp_build_jit_fill_image_dynamic_state(&image->dynamic_state.base);
639 image->dynamic_state.static_state = static_state;
640
641 image->nr_images = nr_images;
642 return &image->base;
643 }
644
645 struct lp_sampler_dynamic_state *
lp_build_sampler_soa_dynamic_state(struct lp_build_sampler_soa * _sampler)646 lp_build_sampler_soa_dynamic_state(struct lp_build_sampler_soa *_sampler)
647 {
648 struct lp_bld_llvm_sampler_soa *sampler = (struct lp_bld_llvm_sampler_soa *)_sampler;
649 return &sampler->dynamic_state.base;
650 }
651
652 struct lp_sampler_dynamic_state *
lp_build_image_soa_dynamic_state(struct lp_build_image_soa * _image)653 lp_build_image_soa_dynamic_state(struct lp_build_image_soa *_image)
654 {
655 struct lp_bld_llvm_image_soa *image = (struct lp_bld_llvm_image_soa *)_image;
656 return &image->dynamic_state.base;
657 }
658