1 /*
2 * Copyright 2022 Red Hat.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * on the rights to use, copy, modify, merge, publish, distribute, sub
8 * license, and/or sell copies of the Software, and to permit persons to whom
9 * the Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
19 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
20 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
21 * USE OR OTHER DEALINGS IN THE SOFTWARE.
22 */
23
24 #include "util/compiler.h"
25 #include "gallivm/lp_bld.h"
26 #include "gallivm/lp_bld_init.h"
27 #include "gallivm/lp_bld_struct.h"
28 #include "gallivm/lp_bld_sample.h"
29 #include "gallivm/lp_bld_const.h"
30 #include "gallivm/lp_bld_debug.h"
31 #include "gallivm/lp_bld_ir_common.h"
32 #include "draw/draw_vertex_header.h"
33 #include "lp_bld_jit_types.h"
34
35
36 static LLVMTypeRef
lp_build_create_jit_buffer_type(struct gallivm_state * gallivm)37 lp_build_create_jit_buffer_type(struct gallivm_state *gallivm)
38 {
39 LLVMContextRef lc = gallivm->context;
40 LLVMTypeRef buffer_type;
41 LLVMTypeRef elem_types[LP_JIT_BUFFER_NUM_FIELDS];
42
43 elem_types[LP_JIT_BUFFER_BASE] = LLVMPointerType(LLVMInt32TypeInContext(lc), 0);
44 elem_types[LP_JIT_BUFFER_NUM_ELEMENTS] = LLVMInt32TypeInContext(lc);
45
46 buffer_type = LLVMStructTypeInContext(lc, elem_types,
47 ARRAY_SIZE(elem_types), 0);
48
49 LP_CHECK_MEMBER_OFFSET(struct lp_jit_buffer, f,
50 gallivm->target, buffer_type,
51 LP_JIT_BUFFER_BASE);
52
53 LP_CHECK_MEMBER_OFFSET(struct lp_jit_buffer, num_elements,
54 gallivm->target, buffer_type,
55 LP_JIT_BUFFER_NUM_ELEMENTS);
56 return buffer_type;
57 }
58
59 LLVMValueRef
lp_llvm_descriptor_base(struct gallivm_state * gallivm,LLVMValueRef buffers_ptr,LLVMValueRef index,unsigned buffers_limit)60 lp_llvm_descriptor_base(struct gallivm_state *gallivm,
61 LLVMValueRef buffers_ptr,
62 LLVMValueRef index, unsigned buffers_limit)
63 {
64 LLVMBuilderRef builder = gallivm->builder;
65
66 LLVMValueRef desc_set_index = LLVMBuildExtractValue(builder, index, 0, "");
67 if (LLVMGetTypeKind(LLVMTypeOf(desc_set_index)) == LLVMVectorTypeKind)
68 desc_set_index = LLVMBuildExtractElement(builder, desc_set_index, lp_build_const_int32(gallivm, 0), "");
69 LLVMValueRef desc_set_base = lp_llvm_buffer_base(gallivm, buffers_ptr, desc_set_index, buffers_limit);
70
71 LLVMValueRef binding_index = LLVMBuildExtractValue(builder, index, 1, "");
72 if (LLVMGetTypeKind(LLVMTypeOf(binding_index)) == LLVMVectorTypeKind)
73 binding_index = LLVMBuildExtractElement(builder, binding_index, lp_build_const_int32(gallivm, 0), "");
74
75 LLVMValueRef binding_offset = LLVMBuildMul(builder, binding_index, lp_build_const_int32(gallivm, sizeof(struct lp_descriptor)), "");
76 LLVMTypeRef int64_type = LLVMInt64TypeInContext(gallivm->context);
77 binding_offset = LLVMBuildIntCast2(builder, binding_offset, int64_type, false, "");
78
79 LLVMValueRef desc_ptr = LLVMBuildPtrToInt(builder, desc_set_base, int64_type, "");
80 return LLVMBuildAdd(builder, desc_ptr, binding_offset, "");
81 }
82
83 static LLVMValueRef
lp_llvm_buffer_member(struct gallivm_state * gallivm,LLVMValueRef buffers_ptr,LLVMValueRef buffers_offset,unsigned buffers_limit,unsigned member_index,const char * member_name)84 lp_llvm_buffer_member(struct gallivm_state *gallivm,
85 LLVMValueRef buffers_ptr,
86 LLVMValueRef buffers_offset,
87 unsigned buffers_limit,
88 unsigned member_index,
89 const char *member_name)
90 {
91 LLVMBuilderRef builder = gallivm->builder;
92
93 LLVMTypeRef buffer_type = lp_build_create_jit_buffer_type(gallivm);
94
95 LLVMValueRef ptr;
96 if (LLVMGetTypeKind(LLVMTypeOf(buffers_offset)) == LLVMArrayTypeKind) {
97 LLVMValueRef desc_ptr = lp_llvm_descriptor_base(gallivm, buffers_ptr, buffers_offset, buffers_limit);
98
99 LLVMTypeRef buffer_ptr_type = LLVMPointerType(buffer_type, 0);
100 desc_ptr = LLVMBuildIntToPtr(builder, desc_ptr, buffer_ptr_type, "");
101
102 LLVMValueRef indices[2] = {
103 lp_build_const_int32(gallivm, 0),
104 lp_build_const_int32(gallivm, member_index),
105 };
106 ptr = LLVMBuildGEP2(builder, buffer_type, desc_ptr, indices, ARRAY_SIZE(indices), "");
107 } else {
108 LLVMValueRef indices[3];
109
110 indices[0] = lp_build_const_int32(gallivm, 0);
111 LLVMValueRef cond = LLVMBuildICmp(gallivm->builder, LLVMIntULT, buffers_offset, lp_build_const_int32(gallivm, buffers_limit), "");
112 indices[1] = LLVMBuildSelect(gallivm->builder, cond, buffers_offset, lp_build_const_int32(gallivm, 0), "");
113 indices[2] = lp_build_const_int32(gallivm, member_index);
114
115 LLVMTypeRef buffers_type = LLVMArrayType(buffer_type, buffers_limit);
116 ptr = LLVMBuildGEP2(builder, buffers_type, buffers_ptr, indices, ARRAY_SIZE(indices), "");
117 }
118
119 LLVMTypeRef res_type = LLVMStructGetTypeAtIndex(buffer_type, member_index);
120 LLVMValueRef res = LLVMBuildLoad2(builder, res_type, ptr, "");
121
122 lp_build_name(res, "buffer.%s", member_name);
123
124 return res;
125 }
126
127 LLVMValueRef
lp_llvm_buffer_base(struct gallivm_state * gallivm,LLVMValueRef buffers_ptr,LLVMValueRef buffers_offset,unsigned buffers_limit)128 lp_llvm_buffer_base(struct gallivm_state *gallivm,
129 LLVMValueRef buffers_ptr, LLVMValueRef buffers_offset, unsigned buffers_limit)
130 {
131 return lp_llvm_buffer_member(gallivm, buffers_ptr, buffers_offset, buffers_limit, LP_JIT_BUFFER_BASE, "base");
132 }
133
134 LLVMValueRef
lp_llvm_buffer_num_elements(struct gallivm_state * gallivm,LLVMValueRef buffers_ptr,LLVMValueRef buffers_offset,unsigned buffers_limit)135 lp_llvm_buffer_num_elements(struct gallivm_state *gallivm,
136 LLVMValueRef buffers_ptr, LLVMValueRef buffers_offset, unsigned buffers_limit)
137 {
138 return lp_llvm_buffer_member(gallivm, buffers_ptr, buffers_offset, buffers_limit, LP_JIT_BUFFER_NUM_ELEMENTS, "num_elements");
139 }
140
141 static LLVMTypeRef
lp_build_create_jit_texture_type(struct gallivm_state * gallivm)142 lp_build_create_jit_texture_type(struct gallivm_state *gallivm)
143 {
144 LLVMContextRef lc = gallivm->context;
145 LLVMTypeRef texture_type;
146 LLVMTypeRef elem_types[LP_JIT_TEXTURE_NUM_FIELDS];
147
148 /* struct lp_jit_texture */
149 elem_types[LP_JIT_SAMPLER_INDEX_DUMMY] =
150 elem_types[LP_JIT_TEXTURE_WIDTH] = LLVMInt32TypeInContext(lc);
151 elem_types[LP_JIT_TEXTURE_HEIGHT] =
152 elem_types[LP_JIT_TEXTURE_DEPTH] = LLVMInt16TypeInContext(lc);
153 elem_types[LP_JIT_TEXTURE_FIRST_LEVEL] =
154 elem_types[LP_JIT_TEXTURE_LAST_LEVEL] = LLVMInt8TypeInContext(lc);
155 elem_types[LP_JIT_TEXTURE_BASE] = LLVMPointerType(LLVMInt8TypeInContext(lc), 0);
156 elem_types[LP_JIT_TEXTURE_ROW_STRIDE] =
157 elem_types[LP_JIT_TEXTURE_IMG_STRIDE] =
158 elem_types[LP_JIT_TEXTURE_MIP_OFFSETS] =
159 LLVMArrayType(LLVMInt32TypeInContext(lc), PIPE_MAX_TEXTURE_LEVELS);
160
161 texture_type = LLVMStructTypeInContext(lc, elem_types,
162 ARRAY_SIZE(elem_types), 0);
163
164 LP_CHECK_MEMBER_OFFSET(struct lp_jit_texture, width,
165 gallivm->target, texture_type,
166 LP_JIT_TEXTURE_WIDTH);
167 LP_CHECK_MEMBER_OFFSET(struct lp_jit_texture, height,
168 gallivm->target, texture_type,
169 LP_JIT_TEXTURE_HEIGHT);
170 LP_CHECK_MEMBER_OFFSET(struct lp_jit_texture, depth,
171 gallivm->target, texture_type,
172 LP_JIT_TEXTURE_DEPTH);
173 LP_CHECK_MEMBER_OFFSET(struct lp_jit_texture, base,
174 gallivm->target, texture_type,
175 LP_JIT_TEXTURE_BASE);
176 LP_CHECK_MEMBER_OFFSET(struct lp_jit_texture, row_stride,
177 gallivm->target, texture_type,
178 LP_JIT_TEXTURE_ROW_STRIDE);
179 LP_CHECK_MEMBER_OFFSET(struct lp_jit_texture, img_stride,
180 gallivm->target, texture_type,
181 LP_JIT_TEXTURE_IMG_STRIDE);
182 LP_CHECK_MEMBER_OFFSET(struct lp_jit_texture, first_level,
183 gallivm->target, texture_type,
184 LP_JIT_TEXTURE_FIRST_LEVEL);
185 LP_CHECK_MEMBER_OFFSET(struct lp_jit_texture, last_level,
186 gallivm->target, texture_type,
187 LP_JIT_TEXTURE_LAST_LEVEL);
188 LP_CHECK_MEMBER_OFFSET(struct lp_jit_texture, mip_offsets,
189 gallivm->target, texture_type,
190 LP_JIT_TEXTURE_MIP_OFFSETS);
191 LP_CHECK_STRUCT_SIZE(struct lp_jit_texture,
192 gallivm->target, texture_type);
193 return texture_type;
194 }
195
196 static LLVMTypeRef
lp_build_create_jit_sampler_type(struct gallivm_state * gallivm)197 lp_build_create_jit_sampler_type(struct gallivm_state *gallivm)
198 {
199 LLVMContextRef lc = gallivm->context;
200 LLVMTypeRef sampler_type;
201 LLVMTypeRef elem_types[LP_JIT_SAMPLER_NUM_FIELDS];
202 elem_types[LP_JIT_SAMPLER_MIN_LOD] =
203 elem_types[LP_JIT_SAMPLER_MAX_LOD] =
204 elem_types[LP_JIT_SAMPLER_LOD_BIAS] = LLVMFloatTypeInContext(lc);
205 elem_types[LP_JIT_SAMPLER_BORDER_COLOR] =
206 LLVMArrayType(LLVMFloatTypeInContext(lc), 4);
207
208 sampler_type = LLVMStructTypeInContext(lc, elem_types,
209 ARRAY_SIZE(elem_types), 0);
210
211 LP_CHECK_MEMBER_OFFSET(struct lp_jit_sampler, min_lod,
212 gallivm->target, sampler_type,
213 LP_JIT_SAMPLER_MIN_LOD);
214 LP_CHECK_MEMBER_OFFSET(struct lp_jit_sampler, max_lod,
215 gallivm->target, sampler_type,
216 LP_JIT_SAMPLER_MAX_LOD);
217 LP_CHECK_MEMBER_OFFSET(struct lp_jit_sampler, lod_bias,
218 gallivm->target, sampler_type,
219 LP_JIT_SAMPLER_LOD_BIAS);
220 LP_CHECK_MEMBER_OFFSET(struct lp_jit_sampler, border_color,
221 gallivm->target, sampler_type,
222 LP_JIT_SAMPLER_BORDER_COLOR);
223 LP_CHECK_STRUCT_SIZE(struct lp_jit_sampler,
224 gallivm->target, sampler_type);
225 return sampler_type;
226 }
227
228 static LLVMTypeRef
lp_build_create_jit_image_type(struct gallivm_state * gallivm)229 lp_build_create_jit_image_type(struct gallivm_state *gallivm)
230 {
231 LLVMContextRef lc = gallivm->context;
232 LLVMTypeRef image_type;
233 LLVMTypeRef elem_types[LP_JIT_IMAGE_NUM_FIELDS];
234 elem_types[LP_JIT_IMAGE_WIDTH] = LLVMInt32TypeInContext(lc);
235 elem_types[LP_JIT_IMAGE_HEIGHT] =
236 elem_types[LP_JIT_IMAGE_DEPTH] = LLVMInt16TypeInContext(lc);
237 elem_types[LP_JIT_IMAGE_NUM_SAMPLES] = LLVMInt8TypeInContext(lc);
238 elem_types[LP_JIT_IMAGE_BASE] =
239 elem_types[LP_JIT_IMAGE_RESIDENCY] = LLVMPointerType(LLVMInt8TypeInContext(lc), 0);
240 elem_types[LP_JIT_IMAGE_ROW_STRIDE] =
241 elem_types[LP_JIT_IMAGE_IMG_STRIDE] =
242 elem_types[LP_JIT_IMAGE_SAMPLE_STRIDE] =
243 elem_types[LP_JIT_IMAGE_BASE_OFFSET] = LLVMInt32TypeInContext(lc);
244
245 image_type = LLVMStructTypeInContext(lc, elem_types,
246 ARRAY_SIZE(elem_types), 0);
247 LP_CHECK_MEMBER_OFFSET(struct lp_jit_image, width,
248 gallivm->target, image_type,
249 LP_JIT_IMAGE_WIDTH);
250 LP_CHECK_MEMBER_OFFSET(struct lp_jit_image, height,
251 gallivm->target, image_type,
252 LP_JIT_IMAGE_HEIGHT);
253 LP_CHECK_MEMBER_OFFSET(struct lp_jit_image, depth,
254 gallivm->target, image_type,
255 LP_JIT_IMAGE_DEPTH);
256 LP_CHECK_MEMBER_OFFSET(struct lp_jit_image, base,
257 gallivm->target, image_type,
258 LP_JIT_IMAGE_BASE);
259 LP_CHECK_MEMBER_OFFSET(struct lp_jit_image, row_stride,
260 gallivm->target, image_type,
261 LP_JIT_IMAGE_ROW_STRIDE);
262 LP_CHECK_MEMBER_OFFSET(struct lp_jit_image, img_stride,
263 gallivm->target, image_type,
264 LP_JIT_IMAGE_IMG_STRIDE);
265 LP_CHECK_MEMBER_OFFSET(struct lp_jit_image, num_samples,
266 gallivm->target, image_type,
267 LP_JIT_IMAGE_NUM_SAMPLES);
268 LP_CHECK_MEMBER_OFFSET(struct lp_jit_image, sample_stride,
269 gallivm->target, image_type,
270 LP_JIT_IMAGE_SAMPLE_STRIDE);
271 return image_type;
272 }
273
274 LLVMTypeRef
lp_build_jit_resources_type(struct gallivm_state * gallivm)275 lp_build_jit_resources_type(struct gallivm_state *gallivm)
276 {
277 LLVMTypeRef elem_types[LP_JIT_RES_COUNT];
278 LLVMTypeRef resources_type;
279 LLVMTypeRef texture_type, sampler_type, image_type, buffer_type;
280
281 buffer_type = lp_build_create_jit_buffer_type(gallivm);
282 texture_type = lp_build_create_jit_texture_type(gallivm);
283 sampler_type = lp_build_create_jit_sampler_type(gallivm);
284 image_type = lp_build_create_jit_image_type(gallivm);
285 elem_types[LP_JIT_RES_CONSTANTS] = LLVMArrayType(buffer_type,
286 LP_MAX_TGSI_CONST_BUFFERS);
287 elem_types[LP_JIT_RES_SSBOS] =
288 LLVMArrayType(buffer_type, LP_MAX_TGSI_SHADER_BUFFERS);
289 elem_types[LP_JIT_RES_TEXTURES] = LLVMArrayType(texture_type,
290 PIPE_MAX_SHADER_SAMPLER_VIEWS);
291 elem_types[LP_JIT_RES_SAMPLERS] = LLVMArrayType(sampler_type,
292 PIPE_MAX_SAMPLERS);
293 elem_types[LP_JIT_RES_IMAGES] = LLVMArrayType(image_type,
294 PIPE_MAX_SHADER_IMAGES);
295
296 resources_type = LLVMStructTypeInContext(gallivm->context, elem_types,
297 ARRAY_SIZE(elem_types), 0);
298
299 LP_CHECK_MEMBER_OFFSET(struct lp_jit_resources, constants,
300 gallivm->target, resources_type,
301 LP_JIT_RES_CONSTANTS);
302 LP_CHECK_MEMBER_OFFSET(struct lp_jit_resources, ssbos,
303 gallivm->target, resources_type,
304 LP_JIT_RES_SSBOS);
305 LP_CHECK_MEMBER_OFFSET(struct lp_jit_resources, textures,
306 gallivm->target, resources_type,
307 LP_JIT_RES_TEXTURES);
308 LP_CHECK_MEMBER_OFFSET(struct lp_jit_resources, samplers,
309 gallivm->target, resources_type,
310 LP_JIT_RES_SAMPLERS);
311 LP_CHECK_MEMBER_OFFSET(struct lp_jit_resources, images,
312 gallivm->target, resources_type,
313 LP_JIT_RES_IMAGES);
314
315 return resources_type;
316 }
317
318
319 /**
320 * Fetch the specified member of the lp_jit_texture structure.
321 * \param emit_load if TRUE, emit the LLVM load instruction to actually
322 * fetch the field's value. Otherwise, just emit the
323 * GEP code to address the field.
324 *
325 * @sa http://llvm.org/docs/GetElementPtr.html
326 */
327 static LLVMValueRef
lp_build_llvm_texture_member(struct gallivm_state * gallivm,LLVMTypeRef resources_type,LLVMValueRef resources_ptr,unsigned texture_unit,LLVMValueRef texture_unit_offset,unsigned member_index,const char * member_name,bool emit_load,LLVMTypeRef * out_type)328 lp_build_llvm_texture_member(struct gallivm_state *gallivm,
329 LLVMTypeRef resources_type,
330 LLVMValueRef resources_ptr,
331 unsigned texture_unit,
332 LLVMValueRef texture_unit_offset,
333 unsigned member_index,
334 const char *member_name,
335 bool emit_load,
336 LLVMTypeRef *out_type)
337 {
338 LLVMBuilderRef builder = gallivm->builder;
339
340 LLVMValueRef ptr;
341 if (gallivm->texture_descriptor) {
342 static_assert(offsetof(struct lp_descriptor, texture) == 0, "Invalid texture offset!");
343 LLVMValueRef texture_ptr = gallivm->texture_descriptor;
344
345 LLVMTypeRef texture_ptr_type = LLVMStructGetTypeAtIndex(resources_type, LP_JIT_RES_TEXTURES);
346 LLVMTypeRef texture_type = LLVMGetElementType(texture_ptr_type);
347 texture_ptr_type = LLVMPointerType(texture_type, 0);
348
349 texture_ptr = LLVMBuildIntToPtr(builder, texture_ptr, texture_ptr_type, "");
350
351 LLVMValueRef indices[2] = {
352 lp_build_const_int32(gallivm, 0),
353 lp_build_const_int32(gallivm, member_index),
354 };
355 ptr = LLVMBuildGEP2(builder, texture_type, texture_ptr, indices, ARRAY_SIZE(indices), "");
356 } else {
357 LLVMValueRef indices[4];
358
359 assert(texture_unit < PIPE_MAX_SHADER_SAMPLER_VIEWS);
360
361 /* resources[0] */
362 indices[0] = lp_build_const_int32(gallivm, 0);
363 /* resources[0].textures */
364 indices[1] = lp_build_const_int32(gallivm, LP_JIT_RES_TEXTURES);
365 /* resources[0].textures[unit] */
366 indices[2] = lp_build_const_int32(gallivm, texture_unit);
367 if (texture_unit_offset) {
368 indices[2] = LLVMBuildAdd(gallivm->builder, indices[2],
369 texture_unit_offset, "");
370 LLVMValueRef cond =
371 LLVMBuildICmp(gallivm->builder, LLVMIntULT,
372 indices[2],
373 lp_build_const_int32(gallivm,
374 PIPE_MAX_SHADER_SAMPLER_VIEWS), "");
375 indices[2] = LLVMBuildSelect(gallivm->builder, cond, indices[2],
376 lp_build_const_int32(gallivm,
377 texture_unit), "");
378 }
379 /* resources[0].textures[unit].member */
380 indices[3] = lp_build_const_int32(gallivm, member_index);
381
382 ptr = LLVMBuildGEP2(builder, resources_type, resources_ptr, indices, ARRAY_SIZE(indices), "");
383 }
384
385 LLVMValueRef res;
386 if (emit_load) {
387 LLVMTypeRef tex_type = LLVMStructGetTypeAtIndex(resources_type, LP_JIT_RES_TEXTURES);
388 LLVMTypeRef res_type = LLVMStructGetTypeAtIndex(LLVMGetElementType(tex_type), member_index);
389 res = LLVMBuildLoad2(builder, res_type, ptr, "");
390 } else
391 res = ptr;
392
393 if (out_type) {
394 LLVMTypeRef tex_type = LLVMStructGetTypeAtIndex(resources_type, LP_JIT_RES_TEXTURES);
395 LLVMTypeRef res_type = LLVMStructGetTypeAtIndex(LLVMGetElementType(tex_type), member_index);
396 *out_type = res_type;
397 }
398
399 lp_build_name(res, "resources.texture%u.%s", texture_unit, member_name);
400
401 return res;
402 }
403
404 static LLVMValueRef
lp_build_llvm_texture_residency(struct gallivm_state * gallivm,LLVMTypeRef resources_type,LLVMValueRef resources_ptr,unsigned texture_unit,LLVMValueRef texture_unit_offset)405 lp_build_llvm_texture_residency(struct gallivm_state *gallivm,
406 LLVMTypeRef resources_type,
407 LLVMValueRef resources_ptr,
408 unsigned texture_unit,
409 LLVMValueRef texture_unit_offset)
410 {
411 LLVMBuilderRef builder = gallivm->builder;
412
413 static_assert(offsetof(struct lp_descriptor, texture) == 0, "Invalid texture offset");
414 LLVMValueRef texture_ptr = gallivm->texture_descriptor;
415
416 LLVMTypeRef texture_ptr_type = LLVMStructGetTypeAtIndex(resources_type, LP_JIT_RES_TEXTURES);
417 LLVMTypeRef texture_type = LLVMGetElementType(texture_ptr_type);
418 texture_ptr_type = LLVMPointerType(texture_type, 0);
419
420 texture_ptr = LLVMBuildIntToPtr(builder, texture_ptr, texture_ptr_type, "");
421
422 static_assert(offsetof(struct lp_jit_texture, row_stride) == offsetof(struct lp_jit_texture, residency),
423 "Invalid texture descriptor layout");
424 LLVMValueRef indices[2] = {
425 lp_build_const_int32(gallivm, 0),
426 lp_build_const_int32(gallivm, LP_JIT_TEXTURE_ROW_STRIDE),
427 };
428 LLVMValueRef ptr = LLVMBuildGEP2(builder, texture_type, texture_ptr, indices, ARRAY_SIZE(indices), "");
429
430 LLVMTypeRef residency_type = LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0);
431 ptr = LLVMBuildBitCast(builder, ptr, LLVMPointerType(residency_type, 0), "");
432
433 return LLVMBuildLoad2(builder, residency_type, ptr, "");
434 }
435
436
437 /**
438 * Helper macro to instantiate the functions that generate the code to
439 * fetch the members of lp_jit_texture to fulfill the sampler code
440 * generator requests.
441 *
442 * This complexity is the price we have to pay to keep the texture
443 * sampler code generator a reusable module without dependencies to
444 * llvmpipe internals.
445 */
446 #define LP_BUILD_LLVM_TEXTURE_MEMBER(_name, _index, _emit_load) \
447 static LLVMValueRef \
448 lp_build_llvm_texture_##_name(struct gallivm_state *gallivm, \
449 LLVMTypeRef resources_type, \
450 LLVMValueRef resources_ptr, \
451 unsigned texture_unit, \
452 LLVMValueRef texture_unit_offset) \
453 { \
454 return lp_build_llvm_texture_member(gallivm, resources_type, resources_ptr, \
455 texture_unit, texture_unit_offset, \
456 _index, #_name, _emit_load, NULL ); \
457 }
458
459 #define LP_BUILD_LLVM_TEXTURE_MEMBER_OUTTYPE(_name, _index, _emit_load) \
460 static LLVMValueRef \
461 lp_build_llvm_texture_##_name(struct gallivm_state *gallivm, \
462 LLVMTypeRef resources_type, \
463 LLVMValueRef resources_ptr, \
464 unsigned texture_unit, \
465 LLVMValueRef texture_unit_offset, \
466 LLVMTypeRef *out_type) \
467 { \
468 return lp_build_llvm_texture_member(gallivm, resources_type, resources_ptr, \
469 texture_unit, texture_unit_offset, \
470 _index, #_name, _emit_load, out_type ); \
471 }
472
LP_BUILD_LLVM_TEXTURE_MEMBER(width,LP_JIT_TEXTURE_WIDTH,true)473 LP_BUILD_LLVM_TEXTURE_MEMBER(width, LP_JIT_TEXTURE_WIDTH, true)
474 LP_BUILD_LLVM_TEXTURE_MEMBER(height, LP_JIT_TEXTURE_HEIGHT, true)
475 LP_BUILD_LLVM_TEXTURE_MEMBER(depth, LP_JIT_TEXTURE_DEPTH, true)
476 LP_BUILD_LLVM_TEXTURE_MEMBER(first_level, LP_JIT_TEXTURE_FIRST_LEVEL, true)
477 LP_BUILD_LLVM_TEXTURE_MEMBER(last_level, LP_JIT_TEXTURE_LAST_LEVEL, true)
478 LP_BUILD_LLVM_TEXTURE_MEMBER(base_ptr, LP_JIT_TEXTURE_BASE, true)
479 LP_BUILD_LLVM_TEXTURE_MEMBER_OUTTYPE(row_stride, LP_JIT_TEXTURE_ROW_STRIDE, false)
480 LP_BUILD_LLVM_TEXTURE_MEMBER_OUTTYPE(img_stride, LP_JIT_TEXTURE_IMG_STRIDE, false)
481 LP_BUILD_LLVM_TEXTURE_MEMBER_OUTTYPE(mip_offsets, LP_JIT_TEXTURE_MIP_OFFSETS, false)
482
483 /**
484 * Fetch the specified member of the lp_jit_sampler structure.
485 * \param emit_load if TRUE, emit the LLVM load instruction to actually
486 * fetch the field's value. Otherwise, just emit the
487 * GEP code to address the field.
488 *
489 * @sa http://llvm.org/docs/GetElementPtr.html
490 */
491 static LLVMValueRef
492 lp_build_llvm_sampler_member(struct gallivm_state *gallivm,
493 LLVMTypeRef resources_type,
494 LLVMValueRef resources_ptr,
495 unsigned sampler_unit,
496 unsigned member_index,
497 const char *member_name,
498 bool emit_load)
499 {
500 LLVMBuilderRef builder = gallivm->builder;
501
502 LLVMValueRef ptr;
503 if (gallivm->sampler_descriptor) {
504 LLVMValueRef sampler_offset = lp_build_const_int64(gallivm, offsetof(struct lp_descriptor, sampler));
505 LLVMValueRef sampler_ptr = LLVMBuildAdd(builder, gallivm->sampler_descriptor, sampler_offset, "");
506
507 LLVMTypeRef sampler_ptr_type = LLVMStructGetTypeAtIndex(resources_type, LP_JIT_RES_SAMPLERS);
508 LLVMTypeRef sampler_type = LLVMGetElementType(sampler_ptr_type);
509 sampler_ptr_type = LLVMPointerType(sampler_type, 0);
510
511 sampler_ptr = LLVMBuildIntToPtr(builder, sampler_ptr, sampler_ptr_type, "");
512
513 LLVMValueRef indices[2] = {
514 lp_build_const_int32(gallivm, 0),
515 lp_build_const_int32(gallivm, member_index),
516 };
517 ptr = LLVMBuildGEP2(builder, sampler_type, sampler_ptr, indices, ARRAY_SIZE(indices), "");
518 } else {
519 LLVMValueRef indices[4];
520
521 assert(sampler_unit < PIPE_MAX_SAMPLERS);
522
523 /* resources[0] */
524 indices[0] = lp_build_const_int32(gallivm, 0);
525 /* resources[0].samplers */
526 indices[1] = lp_build_const_int32(gallivm, LP_JIT_RES_SAMPLERS);
527 /* resources[0].samplers[unit] */
528 indices[2] = lp_build_const_int32(gallivm, sampler_unit);
529 /* resources[0].samplers[unit].member */
530 indices[3] = lp_build_const_int32(gallivm, member_index);
531
532 ptr = LLVMBuildGEP2(builder, resources_type, resources_ptr, indices, ARRAY_SIZE(indices), "");
533 }
534
535 LLVMValueRef res;
536 if (emit_load) {
537 LLVMTypeRef samp_type = LLVMStructGetTypeAtIndex(resources_type, LP_JIT_RES_SAMPLERS);
538 LLVMTypeRef res_type = LLVMStructGetTypeAtIndex(LLVMGetElementType(samp_type), member_index);
539 res = LLVMBuildLoad2(builder, res_type, ptr, "");
540 } else
541 res = ptr;
542
543 lp_build_name(res, "resources.sampler%u.%s", sampler_unit, member_name);
544
545 return res;
546 }
547
548
549 #define LP_BUILD_LLVM_SAMPLER_MEMBER(_name, _index, _emit_load) \
550 static LLVMValueRef \
551 lp_build_llvm_sampler_##_name( struct gallivm_state *gallivm, \
552 LLVMTypeRef resources_type, \
553 LLVMValueRef resources_ptr, \
554 unsigned sampler_unit) \
555 { \
556 return lp_build_llvm_sampler_member(gallivm, resources_type, resources_ptr, \
557 sampler_unit, _index, #_name, _emit_load ); \
558 }
559
560
LP_BUILD_LLVM_SAMPLER_MEMBER(min_lod,LP_JIT_SAMPLER_MIN_LOD,true)561 LP_BUILD_LLVM_SAMPLER_MEMBER(min_lod, LP_JIT_SAMPLER_MIN_LOD, true)
562 LP_BUILD_LLVM_SAMPLER_MEMBER(max_lod, LP_JIT_SAMPLER_MAX_LOD, true)
563 LP_BUILD_LLVM_SAMPLER_MEMBER(lod_bias, LP_JIT_SAMPLER_LOD_BIAS, true)
564 LP_BUILD_LLVM_SAMPLER_MEMBER(border_color, LP_JIT_SAMPLER_BORDER_COLOR, false)
565
566 /**
567 * Fetch the specified member of the lp_jit_image structure.
568 * \param emit_load if TRUE, emit the LLVM load instruction to actually
569 * fetch the field's value. Otherwise, just emit the
570 * GEP code to address the field.
571 *
572 * @sa http://llvm.org/docs/GetElementPtr.html
573 */
574 static LLVMValueRef
575 lp_build_llvm_image_member(struct gallivm_state *gallivm,
576 LLVMTypeRef resources_type,
577 LLVMValueRef resources_ptr,
578 unsigned image_unit,
579 LLVMValueRef image_unit_offset,
580 unsigned member_index,
581 const char *member_name,
582 bool emit_load)
583 {
584 LLVMBuilderRef builder = gallivm->builder;
585
586 LLVMValueRef ptr;
587 if (gallivm->texture_descriptor) {
588 LLVMValueRef image_offset = lp_build_const_int64(gallivm, offsetof(struct lp_descriptor, image));
589 LLVMValueRef image_ptr = LLVMBuildAdd(builder, gallivm->texture_descriptor, image_offset, "");
590
591 LLVMTypeRef image_ptr_type = LLVMStructGetTypeAtIndex(resources_type, LP_JIT_RES_IMAGES);
592 LLVMTypeRef image_type = LLVMGetElementType(image_ptr_type);
593 image_ptr_type = LLVMPointerType(image_type, 0);
594
595 image_ptr = LLVMBuildIntToPtr(builder, image_ptr, image_ptr_type, "");
596
597 LLVMValueRef indices[2] = {
598 lp_build_const_int32(gallivm, 0),
599 lp_build_const_int32(gallivm, member_index),
600 };
601 ptr = LLVMBuildGEP2(builder, image_type, image_ptr, indices, ARRAY_SIZE(indices), "");
602 } else {
603 LLVMValueRef indices[4];
604
605 assert(image_unit < PIPE_MAX_SHADER_IMAGES);
606
607 /* resources[0] */
608 indices[0] = lp_build_const_int32(gallivm, 0);
609 /* resources[0].images */
610 indices[1] = lp_build_const_int32(gallivm, LP_JIT_RES_IMAGES);
611 /* resources[0].images[unit] */
612 indices[2] = lp_build_const_int32(gallivm, image_unit);
613 if (image_unit_offset) {
614 indices[2] = LLVMBuildAdd(gallivm->builder, indices[2], image_unit_offset, "");
615 LLVMValueRef cond = LLVMBuildICmp(gallivm->builder, LLVMIntULT, indices[2], lp_build_const_int32(gallivm, PIPE_MAX_SHADER_IMAGES), "");
616 indices[2] = LLVMBuildSelect(gallivm->builder, cond, indices[2], lp_build_const_int32(gallivm, image_unit), "");
617 }
618 /* resources[0].images[unit].member */
619 indices[3] = lp_build_const_int32(gallivm, member_index);
620
621 ptr = LLVMBuildGEP2(builder, resources_type, resources_ptr, indices, ARRAY_SIZE(indices), "");
622 }
623
624 LLVMValueRef res;
625 if (emit_load) {
626 LLVMTypeRef img_type = LLVMStructGetTypeAtIndex(resources_type, LP_JIT_RES_IMAGES);
627 LLVMTypeRef res_type = LLVMStructGetTypeAtIndex(LLVMGetElementType(img_type), member_index);
628 res = LLVMBuildLoad2(builder, res_type, ptr, "");
629 } else
630 res = ptr;
631
632 lp_build_name(res, "resources.image%u.%s", image_unit, member_name);
633
634 return res;
635 }
636
637 /**
638 * Helper macro to instantiate the functions that generate the code to
639 * fetch the members of lp_jit_image to fulfill the sampler code
640 * generator requests.
641 *
642 * This complexity is the price we have to pay to keep the image
643 * sampler code generator a reusable module without dependencies to
644 * llvmpipe internals.
645 */
646 #define LP_BUILD_LLVM_IMAGE_MEMBER(_name, _index, _emit_load) \
647 static LLVMValueRef \
648 lp_build_llvm_image_##_name( struct gallivm_state *gallivm, \
649 LLVMTypeRef resources_type, \
650 LLVMValueRef resources_ptr, \
651 unsigned image_unit, LLVMValueRef image_unit_offset) \
652 { \
653 return lp_build_llvm_image_member(gallivm, resources_type, resources_ptr, \
654 image_unit, image_unit_offset, \
655 _index, #_name, _emit_load ); \
656 }
657
658 #define LP_BUILD_LLVM_IMAGE_MEMBER_OUTTYPE(_name, _index, _emit_load) \
659 static LLVMValueRef \
660 lp_build_llvm_image_##_name( struct gallivm_state *gallivm, \
661 LLVMTypeRef resources_type, \
662 LLVMValueRef resources_ptr, \
663 unsigned image_unit, LLVMValueRef image_unit_offset, \
664 LLVMTypeRef *out_type) \
665 { \
666 assert(!out_type); \
667 return lp_build_llvm_image_member(gallivm, resources_type, resources_ptr, \
668 image_unit, image_unit_offset, \
669 _index, #_name, _emit_load ); \
670 }
671
LP_BUILD_LLVM_IMAGE_MEMBER(width,LP_JIT_IMAGE_WIDTH,true)672 LP_BUILD_LLVM_IMAGE_MEMBER(width, LP_JIT_IMAGE_WIDTH, true)
673 LP_BUILD_LLVM_IMAGE_MEMBER(height, LP_JIT_IMAGE_HEIGHT, true)
674 LP_BUILD_LLVM_IMAGE_MEMBER(depth, LP_JIT_IMAGE_DEPTH, true)
675 LP_BUILD_LLVM_IMAGE_MEMBER(base_ptr, LP_JIT_IMAGE_BASE, true)
676 LP_BUILD_LLVM_IMAGE_MEMBER_OUTTYPE(row_stride, LP_JIT_IMAGE_ROW_STRIDE, true)
677 LP_BUILD_LLVM_IMAGE_MEMBER_OUTTYPE(img_stride, LP_JIT_IMAGE_IMG_STRIDE, true)
678 LP_BUILD_LLVM_IMAGE_MEMBER(num_samples, LP_JIT_IMAGE_NUM_SAMPLES, true)
679 LP_BUILD_LLVM_IMAGE_MEMBER(sample_stride, LP_JIT_IMAGE_SAMPLE_STRIDE, true)
680 LP_BUILD_LLVM_IMAGE_MEMBER(residency, LP_JIT_IMAGE_RESIDENCY, true)
681 LP_BUILD_LLVM_IMAGE_MEMBER(base_offset, LP_JIT_IMAGE_BASE_OFFSET, true)
682
683 void
684 lp_build_jit_fill_sampler_dynamic_state(struct lp_sampler_dynamic_state *state)
685 {
686 state->width = lp_build_llvm_texture_width;
687 state->height = lp_build_llvm_texture_height;
688 state->depth = lp_build_llvm_texture_depth;
689 state->first_level = lp_build_llvm_texture_first_level;
690 state->last_level = lp_build_llvm_texture_last_level;
691 state->base_ptr = lp_build_llvm_texture_base_ptr;
692 state->row_stride = lp_build_llvm_texture_row_stride;
693 state->img_stride = lp_build_llvm_texture_img_stride;
694 state->mip_offsets = lp_build_llvm_texture_mip_offsets;
695 state->residency = lp_build_llvm_texture_residency;
696
697 state->min_lod = lp_build_llvm_sampler_min_lod;
698 state->max_lod = lp_build_llvm_sampler_max_lod;
699 state->lod_bias = lp_build_llvm_sampler_lod_bias;
700 state->border_color = lp_build_llvm_sampler_border_color;
701 }
702
703 void
lp_build_jit_fill_image_dynamic_state(struct lp_sampler_dynamic_state * state)704 lp_build_jit_fill_image_dynamic_state(struct lp_sampler_dynamic_state *state)
705 {
706 state->width = lp_build_llvm_image_width;
707 state->height = lp_build_llvm_image_height;
708
709 state->depth = lp_build_llvm_image_depth;
710 state->base_ptr = lp_build_llvm_image_base_ptr;
711 state->row_stride = lp_build_llvm_image_row_stride;
712 state->img_stride = lp_build_llvm_image_img_stride;
713 state->last_level = lp_build_llvm_image_num_samples;
714 state->sample_stride = lp_build_llvm_image_sample_stride;
715 state->residency = lp_build_llvm_image_residency;
716 state->base_offset = lp_build_llvm_image_base_offset;
717 }
718
719 /**
720 * Create LLVM type for struct vertex_header;
721 */
722 LLVMTypeRef
lp_build_create_jit_vertex_header_type(struct gallivm_state * gallivm,int data_elems)723 lp_build_create_jit_vertex_header_type(struct gallivm_state *gallivm, int data_elems)
724 {
725 LLVMTargetDataRef target = gallivm->target;
726 LLVMTypeRef elem_types[3];
727 LLVMTypeRef vertex_header;
728 char struct_name[24];
729
730 snprintf(struct_name, 23, "vertex_header%d", data_elems);
731
732 elem_types[LP_JIT_VERTEX_HEADER_VERTEX_ID] = LLVMIntTypeInContext(gallivm->context, 32);
733 elem_types[LP_JIT_VERTEX_HEADER_CLIP_POS] = LLVMArrayType(LLVMFloatTypeInContext(gallivm->context), 4);
734 elem_types[LP_JIT_VERTEX_HEADER_DATA] = LLVMArrayType(elem_types[1], data_elems);
735
736 vertex_header = LLVMStructTypeInContext(gallivm->context, elem_types,
737 ARRAY_SIZE(elem_types), 0);
738
739 /* these are bit-fields and we can't take address of them
740 LP_CHECK_MEMBER_OFFSET(struct vertex_header, clipmask,
741 target, vertex_header,
742 LP_JIT_VERTEX_HEADER_CLIPMASK);
743 LP_CHECK_MEMBER_OFFSET(struct vertex_header, edgeflag,
744 target, vertex_header,
745 LP_JIT_VERTEX_HEADER_EDGEFLAG);
746 LP_CHECK_MEMBER_OFFSET(struct vertex_header, pad,
747 target, vertex_header,
748 LP_JIT_VERTEX_HEADER_PAD);
749 LP_CHECK_MEMBER_OFFSET(struct vertex_header, vertex_id,
750 target, vertex_header,
751 LP_JIT_VERTEX_HEADER_VERTEX_ID);
752 */
753 (void) target; /* silence unused var warning for non-debug build */
754 LP_CHECK_MEMBER_OFFSET(struct vertex_header, clip_pos,
755 target, vertex_header,
756 LP_JIT_VERTEX_HEADER_CLIP_POS);
757 LP_CHECK_MEMBER_OFFSET(struct vertex_header, data,
758 target, vertex_header,
759 LP_JIT_VERTEX_HEADER_DATA);
760
761 assert(LLVMABISizeOfType(target, vertex_header) ==
762 offsetof(struct vertex_header, data[data_elems]));
763
764 return vertex_header;
765 }
766
767 LLVMTypeRef
lp_build_sample_function_type(struct gallivm_state * gallivm,uint32_t sample_key)768 lp_build_sample_function_type(struct gallivm_state *gallivm, uint32_t sample_key)
769 {
770 struct lp_type type;
771 memset(&type, 0, sizeof type);
772 type.floating = true; /* floating point values */
773 type.sign = true; /* values are signed */
774 type.norm = false; /* values are not limited to [0,1] or [-1,1] */
775 type.width = 32; /* 32-bit float */
776 type.length = MIN2(lp_native_vector_width / 32, 16); /* n*4 elements per vector */
777
778 enum lp_sampler_op_type op_type = (sample_key & LP_SAMPLER_OP_TYPE_MASK) >> LP_SAMPLER_OP_TYPE_SHIFT;
779 enum lp_sampler_lod_control lod_control = (sample_key & LP_SAMPLER_LOD_CONTROL_MASK) >> LP_SAMPLER_LOD_CONTROL_SHIFT;
780
781 LLVMTypeRef arg_types[LP_MAX_TEX_FUNC_ARGS];
782 LLVMTypeRef ret_type;
783 LLVMTypeRef val_type[5];
784 uint32_t num_params = 0;
785
786 LLVMTypeRef coord_type;
787 if (op_type == LP_SAMPLER_OP_FETCH)
788 coord_type = lp_build_vec_type(gallivm, lp_int_type(type));
789 else
790 coord_type = lp_build_vec_type(gallivm, type);
791
792 arg_types[num_params++] = LLVMInt64TypeInContext(gallivm->context);
793 arg_types[num_params++] = LLVMInt64TypeInContext(gallivm->context);
794
795 for (unsigned i = 0; i < 4; i++)
796 arg_types[num_params++] = coord_type;
797
798 if (sample_key & LP_SAMPLER_SHADOW)
799 arg_types[num_params++] = lp_build_vec_type(gallivm, type);
800
801 if (sample_key & LP_SAMPLER_FETCH_MS)
802 arg_types[num_params++] = lp_build_vec_type(gallivm, lp_uint_type(type));
803
804 if (sample_key & LP_SAMPLER_OFFSETS)
805 for (uint32_t i = 0; i < 3; i++)
806 arg_types[num_params++] = lp_build_int_vec_type(gallivm, type);
807
808 if (lod_control == LP_SAMPLER_LOD_BIAS || lod_control == LP_SAMPLER_LOD_EXPLICIT)
809 arg_types[num_params++] = coord_type;
810
811 val_type[0] = val_type[1] = val_type[2] = val_type[3] = lp_build_vec_type(gallivm, type);
812 val_type[4] = lp_build_int_vec_type(gallivm, type);
813 ret_type = LLVMStructTypeInContext(gallivm->context, val_type, 5, 0);
814 return LLVMFunctionType(ret_type, arg_types, num_params, false);
815 }
816
817 LLVMTypeRef
lp_build_size_function_type(struct gallivm_state * gallivm,const struct lp_sampler_size_query_params * params)818 lp_build_size_function_type(struct gallivm_state *gallivm,
819 const struct lp_sampler_size_query_params *params)
820 {
821 struct lp_type type;
822 memset(&type, 0, sizeof type);
823 type.floating = true; /* floating point values */
824 type.sign = true; /* values are signed */
825 type.norm = false; /* values are not limited to [0,1] or [-1,1] */
826 type.width = 32; /* 32-bit float */
827 type.length = MIN2(lp_native_vector_width / 32, 16); /* n*4 elements per vector */
828
829 LLVMTypeRef arg_types[LP_MAX_TEX_FUNC_ARGS];
830 LLVMTypeRef ret_type;
831 LLVMTypeRef val_type[4];
832 uint32_t num_params = 0;
833
834 arg_types[num_params++] = LLVMInt64TypeInContext(gallivm->context);
835
836 if (!params->samples_only)
837 arg_types[num_params++] = lp_build_int_vec_type(gallivm, type);
838
839 val_type[0] = val_type[1] = val_type[2] = val_type[3] = lp_build_int_vec_type(gallivm, type);
840 ret_type = LLVMStructTypeInContext(gallivm->context, val_type, 4, 0);
841 return LLVMFunctionType(ret_type, arg_types, num_params, false);
842 }
843
844 LLVMTypeRef
lp_build_image_function_type(struct gallivm_state * gallivm,const struct lp_img_params * params,bool ms)845 lp_build_image_function_type(struct gallivm_state *gallivm,
846 const struct lp_img_params *params, bool ms)
847 {
848 struct lp_type type;
849 memset(&type, 0, sizeof type);
850 type.floating = true; /* floating point values */
851 type.sign = true; /* values are signed */
852 type.norm = false; /* values are not limited to [0,1] or [-1,1] */
853 type.width = 32; /* 32-bit float */
854 type.length = MIN2(lp_native_vector_width / 32, 16); /* n*4 elements per vector */
855
856 LLVMTypeRef arg_types[LP_MAX_TEX_FUNC_ARGS];
857 LLVMTypeRef ret_type;
858 uint32_t num_params = 0;
859
860 arg_types[num_params++] = LLVMInt64TypeInContext(gallivm->context);
861
862 if (params->img_op != LP_IMG_LOAD && params->img_op != LP_IMG_LOAD_SPARSE)
863 arg_types[num_params++] = lp_build_int_vec_type(gallivm, type);
864
865 for (uint32_t i = 0; i < 3; i++)
866 arg_types[num_params++] = lp_build_vec_type(gallivm, lp_uint_type(type));
867
868 if (ms)
869 arg_types[num_params++] = lp_build_vec_type(gallivm, lp_uint_type(type));
870
871 uint32_t num_inputs = params->img_op != LP_IMG_LOAD && params->img_op != LP_IMG_LOAD_SPARSE ? 4 : 0;
872 if (params->img_op == LP_IMG_ATOMIC_CAS)
873 num_inputs = 8;
874
875 const struct util_format_description *desc = util_format_description(params->format);
876 LLVMTypeRef component_type = lp_build_vec_type(gallivm, lp_build_texel_type(type, desc));
877
878 for (uint32_t i = 0; i < num_inputs; i++)
879 arg_types[num_params++] = component_type;
880
881 if (params->img_op == LP_IMG_LOAD_SPARSE) {
882 LLVMTypeRef val_type[5];
883 val_type[0] = val_type[1] = val_type[2] = val_type[3] = component_type;
884 val_type[4] = lp_build_int_vec_type(gallivm, type);
885 ret_type = LLVMStructTypeInContext(gallivm->context, val_type, 5, 0);
886 } else if (params->img_op != LP_IMG_STORE) {
887 LLVMTypeRef val_type[4];
888 val_type[0] = val_type[1] = val_type[2] = val_type[3] = component_type;
889 ret_type = LLVMStructTypeInContext(gallivm->context, val_type, 4, 0);
890 } else {
891 ret_type = LLVMVoidTypeInContext(gallivm->context);
892 }
893
894 return LLVMFunctionType(ret_type, arg_types, num_params, false);
895 }
896