1 /*
2 * Copyright 2022 Red Hat.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * on the rights to use, copy, modify, merge, publish, distribute, sub
8 * license, and/or sell copies of the Software, and to permit persons to whom
9 * the Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
19 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
20 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
21 * USE OR OTHER DEALINGS IN THE SOFTWARE.
22 */
23
24 #include "util/compiler.h"
25 #include "gallivm/lp_bld.h"
26 #include "gallivm/lp_bld_init.h"
27 #include "gallivm/lp_bld_struct.h"
28 #include "gallivm/lp_bld_sample.h"
29 #include "gallivm/lp_bld_const.h"
30 #include "gallivm/lp_bld_debug.h"
31 #include "gallivm/lp_bld_ir_common.h"
32 #include "draw/draw_vertex_header.h"
33 #include "lp_bld_jit_types.h"
34
35
36 static LLVMTypeRef
lp_build_create_jit_buffer_type(struct gallivm_state * gallivm)37 lp_build_create_jit_buffer_type(struct gallivm_state *gallivm)
38 {
39 LLVMContextRef lc = gallivm->context;
40 LLVMTypeRef buffer_type;
41 LLVMTypeRef elem_types[LP_JIT_BUFFER_NUM_FIELDS];
42
43 elem_types[LP_JIT_BUFFER_BASE] = LLVMPointerType(LLVMInt32TypeInContext(lc), 0);
44 elem_types[LP_JIT_BUFFER_NUM_ELEMENTS] = LLVMInt32TypeInContext(lc);
45
46 buffer_type = LLVMStructTypeInContext(lc, elem_types,
47 ARRAY_SIZE(elem_types), 0);
48
49 LP_CHECK_MEMBER_OFFSET(struct lp_jit_buffer, f,
50 gallivm->target, buffer_type,
51 LP_JIT_BUFFER_BASE);
52
53 LP_CHECK_MEMBER_OFFSET(struct lp_jit_buffer, num_elements,
54 gallivm->target, buffer_type,
55 LP_JIT_BUFFER_NUM_ELEMENTS);
56 return buffer_type;
57 }
58
59 LLVMValueRef
lp_llvm_descriptor_base(struct gallivm_state * gallivm,LLVMValueRef buffers_ptr,LLVMValueRef index,unsigned buffers_limit)60 lp_llvm_descriptor_base(struct gallivm_state *gallivm,
61 LLVMValueRef buffers_ptr,
62 LLVMValueRef index, unsigned buffers_limit)
63 {
64 LLVMBuilderRef builder = gallivm->builder;
65
66 LLVMValueRef desc_set_index = LLVMBuildExtractValue(builder, index, 0, "");
67 if (LLVMGetTypeKind(LLVMTypeOf(desc_set_index)) == LLVMVectorTypeKind)
68 desc_set_index = LLVMBuildExtractElement(builder, desc_set_index, lp_build_const_int32(gallivm, 0), "");
69 LLVMValueRef desc_set_base = lp_llvm_buffer_base(gallivm, buffers_ptr, desc_set_index, buffers_limit);
70
71 LLVMValueRef binding_index = LLVMBuildExtractValue(builder, index, 1, "");
72 if (LLVMGetTypeKind(LLVMTypeOf(binding_index)) == LLVMVectorTypeKind)
73 binding_index = LLVMBuildExtractElement(builder, binding_index, lp_build_const_int32(gallivm, 0), "");
74
75 LLVMValueRef binding_offset = LLVMBuildMul(builder, binding_index, lp_build_const_int32(gallivm, sizeof(struct lp_descriptor)), "");
76 LLVMTypeRef int64_type = LLVMInt64TypeInContext(gallivm->context);
77 binding_offset = LLVMBuildIntCast2(builder, binding_offset, int64_type, false, "");
78
79 LLVMValueRef desc_ptr = LLVMBuildPtrToInt(builder, desc_set_base, int64_type, "");
80 return LLVMBuildAdd(builder, desc_ptr, binding_offset, "");
81 }
82
83 static LLVMValueRef
lp_llvm_buffer_member(struct gallivm_state * gallivm,LLVMValueRef buffers_ptr,LLVMValueRef buffers_offset,unsigned buffers_limit,unsigned member_index,const char * member_name)84 lp_llvm_buffer_member(struct gallivm_state *gallivm,
85 LLVMValueRef buffers_ptr,
86 LLVMValueRef buffers_offset,
87 unsigned buffers_limit,
88 unsigned member_index,
89 const char *member_name)
90 {
91 LLVMBuilderRef builder = gallivm->builder;
92
93 LLVMTypeRef buffer_type = lp_build_create_jit_buffer_type(gallivm);
94
95 LLVMValueRef ptr;
96 if (LLVMGetTypeKind(LLVMTypeOf(buffers_offset)) == LLVMArrayTypeKind) {
97 LLVMValueRef desc_ptr = lp_llvm_descriptor_base(gallivm, buffers_ptr, buffers_offset, buffers_limit);
98
99 LLVMTypeRef buffer_ptr_type = LLVMPointerType(buffer_type, 0);
100 desc_ptr = LLVMBuildIntToPtr(builder, desc_ptr, buffer_ptr_type, "");
101
102 LLVMValueRef indices[2] = {
103 lp_build_const_int32(gallivm, 0),
104 lp_build_const_int32(gallivm, member_index),
105 };
106 ptr = LLVMBuildGEP2(builder, buffer_type, desc_ptr, indices, ARRAY_SIZE(indices), "");
107 } else {
108 LLVMValueRef indices[3];
109
110 indices[0] = lp_build_const_int32(gallivm, 0);
111 LLVMValueRef cond = LLVMBuildICmp(gallivm->builder, LLVMIntULT, buffers_offset, lp_build_const_int32(gallivm, buffers_limit), "");
112 indices[1] = LLVMBuildSelect(gallivm->builder, cond, buffers_offset, lp_build_const_int32(gallivm, 0), "");
113 indices[2] = lp_build_const_int32(gallivm, member_index);
114
115 LLVMTypeRef buffers_type = LLVMArrayType(buffer_type, buffers_limit);
116 ptr = LLVMBuildGEP2(builder, buffers_type, buffers_ptr, indices, ARRAY_SIZE(indices), "");
117 }
118
119 LLVMTypeRef res_type = LLVMStructGetTypeAtIndex(buffer_type, member_index);
120 LLVMValueRef res = LLVMBuildLoad2(builder, res_type, ptr, "");
121
122 lp_build_name(res, "buffer.%s", member_name);
123
124 return res;
125 }
126
127 LLVMValueRef
lp_llvm_buffer_base(struct gallivm_state * gallivm,LLVMValueRef buffers_ptr,LLVMValueRef buffers_offset,unsigned buffers_limit)128 lp_llvm_buffer_base(struct gallivm_state *gallivm,
129 LLVMValueRef buffers_ptr, LLVMValueRef buffers_offset, unsigned buffers_limit)
130 {
131 return lp_llvm_buffer_member(gallivm, buffers_ptr, buffers_offset, buffers_limit, LP_JIT_BUFFER_BASE, "base");
132 }
133
134 LLVMValueRef
lp_llvm_buffer_num_elements(struct gallivm_state * gallivm,LLVMValueRef buffers_ptr,LLVMValueRef buffers_offset,unsigned buffers_limit)135 lp_llvm_buffer_num_elements(struct gallivm_state *gallivm,
136 LLVMValueRef buffers_ptr, LLVMValueRef buffers_offset, unsigned buffers_limit)
137 {
138 return lp_llvm_buffer_member(gallivm, buffers_ptr, buffers_offset, buffers_limit, LP_JIT_BUFFER_NUM_ELEMENTS, "num_elements");
139 }
140
141 static LLVMTypeRef
lp_build_create_jit_texture_type(struct gallivm_state * gallivm)142 lp_build_create_jit_texture_type(struct gallivm_state *gallivm)
143 {
144 LLVMContextRef lc = gallivm->context;
145 LLVMTypeRef texture_type;
146 LLVMTypeRef elem_types[LP_JIT_TEXTURE_NUM_FIELDS];
147
148 /* struct lp_jit_texture */
149 elem_types[LP_JIT_SAMPLER_INDEX_DUMMY] =
150 elem_types[LP_JIT_TEXTURE_WIDTH] = LLVMInt32TypeInContext(lc);
151 elem_types[LP_JIT_TEXTURE_HEIGHT] =
152 elem_types[LP_JIT_TEXTURE_DEPTH] = LLVMInt16TypeInContext(lc);
153 elem_types[LP_JIT_TEXTURE_FIRST_LEVEL] =
154 elem_types[LP_JIT_TEXTURE_LAST_LEVEL] = LLVMInt8TypeInContext(lc);
155 elem_types[LP_JIT_TEXTURE_BASE] = LLVMPointerType(LLVMInt8TypeInContext(lc), 0);
156 elem_types[LP_JIT_TEXTURE_ROW_STRIDE] =
157 elem_types[LP_JIT_TEXTURE_IMG_STRIDE] =
158 elem_types[LP_JIT_TEXTURE_MIP_OFFSETS] =
159 LLVMArrayType(LLVMInt32TypeInContext(lc), PIPE_MAX_TEXTURE_LEVELS);
160
161 texture_type = LLVMStructTypeInContext(lc, elem_types,
162 ARRAY_SIZE(elem_types), 0);
163
164 LP_CHECK_MEMBER_OFFSET(struct lp_jit_texture, width,
165 gallivm->target, texture_type,
166 LP_JIT_TEXTURE_WIDTH);
167 LP_CHECK_MEMBER_OFFSET(struct lp_jit_texture, height,
168 gallivm->target, texture_type,
169 LP_JIT_TEXTURE_HEIGHT);
170 LP_CHECK_MEMBER_OFFSET(struct lp_jit_texture, depth,
171 gallivm->target, texture_type,
172 LP_JIT_TEXTURE_DEPTH);
173 LP_CHECK_MEMBER_OFFSET(struct lp_jit_texture, base,
174 gallivm->target, texture_type,
175 LP_JIT_TEXTURE_BASE);
176 LP_CHECK_MEMBER_OFFSET(struct lp_jit_texture, row_stride,
177 gallivm->target, texture_type,
178 LP_JIT_TEXTURE_ROW_STRIDE);
179 LP_CHECK_MEMBER_OFFSET(struct lp_jit_texture, img_stride,
180 gallivm->target, texture_type,
181 LP_JIT_TEXTURE_IMG_STRIDE);
182 LP_CHECK_MEMBER_OFFSET(struct lp_jit_texture, first_level,
183 gallivm->target, texture_type,
184 LP_JIT_TEXTURE_FIRST_LEVEL);
185 LP_CHECK_MEMBER_OFFSET(struct lp_jit_texture, last_level,
186 gallivm->target, texture_type,
187 LP_JIT_TEXTURE_LAST_LEVEL);
188 LP_CHECK_MEMBER_OFFSET(struct lp_jit_texture, mip_offsets,
189 gallivm->target, texture_type,
190 LP_JIT_TEXTURE_MIP_OFFSETS);
191 LP_CHECK_STRUCT_SIZE(struct lp_jit_texture,
192 gallivm->target, texture_type);
193 return texture_type;
194 }
195
196 static LLVMTypeRef
lp_build_create_jit_sampler_type(struct gallivm_state * gallivm)197 lp_build_create_jit_sampler_type(struct gallivm_state *gallivm)
198 {
199 LLVMContextRef lc = gallivm->context;
200 LLVMTypeRef sampler_type;
201 LLVMTypeRef elem_types[LP_JIT_SAMPLER_NUM_FIELDS];
202 elem_types[LP_JIT_SAMPLER_MIN_LOD] =
203 elem_types[LP_JIT_SAMPLER_MAX_LOD] =
204 elem_types[LP_JIT_SAMPLER_LOD_BIAS] =
205 elem_types[LP_JIT_SAMPLER_MAX_ANISO] = LLVMFloatTypeInContext(lc);
206 elem_types[LP_JIT_SAMPLER_BORDER_COLOR] =
207 LLVMArrayType(LLVMFloatTypeInContext(lc), 4);
208
209 sampler_type = LLVMStructTypeInContext(lc, elem_types,
210 ARRAY_SIZE(elem_types), 0);
211
212 LP_CHECK_MEMBER_OFFSET(struct lp_jit_sampler, min_lod,
213 gallivm->target, sampler_type,
214 LP_JIT_SAMPLER_MIN_LOD);
215 LP_CHECK_MEMBER_OFFSET(struct lp_jit_sampler, max_lod,
216 gallivm->target, sampler_type,
217 LP_JIT_SAMPLER_MAX_LOD);
218 LP_CHECK_MEMBER_OFFSET(struct lp_jit_sampler, lod_bias,
219 gallivm->target, sampler_type,
220 LP_JIT_SAMPLER_LOD_BIAS);
221 LP_CHECK_MEMBER_OFFSET(struct lp_jit_sampler, border_color,
222 gallivm->target, sampler_type,
223 LP_JIT_SAMPLER_BORDER_COLOR);
224 LP_CHECK_MEMBER_OFFSET(struct lp_jit_sampler, max_aniso,
225 gallivm->target, sampler_type,
226 LP_JIT_SAMPLER_MAX_ANISO);
227 LP_CHECK_STRUCT_SIZE(struct lp_jit_sampler,
228 gallivm->target, sampler_type);
229 return sampler_type;
230 }
231
232 static LLVMTypeRef
lp_build_create_jit_image_type(struct gallivm_state * gallivm)233 lp_build_create_jit_image_type(struct gallivm_state *gallivm)
234 {
235 LLVMContextRef lc = gallivm->context;
236 LLVMTypeRef image_type;
237 LLVMTypeRef elem_types[LP_JIT_IMAGE_NUM_FIELDS];
238 elem_types[LP_JIT_IMAGE_WIDTH] = LLVMInt32TypeInContext(lc);
239 elem_types[LP_JIT_IMAGE_HEIGHT] =
240 elem_types[LP_JIT_IMAGE_DEPTH] = LLVMInt16TypeInContext(lc);
241 elem_types[LP_JIT_IMAGE_NUM_SAMPLES] = LLVMInt8TypeInContext(lc);
242 elem_types[LP_JIT_IMAGE_BASE] = LLVMPointerType(LLVMInt8TypeInContext(lc), 0);
243 elem_types[LP_JIT_IMAGE_ROW_STRIDE] =
244 elem_types[LP_JIT_IMAGE_IMG_STRIDE] =
245 elem_types[LP_JIT_IMAGE_SAMPLE_STRIDE] = LLVMInt32TypeInContext(lc);
246
247 image_type = LLVMStructTypeInContext(lc, elem_types,
248 ARRAY_SIZE(elem_types), 0);
249 LP_CHECK_MEMBER_OFFSET(struct lp_jit_image, width,
250 gallivm->target, image_type,
251 LP_JIT_IMAGE_WIDTH);
252 LP_CHECK_MEMBER_OFFSET(struct lp_jit_image, height,
253 gallivm->target, image_type,
254 LP_JIT_IMAGE_HEIGHT);
255 LP_CHECK_MEMBER_OFFSET(struct lp_jit_image, depth,
256 gallivm->target, image_type,
257 LP_JIT_IMAGE_DEPTH);
258 LP_CHECK_MEMBER_OFFSET(struct lp_jit_image, base,
259 gallivm->target, image_type,
260 LP_JIT_IMAGE_BASE);
261 LP_CHECK_MEMBER_OFFSET(struct lp_jit_image, row_stride,
262 gallivm->target, image_type,
263 LP_JIT_IMAGE_ROW_STRIDE);
264 LP_CHECK_MEMBER_OFFSET(struct lp_jit_image, img_stride,
265 gallivm->target, image_type,
266 LP_JIT_IMAGE_IMG_STRIDE);
267 LP_CHECK_MEMBER_OFFSET(struct lp_jit_image, num_samples,
268 gallivm->target, image_type,
269 LP_JIT_IMAGE_NUM_SAMPLES);
270 LP_CHECK_MEMBER_OFFSET(struct lp_jit_image, sample_stride,
271 gallivm->target, image_type,
272 LP_JIT_IMAGE_SAMPLE_STRIDE);
273 return image_type;
274 }
275
276 LLVMTypeRef
lp_build_jit_resources_type(struct gallivm_state * gallivm)277 lp_build_jit_resources_type(struct gallivm_state *gallivm)
278 {
279 LLVMTypeRef elem_types[LP_JIT_RES_COUNT];
280 LLVMTypeRef resources_type;
281 LLVMTypeRef texture_type, sampler_type, image_type, buffer_type;
282
283 buffer_type = lp_build_create_jit_buffer_type(gallivm);
284 texture_type = lp_build_create_jit_texture_type(gallivm);
285 sampler_type = lp_build_create_jit_sampler_type(gallivm);
286 image_type = lp_build_create_jit_image_type(gallivm);
287 elem_types[LP_JIT_RES_CONSTANTS] = LLVMArrayType(buffer_type,
288 LP_MAX_TGSI_CONST_BUFFERS);
289 elem_types[LP_JIT_RES_SSBOS] =
290 LLVMArrayType(buffer_type, LP_MAX_TGSI_SHADER_BUFFERS);
291 elem_types[LP_JIT_RES_TEXTURES] = LLVMArrayType(texture_type,
292 PIPE_MAX_SHADER_SAMPLER_VIEWS);
293 elem_types[LP_JIT_RES_SAMPLERS] = LLVMArrayType(sampler_type,
294 PIPE_MAX_SAMPLERS);
295 elem_types[LP_JIT_RES_IMAGES] = LLVMArrayType(image_type,
296 PIPE_MAX_SHADER_IMAGES);
297 elem_types[LP_JIT_RES_ANISO_FILTER_TABLE] = LLVMPointerType(LLVMFloatTypeInContext(gallivm->context), 0);
298
299 resources_type = LLVMStructTypeInContext(gallivm->context, elem_types,
300 ARRAY_SIZE(elem_types), 0);
301
302 LP_CHECK_MEMBER_OFFSET(struct lp_jit_resources, constants,
303 gallivm->target, resources_type,
304 LP_JIT_RES_CONSTANTS);
305 LP_CHECK_MEMBER_OFFSET(struct lp_jit_resources, ssbos,
306 gallivm->target, resources_type,
307 LP_JIT_RES_SSBOS);
308 LP_CHECK_MEMBER_OFFSET(struct lp_jit_resources, textures,
309 gallivm->target, resources_type,
310 LP_JIT_RES_TEXTURES);
311 LP_CHECK_MEMBER_OFFSET(struct lp_jit_resources, samplers,
312 gallivm->target, resources_type,
313 LP_JIT_RES_SAMPLERS);
314 LP_CHECK_MEMBER_OFFSET(struct lp_jit_resources, images,
315 gallivm->target, resources_type,
316 LP_JIT_RES_IMAGES);
317 LP_CHECK_MEMBER_OFFSET(struct lp_jit_resources, aniso_filter_table,
318 gallivm->target, resources_type,
319 LP_JIT_RES_ANISO_FILTER_TABLE);
320
321 return resources_type;
322 }
323
324
325 /**
326 * Fetch the specified member of the lp_jit_texture structure.
327 * \param emit_load if TRUE, emit the LLVM load instruction to actually
328 * fetch the field's value. Otherwise, just emit the
329 * GEP code to address the field.
330 *
331 * @sa http://llvm.org/docs/GetElementPtr.html
332 */
333 static LLVMValueRef
lp_build_llvm_texture_member(struct gallivm_state * gallivm,LLVMTypeRef resources_type,LLVMValueRef resources_ptr,unsigned texture_unit,LLVMValueRef texture_unit_offset,unsigned member_index,const char * member_name,bool emit_load,LLVMTypeRef * out_type)334 lp_build_llvm_texture_member(struct gallivm_state *gallivm,
335 LLVMTypeRef resources_type,
336 LLVMValueRef resources_ptr,
337 unsigned texture_unit,
338 LLVMValueRef texture_unit_offset,
339 unsigned member_index,
340 const char *member_name,
341 bool emit_load,
342 LLVMTypeRef *out_type)
343 {
344 LLVMBuilderRef builder = gallivm->builder;
345
346 LLVMValueRef ptr;
347 if (gallivm->texture_descriptor) {
348 static_assert(offsetof(struct lp_descriptor, texture) == 0, "Invalid texture offset!");
349 LLVMValueRef texture_ptr = gallivm->texture_descriptor;
350
351 LLVMTypeRef texture_ptr_type = LLVMStructGetTypeAtIndex(resources_type, LP_JIT_RES_TEXTURES);
352 LLVMTypeRef texture_type = LLVMGetElementType(texture_ptr_type);
353 texture_ptr_type = LLVMPointerType(texture_type, 0);
354
355 texture_ptr = LLVMBuildIntToPtr(builder, texture_ptr, texture_ptr_type, "");
356
357 LLVMValueRef indices[2] = {
358 lp_build_const_int32(gallivm, 0),
359 lp_build_const_int32(gallivm, member_index),
360 };
361 ptr = LLVMBuildGEP2(builder, texture_type, texture_ptr, indices, ARRAY_SIZE(indices), "");
362 } else {
363 LLVMValueRef indices[4];
364
365 assert(texture_unit < PIPE_MAX_SHADER_SAMPLER_VIEWS);
366
367 /* resources[0] */
368 indices[0] = lp_build_const_int32(gallivm, 0);
369 /* resources[0].textures */
370 indices[1] = lp_build_const_int32(gallivm, LP_JIT_RES_TEXTURES);
371 /* resources[0].textures[unit] */
372 indices[2] = lp_build_const_int32(gallivm, texture_unit);
373 if (texture_unit_offset) {
374 indices[2] = LLVMBuildAdd(gallivm->builder, indices[2],
375 texture_unit_offset, "");
376 LLVMValueRef cond =
377 LLVMBuildICmp(gallivm->builder, LLVMIntULT,
378 indices[2],
379 lp_build_const_int32(gallivm,
380 PIPE_MAX_SHADER_SAMPLER_VIEWS), "");
381 indices[2] = LLVMBuildSelect(gallivm->builder, cond, indices[2],
382 lp_build_const_int32(gallivm,
383 texture_unit), "");
384 }
385 /* resources[0].textures[unit].member */
386 indices[3] = lp_build_const_int32(gallivm, member_index);
387
388 ptr = LLVMBuildGEP2(builder, resources_type, resources_ptr, indices, ARRAY_SIZE(indices), "");
389 }
390
391 LLVMValueRef res;
392 if (emit_load) {
393 LLVMTypeRef tex_type = LLVMStructGetTypeAtIndex(resources_type, LP_JIT_RES_TEXTURES);
394 LLVMTypeRef res_type = LLVMStructGetTypeAtIndex(LLVMGetElementType(tex_type), member_index);
395 res = LLVMBuildLoad2(builder, res_type, ptr, "");
396 } else
397 res = ptr;
398
399 if (out_type) {
400 LLVMTypeRef tex_type = LLVMStructGetTypeAtIndex(resources_type, LP_JIT_RES_TEXTURES);
401 LLVMTypeRef res_type = LLVMStructGetTypeAtIndex(LLVMGetElementType(tex_type), member_index);
402 *out_type = res_type;
403 }
404
405 lp_build_name(res, "resources.texture%u.%s", texture_unit, member_name);
406
407 return res;
408 }
409
410
411 /**
412 * Helper macro to instantiate the functions that generate the code to
413 * fetch the members of lp_jit_texture to fulfill the sampler code
414 * generator requests.
415 *
416 * This complexity is the price we have to pay to keep the texture
417 * sampler code generator a reusable module without dependencies to
418 * llvmpipe internals.
419 */
420 #define LP_BUILD_LLVM_TEXTURE_MEMBER(_name, _index, _emit_load) \
421 static LLVMValueRef \
422 lp_build_llvm_texture_##_name(struct gallivm_state *gallivm, \
423 LLVMTypeRef resources_type, \
424 LLVMValueRef resources_ptr, \
425 unsigned texture_unit, \
426 LLVMValueRef texture_unit_offset) \
427 { \
428 return lp_build_llvm_texture_member(gallivm, resources_type, resources_ptr, \
429 texture_unit, texture_unit_offset, \
430 _index, #_name, _emit_load, NULL ); \
431 }
432
433 #define LP_BUILD_LLVM_TEXTURE_MEMBER_OUTTYPE(_name, _index, _emit_load) \
434 static LLVMValueRef \
435 lp_build_llvm_texture_##_name(struct gallivm_state *gallivm, \
436 LLVMTypeRef resources_type, \
437 LLVMValueRef resources_ptr, \
438 unsigned texture_unit, \
439 LLVMValueRef texture_unit_offset, \
440 LLVMTypeRef *out_type) \
441 { \
442 return lp_build_llvm_texture_member(gallivm, resources_type, resources_ptr, \
443 texture_unit, texture_unit_offset, \
444 _index, #_name, _emit_load, out_type ); \
445 }
446
LP_BUILD_LLVM_TEXTURE_MEMBER(width,LP_JIT_TEXTURE_WIDTH,true)447 LP_BUILD_LLVM_TEXTURE_MEMBER(width, LP_JIT_TEXTURE_WIDTH, true)
448 LP_BUILD_LLVM_TEXTURE_MEMBER(height, LP_JIT_TEXTURE_HEIGHT, true)
449 LP_BUILD_LLVM_TEXTURE_MEMBER(depth, LP_JIT_TEXTURE_DEPTH, true)
450 LP_BUILD_LLVM_TEXTURE_MEMBER(first_level, LP_JIT_TEXTURE_FIRST_LEVEL, true)
451 LP_BUILD_LLVM_TEXTURE_MEMBER(last_level, LP_JIT_TEXTURE_LAST_LEVEL, true)
452 LP_BUILD_LLVM_TEXTURE_MEMBER(base_ptr, LP_JIT_TEXTURE_BASE, true)
453 LP_BUILD_LLVM_TEXTURE_MEMBER_OUTTYPE(row_stride, LP_JIT_TEXTURE_ROW_STRIDE, false)
454 LP_BUILD_LLVM_TEXTURE_MEMBER_OUTTYPE(img_stride, LP_JIT_TEXTURE_IMG_STRIDE, false)
455 LP_BUILD_LLVM_TEXTURE_MEMBER_OUTTYPE(mip_offsets, LP_JIT_TEXTURE_MIP_OFFSETS, false)
456
457 /**
458 * Fetch the specified member of the lp_jit_sampler structure.
459 * \param emit_load if TRUE, emit the LLVM load instruction to actually
460 * fetch the field's value. Otherwise, just emit the
461 * GEP code to address the field.
462 *
463 * @sa http://llvm.org/docs/GetElementPtr.html
464 */
465 static LLVMValueRef
466 lp_build_llvm_sampler_member(struct gallivm_state *gallivm,
467 LLVMTypeRef resources_type,
468 LLVMValueRef resources_ptr,
469 unsigned sampler_unit,
470 unsigned member_index,
471 const char *member_name,
472 bool emit_load)
473 {
474 LLVMBuilderRef builder = gallivm->builder;
475
476 LLVMValueRef ptr;
477 if (gallivm->sampler_descriptor) {
478 LLVMValueRef sampler_offset = lp_build_const_int64(gallivm, offsetof(struct lp_descriptor, sampler));
479 LLVMValueRef sampler_ptr = LLVMBuildAdd(builder, gallivm->sampler_descriptor, sampler_offset, "");
480
481 LLVMTypeRef sampler_ptr_type = LLVMStructGetTypeAtIndex(resources_type, LP_JIT_RES_SAMPLERS);
482 LLVMTypeRef sampler_type = LLVMGetElementType(sampler_ptr_type);
483 sampler_ptr_type = LLVMPointerType(sampler_type, 0);
484
485 sampler_ptr = LLVMBuildIntToPtr(builder, sampler_ptr, sampler_ptr_type, "");
486
487 LLVMValueRef indices[2] = {
488 lp_build_const_int32(gallivm, 0),
489 lp_build_const_int32(gallivm, member_index),
490 };
491 ptr = LLVMBuildGEP2(builder, sampler_type, sampler_ptr, indices, ARRAY_SIZE(indices), "");
492 } else {
493 LLVMValueRef indices[4];
494
495 assert(sampler_unit < PIPE_MAX_SAMPLERS);
496
497 /* resources[0] */
498 indices[0] = lp_build_const_int32(gallivm, 0);
499 /* resources[0].samplers */
500 indices[1] = lp_build_const_int32(gallivm, LP_JIT_RES_SAMPLERS);
501 /* resources[0].samplers[unit] */
502 indices[2] = lp_build_const_int32(gallivm, sampler_unit);
503 /* resources[0].samplers[unit].member */
504 indices[3] = lp_build_const_int32(gallivm, member_index);
505
506 ptr = LLVMBuildGEP2(builder, resources_type, resources_ptr, indices, ARRAY_SIZE(indices), "");
507 }
508
509 LLVMValueRef res;
510 if (emit_load) {
511 LLVMTypeRef samp_type = LLVMStructGetTypeAtIndex(resources_type, LP_JIT_RES_SAMPLERS);
512 LLVMTypeRef res_type = LLVMStructGetTypeAtIndex(LLVMGetElementType(samp_type), member_index);
513 res = LLVMBuildLoad2(builder, res_type, ptr, "");
514 } else
515 res = ptr;
516
517 lp_build_name(res, "resources.sampler%u.%s", sampler_unit, member_name);
518
519 return res;
520 }
521
522
523 #define LP_BUILD_LLVM_SAMPLER_MEMBER(_name, _index, _emit_load) \
524 static LLVMValueRef \
525 lp_build_llvm_sampler_##_name( struct gallivm_state *gallivm, \
526 LLVMTypeRef resources_type, \
527 LLVMValueRef resources_ptr, \
528 unsigned sampler_unit) \
529 { \
530 return lp_build_llvm_sampler_member(gallivm, resources_type, resources_ptr, \
531 sampler_unit, _index, #_name, _emit_load ); \
532 }
533
534
LP_BUILD_LLVM_SAMPLER_MEMBER(min_lod,LP_JIT_SAMPLER_MIN_LOD,true)535 LP_BUILD_LLVM_SAMPLER_MEMBER(min_lod, LP_JIT_SAMPLER_MIN_LOD, true)
536 LP_BUILD_LLVM_SAMPLER_MEMBER(max_lod, LP_JIT_SAMPLER_MAX_LOD, true)
537 LP_BUILD_LLVM_SAMPLER_MEMBER(lod_bias, LP_JIT_SAMPLER_LOD_BIAS, true)
538 LP_BUILD_LLVM_SAMPLER_MEMBER(border_color, LP_JIT_SAMPLER_BORDER_COLOR, false)
539 LP_BUILD_LLVM_SAMPLER_MEMBER(max_aniso, LP_JIT_SAMPLER_MAX_ANISO, true)
540
541 /**
542 * Fetch the specified member of the lp_jit_image structure.
543 * \param emit_load if TRUE, emit the LLVM load instruction to actually
544 * fetch the field's value. Otherwise, just emit the
545 * GEP code to address the field.
546 *
547 * @sa http://llvm.org/docs/GetElementPtr.html
548 */
549 static LLVMValueRef
550 lp_build_llvm_image_member(struct gallivm_state *gallivm,
551 LLVMTypeRef resources_type,
552 LLVMValueRef resources_ptr,
553 unsigned image_unit,
554 LLVMValueRef image_unit_offset,
555 unsigned member_index,
556 const char *member_name,
557 bool emit_load)
558 {
559 LLVMBuilderRef builder = gallivm->builder;
560
561 LLVMValueRef ptr;
562 if (gallivm->texture_descriptor) {
563 LLVMValueRef image_offset = lp_build_const_int64(gallivm, offsetof(struct lp_descriptor, image));
564 LLVMValueRef image_ptr = LLVMBuildAdd(builder, gallivm->texture_descriptor, image_offset, "");
565
566 LLVMTypeRef image_ptr_type = LLVMStructGetTypeAtIndex(resources_type, LP_JIT_RES_IMAGES);
567 LLVMTypeRef image_type = LLVMGetElementType(image_ptr_type);
568 image_ptr_type = LLVMPointerType(image_type, 0);
569
570 image_ptr = LLVMBuildIntToPtr(builder, image_ptr, image_ptr_type, "");
571
572 LLVMValueRef indices[2] = {
573 lp_build_const_int32(gallivm, 0),
574 lp_build_const_int32(gallivm, member_index),
575 };
576 ptr = LLVMBuildGEP2(builder, image_type, image_ptr, indices, ARRAY_SIZE(indices), "");
577 } else {
578 LLVMValueRef indices[4];
579
580 assert(image_unit < PIPE_MAX_SHADER_IMAGES);
581
582 /* resources[0] */
583 indices[0] = lp_build_const_int32(gallivm, 0);
584 /* resources[0].images */
585 indices[1] = lp_build_const_int32(gallivm, LP_JIT_RES_IMAGES);
586 /* resources[0].images[unit] */
587 indices[2] = lp_build_const_int32(gallivm, image_unit);
588 if (image_unit_offset) {
589 indices[2] = LLVMBuildAdd(gallivm->builder, indices[2], image_unit_offset, "");
590 LLVMValueRef cond = LLVMBuildICmp(gallivm->builder, LLVMIntULT, indices[2], lp_build_const_int32(gallivm, PIPE_MAX_SHADER_IMAGES), "");
591 indices[2] = LLVMBuildSelect(gallivm->builder, cond, indices[2], lp_build_const_int32(gallivm, image_unit), "");
592 }
593 /* resources[0].images[unit].member */
594 indices[3] = lp_build_const_int32(gallivm, member_index);
595
596 ptr = LLVMBuildGEP2(builder, resources_type, resources_ptr, indices, ARRAY_SIZE(indices), "");
597 }
598
599 LLVMValueRef res;
600 if (emit_load) {
601 LLVMTypeRef img_type = LLVMStructGetTypeAtIndex(resources_type, LP_JIT_RES_IMAGES);
602 LLVMTypeRef res_type = LLVMStructGetTypeAtIndex(LLVMGetElementType(img_type), member_index);
603 res = LLVMBuildLoad2(builder, res_type, ptr, "");
604 } else
605 res = ptr;
606
607 lp_build_name(res, "resources.image%u.%s", image_unit, member_name);
608
609 return res;
610 }
611
612 /**
613 * Helper macro to instantiate the functions that generate the code to
614 * fetch the members of lp_jit_image to fulfill the sampler code
615 * generator requests.
616 *
617 * This complexity is the price we have to pay to keep the image
618 * sampler code generator a reusable module without dependencies to
619 * llvmpipe internals.
620 */
621 #define LP_BUILD_LLVM_IMAGE_MEMBER(_name, _index, _emit_load) \
622 static LLVMValueRef \
623 lp_build_llvm_image_##_name( struct gallivm_state *gallivm, \
624 LLVMTypeRef resources_type, \
625 LLVMValueRef resources_ptr, \
626 unsigned image_unit, LLVMValueRef image_unit_offset) \
627 { \
628 return lp_build_llvm_image_member(gallivm, resources_type, resources_ptr, \
629 image_unit, image_unit_offset, \
630 _index, #_name, _emit_load ); \
631 }
632
633 #define LP_BUILD_LLVM_IMAGE_MEMBER_OUTTYPE(_name, _index, _emit_load) \
634 static LLVMValueRef \
635 lp_build_llvm_image_##_name( struct gallivm_state *gallivm, \
636 LLVMTypeRef resources_type, \
637 LLVMValueRef resources_ptr, \
638 unsigned image_unit, LLVMValueRef image_unit_offset, \
639 LLVMTypeRef *out_type) \
640 { \
641 assert(!out_type); \
642 return lp_build_llvm_image_member(gallivm, resources_type, resources_ptr, \
643 image_unit, image_unit_offset, \
644 _index, #_name, _emit_load ); \
645 }
646
LP_BUILD_LLVM_IMAGE_MEMBER(width,LP_JIT_IMAGE_WIDTH,true)647 LP_BUILD_LLVM_IMAGE_MEMBER(width, LP_JIT_IMAGE_WIDTH, true)
648 LP_BUILD_LLVM_IMAGE_MEMBER(height, LP_JIT_IMAGE_HEIGHT, true)
649 LP_BUILD_LLVM_IMAGE_MEMBER(depth, LP_JIT_IMAGE_DEPTH, true)
650 LP_BUILD_LLVM_IMAGE_MEMBER(base_ptr, LP_JIT_IMAGE_BASE, true)
651 LP_BUILD_LLVM_IMAGE_MEMBER_OUTTYPE(row_stride, LP_JIT_IMAGE_ROW_STRIDE, true)
652 LP_BUILD_LLVM_IMAGE_MEMBER_OUTTYPE(img_stride, LP_JIT_IMAGE_IMG_STRIDE, true)
653 LP_BUILD_LLVM_IMAGE_MEMBER(num_samples, LP_JIT_IMAGE_NUM_SAMPLES, true)
654 LP_BUILD_LLVM_IMAGE_MEMBER(sample_stride, LP_JIT_IMAGE_SAMPLE_STRIDE, true)
655
656 void
657 lp_build_jit_fill_sampler_dynamic_state(struct lp_sampler_dynamic_state *state)
658 {
659 state->width = lp_build_llvm_texture_width;
660 state->height = lp_build_llvm_texture_height;
661 state->depth = lp_build_llvm_texture_depth;
662 state->first_level = lp_build_llvm_texture_first_level;
663 state->last_level = lp_build_llvm_texture_last_level;
664 state->base_ptr = lp_build_llvm_texture_base_ptr;
665 state->row_stride = lp_build_llvm_texture_row_stride;
666 state->img_stride = lp_build_llvm_texture_img_stride;
667 state->mip_offsets = lp_build_llvm_texture_mip_offsets;
668
669 state->min_lod = lp_build_llvm_sampler_min_lod;
670 state->max_lod = lp_build_llvm_sampler_max_lod;
671 state->lod_bias = lp_build_llvm_sampler_lod_bias;
672 state->border_color = lp_build_llvm_sampler_border_color;
673 state->max_aniso = lp_build_llvm_sampler_max_aniso;
674 }
675
676 void
lp_build_jit_fill_image_dynamic_state(struct lp_sampler_dynamic_state * state)677 lp_build_jit_fill_image_dynamic_state(struct lp_sampler_dynamic_state *state)
678 {
679 state->width = lp_build_llvm_image_width;
680 state->height = lp_build_llvm_image_height;
681
682 state->depth = lp_build_llvm_image_depth;
683 state->base_ptr = lp_build_llvm_image_base_ptr;
684 state->row_stride = lp_build_llvm_image_row_stride;
685 state->img_stride = lp_build_llvm_image_img_stride;
686 state->last_level = lp_build_llvm_image_num_samples;
687 state->sample_stride = lp_build_llvm_image_sample_stride;
688 }
689
690 /**
691 * Create LLVM type for struct vertex_header;
692 */
693 LLVMTypeRef
lp_build_create_jit_vertex_header_type(struct gallivm_state * gallivm,int data_elems)694 lp_build_create_jit_vertex_header_type(struct gallivm_state *gallivm, int data_elems)
695 {
696 LLVMTargetDataRef target = gallivm->target;
697 LLVMTypeRef elem_types[3];
698 LLVMTypeRef vertex_header;
699 char struct_name[24];
700
701 snprintf(struct_name, 23, "vertex_header%d", data_elems);
702
703 elem_types[LP_JIT_VERTEX_HEADER_VERTEX_ID] = LLVMIntTypeInContext(gallivm->context, 32);
704 elem_types[LP_JIT_VERTEX_HEADER_CLIP_POS] = LLVMArrayType(LLVMFloatTypeInContext(gallivm->context), 4);
705 elem_types[LP_JIT_VERTEX_HEADER_DATA] = LLVMArrayType(elem_types[1], data_elems);
706
707 vertex_header = LLVMStructTypeInContext(gallivm->context, elem_types,
708 ARRAY_SIZE(elem_types), 0);
709
710 /* these are bit-fields and we can't take address of them
711 LP_CHECK_MEMBER_OFFSET(struct vertex_header, clipmask,
712 target, vertex_header,
713 LP_JIT_VERTEX_HEADER_CLIPMASK);
714 LP_CHECK_MEMBER_OFFSET(struct vertex_header, edgeflag,
715 target, vertex_header,
716 LP_JIT_VERTEX_HEADER_EDGEFLAG);
717 LP_CHECK_MEMBER_OFFSET(struct vertex_header, pad,
718 target, vertex_header,
719 LP_JIT_VERTEX_HEADER_PAD);
720 LP_CHECK_MEMBER_OFFSET(struct vertex_header, vertex_id,
721 target, vertex_header,
722 LP_JIT_VERTEX_HEADER_VERTEX_ID);
723 */
724 (void) target; /* silence unused var warning for non-debug build */
725 LP_CHECK_MEMBER_OFFSET(struct vertex_header, clip_pos,
726 target, vertex_header,
727 LP_JIT_VERTEX_HEADER_CLIP_POS);
728 LP_CHECK_MEMBER_OFFSET(struct vertex_header, data,
729 target, vertex_header,
730 LP_JIT_VERTEX_HEADER_DATA);
731
732 assert(LLVMABISizeOfType(target, vertex_header) ==
733 offsetof(struct vertex_header, data[data_elems]));
734
735 return vertex_header;
736 }
737
738 LLVMTypeRef
lp_build_sample_function_type(struct gallivm_state * gallivm,uint32_t sample_key)739 lp_build_sample_function_type(struct gallivm_state *gallivm, uint32_t sample_key)
740 {
741 struct lp_type type;
742 memset(&type, 0, sizeof type);
743 type.floating = true; /* floating point values */
744 type.sign = true; /* values are signed */
745 type.norm = false; /* values are not limited to [0,1] or [-1,1] */
746 type.width = 32; /* 32-bit float */
747 type.length = MIN2(lp_native_vector_width / 32, 16); /* n*4 elements per vector */
748
749 enum lp_sampler_op_type op_type = (sample_key & LP_SAMPLER_OP_TYPE_MASK) >> LP_SAMPLER_OP_TYPE_SHIFT;
750 enum lp_sampler_lod_control lod_control = (sample_key & LP_SAMPLER_LOD_CONTROL_MASK) >> LP_SAMPLER_LOD_CONTROL_SHIFT;
751
752 LLVMTypeRef arg_types[LP_MAX_TEX_FUNC_ARGS];
753 LLVMTypeRef ret_type;
754 LLVMTypeRef val_type[4];
755 uint32_t num_params = 0;
756
757 LLVMTypeRef coord_type;
758 if (op_type == LP_SAMPLER_OP_FETCH)
759 coord_type = lp_build_vec_type(gallivm, lp_int_type(type));
760 else
761 coord_type = lp_build_vec_type(gallivm, type);
762
763 arg_types[num_params++] = LLVMInt64TypeInContext(gallivm->context);
764 arg_types[num_params++] = LLVMInt64TypeInContext(gallivm->context);
765
766 arg_types[num_params++] = LLVMPointerType(LLVMFloatTypeInContext(gallivm->context), 0);
767
768 for (unsigned i = 0; i < 4; i++)
769 arg_types[num_params++] = coord_type;
770
771 if (sample_key & LP_SAMPLER_SHADOW)
772 arg_types[num_params++] = lp_build_vec_type(gallivm, type);
773
774 if (sample_key & LP_SAMPLER_FETCH_MS)
775 arg_types[num_params++] = lp_build_vec_type(gallivm, lp_uint_type(type));
776
777 if (sample_key & LP_SAMPLER_OFFSETS)
778 for (uint32_t i = 0; i < 3; i++)
779 arg_types[num_params++] = lp_build_int_vec_type(gallivm, type);
780
781 if (lod_control == LP_SAMPLER_LOD_BIAS || lod_control == LP_SAMPLER_LOD_EXPLICIT)
782 arg_types[num_params++] = coord_type;
783
784 val_type[0] = val_type[1] = val_type[2] = val_type[3] = lp_build_vec_type(gallivm, type);
785 ret_type = LLVMStructTypeInContext(gallivm->context, val_type, 4, 0);
786 return LLVMFunctionType(ret_type, arg_types, num_params, false);
787 }
788
789 LLVMTypeRef
lp_build_size_function_type(struct gallivm_state * gallivm,const struct lp_sampler_size_query_params * params)790 lp_build_size_function_type(struct gallivm_state *gallivm,
791 const struct lp_sampler_size_query_params *params)
792 {
793 struct lp_type type;
794 memset(&type, 0, sizeof type);
795 type.floating = true; /* floating point values */
796 type.sign = true; /* values are signed */
797 type.norm = false; /* values are not limited to [0,1] or [-1,1] */
798 type.width = 32; /* 32-bit float */
799 type.length = MIN2(lp_native_vector_width / 32, 16); /* n*4 elements per vector */
800
801 LLVMTypeRef arg_types[LP_MAX_TEX_FUNC_ARGS];
802 LLVMTypeRef ret_type;
803 LLVMTypeRef val_type[4];
804 uint32_t num_params = 0;
805
806 arg_types[num_params++] = LLVMInt64TypeInContext(gallivm->context);
807
808 if (!params->samples_only)
809 arg_types[num_params++] = lp_build_int_vec_type(gallivm, type);
810
811 val_type[0] = val_type[1] = val_type[2] = val_type[3] = lp_build_int_vec_type(gallivm, type);
812 ret_type = LLVMStructTypeInContext(gallivm->context, val_type, 4, 0);
813 return LLVMFunctionType(ret_type, arg_types, num_params, false);
814 }
815
816 LLVMTypeRef
lp_build_image_function_type(struct gallivm_state * gallivm,const struct lp_img_params * params,bool ms)817 lp_build_image_function_type(struct gallivm_state *gallivm,
818 const struct lp_img_params *params, bool ms)
819 {
820 struct lp_type type;
821 memset(&type, 0, sizeof type);
822 type.floating = true; /* floating point values */
823 type.sign = true; /* values are signed */
824 type.norm = false; /* values are not limited to [0,1] or [-1,1] */
825 type.width = 32; /* 32-bit float */
826 type.length = MIN2(lp_native_vector_width / 32, 16); /* n*4 elements per vector */
827
828 LLVMTypeRef arg_types[LP_MAX_TEX_FUNC_ARGS];
829 LLVMTypeRef ret_type;
830 uint32_t num_params = 0;
831
832 arg_types[num_params++] = LLVMInt64TypeInContext(gallivm->context);
833
834 if (params->img_op != LP_IMG_LOAD)
835 arg_types[num_params++] = lp_build_int_vec_type(gallivm, type);
836
837 for (uint32_t i = 0; i < 3; i++)
838 arg_types[num_params++] = lp_build_vec_type(gallivm, lp_uint_type(type));
839
840 if (ms)
841 arg_types[num_params++] = lp_build_vec_type(gallivm, lp_uint_type(type));
842
843 uint32_t num_inputs = params->img_op != LP_IMG_LOAD ? 4 : 0;
844 if (params->img_op == LP_IMG_ATOMIC_CAS)
845 num_inputs = 8;
846
847 const struct util_format_description *desc = util_format_description(params->format);
848 LLVMTypeRef component_type = lp_build_vec_type(gallivm, lp_build_texel_type(type, desc));
849
850 for (uint32_t i = 0; i < num_inputs; i++)
851 arg_types[num_params++] = component_type;
852
853 if (params->img_op != LP_IMG_STORE) {
854 LLVMTypeRef val_type[4];
855 val_type[0] = val_type[1] = val_type[2] = val_type[3] = component_type;
856 ret_type = LLVMStructTypeInContext(gallivm->context, val_type, 4, 0);
857 } else {
858 ret_type = LLVMVoidTypeInContext(gallivm->context);
859 }
860
861 return LLVMFunctionType(ret_type, arg_types, num_params, false);
862 }
863