1 /**************************************************************************
2 *
3 * Copyright 2010 VMware, Inc.
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
21 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27
28 #include "draw_llvm.h"
29
30 #include "draw_context.h"
31 #include "draw_vs.h"
32
33 #include "gallivm/lp_bld_arit.h"
34 #include "gallivm/lp_bld_logic.h"
35 #include "gallivm/lp_bld_const.h"
36 #include "gallivm/lp_bld_swizzle.h"
37 #include "gallivm/lp_bld_struct.h"
38 #include "gallivm/lp_bld_type.h"
39 #include "gallivm/lp_bld_flow.h"
40 #include "gallivm/lp_bld_debug.h"
41 #include "gallivm/lp_bld_tgsi.h"
42 #include "gallivm/lp_bld_printf.h"
43 #include "gallivm/lp_bld_intr.h"
44 #include "gallivm/lp_bld_init.h"
45 #include "gallivm/lp_bld_type.h"
46 #include "gallivm/lp_bld_pack.h"
47 #include "gallivm/lp_bld_format.h"
48
49 #include "tgsi/tgsi_exec.h"
50 #include "tgsi/tgsi_dump.h"
51
52 #include "util/u_math.h"
53 #include "util/u_pointer.h"
54 #include "util/u_string.h"
55 #include "util/u_simple_list.h"
56
57
58 #define DEBUG_STORE 0
59
60
61 static void
62 draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *var,
63 boolean elts);
64
65
66 /**
67 * Create LLVM type for struct draw_jit_texture
68 */
69 static LLVMTypeRef
create_jit_texture_type(struct gallivm_state * gallivm,const char * struct_name)70 create_jit_texture_type(struct gallivm_state *gallivm, const char *struct_name)
71 {
72 LLVMTargetDataRef target = gallivm->target;
73 LLVMTypeRef texture_type;
74 LLVMTypeRef elem_types[DRAW_JIT_TEXTURE_NUM_FIELDS];
75 LLVMTypeRef int32_type = LLVMInt32TypeInContext(gallivm->context);
76
77 elem_types[DRAW_JIT_TEXTURE_WIDTH] =
78 elem_types[DRAW_JIT_TEXTURE_HEIGHT] =
79 elem_types[DRAW_JIT_TEXTURE_DEPTH] =
80 elem_types[DRAW_JIT_TEXTURE_FIRST_LEVEL] =
81 elem_types[DRAW_JIT_TEXTURE_LAST_LEVEL] = int32_type;
82 elem_types[DRAW_JIT_TEXTURE_ROW_STRIDE] =
83 elem_types[DRAW_JIT_TEXTURE_IMG_STRIDE] =
84 LLVMArrayType(int32_type, PIPE_MAX_TEXTURE_LEVELS);
85 elem_types[DRAW_JIT_TEXTURE_DATA] =
86 LLVMArrayType(LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0),
87 PIPE_MAX_TEXTURE_LEVELS);
88 elem_types[DRAW_JIT_TEXTURE_MIN_LOD] =
89 elem_types[DRAW_JIT_TEXTURE_MAX_LOD] =
90 elem_types[DRAW_JIT_TEXTURE_LOD_BIAS] = LLVMFloatTypeInContext(gallivm->context);
91 elem_types[DRAW_JIT_TEXTURE_BORDER_COLOR] =
92 LLVMArrayType(LLVMFloatTypeInContext(gallivm->context), 4);
93
94 texture_type = LLVMStructTypeInContext(gallivm->context, elem_types,
95 Elements(elem_types), 0);
96
97 #if HAVE_LLVM < 0x0300
98 LLVMAddTypeName(gallivm->module, struct_name, texture_type);
99
100 /* Make sure the target's struct layout cache doesn't return
101 * stale/invalid data.
102 */
103 LLVMInvalidateStructLayout(gallivm->target, texture_type);
104 #endif
105
106 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, width,
107 target, texture_type,
108 DRAW_JIT_TEXTURE_WIDTH);
109 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, height,
110 target, texture_type,
111 DRAW_JIT_TEXTURE_HEIGHT);
112 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, depth,
113 target, texture_type,
114 DRAW_JIT_TEXTURE_DEPTH);
115 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, first_level,
116 target, texture_type,
117 DRAW_JIT_TEXTURE_FIRST_LEVEL);
118 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, last_level,
119 target, texture_type,
120 DRAW_JIT_TEXTURE_LAST_LEVEL);
121 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, row_stride,
122 target, texture_type,
123 DRAW_JIT_TEXTURE_ROW_STRIDE);
124 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, img_stride,
125 target, texture_type,
126 DRAW_JIT_TEXTURE_IMG_STRIDE);
127 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, data,
128 target, texture_type,
129 DRAW_JIT_TEXTURE_DATA);
130 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, min_lod,
131 target, texture_type,
132 DRAW_JIT_TEXTURE_MIN_LOD);
133 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, max_lod,
134 target, texture_type,
135 DRAW_JIT_TEXTURE_MAX_LOD);
136 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, lod_bias,
137 target, texture_type,
138 DRAW_JIT_TEXTURE_LOD_BIAS);
139 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, border_color,
140 target, texture_type,
141 DRAW_JIT_TEXTURE_BORDER_COLOR);
142
143 LP_CHECK_STRUCT_SIZE(struct draw_jit_texture, target, texture_type);
144
145 return texture_type;
146 }
147
148
149 /**
150 * Create LLVM type for struct draw_jit_texture
151 */
152 static LLVMTypeRef
create_jit_context_type(struct gallivm_state * gallivm,LLVMTypeRef texture_type,const char * struct_name)153 create_jit_context_type(struct gallivm_state *gallivm,
154 LLVMTypeRef texture_type, const char *struct_name)
155 {
156 LLVMTargetDataRef target = gallivm->target;
157 LLVMTypeRef float_type = LLVMFloatTypeInContext(gallivm->context);
158 LLVMTypeRef elem_types[5];
159 LLVMTypeRef context_type;
160
161 elem_types[0] = LLVMPointerType(float_type, 0); /* vs_constants */
162 elem_types[1] = LLVMPointerType(float_type, 0); /* gs_constants */
163 elem_types[2] = LLVMPointerType(LLVMArrayType(LLVMArrayType(float_type, 4),
164 DRAW_TOTAL_CLIP_PLANES), 0);
165 elem_types[3] = LLVMPointerType(float_type, 0); /* viewport */
166 elem_types[4] = LLVMArrayType(texture_type,
167 PIPE_MAX_SAMPLERS); /* textures */
168 context_type = LLVMStructTypeInContext(gallivm->context, elem_types,
169 Elements(elem_types), 0);
170 #if HAVE_LLVM < 0x0300
171 LLVMAddTypeName(gallivm->module, struct_name, context_type);
172
173 LLVMInvalidateStructLayout(gallivm->target, context_type);
174 #endif
175
176 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, vs_constants,
177 target, context_type, 0);
178 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, gs_constants,
179 target, context_type, 1);
180 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, planes,
181 target, context_type, 2);
182 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, textures,
183 target, context_type,
184 DRAW_JIT_CTX_TEXTURES);
185 LP_CHECK_STRUCT_SIZE(struct draw_jit_context,
186 target, context_type);
187
188 return context_type;
189 }
190
191
192 /**
193 * Create LLVM type for struct pipe_vertex_buffer
194 */
195 static LLVMTypeRef
create_jit_vertex_buffer_type(struct gallivm_state * gallivm,const char * struct_name)196 create_jit_vertex_buffer_type(struct gallivm_state *gallivm, const char *struct_name)
197 {
198 LLVMTargetDataRef target = gallivm->target;
199 LLVMTypeRef elem_types[4];
200 LLVMTypeRef vb_type;
201
202 elem_types[0] =
203 elem_types[1] = LLVMInt32TypeInContext(gallivm->context);
204 elem_types[2] =
205 elem_types[3] = LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0); /* vs_constants */
206
207 vb_type = LLVMStructTypeInContext(gallivm->context, elem_types,
208 Elements(elem_types), 0);
209 #if HAVE_LLVM < 0x0300
210 LLVMAddTypeName(gallivm->module, struct_name, vb_type);
211
212 LLVMInvalidateStructLayout(gallivm->target, vb_type);
213 #endif
214
215 LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, stride,
216 target, vb_type, 0);
217 LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, buffer_offset,
218 target, vb_type, 1);
219
220 LP_CHECK_STRUCT_SIZE(struct pipe_vertex_buffer, target, vb_type);
221
222 return vb_type;
223 }
224
225
226 /**
227 * Create LLVM type for struct vertex_header;
228 */
229 static LLVMTypeRef
create_jit_vertex_header(struct gallivm_state * gallivm,int data_elems)230 create_jit_vertex_header(struct gallivm_state *gallivm, int data_elems)
231 {
232 LLVMTargetDataRef target = gallivm->target;
233 LLVMTypeRef elem_types[4];
234 LLVMTypeRef vertex_header;
235 char struct_name[24];
236
237 util_snprintf(struct_name, 23, "vertex_header%d", data_elems);
238
239 elem_types[DRAW_JIT_VERTEX_VERTEX_ID] = LLVMIntTypeInContext(gallivm->context, 32);
240 elem_types[DRAW_JIT_VERTEX_CLIP] = LLVMArrayType(LLVMFloatTypeInContext(gallivm->context), 4);
241 elem_types[DRAW_JIT_VERTEX_PRE_CLIP_POS] = LLVMArrayType(LLVMFloatTypeInContext(gallivm->context), 4);
242 elem_types[DRAW_JIT_VERTEX_DATA] = LLVMArrayType(elem_types[1], data_elems);
243
244 vertex_header = LLVMStructTypeInContext(gallivm->context, elem_types,
245 Elements(elem_types), 0);
246 #if HAVE_LLVM < 0x0300
247 LLVMAddTypeName(gallivm->module, struct_name, vertex_header);
248
249 LLVMInvalidateStructLayout(gallivm->target, vertex_header);
250 #endif
251
252 /* these are bit-fields and we can't take address of them
253 LP_CHECK_MEMBER_OFFSET(struct vertex_header, clipmask,
254 target, vertex_header,
255 DRAW_JIT_VERTEX_CLIPMASK);
256 LP_CHECK_MEMBER_OFFSET(struct vertex_header, edgeflag,
257 target, vertex_header,
258 DRAW_JIT_VERTEX_EDGEFLAG);
259 LP_CHECK_MEMBER_OFFSET(struct vertex_header, pad,
260 target, vertex_header,
261 DRAW_JIT_VERTEX_PAD);
262 LP_CHECK_MEMBER_OFFSET(struct vertex_header, vertex_id,
263 target, vertex_header,
264 DRAW_JIT_VERTEX_VERTEX_ID);
265 */
266 LP_CHECK_MEMBER_OFFSET(struct vertex_header, clip,
267 target, vertex_header,
268 DRAW_JIT_VERTEX_CLIP);
269 LP_CHECK_MEMBER_OFFSET(struct vertex_header, pre_clip_pos,
270 target, vertex_header,
271 DRAW_JIT_VERTEX_PRE_CLIP_POS);
272 LP_CHECK_MEMBER_OFFSET(struct vertex_header, data,
273 target, vertex_header,
274 DRAW_JIT_VERTEX_DATA);
275
276 assert(LLVMABISizeOfType(target, vertex_header) ==
277 offsetof(struct vertex_header, data[data_elems]));
278
279 return vertex_header;
280 }
281
282
283 /**
284 * Create LLVM types for various structures.
285 */
286 static void
create_jit_types(struct draw_llvm_variant * variant)287 create_jit_types(struct draw_llvm_variant *variant)
288 {
289 struct gallivm_state *gallivm = variant->gallivm;
290 LLVMTypeRef texture_type, context_type, buffer_type, vb_type;
291
292 texture_type = create_jit_texture_type(gallivm, "texture");
293
294 context_type = create_jit_context_type(gallivm, texture_type, "draw_jit_context");
295 variant->context_ptr_type = LLVMPointerType(context_type, 0);
296
297 buffer_type = LLVMPointerType(LLVMIntTypeInContext(gallivm->context, 8), 0);
298 variant->buffer_ptr_type = LLVMPointerType(buffer_type, 0);
299
300 vb_type = create_jit_vertex_buffer_type(gallivm, "pipe_vertex_buffer");
301 variant->vb_ptr_type = LLVMPointerType(vb_type, 0);
302 }
303
304
305 static LLVMTypeRef
get_context_ptr_type(struct draw_llvm_variant * variant)306 get_context_ptr_type(struct draw_llvm_variant *variant)
307 {
308 if (!variant->context_ptr_type)
309 create_jit_types(variant);
310 return variant->context_ptr_type;
311 }
312
313
314 static LLVMTypeRef
get_buffer_ptr_type(struct draw_llvm_variant * variant)315 get_buffer_ptr_type(struct draw_llvm_variant *variant)
316 {
317 if (!variant->buffer_ptr_type)
318 create_jit_types(variant);
319 return variant->buffer_ptr_type;
320 }
321
322
323 static LLVMTypeRef
get_vb_ptr_type(struct draw_llvm_variant * variant)324 get_vb_ptr_type(struct draw_llvm_variant *variant)
325 {
326 if (!variant->vb_ptr_type)
327 create_jit_types(variant);
328 return variant->vb_ptr_type;
329 }
330
331 static LLVMTypeRef
get_vertex_header_ptr_type(struct draw_llvm_variant * variant)332 get_vertex_header_ptr_type(struct draw_llvm_variant *variant)
333 {
334 if (!variant->vertex_header_ptr_type)
335 create_jit_types(variant);
336 return variant->vertex_header_ptr_type;
337 }
338
339
340 /**
341 * Create per-context LLVM info.
342 */
343 struct draw_llvm *
draw_llvm_create(struct draw_context * draw)344 draw_llvm_create(struct draw_context *draw)
345 {
346 struct draw_llvm *llvm;
347
348 llvm = CALLOC_STRUCT( draw_llvm );
349 if (!llvm)
350 return NULL;
351
352 lp_build_init();
353
354 llvm->draw = draw;
355
356 llvm->nr_variants = 0;
357 make_empty_list(&llvm->vs_variants_list);
358
359 return llvm;
360 }
361
362
363 /**
364 * Free per-context LLVM info.
365 */
366 void
draw_llvm_destroy(struct draw_llvm * llvm)367 draw_llvm_destroy(struct draw_llvm *llvm)
368 {
369 /* XXX free other draw_llvm data? */
370 FREE(llvm);
371 }
372
373
374 /**
375 * Create LLVM-generated code for a vertex shader.
376 */
377 struct draw_llvm_variant *
draw_llvm_create_variant(struct draw_llvm * llvm,unsigned num_inputs,const struct draw_llvm_variant_key * key)378 draw_llvm_create_variant(struct draw_llvm *llvm,
379 unsigned num_inputs,
380 const struct draw_llvm_variant_key *key)
381 {
382 struct draw_llvm_variant *variant;
383 struct llvm_vertex_shader *shader =
384 llvm_vertex_shader(llvm->draw->vs.vertex_shader);
385 LLVMTypeRef vertex_header;
386
387 variant = MALLOC(sizeof *variant +
388 shader->variant_key_size -
389 sizeof variant->key);
390 if (variant == NULL)
391 return NULL;
392
393 variant->llvm = llvm;
394
395 variant->gallivm = gallivm_create();
396
397 create_jit_types(variant);
398
399 memcpy(&variant->key, key, shader->variant_key_size);
400
401 vertex_header = create_jit_vertex_header(variant->gallivm, num_inputs);
402
403 variant->vertex_header_ptr_type = LLVMPointerType(vertex_header, 0);
404
405 draw_llvm_generate(llvm, variant, FALSE); /* linear */
406 draw_llvm_generate(llvm, variant, TRUE); /* elts */
407
408 gallivm_compile_module(variant->gallivm);
409
410 variant->jit_func = (draw_jit_vert_func)
411 gallivm_jit_function(variant->gallivm, variant->function);
412
413 variant->jit_func_elts = (draw_jit_vert_func_elts)
414 gallivm_jit_function(variant->gallivm, variant->function_elts);
415
416 variant->shader = shader;
417 variant->list_item_global.base = variant;
418 variant->list_item_local.base = variant;
419 /*variant->no = */shader->variants_created++;
420 variant->list_item_global.base = variant;
421
422 return variant;
423 }
424
425
426 static void
generate_vs(struct draw_llvm_variant * variant,LLVMBuilderRef builder,struct lp_type vs_type,LLVMValueRef (* outputs)[TGSI_NUM_CHANNELS],const LLVMValueRef (* inputs)[TGSI_NUM_CHANNELS],const struct lp_bld_tgsi_system_values * system_values,LLVMValueRef context_ptr,struct lp_build_sampler_soa * draw_sampler,boolean clamp_vertex_color)427 generate_vs(struct draw_llvm_variant *variant,
428 LLVMBuilderRef builder,
429 struct lp_type vs_type,
430 LLVMValueRef (*outputs)[TGSI_NUM_CHANNELS],
431 const LLVMValueRef (*inputs)[TGSI_NUM_CHANNELS],
432 const struct lp_bld_tgsi_system_values *system_values,
433 LLVMValueRef context_ptr,
434 struct lp_build_sampler_soa *draw_sampler,
435 boolean clamp_vertex_color)
436 {
437 struct draw_llvm *llvm = variant->llvm;
438 const struct tgsi_token *tokens = llvm->draw->vs.vertex_shader->state.tokens;
439 LLVMValueRef consts_ptr = draw_jit_context_vs_constants(variant->gallivm, context_ptr);
440 struct lp_build_sampler_soa *sampler = 0;
441
442 if (gallivm_debug & GALLIVM_DEBUG_IR) {
443 tgsi_dump(tokens, 0);
444 }
445
446 if (llvm->draw->num_sampler_views && llvm->draw->num_samplers)
447 sampler = draw_sampler;
448
449 lp_build_tgsi_soa(variant->gallivm,
450 tokens,
451 vs_type,
452 NULL /*struct lp_build_mask_context *mask*/,
453 consts_ptr,
454 system_values,
455 NULL /*pos*/,
456 inputs,
457 outputs,
458 sampler,
459 &llvm->draw->vs.vertex_shader->info);
460
461 {
462 LLVMValueRef out;
463 unsigned chan, attrib;
464 struct lp_build_context bld;
465 struct tgsi_shader_info* info = &llvm->draw->vs.vertex_shader->info;
466 lp_build_context_init(&bld, variant->gallivm, vs_type);
467
468 for (attrib = 0; attrib < info->num_outputs; ++attrib) {
469 for (chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) {
470 if (outputs[attrib][chan]) {
471 switch (info->output_semantic_name[attrib]) {
472 case TGSI_SEMANTIC_COLOR:
473 case TGSI_SEMANTIC_BCOLOR:
474 if (clamp_vertex_color) {
475 out = LLVMBuildLoad(builder, outputs[attrib][chan], "");
476 out = lp_build_clamp(&bld, out, bld.zero, bld.one);
477 LLVMBuildStore(builder, out, outputs[attrib][chan]);
478 }
479 break;
480 case TGSI_SEMANTIC_FOG:
481 if (chan == 1 || chan == 2)
482 LLVMBuildStore(builder, bld.zero, outputs[attrib][chan]);
483 else if (chan == 3)
484 LLVMBuildStore(builder, bld.one, outputs[attrib][chan]);
485 break;
486 }
487 }
488 }
489 }
490 }
491 }
492
493
494 static void
generate_fetch(struct gallivm_state * gallivm,LLVMValueRef vbuffers_ptr,LLVMValueRef * res,struct pipe_vertex_element * velem,LLVMValueRef vbuf,LLVMValueRef index,LLVMValueRef instance_id)495 generate_fetch(struct gallivm_state *gallivm,
496 LLVMValueRef vbuffers_ptr,
497 LLVMValueRef *res,
498 struct pipe_vertex_element *velem,
499 LLVMValueRef vbuf,
500 LLVMValueRef index,
501 LLVMValueRef instance_id)
502 {
503 const struct util_format_description *format_desc = util_format_description(velem->src_format);
504 LLVMValueRef zero = LLVMConstNull(LLVMInt32TypeInContext(gallivm->context));
505 LLVMBuilderRef builder = gallivm->builder;
506 LLVMValueRef indices =
507 LLVMConstInt(LLVMInt64TypeInContext(gallivm->context),
508 velem->vertex_buffer_index, 0);
509 LLVMValueRef vbuffer_ptr = LLVMBuildGEP(builder, vbuffers_ptr,
510 &indices, 1, "");
511 LLVMValueRef vb_stride = draw_jit_vbuffer_stride(gallivm, vbuf);
512 LLVMValueRef vb_buffer_offset = draw_jit_vbuffer_offset(gallivm, vbuf);
513 LLVMValueRef stride;
514
515 if (velem->instance_divisor) {
516 /* array index = instance_id / instance_divisor */
517 index = LLVMBuildUDiv(builder, instance_id,
518 lp_build_const_int32(gallivm, velem->instance_divisor),
519 "instance_divisor");
520 }
521
522 stride = LLVMBuildMul(builder, vb_stride, index, "");
523
524 vbuffer_ptr = LLVMBuildLoad(builder, vbuffer_ptr, "vbuffer");
525
526 stride = LLVMBuildAdd(builder, stride,
527 vb_buffer_offset,
528 "");
529 stride = LLVMBuildAdd(builder, stride,
530 lp_build_const_int32(gallivm, velem->src_offset),
531 "");
532
533 /* lp_build_printf(gallivm, "vbuf index = %d, stride is %d\n", indices, stride);*/
534 vbuffer_ptr = LLVMBuildGEP(builder, vbuffer_ptr, &stride, 1, "");
535
536 *res = lp_build_fetch_rgba_aos(gallivm,
537 format_desc,
538 lp_float32_vec4_type(),
539 vbuffer_ptr,
540 zero, zero, zero);
541 }
542
543 static void
convert_to_soa(struct gallivm_state * gallivm,LLVMValueRef (* src_aos)[LP_MAX_VECTOR_WIDTH/32],LLVMValueRef (* dst_soa)[TGSI_NUM_CHANNELS],unsigned num_attribs,const struct lp_type soa_type)544 convert_to_soa(struct gallivm_state *gallivm,
545 LLVMValueRef (*src_aos)[LP_MAX_VECTOR_WIDTH / 32],
546 LLVMValueRef (*dst_soa)[TGSI_NUM_CHANNELS],
547 unsigned num_attribs, const struct lp_type soa_type)
548 {
549 unsigned i, j, k;
550 struct lp_type aos_channel_type = soa_type;
551
552 debug_assert(TGSI_NUM_CHANNELS == 4);
553 debug_assert((soa_type.length % TGSI_NUM_CHANNELS) == 0);
554
555 aos_channel_type.length >>= 1;
556
557 for (i = 0; i < num_attribs; ++i) {
558 LLVMValueRef aos_channels[TGSI_NUM_CHANNELS];
559 unsigned pixels_per_channel = soa_type.length / TGSI_NUM_CHANNELS;
560
561 for (j = 0; j < TGSI_NUM_CHANNELS; ++j) {
562 LLVMValueRef channel[LP_MAX_VECTOR_LENGTH] = { 0 };
563
564 assert(pixels_per_channel <= LP_MAX_VECTOR_LENGTH);
565
566 for (k = 0; k < pixels_per_channel; ++k) {
567 channel[k] = src_aos[i][j + TGSI_NUM_CHANNELS * k];
568 }
569
570 aos_channels[j] = lp_build_concat(gallivm, channel, aos_channel_type, pixels_per_channel);
571 }
572
573 lp_build_transpose_aos(gallivm, soa_type, aos_channels, dst_soa[i]);
574 }
575 }
576
577
578 static void
store_aos(struct gallivm_state * gallivm,LLVMValueRef io_ptr,LLVMValueRef index,LLVMValueRef value)579 store_aos(struct gallivm_state *gallivm,
580 LLVMValueRef io_ptr,
581 LLVMValueRef index,
582 LLVMValueRef value)
583 {
584 LLVMTypeRef data_ptr_type = LLVMPointerType(lp_build_vec_type(gallivm, lp_float32_vec4_type()), 0);
585 LLVMBuilderRef builder = gallivm->builder;
586 LLVMValueRef data_ptr = draw_jit_header_data(gallivm, io_ptr);
587 LLVMValueRef indices[3];
588
589 indices[0] = lp_build_const_int32(gallivm, 0);
590 indices[1] = index;
591 indices[2] = lp_build_const_int32(gallivm, 0);
592
593 #if DEBUG_STORE
594 lp_build_printf(gallivm, " ---- %p storing attribute %d (io = %p)\n", data_ptr, index, io_ptr);
595 #endif
596
597 data_ptr = LLVMBuildGEP(builder, data_ptr, indices, 3, "");
598 data_ptr = LLVMBuildPointerCast(builder, data_ptr, data_ptr_type, "");
599
600 /* Unaligned store due to the vertex header */
601 lp_set_store_alignment(LLVMBuildStore(builder, value, data_ptr), sizeof(float));
602 }
603
604
605 static void
store_aos_array(struct gallivm_state * gallivm,struct lp_type soa_type,LLVMValueRef io_ptr,LLVMValueRef * aos,int attrib,int num_outputs,LLVMValueRef clipmask,boolean have_clipdist)606 store_aos_array(struct gallivm_state *gallivm,
607 struct lp_type soa_type,
608 LLVMValueRef io_ptr,
609 LLVMValueRef* aos,
610 int attrib,
611 int num_outputs,
612 LLVMValueRef clipmask,
613 boolean have_clipdist)
614 {
615 LLVMBuilderRef builder = gallivm->builder;
616 LLVMValueRef attr_index = lp_build_const_int32(gallivm, attrib);
617 LLVMValueRef inds[LP_MAX_VECTOR_WIDTH / 32];
618 LLVMValueRef io_ptrs[LP_MAX_VECTOR_WIDTH / 32];
619 int vector_length = soa_type.length;
620 int i;
621
622 debug_assert(TGSI_NUM_CHANNELS == 4);
623
624 for (i = 0; i < vector_length; i++) {
625 inds[i] = lp_build_const_int32(gallivm, i);
626 io_ptrs[i] = LLVMBuildGEP(builder, io_ptr, &inds[i], 1, "");
627 }
628
629 if (attrib == 0) {
630 /* store vertex header for each of the n vertices */
631 LLVMValueRef val, cliptmp;
632 int vertex_id_pad_edgeflag;
633
634 /* If this assertion fails, it means we need to update the bit twidding
635 * code here. See struct vertex_header in draw_private.h.
636 */
637 assert(DRAW_TOTAL_CLIP_PLANES==14);
638 /* initialize vertex id:16 = 0xffff, have_clipdist:1 = 0, edgeflag:1 = 1 */
639 vertex_id_pad_edgeflag = (0xffff << 16) | (1 << DRAW_TOTAL_CLIP_PLANES);
640 if (have_clipdist)
641 vertex_id_pad_edgeflag |= 1 << (DRAW_TOTAL_CLIP_PLANES+1);
642 val = lp_build_const_int_vec(gallivm, lp_int_type(soa_type), vertex_id_pad_edgeflag);
643 /* OR with the clipmask */
644 cliptmp = LLVMBuildOr(builder, val, clipmask, "");
645 for (i = 0; i < vector_length; i++) {
646 LLVMValueRef id_ptr = draw_jit_header_id(gallivm, io_ptrs[i]);
647 val = LLVMBuildExtractElement(builder, cliptmp, inds[i], "");
648 LLVMBuildStore(builder, val, id_ptr);
649 #if DEBUG_STORE
650 lp_build_printf(gallivm, "io = %p, index %d\n, clipmask = %x\n",
651 io_ptrs[i], inds[i], val);
652 #endif
653 }
654 }
655
656 /* store for each of the n vertices */
657 for (i = 0; i < vector_length; i++) {
658 store_aos(gallivm, io_ptrs[i], attr_index, aos[i]);
659 }
660 }
661
662
663 static void
convert_to_aos(struct gallivm_state * gallivm,LLVMValueRef io,LLVMValueRef (* outputs)[TGSI_NUM_CHANNELS],LLVMValueRef clipmask,int num_outputs,struct lp_type soa_type,boolean have_clipdist)664 convert_to_aos(struct gallivm_state *gallivm,
665 LLVMValueRef io,
666 LLVMValueRef (*outputs)[TGSI_NUM_CHANNELS],
667 LLVMValueRef clipmask,
668 int num_outputs,
669 struct lp_type soa_type,
670 boolean have_clipdist)
671 {
672 LLVMBuilderRef builder = gallivm->builder;
673 unsigned chan, attrib, i;
674
675 #if DEBUG_STORE
676 lp_build_printf(gallivm, " # storing begin\n");
677 #endif
678 for (attrib = 0; attrib < num_outputs; ++attrib) {
679 LLVMValueRef soa[TGSI_NUM_CHANNELS];
680 LLVMValueRef aos[LP_MAX_VECTOR_WIDTH / 32];
681 for (chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) {
682 if (outputs[attrib][chan]) {
683 LLVMValueRef out = LLVMBuildLoad(builder, outputs[attrib][chan], "");
684 lp_build_name(out, "output%u.%c", attrib, "xyzw"[chan]);
685 #if DEBUG_STORE
686 lp_build_printf(gallivm, "output %d : %d ",
687 LLVMConstInt(LLVMInt32TypeInContext(gallivm->context),
688 attrib, 0),
689 LLVMConstInt(LLVMInt32TypeInContext(gallivm->context),
690 chan, 0));
691 lp_build_print_value(gallivm, "val = ", out);
692 #endif
693 soa[chan] = out;
694 }
695 else {
696 soa[chan] = 0;
697 }
698 }
699
700
701 if (soa_type.length == TGSI_NUM_CHANNELS) {
702 lp_build_transpose_aos(gallivm, soa_type, soa, aos);
703 } else {
704 lp_build_transpose_aos(gallivm, soa_type, soa, soa);
705
706 for (i = 0; i < soa_type.length; ++i) {
707 aos[i] = lp_build_extract_range(gallivm,
708 soa[i % TGSI_NUM_CHANNELS],
709 (i / TGSI_NUM_CHANNELS) * TGSI_NUM_CHANNELS,
710 TGSI_NUM_CHANNELS);
711 }
712 }
713
714 store_aos_array(gallivm,
715 soa_type,
716 io,
717 aos,
718 attrib,
719 num_outputs,
720 clipmask, have_clipdist);
721 }
722 #if DEBUG_STORE
723 lp_build_printf(gallivm, " # storing end\n");
724 #endif
725 }
726
727
728 /**
729 * Stores original vertex positions in clip coordinates
730 */
731 static void
store_clip(struct gallivm_state * gallivm,const struct lp_type vs_type,LLVMValueRef io_ptr,LLVMValueRef (* outputs)[TGSI_NUM_CHANNELS],boolean pre_clip_pos,int idx)732 store_clip(struct gallivm_state *gallivm,
733 const struct lp_type vs_type,
734 LLVMValueRef io_ptr,
735 LLVMValueRef (*outputs)[TGSI_NUM_CHANNELS],
736 boolean pre_clip_pos, int idx)
737 {
738 LLVMBuilderRef builder = gallivm->builder;
739 LLVMValueRef soa[4];
740 LLVMValueRef aos[LP_MAX_VECTOR_LENGTH];
741 LLVMValueRef indices[2];
742 LLVMValueRef io_ptrs[LP_MAX_VECTOR_WIDTH / 32];
743 LLVMValueRef inds[LP_MAX_VECTOR_WIDTH / 32];
744 LLVMValueRef clip_ptrs[LP_MAX_VECTOR_WIDTH / 32];
745 int i, j;
746
747 indices[0] =
748 indices[1] = lp_build_const_int32(gallivm, 0);
749
750 for (i = 0; i < vs_type.length; i++) {
751 inds[i] = lp_build_const_int32(gallivm, i);
752 io_ptrs[i] = LLVMBuildGEP(builder, io_ptr, &inds[i], 1, "");
753 }
754
755 soa[0] = LLVMBuildLoad(builder, outputs[idx][0], ""); /*x0 x1 .. xn*/
756 soa[1] = LLVMBuildLoad(builder, outputs[idx][1], ""); /*y0 y1 .. yn*/
757 soa[2] = LLVMBuildLoad(builder, outputs[idx][2], ""); /*z0 z1 .. zn*/
758 soa[3] = LLVMBuildLoad(builder, outputs[idx][3], ""); /*w0 w1 .. wn*/
759
760 if (!pre_clip_pos) {
761 for (i = 0; i < vs_type.length; i++) {
762 clip_ptrs[i] = draw_jit_header_clip(gallivm, io_ptrs[i]);
763 }
764 } else {
765 for (i = 0; i < vs_type.length; i++) {
766 clip_ptrs[i] = draw_jit_header_pre_clip_pos(gallivm, io_ptrs[i]);
767 }
768 }
769
770 lp_build_transpose_aos(gallivm, vs_type, soa, soa);
771 for (i = 0; i < vs_type.length; ++i) {
772 aos[i] = lp_build_extract_range(gallivm,
773 soa[i % TGSI_NUM_CHANNELS],
774 (i / TGSI_NUM_CHANNELS) * TGSI_NUM_CHANNELS,
775 TGSI_NUM_CHANNELS);
776 }
777
778 for (j = 0; j < vs_type.length; j++) {
779 LLVMTypeRef clip_ptr_type = LLVMPointerType(LLVMVectorType(LLVMFloatTypeInContext(gallivm->context), 4), 0);
780 LLVMValueRef clip_ptr;
781
782 clip_ptr = LLVMBuildGEP(builder, clip_ptrs[j], indices, 2, "clipo");
783 clip_ptr = LLVMBuildPointerCast(builder, clip_ptr, clip_ptr_type, "");
784
785 /* Unaligned store */
786 lp_set_store_alignment(LLVMBuildStore(builder, aos[j], clip_ptr), sizeof(float));
787 }
788 }
789
790
791 /**
792 * Transforms the outputs for viewport mapping
793 */
794 static void
generate_viewport(struct draw_llvm_variant * variant,LLVMBuilderRef builder,struct lp_type vs_type,LLVMValueRef (* outputs)[TGSI_NUM_CHANNELS],LLVMValueRef context_ptr)795 generate_viewport(struct draw_llvm_variant *variant,
796 LLVMBuilderRef builder,
797 struct lp_type vs_type,
798 LLVMValueRef (*outputs)[TGSI_NUM_CHANNELS],
799 LLVMValueRef context_ptr)
800 {
801 int i;
802 struct gallivm_state *gallivm = variant->gallivm;
803 struct lp_type f32_type = vs_type;
804 LLVMTypeRef vs_type_llvm = lp_build_vec_type(gallivm, vs_type);
805 LLVMValueRef out3 = LLVMBuildLoad(builder, outputs[0][3], ""); /*w0 w1 .. wn*/
806 LLVMValueRef const1 = lp_build_const_vec(gallivm, f32_type, 1.0); /*1.0 1.0 1.0 1.0*/
807 LLVMValueRef vp_ptr = draw_jit_context_viewport(gallivm, context_ptr);
808
809 /* for 1/w convention*/
810 out3 = LLVMBuildFDiv(builder, const1, out3, "");
811 LLVMBuildStore(builder, out3, outputs[0][3]);
812
813 /* Viewport Mapping */
814 for (i=0; i<3; i++) {
815 LLVMValueRef out = LLVMBuildLoad(builder, outputs[0][i], ""); /*x0 x1 .. xn*/
816 LLVMValueRef scale;
817 LLVMValueRef trans;
818 LLVMValueRef scale_i;
819 LLVMValueRef trans_i;
820 LLVMValueRef index;
821
822 index = lp_build_const_int32(gallivm, i);
823 scale_i = LLVMBuildGEP(builder, vp_ptr, &index, 1, "");
824
825 index = lp_build_const_int32(gallivm, i+4);
826 trans_i = LLVMBuildGEP(builder, vp_ptr, &index, 1, "");
827
828 scale = lp_build_broadcast(gallivm, vs_type_llvm,
829 LLVMBuildLoad(builder, scale_i, "scale"));
830 trans = lp_build_broadcast(gallivm, vs_type_llvm,
831 LLVMBuildLoad(builder, trans_i, "trans"));
832
833 /* divide by w */
834 out = LLVMBuildFMul(builder, out, out3, "");
835 /* mult by scale */
836 out = LLVMBuildFMul(builder, out, scale, "");
837 /* add translation */
838 out = LLVMBuildFAdd(builder, out, trans, "");
839
840 /* store transformed outputs */
841 LLVMBuildStore(builder, out, outputs[0][i]);
842 }
843
844 }
845
846
847 /**
848 * Returns clipmask as nxi32 bitmask for the n vertices
849 */
850 static LLVMValueRef
generate_clipmask(struct draw_llvm * llvm,struct gallivm_state * gallivm,struct lp_type vs_type,LLVMValueRef (* outputs)[TGSI_NUM_CHANNELS],boolean clip_xy,boolean clip_z,boolean clip_user,boolean clip_halfz,unsigned ucp_enable,LLVMValueRef context_ptr,boolean * have_clipdist)851 generate_clipmask(struct draw_llvm *llvm,
852 struct gallivm_state *gallivm,
853 struct lp_type vs_type,
854 LLVMValueRef (*outputs)[TGSI_NUM_CHANNELS],
855 boolean clip_xy,
856 boolean clip_z,
857 boolean clip_user,
858 boolean clip_halfz,
859 unsigned ucp_enable,
860 LLVMValueRef context_ptr,
861 boolean *have_clipdist)
862 {
863 LLVMBuilderRef builder = gallivm->builder;
864 LLVMValueRef mask; /* stores the <nxi32> clipmasks */
865 LLVMValueRef test, temp;
866 LLVMValueRef zero, shift;
867 LLVMValueRef pos_x, pos_y, pos_z, pos_w;
868 LLVMValueRef cv_x, cv_y, cv_z, cv_w;
869 LLVMValueRef plane1, planes, plane_ptr, sum;
870 struct lp_type f32_type = vs_type;
871 struct lp_type i32_type = lp_int_type(vs_type);
872 const unsigned pos = draw_current_shader_position_output(llvm->draw);
873 const unsigned cv = draw_current_shader_clipvertex_output(llvm->draw);
874 int num_written_clipdistance = llvm->draw->vs.vertex_shader->info.num_written_clipdistance;
875 bool have_cd = false;
876 unsigned cd[2];
877
878 cd[0] = draw_current_shader_clipdistance_output(llvm->draw, 0);
879 cd[1] = draw_current_shader_clipdistance_output(llvm->draw, 1);
880
881 if (cd[0] != pos || cd[1] != pos)
882 have_cd = true;
883
884 mask = lp_build_const_int_vec(gallivm, i32_type, 0);
885 temp = lp_build_const_int_vec(gallivm, i32_type, 0);
886 zero = lp_build_const_vec(gallivm, f32_type, 0); /* 0.0f 0.0f 0.0f 0.0f */
887 shift = lp_build_const_int_vec(gallivm, i32_type, 1); /* 1 1 1 1 */
888
889 /*
890 * load clipvertex and position from correct locations.
891 * if they are the same just load them once.
892 */
893 pos_x = LLVMBuildLoad(builder, outputs[pos][0], ""); /*x0 x1 .. xn */
894 pos_y = LLVMBuildLoad(builder, outputs[pos][1], ""); /*y0 y1 .. yn */
895 pos_z = LLVMBuildLoad(builder, outputs[pos][2], ""); /*z0 z1 .. zn */
896 pos_w = LLVMBuildLoad(builder, outputs[pos][3], ""); /*w0 w1 .. wn */
897
898 if (clip_user && cv != pos) {
899 cv_x = LLVMBuildLoad(builder, outputs[cv][0], ""); /*x0 x1 .. xn */
900 cv_y = LLVMBuildLoad(builder, outputs[cv][1], ""); /*y0 y1 .. yn */
901 cv_z = LLVMBuildLoad(builder, outputs[cv][2], ""); /*z0 z1 .. zn */
902 cv_w = LLVMBuildLoad(builder, outputs[cv][3], ""); /*w0 w1 .. wn */
903 } else {
904 cv_x = pos_x;
905 cv_y = pos_y;
906 cv_z = pos_z;
907 cv_w = pos_w;
908 }
909
910 /* Cliptest, for hardwired planes */
911 if (clip_xy) {
912 /* plane 1 */
913 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, pos_x , pos_w);
914 temp = shift;
915 test = LLVMBuildAnd(builder, test, temp, "");
916 mask = test;
917
918 /* plane 2 */
919 test = LLVMBuildFAdd(builder, pos_x, pos_w, "");
920 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, test);
921 temp = LLVMBuildShl(builder, temp, shift, "");
922 test = LLVMBuildAnd(builder, test, temp, "");
923 mask = LLVMBuildOr(builder, mask, test, "");
924
925 /* plane 3 */
926 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, pos_y, pos_w);
927 temp = LLVMBuildShl(builder, temp, shift, "");
928 test = LLVMBuildAnd(builder, test, temp, "");
929 mask = LLVMBuildOr(builder, mask, test, "");
930
931 /* plane 4 */
932 test = LLVMBuildFAdd(builder, pos_y, pos_w, "");
933 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, test);
934 temp = LLVMBuildShl(builder, temp, shift, "");
935 test = LLVMBuildAnd(builder, test, temp, "");
936 mask = LLVMBuildOr(builder, mask, test, "");
937 }
938
939 if (clip_z) {
940 temp = lp_build_const_int_vec(gallivm, i32_type, 16);
941 if (clip_halfz) {
942 /* plane 5 */
943 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, pos_z);
944 test = LLVMBuildAnd(builder, test, temp, "");
945 mask = LLVMBuildOr(builder, mask, test, "");
946 }
947 else {
948 /* plane 5 */
949 test = LLVMBuildFAdd(builder, pos_z, pos_w, "");
950 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, test);
951 test = LLVMBuildAnd(builder, test, temp, "");
952 mask = LLVMBuildOr(builder, mask, test, "");
953 }
954 /* plane 6 */
955 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, pos_z, pos_w);
956 temp = LLVMBuildShl(builder, temp, shift, "");
957 test = LLVMBuildAnd(builder, test, temp, "");
958 mask = LLVMBuildOr(builder, mask, test, "");
959 }
960
961 if (clip_user) {
962 LLVMValueRef planes_ptr = draw_jit_context_planes(gallivm, context_ptr);
963 LLVMValueRef indices[3];
964
965 /* userclip planes */
966 while (ucp_enable) {
967 unsigned plane_idx = ffs(ucp_enable)-1;
968 ucp_enable &= ~(1 << plane_idx);
969 plane_idx += 6;
970
971 if (have_cd && num_written_clipdistance) {
972 LLVMValueRef clipdist;
973 int i;
974 i = plane_idx - 6;
975
976 *have_clipdist = TRUE;
977 if (i < 4) {
978 clipdist = LLVMBuildLoad(builder, outputs[cd[0]][i], "");
979 } else {
980 clipdist = LLVMBuildLoad(builder, outputs[cd[1]][i-4], "");
981 }
982 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, clipdist);
983 temp = lp_build_const_int_vec(gallivm, i32_type, 1 << plane_idx);
984 test = LLVMBuildAnd(builder, test, temp, "");
985 mask = LLVMBuildOr(builder, mask, test, "");
986 } else {
987 LLVMTypeRef vs_type_llvm = lp_build_vec_type(gallivm, vs_type);
988 indices[0] = lp_build_const_int32(gallivm, 0);
989 indices[1] = lp_build_const_int32(gallivm, plane_idx);
990
991 indices[2] = lp_build_const_int32(gallivm, 0);
992 plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, "");
993 plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_x");
994 planes = lp_build_broadcast(gallivm, vs_type_llvm, plane1);
995 sum = LLVMBuildFMul(builder, planes, cv_x, "");
996
997 indices[2] = lp_build_const_int32(gallivm, 1);
998 plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, "");
999 plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_y");
1000 planes = lp_build_broadcast(gallivm, vs_type_llvm, plane1);
1001 test = LLVMBuildFMul(builder, planes, cv_y, "");
1002 sum = LLVMBuildFAdd(builder, sum, test, "");
1003
1004 indices[2] = lp_build_const_int32(gallivm, 2);
1005 plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, "");
1006 plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_z");
1007 planes = lp_build_broadcast(gallivm, vs_type_llvm, plane1);
1008 test = LLVMBuildFMul(builder, planes, cv_z, "");
1009 sum = LLVMBuildFAdd(builder, sum, test, "");
1010
1011 indices[2] = lp_build_const_int32(gallivm, 3);
1012 plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, "");
1013 plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_w");
1014 planes = lp_build_broadcast(gallivm, vs_type_llvm, plane1);
1015 test = LLVMBuildFMul(builder, planes, cv_w, "");
1016 sum = LLVMBuildFAdd(builder, sum, test, "");
1017
1018 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, sum);
1019 temp = lp_build_const_int_vec(gallivm, i32_type, 1 << plane_idx);
1020 test = LLVMBuildAnd(builder, test, temp, "");
1021 mask = LLVMBuildOr(builder, mask, test, "");
1022 }
1023 }
1024 }
1025 return mask;
1026 }
1027
1028
1029 /**
1030 * Returns boolean if any clipping has occurred
1031 * Used zero/non-zero i32 value to represent boolean
1032 */
1033 static LLVMValueRef
clipmask_booli32(struct gallivm_state * gallivm,const struct lp_type vs_type,LLVMValueRef clipmask_bool_ptr)1034 clipmask_booli32(struct gallivm_state *gallivm,
1035 const struct lp_type vs_type,
1036 LLVMValueRef clipmask_bool_ptr)
1037 {
1038 LLVMBuilderRef builder = gallivm->builder;
1039 LLVMTypeRef int32_type = LLVMInt32TypeInContext(gallivm->context);
1040 LLVMValueRef clipmask_bool = LLVMBuildLoad(builder, clipmask_bool_ptr, "");
1041 LLVMValueRef ret = LLVMConstNull(int32_type);
1042 LLVMValueRef temp;
1043 int i;
1044
1045 /*
1046 * Can do this with log2(vector length) pack instructions and one extract
1047 * (as we don't actually need a or) with sse2 which would be way better.
1048 */
1049 for (i=0; i < vs_type.length; i++) {
1050 temp = LLVMBuildExtractElement(builder, clipmask_bool,
1051 lp_build_const_int32(gallivm, i) , "");
1052 ret = LLVMBuildOr(builder, ret, temp, "");
1053 }
1054 return ret;
1055 }
1056
1057
1058 static void
draw_llvm_generate(struct draw_llvm * llvm,struct draw_llvm_variant * variant,boolean elts)1059 draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *variant,
1060 boolean elts)
1061 {
1062 struct gallivm_state *gallivm = variant->gallivm;
1063 LLVMContextRef context = gallivm->context;
1064 LLVMTypeRef int32_type = LLVMInt32TypeInContext(context);
1065 LLVMTypeRef arg_types[8];
1066 LLVMTypeRef func_type;
1067 LLVMValueRef context_ptr;
1068 LLVMBasicBlockRef block;
1069 LLVMBuilderRef builder;
1070 struct lp_type vs_type;
1071 LLVMValueRef end, start;
1072 LLVMValueRef count, fetch_elts, fetch_count;
1073 LLVMValueRef stride, step, io_itr;
1074 LLVMValueRef io_ptr, vbuffers_ptr, vb_ptr;
1075 LLVMValueRef zero = lp_build_const_int32(gallivm, 0);
1076 LLVMValueRef one = lp_build_const_int32(gallivm, 1);
1077 struct draw_context *draw = llvm->draw;
1078 const struct tgsi_shader_info *vs_info = &draw->vs.vertex_shader->info;
1079 unsigned i, j;
1080 struct lp_build_context bld;
1081 struct lp_build_loop_state lp_loop;
1082 const int vector_length = lp_native_vector_width / 32;
1083 LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][TGSI_NUM_CHANNELS];
1084 LLVMValueRef fetch_max;
1085 struct lp_build_sampler_soa *sampler = 0;
1086 LLVMValueRef ret, clipmask_bool_ptr;
1087 const boolean bypass_viewport = variant->key.bypass_viewport;
1088 const boolean enable_cliptest = variant->key.clip_xy ||
1089 variant->key.clip_z ||
1090 variant->key.clip_user;
1091 LLVMValueRef variant_func;
1092 const unsigned pos = draw_current_shader_position_output(llvm->draw);
1093 const unsigned cv = draw_current_shader_clipvertex_output(llvm->draw);
1094 boolean have_clipdist = FALSE;
1095 struct lp_bld_tgsi_system_values system_values;
1096
1097 memset(&system_values, 0, sizeof(system_values));
1098
1099 arg_types[0] = get_context_ptr_type(variant); /* context */
1100 arg_types[1] = get_vertex_header_ptr_type(variant); /* vertex_header */
1101 arg_types[2] = get_buffer_ptr_type(variant); /* vbuffers */
1102 if (elts)
1103 arg_types[3] = LLVMPointerType(int32_type, 0);/* fetch_elts * */
1104 else
1105 arg_types[3] = int32_type; /* start */
1106 arg_types[4] = int32_type; /* fetch_count / count */
1107 arg_types[5] = int32_type; /* stride */
1108 arg_types[6] = get_vb_ptr_type(variant); /* pipe_vertex_buffer's */
1109 arg_types[7] = int32_type; /* instance_id */
1110
1111 func_type = LLVMFunctionType(int32_type, arg_types, Elements(arg_types), 0);
1112
1113 variant_func = LLVMAddFunction(gallivm->module,
1114 elts ? "draw_llvm_shader_elts" : "draw_llvm_shader",
1115 func_type);
1116
1117 if (elts)
1118 variant->function_elts = variant_func;
1119 else
1120 variant->function = variant_func;
1121
1122 LLVMSetFunctionCallConv(variant_func, LLVMCCallConv);
1123 for (i = 0; i < Elements(arg_types); ++i)
1124 if (LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind)
1125 LLVMAddAttribute(LLVMGetParam(variant_func, i),
1126 LLVMNoAliasAttribute);
1127
1128 context_ptr = LLVMGetParam(variant_func, 0);
1129 io_ptr = LLVMGetParam(variant_func, 1);
1130 vbuffers_ptr = LLVMGetParam(variant_func, 2);
1131 stride = LLVMGetParam(variant_func, 5);
1132 vb_ptr = LLVMGetParam(variant_func, 6);
1133 system_values.instance_id = LLVMGetParam(variant_func, 7);
1134
1135 lp_build_name(context_ptr, "context");
1136 lp_build_name(io_ptr, "io");
1137 lp_build_name(vbuffers_ptr, "vbuffers");
1138 lp_build_name(stride, "stride");
1139 lp_build_name(vb_ptr, "vb");
1140 lp_build_name(system_values.instance_id, "instance_id");
1141
1142 if (elts) {
1143 fetch_elts = LLVMGetParam(variant_func, 3);
1144 fetch_count = LLVMGetParam(variant_func, 4);
1145 lp_build_name(fetch_elts, "fetch_elts");
1146 lp_build_name(fetch_count, "fetch_count");
1147 start = count = NULL;
1148 }
1149 else {
1150 start = LLVMGetParam(variant_func, 3);
1151 count = LLVMGetParam(variant_func, 4);
1152 lp_build_name(start, "start");
1153 lp_build_name(count, "count");
1154 fetch_elts = fetch_count = NULL;
1155 }
1156
1157 /*
1158 * Function body
1159 */
1160
1161 block = LLVMAppendBasicBlockInContext(gallivm->context, variant_func, "entry");
1162 builder = gallivm->builder;
1163 LLVMPositionBuilderAtEnd(builder, block);
1164
1165 lp_build_context_init(&bld, gallivm, lp_type_int(32));
1166
1167 memset(&vs_type, 0, sizeof vs_type);
1168 vs_type.floating = TRUE; /* floating point values */
1169 vs_type.sign = TRUE; /* values are signed */
1170 vs_type.norm = FALSE; /* values are not limited to [0,1] or [-1,1] */
1171 vs_type.width = 32; /* 32-bit float */
1172 vs_type.length = vector_length;
1173
1174 /* hold temporary "bool" clipmask */
1175 clipmask_bool_ptr = lp_build_alloca(gallivm, lp_build_int_vec_type(gallivm, vs_type), "");
1176 LLVMBuildStore(builder, lp_build_zero(gallivm, lp_int_type(vs_type)), clipmask_bool_ptr);
1177
1178 /* code generated texture sampling */
1179 sampler = draw_llvm_sampler_soa_create(
1180 draw_llvm_variant_key_samplers(&variant->key),
1181 context_ptr);
1182
1183 if (elts) {
1184 start = zero;
1185 end = fetch_count;
1186 }
1187 else {
1188 end = lp_build_add(&bld, start, count);
1189 }
1190
1191 step = lp_build_const_int32(gallivm, vector_length);
1192
1193 fetch_max = LLVMBuildSub(builder, end, one, "fetch_max");
1194
1195 lp_build_loop_begin(&lp_loop, gallivm, start);
1196 {
1197 LLVMValueRef inputs[PIPE_MAX_SHADER_INPUTS][TGSI_NUM_CHANNELS];
1198 LLVMValueRef aos_attribs[PIPE_MAX_SHADER_INPUTS][LP_MAX_VECTOR_WIDTH / 32] = { { 0 } };
1199 LLVMValueRef io;
1200 LLVMValueRef clipmask; /* holds the clipmask value */
1201 const LLVMValueRef (*ptr_aos)[TGSI_NUM_CHANNELS];
1202
1203 if (elts)
1204 io_itr = lp_loop.counter;
1205 else
1206 io_itr = LLVMBuildSub(builder, lp_loop.counter, start, "");
1207
1208 io = LLVMBuildGEP(builder, io_ptr, &io_itr, 1, "");
1209 #if DEBUG_STORE
1210 lp_build_printf(gallivm, " --- io %d = %p, loop counter %d\n",
1211 io_itr, io, lp_loop.counter);
1212 #endif
1213 system_values.vertex_id = lp_build_zero(gallivm, lp_type_uint_vec(32, 32*vector_length));
1214 for (i = 0; i < vector_length; ++i) {
1215 LLVMValueRef true_index =
1216 LLVMBuildAdd(builder,
1217 lp_loop.counter,
1218 lp_build_const_int32(gallivm, i), "");
1219
1220 /* make sure we're not out of bounds which can happen
1221 * if fetch_count % 4 != 0, because on the last iteration
1222 * a few of the 4 vertex fetches will be out of bounds */
1223 true_index = lp_build_min(&bld, true_index, fetch_max);
1224
1225 if (elts) {
1226 LLVMValueRef fetch_ptr;
1227 fetch_ptr = LLVMBuildGEP(builder, fetch_elts,
1228 &true_index, 1, "");
1229 true_index = LLVMBuildLoad(builder, fetch_ptr, "fetch_elt");
1230 }
1231
1232 system_values.vertex_id = LLVMBuildInsertElement(gallivm->builder,
1233 system_values.vertex_id, true_index,
1234 lp_build_const_int32(gallivm, i), "");
1235 for (j = 0; j < draw->pt.nr_vertex_elements; ++j) {
1236 struct pipe_vertex_element *velem = &draw->pt.vertex_element[j];
1237 LLVMValueRef vb_index =
1238 lp_build_const_int32(gallivm, velem->vertex_buffer_index);
1239 LLVMValueRef vb = LLVMBuildGEP(builder, vb_ptr, &vb_index, 1, "");
1240 generate_fetch(gallivm, vbuffers_ptr,
1241 &aos_attribs[j][i], velem, vb, true_index,
1242 system_values.instance_id);
1243 }
1244 }
1245 convert_to_soa(gallivm, aos_attribs, inputs,
1246 draw->pt.nr_vertex_elements, vs_type);
1247
1248 ptr_aos = (const LLVMValueRef (*)[TGSI_NUM_CHANNELS]) inputs;
1249 generate_vs(variant,
1250 builder,
1251 vs_type,
1252 outputs,
1253 ptr_aos,
1254 &system_values,
1255 context_ptr,
1256 sampler,
1257 variant->key.clamp_vertex_color);
1258
1259 /* store original positions in clip before further manipulation */
1260 store_clip(gallivm, vs_type, io, outputs, 0, cv);
1261 store_clip(gallivm, vs_type, io, outputs, 1, pos);
1262
1263 /* do cliptest */
1264 if (enable_cliptest) {
1265 LLVMValueRef temp = LLVMBuildLoad(builder, clipmask_bool_ptr, "");
1266 /* allocate clipmask, assign it integer type */
1267 clipmask = generate_clipmask(llvm,
1268 gallivm,
1269 vs_type,
1270 outputs,
1271 variant->key.clip_xy,
1272 variant->key.clip_z,
1273 variant->key.clip_user,
1274 variant->key.clip_halfz,
1275 variant->key.ucp_enable,
1276 context_ptr, &have_clipdist);
1277 temp = LLVMBuildOr(builder, clipmask, temp, "");
1278 /* store temporary clipping boolean value */
1279 LLVMBuildStore(builder, temp, clipmask_bool_ptr);
1280 }
1281 else {
1282 clipmask = lp_build_const_int_vec(gallivm, lp_int_type(vs_type), 0);
1283 }
1284
1285 /* do viewport mapping */
1286 if (!bypass_viewport) {
1287 generate_viewport(variant, builder, vs_type, outputs, context_ptr);
1288 }
1289
1290 /* store clipmask in vertex header,
1291 * original positions in clip
1292 * and transformed positions in data
1293 */
1294 convert_to_aos(gallivm, io, outputs, clipmask,
1295 vs_info->num_outputs, vs_type,
1296 have_clipdist);
1297 }
1298
1299 lp_build_loop_end_cond(&lp_loop, end, step, LLVMIntUGE);
1300
1301 sampler->destroy(sampler);
1302
1303 /* return clipping boolean value for function */
1304 ret = clipmask_booli32(gallivm, vs_type, clipmask_bool_ptr);
1305
1306 LLVMBuildRet(builder, ret);
1307
1308 gallivm_verify_function(gallivm, variant_func);
1309 }
1310
1311
1312 struct draw_llvm_variant_key *
draw_llvm_make_variant_key(struct draw_llvm * llvm,char * store)1313 draw_llvm_make_variant_key(struct draw_llvm *llvm, char *store)
1314 {
1315 unsigned i;
1316 struct draw_llvm_variant_key *key;
1317 struct lp_sampler_static_state *sampler;
1318
1319 key = (struct draw_llvm_variant_key *)store;
1320
1321 key->clamp_vertex_color = llvm->draw->rasterizer->clamp_vertex_color; /**/
1322
1323 /* Presumably all variants of the shader should have the same
1324 * number of vertex elements - ie the number of shader inputs.
1325 */
1326 key->nr_vertex_elements = llvm->draw->pt.nr_vertex_elements;
1327
1328 /* will have to rig this up properly later */
1329 key->clip_xy = llvm->draw->clip_xy;
1330 key->clip_z = llvm->draw->clip_z;
1331 key->clip_user = llvm->draw->clip_user;
1332 key->bypass_viewport = llvm->draw->identity_viewport;
1333 key->clip_halfz = !llvm->draw->rasterizer->gl_rasterization_rules;
1334 key->need_edgeflags = (llvm->draw->vs.edgeflag_output ? TRUE : FALSE);
1335 key->ucp_enable = llvm->draw->rasterizer->clip_plane_enable;
1336 key->pad = 0;
1337
1338 /* All variants of this shader will have the same value for
1339 * nr_samplers. Not yet trying to compact away holes in the
1340 * sampler array.
1341 */
1342 key->nr_samplers = llvm->draw->vs.vertex_shader->info.file_max[TGSI_FILE_SAMPLER] + 1;
1343
1344 sampler = draw_llvm_variant_key_samplers(key);
1345
1346 memcpy(key->vertex_element,
1347 llvm->draw->pt.vertex_element,
1348 sizeof(struct pipe_vertex_element) * key->nr_vertex_elements);
1349
1350 memset(sampler, 0, key->nr_samplers * sizeof *sampler);
1351
1352 for (i = 0 ; i < key->nr_samplers; i++) {
1353 lp_sampler_static_state(&sampler[i],
1354 llvm->draw->sampler_views[PIPE_SHADER_VERTEX][i],
1355 llvm->draw->samplers[PIPE_SHADER_VERTEX][i]);
1356 }
1357
1358 return key;
1359 }
1360
1361
1362 void
draw_llvm_set_mapped_texture(struct draw_context * draw,unsigned sampler_idx,uint32_t width,uint32_t height,uint32_t depth,uint32_t first_level,uint32_t last_level,uint32_t row_stride[PIPE_MAX_TEXTURE_LEVELS],uint32_t img_stride[PIPE_MAX_TEXTURE_LEVELS],const void * data[PIPE_MAX_TEXTURE_LEVELS])1363 draw_llvm_set_mapped_texture(struct draw_context *draw,
1364 unsigned sampler_idx,
1365 uint32_t width, uint32_t height, uint32_t depth,
1366 uint32_t first_level, uint32_t last_level,
1367 uint32_t row_stride[PIPE_MAX_TEXTURE_LEVELS],
1368 uint32_t img_stride[PIPE_MAX_TEXTURE_LEVELS],
1369 const void *data[PIPE_MAX_TEXTURE_LEVELS])
1370 {
1371 unsigned j;
1372 struct draw_jit_texture *jit_tex;
1373
1374 assert(sampler_idx < Elements(draw->llvm->jit_context.textures));
1375
1376 jit_tex = &draw->llvm->jit_context.textures[sampler_idx];
1377
1378 jit_tex->width = width;
1379 jit_tex->height = height;
1380 jit_tex->depth = depth;
1381 jit_tex->first_level = first_level;
1382 jit_tex->last_level = last_level;
1383
1384 for (j = first_level; j <= last_level; j++) {
1385 jit_tex->data[j] = data[j];
1386 jit_tex->row_stride[j] = row_stride[j];
1387 jit_tex->img_stride[j] = img_stride[j];
1388 }
1389 }
1390
1391
1392 void
draw_llvm_set_sampler_state(struct draw_context * draw)1393 draw_llvm_set_sampler_state(struct draw_context *draw)
1394 {
1395 unsigned i;
1396
1397 for (i = 0; i < draw->num_samplers[PIPE_SHADER_VERTEX]; i++) {
1398 struct draw_jit_texture *jit_tex = &draw->llvm->jit_context.textures[i];
1399
1400 if (draw->samplers[i]) {
1401 const struct pipe_sampler_state *s
1402 = draw->samplers[PIPE_SHADER_VERTEX][i];
1403 jit_tex->min_lod = s->min_lod;
1404 jit_tex->max_lod = s->max_lod;
1405 jit_tex->lod_bias = s->lod_bias;
1406 COPY_4V(jit_tex->border_color, s->border_color.f);
1407 }
1408 }
1409 }
1410
1411
1412 void
draw_llvm_destroy_variant(struct draw_llvm_variant * variant)1413 draw_llvm_destroy_variant(struct draw_llvm_variant *variant)
1414 {
1415 struct draw_llvm *llvm = variant->llvm;
1416
1417 if (variant->function_elts) {
1418 gallivm_free_function(variant->gallivm,
1419 variant->function_elts, variant->jit_func_elts);
1420 }
1421
1422 if (variant->function) {
1423 gallivm_free_function(variant->gallivm,
1424 variant->function, variant->jit_func);
1425 }
1426
1427 gallivm_destroy(variant->gallivm);
1428
1429 remove_from_list(&variant->list_item_local);
1430 variant->shader->variants_cached--;
1431 remove_from_list(&variant->list_item_global);
1432 llvm->nr_variants--;
1433 FREE(variant);
1434 }
1435