1 /**************************************************************************
2 *
3 * Copyright 2010 VMware, Inc.
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
21 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27
28 #include "util/u_math.h"
29 #include "util/u_memory.h"
30 #include "util/u_prim.h"
31 #include "draw/draw_context.h"
32 #include "draw/draw_gs.h"
33 #include "draw/draw_tess.h"
34 #include "draw/draw_vbuf.h"
35 #include "draw/draw_vertex.h"
36 #include "draw/draw_pt.h"
37 #include "draw/draw_prim_assembler.h"
38 #include "draw/draw_vs.h"
39 #include "draw/draw_llvm.h"
40 #include "gallivm/lp_bld_init.h"
41 #include "gallivm/lp_bld_debug.h"
42
43
44 struct llvm_middle_end {
45 struct draw_pt_middle_end base;
46 struct draw_context *draw;
47
48 struct pt_emit *emit;
49 struct pt_so_emit *so_emit;
50 struct pt_fetch *fetch;
51 struct pt_post_vs *post_vs;
52
53
54 unsigned vertex_data_offset;
55 unsigned vertex_size;
56 unsigned input_prim;
57 unsigned opt;
58
59 struct draw_llvm *llvm;
60 struct draw_llvm_variant *current_variant;
61 };
62
63
64 /** cast wrapper */
65 static inline struct llvm_middle_end *
llvm_middle_end(struct draw_pt_middle_end * middle)66 llvm_middle_end(struct draw_pt_middle_end *middle)
67 {
68 return (struct llvm_middle_end *) middle;
69 }
70
71
72 static void
llvm_middle_end_prepare_gs(struct llvm_middle_end * fpme)73 llvm_middle_end_prepare_gs(struct llvm_middle_end *fpme)
74 {
75 struct draw_context *draw = fpme->draw;
76 struct draw_llvm *llvm = fpme->llvm;
77 struct draw_geometry_shader *gs = draw->gs.geometry_shader;
78 struct draw_gs_llvm_variant_key *key;
79 struct draw_gs_llvm_variant *variant = NULL;
80 struct draw_gs_llvm_variant_list_item *li;
81 struct llvm_geometry_shader *shader = llvm_geometry_shader(gs);
82 char store[DRAW_GS_LLVM_MAX_VARIANT_KEY_SIZE];
83 unsigned i;
84
85 key = draw_gs_llvm_make_variant_key(llvm, store);
86
87 /* Search shader's list of variants for the key */
88 li = first_elem(&shader->variants);
89 while (!at_end(&shader->variants, li)) {
90 if (memcmp(&li->base->key, key, shader->variant_key_size) == 0) {
91 variant = li->base;
92 break;
93 }
94 li = next_elem(li);
95 }
96
97 if (variant) {
98 /* found the variant, move to head of global list (for LRU) */
99 move_to_head(&llvm->gs_variants_list, &variant->list_item_global);
100 }
101 else {
102 /* Need to create new variant */
103
104 /* First check if we've created too many variants. If so, free
105 * 3.125% of the LRU to avoid using too much memory.
106 */
107 if (llvm->nr_gs_variants >= DRAW_MAX_SHADER_VARIANTS) {
108 if (gallivm_debug & GALLIVM_DEBUG_PERF) {
109 debug_printf("Evicting GS: %u gs variants,\t%u total variants\n",
110 shader->variants_cached, llvm->nr_gs_variants);
111 }
112
113 /*
114 * XXX: should we flush here ?
115 */
116 for (i = 0; i < DRAW_MAX_SHADER_VARIANTS / 32; i++) {
117 struct draw_gs_llvm_variant_list_item *item;
118 if (is_empty_list(&llvm->gs_variants_list)) {
119 break;
120 }
121 item = last_elem(&llvm->gs_variants_list);
122 assert(item);
123 assert(item->base);
124 draw_gs_llvm_destroy_variant(item->base);
125 }
126 }
127
128 variant = draw_gs_llvm_create_variant(llvm, gs->info.num_outputs, key);
129
130 if (variant) {
131 insert_at_head(&shader->variants, &variant->list_item_local);
132 insert_at_head(&llvm->gs_variants_list,
133 &variant->list_item_global);
134 llvm->nr_gs_variants++;
135 shader->variants_cached++;
136 }
137 }
138
139 gs->current_variant = variant;
140 }
141
142 static void
llvm_middle_end_prepare_tcs(struct llvm_middle_end * fpme)143 llvm_middle_end_prepare_tcs(struct llvm_middle_end *fpme)
144 {
145 struct draw_context *draw = fpme->draw;
146 struct draw_llvm *llvm = fpme->llvm;
147 struct draw_tess_ctrl_shader *tcs = draw->tcs.tess_ctrl_shader;
148 struct draw_tcs_llvm_variant_key *key;
149 struct draw_tcs_llvm_variant *variant = NULL;
150 struct draw_tcs_llvm_variant_list_item *li;
151 struct llvm_tess_ctrl_shader *shader = llvm_tess_ctrl_shader(tcs);
152 char store[DRAW_TCS_LLVM_MAX_VARIANT_KEY_SIZE];
153 unsigned i;
154
155 key = draw_tcs_llvm_make_variant_key(llvm, store);
156
157 /* Search shader's list of variants for the key */
158 li = first_elem(&shader->variants);
159 while (!at_end(&shader->variants, li)) {
160 if (memcmp(&li->base->key, key, shader->variant_key_size) == 0) {
161 variant = li->base;
162 break;
163 }
164 li = next_elem(li);
165 }
166
167 if (variant) {
168 /* found the variant, move to head of global list (for LRU) */
169 move_to_head(&llvm->tcs_variants_list, &variant->list_item_global);
170 }
171 else {
172 /* Need to create new variant */
173
174 /* First check if we've created too many variants. If so, free
175 * 3.125% of the LRU to avoid using too much memory.
176 */
177 if (llvm->nr_tcs_variants >= DRAW_MAX_SHADER_VARIANTS) {
178 if (gallivm_debug & GALLIVM_DEBUG_PERF) {
179 debug_printf("Evicting TCS: %u tcs variants,\t%u total variants\n",
180 shader->variants_cached, llvm->nr_tcs_variants);
181 }
182
183 /*
184 * XXX: should we flush here ?
185 */
186 for (i = 0; i < DRAW_MAX_SHADER_VARIANTS / 32; i++) {
187 struct draw_tcs_llvm_variant_list_item *item;
188 if (is_empty_list(&llvm->tcs_variants_list)) {
189 break;
190 }
191 item = last_elem(&llvm->tcs_variants_list);
192 assert(item);
193 assert(item->base);
194 draw_tcs_llvm_destroy_variant(item->base);
195 }
196 }
197
198 variant = draw_tcs_llvm_create_variant(llvm, 0, key);
199
200 if (variant) {
201 insert_at_head(&shader->variants, &variant->list_item_local);
202 insert_at_head(&llvm->tcs_variants_list,
203 &variant->list_item_global);
204 llvm->nr_tcs_variants++;
205 shader->variants_cached++;
206 }
207 }
208
209 tcs->current_variant = variant;
210 }
211
212 static void
llvm_middle_end_prepare_tes(struct llvm_middle_end * fpme)213 llvm_middle_end_prepare_tes(struct llvm_middle_end *fpme)
214 {
215 struct draw_context *draw = fpme->draw;
216 struct draw_llvm *llvm = fpme->llvm;
217 struct draw_tess_eval_shader *tes = draw->tes.tess_eval_shader;
218 struct draw_tes_llvm_variant_key *key;
219 struct draw_tes_llvm_variant *variant = NULL;
220 struct draw_tes_llvm_variant_list_item *li;
221 struct llvm_tess_eval_shader *shader = llvm_tess_eval_shader(tes);
222 char store[DRAW_TES_LLVM_MAX_VARIANT_KEY_SIZE];
223 unsigned i;
224
225 key = draw_tes_llvm_make_variant_key(llvm, store);
226
227 /* Search shader's list of variants for the key */
228 li = first_elem(&shader->variants);
229 while (!at_end(&shader->variants, li)) {
230 if (memcmp(&li->base->key, key, shader->variant_key_size) == 0) {
231 variant = li->base;
232 break;
233 }
234 li = next_elem(li);
235 }
236
237 if (variant) {
238 /* found the variant, move to head of global list (for LRU) */
239 move_to_head(&llvm->tes_variants_list, &variant->list_item_global);
240 }
241 else {
242 /* Need to create new variant */
243
244 /* First check if we've created too many variants. If so, free
245 * 3.125% of the LRU to avoid using too much memory.
246 */
247 if (llvm->nr_tes_variants >= DRAW_MAX_SHADER_VARIANTS) {
248 if (gallivm_debug & GALLIVM_DEBUG_PERF) {
249 debug_printf("Evicting TES: %u tes variants,\t%u total variants\n",
250 shader->variants_cached, llvm->nr_tes_variants);
251 }
252
253 /*
254 * XXX: should we flush here ?
255 */
256 for (i = 0; i < DRAW_MAX_SHADER_VARIANTS / 32; i++) {
257 struct draw_tes_llvm_variant_list_item *item;
258 if (is_empty_list(&llvm->tes_variants_list)) {
259 break;
260 }
261 item = last_elem(&llvm->tes_variants_list);
262 assert(item);
263 assert(item->base);
264 draw_tes_llvm_destroy_variant(item->base);
265 }
266 }
267
268 variant = draw_tes_llvm_create_variant(llvm, tes->info.num_outputs, key);
269
270 if (variant) {
271 insert_at_head(&shader->variants, &variant->list_item_local);
272 insert_at_head(&llvm->tes_variants_list,
273 &variant->list_item_global);
274 llvm->nr_tes_variants++;
275 shader->variants_cached++;
276 }
277 }
278
279 tes->current_variant = variant;
280 }
281
282 /**
283 * Prepare/validate middle part of the vertex pipeline.
284 * NOTE: if you change this function, also look at the non-LLVM
285 * function fetch_pipeline_prepare() for similar changes.
286 */
287 static void
llvm_middle_end_prepare(struct draw_pt_middle_end * middle,unsigned in_prim,unsigned opt,unsigned * max_vertices)288 llvm_middle_end_prepare( struct draw_pt_middle_end *middle,
289 unsigned in_prim,
290 unsigned opt,
291 unsigned *max_vertices )
292 {
293 struct llvm_middle_end *fpme = llvm_middle_end(middle);
294 struct draw_context *draw = fpme->draw;
295 struct draw_llvm *llvm = fpme->llvm;
296 struct draw_vertex_shader *vs = draw->vs.vertex_shader;
297 struct draw_geometry_shader *gs = draw->gs.geometry_shader;
298 struct draw_tess_ctrl_shader *tcs = draw->tcs.tess_ctrl_shader;
299 struct draw_tess_eval_shader *tes = draw->tes.tess_eval_shader;
300 const unsigned out_prim = gs ? gs->output_primitive : tes ? get_tes_output_prim(tes) :
301 u_assembled_prim(in_prim);
302 unsigned point_clip = draw->rasterizer->fill_front == PIPE_POLYGON_MODE_POINT ||
303 out_prim == PIPE_PRIM_POINTS;
304 unsigned nr;
305
306 fpme->input_prim = in_prim;
307 fpme->opt = opt;
308
309 draw_pt_post_vs_prepare( fpme->post_vs,
310 draw->clip_xy,
311 draw->clip_z,
312 draw->clip_user,
313 point_clip ? draw->guard_band_points_xy :
314 draw->guard_band_xy,
315 draw->bypass_viewport,
316 draw->rasterizer->clip_halfz,
317 (draw->vs.edgeflag_output ? TRUE : FALSE) );
318
319 draw_pt_so_emit_prepare( fpme->so_emit, gs == NULL );
320
321 if (!(opt & PT_PIPELINE)) {
322 draw_pt_emit_prepare( fpme->emit, out_prim,
323 max_vertices );
324
325 *max_vertices = MAX2( *max_vertices, 4096 );
326 }
327 else {
328 /* limit max fetches by limiting max_vertices */
329 *max_vertices = 4096;
330 }
331
332 /* Get the number of float[4] attributes per vertex.
333 * Note: this must be done after draw_pt_emit_prepare() since that
334 * can effect the vertex size.
335 */
336 nr = MAX2(vs->info.num_inputs, draw_total_vs_outputs(draw));
337
338 /* Always leave room for the vertex header whether we need it or
339 * not. It's hard to get rid of it in particular because of the
340 * viewport code in draw_pt_post_vs.c.
341 */
342 fpme->vertex_size = sizeof(struct vertex_header) + nr * 4 * sizeof(float);
343
344 /* return even number */
345 *max_vertices = *max_vertices & ~1;
346
347 /* Find/create the vertex shader variant */
348 {
349 struct draw_llvm_variant_key *key;
350 struct draw_llvm_variant *variant = NULL;
351 struct draw_llvm_variant_list_item *li;
352 struct llvm_vertex_shader *shader = llvm_vertex_shader(vs);
353 char store[DRAW_LLVM_MAX_VARIANT_KEY_SIZE];
354 unsigned i;
355
356 key = draw_llvm_make_variant_key(llvm, store);
357
358 /* Search shader's list of variants for the key */
359 li = first_elem(&shader->variants);
360 while (!at_end(&shader->variants, li)) {
361 if (memcmp(&li->base->key, key, shader->variant_key_size) == 0) {
362 variant = li->base;
363 break;
364 }
365 li = next_elem(li);
366 }
367
368 if (variant) {
369 /* found the variant, move to head of global list (for LRU) */
370 move_to_head(&llvm->vs_variants_list, &variant->list_item_global);
371 }
372 else {
373 /* Need to create new variant */
374
375 /* First check if we've created too many variants. If so, free
376 * 3.125% of the LRU to avoid using too much memory.
377 */
378 if (llvm->nr_variants >= DRAW_MAX_SHADER_VARIANTS) {
379 if (gallivm_debug & GALLIVM_DEBUG_PERF) {
380 debug_printf("Evicting VS: %u vs variants,\t%u total variants\n",
381 shader->variants_cached, llvm->nr_variants);
382 }
383
384 /*
385 * XXX: should we flush here ?
386 */
387 for (i = 0; i < DRAW_MAX_SHADER_VARIANTS / 32; i++) {
388 struct draw_llvm_variant_list_item *item;
389 if (is_empty_list(&llvm->vs_variants_list)) {
390 break;
391 }
392 item = last_elem(&llvm->vs_variants_list);
393 assert(item);
394 assert(item->base);
395 draw_llvm_destroy_variant(item->base);
396 }
397 }
398
399 variant = draw_llvm_create_variant(llvm, nr, key);
400
401 if (variant) {
402 insert_at_head(&shader->variants, &variant->list_item_local);
403 insert_at_head(&llvm->vs_variants_list,
404 &variant->list_item_global);
405 llvm->nr_variants++;
406 shader->variants_cached++;
407 }
408 }
409
410 fpme->current_variant = variant;
411 }
412
413 if (gs) {
414 llvm_middle_end_prepare_gs(fpme);
415 }
416 if (tcs) {
417 llvm_middle_end_prepare_tcs(fpme);
418 }
419 if (tes) {
420 llvm_middle_end_prepare_tes(fpme);
421 }
422 }
423
424 static unsigned
get_num_consts_robust(struct draw_context * draw,unsigned * sizes,unsigned idx)425 get_num_consts_robust(struct draw_context *draw, unsigned *sizes, unsigned idx)
426 {
427 unsigned const_bytes = sizes[idx];
428
429 if (const_bytes < sizeof(float))
430 return 0;
431
432 return DIV_ROUND_UP(const_bytes, draw->constant_buffer_stride);
433 }
434
435 /**
436 * Bind/update constant buffer pointers, clip planes and viewport dims.
437 * These are "light weight" parameters which aren't baked into the
438 * generated code. Updating these items is much cheaper than revalidating
439 * and rebuilding the generated pipeline code.
440 */
441 static void
llvm_middle_end_bind_parameters(struct draw_pt_middle_end * middle)442 llvm_middle_end_bind_parameters(struct draw_pt_middle_end *middle)
443 {
444 static const float fake_const_buf[4];
445 struct llvm_middle_end *fpme = llvm_middle_end(middle);
446 struct draw_context *draw = fpme->draw;
447 struct draw_llvm *llvm = fpme->llvm;
448 unsigned i;
449
450 for (i = 0; i < ARRAY_SIZE(llvm->jit_context.vs_constants); ++i) {
451 /*
452 * There could be a potential issue with rounding this up, as the
453 * shader expects 16-byte allocations, the fix is likely to move
454 * to LOAD intrinsic in the future and remove the vec4 constraint.
455 */
456 int num_consts = get_num_consts_robust(draw, draw->pt.user.vs_constants_size, i);
457 llvm->jit_context.vs_constants[i] = draw->pt.user.vs_constants[i];
458 llvm->jit_context.num_vs_constants[i] = num_consts;
459 if (num_consts == 0) {
460 llvm->jit_context.vs_constants[i] = fake_const_buf;
461 }
462 }
463 for (i = 0; i < ARRAY_SIZE(llvm->jit_context.vs_ssbos); ++i) {
464 int num_ssbos = draw->pt.user.vs_ssbos_size[i];
465 llvm->jit_context.vs_ssbos[i] = draw->pt.user.vs_ssbos[i];
466 llvm->jit_context.num_vs_ssbos[i] = num_ssbos;
467 if (num_ssbos == 0) {
468 llvm->jit_context.vs_ssbos[i] = (const uint32_t *)fake_const_buf;
469 }
470 }
471
472 for (i = 0; i < ARRAY_SIZE(llvm->gs_jit_context.constants); ++i) {
473 int num_consts = get_num_consts_robust(draw, draw->pt.user.gs_constants_size, i);
474 llvm->gs_jit_context.constants[i] = draw->pt.user.gs_constants[i];
475 llvm->gs_jit_context.num_constants[i] = num_consts;
476 if (num_consts == 0) {
477 llvm->gs_jit_context.constants[i] = fake_const_buf;
478 }
479 }
480 for (i = 0; i < ARRAY_SIZE(llvm->gs_jit_context.ssbos); ++i) {
481 int num_ssbos = draw->pt.user.gs_ssbos_size[i];
482 llvm->gs_jit_context.ssbos[i] = draw->pt.user.gs_ssbos[i];
483 llvm->gs_jit_context.num_ssbos[i] = num_ssbos;
484 if (num_ssbos == 0) {
485 llvm->gs_jit_context.ssbos[i] = (const uint32_t *)fake_const_buf;
486 }
487 }
488
489 for (i = 0; i < ARRAY_SIZE(llvm->tcs_jit_context.constants); ++i) {
490 int num_consts = get_num_consts_robust(draw, draw->pt.user.tcs_constants_size, i);
491 llvm->tcs_jit_context.constants[i] = draw->pt.user.tcs_constants[i];
492 llvm->tcs_jit_context.num_constants[i] = num_consts;
493 if (num_consts == 0) {
494 llvm->tcs_jit_context.constants[i] = fake_const_buf;
495 }
496 }
497 for (i = 0; i < ARRAY_SIZE(llvm->tcs_jit_context.ssbos); ++i) {
498 int num_ssbos = draw->pt.user.tcs_ssbos_size[i];
499 llvm->tcs_jit_context.ssbos[i] = draw->pt.user.tcs_ssbos[i];
500 llvm->tcs_jit_context.num_ssbos[i] = num_ssbos;
501 if (num_ssbos == 0) {
502 llvm->tcs_jit_context.ssbos[i] = (const uint32_t *)fake_const_buf;
503 }
504 }
505
506 for (i = 0; i < ARRAY_SIZE(llvm->tes_jit_context.constants); ++i) {
507 int num_consts = get_num_consts_robust(draw, draw->pt.user.tes_constants_size, i);
508 llvm->tes_jit_context.constants[i] = draw->pt.user.tes_constants[i];
509 llvm->tes_jit_context.num_constants[i] = num_consts;
510 if (num_consts == 0) {
511 llvm->tes_jit_context.constants[i] = fake_const_buf;
512 }
513 }
514 for (i = 0; i < ARRAY_SIZE(llvm->tes_jit_context.ssbos); ++i) {
515 int num_ssbos = draw->pt.user.tes_ssbos_size[i];
516 llvm->tes_jit_context.ssbos[i] = draw->pt.user.tes_ssbos[i];
517 llvm->tes_jit_context.num_ssbos[i] = num_ssbos;
518 if (num_ssbos == 0) {
519 llvm->tes_jit_context.ssbos[i] = (const uint32_t *)fake_const_buf;
520 }
521 }
522
523 llvm->jit_context.planes =
524 (float (*)[DRAW_TOTAL_CLIP_PLANES][4]) draw->pt.user.planes[0];
525 llvm->gs_jit_context.planes =
526 (float (*)[DRAW_TOTAL_CLIP_PLANES][4]) draw->pt.user.planes[0];
527
528 llvm->jit_context.viewports = draw->viewports;
529 llvm->gs_jit_context.viewports = draw->viewports;
530 }
531
532
533 static void
pipeline(struct llvm_middle_end * llvm,const struct draw_vertex_info * vert_info,const struct draw_prim_info * prim_info)534 pipeline(struct llvm_middle_end *llvm,
535 const struct draw_vertex_info *vert_info,
536 const struct draw_prim_info *prim_info)
537 {
538 if (prim_info->linear)
539 draw_pipeline_run_linear( llvm->draw,
540 vert_info,
541 prim_info);
542 else
543 draw_pipeline_run( llvm->draw,
544 vert_info,
545 prim_info );
546 }
547
548
549 static void
emit(struct pt_emit * emit,const struct draw_vertex_info * vert_info,const struct draw_prim_info * prim_info)550 emit(struct pt_emit *emit,
551 const struct draw_vertex_info *vert_info,
552 const struct draw_prim_info *prim_info)
553 {
554 if (prim_info->linear) {
555 draw_pt_emit_linear(emit, vert_info, prim_info);
556 }
557 else {
558 draw_pt_emit(emit, vert_info, prim_info);
559 }
560 }
561
562
563 static void
llvm_pipeline_generic(struct draw_pt_middle_end * middle,const struct draw_fetch_info * fetch_info,const struct draw_prim_info * in_prim_info)564 llvm_pipeline_generic(struct draw_pt_middle_end *middle,
565 const struct draw_fetch_info *fetch_info,
566 const struct draw_prim_info *in_prim_info)
567 {
568 struct llvm_middle_end *fpme = llvm_middle_end(middle);
569 struct draw_context *draw = fpme->draw;
570 struct draw_geometry_shader *gshader = draw->gs.geometry_shader;
571 struct draw_tess_ctrl_shader *tcs_shader = draw->tcs.tess_ctrl_shader;
572 struct draw_tess_eval_shader *tes_shader = draw->tes.tess_eval_shader;
573 struct draw_prim_info tcs_prim_info;
574 struct draw_prim_info tes_prim_info;
575 struct draw_prim_info gs_prim_info[TGSI_MAX_VERTEX_STREAMS];
576 struct draw_vertex_info llvm_vert_info;
577 struct draw_vertex_info tcs_vert_info;
578 struct draw_vertex_info tes_vert_info;
579 struct draw_vertex_info gs_vert_info[TGSI_MAX_VERTEX_STREAMS];
580 struct draw_vertex_info *vert_info;
581 struct draw_prim_info ia_prim_info;
582 struct draw_vertex_info ia_vert_info;
583 const struct draw_prim_info *prim_info = in_prim_info;
584 boolean free_prim_info = FALSE;
585 unsigned opt = fpme->opt;
586 boolean clipped = 0;
587 unsigned start_or_maxelt, vid_base;
588 const unsigned *elts;
589 ushort *tes_elts_out = NULL;
590
591 memset(&gs_vert_info, 0, sizeof(struct draw_vertex_info) * TGSI_MAX_VERTEX_STREAMS);
592 assert(fetch_info->count > 0);
593 llvm_vert_info.count = fetch_info->count;
594 llvm_vert_info.vertex_size = fpme->vertex_size;
595 llvm_vert_info.stride = fpme->vertex_size;
596 llvm_vert_info.verts = (struct vertex_header *)
597 MALLOC(fpme->vertex_size *
598 align(fetch_info->count, lp_native_vector_width / 32));
599 if (!llvm_vert_info.verts) {
600 assert(0);
601 return;
602 }
603
604 if (draw->collect_statistics) {
605 draw->statistics.ia_vertices += prim_info->count;
606 if (prim_info->prim == PIPE_PRIM_PATCHES)
607 draw->statistics.ia_primitives += prim_info->count / draw->pt.vertices_per_patch;
608 else
609 draw->statistics.ia_primitives +=
610 u_decomposed_prims_for_vertices(prim_info->prim, prim_info->count);
611 draw->statistics.vs_invocations += fetch_info->count;
612 }
613
614 if (fetch_info->linear) {
615 start_or_maxelt = fetch_info->start;
616 vid_base = draw->start_index;
617 elts = NULL;
618 }
619 else {
620 start_or_maxelt = draw->pt.user.eltMax;
621 vid_base = draw->pt.user.eltBias;
622 elts = fetch_info->elts;
623 }
624 clipped = fpme->current_variant->jit_func(&fpme->llvm->jit_context,
625 llvm_vert_info.verts,
626 draw->pt.user.vbuffer,
627 fetch_info->count,
628 start_or_maxelt,
629 fpme->vertex_size,
630 draw->pt.vertex_buffer,
631 draw->instance_id,
632 vid_base,
633 draw->start_instance,
634 elts, draw->pt.user.drawid);
635
636 /* Finished with fetch and vs:
637 */
638 fetch_info = NULL;
639 vert_info = &llvm_vert_info;
640
641 if (opt & PT_SHADE) {
642 struct draw_vertex_shader *vshader = draw->vs.vertex_shader;
643 if (tcs_shader) {
644 draw_tess_ctrl_shader_run(tcs_shader,
645 draw->pt.user.tcs_constants,
646 draw->pt.user.tcs_constants_size,
647 vert_info,
648 prim_info,
649 &vshader->info,
650 &tcs_vert_info,
651 &tcs_prim_info);
652 FREE(vert_info->verts);
653 vert_info = &tcs_vert_info;
654 prim_info = &tcs_prim_info;
655
656 } else if (tes_shader) {
657 unsigned num_prims = prim_info->count / draw->pt.vertices_per_patch;
658 tcs_prim_info = *prim_info;
659 tcs_prim_info.primitive_count = num_prims;
660 prim_info = &tcs_prim_info;
661 }
662
663 if (tes_shader) {
664 draw_tess_eval_shader_run(tes_shader,
665 draw->pt.user.tes_constants,
666 draw->pt.user.tes_constants_size,
667 tcs_shader ? tcs_shader->vertices_out : draw->pt.vertices_per_patch,
668 vert_info,
669 prim_info,
670 tcs_shader ? &tcs_shader->info : &vshader->info,
671 &tes_vert_info,
672 &tes_prim_info, &tes_elts_out);
673
674 FREE(vert_info->verts);
675 vert_info = &tes_vert_info;
676 prim_info = &tes_prim_info;
677 free_prim_info = TRUE;
678
679 /*
680 * pt emit can only handle ushort number of vertices (see
681 * render->allocate_vertices).
682 * vsplit guarantees there's never more than 4096, however GS can
683 * easily blow this up (by a factor of 256 (or even 1024) max).
684 */
685 if (vert_info->count > 65535) {
686 opt |= PT_PIPELINE;
687 }
688 }
689 }
690
691 if ((opt & PT_SHADE) && gshader) {
692 struct draw_vertex_shader *vshader = draw->vs.vertex_shader;
693 draw_geometry_shader_run(gshader,
694 draw->pt.user.gs_constants,
695 draw->pt.user.gs_constants_size,
696 vert_info,
697 prim_info,
698 tes_shader ? &tes_shader->info : &vshader->info,
699 gs_vert_info,
700 gs_prim_info);
701
702 FREE(vert_info->verts);
703 if (free_prim_info) {
704 FREE(prim_info->primitive_lengths);
705 FREE(tes_elts_out);
706 }
707 vert_info = &gs_vert_info[0];
708 prim_info = &gs_prim_info[0];
709 free_prim_info = FALSE;
710 /*
711 * pt emit can only handle ushort number of vertices (see
712 * render->allocate_vertices).
713 * vsplit guarantees there's never more than 4096, however GS can
714 * easily blow this up (by a factor of 256 (or even 1024) max).
715 */
716 if (vert_info->count > 65535) {
717 opt |= PT_PIPELINE;
718 }
719 } else {
720 if (draw_prim_assembler_is_required(draw, prim_info, vert_info)) {
721 draw_prim_assembler_run(draw, prim_info, vert_info,
722 &ia_prim_info, &ia_vert_info);
723
724 if (ia_vert_info.count) {
725 FREE(vert_info->verts);
726 vert_info = &ia_vert_info;
727 prim_info = &ia_prim_info;
728 free_prim_info = TRUE;
729 }
730 }
731 }
732
733 /* stream output needs to be done before clipping */
734 draw_pt_so_emit( fpme->so_emit, gshader ? gshader->num_vertex_streams : 1, vert_info, prim_info );
735
736 if (prim_info->count == 0) {
737 debug_printf("GS/IA didn't emit any vertices!\n");
738 goto out;
739 }
740
741 draw_stats_clipper_primitives(draw, prim_info);
742
743 /*
744 * if there's no position, need to stop now, or the latter stages
745 * will try to access non-existent position output.
746 */
747 if (draw_current_shader_position_output(draw) != -1) {
748 if ((opt & PT_SHADE) && (gshader || tes_shader ||
749 draw->vs.vertex_shader->info.writes_viewport_index)) {
750 clipped = draw_pt_post_vs_run( fpme->post_vs, vert_info, prim_info );
751 }
752 /* "clipped" also includes non-one edgeflag */
753 if (clipped) {
754 opt |= PT_PIPELINE;
755 }
756
757 /* Do we need to run the pipeline? Now will come here if clipped
758 */
759 if (opt & PT_PIPELINE) {
760 pipeline( fpme, vert_info, prim_info );
761 }
762 else {
763 emit( fpme->emit, vert_info, prim_info );
764 }
765 }
766 out:
767 FREE(vert_info->verts);
768 if (gshader && gshader->num_vertex_streams > 1)
769 for (unsigned i = 1; i < gshader->num_vertex_streams; i++)
770 FREE(gs_vert_info[i].verts);
771
772 if (free_prim_info) {
773 FREE(tes_elts_out);
774 FREE(prim_info->primitive_lengths);
775 }
776 }
777
778
779 static inline unsigned
prim_type(unsigned prim,unsigned flags)780 prim_type(unsigned prim, unsigned flags)
781 {
782 if (flags & DRAW_LINE_LOOP_AS_STRIP)
783 return PIPE_PRIM_LINE_STRIP;
784 else
785 return prim;
786 }
787
788
789 static void
llvm_middle_end_run(struct draw_pt_middle_end * middle,const unsigned * fetch_elts,unsigned fetch_count,const ushort * draw_elts,unsigned draw_count,unsigned prim_flags)790 llvm_middle_end_run(struct draw_pt_middle_end *middle,
791 const unsigned *fetch_elts,
792 unsigned fetch_count,
793 const ushort *draw_elts,
794 unsigned draw_count,
795 unsigned prim_flags)
796 {
797 struct llvm_middle_end *fpme = llvm_middle_end(middle);
798 struct draw_fetch_info fetch_info;
799 struct draw_prim_info prim_info;
800
801 fetch_info.linear = FALSE;
802 fetch_info.start = 0;
803 fetch_info.elts = fetch_elts;
804 fetch_info.count = fetch_count;
805
806 prim_info.linear = FALSE;
807 prim_info.start = 0;
808 prim_info.count = draw_count;
809 prim_info.elts = draw_elts;
810 prim_info.prim = prim_type(fpme->input_prim, prim_flags);
811 prim_info.flags = prim_flags;
812 prim_info.primitive_count = 1;
813 prim_info.primitive_lengths = &draw_count;
814
815 llvm_pipeline_generic( middle, &fetch_info, &prim_info );
816 }
817
818
819 static void
llvm_middle_end_linear_run(struct draw_pt_middle_end * middle,unsigned start,unsigned count,unsigned prim_flags)820 llvm_middle_end_linear_run(struct draw_pt_middle_end *middle,
821 unsigned start,
822 unsigned count,
823 unsigned prim_flags)
824 {
825 struct llvm_middle_end *fpme = llvm_middle_end(middle);
826 struct draw_fetch_info fetch_info;
827 struct draw_prim_info prim_info;
828
829 fetch_info.linear = TRUE;
830 fetch_info.start = start;
831 fetch_info.count = count;
832 fetch_info.elts = NULL;
833
834 prim_info.linear = TRUE;
835 prim_info.start = 0;
836 prim_info.count = count;
837 prim_info.elts = NULL;
838 prim_info.prim = prim_type(fpme->input_prim, prim_flags);
839 prim_info.flags = prim_flags;
840 prim_info.primitive_count = 1;
841 prim_info.primitive_lengths = &count;
842
843 llvm_pipeline_generic( middle, &fetch_info, &prim_info );
844 }
845
846
847 static boolean
llvm_middle_end_linear_run_elts(struct draw_pt_middle_end * middle,unsigned start,unsigned count,const ushort * draw_elts,unsigned draw_count,unsigned prim_flags)848 llvm_middle_end_linear_run_elts(struct draw_pt_middle_end *middle,
849 unsigned start,
850 unsigned count,
851 const ushort *draw_elts,
852 unsigned draw_count,
853 unsigned prim_flags)
854 {
855 struct llvm_middle_end *fpme = llvm_middle_end(middle);
856 struct draw_fetch_info fetch_info;
857 struct draw_prim_info prim_info;
858
859 fetch_info.linear = TRUE;
860 fetch_info.start = start;
861 fetch_info.count = count;
862 fetch_info.elts = NULL;
863
864 prim_info.linear = FALSE;
865 prim_info.start = 0;
866 prim_info.count = draw_count;
867 prim_info.elts = draw_elts;
868 prim_info.prim = prim_type(fpme->input_prim, prim_flags);
869 prim_info.flags = prim_flags;
870 prim_info.primitive_count = 1;
871 prim_info.primitive_lengths = &draw_count;
872
873 llvm_pipeline_generic( middle, &fetch_info, &prim_info );
874
875 return TRUE;
876 }
877
878
879 static void
llvm_middle_end_finish(struct draw_pt_middle_end * middle)880 llvm_middle_end_finish(struct draw_pt_middle_end *middle)
881 {
882 /* nothing to do */
883 }
884
885
886 static void
llvm_middle_end_destroy(struct draw_pt_middle_end * middle)887 llvm_middle_end_destroy(struct draw_pt_middle_end *middle)
888 {
889 struct llvm_middle_end *fpme = llvm_middle_end(middle);
890
891 if (fpme->fetch)
892 draw_pt_fetch_destroy( fpme->fetch );
893
894 if (fpme->emit)
895 draw_pt_emit_destroy( fpme->emit );
896
897 if (fpme->so_emit)
898 draw_pt_so_emit_destroy( fpme->so_emit );
899
900 if (fpme->post_vs)
901 draw_pt_post_vs_destroy( fpme->post_vs );
902
903 FREE(middle);
904 }
905
906
907 struct draw_pt_middle_end *
draw_pt_fetch_pipeline_or_emit_llvm(struct draw_context * draw)908 draw_pt_fetch_pipeline_or_emit_llvm(struct draw_context *draw)
909 {
910 struct llvm_middle_end *fpme = 0;
911
912 if (!draw->llvm)
913 return NULL;
914
915 fpme = CALLOC_STRUCT( llvm_middle_end );
916 if (!fpme)
917 goto fail;
918
919 fpme->base.prepare = llvm_middle_end_prepare;
920 fpme->base.bind_parameters = llvm_middle_end_bind_parameters;
921 fpme->base.run = llvm_middle_end_run;
922 fpme->base.run_linear = llvm_middle_end_linear_run;
923 fpme->base.run_linear_elts = llvm_middle_end_linear_run_elts;
924 fpme->base.finish = llvm_middle_end_finish;
925 fpme->base.destroy = llvm_middle_end_destroy;
926
927 fpme->draw = draw;
928
929 fpme->fetch = draw_pt_fetch_create( draw );
930 if (!fpme->fetch)
931 goto fail;
932
933 fpme->post_vs = draw_pt_post_vs_create( draw );
934 if (!fpme->post_vs)
935 goto fail;
936
937 fpme->emit = draw_pt_emit_create( draw );
938 if (!fpme->emit)
939 goto fail;
940
941 fpme->so_emit = draw_pt_so_emit_create( draw );
942 if (!fpme->so_emit)
943 goto fail;
944
945 fpme->llvm = draw->llvm;
946 if (!fpme->llvm)
947 goto fail;
948
949 fpme->current_variant = NULL;
950
951 return &fpme->base;
952
953 fail:
954 if (fpme)
955 llvm_middle_end_destroy( &fpme->base );
956
957 return NULL;
958 }
959