1 /**************************************************************************
2 *
3 * Copyright 2010 VMware, Inc.
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
21 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27
28 #include "util/u_math.h"
29 #include "util/u_memory.h"
30 #include "util/u_prim.h"
31 #include "draw/draw_context.h"
32 #include "draw/draw_gs.h"
33 #include "draw/draw_tess.h"
34 #include "draw/draw_vbuf.h"
35 #include "draw/draw_vertex.h"
36 #include "draw/draw_pt.h"
37 #include "draw/draw_prim_assembler.h"
38 #include "draw/draw_vs.h"
39 #include "draw/draw_llvm.h"
40 #include "gallivm/lp_bld_init.h"
41 #include "gallivm/lp_bld_debug.h"
42
43
44 struct llvm_middle_end {
45 struct draw_pt_middle_end base;
46 struct draw_context *draw;
47
48 struct pt_emit *emit;
49 struct pt_so_emit *so_emit;
50 struct pt_fetch *fetch;
51 struct pt_post_vs *post_vs;
52
53
54 unsigned vertex_data_offset;
55 unsigned vertex_size;
56 enum mesa_prim input_prim;
57 unsigned opt;
58
59 struct draw_llvm *llvm;
60 struct draw_llvm_variant *current_variant;
61 };
62
63
64 /** cast wrapper */
65 static inline struct llvm_middle_end *
llvm_middle_end(struct draw_pt_middle_end * middle)66 llvm_middle_end(struct draw_pt_middle_end *middle)
67 {
68 return (struct llvm_middle_end *) middle;
69 }
70
71
72 static void
llvm_middle_end_prepare_gs(struct llvm_middle_end * fpme)73 llvm_middle_end_prepare_gs(struct llvm_middle_end *fpme)
74 {
75 struct draw_context *draw = fpme->draw;
76 struct draw_llvm *llvm = fpme->llvm;
77 struct draw_geometry_shader *gs = draw->gs.geometry_shader;
78 struct draw_gs_llvm_variant_list_item *li;
79 struct llvm_geometry_shader *shader = llvm_geometry_shader(gs);
80 char store[DRAW_GS_LLVM_MAX_VARIANT_KEY_SIZE];
81 struct draw_gs_llvm_variant_key *key = draw_gs_llvm_make_variant_key(llvm, store);
82
83 /* Search shader's list of variants for the key */
84 struct draw_gs_llvm_variant *variant = NULL;
85 LIST_FOR_EACH_ENTRY(li, &shader->variants.list, list) {
86 if (memcmp(&li->base->key, key, shader->variant_key_size) == 0) {
87 variant = li->base;
88 break;
89 }
90 }
91
92 if (variant) {
93 /* found the variant, move to head of global list (for LRU) */
94 list_move_to(&variant->list_item_global.list, &llvm->gs_variants_list.list);
95 } else {
96 /* Need to create new variant */
97
98 /* First check if we've created too many variants. If so, free
99 * 3.125% of the LRU to avoid using too much memory.
100 */
101 if (llvm->nr_gs_variants >= DRAW_MAX_SHADER_VARIANTS) {
102 if (gallivm_debug & GALLIVM_DEBUG_PERF) {
103 debug_printf("Evicting GS: %u gs variants,\t%u total variants\n",
104 shader->variants_cached, llvm->nr_gs_variants);
105 }
106
107 /*
108 * XXX: should we flush here ?
109 */
110 struct draw_gs_llvm_variant_list_item *item;
111 for (unsigned i = 0; i < DRAW_MAX_SHADER_VARIANTS / 32; i++) {
112 if (list_is_empty(&llvm->gs_variants_list.list)) {
113 break;
114 }
115 item = list_last_entry(&llvm->gs_variants_list.list,
116 struct draw_gs_llvm_variant_list_item, list);
117 assert(item);
118 assert(item->base);
119 draw_gs_llvm_destroy_variant(item->base);
120 }
121 }
122
123 variant = draw_gs_llvm_create_variant(llvm, draw_total_gs_outputs(draw), key);
124
125 if (variant) {
126 list_add(&variant->list_item_local.list, &shader->variants.list);
127 list_add(&variant->list_item_global.list, &llvm->gs_variants_list.list);
128 llvm->nr_gs_variants++;
129 shader->variants_cached++;
130 }
131 }
132
133 gs->current_variant = variant;
134 }
135
136
137 static void
llvm_middle_end_prepare_tcs(struct llvm_middle_end * fpme)138 llvm_middle_end_prepare_tcs(struct llvm_middle_end *fpme)
139 {
140 struct draw_context *draw = fpme->draw;
141 struct draw_llvm *llvm = fpme->llvm;
142 struct draw_tess_ctrl_shader *tcs = draw->tcs.tess_ctrl_shader;
143 struct draw_tcs_llvm_variant_list_item *li;
144 struct llvm_tess_ctrl_shader *shader = llvm_tess_ctrl_shader(tcs);
145 char store[DRAW_TCS_LLVM_MAX_VARIANT_KEY_SIZE];
146 const struct draw_tcs_llvm_variant_key *key =
147 draw_tcs_llvm_make_variant_key(llvm, store);
148
149 /* Search shader's list of variants for the key */
150 struct draw_tcs_llvm_variant *variant = NULL;
151 LIST_FOR_EACH_ENTRY(li, &shader->variants.list, list) {
152 if (memcmp(&li->base->key, key, shader->variant_key_size) == 0) {
153 variant = li->base;
154 break;
155 }
156 }
157
158 if (variant) {
159 /* found the variant, move to head of global list (for LRU) */
160 list_move_to(&variant->list_item_global.list,
161 &llvm->tcs_variants_list.list);
162 } else {
163 /* Need to create new variant */
164
165 /* First check if we've created too many variants. If so, free
166 * 3.125% of the LRU to avoid using too much memory.
167 */
168 if (llvm->nr_tcs_variants >= DRAW_MAX_SHADER_VARIANTS) {
169 if (gallivm_debug & GALLIVM_DEBUG_PERF) {
170 debug_printf("Evicting TCS: %u tcs variants,\t%u total variants\n",
171 shader->variants_cached, llvm->nr_tcs_variants);
172 }
173
174 /*
175 * XXX: should we flush here ?
176 */
177 for (unsigned i = 0; i < DRAW_MAX_SHADER_VARIANTS / 32; i++) {
178 struct draw_tcs_llvm_variant_list_item *item;
179 if (list_is_empty(&llvm->tcs_variants_list.list)) {
180 break;
181 }
182 item = list_last_entry(&llvm->tcs_variants_list.list,
183 struct draw_tcs_llvm_variant_list_item, list);
184 assert(item);
185 assert(item->base);
186 draw_tcs_llvm_destroy_variant(item->base);
187 }
188 }
189
190 variant = draw_tcs_llvm_create_variant(llvm, 0, key);
191
192 if (variant) {
193 list_add(&variant->list_item_local.list, &shader->variants.list);
194 list_add(&variant->list_item_global.list, &llvm->tcs_variants_list.list);
195 llvm->nr_tcs_variants++;
196 shader->variants_cached++;
197 }
198 }
199
200 tcs->current_variant = variant;
201 }
202
203
204 static void
llvm_middle_end_prepare_tes(struct llvm_middle_end * fpme)205 llvm_middle_end_prepare_tes(struct llvm_middle_end *fpme)
206 {
207 struct draw_context *draw = fpme->draw;
208 struct draw_llvm *llvm = fpme->llvm;
209 struct draw_tess_eval_shader *tes = draw->tes.tess_eval_shader;
210 struct draw_tes_llvm_variant *variant = NULL;
211 struct draw_tes_llvm_variant_list_item *li;
212 struct llvm_tess_eval_shader *shader = llvm_tess_eval_shader(tes);
213 char store[DRAW_TES_LLVM_MAX_VARIANT_KEY_SIZE];
214 const struct draw_tes_llvm_variant_key *key =
215 draw_tes_llvm_make_variant_key(llvm, store);
216
217 /* Search shader's list of variants for the key */
218 LIST_FOR_EACH_ENTRY(li, &shader->variants.list, list) {
219 if (memcmp(&li->base->key, key, shader->variant_key_size) == 0) {
220 variant = li->base;
221 break;
222 }
223 }
224
225 if (variant) {
226 /* found the variant, move to head of global list (for LRU) */
227 list_move_to(&variant->list_item_global.list,
228 &llvm->tes_variants_list.list);
229 } else {
230 /* Need to create new variant */
231
232 /* First check if we've created too many variants. If so, free
233 * 3.125% of the LRU to avoid using too much memory.
234 */
235 if (llvm->nr_tes_variants >= DRAW_MAX_SHADER_VARIANTS) {
236 if (gallivm_debug & GALLIVM_DEBUG_PERF) {
237 debug_printf("Evicting TES: %u tes variants,\t%u total variants\n",
238 shader->variants_cached, llvm->nr_tes_variants);
239 }
240
241 /*
242 * XXX: should we flush here ?
243 */
244 for (unsigned i = 0; i < DRAW_MAX_SHADER_VARIANTS / 32; i++) {
245 struct draw_tes_llvm_variant_list_item *item;
246 if (list_is_empty(&llvm->tes_variants_list.list)) {
247 break;
248 }
249 item = list_last_entry(&llvm->tes_variants_list.list,
250 struct draw_tes_llvm_variant_list_item, list);
251 assert(item);
252 assert(item->base);
253 draw_tes_llvm_destroy_variant(item->base);
254 }
255 }
256
257 variant = draw_tes_llvm_create_variant(llvm, draw_total_tes_outputs(draw), key);
258
259 if (variant) {
260 list_add(&variant->list_item_local.list, &shader->variants.list);
261 list_add(&variant->list_item_global.list, &llvm->tes_variants_list.list);
262 llvm->nr_tes_variants++;
263 shader->variants_cached++;
264 }
265 }
266
267 tes->current_variant = variant;
268 }
269
270
271 /**
272 * Prepare/validate middle part of the vertex pipeline.
273 * NOTE: if you change this function, also look at the non-LLVM
274 * function fetch_pipeline_prepare() for similar changes.
275 */
276 static void
llvm_middle_end_prepare(struct draw_pt_middle_end * middle,enum mesa_prim in_prim,unsigned opt,unsigned * max_vertices)277 llvm_middle_end_prepare(struct draw_pt_middle_end *middle,
278 enum mesa_prim in_prim,
279 unsigned opt,
280 unsigned *max_vertices)
281 {
282 struct llvm_middle_end *fpme = llvm_middle_end(middle);
283 struct draw_context *draw = fpme->draw;
284 struct draw_llvm *llvm = fpme->llvm;
285 struct draw_vertex_shader *vs = draw->vs.vertex_shader;
286 struct draw_geometry_shader *gs = draw->gs.geometry_shader;
287 struct draw_tess_ctrl_shader *tcs = draw->tcs.tess_ctrl_shader;
288 struct draw_tess_eval_shader *tes = draw->tes.tess_eval_shader;
289 const enum mesa_prim out_prim =
290 gs ? gs->output_primitive : tes ? get_tes_output_prim(tes) :
291 u_assembled_prim(in_prim);
292 unsigned point_line_clip = draw->rasterizer->fill_front == PIPE_POLYGON_MODE_POINT ||
293 draw->rasterizer->fill_front == PIPE_POLYGON_MODE_LINE ||
294 out_prim == MESA_PRIM_POINTS ||
295 u_reduced_prim(out_prim) == MESA_PRIM_LINES;
296
297 fpme->input_prim = in_prim;
298 fpme->opt = opt;
299
300 draw_pt_post_vs_prepare(fpme->post_vs,
301 draw->clip_xy,
302 draw->clip_z,
303 draw->clip_user,
304 point_line_clip ? draw->guard_band_points_lines_xy :
305 draw->guard_band_xy,
306 draw->bypass_viewport,
307 draw->rasterizer->clip_halfz,
308 (draw->vs.edgeflag_output ? true : false));
309
310 draw_pt_so_emit_prepare(fpme->so_emit, (gs == NULL && tes == NULL));
311
312 if (!(opt & PT_PIPELINE)) {
313 draw_pt_emit_prepare(fpme->emit, out_prim, max_vertices);
314
315 *max_vertices = MAX2(*max_vertices, 4096);
316 } else {
317 /* limit max fetches by limiting max_vertices */
318 *max_vertices = 4096;
319 }
320
321 /* Get the number of float[4] attributes per vertex.
322 * Note: this must be done after draw_pt_emit_prepare() since that
323 * can effect the vertex size.
324 */
325 unsigned nr = MAX2(vs->info.num_inputs, draw_total_vs_outputs(draw));
326
327 /* Always leave room for the vertex header whether we need it or
328 * not. It's hard to get rid of it in particular because of the
329 * viewport code in draw_pt_post_vs.c.
330 */
331 fpme->vertex_size = sizeof(struct vertex_header) + nr * 4 * sizeof(float);
332
333 /* return even number */
334 *max_vertices = *max_vertices & ~1;
335
336 /* Find/create the vertex shader variant */
337 {
338 struct draw_llvm_variant *variant = NULL;
339 struct draw_llvm_variant_list_item *li;
340 struct llvm_vertex_shader *shader = llvm_vertex_shader(vs);
341 char store[DRAW_LLVM_MAX_VARIANT_KEY_SIZE];
342 struct draw_llvm_variant_key *key = draw_llvm_make_variant_key(llvm, store);
343
344 /* Search shader's list of variants for the key */
345 LIST_FOR_EACH_ENTRY(li, &shader->variants.list, list) {
346 if (memcmp(&li->base->key, key, shader->variant_key_size) == 0) {
347 variant = li->base;
348 break;
349 }
350 }
351
352 if (variant) {
353 /* found the variant, move to head of global list (for LRU) */
354 list_move_to(&variant->list_item_global.list, &llvm->vs_variants_list.list);
355 } else {
356 /* Need to create new variant */
357
358 /* First check if we've created too many variants. If so, free
359 * 3.125% of the LRU to avoid using too much memory.
360 */
361 if (llvm->nr_variants >= DRAW_MAX_SHADER_VARIANTS) {
362 if (gallivm_debug & GALLIVM_DEBUG_PERF) {
363 debug_printf("Evicting VS: %u vs variants,\t%u total variants\n",
364 shader->variants_cached, llvm->nr_variants);
365 }
366
367 /*
368 * XXX: should we flush here ?
369 */
370 for (unsigned i = 0; i < DRAW_MAX_SHADER_VARIANTS / 32; i++) {
371 struct draw_llvm_variant_list_item *item;
372 if (list_is_empty(&llvm->vs_variants_list.list)) {
373 break;
374 }
375 item = list_last_entry(&llvm->vs_variants_list.list,
376 struct draw_llvm_variant_list_item, list);
377 assert(item);
378 assert(item->base);
379 draw_llvm_destroy_variant(item->base);
380 }
381 }
382
383 variant = draw_llvm_create_variant(llvm, nr, key);
384
385 if (variant) {
386 list_add(&variant->list_item_local.list, &shader->variants.list);
387 list_add(&variant->list_item_global.list, &llvm->vs_variants_list.list);
388 llvm->nr_variants++;
389 shader->variants_cached++;
390 }
391 }
392
393 fpme->current_variant = variant;
394 }
395
396 if (gs) {
397 llvm_middle_end_prepare_gs(fpme);
398 }
399 if (tcs) {
400 llvm_middle_end_prepare_tcs(fpme);
401 }
402 if (tes) {
403 llvm_middle_end_prepare_tes(fpme);
404 }
405 }
406
407
408 static unsigned
get_num_consts_robust(struct draw_context * draw,struct draw_buffer_info * bufs,unsigned idx)409 get_num_consts_robust(struct draw_context *draw, struct draw_buffer_info *bufs, unsigned idx)
410 {
411 uint64_t const_bytes = bufs[idx].size;
412
413 if (const_bytes < sizeof(float))
414 return 0;
415
416 return DIV_ROUND_UP(const_bytes, draw->constant_buffer_stride);
417 }
418
419 /**
420 * Bind/update constant buffer pointers, clip planes and viewport dims.
421 * These are "light weight" parameters which aren't baked into the
422 * generated code. Updating these items is much cheaper than revalidating
423 * and rebuilding the generated pipeline code.
424 */
425 static void
llvm_middle_end_bind_parameters(struct draw_pt_middle_end * middle)426 llvm_middle_end_bind_parameters(struct draw_pt_middle_end *middle)
427 {
428 static const float fake_const_buf[4];
429 struct llvm_middle_end *fpme = llvm_middle_end(middle);
430 struct draw_context *draw = fpme->draw;
431 struct draw_llvm *llvm = fpme->llvm;
432 unsigned i;
433
434 for (enum pipe_shader_type shader_type = PIPE_SHADER_VERTEX; shader_type <= PIPE_SHADER_GEOMETRY; shader_type++) {
435 for (i = 0; i < ARRAY_SIZE(llvm->jit_resources[shader_type].constants); ++i) {
436 /*
437 * There could be a potential issue with rounding this up, as the
438 * shader expects 16-byte allocations, the fix is likely to move
439 * to LOAD intrinsic in the future and remove the vec4 constraint.
440 */
441 int num_consts = get_num_consts_robust(draw, draw->pt.user.constants[shader_type], i);
442 llvm->jit_resources[shader_type].constants[i].f = draw->pt.user.constants[shader_type][i].ptr;
443 llvm->jit_resources[shader_type].constants[i].num_elements = num_consts;
444 if (num_consts == 0) {
445 llvm->jit_resources[shader_type].constants[i].f = fake_const_buf;
446 }
447 }
448 for (i = 0; i < ARRAY_SIZE(llvm->jit_resources[shader_type].ssbos); ++i) {
449 int num_ssbos = draw->pt.user.ssbos[shader_type][i].size;
450 llvm->jit_resources[shader_type].ssbos[i].u = draw->pt.user.ssbos[shader_type][i].ptr;
451 llvm->jit_resources[shader_type].ssbos[i].num_elements = num_ssbos;
452 if (num_ssbos == 0) {
453 llvm->jit_resources[shader_type].ssbos[i].u = (const uint32_t *)fake_const_buf;
454 }
455 }
456 }
457
458 llvm->vs_jit_context.planes =
459 (float (*)[DRAW_TOTAL_CLIP_PLANES][4]) draw->pt.user.planes[0];
460 llvm->gs_jit_context.planes =
461 (float (*)[DRAW_TOTAL_CLIP_PLANES][4]) draw->pt.user.planes[0];
462
463 llvm->vs_jit_context.viewports = draw->viewports;
464 llvm->gs_jit_context.viewports = draw->viewports;
465 }
466
467
468 static void
pipeline(struct llvm_middle_end * llvm,const struct draw_vertex_info * vert_info,const struct draw_prim_info * prim_info)469 pipeline(struct llvm_middle_end *llvm,
470 const struct draw_vertex_info *vert_info,
471 const struct draw_prim_info *prim_info)
472 {
473 if (prim_info->linear)
474 draw_pipeline_run_linear(llvm->draw, vert_info, prim_info);
475 else
476 draw_pipeline_run(llvm->draw, vert_info, prim_info);
477 }
478
479
480 static void
emit(struct pt_emit * emit,const struct draw_vertex_info * vert_info,const struct draw_prim_info * prim_info)481 emit(struct pt_emit *emit,
482 const struct draw_vertex_info *vert_info,
483 const struct draw_prim_info *prim_info)
484 {
485 if (prim_info->linear)
486 draw_pt_emit_linear(emit, vert_info, prim_info);
487 else
488 draw_pt_emit(emit, vert_info, prim_info);
489 }
490
491
492 static void
llvm_pipeline_generic(struct draw_pt_middle_end * middle,const struct draw_fetch_info * fetch_info,const struct draw_prim_info * in_prim_info)493 llvm_pipeline_generic(struct draw_pt_middle_end *middle,
494 const struct draw_fetch_info *fetch_info,
495 const struct draw_prim_info *in_prim_info)
496 {
497 struct llvm_middle_end *fpme = llvm_middle_end(middle);
498 struct draw_context *draw = fpme->draw;
499 struct draw_geometry_shader *gshader = draw->gs.geometry_shader;
500 struct draw_tess_ctrl_shader *tcs_shader = draw->tcs.tess_ctrl_shader;
501 struct draw_tess_eval_shader *tes_shader = draw->tes.tess_eval_shader;
502 struct draw_prim_info tcs_prim_info;
503 struct draw_prim_info tes_prim_info;
504 struct draw_prim_info gs_prim_info[TGSI_MAX_VERTEX_STREAMS];
505 struct draw_vertex_info llvm_vert_info;
506 struct draw_vertex_info tcs_vert_info;
507 struct draw_vertex_info tes_vert_info;
508 struct draw_vertex_info *vert_info;
509 struct draw_prim_info ia_prim_info;
510 struct draw_vertex_info ia_vert_info;
511 const struct draw_prim_info *prim_info = in_prim_info;
512 bool free_prim_info = false;
513 unsigned opt = fpme->opt;
514 bool clipped = 0;
515 uint16_t *tes_elts_out = NULL;
516
517 assert(fetch_info->count > 0);
518
519 llvm_vert_info.count = fetch_info->count;
520 llvm_vert_info.vertex_size = fpme->vertex_size;
521 llvm_vert_info.stride = fpme->vertex_size;
522 llvm_vert_info.verts = (struct vertex_header *)
523 MALLOC(fpme->vertex_size *
524 align(fetch_info->count, lp_native_vector_width / 32) +
525 DRAW_EXTRA_VERTICES_PADDING);
526 if (!llvm_vert_info.verts) {
527 assert(0);
528 return;
529 }
530
531 if (draw->collect_statistics) {
532 draw->statistics.ia_vertices += prim_info->count;
533 if (prim_info->prim == MESA_PRIM_PATCHES)
534 draw->statistics.ia_primitives +=
535 prim_info->count / draw->pt.vertices_per_patch;
536 else
537 draw->statistics.ia_primitives +=
538 u_decomposed_prims_for_vertices(prim_info->prim, prim_info->count);
539 draw->statistics.vs_invocations += fetch_info->count;
540 }
541
542 {
543 unsigned start, vertex_id_offset;
544 const unsigned *elts;
545
546 if (fetch_info->linear) {
547 start = fetch_info->start;
548 vertex_id_offset = draw->start_index;
549 elts = NULL;
550 } else {
551 start = draw->pt.user.eltMax;
552 vertex_id_offset = draw->pt.user.eltBias;
553 elts = fetch_info->elts;
554 }
555 /* Run vertex fetch shader */
556 clipped = fpme->current_variant->jit_func(&fpme->llvm->vs_jit_context,
557 &fpme->llvm->jit_resources[PIPE_SHADER_VERTEX],
558 llvm_vert_info.verts,
559 draw->pt.user.vbuffer,
560 fetch_info->count,
561 start,
562 fpme->vertex_size,
563 draw->pt.vertex_buffer,
564 draw->instance_id,
565 vertex_id_offset,
566 draw->start_instance,
567 elts,
568 draw->pt.user.drawid,
569 draw->pt.user.viewid);
570
571 /* Finished with fetch and vs */
572 fetch_info = NULL;
573 vert_info = &llvm_vert_info;
574 }
575
576 /* Keep track of the patch lengths if we have a geometry shader, this way we can increment
577 * gl_PrimitiveID once per patch, instead of per tessellation output primitive.
578 * The Vulkan and OpenGL specs say:
579 * In a geometry shader, it (gl_PrimitiveID) will contain the number of primitives presented
580 * as input to the shader since the current set of rendering primitives was started.
581 * This sounds like gl_PrimitiveID will be the index of each output primitive from the
582 * tessellation shader, so multiple per patch. But drivers implement the more specific language
583 * from ARB_tessellation_shader:
584 * In geometry language the input variable gl_PrimitiveIDIn behaves identically to the
585 * tessellation control and evaluation shader input variable gl_PrimitiveID.
586 * Which implies on primitive index per patch. DirectX is also more specific:
587 * The geometry-shader stage can consume the SV_PrimitiveID system-generated value that is
588 * auto-generated by the IA. This allows per-primitive data to be fetched or computed if desired.
589 * The input-assembler (IA) stage happens before the tessellation, so we would expect one index
590 * per patch.
591 */
592 uint32_t *patch_lengths = NULL;
593
594 if (opt & PT_SHADE) {
595 struct draw_vertex_shader *vshader = draw->vs.vertex_shader;
596 if (tcs_shader) {
597 draw_tess_ctrl_shader_run(tcs_shader,
598 vert_info,
599 prim_info,
600 &vshader->info,
601 &tcs_vert_info,
602 &tcs_prim_info);
603 FREE(vert_info->verts);
604 vert_info = &tcs_vert_info;
605 prim_info = &tcs_prim_info;
606
607 } else if (tes_shader) {
608 unsigned num_prims = prim_info->count / draw->pt.vertices_per_patch;
609 tcs_prim_info = *prim_info;
610 tcs_prim_info.primitive_count = num_prims;
611 prim_info = &tcs_prim_info;
612 }
613
614 if (tes_shader) {
615 draw_tess_eval_shader_run(tes_shader,
616 tcs_shader ? tcs_shader->vertices_out : draw->pt.vertices_per_patch,
617 vert_info,
618 prim_info,
619 tcs_shader ? &tcs_shader->info : &vshader->info,
620 &tes_vert_info,
621 &tes_prim_info,
622 gshader ? &patch_lengths : NULL,
623 &tes_elts_out);
624
625 FREE(vert_info->verts);
626 vert_info = &tes_vert_info;
627 prim_info = &tes_prim_info;
628 free_prim_info = true;
629
630 /*
631 * pt emit can only handle ushort number of vertices (see
632 * render->allocate_vertices).
633 * vsplit guarantees there's never more than 4096, however GS can
634 * easily blow this up (by a factor of 256 (or even 1024) max).
635 */
636 if (vert_info->count > 65535) {
637 opt |= PT_PIPELINE;
638 }
639 }
640 }
641
642 struct draw_vertex_info gs_vert_info[TGSI_MAX_VERTEX_STREAMS];
643 memset(&gs_vert_info, 0, sizeof(gs_vert_info));
644
645 if ((opt & PT_SHADE) && gshader) {
646 struct draw_vertex_shader *vshader = draw->vs.vertex_shader;
647 draw_geometry_shader_run(gshader,
648 draw->pt.user.constants[PIPE_SHADER_GEOMETRY],
649 vert_info,
650 prim_info,
651 tes_shader ? &tes_shader->info : &vshader->info,
652 tes_shader ? &patch_lengths : NULL,
653 gs_vert_info,
654 gs_prim_info);
655
656 FREE(vert_info->verts);
657 if (free_prim_info) {
658 FREE(prim_info->primitive_lengths);
659 FREE(tes_elts_out);
660 }
661 vert_info = &gs_vert_info[0];
662 prim_info = &gs_prim_info[0];
663 free_prim_info = false;
664 /*
665 * pt emit can only handle ushort number of vertices (see
666 * render->allocate_vertices).
667 * vsplit guarantees there's never more than 4096, however GS can
668 * easily blow this up (by a factor of 256 (or even 1024) max).
669 */
670 if (vert_info->count > 65535) {
671 opt |= PT_PIPELINE;
672 }
673 } else {
674 if (!tes_shader &&
675 draw_prim_assembler_is_required(draw, prim_info, vert_info)) {
676 draw_prim_assembler_run(draw, prim_info, vert_info,
677 &ia_prim_info, &ia_vert_info);
678
679 if (ia_vert_info.count) {
680 FREE(vert_info->verts);
681 if (free_prim_info) {
682 FREE(prim_info->primitive_lengths);
683 FREE(tes_elts_out);
684 tes_elts_out = NULL;
685 }
686 vert_info = &ia_vert_info;
687 prim_info = &ia_prim_info;
688 free_prim_info = true;
689 }
690 }
691 }
692
693 /* stream output needs to be done before clipping */
694 draw_pt_so_emit(fpme->so_emit,
695 gshader ? gshader->num_vertex_streams : 1,
696 vert_info, prim_info);
697
698 if (prim_info->count == 0) {
699 debug_printf("GS/IA didn't emit any vertices!\n");
700 } else {
701 draw_stats_clipper_primitives(draw, prim_info);
702
703 /*
704 * if there's no position, need to stop now, or the latter stages
705 * will try to access non-existent position output.
706 */
707 if (draw_current_shader_position_output(draw) != -1) {
708 if ((opt & PT_SHADE) &&
709 (gshader || tes_shader ||
710 draw->vs.vertex_shader->info.writes_viewport_index)) {
711 clipped = draw_pt_post_vs_run(fpme->post_vs, vert_info, prim_info);
712 }
713 /* "clipped" also includes non-one edgeflag */
714 if (clipped) {
715 opt |= PT_PIPELINE;
716 }
717
718 /* Do we need to run the pipeline? Now will come here if clipped */
719 if (opt & PT_PIPELINE) {
720 pipeline(fpme, vert_info, prim_info);
721 } else {
722 emit(fpme->emit, vert_info, prim_info);
723 }
724 }
725 }
726
727 FREE(vert_info->verts);
728 if (gshader && gshader->num_vertex_streams > 1) {
729 for (unsigned i = 1; i < gshader->num_vertex_streams; i++)
730 FREE(gs_vert_info[i].verts);
731 }
732
733 if (patch_lengths) {
734 FREE(patch_lengths);
735 }
736
737 if (free_prim_info) {
738 FREE(tes_elts_out);
739 FREE(prim_info->primitive_lengths);
740 }
741 }
742
743
744 static inline enum mesa_prim
prim_type(enum mesa_prim prim,unsigned flags)745 prim_type(enum mesa_prim prim, unsigned flags)
746 {
747 if (flags & DRAW_LINE_LOOP_AS_STRIP)
748 return MESA_PRIM_LINE_STRIP;
749 else
750 return prim;
751 }
752
753
754 static void
llvm_middle_end_run(struct draw_pt_middle_end * middle,const unsigned * fetch_elts,unsigned fetch_count,const uint16_t * draw_elts,unsigned draw_count,unsigned prim_flags)755 llvm_middle_end_run(struct draw_pt_middle_end *middle,
756 const unsigned *fetch_elts,
757 unsigned fetch_count,
758 const uint16_t *draw_elts,
759 unsigned draw_count,
760 unsigned prim_flags)
761 {
762 struct llvm_middle_end *fpme = llvm_middle_end(middle);
763 struct draw_fetch_info fetch_info;
764 struct draw_prim_info prim_info;
765
766 fetch_info.linear = false;
767 fetch_info.start = 0;
768 fetch_info.elts = fetch_elts;
769 fetch_info.count = fetch_count;
770
771 prim_info.linear = false;
772 prim_info.start = 0;
773 prim_info.count = draw_count;
774 prim_info.elts = draw_elts;
775 prim_info.prim = prim_type(fpme->input_prim, prim_flags);
776 prim_info.flags = prim_flags;
777 prim_info.primitive_count = 1;
778 prim_info.primitive_lengths = &draw_count;
779
780 llvm_pipeline_generic(middle, &fetch_info, &prim_info);
781 }
782
783
784 static void
llvm_middle_end_linear_run(struct draw_pt_middle_end * middle,unsigned start,unsigned count,unsigned prim_flags)785 llvm_middle_end_linear_run(struct draw_pt_middle_end *middle,
786 unsigned start,
787 unsigned count,
788 unsigned prim_flags)
789 {
790 struct llvm_middle_end *fpme = llvm_middle_end(middle);
791 struct draw_fetch_info fetch_info;
792 struct draw_prim_info prim_info;
793
794 fetch_info.linear = true;
795 fetch_info.start = start;
796 fetch_info.count = count;
797 fetch_info.elts = NULL;
798
799 prim_info.linear = true;
800 prim_info.start = start;
801 prim_info.count = count;
802 prim_info.elts = NULL;
803 prim_info.prim = prim_type(fpme->input_prim, prim_flags);
804 prim_info.flags = prim_flags;
805 prim_info.primitive_count = 1;
806 prim_info.primitive_lengths = &count;
807
808 llvm_pipeline_generic(middle, &fetch_info, &prim_info);
809 }
810
811
812 static bool
llvm_middle_end_linear_run_elts(struct draw_pt_middle_end * middle,unsigned start,unsigned count,const uint16_t * draw_elts,unsigned draw_count,unsigned prim_flags)813 llvm_middle_end_linear_run_elts(struct draw_pt_middle_end *middle,
814 unsigned start,
815 unsigned count,
816 const uint16_t *draw_elts,
817 unsigned draw_count,
818 unsigned prim_flags)
819 {
820 struct llvm_middle_end *fpme = llvm_middle_end(middle);
821 struct draw_fetch_info fetch_info;
822 struct draw_prim_info prim_info;
823
824 fetch_info.linear = true;
825 fetch_info.start = start;
826 fetch_info.count = count;
827 fetch_info.elts = NULL;
828
829 prim_info.linear = false;
830 prim_info.start = 0;
831 prim_info.count = draw_count;
832 prim_info.elts = draw_elts;
833 prim_info.prim = prim_type(fpme->input_prim, prim_flags);
834 prim_info.flags = prim_flags;
835 prim_info.primitive_count = 1;
836 prim_info.primitive_lengths = &draw_count;
837
838 llvm_pipeline_generic(middle, &fetch_info, &prim_info);
839
840 return true;
841 }
842
843
844 static void
llvm_middle_end_finish(struct draw_pt_middle_end * middle)845 llvm_middle_end_finish(struct draw_pt_middle_end *middle)
846 {
847 /* nothing to do */
848 }
849
850
851 static void
llvm_middle_end_destroy(struct draw_pt_middle_end * middle)852 llvm_middle_end_destroy(struct draw_pt_middle_end *middle)
853 {
854 struct llvm_middle_end *fpme = llvm_middle_end(middle);
855
856 if (fpme->fetch)
857 draw_pt_fetch_destroy(fpme->fetch);
858
859 if (fpme->emit)
860 draw_pt_emit_destroy(fpme->emit);
861
862 if (fpme->so_emit)
863 draw_pt_so_emit_destroy(fpme->so_emit);
864
865 if (fpme->post_vs)
866 draw_pt_post_vs_destroy(fpme->post_vs);
867
868 FREE(middle);
869 }
870
871
872 struct draw_pt_middle_end *
draw_pt_fetch_pipeline_or_emit_llvm(struct draw_context * draw)873 draw_pt_fetch_pipeline_or_emit_llvm(struct draw_context *draw)
874 {
875 struct llvm_middle_end *fpme = 0;
876
877 if (!draw->llvm)
878 return NULL;
879
880 fpme = CALLOC_STRUCT(llvm_middle_end);
881 if (!fpme)
882 goto fail;
883
884 fpme->base.prepare = llvm_middle_end_prepare;
885 fpme->base.bind_parameters = llvm_middle_end_bind_parameters;
886 fpme->base.run = llvm_middle_end_run;
887 fpme->base.run_linear = llvm_middle_end_linear_run;
888 fpme->base.run_linear_elts = llvm_middle_end_linear_run_elts;
889 fpme->base.finish = llvm_middle_end_finish;
890 fpme->base.destroy = llvm_middle_end_destroy;
891
892 fpme->draw = draw;
893
894 fpme->fetch = draw_pt_fetch_create(draw);
895 if (!fpme->fetch)
896 goto fail;
897
898 fpme->post_vs = draw_pt_post_vs_create(draw);
899 if (!fpme->post_vs)
900 goto fail;
901
902 fpme->emit = draw_pt_emit_create(draw);
903 if (!fpme->emit)
904 goto fail;
905
906 fpme->so_emit = draw_pt_so_emit_create(draw);
907 if (!fpme->so_emit)
908 goto fail;
909
910 fpme->llvm = draw->llvm;
911 if (!fpme->llvm)
912 goto fail;
913
914 fpme->current_variant = NULL;
915
916 return &fpme->base;
917
918 fail:
919 if (fpme)
920 llvm_middle_end_destroy(&fpme->base);
921
922 return NULL;
923 }
924