1 /**************************************************************************
2 *
3 * Copyright 2009 VMware, Inc.
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
21 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27
28 #include "draw_gs.h"
29
30 #include "draw_private.h"
31 #include "draw_context.h"
32 #ifdef LLVM_AVAILABLE
33 #include "draw_llvm.h"
34 #endif
35
36 #include "tgsi/tgsi_parse.h"
37 #include "tgsi/tgsi_exec.h"
38 #include "nir/nir_to_tgsi_info.h"
39 #include "pipe/p_shader_tokens.h"
40
41 #include "util/u_math.h"
42 #include "util/u_memory.h"
43 #include "util/u_prim.h"
44 #include "util/ralloc.h"
45 /* fixme: move it from here */
46 #define MAX_PRIMITIVES 64
47
48 static inline int
draw_gs_get_input_index(int semantic,int index,const struct tgsi_shader_info * input_info)49 draw_gs_get_input_index(int semantic, int index,
50 const struct tgsi_shader_info *input_info)
51 {
52 int i;
53 const ubyte *input_semantic_names = input_info->output_semantic_name;
54 const ubyte *input_semantic_indices = input_info->output_semantic_index;
55 for (i = 0; i < PIPE_MAX_SHADER_OUTPUTS; i++) {
56 if (input_semantic_names[i] == semantic &&
57 input_semantic_indices[i] == index)
58 return i;
59 }
60 return -1;
61 }
62
63 /**
64 * We execute geometry shaders in the SOA mode, so ideally we want to
65 * flush when the number of currently fetched primitives is equal to
66 * the number of elements in the SOA vector. This ensures that the
67 * throughput is optimized for the given vector instruction set.
68 */
69 static inline boolean
draw_gs_should_flush(struct draw_geometry_shader * shader)70 draw_gs_should_flush(struct draw_geometry_shader *shader)
71 {
72 return (shader->fetched_prim_count == shader->vector_length || shader->num_invocations > 1);
73 }
74
75 /*#define DEBUG_OUTPUTS 1*/
76 static void
tgsi_fetch_gs_outputs(struct draw_geometry_shader * shader,unsigned stream,unsigned num_primitives,float (** p_output)[4])77 tgsi_fetch_gs_outputs(struct draw_geometry_shader *shader,
78 unsigned stream,
79 unsigned num_primitives,
80 float (**p_output)[4])
81 {
82 struct tgsi_exec_machine *machine = shader->machine;
83 unsigned prim_idx, j, slot;
84 float (*output)[4];
85
86 output = *p_output;
87
88 /* Unswizzle all output results.
89 */
90
91 for (prim_idx = 0; prim_idx < num_primitives; ++prim_idx) {
92 unsigned num_verts_per_prim = machine->Primitives[stream][prim_idx];
93 unsigned prim_offset = machine->PrimitiveOffsets[stream][prim_idx];
94 shader->stream[stream].primitive_lengths[prim_idx + shader->stream[stream].emitted_primitives] =
95 machine->Primitives[stream][prim_idx];
96 shader->stream[stream].emitted_vertices += num_verts_per_prim;
97
98 for (j = 0; j < num_verts_per_prim; j++) {
99 int idx = prim_offset + j * shader->info.num_outputs;
100 #ifdef DEBUG_OUTPUTS
101 debug_printf("%d/%d) Output vert:\n", stream, idx / shader->info.num_outputs);
102 #endif
103 for (slot = 0; slot < shader->info.num_outputs; slot++) {
104 output[slot][0] = machine->Outputs[idx + slot].xyzw[0].f[0];
105 output[slot][1] = machine->Outputs[idx + slot].xyzw[1].f[0];
106 output[slot][2] = machine->Outputs[idx + slot].xyzw[2].f[0];
107 output[slot][3] = machine->Outputs[idx + slot].xyzw[3].f[0];
108 #ifdef DEBUG_OUTPUTS
109 debug_printf("\t%d: %f %f %f %f\n", slot,
110 output[slot][0],
111 output[slot][1],
112 output[slot][2],
113 output[slot][3]);
114 #endif
115 }
116 output = (float (*)[4])((char *)output + shader->vertex_size);
117 }
118 }
119 *p_output = output;
120 shader->stream[stream].emitted_primitives += num_primitives;
121 }
122
123 /*#define DEBUG_INPUTS 1*/
tgsi_fetch_gs_input(struct draw_geometry_shader * shader,unsigned * indices,unsigned num_vertices,unsigned prim_idx)124 static void tgsi_fetch_gs_input(struct draw_geometry_shader *shader,
125 unsigned *indices,
126 unsigned num_vertices,
127 unsigned prim_idx)
128 {
129 struct tgsi_exec_machine *machine = shader->machine;
130 unsigned slot, i;
131 int vs_slot;
132 unsigned input_vertex_stride = shader->input_vertex_stride;
133 const float (*input_ptr)[4];
134
135 int primid_sv = machine->SysSemanticToIndex[TGSI_SEMANTIC_PRIMID];
136 if (primid_sv != -1) {
137 for (unsigned j = 0; j < TGSI_QUAD_SIZE; j++)
138 machine->SystemValue[primid_sv].xyzw[0].i[j] = shader->in_prim_idx;
139 }
140
141 input_ptr = shader->input;
142
143 for (i = 0; i < num_vertices; ++i) {
144 const float (*input)[4];
145 #if DEBUG_INPUTS
146 debug_printf("%d) vertex index = %d (prim idx = %d)\n",
147 i, indices[i], prim_idx);
148 #endif
149 input = (const float (*)[4])(
150 (const char *)input_ptr + (indices[i] * input_vertex_stride));
151 for (slot = 0, vs_slot = 0; slot < shader->info.num_inputs; ++slot) {
152 unsigned idx = i * TGSI_EXEC_MAX_INPUT_ATTRIBS + slot;
153 if (shader->info.input_semantic_name[slot] == TGSI_SEMANTIC_PRIMID) {
154 machine->Inputs[idx].xyzw[0].u[prim_idx] = shader->in_prim_idx;
155 machine->Inputs[idx].xyzw[1].u[prim_idx] = shader->in_prim_idx;
156 machine->Inputs[idx].xyzw[2].u[prim_idx] = shader->in_prim_idx;
157 machine->Inputs[idx].xyzw[3].u[prim_idx] = shader->in_prim_idx;
158 } else {
159 vs_slot = draw_gs_get_input_index(
160 shader->info.input_semantic_name[slot],
161 shader->info.input_semantic_index[slot],
162 shader->input_info);
163 if (vs_slot < 0) {
164 debug_printf("VS/GS signature mismatch!\n");
165 machine->Inputs[idx].xyzw[0].f[prim_idx] = 0;
166 machine->Inputs[idx].xyzw[1].f[prim_idx] = 0;
167 machine->Inputs[idx].xyzw[2].f[prim_idx] = 0;
168 machine->Inputs[idx].xyzw[3].f[prim_idx] = 0;
169 } else {
170 #if DEBUG_INPUTS
171 debug_printf("\tSlot = %d, vs_slot = %d, idx = %d:\n",
172 slot, vs_slot, idx);
173 assert(!util_is_inf_or_nan(input[vs_slot][0]));
174 assert(!util_is_inf_or_nan(input[vs_slot][1]));
175 assert(!util_is_inf_or_nan(input[vs_slot][2]));
176 assert(!util_is_inf_or_nan(input[vs_slot][3]));
177 #endif
178 machine->Inputs[idx].xyzw[0].f[prim_idx] = input[vs_slot][0];
179 machine->Inputs[idx].xyzw[1].f[prim_idx] = input[vs_slot][1];
180 machine->Inputs[idx].xyzw[2].f[prim_idx] = input[vs_slot][2];
181 machine->Inputs[idx].xyzw[3].f[prim_idx] = input[vs_slot][3];
182 #if DEBUG_INPUTS
183 debug_printf("\t\t%f %f %f %f\n",
184 machine->Inputs[idx].xyzw[0].f[prim_idx],
185 machine->Inputs[idx].xyzw[1].f[prim_idx],
186 machine->Inputs[idx].xyzw[2].f[prim_idx],
187 machine->Inputs[idx].xyzw[3].f[prim_idx]);
188 #endif
189 ++vs_slot;
190 }
191 }
192 }
193 }
194 }
195
tgsi_gs_prepare(struct draw_geometry_shader * shader,const void * constants[PIPE_MAX_CONSTANT_BUFFERS],const unsigned constants_size[PIPE_MAX_CONSTANT_BUFFERS])196 static void tgsi_gs_prepare(struct draw_geometry_shader *shader,
197 const void *constants[PIPE_MAX_CONSTANT_BUFFERS],
198 const unsigned constants_size[PIPE_MAX_CONSTANT_BUFFERS])
199 {
200 struct tgsi_exec_machine *machine = shader->machine;
201 tgsi_exec_set_constant_buffers(machine, PIPE_MAX_CONSTANT_BUFFERS,
202 constants, constants_size);
203 }
204
tgsi_gs_run(struct draw_geometry_shader * shader,unsigned input_primitives,unsigned * out_prims)205 static void tgsi_gs_run(struct draw_geometry_shader *shader,
206 unsigned input_primitives,
207 unsigned *out_prims)
208 {
209 struct tgsi_exec_machine *machine = shader->machine;
210 int i;
211
212 if (shader->info.uses_invocationid) {
213 unsigned i = machine->SysSemanticToIndex[TGSI_SEMANTIC_INVOCATIONID];
214 for (int j = 0; j < TGSI_QUAD_SIZE; j++)
215 machine->SystemValue[i].xyzw[0].i[j] = shader->invocation_id;
216 }
217
218 /* run interpreter */
219 tgsi_exec_machine_run(machine, 0);
220
221 for (i = 0; i < 4; i++) {
222 int prim_i;
223 int prim_c;
224 switch (i) {
225 case 0:
226 prim_i = TGSI_EXEC_TEMP_PRIMITIVE_I;
227 prim_c = TGSI_EXEC_TEMP_PRIMITIVE_C;
228 break;
229 case 1:
230 prim_i = TGSI_EXEC_TEMP_PRIMITIVE_S1_I;
231 prim_c = TGSI_EXEC_TEMP_PRIMITIVE_S1_C;
232 break;
233 case 2:
234 prim_i = TGSI_EXEC_TEMP_PRIMITIVE_S2_I;
235 prim_c = TGSI_EXEC_TEMP_PRIMITIVE_S2_C;
236 break;
237 case 3:
238 prim_i = TGSI_EXEC_TEMP_PRIMITIVE_S3_I;
239 prim_c = TGSI_EXEC_TEMP_PRIMITIVE_S3_C;
240 break;
241 };
242
243 out_prims[i] = machine->Temps[prim_i].xyzw[prim_c].u[0];
244 }
245 }
246
247 #ifdef LLVM_AVAILABLE
248
249 static void
llvm_fetch_gs_input(struct draw_geometry_shader * shader,unsigned * indices,unsigned num_vertices,unsigned prim_idx)250 llvm_fetch_gs_input(struct draw_geometry_shader *shader,
251 unsigned *indices,
252 unsigned num_vertices,
253 unsigned prim_idx)
254 {
255 unsigned slot, i;
256 int vs_slot;
257 unsigned input_vertex_stride = shader->input_vertex_stride;
258 const float (*input_ptr)[4];
259 float (*input_data)[6][PIPE_MAX_SHADER_INPUTS][TGSI_NUM_CHANNELS][TGSI_NUM_CHANNELS] = &shader->gs_input->data;
260
261 shader->llvm_prim_ids[shader->fetched_prim_count] = shader->in_prim_idx;
262
263 input_ptr = shader->input;
264
265 for (i = 0; i < num_vertices; ++i) {
266 const float (*input)[4];
267 #if DEBUG_INPUTS
268 debug_printf("%d) vertex index = %d (prim idx = %d)\n",
269 i, indices[i], prim_idx);
270 #endif
271 input = (const float (*)[4])(
272 (const char *)input_ptr + (indices[i] * input_vertex_stride));
273 for (slot = 0, vs_slot = 0; slot < shader->info.num_inputs; ++slot) {
274 if (shader->info.input_semantic_name[slot] == TGSI_SEMANTIC_PRIMID) {
275 /* skip. we handle system values through gallivm */
276 /* NOTE: If we hit this case here it's an ordinary input not a sv,
277 * even though it probably should be a sv.
278 * Not sure how to set it up as regular input however if that even,
279 * would make sense so hack around this later in gallivm.
280 */
281 } else {
282 vs_slot = draw_gs_get_input_index(
283 shader->info.input_semantic_name[slot],
284 shader->info.input_semantic_index[slot],
285 shader->input_info);
286 if (vs_slot < 0) {
287 debug_printf("VS/GS signature mismatch!\n");
288 (*input_data)[i][slot][0][prim_idx] = 0;
289 (*input_data)[i][slot][1][prim_idx] = 0;
290 (*input_data)[i][slot][2][prim_idx] = 0;
291 (*input_data)[i][slot][3][prim_idx] = 0;
292 } else {
293 #if DEBUG_INPUTS
294 debug_printf("\tSlot = %d, vs_slot = %d, i = %d:\n",
295 slot, vs_slot, i);
296 assert(!util_is_inf_or_nan(input[vs_slot][0]));
297 assert(!util_is_inf_or_nan(input[vs_slot][1]));
298 assert(!util_is_inf_or_nan(input[vs_slot][2]));
299 assert(!util_is_inf_or_nan(input[vs_slot][3]));
300 #endif
301 (*input_data)[i][slot][0][prim_idx] = input[vs_slot][0];
302 (*input_data)[i][slot][1][prim_idx] = input[vs_slot][1];
303 (*input_data)[i][slot][2][prim_idx] = input[vs_slot][2];
304 (*input_data)[i][slot][3][prim_idx] = input[vs_slot][3];
305 #if DEBUG_INPUTS
306 debug_printf("\t\t%f %f %f %f\n",
307 (*input_data)[i][slot][0][prim_idx],
308 (*input_data)[i][slot][1][prim_idx],
309 (*input_data)[i][slot][2][prim_idx],
310 (*input_data)[i][slot][3][prim_idx]);
311 #endif
312 ++vs_slot;
313 }
314 }
315 }
316 }
317 }
318
319 static void
llvm_fetch_gs_outputs(struct draw_geometry_shader * shader,unsigned stream,unsigned num_primitives,float (** p_output)[4])320 llvm_fetch_gs_outputs(struct draw_geometry_shader *shader,
321 unsigned stream,
322 unsigned num_primitives,
323 float (**p_output)[4])
324 {
325 int total_verts = 0;
326 int vertex_count = 0;
327 int total_prims = 0;
328 int max_prims_per_invocation = 0;
329 char *output_ptr = (char*)shader->gs_output[stream];
330 int i, j, prim_idx;
331 unsigned next_prim_boundary = shader->primitive_boundary;
332
333 for (i = 0; i < shader->vector_length; ++i) {
334 int prims = shader->llvm_emitted_primitives[i + (stream * shader->vector_length)];
335 total_prims += prims;
336 max_prims_per_invocation = MAX2(max_prims_per_invocation, prims);
337 }
338 for (i = 0; i < shader->vector_length; ++i) {
339 total_verts += shader->llvm_emitted_vertices[i + (stream * shader->vector_length)];
340 }
341
342 output_ptr += shader->stream[stream].emitted_vertices * shader->vertex_size;
343 for (i = 0; i < shader->vector_length - 1; ++i) {
344 int current_verts = shader->llvm_emitted_vertices[i + (stream * shader->vector_length)];
345 int next_verts = shader->llvm_emitted_vertices[i + 1 + (stream * shader->vector_length)];
346 #if 0
347 int j;
348 for (j = 0; j < current_verts; ++j) {
349 struct vertex_header *vh = (struct vertex_header *)
350 (output_ptr + shader->vertex_size * (i * next_prim_boundary + j));
351 debug_printf("--- %d) [%f, %f, %f, %f]\n", j + vertex_count,
352 vh->data[0][0], vh->data[0][1], vh->data[0][2], vh->data[0][3]);
353
354 }
355 #endif
356 debug_assert(current_verts <= shader->max_output_vertices);
357 debug_assert(next_verts <= shader->max_output_vertices);
358 if (next_verts) {
359 memmove(output_ptr + (vertex_count + current_verts) * shader->vertex_size,
360 output_ptr + ((i + 1) * next_prim_boundary) * shader->vertex_size,
361 shader->vertex_size * next_verts);
362 }
363 vertex_count += current_verts;
364 }
365
366 #if 0
367 {
368 int i;
369 for (i = 0; i < total_verts; ++i) {
370 struct vertex_header *vh = (struct vertex_header *)(output_ptr + shader->vertex_size * i);
371 debug_printf("%d) Vertex:\n", i);
372 for (j = 0; j < shader->info.num_outputs; ++j) {
373 unsigned *udata = (unsigned*)vh->data[j];
374 debug_printf(" %d) [%f, %f, %f, %f] [%d, %d, %d, %d]\n", j,
375 vh->data[j][0], vh->data[j][1], vh->data[j][2], vh->data[j][3],
376 udata[0], udata[1], udata[2], udata[3]);
377 }
378
379 }
380 }
381 #endif
382
383 prim_idx = 0;
384 for (i = 0; i < shader->vector_length; ++i) {
385 int num_prims = shader->llvm_emitted_primitives[i + (stream * shader->vector_length)];
386 for (j = 0; j < num_prims; ++j) {
387 int prim_length =
388 shader->llvm_prim_lengths[j * shader->num_vertex_streams + stream][i];
389 shader->stream[stream].primitive_lengths[shader->stream[stream].emitted_primitives + prim_idx] =
390 prim_length;
391 ++prim_idx;
392 }
393 }
394
395 shader->stream[stream].emitted_primitives += total_prims;
396 shader->stream[stream].emitted_vertices += total_verts;
397 }
398
399 static void
llvm_gs_prepare(struct draw_geometry_shader * shader,const void * constants[PIPE_MAX_CONSTANT_BUFFERS],const unsigned constants_size[PIPE_MAX_CONSTANT_BUFFERS])400 llvm_gs_prepare(struct draw_geometry_shader *shader,
401 const void *constants[PIPE_MAX_CONSTANT_BUFFERS],
402 const unsigned constants_size[PIPE_MAX_CONSTANT_BUFFERS])
403 {
404 }
405
406 static void
llvm_gs_run(struct draw_geometry_shader * shader,unsigned input_primitives,unsigned * out_prims)407 llvm_gs_run(struct draw_geometry_shader *shader,
408 unsigned input_primitives, unsigned *out_prims)
409 {
410 struct vertex_header *input[PIPE_MAX_VERTEX_STREAMS];
411 for (unsigned i = 0; i < shader->num_vertex_streams; i++) {
412 char *tmp = (char *)shader->gs_output[i];
413 tmp += shader->stream[i].emitted_vertices * shader->vertex_size;
414 input[i] = (struct vertex_header *)tmp;
415 }
416
417 shader->current_variant->jit_func(
418 shader->jit_context, shader->gs_input->data,
419 input,
420 input_primitives,
421 shader->draw->instance_id,
422 shader->llvm_prim_ids,
423 shader->invocation_id);
424
425 for (unsigned i = 0; i < shader->num_vertex_streams; i++) {
426 out_prims[i] = shader->jit_context->emitted_prims[i];
427 }
428 }
429
430 #endif
431
gs_flush(struct draw_geometry_shader * shader)432 static void gs_flush(struct draw_geometry_shader *shader)
433 {
434 unsigned out_prim_count[TGSI_MAX_VERTEX_STREAMS];
435 unsigned i;
436 unsigned input_primitives = shader->fetched_prim_count;
437
438 if (shader->draw->collect_statistics) {
439 shader->draw->statistics.gs_invocations += input_primitives;
440 }
441
442 debug_assert(input_primitives > 0 &&
443 input_primitives <= 4);
444
445 for (unsigned invocation = 0; invocation < shader->num_invocations; invocation++) {
446 shader->invocation_id = invocation;
447 shader->run(shader, input_primitives, out_prim_count);
448 for (i = 0; i < shader->num_vertex_streams; i++) {
449 shader->fetch_outputs(shader, i, out_prim_count[i],
450 &shader->stream[i].tmp_output);
451 }
452 }
453
454 #if 0
455 for (i = 0; i < shader->num_vertex_streams; i++) {
456 debug_printf("stream %d: PRIM emitted prims = %d (verts=%d), cur prim count = %d\n",
457 i,
458 shader->stream[i].emitted_primitives, shader->stream[i].emitted_vertices,
459 out_prim_count[i]);
460 }
461 #endif
462
463 shader->fetched_prim_count = 0;
464 }
465
gs_point(struct draw_geometry_shader * shader,int idx)466 static void gs_point(struct draw_geometry_shader *shader,
467 int idx)
468 {
469 unsigned indices[1];
470
471 indices[0] = idx;
472
473 shader->fetch_inputs(shader, indices, 1,
474 shader->fetched_prim_count);
475 ++shader->in_prim_idx;
476 ++shader->fetched_prim_count;
477
478 if (draw_gs_should_flush(shader))
479 gs_flush(shader);
480 }
481
gs_line(struct draw_geometry_shader * shader,int i0,int i1)482 static void gs_line(struct draw_geometry_shader *shader,
483 int i0, int i1)
484 {
485 unsigned indices[2];
486
487 indices[0] = i0;
488 indices[1] = i1;
489
490 shader->fetch_inputs(shader, indices, 2,
491 shader->fetched_prim_count);
492 ++shader->in_prim_idx;
493 ++shader->fetched_prim_count;
494
495 if (draw_gs_should_flush(shader))
496 gs_flush(shader);
497 }
498
gs_line_adj(struct draw_geometry_shader * shader,int i0,int i1,int i2,int i3)499 static void gs_line_adj(struct draw_geometry_shader *shader,
500 int i0, int i1, int i2, int i3)
501 {
502 unsigned indices[4];
503
504 indices[0] = i0;
505 indices[1] = i1;
506 indices[2] = i2;
507 indices[3] = i3;
508
509 shader->fetch_inputs(shader, indices, 4,
510 shader->fetched_prim_count);
511 ++shader->in_prim_idx;
512 ++shader->fetched_prim_count;
513
514 if (draw_gs_should_flush(shader))
515 gs_flush(shader);
516 }
517
gs_tri(struct draw_geometry_shader * shader,int i0,int i1,int i2)518 static void gs_tri(struct draw_geometry_shader *shader,
519 int i0, int i1, int i2)
520 {
521 unsigned indices[3];
522
523 indices[0] = i0;
524 indices[1] = i1;
525 indices[2] = i2;
526
527 shader->fetch_inputs(shader, indices, 3,
528 shader->fetched_prim_count);
529 ++shader->in_prim_idx;
530 ++shader->fetched_prim_count;
531
532 if (draw_gs_should_flush(shader))
533 gs_flush(shader);
534 }
535
gs_tri_adj(struct draw_geometry_shader * shader,int i0,int i1,int i2,int i3,int i4,int i5)536 static void gs_tri_adj(struct draw_geometry_shader *shader,
537 int i0, int i1, int i2,
538 int i3, int i4, int i5)
539 {
540 unsigned indices[6];
541
542 indices[0] = i0;
543 indices[1] = i1;
544 indices[2] = i2;
545 indices[3] = i3;
546 indices[4] = i4;
547 indices[5] = i5;
548
549 shader->fetch_inputs(shader, indices, 6,
550 shader->fetched_prim_count);
551 ++shader->in_prim_idx;
552 ++shader->fetched_prim_count;
553
554 if (draw_gs_should_flush(shader))
555 gs_flush(shader);
556 }
557
558 #define FUNC gs_run
559 #define GET_ELT(idx) (idx)
560 #include "draw_gs_tmp.h"
561
562
563 #define FUNC gs_run_elts
564 #define LOCAL_VARS const ushort *elts = input_prims->elts;
565 #define GET_ELT(idx) (elts[idx])
566 #include "draw_gs_tmp.h"
567
568
569 /**
570 * Execute geometry shader.
571 */
draw_geometry_shader_run(struct draw_geometry_shader * shader,const void * constants[PIPE_MAX_CONSTANT_BUFFERS],const unsigned constants_size[PIPE_MAX_CONSTANT_BUFFERS],const struct draw_vertex_info * input_verts,const struct draw_prim_info * input_prim,const struct tgsi_shader_info * input_info,struct draw_vertex_info * output_verts,struct draw_prim_info * output_prims)572 int draw_geometry_shader_run(struct draw_geometry_shader *shader,
573 const void *constants[PIPE_MAX_CONSTANT_BUFFERS],
574 const unsigned constants_size[PIPE_MAX_CONSTANT_BUFFERS],
575 const struct draw_vertex_info *input_verts,
576 const struct draw_prim_info *input_prim,
577 const struct tgsi_shader_info *input_info,
578 struct draw_vertex_info *output_verts,
579 struct draw_prim_info *output_prims )
580 {
581 const float (*input)[4] = (const float (*)[4])input_verts->verts->data;
582 unsigned input_stride = input_verts->vertex_size;
583 unsigned num_outputs = draw_total_gs_outputs(shader->draw);
584 unsigned vertex_size = sizeof(struct vertex_header) + num_outputs * 4 * sizeof(float);
585 unsigned num_input_verts = input_prim->linear ?
586 input_verts->count :
587 input_prim->count;
588 unsigned num_in_primitives =
589 align(
590 MAX2(u_decomposed_prims_for_vertices(input_prim->prim,
591 num_input_verts),
592 u_decomposed_prims_for_vertices(shader->input_primitive,
593 num_input_verts)),
594 shader->vector_length);
595 unsigned max_out_prims =
596 u_decomposed_prims_for_vertices(shader->output_primitive,
597 shader->max_output_vertices)
598 * num_in_primitives;
599 /* we allocate exactly one extra vertex per primitive to allow the GS to emit
600 * overflown vertices into some area where they won't harm anyone */
601 unsigned total_verts_per_buffer = shader->primitive_boundary *
602 num_in_primitives;
603 int i;
604 //Assume at least one primitive
605 max_out_prims = MAX2(max_out_prims, 1);
606
607 for (i = 0; i < shader->num_vertex_streams; i++) {
608 /* write all the vertex data into all the streams */
609 output_verts[i].vertex_size = vertex_size;
610 output_verts[i].stride = output_verts[i].vertex_size;
611 output_verts[i].verts =
612 (struct vertex_header *)MALLOC(output_verts[i].vertex_size *
613 total_verts_per_buffer * shader->num_invocations);
614 debug_assert(output_verts[i].verts);
615 }
616
617 #if 0
618 debug_printf("%s count = %d (in prims # = %d, invocs = %d, streams = %d)\n",
619 __FUNCTION__, num_input_verts, num_in_primitives,
620 shader->num_invocations, shader->num_vertex_streams);
621 debug_printf("\tlinear = %d, prim_info->count = %d\n",
622 input_prim->linear, input_prim->count);
623 debug_printf("\tprim pipe = %s, shader in = %s, shader out = %s\n",
624 u_prim_name(input_prim->prim),
625 u_prim_name(shader->input_primitive),
626 u_prim_name(shader->output_primitive));
627 debug_printf("\tmaxv = %d, maxp = %d, primitive_boundary = %d, "
628 "vertex_size = %d, tverts = %d\n",
629 shader->max_output_vertices, max_out_prims,
630 shader->primitive_boundary, output_verts->vertex_size,
631 total_verts_per_buffer);
632 #endif
633
634 for (i = 0; i < shader->num_vertex_streams; i++) {
635 shader->stream[i].emitted_vertices = 0;
636 shader->stream[i].emitted_primitives = 0;
637 FREE(shader->stream[i].primitive_lengths);
638 shader->stream[i].primitive_lengths = MALLOC(max_out_prims * sizeof(unsigned) * shader->num_invocations);
639 shader->stream[i].tmp_output = (float (*)[4])output_verts[i].verts->data;
640 }
641 shader->vertex_size = vertex_size;
642 shader->fetched_prim_count = 0;
643 shader->input_vertex_stride = input_stride;
644 shader->input = input;
645 shader->input_info = input_info;
646
647 #ifdef LLVM_AVAILABLE
648 if (shader->draw->llvm) {
649 for (i = 0; i < shader->num_vertex_streams; i++) {
650 shader->gs_output[i] = output_verts[i].verts;
651 }
652 if (max_out_prims > shader->max_out_prims) {
653 unsigned i;
654 if (shader->llvm_prim_lengths) {
655 for (i = 0; i < shader->num_vertex_streams * shader->max_out_prims; ++i) {
656 align_free(shader->llvm_prim_lengths[i]);
657 }
658 FREE(shader->llvm_prim_lengths);
659 }
660
661 shader->llvm_prim_lengths = MALLOC(shader->num_vertex_streams * max_out_prims * sizeof(unsigned*));
662 for (i = 0; i < shader->num_vertex_streams * max_out_prims; ++i) {
663 int vector_size = shader->vector_length * sizeof(unsigned);
664 shader->llvm_prim_lengths[i] =
665 align_malloc(vector_size, vector_size);
666 }
667
668 shader->max_out_prims = max_out_prims;
669 }
670 shader->jit_context->prim_lengths = shader->llvm_prim_lengths;
671 shader->jit_context->emitted_vertices = shader->llvm_emitted_vertices;
672 shader->jit_context->emitted_prims = shader->llvm_emitted_primitives;
673 }
674 #endif
675
676 shader->prepare(shader, constants, constants_size);
677
678 if (input_prim->linear)
679 gs_run(shader, input_prim, input_verts,
680 output_prims, output_verts);
681 else
682 gs_run_elts(shader, input_prim, input_verts,
683 output_prims, output_verts);
684
685 /* Flush the remaining primitives. Will happen if
686 * num_input_primitives % 4 != 0
687 */
688 if (shader->fetched_prim_count > 0) {
689 gs_flush(shader);
690 }
691 debug_assert(shader->fetched_prim_count == 0);
692
693 /* Update prim_info:
694 */
695 for (i = 0; i < shader->num_vertex_streams; i++) {
696 output_prims[i].linear = TRUE;
697 output_prims[i].elts = NULL;
698 output_prims[i].start = 0;
699 output_prims[i].count = shader->stream[i].emitted_vertices;
700 output_prims[i].prim = shader->output_primitive;
701 output_prims[i].flags = 0x0;
702 output_prims[i].primitive_lengths = shader->stream[i].primitive_lengths;
703 output_prims[i].primitive_count = shader->stream[i].emitted_primitives;
704 output_verts[i].count = shader->stream[i].emitted_vertices;
705
706 if (shader->draw->collect_statistics) {
707 unsigned j;
708 for (j = 0; j < shader->stream[i].emitted_primitives; ++j) {
709 shader->draw->statistics.gs_primitives +=
710 u_decomposed_prims_for_vertices(shader->output_primitive,
711 shader->stream[i].primitive_lengths[j]);
712 }
713 }
714 }
715
716 #if 0
717 debug_printf("GS finished\n");
718 for (i = 0; i < 4; i++)
719 debug_printf("stream %d: prims = %d verts = %d\n", i, output_prims[i].primitive_count, output_verts[i].count);
720 #endif
721
722 return 0;
723 }
724
draw_geometry_shader_prepare(struct draw_geometry_shader * shader,struct draw_context * draw)725 void draw_geometry_shader_prepare(struct draw_geometry_shader *shader,
726 struct draw_context *draw)
727 {
728 boolean use_llvm = draw->llvm != NULL;
729 if (!use_llvm && shader && shader->machine->Tokens != shader->state.tokens) {
730 tgsi_exec_machine_bind_shader(shader->machine,
731 shader->state.tokens,
732 draw->gs.tgsi.sampler,
733 draw->gs.tgsi.image,
734 draw->gs.tgsi.buffer);
735 }
736 }
737
738
739 boolean
draw_gs_init(struct draw_context * draw)740 draw_gs_init( struct draw_context *draw )
741 {
742 if (!draw->llvm) {
743 draw->gs.tgsi.machine = tgsi_exec_machine_create(PIPE_SHADER_GEOMETRY);
744
745 for (unsigned i = 0; i < TGSI_MAX_VERTEX_STREAMS; i++) {
746 draw->gs.tgsi.machine->Primitives[i] = align_malloc(
747 MAX_PRIMITIVES * sizeof(struct tgsi_exec_vector), 16);
748 draw->gs.tgsi.machine->PrimitiveOffsets[i] = align_malloc(
749 MAX_PRIMITIVES * sizeof(struct tgsi_exec_vector), 16);
750 if (!draw->gs.tgsi.machine->Primitives[i] || !draw->gs.tgsi.machine->PrimitiveOffsets[i])
751 return FALSE;
752 memset(draw->gs.tgsi.machine->Primitives[i], 0,
753 MAX_PRIMITIVES * sizeof(struct tgsi_exec_vector));
754 memset(draw->gs.tgsi.machine->PrimitiveOffsets[i], 0,
755 MAX_PRIMITIVES * sizeof(struct tgsi_exec_vector));
756 }
757 }
758
759 return TRUE;
760 }
761
draw_gs_destroy(struct draw_context * draw)762 void draw_gs_destroy( struct draw_context *draw )
763 {
764 int i;
765 if (draw->gs.tgsi.machine) {
766 for (i = 0; i < TGSI_MAX_VERTEX_STREAMS; i++) {
767 align_free(draw->gs.tgsi.machine->Primitives[i]);
768 align_free(draw->gs.tgsi.machine->PrimitiveOffsets[i]);
769 }
770 tgsi_exec_machine_destroy(draw->gs.tgsi.machine);
771 }
772 }
773
774 struct draw_geometry_shader *
draw_create_geometry_shader(struct draw_context * draw,const struct pipe_shader_state * state)775 draw_create_geometry_shader(struct draw_context *draw,
776 const struct pipe_shader_state *state)
777 {
778 #ifdef LLVM_AVAILABLE
779 boolean use_llvm = draw->llvm != NULL;
780 struct llvm_geometry_shader *llvm_gs = NULL;
781 #endif
782 struct draw_geometry_shader *gs;
783 unsigned i;
784
785 #ifdef LLVM_AVAILABLE
786 if (use_llvm) {
787 llvm_gs = CALLOC_STRUCT(llvm_geometry_shader);
788
789 if (!llvm_gs)
790 return NULL;
791
792 gs = &llvm_gs->base;
793
794 make_empty_list(&llvm_gs->variants);
795 } else
796 #endif
797 {
798 gs = CALLOC_STRUCT(draw_geometry_shader);
799 }
800
801 if (!gs)
802 return NULL;
803
804 gs->draw = draw;
805 gs->state = *state;
806
807 if (state->type == PIPE_SHADER_IR_TGSI) {
808 gs->state.tokens = tgsi_dup_tokens(state->tokens);
809 if (!gs->state.tokens) {
810 FREE(gs);
811 return NULL;
812 }
813
814 tgsi_scan_shader(state->tokens, &gs->info);
815 } else
816 nir_tgsi_scan_shader(state->ir.nir, &gs->info, true);
817
818 /* setup the defaults */
819 gs->max_out_prims = 0;
820
821 #ifdef LLVM_AVAILABLE
822 if (use_llvm) {
823 /* TODO: change the input array to handle the following
824 vector length, instead of the currently hardcoded
825 TGSI_NUM_CHANNELS
826 gs->vector_length = lp_native_vector_width / 32;*/
827 gs->vector_length = TGSI_NUM_CHANNELS;
828 } else
829 #endif
830 {
831 gs->vector_length = 1;
832 }
833
834 gs->input_primitive =
835 gs->info.properties[TGSI_PROPERTY_GS_INPUT_PRIM];
836 gs->output_primitive =
837 gs->info.properties[TGSI_PROPERTY_GS_OUTPUT_PRIM];
838 gs->max_output_vertices =
839 gs->info.properties[TGSI_PROPERTY_GS_MAX_OUTPUT_VERTICES];
840 gs->num_invocations =
841 gs->info.properties[TGSI_PROPERTY_GS_INVOCATIONS];
842 if (!gs->max_output_vertices)
843 gs->max_output_vertices = 32;
844
845 /* Primitive boundary is bigger than max_output_vertices by one, because
846 * the specification says that the geometry shader should exit if the
847 * number of emitted vertices is bigger or equal to max_output_vertices and
848 * we can't do that because we're running in the SoA mode, which means that
849 * our storing routines will keep getting called on channels that have
850 * overflown.
851 * So we need some scratch area where we can keep writing the overflown
852 * vertices without overwriting anything important or crashing.
853 */
854 gs->primitive_boundary = gs->max_output_vertices + 1;
855
856 gs->position_output = -1;
857 for (i = 0; i < gs->info.num_outputs; i++) {
858 if (gs->info.output_semantic_name[i] == TGSI_SEMANTIC_POSITION &&
859 gs->info.output_semantic_index[i] == 0)
860 gs->position_output = i;
861 if (gs->info.output_semantic_name[i] == TGSI_SEMANTIC_VIEWPORT_INDEX)
862 gs->viewport_index_output = i;
863 if (gs->info.output_semantic_name[i] == TGSI_SEMANTIC_CLIPDIST) {
864 debug_assert(gs->info.output_semantic_index[i] <
865 PIPE_MAX_CLIP_OR_CULL_DISTANCE_ELEMENT_COUNT);
866 gs->ccdistance_output[gs->info.output_semantic_index[i]] = i;
867 }
868 }
869
870 gs->machine = draw->gs.tgsi.machine;
871
872 gs->num_vertex_streams = 1;
873 for (i = 0; i < gs->state.stream_output.num_outputs; i++) {
874 if (gs->state.stream_output.output[i].stream >= gs->num_vertex_streams)
875 gs->num_vertex_streams = gs->state.stream_output.output[i].stream + 1;
876 }
877
878 #ifdef LLVM_AVAILABLE
879 if (use_llvm) {
880 int vector_size = gs->vector_length * sizeof(float);
881 gs->gs_input = align_malloc(sizeof(struct draw_gs_inputs), 16);
882 memset(gs->gs_input, 0, sizeof(struct draw_gs_inputs));
883 gs->llvm_prim_lengths = 0;
884
885 gs->llvm_emitted_primitives = align_malloc(vector_size * gs->num_vertex_streams, vector_size);
886 gs->llvm_emitted_vertices = align_malloc(vector_size * gs->num_vertex_streams, vector_size);
887 gs->llvm_prim_ids = align_calloc(vector_size, vector_size);
888
889 gs->fetch_outputs = llvm_fetch_gs_outputs;
890 gs->fetch_inputs = llvm_fetch_gs_input;
891 gs->prepare = llvm_gs_prepare;
892 gs->run = llvm_gs_run;
893
894 gs->jit_context = &draw->llvm->gs_jit_context;
895
896
897 llvm_gs->variant_key_size =
898 draw_gs_llvm_variant_key_size(
899 MAX2(gs->info.file_max[TGSI_FILE_SAMPLER]+1,
900 gs->info.file_max[TGSI_FILE_SAMPLER_VIEW]+1),
901 gs->info.file_max[TGSI_FILE_IMAGE]+1);
902 } else
903 #endif
904 {
905 gs->fetch_outputs = tgsi_fetch_gs_outputs;
906 gs->fetch_inputs = tgsi_fetch_gs_input;
907 gs->prepare = tgsi_gs_prepare;
908 gs->run = tgsi_gs_run;
909 }
910
911 return gs;
912 }
913
draw_bind_geometry_shader(struct draw_context * draw,struct draw_geometry_shader * dgs)914 void draw_bind_geometry_shader(struct draw_context *draw,
915 struct draw_geometry_shader *dgs)
916 {
917 draw_do_flush(draw, DRAW_FLUSH_STATE_CHANGE);
918
919 if (dgs) {
920 draw->gs.geometry_shader = dgs;
921 draw->gs.num_gs_outputs = dgs->info.num_outputs;
922 draw->gs.position_output = dgs->position_output;
923 draw_geometry_shader_prepare(dgs, draw);
924 }
925 else {
926 draw->gs.geometry_shader = NULL;
927 draw->gs.num_gs_outputs = 0;
928 }
929 }
930
draw_delete_geometry_shader(struct draw_context * draw,struct draw_geometry_shader * dgs)931 void draw_delete_geometry_shader(struct draw_context *draw,
932 struct draw_geometry_shader *dgs)
933 {
934 int i;
935 if (!dgs) {
936 return;
937 }
938 #ifdef LLVM_AVAILABLE
939 if (draw->llvm) {
940 struct llvm_geometry_shader *shader = llvm_geometry_shader(dgs);
941 struct draw_gs_llvm_variant_list_item *li;
942
943 li = first_elem(&shader->variants);
944 while(!at_end(&shader->variants, li)) {
945 struct draw_gs_llvm_variant_list_item *next = next_elem(li);
946 draw_gs_llvm_destroy_variant(li->base);
947 li = next;
948 }
949
950 assert(shader->variants_cached == 0);
951
952 if (dgs->llvm_prim_lengths) {
953 unsigned i;
954 for (i = 0; i < dgs->num_vertex_streams * dgs->max_out_prims; ++i) {
955 align_free(dgs->llvm_prim_lengths[i]);
956 }
957 FREE(dgs->llvm_prim_lengths);
958 }
959 align_free(dgs->llvm_emitted_primitives);
960 align_free(dgs->llvm_emitted_vertices);
961 align_free(dgs->llvm_prim_ids);
962
963 align_free(dgs->gs_input);
964 }
965 #endif
966
967 if (draw->gs.tgsi.machine && draw->gs.tgsi.machine->Tokens == dgs->state.tokens)
968 draw->gs.tgsi.machine->Tokens = NULL;
969
970 for (i = 0; i < TGSI_MAX_VERTEX_STREAMS; i++)
971 FREE(dgs->stream[i].primitive_lengths);
972
973 if (dgs->state.ir.nir)
974 ralloc_free(dgs->state.ir.nir);
975 FREE((void*) dgs->state.tokens);
976 FREE(dgs);
977 }
978
979
980 #ifdef LLVM_AVAILABLE
draw_gs_set_current_variant(struct draw_geometry_shader * shader,struct draw_gs_llvm_variant * variant)981 void draw_gs_set_current_variant(struct draw_geometry_shader *shader,
982 struct draw_gs_llvm_variant *variant)
983 {
984 shader->current_variant = variant;
985 }
986 #endif
987
988 /*
989 * Called at the very begin of the draw call with a new instance
990 * Used to reset state that should persist between primitive restart.
991 */
992 void
draw_geometry_shader_new_instance(struct draw_geometry_shader * gs)993 draw_geometry_shader_new_instance(struct draw_geometry_shader *gs)
994 {
995 if (!gs)
996 return;
997
998 gs->in_prim_idx = 0;
999 }
1000