• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright © 2013 Intel Corporation
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the next
12  * paragraph) shall be included in all copies or substantial portions of the
13  * Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21  * DEALINGS IN THE SOFTWARE.
22  */
23 
24 /**
25  * \file brw_vec4_gs_visitor.cpp
26  *
27  * Geometry-shader-specific code derived from the vec4_visitor class.
28  */
29 
30 #include "brw_vec4_gs_visitor.h"
31 #include "gen6_gs_visitor.h"
32 #include "brw_fs.h"
33 #include "brw_nir.h"
34 
35 namespace brw {
36 
vec4_gs_visitor(const struct brw_compiler * compiler,void * log_data,struct brw_gs_compile * c,struct brw_gs_prog_data * prog_data,const nir_shader * shader,void * mem_ctx,bool no_spills,int shader_time_index)37 vec4_gs_visitor::vec4_gs_visitor(const struct brw_compiler *compiler,
38                                  void *log_data,
39                                  struct brw_gs_compile *c,
40                                  struct brw_gs_prog_data *prog_data,
41                                  const nir_shader *shader,
42                                  void *mem_ctx,
43                                  bool no_spills,
44                                  int shader_time_index)
45    : vec4_visitor(compiler, log_data, &c->key.tex,
46                   &prog_data->base, shader,  mem_ctx,
47                   no_spills, shader_time_index),
48      c(c),
49      gs_prog_data(prog_data)
50 {
51 }
52 
53 
54 dst_reg *
make_reg_for_system_value(int location)55 vec4_gs_visitor::make_reg_for_system_value(int location)
56 {
57    dst_reg *reg = new(mem_ctx) dst_reg(this, glsl_type::int_type);
58 
59    switch (location) {
60    case SYSTEM_VALUE_INVOCATION_ID:
61       this->current_annotation = "initialize gl_InvocationID";
62       if (gs_prog_data->invocations > 1)
63          emit(GS_OPCODE_GET_INSTANCE_ID, *reg);
64       else
65          emit(MOV(*reg, brw_imm_ud(0)));
66       break;
67    default:
68       unreachable("not reached");
69    }
70 
71    return reg;
72 }
73 
74 
75 int
setup_varying_inputs(int payload_reg,int * attribute_map,int attributes_per_reg)76 vec4_gs_visitor::setup_varying_inputs(int payload_reg, int *attribute_map,
77                                       int attributes_per_reg)
78 {
79    /* For geometry shaders there are N copies of the input attributes, where N
80     * is the number of input vertices.  attribute_map[BRW_VARYING_SLOT_COUNT *
81     * i + j] represents attribute j for vertex i.
82     *
83     * Note that GS inputs are read from the VUE 256 bits (2 vec4's) at a time,
84     * so the total number of input slots that will be delivered to the GS (and
85     * thus the stride of the input arrays) is urb_read_length * 2.
86     */
87    const unsigned num_input_vertices = nir->info->gs.vertices_in;
88    assert(num_input_vertices <= MAX_GS_INPUT_VERTICES);
89    unsigned input_array_stride = prog_data->urb_read_length * 2;
90 
91    for (int slot = 0; slot < c->input_vue_map.num_slots; slot++) {
92       int varying = c->input_vue_map.slot_to_varying[slot];
93       for (unsigned vertex = 0; vertex < num_input_vertices; vertex++) {
94          attribute_map[BRW_VARYING_SLOT_COUNT * vertex + varying] =
95             attributes_per_reg * payload_reg + input_array_stride * vertex +
96             slot;
97       }
98    }
99 
100    int regs_used = ALIGN(input_array_stride * num_input_vertices,
101                          attributes_per_reg) / attributes_per_reg;
102    return payload_reg + regs_used;
103 }
104 
105 
106 void
setup_payload()107 vec4_gs_visitor::setup_payload()
108 {
109    int attribute_map[BRW_VARYING_SLOT_COUNT * MAX_GS_INPUT_VERTICES];
110 
111    /* If we are in dual instanced or single mode, then attributes are going
112     * to be interleaved, so one register contains two attribute slots.
113     */
114    int attributes_per_reg =
115       prog_data->dispatch_mode == DISPATCH_MODE_4X2_DUAL_OBJECT ? 1 : 2;
116 
117    /* If a geometry shader tries to read from an input that wasn't written by
118     * the vertex shader, that produces undefined results, but it shouldn't
119     * crash anything.  So initialize attribute_map to zeros--that ensures that
120     * these undefined results are read from r0.
121     */
122    memset(attribute_map, 0, sizeof(attribute_map));
123 
124    int reg = 0;
125 
126    /* The payload always contains important data in r0, which contains
127     * the URB handles that are passed on to the URB write at the end
128     * of the thread.
129     */
130    reg++;
131 
132    /* If the shader uses gl_PrimitiveIDIn, that goes in r1. */
133    if (gs_prog_data->include_primitive_id)
134       attribute_map[VARYING_SLOT_PRIMITIVE_ID] = attributes_per_reg * reg++;
135 
136    reg = setup_uniforms(reg);
137 
138    reg = setup_varying_inputs(reg, attribute_map, attributes_per_reg);
139 
140    lower_attributes_to_hw_regs(attribute_map, attributes_per_reg > 1);
141 
142    this->first_non_payload_grf = reg;
143 }
144 
145 
146 void
emit_prolog()147 vec4_gs_visitor::emit_prolog()
148 {
149    /* In vertex shaders, r0.2 is guaranteed to be initialized to zero.  In
150     * geometry shaders, it isn't (it contains a bunch of information we don't
151     * need, like the input primitive type).  We need r0.2 to be zero in order
152     * to build scratch read/write messages correctly (otherwise this value
153     * will be interpreted as a global offset, causing us to do our scratch
154     * reads/writes to garbage memory).  So just set it to zero at the top of
155     * the shader.
156     */
157    this->current_annotation = "clear r0.2";
158    dst_reg r0(retype(brw_vec4_grf(0, 0), BRW_REGISTER_TYPE_UD));
159    vec4_instruction *inst = emit(GS_OPCODE_SET_DWORD_2, r0, brw_imm_ud(0u));
160    inst->force_writemask_all = true;
161 
162    /* Create a virtual register to hold the vertex count */
163    this->vertex_count = src_reg(this, glsl_type::uint_type);
164 
165    /* Initialize the vertex_count register to 0 */
166    this->current_annotation = "initialize vertex_count";
167    inst = emit(MOV(dst_reg(this->vertex_count), brw_imm_ud(0u)));
168    inst->force_writemask_all = true;
169 
170    if (c->control_data_header_size_bits > 0) {
171       /* Create a virtual register to hold the current set of control data
172        * bits.
173        */
174       this->control_data_bits = src_reg(this, glsl_type::uint_type);
175 
176       /* If we're outputting more than 32 control data bits, then EmitVertex()
177        * will set control_data_bits to 0 after emitting the first vertex.
178        * Otherwise, we need to initialize it to 0 here.
179        */
180       if (c->control_data_header_size_bits <= 32) {
181          this->current_annotation = "initialize control data bits";
182          inst = emit(MOV(dst_reg(this->control_data_bits), brw_imm_ud(0u)));
183          inst->force_writemask_all = true;
184       }
185    }
186 
187    this->current_annotation = NULL;
188 }
189 
190 void
emit_thread_end()191 vec4_gs_visitor::emit_thread_end()
192 {
193    if (c->control_data_header_size_bits > 0) {
194       /* During shader execution, we only ever call emit_control_data_bits()
195        * just prior to outputting a vertex.  Therefore, the control data bits
196        * corresponding to the most recently output vertex still need to be
197        * emitted.
198        */
199       current_annotation = "thread end: emit control data bits";
200       emit_control_data_bits();
201    }
202 
203    /* MRF 0 is reserved for the debugger, so start with message header
204     * in MRF 1.
205     */
206    int base_mrf = 1;
207 
208    bool static_vertex_count = gs_prog_data->static_vertex_count != -1;
209 
210    /* If the previous instruction was a URB write, we don't need to issue
211     * a second one - we can just set the EOT bit on the previous write.
212     *
213     * Skip this on Gen8+ unless there's a static vertex count, as we also
214     * need to write the vertex count out, and combining the two may not be
215     * possible (or at least not straightforward).
216     */
217    vec4_instruction *last = (vec4_instruction *) instructions.get_tail();
218    if (last && last->opcode == GS_OPCODE_URB_WRITE &&
219        !(INTEL_DEBUG & DEBUG_SHADER_TIME) &&
220        devinfo->gen >= 8 && static_vertex_count) {
221       last->urb_write_flags = BRW_URB_WRITE_EOT | last->urb_write_flags;
222       return;
223    }
224 
225    current_annotation = "thread end";
226    dst_reg mrf_reg(MRF, base_mrf);
227    src_reg r0(retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD));
228    vec4_instruction *inst = emit(MOV(mrf_reg, r0));
229    inst->force_writemask_all = true;
230    if (devinfo->gen < 8 || !static_vertex_count)
231       emit(GS_OPCODE_SET_VERTEX_COUNT, mrf_reg, this->vertex_count);
232    if (INTEL_DEBUG & DEBUG_SHADER_TIME)
233       emit_shader_time_end();
234    inst = emit(GS_OPCODE_THREAD_END);
235    inst->base_mrf = base_mrf;
236    inst->mlen = devinfo->gen >= 8 && !static_vertex_count ? 2 : 1;
237 }
238 
239 
240 void
emit_urb_write_header(int mrf)241 vec4_gs_visitor::emit_urb_write_header(int mrf)
242 {
243    /* The SEND instruction that writes the vertex data to the VUE will use
244     * per_slot_offset=true, which means that DWORDs 3 and 4 of the message
245     * header specify an offset (in multiples of 256 bits) into the URB entry
246     * at which the write should take place.
247     *
248     * So we have to prepare a message header with the appropriate offset
249     * values.
250     */
251    dst_reg mrf_reg(MRF, mrf);
252    src_reg r0(retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD));
253    this->current_annotation = "URB write header";
254    vec4_instruction *inst = emit(MOV(mrf_reg, r0));
255    inst->force_writemask_all = true;
256    emit(GS_OPCODE_SET_WRITE_OFFSET, mrf_reg, this->vertex_count,
257         brw_imm_ud(gs_prog_data->output_vertex_size_hwords));
258 }
259 
260 
261 vec4_instruction *
emit_urb_write_opcode(bool complete)262 vec4_gs_visitor::emit_urb_write_opcode(bool complete)
263 {
264    /* We don't care whether the vertex is complete, because in general
265     * geometry shaders output multiple vertices, and we don't terminate the
266     * thread until all vertices are complete.
267     */
268    (void) complete;
269 
270    vec4_instruction *inst = emit(GS_OPCODE_URB_WRITE);
271    inst->offset = gs_prog_data->control_data_header_size_hwords;
272 
273    /* We need to increment Global Offset by 1 to make room for Broadwell's
274     * extra "Vertex Count" payload at the beginning of the URB entry.
275     */
276    if (devinfo->gen >= 8 && gs_prog_data->static_vertex_count == -1)
277       inst->offset++;
278 
279    inst->urb_write_flags = BRW_URB_WRITE_PER_SLOT_OFFSET;
280    return inst;
281 }
282 
283 
284 /**
285  * Write out a batch of 32 control data bits from the control_data_bits
286  * register to the URB.
287  *
288  * The current value of the vertex_count register determines which DWORD in
289  * the URB receives the control data bits.  The control_data_bits register is
290  * assumed to contain the correct data for the vertex that was most recently
291  * output, and all previous vertices that share the same DWORD.
292  *
293  * This function takes care of ensuring that if no vertices have been output
294  * yet, no control bits are emitted.
295  */
296 void
emit_control_data_bits()297 vec4_gs_visitor::emit_control_data_bits()
298 {
299    assert(c->control_data_bits_per_vertex != 0);
300 
301    /* Since the URB_WRITE_OWORD message operates with 128-bit (vec4 sized)
302     * granularity, we need to use two tricks to ensure that the batch of 32
303     * control data bits is written to the appropriate DWORD in the URB.  To
304     * select which vec4 we are writing to, we use the "slot {0,1} offset"
305     * fields of the message header.  To select which DWORD in the vec4 we are
306     * writing to, we use the channel mask fields of the message header.  To
307     * avoid penalizing geometry shaders that emit a small number of vertices
308     * with extra bookkeeping, we only do each of these tricks when
309     * c->prog_data.control_data_header_size_bits is large enough to make it
310     * necessary.
311     *
312     * Note: this means that if we're outputting just a single DWORD of control
313     * data bits, we'll actually replicate it four times since we won't do any
314     * channel masking.  But that's not a problem since in this case the
315     * hardware only pays attention to the first DWORD.
316     */
317    enum brw_urb_write_flags urb_write_flags = BRW_URB_WRITE_OWORD;
318    if (c->control_data_header_size_bits > 32)
319       urb_write_flags = urb_write_flags | BRW_URB_WRITE_USE_CHANNEL_MASKS;
320    if (c->control_data_header_size_bits > 128)
321       urb_write_flags = urb_write_flags | BRW_URB_WRITE_PER_SLOT_OFFSET;
322 
323    /* If we are using either channel masks or a per-slot offset, then we
324     * need to figure out which DWORD we are trying to write to, using the
325     * formula:
326     *
327     *     dword_index = (vertex_count - 1) * bits_per_vertex / 32
328     *
329     * Since bits_per_vertex is a power of two, and is known at compile
330     * time, this can be optimized to:
331     *
332     *     dword_index = (vertex_count - 1) >> (6 - log2(bits_per_vertex))
333     */
334    src_reg dword_index(this, glsl_type::uint_type);
335    if (urb_write_flags) {
336       src_reg prev_count(this, glsl_type::uint_type);
337       emit(ADD(dst_reg(prev_count), this->vertex_count,
338                brw_imm_ud(0xffffffffu)));
339       unsigned log2_bits_per_vertex =
340          util_last_bit(c->control_data_bits_per_vertex);
341       emit(SHR(dst_reg(dword_index), prev_count,
342                brw_imm_ud(6 - log2_bits_per_vertex)));
343    }
344 
345    /* Start building the URB write message.  The first MRF gets a copy of
346     * R0.
347     */
348    int base_mrf = 1;
349    dst_reg mrf_reg(MRF, base_mrf);
350    src_reg r0(retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD));
351    vec4_instruction *inst = emit(MOV(mrf_reg, r0));
352    inst->force_writemask_all = true;
353 
354    if (urb_write_flags & BRW_URB_WRITE_PER_SLOT_OFFSET) {
355       /* Set the per-slot offset to dword_index / 4, to that we'll write to
356        * the appropriate OWORD within the control data header.
357        */
358       src_reg per_slot_offset(this, glsl_type::uint_type);
359       emit(SHR(dst_reg(per_slot_offset), dword_index, brw_imm_ud(2u)));
360       emit(GS_OPCODE_SET_WRITE_OFFSET, mrf_reg, per_slot_offset,
361            brw_imm_ud(1u));
362    }
363 
364    if (urb_write_flags & BRW_URB_WRITE_USE_CHANNEL_MASKS) {
365       /* Set the channel masks to 1 << (dword_index % 4), so that we'll
366        * write to the appropriate DWORD within the OWORD.  We need to do
367        * this computation with force_writemask_all, otherwise garbage data
368        * from invocation 0 might clobber the mask for invocation 1 when
369        * GS_OPCODE_PREPARE_CHANNEL_MASKS tries to OR the two masks
370        * together.
371        */
372       src_reg channel(this, glsl_type::uint_type);
373       inst = emit(AND(dst_reg(channel), dword_index, brw_imm_ud(3u)));
374       inst->force_writemask_all = true;
375       src_reg one(this, glsl_type::uint_type);
376       inst = emit(MOV(dst_reg(one), brw_imm_ud(1u)));
377       inst->force_writemask_all = true;
378       src_reg channel_mask(this, glsl_type::uint_type);
379       inst = emit(SHL(dst_reg(channel_mask), one, channel));
380       inst->force_writemask_all = true;
381       emit(GS_OPCODE_PREPARE_CHANNEL_MASKS, dst_reg(channel_mask),
382                                             channel_mask);
383       emit(GS_OPCODE_SET_CHANNEL_MASKS, mrf_reg, channel_mask);
384    }
385 
386    /* Store the control data bits in the message payload and send it. */
387    dst_reg mrf_reg2(MRF, base_mrf + 1);
388    inst = emit(MOV(mrf_reg2, this->control_data_bits));
389    inst->force_writemask_all = true;
390    inst = emit(GS_OPCODE_URB_WRITE);
391    inst->urb_write_flags = urb_write_flags;
392    /* We need to increment Global Offset by 256-bits to make room for
393     * Broadwell's extra "Vertex Count" payload at the beginning of the
394     * URB entry.  Since this is an OWord message, Global Offset is counted
395     * in 128-bit units, so we must set it to 2.
396     */
397    if (devinfo->gen >= 8 && gs_prog_data->static_vertex_count == -1)
398       inst->offset = 2;
399    inst->base_mrf = base_mrf;
400    inst->mlen = 2;
401 }
402 
403 void
set_stream_control_data_bits(unsigned stream_id)404 vec4_gs_visitor::set_stream_control_data_bits(unsigned stream_id)
405 {
406    /* control_data_bits |= stream_id << ((2 * (vertex_count - 1)) % 32) */
407 
408    /* Note: we are calling this *before* increasing vertex_count, so
409     * this->vertex_count == vertex_count - 1 in the formula above.
410     */
411 
412    /* Stream mode uses 2 bits per vertex */
413    assert(c->control_data_bits_per_vertex == 2);
414 
415    /* Must be a valid stream */
416    assert(stream_id >= 0 && stream_id < MAX_VERTEX_STREAMS);
417 
418    /* Control data bits are initialized to 0 so we don't have to set any
419     * bits when sending vertices to stream 0.
420     */
421    if (stream_id == 0)
422       return;
423 
424    /* reg::sid = stream_id */
425    src_reg sid(this, glsl_type::uint_type);
426    emit(MOV(dst_reg(sid), brw_imm_ud(stream_id)));
427 
428    /* reg:shift_count = 2 * (vertex_count - 1) */
429    src_reg shift_count(this, glsl_type::uint_type);
430    emit(SHL(dst_reg(shift_count), this->vertex_count, brw_imm_ud(1u)));
431 
432    /* Note: we're relying on the fact that the GEN SHL instruction only pays
433     * attention to the lower 5 bits of its second source argument, so on this
434     * architecture, stream_id << 2 * (vertex_count - 1) is equivalent to
435     * stream_id << ((2 * (vertex_count - 1)) % 32).
436     */
437    src_reg mask(this, glsl_type::uint_type);
438    emit(SHL(dst_reg(mask), sid, shift_count));
439    emit(OR(dst_reg(this->control_data_bits), this->control_data_bits, mask));
440 }
441 
442 void
gs_emit_vertex(int stream_id)443 vec4_gs_visitor::gs_emit_vertex(int stream_id)
444 {
445    this->current_annotation = "emit vertex: safety check";
446 
447    /* Haswell and later hardware ignores the "Render Stream Select" bits
448     * from the 3DSTATE_STREAMOUT packet when the SOL stage is disabled,
449     * and instead sends all primitives down the pipeline for rasterization.
450     * If the SOL stage is enabled, "Render Stream Select" is honored and
451     * primitives bound to non-zero streams are discarded after stream output.
452     *
453     * Since the only purpose of primives sent to non-zero streams is to
454     * be recorded by transform feedback, we can simply discard all geometry
455     * bound to these streams when transform feedback is disabled.
456     */
457    if (stream_id > 0 && !nir->info->has_transform_feedback_varyings)
458       return;
459 
460    /* If we're outputting 32 control data bits or less, then we can wait
461     * until the shader is over to output them all.  Otherwise we need to
462     * output them as we go.  Now is the time to do it, since we're about to
463     * output the vertex_count'th vertex, so it's guaranteed that the
464     * control data bits associated with the (vertex_count - 1)th vertex are
465     * correct.
466     */
467    if (c->control_data_header_size_bits > 32) {
468       this->current_annotation = "emit vertex: emit control data bits";
469       /* Only emit control data bits if we've finished accumulating a batch
470        * of 32 bits.  This is the case when:
471        *
472        *     (vertex_count * bits_per_vertex) % 32 == 0
473        *
474        * (in other words, when the last 5 bits of vertex_count *
475        * bits_per_vertex are 0).  Assuming bits_per_vertex == 2^n for some
476        * integer n (which is always the case, since bits_per_vertex is
477        * always 1 or 2), this is equivalent to requiring that the last 5-n
478        * bits of vertex_count are 0:
479        *
480        *     vertex_count & (2^(5-n) - 1) == 0
481        *
482        * 2^(5-n) == 2^5 / 2^n == 32 / bits_per_vertex, so this is
483        * equivalent to:
484        *
485        *     vertex_count & (32 / bits_per_vertex - 1) == 0
486        */
487       vec4_instruction *inst =
488          emit(AND(dst_null_ud(), this->vertex_count,
489                   brw_imm_ud(32 / c->control_data_bits_per_vertex - 1)));
490       inst->conditional_mod = BRW_CONDITIONAL_Z;
491 
492       emit(IF(BRW_PREDICATE_NORMAL));
493       {
494          /* If vertex_count is 0, then no control data bits have been
495           * accumulated yet, so we skip emitting them.
496           */
497          emit(CMP(dst_null_ud(), this->vertex_count, brw_imm_ud(0u),
498                   BRW_CONDITIONAL_NEQ));
499          emit(IF(BRW_PREDICATE_NORMAL));
500          emit_control_data_bits();
501          emit(BRW_OPCODE_ENDIF);
502 
503          /* Reset control_data_bits to 0 so we can start accumulating a new
504           * batch.
505           *
506           * Note: in the case where vertex_count == 0, this neutralizes the
507           * effect of any call to EndPrimitive() that the shader may have
508           * made before outputting its first vertex.
509           */
510          inst = emit(MOV(dst_reg(this->control_data_bits), brw_imm_ud(0u)));
511          inst->force_writemask_all = true;
512       }
513       emit(BRW_OPCODE_ENDIF);
514    }
515 
516    this->current_annotation = "emit vertex: vertex data";
517    emit_vertex();
518 
519    /* In stream mode we have to set control data bits for all vertices
520     * unless we have disabled control data bits completely (which we do
521     * do for GL_POINTS outputs that don't use streams).
522     */
523    if (c->control_data_header_size_bits > 0 &&
524        gs_prog_data->control_data_format ==
525           GEN7_GS_CONTROL_DATA_FORMAT_GSCTL_SID) {
526        this->current_annotation = "emit vertex: Stream control data bits";
527        set_stream_control_data_bits(stream_id);
528    }
529 
530    this->current_annotation = NULL;
531 }
532 
533 void
gs_end_primitive()534 vec4_gs_visitor::gs_end_primitive()
535 {
536    /* We can only do EndPrimitive() functionality when the control data
537     * consists of cut bits.  Fortunately, the only time it isn't is when the
538     * output type is points, in which case EndPrimitive() is a no-op.
539     */
540    if (gs_prog_data->control_data_format !=
541        GEN7_GS_CONTROL_DATA_FORMAT_GSCTL_CUT) {
542       return;
543    }
544 
545    if (c->control_data_header_size_bits == 0)
546       return;
547 
548    /* Cut bits use one bit per vertex. */
549    assert(c->control_data_bits_per_vertex == 1);
550 
551    /* Cut bit n should be set to 1 if EndPrimitive() was called after emitting
552     * vertex n, 0 otherwise.  So all we need to do here is mark bit
553     * (vertex_count - 1) % 32 in the cut_bits register to indicate that
554     * EndPrimitive() was called after emitting vertex (vertex_count - 1);
555     * vec4_gs_visitor::emit_control_data_bits() will take care of the rest.
556     *
557     * Note that if EndPrimitve() is called before emitting any vertices, this
558     * will cause us to set bit 31 of the control_data_bits register to 1.
559     * That's fine because:
560     *
561     * - If max_vertices < 32, then vertex number 31 (zero-based) will never be
562     *   output, so the hardware will ignore cut bit 31.
563     *
564     * - If max_vertices == 32, then vertex number 31 is guaranteed to be the
565     *   last vertex, so setting cut bit 31 has no effect (since the primitive
566     *   is automatically ended when the GS terminates).
567     *
568     * - If max_vertices > 32, then the ir_emit_vertex visitor will reset the
569     *   control_data_bits register to 0 when the first vertex is emitted.
570     */
571 
572    /* control_data_bits |= 1 << ((vertex_count - 1) % 32) */
573    src_reg one(this, glsl_type::uint_type);
574    emit(MOV(dst_reg(one), brw_imm_ud(1u)));
575    src_reg prev_count(this, glsl_type::uint_type);
576    emit(ADD(dst_reg(prev_count), this->vertex_count, brw_imm_ud(0xffffffffu)));
577    src_reg mask(this, glsl_type::uint_type);
578    /* Note: we're relying on the fact that the GEN SHL instruction only pays
579     * attention to the lower 5 bits of its second source argument, so on this
580     * architecture, 1 << (vertex_count - 1) is equivalent to 1 <<
581     * ((vertex_count - 1) % 32).
582     */
583    emit(SHL(dst_reg(mask), one, prev_count));
584    emit(OR(dst_reg(this->control_data_bits), this->control_data_bits, mask));
585 }
586 
587 extern "C" const unsigned *
brw_compile_gs(const struct brw_compiler * compiler,void * log_data,void * mem_ctx,const struct brw_gs_prog_key * key,struct brw_gs_prog_data * prog_data,const nir_shader * src_shader,struct gl_program * prog,int shader_time_index,unsigned * final_assembly_size,char ** error_str)588 brw_compile_gs(const struct brw_compiler *compiler, void *log_data,
589                void *mem_ctx,
590                const struct brw_gs_prog_key *key,
591                struct brw_gs_prog_data *prog_data,
592                const nir_shader *src_shader,
593                struct gl_program *prog,
594                int shader_time_index,
595                unsigned *final_assembly_size,
596                char **error_str)
597 {
598    struct brw_gs_compile c;
599    memset(&c, 0, sizeof(c));
600    c.key = *key;
601 
602    const bool is_scalar = compiler->scalar_stage[MESA_SHADER_GEOMETRY];
603    nir_shader *shader = nir_shader_clone(mem_ctx, src_shader);
604 
605    /* The GLSL linker will have already matched up GS inputs and the outputs
606     * of prior stages.  The driver does extend VS outputs in some cases, but
607     * only for legacy OpenGL or Gen4-5 hardware, neither of which offer
608     * geometry shader support.  So we can safely ignore that.
609     *
610     * For SSO pipelines, we use a fixed VUE map layout based on variable
611     * locations, so we can rely on rendezvous-by-location making this work.
612     */
613    GLbitfield64 inputs_read = shader->info->inputs_read;
614    brw_compute_vue_map(compiler->devinfo,
615                        &c.input_vue_map, inputs_read,
616                        shader->info->separate_shader);
617 
618    shader = brw_nir_apply_sampler_key(shader, compiler, &key->tex, is_scalar);
619    brw_nir_lower_vue_inputs(shader, is_scalar, &c.input_vue_map);
620    brw_nir_lower_vue_outputs(shader, is_scalar);
621    shader = brw_postprocess_nir(shader, compiler, is_scalar);
622 
623    prog_data->base.clip_distance_mask =
624       ((1 << shader->info->clip_distance_array_size) - 1);
625    prog_data->base.cull_distance_mask =
626       ((1 << shader->info->cull_distance_array_size) - 1) <<
627       shader->info->clip_distance_array_size;
628 
629    prog_data->include_primitive_id =
630       (shader->info->system_values_read & (1 << SYSTEM_VALUE_PRIMITIVE_ID)) != 0;
631 
632    prog_data->invocations = shader->info->gs.invocations;
633 
634    if (compiler->devinfo->gen >= 8)
635       prog_data->static_vertex_count = nir_gs_count_vertices(shader);
636 
637    if (compiler->devinfo->gen >= 7) {
638       if (shader->info->gs.output_primitive == GL_POINTS) {
639          /* When the output type is points, the geometry shader may output data
640           * to multiple streams, and EndPrimitive() has no effect.  So we
641           * configure the hardware to interpret the control data as stream ID.
642           */
643          prog_data->control_data_format = GEN7_GS_CONTROL_DATA_FORMAT_GSCTL_SID;
644 
645          /* We only have to emit control bits if we are using streams */
646          if (prog && prog->info.gs.uses_streams)
647             c.control_data_bits_per_vertex = 2;
648          else
649             c.control_data_bits_per_vertex = 0;
650       } else {
651          /* When the output type is triangle_strip or line_strip, EndPrimitive()
652           * may be used to terminate the current strip and start a new one
653           * (similar to primitive restart), and outputting data to multiple
654           * streams is not supported.  So we configure the hardware to interpret
655           * the control data as EndPrimitive information (a.k.a. "cut bits").
656           */
657          prog_data->control_data_format = GEN7_GS_CONTROL_DATA_FORMAT_GSCTL_CUT;
658 
659          /* We only need to output control data if the shader actually calls
660           * EndPrimitive().
661           */
662          c.control_data_bits_per_vertex =
663             shader->info->gs.uses_end_primitive ? 1 : 0;
664       }
665    } else {
666       /* There are no control data bits in gen6. */
667       c.control_data_bits_per_vertex = 0;
668    }
669    c.control_data_header_size_bits =
670       shader->info->gs.vertices_out * c.control_data_bits_per_vertex;
671 
672    /* 1 HWORD = 32 bytes = 256 bits */
673    prog_data->control_data_header_size_hwords =
674       ALIGN(c.control_data_header_size_bits, 256) / 256;
675 
676    /* Compute the output vertex size.
677     *
678     * From the Ivy Bridge PRM, Vol2 Part1 7.2.1.1 STATE_GS - Output Vertex
679     * Size (p168):
680     *
681     *     [0,62] indicating [1,63] 16B units
682     *
683     *     Specifies the size of each vertex stored in the GS output entry
684     *     (following any Control Header data) as a number of 128-bit units
685     *     (minus one).
686     *
687     *     Programming Restrictions: The vertex size must be programmed as a
688     *     multiple of 32B units with the following exception: Rendering is
689     *     disabled (as per SOL stage state) and the vertex size output by the
690     *     GS thread is 16B.
691     *
692     *     If rendering is enabled (as per SOL state) the vertex size must be
693     *     programmed as a multiple of 32B units. In other words, the only time
694     *     software can program a vertex size with an odd number of 16B units
695     *     is when rendering is disabled.
696     *
697     * Note: B=bytes in the above text.
698     *
699     * It doesn't seem worth the extra trouble to optimize the case where the
700     * vertex size is 16B (especially since this would require special-casing
701     * the GEN assembly that writes to the URB).  So we just set the vertex
702     * size to a multiple of 32B (2 vec4's) in all cases.
703     *
704     * The maximum output vertex size is 62*16 = 992 bytes (31 hwords).  We
705     * budget that as follows:
706     *
707     *   512 bytes for varyings (a varying component is 4 bytes and
708     *             gl_MaxGeometryOutputComponents = 128)
709     *    16 bytes overhead for VARYING_SLOT_PSIZ (each varying slot is 16
710     *             bytes)
711     *    16 bytes overhead for gl_Position (we allocate it a slot in the VUE
712     *             even if it's not used)
713     *    32 bytes overhead for gl_ClipDistance (we allocate it 2 VUE slots
714     *             whenever clip planes are enabled, even if the shader doesn't
715     *             write to gl_ClipDistance)
716     *    16 bytes overhead since the VUE size must be a multiple of 32 bytes
717     *             (see below)--this causes up to 1 VUE slot to be wasted
718     *   400 bytes available for varying packing overhead
719     *
720     * Worst-case varying packing overhead is 3/4 of a varying slot (12 bytes)
721     * per interpolation type, so this is plenty.
722     *
723     */
724    unsigned output_vertex_size_bytes = prog_data->base.vue_map.num_slots * 16;
725    assert(compiler->devinfo->gen == 6 ||
726           output_vertex_size_bytes <= GEN7_MAX_GS_OUTPUT_VERTEX_SIZE_BYTES);
727    prog_data->output_vertex_size_hwords =
728       ALIGN(output_vertex_size_bytes, 32) / 32;
729 
730    /* Compute URB entry size.  The maximum allowed URB entry size is 32k.
731     * That divides up as follows:
732     *
733     *     64 bytes for the control data header (cut indices or StreamID bits)
734     *   4096 bytes for varyings (a varying component is 4 bytes and
735     *              gl_MaxGeometryTotalOutputComponents = 1024)
736     *   4096 bytes overhead for VARYING_SLOT_PSIZ (each varying slot is 16
737     *              bytes/vertex and gl_MaxGeometryOutputVertices is 256)
738     *   4096 bytes overhead for gl_Position (we allocate it a slot in the VUE
739     *              even if it's not used)
740     *   8192 bytes overhead for gl_ClipDistance (we allocate it 2 VUE slots
741     *              whenever clip planes are enabled, even if the shader doesn't
742     *              write to gl_ClipDistance)
743     *   4096 bytes overhead since the VUE size must be a multiple of 32
744     *              bytes (see above)--this causes up to 1 VUE slot to be wasted
745     *   8128 bytes available for varying packing overhead
746     *
747     * Worst-case varying packing overhead is 3/4 of a varying slot per
748     * interpolation type, which works out to 3072 bytes, so this would allow
749     * us to accommodate 2 interpolation types without any danger of running
750     * out of URB space.
751     *
752     * In practice, the risk of running out of URB space is very small, since
753     * the above figures are all worst-case, and most of them scale with the
754     * number of output vertices.  So we'll just calculate the amount of space
755     * we need, and if it's too large, fail to compile.
756     *
757     * The above is for gen7+ where we have a single URB entry that will hold
758     * all the output. In gen6, we will have to allocate URB entries for every
759     * vertex we emit, so our URB entries only need to be large enough to hold
760     * a single vertex. Also, gen6 does not have a control data header.
761     */
762    unsigned output_size_bytes;
763    if (compiler->devinfo->gen >= 7) {
764       output_size_bytes =
765          prog_data->output_vertex_size_hwords * 32 * shader->info->gs.vertices_out;
766       output_size_bytes += 32 * prog_data->control_data_header_size_hwords;
767    } else {
768       output_size_bytes = prog_data->output_vertex_size_hwords * 32;
769    }
770 
771    /* Broadwell stores "Vertex Count" as a full 8 DWord (32 byte) URB output,
772     * which comes before the control header.
773     */
774    if (compiler->devinfo->gen >= 8)
775       output_size_bytes += 32;
776 
777    /* Shaders can technically set max_vertices = 0, at which point we
778     * may have a URB size of 0 bytes.  Nothing good can come from that,
779     * so enforce a minimum size.
780     */
781    if (output_size_bytes == 0)
782       output_size_bytes = 1;
783 
784    unsigned max_output_size_bytes = GEN7_MAX_GS_URB_ENTRY_SIZE_BYTES;
785    if (compiler->devinfo->gen == 6)
786       max_output_size_bytes = GEN6_MAX_GS_URB_ENTRY_SIZE_BYTES;
787    if (output_size_bytes > max_output_size_bytes)
788       return NULL;
789 
790 
791    /* URB entry sizes are stored as a multiple of 64 bytes in gen7+ and
792     * a multiple of 128 bytes in gen6.
793     */
794    if (compiler->devinfo->gen >= 7)
795       prog_data->base.urb_entry_size = ALIGN(output_size_bytes, 64) / 64;
796    else
797       prog_data->base.urb_entry_size = ALIGN(output_size_bytes, 128) / 128;
798 
799    prog_data->output_topology =
800       get_hw_prim_for_gl_prim(shader->info->gs.output_primitive);
801 
802    prog_data->vertices_in = shader->info->gs.vertices_in;
803 
804    /* GS inputs are read from the VUE 256 bits (2 vec4's) at a time, so we
805     * need to program a URB read length of ceiling(num_slots / 2).
806     */
807    prog_data->base.urb_read_length = (c.input_vue_map.num_slots + 1) / 2;
808 
809    /* Now that prog_data setup is done, we are ready to actually compile the
810     * program.
811     */
812    if (unlikely(INTEL_DEBUG & DEBUG_GS)) {
813       fprintf(stderr, "GS Input ");
814       brw_print_vue_map(stderr, &c.input_vue_map);
815       fprintf(stderr, "GS Output ");
816       brw_print_vue_map(stderr, &prog_data->base.vue_map);
817    }
818 
819    if (is_scalar) {
820       fs_visitor v(compiler, log_data, mem_ctx, &c, prog_data, shader,
821                    shader_time_index);
822       if (v.run_gs()) {
823          prog_data->base.dispatch_mode = DISPATCH_MODE_SIMD8;
824          prog_data->base.base.dispatch_grf_start_reg = v.payload.num_regs;
825 
826          fs_generator g(compiler, log_data, mem_ctx, &c.key,
827                         &prog_data->base.base, v.promoted_constants,
828                         false, MESA_SHADER_GEOMETRY);
829          if (unlikely(INTEL_DEBUG & DEBUG_GS)) {
830             const char *label =
831                shader->info->label ? shader->info->label : "unnamed";
832             char *name = ralloc_asprintf(mem_ctx, "%s geometry shader %s",
833                                          label, shader->info->name);
834             g.enable_debug(name);
835          }
836          g.generate_code(v.cfg, 8);
837          return g.get_assembly(final_assembly_size);
838       }
839    }
840 
841    if (compiler->devinfo->gen >= 7) {
842       /* Compile the geometry shader in DUAL_OBJECT dispatch mode, if we can do
843        * so without spilling. If the GS invocations count > 1, then we can't use
844        * dual object mode.
845        */
846       if (prog_data->invocations <= 1 &&
847           likely(!(INTEL_DEBUG & DEBUG_NO_DUAL_OBJECT_GS))) {
848          prog_data->base.dispatch_mode = DISPATCH_MODE_4X2_DUAL_OBJECT;
849 
850          vec4_gs_visitor v(compiler, log_data, &c, prog_data, shader,
851                            mem_ctx, true /* no_spills */, shader_time_index);
852          if (v.run()) {
853             return brw_vec4_generate_assembly(compiler, log_data, mem_ctx,
854                                               shader, &prog_data->base, v.cfg,
855                                               final_assembly_size);
856          }
857       }
858    }
859 
860    /* Either we failed to compile in DUAL_OBJECT mode (probably because it
861     * would have required spilling) or DUAL_OBJECT mode is disabled.  So fall
862     * back to DUAL_INSTANCED or SINGLE mode, which consumes fewer registers.
863     *
864     * FIXME: Single dispatch mode requires that the driver can handle
865     * interleaving of input registers, but this is already supported (dual
866     * instance mode has the same requirement). However, to take full advantage
867     * of single dispatch mode to reduce register pressure we would also need to
868     * do interleaved outputs, but currently, the vec4 visitor and generator
869     * classes do not support this, so at the moment register pressure in
870     * single and dual instance modes is the same.
871     *
872     * From the Ivy Bridge PRM, Vol2 Part1 7.2.1.1 "3DSTATE_GS"
873     * "If InstanceCount>1, DUAL_OBJECT mode is invalid. Software will likely
874     * want to use DUAL_INSTANCE mode for higher performance, but SINGLE mode
875     * is also supported. When InstanceCount=1 (one instance per object) software
876     * can decide which dispatch mode to use. DUAL_OBJECT mode would likely be
877     * the best choice for performance, followed by SINGLE mode."
878     *
879     * So SINGLE mode is more performant when invocations == 1 and DUAL_INSTANCE
880     * mode is more performant when invocations > 1. Gen6 only supports
881     * SINGLE mode.
882     */
883    if (prog_data->invocations <= 1 || compiler->devinfo->gen < 7)
884       prog_data->base.dispatch_mode = DISPATCH_MODE_4X1_SINGLE;
885    else
886       prog_data->base.dispatch_mode = DISPATCH_MODE_4X2_DUAL_INSTANCE;
887 
888    vec4_gs_visitor *gs = NULL;
889    const unsigned *ret = NULL;
890 
891    if (compiler->devinfo->gen >= 7)
892       gs = new vec4_gs_visitor(compiler, log_data, &c, prog_data,
893                                shader, mem_ctx, false /* no_spills */,
894                                shader_time_index);
895    else
896       gs = new gen6_gs_visitor(compiler, log_data, &c, prog_data, prog,
897                                shader, mem_ctx, false /* no_spills */,
898                                shader_time_index);
899 
900    if (!gs->run()) {
901       if (error_str)
902          *error_str = ralloc_strdup(mem_ctx, gs->fail_msg);
903    } else {
904       ret = brw_vec4_generate_assembly(compiler, log_data, mem_ctx, shader,
905                                        &prog_data->base, gs->cfg,
906                                        final_assembly_size);
907    }
908 
909    delete gs;
910    return ret;
911 }
912 
913 
914 } /* namespace brw */
915