• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright © 2010 Intel Corporation
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the next
12  * paragraph) shall be included in all copies or substantial portions of the
13  * Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21  * IN THE SOFTWARE.
22  */
23 
24 /** @file brw_fs_visitor.cpp
25  *
26  * This file supports generating the FS LIR from the GLSL IR.  The LIR
27  * makes it easier to do backend-specific optimizations than doing so
28  * in the GLSL IR or in the native code.
29  */
30 #include "brw_fs.h"
31 #include "compiler/glsl_types.h"
32 
33 using namespace brw;
34 
35 /* Sample from the MCS surface attached to this multisample texture. */
36 fs_reg
emit_mcs_fetch(const fs_reg & coordinate,unsigned components,const fs_reg & texture,const fs_reg & texture_handle)37 fs_visitor::emit_mcs_fetch(const fs_reg &coordinate, unsigned components,
38                            const fs_reg &texture,
39                            const fs_reg &texture_handle)
40 {
41    const fs_reg dest = vgrf(glsl_type::uvec4_type);
42 
43    fs_reg srcs[TEX_LOGICAL_NUM_SRCS];
44    srcs[TEX_LOGICAL_SRC_COORDINATE] = coordinate;
45    srcs[TEX_LOGICAL_SRC_SURFACE] = texture;
46    srcs[TEX_LOGICAL_SRC_SAMPLER] = brw_imm_ud(0);
47    srcs[TEX_LOGICAL_SRC_SURFACE_HANDLE] = texture_handle;
48    srcs[TEX_LOGICAL_SRC_COORD_COMPONENTS] = brw_imm_d(components);
49    srcs[TEX_LOGICAL_SRC_GRAD_COMPONENTS] = brw_imm_d(0);
50 
51    fs_inst *inst = bld.emit(SHADER_OPCODE_TXF_MCS_LOGICAL, dest, srcs,
52                             ARRAY_SIZE(srcs));
53 
54    /* We only care about one or two regs of response, but the sampler always
55     * writes 4/8.
56     */
57    inst->size_written = 4 * dest.component_size(inst->exec_size);
58 
59    return dest;
60 }
61 
62 /**
63  * Apply workarounds for Gen6 gather with UINT/SINT
64  */
65 void
emit_gen6_gather_wa(uint8_t wa,fs_reg dst)66 fs_visitor::emit_gen6_gather_wa(uint8_t wa, fs_reg dst)
67 {
68    if (!wa)
69       return;
70 
71    int width = (wa & WA_8BIT) ? 8 : 16;
72 
73    for (int i = 0; i < 4; i++) {
74       fs_reg dst_f = retype(dst, BRW_REGISTER_TYPE_F);
75       /* Convert from UNORM to UINT */
76       bld.MUL(dst_f, dst_f, brw_imm_f((1 << width) - 1));
77       bld.MOV(dst, dst_f);
78 
79       if (wa & WA_SIGN) {
80          /* Reinterpret the UINT value as a signed INT value by
81           * shifting the sign bit into place, then shifting back
82           * preserving sign.
83           */
84          bld.SHL(dst, dst, brw_imm_d(32 - width));
85          bld.ASR(dst, dst, brw_imm_d(32 - width));
86       }
87 
88       dst = offset(dst, bld, 1);
89    }
90 }
91 
92 /** Emits a dummy fragment shader consisting of magenta for bringup purposes. */
93 void
emit_dummy_fs()94 fs_visitor::emit_dummy_fs()
95 {
96    int reg_width = dispatch_width / 8;
97 
98    /* Everyone's favorite color. */
99    const float color[4] = { 1.0, 0.0, 1.0, 0.0 };
100    for (int i = 0; i < 4; i++) {
101       bld.MOV(fs_reg(MRF, 2 + i * reg_width, BRW_REGISTER_TYPE_F),
102               brw_imm_f(color[i]));
103    }
104 
105    fs_inst *write;
106    write = bld.emit(FS_OPCODE_FB_WRITE);
107    write->eot = true;
108    write->last_rt = true;
109    if (devinfo->gen >= 6) {
110       write->base_mrf = 2;
111       write->mlen = 4 * reg_width;
112    } else {
113       write->header_size = 2;
114       write->base_mrf = 0;
115       write->mlen = 2 + 4 * reg_width;
116    }
117 
118    /* Tell the SF we don't have any inputs.  Gen4-5 require at least one
119     * varying to avoid GPU hangs, so set that.
120     */
121    struct brw_wm_prog_data *wm_prog_data = brw_wm_prog_data(this->prog_data);
122    wm_prog_data->num_varying_inputs = devinfo->gen < 6 ? 1 : 0;
123    memset(wm_prog_data->urb_setup, -1,
124           sizeof(wm_prog_data->urb_setup[0]) * VARYING_SLOT_MAX);
125    brw_compute_urb_setup_index(wm_prog_data);
126 
127    /* We don't have any uniforms. */
128    stage_prog_data->nr_params = 0;
129    stage_prog_data->nr_pull_params = 0;
130    stage_prog_data->curb_read_length = 0;
131    stage_prog_data->dispatch_grf_start_reg = 2;
132    wm_prog_data->dispatch_grf_start_reg_16 = 2;
133    wm_prog_data->dispatch_grf_start_reg_32 = 2;
134    grf_used = 1; /* Gen4-5 don't allow zero GRF blocks */
135 
136    calculate_cfg();
137 }
138 
139 /* The register location here is relative to the start of the URB
140  * data.  It will get adjusted to be a real location before
141  * generate_code() time.
142  */
143 fs_reg
interp_reg(int location,int channel)144 fs_visitor::interp_reg(int location, int channel)
145 {
146    assert(stage == MESA_SHADER_FRAGMENT);
147    struct brw_wm_prog_data *prog_data = brw_wm_prog_data(this->prog_data);
148    int regnr = prog_data->urb_setup[location] * 4 + channel;
149    assert(prog_data->urb_setup[location] != -1);
150 
151    return fs_reg(ATTR, regnr, BRW_REGISTER_TYPE_F);
152 }
153 
154 /** Emits the interpolation for the varying inputs. */
155 void
emit_interpolation_setup_gen4()156 fs_visitor::emit_interpolation_setup_gen4()
157 {
158    struct brw_reg g1_uw = retype(brw_vec1_grf(1, 0), BRW_REGISTER_TYPE_UW);
159 
160    fs_builder abld = bld.annotate("compute pixel centers");
161    this->pixel_x = vgrf(glsl_type::uint_type);
162    this->pixel_y = vgrf(glsl_type::uint_type);
163    this->pixel_x.type = BRW_REGISTER_TYPE_UW;
164    this->pixel_y.type = BRW_REGISTER_TYPE_UW;
165    abld.ADD(this->pixel_x,
166             fs_reg(stride(suboffset(g1_uw, 4), 2, 4, 0)),
167             fs_reg(brw_imm_v(0x10101010)));
168    abld.ADD(this->pixel_y,
169             fs_reg(stride(suboffset(g1_uw, 5), 2, 4, 0)),
170             fs_reg(brw_imm_v(0x11001100)));
171 
172    abld = bld.annotate("compute pixel deltas from v0");
173 
174    this->delta_xy[BRW_BARYCENTRIC_PERSPECTIVE_PIXEL] =
175       vgrf(glsl_type::vec2_type);
176    const fs_reg &delta_xy = this->delta_xy[BRW_BARYCENTRIC_PERSPECTIVE_PIXEL];
177    const fs_reg xstart(negate(brw_vec1_grf(1, 0)));
178    const fs_reg ystart(negate(brw_vec1_grf(1, 1)));
179 
180    if (devinfo->has_pln) {
181       for (unsigned i = 0; i < dispatch_width / 8; i++) {
182          abld.quarter(i).ADD(quarter(offset(delta_xy, abld, 0), i),
183                              quarter(this->pixel_x, i), xstart);
184          abld.quarter(i).ADD(quarter(offset(delta_xy, abld, 1), i),
185                              quarter(this->pixel_y, i), ystart);
186       }
187    } else {
188       abld.ADD(offset(delta_xy, abld, 0), this->pixel_x, xstart);
189       abld.ADD(offset(delta_xy, abld, 1), this->pixel_y, ystart);
190    }
191 
192    abld = bld.annotate("compute pos.w and 1/pos.w");
193    /* Compute wpos.w.  It's always in our setup, since it's needed to
194     * interpolate the other attributes.
195     */
196    this->wpos_w = vgrf(glsl_type::float_type);
197    abld.emit(FS_OPCODE_LINTERP, wpos_w, delta_xy,
198              component(interp_reg(VARYING_SLOT_POS, 3), 0));
199    /* Compute the pixel 1/W value from wpos.w. */
200    this->pixel_w = vgrf(glsl_type::float_type);
201    abld.emit(SHADER_OPCODE_RCP, this->pixel_w, wpos_w);
202 }
203 
204 static unsigned
brw_rnd_mode_from_nir(unsigned mode,unsigned * mask)205 brw_rnd_mode_from_nir(unsigned mode, unsigned *mask)
206 {
207    unsigned brw_mode = 0;
208    *mask = 0;
209 
210    if ((FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP16 |
211         FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP32 |
212         FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP64) &
213        mode) {
214       brw_mode |= BRW_RND_MODE_RTZ << BRW_CR0_RND_MODE_SHIFT;
215       *mask |= BRW_CR0_RND_MODE_MASK;
216    }
217    if ((FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP16 |
218         FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP32 |
219         FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP64) &
220        mode) {
221       brw_mode |= BRW_RND_MODE_RTNE << BRW_CR0_RND_MODE_SHIFT;
222       *mask |= BRW_CR0_RND_MODE_MASK;
223    }
224    if (mode & FLOAT_CONTROLS_DENORM_PRESERVE_FP16) {
225       brw_mode |= BRW_CR0_FP16_DENORM_PRESERVE;
226       *mask |= BRW_CR0_FP16_DENORM_PRESERVE;
227    }
228    if (mode & FLOAT_CONTROLS_DENORM_PRESERVE_FP32) {
229       brw_mode |= BRW_CR0_FP32_DENORM_PRESERVE;
230       *mask |= BRW_CR0_FP32_DENORM_PRESERVE;
231    }
232    if (mode & FLOAT_CONTROLS_DENORM_PRESERVE_FP64) {
233       brw_mode |= BRW_CR0_FP64_DENORM_PRESERVE;
234       *mask |= BRW_CR0_FP64_DENORM_PRESERVE;
235    }
236    if (mode & FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP16)
237       *mask |= BRW_CR0_FP16_DENORM_PRESERVE;
238    if (mode & FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP32)
239       *mask |= BRW_CR0_FP32_DENORM_PRESERVE;
240    if (mode & FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP64)
241       *mask |= BRW_CR0_FP64_DENORM_PRESERVE;
242    if (mode == FLOAT_CONTROLS_DEFAULT_FLOAT_CONTROL_MODE)
243       *mask |= BRW_CR0_FP_MODE_MASK;
244 
245    if (*mask != 0)
246       assert((*mask & brw_mode) == brw_mode);
247 
248    return brw_mode;
249 }
250 
251 void
emit_shader_float_controls_execution_mode()252 fs_visitor::emit_shader_float_controls_execution_mode()
253 {
254    unsigned execution_mode = this->nir->info.float_controls_execution_mode;
255    if (execution_mode == FLOAT_CONTROLS_DEFAULT_FLOAT_CONTROL_MODE)
256       return;
257 
258    fs_builder abld = bld.annotate("shader floats control execution mode");
259    unsigned mask, mode = brw_rnd_mode_from_nir(execution_mode, &mask);
260 
261    if (mask == 0)
262       return;
263 
264    abld.emit(SHADER_OPCODE_FLOAT_CONTROL_MODE, bld.null_reg_ud(),
265              brw_imm_d(mode), brw_imm_d(mask));
266 }
267 
268 /** Emits the interpolation for the varying inputs. */
269 void
emit_interpolation_setup_gen6()270 fs_visitor::emit_interpolation_setup_gen6()
271 {
272    fs_builder abld = bld.annotate("compute pixel centers");
273 
274    this->pixel_x = vgrf(glsl_type::float_type);
275    this->pixel_y = vgrf(glsl_type::float_type);
276 
277    for (unsigned i = 0; i < DIV_ROUND_UP(dispatch_width, 16); i++) {
278       const fs_builder hbld = abld.group(MIN2(16, dispatch_width), i);
279       struct brw_reg gi_uw = retype(brw_vec1_grf(1 + i, 0), BRW_REGISTER_TYPE_UW);
280 
281       if (devinfo->gen >= 8 || dispatch_width == 8) {
282          /* The "Register Region Restrictions" page says for BDW (and newer,
283           * presumably):
284           *
285           *     "When destination spans two registers, the source may be one or
286           *      two registers. The destination elements must be evenly split
287           *      between the two registers."
288           *
289           * Thus we can do a single add(16) in SIMD8 or an add(32) in SIMD16
290           * to compute our pixel centers.
291           */
292          const fs_builder dbld =
293             abld.exec_all().group(hbld.dispatch_width() * 2, 0);
294          fs_reg int_pixel_xy = dbld.vgrf(BRW_REGISTER_TYPE_UW);
295 
296          dbld.ADD(int_pixel_xy,
297                   fs_reg(stride(suboffset(gi_uw, 4), 1, 4, 0)),
298                   fs_reg(brw_imm_v(0x11001010)));
299 
300          hbld.emit(FS_OPCODE_PIXEL_X, offset(pixel_x, hbld, i), int_pixel_xy);
301          hbld.emit(FS_OPCODE_PIXEL_Y, offset(pixel_y, hbld, i), int_pixel_xy);
302       } else {
303          /* The "Register Region Restrictions" page says for SNB, IVB, HSW:
304           *
305           *     "When destination spans two registers, the source MUST span
306           *      two registers."
307           *
308           * Since the GRF source of the ADD will only read a single register,
309           * we must do two separate ADDs in SIMD16.
310           */
311          const fs_reg int_pixel_x = hbld.vgrf(BRW_REGISTER_TYPE_UW);
312          const fs_reg int_pixel_y = hbld.vgrf(BRW_REGISTER_TYPE_UW);
313 
314          hbld.ADD(int_pixel_x,
315                   fs_reg(stride(suboffset(gi_uw, 4), 2, 4, 0)),
316                   fs_reg(brw_imm_v(0x10101010)));
317          hbld.ADD(int_pixel_y,
318                   fs_reg(stride(suboffset(gi_uw, 5), 2, 4, 0)),
319                   fs_reg(brw_imm_v(0x11001100)));
320 
321          /* As of gen6, we can no longer mix float and int sources.  We have
322           * to turn the integer pixel centers into floats for their actual
323           * use.
324           */
325          hbld.MOV(offset(pixel_x, hbld, i), int_pixel_x);
326          hbld.MOV(offset(pixel_y, hbld, i), int_pixel_y);
327       }
328    }
329 
330    abld = bld.annotate("compute pos.w");
331    this->pixel_w = fetch_payload_reg(abld, payload.source_w_reg);
332    this->wpos_w = vgrf(glsl_type::float_type);
333    abld.emit(SHADER_OPCODE_RCP, this->wpos_w, this->pixel_w);
334 
335    struct brw_wm_prog_data *wm_prog_data = brw_wm_prog_data(prog_data);
336 
337    for (int i = 0; i < BRW_BARYCENTRIC_MODE_COUNT; ++i) {
338       this->delta_xy[i] = fetch_barycentric_reg(
339          bld, payload.barycentric_coord_reg[i]);
340    }
341 
342    uint32_t centroid_modes = wm_prog_data->barycentric_interp_modes &
343       (1 << BRW_BARYCENTRIC_PERSPECTIVE_CENTROID |
344        1 << BRW_BARYCENTRIC_NONPERSPECTIVE_CENTROID);
345 
346    if (devinfo->needs_unlit_centroid_workaround && centroid_modes) {
347       /* Get the pixel/sample mask into f0 so that we know which
348        * pixels are lit.  Then, for each channel that is unlit,
349        * replace the centroid data with non-centroid data.
350        */
351       for (unsigned i = 0; i < DIV_ROUND_UP(dispatch_width, 16); i++) {
352          bld.exec_all().group(1, 0)
353             .MOV(retype(brw_flag_reg(0, i), BRW_REGISTER_TYPE_UW),
354                  retype(brw_vec1_grf(1 + i, 7), BRW_REGISTER_TYPE_UW));
355       }
356 
357       for (int i = 0; i < BRW_BARYCENTRIC_MODE_COUNT; ++i) {
358          if (!(centroid_modes & (1 << i)))
359             continue;
360 
361          const fs_reg centroid_delta_xy = delta_xy[i];
362          const fs_reg &pixel_delta_xy = delta_xy[i - 1];
363 
364          delta_xy[i] = bld.vgrf(BRW_REGISTER_TYPE_F, 2);
365 
366          for (unsigned c = 0; c < 2; c++) {
367             for (unsigned q = 0; q < dispatch_width / 8; q++) {
368                set_predicate(BRW_PREDICATE_NORMAL,
369                   bld.quarter(q).SEL(
370                      quarter(offset(delta_xy[i], bld, c), q),
371                      quarter(offset(centroid_delta_xy, bld, c), q),
372                      quarter(offset(pixel_delta_xy, bld, c), q)));
373             }
374          }
375       }
376    }
377 }
378 
379 static enum brw_conditional_mod
cond_for_alpha_func(GLenum func)380 cond_for_alpha_func(GLenum func)
381 {
382    switch(func) {
383       case GL_GREATER:
384          return BRW_CONDITIONAL_G;
385       case GL_GEQUAL:
386          return BRW_CONDITIONAL_GE;
387       case GL_LESS:
388          return BRW_CONDITIONAL_L;
389       case GL_LEQUAL:
390          return BRW_CONDITIONAL_LE;
391       case GL_EQUAL:
392          return BRW_CONDITIONAL_EQ;
393       case GL_NOTEQUAL:
394          return BRW_CONDITIONAL_NEQ;
395       default:
396          unreachable("Not reached");
397    }
398 }
399 
400 /**
401  * Alpha test support for when we compile it into the shader instead
402  * of using the normal fixed-function alpha test.
403  */
404 void
emit_alpha_test()405 fs_visitor::emit_alpha_test()
406 {
407    assert(stage == MESA_SHADER_FRAGMENT);
408    brw_wm_prog_key *key = (brw_wm_prog_key*) this->key;
409    const fs_builder abld = bld.annotate("Alpha test");
410 
411    fs_inst *cmp;
412    if (key->alpha_test_func == GL_ALWAYS)
413       return;
414 
415    if (key->alpha_test_func == GL_NEVER) {
416       /* f0.1 = 0 */
417       fs_reg some_reg = fs_reg(retype(brw_vec8_grf(0, 0),
418                                       BRW_REGISTER_TYPE_UW));
419       cmp = abld.CMP(bld.null_reg_f(), some_reg, some_reg,
420                      BRW_CONDITIONAL_NEQ);
421    } else {
422       /* RT0 alpha */
423       fs_reg color = offset(outputs[0], bld, 3);
424 
425       /* f0.1 &= func(color, ref) */
426       cmp = abld.CMP(bld.null_reg_f(), color, brw_imm_f(key->alpha_test_ref),
427                      cond_for_alpha_func(key->alpha_test_func));
428    }
429    cmp->predicate = BRW_PREDICATE_NORMAL;
430    cmp->flag_subreg = 1;
431 }
432 
433 fs_inst *
emit_single_fb_write(const fs_builder & bld,fs_reg color0,fs_reg color1,fs_reg src0_alpha,unsigned components)434 fs_visitor::emit_single_fb_write(const fs_builder &bld,
435                                  fs_reg color0, fs_reg color1,
436                                  fs_reg src0_alpha, unsigned components)
437 {
438    assert(stage == MESA_SHADER_FRAGMENT);
439    struct brw_wm_prog_data *prog_data = brw_wm_prog_data(this->prog_data);
440 
441    /* Hand over gl_FragDepth or the payload depth. */
442    const fs_reg dst_depth = fetch_payload_reg(bld, payload.dest_depth_reg);
443    fs_reg src_depth, src_stencil;
444 
445    if (source_depth_to_render_target) {
446       if (nir->info.outputs_written & BITFIELD64_BIT(FRAG_RESULT_DEPTH))
447          src_depth = frag_depth;
448       else
449          src_depth = fetch_payload_reg(bld, payload.source_depth_reg);
450    }
451 
452    if (nir->info.outputs_written & BITFIELD64_BIT(FRAG_RESULT_STENCIL))
453       src_stencil = frag_stencil;
454 
455    const fs_reg sources[] = {
456       color0, color1, src0_alpha, src_depth, dst_depth, src_stencil,
457       (prog_data->uses_omask ? sample_mask : fs_reg()),
458       brw_imm_ud(components)
459    };
460    assert(ARRAY_SIZE(sources) - 1 == FB_WRITE_LOGICAL_SRC_COMPONENTS);
461    fs_inst *write = bld.emit(FS_OPCODE_FB_WRITE_LOGICAL, fs_reg(),
462                              sources, ARRAY_SIZE(sources));
463 
464    if (prog_data->uses_kill) {
465       write->predicate = BRW_PREDICATE_NORMAL;
466       write->flag_subreg = sample_mask_flag_subreg(this);
467    }
468 
469    return write;
470 }
471 
472 void
emit_fb_writes()473 fs_visitor::emit_fb_writes()
474 {
475    assert(stage == MESA_SHADER_FRAGMENT);
476    struct brw_wm_prog_data *prog_data = brw_wm_prog_data(this->prog_data);
477    brw_wm_prog_key *key = (brw_wm_prog_key*) this->key;
478 
479    fs_inst *inst = NULL;
480 
481    if (source_depth_to_render_target && devinfo->gen == 6) {
482       /* For outputting oDepth on gen6, SIMD8 writes have to be used.  This
483        * would require SIMD8 moves of each half to message regs, e.g. by using
484        * the SIMD lowering pass.  Unfortunately this is more difficult than it
485        * sounds because the SIMD8 single-source message lacks channel selects
486        * for the second and third subspans.
487        */
488       limit_dispatch_width(8, "Depth writes unsupported in SIMD16+ mode.\n");
489    }
490 
491    if (nir->info.outputs_written & BITFIELD64_BIT(FRAG_RESULT_STENCIL)) {
492       /* From the 'Render Target Write message' section of the docs:
493        * "Output Stencil is not supported with SIMD16 Render Target Write
494        * Messages."
495        */
496       limit_dispatch_width(8, "gl_FragStencilRefARB unsupported "
497                            "in SIMD16+ mode.\n");
498    }
499 
500    /* ANV doesn't know about sample mask output during the wm key creation
501     * so we compute if we need replicate alpha and emit alpha to coverage
502     * workaround here.
503     */
504    const bool replicate_alpha = key->alpha_test_replicate_alpha ||
505       (key->nr_color_regions > 1 && key->alpha_to_coverage &&
506        (sample_mask.file == BAD_FILE || devinfo->gen == 6));
507 
508    for (int target = 0; target < key->nr_color_regions; target++) {
509       /* Skip over outputs that weren't written. */
510       if (this->outputs[target].file == BAD_FILE)
511          continue;
512 
513       const fs_builder abld = bld.annotate(
514          ralloc_asprintf(this->mem_ctx, "FB write target %d", target));
515 
516       fs_reg src0_alpha;
517       if (devinfo->gen >= 6 && replicate_alpha && target != 0)
518          src0_alpha = offset(outputs[0], bld, 3);
519 
520       inst = emit_single_fb_write(abld, this->outputs[target],
521                                   this->dual_src_output, src0_alpha, 4);
522       inst->target = target;
523    }
524 
525    prog_data->dual_src_blend = (this->dual_src_output.file != BAD_FILE &&
526                                 this->outputs[0].file != BAD_FILE);
527    assert(!prog_data->dual_src_blend || key->nr_color_regions == 1);
528 
529    if (inst == NULL) {
530       /* Even if there's no color buffers enabled, we still need to send
531        * alpha out the pipeline to our null renderbuffer to support
532        * alpha-testing, alpha-to-coverage, and so on.
533        */
534       /* FINISHME: Factor out this frequently recurring pattern into a
535        * helper function.
536        */
537       const fs_reg srcs[] = { reg_undef, reg_undef,
538                               reg_undef, offset(this->outputs[0], bld, 3) };
539       const fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_UD, 4);
540       bld.LOAD_PAYLOAD(tmp, srcs, 4, 0);
541 
542       inst = emit_single_fb_write(bld, tmp, reg_undef, reg_undef, 4);
543       inst->target = 0;
544    }
545 
546    inst->last_rt = true;
547    inst->eot = true;
548 
549    if (devinfo->gen >= 11 && devinfo->gen <= 12 &&
550        prog_data->dual_src_blend) {
551       /* The dual-source RT write messages fail to release the thread
552        * dependency on ICL and TGL with SIMD32 dispatch, leading to hangs.
553        *
554        * XXX - Emit an extra single-source NULL RT-write marked LastRT in
555        *       order to release the thread dependency without disabling
556        *       SIMD32.
557        *
558        * The dual-source RT write messages may lead to hangs with SIMD16
559        * dispatch on ICL due some unknown reasons, see
560        * https://gitlab.freedesktop.org/mesa/mesa/-/issues/2183
561        */
562       limit_dispatch_width(8, "Dual source blending unsupported "
563                            "in SIMD16 and SIMD32 modes.\n");
564    }
565 }
566 
567 void
emit_urb_writes(const fs_reg & gs_vertex_count)568 fs_visitor::emit_urb_writes(const fs_reg &gs_vertex_count)
569 {
570    int slot, urb_offset, length;
571    int starting_urb_offset = 0;
572    const struct brw_vue_prog_data *vue_prog_data =
573       brw_vue_prog_data(this->prog_data);
574    const struct brw_vs_prog_key *vs_key =
575       (const struct brw_vs_prog_key *) this->key;
576    const GLbitfield64 psiz_mask =
577       VARYING_BIT_LAYER | VARYING_BIT_VIEWPORT | VARYING_BIT_PSIZ;
578    const struct brw_vue_map *vue_map = &vue_prog_data->vue_map;
579    bool flush;
580    fs_reg sources[8];
581    fs_reg urb_handle;
582 
583    if (stage == MESA_SHADER_TESS_EVAL)
584       urb_handle = fs_reg(retype(brw_vec8_grf(4, 0), BRW_REGISTER_TYPE_UD));
585    else
586       urb_handle = fs_reg(retype(brw_vec8_grf(1, 0), BRW_REGISTER_TYPE_UD));
587 
588    opcode opcode = SHADER_OPCODE_URB_WRITE_SIMD8;
589    int header_size = 1;
590    fs_reg per_slot_offsets;
591 
592    if (stage == MESA_SHADER_GEOMETRY) {
593       const struct brw_gs_prog_data *gs_prog_data =
594          brw_gs_prog_data(this->prog_data);
595 
596       /* We need to increment the Global Offset to skip over the control data
597        * header and the extra "Vertex Count" field (1 HWord) at the beginning
598        * of the VUE.  We're counting in OWords, so the units are doubled.
599        */
600       starting_urb_offset = 2 * gs_prog_data->control_data_header_size_hwords;
601       if (gs_prog_data->static_vertex_count == -1)
602          starting_urb_offset += 2;
603 
604       /* We also need to use per-slot offsets.  The per-slot offset is the
605        * Vertex Count.  SIMD8 mode processes 8 different primitives at a
606        * time; each may output a different number of vertices.
607        */
608       opcode = SHADER_OPCODE_URB_WRITE_SIMD8_PER_SLOT;
609       header_size++;
610 
611       /* The URB offset is in 128-bit units, so we need to multiply by 2 */
612       const int output_vertex_size_owords =
613          gs_prog_data->output_vertex_size_hwords * 2;
614 
615       if (gs_vertex_count.file == IMM) {
616          per_slot_offsets = brw_imm_ud(output_vertex_size_owords *
617                                        gs_vertex_count.ud);
618       } else {
619          per_slot_offsets = vgrf(glsl_type::uint_type);
620          bld.MUL(per_slot_offsets, gs_vertex_count,
621                  brw_imm_ud(output_vertex_size_owords));
622       }
623    }
624 
625    length = 0;
626    urb_offset = starting_urb_offset;
627    flush = false;
628 
629    /* SSO shaders can have VUE slots allocated which are never actually
630     * written to, so ignore them when looking for the last (written) slot.
631     */
632    int last_slot = vue_map->num_slots - 1;
633    while (last_slot > 0 &&
634           (vue_map->slot_to_varying[last_slot] == BRW_VARYING_SLOT_PAD ||
635            outputs[vue_map->slot_to_varying[last_slot]].file == BAD_FILE)) {
636       last_slot--;
637    }
638 
639    bool urb_written = false;
640    for (slot = 0; slot < vue_map->num_slots; slot++) {
641       int varying = vue_map->slot_to_varying[slot];
642       switch (varying) {
643       case VARYING_SLOT_PSIZ: {
644          /* The point size varying slot is the vue header and is always in the
645           * vue map.  But often none of the special varyings that live there
646           * are written and in that case we can skip writing to the vue
647           * header, provided the corresponding state properly clamps the
648           * values further down the pipeline. */
649          if ((vue_map->slots_valid & psiz_mask) == 0) {
650             assert(length == 0);
651             urb_offset++;
652             break;
653          }
654 
655          fs_reg zero(VGRF, alloc.allocate(1), BRW_REGISTER_TYPE_UD);
656          bld.MOV(zero, brw_imm_ud(0u));
657 
658          sources[length++] = zero;
659          if (vue_map->slots_valid & VARYING_BIT_LAYER)
660             sources[length++] = this->outputs[VARYING_SLOT_LAYER];
661          else
662             sources[length++] = zero;
663 
664          if (vue_map->slots_valid & VARYING_BIT_VIEWPORT)
665             sources[length++] = this->outputs[VARYING_SLOT_VIEWPORT];
666          else
667             sources[length++] = zero;
668 
669          if (vue_map->slots_valid & VARYING_BIT_PSIZ)
670             sources[length++] = this->outputs[VARYING_SLOT_PSIZ];
671          else
672             sources[length++] = zero;
673          break;
674       }
675       case BRW_VARYING_SLOT_NDC:
676       case VARYING_SLOT_EDGE:
677          unreachable("unexpected scalar vs output");
678          break;
679 
680       default:
681          /* gl_Position is always in the vue map, but isn't always written by
682           * the shader.  Other varyings (clip distances) get added to the vue
683           * map but don't always get written.  In those cases, the
684           * corresponding this->output[] slot will be invalid we and can skip
685           * the urb write for the varying.  If we've already queued up a vue
686           * slot for writing we flush a mlen 5 urb write, otherwise we just
687           * advance the urb_offset.
688           */
689          if (varying == BRW_VARYING_SLOT_PAD ||
690              this->outputs[varying].file == BAD_FILE) {
691             if (length > 0)
692                flush = true;
693             else
694                urb_offset++;
695             break;
696          }
697 
698          if (stage == MESA_SHADER_VERTEX && vs_key->clamp_vertex_color &&
699              (varying == VARYING_SLOT_COL0 ||
700               varying == VARYING_SLOT_COL1 ||
701               varying == VARYING_SLOT_BFC0 ||
702               varying == VARYING_SLOT_BFC1)) {
703             /* We need to clamp these guys, so do a saturating MOV into a
704              * temp register and use that for the payload.
705              */
706             for (int i = 0; i < 4; i++) {
707                fs_reg reg = fs_reg(VGRF, alloc.allocate(1), outputs[varying].type);
708                fs_reg src = offset(this->outputs[varying], bld, i);
709                set_saturate(true, bld.MOV(reg, src));
710                sources[length++] = reg;
711             }
712          } else {
713             int slot_offset = 0;
714 
715             /* When using Primitive Replication, there may be multiple slots
716              * assigned to POS.
717              */
718             if (varying == VARYING_SLOT_POS)
719                slot_offset = slot - vue_map->varying_to_slot[VARYING_SLOT_POS];
720 
721             for (unsigned i = 0; i < 4; i++) {
722                sources[length++] = offset(this->outputs[varying], bld,
723                                           i + (slot_offset * 4));
724             }
725          }
726          break;
727       }
728 
729       const fs_builder abld = bld.annotate("URB write");
730 
731       /* If we've queued up 8 registers of payload (2 VUE slots), if this is
732        * the last slot or if we need to flush (see BAD_FILE varying case
733        * above), emit a URB write send now to flush out the data.
734        */
735       if (length == 8 || (length > 0 && slot == last_slot))
736          flush = true;
737       if (flush) {
738          fs_reg *payload_sources =
739             ralloc_array(mem_ctx, fs_reg, length + header_size);
740          fs_reg payload = fs_reg(VGRF, alloc.allocate(length + header_size),
741                                  BRW_REGISTER_TYPE_F);
742          payload_sources[0] = urb_handle;
743 
744          if (opcode == SHADER_OPCODE_URB_WRITE_SIMD8_PER_SLOT)
745             payload_sources[1] = per_slot_offsets;
746 
747          memcpy(&payload_sources[header_size], sources,
748                 length * sizeof sources[0]);
749 
750          abld.LOAD_PAYLOAD(payload, payload_sources, length + header_size,
751                            header_size);
752 
753          fs_inst *inst = abld.emit(opcode, reg_undef, payload);
754 
755          /* For ICL WA 1805992985 one needs additional write in the end. */
756          if (devinfo->gen == 11 && stage == MESA_SHADER_TESS_EVAL)
757             inst->eot = false;
758          else
759             inst->eot = slot == last_slot && stage != MESA_SHADER_GEOMETRY;
760 
761          inst->mlen = length + header_size;
762          inst->offset = urb_offset;
763          urb_offset = starting_urb_offset + slot + 1;
764          length = 0;
765          flush = false;
766          urb_written = true;
767       }
768    }
769 
770    /* If we don't have any valid slots to write, just do a minimal urb write
771     * send to terminate the shader.  This includes 1 slot of undefined data,
772     * because it's invalid to write 0 data:
773     *
774     * From the Broadwell PRM, Volume 7: 3D Media GPGPU, Shared Functions -
775     * Unified Return Buffer (URB) > URB_SIMD8_Write and URB_SIMD8_Read >
776     * Write Data Payload:
777     *
778     *    "The write data payload can be between 1 and 8 message phases long."
779     */
780    if (!urb_written) {
781       /* For GS, just turn EmitVertex() into a no-op.  We don't want it to
782        * end the thread, and emit_gs_thread_end() already emits a SEND with
783        * EOT at the end of the program for us.
784        */
785       if (stage == MESA_SHADER_GEOMETRY)
786          return;
787 
788       fs_reg payload = fs_reg(VGRF, alloc.allocate(2), BRW_REGISTER_TYPE_UD);
789       bld.exec_all().MOV(payload, urb_handle);
790 
791       fs_inst *inst = bld.emit(SHADER_OPCODE_URB_WRITE_SIMD8, reg_undef, payload);
792       inst->eot = true;
793       inst->mlen = 2;
794       inst->offset = 1;
795       return;
796    }
797 
798    /* ICL WA 1805992985:
799     *
800     * ICLLP GPU hangs on one of tessellation vkcts tests with DS not done. The
801     * send cycle, which is a urb write with an eot must be 4 phases long and
802     * all 8 lanes must valid.
803     */
804    if (devinfo->gen == 11 && stage == MESA_SHADER_TESS_EVAL) {
805       fs_reg payload = fs_reg(VGRF, alloc.allocate(6), BRW_REGISTER_TYPE_UD);
806 
807       /* Workaround requires all 8 channels (lanes) to be valid. This is
808        * understood to mean they all need to be alive. First trick is to find
809        * a live channel and copy its urb handle for all the other channels to
810        * make sure all handles are valid.
811        */
812       bld.exec_all().MOV(payload, bld.emit_uniformize(urb_handle));
813 
814       /* Second trick is to use masked URB write where one can tell the HW to
815        * actually write data only for selected channels even though all are
816        * active.
817        * Third trick is to take advantage of the must-be-zero (MBZ) area in
818        * the very beginning of the URB.
819        *
820        * One masks data to be written only for the first channel and uses
821        * offset zero explicitly to land data to the MBZ area avoiding trashing
822        * any other part of the URB.
823        *
824        * Since the WA says that the write needs to be 4 phases long one uses
825        * 4 slots data. All are explicitly zeros in order to to keep the MBZ
826        * area written as zeros.
827        */
828       bld.exec_all().MOV(offset(payload, bld, 1), brw_imm_ud(0x10000u));
829       bld.exec_all().MOV(offset(payload, bld, 2), brw_imm_ud(0u));
830       bld.exec_all().MOV(offset(payload, bld, 3), brw_imm_ud(0u));
831       bld.exec_all().MOV(offset(payload, bld, 4), brw_imm_ud(0u));
832       bld.exec_all().MOV(offset(payload, bld, 5), brw_imm_ud(0u));
833 
834       fs_inst *inst = bld.exec_all().emit(SHADER_OPCODE_URB_WRITE_SIMD8_MASKED,
835                                           reg_undef, payload);
836       inst->eot = true;
837       inst->mlen = 6;
838       inst->offset = 0;
839    }
840 }
841 
842 void
emit_cs_terminate()843 fs_visitor::emit_cs_terminate()
844 {
845    assert(devinfo->gen >= 7);
846 
847    /* We are getting the thread ID from the compute shader header */
848    assert(stage == MESA_SHADER_COMPUTE || stage == MESA_SHADER_KERNEL);
849 
850    /* We can't directly send from g0, since sends with EOT have to use
851     * g112-127. So, copy it to a virtual register, The register allocator will
852     * make sure it uses the appropriate register range.
853     */
854    struct brw_reg g0 = retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD);
855    fs_reg payload = fs_reg(VGRF, alloc.allocate(1), BRW_REGISTER_TYPE_UD);
856    bld.group(8, 0).exec_all().MOV(payload, g0);
857 
858    /* Send a message to the thread spawner to terminate the thread. */
859    fs_inst *inst = bld.exec_all()
860                       .emit(CS_OPCODE_CS_TERMINATE, reg_undef, payload);
861    inst->eot = true;
862 }
863 
864 void
emit_barrier()865 fs_visitor::emit_barrier()
866 {
867    uint32_t barrier_id_mask;
868    switch (devinfo->gen) {
869    case 7:
870    case 8:
871       barrier_id_mask = 0x0f000000u; break;
872    case 9:
873       barrier_id_mask = 0x8f000000u; break;
874    case 11:
875    case 12:
876       barrier_id_mask = 0x7f000000u; break;
877    default:
878       unreachable("barrier is only available on gen >= 7");
879    }
880 
881    /* We are getting the barrier ID from the compute shader header */
882    assert(stage == MESA_SHADER_COMPUTE || stage == MESA_SHADER_KERNEL);
883 
884    fs_reg payload = fs_reg(VGRF, alloc.allocate(1), BRW_REGISTER_TYPE_UD);
885 
886    /* Clear the message payload */
887    bld.exec_all().group(8, 0).MOV(payload, brw_imm_ud(0u));
888 
889    /* Copy the barrier id from r0.2 to the message payload reg.2 */
890    fs_reg r0_2 = fs_reg(retype(brw_vec1_grf(0, 2), BRW_REGISTER_TYPE_UD));
891    bld.exec_all().group(1, 0).AND(component(payload, 2), r0_2,
892                                   brw_imm_ud(barrier_id_mask));
893 
894    /* Emit a gateway "barrier" message using the payload we set up, followed
895     * by a wait instruction.
896     */
897    bld.exec_all().emit(SHADER_OPCODE_BARRIER, reg_undef, payload);
898 }
899 
fs_visitor(const struct brw_compiler * compiler,void * log_data,void * mem_ctx,const brw_base_prog_key * key,struct brw_stage_prog_data * prog_data,const nir_shader * shader,unsigned dispatch_width,int shader_time_index,const struct brw_vue_map * input_vue_map)900 fs_visitor::fs_visitor(const struct brw_compiler *compiler, void *log_data,
901                        void *mem_ctx,
902                        const brw_base_prog_key *key,
903                        struct brw_stage_prog_data *prog_data,
904                        const nir_shader *shader,
905                        unsigned dispatch_width,
906                        int shader_time_index,
907                        const struct brw_vue_map *input_vue_map)
908    : backend_shader(compiler, log_data, mem_ctx, shader, prog_data),
909      key(key), gs_compile(NULL), prog_data(prog_data),
910      input_vue_map(input_vue_map),
911      live_analysis(this), regpressure_analysis(this),
912      performance_analysis(this),
913      dispatch_width(dispatch_width),
914      shader_time_index(shader_time_index),
915      bld(fs_builder(this, dispatch_width).at_end())
916 {
917    init();
918 }
919 
fs_visitor(const struct brw_compiler * compiler,void * log_data,void * mem_ctx,struct brw_gs_compile * c,struct brw_gs_prog_data * prog_data,const nir_shader * shader,int shader_time_index)920 fs_visitor::fs_visitor(const struct brw_compiler *compiler, void *log_data,
921                        void *mem_ctx,
922                        struct brw_gs_compile *c,
923                        struct brw_gs_prog_data *prog_data,
924                        const nir_shader *shader,
925                        int shader_time_index)
926    : backend_shader(compiler, log_data, mem_ctx, shader,
927                     &prog_data->base.base),
928      key(&c->key.base), gs_compile(c),
929      prog_data(&prog_data->base.base),
930      live_analysis(this), regpressure_analysis(this),
931      performance_analysis(this),
932      dispatch_width(8),
933      shader_time_index(shader_time_index),
934      bld(fs_builder(this, dispatch_width).at_end())
935 {
936    init();
937 }
938 
939 
940 void
init()941 fs_visitor::init()
942 {
943    if (key)
944       this->key_tex = &key->tex;
945    else
946       this->key_tex = NULL;
947 
948    this->max_dispatch_width = 32;
949    this->prog_data = this->stage_prog_data;
950 
951    this->failed = false;
952    this->fail_msg = NULL;
953 
954    this->nir_locals = NULL;
955    this->nir_ssa_values = NULL;
956    this->nir_system_values = NULL;
957 
958    memset(&this->payload, 0, sizeof(this->payload));
959    this->source_depth_to_render_target = false;
960    this->runtime_check_aads_emit = false;
961    this->first_non_payload_grf = 0;
962    this->max_grf = devinfo->gen >= 7 ? GEN7_MRF_HACK_START : BRW_MAX_GRF;
963 
964    this->uniforms = 0;
965    this->last_scratch = 0;
966    this->pull_constant_loc = NULL;
967    this->push_constant_loc = NULL;
968 
969    this->shader_stats.scheduler_mode = NULL;
970    this->shader_stats.promoted_constants = 0,
971 
972    this->grf_used = 0;
973    this->spilled_any_registers = false;
974 }
975 
~fs_visitor()976 fs_visitor::~fs_visitor()
977 {
978 }
979