• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  Copyright (C) Intel Corp.  2006.  All Rights Reserved.
3  Intel funded Tungsten Graphics to
4  develop this 3D driver.
5 
6  Permission is hereby granted, free of charge, to any person obtaining
7  a copy of this software and associated documentation files (the
8  "Software"), to deal in the Software without restriction, including
9  without limitation the rights to use, copy, modify, merge, publish,
10  distribute, sublicense, and/or sell copies of the Software, and to
11  permit persons to whom the Software is furnished to do so, subject to
12  the following conditions:
13 
14  The above copyright notice and this permission notice (including the
15  next paragraph) shall be included in all copies or substantial
16  portions of the Software.
17 
18  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
19  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
21  IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
22  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
23  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
24  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25 
26  **********************************************************************/
27  /*
28   * Authors:
29   *   Keith Whitwell <keithw@vmware.com>
30   */
31 
32 
33 #include "brw_context.h"
34 #include "brw_defines.h"
35 #include "brw_eu.h"
36 
37 #include "util/ralloc.h"
38 
39 /**
40  * Prior to Sandybridge, the SEND instruction accepted non-MRF source
41  * registers, implicitly moving the operand to a message register.
42  *
43  * On Sandybridge, this is no longer the case.  This function performs the
44  * explicit move; it should be called before emitting a SEND instruction.
45  */
46 void
gen6_resolve_implied_move(struct brw_codegen * p,struct brw_reg * src,unsigned msg_reg_nr)47 gen6_resolve_implied_move(struct brw_codegen *p,
48 			  struct brw_reg *src,
49 			  unsigned msg_reg_nr)
50 {
51    const struct gen_device_info *devinfo = p->devinfo;
52    if (devinfo->gen < 6)
53       return;
54 
55    if (src->file == BRW_MESSAGE_REGISTER_FILE)
56       return;
57 
58    if (src->file != BRW_ARCHITECTURE_REGISTER_FILE || src->nr != BRW_ARF_NULL) {
59       brw_push_insn_state(p);
60       brw_set_default_exec_size(p, BRW_EXECUTE_8);
61       brw_set_default_mask_control(p, BRW_MASK_DISABLE);
62       brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
63       brw_MOV(p, retype(brw_message_reg(msg_reg_nr), BRW_REGISTER_TYPE_UD),
64 	      retype(*src, BRW_REGISTER_TYPE_UD));
65       brw_pop_insn_state(p);
66    }
67    *src = brw_message_reg(msg_reg_nr);
68 }
69 
70 static void
gen7_convert_mrf_to_grf(struct brw_codegen * p,struct brw_reg * reg)71 gen7_convert_mrf_to_grf(struct brw_codegen *p, struct brw_reg *reg)
72 {
73    /* From the Ivybridge PRM, Volume 4 Part 3, page 218 ("send"):
74     * "The send with EOT should use register space R112-R127 for <src>. This is
75     *  to enable loading of a new thread into the same slot while the message
76     *  with EOT for current thread is pending dispatch."
77     *
78     * Since we're pretending to have 16 MRFs anyway, we may as well use the
79     * registers required for messages with EOT.
80     */
81    const struct gen_device_info *devinfo = p->devinfo;
82    if (devinfo->gen >= 7 && reg->file == BRW_MESSAGE_REGISTER_FILE) {
83       reg->file = BRW_GENERAL_REGISTER_FILE;
84       reg->nr += GEN7_MRF_HACK_START;
85    }
86 }
87 
88 /**
89  * Convert a brw_reg_type enumeration value into the hardware representation.
90  *
91  * The hardware encoding may depend on whether the value is an immediate.
92  */
93 unsigned
brw_reg_type_to_hw_type(const struct gen_device_info * devinfo,enum brw_reg_type type,enum brw_reg_file file)94 brw_reg_type_to_hw_type(const struct gen_device_info *devinfo,
95                         enum brw_reg_type type, enum brw_reg_file file)
96 {
97    if (file == BRW_IMMEDIATE_VALUE) {
98       static const int imm_hw_types[] = {
99          [BRW_REGISTER_TYPE_UD] = BRW_HW_REG_TYPE_UD,
100          [BRW_REGISTER_TYPE_D]  = BRW_HW_REG_TYPE_D,
101          [BRW_REGISTER_TYPE_UW] = BRW_HW_REG_TYPE_UW,
102          [BRW_REGISTER_TYPE_W]  = BRW_HW_REG_TYPE_W,
103          [BRW_REGISTER_TYPE_F]  = BRW_HW_REG_TYPE_F,
104          [BRW_REGISTER_TYPE_UB] = -1,
105          [BRW_REGISTER_TYPE_B]  = -1,
106          [BRW_REGISTER_TYPE_UV] = BRW_HW_REG_IMM_TYPE_UV,
107          [BRW_REGISTER_TYPE_VF] = BRW_HW_REG_IMM_TYPE_VF,
108          [BRW_REGISTER_TYPE_V]  = BRW_HW_REG_IMM_TYPE_V,
109          [BRW_REGISTER_TYPE_DF] = GEN8_HW_REG_IMM_TYPE_DF,
110          [BRW_REGISTER_TYPE_HF] = GEN8_HW_REG_IMM_TYPE_HF,
111          [BRW_REGISTER_TYPE_UQ] = GEN8_HW_REG_TYPE_UQ,
112          [BRW_REGISTER_TYPE_Q]  = GEN8_HW_REG_TYPE_Q,
113       };
114       assert(type < ARRAY_SIZE(imm_hw_types));
115       assert(imm_hw_types[type] != -1);
116       assert(devinfo->gen >= 8 || type < BRW_REGISTER_TYPE_DF);
117       return imm_hw_types[type];
118    } else {
119       /* Non-immediate registers */
120       static const int hw_types[] = {
121          [BRW_REGISTER_TYPE_UD] = BRW_HW_REG_TYPE_UD,
122          [BRW_REGISTER_TYPE_D]  = BRW_HW_REG_TYPE_D,
123          [BRW_REGISTER_TYPE_UW] = BRW_HW_REG_TYPE_UW,
124          [BRW_REGISTER_TYPE_W]  = BRW_HW_REG_TYPE_W,
125          [BRW_REGISTER_TYPE_UB] = BRW_HW_REG_NON_IMM_TYPE_UB,
126          [BRW_REGISTER_TYPE_B]  = BRW_HW_REG_NON_IMM_TYPE_B,
127          [BRW_REGISTER_TYPE_F]  = BRW_HW_REG_TYPE_F,
128          [BRW_REGISTER_TYPE_UV] = -1,
129          [BRW_REGISTER_TYPE_VF] = -1,
130          [BRW_REGISTER_TYPE_V]  = -1,
131          [BRW_REGISTER_TYPE_DF] = GEN7_HW_REG_NON_IMM_TYPE_DF,
132          [BRW_REGISTER_TYPE_HF] = GEN8_HW_REG_NON_IMM_TYPE_HF,
133          [BRW_REGISTER_TYPE_UQ] = GEN8_HW_REG_TYPE_UQ,
134          [BRW_REGISTER_TYPE_Q]  = GEN8_HW_REG_TYPE_Q,
135       };
136       assert(type < ARRAY_SIZE(hw_types));
137       assert(hw_types[type] != -1);
138       assert(devinfo->gen >= 7 || type < BRW_REGISTER_TYPE_DF);
139       assert(devinfo->gen >= 8 || type < BRW_REGISTER_TYPE_HF);
140       return hw_types[type];
141    }
142 }
143 
144 void
brw_set_dest(struct brw_codegen * p,brw_inst * inst,struct brw_reg dest)145 brw_set_dest(struct brw_codegen *p, brw_inst *inst, struct brw_reg dest)
146 {
147    const struct gen_device_info *devinfo = p->devinfo;
148 
149    if (dest.file == BRW_MESSAGE_REGISTER_FILE)
150       assert((dest.nr & ~BRW_MRF_COMPR4) < BRW_MAX_MRF(devinfo->gen));
151    else if (dest.file != BRW_ARCHITECTURE_REGISTER_FILE)
152       assert(dest.nr < 128);
153 
154    gen7_convert_mrf_to_grf(p, &dest);
155 
156    brw_inst_set_dst_reg_file(devinfo, inst, dest.file);
157    brw_inst_set_dst_reg_type(devinfo, inst,
158                              brw_reg_type_to_hw_type(devinfo, dest.type,
159                                                      dest.file));
160    brw_inst_set_dst_address_mode(devinfo, inst, dest.address_mode);
161 
162    if (dest.address_mode == BRW_ADDRESS_DIRECT) {
163       brw_inst_set_dst_da_reg_nr(devinfo, inst, dest.nr);
164 
165       if (brw_inst_access_mode(devinfo, inst) == BRW_ALIGN_1) {
166          brw_inst_set_dst_da1_subreg_nr(devinfo, inst, dest.subnr);
167 	 if (dest.hstride == BRW_HORIZONTAL_STRIDE_0)
168 	    dest.hstride = BRW_HORIZONTAL_STRIDE_1;
169          brw_inst_set_dst_hstride(devinfo, inst, dest.hstride);
170       } else {
171          brw_inst_set_dst_da16_subreg_nr(devinfo, inst, dest.subnr / 16);
172          brw_inst_set_da16_writemask(devinfo, inst, dest.writemask);
173          if (dest.file == BRW_GENERAL_REGISTER_FILE ||
174              dest.file == BRW_MESSAGE_REGISTER_FILE) {
175             assert(dest.writemask != 0);
176          }
177 	 /* From the Ivybridge PRM, Vol 4, Part 3, Section 5.2.4.1:
178 	  *    Although Dst.HorzStride is a don't care for Align16, HW needs
179 	  *    this to be programmed as "01".
180 	  */
181          brw_inst_set_dst_hstride(devinfo, inst, 1);
182       }
183    } else {
184       brw_inst_set_dst_ia_subreg_nr(devinfo, inst, dest.subnr);
185 
186       /* These are different sizes in align1 vs align16:
187        */
188       if (brw_inst_access_mode(devinfo, inst) == BRW_ALIGN_1) {
189          brw_inst_set_dst_ia1_addr_imm(devinfo, inst,
190                                        dest.indirect_offset);
191 	 if (dest.hstride == BRW_HORIZONTAL_STRIDE_0)
192 	    dest.hstride = BRW_HORIZONTAL_STRIDE_1;
193          brw_inst_set_dst_hstride(devinfo, inst, dest.hstride);
194       } else {
195          brw_inst_set_dst_ia16_addr_imm(devinfo, inst,
196                                         dest.indirect_offset);
197 	 /* even ignored in da16, still need to set as '01' */
198          brw_inst_set_dst_hstride(devinfo, inst, 1);
199       }
200    }
201 
202    /* Generators should set a default exec_size of either 8 (SIMD4x2 or SIMD8)
203     * or 16 (SIMD16), as that's normally correct.  However, when dealing with
204     * small registers, we automatically reduce it to match the register size.
205     *
206     * In platforms that support fp64 we can emit instructions with a width of
207     * 4 that need two SIMD8 registers and an exec_size of 8 or 16. In these
208     * cases we need to make sure that these instructions have their exec sizes
209     * set properly when they are emitted and we can't rely on this code to fix
210     * it.
211     */
212    bool fix_exec_size;
213    if (devinfo->gen >= 6)
214       fix_exec_size = dest.width < BRW_EXECUTE_4;
215    else
216       fix_exec_size = dest.width < BRW_EXECUTE_8;
217 
218    if (fix_exec_size)
219       brw_inst_set_exec_size(devinfo, inst, dest.width);
220 }
221 
222 extern int reg_type_size[];
223 
224 static void
validate_reg(const struct gen_device_info * devinfo,brw_inst * inst,struct brw_reg reg)225 validate_reg(const struct gen_device_info *devinfo,
226              brw_inst *inst, struct brw_reg reg)
227 {
228    const int hstride_for_reg[] = {0, 1, 2, 4};
229    const int vstride_for_reg[] = {0, 1, 2, 4, 8, 16, 32};
230    const int width_for_reg[] = {1, 2, 4, 8, 16};
231    const int execsize_for_reg[] = {1, 2, 4, 8, 16, 32};
232    int width, hstride, vstride, execsize;
233 
234    if (reg.file == BRW_IMMEDIATE_VALUE) {
235       /* 3.3.6: Region Parameters.  Restriction: Immediate vectors
236        * mean the destination has to be 128-bit aligned and the
237        * destination horiz stride has to be a word.
238        */
239       if (reg.type == BRW_REGISTER_TYPE_V) {
240          assert(hstride_for_reg[brw_inst_dst_hstride(devinfo, inst)] *
241                 reg_type_size[brw_inst_dst_reg_type(devinfo, inst)] == 2);
242       }
243 
244       return;
245    }
246 
247    if (reg.file == BRW_ARCHITECTURE_REGISTER_FILE &&
248        reg.file == BRW_ARF_NULL)
249       return;
250 
251    /* From the IVB PRM Vol. 4, Pt. 3, Section 3.3.3.5:
252     *
253     *    "Swizzling is not allowed when an accumulator is used as an implicit
254     *    source or an explicit source in an instruction."
255     */
256    if (reg.file == BRW_ARCHITECTURE_REGISTER_FILE &&
257        reg.nr == BRW_ARF_ACCUMULATOR)
258       assert(reg.swizzle == BRW_SWIZZLE_XYZW);
259 
260    assert(reg.hstride >= 0 && reg.hstride < ARRAY_SIZE(hstride_for_reg));
261    hstride = hstride_for_reg[reg.hstride];
262 
263    if (reg.vstride == 0xf) {
264       vstride = -1;
265    } else {
266       assert(reg.vstride >= 0 && reg.vstride < ARRAY_SIZE(vstride_for_reg));
267       vstride = vstride_for_reg[reg.vstride];
268    }
269 
270    assert(reg.width >= 0 && reg.width < ARRAY_SIZE(width_for_reg));
271    width = width_for_reg[reg.width];
272 
273    assert(brw_inst_exec_size(devinfo, inst) >= 0 &&
274           brw_inst_exec_size(devinfo, inst) < ARRAY_SIZE(execsize_for_reg));
275    execsize = execsize_for_reg[brw_inst_exec_size(devinfo, inst)];
276 
277    /* Restrictions from 3.3.10: Register Region Restrictions. */
278    /* 3. */
279    assert(execsize >= width);
280 
281    /* 4. */
282    if (execsize == width && hstride != 0) {
283       assert(vstride == -1 || vstride == width * hstride);
284    }
285 
286    /* 5. */
287    if (execsize == width && hstride == 0) {
288       /* no restriction on vstride. */
289    }
290 
291    /* 6. */
292    if (width == 1) {
293       assert(hstride == 0);
294    }
295 
296    /* 7. */
297    if (execsize == 1 && width == 1) {
298       assert(hstride == 0);
299       assert(vstride == 0);
300    }
301 
302    /* 8. */
303    if (vstride == 0 && hstride == 0) {
304       assert(width == 1);
305    }
306 
307    /* 10. Check destination issues. */
308 }
309 
310 static bool
is_compactable_immediate(unsigned imm)311 is_compactable_immediate(unsigned imm)
312 {
313    /* We get the low 12 bits as-is. */
314    imm &= ~0xfff;
315 
316    /* We get one bit replicated through the top 20 bits. */
317    return imm == 0 || imm == 0xfffff000;
318 }
319 
320 void
brw_set_src0(struct brw_codegen * p,brw_inst * inst,struct brw_reg reg)321 brw_set_src0(struct brw_codegen *p, brw_inst *inst, struct brw_reg reg)
322 {
323    const struct gen_device_info *devinfo = p->devinfo;
324 
325    if (reg.file == BRW_MESSAGE_REGISTER_FILE)
326       assert((reg.nr & ~BRW_MRF_COMPR4) < BRW_MAX_MRF(devinfo->gen));
327    else if (reg.file != BRW_ARCHITECTURE_REGISTER_FILE)
328       assert(reg.nr < 128);
329 
330    gen7_convert_mrf_to_grf(p, &reg);
331 
332    if (devinfo->gen >= 6 && (brw_inst_opcode(devinfo, inst) == BRW_OPCODE_SEND ||
333                              brw_inst_opcode(devinfo, inst) == BRW_OPCODE_SENDC)) {
334       /* Any source modifiers or regions will be ignored, since this just
335        * identifies the MRF/GRF to start reading the message contents from.
336        * Check for some likely failures.
337        */
338       assert(!reg.negate);
339       assert(!reg.abs);
340       assert(reg.address_mode == BRW_ADDRESS_DIRECT);
341    }
342 
343    validate_reg(devinfo, inst, reg);
344 
345    brw_inst_set_src0_reg_file(devinfo, inst, reg.file);
346    brw_inst_set_src0_reg_type(devinfo, inst,
347                               brw_reg_type_to_hw_type(devinfo, reg.type, reg.file));
348    brw_inst_set_src0_abs(devinfo, inst, reg.abs);
349    brw_inst_set_src0_negate(devinfo, inst, reg.negate);
350    brw_inst_set_src0_address_mode(devinfo, inst, reg.address_mode);
351 
352    if (reg.file == BRW_IMMEDIATE_VALUE) {
353       if (reg.type == BRW_REGISTER_TYPE_DF ||
354           brw_inst_opcode(devinfo, inst) == BRW_OPCODE_DIM)
355          brw_inst_set_imm_df(devinfo, inst, reg.df);
356       else
357          brw_inst_set_imm_ud(devinfo, inst, reg.ud);
358 
359       /* The Bspec's section titled "Non-present Operands" claims that if src0
360        * is an immediate that src1's type must be the same as that of src0.
361        *
362        * The SNB+ DataTypeIndex instruction compaction tables contain mappings
363        * that do not follow this rule. E.g., from the IVB/HSW table:
364        *
365        *  DataTypeIndex   18-Bit Mapping       Mapped Meaning
366        *        3         001000001011111101   r:f | i:vf | a:ud | <1> | dir |
367        *
368        * And from the SNB table:
369        *
370        *  DataTypeIndex   18-Bit Mapping       Mapped Meaning
371        *        8         001000000111101100   a:w | i:w | a:ud | <1> | dir |
372        *
373        * Neither of these cause warnings from the simulator when used,
374        * compacted or otherwise. In fact, all compaction mappings that have an
375        * immediate in src0 use a:ud for src1.
376        *
377        * The GM45 instruction compaction tables do not contain mapped meanings
378        * so it's not clear whether it has the restriction. We'll assume it was
379        * lifted on SNB. (FINISHME: decode the GM45 tables and check.)
380        *
381        * Don't do any of this for 64-bit immediates, since the src1 fields
382        * overlap with the immediate and setting them would overwrite the
383        * immediate we set.
384        */
385       if (type_sz(reg.type) < 8) {
386          brw_inst_set_src1_reg_file(devinfo, inst,
387                                     BRW_ARCHITECTURE_REGISTER_FILE);
388          if (devinfo->gen < 6) {
389             brw_inst_set_src1_reg_type(devinfo, inst,
390                                        brw_inst_src0_reg_type(devinfo, inst));
391          } else {
392             brw_inst_set_src1_reg_type(devinfo, inst, BRW_HW_REG_TYPE_UD);
393          }
394       }
395 
396       /* Compacted instructions only have 12-bits (plus 1 for the other 20)
397        * for immediate values. Presumably the hardware engineers realized
398        * that the only useful floating-point value that could be represented
399        * in this format is 0.0, which can also be represented as a VF-typed
400        * immediate, so they gave us the previously mentioned mapping on IVB+.
401        *
402        * Strangely, we do have a mapping for imm:f in src1, so we don't need
403        * to do this there.
404        *
405        * If we see a 0.0:F, change the type to VF so that it can be compacted.
406        */
407       if (brw_inst_imm_ud(devinfo, inst) == 0x0 &&
408           brw_inst_src0_reg_type(devinfo, inst) == BRW_HW_REG_TYPE_F) {
409          brw_inst_set_src0_reg_type(devinfo, inst, BRW_HW_REG_IMM_TYPE_VF);
410       }
411 
412       /* There are no mappings for dst:d | i:d, so if the immediate is suitable
413        * set the types to :UD so the instruction can be compacted.
414        */
415       if (is_compactable_immediate(brw_inst_imm_ud(devinfo, inst)) &&
416           brw_inst_cond_modifier(devinfo, inst) == BRW_CONDITIONAL_NONE &&
417           brw_inst_src0_reg_type(devinfo, inst) == BRW_HW_REG_TYPE_D &&
418           brw_inst_dst_reg_type(devinfo, inst) == BRW_HW_REG_TYPE_D) {
419          brw_inst_set_src0_reg_type(devinfo, inst, BRW_HW_REG_TYPE_UD);
420          brw_inst_set_dst_reg_type(devinfo, inst, BRW_HW_REG_TYPE_UD);
421       }
422    } else {
423       if (reg.address_mode == BRW_ADDRESS_DIRECT) {
424          brw_inst_set_src0_da_reg_nr(devinfo, inst, reg.nr);
425          if (brw_inst_access_mode(devinfo, inst) == BRW_ALIGN_1) {
426              brw_inst_set_src0_da1_subreg_nr(devinfo, inst, reg.subnr);
427 	 } else {
428             brw_inst_set_src0_da16_subreg_nr(devinfo, inst, reg.subnr / 16);
429 	 }
430       } else {
431          brw_inst_set_src0_ia_subreg_nr(devinfo, inst, reg.subnr);
432 
433          if (brw_inst_access_mode(devinfo, inst) == BRW_ALIGN_1) {
434             brw_inst_set_src0_ia1_addr_imm(devinfo, inst, reg.indirect_offset);
435 	 } else {
436             brw_inst_set_src0_ia16_addr_imm(devinfo, inst, reg.indirect_offset);
437 	 }
438       }
439 
440       if (brw_inst_access_mode(devinfo, inst) == BRW_ALIGN_1) {
441 	 if (reg.width == BRW_WIDTH_1 &&
442              brw_inst_exec_size(devinfo, inst) == BRW_EXECUTE_1) {
443             brw_inst_set_src0_hstride(devinfo, inst, BRW_HORIZONTAL_STRIDE_0);
444             brw_inst_set_src0_width(devinfo, inst, BRW_WIDTH_1);
445             brw_inst_set_src0_vstride(devinfo, inst, BRW_VERTICAL_STRIDE_0);
446 	 } else {
447             brw_inst_set_src0_hstride(devinfo, inst, reg.hstride);
448             brw_inst_set_src0_width(devinfo, inst, reg.width);
449             brw_inst_set_src0_vstride(devinfo, inst, reg.vstride);
450 	 }
451       } else {
452          brw_inst_set_src0_da16_swiz_x(devinfo, inst,
453             BRW_GET_SWZ(reg.swizzle, BRW_CHANNEL_X));
454          brw_inst_set_src0_da16_swiz_y(devinfo, inst,
455             BRW_GET_SWZ(reg.swizzle, BRW_CHANNEL_Y));
456          brw_inst_set_src0_da16_swiz_z(devinfo, inst,
457             BRW_GET_SWZ(reg.swizzle, BRW_CHANNEL_Z));
458          brw_inst_set_src0_da16_swiz_w(devinfo, inst,
459             BRW_GET_SWZ(reg.swizzle, BRW_CHANNEL_W));
460 
461 	 /* This is an oddity of the fact we're using the same
462 	  * descriptions for registers in align_16 as align_1:
463 	  */
464 	 if (reg.vstride == BRW_VERTICAL_STRIDE_8)
465             brw_inst_set_src0_vstride(devinfo, inst, BRW_VERTICAL_STRIDE_4);
466 	 else
467             brw_inst_set_src0_vstride(devinfo, inst, reg.vstride);
468       }
469    }
470 }
471 
472 
473 void
brw_set_src1(struct brw_codegen * p,brw_inst * inst,struct brw_reg reg)474 brw_set_src1(struct brw_codegen *p, brw_inst *inst, struct brw_reg reg)
475 {
476    const struct gen_device_info *devinfo = p->devinfo;
477 
478    if (reg.file != BRW_ARCHITECTURE_REGISTER_FILE)
479       assert(reg.nr < 128);
480 
481    /* From the IVB PRM Vol. 4, Pt. 3, Section 3.3.3.5:
482     *
483     *    "Accumulator registers may be accessed explicitly as src0
484     *    operands only."
485     */
486    assert(reg.file != BRW_ARCHITECTURE_REGISTER_FILE ||
487           reg.nr != BRW_ARF_ACCUMULATOR);
488 
489    gen7_convert_mrf_to_grf(p, &reg);
490    assert(reg.file != BRW_MESSAGE_REGISTER_FILE);
491 
492    validate_reg(devinfo, inst, reg);
493 
494    brw_inst_set_src1_reg_file(devinfo, inst, reg.file);
495    brw_inst_set_src1_reg_type(devinfo, inst,
496                               brw_reg_type_to_hw_type(devinfo, reg.type, reg.file));
497    brw_inst_set_src1_abs(devinfo, inst, reg.abs);
498    brw_inst_set_src1_negate(devinfo, inst, reg.negate);
499 
500    /* Only src1 can be immediate in two-argument instructions.
501     */
502    assert(brw_inst_src0_reg_file(devinfo, inst) != BRW_IMMEDIATE_VALUE);
503 
504    if (reg.file == BRW_IMMEDIATE_VALUE) {
505       /* two-argument instructions can only use 32-bit immediates */
506       assert(type_sz(reg.type) < 8);
507       brw_inst_set_imm_ud(devinfo, inst, reg.ud);
508    } else {
509       /* This is a hardware restriction, which may or may not be lifted
510        * in the future:
511        */
512       assert (reg.address_mode == BRW_ADDRESS_DIRECT);
513       /* assert (reg.file == BRW_GENERAL_REGISTER_FILE); */
514 
515       brw_inst_set_src1_da_reg_nr(devinfo, inst, reg.nr);
516       if (brw_inst_access_mode(devinfo, inst) == BRW_ALIGN_1) {
517          brw_inst_set_src1_da1_subreg_nr(devinfo, inst, reg.subnr);
518       } else {
519          brw_inst_set_src1_da16_subreg_nr(devinfo, inst, reg.subnr / 16);
520       }
521 
522       if (brw_inst_access_mode(devinfo, inst) == BRW_ALIGN_1) {
523 	 if (reg.width == BRW_WIDTH_1 &&
524              brw_inst_exec_size(devinfo, inst) == BRW_EXECUTE_1) {
525             brw_inst_set_src1_hstride(devinfo, inst, BRW_HORIZONTAL_STRIDE_0);
526             brw_inst_set_src1_width(devinfo, inst, BRW_WIDTH_1);
527             brw_inst_set_src1_vstride(devinfo, inst, BRW_VERTICAL_STRIDE_0);
528 	 } else {
529             brw_inst_set_src1_hstride(devinfo, inst, reg.hstride);
530             brw_inst_set_src1_width(devinfo, inst, reg.width);
531             brw_inst_set_src1_vstride(devinfo, inst, reg.vstride);
532 	 }
533       } else {
534          brw_inst_set_src1_da16_swiz_x(devinfo, inst,
535             BRW_GET_SWZ(reg.swizzle, BRW_CHANNEL_X));
536          brw_inst_set_src1_da16_swiz_y(devinfo, inst,
537             BRW_GET_SWZ(reg.swizzle, BRW_CHANNEL_Y));
538          brw_inst_set_src1_da16_swiz_z(devinfo, inst,
539             BRW_GET_SWZ(reg.swizzle, BRW_CHANNEL_Z));
540          brw_inst_set_src1_da16_swiz_w(devinfo, inst,
541             BRW_GET_SWZ(reg.swizzle, BRW_CHANNEL_W));
542 
543 	 /* This is an oddity of the fact we're using the same
544 	  * descriptions for registers in align_16 as align_1:
545 	  */
546 	 if (reg.vstride == BRW_VERTICAL_STRIDE_8)
547             brw_inst_set_src1_vstride(devinfo, inst, BRW_VERTICAL_STRIDE_4);
548 	 else
549             brw_inst_set_src1_vstride(devinfo, inst, reg.vstride);
550       }
551    }
552 }
553 
554 /**
555  * Set the Message Descriptor and Extended Message Descriptor fields
556  * for SEND messages.
557  *
558  * \note This zeroes out the Function Control bits, so it must be called
559  *       \b before filling out any message-specific data.  Callers can
560  *       choose not to fill in irrelevant bits; they will be zero.
561  */
562 void
brw_set_message_descriptor(struct brw_codegen * p,brw_inst * inst,enum brw_message_target sfid,unsigned msg_length,unsigned response_length,bool header_present,bool end_of_thread)563 brw_set_message_descriptor(struct brw_codegen *p,
564 			   brw_inst *inst,
565 			   enum brw_message_target sfid,
566 			   unsigned msg_length,
567 			   unsigned response_length,
568 			   bool header_present,
569 			   bool end_of_thread)
570 {
571    const struct gen_device_info *devinfo = p->devinfo;
572 
573    brw_set_src1(p, inst, brw_imm_d(0));
574 
575    /* For indirect sends, `inst` will not be the SEND/SENDC instruction
576     * itself; instead, it will be a MOV/OR into the address register.
577     *
578     * In this case, we avoid setting the extended message descriptor bits,
579     * since they go on the later SEND/SENDC instead and if set here would
580     * instead clobber the conditionalmod bits.
581     */
582    unsigned opcode = brw_inst_opcode(devinfo, inst);
583    if (opcode == BRW_OPCODE_SEND || opcode == BRW_OPCODE_SENDC) {
584       brw_inst_set_sfid(devinfo, inst, sfid);
585    }
586 
587    brw_inst_set_mlen(devinfo, inst, msg_length);
588    brw_inst_set_rlen(devinfo, inst, response_length);
589    brw_inst_set_eot(devinfo, inst, end_of_thread);
590 
591    if (devinfo->gen >= 5) {
592       brw_inst_set_header_present(devinfo, inst, header_present);
593    }
594 }
595 
brw_set_math_message(struct brw_codegen * p,brw_inst * inst,unsigned function,unsigned integer_type,bool low_precision,unsigned dataType)596 static void brw_set_math_message( struct brw_codegen *p,
597 				  brw_inst *inst,
598 				  unsigned function,
599 				  unsigned integer_type,
600 				  bool low_precision,
601 				  unsigned dataType )
602 {
603    const struct gen_device_info *devinfo = p->devinfo;
604    unsigned msg_length;
605    unsigned response_length;
606 
607    /* Infer message length from the function */
608    switch (function) {
609    case BRW_MATH_FUNCTION_POW:
610    case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT:
611    case BRW_MATH_FUNCTION_INT_DIV_REMAINDER:
612    case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER:
613       msg_length = 2;
614       break;
615    default:
616       msg_length = 1;
617       break;
618    }
619 
620    /* Infer response length from the function */
621    switch (function) {
622    case BRW_MATH_FUNCTION_SINCOS:
623    case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER:
624       response_length = 2;
625       break;
626    default:
627       response_length = 1;
628       break;
629    }
630 
631 
632    brw_set_message_descriptor(p, inst, BRW_SFID_MATH,
633 			      msg_length, response_length, false, false);
634    brw_inst_set_math_msg_function(devinfo, inst, function);
635    brw_inst_set_math_msg_signed_int(devinfo, inst, integer_type);
636    brw_inst_set_math_msg_precision(devinfo, inst, low_precision);
637    brw_inst_set_math_msg_saturate(devinfo, inst, brw_inst_saturate(devinfo, inst));
638    brw_inst_set_math_msg_data_type(devinfo, inst, dataType);
639    brw_inst_set_saturate(devinfo, inst, 0);
640 }
641 
642 
brw_set_ff_sync_message(struct brw_codegen * p,brw_inst * insn,bool allocate,unsigned response_length,bool end_of_thread)643 static void brw_set_ff_sync_message(struct brw_codegen *p,
644 				    brw_inst *insn,
645 				    bool allocate,
646 				    unsigned response_length,
647 				    bool end_of_thread)
648 {
649    const struct gen_device_info *devinfo = p->devinfo;
650 
651    brw_set_message_descriptor(p, insn, BRW_SFID_URB,
652 			      1, response_length, true, end_of_thread);
653    brw_inst_set_urb_opcode(devinfo, insn, 1); /* FF_SYNC */
654    brw_inst_set_urb_allocate(devinfo, insn, allocate);
655    /* The following fields are not used by FF_SYNC: */
656    brw_inst_set_urb_global_offset(devinfo, insn, 0);
657    brw_inst_set_urb_swizzle_control(devinfo, insn, 0);
658    brw_inst_set_urb_used(devinfo, insn, 0);
659    brw_inst_set_urb_complete(devinfo, insn, 0);
660 }
661 
brw_set_urb_message(struct brw_codegen * p,brw_inst * insn,enum brw_urb_write_flags flags,unsigned msg_length,unsigned response_length,unsigned offset,unsigned swizzle_control)662 static void brw_set_urb_message( struct brw_codegen *p,
663 				 brw_inst *insn,
664                                  enum brw_urb_write_flags flags,
665 				 unsigned msg_length,
666 				 unsigned response_length,
667 				 unsigned offset,
668 				 unsigned swizzle_control )
669 {
670    const struct gen_device_info *devinfo = p->devinfo;
671 
672    assert(devinfo->gen < 7 || swizzle_control != BRW_URB_SWIZZLE_TRANSPOSE);
673    assert(devinfo->gen < 7 || !(flags & BRW_URB_WRITE_ALLOCATE));
674    assert(devinfo->gen >= 7 || !(flags & BRW_URB_WRITE_PER_SLOT_OFFSET));
675 
676    brw_set_message_descriptor(p, insn, BRW_SFID_URB,
677 			      msg_length, response_length, true,
678                               flags & BRW_URB_WRITE_EOT);
679 
680    if (flags & BRW_URB_WRITE_OWORD) {
681       assert(msg_length == 2); /* header + one OWORD of data */
682       brw_inst_set_urb_opcode(devinfo, insn, BRW_URB_OPCODE_WRITE_OWORD);
683    } else {
684       brw_inst_set_urb_opcode(devinfo, insn, BRW_URB_OPCODE_WRITE_HWORD);
685    }
686 
687    brw_inst_set_urb_global_offset(devinfo, insn, offset);
688    brw_inst_set_urb_swizzle_control(devinfo, insn, swizzle_control);
689 
690    if (devinfo->gen < 8) {
691       brw_inst_set_urb_complete(devinfo, insn, !!(flags & BRW_URB_WRITE_COMPLETE));
692    }
693 
694    if (devinfo->gen < 7) {
695       brw_inst_set_urb_allocate(devinfo, insn, !!(flags & BRW_URB_WRITE_ALLOCATE));
696       brw_inst_set_urb_used(devinfo, insn, !(flags & BRW_URB_WRITE_UNUSED));
697    } else {
698       brw_inst_set_urb_per_slot_offset(devinfo, insn,
699          !!(flags & BRW_URB_WRITE_PER_SLOT_OFFSET));
700    }
701 }
702 
703 void
brw_set_dp_write_message(struct brw_codegen * p,brw_inst * insn,unsigned binding_table_index,unsigned msg_control,unsigned msg_type,unsigned target_cache,unsigned msg_length,bool header_present,unsigned last_render_target,unsigned response_length,unsigned end_of_thread,unsigned send_commit_msg)704 brw_set_dp_write_message(struct brw_codegen *p,
705 			 brw_inst *insn,
706 			 unsigned binding_table_index,
707 			 unsigned msg_control,
708 			 unsigned msg_type,
709                          unsigned target_cache,
710 			 unsigned msg_length,
711 			 bool header_present,
712 			 unsigned last_render_target,
713 			 unsigned response_length,
714 			 unsigned end_of_thread,
715 			 unsigned send_commit_msg)
716 {
717    const struct gen_device_info *devinfo = p->devinfo;
718    const unsigned sfid = (devinfo->gen >= 6 ? target_cache :
719                           BRW_SFID_DATAPORT_WRITE);
720 
721    brw_set_message_descriptor(p, insn, sfid, msg_length, response_length,
722 			      header_present, end_of_thread);
723 
724    brw_inst_set_binding_table_index(devinfo, insn, binding_table_index);
725    brw_inst_set_dp_write_msg_type(devinfo, insn, msg_type);
726    brw_inst_set_dp_write_msg_control(devinfo, insn, msg_control);
727    brw_inst_set_rt_last(devinfo, insn, last_render_target);
728    if (devinfo->gen < 7) {
729       brw_inst_set_dp_write_commit(devinfo, insn, send_commit_msg);
730    }
731 }
732 
733 void
brw_set_dp_read_message(struct brw_codegen * p,brw_inst * insn,unsigned binding_table_index,unsigned msg_control,unsigned msg_type,unsigned target_cache,unsigned msg_length,bool header_present,unsigned response_length)734 brw_set_dp_read_message(struct brw_codegen *p,
735 			brw_inst *insn,
736 			unsigned binding_table_index,
737 			unsigned msg_control,
738 			unsigned msg_type,
739 			unsigned target_cache,
740 			unsigned msg_length,
741                         bool header_present,
742 			unsigned response_length)
743 {
744    const struct gen_device_info *devinfo = p->devinfo;
745    const unsigned sfid = (devinfo->gen >= 6 ? target_cache :
746                           BRW_SFID_DATAPORT_READ);
747 
748    brw_set_message_descriptor(p, insn, sfid, msg_length, response_length,
749 			      header_present, false);
750 
751    brw_inst_set_binding_table_index(devinfo, insn, binding_table_index);
752    brw_inst_set_dp_read_msg_type(devinfo, insn, msg_type);
753    brw_inst_set_dp_read_msg_control(devinfo, insn, msg_control);
754    if (devinfo->gen < 6)
755       brw_inst_set_dp_read_target_cache(devinfo, insn, target_cache);
756 }
757 
758 void
brw_set_sampler_message(struct brw_codegen * p,brw_inst * inst,unsigned binding_table_index,unsigned sampler,unsigned msg_type,unsigned response_length,unsigned msg_length,unsigned header_present,unsigned simd_mode,unsigned return_format)759 brw_set_sampler_message(struct brw_codegen *p,
760                         brw_inst *inst,
761                         unsigned binding_table_index,
762                         unsigned sampler,
763                         unsigned msg_type,
764                         unsigned response_length,
765                         unsigned msg_length,
766                         unsigned header_present,
767                         unsigned simd_mode,
768                         unsigned return_format)
769 {
770    const struct gen_device_info *devinfo = p->devinfo;
771 
772    brw_set_message_descriptor(p, inst, BRW_SFID_SAMPLER, msg_length,
773 			      response_length, header_present, false);
774 
775    brw_inst_set_binding_table_index(devinfo, inst, binding_table_index);
776    brw_inst_set_sampler(devinfo, inst, sampler);
777    brw_inst_set_sampler_msg_type(devinfo, inst, msg_type);
778    if (devinfo->gen >= 5) {
779       brw_inst_set_sampler_simd_mode(devinfo, inst, simd_mode);
780    } else if (devinfo->gen == 4 && !devinfo->is_g4x) {
781       brw_inst_set_sampler_return_format(devinfo, inst, return_format);
782    }
783 }
784 
785 static void
gen7_set_dp_scratch_message(struct brw_codegen * p,brw_inst * inst,bool write,bool dword,bool invalidate_after_read,unsigned num_regs,unsigned addr_offset,unsigned mlen,unsigned rlen,bool header_present)786 gen7_set_dp_scratch_message(struct brw_codegen *p,
787                             brw_inst *inst,
788                             bool write,
789                             bool dword,
790                             bool invalidate_after_read,
791                             unsigned num_regs,
792                             unsigned addr_offset,
793                             unsigned mlen,
794                             unsigned rlen,
795                             bool header_present)
796 {
797    const struct gen_device_info *devinfo = p->devinfo;
798    assert(num_regs == 1 || num_regs == 2 || num_regs == 4 ||
799           (devinfo->gen >= 8 && num_regs == 8));
800    const unsigned block_size = (devinfo->gen >= 8 ? _mesa_logbase2(num_regs) :
801                                 num_regs - 1);
802 
803    brw_set_message_descriptor(p, inst, GEN7_SFID_DATAPORT_DATA_CACHE,
804                               mlen, rlen, header_present, false);
805    brw_inst_set_dp_category(devinfo, inst, 1); /* Scratch Block Read/Write msgs */
806    brw_inst_set_scratch_read_write(devinfo, inst, write);
807    brw_inst_set_scratch_type(devinfo, inst, dword);
808    brw_inst_set_scratch_invalidate_after_read(devinfo, inst, invalidate_after_read);
809    brw_inst_set_scratch_block_size(devinfo, inst, block_size);
810    brw_inst_set_scratch_addr_offset(devinfo, inst, addr_offset);
811 }
812 
813 #define next_insn brw_next_insn
814 brw_inst *
brw_next_insn(struct brw_codegen * p,unsigned opcode)815 brw_next_insn(struct brw_codegen *p, unsigned opcode)
816 {
817    const struct gen_device_info *devinfo = p->devinfo;
818    brw_inst *insn;
819 
820    if (p->nr_insn + 1 > p->store_size) {
821       p->store_size <<= 1;
822       p->store = reralloc(p->mem_ctx, p->store, brw_inst, p->store_size);
823    }
824 
825    p->next_insn_offset += 16;
826    insn = &p->store[p->nr_insn++];
827    memcpy(insn, p->current, sizeof(*insn));
828 
829    brw_inst_set_opcode(devinfo, insn, opcode);
830    return insn;
831 }
832 
833 static brw_inst *
brw_alu1(struct brw_codegen * p,unsigned opcode,struct brw_reg dest,struct brw_reg src)834 brw_alu1(struct brw_codegen *p, unsigned opcode,
835          struct brw_reg dest, struct brw_reg src)
836 {
837    brw_inst *insn = next_insn(p, opcode);
838    brw_set_dest(p, insn, dest);
839    brw_set_src0(p, insn, src);
840    return insn;
841 }
842 
843 static brw_inst *
brw_alu2(struct brw_codegen * p,unsigned opcode,struct brw_reg dest,struct brw_reg src0,struct brw_reg src1)844 brw_alu2(struct brw_codegen *p, unsigned opcode,
845          struct brw_reg dest, struct brw_reg src0, struct brw_reg src1)
846 {
847    /* 64-bit immediates are only supported on 1-src instructions */
848    assert(src0.file != BRW_IMMEDIATE_VALUE || type_sz(src0.type) <= 4);
849    assert(src1.file != BRW_IMMEDIATE_VALUE || type_sz(src1.type) <= 4);
850 
851    brw_inst *insn = next_insn(p, opcode);
852    brw_set_dest(p, insn, dest);
853    brw_set_src0(p, insn, src0);
854    brw_set_src1(p, insn, src1);
855    return insn;
856 }
857 
858 static int
get_3src_subreg_nr(struct brw_reg reg)859 get_3src_subreg_nr(struct brw_reg reg)
860 {
861    /* Normally, SubRegNum is in bytes (0..31).  However, 3-src instructions
862     * use 32-bit units (components 0..7).  Since they only support F/D/UD
863     * types, this doesn't lose any flexibility, but uses fewer bits.
864     */
865    return reg.subnr / 4;
866 }
867 
868 static brw_inst *
brw_alu3(struct brw_codegen * p,unsigned opcode,struct brw_reg dest,struct brw_reg src0,struct brw_reg src1,struct brw_reg src2)869 brw_alu3(struct brw_codegen *p, unsigned opcode, struct brw_reg dest,
870          struct brw_reg src0, struct brw_reg src1, struct brw_reg src2)
871 {
872    const struct gen_device_info *devinfo = p->devinfo;
873    brw_inst *inst = next_insn(p, opcode);
874 
875    gen7_convert_mrf_to_grf(p, &dest);
876 
877    assert(brw_inst_access_mode(devinfo, inst) == BRW_ALIGN_16);
878 
879    assert(dest.file == BRW_GENERAL_REGISTER_FILE ||
880 	  dest.file == BRW_MESSAGE_REGISTER_FILE);
881    assert(dest.nr < 128);
882    assert(dest.address_mode == BRW_ADDRESS_DIRECT);
883    assert(dest.type == BRW_REGISTER_TYPE_F  ||
884           dest.type == BRW_REGISTER_TYPE_DF ||
885           dest.type == BRW_REGISTER_TYPE_D  ||
886           dest.type == BRW_REGISTER_TYPE_UD);
887    if (devinfo->gen == 6) {
888       brw_inst_set_3src_dst_reg_file(devinfo, inst,
889                                      dest.file == BRW_MESSAGE_REGISTER_FILE);
890    }
891    brw_inst_set_3src_dst_reg_nr(devinfo, inst, dest.nr);
892    brw_inst_set_3src_dst_subreg_nr(devinfo, inst, dest.subnr / 16);
893    brw_inst_set_3src_dst_writemask(devinfo, inst, dest.writemask);
894 
895    assert(src0.file == BRW_GENERAL_REGISTER_FILE);
896    assert(src0.address_mode == BRW_ADDRESS_DIRECT);
897    assert(src0.nr < 128);
898    brw_inst_set_3src_src0_swizzle(devinfo, inst, src0.swizzle);
899    brw_inst_set_3src_src0_subreg_nr(devinfo, inst, get_3src_subreg_nr(src0));
900    brw_inst_set_3src_src0_reg_nr(devinfo, inst, src0.nr);
901    brw_inst_set_3src_src0_abs(devinfo, inst, src0.abs);
902    brw_inst_set_3src_src0_negate(devinfo, inst, src0.negate);
903    brw_inst_set_3src_src0_rep_ctrl(devinfo, inst,
904                                    src0.vstride == BRW_VERTICAL_STRIDE_0);
905 
906    assert(src1.file == BRW_GENERAL_REGISTER_FILE);
907    assert(src1.address_mode == BRW_ADDRESS_DIRECT);
908    assert(src1.nr < 128);
909    brw_inst_set_3src_src1_swizzle(devinfo, inst, src1.swizzle);
910    brw_inst_set_3src_src1_subreg_nr(devinfo, inst, get_3src_subreg_nr(src1));
911    brw_inst_set_3src_src1_reg_nr(devinfo, inst, src1.nr);
912    brw_inst_set_3src_src1_abs(devinfo, inst, src1.abs);
913    brw_inst_set_3src_src1_negate(devinfo, inst, src1.negate);
914    brw_inst_set_3src_src1_rep_ctrl(devinfo, inst,
915                                    src1.vstride == BRW_VERTICAL_STRIDE_0);
916 
917    assert(src2.file == BRW_GENERAL_REGISTER_FILE);
918    assert(src2.address_mode == BRW_ADDRESS_DIRECT);
919    assert(src2.nr < 128);
920    brw_inst_set_3src_src2_swizzle(devinfo, inst, src2.swizzle);
921    brw_inst_set_3src_src2_subreg_nr(devinfo, inst, get_3src_subreg_nr(src2));
922    brw_inst_set_3src_src2_reg_nr(devinfo, inst, src2.nr);
923    brw_inst_set_3src_src2_abs(devinfo, inst, src2.abs);
924    brw_inst_set_3src_src2_negate(devinfo, inst, src2.negate);
925    brw_inst_set_3src_src2_rep_ctrl(devinfo, inst,
926                                    src2.vstride == BRW_VERTICAL_STRIDE_0);
927 
928    if (devinfo->gen >= 7) {
929       /* Set both the source and destination types based on dest.type,
930        * ignoring the source register types.  The MAD and LRP emitters ensure
931        * that all four types are float.  The BFE and BFI2 emitters, however,
932        * may send us mixed D and UD types and want us to ignore that and use
933        * the destination type.
934        */
935       switch (dest.type) {
936       case BRW_REGISTER_TYPE_F:
937          brw_inst_set_3src_src_type(devinfo, inst, BRW_3SRC_TYPE_F);
938          brw_inst_set_3src_dst_type(devinfo, inst, BRW_3SRC_TYPE_F);
939          break;
940       case BRW_REGISTER_TYPE_DF:
941          brw_inst_set_3src_src_type(devinfo, inst, BRW_3SRC_TYPE_DF);
942          brw_inst_set_3src_dst_type(devinfo, inst, BRW_3SRC_TYPE_DF);
943          break;
944       case BRW_REGISTER_TYPE_D:
945          brw_inst_set_3src_src_type(devinfo, inst, BRW_3SRC_TYPE_D);
946          brw_inst_set_3src_dst_type(devinfo, inst, BRW_3SRC_TYPE_D);
947          break;
948       case BRW_REGISTER_TYPE_UD:
949          brw_inst_set_3src_src_type(devinfo, inst, BRW_3SRC_TYPE_UD);
950          brw_inst_set_3src_dst_type(devinfo, inst, BRW_3SRC_TYPE_UD);
951          break;
952       default:
953          unreachable("not reached");
954       }
955    }
956 
957    return inst;
958 }
959 
960 
961 /***********************************************************************
962  * Convenience routines.
963  */
964 #define ALU1(OP)					\
965 brw_inst *brw_##OP(struct brw_codegen *p,		\
966 	      struct brw_reg dest,			\
967 	      struct brw_reg src0)   			\
968 {							\
969    return brw_alu1(p, BRW_OPCODE_##OP, dest, src0);    	\
970 }
971 
972 #define ALU2(OP)					\
973 brw_inst *brw_##OP(struct brw_codegen *p,		\
974 	      struct brw_reg dest,			\
975 	      struct brw_reg src0,			\
976 	      struct brw_reg src1)   			\
977 {							\
978    return brw_alu2(p, BRW_OPCODE_##OP, dest, src0, src1);	\
979 }
980 
981 #define ALU3(OP)					\
982 brw_inst *brw_##OP(struct brw_codegen *p,		\
983 	      struct brw_reg dest,			\
984 	      struct brw_reg src0,			\
985 	      struct brw_reg src1,			\
986 	      struct brw_reg src2)   			\
987 {							\
988    return brw_alu3(p, BRW_OPCODE_##OP, dest, src0, src1, src2);	\
989 }
990 
991 #define ALU3F(OP)                                               \
992 brw_inst *brw_##OP(struct brw_codegen *p,         \
993                                  struct brw_reg dest,           \
994                                  struct brw_reg src0,           \
995                                  struct brw_reg src1,           \
996                                  struct brw_reg src2)           \
997 {                                                               \
998    assert(dest.type == BRW_REGISTER_TYPE_F ||                   \
999           dest.type == BRW_REGISTER_TYPE_DF);                   \
1000    if (dest.type == BRW_REGISTER_TYPE_F) {                      \
1001       assert(src0.type == BRW_REGISTER_TYPE_F);                 \
1002       assert(src1.type == BRW_REGISTER_TYPE_F);                 \
1003       assert(src2.type == BRW_REGISTER_TYPE_F);                 \
1004    } else if (dest.type == BRW_REGISTER_TYPE_DF) {              \
1005       assert(src0.type == BRW_REGISTER_TYPE_DF);                \
1006       assert(src1.type == BRW_REGISTER_TYPE_DF);                \
1007       assert(src2.type == BRW_REGISTER_TYPE_DF);                \
1008    }                                                            \
1009    return brw_alu3(p, BRW_OPCODE_##OP, dest, src0, src1, src2); \
1010 }
1011 
1012 /* Rounding operations (other than RNDD) require two instructions - the first
1013  * stores a rounded value (possibly the wrong way) in the dest register, but
1014  * also sets a per-channel "increment bit" in the flag register.  A predicated
1015  * add of 1.0 fixes dest to contain the desired result.
1016  *
1017  * Sandybridge and later appear to round correctly without an ADD.
1018  */
1019 #define ROUND(OP)							      \
1020 void brw_##OP(struct brw_codegen *p,					      \
1021 	      struct brw_reg dest,					      \
1022 	      struct brw_reg src)					      \
1023 {									      \
1024    const struct gen_device_info *devinfo = p->devinfo;					      \
1025    brw_inst *rnd, *add;							      \
1026    rnd = next_insn(p, BRW_OPCODE_##OP);					      \
1027    brw_set_dest(p, rnd, dest);						      \
1028    brw_set_src0(p, rnd, src);						      \
1029 									      \
1030    if (devinfo->gen < 6) {							      \
1031       /* turn on round-increments */					      \
1032       brw_inst_set_cond_modifier(devinfo, rnd, BRW_CONDITIONAL_R);            \
1033       add = brw_ADD(p, dest, dest, brw_imm_f(1.0f));			      \
1034       brw_inst_set_pred_control(devinfo, add, BRW_PREDICATE_NORMAL);          \
1035    }									      \
1036 }
1037 
1038 
1039 ALU1(MOV)
ALU2(SEL)1040 ALU2(SEL)
1041 ALU1(NOT)
1042 ALU2(AND)
1043 ALU2(OR)
1044 ALU2(XOR)
1045 ALU2(SHR)
1046 ALU2(SHL)
1047 ALU1(DIM)
1048 ALU2(ASR)
1049 ALU1(FRC)
1050 ALU1(RNDD)
1051 ALU2(MAC)
1052 ALU2(MACH)
1053 ALU1(LZD)
1054 ALU2(DP4)
1055 ALU2(DPH)
1056 ALU2(DP3)
1057 ALU2(DP2)
1058 ALU3F(MAD)
1059 ALU3F(LRP)
1060 ALU1(BFREV)
1061 ALU3(BFE)
1062 ALU2(BFI1)
1063 ALU3(BFI2)
1064 ALU1(FBH)
1065 ALU1(FBL)
1066 ALU1(CBIT)
1067 ALU2(ADDC)
1068 ALU2(SUBB)
1069 
1070 ROUND(RNDZ)
1071 ROUND(RNDE)
1072 
1073 
1074 brw_inst *
1075 brw_ADD(struct brw_codegen *p, struct brw_reg dest,
1076         struct brw_reg src0, struct brw_reg src1)
1077 {
1078    /* 6.2.2: add */
1079    if (src0.type == BRW_REGISTER_TYPE_F ||
1080        (src0.file == BRW_IMMEDIATE_VALUE &&
1081 	src0.type == BRW_REGISTER_TYPE_VF)) {
1082       assert(src1.type != BRW_REGISTER_TYPE_UD);
1083       assert(src1.type != BRW_REGISTER_TYPE_D);
1084    }
1085 
1086    if (src1.type == BRW_REGISTER_TYPE_F ||
1087        (src1.file == BRW_IMMEDIATE_VALUE &&
1088 	src1.type == BRW_REGISTER_TYPE_VF)) {
1089       assert(src0.type != BRW_REGISTER_TYPE_UD);
1090       assert(src0.type != BRW_REGISTER_TYPE_D);
1091    }
1092 
1093    return brw_alu2(p, BRW_OPCODE_ADD, dest, src0, src1);
1094 }
1095 
1096 brw_inst *
brw_AVG(struct brw_codegen * p,struct brw_reg dest,struct brw_reg src0,struct brw_reg src1)1097 brw_AVG(struct brw_codegen *p, struct brw_reg dest,
1098         struct brw_reg src0, struct brw_reg src1)
1099 {
1100    assert(dest.type == src0.type);
1101    assert(src0.type == src1.type);
1102    switch (src0.type) {
1103    case BRW_REGISTER_TYPE_B:
1104    case BRW_REGISTER_TYPE_UB:
1105    case BRW_REGISTER_TYPE_W:
1106    case BRW_REGISTER_TYPE_UW:
1107    case BRW_REGISTER_TYPE_D:
1108    case BRW_REGISTER_TYPE_UD:
1109       break;
1110    default:
1111       unreachable("Bad type for brw_AVG");
1112    }
1113 
1114    return brw_alu2(p, BRW_OPCODE_AVG, dest, src0, src1);
1115 }
1116 
1117 brw_inst *
brw_MUL(struct brw_codegen * p,struct brw_reg dest,struct brw_reg src0,struct brw_reg src1)1118 brw_MUL(struct brw_codegen *p, struct brw_reg dest,
1119         struct brw_reg src0, struct brw_reg src1)
1120 {
1121    /* 6.32.38: mul */
1122    if (src0.type == BRW_REGISTER_TYPE_D ||
1123        src0.type == BRW_REGISTER_TYPE_UD ||
1124        src1.type == BRW_REGISTER_TYPE_D ||
1125        src1.type == BRW_REGISTER_TYPE_UD) {
1126       assert(dest.type != BRW_REGISTER_TYPE_F);
1127    }
1128 
1129    if (src0.type == BRW_REGISTER_TYPE_F ||
1130        (src0.file == BRW_IMMEDIATE_VALUE &&
1131 	src0.type == BRW_REGISTER_TYPE_VF)) {
1132       assert(src1.type != BRW_REGISTER_TYPE_UD);
1133       assert(src1.type != BRW_REGISTER_TYPE_D);
1134    }
1135 
1136    if (src1.type == BRW_REGISTER_TYPE_F ||
1137        (src1.file == BRW_IMMEDIATE_VALUE &&
1138 	src1.type == BRW_REGISTER_TYPE_VF)) {
1139       assert(src0.type != BRW_REGISTER_TYPE_UD);
1140       assert(src0.type != BRW_REGISTER_TYPE_D);
1141    }
1142 
1143    assert(src0.file != BRW_ARCHITECTURE_REGISTER_FILE ||
1144 	  src0.nr != BRW_ARF_ACCUMULATOR);
1145    assert(src1.file != BRW_ARCHITECTURE_REGISTER_FILE ||
1146 	  src1.nr != BRW_ARF_ACCUMULATOR);
1147 
1148    return brw_alu2(p, BRW_OPCODE_MUL, dest, src0, src1);
1149 }
1150 
1151 brw_inst *
brw_LINE(struct brw_codegen * p,struct brw_reg dest,struct brw_reg src0,struct brw_reg src1)1152 brw_LINE(struct brw_codegen *p, struct brw_reg dest,
1153          struct brw_reg src0, struct brw_reg src1)
1154 {
1155    src0.vstride = BRW_VERTICAL_STRIDE_0;
1156    src0.width = BRW_WIDTH_1;
1157    src0.hstride = BRW_HORIZONTAL_STRIDE_0;
1158    return brw_alu2(p, BRW_OPCODE_LINE, dest, src0, src1);
1159 }
1160 
1161 brw_inst *
brw_PLN(struct brw_codegen * p,struct brw_reg dest,struct brw_reg src0,struct brw_reg src1)1162 brw_PLN(struct brw_codegen *p, struct brw_reg dest,
1163         struct brw_reg src0, struct brw_reg src1)
1164 {
1165    src0.vstride = BRW_VERTICAL_STRIDE_0;
1166    src0.width = BRW_WIDTH_1;
1167    src0.hstride = BRW_HORIZONTAL_STRIDE_0;
1168    src1.vstride = BRW_VERTICAL_STRIDE_8;
1169    src1.width = BRW_WIDTH_8;
1170    src1.hstride = BRW_HORIZONTAL_STRIDE_1;
1171    return brw_alu2(p, BRW_OPCODE_PLN, dest, src0, src1);
1172 }
1173 
1174 brw_inst *
brw_F32TO16(struct brw_codegen * p,struct brw_reg dst,struct brw_reg src)1175 brw_F32TO16(struct brw_codegen *p, struct brw_reg dst, struct brw_reg src)
1176 {
1177    const struct gen_device_info *devinfo = p->devinfo;
1178    const bool align16 = brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_16;
1179    /* The F32TO16 instruction doesn't support 32-bit destination types in
1180     * Align1 mode, and neither does the Gen8 implementation in terms of a
1181     * converting MOV.  Gen7 does zero out the high 16 bits in Align16 mode as
1182     * an undocumented feature.
1183     */
1184    const bool needs_zero_fill = (dst.type == BRW_REGISTER_TYPE_UD &&
1185                                  (!align16 || devinfo->gen >= 8));
1186    brw_inst *inst;
1187 
1188    if (align16) {
1189       assert(dst.type == BRW_REGISTER_TYPE_UD);
1190    } else {
1191       assert(dst.type == BRW_REGISTER_TYPE_UD ||
1192              dst.type == BRW_REGISTER_TYPE_W ||
1193              dst.type == BRW_REGISTER_TYPE_UW ||
1194              dst.type == BRW_REGISTER_TYPE_HF);
1195    }
1196 
1197    brw_push_insn_state(p);
1198 
1199    if (needs_zero_fill) {
1200       brw_set_default_access_mode(p, BRW_ALIGN_1);
1201       dst = spread(retype(dst, BRW_REGISTER_TYPE_W), 2);
1202    }
1203 
1204    if (devinfo->gen >= 8) {
1205       inst = brw_MOV(p, retype(dst, BRW_REGISTER_TYPE_HF), src);
1206    } else {
1207       assert(devinfo->gen == 7);
1208       inst = brw_alu1(p, BRW_OPCODE_F32TO16, dst, src);
1209    }
1210 
1211    if (needs_zero_fill) {
1212       brw_inst_set_no_dd_clear(devinfo, inst, true);
1213       inst = brw_MOV(p, suboffset(dst, 1), brw_imm_ud(0u));
1214       brw_inst_set_no_dd_check(devinfo, inst, true);
1215    }
1216 
1217    brw_pop_insn_state(p);
1218    return inst;
1219 }
1220 
1221 brw_inst *
brw_F16TO32(struct brw_codegen * p,struct brw_reg dst,struct brw_reg src)1222 brw_F16TO32(struct brw_codegen *p, struct brw_reg dst, struct brw_reg src)
1223 {
1224    const struct gen_device_info *devinfo = p->devinfo;
1225    bool align16 = brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_16;
1226 
1227    if (align16) {
1228       assert(src.type == BRW_REGISTER_TYPE_UD);
1229    } else {
1230       /* From the Ivybridge PRM, Vol4, Part3, Section 6.26 f16to32:
1231        *
1232        *   Because this instruction does not have a 16-bit floating-point
1233        *   type, the source data type must be Word (W). The destination type
1234        *   must be F (Float).
1235        */
1236       if (src.type == BRW_REGISTER_TYPE_UD)
1237          src = spread(retype(src, BRW_REGISTER_TYPE_W), 2);
1238 
1239       assert(src.type == BRW_REGISTER_TYPE_W ||
1240              src.type == BRW_REGISTER_TYPE_UW ||
1241              src.type == BRW_REGISTER_TYPE_HF);
1242    }
1243 
1244    if (devinfo->gen >= 8) {
1245       return brw_MOV(p, dst, retype(src, BRW_REGISTER_TYPE_HF));
1246    } else {
1247       assert(devinfo->gen == 7);
1248       return brw_alu1(p, BRW_OPCODE_F16TO32, dst, src);
1249    }
1250 }
1251 
1252 
brw_NOP(struct brw_codegen * p)1253 void brw_NOP(struct brw_codegen *p)
1254 {
1255    brw_inst *insn = next_insn(p, BRW_OPCODE_NOP);
1256    memset(insn, 0, sizeof(*insn));
1257    brw_inst_set_opcode(p->devinfo, insn, BRW_OPCODE_NOP);
1258 }
1259 
1260 
1261 
1262 
1263 
1264 /***********************************************************************
1265  * Comparisons, if/else/endif
1266  */
1267 
1268 brw_inst *
brw_JMPI(struct brw_codegen * p,struct brw_reg index,unsigned predicate_control)1269 brw_JMPI(struct brw_codegen *p, struct brw_reg index,
1270          unsigned predicate_control)
1271 {
1272    const struct gen_device_info *devinfo = p->devinfo;
1273    struct brw_reg ip = brw_ip_reg();
1274    brw_inst *inst = brw_alu2(p, BRW_OPCODE_JMPI, ip, ip, index);
1275 
1276    brw_inst_set_exec_size(devinfo, inst, BRW_EXECUTE_2);
1277    brw_inst_set_qtr_control(devinfo, inst, BRW_COMPRESSION_NONE);
1278    brw_inst_set_mask_control(devinfo, inst, BRW_MASK_DISABLE);
1279    brw_inst_set_pred_control(devinfo, inst, predicate_control);
1280 
1281    return inst;
1282 }
1283 
1284 static void
push_if_stack(struct brw_codegen * p,brw_inst * inst)1285 push_if_stack(struct brw_codegen *p, brw_inst *inst)
1286 {
1287    p->if_stack[p->if_stack_depth] = inst - p->store;
1288 
1289    p->if_stack_depth++;
1290    if (p->if_stack_array_size <= p->if_stack_depth) {
1291       p->if_stack_array_size *= 2;
1292       p->if_stack = reralloc(p->mem_ctx, p->if_stack, int,
1293 			     p->if_stack_array_size);
1294    }
1295 }
1296 
1297 static brw_inst *
pop_if_stack(struct brw_codegen * p)1298 pop_if_stack(struct brw_codegen *p)
1299 {
1300    p->if_stack_depth--;
1301    return &p->store[p->if_stack[p->if_stack_depth]];
1302 }
1303 
1304 static void
push_loop_stack(struct brw_codegen * p,brw_inst * inst)1305 push_loop_stack(struct brw_codegen *p, brw_inst *inst)
1306 {
1307    if (p->loop_stack_array_size <= (p->loop_stack_depth + 1)) {
1308       p->loop_stack_array_size *= 2;
1309       p->loop_stack = reralloc(p->mem_ctx, p->loop_stack, int,
1310 			       p->loop_stack_array_size);
1311       p->if_depth_in_loop = reralloc(p->mem_ctx, p->if_depth_in_loop, int,
1312 				     p->loop_stack_array_size);
1313    }
1314 
1315    p->loop_stack[p->loop_stack_depth] = inst - p->store;
1316    p->loop_stack_depth++;
1317    p->if_depth_in_loop[p->loop_stack_depth] = 0;
1318 }
1319 
1320 static brw_inst *
get_inner_do_insn(struct brw_codegen * p)1321 get_inner_do_insn(struct brw_codegen *p)
1322 {
1323    return &p->store[p->loop_stack[p->loop_stack_depth - 1]];
1324 }
1325 
1326 /* EU takes the value from the flag register and pushes it onto some
1327  * sort of a stack (presumably merging with any flag value already on
1328  * the stack).  Within an if block, the flags at the top of the stack
1329  * control execution on each channel of the unit, eg. on each of the
1330  * 16 pixel values in our wm programs.
1331  *
1332  * When the matching 'else' instruction is reached (presumably by
1333  * countdown of the instruction count patched in by our ELSE/ENDIF
1334  * functions), the relevant flags are inverted.
1335  *
1336  * When the matching 'endif' instruction is reached, the flags are
1337  * popped off.  If the stack is now empty, normal execution resumes.
1338  */
1339 brw_inst *
brw_IF(struct brw_codegen * p,unsigned execute_size)1340 brw_IF(struct brw_codegen *p, unsigned execute_size)
1341 {
1342    const struct gen_device_info *devinfo = p->devinfo;
1343    brw_inst *insn;
1344 
1345    insn = next_insn(p, BRW_OPCODE_IF);
1346 
1347    /* Override the defaults for this instruction:
1348     */
1349    if (devinfo->gen < 6) {
1350       brw_set_dest(p, insn, brw_ip_reg());
1351       brw_set_src0(p, insn, brw_ip_reg());
1352       brw_set_src1(p, insn, brw_imm_d(0x0));
1353    } else if (devinfo->gen == 6) {
1354       brw_set_dest(p, insn, brw_imm_w(0));
1355       brw_inst_set_gen6_jump_count(devinfo, insn, 0);
1356       brw_set_src0(p, insn, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D)));
1357       brw_set_src1(p, insn, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D)));
1358    } else if (devinfo->gen == 7) {
1359       brw_set_dest(p, insn, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D)));
1360       brw_set_src0(p, insn, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D)));
1361       brw_set_src1(p, insn, brw_imm_w(0));
1362       brw_inst_set_jip(devinfo, insn, 0);
1363       brw_inst_set_uip(devinfo, insn, 0);
1364    } else {
1365       brw_set_dest(p, insn, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D)));
1366       brw_set_src0(p, insn, brw_imm_d(0));
1367       brw_inst_set_jip(devinfo, insn, 0);
1368       brw_inst_set_uip(devinfo, insn, 0);
1369    }
1370 
1371    brw_inst_set_exec_size(devinfo, insn, execute_size);
1372    brw_inst_set_qtr_control(devinfo, insn, BRW_COMPRESSION_NONE);
1373    brw_inst_set_pred_control(devinfo, insn, BRW_PREDICATE_NORMAL);
1374    brw_inst_set_mask_control(devinfo, insn, BRW_MASK_ENABLE);
1375    if (!p->single_program_flow && devinfo->gen < 6)
1376       brw_inst_set_thread_control(devinfo, insn, BRW_THREAD_SWITCH);
1377 
1378    push_if_stack(p, insn);
1379    p->if_depth_in_loop[p->loop_stack_depth]++;
1380    return insn;
1381 }
1382 
1383 /* This function is only used for gen6-style IF instructions with an
1384  * embedded comparison (conditional modifier).  It is not used on gen7.
1385  */
1386 brw_inst *
gen6_IF(struct brw_codegen * p,enum brw_conditional_mod conditional,struct brw_reg src0,struct brw_reg src1)1387 gen6_IF(struct brw_codegen *p, enum brw_conditional_mod conditional,
1388 	struct brw_reg src0, struct brw_reg src1)
1389 {
1390    const struct gen_device_info *devinfo = p->devinfo;
1391    brw_inst *insn;
1392 
1393    insn = next_insn(p, BRW_OPCODE_IF);
1394 
1395    brw_set_dest(p, insn, brw_imm_w(0));
1396    brw_inst_set_exec_size(devinfo, insn,
1397                           brw_inst_exec_size(devinfo, p->current));
1398    brw_inst_set_gen6_jump_count(devinfo, insn, 0);
1399    brw_set_src0(p, insn, src0);
1400    brw_set_src1(p, insn, src1);
1401 
1402    assert(brw_inst_qtr_control(devinfo, insn) == BRW_COMPRESSION_NONE);
1403    assert(brw_inst_pred_control(devinfo, insn) == BRW_PREDICATE_NONE);
1404    brw_inst_set_cond_modifier(devinfo, insn, conditional);
1405 
1406    push_if_stack(p, insn);
1407    return insn;
1408 }
1409 
1410 /**
1411  * In single-program-flow (SPF) mode, convert IF and ELSE into ADDs.
1412  */
1413 static void
convert_IF_ELSE_to_ADD(struct brw_codegen * p,brw_inst * if_inst,brw_inst * else_inst)1414 convert_IF_ELSE_to_ADD(struct brw_codegen *p,
1415                        brw_inst *if_inst, brw_inst *else_inst)
1416 {
1417    const struct gen_device_info *devinfo = p->devinfo;
1418 
1419    /* The next instruction (where the ENDIF would be, if it existed) */
1420    brw_inst *next_inst = &p->store[p->nr_insn];
1421 
1422    assert(p->single_program_flow);
1423    assert(if_inst != NULL && brw_inst_opcode(devinfo, if_inst) == BRW_OPCODE_IF);
1424    assert(else_inst == NULL || brw_inst_opcode(devinfo, else_inst) == BRW_OPCODE_ELSE);
1425    assert(brw_inst_exec_size(devinfo, if_inst) == BRW_EXECUTE_1);
1426 
1427    /* Convert IF to an ADD instruction that moves the instruction pointer
1428     * to the first instruction of the ELSE block.  If there is no ELSE
1429     * block, point to where ENDIF would be.  Reverse the predicate.
1430     *
1431     * There's no need to execute an ENDIF since we don't need to do any
1432     * stack operations, and if we're currently executing, we just want to
1433     * continue normally.
1434     */
1435    brw_inst_set_opcode(devinfo, if_inst, BRW_OPCODE_ADD);
1436    brw_inst_set_pred_inv(devinfo, if_inst, true);
1437 
1438    if (else_inst != NULL) {
1439       /* Convert ELSE to an ADD instruction that points where the ENDIF
1440        * would be.
1441        */
1442       brw_inst_set_opcode(devinfo, else_inst, BRW_OPCODE_ADD);
1443 
1444       brw_inst_set_imm_ud(devinfo, if_inst, (else_inst - if_inst + 1) * 16);
1445       brw_inst_set_imm_ud(devinfo, else_inst, (next_inst - else_inst) * 16);
1446    } else {
1447       brw_inst_set_imm_ud(devinfo, if_inst, (next_inst - if_inst) * 16);
1448    }
1449 }
1450 
1451 /**
1452  * Patch IF and ELSE instructions with appropriate jump targets.
1453  */
1454 static void
patch_IF_ELSE(struct brw_codegen * p,brw_inst * if_inst,brw_inst * else_inst,brw_inst * endif_inst)1455 patch_IF_ELSE(struct brw_codegen *p,
1456               brw_inst *if_inst, brw_inst *else_inst, brw_inst *endif_inst)
1457 {
1458    const struct gen_device_info *devinfo = p->devinfo;
1459 
1460    /* We shouldn't be patching IF and ELSE instructions in single program flow
1461     * mode when gen < 6, because in single program flow mode on those
1462     * platforms, we convert flow control instructions to conditional ADDs that
1463     * operate on IP (see brw_ENDIF).
1464     *
1465     * However, on Gen6, writing to IP doesn't work in single program flow mode
1466     * (see the SandyBridge PRM, Volume 4 part 2, p79: "When SPF is ON, IP may
1467     * not be updated by non-flow control instructions.").  And on later
1468     * platforms, there is no significant benefit to converting control flow
1469     * instructions to conditional ADDs.  So we do patch IF and ELSE
1470     * instructions in single program flow mode on those platforms.
1471     */
1472    if (devinfo->gen < 6)
1473       assert(!p->single_program_flow);
1474 
1475    assert(if_inst != NULL && brw_inst_opcode(devinfo, if_inst) == BRW_OPCODE_IF);
1476    assert(endif_inst != NULL);
1477    assert(else_inst == NULL || brw_inst_opcode(devinfo, else_inst) == BRW_OPCODE_ELSE);
1478 
1479    unsigned br = brw_jump_scale(devinfo);
1480 
1481    assert(brw_inst_opcode(devinfo, endif_inst) == BRW_OPCODE_ENDIF);
1482    brw_inst_set_exec_size(devinfo, endif_inst, brw_inst_exec_size(devinfo, if_inst));
1483 
1484    if (else_inst == NULL) {
1485       /* Patch IF -> ENDIF */
1486       if (devinfo->gen < 6) {
1487 	 /* Turn it into an IFF, which means no mask stack operations for
1488 	  * all-false and jumping past the ENDIF.
1489 	  */
1490          brw_inst_set_opcode(devinfo, if_inst, BRW_OPCODE_IFF);
1491          brw_inst_set_gen4_jump_count(devinfo, if_inst,
1492                                       br * (endif_inst - if_inst + 1));
1493          brw_inst_set_gen4_pop_count(devinfo, if_inst, 0);
1494       } else if (devinfo->gen == 6) {
1495 	 /* As of gen6, there is no IFF and IF must point to the ENDIF. */
1496          brw_inst_set_gen6_jump_count(devinfo, if_inst, br*(endif_inst - if_inst));
1497       } else {
1498          brw_inst_set_uip(devinfo, if_inst, br * (endif_inst - if_inst));
1499          brw_inst_set_jip(devinfo, if_inst, br * (endif_inst - if_inst));
1500       }
1501    } else {
1502       brw_inst_set_exec_size(devinfo, else_inst, brw_inst_exec_size(devinfo, if_inst));
1503 
1504       /* Patch IF -> ELSE */
1505       if (devinfo->gen < 6) {
1506          brw_inst_set_gen4_jump_count(devinfo, if_inst,
1507                                       br * (else_inst - if_inst));
1508          brw_inst_set_gen4_pop_count(devinfo, if_inst, 0);
1509       } else if (devinfo->gen == 6) {
1510          brw_inst_set_gen6_jump_count(devinfo, if_inst,
1511                                       br * (else_inst - if_inst + 1));
1512       }
1513 
1514       /* Patch ELSE -> ENDIF */
1515       if (devinfo->gen < 6) {
1516 	 /* BRW_OPCODE_ELSE pre-gen6 should point just past the
1517 	  * matching ENDIF.
1518 	  */
1519          brw_inst_set_gen4_jump_count(devinfo, else_inst,
1520                                       br * (endif_inst - else_inst + 1));
1521          brw_inst_set_gen4_pop_count(devinfo, else_inst, 1);
1522       } else if (devinfo->gen == 6) {
1523 	 /* BRW_OPCODE_ELSE on gen6 should point to the matching ENDIF. */
1524          brw_inst_set_gen6_jump_count(devinfo, else_inst,
1525                                       br * (endif_inst - else_inst));
1526       } else {
1527 	 /* The IF instruction's JIP should point just past the ELSE */
1528          brw_inst_set_jip(devinfo, if_inst, br * (else_inst - if_inst + 1));
1529 	 /* The IF instruction's UIP and ELSE's JIP should point to ENDIF */
1530          brw_inst_set_uip(devinfo, if_inst, br * (endif_inst - if_inst));
1531          brw_inst_set_jip(devinfo, else_inst, br * (endif_inst - else_inst));
1532          if (devinfo->gen >= 8) {
1533             /* Since we don't set branch_ctrl, the ELSE's JIP and UIP both
1534              * should point to ENDIF.
1535              */
1536             brw_inst_set_uip(devinfo, else_inst, br * (endif_inst - else_inst));
1537          }
1538       }
1539    }
1540 }
1541 
1542 void
brw_ELSE(struct brw_codegen * p)1543 brw_ELSE(struct brw_codegen *p)
1544 {
1545    const struct gen_device_info *devinfo = p->devinfo;
1546    brw_inst *insn;
1547 
1548    insn = next_insn(p, BRW_OPCODE_ELSE);
1549 
1550    if (devinfo->gen < 6) {
1551       brw_set_dest(p, insn, brw_ip_reg());
1552       brw_set_src0(p, insn, brw_ip_reg());
1553       brw_set_src1(p, insn, brw_imm_d(0x0));
1554    } else if (devinfo->gen == 6) {
1555       brw_set_dest(p, insn, brw_imm_w(0));
1556       brw_inst_set_gen6_jump_count(devinfo, insn, 0);
1557       brw_set_src0(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1558       brw_set_src1(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1559    } else if (devinfo->gen == 7) {
1560       brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1561       brw_set_src0(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1562       brw_set_src1(p, insn, brw_imm_w(0));
1563       brw_inst_set_jip(devinfo, insn, 0);
1564       brw_inst_set_uip(devinfo, insn, 0);
1565    } else {
1566       brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1567       brw_set_src0(p, insn, brw_imm_d(0));
1568       brw_inst_set_jip(devinfo, insn, 0);
1569       brw_inst_set_uip(devinfo, insn, 0);
1570    }
1571 
1572    brw_inst_set_qtr_control(devinfo, insn, BRW_COMPRESSION_NONE);
1573    brw_inst_set_mask_control(devinfo, insn, BRW_MASK_ENABLE);
1574    if (!p->single_program_flow && devinfo->gen < 6)
1575       brw_inst_set_thread_control(devinfo, insn, BRW_THREAD_SWITCH);
1576 
1577    push_if_stack(p, insn);
1578 }
1579 
1580 void
brw_ENDIF(struct brw_codegen * p)1581 brw_ENDIF(struct brw_codegen *p)
1582 {
1583    const struct gen_device_info *devinfo = p->devinfo;
1584    brw_inst *insn = NULL;
1585    brw_inst *else_inst = NULL;
1586    brw_inst *if_inst = NULL;
1587    brw_inst *tmp;
1588    bool emit_endif = true;
1589 
1590    /* In single program flow mode, we can express IF and ELSE instructions
1591     * equivalently as ADD instructions that operate on IP.  On platforms prior
1592     * to Gen6, flow control instructions cause an implied thread switch, so
1593     * this is a significant savings.
1594     *
1595     * However, on Gen6, writing to IP doesn't work in single program flow mode
1596     * (see the SandyBridge PRM, Volume 4 part 2, p79: "When SPF is ON, IP may
1597     * not be updated by non-flow control instructions.").  And on later
1598     * platforms, there is no significant benefit to converting control flow
1599     * instructions to conditional ADDs.  So we only do this trick on Gen4 and
1600     * Gen5.
1601     */
1602    if (devinfo->gen < 6 && p->single_program_flow)
1603       emit_endif = false;
1604 
1605    /*
1606     * A single next_insn() may change the base address of instruction store
1607     * memory(p->store), so call it first before referencing the instruction
1608     * store pointer from an index
1609     */
1610    if (emit_endif)
1611       insn = next_insn(p, BRW_OPCODE_ENDIF);
1612 
1613    /* Pop the IF and (optional) ELSE instructions from the stack */
1614    p->if_depth_in_loop[p->loop_stack_depth]--;
1615    tmp = pop_if_stack(p);
1616    if (brw_inst_opcode(devinfo, tmp) == BRW_OPCODE_ELSE) {
1617       else_inst = tmp;
1618       tmp = pop_if_stack(p);
1619    }
1620    if_inst = tmp;
1621 
1622    if (!emit_endif) {
1623       /* ENDIF is useless; don't bother emitting it. */
1624       convert_IF_ELSE_to_ADD(p, if_inst, else_inst);
1625       return;
1626    }
1627 
1628    if (devinfo->gen < 6) {
1629       brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1630       brw_set_src0(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1631       brw_set_src1(p, insn, brw_imm_d(0x0));
1632    } else if (devinfo->gen == 6) {
1633       brw_set_dest(p, insn, brw_imm_w(0));
1634       brw_set_src0(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1635       brw_set_src1(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1636    } else if (devinfo->gen == 7) {
1637       brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1638       brw_set_src0(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1639       brw_set_src1(p, insn, brw_imm_w(0));
1640    } else {
1641       brw_set_src0(p, insn, brw_imm_d(0));
1642    }
1643 
1644    brw_inst_set_qtr_control(devinfo, insn, BRW_COMPRESSION_NONE);
1645    brw_inst_set_mask_control(devinfo, insn, BRW_MASK_ENABLE);
1646    if (devinfo->gen < 6)
1647       brw_inst_set_thread_control(devinfo, insn, BRW_THREAD_SWITCH);
1648 
1649    /* Also pop item off the stack in the endif instruction: */
1650    if (devinfo->gen < 6) {
1651       brw_inst_set_gen4_jump_count(devinfo, insn, 0);
1652       brw_inst_set_gen4_pop_count(devinfo, insn, 1);
1653    } else if (devinfo->gen == 6) {
1654       brw_inst_set_gen6_jump_count(devinfo, insn, 2);
1655    } else {
1656       brw_inst_set_jip(devinfo, insn, 2);
1657    }
1658    patch_IF_ELSE(p, if_inst, else_inst, insn);
1659 }
1660 
1661 brw_inst *
brw_BREAK(struct brw_codegen * p)1662 brw_BREAK(struct brw_codegen *p)
1663 {
1664    const struct gen_device_info *devinfo = p->devinfo;
1665    brw_inst *insn;
1666 
1667    insn = next_insn(p, BRW_OPCODE_BREAK);
1668    if (devinfo->gen >= 8) {
1669       brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1670       brw_set_src0(p, insn, brw_imm_d(0x0));
1671    } else if (devinfo->gen >= 6) {
1672       brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1673       brw_set_src0(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1674       brw_set_src1(p, insn, brw_imm_d(0x0));
1675    } else {
1676       brw_set_dest(p, insn, brw_ip_reg());
1677       brw_set_src0(p, insn, brw_ip_reg());
1678       brw_set_src1(p, insn, brw_imm_d(0x0));
1679       brw_inst_set_gen4_pop_count(devinfo, insn,
1680                                   p->if_depth_in_loop[p->loop_stack_depth]);
1681    }
1682    brw_inst_set_qtr_control(devinfo, insn, BRW_COMPRESSION_NONE);
1683    brw_inst_set_exec_size(devinfo, insn,
1684                           brw_inst_exec_size(devinfo, p->current));
1685 
1686    return insn;
1687 }
1688 
1689 brw_inst *
brw_CONT(struct brw_codegen * p)1690 brw_CONT(struct brw_codegen *p)
1691 {
1692    const struct gen_device_info *devinfo = p->devinfo;
1693    brw_inst *insn;
1694 
1695    insn = next_insn(p, BRW_OPCODE_CONTINUE);
1696    brw_set_dest(p, insn, brw_ip_reg());
1697    if (devinfo->gen >= 8) {
1698       brw_set_src0(p, insn, brw_imm_d(0x0));
1699    } else {
1700       brw_set_src0(p, insn, brw_ip_reg());
1701       brw_set_src1(p, insn, brw_imm_d(0x0));
1702    }
1703 
1704    if (devinfo->gen < 6) {
1705       brw_inst_set_gen4_pop_count(devinfo, insn,
1706                                   p->if_depth_in_loop[p->loop_stack_depth]);
1707    }
1708    brw_inst_set_qtr_control(devinfo, insn, BRW_COMPRESSION_NONE);
1709    brw_inst_set_exec_size(devinfo, insn,
1710                           brw_inst_exec_size(devinfo, p->current));
1711    return insn;
1712 }
1713 
1714 brw_inst *
gen6_HALT(struct brw_codegen * p)1715 gen6_HALT(struct brw_codegen *p)
1716 {
1717    const struct gen_device_info *devinfo = p->devinfo;
1718    brw_inst *insn;
1719 
1720    insn = next_insn(p, BRW_OPCODE_HALT);
1721    brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1722    if (devinfo->gen >= 8) {
1723       brw_set_src0(p, insn, brw_imm_d(0x0));
1724    } else {
1725       brw_set_src0(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1726       brw_set_src1(p, insn, brw_imm_d(0x0)); /* UIP and JIP, updated later. */
1727    }
1728 
1729    brw_inst_set_qtr_control(devinfo, insn, BRW_COMPRESSION_NONE);
1730    brw_inst_set_exec_size(devinfo, insn,
1731                           brw_inst_exec_size(devinfo, p->current));
1732    return insn;
1733 }
1734 
1735 /* DO/WHILE loop:
1736  *
1737  * The DO/WHILE is just an unterminated loop -- break or continue are
1738  * used for control within the loop.  We have a few ways they can be
1739  * done.
1740  *
1741  * For uniform control flow, the WHILE is just a jump, so ADD ip, ip,
1742  * jip and no DO instruction.
1743  *
1744  * For non-uniform control flow pre-gen6, there's a DO instruction to
1745  * push the mask, and a WHILE to jump back, and BREAK to get out and
1746  * pop the mask.
1747  *
1748  * For gen6, there's no more mask stack, so no need for DO.  WHILE
1749  * just points back to the first instruction of the loop.
1750  */
1751 brw_inst *
brw_DO(struct brw_codegen * p,unsigned execute_size)1752 brw_DO(struct brw_codegen *p, unsigned execute_size)
1753 {
1754    const struct gen_device_info *devinfo = p->devinfo;
1755 
1756    if (devinfo->gen >= 6 || p->single_program_flow) {
1757       push_loop_stack(p, &p->store[p->nr_insn]);
1758       return &p->store[p->nr_insn];
1759    } else {
1760       brw_inst *insn = next_insn(p, BRW_OPCODE_DO);
1761 
1762       push_loop_stack(p, insn);
1763 
1764       /* Override the defaults for this instruction:
1765        */
1766       brw_set_dest(p, insn, brw_null_reg());
1767       brw_set_src0(p, insn, brw_null_reg());
1768       brw_set_src1(p, insn, brw_null_reg());
1769 
1770       brw_inst_set_qtr_control(devinfo, insn, BRW_COMPRESSION_NONE);
1771       brw_inst_set_exec_size(devinfo, insn, execute_size);
1772       brw_inst_set_pred_control(devinfo, insn, BRW_PREDICATE_NONE);
1773 
1774       return insn;
1775    }
1776 }
1777 
1778 /**
1779  * For pre-gen6, we patch BREAK/CONT instructions to point at the WHILE
1780  * instruction here.
1781  *
1782  * For gen6+, see brw_set_uip_jip(), which doesn't care so much about the loop
1783  * nesting, since it can always just point to the end of the block/current loop.
1784  */
1785 static void
brw_patch_break_cont(struct brw_codegen * p,brw_inst * while_inst)1786 brw_patch_break_cont(struct brw_codegen *p, brw_inst *while_inst)
1787 {
1788    const struct gen_device_info *devinfo = p->devinfo;
1789    brw_inst *do_inst = get_inner_do_insn(p);
1790    brw_inst *inst;
1791    unsigned br = brw_jump_scale(devinfo);
1792 
1793    assert(devinfo->gen < 6);
1794 
1795    for (inst = while_inst - 1; inst != do_inst; inst--) {
1796       /* If the jump count is != 0, that means that this instruction has already
1797        * been patched because it's part of a loop inside of the one we're
1798        * patching.
1799        */
1800       if (brw_inst_opcode(devinfo, inst) == BRW_OPCODE_BREAK &&
1801           brw_inst_gen4_jump_count(devinfo, inst) == 0) {
1802          brw_inst_set_gen4_jump_count(devinfo, inst, br*((while_inst - inst) + 1));
1803       } else if (brw_inst_opcode(devinfo, inst) == BRW_OPCODE_CONTINUE &&
1804                  brw_inst_gen4_jump_count(devinfo, inst) == 0) {
1805          brw_inst_set_gen4_jump_count(devinfo, inst, br * (while_inst - inst));
1806       }
1807    }
1808 }
1809 
1810 brw_inst *
brw_WHILE(struct brw_codegen * p)1811 brw_WHILE(struct brw_codegen *p)
1812 {
1813    const struct gen_device_info *devinfo = p->devinfo;
1814    brw_inst *insn, *do_insn;
1815    unsigned br = brw_jump_scale(devinfo);
1816 
1817    if (devinfo->gen >= 6) {
1818       insn = next_insn(p, BRW_OPCODE_WHILE);
1819       do_insn = get_inner_do_insn(p);
1820 
1821       if (devinfo->gen >= 8) {
1822          brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1823          brw_set_src0(p, insn, brw_imm_d(0));
1824          brw_inst_set_jip(devinfo, insn, br * (do_insn - insn));
1825       } else if (devinfo->gen == 7) {
1826          brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1827          brw_set_src0(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1828          brw_set_src1(p, insn, brw_imm_w(0));
1829          brw_inst_set_jip(devinfo, insn, br * (do_insn - insn));
1830       } else {
1831          brw_set_dest(p, insn, brw_imm_w(0));
1832          brw_inst_set_gen6_jump_count(devinfo, insn, br * (do_insn - insn));
1833          brw_set_src0(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1834          brw_set_src1(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
1835       }
1836 
1837       brw_inst_set_exec_size(devinfo, insn,
1838                              brw_inst_exec_size(devinfo, p->current));
1839 
1840    } else {
1841       if (p->single_program_flow) {
1842 	 insn = next_insn(p, BRW_OPCODE_ADD);
1843          do_insn = get_inner_do_insn(p);
1844 
1845 	 brw_set_dest(p, insn, brw_ip_reg());
1846 	 brw_set_src0(p, insn, brw_ip_reg());
1847 	 brw_set_src1(p, insn, brw_imm_d((do_insn - insn) * 16));
1848          brw_inst_set_exec_size(devinfo, insn, BRW_EXECUTE_1);
1849       } else {
1850 	 insn = next_insn(p, BRW_OPCODE_WHILE);
1851          do_insn = get_inner_do_insn(p);
1852 
1853          assert(brw_inst_opcode(devinfo, do_insn) == BRW_OPCODE_DO);
1854 
1855 	 brw_set_dest(p, insn, brw_ip_reg());
1856 	 brw_set_src0(p, insn, brw_ip_reg());
1857 	 brw_set_src1(p, insn, brw_imm_d(0));
1858 
1859          brw_inst_set_exec_size(devinfo, insn, brw_inst_exec_size(devinfo, do_insn));
1860          brw_inst_set_gen4_jump_count(devinfo, insn, br * (do_insn - insn + 1));
1861          brw_inst_set_gen4_pop_count(devinfo, insn, 0);
1862 
1863 	 brw_patch_break_cont(p, insn);
1864       }
1865    }
1866    brw_inst_set_qtr_control(devinfo, insn, BRW_COMPRESSION_NONE);
1867 
1868    p->loop_stack_depth--;
1869 
1870    return insn;
1871 }
1872 
1873 /* FORWARD JUMPS:
1874  */
brw_land_fwd_jump(struct brw_codegen * p,int jmp_insn_idx)1875 void brw_land_fwd_jump(struct brw_codegen *p, int jmp_insn_idx)
1876 {
1877    const struct gen_device_info *devinfo = p->devinfo;
1878    brw_inst *jmp_insn = &p->store[jmp_insn_idx];
1879    unsigned jmpi = 1;
1880 
1881    if (devinfo->gen >= 5)
1882       jmpi = 2;
1883 
1884    assert(brw_inst_opcode(devinfo, jmp_insn) == BRW_OPCODE_JMPI);
1885    assert(brw_inst_src1_reg_file(devinfo, jmp_insn) == BRW_IMMEDIATE_VALUE);
1886 
1887    brw_inst_set_gen4_jump_count(devinfo, jmp_insn,
1888                                 jmpi * (p->nr_insn - jmp_insn_idx - 1));
1889 }
1890 
1891 /* To integrate with the above, it makes sense that the comparison
1892  * instruction should populate the flag register.  It might be simpler
1893  * just to use the flag reg for most WM tasks?
1894  */
brw_CMP(struct brw_codegen * p,struct brw_reg dest,unsigned conditional,struct brw_reg src0,struct brw_reg src1)1895 void brw_CMP(struct brw_codegen *p,
1896 	     struct brw_reg dest,
1897 	     unsigned conditional,
1898 	     struct brw_reg src0,
1899 	     struct brw_reg src1)
1900 {
1901    const struct gen_device_info *devinfo = p->devinfo;
1902    brw_inst *insn = next_insn(p, BRW_OPCODE_CMP);
1903 
1904    brw_inst_set_cond_modifier(devinfo, insn, conditional);
1905    brw_set_dest(p, insn, dest);
1906    brw_set_src0(p, insn, src0);
1907    brw_set_src1(p, insn, src1);
1908 
1909    /* Item WaCMPInstNullDstForcesThreadSwitch in the Haswell Bspec workarounds
1910     * page says:
1911     *    "Any CMP instruction with a null destination must use a {switch}."
1912     *
1913     * It also applies to other Gen7 platforms (IVB, BYT) even though it isn't
1914     * mentioned on their work-arounds pages.
1915     */
1916    if (devinfo->gen == 7) {
1917       if (dest.file == BRW_ARCHITECTURE_REGISTER_FILE &&
1918           dest.nr == BRW_ARF_NULL) {
1919          brw_inst_set_thread_control(devinfo, insn, BRW_THREAD_SWITCH);
1920       }
1921    }
1922 }
1923 
1924 /***********************************************************************
1925  * Helpers for the various SEND message types:
1926  */
1927 
1928 /** Extended math function, float[8].
1929  */
gen4_math(struct brw_codegen * p,struct brw_reg dest,unsigned function,unsigned msg_reg_nr,struct brw_reg src,unsigned precision)1930 void gen4_math(struct brw_codegen *p,
1931 	       struct brw_reg dest,
1932 	       unsigned function,
1933 	       unsigned msg_reg_nr,
1934 	       struct brw_reg src,
1935 	       unsigned precision )
1936 {
1937    const struct gen_device_info *devinfo = p->devinfo;
1938    brw_inst *insn = next_insn(p, BRW_OPCODE_SEND);
1939    unsigned data_type;
1940    if (has_scalar_region(src)) {
1941       data_type = BRW_MATH_DATA_SCALAR;
1942    } else {
1943       data_type = BRW_MATH_DATA_VECTOR;
1944    }
1945 
1946    assert(devinfo->gen < 6);
1947 
1948    /* Example code doesn't set predicate_control for send
1949     * instructions.
1950     */
1951    brw_inst_set_pred_control(devinfo, insn, 0);
1952    brw_inst_set_base_mrf(devinfo, insn, msg_reg_nr);
1953 
1954    brw_set_dest(p, insn, dest);
1955    brw_set_src0(p, insn, src);
1956    brw_set_math_message(p,
1957                         insn,
1958                         function,
1959                         src.type == BRW_REGISTER_TYPE_D,
1960                         precision,
1961                         data_type);
1962 }
1963 
gen6_math(struct brw_codegen * p,struct brw_reg dest,unsigned function,struct brw_reg src0,struct brw_reg src1)1964 void gen6_math(struct brw_codegen *p,
1965 	       struct brw_reg dest,
1966 	       unsigned function,
1967 	       struct brw_reg src0,
1968 	       struct brw_reg src1)
1969 {
1970    const struct gen_device_info *devinfo = p->devinfo;
1971    brw_inst *insn = next_insn(p, BRW_OPCODE_MATH);
1972 
1973    assert(devinfo->gen >= 6);
1974 
1975    assert(dest.file == BRW_GENERAL_REGISTER_FILE ||
1976           (devinfo->gen >= 7 && dest.file == BRW_MESSAGE_REGISTER_FILE));
1977    assert(src0.file == BRW_GENERAL_REGISTER_FILE ||
1978           (devinfo->gen >= 8 && src0.file == BRW_IMMEDIATE_VALUE));
1979 
1980    assert(dest.hstride == BRW_HORIZONTAL_STRIDE_1);
1981    if (devinfo->gen == 6) {
1982       assert(src0.hstride == BRW_HORIZONTAL_STRIDE_1);
1983       assert(src1.hstride == BRW_HORIZONTAL_STRIDE_1);
1984    }
1985 
1986    if (function == BRW_MATH_FUNCTION_INT_DIV_QUOTIENT ||
1987        function == BRW_MATH_FUNCTION_INT_DIV_REMAINDER ||
1988        function == BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER) {
1989       assert(src0.type != BRW_REGISTER_TYPE_F);
1990       assert(src1.type != BRW_REGISTER_TYPE_F);
1991       assert(src1.file == BRW_GENERAL_REGISTER_FILE ||
1992              (devinfo->gen >= 8 && src1.file == BRW_IMMEDIATE_VALUE));
1993    } else {
1994       assert(src0.type == BRW_REGISTER_TYPE_F);
1995       assert(src1.type == BRW_REGISTER_TYPE_F);
1996       if (function == BRW_MATH_FUNCTION_POW) {
1997          assert(src1.file == BRW_GENERAL_REGISTER_FILE ||
1998                 (devinfo->gen >= 8 && src1.file == BRW_IMMEDIATE_VALUE));
1999       } else {
2000          assert(src1.file == BRW_ARCHITECTURE_REGISTER_FILE &&
2001                 src1.nr == BRW_ARF_NULL);
2002       }
2003    }
2004 
2005    /* Source modifiers are ignored for extended math instructions on Gen6. */
2006    if (devinfo->gen == 6) {
2007       assert(!src0.negate);
2008       assert(!src0.abs);
2009       assert(!src1.negate);
2010       assert(!src1.abs);
2011    }
2012 
2013    brw_inst_set_math_function(devinfo, insn, function);
2014 
2015    brw_set_dest(p, insn, dest);
2016    brw_set_src0(p, insn, src0);
2017    brw_set_src1(p, insn, src1);
2018 }
2019 
2020 /**
2021  * Return the right surface index to access the thread scratch space using
2022  * stateless dataport messages.
2023  */
2024 unsigned
brw_scratch_surface_idx(const struct brw_codegen * p)2025 brw_scratch_surface_idx(const struct brw_codegen *p)
2026 {
2027    /* The scratch space is thread-local so IA coherency is unnecessary. */
2028    if (p->devinfo->gen >= 8)
2029       return GEN8_BTI_STATELESS_NON_COHERENT;
2030    else
2031       return BRW_BTI_STATELESS;
2032 }
2033 
2034 /**
2035  * Write a block of OWORDs (half a GRF each) from the scratch buffer,
2036  * using a constant offset per channel.
2037  *
2038  * The offset must be aligned to oword size (16 bytes).  Used for
2039  * register spilling.
2040  */
brw_oword_block_write_scratch(struct brw_codegen * p,struct brw_reg mrf,int num_regs,unsigned offset)2041 void brw_oword_block_write_scratch(struct brw_codegen *p,
2042 				   struct brw_reg mrf,
2043 				   int num_regs,
2044 				   unsigned offset)
2045 {
2046    const struct gen_device_info *devinfo = p->devinfo;
2047    const unsigned target_cache =
2048       (devinfo->gen >= 7 ? GEN7_SFID_DATAPORT_DATA_CACHE :
2049        devinfo->gen >= 6 ? GEN6_SFID_DATAPORT_RENDER_CACHE :
2050        BRW_DATAPORT_READ_TARGET_RENDER_CACHE);
2051    uint32_t msg_type;
2052 
2053    if (devinfo->gen >= 6)
2054       offset /= 16;
2055 
2056    mrf = retype(mrf, BRW_REGISTER_TYPE_UD);
2057 
2058    const unsigned mlen = 1 + num_regs;
2059 
2060    /* Set up the message header.  This is g0, with g0.2 filled with
2061     * the offset.  We don't want to leave our offset around in g0 or
2062     * it'll screw up texture samples, so set it up inside the message
2063     * reg.
2064     */
2065    {
2066       brw_push_insn_state(p);
2067       brw_set_default_exec_size(p, BRW_EXECUTE_8);
2068       brw_set_default_mask_control(p, BRW_MASK_DISABLE);
2069       brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
2070 
2071       brw_MOV(p, mrf, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD));
2072 
2073       /* set message header global offset field (reg 0, element 2) */
2074       brw_MOV(p,
2075 	      retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE,
2076 				  mrf.nr,
2077 				  2), BRW_REGISTER_TYPE_UD),
2078 	      brw_imm_ud(offset));
2079 
2080       brw_pop_insn_state(p);
2081    }
2082 
2083    {
2084       struct brw_reg dest;
2085       brw_inst *insn = next_insn(p, BRW_OPCODE_SEND);
2086       int send_commit_msg;
2087       struct brw_reg src_header = retype(brw_vec8_grf(0, 0),
2088 					 BRW_REGISTER_TYPE_UW);
2089 
2090       brw_inst_set_compression(devinfo, insn, false);
2091 
2092       if (brw_inst_exec_size(devinfo, insn) >= 16)
2093 	 src_header = vec16(src_header);
2094 
2095       assert(brw_inst_pred_control(devinfo, insn) == BRW_PREDICATE_NONE);
2096       if (devinfo->gen < 6)
2097          brw_inst_set_base_mrf(devinfo, insn, mrf.nr);
2098 
2099       /* Until gen6, writes followed by reads from the same location
2100        * are not guaranteed to be ordered unless write_commit is set.
2101        * If set, then a no-op write is issued to the destination
2102        * register to set a dependency, and a read from the destination
2103        * can be used to ensure the ordering.
2104        *
2105        * For gen6, only writes between different threads need ordering
2106        * protection.  Our use of DP writes is all about register
2107        * spilling within a thread.
2108        */
2109       if (devinfo->gen >= 6) {
2110 	 dest = retype(vec16(brw_null_reg()), BRW_REGISTER_TYPE_UW);
2111 	 send_commit_msg = 0;
2112       } else {
2113 	 dest = src_header;
2114 	 send_commit_msg = 1;
2115       }
2116 
2117       brw_set_dest(p, insn, dest);
2118       if (devinfo->gen >= 6) {
2119 	 brw_set_src0(p, insn, mrf);
2120       } else {
2121 	 brw_set_src0(p, insn, brw_null_reg());
2122       }
2123 
2124       if (devinfo->gen >= 6)
2125 	 msg_type = GEN6_DATAPORT_WRITE_MESSAGE_OWORD_BLOCK_WRITE;
2126       else
2127 	 msg_type = BRW_DATAPORT_WRITE_MESSAGE_OWORD_BLOCK_WRITE;
2128 
2129       brw_set_dp_write_message(p,
2130 			       insn,
2131                                brw_scratch_surface_idx(p),
2132 			       BRW_DATAPORT_OWORD_BLOCK_DWORDS(num_regs * 8),
2133 			       msg_type,
2134                                target_cache,
2135 			       mlen,
2136 			       true, /* header_present */
2137 			       0, /* not a render target */
2138 			       send_commit_msg, /* response_length */
2139 			       0, /* eot */
2140 			       send_commit_msg);
2141    }
2142 }
2143 
2144 
2145 /**
2146  * Read a block of owords (half a GRF each) from the scratch buffer
2147  * using a constant index per channel.
2148  *
2149  * Offset must be aligned to oword size (16 bytes).  Used for register
2150  * spilling.
2151  */
2152 void
brw_oword_block_read_scratch(struct brw_codegen * p,struct brw_reg dest,struct brw_reg mrf,int num_regs,unsigned offset)2153 brw_oword_block_read_scratch(struct brw_codegen *p,
2154 			     struct brw_reg dest,
2155 			     struct brw_reg mrf,
2156 			     int num_regs,
2157 			     unsigned offset)
2158 {
2159    const struct gen_device_info *devinfo = p->devinfo;
2160 
2161    if (devinfo->gen >= 6)
2162       offset /= 16;
2163 
2164    if (p->devinfo->gen >= 7) {
2165       /* On gen 7 and above, we no longer have message registers and we can
2166        * send from any register we want.  By using the destination register
2167        * for the message, we guarantee that the implied message write won't
2168        * accidentally overwrite anything.  This has been a problem because
2169        * the MRF registers and source for the final FB write are both fixed
2170        * and may overlap.
2171        */
2172       mrf = retype(dest, BRW_REGISTER_TYPE_UD);
2173    } else {
2174       mrf = retype(mrf, BRW_REGISTER_TYPE_UD);
2175    }
2176    dest = retype(dest, BRW_REGISTER_TYPE_UW);
2177 
2178    const unsigned rlen = num_regs;
2179    const unsigned target_cache =
2180       (devinfo->gen >= 7 ? GEN7_SFID_DATAPORT_DATA_CACHE :
2181        devinfo->gen >= 6 ? GEN6_SFID_DATAPORT_RENDER_CACHE :
2182        BRW_DATAPORT_READ_TARGET_RENDER_CACHE);
2183 
2184    {
2185       brw_push_insn_state(p);
2186       brw_set_default_exec_size(p, BRW_EXECUTE_8);
2187       brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
2188       brw_set_default_mask_control(p, BRW_MASK_DISABLE);
2189 
2190       brw_MOV(p, mrf, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD));
2191 
2192       /* set message header global offset field (reg 0, element 2) */
2193       brw_MOV(p, get_element_ud(mrf, 2), brw_imm_ud(offset));
2194 
2195       brw_pop_insn_state(p);
2196    }
2197 
2198    {
2199       brw_inst *insn = next_insn(p, BRW_OPCODE_SEND);
2200 
2201       assert(brw_inst_pred_control(devinfo, insn) == 0);
2202       brw_inst_set_compression(devinfo, insn, false);
2203 
2204       brw_set_dest(p, insn, dest);	/* UW? */
2205       if (devinfo->gen >= 6) {
2206 	 brw_set_src0(p, insn, mrf);
2207       } else {
2208 	 brw_set_src0(p, insn, brw_null_reg());
2209          brw_inst_set_base_mrf(devinfo, insn, mrf.nr);
2210       }
2211 
2212       brw_set_dp_read_message(p,
2213 			      insn,
2214                               brw_scratch_surface_idx(p),
2215 			      BRW_DATAPORT_OWORD_BLOCK_DWORDS(num_regs * 8),
2216 			      BRW_DATAPORT_READ_MESSAGE_OWORD_BLOCK_READ, /* msg_type */
2217 			      target_cache,
2218 			      1, /* msg_length */
2219                               true, /* header_present */
2220 			      rlen);
2221    }
2222 }
2223 
2224 void
gen7_block_read_scratch(struct brw_codegen * p,struct brw_reg dest,int num_regs,unsigned offset)2225 gen7_block_read_scratch(struct brw_codegen *p,
2226                         struct brw_reg dest,
2227                         int num_regs,
2228                         unsigned offset)
2229 {
2230    brw_inst *insn = next_insn(p, BRW_OPCODE_SEND);
2231    assert(brw_inst_pred_control(p->devinfo, insn) == BRW_PREDICATE_NONE);
2232 
2233    brw_set_dest(p, insn, retype(dest, BRW_REGISTER_TYPE_UW));
2234 
2235    /* The HW requires that the header is present; this is to get the g0.5
2236     * scratch offset.
2237     */
2238    brw_set_src0(p, insn, brw_vec8_grf(0, 0));
2239 
2240    /* According to the docs, offset is "A 12-bit HWord offset into the memory
2241     * Immediate Memory buffer as specified by binding table 0xFF."  An HWORD
2242     * is 32 bytes, which happens to be the size of a register.
2243     */
2244    offset /= REG_SIZE;
2245    assert(offset < (1 << 12));
2246 
2247    gen7_set_dp_scratch_message(p, insn,
2248                                false, /* scratch read */
2249                                false, /* OWords */
2250                                false, /* invalidate after read */
2251                                num_regs,
2252                                offset,
2253                                1,        /* mlen: just g0 */
2254                                num_regs, /* rlen */
2255                                true);    /* header present */
2256 }
2257 
2258 /**
2259  * Read float[4] vectors from the data port constant cache.
2260  * Location (in buffer) should be a multiple of 16.
2261  * Used for fetching shader constants.
2262  */
brw_oword_block_read(struct brw_codegen * p,struct brw_reg dest,struct brw_reg mrf,uint32_t offset,uint32_t bind_table_index)2263 void brw_oword_block_read(struct brw_codegen *p,
2264 			  struct brw_reg dest,
2265 			  struct brw_reg mrf,
2266 			  uint32_t offset,
2267 			  uint32_t bind_table_index)
2268 {
2269    const struct gen_device_info *devinfo = p->devinfo;
2270    const unsigned target_cache =
2271       (devinfo->gen >= 6 ? GEN6_SFID_DATAPORT_CONSTANT_CACHE :
2272        BRW_DATAPORT_READ_TARGET_DATA_CACHE);
2273    const unsigned exec_size = 1 << brw_inst_exec_size(devinfo, p->current);
2274 
2275    /* On newer hardware, offset is in units of owords. */
2276    if (devinfo->gen >= 6)
2277       offset /= 16;
2278 
2279    mrf = retype(mrf, BRW_REGISTER_TYPE_UD);
2280 
2281    brw_push_insn_state(p);
2282    brw_set_default_predicate_control(p, BRW_PREDICATE_NONE);
2283    brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
2284    brw_set_default_mask_control(p, BRW_MASK_DISABLE);
2285 
2286    brw_push_insn_state(p);
2287    brw_set_default_exec_size(p, BRW_EXECUTE_8);
2288    brw_MOV(p, mrf, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD));
2289 
2290    /* set message header global offset field (reg 0, element 2) */
2291    brw_MOV(p,
2292 	   retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE,
2293 			       mrf.nr,
2294 			       2), BRW_REGISTER_TYPE_UD),
2295 	   brw_imm_ud(offset));
2296    brw_pop_insn_state(p);
2297 
2298    brw_inst *insn = next_insn(p, BRW_OPCODE_SEND);
2299 
2300    /* cast dest to a uword[8] vector */
2301    dest = retype(vec8(dest), BRW_REGISTER_TYPE_UW);
2302 
2303    brw_set_dest(p, insn, dest);
2304    if (devinfo->gen >= 6) {
2305       brw_set_src0(p, insn, mrf);
2306    } else {
2307       brw_set_src0(p, insn, brw_null_reg());
2308       brw_inst_set_base_mrf(devinfo, insn, mrf.nr);
2309    }
2310 
2311    brw_set_dp_read_message(p, insn, bind_table_index,
2312                            BRW_DATAPORT_OWORD_BLOCK_DWORDS(exec_size),
2313 			   BRW_DATAPORT_READ_MESSAGE_OWORD_BLOCK_READ,
2314 			   target_cache,
2315 			   1, /* msg_length */
2316                            true, /* header_present */
2317 			   DIV_ROUND_UP(exec_size, 8)); /* response_length */
2318 
2319    brw_pop_insn_state(p);
2320 }
2321 
2322 
brw_fb_WRITE(struct brw_codegen * p,struct brw_reg payload,struct brw_reg implied_header,unsigned msg_control,unsigned binding_table_index,unsigned msg_length,unsigned response_length,bool eot,bool last_render_target,bool header_present)2323 void brw_fb_WRITE(struct brw_codegen *p,
2324                   struct brw_reg payload,
2325                   struct brw_reg implied_header,
2326                   unsigned msg_control,
2327                   unsigned binding_table_index,
2328                   unsigned msg_length,
2329                   unsigned response_length,
2330                   bool eot,
2331                   bool last_render_target,
2332                   bool header_present)
2333 {
2334    const struct gen_device_info *devinfo = p->devinfo;
2335    const unsigned target_cache =
2336       (devinfo->gen >= 6 ? GEN6_SFID_DATAPORT_RENDER_CACHE :
2337        BRW_DATAPORT_READ_TARGET_RENDER_CACHE);
2338    brw_inst *insn;
2339    unsigned msg_type;
2340    struct brw_reg dest, src0;
2341 
2342    if (brw_inst_exec_size(devinfo, p->current) >= BRW_EXECUTE_16)
2343       dest = retype(vec16(brw_null_reg()), BRW_REGISTER_TYPE_UW);
2344    else
2345       dest = retype(vec8(brw_null_reg()), BRW_REGISTER_TYPE_UW);
2346 
2347    if (devinfo->gen >= 6) {
2348       insn = next_insn(p, BRW_OPCODE_SENDC);
2349    } else {
2350       insn = next_insn(p, BRW_OPCODE_SEND);
2351    }
2352    brw_inst_set_compression(devinfo, insn, false);
2353 
2354    if (devinfo->gen >= 6) {
2355       /* headerless version, just submit color payload */
2356       src0 = payload;
2357 
2358       msg_type = GEN6_DATAPORT_WRITE_MESSAGE_RENDER_TARGET_WRITE;
2359    } else {
2360       assert(payload.file == BRW_MESSAGE_REGISTER_FILE);
2361       brw_inst_set_base_mrf(devinfo, insn, payload.nr);
2362       src0 = implied_header;
2363 
2364       msg_type = BRW_DATAPORT_WRITE_MESSAGE_RENDER_TARGET_WRITE;
2365    }
2366 
2367    brw_set_dest(p, insn, dest);
2368    brw_set_src0(p, insn, src0);
2369    brw_set_dp_write_message(p,
2370 			    insn,
2371 			    binding_table_index,
2372 			    msg_control,
2373 			    msg_type,
2374                             target_cache,
2375 			    msg_length,
2376 			    header_present,
2377 			    last_render_target,
2378 			    response_length,
2379 			    eot,
2380 			    0 /* send_commit_msg */);
2381 }
2382 
2383 brw_inst *
gen9_fb_READ(struct brw_codegen * p,struct brw_reg dst,struct brw_reg payload,unsigned binding_table_index,unsigned msg_length,unsigned response_length,bool per_sample)2384 gen9_fb_READ(struct brw_codegen *p,
2385              struct brw_reg dst,
2386              struct brw_reg payload,
2387              unsigned binding_table_index,
2388              unsigned msg_length,
2389              unsigned response_length,
2390              bool per_sample)
2391 {
2392    const struct gen_device_info *devinfo = p->devinfo;
2393    assert(devinfo->gen >= 9);
2394    const unsigned msg_subtype =
2395       brw_inst_exec_size(devinfo, p->current) == BRW_EXECUTE_16 ? 0 : 1;
2396    brw_inst *insn = next_insn(p, BRW_OPCODE_SENDC);
2397 
2398    brw_set_dest(p, insn, dst);
2399    brw_set_src0(p, insn, payload);
2400    brw_set_dp_read_message(p, insn, binding_table_index,
2401                            per_sample << 5 | msg_subtype,
2402                            GEN9_DATAPORT_RC_RENDER_TARGET_READ,
2403                            GEN6_SFID_DATAPORT_RENDER_CACHE,
2404                            msg_length, true /* header_present */,
2405                            response_length);
2406    brw_inst_set_rt_slot_group(devinfo, insn,
2407                               brw_inst_qtr_control(devinfo, p->current) / 2);
2408 
2409    return insn;
2410 }
2411 
2412 /**
2413  * Texture sample instruction.
2414  * Note: the msg_type plus msg_length values determine exactly what kind
2415  * of sampling operation is performed.  See volume 4, page 161 of docs.
2416  */
brw_SAMPLE(struct brw_codegen * p,struct brw_reg dest,unsigned msg_reg_nr,struct brw_reg src0,unsigned binding_table_index,unsigned sampler,unsigned msg_type,unsigned response_length,unsigned msg_length,unsigned header_present,unsigned simd_mode,unsigned return_format)2417 void brw_SAMPLE(struct brw_codegen *p,
2418 		struct brw_reg dest,
2419 		unsigned msg_reg_nr,
2420 		struct brw_reg src0,
2421 		unsigned binding_table_index,
2422 		unsigned sampler,
2423 		unsigned msg_type,
2424 		unsigned response_length,
2425 		unsigned msg_length,
2426 		unsigned header_present,
2427 		unsigned simd_mode,
2428 		unsigned return_format)
2429 {
2430    const struct gen_device_info *devinfo = p->devinfo;
2431    brw_inst *insn;
2432 
2433    if (msg_reg_nr != -1)
2434       gen6_resolve_implied_move(p, &src0, msg_reg_nr);
2435 
2436    insn = next_insn(p, BRW_OPCODE_SEND);
2437    brw_inst_set_pred_control(devinfo, insn, BRW_PREDICATE_NONE); /* XXX */
2438 
2439    /* From the 965 PRM (volume 4, part 1, section 14.2.41):
2440     *
2441     *    "Instruction compression is not allowed for this instruction (that
2442     *     is, send). The hardware behavior is undefined if this instruction is
2443     *     set as compressed. However, compress control can be set to "SecHalf"
2444     *     to affect the EMask generation."
2445     *
2446     * No similar wording is found in later PRMs, but there are examples
2447     * utilizing send with SecHalf.  More importantly, SIMD8 sampler messages
2448     * are allowed in SIMD16 mode and they could not work without SecHalf.  For
2449     * these reasons, we allow BRW_COMPRESSION_2NDHALF here.
2450     */
2451    brw_inst_set_compression(devinfo, insn, false);
2452 
2453    if (devinfo->gen < 6)
2454       brw_inst_set_base_mrf(devinfo, insn, msg_reg_nr);
2455 
2456    brw_set_dest(p, insn, dest);
2457    brw_set_src0(p, insn, src0);
2458    brw_set_sampler_message(p, insn,
2459                            binding_table_index,
2460                            sampler,
2461                            msg_type,
2462                            response_length,
2463                            msg_length,
2464                            header_present,
2465                            simd_mode,
2466                            return_format);
2467 }
2468 
2469 /* Adjust the message header's sampler state pointer to
2470  * select the correct group of 16 samplers.
2471  */
brw_adjust_sampler_state_pointer(struct brw_codegen * p,struct brw_reg header,struct brw_reg sampler_index)2472 void brw_adjust_sampler_state_pointer(struct brw_codegen *p,
2473                                       struct brw_reg header,
2474                                       struct brw_reg sampler_index)
2475 {
2476    /* The "Sampler Index" field can only store values between 0 and 15.
2477     * However, we can add an offset to the "Sampler State Pointer"
2478     * field, effectively selecting a different set of 16 samplers.
2479     *
2480     * The "Sampler State Pointer" needs to be aligned to a 32-byte
2481     * offset, and each sampler state is only 16-bytes, so we can't
2482     * exclusively use the offset - we have to use both.
2483     */
2484 
2485    const struct gen_device_info *devinfo = p->devinfo;
2486 
2487    if (sampler_index.file == BRW_IMMEDIATE_VALUE) {
2488       const int sampler_state_size = 16; /* 16 bytes */
2489       uint32_t sampler = sampler_index.ud;
2490 
2491       if (sampler >= 16) {
2492          assert(devinfo->is_haswell || devinfo->gen >= 8);
2493          brw_ADD(p,
2494                  get_element_ud(header, 3),
2495                  get_element_ud(brw_vec8_grf(0, 0), 3),
2496                  brw_imm_ud(16 * (sampler / 16) * sampler_state_size));
2497       }
2498    } else {
2499       /* Non-const sampler array indexing case */
2500       if (devinfo->gen < 8 && !devinfo->is_haswell) {
2501          return;
2502       }
2503 
2504       struct brw_reg temp = get_element_ud(header, 3);
2505 
2506       brw_AND(p, temp, get_element_ud(sampler_index, 0), brw_imm_ud(0x0f0));
2507       brw_SHL(p, temp, temp, brw_imm_ud(4));
2508       brw_ADD(p,
2509               get_element_ud(header, 3),
2510               get_element_ud(brw_vec8_grf(0, 0), 3),
2511               temp);
2512    }
2513 }
2514 
2515 /* All these variables are pretty confusing - we might be better off
2516  * using bitmasks and macros for this, in the old style.  Or perhaps
2517  * just having the caller instantiate the fields in dword3 itself.
2518  */
brw_urb_WRITE(struct brw_codegen * p,struct brw_reg dest,unsigned msg_reg_nr,struct brw_reg src0,enum brw_urb_write_flags flags,unsigned msg_length,unsigned response_length,unsigned offset,unsigned swizzle)2519 void brw_urb_WRITE(struct brw_codegen *p,
2520 		   struct brw_reg dest,
2521 		   unsigned msg_reg_nr,
2522 		   struct brw_reg src0,
2523                    enum brw_urb_write_flags flags,
2524 		   unsigned msg_length,
2525 		   unsigned response_length,
2526 		   unsigned offset,
2527 		   unsigned swizzle)
2528 {
2529    const struct gen_device_info *devinfo = p->devinfo;
2530    brw_inst *insn;
2531 
2532    gen6_resolve_implied_move(p, &src0, msg_reg_nr);
2533 
2534    if (devinfo->gen >= 7 && !(flags & BRW_URB_WRITE_USE_CHANNEL_MASKS)) {
2535       /* Enable Channel Masks in the URB_WRITE_HWORD message header */
2536       brw_push_insn_state(p);
2537       brw_set_default_access_mode(p, BRW_ALIGN_1);
2538       brw_set_default_mask_control(p, BRW_MASK_DISABLE);
2539       brw_OR(p, retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE, msg_reg_nr, 5),
2540 		       BRW_REGISTER_TYPE_UD),
2541 	        retype(brw_vec1_grf(0, 5), BRW_REGISTER_TYPE_UD),
2542 		brw_imm_ud(0xff00));
2543       brw_pop_insn_state(p);
2544    }
2545 
2546    insn = next_insn(p, BRW_OPCODE_SEND);
2547 
2548    assert(msg_length < BRW_MAX_MRF(devinfo->gen));
2549 
2550    brw_set_dest(p, insn, dest);
2551    brw_set_src0(p, insn, src0);
2552    brw_set_src1(p, insn, brw_imm_d(0));
2553 
2554    if (devinfo->gen < 6)
2555       brw_inst_set_base_mrf(devinfo, insn, msg_reg_nr);
2556 
2557    brw_set_urb_message(p,
2558 		       insn,
2559 		       flags,
2560 		       msg_length,
2561 		       response_length,
2562 		       offset,
2563 		       swizzle);
2564 }
2565 
2566 struct brw_inst *
brw_send_indirect_message(struct brw_codegen * p,unsigned sfid,struct brw_reg dst,struct brw_reg payload,struct brw_reg desc)2567 brw_send_indirect_message(struct brw_codegen *p,
2568                           unsigned sfid,
2569                           struct brw_reg dst,
2570                           struct brw_reg payload,
2571                           struct brw_reg desc)
2572 {
2573    const struct gen_device_info *devinfo = p->devinfo;
2574    struct brw_inst *send;
2575    int setup;
2576 
2577    dst = retype(dst, BRW_REGISTER_TYPE_UW);
2578 
2579    assert(desc.type == BRW_REGISTER_TYPE_UD);
2580 
2581    /* We hold on to the setup instruction (the SEND in the direct case, the OR
2582     * in the indirect case) by its index in the instruction store.  The
2583     * pointer returned by next_insn() may become invalid if emitting the SEND
2584     * in the indirect case reallocs the store.
2585     */
2586 
2587    if (desc.file == BRW_IMMEDIATE_VALUE) {
2588       setup = p->nr_insn;
2589       send = next_insn(p, BRW_OPCODE_SEND);
2590       brw_set_src1(p, send, desc);
2591 
2592    } else {
2593       struct brw_reg addr = retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD);
2594 
2595       brw_push_insn_state(p);
2596       brw_set_default_access_mode(p, BRW_ALIGN_1);
2597       brw_set_default_mask_control(p, BRW_MASK_DISABLE);
2598       brw_set_default_predicate_control(p, BRW_PREDICATE_NONE);
2599 
2600       /* Load the indirect descriptor to an address register using OR so the
2601        * caller can specify additional descriptor bits with the usual
2602        * brw_set_*_message() helper functions.
2603        */
2604       setup = p->nr_insn;
2605       brw_OR(p, addr, desc, brw_imm_ud(0));
2606 
2607       brw_pop_insn_state(p);
2608 
2609       send = next_insn(p, BRW_OPCODE_SEND);
2610       brw_set_src1(p, send, addr);
2611    }
2612 
2613    if (dst.width < BRW_EXECUTE_8)
2614       brw_inst_set_exec_size(devinfo, send, dst.width);
2615 
2616    brw_set_dest(p, send, dst);
2617    brw_set_src0(p, send, retype(payload, BRW_REGISTER_TYPE_UD));
2618    brw_inst_set_sfid(devinfo, send, sfid);
2619 
2620    return &p->store[setup];
2621 }
2622 
2623 static struct brw_inst *
brw_send_indirect_surface_message(struct brw_codegen * p,unsigned sfid,struct brw_reg dst,struct brw_reg payload,struct brw_reg surface,unsigned message_len,unsigned response_len,bool header_present)2624 brw_send_indirect_surface_message(struct brw_codegen *p,
2625                                   unsigned sfid,
2626                                   struct brw_reg dst,
2627                                   struct brw_reg payload,
2628                                   struct brw_reg surface,
2629                                   unsigned message_len,
2630                                   unsigned response_len,
2631                                   bool header_present)
2632 {
2633    const struct gen_device_info *devinfo = p->devinfo;
2634    struct brw_inst *insn;
2635 
2636    if (surface.file != BRW_IMMEDIATE_VALUE) {
2637       struct brw_reg addr = retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD);
2638 
2639       brw_push_insn_state(p);
2640       brw_set_default_access_mode(p, BRW_ALIGN_1);
2641       brw_set_default_mask_control(p, BRW_MASK_DISABLE);
2642       brw_set_default_predicate_control(p, BRW_PREDICATE_NONE);
2643 
2644       /* Mask out invalid bits from the surface index to avoid hangs e.g. when
2645        * some surface array is accessed out of bounds.
2646        */
2647       insn = brw_AND(p, addr,
2648                      suboffset(vec1(retype(surface, BRW_REGISTER_TYPE_UD)),
2649                                BRW_GET_SWZ(surface.swizzle, 0)),
2650                      brw_imm_ud(0xff));
2651 
2652       brw_pop_insn_state(p);
2653 
2654       surface = addr;
2655    }
2656 
2657    insn = brw_send_indirect_message(p, sfid, dst, payload, surface);
2658    brw_inst_set_mlen(devinfo, insn, message_len);
2659    brw_inst_set_rlen(devinfo, insn, response_len);
2660    brw_inst_set_header_present(devinfo, insn, header_present);
2661 
2662    return insn;
2663 }
2664 
2665 static bool
while_jumps_before_offset(const struct gen_device_info * devinfo,brw_inst * insn,int while_offset,int start_offset)2666 while_jumps_before_offset(const struct gen_device_info *devinfo,
2667                           brw_inst *insn, int while_offset, int start_offset)
2668 {
2669    int scale = 16 / brw_jump_scale(devinfo);
2670    int jip = devinfo->gen == 6 ? brw_inst_gen6_jump_count(devinfo, insn)
2671                                : brw_inst_jip(devinfo, insn);
2672    return while_offset + jip * scale <= start_offset;
2673 }
2674 
2675 
2676 static int
brw_find_next_block_end(struct brw_codegen * p,int start_offset)2677 brw_find_next_block_end(struct brw_codegen *p, int start_offset)
2678 {
2679    int offset;
2680    void *store = p->store;
2681    const struct gen_device_info *devinfo = p->devinfo;
2682 
2683    int depth = 0;
2684 
2685    for (offset = next_offset(devinfo, store, start_offset);
2686         offset < p->next_insn_offset;
2687         offset = next_offset(devinfo, store, offset)) {
2688       brw_inst *insn = store + offset;
2689 
2690       switch (brw_inst_opcode(devinfo, insn)) {
2691       case BRW_OPCODE_IF:
2692          depth++;
2693          break;
2694       case BRW_OPCODE_ENDIF:
2695          if (depth == 0)
2696             return offset;
2697          depth--;
2698          break;
2699       case BRW_OPCODE_WHILE:
2700          /* If the while doesn't jump before our instruction, it's the end
2701           * of a sibling do...while loop.  Ignore it.
2702           */
2703          if (!while_jumps_before_offset(devinfo, insn, offset, start_offset))
2704             continue;
2705          /* fallthrough */
2706       case BRW_OPCODE_ELSE:
2707       case BRW_OPCODE_HALT:
2708          if (depth == 0)
2709             return offset;
2710       }
2711    }
2712 
2713    return 0;
2714 }
2715 
2716 /* There is no DO instruction on gen6, so to find the end of the loop
2717  * we have to see if the loop is jumping back before our start
2718  * instruction.
2719  */
2720 static int
brw_find_loop_end(struct brw_codegen * p,int start_offset)2721 brw_find_loop_end(struct brw_codegen *p, int start_offset)
2722 {
2723    const struct gen_device_info *devinfo = p->devinfo;
2724    int offset;
2725    void *store = p->store;
2726 
2727    assert(devinfo->gen >= 6);
2728 
2729    /* Always start after the instruction (such as a WHILE) we're trying to fix
2730     * up.
2731     */
2732    for (offset = next_offset(devinfo, store, start_offset);
2733         offset < p->next_insn_offset;
2734         offset = next_offset(devinfo, store, offset)) {
2735       brw_inst *insn = store + offset;
2736 
2737       if (brw_inst_opcode(devinfo, insn) == BRW_OPCODE_WHILE) {
2738 	 if (while_jumps_before_offset(devinfo, insn, offset, start_offset))
2739 	    return offset;
2740       }
2741    }
2742    assert(!"not reached");
2743    return start_offset;
2744 }
2745 
2746 /* After program generation, go back and update the UIP and JIP of
2747  * BREAK, CONT, and HALT instructions to their correct locations.
2748  */
2749 void
brw_set_uip_jip(struct brw_codegen * p,int start_offset)2750 brw_set_uip_jip(struct brw_codegen *p, int start_offset)
2751 {
2752    const struct gen_device_info *devinfo = p->devinfo;
2753    int offset;
2754    int br = brw_jump_scale(devinfo);
2755    int scale = 16 / br;
2756    void *store = p->store;
2757 
2758    if (devinfo->gen < 6)
2759       return;
2760 
2761    for (offset = start_offset; offset < p->next_insn_offset; offset += 16) {
2762       brw_inst *insn = store + offset;
2763       assert(brw_inst_cmpt_control(devinfo, insn) == 0);
2764 
2765       int block_end_offset = brw_find_next_block_end(p, offset);
2766       switch (brw_inst_opcode(devinfo, insn)) {
2767       case BRW_OPCODE_BREAK:
2768          assert(block_end_offset != 0);
2769          brw_inst_set_jip(devinfo, insn, (block_end_offset - offset) / scale);
2770 	 /* Gen7 UIP points to WHILE; Gen6 points just after it */
2771          brw_inst_set_uip(devinfo, insn,
2772 	    (brw_find_loop_end(p, offset) - offset +
2773              (devinfo->gen == 6 ? 16 : 0)) / scale);
2774 	 break;
2775       case BRW_OPCODE_CONTINUE:
2776          assert(block_end_offset != 0);
2777          brw_inst_set_jip(devinfo, insn, (block_end_offset - offset) / scale);
2778          brw_inst_set_uip(devinfo, insn,
2779             (brw_find_loop_end(p, offset) - offset) / scale);
2780 
2781          assert(brw_inst_uip(devinfo, insn) != 0);
2782          assert(brw_inst_jip(devinfo, insn) != 0);
2783 	 break;
2784 
2785       case BRW_OPCODE_ENDIF: {
2786          int32_t jump = (block_end_offset == 0) ?
2787                         1 * br : (block_end_offset - offset) / scale;
2788          if (devinfo->gen >= 7)
2789             brw_inst_set_jip(devinfo, insn, jump);
2790          else
2791             brw_inst_set_gen6_jump_count(devinfo, insn, jump);
2792 	 break;
2793       }
2794 
2795       case BRW_OPCODE_HALT:
2796 	 /* From the Sandy Bridge PRM (volume 4, part 2, section 8.3.19):
2797 	  *
2798 	  *    "In case of the halt instruction not inside any conditional
2799 	  *     code block, the value of <JIP> and <UIP> should be the
2800 	  *     same. In case of the halt instruction inside conditional code
2801 	  *     block, the <UIP> should be the end of the program, and the
2802 	  *     <JIP> should be end of the most inner conditional code block."
2803 	  *
2804 	  * The uip will have already been set by whoever set up the
2805 	  * instruction.
2806 	  */
2807 	 if (block_end_offset == 0) {
2808             brw_inst_set_jip(devinfo, insn, brw_inst_uip(devinfo, insn));
2809 	 } else {
2810             brw_inst_set_jip(devinfo, insn, (block_end_offset - offset) / scale);
2811 	 }
2812          assert(brw_inst_uip(devinfo, insn) != 0);
2813          assert(brw_inst_jip(devinfo, insn) != 0);
2814 	 break;
2815       }
2816    }
2817 }
2818 
brw_ff_sync(struct brw_codegen * p,struct brw_reg dest,unsigned msg_reg_nr,struct brw_reg src0,bool allocate,unsigned response_length,bool eot)2819 void brw_ff_sync(struct brw_codegen *p,
2820 		   struct brw_reg dest,
2821 		   unsigned msg_reg_nr,
2822 		   struct brw_reg src0,
2823 		   bool allocate,
2824 		   unsigned response_length,
2825 		   bool eot)
2826 {
2827    const struct gen_device_info *devinfo = p->devinfo;
2828    brw_inst *insn;
2829 
2830    gen6_resolve_implied_move(p, &src0, msg_reg_nr);
2831 
2832    insn = next_insn(p, BRW_OPCODE_SEND);
2833    brw_set_dest(p, insn, dest);
2834    brw_set_src0(p, insn, src0);
2835    brw_set_src1(p, insn, brw_imm_d(0));
2836 
2837    if (devinfo->gen < 6)
2838       brw_inst_set_base_mrf(devinfo, insn, msg_reg_nr);
2839 
2840    brw_set_ff_sync_message(p,
2841 			   insn,
2842 			   allocate,
2843 			   response_length,
2844 			   eot);
2845 }
2846 
2847 /**
2848  * Emit the SEND instruction necessary to generate stream output data on Gen6
2849  * (for transform feedback).
2850  *
2851  * If send_commit_msg is true, this is the last piece of stream output data
2852  * from this thread, so send the data as a committed write.  According to the
2853  * Sandy Bridge PRM (volume 2 part 1, section 4.5.1):
2854  *
2855  *   "Prior to End of Thread with a URB_WRITE, the kernel must ensure all
2856  *   writes are complete by sending the final write as a committed write."
2857  */
2858 void
brw_svb_write(struct brw_codegen * p,struct brw_reg dest,unsigned msg_reg_nr,struct brw_reg src0,unsigned binding_table_index,bool send_commit_msg)2859 brw_svb_write(struct brw_codegen *p,
2860               struct brw_reg dest,
2861               unsigned msg_reg_nr,
2862               struct brw_reg src0,
2863               unsigned binding_table_index,
2864               bool   send_commit_msg)
2865 {
2866    const struct gen_device_info *devinfo = p->devinfo;
2867    const unsigned target_cache =
2868       (devinfo->gen >= 7 ? GEN7_SFID_DATAPORT_DATA_CACHE :
2869        devinfo->gen >= 6 ? GEN6_SFID_DATAPORT_RENDER_CACHE :
2870        BRW_DATAPORT_READ_TARGET_RENDER_CACHE);
2871    brw_inst *insn;
2872 
2873    gen6_resolve_implied_move(p, &src0, msg_reg_nr);
2874 
2875    insn = next_insn(p, BRW_OPCODE_SEND);
2876    brw_set_dest(p, insn, dest);
2877    brw_set_src0(p, insn, src0);
2878    brw_set_src1(p, insn, brw_imm_d(0));
2879    brw_set_dp_write_message(p, insn,
2880                             binding_table_index,
2881                             0, /* msg_control: ignored */
2882                             GEN6_DATAPORT_WRITE_MESSAGE_STREAMED_VB_WRITE,
2883                             target_cache,
2884                             1, /* msg_length */
2885                             true, /* header_present */
2886                             0, /* last_render_target: ignored */
2887                             send_commit_msg, /* response_length */
2888                             0, /* end_of_thread */
2889                             send_commit_msg); /* send_commit_msg */
2890 }
2891 
2892 static unsigned
brw_surface_payload_size(struct brw_codegen * p,unsigned num_channels,bool has_simd4x2,bool has_simd16)2893 brw_surface_payload_size(struct brw_codegen *p,
2894                          unsigned num_channels,
2895                          bool has_simd4x2,
2896                          bool has_simd16)
2897 {
2898    if (has_simd4x2 &&
2899        brw_inst_access_mode(p->devinfo, p->current) == BRW_ALIGN_16)
2900       return 1;
2901    else if (has_simd16 &&
2902             brw_inst_exec_size(p->devinfo, p->current) == BRW_EXECUTE_16)
2903       return 2 * num_channels;
2904    else
2905       return num_channels;
2906 }
2907 
2908 static void
brw_set_dp_untyped_atomic_message(struct brw_codegen * p,brw_inst * insn,unsigned atomic_op,bool response_expected)2909 brw_set_dp_untyped_atomic_message(struct brw_codegen *p,
2910                                   brw_inst *insn,
2911                                   unsigned atomic_op,
2912                                   bool response_expected)
2913 {
2914    const struct gen_device_info *devinfo = p->devinfo;
2915    unsigned msg_control =
2916       atomic_op | /* Atomic Operation Type: BRW_AOP_* */
2917       (response_expected ? 1 << 5 : 0); /* Return data expected */
2918 
2919    if (devinfo->gen >= 8 || devinfo->is_haswell) {
2920       if (brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1) {
2921          if (brw_inst_exec_size(devinfo, p->current) != BRW_EXECUTE_16)
2922             msg_control |= 1 << 4; /* SIMD8 mode */
2923 
2924          brw_inst_set_dp_msg_type(devinfo, insn,
2925                                   HSW_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_OP);
2926       } else {
2927          brw_inst_set_dp_msg_type(devinfo, insn,
2928             HSW_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_OP_SIMD4X2);
2929       }
2930    } else {
2931       brw_inst_set_dp_msg_type(devinfo, insn,
2932                                GEN7_DATAPORT_DC_UNTYPED_ATOMIC_OP);
2933 
2934       if (brw_inst_exec_size(devinfo, p->current) != BRW_EXECUTE_16)
2935          msg_control |= 1 << 4; /* SIMD8 mode */
2936    }
2937 
2938    brw_inst_set_dp_msg_control(devinfo, insn, msg_control);
2939 }
2940 
2941 void
brw_untyped_atomic(struct brw_codegen * p,struct brw_reg dst,struct brw_reg payload,struct brw_reg surface,unsigned atomic_op,unsigned msg_length,bool response_expected)2942 brw_untyped_atomic(struct brw_codegen *p,
2943                    struct brw_reg dst,
2944                    struct brw_reg payload,
2945                    struct brw_reg surface,
2946                    unsigned atomic_op,
2947                    unsigned msg_length,
2948                    bool response_expected)
2949 {
2950    const struct gen_device_info *devinfo = p->devinfo;
2951    const unsigned sfid = (devinfo->gen >= 8 || devinfo->is_haswell ?
2952                           HSW_SFID_DATAPORT_DATA_CACHE_1 :
2953                           GEN7_SFID_DATAPORT_DATA_CACHE);
2954    const bool align1 = brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1;
2955    /* Mask out unused components -- This is especially important in Align16
2956     * mode on generations that don't have native support for SIMD4x2 atomics,
2957     * because unused but enabled components will cause the dataport to perform
2958     * additional atomic operations on the addresses that happen to be in the
2959     * uninitialized Y, Z and W coordinates of the payload.
2960     */
2961    const unsigned mask = align1 ? WRITEMASK_XYZW : WRITEMASK_X;
2962    struct brw_inst *insn = brw_send_indirect_surface_message(
2963       p, sfid, brw_writemask(dst, mask), payload, surface, msg_length,
2964       brw_surface_payload_size(p, response_expected,
2965                                devinfo->gen >= 8 || devinfo->is_haswell, true),
2966       align1);
2967 
2968    brw_set_dp_untyped_atomic_message(
2969       p, insn, atomic_op, response_expected);
2970 }
2971 
2972 static void
brw_set_dp_untyped_surface_read_message(struct brw_codegen * p,struct brw_inst * insn,unsigned num_channels)2973 brw_set_dp_untyped_surface_read_message(struct brw_codegen *p,
2974                                         struct brw_inst *insn,
2975                                         unsigned num_channels)
2976 {
2977    const struct gen_device_info *devinfo = p->devinfo;
2978    /* Set mask of 32-bit channels to drop. */
2979    unsigned msg_control = 0xf & (0xf << num_channels);
2980 
2981    if (brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1) {
2982       if (brw_inst_exec_size(devinfo, p->current) == BRW_EXECUTE_16)
2983          msg_control |= 1 << 4; /* SIMD16 mode */
2984       else
2985          msg_control |= 2 << 4; /* SIMD8 mode */
2986    }
2987 
2988    brw_inst_set_dp_msg_type(devinfo, insn,
2989                             (devinfo->gen >= 8 || devinfo->is_haswell ?
2990                              HSW_DATAPORT_DC_PORT1_UNTYPED_SURFACE_READ :
2991                              GEN7_DATAPORT_DC_UNTYPED_SURFACE_READ));
2992    brw_inst_set_dp_msg_control(devinfo, insn, msg_control);
2993 }
2994 
2995 void
brw_untyped_surface_read(struct brw_codegen * p,struct brw_reg dst,struct brw_reg payload,struct brw_reg surface,unsigned msg_length,unsigned num_channels)2996 brw_untyped_surface_read(struct brw_codegen *p,
2997                          struct brw_reg dst,
2998                          struct brw_reg payload,
2999                          struct brw_reg surface,
3000                          unsigned msg_length,
3001                          unsigned num_channels)
3002 {
3003    const struct gen_device_info *devinfo = p->devinfo;
3004    const unsigned sfid = (devinfo->gen >= 8 || devinfo->is_haswell ?
3005                           HSW_SFID_DATAPORT_DATA_CACHE_1 :
3006                           GEN7_SFID_DATAPORT_DATA_CACHE);
3007    struct brw_inst *insn = brw_send_indirect_surface_message(
3008       p, sfid, dst, payload, surface, msg_length,
3009       brw_surface_payload_size(p, num_channels, true, true),
3010       false);
3011 
3012    brw_set_dp_untyped_surface_read_message(
3013       p, insn, num_channels);
3014 }
3015 
3016 static void
brw_set_dp_untyped_surface_write_message(struct brw_codegen * p,struct brw_inst * insn,unsigned num_channels)3017 brw_set_dp_untyped_surface_write_message(struct brw_codegen *p,
3018                                          struct brw_inst *insn,
3019                                          unsigned num_channels)
3020 {
3021    const struct gen_device_info *devinfo = p->devinfo;
3022    /* Set mask of 32-bit channels to drop. */
3023    unsigned msg_control = 0xf & (0xf << num_channels);
3024 
3025    if (brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1) {
3026       if (brw_inst_exec_size(devinfo, p->current) == BRW_EXECUTE_16)
3027          msg_control |= 1 << 4; /* SIMD16 mode */
3028       else
3029          msg_control |= 2 << 4; /* SIMD8 mode */
3030    } else {
3031       if (devinfo->gen >= 8 || devinfo->is_haswell)
3032          msg_control |= 0 << 4; /* SIMD4x2 mode */
3033       else
3034          msg_control |= 2 << 4; /* SIMD8 mode */
3035    }
3036 
3037    brw_inst_set_dp_msg_type(devinfo, insn,
3038                             devinfo->gen >= 8 || devinfo->is_haswell ?
3039                              HSW_DATAPORT_DC_PORT1_UNTYPED_SURFACE_WRITE :
3040                              GEN7_DATAPORT_DC_UNTYPED_SURFACE_WRITE);
3041    brw_inst_set_dp_msg_control(devinfo, insn, msg_control);
3042 }
3043 
3044 void
brw_untyped_surface_write(struct brw_codegen * p,struct brw_reg payload,struct brw_reg surface,unsigned msg_length,unsigned num_channels)3045 brw_untyped_surface_write(struct brw_codegen *p,
3046                           struct brw_reg payload,
3047                           struct brw_reg surface,
3048                           unsigned msg_length,
3049                           unsigned num_channels)
3050 {
3051    const struct gen_device_info *devinfo = p->devinfo;
3052    const unsigned sfid = (devinfo->gen >= 8 || devinfo->is_haswell ?
3053                           HSW_SFID_DATAPORT_DATA_CACHE_1 :
3054                           GEN7_SFID_DATAPORT_DATA_CACHE);
3055    const bool align1 = brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1;
3056    /* Mask out unused components -- See comment in brw_untyped_atomic(). */
3057    const unsigned mask = devinfo->gen == 7 && !devinfo->is_haswell && !align1 ?
3058                           WRITEMASK_X : WRITEMASK_XYZW;
3059    struct brw_inst *insn = brw_send_indirect_surface_message(
3060       p, sfid, brw_writemask(brw_null_reg(), mask),
3061       payload, surface, msg_length, 0, align1);
3062 
3063    brw_set_dp_untyped_surface_write_message(
3064       p, insn, num_channels);
3065 }
3066 
3067 static void
brw_set_dp_typed_atomic_message(struct brw_codegen * p,struct brw_inst * insn,unsigned atomic_op,bool response_expected)3068 brw_set_dp_typed_atomic_message(struct brw_codegen *p,
3069                                 struct brw_inst *insn,
3070                                 unsigned atomic_op,
3071                                 bool response_expected)
3072 {
3073    const struct gen_device_info *devinfo = p->devinfo;
3074    unsigned msg_control =
3075       atomic_op | /* Atomic Operation Type: BRW_AOP_* */
3076       (response_expected ? 1 << 5 : 0); /* Return data expected */
3077 
3078    if (devinfo->gen >= 8 || devinfo->is_haswell) {
3079       if (brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1) {
3080          if (brw_inst_qtr_control(devinfo, p->current) % 2 == 1)
3081             msg_control |= 1 << 4; /* Use high 8 slots of the sample mask */
3082 
3083          brw_inst_set_dp_msg_type(devinfo, insn,
3084                                   HSW_DATAPORT_DC_PORT1_TYPED_ATOMIC_OP);
3085       } else {
3086          brw_inst_set_dp_msg_type(devinfo, insn,
3087                                   HSW_DATAPORT_DC_PORT1_TYPED_ATOMIC_OP_SIMD4X2);
3088       }
3089 
3090    } else {
3091       brw_inst_set_dp_msg_type(devinfo, insn,
3092                                GEN7_DATAPORT_RC_TYPED_ATOMIC_OP);
3093 
3094       if (brw_inst_qtr_control(devinfo, p->current) % 2 == 1)
3095          msg_control |= 1 << 4; /* Use high 8 slots of the sample mask */
3096    }
3097 
3098    brw_inst_set_dp_msg_control(devinfo, insn, msg_control);
3099 }
3100 
3101 void
brw_typed_atomic(struct brw_codegen * p,struct brw_reg dst,struct brw_reg payload,struct brw_reg surface,unsigned atomic_op,unsigned msg_length,bool response_expected)3102 brw_typed_atomic(struct brw_codegen *p,
3103                  struct brw_reg dst,
3104                  struct brw_reg payload,
3105                  struct brw_reg surface,
3106                  unsigned atomic_op,
3107                  unsigned msg_length,
3108                  bool response_expected) {
3109    const struct gen_device_info *devinfo = p->devinfo;
3110    const unsigned sfid = (devinfo->gen >= 8 || devinfo->is_haswell ?
3111                           HSW_SFID_DATAPORT_DATA_CACHE_1 :
3112                           GEN6_SFID_DATAPORT_RENDER_CACHE);
3113    const bool align1 = (brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1);
3114    /* Mask out unused components -- See comment in brw_untyped_atomic(). */
3115    const unsigned mask = align1 ? WRITEMASK_XYZW : WRITEMASK_X;
3116    struct brw_inst *insn = brw_send_indirect_surface_message(
3117       p, sfid, brw_writemask(dst, mask), payload, surface, msg_length,
3118       brw_surface_payload_size(p, response_expected,
3119                                devinfo->gen >= 8 || devinfo->is_haswell, false),
3120       true);
3121 
3122    brw_set_dp_typed_atomic_message(
3123       p, insn, atomic_op, response_expected);
3124 }
3125 
3126 static void
brw_set_dp_typed_surface_read_message(struct brw_codegen * p,struct brw_inst * insn,unsigned num_channels)3127 brw_set_dp_typed_surface_read_message(struct brw_codegen *p,
3128                                       struct brw_inst *insn,
3129                                       unsigned num_channels)
3130 {
3131    const struct gen_device_info *devinfo = p->devinfo;
3132    /* Set mask of unused channels. */
3133    unsigned msg_control = 0xf & (0xf << num_channels);
3134 
3135    if (devinfo->gen >= 8 || devinfo->is_haswell) {
3136       if (brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1) {
3137          if (brw_inst_qtr_control(devinfo, p->current) % 2 == 1)
3138             msg_control |= 2 << 4; /* Use high 8 slots of the sample mask */
3139          else
3140             msg_control |= 1 << 4; /* Use low 8 slots of the sample mask */
3141       }
3142 
3143       brw_inst_set_dp_msg_type(devinfo, insn,
3144                                HSW_DATAPORT_DC_PORT1_TYPED_SURFACE_READ);
3145    } else {
3146       if (brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1) {
3147          if (brw_inst_qtr_control(devinfo, p->current) % 2 == 1)
3148             msg_control |= 1 << 5; /* Use high 8 slots of the sample mask */
3149       }
3150 
3151       brw_inst_set_dp_msg_type(devinfo, insn,
3152                                GEN7_DATAPORT_RC_TYPED_SURFACE_READ);
3153    }
3154 
3155    brw_inst_set_dp_msg_control(devinfo, insn, msg_control);
3156 }
3157 
3158 void
brw_typed_surface_read(struct brw_codegen * p,struct brw_reg dst,struct brw_reg payload,struct brw_reg surface,unsigned msg_length,unsigned num_channels)3159 brw_typed_surface_read(struct brw_codegen *p,
3160                        struct brw_reg dst,
3161                        struct brw_reg payload,
3162                        struct brw_reg surface,
3163                        unsigned msg_length,
3164                        unsigned num_channels)
3165 {
3166    const struct gen_device_info *devinfo = p->devinfo;
3167    const unsigned sfid = (devinfo->gen >= 8 || devinfo->is_haswell ?
3168                           HSW_SFID_DATAPORT_DATA_CACHE_1 :
3169                           GEN6_SFID_DATAPORT_RENDER_CACHE);
3170    struct brw_inst *insn = brw_send_indirect_surface_message(
3171       p, sfid, dst, payload, surface, msg_length,
3172       brw_surface_payload_size(p, num_channels,
3173                                devinfo->gen >= 8 || devinfo->is_haswell, false),
3174       true);
3175 
3176    brw_set_dp_typed_surface_read_message(
3177       p, insn, num_channels);
3178 }
3179 
3180 static void
brw_set_dp_typed_surface_write_message(struct brw_codegen * p,struct brw_inst * insn,unsigned num_channels)3181 brw_set_dp_typed_surface_write_message(struct brw_codegen *p,
3182                                        struct brw_inst *insn,
3183                                        unsigned num_channels)
3184 {
3185    const struct gen_device_info *devinfo = p->devinfo;
3186    /* Set mask of unused channels. */
3187    unsigned msg_control = 0xf & (0xf << num_channels);
3188 
3189    if (devinfo->gen >= 8 || devinfo->is_haswell) {
3190       if (brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1) {
3191          if (brw_inst_qtr_control(devinfo, p->current) % 2 == 1)
3192             msg_control |= 2 << 4; /* Use high 8 slots of the sample mask */
3193          else
3194             msg_control |= 1 << 4; /* Use low 8 slots of the sample mask */
3195       }
3196 
3197       brw_inst_set_dp_msg_type(devinfo, insn,
3198                                HSW_DATAPORT_DC_PORT1_TYPED_SURFACE_WRITE);
3199 
3200    } else {
3201       if (brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1) {
3202          if (brw_inst_qtr_control(devinfo, p->current) % 2 == 1)
3203             msg_control |= 1 << 5; /* Use high 8 slots of the sample mask */
3204       }
3205 
3206       brw_inst_set_dp_msg_type(devinfo, insn,
3207                                GEN7_DATAPORT_RC_TYPED_SURFACE_WRITE);
3208    }
3209 
3210    brw_inst_set_dp_msg_control(devinfo, insn, msg_control);
3211 }
3212 
3213 void
brw_typed_surface_write(struct brw_codegen * p,struct brw_reg payload,struct brw_reg surface,unsigned msg_length,unsigned num_channels)3214 brw_typed_surface_write(struct brw_codegen *p,
3215                         struct brw_reg payload,
3216                         struct brw_reg surface,
3217                         unsigned msg_length,
3218                         unsigned num_channels)
3219 {
3220    const struct gen_device_info *devinfo = p->devinfo;
3221    const unsigned sfid = (devinfo->gen >= 8 || devinfo->is_haswell ?
3222                           HSW_SFID_DATAPORT_DATA_CACHE_1 :
3223                           GEN6_SFID_DATAPORT_RENDER_CACHE);
3224    const bool align1 = (brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1);
3225    /* Mask out unused components -- See comment in brw_untyped_atomic(). */
3226    const unsigned mask = (devinfo->gen == 7 && !devinfo->is_haswell && !align1 ?
3227                           WRITEMASK_X : WRITEMASK_XYZW);
3228    struct brw_inst *insn = brw_send_indirect_surface_message(
3229       p, sfid, brw_writemask(brw_null_reg(), mask),
3230       payload, surface, msg_length, 0, true);
3231 
3232    brw_set_dp_typed_surface_write_message(
3233       p, insn, num_channels);
3234 }
3235 
3236 static void
brw_set_memory_fence_message(struct brw_codegen * p,struct brw_inst * insn,enum brw_message_target sfid,bool commit_enable)3237 brw_set_memory_fence_message(struct brw_codegen *p,
3238                              struct brw_inst *insn,
3239                              enum brw_message_target sfid,
3240                              bool commit_enable)
3241 {
3242    const struct gen_device_info *devinfo = p->devinfo;
3243 
3244    brw_set_message_descriptor(p, insn, sfid,
3245                               1 /* message length */,
3246                               (commit_enable ? 1 : 0) /* response length */,
3247                               true /* header present */,
3248                               false);
3249 
3250    switch (sfid) {
3251    case GEN6_SFID_DATAPORT_RENDER_CACHE:
3252       brw_inst_set_dp_msg_type(devinfo, insn, GEN7_DATAPORT_RC_MEMORY_FENCE);
3253       break;
3254    case GEN7_SFID_DATAPORT_DATA_CACHE:
3255       brw_inst_set_dp_msg_type(devinfo, insn, GEN7_DATAPORT_DC_MEMORY_FENCE);
3256       break;
3257    default:
3258       unreachable("Not reached");
3259    }
3260 
3261    if (commit_enable)
3262       brw_inst_set_dp_msg_control(devinfo, insn, 1 << 5);
3263 }
3264 
3265 void
brw_memory_fence(struct brw_codegen * p,struct brw_reg dst)3266 brw_memory_fence(struct brw_codegen *p,
3267                  struct brw_reg dst)
3268 {
3269    const struct gen_device_info *devinfo = p->devinfo;
3270    const bool commit_enable = devinfo->gen == 7 && !devinfo->is_haswell;
3271    struct brw_inst *insn;
3272 
3273    brw_push_insn_state(p);
3274    brw_set_default_mask_control(p, BRW_MASK_DISABLE);
3275    brw_set_default_exec_size(p, BRW_EXECUTE_1);
3276    dst = vec1(dst);
3277 
3278    /* Set dst as destination for dependency tracking, the MEMORY_FENCE
3279     * message doesn't write anything back.
3280     */
3281    insn = next_insn(p, BRW_OPCODE_SEND);
3282    dst = retype(dst, BRW_REGISTER_TYPE_UW);
3283    brw_set_dest(p, insn, dst);
3284    brw_set_src0(p, insn, dst);
3285    brw_set_memory_fence_message(p, insn, GEN7_SFID_DATAPORT_DATA_CACHE,
3286                                 commit_enable);
3287 
3288    if (devinfo->gen == 7 && !devinfo->is_haswell) {
3289       /* IVB does typed surface access through the render cache, so we need to
3290        * flush it too.  Use a different register so both flushes can be
3291        * pipelined by the hardware.
3292        */
3293       insn = next_insn(p, BRW_OPCODE_SEND);
3294       brw_set_dest(p, insn, offset(dst, 1));
3295       brw_set_src0(p, insn, offset(dst, 1));
3296       brw_set_memory_fence_message(p, insn, GEN6_SFID_DATAPORT_RENDER_CACHE,
3297                                    commit_enable);
3298 
3299       /* Now write the response of the second message into the response of the
3300        * first to trigger a pipeline stall -- This way future render and data
3301        * cache messages will be properly ordered with respect to past data and
3302        * render cache messages.
3303        */
3304       brw_MOV(p, dst, offset(dst, 1));
3305    }
3306 
3307    brw_pop_insn_state(p);
3308 }
3309 
3310 void
brw_pixel_interpolator_query(struct brw_codegen * p,struct brw_reg dest,struct brw_reg mrf,bool noperspective,unsigned mode,struct brw_reg data,unsigned msg_length,unsigned response_length)3311 brw_pixel_interpolator_query(struct brw_codegen *p,
3312                              struct brw_reg dest,
3313                              struct brw_reg mrf,
3314                              bool noperspective,
3315                              unsigned mode,
3316                              struct brw_reg data,
3317                              unsigned msg_length,
3318                              unsigned response_length)
3319 {
3320    const struct gen_device_info *devinfo = p->devinfo;
3321    struct brw_inst *insn;
3322    const uint16_t exec_size = brw_inst_exec_size(devinfo, p->current);
3323 
3324    /* brw_send_indirect_message will automatically use a direct send message
3325     * if data is actually immediate.
3326     */
3327    insn = brw_send_indirect_message(p,
3328                                     GEN7_SFID_PIXEL_INTERPOLATOR,
3329                                     dest,
3330                                     mrf,
3331                                     vec1(data));
3332    brw_inst_set_mlen(devinfo, insn, msg_length);
3333    brw_inst_set_rlen(devinfo, insn, response_length);
3334 
3335    brw_inst_set_pi_simd_mode(devinfo, insn, exec_size == BRW_EXECUTE_16);
3336    brw_inst_set_pi_slot_group(devinfo, insn, 0); /* zero unless 32/64px dispatch */
3337    brw_inst_set_pi_nopersp(devinfo, insn, noperspective);
3338    brw_inst_set_pi_message_type(devinfo, insn, mode);
3339 }
3340 
3341 void
brw_find_live_channel(struct brw_codegen * p,struct brw_reg dst,struct brw_reg mask)3342 brw_find_live_channel(struct brw_codegen *p, struct brw_reg dst,
3343                       struct brw_reg mask)
3344 {
3345    const struct gen_device_info *devinfo = p->devinfo;
3346    const unsigned exec_size = 1 << brw_inst_exec_size(devinfo, p->current);
3347    const unsigned qtr_control = brw_inst_qtr_control(devinfo, p->current);
3348    brw_inst *inst;
3349 
3350    assert(devinfo->gen >= 7);
3351    assert(mask.type == BRW_REGISTER_TYPE_UD);
3352 
3353    brw_push_insn_state(p);
3354 
3355    if (brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1) {
3356       brw_set_default_mask_control(p, BRW_MASK_DISABLE);
3357 
3358       if (devinfo->gen >= 8) {
3359          /* Getting the first active channel index is easy on Gen8: Just find
3360           * the first bit set in the execution mask.  The register exists on
3361           * HSW already but it reads back as all ones when the current
3362           * instruction has execution masking disabled, so it's kind of
3363           * useless.
3364           */
3365          struct brw_reg exec_mask =
3366             retype(brw_mask_reg(0), BRW_REGISTER_TYPE_UD);
3367 
3368          if (mask.file != BRW_IMMEDIATE_VALUE || mask.ud != 0xffffffff) {
3369             /* Unfortunately, ce0 does not take into account the thread
3370              * dispatch mask, which may be a problem in cases where it's not
3371              * tightly packed (i.e. it doesn't have the form '2^n - 1' for
3372              * some n).  Combine ce0 with the given dispatch (or vector) mask
3373              * to mask off those channels which were never dispatched by the
3374              * hardware.
3375              */
3376             brw_SHR(p, vec1(dst), mask, brw_imm_ud(qtr_control * 8));
3377             brw_AND(p, vec1(dst), exec_mask, vec1(dst));
3378             exec_mask = vec1(dst);
3379          }
3380 
3381          /* Quarter control has the effect of magically shifting the value of
3382           * ce0 so you'll get the first active channel relative to the
3383           * specified quarter control as result.
3384           */
3385          inst = brw_FBL(p, vec1(dst), exec_mask);
3386       } else {
3387          const struct brw_reg flag = brw_flag_reg(1, 0);
3388 
3389          brw_MOV(p, retype(flag, BRW_REGISTER_TYPE_UD), brw_imm_ud(0));
3390 
3391          /* Run enough instructions returning zero with execution masking and
3392           * a conditional modifier enabled in order to get the full execution
3393           * mask in f1.0.  We could use a single 32-wide move here if it
3394           * weren't because of the hardware bug that causes channel enables to
3395           * be applied incorrectly to the second half of 32-wide instructions
3396           * on Gen7.
3397           */
3398          const unsigned lower_size = MIN2(16, exec_size);
3399          for (unsigned i = 0; i < exec_size / lower_size; i++) {
3400             inst = brw_MOV(p, retype(brw_null_reg(), BRW_REGISTER_TYPE_UW),
3401                            brw_imm_uw(0));
3402             brw_inst_set_mask_control(devinfo, inst, BRW_MASK_ENABLE);
3403             brw_inst_set_group(devinfo, inst, lower_size * i + 8 * qtr_control);
3404             brw_inst_set_cond_modifier(devinfo, inst, BRW_CONDITIONAL_Z);
3405             brw_inst_set_flag_reg_nr(devinfo, inst, 1);
3406             brw_inst_set_exec_size(devinfo, inst, cvt(lower_size) - 1);
3407          }
3408 
3409          /* Find the first bit set in the exec_size-wide portion of the flag
3410           * register that was updated by the last sequence of MOV
3411           * instructions.
3412           */
3413          const enum brw_reg_type type = brw_int_type(exec_size / 8, false);
3414          brw_FBL(p, vec1(dst), byte_offset(retype(flag, type), qtr_control));
3415       }
3416    } else {
3417       brw_set_default_mask_control(p, BRW_MASK_DISABLE);
3418 
3419       if (devinfo->gen >= 8 &&
3420           mask.file == BRW_IMMEDIATE_VALUE && mask.ud == 0xffffffff) {
3421          /* In SIMD4x2 mode the first active channel index is just the
3422           * negation of the first bit of the mask register.  Note that ce0
3423           * doesn't take into account the dispatch mask, so the Gen7 path
3424           * should be used instead unless you have the guarantee that the
3425           * dispatch mask is tightly packed (i.e. it has the form '2^n - 1'
3426           * for some n).
3427           */
3428          inst = brw_AND(p, brw_writemask(dst, WRITEMASK_X),
3429                         negate(retype(brw_mask_reg(0), BRW_REGISTER_TYPE_UD)),
3430                         brw_imm_ud(1));
3431 
3432       } else {
3433          /* Overwrite the destination without and with execution masking to
3434           * find out which of the channels is active.
3435           */
3436          brw_push_insn_state(p);
3437          brw_set_default_exec_size(p, BRW_EXECUTE_4);
3438          brw_MOV(p, brw_writemask(vec4(dst), WRITEMASK_X),
3439                  brw_imm_ud(1));
3440 
3441          inst = brw_MOV(p, brw_writemask(vec4(dst), WRITEMASK_X),
3442                         brw_imm_ud(0));
3443          brw_pop_insn_state(p);
3444          brw_inst_set_mask_control(devinfo, inst, BRW_MASK_ENABLE);
3445       }
3446    }
3447 
3448    brw_pop_insn_state(p);
3449 }
3450 
3451 void
brw_broadcast(struct brw_codegen * p,struct brw_reg dst,struct brw_reg src,struct brw_reg idx)3452 brw_broadcast(struct brw_codegen *p,
3453               struct brw_reg dst,
3454               struct brw_reg src,
3455               struct brw_reg idx)
3456 {
3457    const struct gen_device_info *devinfo = p->devinfo;
3458    const bool align1 = brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1;
3459    brw_inst *inst;
3460 
3461    brw_push_insn_state(p);
3462    brw_set_default_mask_control(p, BRW_MASK_DISABLE);
3463    brw_set_default_exec_size(p, align1 ? BRW_EXECUTE_1 : BRW_EXECUTE_4);
3464 
3465    assert(src.file == BRW_GENERAL_REGISTER_FILE &&
3466           src.address_mode == BRW_ADDRESS_DIRECT);
3467 
3468    if ((src.vstride == 0 && (src.hstride == 0 || !align1)) ||
3469        idx.file == BRW_IMMEDIATE_VALUE) {
3470       /* Trivial, the source is already uniform or the index is a constant.
3471        * We will typically not get here if the optimizer is doing its job, but
3472        * asserting would be mean.
3473        */
3474       const unsigned i = idx.file == BRW_IMMEDIATE_VALUE ? idx.ud : 0;
3475       brw_MOV(p, dst,
3476               (align1 ? stride(suboffset(src, i), 0, 1, 0) :
3477                stride(suboffset(src, 4 * i), 0, 4, 1)));
3478    } else {
3479       if (align1) {
3480          const struct brw_reg addr =
3481             retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD);
3482          const unsigned offset = src.nr * REG_SIZE + src.subnr;
3483          /* Limit in bytes of the signed indirect addressing immediate. */
3484          const unsigned limit = 512;
3485 
3486          brw_push_insn_state(p);
3487          brw_set_default_mask_control(p, BRW_MASK_DISABLE);
3488          brw_set_default_predicate_control(p, BRW_PREDICATE_NONE);
3489 
3490          /* Take into account the component size and horizontal stride. */
3491          assert(src.vstride == src.hstride + src.width);
3492          brw_SHL(p, addr, vec1(idx),
3493                  brw_imm_ud(_mesa_logbase2(type_sz(src.type)) +
3494                             src.hstride - 1));
3495 
3496          /* We can only address up to limit bytes using the indirect
3497           * addressing immediate, account for the difference if the source
3498           * register is above this limit.
3499           */
3500          if (offset >= limit)
3501             brw_ADD(p, addr, addr, brw_imm_ud(offset - offset % limit));
3502 
3503          brw_pop_insn_state(p);
3504 
3505          /* Use indirect addressing to fetch the specified component. */
3506          brw_MOV(p, dst,
3507                  retype(brw_vec1_indirect(addr.subnr, offset % limit),
3508                         src.type));
3509       } else {
3510          /* In SIMD4x2 mode the index can be either zero or one, replicate it
3511           * to all bits of a flag register,
3512           */
3513          inst = brw_MOV(p,
3514                         brw_null_reg(),
3515                         stride(brw_swizzle(idx, BRW_SWIZZLE_XXXX), 4, 4, 1));
3516          brw_inst_set_pred_control(devinfo, inst, BRW_PREDICATE_NONE);
3517          brw_inst_set_cond_modifier(devinfo, inst, BRW_CONDITIONAL_NZ);
3518          brw_inst_set_flag_reg_nr(devinfo, inst, 1);
3519 
3520          /* and use predicated SEL to pick the right channel. */
3521          inst = brw_SEL(p, dst,
3522                         stride(suboffset(src, 4), 4, 4, 1),
3523                         stride(src, 4, 4, 1));
3524          brw_inst_set_pred_control(devinfo, inst, BRW_PREDICATE_NORMAL);
3525          brw_inst_set_flag_reg_nr(devinfo, inst, 1);
3526       }
3527    }
3528 
3529    brw_pop_insn_state(p);
3530 }
3531 
3532 /**
3533  * This instruction is generated as a single-channel align1 instruction by
3534  * both the VS and FS stages when using INTEL_DEBUG=shader_time.
3535  *
3536  * We can't use the typed atomic op in the FS because that has the execution
3537  * mask ANDed with the pixel mask, but we just want to write the one dword for
3538  * all the pixels.
3539  *
3540  * We don't use the SIMD4x2 atomic ops in the VS because want to just write
3541  * one u32.  So we use the same untyped atomic write message as the pixel
3542  * shader.
3543  *
3544  * The untyped atomic operation requires a BUFFER surface type with RAW
3545  * format, and is only accessible through the legacy DATA_CACHE dataport
3546  * messages.
3547  */
brw_shader_time_add(struct brw_codegen * p,struct brw_reg payload,uint32_t surf_index)3548 void brw_shader_time_add(struct brw_codegen *p,
3549                          struct brw_reg payload,
3550                          uint32_t surf_index)
3551 {
3552    const unsigned sfid = (p->devinfo->gen >= 8 || p->devinfo->is_haswell ?
3553                           HSW_SFID_DATAPORT_DATA_CACHE_1 :
3554                           GEN7_SFID_DATAPORT_DATA_CACHE);
3555    assert(p->devinfo->gen >= 7);
3556 
3557    brw_push_insn_state(p);
3558    brw_set_default_access_mode(p, BRW_ALIGN_1);
3559    brw_set_default_mask_control(p, BRW_MASK_DISABLE);
3560    brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
3561    brw_inst *send = brw_next_insn(p, BRW_OPCODE_SEND);
3562 
3563    /* We use brw_vec1_reg and unmasked because we want to increment the given
3564     * offset only once.
3565     */
3566    brw_set_dest(p, send, brw_vec1_reg(BRW_ARCHITECTURE_REGISTER_FILE,
3567                                       BRW_ARF_NULL, 0));
3568    brw_set_src0(p, send, brw_vec1_reg(payload.file,
3569                                       payload.nr, 0));
3570    brw_set_src1(p, send, brw_imm_ud(0));
3571    brw_set_message_descriptor(p, send, sfid, 2, 0, false, false);
3572    brw_inst_set_binding_table_index(p->devinfo, send, surf_index);
3573    brw_set_dp_untyped_atomic_message(p, send, BRW_AOP_ADD, false);
3574 
3575    brw_pop_insn_state(p);
3576 }
3577 
3578 
3579 /**
3580  * Emit the SEND message for a barrier
3581  */
3582 void
brw_barrier(struct brw_codegen * p,struct brw_reg src)3583 brw_barrier(struct brw_codegen *p, struct brw_reg src)
3584 {
3585    const struct gen_device_info *devinfo = p->devinfo;
3586    struct brw_inst *inst;
3587 
3588    assert(devinfo->gen >= 7);
3589 
3590    brw_push_insn_state(p);
3591    brw_set_default_access_mode(p, BRW_ALIGN_1);
3592    inst = next_insn(p, BRW_OPCODE_SEND);
3593    brw_set_dest(p, inst, retype(brw_null_reg(), BRW_REGISTER_TYPE_UW));
3594    brw_set_src0(p, inst, src);
3595    brw_set_src1(p, inst, brw_null_reg());
3596 
3597    brw_set_message_descriptor(p, inst, BRW_SFID_MESSAGE_GATEWAY,
3598                               1 /* msg_length */,
3599                               0 /* response_length */,
3600                               false /* header_present */,
3601                               false /* end_of_thread */);
3602 
3603    brw_inst_set_gateway_notify(devinfo, inst, 1);
3604    brw_inst_set_gateway_subfuncid(devinfo, inst,
3605                                   BRW_MESSAGE_GATEWAY_SFID_BARRIER_MSG);
3606 
3607    brw_inst_set_mask_control(devinfo, inst, BRW_MASK_DISABLE);
3608    brw_pop_insn_state(p);
3609 }
3610 
3611 
3612 /**
3613  * Emit the wait instruction for a barrier
3614  */
3615 void
brw_WAIT(struct brw_codegen * p)3616 brw_WAIT(struct brw_codegen *p)
3617 {
3618    const struct gen_device_info *devinfo = p->devinfo;
3619    struct brw_inst *insn;
3620 
3621    struct brw_reg src = brw_notification_reg();
3622 
3623    insn = next_insn(p, BRW_OPCODE_WAIT);
3624    brw_set_dest(p, insn, src);
3625    brw_set_src0(p, insn, src);
3626    brw_set_src1(p, insn, brw_null_reg());
3627 
3628    brw_inst_set_exec_size(devinfo, insn, BRW_EXECUTE_1);
3629    brw_inst_set_mask_control(devinfo, insn, BRW_MASK_DISABLE);
3630 }
3631