1 /*
2 * Copyright © 2012 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 /** @file
25 *
26 * Support for global copy propagation in two passes: A local pass that does
27 * intra-block copy (and constant) propagation, and a global pass that uses
28 * dataflow analysis on the copies available at the end of each block to re-do
29 * local copy propagation with more copies available.
30 *
31 * See Muchnick's Advanced Compiler Design and Implementation, section
32 * 12.5 (p356).
33 */
34
35 #include "util/bitset.h"
36 #include "util/u_math.h"
37 #include "util/rb_tree.h"
38 #include "brw_fs.h"
39 #include "brw_fs_live_variables.h"
40 #include "brw_cfg.h"
41 #include "brw_eu.h"
42
43 using namespace brw;
44
45 namespace { /* avoid conflict with opt_copy_propagation_elements */
46 struct acp_entry {
47 struct rb_node by_dst;
48 struct rb_node by_src;
49 brw_reg dst;
50 brw_reg src;
51 unsigned global_idx;
52 unsigned size_written;
53 unsigned size_read;
54 enum opcode opcode;
55 bool is_partial_write;
56 bool force_writemask_all;
57 };
58
59 /**
60 * Compare two acp_entry::src.nr
61 *
62 * This is intended to be used as the comparison function for rb_tree.
63 */
64 static int
cmp_entry_dst_entry_dst(const struct rb_node * a_node,const struct rb_node * b_node)65 cmp_entry_dst_entry_dst(const struct rb_node *a_node, const struct rb_node *b_node)
66 {
67 const struct acp_entry *a_entry =
68 rb_node_data(struct acp_entry, a_node, by_dst);
69
70 const struct acp_entry *b_entry =
71 rb_node_data(struct acp_entry, b_node, by_dst);
72
73 return a_entry->dst.nr - b_entry->dst.nr;
74 }
75
76 static int
cmp_entry_dst_nr(const struct rb_node * a_node,const void * b_key)77 cmp_entry_dst_nr(const struct rb_node *a_node, const void *b_key)
78 {
79 const struct acp_entry *a_entry =
80 rb_node_data(struct acp_entry, a_node, by_dst);
81
82 return a_entry->dst.nr - (uintptr_t) b_key;
83 }
84
85 static int
cmp_entry_src_entry_src(const struct rb_node * a_node,const struct rb_node * b_node)86 cmp_entry_src_entry_src(const struct rb_node *a_node, const struct rb_node *b_node)
87 {
88 const struct acp_entry *a_entry =
89 rb_node_data(struct acp_entry, a_node, by_src);
90
91 const struct acp_entry *b_entry =
92 rb_node_data(struct acp_entry, b_node, by_src);
93
94 return a_entry->src.nr - b_entry->src.nr;
95 }
96
97 /**
98 * Compare an acp_entry::src.nr with a raw nr.
99 *
100 * This is intended to be used as the comparison function for rb_tree.
101 */
102 static int
cmp_entry_src_nr(const struct rb_node * a_node,const void * b_key)103 cmp_entry_src_nr(const struct rb_node *a_node, const void *b_key)
104 {
105 const struct acp_entry *a_entry =
106 rb_node_data(struct acp_entry, a_node, by_src);
107
108 return a_entry->src.nr - (uintptr_t) b_key;
109 }
110
111 class acp_forward_iterator {
112 public:
acp_forward_iterator(struct rb_node * n,unsigned offset)113 acp_forward_iterator(struct rb_node *n, unsigned offset)
114 : curr(n), next(nullptr), offset(offset)
115 {
116 next = rb_node_next_or_null(curr);
117 }
118
operator ++()119 acp_forward_iterator &operator++()
120 {
121 curr = next;
122 next = rb_node_next_or_null(curr);
123
124 return *this;
125 }
126
operator !=(const acp_forward_iterator & other) const127 bool operator!=(const acp_forward_iterator &other) const
128 {
129 return curr != other.curr;
130 }
131
operator *() const132 struct acp_entry *operator*() const
133 {
134 /* This open-codes part of rb_node_data. */
135 return curr != NULL ? (struct acp_entry *)(((char *)curr) - offset)
136 : NULL;
137 }
138
139 private:
140 struct rb_node *curr;
141 struct rb_node *next;
142 unsigned offset;
143 };
144
145 struct acp {
146 DECLARE_LINEAR_ALLOC_CXX_OPERATORS(acp);
147
148 struct rb_tree by_dst;
149 struct rb_tree by_src;
150
acp__anon7a832aa70111::acp151 acp()
152 {
153 rb_tree_init(&by_dst);
154 rb_tree_init(&by_src);
155 }
156
begin__anon7a832aa70111::acp157 acp_forward_iterator begin()
158 {
159 return acp_forward_iterator(rb_tree_first(&by_src),
160 rb_tree_offsetof(struct acp_entry, by_src, 0));
161 }
162
end__anon7a832aa70111::acp163 const acp_forward_iterator end() const
164 {
165 return acp_forward_iterator(nullptr, 0);
166 }
167
length__anon7a832aa70111::acp168 unsigned length()
169 {
170 unsigned l = 0;
171
172 for (rb_node *iter = rb_tree_first(&by_src);
173 iter != NULL; iter = rb_node_next(iter))
174 l++;
175
176 return l;
177 }
178
add__anon7a832aa70111::acp179 void add(acp_entry *entry)
180 {
181 rb_tree_insert(&by_dst, &entry->by_dst, cmp_entry_dst_entry_dst);
182 rb_tree_insert(&by_src, &entry->by_src, cmp_entry_src_entry_src);
183 }
184
remove__anon7a832aa70111::acp185 void remove(acp_entry *entry)
186 {
187 rb_tree_remove(&by_dst, &entry->by_dst);
188 rb_tree_remove(&by_src, &entry->by_src);
189 }
190
find_by_src__anon7a832aa70111::acp191 acp_forward_iterator find_by_src(unsigned nr)
192 {
193 struct rb_node *rbn = rb_tree_search(&by_src,
194 (void *)(uintptr_t) nr,
195 cmp_entry_src_nr);
196
197 return acp_forward_iterator(rbn, rb_tree_offsetof(struct acp_entry,
198 by_src, rbn));
199 }
200
find_by_dst__anon7a832aa70111::acp201 acp_forward_iterator find_by_dst(unsigned nr)
202 {
203 struct rb_node *rbn = rb_tree_search(&by_dst,
204 (void *)(uintptr_t) nr,
205 cmp_entry_dst_nr);
206
207 return acp_forward_iterator(rbn, rb_tree_offsetof(struct acp_entry,
208 by_dst, rbn));
209 }
210 };
211
212 struct block_data {
213 /**
214 * Which entries in the fs_copy_prop_dataflow acp table are live at the
215 * start of this block. This is the useful output of the analysis, since
216 * it lets us plug those into the local copy propagation on the second
217 * pass.
218 */
219 BITSET_WORD *livein;
220
221 /**
222 * Which entries in the fs_copy_prop_dataflow acp table are live at the end
223 * of this block. This is done in initial setup from the per-block acps
224 * returned by the first local copy prop pass.
225 */
226 BITSET_WORD *liveout;
227
228 /**
229 * Which entries in the fs_copy_prop_dataflow acp table are generated by
230 * instructions in this block which reach the end of the block without
231 * being killed.
232 */
233 BITSET_WORD *copy;
234
235 /**
236 * Which entries in the fs_copy_prop_dataflow acp table are killed over the
237 * course of this block.
238 */
239 BITSET_WORD *kill;
240
241 /**
242 * Which entries in the fs_copy_prop_dataflow acp table are guaranteed to
243 * have a fully uninitialized destination at the end of this block.
244 */
245 BITSET_WORD *undef;
246
247 /**
248 * Which entries in the fs_copy_prop_dataflow acp table can the
249 * start of this block be reached from. Note that this is a weaker
250 * condition than livein.
251 */
252 BITSET_WORD *reachin;
253
254 /**
255 * Which entries in the fs_copy_prop_dataflow acp table are
256 * overwritten by an instruction with channel masks inconsistent
257 * with the copy instruction (e.g. due to force_writemask_all).
258 * Such an overwrite can cause the copy entry to become invalid
259 * even if the copy instruction is subsequently re-executed for any
260 * given channel i, since the execution of the overwrite for
261 * channel i may corrupt other channels j!=i inactive for the
262 * subsequent copy.
263 */
264 BITSET_WORD *exec_mismatch;
265 };
266
267 class fs_copy_prop_dataflow
268 {
269 public:
270 fs_copy_prop_dataflow(linear_ctx *lin_ctx, cfg_t *cfg,
271 const fs_live_variables &live,
272 struct acp *out_acp);
273
274 void setup_initial_values();
275 void run();
276
277 void dump_block_data() const UNUSED;
278
279 cfg_t *cfg;
280 const fs_live_variables &live;
281
282 acp_entry **acp;
283 int num_acp;
284 int bitset_words;
285
286 struct block_data *bd;
287 };
288 } /* anonymous namespace */
289
fs_copy_prop_dataflow(linear_ctx * lin_ctx,cfg_t * cfg,const fs_live_variables & live,struct acp * out_acp)290 fs_copy_prop_dataflow::fs_copy_prop_dataflow(linear_ctx *lin_ctx, cfg_t *cfg,
291 const fs_live_variables &live,
292 struct acp *out_acp)
293 : cfg(cfg), live(live)
294 {
295 bd = linear_zalloc_array(lin_ctx, struct block_data, cfg->num_blocks);
296
297 num_acp = 0;
298 foreach_block (block, cfg)
299 num_acp += out_acp[block->num].length();
300
301 bitset_words = BITSET_WORDS(num_acp);
302
303 foreach_block (block, cfg) {
304 bd[block->num].livein = linear_zalloc_array(lin_ctx, BITSET_WORD, bitset_words);
305 bd[block->num].liveout = linear_zalloc_array(lin_ctx, BITSET_WORD, bitset_words);
306 bd[block->num].copy = linear_zalloc_array(lin_ctx, BITSET_WORD, bitset_words);
307 bd[block->num].kill = linear_zalloc_array(lin_ctx, BITSET_WORD, bitset_words);
308 bd[block->num].undef = linear_zalloc_array(lin_ctx, BITSET_WORD, bitset_words);
309 bd[block->num].reachin = linear_zalloc_array(lin_ctx, BITSET_WORD, bitset_words);
310 bd[block->num].exec_mismatch = linear_zalloc_array(lin_ctx, BITSET_WORD, bitset_words);
311 }
312
313 acp = linear_zalloc_array(lin_ctx, struct acp_entry *, num_acp);
314
315 int next_acp = 0;
316 foreach_block (block, cfg) {
317 for (auto iter = out_acp[block->num].begin();
318 iter != out_acp[block->num].end(); ++iter) {
319 acp[next_acp] = *iter;
320
321 (*iter)->global_idx = next_acp;
322
323 /* opt_copy_propagation_local populates out_acp with copies created
324 * in a block which are still live at the end of the block. This
325 * is exactly what we want in the COPY set.
326 */
327 BITSET_SET(bd[block->num].copy, next_acp);
328
329 next_acp++;
330 }
331 }
332
333 assert(next_acp == num_acp);
334
335 setup_initial_values();
336 run();
337 }
338
339 /**
340 * Like reg_offset, but register must be VGRF or FIXED_GRF.
341 */
342 static inline unsigned
grf_reg_offset(const brw_reg & r)343 grf_reg_offset(const brw_reg &r)
344 {
345 return (r.file == VGRF ? 0 : r.nr) * REG_SIZE +
346 r.offset +
347 (r.file == FIXED_GRF ? r.subnr : 0);
348 }
349
350 /**
351 * Like regions_overlap, but register must be VGRF or FIXED_GRF.
352 */
353 static inline bool
grf_regions_overlap(const brw_reg & r,unsigned dr,const brw_reg & s,unsigned ds)354 grf_regions_overlap(const brw_reg &r, unsigned dr, const brw_reg &s, unsigned ds)
355 {
356 return reg_space(r) == reg_space(s) &&
357 !(grf_reg_offset(r) + dr <= grf_reg_offset(s) ||
358 grf_reg_offset(s) + ds <= grf_reg_offset(r));
359 }
360
361 /**
362 * Set up initial values for each of the data flow sets, prior to running
363 * the fixed-point algorithm.
364 */
365 void
setup_initial_values()366 fs_copy_prop_dataflow::setup_initial_values()
367 {
368 /* Initialize the COPY and KILL sets. */
369 {
370 struct acp acp_table;
371
372 /* First, get all the KILLs for instructions which overwrite ACP
373 * destinations.
374 */
375 for (int i = 0; i < num_acp; i++)
376 acp_table.add(acp[i]);
377
378 foreach_block (block, cfg) {
379 foreach_inst_in_block(fs_inst, inst, block) {
380 if (inst->dst.file != VGRF &&
381 inst->dst.file != FIXED_GRF)
382 continue;
383
384 for (auto iter = acp_table.find_by_src(inst->dst.nr);
385 iter != acp_table.end() && (*iter)->src.nr == inst->dst.nr;
386 ++iter) {
387 if (grf_regions_overlap(inst->dst, inst->size_written,
388 (*iter)->src, (*iter)->size_read)) {
389 BITSET_SET(bd[block->num].kill, (*iter)->global_idx);
390 if (inst->force_writemask_all && !(*iter)->force_writemask_all)
391 BITSET_SET(bd[block->num].exec_mismatch, (*iter)->global_idx);
392 }
393 }
394
395 if (inst->dst.file != VGRF)
396 continue;
397
398 for (auto iter = acp_table.find_by_dst(inst->dst.nr);
399 iter != acp_table.end() && (*iter)->dst.nr == inst->dst.nr;
400 ++iter) {
401 if (grf_regions_overlap(inst->dst, inst->size_written,
402 (*iter)->dst, (*iter)->size_written)) {
403 BITSET_SET(bd[block->num].kill, (*iter)->global_idx);
404 if (inst->force_writemask_all && !(*iter)->force_writemask_all)
405 BITSET_SET(bd[block->num].exec_mismatch, (*iter)->global_idx);
406 }
407 }
408 }
409 }
410 }
411
412 /* Populate the initial values for the livein and liveout sets. For the
413 * block at the start of the program, livein = 0 and liveout = copy.
414 * For the others, set liveout and livein to ~0 (the universal set).
415 */
416 foreach_block (block, cfg) {
417 if (block->parents.is_empty()) {
418 for (int i = 0; i < bitset_words; i++) {
419 bd[block->num].livein[i] = 0u;
420 bd[block->num].liveout[i] = bd[block->num].copy[i];
421 }
422 } else {
423 for (int i = 0; i < bitset_words; i++) {
424 bd[block->num].liveout[i] = ~0u;
425 bd[block->num].livein[i] = ~0u;
426 }
427 }
428 }
429
430 /* Initialize the undef set. */
431 foreach_block (block, cfg) {
432 for (int i = 0; i < num_acp; i++) {
433 BITSET_SET(bd[block->num].undef, i);
434 for (unsigned off = 0; off < acp[i]->size_written; off += REG_SIZE) {
435 if (BITSET_TEST(live.block_data[block->num].defout,
436 live.var_from_reg(byte_offset(acp[i]->dst, off))))
437 BITSET_CLEAR(bd[block->num].undef, i);
438 }
439 }
440 }
441 }
442
443 /**
444 * Walk the set of instructions in the block, marking which entries in the acp
445 * are killed by the block.
446 */
447 void
run()448 fs_copy_prop_dataflow::run()
449 {
450 bool progress;
451
452 do {
453 progress = false;
454
455 foreach_block (block, cfg) {
456 if (block->parents.is_empty())
457 continue;
458
459 for (int i = 0; i < bitset_words; i++) {
460 const BITSET_WORD old_liveout = bd[block->num].liveout[i];
461 const BITSET_WORD old_reachin = bd[block->num].reachin[i];
462 BITSET_WORD livein_from_any_block = 0;
463
464 /* Update livein for this block. If a copy is live out of all
465 * parent blocks, it's live coming in to this block.
466 */
467 bd[block->num].livein[i] = ~0u;
468 foreach_list_typed(bblock_link, parent_link, link, &block->parents) {
469 bblock_t *parent = parent_link->block;
470 /* Consider ACP entries with a known-undefined destination to
471 * be available from the parent. This is valid because we're
472 * free to set the undefined variable equal to the source of
473 * the ACP entry without breaking the application's
474 * expectations, since the variable is undefined.
475 */
476 bd[block->num].livein[i] &= (bd[parent->num].liveout[i] |
477 bd[parent->num].undef[i]);
478 livein_from_any_block |= bd[parent->num].liveout[i];
479
480 /* Update reachin for this block. If the end of any
481 * parent block is reachable from the copy, the start
482 * of this block is reachable from it as well.
483 */
484 bd[block->num].reachin[i] |= (bd[parent->num].reachin[i] |
485 bd[parent->num].copy[i]);
486 }
487
488 /* Limit to the set of ACP entries that can possibly be available
489 * at the start of the block, since propagating from a variable
490 * which is guaranteed to be undefined (rather than potentially
491 * undefined for some dynamic control-flow paths) doesn't seem
492 * particularly useful.
493 */
494 bd[block->num].livein[i] &= livein_from_any_block;
495
496 /* Update liveout for this block. */
497 bd[block->num].liveout[i] =
498 bd[block->num].copy[i] | (bd[block->num].livein[i] &
499 ~bd[block->num].kill[i]);
500
501 if (old_liveout != bd[block->num].liveout[i] ||
502 old_reachin != bd[block->num].reachin[i])
503 progress = true;
504 }
505 }
506 } while (progress);
507
508 /* Perform a second fixed-point pass in order to propagate the
509 * exec_mismatch bitsets. Note that this requires an accurate
510 * value of the reachin bitsets as input, which isn't available
511 * until the end of the first propagation pass, so this loop cannot
512 * be folded into the previous one.
513 */
514 do {
515 progress = false;
516
517 foreach_block (block, cfg) {
518 for (int i = 0; i < bitset_words; i++) {
519 const BITSET_WORD old_exec_mismatch = bd[block->num].exec_mismatch[i];
520
521 /* Update exec_mismatch for this block. If the end of a
522 * parent block is reachable by an overwrite with
523 * inconsistent execution masking, the start of this block
524 * is reachable by such an overwrite as well.
525 */
526 foreach_list_typed(bblock_link, parent_link, link, &block->parents) {
527 bblock_t *parent = parent_link->block;
528 bd[block->num].exec_mismatch[i] |= (bd[parent->num].exec_mismatch[i] &
529 bd[parent->num].reachin[i]);
530 }
531
532 /* Only consider overwrites with inconsistent execution
533 * masking if they are reachable from the copy, since
534 * overwrites unreachable from a copy are harmless to that
535 * copy.
536 */
537 bd[block->num].exec_mismatch[i] &= bd[block->num].reachin[i];
538 if (old_exec_mismatch != bd[block->num].exec_mismatch[i])
539 progress = true;
540 }
541 }
542 } while (progress);
543 }
544
545 void
dump_block_data() const546 fs_copy_prop_dataflow::dump_block_data() const
547 {
548 foreach_block (block, cfg) {
549 fprintf(stderr, "Block %d [%d, %d] (parents ", block->num,
550 block->start_ip, block->end_ip);
551 foreach_list_typed(bblock_link, link, link, &block->parents) {
552 bblock_t *parent = link->block;
553 fprintf(stderr, "%d ", parent->num);
554 }
555 fprintf(stderr, "):\n");
556 fprintf(stderr, " livein = 0x");
557 for (int i = 0; i < bitset_words; i++)
558 fprintf(stderr, "%08x", bd[block->num].livein[i]);
559 fprintf(stderr, ", liveout = 0x");
560 for (int i = 0; i < bitset_words; i++)
561 fprintf(stderr, "%08x", bd[block->num].liveout[i]);
562 fprintf(stderr, ",\n copy = 0x");
563 for (int i = 0; i < bitset_words; i++)
564 fprintf(stderr, "%08x", bd[block->num].copy[i]);
565 fprintf(stderr, ", kill = 0x");
566 for (int i = 0; i < bitset_words; i++)
567 fprintf(stderr, "%08x", bd[block->num].kill[i]);
568 fprintf(stderr, "\n");
569 }
570 }
571
572 static bool
is_logic_op(enum opcode opcode)573 is_logic_op(enum opcode opcode)
574 {
575 return (opcode == BRW_OPCODE_AND ||
576 opcode == BRW_OPCODE_OR ||
577 opcode == BRW_OPCODE_XOR ||
578 opcode == BRW_OPCODE_NOT);
579 }
580
581 static bool
can_take_stride(fs_inst * inst,brw_reg_type dst_type,unsigned arg,unsigned stride,const struct brw_compiler * compiler)582 can_take_stride(fs_inst *inst, brw_reg_type dst_type,
583 unsigned arg, unsigned stride,
584 const struct brw_compiler *compiler)
585 {
586 const struct intel_device_info *devinfo = compiler->devinfo;
587
588 if (stride > 4)
589 return false;
590
591 /* Bail if the channels of the source need to be aligned to the byte offset
592 * of the corresponding channel of the destination, and the provided stride
593 * would break this restriction.
594 */
595 if (has_dst_aligned_region_restriction(devinfo, inst, dst_type) &&
596 !(brw_type_size_bytes(inst->src[arg].type) * stride ==
597 brw_type_size_bytes(dst_type) * inst->dst.stride ||
598 stride == 0))
599 return false;
600
601 /* 3-source instructions can only be Align16, which restricts what strides
602 * they can take. They can only take a stride of 1 (the usual case), or 0
603 * with a special "repctrl" bit. But the repctrl bit doesn't work for
604 * 64-bit datatypes, so if the source type is 64-bit then only a stride of
605 * 1 is allowed. From the Broadwell PRM, Volume 7 "3D Media GPGPU", page
606 * 944:
607 *
608 * This is applicable to 32b datatypes and 16b datatype. 64b datatypes
609 * cannot use the replicate control.
610 */
611 if (inst->is_3src(compiler)) {
612 if (brw_type_size_bytes(inst->src[arg].type) > 4)
613 return stride == 1;
614 else
615 return stride == 1 || stride == 0;
616 }
617
618 if (inst->is_math()) {
619 /* Wa_22016140776:
620 *
621 * Scalar broadcast on HF math (packed or unpacked) must not be used.
622 * Compiler must use a mov instruction to expand the scalar value to
623 * a vector before using in a HF (packed or unpacked) math operation.
624 *
625 * Prevent copy propagating a scalar value into a math instruction.
626 */
627 if (intel_needs_workaround(devinfo, 22016140776) &&
628 stride == 0 && inst->src[arg].type == BRW_TYPE_HF) {
629 return false;
630 }
631
632 /* From the Broadwell PRM, Volume 2a "Command Reference - Instructions",
633 * page 391 ("Extended Math Function"):
634 *
635 * The following restrictions apply for align1 mode: Scalar source
636 * is supported. Source and destination horizontal stride must be
637 * the same.
638 */
639 return stride == inst->dst.stride || stride == 0;
640 }
641
642 return true;
643 }
644
645 static bool
instruction_requires_packed_data(fs_inst * inst)646 instruction_requires_packed_data(fs_inst *inst)
647 {
648 switch (inst->opcode) {
649 case FS_OPCODE_DDX_FINE:
650 case FS_OPCODE_DDX_COARSE:
651 case FS_OPCODE_DDY_FINE:
652 case FS_OPCODE_DDY_COARSE:
653 case SHADER_OPCODE_QUAD_SWIZZLE:
654 case SHADER_OPCODE_QUAD_SWAP:
655 return true;
656 default:
657 return false;
658 }
659 }
660
661 static bool
try_copy_propagate(const brw_compiler * compiler,fs_inst * inst,acp_entry * entry,int arg,const brw::simple_allocator & alloc,uint8_t max_polygons)662 try_copy_propagate(const brw_compiler *compiler, fs_inst *inst,
663 acp_entry *entry, int arg,
664 const brw::simple_allocator &alloc,
665 uint8_t max_polygons)
666 {
667 if (inst->src[arg].file != VGRF)
668 return false;
669
670 const struct intel_device_info *devinfo = compiler->devinfo;
671
672 assert(entry->src.file == VGRF || entry->src.file == UNIFORM ||
673 entry->src.file == ATTR || entry->src.file == FIXED_GRF);
674
675 /* Avoid propagating a LOAD_PAYLOAD instruction into another if there is a
676 * good chance that we'll be able to eliminate the latter through register
677 * coalescing. If only part of the sources of the second LOAD_PAYLOAD can
678 * be simplified through copy propagation we would be making register
679 * coalescing impossible, ending up with unnecessary copies in the program.
680 * This is also the case for is_multi_copy_payload() copies that can only
681 * be coalesced when the instruction is lowered into a sequence of MOVs.
682 *
683 * Worse -- In cases where the ACP entry was the result of CSE combining
684 * multiple LOAD_PAYLOAD subexpressions, propagating the first LOAD_PAYLOAD
685 * into the second would undo the work of CSE, leading to an infinite
686 * optimization loop. Avoid this by detecting LOAD_PAYLOAD copies from CSE
687 * temporaries which should match is_coalescing_payload().
688 */
689 if (entry->opcode == SHADER_OPCODE_LOAD_PAYLOAD &&
690 (is_coalescing_payload(devinfo, alloc, inst) ||
691 is_multi_copy_payload(devinfo, inst)))
692 return false;
693
694 assert(entry->dst.file == VGRF);
695 if (inst->src[arg].nr != entry->dst.nr)
696 return false;
697
698 /* Bail if inst is reading a range that isn't contained in the range
699 * that entry is writing.
700 */
701 if (!region_contained_in(inst->src[arg], inst->size_read(devinfo, arg),
702 entry->dst, entry->size_written))
703 return false;
704
705 /* Send messages with EOT set are restricted to use g112-g127 (and we
706 * sometimes need g127 for other purposes), so avoid copy propagating
707 * anything that would make it impossible to satisfy that restriction.
708 */
709 if (inst->eot) {
710 /* Don't propagate things that are already pinned. */
711 if (entry->src.file != VGRF)
712 return false;
713
714 /* We might be propagating from a large register, while the SEND only
715 * is reading a portion of it (say the .A channel in an RGBA value).
716 * We need to pin both split SEND sources in g112-g126/127, so only
717 * allow this if the registers aren't too large.
718 */
719 if (inst->opcode == SHADER_OPCODE_SEND && inst->sources >= 4 &&
720 entry->src.file == VGRF) {
721 int other_src = arg == 2 ? 3 : 2;
722 unsigned other_size = inst->src[other_src].file == VGRF ?
723 alloc.sizes[inst->src[other_src].nr] :
724 inst->size_read(devinfo, other_src);
725 unsigned prop_src_size = alloc.sizes[entry->src.nr];
726 if (other_size + prop_src_size > 15)
727 return false;
728 }
729 }
730
731 /* we can't generally copy-propagate UD negations because we
732 * can end up accessing the resulting values as signed integers
733 * instead. See also resolve_ud_negate() and comment in
734 * brw_generator::generate_code.
735 */
736 if (entry->src.type == BRW_TYPE_UD &&
737 entry->src.negate)
738 return false;
739
740 bool has_source_modifiers = entry->src.abs || entry->src.negate;
741
742 if (has_source_modifiers && !inst->can_do_source_mods(devinfo))
743 return false;
744
745 /* Reject cases that would violate register regioning restrictions. */
746 if ((entry->src.file == UNIFORM || !entry->src.is_contiguous()) &&
747 (inst->is_send_from_grf() ||
748 inst->uses_indirect_addressing())) {
749 return false;
750 }
751
752 /* Some instructions implemented in the generator backend, such as
753 * derivatives, assume that their operands are packed so we can't
754 * generally propagate strided regions to them.
755 */
756 const unsigned entry_stride = (entry->src.file == FIXED_GRF ? 1 :
757 entry->src.stride);
758 if (instruction_requires_packed_data(inst) && entry_stride != 1)
759 return false;
760
761 const brw_reg_type dst_type = (has_source_modifiers &&
762 entry->dst.type != inst->src[arg].type) ?
763 entry->dst.type : inst->dst.type;
764
765 /* Bail if the result of composing both strides would exceed the
766 * hardware limit.
767 */
768 if (!can_take_stride(inst, dst_type, arg,
769 entry_stride * inst->src[arg].stride,
770 compiler))
771 return false;
772
773 /* From the Cherry Trail/Braswell PRMs, Volume 7: 3D Media GPGPU:
774 * EU Overview
775 * Register Region Restrictions
776 * Special Requirements for Handling Double Precision Data Types :
777 *
778 * "When source or destination datatype is 64b or operation is integer
779 * DWord multiply, regioning in Align1 must follow these rules:
780 *
781 * 1. Source and Destination horizontal stride must be aligned to the
782 * same qword.
783 * 2. Regioning must ensure Src.Vstride = Src.Width * Src.Hstride.
784 * 3. Source and Destination offset must be the same, except the case
785 * of scalar source."
786 *
787 * Most of this is already checked in can_take_stride(), we're only left
788 * with checking 3.
789 */
790 if (has_dst_aligned_region_restriction(devinfo, inst, dst_type) &&
791 entry_stride != 0 &&
792 (reg_offset(inst->dst) % (REG_SIZE * reg_unit(devinfo))) != (reg_offset(entry->src) % (REG_SIZE * reg_unit(devinfo))))
793 return false;
794
795 /*
796 * Bail if the composition of both regions would be affected by the Xe2+
797 * regioning restrictions that apply to integer types smaller than a dword.
798 * See BSpec #56640 for details.
799 */
800 const brw_reg tmp = horiz_stride(entry->src, inst->src[arg].stride);
801 if (has_subdword_integer_region_restriction(devinfo, inst, &tmp, 1))
802 return false;
803
804 /* The <8;8,0> regions used for FS attributes in multipolygon
805 * dispatch mode could violate regioning restrictions, don't copy
806 * propagate them in such cases.
807 */
808 if (entry->src.file == ATTR && max_polygons > 1 &&
809 (has_dst_aligned_region_restriction(devinfo, inst, dst_type) ||
810 instruction_requires_packed_data(inst) ||
811 (inst->is_3src(compiler) && arg == 2) ||
812 entry->dst.type != inst->src[arg].type))
813 return false;
814
815 /* Bail if the source FIXED_GRF region of the copy cannot be trivially
816 * composed with the source region of the instruction -- E.g. because the
817 * copy uses some extended stride greater than 4 not supported natively by
818 * the hardware as a horizontal stride, or because instruction compression
819 * could require us to use a vertical stride shorter than a GRF.
820 */
821 if (entry->src.file == FIXED_GRF &&
822 (inst->src[arg].stride > 4 ||
823 inst->dst.component_size(inst->exec_size) >
824 inst->src[arg].component_size(inst->exec_size)))
825 return false;
826
827 /* Bail if the instruction type is larger than the execution type of the
828 * copy, what implies that each channel is reading multiple channels of the
829 * destination of the copy, and simply replacing the sources would give a
830 * program with different semantics.
831 */
832 if (brw_type_size_bits(entry->dst.type) < brw_type_size_bits(inst->src[arg].type) ||
833 (entry->is_partial_write && inst->opcode != BRW_OPCODE_MOV)) {
834 return false;
835 }
836
837 /* Bail if the result of composing both strides cannot be expressed
838 * as another stride. This avoids, for example, trying to transform
839 * this:
840 *
841 * MOV (8) rX<1>UD rY<0;1,0>UD
842 * FOO (8) ... rX<8;8,1>UW
843 *
844 * into this:
845 *
846 * FOO (8) ... rY<0;1,0>UW
847 *
848 * Which would have different semantics.
849 */
850 if (entry_stride != 1 &&
851 (inst->src[arg].stride *
852 brw_type_size_bytes(inst->src[arg].type)) % brw_type_size_bytes(entry->src.type) != 0)
853 return false;
854
855 /* Since semantics of source modifiers are type-dependent we need to
856 * ensure that the meaning of the instruction remains the same if we
857 * change the type. If the sizes of the types are different the new
858 * instruction will read a different amount of data than the original
859 * and the semantics will always be different.
860 */
861 if (has_source_modifiers &&
862 entry->dst.type != inst->src[arg].type &&
863 (!inst->can_change_types() ||
864 brw_type_size_bits(entry->dst.type) != brw_type_size_bits(inst->src[arg].type)))
865 return false;
866
867 if ((entry->src.negate || entry->src.abs) &&
868 is_logic_op(inst->opcode)) {
869 return false;
870 }
871
872 /* Save the offset of inst->src[arg] relative to entry->dst for it to be
873 * applied later.
874 */
875 const unsigned rel_offset = inst->src[arg].offset - entry->dst.offset;
876
877 /* Fold the copy into the instruction consuming it. */
878 inst->src[arg].file = entry->src.file;
879 inst->src[arg].nr = entry->src.nr;
880 inst->src[arg].subnr = entry->src.subnr;
881 inst->src[arg].offset = entry->src.offset;
882
883 /* Compose the strides of both regions. */
884 if (entry->src.file == FIXED_GRF) {
885 if (inst->src[arg].stride) {
886 const unsigned orig_width = 1 << entry->src.width;
887 const unsigned reg_width =
888 REG_SIZE / (brw_type_size_bytes(inst->src[arg].type) *
889 inst->src[arg].stride);
890 inst->src[arg].width = cvt(MIN2(orig_width, reg_width)) - 1;
891 inst->src[arg].hstride = cvt(inst->src[arg].stride);
892 inst->src[arg].vstride = inst->src[arg].hstride + inst->src[arg].width;
893 } else {
894 inst->src[arg].vstride = inst->src[arg].hstride =
895 inst->src[arg].width = 0;
896 }
897
898 inst->src[arg].stride = 1;
899
900 /* Hopefully no Align16 around here... */
901 assert(entry->src.swizzle == BRW_SWIZZLE_XYZW);
902 inst->src[arg].swizzle = entry->src.swizzle;
903 } else {
904 inst->src[arg].stride *= entry->src.stride;
905 }
906
907 /* Compute the first component of the copy that the instruction is
908 * reading, and the base byte offset within that component.
909 */
910 assert(entry->dst.stride == 1);
911 const unsigned component = rel_offset / brw_type_size_bytes(entry->dst.type);
912 const unsigned suboffset = rel_offset % brw_type_size_bytes(entry->dst.type);
913
914 /* Calculate the byte offset at the origin of the copy of the given
915 * component and suboffset.
916 */
917 inst->src[arg] = byte_offset(inst->src[arg],
918 component * entry_stride * brw_type_size_bytes(entry->src.type) + suboffset);
919
920 if (has_source_modifiers) {
921 if (entry->dst.type != inst->src[arg].type) {
922 /* We are propagating source modifiers from a MOV with a different
923 * type. If we got here, then we can just change the source and
924 * destination types of the instruction and keep going.
925 */
926 for (int i = 0; i < inst->sources; i++) {
927 inst->src[i].type = entry->dst.type;
928 }
929 inst->dst.type = entry->dst.type;
930 }
931
932 if (!inst->src[arg].abs) {
933 inst->src[arg].abs = entry->src.abs;
934 inst->src[arg].negate ^= entry->src.negate;
935 }
936 }
937
938 return true;
939 }
940
941 static bool
try_constant_propagate_value(brw_reg val,brw_reg_type dst_type,fs_inst * inst,int arg)942 try_constant_propagate_value(brw_reg val, brw_reg_type dst_type,
943 fs_inst *inst, int arg)
944 {
945 bool progress = false;
946
947 if (brw_type_size_bytes(val.type) > 4)
948 return false;
949
950 /* If the size of the use type is smaller than the size of the entry,
951 * clamp the value to the range of the use type. This enables constant
952 * copy propagation in cases like
953 *
954 *
955 * mov(8) g12<1>UD 0x0000000cUD
956 * ...
957 * mul(8) g47<1>D g86<8,8,1>D g12<16,8,2>W
958 */
959 if (brw_type_size_bits(inst->src[arg].type) <
960 brw_type_size_bits(dst_type)) {
961 if (brw_type_size_bytes(inst->src[arg].type) != 2 ||
962 brw_type_size_bytes(dst_type) != 4)
963 return false;
964
965 assert(inst->src[arg].subnr == 0 || inst->src[arg].subnr == 2);
966
967 /* When subnr is 0, we want the lower 16-bits, and when it's 2, we
968 * want the upper 16-bits. No other values of subnr are valid for a
969 * UD source.
970 */
971 const uint16_t v = inst->src[arg].subnr == 2 ? val.ud >> 16 : val.ud;
972
973 val.ud = v | (uint32_t(v) << 16);
974 }
975
976 val.type = inst->src[arg].type;
977
978 if (inst->src[arg].abs) {
979 if (is_logic_op(inst->opcode) ||
980 !brw_reg_abs_immediate(&val)) {
981 return false;
982 }
983 }
984
985 if (inst->src[arg].negate) {
986 if (is_logic_op(inst->opcode) ||
987 !brw_reg_negate_immediate(&val)) {
988 return false;
989 }
990 }
991
992 switch (inst->opcode) {
993 case BRW_OPCODE_MOV:
994 case SHADER_OPCODE_LOAD_PAYLOAD:
995 case SHADER_OPCODE_POW:
996 case FS_OPCODE_PACK:
997 inst->src[arg] = val;
998 progress = true;
999 break;
1000
1001 case BRW_OPCODE_SUBB:
1002 if (arg == 1) {
1003 inst->src[arg] = val;
1004 progress = true;
1005 }
1006 break;
1007
1008 case BRW_OPCODE_MACH:
1009 case BRW_OPCODE_MUL:
1010 case SHADER_OPCODE_MULH:
1011 case BRW_OPCODE_ADD:
1012 case BRW_OPCODE_XOR:
1013 case BRW_OPCODE_ADDC:
1014 if (arg == 1) {
1015 inst->src[arg] = val;
1016 progress = true;
1017 } else if (arg == 0 && inst->src[1].file != IMM) {
1018 /* We used to not copy propagate the constant in situations like
1019 *
1020 * mov(8) g8<1>D 0x7fffffffD
1021 * mul(8) g16<1>D g8<8,8,1>D g15<16,8,2>W
1022 *
1023 * On platforms that only have a 32x16 multiplier, this would
1024 * result in lowering the multiply to
1025 *
1026 * mul(8) g15<1>D g14<8,8,1>D 0xffffUW
1027 * mul(8) g16<1>D g14<8,8,1>D 0x7fffUW
1028 * add(8) g15.1<2>UW g15.1<16,8,2>UW g16<16,8,2>UW
1029 *
1030 * On Gfx8 and Gfx9, which have the full 32x32 multiplier, it
1031 * would results in
1032 *
1033 * mul(8) g16<1>D g15<16,8,2>W 0x7fffffffD
1034 *
1035 * Volume 2a of the Skylake PRM says:
1036 *
1037 * When multiplying a DW and any lower precision integer, the
1038 * DW operand must on src0.
1039 *
1040 * So it would have been invalid. However, brw_fs_combine_constants
1041 * will now "fix" the constant.
1042 */
1043 if (inst->opcode == BRW_OPCODE_MUL &&
1044 brw_type_size_bytes(inst->src[1].type) < 4 &&
1045 (inst->src[0].type == BRW_TYPE_D ||
1046 inst->src[0].type == BRW_TYPE_UD)) {
1047 inst->src[0] = val;
1048 inst->src[0].type = BRW_TYPE_D;
1049 progress = true;
1050 break;
1051 }
1052
1053 /* Fit this constant in by commuting the operands.
1054 * Exception: we can't do this for 32-bit integer MUL/MACH
1055 * because it's asymmetric.
1056 *
1057 * The BSpec says for Broadwell that
1058 *
1059 * "When multiplying DW x DW, the dst cannot be accumulator."
1060 *
1061 * Integer MUL with a non-accumulator destination will be lowered
1062 * by lower_integer_multiplication(), so don't restrict it.
1063 */
1064 if (((inst->opcode == BRW_OPCODE_MUL &&
1065 inst->dst.is_accumulator()) ||
1066 inst->opcode == BRW_OPCODE_MACH) &&
1067 (inst->src[1].type == BRW_TYPE_D ||
1068 inst->src[1].type == BRW_TYPE_UD))
1069 break;
1070 inst->src[0] = inst->src[1];
1071 inst->src[1] = val;
1072 progress = true;
1073 }
1074 break;
1075
1076 case BRW_OPCODE_CMP:
1077 if (arg == 1) {
1078 inst->src[arg] = val;
1079 progress = true;
1080 } else if (arg == 0 && inst->src[1].file != IMM) {
1081 enum brw_conditional_mod new_cmod;
1082
1083 new_cmod = brw_swap_cmod(inst->conditional_mod);
1084 if (new_cmod != BRW_CONDITIONAL_NONE) {
1085 /* Fit this constant in by swapping the operands and
1086 * flipping the test
1087 */
1088 inst->src[0] = inst->src[1];
1089 inst->src[1] = val;
1090 inst->conditional_mod = new_cmod;
1091 progress = true;
1092 }
1093 }
1094 break;
1095
1096 case BRW_OPCODE_SEL:
1097 if (arg == 1) {
1098 inst->src[arg] = val;
1099 progress = true;
1100 } else if (arg == 0) {
1101 if (inst->src[1].file != IMM &&
1102 (inst->conditional_mod == BRW_CONDITIONAL_NONE ||
1103 /* Only GE and L are commutative. */
1104 inst->conditional_mod == BRW_CONDITIONAL_GE ||
1105 inst->conditional_mod == BRW_CONDITIONAL_L)) {
1106 inst->src[0] = inst->src[1];
1107 inst->src[1] = val;
1108
1109 /* If this was predicated, flipping operands means
1110 * we also need to flip the predicate.
1111 */
1112 if (inst->conditional_mod == BRW_CONDITIONAL_NONE) {
1113 inst->predicate_inverse =
1114 !inst->predicate_inverse;
1115 }
1116 } else {
1117 inst->src[0] = val;
1118 }
1119
1120 progress = true;
1121 }
1122 break;
1123
1124 case BRW_OPCODE_CSEL:
1125 assert(inst->conditional_mod != BRW_CONDITIONAL_NONE);
1126
1127 if (arg == 0 &&
1128 inst->src[1].file != IMM &&
1129 (!brw_type_is_float(inst->src[1].type) ||
1130 inst->conditional_mod == BRW_CONDITIONAL_NZ ||
1131 inst->conditional_mod == BRW_CONDITIONAL_Z)) {
1132 /* Only EQ and NE are commutative due to NaN issues. */
1133 inst->src[0] = inst->src[1];
1134 inst->src[1] = val;
1135 inst->conditional_mod = brw_negate_cmod(inst->conditional_mod);
1136 } else {
1137 /* While CSEL is a 3-source instruction, the last source should never
1138 * be a constant. We'll support that, but should it ever happen, we
1139 * should add support to the constant folding pass.
1140 */
1141 inst->src[arg] = val;
1142 }
1143
1144 progress = true;
1145 break;
1146
1147 case FS_OPCODE_FB_WRITE_LOGICAL:
1148 /* The stencil and omask sources of FS_OPCODE_FB_WRITE_LOGICAL are
1149 * bit-cast using a strided region so they cannot be immediates.
1150 */
1151 if (arg != FB_WRITE_LOGICAL_SRC_SRC_STENCIL &&
1152 arg != FB_WRITE_LOGICAL_SRC_OMASK) {
1153 inst->src[arg] = val;
1154 progress = true;
1155 }
1156 break;
1157
1158 case SHADER_OPCODE_INT_QUOTIENT:
1159 case SHADER_OPCODE_INT_REMAINDER:
1160 case BRW_OPCODE_ADD3:
1161 case BRW_OPCODE_AND:
1162 case BRW_OPCODE_ASR:
1163 case BRW_OPCODE_BFE:
1164 case BRW_OPCODE_BFI1:
1165 case BRW_OPCODE_BFI2:
1166 case BRW_OPCODE_ROL:
1167 case BRW_OPCODE_ROR:
1168 case BRW_OPCODE_SHL:
1169 case BRW_OPCODE_SHR:
1170 case BRW_OPCODE_OR:
1171 case SHADER_OPCODE_TEX_LOGICAL:
1172 case SHADER_OPCODE_TXD_LOGICAL:
1173 case SHADER_OPCODE_TXF_LOGICAL:
1174 case SHADER_OPCODE_TXL_LOGICAL:
1175 case SHADER_OPCODE_TXS_LOGICAL:
1176 case FS_OPCODE_TXB_LOGICAL:
1177 case SHADER_OPCODE_TXF_CMS_W_LOGICAL:
1178 case SHADER_OPCODE_TXF_CMS_W_GFX12_LOGICAL:
1179 case SHADER_OPCODE_TXF_MCS_LOGICAL:
1180 case SHADER_OPCODE_LOD_LOGICAL:
1181 case SHADER_OPCODE_TG4_BIAS_LOGICAL:
1182 case SHADER_OPCODE_TG4_EXPLICIT_LOD_LOGICAL:
1183 case SHADER_OPCODE_TG4_IMPLICIT_LOD_LOGICAL:
1184 case SHADER_OPCODE_TG4_LOGICAL:
1185 case SHADER_OPCODE_TG4_OFFSET_LOGICAL:
1186 case SHADER_OPCODE_TG4_OFFSET_LOD_LOGICAL:
1187 case SHADER_OPCODE_TG4_OFFSET_BIAS_LOGICAL:
1188 case SHADER_OPCODE_SAMPLEINFO_LOGICAL:
1189 case SHADER_OPCODE_IMAGE_SIZE_LOGICAL:
1190 case SHADER_OPCODE_MEMORY_LOAD_LOGICAL:
1191 case SHADER_OPCODE_MEMORY_STORE_LOGICAL:
1192 case SHADER_OPCODE_MEMORY_ATOMIC_LOGICAL:
1193 case FS_OPCODE_UNIFORM_PULL_CONSTANT_LOAD:
1194 case FS_OPCODE_VARYING_PULL_CONSTANT_LOAD_LOGICAL:
1195 case SHADER_OPCODE_BROADCAST:
1196 case BRW_OPCODE_MAD:
1197 case BRW_OPCODE_LRP:
1198 case FS_OPCODE_PACK_HALF_2x16_SPLIT:
1199 case SHADER_OPCODE_SHUFFLE:
1200 case SHADER_OPCODE_BALLOT:
1201 inst->src[arg] = val;
1202 progress = true;
1203 break;
1204
1205 default:
1206 break;
1207 }
1208
1209 return progress;
1210 }
1211
1212
1213 static bool
try_constant_propagate(const struct intel_device_info * devinfo,fs_inst * inst,acp_entry * entry,int arg)1214 try_constant_propagate(const struct intel_device_info *devinfo,
1215 fs_inst *inst, acp_entry *entry, int arg)
1216 {
1217 if (inst->src[arg].file != VGRF)
1218 return false;
1219
1220 assert(entry->dst.file == VGRF);
1221 if (inst->src[arg].nr != entry->dst.nr)
1222 return false;
1223
1224 /* Bail if inst is reading a range that isn't contained in the range
1225 * that entry is writing.
1226 */
1227 if (!region_contained_in(inst->src[arg], inst->size_read(devinfo, arg),
1228 entry->dst, entry->size_written))
1229 return false;
1230
1231 /* If the size of the use type is larger than the size of the entry
1232 * type, the entry doesn't contain all of the data that the user is
1233 * trying to use.
1234 */
1235 if (brw_type_size_bits(inst->src[arg].type) >
1236 brw_type_size_bits(entry->dst.type))
1237 return false;
1238
1239 return try_constant_propagate_value(entry->src, entry->dst.type, inst, arg);
1240 }
1241
1242 static bool
can_propagate_from(const struct intel_device_info * devinfo,fs_inst * inst)1243 can_propagate_from(const struct intel_device_info *devinfo, fs_inst *inst)
1244 {
1245 return (inst->opcode == BRW_OPCODE_MOV &&
1246 inst->dst.file == VGRF &&
1247 ((inst->src[0].file == VGRF &&
1248 !grf_regions_overlap(inst->dst, inst->size_written,
1249 inst->src[0], inst->size_read(devinfo, 0))) ||
1250 inst->src[0].file == ATTR ||
1251 inst->src[0].file == UNIFORM ||
1252 inst->src[0].file == IMM ||
1253 (inst->src[0].file == FIXED_GRF &&
1254 inst->src[0].is_contiguous())) &&
1255 /* is_raw_move also rejects source modifiers, but copy propagation
1256 * can handle that if the types are the same.
1257 */
1258 ((inst->src[0].type == inst->dst.type &&
1259 !inst->saturate) ||
1260 inst->is_raw_move()) &&
1261 /* Subset of !is_partial_write() conditions. */
1262 !inst->predicate && inst->dst.is_contiguous()) ||
1263 is_identity_payload(devinfo, FIXED_GRF, inst);
1264 }
1265
1266 static void
swap_srcs(fs_inst * inst,unsigned a,unsigned b)1267 swap_srcs(fs_inst *inst, unsigned a, unsigned b)
1268 {
1269 const auto tmp = inst->src[a];
1270 inst->src[a] = inst->src[b];
1271 inst->src[b] = tmp;
1272 }
1273
1274 static void
commute_immediates(fs_inst * inst)1275 commute_immediates(fs_inst *inst)
1276 {
1277 /* ADD3 can have the immediate as src0 or src2. Using one or the other
1278 * consistently makes assembly dumps more readable, so we arbitrarily
1279 * prefer src0.
1280 */
1281 if (inst->opcode == BRW_OPCODE_ADD3) {
1282 if (inst->src[1].file == IMM) {
1283 if (inst->src[0].file != IMM)
1284 swap_srcs(inst, 0, 1);
1285 else if (inst->src[2].file != IMM)
1286 swap_srcs(inst, 1, 2);
1287 }
1288 }
1289
1290 /* MAD can only have mutliplicand immediate in src2. */
1291 if (inst->opcode == BRW_OPCODE_MAD) {
1292 if (inst->src[1].file == IMM && inst->src[2].file != IMM)
1293 swap_srcs(inst, 1, 2);
1294 }
1295
1296 /* If only one of the sources of a 2-source, commutative instruction (e.g.,
1297 * AND) is immediate, it must be src1. If both are immediate, opt_algebraic
1298 * should fold it away.
1299 */
1300 if (inst->sources == 2 && inst->is_commutative() &&
1301 inst->src[0].file == IMM && inst->src[1].file != IMM) {
1302 const auto src1 = inst->src[1];
1303 inst->src[1] = inst->src[0];
1304 inst->src[0] = src1;
1305 }
1306 }
1307
1308 /* Walks a basic block and does copy propagation on it using the acp
1309 * list.
1310 */
1311 static bool
opt_copy_propagation_local(const brw_compiler * compiler,linear_ctx * lin_ctx,bblock_t * block,struct acp & acp,const brw::simple_allocator & alloc,uint8_t max_polygons)1312 opt_copy_propagation_local(const brw_compiler *compiler, linear_ctx *lin_ctx,
1313 bblock_t *block, struct acp &acp,
1314 const brw::simple_allocator &alloc,
1315 uint8_t max_polygons)
1316 {
1317 const struct intel_device_info *devinfo = compiler->devinfo;
1318 bool progress = false;
1319
1320 foreach_inst_in_block(fs_inst, inst, block) {
1321 /* Try propagating into this instruction. */
1322 bool constant_progress = false;
1323 for (int i = inst->sources - 1; i >= 0; i--) {
1324 if (inst->src[i].file != VGRF)
1325 continue;
1326
1327 for (auto iter = acp.find_by_dst(inst->src[i].nr);
1328 iter != acp.end() && (*iter)->dst.nr == inst->src[i].nr;
1329 ++iter) {
1330 if ((*iter)->src.file == IMM) {
1331 if (try_constant_propagate(devinfo, inst, *iter, i)) {
1332 constant_progress = true;
1333 break;
1334 }
1335 } else {
1336 if (try_copy_propagate(compiler, inst, *iter, i, alloc,
1337 max_polygons)) {
1338 progress = true;
1339 break;
1340 }
1341 }
1342 }
1343 }
1344
1345 if (constant_progress) {
1346 commute_immediates(inst);
1347 brw_opt_constant_fold_instruction(compiler->devinfo, inst);
1348 progress = true;
1349 }
1350
1351 /* kill the destination from the ACP */
1352 if (inst->dst.file == VGRF || inst->dst.file == FIXED_GRF) {
1353 for (auto iter = acp.find_by_dst(inst->dst.nr);
1354 iter != acp.end() && (*iter)->dst.nr == inst->dst.nr;
1355 ++iter) {
1356 if (grf_regions_overlap((*iter)->dst, (*iter)->size_written,
1357 inst->dst, inst->size_written))
1358 acp.remove(*iter);
1359 }
1360
1361 for (auto iter = acp.find_by_src(inst->dst.nr);
1362 iter != acp.end() && (*iter)->src.nr == inst->dst.nr;
1363 ++iter) {
1364 /* Make sure we kill the entry if this instruction overwrites
1365 * _any_ of the registers that it reads
1366 */
1367 if (grf_regions_overlap((*iter)->src, (*iter)->size_read,
1368 inst->dst, inst->size_written))
1369 acp.remove(*iter);
1370 }
1371 }
1372
1373 /* If this instruction's source could potentially be folded into the
1374 * operand of another instruction, add it to the ACP.
1375 */
1376 if (can_propagate_from(devinfo, inst)) {
1377 acp_entry *entry = linear_zalloc(lin_ctx, acp_entry);
1378 entry->dst = inst->dst;
1379 entry->src = inst->src[0];
1380 entry->size_written = inst->size_written;
1381 for (unsigned i = 0; i < inst->sources; i++)
1382 entry->size_read += inst->size_read(devinfo, i);
1383 entry->opcode = inst->opcode;
1384 entry->is_partial_write = inst->is_partial_write();
1385 entry->force_writemask_all = inst->force_writemask_all;
1386 acp.add(entry);
1387 } else if (inst->opcode == SHADER_OPCODE_LOAD_PAYLOAD &&
1388 inst->dst.file == VGRF) {
1389 int offset = 0;
1390 for (int i = 0; i < inst->sources; i++) {
1391 int effective_width = i < inst->header_size ? 8 : inst->exec_size;
1392 const unsigned size_written =
1393 effective_width * brw_type_size_bytes(inst->src[i].type);
1394 if (inst->src[i].file == VGRF ||
1395 (inst->src[i].file == FIXED_GRF &&
1396 inst->src[i].is_contiguous())) {
1397 const brw_reg_type t = i < inst->header_size ?
1398 BRW_TYPE_UD : inst->src[i].type;
1399 brw_reg dst = byte_offset(retype(inst->dst, t), offset);
1400 if (!dst.equals(inst->src[i])) {
1401 acp_entry *entry = linear_zalloc(lin_ctx, acp_entry);
1402 entry->dst = dst;
1403 entry->src = retype(inst->src[i], t);
1404 entry->size_written = size_written;
1405 entry->size_read = inst->size_read(devinfo, i);
1406 entry->opcode = inst->opcode;
1407 entry->force_writemask_all = inst->force_writemask_all;
1408 acp.add(entry);
1409 }
1410 }
1411 offset += size_written;
1412 }
1413 }
1414 }
1415
1416 return progress;
1417 }
1418
1419 bool
brw_opt_copy_propagation(fs_visitor & s)1420 brw_opt_copy_propagation(fs_visitor &s)
1421 {
1422 bool progress = false;
1423 void *copy_prop_ctx = ralloc_context(NULL);
1424 linear_ctx *lin_ctx = linear_context(copy_prop_ctx);
1425 struct acp *out_acp = new (lin_ctx) acp[s.cfg->num_blocks];
1426
1427 const fs_live_variables &live = s.live_analysis.require();
1428
1429 /* First, walk through each block doing local copy propagation and getting
1430 * the set of copies available at the end of the block.
1431 */
1432 foreach_block (block, s.cfg) {
1433 progress = opt_copy_propagation_local(s.compiler, lin_ctx, block,
1434 out_acp[block->num], s.alloc,
1435 s.max_polygons) || progress;
1436
1437 /* If the destination of an ACP entry exists only within this block,
1438 * then there's no need to keep it for dataflow analysis. We can delete
1439 * it from the out_acp table and avoid growing the bitsets any bigger
1440 * than we absolutely have to.
1441 *
1442 * Because nothing in opt_copy_propagation_local touches the block
1443 * start/end IPs and opt_copy_propagation_local is incapable of
1444 * extending the live range of an ACP destination beyond the block,
1445 * it's safe to use the liveness information in this way.
1446 */
1447 for (auto iter = out_acp[block->num].begin();
1448 iter != out_acp[block->num].end(); ++iter) {
1449 assert((*iter)->dst.file == VGRF);
1450 if (block->start_ip <= live.vgrf_start[(*iter)->dst.nr] &&
1451 live.vgrf_end[(*iter)->dst.nr] <= block->end_ip) {
1452 out_acp[block->num].remove(*iter);
1453 }
1454 }
1455 }
1456
1457 /* Do dataflow analysis for those available copies. */
1458 fs_copy_prop_dataflow dataflow(lin_ctx, s.cfg, live, out_acp);
1459
1460 /* Next, re-run local copy propagation, this time with the set of copies
1461 * provided by the dataflow analysis available at the start of a block.
1462 */
1463 foreach_block (block, s.cfg) {
1464 struct acp in_acp;
1465
1466 for (int i = 0; i < dataflow.num_acp; i++) {
1467 if (BITSET_TEST(dataflow.bd[block->num].livein, i) &&
1468 !BITSET_TEST(dataflow.bd[block->num].exec_mismatch, i)) {
1469 struct acp_entry *entry = dataflow.acp[i];
1470 in_acp.add(entry);
1471 }
1472 }
1473
1474 progress = opt_copy_propagation_local(s.compiler, lin_ctx, block,
1475 in_acp, s.alloc, s.max_polygons) ||
1476 progress;
1477 }
1478
1479 ralloc_free(copy_prop_ctx);
1480
1481 if (progress)
1482 s.invalidate_analysis(DEPENDENCY_INSTRUCTION_DATA_FLOW |
1483 DEPENDENCY_INSTRUCTION_DETAIL);
1484
1485 return progress;
1486 }
1487
1488 static bool
try_copy_propagate_def(const brw_compiler * compiler,const brw::simple_allocator & alloc,fs_inst * def,const brw_reg & val,fs_inst * inst,int arg,uint8_t max_polygons)1489 try_copy_propagate_def(const brw_compiler *compiler,
1490 const brw::simple_allocator &alloc,
1491 fs_inst *def, const brw_reg &val,
1492 fs_inst *inst, int arg,
1493 uint8_t max_polygons)
1494 {
1495 const struct intel_device_info *devinfo = compiler->devinfo;
1496
1497 assert(val.file != BAD_FILE);
1498
1499 /* We can't generally copy-propagate UD negations because we can end up
1500 * accessing the resulting values as signed integers instead.
1501 */
1502 if (val.negate && val.type == BRW_TYPE_UD)
1503 return false;
1504
1505 /* Bail if the instruction type is larger than the execution type of the
1506 * copy, what implies that each channel is reading multiple channels of the
1507 * destination of the copy, and simply replacing the sources would give a
1508 * program with different semantics.
1509 */
1510 if (brw_type_size_bits(def->dst.type) <
1511 brw_type_size_bits(inst->src[arg].type))
1512 return false;
1513
1514 const bool has_source_modifiers = val.abs || val.negate;
1515
1516 if (has_source_modifiers) {
1517 if (is_logic_op(inst->opcode) || !inst->can_do_source_mods(devinfo))
1518 return false;
1519
1520 /* Since semantics of source modifiers are type-dependent we need to
1521 * ensure that the meaning of the instruction remains the same if we
1522 * change the type. If the sizes of the types are different the new
1523 * instruction will read a different amount of data than the original
1524 * and the semantics will always be different.
1525 */
1526 if (def->dst.type != inst->src[arg].type &&
1527 (!inst->can_change_types() ||
1528 brw_type_size_bits(def->dst.type) !=
1529 brw_type_size_bits(inst->src[arg].type)))
1530 return false;
1531 }
1532
1533 /* Send messages with EOT set are restricted to use g112-g127 (and we
1534 * sometimes need g127 for other purposes), so avoid copy propagating
1535 * anything that would make it impossible to satisfy that restriction.
1536 */
1537 if (inst->eot) {
1538 /* Don't propagate things that are already pinned. */
1539 if (val.file != VGRF)
1540 return false;
1541
1542 /* We might be propagating from a large register, while the SEND only
1543 * is reading a portion of it (say the .A channel in an RGBA value).
1544 * We need to pin both split SEND sources in g112-g126/127, so only
1545 * allow this if the registers aren't too large.
1546 */
1547 if (inst->opcode == SHADER_OPCODE_SEND && inst->sources >= 4 &&
1548 val.file == VGRF) {
1549 int other_src = arg == 2 ? 3 : 2;
1550 unsigned other_size = inst->src[other_src].file == VGRF ?
1551 alloc.sizes[inst->src[other_src].nr] :
1552 inst->size_read(devinfo, other_src);
1553 unsigned prop_src_size = alloc.sizes[val.nr];
1554 if (other_size + prop_src_size > 15)
1555 return false;
1556 }
1557 }
1558
1559 /* Reject cases that would violate register regioning restrictions. */
1560 if (inst->opcode == SHADER_OPCODE_BROADCAST) {
1561 if (arg == 0)
1562 return false;
1563
1564 assert(inst->src[arg].stride == 0);
1565 } else if ((val.file == UNIFORM || !val.is_contiguous()) &&
1566 (inst->is_send_from_grf() || inst->uses_indirect_addressing())) {
1567 return false;
1568 }
1569
1570 /* Some instructions implemented in the generator backend, such as
1571 * derivatives, assume that their operands are packed so we can't
1572 * generally propagate strided regions to them.
1573 */
1574 const unsigned entry_stride = val.file == FIXED_GRF ? 1 : val.stride;
1575 if (instruction_requires_packed_data(inst) && entry_stride != 1)
1576 return false;
1577
1578 const brw_reg_type dst_type = (has_source_modifiers &&
1579 def->dst.type != inst->src[arg].type) ?
1580 def->dst.type : inst->dst.type;
1581
1582 /* Bail if the result of composing both strides would exceed the
1583 * hardware limit.
1584 */
1585 if (!can_take_stride(inst, dst_type, arg,
1586 entry_stride * inst->src[arg].stride,
1587 compiler))
1588 return false;
1589
1590 /* Bail if the source FIXED_GRF region of the copy cannot be trivially
1591 * composed with the source region of the instruction -- E.g. because the
1592 * copy uses some extended stride greater than 4 not supported natively by
1593 * the hardware as a horizontal stride, or because instruction compression
1594 * could require us to use a vertical stride shorter than a GRF.
1595 */
1596 if (val.file == FIXED_GRF &&
1597 (inst->src[arg].stride > 4 ||
1598 inst->dst.component_size(inst->exec_size) >
1599 inst->src[arg].component_size(inst->exec_size)))
1600 return false;
1601
1602 /* Bail if the result of composing both strides cannot be expressed
1603 * as another stride. This avoids, for example, trying to transform
1604 * this:
1605 *
1606 * MOV (8) rX<1>UD rY<0;1,0>UD
1607 * FOO (8) ... rX<8;8,1>UW
1608 *
1609 * into this:
1610 *
1611 * FOO (8) ... rY<0;1,0>UW
1612 *
1613 * Which would have different semantics.
1614 */
1615 if (entry_stride != 1 &&
1616 (inst->src[arg].stride *
1617 brw_type_size_bytes(inst->src[arg].type)) % brw_type_size_bytes(val.type) != 0)
1618 return false;
1619
1620 /* From the Cherry Trail/Braswell PRMs, Volume 7: 3D Media GPGPU:
1621 * EU Overview
1622 * Register Region Restrictions
1623 * Special Requirements for Handling Double Precision Data Types :
1624 *
1625 * "When source or destination datatype is 64b or operation is integer
1626 * DWord multiply, regioning in Align1 must follow these rules:
1627 *
1628 * 1. Source and Destination horizontal stride must be aligned to the
1629 * same qword.
1630 * 2. Regioning must ensure Src.Vstride = Src.Width * Src.Hstride.
1631 * 3. Source and Destination offset must be the same, except the case
1632 * of scalar source."
1633 *
1634 * Most of this is already checked in can_take_stride(), we're only left
1635 * with checking 3.
1636 */
1637 if (has_dst_aligned_region_restriction(devinfo, inst, dst_type) &&
1638 entry_stride != 0 &&
1639 (reg_offset(inst->dst) % (REG_SIZE * reg_unit(devinfo))) != (reg_offset(val) % (REG_SIZE * reg_unit(devinfo))))
1640 return false;
1641
1642 /* The <8;8,0> regions used for FS attributes in multipolygon
1643 * dispatch mode could violate regioning restrictions, don't copy
1644 * propagate them in such cases.
1645 */
1646 if (max_polygons > 1 && val.file == ATTR &&
1647 (has_dst_aligned_region_restriction(devinfo, inst, dst_type) ||
1648 instruction_requires_packed_data(inst) ||
1649 (inst->is_3src(compiler) && arg == 2) ||
1650 def->dst.type != inst->src[arg].type))
1651 return false;
1652
1653 /* Fold the copy into the instruction consuming it. */
1654 inst->src[arg].file = val.file;
1655 inst->src[arg].nr = val.nr;
1656 inst->src[arg].subnr = val.subnr;
1657 inst->src[arg].offset = val.offset;
1658
1659 /* Compose the strides of both regions. */
1660 if (val.file == FIXED_GRF) {
1661 if (inst->src[arg].stride) {
1662 const unsigned orig_width = 1 << val.width;
1663 const unsigned reg_width =
1664 REG_SIZE / (brw_type_size_bytes(inst->src[arg].type) *
1665 inst->src[arg].stride);
1666 inst->src[arg].width = cvt(MIN2(orig_width, reg_width)) - 1;
1667 inst->src[arg].hstride = cvt(inst->src[arg].stride);
1668 inst->src[arg].vstride = inst->src[arg].hstride + inst->src[arg].width;
1669 } else {
1670 inst->src[arg].vstride = inst->src[arg].hstride =
1671 inst->src[arg].width = 0;
1672 }
1673
1674 inst->src[arg].stride = 1;
1675
1676 /* Hopefully no Align16 around here... */
1677 assert(val.swizzle == BRW_SWIZZLE_XYZW);
1678 inst->src[arg].swizzle = val.swizzle;
1679 } else {
1680 inst->src[arg].stride *= val.stride;
1681 }
1682
1683 /* Handle NoMask cases where the def replicates a small scalar to a number
1684 * of channels, but the use is a lower SIMD width but larger type, so each
1685 * invocation reads multiple channels worth of data, e.g.
1686 *
1687 * mov(16) vgrf1:UW, u0<0>:UW NoMask
1688 * mov(8) vgrf2:UD, vgrf1:UD NoMask group0
1689 *
1690 * In this case, we should just use the scalar's type.
1691 */
1692 if (val.stride == 0 &&
1693 inst->opcode == BRW_OPCODE_MOV &&
1694 inst->force_writemask_all && def->force_writemask_all &&
1695 inst->exec_size < def->exec_size &&
1696 (inst->exec_size * brw_type_size_bytes(inst->src[arg].type) ==
1697 def->exec_size * brw_type_size_bytes(val.type))) {
1698 inst->src[arg].type = val.type;
1699 inst->dst.type = val.type;
1700 inst->exec_size = def->exec_size;
1701 }
1702
1703 if (has_source_modifiers) {
1704 if (def->dst.type != inst->src[arg].type) {
1705 /* We are propagating source modifiers from a MOV with a different
1706 * type. If we got here, then we can just change the source and
1707 * destination types of the instruction and keep going.
1708 */
1709 for (int i = 0; i < inst->sources; i++) {
1710 inst->src[i].type = def->dst.type;
1711 }
1712 inst->dst.type = def->dst.type;
1713 }
1714
1715 if (!inst->src[arg].abs) {
1716 inst->src[arg].abs = val.abs;
1717 inst->src[arg].negate ^= val.negate;
1718 }
1719 }
1720
1721 return true;
1722 }
1723
1724 static bool
try_constant_propagate_def(const struct intel_device_info * devinfo,fs_inst * def,brw_reg val,fs_inst * inst,int arg)1725 try_constant_propagate_def(const struct intel_device_info *devinfo,
1726 fs_inst *def, brw_reg val, fs_inst *inst, int arg)
1727 {
1728 /* Bail if inst is reading more than a single vector component of entry */
1729 if (inst->size_read(devinfo, arg) > def->dst.component_size(inst->exec_size))
1730 return false;
1731
1732 return try_constant_propagate_value(val, def->dst.type, inst, arg);
1733 }
1734
1735 /**
1736 * Handle cases like UW subreads of a UD immediate, with an offset.
1737 */
1738 static brw_reg
extract_imm(brw_reg val,brw_reg_type type,unsigned offset)1739 extract_imm(brw_reg val, brw_reg_type type, unsigned offset)
1740 {
1741 assert(val.file == IMM);
1742
1743 const unsigned bitsize = brw_type_size_bits(type);
1744
1745 if (offset == 0 || bitsize == brw_type_size_bits(val.type))
1746 return val;
1747
1748 /* The whole extracted value must come from bits that acutally exist in the
1749 * original immediate value.
1750 */
1751 assert((8 * offset) + bitsize <= brw_type_size_bits(val.type));
1752
1753 val.u64 = (val.u64 >> (8 * offset)) & ((1ull << bitsize) - 1);
1754
1755 return val;
1756 }
1757
1758 static brw_reg
find_value_for_offset(fs_inst * def,const brw_reg & src,unsigned src_size)1759 find_value_for_offset(fs_inst *def, const brw_reg &src, unsigned src_size)
1760 {
1761 brw_reg val;
1762
1763 switch (def->opcode) {
1764 case BRW_OPCODE_MOV:
1765 /* is_raw_move also rejects source modifiers, but copy propagation
1766 * can handle that if the tyeps are the same.
1767 */
1768 if ((def->dst.type == def->src[0].type || def->is_raw_move()) &&
1769 def->src[0].stride <= 1) {
1770 val = def->src[0];
1771
1772 unsigned rel_offset = src.offset - def->dst.offset;
1773
1774 if (val.stride == 0)
1775 rel_offset %= brw_type_size_bytes(def->dst.type);
1776
1777 if (val.file == IMM)
1778 val = extract_imm(val, src.type, rel_offset);
1779 else
1780 val = byte_offset(def->src[0], rel_offset);
1781 }
1782 break;
1783 case SHADER_OPCODE_LOAD_PAYLOAD: {
1784 unsigned offset = 0;
1785 for (int i = def->header_size; i < def->sources; i++) {
1786 /* Ignore the source splat if the source is a scalar. In that case
1787 * always use just the first component.
1788 */
1789 const unsigned splat =
1790 (def->src[i].stride == 0 && !src.is_scalar) || def->src[i].file == IMM ? def->exec_size : 1;
1791 const unsigned component_size =
1792 def->src[i].component_size(def->exec_size);
1793
1794 if (offset == src.offset) {
1795 if (def->dst.type == def->src[i].type &&
1796 def->src[i].stride <= 1 &&
1797 (component_size * splat == src_size ||
1798 (def->src[i].file == IMM && component_size == src_size)))
1799 val = def->src[i];
1800
1801 break;
1802 }
1803
1804 offset += def->exec_size * brw_type_size_bytes(def->src[i].type);
1805 }
1806 break;
1807 }
1808 default:
1809 break;
1810 }
1811
1812 return val;
1813 }
1814
1815 bool
brw_opt_copy_propagation_defs(fs_visitor & s)1816 brw_opt_copy_propagation_defs(fs_visitor &s)
1817 {
1818 const brw::def_analysis &defs = s.def_analysis.require();
1819 unsigned *uses_deleted = new unsigned[defs.count()]();
1820 bool progress = false;
1821
1822 foreach_block_and_inst_safe(block, fs_inst, inst, s.cfg) {
1823 /* Try propagating into this instruction. */
1824 bool constant_progress = false;
1825
1826 for (int i = inst->sources - 1; i >= 0; i--) {
1827 fs_inst *def = defs.get(inst->src[i]);
1828
1829 if (!def || def->saturate)
1830 continue;
1831
1832 bool source_progress = false;
1833
1834 if (def->opcode == SHADER_OPCODE_LOAD_PAYLOAD) {
1835 if (inst->size_read(s.devinfo, i) == def->size_written &&
1836 def->src[0].file != BAD_FILE && def->src[0].file != IMM &&
1837 is_identity_payload(s.devinfo, def->src[0].file, def)) {
1838 source_progress =
1839 try_copy_propagate_def(s.compiler, s.alloc, def, def->src[0],
1840 inst, i, s.max_polygons);
1841
1842 if (source_progress) {
1843 progress = true;
1844 ++uses_deleted[def->dst.nr];
1845 if (defs.get_use_count(def->dst) == uses_deleted[def->dst.nr])
1846 def->remove(defs.get_block(def->dst), true);
1847 }
1848
1849 continue;
1850 }
1851 }
1852
1853 brw_reg val =
1854 find_value_for_offset(def, inst->src[i], inst->size_read(s.devinfo, i));
1855
1856 if (val.file == IMM) {
1857 if (try_constant_propagate_def(s.devinfo, def, val, inst, i)) {
1858 source_progress = true;
1859 constant_progress = true;
1860 }
1861 } else if (val.file == VGRF ||
1862 val.file == ATTR || val.file == UNIFORM ||
1863 (val.file == FIXED_GRF && val.is_contiguous())) {
1864 source_progress =
1865 try_copy_propagate_def(s.compiler, s.alloc, def, val, inst, i,
1866 s.max_polygons);
1867 }
1868
1869 if (source_progress) {
1870 progress = true;
1871 ++uses_deleted[def->dst.nr];
1872
1873 /* We can copy propagate through an instruction like
1874 *
1875 * mov.nz.f0.0(8) %2:D, -%78:D
1876 *
1877 * but deleting the instruction may alter the program.
1878 */
1879 if (def->conditional_mod == BRW_CONDITIONAL_NONE &&
1880 defs.get_use_count(def->dst) == uses_deleted[def->dst.nr]) {
1881 def->remove(defs.get_block(def->dst), true);
1882 }
1883 }
1884 }
1885
1886 if (constant_progress) {
1887 commute_immediates(inst);
1888 brw_opt_constant_fold_instruction(s.compiler->devinfo, inst);
1889 }
1890 }
1891
1892 if (progress) {
1893 s.cfg->adjust_block_ips();
1894 s.invalidate_analysis(DEPENDENCY_INSTRUCTION_DATA_FLOW |
1895 DEPENDENCY_INSTRUCTION_DETAIL);
1896 }
1897
1898 delete [] uses_deleted;
1899
1900 return progress;
1901 }
1902