1 /*
2 * Copyright © 2011 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 /**
25 * @file brw_vec4_copy_propagation.cpp
26 *
27 * Implements tracking of values copied between registers, and
28 * optimizations based on that: copy propagation and constant
29 * propagation.
30 */
31
32 #include "brw_vec4.h"
33 #include "brw_cfg.h"
34 #include "brw_eu.h"
35
36 namespace brw {
37
38 struct copy_entry {
39 src_reg *value[4];
40 int saturatemask;
41 };
42
43 static bool
is_direct_copy(vec4_instruction * inst)44 is_direct_copy(vec4_instruction *inst)
45 {
46 return (inst->opcode == BRW_OPCODE_MOV &&
47 !inst->predicate &&
48 inst->dst.file == VGRF &&
49 inst->dst.offset % REG_SIZE == 0 &&
50 !inst->dst.reladdr &&
51 !inst->src[0].reladdr &&
52 (inst->dst.type == inst->src[0].type ||
53 (inst->dst.type == BRW_REGISTER_TYPE_F &&
54 inst->src[0].type == BRW_REGISTER_TYPE_VF)));
55 }
56
57 static bool
is_dominated_by_previous_instruction(vec4_instruction * inst)58 is_dominated_by_previous_instruction(vec4_instruction *inst)
59 {
60 return (inst->opcode != BRW_OPCODE_DO &&
61 inst->opcode != BRW_OPCODE_WHILE &&
62 inst->opcode != BRW_OPCODE_ELSE &&
63 inst->opcode != BRW_OPCODE_ENDIF);
64 }
65
66 static bool
is_channel_updated(vec4_instruction * inst,src_reg * values[4],int ch)67 is_channel_updated(vec4_instruction *inst, src_reg *values[4], int ch)
68 {
69 const src_reg *src = values[ch];
70
71 /* consider GRF only */
72 assert(inst->dst.file == VGRF);
73 if (!src || src->file != VGRF)
74 return false;
75
76 return regions_overlap(*src, REG_SIZE, inst->dst, inst->size_written) &&
77 (inst->dst.offset != src->offset ||
78 inst->dst.writemask & (1 << BRW_GET_SWZ(src->swizzle, ch)));
79 }
80
81 /**
82 * Get the origin of a copy as a single register if all components present in
83 * the given readmask originate from the same register and have compatible
84 * regions, otherwise return a BAD_FILE register.
85 */
86 static src_reg
get_copy_value(const copy_entry & entry,unsigned readmask)87 get_copy_value(const copy_entry &entry, unsigned readmask)
88 {
89 unsigned swz[4] = {};
90 src_reg value;
91
92 for (unsigned i = 0; i < 4; i++) {
93 if (readmask & (1 << i)) {
94 if (entry.value[i]) {
95 src_reg src = *entry.value[i];
96
97 if (src.file == IMM) {
98 swz[i] = i;
99 } else {
100 swz[i] = BRW_GET_SWZ(src.swizzle, i);
101 /* Overwrite the original swizzle so the src_reg::equals call
102 * below doesn't care about it, the correct swizzle will be
103 * calculated once the swizzles of all components are known.
104 */
105 src.swizzle = BRW_SWIZZLE_XYZW;
106 }
107
108 if (value.file == BAD_FILE) {
109 value = src;
110 } else if (!value.equals(src)) {
111 return src_reg();
112 }
113 } else {
114 return src_reg();
115 }
116 }
117 }
118
119 return swizzle(value,
120 brw_compose_swizzle(brw_swizzle_for_mask(readmask),
121 BRW_SWIZZLE4(swz[0], swz[1],
122 swz[2], swz[3])));
123 }
124
125 static bool
try_constant_propagate(vec4_instruction * inst,int arg,const copy_entry * entry)126 try_constant_propagate(vec4_instruction *inst,
127 int arg, const copy_entry *entry)
128 {
129 /* For constant propagation, we only handle the same constant
130 * across all 4 channels. Some day, we should handle the 8-bit
131 * float vector format, which would let us constant propagate
132 * vectors better.
133 * We could be more aggressive here -- some channels might not get used
134 * based on the destination writemask.
135 */
136 src_reg value =
137 get_copy_value(*entry,
138 brw_apply_inv_swizzle_to_mask(inst->src[arg].swizzle,
139 WRITEMASK_XYZW));
140
141 if (value.file != IMM)
142 return false;
143
144 /* 64-bit types can't be used except for one-source instructions, which
145 * higher levels should have constant folded away, so there's no point in
146 * propagating immediates here.
147 */
148 if (type_sz(value.type) == 8 || type_sz(inst->src[arg].type) == 8)
149 return false;
150
151 if (value.type == BRW_REGISTER_TYPE_VF) {
152 /* The result of bit-casting the component values of a vector float
153 * cannot in general be represented as an immediate.
154 */
155 if (inst->src[arg].type != BRW_REGISTER_TYPE_F)
156 return false;
157 } else {
158 value.type = inst->src[arg].type;
159 }
160
161 if (inst->src[arg].abs) {
162 if (!brw_abs_immediate(value.type, &value.as_brw_reg()))
163 return false;
164 }
165
166 if (inst->src[arg].negate) {
167 if (!brw_negate_immediate(value.type, &value.as_brw_reg()))
168 return false;
169 }
170
171 value = swizzle(value, inst->src[arg].swizzle);
172
173 switch (inst->opcode) {
174 case BRW_OPCODE_MOV:
175 case SHADER_OPCODE_BROADCAST:
176 inst->src[arg] = value;
177 return true;
178
179 case VEC4_OPCODE_UNTYPED_ATOMIC:
180 if (arg == 1) {
181 inst->src[arg] = value;
182 return true;
183 }
184 break;
185
186 case SHADER_OPCODE_POW:
187 case SHADER_OPCODE_INT_QUOTIENT:
188 case SHADER_OPCODE_INT_REMAINDER:
189 break;
190 case BRW_OPCODE_DP2:
191 case BRW_OPCODE_DP3:
192 case BRW_OPCODE_DP4:
193 case BRW_OPCODE_DPH:
194 case BRW_OPCODE_BFI1:
195 case BRW_OPCODE_ASR:
196 case BRW_OPCODE_SHL:
197 case BRW_OPCODE_SHR:
198 case BRW_OPCODE_SUBB:
199 if (arg == 1) {
200 inst->src[arg] = value;
201 return true;
202 }
203 break;
204
205 case BRW_OPCODE_MACH:
206 case BRW_OPCODE_MUL:
207 case SHADER_OPCODE_MULH:
208 case BRW_OPCODE_ADD:
209 case BRW_OPCODE_OR:
210 case BRW_OPCODE_AND:
211 case BRW_OPCODE_XOR:
212 case BRW_OPCODE_ADDC:
213 if (arg == 1) {
214 inst->src[arg] = value;
215 return true;
216 } else if (arg == 0 && inst->src[1].file != IMM) {
217 /* Fit this constant in by commuting the operands. Exception: we
218 * can't do this for 32-bit integer MUL/MACH because it's asymmetric.
219 */
220 if ((inst->opcode == BRW_OPCODE_MUL ||
221 inst->opcode == BRW_OPCODE_MACH) &&
222 (inst->src[1].type == BRW_REGISTER_TYPE_D ||
223 inst->src[1].type == BRW_REGISTER_TYPE_UD))
224 break;
225 inst->src[0] = inst->src[1];
226 inst->src[1] = value;
227 return true;
228 }
229 break;
230 case GS_OPCODE_SET_WRITE_OFFSET:
231 /* This is just a multiply by a constant with special strides.
232 * The generator will handle immediates in both arguments (generating
233 * a single MOV of the product). So feel free to propagate in src0.
234 */
235 inst->src[arg] = value;
236 return true;
237
238 case BRW_OPCODE_CMP:
239 if (arg == 1) {
240 inst->src[arg] = value;
241 return true;
242 } else if (arg == 0 && inst->src[1].file != IMM) {
243 enum brw_conditional_mod new_cmod;
244
245 new_cmod = brw_swap_cmod(inst->conditional_mod);
246 if (new_cmod != BRW_CONDITIONAL_NONE) {
247 /* Fit this constant in by swapping the operands and
248 * flipping the test.
249 */
250 inst->src[0] = inst->src[1];
251 inst->src[1] = value;
252 inst->conditional_mod = new_cmod;
253 return true;
254 }
255 }
256 break;
257
258 case BRW_OPCODE_SEL:
259 if (arg == 1) {
260 inst->src[arg] = value;
261 return true;
262 } else if (arg == 0 && inst->src[1].file != IMM) {
263 inst->src[0] = inst->src[1];
264 inst->src[1] = value;
265
266 /* If this was predicated, flipping operands means
267 * we also need to flip the predicate.
268 */
269 if (inst->conditional_mod == BRW_CONDITIONAL_NONE) {
270 inst->predicate_inverse = !inst->predicate_inverse;
271 }
272 return true;
273 }
274 break;
275
276 default:
277 break;
278 }
279
280 return false;
281 }
282
283 static bool
is_align1_opcode(unsigned opcode)284 is_align1_opcode(unsigned opcode)
285 {
286 switch (opcode) {
287 case VEC4_OPCODE_DOUBLE_TO_F32:
288 case VEC4_OPCODE_DOUBLE_TO_D32:
289 case VEC4_OPCODE_DOUBLE_TO_U32:
290 case VEC4_OPCODE_TO_DOUBLE:
291 case VEC4_OPCODE_PICK_LOW_32BIT:
292 case VEC4_OPCODE_PICK_HIGH_32BIT:
293 case VEC4_OPCODE_SET_LOW_32BIT:
294 case VEC4_OPCODE_SET_HIGH_32BIT:
295 return true;
296 default:
297 return false;
298 }
299 }
300
301 static bool
try_copy_propagate(const struct gen_device_info * devinfo,vec4_instruction * inst,int arg,const copy_entry * entry,int attributes_per_reg)302 try_copy_propagate(const struct gen_device_info *devinfo,
303 vec4_instruction *inst, int arg,
304 const copy_entry *entry, int attributes_per_reg)
305 {
306 /* Build up the value we are propagating as if it were the source of a
307 * single MOV
308 */
309 src_reg value =
310 get_copy_value(*entry,
311 brw_apply_inv_swizzle_to_mask(inst->src[arg].swizzle,
312 WRITEMASK_XYZW));
313
314 /* Check that we can propagate that value */
315 if (value.file != UNIFORM &&
316 value.file != VGRF &&
317 value.file != ATTR)
318 return false;
319
320 /* Instructions that write 2 registers also need to read 2 registers. Make
321 * sure we don't break that restriction by copy propagating from a uniform.
322 */
323 if (inst->size_written > REG_SIZE && is_uniform(value))
324 return false;
325
326 /* There is a regioning restriction such that if execsize == width
327 * and hstride != 0 then the vstride can't be 0. When we split instrutions
328 * that take a single-precision source (like F->DF conversions) we end up
329 * with a 4-wide source on an instruction with an execution size of 4.
330 * If we then copy-propagate the source from a uniform we also end up with a
331 * vstride of 0 and we violate the restriction.
332 */
333 if (inst->exec_size == 4 && value.file == UNIFORM &&
334 type_sz(value.type) == 4)
335 return false;
336
337 /* If the type of the copy value is different from the type of the
338 * instruction then the swizzles and writemasks involved don't have the same
339 * meaning and simply replacing the source would produce different semantics.
340 */
341 if (type_sz(value.type) != type_sz(inst->src[arg].type))
342 return false;
343
344 if (inst->src[arg].offset % REG_SIZE || value.offset % REG_SIZE)
345 return false;
346
347 bool has_source_modifiers = value.negate || value.abs;
348
349 /* gen6 math and gen7+ SENDs from GRFs ignore source modifiers on
350 * instructions.
351 */
352 if ((has_source_modifiers || value.file == UNIFORM ||
353 value.swizzle != BRW_SWIZZLE_XYZW) && !inst->can_do_source_mods(devinfo))
354 return false;
355
356 if (has_source_modifiers &&
357 value.type != inst->src[arg].type &&
358 !inst->can_change_types())
359 return false;
360
361 if (has_source_modifiers &&
362 (inst->opcode == SHADER_OPCODE_GEN4_SCRATCH_WRITE ||
363 inst->opcode == VEC4_OPCODE_PICK_HIGH_32BIT))
364 return false;
365
366 unsigned composed_swizzle = brw_compose_swizzle(inst->src[arg].swizzle,
367 value.swizzle);
368
369 /* Instructions that operate on vectors in ALIGN1 mode will ignore swizzles
370 * so copy-propagation won't be safe if the composed swizzle is anything
371 * other than the identity.
372 */
373 if (is_align1_opcode(inst->opcode) && composed_swizzle != BRW_SWIZZLE_XYZW)
374 return false;
375
376 if (inst->is_3src(devinfo) &&
377 (value.file == UNIFORM ||
378 (value.file == ATTR && attributes_per_reg != 1)) &&
379 !brw_is_single_value_swizzle(composed_swizzle))
380 return false;
381
382 if (inst->is_send_from_grf())
383 return false;
384
385 /* we can't generally copy-propagate UD negations becuse we
386 * end up accessing the resulting values as signed integers
387 * instead. See also resolve_ud_negate().
388 */
389 if (value.negate &&
390 value.type == BRW_REGISTER_TYPE_UD)
391 return false;
392
393 /* Don't report progress if this is a noop. */
394 if (value.equals(inst->src[arg]))
395 return false;
396
397 const unsigned dst_saturate_mask = inst->dst.writemask &
398 brw_apply_swizzle_to_mask(inst->src[arg].swizzle, entry->saturatemask);
399
400 if (dst_saturate_mask) {
401 /* We either saturate all or nothing. */
402 if (dst_saturate_mask != inst->dst.writemask)
403 return false;
404
405 /* Limit saturate propagation only to SEL with src1 bounded within 0.0
406 * and 1.0, otherwise skip copy propagate altogether.
407 */
408 switch(inst->opcode) {
409 case BRW_OPCODE_SEL:
410 if (arg != 0 ||
411 inst->src[0].type != BRW_REGISTER_TYPE_F ||
412 inst->src[1].file != IMM ||
413 inst->src[1].type != BRW_REGISTER_TYPE_F ||
414 inst->src[1].f < 0.0 ||
415 inst->src[1].f > 1.0) {
416 return false;
417 }
418 if (!inst->saturate)
419 inst->saturate = true;
420 break;
421 default:
422 return false;
423 }
424 }
425
426 /* Build the final value */
427 if (inst->src[arg].abs) {
428 value.negate = false;
429 value.abs = true;
430 }
431 if (inst->src[arg].negate)
432 value.negate = !value.negate;
433
434 value.swizzle = composed_swizzle;
435 if (has_source_modifiers &&
436 value.type != inst->src[arg].type) {
437 assert(inst->can_change_types());
438 for (int i = 0; i < 3; i++) {
439 inst->src[i].type = value.type;
440 }
441 inst->dst.type = value.type;
442 } else {
443 value.type = inst->src[arg].type;
444 }
445
446 inst->src[arg] = value;
447 return true;
448 }
449
450 bool
opt_copy_propagation(bool do_constant_prop)451 vec4_visitor::opt_copy_propagation(bool do_constant_prop)
452 {
453 /* If we are in dual instanced or single mode, then attributes are going
454 * to be interleaved, so one register contains two attribute slots.
455 */
456 const int attributes_per_reg =
457 prog_data->dispatch_mode == DISPATCH_MODE_4X2_DUAL_OBJECT ? 1 : 2;
458 bool progress = false;
459 struct copy_entry entries[alloc.total_size];
460
461 memset(&entries, 0, sizeof(entries));
462
463 foreach_block_and_inst(block, vec4_instruction, inst, cfg) {
464 /* This pass only works on basic blocks. If there's flow
465 * control, throw out all our information and start from
466 * scratch.
467 *
468 * This should really be fixed by using a structure like in
469 * src/glsl/opt_copy_propagation.cpp to track available copies.
470 */
471 if (!is_dominated_by_previous_instruction(inst)) {
472 memset(&entries, 0, sizeof(entries));
473 continue;
474 }
475
476 /* For each source arg, see if each component comes from a copy
477 * from the same type file (IMM, VGRF, UNIFORM), and try
478 * optimizing out access to the copy result
479 */
480 for (int i = 2; i >= 0; i--) {
481 /* Copied values end up in GRFs, and we don't track reladdr
482 * accesses.
483 */
484 if (inst->src[i].file != VGRF ||
485 inst->src[i].reladdr)
486 continue;
487
488 /* We only handle register-aligned single GRF copies. */
489 if (inst->size_read(i) != REG_SIZE ||
490 inst->src[i].offset % REG_SIZE)
491 continue;
492
493 const unsigned reg = (alloc.offsets[inst->src[i].nr] +
494 inst->src[i].offset / REG_SIZE);
495 const copy_entry &entry = entries[reg];
496
497 if (do_constant_prop && try_constant_propagate(inst, i, &entry))
498 progress = true;
499 else if (try_copy_propagate(devinfo, inst, i, &entry, attributes_per_reg))
500 progress = true;
501 }
502
503 /* Track available source registers. */
504 if (inst->dst.file == VGRF) {
505 const int reg =
506 alloc.offsets[inst->dst.nr] + inst->dst.offset / REG_SIZE;
507
508 /* Update our destination's current channel values. For a direct copy,
509 * the value is the newly propagated source. Otherwise, we don't know
510 * the new value, so clear it.
511 */
512 bool direct_copy = is_direct_copy(inst);
513 entries[reg].saturatemask &= ~inst->dst.writemask;
514 for (int i = 0; i < 4; i++) {
515 if (inst->dst.writemask & (1 << i)) {
516 entries[reg].value[i] = direct_copy ? &inst->src[0] : NULL;
517 entries[reg].saturatemask |=
518 inst->saturate && direct_copy ? 1 << i : 0;
519 }
520 }
521
522 /* Clear the records for any registers whose current value came from
523 * our destination's updated channels, as the two are no longer equal.
524 */
525 if (inst->dst.reladdr)
526 memset(&entries, 0, sizeof(entries));
527 else {
528 for (unsigned i = 0; i < alloc.total_size; i++) {
529 for (int j = 0; j < 4; j++) {
530 if (is_channel_updated(inst, entries[i].value, j)) {
531 entries[i].value[j] = NULL;
532 entries[i].saturatemask &= ~(1 << j);
533 }
534 }
535 }
536 }
537 }
538 }
539
540 if (progress)
541 invalidate_analysis(DEPENDENCY_INSTRUCTION_DATA_FLOW |
542 DEPENDENCY_INSTRUCTION_DETAIL);
543
544 return progress;
545 }
546
547 } /* namespace brw */
548