1 /* -*- mode: C; c-file-style: "k&r"; tab-width 4; indent-tabs-mode: t; -*- */
2
3 /*
4 * Copyright (C) 2014 Rob Clark <robclark@freedesktop.org>
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice (including the next
14 * paragraph) shall be included in all copies or substantial portions of the
15 * Software.
16 *
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
23 * SOFTWARE.
24 *
25 * Authors:
26 * Rob Clark <robclark@freedesktop.org>
27 */
28
29 #include "util/u_math.h"
30
31 #include "freedreno_util.h"
32
33 #include "ir3.h"
34
35 /*
36 * Legalize:
37 *
38 * We currently require that scheduling ensures that we have enough nop's
39 * in all the right places. The legalize step mostly handles fixing up
40 * instruction flags ((ss)/(sy)/(ei)), and collapses sequences of nop's
41 * into fewer nop's w/ rpt flag.
42 */
43
44 struct ir3_legalize_ctx {
45 bool has_samp;
46 bool has_ssbo;
47 int max_bary;
48 };
49
50 /* We want to evaluate each block from the position of any other
51 * predecessor block, in order that the flags set are the union
52 * of all possible program paths. For stopping condition, we
53 * want to stop when the pair of <pred-block, current-block> has
54 * been visited already.
55 *
56 * XXX is that completely true? We could have different needs_xyz
57 * flags set depending on path leading to pred-block.. we could
58 * do *most* of this based on chasing src instructions ptrs (and
59 * following all phi srcs).. except the write-after-read hazzard.
60 *
61 * For now we just set ss/sy flag on first instruction on block,
62 * and handle everything within the block as before.
63 */
64
65 static void
legalize_block(struct ir3_legalize_ctx * ctx,struct ir3_block * block)66 legalize_block(struct ir3_legalize_ctx *ctx, struct ir3_block *block)
67 {
68 struct ir3_instruction *last_input = NULL;
69 struct ir3_instruction *last_rel = NULL;
70 struct ir3_instruction *last_n = NULL;
71 struct list_head instr_list;
72 regmask_t needs_ss_war; /* write after read */
73 regmask_t needs_ss;
74 regmask_t needs_sy;
75
76 regmask_init(&needs_ss_war);
77 regmask_init(&needs_ss);
78 regmask_init(&needs_sy);
79
80 /* remove all the instructions from the list, we'll be adding
81 * them back in as we go
82 */
83 list_replace(&block->instr_list, &instr_list);
84 list_inithead(&block->instr_list);
85
86 list_for_each_entry_safe (struct ir3_instruction, n, &instr_list, node) {
87 struct ir3_register *reg;
88 unsigned i;
89
90 if (is_meta(n))
91 continue;
92
93 if (is_input(n)) {
94 struct ir3_register *inloc = n->regs[1];
95 assert(inloc->flags & IR3_REG_IMMED);
96 ctx->max_bary = MAX2(ctx->max_bary, inloc->iim_val);
97 }
98
99 if (last_n && is_barrier(last_n))
100 n->flags |= IR3_INSTR_SS | IR3_INSTR_SY;
101
102 /* NOTE: consider dst register too.. it could happen that
103 * texture sample instruction (for example) writes some
104 * components which are unused. A subsequent instruction
105 * that writes the same register can race w/ the sam instr
106 * resulting in undefined results:
107 */
108 for (i = 0; i < n->regs_count; i++) {
109 reg = n->regs[i];
110
111 if (reg_gpr(reg)) {
112
113 /* TODO: we probably only need (ss) for alu
114 * instr consuming sfu result.. need to make
115 * some tests for both this and (sy)..
116 */
117 if (regmask_get(&needs_ss, reg)) {
118 n->flags |= IR3_INSTR_SS;
119 regmask_init(&needs_ss);
120 }
121
122 if (regmask_get(&needs_sy, reg)) {
123 n->flags |= IR3_INSTR_SY;
124 regmask_init(&needs_sy);
125 }
126 }
127
128 /* TODO: is it valid to have address reg loaded from a
129 * relative src (ie. mova a0, c<a0.x+4>)? If so, the
130 * last_rel check below should be moved ahead of this:
131 */
132 if (reg->flags & IR3_REG_RELATIV)
133 last_rel = n;
134 }
135
136 if (n->regs_count > 0) {
137 reg = n->regs[0];
138 if (regmask_get(&needs_ss_war, reg)) {
139 n->flags |= IR3_INSTR_SS;
140 regmask_init(&needs_ss_war); // ??? I assume?
141 }
142
143 if (last_rel && (reg->num == regid(REG_A0, 0))) {
144 last_rel->flags |= IR3_INSTR_UL;
145 last_rel = NULL;
146 }
147 }
148
149 /* cat5+ does not have an (ss) bit, if needed we need to
150 * insert a nop to carry the sync flag. Would be kinda
151 * clever if we were aware of this during scheduling, but
152 * this should be a pretty rare case:
153 */
154 if ((n->flags & IR3_INSTR_SS) && (opc_cat(n->opc) >= 5)) {
155 struct ir3_instruction *nop;
156 nop = ir3_NOP(block);
157 nop->flags |= IR3_INSTR_SS;
158 n->flags &= ~IR3_INSTR_SS;
159 }
160
161 /* need to be able to set (ss) on first instruction: */
162 if (list_empty(&block->instr_list) && (opc_cat(n->opc) >= 5))
163 ir3_NOP(block);
164
165 if (is_nop(n) && !list_empty(&block->instr_list)) {
166 struct ir3_instruction *last = list_last_entry(&block->instr_list,
167 struct ir3_instruction, node);
168 if (is_nop(last) && (last->repeat < 5)) {
169 last->repeat++;
170 last->flags |= n->flags;
171 continue;
172 }
173 }
174
175 list_addtail(&n->node, &block->instr_list);
176
177 if (is_sfu(n))
178 regmask_set(&needs_ss, n->regs[0]);
179
180 if (is_tex(n)) {
181 /* this ends up being the # of samp instructions.. but that
182 * is ok, everything else only cares whether it is zero or
183 * not. We do this here, rather than when we encounter a
184 * SAMP decl, because (especially in binning pass shader)
185 * the samp instruction(s) could get eliminated if the
186 * result is not used.
187 */
188 ctx->has_samp = true;
189 regmask_set(&needs_sy, n->regs[0]);
190 } else if (n->opc == OPC_RESINFO) {
191 regmask_set(&needs_ss, n->regs[0]);
192 ir3_NOP(block)->flags |= IR3_INSTR_SS;
193 } else if (is_load(n)) {
194 /* seems like ldlv needs (ss) bit instead?? which is odd but
195 * makes a bunch of flat-varying tests start working on a4xx.
196 */
197 if ((n->opc == OPC_LDLV) || (n->opc == OPC_LDL))
198 regmask_set(&needs_ss, n->regs[0]);
199 else
200 regmask_set(&needs_sy, n->regs[0]);
201 } else if (is_atomic(n->opc)) {
202 if (n->flags & IR3_INSTR_G)
203 regmask_set(&needs_sy, n->regs[0]);
204 else
205 regmask_set(&needs_ss, n->regs[0]);
206 }
207
208 if (is_ssbo(n->opc) || (is_atomic(n->opc) && (n->flags & IR3_INSTR_G)))
209 ctx->has_ssbo = true;
210
211 /* both tex/sfu appear to not always immediately consume
212 * their src register(s):
213 */
214 if (is_tex(n) || is_sfu(n) || is_mem(n)) {
215 foreach_src(reg, n) {
216 if (reg_gpr(reg))
217 regmask_set(&needs_ss_war, reg);
218 }
219 }
220
221 if (is_input(n))
222 last_input = n;
223
224 last_n = n;
225 }
226
227 if (last_input) {
228 /* special hack.. if using ldlv to bypass interpolation,
229 * we need to insert a dummy bary.f on which we can set
230 * the (ei) flag:
231 */
232 if (is_mem(last_input) && (last_input->opc == OPC_LDLV)) {
233 struct ir3_instruction *baryf;
234
235 /* (ss)bary.f (ei)r63.x, 0, r0.x */
236 baryf = ir3_instr_create(block, OPC_BARY_F);
237 baryf->flags |= IR3_INSTR_SS;
238 ir3_reg_create(baryf, regid(63, 0), 0);
239 ir3_reg_create(baryf, 0, IR3_REG_IMMED)->iim_val = 0;
240 ir3_reg_create(baryf, regid(0, 0), 0);
241
242 /* insert the dummy bary.f after last_input: */
243 list_delinit(&baryf->node);
244 list_add(&baryf->node, &last_input->node);
245
246 last_input = baryf;
247 }
248 last_input->regs[0]->flags |= IR3_REG_EI;
249 }
250
251 if (last_rel)
252 last_rel->flags |= IR3_INSTR_UL;
253
254 list_first_entry(&block->instr_list, struct ir3_instruction, node)
255 ->flags |= IR3_INSTR_SS | IR3_INSTR_SY;
256 }
257
258 /* NOTE: branch instructions are always the last instruction(s)
259 * in the block. We take advantage of this as we resolve the
260 * branches, since "if (foo) break;" constructs turn into
261 * something like:
262 *
263 * block3 {
264 * ...
265 * 0029:021: mov.s32s32 r62.x, r1.y
266 * 0082:022: br !p0.x, target=block5
267 * 0083:023: br p0.x, target=block4
268 * // succs: if _[0029:021: mov.s32s32] block4; else block5;
269 * }
270 * block4 {
271 * 0084:024: jump, target=block6
272 * // succs: block6;
273 * }
274 * block5 {
275 * 0085:025: jump, target=block7
276 * // succs: block7;
277 * }
278 *
279 * ie. only instruction in block4/block5 is a jump, so when
280 * resolving branches we can easily detect this by checking
281 * that the first instruction in the target block is itself
282 * a jump, and setup the br directly to the jump's target
283 * (and strip back out the now unreached jump)
284 *
285 * TODO sometimes we end up with things like:
286 *
287 * br !p0.x, #2
288 * br p0.x, #12
289 * add.u r0.y, r0.y, 1
290 *
291 * If we swapped the order of the branches, we could drop one.
292 */
293 static struct ir3_block *
resolve_dest_block(struct ir3_block * block)294 resolve_dest_block(struct ir3_block *block)
295 {
296 /* special case for last block: */
297 if (!block->successors[0])
298 return block;
299
300 /* NOTE that we may or may not have inserted the jump
301 * in the target block yet, so conditions to resolve
302 * the dest to the dest block's successor are:
303 *
304 * (1) successor[1] == NULL &&
305 * (2) (block-is-empty || only-instr-is-jump)
306 */
307 if (block->successors[1] == NULL) {
308 if (list_empty(&block->instr_list)) {
309 return block->successors[0];
310 } else if (list_length(&block->instr_list) == 1) {
311 struct ir3_instruction *instr = list_first_entry(
312 &block->instr_list, struct ir3_instruction, node);
313 if (instr->opc == OPC_JUMP)
314 return block->successors[0];
315 }
316 }
317 return block;
318 }
319
320 static bool
resolve_jump(struct ir3_instruction * instr)321 resolve_jump(struct ir3_instruction *instr)
322 {
323 struct ir3_block *tblock =
324 resolve_dest_block(instr->cat0.target);
325 struct ir3_instruction *target;
326
327 if (tblock != instr->cat0.target) {
328 list_delinit(&instr->cat0.target->node);
329 instr->cat0.target = tblock;
330 return true;
331 }
332
333 target = list_first_entry(&tblock->instr_list,
334 struct ir3_instruction, node);
335
336 if ((!target) || (target->ip == (instr->ip + 1))) {
337 list_delinit(&instr->node);
338 return true;
339 } else {
340 instr->cat0.immed =
341 (int)target->ip - (int)instr->ip;
342 }
343 return false;
344 }
345
346 /* resolve jumps, removing jumps/branches to immediately following
347 * instruction which we end up with from earlier stages. Since
348 * removing an instruction can invalidate earlier instruction's
349 * branch offsets, we need to do this iteratively until no more
350 * branches are removed.
351 */
352 static bool
resolve_jumps(struct ir3 * ir)353 resolve_jumps(struct ir3 *ir)
354 {
355 list_for_each_entry (struct ir3_block, block, &ir->block_list, node)
356 list_for_each_entry (struct ir3_instruction, instr, &block->instr_list, node)
357 if (is_flow(instr) && instr->cat0.target)
358 if (resolve_jump(instr))
359 return true;
360
361 return false;
362 }
363
364 /* we want to mark points where divergent flow control re-converges
365 * with (jp) flags. For now, since we don't do any optimization for
366 * things that start out as a 'do {} while()', re-convergence points
367 * will always be a branch or jump target. Note that this is overly
368 * conservative, since unconditional jump targets are not convergence
369 * points, we are just assuming that the other path to reach the jump
370 * target was divergent. If we were clever enough to optimize the
371 * jump at end of a loop back to a conditional branch into a single
372 * conditional branch, ie. like:
373 *
374 * add.f r1.w, r0.x, (neg)(r)c2.x <= loop start
375 * mul.f r1.z, r1.z, r0.x
376 * mul.f r1.y, r1.y, r0.x
377 * mul.f r0.z, r1.x, r0.x
378 * mul.f r0.w, r0.y, r0.x
379 * cmps.f.ge r0.x, (r)c2.y, (r)r1.w
380 * add.s r0.x, (r)r0.x, (r)-1
381 * sel.f32 r0.x, (r)c3.y, (r)r0.x, c3.x
382 * cmps.f.eq p0.x, r0.x, c3.y
383 * mov.f32f32 r0.x, r1.w
384 * mov.f32f32 r0.y, r0.w
385 * mov.f32f32 r1.x, r0.z
386 * (rpt2)nop
387 * br !p0.x, #-13
388 * (jp)mul.f r0.x, c263.y, r1.y
389 *
390 * Then we'd have to be more clever, as the convergence point is no
391 * longer a branch or jump target.
392 */
393 static void
mark_convergence_points(struct ir3 * ir)394 mark_convergence_points(struct ir3 *ir)
395 {
396 list_for_each_entry (struct ir3_block, block, &ir->block_list, node) {
397 list_for_each_entry (struct ir3_instruction, instr, &block->instr_list, node) {
398 if (is_flow(instr) && instr->cat0.target) {
399 struct ir3_instruction *target =
400 list_first_entry(&instr->cat0.target->instr_list,
401 struct ir3_instruction, node);
402 target->flags |= IR3_INSTR_JP;
403 }
404 }
405 }
406 }
407
408 void
ir3_legalize(struct ir3 * ir,bool * has_samp,bool * has_ssbo,int * max_bary)409 ir3_legalize(struct ir3 *ir, bool *has_samp, bool *has_ssbo, int *max_bary)
410 {
411 struct ir3_legalize_ctx ctx = {
412 .max_bary = -1,
413 };
414
415 list_for_each_entry (struct ir3_block, block, &ir->block_list, node) {
416 legalize_block(&ctx, block);
417 }
418
419 *has_samp = ctx.has_samp;
420 *has_ssbo = ctx.has_ssbo;
421 *max_bary = ctx.max_bary;
422
423 do {
424 ir3_count_instructions(ir);
425 } while(resolve_jumps(ir));
426
427 mark_convergence_points(ir);
428 }
429