1 /*
2 * Copyright (C) 2018-2019 Alyssa Rosenzweig <alyssa@rosenzweig.io>
3 * Copyright (C) 2019-2020 Collabora, Ltd.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
14 * Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24
25 #include "compiler.h"
26 #include "midgard_ops.h"
27 #include "midgard_quirks.h"
28
29 static midgard_int_mod
mir_get_imod(bool shift,nir_alu_type T,bool half,bool scalar)30 mir_get_imod(bool shift, nir_alu_type T, bool half, bool scalar)
31 {
32 if (!half) {
33 assert(!shift);
34 /* Doesn't matter, src mods are only used when expanding */
35 return midgard_int_sign_extend;
36 }
37
38 if (shift)
39 return midgard_int_left_shift;
40
41 if (nir_alu_type_get_base_type(T) == nir_type_int)
42 return midgard_int_sign_extend;
43 else
44 return midgard_int_zero_extend;
45 }
46
47 void
midgard_pack_ubo_index_imm(midgard_load_store_word * word,unsigned index)48 midgard_pack_ubo_index_imm(midgard_load_store_word *word, unsigned index)
49 {
50 word->arg_comp = index & 0x3;
51 word->arg_reg = (index >> 2) & 0x7;
52 word->bitsize_toggle = (index >> 5) & 0x1;
53 word->index_format = (index >> 6) & 0x3;
54 }
55
56 unsigned
midgard_unpack_ubo_index_imm(midgard_load_store_word word)57 midgard_unpack_ubo_index_imm(midgard_load_store_word word)
58 {
59 unsigned ubo = word.arg_comp |
60 (word.arg_reg << 2) |
61 (word.bitsize_toggle << 5) |
62 (word.index_format << 6);
63
64 return ubo;
65 }
66
midgard_pack_varying_params(midgard_load_store_word * word,midgard_varying_params p)67 void midgard_pack_varying_params(midgard_load_store_word *word, midgard_varying_params p)
68 {
69 /* Currently these parameters are not supported. */
70 assert(p.direct_sample_pos_x == 0 && p.direct_sample_pos_y == 0);
71
72 unsigned u;
73 memcpy(&u, &p, sizeof(p));
74
75 word->signed_offset |= u & 0x1FF;
76 }
77
midgard_unpack_varying_params(midgard_load_store_word word)78 midgard_varying_params midgard_unpack_varying_params(midgard_load_store_word word)
79 {
80 unsigned params = word.signed_offset & 0x1FF;
81
82 midgard_varying_params p;
83 memcpy(&p, ¶ms, sizeof(p));
84
85 return p;
86 }
87
88 unsigned
mir_pack_mod(midgard_instruction * ins,unsigned i,bool scalar)89 mir_pack_mod(midgard_instruction *ins, unsigned i, bool scalar)
90 {
91 bool integer = midgard_is_integer_op(ins->op);
92 unsigned base_size = max_bitsize_for_alu(ins);
93 unsigned sz = nir_alu_type_get_type_size(ins->src_types[i]);
94 bool half = (sz == (base_size >> 1));
95
96 return integer ?
97 mir_get_imod(ins->src_shift[i], ins->src_types[i], half, scalar) :
98 ((ins->src_abs[i] << 0) |
99 ((ins->src_neg[i] << 1)));
100 }
101
102 /* Midgard IR only knows vector ALU types, but we sometimes need to actually
103 * use scalar ALU instructions, for functional or performance reasons. To do
104 * this, we just demote vector ALU payloads to scalar. */
105
106 static int
component_from_mask(unsigned mask)107 component_from_mask(unsigned mask)
108 {
109 for (int c = 0; c < 8; ++c) {
110 if (mask & (1 << c))
111 return c;
112 }
113
114 assert(0);
115 return 0;
116 }
117
118 static unsigned
mir_pack_scalar_source(unsigned mod,bool is_full,unsigned component)119 mir_pack_scalar_source(unsigned mod, bool is_full, unsigned component)
120 {
121 midgard_scalar_alu_src s = {
122 .mod = mod,
123 .full = is_full,
124 .component = component << (is_full ? 1 : 0)
125 };
126
127 unsigned o;
128 memcpy(&o, &s, sizeof(s));
129
130 return o & ((1 << 6) - 1);
131 }
132
133 static midgard_scalar_alu
vector_to_scalar_alu(midgard_vector_alu v,midgard_instruction * ins)134 vector_to_scalar_alu(midgard_vector_alu v, midgard_instruction *ins)
135 {
136 bool is_full = nir_alu_type_get_type_size(ins->dest_type) == 32;
137
138 bool half_0 = nir_alu_type_get_type_size(ins->src_types[0]) == 16;
139 bool half_1 = nir_alu_type_get_type_size(ins->src_types[1]) == 16;
140 unsigned comp = component_from_mask(ins->mask);
141
142 unsigned packed_src[2] = {
143 mir_pack_scalar_source(mir_pack_mod(ins, 0, true), !half_0, ins->swizzle[0][comp]),
144 mir_pack_scalar_source(mir_pack_mod(ins, 1, true), !half_1, ins->swizzle[1][comp])
145 };
146
147 /* The output component is from the mask */
148 midgard_scalar_alu s = {
149 .op = v.op,
150 .src1 = packed_src[0],
151 .src2 = packed_src[1],
152 .unknown = 0,
153 .outmod = v.outmod,
154 .output_full = is_full,
155 .output_component = comp
156 };
157
158 /* Full components are physically spaced out */
159 if (is_full) {
160 assert(s.output_component < 4);
161 s.output_component <<= 1;
162 }
163
164 /* Inline constant is passed along rather than trying to extract it
165 * from v */
166
167 if (ins->has_inline_constant) {
168 uint16_t imm = 0;
169 int lower_11 = ins->inline_constant & ((1 << 12) - 1);
170 imm |= (lower_11 >> 9) & 3;
171 imm |= (lower_11 >> 6) & 4;
172 imm |= (lower_11 >> 2) & 0x38;
173 imm |= (lower_11 & 63) << 6;
174
175 s.src2 = imm;
176 }
177
178 return s;
179 }
180
181 /* 64-bit swizzles are super easy since there are 2 components of 2 components
182 * in an 8-bit field ... lots of duplication to go around!
183 *
184 * Swizzles of 32-bit vectors accessed from 64-bit instructions are a little
185 * funny -- pack them *as if* they were native 64-bit, using rep_* flags to
186 * flag upper. For instance, xy would become 64-bit XY but that's just xyzw
187 * native. Likewise, zz would become 64-bit XX with rep* so it would be xyxy
188 * with rep. Pretty nifty, huh? */
189
190 static unsigned
mir_pack_swizzle_64(unsigned * swizzle,unsigned max_component)191 mir_pack_swizzle_64(unsigned *swizzle, unsigned max_component)
192 {
193 unsigned packed = 0;
194
195 for (unsigned i = 0; i < 2; ++i) {
196 assert(swizzle[i] <= max_component);
197
198 unsigned a = (swizzle[i] & 1) ?
199 (COMPONENT_W << 2) | COMPONENT_Z :
200 (COMPONENT_Y << 2) | COMPONENT_X;
201
202 packed |= a << (i * 4);
203 }
204
205 return packed;
206 }
207
208 static void
mir_pack_mask_alu(midgard_instruction * ins,midgard_vector_alu * alu)209 mir_pack_mask_alu(midgard_instruction *ins, midgard_vector_alu *alu)
210 {
211 unsigned effective = ins->mask;
212
213 /* If we have a destination override, we need to figure out whether to
214 * override to the lower or upper half, shifting the effective mask in
215 * the latter, so AAAA.... becomes AAAA */
216
217 unsigned inst_size = max_bitsize_for_alu(ins);
218 signed upper_shift = mir_upper_override(ins, inst_size);
219
220 if (upper_shift >= 0) {
221 effective >>= upper_shift;
222 alu->shrink_mode = upper_shift ?
223 midgard_shrink_mode_upper :
224 midgard_shrink_mode_lower;
225 } else {
226 alu->shrink_mode = midgard_shrink_mode_none;
227 }
228
229 if (inst_size == 32)
230 alu->mask = expand_writemask(effective, 2);
231 else if (inst_size == 64)
232 alu->mask = expand_writemask(effective, 1);
233 else
234 alu->mask = effective;
235 }
236
237 static unsigned
mir_pack_swizzle(unsigned mask,unsigned * swizzle,unsigned sz,unsigned base_size,bool op_channeled,midgard_src_expand_mode * expand_mode)238 mir_pack_swizzle(unsigned mask, unsigned *swizzle,
239 unsigned sz, unsigned base_size,
240 bool op_channeled, midgard_src_expand_mode *expand_mode)
241 {
242 unsigned packed = 0;
243
244 *expand_mode = midgard_src_passthrough;
245
246 midgard_reg_mode reg_mode = reg_mode_for_bitsize(base_size);
247
248 if (reg_mode == midgard_reg_mode_64) {
249 assert(sz == 64 || sz == 32);
250 unsigned components = (sz == 32) ? 4 : 2;
251
252 packed = mir_pack_swizzle_64(swizzle, components);
253
254 if (sz == 32) {
255 bool lo = swizzle[0] >= COMPONENT_Z;
256 bool hi = swizzle[1] >= COMPONENT_Z;
257
258 if (mask & 0x1) {
259 /* We can't mix halves... */
260 if (mask & 2)
261 assert(lo == hi);
262
263 *expand_mode = lo ? midgard_src_expand_high :
264 midgard_src_expand_low;
265 } else {
266 *expand_mode = hi ? midgard_src_expand_high :
267 midgard_src_expand_low;
268 }
269 } else if (sz < 32) {
270 unreachable("Cannot encode 8/16 swizzle in 64-bit");
271 }
272 } else {
273 /* For 32-bit, swizzle packing is stupid-simple. For 16-bit,
274 * the strategy is to check whether the nibble we're on is
275 * upper or lower. We need all components to be on the same
276 * "side"; that much is enforced by the ISA and should have
277 * been lowered. TODO: 8-bit packing. TODO: vec8 */
278
279 unsigned first = mask ? ffs(mask) - 1 : 0;
280 bool upper = swizzle[first] > 3;
281
282 if (upper && mask)
283 assert(sz <= 16);
284
285 bool dest_up = !op_channeled && (first >= 4);
286
287 for (unsigned c = (dest_up ? 4 : 0); c < (dest_up ? 8 : 4); ++c) {
288 unsigned v = swizzle[c];
289
290 ASSERTED bool t_upper = v > 3;
291
292 /* Ensure we're doing something sane */
293
294 if (mask & (1 << c)) {
295 assert(t_upper == upper);
296 assert(v <= 7);
297 }
298
299 /* Use the non upper part */
300 v &= 0x3;
301
302 packed |= v << (2 * (c % 4));
303 }
304
305
306 /* Replicate for now.. should really pick a side for
307 * dot products */
308
309 if (reg_mode == midgard_reg_mode_16 && sz == 16) {
310 *expand_mode = upper ? midgard_src_rep_high :
311 midgard_src_rep_low;
312 } else if (reg_mode == midgard_reg_mode_16 && sz == 8) {
313 if (base_size == 16) {
314 *expand_mode = upper ? midgard_src_expand_high :
315 midgard_src_expand_low;
316 } else if (upper) {
317 *expand_mode = midgard_src_swap;
318 }
319 } else if (reg_mode == midgard_reg_mode_32 && sz == 16) {
320 *expand_mode = upper ? midgard_src_expand_high :
321 midgard_src_expand_low;
322 } else if (reg_mode == midgard_reg_mode_8) {
323 unreachable("Unhandled reg mode");
324 }
325 }
326
327 return packed;
328 }
329
330 static void
mir_pack_vector_srcs(midgard_instruction * ins,midgard_vector_alu * alu)331 mir_pack_vector_srcs(midgard_instruction *ins, midgard_vector_alu *alu)
332 {
333 bool channeled = GET_CHANNEL_COUNT(alu_opcode_props[ins->op].props);
334
335 unsigned base_size = max_bitsize_for_alu(ins);
336
337 for (unsigned i = 0; i < 2; ++i) {
338 if (ins->has_inline_constant && (i == 1))
339 continue;
340
341 if (ins->src[i] == ~0)
342 continue;
343
344 unsigned sz = nir_alu_type_get_type_size(ins->src_types[i]);
345 assert((sz == base_size) || (sz == base_size / 2));
346
347 midgard_src_expand_mode expand_mode = midgard_src_passthrough;
348 unsigned swizzle = mir_pack_swizzle(ins->mask, ins->swizzle[i],
349 sz, base_size, channeled,
350 &expand_mode);
351
352 midgard_vector_alu_src pack = {
353 .mod = mir_pack_mod(ins, i, false),
354 .expand_mode = expand_mode,
355 .swizzle = swizzle
356 };
357
358 unsigned p = vector_alu_srco_unsigned(pack);
359
360 if (i == 0)
361 alu->src1 = p;
362 else
363 alu->src2 = p;
364 }
365 }
366
367 static void
mir_pack_swizzle_ldst(midgard_instruction * ins)368 mir_pack_swizzle_ldst(midgard_instruction *ins)
369 {
370 /* TODO: non-32-bit, non-vec4 */
371 for (unsigned c = 0; c < 4; ++c) {
372 unsigned v = ins->swizzle[0][c];
373
374 /* Check vec4 */
375 assert(v <= 3);
376
377 ins->load_store.swizzle |= v << (2 * c);
378 }
379
380 /* TODO: arg_1/2 */
381 }
382
383 static void
mir_pack_swizzle_tex(midgard_instruction * ins)384 mir_pack_swizzle_tex(midgard_instruction *ins)
385 {
386 for (unsigned i = 0; i < 2; ++i) {
387 unsigned packed = 0;
388
389 for (unsigned c = 0; c < 4; ++c) {
390 unsigned v = ins->swizzle[i][c];
391
392 /* Check vec4 */
393 assert(v <= 3);
394
395 packed |= v << (2 * c);
396 }
397
398 if (i == 0)
399 ins->texture.swizzle = packed;
400 else
401 ins->texture.in_reg_swizzle = packed;
402 }
403
404 /* TODO: bias component */
405 }
406
407 /* Up to 3 { ALU, LDST } bundles can execute in parallel with a texture op.
408 * Given a texture op, lookahead to see how many such bundles we can flag for
409 * OoO execution */
410
411 static bool
mir_can_run_ooo(midgard_block * block,midgard_bundle * bundle,unsigned dependency)412 mir_can_run_ooo(midgard_block *block, midgard_bundle *bundle,
413 unsigned dependency)
414 {
415 /* Don't read out of bounds */
416 if (bundle >= (midgard_bundle *) ((char *) block->bundles.data + block->bundles.size))
417 return false;
418
419 /* Texture ops can't execute with other texture ops */
420 if (!IS_ALU(bundle->tag) && bundle->tag != TAG_LOAD_STORE_4)
421 return false;
422
423 /* Ensure there is no read-after-write dependency */
424
425 for (unsigned i = 0; i < bundle->instruction_count; ++i) {
426 midgard_instruction *ins = bundle->instructions[i];
427
428 mir_foreach_src(ins, s) {
429 if (ins->src[s] == dependency)
430 return false;
431 }
432 }
433
434 /* Otherwise, we're okay */
435 return true;
436 }
437
438 static void
mir_pack_tex_ooo(midgard_block * block,midgard_bundle * bundle,midgard_instruction * ins)439 mir_pack_tex_ooo(midgard_block *block, midgard_bundle *bundle, midgard_instruction *ins)
440 {
441 unsigned count = 0;
442
443 for (count = 0; count < 3; ++count) {
444 if (!mir_can_run_ooo(block, bundle + count + 1, ins->dest))
445 break;
446 }
447
448 ins->texture.out_of_order = count;
449 }
450
451 /* Load store masks are 4-bits. Load/store ops pack for that.
452 * For most operations, vec4 is the natural mask width; vec8 is constrained to
453 * be in pairs, vec2 is duplicated. TODO: 8-bit?
454 * For common stores (i.e. ST.*), each bit masks a single byte in the 32-bit
455 * case, 2 bytes in the 64-bit case and 4 bytes in the 128-bit case.
456 */
457
458 static unsigned
midgard_pack_common_store_mask(midgard_instruction * ins)459 midgard_pack_common_store_mask(midgard_instruction *ins) {
460 unsigned comp_sz = nir_alu_type_get_type_size(ins->dest_type);
461 unsigned mask = ins->mask;
462 unsigned packed = 0;
463 unsigned nr_comp;
464
465 switch (ins->op) {
466 case midgard_op_st_u8:
467 packed |= mask & 1;
468 break;
469 case midgard_op_st_u16:
470 nr_comp = 16 / comp_sz;
471 for (int i = 0; i < nr_comp; i++) {
472 if (mask & (1 << i)) {
473 if (comp_sz == 16)
474 packed |= 0x3;
475 else if (comp_sz == 8)
476 packed |= 1 << i;
477 }
478 }
479 break;
480 case midgard_op_st_32:
481 case midgard_op_st_64:
482 case midgard_op_st_128: {
483 unsigned total_sz = 32;
484 if (ins->op == midgard_op_st_128)
485 total_sz = 128;
486 else if (ins->op == midgard_op_st_64)
487 total_sz = 64;
488
489 nr_comp = total_sz / comp_sz;
490
491 /* Each writemask bit masks 1/4th of the value to be stored. */
492 assert(comp_sz >= total_sz / 4);
493
494 for (int i = 0; i < nr_comp; i++) {
495 if (mask & (1 << i)) {
496 if (comp_sz == total_sz)
497 packed |= 0xF;
498 else if (comp_sz == total_sz / 2)
499 packed |= 0x3 << (i * 2);
500 else if (comp_sz == total_sz / 4)
501 packed |= 0x1 << i;
502 }
503 }
504 break;
505 }
506 default:
507 unreachable("unexpected ldst opcode");
508 }
509
510 return packed;
511 }
512
513 static void
mir_pack_ldst_mask(midgard_instruction * ins)514 mir_pack_ldst_mask(midgard_instruction *ins)
515 {
516 unsigned sz = nir_alu_type_get_type_size(ins->dest_type);
517 unsigned packed = ins->mask;
518
519 if (OP_IS_COMMON_STORE(ins->op)) {
520 packed = midgard_pack_common_store_mask(ins);
521 } else {
522 if (sz == 64) {
523 packed = ((ins->mask & 0x2) ? (0x8 | 0x4) : 0) |
524 ((ins->mask & 0x1) ? (0x2 | 0x1) : 0);
525 } else if (sz == 16) {
526 packed = 0;
527
528 for (unsigned i = 0; i < 4; ++i) {
529 /* Make sure we're duplicated */
530 bool u = (ins->mask & (1 << (2*i + 0))) != 0;
531 ASSERTED bool v = (ins->mask & (1 << (2*i + 1))) != 0;
532 assert(u == v);
533
534 packed |= (u << i);
535 }
536 } else {
537 assert(sz == 32);
538 }
539 }
540
541 ins->load_store.mask = packed;
542 }
543
544 static void
mir_lower_inverts(midgard_instruction * ins)545 mir_lower_inverts(midgard_instruction *ins)
546 {
547 bool inv[3] = {
548 ins->src_invert[0],
549 ins->src_invert[1],
550 ins->src_invert[2]
551 };
552
553 switch (ins->op) {
554 case midgard_alu_op_iand:
555 /* a & ~b = iandnot(a, b) */
556 /* ~a & ~b = ~(a | b) = inor(a, b) */
557
558 if (inv[0] && inv[1])
559 ins->op = midgard_alu_op_inor;
560 else if (inv[1])
561 ins->op = midgard_alu_op_iandnot;
562
563 break;
564 case midgard_alu_op_ior:
565 /* a | ~b = iornot(a, b) */
566 /* ~a | ~b = ~(a & b) = inand(a, b) */
567
568 if (inv[0] && inv[1])
569 ins->op = midgard_alu_op_inand;
570 else if (inv[1])
571 ins->op = midgard_alu_op_iornot;
572
573 break;
574
575 case midgard_alu_op_ixor:
576 /* ~a ^ b = a ^ ~b = ~(a ^ b) = inxor(a, b) */
577 /* ~a ^ ~b = a ^ b */
578
579 if (inv[0] ^ inv[1])
580 ins->op = midgard_alu_op_inxor;
581
582 break;
583
584 default:
585 break;
586 }
587 }
588
589 /* Opcodes with ROUNDS are the base (rte/0) type so we can just add */
590
591 static void
mir_lower_roundmode(midgard_instruction * ins)592 mir_lower_roundmode(midgard_instruction *ins)
593 {
594 if (alu_opcode_props[ins->op].props & MIDGARD_ROUNDS) {
595 assert(ins->roundmode <= 0x3);
596 ins->op += ins->roundmode;
597 }
598 }
599
600 static midgard_load_store_word
load_store_from_instr(midgard_instruction * ins)601 load_store_from_instr(midgard_instruction *ins)
602 {
603 midgard_load_store_word ldst = ins->load_store;
604 ldst.op = ins->op;
605
606 if (OP_IS_STORE(ldst.op)) {
607 ldst.reg = SSA_REG_FROM_FIXED(ins->src[0]) & 1;
608 } else {
609 ldst.reg = SSA_REG_FROM_FIXED(ins->dest);
610 }
611
612 /* Atomic opcode swizzles have a special meaning:
613 * - The first two bits say which component of the implicit register should be used
614 * - The next two bits say if the implicit register is r26 or r27 */
615 if (OP_IS_ATOMIC(ins->op)) {
616 ldst.swizzle = 0;
617 ldst.swizzle |= ins->swizzle[3][0] & 3;
618 ldst.swizzle |= (SSA_REG_FROM_FIXED(ins->src[3]) & 1 ? 1 : 0) << 2;
619 }
620
621 if (ins->src[1] != ~0) {
622 ldst.arg_reg = SSA_REG_FROM_FIXED(ins->src[1]) - REGISTER_LDST_BASE;
623 unsigned sz = nir_alu_type_get_type_size(ins->src_types[1]);
624 ldst.arg_comp = midgard_ldst_comp(ldst.arg_reg, ins->swizzle[1][0], sz);
625 }
626
627 if (ins->src[2] != ~0) {
628 ldst.index_reg = SSA_REG_FROM_FIXED(ins->src[2]) - REGISTER_LDST_BASE;
629 unsigned sz = nir_alu_type_get_type_size(ins->src_types[2]);
630 ldst.index_comp = midgard_ldst_comp(ldst.index_reg, ins->swizzle[2][0], sz);
631 }
632
633 return ldst;
634 }
635
636 static midgard_texture_word
texture_word_from_instr(midgard_instruction * ins)637 texture_word_from_instr(midgard_instruction *ins)
638 {
639 midgard_texture_word tex = ins->texture;
640 tex.op = ins->op;
641
642 unsigned src1 = ins->src[1] == ~0 ? REGISTER_UNUSED : SSA_REG_FROM_FIXED(ins->src[1]);
643 tex.in_reg_select = src1 & 1;
644
645 unsigned dest = ins->dest == ~0 ? REGISTER_UNUSED : SSA_REG_FROM_FIXED(ins->dest);
646 tex.out_reg_select = dest & 1;
647
648 if (ins->src[2] != ~0) {
649 midgard_tex_register_select sel = {
650 .select = SSA_REG_FROM_FIXED(ins->src[2]) & 1,
651 .full = 1,
652 .component = ins->swizzle[2][0]
653 };
654 uint8_t packed;
655 memcpy(&packed, &sel, sizeof(packed));
656 tex.bias = packed;
657 }
658
659 if (ins->src[3] != ~0) {
660 unsigned x = ins->swizzle[3][0];
661 unsigned y = x + 1;
662 unsigned z = x + 2;
663
664 /* Check range, TODO: half-registers */
665 assert(z < 4);
666
667 unsigned offset_reg = SSA_REG_FROM_FIXED(ins->src[3]);
668 tex.offset =
669 (1) | /* full */
670 (offset_reg & 1) << 1 | /* select */
671 (0 << 2) | /* upper */
672 (x << 3) | /* swizzle */
673 (y << 5) | /* swizzle */
674 (z << 7); /* swizzle */
675 }
676
677 return tex;
678 }
679
680 static midgard_vector_alu
vector_alu_from_instr(midgard_instruction * ins)681 vector_alu_from_instr(midgard_instruction *ins)
682 {
683 midgard_vector_alu alu = {
684 .op = ins->op,
685 .outmod = ins->outmod,
686 .reg_mode = reg_mode_for_bitsize(max_bitsize_for_alu(ins))
687 };
688
689 if (ins->has_inline_constant) {
690 /* Encode inline 16-bit constant. See disassembler for
691 * where the algorithm is from */
692
693 int lower_11 = ins->inline_constant & ((1 << 12) - 1);
694 uint16_t imm = ((lower_11 >> 8) & 0x7) |
695 ((lower_11 & 0xFF) << 3);
696
697 alu.src2 = imm << 2;
698 }
699
700 return alu;
701 }
702
703 static midgard_branch_extended
midgard_create_branch_extended(midgard_condition cond,midgard_jmp_writeout_op op,unsigned dest_tag,signed quadword_offset)704 midgard_create_branch_extended( midgard_condition cond,
705 midgard_jmp_writeout_op op,
706 unsigned dest_tag,
707 signed quadword_offset)
708 {
709 /* The condition code is actually a LUT describing a function to
710 * combine multiple condition codes. However, we only support a single
711 * condition code at the moment, so we just duplicate over a bunch of
712 * times. */
713
714 uint16_t duplicated_cond =
715 (cond << 14) |
716 (cond << 12) |
717 (cond << 10) |
718 (cond << 8) |
719 (cond << 6) |
720 (cond << 4) |
721 (cond << 2) |
722 (cond << 0);
723
724 midgard_branch_extended branch = {
725 .op = op,
726 .dest_tag = dest_tag,
727 .offset = quadword_offset,
728 .cond = duplicated_cond
729 };
730
731 return branch;
732 }
733
734 static void
emit_branch(midgard_instruction * ins,compiler_context * ctx,midgard_block * block,midgard_bundle * bundle,struct util_dynarray * emission)735 emit_branch(midgard_instruction *ins,
736 compiler_context *ctx,
737 midgard_block *block,
738 midgard_bundle *bundle,
739 struct util_dynarray *emission)
740 {
741 /* Parse some basic branch info */
742 bool is_compact = ins->unit == ALU_ENAB_BR_COMPACT;
743 bool is_conditional = ins->branch.conditional;
744 bool is_inverted = ins->branch.invert_conditional;
745 bool is_discard = ins->branch.target_type == TARGET_DISCARD;
746 bool is_tilebuf_wait = ins->branch.target_type == TARGET_TILEBUF_WAIT;
747 bool is_special = is_discard || is_tilebuf_wait;
748 bool is_writeout = ins->writeout;
749
750 /* Determine the block we're jumping to */
751 int target_number = ins->branch.target_block;
752
753 /* Report the destination tag */
754 int dest_tag = is_discard ? 0 :
755 is_tilebuf_wait ? bundle->tag :
756 midgard_get_first_tag_from_block(ctx, target_number);
757
758 /* Count up the number of quadwords we're
759 * jumping over = number of quadwords until
760 * (br_block_idx, target_number) */
761
762 int quadword_offset = 0;
763
764 if (is_discard) {
765 /* Fixed encoding, not actually an offset */
766 quadword_offset = 0x2;
767 } else if (is_tilebuf_wait) {
768 quadword_offset = -1;
769 } else if (target_number > block->base.name) {
770 /* Jump forward */
771
772 for (int idx = block->base.name+1; idx < target_number; ++idx) {
773 midgard_block *blk = mir_get_block(ctx, idx);
774 assert(blk);
775
776 quadword_offset += blk->quadword_count;
777 }
778 } else {
779 /* Jump backwards */
780
781 for (int idx = block->base.name; idx >= target_number; --idx) {
782 midgard_block *blk = mir_get_block(ctx, idx);
783 assert(blk);
784
785 quadword_offset -= blk->quadword_count;
786 }
787 }
788
789 /* Unconditional extended branches (far jumps)
790 * have issues, so we always use a conditional
791 * branch, setting the condition to always for
792 * unconditional. For compact unconditional
793 * branches, cond isn't used so it doesn't
794 * matter what we pick. */
795
796 midgard_condition cond =
797 !is_conditional ? midgard_condition_always :
798 is_inverted ? midgard_condition_false :
799 midgard_condition_true;
800
801 midgard_jmp_writeout_op op =
802 is_discard ? midgard_jmp_writeout_op_discard :
803 is_tilebuf_wait ? midgard_jmp_writeout_op_tilebuffer_pending :
804 is_writeout ? midgard_jmp_writeout_op_writeout :
805 (is_compact && !is_conditional) ?
806 midgard_jmp_writeout_op_branch_uncond :
807 midgard_jmp_writeout_op_branch_cond;
808
809 if (is_compact) {
810 unsigned size = sizeof(midgard_branch_cond);
811
812 if (is_conditional || is_special) {
813 midgard_branch_cond branch = {
814 .op = op,
815 .dest_tag = dest_tag,
816 .offset = quadword_offset,
817 .cond = cond
818 };
819 memcpy(util_dynarray_grow_bytes(emission, size, 1), &branch, size);
820 } else {
821 assert(op == midgard_jmp_writeout_op_branch_uncond);
822 midgard_branch_uncond branch = {
823 .op = op,
824 .dest_tag = dest_tag,
825 .offset = quadword_offset,
826 .unknown = 1
827 };
828 assert(branch.offset == quadword_offset);
829 memcpy(util_dynarray_grow_bytes(emission, size, 1), &branch, size);
830 }
831 } else { /* `ins->compact_branch`, misnomer */
832 unsigned size = sizeof(midgard_branch_extended);
833
834 midgard_branch_extended branch =
835 midgard_create_branch_extended(
836 cond, op,
837 dest_tag,
838 quadword_offset);
839
840 memcpy(util_dynarray_grow_bytes(emission, size, 1), &branch, size);
841 }
842 }
843
844 static void
emit_alu_bundle(compiler_context * ctx,midgard_block * block,midgard_bundle * bundle,struct util_dynarray * emission,unsigned lookahead)845 emit_alu_bundle(compiler_context *ctx,
846 midgard_block *block,
847 midgard_bundle *bundle,
848 struct util_dynarray *emission,
849 unsigned lookahead)
850 {
851 /* Emit the control word */
852 util_dynarray_append(emission, uint32_t, bundle->control | lookahead);
853
854 /* Next up, emit register words */
855 for (unsigned i = 0; i < bundle->instruction_count; ++i) {
856 midgard_instruction *ins = bundle->instructions[i];
857
858 /* Check if this instruction has registers */
859 if (ins->compact_branch) continue;
860
861 unsigned src2_reg = REGISTER_UNUSED;
862 if (ins->has_inline_constant)
863 src2_reg = ins->inline_constant >> 11;
864 else if (ins->src[1] != ~0)
865 src2_reg = SSA_REG_FROM_FIXED(ins->src[1]);
866
867 /* Otherwise, just emit the registers */
868 uint16_t reg_word = 0;
869 midgard_reg_info registers = {
870 .src1_reg = (ins->src[0] == ~0 ?
871 REGISTER_UNUSED :
872 SSA_REG_FROM_FIXED(ins->src[0])),
873 .src2_reg = src2_reg,
874 .src2_imm = ins->has_inline_constant,
875 .out_reg = (ins->dest == ~0 ?
876 REGISTER_UNUSED :
877 SSA_REG_FROM_FIXED(ins->dest)),
878 };
879 memcpy(®_word, ®isters, sizeof(uint16_t));
880 util_dynarray_append(emission, uint16_t, reg_word);
881 }
882
883 /* Now, we emit the body itself */
884 for (unsigned i = 0; i < bundle->instruction_count; ++i) {
885 midgard_instruction *ins = bundle->instructions[i];
886
887 if (!ins->compact_branch) {
888 mir_lower_inverts(ins);
889 mir_lower_roundmode(ins);
890 }
891
892 if (midgard_is_branch_unit(ins->unit)) {
893 emit_branch(ins, ctx, block, bundle, emission);
894 } else if (ins->unit & UNITS_ANY_VECTOR) {
895 midgard_vector_alu source = vector_alu_from_instr(ins);
896 mir_pack_mask_alu(ins, &source);
897 mir_pack_vector_srcs(ins, &source);
898 unsigned size = sizeof(source);
899 memcpy(util_dynarray_grow_bytes(emission, size, 1), &source, size);
900 } else {
901 midgard_scalar_alu source = vector_to_scalar_alu(vector_alu_from_instr(ins), ins);
902 unsigned size = sizeof(source);
903 memcpy(util_dynarray_grow_bytes(emission, size, 1), &source, size);
904 }
905 }
906
907 /* Emit padding (all zero) */
908 if (bundle->padding) {
909 memset(util_dynarray_grow_bytes(emission, bundle->padding, 1),
910 0, bundle->padding);
911 }
912
913 /* Tack on constants */
914
915 if (bundle->has_embedded_constants)
916 util_dynarray_append(emission, midgard_constants, bundle->constants);
917 }
918
919 /* Shift applied to the immediate used as an offset. Probably this is papering
920 * over some other semantic distinction else well, but it unifies things in the
921 * compiler so I don't mind. */
922
923 static void
mir_ldst_pack_offset(midgard_instruction * ins,int offset)924 mir_ldst_pack_offset(midgard_instruction *ins, int offset)
925 {
926 /* These opcodes don't support offsets */
927 assert(!OP_IS_REG2REG_LDST(ins->op) ||
928 ins->op == midgard_op_lea ||
929 ins->op == midgard_op_lea_image);
930
931 if (OP_IS_UBO_READ(ins->op))
932 ins->load_store.signed_offset |= PACK_LDST_UBO_OFS(offset);
933 else if (OP_IS_IMAGE(ins->op))
934 ins->load_store.signed_offset |= PACK_LDST_ATTRIB_OFS(offset);
935 else if (OP_IS_SPECIAL(ins->op))
936 ins->load_store.signed_offset |= PACK_LDST_SELECTOR_OFS(offset);
937 else
938 ins->load_store.signed_offset |= PACK_LDST_MEM_OFS(offset);
939 }
940
941 static enum mali_sampler_type
midgard_sampler_type(nir_alu_type t)942 midgard_sampler_type(nir_alu_type t) {
943 switch (nir_alu_type_get_base_type(t))
944 {
945 case nir_type_float:
946 return MALI_SAMPLER_FLOAT;
947 case nir_type_int:
948 return MALI_SAMPLER_SIGNED;
949 case nir_type_uint:
950 return MALI_SAMPLER_UNSIGNED;
951 default:
952 unreachable("Unknown sampler type");
953 }
954 }
955
956 /* After everything is scheduled, emit whole bundles at a time */
957
958 void
emit_binary_bundle(compiler_context * ctx,midgard_block * block,midgard_bundle * bundle,struct util_dynarray * emission,int next_tag)959 emit_binary_bundle(compiler_context *ctx,
960 midgard_block *block,
961 midgard_bundle *bundle,
962 struct util_dynarray *emission,
963 int next_tag)
964 {
965 int lookahead = next_tag << 4;
966
967 switch (bundle->tag) {
968 case TAG_ALU_4:
969 case TAG_ALU_8:
970 case TAG_ALU_12:
971 case TAG_ALU_16:
972 case TAG_ALU_4 + 4:
973 case TAG_ALU_8 + 4:
974 case TAG_ALU_12 + 4:
975 case TAG_ALU_16 + 4:
976 emit_alu_bundle(ctx, block, bundle, emission, lookahead);
977 break;
978
979 case TAG_LOAD_STORE_4: {
980 /* One or two composing instructions */
981
982 uint64_t current64, next64 = LDST_NOP;
983
984 /* Copy masks */
985
986 for (unsigned i = 0; i < bundle->instruction_count; ++i) {
987 midgard_instruction *ins = bundle->instructions[i];
988 mir_pack_ldst_mask(ins);
989
990 /* Atomic ops don't use this swizzle the same way as other ops */
991 if (!OP_IS_ATOMIC(ins->op))
992 mir_pack_swizzle_ldst(ins);
993
994 /* Apply a constant offset */
995 unsigned offset = ins->constants.u32[0];
996 if (offset)
997 mir_ldst_pack_offset(ins, offset);
998 }
999
1000 midgard_load_store_word ldst0 =
1001 load_store_from_instr(bundle->instructions[0]);
1002 memcpy(¤t64, &ldst0, sizeof(current64));
1003
1004 if (bundle->instruction_count == 2) {
1005 midgard_load_store_word ldst1 =
1006 load_store_from_instr(bundle->instructions[1]);
1007 memcpy(&next64, &ldst1, sizeof(next64));
1008 }
1009
1010 midgard_load_store instruction = {
1011 .type = bundle->tag,
1012 .next_type = next_tag,
1013 .word1 = current64,
1014 .word2 = next64
1015 };
1016
1017 util_dynarray_append(emission, midgard_load_store, instruction);
1018
1019 break;
1020 }
1021
1022 case TAG_TEXTURE_4:
1023 case TAG_TEXTURE_4_VTX:
1024 case TAG_TEXTURE_4_BARRIER: {
1025 /* Texture instructions are easy, since there is no pipelining
1026 * nor VLIW to worry about. We may need to set .cont/.last
1027 * flags. */
1028
1029 midgard_instruction *ins = bundle->instructions[0];
1030
1031 ins->texture.type = bundle->tag;
1032 ins->texture.next_type = next_tag;
1033
1034 /* Nothing else to pack for barriers */
1035 if (ins->op == midgard_tex_op_barrier) {
1036 ins->texture.cont = ins->texture.last = 1;
1037 ins->texture.op = ins->op;
1038 util_dynarray_append(emission, midgard_texture_word, ins->texture);
1039 return;
1040 }
1041
1042 signed override = mir_upper_override(ins, 32);
1043
1044 ins->texture.mask = override > 0 ?
1045 ins->mask >> override :
1046 ins->mask;
1047
1048 mir_pack_swizzle_tex(ins);
1049
1050 if (!(ctx->quirks & MIDGARD_NO_OOO))
1051 mir_pack_tex_ooo(block, bundle, ins);
1052
1053 unsigned osz = nir_alu_type_get_type_size(ins->dest_type);
1054 unsigned isz = nir_alu_type_get_type_size(ins->src_types[1]);
1055
1056 assert(osz == 32 || osz == 16);
1057 assert(isz == 32 || isz == 16);
1058
1059 ins->texture.out_full = (osz == 32);
1060 ins->texture.out_upper = override > 0;
1061 ins->texture.in_reg_full = (isz == 32);
1062 ins->texture.sampler_type = midgard_sampler_type(ins->dest_type);
1063 ins->texture.outmod = ins->outmod;
1064
1065 if (mir_op_computes_derivatives(ctx->stage, ins->op)) {
1066 ins->texture.cont = !ins->helper_terminate;
1067 ins->texture.last = ins->helper_terminate || ins->helper_execute;
1068 } else {
1069 ins->texture.cont = ins->texture.last = 1;
1070 }
1071
1072 midgard_texture_word texture = texture_word_from_instr(ins);
1073 util_dynarray_append(emission, midgard_texture_word, texture);
1074 break;
1075 }
1076
1077 default:
1078 unreachable("Unknown midgard instruction type\n");
1079 }
1080 }
1081