1 #include <vector>
2 #include <algorithm>
3
4 #include "aco_ir.h"
5 #include "aco_builder.h"
6 #include "common/sid.h"
7 #include "ac_shader_util.h"
8 #include "util/memstream.h"
9 #include "util/u_math.h"
10
11 namespace aco {
12
13 struct asm_context {
14 Program *program;
15 enum chip_class chip_class;
16 std::vector<std::pair<int, SOPP_instruction*>> branches;
17 std::vector<unsigned> constaddrs;
18 const int16_t* opcode;
19 // TODO: keep track of branch instructions referring blocks
20 // and, when emitting the block, correct the offset in instr
asm_contextaco::asm_context21 asm_context(Program* program) : program(program), chip_class(program->chip_class) {
22 if (chip_class <= GFX7)
23 opcode = &instr_info.opcode_gfx7[0];
24 else if (chip_class <= GFX9)
25 opcode = &instr_info.opcode_gfx9[0];
26 else if (chip_class >= GFX10)
27 opcode = &instr_info.opcode_gfx10[0];
28 }
29
30 int subvector_begin_pos = -1;
31 };
32
get_sdwa_sel(unsigned sel,PhysReg reg)33 static uint32_t get_sdwa_sel(unsigned sel, PhysReg reg)
34 {
35 if (sel & sdwa_isra) {
36 unsigned size = sdwa_rasize & sel;
37 if (size == 1)
38 return reg.byte();
39 else /* size == 2 */
40 return sdwa_isword | (reg.byte() >> 1);
41 }
42 return sel & sdwa_asuint;
43 }
44
emit_instruction(asm_context & ctx,std::vector<uint32_t> & out,Instruction * instr)45 void emit_instruction(asm_context& ctx, std::vector<uint32_t>& out, Instruction* instr)
46 {
47 /* lower remaining pseudo-instructions */
48 if (instr->opcode == aco_opcode::p_constaddr) {
49 unsigned dest = instr->definitions[0].physReg();
50 unsigned offset = instr->operands[0].constantValue();
51
52 /* s_getpc_b64 dest[0:1] */
53 uint32_t encoding = (0b101111101 << 23);
54 uint32_t opcode = ctx.opcode[(int)aco_opcode::s_getpc_b64];
55 if (opcode >= 55 && ctx.chip_class <= GFX9) {
56 assert(ctx.chip_class == GFX9 && opcode < 60);
57 opcode = opcode - 4;
58 }
59 encoding |= dest << 16;
60 encoding |= opcode << 8;
61 out.push_back(encoding);
62
63 /* s_add_u32 dest[0], dest[0], ... */
64 encoding = (0b10 << 30);
65 encoding |= ctx.opcode[(int)aco_opcode::s_add_u32] << 23;
66 encoding |= dest << 16;
67 encoding |= dest;
68 encoding |= 255 << 8;
69 out.push_back(encoding);
70 ctx.constaddrs.push_back(out.size());
71 out.push_back(offset);
72
73 /* s_addc_u32 dest[1], dest[1], 0 */
74 encoding = (0b10 << 30);
75 encoding |= ctx.opcode[(int)aco_opcode::s_addc_u32] << 23;
76 encoding |= (dest + 1) << 16;
77 encoding |= dest + 1;
78 encoding |= 128 << 8;
79 out.push_back(encoding);
80 return;
81 }
82
83 uint32_t opcode = ctx.opcode[(int)instr->opcode];
84 if (opcode == (uint32_t)-1) {
85 char *out;
86 size_t outsize;
87 struct u_memstream mem;
88 u_memstream_open(&mem, &out, &outsize);
89 FILE *const memf = u_memstream_get(&mem);
90
91 fprintf(memf, "Unsupported opcode: ");
92 aco_print_instr(instr, memf);
93 u_memstream_close(&mem);
94
95 aco_err(ctx.program, out);
96 free(out);
97
98 abort();
99 }
100
101 switch (instr->format) {
102 case Format::SOP2: {
103 uint32_t encoding = (0b10 << 30);
104 encoding |= opcode << 23;
105 encoding |= !instr->definitions.empty() ? instr->definitions[0].physReg() << 16 : 0;
106 encoding |= instr->operands.size() >= 2 ? instr->operands[1].physReg() << 8 : 0;
107 encoding |= !instr->operands.empty() ? instr->operands[0].physReg() : 0;
108 out.push_back(encoding);
109 break;
110 }
111 case Format::SOPK: {
112 SOPK_instruction *sopk = static_cast<SOPK_instruction*>(instr);
113
114 if (instr->opcode == aco_opcode::s_subvector_loop_begin) {
115 assert(ctx.chip_class >= GFX10);
116 assert(ctx.subvector_begin_pos == -1);
117 ctx.subvector_begin_pos = out.size();
118 } else if (instr->opcode == aco_opcode::s_subvector_loop_end) {
119 assert(ctx.chip_class >= GFX10);
120 assert(ctx.subvector_begin_pos != -1);
121 /* Adjust s_subvector_loop_begin instruction to the address after the end */
122 out[ctx.subvector_begin_pos] |= (out.size() - ctx.subvector_begin_pos);
123 /* Adjust s_subvector_loop_end instruction to the address after the beginning */
124 sopk->imm = (uint16_t)(ctx.subvector_begin_pos - (int)out.size());
125 ctx.subvector_begin_pos = -1;
126 }
127
128 uint32_t encoding = (0b1011 << 28);
129 encoding |= opcode << 23;
130 encoding |=
131 !instr->definitions.empty() && !(instr->definitions[0].physReg() == scc) ?
132 instr->definitions[0].physReg() << 16 :
133 !instr->operands.empty() && instr->operands[0].physReg() <= 127 ?
134 instr->operands[0].physReg() << 16 : 0;
135 encoding |= sopk->imm;
136 out.push_back(encoding);
137 break;
138 }
139 case Format::SOP1: {
140 uint32_t encoding = (0b101111101 << 23);
141 if (opcode >= 55 && ctx.chip_class <= GFX9) {
142 assert(ctx.chip_class == GFX9 && opcode < 60);
143 opcode = opcode - 4;
144 }
145 encoding |= !instr->definitions.empty() ? instr->definitions[0].physReg() << 16 : 0;
146 encoding |= opcode << 8;
147 encoding |= !instr->operands.empty() ? instr->operands[0].physReg() : 0;
148 out.push_back(encoding);
149 break;
150 }
151 case Format::SOPC: {
152 uint32_t encoding = (0b101111110 << 23);
153 encoding |= opcode << 16;
154 encoding |= instr->operands.size() == 2 ? instr->operands[1].physReg() << 8 : 0;
155 encoding |= !instr->operands.empty() ? instr->operands[0].physReg() : 0;
156 out.push_back(encoding);
157 break;
158 }
159 case Format::SOPP: {
160 SOPP_instruction* sopp = static_cast<SOPP_instruction*>(instr);
161 uint32_t encoding = (0b101111111 << 23);
162 encoding |= opcode << 16;
163 encoding |= (uint16_t) sopp->imm;
164 if (sopp->block != -1) {
165 sopp->pass_flags = 0;
166 ctx.branches.emplace_back(out.size(), sopp);
167 }
168 out.push_back(encoding);
169 break;
170 }
171 case Format::SMEM: {
172 SMEM_instruction* smem = static_cast<SMEM_instruction*>(instr);
173 bool soe = instr->operands.size() >= (!instr->definitions.empty() ? 3 : 4);
174 bool is_load = !instr->definitions.empty();
175 uint32_t encoding = 0;
176
177 if (ctx.chip_class <= GFX7) {
178 encoding = (0b11000 << 27);
179 encoding |= opcode << 22;
180 encoding |= instr->definitions.size() ? instr->definitions[0].physReg() << 15 : 0;
181 encoding |= instr->operands.size() ? (instr->operands[0].physReg() >> 1) << 9 : 0;
182 if (instr->operands.size() >= 2) {
183 if (!instr->operands[1].isConstant() || instr->operands[1].constantValue() >= 1024) {
184 encoding |= instr->operands[1].physReg().reg();
185 } else {
186 encoding |= instr->operands[1].constantValue() >> 2;
187 encoding |= 1 << 8;
188 }
189 }
190 out.push_back(encoding);
191 /* SMRD instructions can take a literal on GFX6 & GFX7 */
192 if (instr->operands.size() >= 2 && instr->operands[1].isConstant() && instr->operands[1].constantValue() >= 1024)
193 out.push_back(instr->operands[1].constantValue() >> 2);
194 return;
195 }
196
197 if (ctx.chip_class <= GFX9) {
198 encoding = (0b110000 << 26);
199 assert(!smem->dlc); /* Device-level coherent is not supported on GFX9 and lower */
200 encoding |= smem->nv ? 1 << 15 : 0;
201 } else {
202 encoding = (0b111101 << 26);
203 assert(!smem->nv); /* Non-volatile is not supported on GFX10 */
204 encoding |= smem->dlc ? 1 << 14 : 0;
205 }
206
207 encoding |= opcode << 18;
208 encoding |= smem->glc ? 1 << 16 : 0;
209
210 if (ctx.chip_class <= GFX9) {
211 if (instr->operands.size() >= 2)
212 encoding |= instr->operands[1].isConstant() ? 1 << 17 : 0; /* IMM - immediate enable */
213 }
214 if (ctx.chip_class == GFX9) {
215 encoding |= soe ? 1 << 14 : 0;
216 }
217
218 if (is_load || instr->operands.size() >= 3) { /* SDATA */
219 encoding |= (is_load ? instr->definitions[0].physReg() : instr->operands[2].physReg()) << 6;
220 }
221 if (instr->operands.size() >= 1) { /* SBASE */
222 encoding |= instr->operands[0].physReg() >> 1;
223 }
224
225 out.push_back(encoding);
226 encoding = 0;
227
228 int32_t offset = 0;
229 uint32_t soffset = ctx.chip_class >= GFX10
230 ? sgpr_null /* On GFX10 this is disabled by specifying SGPR_NULL */
231 : 0; /* On GFX9, it is disabled by the SOE bit (and it's not present on GFX8 and below) */
232 if (instr->operands.size() >= 2) {
233 const Operand &op_off1 = instr->operands[1];
234 if (ctx.chip_class <= GFX9) {
235 offset = op_off1.isConstant() ? op_off1.constantValue() : op_off1.physReg();
236 } else {
237 /* GFX10 only supports constants in OFFSET, so put the operand in SOFFSET if it's an SGPR */
238 if (op_off1.isConstant()) {
239 offset = op_off1.constantValue();
240 } else {
241 soffset = op_off1.physReg();
242 assert(!soe); /* There is no place to put the other SGPR offset, if any */
243 }
244 }
245
246 if (soe) {
247 const Operand &op_off2 = instr->operands.back();
248 assert(ctx.chip_class >= GFX9); /* GFX8 and below don't support specifying a constant and an SGPR at the same time */
249 assert(!op_off2.isConstant());
250 soffset = op_off2.physReg();
251 }
252 }
253 encoding |= offset;
254 encoding |= soffset << 25;
255
256 out.push_back(encoding);
257 return;
258 }
259 case Format::VOP2: {
260 uint32_t encoding = 0;
261 encoding |= opcode << 25;
262 encoding |= (0xFF & instr->definitions[0].physReg()) << 17;
263 encoding |= (0xFF & instr->operands[1].physReg()) << 9;
264 encoding |= instr->operands[0].physReg();
265 out.push_back(encoding);
266 break;
267 }
268 case Format::VOP1: {
269 uint32_t encoding = (0b0111111 << 25);
270 if (!instr->definitions.empty())
271 encoding |= (0xFF & instr->definitions[0].physReg()) << 17;
272 encoding |= opcode << 9;
273 if (!instr->operands.empty())
274 encoding |= instr->operands[0].physReg();
275 out.push_back(encoding);
276 break;
277 }
278 case Format::VOPC: {
279 uint32_t encoding = (0b0111110 << 25);
280 encoding |= opcode << 17;
281 encoding |= (0xFF & instr->operands[1].physReg()) << 9;
282 encoding |= instr->operands[0].physReg();
283 out.push_back(encoding);
284 break;
285 }
286 case Format::VINTRP: {
287 Interp_instruction* interp = static_cast<Interp_instruction*>(instr);
288 uint32_t encoding = 0;
289
290 if (instr->opcode == aco_opcode::v_interp_p1ll_f16 ||
291 instr->opcode == aco_opcode::v_interp_p1lv_f16 ||
292 instr->opcode == aco_opcode::v_interp_p2_legacy_f16 ||
293 instr->opcode == aco_opcode::v_interp_p2_f16) {
294 if (ctx.chip_class == GFX8 || ctx.chip_class == GFX9) {
295 encoding = (0b110100 << 26);
296 } else if (ctx.chip_class >= GFX10) {
297 encoding = (0b110101 << 26);
298 } else {
299 unreachable("Unknown chip_class.");
300 }
301
302 encoding |= opcode << 16;
303 encoding |= (0xFF & instr->definitions[0].physReg());
304 out.push_back(encoding);
305
306 encoding = 0;
307 encoding |= interp->attribute;
308 encoding |= interp->component << 6;
309 encoding |= instr->operands[0].physReg() << 9;
310 if (instr->opcode == aco_opcode::v_interp_p2_f16 ||
311 instr->opcode == aco_opcode::v_interp_p2_legacy_f16 ||
312 instr->opcode == aco_opcode::v_interp_p1lv_f16) {
313 encoding |= instr->operands[2].physReg() << 18;
314 }
315 out.push_back(encoding);
316 } else {
317 if (ctx.chip_class == GFX8 || ctx.chip_class == GFX9) {
318 encoding = (0b110101 << 26); /* Vega ISA doc says 110010 but it's wrong */
319 } else {
320 encoding = (0b110010 << 26);
321 }
322
323 assert(encoding);
324 encoding |= (0xFF & instr->definitions[0].physReg()) << 18;
325 encoding |= opcode << 16;
326 encoding |= interp->attribute << 10;
327 encoding |= interp->component << 8;
328 if (instr->opcode == aco_opcode::v_interp_mov_f32)
329 encoding |= (0x3 & instr->operands[0].constantValue());
330 else
331 encoding |= (0xFF & instr->operands[0].physReg());
332 out.push_back(encoding);
333 }
334 break;
335 }
336 case Format::DS: {
337 DS_instruction* ds = static_cast<DS_instruction*>(instr);
338 uint32_t encoding = (0b110110 << 26);
339 if (ctx.chip_class == GFX8 || ctx.chip_class == GFX9) {
340 encoding |= opcode << 17;
341 encoding |= (ds->gds ? 1 : 0) << 16;
342 } else {
343 encoding |= opcode << 18;
344 encoding |= (ds->gds ? 1 : 0) << 17;
345 }
346 encoding |= ((0xFF & ds->offset1) << 8);
347 encoding |= (0xFFFF & ds->offset0);
348 out.push_back(encoding);
349 encoding = 0;
350 unsigned reg = !instr->definitions.empty() ? instr->definitions[0].physReg() : 0;
351 encoding |= (0xFF & reg) << 24;
352 reg = instr->operands.size() >= 3 && !(instr->operands[2].physReg() == m0) ? instr->operands[2].physReg() : 0;
353 encoding |= (0xFF & reg) << 16;
354 reg = instr->operands.size() >= 2 && !(instr->operands[1].physReg() == m0) ? instr->operands[1].physReg() : 0;
355 encoding |= (0xFF & reg) << 8;
356 encoding |= (0xFF & instr->operands[0].physReg());
357 out.push_back(encoding);
358 break;
359 }
360 case Format::MUBUF: {
361 MUBUF_instruction* mubuf = static_cast<MUBUF_instruction*>(instr);
362 uint32_t encoding = (0b111000 << 26);
363 encoding |= opcode << 18;
364 encoding |= (mubuf->lds ? 1 : 0) << 16;
365 encoding |= (mubuf->glc ? 1 : 0) << 14;
366 encoding |= (mubuf->idxen ? 1 : 0) << 13;
367 assert(!mubuf->addr64 || ctx.chip_class <= GFX7);
368 if (ctx.chip_class == GFX6 || ctx.chip_class == GFX7)
369 encoding |= (mubuf->addr64 ? 1 : 0) << 15;
370 encoding |= (mubuf->offen ? 1 : 0) << 12;
371 if (ctx.chip_class == GFX8 || ctx.chip_class == GFX9) {
372 assert(!mubuf->dlc); /* Device-level coherent is not supported on GFX9 and lower */
373 encoding |= (mubuf->slc ? 1 : 0) << 17;
374 } else if (ctx.chip_class >= GFX10) {
375 encoding |= (mubuf->dlc ? 1 : 0) << 15;
376 }
377 encoding |= 0x0FFF & mubuf->offset;
378 out.push_back(encoding);
379 encoding = 0;
380 if (ctx.chip_class <= GFX7 || ctx.chip_class >= GFX10) {
381 encoding |= (mubuf->slc ? 1 : 0) << 22;
382 }
383 encoding |= instr->operands[2].physReg() << 24;
384 encoding |= (mubuf->tfe ? 1 : 0) << 23;
385 encoding |= (instr->operands[0].physReg() >> 2) << 16;
386 unsigned reg = instr->operands.size() > 3 ? instr->operands[3].physReg() : instr->definitions[0].physReg();
387 encoding |= (0xFF & reg) << 8;
388 encoding |= (0xFF & instr->operands[1].physReg());
389 out.push_back(encoding);
390 break;
391 }
392 case Format::MTBUF: {
393 MTBUF_instruction* mtbuf = static_cast<MTBUF_instruction*>(instr);
394
395 uint32_t img_format = ac_get_tbuffer_format(ctx.chip_class, mtbuf->dfmt, mtbuf->nfmt);
396 uint32_t encoding = (0b111010 << 26);
397 assert(img_format <= 0x7F);
398 assert(!mtbuf->dlc || ctx.chip_class >= GFX10);
399 encoding |= (mtbuf->dlc ? 1 : 0) << 15; /* DLC bit replaces one bit of the OPCODE on GFX10 */
400 encoding |= (mtbuf->glc ? 1 : 0) << 14;
401 encoding |= (mtbuf->idxen ? 1 : 0) << 13;
402 encoding |= (mtbuf->offen ? 1 : 0) << 12;
403 encoding |= 0x0FFF & mtbuf->offset;
404 encoding |= (img_format << 19); /* Handles both the GFX10 FORMAT and the old NFMT+DFMT */
405
406 if (ctx.chip_class == GFX8 || ctx.chip_class == GFX9) {
407 encoding |= opcode << 15;
408 } else {
409 encoding |= (opcode & 0x07) << 16; /* 3 LSBs of 4-bit OPCODE */
410 }
411
412 out.push_back(encoding);
413 encoding = 0;
414
415 encoding |= instr->operands[2].physReg() << 24;
416 encoding |= (mtbuf->tfe ? 1 : 0) << 23;
417 encoding |= (mtbuf->slc ? 1 : 0) << 22;
418 encoding |= (instr->operands[0].physReg() >> 2) << 16;
419 unsigned reg = instr->operands.size() > 3 ? instr->operands[3].physReg() : instr->definitions[0].physReg();
420 encoding |= (0xFF & reg) << 8;
421 encoding |= (0xFF & instr->operands[1].physReg());
422
423 if (ctx.chip_class >= GFX10) {
424 encoding |= (((opcode & 0x08) >> 3) << 21); /* MSB of 4-bit OPCODE */
425 }
426
427 out.push_back(encoding);
428 break;
429 }
430 case Format::MIMG: {
431 MIMG_instruction* mimg = static_cast<MIMG_instruction*>(instr);
432 uint32_t encoding = (0b111100 << 26);
433 encoding |= mimg->slc ? 1 << 25 : 0;
434 encoding |= opcode << 18;
435 encoding |= mimg->lwe ? 1 << 17 : 0;
436 encoding |= mimg->tfe ? 1 << 16 : 0;
437 encoding |= mimg->glc ? 1 << 13 : 0;
438 encoding |= mimg->unrm ? 1 << 12 : 0;
439 if (ctx.chip_class <= GFX9) {
440 assert(!mimg->dlc); /* Device-level coherent is not supported on GFX9 and lower */
441 assert(!mimg->r128);
442 encoding |= mimg->a16 ? 1 << 15 : 0;
443 encoding |= mimg->da ? 1 << 14 : 0;
444 } else {
445 encoding |= mimg->r128 ? 1 << 15 : 0; /* GFX10: A16 moved to 2nd word, R128 replaces it in 1st word */
446 encoding |= mimg->dim << 3; /* GFX10: dimensionality instead of declare array */
447 encoding |= mimg->dlc ? 1 << 7 : 0;
448 }
449 encoding |= (0xF & mimg->dmask) << 8;
450 out.push_back(encoding);
451 encoding = (0xFF & instr->operands[2].physReg()); /* VADDR */
452 if (!instr->definitions.empty()) {
453 encoding |= (0xFF & instr->definitions[0].physReg()) << 8; /* VDATA */
454 } else if (instr->operands[1].regClass().type() == RegType::vgpr) {
455 encoding |= (0xFF & instr->operands[1].physReg()) << 8; /* VDATA */
456 }
457 encoding |= (0x1F & (instr->operands[0].physReg() >> 2)) << 16; /* T# (resource) */
458 if (instr->operands[1].regClass().type() == RegType::sgpr)
459 encoding |= (0x1F & (instr->operands[1].physReg() >> 2)) << 21; /* sampler */
460
461 assert(!mimg->d16 || ctx.chip_class >= GFX9);
462 encoding |= mimg->d16 ? 1 << 15 : 0;
463 if (ctx.chip_class >= GFX10) {
464 encoding |= mimg->a16 ? 1 << 14 : 0; /* GFX10: A16 still exists, but is in a different place */
465 }
466
467 out.push_back(encoding);
468 break;
469 }
470 case Format::FLAT:
471 case Format::SCRATCH:
472 case Format::GLOBAL: {
473 FLAT_instruction *flat = static_cast<FLAT_instruction*>(instr);
474 uint32_t encoding = (0b110111 << 26);
475 encoding |= opcode << 18;
476 if (ctx.chip_class <= GFX9) {
477 assert(flat->offset <= 0x1fff);
478 encoding |= flat->offset & 0x1fff;
479 } else if (instr->format == Format::FLAT) {
480 /* GFX10 has a 12-bit immediate OFFSET field,
481 * but it has a hw bug: it ignores the offset, called FlatSegmentOffsetBug
482 */
483 assert(flat->offset == 0);
484 } else {
485 assert(flat->offset <= 0xfff);
486 encoding |= flat->offset & 0xfff;
487 }
488 if (instr->format == Format::SCRATCH)
489 encoding |= 1 << 14;
490 else if (instr->format == Format::GLOBAL)
491 encoding |= 2 << 14;
492 encoding |= flat->lds ? 1 << 13 : 0;
493 encoding |= flat->glc ? 1 << 16 : 0;
494 encoding |= flat->slc ? 1 << 17 : 0;
495 if (ctx.chip_class >= GFX10) {
496 assert(!flat->nv);
497 encoding |= flat->dlc ? 1 << 12 : 0;
498 } else {
499 assert(!flat->dlc);
500 }
501 out.push_back(encoding);
502 encoding = (0xFF & instr->operands[0].physReg());
503 if (!instr->definitions.empty())
504 encoding |= (0xFF & instr->definitions[0].physReg()) << 24;
505 if (instr->operands.size() >= 3)
506 encoding |= (0xFF & instr->operands[2].physReg()) << 8;
507 if (!instr->operands[1].isUndefined()) {
508 assert(ctx.chip_class >= GFX10 || instr->operands[1].physReg() != 0x7F);
509 assert(instr->format != Format::FLAT);
510 encoding |= instr->operands[1].physReg() << 16;
511 } else if (instr->format != Format::FLAT || ctx.chip_class >= GFX10) { /* SADDR is actually used with FLAT on GFX10 */
512 if (ctx.chip_class <= GFX9)
513 encoding |= 0x7F << 16;
514 else
515 encoding |= sgpr_null << 16;
516 }
517 encoding |= flat->nv ? 1 << 23 : 0;
518 out.push_back(encoding);
519 break;
520 }
521 case Format::EXP: {
522 Export_instruction* exp = static_cast<Export_instruction*>(instr);
523 uint32_t encoding;
524 if (ctx.chip_class == GFX8 || ctx.chip_class == GFX9) {
525 encoding = (0b110001 << 26);
526 } else {
527 encoding = (0b111110 << 26);
528 }
529
530 encoding |= exp->valid_mask ? 0b1 << 12 : 0;
531 encoding |= exp->done ? 0b1 << 11 : 0;
532 encoding |= exp->compressed ? 0b1 << 10 : 0;
533 encoding |= exp->dest << 4;
534 encoding |= exp->enabled_mask;
535 out.push_back(encoding);
536 encoding = 0xFF & exp->operands[0].physReg();
537 encoding |= (0xFF & exp->operands[1].physReg()) << 8;
538 encoding |= (0xFF & exp->operands[2].physReg()) << 16;
539 encoding |= (0xFF & exp->operands[3].physReg()) << 24;
540 out.push_back(encoding);
541 break;
542 }
543 case Format::PSEUDO:
544 case Format::PSEUDO_BARRIER:
545 if (instr->opcode != aco_opcode::p_unit_test)
546 unreachable("Pseudo instructions should be lowered before assembly.");
547 break;
548 default:
549 if ((uint16_t) instr->format & (uint16_t) Format::VOP3A) {
550 VOP3A_instruction* vop3 = static_cast<VOP3A_instruction*>(instr);
551
552 if ((uint16_t) instr->format & (uint16_t) Format::VOP2) {
553 opcode = opcode + 0x100;
554 } else if ((uint16_t) instr->format & (uint16_t) Format::VOP1) {
555 if (ctx.chip_class == GFX8 || ctx.chip_class == GFX9)
556 opcode = opcode + 0x140;
557 else
558 opcode = opcode + 0x180;
559 } else if ((uint16_t) instr->format & (uint16_t) Format::VOPC) {
560 opcode = opcode + 0x0;
561 } else if ((uint16_t) instr->format & (uint16_t) Format::VINTRP) {
562 opcode = opcode + 0x270;
563 }
564
565 uint32_t encoding;
566 if (ctx.chip_class <= GFX9) {
567 encoding = (0b110100 << 26);
568 } else if (ctx.chip_class >= GFX10) {
569 encoding = (0b110101 << 26);
570 } else {
571 unreachable("Unknown chip_class.");
572 }
573
574 if (ctx.chip_class <= GFX7) {
575 encoding |= opcode << 17;
576 encoding |= (vop3->clamp ? 1 : 0) << 11;
577 } else {
578 encoding |= opcode << 16;
579 encoding |= (vop3->clamp ? 1 : 0) << 15;
580 }
581 encoding |= vop3->opsel << 11;
582 for (unsigned i = 0; i < 3; i++)
583 encoding |= vop3->abs[i] << (8+i);
584 if (instr->definitions.size() == 2)
585 encoding |= instr->definitions[1].physReg() << 8;
586 encoding |= (0xFF & instr->definitions[0].physReg());
587 out.push_back(encoding);
588 encoding = 0;
589 if (instr->opcode == aco_opcode::v_interp_mov_f32) {
590 encoding = 0x3 & instr->operands[0].constantValue();
591 } else {
592 for (unsigned i = 0; i < instr->operands.size(); i++)
593 encoding |= instr->operands[i].physReg() << (i * 9);
594 }
595 encoding |= vop3->omod << 27;
596 for (unsigned i = 0; i < 3; i++)
597 encoding |= vop3->neg[i] << (29+i);
598 out.push_back(encoding);
599
600 } else if (instr->format == Format::VOP3P) {
601 VOP3P_instruction* vop3 = static_cast<VOP3P_instruction*>(instr);
602
603 uint32_t encoding;
604 if (ctx.chip_class == GFX9) {
605 encoding = (0b110100111 << 23);
606 } else if (ctx.chip_class >= GFX10) {
607 encoding = (0b110011 << 26);
608 } else {
609 unreachable("Unknown chip_class.");
610 }
611
612 encoding |= opcode << 16;
613 encoding |= (vop3->clamp ? 1 : 0) << 15;
614 encoding |= vop3->opsel_lo << 11;
615 encoding |= (vop3->opsel_hi & 0x4) ? 1 : 0 << 14;
616 for (unsigned i = 0; i < 3; i++)
617 encoding |= vop3->neg_hi[i] << (8+i);
618 encoding |= (0xFF & instr->definitions[0].physReg());
619 out.push_back(encoding);
620 encoding = 0;
621 for (unsigned i = 0; i < instr->operands.size(); i++)
622 encoding |= instr->operands[i].physReg() << (i * 9);
623 encoding |= vop3->opsel_hi & 0x3 << 27;
624 for (unsigned i = 0; i < 3; i++)
625 encoding |= vop3->neg_lo[i] << (29+i);
626 out.push_back(encoding);
627
628 } else if (instr->isDPP()){
629 assert(ctx.chip_class >= GFX8);
630 /* first emit the instruction without the DPP operand */
631 Operand dpp_op = instr->operands[0];
632 instr->operands[0] = Operand(PhysReg{250}, v1);
633 instr->format = (Format) ((uint16_t) instr->format & ~(uint16_t)Format::DPP);
634 emit_instruction(ctx, out, instr);
635 DPP_instruction* dpp = static_cast<DPP_instruction*>(instr);
636 uint32_t encoding = (0xF & dpp->row_mask) << 28;
637 encoding |= (0xF & dpp->bank_mask) << 24;
638 encoding |= dpp->abs[1] << 23;
639 encoding |= dpp->neg[1] << 22;
640 encoding |= dpp->abs[0] << 21;
641 encoding |= dpp->neg[0] << 20;
642 if (ctx.chip_class >= GFX10)
643 encoding |= 1 << 18; /* set Fetch Inactive to match GFX9 behaviour */
644 encoding |= dpp->bound_ctrl << 19;
645 encoding |= dpp->dpp_ctrl << 8;
646 encoding |= (0xFF) & dpp_op.physReg();
647 out.push_back(encoding);
648 return;
649 } else if (instr->isSDWA()) {
650 /* first emit the instruction without the SDWA operand */
651 Operand sdwa_op = instr->operands[0];
652 instr->operands[0] = Operand(PhysReg{249}, v1);
653 instr->format = (Format) ((uint16_t) instr->format & ~(uint16_t)Format::SDWA);
654 emit_instruction(ctx, out, instr);
655
656 SDWA_instruction* sdwa = static_cast<SDWA_instruction*>(instr);
657 uint32_t encoding = 0;
658
659 if ((uint16_t)instr->format & (uint16_t)Format::VOPC) {
660 if (instr->definitions[0].physReg() != vcc) {
661 encoding |= instr->definitions[0].physReg() << 8;
662 encoding |= 1 << 15;
663 }
664 encoding |= (sdwa->clamp ? 1 : 0) << 13;
665 } else {
666 encoding |= get_sdwa_sel(sdwa->dst_sel, instr->definitions[0].physReg()) << 8;
667 uint32_t dst_u = sdwa->dst_sel & sdwa_sext ? 1 : 0;
668 if (sdwa->dst_preserve || (sdwa->dst_sel & sdwa_isra))
669 dst_u = 2;
670 encoding |= dst_u << 11;
671 encoding |= (sdwa->clamp ? 1 : 0) << 13;
672 encoding |= sdwa->omod << 14;
673 }
674
675 encoding |= get_sdwa_sel(sdwa->sel[0], sdwa_op.physReg()) << 16;
676 encoding |= sdwa->sel[0] & sdwa_sext ? 1 << 19 : 0;
677 encoding |= sdwa->abs[0] << 21;
678 encoding |= sdwa->neg[0] << 20;
679
680 if (instr->operands.size() >= 2) {
681 encoding |= get_sdwa_sel(sdwa->sel[1], instr->operands[1].physReg()) << 24;
682 encoding |= sdwa->sel[1] & sdwa_sext ? 1 << 27 : 0;
683 encoding |= sdwa->abs[1] << 29;
684 encoding |= sdwa->neg[1] << 28;
685 }
686
687 encoding |= 0xFF & sdwa_op.physReg();
688 encoding |= (sdwa_op.physReg() < 256) << 23;
689 if (instr->operands.size() >= 2)
690 encoding |= (instr->operands[1].physReg() < 256) << 31;
691 out.push_back(encoding);
692 } else {
693 unreachable("unimplemented instruction format");
694 }
695 break;
696 }
697
698 /* append literal dword */
699 for (const Operand& op : instr->operands) {
700 if (op.isLiteral()) {
701 out.push_back(op.constantValue());
702 break;
703 }
704 }
705 }
706
emit_block(asm_context & ctx,std::vector<uint32_t> & out,Block & block)707 void emit_block(asm_context& ctx, std::vector<uint32_t>& out, Block& block)
708 {
709 for (aco_ptr<Instruction>& instr : block.instructions) {
710 #if 0
711 int start_idx = out.size();
712 std::cerr << "Encoding:\t" << std::endl;
713 aco_print_instr(&*instr, stderr);
714 std::cerr << std::endl;
715 #endif
716 emit_instruction(ctx, out, instr.get());
717 #if 0
718 for (int i = start_idx; i < out.size(); i++)
719 std::cerr << "encoding: " << "0x" << std::setfill('0') << std::setw(8) << std::hex << out[i] << std::endl;
720 #endif
721 }
722 }
723
fix_exports(asm_context & ctx,std::vector<uint32_t> & out,Program * program)724 void fix_exports(asm_context& ctx, std::vector<uint32_t>& out, Program* program)
725 {
726 bool exported = false;
727 for (Block& block : program->blocks) {
728 if (!(block.kind & block_kind_export_end))
729 continue;
730 std::vector<aco_ptr<Instruction>>::reverse_iterator it = block.instructions.rbegin();
731 while ( it != block.instructions.rend())
732 {
733 if ((*it)->format == Format::EXP) {
734 Export_instruction* exp = static_cast<Export_instruction*>((*it).get());
735 if (program->stage.hw == HWStage::VS || program->stage.hw == HWStage::NGG) {
736 if (exp->dest >= V_008DFC_SQ_EXP_POS && exp->dest <= (V_008DFC_SQ_EXP_POS + 3)) {
737 exp->done = true;
738 exported = true;
739 break;
740 }
741 } else {
742 exp->done = true;
743 exp->valid_mask = true;
744 exported = true;
745 break;
746 }
747 } else if ((*it)->definitions.size() && (*it)->definitions[0].physReg() == exec)
748 break;
749 ++it;
750 }
751 }
752
753 if (!exported) {
754 /* Abort in order to avoid a GPU hang. */
755 bool is_vertex_or_ngg = (program->stage.hw == HWStage::VS || program->stage.hw == HWStage::NGG);
756 aco_err(program, "Missing export in %s shader:", is_vertex_or_ngg ? "vertex or NGG" : "fragment");
757 aco_print_program(program, stderr);
758 abort();
759 }
760 }
761
insert_code(asm_context & ctx,std::vector<uint32_t> & out,unsigned insert_before,unsigned insert_count,const uint32_t * insert_data)762 static void insert_code(asm_context& ctx, std::vector<uint32_t>& out, unsigned insert_before,
763 unsigned insert_count, const uint32_t *insert_data)
764 {
765 out.insert(out.begin() + insert_before, insert_data, insert_data + insert_count);
766
767 /* Update the offset of each affected block */
768 for (Block& block : ctx.program->blocks) {
769 if (block.offset >= insert_before)
770 block.offset += insert_count;
771 }
772
773 /* Find first branch after the inserted code */
774 auto branch_it = std::find_if(ctx.branches.begin(), ctx.branches.end(), [insert_before](const auto &branch) -> bool {
775 return (unsigned)branch.first >= insert_before;
776 });
777
778 /* Update the locations of branches */
779 for (; branch_it != ctx.branches.end(); ++branch_it)
780 branch_it->first += insert_count;
781
782 /* Find first constant address after the inserted code */
783 auto caddr_it = std::find_if(ctx.constaddrs.begin(), ctx.constaddrs.end(), [insert_before](const int &caddr_pos) -> bool {
784 return (unsigned)caddr_pos >= insert_before;
785 });
786
787 /* Update the locations of constant addresses */
788 for (; caddr_it != ctx.constaddrs.end(); ++caddr_it)
789 (*caddr_it) += insert_count;
790 }
791
fix_branches_gfx10(asm_context & ctx,std::vector<uint32_t> & out)792 static void fix_branches_gfx10(asm_context& ctx, std::vector<uint32_t>& out)
793 {
794 /* Branches with an offset of 0x3f are buggy on GFX10, we workaround by inserting NOPs if needed. */
795 bool gfx10_3f_bug = false;
796
797 do {
798 auto buggy_branch_it = std::find_if(ctx.branches.begin(), ctx.branches.end(), [&ctx](const auto &branch) -> bool {
799 return ((int)ctx.program->blocks[branch.second->block].offset - branch.first - 1) == 0x3f;
800 });
801
802 gfx10_3f_bug = buggy_branch_it != ctx.branches.end();
803
804 if (gfx10_3f_bug) {
805 /* Insert an s_nop after the branch */
806 constexpr uint32_t s_nop_0 = 0xbf800000u;
807 insert_code(ctx, out, buggy_branch_it->first + 1, 1, &s_nop_0);
808 }
809 } while (gfx10_3f_bug);
810 }
811
emit_long_jump(asm_context & ctx,SOPP_instruction * branch,bool backwards,std::vector<uint32_t> & out)812 void emit_long_jump(asm_context& ctx, SOPP_instruction *branch, bool backwards, std::vector<uint32_t>& out)
813 {
814 Builder bld(ctx.program);
815
816 Definition def_tmp_lo(branch->definitions[0].physReg(), s1);
817 Operand op_tmp_lo(branch->definitions[0].physReg(), s1);
818 Definition def_tmp_hi(branch->definitions[0].physReg().advance(4), s1);
819 Operand op_tmp_hi(branch->definitions[0].physReg().advance(4), s1);
820
821 aco_ptr<Instruction> instr;
822
823 if (branch->opcode != aco_opcode::s_branch) {
824 /* for conditional branches, skip the long jump if the condition is false */
825 aco_opcode inv;
826 switch (branch->opcode) {
827 case aco_opcode::s_cbranch_scc0:
828 inv = aco_opcode::s_cbranch_scc1;
829 break;
830 case aco_opcode::s_cbranch_scc1:
831 inv = aco_opcode::s_cbranch_scc0;
832 break;
833 case aco_opcode::s_cbranch_vccz:
834 inv = aco_opcode::s_cbranch_vccnz;
835 break;
836 case aco_opcode::s_cbranch_vccnz:
837 inv = aco_opcode::s_cbranch_vccz;
838 break;
839 case aco_opcode::s_cbranch_execz:
840 inv = aco_opcode::s_cbranch_execnz;
841 break;
842 case aco_opcode::s_cbranch_execnz:
843 inv = aco_opcode::s_cbranch_execz;
844 break;
845 default:
846 unreachable("Unhandled long jump.");
847 }
848 instr.reset(bld.sopp(inv, -1, 7));
849 emit_instruction(ctx, out, instr.get());
850 }
851
852 /* create the new PC and stash SCC in the LSB */
853 instr.reset(bld.sop1(aco_opcode::s_getpc_b64, branch->definitions[0]).instr);
854 emit_instruction(ctx, out, instr.get());
855
856 instr.reset(bld.sop2(aco_opcode::s_addc_u32, def_tmp_lo, op_tmp_lo, Operand(0u)).instr);
857 instr->operands[1].setFixed(PhysReg{255}); /* this operand has to be a literal */
858 emit_instruction(ctx, out, instr.get());
859 branch->pass_flags = out.size();
860
861 instr.reset(bld.sop2(aco_opcode::s_addc_u32, def_tmp_hi, op_tmp_hi, Operand(backwards ? UINT32_MAX : 0u)).instr);
862 emit_instruction(ctx, out, instr.get());
863
864 /* restore SCC and clear the LSB of the new PC */
865 instr.reset(bld.sopc(aco_opcode::s_bitcmp1_b32, def_tmp_lo, op_tmp_lo, Operand(0u)).instr);
866 emit_instruction(ctx, out, instr.get());
867 instr.reset(bld.sop1(aco_opcode::s_bitset0_b32, def_tmp_lo, Operand(0u)).instr);
868 emit_instruction(ctx, out, instr.get());
869
870 /* create the s_setpc_b64 to jump */
871 instr.reset(bld.sop1(aco_opcode::s_setpc_b64, Operand(branch->definitions[0].physReg(), s2)).instr);
872 emit_instruction(ctx, out, instr.get());
873 }
874
fix_branches(asm_context & ctx,std::vector<uint32_t> & out)875 void fix_branches(asm_context& ctx, std::vector<uint32_t>& out)
876 {
877 bool repeat = false;
878 do {
879 repeat = false;
880
881 if (ctx.chip_class == GFX10)
882 fix_branches_gfx10(ctx, out);
883
884 for (std::pair<int, SOPP_instruction*> &branch : ctx.branches) {
885 int offset = (int)ctx.program->blocks[branch.second->block].offset - branch.first - 1;
886 if ((offset < INT16_MIN || offset > INT16_MAX) && !branch.second->pass_flags) {
887 std::vector<uint32_t> long_jump;
888 bool backwards = ctx.program->blocks[branch.second->block].offset < (unsigned)branch.first;
889 emit_long_jump(ctx, branch.second, backwards, long_jump);
890
891 out[branch.first] = long_jump[0];
892 insert_code(ctx, out, branch.first + 1, long_jump.size() - 1, long_jump.data() + 1);
893
894 repeat = true;
895 break;
896 }
897
898 if (branch.second->pass_flags) {
899 int after_getpc = branch.first + branch.second->pass_flags - 2;
900 offset = (int)ctx.program->blocks[branch.second->block].offset - after_getpc;
901 out[branch.first + branch.second->pass_flags - 1] = offset * 4;
902 } else {
903 out[branch.first] &= 0xffff0000u;
904 out[branch.first] |= (uint16_t) offset;
905 }
906 }
907 } while (repeat);
908 }
909
fix_constaddrs(asm_context & ctx,std::vector<uint32_t> & out)910 void fix_constaddrs(asm_context& ctx, std::vector<uint32_t>& out)
911 {
912 for (unsigned addr : ctx.constaddrs)
913 out[addr] += (out.size() - addr + 1u) * 4u;
914 }
915
emit_program(Program * program,std::vector<uint32_t> & code)916 unsigned emit_program(Program* program,
917 std::vector<uint32_t>& code)
918 {
919 asm_context ctx(program);
920
921 if (program->stage.hw == HWStage::VS ||
922 program->stage.hw == HWStage::FS ||
923 program->stage.hw == HWStage::NGG)
924 fix_exports(ctx, code, program);
925
926 for (Block& block : program->blocks) {
927 block.offset = code.size();
928 emit_block(ctx, code, block);
929 }
930
931 fix_branches(ctx, code);
932
933 unsigned exec_size = code.size() * sizeof(uint32_t);
934
935 if (program->chip_class >= GFX10) {
936 /* Pad output with s_code_end so instruction prefetching doesn't cause
937 * page faults */
938 unsigned final_size = align(code.size() + 3 * 16, 16);
939 while (code.size() < final_size)
940 code.push_back(0xbf9f0000u);
941 }
942
943 fix_constaddrs(ctx, code);
944
945 while (program->constant_data.size() % 4u)
946 program->constant_data.push_back(0);
947 /* Copy constant data */
948 code.insert(code.end(), (uint32_t*)program->constant_data.data(),
949 (uint32_t*)(program->constant_data.data() + program->constant_data.size()));
950
951 return exec_size;
952 }
953
954 }
955