1 /*
2 * Copyright © 2020 Valve Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 */
24 #include "aco_ir.h"
25 #include "vulkan/radv_shader.h"
26 #include "c11/threads.h"
27 #include "util/debug.h"
28
29 namespace aco {
30
31 uint64_t debug_flags = 0;
32
33 static const struct debug_control aco_debug_options[] = {
34 {"validateir", DEBUG_VALIDATE_IR},
35 {"validatera", DEBUG_VALIDATE_RA},
36 {"perfwarn", DEBUG_PERFWARN},
37 {"force-waitcnt", DEBUG_FORCE_WAITCNT},
38 {"novn", DEBUG_NO_VN},
39 {"noopt", DEBUG_NO_OPT},
40 {"nosched", DEBUG_NO_SCHED},
41 {NULL, 0}
42 };
43
44 static once_flag init_once_flag = ONCE_FLAG_INIT;
45
init_once()46 static void init_once()
47 {
48 debug_flags = parse_debug_string(getenv("ACO_DEBUG"), aco_debug_options);
49
50 #ifndef NDEBUG
51 /* enable some flags by default on debug builds */
52 debug_flags |= aco::DEBUG_VALIDATE_IR;
53 #endif
54 }
55
init()56 void init()
57 {
58 call_once(&init_once_flag, init_once);
59 }
60
init_program(Program * program,Stage stage,struct radv_shader_info * info,enum chip_class chip_class,enum radeon_family family,ac_shader_config * config)61 void init_program(Program *program, Stage stage, struct radv_shader_info *info,
62 enum chip_class chip_class, enum radeon_family family,
63 ac_shader_config *config)
64 {
65 program->stage = stage;
66 program->config = config;
67 program->info = info;
68 program->chip_class = chip_class;
69 if (family == CHIP_UNKNOWN) {
70 switch (chip_class) {
71 case GFX6:
72 program->family = CHIP_TAHITI;
73 break;
74 case GFX7:
75 program->family = CHIP_BONAIRE;
76 break;
77 case GFX8:
78 program->family = CHIP_POLARIS10;
79 break;
80 case GFX9:
81 program->family = CHIP_VEGA10;
82 break;
83 case GFX10:
84 program->family = CHIP_NAVI10;
85 break;
86 default:
87 program->family = CHIP_UNKNOWN;
88 break;
89 }
90 } else {
91 program->family = family;
92 }
93 program->wave_size = info->wave_size;
94 program->lane_mask = program->wave_size == 32 ? s1 : s2;
95
96 program->lds_alloc_granule = chip_class >= GFX7 ? 512 : 256;
97 program->lds_limit = chip_class >= GFX7 ? 65536 : 32768;
98 /* apparently gfx702 also has 16-bank LDS but I can't find a family for that */
99 program->has_16bank_lds = family == CHIP_KABINI || family == CHIP_STONEY;
100
101 program->vgpr_limit = 256;
102 program->vgpr_alloc_granule = 3;
103
104 if (chip_class >= GFX10) {
105 program->physical_sgprs = 2560; /* doesn't matter as long as it's at least 128 * 20 */
106 program->sgpr_alloc_granule = 127;
107 program->sgpr_limit = 106;
108 if (chip_class >= GFX10_3)
109 program->vgpr_alloc_granule = program->wave_size == 32 ? 15 : 7;
110 else
111 program->vgpr_alloc_granule = program->wave_size == 32 ? 7 : 3;
112 } else if (program->chip_class >= GFX8) {
113 program->physical_sgprs = 800;
114 program->sgpr_alloc_granule = 15;
115 if (family == CHIP_TONGA || family == CHIP_ICELAND)
116 program->sgpr_limit = 94; /* workaround hardware bug */
117 else
118 program->sgpr_limit = 102;
119 } else {
120 program->physical_sgprs = 512;
121 program->sgpr_alloc_granule = 7;
122 program->sgpr_limit = 104;
123 }
124
125 program->next_fp_mode.preserve_signed_zero_inf_nan32 = false;
126 program->next_fp_mode.preserve_signed_zero_inf_nan16_64 = false;
127 program->next_fp_mode.must_flush_denorms32 = false;
128 program->next_fp_mode.must_flush_denorms16_64 = false;
129 program->next_fp_mode.care_about_round32 = false;
130 program->next_fp_mode.care_about_round16_64 = false;
131 program->next_fp_mode.denorm16_64 = fp_denorm_keep;
132 program->next_fp_mode.denorm32 = 0;
133 program->next_fp_mode.round16_64 = fp_round_ne;
134 program->next_fp_mode.round32 = fp_round_ne;
135 }
136
get_sync_info(const Instruction * instr)137 memory_sync_info get_sync_info(const Instruction* instr)
138 {
139 switch (instr->format) {
140 case Format::SMEM:
141 return static_cast<const SMEM_instruction*>(instr)->sync;
142 case Format::MUBUF:
143 return static_cast<const MUBUF_instruction*>(instr)->sync;
144 case Format::MIMG:
145 return static_cast<const MIMG_instruction*>(instr)->sync;
146 case Format::MTBUF:
147 return static_cast<const MTBUF_instruction*>(instr)->sync;
148 case Format::FLAT:
149 case Format::GLOBAL:
150 case Format::SCRATCH:
151 return static_cast<const FLAT_instruction*>(instr)->sync;
152 case Format::DS:
153 return static_cast<const DS_instruction*>(instr)->sync;
154 default:
155 return memory_sync_info();
156 }
157 }
158
can_use_SDWA(chip_class chip,const aco_ptr<Instruction> & instr)159 bool can_use_SDWA(chip_class chip, const aco_ptr<Instruction>& instr)
160 {
161 if (!instr->isVALU())
162 return false;
163
164 if (chip < GFX8 || instr->isDPP())
165 return false;
166
167 if (instr->isSDWA())
168 return true;
169
170 if (instr->isVOP3()) {
171 VOP3A_instruction *vop3 = static_cast<VOP3A_instruction*>(instr.get());
172 if (instr->format == Format::VOP3)
173 return false;
174 if (vop3->clamp && instr->format == asVOP3(Format::VOPC) && chip != GFX8)
175 return false;
176 if (vop3->omod && chip < GFX9)
177 return false;
178
179 //TODO: return true if we know we will use vcc
180 if (instr->definitions.size() >= 2)
181 return false;
182
183 for (unsigned i = 1; i < instr->operands.size(); i++) {
184 if (instr->operands[i].isLiteral())
185 return false;
186 if (chip < GFX9 && !instr->operands[i].isOfType(RegType::vgpr))
187 return false;
188 }
189 }
190
191 if (!instr->operands.empty()) {
192 if (instr->operands[0].isLiteral())
193 return false;
194 if (chip < GFX9 && !instr->operands[0].isOfType(RegType::vgpr))
195 return false;
196 }
197
198 bool is_mac = instr->opcode == aco_opcode::v_mac_f32 ||
199 instr->opcode == aco_opcode::v_mac_f16 ||
200 instr->opcode == aco_opcode::v_fmac_f32 ||
201 instr->opcode == aco_opcode::v_fmac_f16;
202
203 if (chip != GFX8 && is_mac)
204 return false;
205
206 //TODO: return true if we know we will use vcc
207 if ((unsigned)instr->format & (unsigned)Format::VOPC)
208 return false;
209 if (instr->operands.size() >= 3 && !is_mac)
210 return false;
211
212 return instr->opcode != aco_opcode::v_madmk_f32 &&
213 instr->opcode != aco_opcode::v_madak_f32 &&
214 instr->opcode != aco_opcode::v_madmk_f16 &&
215 instr->opcode != aco_opcode::v_madak_f16 &&
216 instr->opcode != aco_opcode::v_readfirstlane_b32 &&
217 instr->opcode != aco_opcode::v_clrexcp &&
218 instr->opcode != aco_opcode::v_swap_b32;
219 }
220
221 /* updates "instr" and returns the old instruction (or NULL if no update was needed) */
convert_to_SDWA(chip_class chip,aco_ptr<Instruction> & instr)222 aco_ptr<Instruction> convert_to_SDWA(chip_class chip, aco_ptr<Instruction>& instr)
223 {
224 if (instr->isSDWA())
225 return NULL;
226
227 aco_ptr<Instruction> tmp = std::move(instr);
228 Format format = (Format)(((uint16_t)tmp->format & ~(uint16_t)Format::VOP3) | (uint16_t)Format::SDWA);
229 instr.reset(create_instruction<SDWA_instruction>(tmp->opcode, format, tmp->operands.size(), tmp->definitions.size()));
230 std::copy(tmp->operands.cbegin(), tmp->operands.cend(), instr->operands.begin());
231 std::copy(tmp->definitions.cbegin(), tmp->definitions.cend(), instr->definitions.begin());
232
233 SDWA_instruction *sdwa = static_cast<SDWA_instruction*>(instr.get());
234
235 if (tmp->isVOP3()) {
236 VOP3A_instruction *vop3 = static_cast<VOP3A_instruction*>(tmp.get());
237 memcpy(sdwa->neg, vop3->neg, sizeof(sdwa->neg));
238 memcpy(sdwa->abs, vop3->abs, sizeof(sdwa->abs));
239 sdwa->omod = vop3->omod;
240 sdwa->clamp = vop3->clamp;
241 }
242
243 for (unsigned i = 0; i < instr->operands.size(); i++) {
244 /* SDWA only uses operands 0 and 1. */
245 if (i >= 2)
246 break;
247
248 switch (instr->operands[i].bytes()) {
249 case 1:
250 sdwa->sel[i] = sdwa_ubyte;
251 break;
252 case 2:
253 sdwa->sel[i] = sdwa_uword;
254 break;
255 case 4:
256 sdwa->sel[i] = sdwa_udword;
257 break;
258 }
259 }
260 switch (instr->definitions[0].bytes()) {
261 case 1:
262 sdwa->dst_sel = sdwa_ubyte;
263 sdwa->dst_preserve = true;
264 break;
265 case 2:
266 sdwa->dst_sel = sdwa_uword;
267 sdwa->dst_preserve = true;
268 break;
269 case 4:
270 sdwa->dst_sel = sdwa_udword;
271 break;
272 }
273
274 if (instr->definitions[0].getTemp().type() == RegType::sgpr && chip == GFX8)
275 instr->definitions[0].setFixed(vcc);
276 if (instr->definitions.size() >= 2)
277 instr->definitions[1].setFixed(vcc);
278 if (instr->operands.size() >= 3)
279 instr->operands[2].setFixed(vcc);
280
281 return tmp;
282 }
283
can_use_opsel(chip_class chip,aco_opcode op,int idx,bool high)284 bool can_use_opsel(chip_class chip, aco_opcode op, int idx, bool high)
285 {
286 /* opsel is only GFX9+ */
287 if ((high || idx == -1) && chip < GFX9)
288 return false;
289
290 switch (op) {
291 case aco_opcode::v_div_fixup_f16:
292 case aco_opcode::v_fma_f16:
293 case aco_opcode::v_mad_f16:
294 case aco_opcode::v_mad_u16:
295 case aco_opcode::v_mad_i16:
296 case aco_opcode::v_med3_f16:
297 case aco_opcode::v_med3_i16:
298 case aco_opcode::v_med3_u16:
299 case aco_opcode::v_min3_f16:
300 case aco_opcode::v_min3_i16:
301 case aco_opcode::v_min3_u16:
302 case aco_opcode::v_max3_f16:
303 case aco_opcode::v_max3_i16:
304 case aco_opcode::v_max3_u16:
305 case aco_opcode::v_max_u16_e64:
306 case aco_opcode::v_max_i16_e64:
307 case aco_opcode::v_min_u16_e64:
308 case aco_opcode::v_min_i16_e64:
309 case aco_opcode::v_add_i16:
310 case aco_opcode::v_sub_i16:
311 case aco_opcode::v_add_u16_e64:
312 case aco_opcode::v_sub_u16_e64:
313 case aco_opcode::v_lshlrev_b16_e64:
314 case aco_opcode::v_lshrrev_b16_e64:
315 case aco_opcode::v_ashrrev_i16_e64:
316 case aco_opcode::v_mul_lo_u16_e64:
317 return true;
318 case aco_opcode::v_pack_b32_f16:
319 case aco_opcode::v_cvt_pknorm_i16_f16:
320 case aco_opcode::v_cvt_pknorm_u16_f16:
321 return idx != -1;
322 case aco_opcode::v_mad_u32_u16:
323 case aco_opcode::v_mad_i32_i16:
324 return idx >= 0 && idx < 2;
325 default:
326 return false;
327 }
328 }
329
get_reduction_identity(ReduceOp op,unsigned idx)330 uint32_t get_reduction_identity(ReduceOp op, unsigned idx)
331 {
332 switch (op) {
333 case iadd8:
334 case iadd16:
335 case iadd32:
336 case iadd64:
337 case fadd16:
338 case fadd32:
339 case fadd64:
340 case ior8:
341 case ior16:
342 case ior32:
343 case ior64:
344 case ixor8:
345 case ixor16:
346 case ixor32:
347 case ixor64:
348 case umax8:
349 case umax16:
350 case umax32:
351 case umax64:
352 return 0;
353 case imul8:
354 case imul16:
355 case imul32:
356 case imul64:
357 return idx ? 0 : 1;
358 case fmul16:
359 return 0x3c00u; /* 1.0 */
360 case fmul32:
361 return 0x3f800000u; /* 1.0 */
362 case fmul64:
363 return idx ? 0x3ff00000u : 0u; /* 1.0 */
364 case imin8:
365 return INT8_MAX;
366 case imin16:
367 return INT16_MAX;
368 case imin32:
369 return INT32_MAX;
370 case imin64:
371 return idx ? 0x7fffffffu : 0xffffffffu;
372 case imax8:
373 return INT8_MIN;
374 case imax16:
375 return INT16_MIN;
376 case imax32:
377 return INT32_MIN;
378 case imax64:
379 return idx ? 0x80000000u : 0;
380 case umin8:
381 case umin16:
382 case iand8:
383 case iand16:
384 return 0xffffffffu;
385 case umin32:
386 case umin64:
387 case iand32:
388 case iand64:
389 return 0xffffffffu;
390 case fmin16:
391 return 0x7c00u; /* infinity */
392 case fmin32:
393 return 0x7f800000u; /* infinity */
394 case fmin64:
395 return idx ? 0x7ff00000u : 0u; /* infinity */
396 case fmax16:
397 return 0xfc00u; /* negative infinity */
398 case fmax32:
399 return 0xff800000u; /* negative infinity */
400 case fmax64:
401 return idx ? 0xfff00000u : 0u; /* negative infinity */
402 default:
403 unreachable("Invalid reduction operation");
404 break;
405 }
406 return 0;
407 }
408
409 }
410