1 /*
2 * Copyright 2016 Bas Nieuwenhuizen
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the
6 * "Software"), to deal in the Software without restriction, including
7 * without limitation the rights to use, copy, modify, merge, publish,
8 * distribute, sub license, and/or sell copies of the Software, and to
9 * permit persons to whom the Software is furnished to do so, subject to
10 * the following conditions:
11 *
12 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
13 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
14 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
15 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
16 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
17 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
18 * USE OR OTHER DEALINGS IN THE SOFTWARE.
19 *
20 * The above copyright notice and this permission notice (including the
21 * next paragraph) shall be included in all copies or substantial portions
22 * of the Software.
23 *
24 */
25 #ifndef AC_LLVM_BUILD_H
26 #define AC_LLVM_BUILD_H
27
28 #include "ac_shader_abi.h"
29 #include "ac_shader_args.h"
30 #include "ac_shader_util.h"
31 #include "amd_family.h"
32 #include "compiler/nir/nir.h"
33 #include <llvm-c/Core.h>
34
35 #include <stdbool.h>
36
37 #ifdef __cplusplus
38 extern "C" {
39 #endif
40
41 enum
42 {
43 AC_ADDR_SPACE_FLAT = 0, /* Slower than global. */
44 AC_ADDR_SPACE_GLOBAL = 1,
45 AC_ADDR_SPACE_GDS = 2,
46 AC_ADDR_SPACE_LDS = 3,
47 AC_ADDR_SPACE_CONST = 4, /* Global allowing SMEM. */
48 AC_ADDR_SPACE_CONST_32BIT = 6, /* same as CONST, but the pointer type has 32 bits */
49 };
50
51 #define AC_WAIT_LGKM (1 << 0) /* LDS, GDS, constant, message */
52 #define AC_WAIT_VLOAD (1 << 1) /* VMEM load/sample instructions */
53 #define AC_WAIT_VSTORE (1 << 2) /* VMEM store instructions */
54
55 struct ac_llvm_flow;
56 struct ac_llvm_compiler;
57 enum ac_float_mode;
58
59 struct ac_llvm_flow_state {
60 struct ac_llvm_flow *stack;
61 unsigned depth_max;
62 unsigned depth;
63 };
64
65 struct ac_llvm_context {
66 LLVMContextRef context;
67 LLVMModuleRef module;
68 LLVMBuilderRef builder;
69
70 LLVMValueRef main_function;
71
72 LLVMTypeRef voidt;
73 LLVMTypeRef i1;
74 LLVMTypeRef i8;
75 LLVMTypeRef i16;
76 LLVMTypeRef i32;
77 LLVMTypeRef i64;
78 LLVMTypeRef i128;
79 LLVMTypeRef intptr;
80 LLVMTypeRef f16;
81 LLVMTypeRef f32;
82 LLVMTypeRef f64;
83 LLVMTypeRef v2i16;
84 LLVMTypeRef v4i16;
85 LLVMTypeRef v2f16;
86 LLVMTypeRef v4f16;
87 LLVMTypeRef v2i32;
88 LLVMTypeRef v3i32;
89 LLVMTypeRef v4i32;
90 LLVMTypeRef v2f32;
91 LLVMTypeRef v3f32;
92 LLVMTypeRef v4f32;
93 LLVMTypeRef v8i32;
94 LLVMTypeRef iN_wavemask;
95 LLVMTypeRef iN_ballotmask;
96
97 LLVMValueRef i8_0;
98 LLVMValueRef i8_1;
99 LLVMValueRef i16_0;
100 LLVMValueRef i16_1;
101 LLVMValueRef i32_0;
102 LLVMValueRef i32_1;
103 LLVMValueRef i64_0;
104 LLVMValueRef i64_1;
105 LLVMValueRef i128_0;
106 LLVMValueRef i128_1;
107 LLVMValueRef f16_0;
108 LLVMValueRef f16_1;
109 LLVMValueRef f32_0;
110 LLVMValueRef f32_1;
111 LLVMValueRef f64_0;
112 LLVMValueRef f64_1;
113 LLVMValueRef i1true;
114 LLVMValueRef i1false;
115
116 /* Temporary helper to implement demote_to_helper:
117 * True = live lanes
118 * False = demoted lanes
119 */
120 LLVMValueRef postponed_kill;
121
122 /* Since ac_nir_translate makes a local copy of ac_llvm_context, there
123 * are two ac_llvm_contexts. Declare a pointer here, so that the control
124 * flow stack is shared by both ac_llvm_contexts.
125 */
126 struct ac_llvm_flow_state *flow;
127
128 unsigned range_md_kind;
129 unsigned invariant_load_md_kind;
130 unsigned uniform_md_kind;
131 LLVMValueRef empty_md;
132
133 enum chip_class chip_class;
134 enum radeon_family family;
135
136 unsigned wave_size;
137 unsigned ballot_mask_bits;
138
139 unsigned float_mode;
140
141 LLVMValueRef lds;
142 };
143
144 void ac_llvm_context_init(struct ac_llvm_context *ctx, struct ac_llvm_compiler *compiler,
145 enum chip_class chip_class, enum radeon_family family,
146 enum ac_float_mode float_mode, unsigned wave_size,
147 unsigned ballot_mask_bits);
148
149 void ac_llvm_context_dispose(struct ac_llvm_context *ctx);
150
151 int ac_get_llvm_num_components(LLVMValueRef value);
152
153 int ac_get_elem_bits(struct ac_llvm_context *ctx, LLVMTypeRef type);
154
155 LLVMValueRef ac_llvm_extract_elem(struct ac_llvm_context *ac, LLVMValueRef value, int index);
156
157 unsigned ac_get_type_size(LLVMTypeRef type);
158
159 LLVMTypeRef ac_to_integer_type(struct ac_llvm_context *ctx, LLVMTypeRef t);
160 LLVMValueRef ac_to_integer(struct ac_llvm_context *ctx, LLVMValueRef v);
161 LLVMValueRef ac_to_integer_or_pointer(struct ac_llvm_context *ctx, LLVMValueRef v);
162 LLVMTypeRef ac_to_float_type(struct ac_llvm_context *ctx, LLVMTypeRef t);
163 LLVMValueRef ac_to_float(struct ac_llvm_context *ctx, LLVMValueRef v);
164
165 LLVMValueRef ac_build_intrinsic(struct ac_llvm_context *ctx, const char *name,
166 LLVMTypeRef return_type, LLVMValueRef *params, unsigned param_count,
167 unsigned attrib_mask);
168
169 void ac_build_type_name_for_intr(LLVMTypeRef type, char *buf, unsigned bufsize);
170
171 LLVMValueRef ac_build_phi(struct ac_llvm_context *ctx, LLVMTypeRef type, unsigned count_incoming,
172 LLVMValueRef *values, LLVMBasicBlockRef *blocks);
173
174 void ac_build_s_barrier(struct ac_llvm_context *ctx);
175 void ac_build_optimization_barrier(struct ac_llvm_context *ctx, LLVMValueRef *pvgpr);
176
177 LLVMValueRef ac_build_shader_clock(struct ac_llvm_context *ctx, nir_scope scope);
178
179 LLVMValueRef ac_build_ballot(struct ac_llvm_context *ctx, LLVMValueRef value);
180 LLVMValueRef ac_get_i1_sgpr_mask(struct ac_llvm_context *ctx, LLVMValueRef value);
181
182 LLVMValueRef ac_build_vote_all(struct ac_llvm_context *ctx, LLVMValueRef value);
183
184 LLVMValueRef ac_build_vote_any(struct ac_llvm_context *ctx, LLVMValueRef value);
185
186 LLVMValueRef ac_build_vote_eq(struct ac_llvm_context *ctx, LLVMValueRef value);
187
188 LLVMValueRef ac_build_varying_gather_values(struct ac_llvm_context *ctx, LLVMValueRef *values,
189 unsigned value_count, unsigned component);
190
191 LLVMValueRef ac_build_gather_values_extended(struct ac_llvm_context *ctx, LLVMValueRef *values,
192 unsigned value_count, unsigned value_stride, bool load,
193 bool always_vector);
194 LLVMValueRef ac_build_gather_values(struct ac_llvm_context *ctx, LLVMValueRef *values,
195 unsigned value_count);
196
197 LLVMValueRef ac_extract_components(struct ac_llvm_context *ctx, LLVMValueRef value, unsigned start,
198 unsigned channels);
199
200 LLVMValueRef ac_build_expand_to_vec4(struct ac_llvm_context *ctx, LLVMValueRef value,
201 unsigned num_channels);
202 LLVMValueRef ac_build_round(struct ac_llvm_context *ctx, LLVMValueRef value);
203
204 LLVMValueRef ac_build_fdiv(struct ac_llvm_context *ctx, LLVMValueRef num, LLVMValueRef den);
205
206 LLVMValueRef ac_build_fast_udiv(struct ac_llvm_context *ctx, LLVMValueRef num,
207 LLVMValueRef multiplier, LLVMValueRef pre_shift,
208 LLVMValueRef post_shift, LLVMValueRef increment);
209 LLVMValueRef ac_build_fast_udiv_nuw(struct ac_llvm_context *ctx, LLVMValueRef num,
210 LLVMValueRef multiplier, LLVMValueRef pre_shift,
211 LLVMValueRef post_shift, LLVMValueRef increment);
212 LLVMValueRef ac_build_fast_udiv_u31_d_not_one(struct ac_llvm_context *ctx, LLVMValueRef num,
213 LLVMValueRef multiplier, LLVMValueRef post_shift);
214
215 void ac_prepare_cube_coords(struct ac_llvm_context *ctx, bool is_deriv, bool is_array, bool is_lod,
216 LLVMValueRef *coords_arg, LLVMValueRef *derivs_arg);
217
218 LLVMValueRef ac_build_fs_interp(struct ac_llvm_context *ctx, LLVMValueRef llvm_chan,
219 LLVMValueRef attr_number, LLVMValueRef params, LLVMValueRef i,
220 LLVMValueRef j);
221
222 LLVMValueRef ac_build_fs_interp_f16(struct ac_llvm_context *ctx, LLVMValueRef llvm_chan,
223 LLVMValueRef attr_number, LLVMValueRef params, LLVMValueRef i,
224 LLVMValueRef j);
225
226 LLVMValueRef ac_build_fs_interp_mov(struct ac_llvm_context *ctx, LLVMValueRef parameter,
227 LLVMValueRef llvm_chan, LLVMValueRef attr_number,
228 LLVMValueRef params);
229
230 LLVMValueRef ac_build_gep_ptr(struct ac_llvm_context *ctx, LLVMValueRef base_ptr,
231 LLVMValueRef index);
232
233 LLVMValueRef ac_build_gep0(struct ac_llvm_context *ctx, LLVMValueRef base_ptr, LLVMValueRef index);
234 LLVMValueRef ac_build_pointer_add(struct ac_llvm_context *ctx, LLVMValueRef ptr,
235 LLVMValueRef index);
236
237 void ac_build_indexed_store(struct ac_llvm_context *ctx, LLVMValueRef base_ptr, LLVMValueRef index,
238 LLVMValueRef value);
239
240 LLVMValueRef ac_build_load(struct ac_llvm_context *ctx, LLVMValueRef base_ptr, LLVMValueRef index);
241 LLVMValueRef ac_build_load_invariant(struct ac_llvm_context *ctx, LLVMValueRef base_ptr,
242 LLVMValueRef index);
243 LLVMValueRef ac_build_load_to_sgpr(struct ac_llvm_context *ctx, LLVMValueRef base_ptr,
244 LLVMValueRef index);
245 LLVMValueRef ac_build_load_to_sgpr_uint_wraparound(struct ac_llvm_context *ctx,
246 LLVMValueRef base_ptr, LLVMValueRef index);
247
248 void ac_build_buffer_store_dword(struct ac_llvm_context *ctx, LLVMValueRef rsrc, LLVMValueRef vdata,
249 unsigned num_channels, LLVMValueRef voffset, LLVMValueRef soffset,
250 unsigned inst_offset, unsigned cache_policy);
251
252 void ac_build_buffer_store_format(struct ac_llvm_context *ctx, LLVMValueRef rsrc, LLVMValueRef data,
253 LLVMValueRef vindex, LLVMValueRef voffset, unsigned cache_policy);
254
255 LLVMValueRef ac_build_buffer_load(struct ac_llvm_context *ctx, LLVMValueRef rsrc, int num_channels,
256 LLVMValueRef vindex, LLVMValueRef voffset, LLVMValueRef soffset,
257 unsigned inst_offset, unsigned cache_policy, bool can_speculate,
258 bool allow_smem);
259
260 LLVMValueRef ac_build_buffer_load_format(struct ac_llvm_context *ctx, LLVMValueRef rsrc,
261 LLVMValueRef vindex, LLVMValueRef voffset,
262 unsigned num_channels, unsigned cache_policy,
263 bool can_speculate, bool d16);
264
265 LLVMValueRef ac_build_tbuffer_load_short(struct ac_llvm_context *ctx, LLVMValueRef rsrc,
266 LLVMValueRef voffset, LLVMValueRef soffset,
267 LLVMValueRef immoffset, unsigned cache_policy);
268
269 LLVMValueRef ac_build_tbuffer_load_byte(struct ac_llvm_context *ctx, LLVMValueRef rsrc,
270 LLVMValueRef voffset, LLVMValueRef soffset,
271 LLVMValueRef immoffset, unsigned cache_policy);
272
273 LLVMValueRef ac_build_struct_tbuffer_load(struct ac_llvm_context *ctx, LLVMValueRef rsrc,
274 LLVMValueRef vindex, LLVMValueRef voffset,
275 LLVMValueRef soffset, LLVMValueRef immoffset,
276 unsigned num_channels, unsigned dfmt, unsigned nfmt,
277 unsigned cache_policy, bool can_speculate);
278
279 LLVMValueRef ac_build_raw_tbuffer_load(struct ac_llvm_context *ctx, LLVMValueRef rsrc,
280 LLVMValueRef voffset, LLVMValueRef soffset,
281 LLVMValueRef immoffset, unsigned num_channels, unsigned dfmt,
282 unsigned nfmt, unsigned cache_policy, bool can_speculate);
283
284
285 LLVMValueRef ac_build_opencoded_load_format(struct ac_llvm_context *ctx, unsigned log_size,
286 unsigned num_channels, unsigned format, bool reverse,
287 bool known_aligned, LLVMValueRef rsrc,
288 LLVMValueRef vindex, LLVMValueRef voffset,
289 LLVMValueRef soffset, unsigned cache_policy,
290 bool can_speculate);
291
292 void ac_build_tbuffer_store_short(struct ac_llvm_context *ctx, LLVMValueRef rsrc,
293 LLVMValueRef vdata, LLVMValueRef voffset, LLVMValueRef soffset,
294 unsigned cache_policy);
295
296 void ac_build_tbuffer_store_byte(struct ac_llvm_context *ctx, LLVMValueRef rsrc, LLVMValueRef vdata,
297 LLVMValueRef voffset, LLVMValueRef soffset, unsigned cache_policy);
298
299 void ac_build_struct_tbuffer_store(struct ac_llvm_context *ctx, LLVMValueRef rsrc,
300 LLVMValueRef vdata, LLVMValueRef vindex, LLVMValueRef voffset,
301 LLVMValueRef soffset, LLVMValueRef immoffset,
302 unsigned num_channels, unsigned dfmt, unsigned nfmt,
303 unsigned cache_policy);
304
305 void ac_build_raw_tbuffer_store(struct ac_llvm_context *ctx, LLVMValueRef rsrc, LLVMValueRef vdata,
306 LLVMValueRef voffset, LLVMValueRef soffset, LLVMValueRef immoffset,
307 unsigned num_channels, unsigned dfmt, unsigned nfmt,
308 unsigned cache_policy);
309
310 LLVMValueRef ac_get_thread_id(struct ac_llvm_context *ctx);
311
312 #define AC_TID_MASK_TOP_LEFT 0xfffffffc
313 #define AC_TID_MASK_TOP 0xfffffffd
314 #define AC_TID_MASK_LEFT 0xfffffffe
315
316 LLVMValueRef ac_build_ddxy(struct ac_llvm_context *ctx, uint32_t mask, int idx, LLVMValueRef val);
317
318 #define AC_SENDMSG_GS 2
319 #define AC_SENDMSG_GS_DONE 3
320 #define AC_SENDMSG_GS_ALLOC_REQ 9
321
322 #define AC_SENDMSG_GS_OP_NOP (0 << 4)
323 #define AC_SENDMSG_GS_OP_CUT (1 << 4)
324 #define AC_SENDMSG_GS_OP_EMIT (2 << 4)
325 #define AC_SENDMSG_GS_OP_EMIT_CUT (3 << 4)
326
327 void ac_build_sendmsg(struct ac_llvm_context *ctx, uint32_t msg, LLVMValueRef wave_id);
328
329 LLVMValueRef ac_build_imsb(struct ac_llvm_context *ctx, LLVMValueRef arg, LLVMTypeRef dst_type);
330
331 LLVMValueRef ac_build_umsb(struct ac_llvm_context *ctx, LLVMValueRef arg, LLVMTypeRef dst_type);
332 LLVMValueRef ac_build_fmin(struct ac_llvm_context *ctx, LLVMValueRef a, LLVMValueRef b);
333 LLVMValueRef ac_build_fmax(struct ac_llvm_context *ctx, LLVMValueRef a, LLVMValueRef b);
334 LLVMValueRef ac_build_imin(struct ac_llvm_context *ctx, LLVMValueRef a, LLVMValueRef b);
335 LLVMValueRef ac_build_imax(struct ac_llvm_context *ctx, LLVMValueRef a, LLVMValueRef b);
336 LLVMValueRef ac_build_umin(struct ac_llvm_context *ctx, LLVMValueRef a, LLVMValueRef b);
337 LLVMValueRef ac_build_umax(struct ac_llvm_context *ctx, LLVMValueRef a, LLVMValueRef b);
338 LLVMValueRef ac_build_clamp(struct ac_llvm_context *ctx, LLVMValueRef value);
339
340 struct ac_export_args {
341 LLVMValueRef out[4];
342 unsigned target;
343 unsigned enabled_channels;
344 bool compr;
345 bool done;
346 bool valid_mask;
347 };
348
349 void ac_build_export(struct ac_llvm_context *ctx, struct ac_export_args *a);
350
351 void ac_build_export_null(struct ac_llvm_context *ctx);
352
353 enum ac_image_opcode
354 {
355 ac_image_sample,
356 ac_image_gather4,
357 ac_image_load,
358 ac_image_load_mip,
359 ac_image_store,
360 ac_image_store_mip,
361 ac_image_get_lod,
362 ac_image_get_resinfo,
363 ac_image_atomic,
364 ac_image_atomic_cmpswap,
365 };
366
367 enum ac_atomic_op
368 {
369 ac_atomic_swap,
370 ac_atomic_add,
371 ac_atomic_sub,
372 ac_atomic_smin,
373 ac_atomic_umin,
374 ac_atomic_smax,
375 ac_atomic_umax,
376 ac_atomic_and,
377 ac_atomic_or,
378 ac_atomic_xor,
379 ac_atomic_inc_wrap,
380 ac_atomic_dec_wrap,
381 };
382
383 /* These cache policy bits match the definitions used by the LLVM intrinsics. */
384 enum ac_image_cache_policy
385 {
386 ac_glc = 1 << 0, /* per-CU cache control */
387 ac_slc = 1 << 1, /* global L2 cache control */
388 ac_dlc = 1 << 2, /* per-shader-array cache control */
389 ac_swizzled = 1 << 3, /* the access is swizzled, disabling load/store merging */
390 };
391
392 struct ac_image_args {
393 enum ac_image_opcode opcode : 4;
394 enum ac_atomic_op atomic : 4; /* for the ac_image_atomic opcode */
395 enum ac_image_dim dim : 3;
396 unsigned dmask : 4;
397 unsigned cache_policy : 3;
398 bool unorm : 1;
399 bool level_zero : 1;
400 bool d16 : 1; /* data and return values are 16-bit, requires GFX8+ */
401 unsigned attributes; /* additional call-site specific AC_FUNC_ATTRs */
402
403 LLVMValueRef resource;
404 LLVMValueRef sampler;
405 LLVMValueRef data[2]; /* data[0] is source data (vector); data[1] is cmp for cmpswap */
406 LLVMValueRef offset;
407 LLVMValueRef bias;
408 LLVMValueRef compare;
409 LLVMValueRef derivs[6];
410 LLVMValueRef coords[4];
411 LLVMValueRef lod; // also used by ac_image_get_resinfo
412 LLVMValueRef min_lod;
413 };
414
415 LLVMValueRef ac_build_image_opcode(struct ac_llvm_context *ctx, struct ac_image_args *a);
416 LLVMValueRef ac_build_image_get_sample_count(struct ac_llvm_context *ctx, LLVMValueRef rsrc);
417 LLVMValueRef ac_build_cvt_pkrtz_f16(struct ac_llvm_context *ctx, LLVMValueRef args[2]);
418 LLVMValueRef ac_build_cvt_pknorm_i16(struct ac_llvm_context *ctx, LLVMValueRef args[2]);
419 LLVMValueRef ac_build_cvt_pknorm_u16(struct ac_llvm_context *ctx, LLVMValueRef args[2]);
420 LLVMValueRef ac_build_cvt_pknorm_i16_f16(struct ac_llvm_context *ctx, LLVMValueRef args[2]);
421 LLVMValueRef ac_build_cvt_pknorm_u16_f16(struct ac_llvm_context *ctx, LLVMValueRef args[2]);
422 LLVMValueRef ac_build_cvt_pk_i16(struct ac_llvm_context *ctx, LLVMValueRef args[2], unsigned bits,
423 bool hi);
424 LLVMValueRef ac_build_cvt_pk_u16(struct ac_llvm_context *ctx, LLVMValueRef args[2], unsigned bits,
425 bool hi);
426 LLVMValueRef ac_build_wqm_vote(struct ac_llvm_context *ctx, LLVMValueRef i1);
427 void ac_build_kill_if_false(struct ac_llvm_context *ctx, LLVMValueRef i1);
428 LLVMValueRef ac_build_bfe(struct ac_llvm_context *ctx, LLVMValueRef input, LLVMValueRef offset,
429 LLVMValueRef width, bool is_signed);
430 LLVMValueRef ac_build_imad(struct ac_llvm_context *ctx, LLVMValueRef s0, LLVMValueRef s1,
431 LLVMValueRef s2);
432 LLVMValueRef ac_build_fmad(struct ac_llvm_context *ctx, LLVMValueRef s0, LLVMValueRef s1,
433 LLVMValueRef s2);
434
435 void ac_build_waitcnt(struct ac_llvm_context *ctx, unsigned wait_flags);
436
437 LLVMValueRef ac_build_fract(struct ac_llvm_context *ctx, LLVMValueRef src0, unsigned bitsize);
438 LLVMValueRef ac_const_uint_vec(struct ac_llvm_context *ctx, LLVMTypeRef type, uint64_t value);
439 LLVMValueRef ac_build_isign(struct ac_llvm_context *ctx, LLVMValueRef src0);
440 LLVMValueRef ac_build_fsign(struct ac_llvm_context *ctx, LLVMValueRef src);
441 LLVMValueRef ac_build_bit_count(struct ac_llvm_context *ctx, LLVMValueRef src0);
442
443 LLVMValueRef ac_build_fsat(struct ac_llvm_context *ctx, LLVMValueRef src,
444 LLVMTypeRef type);
445
446 LLVMValueRef ac_build_bitfield_reverse(struct ac_llvm_context *ctx, LLVMValueRef src0);
447
448 void ac_optimize_vs_outputs(struct ac_llvm_context *ac, LLVMValueRef main_fn,
449 uint8_t *vs_output_param_offset, uint32_t num_outputs,
450 uint32_t skip_output_mask, uint8_t *num_param_exports);
451 void ac_init_exec_full_mask(struct ac_llvm_context *ctx);
452
453 void ac_declare_lds_as_pointer(struct ac_llvm_context *ac);
454 LLVMValueRef ac_lds_load(struct ac_llvm_context *ctx, LLVMValueRef dw_addr);
455 void ac_lds_store(struct ac_llvm_context *ctx, LLVMValueRef dw_addr, LLVMValueRef value);
456
457 LLVMValueRef ac_find_lsb(struct ac_llvm_context *ctx, LLVMTypeRef dst_type, LLVMValueRef src0);
458
459 LLVMTypeRef ac_array_in_const_addr_space(LLVMTypeRef elem_type);
460 LLVMTypeRef ac_array_in_const32_addr_space(LLVMTypeRef elem_type);
461
462 void ac_build_bgnloop(struct ac_llvm_context *ctx, int lable_id);
463 void ac_build_break(struct ac_llvm_context *ctx);
464 void ac_build_continue(struct ac_llvm_context *ctx);
465 void ac_build_else(struct ac_llvm_context *ctx, int lable_id);
466 void ac_build_endif(struct ac_llvm_context *ctx, int lable_id);
467 void ac_build_endloop(struct ac_llvm_context *ctx, int lable_id);
468 void ac_build_ifcc(struct ac_llvm_context *ctx, LLVMValueRef cond, int label_id);
469
470 LLVMValueRef ac_build_alloca(struct ac_llvm_context *ac, LLVMTypeRef type, const char *name);
471 LLVMValueRef ac_build_alloca_undef(struct ac_llvm_context *ac, LLVMTypeRef type, const char *name);
472
473 LLVMValueRef ac_cast_ptr(struct ac_llvm_context *ctx, LLVMValueRef ptr, LLVMTypeRef type);
474
475 LLVMValueRef ac_trim_vector(struct ac_llvm_context *ctx, LLVMValueRef value, unsigned count);
476
477 LLVMValueRef ac_unpack_param(struct ac_llvm_context *ctx, LLVMValueRef param, unsigned rshift,
478 unsigned bitwidth);
479
480 void ac_apply_fmask_to_sample(struct ac_llvm_context *ac, LLVMValueRef fmask, LLVMValueRef *addr,
481 bool is_array_tex);
482
483 LLVMValueRef ac_build_ds_swizzle(struct ac_llvm_context *ctx, LLVMValueRef src, unsigned mask);
484
485 LLVMValueRef ac_build_readlane_no_opt_barrier(struct ac_llvm_context *ctx, LLVMValueRef src,
486 LLVMValueRef lane);
487
488 LLVMValueRef ac_build_readlane(struct ac_llvm_context *ctx, LLVMValueRef src, LLVMValueRef lane);
489
490 LLVMValueRef ac_build_writelane(struct ac_llvm_context *ctx, LLVMValueRef src, LLVMValueRef value,
491 LLVMValueRef lane);
492
493 LLVMValueRef ac_build_mbcnt(struct ac_llvm_context *ctx, LLVMValueRef mask);
494
495 LLVMValueRef ac_build_inclusive_scan(struct ac_llvm_context *ctx, LLVMValueRef src, nir_op op);
496
497 LLVMValueRef ac_build_exclusive_scan(struct ac_llvm_context *ctx, LLVMValueRef src, nir_op op);
498
499 LLVMValueRef ac_build_reduce(struct ac_llvm_context *ctx, LLVMValueRef src, nir_op op,
500 unsigned cluster_size);
501
502 /**
503 * Common arguments for a scan/reduce operation that accumulates per-wave
504 * values across an entire workgroup, while respecting the order of waves.
505 */
506 struct ac_wg_scan {
507 bool enable_reduce;
508 bool enable_exclusive;
509 bool enable_inclusive;
510 nir_op op;
511 LLVMValueRef src; /* clobbered! */
512 LLVMValueRef result_reduce;
513 LLVMValueRef result_exclusive;
514 LLVMValueRef result_inclusive;
515 LLVMValueRef extra;
516 LLVMValueRef waveidx;
517 LLVMValueRef numwaves; /* only needed for "reduce" operations */
518
519 /* T addrspace(LDS) pointer to the same type as value, at least maxwaves entries */
520 LLVMValueRef scratch;
521 unsigned maxwaves;
522 };
523
524 void ac_build_wg_wavescan_top(struct ac_llvm_context *ctx, struct ac_wg_scan *ws);
525 void ac_build_wg_wavescan_bottom(struct ac_llvm_context *ctx, struct ac_wg_scan *ws);
526 void ac_build_wg_wavescan(struct ac_llvm_context *ctx, struct ac_wg_scan *ws);
527
528 void ac_build_wg_scan_top(struct ac_llvm_context *ctx, struct ac_wg_scan *ws);
529 void ac_build_wg_scan_bottom(struct ac_llvm_context *ctx, struct ac_wg_scan *ws);
530 void ac_build_wg_scan(struct ac_llvm_context *ctx, struct ac_wg_scan *ws);
531
532 LLVMValueRef ac_build_quad_swizzle(struct ac_llvm_context *ctx, LLVMValueRef src, unsigned lane0,
533 unsigned lane1, unsigned lane2, unsigned lane3);
534
535 LLVMValueRef ac_build_shuffle(struct ac_llvm_context *ctx, LLVMValueRef src, LLVMValueRef index);
536
537 LLVMValueRef ac_build_frexp_exp(struct ac_llvm_context *ctx, LLVMValueRef src0, unsigned bitsize);
538
539 LLVMValueRef ac_build_frexp_mant(struct ac_llvm_context *ctx, LLVMValueRef src0, unsigned bitsize);
540
541 LLVMValueRef ac_build_canonicalize(struct ac_llvm_context *ctx, LLVMValueRef src0,
542 unsigned bitsize);
543
544 LLVMValueRef ac_build_ddxy_interp(struct ac_llvm_context *ctx, LLVMValueRef interp_ij);
545
546 LLVMValueRef ac_build_load_helper_invocation(struct ac_llvm_context *ctx);
547
548 LLVMValueRef ac_build_is_helper_invocation(struct ac_llvm_context *ctx);
549
550 LLVMValueRef ac_build_call(struct ac_llvm_context *ctx, LLVMValueRef func, LLVMValueRef *args,
551 unsigned num_args);
552
553 LLVMValueRef ac_build_atomic_rmw(struct ac_llvm_context *ctx, LLVMAtomicRMWBinOp op,
554 LLVMValueRef ptr, LLVMValueRef val, const char *sync_scope);
555
556 LLVMValueRef ac_build_atomic_cmp_xchg(struct ac_llvm_context *ctx, LLVMValueRef ptr,
557 LLVMValueRef cmp, LLVMValueRef val, const char *sync_scope);
558
559 void ac_export_mrt_z(struct ac_llvm_context *ctx, LLVMValueRef depth, LLVMValueRef stencil,
560 LLVMValueRef samplemask, struct ac_export_args *args);
561
562 void ac_build_sendmsg_gs_alloc_req(struct ac_llvm_context *ctx, LLVMValueRef wave_id,
563 LLVMValueRef vtx_cnt, LLVMValueRef prim_cnt);
564
565 struct ac_ngg_prim {
566 unsigned num_vertices;
567 LLVMValueRef isnull;
568 LLVMValueRef index[3];
569 LLVMValueRef edgeflag[3];
570 LLVMValueRef passthrough;
571 };
572
573 LLVMValueRef ac_pack_prim_export(struct ac_llvm_context *ctx, const struct ac_ngg_prim *prim);
574 void ac_build_export_prim(struct ac_llvm_context *ctx, const struct ac_ngg_prim *prim);
575
ac_get_arg(struct ac_llvm_context * ctx,struct ac_arg arg)576 static inline LLVMValueRef ac_get_arg(struct ac_llvm_context *ctx, struct ac_arg arg)
577 {
578 assert(arg.used);
579 return LLVMGetParam(ctx->main_function, arg.arg_index);
580 }
581
582 enum ac_llvm_calling_convention
583 {
584 AC_LLVM_AMDGPU_VS = 87,
585 AC_LLVM_AMDGPU_GS = 88,
586 AC_LLVM_AMDGPU_PS = 89,
587 AC_LLVM_AMDGPU_CS = 90,
588 AC_LLVM_AMDGPU_HS = 93,
589 };
590
591 LLVMValueRef ac_build_main(const struct ac_shader_args *args, struct ac_llvm_context *ctx,
592 enum ac_llvm_calling_convention convention, const char *name,
593 LLVMTypeRef ret_type, LLVMModuleRef module);
594 void ac_build_s_endpgm(struct ac_llvm_context *ctx);
595
596 LLVMValueRef ac_prefix_bitcount(struct ac_llvm_context *ctx, LLVMValueRef mask, LLVMValueRef index);
597 LLVMValueRef ac_prefix_bitcount_2x64(struct ac_llvm_context *ctx, LLVMValueRef mask[2],
598 LLVMValueRef index);
599 void ac_build_triangle_strip_indices_to_triangle(struct ac_llvm_context *ctx, LLVMValueRef is_odd,
600 LLVMValueRef flatshade_first,
601 LLVMValueRef index[3]);
602
603 #ifdef __cplusplus
604 }
605 #endif
606
607 #endif
608