1 /**************************************************************************
2 *
3 * Copyright 2019 Red Hat.
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included
14 * in all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
17 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 *
24 **************************************************************************/
25
26 #ifndef LP_BLD_NIR_H
27 #define LP_BLD_NIR_H
28
29 #include "gallivm/lp_bld.h"
30 #include "gallivm/lp_bld_limits.h"
31 #include "lp_bld_type.h"
32
33 #include "gallivm/lp_bld_tgsi.h"
34 #include "nir.h"
35
36 struct nir_shader;
37
38 /*
39 * 2 reserved functions args for each function call,
40 * exec mask and context.
41 */
42 #define LP_RESV_FUNC_ARGS 2
43
44 void lp_build_nir_soa(struct gallivm_state *gallivm,
45 struct nir_shader *shader,
46 const struct lp_build_tgsi_params *params,
47 LLVMValueRef (*outputs)[4]);
48
49 void lp_build_nir_soa_func(struct gallivm_state *gallivm,
50 struct nir_shader *shader,
51 nir_function_impl *impl,
52 const struct lp_build_tgsi_params *params,
53 LLVMValueRef (*outputs)[4]);
54
55 void lp_build_nir_aos(struct gallivm_state *gallivm,
56 struct nir_shader *shader,
57 struct lp_type type,
58 const unsigned char swizzles[4],
59 LLVMValueRef consts_ptr,
60 const LLVMValueRef *inputs,
61 LLVMValueRef *outputs,
62 const struct lp_build_sampler_aos *sampler);
63
64 struct lp_build_fn {
65 LLVMTypeRef fn_type;
66 LLVMValueRef fn;
67 };
68
69 struct lp_build_nir_context
70 {
71 struct lp_build_context base;
72 struct lp_build_context uint_bld;
73 struct lp_build_context int_bld;
74 struct lp_build_context uint8_bld;
75 struct lp_build_context int8_bld;
76 struct lp_build_context uint16_bld;
77 struct lp_build_context int16_bld;
78 struct lp_build_context half_bld;
79 struct lp_build_context dbl_bld;
80 struct lp_build_context uint64_bld;
81 struct lp_build_context int64_bld;
82
83 LLVMValueRef *ssa_defs;
84 struct hash_table *regs;
85 struct hash_table *vars;
86 struct hash_table *fns;
87
88 /** Value range analysis hash table used in code generation. */
89 struct hash_table *range_ht;
90
91 LLVMValueRef aniso_filter_table;
92
93 LLVMValueRef func;
94 nir_shader *shader;
95
96 void (*load_ubo)(struct lp_build_nir_context *bld_base,
97 unsigned nc,
98 unsigned bit_size,
99 bool offset_is_uniform,
100 LLVMValueRef index, LLVMValueRef offset, LLVMValueRef result[NIR_MAX_VEC_COMPONENTS]);
101
102 void (*load_kernel_arg)(struct lp_build_nir_context *bld_base,
103 unsigned nc,
104 unsigned bit_size,
105 unsigned offset_bit_size,
106 bool offset_is_uniform,
107 LLVMValueRef offset, LLVMValueRef result[NIR_MAX_VEC_COMPONENTS]);
108
109 void (*load_global)(struct lp_build_nir_context *bld_base,
110 unsigned nc, unsigned bit_size,
111 unsigned offset_bit_size,
112 bool offset_is_global,
113 LLVMValueRef offset, LLVMValueRef result[NIR_MAX_VEC_COMPONENTS]);
114
115 void (*store_global)(struct lp_build_nir_context *bld_base,
116 unsigned writemask,
117 unsigned nc, unsigned bit_size,
118 unsigned addr_bit_size,
119 LLVMValueRef addr, LLVMValueRef dst);
120
121 void (*atomic_global)(struct lp_build_nir_context *bld_base,
122 nir_atomic_op nir_op,
123 unsigned addr_bit_size,
124 unsigned val_bit_size,
125 LLVMValueRef addr,
126 LLVMValueRef val, LLVMValueRef val2,
127 LLVMValueRef *result);
128
129 /* for SSBO and shared memory */
130 void (*load_mem)(struct lp_build_nir_context *bld_base,
131 unsigned nc, unsigned bit_size,
132 bool index_and_offset_are_uniform, bool payload,
133 LLVMValueRef index, LLVMValueRef offset, LLVMValueRef result[NIR_MAX_VEC_COMPONENTS]);
134 void (*store_mem)(struct lp_build_nir_context *bld_base,
135 unsigned writemask, unsigned nc, unsigned bit_size,
136 bool index_and_offset_are_uniform, bool payload,
137 LLVMValueRef index, LLVMValueRef offset, LLVMValueRef dst);
138
139 void (*atomic_mem)(struct lp_build_nir_context *bld_base,
140 nir_atomic_op op,
141 unsigned bit_size,
142 bool payload,
143 LLVMValueRef index, LLVMValueRef offset,
144 LLVMValueRef val, LLVMValueRef val2,
145 LLVMValueRef *result);
146
147 void (*barrier)(struct lp_build_nir_context *bld_base);
148
149 void (*image_op)(struct lp_build_nir_context *bld_base,
150 struct lp_img_params *params);
151 void (*image_size)(struct lp_build_nir_context *bld_base,
152 struct lp_sampler_size_query_params *params);
153 LLVMValueRef (*get_ssbo_size)(struct lp_build_nir_context *bld_base,
154 LLVMValueRef index);
155
156 void (*load_const)(struct lp_build_nir_context *bld_base,
157 const nir_load_const_instr *instr,
158 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS]);
159 void (*load_var)(struct lp_build_nir_context *bld_base,
160 nir_variable_mode deref_mode,
161 unsigned num_components,
162 unsigned bit_size,
163 nir_variable *var,
164 unsigned vertex_index,
165 LLVMValueRef indir_vertex_index,
166 unsigned const_index,
167 LLVMValueRef indir_index,
168 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS]);
169 void (*store_var)(struct lp_build_nir_context *bld_base,
170 nir_variable_mode deref_mode,
171 unsigned num_components,
172 unsigned bit_size,
173 nir_variable *var,
174 unsigned writemask,
175 LLVMValueRef indir_vertex_index,
176 unsigned const_index,
177 LLVMValueRef indir_index,
178 LLVMValueRef dst);
179
180 LLVMValueRef (*load_reg)(struct lp_build_nir_context *bld_base,
181 struct lp_build_context *reg_bld,
182 const nir_intrinsic_instr *decl,
183 unsigned base,
184 LLVMValueRef indir_src,
185 LLVMValueRef reg_storage);
186 void (*store_reg)(struct lp_build_nir_context *bld_base,
187 struct lp_build_context *reg_bld,
188 const nir_intrinsic_instr *decl,
189 unsigned writemask,
190 unsigned base,
191 LLVMValueRef indir_src,
192 LLVMValueRef reg_storage,
193 LLVMValueRef dst[NIR_MAX_VEC_COMPONENTS]);
194
195 void (*load_scratch)(struct lp_build_nir_context *bld_base,
196 unsigned nc, unsigned bit_size,
197 LLVMValueRef offset,
198 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS]);
199 void (*store_scratch)(struct lp_build_nir_context *bld_base,
200 unsigned writemask, unsigned nc,
201 unsigned bit_size, LLVMValueRef offset,
202 LLVMValueRef val);
203
204 void (*emit_var_decl)(struct lp_build_nir_context *bld_base,
205 nir_variable *var);
206
207 void (*tex)(struct lp_build_nir_context *bld_base,
208 struct lp_sampler_params *params);
209
210 void (*tex_size)(struct lp_build_nir_context *bld_base,
211 struct lp_sampler_size_query_params *params);
212
213 void (*sysval_intrin)(struct lp_build_nir_context *bld_base,
214 nir_intrinsic_instr *instr,
215 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS]);
216 void (*discard)(struct lp_build_nir_context *bld_base,
217 LLVMValueRef cond);
218
219 void (*bgnloop)(struct lp_build_nir_context *bld_base);
220 void (*endloop)(struct lp_build_nir_context *bld_base);
221 void (*if_cond)(struct lp_build_nir_context *bld_base, LLVMValueRef cond);
222 void (*else_stmt)(struct lp_build_nir_context *bld_base);
223 void (*endif_stmt)(struct lp_build_nir_context *bld_base);
224 void (*break_stmt)(struct lp_build_nir_context *bld_base);
225 void (*continue_stmt)(struct lp_build_nir_context *bld_base);
226
227 void (*emit_vertex)(struct lp_build_nir_context *bld_base, uint32_t stream_id);
228 void (*end_primitive)(struct lp_build_nir_context *bld_base, uint32_t stream_id);
229
230 void (*vote)(struct lp_build_nir_context *bld_base, LLVMValueRef src, nir_intrinsic_instr *instr, LLVMValueRef dst[4]);
231 void (*elect)(struct lp_build_nir_context *bld_base, LLVMValueRef dst[4]);
232 void (*reduce)(struct lp_build_nir_context *bld_base, LLVMValueRef src, nir_intrinsic_instr *instr, LLVMValueRef dst[4]);
233 void (*ballot)(struct lp_build_nir_context *bld_base, LLVMValueRef src, nir_intrinsic_instr *instr, LLVMValueRef dst[4]);
234 #if LLVM_VERSION_MAJOR >= 10
235 void (*shuffle)(struct lp_build_nir_context *bld_base,
236 LLVMValueRef src,
237 LLVMValueRef index,
238 nir_intrinsic_instr *instr,
239 LLVMValueRef dst[4]);
240 #endif
241 void (*read_invocation)(struct lp_build_nir_context *bld_base,
242 LLVMValueRef src, unsigned bit_size, LLVMValueRef invoc,
243 LLVMValueRef dst[4]);
244 void (*helper_invocation)(struct lp_build_nir_context *bld_base, LLVMValueRef *dst);
245
246 void (*clock)(struct lp_build_nir_context *bld_Base, LLVMValueRef dst[4]);
247 void (*interp_at)(struct lp_build_nir_context *bld_base,
248 unsigned num_components,
249 nir_variable *var,
250 bool centroid, bool sample,
251 unsigned const_index,
252 LLVMValueRef indir_index,
253 LLVMValueRef offsets[2], LLVMValueRef dst[4]);
254 void (*set_vertex_and_primitive_count)(struct lp_build_nir_context *bld_base,
255 LLVMValueRef vert_count,
256 LLVMValueRef prim_count);
257 void (*launch_mesh_workgroups)(struct lp_build_nir_context *bld_base,
258 LLVMValueRef launch_grid);
259
260 void (*call)(struct lp_build_nir_context *bld_base,
261 struct lp_build_fn *fn,
262 int num_args,
263 LLVMValueRef *args);
264 // LLVMValueRef main_function
265 };
266
267 struct lp_build_nir_soa_context
268 {
269 struct lp_build_nir_context bld_base;
270
271 /* Builder for scalar elements of shader's data type (float) */
272 struct lp_build_context elem_bld;
273 struct lp_build_context uint_elem_bld;
274
275 LLVMValueRef consts_ptr;
276 const LLVMValueRef (*inputs)[TGSI_NUM_CHANNELS];
277 LLVMValueRef (*outputs)[TGSI_NUM_CHANNELS];
278 int num_inputs;
279 LLVMTypeRef context_type;
280 LLVMValueRef context_ptr;
281 LLVMTypeRef resources_type;
282 LLVMValueRef resources_ptr;
283 LLVMTypeRef thread_data_type;
284 LLVMValueRef thread_data_ptr;
285
286 LLVMValueRef ssbo_ptr;
287
288 LLVMValueRef shared_ptr;
289 LLVMValueRef payload_ptr;
290 LLVMValueRef scratch_ptr;
291 unsigned scratch_size;
292
293 const struct lp_build_coro_suspend_info *coro;
294
295 const struct lp_build_sampler_soa *sampler;
296 const struct lp_build_image_soa *image;
297
298 const struct lp_build_gs_iface *gs_iface;
299 const struct lp_build_tcs_iface *tcs_iface;
300 const struct lp_build_tes_iface *tes_iface;
301 const struct lp_build_fs_iface *fs_iface;
302 const struct lp_build_mesh_iface *mesh_iface;
303
304 LLVMValueRef emitted_prims_vec_ptr[PIPE_MAX_VERTEX_STREAMS];
305 LLVMValueRef total_emitted_vertices_vec_ptr[PIPE_MAX_VERTEX_STREAMS];
306 LLVMValueRef emitted_vertices_vec_ptr[PIPE_MAX_VERTEX_STREAMS];
307 LLVMValueRef max_output_vertices_vec;
308 struct lp_bld_tgsi_system_values system_values;
309
310 nir_variable_mode indirects;
311 struct lp_build_mask_context *mask;
312 struct lp_exec_mask exec_mask;
313
314 /* We allocate/use this array of inputs if (indirects & nir_var_shader_in) is
315 * set. The inputs[] array above is unused then.
316 */
317 LLVMValueRef inputs_array;
318
319 LLVMValueRef kernel_args_ptr;
320 unsigned gs_vertex_streams;
321
322 LLVMTypeRef call_context_type;
323 LLVMValueRef call_context_ptr;
324 };
325
326 void
327 lp_build_nir_prepasses(struct nir_shader *nir);
328
329 bool
330 lp_build_nir_llvm(struct lp_build_nir_context *bld_base,
331 struct nir_shader *nir,
332 nir_function_impl *impl);
333
334 void
335 lp_build_opt_nir(struct nir_shader *nir);
336
337
338 static inline LLVMValueRef
lp_nir_array_build_gather_values(LLVMBuilderRef builder,LLVMValueRef * values,unsigned value_count)339 lp_nir_array_build_gather_values(LLVMBuilderRef builder,
340 LLVMValueRef * values,
341 unsigned value_count)
342 {
343 LLVMTypeRef arr_type = LLVMArrayType(LLVMTypeOf(values[0]), value_count);
344 LLVMValueRef arr = LLVMGetUndef(arr_type);
345
346 for (unsigned i = 0; i < value_count; i++) {
347 arr = LLVMBuildInsertValue(builder, arr, values[i], i, "");
348 }
349 return arr;
350 }
351
352
353 static inline struct lp_build_context *
get_flt_bld(struct lp_build_nir_context * bld_base,unsigned op_bit_size)354 get_flt_bld(struct lp_build_nir_context *bld_base,
355 unsigned op_bit_size)
356 {
357 switch (op_bit_size) {
358 case 64:
359 return &bld_base->dbl_bld;
360 case 16:
361 return &bld_base->half_bld;
362 default:
363 case 32:
364 return &bld_base->base;
365 }
366 }
367
368
369 static inline struct lp_build_context *
get_int_bld(struct lp_build_nir_context * bld_base,bool is_unsigned,unsigned op_bit_size)370 get_int_bld(struct lp_build_nir_context *bld_base,
371 bool is_unsigned,
372 unsigned op_bit_size)
373 {
374 if (is_unsigned) {
375 switch (op_bit_size) {
376 case 64:
377 return &bld_base->uint64_bld;
378 case 32:
379 default:
380 return &bld_base->uint_bld;
381 case 16:
382 return &bld_base->uint16_bld;
383 case 8:
384 return &bld_base->uint8_bld;
385 }
386 } else {
387 switch (op_bit_size) {
388 case 64:
389 return &bld_base->int64_bld;
390 default:
391 case 32:
392 return &bld_base->int_bld;
393 case 16:
394 return &bld_base->int16_bld;
395 case 8:
396 return &bld_base->int8_bld;
397 }
398 }
399 }
400
401
402 unsigned
403 lp_nir_aos_swizzle(struct lp_build_nir_context *bld_base, unsigned chan);
404
405 LLVMAtomicRMWBinOp
406 lp_translate_atomic_op(nir_atomic_op op);
407
408 uint32_t
409 lp_build_nir_sample_key(gl_shader_stage stage, nir_tex_instr *instr);
410
411
412 void lp_img_op_from_intrinsic(struct lp_img_params *params, nir_intrinsic_instr *instr);
413
414 enum lp_nir_call_context_args {
415 LP_NIR_CALL_CONTEXT_CONTEXT,
416 LP_NIR_CALL_CONTEXT_RESOURCES,
417 LP_NIR_CALL_CONTEXT_SHARED,
418 LP_NIR_CALL_CONTEXT_SCRATCH,
419 LP_NIR_CALL_CONTEXT_WORK_DIM,
420 LP_NIR_CALL_CONTEXT_THREAD_ID_0,
421 LP_NIR_CALL_CONTEXT_THREAD_ID_1,
422 LP_NIR_CALL_CONTEXT_THREAD_ID_2,
423 LP_NIR_CALL_CONTEXT_BLOCK_ID_0,
424 LP_NIR_CALL_CONTEXT_BLOCK_ID_1,
425 LP_NIR_CALL_CONTEXT_BLOCK_ID_2,
426 LP_NIR_CALL_CONTEXT_GRID_SIZE_0,
427 LP_NIR_CALL_CONTEXT_GRID_SIZE_1,
428 LP_NIR_CALL_CONTEXT_GRID_SIZE_2,
429 LP_NIR_CALL_CONTEXT_BLOCK_SIZE_0,
430 LP_NIR_CALL_CONTEXT_BLOCK_SIZE_1,
431 LP_NIR_CALL_CONTEXT_BLOCK_SIZE_2,
432 LP_NIR_CALL_CONTEXT_MAX_ARGS,
433 };
434
435 LLVMTypeRef
436 lp_build_cs_func_call_context(struct gallivm_state *gallivm, int length,
437 LLVMTypeRef context_type, LLVMTypeRef resources_type);
438
439
440
441 #endif
442