1 /**************************************************************************
2 *
3 * Copyright 2019 Red Hat.
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included
14 * in all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
17 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 *
24 **************************************************************************/
25
26 #ifndef LP_BLD_NIR_H
27 #define LP_BLD_NIR_H
28
29 #include "gallivm/lp_bld.h"
30 #include "gallivm/lp_bld_limits.h"
31 #include "gallivm/lp_bld_flow.h"
32 #include "lp_bld_type.h"
33
34 #include "gallivm/lp_bld_tgsi.h"
35 #include "nir.h"
36
37 struct nir_shader;
38
39 /*
40 * 2 reserved functions args for each function call,
41 * exec mask and context.
42 */
43 #define LP_RESV_FUNC_ARGS 2
44
45 void lp_build_nir_soa(struct gallivm_state *gallivm,
46 struct nir_shader *shader,
47 const struct lp_build_tgsi_params *params,
48 LLVMValueRef (*outputs)[4]);
49
50 void lp_build_nir_soa_func(struct gallivm_state *gallivm,
51 struct nir_shader *shader,
52 nir_function_impl *impl,
53 const struct lp_build_tgsi_params *params,
54 LLVMValueRef (*outputs)[4]);
55
56 void lp_build_nir_aos(struct gallivm_state *gallivm,
57 struct nir_shader *shader,
58 struct lp_type type,
59 const unsigned char swizzles[4],
60 LLVMValueRef consts_ptr,
61 const LLVMValueRef *inputs,
62 LLVMValueRef *outputs,
63 const struct lp_build_sampler_aos *sampler);
64
65 struct lp_build_fn {
66 LLVMTypeRef fn_type;
67 LLVMValueRef fn;
68 };
69
70 struct lp_build_nir_context
71 {
72 struct lp_build_context base;
73 struct lp_build_context uint_bld;
74 struct lp_build_context int_bld;
75 struct lp_build_context uint8_bld;
76 struct lp_build_context int8_bld;
77 struct lp_build_context uint16_bld;
78 struct lp_build_context int16_bld;
79 struct lp_build_context half_bld;
80 struct lp_build_context dbl_bld;
81 struct lp_build_context uint64_bld;
82 struct lp_build_context int64_bld;
83
84 LLVMValueRef *ssa_defs;
85 struct hash_table *regs;
86 struct hash_table *vars;
87 struct hash_table *fns;
88
89 /** Value range analysis hash table used in code generation. */
90 struct hash_table *range_ht;
91
92 LLVMValueRef func;
93 nir_shader *shader;
94
95 struct lp_build_if_state if_stack[LP_MAX_TGSI_NESTING];
96 uint32_t if_stack_size;
97
98 void (*load_ubo)(struct lp_build_nir_context *bld_base,
99 unsigned nc,
100 unsigned bit_size,
101 bool offset_is_uniform,
102 LLVMValueRef index, LLVMValueRef offset, LLVMValueRef result[NIR_MAX_VEC_COMPONENTS]);
103
104 void (*load_kernel_arg)(struct lp_build_nir_context *bld_base,
105 unsigned nc,
106 unsigned bit_size,
107 unsigned offset_bit_size,
108 bool offset_is_uniform,
109 LLVMValueRef offset, LLVMValueRef result[NIR_MAX_VEC_COMPONENTS]);
110
111 void (*load_global)(struct lp_build_nir_context *bld_base,
112 unsigned nc, unsigned bit_size,
113 unsigned offset_bit_size,
114 bool offset_is_global,
115 LLVMValueRef offset, LLVMValueRef result[NIR_MAX_VEC_COMPONENTS]);
116
117 void (*store_global)(struct lp_build_nir_context *bld_base,
118 unsigned writemask,
119 unsigned nc, unsigned bit_size,
120 unsigned addr_bit_size,
121 LLVMValueRef addr, LLVMValueRef dst);
122
123 void (*atomic_global)(struct lp_build_nir_context *bld_base,
124 nir_atomic_op nir_op,
125 unsigned addr_bit_size,
126 unsigned val_bit_size,
127 LLVMValueRef addr,
128 LLVMValueRef val, LLVMValueRef val2,
129 LLVMValueRef *result);
130
131 /* for SSBO and shared memory */
132 void (*load_mem)(struct lp_build_nir_context *bld_base,
133 unsigned nc, unsigned bit_size,
134 bool index_and_offset_are_uniform, bool payload,
135 LLVMValueRef index, LLVMValueRef offset, LLVMValueRef result[NIR_MAX_VEC_COMPONENTS]);
136 void (*store_mem)(struct lp_build_nir_context *bld_base,
137 unsigned writemask, unsigned nc, unsigned bit_size,
138 bool index_and_offset_are_uniform, bool payload,
139 LLVMValueRef index, LLVMValueRef offset, LLVMValueRef dst);
140
141 void (*atomic_mem)(struct lp_build_nir_context *bld_base,
142 nir_atomic_op op,
143 unsigned bit_size,
144 bool payload,
145 LLVMValueRef index, LLVMValueRef offset,
146 LLVMValueRef val, LLVMValueRef val2,
147 LLVMValueRef *result);
148
149 void (*barrier)(struct lp_build_nir_context *bld_base);
150
151 void (*image_op)(struct lp_build_nir_context *bld_base,
152 struct lp_img_params *params);
153 void (*image_size)(struct lp_build_nir_context *bld_base,
154 struct lp_sampler_size_query_params *params);
155 LLVMValueRef (*get_ssbo_size)(struct lp_build_nir_context *bld_base,
156 LLVMValueRef index);
157
158 void (*load_const)(struct lp_build_nir_context *bld_base,
159 const nir_load_const_instr *instr,
160 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS]);
161 void (*load_var)(struct lp_build_nir_context *bld_base,
162 nir_variable_mode deref_mode,
163 unsigned num_components,
164 unsigned bit_size,
165 nir_variable *var,
166 unsigned vertex_index,
167 LLVMValueRef indir_vertex_index,
168 unsigned const_index,
169 LLVMValueRef indir_index,
170 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS]);
171 void (*store_var)(struct lp_build_nir_context *bld_base,
172 nir_variable_mode deref_mode,
173 unsigned num_components,
174 unsigned bit_size,
175 nir_variable *var,
176 unsigned writemask,
177 LLVMValueRef indir_vertex_index,
178 unsigned const_index,
179 LLVMValueRef indir_index,
180 LLVMValueRef dst);
181
182 LLVMValueRef (*load_reg)(struct lp_build_nir_context *bld_base,
183 struct lp_build_context *reg_bld,
184 const nir_intrinsic_instr *decl,
185 unsigned base,
186 LLVMValueRef indir_src,
187 LLVMValueRef reg_storage);
188 void (*store_reg)(struct lp_build_nir_context *bld_base,
189 struct lp_build_context *reg_bld,
190 const nir_intrinsic_instr *decl,
191 unsigned writemask,
192 unsigned base,
193 LLVMValueRef indir_src,
194 LLVMValueRef reg_storage,
195 LLVMValueRef dst[NIR_MAX_VEC_COMPONENTS]);
196
197 void (*load_scratch)(struct lp_build_nir_context *bld_base,
198 unsigned nc, unsigned bit_size,
199 LLVMValueRef offset,
200 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS]);
201 void (*store_scratch)(struct lp_build_nir_context *bld_base,
202 unsigned writemask, unsigned nc,
203 unsigned bit_size, LLVMValueRef offset,
204 LLVMValueRef val);
205
206 void (*emit_var_decl)(struct lp_build_nir_context *bld_base,
207 nir_variable *var);
208
209 void (*tex)(struct lp_build_nir_context *bld_base,
210 struct lp_sampler_params *params);
211
212 void (*tex_size)(struct lp_build_nir_context *bld_base,
213 struct lp_sampler_size_query_params *params);
214
215 void (*sysval_intrin)(struct lp_build_nir_context *bld_base,
216 nir_intrinsic_instr *instr,
217 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS]);
218 void (*discard)(struct lp_build_nir_context *bld_base,
219 LLVMValueRef cond);
220
221 void (*bgnloop)(struct lp_build_nir_context *bld_base);
222 void (*endloop)(struct lp_build_nir_context *bld_base);
223 void (*if_cond)(struct lp_build_nir_context *bld_base, LLVMValueRef cond, bool flatten);
224 void (*else_stmt)(struct lp_build_nir_context *bld_base, bool flatten_then, bool flatten_else);
225 void (*endif_stmt)(struct lp_build_nir_context *bld_base, bool flatten);
226 void (*break_stmt)(struct lp_build_nir_context *bld_base);
227 void (*continue_stmt)(struct lp_build_nir_context *bld_base);
228
229 void (*emit_vertex)(struct lp_build_nir_context *bld_base, uint32_t stream_id);
230 void (*end_primitive)(struct lp_build_nir_context *bld_base, uint32_t stream_id);
231
232 void (*vote)(struct lp_build_nir_context *bld_base, LLVMValueRef src, nir_intrinsic_instr *instr, LLVMValueRef dst[4]);
233 void (*elect)(struct lp_build_nir_context *bld_base, LLVMValueRef dst[4]);
234 void (*reduce)(struct lp_build_nir_context *bld_base, LLVMValueRef src, nir_intrinsic_instr *instr, LLVMValueRef dst[4]);
235 void (*ballot)(struct lp_build_nir_context *bld_base, LLVMValueRef src, nir_intrinsic_instr *instr, LLVMValueRef dst[4]);
236 #if LLVM_VERSION_MAJOR >= 10
237 void (*shuffle)(struct lp_build_nir_context *bld_base,
238 LLVMValueRef src,
239 LLVMValueRef index,
240 nir_intrinsic_instr *instr,
241 LLVMValueRef dst[4]);
242 #endif
243 void (*read_invocation)(struct lp_build_nir_context *bld_base,
244 LLVMValueRef src, unsigned bit_size, LLVMValueRef invoc,
245 LLVMValueRef dst[4]);
246 void (*helper_invocation)(struct lp_build_nir_context *bld_base, LLVMValueRef *dst);
247
248 void (*clock)(struct lp_build_nir_context *bld_Base, LLVMValueRef dst[4]);
249 void (*interp_at)(struct lp_build_nir_context *bld_base,
250 unsigned num_components,
251 nir_variable *var,
252 bool centroid, bool sample,
253 unsigned const_index,
254 LLVMValueRef indir_index,
255 LLVMValueRef offsets[2], LLVMValueRef dst[4]);
256 void (*set_vertex_and_primitive_count)(struct lp_build_nir_context *bld_base,
257 LLVMValueRef vert_count,
258 LLVMValueRef prim_count);
259 void (*launch_mesh_workgroups)(struct lp_build_nir_context *bld_base,
260 LLVMValueRef launch_grid);
261
262 void (*call)(struct lp_build_nir_context *bld_base,
263 struct lp_build_fn *fn,
264 int num_args,
265 LLVMValueRef *args);
266 // LLVMValueRef main_function
267 };
268
269 struct lp_build_nir_soa_context
270 {
271 struct lp_build_nir_context bld_base;
272
273 /* Builder for scalar elements of shader's data type (float) */
274 struct lp_build_context elem_bld;
275 struct lp_build_context uint_elem_bld;
276
277 LLVMValueRef consts_ptr;
278 const LLVMValueRef (*inputs)[TGSI_NUM_CHANNELS];
279 LLVMValueRef (*outputs)[TGSI_NUM_CHANNELS];
280 int num_inputs;
281 LLVMTypeRef context_type;
282 LLVMValueRef context_ptr;
283 LLVMTypeRef resources_type;
284 LLVMValueRef resources_ptr;
285 LLVMTypeRef thread_data_type;
286 LLVMValueRef thread_data_ptr;
287
288 LLVMValueRef ssbo_ptr;
289
290 LLVMValueRef shared_ptr;
291 LLVMValueRef payload_ptr;
292 LLVMValueRef scratch_ptr;
293 unsigned scratch_size;
294
295 const struct lp_build_coro_suspend_info *coro;
296
297 const struct lp_build_sampler_soa *sampler;
298 const struct lp_build_image_soa *image;
299
300 const struct lp_build_gs_iface *gs_iface;
301 const struct lp_build_tcs_iface *tcs_iface;
302 const struct lp_build_tes_iface *tes_iface;
303 const struct lp_build_fs_iface *fs_iface;
304 const struct lp_build_mesh_iface *mesh_iface;
305
306 LLVMValueRef emitted_prims_vec_ptr[PIPE_MAX_VERTEX_STREAMS];
307 LLVMValueRef total_emitted_vertices_vec_ptr[PIPE_MAX_VERTEX_STREAMS];
308 LLVMValueRef emitted_vertices_vec_ptr[PIPE_MAX_VERTEX_STREAMS];
309 LLVMValueRef max_output_vertices_vec;
310 struct lp_bld_tgsi_system_values system_values;
311
312 nir_variable_mode indirects;
313 struct lp_build_mask_context *mask;
314 struct lp_exec_mask exec_mask;
315
316 /* We allocate/use this array of inputs if (indirects & nir_var_shader_in) is
317 * set. The inputs[] array above is unused then.
318 */
319 LLVMValueRef inputs_array;
320
321 LLVMValueRef kernel_args_ptr;
322 unsigned gs_vertex_streams;
323
324 LLVMTypeRef call_context_type;
325 LLVMValueRef call_context_ptr;
326 };
327
328 void
329 lp_build_nir_prepasses(struct nir_shader *nir);
330
331 bool
332 lp_build_nir_llvm(struct lp_build_nir_context *bld_base,
333 struct nir_shader *nir,
334 nir_function_impl *impl);
335
336 void
337 lp_build_opt_nir(struct nir_shader *nir);
338
339
340 static inline LLVMValueRef
lp_nir_array_build_gather_values(LLVMBuilderRef builder,LLVMValueRef * values,unsigned value_count)341 lp_nir_array_build_gather_values(LLVMBuilderRef builder,
342 LLVMValueRef * values,
343 unsigned value_count)
344 {
345 LLVMTypeRef arr_type = LLVMArrayType(LLVMTypeOf(values[0]), value_count);
346 LLVMValueRef arr = LLVMGetUndef(arr_type);
347
348 for (unsigned i = 0; i < value_count; i++) {
349 arr = LLVMBuildInsertValue(builder, arr, values[i], i, "");
350 }
351 return arr;
352 }
353
354
355 static inline struct lp_build_context *
get_flt_bld(struct lp_build_nir_context * bld_base,unsigned op_bit_size)356 get_flt_bld(struct lp_build_nir_context *bld_base,
357 unsigned op_bit_size)
358 {
359 switch (op_bit_size) {
360 case 64:
361 return &bld_base->dbl_bld;
362 case 16:
363 return &bld_base->half_bld;
364 default:
365 case 32:
366 return &bld_base->base;
367 }
368 }
369
370
371 static inline struct lp_build_context *
get_int_bld(struct lp_build_nir_context * bld_base,bool is_unsigned,unsigned op_bit_size)372 get_int_bld(struct lp_build_nir_context *bld_base,
373 bool is_unsigned,
374 unsigned op_bit_size)
375 {
376 if (is_unsigned) {
377 switch (op_bit_size) {
378 case 64:
379 return &bld_base->uint64_bld;
380 case 32:
381 default:
382 return &bld_base->uint_bld;
383 case 16:
384 return &bld_base->uint16_bld;
385 case 8:
386 return &bld_base->uint8_bld;
387 }
388 } else {
389 switch (op_bit_size) {
390 case 64:
391 return &bld_base->int64_bld;
392 default:
393 case 32:
394 return &bld_base->int_bld;
395 case 16:
396 return &bld_base->int16_bld;
397 case 8:
398 return &bld_base->int8_bld;
399 }
400 }
401 }
402
403
404 unsigned
405 lp_nir_aos_swizzle(struct lp_build_nir_context *bld_base, unsigned chan);
406
407 LLVMAtomicRMWBinOp
408 lp_translate_atomic_op(nir_atomic_op op);
409
410 uint32_t
411 lp_build_nir_sample_key(gl_shader_stage stage, nir_tex_instr *instr);
412
413
414 void lp_img_op_from_intrinsic(struct lp_img_params *params, nir_intrinsic_instr *instr);
415
416 enum lp_nir_call_context_args {
417 LP_NIR_CALL_CONTEXT_CONTEXT,
418 LP_NIR_CALL_CONTEXT_RESOURCES,
419 LP_NIR_CALL_CONTEXT_SHARED,
420 LP_NIR_CALL_CONTEXT_SCRATCH,
421 LP_NIR_CALL_CONTEXT_WORK_DIM,
422 LP_NIR_CALL_CONTEXT_THREAD_ID_0,
423 LP_NIR_CALL_CONTEXT_THREAD_ID_1,
424 LP_NIR_CALL_CONTEXT_THREAD_ID_2,
425 LP_NIR_CALL_CONTEXT_BLOCK_ID_0,
426 LP_NIR_CALL_CONTEXT_BLOCK_ID_1,
427 LP_NIR_CALL_CONTEXT_BLOCK_ID_2,
428 LP_NIR_CALL_CONTEXT_GRID_SIZE_0,
429 LP_NIR_CALL_CONTEXT_GRID_SIZE_1,
430 LP_NIR_CALL_CONTEXT_GRID_SIZE_2,
431 LP_NIR_CALL_CONTEXT_BLOCK_SIZE_0,
432 LP_NIR_CALL_CONTEXT_BLOCK_SIZE_1,
433 LP_NIR_CALL_CONTEXT_BLOCK_SIZE_2,
434 LP_NIR_CALL_CONTEXT_MAX_ARGS,
435 };
436
437 LLVMTypeRef
438 lp_build_cs_func_call_context(struct gallivm_state *gallivm, int length,
439 LLVMTypeRef context_type, LLVMTypeRef resources_type);
440
441
442
443 #endif
444