1 /**************************************************************************
2 *
3 * Copyright 2019 Red Hat.
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included
14 * in all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
17 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 *
24 **************************************************************************/
25
26 #ifndef LP_BLD_NIR_H
27 #define LP_BLD_NIR_H
28
29 #include "gallivm/lp_bld.h"
30 #include "gallivm/lp_bld_limits.h"
31 #include "lp_bld_type.h"
32
33 #include "gallivm/lp_bld_tgsi.h"
34 #include "nir.h"
35
36 struct nir_shader;
37
38 void lp_build_nir_soa(struct gallivm_state *gallivm,
39 struct nir_shader *shader,
40 const struct lp_build_tgsi_params *params,
41 LLVMValueRef (*outputs)[4]);
42
43 void lp_build_nir_aos(struct gallivm_state *gallivm,
44 struct nir_shader *shader,
45 struct lp_type type,
46 const unsigned char swizzles[4],
47 LLVMValueRef consts_ptr,
48 const LLVMValueRef *inputs,
49 LLVMValueRef *outputs,
50 const struct lp_build_sampler_aos *sampler,
51 const struct tgsi_shader_info *info);
52
53 struct lp_build_nir_context
54 {
55 struct lp_build_context base;
56 struct lp_build_context uint_bld;
57 struct lp_build_context int_bld;
58 struct lp_build_context uint8_bld;
59 struct lp_build_context int8_bld;
60 struct lp_build_context uint16_bld;
61 struct lp_build_context int16_bld;
62 struct lp_build_context half_bld;
63 struct lp_build_context dbl_bld;
64 struct lp_build_context uint64_bld;
65 struct lp_build_context int64_bld;
66
67 LLVMValueRef *ssa_defs;
68 struct hash_table *regs;
69 struct hash_table *vars;
70
71 /** Value range analysis hash table used in code generation. */
72 struct hash_table *range_ht;
73
74 LLVMValueRef aniso_filter_table;
75
76 nir_shader *shader;
77
78 void (*load_ubo)(struct lp_build_nir_context *bld_base,
79 unsigned nc,
80 unsigned bit_size,
81 bool offset_is_uniform,
82 LLVMValueRef index, LLVMValueRef offset, LLVMValueRef result[NIR_MAX_VEC_COMPONENTS]);
83
84 void (*load_kernel_arg)(struct lp_build_nir_context *bld_base,
85 unsigned nc,
86 unsigned bit_size,
87 unsigned offset_bit_size,
88 bool offset_is_uniform,
89 LLVMValueRef offset, LLVMValueRef result[NIR_MAX_VEC_COMPONENTS]);
90
91 void (*load_global)(struct lp_build_nir_context *bld_base,
92 unsigned nc, unsigned bit_size,
93 unsigned offset_bit_size,
94 bool offset_is_global,
95 LLVMValueRef offset, LLVMValueRef result[NIR_MAX_VEC_COMPONENTS]);
96
97 void (*store_global)(struct lp_build_nir_context *bld_base,
98 unsigned writemask,
99 unsigned nc, unsigned bit_size,
100 unsigned addr_bit_size,
101 LLVMValueRef addr, LLVMValueRef dst);
102
103 void (*atomic_global)(struct lp_build_nir_context *bld_base,
104 nir_intrinsic_op op,
105 unsigned addr_bit_size,
106 unsigned val_bit_size,
107 LLVMValueRef addr,
108 LLVMValueRef val, LLVMValueRef val2,
109 LLVMValueRef *result);
110
111 /* for SSBO and shared memory */
112 void (*load_mem)(struct lp_build_nir_context *bld_base,
113 unsigned nc, unsigned bit_size,
114 bool index_and_offset_are_uniform,
115 LLVMValueRef index, LLVMValueRef offset, LLVMValueRef result[NIR_MAX_VEC_COMPONENTS]);
116 void (*store_mem)(struct lp_build_nir_context *bld_base,
117 unsigned writemask, unsigned nc, unsigned bit_size,
118 bool index_and_offset_are_uniform,
119 LLVMValueRef index, LLVMValueRef offset, LLVMValueRef dst);
120
121 void (*atomic_mem)(struct lp_build_nir_context *bld_base,
122 nir_intrinsic_op op,
123 unsigned bit_size,
124 LLVMValueRef index, LLVMValueRef offset,
125 LLVMValueRef val, LLVMValueRef val2,
126 LLVMValueRef *result);
127
128 void (*barrier)(struct lp_build_nir_context *bld_base);
129
130 void (*image_op)(struct lp_build_nir_context *bld_base,
131 struct lp_img_params *params);
132 void (*image_size)(struct lp_build_nir_context *bld_base,
133 struct lp_sampler_size_query_params *params);
134 LLVMValueRef (*get_ssbo_size)(struct lp_build_nir_context *bld_base,
135 LLVMValueRef index);
136
137 void (*load_const)(struct lp_build_nir_context *bld_base,
138 const nir_load_const_instr *instr,
139 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS]);
140 void (*load_var)(struct lp_build_nir_context *bld_base,
141 nir_variable_mode deref_mode,
142 unsigned num_components,
143 unsigned bit_size,
144 nir_variable *var,
145 unsigned vertex_index,
146 LLVMValueRef indir_vertex_index,
147 unsigned const_index,
148 LLVMValueRef indir_index,
149 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS]);
150 void (*store_var)(struct lp_build_nir_context *bld_base,
151 nir_variable_mode deref_mode,
152 unsigned num_components,
153 unsigned bit_size,
154 nir_variable *var,
155 unsigned writemask,
156 LLVMValueRef indir_vertex_index,
157 unsigned const_index,
158 LLVMValueRef indir_index,
159 LLVMValueRef dst);
160
161 LLVMValueRef (*load_reg)(struct lp_build_nir_context *bld_base,
162 struct lp_build_context *reg_bld,
163 const nir_reg_src *reg,
164 LLVMValueRef indir_src,
165 LLVMValueRef reg_storage);
166 void (*store_reg)(struct lp_build_nir_context *bld_base,
167 struct lp_build_context *reg_bld,
168 const nir_reg_dest *reg,
169 unsigned writemask,
170 LLVMValueRef indir_src,
171 LLVMValueRef reg_storage,
172 LLVMValueRef dst[NIR_MAX_VEC_COMPONENTS]);
173
174 void (*load_scratch)(struct lp_build_nir_context *bld_base,
175 unsigned nc, unsigned bit_size,
176 LLVMValueRef offset,
177 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS]);
178 void (*store_scratch)(struct lp_build_nir_context *bld_base,
179 unsigned writemask, unsigned nc,
180 unsigned bit_size, LLVMValueRef offset,
181 LLVMValueRef val);
182
183 void (*emit_var_decl)(struct lp_build_nir_context *bld_base,
184 nir_variable *var);
185
186 void (*tex)(struct lp_build_nir_context *bld_base,
187 struct lp_sampler_params *params);
188
189 void (*tex_size)(struct lp_build_nir_context *bld_base,
190 struct lp_sampler_size_query_params *params);
191
192 void (*sysval_intrin)(struct lp_build_nir_context *bld_base,
193 nir_intrinsic_instr *instr,
194 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS]);
195 void (*discard)(struct lp_build_nir_context *bld_base,
196 LLVMValueRef cond);
197
198 void (*bgnloop)(struct lp_build_nir_context *bld_base);
199 void (*endloop)(struct lp_build_nir_context *bld_base);
200 void (*if_cond)(struct lp_build_nir_context *bld_base, LLVMValueRef cond);
201 void (*else_stmt)(struct lp_build_nir_context *bld_base);
202 void (*endif_stmt)(struct lp_build_nir_context *bld_base);
203 void (*break_stmt)(struct lp_build_nir_context *bld_base);
204 void (*continue_stmt)(struct lp_build_nir_context *bld_base);
205
206 void (*emit_vertex)(struct lp_build_nir_context *bld_base, uint32_t stream_id);
207 void (*end_primitive)(struct lp_build_nir_context *bld_base, uint32_t stream_id);
208
209 void (*vote)(struct lp_build_nir_context *bld_base, LLVMValueRef src, nir_intrinsic_instr *instr, LLVMValueRef dst[4]);
210 void (*elect)(struct lp_build_nir_context *bld_base, LLVMValueRef dst[4]);
211 void (*reduce)(struct lp_build_nir_context *bld_base, LLVMValueRef src, nir_intrinsic_instr *instr, LLVMValueRef dst[4]);
212 void (*ballot)(struct lp_build_nir_context *bld_base, LLVMValueRef src, nir_intrinsic_instr *instr, LLVMValueRef dst[4]);
213 #if LLVM_VERSION_MAJOR >= 10
214 void (*shuffle)(struct lp_build_nir_context *bld_base,
215 LLVMValueRef src,
216 LLVMValueRef index,
217 nir_intrinsic_instr *instr,
218 LLVMValueRef dst[4]);
219 #endif
220 void (*read_invocation)(struct lp_build_nir_context *bld_base,
221 LLVMValueRef src, unsigned bit_size, LLVMValueRef invoc,
222 LLVMValueRef dst[4]);
223 void (*helper_invocation)(struct lp_build_nir_context *bld_base, LLVMValueRef *dst);
224
225 void (*interp_at)(struct lp_build_nir_context *bld_base,
226 unsigned num_components,
227 nir_variable *var,
228 bool centroid, bool sample,
229 unsigned const_index,
230 LLVMValueRef indir_index,
231 LLVMValueRef offsets[2], LLVMValueRef dst[4]);
232 // LLVMValueRef main_function
233 };
234
235 struct lp_build_nir_soa_context
236 {
237 struct lp_build_nir_context bld_base;
238
239 /* Builder for scalar elements of shader's data type (float) */
240 struct lp_build_context elem_bld;
241 struct lp_build_context uint_elem_bld;
242
243 LLVMValueRef consts_ptr;
244 LLVMValueRef const_sizes_ptr;
245 LLVMValueRef consts[LP_MAX_TGSI_CONST_BUFFERS];
246 LLVMValueRef consts_sizes[LP_MAX_TGSI_CONST_BUFFERS];
247 const LLVMValueRef (*inputs)[TGSI_NUM_CHANNELS];
248 LLVMValueRef (*outputs)[TGSI_NUM_CHANNELS];
249 LLVMValueRef context_ptr;
250 LLVMValueRef thread_data_ptr;
251
252 LLVMValueRef ssbo_ptr;
253 LLVMValueRef ssbo_sizes_ptr;
254 LLVMValueRef ssbos[LP_MAX_TGSI_SHADER_BUFFERS];
255 LLVMValueRef ssbo_sizes[LP_MAX_TGSI_SHADER_BUFFERS];
256
257 LLVMValueRef shared_ptr;
258 LLVMValueRef scratch_ptr;
259 unsigned scratch_size;
260
261 const struct lp_build_coro_suspend_info *coro;
262
263 const struct lp_build_sampler_soa *sampler;
264 const struct lp_build_image_soa *image;
265
266 const struct lp_build_gs_iface *gs_iface;
267 const struct lp_build_tcs_iface *tcs_iface;
268 const struct lp_build_tes_iface *tes_iface;
269 const struct lp_build_fs_iface *fs_iface;
270 LLVMValueRef emitted_prims_vec_ptr[PIPE_MAX_VERTEX_STREAMS];
271 LLVMValueRef total_emitted_vertices_vec_ptr[PIPE_MAX_VERTEX_STREAMS];
272 LLVMValueRef emitted_vertices_vec_ptr[PIPE_MAX_VERTEX_STREAMS];
273 LLVMValueRef max_output_vertices_vec;
274 struct lp_bld_tgsi_system_values system_values;
275
276 nir_variable_mode indirects;
277 struct lp_build_mask_context *mask;
278 struct lp_exec_mask exec_mask;
279
280 /* We allocate/use this array of inputs if (indirects & nir_var_shader_in) is
281 * set. The inputs[] array above is unused then.
282 */
283 LLVMValueRef inputs_array;
284
285 LLVMValueRef kernel_args_ptr;
286 unsigned gs_vertex_streams;
287 };
288
289 struct lp_build_nir_aos_context
290 {
291 struct lp_build_nir_context bld_base;
292
293 /* Builder for integer masks and indices */
294 struct lp_build_context int_bld;
295
296 /*
297 * AoS swizzle used:
298 * - swizzles[0] = red index
299 * - swizzles[1] = green index
300 * - swizzles[2] = blue index
301 * - swizzles[3] = alpha index
302 */
303 unsigned char swizzles[4];
304 unsigned char inv_swizzles[4];
305
306 LLVMValueRef consts_ptr;
307 const LLVMValueRef *inputs;
308 LLVMValueRef *outputs;
309
310 const struct lp_build_sampler_aos *sampler;
311 };
312
313
314 bool
315 lp_build_nir_llvm(struct lp_build_nir_context *bld_base,
316 struct nir_shader *nir);
317
318 void
319 lp_build_opt_nir(struct nir_shader *nir);
320
321
322 static inline LLVMValueRef
lp_nir_array_build_gather_values(LLVMBuilderRef builder,LLVMValueRef * values,unsigned value_count)323 lp_nir_array_build_gather_values(LLVMBuilderRef builder,
324 LLVMValueRef * values,
325 unsigned value_count)
326 {
327 LLVMTypeRef arr_type = LLVMArrayType(LLVMTypeOf(values[0]), value_count);
328 LLVMValueRef arr = LLVMGetUndef(arr_type);
329
330 for (unsigned i = 0; i < value_count; i++) {
331 arr = LLVMBuildInsertValue(builder, arr, values[i], i, "");
332 }
333 return arr;
334 }
335
336
337 static inline struct lp_build_context *
get_flt_bld(struct lp_build_nir_context * bld_base,unsigned op_bit_size)338 get_flt_bld(struct lp_build_nir_context *bld_base,
339 unsigned op_bit_size)
340 {
341 switch (op_bit_size) {
342 case 64:
343 return &bld_base->dbl_bld;
344 case 16:
345 return &bld_base->half_bld;
346 default:
347 case 32:
348 return &bld_base->base;
349 }
350 }
351
352
353 static inline struct lp_build_context *
get_int_bld(struct lp_build_nir_context * bld_base,bool is_unsigned,unsigned op_bit_size)354 get_int_bld(struct lp_build_nir_context *bld_base,
355 bool is_unsigned,
356 unsigned op_bit_size)
357 {
358 if (is_unsigned) {
359 switch (op_bit_size) {
360 case 64:
361 return &bld_base->uint64_bld;
362 case 32:
363 default:
364 return &bld_base->uint_bld;
365 case 16:
366 return &bld_base->uint16_bld;
367 case 8:
368 return &bld_base->uint8_bld;
369 }
370 } else {
371 switch (op_bit_size) {
372 case 64:
373 return &bld_base->int64_bld;
374 default:
375 case 32:
376 return &bld_base->int_bld;
377 case 16:
378 return &bld_base->int16_bld;
379 case 8:
380 return &bld_base->int8_bld;
381 }
382 }
383 }
384
385
386 static inline struct lp_build_nir_aos_context *
lp_nir_aos_context(struct lp_build_nir_context * bld_base)387 lp_nir_aos_context(struct lp_build_nir_context *bld_base)
388 {
389 return (struct lp_build_nir_aos_context *) bld_base;
390 }
391
392
393 LLVMValueRef
394 lp_nir_aos_conv_const(struct gallivm_state *gallivm,
395 LLVMValueRef constval, int nc);
396
397 #endif
398