1 /**************************************************************************
2 *
3 * Copyright 2019 Red Hat.
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included
14 * in all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
17 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 *
24 **************************************************************************/
25
26 #ifndef LP_BLD_NIR_H
27 #define LP_BLD_NIR_H
28
29 #include "gallivm/lp_bld.h"
30 #include "gallivm/lp_bld_limits.h"
31 #include "lp_bld_type.h"
32
33 #include "gallivm/lp_bld_tgsi.h"
34 #include "nir.h"
35
36 struct nir_shader;
37
38 void lp_build_nir_soa(struct gallivm_state *gallivm,
39 struct nir_shader *shader,
40 const struct lp_build_tgsi_params *params,
41 LLVMValueRef (*outputs)[4]);
42
43 struct lp_build_nir_context
44 {
45 struct lp_build_context base;
46 struct lp_build_context uint_bld;
47 struct lp_build_context int_bld;
48 struct lp_build_context uint8_bld;
49 struct lp_build_context int8_bld;
50 struct lp_build_context uint16_bld;
51 struct lp_build_context int16_bld;
52 struct lp_build_context half_bld;
53 struct lp_build_context dbl_bld;
54 struct lp_build_context uint64_bld;
55 struct lp_build_context int64_bld;
56
57 LLVMValueRef *ssa_defs;
58 struct hash_table *regs;
59 struct hash_table *vars;
60
61 /** Value range analysis hash table used in code generation. */
62 struct hash_table *range_ht;
63
64 LLVMValueRef aniso_filter_table;
65
66 nir_shader *shader;
67
68 void (*load_ubo)(struct lp_build_nir_context *bld_base,
69 unsigned nc,
70 unsigned bit_size,
71 bool offset_is_uniform,
72 LLVMValueRef index, LLVMValueRef offset, LLVMValueRef result[NIR_MAX_VEC_COMPONENTS]);
73
74 void (*load_kernel_arg)(struct lp_build_nir_context *bld_base,
75 unsigned nc,
76 unsigned bit_size,
77 unsigned offset_bit_size,
78 bool offset_is_uniform,
79 LLVMValueRef offset, LLVMValueRef result[NIR_MAX_VEC_COMPONENTS]);
80
81 void (*load_global)(struct lp_build_nir_context *bld_base,
82 unsigned nc, unsigned bit_size,
83 unsigned offset_bit_size,
84 LLVMValueRef offset, LLVMValueRef result[NIR_MAX_VEC_COMPONENTS]);
85
86 void (*store_global)(struct lp_build_nir_context *bld_base,
87 unsigned writemask,
88 unsigned nc, unsigned bit_size,
89 unsigned addr_bit_size,
90 LLVMValueRef addr, LLVMValueRef dst);
91
92 void (*atomic_global)(struct lp_build_nir_context *bld_base,
93 nir_intrinsic_op op,
94 unsigned addr_bit_size,
95 unsigned val_bit_size,
96 LLVMValueRef addr,
97 LLVMValueRef val, LLVMValueRef val2,
98 LLVMValueRef *result);
99
100 /* for SSBO and shared memory */
101 void (*load_mem)(struct lp_build_nir_context *bld_base,
102 unsigned nc, unsigned bit_size,
103 LLVMValueRef index, LLVMValueRef offset, LLVMValueRef result[NIR_MAX_VEC_COMPONENTS]);
104 void (*store_mem)(struct lp_build_nir_context *bld_base,
105 unsigned writemask, unsigned nc, unsigned bit_size,
106 LLVMValueRef index, LLVMValueRef offset, LLVMValueRef dst);
107
108 void (*atomic_mem)(struct lp_build_nir_context *bld_base,
109 nir_intrinsic_op op,
110 unsigned bit_size,
111 LLVMValueRef index, LLVMValueRef offset,
112 LLVMValueRef val, LLVMValueRef val2,
113 LLVMValueRef *result);
114
115 void (*barrier)(struct lp_build_nir_context *bld_base);
116
117 void (*image_op)(struct lp_build_nir_context *bld_base,
118 struct lp_img_params *params);
119 void (*image_size)(struct lp_build_nir_context *bld_base,
120 struct lp_sampler_size_query_params *params);
121 LLVMValueRef (*get_ssbo_size)(struct lp_build_nir_context *bld_base,
122 LLVMValueRef index);
123
124 void (*load_var)(struct lp_build_nir_context *bld_base,
125 nir_variable_mode deref_mode,
126 unsigned num_components,
127 unsigned bit_size,
128 nir_variable *var,
129 unsigned vertex_index,
130 LLVMValueRef indir_vertex_index,
131 unsigned const_index,
132 LLVMValueRef indir_index,
133 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS]);
134 void (*store_var)(struct lp_build_nir_context *bld_base,
135 nir_variable_mode deref_mode,
136 unsigned num_components,
137 unsigned bit_size,
138 nir_variable *var,
139 unsigned writemask,
140 LLVMValueRef indir_vertex_index,
141 unsigned const_index,
142 LLVMValueRef indir_index,
143 LLVMValueRef dst);
144
145 LLVMValueRef (*load_reg)(struct lp_build_nir_context *bld_base,
146 struct lp_build_context *reg_bld,
147 const nir_reg_src *reg,
148 LLVMValueRef indir_src,
149 LLVMValueRef reg_storage);
150 void (*store_reg)(struct lp_build_nir_context *bld_base,
151 struct lp_build_context *reg_bld,
152 const nir_reg_dest *reg,
153 unsigned writemask,
154 LLVMValueRef indir_src,
155 LLVMValueRef reg_storage,
156 LLVMValueRef dst[NIR_MAX_VEC_COMPONENTS]);
157
158 void (*load_scratch)(struct lp_build_nir_context *bld_base,
159 unsigned nc, unsigned bit_size,
160 LLVMValueRef offset,
161 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS]);
162 void (*store_scratch)(struct lp_build_nir_context *bld_base,
163 unsigned writemask, unsigned nc,
164 unsigned bit_size, LLVMValueRef offset,
165 LLVMValueRef val);
166
167 void (*emit_var_decl)(struct lp_build_nir_context *bld_base,
168 nir_variable *var);
169
170 void (*tex)(struct lp_build_nir_context *bld_base,
171 struct lp_sampler_params *params);
172
173 void (*tex_size)(struct lp_build_nir_context *bld_base,
174 struct lp_sampler_size_query_params *params);
175
176 void (*sysval_intrin)(struct lp_build_nir_context *bld_base,
177 nir_intrinsic_instr *instr,
178 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS]);
179 void (*discard)(struct lp_build_nir_context *bld_base,
180 LLVMValueRef cond);
181
182 void (*bgnloop)(struct lp_build_nir_context *bld_base);
183 void (*endloop)(struct lp_build_nir_context *bld_base);
184 void (*if_cond)(struct lp_build_nir_context *bld_base, LLVMValueRef cond);
185 void (*else_stmt)(struct lp_build_nir_context *bld_base);
186 void (*endif_stmt)(struct lp_build_nir_context *bld_base);
187 void (*break_stmt)(struct lp_build_nir_context *bld_base);
188 void (*continue_stmt)(struct lp_build_nir_context *bld_base);
189
190 void (*emit_vertex)(struct lp_build_nir_context *bld_base, uint32_t stream_id);
191 void (*end_primitive)(struct lp_build_nir_context *bld_base, uint32_t stream_id);
192
193 void (*vote)(struct lp_build_nir_context *bld_base, LLVMValueRef src, nir_intrinsic_instr *instr, LLVMValueRef dst[4]);
194 void (*elect)(struct lp_build_nir_context *bld_base, LLVMValueRef dst[4]);
195 void (*reduce)(struct lp_build_nir_context *bld_base, LLVMValueRef src, nir_intrinsic_instr *instr, LLVMValueRef dst[4]);
196 void (*ballot)(struct lp_build_nir_context *bld_base, LLVMValueRef src, nir_intrinsic_instr *instr, LLVMValueRef dst[4]);
197 void (*read_invocation)(struct lp_build_nir_context *bld_base,
198 LLVMValueRef src, unsigned bit_size, LLVMValueRef invoc,
199 LLVMValueRef dst[4]);
200 void (*helper_invocation)(struct lp_build_nir_context *bld_base, LLVMValueRef *dst);
201
202 void (*interp_at)(struct lp_build_nir_context *bld_base,
203 unsigned num_components,
204 nir_variable *var,
205 bool centroid, bool sample,
206 unsigned const_index,
207 LLVMValueRef indir_index,
208 LLVMValueRef offsets[2], LLVMValueRef dst[4]);
209 // LLVMValueRef main_function
210 };
211
212 struct lp_build_nir_soa_context
213 {
214 struct lp_build_nir_context bld_base;
215
216 /* Builder for scalar elements of shader's data type (float) */
217 struct lp_build_context elem_bld;
218 struct lp_build_context uint_elem_bld;
219
220 LLVMValueRef consts_ptr;
221 LLVMValueRef const_sizes_ptr;
222 LLVMValueRef consts[LP_MAX_TGSI_CONST_BUFFERS];
223 LLVMValueRef consts_sizes[LP_MAX_TGSI_CONST_BUFFERS];
224 const LLVMValueRef (*inputs)[TGSI_NUM_CHANNELS];
225 LLVMValueRef (*outputs)[TGSI_NUM_CHANNELS];
226 LLVMValueRef context_ptr;
227 LLVMValueRef thread_data_ptr;
228
229 LLVMValueRef ssbo_ptr;
230 LLVMValueRef ssbo_sizes_ptr;
231 LLVMValueRef ssbos[LP_MAX_TGSI_SHADER_BUFFERS];
232 LLVMValueRef ssbo_sizes[LP_MAX_TGSI_SHADER_BUFFERS];
233
234 LLVMValueRef shared_ptr;
235 LLVMValueRef scratch_ptr;
236 unsigned scratch_size;
237
238 const struct lp_build_coro_suspend_info *coro;
239
240 const struct lp_build_sampler_soa *sampler;
241 const struct lp_build_image_soa *image;
242
243 const struct lp_build_gs_iface *gs_iface;
244 const struct lp_build_tcs_iface *tcs_iface;
245 const struct lp_build_tes_iface *tes_iface;
246 const struct lp_build_fs_iface *fs_iface;
247 LLVMValueRef emitted_prims_vec_ptr[PIPE_MAX_VERTEX_STREAMS];
248 LLVMValueRef total_emitted_vertices_vec_ptr[PIPE_MAX_VERTEX_STREAMS];
249 LLVMValueRef emitted_vertices_vec_ptr[PIPE_MAX_VERTEX_STREAMS];
250 LLVMValueRef max_output_vertices_vec;
251 struct lp_bld_tgsi_system_values system_values;
252
253 nir_variable_mode indirects;
254 struct lp_build_mask_context *mask;
255 struct lp_exec_mask exec_mask;
256
257 /* We allocate/use this array of inputs if (indirects & nir_var_shader_in) is
258 * set. The inputs[] array above is unused then.
259 */
260 LLVMValueRef inputs_array;
261
262 LLVMValueRef kernel_args_ptr;
263 unsigned gs_vertex_streams;
264 };
265
266 bool
267 lp_build_nir_llvm(struct lp_build_nir_context *bld_base,
268 struct nir_shader *nir);
269
270 void lp_build_opt_nir(struct nir_shader *nir);
271
272 static inline LLVMValueRef
lp_nir_array_build_gather_values(LLVMBuilderRef builder,LLVMValueRef * values,unsigned value_count)273 lp_nir_array_build_gather_values(LLVMBuilderRef builder,
274 LLVMValueRef * values,
275 unsigned value_count)
276 {
277 LLVMTypeRef arr_type = LLVMArrayType(LLVMTypeOf(values[0]), value_count);
278 LLVMValueRef arr = LLVMGetUndef(arr_type);
279 unsigned i;
280
281 for (i = 0; i < value_count; i++) {
282 arr = LLVMBuildInsertValue(builder, arr, values[i], i, "");
283 }
284 return arr;
285 }
286
get_flt_bld(struct lp_build_nir_context * bld_base,unsigned op_bit_size)287 static inline struct lp_build_context *get_flt_bld(struct lp_build_nir_context *bld_base,
288 unsigned op_bit_size)
289 {
290 switch (op_bit_size) {
291 case 64:
292 return &bld_base->dbl_bld;
293 case 16:
294 return &bld_base->half_bld;
295 default:
296 case 32:
297 return &bld_base->base;
298 }
299 }
300
get_int_bld(struct lp_build_nir_context * bld_base,bool is_unsigned,unsigned op_bit_size)301 static inline struct lp_build_context *get_int_bld(struct lp_build_nir_context *bld_base,
302 bool is_unsigned,
303 unsigned op_bit_size)
304 {
305 if (is_unsigned) {
306 switch (op_bit_size) {
307 case 64:
308 return &bld_base->uint64_bld;
309 case 32:
310 default:
311 return &bld_base->uint_bld;
312 case 16:
313 return &bld_base->uint16_bld;
314 case 8:
315 return &bld_base->uint8_bld;
316 }
317 } else {
318 switch (op_bit_size) {
319 case 64:
320 return &bld_base->int64_bld;
321 default:
322 case 32:
323 return &bld_base->int_bld;
324 case 16:
325 return &bld_base->int16_bld;
326 case 8:
327 return &bld_base->int8_bld;
328 }
329 }
330 }
331
332 #endif
333