1 /*
2 * Copyright © 2022 Konstantin Seurer
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "nir/nir.h"
25 #include "nir/nir_builder.h"
26
27 #include "util/hash_table.h"
28
29 #include "radv_acceleration_structure.h"
30 #include "radv_private.h"
31 #include "radv_rt_common.h"
32 #include "radv_shader.h"
33
34 typedef struct {
35 nir_variable *variable;
36 unsigned array_length;
37 } rq_variable;
38
39 static rq_variable *
rq_variable_create(nir_shader * shader,nir_function_impl * impl,unsigned array_length,const struct glsl_type * type,const char * name)40 rq_variable_create(nir_shader *shader, nir_function_impl *impl, unsigned array_length,
41 const struct glsl_type *type, const char *name)
42 {
43 rq_variable *result = ralloc(shader ? (void *)shader : (void *)impl, rq_variable);
44 result->array_length = array_length;
45
46 const struct glsl_type *variable_type = type;
47 if (array_length != 1)
48 variable_type = glsl_array_type(type, array_length, glsl_get_explicit_stride(type));
49
50 if (shader) {
51 result->variable = nir_variable_create(shader, nir_var_shader_temp, variable_type, name);
52 } else {
53 result->variable = nir_local_variable_create(impl, variable_type, name);
54 }
55
56 return result;
57 }
58
59 static nir_ssa_def *
nir_load_array(nir_builder * b,nir_variable * array,nir_ssa_def * index)60 nir_load_array(nir_builder *b, nir_variable *array, nir_ssa_def *index)
61 {
62 return nir_load_deref(b, nir_build_deref_array(b, nir_build_deref_var(b, array), index));
63 }
64
65 static void
nir_store_array(nir_builder * b,nir_variable * array,nir_ssa_def * index,nir_ssa_def * value,unsigned writemask)66 nir_store_array(nir_builder *b, nir_variable *array, nir_ssa_def *index, nir_ssa_def *value,
67 unsigned writemask)
68 {
69 nir_store_deref(b, nir_build_deref_array(b, nir_build_deref_var(b, array), index), value,
70 writemask);
71 }
72
73 static nir_ssa_def *
rq_load_var(nir_builder * b,nir_ssa_def * index,rq_variable * var)74 rq_load_var(nir_builder *b, nir_ssa_def *index, rq_variable *var)
75 {
76 if (var->array_length == 1)
77 return nir_load_var(b, var->variable);
78
79 return nir_load_array(b, var->variable, index);
80 }
81
82 static void
rq_store_var(nir_builder * b,nir_ssa_def * index,rq_variable * var,nir_ssa_def * value,unsigned writemask)83 rq_store_var(nir_builder *b, nir_ssa_def *index, rq_variable *var, nir_ssa_def *value,
84 unsigned writemask)
85 {
86 if (var->array_length == 1) {
87 nir_store_var(b, var->variable, value, writemask);
88 } else {
89 nir_store_array(b, var->variable, index, value, writemask);
90 }
91 }
92
93 static void
rq_copy_var(nir_builder * b,nir_ssa_def * index,rq_variable * dst,rq_variable * src,unsigned mask)94 rq_copy_var(nir_builder *b, nir_ssa_def *index, rq_variable *dst, rq_variable *src, unsigned mask)
95 {
96 rq_store_var(b, index, dst, rq_load_var(b, index, src), mask);
97 }
98
99 static nir_ssa_def *
rq_load_array(nir_builder * b,nir_ssa_def * index,rq_variable * var,nir_ssa_def * array_index)100 rq_load_array(nir_builder *b, nir_ssa_def *index, rq_variable *var, nir_ssa_def *array_index)
101 {
102 if (var->array_length == 1)
103 return nir_load_array(b, var->variable, array_index);
104
105 return nir_load_deref(
106 b,
107 nir_build_deref_array(
108 b, nir_build_deref_array(b, nir_build_deref_var(b, var->variable), index), array_index));
109 }
110
111 static void
rq_store_array(nir_builder * b,nir_ssa_def * index,rq_variable * var,nir_ssa_def * array_index,nir_ssa_def * value,unsigned writemask)112 rq_store_array(nir_builder *b, nir_ssa_def *index, rq_variable *var, nir_ssa_def *array_index,
113 nir_ssa_def *value, unsigned writemask)
114 {
115 if (var->array_length == 1) {
116 nir_store_array(b, var->variable, array_index, value, writemask);
117 } else {
118 nir_store_deref(
119 b,
120 nir_build_deref_array(
121 b, nir_build_deref_array(b, nir_build_deref_var(b, var->variable), index), array_index),
122 value, writemask);
123 }
124 }
125
126 struct ray_query_traversal_vars {
127 rq_variable *origin;
128 rq_variable *direction;
129
130 rq_variable *inv_dir;
131 rq_variable *bvh_base;
132 rq_variable *stack;
133 rq_variable *top_stack;
134 };
135
136 struct ray_query_intersection_vars {
137 rq_variable *primitive_id;
138 rq_variable *geometry_id_and_flags;
139 rq_variable *instance_id;
140 rq_variable *instance_addr;
141 rq_variable *intersection_type;
142 rq_variable *opaque;
143 rq_variable *frontface;
144 rq_variable *custom_instance_and_mask;
145 rq_variable *sbt_offset_and_flags;
146 rq_variable *barycentrics;
147 rq_variable *t;
148 };
149
150 struct ray_query_vars {
151 rq_variable *accel_struct;
152 rq_variable *flags;
153 rq_variable *cull_mask;
154 rq_variable *origin;
155 rq_variable *tmin;
156 rq_variable *direction;
157
158 rq_variable *incomplete;
159
160 struct ray_query_intersection_vars closest;
161 struct ray_query_intersection_vars candidate;
162
163 struct ray_query_traversal_vars trav;
164
165 rq_variable *stack;
166 };
167
168 #define VAR_NAME(name) \
169 strcat(strcpy(ralloc_size(impl, strlen(base_name) + strlen(name) + 1), base_name), name)
170
171 static struct ray_query_traversal_vars
init_ray_query_traversal_vars(nir_shader * shader,nir_function_impl * impl,unsigned array_length,const char * base_name)172 init_ray_query_traversal_vars(nir_shader *shader, nir_function_impl *impl, unsigned array_length,
173 const char *base_name)
174 {
175 struct ray_query_traversal_vars result;
176
177 const struct glsl_type *vec3_type = glsl_vector_type(GLSL_TYPE_FLOAT, 3);
178
179 result.origin = rq_variable_create(shader, impl, array_length, vec3_type, VAR_NAME("_origin"));
180 result.direction =
181 rq_variable_create(shader, impl, array_length, vec3_type, VAR_NAME("_direction"));
182
183 result.inv_dir = rq_variable_create(shader, impl, array_length, vec3_type, VAR_NAME("_inv_dir"));
184 result.bvh_base =
185 rq_variable_create(shader, impl, array_length, glsl_uint64_t_type(), VAR_NAME("_bvh_base"));
186 result.stack =
187 rq_variable_create(shader, impl, array_length, glsl_uint_type(), VAR_NAME("_stack"));
188 result.top_stack =
189 rq_variable_create(shader, impl, array_length, glsl_uint_type(), VAR_NAME("_top_stack"));
190
191 return result;
192 }
193
194 static struct ray_query_intersection_vars
init_ray_query_intersection_vars(nir_shader * shader,nir_function_impl * impl,unsigned array_length,const char * base_name)195 init_ray_query_intersection_vars(nir_shader *shader, nir_function_impl *impl, unsigned array_length,
196 const char *base_name)
197 {
198 struct ray_query_intersection_vars result;
199
200 const struct glsl_type *vec2_type = glsl_vector_type(GLSL_TYPE_FLOAT, 2);
201
202 result.primitive_id =
203 rq_variable_create(shader, impl, array_length, glsl_uint_type(), VAR_NAME("_primitive_id"));
204 result.geometry_id_and_flags = rq_variable_create(shader, impl, array_length, glsl_uint_type(),
205 VAR_NAME("_geometry_id_and_flags"));
206 result.instance_id =
207 rq_variable_create(shader, impl, array_length, glsl_uint_type(), VAR_NAME("_instance_id"));
208 result.instance_addr = rq_variable_create(shader, impl, array_length, glsl_uint64_t_type(),
209 VAR_NAME("_instance_addr"));
210 result.intersection_type = rq_variable_create(shader, impl, array_length, glsl_uint_type(),
211 VAR_NAME("_intersection_type"));
212 result.opaque =
213 rq_variable_create(shader, impl, array_length, glsl_bool_type(), VAR_NAME("_opaque"));
214 result.frontface =
215 rq_variable_create(shader, impl, array_length, glsl_bool_type(), VAR_NAME("_frontface"));
216 result.custom_instance_and_mask = rq_variable_create(
217 shader, impl, array_length, glsl_uint_type(), VAR_NAME("_custom_instance_and_mask"));
218 result.sbt_offset_and_flags = rq_variable_create(shader, impl, array_length, glsl_uint_type(),
219 VAR_NAME("_sbt_offset_and_flags"));
220 result.barycentrics =
221 rq_variable_create(shader, impl, array_length, vec2_type, VAR_NAME("_barycentrics"));
222 result.t = rq_variable_create(shader, impl, array_length, glsl_float_type(), VAR_NAME("_t"));
223
224 return result;
225 }
226
227 static void
init_ray_query_vars(nir_shader * shader,nir_function_impl * impl,unsigned array_length,struct ray_query_vars * dst,const char * base_name)228 init_ray_query_vars(nir_shader *shader, nir_function_impl *impl, unsigned array_length,
229 struct ray_query_vars *dst, const char *base_name)
230 {
231 const struct glsl_type *vec3_type = glsl_vector_type(GLSL_TYPE_FLOAT, 3);
232
233 dst->accel_struct = rq_variable_create(shader, impl, array_length, glsl_uint64_t_type(),
234 VAR_NAME("_accel_struct"));
235 dst->flags =
236 rq_variable_create(shader, impl, array_length, glsl_uint_type(), VAR_NAME("_flags"));
237 dst->cull_mask =
238 rq_variable_create(shader, impl, array_length, glsl_uint_type(), VAR_NAME("_cull_mask"));
239 dst->origin = rq_variable_create(shader, impl, array_length, vec3_type, VAR_NAME("_origin"));
240 dst->tmin = rq_variable_create(shader, impl, array_length, glsl_float_type(), VAR_NAME("_tmin"));
241 dst->direction =
242 rq_variable_create(shader, impl, array_length, vec3_type, VAR_NAME("_direction"));
243
244 dst->incomplete =
245 rq_variable_create(shader, impl, array_length, glsl_bool_type(), VAR_NAME("_incomplete"));
246
247 dst->closest =
248 init_ray_query_intersection_vars(shader, impl, array_length, VAR_NAME("_closest"));
249 dst->candidate =
250 init_ray_query_intersection_vars(shader, impl, array_length, VAR_NAME("_candidate"));
251
252 dst->trav = init_ray_query_traversal_vars(shader, impl, array_length, VAR_NAME("_top"));
253
254 dst->stack = rq_variable_create(shader, impl, array_length,
255 glsl_array_type(glsl_uint_type(), MAX_STACK_ENTRY_COUNT,
256 glsl_get_explicit_stride(glsl_uint_type())),
257 VAR_NAME("_stack"));
258 }
259
260 #undef VAR_NAME
261
262 static void
lower_ray_query(nir_shader * shader,nir_function_impl * impl,nir_variable * ray_query,struct hash_table * ht)263 lower_ray_query(nir_shader *shader, nir_function_impl *impl, nir_variable *ray_query,
264 struct hash_table *ht)
265 {
266 struct ray_query_vars *vars = ralloc(impl, struct ray_query_vars);
267
268 unsigned array_length = 1;
269 if (glsl_type_is_array(ray_query->type))
270 array_length = glsl_get_length(ray_query->type);
271
272 init_ray_query_vars(shader, impl, array_length, vars,
273 ray_query->name == NULL ? "" : ray_query->name);
274
275 _mesa_hash_table_insert(ht, ray_query, vars);
276 }
277
278 static void
copy_candidate_to_closest(nir_builder * b,nir_ssa_def * index,struct ray_query_vars * vars)279 copy_candidate_to_closest(nir_builder *b, nir_ssa_def *index, struct ray_query_vars *vars)
280 {
281 rq_copy_var(b, index, vars->closest.barycentrics, vars->candidate.barycentrics, 0x3);
282 rq_copy_var(b, index, vars->closest.custom_instance_and_mask,
283 vars->candidate.custom_instance_and_mask, 0x1);
284 rq_copy_var(b, index, vars->closest.geometry_id_and_flags, vars->candidate.geometry_id_and_flags,
285 0x1);
286 rq_copy_var(b, index, vars->closest.instance_addr, vars->candidate.instance_addr, 0x1);
287 rq_copy_var(b, index, vars->closest.instance_id, vars->candidate.instance_id, 0x1);
288 rq_copy_var(b, index, vars->closest.intersection_type, vars->candidate.intersection_type, 0x1);
289 rq_copy_var(b, index, vars->closest.opaque, vars->candidate.opaque, 0x1);
290 rq_copy_var(b, index, vars->closest.frontface, vars->candidate.frontface, 0x1);
291 rq_copy_var(b, index, vars->closest.sbt_offset_and_flags, vars->candidate.sbt_offset_and_flags,
292 0x1);
293 rq_copy_var(b, index, vars->closest.primitive_id, vars->candidate.primitive_id, 0x1);
294 rq_copy_var(b, index, vars->closest.t, vars->candidate.t, 0x1);
295 }
296
297 static void
insert_terminate_on_first_hit(nir_builder * b,nir_ssa_def * index,struct ray_query_vars * vars,bool break_on_terminate)298 insert_terminate_on_first_hit(nir_builder *b, nir_ssa_def *index, struct ray_query_vars *vars,
299 bool break_on_terminate)
300 {
301 nir_ssa_def *terminate_on_first_hit =
302 nir_test_mask(b, rq_load_var(b, index, vars->flags), SpvRayFlagsTerminateOnFirstHitKHRMask);
303 nir_push_if(b, terminate_on_first_hit);
304 {
305 rq_store_var(b, index, vars->incomplete, nir_imm_bool(b, false), 0x1);
306 if (break_on_terminate)
307 nir_jump(b, nir_jump_break);
308 }
309 nir_pop_if(b, NULL);
310 }
311
312 static void
lower_rq_confirm_intersection(nir_builder * b,nir_ssa_def * index,nir_intrinsic_instr * instr,struct ray_query_vars * vars)313 lower_rq_confirm_intersection(nir_builder *b, nir_ssa_def *index, nir_intrinsic_instr *instr,
314 struct ray_query_vars *vars)
315 {
316 copy_candidate_to_closest(b, index, vars);
317 insert_terminate_on_first_hit(b, index, vars, false);
318 }
319
320 static void
lower_rq_generate_intersection(nir_builder * b,nir_ssa_def * index,nir_intrinsic_instr * instr,struct ray_query_vars * vars)321 lower_rq_generate_intersection(nir_builder *b, nir_ssa_def *index, nir_intrinsic_instr *instr,
322 struct ray_query_vars *vars)
323 {
324 nir_push_if(b, nir_iand(b, nir_fge(b, rq_load_var(b, index, vars->closest.t), instr->src[1].ssa),
325 nir_fge(b, instr->src[1].ssa, rq_load_var(b, index, vars->tmin))));
326 {
327 copy_candidate_to_closest(b, index, vars);
328 insert_terminate_on_first_hit(b, index, vars, false);
329 rq_store_var(b, index, vars->closest.t, instr->src[1].ssa, 0x1);
330 }
331 nir_pop_if(b, NULL);
332 }
333
334 enum rq_intersection_type {
335 intersection_type_none,
336 intersection_type_triangle,
337 intersection_type_aabb
338 };
339
340 static void
lower_rq_initialize(nir_builder * b,nir_ssa_def * index,nir_intrinsic_instr * instr,struct ray_query_vars * vars)341 lower_rq_initialize(nir_builder *b, nir_ssa_def *index, nir_intrinsic_instr *instr,
342 struct ray_query_vars *vars)
343 {
344 rq_store_var(b, index, vars->accel_struct, instr->src[1].ssa, 0x1);
345 rq_store_var(b, index, vars->flags, instr->src[2].ssa, 0x1);
346 rq_store_var(b, index, vars->cull_mask, nir_iand_imm(b, instr->src[3].ssa, 0xff), 0x1);
347
348 rq_store_var(b, index, vars->origin, instr->src[4].ssa, 0x7);
349 rq_store_var(b, index, vars->trav.origin, instr->src[4].ssa, 0x7);
350
351 rq_store_var(b, index, vars->tmin, instr->src[5].ssa, 0x1);
352
353 rq_store_var(b, index, vars->direction, instr->src[6].ssa, 0x7);
354 rq_store_var(b, index, vars->trav.direction, instr->src[6].ssa, 0x7);
355
356 nir_ssa_def *vec3ones = nir_channels(b, nir_imm_vec4(b, 1.0, 1.0, 1.0, 1.0), 0x7);
357 rq_store_var(b, index, vars->trav.inv_dir, nir_fdiv(b, vec3ones, instr->src[6].ssa), 0x7);
358
359 rq_store_var(b, index, vars->closest.t, instr->src[7].ssa, 0x1);
360 rq_store_var(b, index, vars->closest.intersection_type, nir_imm_int(b, intersection_type_none),
361 0x1);
362
363 nir_ssa_def *accel_struct = rq_load_var(b, index, vars->accel_struct);
364
365 nir_push_if(b, nir_ine_imm(b, accel_struct, 0));
366 {
367 rq_store_var(b, index, vars->trav.bvh_base, build_addr_to_node(b, accel_struct), 1);
368
369 nir_ssa_def *bvh_root =
370 nir_build_load_global(b, 1, 32, accel_struct, .access = ACCESS_NON_WRITEABLE,
371 .align_mul = 64, .align_offset = 0);
372
373 rq_store_var(b, index, vars->trav.stack, nir_imm_int(b, 1), 0x1);
374 rq_store_array(b, index, vars->stack, nir_imm_int(b, 0), bvh_root, 0x1);
375
376 rq_store_var(b, index, vars->trav.top_stack, nir_imm_int(b, 0), 1);
377
378 rq_store_var(b, index, vars->incomplete, nir_imm_bool(b, true), 0x1);
379 }
380 nir_push_else(b, NULL);
381 {
382 rq_store_var(b, index, vars->incomplete, nir_imm_bool(b, false), 0x1);
383 }
384 nir_pop_if(b, NULL);
385 }
386
387 static nir_ssa_def *
lower_rq_load(nir_builder * b,nir_ssa_def * index,struct ray_query_vars * vars,nir_ssa_def * committed,nir_ray_query_value value,unsigned column)388 lower_rq_load(nir_builder *b, nir_ssa_def *index, struct ray_query_vars *vars,
389 nir_ssa_def *committed, nir_ray_query_value value, unsigned column)
390 {
391 switch (value) {
392 case nir_ray_query_value_flags:
393 return rq_load_var(b, index, vars->flags);
394 case nir_ray_query_value_intersection_barycentrics:
395 return nir_bcsel(b, committed, rq_load_var(b, index, vars->closest.barycentrics),
396 rq_load_var(b, index, vars->candidate.barycentrics));
397 case nir_ray_query_value_intersection_candidate_aabb_opaque:
398 return nir_iand(b, rq_load_var(b, index, vars->candidate.opaque),
399 nir_ieq_imm(b, rq_load_var(b, index, vars->candidate.intersection_type),
400 intersection_type_aabb));
401 case nir_ray_query_value_intersection_front_face:
402 return nir_bcsel(b, committed, rq_load_var(b, index, vars->closest.frontface),
403 rq_load_var(b, index, vars->candidate.frontface));
404 case nir_ray_query_value_intersection_geometry_index:
405 return nir_iand_imm(
406 b,
407 nir_bcsel(b, committed, rq_load_var(b, index, vars->closest.geometry_id_and_flags),
408 rq_load_var(b, index, vars->candidate.geometry_id_and_flags)),
409 0xFFFFFF);
410 case nir_ray_query_value_intersection_instance_custom_index:
411 return nir_iand_imm(
412 b,
413 nir_bcsel(b, committed, rq_load_var(b, index, vars->closest.custom_instance_and_mask),
414 rq_load_var(b, index, vars->candidate.custom_instance_and_mask)),
415 0xFFFFFF);
416 case nir_ray_query_value_intersection_instance_id:
417 return nir_bcsel(b, committed, rq_load_var(b, index, vars->closest.instance_id),
418 rq_load_var(b, index, vars->candidate.instance_id));
419 case nir_ray_query_value_intersection_instance_sbt_index:
420 return nir_iand_imm(
421 b,
422 nir_bcsel(b, committed, rq_load_var(b, index, vars->closest.sbt_offset_and_flags),
423 rq_load_var(b, index, vars->candidate.sbt_offset_and_flags)),
424 0xFFFFFF);
425 case nir_ray_query_value_intersection_object_ray_direction: {
426 nir_ssa_def *instance_node_addr =
427 nir_bcsel(b, committed, rq_load_var(b, index, vars->closest.instance_addr),
428 rq_load_var(b, index, vars->candidate.instance_addr));
429 nir_ssa_def *wto_matrix[3];
430 nir_build_wto_matrix_load(b, instance_node_addr, wto_matrix);
431 return nir_build_vec3_mat_mult(b, rq_load_var(b, index, vars->direction), wto_matrix, false);
432 }
433 case nir_ray_query_value_intersection_object_ray_origin: {
434 nir_ssa_def *instance_node_addr =
435 nir_bcsel(b, committed, rq_load_var(b, index, vars->closest.instance_addr),
436 rq_load_var(b, index, vars->candidate.instance_addr));
437 nir_ssa_def *wto_matrix[] = {
438 nir_build_load_global(b, 4, 32, nir_iadd_imm(b, instance_node_addr, 16), .align_mul = 64,
439 .align_offset = 16),
440 nir_build_load_global(b, 4, 32, nir_iadd_imm(b, instance_node_addr, 32), .align_mul = 64,
441 .align_offset = 32),
442 nir_build_load_global(b, 4, 32, nir_iadd_imm(b, instance_node_addr, 48), .align_mul = 64,
443 .align_offset = 48)};
444 return nir_build_vec3_mat_mult_pre(b, rq_load_var(b, index, vars->origin), wto_matrix);
445 }
446 case nir_ray_query_value_intersection_object_to_world: {
447 nir_ssa_def *instance_node_addr =
448 nir_bcsel(b, committed, rq_load_var(b, index, vars->closest.instance_addr),
449 rq_load_var(b, index, vars->candidate.instance_addr));
450
451 if (column == 3) {
452 nir_ssa_def *wto_matrix[3];
453 nir_build_wto_matrix_load(b, instance_node_addr, wto_matrix);
454
455 nir_ssa_def *vals[3];
456 for (unsigned i = 0; i < 3; ++i)
457 vals[i] = nir_channel(b, wto_matrix[i], column);
458
459 return nir_vec(b, vals, 3);
460 }
461
462 return nir_build_load_global(b, 3, 32, nir_iadd_imm(b, instance_node_addr, 92 + column * 12));
463 }
464 case nir_ray_query_value_intersection_primitive_index:
465 return nir_bcsel(b, committed, rq_load_var(b, index, vars->closest.primitive_id),
466 rq_load_var(b, index, vars->candidate.primitive_id));
467 case nir_ray_query_value_intersection_t:
468 return nir_bcsel(b, committed, rq_load_var(b, index, vars->closest.t),
469 rq_load_var(b, index, vars->candidate.t));
470 case nir_ray_query_value_intersection_type:
471 return nir_bcsel(
472 b, committed, rq_load_var(b, index, vars->closest.intersection_type),
473 nir_iadd_imm(b, rq_load_var(b, index, vars->candidate.intersection_type), -1));
474 case nir_ray_query_value_intersection_world_to_object: {
475 nir_ssa_def *instance_node_addr =
476 nir_bcsel(b, committed, rq_load_var(b, index, vars->closest.instance_addr),
477 rq_load_var(b, index, vars->candidate.instance_addr));
478
479 nir_ssa_def *wto_matrix[3];
480 nir_build_wto_matrix_load(b, instance_node_addr, wto_matrix);
481
482 nir_ssa_def *vals[3];
483 for (unsigned i = 0; i < 3; ++i)
484 vals[i] = nir_channel(b, wto_matrix[i], column);
485
486 if (column == 3)
487 return nir_fneg(b, nir_build_vec3_mat_mult(b, nir_vec(b, vals, 3), wto_matrix, false));
488
489 return nir_vec(b, vals, 3);
490 }
491 case nir_ray_query_value_tmin:
492 return rq_load_var(b, index, vars->tmin);
493 case nir_ray_query_value_world_ray_direction:
494 return rq_load_var(b, index, vars->direction);
495 case nir_ray_query_value_world_ray_origin:
496 return rq_load_var(b, index, vars->origin);
497 default:
498 unreachable("Invalid nir_ray_query_value!");
499 }
500
501 return NULL;
502 }
503
504 static void
insert_traversal_triangle_case(struct radv_device * device,nir_builder * b,nir_ssa_def * index,nir_ssa_def * result,struct ray_query_vars * vars,nir_ssa_def * bvh_node)505 insert_traversal_triangle_case(struct radv_device *device, nir_builder *b, nir_ssa_def *index,
506 nir_ssa_def *result, struct ray_query_vars *vars,
507 nir_ssa_def *bvh_node)
508 {
509 nir_ssa_def *dist = nir_channel(b, result, 0);
510 nir_ssa_def *div = nir_channel(b, result, 1);
511 dist = nir_fdiv(b, dist, div);
512 nir_ssa_def *frontface = nir_flt(b, nir_imm_float(b, 0), div);
513 nir_ssa_def *switch_ccw =
514 nir_test_mask(b, rq_load_var(b, index, vars->candidate.sbt_offset_and_flags),
515 VK_GEOMETRY_INSTANCE_TRIANGLE_FLIP_FACING_BIT_KHR << 24);
516 frontface = nir_ixor(b, frontface, switch_ccw);
517 rq_store_var(b, index, vars->candidate.frontface, frontface, 0x1);
518
519 nir_ssa_def *not_cull = nir_inot(
520 b, nir_test_mask(b, rq_load_var(b, index, vars->flags), SpvRayFlagsSkipTrianglesKHRMask));
521 nir_ssa_def *not_facing_cull = nir_ieq_imm(
522 b,
523 nir_iand(b, rq_load_var(b, index, vars->flags),
524 nir_bcsel(b, frontface, nir_imm_int(b, SpvRayFlagsCullFrontFacingTrianglesKHRMask),
525 nir_imm_int(b, SpvRayFlagsCullBackFacingTrianglesKHRMask))),
526 0);
527
528 not_cull = nir_iand(
529 b, not_cull,
530 nir_ior(b, not_facing_cull,
531 nir_test_mask(b, rq_load_var(b, index, vars->candidate.sbt_offset_and_flags),
532 VK_GEOMETRY_INSTANCE_TRIANGLE_FACING_CULL_DISABLE_BIT_KHR << 24)));
533
534 nir_push_if(b, nir_iand(b,
535 nir_iand(b, nir_fge(b, rq_load_var(b, index, vars->closest.t), dist),
536 nir_fge(b, dist, rq_load_var(b, index, vars->tmin))),
537 not_cull));
538 {
539 nir_ssa_def *triangle_info =
540 nir_build_load_global(b, 2, 32,
541 nir_iadd_imm(b, build_node_to_addr(device, b, bvh_node),
542 offsetof(struct radv_bvh_triangle_node, triangle_id)));
543 nir_ssa_def *primitive_id = nir_channel(b, triangle_info, 0);
544 nir_ssa_def *geometry_id_and_flags = nir_channel(b, triangle_info, 1);
545 nir_ssa_def *is_opaque =
546 hit_is_opaque(b, rq_load_var(b, index, vars->candidate.sbt_offset_and_flags),
547 rq_load_var(b, index, vars->flags), geometry_id_and_flags);
548
549 not_cull =
550 nir_ieq_imm(b,
551 nir_iand(b, rq_load_var(b, index, vars->flags),
552 nir_bcsel(b, is_opaque, nir_imm_int(b, SpvRayFlagsCullOpaqueKHRMask),
553 nir_imm_int(b, SpvRayFlagsCullNoOpaqueKHRMask))),
554 0);
555 nir_push_if(b, not_cull);
556 {
557 nir_ssa_def *divs[2] = {div, div};
558 nir_ssa_def *ij = nir_fdiv(b, nir_channels(b, result, 0xc), nir_vec(b, divs, 2));
559
560 rq_store_var(b, index, vars->candidate.barycentrics, ij, 3);
561 rq_store_var(b, index, vars->candidate.primitive_id, primitive_id, 1);
562 rq_store_var(b, index, vars->candidate.geometry_id_and_flags, geometry_id_and_flags, 1);
563 rq_store_var(b, index, vars->candidate.t, dist, 0x1);
564 rq_store_var(b, index, vars->candidate.opaque, is_opaque, 0x1);
565 rq_store_var(b, index, vars->candidate.intersection_type,
566 nir_imm_int(b, intersection_type_triangle), 0x1);
567
568 nir_push_if(b, is_opaque);
569 {
570 copy_candidate_to_closest(b, index, vars);
571 insert_terminate_on_first_hit(b, index, vars, true);
572 }
573 nir_push_else(b, NULL);
574 {
575 nir_jump(b, nir_jump_break);
576 }
577 nir_pop_if(b, NULL);
578 }
579 nir_pop_if(b, NULL);
580 }
581 nir_pop_if(b, NULL);
582 }
583
584 static void
insert_traversal_aabb_case(struct radv_device * device,nir_builder * b,nir_ssa_def * index,struct ray_query_vars * vars,nir_ssa_def * bvh_node)585 insert_traversal_aabb_case(struct radv_device *device, nir_builder *b, nir_ssa_def *index,
586 struct ray_query_vars *vars, nir_ssa_def *bvh_node)
587 {
588 nir_ssa_def *node_addr = build_node_to_addr(device, b, bvh_node);
589 nir_ssa_def *triangle_info = nir_build_load_global(b, 2, 32, nir_iadd_imm(b, node_addr, 24));
590 nir_ssa_def *primitive_id = nir_channel(b, triangle_info, 0);
591 nir_ssa_def *geometry_id_and_flags = nir_channel(b, triangle_info, 1);
592 nir_ssa_def *is_opaque =
593 hit_is_opaque(b, rq_load_var(b, index, vars->candidate.sbt_offset_and_flags),
594 rq_load_var(b, index, vars->flags), geometry_id_and_flags);
595
596 nir_ssa_def *not_skip_aabb = nir_inot(
597 b, nir_test_mask(b, rq_load_var(b, index, vars->flags), SpvRayFlagsSkipAABBsKHRMask));
598 nir_ssa_def *not_cull = nir_iand(
599 b, not_skip_aabb,
600 nir_ieq_imm(b,
601 nir_iand(b, rq_load_var(b, index, vars->flags),
602 nir_bcsel(b, is_opaque, nir_imm_int(b, SpvRayFlagsCullOpaqueKHRMask),
603 nir_imm_int(b, SpvRayFlagsCullNoOpaqueKHRMask))),
604 0));
605 nir_push_if(b, not_cull);
606 {
607 nir_ssa_def *vec3_zero = nir_channels(b, nir_imm_vec4(b, 0, 0, 0, 0), 0x7);
608 nir_ssa_def *vec3_inf =
609 nir_channels(b, nir_imm_vec4(b, INFINITY, INFINITY, INFINITY, 0), 0x7);
610
611 nir_ssa_def *bvh_lo = nir_build_load_global(b, 3, 32, nir_iadd_imm(b, node_addr, 0));
612 nir_ssa_def *bvh_hi = nir_build_load_global(b, 3, 32, nir_iadd_imm(b, node_addr, 12));
613
614 bvh_lo = nir_fsub(b, bvh_lo, rq_load_var(b, index, vars->trav.origin));
615 bvh_hi = nir_fsub(b, bvh_hi, rq_load_var(b, index, vars->trav.origin));
616 nir_ssa_def *t_vec =
617 nir_fmin(b, nir_fmul(b, bvh_lo, rq_load_var(b, index, vars->trav.inv_dir)),
618 nir_fmul(b, bvh_hi, rq_load_var(b, index, vars->trav.inv_dir)));
619 nir_ssa_def *t2_vec =
620 nir_fmax(b, nir_fmul(b, bvh_lo, rq_load_var(b, index, vars->trav.inv_dir)),
621 nir_fmul(b, bvh_hi, rq_load_var(b, index, vars->trav.inv_dir)));
622 /* If we run parallel to one of the edges the range should be [0, inf) not [0,0] */
623 t2_vec = nir_bcsel(b, nir_feq(b, rq_load_var(b, index, vars->trav.direction), vec3_zero),
624 vec3_inf, t2_vec);
625
626 nir_ssa_def *t_min = nir_fmax(b, nir_channel(b, t_vec, 0), nir_channel(b, t_vec, 1));
627 t_min = nir_fmax(b, t_min, nir_channel(b, t_vec, 2));
628
629 nir_ssa_def *t_max = nir_fmin(b, nir_channel(b, t2_vec, 0), nir_channel(b, t2_vec, 1));
630 t_max = nir_fmin(b, t_max, nir_channel(b, t2_vec, 2));
631
632 nir_push_if(b, nir_iand(b, nir_fge(b, rq_load_var(b, index, vars->closest.t), t_min),
633 nir_fge(b, t_max, rq_load_var(b, index, vars->tmin))));
634 {
635 rq_store_var(b, index, vars->candidate.t,
636 nir_fmax(b, t_min, rq_load_var(b, index, vars->tmin)), 0x1);
637 rq_store_var(b, index, vars->candidate.primitive_id, primitive_id, 1);
638 rq_store_var(b, index, vars->candidate.geometry_id_and_flags, geometry_id_and_flags, 1);
639 rq_store_var(b, index, vars->candidate.opaque, is_opaque, 0x1);
640 rq_store_var(b, index, vars->candidate.intersection_type,
641 nir_imm_int(b, intersection_type_aabb), 0x1);
642
643 nir_push_if(b, is_opaque);
644 {
645 copy_candidate_to_closest(b, index, vars);
646 }
647 nir_pop_if(b, NULL);
648
649 nir_jump(b, nir_jump_break);
650 }
651 nir_pop_if(b, NULL);
652 }
653 nir_pop_if(b, NULL);
654 }
655
656 static nir_ssa_def *
lower_rq_proceed(nir_builder * b,nir_ssa_def * index,struct ray_query_vars * vars,struct radv_device * device)657 lower_rq_proceed(nir_builder *b, nir_ssa_def *index, struct ray_query_vars *vars,
658 struct radv_device *device)
659 {
660 nir_push_if(b, rq_load_var(b, index, vars->incomplete));
661 {
662 nir_ssa_def *desc = create_bvh_descriptor(b);
663 nir_ssa_def *vec3ones = nir_channels(b, nir_imm_vec4(b, 1.0, 1.0, 1.0, 1.0), 0x7);
664
665 nir_push_loop(b);
666 {
667 nir_push_if(b, nir_uge(b, rq_load_var(b, index, vars->trav.top_stack),
668 rq_load_var(b, index, vars->trav.stack)));
669 {
670 nir_push_if(b, nir_ieq_imm(b, rq_load_var(b, index, vars->trav.stack), 0));
671 {
672 rq_store_var(b, index, vars->incomplete, nir_imm_bool(b, false), 0x1);
673 nir_jump(b, nir_jump_break);
674 }
675 nir_pop_if(b, NULL);
676
677 rq_store_var(b, index, vars->trav.top_stack, nir_imm_int(b, 0), 1);
678 rq_store_var(b, index, vars->trav.bvh_base,
679 build_addr_to_node(b, rq_load_var(b, index, vars->accel_struct)), 1);
680 rq_store_var(b, index, vars->trav.origin, rq_load_var(b, index, vars->origin), 7);
681 rq_store_var(b, index, vars->trav.direction, rq_load_var(b, index, vars->direction), 7);
682 rq_store_var(b, index, vars->trav.inv_dir,
683 nir_fdiv(b, vec3ones, rq_load_var(b, index, vars->direction)), 7);
684 }
685 nir_pop_if(b, NULL);
686
687 rq_store_var(b, index, vars->trav.stack,
688 nir_iadd_imm(b, rq_load_var(b, index, vars->trav.stack), -1), 1);
689
690 nir_ssa_def *bvh_node =
691 rq_load_array(b, index, vars->stack, rq_load_var(b, index, vars->trav.stack));
692 nir_ssa_def *bvh_node_type = bvh_node;
693
694 bvh_node =
695 nir_iadd(b, rq_load_var(b, index, vars->trav.bvh_base), nir_u2u(b, bvh_node, 64));
696 nir_ssa_def *intrinsic_result = NULL;
697 if (!radv_emulate_rt(device->physical_device)) {
698 intrinsic_result = nir_bvh64_intersect_ray_amd(
699 b, 32, desc, nir_unpack_64_2x32(b, bvh_node), rq_load_var(b, index, vars->closest.t),
700 rq_load_var(b, index, vars->trav.origin),
701 rq_load_var(b, index, vars->trav.direction),
702 rq_load_var(b, index, vars->trav.inv_dir));
703 }
704
705 /* if (node.type_flags & aabb) */
706 nir_push_if(b, nir_ine_imm(b, nir_iand_imm(b, bvh_node_type, 4), 0));
707 {
708 /* if (node.type_flags & leaf) */
709 nir_push_if(b, nir_ine_imm(b, nir_iand_imm(b, bvh_node_type, 2), 0));
710 {
711 /* custom */
712 nir_push_if(b, nir_ine_imm(b, nir_iand_imm(b, bvh_node_type, 1), 0));
713 {
714 insert_traversal_aabb_case(device, b, index, vars, bvh_node);
715 }
716 nir_push_else(b, NULL);
717 {
718 /* instance */
719 nir_ssa_def *instance_node_addr = build_node_to_addr(device, b, bvh_node);
720 nir_ssa_def *instance_data = nir_build_load_global(
721 b, 4, 32, instance_node_addr, .align_mul = 64, .align_offset = 0);
722 nir_ssa_def *instance_and_mask = nir_channel(b, instance_data, 2);
723 nir_ssa_def *instance_mask = nir_ushr_imm(b, instance_and_mask, 24);
724
725 nir_push_if(
726 b,
727 nir_ieq_imm(
728 b, nir_iand(b, instance_mask, rq_load_var(b, index, vars->cull_mask)), 0));
729 {
730 nir_jump(b, nir_jump_continue);
731 }
732 nir_pop_if(b, NULL);
733
734 nir_ssa_def *wto_matrix[] = {
735 nir_build_load_global(b, 4, 32, nir_iadd_imm(b, instance_node_addr, 16),
736 .align_mul = 64, .align_offset = 16),
737 nir_build_load_global(b, 4, 32, nir_iadd_imm(b, instance_node_addr, 32),
738 .align_mul = 64, .align_offset = 32),
739 nir_build_load_global(b, 4, 32, nir_iadd_imm(b, instance_node_addr, 48),
740 .align_mul = 64, .align_offset = 48)};
741 nir_ssa_def *instance_id =
742 nir_build_load_global(b, 1, 32, nir_iadd_imm(b, instance_node_addr, 88));
743
744 rq_store_var(b, index, vars->trav.top_stack,
745 rq_load_var(b, index, vars->trav.stack), 1);
746 rq_store_var(b, index, vars->trav.bvh_base,
747 build_addr_to_node(
748 b, nir_pack_64_2x32(b, nir_channels(b, instance_data, 0x3))),
749 1);
750
751 rq_store_array(b, index, vars->stack, rq_load_var(b, index, vars->trav.stack),
752 nir_iand_imm(b, nir_channel(b, instance_data, 0), 63), 0x1);
753 rq_store_var(b, index, vars->trav.stack,
754 nir_iadd_imm(b, rq_load_var(b, index, vars->trav.stack), 1), 1);
755
756 rq_store_var(b, index, vars->trav.origin,
757 nir_build_vec3_mat_mult_pre(b, rq_load_var(b, index, vars->origin),
758 wto_matrix),
759 7);
760 rq_store_var(b, index, vars->trav.direction,
761 nir_build_vec3_mat_mult(b, rq_load_var(b, index, vars->direction),
762 wto_matrix, false),
763 7);
764 rq_store_var(b, index, vars->trav.inv_dir,
765 nir_fdiv(b, vec3ones, rq_load_var(b, index, vars->trav.direction)),
766 7);
767
768 rq_store_var(b, index, vars->candidate.sbt_offset_and_flags,
769 nir_channel(b, instance_data, 3), 1);
770 rq_store_var(b, index, vars->candidate.custom_instance_and_mask,
771 instance_and_mask, 1);
772 rq_store_var(b, index, vars->candidate.instance_id, instance_id, 1);
773 rq_store_var(b, index, vars->candidate.instance_addr, instance_node_addr, 1);
774 }
775 nir_pop_if(b, NULL);
776 }
777 nir_push_else(b, NULL);
778 {
779 nir_ssa_def *result = intrinsic_result;
780 if (!result) {
781 /* If we didn't run the intrinsic cause the hardware didn't support it,
782 * emulate ray/box intersection here */
783 result = intersect_ray_amd_software_box(
784 device, b, bvh_node, rq_load_var(b, index, vars->closest.t),
785 rq_load_var(b, index, vars->trav.origin),
786 rq_load_var(b, index, vars->trav.direction),
787 rq_load_var(b, index, vars->trav.inv_dir));
788 }
789
790 /* box */
791 for (unsigned i = 4; i-- > 0;) {
792 nir_ssa_def *new_node = nir_channel(b, result, i);
793 nir_push_if(b, nir_ine_imm(b, new_node, 0xffffffff));
794 {
795 rq_store_array(b, index, vars->stack, rq_load_var(b, index, vars->trav.stack),
796 new_node, 0x1);
797 rq_store_var(b, index, vars->trav.stack,
798 nir_iadd_imm(b, rq_load_var(b, index, vars->trav.stack), 1), 1);
799 }
800 nir_pop_if(b, NULL);
801 }
802 }
803 nir_pop_if(b, NULL);
804 }
805 nir_push_else(b, NULL);
806 {
807 nir_ssa_def *result = intrinsic_result;
808 if (!result) {
809 /* If we didn't run the intrinsic cause the hardware didn't support it,
810 * emulate ray/tri intersection here */
811 result = intersect_ray_amd_software_tri(device, b, bvh_node,
812 rq_load_var(b, index, vars->closest.t),
813 rq_load_var(b, index, vars->trav.origin),
814 rq_load_var(b, index, vars->trav.direction),
815 rq_load_var(b, index, vars->trav.inv_dir));
816 }
817 insert_traversal_triangle_case(device, b, index, result, vars, bvh_node);
818 }
819 nir_pop_if(b, NULL);
820 }
821 nir_pop_loop(b, NULL);
822 }
823 nir_pop_if(b, NULL);
824
825 return rq_load_var(b, index, vars->incomplete);
826 }
827
828 static void
lower_rq_terminate(nir_builder * b,nir_ssa_def * index,nir_intrinsic_instr * instr,struct ray_query_vars * vars)829 lower_rq_terminate(nir_builder *b, nir_ssa_def *index, nir_intrinsic_instr *instr,
830 struct ray_query_vars *vars)
831 {
832 rq_store_var(b, index, vars->incomplete, nir_imm_bool(b, false), 0x1);
833 }
834
835 static bool
is_rq_intrinsic(nir_intrinsic_op intrinsic)836 is_rq_intrinsic(nir_intrinsic_op intrinsic)
837 {
838 switch (intrinsic) {
839 case nir_intrinsic_rq_confirm_intersection:
840 case nir_intrinsic_rq_generate_intersection:
841 case nir_intrinsic_rq_initialize:
842 case nir_intrinsic_rq_load:
843 case nir_intrinsic_rq_proceed:
844 case nir_intrinsic_rq_terminate:
845 return true;
846 default:
847 return false;
848 }
849 }
850
851 bool
radv_nir_lower_ray_queries(struct nir_shader * shader,struct radv_device * device)852 radv_nir_lower_ray_queries(struct nir_shader *shader, struct radv_device *device)
853 {
854 bool contains_ray_query = false;
855 struct hash_table *query_ht = _mesa_pointer_hash_table_create(NULL);
856
857 nir_foreach_variable_in_list (var, &shader->variables) {
858 if (!var->data.ray_query)
859 continue;
860
861 lower_ray_query(shader, NULL, var, query_ht);
862 contains_ray_query = true;
863 }
864
865 nir_foreach_function (function, shader) {
866 if (!function->impl)
867 continue;
868
869 nir_builder builder;
870 nir_builder_init(&builder, function->impl);
871
872 nir_foreach_variable_in_list (var, &function->impl->locals) {
873 if (!var->data.ray_query)
874 continue;
875
876 lower_ray_query(NULL, function->impl, var, query_ht);
877 contains_ray_query = true;
878 }
879
880 if (!contains_ray_query)
881 continue;
882
883 nir_foreach_block (block, function->impl) {
884 nir_foreach_instr_safe (instr, block) {
885 if (instr->type != nir_instr_type_intrinsic)
886 continue;
887
888 nir_intrinsic_instr *intrinsic = nir_instr_as_intrinsic(instr);
889
890 if (!is_rq_intrinsic(intrinsic->intrinsic))
891 continue;
892
893 nir_deref_instr *ray_query_deref =
894 nir_instr_as_deref(intrinsic->src[0].ssa->parent_instr);
895 nir_ssa_def *index = NULL;
896
897 if (ray_query_deref->deref_type == nir_deref_type_array) {
898 index = ray_query_deref->arr.index.ssa;
899 ray_query_deref = nir_instr_as_deref(ray_query_deref->parent.ssa->parent_instr);
900 }
901
902 assert(ray_query_deref->deref_type == nir_deref_type_var);
903
904 struct ray_query_vars *vars =
905 (struct ray_query_vars *)_mesa_hash_table_search(query_ht, ray_query_deref->var)
906 ->data;
907
908 builder.cursor = nir_before_instr(instr);
909
910 nir_ssa_def *new_dest = NULL;
911
912 switch (intrinsic->intrinsic) {
913 case nir_intrinsic_rq_confirm_intersection:
914 lower_rq_confirm_intersection(&builder, index, intrinsic, vars);
915 break;
916 case nir_intrinsic_rq_generate_intersection:
917 lower_rq_generate_intersection(&builder, index, intrinsic, vars);
918 break;
919 case nir_intrinsic_rq_initialize:
920 lower_rq_initialize(&builder, index, intrinsic, vars);
921 break;
922 case nir_intrinsic_rq_load:
923 new_dest = lower_rq_load(&builder, index, vars, intrinsic->src[1].ssa,
924 (nir_ray_query_value)nir_intrinsic_base(intrinsic),
925 nir_intrinsic_column(intrinsic));
926 break;
927 case nir_intrinsic_rq_proceed:
928 new_dest = lower_rq_proceed(&builder, index, vars, device);
929 break;
930 case nir_intrinsic_rq_terminate:
931 lower_rq_terminate(&builder, index, intrinsic, vars);
932 break;
933 default:
934 unreachable("Unsupported ray query intrinsic!");
935 }
936
937 if (new_dest)
938 nir_ssa_def_rewrite_uses(&intrinsic->dest.ssa, new_dest);
939
940 nir_instr_remove(instr);
941 nir_instr_free(instr);
942 }
943 }
944
945 nir_metadata_preserve(function->impl, nir_metadata_none);
946 }
947
948 ralloc_free(query_ht);
949
950 return contains_ray_query;
951 }
952