1 /*
2 * Copyright © 2015 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Jason Ekstrand (jason@jlekstrand.net)
25 *
26 */
27
28 #include "vtn_private.h"
29 #include "spirv_info.h"
30
31 static struct vtn_access_chain *
vtn_access_chain_extend(struct vtn_builder * b,struct vtn_access_chain * old,unsigned new_ids)32 vtn_access_chain_extend(struct vtn_builder *b, struct vtn_access_chain *old,
33 unsigned new_ids)
34 {
35 struct vtn_access_chain *chain;
36
37 unsigned new_len = old->length + new_ids;
38 /* TODO: don't use rzalloc */
39 chain = rzalloc_size(b, sizeof(*chain) + new_len * sizeof(chain->link[0]));
40
41 chain->var = old->var;
42 chain->length = new_len;
43
44 for (unsigned i = 0; i < old->length; i++)
45 chain->link[i] = old->link[i];
46
47 return chain;
48 }
49
50 static nir_ssa_def *
vtn_access_link_as_ssa(struct vtn_builder * b,struct vtn_access_link link,unsigned stride)51 vtn_access_link_as_ssa(struct vtn_builder *b, struct vtn_access_link link,
52 unsigned stride)
53 {
54 assert(stride > 0);
55 if (link.mode == vtn_access_mode_literal) {
56 return nir_imm_int(&b->nb, link.id * stride);
57 } else if (stride == 1) {
58 return vtn_ssa_value(b, link.id)->def;
59 } else {
60 return nir_imul(&b->nb, vtn_ssa_value(b, link.id)->def,
61 nir_imm_int(&b->nb, stride));
62 }
63 }
64
65 static struct vtn_type *
vtn_access_chain_tail_type(struct vtn_builder * b,struct vtn_access_chain * chain)66 vtn_access_chain_tail_type(struct vtn_builder *b,
67 struct vtn_access_chain *chain)
68 {
69 struct vtn_type *type = chain->var->type;
70 for (unsigned i = 0; i < chain->length; i++) {
71 if (glsl_type_is_struct(type->type)) {
72 assert(chain->link[i].mode == vtn_access_mode_literal);
73 type = type->members[chain->link[i].id];
74 } else {
75 type = type->array_element;
76 }
77 }
78 return type;
79 }
80
81 /* Crawls a chain of array derefs and rewrites the types so that the
82 * lengths stay the same but the terminal type is the one given by
83 * tail_type. This is useful for split structures.
84 */
85 static void
rewrite_deref_types(nir_deref * deref,const struct glsl_type * type)86 rewrite_deref_types(nir_deref *deref, const struct glsl_type *type)
87 {
88 deref->type = type;
89 if (deref->child) {
90 assert(deref->child->deref_type == nir_deref_type_array);
91 assert(glsl_type_is_array(deref->type));
92 rewrite_deref_types(deref->child, glsl_get_array_element(type));
93 }
94 }
95
96 nir_deref_var *
vtn_access_chain_to_deref(struct vtn_builder * b,struct vtn_access_chain * chain)97 vtn_access_chain_to_deref(struct vtn_builder *b, struct vtn_access_chain *chain)
98 {
99 nir_deref_var *deref_var;
100 if (chain->var->var) {
101 deref_var = nir_deref_var_create(b, chain->var->var);
102 } else {
103 assert(chain->var->members);
104 /* Create the deref_var manually. It will get filled out later. */
105 deref_var = rzalloc(b, nir_deref_var);
106 deref_var->deref.deref_type = nir_deref_type_var;
107 }
108
109 struct vtn_type *deref_type = chain->var->type;
110 nir_deref *tail = &deref_var->deref;
111 nir_variable **members = chain->var->members;
112
113 for (unsigned i = 0; i < chain->length; i++) {
114 enum glsl_base_type base_type = glsl_get_base_type(deref_type->type);
115 switch (base_type) {
116 case GLSL_TYPE_UINT:
117 case GLSL_TYPE_INT:
118 case GLSL_TYPE_FLOAT:
119 case GLSL_TYPE_DOUBLE:
120 case GLSL_TYPE_BOOL:
121 case GLSL_TYPE_ARRAY: {
122 deref_type = deref_type->array_element;
123
124 nir_deref_array *deref_arr = nir_deref_array_create(b);
125 deref_arr->deref.type = deref_type->type;
126
127 if (chain->link[i].mode == vtn_access_mode_literal) {
128 deref_arr->deref_array_type = nir_deref_array_type_direct;
129 deref_arr->base_offset = chain->link[i].id;
130 } else {
131 assert(chain->link[i].mode == vtn_access_mode_id);
132 deref_arr->deref_array_type = nir_deref_array_type_indirect;
133 deref_arr->base_offset = 0;
134 deref_arr->indirect =
135 nir_src_for_ssa(vtn_ssa_value(b, chain->link[i].id)->def);
136 }
137 tail->child = &deref_arr->deref;
138 tail = tail->child;
139 break;
140 }
141
142 case GLSL_TYPE_STRUCT: {
143 assert(chain->link[i].mode == vtn_access_mode_literal);
144 unsigned idx = chain->link[i].id;
145 deref_type = deref_type->members[idx];
146 if (members) {
147 /* This is a pre-split structure. */
148 deref_var->var = members[idx];
149 rewrite_deref_types(&deref_var->deref, members[idx]->type);
150 assert(tail->type == deref_type->type);
151 members = NULL;
152 } else {
153 nir_deref_struct *deref_struct = nir_deref_struct_create(b, idx);
154 deref_struct->deref.type = deref_type->type;
155 tail->child = &deref_struct->deref;
156 tail = tail->child;
157 }
158 break;
159 }
160 default:
161 unreachable("Invalid type for deref");
162 }
163 }
164
165 assert(members == NULL);
166 return deref_var;
167 }
168
169 static void
_vtn_local_load_store(struct vtn_builder * b,bool load,nir_deref_var * deref,nir_deref * tail,struct vtn_ssa_value * inout)170 _vtn_local_load_store(struct vtn_builder *b, bool load, nir_deref_var *deref,
171 nir_deref *tail, struct vtn_ssa_value *inout)
172 {
173 /* The deref tail may contain a deref to select a component of a vector (in
174 * other words, it might not be an actual tail) so we have to save it away
175 * here since we overwrite it later.
176 */
177 nir_deref *old_child = tail->child;
178
179 if (glsl_type_is_vector_or_scalar(tail->type)) {
180 /* Terminate the deref chain in case there is one more link to pick
181 * off a component of the vector.
182 */
183 tail->child = NULL;
184
185 nir_intrinsic_op op = load ? nir_intrinsic_load_var :
186 nir_intrinsic_store_var;
187
188 nir_intrinsic_instr *intrin = nir_intrinsic_instr_create(b->shader, op);
189 intrin->variables[0] = nir_deref_var_clone(deref, intrin);
190 intrin->num_components = glsl_get_vector_elements(tail->type);
191
192 if (load) {
193 nir_ssa_dest_init(&intrin->instr, &intrin->dest,
194 intrin->num_components,
195 glsl_get_bit_size(tail->type),
196 NULL);
197 inout->def = &intrin->dest.ssa;
198 } else {
199 nir_intrinsic_set_write_mask(intrin, (1 << intrin->num_components) - 1);
200 intrin->src[0] = nir_src_for_ssa(inout->def);
201 }
202
203 nir_builder_instr_insert(&b->nb, &intrin->instr);
204 } else if (glsl_get_base_type(tail->type) == GLSL_TYPE_ARRAY ||
205 glsl_type_is_matrix(tail->type)) {
206 unsigned elems = glsl_get_length(tail->type);
207 nir_deref_array *deref_arr = nir_deref_array_create(b);
208 deref_arr->deref_array_type = nir_deref_array_type_direct;
209 deref_arr->deref.type = glsl_get_array_element(tail->type);
210 tail->child = &deref_arr->deref;
211 for (unsigned i = 0; i < elems; i++) {
212 deref_arr->base_offset = i;
213 _vtn_local_load_store(b, load, deref, tail->child, inout->elems[i]);
214 }
215 } else {
216 assert(glsl_get_base_type(tail->type) == GLSL_TYPE_STRUCT);
217 unsigned elems = glsl_get_length(tail->type);
218 nir_deref_struct *deref_struct = nir_deref_struct_create(b, 0);
219 tail->child = &deref_struct->deref;
220 for (unsigned i = 0; i < elems; i++) {
221 deref_struct->index = i;
222 deref_struct->deref.type = glsl_get_struct_field(tail->type, i);
223 _vtn_local_load_store(b, load, deref, tail->child, inout->elems[i]);
224 }
225 }
226
227 tail->child = old_child;
228 }
229
230 nir_deref_var *
vtn_nir_deref(struct vtn_builder * b,uint32_t id)231 vtn_nir_deref(struct vtn_builder *b, uint32_t id)
232 {
233 struct vtn_access_chain *chain =
234 vtn_value(b, id, vtn_value_type_access_chain)->access_chain;
235
236 return vtn_access_chain_to_deref(b, chain);
237 }
238
239 /*
240 * Gets the NIR-level deref tail, which may have as a child an array deref
241 * selecting which component due to OpAccessChain supporting per-component
242 * indexing in SPIR-V.
243 */
244 static nir_deref *
get_deref_tail(nir_deref_var * deref)245 get_deref_tail(nir_deref_var *deref)
246 {
247 nir_deref *cur = &deref->deref;
248 while (!glsl_type_is_vector_or_scalar(cur->type) && cur->child)
249 cur = cur->child;
250
251 return cur;
252 }
253
254 struct vtn_ssa_value *
vtn_local_load(struct vtn_builder * b,nir_deref_var * src)255 vtn_local_load(struct vtn_builder *b, nir_deref_var *src)
256 {
257 nir_deref *src_tail = get_deref_tail(src);
258 struct vtn_ssa_value *val = vtn_create_ssa_value(b, src_tail->type);
259 _vtn_local_load_store(b, true, src, src_tail, val);
260
261 if (src_tail->child) {
262 nir_deref_array *vec_deref = nir_deref_as_array(src_tail->child);
263 assert(vec_deref->deref.child == NULL);
264 val->type = vec_deref->deref.type;
265 if (vec_deref->deref_array_type == nir_deref_array_type_direct)
266 val->def = vtn_vector_extract(b, val->def, vec_deref->base_offset);
267 else
268 val->def = vtn_vector_extract_dynamic(b, val->def,
269 vec_deref->indirect.ssa);
270 }
271
272 return val;
273 }
274
275 void
vtn_local_store(struct vtn_builder * b,struct vtn_ssa_value * src,nir_deref_var * dest)276 vtn_local_store(struct vtn_builder *b, struct vtn_ssa_value *src,
277 nir_deref_var *dest)
278 {
279 nir_deref *dest_tail = get_deref_tail(dest);
280
281 if (dest_tail->child) {
282 struct vtn_ssa_value *val = vtn_create_ssa_value(b, dest_tail->type);
283 _vtn_local_load_store(b, true, dest, dest_tail, val);
284 nir_deref_array *deref = nir_deref_as_array(dest_tail->child);
285 assert(deref->deref.child == NULL);
286 if (deref->deref_array_type == nir_deref_array_type_direct)
287 val->def = vtn_vector_insert(b, val->def, src->def,
288 deref->base_offset);
289 else
290 val->def = vtn_vector_insert_dynamic(b, val->def, src->def,
291 deref->indirect.ssa);
292 _vtn_local_load_store(b, false, dest, dest_tail, val);
293 } else {
294 _vtn_local_load_store(b, false, dest, dest_tail, src);
295 }
296 }
297
298 static nir_ssa_def *
get_vulkan_resource_index(struct vtn_builder * b,struct vtn_access_chain * chain,struct vtn_type ** type,unsigned * chain_idx)299 get_vulkan_resource_index(struct vtn_builder *b, struct vtn_access_chain *chain,
300 struct vtn_type **type, unsigned *chain_idx)
301 {
302 /* Push constants have no explicit binding */
303 if (chain->var->mode == vtn_variable_mode_push_constant) {
304 *chain_idx = 0;
305 *type = chain->var->type;
306 return NULL;
307 }
308
309 nir_ssa_def *array_index;
310 if (glsl_type_is_array(chain->var->type->type)) {
311 assert(chain->length > 0);
312 array_index = vtn_access_link_as_ssa(b, chain->link[0], 1);
313 *chain_idx = 1;
314 *type = chain->var->type->array_element;
315 } else {
316 array_index = nir_imm_int(&b->nb, 0);
317 *chain_idx = 0;
318 *type = chain->var->type;
319 }
320
321 nir_intrinsic_instr *instr =
322 nir_intrinsic_instr_create(b->nb.shader,
323 nir_intrinsic_vulkan_resource_index);
324 instr->src[0] = nir_src_for_ssa(array_index);
325 nir_intrinsic_set_desc_set(instr, chain->var->descriptor_set);
326 nir_intrinsic_set_binding(instr, chain->var->binding);
327
328 nir_ssa_dest_init(&instr->instr, &instr->dest, 1, 32, NULL);
329 nir_builder_instr_insert(&b->nb, &instr->instr);
330
331 return &instr->dest.ssa;
332 }
333
334 nir_ssa_def *
vtn_access_chain_to_offset(struct vtn_builder * b,struct vtn_access_chain * chain,nir_ssa_def ** index_out,struct vtn_type ** type_out,unsigned * end_idx_out,bool stop_at_matrix)335 vtn_access_chain_to_offset(struct vtn_builder *b,
336 struct vtn_access_chain *chain,
337 nir_ssa_def **index_out, struct vtn_type **type_out,
338 unsigned *end_idx_out, bool stop_at_matrix)
339 {
340 unsigned idx = 0;
341 struct vtn_type *type;
342 *index_out = get_vulkan_resource_index(b, chain, &type, &idx);
343
344 nir_ssa_def *offset = nir_imm_int(&b->nb, 0);
345 for (; idx < chain->length; idx++) {
346 enum glsl_base_type base_type = glsl_get_base_type(type->type);
347 switch (base_type) {
348 case GLSL_TYPE_UINT:
349 case GLSL_TYPE_INT:
350 case GLSL_TYPE_FLOAT:
351 case GLSL_TYPE_DOUBLE:
352 case GLSL_TYPE_BOOL:
353 /* Some users may not want matrix or vector derefs */
354 if (stop_at_matrix)
355 goto end;
356 /* Fall through */
357
358 case GLSL_TYPE_ARRAY:
359 offset = nir_iadd(&b->nb, offset,
360 vtn_access_link_as_ssa(b, chain->link[idx],
361 type->stride));
362
363 type = type->array_element;
364 break;
365
366 case GLSL_TYPE_STRUCT: {
367 assert(chain->link[idx].mode == vtn_access_mode_literal);
368 unsigned member = chain->link[idx].id;
369 offset = nir_iadd(&b->nb, offset,
370 nir_imm_int(&b->nb, type->offsets[member]));
371 type = type->members[member];
372 break;
373 }
374
375 default:
376 unreachable("Invalid type for deref");
377 }
378 }
379
380 end:
381 *type_out = type;
382 if (end_idx_out)
383 *end_idx_out = idx;
384
385 return offset;
386 }
387
388 /* Tries to compute the size of an interface block based on the strides and
389 * offsets that are provided to us in the SPIR-V source.
390 */
391 static unsigned
vtn_type_block_size(struct vtn_type * type)392 vtn_type_block_size(struct vtn_type *type)
393 {
394 enum glsl_base_type base_type = glsl_get_base_type(type->type);
395 switch (base_type) {
396 case GLSL_TYPE_UINT:
397 case GLSL_TYPE_INT:
398 case GLSL_TYPE_FLOAT:
399 case GLSL_TYPE_BOOL:
400 case GLSL_TYPE_DOUBLE: {
401 unsigned cols = type->row_major ? glsl_get_vector_elements(type->type) :
402 glsl_get_matrix_columns(type->type);
403 if (cols > 1) {
404 assert(type->stride > 0);
405 return type->stride * cols;
406 } else if (base_type == GLSL_TYPE_DOUBLE) {
407 return glsl_get_vector_elements(type->type) * 8;
408 } else {
409 return glsl_get_vector_elements(type->type) * 4;
410 }
411 }
412
413 case GLSL_TYPE_STRUCT:
414 case GLSL_TYPE_INTERFACE: {
415 unsigned size = 0;
416 unsigned num_fields = glsl_get_length(type->type);
417 for (unsigned f = 0; f < num_fields; f++) {
418 unsigned field_end = type->offsets[f] +
419 vtn_type_block_size(type->members[f]);
420 size = MAX2(size, field_end);
421 }
422 return size;
423 }
424
425 case GLSL_TYPE_ARRAY:
426 assert(type->stride > 0);
427 assert(glsl_get_length(type->type) > 0);
428 return type->stride * glsl_get_length(type->type);
429
430 default:
431 assert(!"Invalid block type");
432 return 0;
433 }
434 }
435
436 static void
vtn_access_chain_get_offset_size(struct vtn_access_chain * chain,unsigned * access_offset,unsigned * access_size)437 vtn_access_chain_get_offset_size(struct vtn_access_chain *chain,
438 unsigned *access_offset,
439 unsigned *access_size)
440 {
441 /* Only valid for push constants accesses now. */
442 assert(chain->var->mode == vtn_variable_mode_push_constant);
443
444 struct vtn_type *type = chain->var->type;
445
446 *access_offset = 0;
447
448 for (unsigned i = 0; i < chain->length; i++) {
449 if (chain->link[i].mode != vtn_access_mode_literal)
450 break;
451
452 if (glsl_type_is_struct(type->type)) {
453 *access_offset += type->offsets[chain->link[i].id];
454 type = type->members[chain->link[i].id];
455 } else {
456 *access_offset += type->stride * chain->link[i].id;
457 type = type->array_element;
458 }
459 }
460
461 *access_size = vtn_type_block_size(type);
462 }
463
464 static void
_vtn_load_store_tail(struct vtn_builder * b,nir_intrinsic_op op,bool load,nir_ssa_def * index,nir_ssa_def * offset,unsigned access_offset,unsigned access_size,struct vtn_ssa_value ** inout,const struct glsl_type * type)465 _vtn_load_store_tail(struct vtn_builder *b, nir_intrinsic_op op, bool load,
466 nir_ssa_def *index, nir_ssa_def *offset,
467 unsigned access_offset, unsigned access_size,
468 struct vtn_ssa_value **inout, const struct glsl_type *type)
469 {
470 nir_intrinsic_instr *instr = nir_intrinsic_instr_create(b->nb.shader, op);
471 instr->num_components = glsl_get_vector_elements(type);
472
473 int src = 0;
474 if (!load) {
475 nir_intrinsic_set_write_mask(instr, (1 << instr->num_components) - 1);
476 instr->src[src++] = nir_src_for_ssa((*inout)->def);
477 }
478
479 if (op == nir_intrinsic_load_push_constant) {
480 assert(access_offset % 4 == 0);
481
482 nir_intrinsic_set_base(instr, access_offset);
483 nir_intrinsic_set_range(instr, access_size);
484 }
485
486 if (index)
487 instr->src[src++] = nir_src_for_ssa(index);
488
489 if (op == nir_intrinsic_load_push_constant) {
490 /* We need to subtract the offset from where the intrinsic will load the
491 * data. */
492 instr->src[src++] =
493 nir_src_for_ssa(nir_isub(&b->nb, offset,
494 nir_imm_int(&b->nb, access_offset)));
495 } else {
496 instr->src[src++] = nir_src_for_ssa(offset);
497 }
498
499 if (load) {
500 nir_ssa_dest_init(&instr->instr, &instr->dest,
501 instr->num_components,
502 glsl_get_bit_size(type), NULL);
503 (*inout)->def = &instr->dest.ssa;
504 }
505
506 nir_builder_instr_insert(&b->nb, &instr->instr);
507
508 if (load && glsl_get_base_type(type) == GLSL_TYPE_BOOL)
509 (*inout)->def = nir_ine(&b->nb, (*inout)->def, nir_imm_int(&b->nb, 0));
510 }
511
512 static void
_vtn_block_load_store(struct vtn_builder * b,nir_intrinsic_op op,bool load,nir_ssa_def * index,nir_ssa_def * offset,unsigned access_offset,unsigned access_size,struct vtn_access_chain * chain,unsigned chain_idx,struct vtn_type * type,struct vtn_ssa_value ** inout)513 _vtn_block_load_store(struct vtn_builder *b, nir_intrinsic_op op, bool load,
514 nir_ssa_def *index, nir_ssa_def *offset,
515 unsigned access_offset, unsigned access_size,
516 struct vtn_access_chain *chain, unsigned chain_idx,
517 struct vtn_type *type, struct vtn_ssa_value **inout)
518 {
519 if (chain && chain_idx >= chain->length)
520 chain = NULL;
521
522 if (load && chain == NULL && *inout == NULL)
523 *inout = vtn_create_ssa_value(b, type->type);
524
525 enum glsl_base_type base_type = glsl_get_base_type(type->type);
526 switch (base_type) {
527 case GLSL_TYPE_UINT:
528 case GLSL_TYPE_INT:
529 case GLSL_TYPE_FLOAT:
530 case GLSL_TYPE_DOUBLE:
531 case GLSL_TYPE_BOOL:
532 /* This is where things get interesting. At this point, we've hit
533 * a vector, a scalar, or a matrix.
534 */
535 if (glsl_type_is_matrix(type->type)) {
536 if (chain == NULL) {
537 /* Loading the whole matrix */
538 struct vtn_ssa_value *transpose;
539 unsigned num_ops, vec_width;
540 if (type->row_major) {
541 num_ops = glsl_get_vector_elements(type->type);
542 vec_width = glsl_get_matrix_columns(type->type);
543 if (load) {
544 const struct glsl_type *transpose_type =
545 glsl_matrix_type(base_type, vec_width, num_ops);
546 *inout = vtn_create_ssa_value(b, transpose_type);
547 } else {
548 transpose = vtn_ssa_transpose(b, *inout);
549 inout = &transpose;
550 }
551 } else {
552 num_ops = glsl_get_matrix_columns(type->type);
553 vec_width = glsl_get_vector_elements(type->type);
554 }
555
556 for (unsigned i = 0; i < num_ops; i++) {
557 nir_ssa_def *elem_offset =
558 nir_iadd(&b->nb, offset,
559 nir_imm_int(&b->nb, i * type->stride));
560 _vtn_load_store_tail(b, op, load, index, elem_offset,
561 access_offset, access_size,
562 &(*inout)->elems[i],
563 glsl_vector_type(base_type, vec_width));
564 }
565
566 if (load && type->row_major)
567 *inout = vtn_ssa_transpose(b, *inout);
568 } else if (type->row_major) {
569 /* Row-major but with an access chiain. */
570 nir_ssa_def *col_offset =
571 vtn_access_link_as_ssa(b, chain->link[chain_idx],
572 type->array_element->stride);
573 offset = nir_iadd(&b->nb, offset, col_offset);
574
575 if (chain_idx + 1 < chain->length) {
576 /* Picking off a single element */
577 nir_ssa_def *row_offset =
578 vtn_access_link_as_ssa(b, chain->link[chain_idx + 1],
579 type->stride);
580 offset = nir_iadd(&b->nb, offset, row_offset);
581 if (load)
582 *inout = vtn_create_ssa_value(b, glsl_scalar_type(base_type));
583 _vtn_load_store_tail(b, op, load, index, offset,
584 access_offset, access_size,
585 inout, glsl_scalar_type(base_type));
586 } else {
587 /* Grabbing a column; picking one element off each row */
588 unsigned num_comps = glsl_get_vector_elements(type->type);
589 const struct glsl_type *column_type =
590 glsl_get_column_type(type->type);
591
592 nir_ssa_def *comps[4];
593 for (unsigned i = 0; i < num_comps; i++) {
594 nir_ssa_def *elem_offset =
595 nir_iadd(&b->nb, offset,
596 nir_imm_int(&b->nb, i * type->stride));
597
598 struct vtn_ssa_value *comp, temp_val;
599 if (!load) {
600 temp_val.def = nir_channel(&b->nb, (*inout)->def, i);
601 temp_val.type = glsl_scalar_type(base_type);
602 }
603 comp = &temp_val;
604 _vtn_load_store_tail(b, op, load, index, elem_offset,
605 access_offset, access_size,
606 &comp, glsl_scalar_type(base_type));
607 comps[i] = comp->def;
608 }
609
610 if (load) {
611 if (*inout == NULL)
612 *inout = vtn_create_ssa_value(b, column_type);
613
614 (*inout)->def = nir_vec(&b->nb, comps, num_comps);
615 }
616 }
617 } else {
618 /* Column-major with a deref. Fall through to array case. */
619 nir_ssa_def *col_offset =
620 vtn_access_link_as_ssa(b, chain->link[chain_idx], type->stride);
621 offset = nir_iadd(&b->nb, offset, col_offset);
622
623 _vtn_block_load_store(b, op, load, index, offset,
624 access_offset, access_size,
625 chain, chain_idx + 1,
626 type->array_element, inout);
627 }
628 } else if (chain == NULL) {
629 /* Single whole vector */
630 assert(glsl_type_is_vector_or_scalar(type->type));
631 _vtn_load_store_tail(b, op, load, index, offset,
632 access_offset, access_size,
633 inout, type->type);
634 } else {
635 /* Single component of a vector. Fall through to array case. */
636 nir_ssa_def *elem_offset =
637 vtn_access_link_as_ssa(b, chain->link[chain_idx], type->stride);
638 offset = nir_iadd(&b->nb, offset, elem_offset);
639
640 _vtn_block_load_store(b, op, load, index, offset,
641 access_offset, access_size,
642 NULL, 0,
643 type->array_element, inout);
644 }
645 return;
646
647 case GLSL_TYPE_ARRAY: {
648 unsigned elems = glsl_get_length(type->type);
649 for (unsigned i = 0; i < elems; i++) {
650 nir_ssa_def *elem_off =
651 nir_iadd(&b->nb, offset, nir_imm_int(&b->nb, i * type->stride));
652 _vtn_block_load_store(b, op, load, index, elem_off,
653 access_offset, access_size,
654 NULL, 0,
655 type->array_element, &(*inout)->elems[i]);
656 }
657 return;
658 }
659
660 case GLSL_TYPE_STRUCT: {
661 unsigned elems = glsl_get_length(type->type);
662 for (unsigned i = 0; i < elems; i++) {
663 nir_ssa_def *elem_off =
664 nir_iadd(&b->nb, offset, nir_imm_int(&b->nb, type->offsets[i]));
665 _vtn_block_load_store(b, op, load, index, elem_off,
666 access_offset, access_size,
667 NULL, 0,
668 type->members[i], &(*inout)->elems[i]);
669 }
670 return;
671 }
672
673 default:
674 unreachable("Invalid block member type");
675 }
676 }
677
678 static struct vtn_ssa_value *
vtn_block_load(struct vtn_builder * b,struct vtn_access_chain * src)679 vtn_block_load(struct vtn_builder *b, struct vtn_access_chain *src)
680 {
681 nir_intrinsic_op op;
682 unsigned access_offset = 0, access_size = 0;
683 switch (src->var->mode) {
684 case vtn_variable_mode_ubo:
685 op = nir_intrinsic_load_ubo;
686 break;
687 case vtn_variable_mode_ssbo:
688 op = nir_intrinsic_load_ssbo;
689 break;
690 case vtn_variable_mode_push_constant:
691 op = nir_intrinsic_load_push_constant;
692 vtn_access_chain_get_offset_size(src, &access_offset, &access_size);
693 break;
694 default:
695 assert(!"Invalid block variable mode");
696 }
697
698 nir_ssa_def *offset, *index = NULL;
699 struct vtn_type *type;
700 unsigned chain_idx;
701 offset = vtn_access_chain_to_offset(b, src, &index, &type, &chain_idx, true);
702
703 struct vtn_ssa_value *value = NULL;
704 _vtn_block_load_store(b, op, true, index, offset,
705 access_offset, access_size,
706 src, chain_idx, type, &value);
707 return value;
708 }
709
710 static void
vtn_block_store(struct vtn_builder * b,struct vtn_ssa_value * src,struct vtn_access_chain * dst)711 vtn_block_store(struct vtn_builder *b, struct vtn_ssa_value *src,
712 struct vtn_access_chain *dst)
713 {
714 nir_ssa_def *offset, *index = NULL;
715 struct vtn_type *type;
716 unsigned chain_idx;
717 offset = vtn_access_chain_to_offset(b, dst, &index, &type, &chain_idx, true);
718
719 _vtn_block_load_store(b, nir_intrinsic_store_ssbo, false, index, offset,
720 0, 0, dst, chain_idx, type, &src);
721 }
722
723 static bool
vtn_variable_is_external_block(struct vtn_variable * var)724 vtn_variable_is_external_block(struct vtn_variable *var)
725 {
726 return var->mode == vtn_variable_mode_ssbo ||
727 var->mode == vtn_variable_mode_ubo ||
728 var->mode == vtn_variable_mode_push_constant;
729 }
730
731 static void
_vtn_variable_load_store(struct vtn_builder * b,bool load,struct vtn_access_chain * chain,struct vtn_type * tail_type,struct vtn_ssa_value ** inout)732 _vtn_variable_load_store(struct vtn_builder *b, bool load,
733 struct vtn_access_chain *chain,
734 struct vtn_type *tail_type,
735 struct vtn_ssa_value **inout)
736 {
737 enum glsl_base_type base_type = glsl_get_base_type(tail_type->type);
738 switch (base_type) {
739 case GLSL_TYPE_UINT:
740 case GLSL_TYPE_INT:
741 case GLSL_TYPE_FLOAT:
742 case GLSL_TYPE_BOOL:
743 case GLSL_TYPE_DOUBLE:
744 /* At this point, we have a scalar, vector, or matrix so we know that
745 * there cannot be any structure splitting still in the way. By
746 * stopping at the matrix level rather than the vector level, we
747 * ensure that matrices get loaded in the optimal way even if they
748 * are storred row-major in a UBO.
749 */
750 if (load) {
751 *inout = vtn_local_load(b, vtn_access_chain_to_deref(b, chain));
752 } else {
753 vtn_local_store(b, *inout, vtn_access_chain_to_deref(b, chain));
754 }
755 return;
756
757 case GLSL_TYPE_ARRAY:
758 case GLSL_TYPE_STRUCT: {
759 struct vtn_access_chain *new_chain =
760 vtn_access_chain_extend(b, chain, 1);
761 new_chain->link[chain->length].mode = vtn_access_mode_literal;
762 unsigned elems = glsl_get_length(tail_type->type);
763 if (load) {
764 assert(*inout == NULL);
765 *inout = rzalloc(b, struct vtn_ssa_value);
766 (*inout)->type = tail_type->type;
767 (*inout)->elems = rzalloc_array(b, struct vtn_ssa_value *, elems);
768 }
769 for (unsigned i = 0; i < elems; i++) {
770 new_chain->link[chain->length].id = i;
771 struct vtn_type *elem_type = base_type == GLSL_TYPE_ARRAY ?
772 tail_type->array_element : tail_type->members[i];
773 _vtn_variable_load_store(b, load, new_chain, elem_type,
774 &(*inout)->elems[i]);
775 }
776 return;
777 }
778
779 default:
780 unreachable("Invalid access chain type");
781 }
782 }
783
784 struct vtn_ssa_value *
vtn_variable_load(struct vtn_builder * b,struct vtn_access_chain * src)785 vtn_variable_load(struct vtn_builder *b, struct vtn_access_chain *src)
786 {
787 if (vtn_variable_is_external_block(src->var)) {
788 return vtn_block_load(b, src);
789 } else {
790 struct vtn_type *tail_type = vtn_access_chain_tail_type(b, src);
791 struct vtn_ssa_value *val = NULL;
792 _vtn_variable_load_store(b, true, src, tail_type, &val);
793 return val;
794 }
795 }
796
797 void
vtn_variable_store(struct vtn_builder * b,struct vtn_ssa_value * src,struct vtn_access_chain * dest)798 vtn_variable_store(struct vtn_builder *b, struct vtn_ssa_value *src,
799 struct vtn_access_chain *dest)
800 {
801 if (vtn_variable_is_external_block(dest->var)) {
802 assert(dest->var->mode == vtn_variable_mode_ssbo);
803 vtn_block_store(b, src, dest);
804 } else {
805 struct vtn_type *tail_type = vtn_access_chain_tail_type(b, dest);
806 _vtn_variable_load_store(b, false, dest, tail_type, &src);
807 }
808 }
809
810 static void
_vtn_variable_copy(struct vtn_builder * b,struct vtn_access_chain * dest,struct vtn_access_chain * src,struct vtn_type * tail_type)811 _vtn_variable_copy(struct vtn_builder *b, struct vtn_access_chain *dest,
812 struct vtn_access_chain *src, struct vtn_type *tail_type)
813 {
814 enum glsl_base_type base_type = glsl_get_base_type(tail_type->type);
815 switch (base_type) {
816 case GLSL_TYPE_UINT:
817 case GLSL_TYPE_INT:
818 case GLSL_TYPE_FLOAT:
819 case GLSL_TYPE_DOUBLE:
820 case GLSL_TYPE_BOOL:
821 /* At this point, we have a scalar, vector, or matrix so we know that
822 * there cannot be any structure splitting still in the way. By
823 * stopping at the matrix level rather than the vector level, we
824 * ensure that matrices get loaded in the optimal way even if they
825 * are storred row-major in a UBO.
826 */
827 vtn_variable_store(b, vtn_variable_load(b, src), dest);
828 return;
829
830 case GLSL_TYPE_ARRAY:
831 case GLSL_TYPE_STRUCT: {
832 struct vtn_access_chain *new_src, *new_dest;
833 new_src = vtn_access_chain_extend(b, src, 1);
834 new_dest = vtn_access_chain_extend(b, dest, 1);
835 new_src->link[src->length].mode = vtn_access_mode_literal;
836 new_dest->link[dest->length].mode = vtn_access_mode_literal;
837 unsigned elems = glsl_get_length(tail_type->type);
838 for (unsigned i = 0; i < elems; i++) {
839 new_src->link[src->length].id = i;
840 new_dest->link[dest->length].id = i;
841 struct vtn_type *elem_type = base_type == GLSL_TYPE_ARRAY ?
842 tail_type->array_element : tail_type->members[i];
843 _vtn_variable_copy(b, new_dest, new_src, elem_type);
844 }
845 return;
846 }
847
848 default:
849 unreachable("Invalid access chain type");
850 }
851 }
852
853 static void
vtn_variable_copy(struct vtn_builder * b,struct vtn_access_chain * dest,struct vtn_access_chain * src)854 vtn_variable_copy(struct vtn_builder *b, struct vtn_access_chain *dest,
855 struct vtn_access_chain *src)
856 {
857 struct vtn_type *tail_type = vtn_access_chain_tail_type(b, src);
858 assert(vtn_access_chain_tail_type(b, dest)->type == tail_type->type);
859
860 /* TODO: At some point, we should add a special-case for when we can
861 * just emit a copy_var intrinsic.
862 */
863 _vtn_variable_copy(b, dest, src, tail_type);
864 }
865
866 static void
set_mode_system_value(nir_variable_mode * mode)867 set_mode_system_value(nir_variable_mode *mode)
868 {
869 assert(*mode == nir_var_system_value || *mode == nir_var_shader_in);
870 *mode = nir_var_system_value;
871 }
872
873 static void
vtn_get_builtin_location(struct vtn_builder * b,SpvBuiltIn builtin,int * location,nir_variable_mode * mode)874 vtn_get_builtin_location(struct vtn_builder *b,
875 SpvBuiltIn builtin, int *location,
876 nir_variable_mode *mode)
877 {
878 switch (builtin) {
879 case SpvBuiltInPosition:
880 *location = VARYING_SLOT_POS;
881 break;
882 case SpvBuiltInPointSize:
883 *location = VARYING_SLOT_PSIZ;
884 break;
885 case SpvBuiltInClipDistance:
886 *location = VARYING_SLOT_CLIP_DIST0; /* XXX CLIP_DIST1? */
887 break;
888 case SpvBuiltInCullDistance:
889 *location = VARYING_SLOT_CULL_DIST0;
890 break;
891 case SpvBuiltInVertexIndex:
892 *location = SYSTEM_VALUE_VERTEX_ID;
893 set_mode_system_value(mode);
894 break;
895 case SpvBuiltInVertexId:
896 /* Vulkan defines VertexID to be zero-based and reserves the new
897 * builtin keyword VertexIndex to indicate the non-zero-based value.
898 */
899 *location = SYSTEM_VALUE_VERTEX_ID_ZERO_BASE;
900 set_mode_system_value(mode);
901 break;
902 case SpvBuiltInInstanceIndex:
903 *location = SYSTEM_VALUE_INSTANCE_INDEX;
904 set_mode_system_value(mode);
905 break;
906 case SpvBuiltInInstanceId:
907 *location = SYSTEM_VALUE_INSTANCE_ID;
908 set_mode_system_value(mode);
909 break;
910 case SpvBuiltInPrimitiveId:
911 if (b->shader->stage == MESA_SHADER_FRAGMENT) {
912 assert(*mode == nir_var_shader_in);
913 *location = VARYING_SLOT_PRIMITIVE_ID;
914 } else if (*mode == nir_var_shader_out) {
915 *location = VARYING_SLOT_PRIMITIVE_ID;
916 } else {
917 *location = SYSTEM_VALUE_PRIMITIVE_ID;
918 set_mode_system_value(mode);
919 }
920 break;
921 case SpvBuiltInInvocationId:
922 *location = SYSTEM_VALUE_INVOCATION_ID;
923 set_mode_system_value(mode);
924 break;
925 case SpvBuiltInLayer:
926 *location = VARYING_SLOT_LAYER;
927 if (b->shader->stage == MESA_SHADER_FRAGMENT)
928 *mode = nir_var_shader_in;
929 else if (b->shader->stage == MESA_SHADER_GEOMETRY)
930 *mode = nir_var_shader_out;
931 else
932 unreachable("invalid stage for SpvBuiltInLayer");
933 break;
934 case SpvBuiltInViewportIndex:
935 *location = VARYING_SLOT_VIEWPORT;
936 if (b->shader->stage == MESA_SHADER_GEOMETRY)
937 *mode = nir_var_shader_out;
938 else if (b->shader->stage == MESA_SHADER_FRAGMENT)
939 *mode = nir_var_shader_in;
940 else
941 unreachable("invalid stage for SpvBuiltInViewportIndex");
942 break;
943 case SpvBuiltInTessLevelOuter:
944 *location = VARYING_SLOT_TESS_LEVEL_OUTER;
945 break;
946 case SpvBuiltInTessLevelInner:
947 *location = VARYING_SLOT_TESS_LEVEL_INNER;
948 break;
949 case SpvBuiltInTessCoord:
950 *location = SYSTEM_VALUE_TESS_COORD;
951 set_mode_system_value(mode);
952 break;
953 case SpvBuiltInPatchVertices:
954 *location = SYSTEM_VALUE_VERTICES_IN;
955 set_mode_system_value(mode);
956 break;
957 case SpvBuiltInFragCoord:
958 *location = VARYING_SLOT_POS;
959 assert(*mode == nir_var_shader_in);
960 break;
961 case SpvBuiltInPointCoord:
962 *location = VARYING_SLOT_PNTC;
963 assert(*mode == nir_var_shader_in);
964 break;
965 case SpvBuiltInFrontFacing:
966 *location = SYSTEM_VALUE_FRONT_FACE;
967 set_mode_system_value(mode);
968 break;
969 case SpvBuiltInSampleId:
970 *location = SYSTEM_VALUE_SAMPLE_ID;
971 set_mode_system_value(mode);
972 break;
973 case SpvBuiltInSamplePosition:
974 *location = SYSTEM_VALUE_SAMPLE_POS;
975 set_mode_system_value(mode);
976 break;
977 case SpvBuiltInSampleMask:
978 *location = SYSTEM_VALUE_SAMPLE_MASK_IN; /* XXX out? */
979 set_mode_system_value(mode);
980 break;
981 case SpvBuiltInFragDepth:
982 *location = FRAG_RESULT_DEPTH;
983 assert(*mode == nir_var_shader_out);
984 break;
985 case SpvBuiltInNumWorkgroups:
986 *location = SYSTEM_VALUE_NUM_WORK_GROUPS;
987 set_mode_system_value(mode);
988 break;
989 case SpvBuiltInWorkgroupSize:
990 /* This should already be handled */
991 unreachable("unsupported builtin");
992 break;
993 case SpvBuiltInWorkgroupId:
994 *location = SYSTEM_VALUE_WORK_GROUP_ID;
995 set_mode_system_value(mode);
996 break;
997 case SpvBuiltInLocalInvocationId:
998 *location = SYSTEM_VALUE_LOCAL_INVOCATION_ID;
999 set_mode_system_value(mode);
1000 break;
1001 case SpvBuiltInLocalInvocationIndex:
1002 *location = SYSTEM_VALUE_LOCAL_INVOCATION_INDEX;
1003 set_mode_system_value(mode);
1004 break;
1005 case SpvBuiltInGlobalInvocationId:
1006 *location = SYSTEM_VALUE_GLOBAL_INVOCATION_ID;
1007 set_mode_system_value(mode);
1008 break;
1009 case SpvBuiltInHelperInvocation:
1010 default:
1011 unreachable("unsupported builtin");
1012 }
1013 }
1014
1015 static void
apply_var_decoration(struct vtn_builder * b,nir_variable * nir_var,const struct vtn_decoration * dec)1016 apply_var_decoration(struct vtn_builder *b, nir_variable *nir_var,
1017 const struct vtn_decoration *dec)
1018 {
1019 switch (dec->decoration) {
1020 case SpvDecorationRelaxedPrecision:
1021 break; /* FIXME: Do nothing with this for now. */
1022 case SpvDecorationNoPerspective:
1023 nir_var->data.interpolation = INTERP_MODE_NOPERSPECTIVE;
1024 break;
1025 case SpvDecorationFlat:
1026 nir_var->data.interpolation = INTERP_MODE_FLAT;
1027 break;
1028 case SpvDecorationCentroid:
1029 nir_var->data.centroid = true;
1030 break;
1031 case SpvDecorationSample:
1032 nir_var->data.sample = true;
1033 break;
1034 case SpvDecorationInvariant:
1035 nir_var->data.invariant = true;
1036 break;
1037 case SpvDecorationConstant:
1038 assert(nir_var->constant_initializer != NULL);
1039 nir_var->data.read_only = true;
1040 break;
1041 case SpvDecorationNonWritable:
1042 nir_var->data.read_only = true;
1043 break;
1044 case SpvDecorationComponent:
1045 nir_var->data.location_frac = dec->literals[0];
1046 break;
1047 case SpvDecorationIndex:
1048 nir_var->data.index = dec->literals[0];
1049 break;
1050 case SpvDecorationBuiltIn: {
1051 SpvBuiltIn builtin = dec->literals[0];
1052
1053 if (builtin == SpvBuiltInWorkgroupSize) {
1054 /* This shouldn't be a builtin. It's actually a constant. */
1055 nir_var->data.mode = nir_var_global;
1056 nir_var->data.read_only = true;
1057
1058 nir_constant *c = rzalloc(nir_var, nir_constant);
1059 c->values[0].u32[0] = b->shader->info->cs.local_size[0];
1060 c->values[0].u32[1] = b->shader->info->cs.local_size[1];
1061 c->values[0].u32[2] = b->shader->info->cs.local_size[2];
1062 nir_var->constant_initializer = c;
1063 break;
1064 }
1065
1066 nir_variable_mode mode = nir_var->data.mode;
1067 vtn_get_builtin_location(b, builtin, &nir_var->data.location, &mode);
1068 nir_var->data.mode = mode;
1069
1070 switch (builtin) {
1071 case SpvBuiltInTessLevelOuter:
1072 case SpvBuiltInTessLevelInner:
1073 nir_var->data.compact = true;
1074 break;
1075 case SpvBuiltInSamplePosition:
1076 nir_var->data.origin_upper_left = b->origin_upper_left;
1077 /* fallthrough */
1078 case SpvBuiltInFragCoord:
1079 nir_var->data.pixel_center_integer = b->pixel_center_integer;
1080 break;
1081 default:
1082 break;
1083 }
1084 }
1085
1086 case SpvDecorationSpecId:
1087 case SpvDecorationRowMajor:
1088 case SpvDecorationColMajor:
1089 case SpvDecorationMatrixStride:
1090 case SpvDecorationRestrict:
1091 case SpvDecorationAliased:
1092 case SpvDecorationVolatile:
1093 case SpvDecorationCoherent:
1094 case SpvDecorationNonReadable:
1095 case SpvDecorationUniform:
1096 case SpvDecorationStream:
1097 case SpvDecorationOffset:
1098 case SpvDecorationLinkageAttributes:
1099 break; /* Do nothing with these here */
1100
1101 case SpvDecorationPatch:
1102 nir_var->data.patch = true;
1103 break;
1104
1105 case SpvDecorationLocation:
1106 unreachable("Handled above");
1107
1108 case SpvDecorationBlock:
1109 case SpvDecorationBufferBlock:
1110 case SpvDecorationArrayStride:
1111 case SpvDecorationGLSLShared:
1112 case SpvDecorationGLSLPacked:
1113 break; /* These can apply to a type but we don't care about them */
1114
1115 case SpvDecorationBinding:
1116 case SpvDecorationDescriptorSet:
1117 case SpvDecorationNoContraction:
1118 case SpvDecorationInputAttachmentIndex:
1119 vtn_warn("Decoration not allowed for variable or structure member: %s",
1120 spirv_decoration_to_string(dec->decoration));
1121 break;
1122
1123 case SpvDecorationXfbBuffer:
1124 case SpvDecorationXfbStride:
1125 vtn_warn("Vulkan does not have transform feedback: %s",
1126 spirv_decoration_to_string(dec->decoration));
1127 break;
1128
1129 case SpvDecorationCPacked:
1130 case SpvDecorationSaturatedConversion:
1131 case SpvDecorationFuncParamAttr:
1132 case SpvDecorationFPRoundingMode:
1133 case SpvDecorationFPFastMathMode:
1134 case SpvDecorationAlignment:
1135 vtn_warn("Decoraiton only allowed for CL-style kernels: %s",
1136 spirv_decoration_to_string(dec->decoration));
1137 break;
1138 }
1139 }
1140
1141 static void
var_is_patch_cb(struct vtn_builder * b,struct vtn_value * val,int member,const struct vtn_decoration * dec,void * out_is_patch)1142 var_is_patch_cb(struct vtn_builder *b, struct vtn_value *val, int member,
1143 const struct vtn_decoration *dec, void *out_is_patch)
1144 {
1145 if (dec->decoration == SpvDecorationPatch) {
1146 *((bool *) out_is_patch) = true;
1147 }
1148 }
1149
1150 static void
var_decoration_cb(struct vtn_builder * b,struct vtn_value * val,int member,const struct vtn_decoration * dec,void * void_var)1151 var_decoration_cb(struct vtn_builder *b, struct vtn_value *val, int member,
1152 const struct vtn_decoration *dec, void *void_var)
1153 {
1154 struct vtn_variable *vtn_var = void_var;
1155
1156 /* Handle decorations that apply to a vtn_variable as a whole */
1157 switch (dec->decoration) {
1158 case SpvDecorationBinding:
1159 vtn_var->binding = dec->literals[0];
1160 return;
1161 case SpvDecorationDescriptorSet:
1162 vtn_var->descriptor_set = dec->literals[0];
1163 return;
1164 case SpvDecorationInputAttachmentIndex:
1165 vtn_var->input_attachment_index = dec->literals[0];
1166 return;
1167 case SpvDecorationPatch:
1168 vtn_var->patch = true;
1169 break;
1170 default:
1171 break;
1172 }
1173
1174 if (val->value_type == vtn_value_type_access_chain) {
1175 assert(val->access_chain->length == 0);
1176 assert(val->access_chain->var == void_var);
1177 assert(member == -1);
1178 } else {
1179 assert(val->value_type == vtn_value_type_type);
1180 }
1181
1182 /* Location is odd. If applied to a split structure, we have to walk the
1183 * whole thing and accumulate the location. It's easier to handle as a
1184 * special case.
1185 */
1186 if (dec->decoration == SpvDecorationLocation) {
1187 unsigned location = dec->literals[0];
1188 bool is_vertex_input;
1189 if (b->shader->stage == MESA_SHADER_FRAGMENT &&
1190 vtn_var->mode == vtn_variable_mode_output) {
1191 is_vertex_input = false;
1192 location += FRAG_RESULT_DATA0;
1193 } else if (b->shader->stage == MESA_SHADER_VERTEX &&
1194 vtn_var->mode == vtn_variable_mode_input) {
1195 is_vertex_input = true;
1196 location += VERT_ATTRIB_GENERIC0;
1197 } else if (vtn_var->mode == vtn_variable_mode_input ||
1198 vtn_var->mode == vtn_variable_mode_output) {
1199 is_vertex_input = false;
1200 location += vtn_var->patch ? VARYING_SLOT_PATCH0 : VARYING_SLOT_VAR0;
1201 } else {
1202 vtn_warn("Location must be on input or output variable");
1203 return;
1204 }
1205
1206 if (vtn_var->var) {
1207 /* This handles the member and lone variable cases */
1208 vtn_var->var->data.location = location;
1209 } else {
1210 /* This handles the structure member case */
1211 assert(vtn_var->members);
1212 unsigned length =
1213 glsl_get_length(glsl_without_array(vtn_var->type->type));
1214 for (unsigned i = 0; i < length; i++) {
1215 vtn_var->members[i]->data.location = location;
1216 location +=
1217 glsl_count_attribute_slots(vtn_var->members[i]->interface_type,
1218 is_vertex_input);
1219 }
1220 }
1221 return;
1222 } else {
1223 if (vtn_var->var) {
1224 assert(member <= 0);
1225 apply_var_decoration(b, vtn_var->var, dec);
1226 } else if (vtn_var->members) {
1227 if (member >= 0) {
1228 assert(vtn_var->members);
1229 apply_var_decoration(b, vtn_var->members[member], dec);
1230 } else {
1231 unsigned length =
1232 glsl_get_length(glsl_without_array(vtn_var->type->type));
1233 for (unsigned i = 0; i < length; i++)
1234 apply_var_decoration(b, vtn_var->members[i], dec);
1235 }
1236 } else {
1237 /* A few variables, those with external storage, have no actual
1238 * nir_variables associated with them. Fortunately, all decorations
1239 * we care about for those variables are on the type only.
1240 */
1241 assert(vtn_var->mode == vtn_variable_mode_ubo ||
1242 vtn_var->mode == vtn_variable_mode_ssbo ||
1243 vtn_var->mode == vtn_variable_mode_push_constant);
1244 }
1245 }
1246 }
1247
1248 static bool
is_per_vertex_inout(const struct vtn_variable * var,gl_shader_stage stage)1249 is_per_vertex_inout(const struct vtn_variable *var, gl_shader_stage stage)
1250 {
1251 if (var->patch || !glsl_type_is_array(var->type->type))
1252 return false;
1253
1254 if (var->mode == vtn_variable_mode_input) {
1255 return stage == MESA_SHADER_TESS_CTRL ||
1256 stage == MESA_SHADER_TESS_EVAL ||
1257 stage == MESA_SHADER_GEOMETRY;
1258 }
1259
1260 if (var->mode == vtn_variable_mode_output)
1261 return stage == MESA_SHADER_TESS_CTRL;
1262
1263 return false;
1264 }
1265
1266 void
vtn_handle_variables(struct vtn_builder * b,SpvOp opcode,const uint32_t * w,unsigned count)1267 vtn_handle_variables(struct vtn_builder *b, SpvOp opcode,
1268 const uint32_t *w, unsigned count)
1269 {
1270 switch (opcode) {
1271 case SpvOpUndef: {
1272 struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_undef);
1273 val->type = vtn_value(b, w[1], vtn_value_type_type)->type;
1274 break;
1275 }
1276
1277 case SpvOpVariable: {
1278 struct vtn_variable *var = rzalloc(b, struct vtn_variable);
1279 var->type = vtn_value(b, w[1], vtn_value_type_type)->type;
1280
1281 var->chain.var = var;
1282 var->chain.length = 0;
1283
1284 struct vtn_value *val =
1285 vtn_push_value(b, w[2], vtn_value_type_access_chain);
1286 val->access_chain = &var->chain;
1287
1288 struct vtn_type *without_array = var->type;
1289 while(glsl_type_is_array(without_array->type))
1290 without_array = without_array->array_element;
1291
1292 nir_variable_mode nir_mode;
1293 switch ((SpvStorageClass)w[3]) {
1294 case SpvStorageClassUniform:
1295 case SpvStorageClassUniformConstant:
1296 if (without_array->block) {
1297 var->mode = vtn_variable_mode_ubo;
1298 b->shader->info->num_ubos++;
1299 } else if (without_array->buffer_block) {
1300 var->mode = vtn_variable_mode_ssbo;
1301 b->shader->info->num_ssbos++;
1302 } else if (glsl_type_is_image(without_array->type)) {
1303 var->mode = vtn_variable_mode_image;
1304 nir_mode = nir_var_uniform;
1305 b->shader->info->num_images++;
1306 } else if (glsl_type_is_sampler(without_array->type)) {
1307 var->mode = vtn_variable_mode_sampler;
1308 nir_mode = nir_var_uniform;
1309 b->shader->info->num_textures++;
1310 } else {
1311 assert(!"Invalid uniform variable type");
1312 }
1313 break;
1314 case SpvStorageClassPushConstant:
1315 var->mode = vtn_variable_mode_push_constant;
1316 assert(b->shader->num_uniforms == 0);
1317 b->shader->num_uniforms = vtn_type_block_size(var->type);
1318 break;
1319 case SpvStorageClassInput:
1320 var->mode = vtn_variable_mode_input;
1321 nir_mode = nir_var_shader_in;
1322 break;
1323 case SpvStorageClassOutput:
1324 var->mode = vtn_variable_mode_output;
1325 nir_mode = nir_var_shader_out;
1326 break;
1327 case SpvStorageClassPrivate:
1328 var->mode = vtn_variable_mode_global;
1329 nir_mode = nir_var_global;
1330 break;
1331 case SpvStorageClassFunction:
1332 var->mode = vtn_variable_mode_local;
1333 nir_mode = nir_var_local;
1334 break;
1335 case SpvStorageClassWorkgroup:
1336 var->mode = vtn_variable_mode_workgroup;
1337 nir_mode = nir_var_shared;
1338 break;
1339 case SpvStorageClassCrossWorkgroup:
1340 case SpvStorageClassGeneric:
1341 case SpvStorageClassAtomicCounter:
1342 default:
1343 unreachable("Unhandled variable storage class");
1344 }
1345
1346 switch (var->mode) {
1347 case vtn_variable_mode_local:
1348 case vtn_variable_mode_global:
1349 case vtn_variable_mode_image:
1350 case vtn_variable_mode_sampler:
1351 case vtn_variable_mode_workgroup:
1352 /* For these, we create the variable normally */
1353 var->var = rzalloc(b->shader, nir_variable);
1354 var->var->name = ralloc_strdup(var->var, val->name);
1355 var->var->type = var->type->type;
1356 var->var->data.mode = nir_mode;
1357
1358 switch (var->mode) {
1359 case vtn_variable_mode_image:
1360 case vtn_variable_mode_sampler:
1361 var->var->interface_type = without_array->type;
1362 break;
1363 default:
1364 var->var->interface_type = NULL;
1365 break;
1366 }
1367 break;
1368
1369 case vtn_variable_mode_input:
1370 case vtn_variable_mode_output: {
1371 /* In order to know whether or not we're a per-vertex inout, we need
1372 * the patch qualifier. This means walking the variable decorations
1373 * early before we actually create any variables. Not a big deal.
1374 *
1375 * GLSLang really likes to place decorations in the most interior
1376 * thing it possibly can. In particular, if you have a struct, it
1377 * will place the patch decorations on the struct members. This
1378 * should be handled by the variable splitting below just fine.
1379 *
1380 * If you have an array-of-struct, things get even more weird as it
1381 * will place the patch decorations on the struct even though it's
1382 * inside an array and some of the members being patch and others not
1383 * makes no sense whatsoever. Since the only sensible thing is for
1384 * it to be all or nothing, we'll call it patch if any of the members
1385 * are declared patch.
1386 */
1387 var->patch = false;
1388 vtn_foreach_decoration(b, val, var_is_patch_cb, &var->patch);
1389 if (glsl_type_is_array(var->type->type) &&
1390 glsl_type_is_struct(without_array->type)) {
1391 vtn_foreach_decoration(b, without_array->val,
1392 var_is_patch_cb, &var->patch);
1393 }
1394
1395 /* For inputs and outputs, we immediately split structures. This
1396 * is for a couple of reasons. For one, builtins may all come in
1397 * a struct and we really want those split out into separate
1398 * variables. For another, interpolation qualifiers can be
1399 * applied to members of the top-level struct ane we need to be
1400 * able to preserve that information.
1401 */
1402
1403 int array_length = -1;
1404 struct vtn_type *interface_type = var->type;
1405 if (is_per_vertex_inout(var, b->shader->stage)) {
1406 /* In Geometry shaders (and some tessellation), inputs come
1407 * in per-vertex arrays. However, some builtins come in
1408 * non-per-vertex, hence the need for the is_array check. In
1409 * any case, there are no non-builtin arrays allowed so this
1410 * check should be sufficient.
1411 */
1412 interface_type = var->type->array_element;
1413 array_length = glsl_get_length(var->type->type);
1414 }
1415
1416 if (glsl_type_is_struct(interface_type->type)) {
1417 /* It's a struct. Split it. */
1418 unsigned num_members = glsl_get_length(interface_type->type);
1419 var->members = ralloc_array(b, nir_variable *, num_members);
1420
1421 for (unsigned i = 0; i < num_members; i++) {
1422 const struct glsl_type *mtype = interface_type->members[i]->type;
1423 if (array_length >= 0)
1424 mtype = glsl_array_type(mtype, array_length);
1425
1426 var->members[i] = rzalloc(b->shader, nir_variable);
1427 var->members[i]->name =
1428 ralloc_asprintf(var->members[i], "%s.%d", val->name, i);
1429 var->members[i]->type = mtype;
1430 var->members[i]->interface_type =
1431 interface_type->members[i]->type;
1432 var->members[i]->data.mode = nir_mode;
1433 var->members[i]->data.patch = var->patch;
1434 }
1435 } else {
1436 var->var = rzalloc(b->shader, nir_variable);
1437 var->var->name = ralloc_strdup(var->var, val->name);
1438 var->var->type = var->type->type;
1439 var->var->interface_type = interface_type->type;
1440 var->var->data.mode = nir_mode;
1441 var->var->data.patch = var->patch;
1442 }
1443
1444 /* For inputs and outputs, we need to grab locations and builtin
1445 * information from the interface type.
1446 */
1447 vtn_foreach_decoration(b, interface_type->val, var_decoration_cb, var);
1448 break;
1449
1450 case vtn_variable_mode_param:
1451 unreachable("Not created through OpVariable");
1452 }
1453
1454 case vtn_variable_mode_ubo:
1455 case vtn_variable_mode_ssbo:
1456 case vtn_variable_mode_push_constant:
1457 /* These don't need actual variables. */
1458 break;
1459 }
1460
1461 if (count > 4) {
1462 assert(count == 5);
1463 nir_constant *constant =
1464 vtn_value(b, w[4], vtn_value_type_constant)->constant;
1465 var->var->constant_initializer =
1466 nir_constant_clone(constant, var->var);
1467 }
1468
1469 vtn_foreach_decoration(b, val, var_decoration_cb, var);
1470
1471 if (var->mode == vtn_variable_mode_image ||
1472 var->mode == vtn_variable_mode_sampler) {
1473 /* XXX: We still need the binding information in the nir_variable
1474 * for these. We should fix that.
1475 */
1476 var->var->data.binding = var->binding;
1477 var->var->data.descriptor_set = var->descriptor_set;
1478 var->var->data.index = var->input_attachment_index;
1479
1480 if (var->mode == vtn_variable_mode_image)
1481 var->var->data.image.format = without_array->image_format;
1482 }
1483
1484 if (var->mode == vtn_variable_mode_local) {
1485 assert(var->members == NULL && var->var != NULL);
1486 nir_function_impl_add_variable(b->impl, var->var);
1487 } else if (var->var) {
1488 nir_shader_add_variable(b->shader, var->var);
1489 } else if (var->members) {
1490 unsigned count = glsl_get_length(without_array->type);
1491 for (unsigned i = 0; i < count; i++) {
1492 assert(var->members[i]->data.mode != nir_var_local);
1493 nir_shader_add_variable(b->shader, var->members[i]);
1494 }
1495 } else {
1496 assert(var->mode == vtn_variable_mode_ubo ||
1497 var->mode == vtn_variable_mode_ssbo ||
1498 var->mode == vtn_variable_mode_push_constant);
1499 }
1500 break;
1501 }
1502
1503 case SpvOpAccessChain:
1504 case SpvOpInBoundsAccessChain: {
1505 struct vtn_access_chain *base, *chain;
1506 struct vtn_value *base_val = vtn_untyped_value(b, w[3]);
1507 if (base_val->value_type == vtn_value_type_sampled_image) {
1508 /* This is rather insane. SPIR-V allows you to use OpSampledImage
1509 * to combine an array of images with a single sampler to get an
1510 * array of sampled images that all share the same sampler.
1511 * Fortunately, this means that we can more-or-less ignore the
1512 * sampler when crawling the access chain, but it does leave us
1513 * with this rather awkward little special-case.
1514 */
1515 base = base_val->sampled_image->image;
1516 } else {
1517 assert(base_val->value_type == vtn_value_type_access_chain);
1518 base = base_val->access_chain;
1519 }
1520
1521 chain = vtn_access_chain_extend(b, base, count - 4);
1522
1523 unsigned idx = base->length;
1524 for (int i = 4; i < count; i++) {
1525 struct vtn_value *link_val = vtn_untyped_value(b, w[i]);
1526 if (link_val->value_type == vtn_value_type_constant) {
1527 chain->link[idx].mode = vtn_access_mode_literal;
1528 chain->link[idx].id = link_val->constant->values[0].u32[0];
1529 } else {
1530 chain->link[idx].mode = vtn_access_mode_id;
1531 chain->link[idx].id = w[i];
1532 }
1533 idx++;
1534 }
1535
1536 if (base_val->value_type == vtn_value_type_sampled_image) {
1537 struct vtn_value *val =
1538 vtn_push_value(b, w[2], vtn_value_type_sampled_image);
1539 val->sampled_image = ralloc(b, struct vtn_sampled_image);
1540 val->sampled_image->image = chain;
1541 val->sampled_image->sampler = base_val->sampled_image->sampler;
1542 } else {
1543 struct vtn_value *val =
1544 vtn_push_value(b, w[2], vtn_value_type_access_chain);
1545 val->access_chain = chain;
1546 }
1547 break;
1548 }
1549
1550 case SpvOpCopyMemory: {
1551 struct vtn_value *dest = vtn_value(b, w[1], vtn_value_type_access_chain);
1552 struct vtn_value *src = vtn_value(b, w[2], vtn_value_type_access_chain);
1553
1554 vtn_variable_copy(b, dest->access_chain, src->access_chain);
1555 break;
1556 }
1557
1558 case SpvOpLoad: {
1559 struct vtn_access_chain *src =
1560 vtn_value(b, w[3], vtn_value_type_access_chain)->access_chain;
1561
1562 if (src->var->mode == vtn_variable_mode_image ||
1563 src->var->mode == vtn_variable_mode_sampler) {
1564 vtn_push_value(b, w[2], vtn_value_type_access_chain)->access_chain = src;
1565 return;
1566 }
1567
1568 struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_ssa);
1569 val->ssa = vtn_variable_load(b, src);
1570 break;
1571 }
1572
1573 case SpvOpStore: {
1574 struct vtn_access_chain *dest =
1575 vtn_value(b, w[1], vtn_value_type_access_chain)->access_chain;
1576 struct vtn_ssa_value *src = vtn_ssa_value(b, w[2]);
1577 vtn_variable_store(b, src, dest);
1578 break;
1579 }
1580
1581 case SpvOpArrayLength: {
1582 struct vtn_access_chain *chain =
1583 vtn_value(b, w[3], vtn_value_type_access_chain)->access_chain;
1584
1585 const uint32_t offset = chain->var->type->offsets[w[4]];
1586 const uint32_t stride = chain->var->type->members[w[4]]->stride;
1587
1588 unsigned chain_idx;
1589 struct vtn_type *type;
1590 nir_ssa_def *index =
1591 get_vulkan_resource_index(b, chain, &type, &chain_idx);
1592
1593 nir_intrinsic_instr *instr =
1594 nir_intrinsic_instr_create(b->nb.shader,
1595 nir_intrinsic_get_buffer_size);
1596 instr->src[0] = nir_src_for_ssa(index);
1597 nir_ssa_dest_init(&instr->instr, &instr->dest, 1, 32, NULL);
1598 nir_builder_instr_insert(&b->nb, &instr->instr);
1599 nir_ssa_def *buf_size = &instr->dest.ssa;
1600
1601 /* array_length = max(buffer_size - offset, 0) / stride */
1602 nir_ssa_def *array_length =
1603 nir_idiv(&b->nb,
1604 nir_imax(&b->nb,
1605 nir_isub(&b->nb,
1606 buf_size,
1607 nir_imm_int(&b->nb, offset)),
1608 nir_imm_int(&b->nb, 0u)),
1609 nir_imm_int(&b->nb, stride));
1610
1611 struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_ssa);
1612 val->ssa = vtn_create_ssa_value(b, glsl_uint_type());
1613 val->ssa->def = array_length;
1614 break;
1615 }
1616
1617 case SpvOpCopyMemorySized:
1618 default:
1619 unreachable("Unhandled opcode");
1620 }
1621 }
1622