1 /*
2 * Copyright © 2020 Raspberry Pi Ltd
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "compiler/v3d_compiler.h"
25 #include "compiler/nir/nir_builder.h"
26
27 static void
rewrite_offset(nir_builder * b,nir_intrinsic_instr * instr,uint32_t buffer_idx,uint32_t offset_src,nir_intrinsic_op buffer_size_op)28 rewrite_offset(nir_builder *b,
29 nir_intrinsic_instr *instr,
30 uint32_t buffer_idx,
31 uint32_t offset_src,
32 nir_intrinsic_op buffer_size_op)
33 {
34 b->cursor = nir_before_instr(&instr->instr);
35
36 /* Get size of the buffer */
37 nir_intrinsic_instr *size =
38 nir_intrinsic_instr_create(b->shader, buffer_size_op);
39 size->src[0] = nir_src_for_ssa(nir_imm_int(b, buffer_idx));
40 nir_ssa_dest_init(&size->instr, &size->dest, 1, 32, NULL);
41 nir_builder_instr_insert(b, &size->instr);
42
43 /* All out TMU accesses are 32-bit aligned */
44 nir_ssa_def *aligned_buffer_size =
45 nir_iand(b, &size->dest.ssa, nir_imm_int(b, 0xfffffffc));
46
47 /* Rewrite offset */
48 nir_ssa_def *offset =
49 nir_umin(b, instr->src[offset_src].ssa, aligned_buffer_size);
50 nir_instr_rewrite_src(&instr->instr, &instr->src[offset_src],
51 nir_src_for_ssa(offset));
52 }
53
54 static void
lower_load(struct v3d_compile * c,nir_builder * b,nir_intrinsic_instr * instr)55 lower_load(struct v3d_compile *c,
56 nir_builder *b,
57 nir_intrinsic_instr *instr)
58 {
59 uint32_t index = nir_src_comp_as_uint(instr->src[0], 0);
60
61 nir_intrinsic_op op;
62 if (instr->intrinsic == nir_intrinsic_load_ubo) {
63 op = nir_intrinsic_get_ubo_size;
64 if (c->key->environment == V3D_ENVIRONMENT_VULKAN)
65 index--;
66 } else {
67 op = nir_intrinsic_get_ssbo_size;
68 }
69
70 rewrite_offset(b, instr, index, 1, op);
71 }
72
73 static void
lower_store(struct v3d_compile * c,nir_builder * b,nir_intrinsic_instr * instr)74 lower_store(struct v3d_compile *c,
75 nir_builder *b,
76 nir_intrinsic_instr *instr)
77 {
78 uint32_t index = nir_src_comp_as_uint(instr->src[1], 0);
79 rewrite_offset(b, instr, index, 2, nir_intrinsic_get_ssbo_size);
80 }
81
82 static void
lower_atomic(struct v3d_compile * c,nir_builder * b,nir_intrinsic_instr * instr)83 lower_atomic(struct v3d_compile *c,
84 nir_builder *b,
85 nir_intrinsic_instr *instr)
86 {
87 uint32_t index = nir_src_comp_as_uint(instr->src[0], 0);
88 rewrite_offset(b, instr, index, 1, nir_intrinsic_get_ssbo_size);
89 }
90
91 static void
lower_shared(struct v3d_compile * c,nir_builder * b,nir_intrinsic_instr * instr)92 lower_shared(struct v3d_compile *c,
93 nir_builder *b,
94 nir_intrinsic_instr *instr)
95 {
96 b->cursor = nir_before_instr(&instr->instr);
97 nir_ssa_def *aligned_size =
98 nir_imm_int(b, c->s->info.shared_size & 0xfffffffc);
99 nir_ssa_def *offset = nir_umin(b, instr->src[0].ssa, aligned_size);
100 nir_instr_rewrite_src(&instr->instr, &instr->src[0],
101 nir_src_for_ssa(offset));
102 }
103
104 static bool
lower_instr(nir_builder * b,nir_instr * instr,void * _state)105 lower_instr(nir_builder *b, nir_instr *instr, void *_state)
106 {
107 struct v3d_compile *c = _state;
108
109 if (instr->type != nir_instr_type_intrinsic)
110 return false;
111 nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
112
113 switch (intr->intrinsic) {
114 case nir_intrinsic_load_ubo:
115 case nir_intrinsic_load_ssbo:
116 lower_load(c, b, intr);
117 return true;
118 case nir_intrinsic_store_ssbo:
119 lower_store(c, b, intr);
120 return true;
121 case nir_intrinsic_ssbo_atomic_add:
122 case nir_intrinsic_ssbo_atomic_imin:
123 case nir_intrinsic_ssbo_atomic_umin:
124 case nir_intrinsic_ssbo_atomic_imax:
125 case nir_intrinsic_ssbo_atomic_umax:
126 case nir_intrinsic_ssbo_atomic_and:
127 case nir_intrinsic_ssbo_atomic_or:
128 case nir_intrinsic_ssbo_atomic_xor:
129 case nir_intrinsic_ssbo_atomic_exchange:
130 case nir_intrinsic_ssbo_atomic_comp_swap:
131 lower_atomic(c, b, intr);
132 return true;
133 case nir_intrinsic_load_shared:
134 case nir_intrinsic_shared_atomic_add:
135 case nir_intrinsic_shared_atomic_imin:
136 case nir_intrinsic_shared_atomic_umin:
137 case nir_intrinsic_shared_atomic_imax:
138 case nir_intrinsic_shared_atomic_umax:
139 case nir_intrinsic_shared_atomic_and:
140 case nir_intrinsic_shared_atomic_or:
141 case nir_intrinsic_shared_atomic_xor:
142 case nir_intrinsic_shared_atomic_exchange:
143 case nir_intrinsic_shared_atomic_comp_swap:
144 lower_shared(c, b, intr);
145 return true;
146 default:
147 return false;
148 }
149 }
150
151 bool
v3d_nir_lower_robust_buffer_access(nir_shader * s,struct v3d_compile * c)152 v3d_nir_lower_robust_buffer_access(nir_shader *s, struct v3d_compile *c)
153 {
154 return nir_shader_instructions_pass(s, lower_instr,
155 nir_metadata_block_index |
156 nir_metadata_dominance, c);
157 }
158