1 /*
2 * Copyright © 2017-2018 Rob Clark <robclark@freedesktop.org>
3 * SPDX-License-Identifier: MIT
4 *
5 * Authors:
6 * Rob Clark <robclark@freedesktop.org>
7 */
8
9 /* 500 gets us LDIB but doesn't change any other a4xx instructions */
10 #define GPU 500
11
12 #include "ir3_context.h"
13 #include "ir3_image.h"
14
15 /* SSBO data is available at this CB address, addressed like regular consts
16 * containing the following data in each vec4:
17 *
18 * [ base address, pitch, array_pitch, cpp ]
19 *
20 * These mirror the values uploaded to A4XX_SSBO_0 state. For A5XX, these are
21 * uploaded manually by the driver.
22 */
23 #define A4XX_SSBO_CB_BASE(i) (0x700 + ((i) << 2))
24
25 /*
26 * Handlers for instructions changed/added in a4xx:
27 */
28
29 /* Convert byte offset to address of appropriate width for GPU */
30 static struct ir3_instruction *
byte_offset_to_address(struct ir3_context * ctx,nir_src * ssbo,struct ir3_instruction * byte_offset)31 byte_offset_to_address(struct ir3_context *ctx,
32 nir_src *ssbo,
33 struct ir3_instruction *byte_offset)
34 {
35 struct ir3_builder *b = &ctx->build;
36
37 if (ctx->compiler->gen == 4) {
38 uint32_t index = nir_src_as_uint(*ssbo);
39 unsigned cb = A4XX_SSBO_CB_BASE(index);
40 byte_offset = ir3_ADD_U(b, create_uniform(b, cb), 0, byte_offset, 0);
41 }
42
43 if (ctx->compiler->is_64bit) {
44 return ir3_collect(b, byte_offset, create_immed(b, 0));
45 } else {
46 return byte_offset;
47 }
48 }
49
50 /* src[] = { buffer_index, offset }. No const_index */
51 static void
emit_intrinsic_load_ssbo(struct ir3_context * ctx,nir_intrinsic_instr * intr,struct ir3_instruction ** dst)52 emit_intrinsic_load_ssbo(struct ir3_context *ctx, nir_intrinsic_instr *intr,
53 struct ir3_instruction **dst)
54 {
55 struct ir3_builder *b = &ctx->build;
56 struct ir3_instruction *ldgb, *src0, *src1, *byte_offset, *offset;
57
58 struct ir3_instruction *ssbo = ir3_ssbo_to_ibo(ctx, intr->src[0]);
59
60 byte_offset = ir3_get_src(ctx, &intr->src[1])[0];
61 offset = ir3_get_src(ctx, &intr->src[2])[0];
62
63 /* src0 is uvec2(offset*4, 0), src1 is offset.. nir already *= 4: */
64 src0 = byte_offset_to_address(ctx, &intr->src[0], byte_offset);
65 src1 = offset;
66
67 ldgb = ir3_LDGB(b, ssbo, 0, src0, 0, src1, 0);
68 ldgb->dsts[0]->wrmask = MASK(intr->num_components);
69 ldgb->cat6.iim_val = intr->num_components;
70 ldgb->cat6.d = 4;
71 ldgb->cat6.type = TYPE_U32;
72 ldgb->barrier_class = IR3_BARRIER_BUFFER_R;
73 ldgb->barrier_conflict = IR3_BARRIER_BUFFER_W;
74
75 ir3_split_dest(b, dst, ldgb, 0, intr->num_components);
76 }
77
78 /* src[] = { value, block_index, offset }. const_index[] = { write_mask } */
79 static void
emit_intrinsic_store_ssbo(struct ir3_context * ctx,nir_intrinsic_instr * intr)80 emit_intrinsic_store_ssbo(struct ir3_context *ctx, nir_intrinsic_instr *intr)
81 {
82 struct ir3_builder *b = &ctx->build;
83 struct ir3_instruction *stgb, *src0, *src1, *src2, *byte_offset, *offset;
84 unsigned wrmask = nir_intrinsic_write_mask(intr);
85 unsigned ncomp = ffs(~wrmask) - 1;
86
87 assert(wrmask == BITFIELD_MASK(intr->num_components));
88
89 struct ir3_instruction *ssbo = ir3_ssbo_to_ibo(ctx, intr->src[1]);
90
91 byte_offset = ir3_get_src(ctx, &intr->src[2])[0];
92 offset = ir3_get_src(ctx, &intr->src[3])[0];
93
94 /* src0 is value, src1 is offset, src2 is uvec2(offset*4, 0)..
95 * nir already *= 4:
96 */
97 src0 = ir3_create_collect(b, ir3_get_src(ctx, &intr->src[0]), ncomp);
98 src1 = offset;
99 src2 = byte_offset_to_address(ctx, &intr->src[1], byte_offset);
100
101 stgb = ir3_STGB(b, ssbo, 0, src0, 0, src1, 0, src2, 0);
102 stgb->cat6.iim_val = ncomp;
103 stgb->cat6.d = 4;
104 stgb->cat6.type = TYPE_U32;
105 stgb->barrier_class = IR3_BARRIER_BUFFER_W;
106 stgb->barrier_conflict = IR3_BARRIER_BUFFER_R | IR3_BARRIER_BUFFER_W;
107
108 array_insert(ctx->block, ctx->block->keeps, stgb);
109 }
110
111 static struct ir3_instruction *
emit_atomic(struct ir3_builder * b,nir_atomic_op op,struct ir3_instruction * bo,struct ir3_instruction * data,struct ir3_instruction * offset,struct ir3_instruction * byte_offset)112 emit_atomic(struct ir3_builder *b, nir_atomic_op op, struct ir3_instruction *bo,
113 struct ir3_instruction *data, struct ir3_instruction *offset,
114 struct ir3_instruction *byte_offset)
115 {
116 switch (op) {
117 case nir_atomic_op_iadd:
118 return ir3_ATOMIC_S_ADD(b, bo, 0, data, 0, offset, 0, byte_offset, 0);
119 case nir_atomic_op_imin:
120 return ir3_ATOMIC_S_MIN(b, bo, 0, data, 0, offset, 0, byte_offset, 0);
121 case nir_atomic_op_umin:
122 return ir3_ATOMIC_S_MIN(b, bo, 0, data, 0, offset, 0, byte_offset, 0);
123 case nir_atomic_op_imax:
124 return ir3_ATOMIC_S_MAX(b, bo, 0, data, 0, offset, 0, byte_offset, 0);
125 case nir_atomic_op_umax:
126 return ir3_ATOMIC_S_MAX(b, bo, 0, data, 0, offset, 0, byte_offset, 0);
127 case nir_atomic_op_iand:
128 return ir3_ATOMIC_S_AND(b, bo, 0, data, 0, offset, 0, byte_offset, 0);
129 case nir_atomic_op_ior:
130 return ir3_ATOMIC_S_OR(b, bo, 0, data, 0, offset, 0, byte_offset, 0);
131 case nir_atomic_op_ixor:
132 return ir3_ATOMIC_S_XOR(b, bo, 0, data, 0, offset, 0, byte_offset, 0);
133 case nir_atomic_op_xchg:
134 return ir3_ATOMIC_S_XCHG(b, bo, 0, data, 0, offset, 0, byte_offset, 0);
135 case nir_atomic_op_cmpxchg:
136 return ir3_ATOMIC_S_CMPXCHG(b, bo, 0, data, 0, offset, 0, byte_offset, 0);
137 default:
138 unreachable("boo");
139 }
140 }
141
142 /*
143 * SSBO atomic intrinsics
144 *
145 * All of the SSBO atomic memory operations read a value from memory,
146 * compute a new value using one of the operations below, write the new
147 * value to memory, and return the original value read.
148 *
149 * All operations take 3 sources except CompSwap that takes 4. These
150 * sources represent:
151 *
152 * 0: The SSBO buffer index.
153 * 1: The byte offset into the SSBO buffer of the variable that the atomic
154 * operation will operate on.
155 * 2: The data parameter to the atomic function (i.e. the value to add
156 * in, etc).
157 * 3: CompSwap: the second data parameter.
158 * Non-CompSwap: The dword offset into the SSBO buffer variable.
159 * 4: CompSwap: The dword offset into the SSBO buffer variable.
160 *
161 * We use custom ssbo_*_ir3 intrinsics generated by ir3_nir_lower_io_offsets()
162 * so we can have the dword offset generated in NIR.
163 */
164 static struct ir3_instruction *
emit_intrinsic_atomic_ssbo(struct ir3_context * ctx,nir_intrinsic_instr * intr)165 emit_intrinsic_atomic_ssbo(struct ir3_context *ctx, nir_intrinsic_instr *intr)
166 {
167 struct ir3_builder *b = &ctx->build;
168 nir_atomic_op op = nir_intrinsic_atomic_op(intr);
169 type_t type = nir_atomic_op_type(op) == nir_type_int ? TYPE_S32 : TYPE_U32;
170
171 struct ir3_instruction *ssbo = ir3_ssbo_to_ibo(ctx, intr->src[0]);
172
173 struct ir3_instruction *data = ir3_get_src(ctx, &intr->src[2])[0];
174 /* 64b byte offset */
175 struct ir3_instruction *byte_offset =
176 byte_offset_to_address(ctx, &intr->src[0], ir3_get_src(ctx, &intr->src[1])[0]);
177 /* dword offset for everything but cmpxchg */
178 struct ir3_instruction *src3 = ir3_get_src(ctx, &intr->src[3])[0];
179
180 if (op == nir_atomic_op_cmpxchg) {
181 /* for cmpxchg, src0 is [ui]vec2(data, compare): */
182 data = ir3_collect(b, src3, data);
183 src3 = ir3_get_src(ctx, &intr->src[4])[0];
184 }
185
186 struct ir3_instruction *atomic =
187 emit_atomic(b, op, ssbo, data, src3, byte_offset);
188
189 atomic->cat6.iim_val = 1;
190 atomic->cat6.d = 4;
191 atomic->cat6.type = type;
192 atomic->barrier_class = IR3_BARRIER_BUFFER_W;
193 atomic->barrier_conflict = IR3_BARRIER_BUFFER_R | IR3_BARRIER_BUFFER_W;
194
195 /* even if nothing consume the result, we can't DCE the instruction: */
196 array_insert(ctx->block, ctx->block->keeps, atomic);
197
198 return atomic;
199 }
200
201 static struct ir3_instruction *
get_image_offset(struct ir3_context * ctx,const nir_intrinsic_instr * instr,struct ir3_instruction * const * coords,bool byteoff)202 get_image_offset(struct ir3_context *ctx, const nir_intrinsic_instr *instr,
203 struct ir3_instruction *const *coords, bool byteoff)
204 {
205 struct ir3_builder *b = &ctx->build;
206 struct ir3_instruction *offset;
207 unsigned index = nir_src_as_uint(instr->src[0]);
208 unsigned ncoords = ir3_get_image_coords(instr, NULL);
209
210 /* to calculate the byte offset (yes, uggg) we need (up to) three
211 * const values to know the bytes per pixel, and y and z stride:
212 */
213 unsigned cb;
214 if (ctx->compiler->gen > 4) {
215 const struct ir3_const_state *const_state = ir3_const_state(ctx->so);
216 assert(const_state->image_dims.mask & (1 << index));
217
218 cb = ir3_const_reg(const_state, IR3_CONST_ALLOC_IMAGE_DIMS, 0) +
219 const_state->image_dims.off[index];
220 } else {
221 index += ctx->s->info.num_ssbos;
222 cb = A4XX_SSBO_CB_BASE(index);
223 }
224
225 /* offset = coords.x * bytes_per_pixel: */
226 if (ctx->compiler->gen == 4)
227 offset = ir3_MUL_S24(b, coords[0], 0, create_uniform(b, cb + 3), 0);
228 else
229 offset = ir3_MUL_S24(b, coords[0], 0, create_uniform(b, cb + 0), 0);
230 if (ncoords > 1) {
231 /* offset += coords.y * y_pitch: */
232 offset =
233 ir3_MAD_S24(b, create_uniform(b, cb + 1), 0, coords[1], 0, offset, 0);
234 }
235 if (ncoords > 2) {
236 /* offset += coords.z * z_pitch: */
237 offset =
238 ir3_MAD_S24(b, create_uniform(b, cb + 2), 0, coords[2], 0, offset, 0);
239 }
240
241 /* a4xx: must add in the base address: */
242 if (ctx->compiler->gen == 4)
243 offset = ir3_ADD_U(b, offset, 0, create_uniform(b, cb + 0), 0);
244
245 if (!byteoff) {
246 /* Some cases, like atomics, seem to use dword offset instead
247 * of byte offsets.. blob just puts an extra shr.b in there
248 * in those cases:
249 */
250 offset = ir3_SHR_B(b, offset, 0, create_immed(b, 2), 0);
251 }
252
253 if (ctx->compiler->is_64bit)
254 return ir3_collect(b, offset, create_immed(b, 0));
255 else
256 return offset;
257 }
258
259 /* src[] = { deref, coord, sample_index }. const_index[] = {} */
260 static void
emit_intrinsic_load_image(struct ir3_context * ctx,nir_intrinsic_instr * intr,struct ir3_instruction ** dst)261 emit_intrinsic_load_image(struct ir3_context *ctx, nir_intrinsic_instr *intr,
262 struct ir3_instruction **dst)
263 {
264 struct ir3_builder *b = &ctx->build;
265 struct ir3_instruction *const *coords = ir3_get_src(ctx, &intr->src[1]);
266 struct ir3_instruction *ibo = ir3_image_to_ibo(ctx, intr->src[0]);
267 struct ir3_instruction *offset = get_image_offset(ctx, intr, coords, true);
268 unsigned ncoords = ir3_get_image_coords(intr, NULL);
269 unsigned ncomp =
270 ir3_get_num_components_for_image_format(nir_intrinsic_format(intr));
271
272 struct ir3_instruction *ldib;
273 /* At least A420 does not have LDIB. Use LDGB and perform conversion
274 * ourselves.
275 *
276 * TODO: Actually do the conversion. ES 3.1 only requires this for
277 * single-component 32-bit types anyways.
278 */
279 if (ctx->compiler->gen > 4) {
280 ldib = ir3_LDIB(
281 b, ibo, 0, offset, 0, ir3_create_collect(b, coords, ncoords), 0);
282 } else {
283 ldib = ir3_LDGB(
284 b, ibo, 0, offset, 0, ir3_create_collect(b, coords, ncoords), 0);
285 switch (nir_intrinsic_format(intr)) {
286 case PIPE_FORMAT_R32_UINT:
287 case PIPE_FORMAT_R32_SINT:
288 case PIPE_FORMAT_R32_FLOAT:
289 break;
290 default:
291 /* For some reason even more 32-bit components don't work. */
292 assert(0);
293 break;
294 }
295 }
296 ldib->dsts[0]->wrmask = MASK(intr->num_components);
297 ldib->cat6.iim_val = ncomp;
298 ldib->cat6.d = ncoords;
299 ldib->cat6.type = ir3_get_type_for_image_intrinsic(intr);
300 ldib->cat6.typed = true;
301 ldib->barrier_class = IR3_BARRIER_IMAGE_R;
302 ldib->barrier_conflict = IR3_BARRIER_IMAGE_W;
303
304 ir3_split_dest(b, dst, ldib, 0, intr->num_components);
305 }
306
307 /* src[] = { index, coord, sample_index, value }. const_index[] = {} */
308 static void
emit_intrinsic_store_image(struct ir3_context * ctx,nir_intrinsic_instr * intr)309 emit_intrinsic_store_image(struct ir3_context *ctx, nir_intrinsic_instr *intr)
310 {
311 struct ir3_builder *b = &ctx->build;
312 struct ir3_instruction *stib, *offset;
313 struct ir3_instruction *const *value = ir3_get_src(ctx, &intr->src[3]);
314 struct ir3_instruction *const *coords = ir3_get_src(ctx, &intr->src[1]);
315 struct ir3_instruction *ibo = ir3_image_to_ibo(ctx, intr->src[0]);
316 unsigned ncoords = ir3_get_image_coords(intr, NULL);
317 unsigned ncomp =
318 ir3_get_num_components_for_image_format(nir_intrinsic_format(intr));
319
320 /* src0 is value
321 * src1 is coords
322 * src2 is 64b byte offset
323 */
324
325 offset = get_image_offset(ctx, intr, coords, true);
326
327 /* NOTE: stib seems to take byte offset, but stgb.typed can be used
328 * too and takes a dword offset.. not quite sure yet why blob uses
329 * one over the other in various cases.
330 */
331
332 stib = ir3_STIB(b, ibo, 0, ir3_create_collect(b, value, ncomp), 0,
333 ir3_create_collect(b, coords, ncoords), 0, offset, 0);
334 stib->cat6.iim_val = ncomp;
335 stib->cat6.d = ncoords;
336 stib->cat6.type = ir3_get_type_for_image_intrinsic(intr);
337 stib->cat6.typed = true;
338 stib->barrier_class = IR3_BARRIER_IMAGE_W;
339 stib->barrier_conflict = IR3_BARRIER_IMAGE_R | IR3_BARRIER_IMAGE_W;
340
341 array_insert(ctx->block, ctx->block->keeps, stib);
342 }
343
344 /* src[] = { deref, coord, sample_index, value, compare }. const_index[] = {} */
345 static struct ir3_instruction *
emit_intrinsic_atomic_image(struct ir3_context * ctx,nir_intrinsic_instr * intr)346 emit_intrinsic_atomic_image(struct ir3_context *ctx, nir_intrinsic_instr *intr)
347 {
348 struct ir3_builder *b = &ctx->build;
349 struct ir3_instruction *atomic, *src0, *src1, *src2;
350 struct ir3_instruction *const *coords = ir3_get_src(ctx, &intr->src[1]);
351 struct ir3_instruction *image = ir3_image_to_ibo(ctx, intr->src[0]);
352 unsigned ncoords = ir3_get_image_coords(intr, NULL);
353 nir_atomic_op op = nir_intrinsic_atomic_op(intr);
354
355 /* src0 is value (or uvec2(value, compare))
356 * src1 is coords
357 * src2 is 64b byte offset
358 */
359 src0 = ir3_get_src(ctx, &intr->src[3])[0];
360 src1 = ir3_create_collect(b, coords, ncoords);
361 src2 = get_image_offset(ctx, intr, coords, ctx->compiler->gen == 4);
362
363 if (op == nir_atomic_op_cmpxchg)
364 src0 = ir3_collect(b, ir3_get_src(ctx, &intr->src[4])[0], src0);
365
366 atomic = emit_atomic(b, op, image, src0, src1, src2);
367 atomic->cat6.iim_val = 1;
368 atomic->cat6.d = ncoords;
369 atomic->cat6.type = ir3_get_type_for_image_intrinsic(intr);
370 atomic->cat6.typed = ctx->compiler->gen == 5;
371 atomic->barrier_class = IR3_BARRIER_IMAGE_W;
372 atomic->barrier_conflict = IR3_BARRIER_IMAGE_R | IR3_BARRIER_IMAGE_W;
373
374 /* even if nothing consume the result, we can't DCE the instruction: */
375 array_insert(ctx->block, ctx->block->keeps, atomic);
376
377 return atomic;
378 }
379
380 static struct ir3_instruction *
emit_intrinsic_atomic_global(struct ir3_context * ctx,nir_intrinsic_instr * intr)381 emit_intrinsic_atomic_global(struct ir3_context *ctx, nir_intrinsic_instr *intr)
382 {
383 unreachable("Global atomic are unimplemented on A5xx");
384 }
385
386 const struct ir3_context_funcs ir3_a4xx_funcs = {
387 .emit_intrinsic_load_ssbo = emit_intrinsic_load_ssbo,
388 .emit_intrinsic_store_ssbo = emit_intrinsic_store_ssbo,
389 .emit_intrinsic_atomic_ssbo = emit_intrinsic_atomic_ssbo,
390 .emit_intrinsic_load_image = emit_intrinsic_load_image,
391 .emit_intrinsic_store_image = emit_intrinsic_store_image,
392 .emit_intrinsic_atomic_image = emit_intrinsic_atomic_image,
393 .emit_intrinsic_image_size = emit_intrinsic_image_size_tex,
394 .emit_intrinsic_load_global_ir3 = NULL,
395 .emit_intrinsic_store_global_ir3 = NULL,
396 .emit_intrinsic_atomic_global = emit_intrinsic_atomic_global,
397 };
398