1 /*
2 * Copyright © 2024 Igalia
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "nir.h"
25 #include "nir_builder.h"
26 #include "nir_builder_opcodes.h"
27 #include "nir_intrinsics.h"
28
29 /*
30 * Convert atomic arithmetic to regular arithmetic along with cmpxchg
31 * by repeating the operation until the result is expected.
32 *
33 * eg:
34 * atomicAdd(a[0], 1) ->
35 *
36 * uint expected = a[0];
37 * while (true) {
38 * uint before = expected;
39 * expected += 1;
40 * uint original = atomicCompareExchange(a[0], before, expected);
41 * if (original == before) {break;}
42 * expected = original;
43 * }
44 */
45
46 static nir_def *
build_atomic(nir_builder * b,nir_intrinsic_instr * intr,bool ssbo)47 build_atomic(nir_builder *b, nir_intrinsic_instr *intr, bool ssbo)
48 {
49 nir_def *load = ssbo ? nir_load_ssbo(b, 1, intr->def.bit_size, intr->src[0].ssa,
50 intr->src[1].ssa,
51 .align_mul = intr->def.bit_size / 8,
52 .align_offset = 0)
53 : nir_load_global(b, intr->src[0].ssa, 8, 1, intr->def.bit_size);
54 nir_def *data = ssbo ? intr->src[2].ssa : intr->src[1].ssa;
55 nir_loop *loop = nir_push_loop(b);
56 nir_def *xchg;
57 {
58 nir_phi_instr *phi = nir_phi_instr_create(b->shader);
59 nir_def_init(&phi->instr, &phi->def, 1, intr->def.bit_size);
60 nir_phi_instr_add_src(phi, load->parent_instr->block, load);
61 nir_def *before = &phi->def;
62 nir_def *expected = nir_build_alu2(
63 b, nir_atomic_op_to_alu(nir_intrinsic_atomic_op(intr)), before, data);
64 nir_alu_instr* op = nir_instr_as_alu(expected->parent_instr);
65 op->exact = true;
66 op->fp_fast_math = 0;
67 if (ssbo) {
68 xchg = nir_ssbo_atomic_swap(b, intr->def.bit_size, intr->src[0].ssa, intr->src[1].ssa,
69 before, expected,
70 .atomic_op = nir_atomic_op_cmpxchg);
71 } else {
72 xchg =
73 nir_global_atomic_swap(b, intr->def.bit_size, intr->src[0].ssa, before, expected,
74 .atomic_op = nir_atomic_op_cmpxchg);
75 }
76 nir_break_if(b, nir_ieq(b, xchg, before));
77 nir_phi_instr_add_src(phi, nir_loop_last_block(loop), xchg);
78 b->cursor = nir_before_block(nir_loop_first_block(loop));
79 nir_builder_instr_insert(b, &phi->instr);
80 }
81 nir_pop_loop(b, loop);
82 return xchg;
83 }
84
85 static bool
lower_atomics(struct nir_builder * b,nir_intrinsic_instr * intr,void * supported)86 lower_atomics(struct nir_builder *b, nir_intrinsic_instr *intr,
87 void *supported)
88 {
89 nir_instr_filter_cb supported_cb = supported;
90
91 if (intr->intrinsic != nir_intrinsic_ssbo_atomic &&
92 intr->intrinsic != nir_intrinsic_global_atomic)
93 return false;
94 if (supported_cb(&intr->instr, NULL))
95 return false;
96 b->cursor = nir_before_instr(&intr->instr);
97 switch (nir_intrinsic_atomic_op(intr)) {
98 case nir_atomic_op_imin:
99 case nir_atomic_op_umin:
100 case nir_atomic_op_imax:
101 case nir_atomic_op_umax:
102 case nir_atomic_op_iand:
103 case nir_atomic_op_ior:
104 case nir_atomic_op_ixor:
105 case nir_atomic_op_fadd:
106 case nir_atomic_op_fmin:
107 case nir_atomic_op_fmax:
108 case nir_atomic_op_iadd: {
109 nir_def_replace(&intr->def, build_atomic(
110 b, intr, intr->intrinsic == nir_intrinsic_ssbo_atomic));
111 return true;
112 }
113 case nir_atomic_op_cmpxchg:
114 case nir_atomic_op_xchg:
115 return false;
116 case nir_atomic_op_fcmpxchg: /* unimplemented */
117 default:
118 unreachable("Invalid nir_atomic_op");
119 }
120 }
121
122 bool
nir_lower_atomics(nir_shader * shader,nir_instr_filter_cb supported)123 nir_lower_atomics(nir_shader *shader, nir_instr_filter_cb supported)
124 {
125 return nir_shader_intrinsics_pass(shader, lower_atomics,
126 nir_metadata_none, supported);
127 }
128