1 /*
2 * Copyright © 2020 Valve Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 */
24
25 #include "nir.h"
26 #include "nir_builder.h"
27
28 static bool
lower_discard_to_demote(nir_builder * b,nir_intrinsic_instr * intrin,void * data)29 lower_discard_to_demote(nir_builder *b, nir_intrinsic_instr *intrin, void *data)
30 {
31 switch (intrin->intrinsic) {
32 case nir_intrinsic_discard:
33 intrin->intrinsic = nir_intrinsic_demote;
34 return true;
35 case nir_intrinsic_discard_if:
36 intrin->intrinsic = nir_intrinsic_demote_if;
37 return true;
38 case nir_intrinsic_load_helper_invocation:
39 intrin->intrinsic = nir_intrinsic_is_helper_invocation;
40 return true;
41 default:
42 return false;
43 }
44 }
45
46 static bool
lower_demote_to_discard(nir_builder * b,nir_intrinsic_instr * intrin,void * data)47 lower_demote_to_discard(nir_builder *b, nir_intrinsic_instr *intrin, void *data)
48 {
49 switch (intrin->intrinsic) {
50 case nir_intrinsic_demote:
51 intrin->intrinsic = nir_intrinsic_discard;
52 return true;
53 case nir_intrinsic_demote_if:
54 intrin->intrinsic = nir_intrinsic_discard_if;
55 return true;
56 case nir_intrinsic_is_helper_invocation:
57 case nir_intrinsic_load_helper_invocation: {
58 /* If the shader doesn't need helper invocations,
59 * we can assume there are none */
60 b->cursor = nir_before_instr(&intrin->instr);
61 nir_def *zero = nir_imm_false(b);
62 nir_def_rewrite_uses(&intrin->def, zero);
63 nir_instr_remove(&intrin->instr);
64 return true;
65 }
66 default:
67 return false;
68 }
69 }
70
71 static nir_def *
insert_is_helper(nir_builder * b,nir_instr * instr)72 insert_is_helper(nir_builder *b, nir_instr *instr)
73 {
74 /* find best place to insert is_helper */
75 nir_cf_node *node = &instr->block->cf_node;
76 while (node->parent->type != nir_cf_node_function)
77 node = nir_cf_node_prev(node->parent);
78 nir_block *block = nir_cf_node_as_block(node);
79 if (block == instr->block) {
80 b->cursor = nir_before_instr(instr);
81 } else {
82 b->cursor = nir_after_block_before_jump(block);
83 }
84 return nir_is_helper_invocation(b, 1);
85 }
86
87 static bool
lower_load_helper_to_is_helper(nir_builder * b,nir_intrinsic_instr * intrin,void * data)88 lower_load_helper_to_is_helper(nir_builder *b,
89 nir_intrinsic_instr *intrin, void *data)
90 {
91 nir_def *is_helper = *(nir_def **)data;
92 switch (intrin->intrinsic) {
93 case nir_intrinsic_demote:
94 case nir_intrinsic_demote_if:
95 /* insert is_helper at last top level occasion */
96 if (is_helper == NULL) {
97 is_helper = insert_is_helper(b, &intrin->instr);
98 *(nir_def **)data = is_helper;
99 return true;
100 } else {
101 return false;
102 }
103 case nir_intrinsic_load_helper_invocation:
104 /* Don't update data: as long as we didn't encounter any demote(),
105 * we can insert new is_helper() intrinsics. These are placed at
106 * top-level blocks to ensure correct behavior w.r.t. loops */
107 if (is_helper == NULL)
108 is_helper = insert_is_helper(b, &intrin->instr);
109 nir_def_rewrite_uses(&intrin->def, is_helper);
110 nir_instr_remove(&intrin->instr);
111 return true;
112 default:
113 return false;
114 }
115 }
116
117 /**
118 * Optimize discard and demote opcodes.
119 *
120 * If force_correct_quad_ops_after_discard is true and quad operations are
121 * used, discard() will be converted to demote() and gl_HelperInvocation will
122 * be lowered to helperInvocationEXT(). This is intended as workaround for
123 * game bugs to force correct derivatives after kill. This lowering is not
124 * valid in the general case as it might change the result of subgroup
125 * operations and loop behavior.
126 *
127 * Otherwise, if demote is used and no ops need helper invocations, demote()
128 * will be converted to discard() as an optimization.
129 */
130 bool
nir_lower_discard_or_demote(nir_shader * shader,bool force_correct_quad_ops_after_discard)131 nir_lower_discard_or_demote(nir_shader *shader,
132 bool force_correct_quad_ops_after_discard)
133 {
134 if (shader->info.stage != MESA_SHADER_FRAGMENT)
135 return false;
136
137 /* We need uses_discard/demote and needs_*_helper_invocations. */
138 nir_shader_gather_info(shader, nir_shader_get_entrypoint(shader));
139 /* Validate that if uses_demote is set, uses_discard is also be set. */
140 assert(!shader->info.fs.uses_demote || shader->info.fs.uses_discard);
141
142 /* Quick skip. */
143 if (!shader->info.fs.uses_discard)
144 return false;
145
146 bool progress = false;
147
148 if (force_correct_quad_ops_after_discard &&
149 shader->info.fs.needs_quad_helper_invocations) {
150 /* If we need correct derivatives, convert discard to demote only when
151 * derivatives are actually used.
152 */
153 progress = nir_shader_intrinsics_pass(shader, lower_discard_to_demote,
154 nir_metadata_block_index |
155 nir_metadata_dominance |
156 nir_metadata_live_defs |
157 nir_metadata_instr_index,
158 NULL);
159 shader->info.fs.uses_demote = true;
160 } else if (!shader->info.fs.needs_quad_helper_invocations &&
161 !shader->info.uses_wide_subgroup_intrinsics &&
162 shader->info.fs.uses_demote) {
163 /* If we don't need any helper invocations, convert demote to discard. */
164 progress = nir_shader_intrinsics_pass(shader, lower_demote_to_discard,
165 nir_metadata_block_index |
166 nir_metadata_dominance,
167 NULL);
168 shader->info.fs.uses_demote = false;
169 } else if (shader->info.fs.uses_demote &&
170 BITSET_TEST(shader->info.system_values_read,
171 nir_system_value_from_intrinsic(nir_intrinsic_load_helper_invocation))) {
172 /* load_helper needs to preserve the value (whether an invocation is
173 * a helper lane) from the beginning of the shader. */
174 nir_def *is_helper = NULL;
175 progress = nir_shader_intrinsics_pass(shader,
176 lower_load_helper_to_is_helper,
177 nir_metadata_block_index |
178 nir_metadata_dominance,
179 &is_helper);
180 BITSET_CLEAR(shader->info.system_values_read,
181 nir_system_value_from_intrinsic(nir_intrinsic_load_helper_invocation));
182 }
183
184 /* Validate again that if uses_demote is set, uses_discard is also be set. */
185 assert(!shader->info.fs.uses_demote || shader->info.fs.uses_discard);
186 return progress;
187 }
188