1 /**************************************************************************
2 *
3 * Copyright 2022 Red Hat
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included
14 * in all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
17 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 *
24 **************************************************************************/
25
26 #include "lp_bld_nir.h"
27 #include "lp_bld_init.h"
28 #include "lp_bld_const.h"
29 #include "lp_bld_flow.h"
30 #include "lp_bld_struct.h"
31 #include "lp_bld_swizzle.h"
32 #include "lp_bld_debug.h"
33 #include "util/u_math.h"
34
35
36 static LLVMValueRef
swizzle_aos(struct lp_build_nir_context * bld_base,LLVMValueRef a,unsigned swizzle_x,unsigned swizzle_y,unsigned swizzle_z,unsigned swizzle_w)37 swizzle_aos(struct lp_build_nir_context *bld_base,
38 LLVMValueRef a,
39 unsigned swizzle_x,
40 unsigned swizzle_y,
41 unsigned swizzle_z,
42 unsigned swizzle_w)
43 {
44 unsigned char swizzles[4];
45 struct lp_build_nir_aos_context *bld = lp_nir_aos_context(bld_base);
46
47 assert(swizzle_x < 4);
48 assert(swizzle_y < 4);
49 assert(swizzle_z < 4);
50 assert(swizzle_w < 4);
51
52 swizzles[bld->inv_swizzles[0]] = bld->swizzles[swizzle_x];
53 swizzles[bld->inv_swizzles[1]] = bld->swizzles[swizzle_y];
54 swizzles[bld->inv_swizzles[2]] = bld->swizzles[swizzle_z];
55 swizzles[bld->inv_swizzles[3]] = bld->swizzles[swizzle_w];
56
57 return lp_build_swizzle_aos(&bld->bld_base.base, a, swizzles);
58 }
59
60
61 LLVMValueRef
lp_nir_aos_conv_const(struct gallivm_state * gallivm,LLVMValueRef constval,int nc)62 lp_nir_aos_conv_const(struct gallivm_state *gallivm,
63 LLVMValueRef constval, int nc)
64 {
65 LLVMValueRef elems[16];
66 uint8_t val = 0;
67 /* convert from 1..4 x f32 to 16 x unorm8 */
68 for (unsigned i = 0; i < nc; i++) {
69 LLVMValueRef value =
70 LLVMBuildExtractElement(gallivm->builder, constval,
71 lp_build_const_int32(gallivm, i), "");
72 assert(LLVMIsConstant(value));
73 unsigned uval = LLVMConstIntGetZExtValue(value);
74 float f = uif(uval);
75 val = float_to_ubyte(f);
76 for (unsigned j = 0; j < 4; j++) {
77 elems[j * 4 + i] =
78 LLVMConstInt(LLVMInt8TypeInContext(gallivm->context), val, 0);
79 }
80 }
81 for (unsigned i = nc; i < 4; i++) {
82 for (unsigned j = 0; j < 4; j++) {
83 elems[j * 4 + i] =
84 LLVMConstInt(LLVMInt8TypeInContext(gallivm->context), val, 0);
85 }
86 }
87 return LLVMConstVector(elems, 16);
88 }
89
90
91 static void
init_var_slots(struct lp_build_nir_context * bld_base,nir_variable * var)92 init_var_slots(struct lp_build_nir_context *bld_base,
93 nir_variable *var)
94 {
95 struct lp_build_nir_aos_context *bld =
96 (struct lp_build_nir_aos_context *)bld_base;
97
98 if (!bld->outputs)
99 return;
100 unsigned this_loc = var->data.driver_location;
101
102 bld->outputs[this_loc] = lp_build_alloca(bld_base->base.gallivm,
103 bld_base->base.vec_type,
104 "output");
105 }
106
107
108 static void
emit_var_decl(struct lp_build_nir_context * bld_base,nir_variable * var)109 emit_var_decl(struct lp_build_nir_context *bld_base,
110 nir_variable *var)
111 {
112 if (var->data.mode == nir_var_shader_out) {
113 init_var_slots(bld_base, var);
114 }
115 }
116
117
118 static void
emit_load_var(struct lp_build_nir_context * bld_base,nir_variable_mode deref_mode,unsigned num_components,unsigned bit_size,nir_variable * var,unsigned vertex_index,LLVMValueRef indir_vertex_index,unsigned const_index,LLVMValueRef indir_index,LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])119 emit_load_var(struct lp_build_nir_context *bld_base,
120 nir_variable_mode deref_mode,
121 unsigned num_components,
122 unsigned bit_size,
123 nir_variable *var,
124 unsigned vertex_index,
125 LLVMValueRef indir_vertex_index,
126 unsigned const_index,
127 LLVMValueRef indir_index,
128 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
129 {
130 struct lp_build_nir_aos_context *bld =
131 (struct lp_build_nir_aos_context *)bld_base;
132 unsigned location = var->data.driver_location;
133
134 if (deref_mode == nir_var_shader_in) {
135 result[0] = bld->inputs[location];
136 }
137 }
138
139
140 static void
emit_store_var(struct lp_build_nir_context * bld_base,nir_variable_mode deref_mode,unsigned num_components,unsigned bit_size,nir_variable * var,unsigned writemask,LLVMValueRef indir_vertex_index,unsigned const_index,LLVMValueRef indir_index,LLVMValueRef vals)141 emit_store_var(struct lp_build_nir_context *bld_base,
142 nir_variable_mode deref_mode,
143 unsigned num_components,
144 unsigned bit_size,
145 nir_variable *var,
146 unsigned writemask,
147 LLVMValueRef indir_vertex_index,
148 unsigned const_index,
149 LLVMValueRef indir_index,
150 LLVMValueRef vals)
151 {
152 struct lp_build_nir_aos_context *bld =
153 (struct lp_build_nir_aos_context *)bld_base;
154 struct gallivm_state *gallivm = bld_base->base.gallivm;
155 unsigned location = var->data.driver_location;
156
157 if (LLVMIsConstant(vals)) {
158 vals = lp_nir_aos_conv_const(gallivm, vals, num_components);
159 }
160
161 if (deref_mode == nir_var_shader_out) {
162 LLVMBuildStore(gallivm->builder, vals, bld->outputs[location]);
163 }
164 }
165
166
167 static LLVMValueRef
emit_load_reg(struct lp_build_nir_context * bld_base,struct lp_build_context * reg_bld,const nir_reg_src * reg,LLVMValueRef indir_src,LLVMValueRef reg_storage)168 emit_load_reg(struct lp_build_nir_context *bld_base,
169 struct lp_build_context *reg_bld,
170 const nir_reg_src *reg,
171 LLVMValueRef indir_src,
172 LLVMValueRef reg_storage)
173 {
174 struct gallivm_state *gallivm = bld_base->base.gallivm;
175 return LLVMBuildLoad(gallivm->builder, reg_storage, "");
176 }
177
178
179 static void
emit_store_reg(struct lp_build_nir_context * bld_base,struct lp_build_context * reg_bld,const nir_reg_dest * reg,unsigned writemask,LLVMValueRef indir_src,LLVMValueRef reg_storage,LLVMValueRef vals[NIR_MAX_VEC_COMPONENTS])180 emit_store_reg(struct lp_build_nir_context *bld_base,
181 struct lp_build_context *reg_bld,
182 const nir_reg_dest *reg,
183 unsigned writemask,
184 LLVMValueRef indir_src,
185 LLVMValueRef reg_storage,
186 LLVMValueRef vals[NIR_MAX_VEC_COMPONENTS])
187 {
188 struct gallivm_state *gallivm = bld_base->base.gallivm;
189
190 if (LLVMIsConstant(vals[0]))
191 vals[0] = lp_nir_aos_conv_const(gallivm, vals[0], 1);
192
193 if (writemask == 0xf) {
194 LLVMBuildStore(gallivm->builder, vals[0], reg_storage);
195 return;
196 }
197
198 LLVMValueRef cur = LLVMBuildLoad(gallivm->builder, reg_storage, "");
199 LLVMTypeRef i32t = LLVMInt32TypeInContext(gallivm->context);
200 LLVMValueRef shuffles[LP_MAX_VECTOR_LENGTH];
201 for (unsigned j = 0; j < 16; j++) {
202 unsigned comp = j % 4;
203 if (writemask & (1 << comp)) {
204 shuffles[j] = LLVMConstInt(i32t, 16 + j, 0); // new val
205 } else {
206 shuffles[j] = LLVMConstInt(i32t, j, 0); // cur val
207 }
208 }
209 cur = LLVMBuildShuffleVector(gallivm->builder, cur, vals[0],
210 LLVMConstVector(shuffles, 16), "");
211
212 LLVMBuildStore(gallivm->builder, cur, reg_storage);
213 }
214
215
216 static void
emit_load_ubo(struct lp_build_nir_context * bld_base,unsigned nc,unsigned bit_size,bool offset_is_uniform,LLVMValueRef index,LLVMValueRef offset,LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])217 emit_load_ubo(struct lp_build_nir_context *bld_base,
218 unsigned nc,
219 unsigned bit_size,
220 bool offset_is_uniform,
221 LLVMValueRef index,
222 LLVMValueRef offset,
223 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
224 {
225 struct lp_build_nir_aos_context *bld =
226 (struct lp_build_nir_aos_context *)bld_base;
227 LLVMBuilderRef builder = bld_base->base.gallivm->builder;
228 struct gallivm_state *gallivm = bld_base->base.gallivm;
229 struct lp_type type = bld_base->base.type;
230 LLVMValueRef res;
231
232 res = bld->bld_base.base.undef;
233 offset = LLVMBuildExtractElement(builder, offset,
234 lp_build_const_int32(gallivm, 0), "");
235 assert(LLVMIsConstant(offset));
236 unsigned offset_val = LLVMConstIntGetZExtValue(offset) >> 2;
237 for (unsigned chan = 0; chan < nc; ++chan) {
238 LLVMValueRef this_offset = lp_build_const_int32(gallivm,
239 offset_val + chan);
240
241 LLVMValueRef scalar_ptr = LLVMBuildGEP(builder, bld->consts_ptr,
242 &this_offset, 1, "");
243
244 LLVMValueRef scalar = LLVMBuildLoad(builder, scalar_ptr, "");
245
246 lp_build_name(scalar, "const[%u].%c", offset_val, "xyzw"[chan]);
247
248 LLVMValueRef swizzle = lp_build_const_int32(bld->bld_base.base.gallivm,
249 nc == 1 ? 0 : bld->swizzles[chan]);
250
251 res = LLVMBuildInsertElement(builder, res, scalar, swizzle, "");
252 }
253
254 if (type.length > 4) {
255 LLVMValueRef shuffles[LP_MAX_VECTOR_LENGTH];
256
257 for (unsigned chan = 0; chan < nc; ++chan) {
258 shuffles[chan] =
259 lp_build_const_int32(bld->bld_base.base.gallivm, chan);
260 }
261
262 for (unsigned i = nc; i < type.length; ++i) {
263 shuffles[i] = shuffles[i % nc];
264 }
265
266 res = LLVMBuildShuffleVector(builder, res, bld->bld_base.base.undef,
267 LLVMConstVector(shuffles, type.length),
268 "");
269 }
270
271 if (nc == 4)
272 swizzle_aos(bld_base, res, 0, 1, 2, 3);
273
274 result[0] = res;
275 }
276
277
278 static void
emit_tex(struct lp_build_nir_context * bld_base,struct lp_sampler_params * params)279 emit_tex(struct lp_build_nir_context *bld_base,
280 struct lp_sampler_params *params)
281 {
282 struct lp_build_nir_aos_context *bld =
283 (struct lp_build_nir_aos_context *)bld_base;
284 static const struct lp_derivatives derivs = { 0 };
285 params->type = bld_base->base.type;
286 params->texel[0] = bld->sampler->emit_fetch_texel(bld->sampler,
287 &bld->bld_base.base,
288 PIPE_TEXTURE_2D,
289 params->texture_index,
290 params->coords[0],
291 params->derivs ? params->derivs[0] : derivs,
292 LP_BLD_TEX_MODIFIER_NONE);
293 }
294
295
296 static void
emit_load_const(struct lp_build_nir_context * bld_base,const nir_load_const_instr * instr,LLVMValueRef outval[NIR_MAX_VEC_COMPONENTS])297 emit_load_const(struct lp_build_nir_context *bld_base,
298 const nir_load_const_instr *instr,
299 LLVMValueRef outval[NIR_MAX_VEC_COMPONENTS])
300 {
301 struct lp_build_nir_aos_context *bld = lp_nir_aos_context(bld_base);
302 struct gallivm_state *gallivm = bld_base->base.gallivm;
303 LLVMValueRef elems[4];
304 const int nc = instr->def.num_components;
305 bool do_swizzle = false;
306
307 if (nc == 4)
308 do_swizzle = true;
309
310 for (unsigned i = 0; i < nc; i++) {
311 int idx = do_swizzle ? bld->swizzles[i] : i;
312 elems[idx] = LLVMConstInt(LLVMInt32TypeInContext(gallivm->context),
313 instr->value[i].u32,
314 bld_base->base.type.sign ? 1 : 0);
315 }
316 outval[0] = LLVMConstVector(elems, nc);
317 }
318
319
320 void
lp_build_nir_aos(struct gallivm_state * gallivm,struct nir_shader * shader,struct lp_type type,const unsigned char swizzles[4],LLVMValueRef consts_ptr,const LLVMValueRef * inputs,LLVMValueRef * outputs,const struct lp_build_sampler_aos * sampler,const struct tgsi_shader_info * info)321 lp_build_nir_aos(struct gallivm_state *gallivm,
322 struct nir_shader *shader,
323 struct lp_type type,
324 const unsigned char swizzles[4],
325 LLVMValueRef consts_ptr,
326 const LLVMValueRef *inputs,
327 LLVMValueRef *outputs,
328 const struct lp_build_sampler_aos *sampler,
329 const struct tgsi_shader_info *info)
330 {
331 struct lp_build_nir_aos_context bld;
332
333 memset(&bld, 0, sizeof bld);
334 lp_build_context_init(&bld.bld_base.base, gallivm, type);
335 lp_build_context_init(&bld.bld_base.uint_bld, gallivm, lp_uint_type(type));
336 lp_build_context_init(&bld.bld_base.int_bld, gallivm, lp_int_type(type));
337
338 for (unsigned chan = 0; chan < 4; ++chan) {
339 bld.swizzles[chan] = swizzles[chan];
340 bld.inv_swizzles[swizzles[chan]] = chan;
341 }
342 bld.sampler = sampler;
343
344 bld.bld_base.shader = shader;
345
346 bld.inputs = inputs;
347 bld.outputs = outputs;
348 bld.consts_ptr = consts_ptr;
349
350 bld.bld_base.load_var = emit_load_var;
351 bld.bld_base.store_var = emit_store_var;
352 bld.bld_base.load_reg = emit_load_reg;
353 bld.bld_base.store_reg = emit_store_reg;
354 bld.bld_base.load_ubo = emit_load_ubo;
355 bld.bld_base.load_const = emit_load_const;
356
357 bld.bld_base.tex = emit_tex;
358 bld.bld_base.emit_var_decl = emit_var_decl;
359
360 lp_build_nir_llvm(&bld.bld_base, shader);
361 }
362