1 /**************************************************************************
2 *
3 * Copyright 2010-2021 VMware, Inc.
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
21 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27
28 #include <limits.h>
29
30 #include "pipe/p_defines.h"
31 #include "util/u_inlines.h"
32 #include "util/u_memory.h"
33 #include "util/u_pointer.h"
34 #include "util/format/u_format.h"
35 #include "util/u_dump.h"
36 #include "util/u_string.h"
37 #include "util/os_time.h"
38 #include "pipe/p_shader_tokens.h"
39 #include "draw/draw_context.h"
40 #include "tgsi/tgsi_dump.h"
41 #include "tgsi/tgsi_scan.h"
42 #include "tgsi/tgsi_parse.h"
43 #include "gallivm/lp_bld_type.h"
44 #include "gallivm/lp_bld_const.h"
45 #include "gallivm/lp_bld_conv.h"
46 #include "gallivm/lp_bld_init.h"
47 #include "gallivm/lp_bld_intr.h"
48 #include "gallivm/lp_bld_logic.h"
49 #include "gallivm/lp_bld_tgsi.h"
50 #include "gallivm/lp_bld_swizzle.h"
51 #include "gallivm/lp_bld_flow.h"
52 #include "gallivm/lp_bld_printf.h"
53 #include "gallivm/lp_bld_debug.h"
54 #include "gallivm/lp_bld_nir.h"
55
56 #include "lp_bld_alpha.h"
57 #include "lp_bld_blend.h"
58 #include "lp_bld_depth.h"
59 #include "lp_bld_interp.h"
60 #include "lp_context.h"
61 #include "lp_debug.h"
62 #include "lp_perf.h"
63 #include "lp_screen.h"
64 #include "lp_setup.h"
65 #include "lp_state.h"
66 #include "lp_tex_sample.h"
67 #include "lp_flush.h"
68 #include "lp_state_fs.h"
69
70
71 /**
72 * Sampler.
73 */
74 struct linear_sampler
75 {
76 struct lp_build_sampler_aos base;
77 LLVMValueRef texels_ptrs[LP_MAX_LINEAR_TEXTURES];
78 LLVMValueRef counter;
79 unsigned instance;
80 };
81
82
83 /**
84 * Provide texels to the TGSI translation.
85 *
86 * We don't actually do any texture sampling here, but simply hand the
87 * precomputed row of texels.
88 */
89 static LLVMValueRef
emit_fetch_texel_linear(const struct lp_build_sampler_aos * base,struct lp_build_context * bld,enum tgsi_texture_type target,unsigned unit,LLVMValueRef coords,const struct lp_derivatives derivs,enum lp_build_tex_modifier modifier)90 emit_fetch_texel_linear(const struct lp_build_sampler_aos *base,
91 struct lp_build_context *bld,
92 enum tgsi_texture_type target,
93 unsigned unit,
94 LLVMValueRef coords,
95 const struct lp_derivatives derivs,
96 enum lp_build_tex_modifier modifier)
97 {
98 struct linear_sampler *sampler = (struct linear_sampler *)base;
99
100 if (sampler->instance >= LP_MAX_LINEAR_TEXTURES) {
101 assert(FALSE);
102 return bld->undef;
103 }
104
105 /* Pointer to a row of texels */
106 LLVMValueRef texels_ptr = sampler->texels_ptrs[sampler->instance];
107
108 LLVMValueRef texel = lp_build_pointer_get(bld->gallivm->builder,
109 texels_ptr, sampler->counter);
110 assert(LLVMTypeOf(texel) == bld->vec_type);
111
112 /*
113 * We have a struct lp_linear_sampler instance per TEX instruction,
114 * _not_ per unit, as each TEX instruction will need separate storage
115 * for the texels.
116 */
117 (void)unit;
118 ++sampler->instance;
119
120 return texel;
121 }
122
123
124 /**
125 * Generates the main body of the fragment shader
126 * Supports generating code for 4 pixel blocks and individual pixels
127 */
128 static LLVMValueRef
llvm_fragment_body(struct lp_build_context * bld,struct lp_fragment_shader * shader,struct lp_fragment_shader_variant * variant,struct linear_sampler * sampler,LLVMValueRef * inputs_ptrs,LLVMValueRef consts_ptr,LLVMValueRef blend_color,LLVMValueRef alpha_ref,struct lp_type fs_type,LLVMValueRef dst)129 llvm_fragment_body(struct lp_build_context *bld,
130 struct lp_fragment_shader *shader,
131 struct lp_fragment_shader_variant *variant,
132 struct linear_sampler* sampler,
133 LLVMValueRef *inputs_ptrs,
134 LLVMValueRef consts_ptr,
135 LLVMValueRef blend_color,
136 LLVMValueRef alpha_ref,
137 struct lp_type fs_type,
138 LLVMValueRef dst)
139 {
140 static const unsigned char bgra_swizzles[4] = {2, 1, 0, 3};
141 LLVMValueRef inputs[PIPE_MAX_SHADER_INPUTS];
142 LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS];
143 LLVMBuilderRef builder = bld->gallivm->builder;
144 struct gallivm_state *gallivm = bld->gallivm;
145 LLVMValueRef result = NULL;
146
147 sampler->instance = 0;
148
149 /*
150 * Advance inputs
151 */
152 unsigned i;
153 for (i = 0; i < shader->info.base.num_inputs; ++i) {
154 inputs[i] =
155 lp_build_pointer_get(builder, inputs_ptrs[i], sampler->counter);
156 assert(LLVMTypeOf(inputs[i]) == bld->vec_type);
157 }
158 for ( ; i < PIPE_MAX_SHADER_INPUTS; ++i) {
159 inputs[i] = bld->undef;
160 }
161
162 for (i = 0; i < PIPE_MAX_SHADER_OUTPUTS; ++i) {
163 outputs[i] = bld->undef;
164 }
165
166 if (shader->base.type == PIPE_SHADER_IR_TGSI) {
167 lp_build_tgsi_aos(gallivm, shader->base.tokens, fs_type,
168 bgra_swizzles,
169 consts_ptr, inputs, outputs,
170 &sampler->base,
171 &shader->info.base);
172 } else {
173 nir_shader *clone = nir_shader_clone(NULL, shader->base.ir.nir);
174 lp_build_nir_aos(gallivm, clone, fs_type,
175 bgra_swizzles,
176 consts_ptr, inputs, outputs,
177 &sampler->base,
178 &shader->info.base);
179 ralloc_free(clone);
180 }
181
182 /*
183 * Blend output color
184 */
185 for (i = 0; i < shader->info.base.num_outputs; ++i) {
186 if (!outputs[i])
187 continue;
188
189 LLVMValueRef output = LLVMBuildLoad(builder, outputs[i], "");
190 lp_build_name(output, "output%u", i);
191
192 unsigned cbuf = shader->info.base.output_semantic_index[i];
193 lp_build_name(output, "cbuf%u", cbuf);
194
195 if (shader->info.base.output_semantic_name[i]
196 != TGSI_SEMANTIC_COLOR || cbuf != 0) {
197 continue;
198 }
199
200 /* Perform alpha test if necessary */
201 LLVMValueRef mask = NULL;
202 if (variant->key.alpha.enabled) {
203 LLVMTypeRef vec_type = lp_build_vec_type(gallivm, fs_type);
204 LLVMValueRef broadcast_alpha = lp_build_broadcast(gallivm, vec_type,
205 alpha_ref);
206
207 mask = lp_build_cmp(bld, variant->key.alpha.func, output,
208 broadcast_alpha);
209 /* XXX is 4 correct? */
210 mask = lp_build_swizzle_scalar_aos(bld, mask, bgra_swizzles[3], 4);
211
212 lp_build_name(mask, "alpha_test_mask");
213 }
214
215 LLVMValueRef src1 = lp_build_zero(gallivm, fs_type);
216
217 result = lp_build_blend_aos(gallivm,
218 &variant->key.blend,
219 variant->key.cbuf_format[i],
220 fs_type,
221 cbuf, /* rt */
222 output, /* src */
223 NULL, /* src_alpha */
224 src1, /* src1 */
225 NULL, /* src1_alpha */
226 dst,
227 mask,
228 blend_color, /* const_ */
229 NULL, /* const_alpha */
230 bgra_swizzles,
231 4);
232 }
233
234 return result;
235 }
236
237
238 /**
239 * Generate a function that executes the fragment shader in a linear fashion.
240 * The shader operates on unorm8[16] vectors.
241 * See lp_state_fs_analysis for the "linear" conditions.
242 */
243 void
llvmpipe_fs_variant_linear_llvm(struct llvmpipe_context * lp,struct lp_fragment_shader * shader,struct lp_fragment_shader_variant * variant)244 llvmpipe_fs_variant_linear_llvm(struct llvmpipe_context *lp,
245 struct lp_fragment_shader *shader,
246 struct lp_fragment_shader_variant *variant)
247 {
248 assert(shader->kind == LP_FS_KIND_BLIT_RGBA ||
249 shader->kind == LP_FS_KIND_BLIT_RGB1 ||
250 shader->kind == LP_FS_KIND_LLVM_LINEAR);
251
252 struct gallivm_state *gallivm = variant->gallivm;
253 LLVMTypeRef int8t = LLVMInt8TypeInContext(gallivm->context);
254 LLVMTypeRef int32t = LLVMInt32TypeInContext(gallivm->context);
255 LLVMTypeRef pint8t = LLVMPointerType(int8t, 0);
256 LLVMTypeRef pixelt = LLVMVectorType(int32t, 4);
257
258 // unorm8[16] vector type
259 struct lp_type fs_type;
260 memset(&fs_type, 0, sizeof fs_type);
261 fs_type.floating = FALSE;
262 fs_type.sign = FALSE;
263 fs_type.norm = TRUE;
264 fs_type.width = 8;
265 fs_type.length = 16;
266
267 if (LP_DEBUG & DEBUG_TGSI) {
268 if (shader->base.tokens) {
269 tgsi_dump(shader->base.tokens, 0);
270 }
271 if (shader->base.ir.nir) {
272 nir_print_shader(shader->base.ir.nir, stderr);
273 }
274 }
275
276 /*
277 * Generate the function prototype. Any change here must be reflected in
278 * lp_jit.h's lp_jit_frag_func function pointer type, and vice-versa.
279 */
280
281 char func_name[256];
282 snprintf(func_name, sizeof(func_name), "fs_variant_linear");
283
284 LLVMTypeRef ret_type = pint8t;
285 LLVMTypeRef arg_types[4];
286 arg_types[0] = variant->jit_linear_context_ptr_type; /* context */
287 arg_types[1] = int32t; /* x */
288 arg_types[2] = int32t; /* y */
289 arg_types[3] = int32t; /* width */
290
291 LLVMTypeRef func_type =
292 LLVMFunctionType(ret_type, arg_types, ARRAY_SIZE(arg_types), 0);
293
294 LLVMValueRef function =
295 LLVMAddFunction(gallivm->module, func_name, func_type);
296 LLVMSetFunctionCallConv(function, LLVMCCallConv);
297
298 variant->linear_function = function;
299
300 /* XXX: need to propagate noalias down into color param now we are
301 * passing a pointer-to-pointer?
302 */
303 for (unsigned i = 0; i < ARRAY_SIZE(arg_types); ++i) {
304 if (LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind) {
305 lp_add_function_attr(function, i + 1, LP_FUNC_ATTR_NOALIAS);
306 }
307 }
308
309 if (variant->gallivm->cache->data_size)
310 return;
311
312 LLVMValueRef context_ptr = LLVMGetParam(function, 0);
313 LLVMValueRef x = LLVMGetParam(function, 1);
314 LLVMValueRef y = LLVMGetParam(function, 2);
315 LLVMValueRef width = LLVMGetParam(function, 3);
316
317 lp_build_name(context_ptr, "context");
318 lp_build_name(x, "x");
319 lp_build_name(y, "y");
320 lp_build_name(width, "width");
321
322 /*
323 * Function body
324 */
325
326 LLVMBasicBlockRef block =
327 LLVMAppendBasicBlockInContext(gallivm->context, function, "entry");
328 LLVMBuilderRef builder = gallivm->builder;
329
330 LLVMPositionBuilderAtEnd(builder, block);
331
332 struct lp_build_context bld;
333 lp_build_context_init(&bld, gallivm, fs_type);
334
335 /*
336 * Get context data
337 */
338 LLVMValueRef consts_ptr =
339 lp_jit_linear_context_constants(gallivm, context_ptr);
340 LLVMValueRef interpolators_ptr =
341 lp_jit_linear_context_inputs(gallivm, context_ptr);
342 LLVMValueRef samplers_ptr =
343 lp_jit_linear_context_tex(gallivm, context_ptr);
344
345 LLVMValueRef color0_ptr =
346 lp_jit_linear_context_color0(gallivm, context_ptr);
347 color0_ptr = LLVMBuildLoad(builder, color0_ptr, "");
348 color0_ptr = LLVMBuildBitCast(builder, color0_ptr,
349 LLVMPointerType(bld.vec_type, 0), "");
350
351 LLVMValueRef blend_color =
352 lp_jit_linear_context_blend_color(gallivm, context_ptr);
353 blend_color = LLVMBuildLoad(builder, blend_color, "");
354 blend_color = lp_build_broadcast(gallivm, LLVMVectorType(int32t, 4),
355 blend_color);
356 blend_color = LLVMBuildBitCast(builder, blend_color,
357 LLVMVectorType(int8t, 16), "");
358
359 LLVMValueRef alpha_ref =
360 lp_jit_linear_context_alpha_ref(gallivm, context_ptr);
361 alpha_ref = LLVMBuildLoad(builder, alpha_ref, "");
362
363 /*
364 * Invoke the input interpolators
365 */
366 LLVMValueRef inputs_ptrs[LP_MAX_LINEAR_INPUTS];
367
368 for (unsigned attrib = 0; attrib < shader->info.base.num_inputs; ++attrib) {
369 assert(attrib < LP_MAX_LINEAR_INPUTS);
370 if (attrib >= LP_MAX_LINEAR_INPUTS) {
371 break;
372 }
373
374 LLVMValueRef index = LLVMConstInt(int32t, attrib, 0);
375
376 LLVMValueRef elem =
377 lp_build_array_get(bld.gallivm, interpolators_ptr, index);
378 assert(LLVMGetTypeKind(LLVMTypeOf(elem)) == LLVMPointerTypeKind);
379
380 LLVMValueRef fetch_ptr = lp_build_pointer_get(builder, elem,
381 LLVMConstInt(int32t, 0, 0));
382 assert(LLVMGetTypeKind(LLVMTypeOf(fetch_ptr)) == LLVMPointerTypeKind);
383
384 /* Pointer to a row of interpolated inputs */
385 elem = LLVMBuildBitCast(builder, elem, pint8t, "");
386 LLVMValueRef inputs_ptr = LLVMBuildCall(builder, fetch_ptr, &elem, 1, "");
387 assert(LLVMGetTypeKind(LLVMTypeOf(inputs_ptr)) == LLVMPointerTypeKind);
388
389 /* Mark the function read-only so that LLVM can optimize it away */
390 lp_add_function_attr(inputs_ptr, -1, LP_FUNC_ATTR_READONLY);
391 lp_add_function_attr(inputs_ptr, -1, LP_FUNC_ATTR_NOUNWIND);
392
393 lp_build_name(inputs_ptr, "input%u_ptr", attrib);
394
395 inputs_ptrs[attrib] = inputs_ptr;
396 }
397
398 /*
399 * Invoke and hook up the texture samplers.
400 */
401
402 struct linear_sampler sampler;
403 memset(&sampler, 0, sizeof sampler);
404 sampler.base.emit_fetch_texel = &emit_fetch_texel_linear;
405
406 for (unsigned attrib = 0; attrib < shader->info.num_texs; ++attrib) {
407 assert(attrib < LP_MAX_LINEAR_TEXTURES);
408 if (attrib >= LP_MAX_LINEAR_TEXTURES) {
409 break;
410 }
411
412 LLVMValueRef index = LLVMConstInt(int32t, attrib, 0);
413
414 LLVMValueRef elem = lp_build_array_get(bld.gallivm, samplers_ptr, index);
415 assert(LLVMGetTypeKind(LLVMTypeOf(elem)) == LLVMPointerTypeKind);
416
417 LLVMValueRef fetch_ptr =
418 lp_build_pointer_get(builder, elem, LLVMConstInt(int32t, 0, 0));
419 assert(LLVMGetTypeKind(LLVMTypeOf(fetch_ptr)) == LLVMPointerTypeKind);
420
421 /* Pointer to a row of texels */
422 elem = LLVMBuildBitCast(builder, elem, pint8t, "");
423 LLVMValueRef texels_ptr = LLVMBuildCall(builder, fetch_ptr, &elem, 1, "");
424 assert(LLVMGetTypeKind(LLVMTypeOf(texels_ptr)) == LLVMPointerTypeKind);
425
426 /* Mark the function read-only so that LLVM can optimize it away */
427 lp_add_function_attr(texels_ptr, -1, LP_FUNC_ATTR_READONLY);
428 lp_add_function_attr(texels_ptr, -1, LP_FUNC_ATTR_NOUNWIND);
429
430 lp_build_name(texels_ptr, "tex%u_ptr", attrib);
431
432 sampler.texels_ptrs[attrib] = texels_ptr;
433 }
434
435 /* excess = width & 0x3 */
436 LLVMValueRef excess =
437 LLVMBuildAnd(builder, width, LLVMConstInt(int32t, 3, 0), "");
438 /* width *= 4 */
439 width = LLVMBuildLShr(builder, width, LLVMConstInt(int32t, 2, 0), "");
440
441 /* Loop over blocks of 4 pixels */
442 /* for loop.counter = 0; loop.counter < width; loop.counter++) { */
443 struct lp_build_for_loop_state loop;
444 lp_build_for_loop_begin(&loop, gallivm, LLVMConstInt(int32t, 0, 0),
445 LLVMIntULT, width, LLVMConstInt(int32t, 1, 0));
446 {
447 LLVMValueRef value;
448 sampler.counter = loop.counter;
449
450 /* Read 4 pixels */
451 value = lp_build_pointer_get_unaligned(builder, color0_ptr,
452 loop.counter, 4);
453
454 /* Perform fragment shader body */
455 value = llvm_fragment_body(&bld, shader, variant, &sampler, inputs_ptrs,
456 consts_ptr, blend_color, alpha_ref, fs_type,
457 value);
458
459 /* Write 4 pixels */
460 lp_build_pointer_set_unaligned(builder, color0_ptr, loop.counter,
461 value, 4);
462 }
463 lp_build_for_loop_end(&loop);
464
465 /* Compute the edge pixels (width % 4) */
466 struct lp_build_if_state ifstate;
467 lp_build_if(&ifstate, gallivm, LLVMBuildICmp(builder, LLVMIntNE, excess,
468 LLVMConstInt(int32t, 0, 0), ""));
469 {
470 struct lp_build_loop_state loop_read, loop_write;
471 LLVMValueRef buf, elem, result, pixel_ptr;
472 LLVMValueRef buf_ptr = lp_build_alloca(gallivm, pixelt, "");
473
474 sampler.counter = width;
475
476 /* Get the i32* pixel pointer from the <i16x8>* element pointer */
477 pixel_ptr = LLVMBuildGEP(gallivm->builder, color0_ptr, &width, 1, "");
478 pixel_ptr = LLVMBuildBitCast(gallivm->builder, pixel_ptr,
479 LLVMPointerType(int32t, 0), "");
480
481 /* Copy individual pixels from memory to local buffer */
482 lp_build_loop_begin(&loop_read, gallivm, LLVMConstInt(int32t, 0, 0));
483 {
484 elem = lp_build_pointer_get(gallivm->builder,
485 pixel_ptr, loop_read.counter);
486
487 buf = LLVMBuildLoad(gallivm->builder, buf_ptr, "");
488 buf = LLVMBuildInsertElement(builder, buf, elem,
489 loop_read.counter, "");
490 LLVMBuildStore(builder, buf, buf_ptr);
491 }
492 lp_build_loop_end_cond(&loop_read, excess,
493 LLVMConstInt(int32t, 1, 0), LLVMIntUGE);
494
495 /* Perform fragment shader body */
496 buf = LLVMBuildLoad(gallivm->builder, buf_ptr, "");
497 buf = LLVMBuildBitCast(builder, buf, bld.vec_type, "");
498
499 result = llvm_fragment_body(&bld, shader, variant, &sampler,
500 inputs_ptrs, consts_ptr, blend_color,
501 alpha_ref, fs_type, buf);
502 result = LLVMBuildBitCast(builder, result, pixelt, "");
503
504 /* Write individual pixels from local buffer to the memory */
505 lp_build_loop_begin(&loop_write, gallivm, LLVMConstInt(int32t, 0, 0));
506 {
507 elem = LLVMBuildExtractElement(builder, result,
508 loop_write.counter, "");
509
510 lp_build_pointer_set(gallivm->builder, pixel_ptr,
511 loop_write.counter, elem);
512 }
513 lp_build_loop_end_cond(&loop_write, excess,
514 LLVMConstInt(int32t, 1, 0), LLVMIntUGE);
515 }
516 lp_build_endif(&ifstate);
517
518 color0_ptr = LLVMBuildBitCast(builder, color0_ptr, pint8t, "");
519
520 LLVMBuildRet(builder, color0_ptr);
521
522 gallivm_verify_function(gallivm, function);
523 }
524