1 /**************************************************************************
2 *
3 * Copyright 2009 VMware, Inc.
4 * Copyright 2007 VMware, Inc.
5 * All Rights Reserved.
6 *
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the
9 * "Software"), to deal in the Software without restriction, including
10 * without limitation the rights to use, copy, modify, merge, publish,
11 * distribute, sub license, and/or sell copies of the Software, and to
12 * permit persons to whom the Software is furnished to do so, subject to
13 * the following conditions:
14 *
15 * The above copyright notice and this permission notice (including the
16 * next paragraph) shall be included in all copies or substantial portions
17 * of the Software.
18 *
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
20 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
21 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
22 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
23 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
24 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
25 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 *
27 **************************************************************************/
28
29 /**
30 * @file
31 * Code generate the whole fragment pipeline.
32 *
33 * The fragment pipeline consists of the following stages:
34 * - early depth test
35 * - fragment shader
36 * - alpha test
37 * - depth/stencil test
38 * - blending
39 *
40 * This file has only the glue to assemble the fragment pipeline. The actual
41 * plumbing of converting Gallium state into LLVM IR is done elsewhere, in the
42 * lp_bld_*.[ch] files, and in a complete generic and reusable way. Here we
43 * muster the LLVM JIT execution engine to create a function that follows an
44 * established binary interface and that can be called from C directly.
45 *
46 * A big source of complexity here is that we often want to run different
47 * stages with different precisions and data types and precisions. For example,
48 * the fragment shader needs typically to be done in floats, but the
49 * depth/stencil test and blending is better done in the type that most closely
50 * matches the depth/stencil and color buffer respectively.
51 *
52 * Since the width of a SIMD vector register stays the same regardless of the
53 * element type, different types imply different number of elements, so we must
54 * code generate more instances of the stages with larger types to be able to
55 * feed/consume the stages with smaller types.
56 *
57 * @author Jose Fonseca <jfonseca@vmware.com>
58 */
59
60 #include <limits.h>
61 #include "pipe/p_defines.h"
62 #include "util/u_inlines.h"
63 #include "util/u_memory.h"
64 #include "util/u_pointer.h"
65 #include "util/format/u_format.h"
66 #include "util/u_dump.h"
67 #include "util/u_string.h"
68 #include "util/u_dual_blend.h"
69 #include "util/u_upload_mgr.h"
70 #include "util/os_time.h"
71 #include "pipe/p_shader_tokens.h"
72 #include "draw/draw_context.h"
73 #include "tgsi/tgsi_dump.h"
74 #include "tgsi/tgsi_scan.h"
75 #include "tgsi/tgsi_parse.h"
76 #include "gallivm/lp_bld_type.h"
77 #include "gallivm/lp_bld_const.h"
78 #include "gallivm/lp_bld_conv.h"
79 #include "gallivm/lp_bld_init.h"
80 #include "gallivm/lp_bld_intr.h"
81 #include "gallivm/lp_bld_logic.h"
82 #include "gallivm/lp_bld_tgsi.h"
83 #include "gallivm/lp_bld_nir.h"
84 #include "gallivm/lp_bld_swizzle.h"
85 #include "gallivm/lp_bld_flow.h"
86 #include "gallivm/lp_bld_debug.h"
87 #include "gallivm/lp_bld_arit.h"
88 #include "gallivm/lp_bld_bitarit.h"
89 #include "gallivm/lp_bld_pack.h"
90 #include "gallivm/lp_bld_format.h"
91 #include "gallivm/lp_bld_quad.h"
92 #include "gallivm/lp_bld_gather.h"
93
94 #include "lp_bld_alpha.h"
95 #include "lp_bld_blend.h"
96 #include "lp_bld_depth.h"
97 #include "lp_bld_interp.h"
98 #include "lp_context.h"
99 #include "lp_debug.h"
100 #include "lp_perf.h"
101 #include "lp_setup.h"
102 #include "lp_state.h"
103 #include "lp_tex_sample.h"
104 #include "lp_flush.h"
105 #include "lp_state_fs.h"
106 #include "lp_rast.h"
107 #include "nir/nir_to_tgsi_info.h"
108
109 #include "lp_screen.h"
110 #include "compiler/nir/nir_serialize.h"
111 #include "util/mesa-sha1.h"
112
113
114 /** Fragment shader number (for debugging) */
115 static unsigned fs_no = 0;
116
117
118 static void
119 load_unswizzled_block(struct gallivm_state *gallivm,
120 LLVMValueRef base_ptr,
121 LLVMValueRef stride,
122 unsigned block_width,
123 unsigned block_height,
124 LLVMValueRef* dst,
125 struct lp_type dst_type,
126 unsigned dst_count,
127 unsigned dst_alignment);
128 /**
129 * Checks if a format description is an arithmetic format
130 *
131 * A format which has irregular channel sizes such as R3_G3_B2 or R5_G6_B5.
132 */
133 static inline boolean
is_arithmetic_format(const struct util_format_description * format_desc)134 is_arithmetic_format(const struct util_format_description *format_desc)
135 {
136 boolean arith = false;
137
138 for (unsigned i = 0; i < format_desc->nr_channels; ++i) {
139 arith |= format_desc->channel[i].size != format_desc->channel[0].size;
140 arith |= (format_desc->channel[i].size % 8) != 0;
141 }
142
143 return arith;
144 }
145
146
147 /**
148 * Checks if this format requires special handling due to required expansion
149 * to floats for blending, and furthermore has "natural" packed AoS -> unpacked
150 * SoA conversion.
151 */
152 static inline boolean
format_expands_to_float_soa(const struct util_format_description * format_desc)153 format_expands_to_float_soa(const struct util_format_description *format_desc)
154 {
155 if (format_desc->format == PIPE_FORMAT_R11G11B10_FLOAT ||
156 format_desc->colorspace == UTIL_FORMAT_COLORSPACE_SRGB) {
157 return true;
158 }
159 return false;
160 }
161
162
163 /**
164 * Retrieves the type representing the memory layout for a format
165 *
166 * e.g. RGBA16F = 4x half-float and R3G3B2 = 1x byte
167 */
168 static inline void
lp_mem_type_from_format_desc(const struct util_format_description * format_desc,struct lp_type * type)169 lp_mem_type_from_format_desc(const struct util_format_description *format_desc,
170 struct lp_type* type)
171 {
172 unsigned i;
173 unsigned chan;
174
175 if (format_expands_to_float_soa(format_desc)) {
176 /* just make this a uint with width of block */
177 type->floating = false;
178 type->fixed = false;
179 type->sign = false;
180 type->norm = false;
181 type->width = format_desc->block.bits;
182 type->length = 1;
183 return;
184 }
185
186 for (i = 0; i < 4; i++) {
187 if (format_desc->channel[i].type != UTIL_FORMAT_TYPE_VOID)
188 break;
189 }
190 chan = i;
191
192 memset(type, 0, sizeof(struct lp_type));
193 type->floating = format_desc->channel[chan].type == UTIL_FORMAT_TYPE_FLOAT;
194 type->fixed = format_desc->channel[chan].type == UTIL_FORMAT_TYPE_FIXED;
195 type->sign = format_desc->channel[chan].type != UTIL_FORMAT_TYPE_UNSIGNED;
196 type->norm = format_desc->channel[chan].normalized;
197
198 if (is_arithmetic_format(format_desc)) {
199 type->width = 0;
200 type->length = 1;
201
202 for (unsigned i = 0; i < format_desc->nr_channels; ++i) {
203 type->width += format_desc->channel[i].size;
204 }
205 } else {
206 type->width = format_desc->channel[chan].size;
207 type->length = format_desc->nr_channels;
208 }
209 }
210
211 /**
212 * Expand the relevant bits of mask_input to a n*4-dword mask for the
213 * n*four pixels in n 2x2 quads. This will set the n*four elements of the
214 * quad mask vector to 0 or ~0.
215 * Grouping is 01, 23 for 2 quad mode hence only 0 and 2 are valid
216 * quad arguments with fs length 8.
217 *
218 * \param first_quad which quad(s) of the quad group to test, in [0,3]
219 * \param mask_input bitwise mask for the whole 4x4 stamp
220 */
221 static LLVMValueRef
generate_quad_mask(struct gallivm_state * gallivm,struct lp_type fs_type,unsigned first_quad,unsigned sample,LLVMValueRef mask_input)222 generate_quad_mask(struct gallivm_state *gallivm,
223 struct lp_type fs_type,
224 unsigned first_quad,
225 unsigned sample,
226 LLVMValueRef mask_input) /* int64 */
227 {
228 LLVMBuilderRef builder = gallivm->builder;
229 LLVMTypeRef i32t = LLVMInt32TypeInContext(gallivm->context);
230 LLVMValueRef bits[16];
231 LLVMValueRef mask, bits_vec;
232
233 /*
234 * XXX: We'll need a different path for 16 x u8
235 */
236 assert(fs_type.width == 32);
237 assert(fs_type.length <= ARRAY_SIZE(bits));
238 struct lp_type mask_type = lp_int_type(fs_type);
239
240 /*
241 * mask_input >>= (quad * 4)
242 */
243 int shift;
244 switch (first_quad) {
245 case 0:
246 shift = 0;
247 break;
248 case 1:
249 assert(fs_type.length == 4);
250 shift = 2;
251 break;
252 case 2:
253 shift = 8;
254 break;
255 case 3:
256 assert(fs_type.length == 4);
257 shift = 10;
258 break;
259 default:
260 assert(0);
261 shift = 0;
262 }
263
264 mask_input = LLVMBuildLShr(builder, mask_input,
265 lp_build_const_int64(gallivm, 16 * sample), "");
266 mask_input = LLVMBuildTrunc(builder, mask_input, i32t, "");
267 mask_input = LLVMBuildAnd(builder, mask_input,
268 lp_build_const_int32(gallivm, 0xffff), "");
269 mask_input = LLVMBuildLShr(builder, mask_input,
270 LLVMConstInt(i32t, shift, 0), "");
271
272 /*
273 * mask = { mask_input & (1 << i), for i in [0,3] }
274 */
275 mask = lp_build_broadcast(gallivm,
276 lp_build_vec_type(gallivm, mask_type),
277 mask_input);
278
279 for (int i = 0; i < fs_type.length / 4; i++) {
280 unsigned j = 2 * (i % 2) + (i / 2) * 8;
281 bits[4*i + 0] = LLVMConstInt(i32t, 1ULL << (j + 0), 0);
282 bits[4*i + 1] = LLVMConstInt(i32t, 1ULL << (j + 1), 0);
283 bits[4*i + 2] = LLVMConstInt(i32t, 1ULL << (j + 4), 0);
284 bits[4*i + 3] = LLVMConstInt(i32t, 1ULL << (j + 5), 0);
285 }
286 bits_vec = LLVMConstVector(bits, fs_type.length);
287 mask = LLVMBuildAnd(builder, mask, bits_vec, "");
288
289 /*
290 * mask = mask == bits ? ~0 : 0
291 */
292 mask = lp_build_compare(gallivm,
293 mask_type, PIPE_FUNC_EQUAL,
294 mask, bits_vec);
295
296 return mask;
297 }
298
299
300 #define EARLY_DEPTH_TEST 0x1
301 #define LATE_DEPTH_TEST 0x2
302 #define EARLY_DEPTH_WRITE 0x4
303 #define LATE_DEPTH_WRITE 0x8
304 #define EARLY_DEPTH_TEST_INFERRED 0x10 //only with EARLY_DEPTH_TEST
305
306
307 static int
find_output_by_semantic(const struct tgsi_shader_info * info,enum tgsi_semantic semantic,unsigned index)308 find_output_by_semantic(const struct tgsi_shader_info *info,
309 enum tgsi_semantic semantic,
310 unsigned index)
311 {
312 for (int i = 0; i < info->num_outputs; i++)
313 if (info->output_semantic_name[i] == semantic &&
314 info->output_semantic_index[i] == index)
315 return i;
316
317 return -1;
318 }
319
320
321 /**
322 * Fetch the specified lp_jit_viewport structure for a given viewport_index.
323 */
324 static LLVMValueRef
lp_llvm_viewport(LLVMValueRef context_ptr,struct gallivm_state * gallivm,LLVMValueRef viewport_index)325 lp_llvm_viewport(LLVMValueRef context_ptr,
326 struct gallivm_state *gallivm,
327 LLVMValueRef viewport_index)
328 {
329 LLVMBuilderRef builder = gallivm->builder;
330 LLVMValueRef ptr;
331 LLVMValueRef res;
332 struct lp_type viewport_type =
333 lp_type_float_vec(32, 32 * LP_JIT_VIEWPORT_NUM_FIELDS);
334
335 ptr = lp_jit_context_viewports(gallivm, context_ptr);
336 ptr = LLVMBuildPointerCast(builder, ptr,
337 LLVMPointerType(lp_build_vec_type(gallivm, viewport_type), 0), "");
338
339 res = lp_build_pointer_get(builder, ptr, viewport_index);
340
341 return res;
342 }
343
344
345 static LLVMValueRef
lp_build_depth_clamp(struct gallivm_state * gallivm,LLVMBuilderRef builder,bool depth_clamp,bool restrict_depth,struct lp_type type,LLVMValueRef context_ptr,LLVMValueRef thread_data_ptr,LLVMValueRef z)346 lp_build_depth_clamp(struct gallivm_state *gallivm,
347 LLVMBuilderRef builder,
348 bool depth_clamp,
349 bool restrict_depth,
350 struct lp_type type,
351 LLVMValueRef context_ptr,
352 LLVMValueRef thread_data_ptr,
353 LLVMValueRef z)
354 {
355 LLVMValueRef viewport, min_depth, max_depth;
356 LLVMValueRef viewport_index;
357 struct lp_build_context f32_bld;
358
359 assert(type.floating);
360 lp_build_context_init(&f32_bld, gallivm, type);
361
362 if (restrict_depth)
363 z = lp_build_clamp(&f32_bld, z, f32_bld.zero, f32_bld.one);
364
365 if (!depth_clamp)
366 return z;
367
368 /*
369 * Assumes clamping of the viewport index will occur in setup/gs. Value
370 * is passed through the rasterization stage via lp_rast_shader_inputs.
371 *
372 * See: draw_clamp_viewport_idx and lp_clamp_viewport_idx for clamping
373 * semantics.
374 */
375 viewport_index = lp_jit_thread_data_raster_state_viewport_index(gallivm,
376 thread_data_ptr);
377
378 /*
379 * Load the min and max depth from the lp_jit_context.viewports
380 * array of lp_jit_viewport structures.
381 */
382 viewport = lp_llvm_viewport(context_ptr, gallivm, viewport_index);
383
384 /* viewports[viewport_index].min_depth */
385 min_depth = LLVMBuildExtractElement(builder, viewport,
386 lp_build_const_int32(gallivm, LP_JIT_VIEWPORT_MIN_DEPTH), "");
387 min_depth = lp_build_broadcast_scalar(&f32_bld, min_depth);
388
389 /* viewports[viewport_index].max_depth */
390 max_depth = LLVMBuildExtractElement(builder, viewport,
391 lp_build_const_int32(gallivm, LP_JIT_VIEWPORT_MAX_DEPTH), "");
392 max_depth = lp_build_broadcast_scalar(&f32_bld, max_depth);
393
394 /*
395 * Clamp to the min and max depth values for the given viewport.
396 */
397 return lp_build_clamp(&f32_bld, z, min_depth, max_depth);
398 }
399
400
401 static void
lp_build_sample_alpha_to_coverage(struct gallivm_state * gallivm,struct lp_type type,unsigned coverage_samples,LLVMValueRef num_loop,LLVMValueRef loop_counter,LLVMValueRef coverage_mask_store,LLVMValueRef alpha)402 lp_build_sample_alpha_to_coverage(struct gallivm_state *gallivm,
403 struct lp_type type,
404 unsigned coverage_samples,
405 LLVMValueRef num_loop,
406 LLVMValueRef loop_counter,
407 LLVMValueRef coverage_mask_store,
408 LLVMValueRef alpha)
409 {
410 struct lp_build_context bld;
411 LLVMBuilderRef builder = gallivm->builder;
412 float step = 1.0 / coverage_samples;
413
414 lp_build_context_init(&bld, gallivm, type);
415 for (unsigned s = 0; s < coverage_samples; s++) {
416 LLVMValueRef alpha_ref_value = lp_build_const_vec(gallivm, type, step * s);
417 LLVMValueRef test = lp_build_cmp(&bld, PIPE_FUNC_GREATER, alpha, alpha_ref_value);
418
419 LLVMValueRef s_mask_idx = LLVMBuildMul(builder, lp_build_const_int32(gallivm, s), num_loop, "");
420 s_mask_idx = LLVMBuildAdd(builder, s_mask_idx, loop_counter, "");
421 LLVMValueRef s_mask_ptr = LLVMBuildGEP(builder, coverage_mask_store, &s_mask_idx, 1, "");
422 LLVMValueRef s_mask = LLVMBuildLoad(builder, s_mask_ptr, "");
423 s_mask = LLVMBuildAnd(builder, s_mask, test, "");
424 LLVMBuildStore(builder, s_mask, s_mask_ptr);
425 }
426 };
427
428
429 struct lp_build_fs_llvm_iface {
430 struct lp_build_fs_iface base;
431 struct lp_build_interp_soa_context *interp;
432 struct lp_build_for_loop_state *loop_state;
433 LLVMValueRef mask_store;
434 LLVMValueRef sample_id;
435 LLVMValueRef color_ptr_ptr;
436 LLVMValueRef color_stride_ptr;
437 LLVMValueRef color_sample_stride_ptr;
438 LLVMValueRef zs_base_ptr;
439 LLVMValueRef zs_stride;
440 LLVMValueRef zs_sample_stride;
441 const struct lp_fragment_shader_variant_key *key;
442 };
443
444
445 static LLVMValueRef
fs_interp(const struct lp_build_fs_iface * iface,struct lp_build_context * bld,unsigned attrib,unsigned chan,bool centroid,bool sample,LLVMValueRef attrib_indir,LLVMValueRef offsets[2])446 fs_interp(const struct lp_build_fs_iface *iface,
447 struct lp_build_context *bld,
448 unsigned attrib, unsigned chan,
449 bool centroid, bool sample,
450 LLVMValueRef attrib_indir,
451 LLVMValueRef offsets[2])
452 {
453 struct lp_build_fs_llvm_iface *fs_iface = (struct lp_build_fs_llvm_iface *)iface;
454 struct lp_build_interp_soa_context *interp = fs_iface->interp;
455 unsigned loc = TGSI_INTERPOLATE_LOC_CENTER;
456 if (centroid)
457 loc = TGSI_INTERPOLATE_LOC_CENTROID;
458 if (sample)
459 loc = TGSI_INTERPOLATE_LOC_SAMPLE;
460
461 return lp_build_interp_soa(interp, bld->gallivm, fs_iface->loop_state->counter,
462 fs_iface->mask_store,
463 attrib, chan, loc, attrib_indir, offsets);
464 }
465
466 /* Convert depth-stencil format to a single component one, returning
467 * PIPE_FORMAT_NONE if it doesn't contain the required component. */
468 static enum pipe_format
select_zs_component_format(enum pipe_format format,bool fetch_stencil)469 select_zs_component_format(enum pipe_format format,
470 bool fetch_stencil)
471 {
472 const struct util_format_description* desc = util_format_description(format);
473 if (fetch_stencil && !util_format_has_stencil(desc))
474 return PIPE_FORMAT_NONE;
475 if (!fetch_stencil && !util_format_has_depth(desc))
476 return PIPE_FORMAT_NONE;
477
478 switch (format) {
479 case PIPE_FORMAT_Z24_UNORM_S8_UINT:
480 return fetch_stencil ? PIPE_FORMAT_X24S8_UINT : PIPE_FORMAT_Z24X8_UNORM;
481 case PIPE_FORMAT_S8_UINT_Z24_UNORM:
482 return fetch_stencil ? PIPE_FORMAT_S8X24_UINT : PIPE_FORMAT_X8Z24_UNORM;
483 case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT:
484 return fetch_stencil ? PIPE_FORMAT_X32_S8X24_UINT : format;
485 default:
486 return format;
487 }
488 }
489
490 static void
fs_fb_fetch(const struct lp_build_fs_iface * iface,struct lp_build_context * bld,int location,LLVMValueRef result[4])491 fs_fb_fetch(const struct lp_build_fs_iface *iface,
492 struct lp_build_context *bld,
493 int location,
494 LLVMValueRef result[4])
495 {
496 struct lp_build_fs_llvm_iface *fs_iface = (struct lp_build_fs_llvm_iface *)iface;
497 struct gallivm_state *gallivm = bld->gallivm;
498 LLVMBuilderRef builder = gallivm->builder;
499 const struct lp_fragment_shader_variant_key *key = fs_iface->key;
500
501 LLVMValueRef buf_ptr;
502 LLVMValueRef stride;
503 enum pipe_format buf_format;
504
505 const bool fetch_stencil = location == FRAG_RESULT_STENCIL;
506 const bool fetch_zs = fetch_stencil || location == FRAG_RESULT_DEPTH;
507 if (fetch_zs) {
508 buf_ptr = fs_iface->zs_base_ptr;
509 stride = fs_iface->zs_stride;
510 buf_format = select_zs_component_format(key->zsbuf_format, fetch_stencil);
511 }
512 else {
513 assert(location >= FRAG_RESULT_DATA0 && location <= FRAG_RESULT_DATA7);
514 const int cbuf = location - FRAG_RESULT_DATA0;
515 LLVMValueRef index = lp_build_const_int32(gallivm, cbuf);
516
517 buf_ptr = LLVMBuildLoad(builder, LLVMBuildGEP(builder, fs_iface->color_ptr_ptr, &index, 1, ""), "");
518 stride = LLVMBuildLoad(builder, LLVMBuildGEP(builder, fs_iface->color_stride_ptr, &index, 1, ""), "");
519 buf_format = key->cbuf_format[cbuf];
520 }
521
522 const struct util_format_description* out_format_desc = util_format_description(buf_format);
523 if (out_format_desc->format == PIPE_FORMAT_NONE) {
524 result[0] = result[1] = result[2] = result[3] = bld->undef;
525 return;
526 }
527
528 unsigned block_size = bld->type.length;
529 unsigned block_height = key->resource_1d ? 1 : 2;
530 unsigned block_width = block_size / block_height;
531
532 if (key->multisample) {
533 LLVMValueRef sample_stride;
534
535 if (fetch_zs) {
536 sample_stride = fs_iface->zs_sample_stride;
537 }
538 else {
539 LLVMValueRef index = lp_build_const_int32(gallivm, location - FRAG_RESULT_DATA0);
540 sample_stride = LLVMBuildLoad(builder,
541 LLVMBuildGEP(builder, fs_iface->color_sample_stride_ptr,
542 &index, 1, ""), "");
543 }
544
545 LLVMValueRef sample_offset = LLVMBuildMul(builder, sample_stride, fs_iface->sample_id, "");
546 buf_ptr = LLVMBuildGEP(builder, buf_ptr, &sample_offset, 1, "");
547 }
548
549 /* fragment shader executes on 4x4 blocks. depending on vector width it can
550 * execute 2 or 4 iterations. only move to the next row once the top row
551 * has completed 8 wide 1 iteration, 4 wide 2 iterations */
552 LLVMValueRef x_offset = NULL, y_offset = NULL;
553 if (!key->resource_1d) {
554 LLVMValueRef counter = fs_iface->loop_state->counter;
555
556 if (block_size == 4) {
557 x_offset = LLVMBuildShl(builder,
558 LLVMBuildAnd(builder, fs_iface->loop_state->counter, lp_build_const_int32(gallivm, 1), ""),
559 lp_build_const_int32(gallivm, 1), "");
560 counter = LLVMBuildLShr(builder, fs_iface->loop_state->counter, lp_build_const_int32(gallivm, 1), "");
561 }
562 y_offset = LLVMBuildMul(builder, counter, lp_build_const_int32(gallivm, 2), "");
563 }
564
565 LLVMValueRef offsets[4 * 4];
566 for (unsigned i = 0; i < block_size; i++) {
567 unsigned x = i % block_width;
568 unsigned y = i / block_width;
569
570 if (block_size == 8) {
571 /* remap the raw slots into the fragment shader execution mode. */
572 /* this math took me way too long to work out, I'm sure it's overkill. */
573 x = (i & 1) + ((i >> 2) << 1);
574 if (!key->resource_1d)
575 y = (i & 2) >> 1;
576 }
577
578 LLVMValueRef x_val;
579 if (x_offset) {
580 x_val = LLVMBuildAdd(builder, lp_build_const_int32(gallivm, x), x_offset, "");
581 x_val = LLVMBuildMul(builder, x_val, lp_build_const_int32(gallivm, out_format_desc->block.bits / 8), "");
582 } else {
583 x_val = lp_build_const_int32(gallivm, x * (out_format_desc->block.bits / 8));
584 }
585
586 LLVMValueRef y_val = lp_build_const_int32(gallivm, y);
587 if (y_offset)
588 y_val = LLVMBuildAdd(builder, y_val, y_offset, "");
589 y_val = LLVMBuildMul(builder, y_val, stride, "");
590
591 offsets[i] = LLVMBuildAdd(builder, x_val, y_val, "");
592 }
593 LLVMValueRef offset = lp_build_gather_values(gallivm, offsets, block_size);
594
595 struct lp_type texel_type = bld->type;
596 if (out_format_desc->colorspace == UTIL_FORMAT_COLORSPACE_RGB &&
597 out_format_desc->channel[0].pure_integer) {
598 if (out_format_desc->channel[0].type == UTIL_FORMAT_TYPE_SIGNED) {
599 texel_type = lp_type_int_vec(bld->type.width, bld->type.width * bld->type.length);
600 }
601 else if (out_format_desc->channel[0].type == UTIL_FORMAT_TYPE_UNSIGNED) {
602 texel_type = lp_type_uint_vec(bld->type.width, bld->type.width * bld->type.length);
603 }
604 } else if (fetch_stencil) {
605 texel_type = lp_type_uint_vec(bld->type.width, bld->type.width * bld->type.length);
606 }
607
608 lp_build_fetch_rgba_soa(gallivm, out_format_desc, texel_type,
609 true, buf_ptr, offset,
610 NULL, NULL, NULL, result);
611 }
612
613
614 /**
615 * Generate the fragment shader, depth/stencil test, and alpha tests.
616 */
617 static void
generate_fs_loop(struct gallivm_state * gallivm,struct lp_fragment_shader * shader,const struct lp_fragment_shader_variant_key * key,LLVMBuilderRef builder,struct lp_type type,LLVMValueRef context_ptr,LLVMValueRef sample_pos_array,LLVMValueRef num_loop,struct lp_build_interp_soa_context * interp,const struct lp_build_sampler_soa * sampler,const struct lp_build_image_soa * image,LLVMValueRef mask_store,LLVMValueRef (* out_color)[4],LLVMValueRef depth_base_ptr,LLVMValueRef depth_stride,LLVMValueRef depth_sample_stride,LLVMValueRef color_ptr_ptr,LLVMValueRef color_stride_ptr,LLVMValueRef color_sample_stride_ptr,LLVMValueRef facing,LLVMValueRef thread_data_ptr)618 generate_fs_loop(struct gallivm_state *gallivm,
619 struct lp_fragment_shader *shader,
620 const struct lp_fragment_shader_variant_key *key,
621 LLVMBuilderRef builder,
622 struct lp_type type,
623 LLVMValueRef context_ptr,
624 LLVMValueRef sample_pos_array,
625 LLVMValueRef num_loop,
626 struct lp_build_interp_soa_context *interp,
627 const struct lp_build_sampler_soa *sampler,
628 const struct lp_build_image_soa *image,
629 LLVMValueRef mask_store,
630 LLVMValueRef (*out_color)[4],
631 LLVMValueRef depth_base_ptr,
632 LLVMValueRef depth_stride,
633 LLVMValueRef depth_sample_stride,
634 LLVMValueRef color_ptr_ptr,
635 LLVMValueRef color_stride_ptr,
636 LLVMValueRef color_sample_stride_ptr,
637 LLVMValueRef facing,
638 LLVMValueRef thread_data_ptr)
639 {
640 const struct tgsi_token *tokens = shader->base.tokens;
641 struct lp_type int_type = lp_int_type(type);
642 LLVMValueRef mask_ptr = NULL, mask_val = NULL;
643 LLVMValueRef z;
644 LLVMValueRef z_value, s_value;
645 LLVMValueRef z_fb, s_fb;
646 LLVMValueRef zs_samples = lp_build_const_int32(gallivm, key->zsbuf_nr_samples);
647 LLVMValueRef z_out = NULL, s_out = NULL;
648 struct lp_build_for_loop_state loop_state, sample_loop_state = {0};
649 struct lp_build_mask_context mask;
650 /*
651 * TODO: figure out if simple_shader optimization is really worthwile to
652 * keep. Disabled because it may hide some real bugs in the (depth/stencil)
653 * code since tests tend to take another codepath than real shaders.
654 */
655 boolean simple_shader = (shader->info.base.file_count[TGSI_FILE_SAMPLER] == 0 &&
656 shader->info.base.num_inputs < 3 &&
657 shader->info.base.num_instructions < 8) && 0;
658 const boolean dual_source_blend = key->blend.rt[0].blend_enable &&
659 util_blend_state_is_dual(&key->blend, 0);
660 const bool post_depth_coverage = shader->info.base.properties[TGSI_PROPERTY_FS_POST_DEPTH_COVERAGE];
661
662 struct lp_bld_tgsi_system_values system_values;
663
664 memset(&system_values, 0, sizeof(system_values));
665
666 /* truncate then sign extend. */
667 system_values.front_facing =
668 LLVMBuildTrunc(gallivm->builder, facing,
669 LLVMInt1TypeInContext(gallivm->context), "");
670 system_values.front_facing =
671 LLVMBuildSExt(gallivm->builder, system_values.front_facing,
672 LLVMInt32TypeInContext(gallivm->context), "");
673 system_values.view_index =
674 lp_jit_thread_data_raster_state_view_index(gallivm, thread_data_ptr);
675
676 unsigned depth_mode;
677 const struct util_format_description *zs_format_desc = NULL;
678 if (key->depth.enabled ||
679 key->stencil[0].enabled) {
680 zs_format_desc = util_format_description(key->zsbuf_format);
681
682 if (shader->info.base.properties[TGSI_PROPERTY_FS_EARLY_DEPTH_STENCIL])
683 depth_mode = EARLY_DEPTH_TEST | EARLY_DEPTH_WRITE;
684 else if (!shader->info.base.writes_z && !shader->info.base.writes_stencil &&
685 !shader->info.base.uses_fbfetch && !shader->info.base.writes_memory) {
686 if (key->alpha.enabled ||
687 key->blend.alpha_to_coverage ||
688 shader->info.base.uses_kill ||
689 shader->info.base.writes_samplemask) {
690 /* With alpha test and kill, can do the depth test early
691 * and hopefully eliminate some quads. But need to do a
692 * special deferred depth write once the final mask value
693 * is known. This only works though if there's either no
694 * stencil test or the stencil value isn't written.
695 */
696 if (key->stencil[0].enabled && (key->stencil[0].writemask ||
697 (key->stencil[1].enabled &&
698 key->stencil[1].writemask)))
699 depth_mode = LATE_DEPTH_TEST | LATE_DEPTH_WRITE;
700 else
701 depth_mode = EARLY_DEPTH_TEST | LATE_DEPTH_WRITE | EARLY_DEPTH_TEST_INFERRED;
702 }
703 else
704 depth_mode = EARLY_DEPTH_TEST | EARLY_DEPTH_WRITE | EARLY_DEPTH_TEST_INFERRED;
705 }
706 else {
707 depth_mode = LATE_DEPTH_TEST | LATE_DEPTH_WRITE;
708 }
709
710 if (!(key->depth.enabled && key->depth.writemask) &&
711 !(key->stencil[0].enabled && (key->stencil[0].writemask ||
712 (key->stencil[1].enabled &&
713 key->stencil[1].writemask))))
714 depth_mode &= ~(LATE_DEPTH_WRITE | EARLY_DEPTH_WRITE);
715 }
716 else {
717 depth_mode = 0;
718 }
719
720 LLVMTypeRef vec_type = lp_build_vec_type(gallivm, type);
721 LLVMTypeRef int_vec_type = lp_build_vec_type(gallivm, int_type);
722
723 LLVMValueRef stencil_refs[2];
724 stencil_refs[0] = lp_jit_context_stencil_ref_front_value(gallivm, context_ptr);
725 stencil_refs[1] = lp_jit_context_stencil_ref_back_value(gallivm, context_ptr);
726 /* convert scalar stencil refs into vectors */
727 stencil_refs[0] = lp_build_broadcast(gallivm, int_vec_type, stencil_refs[0]);
728 stencil_refs[1] = lp_build_broadcast(gallivm, int_vec_type, stencil_refs[1]);
729
730 LLVMValueRef consts_ptr = lp_jit_context_constants(gallivm, context_ptr);
731 LLVMValueRef num_consts_ptr = lp_jit_context_num_constants(gallivm,
732 context_ptr);
733
734 LLVMValueRef ssbo_ptr = lp_jit_context_ssbos(gallivm, context_ptr);
735 LLVMValueRef num_ssbo_ptr = lp_jit_context_num_ssbos(gallivm, context_ptr);
736
737 LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][TGSI_NUM_CHANNELS];
738 memset(outputs, 0, sizeof outputs);
739
740 /* Allocate color storage for each fragment sample */
741 LLVMValueRef color_store_size = num_loop;
742 if (key->min_samples > 1)
743 color_store_size = LLVMBuildMul(builder, num_loop, lp_build_const_int32(gallivm, key->min_samples), "");
744
745 for (unsigned cbuf = 0; cbuf < key->nr_cbufs; cbuf++) {
746 for (unsigned chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) {
747 out_color[cbuf][chan] = lp_build_array_alloca(gallivm,
748 lp_build_vec_type(gallivm,
749 type),
750 color_store_size, "color");
751 }
752 }
753 if (dual_source_blend) {
754 assert(key->nr_cbufs <= 1);
755 for (unsigned chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) {
756 out_color[1][chan] = lp_build_array_alloca(gallivm,
757 lp_build_vec_type(gallivm,
758 type),
759 color_store_size, "color1");
760 }
761 }
762 if (shader->info.base.writes_z) {
763 z_out = lp_build_array_alloca(gallivm,
764 lp_build_vec_type(gallivm, type),
765 color_store_size, "depth");
766 }
767
768 if (shader->info.base.writes_stencil) {
769 s_out = lp_build_array_alloca(gallivm,
770 lp_build_vec_type(gallivm, type),
771 color_store_size, "depth");
772 }
773
774 lp_build_for_loop_begin(&loop_state, gallivm,
775 lp_build_const_int32(gallivm, 0),
776 LLVMIntULT,
777 num_loop,
778 lp_build_const_int32(gallivm, 1));
779
780 LLVMValueRef sample_mask_in;
781 if (key->multisample) {
782 sample_mask_in = lp_build_const_int_vec(gallivm, type, 0);
783 /* create shader execution mask by combining all sample masks. */
784 for (unsigned s = 0; s < key->coverage_samples; s++) {
785 LLVMValueRef s_mask_idx = LLVMBuildMul(builder, num_loop, lp_build_const_int32(gallivm, s), "");
786 s_mask_idx = LLVMBuildAdd(builder, s_mask_idx, loop_state.counter, "");
787 LLVMValueRef s_mask = lp_build_pointer_get(builder, mask_store, s_mask_idx);
788 if (s == 0)
789 mask_val = s_mask;
790 else
791 mask_val = LLVMBuildOr(builder, s_mask, mask_val, "");
792
793 LLVMValueRef mask_in = LLVMBuildAnd(builder, s_mask, lp_build_const_int_vec(gallivm, type, (1ll << s)), "");
794 sample_mask_in = LLVMBuildOr(builder, sample_mask_in, mask_in, "");
795 }
796 } else {
797 sample_mask_in = lp_build_const_int_vec(gallivm, type, 1);
798 mask_ptr = LLVMBuildGEP(builder, mask_store,
799 &loop_state.counter, 1, "mask_ptr");
800 mask_val = LLVMBuildLoad(builder, mask_ptr, "");
801
802 LLVMValueRef mask_in = LLVMBuildAnd(builder, mask_val, lp_build_const_int_vec(gallivm, type, 1), "");
803 sample_mask_in = LLVMBuildOr(builder, sample_mask_in, mask_in, "");
804 }
805
806 /* 'mask' will control execution based on quad's pixel alive/killed state */
807 lp_build_mask_begin(&mask, gallivm, type, mask_val);
808
809 if (!(depth_mode & EARLY_DEPTH_TEST) && !simple_shader)
810 lp_build_mask_check(&mask);
811
812 /* Create storage for recombining sample masks after early Z pass. */
813 LLVMValueRef s_mask_or = lp_build_alloca(gallivm, lp_build_int_vec_type(gallivm, type), "cov_mask_early_depth");
814 LLVMBuildStore(builder, LLVMConstNull(lp_build_int_vec_type(gallivm, type)), s_mask_or);
815
816 /* Create storage for post depth sample mask */
817 LLVMValueRef post_depth_sample_mask_in = NULL;
818 if (post_depth_coverage)
819 post_depth_sample_mask_in = lp_build_alloca(gallivm, int_vec_type, "post_depth_sample_mask_in");
820
821 LLVMValueRef s_mask = NULL, s_mask_ptr = NULL;
822 LLVMValueRef z_sample_value_store = NULL, s_sample_value_store = NULL;
823 LLVMValueRef z_fb_store = NULL, s_fb_store = NULL;
824 LLVMTypeRef z_type = NULL, z_fb_type = NULL;
825
826 /* Run early depth once per sample */
827 if (key->multisample) {
828
829 if (zs_format_desc) {
830 struct lp_type zs_type = lp_depth_type(zs_format_desc, type.length);
831 struct lp_type z_type = zs_type;
832 struct lp_type s_type = zs_type;
833 if (zs_format_desc->block.bits < type.width)
834 z_type.width = type.width;
835 if (zs_format_desc->block.bits == 8)
836 s_type.width = type.width;
837
838 else if (zs_format_desc->block.bits > 32) {
839 z_type.width = z_type.width / 2;
840 s_type.width = s_type.width / 2;
841 s_type.floating = 0;
842 }
843 z_sample_value_store = lp_build_array_alloca(gallivm, lp_build_int_vec_type(gallivm, type),
844 zs_samples, "z_sample_store");
845 s_sample_value_store = lp_build_array_alloca(gallivm, lp_build_int_vec_type(gallivm, type),
846 zs_samples, "s_sample_store");
847 z_fb_store = lp_build_array_alloca(gallivm, lp_build_vec_type(gallivm, z_type),
848 zs_samples, "z_fb_store");
849 s_fb_store = lp_build_array_alloca(gallivm, lp_build_vec_type(gallivm, s_type),
850 zs_samples, "s_fb_store");
851 }
852 lp_build_for_loop_begin(&sample_loop_state, gallivm,
853 lp_build_const_int32(gallivm, 0),
854 LLVMIntULT, lp_build_const_int32(gallivm, key->coverage_samples),
855 lp_build_const_int32(gallivm, 1));
856
857 LLVMValueRef s_mask_idx = LLVMBuildMul(builder, sample_loop_state.counter, num_loop, "");
858 s_mask_idx = LLVMBuildAdd(builder, s_mask_idx, loop_state.counter, "");
859 s_mask_ptr = LLVMBuildGEP(builder, mask_store, &s_mask_idx, 1, "");
860
861 s_mask = LLVMBuildLoad(builder, s_mask_ptr, "");
862 s_mask = LLVMBuildAnd(builder, s_mask, mask_val, "");
863 }
864
865
866 /* for multisample Z needs to be interpolated at sample points for testing. */
867 lp_build_interp_soa_update_pos_dyn(interp, gallivm, loop_state.counter,
868 key->multisample
869 ? sample_loop_state.counter : NULL);
870 z = interp->pos[2];
871
872 LLVMValueRef depth_ptr = depth_base_ptr;
873 if (key->multisample) {
874 LLVMValueRef sample_offset =
875 LLVMBuildMul(builder, sample_loop_state.counter,
876 depth_sample_stride, "");
877 depth_ptr = LLVMBuildGEP(builder, depth_ptr, &sample_offset, 1, "");
878 }
879
880 if (depth_mode & EARLY_DEPTH_TEST) {
881 z = lp_build_depth_clamp(gallivm, builder, key->depth_clamp,
882 key->restrict_depth_values, type, context_ptr,
883 thread_data_ptr, z);
884
885 lp_build_depth_stencil_load_swizzled(gallivm, type,
886 zs_format_desc, key->resource_1d,
887 depth_ptr, depth_stride,
888 &z_fb, &s_fb, loop_state.counter);
889 lp_build_depth_stencil_test(gallivm,
890 &key->depth,
891 key->stencil,
892 type,
893 zs_format_desc,
894 key->multisample ? NULL : &mask,
895 &s_mask,
896 stencil_refs,
897 z, z_fb, s_fb,
898 facing,
899 &z_value, &s_value,
900 !simple_shader && !key->multisample,
901 key->restrict_depth_values);
902
903 if (depth_mode & EARLY_DEPTH_WRITE) {
904 lp_build_depth_stencil_write_swizzled(gallivm, type,
905 zs_format_desc, key->resource_1d,
906 NULL, NULL, NULL, loop_state.counter,
907 depth_ptr, depth_stride,
908 z_value, s_value);
909 }
910 /*
911 * Note mask check if stencil is enabled must be after ds write not after
912 * stencil test otherwise new stencil values may not get written if all
913 * fragments got killed by depth/stencil test.
914 */
915 if (!simple_shader && key->stencil[0].enabled && !key->multisample)
916 lp_build_mask_check(&mask);
917
918 if (key->multisample) {
919 z_fb_type = LLVMTypeOf(z_fb);
920 z_type = LLVMTypeOf(z_value);
921 lp_build_pointer_set(builder, z_sample_value_store, sample_loop_state.counter, LLVMBuildBitCast(builder, z_value, lp_build_int_vec_type(gallivm, type), ""));
922 lp_build_pointer_set(builder, s_sample_value_store, sample_loop_state.counter, LLVMBuildBitCast(builder, s_value, lp_build_int_vec_type(gallivm, type), ""));
923 lp_build_pointer_set(builder, z_fb_store, sample_loop_state.counter, z_fb);
924 lp_build_pointer_set(builder, s_fb_store, sample_loop_state.counter, s_fb);
925 }
926 }
927
928 if (key->multisample) {
929 /*
930 * Store the post-early Z coverage mask.
931 * Recombine the resulting coverage masks post early Z into the fragment
932 * shader execution mask.
933 */
934 LLVMValueRef tmp_s_mask_or = LLVMBuildLoad(builder, s_mask_or, "");
935 tmp_s_mask_or = LLVMBuildOr(builder, tmp_s_mask_or, s_mask, "");
936 LLVMBuildStore(builder, tmp_s_mask_or, s_mask_or);
937
938 if (post_depth_coverage) {
939 LLVMValueRef mask_bit_idx = LLVMBuildShl(builder, lp_build_const_int32(gallivm, 1), sample_loop_state.counter, "");
940 LLVMValueRef post_depth_mask_in = LLVMBuildLoad(builder, post_depth_sample_mask_in, "");
941 mask_bit_idx = LLVMBuildAnd(builder, s_mask, lp_build_broadcast(gallivm, int_vec_type, mask_bit_idx), "");
942 post_depth_mask_in = LLVMBuildOr(builder, post_depth_mask_in, mask_bit_idx, "");
943 LLVMBuildStore(builder, post_depth_mask_in, post_depth_sample_mask_in);
944 }
945
946 LLVMBuildStore(builder, s_mask, s_mask_ptr);
947
948 lp_build_for_loop_end(&sample_loop_state);
949
950 /* recombined all the coverage masks in the shader exec mask. */
951 tmp_s_mask_or = LLVMBuildLoad(builder, s_mask_or, "");
952 lp_build_mask_update(&mask, tmp_s_mask_or);
953
954 if (key->min_samples == 1) {
955 /* for multisample Z needs to be re interpolated at pixel center */
956 lp_build_interp_soa_update_pos_dyn(interp, gallivm, loop_state.counter, NULL);
957 z = interp->pos[2];
958 lp_build_mask_update(&mask, tmp_s_mask_or);
959 }
960 } else {
961 if (post_depth_coverage) {
962 LLVMValueRef post_depth_mask_in = LLVMBuildAnd(builder, lp_build_mask_value(&mask), lp_build_const_int_vec(gallivm, type, 1), "");
963 LLVMBuildStore(builder, post_depth_mask_in, post_depth_sample_mask_in);
964 }
965 }
966
967 LLVMValueRef out_sample_mask_storage = NULL;
968 if (shader->info.base.writes_samplemask) {
969 out_sample_mask_storage = lp_build_alloca(gallivm, int_vec_type, "write_mask");
970 if (key->min_samples > 1)
971 LLVMBuildStore(builder, LLVMConstNull(int_vec_type), out_sample_mask_storage);
972 }
973
974 if (post_depth_coverage) {
975 system_values.sample_mask_in = LLVMBuildLoad(builder, post_depth_sample_mask_in, "");
976 }
977 else
978 system_values.sample_mask_in = sample_mask_in;
979 if (key->multisample && key->min_samples > 1) {
980 lp_build_for_loop_begin(&sample_loop_state, gallivm,
981 lp_build_const_int32(gallivm, 0),
982 LLVMIntULT,
983 lp_build_const_int32(gallivm, key->min_samples),
984 lp_build_const_int32(gallivm, 1));
985
986 LLVMValueRef s_mask_idx = LLVMBuildMul(builder, sample_loop_state.counter, num_loop, "");
987 s_mask_idx = LLVMBuildAdd(builder, s_mask_idx, loop_state.counter, "");
988 s_mask_ptr = LLVMBuildGEP(builder, mask_store, &s_mask_idx, 1, "");
989 s_mask = LLVMBuildLoad(builder, s_mask_ptr, "");
990 lp_build_mask_force(&mask, s_mask);
991 lp_build_interp_soa_update_pos_dyn(interp, gallivm, loop_state.counter, sample_loop_state.counter);
992 system_values.sample_id = sample_loop_state.counter;
993 system_values.sample_mask_in = LLVMBuildAnd(builder, system_values.sample_mask_in,
994 lp_build_broadcast(gallivm, int_vec_type,
995 LLVMBuildShl(builder, lp_build_const_int32(gallivm, 1), sample_loop_state.counter, "")), "");
996 } else {
997 system_values.sample_id = lp_build_const_int32(gallivm, 0);
998
999 }
1000 system_values.sample_pos = sample_pos_array;
1001
1002 lp_build_interp_soa_update_inputs_dyn(interp, gallivm, loop_state.counter, mask_store, sample_loop_state.counter);
1003
1004 struct lp_build_fs_llvm_iface fs_iface = {
1005 .base.interp_fn = fs_interp,
1006 .base.fb_fetch = fs_fb_fetch,
1007 .interp = interp,
1008 .loop_state = &loop_state,
1009 .sample_id = system_values.sample_id,
1010 .mask_store = mask_store,
1011 .color_ptr_ptr = color_ptr_ptr,
1012 .color_stride_ptr = color_stride_ptr,
1013 .color_sample_stride_ptr = color_sample_stride_ptr,
1014 .zs_base_ptr = depth_base_ptr,
1015 .zs_stride = depth_stride,
1016 .zs_sample_stride = depth_sample_stride,
1017 .key = key,
1018 };
1019
1020 struct lp_build_tgsi_params params;
1021 memset(¶ms, 0, sizeof(params));
1022
1023 params.type = type;
1024 params.mask = &mask;
1025 params.fs_iface = &fs_iface.base;
1026 params.consts_ptr = consts_ptr;
1027 params.const_sizes_ptr = num_consts_ptr;
1028 params.system_values = &system_values;
1029 params.inputs = interp->inputs;
1030 params.context_ptr = context_ptr;
1031 params.thread_data_ptr = thread_data_ptr;
1032 params.sampler = sampler;
1033 params.info = &shader->info.base;
1034 params.ssbo_ptr = ssbo_ptr;
1035 params.ssbo_sizes_ptr = num_ssbo_ptr;
1036 params.image = image;
1037 params.aniso_filter_table = lp_jit_context_aniso_filter_table(gallivm, context_ptr);
1038
1039 /* Build the actual shader */
1040 if (shader->base.type == PIPE_SHADER_IR_TGSI)
1041 lp_build_tgsi_soa(gallivm, tokens, ¶ms,
1042 outputs);
1043 else
1044 lp_build_nir_soa(gallivm, shader->base.ir.nir, ¶ms,
1045 outputs);
1046
1047 /* Alpha test */
1048 if (key->alpha.enabled) {
1049 int color0 = find_output_by_semantic(&shader->info.base,
1050 TGSI_SEMANTIC_COLOR,
1051 0);
1052
1053 if (color0 != -1 && outputs[color0][3]) {
1054 const struct util_format_description *cbuf_format_desc;
1055 LLVMValueRef alpha = LLVMBuildLoad(builder, outputs[color0][3], "alpha");
1056 LLVMValueRef alpha_ref_value;
1057
1058 alpha_ref_value = lp_jit_context_alpha_ref_value(gallivm, context_ptr);
1059 alpha_ref_value = lp_build_broadcast(gallivm, vec_type, alpha_ref_value);
1060
1061 cbuf_format_desc = util_format_description(key->cbuf_format[0]);
1062
1063 lp_build_alpha_test(gallivm, key->alpha.func, type, cbuf_format_desc,
1064 &mask, alpha, alpha_ref_value,
1065 ((depth_mode & LATE_DEPTH_TEST) != 0) && !key->multisample);
1066 }
1067 }
1068
1069 /* Emulate Alpha to Coverage with Alpha test */
1070 if (key->blend.alpha_to_coverage) {
1071 int color0 = find_output_by_semantic(&shader->info.base,
1072 TGSI_SEMANTIC_COLOR,
1073 0);
1074
1075 if (color0 != -1 && outputs[color0][3]) {
1076 LLVMValueRef alpha = LLVMBuildLoad(builder, outputs[color0][3], "alpha");
1077
1078 if (!key->multisample) {
1079 lp_build_alpha_to_coverage(gallivm, type,
1080 &mask, alpha,
1081 (depth_mode & LATE_DEPTH_TEST) != 0);
1082 } else {
1083 lp_build_sample_alpha_to_coverage(gallivm, type, key->coverage_samples, num_loop,
1084 loop_state.counter,
1085 mask_store, alpha);
1086 }
1087 }
1088 }
1089
1090 if (key->blend.alpha_to_one) {
1091 for (unsigned attrib = 0; attrib < shader->info.base.num_outputs; ++attrib) {
1092 unsigned cbuf = shader->info.base.output_semantic_index[attrib];
1093 if ((shader->info.base.output_semantic_name[attrib] == TGSI_SEMANTIC_COLOR) &&
1094 ((cbuf < key->nr_cbufs) || (cbuf == 1 && dual_source_blend)))
1095 if (outputs[cbuf][3]) {
1096 LLVMBuildStore(builder, lp_build_const_vec(gallivm, type, 1.0),
1097 outputs[cbuf][3]);
1098 }
1099 }
1100 }
1101
1102 if (shader->info.base.writes_samplemask) {
1103 LLVMValueRef output_smask = NULL;
1104 int smaski = find_output_by_semantic(&shader->info.base,
1105 TGSI_SEMANTIC_SAMPLEMASK,
1106 0);
1107 struct lp_build_context smask_bld;
1108 lp_build_context_init(&smask_bld, gallivm, int_type);
1109
1110 assert(smaski >= 0);
1111 output_smask = LLVMBuildLoad(builder, outputs[smaski][0], "smask");
1112 output_smask = LLVMBuildBitCast(builder, output_smask, smask_bld.vec_type, "");
1113 if (!key->multisample && key->no_ms_sample_mask_out) {
1114 output_smask = lp_build_and(&smask_bld, output_smask, smask_bld.one);
1115 output_smask = lp_build_cmp(&smask_bld, PIPE_FUNC_NOTEQUAL, output_smask, smask_bld.zero);
1116 lp_build_mask_update(&mask, output_smask);
1117 }
1118
1119 if (key->min_samples > 1) {
1120 /* only the bit corresponding to this sample is to be used. */
1121 LLVMValueRef tmp_mask = LLVMBuildLoad(builder, out_sample_mask_storage, "tmp_mask");
1122 LLVMValueRef out_smask_idx = LLVMBuildShl(builder, lp_build_const_int32(gallivm, 1), sample_loop_state.counter, "");
1123 LLVMValueRef smask_bit = LLVMBuildAnd(builder, output_smask, lp_build_broadcast(gallivm, int_vec_type, out_smask_idx), "");
1124 output_smask = LLVMBuildOr(builder, tmp_mask, smask_bit, "");
1125 }
1126
1127 LLVMBuildStore(builder, output_smask, out_sample_mask_storage);
1128 }
1129
1130 if (shader->info.base.writes_z) {
1131 int pos0 = find_output_by_semantic(&shader->info.base,
1132 TGSI_SEMANTIC_POSITION,
1133 0);
1134 LLVMValueRef out = LLVMBuildLoad(builder, outputs[pos0][2], "");
1135 LLVMValueRef idx = loop_state.counter;
1136 if (key->min_samples > 1)
1137 idx = LLVMBuildAdd(builder, idx,
1138 LLVMBuildMul(builder, sample_loop_state.counter, num_loop, ""), "");
1139 LLVMValueRef ptr = LLVMBuildGEP(builder, z_out, &idx, 1, "");
1140 LLVMBuildStore(builder, out, ptr);
1141 }
1142
1143 if (shader->info.base.writes_stencil) {
1144 int sten_out = find_output_by_semantic(&shader->info.base,
1145 TGSI_SEMANTIC_STENCIL,
1146 0);
1147 LLVMValueRef out = LLVMBuildLoad(builder, outputs[sten_out][1], "output.s");
1148 LLVMValueRef idx = loop_state.counter;
1149 if (key->min_samples > 1)
1150 idx = LLVMBuildAdd(builder, idx,
1151 LLVMBuildMul(builder, sample_loop_state.counter, num_loop, ""), "");
1152 LLVMValueRef ptr = LLVMBuildGEP(builder, s_out, &idx, 1, "");
1153 LLVMBuildStore(builder, out, ptr);
1154 }
1155
1156 bool has_cbuf0_write = false;
1157 /* Color write - per fragment sample */
1158 for (unsigned attrib = 0; attrib < shader->info.base.num_outputs; ++attrib) {
1159 unsigned cbuf = shader->info.base.output_semantic_index[attrib];
1160 if ((shader->info.base.output_semantic_name[attrib]
1161 == TGSI_SEMANTIC_COLOR) &&
1162 ((cbuf < key->nr_cbufs) || (cbuf == 1 && dual_source_blend))) {
1163 if (cbuf == 0 &&
1164 shader->info.base.properties[TGSI_PROPERTY_FS_COLOR0_WRITES_ALL_CBUFS]) {
1165 /* XXX: there is an edge case with FB fetch where gl_FragColor and
1166 * gl_LastFragData[0] are used together. This creates both
1167 * FRAG_RESULT_COLOR and FRAG_RESULT_DATA* output variables. This
1168 * loop then writes to cbuf 0 twice, owerwriting the correct value
1169 * from gl_FragColor with some garbage. This case is excercised in
1170 * one of deqp tests. A similar bug can happen if
1171 * gl_SecondaryFragColorEXT and gl_LastFragData[1] are mixed in
1172 * the same fashion... This workaround will break if
1173 * gl_LastFragData[0] goes in outputs list before
1174 * gl_FragColor. This doesn't seem to happen though.
1175 */
1176 if (has_cbuf0_write)
1177 continue;
1178 has_cbuf0_write = true;
1179 }
1180
1181 for (unsigned chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) {
1182 if (outputs[attrib][chan]) {
1183 /* XXX: just initialize outputs to point at colors[] and
1184 * skip this.
1185 */
1186 LLVMValueRef out = LLVMBuildLoad(builder, outputs[attrib][chan], "");
1187 LLVMValueRef color_ptr;
1188 LLVMValueRef color_idx = loop_state.counter;
1189 if (key->min_samples > 1)
1190 color_idx = LLVMBuildAdd(builder, color_idx,
1191 LLVMBuildMul(builder, sample_loop_state.counter, num_loop, ""), "");
1192 color_ptr = LLVMBuildGEP(builder, out_color[cbuf][chan],
1193 &color_idx, 1, "");
1194 lp_build_name(out, "color%u.%c", attrib, "rgba"[chan]);
1195 LLVMBuildStore(builder, out, color_ptr);
1196 }
1197 }
1198 }
1199 }
1200
1201 if (key->multisample && key->min_samples > 1) {
1202 LLVMBuildStore(builder, lp_build_mask_value(&mask), s_mask_ptr);
1203 lp_build_for_loop_end(&sample_loop_state);
1204 }
1205
1206 if (key->multisample) {
1207 /* execute depth test for each sample */
1208 lp_build_for_loop_begin(&sample_loop_state, gallivm,
1209 lp_build_const_int32(gallivm, 0),
1210 LLVMIntULT, lp_build_const_int32(gallivm, key->coverage_samples),
1211 lp_build_const_int32(gallivm, 1));
1212
1213 /* load the per-sample coverage mask */
1214 LLVMValueRef s_mask_idx = LLVMBuildMul(builder, sample_loop_state.counter, num_loop, "");
1215 s_mask_idx = LLVMBuildAdd(builder, s_mask_idx, loop_state.counter, "");
1216 s_mask_ptr = LLVMBuildGEP(builder, mask_store, &s_mask_idx, 1, "");
1217
1218 /* combine the execution mask post fragment shader with the coverage mask. */
1219 s_mask = LLVMBuildLoad(builder, s_mask_ptr, "");
1220 if (key->min_samples == 1)
1221 s_mask = LLVMBuildAnd(builder, s_mask, lp_build_mask_value(&mask), "");
1222
1223 /* if the shader writes sample mask use that,
1224 * but only if this isn't genuine early-depth to avoid breaking occlusion query */
1225 if (shader->info.base.writes_samplemask &&
1226 (!(depth_mode & EARLY_DEPTH_TEST) || (depth_mode & (EARLY_DEPTH_TEST_INFERRED)))) {
1227 LLVMValueRef out_smask_idx = LLVMBuildShl(builder, lp_build_const_int32(gallivm, 1), sample_loop_state.counter, "");
1228 out_smask_idx = lp_build_broadcast(gallivm, int_vec_type, out_smask_idx);
1229 LLVMValueRef output_smask = LLVMBuildLoad(builder, out_sample_mask_storage, "");
1230 LLVMValueRef smask_bit = LLVMBuildAnd(builder, output_smask, out_smask_idx, "");
1231 LLVMValueRef cmp = LLVMBuildICmp(builder, LLVMIntNE, smask_bit, lp_build_const_int_vec(gallivm, int_type, 0), "");
1232 smask_bit = LLVMBuildSExt(builder, cmp, int_vec_type, "");
1233
1234 s_mask = LLVMBuildAnd(builder, s_mask, smask_bit, "");
1235 }
1236 }
1237
1238 depth_ptr = depth_base_ptr;
1239 if (key->multisample) {
1240 LLVMValueRef sample_offset = LLVMBuildMul(builder, sample_loop_state.counter, depth_sample_stride, "");
1241 depth_ptr = LLVMBuildGEP(builder, depth_ptr, &sample_offset, 1, "");
1242 }
1243
1244 /* Late Z test */
1245 if (depth_mode & LATE_DEPTH_TEST) {
1246 if (shader->info.base.writes_z) {
1247 LLVMValueRef idx = loop_state.counter;
1248 if (key->min_samples > 1)
1249 idx = LLVMBuildAdd(builder, idx,
1250 LLVMBuildMul(builder, sample_loop_state.counter, num_loop, ""), "");
1251 LLVMValueRef ptr = LLVMBuildGEP(builder, z_out, &idx, 1, "");
1252 z = LLVMBuildLoad(builder, ptr, "output.z");
1253 } else {
1254 if (key->multisample) {
1255 lp_build_interp_soa_update_pos_dyn(interp, gallivm, loop_state.counter, key->multisample ? sample_loop_state.counter : NULL);
1256 z = interp->pos[2];
1257 }
1258 }
1259
1260 /*
1261 * Clamp according to ARB_depth_clamp semantics.
1262 */
1263 z = lp_build_depth_clamp(gallivm, builder, key->depth_clamp,
1264 key->restrict_depth_values, type, context_ptr,
1265 thread_data_ptr, z);
1266
1267 if (shader->info.base.writes_stencil) {
1268 LLVMValueRef idx = loop_state.counter;
1269 if (key->min_samples > 1)
1270 idx = LLVMBuildAdd(builder, idx,
1271 LLVMBuildMul(builder, sample_loop_state.counter, num_loop, ""), "");
1272 LLVMValueRef ptr = LLVMBuildGEP(builder, s_out, &idx, 1, "");
1273 stencil_refs[0] = LLVMBuildLoad(builder, ptr, "output.s");
1274 /* there's only one value, and spec says to discard additional bits */
1275 LLVMValueRef s_max_mask = lp_build_const_int_vec(gallivm, int_type, 255);
1276 stencil_refs[0] = LLVMBuildBitCast(builder, stencil_refs[0], int_vec_type, "");
1277 stencil_refs[0] = LLVMBuildAnd(builder, stencil_refs[0], s_max_mask, "");
1278 stencil_refs[1] = stencil_refs[0];
1279 }
1280
1281 lp_build_depth_stencil_load_swizzled(gallivm, type,
1282 zs_format_desc, key->resource_1d,
1283 depth_ptr, depth_stride,
1284 &z_fb, &s_fb, loop_state.counter);
1285
1286 lp_build_depth_stencil_test(gallivm,
1287 &key->depth,
1288 key->stencil,
1289 type,
1290 zs_format_desc,
1291 key->multisample ? NULL : &mask,
1292 &s_mask,
1293 stencil_refs,
1294 z, z_fb, s_fb,
1295 facing,
1296 &z_value, &s_value,
1297 !simple_shader,
1298 key->restrict_depth_values);
1299 /* Late Z write */
1300 if (depth_mode & LATE_DEPTH_WRITE) {
1301 lp_build_depth_stencil_write_swizzled(gallivm, type,
1302 zs_format_desc, key->resource_1d,
1303 NULL, NULL, NULL, loop_state.counter,
1304 depth_ptr, depth_stride,
1305 z_value, s_value);
1306 }
1307 }
1308 else if ((depth_mode & EARLY_DEPTH_TEST) &&
1309 (depth_mode & LATE_DEPTH_WRITE))
1310 {
1311 /* Need to apply a reduced mask to the depth write. Reload the
1312 * depth value, update from zs_value with the new mask value and
1313 * write that out.
1314 */
1315 if (key->multisample) {
1316 z_value = LLVMBuildBitCast(builder, lp_build_pointer_get(builder, z_sample_value_store, sample_loop_state.counter), z_type, "");;
1317 s_value = lp_build_pointer_get(builder, s_sample_value_store, sample_loop_state.counter);
1318 z_fb = LLVMBuildBitCast(builder, lp_build_pointer_get(builder, z_fb_store, sample_loop_state.counter), z_fb_type, "");
1319 s_fb = lp_build_pointer_get(builder, s_fb_store, sample_loop_state.counter);
1320 }
1321 lp_build_depth_stencil_write_swizzled(gallivm, type,
1322 zs_format_desc, key->resource_1d,
1323 key->multisample ? s_mask : lp_build_mask_value(&mask), z_fb, s_fb, loop_state.counter,
1324 depth_ptr, depth_stride,
1325 z_value, s_value);
1326 }
1327
1328 if (key->occlusion_count) {
1329 LLVMValueRef counter = lp_jit_thread_data_counter(gallivm, thread_data_ptr);
1330 lp_build_name(counter, "counter");
1331
1332 lp_build_occlusion_count(gallivm, type,
1333 key->multisample ? s_mask : lp_build_mask_value(&mask), counter);
1334 }
1335
1336 /* if this is genuine early-depth in the shader, write samplemask now
1337 * after occlusion count has been updated
1338 */
1339 if (key->multisample && shader->info.base.writes_samplemask &&
1340 (depth_mode & (EARLY_DEPTH_TEST_INFERRED | EARLY_DEPTH_TEST)) == EARLY_DEPTH_TEST) {
1341 /* if the shader writes sample mask use that */
1342 LLVMValueRef out_smask_idx = LLVMBuildShl(builder, lp_build_const_int32(gallivm, 1), sample_loop_state.counter, "");
1343 out_smask_idx = lp_build_broadcast(gallivm, int_vec_type, out_smask_idx);
1344 LLVMValueRef output_smask = LLVMBuildLoad(builder, out_sample_mask_storage, "");
1345 LLVMValueRef smask_bit = LLVMBuildAnd(builder, output_smask, out_smask_idx, "");
1346 LLVMValueRef cmp = LLVMBuildICmp(builder, LLVMIntNE, smask_bit, lp_build_const_int_vec(gallivm, int_type, 0), "");
1347 smask_bit = LLVMBuildSExt(builder, cmp, int_vec_type, "");
1348
1349 s_mask = LLVMBuildAnd(builder, s_mask, smask_bit, "");
1350 }
1351
1352
1353 if (key->multisample) {
1354 /* store the sample mask for this loop */
1355 LLVMBuildStore(builder, s_mask, s_mask_ptr);
1356 lp_build_for_loop_end(&sample_loop_state);
1357 }
1358
1359 mask_val = lp_build_mask_end(&mask);
1360 if (!key->multisample)
1361 LLVMBuildStore(builder, mask_val, mask_ptr);
1362 lp_build_for_loop_end(&loop_state);
1363 }
1364
1365
1366 /**
1367 * This function will reorder pixels from the fragment shader SoA to memory layout AoS
1368 *
1369 * Fragment Shader outputs pixels in small 2x2 blocks
1370 * e.g. (0, 0), (1, 0), (0, 1), (1, 1) ; (2, 0) ...
1371 *
1372 * However in memory pixels are stored in rows
1373 * e.g. (0, 0), (1, 0), (2, 0), (3, 0) ; (0, 1) ...
1374 *
1375 * @param type fragment shader type (4x or 8x float)
1376 * @param num_fs number of fs_src
1377 * @param is_1d whether we're outputting to a 1d resource
1378 * @param dst_channels number of output channels
1379 * @param fs_src output from fragment shader
1380 * @param dst pointer to store result
1381 * @param pad_inline is channel padding inline or at end of row
1382 * @return the number of dsts
1383 */
1384 static int
generate_fs_twiddle(struct gallivm_state * gallivm,struct lp_type type,unsigned num_fs,unsigned dst_channels,LLVMValueRef fs_src[][4],LLVMValueRef * dst,bool pad_inline)1385 generate_fs_twiddle(struct gallivm_state *gallivm,
1386 struct lp_type type,
1387 unsigned num_fs,
1388 unsigned dst_channels,
1389 LLVMValueRef fs_src[][4],
1390 LLVMValueRef* dst,
1391 bool pad_inline)
1392 {
1393 LLVMValueRef src[16];
1394
1395 bool swizzle_pad;
1396 bool twiddle;
1397 bool split;
1398
1399 unsigned pixels = type.length / 4;
1400 unsigned reorder_group;
1401 unsigned src_channels;
1402 unsigned src_count;
1403 unsigned i;
1404
1405 src_channels = dst_channels < 3 ? dst_channels : 4;
1406 src_count = num_fs * src_channels;
1407
1408 assert(pixels == 2 || pixels == 1);
1409 assert(num_fs * src_channels <= ARRAY_SIZE(src));
1410
1411 /*
1412 * Transpose from SoA -> AoS
1413 */
1414 for (i = 0; i < num_fs; ++i) {
1415 lp_build_transpose_aos_n(gallivm, type, &fs_src[i][0], src_channels, &src[i * src_channels]);
1416 }
1417
1418 /*
1419 * Pick transformation options
1420 */
1421 swizzle_pad = false;
1422 twiddle = false;
1423 split = false;
1424 reorder_group = 0;
1425
1426 if (dst_channels == 1) {
1427 twiddle = true;
1428
1429 if (pixels == 2) {
1430 split = true;
1431 }
1432 } else if (dst_channels == 2) {
1433 if (pixels == 1) {
1434 reorder_group = 1;
1435 }
1436 } else if (dst_channels > 2) {
1437 if (pixels == 1) {
1438 reorder_group = 2;
1439 } else {
1440 twiddle = true;
1441 }
1442
1443 if (!pad_inline && dst_channels == 3 && pixels > 1) {
1444 swizzle_pad = true;
1445 }
1446 }
1447
1448 /*
1449 * Split the src in half
1450 */
1451 if (split) {
1452 for (i = num_fs; i > 0; --i) {
1453 src[(i - 1)*2 + 1] = lp_build_extract_range(gallivm, src[i - 1], 4, 4);
1454 src[(i - 1)*2 + 0] = lp_build_extract_range(gallivm, src[i - 1], 0, 4);
1455 }
1456
1457 src_count *= 2;
1458 type.length = 4;
1459 }
1460
1461 /*
1462 * Ensure pixels are in memory order
1463 */
1464 if (reorder_group) {
1465 /* Twiddle pixels by reordering the array, e.g.:
1466 *
1467 * src_count = 8 -> 0 2 1 3 4 6 5 7
1468 * src_count = 16 -> 0 1 4 5 2 3 6 7 8 9 12 13 10 11 14 15
1469 */
1470 const unsigned reorder_sw[] = { 0, 2, 1, 3 };
1471
1472 for (i = 0; i < src_count; ++i) {
1473 unsigned group = i / reorder_group;
1474 unsigned block = (group / 4) * 4 * reorder_group;
1475 unsigned j = block + (reorder_sw[group % 4] * reorder_group) + (i % reorder_group);
1476 dst[i] = src[j];
1477 }
1478 } else if (twiddle) {
1479 /* Twiddle pixels across elements of array */
1480 /*
1481 * XXX: we should avoid this in some cases, but would need to tell
1482 * lp_build_conv to reorder (or deal with it ourselves).
1483 */
1484 lp_bld_quad_twiddle(gallivm, type, src, src_count, dst);
1485 } else {
1486 /* Do nothing */
1487 memcpy(dst, src, sizeof(LLVMValueRef) * src_count);
1488 }
1489
1490 /*
1491 * Moves any padding between pixels to the end
1492 * e.g. RGBXRGBX -> RGBRGBXX
1493 */
1494 if (swizzle_pad) {
1495 unsigned char swizzles[16];
1496 unsigned elems = pixels * dst_channels;
1497
1498 for (i = 0; i < type.length; ++i) {
1499 if (i < elems)
1500 swizzles[i] = i % dst_channels + (i / dst_channels) * 4;
1501 else
1502 swizzles[i] = LP_BLD_SWIZZLE_DONTCARE;
1503 }
1504
1505 for (i = 0; i < src_count; ++i) {
1506 dst[i] = lp_build_swizzle_aos_n(gallivm, dst[i], swizzles, type.length, type.length);
1507 }
1508 }
1509
1510 return src_count;
1511 }
1512
1513
1514 /*
1515 * Untwiddle and transpose, much like the above.
1516 * However, this is after conversion, so we get packed vectors.
1517 * At this time only handle 4x16i8 rgba / 2x16i8 rg / 1x16i8 r data,
1518 * the vectors will look like:
1519 * r0r1r4r5r2r3r6r7r8r9r12... (albeit color channels may
1520 * be swizzled here). Extending to 16bit should be trivial.
1521 * Should also be extended to handle twice wide vectors with AVX2...
1522 */
1523 static void
fs_twiddle_transpose(struct gallivm_state * gallivm,struct lp_type type,LLVMValueRef * src,unsigned src_count,LLVMValueRef * dst)1524 fs_twiddle_transpose(struct gallivm_state *gallivm,
1525 struct lp_type type,
1526 LLVMValueRef *src,
1527 unsigned src_count,
1528 LLVMValueRef *dst)
1529 {
1530 struct lp_type type64, type16, type32;
1531 LLVMTypeRef type64_t, type8_t, type16_t, type32_t;
1532 LLVMBuilderRef builder = gallivm->builder;
1533 LLVMValueRef tmp[4], shuf[8];
1534 for (unsigned j = 0; j < 2; j++) {
1535 shuf[j*4 + 0] = lp_build_const_int32(gallivm, j*4 + 0);
1536 shuf[j*4 + 1] = lp_build_const_int32(gallivm, j*4 + 2);
1537 shuf[j*4 + 2] = lp_build_const_int32(gallivm, j*4 + 1);
1538 shuf[j*4 + 3] = lp_build_const_int32(gallivm, j*4 + 3);
1539 }
1540
1541 assert(src_count == 4 || src_count == 2 || src_count == 1);
1542 assert(type.width == 8);
1543 assert(type.length == 16);
1544
1545 type8_t = lp_build_vec_type(gallivm, type);
1546
1547 type64 = type;
1548 type64.length /= 8;
1549 type64.width *= 8;
1550 type64_t = lp_build_vec_type(gallivm, type64);
1551
1552 type16 = type;
1553 type16.length /= 2;
1554 type16.width *= 2;
1555 type16_t = lp_build_vec_type(gallivm, type16);
1556
1557 type32 = type;
1558 type32.length /= 4;
1559 type32.width *= 4;
1560 type32_t = lp_build_vec_type(gallivm, type32);
1561
1562 lp_build_transpose_aos_n(gallivm, type, src, src_count, tmp);
1563
1564 if (src_count == 1) {
1565 /* transpose was no-op, just untwiddle */
1566 LLVMValueRef shuf_vec;
1567 shuf_vec = LLVMConstVector(shuf, 8);
1568 tmp[0] = LLVMBuildBitCast(builder, src[0], type16_t, "");
1569 tmp[0] = LLVMBuildShuffleVector(builder, tmp[0], tmp[0], shuf_vec, "");
1570 dst[0] = LLVMBuildBitCast(builder, tmp[0], type8_t, "");
1571 } else if (src_count == 2) {
1572 LLVMValueRef shuf_vec;
1573 shuf_vec = LLVMConstVector(shuf, 4);
1574
1575 for (unsigned i = 0; i < 2; i++) {
1576 tmp[i] = LLVMBuildBitCast(builder, tmp[i], type32_t, "");
1577 tmp[i] = LLVMBuildShuffleVector(builder, tmp[i], tmp[i], shuf_vec, "");
1578 dst[i] = LLVMBuildBitCast(builder, tmp[i], type8_t, "");
1579 }
1580 } else {
1581 for (unsigned j = 0; j < 2; j++) {
1582 LLVMValueRef lo, hi, lo2, hi2;
1583 /*
1584 * Note that if we only really have 3 valid channels (rgb)
1585 * and we don't need alpha we could substitute a undef here
1586 * for the respective channel (causing llvm to drop conversion
1587 * for alpha).
1588 */
1589 /* we now have rgba0rgba1rgba4rgba5 etc, untwiddle */
1590 lo2 = LLVMBuildBitCast(builder, tmp[j*2], type64_t, "");
1591 hi2 = LLVMBuildBitCast(builder, tmp[j*2 + 1], type64_t, "");
1592 lo = lp_build_interleave2(gallivm, type64, lo2, hi2, 0);
1593 hi = lp_build_interleave2(gallivm, type64, lo2, hi2, 1);
1594 dst[j*2] = LLVMBuildBitCast(builder, lo, type8_t, "");
1595 dst[j*2 + 1] = LLVMBuildBitCast(builder, hi, type8_t, "");
1596 }
1597 }
1598 }
1599
1600
1601 /**
1602 * Load an unswizzled block of pixels from memory
1603 */
1604 static void
load_unswizzled_block(struct gallivm_state * gallivm,LLVMValueRef base_ptr,LLVMValueRef stride,unsigned block_width,unsigned block_height,LLVMValueRef * dst,struct lp_type dst_type,unsigned dst_count,unsigned dst_alignment)1605 load_unswizzled_block(struct gallivm_state *gallivm,
1606 LLVMValueRef base_ptr,
1607 LLVMValueRef stride,
1608 unsigned block_width,
1609 unsigned block_height,
1610 LLVMValueRef* dst,
1611 struct lp_type dst_type,
1612 unsigned dst_count,
1613 unsigned dst_alignment)
1614 {
1615 LLVMBuilderRef builder = gallivm->builder;
1616 const unsigned row_size = dst_count / block_height;
1617
1618 /* Ensure block exactly fits into dst */
1619 assert((block_width * block_height) % dst_count == 0);
1620
1621 for (unsigned i = 0; i < dst_count; ++i) {
1622 unsigned x = i % row_size;
1623 unsigned y = i / row_size;
1624
1625 LLVMValueRef bx = lp_build_const_int32(gallivm, x * (dst_type.width / 8) * dst_type.length);
1626 LLVMValueRef by = LLVMBuildMul(builder, lp_build_const_int32(gallivm, y), stride, "");
1627
1628 LLVMValueRef gep[2];
1629 LLVMValueRef dst_ptr;
1630
1631 gep[0] = lp_build_const_int32(gallivm, 0);
1632 gep[1] = LLVMBuildAdd(builder, bx, by, "");
1633
1634 dst_ptr = LLVMBuildGEP(builder, base_ptr, gep, 2, "");
1635 dst_ptr = LLVMBuildBitCast(builder, dst_ptr,
1636 LLVMPointerType(lp_build_vec_type(gallivm, dst_type), 0), "");
1637
1638 dst[i] = LLVMBuildLoad(builder, dst_ptr, "");
1639
1640 LLVMSetAlignment(dst[i], dst_alignment);
1641 }
1642 }
1643
1644
1645 /**
1646 * Store an unswizzled block of pixels to memory
1647 */
1648 static void
store_unswizzled_block(struct gallivm_state * gallivm,LLVMValueRef base_ptr,LLVMValueRef stride,unsigned block_width,unsigned block_height,LLVMValueRef * src,struct lp_type src_type,unsigned src_count,unsigned src_alignment)1649 store_unswizzled_block(struct gallivm_state *gallivm,
1650 LLVMValueRef base_ptr,
1651 LLVMValueRef stride,
1652 unsigned block_width,
1653 unsigned block_height,
1654 LLVMValueRef* src,
1655 struct lp_type src_type,
1656 unsigned src_count,
1657 unsigned src_alignment)
1658 {
1659 LLVMBuilderRef builder = gallivm->builder;
1660 const unsigned row_size = src_count / block_height;
1661
1662 /* Ensure src exactly fits into block */
1663 assert((block_width * block_height) % src_count == 0);
1664
1665 for (unsigned i = 0; i < src_count; ++i) {
1666 unsigned x = i % row_size;
1667 unsigned y = i / row_size;
1668
1669 LLVMValueRef bx = lp_build_const_int32(gallivm, x * (src_type.width / 8) * src_type.length);
1670 LLVMValueRef by = LLVMBuildMul(builder, lp_build_const_int32(gallivm, y), stride, "");
1671
1672 LLVMValueRef gep[2];
1673 LLVMValueRef src_ptr;
1674
1675 gep[0] = lp_build_const_int32(gallivm, 0);
1676 gep[1] = LLVMBuildAdd(builder, bx, by, "");
1677
1678 src_ptr = LLVMBuildGEP(builder, base_ptr, gep, 2, "");
1679 src_ptr = LLVMBuildBitCast(builder, src_ptr,
1680 LLVMPointerType(lp_build_vec_type(gallivm, src_type), 0), "");
1681
1682 src_ptr = LLVMBuildStore(builder, src[i], src_ptr);
1683
1684 LLVMSetAlignment(src_ptr, src_alignment);
1685 }
1686 }
1687
1688
1689
1690 /**
1691 * Retrieves the type for a format which is usable in the blending code.
1692 *
1693 * e.g. RGBA16F = 4x float, R3G3B2 = 3x byte
1694 */
1695 static inline void
lp_blend_type_from_format_desc(const struct util_format_description * format_desc,struct lp_type * type)1696 lp_blend_type_from_format_desc(const struct util_format_description *format_desc,
1697 struct lp_type* type)
1698 {
1699 if (format_expands_to_float_soa(format_desc)) {
1700 /* always use ordinary floats for blending */
1701 type->floating = true;
1702 type->fixed = false;
1703 type->sign = true;
1704 type->norm = false;
1705 type->width = 32;
1706 type->length = 4;
1707 return;
1708 }
1709
1710 unsigned i;
1711 for (i = 0; i < 4; i++)
1712 if (format_desc->channel[i].type != UTIL_FORMAT_TYPE_VOID)
1713 break;
1714 const unsigned chan = i;
1715
1716 memset(type, 0, sizeof(struct lp_type));
1717 type->floating = format_desc->channel[chan].type == UTIL_FORMAT_TYPE_FLOAT;
1718 type->fixed = format_desc->channel[chan].type == UTIL_FORMAT_TYPE_FIXED;
1719 type->sign = format_desc->channel[chan].type != UTIL_FORMAT_TYPE_UNSIGNED;
1720 type->norm = format_desc->channel[chan].normalized;
1721 type->width = format_desc->channel[chan].size;
1722 type->length = format_desc->nr_channels;
1723
1724 for (unsigned i = 1; i < format_desc->nr_channels; ++i) {
1725 if (format_desc->channel[i].size > type->width)
1726 type->width = format_desc->channel[i].size;
1727 }
1728
1729 if (type->floating) {
1730 type->width = 32;
1731 } else {
1732 if (type->width <= 8) {
1733 type->width = 8;
1734 } else if (type->width <= 16) {
1735 type->width = 16;
1736 } else {
1737 type->width = 32;
1738 }
1739 }
1740
1741 if (is_arithmetic_format(format_desc) && type->length == 3) {
1742 type->length = 4;
1743 }
1744 }
1745
1746
1747 /**
1748 * Scale a normalized value from src_bits to dst_bits.
1749 *
1750 * The exact calculation is
1751 *
1752 * dst = iround(src * dst_mask / src_mask)
1753 *
1754 * or with integer rounding
1755 *
1756 * dst = src * (2*dst_mask + sign(src)*src_mask) / (2*src_mask)
1757 *
1758 * where
1759 *
1760 * src_mask = (1 << src_bits) - 1
1761 * dst_mask = (1 << dst_bits) - 1
1762 *
1763 * but we try to avoid division and multiplication through shifts.
1764 */
1765 static inline LLVMValueRef
scale_bits(struct gallivm_state * gallivm,int src_bits,int dst_bits,LLVMValueRef src,struct lp_type src_type)1766 scale_bits(struct gallivm_state *gallivm,
1767 int src_bits,
1768 int dst_bits,
1769 LLVMValueRef src,
1770 struct lp_type src_type)
1771 {
1772 LLVMBuilderRef builder = gallivm->builder;
1773 LLVMValueRef result = src;
1774
1775 if (dst_bits < src_bits) {
1776 int delta_bits = src_bits - dst_bits;
1777
1778 if (delta_bits <= dst_bits) {
1779
1780 if (dst_bits == 4) {
1781 struct lp_type flt_type = lp_type_float_vec(32, src_type.length * 32);
1782
1783 result = lp_build_unsigned_norm_to_float(gallivm, src_bits, flt_type, src);
1784 result = lp_build_clamped_float_to_unsigned_norm(gallivm, flt_type, dst_bits, result);
1785 result = LLVMBuildTrunc(gallivm->builder, result, lp_build_int_vec_type(gallivm, src_type), "");
1786 return result;
1787 }
1788
1789 /*
1790 * Approximate the rescaling with a single shift.
1791 *
1792 * This gives the wrong rounding.
1793 */
1794
1795 result = LLVMBuildLShr(builder,
1796 src,
1797 lp_build_const_int_vec(gallivm, src_type, delta_bits),
1798 "");
1799
1800 } else {
1801 /*
1802 * Try more accurate rescaling.
1803 */
1804
1805 /*
1806 * Drop the least significant bits to make space for the multiplication.
1807 *
1808 * XXX: A better approach would be to use a wider integer type as intermediate. But
1809 * this is enough to convert alpha from 16bits -> 2 when rendering to
1810 * PIPE_FORMAT_R10G10B10A2_UNORM.
1811 */
1812 result = LLVMBuildLShr(builder,
1813 src,
1814 lp_build_const_int_vec(gallivm, src_type, dst_bits),
1815 "");
1816
1817
1818 result = LLVMBuildMul(builder,
1819 result,
1820 lp_build_const_int_vec(gallivm, src_type, (1LL << dst_bits) - 1),
1821 "");
1822
1823 /*
1824 * Add a rounding term before the division.
1825 *
1826 * TODO: Handle signed integers too.
1827 */
1828 if (!src_type.sign) {
1829 result = LLVMBuildAdd(builder,
1830 result,
1831 lp_build_const_int_vec(gallivm, src_type, (1LL << (delta_bits - 1))),
1832 "");
1833 }
1834
1835 /*
1836 * Approximate the division by src_mask with a src_bits shift.
1837 *
1838 * Given the src has already been shifted by dst_bits, all we need
1839 * to do is to shift by the difference.
1840 */
1841
1842 result = LLVMBuildLShr(builder,
1843 result,
1844 lp_build_const_int_vec(gallivm, src_type, delta_bits),
1845 "");
1846 }
1847
1848 } else if (dst_bits > src_bits) {
1849 /* Scale up bits */
1850 int db = dst_bits - src_bits;
1851
1852 /* Shift left by difference in bits */
1853 result = LLVMBuildShl(builder,
1854 src,
1855 lp_build_const_int_vec(gallivm, src_type, db),
1856 "");
1857
1858 if (db <= src_bits) {
1859 /* Enough bits in src to fill the remainder */
1860 LLVMValueRef lower = LLVMBuildLShr(builder,
1861 src,
1862 lp_build_const_int_vec(gallivm, src_type, src_bits - db),
1863 "");
1864
1865 result = LLVMBuildOr(builder, result, lower, "");
1866 } else if (db > src_bits) {
1867 /* Need to repeatedly copy src bits to fill remainder in dst */
1868 unsigned n;
1869
1870 for (n = src_bits; n < dst_bits; n *= 2) {
1871 LLVMValueRef shuv = lp_build_const_int_vec(gallivm, src_type, n);
1872
1873 result = LLVMBuildOr(builder,
1874 result,
1875 LLVMBuildLShr(builder, result, shuv, ""),
1876 "");
1877 }
1878 }
1879 }
1880
1881 return result;
1882 }
1883
1884 /**
1885 * If RT is a smallfloat (needing denorms) format
1886 */
1887 static inline int
have_smallfloat_format(struct lp_type dst_type,enum pipe_format format)1888 have_smallfloat_format(struct lp_type dst_type,
1889 enum pipe_format format)
1890 {
1891 return ((dst_type.floating && dst_type.width != 32) ||
1892 /* due to format handling hacks this format doesn't have floating set
1893 * here (and actually has width set to 32 too) so special case this. */
1894 (format == PIPE_FORMAT_R11G11B10_FLOAT));
1895 }
1896
1897
1898 /**
1899 * Convert from memory format to blending format
1900 *
1901 * e.g. GL_R3G3B2 is 1 byte in memory but 3 bytes for blending
1902 */
1903 static void
convert_to_blend_type(struct gallivm_state * gallivm,unsigned block_size,const struct util_format_description * src_fmt,struct lp_type src_type,struct lp_type dst_type,LLVMValueRef * src,unsigned num_srcs)1904 convert_to_blend_type(struct gallivm_state *gallivm,
1905 unsigned block_size,
1906 const struct util_format_description *src_fmt,
1907 struct lp_type src_type,
1908 struct lp_type dst_type,
1909 LLVMValueRef* src, // and dst
1910 unsigned num_srcs)
1911 {
1912 LLVMValueRef *dst = src;
1913 LLVMBuilderRef builder = gallivm->builder;
1914 struct lp_type blend_type;
1915 struct lp_type mem_type;
1916 unsigned i, j;
1917 unsigned pixels = block_size / num_srcs;
1918 bool is_arith;
1919
1920 /*
1921 * full custom path for packed floats and srgb formats - none of the later
1922 * functions would do anything useful, and given the lp_type representation they
1923 * can't be fixed. Should really have some SoA blend path for these kind of
1924 * formats rather than hacking them in here.
1925 */
1926 if (format_expands_to_float_soa(src_fmt)) {
1927 LLVMValueRef tmpsrc[4];
1928 /*
1929 * This is pretty suboptimal for this case blending in SoA would be much
1930 * better, since conversion gets us SoA values so need to convert back.
1931 */
1932 assert(src_type.width == 32 || src_type.width == 16);
1933 assert(dst_type.floating);
1934 assert(dst_type.width == 32);
1935 assert(dst_type.length % 4 == 0);
1936 assert(num_srcs % 4 == 0);
1937
1938 if (src_type.width == 16) {
1939 /* expand 4x16bit values to 4x32bit */
1940 struct lp_type type32x4 = src_type;
1941 LLVMTypeRef ltype32x4;
1942 unsigned num_fetch = dst_type.length == 8 ? num_srcs / 2 : num_srcs / 4;
1943 type32x4.width = 32;
1944 ltype32x4 = lp_build_vec_type(gallivm, type32x4);
1945 for (i = 0; i < num_fetch; i++) {
1946 src[i] = LLVMBuildZExt(builder, src[i], ltype32x4, "");
1947 }
1948 src_type.width = 32;
1949 }
1950 for (i = 0; i < 4; i++) {
1951 tmpsrc[i] = src[i];
1952 }
1953 for (i = 0; i < num_srcs / 4; i++) {
1954 LLVMValueRef tmpsoa[4];
1955 LLVMValueRef tmps = tmpsrc[i];
1956 if (dst_type.length == 8) {
1957 LLVMValueRef shuffles[8];
1958 unsigned j;
1959 /* fetch was 4 values but need 8-wide output values */
1960 tmps = lp_build_concat(gallivm, &tmpsrc[i * 2], src_type, 2);
1961 /*
1962 * for 8-wide aos transpose would give us wrong order not matching
1963 * incoming converted fs values and mask. ARGH.
1964 */
1965 for (j = 0; j < 4; j++) {
1966 shuffles[j] = lp_build_const_int32(gallivm, j * 2);
1967 shuffles[j + 4] = lp_build_const_int32(gallivm, j * 2 + 1);
1968 }
1969 tmps = LLVMBuildShuffleVector(builder, tmps, tmps,
1970 LLVMConstVector(shuffles, 8), "");
1971 }
1972 if (src_fmt->format == PIPE_FORMAT_R11G11B10_FLOAT) {
1973 lp_build_r11g11b10_to_float(gallivm, tmps, tmpsoa);
1974 }
1975 else {
1976 lp_build_unpack_rgba_soa(gallivm, src_fmt, dst_type, tmps, tmpsoa);
1977 }
1978 lp_build_transpose_aos(gallivm, dst_type, tmpsoa, &src[i * 4]);
1979 }
1980 return;
1981 }
1982
1983 lp_mem_type_from_format_desc(src_fmt, &mem_type);
1984 lp_blend_type_from_format_desc(src_fmt, &blend_type);
1985
1986 /* Is the format arithmetic */
1987 is_arith = blend_type.length * blend_type.width != mem_type.width * mem_type.length;
1988 is_arith &= !(mem_type.width == 16 && mem_type.floating);
1989
1990 /* Pad if necessary */
1991 if (!is_arith && src_type.length < dst_type.length) {
1992 for (i = 0; i < num_srcs; ++i) {
1993 dst[i] = lp_build_pad_vector(gallivm, src[i], dst_type.length);
1994 }
1995
1996 src_type.length = dst_type.length;
1997 }
1998
1999 /* Special case for half-floats */
2000 if (mem_type.width == 16 && mem_type.floating) {
2001 assert(blend_type.width == 32 && blend_type.floating);
2002 lp_build_conv_auto(gallivm, src_type, &dst_type, dst, num_srcs, dst);
2003 is_arith = false;
2004 }
2005
2006 if (!is_arith) {
2007 return;
2008 }
2009
2010 src_type.width = blend_type.width * blend_type.length;
2011 blend_type.length *= pixels;
2012 src_type.length *= pixels / (src_type.length / mem_type.length);
2013
2014 for (i = 0; i < num_srcs; ++i) {
2015 LLVMValueRef chans;
2016 LLVMValueRef res = NULL;
2017
2018 dst[i] = LLVMBuildZExt(builder, src[i], lp_build_vec_type(gallivm, src_type), "");
2019
2020 for (j = 0; j < src_fmt->nr_channels; ++j) {
2021 unsigned mask = 0;
2022 unsigned sa = src_fmt->channel[j].shift;
2023 #if UTIL_ARCH_LITTLE_ENDIAN
2024 unsigned from_lsb = j;
2025 #else
2026 unsigned from_lsb = (blend_type.length / pixels) - j - 1;
2027 #endif
2028
2029 mask = (1 << src_fmt->channel[j].size) - 1;
2030
2031 /* Extract bits from source */
2032 chans = LLVMBuildLShr(builder,
2033 dst[i],
2034 lp_build_const_int_vec(gallivm, src_type, sa),
2035 "");
2036
2037 chans = LLVMBuildAnd(builder,
2038 chans,
2039 lp_build_const_int_vec(gallivm, src_type, mask),
2040 "");
2041
2042 /* Scale bits */
2043 if (src_type.norm) {
2044 chans = scale_bits(gallivm, src_fmt->channel[j].size,
2045 blend_type.width, chans, src_type);
2046 }
2047
2048 /* Insert bits into correct position */
2049 chans = LLVMBuildShl(builder,
2050 chans,
2051 lp_build_const_int_vec(gallivm, src_type, from_lsb * blend_type.width),
2052 "");
2053
2054 if (j == 0) {
2055 res = chans;
2056 } else {
2057 res = LLVMBuildOr(builder, res, chans, "");
2058 }
2059 }
2060
2061 dst[i] = LLVMBuildBitCast(builder, res, lp_build_vec_type(gallivm, blend_type), "");
2062 }
2063 }
2064
2065
2066 /**
2067 * Convert from blending format to memory format
2068 *
2069 * e.g. GL_R3G3B2 is 3 bytes for blending but 1 byte in memory
2070 */
2071 static void
convert_from_blend_type(struct gallivm_state * gallivm,unsigned block_size,const struct util_format_description * src_fmt,struct lp_type src_type,struct lp_type dst_type,LLVMValueRef * src,unsigned num_srcs)2072 convert_from_blend_type(struct gallivm_state *gallivm,
2073 unsigned block_size,
2074 const struct util_format_description *src_fmt,
2075 struct lp_type src_type,
2076 struct lp_type dst_type,
2077 LLVMValueRef* src, // and dst
2078 unsigned num_srcs)
2079 {
2080 LLVMValueRef* dst = src;
2081 unsigned i, j, k;
2082 struct lp_type mem_type;
2083 struct lp_type blend_type;
2084 LLVMBuilderRef builder = gallivm->builder;
2085 unsigned pixels = block_size / num_srcs;
2086 bool is_arith;
2087
2088 /*
2089 * full custom path for packed floats and srgb formats - none of the later
2090 * functions would do anything useful, and given the lp_type representation they
2091 * can't be fixed. Should really have some SoA blend path for these kind of
2092 * formats rather than hacking them in here.
2093 */
2094 if (format_expands_to_float_soa(src_fmt)) {
2095 /*
2096 * This is pretty suboptimal for this case blending in SoA would be much
2097 * better - we need to transpose the AoS values back to SoA values for
2098 * conversion/packing.
2099 */
2100 assert(src_type.floating);
2101 assert(src_type.width == 32);
2102 assert(src_type.length % 4 == 0);
2103 assert(dst_type.width == 32 || dst_type.width == 16);
2104
2105 for (i = 0; i < num_srcs / 4; i++) {
2106 LLVMValueRef tmpsoa[4], tmpdst;
2107 lp_build_transpose_aos(gallivm, src_type, &src[i * 4], tmpsoa);
2108 /* really really need SoA here */
2109
2110 if (src_fmt->format == PIPE_FORMAT_R11G11B10_FLOAT) {
2111 tmpdst = lp_build_float_to_r11g11b10(gallivm, tmpsoa);
2112 }
2113 else {
2114 tmpdst = lp_build_float_to_srgb_packed(gallivm, src_fmt,
2115 src_type, tmpsoa);
2116 }
2117
2118 if (src_type.length == 8) {
2119 LLVMValueRef tmpaos, shuffles[8];
2120 unsigned j;
2121 /*
2122 * for 8-wide aos transpose has given us wrong order not matching
2123 * output order. HMPF. Also need to split the output values manually.
2124 */
2125 for (j = 0; j < 4; j++) {
2126 shuffles[j * 2] = lp_build_const_int32(gallivm, j);
2127 shuffles[j * 2 + 1] = lp_build_const_int32(gallivm, j + 4);
2128 }
2129 tmpaos = LLVMBuildShuffleVector(builder, tmpdst, tmpdst,
2130 LLVMConstVector(shuffles, 8), "");
2131 src[i * 2] = lp_build_extract_range(gallivm, tmpaos, 0, 4);
2132 src[i * 2 + 1] = lp_build_extract_range(gallivm, tmpaos, 4, 4);
2133 }
2134 else {
2135 src[i] = tmpdst;
2136 }
2137 }
2138 if (dst_type.width == 16) {
2139 struct lp_type type16x8 = dst_type;
2140 struct lp_type type32x4 = dst_type;
2141 LLVMTypeRef ltype16x4, ltypei64, ltypei128;
2142 unsigned num_fetch = src_type.length == 8 ? num_srcs / 2 : num_srcs / 4;
2143 type16x8.length = 8;
2144 type32x4.width = 32;
2145 ltypei128 = LLVMIntTypeInContext(gallivm->context, 128);
2146 ltypei64 = LLVMIntTypeInContext(gallivm->context, 64);
2147 ltype16x4 = lp_build_vec_type(gallivm, dst_type);
2148 /* We could do vector truncation but it doesn't generate very good code */
2149 for (i = 0; i < num_fetch; i++) {
2150 src[i] = lp_build_pack2(gallivm, type32x4, type16x8,
2151 src[i], lp_build_zero(gallivm, type32x4));
2152 src[i] = LLVMBuildBitCast(builder, src[i], ltypei128, "");
2153 src[i] = LLVMBuildTrunc(builder, src[i], ltypei64, "");
2154 src[i] = LLVMBuildBitCast(builder, src[i], ltype16x4, "");
2155 }
2156 }
2157 return;
2158 }
2159
2160 lp_mem_type_from_format_desc(src_fmt, &mem_type);
2161 lp_blend_type_from_format_desc(src_fmt, &blend_type);
2162
2163 is_arith = (blend_type.length * blend_type.width != mem_type.width * mem_type.length);
2164
2165 /* Special case for half-floats */
2166 if (mem_type.width == 16 && mem_type.floating) {
2167 int length = dst_type.length;
2168 assert(blend_type.width == 32 && blend_type.floating);
2169
2170 dst_type.length = src_type.length;
2171
2172 lp_build_conv_auto(gallivm, src_type, &dst_type, dst, num_srcs, dst);
2173
2174 dst_type.length = length;
2175 is_arith = false;
2176 }
2177
2178 /* Remove any padding */
2179 if (!is_arith && (src_type.length % mem_type.length)) {
2180 src_type.length -= (src_type.length % mem_type.length);
2181
2182 for (i = 0; i < num_srcs; ++i) {
2183 dst[i] = lp_build_extract_range(gallivm, dst[i], 0, src_type.length);
2184 }
2185 }
2186
2187 /* No bit arithmetic to do */
2188 if (!is_arith) {
2189 return;
2190 }
2191
2192 src_type.length = pixels;
2193 src_type.width = blend_type.length * blend_type.width;
2194 dst_type.length = pixels;
2195
2196 for (i = 0; i < num_srcs; ++i) {
2197 LLVMValueRef chans;
2198 LLVMValueRef res = NULL;
2199
2200 dst[i] = LLVMBuildBitCast(builder, src[i], lp_build_vec_type(gallivm, src_type), "");
2201
2202 for (j = 0; j < src_fmt->nr_channels; ++j) {
2203 unsigned mask = 0;
2204 unsigned sa = src_fmt->channel[j].shift;
2205 unsigned sz_a = src_fmt->channel[j].size;
2206 #if UTIL_ARCH_LITTLE_ENDIAN
2207 unsigned from_lsb = j;
2208 #else
2209 unsigned from_lsb = blend_type.length - j - 1;
2210 #endif
2211
2212 assert(blend_type.width > src_fmt->channel[j].size);
2213
2214 for (k = 0; k < blend_type.width; ++k) {
2215 mask |= 1 << k;
2216 }
2217
2218 /* Extract bits */
2219 chans = LLVMBuildLShr(builder,
2220 dst[i],
2221 lp_build_const_int_vec(gallivm, src_type,
2222 from_lsb * blend_type.width),
2223 "");
2224
2225 chans = LLVMBuildAnd(builder,
2226 chans,
2227 lp_build_const_int_vec(gallivm, src_type, mask),
2228 "");
2229
2230 /* Scale down bits */
2231 if (src_type.norm) {
2232 chans = scale_bits(gallivm, blend_type.width,
2233 src_fmt->channel[j].size, chans, src_type);
2234 } else if (!src_type.floating && sz_a < blend_type.width) {
2235 LLVMValueRef mask_val = lp_build_const_int_vec(gallivm, src_type, (1UL << sz_a) - 1);
2236 LLVMValueRef mask = LLVMBuildICmp(builder, LLVMIntUGT, chans, mask_val, "");
2237 chans = LLVMBuildSelect(builder, mask, mask_val, chans, "");
2238 }
2239
2240 /* Insert bits */
2241 chans = LLVMBuildShl(builder,
2242 chans,
2243 lp_build_const_int_vec(gallivm, src_type, sa),
2244 "");
2245
2246 sa += src_fmt->channel[j].size;
2247
2248 if (j == 0) {
2249 res = chans;
2250 } else {
2251 res = LLVMBuildOr(builder, res, chans, "");
2252 }
2253 }
2254
2255 assert (dst_type.width != 24);
2256
2257 dst[i] = LLVMBuildTrunc(builder, res, lp_build_vec_type(gallivm, dst_type), "");
2258 }
2259 }
2260
2261
2262 /**
2263 * Convert alpha to same blend type as src
2264 */
2265 static void
convert_alpha(struct gallivm_state * gallivm,struct lp_type row_type,struct lp_type alpha_type,const unsigned block_size,const unsigned block_height,const unsigned src_count,const unsigned dst_channels,const bool pad_inline,LLVMValueRef * src_alpha)2266 convert_alpha(struct gallivm_state *gallivm,
2267 struct lp_type row_type,
2268 struct lp_type alpha_type,
2269 const unsigned block_size,
2270 const unsigned block_height,
2271 const unsigned src_count,
2272 const unsigned dst_channels,
2273 const bool pad_inline,
2274 LLVMValueRef* src_alpha)
2275 {
2276 LLVMBuilderRef builder = gallivm->builder;
2277 unsigned i, j;
2278 unsigned length = row_type.length;
2279 row_type.length = alpha_type.length;
2280
2281 /* Twiddle the alpha to match pixels */
2282 lp_bld_quad_twiddle(gallivm, alpha_type, src_alpha, block_height, src_alpha);
2283
2284 /*
2285 * TODO this should use single lp_build_conv call for
2286 * src_count == 1 && dst_channels == 1 case (dropping the concat below)
2287 */
2288 for (i = 0; i < block_height; ++i) {
2289 lp_build_conv(gallivm, alpha_type, row_type, &src_alpha[i], 1, &src_alpha[i], 1);
2290 }
2291
2292 alpha_type = row_type;
2293 row_type.length = length;
2294
2295 /* If only one channel we can only need the single alpha value per pixel */
2296 if (src_count == 1 && dst_channels == 1) {
2297
2298 lp_build_concat_n(gallivm, alpha_type, src_alpha, block_height, src_alpha, src_count);
2299 } else {
2300 /* If there are more srcs than rows then we need to split alpha up */
2301 if (src_count > block_height) {
2302 for (i = src_count; i > 0; --i) {
2303 unsigned pixels = block_size / src_count;
2304 unsigned idx = i - 1;
2305
2306 src_alpha[idx] = lp_build_extract_range(gallivm, src_alpha[(idx * pixels) / 4],
2307 (idx * pixels) % 4, pixels);
2308 }
2309 }
2310
2311 /* If there is a src for each pixel broadcast the alpha across whole row */
2312 if (src_count == block_size) {
2313 for (i = 0; i < src_count; ++i) {
2314 src_alpha[i] = lp_build_broadcast(gallivm,
2315 lp_build_vec_type(gallivm, row_type), src_alpha[i]);
2316 }
2317 } else {
2318 unsigned pixels = block_size / src_count;
2319 unsigned channels = pad_inline ? TGSI_NUM_CHANNELS : dst_channels;
2320 unsigned alpha_span = 1;
2321 LLVMValueRef shuffles[LP_MAX_VECTOR_LENGTH];
2322
2323 /* Check if we need 2 src_alphas for our shuffles */
2324 if (pixels > alpha_type.length) {
2325 alpha_span = 2;
2326 }
2327
2328 /* Broadcast alpha across all channels, e.g. a1a2 to a1a1a1a1a2a2a2a2 */
2329 for (j = 0; j < row_type.length; ++j) {
2330 if (j < pixels * channels) {
2331 shuffles[j] = lp_build_const_int32(gallivm, j / channels);
2332 } else {
2333 shuffles[j] = LLVMGetUndef(LLVMInt32TypeInContext(gallivm->context));
2334 }
2335 }
2336
2337 for (i = 0; i < src_count; ++i) {
2338 unsigned idx1 = i, idx2 = i;
2339
2340 if (alpha_span > 1){
2341 idx1 *= alpha_span;
2342 idx2 = idx1 + 1;
2343 }
2344
2345 src_alpha[i] = LLVMBuildShuffleVector(builder,
2346 src_alpha[idx1],
2347 src_alpha[idx2],
2348 LLVMConstVector(shuffles, row_type.length),
2349 "");
2350 }
2351 }
2352 }
2353 }
2354
2355
2356 /**
2357 * Generates the blend function for unswizzled colour buffers
2358 * Also generates the read & write from colour buffer
2359 */
2360 static void
generate_unswizzled_blend(struct gallivm_state * gallivm,unsigned rt,struct lp_fragment_shader_variant * variant,enum pipe_format out_format,unsigned int num_fs,struct lp_type fs_type,LLVMValueRef * fs_mask,LLVMValueRef fs_out_color[PIPE_MAX_COLOR_BUFS][TGSI_NUM_CHANNELS][4],LLVMValueRef context_ptr,LLVMValueRef color_ptr,LLVMValueRef stride,unsigned partial_mask,boolean do_branch)2361 generate_unswizzled_blend(struct gallivm_state *gallivm,
2362 unsigned rt,
2363 struct lp_fragment_shader_variant *variant,
2364 enum pipe_format out_format,
2365 unsigned int num_fs,
2366 struct lp_type fs_type,
2367 LLVMValueRef* fs_mask,
2368 LLVMValueRef fs_out_color[PIPE_MAX_COLOR_BUFS][TGSI_NUM_CHANNELS][4],
2369 LLVMValueRef context_ptr,
2370 LLVMValueRef color_ptr,
2371 LLVMValueRef stride,
2372 unsigned partial_mask,
2373 boolean do_branch)
2374 {
2375 const unsigned alpha_channel = 3;
2376 const unsigned block_width = LP_RASTER_BLOCK_SIZE;
2377 const unsigned block_height = LP_RASTER_BLOCK_SIZE;
2378 const unsigned block_size = block_width * block_height;
2379 const unsigned lp_integer_vector_width = 128;
2380
2381 LLVMBuilderRef builder = gallivm->builder;
2382 LLVMValueRef fs_src[4][TGSI_NUM_CHANNELS];
2383 LLVMValueRef fs_src1[4][TGSI_NUM_CHANNELS];
2384 LLVMValueRef src_alpha[4 * 4];
2385 LLVMValueRef src1_alpha[4 * 4] = { NULL };
2386 LLVMValueRef src_mask[4 * 4];
2387 LLVMValueRef src[4 * 4];
2388 LLVMValueRef src1[4 * 4];
2389 LLVMValueRef dst[4 * 4];
2390 LLVMValueRef blend_color;
2391 LLVMValueRef blend_alpha;
2392 LLVMValueRef i32_zero;
2393 LLVMValueRef check_mask;
2394 LLVMValueRef undef_src_val;
2395
2396 struct lp_build_mask_context mask_ctx;
2397 struct lp_type mask_type;
2398 struct lp_type blend_type;
2399 struct lp_type row_type;
2400 struct lp_type dst_type;
2401 struct lp_type ls_type;
2402
2403 unsigned char swizzle[TGSI_NUM_CHANNELS];
2404 unsigned vector_width;
2405 unsigned src_channels = TGSI_NUM_CHANNELS;
2406 unsigned dst_channels;
2407 unsigned dst_count;
2408 unsigned src_count;
2409
2410 const struct util_format_description* out_format_desc = util_format_description(out_format);
2411
2412 unsigned dst_alignment;
2413
2414 bool pad_inline = is_arithmetic_format(out_format_desc);
2415 bool has_alpha = false;
2416 const boolean dual_source_blend = variant->key.blend.rt[0].blend_enable &&
2417 util_blend_state_is_dual(&variant->key.blend, 0);
2418
2419 const boolean is_1d = variant->key.resource_1d;
2420 boolean twiddle_after_convert = FALSE;
2421 unsigned num_fullblock_fs = is_1d ? 2 * num_fs : num_fs;
2422 LLVMValueRef fpstate = 0;
2423
2424 /* Get type from output format */
2425 lp_blend_type_from_format_desc(out_format_desc, &row_type);
2426 lp_mem_type_from_format_desc(out_format_desc, &dst_type);
2427
2428 /*
2429 * Technically this code should go into lp_build_smallfloat_to_float
2430 * and lp_build_float_to_smallfloat but due to the
2431 * http://llvm.org/bugs/show_bug.cgi?id=6393
2432 * llvm reorders the mxcsr intrinsics in a way that breaks the code.
2433 * So the ordering is important here and there shouldn't be any
2434 * llvm ir instrunctions in this function before
2435 * this, otherwise half-float format conversions won't work
2436 * (again due to llvm bug #6393).
2437 */
2438 if (have_smallfloat_format(dst_type, out_format)) {
2439 /* We need to make sure that denorms are ok for half float
2440 conversions */
2441 fpstate = lp_build_fpstate_get(gallivm);
2442 lp_build_fpstate_set_denorms_zero(gallivm, FALSE);
2443 }
2444
2445 mask_type = lp_int32_vec4_type();
2446 mask_type.length = fs_type.length;
2447
2448 for (unsigned i = num_fs; i < num_fullblock_fs; i++) {
2449 fs_mask[i] = lp_build_zero(gallivm, mask_type);
2450 }
2451
2452 /* Do not bother executing code when mask is empty.. */
2453 if (do_branch) {
2454 check_mask = LLVMConstNull(lp_build_int_vec_type(gallivm, mask_type));
2455
2456 for (unsigned i = 0; i < num_fullblock_fs; ++i) {
2457 check_mask = LLVMBuildOr(builder, check_mask, fs_mask[i], "");
2458 }
2459
2460 lp_build_mask_begin(&mask_ctx, gallivm, mask_type, check_mask);
2461 lp_build_mask_check(&mask_ctx);
2462 }
2463
2464 partial_mask |= !variant->opaque;
2465 i32_zero = lp_build_const_int32(gallivm, 0);
2466
2467 undef_src_val = lp_build_undef(gallivm, fs_type);
2468
2469 row_type.length = fs_type.length;
2470 vector_width = dst_type.floating ? lp_native_vector_width : lp_integer_vector_width;
2471
2472 /* Compute correct swizzle and count channels */
2473 memset(swizzle, LP_BLD_SWIZZLE_DONTCARE, TGSI_NUM_CHANNELS);
2474 dst_channels = 0;
2475
2476 for (unsigned i = 0; i < TGSI_NUM_CHANNELS; ++i) {
2477 /* Ensure channel is used */
2478 if (out_format_desc->swizzle[i] >= TGSI_NUM_CHANNELS) {
2479 continue;
2480 }
2481
2482 /* Ensure not already written to (happens in case with GL_ALPHA) */
2483 if (swizzle[out_format_desc->swizzle[i]] < TGSI_NUM_CHANNELS) {
2484 continue;
2485 }
2486
2487 /* Ensure we haven't already found all channels */
2488 if (dst_channels >= out_format_desc->nr_channels) {
2489 continue;
2490 }
2491
2492 swizzle[out_format_desc->swizzle[i]] = i;
2493 ++dst_channels;
2494
2495 if (i == alpha_channel) {
2496 has_alpha = true;
2497 }
2498 }
2499
2500 if (format_expands_to_float_soa(out_format_desc)) {
2501 /*
2502 * the code above can't work for layout_other
2503 * for srgb it would sort of work but we short-circuit swizzles, etc.
2504 * as that is done as part of unpack / pack.
2505 */
2506 dst_channels = 4; /* HACK: this is fake 4 really but need it due to transpose stuff later */
2507 has_alpha = true;
2508 swizzle[0] = 0;
2509 swizzle[1] = 1;
2510 swizzle[2] = 2;
2511 swizzle[3] = 3;
2512 pad_inline = true; /* HACK: prevent rgbxrgbx->rgbrgbxx conversion later */
2513 }
2514
2515 /* If 3 channels then pad to include alpha for 4 element transpose */
2516 if (dst_channels == 3) {
2517 assert (!has_alpha);
2518 for (unsigned i = 0; i < TGSI_NUM_CHANNELS; i++) {
2519 if (swizzle[i] > TGSI_NUM_CHANNELS)
2520 swizzle[i] = 3;
2521 }
2522 if (out_format_desc->nr_channels == 4) {
2523 dst_channels = 4;
2524 /*
2525 * We use alpha from the color conversion, not separate one.
2526 * We had to include it for transpose, hence it will get converted
2527 * too (albeit when doing transpose after conversion, that would
2528 * no longer be the case necessarily).
2529 * (It works only with 4 channel dsts, e.g. rgbx formats, because
2530 * otherwise we really have padding, not alpha, included.)
2531 */
2532 has_alpha = true;
2533 }
2534 }
2535
2536 /*
2537 * Load shader output
2538 */
2539 for (unsigned i = 0; i < num_fullblock_fs; ++i) {
2540 /* Always load alpha for use in blending */
2541 LLVMValueRef alpha;
2542 if (i < num_fs) {
2543 alpha = LLVMBuildLoad(builder, fs_out_color[rt][alpha_channel][i], "");
2544 }
2545 else {
2546 alpha = undef_src_val;
2547 }
2548
2549 /* Load each channel */
2550 for (unsigned j = 0; j < dst_channels; ++j) {
2551 assert(swizzle[j] < 4);
2552 if (i < num_fs) {
2553 fs_src[i][j] = LLVMBuildLoad(builder, fs_out_color[rt][swizzle[j]][i], "");
2554 }
2555 else {
2556 fs_src[i][j] = undef_src_val;
2557 }
2558 }
2559
2560 /* If 3 channels then pad to include alpha for 4 element transpose */
2561 /*
2562 * XXX If we include that here maybe could actually use it instead of
2563 * separate alpha for blending?
2564 * (Difficult though we actually convert pad channels, not alpha.)
2565 */
2566 if (dst_channels == 3 && !has_alpha) {
2567 fs_src[i][3] = alpha;
2568 }
2569
2570 /* We split the row_mask and row_alpha as we want 128bit interleave */
2571 if (fs_type.length == 8) {
2572 src_mask[i*2 + 0] = lp_build_extract_range(gallivm, fs_mask[i],
2573 0, src_channels);
2574 src_mask[i*2 + 1] = lp_build_extract_range(gallivm, fs_mask[i],
2575 src_channels, src_channels);
2576
2577 src_alpha[i*2 + 0] = lp_build_extract_range(gallivm, alpha, 0, src_channels);
2578 src_alpha[i*2 + 1] = lp_build_extract_range(gallivm, alpha,
2579 src_channels, src_channels);
2580 } else {
2581 src_mask[i] = fs_mask[i];
2582 src_alpha[i] = alpha;
2583 }
2584 }
2585 if (dual_source_blend) {
2586 /* same as above except different src/dst, skip masks and comments... */
2587 for (unsigned i = 0; i < num_fullblock_fs; ++i) {
2588 LLVMValueRef alpha;
2589 if (i < num_fs) {
2590 alpha = LLVMBuildLoad(builder, fs_out_color[1][alpha_channel][i], "");
2591 }
2592 else {
2593 alpha = undef_src_val;
2594 }
2595
2596 for (unsigned j = 0; j < dst_channels; ++j) {
2597 assert(swizzle[j] < 4);
2598 if (i < num_fs) {
2599 fs_src1[i][j] = LLVMBuildLoad(builder, fs_out_color[1][swizzle[j]][i], "");
2600 }
2601 else {
2602 fs_src1[i][j] = undef_src_val;
2603 }
2604 }
2605 if (dst_channels == 3 && !has_alpha) {
2606 fs_src1[i][3] = alpha;
2607 }
2608 if (fs_type.length == 8) {
2609 src1_alpha[i*2 + 0] = lp_build_extract_range(gallivm, alpha, 0, src_channels);
2610 src1_alpha[i*2 + 1] = lp_build_extract_range(gallivm, alpha,
2611 src_channels, src_channels);
2612 } else {
2613 src1_alpha[i] = alpha;
2614 }
2615 }
2616 }
2617
2618 if (util_format_is_pure_integer(out_format)) {
2619 /*
2620 * In this case fs_type was really ints or uints disguised as floats,
2621 * fix that up now.
2622 */
2623 fs_type.floating = 0;
2624 fs_type.sign = dst_type.sign;
2625 for (unsigned i = 0; i < num_fullblock_fs; ++i) {
2626 for (unsigned j = 0; j < dst_channels; ++j) {
2627 fs_src[i][j] = LLVMBuildBitCast(builder, fs_src[i][j],
2628 lp_build_vec_type(gallivm, fs_type), "");
2629 }
2630 if (dst_channels == 3 && !has_alpha) {
2631 fs_src[i][3] = LLVMBuildBitCast(builder, fs_src[i][3],
2632 lp_build_vec_type(gallivm, fs_type), "");
2633 }
2634 }
2635 }
2636
2637 /*
2638 * We actually should generally do conversion first (for non-1d cases)
2639 * when the blend format is 8 or 16 bits. The reason is obvious,
2640 * there's 2 or 4 times less vectors to deal with for the interleave...
2641 * Albeit for the AVX (not AVX2) case there's no benefit with 16 bit
2642 * vectors (as it can do 32bit unpack with 256bit vectors, but 8/16bit
2643 * unpack only with 128bit vectors).
2644 * Note: for 16bit sizes really need matching pack conversion code
2645 */
2646 if (!is_1d && dst_channels != 3 && dst_type.width == 8) {
2647 twiddle_after_convert = TRUE;
2648 }
2649
2650 /*
2651 * Pixel twiddle from fragment shader order to memory order
2652 */
2653 if (!twiddle_after_convert) {
2654 src_count = generate_fs_twiddle(gallivm, fs_type, num_fullblock_fs,
2655 dst_channels, fs_src, src, pad_inline);
2656 if (dual_source_blend) {
2657 generate_fs_twiddle(gallivm, fs_type, num_fullblock_fs, dst_channels,
2658 fs_src1, src1, pad_inline);
2659 }
2660 } else {
2661 src_count = num_fullblock_fs * dst_channels;
2662 /*
2663 * We reorder things a bit here, so the cases for 4-wide and 8-wide
2664 * (AVX) turn out the same later when untwiddling/transpose (albeit
2665 * for true AVX2 path untwiddle needs to be different).
2666 * For now just order by colors first (so we can use unpack later).
2667 */
2668 for (unsigned j = 0; j < num_fullblock_fs; j++) {
2669 for (unsigned i = 0; i < dst_channels; i++) {
2670 src[i*num_fullblock_fs + j] = fs_src[j][i];
2671 if (dual_source_blend) {
2672 src1[i*num_fullblock_fs + j] = fs_src1[j][i];
2673 }
2674 }
2675 }
2676 }
2677
2678 src_channels = dst_channels < 3 ? dst_channels : 4;
2679 if (src_count != num_fullblock_fs * src_channels) {
2680 unsigned ds = src_count / (num_fullblock_fs * src_channels);
2681 row_type.length /= ds;
2682 fs_type.length = row_type.length;
2683 }
2684
2685 blend_type = row_type;
2686 mask_type.length = 4;
2687
2688 /* Convert src to row_type */
2689 if (dual_source_blend) {
2690 struct lp_type old_row_type = row_type;
2691 lp_build_conv_auto(gallivm, fs_type, &row_type, src, src_count, src);
2692 src_count = lp_build_conv_auto(gallivm, fs_type, &old_row_type, src1, src_count, src1);
2693 }
2694 else {
2695 src_count = lp_build_conv_auto(gallivm, fs_type, &row_type, src, src_count, src);
2696 }
2697
2698 /* If the rows are not an SSE vector, combine them to become SSE size! */
2699 if ((row_type.width * row_type.length) % 128) {
2700 unsigned bits = row_type.width * row_type.length;
2701 unsigned combined;
2702
2703 assert(src_count >= (vector_width / bits));
2704
2705 dst_count = src_count / (vector_width / bits);
2706
2707 combined = lp_build_concat_n(gallivm, row_type, src, src_count, src, dst_count);
2708 if (dual_source_blend) {
2709 lp_build_concat_n(gallivm, row_type, src1, src_count, src1, dst_count);
2710 }
2711
2712 row_type.length *= combined;
2713 src_count /= combined;
2714
2715 bits = row_type.width * row_type.length;
2716 assert(bits == 128 || bits == 256);
2717 }
2718
2719 if (twiddle_after_convert) {
2720 fs_twiddle_transpose(gallivm, row_type, src, src_count, src);
2721 if (dual_source_blend) {
2722 fs_twiddle_transpose(gallivm, row_type, src1, src_count, src1);
2723 }
2724 }
2725
2726 /*
2727 * Blend Colour conversion
2728 */
2729 blend_color = lp_jit_context_f_blend_color(gallivm, context_ptr);
2730 blend_color = LLVMBuildPointerCast(builder, blend_color,
2731 LLVMPointerType(lp_build_vec_type(gallivm, fs_type), 0), "");
2732 blend_color = LLVMBuildLoad(builder, LLVMBuildGEP(builder, blend_color,
2733 &i32_zero, 1, ""), "");
2734
2735 /* Convert */
2736 lp_build_conv(gallivm, fs_type, blend_type, &blend_color, 1, &blend_color, 1);
2737
2738 if (out_format_desc->colorspace == UTIL_FORMAT_COLORSPACE_SRGB) {
2739 /*
2740 * since blending is done with floats, there was no conversion.
2741 * However, the rules according to fixed point renderbuffers still
2742 * apply, that is we must clamp inputs to 0.0/1.0.
2743 * (This would apply to separate alpha conversion too but we currently
2744 * force has_alpha to be true.)
2745 * TODO: should skip this with "fake" blend, since post-blend conversion
2746 * will clamp anyway.
2747 * TODO: could also skip this if fragment color clamping is enabled.
2748 * We don't support it natively so it gets baked into the shader
2749 * however, so can't really tell here.
2750 */
2751 struct lp_build_context f32_bld;
2752 assert(row_type.floating);
2753 lp_build_context_init(&f32_bld, gallivm, row_type);
2754 for (unsigned i = 0; i < src_count; i++) {
2755 src[i] = lp_build_clamp_zero_one_nanzero(&f32_bld, src[i]);
2756 }
2757 if (dual_source_blend) {
2758 for (unsigned i = 0; i < src_count; i++) {
2759 src1[i] = lp_build_clamp_zero_one_nanzero(&f32_bld, src1[i]);
2760 }
2761 }
2762 /* probably can't be different than row_type but better safe than sorry... */
2763 lp_build_context_init(&f32_bld, gallivm, blend_type);
2764 blend_color = lp_build_clamp(&f32_bld, blend_color, f32_bld.zero, f32_bld.one);
2765 }
2766
2767 /* Extract alpha */
2768 blend_alpha = lp_build_extract_broadcast(gallivm, blend_type, row_type, blend_color, lp_build_const_int32(gallivm, 3));
2769
2770 /* Swizzle to appropriate channels, e.g. from RGBA to BGRA BGRA */
2771 pad_inline &= (dst_channels * (block_size / src_count) * row_type.width) != vector_width;
2772 if (pad_inline) {
2773 /* Use all 4 channels e.g. from RGBA RGBA to RGxx RGxx */
2774 blend_color = lp_build_swizzle_aos_n(gallivm, blend_color, swizzle, TGSI_NUM_CHANNELS, row_type.length);
2775 } else {
2776 /* Only use dst_channels e.g. RGBA RGBA to RG RG xxxx */
2777 blend_color = lp_build_swizzle_aos_n(gallivm, blend_color, swizzle, dst_channels, row_type.length);
2778 }
2779
2780 /*
2781 * Mask conversion
2782 */
2783 lp_bld_quad_twiddle(gallivm, mask_type, &src_mask[0], block_height, &src_mask[0]);
2784
2785 if (src_count < block_height) {
2786 lp_build_concat_n(gallivm, mask_type, src_mask, 4, src_mask, src_count);
2787 } else if (src_count > block_height) {
2788 for (unsigned i = src_count; i > 0; --i) {
2789 unsigned pixels = block_size / src_count;
2790 unsigned idx = i - 1;
2791
2792 src_mask[idx] = lp_build_extract_range(gallivm, src_mask[(idx * pixels) / 4],
2793 (idx * pixels) % 4, pixels);
2794 }
2795 }
2796
2797 assert(mask_type.width == 32);
2798
2799 for (unsigned i = 0; i < src_count; ++i) {
2800 unsigned pixels = block_size / src_count;
2801 unsigned pixel_width = row_type.width * dst_channels;
2802
2803 if (pixel_width == 24) {
2804 mask_type.width = 8;
2805 mask_type.length = vector_width / mask_type.width;
2806 } else {
2807 mask_type.length = pixels;
2808 mask_type.width = row_type.width * dst_channels;
2809
2810 /*
2811 * If mask_type width is smaller than 32bit, this doesn't quite
2812 * generate the most efficient code (could use some pack).
2813 */
2814 src_mask[i] = LLVMBuildIntCast(builder, src_mask[i],
2815 lp_build_int_vec_type(gallivm, mask_type), "");
2816
2817 mask_type.length *= dst_channels;
2818 mask_type.width /= dst_channels;
2819 }
2820
2821 src_mask[i] = LLVMBuildBitCast(builder, src_mask[i],
2822 lp_build_int_vec_type(gallivm, mask_type), "");
2823 src_mask[i] = lp_build_pad_vector(gallivm, src_mask[i], row_type.length);
2824 }
2825
2826 /*
2827 * Alpha conversion
2828 */
2829 if (!has_alpha) {
2830 struct lp_type alpha_type = fs_type;
2831 alpha_type.length = 4;
2832 convert_alpha(gallivm, row_type, alpha_type,
2833 block_size, block_height,
2834 src_count, dst_channels,
2835 pad_inline, src_alpha);
2836 if (dual_source_blend) {
2837 convert_alpha(gallivm, row_type, alpha_type,
2838 block_size, block_height,
2839 src_count, dst_channels,
2840 pad_inline, src1_alpha);
2841 }
2842 }
2843
2844
2845 /*
2846 * Load dst from memory
2847 */
2848 if (src_count < block_height) {
2849 dst_count = block_height;
2850 } else {
2851 dst_count = src_count;
2852 }
2853
2854 dst_type.length *= block_size / dst_count;
2855
2856 if (format_expands_to_float_soa(out_format_desc)) {
2857 /*
2858 * we need multiple values at once for the conversion, so can as well
2859 * load them vectorized here too instead of concatenating later.
2860 * (Still need concatenation later for 8-wide vectors).
2861 */
2862 dst_count = block_height;
2863 dst_type.length = block_width;
2864 }
2865
2866 /*
2867 * Compute the alignment of the destination pointer in bytes
2868 * We fetch 1-4 pixels, if the format has pot alignment then those fetches
2869 * are always aligned by MIN2(16, fetch_width) except for buffers (not
2870 * 1d tex but can't distinguish here) so need to stick with per-pixel
2871 * alignment in this case.
2872 */
2873 if (is_1d) {
2874 dst_alignment = (out_format_desc->block.bits + 7)/(out_format_desc->block.width * 8);
2875 }
2876 else {
2877 dst_alignment = dst_type.length * dst_type.width / 8;
2878 }
2879 /* Force power-of-two alignment by extracting only the least-significant-bit */
2880 dst_alignment = 1 << (ffs(dst_alignment) - 1);
2881 /*
2882 * Resource base and stride pointers are aligned to 16 bytes, so that's
2883 * the maximum alignment we can guarantee
2884 */
2885 dst_alignment = MIN2(16, dst_alignment);
2886
2887 ls_type = dst_type;
2888
2889 if (dst_count > src_count) {
2890 if ((dst_type.width == 8 || dst_type.width == 16) &&
2891 util_is_power_of_two_or_zero(dst_type.length) &&
2892 dst_type.length * dst_type.width < 128) {
2893 /*
2894 * Never try to load values as 4xi8 which we will then
2895 * concatenate to larger vectors. This gives llvm a real
2896 * headache (the problem is the type legalizer (?) will
2897 * try to load that as 4xi8 zext to 4xi32 to fill the vector,
2898 * then the shuffles to concatenate are more or less impossible
2899 * - llvm is easily capable of generating a sequence of 32
2900 * pextrb/pinsrb instructions for that. Albeit it appears to
2901 * be fixed in llvm 4.0. So, load and concatenate with 32bit
2902 * width to avoid the trouble (16bit seems not as bad, llvm
2903 * probably recognizes the load+shuffle as only one shuffle
2904 * is necessary, but we can do just the same anyway).
2905 */
2906 ls_type.length = dst_type.length * dst_type.width / 32;
2907 ls_type.width = 32;
2908 }
2909 }
2910
2911 if (is_1d) {
2912 load_unswizzled_block(gallivm, color_ptr, stride, block_width, 1,
2913 dst, ls_type, dst_count / 4, dst_alignment);
2914 for (unsigned i = dst_count / 4; i < dst_count; i++) {
2915 dst[i] = lp_build_undef(gallivm, ls_type);
2916 }
2917
2918 }
2919 else {
2920 load_unswizzled_block(gallivm, color_ptr, stride, block_width, block_height,
2921 dst, ls_type, dst_count, dst_alignment);
2922 }
2923
2924
2925 /*
2926 * Convert from dst/output format to src/blending format.
2927 *
2928 * This is necessary as we can only read 1 row from memory at a time,
2929 * so the minimum dst_count will ever be at this point is 4.
2930 *
2931 * With, for example, R8 format you can have all 16 pixels in a 128 bit
2932 * vector, this will take the 4 dsts and combine them into 1 src so we can
2933 * perform blending on all 16 pixels in that single vector at once.
2934 */
2935 if (dst_count > src_count) {
2936 if (ls_type.length != dst_type.length && ls_type.length == 1) {
2937 LLVMTypeRef elem_type = lp_build_elem_type(gallivm, ls_type);
2938 LLVMTypeRef ls_vec_type = LLVMVectorType(elem_type, 1);
2939 for (unsigned i = 0; i < dst_count; i++) {
2940 dst[i] = LLVMBuildBitCast(builder, dst[i], ls_vec_type, "");
2941 }
2942 }
2943
2944 lp_build_concat_n(gallivm, ls_type, dst, 4, dst, src_count);
2945
2946 if (ls_type.length != dst_type.length) {
2947 struct lp_type tmp_type = dst_type;
2948 tmp_type.length = dst_type.length * 4 / src_count;
2949 for (unsigned i = 0; i < src_count; i++) {
2950 dst[i] = LLVMBuildBitCast(builder, dst[i],
2951 lp_build_vec_type(gallivm, tmp_type), "");
2952 }
2953 }
2954 }
2955
2956 /*
2957 * Blending
2958 */
2959 /* XXX this is broken for RGB8 formats -
2960 * they get expanded from 12 to 16 elements (to include alpha)
2961 * by convert_to_blend_type then reduced to 15 instead of 12
2962 * by convert_from_blend_type (a simple fix though breaks A8...).
2963 * R16G16B16 also crashes differently however something going wrong
2964 * inside llvm handling npot vector sizes seemingly.
2965 * It seems some cleanup could be done here (like skipping conversion/blend
2966 * when not needed).
2967 */
2968 convert_to_blend_type(gallivm, block_size, out_format_desc, dst_type,
2969 row_type, dst, src_count);
2970
2971 /*
2972 * FIXME: Really should get logic ops / masks out of generic blend / row
2973 * format. Logic ops will definitely not work on the blend float format
2974 * used for SRGB here and I think OpenGL expects this to work as expected
2975 * (that is incoming values converted to srgb then logic op applied).
2976 */
2977 for (unsigned i = 0; i < src_count; ++i) {
2978 dst[i] = lp_build_blend_aos(gallivm,
2979 &variant->key.blend,
2980 out_format,
2981 row_type,
2982 rt,
2983 src[i],
2984 has_alpha ? NULL : src_alpha[i],
2985 src1[i],
2986 has_alpha ? NULL : src1_alpha[i],
2987 dst[i],
2988 partial_mask ? src_mask[i] : NULL,
2989 blend_color,
2990 has_alpha ? NULL : blend_alpha,
2991 swizzle,
2992 pad_inline ? 4 : dst_channels);
2993 }
2994
2995 convert_from_blend_type(gallivm, block_size, out_format_desc,
2996 row_type, dst_type, dst, src_count);
2997
2998 /* Split the blend rows back to memory rows */
2999 if (dst_count > src_count) {
3000 row_type.length = dst_type.length * (dst_count / src_count);
3001
3002 if (src_count == 1) {
3003 dst[1] = lp_build_extract_range(gallivm, dst[0], row_type.length / 2, row_type.length / 2);
3004 dst[0] = lp_build_extract_range(gallivm, dst[0], 0, row_type.length / 2);
3005
3006 row_type.length /= 2;
3007 src_count *= 2;
3008 }
3009
3010 dst[3] = lp_build_extract_range(gallivm, dst[1], row_type.length / 2, row_type.length / 2);
3011 dst[2] = lp_build_extract_range(gallivm, dst[1], 0, row_type.length / 2);
3012 dst[1] = lp_build_extract_range(gallivm, dst[0], row_type.length / 2, row_type.length / 2);
3013 dst[0] = lp_build_extract_range(gallivm, dst[0], 0, row_type.length / 2);
3014
3015 row_type.length /= 2;
3016 src_count *= 2;
3017 }
3018
3019 /*
3020 * Store blend result to memory
3021 */
3022 if (is_1d) {
3023 store_unswizzled_block(gallivm, color_ptr, stride, block_width, 1,
3024 dst, dst_type, dst_count / 4, dst_alignment);
3025 }
3026 else {
3027 store_unswizzled_block(gallivm, color_ptr, stride, block_width, block_height,
3028 dst, dst_type, dst_count, dst_alignment);
3029 }
3030
3031 if (have_smallfloat_format(dst_type, out_format)) {
3032 lp_build_fpstate_set(gallivm, fpstate);
3033 }
3034
3035 if (do_branch) {
3036 lp_build_mask_end(&mask_ctx);
3037 }
3038 }
3039
3040
3041 /**
3042 * Generate the runtime callable function for the whole fragment pipeline.
3043 * Note that the function which we generate operates on a block of 16
3044 * pixels at at time. The block contains 2x2 quads. Each quad contains
3045 * 2x2 pixels.
3046 */
3047 static void
generate_fragment(struct llvmpipe_context * lp,struct lp_fragment_shader * shader,struct lp_fragment_shader_variant * variant,unsigned partial_mask)3048 generate_fragment(struct llvmpipe_context *lp,
3049 struct lp_fragment_shader *shader,
3050 struct lp_fragment_shader_variant *variant,
3051 unsigned partial_mask)
3052 {
3053 assert(partial_mask == RAST_WHOLE ||
3054 partial_mask == RAST_EDGE_TEST);
3055
3056 struct gallivm_state *gallivm = variant->gallivm;
3057 struct lp_fragment_shader_variant_key *key = &variant->key;
3058 struct lp_shader_input inputs[PIPE_MAX_SHADER_INPUTS];
3059 LLVMTypeRef fs_elem_type;
3060 LLVMTypeRef blend_vec_type;
3061 LLVMTypeRef arg_types[15];
3062 LLVMTypeRef func_type;
3063 LLVMTypeRef int32_type = LLVMInt32TypeInContext(gallivm->context);
3064 LLVMTypeRef int8_type = LLVMInt8TypeInContext(gallivm->context);
3065 LLVMValueRef context_ptr;
3066 LLVMValueRef x;
3067 LLVMValueRef y;
3068 LLVMValueRef a0_ptr;
3069 LLVMValueRef dadx_ptr;
3070 LLVMValueRef dady_ptr;
3071 LLVMValueRef color_ptr_ptr;
3072 LLVMValueRef stride_ptr;
3073 LLVMValueRef color_sample_stride_ptr;
3074 LLVMValueRef depth_ptr;
3075 LLVMValueRef depth_stride;
3076 LLVMValueRef depth_sample_stride;
3077 LLVMValueRef mask_input;
3078 LLVMValueRef thread_data_ptr;
3079 LLVMBasicBlockRef block;
3080 LLVMBuilderRef builder;
3081 struct lp_build_interp_soa_context interp;
3082 LLVMValueRef fs_mask[(16 / 4) * LP_MAX_SAMPLES];
3083 LLVMValueRef fs_out_color[LP_MAX_SAMPLES][PIPE_MAX_COLOR_BUFS][TGSI_NUM_CHANNELS][16 / 4];
3084 LLVMValueRef function;
3085 LLVMValueRef facing;
3086 boolean cbuf0_write_all;
3087 const boolean dual_source_blend = key->blend.rt[0].blend_enable &&
3088 util_blend_state_is_dual(&key->blend, 0);
3089
3090 assert(lp_native_vector_width / 32 >= 4);
3091
3092 /* Adjust color input interpolation according to flatshade state:
3093 */
3094 memcpy(inputs, shader->inputs, shader->info.base.num_inputs * sizeof inputs[0]);
3095 for (unsigned i = 0; i < shader->info.base.num_inputs; i++) {
3096 if (inputs[i].interp == LP_INTERP_COLOR) {
3097 if (key->flatshade)
3098 inputs[i].interp = LP_INTERP_CONSTANT;
3099 else
3100 inputs[i].interp = LP_INTERP_PERSPECTIVE;
3101 }
3102 }
3103
3104 /* check if writes to cbuf[0] are to be copied to all cbufs */
3105 cbuf0_write_all =
3106 shader->info.base.properties[TGSI_PROPERTY_FS_COLOR0_WRITES_ALL_CBUFS];
3107
3108 /* TODO: actually pick these based on the fs and color buffer
3109 * characteristics. */
3110
3111 struct lp_type fs_type;
3112 memset(&fs_type, 0, sizeof fs_type);
3113 fs_type.floating = TRUE; /* floating point values */
3114 fs_type.sign = TRUE; /* values are signed */
3115 fs_type.norm = FALSE; /* values are not limited to [0,1] or [-1,1] */
3116 fs_type.width = 32; /* 32-bit float */
3117 fs_type.length = MIN2(lp_native_vector_width / 32, 16); /* n*4 elements per vector */
3118
3119 struct lp_type blend_type;
3120 memset(&blend_type, 0, sizeof blend_type);
3121 blend_type.floating = FALSE; /* values are integers */
3122 blend_type.sign = FALSE; /* values are unsigned */
3123 blend_type.norm = TRUE; /* values are in [0,1] or [-1,1] */
3124 blend_type.width = 8; /* 8-bit ubyte values */
3125 blend_type.length = 16; /* 16 elements per vector */
3126
3127 /*
3128 * Generate the function prototype. Any change here must be reflected in
3129 * lp_jit.h's lp_jit_frag_func function pointer type, and vice-versa.
3130 */
3131
3132 fs_elem_type = lp_build_elem_type(gallivm, fs_type);
3133
3134 blend_vec_type = lp_build_vec_type(gallivm, blend_type);
3135
3136 char func_name[64];
3137 snprintf(func_name, sizeof(func_name), "fs_variant_%s",
3138 partial_mask ? "partial" : "whole");
3139
3140 arg_types[0] = variant->jit_context_ptr_type; /* context */
3141 arg_types[1] = int32_type; /* x */
3142 arg_types[2] = int32_type; /* y */
3143 arg_types[3] = int32_type; /* facing */
3144 arg_types[4] = LLVMPointerType(fs_elem_type, 0); /* a0 */
3145 arg_types[5] = LLVMPointerType(fs_elem_type, 0); /* dadx */
3146 arg_types[6] = LLVMPointerType(fs_elem_type, 0); /* dady */
3147 arg_types[7] = LLVMPointerType(LLVMPointerType(int8_type, 0), 0); /* color */
3148 arg_types[8] = LLVMPointerType(int8_type, 0); /* depth */
3149 arg_types[9] = LLVMInt64TypeInContext(gallivm->context); /* mask_input */
3150 arg_types[10] = variant->jit_thread_data_ptr_type; /* per thread data */
3151 arg_types[11] = LLVMPointerType(int32_type, 0); /* stride */
3152 arg_types[12] = int32_type; /* depth_stride */
3153 arg_types[13] = LLVMPointerType(int32_type, 0); /* color sample strides */
3154 arg_types[14] = int32_type; /* depth sample stride */
3155
3156 func_type = LLVMFunctionType(LLVMVoidTypeInContext(gallivm->context),
3157 arg_types, ARRAY_SIZE(arg_types), 0);
3158
3159 function = LLVMAddFunction(gallivm->module, func_name, func_type);
3160 LLVMSetFunctionCallConv(function, LLVMCCallConv);
3161
3162 variant->function[partial_mask] = function;
3163
3164 /* XXX: need to propagate noalias down into color param now we are
3165 * passing a pointer-to-pointer?
3166 */
3167 for (unsigned i = 0; i < ARRAY_SIZE(arg_types); ++i)
3168 if (LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind)
3169 lp_add_function_attr(function, i + 1, LP_FUNC_ATTR_NOALIAS);
3170
3171 if (variant->gallivm->cache->data_size)
3172 return;
3173
3174 context_ptr = LLVMGetParam(function, 0);
3175 x = LLVMGetParam(function, 1);
3176 y = LLVMGetParam(function, 2);
3177 facing = LLVMGetParam(function, 3);
3178 a0_ptr = LLVMGetParam(function, 4);
3179 dadx_ptr = LLVMGetParam(function, 5);
3180 dady_ptr = LLVMGetParam(function, 6);
3181 color_ptr_ptr = LLVMGetParam(function, 7);
3182 depth_ptr = LLVMGetParam(function, 8);
3183 mask_input = LLVMGetParam(function, 9);
3184 thread_data_ptr = LLVMGetParam(function, 10);
3185 stride_ptr = LLVMGetParam(function, 11);
3186 depth_stride = LLVMGetParam(function, 12);
3187 color_sample_stride_ptr = LLVMGetParam(function, 13);
3188 depth_sample_stride = LLVMGetParam(function, 14);
3189
3190 lp_build_name(context_ptr, "context");
3191 lp_build_name(x, "x");
3192 lp_build_name(y, "y");
3193 lp_build_name(a0_ptr, "a0");
3194 lp_build_name(dadx_ptr, "dadx");
3195 lp_build_name(dady_ptr, "dady");
3196 lp_build_name(color_ptr_ptr, "color_ptr_ptr");
3197 lp_build_name(depth_ptr, "depth");
3198 lp_build_name(mask_input, "mask_input");
3199 lp_build_name(thread_data_ptr, "thread_data");
3200 lp_build_name(stride_ptr, "stride_ptr");
3201 lp_build_name(depth_stride, "depth_stride");
3202 lp_build_name(color_sample_stride_ptr, "color_sample_stride_ptr");
3203 lp_build_name(depth_sample_stride, "depth_sample_stride");
3204
3205 /*
3206 * Function body
3207 */
3208
3209 block = LLVMAppendBasicBlockInContext(gallivm->context, function, "entry");
3210 builder = gallivm->builder;
3211 assert(builder);
3212 LLVMPositionBuilderAtEnd(builder, block);
3213
3214 /*
3215 * Must not count ps invocations if there's a null shader.
3216 * (It would be ok to count with null shader if there's d/s tests,
3217 * but only if there's d/s buffers too, which is different
3218 * to implicit rasterization disable which must not depend
3219 * on the d/s buffers.)
3220 * Could use popcount on mask, but pixel accuracy is not required.
3221 * Could disable if there's no stats query, but maybe not worth it.
3222 */
3223 if (shader->info.base.num_instructions > 1) {
3224 LLVMValueRef invocs, val;
3225 invocs = lp_jit_thread_data_invocations(gallivm, thread_data_ptr);
3226 val = LLVMBuildLoad(builder, invocs, "");
3227 val = LLVMBuildAdd(builder, val,
3228 LLVMConstInt(LLVMInt64TypeInContext(gallivm->context), 1, 0),
3229 "invoc_count");
3230 LLVMBuildStore(builder, val, invocs);
3231 }
3232
3233 /* code generated texture sampling */
3234 struct lp_build_sampler_soa *sampler =
3235 lp_llvm_sampler_soa_create(lp_fs_variant_key_samplers(key),
3236 MAX2(key->nr_samplers,
3237 key->nr_sampler_views));
3238 struct lp_build_image_soa *image =
3239 lp_llvm_image_soa_create(lp_fs_variant_key_images(key), key->nr_images);
3240
3241 unsigned num_fs = 16 / fs_type.length; /* number of loops per 4x4 stamp */
3242 /* for 1d resources only run "upper half" of stamp */
3243 if (key->resource_1d)
3244 num_fs /= 2;
3245
3246 {
3247 LLVMValueRef num_loop = lp_build_const_int32(gallivm, num_fs);
3248 LLVMTypeRef mask_type = lp_build_int_vec_type(gallivm, fs_type);
3249 LLVMValueRef num_loop_samp =
3250 lp_build_const_int32(gallivm, num_fs * key->coverage_samples);
3251 LLVMValueRef mask_store =
3252 lp_build_array_alloca(gallivm, mask_type,
3253 num_loop_samp, "mask_store");
3254 LLVMTypeRef flt_type = LLVMFloatTypeInContext(gallivm->context);
3255 LLVMValueRef glob_sample_pos =
3256 LLVMAddGlobal(gallivm->module,
3257 LLVMArrayType(flt_type, key->coverage_samples * 2), "");
3258 LLVMValueRef sample_pos_array;
3259
3260 if (key->multisample && key->coverage_samples == 4) {
3261 LLVMValueRef sample_pos_arr[8];
3262 for (unsigned i = 0; i < 4; i++) {
3263 sample_pos_arr[i * 2] = LLVMConstReal(flt_type, lp_sample_pos_4x[i][0]);
3264 sample_pos_arr[i * 2 + 1] = LLVMConstReal(flt_type, lp_sample_pos_4x[i][1]);
3265 }
3266 sample_pos_array = LLVMConstArray(LLVMFloatTypeInContext(gallivm->context), sample_pos_arr, 8);
3267 } else {
3268 LLVMValueRef sample_pos_arr[2];
3269 sample_pos_arr[0] = LLVMConstReal(flt_type, 0.5);
3270 sample_pos_arr[1] = LLVMConstReal(flt_type, 0.5);
3271 sample_pos_array = LLVMConstArray(LLVMFloatTypeInContext(gallivm->context), sample_pos_arr, 2);
3272 }
3273 LLVMSetInitializer(glob_sample_pos, sample_pos_array);
3274
3275 LLVMValueRef color_store[PIPE_MAX_COLOR_BUFS][TGSI_NUM_CHANNELS];
3276 boolean pixel_center_integer =
3277 shader->info.base.properties[TGSI_PROPERTY_FS_COORD_PIXEL_CENTER];
3278
3279 /*
3280 * The shader input interpolation info is not explicitely baked in the
3281 * shader key, but everything it derives from (TGSI, and flatshade) is
3282 * already included in the shader key.
3283 */
3284 lp_build_interp_soa_init(&interp,
3285 gallivm,
3286 shader->info.base.num_inputs,
3287 inputs,
3288 pixel_center_integer,
3289 key->coverage_samples, glob_sample_pos,
3290 num_loop,
3291 builder, fs_type,
3292 a0_ptr, dadx_ptr, dady_ptr,
3293 x, y);
3294
3295 for (unsigned i = 0; i < num_fs; i++) {
3296 if (key->multisample) {
3297 LLVMValueRef smask_val = LLVMBuildLoad(builder, lp_jit_context_sample_mask(gallivm, context_ptr), "");
3298
3299 /*
3300 * For multisampling, extract the per-sample mask from the
3301 * incoming 64-bit mask, store to the per sample mask storage. Or
3302 * all of them together to generate the fragment shader
3303 * mask. (sample shading TODO). Take the incoming state coverage
3304 * mask into account.
3305 */
3306 for (unsigned s = 0; s < key->coverage_samples; s++) {
3307 LLVMValueRef sindexi = lp_build_const_int32(gallivm, i + (s * num_fs));
3308 LLVMValueRef sample_mask_ptr = LLVMBuildGEP(builder, mask_store,
3309 &sindexi, 1, "sample_mask_ptr");
3310 LLVMValueRef s_mask = generate_quad_mask(gallivm, fs_type,
3311 i*fs_type.length/4, s, mask_input);
3312
3313 LLVMValueRef smask_bit = LLVMBuildAnd(builder, smask_val, lp_build_const_int32(gallivm, (1 << s)), "");
3314 LLVMValueRef cmp = LLVMBuildICmp(builder, LLVMIntNE, smask_bit, lp_build_const_int32(gallivm, 0), "");
3315 smask_bit = LLVMBuildSExt(builder, cmp, int32_type, "");
3316 smask_bit = lp_build_broadcast(gallivm, mask_type, smask_bit);
3317
3318 s_mask = LLVMBuildAnd(builder, s_mask, smask_bit, "");
3319 LLVMBuildStore(builder, s_mask, sample_mask_ptr);
3320 }
3321 } else {
3322 LLVMValueRef mask;
3323 LLVMValueRef indexi = lp_build_const_int32(gallivm, i);
3324 LLVMValueRef mask_ptr = LLVMBuildGEP(builder, mask_store,
3325 &indexi, 1, "mask_ptr");
3326
3327 if (partial_mask) {
3328 mask = generate_quad_mask(gallivm, fs_type,
3329 i*fs_type.length/4, 0, mask_input);
3330 }
3331 else {
3332 mask = lp_build_const_int_vec(gallivm, fs_type, ~0);
3333 }
3334 LLVMBuildStore(builder, mask, mask_ptr);
3335 }
3336 }
3337
3338 generate_fs_loop(gallivm,
3339 shader, key,
3340 builder,
3341 fs_type,
3342 context_ptr,
3343 glob_sample_pos,
3344 num_loop,
3345 &interp,
3346 sampler,
3347 image,
3348 mask_store, /* output */
3349 color_store,
3350 depth_ptr,
3351 depth_stride,
3352 depth_sample_stride,
3353 color_ptr_ptr,
3354 stride_ptr,
3355 color_sample_stride_ptr,
3356 facing,
3357 thread_data_ptr);
3358
3359 for (unsigned i = 0; i < num_fs; i++) {
3360 LLVMValueRef ptr;
3361 for (unsigned s = 0; s < key->coverage_samples; s++) {
3362 int idx = (i + (s * num_fs));
3363 LLVMValueRef sindexi = lp_build_const_int32(gallivm, idx);
3364 ptr = LLVMBuildGEP(builder, mask_store, &sindexi, 1, "");
3365
3366 fs_mask[idx] = LLVMBuildLoad(builder, ptr, "smask");
3367 }
3368
3369 for (unsigned s = 0; s < key->min_samples; s++) {
3370 /* This is fucked up need to reorganize things */
3371 int idx = s * num_fs + i;
3372 LLVMValueRef sindexi = lp_build_const_int32(gallivm, idx);
3373 for (unsigned cbuf = 0; cbuf < key->nr_cbufs; cbuf++) {
3374 for (unsigned chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) {
3375 ptr = LLVMBuildGEP(builder,
3376 color_store[cbuf * !cbuf0_write_all][chan],
3377 &sindexi, 1, "");
3378 fs_out_color[s][cbuf][chan][i] = ptr;
3379 }
3380 }
3381 if (dual_source_blend) {
3382 /* only support one dual source blend target hence always use output 1 */
3383 for (unsigned chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) {
3384 ptr = LLVMBuildGEP(builder,
3385 color_store[1][chan],
3386 &sindexi, 1, "");
3387 fs_out_color[s][1][chan][i] = ptr;
3388 }
3389 }
3390 }
3391 }
3392 }
3393
3394 sampler->destroy(sampler);
3395 image->destroy(image);
3396
3397 /* Loop over color outputs / color buffers to do blending */
3398 for (unsigned cbuf = 0; cbuf < key->nr_cbufs; cbuf++) {
3399 if (key->cbuf_format[cbuf] != PIPE_FORMAT_NONE) {
3400 LLVMValueRef color_ptr;
3401 LLVMValueRef stride;
3402 LLVMValueRef sample_stride = NULL;
3403 LLVMValueRef index = lp_build_const_int32(gallivm, cbuf);
3404
3405 boolean do_branch = ((key->depth.enabled
3406 || key->stencil[0].enabled
3407 || key->alpha.enabled)
3408 && !shader->info.base.uses_kill);
3409
3410 color_ptr = LLVMBuildLoad(builder,
3411 LLVMBuildGEP(builder, color_ptr_ptr,
3412 &index, 1, ""),
3413 "");
3414
3415 stride = LLVMBuildLoad(builder,
3416 LLVMBuildGEP(builder, stride_ptr,
3417 &index, 1, ""),
3418 "");
3419
3420 if (key->cbuf_nr_samples[cbuf] > 1)
3421 sample_stride = LLVMBuildLoad(builder,
3422 LLVMBuildGEP(builder,
3423 color_sample_stride_ptr,
3424 &index, 1, ""), "");
3425
3426 for (unsigned s = 0; s < key->cbuf_nr_samples[cbuf]; s++) {
3427 unsigned mask_idx = num_fs * (key->multisample ? s : 0);
3428 unsigned out_idx = key->min_samples == 1 ? 0 : s;
3429 LLVMValueRef out_ptr = color_ptr;;
3430
3431 if (sample_stride) {
3432 LLVMValueRef sample_offset =
3433 LLVMBuildMul(builder, sample_stride,
3434 lp_build_const_int32(gallivm, s), "");
3435 out_ptr = LLVMBuildGEP(builder, out_ptr, &sample_offset, 1, "");
3436 }
3437 out_ptr = LLVMBuildBitCast(builder, out_ptr,
3438 LLVMPointerType(blend_vec_type, 0), "");
3439
3440 lp_build_name(out_ptr, "color_ptr%d", cbuf);
3441
3442 generate_unswizzled_blend(gallivm, cbuf, variant,
3443 key->cbuf_format[cbuf],
3444 num_fs, fs_type, &fs_mask[mask_idx],
3445 fs_out_color[out_idx],
3446 context_ptr, out_ptr, stride,
3447 partial_mask, do_branch);
3448 }
3449 }
3450 }
3451
3452 LLVMBuildRetVoid(builder);
3453
3454 gallivm_verify_function(gallivm, function);
3455 }
3456
3457
3458 static void
dump_fs_variant_key(struct lp_fragment_shader_variant_key * key)3459 dump_fs_variant_key(struct lp_fragment_shader_variant_key *key)
3460 {
3461 debug_printf("fs variant %p:\n", (void *) key);
3462
3463 if (key->flatshade) {
3464 debug_printf("flatshade = 1\n");
3465 }
3466 if (key->depth_clamp)
3467 debug_printf("depth_clamp = 1\n");
3468
3469 if (key->restrict_depth_values)
3470 debug_printf("restrict_depth_values = 1\n");
3471
3472 if (key->multisample) {
3473 debug_printf("multisample = 1\n");
3474 debug_printf("coverage samples = %d\n", key->coverage_samples);
3475 debug_printf("min samples = %d\n", key->min_samples);
3476 }
3477 for (unsigned i = 0; i < key->nr_cbufs; ++i) {
3478 debug_printf("cbuf_format[%u] = %s\n", i, util_format_name(key->cbuf_format[i]));
3479 debug_printf("cbuf nr_samples[%u] = %d\n", i, key->cbuf_nr_samples[i]);
3480 }
3481 if (key->depth.enabled || key->stencil[0].enabled) {
3482 debug_printf("depth.format = %s\n", util_format_name(key->zsbuf_format));
3483 debug_printf("depth nr_samples = %d\n", key->zsbuf_nr_samples);
3484 }
3485 if (key->depth.enabled) {
3486 debug_printf("depth.func = %s\n", util_str_func(key->depth.func, TRUE));
3487 debug_printf("depth.writemask = %u\n", key->depth.writemask);
3488 }
3489
3490 for (unsigned i = 0; i < 2; ++i) {
3491 if (key->stencil[i].enabled) {
3492 debug_printf("stencil[%u].func = %s\n", i, util_str_func(key->stencil[i].func, TRUE));
3493 debug_printf("stencil[%u].fail_op = %s\n", i, util_str_stencil_op(key->stencil[i].fail_op, TRUE));
3494 debug_printf("stencil[%u].zpass_op = %s\n", i, util_str_stencil_op(key->stencil[i].zpass_op, TRUE));
3495 debug_printf("stencil[%u].zfail_op = %s\n", i, util_str_stencil_op(key->stencil[i].zfail_op, TRUE));
3496 debug_printf("stencil[%u].valuemask = 0x%x\n", i, key->stencil[i].valuemask);
3497 debug_printf("stencil[%u].writemask = 0x%x\n", i, key->stencil[i].writemask);
3498 }
3499 }
3500
3501 if (key->alpha.enabled) {
3502 debug_printf("alpha.func = %s\n", util_str_func(key->alpha.func, TRUE));
3503 }
3504
3505 if (key->occlusion_count) {
3506 debug_printf("occlusion_count = 1\n");
3507 }
3508
3509 if (key->blend.logicop_enable) {
3510 debug_printf("blend.logicop_func = %s\n", util_str_logicop(key->blend.logicop_func, TRUE));
3511 }
3512 else if (key->blend.rt[0].blend_enable) {
3513 debug_printf("blend.rgb_func = %s\n", util_str_blend_func (key->blend.rt[0].rgb_func, TRUE));
3514 debug_printf("blend.rgb_src_factor = %s\n", util_str_blend_factor(key->blend.rt[0].rgb_src_factor, TRUE));
3515 debug_printf("blend.rgb_dst_factor = %s\n", util_str_blend_factor(key->blend.rt[0].rgb_dst_factor, TRUE));
3516 debug_printf("blend.alpha_func = %s\n", util_str_blend_func (key->blend.rt[0].alpha_func, TRUE));
3517 debug_printf("blend.alpha_src_factor = %s\n", util_str_blend_factor(key->blend.rt[0].alpha_src_factor, TRUE));
3518 debug_printf("blend.alpha_dst_factor = %s\n", util_str_blend_factor(key->blend.rt[0].alpha_dst_factor, TRUE));
3519 }
3520 debug_printf("blend.colormask = 0x%x\n", key->blend.rt[0].colormask);
3521 if (key->blend.alpha_to_coverage) {
3522 debug_printf("blend.alpha_to_coverage is enabled\n");
3523 }
3524 for (unsigned i = 0; i < key->nr_samplers; ++i) {
3525 const struct lp_sampler_static_state *samplers = lp_fs_variant_key_samplers(key);
3526 const struct lp_static_sampler_state *sampler = &samplers[i].sampler_state;
3527 debug_printf("sampler[%u] = \n", i);
3528 debug_printf(" .wrap = %s %s %s\n",
3529 util_str_tex_wrap(sampler->wrap_s, TRUE),
3530 util_str_tex_wrap(sampler->wrap_t, TRUE),
3531 util_str_tex_wrap(sampler->wrap_r, TRUE));
3532 debug_printf(" .min_img_filter = %s\n",
3533 util_str_tex_filter(sampler->min_img_filter, TRUE));
3534 debug_printf(" .min_mip_filter = %s\n",
3535 util_str_tex_mipfilter(sampler->min_mip_filter, TRUE));
3536 debug_printf(" .mag_img_filter = %s\n",
3537 util_str_tex_filter(sampler->mag_img_filter, TRUE));
3538 if (sampler->compare_mode != PIPE_TEX_COMPARE_NONE)
3539 debug_printf(" .compare_func = %s\n", util_str_func(sampler->compare_func, TRUE));
3540 debug_printf(" .normalized_coords = %u\n", sampler->normalized_coords);
3541 debug_printf(" .min_max_lod_equal = %u\n", sampler->min_max_lod_equal);
3542 debug_printf(" .lod_bias_non_zero = %u\n", sampler->lod_bias_non_zero);
3543 debug_printf(" .apply_min_lod = %u\n", sampler->apply_min_lod);
3544 debug_printf(" .apply_max_lod = %u\n", sampler->apply_max_lod);
3545 debug_printf(" .reduction_mode = %u\n", sampler->reduction_mode);
3546 debug_printf(" .aniso = %u\n", sampler->aniso);
3547 }
3548 for (unsigned i = 0; i < key->nr_sampler_views; ++i) {
3549 const struct lp_sampler_static_state *samplers = lp_fs_variant_key_samplers(key);
3550 const struct lp_static_texture_state *texture = &samplers[i].texture_state;
3551 debug_printf("texture[%u] = \n", i);
3552 debug_printf(" .format = %s\n",
3553 util_format_name(texture->format));
3554 debug_printf(" .target = %s\n",
3555 util_str_tex_target(texture->target, TRUE));
3556 debug_printf(" .level_zero_only = %u\n",
3557 texture->level_zero_only);
3558 debug_printf(" .pot = %u %u %u\n",
3559 texture->pot_width,
3560 texture->pot_height,
3561 texture->pot_depth);
3562 }
3563 struct lp_image_static_state *images = lp_fs_variant_key_images(key);
3564 for (unsigned i = 0; i < key->nr_images; ++i) {
3565 const struct lp_static_texture_state *image = &images[i].image_state;
3566 debug_printf("image[%u] = \n", i);
3567 debug_printf(" .format = %s\n",
3568 util_format_name(image->format));
3569 debug_printf(" .target = %s\n",
3570 util_str_tex_target(image->target, TRUE));
3571 debug_printf(" .level_zero_only = %u\n",
3572 image->level_zero_only);
3573 debug_printf(" .pot = %u %u %u\n",
3574 image->pot_width,
3575 image->pot_height,
3576 image->pot_depth);
3577 }
3578 }
3579
3580
3581 const char *
lp_debug_fs_kind(enum lp_fs_kind kind)3582 lp_debug_fs_kind(enum lp_fs_kind kind)
3583 {
3584 switch (kind) {
3585 case LP_FS_KIND_GENERAL:
3586 return "GENERAL";
3587 case LP_FS_KIND_BLIT_RGBA:
3588 return "BLIT_RGBA";
3589 case LP_FS_KIND_BLIT_RGB1:
3590 return "BLIT_RGB1";
3591 case LP_FS_KIND_AERO_MINIFICATION:
3592 return "AERO_MINIFICATION";
3593 case LP_FS_KIND_LLVM_LINEAR:
3594 return "LLVM_LINEAR";
3595 default:
3596 return "unknown";
3597 }
3598 }
3599
3600
3601 void
lp_debug_fs_variant(struct lp_fragment_shader_variant * variant)3602 lp_debug_fs_variant(struct lp_fragment_shader_variant *variant)
3603 {
3604 debug_printf("llvmpipe: Fragment shader #%u variant #%u:\n",
3605 variant->shader->no, variant->no);
3606 if (variant->shader->base.type == PIPE_SHADER_IR_TGSI)
3607 tgsi_dump(variant->shader->base.tokens, 0);
3608 else
3609 nir_print_shader(variant->shader->base.ir.nir, stderr);
3610 dump_fs_variant_key(&variant->key);
3611 debug_printf("variant->opaque = %u\n", variant->opaque);
3612 debug_printf("variant->potentially_opaque = %u\n", variant->potentially_opaque);
3613 debug_printf("variant->blit = %u\n", variant->blit);
3614 debug_printf("shader->kind = %s\n", lp_debug_fs_kind(variant->shader->kind));
3615 debug_printf("\n");
3616 }
3617
3618
3619 static void
lp_fs_get_ir_cache_key(struct lp_fragment_shader_variant * variant,unsigned char ir_sha1_cache_key[20])3620 lp_fs_get_ir_cache_key(struct lp_fragment_shader_variant *variant,
3621 unsigned char ir_sha1_cache_key[20])
3622 {
3623 struct blob blob = { 0 };
3624 unsigned ir_size;
3625 void *ir_binary;
3626
3627 blob_init(&blob);
3628 nir_serialize(&blob, variant->shader->base.ir.nir, true);
3629 ir_binary = blob.data;
3630 ir_size = blob.size;
3631
3632 struct mesa_sha1 ctx;
3633 _mesa_sha1_init(&ctx);
3634 _mesa_sha1_update(&ctx, &variant->key, variant->shader->variant_key_size);
3635 _mesa_sha1_update(&ctx, ir_binary, ir_size);
3636 _mesa_sha1_final(&ctx, ir_sha1_cache_key);
3637
3638 blob_finish(&blob);
3639 }
3640
3641
3642 /**
3643 * Generate a new fragment shader variant from the shader code and
3644 * other state indicated by the key.
3645 */
3646 static struct lp_fragment_shader_variant *
generate_variant(struct llvmpipe_context * lp,struct lp_fragment_shader * shader,const struct lp_fragment_shader_variant_key * key)3647 generate_variant(struct llvmpipe_context *lp,
3648 struct lp_fragment_shader *shader,
3649 const struct lp_fragment_shader_variant_key *key)
3650 {
3651 struct lp_fragment_shader_variant *variant =
3652 MALLOC(sizeof *variant + shader->variant_key_size - sizeof variant->key);
3653 if (!variant)
3654 return NULL;
3655
3656 memset(variant, 0, sizeof(*variant));
3657
3658 pipe_reference_init(&variant->reference, 1);
3659 lp_fs_reference(lp, &variant->shader, shader);
3660
3661 memcpy(&variant->key, key, shader->variant_key_size);
3662
3663 struct llvmpipe_screen *screen = llvmpipe_screen(lp->pipe.screen);
3664 struct lp_cached_code cached = { 0 };
3665 unsigned char ir_sha1_cache_key[20];
3666 bool needs_caching = false;
3667 if (shader->base.ir.nir) {
3668 lp_fs_get_ir_cache_key(variant, ir_sha1_cache_key);
3669
3670 lp_disk_cache_find_shader(screen, &cached, ir_sha1_cache_key);
3671 if (!cached.data_size)
3672 needs_caching = true;
3673 }
3674
3675 char module_name[64];
3676 snprintf(module_name, sizeof(module_name), "fs%u_variant%u",
3677 shader->no, shader->variants_created);
3678 variant->gallivm = gallivm_create(module_name, lp->context, &cached);
3679 if (!variant->gallivm) {
3680 FREE(variant);
3681 return NULL;
3682 }
3683
3684 variant->list_item_global.base = variant;
3685 variant->list_item_local.base = variant;
3686 variant->no = shader->variants_created++;
3687
3688 /*
3689 * Determine whether we are touching all channels in the color buffer.
3690 */
3691 const struct util_format_description *cbuf0_format_desc = NULL;
3692 boolean fullcolormask = FALSE;
3693 if (key->nr_cbufs == 1) {
3694 cbuf0_format_desc = util_format_description(key->cbuf_format[0]);
3695 fullcolormask = util_format_colormask_full(cbuf0_format_desc,
3696 key->blend.rt[0].colormask);
3697 }
3698
3699 /* The scissor is ignored here as only tiles inside the scissoring
3700 * rectangle will refer to this */
3701 const boolean no_kill =
3702 fullcolormask &&
3703 !key->stencil[0].enabled &&
3704 !key->alpha.enabled &&
3705 !key->multisample &&
3706 !key->blend.alpha_to_coverage &&
3707 !key->depth.enabled &&
3708 !shader->info.base.uses_kill &&
3709 !shader->info.base.writes_samplemask &&
3710 !shader->info.base.uses_fbfetch;
3711
3712 variant->opaque =
3713 no_kill &&
3714 !key->blend.logicop_enable &&
3715 !key->blend.rt[0].blend_enable
3716 ? TRUE : FALSE;
3717
3718 variant->potentially_opaque =
3719 no_kill &&
3720 !key->blend.logicop_enable &&
3721 key->blend.rt[0].blend_enable &&
3722 key->blend.rt[0].rgb_func == PIPE_BLEND_ADD &&
3723 key->blend.rt[0].rgb_dst_factor == PIPE_BLENDFACTOR_INV_SRC_ALPHA &&
3724 key->blend.rt[0].alpha_func == key->blend.rt[0].rgb_func &&
3725 key->blend.rt[0].alpha_dst_factor == key->blend.rt[0].rgb_dst_factor &&
3726 shader->base.type == PIPE_SHADER_IR_TGSI &&
3727 /*
3728 * FIXME: for NIR, all of the fields of info.xxx (except info.base)
3729 * are zeros, hence shader analysis (here and elsewhere) using these
3730 * bits cannot work and will silently fail (cbuf is the only pointer
3731 * field, hence causing a crash).
3732 */
3733 shader->info.cbuf[0][3].file != TGSI_FILE_NULL
3734 ? TRUE : FALSE;
3735
3736 /* We only care about opaque blits for now */
3737 if (variant->opaque &&
3738 (shader->kind == LP_FS_KIND_BLIT_RGBA ||
3739 shader->kind == LP_FS_KIND_BLIT_RGB1)) {
3740 const struct lp_sampler_static_state *samp0 =
3741 lp_fs_variant_key_sampler_idx(key, 0);
3742 assert(samp0);
3743
3744 const enum pipe_format texture_format = samp0->texture_state.format;
3745 const enum pipe_texture_target target = samp0->texture_state.target;
3746 const unsigned min_img_filter = samp0->sampler_state.min_img_filter;
3747 const unsigned mag_img_filter = samp0->sampler_state.mag_img_filter;
3748
3749 unsigned min_mip_filter;
3750 if (samp0->texture_state.level_zero_only) {
3751 min_mip_filter = PIPE_TEX_MIPFILTER_NONE;
3752 } else {
3753 min_mip_filter = samp0->sampler_state.min_mip_filter;
3754 }
3755
3756 if (target == PIPE_TEXTURE_2D &&
3757 min_img_filter == PIPE_TEX_FILTER_NEAREST &&
3758 mag_img_filter == PIPE_TEX_FILTER_NEAREST &&
3759 min_mip_filter == PIPE_TEX_MIPFILTER_NONE &&
3760 ((texture_format &&
3761 util_is_format_compatible(util_format_description(texture_format),
3762 cbuf0_format_desc)) ||
3763 (shader->kind == LP_FS_KIND_BLIT_RGB1 &&
3764 (texture_format == PIPE_FORMAT_B8G8R8A8_UNORM ||
3765 texture_format == PIPE_FORMAT_B8G8R8X8_UNORM) &&
3766 (key->cbuf_format[0] == PIPE_FORMAT_B8G8R8A8_UNORM ||
3767 key->cbuf_format[0] == PIPE_FORMAT_B8G8R8X8_UNORM)))) {
3768 variant->blit = 1;
3769 }
3770 }
3771
3772 /* Determine whether this shader + pipeline state is a candidate for
3773 * the linear path.
3774 */
3775 const boolean linear_pipeline =
3776 !key->stencil[0].enabled &&
3777 !key->depth.enabled &&
3778 !shader->info.base.uses_kill &&
3779 !key->blend.logicop_enable &&
3780 (key->cbuf_format[0] == PIPE_FORMAT_B8G8R8A8_UNORM ||
3781 key->cbuf_format[0] == PIPE_FORMAT_B8G8R8X8_UNORM);
3782
3783 memcpy(&variant->key, key, sizeof *key);
3784
3785 if ((LP_DEBUG & DEBUG_FS) || (gallivm_debug & GALLIVM_DEBUG_IR)) {
3786 lp_debug_fs_variant(variant);
3787 }
3788
3789 llvmpipe_fs_variant_fastpath(variant);
3790
3791 lp_jit_init_types(variant);
3792
3793 if (variant->jit_function[RAST_EDGE_TEST] == NULL)
3794 generate_fragment(lp, shader, variant, RAST_EDGE_TEST);
3795
3796 if (variant->jit_function[RAST_WHOLE] == NULL) {
3797 if (variant->opaque) {
3798 /* Specialized shader, which doesn't need to read the color buffer. */
3799 generate_fragment(lp, shader, variant, RAST_WHOLE);
3800 }
3801 }
3802
3803 if (linear_pipeline) {
3804 /* Currently keeping both the old fastpaths and new linear path
3805 * active. The older code is still somewhat faster for the cases
3806 * it covers.
3807 *
3808 * XXX: consider restricting this to aero-mode only.
3809 */
3810 if (fullcolormask &&
3811 !key->alpha.enabled &&
3812 !key->blend.alpha_to_coverage) {
3813 llvmpipe_fs_variant_linear_fastpath(variant);
3814 }
3815
3816 /* If the original fastpath doesn't cover this variant, try the new
3817 * code:
3818 */
3819 if (variant->jit_linear == NULL) {
3820 if (shader->kind == LP_FS_KIND_BLIT_RGBA ||
3821 shader->kind == LP_FS_KIND_BLIT_RGB1 ||
3822 shader->kind == LP_FS_KIND_LLVM_LINEAR) {
3823 llvmpipe_fs_variant_linear_llvm(lp, shader, variant);
3824 }
3825 }
3826 } else {
3827 if (LP_DEBUG & DEBUG_LINEAR) {
3828 lp_debug_fs_variant(variant);
3829 debug_printf(" ----> no linear path for this variant\n");
3830 }
3831 }
3832
3833 /*
3834 * Compile everything
3835 */
3836
3837 gallivm_compile_module(variant->gallivm);
3838
3839 variant->nr_instrs += lp_build_count_ir_module(variant->gallivm->module);
3840
3841 if (variant->function[RAST_EDGE_TEST]) {
3842 variant->jit_function[RAST_EDGE_TEST] = (lp_jit_frag_func)
3843 gallivm_jit_function(variant->gallivm,
3844 variant->function[RAST_EDGE_TEST]);
3845 }
3846
3847 if (variant->function[RAST_WHOLE]) {
3848 variant->jit_function[RAST_WHOLE] = (lp_jit_frag_func)
3849 gallivm_jit_function(variant->gallivm,
3850 variant->function[RAST_WHOLE]);
3851 } else if (!variant->jit_function[RAST_WHOLE]) {
3852 variant->jit_function[RAST_WHOLE] = (lp_jit_frag_func)
3853 variant->jit_function[RAST_EDGE_TEST];
3854 }
3855
3856 if (linear_pipeline) {
3857 if (variant->linear_function) {
3858 variant->jit_linear_llvm = (lp_jit_linear_llvm_func)
3859 gallivm_jit_function(variant->gallivm, variant->linear_function);
3860 }
3861
3862 /*
3863 * This must be done after LLVM compilation, as it will call the JIT'ed
3864 * code to determine active inputs.
3865 */
3866 lp_linear_check_variant(variant);
3867 }
3868
3869 if (needs_caching) {
3870 lp_disk_cache_insert_shader(screen, &cached, ir_sha1_cache_key);
3871 }
3872
3873 gallivm_free_ir(variant->gallivm);
3874
3875 return variant;
3876 }
3877
3878
3879 static void *
llvmpipe_create_fs_state(struct pipe_context * pipe,const struct pipe_shader_state * templ)3880 llvmpipe_create_fs_state(struct pipe_context *pipe,
3881 const struct pipe_shader_state *templ)
3882 {
3883 struct llvmpipe_context *llvmpipe = llvmpipe_context(pipe);
3884
3885 struct lp_fragment_shader *shader = CALLOC_STRUCT(lp_fragment_shader);
3886 if (!shader)
3887 return NULL;
3888
3889 pipe_reference_init(&shader->reference, 1);
3890 shader->no = fs_no++;
3891 list_inithead(&shader->variants.list);
3892
3893 shader->base.type = templ->type;
3894 if (templ->type == PIPE_SHADER_IR_TGSI) {
3895 /* get/save the summary info for this shader */
3896 lp_build_tgsi_info(templ->tokens, &shader->info);
3897
3898 /* we need to keep a local copy of the tokens */
3899 shader->base.tokens = tgsi_dup_tokens(templ->tokens);
3900 } else {
3901 shader->base.ir.nir = templ->ir.nir;
3902 nir_tgsi_scan_shader(templ->ir.nir, &shader->info.base, true);
3903 }
3904
3905 shader->draw_data = draw_create_fragment_shader(llvmpipe->draw, templ);
3906 if (shader->draw_data == NULL) {
3907 FREE((void *) shader->base.tokens);
3908 FREE(shader);
3909 return NULL;
3910 }
3911
3912 const int nr_samplers = shader->info.base.file_max[TGSI_FILE_SAMPLER] + 1;
3913 const int nr_sampler_views =
3914 shader->info.base.file_max[TGSI_FILE_SAMPLER_VIEW] + 1;
3915 const int nr_images = shader->info.base.file_max[TGSI_FILE_IMAGE] + 1;
3916
3917 shader->variant_key_size = lp_fs_variant_key_size(MAX2(nr_samplers,
3918 nr_sampler_views),
3919 nr_images);
3920
3921 for (int i = 0; i < shader->info.base.num_inputs; i++) {
3922 shader->inputs[i].usage_mask = shader->info.base.input_usage_mask[i];
3923 shader->inputs[i].location = shader->info.base.input_interpolate_loc[i];
3924
3925 switch (shader->info.base.input_interpolate[i]) {
3926 case TGSI_INTERPOLATE_CONSTANT:
3927 shader->inputs[i].interp = LP_INTERP_CONSTANT;
3928 break;
3929 case TGSI_INTERPOLATE_LINEAR:
3930 shader->inputs[i].interp = LP_INTERP_LINEAR;
3931 break;
3932 case TGSI_INTERPOLATE_PERSPECTIVE:
3933 shader->inputs[i].interp = LP_INTERP_PERSPECTIVE;
3934 break;
3935 case TGSI_INTERPOLATE_COLOR:
3936 shader->inputs[i].interp = LP_INTERP_COLOR;
3937 break;
3938 default:
3939 assert(0);
3940 break;
3941 }
3942
3943 switch (shader->info.base.input_semantic_name[i]) {
3944 case TGSI_SEMANTIC_FACE:
3945 shader->inputs[i].interp = LP_INTERP_FACING;
3946 break;
3947 case TGSI_SEMANTIC_POSITION:
3948 /* Position was already emitted above
3949 */
3950 shader->inputs[i].interp = LP_INTERP_POSITION;
3951 shader->inputs[i].src_index = 0;
3952 continue;
3953 }
3954
3955 /* XXX this is a completely pointless index map... */
3956 shader->inputs[i].src_index = i+1;
3957 }
3958
3959 if (LP_DEBUG & DEBUG_TGSI && templ->type == PIPE_SHADER_IR_TGSI) {
3960 debug_printf("llvmpipe: Create fragment shader #%u %p:\n",
3961 shader->no, (void *) shader);
3962 tgsi_dump(templ->tokens, 0);
3963 debug_printf("usage masks:\n");
3964 for (unsigned attrib = 0; attrib < shader->info.base.num_inputs; ++attrib) {
3965 unsigned usage_mask = shader->info.base.input_usage_mask[attrib];
3966 debug_printf(" IN[%u].%s%s%s%s\n",
3967 attrib,
3968 usage_mask & TGSI_WRITEMASK_X ? "x" : "",
3969 usage_mask & TGSI_WRITEMASK_Y ? "y" : "",
3970 usage_mask & TGSI_WRITEMASK_Z ? "z" : "",
3971 usage_mask & TGSI_WRITEMASK_W ? "w" : "");
3972 }
3973 debug_printf("\n");
3974 }
3975
3976 /* This will put a derived copy of the tokens into shader->base.tokens */
3977 if (templ->type == PIPE_SHADER_IR_TGSI)
3978 llvmpipe_fs_analyse(shader, templ->tokens);
3979 else
3980 llvmpipe_fs_analyse_nir(shader);
3981
3982 return shader;
3983 }
3984
3985
3986 static void
llvmpipe_bind_fs_state(struct pipe_context * pipe,void * fs)3987 llvmpipe_bind_fs_state(struct pipe_context *pipe, void *fs)
3988 {
3989 struct llvmpipe_context *llvmpipe = llvmpipe_context(pipe);
3990 struct lp_fragment_shader *lp_fs = (struct lp_fragment_shader *)fs;
3991 if (llvmpipe->fs == lp_fs)
3992 return;
3993
3994 draw_bind_fragment_shader(llvmpipe->draw,
3995 (lp_fs ? lp_fs->draw_data : NULL));
3996
3997 lp_fs_reference(llvmpipe, &llvmpipe->fs, lp_fs);
3998
3999 /* invalidate the setup link, NEW_FS will make it update */
4000 lp_setup_set_fs_variant(llvmpipe->setup, NULL);
4001 llvmpipe->dirty |= LP_NEW_FS;
4002 }
4003
4004
4005 /**
4006 * Remove shader variant from two lists: the shader's variant list
4007 * and the context's variant list.
4008 */
4009 static void
llvmpipe_remove_shader_variant(struct llvmpipe_context * lp,struct lp_fragment_shader_variant * variant)4010 llvmpipe_remove_shader_variant(struct llvmpipe_context *lp,
4011 struct lp_fragment_shader_variant *variant)
4012 {
4013 if ((LP_DEBUG & DEBUG_FS) || (gallivm_debug & GALLIVM_DEBUG_IR)) {
4014 debug_printf("llvmpipe: del fs #%u var %u v created %u v cached %u "
4015 "v total cached %u inst %u total inst %u\n",
4016 variant->shader->no, variant->no,
4017 variant->shader->variants_created,
4018 variant->shader->variants_cached,
4019 lp->nr_fs_variants, variant->nr_instrs, lp->nr_fs_instrs);
4020 }
4021
4022 /* remove from shader's list */
4023 list_del(&variant->list_item_local.list);
4024 variant->shader->variants_cached--;
4025
4026 /* remove from context's list */
4027 list_del(&variant->list_item_global.list);
4028 lp->nr_fs_variants--;
4029 lp->nr_fs_instrs -= variant->nr_instrs;
4030 }
4031
4032
4033 void
llvmpipe_destroy_shader_variant(struct llvmpipe_context * lp,struct lp_fragment_shader_variant * variant)4034 llvmpipe_destroy_shader_variant(struct llvmpipe_context *lp,
4035 struct lp_fragment_shader_variant *variant)
4036 {
4037 gallivm_destroy(variant->gallivm);
4038 lp_fs_reference(lp, &variant->shader, NULL);
4039 FREE(variant);
4040 }
4041
4042
4043 void
llvmpipe_destroy_fs(struct llvmpipe_context * llvmpipe,struct lp_fragment_shader * shader)4044 llvmpipe_destroy_fs(struct llvmpipe_context *llvmpipe,
4045 struct lp_fragment_shader *shader)
4046 {
4047 /* Delete draw module's data */
4048 draw_delete_fragment_shader(llvmpipe->draw, shader->draw_data);
4049
4050 if (shader->base.ir.nir)
4051 ralloc_free(shader->base.ir.nir);
4052 assert(shader->variants_cached == 0);
4053 FREE((void *) shader->base.tokens);
4054 FREE(shader);
4055 }
4056
4057
4058 static void
llvmpipe_delete_fs_state(struct pipe_context * pipe,void * fs)4059 llvmpipe_delete_fs_state(struct pipe_context *pipe, void *fs)
4060 {
4061 struct llvmpipe_context *llvmpipe = llvmpipe_context(pipe);
4062 struct lp_fragment_shader *shader = fs;
4063 struct lp_fs_variant_list_item *li, *next;
4064
4065 /* Delete all the variants */
4066 LIST_FOR_EACH_ENTRY_SAFE(li, next, &shader->variants.list, list) {
4067 struct lp_fragment_shader_variant *variant;
4068 variant = li->base;
4069 llvmpipe_remove_shader_variant(llvmpipe, li->base);
4070 lp_fs_variant_reference(llvmpipe, &variant, NULL);
4071 }
4072
4073 lp_fs_reference(llvmpipe, &shader, NULL);
4074 }
4075
4076
4077 static void
llvmpipe_set_constant_buffer(struct pipe_context * pipe,enum pipe_shader_type shader,uint index,bool take_ownership,const struct pipe_constant_buffer * cb)4078 llvmpipe_set_constant_buffer(struct pipe_context *pipe,
4079 enum pipe_shader_type shader, uint index,
4080 bool take_ownership,
4081 const struct pipe_constant_buffer *cb)
4082 {
4083 struct llvmpipe_context *llvmpipe = llvmpipe_context(pipe);
4084 struct pipe_constant_buffer *constants = &llvmpipe->constants[shader][index];
4085
4086 assert(shader < PIPE_SHADER_TYPES);
4087 assert(index < ARRAY_SIZE(llvmpipe->constants[shader]));
4088
4089 /* note: reference counting */
4090 util_copy_constant_buffer(&llvmpipe->constants[shader][index], cb,
4091 take_ownership);
4092
4093 /* user_buffer is only valid until the next set_constant_buffer (at most,
4094 * possibly until shader deletion), so we need to upload it now to make sure
4095 * it doesn't get updated/freed out from under us.
4096 */
4097 if (constants->user_buffer) {
4098 u_upload_data(llvmpipe->pipe.const_uploader, 0, constants->buffer_size,
4099 16, constants->user_buffer, &constants->buffer_offset,
4100 &constants->buffer);
4101 }
4102 if (constants->buffer) {
4103 if (!(constants->buffer->bind & PIPE_BIND_CONSTANT_BUFFER)) {
4104 debug_printf("Illegal set constant without bind flag\n");
4105 constants->buffer->bind |= PIPE_BIND_CONSTANT_BUFFER;
4106 }
4107 }
4108
4109 if (shader == PIPE_SHADER_VERTEX ||
4110 shader == PIPE_SHADER_GEOMETRY ||
4111 shader == PIPE_SHADER_TESS_CTRL ||
4112 shader == PIPE_SHADER_TESS_EVAL) {
4113 /* Pass the constants to the 'draw' module */
4114 const unsigned size = cb ? cb->buffer_size : 0;
4115
4116 const ubyte *data = NULL;
4117 if (constants->buffer) {
4118 data = (ubyte *) llvmpipe_resource_data(constants->buffer)
4119 + constants->buffer_offset;
4120 }
4121
4122 draw_set_mapped_constant_buffer(llvmpipe->draw, shader,
4123 index, data, size);
4124 } else if (shader == PIPE_SHADER_COMPUTE) {
4125 llvmpipe->cs_dirty |= LP_CSNEW_CONSTANTS;
4126 } else {
4127 llvmpipe->dirty |= LP_NEW_FS_CONSTANTS;
4128 }
4129 }
4130
4131
4132 static void
llvmpipe_set_shader_buffers(struct pipe_context * pipe,enum pipe_shader_type shader,unsigned start_slot,unsigned count,const struct pipe_shader_buffer * buffers,unsigned writable_bitmask)4133 llvmpipe_set_shader_buffers(struct pipe_context *pipe,
4134 enum pipe_shader_type shader, unsigned start_slot,
4135 unsigned count,
4136 const struct pipe_shader_buffer *buffers,
4137 unsigned writable_bitmask)
4138 {
4139 struct llvmpipe_context *llvmpipe = llvmpipe_context(pipe);
4140
4141 unsigned i, idx;
4142 for (i = start_slot, idx = 0; i < start_slot + count; i++, idx++) {
4143 const struct pipe_shader_buffer *buffer = buffers ? &buffers[idx] : NULL;
4144
4145 util_copy_shader_buffer(&llvmpipe->ssbos[shader][i], buffer);
4146
4147 if (buffer && buffer->buffer) {
4148 boolean read_only = !(writable_bitmask & (1 << idx));
4149 llvmpipe_flush_resource(pipe, buffer->buffer, 0, read_only, false,
4150 false, "buffer");
4151 }
4152
4153 if (shader == PIPE_SHADER_VERTEX ||
4154 shader == PIPE_SHADER_GEOMETRY ||
4155 shader == PIPE_SHADER_TESS_CTRL ||
4156 shader == PIPE_SHADER_TESS_EVAL) {
4157 const unsigned size = buffer ? buffer->buffer_size : 0;
4158 const ubyte *data = NULL;
4159 if (buffer && buffer->buffer)
4160 data = (ubyte *) llvmpipe_resource_data(buffer->buffer);
4161 if (data)
4162 data += buffer->buffer_offset;
4163 draw_set_mapped_shader_buffer(llvmpipe->draw, shader,
4164 i, data, size);
4165 } else if (shader == PIPE_SHADER_COMPUTE) {
4166 llvmpipe->cs_dirty |= LP_CSNEW_SSBOS;
4167 } else if (shader == PIPE_SHADER_FRAGMENT) {
4168 llvmpipe->fs_ssbo_write_mask &= ~(((1 << count) - 1) << start_slot);
4169 llvmpipe->fs_ssbo_write_mask |= writable_bitmask << start_slot;
4170 llvmpipe->dirty |= LP_NEW_FS_SSBOS;
4171 }
4172 }
4173 }
4174
4175
4176 static void
llvmpipe_set_shader_images(struct pipe_context * pipe,enum pipe_shader_type shader,unsigned start_slot,unsigned count,unsigned unbind_num_trailing_slots,const struct pipe_image_view * images)4177 llvmpipe_set_shader_images(struct pipe_context *pipe,
4178 enum pipe_shader_type shader, unsigned start_slot,
4179 unsigned count, unsigned unbind_num_trailing_slots,
4180 const struct pipe_image_view *images)
4181 {
4182 struct llvmpipe_context *llvmpipe = llvmpipe_context(pipe);
4183 unsigned i, idx;
4184
4185 draw_flush(llvmpipe->draw);
4186 for (i = start_slot, idx = 0; i < start_slot + count; i++, idx++) {
4187 const struct pipe_image_view *image = images ? &images[idx] : NULL;
4188
4189 util_copy_image_view(&llvmpipe->images[shader][i], image);
4190
4191 if (image && image->resource) {
4192 bool read_only = !(image->access & PIPE_IMAGE_ACCESS_WRITE);
4193 llvmpipe_flush_resource(pipe, image->resource, 0, read_only, false,
4194 false, "image");
4195 }
4196 }
4197
4198 llvmpipe->num_images[shader] = start_slot + count;
4199 if (shader == PIPE_SHADER_VERTEX ||
4200 shader == PIPE_SHADER_GEOMETRY ||
4201 shader == PIPE_SHADER_TESS_CTRL ||
4202 shader == PIPE_SHADER_TESS_EVAL) {
4203 draw_set_images(llvmpipe->draw,
4204 shader,
4205 llvmpipe->images[shader],
4206 start_slot + count);
4207 } else if (shader == PIPE_SHADER_COMPUTE) {
4208 llvmpipe->cs_dirty |= LP_CSNEW_IMAGES;
4209 } else {
4210 llvmpipe->dirty |= LP_NEW_FS_IMAGES;
4211 }
4212
4213 if (unbind_num_trailing_slots) {
4214 llvmpipe_set_shader_images(pipe, shader, start_slot + count,
4215 unbind_num_trailing_slots, 0, NULL);
4216 }
4217 }
4218
4219
4220 /**
4221 * Return the blend factor equivalent to a destination alpha of one.
4222 */
4223 static inline enum pipe_blendfactor
force_dst_alpha_one(enum pipe_blendfactor factor,boolean clamped_zero)4224 force_dst_alpha_one(enum pipe_blendfactor factor, boolean clamped_zero)
4225 {
4226 switch (factor) {
4227 case PIPE_BLENDFACTOR_DST_ALPHA:
4228 return PIPE_BLENDFACTOR_ONE;
4229 case PIPE_BLENDFACTOR_INV_DST_ALPHA:
4230 return PIPE_BLENDFACTOR_ZERO;
4231 case PIPE_BLENDFACTOR_SRC_ALPHA_SATURATE:
4232 if (clamped_zero)
4233 return PIPE_BLENDFACTOR_ZERO;
4234 else
4235 return PIPE_BLENDFACTOR_SRC_ALPHA_SATURATE;
4236 default:
4237 return factor;
4238 }
4239 }
4240
4241
4242 /**
4243 * We need to generate several variants of the fragment pipeline to match
4244 * all the combinations of the contributing state atoms.
4245 *
4246 * TODO: there is actually no reason to tie this to context state -- the
4247 * generated code could be cached globally in the screen.
4248 */
4249 static struct lp_fragment_shader_variant_key *
make_variant_key(struct llvmpipe_context * lp,struct lp_fragment_shader * shader,char * store)4250 make_variant_key(struct llvmpipe_context *lp,
4251 struct lp_fragment_shader *shader,
4252 char *store)
4253 {
4254 struct lp_fragment_shader_variant_key *key =
4255 (struct lp_fragment_shader_variant_key *)store;
4256
4257 memset(key, 0, sizeof(*key));
4258
4259 if (lp->framebuffer.zsbuf) {
4260 const enum pipe_format zsbuf_format = lp->framebuffer.zsbuf->format;
4261 const struct util_format_description *zsbuf_desc =
4262 util_format_description(zsbuf_format);
4263
4264 if (lp->depth_stencil->depth_enabled &&
4265 util_format_has_depth(zsbuf_desc)) {
4266 key->zsbuf_format = zsbuf_format;
4267 key->depth.enabled = lp->depth_stencil->depth_enabled;
4268 key->depth.writemask = lp->depth_stencil->depth_writemask;
4269 key->depth.func = lp->depth_stencil->depth_func;
4270 }
4271 if (lp->depth_stencil->stencil[0].enabled &&
4272 util_format_has_stencil(zsbuf_desc)) {
4273 key->zsbuf_format = zsbuf_format;
4274 memcpy(&key->stencil, &lp->depth_stencil->stencil,
4275 sizeof key->stencil);
4276 }
4277 if (llvmpipe_resource_is_1d(lp->framebuffer.zsbuf->texture)) {
4278 key->resource_1d = TRUE;
4279 }
4280 key->zsbuf_nr_samples =
4281 util_res_sample_count(lp->framebuffer.zsbuf->texture);
4282
4283 /*
4284 * Restrict depth values if the API is clamped (GL, VK with ext)
4285 * for non float Z buffer
4286 */
4287 key->restrict_depth_values =
4288 !(lp->rasterizer->unclamped_fragment_depth_values &&
4289 util_format_get_depth_only(zsbuf_format) == PIPE_FORMAT_Z32_FLOAT);
4290 }
4291
4292 /*
4293 * Propagate the depth clamp setting from the rasterizer state.
4294 */
4295 key->depth_clamp = lp->rasterizer->depth_clamp;
4296
4297 /* alpha test only applies if render buffer 0 is non-integer
4298 * (or does not exist)
4299 */
4300 if (!lp->framebuffer.nr_cbufs ||
4301 !lp->framebuffer.cbufs[0] ||
4302 !util_format_is_pure_integer(lp->framebuffer.cbufs[0]->format)) {
4303 key->alpha.enabled = lp->depth_stencil->alpha_enabled;
4304 }
4305 if (key->alpha.enabled) {
4306 key->alpha.func = lp->depth_stencil->alpha_func;
4307 /* alpha.ref_value is passed in jit_context */
4308 }
4309
4310 key->flatshade = lp->rasterizer->flatshade;
4311 key->multisample = lp->rasterizer->multisample;
4312 key->no_ms_sample_mask_out = lp->rasterizer->no_ms_sample_mask_out;
4313 if (lp->active_occlusion_queries && !lp->queries_disabled) {
4314 key->occlusion_count = TRUE;
4315 }
4316
4317 memcpy(&key->blend, lp->blend, sizeof key->blend);
4318
4319 key->coverage_samples = 1;
4320 key->min_samples = 1;
4321 if (key->multisample) {
4322 key->coverage_samples =
4323 util_framebuffer_get_num_samples(&lp->framebuffer);
4324 /* Per EXT_shader_framebuffer_fetch spec:
4325 *
4326 * "1. How is framebuffer data treated during multisample rendering?
4327 *
4328 * RESOLVED: Reading the value of gl_LastFragData produces a different
4329 * result for each sample. This implies that all or part of the shader be
4330 * run once for each sample, but has no additional implications on fragment
4331 * shader input variables which may still be interpolated per pixel by the
4332 * implementation."
4333 *
4334 * ARM_shader_framebuffer_fetch_depth_stencil spec further says:
4335 *
4336 * "(1) When multisampling is enabled, does the shader run per sample?
4337 *
4338 * RESOLVED.
4339 *
4340 * This behavior is inherited from either EXT_shader_framebuffer_fetch or
4341 * ARM_shader_framebuffer_fetch as described in the interactions section.
4342 * If neither extension is supported, the shader runs once per fragment."
4343 *
4344 * Therefore we should always enable per-sample shading when FB fetch is used.
4345 */
4346 if (lp->min_samples > 1 || shader->info.base.uses_fbfetch)
4347 key->min_samples = key->coverage_samples;
4348 }
4349 key->nr_cbufs = lp->framebuffer.nr_cbufs;
4350
4351 if (!key->blend.independent_blend_enable) {
4352 // we always need independent blend otherwise the fixups below won't work
4353 for (unsigned i = 1; i < key->nr_cbufs; i++) {
4354 memcpy(&key->blend.rt[i], &key->blend.rt[0],
4355 sizeof(key->blend.rt[0]));
4356 }
4357 key->blend.independent_blend_enable = 1;
4358 }
4359
4360 for (unsigned i = 0; i < lp->framebuffer.nr_cbufs; i++) {
4361 struct pipe_rt_blend_state *blend_rt = &key->blend.rt[i];
4362
4363 if (lp->framebuffer.cbufs[i]) {
4364 const enum pipe_format format = lp->framebuffer.cbufs[i]->format;
4365
4366 key->cbuf_format[i] = format;
4367 key->cbuf_nr_samples[i] =
4368 util_res_sample_count(lp->framebuffer.cbufs[i]->texture);
4369
4370 /*
4371 * Figure out if this is a 1d resource. Note that OpenGL allows crazy
4372 * mixing of 2d textures with height 1 and 1d textures, so make sure
4373 * we pick 1d if any cbuf or zsbuf is 1d.
4374 */
4375 if (llvmpipe_resource_is_1d(lp->framebuffer.cbufs[i]->texture)) {
4376 key->resource_1d = TRUE;
4377 }
4378
4379 const struct util_format_description *format_desc =
4380 util_format_description(format);
4381 assert(format_desc->colorspace == UTIL_FORMAT_COLORSPACE_RGB ||
4382 format_desc->colorspace == UTIL_FORMAT_COLORSPACE_SRGB);
4383
4384 /*
4385 * Mask out color channels not present in the color buffer.
4386 */
4387 blend_rt->colormask &= util_format_colormask(format_desc);
4388
4389 /*
4390 * Disable blend for integer formats.
4391 */
4392 if (util_format_is_pure_integer(format)) {
4393 blend_rt->blend_enable = 0;
4394 }
4395
4396 /*
4397 * Our swizzled render tiles always have an alpha channel, but the
4398 * linear render target format often does not, so force here the dst
4399 * alpha to be one.
4400 *
4401 * This is not a mere optimization. Wrong results will be produced if
4402 * the dst alpha is used, the dst format does not have alpha, and the
4403 * previous rendering was not flushed from the swizzled to linear
4404 * buffer. For example, NonPowTwo DCT.
4405 *
4406 * TODO: This should be generalized to all channels for better
4407 * performance, but only alpha causes correctness issues.
4408 *
4409 * Also, force rgb/alpha func/factors match, to make AoS blending
4410 * easier.
4411 */
4412 if (format_desc->swizzle[3] > PIPE_SWIZZLE_W ||
4413 format_desc->swizzle[3] == format_desc->swizzle[0]) {
4414 // Doesn't cover mixed snorm/unorm but can't render to them anyway
4415 boolean clamped_zero = !util_format_is_float(format) &&
4416 !util_format_is_snorm(format);
4417 blend_rt->rgb_src_factor =
4418 force_dst_alpha_one(blend_rt->rgb_src_factor, clamped_zero);
4419 blend_rt->rgb_dst_factor =
4420 force_dst_alpha_one(blend_rt->rgb_dst_factor, clamped_zero);
4421 blend_rt->alpha_func = blend_rt->rgb_func;
4422 blend_rt->alpha_src_factor = blend_rt->rgb_src_factor;
4423 blend_rt->alpha_dst_factor = blend_rt->rgb_dst_factor;
4424 }
4425 }
4426 else {
4427 /* no color buffer for this fragment output */
4428 key->cbuf_format[i] = PIPE_FORMAT_NONE;
4429 key->cbuf_nr_samples[i] = 0;
4430 blend_rt->colormask = 0x0;
4431 blend_rt->blend_enable = 0;
4432 }
4433 }
4434
4435 /* This value will be the same for all the variants of a given shader:
4436 */
4437 key->nr_samplers = shader->info.base.file_max[TGSI_FILE_SAMPLER] + 1;
4438
4439 if (shader->info.base.file_max[TGSI_FILE_SAMPLER_VIEW] != -1) {
4440 key->nr_sampler_views =
4441 shader->info.base.file_max[TGSI_FILE_SAMPLER_VIEW] + 1;
4442 }
4443
4444 struct lp_sampler_static_state *fs_sampler =
4445 lp_fs_variant_key_samplers(key);
4446
4447 memset(fs_sampler, 0,
4448 MAX2(key->nr_samplers, key->nr_sampler_views) * sizeof *fs_sampler);
4449
4450 for (unsigned i = 0; i < key->nr_samplers; ++i) {
4451 if (shader->info.base.file_mask[TGSI_FILE_SAMPLER] & (1 << i)) {
4452 lp_sampler_static_sampler_state(&fs_sampler[i].sampler_state,
4453 lp->samplers[PIPE_SHADER_FRAGMENT][i]);
4454 }
4455 }
4456
4457 /*
4458 * XXX If TGSI_FILE_SAMPLER_VIEW exists assume all texture opcodes
4459 * are dx10-style? Can't really have mixed opcodes, at least not
4460 * if we want to skip the holes here (without rescanning tgsi).
4461 */
4462 if (shader->info.base.file_max[TGSI_FILE_SAMPLER_VIEW] != -1) {
4463 for (unsigned i = 0; i < key->nr_sampler_views; ++i) {
4464 /*
4465 * Note sview may exceed what's representable by file_mask.
4466 * This will still work, the only downside is that not actually
4467 * used views may be included in the shader key.
4468 */
4469 if ((shader->info.base.file_mask[TGSI_FILE_SAMPLER_VIEW]
4470 & (1u << (i & 31))) || i > 31) {
4471 lp_sampler_static_texture_state(&fs_sampler[i].texture_state,
4472 lp->sampler_views[PIPE_SHADER_FRAGMENT][i]);
4473 }
4474 }
4475 }
4476 else {
4477 key->nr_sampler_views = key->nr_samplers;
4478 for (unsigned i = 0; i < key->nr_sampler_views; ++i) {
4479 if ((shader->info.base.file_mask[TGSI_FILE_SAMPLER] & (1 << i)) || i > 31) {
4480 lp_sampler_static_texture_state(&fs_sampler[i].texture_state,
4481 lp->sampler_views[PIPE_SHADER_FRAGMENT][i]);
4482 }
4483 }
4484 }
4485
4486 struct lp_image_static_state *lp_image = lp_fs_variant_key_images(key);
4487 key->nr_images = shader->info.base.file_max[TGSI_FILE_IMAGE] + 1;
4488
4489 if (key->nr_images)
4490 memset(lp_image, 0,
4491 key->nr_images * sizeof *lp_image);
4492 for (unsigned i = 0; i < key->nr_images; ++i) {
4493 if ((shader->info.base.file_mask[TGSI_FILE_IMAGE] & (1 << i)) || i > 31) {
4494 lp_sampler_static_texture_state_image(&lp_image[i].image_state,
4495 &lp->images[PIPE_SHADER_FRAGMENT][i]);
4496 }
4497 }
4498
4499 if (shader->kind == LP_FS_KIND_AERO_MINIFICATION) {
4500 struct lp_sampler_static_state *samp0 =
4501 lp_fs_variant_key_sampler_idx(key, 0);
4502 assert(samp0);
4503 samp0->sampler_state.min_img_filter = PIPE_TEX_FILTER_NEAREST;
4504 samp0->sampler_state.mag_img_filter = PIPE_TEX_FILTER_NEAREST;
4505 }
4506
4507 return key;
4508 }
4509
4510
4511 /**
4512 * Update fragment shader state. This is called just prior to drawing
4513 * something when some fragment-related state has changed.
4514 */
4515 void
llvmpipe_update_fs(struct llvmpipe_context * lp)4516 llvmpipe_update_fs(struct llvmpipe_context *lp)
4517 {
4518 struct lp_fragment_shader *shader = lp->fs;
4519
4520 char store[LP_FS_MAX_VARIANT_KEY_SIZE];
4521 const struct lp_fragment_shader_variant_key *key =
4522 make_variant_key(lp, shader, store);
4523
4524 struct lp_fragment_shader_variant *variant = NULL;
4525 struct lp_fs_variant_list_item *li;
4526 /* Search the variants for one which matches the key */
4527 LIST_FOR_EACH_ENTRY(li, &shader->variants.list, list) {
4528 if (memcmp(&li->base->key, key, shader->variant_key_size) == 0) {
4529 variant = li->base;
4530 break;
4531 }
4532 }
4533
4534 if (variant) {
4535 /* Move this variant to the head of the list to implement LRU
4536 * deletion of shader's when we have too many.
4537 */
4538 list_move_to(&variant->list_item_global.list, &lp->fs_variants_list.list);
4539 }
4540 else {
4541 /* variant not found, create it now */
4542
4543 if (LP_DEBUG & DEBUG_FS) {
4544 debug_printf("%u variants,\t%u instrs,\t%u instrs/variant\n",
4545 lp->nr_fs_variants,
4546 lp->nr_fs_instrs,
4547 lp->nr_fs_variants ? lp->nr_fs_instrs / lp->nr_fs_variants : 0);
4548 }
4549
4550 /* First, check if we've exceeded the max number of shader variants.
4551 * If so, free 6.25% of them (the least recently used ones).
4552 */
4553 const unsigned variants_to_cull =
4554 lp->nr_fs_variants >= LP_MAX_SHADER_VARIANTS
4555 ? LP_MAX_SHADER_VARIANTS / 16 : 0;
4556
4557 if (variants_to_cull ||
4558 lp->nr_fs_instrs >= LP_MAX_SHADER_INSTRUCTIONS) {
4559 if (gallivm_debug & GALLIVM_DEBUG_PERF) {
4560 debug_printf("Evicting FS: %u fs variants,\t%u total variants,"
4561 "\t%u instrs,\t%u instrs/variant\n",
4562 shader->variants_cached,
4563 lp->nr_fs_variants, lp->nr_fs_instrs,
4564 lp->nr_fs_instrs / lp->nr_fs_variants);
4565 }
4566
4567 /*
4568 * We need to re-check lp->nr_fs_variants because an arbitrarliy large
4569 * number of shader variants (potentially all of them) could be
4570 * pending for destruction on flush.
4571 */
4572
4573 for (unsigned i = 0;
4574 i < variants_to_cull ||
4575 lp->nr_fs_instrs >= LP_MAX_SHADER_INSTRUCTIONS;
4576 i++) {
4577 struct lp_fs_variant_list_item *item;
4578 if (list_is_empty(&lp->fs_variants_list.list)) {
4579 break;
4580 }
4581 item = list_last_entry(&lp->fs_variants_list.list,
4582 struct lp_fs_variant_list_item, list);
4583 assert(item);
4584 assert(item->base);
4585 llvmpipe_remove_shader_variant(lp, item->base);
4586 struct lp_fragment_shader_variant *variant = item->base;
4587 lp_fs_variant_reference(lp, &variant, NULL);
4588 }
4589 }
4590
4591 /*
4592 * Generate the new variant.
4593 */
4594 int64_t t0 = os_time_get();
4595 variant = generate_variant(lp, shader, key);
4596 int64_t t1 = os_time_get();
4597 int64_t dt = t1 - t0;
4598 LP_COUNT_ADD(llvm_compile_time, dt);
4599 LP_COUNT_ADD(nr_llvm_compiles, 2); /* emit vs. omit in/out test */
4600
4601 /* Put the new variant into the list */
4602 if (variant) {
4603 list_add(&variant->list_item_local.list, &shader->variants.list);
4604 list_add(&variant->list_item_global.list, &lp->fs_variants_list.list);
4605 lp->nr_fs_variants++;
4606 lp->nr_fs_instrs += variant->nr_instrs;
4607 shader->variants_cached++;
4608 }
4609 }
4610
4611 /* Bind this variant */
4612 lp_setup_set_fs_variant(lp->setup, variant);
4613 }
4614
4615
4616 void
llvmpipe_init_fs_funcs(struct llvmpipe_context * llvmpipe)4617 llvmpipe_init_fs_funcs(struct llvmpipe_context *llvmpipe)
4618 {
4619 llvmpipe->pipe.create_fs_state = llvmpipe_create_fs_state;
4620 llvmpipe->pipe.bind_fs_state = llvmpipe_bind_fs_state;
4621 llvmpipe->pipe.delete_fs_state = llvmpipe_delete_fs_state;
4622 llvmpipe->pipe.set_constant_buffer = llvmpipe_set_constant_buffer;
4623 llvmpipe->pipe.set_shader_buffers = llvmpipe_set_shader_buffers;
4624 llvmpipe->pipe.set_shader_images = llvmpipe_set_shader_images;
4625 }
4626