1 /**************************************************************************
2 *
3 * Copyright 2009-2010 VMware, Inc.
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
21 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27
28 /**
29 * @file
30 * Depth/stencil testing to LLVM IR translation.
31 *
32 * To be done accurately/efficiently the depth/stencil test must be done with
33 * the same type/format of the depth/stencil buffer, which implies massaging
34 * the incoming depths to fit into place. Using a more straightforward
35 * type/format for depth/stencil values internally and only convert when
36 * flushing would avoid this, but it would most likely result in depth fighting
37 * artifacts.
38 *
39 * Since we're using linear layout for everything, but we need to deal with
40 * 2x2 quads, we need to load/store multiple values and swizzle them into
41 * place (we could avoid this by doing depth/stencil testing in linear format,
42 * which would be easy for late depth/stencil test as we could do that after
43 * the fragment shader loop just as we do for color buffers, but more tricky
44 * for early depth test as we'd need both masks and interpolated depth in
45 * linear format).
46 *
47 *
48 * @author Jose Fonseca <jfonseca@vmware.com>
49 * @author Brian Paul <jfonseca@vmware.com>
50 */
51
52 #include "pipe/p_state.h"
53 #include "util/u_format.h"
54 #include "util/u_cpu_detect.h"
55
56 #include "gallivm/lp_bld_type.h"
57 #include "gallivm/lp_bld_arit.h"
58 #include "gallivm/lp_bld_bitarit.h"
59 #include "gallivm/lp_bld_const.h"
60 #include "gallivm/lp_bld_conv.h"
61 #include "gallivm/lp_bld_logic.h"
62 #include "gallivm/lp_bld_flow.h"
63 #include "gallivm/lp_bld_intr.h"
64 #include "gallivm/lp_bld_debug.h"
65 #include "gallivm/lp_bld_swizzle.h"
66 #include "gallivm/lp_bld_pack.h"
67
68 #include "lp_bld_depth.h"
69
70
71 /** Used to select fields from pipe_stencil_state */
72 enum stencil_op {
73 S_FAIL_OP,
74 Z_FAIL_OP,
75 Z_PASS_OP
76 };
77
78
79
80 /**
81 * Do the stencil test comparison (compare FB stencil values against ref value).
82 * This will be used twice when generating two-sided stencil code.
83 * \param stencil the front/back stencil state
84 * \param stencilRef the stencil reference value, replicated as a vector
85 * \param stencilVals vector of stencil values from framebuffer
86 * \return vector mask of pass/fail values (~0 or 0)
87 */
88 static LLVMValueRef
lp_build_stencil_test_single(struct lp_build_context * bld,const struct pipe_stencil_state * stencil,LLVMValueRef stencilRef,LLVMValueRef stencilVals)89 lp_build_stencil_test_single(struct lp_build_context *bld,
90 const struct pipe_stencil_state *stencil,
91 LLVMValueRef stencilRef,
92 LLVMValueRef stencilVals)
93 {
94 LLVMBuilderRef builder = bld->gallivm->builder;
95 const unsigned stencilMax = 255; /* XXX fix */
96 struct lp_type type = bld->type;
97 LLVMValueRef res;
98
99 /*
100 * SSE2 has intrinsics for signed comparisons, but not unsigned ones. Values
101 * are between 0..255 so ensure we generate the fastest comparisons for
102 * wider elements.
103 */
104 if (type.width <= 8) {
105 assert(!type.sign);
106 } else {
107 assert(type.sign);
108 }
109
110 assert(stencil->enabled);
111
112 if (stencil->valuemask != stencilMax) {
113 /* compute stencilRef = stencilRef & valuemask */
114 LLVMValueRef valuemask = lp_build_const_int_vec(bld->gallivm, type, stencil->valuemask);
115 stencilRef = LLVMBuildAnd(builder, stencilRef, valuemask, "");
116 /* compute stencilVals = stencilVals & valuemask */
117 stencilVals = LLVMBuildAnd(builder, stencilVals, valuemask, "");
118 }
119
120 res = lp_build_cmp(bld, stencil->func, stencilRef, stencilVals);
121
122 return res;
123 }
124
125
126 /**
127 * Do the one or two-sided stencil test comparison.
128 * \sa lp_build_stencil_test_single
129 * \param front_facing an integer vector mask, indicating front (~0) or back
130 * (0) facing polygon. If NULL, assume front-facing.
131 */
132 static LLVMValueRef
lp_build_stencil_test(struct lp_build_context * bld,const struct pipe_stencil_state stencil[2],LLVMValueRef stencilRefs[2],LLVMValueRef stencilVals,LLVMValueRef front_facing)133 lp_build_stencil_test(struct lp_build_context *bld,
134 const struct pipe_stencil_state stencil[2],
135 LLVMValueRef stencilRefs[2],
136 LLVMValueRef stencilVals,
137 LLVMValueRef front_facing)
138 {
139 LLVMValueRef res;
140
141 assert(stencil[0].enabled);
142
143 /* do front face test */
144 res = lp_build_stencil_test_single(bld, &stencil[0],
145 stencilRefs[0], stencilVals);
146
147 if (stencil[1].enabled && front_facing != NULL) {
148 /* do back face test */
149 LLVMValueRef back_res;
150
151 back_res = lp_build_stencil_test_single(bld, &stencil[1],
152 stencilRefs[1], stencilVals);
153
154 res = lp_build_select(bld, front_facing, res, back_res);
155 }
156
157 return res;
158 }
159
160
161 /**
162 * Apply the stencil operator (add/sub/keep/etc) to the given vector
163 * of stencil values.
164 * \return new stencil values vector
165 */
166 static LLVMValueRef
lp_build_stencil_op_single(struct lp_build_context * bld,const struct pipe_stencil_state * stencil,enum stencil_op op,LLVMValueRef stencilRef,LLVMValueRef stencilVals)167 lp_build_stencil_op_single(struct lp_build_context *bld,
168 const struct pipe_stencil_state *stencil,
169 enum stencil_op op,
170 LLVMValueRef stencilRef,
171 LLVMValueRef stencilVals)
172
173 {
174 LLVMBuilderRef builder = bld->gallivm->builder;
175 struct lp_type type = bld->type;
176 LLVMValueRef res;
177 LLVMValueRef max = lp_build_const_int_vec(bld->gallivm, type, 0xff);
178 unsigned stencil_op;
179
180 assert(type.sign);
181
182 switch (op) {
183 case S_FAIL_OP:
184 stencil_op = stencil->fail_op;
185 break;
186 case Z_FAIL_OP:
187 stencil_op = stencil->zfail_op;
188 break;
189 case Z_PASS_OP:
190 stencil_op = stencil->zpass_op;
191 break;
192 default:
193 assert(0 && "Invalid stencil_op mode");
194 stencil_op = PIPE_STENCIL_OP_KEEP;
195 }
196
197 switch (stencil_op) {
198 case PIPE_STENCIL_OP_KEEP:
199 res = stencilVals;
200 /* we can return early for this case */
201 return res;
202 case PIPE_STENCIL_OP_ZERO:
203 res = bld->zero;
204 break;
205 case PIPE_STENCIL_OP_REPLACE:
206 res = stencilRef;
207 break;
208 case PIPE_STENCIL_OP_INCR:
209 res = lp_build_add(bld, stencilVals, bld->one);
210 res = lp_build_min(bld, res, max);
211 break;
212 case PIPE_STENCIL_OP_DECR:
213 res = lp_build_sub(bld, stencilVals, bld->one);
214 res = lp_build_max(bld, res, bld->zero);
215 break;
216 case PIPE_STENCIL_OP_INCR_WRAP:
217 res = lp_build_add(bld, stencilVals, bld->one);
218 res = LLVMBuildAnd(builder, res, max, "");
219 break;
220 case PIPE_STENCIL_OP_DECR_WRAP:
221 res = lp_build_sub(bld, stencilVals, bld->one);
222 res = LLVMBuildAnd(builder, res, max, "");
223 break;
224 case PIPE_STENCIL_OP_INVERT:
225 res = LLVMBuildNot(builder, stencilVals, "");
226 res = LLVMBuildAnd(builder, res, max, "");
227 break;
228 default:
229 assert(0 && "bad stencil op mode");
230 res = bld->undef;
231 }
232
233 return res;
234 }
235
236
237 /**
238 * Do the one or two-sided stencil test op/update.
239 */
240 static LLVMValueRef
lp_build_stencil_op(struct lp_build_context * bld,const struct pipe_stencil_state stencil[2],enum stencil_op op,LLVMValueRef stencilRefs[2],LLVMValueRef stencilVals,LLVMValueRef mask,LLVMValueRef front_facing)241 lp_build_stencil_op(struct lp_build_context *bld,
242 const struct pipe_stencil_state stencil[2],
243 enum stencil_op op,
244 LLVMValueRef stencilRefs[2],
245 LLVMValueRef stencilVals,
246 LLVMValueRef mask,
247 LLVMValueRef front_facing)
248
249 {
250 LLVMBuilderRef builder = bld->gallivm->builder;
251 LLVMValueRef res;
252
253 assert(stencil[0].enabled);
254
255 /* do front face op */
256 res = lp_build_stencil_op_single(bld, &stencil[0], op,
257 stencilRefs[0], stencilVals);
258
259 if (stencil[1].enabled && front_facing != NULL) {
260 /* do back face op */
261 LLVMValueRef back_res;
262
263 back_res = lp_build_stencil_op_single(bld, &stencil[1], op,
264 stencilRefs[1], stencilVals);
265
266 res = lp_build_select(bld, front_facing, res, back_res);
267 }
268
269 if (stencil[0].writemask != 0xff ||
270 (stencil[1].enabled && front_facing != NULL && stencil[1].writemask != 0xff)) {
271 /* mask &= stencil[0].writemask */
272 LLVMValueRef writemask = lp_build_const_int_vec(bld->gallivm, bld->type,
273 stencil[0].writemask);
274 if (stencil[1].enabled && stencil[1].writemask != stencil[0].writemask && front_facing != NULL) {
275 LLVMValueRef back_writemask = lp_build_const_int_vec(bld->gallivm, bld->type,
276 stencil[1].writemask);
277 writemask = lp_build_select(bld, front_facing, writemask, back_writemask);
278 }
279
280 mask = LLVMBuildAnd(builder, mask, writemask, "");
281 /* res = (res & mask) | (stencilVals & ~mask) */
282 res = lp_build_select_bitwise(bld, mask, res, stencilVals);
283 }
284 else {
285 /* res = mask ? res : stencilVals */
286 res = lp_build_select(bld, mask, res, stencilVals);
287 }
288
289 return res;
290 }
291
292
293
294 /**
295 * Return a type that matches the depth/stencil format.
296 */
297 struct lp_type
lp_depth_type(const struct util_format_description * format_desc,unsigned length)298 lp_depth_type(const struct util_format_description *format_desc,
299 unsigned length)
300 {
301 struct lp_type type;
302 unsigned z_swizzle;
303
304 assert(format_desc->colorspace == UTIL_FORMAT_COLORSPACE_ZS);
305 assert(format_desc->block.width == 1);
306 assert(format_desc->block.height == 1);
307
308 memset(&type, 0, sizeof type);
309 type.width = format_desc->block.bits;
310
311 z_swizzle = format_desc->swizzle[0];
312 if (z_swizzle < 4) {
313 if (format_desc->channel[z_swizzle].type == UTIL_FORMAT_TYPE_FLOAT) {
314 type.floating = TRUE;
315 assert(z_swizzle == 0);
316 assert(format_desc->channel[z_swizzle].size == 32);
317 }
318 else if(format_desc->channel[z_swizzle].type == UTIL_FORMAT_TYPE_UNSIGNED) {
319 assert(format_desc->block.bits <= 32);
320 assert(format_desc->channel[z_swizzle].normalized);
321 if (format_desc->channel[z_swizzle].size < format_desc->block.bits) {
322 /* Prefer signed integers when possible, as SSE has less support
323 * for unsigned comparison;
324 */
325 type.sign = TRUE;
326 }
327 }
328 else
329 assert(0);
330 }
331
332 type.length = length;
333
334 return type;
335 }
336
337
338 /**
339 * Compute bitmask and bit shift to apply to the incoming fragment Z values
340 * and the Z buffer values needed before doing the Z comparison.
341 *
342 * Note that we leave the Z bits in the position that we find them
343 * in the Z buffer (typically 0xffffff00 or 0x00ffffff). That lets us
344 * get by with fewer bit twiddling steps.
345 */
346 static boolean
get_z_shift_and_mask(const struct util_format_description * format_desc,unsigned * shift,unsigned * width,unsigned * mask)347 get_z_shift_and_mask(const struct util_format_description *format_desc,
348 unsigned *shift, unsigned *width, unsigned *mask)
349 {
350 unsigned total_bits;
351 unsigned z_swizzle;
352
353 assert(format_desc->colorspace == UTIL_FORMAT_COLORSPACE_ZS);
354 assert(format_desc->block.width == 1);
355 assert(format_desc->block.height == 1);
356
357 /* 64bit d/s format is special already extracted 32 bits */
358 total_bits = format_desc->block.bits > 32 ? 32 : format_desc->block.bits;
359
360 z_swizzle = format_desc->swizzle[0];
361
362 if (z_swizzle == PIPE_SWIZZLE_NONE)
363 return FALSE;
364
365 *width = format_desc->channel[z_swizzle].size;
366 /* & 31 is for the same reason as the 32-bit limit above */
367 *shift = format_desc->channel[z_swizzle].shift & 31;
368
369 if (*width == total_bits) {
370 *mask = 0xffffffff;
371 } else {
372 *mask = ((1 << *width) - 1) << *shift;
373 }
374
375 return TRUE;
376 }
377
378
379 /**
380 * Compute bitmask and bit shift to apply to the framebuffer pixel values
381 * to put the stencil bits in the least significant position.
382 * (i.e. 0x000000ff)
383 */
384 static boolean
get_s_shift_and_mask(const struct util_format_description * format_desc,unsigned * shift,unsigned * mask)385 get_s_shift_and_mask(const struct util_format_description *format_desc,
386 unsigned *shift, unsigned *mask)
387 {
388 unsigned s_swizzle;
389 unsigned sz;
390
391 s_swizzle = format_desc->swizzle[1];
392
393 if (s_swizzle == PIPE_SWIZZLE_NONE)
394 return FALSE;
395
396 /* just special case 64bit d/s format */
397 if (format_desc->block.bits > 32) {
398 /* XXX big-endian? */
399 assert(format_desc->format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT);
400 *shift = 0;
401 *mask = 0xff;
402 return TRUE;
403 }
404
405 *shift = format_desc->channel[s_swizzle].shift;
406 sz = format_desc->channel[s_swizzle].size;
407 *mask = (1U << sz) - 1U;
408
409 return TRUE;
410 }
411
412
413 /**
414 * Perform the occlusion test and increase the counter.
415 * Test the depth mask. Add the number of channel which has none zero mask
416 * into the occlusion counter. e.g. maskvalue is {-1, -1, -1, -1}.
417 * The counter will add 4.
418 * TODO: could get that out of the fs loop.
419 *
420 * \param type holds element type of the mask vector.
421 * \param maskvalue is the depth test mask.
422 * \param counter is a pointer of the uint32 counter.
423 */
424 void
lp_build_occlusion_count(struct gallivm_state * gallivm,struct lp_type type,LLVMValueRef maskvalue,LLVMValueRef counter)425 lp_build_occlusion_count(struct gallivm_state *gallivm,
426 struct lp_type type,
427 LLVMValueRef maskvalue,
428 LLVMValueRef counter)
429 {
430 LLVMBuilderRef builder = gallivm->builder;
431 LLVMContextRef context = gallivm->context;
432 LLVMValueRef countmask = lp_build_const_int_vec(gallivm, type, 1);
433 LLVMValueRef count, newcount;
434
435 assert(type.length <= 16);
436 assert(type.floating);
437
438 if(util_cpu_caps.has_sse && type.length == 4) {
439 const char *movmskintr = "llvm.x86.sse.movmsk.ps";
440 const char *popcntintr = "llvm.ctpop.i32";
441 LLVMValueRef bits = LLVMBuildBitCast(builder, maskvalue,
442 lp_build_vec_type(gallivm, type), "");
443 bits = lp_build_intrinsic_unary(builder, movmskintr,
444 LLVMInt32TypeInContext(context), bits);
445 count = lp_build_intrinsic_unary(builder, popcntintr,
446 LLVMInt32TypeInContext(context), bits);
447 count = LLVMBuildZExt(builder, count, LLVMIntTypeInContext(context, 64), "");
448 }
449 else if(util_cpu_caps.has_avx && type.length == 8) {
450 const char *movmskintr = "llvm.x86.avx.movmsk.ps.256";
451 const char *popcntintr = "llvm.ctpop.i32";
452 LLVMValueRef bits = LLVMBuildBitCast(builder, maskvalue,
453 lp_build_vec_type(gallivm, type), "");
454 bits = lp_build_intrinsic_unary(builder, movmskintr,
455 LLVMInt32TypeInContext(context), bits);
456 count = lp_build_intrinsic_unary(builder, popcntintr,
457 LLVMInt32TypeInContext(context), bits);
458 count = LLVMBuildZExt(builder, count, LLVMIntTypeInContext(context, 64), "");
459 }
460 else {
461 unsigned i;
462 LLVMValueRef countv = LLVMBuildAnd(builder, maskvalue, countmask, "countv");
463 LLVMTypeRef counttype = LLVMIntTypeInContext(context, type.length * 8);
464 LLVMTypeRef i8vntype = LLVMVectorType(LLVMInt8TypeInContext(context), type.length * 4);
465 LLVMValueRef shufflev, countd;
466 LLVMValueRef shuffles[16];
467 const char *popcntintr = NULL;
468
469 countv = LLVMBuildBitCast(builder, countv, i8vntype, "");
470
471 for (i = 0; i < type.length; i++) {
472 shuffles[i] = lp_build_const_int32(gallivm, 4*i);
473 }
474
475 shufflev = LLVMConstVector(shuffles, type.length);
476 countd = LLVMBuildShuffleVector(builder, countv, LLVMGetUndef(i8vntype), shufflev, "");
477 countd = LLVMBuildBitCast(builder, countd, counttype, "countd");
478
479 /*
480 * XXX FIXME
481 * this is bad on cpus without popcount (on x86 supported by intel
482 * nehalem, amd barcelona, and up - not tied to sse42).
483 * Would be much faster to just sum the 4 elements of the vector with
484 * some horizontal add (shuffle/add/shuffle/add after the initial and).
485 */
486 switch (type.length) {
487 case 4:
488 popcntintr = "llvm.ctpop.i32";
489 break;
490 case 8:
491 popcntintr = "llvm.ctpop.i64";
492 break;
493 case 16:
494 popcntintr = "llvm.ctpop.i128";
495 break;
496 default:
497 assert(0);
498 }
499 count = lp_build_intrinsic_unary(builder, popcntintr, counttype, countd);
500
501 if (type.length > 8) {
502 count = LLVMBuildTrunc(builder, count, LLVMIntTypeInContext(context, 64), "");
503 }
504 else if (type.length < 8) {
505 count = LLVMBuildZExt(builder, count, LLVMIntTypeInContext(context, 64), "");
506 }
507 }
508 newcount = LLVMBuildLoad(builder, counter, "origcount");
509 newcount = LLVMBuildAdd(builder, newcount, count, "newcount");
510 LLVMBuildStore(builder, newcount, counter);
511 }
512
513
514 /**
515 * Load depth/stencil values.
516 * The stored values are linear, swizzle them.
517 *
518 * \param type the data type of the fragment depth/stencil values
519 * \param format_desc description of the depth/stencil surface
520 * \param is_1d whether this resource has only one dimension
521 * \param loop_counter the current loop iteration
522 * \param depth_ptr pointer to the depth/stencil values of this 4x4 block
523 * \param depth_stride stride of the depth/stencil buffer
524 * \param z_fb contains z values loaded from fb (may include padding)
525 * \param s_fb contains s values loaded from fb (may include padding)
526 */
527 void
lp_build_depth_stencil_load_swizzled(struct gallivm_state * gallivm,struct lp_type z_src_type,const struct util_format_description * format_desc,boolean is_1d,LLVMValueRef depth_ptr,LLVMValueRef depth_stride,LLVMValueRef * z_fb,LLVMValueRef * s_fb,LLVMValueRef loop_counter)528 lp_build_depth_stencil_load_swizzled(struct gallivm_state *gallivm,
529 struct lp_type z_src_type,
530 const struct util_format_description *format_desc,
531 boolean is_1d,
532 LLVMValueRef depth_ptr,
533 LLVMValueRef depth_stride,
534 LLVMValueRef *z_fb,
535 LLVMValueRef *s_fb,
536 LLVMValueRef loop_counter)
537 {
538 LLVMBuilderRef builder = gallivm->builder;
539 LLVMValueRef shuffles[LP_MAX_VECTOR_LENGTH / 4];
540 LLVMValueRef zs_dst1, zs_dst2;
541 LLVMValueRef zs_dst_ptr;
542 LLVMValueRef depth_offset1, depth_offset2;
543 LLVMTypeRef load_ptr_type;
544 unsigned depth_bytes = format_desc->block.bits / 8;
545 struct lp_type zs_type = lp_depth_type(format_desc, z_src_type.length);
546 struct lp_type zs_load_type = zs_type;
547
548 zs_load_type.length = zs_load_type.length / 2;
549 load_ptr_type = LLVMPointerType(lp_build_vec_type(gallivm, zs_load_type), 0);
550
551 if (z_src_type.length == 4) {
552 unsigned i;
553 LLVMValueRef looplsb = LLVMBuildAnd(builder, loop_counter,
554 lp_build_const_int32(gallivm, 1), "");
555 LLVMValueRef loopmsb = LLVMBuildAnd(builder, loop_counter,
556 lp_build_const_int32(gallivm, 2), "");
557 LLVMValueRef offset2 = LLVMBuildMul(builder, loopmsb,
558 depth_stride, "");
559 depth_offset1 = LLVMBuildMul(builder, looplsb,
560 lp_build_const_int32(gallivm, depth_bytes * 2), "");
561 depth_offset1 = LLVMBuildAdd(builder, depth_offset1, offset2, "");
562
563 /* just concatenate the loaded 2x2 values into 4-wide vector */
564 for (i = 0; i < 4; i++) {
565 shuffles[i] = lp_build_const_int32(gallivm, i);
566 }
567 }
568 else {
569 unsigned i;
570 LLVMValueRef loopx2 = LLVMBuildShl(builder, loop_counter,
571 lp_build_const_int32(gallivm, 1), "");
572 assert(z_src_type.length == 8);
573 depth_offset1 = LLVMBuildMul(builder, loopx2, depth_stride, "");
574 /*
575 * We load 2x4 values, and need to swizzle them (order
576 * 0,1,4,5,2,3,6,7) - not so hot with avx unfortunately.
577 */
578 for (i = 0; i < 8; i++) {
579 shuffles[i] = lp_build_const_int32(gallivm, (i&1) + (i&2) * 2 + (i&4) / 2);
580 }
581 }
582
583 depth_offset2 = LLVMBuildAdd(builder, depth_offset1, depth_stride, "");
584
585 /* Load current z/stencil values from z/stencil buffer */
586 zs_dst_ptr = LLVMBuildGEP(builder, depth_ptr, &depth_offset1, 1, "");
587 zs_dst_ptr = LLVMBuildBitCast(builder, zs_dst_ptr, load_ptr_type, "");
588 zs_dst1 = LLVMBuildLoad(builder, zs_dst_ptr, "");
589 if (is_1d) {
590 zs_dst2 = lp_build_undef(gallivm, zs_load_type);
591 }
592 else {
593 zs_dst_ptr = LLVMBuildGEP(builder, depth_ptr, &depth_offset2, 1, "");
594 zs_dst_ptr = LLVMBuildBitCast(builder, zs_dst_ptr, load_ptr_type, "");
595 zs_dst2 = LLVMBuildLoad(builder, zs_dst_ptr, "");
596 }
597
598 *z_fb = LLVMBuildShuffleVector(builder, zs_dst1, zs_dst2,
599 LLVMConstVector(shuffles, zs_type.length), "");
600 *s_fb = *z_fb;
601
602 if (format_desc->block.bits < z_src_type.width) {
603 /* Extend destination ZS values (e.g., when reading from Z16_UNORM) */
604 *z_fb = LLVMBuildZExt(builder, *z_fb,
605 lp_build_int_vec_type(gallivm, z_src_type), "");
606 }
607
608 else if (format_desc->block.bits > 32) {
609 /* rely on llvm to handle too wide vector we have here nicely */
610 unsigned i;
611 struct lp_type typex2 = zs_type;
612 struct lp_type s_type = zs_type;
613 LLVMValueRef shuffles1[LP_MAX_VECTOR_LENGTH / 4];
614 LLVMValueRef shuffles2[LP_MAX_VECTOR_LENGTH / 4];
615 LLVMValueRef tmp;
616
617 typex2.width = typex2.width / 2;
618 typex2.length = typex2.length * 2;
619 s_type.width = s_type.width / 2;
620 s_type.floating = 0;
621
622 tmp = LLVMBuildBitCast(builder, *z_fb,
623 lp_build_vec_type(gallivm, typex2), "");
624
625 for (i = 0; i < zs_type.length; i++) {
626 shuffles1[i] = lp_build_const_int32(gallivm, i * 2);
627 shuffles2[i] = lp_build_const_int32(gallivm, i * 2 + 1);
628 }
629 *z_fb = LLVMBuildShuffleVector(builder, tmp, tmp,
630 LLVMConstVector(shuffles1, zs_type.length), "");
631 *s_fb = LLVMBuildShuffleVector(builder, tmp, tmp,
632 LLVMConstVector(shuffles2, zs_type.length), "");
633 *s_fb = LLVMBuildBitCast(builder, *s_fb,
634 lp_build_vec_type(gallivm, s_type), "");
635 lp_build_name(*s_fb, "s_dst");
636 }
637
638 lp_build_name(*z_fb, "z_dst");
639 lp_build_name(*s_fb, "s_dst");
640 lp_build_name(*z_fb, "z_dst");
641 }
642
643 /**
644 * Store depth/stencil values.
645 * Incoming values are swizzled (typically n 2x2 quads), stored linear.
646 * If there's a mask it will do select/store otherwise just store.
647 *
648 * \param type the data type of the fragment depth/stencil values
649 * \param format_desc description of the depth/stencil surface
650 * \param is_1d whether this resource has only one dimension
651 * \param mask the alive/dead pixel mask for the quad (vector)
652 * \param z_fb z values read from fb (with padding)
653 * \param s_fb s values read from fb (with padding)
654 * \param loop_counter the current loop iteration
655 * \param depth_ptr pointer to the depth/stencil values of this 4x4 block
656 * \param depth_stride stride of the depth/stencil buffer
657 * \param z_value the depth values to store (with padding)
658 * \param s_value the stencil values to store (with padding)
659 */
660 void
lp_build_depth_stencil_write_swizzled(struct gallivm_state * gallivm,struct lp_type z_src_type,const struct util_format_description * format_desc,boolean is_1d,struct lp_build_mask_context * mask,LLVMValueRef z_fb,LLVMValueRef s_fb,LLVMValueRef loop_counter,LLVMValueRef depth_ptr,LLVMValueRef depth_stride,LLVMValueRef z_value,LLVMValueRef s_value)661 lp_build_depth_stencil_write_swizzled(struct gallivm_state *gallivm,
662 struct lp_type z_src_type,
663 const struct util_format_description *format_desc,
664 boolean is_1d,
665 struct lp_build_mask_context *mask,
666 LLVMValueRef z_fb,
667 LLVMValueRef s_fb,
668 LLVMValueRef loop_counter,
669 LLVMValueRef depth_ptr,
670 LLVMValueRef depth_stride,
671 LLVMValueRef z_value,
672 LLVMValueRef s_value)
673 {
674 struct lp_build_context z_bld;
675 LLVMValueRef shuffles[LP_MAX_VECTOR_LENGTH / 4];
676 LLVMBuilderRef builder = gallivm->builder;
677 LLVMValueRef mask_value = NULL;
678 LLVMValueRef zs_dst1, zs_dst2;
679 LLVMValueRef zs_dst_ptr1, zs_dst_ptr2;
680 LLVMValueRef depth_offset1, depth_offset2;
681 LLVMTypeRef load_ptr_type;
682 unsigned depth_bytes = format_desc->block.bits / 8;
683 struct lp_type zs_type = lp_depth_type(format_desc, z_src_type.length);
684 struct lp_type z_type = zs_type;
685 struct lp_type zs_load_type = zs_type;
686
687 zs_load_type.length = zs_load_type.length / 2;
688 load_ptr_type = LLVMPointerType(lp_build_vec_type(gallivm, zs_load_type), 0);
689
690 z_type.width = z_src_type.width;
691
692 lp_build_context_init(&z_bld, gallivm, z_type);
693
694 /*
695 * This is far from ideal, at least for late depth write we should do this
696 * outside the fs loop to avoid all the swizzle stuff.
697 */
698 if (z_src_type.length == 4) {
699 LLVMValueRef looplsb = LLVMBuildAnd(builder, loop_counter,
700 lp_build_const_int32(gallivm, 1), "");
701 LLVMValueRef loopmsb = LLVMBuildAnd(builder, loop_counter,
702 lp_build_const_int32(gallivm, 2), "");
703 LLVMValueRef offset2 = LLVMBuildMul(builder, loopmsb,
704 depth_stride, "");
705 depth_offset1 = LLVMBuildMul(builder, looplsb,
706 lp_build_const_int32(gallivm, depth_bytes * 2), "");
707 depth_offset1 = LLVMBuildAdd(builder, depth_offset1, offset2, "");
708 }
709 else {
710 unsigned i;
711 LLVMValueRef loopx2 = LLVMBuildShl(builder, loop_counter,
712 lp_build_const_int32(gallivm, 1), "");
713 assert(z_src_type.length == 8);
714 depth_offset1 = LLVMBuildMul(builder, loopx2, depth_stride, "");
715 /*
716 * We load 2x4 values, and need to swizzle them (order
717 * 0,1,4,5,2,3,6,7) - not so hot with avx unfortunately.
718 */
719 for (i = 0; i < 8; i++) {
720 shuffles[i] = lp_build_const_int32(gallivm, (i&1) + (i&2) * 2 + (i&4) / 2);
721 }
722 }
723
724 depth_offset2 = LLVMBuildAdd(builder, depth_offset1, depth_stride, "");
725
726 zs_dst_ptr1 = LLVMBuildGEP(builder, depth_ptr, &depth_offset1, 1, "");
727 zs_dst_ptr1 = LLVMBuildBitCast(builder, zs_dst_ptr1, load_ptr_type, "");
728 zs_dst_ptr2 = LLVMBuildGEP(builder, depth_ptr, &depth_offset2, 1, "");
729 zs_dst_ptr2 = LLVMBuildBitCast(builder, zs_dst_ptr2, load_ptr_type, "");
730
731 if (format_desc->block.bits > 32) {
732 s_value = LLVMBuildBitCast(builder, s_value, z_bld.vec_type, "");
733 }
734
735 if (mask) {
736 mask_value = lp_build_mask_value(mask);
737 z_value = lp_build_select(&z_bld, mask_value, z_value, z_fb);
738 if (format_desc->block.bits > 32) {
739 s_fb = LLVMBuildBitCast(builder, s_fb, z_bld.vec_type, "");
740 s_value = lp_build_select(&z_bld, mask_value, s_value, s_fb);
741 }
742 }
743
744 if (zs_type.width < z_src_type.width) {
745 /* Truncate ZS values (e.g., when writing to Z16_UNORM) */
746 z_value = LLVMBuildTrunc(builder, z_value,
747 lp_build_int_vec_type(gallivm, zs_type), "");
748 }
749
750 if (format_desc->block.bits <= 32) {
751 if (z_src_type.length == 4) {
752 zs_dst1 = lp_build_extract_range(gallivm, z_value, 0, 2);
753 zs_dst2 = lp_build_extract_range(gallivm, z_value, 2, 2);
754 }
755 else {
756 assert(z_src_type.length == 8);
757 zs_dst1 = LLVMBuildShuffleVector(builder, z_value, z_value,
758 LLVMConstVector(&shuffles[0],
759 zs_load_type.length), "");
760 zs_dst2 = LLVMBuildShuffleVector(builder, z_value, z_value,
761 LLVMConstVector(&shuffles[4],
762 zs_load_type.length), "");
763 }
764 }
765 else {
766 if (z_src_type.length == 4) {
767 zs_dst1 = lp_build_interleave2(gallivm, z_type,
768 z_value, s_value, 0);
769 zs_dst2 = lp_build_interleave2(gallivm, z_type,
770 z_value, s_value, 1);
771 }
772 else {
773 unsigned i;
774 LLVMValueRef shuffles[LP_MAX_VECTOR_LENGTH / 2];
775 assert(z_src_type.length == 8);
776 for (i = 0; i < 8; i++) {
777 shuffles[i*2] = lp_build_const_int32(gallivm, (i&1) + (i&2) * 2 + (i&4) / 2);
778 shuffles[i*2+1] = lp_build_const_int32(gallivm, (i&1) + (i&2) * 2 + (i&4) / 2 +
779 z_src_type.length);
780 }
781 zs_dst1 = LLVMBuildShuffleVector(builder, z_value, s_value,
782 LLVMConstVector(&shuffles[0],
783 z_src_type.length), "");
784 zs_dst2 = LLVMBuildShuffleVector(builder, z_value, s_value,
785 LLVMConstVector(&shuffles[8],
786 z_src_type.length), "");
787 }
788 zs_dst1 = LLVMBuildBitCast(builder, zs_dst1,
789 lp_build_vec_type(gallivm, zs_load_type), "");
790 zs_dst2 = LLVMBuildBitCast(builder, zs_dst2,
791 lp_build_vec_type(gallivm, zs_load_type), "");
792 }
793
794 LLVMBuildStore(builder, zs_dst1, zs_dst_ptr1);
795 if (!is_1d) {
796 LLVMBuildStore(builder, zs_dst2, zs_dst_ptr2);
797 }
798 }
799
800 /**
801 * Generate code for performing depth and/or stencil tests.
802 * We operate on a vector of values (typically n 2x2 quads).
803 *
804 * \param depth the depth test state
805 * \param stencil the front/back stencil state
806 * \param type the data type of the fragment depth/stencil values
807 * \param format_desc description of the depth/stencil surface
808 * \param mask the alive/dead pixel mask for the quad (vector)
809 * \param stencil_refs the front/back stencil ref values (scalar)
810 * \param z_src the incoming depth/stencil values (n 2x2 quad values, float32)
811 * \param zs_dst the depth/stencil values in framebuffer
812 * \param face contains boolean value indicating front/back facing polygon
813 */
814 void
lp_build_depth_stencil_test(struct gallivm_state * gallivm,const struct pipe_depth_state * depth,const struct pipe_stencil_state stencil[2],struct lp_type z_src_type,const struct util_format_description * format_desc,struct lp_build_mask_context * mask,LLVMValueRef stencil_refs[2],LLVMValueRef z_src,LLVMValueRef z_fb,LLVMValueRef s_fb,LLVMValueRef face,LLVMValueRef * z_value,LLVMValueRef * s_value,boolean do_branch)815 lp_build_depth_stencil_test(struct gallivm_state *gallivm,
816 const struct pipe_depth_state *depth,
817 const struct pipe_stencil_state stencil[2],
818 struct lp_type z_src_type,
819 const struct util_format_description *format_desc,
820 struct lp_build_mask_context *mask,
821 LLVMValueRef stencil_refs[2],
822 LLVMValueRef z_src,
823 LLVMValueRef z_fb,
824 LLVMValueRef s_fb,
825 LLVMValueRef face,
826 LLVMValueRef *z_value,
827 LLVMValueRef *s_value,
828 boolean do_branch)
829 {
830 LLVMBuilderRef builder = gallivm->builder;
831 struct lp_type z_type;
832 struct lp_build_context z_bld;
833 struct lp_build_context s_bld;
834 struct lp_type s_type;
835 unsigned z_shift = 0, z_width = 0, z_mask = 0;
836 LLVMValueRef z_dst = NULL;
837 LLVMValueRef stencil_vals = NULL;
838 LLVMValueRef z_bitmask = NULL, stencil_shift = NULL;
839 LLVMValueRef z_pass = NULL, s_pass_mask = NULL;
840 LLVMValueRef current_mask = lp_build_mask_value(mask);
841 LLVMValueRef front_facing = NULL;
842 boolean have_z, have_s;
843
844 /*
845 * Depths are expected to be between 0 and 1, even if they are stored in
846 * floats. Setting these bits here will ensure that the lp_build_conv() call
847 * below won't try to unnecessarily clamp the incoming values.
848 */
849 if(z_src_type.floating) {
850 z_src_type.sign = FALSE;
851 z_src_type.norm = TRUE;
852 }
853 else {
854 assert(!z_src_type.sign);
855 assert(z_src_type.norm);
856 }
857
858 /* Pick the type matching the depth-stencil format. */
859 z_type = lp_depth_type(format_desc, z_src_type.length);
860
861 /* Pick the intermediate type for depth operations. */
862 z_type.width = z_src_type.width;
863 assert(z_type.length == z_src_type.length);
864
865 /* FIXME: for non-float depth/stencil might generate better code
866 * if we'd always split it up to use 128bit operations.
867 * For stencil we'd almost certainly want to pack to 8xi16 values,
868 * for z just run twice.
869 */
870
871 /* Sanity checking */
872 {
873 const unsigned z_swizzle = format_desc->swizzle[0];
874 const unsigned s_swizzle = format_desc->swizzle[1];
875
876 assert(z_swizzle != PIPE_SWIZZLE_NONE ||
877 s_swizzle != PIPE_SWIZZLE_NONE);
878
879 assert(depth->enabled || stencil[0].enabled);
880
881 assert(format_desc->colorspace == UTIL_FORMAT_COLORSPACE_ZS);
882 assert(format_desc->block.width == 1);
883 assert(format_desc->block.height == 1);
884
885 if (stencil[0].enabled) {
886 assert(s_swizzle < 4);
887 assert(format_desc->channel[s_swizzle].type == UTIL_FORMAT_TYPE_UNSIGNED);
888 assert(format_desc->channel[s_swizzle].pure_integer);
889 assert(!format_desc->channel[s_swizzle].normalized);
890 assert(format_desc->channel[s_swizzle].size == 8);
891 }
892
893 if (depth->enabled) {
894 assert(z_swizzle < 4);
895 if (z_type.floating) {
896 assert(z_swizzle == 0);
897 assert(format_desc->channel[z_swizzle].type ==
898 UTIL_FORMAT_TYPE_FLOAT);
899 assert(format_desc->channel[z_swizzle].size == 32);
900 }
901 else {
902 assert(format_desc->channel[z_swizzle].type ==
903 UTIL_FORMAT_TYPE_UNSIGNED);
904 assert(format_desc->channel[z_swizzle].normalized);
905 assert(!z_type.fixed);
906 }
907 }
908 }
909
910
911 /* Setup build context for Z vals */
912 lp_build_context_init(&z_bld, gallivm, z_type);
913
914 /* Setup build context for stencil vals */
915 s_type = lp_int_type(z_type);
916 lp_build_context_init(&s_bld, gallivm, s_type);
917
918 /* Compute and apply the Z/stencil bitmasks and shifts.
919 */
920 {
921 unsigned s_shift, s_mask;
922
923 z_dst = z_fb;
924 stencil_vals = s_fb;
925
926 have_z = get_z_shift_and_mask(format_desc, &z_shift, &z_width, &z_mask);
927 have_s = get_s_shift_and_mask(format_desc, &s_shift, &s_mask);
928
929 if (have_z) {
930 if (z_mask != 0xffffffff) {
931 z_bitmask = lp_build_const_int_vec(gallivm, z_type, z_mask);
932 }
933
934 /*
935 * Align the framebuffer Z 's LSB to the right.
936 */
937 if (z_shift) {
938 LLVMValueRef shift = lp_build_const_int_vec(gallivm, z_type, z_shift);
939 z_dst = LLVMBuildLShr(builder, z_dst, shift, "z_dst");
940 } else if (z_bitmask) {
941 z_dst = LLVMBuildAnd(builder, z_dst, z_bitmask, "z_dst");
942 } else {
943 lp_build_name(z_dst, "z_dst");
944 }
945 }
946
947 if (have_s) {
948 if (s_shift) {
949 LLVMValueRef shift = lp_build_const_int_vec(gallivm, s_type, s_shift);
950 stencil_vals = LLVMBuildLShr(builder, stencil_vals, shift, "");
951 stencil_shift = shift; /* used below */
952 }
953
954 if (s_mask != 0xffffffff) {
955 LLVMValueRef mask = lp_build_const_int_vec(gallivm, s_type, s_mask);
956 stencil_vals = LLVMBuildAnd(builder, stencil_vals, mask, "");
957 }
958
959 lp_build_name(stencil_vals, "s_dst");
960 }
961 }
962
963 if (stencil[0].enabled) {
964
965 if (face) {
966 if (0) {
967 /*
968 * XXX: the scalar expansion below produces atrocious code
969 * (basically producing a 64bit scalar value, then moving the 2
970 * 32bit pieces separately to simd, plus 4 shuffles, which is
971 * seriously lame). But the scalar-simd transitions are always
972 * tricky, so no big surprise there.
973 * This here would be way better, however llvm has some serious
974 * trouble later using it in the select, probably because it will
975 * recognize the expression as constant and move the simd value
976 * away (out of the loop) - and then it will suddenly try
977 * constructing i1 high-bit masks out of it later...
978 * (Try piglit stencil-twoside.)
979 * Note this is NOT due to using SExt/Trunc, it fails exactly the
980 * same even when using native compare/select.
981 * I cannot reproduce this problem when using stand-alone compiler
982 * though, suggesting some problem with optimization passes...
983 * (With stand-alone compilation, the construction of this mask
984 * value, no matter if the easy 3 instruction here or the complex
985 * 16+ one below, never gets separated from where it's used.)
986 * The scalar code still has the same problem, but the generated
987 * code looks a bit better at least for some reason, even if
988 * mostly by luck (the fundamental issue clearly is the same).
989 */
990 front_facing = lp_build_broadcast(gallivm, s_bld.vec_type, face);
991 /* front_facing = face != 0 ? ~0 : 0 */
992 front_facing = lp_build_compare(gallivm, s_bld.type,
993 PIPE_FUNC_NOTEQUAL,
994 front_facing, s_bld.zero);
995 } else {
996 LLVMValueRef zero = lp_build_const_int32(gallivm, 0);
997
998 /* front_facing = face != 0 ? ~0 : 0 */
999 front_facing = LLVMBuildICmp(builder, LLVMIntNE, face, zero, "");
1000 front_facing = LLVMBuildSExt(builder, front_facing,
1001 LLVMIntTypeInContext(gallivm->context,
1002 s_bld.type.length*s_bld.type.width),
1003 "");
1004 front_facing = LLVMBuildBitCast(builder, front_facing,
1005 s_bld.int_vec_type, "");
1006
1007 }
1008 }
1009
1010 s_pass_mask = lp_build_stencil_test(&s_bld, stencil,
1011 stencil_refs, stencil_vals,
1012 front_facing);
1013
1014 /* apply stencil-fail operator */
1015 {
1016 LLVMValueRef s_fail_mask = lp_build_andnot(&s_bld, current_mask, s_pass_mask);
1017 stencil_vals = lp_build_stencil_op(&s_bld, stencil, S_FAIL_OP,
1018 stencil_refs, stencil_vals,
1019 s_fail_mask, front_facing);
1020 }
1021 }
1022
1023 if (depth->enabled) {
1024 /*
1025 * Convert fragment Z to the desired type, aligning the LSB to the right.
1026 */
1027
1028 assert(z_type.width == z_src_type.width);
1029 assert(z_type.length == z_src_type.length);
1030 assert(lp_check_value(z_src_type, z_src));
1031 if (z_src_type.floating) {
1032 /*
1033 * Convert from floating point values
1034 */
1035
1036 if (!z_type.floating) {
1037 z_src = lp_build_clamped_float_to_unsigned_norm(gallivm,
1038 z_src_type,
1039 z_width,
1040 z_src);
1041 }
1042 } else {
1043 /*
1044 * Convert from unsigned normalized values.
1045 */
1046
1047 assert(!z_src_type.sign);
1048 assert(!z_src_type.fixed);
1049 assert(z_src_type.norm);
1050 assert(!z_type.floating);
1051 if (z_src_type.width > z_width) {
1052 LLVMValueRef shift = lp_build_const_int_vec(gallivm, z_src_type,
1053 z_src_type.width - z_width);
1054 z_src = LLVMBuildLShr(builder, z_src, shift, "");
1055 }
1056 }
1057 assert(lp_check_value(z_type, z_src));
1058
1059 lp_build_name(z_src, "z_src");
1060
1061 /* compare src Z to dst Z, returning 'pass' mask */
1062 z_pass = lp_build_cmp(&z_bld, depth->func, z_src, z_dst);
1063
1064 /* mask off bits that failed stencil test */
1065 if (s_pass_mask) {
1066 current_mask = LLVMBuildAnd(builder, current_mask, s_pass_mask, "");
1067 }
1068
1069 if (!stencil[0].enabled) {
1070 /* We can potentially skip all remaining operations here, but only
1071 * if stencil is disabled because we still need to update the stencil
1072 * buffer values. Don't need to update Z buffer values.
1073 */
1074 lp_build_mask_update(mask, z_pass);
1075
1076 if (do_branch) {
1077 lp_build_mask_check(mask);
1078 }
1079 }
1080
1081 if (depth->writemask) {
1082 LLVMValueRef z_pass_mask;
1083
1084 /* mask off bits that failed Z test */
1085 z_pass_mask = LLVMBuildAnd(builder, current_mask, z_pass, "");
1086
1087 /* Mix the old and new Z buffer values.
1088 * z_dst[i] = zselectmask[i] ? z_src[i] : z_dst[i]
1089 */
1090 z_dst = lp_build_select(&z_bld, z_pass_mask, z_src, z_dst);
1091 }
1092
1093 if (stencil[0].enabled) {
1094 /* update stencil buffer values according to z pass/fail result */
1095 LLVMValueRef z_fail_mask, z_pass_mask;
1096
1097 /* apply Z-fail operator */
1098 z_fail_mask = lp_build_andnot(&s_bld, current_mask, z_pass);
1099 stencil_vals = lp_build_stencil_op(&s_bld, stencil, Z_FAIL_OP,
1100 stencil_refs, stencil_vals,
1101 z_fail_mask, front_facing);
1102
1103 /* apply Z-pass operator */
1104 z_pass_mask = LLVMBuildAnd(builder, current_mask, z_pass, "");
1105 stencil_vals = lp_build_stencil_op(&s_bld, stencil, Z_PASS_OP,
1106 stencil_refs, stencil_vals,
1107 z_pass_mask, front_facing);
1108 }
1109 }
1110 else {
1111 /* No depth test: apply Z-pass operator to stencil buffer values which
1112 * passed the stencil test.
1113 */
1114 s_pass_mask = LLVMBuildAnd(builder, current_mask, s_pass_mask, "");
1115 stencil_vals = lp_build_stencil_op(&s_bld, stencil, Z_PASS_OP,
1116 stencil_refs, stencil_vals,
1117 s_pass_mask, front_facing);
1118 }
1119
1120 /* Put Z and stencil bits in the right place */
1121 if (have_z && z_shift) {
1122 LLVMValueRef shift = lp_build_const_int_vec(gallivm, z_type, z_shift);
1123 z_dst = LLVMBuildShl(builder, z_dst, shift, "");
1124 }
1125 if (stencil_vals && stencil_shift)
1126 stencil_vals = LLVMBuildShl(builder, stencil_vals,
1127 stencil_shift, "");
1128
1129 /* Finally, merge the z/stencil values */
1130 if (format_desc->block.bits <= 32) {
1131 if (have_z && have_s)
1132 *z_value = LLVMBuildOr(builder, z_dst, stencil_vals, "");
1133 else if (have_z)
1134 *z_value = z_dst;
1135 else
1136 *z_value = stencil_vals;
1137 *s_value = *z_value;
1138 }
1139 else {
1140 *z_value = z_dst;
1141 *s_value = stencil_vals;
1142 }
1143
1144 if (s_pass_mask)
1145 lp_build_mask_update(mask, s_pass_mask);
1146
1147 if (depth->enabled && stencil[0].enabled)
1148 lp_build_mask_update(mask, z_pass);
1149 }
1150
1151