• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /**************************************************************************
2  *
3  * Copyright 2009 VMware, Inc.
4  * All Rights Reserved.
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a
7  * copy of this software and associated documentation files (the
8  * "Software"), to deal in the Software without restriction, including
9  * without limitation the rights to use, copy, modify, merge, publish,
10  * distribute, sub license, and/or sell copies of the Software, and to
11  * permit persons to whom the Software is furnished to do so, subject to
12  * the following conditions:
13  *
14  * The above copyright notice and this permission notice (including the
15  * next paragraph) shall be included in all copies or substantial portions
16  * of the Software.
17  *
18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
19  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
21  * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
22  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
23  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
24  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25  *
26  **************************************************************************/
27 
28 /**
29  * @file
30  * Texture sampling -- SoA.
31  *
32  * @author Jose Fonseca <jfonseca@vmware.com>
33  * @author Brian Paul <brianp@vmware.com>
34  */
35 
36 #include "pipe/p_defines.h"
37 #include "pipe/p_state.h"
38 #include "pipe/p_shader_tokens.h"
39 #include "util/u_debug.h"
40 #include "util/u_dump.h"
41 #include "util/u_memory.h"
42 #include "util/u_math.h"
43 #include "util/format/u_format.h"
44 #include "util/u_cpu_detect.h"
45 #include "util/format_rgb9e5.h"
46 #include "lp_bld_debug.h"
47 #include "lp_bld_type.h"
48 #include "lp_bld_const.h"
49 #include "lp_bld_conv.h"
50 #include "lp_bld_arit.h"
51 #include "lp_bld_bitarit.h"
52 #include "lp_bld_logic.h"
53 #include "lp_bld_printf.h"
54 #include "lp_bld_swizzle.h"
55 #include "lp_bld_flow.h"
56 #include "lp_bld_gather.h"
57 #include "lp_bld_format.h"
58 #include "lp_bld_sample.h"
59 #include "lp_bld_sample_aos.h"
60 #include "lp_bld_struct.h"
61 #include "lp_bld_quad.h"
62 #include "lp_bld_pack.h"
63 #include "lp_bld_intr.h"
64 #include "lp_bld_misc.h"
65 
66 
67 /**
68  * Generate code to fetch a texel from a texture at int coords (x, y, z).
69  * The computation depends on whether the texture is 1D, 2D or 3D.
70  * The result, texel, will be float vectors:
71  *   texel[0] = red values
72  *   texel[1] = green values
73  *   texel[2] = blue values
74  *   texel[3] = alpha values
75  */
76 static void
lp_build_sample_texel_soa(struct lp_build_sample_context * bld,LLVMValueRef width,LLVMValueRef height,LLVMValueRef depth,LLVMValueRef x,LLVMValueRef y,LLVMValueRef z,LLVMValueRef y_stride,LLVMValueRef z_stride,LLVMValueRef data_ptr,LLVMValueRef mipoffsets,LLVMValueRef texel_out[4])77 lp_build_sample_texel_soa(struct lp_build_sample_context *bld,
78                           LLVMValueRef width,
79                           LLVMValueRef height,
80                           LLVMValueRef depth,
81                           LLVMValueRef x,
82                           LLVMValueRef y,
83                           LLVMValueRef z,
84                           LLVMValueRef y_stride,
85                           LLVMValueRef z_stride,
86                           LLVMValueRef data_ptr,
87                           LLVMValueRef mipoffsets,
88                           LLVMValueRef texel_out[4])
89 {
90    const struct lp_static_sampler_state *static_state = bld->static_sampler_state;
91    const unsigned dims = bld->dims;
92    struct lp_build_context *int_coord_bld = &bld->int_coord_bld;
93    LLVMBuilderRef builder = bld->gallivm->builder;
94    LLVMValueRef offset;
95    LLVMValueRef i, j;
96    LLVMValueRef use_border = NULL;
97 
98    /* use_border = x < 0 || x >= width || y < 0 || y >= height */
99    if (lp_sampler_wrap_mode_uses_border_color(static_state->wrap_s,
100                                               static_state->min_img_filter,
101                                               static_state->mag_img_filter)) {
102       LLVMValueRef b1, b2;
103       b1 = lp_build_cmp(int_coord_bld, PIPE_FUNC_LESS, x, int_coord_bld->zero);
104       b2 = lp_build_cmp(int_coord_bld, PIPE_FUNC_GEQUAL, x, width);
105       use_border = LLVMBuildOr(builder, b1, b2, "b1_or_b2");
106    }
107 
108    if (dims >= 2 &&
109        lp_sampler_wrap_mode_uses_border_color(static_state->wrap_t,
110                                               static_state->min_img_filter,
111                                               static_state->mag_img_filter)) {
112       LLVMValueRef b1, b2;
113       b1 = lp_build_cmp(int_coord_bld, PIPE_FUNC_LESS, y, int_coord_bld->zero);
114       b2 = lp_build_cmp(int_coord_bld, PIPE_FUNC_GEQUAL, y, height);
115       if (use_border) {
116          use_border = LLVMBuildOr(builder, use_border, b1, "ub_or_b1");
117          use_border = LLVMBuildOr(builder, use_border, b2, "ub_or_b2");
118       }
119       else {
120          use_border = LLVMBuildOr(builder, b1, b2, "b1_or_b2");
121       }
122    }
123 
124    if (dims == 3 &&
125        lp_sampler_wrap_mode_uses_border_color(static_state->wrap_r,
126                                               static_state->min_img_filter,
127                                               static_state->mag_img_filter)) {
128       LLVMValueRef b1, b2;
129       b1 = lp_build_cmp(int_coord_bld, PIPE_FUNC_LESS, z, int_coord_bld->zero);
130       b2 = lp_build_cmp(int_coord_bld, PIPE_FUNC_GEQUAL, z, depth);
131       if (use_border) {
132          use_border = LLVMBuildOr(builder, use_border, b1, "ub_or_b1");
133          use_border = LLVMBuildOr(builder, use_border, b2, "ub_or_b2");
134       }
135       else {
136          use_border = LLVMBuildOr(builder, b1, b2, "b1_or_b2");
137       }
138    }
139 
140    /* convert x,y,z coords to linear offset from start of texture, in bytes */
141    lp_build_sample_offset(&bld->int_coord_bld,
142                           bld->format_desc,
143                           x, y, z, y_stride, z_stride,
144                           &offset, &i, &j);
145    if (mipoffsets) {
146       offset = lp_build_add(&bld->int_coord_bld, offset, mipoffsets);
147    }
148 
149    if (use_border) {
150       /* If we can sample the border color, it means that texcoords may
151        * lie outside the bounds of the texture image.  We need to do
152        * something to prevent reading out of bounds and causing a segfault.
153        *
154        * Simply AND the texture coords with !use_border.  This will cause
155        * coords which are out of bounds to become zero.  Zero's guaranteed
156        * to be inside the texture image.
157        */
158       offset = lp_build_andnot(&bld->int_coord_bld, offset, use_border);
159    }
160 
161    lp_build_fetch_rgba_soa(bld->gallivm,
162                            bld->format_desc,
163                            bld->texel_type, TRUE,
164                            data_ptr, offset,
165                            i, j,
166                            bld->cache,
167                            texel_out);
168 
169    /*
170     * Note: if we find an app which frequently samples the texture border
171     * we might want to implement a true conditional here to avoid sampling
172     * the texture whenever possible (since that's quite a bit of code).
173     * Ex:
174     *   if (use_border) {
175     *      texel = border_color;
176     *   }
177     *   else {
178     *      texel = sample_texture(coord);
179     *   }
180     * As it is now, we always sample the texture, then selectively replace
181     * the texel color results with the border color.
182     */
183 
184    if (use_border) {
185       /* select texel color or border color depending on use_border. */
186       const struct util_format_description *format_desc = bld->format_desc;
187       int chan;
188       struct lp_type border_type = bld->texel_type;
189       border_type.length = 4;
190       /*
191        * Only replace channels which are actually present. The others should
192        * get optimized away eventually by sampler_view swizzle anyway but it's
193        * easier too.
194        */
195       for (chan = 0; chan < 4; chan++) {
196          unsigned chan_s;
197          /* reverse-map channel... */
198          if (util_format_has_stencil(format_desc)) {
199             if (chan == 0)
200                chan_s = 0;
201             else
202                break;
203          }
204          else {
205             for (chan_s = 0; chan_s < 4; chan_s++) {
206                if (chan_s == format_desc->swizzle[chan]) {
207                   break;
208                }
209             }
210          }
211          if (chan_s <= 3) {
212             /* use the already clamped color */
213             LLVMValueRef idx = lp_build_const_int32(bld->gallivm, chan);
214             LLVMValueRef border_chan;
215 
216             border_chan = lp_build_extract_broadcast(bld->gallivm,
217                                                      border_type,
218                                                      bld->texel_type,
219                                                      bld->border_color_clamped,
220                                                      idx);
221             texel_out[chan] = lp_build_select(&bld->texel_bld, use_border,
222                                               border_chan, texel_out[chan]);
223          }
224       }
225    }
226 }
227 
228 
229 /**
230  * Helper to compute the mirror function for the PIPE_WRAP_MIRROR_REPEAT mode.
231  * (Note that with pot sizes could do this much more easily post-scale
232  * with some bit arithmetic.)
233  */
234 static LLVMValueRef
lp_build_coord_mirror(struct lp_build_sample_context * bld,LLVMValueRef coord,boolean posOnly)235 lp_build_coord_mirror(struct lp_build_sample_context *bld,
236                       LLVMValueRef coord, boolean posOnly)
237 {
238    struct lp_build_context *coord_bld = &bld->coord_bld;
239    LLVMValueRef fract;
240    LLVMValueRef half = lp_build_const_vec(bld->gallivm, coord_bld->type, 0.5);
241 
242    /*
243     * We can just use 2*(x - round(0.5*x)) to do all the mirroring,
244     * it all works out. (The result is in range [-1, 1.0], negative if
245     * the coord is in the "odd" section, otherwise positive.)
246     */
247 
248    coord = lp_build_mul(coord_bld, coord, half);
249    fract = lp_build_round(coord_bld, coord);
250    fract = lp_build_sub(coord_bld, coord, fract);
251    coord = lp_build_add(coord_bld, fract, fract);
252 
253    if (posOnly) {
254       /*
255        * Theoretically it's not quite 100% accurate because the spec says
256        * that ultimately a scaled coord of -x.0 should map to int coord
257        * -x + 1 with mirroring, not -x (this does not matter for bilinear
258        * filtering).
259        */
260       coord = lp_build_abs(coord_bld, coord);
261       /* kill off NaNs */
262       /* XXX: not safe without arch rounding, fract can be anything. */
263       coord = lp_build_max_ext(coord_bld, coord, coord_bld->zero,
264                                GALLIVM_NAN_RETURN_OTHER_SECOND_NONNAN);
265    }
266 
267    return coord;
268 }
269 
270 
271 /**
272  * Helper to compute the first coord and the weight for
273  * linear wrap repeat npot textures
274  */
275 void
lp_build_coord_repeat_npot_linear(struct lp_build_sample_context * bld,LLVMValueRef coord_f,LLVMValueRef length_i,LLVMValueRef length_f,LLVMValueRef * coord0_i,LLVMValueRef * weight_f)276 lp_build_coord_repeat_npot_linear(struct lp_build_sample_context *bld,
277                                   LLVMValueRef coord_f,
278                                   LLVMValueRef length_i,
279                                   LLVMValueRef length_f,
280                                   LLVMValueRef *coord0_i,
281                                   LLVMValueRef *weight_f)
282 {
283    struct lp_build_context *coord_bld = &bld->coord_bld;
284    struct lp_build_context *int_coord_bld = &bld->int_coord_bld;
285    LLVMValueRef half = lp_build_const_vec(bld->gallivm, coord_bld->type, 0.5);
286    LLVMValueRef length_minus_one = lp_build_sub(int_coord_bld, length_i,
287                                                 int_coord_bld->one);
288    LLVMValueRef mask;
289    /* wrap with normalized floats is just fract */
290    coord_f = lp_build_fract(coord_bld, coord_f);
291    /* mul by size and subtract 0.5 */
292    coord_f = lp_build_mul(coord_bld, coord_f, length_f);
293    coord_f = lp_build_sub(coord_bld, coord_f, half);
294    /*
295     * we avoided the 0.5/length division before the repeat wrap,
296     * now need to fix up edge cases with selects
297     */
298    /*
299     * Note we do a float (unordered) compare so we can eliminate NaNs.
300     * (Otherwise would need fract_safe above).
301     */
302    mask = lp_build_compare(coord_bld->gallivm, coord_bld->type,
303                            PIPE_FUNC_LESS, coord_f, coord_bld->zero);
304 
305    /* convert to int, compute lerp weight */
306    lp_build_ifloor_fract(coord_bld, coord_f, coord0_i, weight_f);
307    *coord0_i = lp_build_select(int_coord_bld, mask, length_minus_one, *coord0_i);
308 }
309 
310 
311 /**
312  * Build LLVM code for texture wrap mode for linear filtering.
313  * \param x0_out  returns first integer texcoord
314  * \param x1_out  returns second integer texcoord
315  * \param weight_out  returns linear interpolation weight
316  */
317 static void
lp_build_sample_wrap_linear(struct lp_build_sample_context * bld,boolean is_gather,LLVMValueRef coord,LLVMValueRef length,LLVMValueRef length_f,LLVMValueRef offset,boolean is_pot,unsigned wrap_mode,LLVMValueRef * x0_out,LLVMValueRef * x1_out,LLVMValueRef * weight_out)318 lp_build_sample_wrap_linear(struct lp_build_sample_context *bld,
319                             boolean is_gather,
320                             LLVMValueRef coord,
321                             LLVMValueRef length,
322                             LLVMValueRef length_f,
323                             LLVMValueRef offset,
324                             boolean is_pot,
325                             unsigned wrap_mode,
326                             LLVMValueRef *x0_out,
327                             LLVMValueRef *x1_out,
328                             LLVMValueRef *weight_out)
329 {
330    struct lp_build_context *coord_bld = &bld->coord_bld;
331    struct lp_build_context *int_coord_bld = &bld->int_coord_bld;
332    LLVMBuilderRef builder = bld->gallivm->builder;
333    LLVMValueRef half = lp_build_const_vec(bld->gallivm, coord_bld->type, 0.5);
334    LLVMValueRef length_minus_one = lp_build_sub(int_coord_bld, length, int_coord_bld->one);
335    LLVMValueRef coord0, coord1, weight;
336 
337    switch(wrap_mode) {
338    case PIPE_TEX_WRAP_REPEAT:
339       if (is_pot) {
340          /* mul by size and subtract 0.5 */
341          coord = lp_build_mul(coord_bld, coord, length_f);
342          coord = lp_build_sub(coord_bld, coord, half);
343          if (offset) {
344             offset = lp_build_int_to_float(coord_bld, offset);
345             coord = lp_build_add(coord_bld, coord, offset);
346          }
347          /* convert to int, compute lerp weight */
348          lp_build_ifloor_fract(coord_bld, coord, &coord0, &weight);
349          coord1 = lp_build_add(int_coord_bld, coord0, int_coord_bld->one);
350          /* repeat wrap */
351          coord0 = LLVMBuildAnd(builder, coord0, length_minus_one, "");
352          coord1 = LLVMBuildAnd(builder, coord1, length_minus_one, "");
353       }
354       else {
355          LLVMValueRef mask;
356          if (offset) {
357             offset = lp_build_int_to_float(coord_bld, offset);
358             offset = lp_build_div(coord_bld, offset, length_f);
359             coord = lp_build_add(coord_bld, coord, offset);
360          }
361          lp_build_coord_repeat_npot_linear(bld, coord,
362                                            length, length_f,
363                                            &coord0, &weight);
364          mask = lp_build_compare(int_coord_bld->gallivm, int_coord_bld->type,
365                                  PIPE_FUNC_NOTEQUAL, coord0, length_minus_one);
366          coord1 = LLVMBuildAnd(builder,
367                                lp_build_add(int_coord_bld, coord0, int_coord_bld->one),
368                                mask, "");
369       }
370       break;
371 
372    case PIPE_TEX_WRAP_CLAMP:
373       if (bld->static_sampler_state->normalized_coords) {
374          /* scale coord to length */
375          coord = lp_build_mul(coord_bld, coord, length_f);
376       }
377       if (offset) {
378          offset = lp_build_int_to_float(coord_bld, offset);
379          coord = lp_build_add(coord_bld, coord, offset);
380       }
381 
382       /*
383        * clamp to [0, length]
384        *
385        * Unlike some other wrap modes, this should be correct for gather
386        * too. GL_CLAMP explicitly does this clamp on the coord prior to
387        * actual wrapping (which is per sample).
388        */
389       coord = lp_build_clamp(coord_bld, coord, coord_bld->zero, length_f);
390 
391       coord = lp_build_sub(coord_bld, coord, half);
392 
393       /* convert to int, compute lerp weight */
394       lp_build_ifloor_fract(coord_bld, coord, &coord0, &weight);
395       coord1 = lp_build_add(int_coord_bld, coord0, int_coord_bld->one);
396       break;
397 
398    case PIPE_TEX_WRAP_CLAMP_TO_EDGE:
399       {
400          struct lp_build_context abs_coord_bld = bld->coord_bld;
401          abs_coord_bld.type.sign = FALSE;
402 
403          if (bld->static_sampler_state->normalized_coords) {
404             /* mul by tex size */
405             coord = lp_build_mul(coord_bld, coord, length_f);
406          }
407          if (offset) {
408             offset = lp_build_int_to_float(coord_bld, offset);
409             coord = lp_build_add(coord_bld, coord, offset);
410          }
411 
412          /* clamp to length max */
413          coord = lp_build_min_ext(coord_bld, coord, length_f,
414                                   GALLIVM_NAN_RETURN_OTHER_SECOND_NONNAN);
415          if (!is_gather) {
416             /* subtract 0.5 */
417             coord = lp_build_sub(coord_bld, coord, half);
418             /* clamp to [0, length - 0.5] */
419             coord = lp_build_max(coord_bld, coord, coord_bld->zero);
420             /* convert to int, compute lerp weight */
421             lp_build_ifloor_fract(&abs_coord_bld, coord, &coord0, &weight);
422             coord1 = lp_build_add(int_coord_bld, coord0, int_coord_bld->one);
423          } else {
424             /*
425              * The non-gather path will end up with coords 0, 1 if coord was
426              * smaller than 0.5 (with corresponding weight 0.0 so it doesn't
427              * really matter what the second coord is). But for gather, we
428              * really need to end up with coords 0, 0.
429              */
430             coord = lp_build_max(coord_bld, coord, coord_bld->zero);
431             coord0 = lp_build_sub(coord_bld, coord, half);
432             coord1 = lp_build_add(coord_bld, coord, half);
433             /* Values range ([-0.5, length_f - 0.5], [0.5, length_f + 0.5] */
434             coord0 = lp_build_itrunc(coord_bld, coord0);
435             coord1 = lp_build_itrunc(coord_bld, coord1);
436             weight = coord_bld->undef;
437          }
438          /* coord1 = min(coord1, length-1) */
439          coord1 = lp_build_min(int_coord_bld, coord1, length_minus_one);
440          break;
441       }
442 
443    case PIPE_TEX_WRAP_CLAMP_TO_BORDER:
444       if (bld->static_sampler_state->normalized_coords) {
445          /* scale coord to length */
446          coord = lp_build_mul(coord_bld, coord, length_f);
447       }
448       if (offset) {
449          offset = lp_build_int_to_float(coord_bld, offset);
450          coord = lp_build_add(coord_bld, coord, offset);
451       }
452       /*
453        * We don't need any clamp. Technically, for very large (pos or neg)
454        * (or infinite) values, clamp against [-length, length] would be
455        * correct, but we don't need to guarantee any specific
456        * result for such coords (the ifloor will be undefined, but for modes
457        * requiring border all resulting coords are safe).
458        */
459       coord = lp_build_sub(coord_bld, coord, half);
460       /* convert to int, compute lerp weight */
461       lp_build_ifloor_fract(coord_bld, coord, &coord0, &weight);
462       coord1 = lp_build_add(int_coord_bld, coord0, int_coord_bld->one);
463       break;
464 
465    case PIPE_TEX_WRAP_MIRROR_REPEAT:
466       if (offset) {
467          offset = lp_build_int_to_float(coord_bld, offset);
468          offset = lp_build_div(coord_bld, offset, length_f);
469          coord = lp_build_add(coord_bld, coord, offset);
470       }
471       if (!is_gather) {
472          /* compute mirror function */
473          coord = lp_build_coord_mirror(bld, coord, TRUE);
474 
475          /* scale coord to length */
476          coord = lp_build_mul(coord_bld, coord, length_f);
477          coord = lp_build_sub(coord_bld, coord, half);
478 
479          /* convert to int, compute lerp weight */
480          lp_build_ifloor_fract(coord_bld, coord, &coord0, &weight);
481          coord1 = lp_build_add(int_coord_bld, coord0, int_coord_bld->one);
482 
483          /* coord0 = max(coord0, 0) */
484          coord0 = lp_build_max(int_coord_bld, coord0, int_coord_bld->zero);
485          /* coord1 = min(coord1, length-1) */
486          coord1 = lp_build_min(int_coord_bld, coord1, length_minus_one);
487       } else {
488          /*
489           * This is pretty reasonable in the end,  all what the tests care
490           * about is nasty edge cases (scaled coords x.5, so the individual
491           * coords are actually integers, which is REALLY tricky to get right
492           * due to this working differently both for negative numbers as well
493           * as for even/odd cases). But with enough magic it's not too complex
494           * after all.
495           * Maybe should try a bit arithmetic one though for POT textures...
496           */
497          LLVMValueRef isNeg;
498          /*
499           * Wrapping just once still works, even though it means we can
500           * get "wrong" sign due to performing mirror in the middle of the
501           * two coords (because this can only happen very near the odd/even
502           * edges, so both coords will actually end up as 0 or length - 1
503           * in the end).
504           * For GL4 gather with per-sample offsets we'd need to the mirroring
505           * per coord too.
506           */
507          coord = lp_build_coord_mirror(bld, coord, FALSE);
508          coord = lp_build_mul(coord_bld, coord, length_f);
509 
510          /*
511           * NaNs should be safe here, we'll do away with them with
512           * the ones' complement plus min.
513           */
514          coord0 = lp_build_sub(coord_bld, coord, half);
515          coord0 = lp_build_ifloor(coord_bld, coord0);
516          coord1 = lp_build_add(int_coord_bld, coord0, int_coord_bld->one);
517          /* ones complement for neg numbers (mirror(negX) = X - 1)  */
518          isNeg = lp_build_cmp(int_coord_bld, PIPE_FUNC_LESS,
519                               coord0, int_coord_bld->zero);
520          coord0 = lp_build_xor(int_coord_bld, coord0, isNeg);
521          isNeg = lp_build_cmp(int_coord_bld, PIPE_FUNC_LESS,
522                               coord1, int_coord_bld->zero);
523          coord1 = lp_build_xor(int_coord_bld, coord1, isNeg);
524          coord0 = lp_build_min(int_coord_bld, coord0, length_minus_one);
525          coord1 = lp_build_min(int_coord_bld, coord1, length_minus_one);
526 
527          weight = coord_bld->undef;
528       }
529       break;
530 
531    case PIPE_TEX_WRAP_MIRROR_CLAMP:
532       if (bld->static_sampler_state->normalized_coords) {
533          /* scale coord to length */
534          coord = lp_build_mul(coord_bld, coord, length_f);
535       }
536       if (offset) {
537          offset = lp_build_int_to_float(coord_bld, offset);
538          coord = lp_build_add(coord_bld, coord, offset);
539       }
540       /*
541        * XXX: probably not correct for gather, albeit I'm not
542        * entirely sure as it's poorly specified. The wrapping looks
543        * correct according to the spec which is against gl 1.2.1,
544        * however negative values will be swapped - gl re-specified
545        * wrapping with newer versions (no more pre-clamp except with
546        * GL_CLAMP).
547        */
548       coord = lp_build_abs(coord_bld, coord);
549 
550       /* clamp to [0, length] */
551       coord = lp_build_min_ext(coord_bld, coord, length_f,
552                                GALLIVM_NAN_RETURN_OTHER_SECOND_NONNAN);
553 
554       coord = lp_build_sub(coord_bld, coord, half);
555 
556       /* convert to int, compute lerp weight */
557       lp_build_ifloor_fract(coord_bld, coord, &coord0, &weight);
558       coord1 = lp_build_add(int_coord_bld, coord0, int_coord_bld->one);
559       break;
560 
561    case PIPE_TEX_WRAP_MIRROR_CLAMP_TO_EDGE:
562       {
563          struct lp_build_context abs_coord_bld = bld->coord_bld;
564          abs_coord_bld.type.sign = FALSE;
565 
566          if (bld->static_sampler_state->normalized_coords) {
567             /* scale coord to length */
568             coord = lp_build_mul(coord_bld, coord, length_f);
569          }
570          if (offset) {
571             offset = lp_build_int_to_float(coord_bld, offset);
572             coord = lp_build_add(coord_bld, coord, offset);
573          }
574          if (!is_gather) {
575             coord = lp_build_abs(coord_bld, coord);
576 
577             /* clamp to length max */
578             coord = lp_build_min_ext(coord_bld, coord, length_f,
579                                      GALLIVM_NAN_RETURN_OTHER_SECOND_NONNAN);
580             /* subtract 0.5 */
581             coord = lp_build_sub(coord_bld, coord, half);
582             /* clamp to [0, length - 0.5] */
583             coord = lp_build_max(coord_bld, coord, coord_bld->zero);
584 
585             /* convert to int, compute lerp weight */
586             lp_build_ifloor_fract(&abs_coord_bld, coord, &coord0, &weight);
587             coord1 = lp_build_add(int_coord_bld, coord0, int_coord_bld->one);
588             /* coord1 = min(coord1, length-1) */
589             coord1 = lp_build_min(int_coord_bld, coord1, length_minus_one);
590          } else {
591             /*
592              * The non-gather path will swap coord0/1 if coord was negative,
593              * which is ok for filtering since the filter weight matches
594              * accordingly. Also, if coord is close to zero, coord0/1 will
595              * be 0 and 1, instead of 0 and 0 (again ok due to filter
596              * weight being 0.0). Both issues need to be fixed for gather.
597              */
598             LLVMValueRef isNeg;
599 
600             /*
601              * Actually wanted to cheat here and use:
602              * coord1 = lp_build_iround(coord_bld, coord);
603              * but it's not good enough for some tests (even piglit
604              * textureGather is set up in a way so the coords area always
605              * .5, that is right at the crossover points).
606              * So do ordinary sub/floor, then do ones' complement
607              * for negative numbers.
608              * (Note can't just do sub|add/abs/itrunc per coord neither -
609              * because the spec demands that mirror(3.0) = 3 but
610              * mirror(-3.0) = 2.)
611              */
612             coord = lp_build_sub(coord_bld, coord, half);
613             coord0 = lp_build_ifloor(coord_bld, coord);
614             coord1 = lp_build_add(int_coord_bld, coord0, int_coord_bld->one);
615             isNeg = lp_build_cmp(int_coord_bld, PIPE_FUNC_LESS, coord0,
616                                  int_coord_bld->zero);
617             coord0 = lp_build_xor(int_coord_bld, isNeg, coord0);
618             coord0 = lp_build_min(int_coord_bld, coord0, length_minus_one);
619 
620             isNeg = lp_build_cmp(int_coord_bld, PIPE_FUNC_LESS, coord1,
621                                  int_coord_bld->zero);
622             coord1 = lp_build_xor(int_coord_bld, isNeg, coord1);
623             coord1 = lp_build_min(int_coord_bld, coord1, length_minus_one);
624 
625             weight = coord_bld->undef;
626          }
627       }
628       break;
629 
630    case PIPE_TEX_WRAP_MIRROR_CLAMP_TO_BORDER:
631       {
632          if (bld->static_sampler_state->normalized_coords) {
633             /* scale coord to length */
634             coord = lp_build_mul(coord_bld, coord, length_f);
635          }
636          if (offset) {
637             offset = lp_build_int_to_float(coord_bld, offset);
638             coord = lp_build_add(coord_bld, coord, offset);
639          }
640          /*
641           * XXX: probably not correct for gather due to swapped
642           * order if coord is negative (same rationale as for
643           * MIRROR_CLAMP).
644           */
645          coord = lp_build_abs(coord_bld, coord);
646 
647          /*
648           * We don't need any clamp. Technically, for very large
649           * (or infinite) values, clamp against length would be
650           * correct, but we don't need to guarantee any specific
651           * result for such coords (the ifloor will be undefined, but
652           * for modes requiring border all resulting coords are safe).
653           */
654          coord = lp_build_sub(coord_bld, coord, half);
655 
656          /* convert to int, compute lerp weight */
657          lp_build_ifloor_fract(coord_bld, coord, &coord0, &weight);
658          coord1 = lp_build_add(int_coord_bld, coord0, int_coord_bld->one);
659       }
660       break;
661 
662    default:
663       assert(0);
664       coord0 = NULL;
665       coord1 = NULL;
666       weight = NULL;
667    }
668 
669    *x0_out = coord0;
670    *x1_out = coord1;
671    *weight_out = weight;
672 }
673 
674 
675 /**
676  * Build LLVM code for texture wrap mode for nearest filtering.
677  * \param coord  the incoming texcoord (nominally in [0,1])
678  * \param length  the texture size along one dimension, as int vector
679  * \param length_f  the texture size along one dimension, as float vector
680  * \param offset  texel offset along one dimension (as int vector)
681  * \param is_pot  if TRUE, length is a power of two
682  * \param wrap_mode  one of PIPE_TEX_WRAP_x
683  */
684 static LLVMValueRef
lp_build_sample_wrap_nearest(struct lp_build_sample_context * bld,LLVMValueRef coord,LLVMValueRef length,LLVMValueRef length_f,LLVMValueRef offset,boolean is_pot,unsigned wrap_mode)685 lp_build_sample_wrap_nearest(struct lp_build_sample_context *bld,
686                              LLVMValueRef coord,
687                              LLVMValueRef length,
688                              LLVMValueRef length_f,
689                              LLVMValueRef offset,
690                              boolean is_pot,
691                              unsigned wrap_mode)
692 {
693    struct lp_build_context *coord_bld = &bld->coord_bld;
694    struct lp_build_context *int_coord_bld = &bld->int_coord_bld;
695    LLVMBuilderRef builder = bld->gallivm->builder;
696    LLVMValueRef length_minus_one = lp_build_sub(int_coord_bld, length, int_coord_bld->one);
697    LLVMValueRef icoord;
698 
699    switch(wrap_mode) {
700    case PIPE_TEX_WRAP_REPEAT:
701       if (is_pot) {
702          coord = lp_build_mul(coord_bld, coord, length_f);
703          icoord = lp_build_ifloor(coord_bld, coord);
704          if (offset) {
705             icoord = lp_build_add(int_coord_bld, icoord, offset);
706          }
707          icoord = LLVMBuildAnd(builder, icoord, length_minus_one, "");
708       }
709       else {
710           if (offset) {
711              offset = lp_build_int_to_float(coord_bld, offset);
712              offset = lp_build_div(coord_bld, offset, length_f);
713              coord = lp_build_add(coord_bld, coord, offset);
714           }
715           /* take fraction, unnormalize */
716           coord = lp_build_fract_safe(coord_bld, coord);
717           coord = lp_build_mul(coord_bld, coord, length_f);
718           icoord = lp_build_itrunc(coord_bld, coord);
719       }
720       break;
721 
722    case PIPE_TEX_WRAP_CLAMP:
723    case PIPE_TEX_WRAP_CLAMP_TO_EDGE:
724       if (bld->static_sampler_state->normalized_coords) {
725          /* scale coord to length */
726          coord = lp_build_mul(coord_bld, coord, length_f);
727       }
728 
729       if (offset) {
730          offset = lp_build_int_to_float(coord_bld, offset);
731          coord = lp_build_add(coord_bld, coord, offset);
732       }
733       /* floor */
734       /* use itrunc instead since we clamp to 0 anyway */
735       icoord = lp_build_itrunc(coord_bld, coord);
736 
737       /* clamp to [0, length - 1]. */
738       icoord = lp_build_clamp(int_coord_bld, icoord, int_coord_bld->zero,
739                               length_minus_one);
740       break;
741 
742    case PIPE_TEX_WRAP_CLAMP_TO_BORDER:
743       if (bld->static_sampler_state->normalized_coords) {
744          /* scale coord to length */
745          coord = lp_build_mul(coord_bld, coord, length_f);
746       }
747       /* no clamp necessary, border masking will handle this */
748       icoord = lp_build_ifloor(coord_bld, coord);
749       if (offset) {
750          icoord = lp_build_add(int_coord_bld, icoord, offset);
751       }
752       break;
753 
754    case PIPE_TEX_WRAP_MIRROR_REPEAT:
755       if (offset) {
756          offset = lp_build_int_to_float(coord_bld, offset);
757          offset = lp_build_div(coord_bld, offset, length_f);
758          coord = lp_build_add(coord_bld, coord, offset);
759       }
760       /* compute mirror function */
761       coord = lp_build_coord_mirror(bld, coord, TRUE);
762 
763       /* scale coord to length */
764       assert(bld->static_sampler_state->normalized_coords);
765       coord = lp_build_mul(coord_bld, coord, length_f);
766 
767       /* itrunc == ifloor here */
768       icoord = lp_build_itrunc(coord_bld, coord);
769 
770       /* clamp to [0, length - 1] */
771       icoord = lp_build_min(int_coord_bld, icoord, length_minus_one);
772       break;
773 
774    case PIPE_TEX_WRAP_MIRROR_CLAMP:
775    case PIPE_TEX_WRAP_MIRROR_CLAMP_TO_EDGE:
776       if (bld->static_sampler_state->normalized_coords) {
777          /* scale coord to length */
778          coord = lp_build_mul(coord_bld, coord, length_f);
779       }
780       if (offset) {
781          offset = lp_build_int_to_float(coord_bld, offset);
782          coord = lp_build_add(coord_bld, coord, offset);
783       }
784       coord = lp_build_abs(coord_bld, coord);
785 
786       /* itrunc == ifloor here */
787       icoord = lp_build_itrunc(coord_bld, coord);
788       /*
789        * Use unsigned min due to possible undef values (NaNs, overflow)
790        */
791       {
792          struct lp_build_context abs_coord_bld = *int_coord_bld;
793          abs_coord_bld.type.sign = FALSE;
794          /* clamp to [0, length - 1] */
795          icoord = lp_build_min(&abs_coord_bld, icoord, length_minus_one);
796       }
797       break;
798 
799    case PIPE_TEX_WRAP_MIRROR_CLAMP_TO_BORDER:
800       if (bld->static_sampler_state->normalized_coords) {
801          /* scale coord to length */
802          coord = lp_build_mul(coord_bld, coord, length_f);
803       }
804       if (offset) {
805          offset = lp_build_int_to_float(coord_bld, offset);
806          coord = lp_build_add(coord_bld, coord, offset);
807       }
808       coord = lp_build_abs(coord_bld, coord);
809 
810       /* itrunc == ifloor here */
811       icoord = lp_build_itrunc(coord_bld, coord);
812       break;
813 
814    default:
815       assert(0);
816       icoord = NULL;
817    }
818 
819    return icoord;
820 }
821 
822 
823 /**
824  * Do shadow test/comparison.
825  * \param p shadow ref value
826  * \param texel  the texel to compare against
827  */
828 static LLVMValueRef
lp_build_sample_comparefunc(struct lp_build_sample_context * bld,LLVMValueRef p,LLVMValueRef texel)829 lp_build_sample_comparefunc(struct lp_build_sample_context *bld,
830                             LLVMValueRef p,
831                             LLVMValueRef texel)
832 {
833    struct lp_build_context *texel_bld = &bld->texel_bld;
834    LLVMValueRef res;
835 
836    if (0) {
837       //lp_build_print_value(bld->gallivm, "shadow cmp coord", p);
838       lp_build_print_value(bld->gallivm, "shadow cmp texel", texel);
839    }
840 
841    /* result = (p FUNC texel) ? 1 : 0 */
842    /*
843     * honor d3d10 floating point rules here, which state that comparisons
844     * are ordered except NOT_EQUAL which is unordered.
845     */
846    if (bld->static_sampler_state->compare_func != PIPE_FUNC_NOTEQUAL) {
847       res = lp_build_cmp_ordered(texel_bld, bld->static_sampler_state->compare_func,
848                                  p, texel);
849    }
850    else {
851       res = lp_build_cmp(texel_bld, bld->static_sampler_state->compare_func,
852                          p, texel);
853    }
854    return res;
855 }
856 
857 
858 /**
859  * Generate code to sample a mipmap level with nearest filtering.
860  * If sampling a cube texture, r = cube face in [0,5].
861  */
862 static void
lp_build_sample_image_nearest(struct lp_build_sample_context * bld,LLVMValueRef size,LLVMValueRef row_stride_vec,LLVMValueRef img_stride_vec,LLVMValueRef data_ptr,LLVMValueRef mipoffsets,const LLVMValueRef * coords,const LLVMValueRef * offsets,LLVMValueRef colors_out[4])863 lp_build_sample_image_nearest(struct lp_build_sample_context *bld,
864                               LLVMValueRef size,
865                               LLVMValueRef row_stride_vec,
866                               LLVMValueRef img_stride_vec,
867                               LLVMValueRef data_ptr,
868                               LLVMValueRef mipoffsets,
869                               const LLVMValueRef *coords,
870                               const LLVMValueRef *offsets,
871                               LLVMValueRef colors_out[4])
872 {
873    const unsigned dims = bld->dims;
874    LLVMValueRef width_vec;
875    LLVMValueRef height_vec;
876    LLVMValueRef depth_vec;
877    LLVMValueRef flt_size;
878    LLVMValueRef flt_width_vec;
879    LLVMValueRef flt_height_vec;
880    LLVMValueRef flt_depth_vec;
881    LLVMValueRef x, y = NULL, z = NULL;
882 
883    lp_build_extract_image_sizes(bld,
884                                 &bld->int_size_bld,
885                                 bld->int_coord_type,
886                                 size,
887                                 &width_vec, &height_vec, &depth_vec);
888 
889    flt_size = lp_build_int_to_float(&bld->float_size_bld, size);
890 
891    lp_build_extract_image_sizes(bld,
892                                 &bld->float_size_bld,
893                                 bld->coord_type,
894                                 flt_size,
895                                 &flt_width_vec, &flt_height_vec, &flt_depth_vec);
896 
897    /*
898     * Compute integer texcoords.
899     */
900    x = lp_build_sample_wrap_nearest(bld, coords[0], width_vec,
901                                     flt_width_vec, offsets[0],
902                                     bld->static_texture_state->pot_width,
903                                     bld->static_sampler_state->wrap_s);
904    lp_build_name(x, "tex.x.wrapped");
905 
906    if (dims >= 2) {
907       y = lp_build_sample_wrap_nearest(bld, coords[1], height_vec,
908                                        flt_height_vec, offsets[1],
909                                        bld->static_texture_state->pot_height,
910                                        bld->static_sampler_state->wrap_t);
911       lp_build_name(y, "tex.y.wrapped");
912 
913       if (dims == 3) {
914          z = lp_build_sample_wrap_nearest(bld, coords[2], depth_vec,
915                                           flt_depth_vec, offsets[2],
916                                           bld->static_texture_state->pot_depth,
917                                           bld->static_sampler_state->wrap_r);
918          lp_build_name(z, "tex.z.wrapped");
919       }
920    }
921    if (has_layer_coord(bld->static_texture_state->target)) {
922       if (bld->static_texture_state->target == PIPE_TEXTURE_CUBE_ARRAY) {
923          /* add cube layer to face */
924          z = lp_build_add(&bld->int_coord_bld, coords[2], coords[3]);
925       }
926       else {
927          z = coords[2];
928       }
929       lp_build_name(z, "tex.z.layer");
930    }
931 
932    /*
933     * Get texture colors.
934     */
935    lp_build_sample_texel_soa(bld,
936                              width_vec, height_vec, depth_vec,
937                              x, y, z,
938                              row_stride_vec, img_stride_vec,
939                              data_ptr, mipoffsets, colors_out);
940 
941    if (bld->static_sampler_state->compare_mode != PIPE_TEX_COMPARE_NONE) {
942       LLVMValueRef cmpval;
943       cmpval = lp_build_sample_comparefunc(bld, coords[4], colors_out[0]);
944       /* this is really just a AND 1.0, cmpval but llvm is clever enough */
945       colors_out[0] = lp_build_select(&bld->texel_bld, cmpval,
946                                       bld->texel_bld.one, bld->texel_bld.zero);
947       colors_out[1] = colors_out[2] = colors_out[3] = colors_out[0];
948    }
949 
950 }
951 
952 
953 /**
954  * Like a lerp, but inputs are 0/~0 masks, so can simplify slightly.
955  */
956 static LLVMValueRef
lp_build_masklerp(struct lp_build_context * bld,LLVMValueRef weight,LLVMValueRef mask0,LLVMValueRef mask1)957 lp_build_masklerp(struct lp_build_context *bld,
958                  LLVMValueRef weight,
959                  LLVMValueRef mask0,
960                  LLVMValueRef mask1)
961 {
962    struct gallivm_state *gallivm = bld->gallivm;
963    LLVMBuilderRef builder = gallivm->builder;
964    LLVMValueRef weight2;
965 
966    weight2 = lp_build_sub(bld, bld->one, weight);
967    weight = LLVMBuildBitCast(builder, weight,
968                               lp_build_int_vec_type(gallivm, bld->type), "");
969    weight2 = LLVMBuildBitCast(builder, weight2,
970                               lp_build_int_vec_type(gallivm, bld->type), "");
971    weight = LLVMBuildAnd(builder, weight, mask1, "");
972    weight2 = LLVMBuildAnd(builder, weight2, mask0, "");
973    weight = LLVMBuildBitCast(builder, weight, bld->vec_type, "");
974    weight2 = LLVMBuildBitCast(builder, weight2, bld->vec_type, "");
975    return lp_build_add(bld, weight, weight2);
976 }
977 
978 /**
979  * Like a 2d lerp, but inputs are 0/~0 masks, so can simplify slightly.
980  */
981 static LLVMValueRef
lp_build_masklerp2d(struct lp_build_context * bld,LLVMValueRef weight0,LLVMValueRef weight1,LLVMValueRef mask00,LLVMValueRef mask01,LLVMValueRef mask10,LLVMValueRef mask11)982 lp_build_masklerp2d(struct lp_build_context *bld,
983                     LLVMValueRef weight0,
984                     LLVMValueRef weight1,
985                     LLVMValueRef mask00,
986                     LLVMValueRef mask01,
987                     LLVMValueRef mask10,
988                     LLVMValueRef mask11)
989 {
990    LLVMValueRef val0 = lp_build_masklerp(bld, weight0, mask00, mask01);
991    LLVMValueRef val1 = lp_build_masklerp(bld, weight0, mask10, mask11);
992    return lp_build_lerp(bld, weight1, val0, val1, 0);
993 }
994 
995 /*
996  * this is a bit excessive code for something OpenGL just recommends
997  * but does not require.
998  */
999 #define ACCURATE_CUBE_CORNERS 1
1000 
1001 /**
1002  * Generate code to sample a mipmap level with linear filtering.
1003  * If sampling a cube texture, r = cube face in [0,5].
1004  * If linear_mask is present, only pixels having their mask set
1005  * will receive linear filtering, the rest will use nearest.
1006  */
1007 static void
lp_build_sample_image_linear(struct lp_build_sample_context * bld,boolean is_gather,LLVMValueRef size,LLVMValueRef linear_mask,LLVMValueRef row_stride_vec,LLVMValueRef img_stride_vec,LLVMValueRef data_ptr,LLVMValueRef mipoffsets,const LLVMValueRef * coords,const LLVMValueRef * offsets,LLVMValueRef colors_out[4])1008 lp_build_sample_image_linear(struct lp_build_sample_context *bld,
1009                              boolean is_gather,
1010                              LLVMValueRef size,
1011                              LLVMValueRef linear_mask,
1012                              LLVMValueRef row_stride_vec,
1013                              LLVMValueRef img_stride_vec,
1014                              LLVMValueRef data_ptr,
1015                              LLVMValueRef mipoffsets,
1016                              const LLVMValueRef *coords,
1017                              const LLVMValueRef *offsets,
1018                              LLVMValueRef colors_out[4])
1019 {
1020    LLVMBuilderRef builder = bld->gallivm->builder;
1021    struct lp_build_context *ivec_bld = &bld->int_coord_bld;
1022    struct lp_build_context *coord_bld = &bld->coord_bld;
1023    struct lp_build_context *texel_bld = &bld->texel_bld;
1024    const unsigned dims = bld->dims;
1025    LLVMValueRef width_vec;
1026    LLVMValueRef height_vec;
1027    LLVMValueRef depth_vec;
1028    LLVMValueRef flt_size;
1029    LLVMValueRef flt_width_vec;
1030    LLVMValueRef flt_height_vec;
1031    LLVMValueRef flt_depth_vec;
1032    LLVMValueRef fall_off[4], have_corners;
1033    LLVMValueRef z1 = NULL;
1034    LLVMValueRef z00 = NULL, z01 = NULL, z10 = NULL, z11 = NULL;
1035    LLVMValueRef x00 = NULL, x01 = NULL, x10 = NULL, x11 = NULL;
1036    LLVMValueRef y00 = NULL, y01 = NULL, y10 = NULL, y11 = NULL;
1037    LLVMValueRef s_fpart, t_fpart = NULL, r_fpart = NULL;
1038    LLVMValueRef xs[4], ys[4], zs[4];
1039    LLVMValueRef neighbors[2][2][4];
1040    int chan, texel_index;
1041    boolean seamless_cube_filter, accurate_cube_corners;
1042    unsigned chan_swiz = bld->static_texture_state->swizzle_r;
1043 
1044    if (is_gather) {
1045       switch (bld->gather_comp) {
1046       case 0: chan_swiz = bld->static_texture_state->swizzle_r; break;
1047       case 1: chan_swiz = bld->static_texture_state->swizzle_g; break;
1048       case 2: chan_swiz = bld->static_texture_state->swizzle_b; break;
1049       case 3: chan_swiz = bld->static_texture_state->swizzle_a; break;
1050       default:
1051 	 break;
1052       }
1053    }
1054 
1055    seamless_cube_filter = (bld->static_texture_state->target == PIPE_TEXTURE_CUBE ||
1056                            bld->static_texture_state->target == PIPE_TEXTURE_CUBE_ARRAY) &&
1057                           bld->static_sampler_state->seamless_cube_map;
1058 
1059    /*
1060     * Disable accurate cube corners for integer textures, which should only
1061     * get here in the gather path.
1062     */
1063    accurate_cube_corners = ACCURATE_CUBE_CORNERS && seamless_cube_filter &&
1064      !util_format_is_pure_integer(bld->static_texture_state->format);
1065 
1066    lp_build_extract_image_sizes(bld,
1067                                 &bld->int_size_bld,
1068                                 bld->int_coord_type,
1069                                 size,
1070                                 &width_vec, &height_vec, &depth_vec);
1071 
1072    flt_size = lp_build_int_to_float(&bld->float_size_bld, size);
1073 
1074    lp_build_extract_image_sizes(bld,
1075                                 &bld->float_size_bld,
1076                                 bld->coord_type,
1077                                 flt_size,
1078                                 &flt_width_vec, &flt_height_vec, &flt_depth_vec);
1079 
1080    /*
1081     * Compute integer texcoords.
1082     */
1083 
1084    if (!seamless_cube_filter) {
1085       lp_build_sample_wrap_linear(bld, is_gather, coords[0], width_vec,
1086                                   flt_width_vec, offsets[0],
1087                                   bld->static_texture_state->pot_width,
1088                                   bld->static_sampler_state->wrap_s,
1089                                   &x00, &x01, &s_fpart);
1090       lp_build_name(x00, "tex.x0.wrapped");
1091       lp_build_name(x01, "tex.x1.wrapped");
1092       x10 = x00;
1093       x11 = x01;
1094 
1095       if (dims >= 2) {
1096          lp_build_sample_wrap_linear(bld, is_gather, coords[1], height_vec,
1097                                      flt_height_vec, offsets[1],
1098                                      bld->static_texture_state->pot_height,
1099                                      bld->static_sampler_state->wrap_t,
1100                                      &y00, &y10, &t_fpart);
1101          lp_build_name(y00, "tex.y0.wrapped");
1102          lp_build_name(y10, "tex.y1.wrapped");
1103          y01 = y00;
1104          y11 = y10;
1105 
1106          if (dims == 3) {
1107             lp_build_sample_wrap_linear(bld, is_gather, coords[2], depth_vec,
1108                                         flt_depth_vec, offsets[2],
1109                                         bld->static_texture_state->pot_depth,
1110                                         bld->static_sampler_state->wrap_r,
1111                                         &z00, &z1, &r_fpart);
1112             z01 = z10 = z11 = z00;
1113             lp_build_name(z00, "tex.z0.wrapped");
1114             lp_build_name(z1, "tex.z1.wrapped");
1115          }
1116       }
1117       if (has_layer_coord(bld->static_texture_state->target)) {
1118          if (bld->static_texture_state->target == PIPE_TEXTURE_CUBE_ARRAY) {
1119             /* add cube layer to face */
1120             z00 = z01 = z10 = z11 = z1 =
1121                lp_build_add(&bld->int_coord_bld, coords[2], coords[3]);
1122          }
1123          else {
1124             z00 = z01 = z10 = z11 = z1 = coords[2];  /* cube face or layer */
1125          }
1126          lp_build_name(z00, "tex.z0.layer");
1127          lp_build_name(z1, "tex.z1.layer");
1128       }
1129    }
1130    else {
1131       struct lp_build_if_state edge_if;
1132       LLVMTypeRef int1t;
1133       LLVMValueRef new_faces[4], new_xcoords[4][2], new_ycoords[4][2];
1134       LLVMValueRef coord0, coord1, have_edge, have_corner;
1135       LLVMValueRef fall_off_ym_notxm, fall_off_ym_notxp, fall_off_x, fall_off_y;
1136       LLVMValueRef fall_off_yp_notxm, fall_off_yp_notxp;
1137       LLVMValueRef x0, x1, y0, y1, y0_clamped, y1_clamped;
1138       LLVMValueRef face = coords[2];
1139       LLVMValueRef half = lp_build_const_vec(bld->gallivm, coord_bld->type, 0.5f);
1140       LLVMValueRef length_minus_one = lp_build_sub(ivec_bld, width_vec, ivec_bld->one);
1141       /* XXX drop height calcs. Could (should) do this without seamless filtering too */
1142       height_vec = width_vec;
1143       flt_height_vec = flt_width_vec;
1144 
1145       /* XXX the overflow logic is actually sort of duplicated with trilinear,
1146        * since an overflow in one mip should also have a corresponding overflow
1147        * in another.
1148        */
1149       /* should always have normalized coords, and offsets are undefined */
1150       assert(bld->static_sampler_state->normalized_coords);
1151       /*
1152        * The coords should all be between [0,1] however we can have NaNs,
1153        * which will wreak havoc. In particular the y1_clamped value below
1154        * can be -INT_MAX (on x86) and be propagated right through (probably
1155        * other values might be bogus in the end too).
1156        * So kill off the NaNs here.
1157        */
1158       coord0 = lp_build_max_ext(coord_bld, coords[0], coord_bld->zero,
1159                                 GALLIVM_NAN_RETURN_OTHER_SECOND_NONNAN);
1160       coord0 = lp_build_mul(coord_bld, coord0, flt_width_vec);
1161       /* instead of clamp, build mask if overflowed */
1162       coord0 = lp_build_sub(coord_bld, coord0, half);
1163       /* convert to int, compute lerp weight */
1164       /* not ideal with AVX (and no AVX2) */
1165       lp_build_ifloor_fract(coord_bld, coord0, &x0, &s_fpart);
1166       x1 = lp_build_add(ivec_bld, x0, ivec_bld->one);
1167       coord1 = lp_build_max_ext(coord_bld, coords[1], coord_bld->zero,
1168                                 GALLIVM_NAN_RETURN_OTHER_SECOND_NONNAN);
1169       coord1 = lp_build_mul(coord_bld, coord1, flt_height_vec);
1170       coord1 = lp_build_sub(coord_bld, coord1, half);
1171       lp_build_ifloor_fract(coord_bld, coord1, &y0, &t_fpart);
1172       y1 = lp_build_add(ivec_bld, y0, ivec_bld->one);
1173 
1174       fall_off[0] = lp_build_cmp(ivec_bld, PIPE_FUNC_LESS, x0, ivec_bld->zero);
1175       fall_off[1] = lp_build_cmp(ivec_bld, PIPE_FUNC_GREATER, x1, length_minus_one);
1176       fall_off[2] = lp_build_cmp(ivec_bld, PIPE_FUNC_LESS, y0, ivec_bld->zero);
1177       fall_off[3] = lp_build_cmp(ivec_bld, PIPE_FUNC_GREATER, y1, length_minus_one);
1178 
1179       fall_off_x = lp_build_or(ivec_bld, fall_off[0], fall_off[1]);
1180       fall_off_y = lp_build_or(ivec_bld, fall_off[2], fall_off[3]);
1181       have_edge = lp_build_or(ivec_bld, fall_off_x, fall_off_y);
1182       have_edge = lp_build_any_true_range(ivec_bld, ivec_bld->type.length, have_edge);
1183 
1184       /* needed for accurate corner filtering branch later, rely on 0 init */
1185       int1t = LLVMInt1TypeInContext(bld->gallivm->context);
1186       have_corners = lp_build_alloca(bld->gallivm, int1t, "have_corner");
1187 
1188       for (texel_index = 0; texel_index < 4; texel_index++) {
1189          xs[texel_index] = lp_build_alloca(bld->gallivm, ivec_bld->vec_type, "xs");
1190          ys[texel_index] = lp_build_alloca(bld->gallivm, ivec_bld->vec_type, "ys");
1191          zs[texel_index] = lp_build_alloca(bld->gallivm, ivec_bld->vec_type, "zs");
1192       }
1193 
1194       lp_build_if(&edge_if, bld->gallivm, have_edge);
1195 
1196       have_corner = lp_build_and(ivec_bld, fall_off_x, fall_off_y);
1197       have_corner = lp_build_any_true_range(ivec_bld, ivec_bld->type.length, have_corner);
1198       LLVMBuildStore(builder, have_corner, have_corners);
1199 
1200       /*
1201        * Need to feed clamped values here for cheap corner handling,
1202        * but only for y coord (as when falling off both edges we only
1203        * fall off the x one) - this should be sufficient.
1204        */
1205       y0_clamped = lp_build_max(ivec_bld, y0, ivec_bld->zero);
1206       y1_clamped = lp_build_min(ivec_bld, y1, length_minus_one);
1207 
1208       /*
1209        * Get all possible new coords.
1210        */
1211       lp_build_cube_new_coords(ivec_bld, face,
1212                                x0, x1, y0_clamped, y1_clamped,
1213                                length_minus_one,
1214                                new_faces, new_xcoords, new_ycoords);
1215 
1216       /* handle fall off x-, x+ direction */
1217       /* determine new coords, face (not both fall_off vars can be true at same time) */
1218       x00 = lp_build_select(ivec_bld, fall_off[0], new_xcoords[0][0], x0);
1219       y00 = lp_build_select(ivec_bld, fall_off[0], new_ycoords[0][0], y0_clamped);
1220       x10 = lp_build_select(ivec_bld, fall_off[0], new_xcoords[0][1], x0);
1221       y10 = lp_build_select(ivec_bld, fall_off[0], new_ycoords[0][1], y1_clamped);
1222       x01 = lp_build_select(ivec_bld, fall_off[1], new_xcoords[1][0], x1);
1223       y01 = lp_build_select(ivec_bld, fall_off[1], new_ycoords[1][0], y0_clamped);
1224       x11 = lp_build_select(ivec_bld, fall_off[1], new_xcoords[1][1], x1);
1225       y11 = lp_build_select(ivec_bld, fall_off[1], new_ycoords[1][1], y1_clamped);
1226 
1227       z00 = z10 = lp_build_select(ivec_bld, fall_off[0], new_faces[0], face);
1228       z01 = z11 = lp_build_select(ivec_bld, fall_off[1], new_faces[1], face);
1229 
1230       /* handle fall off y-, y+ direction */
1231       /*
1232        * Cheap corner logic: just hack up things so a texel doesn't fall
1233        * off both sides (which means filter weights will be wrong but we'll only
1234        * use valid texels in the filter).
1235        * This means however (y) coords must additionally be clamped (see above).
1236        * This corner handling should be fully OpenGL (but not d3d10) compliant.
1237        */
1238       fall_off_ym_notxm = lp_build_andnot(ivec_bld, fall_off[2], fall_off[0]);
1239       fall_off_ym_notxp = lp_build_andnot(ivec_bld, fall_off[2], fall_off[1]);
1240       fall_off_yp_notxm = lp_build_andnot(ivec_bld, fall_off[3], fall_off[0]);
1241       fall_off_yp_notxp = lp_build_andnot(ivec_bld, fall_off[3], fall_off[1]);
1242 
1243       x00 = lp_build_select(ivec_bld, fall_off_ym_notxm, new_xcoords[2][0], x00);
1244       y00 = lp_build_select(ivec_bld, fall_off_ym_notxm, new_ycoords[2][0], y00);
1245       x01 = lp_build_select(ivec_bld, fall_off_ym_notxp, new_xcoords[2][1], x01);
1246       y01 = lp_build_select(ivec_bld, fall_off_ym_notxp, new_ycoords[2][1], y01);
1247       x10 = lp_build_select(ivec_bld, fall_off_yp_notxm, new_xcoords[3][0], x10);
1248       y10 = lp_build_select(ivec_bld, fall_off_yp_notxm, new_ycoords[3][0], y10);
1249       x11 = lp_build_select(ivec_bld, fall_off_yp_notxp, new_xcoords[3][1], x11);
1250       y11 = lp_build_select(ivec_bld, fall_off_yp_notxp, new_ycoords[3][1], y11);
1251 
1252       z00 = lp_build_select(ivec_bld, fall_off_ym_notxm, new_faces[2], z00);
1253       z01 = lp_build_select(ivec_bld, fall_off_ym_notxp, new_faces[2], z01);
1254       z10 = lp_build_select(ivec_bld, fall_off_yp_notxm, new_faces[3], z10);
1255       z11 = lp_build_select(ivec_bld, fall_off_yp_notxp, new_faces[3], z11);
1256 
1257       if (bld->static_texture_state->target == PIPE_TEXTURE_CUBE_ARRAY) {
1258          /* now can add cube layer to face (per sample) */
1259          z00 = lp_build_add(ivec_bld, z00, coords[3]);
1260          z01 = lp_build_add(ivec_bld, z01, coords[3]);
1261          z10 = lp_build_add(ivec_bld, z10, coords[3]);
1262          z11 = lp_build_add(ivec_bld, z11, coords[3]);
1263       }
1264 
1265       LLVMBuildStore(builder, x00, xs[0]);
1266       LLVMBuildStore(builder, x01, xs[1]);
1267       LLVMBuildStore(builder, x10, xs[2]);
1268       LLVMBuildStore(builder, x11, xs[3]);
1269       LLVMBuildStore(builder, y00, ys[0]);
1270       LLVMBuildStore(builder, y01, ys[1]);
1271       LLVMBuildStore(builder, y10, ys[2]);
1272       LLVMBuildStore(builder, y11, ys[3]);
1273       LLVMBuildStore(builder, z00, zs[0]);
1274       LLVMBuildStore(builder, z01, zs[1]);
1275       LLVMBuildStore(builder, z10, zs[2]);
1276       LLVMBuildStore(builder, z11, zs[3]);
1277 
1278       lp_build_else(&edge_if);
1279 
1280       LLVMBuildStore(builder, x0, xs[0]);
1281       LLVMBuildStore(builder, x1, xs[1]);
1282       LLVMBuildStore(builder, x0, xs[2]);
1283       LLVMBuildStore(builder, x1, xs[3]);
1284       LLVMBuildStore(builder, y0, ys[0]);
1285       LLVMBuildStore(builder, y0, ys[1]);
1286       LLVMBuildStore(builder, y1, ys[2]);
1287       LLVMBuildStore(builder, y1, ys[3]);
1288       if (bld->static_texture_state->target == PIPE_TEXTURE_CUBE_ARRAY) {
1289          LLVMValueRef cube_layer = lp_build_add(ivec_bld, face, coords[3]);
1290          LLVMBuildStore(builder, cube_layer, zs[0]);
1291          LLVMBuildStore(builder, cube_layer, zs[1]);
1292          LLVMBuildStore(builder, cube_layer, zs[2]);
1293          LLVMBuildStore(builder, cube_layer, zs[3]);
1294       }
1295       else {
1296          LLVMBuildStore(builder, face, zs[0]);
1297          LLVMBuildStore(builder, face, zs[1]);
1298          LLVMBuildStore(builder, face, zs[2]);
1299          LLVMBuildStore(builder, face, zs[3]);
1300       }
1301 
1302       lp_build_endif(&edge_if);
1303 
1304       x00 = LLVMBuildLoad(builder, xs[0], "");
1305       x01 = LLVMBuildLoad(builder, xs[1], "");
1306       x10 = LLVMBuildLoad(builder, xs[2], "");
1307       x11 = LLVMBuildLoad(builder, xs[3], "");
1308       y00 = LLVMBuildLoad(builder, ys[0], "");
1309       y01 = LLVMBuildLoad(builder, ys[1], "");
1310       y10 = LLVMBuildLoad(builder, ys[2], "");
1311       y11 = LLVMBuildLoad(builder, ys[3], "");
1312       z00 = LLVMBuildLoad(builder, zs[0], "");
1313       z01 = LLVMBuildLoad(builder, zs[1], "");
1314       z10 = LLVMBuildLoad(builder, zs[2], "");
1315       z11 = LLVMBuildLoad(builder, zs[3], "");
1316    }
1317 
1318    if (linear_mask) {
1319       /*
1320        * Whack filter weights into place. Whatever texel had more weight is
1321        * the one which should have been selected by nearest filtering hence
1322        * just use 100% weight for it.
1323        */
1324       struct lp_build_context *c_bld = &bld->coord_bld;
1325       LLVMValueRef w1_mask, w1_weight;
1326       LLVMValueRef half = lp_build_const_vec(bld->gallivm, c_bld->type, 0.5f);
1327 
1328       w1_mask = lp_build_cmp(c_bld, PIPE_FUNC_GREATER, s_fpart, half);
1329       /* this select is really just a "and" */
1330       w1_weight = lp_build_select(c_bld, w1_mask, c_bld->one, c_bld->zero);
1331       s_fpart = lp_build_select(c_bld, linear_mask, s_fpart, w1_weight);
1332       if (dims >= 2) {
1333          w1_mask = lp_build_cmp(c_bld, PIPE_FUNC_GREATER, t_fpart, half);
1334          w1_weight = lp_build_select(c_bld, w1_mask, c_bld->one, c_bld->zero);
1335          t_fpart = lp_build_select(c_bld, linear_mask, t_fpart, w1_weight);
1336          if (dims == 3) {
1337             w1_mask = lp_build_cmp(c_bld, PIPE_FUNC_GREATER, r_fpart, half);
1338             w1_weight = lp_build_select(c_bld, w1_mask, c_bld->one, c_bld->zero);
1339             r_fpart = lp_build_select(c_bld, linear_mask, r_fpart, w1_weight);
1340          }
1341       }
1342    }
1343 
1344    /*
1345     * Get texture colors.
1346     */
1347    /* get x0/x1 texels */
1348    lp_build_sample_texel_soa(bld,
1349                              width_vec, height_vec, depth_vec,
1350                              x00, y00, z00,
1351                              row_stride_vec, img_stride_vec,
1352                              data_ptr, mipoffsets, neighbors[0][0]);
1353    lp_build_sample_texel_soa(bld,
1354                              width_vec, height_vec, depth_vec,
1355                              x01, y01, z01,
1356                              row_stride_vec, img_stride_vec,
1357                              data_ptr, mipoffsets, neighbors[0][1]);
1358 
1359    if (dims == 1) {
1360       assert(!is_gather);
1361       if (bld->static_sampler_state->compare_mode == PIPE_TEX_COMPARE_NONE) {
1362          /* Interpolate two samples from 1D image to produce one color */
1363          for (chan = 0; chan < 4; chan++) {
1364             colors_out[chan] = lp_build_lerp(texel_bld, s_fpart,
1365                                              neighbors[0][0][chan],
1366                                              neighbors[0][1][chan],
1367                                              0);
1368          }
1369       }
1370       else {
1371          LLVMValueRef cmpval0, cmpval1;
1372          cmpval0 = lp_build_sample_comparefunc(bld, coords[4], neighbors[0][0][0]);
1373          cmpval1 = lp_build_sample_comparefunc(bld, coords[4], neighbors[0][1][0]);
1374          /* simplified lerp, AND mask with weight and add */
1375          colors_out[0] = lp_build_masklerp(texel_bld, s_fpart,
1376                                            cmpval0, cmpval1);
1377          colors_out[1] = colors_out[2] = colors_out[3] = colors_out[0];
1378       }
1379    }
1380    else {
1381       /* 2D/3D texture */
1382       struct lp_build_if_state corner_if;
1383       LLVMValueRef colors0[4], colorss[4];
1384 
1385       /* get x0/x1 texels at y1 */
1386       lp_build_sample_texel_soa(bld,
1387                                 width_vec, height_vec, depth_vec,
1388                                 x10, y10, z10,
1389                                 row_stride_vec, img_stride_vec,
1390                                 data_ptr, mipoffsets, neighbors[1][0]);
1391       lp_build_sample_texel_soa(bld,
1392                                 width_vec, height_vec, depth_vec,
1393                                 x11, y11, z11,
1394                                 row_stride_vec, img_stride_vec,
1395                                 data_ptr, mipoffsets, neighbors[1][1]);
1396 
1397       /*
1398        * To avoid having to duplicate linear_mask / fetch code use
1399        * another branch (with corner condition though edge would work
1400        * as well) here.
1401        */
1402       if (accurate_cube_corners) {
1403          LLVMValueRef c00, c01, c10, c11, c00f, c01f, c10f, c11f;
1404          LLVMValueRef have_corner, one_third;
1405 
1406          colorss[0] = lp_build_alloca(bld->gallivm, coord_bld->vec_type, "cs0");
1407          colorss[1] = lp_build_alloca(bld->gallivm, coord_bld->vec_type, "cs1");
1408          colorss[2] = lp_build_alloca(bld->gallivm, coord_bld->vec_type, "cs2");
1409          colorss[3] = lp_build_alloca(bld->gallivm, coord_bld->vec_type, "cs3");
1410 
1411          have_corner = LLVMBuildLoad(builder, have_corners, "");
1412 
1413          lp_build_if(&corner_if, bld->gallivm, have_corner);
1414 
1415          one_third = lp_build_const_vec(bld->gallivm, coord_bld->type,
1416                                         1.0f/3.0f);
1417 
1418          /* find corner */
1419          c00 = lp_build_and(ivec_bld, fall_off[0], fall_off[2]);
1420          c00f = LLVMBuildBitCast(builder, c00, coord_bld->vec_type, "");
1421          c01 = lp_build_and(ivec_bld, fall_off[1], fall_off[2]);
1422          c01f = LLVMBuildBitCast(builder, c01, coord_bld->vec_type, "");
1423          c10 = lp_build_and(ivec_bld, fall_off[0], fall_off[3]);
1424          c10f = LLVMBuildBitCast(builder, c10, coord_bld->vec_type, "");
1425          c11 = lp_build_and(ivec_bld, fall_off[1], fall_off[3]);
1426          c11f = LLVMBuildBitCast(builder, c11, coord_bld->vec_type, "");
1427 
1428          if (!is_gather) {
1429             /*
1430              * we can't use standard 2d lerp as we need per-element weight
1431              * in case of corners, so just calculate bilinear result as
1432              * w00*s00 + w01*s01 + w10*s10 + w11*s11.
1433              * (This is actually less work than using 2d lerp, 7 vs. 9
1434              * instructions, however calculating the weights needs another 6,
1435              * so actually probably not slower than 2d lerp only for 4 channels
1436              * as weights only need to be calculated once - of course fixing
1437              * the weights has additional cost.)
1438              */
1439             LLVMValueRef w00, w01, w10, w11, wx0, wy0, c_weight, tmp;
1440             wx0 = lp_build_sub(coord_bld, coord_bld->one, s_fpart);
1441             wy0 = lp_build_sub(coord_bld, coord_bld->one, t_fpart);
1442             w00 = lp_build_mul(coord_bld, wx0, wy0);
1443             w01 = lp_build_mul(coord_bld, s_fpart, wy0);
1444             w10 = lp_build_mul(coord_bld, wx0, t_fpart);
1445             w11 = lp_build_mul(coord_bld, s_fpart, t_fpart);
1446 
1447             /* find corner weight */
1448             c_weight = lp_build_select(coord_bld, c00, w00, coord_bld->zero);
1449             c_weight = lp_build_select(coord_bld, c01, w01, c_weight);
1450             c_weight = lp_build_select(coord_bld, c10, w10, c_weight);
1451             c_weight = lp_build_select(coord_bld, c11, w11, c_weight);
1452 
1453             /*
1454              * add 1/3 of the corner weight to the weight of the 3 other
1455              * samples and null out corner weight.
1456              */
1457             c_weight = lp_build_mul(coord_bld, c_weight, one_third);
1458             w00 = lp_build_add(coord_bld, w00, c_weight);
1459             w00 = lp_build_andnot(coord_bld, w00, c00f);
1460             w01 = lp_build_add(coord_bld, w01, c_weight);
1461             w01 = lp_build_andnot(coord_bld, w01, c01f);
1462             w10 = lp_build_add(coord_bld, w10, c_weight);
1463             w10 = lp_build_andnot(coord_bld, w10, c10f);
1464             w11 = lp_build_add(coord_bld, w11, c_weight);
1465             w11 = lp_build_andnot(coord_bld, w11, c11f);
1466 
1467             if (bld->static_sampler_state->compare_mode ==
1468                 PIPE_TEX_COMPARE_NONE) {
1469                for (chan = 0; chan < 4; chan++) {
1470                   colors0[chan] = lp_build_mul(coord_bld, w00,
1471                                                neighbors[0][0][chan]);
1472                   tmp = lp_build_mul(coord_bld, w01, neighbors[0][1][chan]);
1473                   colors0[chan] = lp_build_add(coord_bld, tmp, colors0[chan]);
1474                   tmp = lp_build_mul(coord_bld, w10, neighbors[1][0][chan]);
1475                   colors0[chan] = lp_build_add(coord_bld, tmp, colors0[chan]);
1476                   tmp = lp_build_mul(coord_bld, w11, neighbors[1][1][chan]);
1477                   colors0[chan] = lp_build_add(coord_bld, tmp, colors0[chan]);
1478                }
1479             }
1480             else {
1481                LLVMValueRef cmpval00, cmpval01, cmpval10, cmpval11;
1482                cmpval00 = lp_build_sample_comparefunc(bld, coords[4],
1483                                                       neighbors[0][0][0]);
1484                cmpval01 = lp_build_sample_comparefunc(bld, coords[4],
1485                                                       neighbors[0][1][0]);
1486                cmpval10 = lp_build_sample_comparefunc(bld, coords[4],
1487                                                       neighbors[1][0][0]);
1488                cmpval11 = lp_build_sample_comparefunc(bld, coords[4],
1489                                                       neighbors[1][1][0]);
1490                /*
1491                 * inputs to interpolation are just masks so just add
1492                 * masked weights together
1493                 */
1494                cmpval00 = LLVMBuildBitCast(builder, cmpval00,
1495                                            coord_bld->vec_type, "");
1496                cmpval01 = LLVMBuildBitCast(builder, cmpval01,
1497                                            coord_bld->vec_type, "");
1498                cmpval10 = LLVMBuildBitCast(builder, cmpval10,
1499                                            coord_bld->vec_type, "");
1500                cmpval11 = LLVMBuildBitCast(builder, cmpval11,
1501                                            coord_bld->vec_type, "");
1502                colors0[0] = lp_build_and(coord_bld, w00, cmpval00);
1503                tmp = lp_build_and(coord_bld, w01, cmpval01);
1504                colors0[0] = lp_build_add(coord_bld, tmp, colors0[0]);
1505                tmp = lp_build_and(coord_bld, w10, cmpval10);
1506                colors0[0] = lp_build_add(coord_bld, tmp, colors0[0]);
1507                tmp = lp_build_and(coord_bld, w11, cmpval11);
1508                colors0[0] = lp_build_add(coord_bld, tmp, colors0[0]);
1509                colors0[1] = colors0[2] = colors0[3] = colors0[0];
1510             }
1511          }
1512          else {
1513             /*
1514              * We don't have any weights to adjust, so instead calculate
1515              * the fourth texel as simply the average of the other 3.
1516              * (This would work for non-gather too, however we'd have
1517              * a boatload more of the select stuff due to there being
1518              * 4 times as many colors as weights.)
1519              */
1520             LLVMValueRef col00, col01, col10, col11;
1521             LLVMValueRef colc, colc0, colc1;
1522             col10 = lp_build_swizzle_soa_channel(texel_bld,
1523                                                  neighbors[1][0], chan_swiz);
1524             col11 = lp_build_swizzle_soa_channel(texel_bld,
1525                                                  neighbors[1][1], chan_swiz);
1526             col01 = lp_build_swizzle_soa_channel(texel_bld,
1527                                                  neighbors[0][1], chan_swiz);
1528             col00 = lp_build_swizzle_soa_channel(texel_bld,
1529                                                  neighbors[0][0], chan_swiz);
1530 
1531             /*
1532              * The spec says for comparison filtering, the comparison
1533              * must happen before synthesizing the new value.
1534              * This means all gathered values are always 0 or 1,
1535              * except for the non-existing texel, which can be 0,1/3,2/3,1...
1536              * Seems like we'd be allowed to just return 0 or 1 too, so we
1537              * could simplify and pass down the compare mask values to the
1538              * end (using int arithmetic/compare on the mask values to
1539              * construct the fourth texel) and only there convert to floats
1540              * but it's probably not worth it (it might be easier for the cpu
1541              * but not for the code)...
1542              */
1543             if (bld->static_sampler_state->compare_mode !=
1544                 PIPE_TEX_COMPARE_NONE) {
1545                LLVMValueRef cmpval00, cmpval01, cmpval10, cmpval11;
1546                cmpval00 = lp_build_sample_comparefunc(bld, coords[4], col00);
1547                cmpval01 = lp_build_sample_comparefunc(bld, coords[4], col01);
1548                cmpval10 = lp_build_sample_comparefunc(bld, coords[4], col10);
1549                cmpval11 = lp_build_sample_comparefunc(bld, coords[4], col11);
1550                col00 = lp_build_select(texel_bld, cmpval00,
1551                                        texel_bld->one, texel_bld->zero);
1552                col01 = lp_build_select(texel_bld, cmpval01,
1553                                        texel_bld->one, texel_bld->zero);
1554                col10 = lp_build_select(texel_bld, cmpval10,
1555                                        texel_bld->one, texel_bld->zero);
1556                col11 = lp_build_select(texel_bld, cmpval11,
1557                                        texel_bld->one, texel_bld->zero);
1558             }
1559 
1560             /*
1561              * Null out corner color.
1562              */
1563             col00 = lp_build_andnot(coord_bld, col00, c00f);
1564             col01 = lp_build_andnot(coord_bld, col01, c01f);
1565             col10 = lp_build_andnot(coord_bld, col10, c10f);
1566             col11 = lp_build_andnot(coord_bld, col11, c11f);
1567 
1568             /*
1569              * New corner texel color is all colors added / 3.
1570              */
1571             colc0 = lp_build_add(coord_bld, col00, col01);
1572             colc1 = lp_build_add(coord_bld, col10, col11);
1573             colc = lp_build_add(coord_bld, colc0, colc1);
1574             colc = lp_build_mul(coord_bld, one_third, colc);
1575 
1576             /*
1577              * Replace the corner texel color with the new value.
1578              */
1579             col00 = lp_build_select(coord_bld, c00, colc, col00);
1580             col01 = lp_build_select(coord_bld, c01, colc, col01);
1581             col10 = lp_build_select(coord_bld, c10, colc, col10);
1582             col11 = lp_build_select(coord_bld, c11, colc, col11);
1583 
1584             colors0[0] = col10;
1585             colors0[1] = col11;
1586             colors0[2] = col01;
1587             colors0[3] = col00;
1588          }
1589 
1590          LLVMBuildStore(builder, colors0[0], colorss[0]);
1591          LLVMBuildStore(builder, colors0[1], colorss[1]);
1592          LLVMBuildStore(builder, colors0[2], colorss[2]);
1593          LLVMBuildStore(builder, colors0[3], colorss[3]);
1594 
1595          lp_build_else(&corner_if);
1596       }
1597 
1598       if (bld->static_sampler_state->compare_mode == PIPE_TEX_COMPARE_NONE) {
1599          if (is_gather) {
1600             /*
1601              * Just assign the red channel (no component selection yet).
1602              * This is a bit hackish, we usually do the swizzle at the
1603              * end of sampling (much less values to swizzle), but this
1604              * obviously cannot work when using gather.
1605              */
1606             colors0[0] = lp_build_swizzle_soa_channel(texel_bld,
1607                                                       neighbors[1][0],
1608                                                       chan_swiz);
1609             colors0[1] = lp_build_swizzle_soa_channel(texel_bld,
1610                                                       neighbors[1][1],
1611                                                       chan_swiz);
1612             colors0[2] = lp_build_swizzle_soa_channel(texel_bld,
1613                                                       neighbors[0][1],
1614                                                       chan_swiz);
1615             colors0[3] = lp_build_swizzle_soa_channel(texel_bld,
1616                                                       neighbors[0][0],
1617                                                       chan_swiz);
1618          }
1619          else {
1620             /* Bilinear interpolate the four samples from the 2D image / 3D slice */
1621             for (chan = 0; chan < 4; chan++) {
1622                colors0[chan] = lp_build_lerp_2d(texel_bld,
1623                                                 s_fpart, t_fpart,
1624                                                 neighbors[0][0][chan],
1625                                                 neighbors[0][1][chan],
1626                                                 neighbors[1][0][chan],
1627                                                 neighbors[1][1][chan],
1628                                                 0);
1629             }
1630          }
1631       }
1632       else {
1633          LLVMValueRef cmpval00, cmpval01, cmpval10, cmpval11;
1634          cmpval00 = lp_build_sample_comparefunc(bld, coords[4], neighbors[0][0][0]);
1635          cmpval01 = lp_build_sample_comparefunc(bld, coords[4], neighbors[0][1][0]);
1636          cmpval10 = lp_build_sample_comparefunc(bld, coords[4], neighbors[1][0][0]);
1637          cmpval11 = lp_build_sample_comparefunc(bld, coords[4], neighbors[1][1][0]);
1638 
1639          if (is_gather) {
1640             /* more hacks for swizzling, should be X, ONE or ZERO... */
1641             colors0[0] = lp_build_select(texel_bld, cmpval10,
1642                                          texel_bld->one, texel_bld->zero);
1643             colors0[1] = lp_build_select(texel_bld, cmpval11,
1644                                          texel_bld->one, texel_bld->zero);
1645             colors0[2] = lp_build_select(texel_bld, cmpval01,
1646                                          texel_bld->one, texel_bld->zero);
1647             colors0[3] = lp_build_select(texel_bld, cmpval00,
1648                                          texel_bld->one, texel_bld->zero);
1649          }
1650          else {
1651             colors0[0] = lp_build_masklerp2d(texel_bld, s_fpart, t_fpart,
1652                                              cmpval00, cmpval01, cmpval10, cmpval11);
1653             colors0[1] = colors0[2] = colors0[3] = colors0[0];
1654          }
1655       }
1656 
1657       if (accurate_cube_corners) {
1658          LLVMBuildStore(builder, colors0[0], colorss[0]);
1659          LLVMBuildStore(builder, colors0[1], colorss[1]);
1660          LLVMBuildStore(builder, colors0[2], colorss[2]);
1661          LLVMBuildStore(builder, colors0[3], colorss[3]);
1662 
1663          lp_build_endif(&corner_if);
1664 
1665          colors0[0] = LLVMBuildLoad(builder, colorss[0], "");
1666          colors0[1] = LLVMBuildLoad(builder, colorss[1], "");
1667          colors0[2] = LLVMBuildLoad(builder, colorss[2], "");
1668          colors0[3] = LLVMBuildLoad(builder, colorss[3], "");
1669       }
1670 
1671       if (dims == 3) {
1672          LLVMValueRef neighbors1[2][2][4];
1673          LLVMValueRef colors1[4];
1674 
1675          assert(!is_gather);
1676 
1677          /* get x0/x1/y0/y1 texels at z1 */
1678          lp_build_sample_texel_soa(bld,
1679                                    width_vec, height_vec, depth_vec,
1680                                    x00, y00, z1,
1681                                    row_stride_vec, img_stride_vec,
1682                                    data_ptr, mipoffsets, neighbors1[0][0]);
1683          lp_build_sample_texel_soa(bld,
1684                                    width_vec, height_vec, depth_vec,
1685                                    x01, y01, z1,
1686                                    row_stride_vec, img_stride_vec,
1687                                    data_ptr, mipoffsets, neighbors1[0][1]);
1688          lp_build_sample_texel_soa(bld,
1689                                    width_vec, height_vec, depth_vec,
1690                                    x10, y10, z1,
1691                                    row_stride_vec, img_stride_vec,
1692                                    data_ptr, mipoffsets, neighbors1[1][0]);
1693          lp_build_sample_texel_soa(bld,
1694                                    width_vec, height_vec, depth_vec,
1695                                    x11, y11, z1,
1696                                    row_stride_vec, img_stride_vec,
1697                                    data_ptr, mipoffsets, neighbors1[1][1]);
1698 
1699          if (bld->static_sampler_state->compare_mode == PIPE_TEX_COMPARE_NONE) {
1700             /* Bilinear interpolate the four samples from the second Z slice */
1701             for (chan = 0; chan < 4; chan++) {
1702                colors1[chan] = lp_build_lerp_2d(texel_bld,
1703                                                 s_fpart, t_fpart,
1704                                                 neighbors1[0][0][chan],
1705                                                 neighbors1[0][1][chan],
1706                                                 neighbors1[1][0][chan],
1707                                                 neighbors1[1][1][chan],
1708                                                 0);
1709             }
1710             /* Linearly interpolate the two samples from the two 3D slices */
1711             for (chan = 0; chan < 4; chan++) {
1712                colors_out[chan] = lp_build_lerp(texel_bld,
1713                                                 r_fpart,
1714                                                 colors0[chan], colors1[chan],
1715                                                 0);
1716             }
1717          }
1718          else {
1719             LLVMValueRef cmpval00, cmpval01, cmpval10, cmpval11;
1720             cmpval00 = lp_build_sample_comparefunc(bld, coords[4], neighbors[0][0][0]);
1721             cmpval01 = lp_build_sample_comparefunc(bld, coords[4], neighbors[0][1][0]);
1722             cmpval10 = lp_build_sample_comparefunc(bld, coords[4], neighbors[1][0][0]);
1723             cmpval11 = lp_build_sample_comparefunc(bld, coords[4], neighbors[1][1][0]);
1724             colors1[0] = lp_build_masklerp2d(texel_bld, s_fpart, t_fpart,
1725                                              cmpval00, cmpval01, cmpval10, cmpval11);
1726             /* Linearly interpolate the two samples from the two 3D slices */
1727             colors_out[0] = lp_build_lerp(texel_bld,
1728                                           r_fpart,
1729                                           colors0[0], colors1[0],
1730                                           0);
1731             colors_out[1] = colors_out[2] = colors_out[3] = colors_out[0];
1732          }
1733       }
1734       else {
1735          /* 2D tex */
1736          for (chan = 0; chan < 4; chan++) {
1737             colors_out[chan] = colors0[chan];
1738          }
1739       }
1740    }
1741    if (is_gather) {
1742       /*
1743        * For gather, we can't do our usual channel swizzling done later,
1744        * so do it here. It only really matters for 0/1 swizzles in case
1745        * of comparison filtering, since in this case the results would be
1746        * wrong, without comparison it should all work out alright but it
1747        * can't hurt to do that here, since it will instantly drop all
1748        * calculations above, though it's a rather stupid idea to do
1749        * gather on a channel which will always return 0 or 1 in any case...
1750        */
1751       if (chan_swiz == PIPE_SWIZZLE_1) {
1752          for (chan = 0; chan < 4; chan++) {
1753             colors_out[chan] = texel_bld->one;
1754          }
1755       } else if (chan_swiz == PIPE_SWIZZLE_0) {
1756          for (chan = 0; chan < 4; chan++) {
1757             colors_out[chan] = texel_bld->zero;
1758          }
1759       }
1760    }
1761 }
1762 
1763 
1764 /**
1765  * Sample the texture/mipmap using given image filter and mip filter.
1766  * ilevel0 and ilevel1 indicate the two mipmap levels to sample
1767  * from (vectors or scalars).
1768  * If we're using nearest miplevel sampling the '1' values will be null/unused.
1769  */
1770 static void
lp_build_sample_mipmap(struct lp_build_sample_context * bld,unsigned img_filter,unsigned mip_filter,boolean is_gather,const LLVMValueRef * coords,const LLVMValueRef * offsets,LLVMValueRef ilevel0,LLVMValueRef ilevel1,LLVMValueRef lod_fpart,LLVMValueRef * colors_out)1771 lp_build_sample_mipmap(struct lp_build_sample_context *bld,
1772                        unsigned img_filter,
1773                        unsigned mip_filter,
1774                        boolean is_gather,
1775                        const LLVMValueRef *coords,
1776                        const LLVMValueRef *offsets,
1777                        LLVMValueRef ilevel0,
1778                        LLVMValueRef ilevel1,
1779                        LLVMValueRef lod_fpart,
1780                        LLVMValueRef *colors_out)
1781 {
1782    LLVMBuilderRef builder = bld->gallivm->builder;
1783    LLVMValueRef size0 = NULL;
1784    LLVMValueRef size1 = NULL;
1785    LLVMValueRef row_stride0_vec = NULL;
1786    LLVMValueRef row_stride1_vec = NULL;
1787    LLVMValueRef img_stride0_vec = NULL;
1788    LLVMValueRef img_stride1_vec = NULL;
1789    LLVMValueRef data_ptr0 = NULL;
1790    LLVMValueRef data_ptr1 = NULL;
1791    LLVMValueRef mipoff0 = NULL;
1792    LLVMValueRef mipoff1 = NULL;
1793    LLVMValueRef colors0[4], colors1[4];
1794    unsigned chan;
1795 
1796    /* sample the first mipmap level */
1797    lp_build_mipmap_level_sizes(bld, ilevel0,
1798                                &size0,
1799                                &row_stride0_vec, &img_stride0_vec);
1800    if (bld->num_mips == 1) {
1801       data_ptr0 = lp_build_get_mipmap_level(bld, ilevel0);
1802    }
1803    else {
1804       /* This path should work for num_lods 1 too but slightly less efficient */
1805       data_ptr0 = bld->base_ptr;
1806       mipoff0 = lp_build_get_mip_offsets(bld, ilevel0);
1807    }
1808    if (img_filter == PIPE_TEX_FILTER_NEAREST) {
1809       lp_build_sample_image_nearest(bld, size0,
1810                                     row_stride0_vec, img_stride0_vec,
1811                                     data_ptr0, mipoff0, coords, offsets,
1812                                     colors0);
1813    }
1814    else {
1815       assert(img_filter == PIPE_TEX_FILTER_LINEAR);
1816       lp_build_sample_image_linear(bld, is_gather, size0, NULL,
1817                                    row_stride0_vec, img_stride0_vec,
1818                                    data_ptr0, mipoff0, coords, offsets,
1819                                    colors0);
1820    }
1821 
1822    /* Store the first level's colors in the output variables */
1823    for (chan = 0; chan < 4; chan++) {
1824        LLVMBuildStore(builder, colors0[chan], colors_out[chan]);
1825    }
1826 
1827    if (mip_filter == PIPE_TEX_MIPFILTER_LINEAR) {
1828       struct lp_build_if_state if_ctx;
1829       LLVMValueRef need_lerp;
1830 
1831       /* need_lerp = lod_fpart > 0 */
1832       if (bld->num_lods == 1) {
1833          need_lerp = LLVMBuildFCmp(builder, LLVMRealUGT,
1834                                    lod_fpart, bld->lodf_bld.zero,
1835                                    "need_lerp");
1836       }
1837       else {
1838          /*
1839           * We'll do mip filtering if any of the quads (or individual
1840           * pixel in case of per-pixel lod) need it.
1841           * It might be better to split the vectors here and only fetch/filter
1842           * quads which need it (if there's one lod per quad).
1843           */
1844          need_lerp = lp_build_compare(bld->gallivm, bld->lodf_bld.type,
1845                                       PIPE_FUNC_GREATER,
1846                                       lod_fpart, bld->lodf_bld.zero);
1847          need_lerp = lp_build_any_true_range(&bld->lodi_bld, bld->num_lods, need_lerp);
1848          lp_build_name(need_lerp, "need_lerp");
1849       }
1850 
1851       lp_build_if(&if_ctx, bld->gallivm, need_lerp);
1852       {
1853          /*
1854           * We unfortunately need to clamp lod_fpart here since we can get
1855           * negative values which would screw up filtering if not all
1856           * lod_fpart values have same sign.
1857           */
1858          lod_fpart = lp_build_max(&bld->lodf_bld, lod_fpart,
1859                                   bld->lodf_bld.zero);
1860          /* sample the second mipmap level */
1861          lp_build_mipmap_level_sizes(bld, ilevel1,
1862                                      &size1,
1863                                      &row_stride1_vec, &img_stride1_vec);
1864          if (bld->num_mips == 1) {
1865             data_ptr1 = lp_build_get_mipmap_level(bld, ilevel1);
1866          }
1867          else {
1868             data_ptr1 = bld->base_ptr;
1869             mipoff1 = lp_build_get_mip_offsets(bld, ilevel1);
1870          }
1871          if (img_filter == PIPE_TEX_FILTER_NEAREST) {
1872             lp_build_sample_image_nearest(bld, size1,
1873                                           row_stride1_vec, img_stride1_vec,
1874                                           data_ptr1, mipoff1, coords, offsets,
1875                                           colors1);
1876          }
1877          else {
1878             lp_build_sample_image_linear(bld, FALSE, size1, NULL,
1879                                          row_stride1_vec, img_stride1_vec,
1880                                          data_ptr1, mipoff1, coords, offsets,
1881                                          colors1);
1882          }
1883 
1884          /* interpolate samples from the two mipmap levels */
1885 
1886          if (bld->num_lods != bld->coord_type.length)
1887             lod_fpart = lp_build_unpack_broadcast_aos_scalars(bld->gallivm,
1888                                                               bld->lodf_bld.type,
1889                                                               bld->texel_bld.type,
1890                                                               lod_fpart);
1891 
1892          for (chan = 0; chan < 4; chan++) {
1893             colors0[chan] = lp_build_lerp(&bld->texel_bld, lod_fpart,
1894                                           colors0[chan], colors1[chan],
1895                                           0);
1896             LLVMBuildStore(builder, colors0[chan], colors_out[chan]);
1897          }
1898       }
1899       lp_build_endif(&if_ctx);
1900    }
1901 }
1902 
1903 
1904 /**
1905  * Sample the texture/mipmap using given mip filter, and using
1906  * both nearest and linear filtering at the same time depending
1907  * on linear_mask.
1908  * lod can be per quad but linear_mask is always per pixel.
1909  * ilevel0 and ilevel1 indicate the two mipmap levels to sample
1910  * from (vectors or scalars).
1911  * If we're using nearest miplevel sampling the '1' values will be null/unused.
1912  */
1913 static void
lp_build_sample_mipmap_both(struct lp_build_sample_context * bld,LLVMValueRef linear_mask,unsigned mip_filter,const LLVMValueRef * coords,const LLVMValueRef * offsets,LLVMValueRef ilevel0,LLVMValueRef ilevel1,LLVMValueRef lod_fpart,LLVMValueRef lod_positive,LLVMValueRef * colors_out)1914 lp_build_sample_mipmap_both(struct lp_build_sample_context *bld,
1915                             LLVMValueRef linear_mask,
1916                             unsigned mip_filter,
1917                             const LLVMValueRef *coords,
1918                             const LLVMValueRef *offsets,
1919                             LLVMValueRef ilevel0,
1920                             LLVMValueRef ilevel1,
1921                             LLVMValueRef lod_fpart,
1922                             LLVMValueRef lod_positive,
1923                             LLVMValueRef *colors_out)
1924 {
1925    LLVMBuilderRef builder = bld->gallivm->builder;
1926    LLVMValueRef size0 = NULL;
1927    LLVMValueRef size1 = NULL;
1928    LLVMValueRef row_stride0_vec = NULL;
1929    LLVMValueRef row_stride1_vec = NULL;
1930    LLVMValueRef img_stride0_vec = NULL;
1931    LLVMValueRef img_stride1_vec = NULL;
1932    LLVMValueRef data_ptr0 = NULL;
1933    LLVMValueRef data_ptr1 = NULL;
1934    LLVMValueRef mipoff0 = NULL;
1935    LLVMValueRef mipoff1 = NULL;
1936    LLVMValueRef colors0[4], colors1[4];
1937    unsigned chan;
1938 
1939    /* sample the first mipmap level */
1940    lp_build_mipmap_level_sizes(bld, ilevel0,
1941                                &size0,
1942                                &row_stride0_vec, &img_stride0_vec);
1943    if (bld->num_mips == 1) {
1944       data_ptr0 = lp_build_get_mipmap_level(bld, ilevel0);
1945    }
1946    else {
1947       /* This path should work for num_lods 1 too but slightly less efficient */
1948       data_ptr0 = bld->base_ptr;
1949       mipoff0 = lp_build_get_mip_offsets(bld, ilevel0);
1950    }
1951 
1952    lp_build_sample_image_linear(bld, FALSE, size0, linear_mask,
1953                                 row_stride0_vec, img_stride0_vec,
1954                                 data_ptr0, mipoff0, coords, offsets,
1955                                 colors0);
1956 
1957    /* Store the first level's colors in the output variables */
1958    for (chan = 0; chan < 4; chan++) {
1959        LLVMBuildStore(builder, colors0[chan], colors_out[chan]);
1960    }
1961 
1962    if (mip_filter == PIPE_TEX_MIPFILTER_LINEAR) {
1963       struct lp_build_if_state if_ctx;
1964       LLVMValueRef need_lerp;
1965 
1966       /*
1967        * We'll do mip filtering if any of the quads (or individual
1968        * pixel in case of per-pixel lod) need it.
1969        * Note using lod_positive here not lod_fpart since it may be the same
1970        * condition as that used in the outer "if" in the caller hence llvm
1971        * should be able to merge the branches in this case.
1972        */
1973       need_lerp = lp_build_any_true_range(&bld->lodi_bld, bld->num_lods, lod_positive);
1974       lp_build_name(need_lerp, "need_lerp");
1975 
1976       lp_build_if(&if_ctx, bld->gallivm, need_lerp);
1977       {
1978          /*
1979           * We unfortunately need to clamp lod_fpart here since we can get
1980           * negative values which would screw up filtering if not all
1981           * lod_fpart values have same sign.
1982           */
1983          lod_fpart = lp_build_max(&bld->lodf_bld, lod_fpart,
1984                                   bld->lodf_bld.zero);
1985          /* sample the second mipmap level */
1986          lp_build_mipmap_level_sizes(bld, ilevel1,
1987                                      &size1,
1988                                      &row_stride1_vec, &img_stride1_vec);
1989          if (bld->num_mips == 1) {
1990             data_ptr1 = lp_build_get_mipmap_level(bld, ilevel1);
1991          }
1992          else {
1993             data_ptr1 = bld->base_ptr;
1994             mipoff1 = lp_build_get_mip_offsets(bld, ilevel1);
1995          }
1996 
1997          lp_build_sample_image_linear(bld, FALSE, size1, linear_mask,
1998                                       row_stride1_vec, img_stride1_vec,
1999                                       data_ptr1, mipoff1, coords, offsets,
2000                                       colors1);
2001 
2002          /* interpolate samples from the two mipmap levels */
2003 
2004          if (bld->num_lods != bld->coord_type.length)
2005             lod_fpart = lp_build_unpack_broadcast_aos_scalars(bld->gallivm,
2006                                                               bld->lodf_bld.type,
2007                                                               bld->texel_bld.type,
2008                                                               lod_fpart);
2009 
2010          for (chan = 0; chan < 4; chan++) {
2011             colors0[chan] = lp_build_lerp(&bld->texel_bld, lod_fpart,
2012                                           colors0[chan], colors1[chan],
2013                                           0);
2014             LLVMBuildStore(builder, colors0[chan], colors_out[chan]);
2015          }
2016       }
2017       lp_build_endif(&if_ctx);
2018    }
2019 }
2020 
2021 
2022 /**
2023  * Build (per-coord) layer value.
2024  * Either clamp layer to valid values or fill in optional out_of_bounds
2025  * value and just return value unclamped.
2026  */
2027 static LLVMValueRef
lp_build_layer_coord(struct lp_build_sample_context * bld,unsigned texture_unit,boolean is_cube_array,LLVMValueRef layer,LLVMValueRef * out_of_bounds)2028 lp_build_layer_coord(struct lp_build_sample_context *bld,
2029                      unsigned texture_unit,
2030                      boolean is_cube_array,
2031                      LLVMValueRef layer,
2032                      LLVMValueRef *out_of_bounds)
2033 {
2034    LLVMValueRef num_layers;
2035    struct lp_build_context *int_coord_bld = &bld->int_coord_bld;
2036 
2037    num_layers = bld->dynamic_state->depth(bld->dynamic_state, bld->gallivm,
2038                                           bld->context_ptr, texture_unit, NULL);
2039 
2040    if (out_of_bounds) {
2041       LLVMValueRef out1, out;
2042       assert(!is_cube_array);
2043       num_layers = lp_build_broadcast_scalar(int_coord_bld, num_layers);
2044       out = lp_build_cmp(int_coord_bld, PIPE_FUNC_LESS, layer, int_coord_bld->zero);
2045       out1 = lp_build_cmp(int_coord_bld, PIPE_FUNC_GEQUAL, layer, num_layers);
2046       *out_of_bounds = lp_build_or(int_coord_bld, out, out1);
2047       return layer;
2048    }
2049    else {
2050       LLVMValueRef maxlayer;
2051       LLVMValueRef s = is_cube_array ? lp_build_const_int32(bld->gallivm, 6) :
2052                                        bld->int_bld.one;
2053       maxlayer = lp_build_sub(&bld->int_bld, num_layers, s);
2054       maxlayer = lp_build_broadcast_scalar(int_coord_bld, maxlayer);
2055       return lp_build_clamp(int_coord_bld, layer, int_coord_bld->zero, maxlayer);
2056    }
2057 }
2058 
2059 
2060 /**
2061  * Calculate cube face, lod, mip levels.
2062  */
2063 static void
lp_build_sample_common(struct lp_build_sample_context * bld,boolean is_lodq,unsigned texture_index,unsigned sampler_index,LLVMValueRef * coords,const struct lp_derivatives * derivs,LLVMValueRef lod_bias,LLVMValueRef explicit_lod,LLVMValueRef * lod_pos_or_zero,LLVMValueRef * lod,LLVMValueRef * lod_fpart,LLVMValueRef * ilevel0,LLVMValueRef * ilevel1)2064 lp_build_sample_common(struct lp_build_sample_context *bld,
2065                        boolean is_lodq,
2066                        unsigned texture_index,
2067                        unsigned sampler_index,
2068                        LLVMValueRef *coords,
2069                        const struct lp_derivatives *derivs, /* optional */
2070                        LLVMValueRef lod_bias, /* optional */
2071                        LLVMValueRef explicit_lod, /* optional */
2072                        LLVMValueRef *lod_pos_or_zero,
2073                        LLVMValueRef *lod,
2074                        LLVMValueRef *lod_fpart,
2075                        LLVMValueRef *ilevel0,
2076                        LLVMValueRef *ilevel1)
2077 {
2078    const unsigned mip_filter = bld->static_sampler_state->min_mip_filter;
2079    const unsigned min_filter = bld->static_sampler_state->min_img_filter;
2080    const unsigned mag_filter = bld->static_sampler_state->mag_img_filter;
2081    const unsigned target = bld->static_texture_state->target;
2082    LLVMValueRef first_level, cube_rho = NULL;
2083    LLVMValueRef lod_ipart = NULL;
2084    struct lp_derivatives cube_derivs;
2085 
2086    /*
2087    printf("%s mip %d  min %d  mag %d\n", __FUNCTION__,
2088           mip_filter, min_filter, mag_filter);
2089    */
2090 
2091    /*
2092     * Choose cube face, recompute texcoords for the chosen face and
2093     * compute rho here too (as it requires transform of derivatives).
2094     */
2095    if (target == PIPE_TEXTURE_CUBE || target == PIPE_TEXTURE_CUBE_ARRAY) {
2096       boolean need_derivs;
2097       need_derivs = ((min_filter != mag_filter ||
2098                       mip_filter != PIPE_TEX_MIPFILTER_NONE) &&
2099                       !bld->static_sampler_state->min_max_lod_equal &&
2100                       !explicit_lod);
2101       lp_build_cube_lookup(bld, coords, derivs, &cube_rho, &cube_derivs, need_derivs);
2102       derivs = &cube_derivs;
2103       if (target == PIPE_TEXTURE_CUBE_ARRAY && !is_lodq) {
2104          /* calculate cube layer coord now */
2105          LLVMValueRef layer = lp_build_iround(&bld->coord_bld, coords[3]);
2106          LLVMValueRef six = lp_build_const_int_vec(bld->gallivm, bld->int_coord_type, 6);
2107          layer = lp_build_mul(&bld->int_coord_bld, layer, six);
2108          coords[3] = lp_build_layer_coord(bld, texture_index, TRUE, layer, NULL);
2109          /* because of seamless filtering can't add it to face (coords[2]) here. */
2110       }
2111    }
2112    else if ((target == PIPE_TEXTURE_1D_ARRAY ||
2113              target == PIPE_TEXTURE_2D_ARRAY) && !is_lodq) {
2114       coords[2] = lp_build_iround(&bld->coord_bld, coords[2]);
2115       coords[2] = lp_build_layer_coord(bld, texture_index, FALSE, coords[2], NULL);
2116    }
2117 
2118    if (bld->static_sampler_state->compare_mode != PIPE_TEX_COMPARE_NONE) {
2119       /*
2120        * Clamp p coords to [0,1] for fixed function depth texture format here.
2121        * Technically this is not entirely correct for unorm depth as the ref value
2122        * should be converted to the depth format (quantization!) and comparison
2123        * then done in texture format. This would actually help performance (since
2124        * only need to do it once and could save the per-sample conversion of texels
2125        * to floats instead), but it would need more messy code (would need to push
2126        * at least some bits down to actual fetch so conversion could be skipped,
2127        * and would have ugly interaction with border color, would need to convert
2128        * border color to that format too or do some other tricks to make it work).
2129        */
2130       const struct util_format_description *format_desc = bld->format_desc;
2131       unsigned chan_type;
2132       /* not entirely sure we couldn't end up with non-valid swizzle here */
2133       chan_type = format_desc->swizzle[0] <= PIPE_SWIZZLE_W ?
2134                      format_desc->channel[format_desc->swizzle[0]].type :
2135                      UTIL_FORMAT_TYPE_FLOAT;
2136       if (chan_type != UTIL_FORMAT_TYPE_FLOAT) {
2137          coords[4] = lp_build_clamp(&bld->coord_bld, coords[4],
2138                                     bld->coord_bld.zero, bld->coord_bld.one);
2139       }
2140    }
2141 
2142    /*
2143     * Compute the level of detail (float).
2144     */
2145    if (min_filter != mag_filter ||
2146        mip_filter != PIPE_TEX_MIPFILTER_NONE || is_lodq) {
2147       /* Need to compute lod either to choose mipmap levels or to
2148        * distinguish between minification/magnification with one mipmap level.
2149        */
2150       lp_build_lod_selector(bld, is_lodq, texture_index, sampler_index,
2151                             coords[0], coords[1], coords[2], cube_rho,
2152                             derivs, lod_bias, explicit_lod,
2153                             mip_filter, lod,
2154                             &lod_ipart, lod_fpart, lod_pos_or_zero);
2155       if (is_lodq) {
2156          LLVMValueRef last_level;
2157          last_level = bld->dynamic_state->last_level(bld->dynamic_state,
2158                                                      bld->gallivm,
2159                                                      bld->context_ptr,
2160                                                      texture_index, NULL);
2161          first_level = bld->dynamic_state->first_level(bld->dynamic_state,
2162                                                        bld->gallivm,
2163                                                        bld->context_ptr,
2164                                                        texture_index, NULL);
2165          last_level = lp_build_sub(&bld->int_bld, last_level, first_level);
2166          last_level = lp_build_int_to_float(&bld->float_bld, last_level);
2167          last_level = lp_build_broadcast_scalar(&bld->lodf_bld, last_level);
2168 
2169          switch (mip_filter) {
2170          case PIPE_TEX_MIPFILTER_NONE:
2171             *lod_fpart = bld->lodf_bld.zero;
2172             break;
2173          case PIPE_TEX_MIPFILTER_NEAREST:
2174              *lod_fpart = lp_build_round(&bld->lodf_bld, *lod_fpart);
2175              /* fallthrough */
2176          case PIPE_TEX_MIPFILTER_LINEAR:
2177             *lod_fpart = lp_build_clamp(&bld->lodf_bld, *lod_fpart,
2178                                         bld->lodf_bld.zero, last_level);
2179             break;
2180          }
2181          return;
2182       }
2183 
2184    } else {
2185       lod_ipart = bld->lodi_bld.zero;
2186       *lod_pos_or_zero = bld->lodi_bld.zero;
2187    }
2188 
2189    if (bld->num_lods != bld->num_mips) {
2190       /* only makes sense if there's just a single mip level */
2191       assert(bld->num_mips == 1);
2192       lod_ipart = lp_build_extract_range(bld->gallivm, lod_ipart, 0, 1);
2193    }
2194 
2195    /*
2196     * Compute integer mipmap level(s) to fetch texels from: ilevel0, ilevel1
2197     */
2198    switch (mip_filter) {
2199    default:
2200       assert(0 && "bad mip_filter value in lp_build_sample_soa()");
2201       /* fall-through */
2202    case PIPE_TEX_MIPFILTER_NONE:
2203       /* always use mip level 0 */
2204       first_level = bld->dynamic_state->first_level(bld->dynamic_state,
2205                                                     bld->gallivm, bld->context_ptr,
2206                                                     texture_index, NULL);
2207       first_level = lp_build_broadcast_scalar(&bld->leveli_bld, first_level);
2208       *ilevel0 = first_level;
2209       break;
2210    case PIPE_TEX_MIPFILTER_NEAREST:
2211       assert(lod_ipart);
2212       lp_build_nearest_mip_level(bld, texture_index, lod_ipart, ilevel0, NULL);
2213       break;
2214    case PIPE_TEX_MIPFILTER_LINEAR:
2215       assert(lod_ipart);
2216       assert(*lod_fpart);
2217       lp_build_linear_mip_levels(bld, texture_index,
2218                                  lod_ipart, lod_fpart,
2219                                  ilevel0, ilevel1);
2220       break;
2221    }
2222 }
2223 
2224 static void
lp_build_clamp_border_color(struct lp_build_sample_context * bld,unsigned sampler_unit)2225 lp_build_clamp_border_color(struct lp_build_sample_context *bld,
2226                             unsigned sampler_unit)
2227 {
2228    struct gallivm_state *gallivm = bld->gallivm;
2229    LLVMBuilderRef builder = gallivm->builder;
2230    LLVMValueRef border_color_ptr =
2231       bld->dynamic_state->border_color(bld->dynamic_state, gallivm,
2232                                        bld->context_ptr, sampler_unit);
2233    LLVMValueRef border_color;
2234    const struct util_format_description *format_desc = bld->format_desc;
2235    struct lp_type vec4_type = bld->texel_type;
2236    struct lp_build_context vec4_bld;
2237    LLVMValueRef min_clamp = NULL;
2238    LLVMValueRef max_clamp = NULL;
2239 
2240    /*
2241     * For normalized format need to clamp border color (technically
2242     * probably should also quantize the data). Really sucks doing this
2243     * here but can't avoid at least for now since this is part of
2244     * sampler state and texture format is part of sampler_view state.
2245     * GL expects also expects clamping for uint/sint formats too so
2246     * do that as well (d3d10 can't end up here with uint/sint since it
2247     * only supports them with ld).
2248     */
2249    vec4_type.length = 4;
2250    lp_build_context_init(&vec4_bld, gallivm, vec4_type);
2251 
2252    /*
2253     * Vectorized clamping of border color. Loading is a bit of a hack since
2254     * we just cast the pointer to float array to pointer to vec4
2255     * (int or float).
2256     */
2257    border_color_ptr = lp_build_array_get_ptr(gallivm, border_color_ptr,
2258                                              lp_build_const_int32(gallivm, 0));
2259    border_color_ptr = LLVMBuildBitCast(builder, border_color_ptr,
2260                                        LLVMPointerType(vec4_bld.vec_type, 0), "");
2261    border_color = LLVMBuildLoad(builder, border_color_ptr, "");
2262    /* we don't have aligned type in the dynamic state unfortunately */
2263    LLVMSetAlignment(border_color, 4);
2264 
2265    /*
2266     * Instead of having some incredibly complex logic which will try to figure out
2267     * clamping necessary for each channel, simply use the first channel, and treat
2268     * mixed signed/unsigned normalized formats specially.
2269     * (Mixed non-normalized, which wouldn't work at all here, do not exist for a
2270     * good reason.)
2271     */
2272    if (format_desc->layout == UTIL_FORMAT_LAYOUT_PLAIN) {
2273       int chan;
2274       /* d/s needs special handling because both present means just sampling depth */
2275       if (util_format_is_depth_and_stencil(format_desc->format)) {
2276          chan = format_desc->swizzle[0];
2277       }
2278       else {
2279          chan = util_format_get_first_non_void_channel(format_desc->format);
2280       }
2281       if (chan >= 0 && chan <= PIPE_SWIZZLE_W) {
2282          unsigned chan_type = format_desc->channel[chan].type;
2283          unsigned chan_norm = format_desc->channel[chan].normalized;
2284          unsigned chan_pure = format_desc->channel[chan].pure_integer;
2285          if (chan_type == UTIL_FORMAT_TYPE_SIGNED) {
2286             if (chan_norm) {
2287                min_clamp = lp_build_const_vec(gallivm, vec4_type, -1.0F);
2288                max_clamp = vec4_bld.one;
2289             }
2290             else if (chan_pure) {
2291                /*
2292                 * Border color was stored as int, hence need min/max clamp
2293                 * only if chan has less than 32 bits..
2294                 */
2295                unsigned chan_size = format_desc->channel[chan].size;
2296                if (chan_size < 32) {
2297                   min_clamp = lp_build_const_int_vec(gallivm, vec4_type,
2298                                                      0 - (1 << (chan_size - 1)));
2299                   max_clamp = lp_build_const_int_vec(gallivm, vec4_type,
2300                                                      (1 << (chan_size - 1)) - 1);
2301                }
2302             }
2303             /* TODO: no idea about non-pure, non-normalized! */
2304          }
2305          else if (chan_type == UTIL_FORMAT_TYPE_UNSIGNED) {
2306             if (chan_norm) {
2307                min_clamp = vec4_bld.zero;
2308                max_clamp = vec4_bld.one;
2309             }
2310             /*
2311              * Need a ugly hack here, because we don't have Z32_FLOAT_X8X24
2312              * we use Z32_FLOAT_S8X24 to imply sampling depth component
2313              * and ignoring stencil, which will blow up here if we try to
2314              * do a uint clamp in a float texel build...
2315              * And even if we had that format, mesa st also thinks using z24s8
2316              * means depth sampling ignoring stencil.
2317              */
2318             else if (chan_pure) {
2319                /*
2320                 * Border color was stored as uint, hence never need min
2321                 * clamp, and only need max clamp if chan has less than 32 bits.
2322                 */
2323                unsigned chan_size = format_desc->channel[chan].size;
2324                if (chan_size < 32) {
2325                   max_clamp = lp_build_const_int_vec(gallivm, vec4_type,
2326                                                      (1 << chan_size) - 1);
2327                }
2328                /* TODO: no idea about non-pure, non-normalized! */
2329             }
2330          }
2331          else if (chan_type == UTIL_FORMAT_TYPE_FIXED) {
2332             /* TODO: I have no idea what clamp this would need if any! */
2333          }
2334       }
2335       /* mixed plain formats (or different pure size) */
2336       switch (format_desc->format) {
2337       case PIPE_FORMAT_B10G10R10A2_UINT:
2338       case PIPE_FORMAT_R10G10B10A2_UINT:
2339       {
2340          unsigned max10 = (1 << 10) - 1;
2341          max_clamp = lp_build_const_aos(gallivm, vec4_type, max10, max10,
2342                                         max10, (1 << 2) - 1, NULL);
2343       }
2344          break;
2345       case PIPE_FORMAT_R10SG10SB10SA2U_NORM:
2346          min_clamp = lp_build_const_aos(gallivm, vec4_type, -1.0F, -1.0F,
2347                                         -1.0F, 0.0F, NULL);
2348          max_clamp = vec4_bld.one;
2349          break;
2350       case PIPE_FORMAT_R8SG8SB8UX8U_NORM:
2351       case PIPE_FORMAT_R5SG5SB6U_NORM:
2352          min_clamp = lp_build_const_aos(gallivm, vec4_type, -1.0F, -1.0F,
2353                                         0.0F, 0.0F, NULL);
2354          max_clamp = vec4_bld.one;
2355          break;
2356       default:
2357          break;
2358       }
2359    }
2360    else {
2361       /* cannot figure this out from format description */
2362       if (format_desc->layout == UTIL_FORMAT_LAYOUT_S3TC) {
2363          /* s3tc formats are always unorm */
2364          min_clamp = vec4_bld.zero;
2365          max_clamp = vec4_bld.one;
2366       }
2367       else if (format_desc->layout == UTIL_FORMAT_LAYOUT_RGTC ||
2368                format_desc->layout == UTIL_FORMAT_LAYOUT_ETC ||
2369                format_desc->layout == UTIL_FORMAT_LAYOUT_BPTC) {
2370          switch (format_desc->format) {
2371          case PIPE_FORMAT_RGTC1_UNORM:
2372          case PIPE_FORMAT_RGTC2_UNORM:
2373          case PIPE_FORMAT_LATC1_UNORM:
2374          case PIPE_FORMAT_LATC2_UNORM:
2375          case PIPE_FORMAT_ETC1_RGB8:
2376          case PIPE_FORMAT_BPTC_RGBA_UNORM:
2377          case PIPE_FORMAT_BPTC_SRGBA:
2378             min_clamp = vec4_bld.zero;
2379             max_clamp = vec4_bld.one;
2380             break;
2381          case PIPE_FORMAT_RGTC1_SNORM:
2382          case PIPE_FORMAT_RGTC2_SNORM:
2383          case PIPE_FORMAT_LATC1_SNORM:
2384          case PIPE_FORMAT_LATC2_SNORM:
2385             min_clamp = lp_build_const_vec(gallivm, vec4_type, -1.0F);
2386             max_clamp = vec4_bld.one;
2387             break;
2388          case PIPE_FORMAT_BPTC_RGB_FLOAT:
2389             /* not sure if we should clamp to max half float? */
2390             break;
2391          case PIPE_FORMAT_BPTC_RGB_UFLOAT:
2392             min_clamp = vec4_bld.zero;
2393             break;
2394          default:
2395             assert(0);
2396             break;
2397          }
2398       }
2399       /*
2400        * all others from subsampled/other group, though we don't care
2401        * about yuv (and should not have any from zs here)
2402        */
2403       else if (format_desc->colorspace != UTIL_FORMAT_COLORSPACE_YUV){
2404          switch (format_desc->format) {
2405          case PIPE_FORMAT_R8G8_B8G8_UNORM:
2406          case PIPE_FORMAT_G8R8_G8B8_UNORM:
2407          case PIPE_FORMAT_G8R8_B8R8_UNORM:
2408          case PIPE_FORMAT_R8G8_R8B8_UNORM:
2409          case PIPE_FORMAT_R1_UNORM: /* doesn't make sense but ah well */
2410             min_clamp = vec4_bld.zero;
2411             max_clamp = vec4_bld.one;
2412             break;
2413          case PIPE_FORMAT_R8G8Bx_SNORM:
2414             min_clamp = lp_build_const_vec(gallivm, vec4_type, -1.0F);
2415             max_clamp = vec4_bld.one;
2416             break;
2417             /*
2418              * Note smallfloat formats usually don't need clamping
2419              * (they still have infinite range) however this is not
2420              * true for r11g11b10 and r9g9b9e5, which can't represent
2421              * negative numbers (and additionally r9g9b9e5 can't represent
2422              * very large numbers). d3d10 seems happy without clamping in
2423              * this case, but gl spec is pretty clear: "for floating
2424              * point and integer formats, border values are clamped to
2425              * the representable range of the format" so do that here.
2426              */
2427          case PIPE_FORMAT_R11G11B10_FLOAT:
2428             min_clamp = vec4_bld.zero;
2429             break;
2430          case PIPE_FORMAT_R9G9B9E5_FLOAT:
2431             min_clamp = vec4_bld.zero;
2432             max_clamp = lp_build_const_vec(gallivm, vec4_type, MAX_RGB9E5);
2433             break;
2434          default:
2435             assert(0);
2436             break;
2437          }
2438       }
2439    }
2440 
2441    if (min_clamp) {
2442       border_color = lp_build_max(&vec4_bld, border_color, min_clamp);
2443    }
2444    if (max_clamp) {
2445       border_color = lp_build_min(&vec4_bld, border_color, max_clamp);
2446    }
2447 
2448    bld->border_color_clamped = border_color;
2449 }
2450 
2451 
2452 /**
2453  * General texture sampling codegen.
2454  * This function handles texture sampling for all texture targets (1D,
2455  * 2D, 3D, cube) and all filtering modes.
2456  */
2457 static void
lp_build_sample_general(struct lp_build_sample_context * bld,unsigned sampler_unit,boolean is_gather,const LLVMValueRef * coords,const LLVMValueRef * offsets,LLVMValueRef lod_positive,LLVMValueRef lod_fpart,LLVMValueRef ilevel0,LLVMValueRef ilevel1,LLVMValueRef * colors_out)2458 lp_build_sample_general(struct lp_build_sample_context *bld,
2459                         unsigned sampler_unit,
2460                         boolean is_gather,
2461                         const LLVMValueRef *coords,
2462                         const LLVMValueRef *offsets,
2463                         LLVMValueRef lod_positive,
2464                         LLVMValueRef lod_fpart,
2465                         LLVMValueRef ilevel0,
2466                         LLVMValueRef ilevel1,
2467                         LLVMValueRef *colors_out)
2468 {
2469    LLVMBuilderRef builder = bld->gallivm->builder;
2470    const struct lp_static_sampler_state *sampler_state = bld->static_sampler_state;
2471    const unsigned mip_filter = sampler_state->min_mip_filter;
2472    const unsigned min_filter = sampler_state->min_img_filter;
2473    const unsigned mag_filter = sampler_state->mag_img_filter;
2474    LLVMValueRef texels[4];
2475    unsigned chan;
2476 
2477    /* if we need border color, (potentially) clamp it now */
2478    if (lp_sampler_wrap_mode_uses_border_color(sampler_state->wrap_s,
2479                                               min_filter,
2480                                               mag_filter) ||
2481        (bld->dims > 1 &&
2482            lp_sampler_wrap_mode_uses_border_color(sampler_state->wrap_t,
2483                                                   min_filter,
2484                                                   mag_filter)) ||
2485        (bld->dims > 2 &&
2486            lp_sampler_wrap_mode_uses_border_color(sampler_state->wrap_r,
2487                                                   min_filter,
2488                                                   mag_filter))) {
2489       lp_build_clamp_border_color(bld, sampler_unit);
2490    }
2491 
2492 
2493    /*
2494     * Get/interpolate texture colors.
2495     */
2496 
2497    for (chan = 0; chan < 4; ++chan) {
2498      texels[chan] = lp_build_alloca(bld->gallivm, bld->texel_bld.vec_type, "");
2499      lp_build_name(texels[chan], "sampler%u_texel_%c_var", sampler_unit, "xyzw"[chan]);
2500    }
2501 
2502    if (min_filter == mag_filter) {
2503       /* no need to distinguish between minification and magnification */
2504       lp_build_sample_mipmap(bld, min_filter, mip_filter,
2505                              is_gather,
2506                              coords, offsets,
2507                              ilevel0, ilevel1, lod_fpart,
2508                              texels);
2509    }
2510    else {
2511       /*
2512        * Could also get rid of the if-logic and always use mipmap_both, both
2513        * for the single lod and multi-lod case if nothing really uses this.
2514        */
2515       if (bld->num_lods == 1) {
2516          /* Emit conditional to choose min image filter or mag image filter
2517           * depending on the lod being > 0 or <= 0, respectively.
2518           */
2519          struct lp_build_if_state if_ctx;
2520 
2521          lod_positive = LLVMBuildTrunc(builder, lod_positive,
2522                                        LLVMInt1TypeInContext(bld->gallivm->context),
2523                                        "lod_pos");
2524 
2525          lp_build_if(&if_ctx, bld->gallivm, lod_positive);
2526          {
2527             /* Use the minification filter */
2528             lp_build_sample_mipmap(bld, min_filter, mip_filter, FALSE,
2529                                    coords, offsets,
2530                                    ilevel0, ilevel1, lod_fpart,
2531                                    texels);
2532          }
2533          lp_build_else(&if_ctx);
2534          {
2535             /* Use the magnification filter */
2536             lp_build_sample_mipmap(bld, mag_filter, PIPE_TEX_MIPFILTER_NONE,
2537                                    FALSE,
2538                                    coords, offsets,
2539                                    ilevel0, NULL, NULL,
2540                                    texels);
2541          }
2542          lp_build_endif(&if_ctx);
2543       }
2544       else {
2545          LLVMValueRef need_linear, linear_mask;
2546          unsigned mip_filter_for_nearest;
2547          struct lp_build_if_state if_ctx;
2548 
2549          if (min_filter == PIPE_TEX_FILTER_LINEAR) {
2550             linear_mask = lod_positive;
2551             mip_filter_for_nearest = PIPE_TEX_MIPFILTER_NONE;
2552          }
2553          else {
2554             linear_mask = lp_build_not(&bld->lodi_bld, lod_positive);
2555             mip_filter_for_nearest = mip_filter;
2556          }
2557          need_linear = lp_build_any_true_range(&bld->lodi_bld, bld->num_lods,
2558                                                linear_mask);
2559          lp_build_name(need_linear, "need_linear");
2560 
2561          if (bld->num_lods != bld->coord_type.length) {
2562             linear_mask = lp_build_unpack_broadcast_aos_scalars(bld->gallivm,
2563                                                                 bld->lodi_type,
2564                                                                 bld->int_coord_type,
2565                                                                 linear_mask);
2566          }
2567 
2568          lp_build_if(&if_ctx, bld->gallivm, need_linear);
2569          {
2570             /*
2571              * Do sampling with both filters simultaneously. This means using
2572              * a linear filter and doing some tricks (with weights) for the pixels
2573              * which need nearest filter.
2574              * Note that it's probably rare some pixels need nearest and some
2575              * linear filter but the fixups required for the nearest pixels
2576              * aren't all that complicated so just always run a combined path
2577              * if at least some pixels require linear.
2578              */
2579             lp_build_sample_mipmap_both(bld, linear_mask, mip_filter,
2580                                         coords, offsets,
2581                                         ilevel0, ilevel1,
2582                                         lod_fpart, lod_positive,
2583                                         texels);
2584          }
2585          lp_build_else(&if_ctx);
2586          {
2587             /*
2588              * All pixels require just nearest filtering, which is way
2589              * cheaper than linear, hence do a separate path for that.
2590              */
2591             lp_build_sample_mipmap(bld, PIPE_TEX_FILTER_NEAREST,
2592                                    mip_filter_for_nearest, FALSE,
2593                                    coords, offsets,
2594                                    ilevel0, ilevel1, lod_fpart,
2595                                    texels);
2596          }
2597          lp_build_endif(&if_ctx);
2598       }
2599    }
2600 
2601    for (chan = 0; chan < 4; ++chan) {
2602      colors_out[chan] = LLVMBuildLoad(builder, texels[chan], "");
2603      lp_build_name(colors_out[chan], "sampler%u_texel_%c", sampler_unit, "xyzw"[chan]);
2604    }
2605 }
2606 
2607 
2608 /**
2609  * Texel fetch function.
2610  * In contrast to general sampling there is no filtering, no coord minification,
2611  * lod (if any) is always explicit uint, coords are uints (in terms of texel units)
2612  * directly to be applied to the selected mip level (after adding texel offsets).
2613  * This function handles texel fetch for all targets where texel fetch is supported
2614  * (no cube maps, but 1d, 2d, 3d are supported, arrays and buffers should be too).
2615  */
2616 static void
lp_build_fetch_texel(struct lp_build_sample_context * bld,unsigned texture_unit,LLVMValueRef ms_index,const LLVMValueRef * coords,LLVMValueRef explicit_lod,const LLVMValueRef * offsets,LLVMValueRef * colors_out)2617 lp_build_fetch_texel(struct lp_build_sample_context *bld,
2618                      unsigned texture_unit,
2619                      LLVMValueRef ms_index,
2620                      const LLVMValueRef *coords,
2621                      LLVMValueRef explicit_lod,
2622                      const LLVMValueRef *offsets,
2623                      LLVMValueRef *colors_out)
2624 {
2625    struct lp_build_context *perquadi_bld = &bld->lodi_bld;
2626    struct lp_build_context *int_coord_bld = &bld->int_coord_bld;
2627    unsigned dims = bld->dims, chan;
2628    unsigned target = bld->static_texture_state->target;
2629    boolean out_of_bound_ret_zero = TRUE;
2630    LLVMValueRef size, ilevel;
2631    LLVMValueRef row_stride_vec = NULL, img_stride_vec = NULL;
2632    LLVMValueRef x = coords[0], y = coords[1], z = coords[2];
2633    LLVMValueRef width, height, depth, i, j;
2634    LLVMValueRef offset, out_of_bounds, out1;
2635 
2636    out_of_bounds = int_coord_bld->zero;
2637 
2638    if (explicit_lod && bld->static_texture_state->target != PIPE_BUFFER) {
2639       if (bld->num_mips != int_coord_bld->type.length) {
2640          ilevel = lp_build_pack_aos_scalars(bld->gallivm, int_coord_bld->type,
2641                                             perquadi_bld->type, explicit_lod, 0);
2642       }
2643       else {
2644          ilevel = explicit_lod;
2645       }
2646       lp_build_nearest_mip_level(bld, texture_unit, ilevel, &ilevel,
2647                                  out_of_bound_ret_zero ? &out_of_bounds : NULL);
2648    }
2649    else {
2650       assert(bld->num_mips == 1);
2651       if (bld->static_texture_state->target != PIPE_BUFFER) {
2652          ilevel = bld->dynamic_state->first_level(bld->dynamic_state, bld->gallivm,
2653                                                   bld->context_ptr, texture_unit, NULL);
2654       }
2655       else {
2656          ilevel = lp_build_const_int32(bld->gallivm, 0);
2657       }
2658    }
2659    lp_build_mipmap_level_sizes(bld, ilevel,
2660                                &size,
2661                                &row_stride_vec, &img_stride_vec);
2662    lp_build_extract_image_sizes(bld, &bld->int_size_bld, int_coord_bld->type,
2663                                 size, &width, &height, &depth);
2664 
2665    if (target == PIPE_TEXTURE_1D_ARRAY ||
2666        target == PIPE_TEXTURE_2D_ARRAY) {
2667       if (out_of_bound_ret_zero) {
2668          z = lp_build_layer_coord(bld, texture_unit, FALSE, z, &out1);
2669          out_of_bounds = lp_build_or(int_coord_bld, out_of_bounds, out1);
2670       }
2671       else {
2672          z = lp_build_layer_coord(bld, texture_unit, FALSE, z, NULL);
2673       }
2674    }
2675 
2676    /* This is a lot like border sampling */
2677    if (offsets[0]) {
2678       /*
2679        * coords are really unsigned, offsets are signed, but I don't think
2680        * exceeding 31 bits is possible
2681        */
2682       x = lp_build_add(int_coord_bld, x, offsets[0]);
2683    }
2684    out1 = lp_build_cmp(int_coord_bld, PIPE_FUNC_LESS, x, int_coord_bld->zero);
2685    out_of_bounds = lp_build_or(int_coord_bld, out_of_bounds, out1);
2686    out1 = lp_build_cmp(int_coord_bld, PIPE_FUNC_GEQUAL, x, width);
2687    out_of_bounds = lp_build_or(int_coord_bld, out_of_bounds, out1);
2688 
2689    if (dims >= 2) {
2690       if (offsets[1]) {
2691          y = lp_build_add(int_coord_bld, y, offsets[1]);
2692       }
2693       out1 = lp_build_cmp(int_coord_bld, PIPE_FUNC_LESS, y, int_coord_bld->zero);
2694       out_of_bounds = lp_build_or(int_coord_bld, out_of_bounds, out1);
2695       out1 = lp_build_cmp(int_coord_bld, PIPE_FUNC_GEQUAL, y, height);
2696       out_of_bounds = lp_build_or(int_coord_bld, out_of_bounds, out1);
2697 
2698       if (dims >= 3) {
2699          if (offsets[2]) {
2700             z = lp_build_add(int_coord_bld, z, offsets[2]);
2701          }
2702          out1 = lp_build_cmp(int_coord_bld, PIPE_FUNC_LESS, z, int_coord_bld->zero);
2703          out_of_bounds = lp_build_or(int_coord_bld, out_of_bounds, out1);
2704          out1 = lp_build_cmp(int_coord_bld, PIPE_FUNC_GEQUAL, z, depth);
2705          out_of_bounds = lp_build_or(int_coord_bld, out_of_bounds, out1);
2706       }
2707    }
2708 
2709    lp_build_sample_offset(int_coord_bld,
2710                           bld->format_desc,
2711                           x, y, z, row_stride_vec, img_stride_vec,
2712                           &offset, &i, &j);
2713 
2714    if (bld->static_texture_state->target != PIPE_BUFFER) {
2715       offset = lp_build_add(int_coord_bld, offset,
2716                             lp_build_get_mip_offsets(bld, ilevel));
2717    }
2718 
2719    if (bld->fetch_ms) {
2720       LLVMValueRef num_samples;
2721       num_samples = bld->dynamic_state->num_samples(bld->dynamic_state, bld->gallivm,
2722                                                     bld->context_ptr, texture_unit, NULL);
2723       out1 = lp_build_cmp(int_coord_bld, PIPE_FUNC_LESS, ms_index, int_coord_bld->zero);
2724       out_of_bounds = lp_build_or(int_coord_bld, out_of_bounds, out1);
2725       out1 = lp_build_cmp(int_coord_bld, PIPE_FUNC_GEQUAL, ms_index, lp_build_broadcast_scalar(int_coord_bld, num_samples));
2726       out_of_bounds = lp_build_or(int_coord_bld, out_of_bounds, out1);
2727       offset = lp_build_add(int_coord_bld, offset,
2728                             lp_build_mul(int_coord_bld, bld->sample_stride, ms_index));
2729    }
2730 
2731    offset = lp_build_andnot(int_coord_bld, offset, out_of_bounds);
2732 
2733    lp_build_fetch_rgba_soa(bld->gallivm,
2734                            bld->format_desc,
2735                            bld->texel_type, TRUE,
2736                            bld->base_ptr, offset,
2737                            i, j,
2738                            bld->cache,
2739                            colors_out);
2740 
2741    if (out_of_bound_ret_zero) {
2742       /*
2743        * Only needed for ARB_robust_buffer_access_behavior and d3d10.
2744        * Could use min/max above instead of out-of-bounds comparisons
2745        * if we don't care about the result returned for out-of-bounds.
2746        */
2747       for (chan = 0; chan < 4; chan++) {
2748          colors_out[chan] = lp_build_select(&bld->texel_bld, out_of_bounds,
2749                                             bld->texel_bld.zero, colors_out[chan]);
2750       }
2751    }
2752 }
2753 
2754 
2755 /**
2756  * Just set texels to white instead of actually sampling the texture.
2757  * For debugging.
2758  */
2759 void
lp_build_sample_nop(struct gallivm_state * gallivm,struct lp_type type,const LLVMValueRef * coords,LLVMValueRef texel_out[4])2760 lp_build_sample_nop(struct gallivm_state *gallivm,
2761                     struct lp_type type,
2762                     const LLVMValueRef *coords,
2763                     LLVMValueRef texel_out[4])
2764 {
2765    LLVMValueRef one = lp_build_one(gallivm, type);
2766    unsigned chan;
2767 
2768    for (chan = 0; chan < 4; chan++) {
2769       texel_out[chan] = one;
2770    }
2771 }
2772 
2773 static struct lp_type
lp_build_texel_type(struct lp_type texel_type,const struct util_format_description * format_desc)2774 lp_build_texel_type(struct lp_type texel_type,
2775                     const struct util_format_description *format_desc)
2776 {
2777    /* always using the first channel hopefully should be safe,
2778     * if not things WILL break in other places anyway.
2779     */
2780    if (format_desc->colorspace == UTIL_FORMAT_COLORSPACE_RGB &&
2781        format_desc->channel[0].pure_integer) {
2782       if (format_desc->channel[0].type == UTIL_FORMAT_TYPE_SIGNED) {
2783          texel_type = lp_type_int_vec(texel_type.width, texel_type.width * texel_type.length);
2784       } else if (format_desc->channel[0].type == UTIL_FORMAT_TYPE_UNSIGNED) {
2785          texel_type = lp_type_uint_vec(texel_type.width, texel_type.width * texel_type.length);
2786       }
2787    }
2788    else if (util_format_has_stencil(format_desc) &&
2789        !util_format_has_depth(format_desc)) {
2790       /* for stencil only formats, sample stencil (uint) */
2791       texel_type = lp_type_uint_vec(texel_type.width, texel_type.width * texel_type.length);
2792    }
2793    return texel_type;
2794 }
2795 
2796 
2797 /**
2798  * Build the actual texture sampling code.
2799  * 'texel' will return a vector of four LLVMValueRefs corresponding to
2800  * R, G, B, A.
2801  * \param type  vector float type to use for coords, etc.
2802  * \param sample_key
2803  * \param derivs  partial derivatives of (s,t,r,q) with respect to x and y
2804  */
2805 static void
lp_build_sample_soa_code(struct gallivm_state * gallivm,const struct lp_static_texture_state * static_texture_state,const struct lp_static_sampler_state * static_sampler_state,struct lp_sampler_dynamic_state * dynamic_state,struct lp_type type,unsigned sample_key,unsigned texture_index,unsigned sampler_index,LLVMValueRef context_ptr,LLVMValueRef thread_data_ptr,const LLVMValueRef * coords,const LLVMValueRef * offsets,const struct lp_derivatives * derivs,LLVMValueRef lod,LLVMValueRef ms_index,LLVMValueRef texel_out[4])2806 lp_build_sample_soa_code(struct gallivm_state *gallivm,
2807                          const struct lp_static_texture_state *static_texture_state,
2808                          const struct lp_static_sampler_state *static_sampler_state,
2809                          struct lp_sampler_dynamic_state *dynamic_state,
2810                          struct lp_type type,
2811                          unsigned sample_key,
2812                          unsigned texture_index,
2813                          unsigned sampler_index,
2814                          LLVMValueRef context_ptr,
2815                          LLVMValueRef thread_data_ptr,
2816                          const LLVMValueRef *coords,
2817                          const LLVMValueRef *offsets,
2818                          const struct lp_derivatives *derivs, /* optional */
2819                          LLVMValueRef lod, /* optional */
2820                          LLVMValueRef ms_index, /* optional */
2821                          LLVMValueRef texel_out[4])
2822 {
2823    unsigned target = static_texture_state->target;
2824    unsigned dims = texture_dims(target);
2825    unsigned num_quads = type.length / 4;
2826    unsigned mip_filter, min_img_filter, mag_img_filter, i;
2827    struct lp_build_sample_context bld;
2828    struct lp_static_sampler_state derived_sampler_state = *static_sampler_state;
2829    LLVMTypeRef i32t = LLVMInt32TypeInContext(gallivm->context);
2830    LLVMBuilderRef builder = gallivm->builder;
2831    LLVMValueRef tex_width, newcoords[5];
2832    enum lp_sampler_lod_property lod_property;
2833    enum lp_sampler_lod_control lod_control;
2834    enum lp_sampler_op_type op_type;
2835    LLVMValueRef lod_bias = NULL;
2836    LLVMValueRef explicit_lod = NULL;
2837    boolean op_is_tex, op_is_lodq, op_is_gather, fetch_ms;
2838 
2839    if (0) {
2840       enum pipe_format fmt = static_texture_state->format;
2841       debug_printf("Sample from %s\n", util_format_name(fmt));
2842    }
2843 
2844    lod_property = (sample_key & LP_SAMPLER_LOD_PROPERTY_MASK) >>
2845                      LP_SAMPLER_LOD_PROPERTY_SHIFT;
2846    lod_control = (sample_key & LP_SAMPLER_LOD_CONTROL_MASK) >>
2847                     LP_SAMPLER_LOD_CONTROL_SHIFT;
2848    op_type = (sample_key & LP_SAMPLER_OP_TYPE_MASK) >>
2849                  LP_SAMPLER_OP_TYPE_SHIFT;
2850    fetch_ms = !!(sample_key & LP_SAMPLER_FETCH_MS);
2851 
2852    op_is_tex = op_type == LP_SAMPLER_OP_TEXTURE;
2853    op_is_lodq = op_type == LP_SAMPLER_OP_LODQ;
2854    op_is_gather = op_type == LP_SAMPLER_OP_GATHER;
2855 
2856    if (lod_control == LP_SAMPLER_LOD_BIAS) {
2857       lod_bias = lod;
2858       assert(lod);
2859       assert(derivs == NULL);
2860    }
2861    else if (lod_control == LP_SAMPLER_LOD_EXPLICIT) {
2862       explicit_lod = lod;
2863       assert(lod);
2864       assert(derivs == NULL);
2865    }
2866    else if (lod_control == LP_SAMPLER_LOD_DERIVATIVES) {
2867       assert(derivs);
2868       assert(lod == NULL);
2869    }
2870    else {
2871       assert(derivs == NULL);
2872       assert(lod == NULL);
2873    }
2874 
2875    if (static_texture_state->format == PIPE_FORMAT_NONE) {
2876       /*
2877        * If there's nothing bound, format is NONE, and we must return
2878        * all zero as mandated by d3d10 in this case.
2879        */
2880       unsigned chan;
2881       LLVMValueRef zero = lp_build_zero(gallivm, type);
2882       for (chan = 0; chan < 4; chan++) {
2883          texel_out[chan] = zero;
2884       }
2885       return;
2886    }
2887 
2888    assert(type.floating);
2889 
2890    /* Setup our build context */
2891    memset(&bld, 0, sizeof bld);
2892    bld.gallivm = gallivm;
2893    bld.context_ptr = context_ptr;
2894    bld.static_sampler_state = &derived_sampler_state;
2895    bld.static_texture_state = static_texture_state;
2896    bld.dynamic_state = dynamic_state;
2897    bld.format_desc = util_format_description(static_texture_state->format);
2898    bld.dims = dims;
2899 
2900    if (gallivm_perf & GALLIVM_PERF_NO_QUAD_LOD || op_is_lodq) {
2901       bld.no_quad_lod = TRUE;
2902    }
2903    if (gallivm_perf & GALLIVM_PERF_NO_RHO_APPROX || op_is_lodq) {
2904       bld.no_rho_approx = TRUE;
2905    }
2906    if (gallivm_perf & GALLIVM_PERF_NO_BRILINEAR || op_is_lodq || lod_bias || explicit_lod) {
2907       bld.no_brilinear = TRUE;
2908    }
2909 
2910    bld.vector_width = lp_type_width(type);
2911 
2912    bld.float_type = lp_type_float(32);
2913    bld.int_type = lp_type_int(32);
2914    bld.coord_type = type;
2915    bld.int_coord_type = lp_int_type(type);
2916    bld.float_size_in_type = lp_type_float(32);
2917    bld.float_size_in_type.length = dims > 1 ? 4 : 1;
2918    bld.int_size_in_type = lp_int_type(bld.float_size_in_type);
2919 
2920    bld.texel_type = lp_build_texel_type(type, bld.format_desc);
2921 
2922    if (!static_texture_state->level_zero_only ||
2923        !static_sampler_state->max_lod_pos || op_is_lodq) {
2924       derived_sampler_state.min_mip_filter = static_sampler_state->min_mip_filter;
2925    } else {
2926       derived_sampler_state.min_mip_filter = PIPE_TEX_MIPFILTER_NONE;
2927    }
2928    if (op_is_gather) {
2929       /*
2930        * gather4 is exactly like GL_LINEAR filtering but in the end skipping
2931        * the actual filtering. Using mostly the same paths, so cube face
2932        * selection, coord wrapping etc. all naturally uses the same code.
2933        */
2934       derived_sampler_state.min_mip_filter = PIPE_TEX_MIPFILTER_NONE;
2935       derived_sampler_state.min_img_filter = PIPE_TEX_FILTER_LINEAR;
2936       derived_sampler_state.mag_img_filter = PIPE_TEX_FILTER_LINEAR;
2937    }
2938    mip_filter = derived_sampler_state.min_mip_filter;
2939 
2940    if (0) {
2941       debug_printf("  .min_mip_filter = %u\n", derived_sampler_state.min_mip_filter);
2942    }
2943 
2944    if (static_texture_state->target == PIPE_TEXTURE_CUBE ||
2945        static_texture_state->target == PIPE_TEXTURE_CUBE_ARRAY)
2946    {
2947       /*
2948        * Seamless filtering ignores wrap modes.
2949        * Setting to CLAMP_TO_EDGE is correct for nearest filtering, for
2950        * bilinear it's not correct but way better than using for instance repeat.
2951        * Note we even set this for non-seamless. Technically GL allows any wrap
2952        * mode, which made sense when supporting true borders (can get seamless
2953        * effect with border and CLAMP_TO_BORDER), but gallium doesn't support
2954        * borders and d3d9 requires wrap modes to be ignored and it's a pain to fix
2955        * up the sampler state (as it makes it texture dependent).
2956        */
2957       derived_sampler_state.wrap_s = PIPE_TEX_WRAP_CLAMP_TO_EDGE;
2958       derived_sampler_state.wrap_t = PIPE_TEX_WRAP_CLAMP_TO_EDGE;
2959    }
2960    /*
2961     * We could force CLAMP to CLAMP_TO_EDGE here if min/mag filter is nearest,
2962     * so AoS path could be used. Not sure it's worth the trouble...
2963     */
2964 
2965    min_img_filter = derived_sampler_state.min_img_filter;
2966    mag_img_filter = derived_sampler_state.mag_img_filter;
2967 
2968 
2969    /*
2970     * This is all a bit complicated different paths are chosen for performance
2971     * reasons.
2972     * Essentially, there can be 1 lod per element, 1 lod per quad or 1 lod for
2973     * everything (the last two options are equivalent for 4-wide case).
2974     * If there's per-quad lod but we split to 4-wide so we can use AoS, per-quad
2975     * lod is calculated then the lod value extracted afterwards so making this
2976     * case basically the same as far as lod handling is concerned for the
2977     * further sample/filter code as the 1 lod for everything case.
2978     * Different lod handling mostly shows up when building mipmap sizes
2979     * (lp_build_mipmap_level_sizes() and friends) and also in filtering
2980     * (getting the fractional part of the lod to the right texels).
2981     */
2982 
2983    /*
2984     * There are other situations where at least the multiple int lods could be
2985     * avoided like min and max lod being equal.
2986     */
2987    bld.num_mips = bld.num_lods = 1;
2988 
2989    if (bld.no_quad_lod && bld.no_rho_approx &&
2990        ((mip_filter != PIPE_TEX_MIPFILTER_NONE && op_is_tex &&
2991          (static_texture_state->target == PIPE_TEXTURE_CUBE ||
2992           static_texture_state->target == PIPE_TEXTURE_CUBE_ARRAY)) ||
2993         op_is_lodq)) {
2994       /*
2995        * special case for using per-pixel lod even for implicit lod,
2996        * which is generally never required (ok by APIs) except to please
2997        * some (somewhat broken imho) tests (because per-pixel face selection
2998        * can cause derivatives to be different for pixels outside the primitive
2999        * due to the major axis division even if pre-project derivatives are
3000        * looking normal).
3001        * For lodq, we do it to simply avoid scalar pack / unpack (albeit for
3002        * cube maps we do indeed get per-pixel lod values).
3003        */
3004       bld.num_mips = type.length;
3005       bld.num_lods = type.length;
3006    }
3007    else if (lod_property == LP_SAMPLER_LOD_PER_ELEMENT ||
3008        (explicit_lod || lod_bias || derivs)) {
3009       if ((!op_is_tex && target != PIPE_BUFFER) ||
3010           (op_is_tex && mip_filter != PIPE_TEX_MIPFILTER_NONE)) {
3011          bld.num_mips = type.length;
3012          bld.num_lods = type.length;
3013       }
3014       else if (op_is_tex && min_img_filter != mag_img_filter) {
3015          bld.num_mips = 1;
3016          bld.num_lods = type.length;
3017       }
3018    }
3019    /* TODO: for true scalar_lod should only use 1 lod value */
3020    else if ((!op_is_tex && explicit_lod && target != PIPE_BUFFER) ||
3021             (op_is_tex && mip_filter != PIPE_TEX_MIPFILTER_NONE)) {
3022       bld.num_mips = num_quads;
3023       bld.num_lods = num_quads;
3024    }
3025    else if (op_is_tex && min_img_filter != mag_img_filter) {
3026       bld.num_mips = 1;
3027       bld.num_lods = num_quads;
3028    }
3029 
3030    bld.fetch_ms = fetch_ms;
3031    if (op_is_gather)
3032       bld.gather_comp = (sample_key & LP_SAMPLER_GATHER_COMP_MASK) >> LP_SAMPLER_GATHER_COMP_SHIFT;
3033    bld.lodf_type = type;
3034    /* we want native vector size to be able to use our intrinsics */
3035    if (bld.num_lods != type.length) {
3036       /* TODO: this currently always has to be per-quad or per-element */
3037       bld.lodf_type.length = type.length > 4 ? ((type.length + 15) / 16) * 4 : 1;
3038    }
3039    bld.lodi_type = lp_int_type(bld.lodf_type);
3040    bld.levelf_type = bld.lodf_type;
3041    if (bld.num_mips == 1) {
3042       bld.levelf_type.length = 1;
3043    }
3044    bld.leveli_type = lp_int_type(bld.levelf_type);
3045    bld.float_size_type = bld.float_size_in_type;
3046    /* Note: size vectors may not be native. They contain minified w/h/d/_ values,
3047     * with per-element lod that is w0/h0/d0/_/w1/h1/d1_/... so up to 8x4f32 */
3048    if (bld.num_mips > 1) {
3049       bld.float_size_type.length = bld.num_mips == type.length ?
3050                                       bld.num_mips * bld.float_size_in_type.length :
3051                                       type.length;
3052    }
3053    bld.int_size_type = lp_int_type(bld.float_size_type);
3054 
3055    lp_build_context_init(&bld.float_bld, gallivm, bld.float_type);
3056    lp_build_context_init(&bld.float_vec_bld, gallivm, type);
3057    lp_build_context_init(&bld.int_bld, gallivm, bld.int_type);
3058    lp_build_context_init(&bld.coord_bld, gallivm, bld.coord_type);
3059    lp_build_context_init(&bld.int_coord_bld, gallivm, bld.int_coord_type);
3060    lp_build_context_init(&bld.int_size_in_bld, gallivm, bld.int_size_in_type);
3061    lp_build_context_init(&bld.float_size_in_bld, gallivm, bld.float_size_in_type);
3062    lp_build_context_init(&bld.int_size_bld, gallivm, bld.int_size_type);
3063    lp_build_context_init(&bld.float_size_bld, gallivm, bld.float_size_type);
3064    lp_build_context_init(&bld.texel_bld, gallivm, bld.texel_type);
3065    lp_build_context_init(&bld.levelf_bld, gallivm, bld.levelf_type);
3066    lp_build_context_init(&bld.leveli_bld, gallivm, bld.leveli_type);
3067    lp_build_context_init(&bld.lodf_bld, gallivm, bld.lodf_type);
3068    lp_build_context_init(&bld.lodi_bld, gallivm, bld.lodi_type);
3069 
3070    /* Get the dynamic state */
3071    tex_width = dynamic_state->width(dynamic_state, gallivm,
3072                                     context_ptr, texture_index, NULL);
3073    bld.row_stride_array = dynamic_state->row_stride(dynamic_state, gallivm,
3074                                                     context_ptr, texture_index, NULL);
3075    bld.img_stride_array = dynamic_state->img_stride(dynamic_state, gallivm,
3076                                                     context_ptr, texture_index, NULL);
3077    bld.base_ptr = dynamic_state->base_ptr(dynamic_state, gallivm,
3078                                           context_ptr, texture_index, NULL);
3079    bld.mip_offsets = dynamic_state->mip_offsets(dynamic_state, gallivm,
3080                                                 context_ptr, texture_index, NULL);
3081 
3082    if (fetch_ms)
3083       bld.sample_stride = lp_build_broadcast_scalar(&bld.int_coord_bld, dynamic_state->sample_stride(dynamic_state, gallivm,
3084                                                                                                      context_ptr, texture_index, NULL));
3085    /* Note that mip_offsets is an array[level] of offsets to texture images */
3086 
3087    if (dynamic_state->cache_ptr && thread_data_ptr) {
3088       bld.cache = dynamic_state->cache_ptr(dynamic_state, gallivm,
3089                                            thread_data_ptr, texture_index);
3090    }
3091 
3092    /* width, height, depth as single int vector */
3093    if (dims <= 1) {
3094       bld.int_size = tex_width;
3095    }
3096    else {
3097       bld.int_size = LLVMBuildInsertElement(builder, bld.int_size_in_bld.undef,
3098                                             tex_width,
3099                                             LLVMConstInt(i32t, 0, 0), "");
3100       if (dims >= 2) {
3101          LLVMValueRef tex_height =
3102             dynamic_state->height(dynamic_state, gallivm,
3103                                   context_ptr, texture_index, NULL);
3104          bld.int_size = LLVMBuildInsertElement(builder, bld.int_size,
3105                                                tex_height,
3106                                                LLVMConstInt(i32t, 1, 0), "");
3107          if (dims >= 3) {
3108             LLVMValueRef tex_depth =
3109                dynamic_state->depth(dynamic_state, gallivm, context_ptr,
3110                                     texture_index, NULL);
3111             bld.int_size = LLVMBuildInsertElement(builder, bld.int_size,
3112                                                   tex_depth,
3113                                                   LLVMConstInt(i32t, 2, 0), "");
3114          }
3115       }
3116    }
3117 
3118    for (i = 0; i < 5; i++) {
3119       newcoords[i] = coords[i];
3120    }
3121 
3122    if (util_format_is_pure_integer(static_texture_state->format) &&
3123        !util_format_has_depth(bld.format_desc) && op_is_tex &&
3124        (static_sampler_state->min_mip_filter == PIPE_TEX_MIPFILTER_LINEAR ||
3125         static_sampler_state->min_img_filter == PIPE_TEX_FILTER_LINEAR ||
3126         static_sampler_state->mag_img_filter == PIPE_TEX_FILTER_LINEAR)) {
3127       /*
3128        * Bail if impossible filtering is specified (the awkard additional
3129        * depth check is because it is legal in gallium to have things like S8Z24
3130        * here which would say it's pure int despite such formats should sample
3131        * the depth component).
3132        * In GL such filters make the texture incomplete, this makes it robust
3133        * against gallium frontends which set this up regardless (we'd crash in the
3134        * lerp later otherwise).
3135        * At least in some apis it may be legal to use such filters with lod
3136        * queries and/or gather (at least for gather d3d10 says only the wrap
3137        * bits are really used hence filter bits are likely simply ignored).
3138        * For fetch, we don't get valid samplers either way here.
3139        */
3140       unsigned chan;
3141       LLVMValueRef zero = lp_build_zero(gallivm, type);
3142       for (chan = 0; chan < 4; chan++) {
3143          texel_out[chan] = zero;
3144       }
3145       return;
3146    }
3147 
3148    if (0) {
3149       /* For debug: no-op texture sampling */
3150       lp_build_sample_nop(gallivm,
3151                           bld.texel_type,
3152                           newcoords,
3153                           texel_out);
3154    }
3155 
3156    else if (op_type == LP_SAMPLER_OP_FETCH) {
3157       lp_build_fetch_texel(&bld, texture_index, ms_index, newcoords,
3158                            lod, offsets,
3159                            texel_out);
3160    }
3161 
3162    else {
3163       LLVMValueRef lod_fpart = NULL, lod_positive = NULL;
3164       LLVMValueRef ilevel0 = NULL, ilevel1 = NULL, lod = NULL;
3165       boolean use_aos;
3166 
3167       use_aos = util_format_fits_8unorm(bld.format_desc) &&
3168                 op_is_tex &&
3169                 /* not sure this is strictly needed or simply impossible */
3170                 derived_sampler_state.compare_mode == PIPE_TEX_COMPARE_NONE &&
3171                 lp_is_simple_wrap_mode(derived_sampler_state.wrap_s);
3172 
3173       use_aos &= bld.num_lods <= num_quads ||
3174                  derived_sampler_state.min_img_filter ==
3175                     derived_sampler_state.mag_img_filter;
3176 
3177       if(gallivm_perf & GALLIVM_PERF_NO_AOS_SAMPLING) {
3178          use_aos = 0;
3179       }
3180 
3181       if (dims > 1) {
3182          use_aos &= lp_is_simple_wrap_mode(derived_sampler_state.wrap_t);
3183          if (dims > 2) {
3184             use_aos &= lp_is_simple_wrap_mode(derived_sampler_state.wrap_r);
3185          }
3186       }
3187       if ((static_texture_state->target == PIPE_TEXTURE_CUBE ||
3188            static_texture_state->target == PIPE_TEXTURE_CUBE_ARRAY) &&
3189           derived_sampler_state.seamless_cube_map &&
3190           (derived_sampler_state.min_img_filter == PIPE_TEX_FILTER_LINEAR ||
3191            derived_sampler_state.mag_img_filter == PIPE_TEX_FILTER_LINEAR)) {
3192          /* theoretically possible with AoS filtering but not implemented (complex!) */
3193          use_aos = 0;
3194       }
3195 
3196       if ((gallivm_debug & GALLIVM_DEBUG_PERF) &&
3197           !use_aos && util_format_fits_8unorm(bld.format_desc)) {
3198          debug_printf("%s: using floating point linear filtering for %s\n",
3199                       __FUNCTION__, bld.format_desc->short_name);
3200          debug_printf("  min_img %d  mag_img %d  mip %d  target %d  seamless %d"
3201                       "  wraps %d  wrapt %d  wrapr %d\n",
3202                       derived_sampler_state.min_img_filter,
3203                       derived_sampler_state.mag_img_filter,
3204                       derived_sampler_state.min_mip_filter,
3205                       static_texture_state->target,
3206                       derived_sampler_state.seamless_cube_map,
3207                       derived_sampler_state.wrap_s,
3208                       derived_sampler_state.wrap_t,
3209                       derived_sampler_state.wrap_r);
3210       }
3211 
3212       lp_build_sample_common(&bld, op_is_lodq, texture_index, sampler_index,
3213                              newcoords,
3214                              derivs, lod_bias, explicit_lod,
3215                              &lod_positive, &lod, &lod_fpart,
3216                              &ilevel0, &ilevel1);
3217 
3218       if (op_is_lodq) {
3219          texel_out[0] = lod_fpart;
3220          texel_out[1] = lod;
3221          texel_out[2] = texel_out[3] = bld.coord_bld.zero;
3222          return;
3223       }
3224 
3225       if (use_aos && static_texture_state->target == PIPE_TEXTURE_CUBE_ARRAY) {
3226          /* The aos path doesn't do seamless filtering so simply add cube layer
3227           * to face now.
3228           */
3229          newcoords[2] = lp_build_add(&bld.int_coord_bld, newcoords[2], newcoords[3]);
3230       }
3231 
3232       /*
3233        * we only try 8-wide sampling with soa or if we have AVX2
3234        * as it appears to be a loss with just AVX)
3235        */
3236       if (num_quads == 1 || !use_aos ||
3237           (util_cpu_caps.has_avx2 &&
3238            (bld.num_lods == 1 ||
3239             derived_sampler_state.min_img_filter == derived_sampler_state.mag_img_filter))) {
3240          if (use_aos) {
3241             /* do sampling/filtering with fixed pt arithmetic */
3242             lp_build_sample_aos(&bld, sampler_index,
3243                                 newcoords[0], newcoords[1],
3244                                 newcoords[2],
3245                                 offsets, lod_positive, lod_fpart,
3246                                 ilevel0, ilevel1,
3247                                 texel_out);
3248          }
3249 
3250          else {
3251             lp_build_sample_general(&bld, sampler_index,
3252                                     op_type == LP_SAMPLER_OP_GATHER,
3253                                     newcoords, offsets,
3254                                     lod_positive, lod_fpart,
3255                                     ilevel0, ilevel1,
3256                                     texel_out);
3257          }
3258       }
3259       else {
3260          unsigned j;
3261          struct lp_build_sample_context bld4;
3262          struct lp_type type4 = type;
3263          unsigned i;
3264          LLVMValueRef texelout4[4];
3265          LLVMValueRef texelouttmp[4][LP_MAX_VECTOR_LENGTH/16];
3266 
3267          type4.length = 4;
3268 
3269          /* Setup our build context */
3270          memset(&bld4, 0, sizeof bld4);
3271          bld4.no_quad_lod = bld.no_quad_lod;
3272          bld4.no_rho_approx = bld.no_rho_approx;
3273          bld4.no_brilinear = bld.no_brilinear;
3274          bld4.gallivm = bld.gallivm;
3275          bld4.context_ptr = bld.context_ptr;
3276          bld4.static_texture_state = bld.static_texture_state;
3277          bld4.static_sampler_state = bld.static_sampler_state;
3278          bld4.dynamic_state = bld.dynamic_state;
3279          bld4.format_desc = bld.format_desc;
3280          bld4.dims = bld.dims;
3281          bld4.row_stride_array = bld.row_stride_array;
3282          bld4.img_stride_array = bld.img_stride_array;
3283          bld4.base_ptr = bld.base_ptr;
3284          bld4.mip_offsets = bld.mip_offsets;
3285          bld4.int_size = bld.int_size;
3286          bld4.cache = bld.cache;
3287 
3288          bld4.vector_width = lp_type_width(type4);
3289 
3290          bld4.float_type = lp_type_float(32);
3291          bld4.int_type = lp_type_int(32);
3292          bld4.coord_type = type4;
3293          bld4.int_coord_type = lp_int_type(type4);
3294          bld4.float_size_in_type = lp_type_float(32);
3295          bld4.float_size_in_type.length = dims > 1 ? 4 : 1;
3296          bld4.int_size_in_type = lp_int_type(bld4.float_size_in_type);
3297          bld4.texel_type = bld.texel_type;
3298          bld4.texel_type.length = 4;
3299 
3300          bld4.num_mips = bld4.num_lods = 1;
3301          if (bld4.no_quad_lod && bld4.no_rho_approx &&
3302              (static_texture_state->target == PIPE_TEXTURE_CUBE ||
3303               static_texture_state->target == PIPE_TEXTURE_CUBE_ARRAY) &&
3304              (op_is_tex && mip_filter != PIPE_TEX_MIPFILTER_NONE)) {
3305             bld4.num_mips = type4.length;
3306             bld4.num_lods = type4.length;
3307          }
3308          if (lod_property == LP_SAMPLER_LOD_PER_ELEMENT &&
3309              (explicit_lod || lod_bias || derivs)) {
3310             if ((!op_is_tex && target != PIPE_BUFFER) ||
3311                 (op_is_tex && mip_filter != PIPE_TEX_MIPFILTER_NONE)) {
3312                bld4.num_mips = type4.length;
3313                bld4.num_lods = type4.length;
3314             }
3315             else if (op_is_tex && min_img_filter != mag_img_filter) {
3316                bld4.num_mips = 1;
3317                bld4.num_lods = type4.length;
3318             }
3319          }
3320 
3321          /* we want native vector size to be able to use our intrinsics */
3322          bld4.lodf_type = type4;
3323          if (bld4.num_lods != type4.length) {
3324             bld4.lodf_type.length = 1;
3325          }
3326          bld4.lodi_type = lp_int_type(bld4.lodf_type);
3327          bld4.levelf_type = type4;
3328          if (bld4.num_mips != type4.length) {
3329             bld4.levelf_type.length = 1;
3330          }
3331          bld4.leveli_type = lp_int_type(bld4.levelf_type);
3332          bld4.float_size_type = bld4.float_size_in_type;
3333          if (bld4.num_mips > 1) {
3334             bld4.float_size_type.length = bld4.num_mips == type4.length ?
3335                                             bld4.num_mips * bld4.float_size_in_type.length :
3336                                             type4.length;
3337          }
3338          bld4.int_size_type = lp_int_type(bld4.float_size_type);
3339 
3340          lp_build_context_init(&bld4.float_bld, gallivm, bld4.float_type);
3341          lp_build_context_init(&bld4.float_vec_bld, gallivm, type4);
3342          lp_build_context_init(&bld4.int_bld, gallivm, bld4.int_type);
3343          lp_build_context_init(&bld4.coord_bld, gallivm, bld4.coord_type);
3344          lp_build_context_init(&bld4.int_coord_bld, gallivm, bld4.int_coord_type);
3345          lp_build_context_init(&bld4.int_size_in_bld, gallivm, bld4.int_size_in_type);
3346          lp_build_context_init(&bld4.float_size_in_bld, gallivm, bld4.float_size_in_type);
3347          lp_build_context_init(&bld4.int_size_bld, gallivm, bld4.int_size_type);
3348          lp_build_context_init(&bld4.float_size_bld, gallivm, bld4.float_size_type);
3349          lp_build_context_init(&bld4.texel_bld, gallivm, bld4.texel_type);
3350          lp_build_context_init(&bld4.levelf_bld, gallivm, bld4.levelf_type);
3351          lp_build_context_init(&bld4.leveli_bld, gallivm, bld4.leveli_type);
3352          lp_build_context_init(&bld4.lodf_bld, gallivm, bld4.lodf_type);
3353          lp_build_context_init(&bld4.lodi_bld, gallivm, bld4.lodi_type);
3354 
3355          for (i = 0; i < num_quads; i++) {
3356             LLVMValueRef s4, t4, r4;
3357             LLVMValueRef lod_positive4, lod_fpart4 = NULL;
3358             LLVMValueRef ilevel04, ilevel14 = NULL;
3359             LLVMValueRef offsets4[4] = { NULL };
3360             unsigned num_lods = bld4.num_lods;
3361 
3362             s4 = lp_build_extract_range(gallivm, newcoords[0], 4*i, 4);
3363             t4 = lp_build_extract_range(gallivm, newcoords[1], 4*i, 4);
3364             r4 = lp_build_extract_range(gallivm, newcoords[2], 4*i, 4);
3365 
3366             if (offsets[0]) {
3367                offsets4[0] = lp_build_extract_range(gallivm, offsets[0], 4*i, 4);
3368                if (dims > 1) {
3369                   offsets4[1] = lp_build_extract_range(gallivm, offsets[1], 4*i, 4);
3370                   if (dims > 2) {
3371                      offsets4[2] = lp_build_extract_range(gallivm, offsets[2], 4*i, 4);
3372                   }
3373                }
3374             }
3375             lod_positive4 = lp_build_extract_range(gallivm, lod_positive, num_lods * i, num_lods);
3376             ilevel04 = bld.num_mips == 1 ? ilevel0 :
3377                           lp_build_extract_range(gallivm, ilevel0, num_lods * i, num_lods);
3378             if (mip_filter == PIPE_TEX_MIPFILTER_LINEAR) {
3379                ilevel14 = lp_build_extract_range(gallivm, ilevel1, num_lods * i, num_lods);
3380                lod_fpart4 = lp_build_extract_range(gallivm, lod_fpart, num_lods * i, num_lods);
3381             }
3382 
3383             if (use_aos) {
3384                /* do sampling/filtering with fixed pt arithmetic */
3385                lp_build_sample_aos(&bld4, sampler_index,
3386                                    s4, t4, r4, offsets4,
3387                                    lod_positive4, lod_fpart4,
3388                                    ilevel04, ilevel14,
3389                                    texelout4);
3390             }
3391 
3392             else {
3393                /* this path is currently unreachable and hence might break easily... */
3394                LLVMValueRef newcoords4[5];
3395                newcoords4[0] = s4;
3396                newcoords4[1] = t4;
3397                newcoords4[2] = r4;
3398                newcoords4[3] = lp_build_extract_range(gallivm, newcoords[3], 4*i, 4);
3399                newcoords4[4] = lp_build_extract_range(gallivm, newcoords[4], 4*i, 4);
3400 
3401                lp_build_sample_general(&bld4, sampler_index,
3402                                        op_type == LP_SAMPLER_OP_GATHER,
3403                                        newcoords4, offsets4,
3404                                        lod_positive4, lod_fpart4,
3405                                        ilevel04, ilevel14,
3406                                        texelout4);
3407             }
3408             for (j = 0; j < 4; j++) {
3409                texelouttmp[j][i] = texelout4[j];
3410             }
3411          }
3412 
3413          for (j = 0; j < 4; j++) {
3414             texel_out[j] = lp_build_concat(gallivm, texelouttmp[j], type4, num_quads);
3415          }
3416       }
3417    }
3418 
3419    if (target != PIPE_BUFFER && op_type != LP_SAMPLER_OP_GATHER) {
3420       apply_sampler_swizzle(&bld, texel_out);
3421    }
3422 
3423    /*
3424     * texel type can be a (32bit) int/uint (for pure int formats only),
3425     * however we are expected to always return floats (storage is untyped).
3426     */
3427    if (!bld.texel_type.floating) {
3428       unsigned chan;
3429       for (chan = 0; chan < 4; chan++) {
3430          texel_out[chan] = LLVMBuildBitCast(builder, texel_out[chan],
3431                                             lp_build_vec_type(gallivm, type), "");
3432       }
3433    }
3434 }
3435 
3436 
3437 #define USE_TEX_FUNC_CALL 1
3438 
3439 #define LP_MAX_TEX_FUNC_ARGS 32
3440 
3441 static inline void
get_target_info(enum pipe_texture_target target,unsigned * num_coords,unsigned * num_derivs,unsigned * num_offsets,unsigned * layer)3442 get_target_info(enum pipe_texture_target target,
3443                 unsigned *num_coords, unsigned *num_derivs,
3444                 unsigned *num_offsets, unsigned *layer)
3445 {
3446    unsigned dims = texture_dims(target);
3447    *num_coords = dims;
3448    *num_offsets = dims;
3449    *num_derivs = (target == PIPE_TEXTURE_CUBE ||
3450                   target == PIPE_TEXTURE_CUBE_ARRAY) ? 3 : dims;
3451    *layer = has_layer_coord(target) ? 2: 0;
3452    if (target == PIPE_TEXTURE_CUBE_ARRAY) {
3453       /*
3454        * dims doesn't include r coord for cubes - this is handled
3455        * by layer instead, but need to fix up for cube arrays...
3456        */
3457       *layer = 3;
3458       *num_coords = 3;
3459    }
3460 }
3461 
3462 
3463 /**
3464  * Generate the function body for a texture sampling function.
3465  */
3466 static void
lp_build_sample_gen_func(struct gallivm_state * gallivm,const struct lp_static_texture_state * static_texture_state,const struct lp_static_sampler_state * static_sampler_state,struct lp_sampler_dynamic_state * dynamic_state,struct lp_type type,unsigned texture_index,unsigned sampler_index,LLVMValueRef function,unsigned num_args,unsigned sample_key)3467 lp_build_sample_gen_func(struct gallivm_state *gallivm,
3468                          const struct lp_static_texture_state *static_texture_state,
3469                          const struct lp_static_sampler_state *static_sampler_state,
3470                          struct lp_sampler_dynamic_state *dynamic_state,
3471                          struct lp_type type,
3472                          unsigned texture_index,
3473                          unsigned sampler_index,
3474                          LLVMValueRef function,
3475                          unsigned num_args,
3476                          unsigned sample_key)
3477 {
3478    LLVMBuilderRef old_builder;
3479    LLVMBasicBlockRef block;
3480    LLVMValueRef coords[5];
3481    LLVMValueRef offsets[3] = { NULL };
3482    LLVMValueRef lod = NULL;
3483    LLVMValueRef ms_index = NULL;
3484    LLVMValueRef context_ptr;
3485    LLVMValueRef thread_data_ptr = NULL;
3486    LLVMValueRef texel_out[4];
3487    struct lp_derivatives derivs;
3488    struct lp_derivatives *deriv_ptr = NULL;
3489    unsigned num_param = 0;
3490    unsigned i, num_coords, num_derivs, num_offsets, layer;
3491    enum lp_sampler_lod_control lod_control;
3492    enum lp_sampler_op_type op_type;
3493    boolean need_cache = FALSE;
3494 
3495    lod_control = (sample_key & LP_SAMPLER_LOD_CONTROL_MASK) >>
3496                     LP_SAMPLER_LOD_CONTROL_SHIFT;
3497 
3498    op_type = (sample_key & LP_SAMPLER_OP_TYPE_MASK) >>
3499                     LP_SAMPLER_OP_TYPE_SHIFT;
3500 
3501    get_target_info(static_texture_state->target,
3502                    &num_coords, &num_derivs, &num_offsets, &layer);
3503 
3504    /* lod query doesn't take a layer */
3505    if (layer && op_type == LP_SAMPLER_OP_LODQ)
3506       layer = 0;
3507 
3508    if (dynamic_state->cache_ptr) {
3509       const struct util_format_description *format_desc;
3510       format_desc = util_format_description(static_texture_state->format);
3511       if (format_desc && format_desc->layout == UTIL_FORMAT_LAYOUT_S3TC) {
3512          need_cache = TRUE;
3513       }
3514    }
3515 
3516    /* "unpack" arguments */
3517    context_ptr = LLVMGetParam(function, num_param++);
3518    if (need_cache) {
3519       thread_data_ptr = LLVMGetParam(function, num_param++);
3520    }
3521    for (i = 0; i < num_coords; i++) {
3522       coords[i] = LLVMGetParam(function, num_param++);
3523    }
3524    for (i = num_coords; i < 5; i++) {
3525       /* This is rather unfortunate... */
3526       coords[i] = lp_build_undef(gallivm, type);
3527    }
3528    if (layer) {
3529       coords[layer] = LLVMGetParam(function, num_param++);
3530    }
3531    if (sample_key & LP_SAMPLER_SHADOW) {
3532       coords[4] = LLVMGetParam(function, num_param++);
3533    }
3534    if (sample_key & LP_SAMPLER_FETCH_MS) {
3535       ms_index = LLVMGetParam(function, num_param++);
3536    }
3537    if (sample_key & LP_SAMPLER_OFFSETS) {
3538       for (i = 0; i < num_offsets; i++) {
3539          offsets[i] = LLVMGetParam(function, num_param++);
3540       }
3541    }
3542    if (lod_control == LP_SAMPLER_LOD_BIAS ||
3543        lod_control == LP_SAMPLER_LOD_EXPLICIT) {
3544       lod = LLVMGetParam(function, num_param++);
3545    }
3546    else if (lod_control == LP_SAMPLER_LOD_DERIVATIVES) {
3547       for (i = 0; i < num_derivs; i++) {
3548          derivs.ddx[i] = LLVMGetParam(function, num_param++);
3549          derivs.ddy[i] = LLVMGetParam(function, num_param++);
3550       }
3551       deriv_ptr = &derivs;
3552    }
3553 
3554    assert(num_args == num_param);
3555 
3556    /*
3557     * Function body
3558     */
3559 
3560    old_builder = gallivm->builder;
3561    block = LLVMAppendBasicBlockInContext(gallivm->context, function, "entry");
3562    gallivm->builder = LLVMCreateBuilderInContext(gallivm->context);
3563    LLVMPositionBuilderAtEnd(gallivm->builder, block);
3564 
3565    lp_build_sample_soa_code(gallivm,
3566                             static_texture_state,
3567                             static_sampler_state,
3568                             dynamic_state,
3569                             type,
3570                             sample_key,
3571                             texture_index,
3572                             sampler_index,
3573                             context_ptr,
3574                             thread_data_ptr,
3575                             coords,
3576                             offsets,
3577                             deriv_ptr,
3578                             lod,
3579                             ms_index,
3580                             texel_out);
3581 
3582    LLVMBuildAggregateRet(gallivm->builder, texel_out, 4);
3583 
3584    LLVMDisposeBuilder(gallivm->builder);
3585    gallivm->builder = old_builder;
3586 
3587    gallivm_verify_function(gallivm, function);
3588 }
3589 
3590 
3591 /**
3592  * Call the matching function for texture sampling.
3593  * If there's no match, generate a new one.
3594  */
3595 static void
lp_build_sample_soa_func(struct gallivm_state * gallivm,const struct lp_static_texture_state * static_texture_state,const struct lp_static_sampler_state * static_sampler_state,struct lp_sampler_dynamic_state * dynamic_state,const struct lp_sampler_params * params,int texture_index,int sampler_index,LLVMValueRef * tex_ret)3596 lp_build_sample_soa_func(struct gallivm_state *gallivm,
3597                          const struct lp_static_texture_state *static_texture_state,
3598                          const struct lp_static_sampler_state *static_sampler_state,
3599                          struct lp_sampler_dynamic_state *dynamic_state,
3600                          const struct lp_sampler_params *params,
3601                          int texture_index, int sampler_index,
3602                          LLVMValueRef *tex_ret)
3603 {
3604    LLVMBuilderRef builder = gallivm->builder;
3605    LLVMModuleRef module = LLVMGetGlobalParent(LLVMGetBasicBlockParent(
3606                              LLVMGetInsertBlock(builder)));
3607    LLVMValueRef function, inst;
3608    LLVMValueRef args[LP_MAX_TEX_FUNC_ARGS];
3609    LLVMBasicBlockRef bb;
3610    unsigned num_args = 0;
3611    char func_name[64];
3612    unsigned i, num_coords, num_derivs, num_offsets, layer;
3613    unsigned sample_key = params->sample_key;
3614    const LLVMValueRef *coords = params->coords;
3615    const LLVMValueRef *offsets = params->offsets;
3616    const struct lp_derivatives *derivs = params->derivs;
3617    enum lp_sampler_lod_control lod_control;
3618    enum lp_sampler_op_type op_type;
3619    boolean need_cache = FALSE;
3620 
3621    lod_control = (sample_key & LP_SAMPLER_LOD_CONTROL_MASK) >>
3622                     LP_SAMPLER_LOD_CONTROL_SHIFT;
3623 
3624    op_type = (sample_key & LP_SAMPLER_OP_TYPE_MASK) >>
3625                     LP_SAMPLER_OP_TYPE_SHIFT;
3626 
3627    get_target_info(static_texture_state->target,
3628                    &num_coords, &num_derivs, &num_offsets, &layer);
3629 
3630    /* lod query doesn't take a layer */
3631    if (layer && op_type == LP_SAMPLER_OP_LODQ)
3632       layer = 0;
3633 
3634    if (dynamic_state->cache_ptr) {
3635       const struct util_format_description *format_desc;
3636       format_desc = util_format_description(static_texture_state->format);
3637       if (format_desc && format_desc->layout == UTIL_FORMAT_LAYOUT_S3TC) {
3638          need_cache = TRUE;
3639       }
3640    }
3641    /*
3642     * texture function matches are found by name.
3643     * Thus the name has to include both the texture and sampler unit
3644     * (which covers all static state) plus the actual texture function
3645     * (including things like offsets, shadow coord, lod control).
3646     * Additionally lod_property has to be included too.
3647     */
3648 
3649    snprintf(func_name, sizeof(func_name), "texfunc_res_%d_sam_%d_%x",
3650             texture_index, sampler_index, sample_key);
3651 
3652    function = LLVMGetNamedFunction(module, func_name);
3653 
3654    if(!function) {
3655       LLVMTypeRef arg_types[LP_MAX_TEX_FUNC_ARGS];
3656       LLVMTypeRef ret_type;
3657       LLVMTypeRef function_type;
3658       LLVMTypeRef val_type[4];
3659       unsigned num_param = 0;
3660 
3661       /*
3662        * Generate the function prototype.
3663        */
3664 
3665       arg_types[num_param++] = LLVMTypeOf(params->context_ptr);
3666       if (need_cache) {
3667          arg_types[num_param++] = LLVMTypeOf(params->thread_data_ptr);
3668       }
3669       for (i = 0; i < num_coords; i++) {
3670          arg_types[num_param++] = LLVMTypeOf(coords[0]);
3671          assert(LLVMTypeOf(coords[0]) == LLVMTypeOf(coords[i]));
3672       }
3673       if (layer) {
3674          arg_types[num_param++] = LLVMTypeOf(coords[layer]);
3675          assert(LLVMTypeOf(coords[0]) == LLVMTypeOf(coords[layer]));
3676       }
3677       if (sample_key & LP_SAMPLER_SHADOW) {
3678          arg_types[num_param++] = LLVMTypeOf(coords[0]);
3679       }
3680       if (sample_key & LP_SAMPLER_FETCH_MS) {
3681          arg_types[num_param++] = LLVMTypeOf(params->ms_index);
3682       }
3683       if (sample_key & LP_SAMPLER_OFFSETS) {
3684          for (i = 0; i < num_offsets; i++) {
3685             arg_types[num_param++] = LLVMTypeOf(offsets[0]);
3686             assert(LLVMTypeOf(offsets[0]) == LLVMTypeOf(offsets[i]));
3687          }
3688       }
3689       if (lod_control == LP_SAMPLER_LOD_BIAS ||
3690           lod_control == LP_SAMPLER_LOD_EXPLICIT) {
3691          arg_types[num_param++] = LLVMTypeOf(params->lod);
3692       }
3693       else if (lod_control == LP_SAMPLER_LOD_DERIVATIVES) {
3694          for (i = 0; i < num_derivs; i++) {
3695             arg_types[num_param++] = LLVMTypeOf(derivs->ddx[i]);
3696             arg_types[num_param++] = LLVMTypeOf(derivs->ddy[i]);
3697             assert(LLVMTypeOf(derivs->ddx[0]) == LLVMTypeOf(derivs->ddx[i]));
3698             assert(LLVMTypeOf(derivs->ddy[0]) == LLVMTypeOf(derivs->ddy[i]));
3699          }
3700       }
3701 
3702       val_type[0] = val_type[1] = val_type[2] = val_type[3] =
3703          lp_build_vec_type(gallivm, params->type);
3704       ret_type = LLVMStructTypeInContext(gallivm->context, val_type, 4, 0);
3705       function_type = LLVMFunctionType(ret_type, arg_types, num_param, 0);
3706       function = LLVMAddFunction(module, func_name, function_type);
3707 
3708       for (i = 0; i < num_param; ++i) {
3709          if(LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind) {
3710 
3711             lp_add_function_attr(function, i + 1, LP_FUNC_ATTR_NOALIAS);
3712          }
3713       }
3714 
3715       LLVMSetFunctionCallConv(function, LLVMFastCallConv);
3716       LLVMSetLinkage(function, LLVMInternalLinkage);
3717 
3718       lp_build_sample_gen_func(gallivm,
3719                                static_texture_state,
3720                                static_sampler_state,
3721                                dynamic_state,
3722                                params->type,
3723                                texture_index,
3724                                sampler_index,
3725                                function,
3726                                num_param,
3727                                sample_key);
3728    }
3729 
3730    num_args = 0;
3731    args[num_args++] = params->context_ptr;
3732    if (need_cache) {
3733       args[num_args++] = params->thread_data_ptr;
3734    }
3735    for (i = 0; i < num_coords; i++) {
3736       args[num_args++] = coords[i];
3737    }
3738    if (layer) {
3739       args[num_args++] = coords[layer];
3740    }
3741    if (sample_key & LP_SAMPLER_SHADOW) {
3742       args[num_args++] = coords[4];
3743    }
3744    if (sample_key & LP_SAMPLER_FETCH_MS) {
3745       args[num_args++] = params->ms_index;
3746    }
3747    if (sample_key & LP_SAMPLER_OFFSETS) {
3748       for (i = 0; i < num_offsets; i++) {
3749          args[num_args++] = offsets[i];
3750       }
3751    }
3752    if (lod_control == LP_SAMPLER_LOD_BIAS ||
3753        lod_control == LP_SAMPLER_LOD_EXPLICIT) {
3754       args[num_args++] = params->lod;
3755    }
3756    else if (lod_control == LP_SAMPLER_LOD_DERIVATIVES) {
3757       for (i = 0; i < num_derivs; i++) {
3758          args[num_args++] = derivs->ddx[i];
3759          args[num_args++] = derivs->ddy[i];
3760       }
3761    }
3762 
3763    assert(num_args <= LP_MAX_TEX_FUNC_ARGS);
3764 
3765    *tex_ret = LLVMBuildCall(builder, function, args, num_args, "");
3766    bb = LLVMGetInsertBlock(builder);
3767    inst = LLVMGetLastInstruction(bb);
3768    LLVMSetInstructionCallConv(inst, LLVMFastCallConv);
3769 
3770 }
3771 
3772 
3773 /**
3774  * Build texture sampling code.
3775  * Either via a function call or inline it directly.
3776  */
3777 void
lp_build_sample_soa(const struct lp_static_texture_state * static_texture_state,const struct lp_static_sampler_state * static_sampler_state,struct lp_sampler_dynamic_state * dynamic_state,struct gallivm_state * gallivm,const struct lp_sampler_params * params)3778 lp_build_sample_soa(const struct lp_static_texture_state *static_texture_state,
3779                     const struct lp_static_sampler_state *static_sampler_state,
3780                     struct lp_sampler_dynamic_state *dynamic_state,
3781                     struct gallivm_state *gallivm,
3782                     const struct lp_sampler_params *params)
3783 {
3784    boolean use_tex_func = FALSE;
3785 
3786    /*
3787     * Do not use a function call if the sampling is "simple enough".
3788     * We define this by
3789     * a) format
3790     * b) no mips (either one level only or no mip filter)
3791     * No mips will definitely make the code smaller, though
3792     * the format requirement is a bit iffy - there's some (SoA) formats
3793     * which definitely generate less code. This does happen to catch
3794     * some important cases though which are hurt quite a bit by using
3795     * a call (though not really because of the call overhead but because
3796     * they are reusing the same texture unit with some of the same
3797     * parameters).
3798     * Ideally we'd let llvm recognize this stuff by doing IPO passes.
3799     */
3800 
3801    if (USE_TEX_FUNC_CALL) {
3802       const struct util_format_description *format_desc;
3803       boolean simple_format;
3804       boolean simple_tex;
3805       enum lp_sampler_op_type op_type;
3806       format_desc = util_format_description(static_texture_state->format);
3807       simple_format = !format_desc ||
3808                          (util_format_is_rgba8_variant(format_desc) &&
3809                           format_desc->colorspace == UTIL_FORMAT_COLORSPACE_RGB);
3810 
3811       op_type = (params->sample_key & LP_SAMPLER_OP_TYPE_MASK) >>
3812                     LP_SAMPLER_OP_TYPE_SHIFT;
3813       simple_tex =
3814          op_type != LP_SAMPLER_OP_TEXTURE ||
3815            ((static_sampler_state->min_mip_filter == PIPE_TEX_MIPFILTER_NONE ||
3816              static_texture_state->level_zero_only == TRUE) &&
3817             static_sampler_state->min_img_filter == static_sampler_state->mag_img_filter);
3818 
3819       use_tex_func = format_desc && !(simple_format && simple_tex);
3820    }
3821 
3822    if (use_tex_func) {
3823       LLVMValueRef tex_ret;
3824       lp_build_sample_soa_func(gallivm,
3825                                static_texture_state,
3826                                static_sampler_state,
3827                                dynamic_state,
3828                                params, params->texture_index, params->sampler_index, &tex_ret);
3829 
3830       for (unsigned i = 0; i < 4; i++) {
3831          params->texel[i] = LLVMBuildExtractValue(gallivm->builder, tex_ret, i, "");
3832       }
3833    }
3834    else {
3835       lp_build_sample_soa_code(gallivm,
3836                                static_texture_state,
3837                                static_sampler_state,
3838                                dynamic_state,
3839                                params->type,
3840                                params->sample_key,
3841                                params->texture_index,
3842                                params->sampler_index,
3843                                params->context_ptr,
3844                                params->thread_data_ptr,
3845                                params->coords,
3846                                params->offsets,
3847                                params->derivs,
3848                                params->lod,
3849                                params->ms_index,
3850                                params->texel);
3851    }
3852 }
3853 
3854 
3855 void
lp_build_size_query_soa(struct gallivm_state * gallivm,const struct lp_static_texture_state * static_state,struct lp_sampler_dynamic_state * dynamic_state,const struct lp_sampler_size_query_params * params)3856 lp_build_size_query_soa(struct gallivm_state *gallivm,
3857                         const struct lp_static_texture_state *static_state,
3858                         struct lp_sampler_dynamic_state *dynamic_state,
3859                         const struct lp_sampler_size_query_params *params)
3860 {
3861    LLVMValueRef lod, level = 0, size;
3862    LLVMValueRef first_level = NULL;
3863    int dims, i;
3864    boolean has_array;
3865    unsigned num_lods = 1;
3866    struct lp_build_context bld_int_vec4;
3867    LLVMValueRef context_ptr = params->context_ptr;
3868    unsigned texture_unit = params->texture_unit;
3869    unsigned target = params->target;
3870    LLVMValueRef texture_unit_offset = params->texture_unit_offset;
3871 
3872    if (static_state->format == PIPE_FORMAT_NONE) {
3873       /*
3874        * If there's nothing bound, format is NONE, and we must return
3875        * all zero as mandated by d3d10 in this case.
3876        */
3877       unsigned chan;
3878       LLVMValueRef zero = lp_build_const_vec(gallivm, params->int_type, 0.0F);
3879       for (chan = 0; chan < 4; chan++) {
3880          params->sizes_out[chan] = zero;
3881       }
3882       return;
3883    }
3884 
3885    /*
3886     * Do some sanity verification about bound texture and shader dcl target.
3887     * Not entirely sure what's possible but assume array/non-array
3888     * always compatible (probably not ok for OpenGL but d3d10 has no
3889     * distinction of arrays at the resource level).
3890     * Everything else looks bogus (though not entirely sure about rect/2d).
3891     * Currently disabled because it causes assertion failures if there's
3892     * nothing bound (or rather a dummy texture, not that this case would
3893     * return the right values).
3894     */
3895    if (0 && static_state->target != target) {
3896       if (static_state->target == PIPE_TEXTURE_1D)
3897          assert(target == PIPE_TEXTURE_1D_ARRAY);
3898       else if (static_state->target == PIPE_TEXTURE_1D_ARRAY)
3899          assert(target == PIPE_TEXTURE_1D);
3900       else if (static_state->target == PIPE_TEXTURE_2D)
3901          assert(target == PIPE_TEXTURE_2D_ARRAY);
3902       else if (static_state->target == PIPE_TEXTURE_2D_ARRAY)
3903          assert(target == PIPE_TEXTURE_2D);
3904       else if (static_state->target == PIPE_TEXTURE_CUBE)
3905          assert(target == PIPE_TEXTURE_CUBE_ARRAY);
3906       else if (static_state->target == PIPE_TEXTURE_CUBE_ARRAY)
3907          assert(target == PIPE_TEXTURE_CUBE);
3908       else
3909          assert(0);
3910    }
3911 
3912    dims = texture_dims(target);
3913 
3914    switch (target) {
3915    case PIPE_TEXTURE_1D_ARRAY:
3916    case PIPE_TEXTURE_2D_ARRAY:
3917    case PIPE_TEXTURE_CUBE_ARRAY:
3918       has_array = TRUE;
3919       break;
3920    default:
3921       has_array = FALSE;
3922       break;
3923    }
3924 
3925    assert(!params->int_type.floating);
3926 
3927    lp_build_context_init(&bld_int_vec4, gallivm, lp_type_int_vec(32, 128));
3928 
3929    if (params->samples_only) {
3930       params->sizes_out[0] = lp_build_broadcast(gallivm, lp_build_vec_type(gallivm, params->int_type),
3931                                                 dynamic_state->num_samples(dynamic_state, gallivm,
3932                                                                            context_ptr, texture_unit,
3933                                                                            texture_unit_offset));
3934       return;
3935    }
3936    if (params->explicit_lod) {
3937       /* FIXME: this needs to honor per-element lod */
3938       lod = LLVMBuildExtractElement(gallivm->builder, params->explicit_lod,
3939                                     lp_build_const_int32(gallivm, 0), "");
3940       first_level = dynamic_state->first_level(dynamic_state, gallivm,
3941                                                context_ptr, texture_unit, texture_unit_offset);
3942       level = LLVMBuildAdd(gallivm->builder, lod, first_level, "level");
3943       lod = lp_build_broadcast_scalar(&bld_int_vec4, level);
3944    } else {
3945       lod = bld_int_vec4.zero;
3946    }
3947 
3948    size = bld_int_vec4.undef;
3949 
3950    size = LLVMBuildInsertElement(gallivm->builder, size,
3951                                  dynamic_state->width(dynamic_state, gallivm,
3952                                                       context_ptr, texture_unit, texture_unit_offset),
3953                                  lp_build_const_int32(gallivm, 0), "");
3954 
3955    if (dims >= 2) {
3956       size = LLVMBuildInsertElement(gallivm->builder, size,
3957                                     dynamic_state->height(dynamic_state, gallivm,
3958                                                           context_ptr, texture_unit, texture_unit_offset),
3959                                     lp_build_const_int32(gallivm, 1), "");
3960    }
3961 
3962    if (dims >= 3) {
3963       size = LLVMBuildInsertElement(gallivm->builder, size,
3964                                     dynamic_state->depth(dynamic_state, gallivm,
3965                                                          context_ptr, texture_unit, texture_unit_offset),
3966                                     lp_build_const_int32(gallivm, 2), "");
3967    }
3968 
3969    size = lp_build_minify(&bld_int_vec4, size, lod, TRUE);
3970 
3971    if (has_array) {
3972       LLVMValueRef layers = dynamic_state->depth(dynamic_state, gallivm,
3973                                                  context_ptr, texture_unit, texture_unit_offset);
3974       if (target == PIPE_TEXTURE_CUBE_ARRAY) {
3975          /*
3976           * It looks like GL wants number of cubes, d3d10.1 has it undefined?
3977           * Could avoid this by passing in number of cubes instead of total
3978           * number of layers (might make things easier elsewhere too).
3979           */
3980          LLVMValueRef six = lp_build_const_int32(gallivm, 6);
3981          layers = LLVMBuildSDiv(gallivm->builder, layers, six, "");
3982       }
3983       size = LLVMBuildInsertElement(gallivm->builder, size, layers,
3984                                     lp_build_const_int32(gallivm, dims), "");
3985    }
3986 
3987    /*
3988     * d3d10 requires zero for x/y/z values (but not w, i.e. mip levels)
3989     * if level is out of bounds (note this can't cover unbound texture
3990     * here, which also requires returning zero).
3991     */
3992    if (params->explicit_lod && params->is_sviewinfo) {
3993       LLVMValueRef last_level, out, out1;
3994       struct lp_build_context leveli_bld;
3995 
3996       /* everything is scalar for now */
3997       lp_build_context_init(&leveli_bld, gallivm, lp_type_int_vec(32, 32));
3998       last_level = dynamic_state->last_level(dynamic_state, gallivm,
3999                                              context_ptr, texture_unit, texture_unit_offset);
4000 
4001       out = lp_build_cmp(&leveli_bld, PIPE_FUNC_LESS, level, first_level);
4002       out1 = lp_build_cmp(&leveli_bld, PIPE_FUNC_GREATER, level, last_level);
4003       out = lp_build_or(&leveli_bld, out, out1);
4004       if (num_lods == 1) {
4005          out = lp_build_broadcast_scalar(&bld_int_vec4, out);
4006       }
4007       else {
4008          /* TODO */
4009          assert(0);
4010       }
4011       size = lp_build_andnot(&bld_int_vec4, size, out);
4012    }
4013    for (i = 0; i < dims + (has_array ? 1 : 0); i++) {
4014       params->sizes_out[i] = lp_build_extract_broadcast(gallivm, bld_int_vec4.type, params->int_type,
4015                                                 size,
4016                                                 lp_build_const_int32(gallivm, i));
4017    }
4018    if (params->is_sviewinfo) {
4019       for (; i < 4; i++) {
4020          params->sizes_out[i] = lp_build_const_vec(gallivm, params->int_type, 0.0);
4021       }
4022    }
4023 
4024    /*
4025     * if there's no explicit_lod (buffers, rects) queries requiring nr of
4026     * mips would be illegal.
4027     */
4028    if (params->is_sviewinfo && params->explicit_lod) {
4029       struct lp_build_context bld_int_scalar;
4030       LLVMValueRef num_levels;
4031       lp_build_context_init(&bld_int_scalar, gallivm, lp_type_int(32));
4032 
4033       if (static_state->level_zero_only) {
4034          num_levels = bld_int_scalar.one;
4035       }
4036       else {
4037          LLVMValueRef last_level;
4038 
4039          last_level = dynamic_state->last_level(dynamic_state, gallivm,
4040                                                 context_ptr, texture_unit, texture_unit_offset);
4041          num_levels = lp_build_sub(&bld_int_scalar, last_level, first_level);
4042          num_levels = lp_build_add(&bld_int_scalar, num_levels, bld_int_scalar.one);
4043       }
4044       params->sizes_out[3] = lp_build_broadcast(gallivm, lp_build_vec_type(gallivm, params->int_type),
4045                                         num_levels);
4046    }
4047 }
4048 
4049 static void
lp_build_do_atomic_soa(struct gallivm_state * gallivm,const struct util_format_description * format_desc,struct lp_type type,LLVMValueRef exec_mask,LLVMValueRef base_ptr,LLVMValueRef offset,LLVMValueRef out_of_bounds,unsigned img_op,LLVMAtomicRMWBinOp op,const LLVMValueRef rgba_in[4],const LLVMValueRef rgba2_in[4],LLVMValueRef atomic_result[4])4050 lp_build_do_atomic_soa(struct gallivm_state *gallivm,
4051                        const struct util_format_description *format_desc,
4052                        struct lp_type type,
4053                        LLVMValueRef exec_mask,
4054                        LLVMValueRef base_ptr,
4055                        LLVMValueRef offset,
4056                        LLVMValueRef out_of_bounds,
4057                        unsigned img_op,
4058                        LLVMAtomicRMWBinOp op,
4059                        const LLVMValueRef rgba_in[4],
4060                        const LLVMValueRef rgba2_in[4],
4061                        LLVMValueRef atomic_result[4])
4062 {
4063    enum pipe_format format = format_desc->format;
4064 
4065    if (format != PIPE_FORMAT_R32_UINT && format != PIPE_FORMAT_R32_SINT && format != PIPE_FORMAT_R32_FLOAT) {
4066       atomic_result[0] = lp_build_zero(gallivm, type);
4067       return;
4068    }
4069 
4070    LLVMValueRef atom_res = lp_build_alloca(gallivm,
4071                                            LLVMVectorType(LLVMInt32TypeInContext(gallivm->context), type.length), "");
4072 
4073    offset = LLVMBuildGEP(gallivm->builder, base_ptr, &offset, 1, "");
4074    struct lp_build_loop_state loop_state;
4075    lp_build_loop_begin(&loop_state, gallivm, lp_build_const_int32(gallivm, 0));
4076    struct lp_build_if_state ifthen;
4077    LLVMValueRef cond;
4078    LLVMValueRef packed = rgba_in[0], packed2 = rgba2_in[0];
4079 
4080    LLVMValueRef should_store_mask = LLVMBuildAnd(gallivm->builder, exec_mask, LLVMBuildNot(gallivm->builder, out_of_bounds, ""), "store_mask");
4081    assert(exec_mask);
4082 
4083    cond = LLVMBuildICmp(gallivm->builder, LLVMIntNE, should_store_mask, lp_build_const_int_vec(gallivm, type, 0), "");
4084    cond = LLVMBuildExtractElement(gallivm->builder, cond, loop_state.counter, "");
4085    lp_build_if(&ifthen, gallivm, cond);
4086 
4087    LLVMValueRef data = LLVMBuildExtractElement(gallivm->builder, packed, loop_state.counter, "");
4088    LLVMValueRef cast_base_ptr = LLVMBuildExtractElement(gallivm->builder, offset, loop_state.counter, "");
4089    cast_base_ptr = LLVMBuildBitCast(gallivm->builder, cast_base_ptr, LLVMPointerType(LLVMInt32TypeInContext(gallivm->context), 0), "");
4090    data = LLVMBuildBitCast(gallivm->builder, data, LLVMInt32TypeInContext(gallivm->context), "");
4091 
4092    if (img_op == LP_IMG_ATOMIC_CAS) {
4093       LLVMValueRef cas_src_ptr = LLVMBuildExtractElement(gallivm->builder, packed2, loop_state.counter, "");
4094       LLVMValueRef cas_src = LLVMBuildBitCast(gallivm->builder, cas_src_ptr, LLVMInt32TypeInContext(gallivm->context), "");
4095       data = LLVMBuildAtomicCmpXchg(gallivm->builder, cast_base_ptr, data,
4096                                     cas_src,
4097                                     LLVMAtomicOrderingSequentiallyConsistent,
4098                                     LLVMAtomicOrderingSequentiallyConsistent,
4099                                     false);
4100       data = LLVMBuildExtractValue(gallivm->builder, data, 0, "");
4101    } else {
4102       data = LLVMBuildAtomicRMW(gallivm->builder, op,
4103                                 cast_base_ptr, data,
4104                                 LLVMAtomicOrderingSequentiallyConsistent,
4105                                 false);
4106    }
4107 
4108    LLVMValueRef temp_res = LLVMBuildLoad(gallivm->builder, atom_res, "");
4109    temp_res = LLVMBuildInsertElement(gallivm->builder, temp_res, data, loop_state.counter, "");
4110    LLVMBuildStore(gallivm->builder, temp_res, atom_res);
4111 
4112    lp_build_endif(&ifthen);
4113    lp_build_loop_end_cond(&loop_state, lp_build_const_int32(gallivm, type.length),
4114                           NULL, LLVMIntUGE);
4115    atomic_result[0] = LLVMBuildLoad(gallivm->builder, atom_res, "");
4116 }
4117 
4118 static void
lp_build_img_op_no_format(struct gallivm_state * gallivm,const struct lp_img_params * params,LLVMValueRef outdata[4])4119 lp_build_img_op_no_format(struct gallivm_state *gallivm,
4120                           const struct lp_img_params *params,
4121                           LLVMValueRef outdata[4])
4122 {
4123    /*
4124     * If there's nothing bound, format is NONE, and we must return
4125     * all zero as mandated by d3d10 in this case.
4126        */
4127    if (params->img_op != LP_IMG_STORE) {
4128       LLVMValueRef zero = lp_build_zero(gallivm, params->type);
4129       for (unsigned chan = 0; chan < (params->img_op == LP_IMG_LOAD ? 4 : 1); chan++) {
4130          outdata[chan] = zero;
4131       }
4132    }
4133 }
4134 
4135 void
lp_build_img_op_soa(const struct lp_static_texture_state * static_texture_state,struct lp_sampler_dynamic_state * dynamic_state,struct gallivm_state * gallivm,const struct lp_img_params * params,LLVMValueRef outdata[4])4136 lp_build_img_op_soa(const struct lp_static_texture_state *static_texture_state,
4137                     struct lp_sampler_dynamic_state *dynamic_state,
4138                     struct gallivm_state *gallivm,
4139                     const struct lp_img_params *params,
4140                     LLVMValueRef outdata[4])
4141 {
4142    unsigned target = params->target;
4143    unsigned dims = texture_dims(target);
4144    /** regular scalar int type */
4145    struct lp_type int_type, int_coord_type;
4146    struct lp_build_context int_bld, int_coord_bld;
4147    const struct util_format_description *format_desc = util_format_description(static_texture_state->format);
4148    LLVMValueRef x = params->coords[0], y = params->coords[1], z = params->coords[2];
4149    LLVMValueRef ms_index = params->ms_index;
4150    LLVMValueRef row_stride_vec = NULL, img_stride_vec = NULL;
4151    int_type = lp_type_int(32);
4152    int_coord_type = lp_int_type(params->type);
4153    lp_build_context_init(&int_bld, gallivm, int_type);
4154    lp_build_context_init(&int_coord_bld, gallivm, int_coord_type);
4155 
4156    if (static_texture_state->format == PIPE_FORMAT_NONE) {
4157       lp_build_img_op_no_format(gallivm, params, outdata);
4158       return;
4159    }
4160    LLVMValueRef offset, i, j;
4161 
4162    LLVMValueRef row_stride = dynamic_state->row_stride(dynamic_state, gallivm,
4163                                                        params->context_ptr, params->image_index, NULL);
4164    LLVMValueRef img_stride = dynamic_state->img_stride(dynamic_state, gallivm,
4165                                                        params->context_ptr, params->image_index, NULL);
4166    LLVMValueRef base_ptr = dynamic_state->base_ptr(dynamic_state, gallivm,
4167                                                    params->context_ptr, params->image_index, NULL);
4168    LLVMValueRef width = dynamic_state->width(dynamic_state, gallivm,
4169                                                 params->context_ptr, params->image_index, NULL);
4170    LLVMValueRef height = dynamic_state->height(dynamic_state, gallivm,
4171                                                params->context_ptr, params->image_index, NULL);
4172    LLVMValueRef depth = dynamic_state->depth(dynamic_state, gallivm,
4173                                               params->context_ptr, params->image_index, NULL);
4174    LLVMValueRef num_samples = NULL, sample_stride = NULL;
4175    if (ms_index) {
4176       num_samples = dynamic_state->num_samples(dynamic_state, gallivm,
4177                                                params->context_ptr, params->image_index, NULL);
4178       sample_stride = dynamic_state->sample_stride(dynamic_state, gallivm,
4179                                                    params->context_ptr, params->image_index, NULL);
4180    }
4181 
4182    boolean layer_coord = has_layer_coord(target);
4183 
4184    width = lp_build_broadcast_scalar(&int_coord_bld, width);
4185    if (dims >= 2) {
4186       height = lp_build_broadcast_scalar(&int_coord_bld, height);
4187       row_stride_vec = lp_build_broadcast_scalar(&int_coord_bld, row_stride);
4188    }
4189    if (dims >= 3 || layer_coord) {
4190       depth = lp_build_broadcast_scalar(&int_coord_bld, depth);
4191       img_stride_vec = lp_build_broadcast_scalar(&int_coord_bld, img_stride);
4192    }
4193 
4194    LLVMValueRef out_of_bounds = int_coord_bld.zero;
4195    LLVMValueRef out1;
4196    out1 = lp_build_cmp(&int_coord_bld, PIPE_FUNC_GEQUAL, x, width);
4197    out_of_bounds = lp_build_or(&int_coord_bld, out_of_bounds, out1);
4198 
4199    if (dims >= 2) {
4200       out1 = lp_build_cmp(&int_coord_bld, PIPE_FUNC_GEQUAL, y, height);
4201       out_of_bounds = lp_build_or(&int_coord_bld, out_of_bounds, out1);
4202    }
4203    if (dims >= 3) {
4204       out1 = lp_build_cmp(&int_coord_bld, PIPE_FUNC_GEQUAL, z, depth);
4205       out_of_bounds = lp_build_or(&int_coord_bld, out_of_bounds, out1);
4206    }
4207    lp_build_sample_offset(&int_coord_bld,
4208                           format_desc,
4209                           x, y, z, row_stride_vec, img_stride_vec,
4210                           &offset, &i, &j);
4211 
4212    if (ms_index) {
4213       out1 = lp_build_cmp(&int_coord_bld, PIPE_FUNC_GEQUAL, ms_index, lp_build_broadcast_scalar(&int_coord_bld, num_samples));
4214       out_of_bounds = lp_build_or(&int_coord_bld, out_of_bounds, out1);
4215 
4216       offset = lp_build_add(&int_coord_bld, offset,
4217                             lp_build_mul(&int_coord_bld, lp_build_broadcast_scalar(&int_coord_bld, sample_stride),
4218                                          ms_index));
4219    }
4220    if (params->img_op == LP_IMG_LOAD) {
4221       struct lp_type texel_type = lp_build_texel_type(params->type, format_desc);
4222 
4223       offset = lp_build_andnot(&int_coord_bld, offset, out_of_bounds);
4224       struct lp_build_context texel_bld;
4225       lp_build_context_init(&texel_bld, gallivm, texel_type);
4226       lp_build_fetch_rgba_soa(gallivm,
4227                               format_desc,
4228                               texel_type, TRUE,
4229                               base_ptr, offset,
4230                               i, j,
4231                               NULL,
4232                               outdata);
4233 
4234       for (unsigned chan = 0; chan < 4; chan++) {
4235          outdata[chan] = lp_build_select(&texel_bld, out_of_bounds,
4236                                          texel_bld.zero, outdata[chan]);
4237       }
4238    } else if (params->img_op == LP_IMG_STORE) {
4239       lp_build_store_rgba_soa(gallivm, format_desc, params->type, params->exec_mask, base_ptr, offset, out_of_bounds,
4240                               params->indata);
4241    } else {
4242       lp_build_do_atomic_soa(gallivm, format_desc, params->type, params->exec_mask, base_ptr, offset, out_of_bounds,
4243                              params->img_op, params->op, params->indata, params->indata2, outdata);
4244    }
4245 }
4246 
4247 /*
4248  * These functions are for indirect texture access suppoort.
4249  *
4250  * Indirect textures are implemented using a switch statement, that
4251  * takes the texture index and jumps to the sampler functions for
4252  * that texture unit.
4253  */
4254 
4255 /*
4256  * Initialise an indexed sampler switch block.
4257  *
4258  * This sets up the switch_info state and adds the LLVM flow control pieces.
4259  */
4260 void
lp_build_sample_array_init_soa(struct lp_build_sample_array_switch * switch_info,struct gallivm_state * gallivm,const struct lp_sampler_params * params,LLVMValueRef idx,unsigned base,unsigned range)4261 lp_build_sample_array_init_soa(struct lp_build_sample_array_switch *switch_info,
4262                            struct gallivm_state *gallivm,
4263                            const struct lp_sampler_params *params,
4264                            LLVMValueRef idx,
4265                            unsigned base, unsigned range)
4266 {
4267    switch_info->gallivm = gallivm;
4268    switch_info->params = *params;
4269    switch_info->base = base;
4270    switch_info->range = range;
4271 
4272    /* for generating the switch functions we don't want the texture index offset */
4273    switch_info->params.texture_index_offset = 0;
4274 
4275    LLVMBasicBlockRef initial_block = LLVMGetInsertBlock(gallivm->builder);
4276    switch_info->merge_ref = lp_build_insert_new_block(gallivm, "texmerge");
4277 
4278    switch_info->switch_ref = LLVMBuildSwitch(gallivm->builder, idx,
4279                                              switch_info->merge_ref, range - base);
4280 
4281    LLVMTypeRef val_type[4];
4282    val_type[0] = val_type[1] = val_type[2] = val_type[3] =
4283       lp_build_vec_type(gallivm, params->type);
4284    LLVMTypeRef ret_type = LLVMStructTypeInContext(gallivm->context, val_type, 4, 0);
4285 
4286    LLVMValueRef undef_val = LLVMGetUndef(ret_type);
4287 
4288    LLVMPositionBuilderAtEnd(gallivm->builder, switch_info->merge_ref);
4289 
4290    switch_info->phi = LLVMBuildPhi(gallivm->builder, ret_type, "");
4291    LLVMAddIncoming(switch_info->phi, &undef_val, &initial_block, 1);
4292 }
4293 
4294 /*
4295  * Add an individual entry to the indirect texture switch.
4296  *
4297  * This builds the sample function and links a case for it into the switch statement.
4298  */
4299 void
lp_build_sample_array_case_soa(struct lp_build_sample_array_switch * switch_info,int idx,const struct lp_static_texture_state * static_texture_state,const struct lp_static_sampler_state * static_sampler_state,struct lp_sampler_dynamic_state * dynamic_texture_state)4300 lp_build_sample_array_case_soa(struct lp_build_sample_array_switch *switch_info,
4301                            int idx,
4302                            const struct lp_static_texture_state *static_texture_state,
4303                            const struct lp_static_sampler_state *static_sampler_state,
4304                            struct lp_sampler_dynamic_state *dynamic_texture_state)
4305 {
4306    struct gallivm_state *gallivm = switch_info->gallivm;
4307    LLVMBasicBlockRef this_block = lp_build_insert_new_block(gallivm, "texblock");
4308    LLVMValueRef tex_ret;
4309 
4310    LLVMAddCase(switch_info->switch_ref, LLVMConstInt(LLVMInt32TypeInContext(gallivm->context), idx, 0), this_block);
4311    LLVMPositionBuilderAtEnd(gallivm->builder, this_block);
4312 
4313    lp_build_sample_soa_func(gallivm, static_texture_state,
4314                             static_sampler_state, dynamic_texture_state, &switch_info->params, idx, idx,
4315                             &tex_ret);
4316 
4317    LLVMAddIncoming(switch_info->phi, &tex_ret, &this_block, 1);
4318    LLVMBuildBr(gallivm->builder, switch_info->merge_ref);
4319 }
4320 
4321 /*
4322  * Finish a switch statement.
4323  *
4324  * This handles extract the results from the switch.
4325  */
lp_build_sample_array_fini_soa(struct lp_build_sample_array_switch * switch_info)4326 void lp_build_sample_array_fini_soa(struct lp_build_sample_array_switch *switch_info)
4327 {
4328    struct gallivm_state *gallivm = switch_info->gallivm;
4329 
4330    LLVMPositionBuilderAtEnd(gallivm->builder, switch_info->merge_ref);
4331    for (unsigned i = 0; i < 4; i++)
4332       switch_info->params.texel[i] = LLVMBuildExtractValue(gallivm->builder, switch_info->phi, i, "");
4333 }
4334 
4335 void
lp_build_image_op_switch_soa(struct lp_build_img_op_array_switch * switch_info,struct gallivm_state * gallivm,const struct lp_img_params * params,LLVMValueRef idx,unsigned base,unsigned range)4336 lp_build_image_op_switch_soa(struct lp_build_img_op_array_switch *switch_info,
4337                              struct gallivm_state *gallivm,
4338                              const struct lp_img_params *params,
4339                              LLVMValueRef idx,
4340                              unsigned base, unsigned range)
4341 {
4342    switch_info->gallivm = gallivm;
4343    switch_info->params = *params;
4344    switch_info->base = base;
4345    switch_info->range = range;
4346 
4347    /* for generating the switch functions we don't want the texture index offset */
4348    switch_info->params.image_index_offset = 0;
4349 
4350    LLVMBasicBlockRef initial_block = LLVMGetInsertBlock(gallivm->builder);
4351    switch_info->merge_ref = lp_build_insert_new_block(gallivm, "imgmerge");
4352 
4353    switch_info->switch_ref = LLVMBuildSwitch(gallivm->builder, idx,
4354                                              switch_info->merge_ref, range - base);
4355 
4356    if (params->img_op != LP_IMG_STORE) {
4357       LLVMTypeRef ret_type = lp_build_vec_type(gallivm, params->type);
4358       LLVMValueRef undef_val = LLVMGetUndef(ret_type);
4359 
4360       LLVMPositionBuilderAtEnd(gallivm->builder, switch_info->merge_ref);
4361 
4362       for (unsigned i = 0; i < ((params->img_op == LP_IMG_LOAD) ? 4 : 1); i++) {
4363          switch_info->phi[i] = LLVMBuildPhi(gallivm->builder, ret_type, "");
4364          LLVMAddIncoming(switch_info->phi[i], &undef_val, &initial_block, 1);
4365       }
4366    }
4367 }
4368 
4369 void
lp_build_image_op_array_case(struct lp_build_img_op_array_switch * switch_info,int idx,const struct lp_static_texture_state * static_texture_state,struct lp_sampler_dynamic_state * dynamic_state)4370 lp_build_image_op_array_case(struct lp_build_img_op_array_switch *switch_info,
4371                             int idx,
4372                             const struct lp_static_texture_state *static_texture_state,
4373                             struct lp_sampler_dynamic_state *dynamic_state)
4374 {
4375    struct gallivm_state *gallivm = switch_info->gallivm;
4376    LLVMBasicBlockRef this_block = lp_build_insert_new_block(gallivm, "img");
4377    LLVMValueRef tex_ret[4];
4378 
4379    LLVMAddCase(switch_info->switch_ref, lp_build_const_int32(gallivm, idx), this_block);
4380    LLVMPositionBuilderAtEnd(gallivm->builder, this_block);
4381 
4382    switch_info->params.image_index = idx;
4383 
4384    lp_build_img_op_soa(static_texture_state, dynamic_state, switch_info->gallivm, &switch_info->params, tex_ret);
4385    if (switch_info->params.img_op != LP_IMG_STORE) {
4386       for (unsigned i = 0; i < ((switch_info->params.img_op == LP_IMG_LOAD) ? 4 : 1); i++)
4387          tex_ret[i] = LLVMBuildBitCast(gallivm->builder, tex_ret[i], lp_build_vec_type(gallivm, switch_info->params.type), "");
4388 
4389       this_block = LLVMGetInsertBlock(gallivm->builder);
4390       for (unsigned i = 0; i < ((switch_info->params.img_op == LP_IMG_LOAD) ? 4 : 1); i++) {
4391          LLVMAddIncoming(switch_info->phi[i], &tex_ret[i], &this_block, 1);
4392       }
4393    }
4394    LLVMBuildBr(gallivm->builder, switch_info->merge_ref);
4395 }
4396 
lp_build_image_op_array_fini_soa(struct lp_build_img_op_array_switch * switch_info)4397 void lp_build_image_op_array_fini_soa(struct lp_build_img_op_array_switch *switch_info)
4398 {
4399    struct gallivm_state *gallivm = switch_info->gallivm;
4400 
4401    LLVMPositionBuilderAtEnd(gallivm->builder, switch_info->merge_ref);
4402 
4403    if (switch_info->params.img_op != LP_IMG_STORE) {
4404       for (unsigned i = 0; i < ((switch_info->params.img_op == LP_IMG_LOAD) ? 4 : 1); i++) {
4405          switch_info->params.outdata[i] = switch_info->phi[i];
4406       }
4407    }
4408 }
4409