1 /*
2 * Copyright © 2012 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "blorp_nir_builder.h"
25 #include "compiler/nir/nir_format_convert.h"
26
27 #include "blorp_priv.h"
28
29 #include "util/format_rgb9e5.h"
30 /* header-only include needed for _mesa_unorm_to_float and friends. */
31 #include "mesa/main/format_utils.h"
32 #include "util/u_math.h"
33
34 #define FILE_DEBUG_FLAG DEBUG_BLORP
35
36 static const bool split_blorp_blit_debug = false;
37
38 struct brw_blorp_blit_vars {
39 /* Input values from brw_blorp_wm_inputs */
40 nir_variable *v_discard_rect;
41 nir_variable *v_rect_grid;
42 nir_variable *v_coord_transform;
43 nir_variable *v_src_z;
44 nir_variable *v_src_offset;
45 nir_variable *v_dst_offset;
46 nir_variable *v_src_inv_size;
47 };
48
49 static void
brw_blorp_blit_vars_init(nir_builder * b,struct brw_blorp_blit_vars * v,const struct brw_blorp_blit_prog_key * key)50 brw_blorp_blit_vars_init(nir_builder *b, struct brw_blorp_blit_vars *v,
51 const struct brw_blorp_blit_prog_key *key)
52 {
53 #define LOAD_INPUT(name, type)\
54 v->v_##name = BLORP_CREATE_NIR_INPUT(b->shader, name, type);
55
56 LOAD_INPUT(discard_rect, glsl_vec4_type())
57 LOAD_INPUT(rect_grid, glsl_vec4_type())
58 LOAD_INPUT(coord_transform, glsl_vec4_type())
59 LOAD_INPUT(src_z, glsl_float_type())
60 LOAD_INPUT(src_offset, glsl_vector_type(GLSL_TYPE_UINT, 2))
61 LOAD_INPUT(dst_offset, glsl_vector_type(GLSL_TYPE_UINT, 2))
62 LOAD_INPUT(src_inv_size, glsl_vector_type(GLSL_TYPE_FLOAT, 2))
63
64 #undef LOAD_INPUT
65 }
66
67 static nir_ssa_def *
blorp_blit_get_frag_coords(nir_builder * b,const struct brw_blorp_blit_prog_key * key,struct brw_blorp_blit_vars * v)68 blorp_blit_get_frag_coords(nir_builder *b,
69 const struct brw_blorp_blit_prog_key *key,
70 struct brw_blorp_blit_vars *v)
71 {
72 nir_ssa_def *coord = nir_f2i32(b, nir_load_frag_coord(b));
73
74 /* Account for destination surface intratile offset
75 *
76 * Transformation parameters giving translation from destination to source
77 * coordinates don't take into account possible intra-tile destination
78 * offset. Therefore it has to be first subtracted from the incoming
79 * coordinates. Vertices are set up based on coordinates containing the
80 * intra-tile offset.
81 */
82 if (key->need_dst_offset)
83 coord = nir_isub(b, coord, nir_load_var(b, v->v_dst_offset));
84
85 if (key->persample_msaa_dispatch) {
86 return nir_vec3(b, nir_channel(b, coord, 0), nir_channel(b, coord, 1),
87 nir_load_sample_id(b));
88 } else {
89 return nir_vec2(b, nir_channel(b, coord, 0), nir_channel(b, coord, 1));
90 }
91 }
92
93 /**
94 * Emit code to translate from destination (X, Y) coordinates to source (X, Y)
95 * coordinates.
96 */
97 static nir_ssa_def *
blorp_blit_apply_transform(nir_builder * b,nir_ssa_def * src_pos,struct brw_blorp_blit_vars * v)98 blorp_blit_apply_transform(nir_builder *b, nir_ssa_def *src_pos,
99 struct brw_blorp_blit_vars *v)
100 {
101 nir_ssa_def *coord_transform = nir_load_var(b, v->v_coord_transform);
102
103 nir_ssa_def *offset = nir_vec2(b, nir_channel(b, coord_transform, 1),
104 nir_channel(b, coord_transform, 3));
105 nir_ssa_def *mul = nir_vec2(b, nir_channel(b, coord_transform, 0),
106 nir_channel(b, coord_transform, 2));
107
108 return nir_fadd(b, nir_fmul(b, src_pos, mul), offset);
109 }
110
111 static inline void
blorp_nir_discard_if_outside_rect(nir_builder * b,nir_ssa_def * pos,struct brw_blorp_blit_vars * v)112 blorp_nir_discard_if_outside_rect(nir_builder *b, nir_ssa_def *pos,
113 struct brw_blorp_blit_vars *v)
114 {
115 nir_ssa_def *c0, *c1, *c2, *c3;
116 nir_ssa_def *discard_rect = nir_load_var(b, v->v_discard_rect);
117 nir_ssa_def *dst_x0 = nir_channel(b, discard_rect, 0);
118 nir_ssa_def *dst_x1 = nir_channel(b, discard_rect, 1);
119 nir_ssa_def *dst_y0 = nir_channel(b, discard_rect, 2);
120 nir_ssa_def *dst_y1 = nir_channel(b, discard_rect, 3);
121
122 c0 = nir_ult(b, nir_channel(b, pos, 0), dst_x0);
123 c1 = nir_uge(b, nir_channel(b, pos, 0), dst_x1);
124 c2 = nir_ult(b, nir_channel(b, pos, 1), dst_y0);
125 c3 = nir_uge(b, nir_channel(b, pos, 1), dst_y1);
126
127 nir_ssa_def *oob = nir_ior(b, nir_ior(b, c0, c1), nir_ior(b, c2, c3));
128
129 nir_intrinsic_instr *discard =
130 nir_intrinsic_instr_create(b->shader, nir_intrinsic_discard_if);
131 discard->src[0] = nir_src_for_ssa(oob);
132 nir_builder_instr_insert(b, &discard->instr);
133 }
134
135 static nir_tex_instr *
blorp_create_nir_tex_instr(nir_builder * b,struct brw_blorp_blit_vars * v,nir_texop op,nir_ssa_def * pos,unsigned num_srcs,nir_alu_type dst_type)136 blorp_create_nir_tex_instr(nir_builder *b, struct brw_blorp_blit_vars *v,
137 nir_texop op, nir_ssa_def *pos, unsigned num_srcs,
138 nir_alu_type dst_type)
139 {
140 nir_tex_instr *tex = nir_tex_instr_create(b->shader, num_srcs);
141
142 tex->op = op;
143
144 tex->dest_type = dst_type;
145 tex->is_array = false;
146 tex->is_shadow = false;
147
148 /* Blorp only has one texture and it's bound at unit 0 */
149 tex->texture_index = 0;
150 tex->sampler_index = 0;
151
152 /* To properly handle 3-D and 2-D array textures, we pull the Z component
153 * from an input. TODO: This is a bit magic; we should probably make this
154 * more explicit in the future.
155 */
156 assert(pos->num_components >= 2);
157 if (op == nir_texop_txf || op == nir_texop_txf_ms || op == nir_texop_txf_ms_mcs) {
158 pos = nir_vec3(b, nir_channel(b, pos, 0), nir_channel(b, pos, 1),
159 nir_f2i32(b, nir_load_var(b, v->v_src_z)));
160 } else {
161 pos = nir_vec3(b, nir_channel(b, pos, 0), nir_channel(b, pos, 1),
162 nir_load_var(b, v->v_src_z));
163 }
164
165 tex->src[0].src_type = nir_tex_src_coord;
166 tex->src[0].src = nir_src_for_ssa(pos);
167 tex->coord_components = 3;
168
169 nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, NULL);
170
171 return tex;
172 }
173
174 static nir_ssa_def *
blorp_nir_tex(nir_builder * b,struct brw_blorp_blit_vars * v,const struct brw_blorp_blit_prog_key * key,nir_ssa_def * pos)175 blorp_nir_tex(nir_builder *b, struct brw_blorp_blit_vars *v,
176 const struct brw_blorp_blit_prog_key *key, nir_ssa_def *pos)
177 {
178 if (key->need_src_offset)
179 pos = nir_fadd(b, pos, nir_i2f32(b, nir_load_var(b, v->v_src_offset)));
180
181 /* If the sampler requires normalized coordinates, we need to compensate. */
182 if (key->src_coords_normalized)
183 pos = nir_fmul(b, pos, nir_load_var(b, v->v_src_inv_size));
184
185 nir_tex_instr *tex =
186 blorp_create_nir_tex_instr(b, v, nir_texop_tex, pos, 2,
187 key->texture_data_type);
188
189 assert(pos->num_components == 2);
190 tex->sampler_dim = GLSL_SAMPLER_DIM_2D;
191 tex->src[1].src_type = nir_tex_src_lod;
192 tex->src[1].src = nir_src_for_ssa(nir_imm_int(b, 0));
193
194 nir_builder_instr_insert(b, &tex->instr);
195
196 return &tex->dest.ssa;
197 }
198
199 static nir_ssa_def *
blorp_nir_txf(nir_builder * b,struct brw_blorp_blit_vars * v,nir_ssa_def * pos,nir_alu_type dst_type)200 blorp_nir_txf(nir_builder *b, struct brw_blorp_blit_vars *v,
201 nir_ssa_def *pos, nir_alu_type dst_type)
202 {
203 nir_tex_instr *tex =
204 blorp_create_nir_tex_instr(b, v, nir_texop_txf, pos, 2, dst_type);
205
206 tex->sampler_dim = GLSL_SAMPLER_DIM_3D;
207 tex->src[1].src_type = nir_tex_src_lod;
208 tex->src[1].src = nir_src_for_ssa(nir_imm_int(b, 0));
209
210 nir_builder_instr_insert(b, &tex->instr);
211
212 return &tex->dest.ssa;
213 }
214
215 static nir_ssa_def *
blorp_nir_txf_ms(nir_builder * b,struct brw_blorp_blit_vars * v,nir_ssa_def * pos,nir_ssa_def * mcs,nir_alu_type dst_type)216 blorp_nir_txf_ms(nir_builder *b, struct brw_blorp_blit_vars *v,
217 nir_ssa_def *pos, nir_ssa_def *mcs, nir_alu_type dst_type)
218 {
219 nir_tex_instr *tex =
220 blorp_create_nir_tex_instr(b, v, nir_texop_txf_ms, pos,
221 mcs != NULL ? 3 : 2, dst_type);
222
223 tex->sampler_dim = GLSL_SAMPLER_DIM_MS;
224
225 tex->src[1].src_type = nir_tex_src_ms_index;
226 if (pos->num_components == 2) {
227 tex->src[1].src = nir_src_for_ssa(nir_imm_int(b, 0));
228 } else {
229 assert(pos->num_components == 3);
230 tex->src[1].src = nir_src_for_ssa(nir_channel(b, pos, 2));
231 }
232
233 if (mcs) {
234 tex->src[2].src_type = nir_tex_src_ms_mcs;
235 tex->src[2].src = nir_src_for_ssa(mcs);
236 }
237
238 nir_builder_instr_insert(b, &tex->instr);
239
240 return &tex->dest.ssa;
241 }
242
243 static nir_ssa_def *
blorp_blit_txf_ms_mcs(nir_builder * b,struct brw_blorp_blit_vars * v,nir_ssa_def * pos)244 blorp_blit_txf_ms_mcs(nir_builder *b, struct brw_blorp_blit_vars *v,
245 nir_ssa_def *pos)
246 {
247 nir_tex_instr *tex =
248 blorp_create_nir_tex_instr(b, v, nir_texop_txf_ms_mcs,
249 pos, 1, nir_type_int);
250
251 tex->sampler_dim = GLSL_SAMPLER_DIM_MS;
252
253 nir_builder_instr_insert(b, &tex->instr);
254
255 return &tex->dest.ssa;
256 }
257
258 /**
259 * Emit code to compensate for the difference between Y and W tiling.
260 *
261 * This code modifies the X and Y coordinates according to the formula:
262 *
263 * (X', Y', S') = detile(W-MAJOR, tile(Y-MAJOR, X, Y, S))
264 *
265 * (See brw_blorp_build_nir_shader).
266 */
267 static inline nir_ssa_def *
blorp_nir_retile_y_to_w(nir_builder * b,nir_ssa_def * pos)268 blorp_nir_retile_y_to_w(nir_builder *b, nir_ssa_def *pos)
269 {
270 assert(pos->num_components == 2);
271 nir_ssa_def *x_Y = nir_channel(b, pos, 0);
272 nir_ssa_def *y_Y = nir_channel(b, pos, 1);
273
274 /* Given X and Y coordinates that describe an address using Y tiling,
275 * translate to the X and Y coordinates that describe the same address
276 * using W tiling.
277 *
278 * If we break down the low order bits of X and Y, using a
279 * single letter to represent each low-order bit:
280 *
281 * X = A << 7 | 0bBCDEFGH
282 * Y = J << 5 | 0bKLMNP (1)
283 *
284 * Then we can apply the Y tiling formula to see the memory offset being
285 * addressed:
286 *
287 * offset = (J * tile_pitch + A) << 12 | 0bBCDKLMNPEFGH (2)
288 *
289 * If we apply the W detiling formula to this memory location, that the
290 * corresponding X' and Y' coordinates are:
291 *
292 * X' = A << 6 | 0bBCDPFH (3)
293 * Y' = J << 6 | 0bKLMNEG
294 *
295 * Combining (1) and (3), we see that to transform (X, Y) to (X', Y'),
296 * we need to make the following computation:
297 *
298 * X' = (X & ~0b1011) >> 1 | (Y & 0b1) << 2 | X & 0b1 (4)
299 * Y' = (Y & ~0b1) << 1 | (X & 0b1000) >> 2 | (X & 0b10) >> 1
300 */
301 nir_ssa_def *x_W = nir_imm_int(b, 0);
302 x_W = nir_mask_shift_or(b, x_W, x_Y, 0xfffffff4, -1);
303 x_W = nir_mask_shift_or(b, x_W, y_Y, 0x1, 2);
304 x_W = nir_mask_shift_or(b, x_W, x_Y, 0x1, 0);
305
306 nir_ssa_def *y_W = nir_imm_int(b, 0);
307 y_W = nir_mask_shift_or(b, y_W, y_Y, 0xfffffffe, 1);
308 y_W = nir_mask_shift_or(b, y_W, x_Y, 0x8, -2);
309 y_W = nir_mask_shift_or(b, y_W, x_Y, 0x2, -1);
310
311 return nir_vec2(b, x_W, y_W);
312 }
313
314 /**
315 * Emit code to compensate for the difference between Y and W tiling.
316 *
317 * This code modifies the X and Y coordinates according to the formula:
318 *
319 * (X', Y', S') = detile(Y-MAJOR, tile(W-MAJOR, X, Y, S))
320 *
321 * (See brw_blorp_build_nir_shader).
322 */
323 static inline nir_ssa_def *
blorp_nir_retile_w_to_y(nir_builder * b,nir_ssa_def * pos)324 blorp_nir_retile_w_to_y(nir_builder *b, nir_ssa_def *pos)
325 {
326 assert(pos->num_components == 2);
327 nir_ssa_def *x_W = nir_channel(b, pos, 0);
328 nir_ssa_def *y_W = nir_channel(b, pos, 1);
329
330 /* Applying the same logic as above, but in reverse, we obtain the
331 * formulas:
332 *
333 * X' = (X & ~0b101) << 1 | (Y & 0b10) << 2 | (Y & 0b1) << 1 | X & 0b1
334 * Y' = (Y & ~0b11) >> 1 | (X & 0b100) >> 2
335 */
336 nir_ssa_def *x_Y = nir_imm_int(b, 0);
337 x_Y = nir_mask_shift_or(b, x_Y, x_W, 0xfffffffa, 1);
338 x_Y = nir_mask_shift_or(b, x_Y, y_W, 0x2, 2);
339 x_Y = nir_mask_shift_or(b, x_Y, y_W, 0x1, 1);
340 x_Y = nir_mask_shift_or(b, x_Y, x_W, 0x1, 0);
341
342 nir_ssa_def *y_Y = nir_imm_int(b, 0);
343 y_Y = nir_mask_shift_or(b, y_Y, y_W, 0xfffffffc, -1);
344 y_Y = nir_mask_shift_or(b, y_Y, x_W, 0x4, -2);
345
346 return nir_vec2(b, x_Y, y_Y);
347 }
348
349 /**
350 * Emit code to compensate for the difference between MSAA and non-MSAA
351 * surfaces.
352 *
353 * This code modifies the X and Y coordinates according to the formula:
354 *
355 * (X', Y', S') = encode_msaa(num_samples, IMS, X, Y, S)
356 *
357 * (See brw_blorp_blit_program).
358 */
359 static inline nir_ssa_def *
blorp_nir_encode_msaa(nir_builder * b,nir_ssa_def * pos,unsigned num_samples,enum isl_msaa_layout layout)360 blorp_nir_encode_msaa(nir_builder *b, nir_ssa_def *pos,
361 unsigned num_samples, enum isl_msaa_layout layout)
362 {
363 assert(pos->num_components == 2 || pos->num_components == 3);
364
365 switch (layout) {
366 case ISL_MSAA_LAYOUT_NONE:
367 assert(pos->num_components == 2);
368 return pos;
369 case ISL_MSAA_LAYOUT_ARRAY:
370 /* No translation needed */
371 return pos;
372 case ISL_MSAA_LAYOUT_INTERLEAVED: {
373 nir_ssa_def *x_in = nir_channel(b, pos, 0);
374 nir_ssa_def *y_in = nir_channel(b, pos, 1);
375 nir_ssa_def *s_in = pos->num_components == 2 ? nir_imm_int(b, 0) :
376 nir_channel(b, pos, 2);
377
378 nir_ssa_def *x_out = nir_imm_int(b, 0);
379 nir_ssa_def *y_out = nir_imm_int(b, 0);
380 switch (num_samples) {
381 case 2:
382 case 4:
383 /* encode_msaa(2, IMS, X, Y, S) = (X', Y', 0)
384 * where X' = (X & ~0b1) << 1 | (S & 0b1) << 1 | (X & 0b1)
385 * Y' = Y
386 *
387 * encode_msaa(4, IMS, X, Y, S) = (X', Y', 0)
388 * where X' = (X & ~0b1) << 1 | (S & 0b1) << 1 | (X & 0b1)
389 * Y' = (Y & ~0b1) << 1 | (S & 0b10) | (Y & 0b1)
390 */
391 x_out = nir_mask_shift_or(b, x_out, x_in, 0xfffffffe, 1);
392 x_out = nir_mask_shift_or(b, x_out, s_in, 0x1, 1);
393 x_out = nir_mask_shift_or(b, x_out, x_in, 0x1, 0);
394 if (num_samples == 2) {
395 y_out = y_in;
396 } else {
397 y_out = nir_mask_shift_or(b, y_out, y_in, 0xfffffffe, 1);
398 y_out = nir_mask_shift_or(b, y_out, s_in, 0x2, 0);
399 y_out = nir_mask_shift_or(b, y_out, y_in, 0x1, 0);
400 }
401 break;
402
403 case 8:
404 /* encode_msaa(8, IMS, X, Y, S) = (X', Y', 0)
405 * where X' = (X & ~0b1) << 2 | (S & 0b100) | (S & 0b1) << 1
406 * | (X & 0b1)
407 * Y' = (Y & ~0b1) << 1 | (S & 0b10) | (Y & 0b1)
408 */
409 x_out = nir_mask_shift_or(b, x_out, x_in, 0xfffffffe, 2);
410 x_out = nir_mask_shift_or(b, x_out, s_in, 0x4, 0);
411 x_out = nir_mask_shift_or(b, x_out, s_in, 0x1, 1);
412 x_out = nir_mask_shift_or(b, x_out, x_in, 0x1, 0);
413 y_out = nir_mask_shift_or(b, y_out, y_in, 0xfffffffe, 1);
414 y_out = nir_mask_shift_or(b, y_out, s_in, 0x2, 0);
415 y_out = nir_mask_shift_or(b, y_out, y_in, 0x1, 0);
416 break;
417
418 case 16:
419 /* encode_msaa(16, IMS, X, Y, S) = (X', Y', 0)
420 * where X' = (X & ~0b1) << 2 | (S & 0b100) | (S & 0b1) << 1
421 * | (X & 0b1)
422 * Y' = (Y & ~0b1) << 2 | (S & 0b1000) >> 1 (S & 0b10)
423 * | (Y & 0b1)
424 */
425 x_out = nir_mask_shift_or(b, x_out, x_in, 0xfffffffe, 2);
426 x_out = nir_mask_shift_or(b, x_out, s_in, 0x4, 0);
427 x_out = nir_mask_shift_or(b, x_out, s_in, 0x1, 1);
428 x_out = nir_mask_shift_or(b, x_out, x_in, 0x1, 0);
429 y_out = nir_mask_shift_or(b, y_out, y_in, 0xfffffffe, 2);
430 y_out = nir_mask_shift_or(b, y_out, s_in, 0x8, -1);
431 y_out = nir_mask_shift_or(b, y_out, s_in, 0x2, 0);
432 y_out = nir_mask_shift_or(b, y_out, y_in, 0x1, 0);
433 break;
434
435 default:
436 unreachable("Invalid number of samples for IMS layout");
437 }
438
439 return nir_vec2(b, x_out, y_out);
440 }
441
442 default:
443 unreachable("Invalid MSAA layout");
444 }
445 }
446
447 /**
448 * Emit code to compensate for the difference between MSAA and non-MSAA
449 * surfaces.
450 *
451 * This code modifies the X and Y coordinates according to the formula:
452 *
453 * (X', Y', S) = decode_msaa(num_samples, IMS, X, Y, S)
454 *
455 * (See brw_blorp_blit_program).
456 */
457 static inline nir_ssa_def *
blorp_nir_decode_msaa(nir_builder * b,nir_ssa_def * pos,unsigned num_samples,enum isl_msaa_layout layout)458 blorp_nir_decode_msaa(nir_builder *b, nir_ssa_def *pos,
459 unsigned num_samples, enum isl_msaa_layout layout)
460 {
461 assert(pos->num_components == 2 || pos->num_components == 3);
462
463 switch (layout) {
464 case ISL_MSAA_LAYOUT_NONE:
465 /* No translation necessary, and S should already be zero. */
466 assert(pos->num_components == 2);
467 return pos;
468 case ISL_MSAA_LAYOUT_ARRAY:
469 /* No translation necessary. */
470 return pos;
471 case ISL_MSAA_LAYOUT_INTERLEAVED: {
472 assert(pos->num_components == 2);
473
474 nir_ssa_def *x_in = nir_channel(b, pos, 0);
475 nir_ssa_def *y_in = nir_channel(b, pos, 1);
476
477 nir_ssa_def *x_out = nir_imm_int(b, 0);
478 nir_ssa_def *y_out = nir_imm_int(b, 0);
479 nir_ssa_def *s_out = nir_imm_int(b, 0);
480 switch (num_samples) {
481 case 2:
482 case 4:
483 /* decode_msaa(2, IMS, X, Y, 0) = (X', Y', S)
484 * where X' = (X & ~0b11) >> 1 | (X & 0b1)
485 * S = (X & 0b10) >> 1
486 *
487 * decode_msaa(4, IMS, X, Y, 0) = (X', Y', S)
488 * where X' = (X & ~0b11) >> 1 | (X & 0b1)
489 * Y' = (Y & ~0b11) >> 1 | (Y & 0b1)
490 * S = (Y & 0b10) | (X & 0b10) >> 1
491 */
492 x_out = nir_mask_shift_or(b, x_out, x_in, 0xfffffffc, -1);
493 x_out = nir_mask_shift_or(b, x_out, x_in, 0x1, 0);
494 if (num_samples == 2) {
495 y_out = y_in;
496 s_out = nir_mask_shift_or(b, s_out, x_in, 0x2, -1);
497 } else {
498 y_out = nir_mask_shift_or(b, y_out, y_in, 0xfffffffc, -1);
499 y_out = nir_mask_shift_or(b, y_out, y_in, 0x1, 0);
500 s_out = nir_mask_shift_or(b, s_out, x_in, 0x2, -1);
501 s_out = nir_mask_shift_or(b, s_out, y_in, 0x2, 0);
502 }
503 break;
504
505 case 8:
506 /* decode_msaa(8, IMS, X, Y, 0) = (X', Y', S)
507 * where X' = (X & ~0b111) >> 2 | (X & 0b1)
508 * Y' = (Y & ~0b11) >> 1 | (Y & 0b1)
509 * S = (X & 0b100) | (Y & 0b10) | (X & 0b10) >> 1
510 */
511 x_out = nir_mask_shift_or(b, x_out, x_in, 0xfffffff8, -2);
512 x_out = nir_mask_shift_or(b, x_out, x_in, 0x1, 0);
513 y_out = nir_mask_shift_or(b, y_out, y_in, 0xfffffffc, -1);
514 y_out = nir_mask_shift_or(b, y_out, y_in, 0x1, 0);
515 s_out = nir_mask_shift_or(b, s_out, x_in, 0x4, 0);
516 s_out = nir_mask_shift_or(b, s_out, y_in, 0x2, 0);
517 s_out = nir_mask_shift_or(b, s_out, x_in, 0x2, -1);
518 break;
519
520 case 16:
521 /* decode_msaa(16, IMS, X, Y, 0) = (X', Y', S)
522 * where X' = (X & ~0b111) >> 2 | (X & 0b1)
523 * Y' = (Y & ~0b111) >> 2 | (Y & 0b1)
524 * S = (Y & 0b100) << 1 | (X & 0b100) |
525 * (Y & 0b10) | (X & 0b10) >> 1
526 */
527 x_out = nir_mask_shift_or(b, x_out, x_in, 0xfffffff8, -2);
528 x_out = nir_mask_shift_or(b, x_out, x_in, 0x1, 0);
529 y_out = nir_mask_shift_or(b, y_out, y_in, 0xfffffff8, -2);
530 y_out = nir_mask_shift_or(b, y_out, y_in, 0x1, 0);
531 s_out = nir_mask_shift_or(b, s_out, y_in, 0x4, 1);
532 s_out = nir_mask_shift_or(b, s_out, x_in, 0x4, 0);
533 s_out = nir_mask_shift_or(b, s_out, y_in, 0x2, 0);
534 s_out = nir_mask_shift_or(b, s_out, x_in, 0x2, -1);
535 break;
536
537 default:
538 unreachable("Invalid number of samples for IMS layout");
539 }
540
541 return nir_vec3(b, x_out, y_out, s_out);
542 }
543
544 default:
545 unreachable("Invalid MSAA layout");
546 }
547 }
548
549 /**
550 * Count the number of trailing 1 bits in the given value. For example:
551 *
552 * count_trailing_one_bits(0) == 0
553 * count_trailing_one_bits(7) == 3
554 * count_trailing_one_bits(11) == 2
555 */
count_trailing_one_bits(unsigned value)556 static inline int count_trailing_one_bits(unsigned value)
557 {
558 #ifdef HAVE___BUILTIN_CTZ
559 return __builtin_ctz(~value);
560 #else
561 return util_bitcount(value & ~(value + 1));
562 #endif
563 }
564
565 static nir_ssa_def *
blorp_nir_combine_samples(nir_builder * b,struct brw_blorp_blit_vars * v,nir_ssa_def * pos,unsigned tex_samples,enum isl_aux_usage tex_aux_usage,nir_alu_type dst_type,enum blorp_filter filter)566 blorp_nir_combine_samples(nir_builder *b, struct brw_blorp_blit_vars *v,
567 nir_ssa_def *pos, unsigned tex_samples,
568 enum isl_aux_usage tex_aux_usage,
569 nir_alu_type dst_type,
570 enum blorp_filter filter)
571 {
572 nir_variable *color =
573 nir_local_variable_create(b->impl, glsl_vec4_type(), "color");
574
575 nir_ssa_def *mcs = NULL;
576 if (isl_aux_usage_has_mcs(tex_aux_usage))
577 mcs = blorp_blit_txf_ms_mcs(b, v, pos);
578
579 nir_op combine_op;
580 switch (filter) {
581 case BLORP_FILTER_AVERAGE:
582 assert(dst_type == nir_type_float);
583 combine_op = nir_op_fadd;
584 break;
585
586 case BLORP_FILTER_MIN_SAMPLE:
587 switch (dst_type) {
588 case nir_type_int: combine_op = nir_op_imin; break;
589 case nir_type_uint: combine_op = nir_op_umin; break;
590 case nir_type_float: combine_op = nir_op_fmin; break;
591 default: unreachable("Invalid dst_type");
592 }
593 break;
594
595 case BLORP_FILTER_MAX_SAMPLE:
596 switch (dst_type) {
597 case nir_type_int: combine_op = nir_op_imax; break;
598 case nir_type_uint: combine_op = nir_op_umax; break;
599 case nir_type_float: combine_op = nir_op_fmax; break;
600 default: unreachable("Invalid dst_type");
601 }
602 break;
603
604 default:
605 unreachable("Invalid filter");
606 }
607
608 /* If true, we inserted an if statement that we need to pop at at the end.
609 */
610 bool inserted_if = false;
611
612 /* We add together samples using a binary tree structure, e.g. for 4x MSAA:
613 *
614 * result = ((sample[0] + sample[1]) + (sample[2] + sample[3])) / 4
615 *
616 * This ensures that when all samples have the same value, no numerical
617 * precision is lost, since each addition operation always adds two equal
618 * values, and summing two equal floating point values does not lose
619 * precision.
620 *
621 * We perform this computation by treating the texture_data array as a
622 * stack and performing the following operations:
623 *
624 * - push sample 0 onto stack
625 * - push sample 1 onto stack
626 * - add top two stack entries
627 * - push sample 2 onto stack
628 * - push sample 3 onto stack
629 * - add top two stack entries
630 * - add top two stack entries
631 * - divide top stack entry by 4
632 *
633 * Note that after pushing sample i onto the stack, the number of add
634 * operations we do is equal to the number of trailing 1 bits in i. This
635 * works provided the total number of samples is a power of two, which it
636 * always is for i965.
637 *
638 * For integer formats, we replace the add operations with average
639 * operations and skip the final division.
640 */
641 nir_ssa_def *texture_data[5];
642 unsigned stack_depth = 0;
643 for (unsigned i = 0; i < tex_samples; ++i) {
644 assert(stack_depth == util_bitcount(i)); /* Loop invariant */
645
646 /* Push sample i onto the stack */
647 assert(stack_depth < ARRAY_SIZE(texture_data));
648
649 nir_ssa_def *ms_pos = nir_vec3(b, nir_channel(b, pos, 0),
650 nir_channel(b, pos, 1),
651 nir_imm_int(b, i));
652 texture_data[stack_depth++] = blorp_nir_txf_ms(b, v, ms_pos, mcs, dst_type);
653
654 if (i == 0 && isl_aux_usage_has_mcs(tex_aux_usage)) {
655 /* The Ivy Bridge PRM, Vol4 Part1 p27 (Multisample Control Surface)
656 * suggests an optimization:
657 *
658 * "A simple optimization with probable large return in
659 * performance is to compare the MCS value to zero (indicating
660 * all samples are on sample slice 0), and sample only from
661 * sample slice 0 using ld2dss if MCS is zero."
662 *
663 * Note that in the case where the MCS value is zero, sampling from
664 * sample slice 0 using ld2dss and sampling from sample 0 using
665 * ld2dms are equivalent (since all samples are on sample slice 0).
666 * Since we have already sampled from sample 0, all we need to do is
667 * skip the remaining fetches and averaging if MCS is zero.
668 *
669 * It's also trivial to detect when the MCS has the magic clear color
670 * value. In this case, the txf we did on sample 0 will return the
671 * clear color and we can skip the remaining fetches just like we do
672 * when MCS == 0.
673 */
674 nir_ssa_def *mcs_zero = nir_ieq_imm(b, nir_channel(b, mcs, 0), 0);
675 if (tex_samples == 16) {
676 mcs_zero = nir_iand(b, mcs_zero,
677 nir_ieq_imm(b, nir_channel(b, mcs, 1), 0));
678 }
679 nir_ssa_def *mcs_clear =
680 blorp_nir_mcs_is_clear_color(b, mcs, tex_samples);
681
682 nir_push_if(b, nir_ior(b, mcs_zero, mcs_clear));
683 nir_store_var(b, color, texture_data[0], 0xf);
684
685 nir_push_else(b, NULL);
686 inserted_if = true;
687 }
688
689 for (int j = 0; j < count_trailing_one_bits(i); j++) {
690 assert(stack_depth >= 2);
691 --stack_depth;
692
693 texture_data[stack_depth - 1] =
694 nir_build_alu(b, combine_op,
695 texture_data[stack_depth - 1],
696 texture_data[stack_depth],
697 NULL, NULL);
698 }
699 }
700
701 /* We should have just 1 sample on the stack now. */
702 assert(stack_depth == 1);
703
704 if (filter == BLORP_FILTER_AVERAGE) {
705 assert(dst_type == nir_type_float);
706 texture_data[0] = nir_fmul(b, texture_data[0],
707 nir_imm_float(b, 1.0 / tex_samples));
708 }
709
710 nir_store_var(b, color, texture_data[0], 0xf);
711
712 if (inserted_if)
713 nir_pop_if(b, NULL);
714
715 return nir_load_var(b, color);
716 }
717
718 static nir_ssa_def *
blorp_nir_manual_blend_bilinear(nir_builder * b,nir_ssa_def * pos,unsigned tex_samples,const struct brw_blorp_blit_prog_key * key,struct brw_blorp_blit_vars * v)719 blorp_nir_manual_blend_bilinear(nir_builder *b, nir_ssa_def *pos,
720 unsigned tex_samples,
721 const struct brw_blorp_blit_prog_key *key,
722 struct brw_blorp_blit_vars *v)
723 {
724 nir_ssa_def *pos_xy = nir_channels(b, pos, 0x3);
725 nir_ssa_def *rect_grid = nir_load_var(b, v->v_rect_grid);
726 nir_ssa_def *scale = nir_imm_vec2(b, key->x_scale, key->y_scale);
727
728 /* Translate coordinates to lay out the samples in a rectangular grid
729 * roughly corresponding to sample locations.
730 */
731 pos_xy = nir_fmul(b, pos_xy, scale);
732 /* Adjust coordinates so that integers represent pixel centers rather
733 * than pixel edges.
734 */
735 pos_xy = nir_fadd(b, pos_xy, nir_imm_float(b, -0.5));
736 /* Clamp the X, Y texture coordinates to properly handle the sampling of
737 * texels on texture edges.
738 */
739 pos_xy = nir_fmin(b, nir_fmax(b, pos_xy, nir_imm_float(b, 0.0)),
740 nir_vec2(b, nir_channel(b, rect_grid, 0),
741 nir_channel(b, rect_grid, 1)));
742
743 /* Store the fractional parts to be used as bilinear interpolation
744 * coefficients.
745 */
746 nir_ssa_def *frac_xy = nir_ffract(b, pos_xy);
747 /* Round the float coordinates down to nearest integer */
748 pos_xy = nir_fdiv(b, nir_ftrunc(b, pos_xy), scale);
749
750 nir_ssa_def *tex_data[4];
751 for (unsigned i = 0; i < 4; ++i) {
752 float sample_off_x = (float)(i & 0x1) / key->x_scale;
753 float sample_off_y = (float)((i >> 1) & 0x1) / key->y_scale;
754 nir_ssa_def *sample_off = nir_imm_vec2(b, sample_off_x, sample_off_y);
755
756 nir_ssa_def *sample_coords = nir_fadd(b, pos_xy, sample_off);
757 nir_ssa_def *sample_coords_int = nir_f2i32(b, sample_coords);
758
759 /* The MCS value we fetch has to match up with the pixel that we're
760 * sampling from. Since we sample from different pixels in each
761 * iteration of this "for" loop, the call to mcs_fetch() should be
762 * here inside the loop after computing the pixel coordinates.
763 */
764 nir_ssa_def *mcs = NULL;
765 if (isl_aux_usage_has_mcs(key->tex_aux_usage))
766 mcs = blorp_blit_txf_ms_mcs(b, v, sample_coords_int);
767
768 /* Compute sample index and map the sample index to a sample number.
769 * Sample index layout shows the numbering of slots in a rectangular
770 * grid of samples with in a pixel. Sample number layout shows the
771 * rectangular grid of samples roughly corresponding to the real sample
772 * locations with in a pixel.
773 *
774 * In the case of 2x MSAA, the layout of sample indices is reversed from
775 * the layout of sample numbers:
776 *
777 * sample index layout : --------- sample number layout : ---------
778 * | 0 | 1 | | 1 | 0 |
779 * --------- ---------
780 *
781 * In case of 4x MSAA, layout of sample indices matches the layout of
782 * sample numbers:
783 * ---------
784 * | 0 | 1 |
785 * ---------
786 * | 2 | 3 |
787 * ---------
788 *
789 * In case of 8x MSAA the two layouts don't match.
790 * sample index layout : --------- sample number layout : ---------
791 * | 0 | 1 | | 3 | 7 |
792 * --------- ---------
793 * | 2 | 3 | | 5 | 0 |
794 * --------- ---------
795 * | 4 | 5 | | 1 | 2 |
796 * --------- ---------
797 * | 6 | 7 | | 4 | 6 |
798 * --------- ---------
799 *
800 * Fortunately, this can be done fairly easily as:
801 * S' = (0x17306425 >> (S * 4)) & 0xf
802 *
803 * In the case of 16x MSAA the two layouts don't match.
804 * Sample index layout: Sample number layout:
805 * --------------------- ---------------------
806 * | 0 | 1 | 2 | 3 | | 15 | 10 | 9 | 7 |
807 * --------------------- ---------------------
808 * | 4 | 5 | 6 | 7 | | 4 | 1 | 3 | 13 |
809 * --------------------- ---------------------
810 * | 8 | 9 | 10 | 11 | | 12 | 2 | 0 | 6 |
811 * --------------------- ---------------------
812 * | 12 | 13 | 14 | 15 | | 11 | 8 | 5 | 14 |
813 * --------------------- ---------------------
814 *
815 * This is equivalent to
816 * S' = (0xe58b602cd31479af >> (S * 4)) & 0xf
817 */
818 nir_ssa_def *frac = nir_ffract(b, sample_coords);
819 nir_ssa_def *sample =
820 nir_fdot2(b, frac, nir_imm_vec2(b, key->x_scale,
821 key->x_scale * key->y_scale));
822 sample = nir_f2i32(b, sample);
823
824 if (tex_samples == 2) {
825 sample = nir_isub(b, nir_imm_int(b, 1), sample);
826 } else if (tex_samples == 8) {
827 sample = nir_iand(b, nir_ishr(b, nir_imm_int(b, 0x64210573),
828 nir_ishl(b, sample, nir_imm_int(b, 2))),
829 nir_imm_int(b, 0xf));
830 } else if (tex_samples == 16) {
831 nir_ssa_def *sample_low =
832 nir_iand(b, nir_ishr(b, nir_imm_int(b, 0xd31479af),
833 nir_ishl(b, sample, nir_imm_int(b, 2))),
834 nir_imm_int(b, 0xf));
835 nir_ssa_def *sample_high =
836 nir_iand(b, nir_ishr(b, nir_imm_int(b, 0xe58b602c),
837 nir_ishl(b, nir_iadd(b, sample,
838 nir_imm_int(b, -8)),
839 nir_imm_int(b, 2))),
840 nir_imm_int(b, 0xf));
841
842 sample = nir_bcsel(b, nir_ilt(b, sample, nir_imm_int(b, 8)),
843 sample_low, sample_high);
844 }
845 nir_ssa_def *pos_ms = nir_vec3(b, nir_channel(b, sample_coords_int, 0),
846 nir_channel(b, sample_coords_int, 1),
847 sample);
848 tex_data[i] = blorp_nir_txf_ms(b, v, pos_ms, mcs, key->texture_data_type);
849 }
850
851 nir_ssa_def *frac_x = nir_channel(b, frac_xy, 0);
852 nir_ssa_def *frac_y = nir_channel(b, frac_xy, 1);
853 return nir_flrp(b, nir_flrp(b, tex_data[0], tex_data[1], frac_x),
854 nir_flrp(b, tex_data[2], tex_data[3], frac_x),
855 frac_y);
856 }
857
858 /** Perform a color bit-cast operation
859 *
860 * For copy operations involving CCS, we may need to use different formats for
861 * the source and destination surfaces. The two formats must both be UINT
862 * formats and must have the same size but may have different bit layouts.
863 * For instance, we may be copying from R8G8B8A8_UINT to R32_UINT or R32_UINT
864 * to R16G16_UINT. This function generates code to shuffle bits around to get
865 * us from one to the other.
866 */
867 static nir_ssa_def *
bit_cast_color(struct nir_builder * b,nir_ssa_def * color,const struct brw_blorp_blit_prog_key * key)868 bit_cast_color(struct nir_builder *b, nir_ssa_def *color,
869 const struct brw_blorp_blit_prog_key *key)
870 {
871 if (key->src_format == key->dst_format)
872 return color;
873
874 const struct isl_format_layout *src_fmtl =
875 isl_format_get_layout(key->src_format);
876 const struct isl_format_layout *dst_fmtl =
877 isl_format_get_layout(key->dst_format);
878
879 /* They must be formats with the same bit size */
880 assert(src_fmtl->bpb == dst_fmtl->bpb);
881
882 if (src_fmtl->bpb <= 32) {
883 assert(src_fmtl->channels.r.type == ISL_UINT ||
884 src_fmtl->channels.r.type == ISL_UNORM);
885 assert(dst_fmtl->channels.r.type == ISL_UINT ||
886 dst_fmtl->channels.r.type == ISL_UNORM);
887
888 nir_ssa_def *packed = nir_imm_int(b, 0);
889 for (unsigned c = 0; c < 4; c++) {
890 if (src_fmtl->channels_array[c].bits == 0)
891 continue;
892
893 const unsigned chan_start_bit = src_fmtl->channels_array[c].start_bit;
894 const unsigned chan_bits = src_fmtl->channels_array[c].bits;
895
896 nir_ssa_def *chan = nir_channel(b, color, c);
897 if (src_fmtl->channels_array[c].type == ISL_UNORM)
898 chan = nir_format_float_to_unorm(b, chan, &chan_bits);
899
900 packed = nir_ior(b, packed, nir_shift(b, chan, chan_start_bit));
901 }
902
903 nir_ssa_def *chans[4] = { };
904 for (unsigned c = 0; c < 4; c++) {
905 if (dst_fmtl->channels_array[c].bits == 0) {
906 chans[c] = nir_imm_int(b, 0);
907 continue;
908 }
909
910 const unsigned chan_start_bit = dst_fmtl->channels_array[c].start_bit;
911 const unsigned chan_bits = dst_fmtl->channels_array[c].bits;
912 chans[c] = nir_iand(b, nir_shift(b, packed, -(int)chan_start_bit),
913 nir_imm_int(b, BITFIELD_MASK(chan_bits)));
914
915 if (dst_fmtl->channels_array[c].type == ISL_UNORM)
916 chans[c] = nir_format_unorm_to_float(b, chans[c], &chan_bits);
917 }
918 color = nir_vec(b, chans, 4);
919 } else {
920 /* This path only supports UINT formats */
921 assert(src_fmtl->channels.r.type == ISL_UINT);
922 assert(dst_fmtl->channels.r.type == ISL_UINT);
923
924 const unsigned src_bpc = src_fmtl->channels.r.bits;
925 const unsigned dst_bpc = dst_fmtl->channels.r.bits;
926
927 assert(src_fmtl->channels.g.bits == 0 ||
928 src_fmtl->channels.g.bits == src_fmtl->channels.r.bits);
929 assert(src_fmtl->channels.b.bits == 0 ||
930 src_fmtl->channels.b.bits == src_fmtl->channels.r.bits);
931 assert(src_fmtl->channels.a.bits == 0 ||
932 src_fmtl->channels.a.bits == src_fmtl->channels.r.bits);
933 assert(dst_fmtl->channels.g.bits == 0 ||
934 dst_fmtl->channels.g.bits == dst_fmtl->channels.r.bits);
935 assert(dst_fmtl->channels.b.bits == 0 ||
936 dst_fmtl->channels.b.bits == dst_fmtl->channels.r.bits);
937 assert(dst_fmtl->channels.a.bits == 0 ||
938 dst_fmtl->channels.a.bits == dst_fmtl->channels.r.bits);
939
940 /* Restrict to only the channels we actually have */
941 const unsigned src_channels =
942 isl_format_get_num_channels(key->src_format);
943 color = nir_channels(b, color, (1 << src_channels) - 1);
944
945 color = nir_format_bitcast_uvec_unmasked(b, color, src_bpc, dst_bpc);
946 }
947
948 /* Blorp likes to assume that colors are vec4s */
949 nir_ssa_def *u = nir_ssa_undef(b, 1, 32);
950 nir_ssa_def *chans[4] = { u, u, u, u };
951 for (unsigned i = 0; i < color->num_components; i++)
952 chans[i] = nir_channel(b, color, i);
953 return nir_vec4(b, chans[0], chans[1], chans[2], chans[3]);
954 }
955
956 static nir_ssa_def *
select_color_channel(struct nir_builder * b,nir_ssa_def * color,nir_alu_type data_type,enum isl_channel_select chan)957 select_color_channel(struct nir_builder *b, nir_ssa_def *color,
958 nir_alu_type data_type,
959 enum isl_channel_select chan)
960 {
961 if (chan == ISL_CHANNEL_SELECT_ZERO) {
962 return nir_imm_int(b, 0);
963 } else if (chan == ISL_CHANNEL_SELECT_ONE) {
964 switch (data_type) {
965 case nir_type_int:
966 case nir_type_uint:
967 return nir_imm_int(b, 1);
968 case nir_type_float:
969 return nir_imm_float(b, 1);
970 default:
971 unreachable("Invalid data type");
972 }
973 } else {
974 assert((unsigned)(chan - ISL_CHANNEL_SELECT_RED) < 4);
975 return nir_channel(b, color, chan - ISL_CHANNEL_SELECT_RED);
976 }
977 }
978
979 static nir_ssa_def *
swizzle_color(struct nir_builder * b,nir_ssa_def * color,struct isl_swizzle swizzle,nir_alu_type data_type)980 swizzle_color(struct nir_builder *b, nir_ssa_def *color,
981 struct isl_swizzle swizzle, nir_alu_type data_type)
982 {
983 return nir_vec4(b,
984 select_color_channel(b, color, data_type, swizzle.r),
985 select_color_channel(b, color, data_type, swizzle.g),
986 select_color_channel(b, color, data_type, swizzle.b),
987 select_color_channel(b, color, data_type, swizzle.a));
988 }
989
990 static nir_ssa_def *
convert_color(struct nir_builder * b,nir_ssa_def * color,const struct brw_blorp_blit_prog_key * key)991 convert_color(struct nir_builder *b, nir_ssa_def *color,
992 const struct brw_blorp_blit_prog_key *key)
993 {
994 /* All of our color conversions end up generating a single-channel color
995 * value that we need to write out.
996 */
997 nir_ssa_def *value;
998
999 if (key->dst_format == ISL_FORMAT_R24_UNORM_X8_TYPELESS) {
1000 /* The destination image is bound as R32_UINT but the data needs to be
1001 * in R24_UNORM_X8_TYPELESS. The bottom 24 are the actual data and the
1002 * top 8 need to be zero. We can accomplish this by simply multiplying
1003 * by a factor to scale things down.
1004 */
1005 unsigned factor = (1 << 24) - 1;
1006 value = nir_fsat(b, nir_channel(b, color, 0));
1007 value = nir_f2i32(b, nir_fmul(b, value, nir_imm_float(b, factor)));
1008 } else if (key->dst_format == ISL_FORMAT_L8_UNORM_SRGB) {
1009 value = nir_format_linear_to_srgb(b, nir_channel(b, color, 0));
1010 } else if (key->dst_format == ISL_FORMAT_R8G8B8_UNORM_SRGB) {
1011 value = nir_format_linear_to_srgb(b, color);
1012 } else if (key->dst_format == ISL_FORMAT_R9G9B9E5_SHAREDEXP) {
1013 value = nir_format_pack_r9g9b9e5(b, color);
1014 } else {
1015 unreachable("Unsupported format conversion");
1016 }
1017
1018 nir_ssa_def *out_comps[4];
1019 for (unsigned i = 0; i < 4; i++) {
1020 if (i < value->num_components)
1021 out_comps[i] = nir_channel(b, value, i);
1022 else
1023 out_comps[i] = nir_ssa_undef(b, 1, 32);
1024 }
1025 return nir_vec(b, out_comps, 4);
1026 }
1027
1028 /**
1029 * Generator for WM programs used in BLORP blits.
1030 *
1031 * The bulk of the work done by the WM program is to wrap and unwrap the
1032 * coordinate transformations used by the hardware to store surfaces in
1033 * memory. The hardware transforms a pixel location (X, Y, S) (where S is the
1034 * sample index for a multisampled surface) to a memory offset by the
1035 * following formulas:
1036 *
1037 * offset = tile(tiling_format, encode_msaa(num_samples, layout, X, Y, S))
1038 * (X, Y, S) = decode_msaa(num_samples, layout, detile(tiling_format, offset))
1039 *
1040 * For a single-sampled surface, or for a multisampled surface using
1041 * INTEL_MSAA_LAYOUT_UMS, encode_msaa() and decode_msaa are the identity
1042 * function:
1043 *
1044 * encode_msaa(1, NONE, X, Y, 0) = (X, Y, 0)
1045 * decode_msaa(1, NONE, X, Y, 0) = (X, Y, 0)
1046 * encode_msaa(n, UMS, X, Y, S) = (X, Y, S)
1047 * decode_msaa(n, UMS, X, Y, S) = (X, Y, S)
1048 *
1049 * For a 4x multisampled surface using INTEL_MSAA_LAYOUT_IMS, encode_msaa()
1050 * embeds the sample number into bit 1 of the X and Y coordinates:
1051 *
1052 * encode_msaa(4, IMS, X, Y, S) = (X', Y', 0)
1053 * where X' = (X & ~0b1) << 1 | (S & 0b1) << 1 | (X & 0b1)
1054 * Y' = (Y & ~0b1 ) << 1 | (S & 0b10) | (Y & 0b1)
1055 * decode_msaa(4, IMS, X, Y, 0) = (X', Y', S)
1056 * where X' = (X & ~0b11) >> 1 | (X & 0b1)
1057 * Y' = (Y & ~0b11) >> 1 | (Y & 0b1)
1058 * S = (Y & 0b10) | (X & 0b10) >> 1
1059 *
1060 * For an 8x multisampled surface using INTEL_MSAA_LAYOUT_IMS, encode_msaa()
1061 * embeds the sample number into bits 1 and 2 of the X coordinate and bit 1 of
1062 * the Y coordinate:
1063 *
1064 * encode_msaa(8, IMS, X, Y, S) = (X', Y', 0)
1065 * where X' = (X & ~0b1) << 2 | (S & 0b100) | (S & 0b1) << 1 | (X & 0b1)
1066 * Y' = (Y & ~0b1) << 1 | (S & 0b10) | (Y & 0b1)
1067 * decode_msaa(8, IMS, X, Y, 0) = (X', Y', S)
1068 * where X' = (X & ~0b111) >> 2 | (X & 0b1)
1069 * Y' = (Y & ~0b11) >> 1 | (Y & 0b1)
1070 * S = (X & 0b100) | (Y & 0b10) | (X & 0b10) >> 1
1071 *
1072 * For X tiling, tile() combines together the low-order bits of the X and Y
1073 * coordinates in the pattern 0byyyxxxxxxxxx, creating 4k tiles that are 512
1074 * bytes wide and 8 rows high:
1075 *
1076 * tile(x_tiled, X, Y, S) = A
1077 * where A = tile_num << 12 | offset
1078 * tile_num = (Y' >> 3) * tile_pitch + (X' >> 9)
1079 * offset = (Y' & 0b111) << 9
1080 * | (X & 0b111111111)
1081 * X' = X * cpp
1082 * Y' = Y + S * qpitch
1083 * detile(x_tiled, A) = (X, Y, S)
1084 * where X = X' / cpp
1085 * Y = Y' % qpitch
1086 * S = Y' / qpitch
1087 * Y' = (tile_num / tile_pitch) << 3
1088 * | (A & 0b111000000000) >> 9
1089 * X' = (tile_num % tile_pitch) << 9
1090 * | (A & 0b111111111)
1091 *
1092 * (In all tiling formulas, cpp is the number of bytes occupied by a single
1093 * sample ("chars per pixel"), tile_pitch is the number of 4k tiles required
1094 * to fill the width of the surface, and qpitch is the spacing (in rows)
1095 * between array slices).
1096 *
1097 * For Y tiling, tile() combines together the low-order bits of the X and Y
1098 * coordinates in the pattern 0bxxxyyyyyxxxx, creating 4k tiles that are 128
1099 * bytes wide and 32 rows high:
1100 *
1101 * tile(y_tiled, X, Y, S) = A
1102 * where A = tile_num << 12 | offset
1103 * tile_num = (Y' >> 5) * tile_pitch + (X' >> 7)
1104 * offset = (X' & 0b1110000) << 5
1105 * | (Y' & 0b11111) << 4
1106 * | (X' & 0b1111)
1107 * X' = X * cpp
1108 * Y' = Y + S * qpitch
1109 * detile(y_tiled, A) = (X, Y, S)
1110 * where X = X' / cpp
1111 * Y = Y' % qpitch
1112 * S = Y' / qpitch
1113 * Y' = (tile_num / tile_pitch) << 5
1114 * | (A & 0b111110000) >> 4
1115 * X' = (tile_num % tile_pitch) << 7
1116 * | (A & 0b111000000000) >> 5
1117 * | (A & 0b1111)
1118 *
1119 * For W tiling, tile() combines together the low-order bits of the X and Y
1120 * coordinates in the pattern 0bxxxyyyyxyxyx, creating 4k tiles that are 64
1121 * bytes wide and 64 rows high (note that W tiling is only used for stencil
1122 * buffers, which always have cpp = 1 and S=0):
1123 *
1124 * tile(w_tiled, X, Y, S) = A
1125 * where A = tile_num << 12 | offset
1126 * tile_num = (Y' >> 6) * tile_pitch + (X' >> 6)
1127 * offset = (X' & 0b111000) << 6
1128 * | (Y' & 0b111100) << 3
1129 * | (X' & 0b100) << 2
1130 * | (Y' & 0b10) << 2
1131 * | (X' & 0b10) << 1
1132 * | (Y' & 0b1) << 1
1133 * | (X' & 0b1)
1134 * X' = X * cpp = X
1135 * Y' = Y + S * qpitch
1136 * detile(w_tiled, A) = (X, Y, S)
1137 * where X = X' / cpp = X'
1138 * Y = Y' % qpitch = Y'
1139 * S = Y / qpitch = 0
1140 * Y' = (tile_num / tile_pitch) << 6
1141 * | (A & 0b111100000) >> 3
1142 * | (A & 0b1000) >> 2
1143 * | (A & 0b10) >> 1
1144 * X' = (tile_num % tile_pitch) << 6
1145 * | (A & 0b111000000000) >> 6
1146 * | (A & 0b10000) >> 2
1147 * | (A & 0b100) >> 1
1148 * | (A & 0b1)
1149 *
1150 * Finally, for a non-tiled surface, tile() simply combines together the X and
1151 * Y coordinates in the natural way:
1152 *
1153 * tile(untiled, X, Y, S) = A
1154 * where A = Y * pitch + X'
1155 * X' = X * cpp
1156 * Y' = Y + S * qpitch
1157 * detile(untiled, A) = (X, Y, S)
1158 * where X = X' / cpp
1159 * Y = Y' % qpitch
1160 * S = Y' / qpitch
1161 * X' = A % pitch
1162 * Y' = A / pitch
1163 *
1164 * (In these formulas, pitch is the number of bytes occupied by a single row
1165 * of samples).
1166 */
1167 static nir_shader *
brw_blorp_build_nir_shader(struct blorp_context * blorp,void * mem_ctx,const struct brw_blorp_blit_prog_key * key)1168 brw_blorp_build_nir_shader(struct blorp_context *blorp, void *mem_ctx,
1169 const struct brw_blorp_blit_prog_key *key)
1170 {
1171 const struct gen_device_info *devinfo = blorp->isl_dev->info;
1172 nir_ssa_def *src_pos, *dst_pos, *color;
1173
1174 /* Sanity checks */
1175 if (key->dst_tiled_w && key->rt_samples > 1) {
1176 /* If the destination image is W tiled and multisampled, then the thread
1177 * must be dispatched once per sample, not once per pixel. This is
1178 * necessary because after conversion between W and Y tiling, there's no
1179 * guarantee that all samples corresponding to a single pixel will still
1180 * be together.
1181 */
1182 assert(key->persample_msaa_dispatch);
1183 }
1184
1185 if (key->persample_msaa_dispatch) {
1186 /* It only makes sense to do persample dispatch if the render target is
1187 * configured as multisampled.
1188 */
1189 assert(key->rt_samples > 0);
1190 }
1191
1192 /* Make sure layout is consistent with sample count */
1193 assert((key->tex_layout == ISL_MSAA_LAYOUT_NONE) ==
1194 (key->tex_samples <= 1));
1195 assert((key->rt_layout == ISL_MSAA_LAYOUT_NONE) ==
1196 (key->rt_samples <= 1));
1197 assert((key->src_layout == ISL_MSAA_LAYOUT_NONE) ==
1198 (key->src_samples <= 1));
1199 assert((key->dst_layout == ISL_MSAA_LAYOUT_NONE) ==
1200 (key->dst_samples <= 1));
1201
1202 nir_builder b;
1203 blorp_nir_init_shader(&b, mem_ctx, MESA_SHADER_FRAGMENT, NULL);
1204
1205 struct brw_blorp_blit_vars v;
1206 brw_blorp_blit_vars_init(&b, &v, key);
1207
1208 dst_pos = blorp_blit_get_frag_coords(&b, key, &v);
1209
1210 /* Render target and texture hardware don't support W tiling until Gen8. */
1211 const bool rt_tiled_w = false;
1212 const bool tex_tiled_w = devinfo->gen >= 8 && key->src_tiled_w;
1213
1214 /* The address that data will be written to is determined by the
1215 * coordinates supplied to the WM thread and the tiling and sample count of
1216 * the render target, according to the formula:
1217 *
1218 * (X, Y, S) = decode_msaa(rt_samples, detile(rt_tiling, offset))
1219 *
1220 * If the actual tiling and sample count of the destination surface are not
1221 * the same as the configuration of the render target, then these
1222 * coordinates are wrong and we have to adjust them to compensate for the
1223 * difference.
1224 */
1225 if (rt_tiled_w != key->dst_tiled_w ||
1226 key->rt_samples != key->dst_samples ||
1227 key->rt_layout != key->dst_layout) {
1228 dst_pos = blorp_nir_encode_msaa(&b, dst_pos, key->rt_samples,
1229 key->rt_layout);
1230 /* Now (X, Y, S) = detile(rt_tiling, offset) */
1231 if (rt_tiled_w != key->dst_tiled_w)
1232 dst_pos = blorp_nir_retile_y_to_w(&b, dst_pos);
1233 /* Now (X, Y, S) = detile(rt_tiling, offset) */
1234 dst_pos = blorp_nir_decode_msaa(&b, dst_pos, key->dst_samples,
1235 key->dst_layout);
1236 }
1237
1238 nir_ssa_def *comp = NULL;
1239 if (key->dst_rgb) {
1240 /* The destination image is bound as a red texture three times as wide
1241 * as the actual image. Our shader is effectively running one color
1242 * component at a time. We need to save off the component and adjust
1243 * the destination position.
1244 */
1245 assert(dst_pos->num_components == 2);
1246 nir_ssa_def *dst_x = nir_channel(&b, dst_pos, 0);
1247 comp = nir_umod(&b, dst_x, nir_imm_int(&b, 3));
1248 dst_pos = nir_vec2(&b, nir_idiv(&b, dst_x, nir_imm_int(&b, 3)),
1249 nir_channel(&b, dst_pos, 1));
1250 }
1251
1252 /* Now (X, Y, S) = decode_msaa(dst_samples, detile(dst_tiling, offset)).
1253 *
1254 * That is: X, Y and S now contain the true coordinates and sample index of
1255 * the data that the WM thread should output.
1256 *
1257 * If we need to kill pixels that are outside the destination rectangle,
1258 * now is the time to do it.
1259 */
1260 if (key->use_kill)
1261 blorp_nir_discard_if_outside_rect(&b, dst_pos, &v);
1262
1263 src_pos = blorp_blit_apply_transform(&b, nir_i2f32(&b, dst_pos), &v);
1264 if (dst_pos->num_components == 3) {
1265 /* The sample coordinate is an integer that we want left alone but
1266 * blorp_blit_apply_transform() blindly applies the transform to all
1267 * three coordinates. Grab the original sample index.
1268 */
1269 src_pos = nir_vec3(&b, nir_channel(&b, src_pos, 0),
1270 nir_channel(&b, src_pos, 1),
1271 nir_channel(&b, dst_pos, 2));
1272 }
1273
1274 /* If the source image is not multisampled, then we want to fetch sample
1275 * number 0, because that's the only sample there is.
1276 */
1277 if (key->src_samples == 1)
1278 src_pos = nir_channels(&b, src_pos, 0x3);
1279
1280 /* X, Y, and S are now the coordinates of the pixel in the source image
1281 * that we want to texture from. Exception: if we are blending, then S is
1282 * irrelevant, because we are going to fetch all samples.
1283 */
1284 switch (key->filter) {
1285 case BLORP_FILTER_NONE:
1286 case BLORP_FILTER_NEAREST:
1287 case BLORP_FILTER_SAMPLE_0:
1288 /* We're going to use texelFetch, so we need integers */
1289 if (src_pos->num_components == 2) {
1290 src_pos = nir_f2i32(&b, src_pos);
1291 } else {
1292 assert(src_pos->num_components == 3);
1293 src_pos = nir_vec3(&b, nir_channel(&b, nir_f2i32(&b, src_pos), 0),
1294 nir_channel(&b, nir_f2i32(&b, src_pos), 1),
1295 nir_channel(&b, src_pos, 2));
1296 }
1297
1298 /* We aren't blending, which means we just want to fetch a single
1299 * sample from the source surface. The address that we want to fetch
1300 * from is related to the X, Y and S values according to the formula:
1301 *
1302 * (X, Y, S) = decode_msaa(src_samples, detile(src_tiling, offset)).
1303 *
1304 * If the actual tiling and sample count of the source surface are
1305 * not the same as the configuration of the texture, then we need to
1306 * adjust the coordinates to compensate for the difference.
1307 */
1308 if (tex_tiled_w != key->src_tiled_w ||
1309 key->tex_samples != key->src_samples ||
1310 key->tex_layout != key->src_layout) {
1311 src_pos = blorp_nir_encode_msaa(&b, src_pos, key->src_samples,
1312 key->src_layout);
1313 /* Now (X, Y, S) = detile(src_tiling, offset) */
1314 if (tex_tiled_w != key->src_tiled_w)
1315 src_pos = blorp_nir_retile_w_to_y(&b, src_pos);
1316 /* Now (X, Y, S) = detile(tex_tiling, offset) */
1317 src_pos = blorp_nir_decode_msaa(&b, src_pos, key->tex_samples,
1318 key->tex_layout);
1319 }
1320
1321 if (key->need_src_offset)
1322 src_pos = nir_iadd(&b, src_pos, nir_load_var(&b, v.v_src_offset));
1323
1324 /* Now (X, Y, S) = decode_msaa(tex_samples, detile(tex_tiling, offset)).
1325 *
1326 * In other words: X, Y, and S now contain values which, when passed to
1327 * the texturing unit, will cause data to be read from the correct
1328 * memory location. So we can fetch the texel now.
1329 */
1330 if (key->src_samples == 1) {
1331 color = blorp_nir_txf(&b, &v, src_pos, key->texture_data_type);
1332 } else {
1333 nir_ssa_def *mcs = NULL;
1334 if (isl_aux_usage_has_mcs(key->tex_aux_usage))
1335 mcs = blorp_blit_txf_ms_mcs(&b, &v, src_pos);
1336
1337 color = blorp_nir_txf_ms(&b, &v, src_pos, mcs, key->texture_data_type);
1338 }
1339 break;
1340
1341 case BLORP_FILTER_BILINEAR:
1342 assert(!key->src_tiled_w);
1343 assert(key->tex_samples == key->src_samples);
1344 assert(key->tex_layout == key->src_layout);
1345
1346 if (key->src_samples == 1) {
1347 color = blorp_nir_tex(&b, &v, key, src_pos);
1348 } else {
1349 assert(!key->use_kill);
1350 color = blorp_nir_manual_blend_bilinear(&b, src_pos, key->src_samples,
1351 key, &v);
1352 }
1353 break;
1354
1355 case BLORP_FILTER_AVERAGE:
1356 case BLORP_FILTER_MIN_SAMPLE:
1357 case BLORP_FILTER_MAX_SAMPLE:
1358 assert(!key->src_tiled_w);
1359 assert(key->tex_samples == key->src_samples);
1360 assert(key->tex_layout == key->src_layout);
1361
1362 /* Resolves (effecively) use texelFetch, so we need integers and we
1363 * don't care about the sample index if we got one.
1364 */
1365 src_pos = nir_f2i32(&b, nir_channels(&b, src_pos, 0x3));
1366
1367 if (devinfo->gen == 6) {
1368 /* Because gen6 only supports 4x interleved MSAA, we can do all the
1369 * blending we need with a single linear-interpolated texture lookup
1370 * at the center of the sample. The texture coordinates to be odd
1371 * integers so that they correspond to the center of a 2x2 block
1372 * representing the four samples that maxe up a pixel. So we need
1373 * to multiply our X and Y coordinates each by 2 and then add 1.
1374 */
1375 assert(key->src_coords_normalized);
1376 assert(key->filter == BLORP_FILTER_AVERAGE);
1377 src_pos = nir_fadd(&b,
1378 nir_i2f32(&b, src_pos),
1379 nir_imm_float(&b, 0.5f));
1380 color = blorp_nir_tex(&b, &v, key, src_pos);
1381 } else {
1382 /* Gen7+ hardware doesn't automaticaly blend. */
1383 color = blorp_nir_combine_samples(&b, &v, src_pos, key->src_samples,
1384 key->tex_aux_usage,
1385 key->texture_data_type,
1386 key->filter);
1387 }
1388 break;
1389
1390 default:
1391 unreachable("Invalid blorp filter");
1392 }
1393
1394 if (!isl_swizzle_is_identity(key->src_swizzle)) {
1395 color = swizzle_color(&b, color, key->src_swizzle,
1396 key->texture_data_type);
1397 }
1398
1399 if (!isl_swizzle_is_identity(key->dst_swizzle)) {
1400 color = swizzle_color(&b, color, isl_swizzle_invert(key->dst_swizzle),
1401 nir_type_int);
1402 }
1403
1404 if (key->format_bit_cast) {
1405 assert(isl_swizzle_is_identity(key->src_swizzle));
1406 assert(isl_swizzle_is_identity(key->dst_swizzle));
1407 color = bit_cast_color(&b, color, key);
1408 } else if (key->dst_format) {
1409 color = convert_color(&b, color, key);
1410 } else if (key->uint32_to_sint) {
1411 /* Normally the hardware will take care of converting values from/to
1412 * the source and destination formats. But a few cases need help.
1413 *
1414 * The Skylake PRM, volume 07, page 658 has a programming note:
1415 *
1416 * "When using SINT or UINT rendertarget surface formats, Blending
1417 * must be DISABLED. The Pre-Blend Color Clamp Enable and Color
1418 * Clamp Range fields are ignored, and an implied clamp to the
1419 * rendertarget surface format is performed."
1420 *
1421 * For UINT to SINT blits, our sample operation gives us a uint32_t,
1422 * but our render target write expects a signed int32_t number. If we
1423 * simply passed the value along, the hardware would interpret a value
1424 * with bit 31 set as a negative value, clamping it to the largest
1425 * negative number the destination format could represent. But the
1426 * actual source value is a positive number, so we want to clamp it
1427 * to INT_MAX. To fix this, we explicitly take min(color, INT_MAX).
1428 */
1429 color = nir_umin(&b, color, nir_imm_int(&b, INT32_MAX));
1430 } else if (key->sint32_to_uint) {
1431 /* Similar to above, but clamping negative numbers to zero. */
1432 color = nir_imax(&b, color, nir_imm_int(&b, 0));
1433 }
1434
1435 if (key->dst_rgb) {
1436 /* The destination image is bound as a red texture three times as wide
1437 * as the actual image. Our shader is effectively running one color
1438 * component at a time. We need to pick off the appropriate component
1439 * from the source color and write that to destination red.
1440 */
1441 assert(dst_pos->num_components == 2);
1442
1443 nir_ssa_def *color_component =
1444 nir_bcsel(&b, nir_ieq_imm(&b, comp, 0),
1445 nir_channel(&b, color, 0),
1446 nir_bcsel(&b, nir_ieq_imm(&b, comp, 1),
1447 nir_channel(&b, color, 1),
1448 nir_channel(&b, color, 2)));
1449
1450 nir_ssa_def *u = nir_ssa_undef(&b, 1, 32);
1451 color = nir_vec4(&b, color_component, u, u, u);
1452 }
1453
1454 if (key->dst_usage == ISL_SURF_USAGE_RENDER_TARGET_BIT) {
1455 nir_variable *color_out =
1456 nir_variable_create(b.shader, nir_var_shader_out,
1457 glsl_vec4_type(), "gl_FragColor");
1458 color_out->data.location = FRAG_RESULT_COLOR;
1459 nir_store_var(&b, color_out, color, 0xf);
1460 } else if (key->dst_usage == ISL_SURF_USAGE_DEPTH_BIT) {
1461 nir_variable *depth_out =
1462 nir_variable_create(b.shader, nir_var_shader_out,
1463 glsl_float_type(), "gl_FragDepth");
1464 depth_out->data.location = FRAG_RESULT_DEPTH;
1465 nir_store_var(&b, depth_out, nir_channel(&b, color, 0), 0x1);
1466 } else if (key->dst_usage == ISL_SURF_USAGE_STENCIL_BIT) {
1467 nir_variable *stencil_out =
1468 nir_variable_create(b.shader, nir_var_shader_out,
1469 glsl_int_type(), "gl_FragStencilRef");
1470 stencil_out->data.location = FRAG_RESULT_STENCIL;
1471 nir_store_var(&b, stencil_out, nir_channel(&b, color, 0), 0x1);
1472 } else {
1473 unreachable("Invalid destination usage");
1474 }
1475
1476 return b.shader;
1477 }
1478
1479 static bool
brw_blorp_get_blit_kernel(struct blorp_batch * batch,struct blorp_params * params,const struct brw_blorp_blit_prog_key * prog_key)1480 brw_blorp_get_blit_kernel(struct blorp_batch *batch,
1481 struct blorp_params *params,
1482 const struct brw_blorp_blit_prog_key *prog_key)
1483 {
1484 struct blorp_context *blorp = batch->blorp;
1485
1486 if (blorp->lookup_shader(batch, prog_key, sizeof(*prog_key),
1487 ¶ms->wm_prog_kernel, ¶ms->wm_prog_data))
1488 return true;
1489
1490 void *mem_ctx = ralloc_context(NULL);
1491
1492 const unsigned *program;
1493 struct brw_wm_prog_data prog_data;
1494
1495 nir_shader *nir = brw_blorp_build_nir_shader(blorp, mem_ctx, prog_key);
1496 nir->info.name = ralloc_strdup(nir, blorp_shader_type_to_name(prog_key->shader_type));
1497
1498 struct brw_wm_prog_key wm_key;
1499 brw_blorp_init_wm_prog_key(&wm_key);
1500 wm_key.base.tex.compressed_multisample_layout_mask =
1501 isl_aux_usage_has_mcs(prog_key->tex_aux_usage);
1502 wm_key.base.tex.msaa_16 = prog_key->tex_samples == 16;
1503 wm_key.multisample_fbo = prog_key->rt_samples > 1;
1504
1505 program = blorp_compile_fs(blorp, mem_ctx, nir, &wm_key, false,
1506 &prog_data);
1507
1508 bool result =
1509 blorp->upload_shader(batch, MESA_SHADER_FRAGMENT,
1510 prog_key, sizeof(*prog_key),
1511 program, prog_data.base.program_size,
1512 &prog_data.base, sizeof(prog_data),
1513 ¶ms->wm_prog_kernel, ¶ms->wm_prog_data);
1514
1515 ralloc_free(mem_ctx);
1516 return result;
1517 }
1518
1519 static void
brw_blorp_setup_coord_transform(struct brw_blorp_coord_transform * xform,GLfloat src0,GLfloat src1,GLfloat dst0,GLfloat dst1,bool mirror)1520 brw_blorp_setup_coord_transform(struct brw_blorp_coord_transform *xform,
1521 GLfloat src0, GLfloat src1,
1522 GLfloat dst0, GLfloat dst1,
1523 bool mirror)
1524 {
1525 double scale = (double)(src1 - src0) / (double)(dst1 - dst0);
1526 if (!mirror) {
1527 /* When not mirroring a coordinate (say, X), we need:
1528 * src_x - src_x0 = (dst_x - dst_x0 + 0.5) * scale
1529 * Therefore:
1530 * src_x = src_x0 + (dst_x - dst_x0 + 0.5) * scale
1531 *
1532 * blorp program uses "round toward zero" to convert the
1533 * transformed floating point coordinates to integer coordinates,
1534 * whereas the behaviour we actually want is "round to nearest",
1535 * so 0.5 provides the necessary correction.
1536 */
1537 xform->multiplier = scale;
1538 xform->offset = src0 + (-(double)dst0 + 0.5) * scale;
1539 } else {
1540 /* When mirroring X we need:
1541 * src_x - src_x0 = dst_x1 - dst_x - 0.5
1542 * Therefore:
1543 * src_x = src_x0 + (dst_x1 -dst_x - 0.5) * scale
1544 */
1545 xform->multiplier = -scale;
1546 xform->offset = src0 + ((double)dst1 - 0.5) * scale;
1547 }
1548 }
1549
1550 static inline void
surf_get_intratile_offset_px(struct brw_blorp_surface_info * info,uint32_t * tile_x_px,uint32_t * tile_y_px)1551 surf_get_intratile_offset_px(struct brw_blorp_surface_info *info,
1552 uint32_t *tile_x_px, uint32_t *tile_y_px)
1553 {
1554 if (info->surf.msaa_layout == ISL_MSAA_LAYOUT_INTERLEAVED) {
1555 struct isl_extent2d px_size_sa =
1556 isl_get_interleaved_msaa_px_size_sa(info->surf.samples);
1557 assert(info->tile_x_sa % px_size_sa.width == 0);
1558 assert(info->tile_y_sa % px_size_sa.height == 0);
1559 *tile_x_px = info->tile_x_sa / px_size_sa.width;
1560 *tile_y_px = info->tile_y_sa / px_size_sa.height;
1561 } else {
1562 *tile_x_px = info->tile_x_sa;
1563 *tile_y_px = info->tile_y_sa;
1564 }
1565 }
1566
1567 void
blorp_surf_convert_to_single_slice(const struct isl_device * isl_dev,struct brw_blorp_surface_info * info)1568 blorp_surf_convert_to_single_slice(const struct isl_device *isl_dev,
1569 struct brw_blorp_surface_info *info)
1570 {
1571 bool ok UNUSED;
1572
1573 /* It would be insane to try and do this on a compressed surface */
1574 assert(info->aux_usage == ISL_AUX_USAGE_NONE);
1575
1576 /* Just bail if we have nothing to do. */
1577 if (info->surf.dim == ISL_SURF_DIM_2D &&
1578 info->view.base_level == 0 && info->view.base_array_layer == 0 &&
1579 info->surf.levels == 1 && info->surf.logical_level0_px.array_len == 1)
1580 return;
1581
1582 /* If this gets triggered then we've gotten here twice which. This
1583 * shouldn't happen thanks to the above early return.
1584 */
1585 assert(info->tile_x_sa == 0 && info->tile_y_sa == 0);
1586
1587 uint32_t layer = 0, z = 0;
1588 if (info->surf.dim == ISL_SURF_DIM_3D)
1589 z = info->view.base_array_layer + info->z_offset;
1590 else
1591 layer = info->view.base_array_layer;
1592
1593 uint32_t byte_offset;
1594 isl_surf_get_image_surf(isl_dev, &info->surf,
1595 info->view.base_level, layer, z,
1596 &info->surf,
1597 &byte_offset, &info->tile_x_sa, &info->tile_y_sa);
1598 info->addr.offset += byte_offset;
1599
1600 uint32_t tile_x_px, tile_y_px;
1601 surf_get_intratile_offset_px(info, &tile_x_px, &tile_y_px);
1602
1603 /* Instead of using the X/Y Offset fields in RENDER_SURFACE_STATE, we place
1604 * the image at the tile boundary and offset our sampling or rendering.
1605 * For this reason, we need to grow the image by the offset to ensure that
1606 * the hardware doesn't think we've gone past the edge.
1607 */
1608 info->surf.logical_level0_px.w += tile_x_px;
1609 info->surf.logical_level0_px.h += tile_y_px;
1610 info->surf.phys_level0_sa.w += info->tile_x_sa;
1611 info->surf.phys_level0_sa.h += info->tile_y_sa;
1612
1613 /* The view is also different now. */
1614 info->view.base_level = 0;
1615 info->view.levels = 1;
1616 info->view.base_array_layer = 0;
1617 info->view.array_len = 1;
1618 info->z_offset = 0;
1619 }
1620
1621 void
blorp_surf_fake_interleaved_msaa(const struct isl_device * isl_dev,struct brw_blorp_surface_info * info)1622 blorp_surf_fake_interleaved_msaa(const struct isl_device *isl_dev,
1623 struct brw_blorp_surface_info *info)
1624 {
1625 assert(info->surf.msaa_layout == ISL_MSAA_LAYOUT_INTERLEAVED);
1626
1627 /* First, we need to convert it to a simple 1-level 1-layer 2-D surface */
1628 blorp_surf_convert_to_single_slice(isl_dev, info);
1629
1630 info->surf.logical_level0_px = info->surf.phys_level0_sa;
1631 info->surf.samples = 1;
1632 info->surf.msaa_layout = ISL_MSAA_LAYOUT_NONE;
1633 }
1634
1635 void
blorp_surf_retile_w_to_y(const struct isl_device * isl_dev,struct brw_blorp_surface_info * info)1636 blorp_surf_retile_w_to_y(const struct isl_device *isl_dev,
1637 struct brw_blorp_surface_info *info)
1638 {
1639 assert(info->surf.tiling == ISL_TILING_W);
1640
1641 /* First, we need to convert it to a simple 1-level 1-layer 2-D surface */
1642 blorp_surf_convert_to_single_slice(isl_dev, info);
1643
1644 /* On gen7+, we don't have interleaved multisampling for color render
1645 * targets so we have to fake it.
1646 *
1647 * TODO: Are we sure we don't also need to fake it on gen6?
1648 */
1649 if (isl_dev->info->gen > 6 &&
1650 info->surf.msaa_layout == ISL_MSAA_LAYOUT_INTERLEAVED) {
1651 blorp_surf_fake_interleaved_msaa(isl_dev, info);
1652 }
1653
1654 if (isl_dev->info->gen == 6) {
1655 /* Gen6 stencil buffers have a very large alignment coming in from the
1656 * miptree. It's out-of-bounds for what the surface state can handle.
1657 * Since we have a single layer and level, it doesn't really matter as
1658 * long as we don't pass a bogus value into isl_surf_fill_state().
1659 */
1660 info->surf.image_alignment_el = isl_extent3d(4, 2, 1);
1661 }
1662
1663 /* Now that we've converted everything to a simple 2-D surface with only
1664 * one miplevel, we can go about retiling it.
1665 */
1666 const unsigned x_align = 8, y_align = info->surf.samples != 0 ? 8 : 4;
1667 info->surf.tiling = ISL_TILING_Y0;
1668 info->surf.logical_level0_px.width =
1669 ALIGN(info->surf.logical_level0_px.width, x_align) * 2;
1670 info->surf.logical_level0_px.height =
1671 ALIGN(info->surf.logical_level0_px.height, y_align) / 2;
1672 info->tile_x_sa *= 2;
1673 info->tile_y_sa /= 2;
1674 }
1675
1676 static bool
can_shrink_surface(const struct brw_blorp_surface_info * surf)1677 can_shrink_surface(const struct brw_blorp_surface_info *surf)
1678 {
1679 /* The current code doesn't support offsets into the aux buffers. This
1680 * should be possible, but we need to make sure the offset is page
1681 * aligned for both the surface and the aux buffer surface. Generally
1682 * this mean using the page aligned offset for the aux buffer.
1683 *
1684 * Currently the cases where we must split the blit are limited to cases
1685 * where we don't have a aux buffer.
1686 */
1687 if (surf->aux_addr.buffer != NULL)
1688 return false;
1689
1690 /* We can't support splitting the blit for gen <= 7, because the qpitch
1691 * size is calculated by the hardware based on the surface height for
1692 * gen <= 7. In gen >= 8, the qpitch is controlled by the driver.
1693 */
1694 if (surf->surf.msaa_layout == ISL_MSAA_LAYOUT_ARRAY)
1695 return false;
1696
1697 return true;
1698 }
1699
1700 static unsigned
get_max_surface_size(const struct gen_device_info * devinfo,const struct brw_blorp_surface_info * surf)1701 get_max_surface_size(const struct gen_device_info *devinfo,
1702 const struct brw_blorp_surface_info *surf)
1703 {
1704 const unsigned max = devinfo->gen >= 7 ? 16384 : 8192;
1705 if (split_blorp_blit_debug && can_shrink_surface(surf))
1706 return max >> 4; /* A smaller restriction when debug is enabled */
1707 else
1708 return max;
1709 }
1710
1711 struct blt_axis {
1712 double src0, src1, dst0, dst1;
1713 bool mirror;
1714 };
1715
1716 struct blt_coords {
1717 struct blt_axis x, y;
1718 };
1719
1720 static enum isl_format
get_red_format_for_rgb_format(enum isl_format format)1721 get_red_format_for_rgb_format(enum isl_format format)
1722 {
1723 const struct isl_format_layout *fmtl = isl_format_get_layout(format);
1724
1725 switch (fmtl->channels.r.bits) {
1726 case 8:
1727 switch (fmtl->channels.r.type) {
1728 case ISL_UNORM:
1729 return ISL_FORMAT_R8_UNORM;
1730 case ISL_SNORM:
1731 return ISL_FORMAT_R8_SNORM;
1732 case ISL_UINT:
1733 return ISL_FORMAT_R8_UINT;
1734 case ISL_SINT:
1735 return ISL_FORMAT_R8_SINT;
1736 default:
1737 unreachable("Invalid 8-bit RGB channel type");
1738 }
1739 case 16:
1740 switch (fmtl->channels.r.type) {
1741 case ISL_UNORM:
1742 return ISL_FORMAT_R16_UNORM;
1743 case ISL_SNORM:
1744 return ISL_FORMAT_R16_SNORM;
1745 case ISL_SFLOAT:
1746 return ISL_FORMAT_R16_FLOAT;
1747 case ISL_UINT:
1748 return ISL_FORMAT_R16_UINT;
1749 case ISL_SINT:
1750 return ISL_FORMAT_R16_SINT;
1751 default:
1752 unreachable("Invalid 8-bit RGB channel type");
1753 }
1754 case 32:
1755 switch (fmtl->channels.r.type) {
1756 case ISL_SFLOAT:
1757 return ISL_FORMAT_R32_FLOAT;
1758 case ISL_UINT:
1759 return ISL_FORMAT_R32_UINT;
1760 case ISL_SINT:
1761 return ISL_FORMAT_R32_SINT;
1762 default:
1763 unreachable("Invalid 8-bit RGB channel type");
1764 }
1765 default:
1766 unreachable("Invalid number of red channel bits");
1767 }
1768 }
1769
1770 void
surf_fake_rgb_with_red(const struct isl_device * isl_dev,struct brw_blorp_surface_info * info)1771 surf_fake_rgb_with_red(const struct isl_device *isl_dev,
1772 struct brw_blorp_surface_info *info)
1773 {
1774 blorp_surf_convert_to_single_slice(isl_dev, info);
1775
1776 info->surf.logical_level0_px.width *= 3;
1777 info->surf.phys_level0_sa.width *= 3;
1778 info->tile_x_sa *= 3;
1779
1780 enum isl_format red_format =
1781 get_red_format_for_rgb_format(info->view.format);
1782
1783 assert(isl_format_get_layout(red_format)->channels.r.type ==
1784 isl_format_get_layout(info->view.format)->channels.r.type);
1785 assert(isl_format_get_layout(red_format)->channels.r.bits ==
1786 isl_format_get_layout(info->view.format)->channels.r.bits);
1787
1788 info->surf.format = info->view.format = red_format;
1789 }
1790
1791 enum blit_shrink_status {
1792 BLIT_NO_SHRINK = 0,
1793 BLIT_SRC_WIDTH_SHRINK = (1 << 0),
1794 BLIT_DST_WIDTH_SHRINK = (1 << 1),
1795 BLIT_SRC_HEIGHT_SHRINK = (1 << 2),
1796 BLIT_DST_HEIGHT_SHRINK = (1 << 3),
1797 };
1798
1799 /* Try to blit. If the surface parameters exceed the size allowed by hardware,
1800 * then enum blit_shrink_status will be returned. If BLIT_NO_SHRINK is
1801 * returned, then the blit was successful.
1802 */
1803 static enum blit_shrink_status
try_blorp_blit(struct blorp_batch * batch,struct blorp_params * params,struct brw_blorp_blit_prog_key * wm_prog_key,struct blt_coords * coords)1804 try_blorp_blit(struct blorp_batch *batch,
1805 struct blorp_params *params,
1806 struct brw_blorp_blit_prog_key *wm_prog_key,
1807 struct blt_coords *coords)
1808 {
1809 const struct gen_device_info *devinfo = batch->blorp->isl_dev->info;
1810
1811 if (params->dst.surf.usage & ISL_SURF_USAGE_DEPTH_BIT) {
1812 if (devinfo->gen >= 7) {
1813 /* We can render as depth on Gen5 but there's no real advantage since
1814 * it doesn't support MSAA or HiZ. On Gen4, we can't always render
1815 * to depth due to issues with depth buffers and mip-mapping. On
1816 * Gen6, we can do everything but we have weird offsetting for HiZ
1817 * and stencil. It's easier to just render using the color pipe
1818 * on those platforms.
1819 */
1820 wm_prog_key->dst_usage = ISL_SURF_USAGE_DEPTH_BIT;
1821 } else {
1822 wm_prog_key->dst_usage = ISL_SURF_USAGE_RENDER_TARGET_BIT;
1823 }
1824 } else if (params->dst.surf.usage & ISL_SURF_USAGE_STENCIL_BIT) {
1825 assert(params->dst.surf.format == ISL_FORMAT_R8_UINT);
1826 if (devinfo->gen >= 9) {
1827 wm_prog_key->dst_usage = ISL_SURF_USAGE_STENCIL_BIT;
1828 } else {
1829 wm_prog_key->dst_usage = ISL_SURF_USAGE_RENDER_TARGET_BIT;
1830 }
1831 } else {
1832 wm_prog_key->dst_usage = ISL_SURF_USAGE_RENDER_TARGET_BIT;
1833 }
1834
1835 if (isl_format_has_sint_channel(params->src.view.format)) {
1836 wm_prog_key->texture_data_type = nir_type_int;
1837 } else if (isl_format_has_uint_channel(params->src.view.format)) {
1838 wm_prog_key->texture_data_type = nir_type_uint;
1839 } else {
1840 wm_prog_key->texture_data_type = nir_type_float;
1841 }
1842
1843 /* src_samples and dst_samples are the true sample counts */
1844 wm_prog_key->src_samples = params->src.surf.samples;
1845 wm_prog_key->dst_samples = params->dst.surf.samples;
1846
1847 wm_prog_key->tex_aux_usage = params->src.aux_usage;
1848
1849 /* src_layout and dst_layout indicate the true MSAA layout used by src and
1850 * dst.
1851 */
1852 wm_prog_key->src_layout = params->src.surf.msaa_layout;
1853 wm_prog_key->dst_layout = params->dst.surf.msaa_layout;
1854
1855 /* Round floating point values to nearest integer to avoid "off by one texel"
1856 * kind of errors when blitting.
1857 */
1858 params->x0 = params->wm_inputs.discard_rect.x0 = round(coords->x.dst0);
1859 params->y0 = params->wm_inputs.discard_rect.y0 = round(coords->y.dst0);
1860 params->x1 = params->wm_inputs.discard_rect.x1 = round(coords->x.dst1);
1861 params->y1 = params->wm_inputs.discard_rect.y1 = round(coords->y.dst1);
1862
1863 brw_blorp_setup_coord_transform(¶ms->wm_inputs.coord_transform[0],
1864 coords->x.src0, coords->x.src1,
1865 coords->x.dst0, coords->x.dst1,
1866 coords->x.mirror);
1867 brw_blorp_setup_coord_transform(¶ms->wm_inputs.coord_transform[1],
1868 coords->y.src0, coords->y.src1,
1869 coords->y.dst0, coords->y.dst1,
1870 coords->y.mirror);
1871
1872
1873 if (devinfo->gen == 4) {
1874 /* The MinLOD and MinimumArrayElement don't work properly for cube maps.
1875 * Convert them to a single slice on gen4.
1876 */
1877 if (params->dst.surf.usage & ISL_SURF_USAGE_CUBE_BIT) {
1878 blorp_surf_convert_to_single_slice(batch->blorp->isl_dev, ¶ms->dst);
1879 wm_prog_key->need_dst_offset = true;
1880 }
1881
1882 if (params->src.surf.usage & ISL_SURF_USAGE_CUBE_BIT) {
1883 blorp_surf_convert_to_single_slice(batch->blorp->isl_dev, ¶ms->src);
1884 wm_prog_key->need_src_offset = true;
1885 }
1886 }
1887
1888 if (devinfo->gen > 6 &&
1889 !isl_surf_usage_is_depth_or_stencil(wm_prog_key->dst_usage) &&
1890 params->dst.surf.msaa_layout == ISL_MSAA_LAYOUT_INTERLEAVED) {
1891 assert(params->dst.surf.samples > 1);
1892
1893 /* We must expand the rectangle we send through the rendering pipeline,
1894 * to account for the fact that we are mapping the destination region as
1895 * single-sampled when it is in fact multisampled. We must also align
1896 * it to a multiple of the multisampling pattern, because the
1897 * differences between multisampled and single-sampled surface formats
1898 * will mean that pixels are scrambled within the multisampling pattern.
1899 * TODO: what if this makes the coordinates too large?
1900 *
1901 * Note: this only works if the destination surface uses the IMS layout.
1902 * If it's UMS, then we have no choice but to set up the rendering
1903 * pipeline as multisampled.
1904 */
1905 struct isl_extent2d px_size_sa =
1906 isl_get_interleaved_msaa_px_size_sa(params->dst.surf.samples);
1907 params->x0 = ROUND_DOWN_TO(params->x0, 2) * px_size_sa.width;
1908 params->y0 = ROUND_DOWN_TO(params->y0, 2) * px_size_sa.height;
1909 params->x1 = ALIGN(params->x1, 2) * px_size_sa.width;
1910 params->y1 = ALIGN(params->y1, 2) * px_size_sa.height;
1911
1912 blorp_surf_fake_interleaved_msaa(batch->blorp->isl_dev, ¶ms->dst);
1913
1914 wm_prog_key->use_kill = true;
1915 wm_prog_key->need_dst_offset = true;
1916 }
1917
1918 if (params->dst.surf.tiling == ISL_TILING_W &&
1919 wm_prog_key->dst_usage != ISL_SURF_USAGE_STENCIL_BIT) {
1920 /* We must modify the rectangle we send through the rendering pipeline
1921 * (and the size and x/y offset of the destination surface), to account
1922 * for the fact that we are mapping it as Y-tiled when it is in fact
1923 * W-tiled.
1924 *
1925 * Both Y tiling and W tiling can be understood as organizations of
1926 * 32-byte sub-tiles; within each 32-byte sub-tile, the layout of pixels
1927 * is different, but the layout of the 32-byte sub-tiles within the 4k
1928 * tile is the same (8 sub-tiles across by 16 sub-tiles down, in
1929 * column-major order). In Y tiling, the sub-tiles are 16 bytes wide
1930 * and 2 rows high; in W tiling, they are 8 bytes wide and 4 rows high.
1931 *
1932 * Therefore, to account for the layout differences within the 32-byte
1933 * sub-tiles, we must expand the rectangle so the X coordinates of its
1934 * edges are multiples of 8 (the W sub-tile width), and its Y
1935 * coordinates of its edges are multiples of 4 (the W sub-tile height).
1936 * Then we need to scale the X and Y coordinates of the rectangle to
1937 * account for the differences in aspect ratio between the Y and W
1938 * sub-tiles. We need to modify the layer width and height similarly.
1939 *
1940 * A correction needs to be applied when MSAA is in use: since
1941 * INTEL_MSAA_LAYOUT_IMS uses an interleaving pattern whose height is 4,
1942 * we need to align the Y coordinates to multiples of 8, so that when
1943 * they are divided by two they are still multiples of 4.
1944 *
1945 * Note: Since the x/y offset of the surface will be applied using the
1946 * SURFACE_STATE command packet, it will be invisible to the swizzling
1947 * code in the shader; therefore it needs to be in a multiple of the
1948 * 32-byte sub-tile size. Fortunately it is, since the sub-tile is 8
1949 * pixels wide and 4 pixels high (when viewed as a W-tiled stencil
1950 * buffer), and the miplevel alignment used for stencil buffers is 8
1951 * pixels horizontally and either 4 or 8 pixels vertically (see
1952 * intel_horizontal_texture_alignment_unit() and
1953 * intel_vertical_texture_alignment_unit()).
1954 *
1955 * Note: Also, since the SURFACE_STATE command packet can only apply
1956 * offsets that are multiples of 4 pixels horizontally and 2 pixels
1957 * vertically, it is important that the offsets will be multiples of
1958 * these sizes after they are converted into Y-tiled coordinates.
1959 * Fortunately they will be, since we know from above that the offsets
1960 * are a multiple of the 32-byte sub-tile size, and in Y-tiled
1961 * coordinates the sub-tile is 16 pixels wide and 2 pixels high.
1962 *
1963 * TODO: what if this makes the coordinates (or the texture size) too
1964 * large?
1965 */
1966 const unsigned x_align = 8;
1967 const unsigned y_align = params->dst.surf.samples != 0 ? 8 : 4;
1968 params->x0 = ROUND_DOWN_TO(params->x0, x_align) * 2;
1969 params->y0 = ROUND_DOWN_TO(params->y0, y_align) / 2;
1970 params->x1 = ALIGN(params->x1, x_align) * 2;
1971 params->y1 = ALIGN(params->y1, y_align) / 2;
1972
1973 /* Retile the surface to Y-tiled */
1974 blorp_surf_retile_w_to_y(batch->blorp->isl_dev, ¶ms->dst);
1975
1976 wm_prog_key->dst_tiled_w = true;
1977 wm_prog_key->use_kill = true;
1978 wm_prog_key->need_dst_offset = true;
1979
1980 if (params->dst.surf.samples > 1) {
1981 /* If the destination surface is a W-tiled multisampled stencil
1982 * buffer that we're mapping as Y tiled, then we need to arrange for
1983 * the WM program to run once per sample rather than once per pixel,
1984 * because the memory layout of related samples doesn't match between
1985 * W and Y tiling.
1986 */
1987 wm_prog_key->persample_msaa_dispatch = true;
1988 }
1989 }
1990
1991 if (devinfo->gen < 8 && params->src.surf.tiling == ISL_TILING_W) {
1992 /* On Haswell and earlier, we have to fake W-tiled sources as Y-tiled.
1993 * Broadwell adds support for sampling from stencil.
1994 *
1995 * See the comments above concerning x/y offset alignment for the
1996 * destination surface.
1997 *
1998 * TODO: what if this makes the texture size too large?
1999 */
2000 blorp_surf_retile_w_to_y(batch->blorp->isl_dev, ¶ms->src);
2001
2002 wm_prog_key->src_tiled_w = true;
2003 wm_prog_key->need_src_offset = true;
2004 }
2005
2006 /* tex_samples and rt_samples are the sample counts that are set up in
2007 * SURFACE_STATE.
2008 */
2009 wm_prog_key->tex_samples = params->src.surf.samples;
2010 wm_prog_key->rt_samples = params->dst.surf.samples;
2011
2012 /* tex_layout and rt_layout indicate the MSAA layout the GPU pipeline will
2013 * use to access the source and destination surfaces.
2014 */
2015 wm_prog_key->tex_layout = params->src.surf.msaa_layout;
2016 wm_prog_key->rt_layout = params->dst.surf.msaa_layout;
2017
2018 if (params->src.surf.samples > 0 && params->dst.surf.samples > 1) {
2019 /* We are blitting from a multisample buffer to a multisample buffer, so
2020 * we must preserve samples within a pixel. This means we have to
2021 * arrange for the WM program to run once per sample rather than once
2022 * per pixel.
2023 */
2024 wm_prog_key->persample_msaa_dispatch = true;
2025 }
2026
2027 params->num_samples = params->dst.surf.samples;
2028
2029 if ((wm_prog_key->filter == BLORP_FILTER_AVERAGE ||
2030 wm_prog_key->filter == BLORP_FILTER_BILINEAR) &&
2031 batch->blorp->isl_dev->info->gen <= 6) {
2032 /* Gen4-5 don't support non-normalized texture coordinates */
2033 wm_prog_key->src_coords_normalized = true;
2034 params->wm_inputs.src_inv_size[0] =
2035 1.0f / minify(params->src.surf.logical_level0_px.width,
2036 params->src.view.base_level);
2037 params->wm_inputs.src_inv_size[1] =
2038 1.0f / minify(params->src.surf.logical_level0_px.height,
2039 params->src.view.base_level);
2040 }
2041
2042 if (isl_format_get_layout(params->dst.view.format)->bpb % 3 == 0) {
2043 /* We can't render to RGB formats natively because they aren't a
2044 * power-of-two size. Instead, we fake them by using a red format
2045 * with the same channel type and size and emitting shader code to
2046 * only write one channel at a time.
2047 */
2048 params->x0 *= 3;
2049 params->x1 *= 3;
2050
2051 /* If it happens to be sRGB, we need to force a conversion */
2052 if (params->dst.view.format == ISL_FORMAT_R8G8B8_UNORM_SRGB)
2053 wm_prog_key->dst_format = ISL_FORMAT_R8G8B8_UNORM_SRGB;
2054
2055 surf_fake_rgb_with_red(batch->blorp->isl_dev, ¶ms->dst);
2056
2057 wm_prog_key->dst_rgb = true;
2058 wm_prog_key->need_dst_offset = true;
2059 } else if (isl_format_is_rgbx(params->dst.view.format)) {
2060 /* We can handle RGBX formats easily enough by treating them as RGBA */
2061 params->dst.view.format =
2062 isl_format_rgbx_to_rgba(params->dst.view.format);
2063 } else if (params->dst.view.format == ISL_FORMAT_R24_UNORM_X8_TYPELESS &&
2064 wm_prog_key->dst_usage != ISL_SURF_USAGE_DEPTH_BIT) {
2065 wm_prog_key->dst_format = params->dst.view.format;
2066 params->dst.view.format = ISL_FORMAT_R32_UINT;
2067 } else if (params->dst.view.format == ISL_FORMAT_A4B4G4R4_UNORM) {
2068 params->dst.view.swizzle =
2069 isl_swizzle_compose(params->dst.view.swizzle,
2070 ISL_SWIZZLE(ALPHA, RED, GREEN, BLUE));
2071 params->dst.view.format = ISL_FORMAT_B4G4R4A4_UNORM;
2072 } else if (params->dst.view.format == ISL_FORMAT_L8_UNORM_SRGB) {
2073 wm_prog_key->dst_format = params->dst.view.format;
2074 params->dst.view.format = ISL_FORMAT_R8_UNORM;
2075 } else if (params->dst.view.format == ISL_FORMAT_R9G9B9E5_SHAREDEXP) {
2076 wm_prog_key->dst_format = params->dst.view.format;
2077 params->dst.view.format = ISL_FORMAT_R32_UINT;
2078 }
2079
2080 if (devinfo->gen <= 7 && !devinfo->is_haswell &&
2081 !isl_swizzle_is_identity(params->src.view.swizzle)) {
2082 wm_prog_key->src_swizzle = params->src.view.swizzle;
2083 params->src.view.swizzle = ISL_SWIZZLE_IDENTITY;
2084 } else {
2085 wm_prog_key->src_swizzle = ISL_SWIZZLE_IDENTITY;
2086 }
2087
2088 if (!isl_swizzle_supports_rendering(devinfo, params->dst.view.swizzle)) {
2089 wm_prog_key->dst_swizzle = params->dst.view.swizzle;
2090 params->dst.view.swizzle = ISL_SWIZZLE_IDENTITY;
2091 } else {
2092 wm_prog_key->dst_swizzle = ISL_SWIZZLE_IDENTITY;
2093 }
2094
2095 if (params->src.tile_x_sa || params->src.tile_y_sa) {
2096 assert(wm_prog_key->need_src_offset);
2097 surf_get_intratile_offset_px(¶ms->src,
2098 ¶ms->wm_inputs.src_offset.x,
2099 ¶ms->wm_inputs.src_offset.y);
2100 }
2101
2102 if (params->dst.tile_x_sa || params->dst.tile_y_sa) {
2103 assert(wm_prog_key->need_dst_offset);
2104 surf_get_intratile_offset_px(¶ms->dst,
2105 ¶ms->wm_inputs.dst_offset.x,
2106 ¶ms->wm_inputs.dst_offset.y);
2107 params->x0 += params->wm_inputs.dst_offset.x;
2108 params->y0 += params->wm_inputs.dst_offset.y;
2109 params->x1 += params->wm_inputs.dst_offset.x;
2110 params->y1 += params->wm_inputs.dst_offset.y;
2111 }
2112
2113 /* For some texture types, we need to pass the layer through the sampler. */
2114 params->wm_inputs.src_z = params->src.z_offset;
2115
2116 if (!brw_blorp_get_blit_kernel(batch, params, wm_prog_key))
2117 return 0;
2118
2119 if (!blorp_ensure_sf_program(batch, params))
2120 return 0;
2121
2122 unsigned result = 0;
2123 unsigned max_src_surface_size = get_max_surface_size(devinfo, ¶ms->src);
2124 if (params->src.surf.logical_level0_px.width > max_src_surface_size)
2125 result |= BLIT_SRC_WIDTH_SHRINK;
2126 if (params->src.surf.logical_level0_px.height > max_src_surface_size)
2127 result |= BLIT_SRC_HEIGHT_SHRINK;
2128
2129 unsigned max_dst_surface_size = get_max_surface_size(devinfo, ¶ms->dst);
2130 if (params->dst.surf.logical_level0_px.width > max_dst_surface_size)
2131 result |= BLIT_DST_WIDTH_SHRINK;
2132 if (params->dst.surf.logical_level0_px.height > max_dst_surface_size)
2133 result |= BLIT_DST_HEIGHT_SHRINK;
2134
2135 if (result == 0) {
2136 if (wm_prog_key->dst_usage == ISL_SURF_USAGE_DEPTH_BIT) {
2137 params->depth = params->dst;
2138 memset(¶ms->dst, 0, sizeof(params->dst));
2139 } else if (wm_prog_key->dst_usage == ISL_SURF_USAGE_STENCIL_BIT) {
2140 params->stencil = params->dst;
2141 params->stencil_mask = 0xff;
2142 memset(¶ms->dst, 0, sizeof(params->dst));
2143 }
2144
2145 batch->blorp->exec(batch, params);
2146 }
2147
2148 return result;
2149 }
2150
2151 /* Adjust split blit source coordinates for the current destination
2152 * coordinates.
2153 */
2154 static void
adjust_split_source_coords(const struct blt_axis * orig,struct blt_axis * split_coords,double scale)2155 adjust_split_source_coords(const struct blt_axis *orig,
2156 struct blt_axis *split_coords,
2157 double scale)
2158 {
2159 /* When scale is greater than 0, then we are growing from the start, so
2160 * src0 uses delta0, and src1 uses delta1. When scale is less than 0, the
2161 * source range shrinks from the end. In that case src0 is adjusted by
2162 * delta1, and src1 is adjusted by delta0.
2163 */
2164 double delta0 = scale * (split_coords->dst0 - orig->dst0);
2165 double delta1 = scale * (split_coords->dst1 - orig->dst1);
2166 split_coords->src0 = orig->src0 + (scale >= 0.0 ? delta0 : delta1);
2167 split_coords->src1 = orig->src1 + (scale >= 0.0 ? delta1 : delta0);
2168 }
2169
2170 static struct isl_extent2d
get_px_size_sa(const struct isl_surf * surf)2171 get_px_size_sa(const struct isl_surf *surf)
2172 {
2173 static const struct isl_extent2d one_to_one = { .w = 1, .h = 1 };
2174
2175 if (surf->msaa_layout != ISL_MSAA_LAYOUT_INTERLEAVED)
2176 return one_to_one;
2177 else
2178 return isl_get_interleaved_msaa_px_size_sa(surf->samples);
2179 }
2180
2181 static void
shrink_surface_params(const struct isl_device * dev,struct brw_blorp_surface_info * info,double * x0,double * x1,double * y0,double * y1)2182 shrink_surface_params(const struct isl_device *dev,
2183 struct brw_blorp_surface_info *info,
2184 double *x0, double *x1, double *y0, double *y1)
2185 {
2186 uint32_t byte_offset, x_offset_sa, y_offset_sa, size;
2187 struct isl_extent2d px_size_sa;
2188 int adjust;
2189
2190 blorp_surf_convert_to_single_slice(dev, info);
2191
2192 px_size_sa = get_px_size_sa(&info->surf);
2193
2194 /* Because this gets called after we lower compressed images, the tile
2195 * offsets may be non-zero and we need to incorporate them in our
2196 * calculations.
2197 */
2198 x_offset_sa = (uint32_t)*x0 * px_size_sa.w + info->tile_x_sa;
2199 y_offset_sa = (uint32_t)*y0 * px_size_sa.h + info->tile_y_sa;
2200 isl_tiling_get_intratile_offset_sa(info->surf.tiling,
2201 info->surf.format, info->surf.row_pitch_B,
2202 x_offset_sa, y_offset_sa,
2203 &byte_offset,
2204 &info->tile_x_sa, &info->tile_y_sa);
2205
2206 info->addr.offset += byte_offset;
2207
2208 adjust = (int)info->tile_x_sa / px_size_sa.w - (int)*x0;
2209 *x0 += adjust;
2210 *x1 += adjust;
2211 info->tile_x_sa = 0;
2212
2213 adjust = (int)info->tile_y_sa / px_size_sa.h - (int)*y0;
2214 *y0 += adjust;
2215 *y1 += adjust;
2216 info->tile_y_sa = 0;
2217
2218 size = MIN2((uint32_t)ceil(*x1), info->surf.logical_level0_px.width);
2219 info->surf.logical_level0_px.width = size;
2220 info->surf.phys_level0_sa.width = size * px_size_sa.w;
2221
2222 size = MIN2((uint32_t)ceil(*y1), info->surf.logical_level0_px.height);
2223 info->surf.logical_level0_px.height = size;
2224 info->surf.phys_level0_sa.height = size * px_size_sa.h;
2225 }
2226
2227 static void
do_blorp_blit(struct blorp_batch * batch,const struct blorp_params * orig_params,struct brw_blorp_blit_prog_key * wm_prog_key,const struct blt_coords * orig)2228 do_blorp_blit(struct blorp_batch *batch,
2229 const struct blorp_params *orig_params,
2230 struct brw_blorp_blit_prog_key *wm_prog_key,
2231 const struct blt_coords *orig)
2232 {
2233 struct blorp_params params;
2234 struct blt_coords blit_coords;
2235 struct blt_coords split_coords = *orig;
2236 double w = orig->x.dst1 - orig->x.dst0;
2237 double h = orig->y.dst1 - orig->y.dst0;
2238 double x_scale = (orig->x.src1 - orig->x.src0) / w;
2239 double y_scale = (orig->y.src1 - orig->y.src0) / h;
2240 if (orig->x.mirror)
2241 x_scale = -x_scale;
2242 if (orig->y.mirror)
2243 y_scale = -y_scale;
2244
2245 enum blit_shrink_status shrink = BLIT_NO_SHRINK;
2246 if (split_blorp_blit_debug) {
2247 if (can_shrink_surface(&orig_params->src))
2248 shrink |= BLIT_SRC_WIDTH_SHRINK | BLIT_SRC_HEIGHT_SHRINK;
2249 if (can_shrink_surface(&orig_params->dst))
2250 shrink |= BLIT_DST_WIDTH_SHRINK | BLIT_DST_HEIGHT_SHRINK;
2251 }
2252
2253 bool x_done, y_done;
2254 do {
2255 params = *orig_params;
2256 blit_coords = split_coords;
2257
2258 if (shrink & (BLIT_SRC_WIDTH_SHRINK | BLIT_SRC_HEIGHT_SHRINK)) {
2259 shrink_surface_params(batch->blorp->isl_dev, ¶ms.src,
2260 &blit_coords.x.src0, &blit_coords.x.src1,
2261 &blit_coords.y.src0, &blit_coords.y.src1);
2262 wm_prog_key->need_src_offset = false;
2263 }
2264
2265 if (shrink & (BLIT_DST_WIDTH_SHRINK | BLIT_DST_HEIGHT_SHRINK)) {
2266 shrink_surface_params(batch->blorp->isl_dev, ¶ms.dst,
2267 &blit_coords.x.dst0, &blit_coords.x.dst1,
2268 &blit_coords.y.dst0, &blit_coords.y.dst1);
2269 wm_prog_key->need_dst_offset = false;
2270 }
2271
2272 enum blit_shrink_status result =
2273 try_blorp_blit(batch, ¶ms, wm_prog_key, &blit_coords);
2274
2275 if (result & (BLIT_SRC_WIDTH_SHRINK | BLIT_SRC_HEIGHT_SHRINK))
2276 assert(can_shrink_surface(&orig_params->src));
2277
2278 if (result & (BLIT_DST_WIDTH_SHRINK | BLIT_DST_HEIGHT_SHRINK))
2279 assert(can_shrink_surface(&orig_params->dst));
2280
2281 if (result & (BLIT_SRC_WIDTH_SHRINK | BLIT_DST_WIDTH_SHRINK)) {
2282 w /= 2.0;
2283 assert(w >= 1.0);
2284 split_coords.x.dst1 = MIN2(split_coords.x.dst0 + w, orig->x.dst1);
2285 adjust_split_source_coords(&orig->x, &split_coords.x, x_scale);
2286 }
2287 if (result & (BLIT_SRC_HEIGHT_SHRINK | BLIT_DST_HEIGHT_SHRINK)) {
2288 h /= 2.0;
2289 assert(h >= 1.0);
2290 split_coords.y.dst1 = MIN2(split_coords.y.dst0 + h, orig->y.dst1);
2291 adjust_split_source_coords(&orig->y, &split_coords.y, y_scale);
2292 }
2293
2294 if (result) {
2295 /* We may get less bits set on result than we had already, so make
2296 * sure we remember all the ways in which a resize is required.
2297 */
2298 shrink |= result;
2299 continue;
2300 }
2301
2302 y_done = (orig->y.dst1 - split_coords.y.dst1 < 0.5);
2303 x_done = y_done && (orig->x.dst1 - split_coords.x.dst1 < 0.5);
2304 if (x_done) {
2305 break;
2306 } else if (y_done) {
2307 split_coords.x.dst0 += w;
2308 split_coords.x.dst1 = MIN2(split_coords.x.dst0 + w, orig->x.dst1);
2309 split_coords.y.dst0 = orig->y.dst0;
2310 split_coords.y.dst1 = MIN2(split_coords.y.dst0 + h, orig->y.dst1);
2311 adjust_split_source_coords(&orig->x, &split_coords.x, x_scale);
2312 } else {
2313 split_coords.y.dst0 += h;
2314 split_coords.y.dst1 = MIN2(split_coords.y.dst0 + h, orig->y.dst1);
2315 adjust_split_source_coords(&orig->y, &split_coords.y, y_scale);
2316 }
2317 } while (true);
2318 }
2319
2320 void
blorp_blit(struct blorp_batch * batch,const struct blorp_surf * src_surf,unsigned src_level,float src_layer,enum isl_format src_format,struct isl_swizzle src_swizzle,const struct blorp_surf * dst_surf,unsigned dst_level,unsigned dst_layer,enum isl_format dst_format,struct isl_swizzle dst_swizzle,float src_x0,float src_y0,float src_x1,float src_y1,float dst_x0,float dst_y0,float dst_x1,float dst_y1,enum blorp_filter filter,bool mirror_x,bool mirror_y)2321 blorp_blit(struct blorp_batch *batch,
2322 const struct blorp_surf *src_surf,
2323 unsigned src_level, float src_layer,
2324 enum isl_format src_format, struct isl_swizzle src_swizzle,
2325 const struct blorp_surf *dst_surf,
2326 unsigned dst_level, unsigned dst_layer,
2327 enum isl_format dst_format, struct isl_swizzle dst_swizzle,
2328 float src_x0, float src_y0,
2329 float src_x1, float src_y1,
2330 float dst_x0, float dst_y0,
2331 float dst_x1, float dst_y1,
2332 enum blorp_filter filter,
2333 bool mirror_x, bool mirror_y)
2334 {
2335 struct blorp_params params;
2336 blorp_params_init(¶ms);
2337
2338 /* We cannot handle combined depth and stencil. */
2339 if (src_surf->surf->usage & ISL_SURF_USAGE_STENCIL_BIT)
2340 assert(src_surf->surf->format == ISL_FORMAT_R8_UINT);
2341 if (dst_surf->surf->usage & ISL_SURF_USAGE_STENCIL_BIT)
2342 assert(dst_surf->surf->format == ISL_FORMAT_R8_UINT);
2343
2344 if (dst_surf->surf->usage & ISL_SURF_USAGE_STENCIL_BIT) {
2345 assert(src_surf->surf->usage & ISL_SURF_USAGE_STENCIL_BIT);
2346 /* Prior to Broadwell, we can't render to R8_UINT */
2347 if (batch->blorp->isl_dev->info->gen < 8) {
2348 src_format = ISL_FORMAT_R8_UNORM;
2349 dst_format = ISL_FORMAT_R8_UNORM;
2350 }
2351 }
2352
2353 brw_blorp_surface_info_init(batch->blorp, ¶ms.src, src_surf, src_level,
2354 src_layer, src_format, false);
2355 brw_blorp_surface_info_init(batch->blorp, ¶ms.dst, dst_surf, dst_level,
2356 dst_layer, dst_format, true);
2357
2358 params.src.view.swizzle = src_swizzle;
2359 params.dst.view.swizzle = dst_swizzle;
2360
2361 const struct isl_format_layout *src_fmtl =
2362 isl_format_get_layout(params.src.view.format);
2363
2364 struct brw_blorp_blit_prog_key wm_prog_key = {
2365 .shader_type = BLORP_SHADER_TYPE_BLIT,
2366 .filter = filter,
2367 .sint32_to_uint = src_fmtl->channels.r.bits == 32 &&
2368 isl_format_has_sint_channel(params.src.view.format) &&
2369 isl_format_has_uint_channel(params.dst.view.format),
2370 .uint32_to_sint = src_fmtl->channels.r.bits == 32 &&
2371 isl_format_has_uint_channel(params.src.view.format) &&
2372 isl_format_has_sint_channel(params.dst.view.format),
2373 };
2374
2375 /* Scaling factors used for bilinear filtering in multisample scaled
2376 * blits.
2377 */
2378 if (params.src.surf.samples == 16)
2379 wm_prog_key.x_scale = 4.0f;
2380 else
2381 wm_prog_key.x_scale = 2.0f;
2382 wm_prog_key.y_scale = params.src.surf.samples / wm_prog_key.x_scale;
2383
2384 params.wm_inputs.rect_grid.x1 =
2385 minify(params.src.surf.logical_level0_px.width, src_level) *
2386 wm_prog_key.x_scale - 1.0f;
2387 params.wm_inputs.rect_grid.y1 =
2388 minify(params.src.surf.logical_level0_px.height, src_level) *
2389 wm_prog_key.y_scale - 1.0f;
2390
2391 struct blt_coords coords = {
2392 .x = {
2393 .src0 = src_x0,
2394 .src1 = src_x1,
2395 .dst0 = dst_x0,
2396 .dst1 = dst_x1,
2397 .mirror = mirror_x
2398 },
2399 .y = {
2400 .src0 = src_y0,
2401 .src1 = src_y1,
2402 .dst0 = dst_y0,
2403 .dst1 = dst_y1,
2404 .mirror = mirror_y
2405 }
2406 };
2407
2408 do_blorp_blit(batch, ¶ms, &wm_prog_key, &coords);
2409 }
2410
2411 static enum isl_format
get_copy_format_for_bpb(const struct isl_device * isl_dev,unsigned bpb)2412 get_copy_format_for_bpb(const struct isl_device *isl_dev, unsigned bpb)
2413 {
2414 /* The choice of UNORM and UINT formats is very intentional here. Most
2415 * of the time, we want to use a UINT format to avoid any rounding error
2416 * in the blit. For stencil blits, R8_UINT is required by the hardware.
2417 * (It's the only format allowed in conjunction with W-tiling.) Also we
2418 * intentionally use the 4-channel formats whenever we can. This is so
2419 * that, when we do a RGB <-> RGBX copy, the two formats will line up
2420 * even though one of them is 3/4 the size of the other. The choice of
2421 * UNORM vs. UINT is also very intentional because we don't have 8 or
2422 * 16-bit RGB UINT formats until Sky Lake so we have to use UNORM there.
2423 * Fortunately, the only time we should ever use two different formats in
2424 * the table below is for RGB -> RGBA blits and so we will never have any
2425 * UNORM/UINT mismatch.
2426 */
2427 if (ISL_DEV_GEN(isl_dev) >= 9) {
2428 switch (bpb) {
2429 case 8: return ISL_FORMAT_R8_UINT;
2430 case 16: return ISL_FORMAT_R8G8_UINT;
2431 case 24: return ISL_FORMAT_R8G8B8_UINT;
2432 case 32: return ISL_FORMAT_R8G8B8A8_UINT;
2433 case 48: return ISL_FORMAT_R16G16B16_UINT;
2434 case 64: return ISL_FORMAT_R16G16B16A16_UINT;
2435 case 96: return ISL_FORMAT_R32G32B32_UINT;
2436 case 128:return ISL_FORMAT_R32G32B32A32_UINT;
2437 default:
2438 unreachable("Unknown format bpb");
2439 }
2440 } else {
2441 switch (bpb) {
2442 case 8: return ISL_FORMAT_R8_UINT;
2443 case 16: return ISL_FORMAT_R8G8_UINT;
2444 case 24: return ISL_FORMAT_R8G8B8_UNORM;
2445 case 32: return ISL_FORMAT_R8G8B8A8_UNORM;
2446 case 48: return ISL_FORMAT_R16G16B16_UNORM;
2447 case 64: return ISL_FORMAT_R16G16B16A16_UNORM;
2448 case 96: return ISL_FORMAT_R32G32B32_UINT;
2449 case 128:return ISL_FORMAT_R32G32B32A32_UINT;
2450 default:
2451 unreachable("Unknown format bpb");
2452 }
2453 }
2454 }
2455
2456 /** Returns a UINT format that is CCS-compatible with the given format
2457 *
2458 * The PRM's say absolutely nothing about how render compression works. The
2459 * only thing they provide is a list of formats on which it is and is not
2460 * supported. Empirical testing indicates that the compression is only based
2461 * on the bit-layout of the format and the channel encoding doesn't matter.
2462 * So, while texture views don't work in general, you can create a view as
2463 * long as the bit-layout of the formats are the same.
2464 *
2465 * Fortunately, for every render compression capable format, the UINT format
2466 * with the same bit layout also supports render compression. This means that
2467 * we only need to handle UINT formats for copy operations. In order to do
2468 * copies between formats with different bit layouts, we attach both with a
2469 * UINT format and use bit_cast_color() to generate code to do the bit-cast
2470 * operation between the two bit layouts.
2471 */
2472 static enum isl_format
get_ccs_compatible_copy_format(const struct isl_format_layout * fmtl)2473 get_ccs_compatible_copy_format(const struct isl_format_layout *fmtl)
2474 {
2475 switch (fmtl->format) {
2476 case ISL_FORMAT_R32G32B32A32_FLOAT:
2477 case ISL_FORMAT_R32G32B32A32_SINT:
2478 case ISL_FORMAT_R32G32B32A32_UINT:
2479 case ISL_FORMAT_R32G32B32A32_UNORM:
2480 case ISL_FORMAT_R32G32B32A32_SNORM:
2481 case ISL_FORMAT_R32G32B32X32_FLOAT:
2482 return ISL_FORMAT_R32G32B32A32_UINT;
2483
2484 case ISL_FORMAT_R16G16B16A16_UNORM:
2485 case ISL_FORMAT_R16G16B16A16_SNORM:
2486 case ISL_FORMAT_R16G16B16A16_SINT:
2487 case ISL_FORMAT_R16G16B16A16_UINT:
2488 case ISL_FORMAT_R16G16B16A16_FLOAT:
2489 case ISL_FORMAT_R16G16B16X16_UNORM:
2490 case ISL_FORMAT_R16G16B16X16_FLOAT:
2491 return ISL_FORMAT_R16G16B16A16_UINT;
2492
2493 case ISL_FORMAT_R32G32_FLOAT:
2494 case ISL_FORMAT_R32G32_SINT:
2495 case ISL_FORMAT_R32G32_UINT:
2496 case ISL_FORMAT_R32G32_UNORM:
2497 case ISL_FORMAT_R32G32_SNORM:
2498 return ISL_FORMAT_R32G32_UINT;
2499
2500 case ISL_FORMAT_B8G8R8A8_UNORM:
2501 case ISL_FORMAT_B8G8R8A8_UNORM_SRGB:
2502 case ISL_FORMAT_R8G8B8A8_UNORM:
2503 case ISL_FORMAT_R8G8B8A8_UNORM_SRGB:
2504 case ISL_FORMAT_R8G8B8A8_SNORM:
2505 case ISL_FORMAT_R8G8B8A8_SINT:
2506 case ISL_FORMAT_R8G8B8A8_UINT:
2507 case ISL_FORMAT_B8G8R8X8_UNORM:
2508 case ISL_FORMAT_B8G8R8X8_UNORM_SRGB:
2509 case ISL_FORMAT_R8G8B8X8_UNORM:
2510 case ISL_FORMAT_R8G8B8X8_UNORM_SRGB:
2511 return ISL_FORMAT_R8G8B8A8_UINT;
2512
2513 case ISL_FORMAT_R16G16_UNORM:
2514 case ISL_FORMAT_R16G16_SNORM:
2515 case ISL_FORMAT_R16G16_SINT:
2516 case ISL_FORMAT_R16G16_UINT:
2517 case ISL_FORMAT_R16G16_FLOAT:
2518 return ISL_FORMAT_R16G16_UINT;
2519
2520 case ISL_FORMAT_R32_SINT:
2521 case ISL_FORMAT_R32_UINT:
2522 case ISL_FORMAT_R32_FLOAT:
2523 case ISL_FORMAT_R32_UNORM:
2524 case ISL_FORMAT_R32_SNORM:
2525 return ISL_FORMAT_R32_UINT;
2526
2527 case ISL_FORMAT_B10G10R10A2_UNORM:
2528 case ISL_FORMAT_B10G10R10A2_UNORM_SRGB:
2529 case ISL_FORMAT_R10G10B10A2_UNORM:
2530 case ISL_FORMAT_R10G10B10A2_UNORM_SRGB:
2531 case ISL_FORMAT_R10G10B10_FLOAT_A2_UNORM:
2532 case ISL_FORMAT_R10G10B10A2_UINT:
2533 return ISL_FORMAT_R10G10B10A2_UINT;
2534
2535 case ISL_FORMAT_R16_UNORM:
2536 case ISL_FORMAT_R16_SNORM:
2537 case ISL_FORMAT_R16_SINT:
2538 case ISL_FORMAT_R16_UINT:
2539 case ISL_FORMAT_R16_FLOAT:
2540 return ISL_FORMAT_R16_UINT;
2541
2542 case ISL_FORMAT_R8G8_UNORM:
2543 case ISL_FORMAT_R8G8_SNORM:
2544 case ISL_FORMAT_R8G8_SINT:
2545 case ISL_FORMAT_R8G8_UINT:
2546 return ISL_FORMAT_R8G8_UINT;
2547
2548 case ISL_FORMAT_B5G5R5X1_UNORM:
2549 case ISL_FORMAT_B5G5R5X1_UNORM_SRGB:
2550 case ISL_FORMAT_B5G5R5A1_UNORM:
2551 case ISL_FORMAT_B5G5R5A1_UNORM_SRGB:
2552 return ISL_FORMAT_B5G5R5A1_UNORM;
2553
2554 case ISL_FORMAT_A4B4G4R4_UNORM:
2555 case ISL_FORMAT_B4G4R4A4_UNORM:
2556 case ISL_FORMAT_B4G4R4A4_UNORM_SRGB:
2557 return ISL_FORMAT_B4G4R4A4_UNORM;
2558
2559 case ISL_FORMAT_B5G6R5_UNORM:
2560 case ISL_FORMAT_B5G6R5_UNORM_SRGB:
2561 return ISL_FORMAT_B5G6R5_UNORM;
2562
2563 case ISL_FORMAT_A1B5G5R5_UNORM:
2564 return ISL_FORMAT_A1B5G5R5_UNORM;
2565
2566 case ISL_FORMAT_A8_UNORM:
2567 case ISL_FORMAT_R8_UNORM:
2568 case ISL_FORMAT_R8_SNORM:
2569 case ISL_FORMAT_R8_SINT:
2570 case ISL_FORMAT_R8_UINT:
2571 return ISL_FORMAT_R8_UINT;
2572
2573 default:
2574 unreachable("Not a compressible format");
2575 }
2576 }
2577
2578 void
blorp_surf_convert_to_uncompressed(const struct isl_device * isl_dev,struct brw_blorp_surface_info * info,uint32_t * x,uint32_t * y,uint32_t * width,uint32_t * height)2579 blorp_surf_convert_to_uncompressed(const struct isl_device *isl_dev,
2580 struct brw_blorp_surface_info *info,
2581 uint32_t *x, uint32_t *y,
2582 uint32_t *width, uint32_t *height)
2583 {
2584 const struct isl_format_layout *fmtl =
2585 isl_format_get_layout(info->surf.format);
2586
2587 assert(fmtl->bw > 1 || fmtl->bh > 1);
2588
2589 /* This is a compressed surface. We need to convert it to a single
2590 * slice (because compressed layouts don't perfectly match uncompressed
2591 * ones with the same bpb) and divide x, y, width, and height by the
2592 * block size.
2593 */
2594 blorp_surf_convert_to_single_slice(isl_dev, info);
2595
2596 if (width && height) {
2597 #ifndef NDEBUG
2598 uint32_t right_edge_px = info->tile_x_sa + *x + *width;
2599 uint32_t bottom_edge_px = info->tile_y_sa + *y + *height;
2600 assert(*width % fmtl->bw == 0 ||
2601 right_edge_px == info->surf.logical_level0_px.width);
2602 assert(*height % fmtl->bh == 0 ||
2603 bottom_edge_px == info->surf.logical_level0_px.height);
2604 #endif
2605 *width = DIV_ROUND_UP(*width, fmtl->bw);
2606 *height = DIV_ROUND_UP(*height, fmtl->bh);
2607 }
2608
2609 if (x && y) {
2610 assert(*x % fmtl->bw == 0);
2611 assert(*y % fmtl->bh == 0);
2612 *x /= fmtl->bw;
2613 *y /= fmtl->bh;
2614 }
2615
2616 info->surf.logical_level0_px = isl_surf_get_logical_level0_el(&info->surf);
2617 info->surf.phys_level0_sa = isl_surf_get_phys_level0_el(&info->surf);
2618
2619 assert(info->tile_x_sa % fmtl->bw == 0);
2620 assert(info->tile_y_sa % fmtl->bh == 0);
2621 info->tile_x_sa /= fmtl->bw;
2622 info->tile_y_sa /= fmtl->bh;
2623
2624 /* It's now an uncompressed surface so we need an uncompressed format */
2625 info->surf.format = get_copy_format_for_bpb(isl_dev, fmtl->bpb);
2626 }
2627
2628 void
blorp_copy(struct blorp_batch * batch,const struct blorp_surf * src_surf,unsigned src_level,unsigned src_layer,const struct blorp_surf * dst_surf,unsigned dst_level,unsigned dst_layer,uint32_t src_x,uint32_t src_y,uint32_t dst_x,uint32_t dst_y,uint32_t src_width,uint32_t src_height)2629 blorp_copy(struct blorp_batch *batch,
2630 const struct blorp_surf *src_surf,
2631 unsigned src_level, unsigned src_layer,
2632 const struct blorp_surf *dst_surf,
2633 unsigned dst_level, unsigned dst_layer,
2634 uint32_t src_x, uint32_t src_y,
2635 uint32_t dst_x, uint32_t dst_y,
2636 uint32_t src_width, uint32_t src_height)
2637 {
2638 const struct isl_device *isl_dev = batch->blorp->isl_dev;
2639 struct blorp_params params;
2640
2641 if (src_width == 0 || src_height == 0)
2642 return;
2643
2644 blorp_params_init(¶ms);
2645 brw_blorp_surface_info_init(batch->blorp, ¶ms.src, src_surf, src_level,
2646 src_layer, ISL_FORMAT_UNSUPPORTED, false);
2647 brw_blorp_surface_info_init(batch->blorp, ¶ms.dst, dst_surf, dst_level,
2648 dst_layer, ISL_FORMAT_UNSUPPORTED, true);
2649
2650 struct brw_blorp_blit_prog_key wm_prog_key = {
2651 .shader_type = BLORP_SHADER_TYPE_COPY,
2652 .filter = BLORP_FILTER_NONE,
2653 .need_src_offset = src_surf->tile_x_sa || src_surf->tile_y_sa,
2654 .need_dst_offset = dst_surf->tile_x_sa || dst_surf->tile_y_sa,
2655 };
2656
2657 const struct isl_format_layout *src_fmtl =
2658 isl_format_get_layout(params.src.surf.format);
2659 const struct isl_format_layout *dst_fmtl =
2660 isl_format_get_layout(params.dst.surf.format);
2661
2662 assert(params.src.aux_usage == ISL_AUX_USAGE_NONE ||
2663 params.src.aux_usage == ISL_AUX_USAGE_HIZ ||
2664 params.src.aux_usage == ISL_AUX_USAGE_HIZ_CCS_WT ||
2665 params.src.aux_usage == ISL_AUX_USAGE_MCS ||
2666 params.src.aux_usage == ISL_AUX_USAGE_MCS_CCS ||
2667 params.src.aux_usage == ISL_AUX_USAGE_CCS_E ||
2668 params.src.aux_usage == ISL_AUX_USAGE_GEN12_CCS_E ||
2669 params.src.aux_usage == ISL_AUX_USAGE_STC_CCS);
2670
2671 if (isl_aux_usage_has_hiz(params.src.aux_usage)) {
2672 /* In order to use HiZ, we have to use the real format for the source.
2673 * Depth <-> Color copies are not allowed.
2674 */
2675 params.src.view.format = params.src.surf.format;
2676 params.dst.view.format = params.src.surf.format;
2677 } else if ((params.dst.surf.usage & ISL_SURF_USAGE_DEPTH_BIT) &&
2678 isl_dev->info->gen >= 7) {
2679 /* On Gen7 and higher, we use actual depth writes for blits into depth
2680 * buffers so we need the real format.
2681 */
2682 params.src.view.format = params.dst.surf.format;
2683 params.dst.view.format = params.dst.surf.format;
2684 } else if (params.dst.aux_usage == ISL_AUX_USAGE_CCS_E ||
2685 params.dst.aux_usage == ISL_AUX_USAGE_GEN12_CCS_E) {
2686 params.dst.view.format = get_ccs_compatible_copy_format(dst_fmtl);
2687 if (params.src.aux_usage == ISL_AUX_USAGE_CCS_E ||
2688 params.src.aux_usage == ISL_AUX_USAGE_GEN12_CCS_E) {
2689 params.src.view.format = get_ccs_compatible_copy_format(src_fmtl);
2690 } else if (src_fmtl->bpb == dst_fmtl->bpb) {
2691 params.src.view.format = params.dst.view.format;
2692 } else {
2693 params.src.view.format =
2694 get_copy_format_for_bpb(isl_dev, src_fmtl->bpb);
2695 }
2696 } else if (params.src.aux_usage == ISL_AUX_USAGE_CCS_E ||
2697 params.src.aux_usage == ISL_AUX_USAGE_GEN12_CCS_E) {
2698 params.src.view.format = get_ccs_compatible_copy_format(src_fmtl);
2699 if (src_fmtl->bpb == dst_fmtl->bpb) {
2700 params.dst.view.format = params.src.view.format;
2701 } else {
2702 params.dst.view.format =
2703 get_copy_format_for_bpb(isl_dev, dst_fmtl->bpb);
2704 }
2705 } else {
2706 params.dst.view.format = get_copy_format_for_bpb(isl_dev, dst_fmtl->bpb);
2707 params.src.view.format = get_copy_format_for_bpb(isl_dev, src_fmtl->bpb);
2708 }
2709
2710 if (params.src.aux_usage == ISL_AUX_USAGE_CCS_E) {
2711 /* It's safe to do a blorp_copy between things which are sRGB with CCS_E
2712 * enabled even though CCS_E doesn't technically do sRGB on SKL because
2713 * we stomp everything to UINT anyway. The one thing we have to be
2714 * careful of is clear colors. Because fast clear colors for sRGB on
2715 * gen9 are encoded as the float values between format conversion and
2716 * sRGB curve application, a given clear color float will convert to the
2717 * same bits regardless of whether the format is UNORM or sRGB.
2718 * Therefore, we can handle sRGB without any special cases.
2719 */
2720 UNUSED enum isl_format linear_src_format =
2721 isl_format_srgb_to_linear(src_surf->surf->format);
2722 assert(isl_formats_are_ccs_e_compatible(batch->blorp->isl_dev->info,
2723 linear_src_format,
2724 params.src.view.format));
2725 uint32_t packed[4];
2726 isl_color_value_pack(¶ms.src.clear_color,
2727 linear_src_format, packed);
2728 isl_color_value_unpack(¶ms.src.clear_color,
2729 params.src.view.format, packed);
2730 }
2731
2732 if (params.dst.aux_usage == ISL_AUX_USAGE_CCS_E) {
2733 /* See above where we handle linear_src_format */
2734 UNUSED enum isl_format linear_dst_format =
2735 isl_format_srgb_to_linear(dst_surf->surf->format);
2736 assert(isl_formats_are_ccs_e_compatible(batch->blorp->isl_dev->info,
2737 linear_dst_format,
2738 params.dst.view.format));
2739 uint32_t packed[4];
2740 isl_color_value_pack(¶ms.dst.clear_color,
2741 linear_dst_format, packed);
2742 isl_color_value_unpack(¶ms.dst.clear_color,
2743 params.dst.view.format, packed);
2744 }
2745
2746 if (params.src.view.format != params.dst.view.format) {
2747 enum isl_format src_cast_format = params.src.view.format;
2748 enum isl_format dst_cast_format = params.dst.view.format;
2749
2750 /* The BLORP bitcast code gets confused by RGB formats. Just treat them
2751 * as RGBA and then everything will be happy. This is perfectly safe
2752 * because BLORP likes to treat things as if they have vec4 colors all
2753 * the time anyway.
2754 */
2755 if (isl_format_get_layout(src_cast_format)->bpb % 3 == 0)
2756 src_cast_format = isl_format_rgb_to_rgba(src_cast_format);
2757 if (isl_format_get_layout(dst_cast_format)->bpb % 3 == 0)
2758 dst_cast_format = isl_format_rgb_to_rgba(dst_cast_format);
2759
2760 if (src_cast_format != dst_cast_format) {
2761 wm_prog_key.format_bit_cast = true;
2762 wm_prog_key.src_format = src_cast_format;
2763 wm_prog_key.dst_format = dst_cast_format;
2764 }
2765 }
2766
2767 if (src_fmtl->bw > 1 || src_fmtl->bh > 1) {
2768 blorp_surf_convert_to_uncompressed(batch->blorp->isl_dev, ¶ms.src,
2769 &src_x, &src_y,
2770 &src_width, &src_height);
2771 wm_prog_key.need_src_offset = true;
2772 }
2773
2774 if (dst_fmtl->bw > 1 || dst_fmtl->bh > 1) {
2775 blorp_surf_convert_to_uncompressed(batch->blorp->isl_dev, ¶ms.dst,
2776 &dst_x, &dst_y, NULL, NULL);
2777 wm_prog_key.need_dst_offset = true;
2778 }
2779
2780 /* Once both surfaces are stompped to uncompressed as needed, the
2781 * destination size is the same as the source size.
2782 */
2783 uint32_t dst_width = src_width;
2784 uint32_t dst_height = src_height;
2785
2786 struct blt_coords coords = {
2787 .x = {
2788 .src0 = src_x,
2789 .src1 = src_x + src_width,
2790 .dst0 = dst_x,
2791 .dst1 = dst_x + dst_width,
2792 .mirror = false
2793 },
2794 .y = {
2795 .src0 = src_y,
2796 .src1 = src_y + src_height,
2797 .dst0 = dst_y,
2798 .dst1 = dst_y + dst_height,
2799 .mirror = false
2800 }
2801 };
2802
2803 do_blorp_blit(batch, ¶ms, &wm_prog_key, &coords);
2804 }
2805
2806 static enum isl_format
isl_format_for_size(unsigned size_B)2807 isl_format_for_size(unsigned size_B)
2808 {
2809 switch (size_B) {
2810 case 1: return ISL_FORMAT_R8_UINT;
2811 case 2: return ISL_FORMAT_R8G8_UINT;
2812 case 4: return ISL_FORMAT_R8G8B8A8_UINT;
2813 case 8: return ISL_FORMAT_R16G16B16A16_UINT;
2814 case 16: return ISL_FORMAT_R32G32B32A32_UINT;
2815 default:
2816 unreachable("Not a power-of-two format size");
2817 }
2818 }
2819
2820 /**
2821 * Returns the greatest common divisor of a and b that is a power of two.
2822 */
2823 static uint64_t
gcd_pow2_u64(uint64_t a,uint64_t b)2824 gcd_pow2_u64(uint64_t a, uint64_t b)
2825 {
2826 assert(a > 0 || b > 0);
2827
2828 unsigned a_log2 = ffsll(a) - 1;
2829 unsigned b_log2 = ffsll(b) - 1;
2830
2831 /* If either a or b is 0, then a_log2 or b_log2 till be UINT_MAX in which
2832 * case, the MIN2() will take the other one. If both are 0 then we will
2833 * hit the assert above.
2834 */
2835 return 1 << MIN2(a_log2, b_log2);
2836 }
2837
2838 static void
do_buffer_copy(struct blorp_batch * batch,struct blorp_address * src,struct blorp_address * dst,int width,int height,int block_size)2839 do_buffer_copy(struct blorp_batch *batch,
2840 struct blorp_address *src,
2841 struct blorp_address *dst,
2842 int width, int height, int block_size)
2843 {
2844 /* The actual format we pick doesn't matter as blorp will throw it away.
2845 * The only thing that actually matters is the size.
2846 */
2847 enum isl_format format = isl_format_for_size(block_size);
2848
2849 UNUSED bool ok;
2850 struct isl_surf surf;
2851 ok = isl_surf_init(batch->blorp->isl_dev, &surf,
2852 .dim = ISL_SURF_DIM_2D,
2853 .format = format,
2854 .width = width,
2855 .height = height,
2856 .depth = 1,
2857 .levels = 1,
2858 .array_len = 1,
2859 .samples = 1,
2860 .row_pitch_B = width * block_size,
2861 .usage = ISL_SURF_USAGE_TEXTURE_BIT |
2862 ISL_SURF_USAGE_RENDER_TARGET_BIT,
2863 .tiling_flags = ISL_TILING_LINEAR_BIT);
2864 assert(ok);
2865
2866 struct blorp_surf src_blorp_surf = {
2867 .surf = &surf,
2868 .addr = *src,
2869 };
2870
2871 struct blorp_surf dst_blorp_surf = {
2872 .surf = &surf,
2873 .addr = *dst,
2874 };
2875
2876 blorp_copy(batch, &src_blorp_surf, 0, 0, &dst_blorp_surf, 0, 0,
2877 0, 0, 0, 0, width, height);
2878 }
2879
2880 void
blorp_buffer_copy(struct blorp_batch * batch,struct blorp_address src,struct blorp_address dst,uint64_t size)2881 blorp_buffer_copy(struct blorp_batch *batch,
2882 struct blorp_address src,
2883 struct blorp_address dst,
2884 uint64_t size)
2885 {
2886 const struct gen_device_info *devinfo = batch->blorp->isl_dev->info;
2887 uint64_t copy_size = size;
2888
2889 /* This is maximum possible width/height our HW can handle */
2890 uint64_t max_surface_dim = 1 << (devinfo->gen >= 7 ? 14 : 13);
2891
2892 /* First, we compute the biggest format that can be used with the
2893 * given offsets and size.
2894 */
2895 int bs = 16;
2896 bs = gcd_pow2_u64(bs, src.offset);
2897 bs = gcd_pow2_u64(bs, dst.offset);
2898 bs = gcd_pow2_u64(bs, size);
2899
2900 /* First, we make a bunch of max-sized copies */
2901 uint64_t max_copy_size = max_surface_dim * max_surface_dim * bs;
2902 while (copy_size >= max_copy_size) {
2903 do_buffer_copy(batch, &src, &dst, max_surface_dim, max_surface_dim, bs);
2904 copy_size -= max_copy_size;
2905 src.offset += max_copy_size;
2906 dst.offset += max_copy_size;
2907 }
2908
2909 /* Now make a max-width copy */
2910 uint64_t height = copy_size / (max_surface_dim * bs);
2911 assert(height < max_surface_dim);
2912 if (height != 0) {
2913 uint64_t rect_copy_size = height * max_surface_dim * bs;
2914 do_buffer_copy(batch, &src, &dst, max_surface_dim, height, bs);
2915 copy_size -= rect_copy_size;
2916 src.offset += rect_copy_size;
2917 dst.offset += rect_copy_size;
2918 }
2919
2920 /* Finally, make a small copy to finish it off */
2921 if (copy_size != 0) {
2922 do_buffer_copy(batch, &src, &dst, copy_size / bs, 1, bs);
2923 }
2924 }
2925