1 /*
2 * Copyright © 2012 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "blorp_nir_builder.h"
25 #include "compiler/nir/nir_format_convert.h"
26
27 #include "blorp_priv.h"
28 #include "dev/intel_debug.h"
29 #include "dev/intel_device_info.h"
30
31 #include "util/format_rgb9e5.h"
32 #include "util/u_math.h"
33
34 #define FILE_DEBUG_FLAG DEBUG_BLORP
35
36 static const bool split_blorp_blit_debug = false;
37
38 struct blorp_blit_vars {
39 /* Input values from blorp_wm_inputs */
40 nir_variable *v_bounds_rect;
41 nir_variable *v_rect_grid;
42 nir_variable *v_coord_transform;
43 nir_variable *v_src_z;
44 nir_variable *v_src_offset;
45 nir_variable *v_dst_offset;
46 nir_variable *v_src_inv_size;
47 };
48
49 static void
blorp_blit_vars_init(nir_builder * b,struct blorp_blit_vars * v,const struct blorp_blit_prog_key * key)50 blorp_blit_vars_init(nir_builder *b, struct blorp_blit_vars *v,
51 const struct blorp_blit_prog_key *key)
52 {
53 #define LOAD_INPUT(name, type)\
54 v->v_##name = BLORP_CREATE_NIR_INPUT(b->shader, name, type);
55
56 LOAD_INPUT(bounds_rect, glsl_vec4_type())
57 LOAD_INPUT(rect_grid, glsl_vec4_type())
58 LOAD_INPUT(coord_transform, glsl_vec4_type())
59 LOAD_INPUT(src_z, glsl_float_type())
60 LOAD_INPUT(src_offset, glsl_vector_type(GLSL_TYPE_UINT, 2))
61 LOAD_INPUT(dst_offset, glsl_vector_type(GLSL_TYPE_UINT, 2))
62 LOAD_INPUT(src_inv_size, glsl_vector_type(GLSL_TYPE_FLOAT, 2))
63
64 #undef LOAD_INPUT
65 }
66
67 static nir_def *
blorp_blit_get_frag_coords(nir_builder * b,const struct blorp_blit_prog_key * key,struct blorp_blit_vars * v)68 blorp_blit_get_frag_coords(nir_builder *b,
69 const struct blorp_blit_prog_key *key,
70 struct blorp_blit_vars *v)
71 {
72 nir_def *coord = nir_f2i32(b, nir_load_frag_coord(b));
73
74 /* Account for destination surface intratile offset
75 *
76 * Transformation parameters giving translation from destination to source
77 * coordinates don't take into account possible intra-tile destination
78 * offset. Therefore it has to be first subtracted from the incoming
79 * coordinates. Vertices are set up based on coordinates containing the
80 * intra-tile offset.
81 */
82 if (key->need_dst_offset)
83 coord = nir_isub(b, coord, nir_load_var(b, v->v_dst_offset));
84
85 if (key->persample_msaa_dispatch) {
86 b->shader->info.fs.uses_sample_shading = true;
87 return nir_vec3(b, nir_channel(b, coord, 0), nir_channel(b, coord, 1),
88 nir_load_sample_id(b));
89 } else {
90 return nir_trim_vector(b, coord, 2);
91 }
92 }
93
94 static nir_def *
blorp_blit_get_cs_dst_coords(nir_builder * b,const struct blorp_blit_prog_key * key,struct blorp_blit_vars * v)95 blorp_blit_get_cs_dst_coords(nir_builder *b,
96 const struct blorp_blit_prog_key *key,
97 struct blorp_blit_vars *v)
98 {
99 nir_def *coord = nir_load_global_invocation_id(b, 32);
100
101 /* Account for destination surface intratile offset
102 *
103 * Transformation parameters giving translation from destination to source
104 * coordinates don't take into account possible intra-tile destination
105 * offset. Therefore it has to be first subtracted from the incoming
106 * coordinates. Vertices are set up based on coordinates containing the
107 * intra-tile offset.
108 */
109 if (key->need_dst_offset)
110 coord = nir_isub(b, coord, nir_load_var(b, v->v_dst_offset));
111
112 assert(!key->persample_msaa_dispatch);
113 return nir_trim_vector(b, coord, 2);
114 }
115
116 /**
117 * Emit code to translate from destination (X, Y) coordinates to source (X, Y)
118 * coordinates.
119 */
120 static nir_def *
blorp_blit_apply_transform(nir_builder * b,nir_def * src_pos,struct blorp_blit_vars * v)121 blorp_blit_apply_transform(nir_builder *b, nir_def *src_pos,
122 struct blorp_blit_vars *v)
123 {
124 nir_def *coord_transform = nir_load_var(b, v->v_coord_transform);
125
126 nir_def *offset = nir_vec2(b, nir_channel(b, coord_transform, 1),
127 nir_channel(b, coord_transform, 3));
128 nir_def *mul = nir_vec2(b, nir_channel(b, coord_transform, 0),
129 nir_channel(b, coord_transform, 2));
130
131 return nir_fadd(b, nir_fmul(b, src_pos, mul), offset);
132 }
133
134 static nir_tex_instr *
blorp_create_nir_tex_instr(nir_builder * b,struct blorp_blit_vars * v,nir_texop op,nir_def * pos,unsigned num_srcs,nir_alu_type dst_type)135 blorp_create_nir_tex_instr(nir_builder *b, struct blorp_blit_vars *v,
136 nir_texop op, nir_def *pos, unsigned num_srcs,
137 nir_alu_type dst_type)
138 {
139 nir_tex_instr *tex = nir_tex_instr_create(b->shader, num_srcs);
140
141 tex->op = op;
142
143 tex->dest_type = dst_type | 32;
144 tex->is_array = false;
145 tex->is_shadow = false;
146
147 tex->texture_index = BLORP_TEXTURE_BT_INDEX;
148 tex->sampler_index = BLORP_SAMPLER_INDEX;
149
150 /* To properly handle 3-D and 2-D array textures, we pull the Z component
151 * from an input. TODO: This is a bit magic; we should probably make this
152 * more explicit in the future.
153 */
154 assert(pos->num_components >= 2);
155 if (op == nir_texop_txf || op == nir_texop_txf_ms ||
156 op == nir_texop_txf_ms_mcs_intel) {
157 pos = nir_vec3(b, nir_channel(b, pos, 0), nir_channel(b, pos, 1),
158 nir_f2i32(b, nir_load_var(b, v->v_src_z)));
159 } else {
160 pos = nir_vec3(b, nir_channel(b, pos, 0), nir_channel(b, pos, 1),
161 nir_load_var(b, v->v_src_z));
162 }
163
164 tex->src[0] = nir_tex_src_for_ssa(nir_tex_src_coord, pos);
165 tex->coord_components = 3;
166
167 nir_def_init(&tex->instr, &tex->def, 4, 32);
168
169 return tex;
170 }
171
172 static nir_def *
blorp_nir_tex(nir_builder * b,struct blorp_blit_vars * v,const struct blorp_blit_prog_key * key,nir_def * pos)173 blorp_nir_tex(nir_builder *b, struct blorp_blit_vars *v,
174 const struct blorp_blit_prog_key *key, nir_def *pos)
175 {
176 if (key->need_src_offset)
177 pos = nir_fadd(b, pos, nir_i2f32(b, nir_load_var(b, v->v_src_offset)));
178
179 /* If the sampler requires normalized coordinates, we need to compensate. */
180 if (key->src_coords_normalized)
181 pos = nir_fmul(b, pos, nir_load_var(b, v->v_src_inv_size));
182
183 nir_tex_instr *tex =
184 blorp_create_nir_tex_instr(b, v, nir_texop_txl, pos, 2,
185 key->texture_data_type);
186
187 assert(pos->num_components == 2);
188 tex->sampler_dim = GLSL_SAMPLER_DIM_2D;
189 tex->src[1] = nir_tex_src_for_ssa(nir_tex_src_lod, nir_imm_int(b, 0));
190
191 nir_builder_instr_insert(b, &tex->instr);
192
193 return &tex->def;
194 }
195
196 static nir_def *
blorp_nir_txf(nir_builder * b,struct blorp_blit_vars * v,nir_def * pos,nir_alu_type dst_type)197 blorp_nir_txf(nir_builder *b, struct blorp_blit_vars *v,
198 nir_def *pos, nir_alu_type dst_type)
199 {
200 nir_tex_instr *tex =
201 blorp_create_nir_tex_instr(b, v, nir_texop_txf, pos, 2, dst_type);
202
203 tex->sampler_dim = GLSL_SAMPLER_DIM_3D;
204 tex->src[1] = nir_tex_src_for_ssa(nir_tex_src_lod, nir_imm_int(b, 0));
205
206 nir_builder_instr_insert(b, &tex->instr);
207
208 return &tex->def;
209 }
210
211 static nir_def *
blorp_nir_txf_ms(nir_builder * b,struct blorp_blit_vars * v,nir_def * pos,nir_def * mcs,nir_alu_type dst_type)212 blorp_nir_txf_ms(nir_builder *b, struct blorp_blit_vars *v,
213 nir_def *pos, nir_def *mcs, nir_alu_type dst_type)
214 {
215 nir_tex_instr *tex =
216 blorp_create_nir_tex_instr(b, v, nir_texop_txf_ms, pos, 3, dst_type);
217
218 tex->sampler_dim = GLSL_SAMPLER_DIM_MS;
219
220 tex->src[1].src_type = nir_tex_src_ms_index;
221 if (pos->num_components == 2) {
222 tex->src[1].src = nir_src_for_ssa(nir_imm_int(b, 0));
223 } else {
224 assert(pos->num_components == 3);
225 tex->src[1].src = nir_src_for_ssa(nir_channel(b, pos, 2));
226 }
227
228 if (!mcs)
229 mcs = nir_imm_zero(b, 4, 32);
230
231 tex->src[2] = nir_tex_src_for_ssa(nir_tex_src_ms_mcs_intel, mcs);
232
233 nir_builder_instr_insert(b, &tex->instr);
234
235 return &tex->def;
236 }
237
238 static nir_def *
blorp_blit_txf_ms_mcs(nir_builder * b,struct blorp_blit_vars * v,nir_def * pos)239 blorp_blit_txf_ms_mcs(nir_builder *b, struct blorp_blit_vars *v,
240 nir_def *pos)
241 {
242 nir_tex_instr *tex =
243 blorp_create_nir_tex_instr(b, v, nir_texop_txf_ms_mcs_intel,
244 pos, 1, nir_type_int);
245
246 tex->sampler_dim = GLSL_SAMPLER_DIM_MS;
247
248 nir_builder_instr_insert(b, &tex->instr);
249
250 return &tex->def;
251 }
252
253 /**
254 * Emit code to compensate for the difference between Y and W tiling.
255 *
256 * This code modifies the X and Y coordinates according to the formula:
257 *
258 * (X', Y', S') = detile(W-MAJOR, tile(Y-MAJOR, X, Y, S))
259 *
260 * (See blorp_build_nir_shader).
261 */
262 static inline nir_def *
blorp_nir_retile_y_to_w(nir_builder * b,nir_def * pos)263 blorp_nir_retile_y_to_w(nir_builder *b, nir_def *pos)
264 {
265 assert(pos->num_components == 2);
266 nir_def *x_Y = nir_channel(b, pos, 0);
267 nir_def *y_Y = nir_channel(b, pos, 1);
268
269 /* Given X and Y coordinates that describe an address using Y tiling,
270 * translate to the X and Y coordinates that describe the same address
271 * using W tiling.
272 *
273 * If we break down the low order bits of X and Y, using a
274 * single letter to represent each low-order bit:
275 *
276 * X = A << 7 | 0bBCDEFGH
277 * Y = J << 5 | 0bKLMNP (1)
278 *
279 * Then we can apply the Y tiling formula to see the memory offset being
280 * addressed:
281 *
282 * offset = (J * tile_pitch + A) << 12 | 0bBCDKLMNPEFGH (2)
283 *
284 * If we apply the W detiling formula to this memory location, that the
285 * corresponding X' and Y' coordinates are:
286 *
287 * X' = A << 6 | 0bBCDPFH (3)
288 * Y' = J << 6 | 0bKLMNEG
289 *
290 * Combining (1) and (3), we see that to transform (X, Y) to (X', Y'),
291 * we need to make the following computation:
292 *
293 * X' = (X & ~0b1011) >> 1 | (Y & 0b1) << 2 | X & 0b1 (4)
294 * Y' = (Y & ~0b1) << 1 | (X & 0b1000) >> 2 | (X & 0b10) >> 1
295 */
296 nir_def *x_W = nir_imm_int(b, 0);
297 x_W = nir_mask_shift_or(b, x_W, x_Y, 0xfffffff4, -1);
298 x_W = nir_mask_shift_or(b, x_W, y_Y, 0x1, 2);
299 x_W = nir_mask_shift_or(b, x_W, x_Y, 0x1, 0);
300
301 nir_def *y_W = nir_imm_int(b, 0);
302 y_W = nir_mask_shift_or(b, y_W, y_Y, 0xfffffffe, 1);
303 y_W = nir_mask_shift_or(b, y_W, x_Y, 0x8, -2);
304 y_W = nir_mask_shift_or(b, y_W, x_Y, 0x2, -1);
305
306 return nir_vec2(b, x_W, y_W);
307 }
308
309 /**
310 * Emit code to compensate for the difference between Y and W tiling.
311 *
312 * This code modifies the X and Y coordinates according to the formula:
313 *
314 * (X', Y', S') = detile(Y-MAJOR, tile(W-MAJOR, X, Y, S))
315 *
316 * (See blorp_build_nir_shader).
317 */
318 static inline nir_def *
blorp_nir_retile_w_to_y(nir_builder * b,nir_def * pos)319 blorp_nir_retile_w_to_y(nir_builder *b, nir_def *pos)
320 {
321 assert(pos->num_components == 2);
322 nir_def *x_W = nir_channel(b, pos, 0);
323 nir_def *y_W = nir_channel(b, pos, 1);
324
325 /* Applying the same logic as above, but in reverse, we obtain the
326 * formulas:
327 *
328 * X' = (X & ~0b101) << 1 | (Y & 0b10) << 2 | (Y & 0b1) << 1 | X & 0b1
329 * Y' = (Y & ~0b11) >> 1 | (X & 0b100) >> 2
330 */
331 nir_def *x_Y = nir_imm_int(b, 0);
332 x_Y = nir_mask_shift_or(b, x_Y, x_W, 0xfffffffa, 1);
333 x_Y = nir_mask_shift_or(b, x_Y, y_W, 0x2, 2);
334 x_Y = nir_mask_shift_or(b, x_Y, y_W, 0x1, 1);
335 x_Y = nir_mask_shift_or(b, x_Y, x_W, 0x1, 0);
336
337 nir_def *y_Y = nir_imm_int(b, 0);
338 y_Y = nir_mask_shift_or(b, y_Y, y_W, 0xfffffffc, -1);
339 y_Y = nir_mask_shift_or(b, y_Y, x_W, 0x4, -2);
340
341 return nir_vec2(b, x_Y, y_Y);
342 }
343
344 /**
345 * Emit code to compensate for the difference between MSAA and non-MSAA
346 * surfaces.
347 *
348 * This code modifies the X and Y coordinates according to the formula:
349 *
350 * (X', Y', S') = encode_msaa(num_samples, IMS, X, Y, S)
351 */
352 static inline nir_def *
blorp_nir_encode_msaa(nir_builder * b,nir_def * pos,unsigned num_samples,enum isl_msaa_layout layout)353 blorp_nir_encode_msaa(nir_builder *b, nir_def *pos,
354 unsigned num_samples, enum isl_msaa_layout layout)
355 {
356 assert(pos->num_components == 2 || pos->num_components == 3);
357
358 switch (layout) {
359 case ISL_MSAA_LAYOUT_NONE:
360 assert(pos->num_components == 2);
361 return pos;
362 case ISL_MSAA_LAYOUT_ARRAY:
363 /* No translation needed */
364 return pos;
365 case ISL_MSAA_LAYOUT_INTERLEAVED: {
366 nir_def *x_in = nir_channel(b, pos, 0);
367 nir_def *y_in = nir_channel(b, pos, 1);
368 nir_def *s_in = pos->num_components == 2 ? nir_imm_int(b, 0) :
369 nir_channel(b, pos, 2);
370
371 nir_def *x_out = nir_imm_int(b, 0);
372 nir_def *y_out = nir_imm_int(b, 0);
373 switch (num_samples) {
374 case 2:
375 case 4:
376 /* encode_msaa(2, IMS, X, Y, S) = (X', Y', 0)
377 * where X' = (X & ~0b1) << 1 | (S & 0b1) << 1 | (X & 0b1)
378 * Y' = Y
379 *
380 * encode_msaa(4, IMS, X, Y, S) = (X', Y', 0)
381 * where X' = (X & ~0b1) << 1 | (S & 0b1) << 1 | (X & 0b1)
382 * Y' = (Y & ~0b1) << 1 | (S & 0b10) | (Y & 0b1)
383 */
384 x_out = nir_mask_shift_or(b, x_out, x_in, 0xfffffffe, 1);
385 x_out = nir_mask_shift_or(b, x_out, s_in, 0x1, 1);
386 x_out = nir_mask_shift_or(b, x_out, x_in, 0x1, 0);
387 if (num_samples == 2) {
388 y_out = y_in;
389 } else {
390 y_out = nir_mask_shift_or(b, y_out, y_in, 0xfffffffe, 1);
391 y_out = nir_mask_shift_or(b, y_out, s_in, 0x2, 0);
392 y_out = nir_mask_shift_or(b, y_out, y_in, 0x1, 0);
393 }
394 break;
395
396 case 8:
397 /* encode_msaa(8, IMS, X, Y, S) = (X', Y', 0)
398 * where X' = (X & ~0b1) << 2 | (S & 0b100) | (S & 0b1) << 1
399 * | (X & 0b1)
400 * Y' = (Y & ~0b1) << 1 | (S & 0b10) | (Y & 0b1)
401 */
402 x_out = nir_mask_shift_or(b, x_out, x_in, 0xfffffffe, 2);
403 x_out = nir_mask_shift_or(b, x_out, s_in, 0x4, 0);
404 x_out = nir_mask_shift_or(b, x_out, s_in, 0x1, 1);
405 x_out = nir_mask_shift_or(b, x_out, x_in, 0x1, 0);
406 y_out = nir_mask_shift_or(b, y_out, y_in, 0xfffffffe, 1);
407 y_out = nir_mask_shift_or(b, y_out, s_in, 0x2, 0);
408 y_out = nir_mask_shift_or(b, y_out, y_in, 0x1, 0);
409 break;
410
411 case 16:
412 /* encode_msaa(16, IMS, X, Y, S) = (X', Y', 0)
413 * where X' = (X & ~0b1) << 2 | (S & 0b100) | (S & 0b1) << 1
414 * | (X & 0b1)
415 * Y' = (Y & ~0b1) << 2 | (S & 0b1000) >> 1 (S & 0b10)
416 * | (Y & 0b1)
417 */
418 x_out = nir_mask_shift_or(b, x_out, x_in, 0xfffffffe, 2);
419 x_out = nir_mask_shift_or(b, x_out, s_in, 0x4, 0);
420 x_out = nir_mask_shift_or(b, x_out, s_in, 0x1, 1);
421 x_out = nir_mask_shift_or(b, x_out, x_in, 0x1, 0);
422 y_out = nir_mask_shift_or(b, y_out, y_in, 0xfffffffe, 2);
423 y_out = nir_mask_shift_or(b, y_out, s_in, 0x8, -1);
424 y_out = nir_mask_shift_or(b, y_out, s_in, 0x2, 0);
425 y_out = nir_mask_shift_or(b, y_out, y_in, 0x1, 0);
426 break;
427
428 default:
429 unreachable("Invalid number of samples for IMS layout");
430 }
431
432 return nir_vec2(b, x_out, y_out);
433 }
434
435 default:
436 unreachable("Invalid MSAA layout");
437 }
438 }
439
440 /**
441 * Emit code to compensate for the difference between MSAA and non-MSAA
442 * surfaces.
443 *
444 * This code modifies the X and Y coordinates according to the formula:
445 *
446 * (X', Y', S) = decode_msaa(num_samples, IMS, X, Y, S)
447 */
448 static inline nir_def *
blorp_nir_decode_msaa(nir_builder * b,nir_def * pos,unsigned num_samples,enum isl_msaa_layout layout)449 blorp_nir_decode_msaa(nir_builder *b, nir_def *pos,
450 unsigned num_samples, enum isl_msaa_layout layout)
451 {
452 assert(pos->num_components == 2 || pos->num_components == 3);
453
454 switch (layout) {
455 case ISL_MSAA_LAYOUT_NONE:
456 /* No translation necessary, and S should already be zero. */
457 assert(pos->num_components == 2);
458 return pos;
459 case ISL_MSAA_LAYOUT_ARRAY:
460 /* No translation necessary. */
461 return pos;
462 case ISL_MSAA_LAYOUT_INTERLEAVED: {
463 assert(pos->num_components == 2);
464
465 nir_def *x_in = nir_channel(b, pos, 0);
466 nir_def *y_in = nir_channel(b, pos, 1);
467
468 nir_def *x_out = nir_imm_int(b, 0);
469 nir_def *y_out = nir_imm_int(b, 0);
470 nir_def *s_out = nir_imm_int(b, 0);
471 switch (num_samples) {
472 case 2:
473 case 4:
474 /* decode_msaa(2, IMS, X, Y, 0) = (X', Y', S)
475 * where X' = (X & ~0b11) >> 1 | (X & 0b1)
476 * S = (X & 0b10) >> 1
477 *
478 * decode_msaa(4, IMS, X, Y, 0) = (X', Y', S)
479 * where X' = (X & ~0b11) >> 1 | (X & 0b1)
480 * Y' = (Y & ~0b11) >> 1 | (Y & 0b1)
481 * S = (Y & 0b10) | (X & 0b10) >> 1
482 */
483 x_out = nir_mask_shift_or(b, x_out, x_in, 0xfffffffc, -1);
484 x_out = nir_mask_shift_or(b, x_out, x_in, 0x1, 0);
485 if (num_samples == 2) {
486 y_out = y_in;
487 s_out = nir_mask_shift_or(b, s_out, x_in, 0x2, -1);
488 } else {
489 y_out = nir_mask_shift_or(b, y_out, y_in, 0xfffffffc, -1);
490 y_out = nir_mask_shift_or(b, y_out, y_in, 0x1, 0);
491 s_out = nir_mask_shift_or(b, s_out, x_in, 0x2, -1);
492 s_out = nir_mask_shift_or(b, s_out, y_in, 0x2, 0);
493 }
494 break;
495
496 case 8:
497 /* decode_msaa(8, IMS, X, Y, 0) = (X', Y', S)
498 * where X' = (X & ~0b111) >> 2 | (X & 0b1)
499 * Y' = (Y & ~0b11) >> 1 | (Y & 0b1)
500 * S = (X & 0b100) | (Y & 0b10) | (X & 0b10) >> 1
501 */
502 x_out = nir_mask_shift_or(b, x_out, x_in, 0xfffffff8, -2);
503 x_out = nir_mask_shift_or(b, x_out, x_in, 0x1, 0);
504 y_out = nir_mask_shift_or(b, y_out, y_in, 0xfffffffc, -1);
505 y_out = nir_mask_shift_or(b, y_out, y_in, 0x1, 0);
506 s_out = nir_mask_shift_or(b, s_out, x_in, 0x4, 0);
507 s_out = nir_mask_shift_or(b, s_out, y_in, 0x2, 0);
508 s_out = nir_mask_shift_or(b, s_out, x_in, 0x2, -1);
509 break;
510
511 case 16:
512 /* decode_msaa(16, IMS, X, Y, 0) = (X', Y', S)
513 * where X' = (X & ~0b111) >> 2 | (X & 0b1)
514 * Y' = (Y & ~0b111) >> 2 | (Y & 0b1)
515 * S = (Y & 0b100) << 1 | (X & 0b100) |
516 * (Y & 0b10) | (X & 0b10) >> 1
517 */
518 x_out = nir_mask_shift_or(b, x_out, x_in, 0xfffffff8, -2);
519 x_out = nir_mask_shift_or(b, x_out, x_in, 0x1, 0);
520 y_out = nir_mask_shift_or(b, y_out, y_in, 0xfffffff8, -2);
521 y_out = nir_mask_shift_or(b, y_out, y_in, 0x1, 0);
522 s_out = nir_mask_shift_or(b, s_out, y_in, 0x4, 1);
523 s_out = nir_mask_shift_or(b, s_out, x_in, 0x4, 0);
524 s_out = nir_mask_shift_or(b, s_out, y_in, 0x2, 0);
525 s_out = nir_mask_shift_or(b, s_out, x_in, 0x2, -1);
526 break;
527
528 default:
529 unreachable("Invalid number of samples for IMS layout");
530 }
531
532 return nir_vec3(b, x_out, y_out, s_out);
533 }
534
535 default:
536 unreachable("Invalid MSAA layout");
537 }
538 }
539
540 /**
541 * Count the number of trailing 1 bits in the given value. For example:
542 *
543 * count_trailing_one_bits(0) == 0
544 * count_trailing_one_bits(7) == 3
545 * count_trailing_one_bits(11) == 2
546 */
count_trailing_one_bits(unsigned value)547 static inline int count_trailing_one_bits(unsigned value)
548 {
549 #ifdef HAVE___BUILTIN_CTZ
550 return __builtin_ctz(~value);
551 #else
552 return util_bitcount(value & ~(value + 1));
553 #endif
554 }
555
556 static nir_def *
blorp_nir_combine_samples(nir_builder * b,struct blorp_blit_vars * v,nir_def * pos,unsigned tex_samples,enum isl_aux_usage tex_aux_usage,nir_alu_type dst_type,enum blorp_filter filter)557 blorp_nir_combine_samples(nir_builder *b, struct blorp_blit_vars *v,
558 nir_def *pos, unsigned tex_samples,
559 enum isl_aux_usage tex_aux_usage,
560 nir_alu_type dst_type,
561 enum blorp_filter filter)
562 {
563 nir_variable *color =
564 nir_local_variable_create(b->impl, glsl_vec4_type(), "color");
565
566 nir_def *mcs = NULL;
567 if (isl_aux_usage_has_mcs(tex_aux_usage))
568 mcs = blorp_blit_txf_ms_mcs(b, v, pos);
569
570 nir_op combine_op;
571 switch (filter) {
572 case BLORP_FILTER_AVERAGE:
573 assert(dst_type == nir_type_float);
574 combine_op = nir_op_fadd;
575 break;
576
577 case BLORP_FILTER_MIN_SAMPLE:
578 switch (dst_type) {
579 case nir_type_int: combine_op = nir_op_imin; break;
580 case nir_type_uint: combine_op = nir_op_umin; break;
581 case nir_type_float: combine_op = nir_op_fmin; break;
582 default: unreachable("Invalid dst_type");
583 }
584 break;
585
586 case BLORP_FILTER_MAX_SAMPLE:
587 switch (dst_type) {
588 case nir_type_int: combine_op = nir_op_imax; break;
589 case nir_type_uint: combine_op = nir_op_umax; break;
590 case nir_type_float: combine_op = nir_op_fmax; break;
591 default: unreachable("Invalid dst_type");
592 }
593 break;
594
595 default:
596 unreachable("Invalid filter");
597 }
598
599 /* If true, we inserted an if statement that we need to pop at at the end.
600 */
601 bool inserted_if = false;
602
603 /* We add together samples using a binary tree structure, e.g. for 4x MSAA:
604 *
605 * result = ((sample[0] + sample[1]) + (sample[2] + sample[3])) / 4
606 *
607 * This ensures that when all samples have the same value, no numerical
608 * precision is lost, since each addition operation always adds two equal
609 * values, and summing two equal floating point values does not lose
610 * precision.
611 *
612 * We perform this computation by treating the texture_data array as a
613 * stack and performing the following operations:
614 *
615 * - push sample 0 onto stack
616 * - push sample 1 onto stack
617 * - add top two stack entries
618 * - push sample 2 onto stack
619 * - push sample 3 onto stack
620 * - add top two stack entries
621 * - add top two stack entries
622 * - divide top stack entry by 4
623 *
624 * Note that after pushing sample i onto the stack, the number of add
625 * operations we do is equal to the number of trailing 1 bits in i. This
626 * works provided the total number of samples is a power of two, which it
627 * always is for i965.
628 *
629 * For integer formats, we replace the add operations with average
630 * operations and skip the final division.
631 */
632 nir_def *texture_data[5];
633 texture_data[0] = NULL; /* Avoid maybe-uninitialized warning with GCC 10 */
634 unsigned stack_depth = 0;
635 for (unsigned i = 0; i < tex_samples; ++i) {
636 assert(stack_depth == util_bitcount(i)); /* Loop invariant */
637
638 /* Push sample i onto the stack */
639 assert(stack_depth < ARRAY_SIZE(texture_data));
640
641 nir_def *ms_pos = nir_vec3(b, nir_channel(b, pos, 0),
642 nir_channel(b, pos, 1),
643 nir_imm_int(b, i));
644 texture_data[stack_depth++] = blorp_nir_txf_ms(b, v, ms_pos, mcs, dst_type);
645
646 if (i == 0 && isl_aux_usage_has_mcs(tex_aux_usage)) {
647 /* The Ivy Bridge PRM, Vol4 Part1 p27 (Multisample Control Surface)
648 * suggests an optimization:
649 *
650 * "A simple optimization with probable large return in
651 * performance is to compare the MCS value to zero (indicating
652 * all samples are on sample slice 0), and sample only from
653 * sample slice 0 using ld2dss if MCS is zero."
654 *
655 * Note that in the case where the MCS value is zero, sampling from
656 * sample slice 0 using ld2dss and sampling from sample 0 using
657 * ld2dms are equivalent (since all samples are on sample slice 0).
658 * Since we have already sampled from sample 0, all we need to do is
659 * skip the remaining fetches and averaging if MCS is zero.
660 *
661 * It's also trivial to detect when the MCS has the magic clear color
662 * value. In this case, the txf we did on sample 0 will return the
663 * clear color and we can skip the remaining fetches just like we do
664 * when MCS == 0.
665 */
666 nir_def *mcs_zero = nir_ieq_imm(b, nir_channel(b, mcs, 0), 0);
667 if (tex_samples == 16) {
668 mcs_zero = nir_iand(b, mcs_zero,
669 nir_ieq_imm(b, nir_channel(b, mcs, 1), 0));
670 }
671 nir_def *mcs_clear =
672 blorp_nir_mcs_is_clear_color(b, mcs, tex_samples);
673
674 nir_push_if(b, nir_ior(b, mcs_zero, mcs_clear));
675 nir_store_var(b, color, texture_data[0], 0xf);
676
677 nir_push_else(b, NULL);
678 inserted_if = true;
679 }
680
681 for (int j = 0; j < count_trailing_one_bits(i); j++) {
682 assert(stack_depth >= 2);
683 --stack_depth;
684
685 texture_data[stack_depth - 1] =
686 nir_build_alu(b, combine_op,
687 texture_data[stack_depth - 1],
688 texture_data[stack_depth],
689 NULL, NULL);
690 }
691 }
692
693 /* We should have just 1 sample on the stack now. */
694 assert(stack_depth == 1);
695
696 if (filter == BLORP_FILTER_AVERAGE) {
697 assert(dst_type == nir_type_float);
698 texture_data[0] = nir_fmul_imm(b, texture_data[0],
699 1.0 / tex_samples);
700 }
701
702 nir_store_var(b, color, texture_data[0], 0xf);
703
704 if (inserted_if)
705 nir_pop_if(b, NULL);
706
707 return nir_load_var(b, color);
708 }
709
710 static nir_def *
blorp_nir_manual_blend_bilinear(nir_builder * b,nir_def * pos,unsigned tex_samples,const struct blorp_blit_prog_key * key,struct blorp_blit_vars * v)711 blorp_nir_manual_blend_bilinear(nir_builder *b, nir_def *pos,
712 unsigned tex_samples,
713 const struct blorp_blit_prog_key *key,
714 struct blorp_blit_vars *v)
715 {
716 nir_def *pos_xy = nir_trim_vector(b, pos, 2);
717 nir_def *rect_grid = nir_load_var(b, v->v_rect_grid);
718 nir_def *scale = nir_imm_vec2(b, key->x_scale, key->y_scale);
719
720 /* Translate coordinates to lay out the samples in a rectangular grid
721 * roughly corresponding to sample locations.
722 */
723 pos_xy = nir_fmul(b, pos_xy, scale);
724 /* Adjust coordinates so that integers represent pixel centers rather
725 * than pixel edges.
726 */
727 pos_xy = nir_fadd_imm(b, pos_xy, -0.5);
728 /* Clamp the X, Y texture coordinates to properly handle the sampling of
729 * texels on texture edges.
730 */
731 pos_xy = nir_fmin(b, nir_fmax(b, pos_xy, nir_imm_float(b, 0.0)),
732 nir_trim_vector(b, rect_grid, 2));
733
734 /* Store the fractional parts to be used as bilinear interpolation
735 * coefficients.
736 */
737 nir_def *frac_xy = nir_ffract(b, pos_xy);
738 /* Round the float coordinates down to nearest integer */
739 pos_xy = nir_fdiv(b, nir_ftrunc(b, pos_xy), scale);
740
741 nir_def *tex_data[4];
742 for (unsigned i = 0; i < 4; ++i) {
743 float sample_off_x = (float)(i & 0x1) / key->x_scale;
744 float sample_off_y = (float)((i >> 1) & 0x1) / key->y_scale;
745 nir_def *sample_off = nir_imm_vec2(b, sample_off_x, sample_off_y);
746
747 nir_def *sample_coords = nir_fadd(b, pos_xy, sample_off);
748 nir_def *sample_coords_int = nir_f2i32(b, sample_coords);
749
750 /* The MCS value we fetch has to match up with the pixel that we're
751 * sampling from. Since we sample from different pixels in each
752 * iteration of this "for" loop, the call to mcs_fetch() should be
753 * here inside the loop after computing the pixel coordinates.
754 */
755 nir_def *mcs = NULL;
756 if (isl_aux_usage_has_mcs(key->tex_aux_usage))
757 mcs = blorp_blit_txf_ms_mcs(b, v, sample_coords_int);
758
759 /* Compute sample index and map the sample index to a sample number.
760 * Sample index layout shows the numbering of slots in a rectangular
761 * grid of samples with in a pixel. Sample number layout shows the
762 * rectangular grid of samples roughly corresponding to the real sample
763 * locations with in a pixel.
764 *
765 * In the case of 2x MSAA, the layout of sample indices is reversed from
766 * the layout of sample numbers:
767 *
768 * sample index layout : --------- sample number layout : ---------
769 * | 0 | 1 | | 1 | 0 |
770 * --------- ---------
771 *
772 * In case of 4x MSAA, layout of sample indices matches the layout of
773 * sample numbers:
774 * ---------
775 * | 0 | 1 |
776 * ---------
777 * | 2 | 3 |
778 * ---------
779 *
780 * In case of 8x MSAA the two layouts don't match.
781 * sample index layout : --------- sample number layout : ---------
782 * | 0 | 1 | | 3 | 7 |
783 * --------- ---------
784 * | 2 | 3 | | 5 | 0 |
785 * --------- ---------
786 * | 4 | 5 | | 1 | 2 |
787 * --------- ---------
788 * | 6 | 7 | | 4 | 6 |
789 * --------- ---------
790 *
791 * Fortunately, this can be done fairly easily as:
792 * S' = (0x17306425 >> (S * 4)) & 0xf
793 *
794 * In the case of 16x MSAA the two layouts don't match.
795 * Sample index layout: Sample number layout:
796 * --------------------- ---------------------
797 * | 0 | 1 | 2 | 3 | | 15 | 10 | 9 | 7 |
798 * --------------------- ---------------------
799 * | 4 | 5 | 6 | 7 | | 4 | 1 | 3 | 13 |
800 * --------------------- ---------------------
801 * | 8 | 9 | 10 | 11 | | 12 | 2 | 0 | 6 |
802 * --------------------- ---------------------
803 * | 12 | 13 | 14 | 15 | | 11 | 8 | 5 | 14 |
804 * --------------------- ---------------------
805 *
806 * This is equivalent to
807 * S' = (0xe58b602cd31479af >> (S * 4)) & 0xf
808 */
809 nir_def *frac = nir_ffract(b, sample_coords);
810 nir_def *sample =
811 nir_fdot2(b, frac, nir_imm_vec2(b, key->x_scale,
812 key->x_scale * key->y_scale));
813 sample = nir_f2i32(b, sample);
814
815 if (tex_samples == 2) {
816 sample = nir_isub_imm(b, 1, sample);
817 } else if (tex_samples == 8) {
818 sample = nir_iand_imm(b, nir_ishr(b, nir_imm_int(b, 0x64210573),
819 nir_ishl_imm(b, sample, 2)),
820 0xf);
821 } else if (tex_samples == 16) {
822 nir_def *sample_low =
823 nir_iand_imm(b, nir_ishr(b, nir_imm_int(b, 0xd31479af),
824 nir_ishl_imm(b, sample, 2)),
825 0xf);
826 nir_def *sample_high =
827 nir_iand_imm(b, nir_ishr(b, nir_imm_int(b, 0xe58b602c),
828 nir_ishl_imm(b, nir_iadd_imm(b, sample, -8),
829 2)),
830 0xf);
831
832 sample = nir_bcsel(b, nir_ilt_imm(b, sample, 8),
833 sample_low, sample_high);
834 }
835 nir_def *pos_ms = nir_vec3(b, nir_channel(b, sample_coords_int, 0),
836 nir_channel(b, sample_coords_int, 1),
837 sample);
838 tex_data[i] = blorp_nir_txf_ms(b, v, pos_ms, mcs, key->texture_data_type);
839 }
840
841 nir_def *frac_x = nir_channel(b, frac_xy, 0);
842 nir_def *frac_y = nir_channel(b, frac_xy, 1);
843 return nir_flrp(b, nir_flrp(b, tex_data[0], tex_data[1], frac_x),
844 nir_flrp(b, tex_data[2], tex_data[3], frac_x),
845 frac_y);
846 }
847
848 /** Perform a color bit-cast operation
849 *
850 * For copy operations involving CCS, we may need to use different formats for
851 * the source and destination surfaces. The two formats must both be UINT
852 * formats and must have the same size but may have different bit layouts.
853 * For instance, we may be copying from R8G8B8A8_UINT to R32_UINT or R32_UINT
854 * to R16G16_UINT. This function generates code to shuffle bits around to get
855 * us from one to the other.
856 */
857 static nir_def *
bit_cast_color(struct nir_builder * b,nir_def * color,const struct blorp_blit_prog_key * key)858 bit_cast_color(struct nir_builder *b, nir_def *color,
859 const struct blorp_blit_prog_key *key)
860 {
861 if (key->src_format == key->dst_format)
862 return color;
863
864 const struct isl_format_layout *src_fmtl =
865 isl_format_get_layout(key->src_format);
866 const struct isl_format_layout *dst_fmtl =
867 isl_format_get_layout(key->dst_format);
868
869 /* They must be formats with the same bit size */
870 assert(src_fmtl->bpb == dst_fmtl->bpb);
871
872 if (src_fmtl->bpb <= 32) {
873 assert(src_fmtl->uniform_channel_type == ISL_UINT ||
874 src_fmtl->uniform_channel_type == ISL_UNORM);
875 assert(dst_fmtl->uniform_channel_type == ISL_UINT ||
876 dst_fmtl->uniform_channel_type == ISL_UNORM);
877
878 nir_def *packed = nir_imm_int(b, 0);
879 for (unsigned c = 0; c < 4; c++) {
880 if (src_fmtl->channels_array[c].bits == 0)
881 continue;
882
883 const unsigned chan_start_bit = src_fmtl->channels_array[c].start_bit;
884 const unsigned chan_bits = src_fmtl->channels_array[c].bits;
885
886 nir_def *chan = nir_channel(b, color, c);
887 if (src_fmtl->channels_array[c].type == ISL_UNORM) {
888
889 /* Don't touch the alpha channel */
890 if (c < 3 && src_fmtl->colorspace == ISL_COLORSPACE_SRGB)
891 chan = nir_format_linear_to_srgb(b, chan);
892
893 chan = nir_format_float_to_unorm(b, chan, &chan_bits);
894 }
895
896 packed = nir_ior(b, packed, nir_shift_imm(b, chan, chan_start_bit));
897 }
898
899 nir_def *chans[4] = { };
900 for (unsigned c = 0; c < 4; c++) {
901 if (dst_fmtl->channels_array[c].bits == 0) {
902 chans[c] = nir_imm_int(b, 0);
903 continue;
904 }
905
906 const unsigned chan_start_bit = dst_fmtl->channels_array[c].start_bit;
907 const unsigned chan_bits = dst_fmtl->channels_array[c].bits;
908 chans[c] = nir_iand_imm(b, nir_shift_imm(b, packed, -(int)chan_start_bit),
909 BITFIELD_MASK(chan_bits));
910
911 if (dst_fmtl->channels_array[c].type == ISL_UNORM) {
912 chans[c] = nir_format_unorm_to_float(b, chans[c], &chan_bits);
913
914 /* Don't touch the alpha channel */
915 if (c < 3 && dst_fmtl->colorspace == ISL_COLORSPACE_SRGB)
916 chans[c] = nir_format_srgb_to_linear(b, chans[c]);
917 }
918 }
919 color = nir_vec(b, chans, 4);
920 } else {
921 /* This path only supports UINT formats */
922 assert(src_fmtl->channels.r.type == ISL_UINT);
923 assert(dst_fmtl->channels.r.type == ISL_UINT);
924
925 const unsigned src_bpc = src_fmtl->channels.r.bits;
926 const unsigned dst_bpc = dst_fmtl->channels.r.bits;
927
928 assert(src_fmtl->channels.g.bits == 0 ||
929 src_fmtl->channels.g.bits == src_fmtl->channels.r.bits);
930 assert(src_fmtl->channels.b.bits == 0 ||
931 src_fmtl->channels.b.bits == src_fmtl->channels.r.bits);
932 assert(src_fmtl->channels.a.bits == 0 ||
933 src_fmtl->channels.a.bits == src_fmtl->channels.r.bits);
934 assert(dst_fmtl->channels.g.bits == 0 ||
935 dst_fmtl->channels.g.bits == dst_fmtl->channels.r.bits);
936 assert(dst_fmtl->channels.b.bits == 0 ||
937 dst_fmtl->channels.b.bits == dst_fmtl->channels.r.bits);
938 assert(dst_fmtl->channels.a.bits == 0 ||
939 dst_fmtl->channels.a.bits == dst_fmtl->channels.r.bits);
940
941 /* Restrict to only the channels we actually have */
942 const unsigned src_channels =
943 isl_format_get_num_channels(key->src_format);
944 color = nir_trim_vector(b, color, src_channels);
945
946 color = nir_format_bitcast_uvec_unmasked(b, color, src_bpc, dst_bpc);
947 }
948
949 /* Blorp likes to assume that colors are vec4s */
950 nir_def *u = nir_undef(b, 1, 32);
951 nir_def *chans[4] = { u, u, u, u };
952 for (unsigned i = 0; i < color->num_components; i++)
953 chans[i] = nir_channel(b, color, i);
954 return nir_vec4(b, chans[0], chans[1], chans[2], chans[3]);
955 }
956
957 static nir_def *
select_color_channel(struct nir_builder * b,nir_def * color,nir_alu_type data_type,enum isl_channel_select chan)958 select_color_channel(struct nir_builder *b, nir_def *color,
959 nir_alu_type data_type,
960 enum isl_channel_select chan)
961 {
962 if (chan == ISL_CHANNEL_SELECT_ZERO) {
963 return nir_imm_int(b, 0);
964 } else if (chan == ISL_CHANNEL_SELECT_ONE) {
965 switch (data_type) {
966 case nir_type_int:
967 case nir_type_uint:
968 return nir_imm_int(b, 1);
969 case nir_type_float:
970 return nir_imm_float(b, 1);
971 default:
972 unreachable("Invalid data type");
973 }
974 } else {
975 assert((unsigned)(chan - ISL_CHANNEL_SELECT_RED) < 4);
976 return nir_channel(b, color, chan - ISL_CHANNEL_SELECT_RED);
977 }
978 }
979
980 static nir_def *
swizzle_color(struct nir_builder * b,nir_def * color,struct isl_swizzle swizzle,nir_alu_type data_type)981 swizzle_color(struct nir_builder *b, nir_def *color,
982 struct isl_swizzle swizzle, nir_alu_type data_type)
983 {
984 return nir_vec4(b,
985 select_color_channel(b, color, data_type, swizzle.r),
986 select_color_channel(b, color, data_type, swizzle.g),
987 select_color_channel(b, color, data_type, swizzle.b),
988 select_color_channel(b, color, data_type, swizzle.a));
989 }
990
991 static nir_def *
convert_color(struct nir_builder * b,nir_def * color,const struct blorp_blit_prog_key * key)992 convert_color(struct nir_builder *b, nir_def *color,
993 const struct blorp_blit_prog_key *key)
994 {
995 /* All of our color conversions end up generating a single-channel color
996 * value that we need to write out.
997 */
998 nir_def *value;
999
1000 if (key->dst_format == ISL_FORMAT_R24_UNORM_X8_TYPELESS) {
1001 /* The destination image is bound as R32_UINT but the data needs to be
1002 * in R24_UNORM_X8_TYPELESS. The bottom 24 are the actual data and the
1003 * top 8 need to be zero. We can accomplish this by simply multiplying
1004 * by a factor to scale things down.
1005 */
1006 unsigned factor = (1 << 24) - 1;
1007 value = nir_fsat(b, nir_channel(b, color, 0));
1008 value = nir_f2i32(b, nir_fmul_imm(b, value, factor));
1009 } else if (key->dst_format == ISL_FORMAT_L8_UNORM_SRGB) {
1010 value = nir_format_linear_to_srgb(b, nir_channel(b, color, 0));
1011 } else if (key->dst_format == ISL_FORMAT_R8G8B8_UNORM_SRGB) {
1012 value = nir_format_linear_to_srgb(b, color);
1013 } else if (key->dst_format == ISL_FORMAT_R9G9B9E5_SHAREDEXP) {
1014 value = nir_format_pack_r9g9b9e5(b, color);
1015 } else {
1016 unreachable("Unsupported format conversion");
1017 }
1018
1019 nir_def *out_comps[4];
1020 for (unsigned i = 0; i < 4; i++) {
1021 if (i < value->num_components)
1022 out_comps[i] = nir_channel(b, value, i);
1023 else
1024 out_comps[i] = nir_undef(b, 1, 32);
1025 }
1026 return nir_vec(b, out_comps, 4);
1027 }
1028
1029 /**
1030 * Generator for WM programs used in BLORP blits.
1031 *
1032 * The bulk of the work done by the WM program is to wrap and unwrap the
1033 * coordinate transformations used by the hardware to store surfaces in
1034 * memory. The hardware transforms a pixel location (X, Y, S) (where S is the
1035 * sample index for a multisampled surface) to a memory offset by the
1036 * following formulas:
1037 *
1038 * offset = tile(tiling_format, encode_msaa(num_samples, layout, X, Y, S))
1039 * (X, Y, S) = decode_msaa(num_samples, layout, detile(tiling_format, offset))
1040 *
1041 * For a single-sampled surface, or for a multisampled surface using
1042 * INTEL_MSAA_LAYOUT_UMS, encode_msaa() and decode_msaa are the identity
1043 * function:
1044 *
1045 * encode_msaa(1, NONE, X, Y, 0) = (X, Y, 0)
1046 * decode_msaa(1, NONE, X, Y, 0) = (X, Y, 0)
1047 * encode_msaa(n, UMS, X, Y, S) = (X, Y, S)
1048 * decode_msaa(n, UMS, X, Y, S) = (X, Y, S)
1049 *
1050 * For a 4x multisampled surface using INTEL_MSAA_LAYOUT_IMS, encode_msaa()
1051 * embeds the sample number into bit 1 of the X and Y coordinates:
1052 *
1053 * encode_msaa(4, IMS, X, Y, S) = (X', Y', 0)
1054 * where X' = (X & ~0b1) << 1 | (S & 0b1) << 1 | (X & 0b1)
1055 * Y' = (Y & ~0b1 ) << 1 | (S & 0b10) | (Y & 0b1)
1056 * decode_msaa(4, IMS, X, Y, 0) = (X', Y', S)
1057 * where X' = (X & ~0b11) >> 1 | (X & 0b1)
1058 * Y' = (Y & ~0b11) >> 1 | (Y & 0b1)
1059 * S = (Y & 0b10) | (X & 0b10) >> 1
1060 *
1061 * For an 8x multisampled surface using INTEL_MSAA_LAYOUT_IMS, encode_msaa()
1062 * embeds the sample number into bits 1 and 2 of the X coordinate and bit 1 of
1063 * the Y coordinate:
1064 *
1065 * encode_msaa(8, IMS, X, Y, S) = (X', Y', 0)
1066 * where X' = (X & ~0b1) << 2 | (S & 0b100) | (S & 0b1) << 1 | (X & 0b1)
1067 * Y' = (Y & ~0b1) << 1 | (S & 0b10) | (Y & 0b1)
1068 * decode_msaa(8, IMS, X, Y, 0) = (X', Y', S)
1069 * where X' = (X & ~0b111) >> 2 | (X & 0b1)
1070 * Y' = (Y & ~0b11) >> 1 | (Y & 0b1)
1071 * S = (X & 0b100) | (Y & 0b10) | (X & 0b10) >> 1
1072 *
1073 * For X tiling, tile() combines together the low-order bits of the X and Y
1074 * coordinates in the pattern 0byyyxxxxxxxxx, creating 4k tiles that are 512
1075 * bytes wide and 8 rows high:
1076 *
1077 * tile(x_tiled, X, Y, S) = A
1078 * where A = tile_num << 12 | offset
1079 * tile_num = (Y' >> 3) * tile_pitch + (X' >> 9)
1080 * offset = (Y' & 0b111) << 9
1081 * | (X & 0b111111111)
1082 * X' = X * cpp
1083 * Y' = Y + S * qpitch
1084 * detile(x_tiled, A) = (X, Y, S)
1085 * where X = X' / cpp
1086 * Y = Y' % qpitch
1087 * S = Y' / qpitch
1088 * Y' = (tile_num / tile_pitch) << 3
1089 * | (A & 0b111000000000) >> 9
1090 * X' = (tile_num % tile_pitch) << 9
1091 * | (A & 0b111111111)
1092 *
1093 * (In all tiling formulas, cpp is the number of bytes occupied by a single
1094 * sample ("chars per pixel"), tile_pitch is the number of 4k tiles required
1095 * to fill the width of the surface, and qpitch is the spacing (in rows)
1096 * between array slices).
1097 *
1098 * For Y tiling, tile() combines together the low-order bits of the X and Y
1099 * coordinates in the pattern 0bxxxyyyyyxxxx, creating 4k tiles that are 128
1100 * bytes wide and 32 rows high:
1101 *
1102 * tile(y_tiled, X, Y, S) = A
1103 * where A = tile_num << 12 | offset
1104 * tile_num = (Y' >> 5) * tile_pitch + (X' >> 7)
1105 * offset = (X' & 0b1110000) << 5
1106 * | (Y' & 0b11111) << 4
1107 * | (X' & 0b1111)
1108 * X' = X * cpp
1109 * Y' = Y + S * qpitch
1110 * detile(y_tiled, A) = (X, Y, S)
1111 * where X = X' / cpp
1112 * Y = Y' % qpitch
1113 * S = Y' / qpitch
1114 * Y' = (tile_num / tile_pitch) << 5
1115 * | (A & 0b111110000) >> 4
1116 * X' = (tile_num % tile_pitch) << 7
1117 * | (A & 0b111000000000) >> 5
1118 * | (A & 0b1111)
1119 *
1120 * For W tiling, tile() combines together the low-order bits of the X and Y
1121 * coordinates in the pattern 0bxxxyyyyxyxyx, creating 4k tiles that are 64
1122 * bytes wide and 64 rows high (note that W tiling is only used for stencil
1123 * buffers, which always have cpp = 1 and S=0):
1124 *
1125 * tile(w_tiled, X, Y, S) = A
1126 * where A = tile_num << 12 | offset
1127 * tile_num = (Y' >> 6) * tile_pitch + (X' >> 6)
1128 * offset = (X' & 0b111000) << 6
1129 * | (Y' & 0b111100) << 3
1130 * | (X' & 0b100) << 2
1131 * | (Y' & 0b10) << 2
1132 * | (X' & 0b10) << 1
1133 * | (Y' & 0b1) << 1
1134 * | (X' & 0b1)
1135 * X' = X * cpp = X
1136 * Y' = Y + S * qpitch
1137 * detile(w_tiled, A) = (X, Y, S)
1138 * where X = X' / cpp = X'
1139 * Y = Y' % qpitch = Y'
1140 * S = Y / qpitch = 0
1141 * Y' = (tile_num / tile_pitch) << 6
1142 * | (A & 0b111100000) >> 3
1143 * | (A & 0b1000) >> 2
1144 * | (A & 0b10) >> 1
1145 * X' = (tile_num % tile_pitch) << 6
1146 * | (A & 0b111000000000) >> 6
1147 * | (A & 0b10000) >> 2
1148 * | (A & 0b100) >> 1
1149 * | (A & 0b1)
1150 *
1151 * Finally, for a non-tiled surface, tile() simply combines together the X and
1152 * Y coordinates in the natural way:
1153 *
1154 * tile(untiled, X, Y, S) = A
1155 * where A = Y * pitch + X'
1156 * X' = X * cpp
1157 * Y' = Y + S * qpitch
1158 * detile(untiled, A) = (X, Y, S)
1159 * where X = X' / cpp
1160 * Y = Y' % qpitch
1161 * S = Y' / qpitch
1162 * X' = A % pitch
1163 * Y' = A / pitch
1164 *
1165 * (In these formulas, pitch is the number of bytes occupied by a single row
1166 * of samples).
1167 */
1168 static nir_shader *
blorp_build_nir_shader(struct blorp_context * blorp,struct blorp_batch * batch,void * mem_ctx,const struct blorp_blit_prog_key * key)1169 blorp_build_nir_shader(struct blorp_context *blorp,
1170 struct blorp_batch *batch, void *mem_ctx,
1171 const struct blorp_blit_prog_key *key)
1172 {
1173 const struct intel_device_info *devinfo = blorp->isl_dev->info;
1174 nir_def *src_pos, *dst_pos, *color;
1175
1176 /* Sanity checks */
1177 if (key->dst_tiled_w && key->rt_samples > 1) {
1178 /* If the destination image is W tiled and multisampled, then the thread
1179 * must be dispatched once per sample, not once per pixel. This is
1180 * necessary because after conversion between W and Y tiling, there's no
1181 * guarantee that all samples corresponding to a single pixel will still
1182 * be together.
1183 */
1184 assert(key->persample_msaa_dispatch);
1185 }
1186
1187 if (key->persample_msaa_dispatch) {
1188 /* It only makes sense to do persample dispatch if the render target is
1189 * configured as multisampled.
1190 */
1191 assert(key->rt_samples > 0);
1192 }
1193
1194 /* Make sure layout is consistent with sample count */
1195 assert((key->tex_layout == ISL_MSAA_LAYOUT_NONE) ==
1196 (key->tex_samples <= 1));
1197 assert((key->rt_layout == ISL_MSAA_LAYOUT_NONE) ==
1198 (key->rt_samples <= 1));
1199 assert((key->src_layout == ISL_MSAA_LAYOUT_NONE) ==
1200 (key->src_samples <= 1));
1201 assert((key->dst_layout == ISL_MSAA_LAYOUT_NONE) ==
1202 (key->dst_samples <= 1));
1203
1204 nir_builder b;
1205 const bool compute =
1206 key->base.shader_pipeline == BLORP_SHADER_PIPELINE_COMPUTE;
1207 gl_shader_stage stage =
1208 compute ? MESA_SHADER_COMPUTE : MESA_SHADER_FRAGMENT;
1209 blorp_nir_init_shader(&b, blorp, mem_ctx, stage, NULL);
1210
1211 struct blorp_blit_vars v;
1212 blorp_blit_vars_init(&b, &v, key);
1213
1214 dst_pos = compute ?
1215 blorp_blit_get_cs_dst_coords(&b, key, &v) :
1216 blorp_blit_get_frag_coords(&b, key, &v);
1217
1218 /* Render target and texture hardware don't support W tiling until Gfx8. */
1219 const bool rt_tiled_w = false;
1220 const bool tex_tiled_w = devinfo->ver >= 8 && key->src_tiled_w;
1221
1222 /* The address that data will be written to is determined by the
1223 * coordinates supplied to the WM thread and the tiling and sample count of
1224 * the render target, according to the formula:
1225 *
1226 * (X, Y, S) = decode_msaa(rt_samples, detile(rt_tiling, offset))
1227 *
1228 * If the actual tiling and sample count of the destination surface are not
1229 * the same as the configuration of the render target, then these
1230 * coordinates are wrong and we have to adjust them to compensate for the
1231 * difference.
1232 */
1233 if (rt_tiled_w != key->dst_tiled_w ||
1234 key->rt_samples != key->dst_samples ||
1235 key->rt_layout != key->dst_layout) {
1236 dst_pos = blorp_nir_encode_msaa(&b, dst_pos, key->rt_samples,
1237 key->rt_layout);
1238 /* Now (X, Y, S) = detile(rt_tiling, offset) */
1239 if (rt_tiled_w != key->dst_tiled_w)
1240 dst_pos = blorp_nir_retile_y_to_w(&b, dst_pos);
1241 /* Now (X, Y, S) = detile(rt_tiling, offset) */
1242 dst_pos = blorp_nir_decode_msaa(&b, dst_pos, key->dst_samples,
1243 key->dst_layout);
1244 }
1245
1246 nir_def *comp = NULL;
1247 if (key->dst_rgb) {
1248 /* The destination image is bound as a red texture three times as wide
1249 * as the actual image. Our shader is effectively running one color
1250 * component at a time. We need to save off the component and adjust
1251 * the destination position.
1252 */
1253 assert(dst_pos->num_components == 2);
1254 nir_def *dst_x = nir_channel(&b, dst_pos, 0);
1255 comp = nir_umod_imm(&b, dst_x, 3);
1256 dst_pos = nir_vec2(&b, nir_idiv(&b, dst_x, nir_imm_int(&b, 3)),
1257 nir_channel(&b, dst_pos, 1));
1258 }
1259
1260 /* Now (X, Y, S) = decode_msaa(dst_samples, detile(dst_tiling, offset)).
1261 *
1262 * That is: X, Y and S now contain the true coordinates and sample index of
1263 * the data that the WM thread should output.
1264 *
1265 * If we need to kill pixels that are outside the destination rectangle,
1266 * now is the time to do it.
1267 */
1268 nir_if *bounds_if = NULL;
1269 if (key->use_kill) {
1270 nir_def *bounds_rect = nir_load_var(&b, v.v_bounds_rect);
1271 nir_def *in_bounds = blorp_check_in_bounds(&b, bounds_rect,
1272 dst_pos);
1273 if (!compute)
1274 nir_discard_if(&b, nir_inot(&b, in_bounds));
1275 else
1276 bounds_if = nir_push_if(&b, in_bounds);
1277 }
1278
1279 src_pos = blorp_blit_apply_transform(&b, nir_i2f32(&b, dst_pos), &v);
1280 if (dst_pos->num_components == 3) {
1281 /* The sample coordinate is an integer that we want left alone but
1282 * blorp_blit_apply_transform() blindly applies the transform to all
1283 * three coordinates. Grab the original sample index.
1284 */
1285 src_pos = nir_vec3(&b, nir_channel(&b, src_pos, 0),
1286 nir_channel(&b, src_pos, 1),
1287 nir_channel(&b, dst_pos, 2));
1288 }
1289
1290 /* If the source image is not multisampled, then we want to fetch sample
1291 * number 0, because that's the only sample there is.
1292 */
1293 if (key->src_samples == 1)
1294 src_pos = nir_trim_vector(&b, src_pos, 2);
1295
1296 /* X, Y, and S are now the coordinates of the pixel in the source image
1297 * that we want to texture from. Exception: if we are blending, then S is
1298 * irrelevant, because we are going to fetch all samples.
1299 */
1300 switch (key->filter) {
1301 case BLORP_FILTER_NONE:
1302 case BLORP_FILTER_NEAREST:
1303 case BLORP_FILTER_SAMPLE_0:
1304 /* We're going to use texelFetch, so we need integers */
1305 if (src_pos->num_components == 2) {
1306 src_pos = nir_f2i32(&b, src_pos);
1307 } else {
1308 assert(src_pos->num_components == 3);
1309 src_pos = nir_vec3(&b, nir_channel(&b, nir_f2i32(&b, src_pos), 0),
1310 nir_channel(&b, nir_f2i32(&b, src_pos), 1),
1311 nir_channel(&b, src_pos, 2));
1312 }
1313
1314 /* We aren't blending, which means we just want to fetch a single
1315 * sample from the source surface. The address that we want to fetch
1316 * from is related to the X, Y and S values according to the formula:
1317 *
1318 * (X, Y, S) = decode_msaa(src_samples, detile(src_tiling, offset)).
1319 *
1320 * If the actual tiling and sample count of the source surface are
1321 * not the same as the configuration of the texture, then we need to
1322 * adjust the coordinates to compensate for the difference.
1323 */
1324 if (tex_tiled_w != key->src_tiled_w ||
1325 key->tex_samples != key->src_samples ||
1326 key->tex_layout != key->src_layout) {
1327 src_pos = blorp_nir_encode_msaa(&b, src_pos, key->src_samples,
1328 key->src_layout);
1329 /* Now (X, Y, S) = detile(src_tiling, offset) */
1330 if (tex_tiled_w != key->src_tiled_w)
1331 src_pos = blorp_nir_retile_w_to_y(&b, src_pos);
1332 /* Now (X, Y, S) = detile(tex_tiling, offset) */
1333 src_pos = blorp_nir_decode_msaa(&b, src_pos, key->tex_samples,
1334 key->tex_layout);
1335 }
1336
1337 if (key->need_src_offset)
1338 src_pos = nir_iadd(&b, src_pos, nir_load_var(&b, v.v_src_offset));
1339
1340 /* Now (X, Y, S) = decode_msaa(tex_samples, detile(tex_tiling, offset)).
1341 *
1342 * In other words: X, Y, and S now contain values which, when passed to
1343 * the texturing unit, will cause data to be read from the correct
1344 * memory location. So we can fetch the texel now.
1345 */
1346 if (key->src_samples == 1) {
1347 color = blorp_nir_txf(&b, &v, src_pos, key->texture_data_type);
1348 } else {
1349 nir_def *mcs = NULL;
1350 if (isl_aux_usage_has_mcs(key->tex_aux_usage))
1351 mcs = blorp_blit_txf_ms_mcs(&b, &v, src_pos);
1352
1353 color = blorp_nir_txf_ms(&b, &v, src_pos, mcs, key->texture_data_type);
1354 }
1355 break;
1356
1357 case BLORP_FILTER_BILINEAR:
1358 assert(!key->src_tiled_w);
1359 assert(key->tex_samples == key->src_samples);
1360 assert(key->tex_layout == key->src_layout);
1361
1362 if (key->src_samples == 1) {
1363 color = blorp_nir_tex(&b, &v, key, src_pos);
1364 } else {
1365 assert(!key->use_kill);
1366 color = blorp_nir_manual_blend_bilinear(&b, src_pos, key->src_samples,
1367 key, &v);
1368 }
1369 break;
1370
1371 case BLORP_FILTER_AVERAGE:
1372 case BLORP_FILTER_MIN_SAMPLE:
1373 case BLORP_FILTER_MAX_SAMPLE:
1374 assert(!key->src_tiled_w);
1375 assert(key->tex_samples == key->src_samples);
1376 assert(key->tex_layout == key->src_layout);
1377
1378 /* Resolves (effecively) use texelFetch, so we need integers and we
1379 * don't care about the sample index if we got one.
1380 */
1381 src_pos = nir_f2i32(&b, nir_trim_vector(&b, src_pos, 2));
1382
1383 if (devinfo->ver == 6) {
1384 /* Because gfx6 only supports 4x interleved MSAA, we can do all the
1385 * blending we need with a single linear-interpolated texture lookup
1386 * at the center of the sample. The texture coordinates to be odd
1387 * integers so that they correspond to the center of a 2x2 block
1388 * representing the four samples that maxe up a pixel. So we need
1389 * to multiply our X and Y coordinates each by 2 and then add 1.
1390 */
1391 assert(key->src_coords_normalized);
1392 assert(key->filter == BLORP_FILTER_AVERAGE);
1393 src_pos = nir_fadd_imm(&b,
1394 nir_i2f32(&b, src_pos),
1395 0.5f);
1396 color = blorp_nir_tex(&b, &v, key, src_pos);
1397 } else {
1398 /* Gfx7+ hardware doesn't automatically blend. */
1399 color = blorp_nir_combine_samples(&b, &v, src_pos, key->src_samples,
1400 key->tex_aux_usage,
1401 key->texture_data_type,
1402 key->filter);
1403 }
1404 break;
1405
1406 default:
1407 unreachable("Invalid blorp filter");
1408 }
1409
1410 if (!isl_swizzle_is_identity(key->src_swizzle)) {
1411 color = swizzle_color(&b, color, key->src_swizzle,
1412 key->texture_data_type);
1413 }
1414
1415 if (!isl_swizzle_is_identity(key->dst_swizzle)) {
1416 color = swizzle_color(&b, color, isl_swizzle_invert(key->dst_swizzle),
1417 nir_type_int);
1418 }
1419
1420 if (key->format_bit_cast) {
1421 assert(isl_swizzle_is_identity(key->src_swizzle));
1422 assert(isl_swizzle_is_identity(key->dst_swizzle));
1423 color = bit_cast_color(&b, color, key);
1424 } else if (key->dst_format) {
1425 color = convert_color(&b, color, key);
1426 } else if (key->uint32_to_sint) {
1427 /* Normally the hardware will take care of converting values from/to
1428 * the source and destination formats. But a few cases need help.
1429 *
1430 * The Skylake PRM, volume 07, page 658 has a programming note:
1431 *
1432 * "When using SINT or UINT rendertarget surface formats, Blending
1433 * must be DISABLED. The Pre-Blend Color Clamp Enable and Color
1434 * Clamp Range fields are ignored, and an implied clamp to the
1435 * rendertarget surface format is performed."
1436 *
1437 * For UINT to SINT blits, our sample operation gives us a uint32_t,
1438 * but our render target write expects a signed int32_t number. If we
1439 * simply passed the value along, the hardware would interpret a value
1440 * with bit 31 set as a negative value, clamping it to the largest
1441 * negative number the destination format could represent. But the
1442 * actual source value is a positive number, so we want to clamp it
1443 * to INT_MAX. To fix this, we explicitly take min(color, INT_MAX).
1444 */
1445 color = nir_umin(&b, color, nir_imm_int(&b, INT32_MAX));
1446 } else if (key->sint32_to_uint) {
1447 /* Similar to above, but clamping negative numbers to zero. */
1448 color = nir_imax(&b, color, nir_imm_int(&b, 0));
1449 }
1450
1451 if (key->dst_rgb) {
1452 /* The destination image is bound as a red texture three times as wide
1453 * as the actual image. Our shader is effectively running one color
1454 * component at a time. We need to pick off the appropriate component
1455 * from the source color and write that to destination red.
1456 */
1457 assert(dst_pos->num_components == 2);
1458
1459 nir_def *color_component =
1460 nir_bcsel(&b, nir_ieq_imm(&b, comp, 0),
1461 nir_channel(&b, color, 0),
1462 nir_bcsel(&b, nir_ieq_imm(&b, comp, 1),
1463 nir_channel(&b, color, 1),
1464 nir_channel(&b, color, 2)));
1465
1466 nir_def *u = nir_undef(&b, 1, 32);
1467 color = nir_vec4(&b, color_component, u, u, u);
1468 }
1469
1470 if (compute) {
1471 nir_def *store_pos = nir_load_global_invocation_id(&b, 32);
1472 nir_image_store(&b, nir_imm_int(&b, 0),
1473 nir_pad_vector_imm_int(&b, store_pos, 0, 4),
1474 nir_imm_int(&b, 0),
1475 nir_pad_vector_imm_int(&b, color, 0, 4),
1476 nir_imm_int(&b, 0),
1477 .image_dim = GLSL_SAMPLER_DIM_2D,
1478 .image_array = true,
1479 .access = ACCESS_NON_READABLE);
1480 } else if (key->dst_usage == ISL_SURF_USAGE_RENDER_TARGET_BIT) {
1481 nir_variable *color_out =
1482 nir_variable_create(b.shader, nir_var_shader_out,
1483 glsl_vec4_type(), "gl_FragColor");
1484 color_out->data.location = FRAG_RESULT_COLOR;
1485 nir_store_var(&b, color_out, color, 0xf);
1486 } else if (key->dst_usage == ISL_SURF_USAGE_DEPTH_BIT) {
1487 nir_variable *depth_out =
1488 nir_variable_create(b.shader, nir_var_shader_out,
1489 glsl_float_type(), "gl_FragDepth");
1490 depth_out->data.location = FRAG_RESULT_DEPTH;
1491 nir_store_var(&b, depth_out, nir_channel(&b, color, 0), 0x1);
1492 } else if (key->dst_usage == ISL_SURF_USAGE_STENCIL_BIT) {
1493 nir_variable *stencil_out =
1494 nir_variable_create(b.shader, nir_var_shader_out,
1495 glsl_int_type(), "gl_FragStencilRef");
1496 stencil_out->data.location = FRAG_RESULT_STENCIL;
1497 nir_store_var(&b, stencil_out, nir_channel(&b, color, 0), 0x1);
1498 } else {
1499 unreachable("Invalid destination usage");
1500 }
1501
1502 if (bounds_if)
1503 nir_pop_if(&b, bounds_if);
1504
1505 return b.shader;
1506 }
1507
1508 static bool
blorp_get_blit_kernel_fs(struct blorp_batch * batch,struct blorp_params * params,const struct blorp_blit_prog_key * key)1509 blorp_get_blit_kernel_fs(struct blorp_batch *batch,
1510 struct blorp_params *params,
1511 const struct blorp_blit_prog_key *key)
1512 {
1513 struct blorp_context *blorp = batch->blorp;
1514
1515 if (blorp->lookup_shader(batch, key, sizeof(*key),
1516 ¶ms->wm_prog_kernel, ¶ms->wm_prog_data))
1517 return true;
1518
1519 void *mem_ctx = ralloc_context(NULL);
1520
1521 nir_shader *nir = blorp_build_nir_shader(blorp, batch, mem_ctx, key);
1522 nir->info.name =
1523 ralloc_strdup(nir, blorp_shader_type_to_name(key->base.shader_type));
1524
1525 const bool multisample_fbo = key->rt_samples > 1;
1526
1527 const struct blorp_program p =
1528 blorp_compile_fs(blorp, mem_ctx, nir, multisample_fbo, false);
1529
1530 bool result =
1531 blorp->upload_shader(batch, MESA_SHADER_FRAGMENT,
1532 key, sizeof(*key),
1533 p.kernel, p.kernel_size,
1534 p.prog_data, p.prog_data_size,
1535 ¶ms->wm_prog_kernel, ¶ms->wm_prog_data);
1536
1537 ralloc_free(mem_ctx);
1538 return result;
1539 }
1540
1541 static bool
blorp_get_blit_kernel_cs(struct blorp_batch * batch,struct blorp_params * params,const struct blorp_blit_prog_key * prog_key)1542 blorp_get_blit_kernel_cs(struct blorp_batch *batch,
1543 struct blorp_params *params,
1544 const struct blorp_blit_prog_key *prog_key)
1545 {
1546 struct blorp_context *blorp = batch->blorp;
1547
1548 if (blorp->lookup_shader(batch, prog_key, sizeof(*prog_key),
1549 ¶ms->cs_prog_kernel, ¶ms->cs_prog_data))
1550 return true;
1551
1552 void *mem_ctx = ralloc_context(NULL);
1553
1554 nir_shader *nir = blorp_build_nir_shader(blorp, batch, mem_ctx,
1555 prog_key);
1556 nir->info.name = ralloc_strdup(nir, "BLORP-gpgpu-blit");
1557 blorp_set_cs_dims(nir, prog_key->local_y);
1558
1559 assert(prog_key->rt_samples == 1);
1560
1561 const struct blorp_program p =
1562 blorp_compile_cs(blorp, mem_ctx, nir);
1563
1564 bool result =
1565 blorp->upload_shader(batch, MESA_SHADER_COMPUTE,
1566 prog_key, sizeof(*prog_key),
1567 p.kernel, p.kernel_size,
1568 p.prog_data, p.prog_data_size,
1569 ¶ms->cs_prog_kernel, ¶ms->cs_prog_data);
1570
1571 ralloc_free(mem_ctx);
1572 return result;
1573 }
1574
1575 static void
blorp_setup_coord_transform(struct blorp_coord_transform * xform,float src0,float src1,float dst0,float dst1,bool mirror)1576 blorp_setup_coord_transform(struct blorp_coord_transform *xform,
1577 float src0, float src1,
1578 float dst0, float dst1,
1579 bool mirror)
1580 {
1581 double scale = (double)(src1 - src0) / (double)(dst1 - dst0);
1582 if (!mirror) {
1583 /* When not mirroring a coordinate (say, X), we need:
1584 * src_x - src_x0 = (dst_x - dst_x0 + 0.5) * scale
1585 * Therefore:
1586 * src_x = src_x0 + (dst_x - dst_x0 + 0.5) * scale
1587 *
1588 * blorp program uses "round toward zero" to convert the
1589 * transformed floating point coordinates to integer coordinates,
1590 * whereas the behaviour we actually want is "round to nearest",
1591 * so 0.5 provides the necessary correction.
1592 */
1593 xform->multiplier = scale;
1594 xform->offset = src0 + (-(double)dst0 + 0.5) * scale;
1595 } else {
1596 /* When mirroring X we need:
1597 * src_x - src_x0 = dst_x1 - dst_x - 0.5
1598 * Therefore:
1599 * src_x = src_x0 + (dst_x1 -dst_x - 0.5) * scale
1600 */
1601 xform->multiplier = -scale;
1602 xform->offset = src0 + ((double)dst1 - 0.5) * scale;
1603 }
1604 }
1605
1606 static inline void
surf_get_intratile_offset_px(struct blorp_surface_info * info,uint32_t * tile_x_px,uint32_t * tile_y_px)1607 surf_get_intratile_offset_px(struct blorp_surface_info *info,
1608 uint32_t *tile_x_px, uint32_t *tile_y_px)
1609 {
1610 if (info->surf.msaa_layout == ISL_MSAA_LAYOUT_INTERLEAVED) {
1611 struct isl_extent2d px_size_sa =
1612 isl_get_interleaved_msaa_px_size_sa(info->surf.samples);
1613 assert(info->tile_x_sa % px_size_sa.width == 0);
1614 assert(info->tile_y_sa % px_size_sa.height == 0);
1615 *tile_x_px = info->tile_x_sa / px_size_sa.width;
1616 *tile_y_px = info->tile_y_sa / px_size_sa.height;
1617 } else {
1618 *tile_x_px = info->tile_x_sa;
1619 *tile_y_px = info->tile_y_sa;
1620 }
1621 }
1622
1623 void
blorp_surf_convert_to_single_slice(const struct isl_device * isl_dev,struct blorp_surface_info * info)1624 blorp_surf_convert_to_single_slice(const struct isl_device *isl_dev,
1625 struct blorp_surface_info *info)
1626 {
1627 bool ok UNUSED;
1628
1629 /* It would be insane to try and do this on a compressed surface */
1630 assert(info->aux_usage == ISL_AUX_USAGE_NONE);
1631
1632 /* Just bail if we have nothing to do. */
1633 if (info->surf.dim == ISL_SURF_DIM_2D &&
1634 info->view.base_level == 0 && info->view.base_array_layer == 0 &&
1635 info->surf.levels == 1 && info->surf.logical_level0_px.array_len == 1)
1636 return;
1637
1638 /* If this gets triggered then we've gotten here twice which. This
1639 * shouldn't happen thanks to the above early return.
1640 */
1641 assert(info->tile_x_sa == 0 && info->tile_y_sa == 0);
1642
1643 uint32_t layer = 0, z = 0;
1644 if (info->surf.dim == ISL_SURF_DIM_3D)
1645 z = info->view.base_array_layer + info->z_offset;
1646 else
1647 layer = info->view.base_array_layer;
1648
1649 uint64_t offset_B;
1650 isl_surf_get_image_surf(isl_dev, &info->surf,
1651 info->view.base_level, layer, z,
1652 &info->surf,
1653 &offset_B, &info->tile_x_sa, &info->tile_y_sa);
1654 info->addr.offset += offset_B;
1655
1656 uint32_t tile_x_px, tile_y_px;
1657 surf_get_intratile_offset_px(info, &tile_x_px, &tile_y_px);
1658
1659 /* Instead of using the X/Y Offset fields in RENDER_SURFACE_STATE, we place
1660 * the image at the tile boundary and offset our sampling or rendering.
1661 * For this reason, we need to grow the image by the offset to ensure that
1662 * the hardware doesn't think we've gone past the edge.
1663 */
1664 info->surf.logical_level0_px.w += tile_x_px;
1665 info->surf.logical_level0_px.h += tile_y_px;
1666 info->surf.phys_level0_sa.w += info->tile_x_sa;
1667 info->surf.phys_level0_sa.h += info->tile_y_sa;
1668
1669 /* The view is also different now. */
1670 info->view.base_level = 0;
1671 info->view.levels = 1;
1672 info->view.base_array_layer = 0;
1673 info->view.array_len = 1;
1674 info->z_offset = 0;
1675 }
1676
1677 void
blorp_surf_fake_interleaved_msaa(const struct isl_device * isl_dev,struct blorp_surface_info * info)1678 blorp_surf_fake_interleaved_msaa(const struct isl_device *isl_dev,
1679 struct blorp_surface_info *info)
1680 {
1681 assert(info->surf.msaa_layout == ISL_MSAA_LAYOUT_INTERLEAVED);
1682
1683 /* First, we need to convert it to a simple 1-level 1-layer 2-D surface */
1684 blorp_surf_convert_to_single_slice(isl_dev, info);
1685
1686 info->surf.logical_level0_px = info->surf.phys_level0_sa;
1687 info->surf.samples = 1;
1688 info->surf.msaa_layout = ISL_MSAA_LAYOUT_NONE;
1689 }
1690
1691 void
blorp_surf_retile_w_to_y(const struct isl_device * isl_dev,struct blorp_surface_info * info)1692 blorp_surf_retile_w_to_y(const struct isl_device *isl_dev,
1693 struct blorp_surface_info *info)
1694 {
1695 assert(info->surf.tiling == ISL_TILING_W);
1696
1697 /* First, we need to convert it to a simple 1-level 1-layer 2-D surface */
1698 blorp_surf_convert_to_single_slice(isl_dev, info);
1699
1700 /* On gfx7+, we don't have interleaved multisampling for color render
1701 * targets so we have to fake it.
1702 *
1703 * TODO: Are we sure we don't also need to fake it on gfx6?
1704 */
1705 if (isl_dev->info->ver > 6 &&
1706 info->surf.msaa_layout == ISL_MSAA_LAYOUT_INTERLEAVED) {
1707 blorp_surf_fake_interleaved_msaa(isl_dev, info);
1708 }
1709
1710 if (isl_dev->info->ver == 6 || isl_dev->info->ver == 7) {
1711 /* Gfx6-7 stencil buffers have a very large alignment coming in from the
1712 * miptree. It's out-of-bounds for what the surface state can handle.
1713 * Since we have a single layer and level, it doesn't really matter as
1714 * long as we don't pass a bogus value into isl_surf_fill_state().
1715 */
1716 info->surf.image_alignment_el = isl_extent3d(4, 2, 1);
1717 }
1718
1719 /* Now that we've converted everything to a simple 2-D surface with only
1720 * one miplevel, we can go about retiling it.
1721 */
1722 const unsigned x_align = 8, y_align = info->surf.samples != 0 ? 8 : 4;
1723 info->surf.tiling = ISL_TILING_Y0;
1724 info->surf.logical_level0_px.width =
1725 ALIGN(info->surf.logical_level0_px.width, x_align) * 2;
1726 info->surf.logical_level0_px.height =
1727 ALIGN(info->surf.logical_level0_px.height, y_align) / 2;
1728 info->tile_x_sa *= 2;
1729 info->tile_y_sa /= 2;
1730 }
1731
1732 static bool
can_shrink_surface(const struct blorp_surface_info * surf)1733 can_shrink_surface(const struct blorp_surface_info *surf)
1734 {
1735 /* The current code doesn't support offsets into the aux buffers. This
1736 * should be possible, but we need to make sure the offset is page
1737 * aligned for both the surface and the aux buffer surface. Generally
1738 * this mean using the page aligned offset for the aux buffer.
1739 *
1740 * Currently the cases where we must split the blit are limited to cases
1741 * where we don't have a aux buffer.
1742 */
1743 if (surf->aux_addr.buffer != NULL)
1744 return false;
1745
1746 /* We can't support splitting the blit for gen <= 7, because the qpitch
1747 * size is calculated by the hardware based on the surface height for
1748 * gen <= 7. In gen >= 8, the qpitch is controlled by the driver.
1749 */
1750 if (surf->surf.msaa_layout == ISL_MSAA_LAYOUT_ARRAY)
1751 return false;
1752
1753 return true;
1754 }
1755
1756 static unsigned
get_max_surface_size(const struct intel_device_info * devinfo,const struct blorp_surface_info * surf)1757 get_max_surface_size(const struct intel_device_info *devinfo,
1758 const struct blorp_surface_info *surf)
1759 {
1760 const unsigned max = devinfo->ver >= 7 ? 16384 : 8192;
1761 if (split_blorp_blit_debug && can_shrink_surface(surf))
1762 return max >> 4; /* A smaller restriction when debug is enabled */
1763 else
1764 return max;
1765 }
1766
1767 struct blt_axis {
1768 double src0, src1, dst0, dst1;
1769 bool mirror;
1770 };
1771
1772 struct blt_coords {
1773 struct blt_axis x, y;
1774 };
1775
1776 static enum isl_format
get_red_format_for_rgb_format(enum isl_format format)1777 get_red_format_for_rgb_format(enum isl_format format)
1778 {
1779 const struct isl_format_layout *fmtl = isl_format_get_layout(format);
1780
1781 switch (fmtl->channels.r.bits) {
1782 case 8:
1783 switch (fmtl->channels.r.type) {
1784 case ISL_UNORM:
1785 return ISL_FORMAT_R8_UNORM;
1786 case ISL_SNORM:
1787 return ISL_FORMAT_R8_SNORM;
1788 case ISL_UINT:
1789 return ISL_FORMAT_R8_UINT;
1790 case ISL_SINT:
1791 return ISL_FORMAT_R8_SINT;
1792 default:
1793 unreachable("Invalid 8-bit RGB channel type");
1794 }
1795 case 16:
1796 switch (fmtl->channels.r.type) {
1797 case ISL_UNORM:
1798 return ISL_FORMAT_R16_UNORM;
1799 case ISL_SNORM:
1800 return ISL_FORMAT_R16_SNORM;
1801 case ISL_SFLOAT:
1802 return ISL_FORMAT_R16_FLOAT;
1803 case ISL_UINT:
1804 return ISL_FORMAT_R16_UINT;
1805 case ISL_SINT:
1806 return ISL_FORMAT_R16_SINT;
1807 default:
1808 unreachable("Invalid 8-bit RGB channel type");
1809 }
1810 case 32:
1811 switch (fmtl->channels.r.type) {
1812 case ISL_SFLOAT:
1813 return ISL_FORMAT_R32_FLOAT;
1814 case ISL_UINT:
1815 return ISL_FORMAT_R32_UINT;
1816 case ISL_SINT:
1817 return ISL_FORMAT_R32_SINT;
1818 default:
1819 unreachable("Invalid 8-bit RGB channel type");
1820 }
1821 default:
1822 unreachable("Invalid number of red channel bits");
1823 }
1824 }
1825
1826 void
surf_fake_rgb_with_red(const struct isl_device * isl_dev,struct blorp_surface_info * info)1827 surf_fake_rgb_with_red(const struct isl_device *isl_dev,
1828 struct blorp_surface_info *info)
1829 {
1830 blorp_surf_convert_to_single_slice(isl_dev, info);
1831
1832 info->surf.logical_level0_px.width *= 3;
1833 info->surf.phys_level0_sa.width *= 3;
1834 info->tile_x_sa *= 3;
1835
1836 enum isl_format red_format =
1837 get_red_format_for_rgb_format(info->view.format);
1838
1839 assert(isl_format_get_layout(red_format)->channels.r.type ==
1840 isl_format_get_layout(info->view.format)->channels.r.type);
1841 assert(isl_format_get_layout(red_format)->channels.r.bits ==
1842 isl_format_get_layout(info->view.format)->channels.r.bits);
1843
1844 info->surf.format = info->view.format = red_format;
1845
1846 if (isl_dev->info->verx10 >= 125) {
1847 /* The horizontal alignment is in units of texels for NPOT formats, and
1848 * bytes for other formats. Since the only allowed alignment units are
1849 * powers of two, there's no way to convert the alignment.
1850 *
1851 * Thankfully, the value doesn't matter since we're only a single slice.
1852 * Pick one allowed by isl_gfx125_choose_image_alignment_el.
1853 */
1854 info->surf.image_alignment_el.w =
1855 128 / (isl_format_get_layout(red_format)->bpb / 8);
1856 }
1857 }
1858
1859 enum blit_shrink_status {
1860 BLIT_NO_SHRINK = 0,
1861 BLIT_SRC_WIDTH_SHRINK = (1 << 0),
1862 BLIT_DST_WIDTH_SHRINK = (1 << 1),
1863 BLIT_SRC_HEIGHT_SHRINK = (1 << 2),
1864 BLIT_DST_HEIGHT_SHRINK = (1 << 3),
1865 };
1866
1867 /* Try to blit. If the surface parameters exceed the size allowed by hardware,
1868 * then enum blit_shrink_status will be returned. If BLIT_NO_SHRINK is
1869 * returned, then the blit was successful.
1870 */
1871 static enum blit_shrink_status
try_blorp_blit(struct blorp_batch * batch,struct blorp_params * params,struct blorp_blit_prog_key * key,struct blt_coords * coords)1872 try_blorp_blit(struct blorp_batch *batch,
1873 struct blorp_params *params,
1874 struct blorp_blit_prog_key *key,
1875 struct blt_coords *coords)
1876 {
1877 const struct intel_device_info *devinfo = batch->blorp->isl_dev->info;
1878
1879 if (params->dst.surf.usage & ISL_SURF_USAGE_DEPTH_BIT) {
1880 if (devinfo->ver >= 7) {
1881 /* We can render as depth on Gfx5 but there's no real advantage since
1882 * it doesn't support MSAA or HiZ. On Gfx4, we can't always render
1883 * to depth due to issues with depth buffers and mip-mapping. On
1884 * Gfx6, we can do everything but we have weird offsetting for HiZ
1885 * and stencil. It's easier to just render using the color pipe
1886 * on those platforms.
1887 */
1888 key->dst_usage = ISL_SURF_USAGE_DEPTH_BIT;
1889 } else {
1890 key->dst_usage = ISL_SURF_USAGE_RENDER_TARGET_BIT;
1891 }
1892 } else if (params->dst.surf.usage & ISL_SURF_USAGE_STENCIL_BIT) {
1893 assert(params->dst.surf.format == ISL_FORMAT_R8_UINT);
1894 if (devinfo->ver >= 9 && !(batch->flags & BLORP_BATCH_USE_COMPUTE)) {
1895 key->dst_usage = ISL_SURF_USAGE_STENCIL_BIT;
1896 } else {
1897 key->dst_usage = ISL_SURF_USAGE_RENDER_TARGET_BIT;
1898 }
1899 } else {
1900 key->dst_usage = ISL_SURF_USAGE_RENDER_TARGET_BIT;
1901 }
1902
1903 if (isl_format_has_sint_channel(params->src.view.format)) {
1904 key->texture_data_type = nir_type_int;
1905 } else if (isl_format_has_uint_channel(params->src.view.format)) {
1906 key->texture_data_type = nir_type_uint;
1907 } else {
1908 key->texture_data_type = nir_type_float;
1909 }
1910
1911 /* src_samples and dst_samples are the true sample counts */
1912 key->src_samples = params->src.surf.samples;
1913 key->dst_samples = params->dst.surf.samples;
1914
1915 key->tex_aux_usage = params->src.aux_usage;
1916
1917 /* src_layout and dst_layout indicate the true MSAA layout used by src and
1918 * dst.
1919 */
1920 key->src_layout = params->src.surf.msaa_layout;
1921 key->dst_layout = params->dst.surf.msaa_layout;
1922
1923 /* Round floating point values to nearest integer to avoid "off by one texel"
1924 * kind of errors when blitting.
1925 */
1926 params->x0 = params->wm_inputs.bounds_rect.x0 = round(coords->x.dst0);
1927 params->y0 = params->wm_inputs.bounds_rect.y0 = round(coords->y.dst0);
1928 params->x1 = params->wm_inputs.bounds_rect.x1 = round(coords->x.dst1);
1929 params->y1 = params->wm_inputs.bounds_rect.y1 = round(coords->y.dst1);
1930
1931 blorp_setup_coord_transform(¶ms->wm_inputs.coord_transform[0],
1932 coords->x.src0, coords->x.src1,
1933 coords->x.dst0, coords->x.dst1,
1934 coords->x.mirror);
1935 blorp_setup_coord_transform(¶ms->wm_inputs.coord_transform[1],
1936 coords->y.src0, coords->y.src1,
1937 coords->y.dst0, coords->y.dst1,
1938 coords->y.mirror);
1939
1940
1941 if (devinfo->ver == 4) {
1942 /* The MinLOD and MinimumArrayElement don't work properly for cube maps.
1943 * Convert them to a single slice on gfx4.
1944 */
1945 if (params->dst.surf.usage & ISL_SURF_USAGE_CUBE_BIT) {
1946 blorp_surf_convert_to_single_slice(batch->blorp->isl_dev, ¶ms->dst);
1947 key->need_dst_offset = true;
1948 }
1949
1950 if (params->src.surf.usage & ISL_SURF_USAGE_CUBE_BIT) {
1951 blorp_surf_convert_to_single_slice(batch->blorp->isl_dev, ¶ms->src);
1952 key->need_src_offset = true;
1953 }
1954 }
1955
1956 if (devinfo->ver > 6 &&
1957 !isl_surf_usage_is_depth_or_stencil(key->dst_usage) &&
1958 params->dst.surf.msaa_layout == ISL_MSAA_LAYOUT_INTERLEAVED) {
1959 assert(params->dst.surf.samples > 1);
1960
1961 /* We must expand the rectangle we send through the rendering pipeline,
1962 * to account for the fact that we are mapping the destination region as
1963 * single-sampled when it is in fact multisampled. We must also align
1964 * it to a multiple of the multisampling pattern, because the
1965 * differences between multisampled and single-sampled surface formats
1966 * will mean that pixels are scrambled within the multisampling pattern.
1967 * TODO: what if this makes the coordinates too large?
1968 *
1969 * Note: this only works if the destination surface uses the IMS layout.
1970 * If it's UMS, then we have no choice but to set up the rendering
1971 * pipeline as multisampled.
1972 */
1973 struct isl_extent2d px_size_sa =
1974 isl_get_interleaved_msaa_px_size_sa(params->dst.surf.samples);
1975 params->x0 = ROUND_DOWN_TO(params->x0, 2) * px_size_sa.width;
1976 params->y0 = ROUND_DOWN_TO(params->y0, 2) * px_size_sa.height;
1977 params->x1 = ALIGN(params->x1, 2) * px_size_sa.width;
1978 params->y1 = ALIGN(params->y1, 2) * px_size_sa.height;
1979
1980 blorp_surf_fake_interleaved_msaa(batch->blorp->isl_dev, ¶ms->dst);
1981
1982 key->use_kill = true;
1983 key->need_dst_offset = true;
1984 }
1985
1986 if (params->dst.surf.tiling == ISL_TILING_W &&
1987 key->dst_usage != ISL_SURF_USAGE_STENCIL_BIT) {
1988 /* We must modify the rectangle we send through the rendering pipeline
1989 * (and the size and x/y offset of the destination surface), to account
1990 * for the fact that we are mapping it as Y-tiled when it is in fact
1991 * W-tiled.
1992 *
1993 * Both Y tiling and W tiling can be understood as organizations of
1994 * 32-byte sub-tiles; within each 32-byte sub-tile, the layout of pixels
1995 * is different, but the layout of the 32-byte sub-tiles within the 4k
1996 * tile is the same (8 sub-tiles across by 16 sub-tiles down, in
1997 * column-major order). In Y tiling, the sub-tiles are 16 bytes wide
1998 * and 2 rows high; in W tiling, they are 8 bytes wide and 4 rows high.
1999 *
2000 * Therefore, to account for the layout differences within the 32-byte
2001 * sub-tiles, we must expand the rectangle so the X coordinates of its
2002 * edges are multiples of 8 (the W sub-tile width), and its Y
2003 * coordinates of its edges are multiples of 4 (the W sub-tile height).
2004 * Then we need to scale the X and Y coordinates of the rectangle to
2005 * account for the differences in aspect ratio between the Y and W
2006 * sub-tiles. We need to modify the layer width and height similarly.
2007 *
2008 * A correction needs to be applied when MSAA is in use: since
2009 * INTEL_MSAA_LAYOUT_IMS uses an interleaving pattern whose height is 4,
2010 * we need to align the Y coordinates to multiples of 8, so that when
2011 * they are divided by two they are still multiples of 4.
2012 *
2013 * Note: Since the x/y offset of the surface will be applied using the
2014 * SURFACE_STATE command packet, it will be invisible to the swizzling
2015 * code in the shader; therefore it needs to be in a multiple of the
2016 * 32-byte sub-tile size. Fortunately it is, since the sub-tile is 8
2017 * pixels wide and 4 pixels high (when viewed as a W-tiled stencil
2018 * buffer), and the miplevel alignment used for stencil buffers is 8
2019 * pixels horizontally and either 4 or 8 pixels vertically (see
2020 * intel_horizontal_texture_alignment_unit() and
2021 * intel_vertical_texture_alignment_unit()).
2022 *
2023 * Note: Also, since the SURFACE_STATE command packet can only apply
2024 * offsets that are multiples of 4 pixels horizontally and 2 pixels
2025 * vertically, it is important that the offsets will be multiples of
2026 * these sizes after they are converted into Y-tiled coordinates.
2027 * Fortunately they will be, since we know from above that the offsets
2028 * are a multiple of the 32-byte sub-tile size, and in Y-tiled
2029 * coordinates the sub-tile is 16 pixels wide and 2 pixels high.
2030 *
2031 * TODO: what if this makes the coordinates (or the texture size) too
2032 * large?
2033 */
2034 const unsigned x_align = 8;
2035 const unsigned y_align = params->dst.surf.samples != 0 ? 8 : 4;
2036 params->x0 = ROUND_DOWN_TO(params->x0, x_align) * 2;
2037 params->y0 = ROUND_DOWN_TO(params->y0, y_align) / 2;
2038 params->x1 = ALIGN(params->x1, x_align) * 2;
2039 params->y1 = ALIGN(params->y1, y_align) / 2;
2040
2041 /* Retile the surface to Y-tiled */
2042 blorp_surf_retile_w_to_y(batch->blorp->isl_dev, ¶ms->dst);
2043
2044 key->dst_tiled_w = true;
2045 key->use_kill = true;
2046 key->need_dst_offset = true;
2047
2048 if (params->dst.surf.samples > 1) {
2049 /* If the destination surface is a W-tiled multisampled stencil
2050 * buffer that we're mapping as Y tiled, then we need to arrange for
2051 * the WM program to run once per sample rather than once per pixel,
2052 * because the memory layout of related samples doesn't match between
2053 * W and Y tiling.
2054 */
2055 key->persample_msaa_dispatch = true;
2056 }
2057 }
2058
2059 if (devinfo->ver < 8 && params->src.surf.tiling == ISL_TILING_W) {
2060 /* On Haswell and earlier, we have to fake W-tiled sources as Y-tiled.
2061 * Broadwell adds support for sampling from stencil.
2062 *
2063 * See the comments above concerning x/y offset alignment for the
2064 * destination surface.
2065 *
2066 * TODO: what if this makes the texture size too large?
2067 */
2068 blorp_surf_retile_w_to_y(batch->blorp->isl_dev, ¶ms->src);
2069
2070 key->src_tiled_w = true;
2071 key->need_src_offset = true;
2072 }
2073
2074 /* tex_samples and rt_samples are the sample counts that are set up in
2075 * SURFACE_STATE.
2076 */
2077 key->tex_samples = params->src.surf.samples;
2078 key->rt_samples = params->dst.surf.samples;
2079
2080 /* tex_layout and rt_layout indicate the MSAA layout the GPU pipeline will
2081 * use to access the source and destination surfaces.
2082 */
2083 key->tex_layout = params->src.surf.msaa_layout;
2084 key->rt_layout = params->dst.surf.msaa_layout;
2085
2086 if (params->src.surf.samples > 0 && params->dst.surf.samples > 1) {
2087 /* We are blitting from a multisample buffer to a multisample buffer, so
2088 * we must preserve samples within a pixel. This means we have to
2089 * arrange for the WM program to run once per sample rather than once
2090 * per pixel.
2091 */
2092 key->persample_msaa_dispatch = true;
2093 }
2094
2095 params->num_samples = params->dst.surf.samples;
2096
2097 if ((key->filter == BLORP_FILTER_AVERAGE ||
2098 key->filter == BLORP_FILTER_BILINEAR) &&
2099 batch->blorp->isl_dev->info->ver <= 6) {
2100 /* Gfx4-5 don't support non-normalized texture coordinates */
2101 key->src_coords_normalized = true;
2102 params->wm_inputs.src_inv_size[0] =
2103 1.0f / u_minify(params->src.surf.logical_level0_px.width,
2104 params->src.view.base_level);
2105 params->wm_inputs.src_inv_size[1] =
2106 1.0f / u_minify(params->src.surf.logical_level0_px.height,
2107 params->src.view.base_level);
2108 }
2109
2110 if (isl_format_get_layout(params->dst.view.format)->bpb % 3 == 0) {
2111 /* We can't render to RGB formats natively because they aren't a
2112 * power-of-two size. Instead, we fake them by using a red format
2113 * with the same channel type and size and emitting shader code to
2114 * only write one channel at a time.
2115 */
2116 params->x0 *= 3;
2117 params->x1 *= 3;
2118
2119 /* If it happens to be sRGB, we need to force a conversion */
2120 if (params->dst.view.format == ISL_FORMAT_R8G8B8_UNORM_SRGB)
2121 key->dst_format = ISL_FORMAT_R8G8B8_UNORM_SRGB;
2122
2123 surf_fake_rgb_with_red(batch->blorp->isl_dev, ¶ms->dst);
2124
2125 key->dst_rgb = true;
2126 key->need_dst_offset = true;
2127 } else if (isl_format_is_rgbx(params->dst.view.format)) {
2128 /* We can handle RGBX formats easily enough by treating them as RGBA */
2129 params->dst.view.format =
2130 isl_format_rgbx_to_rgba(params->dst.view.format);
2131 } else if (params->dst.view.format == ISL_FORMAT_R24_UNORM_X8_TYPELESS &&
2132 key->dst_usage != ISL_SURF_USAGE_DEPTH_BIT) {
2133 key->dst_format = params->dst.view.format;
2134 params->dst.view.format = ISL_FORMAT_R32_UINT;
2135 } else if (!isl_format_supports_rendering(devinfo, params->dst.view.format)
2136 && params->dst.view.format == ISL_FORMAT_A4B4G4R4_UNORM) {
2137 params->dst.view.swizzle =
2138 isl_swizzle_compose(params->dst.view.swizzle,
2139 ISL_SWIZZLE(ALPHA, RED, GREEN, BLUE));
2140 params->dst.view.format = ISL_FORMAT_B4G4R4A4_UNORM;
2141 } else if (params->dst.view.format == ISL_FORMAT_L8_UNORM_SRGB) {
2142 key->dst_format = params->dst.view.format;
2143 params->dst.view.format = ISL_FORMAT_R8_UNORM;
2144 } else if (params->dst.view.format == ISL_FORMAT_R9G9B9E5_SHAREDEXP) {
2145 key->dst_format = params->dst.view.format;
2146 params->dst.view.format = ISL_FORMAT_R32_UINT;
2147 }
2148
2149 if (devinfo->verx10 <= 70 &&
2150 !isl_swizzle_is_identity(params->src.view.swizzle)) {
2151 key->src_swizzle = params->src.view.swizzle;
2152 params->src.view.swizzle = ISL_SWIZZLE_IDENTITY;
2153 } else {
2154 key->src_swizzle = ISL_SWIZZLE_IDENTITY;
2155 }
2156
2157 if (!isl_swizzle_supports_rendering(devinfo, params->dst.view.swizzle)) {
2158 key->dst_swizzle = params->dst.view.swizzle;
2159 params->dst.view.swizzle = ISL_SWIZZLE_IDENTITY;
2160 } else {
2161 key->dst_swizzle = ISL_SWIZZLE_IDENTITY;
2162 }
2163
2164 if (params->src.tile_x_sa || params->src.tile_y_sa) {
2165 assert(key->need_src_offset);
2166 surf_get_intratile_offset_px(¶ms->src,
2167 ¶ms->wm_inputs.src_offset.x,
2168 ¶ms->wm_inputs.src_offset.y);
2169 }
2170
2171 if (params->dst.tile_x_sa || params->dst.tile_y_sa) {
2172 assert(key->need_dst_offset);
2173 surf_get_intratile_offset_px(¶ms->dst,
2174 ¶ms->wm_inputs.dst_offset.x,
2175 ¶ms->wm_inputs.dst_offset.y);
2176 params->x0 += params->wm_inputs.dst_offset.x;
2177 params->y0 += params->wm_inputs.dst_offset.y;
2178 params->x1 += params->wm_inputs.dst_offset.x;
2179 params->y1 += params->wm_inputs.dst_offset.y;
2180 }
2181
2182 /* For some texture types, we need to pass the layer through the sampler. */
2183 params->wm_inputs.src_z = params->src.z_offset;
2184
2185 const bool compute =
2186 key->base.shader_pipeline == BLORP_SHADER_PIPELINE_COMPUTE;
2187 if (compute) {
2188 key->local_y = blorp_get_cs_local_y(params);
2189
2190 unsigned workgroup_width = 16 / key->local_y;
2191 unsigned workgroup_height = key->local_y;
2192
2193 /* If the rectangle being drawn isn't an exact multiple of the
2194 * workgroup size, we'll get extra invocations that should not
2195 * perform blits. We need to set use_kill to bounds check and
2196 * prevent those invocations from blitting.
2197 */
2198 if ((params->x0 % workgroup_width) != 0 ||
2199 (params->x1 % workgroup_width) != 0 ||
2200 (params->y0 % workgroup_height) != 0 ||
2201 (params->y1 % workgroup_height) != 0)
2202 key->use_kill = true;
2203 }
2204
2205 if (compute) {
2206 if (!blorp_get_blit_kernel_cs(batch, params, key))
2207 return 0;
2208 } else {
2209 if (!blorp_get_blit_kernel_fs(batch, params, key))
2210 return 0;
2211
2212 if (!blorp_ensure_sf_program(batch, params))
2213 return 0;
2214 }
2215
2216 unsigned result = 0;
2217 unsigned max_src_surface_size = get_max_surface_size(devinfo, ¶ms->src);
2218 if (params->src.surf.logical_level0_px.width > max_src_surface_size)
2219 result |= BLIT_SRC_WIDTH_SHRINK;
2220 if (params->src.surf.logical_level0_px.height > max_src_surface_size)
2221 result |= BLIT_SRC_HEIGHT_SHRINK;
2222
2223 unsigned max_dst_surface_size = get_max_surface_size(devinfo, ¶ms->dst);
2224 if (params->dst.surf.logical_level0_px.width > max_dst_surface_size)
2225 result |= BLIT_DST_WIDTH_SHRINK;
2226 if (params->dst.surf.logical_level0_px.height > max_dst_surface_size)
2227 result |= BLIT_DST_HEIGHT_SHRINK;
2228
2229 if (result == 0) {
2230 if (key->dst_usage == ISL_SURF_USAGE_DEPTH_BIT) {
2231 params->depth = params->dst;
2232 memset(¶ms->dst, 0, sizeof(params->dst));
2233 } else if (key->dst_usage == ISL_SURF_USAGE_STENCIL_BIT) {
2234 params->stencil = params->dst;
2235 params->stencil_mask = 0xff;
2236 memset(¶ms->dst, 0, sizeof(params->dst));
2237 }
2238
2239 batch->blorp->exec(batch, params);
2240 }
2241
2242 return result;
2243 }
2244
2245 /* Adjust split blit source coordinates for the current destination
2246 * coordinates.
2247 */
2248 static void
adjust_split_source_coords(const struct blt_axis * orig,struct blt_axis * split_coords,double scale)2249 adjust_split_source_coords(const struct blt_axis *orig,
2250 struct blt_axis *split_coords,
2251 double scale)
2252 {
2253 /* When scale is greater than 0, then we are growing from the start, so
2254 * src0 uses delta0, and src1 uses delta1. When scale is less than 0, the
2255 * source range shrinks from the end. In that case src0 is adjusted by
2256 * delta1, and src1 is adjusted by delta0.
2257 */
2258 double delta0 = scale * (split_coords->dst0 - orig->dst0);
2259 double delta1 = scale * (split_coords->dst1 - orig->dst1);
2260 split_coords->src0 = orig->src0 + (scale >= 0.0 ? delta0 : delta1);
2261 split_coords->src1 = orig->src1 + (scale >= 0.0 ? delta1 : delta0);
2262 }
2263
2264 static struct isl_extent2d
get_px_size_sa(const struct isl_surf * surf)2265 get_px_size_sa(const struct isl_surf *surf)
2266 {
2267 static const struct isl_extent2d one_to_one = { .w = 1, .h = 1 };
2268
2269 if (surf->msaa_layout != ISL_MSAA_LAYOUT_INTERLEAVED)
2270 return one_to_one;
2271 else
2272 return isl_get_interleaved_msaa_px_size_sa(surf->samples);
2273 }
2274
2275 static void
shrink_surface_params(const struct isl_device * dev,struct blorp_surface_info * info,double * x0,double * x1,double * y0,double * y1)2276 shrink_surface_params(const struct isl_device *dev,
2277 struct blorp_surface_info *info,
2278 double *x0, double *x1, double *y0, double *y1)
2279 {
2280 uint64_t offset_B;
2281 uint32_t x_offset_sa, y_offset_sa, size;
2282 struct isl_extent2d px_size_sa;
2283 int adjust;
2284
2285 blorp_surf_convert_to_single_slice(dev, info);
2286
2287 px_size_sa = get_px_size_sa(&info->surf);
2288
2289 /* Because this gets called after we lower compressed images, the tile
2290 * offsets may be non-zero and we need to incorporate them in our
2291 * calculations.
2292 */
2293 x_offset_sa = (uint32_t)*x0 * px_size_sa.w + info->tile_x_sa;
2294 y_offset_sa = (uint32_t)*y0 * px_size_sa.h + info->tile_y_sa;
2295 uint32_t tile_z_sa, tile_a;
2296 isl_tiling_get_intratile_offset_sa(info->surf.tiling, info->surf.dim,
2297 info->surf.msaa_layout,
2298 info->surf.format, info->surf.samples,
2299 info->surf.row_pitch_B,
2300 info->surf.array_pitch_el_rows,
2301 x_offset_sa, y_offset_sa, 0, 0,
2302 &offset_B,
2303 &info->tile_x_sa, &info->tile_y_sa,
2304 &tile_z_sa, &tile_a);
2305 assert(tile_z_sa == 0 && tile_a == 0);
2306
2307 info->addr.offset += offset_B;
2308
2309 adjust = (int)info->tile_x_sa / px_size_sa.w - (int)*x0;
2310 *x0 += adjust;
2311 *x1 += adjust;
2312 info->tile_x_sa = 0;
2313
2314 adjust = (int)info->tile_y_sa / px_size_sa.h - (int)*y0;
2315 *y0 += adjust;
2316 *y1 += adjust;
2317 info->tile_y_sa = 0;
2318
2319 size = MIN2((uint32_t)ceil(*x1), info->surf.logical_level0_px.width);
2320 info->surf.logical_level0_px.width = size;
2321 info->surf.phys_level0_sa.width = size * px_size_sa.w;
2322
2323 size = MIN2((uint32_t)ceil(*y1), info->surf.logical_level0_px.height);
2324 info->surf.logical_level0_px.height = size;
2325 info->surf.phys_level0_sa.height = size * px_size_sa.h;
2326 }
2327
2328 static void
do_blorp_blit(struct blorp_batch * batch,const struct blorp_params * orig_params,struct blorp_blit_prog_key * key,const struct blt_coords * orig)2329 do_blorp_blit(struct blorp_batch *batch,
2330 const struct blorp_params *orig_params,
2331 struct blorp_blit_prog_key *key,
2332 const struct blt_coords *orig)
2333 {
2334 struct blorp_params params;
2335 struct blt_coords blit_coords;
2336 struct blt_coords split_coords = *orig;
2337 double w = orig->x.dst1 - orig->x.dst0;
2338 double h = orig->y.dst1 - orig->y.dst0;
2339 double x_scale = (orig->x.src1 - orig->x.src0) / w;
2340 double y_scale = (orig->y.src1 - orig->y.src0) / h;
2341 if (orig->x.mirror)
2342 x_scale = -x_scale;
2343 if (orig->y.mirror)
2344 y_scale = -y_scale;
2345
2346 enum blit_shrink_status shrink = BLIT_NO_SHRINK;
2347 if (split_blorp_blit_debug) {
2348 if (can_shrink_surface(&orig_params->src))
2349 shrink |= BLIT_SRC_WIDTH_SHRINK | BLIT_SRC_HEIGHT_SHRINK;
2350 if (can_shrink_surface(&orig_params->dst))
2351 shrink |= BLIT_DST_WIDTH_SHRINK | BLIT_DST_HEIGHT_SHRINK;
2352 }
2353
2354 bool x_done, y_done;
2355 do {
2356 params = *orig_params;
2357 blit_coords = split_coords;
2358
2359 if (shrink & (BLIT_SRC_WIDTH_SHRINK | BLIT_SRC_HEIGHT_SHRINK)) {
2360 shrink_surface_params(batch->blorp->isl_dev, ¶ms.src,
2361 &blit_coords.x.src0, &blit_coords.x.src1,
2362 &blit_coords.y.src0, &blit_coords.y.src1);
2363 key->need_src_offset = false;
2364 }
2365
2366 if (shrink & (BLIT_DST_WIDTH_SHRINK | BLIT_DST_HEIGHT_SHRINK)) {
2367 shrink_surface_params(batch->blorp->isl_dev, ¶ms.dst,
2368 &blit_coords.x.dst0, &blit_coords.x.dst1,
2369 &blit_coords.y.dst0, &blit_coords.y.dst1);
2370 key->need_dst_offset = false;
2371 }
2372
2373 enum blit_shrink_status result =
2374 try_blorp_blit(batch, ¶ms, key, &blit_coords);
2375
2376 if (result & (BLIT_SRC_WIDTH_SHRINK | BLIT_SRC_HEIGHT_SHRINK))
2377 assert(can_shrink_surface(&orig_params->src));
2378
2379 if (result & (BLIT_DST_WIDTH_SHRINK | BLIT_DST_HEIGHT_SHRINK))
2380 assert(can_shrink_surface(&orig_params->dst));
2381
2382 if (result & (BLIT_SRC_WIDTH_SHRINK | BLIT_DST_WIDTH_SHRINK)) {
2383 w /= 2.0;
2384 assert(w >= 1.0);
2385 split_coords.x.dst1 = MIN2(split_coords.x.dst0 + w, orig->x.dst1);
2386 adjust_split_source_coords(&orig->x, &split_coords.x, x_scale);
2387 }
2388 if (result & (BLIT_SRC_HEIGHT_SHRINK | BLIT_DST_HEIGHT_SHRINK)) {
2389 h /= 2.0;
2390 assert(h >= 1.0);
2391 split_coords.y.dst1 = MIN2(split_coords.y.dst0 + h, orig->y.dst1);
2392 adjust_split_source_coords(&orig->y, &split_coords.y, y_scale);
2393 }
2394
2395 if (result) {
2396 /* We may get less bits set on result than we had already, so make
2397 * sure we remember all the ways in which a resize is required.
2398 */
2399 shrink |= result;
2400 continue;
2401 }
2402
2403 y_done = (orig->y.dst1 - split_coords.y.dst1 < 0.5);
2404 x_done = y_done && (orig->x.dst1 - split_coords.x.dst1 < 0.5);
2405 if (x_done) {
2406 break;
2407 } else if (y_done) {
2408 split_coords.x.dst0 += w;
2409 split_coords.x.dst1 = MIN2(split_coords.x.dst0 + w, orig->x.dst1);
2410 split_coords.y.dst0 = orig->y.dst0;
2411 split_coords.y.dst1 = MIN2(split_coords.y.dst0 + h, orig->y.dst1);
2412 adjust_split_source_coords(&orig->x, &split_coords.x, x_scale);
2413 } else {
2414 split_coords.y.dst0 += h;
2415 split_coords.y.dst1 = MIN2(split_coords.y.dst0 + h, orig->y.dst1);
2416 adjust_split_source_coords(&orig->y, &split_coords.y, y_scale);
2417 }
2418 } while (true);
2419 }
2420
2421 bool
blorp_blit_supports_compute(struct blorp_context * blorp,const struct isl_surf * src_surf,const struct isl_surf * dst_surf,enum isl_aux_usage dst_aux_usage)2422 blorp_blit_supports_compute(struct blorp_context *blorp,
2423 const struct isl_surf *src_surf,
2424 const struct isl_surf *dst_surf,
2425 enum isl_aux_usage dst_aux_usage)
2426 {
2427 /* Our compiler doesn't currently support typed image writes with MSAA.
2428 * Also, our BLORP compute shaders don't handle multisampling cases.
2429 */
2430 if (dst_surf->samples > 1 || src_surf->samples > 1)
2431 return false;
2432
2433 if (blorp->isl_dev->info->ver >= 12) {
2434 return dst_aux_usage == ISL_AUX_USAGE_FCV_CCS_E ||
2435 dst_aux_usage == ISL_AUX_USAGE_CCS_E ||
2436 dst_aux_usage == ISL_AUX_USAGE_NONE;
2437 } else if (blorp->isl_dev->info->ver >= 7) {
2438 return dst_aux_usage == ISL_AUX_USAGE_NONE;
2439 } else {
2440 /* No compute shader support */
2441 return false;
2442 }
2443 }
2444
2445 bool
blorp_blitter_supports_aux(const struct intel_device_info * devinfo,enum isl_aux_usage aux_usage)2446 blorp_blitter_supports_aux(const struct intel_device_info *devinfo,
2447 enum isl_aux_usage aux_usage)
2448 {
2449 switch (aux_usage) {
2450 case ISL_AUX_USAGE_NONE:
2451 return true;
2452 case ISL_AUX_USAGE_CCS_E:
2453 case ISL_AUX_USAGE_FCV_CCS_E:
2454 case ISL_AUX_USAGE_STC_CCS:
2455 return devinfo->verx10 >= 125;
2456 default:
2457 return false;
2458 }
2459 }
2460
2461 bool
blorp_copy_supports_blitter(struct blorp_context * blorp,const struct isl_surf * src_surf,const struct isl_surf * dst_surf,enum isl_aux_usage src_aux_usage,enum isl_aux_usage dst_aux_usage)2462 blorp_copy_supports_blitter(struct blorp_context *blorp,
2463 const struct isl_surf *src_surf,
2464 const struct isl_surf *dst_surf,
2465 enum isl_aux_usage src_aux_usage,
2466 enum isl_aux_usage dst_aux_usage)
2467 {
2468 const struct intel_device_info *devinfo = blorp->isl_dev->info;
2469
2470 if (devinfo->ver < 12)
2471 return false;
2472
2473 if (dst_surf->samples > 1 || src_surf->samples > 1)
2474 return false;
2475
2476 if (!blorp_blitter_supports_aux(devinfo, dst_aux_usage))
2477 return false;
2478
2479 if (!blorp_blitter_supports_aux(devinfo, src_aux_usage))
2480 return false;
2481
2482 const struct isl_format_layout *fmtl =
2483 isl_format_get_layout(dst_surf->format);
2484
2485 if (fmtl->bpb == 96) {
2486 /* XY_BLOCK_COPY_BLT mentions it doesn't support clear colors for 96bpp
2487 * formats, but none of them support CCS anyway, so it's a moot point.
2488 */
2489 assert(src_aux_usage == ISL_AUX_USAGE_NONE);
2490 assert(dst_aux_usage == ISL_AUX_USAGE_NONE);
2491
2492 /* We can only support linear mode for 96bpp. */
2493 if (src_surf->tiling != ISL_TILING_LINEAR ||
2494 dst_surf->tiling != ISL_TILING_LINEAR)
2495 return false;
2496 }
2497
2498 return true;
2499 }
2500
2501 void
blorp_blit(struct blorp_batch * batch,const struct blorp_surf * src_surf,unsigned src_level,float src_layer,enum isl_format src_format,struct isl_swizzle src_swizzle,const struct blorp_surf * dst_surf,unsigned dst_level,unsigned dst_layer,enum isl_format dst_format,struct isl_swizzle dst_swizzle,float src_x0,float src_y0,float src_x1,float src_y1,float dst_x0,float dst_y0,float dst_x1,float dst_y1,enum blorp_filter filter,bool mirror_x,bool mirror_y)2502 blorp_blit(struct blorp_batch *batch,
2503 const struct blorp_surf *src_surf,
2504 unsigned src_level, float src_layer,
2505 enum isl_format src_format, struct isl_swizzle src_swizzle,
2506 const struct blorp_surf *dst_surf,
2507 unsigned dst_level, unsigned dst_layer,
2508 enum isl_format dst_format, struct isl_swizzle dst_swizzle,
2509 float src_x0, float src_y0,
2510 float src_x1, float src_y1,
2511 float dst_x0, float dst_y0,
2512 float dst_x1, float dst_y1,
2513 enum blorp_filter filter,
2514 bool mirror_x, bool mirror_y)
2515 {
2516 struct blorp_params params;
2517 blorp_params_init(¶ms);
2518 params.op = BLORP_OP_BLIT;
2519 const bool compute = batch->flags & BLORP_BATCH_USE_COMPUTE;
2520 if (compute) {
2521 assert(blorp_blit_supports_compute(batch->blorp,
2522 src_surf->surf, dst_surf->surf,
2523 dst_surf->aux_usage));
2524 }
2525
2526 /* We cannot handle combined depth and stencil. */
2527 if (src_surf->surf->usage & ISL_SURF_USAGE_STENCIL_BIT)
2528 assert(src_surf->surf->format == ISL_FORMAT_R8_UINT);
2529 if (dst_surf->surf->usage & ISL_SURF_USAGE_STENCIL_BIT)
2530 assert(dst_surf->surf->format == ISL_FORMAT_R8_UINT);
2531
2532 if (dst_surf->surf->usage & ISL_SURF_USAGE_STENCIL_BIT) {
2533 assert(src_surf->surf->usage & ISL_SURF_USAGE_STENCIL_BIT);
2534 /* Prior to Broadwell, we can't render to R8_UINT */
2535 if (batch->blorp->isl_dev->info->ver < 8) {
2536 src_format = ISL_FORMAT_R8_UNORM;
2537 dst_format = ISL_FORMAT_R8_UNORM;
2538 }
2539 }
2540
2541 blorp_surface_info_init(batch, ¶ms.src, src_surf, src_level,
2542 src_layer, src_format, false);
2543 blorp_surface_info_init(batch, ¶ms.dst, dst_surf, dst_level,
2544 dst_layer, dst_format, true);
2545
2546 params.src.view.swizzle = src_swizzle;
2547 params.dst.view.swizzle = dst_swizzle;
2548
2549 const struct isl_format_layout *src_fmtl =
2550 isl_format_get_layout(params.src.view.format);
2551
2552 struct blorp_blit_prog_key key = {
2553 .base = BLORP_BASE_KEY_INIT(BLORP_SHADER_TYPE_BLIT),
2554 .base.shader_pipeline = compute ? BLORP_SHADER_PIPELINE_COMPUTE :
2555 BLORP_SHADER_PIPELINE_RENDER,
2556 .filter = filter,
2557 .sint32_to_uint = src_fmtl->channels.r.bits == 32 &&
2558 isl_format_has_sint_channel(params.src.view.format) &&
2559 isl_format_has_uint_channel(params.dst.view.format),
2560 .uint32_to_sint = src_fmtl->channels.r.bits == 32 &&
2561 isl_format_has_uint_channel(params.src.view.format) &&
2562 isl_format_has_sint_channel(params.dst.view.format),
2563 };
2564
2565 params.shader_type = key.base.shader_type;
2566 params.shader_pipeline = key.base.shader_pipeline;
2567
2568 /* Scaling factors used for bilinear filtering in multisample scaled
2569 * blits.
2570 */
2571 if (params.src.surf.samples == 16)
2572 key.x_scale = 4.0f;
2573 else
2574 key.x_scale = 2.0f;
2575 key.y_scale = params.src.surf.samples / key.x_scale;
2576
2577 params.wm_inputs.rect_grid.x1 =
2578 u_minify(params.src.surf.logical_level0_px.width, src_level) *
2579 key.x_scale - 1.0f;
2580 params.wm_inputs.rect_grid.y1 =
2581 u_minify(params.src.surf.logical_level0_px.height, src_level) *
2582 key.y_scale - 1.0f;
2583
2584 struct blt_coords coords = {
2585 .x = {
2586 .src0 = src_x0,
2587 .src1 = src_x1,
2588 .dst0 = dst_x0,
2589 .dst1 = dst_x1,
2590 .mirror = mirror_x
2591 },
2592 .y = {
2593 .src0 = src_y0,
2594 .src1 = src_y1,
2595 .dst0 = dst_y0,
2596 .dst1 = dst_y1,
2597 .mirror = mirror_y
2598 }
2599 };
2600
2601 do_blorp_blit(batch, ¶ms, &key, &coords);
2602 }
2603
2604 static enum isl_format
get_copy_format_for_bpb(const struct isl_device * isl_dev,unsigned bpb)2605 get_copy_format_for_bpb(const struct isl_device *isl_dev, unsigned bpb)
2606 {
2607 /* The choice of UNORM and UINT formats is very intentional here. Most
2608 * of the time, we want to use a UINT format to avoid any rounding error
2609 * in the blit. For stencil blits, R8_UINT is required by the hardware.
2610 * (It's the only format allowed in conjunction with W-tiling.) Also we
2611 * intentionally use the 4-channel formats whenever we can. This is so
2612 * that, when we do a RGB <-> RGBX copy, the two formats will line up
2613 * even though one of them is 3/4 the size of the other. The choice of
2614 * UNORM vs. UINT is also very intentional because we don't have 8 or
2615 * 16-bit RGB UINT formats until Sky Lake so we have to use UNORM there.
2616 * Fortunately, the only time we should ever use two different formats in
2617 * the table below is for RGB -> RGBA blits and so we will never have any
2618 * UNORM/UINT mismatch.
2619 */
2620 if (ISL_GFX_VER(isl_dev) >= 9) {
2621 switch (bpb) {
2622 case 8: return ISL_FORMAT_R8_UINT;
2623 case 16: return ISL_FORMAT_R8G8_UINT;
2624 case 24: return ISL_FORMAT_R8G8B8_UINT;
2625 case 32: return ISL_FORMAT_R8G8B8A8_UINT;
2626 case 48: return ISL_FORMAT_R16G16B16_UINT;
2627 case 64: return ISL_FORMAT_R16G16B16A16_UINT;
2628 case 96: return ISL_FORMAT_R32G32B32_UINT;
2629 case 128:return ISL_FORMAT_R32G32B32A32_UINT;
2630 default:
2631 unreachable("Unknown format bpb");
2632 }
2633 } else {
2634 switch (bpb) {
2635 case 8: return ISL_FORMAT_R8_UINT;
2636 case 16: return ISL_FORMAT_R8G8_UINT;
2637 case 24: return ISL_FORMAT_R8G8B8_UNORM;
2638 case 32: return ISL_FORMAT_R8G8B8A8_UNORM;
2639 case 48: return ISL_FORMAT_R16G16B16_UNORM;
2640 case 64: return ISL_FORMAT_R16G16B16A16_UNORM;
2641 case 96: return ISL_FORMAT_R32G32B32_UINT;
2642 case 128:return ISL_FORMAT_R32G32B32A32_UINT;
2643 default:
2644 unreachable("Unknown format bpb");
2645 }
2646 }
2647 }
2648
2649 /** Returns a UINT format that is CCS-compatible with the given format
2650 *
2651 * The PRM's say absolutely nothing about how render compression works. The
2652 * only thing they provide is a list of formats on which it is and is not
2653 * supported. Empirical testing indicates that the compression is only based
2654 * on the bit-layout of the format and the channel encoding doesn't matter.
2655 * So, while texture views don't work in general, you can create a view as
2656 * long as the bit-layout of the formats are the same.
2657 *
2658 * Fortunately, for every render compression capable format, the UINT format
2659 * with the same bit layout also supports render compression. This means that
2660 * we only need to handle UINT formats for copy operations. In order to do
2661 * copies between formats with different bit layouts, we attach both with a
2662 * UINT format and use bit_cast_color() to generate code to do the bit-cast
2663 * operation between the two bit layouts.
2664 */
2665 static enum isl_format
get_ccs_compatible_uint_format(const struct isl_format_layout * fmtl)2666 get_ccs_compatible_uint_format(const struct isl_format_layout *fmtl)
2667 {
2668 switch (fmtl->format) {
2669 case ISL_FORMAT_R32G32B32A32_FLOAT:
2670 case ISL_FORMAT_R32G32B32A32_SINT:
2671 case ISL_FORMAT_R32G32B32A32_UINT:
2672 case ISL_FORMAT_R32G32B32A32_UNORM:
2673 case ISL_FORMAT_R32G32B32A32_SNORM:
2674 case ISL_FORMAT_R32G32B32X32_FLOAT:
2675 return ISL_FORMAT_R32G32B32A32_UINT;
2676
2677 case ISL_FORMAT_R16G16B16A16_UNORM:
2678 case ISL_FORMAT_R16G16B16A16_SNORM:
2679 case ISL_FORMAT_R16G16B16A16_SINT:
2680 case ISL_FORMAT_R16G16B16A16_UINT:
2681 case ISL_FORMAT_R16G16B16A16_FLOAT:
2682 case ISL_FORMAT_R16G16B16X16_UNORM:
2683 case ISL_FORMAT_R16G16B16X16_FLOAT:
2684 return ISL_FORMAT_R16G16B16A16_UINT;
2685
2686 case ISL_FORMAT_R32G32_FLOAT:
2687 case ISL_FORMAT_R32G32_SINT:
2688 case ISL_FORMAT_R32G32_UINT:
2689 case ISL_FORMAT_R32G32_UNORM:
2690 case ISL_FORMAT_R32G32_SNORM:
2691 return ISL_FORMAT_R32G32_UINT;
2692
2693 case ISL_FORMAT_B8G8R8A8_UNORM:
2694 case ISL_FORMAT_B8G8R8A8_UNORM_SRGB:
2695 case ISL_FORMAT_R8G8B8A8_UNORM:
2696 case ISL_FORMAT_R8G8B8A8_UNORM_SRGB:
2697 case ISL_FORMAT_R8G8B8A8_SNORM:
2698 case ISL_FORMAT_R8G8B8A8_SINT:
2699 case ISL_FORMAT_R8G8B8A8_UINT:
2700 case ISL_FORMAT_B8G8R8X8_UNORM:
2701 case ISL_FORMAT_B8G8R8X8_UNORM_SRGB:
2702 case ISL_FORMAT_R8G8B8X8_UNORM:
2703 case ISL_FORMAT_R8G8B8X8_UNORM_SRGB:
2704 return ISL_FORMAT_R8G8B8A8_UINT;
2705
2706 case ISL_FORMAT_R16G16_UNORM:
2707 case ISL_FORMAT_R16G16_SNORM:
2708 case ISL_FORMAT_R16G16_SINT:
2709 case ISL_FORMAT_R16G16_UINT:
2710 case ISL_FORMAT_R16G16_FLOAT:
2711 return ISL_FORMAT_R16G16_UINT;
2712
2713 case ISL_FORMAT_R32_SINT:
2714 case ISL_FORMAT_R32_UINT:
2715 case ISL_FORMAT_R32_FLOAT:
2716 case ISL_FORMAT_R32_UNORM:
2717 case ISL_FORMAT_R32_SNORM:
2718 return ISL_FORMAT_R32_UINT;
2719
2720 case ISL_FORMAT_R11G11B10_FLOAT:
2721 return ISL_FORMAT_R8G8B8A8_UINT;
2722
2723 case ISL_FORMAT_B10G10R10A2_UNORM:
2724 case ISL_FORMAT_B10G10R10A2_UNORM_SRGB:
2725 case ISL_FORMAT_R10G10B10A2_UNORM:
2726 case ISL_FORMAT_R10G10B10A2_UINT:
2727 return ISL_FORMAT_R10G10B10A2_UINT;
2728
2729 case ISL_FORMAT_R16_SNORM:
2730 case ISL_FORMAT_R16_SINT:
2731 case ISL_FORMAT_R16_FLOAT:
2732 return ISL_FORMAT_R16_UINT;
2733
2734 case ISL_FORMAT_R8G8_SNORM:
2735 case ISL_FORMAT_R8G8_SINT:
2736 return ISL_FORMAT_R8G8_UINT;
2737
2738 case ISL_FORMAT_YCRCB_NORMAL:
2739 case ISL_FORMAT_YCRCB_SWAPY:
2740 case ISL_FORMAT_YCRCB_SWAPUV:
2741 case ISL_FORMAT_YCRCB_SWAPUVY:
2742 /* Tiger Lake starts claiming CCS_E support for certain YCRCB formats.
2743 * BLORP chooses to take the CCS-compatible format path whenever ISL
2744 * claims CCS_E support on a format, not when CCS_E is actually used.
2745 * Therefore, if these formats are going to be used with BLORP, we need
2746 * a CCS-compatible format. R8G8_UINT seems as good as any.
2747 */
2748 return ISL_FORMAT_R8G8_UINT;
2749
2750 case ISL_FORMAT_R8_SNORM:
2751 case ISL_FORMAT_R8_SINT:
2752 return ISL_FORMAT_R8_UINT;
2753
2754 default:
2755 unreachable("Not a compressible format");
2756 }
2757 }
2758
2759 enum isl_format
blorp_copy_get_color_format(const struct isl_device * isl_dev,enum isl_format surf_format)2760 blorp_copy_get_color_format(const struct isl_device *isl_dev,
2761 enum isl_format surf_format)
2762 {
2763 const struct isl_format_layout *fmtl = isl_format_get_layout(surf_format);
2764
2765 if (ISL_GFX_VER(isl_dev) >= 9 && ISL_GFX_VER(isl_dev) <= 12 &&
2766 fmtl->colorspace != ISL_COLORSPACE_YUV &&
2767 fmtl->uniform_channel_type != ISL_SNORM &&
2768 fmtl->uniform_channel_type != ISL_UFLOAT &&
2769 fmtl->uniform_channel_type != ISL_SFLOAT &&
2770 fmtl->uniform_channel_type != ISL_SINT &&
2771 surf_format != ISL_FORMAT_R16G16B16A16_UNORM /* TODO */ &&
2772 isl_format_supports_rendering(isl_dev->info, surf_format)) {
2773 /* On gfx9-12, avoid format re-interpretation in order to support
2774 * non-zero clear colors when copying. On gfx12, also do this in order
2775 * to properly interpret the clear color of imported dmabuf surfaces.
2776 */
2777 return surf_format;
2778 } else if (ISL_GFX_VER(isl_dev) <= 12 &&
2779 isl_format_supports_ccs_e(isl_dev->info, surf_format)) {
2780 /* On gfx9-12, choose a copy format that maintains compatibility with
2781 * CCS_E. Although format reinterpretation doesn't affect compression
2782 * support while rendering on gfx12, the sampler does have reduced
2783 * support for compression when the bits-per-channel changes.
2784 */
2785 return get_ccs_compatible_uint_format(fmtl);
2786 } else {
2787 return get_copy_format_for_bpb(isl_dev, fmtl->bpb);
2788 }
2789 }
2790
2791 void
blorp_surf_convert_to_uncompressed(const struct isl_device * isl_dev,struct blorp_surface_info * info,uint32_t * x,uint32_t * y,uint32_t * width,uint32_t * height)2792 blorp_surf_convert_to_uncompressed(const struct isl_device *isl_dev,
2793 struct blorp_surface_info *info,
2794 uint32_t *x, uint32_t *y,
2795 uint32_t *width, uint32_t *height)
2796 {
2797 const struct isl_format_layout *fmtl =
2798 isl_format_get_layout(info->surf.format);
2799
2800 assert(fmtl->bw > 1 || fmtl->bh > 1);
2801
2802 /* This should be the first modification made to the surface */
2803 assert(info->tile_x_sa == 0 && info->tile_y_sa == 0);
2804
2805 if (width && height) {
2806 ASSERTED const uint32_t level_width =
2807 u_minify(info->surf.logical_level0_px.width, info->view.base_level);
2808 ASSERTED const uint32_t level_height =
2809 u_minify(info->surf.logical_level0_px.height, info->view.base_level);
2810 assert(*width % fmtl->bw == 0 || *x + *width == level_width);
2811 assert(*height % fmtl->bh == 0 || *y + *height == level_height);
2812 *width = DIV_ROUND_UP(*width, fmtl->bw);
2813 *height = DIV_ROUND_UP(*height, fmtl->bh);
2814 }
2815
2816 if (x && y) {
2817 assert(*x % fmtl->bw == 0);
2818 assert(*y % fmtl->bh == 0);
2819 *x /= fmtl->bw;
2820 *y /= fmtl->bh;
2821 }
2822
2823 /* We only want one level and slice */
2824 info->view.levels = 1;
2825 info->view.array_len = 1;
2826
2827 if (info->surf.dim == ISL_SURF_DIM_3D) {
2828 /* Roll the Z offset into the image view */
2829 info->view.base_array_layer += info->z_offset;
2830 info->z_offset = 0;
2831 }
2832
2833 uint64_t offset_B;
2834 ASSERTED bool ok =
2835 isl_surf_get_uncompressed_surf(isl_dev, &info->surf, &info->view,
2836 &info->surf, &info->view, &offset_B,
2837 &info->tile_x_sa, &info->tile_y_sa);
2838 assert(ok);
2839 info->addr.offset += offset_B;
2840
2841 /* BLORP doesn't use the actual intratile offsets. Instead, it needs the
2842 * surface to be a bit bigger and we offset the vertices instead. Standard
2843 * tilings don't need intratile offsets because each subresource is aligned
2844 * to a bpb-based tile boundary or miptail slot offset.
2845 */
2846 if (isl_tiling_is_64(info->surf.tiling) ||
2847 isl_tiling_is_std_y(info->surf.tiling)) {
2848 assert(info->tile_x_sa == 0 && info->tile_y_sa == 0);
2849 } else {
2850 assert(info->surf.dim == ISL_SURF_DIM_2D);
2851 assert(info->surf.logical_level0_px.array_len == 1);
2852 info->surf.logical_level0_px.w += info->tile_x_sa;
2853 info->surf.logical_level0_px.h += info->tile_y_sa;
2854 info->surf.phys_level0_sa.w += info->tile_x_sa;
2855 info->surf.phys_level0_sa.h += info->tile_y_sa;
2856 }
2857 }
2858
2859 bool
blorp_copy_supports_compute(struct blorp_context * blorp,const struct isl_surf * src_surf,const struct isl_surf * dst_surf,enum isl_aux_usage dst_aux_usage)2860 blorp_copy_supports_compute(struct blorp_context *blorp,
2861 const struct isl_surf *src_surf,
2862 const struct isl_surf *dst_surf,
2863 enum isl_aux_usage dst_aux_usage)
2864 {
2865 return blorp_blit_supports_compute(blorp, src_surf, dst_surf, dst_aux_usage);
2866 }
2867
2868 void
blorp_copy_get_formats(const struct isl_device * isl_dev,const struct isl_surf * src_surf,const struct isl_surf * dst_surf,enum isl_format * src_view_format,enum isl_format * dst_view_format)2869 blorp_copy_get_formats(const struct isl_device *isl_dev,
2870 const struct isl_surf *src_surf,
2871 const struct isl_surf *dst_surf,
2872 enum isl_format *src_view_format,
2873 enum isl_format *dst_view_format)
2874 {
2875 const struct isl_format_layout *src_fmtl =
2876 isl_format_get_layout(src_surf->format);
2877 const struct isl_format_layout *dst_fmtl =
2878 isl_format_get_layout(dst_surf->format);
2879
2880 if (ISL_GFX_VER(isl_dev) >= 8 &&
2881 isl_surf_usage_is_depth(src_surf->usage)) {
2882 /* In order to use HiZ, we have to use the real format for the source.
2883 * Depth <-> Color copies are not allowed.
2884 */
2885 *src_view_format = src_surf->format;
2886 *dst_view_format = src_surf->format;
2887 } else if (ISL_GFX_VER(isl_dev) >= 7 &&
2888 isl_surf_usage_is_depth(dst_surf->usage)) {
2889 /* On Gfx7 and higher, we use actual depth writes for blits into depth
2890 * buffers so we need the real format.
2891 */
2892 *src_view_format = dst_surf->format;
2893 *dst_view_format = dst_surf->format;
2894 } else if (isl_surf_usage_is_depth_or_stencil(src_surf->usage) ||
2895 isl_surf_usage_is_depth_or_stencil(dst_surf->usage)) {
2896 assert(src_fmtl->bpb == dst_fmtl->bpb);
2897 *src_view_format =
2898 *dst_view_format =
2899 get_copy_format_for_bpb(isl_dev, dst_fmtl->bpb);
2900 } else {
2901 *src_view_format =
2902 blorp_copy_get_color_format(isl_dev, src_surf->format);
2903 *dst_view_format =
2904 blorp_copy_get_color_format(isl_dev, dst_surf->format);
2905 }
2906 }
2907
2908
2909 void
blorp_copy(struct blorp_batch * batch,const struct blorp_surf * src_surf,unsigned src_level,unsigned src_layer,const struct blorp_surf * dst_surf,unsigned dst_level,unsigned dst_layer,uint32_t src_x,uint32_t src_y,uint32_t dst_x,uint32_t dst_y,uint32_t src_width,uint32_t src_height)2910 blorp_copy(struct blorp_batch *batch,
2911 const struct blorp_surf *src_surf,
2912 unsigned src_level, unsigned src_layer,
2913 const struct blorp_surf *dst_surf,
2914 unsigned dst_level, unsigned dst_layer,
2915 uint32_t src_x, uint32_t src_y,
2916 uint32_t dst_x, uint32_t dst_y,
2917 uint32_t src_width, uint32_t src_height)
2918 {
2919 const struct isl_device *isl_dev = batch->blorp->isl_dev;
2920 const struct intel_device_info *devinfo = isl_dev->info;
2921 struct blorp_params params;
2922
2923 if (src_width == 0 || src_height == 0)
2924 return;
2925
2926 blorp_params_init(¶ms);
2927 params.op = BLORP_OP_COPY;
2928
2929 const bool compute = batch->flags & BLORP_BATCH_USE_COMPUTE;
2930 if (compute) {
2931 assert(blorp_copy_supports_compute(batch->blorp,
2932 src_surf->surf, dst_surf->surf,
2933 dst_surf->aux_usage));
2934 } else if (batch->flags & BLORP_BATCH_USE_BLITTER) {
2935 assert(blorp_copy_supports_blitter(batch->blorp,
2936 src_surf->surf, dst_surf->surf,
2937 src_surf->aux_usage,
2938 dst_surf->aux_usage));
2939 }
2940
2941 blorp_surface_info_init(batch, ¶ms.src, src_surf, src_level,
2942 src_layer, ISL_FORMAT_UNSUPPORTED, false);
2943 blorp_surface_info_init(batch, ¶ms.dst, dst_surf, dst_level,
2944 dst_layer, ISL_FORMAT_UNSUPPORTED, true);
2945
2946 struct blorp_blit_prog_key key = {
2947 .base = BLORP_BASE_KEY_INIT(BLORP_SHADER_TYPE_COPY),
2948 .base.shader_pipeline = compute ? BLORP_SHADER_PIPELINE_COMPUTE :
2949 BLORP_SHADER_PIPELINE_RENDER,
2950 .filter = BLORP_FILTER_NONE,
2951 .need_src_offset = src_surf->tile_x_sa || src_surf->tile_y_sa,
2952 .need_dst_offset = dst_surf->tile_x_sa || dst_surf->tile_y_sa,
2953 };
2954
2955 params.shader_type = key.base.shader_type;
2956 params.shader_pipeline = key.base.shader_pipeline;
2957
2958 const struct isl_format_layout *src_fmtl =
2959 isl_format_get_layout(params.src.surf.format);
2960 const struct isl_format_layout *dst_fmtl =
2961 isl_format_get_layout(params.dst.surf.format);
2962
2963 assert(params.src.aux_usage == ISL_AUX_USAGE_NONE ||
2964 params.src.aux_usage == ISL_AUX_USAGE_HIZ ||
2965 params.src.aux_usage == ISL_AUX_USAGE_HIZ_CCS_WT ||
2966 params.src.aux_usage == ISL_AUX_USAGE_MCS ||
2967 params.src.aux_usage == ISL_AUX_USAGE_MCS_CCS ||
2968 params.src.aux_usage == ISL_AUX_USAGE_CCS_E ||
2969 params.src.aux_usage == ISL_AUX_USAGE_FCV_CCS_E ||
2970 params.src.aux_usage == ISL_AUX_USAGE_STC_CCS);
2971
2972 blorp_copy_get_formats(isl_dev, ¶ms.src.surf, ¶ms.dst.surf,
2973 ¶ms.src.view.format, ¶ms.dst.view.format);
2974
2975 if (isl_aux_usage_has_fast_clears(params.src.aux_usage) &&
2976 isl_dev->ss.clear_color_state_size > 0) {
2977 /* Depending on the format, the sampler may change the location from
2978 * which it fetches the clear color. This can be a problem in some
2979 * cases, so make sure that the view format won't change the location.
2980 */
2981 ASSERTED enum isl_format src_view_fmt = params.src.view.format;
2982 ASSERTED enum isl_format src_surf_fmt = params.src.surf.format;
2983 assert(isl_get_sampler_clear_field_offset(devinfo, src_view_fmt) ==
2984 isl_get_sampler_clear_field_offset(devinfo, src_surf_fmt));
2985 }
2986
2987 if (params.src.view.format != params.dst.view.format) {
2988 enum isl_format src_cast_format = params.src.view.format;
2989 enum isl_format dst_cast_format = params.dst.view.format;
2990
2991 /* The BLORP bitcast code gets confused by RGB formats. Just treat them
2992 * as RGBA and then everything will be happy. This is perfectly safe
2993 * because BLORP likes to treat things as if they have vec4 colors all
2994 * the time anyway.
2995 */
2996 if (isl_format_get_layout(src_cast_format)->bpb % 3 == 0)
2997 src_cast_format = isl_format_rgb_to_rgba(src_cast_format);
2998 if (isl_format_get_layout(dst_cast_format)->bpb % 3 == 0)
2999 dst_cast_format = isl_format_rgb_to_rgba(dst_cast_format);
3000
3001 if (src_cast_format != dst_cast_format) {
3002 key.format_bit_cast = true;
3003 key.src_format = src_cast_format;
3004 key.dst_format = dst_cast_format;
3005 }
3006 }
3007
3008 if (src_fmtl->bw > 1 || src_fmtl->bh > 1) {
3009 blorp_surf_convert_to_uncompressed(batch->blorp->isl_dev, ¶ms.src,
3010 &src_x, &src_y,
3011 &src_width, &src_height);
3012 key.need_src_offset = true;
3013 }
3014
3015 if (dst_fmtl->bw > 1 || dst_fmtl->bh > 1) {
3016 blorp_surf_convert_to_uncompressed(batch->blorp->isl_dev, ¶ms.dst,
3017 &dst_x, &dst_y, NULL, NULL);
3018 key.need_dst_offset = true;
3019 }
3020
3021 /* Once both surfaces are stompped to uncompressed as needed, the
3022 * destination size is the same as the source size.
3023 */
3024 uint32_t dst_width = src_width;
3025 uint32_t dst_height = src_height;
3026
3027 if (batch->flags & BLORP_BATCH_USE_BLITTER) {
3028 if (devinfo->verx10 < 125) {
3029 blorp_surf_convert_to_single_slice(isl_dev, ¶ms.dst);
3030 blorp_surf_convert_to_single_slice(isl_dev, ¶ms.src);
3031 }
3032
3033 params.x0 = dst_x;
3034 params.x1 = dst_x + dst_width;
3035 params.y0 = dst_y;
3036 params.y1 = dst_y + dst_height;
3037 params.wm_inputs.coord_transform[0].offset = dst_x - (float)src_x;
3038 params.wm_inputs.coord_transform[1].offset = dst_y - (float)src_y;
3039 params.wm_inputs.coord_transform[0].multiplier = 1.0f;
3040 params.wm_inputs.coord_transform[1].multiplier = 1.0f;
3041
3042 batch->blorp->exec(batch, ¶ms);
3043 return;
3044 }
3045
3046 struct blt_coords coords = {
3047 .x = {
3048 .src0 = src_x,
3049 .src1 = src_x + src_width,
3050 .dst0 = dst_x,
3051 .dst1 = dst_x + dst_width,
3052 .mirror = false
3053 },
3054 .y = {
3055 .src0 = src_y,
3056 .src1 = src_y + src_height,
3057 .dst0 = dst_y,
3058 .dst1 = dst_y + dst_height,
3059 .mirror = false
3060 }
3061 };
3062
3063 do_blorp_blit(batch, ¶ms, &key, &coords);
3064 }
3065
3066 static enum isl_format
isl_format_for_size(unsigned size_B)3067 isl_format_for_size(unsigned size_B)
3068 {
3069 switch (size_B) {
3070 case 1: return ISL_FORMAT_R8_UINT;
3071 case 2: return ISL_FORMAT_R8G8_UINT;
3072 case 4: return ISL_FORMAT_R8G8B8A8_UINT;
3073 case 8: return ISL_FORMAT_R16G16B16A16_UINT;
3074 case 16: return ISL_FORMAT_R32G32B32A32_UINT;
3075 default:
3076 unreachable("Not a power-of-two format size");
3077 }
3078 }
3079
3080 /**
3081 * Returns the greatest common divisor of a and b that is a power of two.
3082 */
3083 static uint64_t
gcd_pow2_u64(uint64_t a,uint64_t b)3084 gcd_pow2_u64(uint64_t a, uint64_t b)
3085 {
3086 assert(a > 0 || b > 0);
3087
3088 unsigned a_log2 = ffsll(a) - 1;
3089 unsigned b_log2 = ffsll(b) - 1;
3090
3091 /* If either a or b is 0, then a_log2 or b_log2 till be UINT_MAX in which
3092 * case, the MIN2() will take the other one. If both are 0 then we will
3093 * hit the assert above.
3094 */
3095 return 1 << MIN2(a_log2, b_log2);
3096 }
3097
3098 static void
do_buffer_copy(struct blorp_batch * batch,struct blorp_address * src,struct blorp_address * dst,int width,int height,int block_size)3099 do_buffer_copy(struct blorp_batch *batch,
3100 struct blorp_address *src,
3101 struct blorp_address *dst,
3102 int width, int height, int block_size)
3103 {
3104 /* The actual format we pick doesn't matter as blorp will throw it away.
3105 * The only thing that actually matters is the size.
3106 */
3107 enum isl_format format = isl_format_for_size(block_size);
3108
3109 UNUSED bool ok;
3110 struct isl_surf surf;
3111 ok = isl_surf_init(batch->blorp->isl_dev, &surf,
3112 .dim = ISL_SURF_DIM_2D,
3113 .format = format,
3114 .width = width,
3115 .height = height,
3116 .depth = 1,
3117 .levels = 1,
3118 .array_len = 1,
3119 .samples = 1,
3120 .row_pitch_B = width * block_size,
3121 .usage = ISL_SURF_USAGE_TEXTURE_BIT |
3122 ISL_SURF_USAGE_RENDER_TARGET_BIT,
3123 .tiling_flags = ISL_TILING_LINEAR_BIT);
3124 assert(ok);
3125
3126 struct blorp_surf src_blorp_surf = {
3127 .surf = &surf,
3128 .addr = *src,
3129 };
3130
3131 struct blorp_surf dst_blorp_surf = {
3132 .surf = &surf,
3133 .addr = *dst,
3134 };
3135
3136 blorp_copy(batch, &src_blorp_surf, 0, 0, &dst_blorp_surf, 0, 0,
3137 0, 0, 0, 0, width, height);
3138 }
3139
3140 void
blorp_buffer_copy(struct blorp_batch * batch,struct blorp_address src,struct blorp_address dst,uint64_t size)3141 blorp_buffer_copy(struct blorp_batch *batch,
3142 struct blorp_address src,
3143 struct blorp_address dst,
3144 uint64_t size)
3145 {
3146 const struct intel_device_info *devinfo = batch->blorp->isl_dev->info;
3147 uint64_t copy_size = size;
3148
3149 /* This is maximum possible width/height our HW can handle */
3150 uint64_t max_surface_dim = 1 << (devinfo->ver >= 7 ? 14 : 13);
3151
3152 /* First, we compute the biggest format that can be used with the
3153 * given offsets and size.
3154 */
3155 int bs = 16;
3156 bs = gcd_pow2_u64(bs, src.offset);
3157 bs = gcd_pow2_u64(bs, dst.offset);
3158 bs = gcd_pow2_u64(bs, size);
3159
3160 /* First, we make a bunch of max-sized copies */
3161 uint64_t max_copy_size = max_surface_dim * max_surface_dim * bs;
3162 while (copy_size >= max_copy_size) {
3163 do_buffer_copy(batch, &src, &dst, max_surface_dim, max_surface_dim, bs);
3164 copy_size -= max_copy_size;
3165 src.offset += max_copy_size;
3166 dst.offset += max_copy_size;
3167 }
3168
3169 /* Now make a max-width copy */
3170 uint64_t height = copy_size / (max_surface_dim * bs);
3171 assert(height < max_surface_dim);
3172 if (height != 0) {
3173 uint64_t rect_copy_size = height * max_surface_dim * bs;
3174 do_buffer_copy(batch, &src, &dst, max_surface_dim, height, bs);
3175 copy_size -= rect_copy_size;
3176 src.offset += rect_copy_size;
3177 dst.offset += rect_copy_size;
3178 }
3179
3180 /* Finally, make a small copy to finish it off */
3181 if (copy_size != 0) {
3182 do_buffer_copy(batch, &src, &dst, copy_size / bs, 1, bs);
3183 }
3184 }
3185