1 /*
2 * Copyright © 2013 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "util/ralloc.h"
25
26 #include "util/macros.h" /* Needed for MAX3 and MAX2 for format_rgb9e5 */
27 #include "util/format_rgb9e5.h"
28 #include "util/format_srgb.h"
29 #include "util/u_math.h"
30
31 #include "blorp_priv.h"
32 #include "dev/intel_debug.h"
33 #include "dev/intel_device_info.h"
34
35 #include "blorp_nir_builder.h"
36
37 #define FILE_DEBUG_FLAG DEBUG_BLORP
38
39 #pragma pack(push, 1)
40 struct blorp_const_color_prog_key
41 {
42 struct blorp_base_key base;
43 bool use_simd16_replicated_data;
44 bool clear_rgb_as_red;
45 uint8_t local_y;
46 };
47 #pragma pack(pop)
48
49 static bool
blorp_params_get_clear_kernel_fs(struct blorp_batch * batch,struct blorp_params * params,bool use_replicated_data,bool clear_rgb_as_red)50 blorp_params_get_clear_kernel_fs(struct blorp_batch *batch,
51 struct blorp_params *params,
52 bool use_replicated_data,
53 bool clear_rgb_as_red)
54 {
55 struct blorp_context *blorp = batch->blorp;
56
57 const struct blorp_const_color_prog_key blorp_key = {
58 .base = BLORP_BASE_KEY_INIT(BLORP_SHADER_TYPE_CLEAR),
59 .base.shader_pipeline = BLORP_SHADER_PIPELINE_RENDER,
60 .use_simd16_replicated_data = use_replicated_data,
61 .clear_rgb_as_red = clear_rgb_as_red,
62 .local_y = 0,
63 };
64
65 params->shader_type = blorp_key.base.shader_type;
66 params->shader_pipeline = blorp_key.base.shader_pipeline;
67
68 if (blorp->lookup_shader(batch, &blorp_key, sizeof(blorp_key),
69 ¶ms->wm_prog_kernel, ¶ms->wm_prog_data))
70 return true;
71
72 void *mem_ctx = ralloc_context(NULL);
73
74 nir_builder b;
75 blorp_nir_init_shader(&b, mem_ctx, MESA_SHADER_FRAGMENT,
76 blorp_shader_type_to_name(blorp_key.base.shader_type));
77
78 nir_variable *v_color =
79 BLORP_CREATE_NIR_INPUT(b.shader, clear_color, glsl_vec4_type());
80 nir_def *color = nir_load_var(&b, v_color);
81
82 if (clear_rgb_as_red) {
83 nir_def *pos = nir_f2i32(&b, nir_load_frag_coord(&b));
84 nir_def *comp = nir_umod_imm(&b, nir_channel(&b, pos, 0), 3);
85 color = nir_pad_vec4(&b, nir_vector_extract(&b, color, comp));
86 }
87
88 nir_variable *frag_color = nir_variable_create(b.shader, nir_var_shader_out,
89 glsl_vec4_type(),
90 "gl_FragColor");
91 frag_color->data.location = FRAG_RESULT_COLOR;
92 nir_store_var(&b, frag_color, color, 0xf);
93
94 const bool multisample_fbo = false;
95 struct blorp_program p =
96 blorp_compile_fs(blorp, mem_ctx, b.shader, multisample_fbo, use_replicated_data);
97
98 bool result =
99 blorp->upload_shader(batch, MESA_SHADER_FRAGMENT,
100 &blorp_key, sizeof(blorp_key),
101 p.kernel, p.kernel_size,
102 p.prog_data, p.prog_data_size,
103 ¶ms->wm_prog_kernel, ¶ms->wm_prog_data);
104
105 ralloc_free(mem_ctx);
106 return result;
107 }
108
109 static bool
blorp_params_get_clear_kernel_cs(struct blorp_batch * batch,struct blorp_params * params,bool clear_rgb_as_red)110 blorp_params_get_clear_kernel_cs(struct blorp_batch *batch,
111 struct blorp_params *params,
112 bool clear_rgb_as_red)
113 {
114 struct blorp_context *blorp = batch->blorp;
115
116 const struct blorp_const_color_prog_key blorp_key = {
117 .base = BLORP_BASE_KEY_INIT(BLORP_SHADER_TYPE_CLEAR),
118 .base.shader_pipeline = BLORP_SHADER_PIPELINE_COMPUTE,
119 .use_simd16_replicated_data = false,
120 .clear_rgb_as_red = clear_rgb_as_red,
121 .local_y = blorp_get_cs_local_y(params),
122 };
123
124 params->shader_type = blorp_key.base.shader_type;
125 params->shader_pipeline = blorp_key.base.shader_pipeline;
126
127 if (blorp->lookup_shader(batch, &blorp_key, sizeof(blorp_key),
128 ¶ms->cs_prog_kernel, ¶ms->cs_prog_data))
129 return true;
130
131 void *mem_ctx = ralloc_context(NULL);
132
133 nir_builder b;
134 blorp_nir_init_shader(&b, mem_ctx, MESA_SHADER_COMPUTE, "BLORP-gpgpu-clear");
135 blorp_set_cs_dims(b.shader, blorp_key.local_y);
136
137 nir_def *dst_pos = nir_load_global_invocation_id(&b, 32);
138
139 nir_variable *v_color =
140 BLORP_CREATE_NIR_INPUT(b.shader, clear_color, glsl_vec4_type());
141 nir_def *color = nir_load_var(&b, v_color);
142
143 nir_variable *v_bounds_rect =
144 BLORP_CREATE_NIR_INPUT(b.shader, bounds_rect, glsl_vec4_type());
145 nir_def *bounds_rect = nir_load_var(&b, v_bounds_rect);
146 nir_def *in_bounds = blorp_check_in_bounds(&b, bounds_rect, dst_pos);
147
148 if (clear_rgb_as_red) {
149 nir_def *comp = nir_umod_imm(&b, nir_channel(&b, dst_pos, 0), 3);
150 color = nir_pad_vec4(&b, nir_vector_extract(&b, color, comp));
151 }
152
153 nir_push_if(&b, in_bounds);
154
155 nir_image_store(&b, nir_imm_int(&b, 0),
156 nir_pad_vector_imm_int(&b, dst_pos, 0, 4),
157 nir_imm_int(&b, 0),
158 nir_pad_vector_imm_int(&b, color, 0, 4),
159 nir_imm_int(&b, 0),
160 .image_dim = GLSL_SAMPLER_DIM_2D,
161 .image_array = true,
162 .access = ACCESS_NON_READABLE);
163
164 nir_pop_if(&b, NULL);
165
166 const struct blorp_program p =
167 blorp_compile_cs(blorp, mem_ctx, b.shader);
168
169 bool result =
170 blorp->upload_shader(batch, MESA_SHADER_COMPUTE,
171 &blorp_key, sizeof(blorp_key),
172 p.kernel, p.kernel_size,
173 p.prog_data, p.prog_data_size,
174 ¶ms->cs_prog_kernel, ¶ms->cs_prog_data);
175
176 ralloc_free(mem_ctx);
177 return result;
178 }
179
180 static bool
blorp_params_get_clear_kernel(struct blorp_batch * batch,struct blorp_params * params,bool use_replicated_data,bool clear_rgb_as_red)181 blorp_params_get_clear_kernel(struct blorp_batch *batch,
182 struct blorp_params *params,
183 bool use_replicated_data,
184 bool clear_rgb_as_red)
185 {
186 if (batch->flags & BLORP_BATCH_USE_COMPUTE) {
187 assert(!use_replicated_data);
188 return blorp_params_get_clear_kernel_cs(batch, params, clear_rgb_as_red);
189 } else {
190 return blorp_params_get_clear_kernel_fs(batch, params,
191 use_replicated_data,
192 clear_rgb_as_red);
193 }
194 }
195
196 /* The x0, y0, x1, and y1 parameters must already be populated with the render
197 * area of the framebuffer to be cleared.
198 */
199 static void
get_fast_clear_rect(const struct isl_device * dev,const struct isl_surf * surf,const struct isl_surf * aux_surf,unsigned * x0,unsigned * y0,unsigned * x1,unsigned * y1)200 get_fast_clear_rect(const struct isl_device *dev,
201 const struct isl_surf *surf,
202 const struct isl_surf *aux_surf,
203 unsigned *x0, unsigned *y0,
204 unsigned *x1, unsigned *y1)
205 {
206 unsigned int x_align, y_align;
207 unsigned int x_scaledown, y_scaledown;
208
209 /* Only single sampled surfaces need to (and actually can) be resolved. */
210 if (surf->samples == 1) {
211 if (dev->info->verx10 >= 125) {
212 assert(surf->tiling == ISL_TILING_4);
213 /* From Bspec 47709, "MCS/CCS Buffer for Render Target(s)":
214 *
215 * SW must ensure that clearing rectangle dimensions cover the
216 * entire area desired, to accomplish this task initial X/Y
217 * dimensions need to be rounded up to next multiple of scaledown
218 * factor before dividing by scale down factor:
219 *
220 * The X and Y scale down factors in the table that follows are used
221 * for both alignment and scaling down.
222 */
223 const uint32_t bs = isl_format_get_layout(surf->format)->bpb / 8;
224 x_align = x_scaledown = 1024 / bs;
225 y_align = y_scaledown = 16;
226 } else {
227 assert(aux_surf->usage == ISL_SURF_USAGE_CCS_BIT);
228 /* From the Ivy Bridge PRM, Vol2 Part1 11.7 "MCS Buffer for Render
229 * Target(s)", beneath the "Fast Color Clear" bullet (p327):
230 *
231 * Clear pass must have a clear rectangle that must follow
232 * alignment rules in terms of pixels and lines as shown in the
233 * table below. Further, the clear-rectangle height and width
234 * must be multiple of the following dimensions. If the height
235 * and width of the render target being cleared do not meet these
236 * requirements, an MCS buffer can be created such that it
237 * follows the requirement and covers the RT.
238 *
239 * The alignment size in the table that follows is related to the
240 * alignment size that is baked into the CCS surface format but with X
241 * alignment multiplied by 16 and Y alignment multiplied by 32.
242 */
243 x_align = isl_format_get_layout(aux_surf->format)->bw;
244 y_align = isl_format_get_layout(aux_surf->format)->bh;
245
246 x_align *= 16;
247
248 /* The line alignment requirement for Y-tiled is halved at SKL and again
249 * at TGL.
250 */
251 if (dev->info->ver >= 12)
252 y_align *= 8;
253 else if (dev->info->ver >= 9)
254 y_align *= 16;
255 else
256 y_align *= 32;
257
258 /* From the Ivy Bridge PRM, Vol2 Part1 11.7 "MCS Buffer for Render
259 * Target(s)", beneath the "Fast Color Clear" bullet (p327):
260 *
261 * In order to optimize the performance MCS buffer (when bound to
262 * 1X RT) clear similarly to MCS buffer clear for MSRT case,
263 * clear rect is required to be scaled by the following factors
264 * in the horizontal and vertical directions:
265 *
266 * The X and Y scale down factors in the table that follows are each
267 * equal to half the alignment value computed above.
268 */
269 x_scaledown = x_align / 2;
270 y_scaledown = y_align / 2;
271 }
272
273 if (ISL_DEV_IS_HASWELL(dev)) {
274 /* From BSpec: 3D-Media-GPGPU Engine > 3D Pipeline > Pixel > Pixel
275 * Backend > MCS Buffer for Render Target(s) [DevIVB+] > Table "Color
276 * Clear of Non-MultiSampled Render Target Restrictions":
277 *
278 * Clear rectangle must be aligned to two times the number of
279 * pixels in the table shown below due to 16x16 hashing across the
280 * slice.
281 *
282 * This restriction is only documented to exist on HSW GT3 but
283 * empirical evidence suggests that it's also needed GT2.
284 */
285 x_align *= 2;
286 y_align *= 2;
287 }
288 } else {
289 assert(aux_surf->usage == ISL_SURF_USAGE_MCS_BIT);
290
291 /* From the Ivy Bridge PRM, Vol2 Part1 11.7 "MCS Buffer for Render
292 * Target(s)", beneath the "MSAA Compression" bullet (p326):
293 *
294 * Clear pass for this case requires that scaled down primitive
295 * is sent down with upper left coordinate to coincide with
296 * actual rectangle being cleared. For MSAA, clear rectangle’s
297 * height and width need to as show in the following table in
298 * terms of (width,height) of the RT.
299 *
300 * MSAA Width of Clear Rect Height of Clear Rect
301 * 2X Ceil(1/8*width) Ceil(1/2*height)
302 * 4X Ceil(1/8*width) Ceil(1/2*height)
303 * 8X Ceil(1/2*width) Ceil(1/2*height)
304 * 16X width Ceil(1/2*height)
305 *
306 * The text "with upper left coordinate to coincide with actual
307 * rectangle being cleared" is a little confusing--it seems to imply
308 * that to clear a rectangle from (x,y) to (x+w,y+h), one needs to
309 * feed the pipeline using the rectangle (x,y) to
310 * (x+Ceil(w/N),y+Ceil(h/2)), where N is either 2 or 8 depending on
311 * the number of samples. Experiments indicate that this is not
312 * quite correct; actually, what the hardware appears to do is to
313 * align whatever rectangle is sent down the pipeline to the nearest
314 * multiple of 2x2 blocks, and then scale it up by a factor of N
315 * horizontally and 2 vertically. So the resulting alignment is 4
316 * vertically and either 4 or 16 horizontally, and the scaledown
317 * factor is 2 vertically and either 2 or 8 horizontally.
318 */
319 switch (aux_surf->format) {
320 case ISL_FORMAT_MCS_2X:
321 case ISL_FORMAT_MCS_4X:
322 x_scaledown = 8;
323 break;
324 case ISL_FORMAT_MCS_8X:
325 x_scaledown = 2;
326 break;
327 case ISL_FORMAT_MCS_16X:
328 x_scaledown = 1;
329 break;
330 default:
331 unreachable("Unexpected MCS format for fast clear");
332 }
333 y_scaledown = 2;
334 x_align = x_scaledown * 2;
335 y_align = y_scaledown * 2;
336 }
337
338 *x0 = ROUND_DOWN_TO(*x0, x_align) / x_scaledown;
339 *y0 = ROUND_DOWN_TO(*y0, y_align) / y_scaledown;
340 *x1 = ALIGN(*x1, x_align) / x_scaledown;
341 *y1 = ALIGN(*y1, y_align) / y_scaledown;
342 }
343
344 void
blorp_fast_clear(struct blorp_batch * batch,const struct blorp_surf * surf,enum isl_format format,struct isl_swizzle swizzle,uint32_t level,uint32_t start_layer,uint32_t num_layers,uint32_t x0,uint32_t y0,uint32_t x1,uint32_t y1)345 blorp_fast_clear(struct blorp_batch *batch,
346 const struct blorp_surf *surf,
347 enum isl_format format, struct isl_swizzle swizzle,
348 uint32_t level, uint32_t start_layer, uint32_t num_layers,
349 uint32_t x0, uint32_t y0, uint32_t x1, uint32_t y1)
350 {
351 struct blorp_params params;
352 blorp_params_init(¶ms);
353 params.num_layers = num_layers;
354 assert((batch->flags & BLORP_BATCH_USE_COMPUTE) == 0);
355
356 params.x0 = x0;
357 params.y0 = y0;
358 params.x1 = x1;
359 params.y1 = y1;
360
361 memset(¶ms.wm_inputs.clear_color, 0xff, 4*sizeof(float));
362 params.fast_clear_op = ISL_AUX_OP_FAST_CLEAR;
363
364 get_fast_clear_rect(batch->blorp->isl_dev, surf->surf, surf->aux_surf,
365 ¶ms.x0, ¶ms.y0, ¶ms.x1, ¶ms.y1);
366
367 if (!blorp_params_get_clear_kernel(batch, ¶ms, true, false))
368 return;
369
370 blorp_surface_info_init(batch, ¶ms.dst, surf, level,
371 start_layer, format, true);
372 params.num_samples = params.dst.surf.samples;
373
374 assert(params.num_samples != 0);
375 if (params.num_samples == 1)
376 params.op = BLORP_OP_CCS_COLOR_CLEAR;
377 else
378 params.op = BLORP_OP_MCS_COLOR_CLEAR;
379
380 /* If a swizzle was provided, we need to swizzle the clear color so that
381 * the hardware color format conversion will work properly.
382 */
383 params.dst.clear_color =
384 isl_color_value_swizzle_inv(params.dst.clear_color, swizzle);
385
386 batch->blorp->exec(batch, ¶ms);
387 }
388
389 bool
blorp_clear_supports_blitter(struct blorp_context * blorp,const struct blorp_surf * surf,uint8_t color_write_disable,bool blend_enabled)390 blorp_clear_supports_blitter(struct blorp_context *blorp,
391 const struct blorp_surf *surf,
392 uint8_t color_write_disable,
393 bool blend_enabled)
394 {
395 const struct intel_device_info *devinfo = blorp->isl_dev->info;
396
397 if (devinfo->ver < 12)
398 return false;
399
400 if (surf->surf->samples > 1)
401 return false;
402
403 if (color_write_disable != 0 || blend_enabled)
404 return false;
405
406 if (!blorp_blitter_supports_aux(devinfo, surf->aux_usage))
407 return false;
408
409 const struct isl_format_layout *fmtl =
410 isl_format_get_layout(surf->surf->format);
411
412 /* We can only support linear mode for 96bpp. */
413 if (fmtl->bpb == 96 && surf->surf->tiling != ISL_TILING_LINEAR)
414 return false;
415
416 return true;
417 }
418
419 bool
blorp_clear_supports_compute(struct blorp_context * blorp,uint8_t color_write_disable,bool blend_enabled,enum isl_aux_usage aux_usage)420 blorp_clear_supports_compute(struct blorp_context *blorp,
421 uint8_t color_write_disable, bool blend_enabled,
422 enum isl_aux_usage aux_usage)
423 {
424 if (blorp->isl_dev->info->ver < 7)
425 return false;
426 if (color_write_disable != 0 || blend_enabled)
427 return false;
428 if (blorp->isl_dev->info->ver >= 12) {
429 return aux_usage == ISL_AUX_USAGE_FCV_CCS_E ||
430 aux_usage == ISL_AUX_USAGE_CCS_E ||
431 aux_usage == ISL_AUX_USAGE_NONE;
432 } else {
433 return aux_usage == ISL_AUX_USAGE_NONE;
434 }
435 }
436
437 void
blorp_clear(struct blorp_batch * batch,const struct blorp_surf * surf,enum isl_format format,struct isl_swizzle swizzle,uint32_t level,uint32_t start_layer,uint32_t num_layers,uint32_t x0,uint32_t y0,uint32_t x1,uint32_t y1,union isl_color_value clear_color,uint8_t color_write_disable)438 blorp_clear(struct blorp_batch *batch,
439 const struct blorp_surf *surf,
440 enum isl_format format, struct isl_swizzle swizzle,
441 uint32_t level, uint32_t start_layer, uint32_t num_layers,
442 uint32_t x0, uint32_t y0, uint32_t x1, uint32_t y1,
443 union isl_color_value clear_color,
444 uint8_t color_write_disable)
445 {
446 struct blorp_params params;
447 blorp_params_init(¶ms);
448 params.op = BLORP_OP_SLOW_COLOR_CLEAR;
449
450 const bool compute = batch->flags & BLORP_BATCH_USE_COMPUTE;
451 if (compute) {
452 assert(blorp_clear_supports_compute(batch->blorp, color_write_disable,
453 false, surf->aux_usage));
454 } else if (batch->flags & BLORP_BATCH_USE_BLITTER) {
455 assert(blorp_clear_supports_blitter(batch->blorp, surf,
456 color_write_disable, false));
457 }
458
459 /* Manually apply the clear destination swizzle. This way swizzled clears
460 * will work for swizzles which we can't normally use for rendering and it
461 * also ensures that they work on pre-Haswell hardware which can't swizlle
462 * at all.
463 */
464 clear_color = isl_color_value_swizzle_inv(clear_color, swizzle);
465 swizzle = ISL_SWIZZLE_IDENTITY;
466
467 bool clear_rgb_as_red = false;
468 if (format == ISL_FORMAT_R9G9B9E5_SHAREDEXP) {
469 clear_color.u32[0] = float3_to_rgb9e5(clear_color.f32);
470 format = ISL_FORMAT_R32_UINT;
471 } else if (format == ISL_FORMAT_L8_UNORM_SRGB) {
472 clear_color.f32[0] = util_format_linear_to_srgb_float(clear_color.f32[0]);
473 format = ISL_FORMAT_R8_UNORM;
474 } else if (format == ISL_FORMAT_A4B4G4R4_UNORM) {
475 /* Broadwell and earlier cannot render to this format so we need to work
476 * around it by swapping the colors around and using B4G4R4A4 instead.
477 */
478 const struct isl_swizzle ARGB = ISL_SWIZZLE(ALPHA, RED, GREEN, BLUE);
479 clear_color = isl_color_value_swizzle_inv(clear_color, ARGB);
480 format = ISL_FORMAT_B4G4R4A4_UNORM;
481 } else if (isl_format_get_layout(format)->bpb % 3 == 0) {
482 clear_rgb_as_red = true;
483 if (format == ISL_FORMAT_R8G8B8_UNORM_SRGB) {
484 clear_color.f32[0] = util_format_linear_to_srgb_float(clear_color.f32[0]);
485 clear_color.f32[1] = util_format_linear_to_srgb_float(clear_color.f32[1]);
486 clear_color.f32[2] = util_format_linear_to_srgb_float(clear_color.f32[2]);
487 }
488 }
489
490 memcpy(¶ms.wm_inputs.clear_color, clear_color.f32, sizeof(float) * 4);
491
492 bool use_simd16_replicated_data = true;
493
494 /* From the SNB PRM (Vol4_Part1):
495 *
496 * "Replicated data (Message Type = 111) is only supported when
497 * accessing tiled memory. Using this Message Type to access linear
498 * (untiled) memory is UNDEFINED."
499 */
500 if (surf->surf->tiling == ISL_TILING_LINEAR)
501 use_simd16_replicated_data = false;
502
503 /* Replicated clears don't work yet before gfx6 */
504 if (batch->blorp->isl_dev->info->ver < 6)
505 use_simd16_replicated_data = false;
506
507 /* From the BSpec: 47719 (TGL/DG2/MTL) Replicate Data:
508 *
509 * "Replicate Data Render Target Write message should not be used
510 * on all projects TGL+."
511 *
512 * Xe2 spec (57350) does not mention this restriction.
513 *
514 * See 14017879046, 14017880152 for additional information.
515 */
516 if (batch->blorp->isl_dev->info->ver >= 12 &&
517 batch->blorp->isl_dev->info->ver < 20)
518 use_simd16_replicated_data = false;
519
520 if (compute)
521 use_simd16_replicated_data = false;
522
523 /* Constant color writes ignore everything in blend and color calculator
524 * state. This is not documented.
525 */
526 params.color_write_disable = color_write_disable & BITFIELD_MASK(4);
527 if (color_write_disable)
528 use_simd16_replicated_data = false;
529
530 if (!blorp_params_get_clear_kernel(batch, ¶ms,
531 use_simd16_replicated_data,
532 clear_rgb_as_red))
533 return;
534
535 if (!compute && !blorp_ensure_sf_program(batch, ¶ms))
536 return;
537
538 while (num_layers > 0) {
539 blorp_surface_info_init(batch, ¶ms.dst, surf, level,
540 start_layer, format, true);
541 params.dst.view.swizzle = swizzle;
542
543 params.x0 = x0;
544 params.y0 = y0;
545 params.x1 = x1;
546 params.y1 = y1;
547
548 if (compute) {
549 params.wm_inputs.bounds_rect.x0 = x0;
550 params.wm_inputs.bounds_rect.y0 = y0;
551 params.wm_inputs.bounds_rect.x1 = x1;
552 params.wm_inputs.bounds_rect.y1 = y1;
553 }
554
555 if (params.dst.tile_x_sa || params.dst.tile_y_sa) {
556 assert(params.dst.surf.samples == 1);
557 assert(num_layers == 1);
558 params.x0 += params.dst.tile_x_sa;
559 params.y0 += params.dst.tile_y_sa;
560 params.x1 += params.dst.tile_x_sa;
561 params.y1 += params.dst.tile_y_sa;
562 }
563
564 /* The MinLOD and MinimumArrayElement don't work properly for cube maps.
565 * Convert them to a single slice on gfx4.
566 */
567 if (batch->blorp->isl_dev->info->ver == 4 &&
568 (params.dst.surf.usage & ISL_SURF_USAGE_CUBE_BIT)) {
569 blorp_surf_convert_to_single_slice(batch->blorp->isl_dev, ¶ms.dst);
570 }
571
572 if (clear_rgb_as_red) {
573 surf_fake_rgb_with_red(batch->blorp->isl_dev, ¶ms.dst);
574 params.x0 *= 3;
575 params.x1 *= 3;
576 }
577
578 if (isl_format_is_compressed(params.dst.surf.format)) {
579 blorp_surf_convert_to_uncompressed(batch->blorp->isl_dev, ¶ms.dst,
580 NULL, NULL, NULL, NULL);
581 //&dst_x, &dst_y, &dst_w, &dst_h);
582 }
583
584 if (params.dst.tile_x_sa || params.dst.tile_y_sa) {
585 /* Either we're on gfx4 where there is no multisampling or the
586 * surface is compressed which also implies no multisampling.
587 * Therefore, sa == px and we don't need to do a conversion.
588 */
589 assert(params.dst.surf.samples == 1);
590 params.x0 += params.dst.tile_x_sa;
591 params.y0 += params.dst.tile_y_sa;
592 params.x1 += params.dst.tile_x_sa;
593 params.y1 += params.dst.tile_y_sa;
594 }
595
596 params.num_samples = params.dst.surf.samples;
597
598 /* We may be restricted on the number of layers we can bind at any one
599 * time. In particular, Sandy Bridge has a maximum number of layers of
600 * 512 but a maximum 3D texture size is much larger.
601 */
602 params.num_layers = MIN2(params.dst.view.array_len, num_layers);
603
604 const unsigned max_image_width = 16 * 1024;
605 if (params.dst.surf.logical_level0_px.width > max_image_width) {
606 /* Clearing an RGB image as red multiplies the surface width by 3
607 * so it may now be too wide for the hardware surface limits. We
608 * have to break the clear up into pieces in order to clear wide
609 * images.
610 */
611 assert(clear_rgb_as_red);
612 assert(params.dst.surf.dim == ISL_SURF_DIM_2D);
613 assert(params.dst.surf.tiling == ISL_TILING_LINEAR);
614 assert(params.dst.surf.logical_level0_px.depth == 1);
615 assert(params.dst.surf.logical_level0_px.array_len == 1);
616 assert(params.dst.surf.levels == 1);
617 assert(params.dst.surf.samples == 1);
618 assert(params.dst.tile_x_sa == 0 || params.dst.tile_y_sa == 0);
619 assert(params.dst.aux_usage == ISL_AUX_USAGE_NONE);
620
621 /* max_image_width rounded down to a multiple of 3 */
622 const unsigned max_fake_rgb_width = (max_image_width / 3) * 3;
623 const unsigned cpp =
624 isl_format_get_layout(params.dst.surf.format)->bpb / 8;
625
626 params.dst.surf.logical_level0_px.width = max_fake_rgb_width;
627 params.dst.surf.phys_level0_sa.width = max_fake_rgb_width;
628
629 uint32_t orig_x0 = params.x0, orig_x1 = params.x1;
630 uint64_t orig_offset = params.dst.addr.offset;
631 for (uint32_t x = orig_x0; x < orig_x1; x += max_fake_rgb_width) {
632 /* Offset to the surface. It's easy because we're linear */
633 params.dst.addr.offset = orig_offset + x * cpp;
634
635 params.x0 = 0;
636 params.x1 = MIN2(orig_x1 - x, max_image_width);
637
638 batch->blorp->exec(batch, ¶ms);
639 }
640 } else {
641 batch->blorp->exec(batch, ¶ms);
642 }
643
644 start_layer += params.num_layers;
645 num_layers -= params.num_layers;
646 }
647 }
648
649 static bool
blorp_clear_stencil_as_rgba(struct blorp_batch * batch,const struct blorp_surf * surf,uint32_t level,uint32_t start_layer,uint32_t num_layers,uint32_t x0,uint32_t y0,uint32_t x1,uint32_t y1,uint8_t stencil_mask,uint8_t stencil_value)650 blorp_clear_stencil_as_rgba(struct blorp_batch *batch,
651 const struct blorp_surf *surf,
652 uint32_t level, uint32_t start_layer,
653 uint32_t num_layers,
654 uint32_t x0, uint32_t y0, uint32_t x1, uint32_t y1,
655 uint8_t stencil_mask, uint8_t stencil_value)
656 {
657 assert((batch->flags & BLORP_BATCH_USE_COMPUTE) == 0);
658
659 /* Stencil mask support would require piles of shader magic */
660 if (stencil_mask != 0xff)
661 return false;
662
663 /* We only support separate W-tiled stencil for now */
664 if (surf->surf->format != ISL_FORMAT_R8_UINT ||
665 surf->surf->tiling != ISL_TILING_W)
666 return false;
667
668 if (surf->surf->samples > 1) {
669 /* Adjust x0, y0, x1, and y1 to be in units of samples */
670 assert(surf->surf->msaa_layout == ISL_MSAA_LAYOUT_INTERLEAVED);
671 struct isl_extent2d msaa_px_size_sa =
672 isl_get_interleaved_msaa_px_size_sa(surf->surf->samples);
673
674 x0 *= msaa_px_size_sa.w;
675 y0 *= msaa_px_size_sa.h;
676 x1 *= msaa_px_size_sa.w;
677 y1 *= msaa_px_size_sa.h;
678 }
679
680 /* W-tiles and Y-tiles have the same layout as far as cache lines are
681 * concerned: both are 8x8 cache lines laid out Y-major. The difference is
682 * entirely in how the data is arranged within the cache line. W-tiling
683 * is 8x8 pixels in a swizzled pattern while Y-tiling is 16B by 4 rows
684 * regardless of image format size. As long as everything is aligned to 8,
685 * we can just treat the W-tiled image as Y-tiled, ignore the layout
686 * difference within a cache line, and blast out data.
687 */
688 if (x0 % 8 != 0 || y0 % 8 != 0 || x1 % 8 != 0 || y1 % 8 != 0)
689 return false;
690
691 struct blorp_params params;
692 blorp_params_init(¶ms);
693 params.op = BLORP_OP_SLOW_DEPTH_CLEAR;
694
695 if (!blorp_params_get_clear_kernel(batch, ¶ms, true, false))
696 return false;
697
698 memset(¶ms.wm_inputs.clear_color, stencil_value,
699 sizeof(params.wm_inputs.clear_color));
700
701 /* The Sandy Bridge PRM Vol. 4 Pt. 2, section 2.11.2.1.1 has the
702 * following footnote to the format table:
703 *
704 * 128 BPE Formats cannot be Tiled Y when used as render targets
705 *
706 * We have to use RGBA16_UINT on SNB.
707 */
708 enum isl_format wide_format;
709 if (ISL_GFX_VER(batch->blorp->isl_dev) <= 6) {
710 wide_format = ISL_FORMAT_R16G16B16A16_UINT;
711
712 /* For RGBA16_UINT, we need to mask the stencil value otherwise, we risk
713 * clamping giving us the wrong values
714 */
715 for (unsigned i = 0; i < 4; i++)
716 params.wm_inputs.clear_color[i] &= 0xffff;
717 } else {
718 wide_format = ISL_FORMAT_R32G32B32A32_UINT;
719 }
720
721 for (uint32_t a = 0; a < num_layers; a++) {
722 uint32_t layer = start_layer + a;
723
724 blorp_surface_info_init(batch, ¶ms.dst, surf, level,
725 layer, ISL_FORMAT_UNSUPPORTED, true);
726
727 if (surf->surf->samples > 1)
728 blorp_surf_fake_interleaved_msaa(batch->blorp->isl_dev, ¶ms.dst);
729
730 /* Make it Y-tiled */
731 blorp_surf_retile_w_to_y(batch->blorp->isl_dev, ¶ms.dst);
732
733 unsigned wide_Bpp =
734 isl_format_get_layout(wide_format)->bpb / 8;
735
736 params.dst.view.format = params.dst.surf.format = wide_format;
737 assert(params.dst.surf.logical_level0_px.width % wide_Bpp == 0);
738 params.dst.surf.logical_level0_px.width /= wide_Bpp;
739 assert(params.dst.tile_x_sa % wide_Bpp == 0);
740 params.dst.tile_x_sa /= wide_Bpp;
741
742 params.x0 = params.dst.tile_x_sa + x0 / (wide_Bpp / 2);
743 params.y0 = params.dst.tile_y_sa + y0 / 2;
744 params.x1 = params.dst.tile_x_sa + x1 / (wide_Bpp / 2);
745 params.y1 = params.dst.tile_y_sa + y1 / 2;
746
747 batch->blorp->exec(batch, ¶ms);
748 }
749
750 return true;
751 }
752
753 void
blorp_clear_depth_stencil(struct blorp_batch * batch,const struct blorp_surf * depth,const struct blorp_surf * stencil,uint32_t level,uint32_t start_layer,uint32_t num_layers,uint32_t x0,uint32_t y0,uint32_t x1,uint32_t y1,bool clear_depth,float depth_value,uint8_t stencil_mask,uint8_t stencil_value)754 blorp_clear_depth_stencil(struct blorp_batch *batch,
755 const struct blorp_surf *depth,
756 const struct blorp_surf *stencil,
757 uint32_t level, uint32_t start_layer,
758 uint32_t num_layers,
759 uint32_t x0, uint32_t y0, uint32_t x1, uint32_t y1,
760 bool clear_depth, float depth_value,
761 uint8_t stencil_mask, uint8_t stencil_value)
762 {
763 assert((batch->flags & BLORP_BATCH_USE_COMPUTE) == 0);
764
765 if (!clear_depth && blorp_clear_stencil_as_rgba(batch, stencil, level,
766 start_layer, num_layers,
767 x0, y0, x1, y1,
768 stencil_mask,
769 stencil_value))
770 return;
771
772 struct blorp_params params;
773 blorp_params_init(¶ms);
774 params.op = BLORP_OP_SLOW_DEPTH_CLEAR;
775
776 params.x0 = x0;
777 params.y0 = y0;
778 params.x1 = x1;
779 params.y1 = y1;
780
781 if (ISL_GFX_VER(batch->blorp->isl_dev) == 6) {
782 /* For some reason, Sandy Bridge gets occlusion queries wrong if we
783 * don't have a shader. In particular, it records samples even though
784 * we disable statistics in 3DSTATE_WM. Give it the usual clear shader
785 * to work around the issue.
786 */
787 if (!blorp_params_get_clear_kernel(batch, ¶ms, false, false))
788 return;
789 }
790
791 while (num_layers > 0) {
792 params.num_layers = num_layers;
793
794 if (stencil_mask) {
795 blorp_surface_info_init(batch, ¶ms.stencil, stencil,
796 level, start_layer,
797 ISL_FORMAT_UNSUPPORTED, true);
798 params.stencil_mask = stencil_mask;
799 params.stencil_ref = stencil_value;
800
801 params.dst.surf.samples = params.stencil.surf.samples;
802 params.dst.surf.logical_level0_px =
803 params.stencil.surf.logical_level0_px;
804 params.dst.view = params.stencil.view;
805
806 params.num_samples = params.stencil.surf.samples;
807
808 /* We may be restricted on the number of layers we can bind at any
809 * one time. In particular, Sandy Bridge has a maximum number of
810 * layers of 512 but a maximum 3D texture size is much larger.
811 */
812 if (params.stencil.view.array_len < params.num_layers)
813 params.num_layers = params.stencil.view.array_len;
814 }
815
816 if (clear_depth) {
817 blorp_surface_info_init(batch, ¶ms.depth, depth,
818 level, start_layer,
819 ISL_FORMAT_UNSUPPORTED, true);
820 params.z = depth_value;
821 params.depth_format =
822 isl_format_get_depth_format(depth->surf->format, false);
823
824 params.dst.surf.samples = params.depth.surf.samples;
825 params.dst.surf.logical_level0_px =
826 params.depth.surf.logical_level0_px;
827 params.dst.view = params.depth.view;
828
829 params.num_samples = params.depth.surf.samples;
830
831 /* We may be restricted on the number of layers we can bind at any
832 * one time. In particular, Sandy Bridge has a maximum number of
833 * layers of 512 but a maximum 3D texture size is much larger.
834 */
835 if (params.depth.view.array_len < params.num_layers)
836 params.num_layers = params.depth.view.array_len;
837 }
838
839 batch->blorp->exec(batch, ¶ms);
840
841 start_layer += params.num_layers;
842 num_layers -= params.num_layers;
843 }
844 }
845
846 bool
blorp_can_hiz_clear_depth(const struct intel_device_info * devinfo,const struct isl_surf * surf,enum isl_aux_usage aux_usage,uint32_t level,uint32_t layer,uint32_t x0,uint32_t y0,uint32_t x1,uint32_t y1)847 blorp_can_hiz_clear_depth(const struct intel_device_info *devinfo,
848 const struct isl_surf *surf,
849 enum isl_aux_usage aux_usage,
850 uint32_t level, uint32_t layer,
851 uint32_t x0, uint32_t y0, uint32_t x1, uint32_t y1)
852 {
853 /* This function currently doesn't support any gen prior to gfx8 */
854 assert(devinfo->ver >= 8);
855
856 if (devinfo->ver == 8 && surf->format == ISL_FORMAT_R16_UNORM) {
857 /* From the BDW PRM, Vol 7, "Depth Buffer Clear":
858 *
859 * The following restrictions apply only if the depth buffer surface
860 * type is D16_UNORM and software does not use the “full surf clear”:
861 *
862 * If Number of Multisamples is NUMSAMPLES_1, the rectangle must be
863 * aligned to an 8x4 pixel block relative to the upper left corner of
864 * the depth buffer, and contain an integer number of these pixel
865 * blocks, and all 8x4 pixels must be lit.
866 *
867 * Alignment requirements for other sample counts are listed, but they
868 * can all be satisfied by the one mentioned above.
869 */
870 if (x0 % 8 || y0 % 4 || x1 % 8 || y1 % 4)
871 return false;
872 } else if (isl_aux_usage_has_ccs(aux_usage)) {
873 /* We have to set the WM_HZ_OP::FullSurfaceDepthandStencilClear bit
874 * whenever we clear an uninitialized HIZ buffer (as some drivers
875 * currently do). However, this bit seems liable to clear 16x8 pixels in
876 * the ZCS on Gfx12 - greater than the slice alignments of many depth
877 * buffers.
878 *
879 * This is the hypothesis behind some corruption that was seen with the
880 * amd_vertex_shader_layer-layered-depth-texture-render piglit test.
881 *
882 * From the Compressed Depth Buffers section of the Bspec, under the
883 * Gfx12 texture performant and ZCS columns:
884 *
885 * Update with clear at either 16x8 or 8x4 granularity, based on
886 * fs_clr or otherwise.
887 *
888 * There are a number of ways to avoid full surface CCS clears that
889 * overlap other slices, but for now we choose to disable fast-clears
890 * when an initializing clear could hit another miplevel.
891 *
892 * NOTE: Because the CCS compresses the depth buffer and not a version
893 * of it that has been rearranged with different alignments (like Gfx8+
894 * HIZ), we have to make sure that the x0 and y0 are at least 16x8
895 * aligned in the context of the entire surface.
896 */
897 uint32_t slice_x0, slice_y0, slice_z0, slice_a0;
898 isl_surf_get_image_offset_el(surf, level,
899 surf->dim == ISL_SURF_DIM_3D ? 0 : layer,
900 surf->dim == ISL_SURF_DIM_3D ? layer: 0,
901 &slice_x0, &slice_y0, &slice_z0, &slice_a0);
902 const bool max_x1_y1 =
903 x1 == u_minify(surf->logical_level0_px.width, level) &&
904 y1 == u_minify(surf->logical_level0_px.height, level);
905 const uint32_t haligned_x1 = ALIGN(x1, surf->image_alignment_el.w);
906 const uint32_t valigned_y1 = ALIGN(y1, surf->image_alignment_el.h);
907 const bool unaligned = (slice_x0 + x0) % 16 || (slice_y0 + y0) % 8 ||
908 (max_x1_y1 ? haligned_x1 % 16 || valigned_y1 % 8 :
909 x1 % 16 || y1 % 8);
910 const bool partial_clear = x0 > 0 || y0 > 0 || !max_x1_y1;
911 const bool multislice_surf = surf->levels > 1 ||
912 surf->logical_level0_px.depth > 1 ||
913 surf->logical_level0_px.array_len > 1;
914
915 if (unaligned && (partial_clear || multislice_surf))
916 return false;
917 }
918
919 return isl_aux_usage_has_hiz(aux_usage);
920 }
921
922 static bool
blorp_can_clear_full_surface(const struct blorp_surf * depth,const struct blorp_surf * stencil,uint32_t level,uint32_t x0,uint32_t y0,uint32_t x1,uint32_t y1,bool clear_depth,bool clear_stencil)923 blorp_can_clear_full_surface(const struct blorp_surf *depth,
924 const struct blorp_surf *stencil,
925 uint32_t level,
926 uint32_t x0, uint32_t y0,
927 uint32_t x1, uint32_t y1,
928 bool clear_depth,
929 bool clear_stencil)
930 {
931 uint32_t width = 0, height = 0;
932 if (clear_stencil) {
933 width = u_minify(stencil->surf->logical_level0_px.width, level);
934 height = u_minify(stencil->surf->logical_level0_px.height, level);
935 }
936
937 if (clear_depth && !(width || height)) {
938 width = u_minify(depth->surf->logical_level0_px.width, level);
939 height = u_minify(depth->surf->logical_level0_px.height, level);
940 }
941
942 return x0 == 0 && y0 == 0 && width == x1 && height == y1;
943 }
944
945 void
blorp_hiz_clear_depth_stencil(struct blorp_batch * batch,const struct blorp_surf * depth,const struct blorp_surf * stencil,uint32_t level,uint32_t start_layer,uint32_t num_layers,uint32_t x0,uint32_t y0,uint32_t x1,uint32_t y1,bool clear_depth,float depth_value,bool clear_stencil,uint8_t stencil_value)946 blorp_hiz_clear_depth_stencil(struct blorp_batch *batch,
947 const struct blorp_surf *depth,
948 const struct blorp_surf *stencil,
949 uint32_t level,
950 uint32_t start_layer, uint32_t num_layers,
951 uint32_t x0, uint32_t y0,
952 uint32_t x1, uint32_t y1,
953 bool clear_depth, float depth_value,
954 bool clear_stencil, uint8_t stencil_value)
955 {
956 struct blorp_params params;
957 blorp_params_init(¶ms);
958 params.op = BLORP_OP_HIZ_CLEAR;
959
960 /* This requires WM_HZ_OP which only exists on gfx8+ */
961 assert(ISL_GFX_VER(batch->blorp->isl_dev) >= 8);
962
963 params.hiz_op = ISL_AUX_OP_FAST_CLEAR;
964 /* From BSpec: 3DSTATE_WM_HZ_OP_BODY >> Full Surface Depth and Stencil Clear
965 *
966 * "Software must set this only when the APP requires the entire Depth
967 * surface to be cleared."
968 */
969 params.full_surface_hiz_op =
970 blorp_can_clear_full_surface(depth, stencil, level, x0, y0, x1, y1,
971 clear_depth, clear_stencil);
972 params.num_layers = 1;
973
974 params.x0 = x0;
975 params.y0 = y0;
976 params.x1 = x1;
977 params.y1 = y1;
978
979 for (uint32_t l = 0; l < num_layers; l++) {
980 const uint32_t layer = start_layer + l;
981 if (clear_stencil) {
982 blorp_surface_info_init(batch, ¶ms.stencil, stencil,
983 level, layer,
984 ISL_FORMAT_UNSUPPORTED, true);
985 params.stencil_mask = 0xff;
986 params.stencil_ref = stencil_value;
987 params.num_samples = params.stencil.surf.samples;
988 }
989
990 if (clear_depth) {
991 /* If we're clearing depth, we must have HiZ */
992 assert(depth && isl_aux_usage_has_hiz(depth->aux_usage));
993
994 blorp_surface_info_init(batch, ¶ms.depth, depth,
995 level, layer,
996 ISL_FORMAT_UNSUPPORTED, true);
997 params.depth.clear_color.f32[0] = depth_value;
998 params.depth_format =
999 isl_format_get_depth_format(depth->surf->format, false);
1000 params.num_samples = params.depth.surf.samples;
1001 }
1002
1003 batch->blorp->exec(batch, ¶ms);
1004 }
1005 }
1006
1007 /* Given a depth stencil attachment, this function performs a fast depth clear
1008 * on a depth portion and a regular clear on the stencil portion. When
1009 * performing a fast depth clear on the depth portion, the HiZ buffer is simply
1010 * tagged as cleared so the depth clear value is not actually needed.
1011 */
1012 void
blorp_gfx8_hiz_clear_attachments(struct blorp_batch * batch,uint32_t num_samples,uint32_t x0,uint32_t y0,uint32_t x1,uint32_t y1,bool clear_depth,bool clear_stencil,uint8_t stencil_value)1013 blorp_gfx8_hiz_clear_attachments(struct blorp_batch *batch,
1014 uint32_t num_samples,
1015 uint32_t x0, uint32_t y0,
1016 uint32_t x1, uint32_t y1,
1017 bool clear_depth, bool clear_stencil,
1018 uint8_t stencil_value)
1019 {
1020 assert(batch->flags & BLORP_BATCH_NO_EMIT_DEPTH_STENCIL);
1021
1022 struct blorp_params params;
1023 blorp_params_init(¶ms);
1024 params.op = BLORP_OP_HIZ_CLEAR;
1025 params.num_layers = 1;
1026 params.hiz_op = ISL_AUX_OP_FAST_CLEAR;
1027 params.x0 = x0;
1028 params.y0 = y0;
1029 params.x1 = x1;
1030 params.y1 = y1;
1031 params.num_samples = num_samples;
1032 params.depth.enabled = clear_depth;
1033 params.stencil.enabled = clear_stencil;
1034 params.stencil_ref = stencil_value;
1035 batch->blorp->exec(batch, ¶ms);
1036 }
1037
1038 /** Clear active color/depth/stencili attachments
1039 *
1040 * This function performs a clear operation on the currently bound
1041 * color/depth/stencil attachments. It is assumed that any information passed
1042 * in here is valid, consistent, and in-bounds relative to the currently
1043 * attached depth/stencil. The binding_table_offset parameter is the 32-bit
1044 * offset relative to surface state base address where pre-baked binding table
1045 * that we are to use lives. If clear_color is false, binding_table_offset
1046 * must point to a binding table with one entry which is a valid null surface
1047 * that matches the currently bound depth and stencil.
1048 */
1049 void
blorp_clear_attachments(struct blorp_batch * batch,uint32_t binding_table_offset,enum isl_format depth_format,uint32_t num_samples,uint32_t start_layer,uint32_t num_layers,uint32_t x0,uint32_t y0,uint32_t x1,uint32_t y1,bool clear_color,union isl_color_value color_value,bool clear_depth,float depth_value,uint8_t stencil_mask,uint8_t stencil_value)1050 blorp_clear_attachments(struct blorp_batch *batch,
1051 uint32_t binding_table_offset,
1052 enum isl_format depth_format,
1053 uint32_t num_samples,
1054 uint32_t start_layer, uint32_t num_layers,
1055 uint32_t x0, uint32_t y0, uint32_t x1, uint32_t y1,
1056 bool clear_color, union isl_color_value color_value,
1057 bool clear_depth, float depth_value,
1058 uint8_t stencil_mask, uint8_t stencil_value)
1059 {
1060 struct blorp_params params;
1061 blorp_params_init(¶ms);
1062
1063 assert((batch->flags & BLORP_BATCH_USE_COMPUTE) == 0);
1064 assert(batch->flags & BLORP_BATCH_NO_EMIT_DEPTH_STENCIL);
1065
1066 params.x0 = x0;
1067 params.y0 = y0;
1068 params.x1 = x1;
1069 params.y1 = y1;
1070
1071 params.use_pre_baked_binding_table = true;
1072 params.pre_baked_binding_table_offset = binding_table_offset;
1073
1074 params.num_layers = num_layers;
1075 params.num_samples = num_samples;
1076
1077 if (clear_color) {
1078 params.dst.enabled = true;
1079 params.op = BLORP_OP_SLOW_COLOR_CLEAR;
1080
1081 memcpy(¶ms.wm_inputs.clear_color, color_value.f32, sizeof(float) * 4);
1082
1083 /* Unfortunately, without knowing whether or not our destination surface
1084 * is tiled or not, we have to assume it may be linear. This means no
1085 * SIMD16_REPDATA for us. :-(
1086 */
1087 if (!blorp_params_get_clear_kernel(batch, ¶ms, false, false))
1088 return;
1089 }
1090
1091 if (clear_depth) {
1092 params.depth.enabled = true;
1093 params.op = BLORP_OP_SLOW_DEPTH_CLEAR;
1094
1095 params.z = depth_value;
1096 params.depth_format = isl_format_get_depth_format(depth_format, false);
1097 }
1098
1099 if (stencil_mask) {
1100 params.stencil.enabled = true;
1101 params.op = BLORP_OP_SLOW_DEPTH_CLEAR;
1102
1103 params.stencil_mask = stencil_mask;
1104 params.stencil_ref = stencil_value;
1105 }
1106
1107 if (!blorp_params_get_layer_offset_vs(batch, ¶ms))
1108 return;
1109
1110 params.vs_inputs.base_layer = start_layer;
1111
1112 batch->blorp->exec(batch, ¶ms);
1113 }
1114
1115 void
blorp_ccs_resolve(struct blorp_batch * batch,struct blorp_surf * surf,uint32_t level,uint32_t start_layer,uint32_t num_layers,enum isl_format format,enum isl_aux_op resolve_op)1116 blorp_ccs_resolve(struct blorp_batch *batch,
1117 struct blorp_surf *surf, uint32_t level,
1118 uint32_t start_layer, uint32_t num_layers,
1119 enum isl_format format,
1120 enum isl_aux_op resolve_op)
1121 {
1122 assert((batch->flags & BLORP_BATCH_USE_COMPUTE) == 0);
1123 struct blorp_params params;
1124
1125 blorp_params_init(¶ms);
1126 switch(resolve_op) {
1127 case ISL_AUX_OP_AMBIGUATE:
1128 params.op = BLORP_OP_CCS_AMBIGUATE;
1129 break;
1130 case ISL_AUX_OP_FULL_RESOLVE:
1131 params.op = BLORP_OP_CCS_RESOLVE;
1132 break;
1133 case ISL_AUX_OP_PARTIAL_RESOLVE:
1134 params.op = BLORP_OP_CCS_PARTIAL_RESOLVE;
1135 break;
1136 default:
1137 assert(false);
1138 }
1139 blorp_surface_info_init(batch, ¶ms.dst, surf,
1140 level, start_layer, format, true);
1141
1142 params.x0 = params.y0 = 0;
1143 params.x1 = u_minify(params.dst.surf.logical_level0_px.width, level);
1144 params.y1 = u_minify(params.dst.surf.logical_level0_px.height, level);
1145 if (ISL_GFX_VER(batch->blorp->isl_dev) >= 9) {
1146 /* From Bspec 2424, "Render Target Resolve":
1147 *
1148 * The Resolve Rectangle size is same as Clear Rectangle size from
1149 * SKL+.
1150 *
1151 * Note that this differs from Vol7 of the Sky Lake PRM, which only
1152 * specifies aligning by the scaledown factors.
1153 */
1154 get_fast_clear_rect(batch->blorp->isl_dev, surf->surf, surf->aux_surf,
1155 ¶ms.x0, ¶ms.y0, ¶ms.x1, ¶ms.y1);
1156 } else {
1157 /* From the Ivy Bridge PRM, Vol2 Part1 11.9 "Render Target Resolve":
1158 *
1159 * A rectangle primitive must be scaled down by the following factors
1160 * with respect to render target being resolved.
1161 *
1162 * The scaledown factors in the table that follows are related to the
1163 * block size of the CCS format. For IVB and HSW, we divide by two, for
1164 * BDW we multiply by 8 and 16.
1165 */
1166 const struct isl_format_layout *aux_fmtl =
1167 isl_format_get_layout(params.dst.aux_surf.format);
1168 assert(aux_fmtl->txc == ISL_TXC_CCS);
1169
1170 unsigned x_scaledown, y_scaledown;
1171 if (ISL_GFX_VER(batch->blorp->isl_dev) >= 8) {
1172 x_scaledown = aux_fmtl->bw * 8;
1173 y_scaledown = aux_fmtl->bh * 16;
1174 } else {
1175 x_scaledown = aux_fmtl->bw / 2;
1176 y_scaledown = aux_fmtl->bh / 2;
1177 }
1178 params.x1 = ALIGN(params.x1, x_scaledown) / x_scaledown;
1179 params.y1 = ALIGN(params.y1, y_scaledown) / y_scaledown;
1180 }
1181
1182 if (batch->blorp->isl_dev->info->ver >= 10) {
1183 assert(resolve_op == ISL_AUX_OP_FULL_RESOLVE ||
1184 resolve_op == ISL_AUX_OP_PARTIAL_RESOLVE ||
1185 resolve_op == ISL_AUX_OP_AMBIGUATE);
1186 } else if (batch->blorp->isl_dev->info->ver >= 9) {
1187 assert(resolve_op == ISL_AUX_OP_FULL_RESOLVE ||
1188 resolve_op == ISL_AUX_OP_PARTIAL_RESOLVE);
1189 } else {
1190 /* Broadwell and earlier do not have a partial resolve */
1191 assert(resolve_op == ISL_AUX_OP_FULL_RESOLVE);
1192 }
1193 params.fast_clear_op = resolve_op;
1194 params.num_layers = num_layers;
1195
1196 /* Note: there is no need to initialize push constants because it doesn't
1197 * matter what data gets dispatched to the render target. However, we must
1198 * ensure that the fragment shader delivers the data using the "replicated
1199 * color" message.
1200 */
1201
1202 if (!blorp_params_get_clear_kernel(batch, ¶ms, true, false))
1203 return;
1204
1205 batch->blorp->exec(batch, ¶ms);
1206
1207 if (batch->blorp->isl_dev->info->ver <= 8) {
1208 assert(surf->aux_usage == ISL_AUX_USAGE_CCS_D);
1209 assert(resolve_op == ISL_AUX_OP_FULL_RESOLVE);
1210 /* ISL's state-machine of CCS_D describes full resolves as leaving the
1211 * aux buffer in the pass-through state. Hardware doesn't behave this
1212 * way on Broadwell however. On that platform, full resolves transition
1213 * the aux buffer to the resolved state. We assume that gfx7 behaves the
1214 * same. Use an ambiguate to match driver expectations.
1215 */
1216 for (int l = 0; l < num_layers; l++)
1217 blorp_ccs_ambiguate(batch, surf, level, start_layer + l);
1218 }
1219 }
1220
1221 static nir_def *
blorp_nir_bit(nir_builder * b,nir_def * src,unsigned bit)1222 blorp_nir_bit(nir_builder *b, nir_def *src, unsigned bit)
1223 {
1224 return nir_iand_imm(b, nir_ushr_imm(b, src, bit), 1);
1225 }
1226
1227 #pragma pack(push, 1)
1228 struct blorp_mcs_partial_resolve_key
1229 {
1230 struct blorp_base_key base;
1231 bool indirect_clear_color;
1232 bool int_format;
1233 uint32_t num_samples;
1234 };
1235 #pragma pack(pop)
1236
1237 static bool
blorp_params_get_mcs_partial_resolve_kernel(struct blorp_batch * batch,struct blorp_params * params)1238 blorp_params_get_mcs_partial_resolve_kernel(struct blorp_batch *batch,
1239 struct blorp_params *params)
1240 {
1241 struct blorp_context *blorp = batch->blorp;
1242 const struct blorp_mcs_partial_resolve_key blorp_key = {
1243 .base = BLORP_BASE_KEY_INIT(BLORP_SHADER_TYPE_MCS_PARTIAL_RESOLVE),
1244 .indirect_clear_color = params->dst.clear_color_addr.buffer != NULL,
1245 .int_format = isl_format_has_int_channel(params->dst.view.format),
1246 .num_samples = params->num_samples,
1247 };
1248
1249 if (blorp->lookup_shader(batch, &blorp_key, sizeof(blorp_key),
1250 ¶ms->wm_prog_kernel, ¶ms->wm_prog_data))
1251 return true;
1252
1253 void *mem_ctx = ralloc_context(NULL);
1254
1255 nir_builder b;
1256 blorp_nir_init_shader(&b, mem_ctx, MESA_SHADER_FRAGMENT,
1257 blorp_shader_type_to_name(blorp_key.base.shader_type));
1258
1259 nir_variable *v_color =
1260 BLORP_CREATE_NIR_INPUT(b.shader, clear_color, glsl_vec4_type());
1261
1262 nir_variable *frag_color =
1263 nir_variable_create(b.shader, nir_var_shader_out,
1264 glsl_vec4_type(), "gl_FragColor");
1265 frag_color->data.location = FRAG_RESULT_COLOR;
1266
1267 /* Do an MCS fetch and check if it is equal to the magic clear value */
1268 nir_def *mcs =
1269 blorp_nir_txf_ms_mcs(&b, nir_f2i32(&b, nir_load_frag_coord(&b)),
1270 nir_load_layer_id(&b));
1271 nir_def *is_clear =
1272 blorp_nir_mcs_is_clear_color(&b, mcs, blorp_key.num_samples);
1273
1274 /* If we aren't the clear value, discard. */
1275 nir_discard_if(&b, nir_inot(&b, is_clear));
1276
1277 nir_def *clear_color = nir_load_var(&b, v_color);
1278 if (blorp_key.indirect_clear_color && blorp->isl_dev->info->ver <= 8) {
1279 /* Gfx7-8 clear colors are stored as single 0/1 bits */
1280 clear_color = nir_vec4(&b, blorp_nir_bit(&b, clear_color, 31),
1281 blorp_nir_bit(&b, clear_color, 30),
1282 blorp_nir_bit(&b, clear_color, 29),
1283 blorp_nir_bit(&b, clear_color, 28));
1284
1285 if (!blorp_key.int_format)
1286 clear_color = nir_i2f32(&b, clear_color);
1287 }
1288 nir_store_var(&b, frag_color, clear_color, 0xf);
1289
1290 const bool multisample_fbo = true;
1291 const struct blorp_program p =
1292 blorp_compile_fs(blorp, mem_ctx, b.shader, multisample_fbo, false);
1293
1294 bool result =
1295 blorp->upload_shader(batch, MESA_SHADER_FRAGMENT,
1296 &blorp_key, sizeof(blorp_key),
1297 p.kernel, p.kernel_size,
1298 p.prog_data, p.prog_data_size,
1299 ¶ms->wm_prog_kernel, ¶ms->wm_prog_data);
1300
1301 ralloc_free(mem_ctx);
1302 return result;
1303 }
1304
1305 void
blorp_mcs_partial_resolve(struct blorp_batch * batch,struct blorp_surf * surf,enum isl_format format,uint32_t start_layer,uint32_t num_layers)1306 blorp_mcs_partial_resolve(struct blorp_batch *batch,
1307 struct blorp_surf *surf,
1308 enum isl_format format,
1309 uint32_t start_layer, uint32_t num_layers)
1310 {
1311 struct blorp_params params;
1312 blorp_params_init(¶ms);
1313 params.op = BLORP_OP_MCS_PARTIAL_RESOLVE;
1314
1315 assert(batch->blorp->isl_dev->info->ver >= 7);
1316
1317 params.x0 = 0;
1318 params.y0 = 0;
1319 params.x1 = surf->surf->logical_level0_px.width;
1320 params.y1 = surf->surf->logical_level0_px.height;
1321
1322 blorp_surface_info_init(batch, ¶ms.src, surf, 0,
1323 start_layer, format, false);
1324 blorp_surface_info_init(batch, ¶ms.dst, surf, 0,
1325 start_layer, format, true);
1326
1327 params.num_samples = params.dst.surf.samples;
1328 params.num_layers = num_layers;
1329 params.dst_clear_color_as_input = surf->clear_color_addr.buffer != NULL;
1330
1331 memcpy(¶ms.wm_inputs.clear_color,
1332 surf->clear_color.f32, sizeof(float) * 4);
1333
1334 if (!blorp_params_get_mcs_partial_resolve_kernel(batch, ¶ms))
1335 return;
1336
1337 batch->blorp->exec(batch, ¶ms);
1338 }
1339
1340 static uint64_t
get_mcs_ambiguate_pixel(int sample_count)1341 get_mcs_ambiguate_pixel(int sample_count)
1342 {
1343 /* See the Broadwell PRM, Volume 5 "Memory Views", Section "Compressed
1344 * Multisample Surfaces".
1345 */
1346 assert(sample_count >= 2);
1347 assert(sample_count <= 16);
1348
1349 /* Each MCS element contains an array of sample slice (SS) elements. The
1350 * size of this array matches the sample count.
1351 */
1352 const int num_ss_entries = sample_count;
1353
1354 /* The width of each SS entry is just large enough to index every slice. */
1355 const int ss_entry_size_b = util_logbase2(num_ss_entries);
1356
1357 /* The encoding for "ambiguated" has each sample slice value storing its
1358 * index (e.g., SS[0] = 0, SS[1] = 1, etc.). The values are stored in
1359 * little endian order. The unused bits are defined as either Reserved or
1360 * Reserved (MBZ). We choose to interpret both as MBZ.
1361 */
1362 uint64_t ambiguate_pixel = 0;
1363 for (uint64_t entry = 0; entry < num_ss_entries; entry++)
1364 ambiguate_pixel |= entry << (entry * ss_entry_size_b);
1365
1366 return ambiguate_pixel;
1367 }
1368
1369 /** Clear an MCS to the "uncompressed" state
1370 *
1371 * This pass is the MCS equivalent of a "HiZ resolve". It sets the MCS values
1372 * for a given layer of a surface to a sample-count dependent value which is
1373 * the "uncompressed" state which tells the sampler to go look at the main
1374 * surface.
1375 */
1376 void
blorp_mcs_ambiguate(struct blorp_batch * batch,struct blorp_surf * surf,uint32_t start_layer,uint32_t num_layers)1377 blorp_mcs_ambiguate(struct blorp_batch *batch,
1378 struct blorp_surf *surf,
1379 uint32_t start_layer, uint32_t num_layers)
1380 {
1381 assert((batch->flags & BLORP_BATCH_USE_COMPUTE) == 0);
1382
1383 struct blorp_params params;
1384 blorp_params_init(¶ms);
1385 params.op = BLORP_OP_MCS_AMBIGUATE;
1386
1387 assert(ISL_GFX_VER(batch->blorp->isl_dev) >= 7);
1388
1389 enum isl_format renderable_format;
1390 switch (isl_format_get_layout(surf->aux_surf->format)->bpb) {
1391 case 8: renderable_format = ISL_FORMAT_R8_UINT; break;
1392 case 32: renderable_format = ISL_FORMAT_R32_UINT; break;
1393 case 64: renderable_format = ISL_FORMAT_R32G32_UINT; break;
1394 default: unreachable("Unexpected MCS format size for ambiguate");
1395 }
1396
1397 params.dst = (struct blorp_surface_info) {
1398 .enabled = true,
1399 .surf = *surf->aux_surf,
1400 .addr = surf->aux_addr,
1401 .view = {
1402 .usage = ISL_SURF_USAGE_RENDER_TARGET_BIT,
1403 .format = renderable_format,
1404 .base_level = 0,
1405 .base_array_layer = start_layer,
1406 .levels = 1,
1407 .array_len = num_layers,
1408 .swizzle = ISL_SWIZZLE_IDENTITY,
1409 },
1410 };
1411
1412 params.x0 = 0;
1413 params.y0 = 0;
1414 params.x1 = params.dst.surf.logical_level0_px.width;
1415 params.y1 = params.dst.surf.logical_level0_px.height;
1416 params.num_layers = params.dst.view.array_len;
1417
1418 const uint64_t pixel = get_mcs_ambiguate_pixel(surf->surf->samples);
1419 params.wm_inputs.clear_color[0] = pixel & 0xFFFFFFFF;
1420 params.wm_inputs.clear_color[1] = pixel >> 32;
1421
1422 if (!blorp_params_get_clear_kernel(batch, ¶ms, true, false))
1423 return;
1424
1425 batch->blorp->exec(batch, ¶ms);
1426 }
1427
1428 /** Clear a CCS to the "uncompressed" state
1429 *
1430 * This pass is the CCS equivalent of a "HiZ resolve". It sets the CCS values
1431 * for a given layer/level of a surface to 0x0 which is the "uncompressed"
1432 * state which tells the sampler to go look at the main surface.
1433 */
1434 void
blorp_ccs_ambiguate(struct blorp_batch * batch,struct blorp_surf * surf,uint32_t level,uint32_t layer)1435 blorp_ccs_ambiguate(struct blorp_batch *batch,
1436 struct blorp_surf *surf,
1437 uint32_t level, uint32_t layer)
1438 {
1439 assert((batch->flags & BLORP_BATCH_USE_COMPUTE) == 0);
1440
1441 if (ISL_GFX_VER(batch->blorp->isl_dev) >= 10) {
1442 /* On gfx10 and above, we have a hardware resolve op for this */
1443 return blorp_ccs_resolve(batch, surf, level, layer, 1,
1444 surf->surf->format, ISL_AUX_OP_AMBIGUATE);
1445 }
1446
1447 struct blorp_params params;
1448 blorp_params_init(¶ms);
1449 params.op = BLORP_OP_CCS_AMBIGUATE;
1450
1451 assert(ISL_GFX_VER(batch->blorp->isl_dev) >= 7);
1452
1453 const struct isl_format_layout *aux_fmtl =
1454 isl_format_get_layout(surf->aux_surf->format);
1455 assert(aux_fmtl->txc == ISL_TXC_CCS);
1456
1457 params.dst = (struct blorp_surface_info) {
1458 .enabled = true,
1459 .addr = surf->aux_addr,
1460 .view = {
1461 .usage = ISL_SURF_USAGE_RENDER_TARGET_BIT,
1462 .format = ISL_FORMAT_R32G32B32A32_UINT,
1463 .base_level = 0,
1464 .base_array_layer = 0,
1465 .levels = 1,
1466 .array_len = 1,
1467 .swizzle = ISL_SWIZZLE_IDENTITY,
1468 },
1469 };
1470
1471 uint32_t z = 0;
1472 if (surf->surf->dim == ISL_SURF_DIM_3D) {
1473 z = layer;
1474 layer = 0;
1475 }
1476
1477 uint64_t offset_B;
1478 uint32_t x_offset_el, y_offset_el;
1479 isl_surf_get_image_offset_B_tile_el(surf->aux_surf, level, layer, z,
1480 &offset_B, &x_offset_el, &y_offset_el);
1481 params.dst.addr.offset += offset_B;
1482
1483 const uint32_t width_px =
1484 u_minify(surf->aux_surf->logical_level0_px.width, level);
1485 const uint32_t height_px =
1486 u_minify(surf->aux_surf->logical_level0_px.height, level);
1487 const uint32_t width_el = DIV_ROUND_UP(width_px, aux_fmtl->bw);
1488 const uint32_t height_el = DIV_ROUND_UP(height_px, aux_fmtl->bh);
1489
1490 struct isl_tile_info ccs_tile_info;
1491 isl_surf_get_tile_info(surf->aux_surf, &ccs_tile_info);
1492
1493 /* We're going to map it as a regular RGBA32_UINT surface. We need to
1494 * downscale a good deal. We start by computing the area on the CCS to
1495 * clear in units of Y-tiled cache lines.
1496 */
1497 uint32_t x_offset_cl, y_offset_cl, width_cl, height_cl;
1498 if (ISL_GFX_VER(batch->blorp->isl_dev) >= 8) {
1499 /* From the Sky Lake PRM Vol. 12 in the section on planes:
1500 *
1501 * "The Color Control Surface (CCS) contains the compression status
1502 * of the cache-line pairs. The compression state of the cache-line
1503 * pair is specified by 2 bits in the CCS. Each CCS cache-line
1504 * represents an area on the main surface of 16x16 sets of 128 byte
1505 * Y-tiled cache-line-pairs. CCS is always Y tiled."
1506 *
1507 * Each 2-bit surface element in the CCS corresponds to a single
1508 * cache-line pair in the main surface. This means that 16x16 el block
1509 * in the CCS maps to a Y-tiled cache line. Fortunately, CCS layouts
1510 * are calculated with a very large alignment so we can round up to a
1511 * whole cache line without worrying about overdraw.
1512 */
1513
1514 /* On Broadwell and above, a CCS tile is the same as a Y tile when
1515 * viewed at the cache-line granularity. Fortunately, the horizontal
1516 * and vertical alignment requirements of the CCS are such that we can
1517 * align to an entire cache line without worrying about crossing over
1518 * from one LOD to another.
1519 */
1520 const uint32_t x_el_per_cl = ccs_tile_info.logical_extent_el.w / 8;
1521 const uint32_t y_el_per_cl = ccs_tile_info.logical_extent_el.h / 8;
1522 assert(surf->aux_surf->image_alignment_el.w % x_el_per_cl == 0);
1523 assert(surf->aux_surf->image_alignment_el.h % y_el_per_cl == 0);
1524
1525 assert(x_offset_el % x_el_per_cl == 0);
1526 assert(y_offset_el % y_el_per_cl == 0);
1527 x_offset_cl = x_offset_el / x_el_per_cl;
1528 y_offset_cl = y_offset_el / y_el_per_cl;
1529 width_cl = DIV_ROUND_UP(width_el, x_el_per_cl);
1530 height_cl = DIV_ROUND_UP(height_el, y_el_per_cl);
1531 } else {
1532 /* On gfx7, the CCS tiling is not so nice. However, there we are
1533 * guaranteed that we only have a single level and slice so we don't
1534 * have to worry about it and can just align to a whole tile.
1535 */
1536 assert(surf->aux_surf->logical_level0_px.depth == 1);
1537 assert(surf->aux_surf->logical_level0_px.array_len == 1);
1538 assert(x_offset_el == 0 && y_offset_el == 0);
1539 const uint32_t width_tl =
1540 DIV_ROUND_UP(width_el, ccs_tile_info.logical_extent_el.w);
1541 const uint32_t height_tl =
1542 DIV_ROUND_UP(height_el, ccs_tile_info.logical_extent_el.h);
1543 x_offset_cl = 0;
1544 y_offset_cl = 0;
1545 width_cl = width_tl * 8;
1546 height_cl = height_tl * 8;
1547 }
1548
1549 /* We're going to use a RGBA32 format so as to write data as quickly as
1550 * possible. A y-tiled cache line will then be 1x4 px.
1551 */
1552 const uint32_t x_offset_rgba_px = x_offset_cl;
1553 const uint32_t y_offset_rgba_px = y_offset_cl * 4;
1554 const uint32_t width_rgba_px = width_cl;
1555 const uint32_t height_rgba_px = height_cl * 4;
1556
1557 ASSERTED bool ok =
1558 isl_surf_init(batch->blorp->isl_dev, ¶ms.dst.surf,
1559 .dim = ISL_SURF_DIM_2D,
1560 .format = ISL_FORMAT_R32G32B32A32_UINT,
1561 .width = width_rgba_px + x_offset_rgba_px,
1562 .height = height_rgba_px + y_offset_rgba_px,
1563 .depth = 1,
1564 .levels = 1,
1565 .array_len = 1,
1566 .samples = 1,
1567 .row_pitch_B = surf->aux_surf->row_pitch_B,
1568 .usage = ISL_SURF_USAGE_RENDER_TARGET_BIT,
1569 .tiling_flags = ISL_TILING_Y0_BIT);
1570 assert(ok);
1571
1572 params.x0 = x_offset_rgba_px;
1573 params.y0 = y_offset_rgba_px;
1574 params.x1 = x_offset_rgba_px + width_rgba_px;
1575 params.y1 = y_offset_rgba_px + height_rgba_px;
1576
1577 /* A CCS value of 0 means "uncompressed." */
1578 memset(¶ms.wm_inputs.clear_color, 0,
1579 sizeof(params.wm_inputs.clear_color));
1580
1581 if (!blorp_params_get_clear_kernel(batch, ¶ms, true, false))
1582 return;
1583
1584 batch->blorp->exec(batch, ¶ms);
1585 }
1586