1 /*
2 * Copyright © 2012 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "main/context.h"
25 #include "main/teximage.h"
26 #include "main/blend.h"
27 #include "main/bufferobj.h"
28 #include "main/enums.h"
29 #include "main/fbobject.h"
30 #include "main/image.h"
31 #include "main/renderbuffer.h"
32 #include "main/glformats.h"
33
34 #include "brw_blorp.h"
35 #include "brw_context.h"
36 #include "brw_defines.h"
37 #include "brw_meta_util.h"
38 #include "brw_state.h"
39 #include "intel_buffer_objects.h"
40 #include "intel_fbo.h"
41 #include "common/gen_debug.h"
42
43 #define FILE_DEBUG_FLAG DEBUG_BLORP
44
45 static bool
brw_blorp_lookup_shader(struct blorp_context * blorp,const void * key,uint32_t key_size,uint32_t * kernel_out,void * prog_data_out)46 brw_blorp_lookup_shader(struct blorp_context *blorp,
47 const void *key, uint32_t key_size,
48 uint32_t *kernel_out, void *prog_data_out)
49 {
50 struct brw_context *brw = blorp->driver_ctx;
51 return brw_search_cache(&brw->cache, BRW_CACHE_BLORP_PROG,
52 key, key_size, kernel_out, prog_data_out);
53 }
54
55 static bool
brw_blorp_upload_shader(struct blorp_context * blorp,const void * key,uint32_t key_size,const void * kernel,uint32_t kernel_size,const struct brw_stage_prog_data * prog_data,uint32_t prog_data_size,uint32_t * kernel_out,void * prog_data_out)56 brw_blorp_upload_shader(struct blorp_context *blorp,
57 const void *key, uint32_t key_size,
58 const void *kernel, uint32_t kernel_size,
59 const struct brw_stage_prog_data *prog_data,
60 uint32_t prog_data_size,
61 uint32_t *kernel_out, void *prog_data_out)
62 {
63 struct brw_context *brw = blorp->driver_ctx;
64 brw_upload_cache(&brw->cache, BRW_CACHE_BLORP_PROG, key, key_size,
65 kernel, kernel_size, prog_data, prog_data_size,
66 kernel_out, prog_data_out);
67 return true;
68 }
69
70 void
brw_blorp_init(struct brw_context * brw)71 brw_blorp_init(struct brw_context *brw)
72 {
73 const struct gen_device_info *devinfo = &brw->screen->devinfo;
74
75 blorp_init(&brw->blorp, brw, &brw->isl_dev);
76
77 brw->blorp.compiler = brw->screen->compiler;
78
79 switch (devinfo->gen) {
80 case 4:
81 if (devinfo->is_g4x) {
82 brw->blorp.exec = gen45_blorp_exec;
83 } else {
84 brw->blorp.exec = gen4_blorp_exec;
85 }
86 break;
87 case 5:
88 brw->blorp.exec = gen5_blorp_exec;
89 break;
90 case 6:
91 brw->blorp.exec = gen6_blorp_exec;
92 break;
93 case 7:
94 if (devinfo->is_haswell) {
95 brw->blorp.exec = gen75_blorp_exec;
96 } else {
97 brw->blorp.exec = gen7_blorp_exec;
98 }
99 break;
100 case 8:
101 brw->blorp.exec = gen8_blorp_exec;
102 break;
103 case 9:
104 brw->blorp.exec = gen9_blorp_exec;
105 break;
106 case 10:
107 brw->blorp.exec = gen10_blorp_exec;
108 break;
109 default:
110 unreachable("Invalid gen");
111 }
112
113 brw->blorp.lookup_shader = brw_blorp_lookup_shader;
114 brw->blorp.upload_shader = brw_blorp_upload_shader;
115 }
116
117 static void
blorp_surf_for_miptree(struct brw_context * brw,struct blorp_surf * surf,struct intel_mipmap_tree * mt,enum isl_aux_usage aux_usage,bool is_render_target,unsigned * level,unsigned start_layer,unsigned num_layers,struct isl_surf tmp_surfs[1])118 blorp_surf_for_miptree(struct brw_context *brw,
119 struct blorp_surf *surf,
120 struct intel_mipmap_tree *mt,
121 enum isl_aux_usage aux_usage,
122 bool is_render_target,
123 unsigned *level,
124 unsigned start_layer, unsigned num_layers,
125 struct isl_surf tmp_surfs[1])
126 {
127 const struct gen_device_info *devinfo = &brw->screen->devinfo;
128
129 if (mt->surf.msaa_layout == ISL_MSAA_LAYOUT_ARRAY) {
130 const unsigned num_samples = mt->surf.samples;
131 for (unsigned i = 0; i < num_layers; i++) {
132 for (unsigned s = 0; s < num_samples; s++) {
133 const unsigned phys_layer = (start_layer + i) * num_samples + s;
134 intel_miptree_check_level_layer(mt, *level, phys_layer);
135 }
136 }
137 } else {
138 for (unsigned i = 0; i < num_layers; i++)
139 intel_miptree_check_level_layer(mt, *level, start_layer + i);
140 }
141
142 *surf = (struct blorp_surf) {
143 .surf = &mt->surf,
144 .addr = (struct blorp_address) {
145 .buffer = mt->bo,
146 .offset = mt->offset,
147 .reloc_flags = is_render_target ? EXEC_OBJECT_WRITE : 0,
148 .mocs = brw_get_bo_mocs(devinfo, mt->bo),
149 },
150 .aux_usage = aux_usage,
151 };
152
153 struct isl_surf *aux_surf = NULL;
154 if (mt->mcs_buf)
155 aux_surf = &mt->mcs_buf->surf;
156 else if (mt->hiz_buf)
157 aux_surf = &mt->hiz_buf->surf;
158
159 if (mt->format == MESA_FORMAT_S_UINT8 && is_render_target &&
160 devinfo->gen <= 7)
161 mt->r8stencil_needs_update = true;
162
163 if (surf->aux_usage == ISL_AUX_USAGE_HIZ &&
164 !intel_miptree_level_has_hiz(mt, *level))
165 surf->aux_usage = ISL_AUX_USAGE_NONE;
166
167 if (surf->aux_usage != ISL_AUX_USAGE_NONE) {
168 /* We only really need a clear color if we also have an auxiliary
169 * surface. Without one, it does nothing.
170 */
171 surf->clear_color = mt->fast_clear_color;
172
173 surf->aux_surf = aux_surf;
174 surf->aux_addr = (struct blorp_address) {
175 .reloc_flags = is_render_target ? EXEC_OBJECT_WRITE : 0,
176 .mocs = surf->addr.mocs,
177 };
178
179 if (mt->mcs_buf) {
180 surf->aux_addr.buffer = mt->mcs_buf->bo;
181 surf->aux_addr.offset = mt->mcs_buf->offset;
182 } else {
183 assert(mt->hiz_buf);
184 assert(surf->aux_usage == ISL_AUX_USAGE_HIZ);
185
186 surf->aux_addr.buffer = mt->hiz_buf->bo;
187 surf->aux_addr.offset = mt->hiz_buf->offset;
188 }
189 } else {
190 surf->aux_addr = (struct blorp_address) {
191 .buffer = NULL,
192 };
193 memset(&surf->clear_color, 0, sizeof(surf->clear_color));
194 }
195 assert((surf->aux_usage == ISL_AUX_USAGE_NONE) ==
196 (surf->aux_addr.buffer == NULL));
197
198 /* ISL wants real levels, not offset ones. */
199 *level -= mt->first_level;
200 }
201
202 static enum isl_format
brw_blorp_to_isl_format(struct brw_context * brw,mesa_format format,bool is_render_target)203 brw_blorp_to_isl_format(struct brw_context *brw, mesa_format format,
204 bool is_render_target)
205 {
206 switch (format) {
207 case MESA_FORMAT_NONE:
208 return ISL_FORMAT_UNSUPPORTED;
209 case MESA_FORMAT_S_UINT8:
210 return ISL_FORMAT_R8_UINT;
211 case MESA_FORMAT_Z24_UNORM_X8_UINT:
212 case MESA_FORMAT_Z24_UNORM_S8_UINT:
213 return ISL_FORMAT_R24_UNORM_X8_TYPELESS;
214 case MESA_FORMAT_Z_FLOAT32:
215 case MESA_FORMAT_Z32_FLOAT_S8X24_UINT:
216 return ISL_FORMAT_R32_FLOAT;
217 case MESA_FORMAT_Z_UNORM16:
218 return ISL_FORMAT_R16_UNORM;
219 default: {
220 if (is_render_target) {
221 assert(brw->mesa_format_supports_render[format]);
222 return brw->mesa_to_isl_render_format[format];
223 } else {
224 return brw_isl_format_for_mesa_format(format);
225 }
226 break;
227 }
228 }
229 }
230
231 /**
232 * Convert an swizzle enumeration (i.e. SWIZZLE_X) to one of the Gen7.5+
233 * "Shader Channel Select" enumerations (i.e. HSW_SCS_RED). The mappings are
234 *
235 * SWIZZLE_X, SWIZZLE_Y, SWIZZLE_Z, SWIZZLE_W, SWIZZLE_ZERO, SWIZZLE_ONE
236 * 0 1 2 3 4 5
237 * 4 5 6 7 0 1
238 * SCS_RED, SCS_GREEN, SCS_BLUE, SCS_ALPHA, SCS_ZERO, SCS_ONE
239 *
240 * which is simply adding 4 then modding by 8 (or anding with 7).
241 *
242 * We then may need to apply workarounds for textureGather hardware bugs.
243 */
244 static enum isl_channel_select
swizzle_to_scs(GLenum swizzle)245 swizzle_to_scs(GLenum swizzle)
246 {
247 return (enum isl_channel_select)((swizzle + 4) & 7);
248 }
249
250 /**
251 * Note: if the src (or dst) is a 2D multisample array texture on Gen7+ using
252 * INTEL_MSAA_LAYOUT_UMS or INTEL_MSAA_LAYOUT_CMS, src_layer (dst_layer) is
253 * the physical layer holding sample 0. So, for example, if
254 * src_mt->surf.samples == 4, then logical layer n corresponds to src_layer ==
255 * 4*n.
256 */
257 void
brw_blorp_blit_miptrees(struct brw_context * brw,struct intel_mipmap_tree * src_mt,unsigned src_level,unsigned src_layer,mesa_format src_format,int src_swizzle,struct intel_mipmap_tree * dst_mt,unsigned dst_level,unsigned dst_layer,mesa_format dst_format,float src_x0,float src_y0,float src_x1,float src_y1,float dst_x0,float dst_y0,float dst_x1,float dst_y1,GLenum filter,bool mirror_x,bool mirror_y,bool decode_srgb,bool encode_srgb)258 brw_blorp_blit_miptrees(struct brw_context *brw,
259 struct intel_mipmap_tree *src_mt,
260 unsigned src_level, unsigned src_layer,
261 mesa_format src_format, int src_swizzle,
262 struct intel_mipmap_tree *dst_mt,
263 unsigned dst_level, unsigned dst_layer,
264 mesa_format dst_format,
265 float src_x0, float src_y0,
266 float src_x1, float src_y1,
267 float dst_x0, float dst_y0,
268 float dst_x1, float dst_y1,
269 GLenum filter, bool mirror_x, bool mirror_y,
270 bool decode_srgb, bool encode_srgb)
271 {
272 const struct gen_device_info *devinfo = &brw->screen->devinfo;
273
274 DBG("%s from %dx %s mt %p %d %d (%f,%f) (%f,%f)"
275 "to %dx %s mt %p %d %d (%f,%f) (%f,%f) (flip %d,%d)\n",
276 __func__,
277 src_mt->surf.samples, _mesa_get_format_name(src_mt->format), src_mt,
278 src_level, src_layer, src_x0, src_y0, src_x1, src_y1,
279 dst_mt->surf.samples, _mesa_get_format_name(dst_mt->format), dst_mt,
280 dst_level, dst_layer, dst_x0, dst_y0, dst_x1, dst_y1,
281 mirror_x, mirror_y);
282
283 if (!decode_srgb && _mesa_get_format_color_encoding(src_format) == GL_SRGB)
284 src_format = _mesa_get_srgb_format_linear(src_format);
285
286 if (!encode_srgb && _mesa_get_format_color_encoding(dst_format) == GL_SRGB)
287 dst_format = _mesa_get_srgb_format_linear(dst_format);
288
289 /* When doing a multisample resolve of a GL_LUMINANCE32F or GL_INTENSITY32F
290 * texture, the above code configures the source format for L32_FLOAT or
291 * I32_FLOAT, and the destination format for R32_FLOAT. On Sandy Bridge,
292 * the SAMPLE message appears to handle multisampled L32_FLOAT and
293 * I32_FLOAT textures incorrectly, resulting in blocky artifacts. So work
294 * around the problem by using a source format of R32_FLOAT. This
295 * shouldn't affect rendering correctness, since the destination format is
296 * R32_FLOAT, so only the contents of the red channel matters.
297 */
298 if (devinfo->gen == 6 &&
299 src_mt->surf.samples > 1 && dst_mt->surf.samples <= 1 &&
300 src_mt->format == dst_mt->format &&
301 (dst_format == MESA_FORMAT_L_FLOAT32 ||
302 dst_format == MESA_FORMAT_I_FLOAT32)) {
303 src_format = dst_format = MESA_FORMAT_R_FLOAT32;
304 }
305
306 enum isl_format src_isl_format =
307 brw_blorp_to_isl_format(brw, src_format, false);
308 enum isl_aux_usage src_aux_usage =
309 intel_miptree_texture_aux_usage(brw, src_mt, src_isl_format);
310 /* We do format workarounds for some depth formats so we can't reliably
311 * sample with HiZ. One of these days, we should fix that.
312 */
313 if (src_aux_usage == ISL_AUX_USAGE_HIZ)
314 src_aux_usage = ISL_AUX_USAGE_NONE;
315 const bool src_clear_supported =
316 src_aux_usage != ISL_AUX_USAGE_NONE && src_mt->format == src_format;
317 intel_miptree_prepare_access(brw, src_mt, src_level, 1, src_layer, 1,
318 src_aux_usage, src_clear_supported);
319
320 enum isl_format dst_isl_format =
321 brw_blorp_to_isl_format(brw, dst_format, true);
322 enum isl_aux_usage dst_aux_usage =
323 intel_miptree_render_aux_usage(brw, dst_mt, dst_isl_format,
324 false, false);
325 const bool dst_clear_supported = dst_aux_usage != ISL_AUX_USAGE_NONE;
326 intel_miptree_prepare_access(brw, dst_mt, dst_level, 1, dst_layer, 1,
327 dst_aux_usage, dst_clear_supported);
328
329 struct isl_surf tmp_surfs[2];
330 struct blorp_surf src_surf, dst_surf;
331 blorp_surf_for_miptree(brw, &src_surf, src_mt, src_aux_usage, false,
332 &src_level, src_layer, 1, &tmp_surfs[0]);
333 blorp_surf_for_miptree(brw, &dst_surf, dst_mt, dst_aux_usage, true,
334 &dst_level, dst_layer, 1, &tmp_surfs[1]);
335
336 struct isl_swizzle src_isl_swizzle = {
337 .r = swizzle_to_scs(GET_SWZ(src_swizzle, 0)),
338 .g = swizzle_to_scs(GET_SWZ(src_swizzle, 1)),
339 .b = swizzle_to_scs(GET_SWZ(src_swizzle, 2)),
340 .a = swizzle_to_scs(GET_SWZ(src_swizzle, 3)),
341 };
342
343 struct blorp_batch batch;
344 blorp_batch_init(&brw->blorp, &batch, brw, 0);
345 blorp_blit(&batch, &src_surf, src_level, src_layer,
346 src_isl_format, src_isl_swizzle,
347 &dst_surf, dst_level, dst_layer,
348 dst_isl_format, ISL_SWIZZLE_IDENTITY,
349 src_x0, src_y0, src_x1, src_y1,
350 dst_x0, dst_y0, dst_x1, dst_y1,
351 filter, mirror_x, mirror_y);
352 blorp_batch_finish(&batch);
353
354 intel_miptree_finish_write(brw, dst_mt, dst_level, dst_layer, 1,
355 dst_aux_usage);
356 }
357
358 void
brw_blorp_copy_miptrees(struct brw_context * brw,struct intel_mipmap_tree * src_mt,unsigned src_level,unsigned src_layer,struct intel_mipmap_tree * dst_mt,unsigned dst_level,unsigned dst_layer,unsigned src_x,unsigned src_y,unsigned dst_x,unsigned dst_y,unsigned src_width,unsigned src_height)359 brw_blorp_copy_miptrees(struct brw_context *brw,
360 struct intel_mipmap_tree *src_mt,
361 unsigned src_level, unsigned src_layer,
362 struct intel_mipmap_tree *dst_mt,
363 unsigned dst_level, unsigned dst_layer,
364 unsigned src_x, unsigned src_y,
365 unsigned dst_x, unsigned dst_y,
366 unsigned src_width, unsigned src_height)
367 {
368 const struct gen_device_info *devinfo = &brw->screen->devinfo;
369
370 DBG("%s from %dx %s mt %p %d %d (%d,%d) %dx%d"
371 "to %dx %s mt %p %d %d (%d,%d)\n",
372 __func__,
373 src_mt->surf.samples, _mesa_get_format_name(src_mt->format), src_mt,
374 src_level, src_layer, src_x, src_y, src_width, src_height,
375 dst_mt->surf.samples, _mesa_get_format_name(dst_mt->format), dst_mt,
376 dst_level, dst_layer, dst_x, dst_y);
377
378 enum isl_aux_usage src_aux_usage, dst_aux_usage;
379 bool src_clear_supported, dst_clear_supported;
380
381 switch (src_mt->aux_usage) {
382 case ISL_AUX_USAGE_MCS:
383 case ISL_AUX_USAGE_CCS_E:
384 src_aux_usage = src_mt->aux_usage;
385 /* Prior to gen9, fast-clear only supported 0/1 clear colors. Since
386 * we're going to re-interpret the format as an integer format possibly
387 * with a different number of components, we can't handle clear colors
388 * until gen9.
389 */
390 src_clear_supported = devinfo->gen >= 9;
391 break;
392 default:
393 src_aux_usage = ISL_AUX_USAGE_NONE;
394 src_clear_supported = false;
395 break;
396 }
397
398 switch (dst_mt->aux_usage) {
399 case ISL_AUX_USAGE_MCS:
400 case ISL_AUX_USAGE_CCS_E:
401 dst_aux_usage = dst_mt->aux_usage;
402 /* Prior to gen9, fast-clear only supported 0/1 clear colors. Since
403 * we're going to re-interpret the format as an integer format possibly
404 * with a different number of components, we can't handle clear colors
405 * until gen9.
406 */
407 dst_clear_supported = devinfo->gen >= 9;
408 break;
409 default:
410 dst_aux_usage = ISL_AUX_USAGE_NONE;
411 dst_clear_supported = false;
412 break;
413 }
414
415 intel_miptree_prepare_access(brw, src_mt, src_level, 1, src_layer, 1,
416 src_aux_usage, src_clear_supported);
417 intel_miptree_prepare_access(brw, dst_mt, dst_level, 1, dst_layer, 1,
418 dst_aux_usage, dst_clear_supported);
419
420 struct isl_surf tmp_surfs[2];
421 struct blorp_surf src_surf, dst_surf;
422 blorp_surf_for_miptree(brw, &src_surf, src_mt, src_aux_usage, false,
423 &src_level, src_layer, 1, &tmp_surfs[0]);
424 blorp_surf_for_miptree(brw, &dst_surf, dst_mt, dst_aux_usage, true,
425 &dst_level, dst_layer, 1, &tmp_surfs[1]);
426
427 /* The hardware seems to have issues with having a two different format
428 * views of the same texture in the sampler cache at the same time. It's
429 * unclear exactly what the issue is but it hurts glCopyImageSubData
430 * particularly badly because it does a lot of format reinterprets. We
431 * badly need better understanding of the issue and a better fix but this
432 * works for now and fixes CTS tests.
433 *
434 * TODO: Remove this hack!
435 */
436 brw_emit_pipe_control_flush(brw, PIPE_CONTROL_CS_STALL |
437 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
438
439 struct blorp_batch batch;
440 blorp_batch_init(&brw->blorp, &batch, brw, 0);
441 blorp_copy(&batch, &src_surf, src_level, src_layer,
442 &dst_surf, dst_level, dst_layer,
443 src_x, src_y, dst_x, dst_y, src_width, src_height);
444 blorp_batch_finish(&batch);
445
446 brw_emit_pipe_control_flush(brw, PIPE_CONTROL_CS_STALL |
447 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
448
449 intel_miptree_finish_write(brw, dst_mt, dst_level, dst_layer, 1,
450 dst_aux_usage);
451 }
452
453 void
brw_blorp_copy_buffers(struct brw_context * brw,struct brw_bo * src_bo,unsigned src_offset,struct brw_bo * dst_bo,unsigned dst_offset,unsigned size)454 brw_blorp_copy_buffers(struct brw_context *brw,
455 struct brw_bo *src_bo,
456 unsigned src_offset,
457 struct brw_bo *dst_bo,
458 unsigned dst_offset,
459 unsigned size)
460 {
461 DBG("%s %d bytes from %p[%d] to %p[%d]",
462 __func__, size, src_bo, src_offset, dst_bo, dst_offset);
463
464 struct blorp_batch batch;
465 struct blorp_address src = { .buffer = src_bo, .offset = src_offset };
466 struct blorp_address dst = { .buffer = dst_bo, .offset = dst_offset };
467
468 blorp_batch_init(&brw->blorp, &batch, brw, 0);
469 blorp_buffer_copy(&batch, src, dst, size);
470 blorp_batch_finish(&batch);
471 }
472
473
474 static struct intel_mipmap_tree *
find_miptree(GLbitfield buffer_bit,struct intel_renderbuffer * irb)475 find_miptree(GLbitfield buffer_bit, struct intel_renderbuffer *irb)
476 {
477 struct intel_mipmap_tree *mt = irb->mt;
478 if (buffer_bit == GL_STENCIL_BUFFER_BIT && mt->stencil_mt)
479 mt = mt->stencil_mt;
480 return mt;
481 }
482
483 static int
blorp_get_texture_swizzle(const struct intel_renderbuffer * irb)484 blorp_get_texture_swizzle(const struct intel_renderbuffer *irb)
485 {
486 return irb->Base.Base._BaseFormat == GL_RGB ?
487 MAKE_SWIZZLE4(SWIZZLE_X, SWIZZLE_Y, SWIZZLE_Z, SWIZZLE_ONE) :
488 SWIZZLE_XYZW;
489 }
490
491 static void
do_blorp_blit(struct brw_context * brw,GLbitfield buffer_bit,struct intel_renderbuffer * src_irb,mesa_format src_format,struct intel_renderbuffer * dst_irb,mesa_format dst_format,GLfloat srcX0,GLfloat srcY0,GLfloat srcX1,GLfloat srcY1,GLfloat dstX0,GLfloat dstY0,GLfloat dstX1,GLfloat dstY1,GLenum filter,bool mirror_x,bool mirror_y)492 do_blorp_blit(struct brw_context *brw, GLbitfield buffer_bit,
493 struct intel_renderbuffer *src_irb, mesa_format src_format,
494 struct intel_renderbuffer *dst_irb, mesa_format dst_format,
495 GLfloat srcX0, GLfloat srcY0, GLfloat srcX1, GLfloat srcY1,
496 GLfloat dstX0, GLfloat dstY0, GLfloat dstX1, GLfloat dstY1,
497 GLenum filter, bool mirror_x, bool mirror_y)
498 {
499 const struct gl_context *ctx = &brw->ctx;
500
501 /* Find source/dst miptrees */
502 struct intel_mipmap_tree *src_mt = find_miptree(buffer_bit, src_irb);
503 struct intel_mipmap_tree *dst_mt = find_miptree(buffer_bit, dst_irb);
504
505 const bool do_srgb = ctx->Color.sRGBEnabled;
506
507 /* Do the blit */
508 brw_blorp_blit_miptrees(brw,
509 src_mt, src_irb->mt_level, src_irb->mt_layer,
510 src_format, blorp_get_texture_swizzle(src_irb),
511 dst_mt, dst_irb->mt_level, dst_irb->mt_layer,
512 dst_format,
513 srcX0, srcY0, srcX1, srcY1,
514 dstX0, dstY0, dstX1, dstY1,
515 filter, mirror_x, mirror_y,
516 do_srgb, do_srgb);
517
518 dst_irb->need_downsample = true;
519 }
520
521 static bool
try_blorp_blit(struct brw_context * brw,const struct gl_framebuffer * read_fb,const struct gl_framebuffer * draw_fb,GLfloat srcX0,GLfloat srcY0,GLfloat srcX1,GLfloat srcY1,GLfloat dstX0,GLfloat dstY0,GLfloat dstX1,GLfloat dstY1,GLenum filter,GLbitfield buffer_bit)522 try_blorp_blit(struct brw_context *brw,
523 const struct gl_framebuffer *read_fb,
524 const struct gl_framebuffer *draw_fb,
525 GLfloat srcX0, GLfloat srcY0, GLfloat srcX1, GLfloat srcY1,
526 GLfloat dstX0, GLfloat dstY0, GLfloat dstX1, GLfloat dstY1,
527 GLenum filter, GLbitfield buffer_bit)
528 {
529 const struct gen_device_info *devinfo = &brw->screen->devinfo;
530 struct gl_context *ctx = &brw->ctx;
531
532 /* Sync up the state of window system buffers. We need to do this before
533 * we go looking for the buffers.
534 */
535 intel_prepare_render(brw);
536
537 bool mirror_x, mirror_y;
538 if (brw_meta_mirror_clip_and_scissor(ctx, read_fb, draw_fb,
539 &srcX0, &srcY0, &srcX1, &srcY1,
540 &dstX0, &dstY0, &dstX1, &dstY1,
541 &mirror_x, &mirror_y))
542 return true;
543
544 /* Find buffers */
545 struct intel_renderbuffer *src_irb;
546 struct intel_renderbuffer *dst_irb;
547 struct intel_mipmap_tree *src_mt;
548 struct intel_mipmap_tree *dst_mt;
549 switch (buffer_bit) {
550 case GL_COLOR_BUFFER_BIT:
551 src_irb = intel_renderbuffer(read_fb->_ColorReadBuffer);
552 for (unsigned i = 0; i < draw_fb->_NumColorDrawBuffers; ++i) {
553 dst_irb = intel_renderbuffer(draw_fb->_ColorDrawBuffers[i]);
554 if (dst_irb)
555 do_blorp_blit(brw, buffer_bit,
556 src_irb, src_irb->Base.Base.Format,
557 dst_irb, dst_irb->Base.Base.Format,
558 srcX0, srcY0, srcX1, srcY1,
559 dstX0, dstY0, dstX1, dstY1,
560 filter, mirror_x, mirror_y);
561 }
562 break;
563 case GL_DEPTH_BUFFER_BIT:
564 src_irb =
565 intel_renderbuffer(read_fb->Attachment[BUFFER_DEPTH].Renderbuffer);
566 dst_irb =
567 intel_renderbuffer(draw_fb->Attachment[BUFFER_DEPTH].Renderbuffer);
568 src_mt = find_miptree(buffer_bit, src_irb);
569 dst_mt = find_miptree(buffer_bit, dst_irb);
570
571 /* We can't handle format conversions between Z24 and other formats
572 * since we have to lie about the surface format. See the comments in
573 * brw_blorp_surface_info::set().
574 */
575 if ((src_mt->format == MESA_FORMAT_Z24_UNORM_X8_UINT) !=
576 (dst_mt->format == MESA_FORMAT_Z24_UNORM_X8_UINT))
577 return false;
578
579 /* We also can't handle any combined depth-stencil formats because we
580 * have to reinterpret as a color format.
581 */
582 if (_mesa_get_format_base_format(src_mt->format) == GL_DEPTH_STENCIL ||
583 _mesa_get_format_base_format(dst_mt->format) == GL_DEPTH_STENCIL)
584 return false;
585
586 do_blorp_blit(brw, buffer_bit, src_irb, MESA_FORMAT_NONE,
587 dst_irb, MESA_FORMAT_NONE, srcX0, srcY0,
588 srcX1, srcY1, dstX0, dstY0, dstX1, dstY1,
589 filter, mirror_x, mirror_y);
590 break;
591 case GL_STENCIL_BUFFER_BIT:
592 /* Blorp doesn't support combined depth stencil which is all we have
593 * prior to gen6.
594 */
595 if (devinfo->gen < 6)
596 return false;
597
598 src_irb =
599 intel_renderbuffer(read_fb->Attachment[BUFFER_STENCIL].Renderbuffer);
600 dst_irb =
601 intel_renderbuffer(draw_fb->Attachment[BUFFER_STENCIL].Renderbuffer);
602 do_blorp_blit(brw, buffer_bit, src_irb, MESA_FORMAT_NONE,
603 dst_irb, MESA_FORMAT_NONE, srcX0, srcY0,
604 srcX1, srcY1, dstX0, dstY0, dstX1, dstY1,
605 filter, mirror_x, mirror_y);
606 break;
607 default:
608 unreachable("not reached");
609 }
610
611 return true;
612 }
613
614 static void
apply_y_flip(int * y0,int * y1,int height)615 apply_y_flip(int *y0, int *y1, int height)
616 {
617 int tmp = height - *y0;
618 *y0 = height - *y1;
619 *y1 = tmp;
620 }
621
622 bool
brw_blorp_copytexsubimage(struct brw_context * brw,struct gl_renderbuffer * src_rb,struct gl_texture_image * dst_image,int slice,int srcX0,int srcY0,int dstX0,int dstY0,int width,int height)623 brw_blorp_copytexsubimage(struct brw_context *brw,
624 struct gl_renderbuffer *src_rb,
625 struct gl_texture_image *dst_image,
626 int slice,
627 int srcX0, int srcY0,
628 int dstX0, int dstY0,
629 int width, int height)
630 {
631 struct gl_context *ctx = &brw->ctx;
632 struct intel_renderbuffer *src_irb = intel_renderbuffer(src_rb);
633 struct intel_texture_image *intel_image = intel_texture_image(dst_image);
634
635 /* No pixel transfer operations (zoom, bias, mapping), just a blit */
636 if (brw->ctx._ImageTransferState)
637 return false;
638
639 /* Sync up the state of window system buffers. We need to do this before
640 * we go looking at the src renderbuffer's miptree.
641 */
642 intel_prepare_render(brw);
643
644 struct intel_mipmap_tree *src_mt = src_irb->mt;
645 struct intel_mipmap_tree *dst_mt = intel_image->mt;
646
647 /* There is support for only up to eight samples. */
648 if (src_mt->surf.samples > 8 || dst_mt->surf.samples > 8)
649 return false;
650
651 if (_mesa_get_format_base_format(src_rb->Format) !=
652 _mesa_get_format_base_format(dst_image->TexFormat)) {
653 return false;
654 }
655
656 /* We can't handle format conversions between Z24 and other formats since
657 * we have to lie about the surface format. See the comments in
658 * brw_blorp_surface_info::set().
659 */
660 if ((src_mt->format == MESA_FORMAT_Z24_UNORM_X8_UINT) !=
661 (dst_mt->format == MESA_FORMAT_Z24_UNORM_X8_UINT)) {
662 return false;
663 }
664
665 /* We also can't handle any combined depth-stencil formats because we
666 * have to reinterpret as a color format.
667 */
668 if (_mesa_get_format_base_format(src_mt->format) == GL_DEPTH_STENCIL ||
669 _mesa_get_format_base_format(dst_mt->format) == GL_DEPTH_STENCIL)
670 return false;
671
672 if (!brw->mesa_format_supports_render[dst_image->TexFormat])
673 return false;
674
675 /* Source clipping shouldn't be necessary, since copytexsubimage (in
676 * src/mesa/main/teximage.c) calls _mesa_clip_copytexsubimage() which
677 * takes care of it.
678 *
679 * Destination clipping shouldn't be necessary since the restrictions on
680 * glCopyTexSubImage prevent the user from specifying a destination rectangle
681 * that falls outside the bounds of the destination texture.
682 * See error_check_subtexture_dimensions().
683 */
684
685 int srcY1 = srcY0 + height;
686 int srcX1 = srcX0 + width;
687 int dstX1 = dstX0 + width;
688 int dstY1 = dstY0 + height;
689
690 /* Account for the fact that in the system framebuffer, the origin is at
691 * the lower left.
692 */
693 bool mirror_y = _mesa_is_winsys_fbo(ctx->ReadBuffer);
694 if (mirror_y)
695 apply_y_flip(&srcY0, &srcY1, src_rb->Height);
696
697 /* Account for face selection and texture view MinLayer */
698 int dst_slice = slice + dst_image->TexObject->MinLayer + dst_image->Face;
699 int dst_level = dst_image->Level + dst_image->TexObject->MinLevel;
700
701 brw_blorp_blit_miptrees(brw,
702 src_mt, src_irb->mt_level, src_irb->mt_layer,
703 src_rb->Format, blorp_get_texture_swizzle(src_irb),
704 dst_mt, dst_level, dst_slice,
705 dst_image->TexFormat,
706 srcX0, srcY0, srcX1, srcY1,
707 dstX0, dstY0, dstX1, dstY1,
708 GL_NEAREST, false, mirror_y,
709 false, false);
710
711 /* If we're copying to a packed depth stencil texture and the source
712 * framebuffer has separate stencil, we need to also copy the stencil data
713 * over.
714 */
715 src_rb = ctx->ReadBuffer->Attachment[BUFFER_STENCIL].Renderbuffer;
716 if (_mesa_get_format_bits(dst_image->TexFormat, GL_STENCIL_BITS) > 0 &&
717 src_rb != NULL) {
718 src_irb = intel_renderbuffer(src_rb);
719 src_mt = src_irb->mt;
720
721 if (src_mt->stencil_mt)
722 src_mt = src_mt->stencil_mt;
723 if (dst_mt->stencil_mt)
724 dst_mt = dst_mt->stencil_mt;
725
726 if (src_mt != dst_mt) {
727 brw_blorp_blit_miptrees(brw,
728 src_mt, src_irb->mt_level, src_irb->mt_layer,
729 src_mt->format,
730 blorp_get_texture_swizzle(src_irb),
731 dst_mt, dst_level, dst_slice,
732 dst_mt->format,
733 srcX0, srcY0, srcX1, srcY1,
734 dstX0, dstY0, dstX1, dstY1,
735 GL_NEAREST, false, mirror_y,
736 false, false);
737 }
738 }
739
740 return true;
741 }
742
743
744 GLbitfield
brw_blorp_framebuffer(struct brw_context * brw,struct gl_framebuffer * readFb,struct gl_framebuffer * drawFb,GLint srcX0,GLint srcY0,GLint srcX1,GLint srcY1,GLint dstX0,GLint dstY0,GLint dstX1,GLint dstY1,GLbitfield mask,GLenum filter)745 brw_blorp_framebuffer(struct brw_context *brw,
746 struct gl_framebuffer *readFb,
747 struct gl_framebuffer *drawFb,
748 GLint srcX0, GLint srcY0, GLint srcX1, GLint srcY1,
749 GLint dstX0, GLint dstY0, GLint dstX1, GLint dstY1,
750 GLbitfield mask, GLenum filter)
751 {
752 static GLbitfield buffer_bits[] = {
753 GL_COLOR_BUFFER_BIT,
754 GL_DEPTH_BUFFER_BIT,
755 GL_STENCIL_BUFFER_BIT,
756 };
757
758 for (unsigned int i = 0; i < ARRAY_SIZE(buffer_bits); ++i) {
759 if ((mask & buffer_bits[i]) &&
760 try_blorp_blit(brw, readFb, drawFb,
761 srcX0, srcY0, srcX1, srcY1,
762 dstX0, dstY0, dstX1, dstY1,
763 filter, buffer_bits[i])) {
764 mask &= ~buffer_bits[i];
765 }
766 }
767
768 return mask;
769 }
770
771 static struct brw_bo *
blorp_get_client_bo(struct brw_context * brw,unsigned w,unsigned h,unsigned d,GLenum target,GLenum format,GLenum type,const void * pixels,const struct gl_pixelstore_attrib * packing,uint32_t * offset_out,uint32_t * row_stride_out,uint32_t * image_stride_out,bool read_only)772 blorp_get_client_bo(struct brw_context *brw,
773 unsigned w, unsigned h, unsigned d,
774 GLenum target, GLenum format, GLenum type,
775 const void *pixels,
776 const struct gl_pixelstore_attrib *packing,
777 uint32_t *offset_out, uint32_t *row_stride_out,
778 uint32_t *image_stride_out, bool read_only)
779 {
780 /* Account for SKIP_PIXELS, SKIP_ROWS, ALIGNMENT, and SKIP_IMAGES */
781 const GLuint dims = _mesa_get_texture_dimensions(target);
782 const uint32_t first_pixel = _mesa_image_offset(dims, packing, w, h,
783 format, type, 0, 0, 0);
784 const uint32_t last_pixel = _mesa_image_offset(dims, packing, w, h,
785 format, type,
786 d - 1, h - 1, w);
787 const uint32_t stride = _mesa_image_row_stride(packing, w, format, type);
788 const uint32_t cpp = _mesa_bytes_per_pixel(format, type);
789 const uint32_t size = last_pixel - first_pixel;
790
791 *row_stride_out = stride;
792 *image_stride_out = _mesa_image_image_stride(packing, w, h, format, type);
793
794 if (_mesa_is_bufferobj(packing->BufferObj)) {
795 const uint32_t offset = first_pixel + (intptr_t)pixels;
796 if (!read_only && ((offset % cpp) || (stride % cpp))) {
797 perf_debug("Bad PBO alignment; fallback to CPU mapping\n");
798 return NULL;
799 }
800
801 /* This is a user-provided PBO. We just need to get the BO out */
802 struct intel_buffer_object *intel_pbo =
803 intel_buffer_object(packing->BufferObj);
804 struct brw_bo *bo =
805 intel_bufferobj_buffer(brw, intel_pbo, offset, size, !read_only);
806
807 /* We take a reference to the BO so that the caller can just always
808 * unref without having to worry about whether it's a user PBO or one
809 * we created.
810 */
811 brw_bo_reference(bo);
812
813 *offset_out = offset;
814 return bo;
815 } else {
816 /* Someone should have already checked that there is data to upload. */
817 assert(pixels);
818
819 /* Creating a temp buffer currently only works for upload */
820 assert(read_only);
821
822 /* This is not a user-provided PBO. Instead, pixels is a pointer to CPU
823 * data which we need to copy into a BO.
824 */
825 struct brw_bo *bo =
826 brw_bo_alloc(brw->bufmgr, "tmp_tex_subimage_src", size, 64);
827 if (bo == NULL) {
828 perf_debug("intel_texsubimage: temp bo creation failed: size = %u\n",
829 size);
830 return NULL;
831 }
832
833 if (brw_bo_subdata(bo, 0, size, pixels + first_pixel)) {
834 perf_debug("intel_texsubimage: temp bo upload failed\n");
835 brw_bo_unreference(bo);
836 return NULL;
837 }
838
839 *offset_out = 0;
840 return bo;
841 }
842 }
843
844 /* Consider all the restrictions and determine the format of the source. */
845 static mesa_format
blorp_get_client_format(struct brw_context * brw,GLenum format,GLenum type,const struct gl_pixelstore_attrib * packing)846 blorp_get_client_format(struct brw_context *brw,
847 GLenum format, GLenum type,
848 const struct gl_pixelstore_attrib *packing)
849 {
850 if (brw->ctx._ImageTransferState)
851 return MESA_FORMAT_NONE;
852
853 if (packing->SwapBytes || packing->LsbFirst || packing->Invert) {
854 perf_debug("intel_texsubimage_blorp: unsupported gl_pixelstore_attrib\n");
855 return MESA_FORMAT_NONE;
856 }
857
858 if (format != GL_RED &&
859 format != GL_RG &&
860 format != GL_RGB &&
861 format != GL_BGR &&
862 format != GL_RGBA &&
863 format != GL_BGRA &&
864 format != GL_ALPHA &&
865 format != GL_RED_INTEGER &&
866 format != GL_RG_INTEGER &&
867 format != GL_RGB_INTEGER &&
868 format != GL_BGR_INTEGER &&
869 format != GL_RGBA_INTEGER &&
870 format != GL_BGRA_INTEGER) {
871 perf_debug("intel_texsubimage_blorp: %s not supported",
872 _mesa_enum_to_string(format));
873 return MESA_FORMAT_NONE;
874 }
875
876 return _mesa_tex_format_from_format_and_type(&brw->ctx, format, type);
877 }
878
879 static bool
need_signed_unsigned_int_conversion(mesa_format src_format,mesa_format dst_format)880 need_signed_unsigned_int_conversion(mesa_format src_format,
881 mesa_format dst_format)
882 {
883 const GLenum src_type = _mesa_get_format_datatype(src_format);
884 const GLenum dst_type = _mesa_get_format_datatype(dst_format);
885 return (src_type == GL_INT && dst_type == GL_UNSIGNED_INT) ||
886 (src_type == GL_UNSIGNED_INT && dst_type == GL_INT);
887 }
888
889 bool
brw_blorp_upload_miptree(struct brw_context * brw,struct intel_mipmap_tree * dst_mt,mesa_format dst_format,uint32_t level,uint32_t x,uint32_t y,uint32_t z,uint32_t width,uint32_t height,uint32_t depth,GLenum target,GLenum format,GLenum type,const void * pixels,const struct gl_pixelstore_attrib * packing)890 brw_blorp_upload_miptree(struct brw_context *brw,
891 struct intel_mipmap_tree *dst_mt,
892 mesa_format dst_format,
893 uint32_t level, uint32_t x, uint32_t y, uint32_t z,
894 uint32_t width, uint32_t height, uint32_t depth,
895 GLenum target, GLenum format, GLenum type,
896 const void *pixels,
897 const struct gl_pixelstore_attrib *packing)
898 {
899 const mesa_format src_format =
900 blorp_get_client_format(brw, format, type, packing);
901 if (src_format == MESA_FORMAT_NONE)
902 return false;
903
904 if (!brw->mesa_format_supports_render[dst_format]) {
905 perf_debug("intel_texsubimage: can't use %s as render target\n",
906 _mesa_get_format_name(dst_format));
907 return false;
908 }
909
910 /* This function relies on blorp_blit to upload the pixel data to the
911 * miptree. But, blorp_blit doesn't support signed to unsigned or
912 * unsigned to signed integer conversions.
913 */
914 if (need_signed_unsigned_int_conversion(src_format, dst_format))
915 return false;
916
917 uint32_t src_offset, src_row_stride, src_image_stride;
918 struct brw_bo *src_bo =
919 blorp_get_client_bo(brw, width, height, depth,
920 target, format, type, pixels, packing,
921 &src_offset, &src_row_stride,
922 &src_image_stride, true);
923 if (src_bo == NULL)
924 return false;
925
926 /* Now that source is offset to correct starting point, adjust the
927 * given dimensions to treat 1D arrays as 2D.
928 */
929 if (target == GL_TEXTURE_1D_ARRAY) {
930 assert(depth == 1);
931 assert(z == 0);
932 depth = height;
933 height = 1;
934 z = y;
935 y = 0;
936 src_image_stride = src_row_stride;
937 }
938
939 intel_miptree_check_level_layer(dst_mt, level, z + depth - 1);
940
941 bool result = false;
942
943 /* Blit slice-by-slice creating a single-slice miptree for each layer. Even
944 * in case of linear buffers hardware wants image arrays to be aligned by
945 * four rows. This way hardware only gets one image at a time and any
946 * source alignment will do.
947 */
948 for (unsigned i = 0; i < depth; ++i) {
949 struct intel_mipmap_tree *src_mt = intel_miptree_create_for_bo(
950 brw, src_bo, src_format,
951 src_offset + i * src_image_stride,
952 width, height, 1,
953 src_row_stride,
954 ISL_TILING_LINEAR, 0);
955
956 if (!src_mt) {
957 perf_debug("intel_texsubimage: miptree creation for src failed\n");
958 goto err;
959 }
960
961 /* In case exact match is needed, copy using equivalent UINT formats
962 * preventing hardware from changing presentation for SNORM -1.
963 */
964 if (src_mt->format == dst_format) {
965 brw_blorp_copy_miptrees(brw, src_mt, 0, 0,
966 dst_mt, level, z + i,
967 0, 0, x, y, width, height);
968 } else {
969 brw_blorp_blit_miptrees(brw, src_mt, 0, 0,
970 src_format, SWIZZLE_XYZW,
971 dst_mt, level, z + i,
972 dst_format,
973 0, 0, width, height,
974 x, y, x + width, y + height,
975 GL_NEAREST, false, false, false, false);
976 }
977
978 intel_miptree_release(&src_mt);
979 }
980
981 result = true;
982
983 err:
984 brw_bo_unreference(src_bo);
985
986 return result;
987 }
988
989 bool
brw_blorp_download_miptree(struct brw_context * brw,struct intel_mipmap_tree * src_mt,mesa_format src_format,uint32_t src_swizzle,uint32_t level,uint32_t x,uint32_t y,uint32_t z,uint32_t width,uint32_t height,uint32_t depth,GLenum target,GLenum format,GLenum type,bool y_flip,const void * pixels,const struct gl_pixelstore_attrib * packing)990 brw_blorp_download_miptree(struct brw_context *brw,
991 struct intel_mipmap_tree *src_mt,
992 mesa_format src_format, uint32_t src_swizzle,
993 uint32_t level, uint32_t x, uint32_t y, uint32_t z,
994 uint32_t width, uint32_t height, uint32_t depth,
995 GLenum target, GLenum format, GLenum type,
996 bool y_flip, const void *pixels,
997 const struct gl_pixelstore_attrib *packing)
998 {
999 const mesa_format dst_format =
1000 blorp_get_client_format(brw, format, type, packing);
1001 if (dst_format == MESA_FORMAT_NONE)
1002 return false;
1003
1004 if (!brw->mesa_format_supports_render[dst_format]) {
1005 perf_debug("intel_texsubimage: can't use %s as render target\n",
1006 _mesa_get_format_name(dst_format));
1007 return false;
1008 }
1009
1010 /* This function relies on blorp_blit to download the pixel data from the
1011 * miptree. But, blorp_blit doesn't support signed to unsigned or unsigned
1012 * to signed integer conversions.
1013 */
1014 if (need_signed_unsigned_int_conversion(src_format, dst_format))
1015 return false;
1016
1017 /* We can't fetch from LUMINANCE or intensity as that would require a
1018 * non-trivial swizzle.
1019 */
1020 switch (_mesa_get_format_base_format(src_format)) {
1021 case GL_LUMINANCE:
1022 case GL_LUMINANCE_ALPHA:
1023 case GL_INTENSITY:
1024 return false;
1025 default:
1026 break;
1027 }
1028
1029 /* This pass only works for PBOs */
1030 assert(_mesa_is_bufferobj(packing->BufferObj));
1031
1032 uint32_t dst_offset, dst_row_stride, dst_image_stride;
1033 struct brw_bo *dst_bo =
1034 blorp_get_client_bo(brw, width, height, depth,
1035 target, format, type, pixels, packing,
1036 &dst_offset, &dst_row_stride,
1037 &dst_image_stride, false);
1038 if (dst_bo == NULL)
1039 return false;
1040
1041 /* Now that source is offset to correct starting point, adjust the
1042 * given dimensions to treat 1D arrays as 2D.
1043 */
1044 if (target == GL_TEXTURE_1D_ARRAY) {
1045 assert(depth == 1);
1046 assert(z == 0);
1047 depth = height;
1048 height = 1;
1049 z = y;
1050 y = 0;
1051 dst_image_stride = dst_row_stride;
1052 }
1053
1054 intel_miptree_check_level_layer(src_mt, level, z + depth - 1);
1055
1056 int y0 = y;
1057 int y1 = y + height;
1058 if (y_flip) {
1059 apply_y_flip(&y0, &y1, minify(src_mt->surf.phys_level0_sa.height,
1060 level - src_mt->first_level));
1061 }
1062
1063 bool result = false;
1064
1065 /* Blit slice-by-slice creating a single-slice miptree for each layer. Even
1066 * in case of linear buffers hardware wants image arrays to be aligned by
1067 * four rows. This way hardware only gets one image at a time and any
1068 * source alignment will do.
1069 */
1070 for (unsigned i = 0; i < depth; ++i) {
1071 struct intel_mipmap_tree *dst_mt = intel_miptree_create_for_bo(
1072 brw, dst_bo, dst_format,
1073 dst_offset + i * dst_image_stride,
1074 width, height, 1,
1075 dst_row_stride,
1076 ISL_TILING_LINEAR, 0);
1077
1078 if (!dst_mt) {
1079 perf_debug("intel_texsubimage: miptree creation for src failed\n");
1080 goto err;
1081 }
1082
1083 /* In case exact match is needed, copy using equivalent UINT formats
1084 * preventing hardware from changing presentation for SNORM -1.
1085 */
1086 if (dst_mt->format == src_format && !y_flip &&
1087 src_swizzle == SWIZZLE_XYZW) {
1088 brw_blorp_copy_miptrees(brw, src_mt, level, z + i,
1089 dst_mt, 0, 0,
1090 x, y, 0, 0, width, height);
1091 } else {
1092 brw_blorp_blit_miptrees(brw, src_mt, level, z + i,
1093 src_format, src_swizzle,
1094 dst_mt, 0, 0, dst_format,
1095 x, y0, x + width, y1,
1096 0, 0, width, height,
1097 GL_NEAREST, false, y_flip, false, false);
1098 }
1099
1100 intel_miptree_release(&dst_mt);
1101 }
1102
1103 result = true;
1104
1105 /* As we implement PBO transfers by binding the user-provided BO as a
1106 * fake framebuffer and rendering to it. This breaks the invariant of the
1107 * GL that nothing is able to render to a BO, causing nondeterministic
1108 * corruption issues because the render cache is not coherent with a
1109 * number of other caches that the BO could potentially be bound to
1110 * afterwards.
1111 *
1112 * This could be solved in the same way that we guarantee texture
1113 * coherency after a texture is attached to a framebuffer and
1114 * rendered to, but that would involve checking *all* BOs bound to
1115 * the pipeline for the case we need to emit a cache flush due to
1116 * previous rendering to any of them -- Including vertex, index,
1117 * uniform, atomic counter, shader image, transform feedback,
1118 * indirect draw buffers, etc.
1119 *
1120 * That would increase the per-draw call overhead even though it's
1121 * very unlikely that any of the BOs bound to the pipeline has been
1122 * rendered to via a PBO at any point, so it seems better to just
1123 * flush here unconditionally.
1124 */
1125 brw_emit_mi_flush(brw);
1126
1127 err:
1128 brw_bo_unreference(dst_bo);
1129
1130 return result;
1131 }
1132
1133 static bool
set_write_disables(const struct intel_renderbuffer * irb,const GLubyte * color_mask,bool * color_write_disable)1134 set_write_disables(const struct intel_renderbuffer *irb,
1135 const GLubyte *color_mask, bool *color_write_disable)
1136 {
1137 /* Format information in the renderbuffer represents the requirements
1138 * given by the client. There are cases where the backing miptree uses,
1139 * for example, RGBA to represent RGBX. Since the client is only expecting
1140 * RGB we can treat alpha as not used and write whatever we like into it.
1141 */
1142 const GLenum base_format = irb->Base.Base._BaseFormat;
1143 const int components = _mesa_base_format_component_count(base_format);
1144 bool disables = false;
1145
1146 assert(components > 0);
1147
1148 for (int i = 0; i < components; i++) {
1149 color_write_disable[i] = !color_mask[i];
1150 disables = disables || !color_mask[i];
1151 }
1152
1153 return disables;
1154 }
1155
1156 static void
do_single_blorp_clear(struct brw_context * brw,struct gl_framebuffer * fb,struct gl_renderbuffer * rb,unsigned buf,bool partial_clear,bool encode_srgb)1157 do_single_blorp_clear(struct brw_context *brw, struct gl_framebuffer *fb,
1158 struct gl_renderbuffer *rb, unsigned buf,
1159 bool partial_clear, bool encode_srgb)
1160 {
1161 struct gl_context *ctx = &brw->ctx;
1162 struct intel_renderbuffer *irb = intel_renderbuffer(rb);
1163 uint32_t x0, x1, y0, y1;
1164
1165 mesa_format format = irb->Base.Base.Format;
1166 if (!encode_srgb && _mesa_get_format_color_encoding(format) == GL_SRGB)
1167 format = _mesa_get_srgb_format_linear(format);
1168 enum isl_format isl_format = brw->mesa_to_isl_render_format[format];
1169
1170 x0 = fb->_Xmin;
1171 x1 = fb->_Xmax;
1172 if (rb->Name != 0) {
1173 y0 = fb->_Ymin;
1174 y1 = fb->_Ymax;
1175 } else {
1176 y0 = rb->Height - fb->_Ymax;
1177 y1 = rb->Height - fb->_Ymin;
1178 }
1179
1180 /* If the clear region is empty, just return. */
1181 if (x0 == x1 || y0 == y1)
1182 return;
1183
1184 bool can_fast_clear = !partial_clear;
1185
1186 bool color_write_disable[4] = { false, false, false, false };
1187 if (set_write_disables(irb, ctx->Color.ColorMask[buf], color_write_disable))
1188 can_fast_clear = false;
1189
1190 /* We store clear colors as floats or uints as needed. If there are
1191 * texture views in play, the formats will not properly be respected
1192 * during resolves because the resolve operations only know about the
1193 * miptree and not the renderbuffer.
1194 */
1195 if (irb->Base.Base.Format != irb->mt->format)
1196 can_fast_clear = false;
1197
1198 if (!irb->mt->supports_fast_clear ||
1199 !brw_is_color_fast_clear_compatible(brw, irb->mt, &ctx->Color.ClearColor))
1200 can_fast_clear = false;
1201
1202 /* Surface state can only record one fast clear color value. Therefore
1203 * unless different levels/layers agree on the color it can be used to
1204 * represent only single level/layer. Here it will be reserved for the
1205 * first slice (level 0, layer 0).
1206 */
1207 if (irb->layer_count > 1 || irb->mt_level || irb->mt_layer)
1208 can_fast_clear = false;
1209
1210 unsigned level = irb->mt_level;
1211 const unsigned num_layers = fb->MaxNumLayers ? irb->layer_count : 1;
1212
1213 /* If the MCS buffer hasn't been allocated yet, we need to allocate it now.
1214 */
1215 if (can_fast_clear && !irb->mt->mcs_buf) {
1216 assert(irb->mt->aux_usage == ISL_AUX_USAGE_CCS_D);
1217 if (!intel_miptree_alloc_ccs(brw, irb->mt)) {
1218 /* There are a few reasons in addition to out-of-memory, that can
1219 * cause intel_miptree_alloc_non_msrt_mcs to fail. Try to recover by
1220 * falling back to non-fast clear.
1221 */
1222 can_fast_clear = false;
1223 }
1224 }
1225
1226 if (can_fast_clear) {
1227 const enum isl_aux_state aux_state =
1228 intel_miptree_get_aux_state(irb->mt, irb->mt_level, irb->mt_layer);
1229 union isl_color_value clear_color =
1230 brw_meta_convert_fast_clear_color(brw, irb->mt,
1231 &ctx->Color.ClearColor);
1232
1233 bool same_clear_color =
1234 !intel_miptree_set_clear_color(ctx, irb->mt, clear_color);
1235
1236 /* If the buffer is already in INTEL_FAST_CLEAR_STATE_CLEAR, the clear
1237 * is redundant and can be skipped.
1238 */
1239 if (aux_state == ISL_AUX_STATE_CLEAR && same_clear_color)
1240 return;
1241
1242 DBG("%s (fast) to mt %p level %d layers %d+%d\n", __FUNCTION__,
1243 irb->mt, irb->mt_level, irb->mt_layer, num_layers);
1244
1245 /* We can't setup the blorp_surf until we've allocated the MCS above */
1246 struct isl_surf isl_tmp[2];
1247 struct blorp_surf surf;
1248 blorp_surf_for_miptree(brw, &surf, irb->mt, irb->mt->aux_usage, true,
1249 &level, irb->mt_layer, num_layers, isl_tmp);
1250
1251 /* Ivybrigde PRM Vol 2, Part 1, "11.7 MCS Buffer for Render Target(s)":
1252 *
1253 * "Any transition from any value in {Clear, Render, Resolve} to a
1254 * different value in {Clear, Render, Resolve} requires end of pipe
1255 * synchronization."
1256 *
1257 * In other words, fast clear ops are not properly synchronized with
1258 * other drawing. We need to use a PIPE_CONTROL to ensure that the
1259 * contents of the previous draw hit the render target before we resolve
1260 * and again afterwards to ensure that the resolve is complete before we
1261 * do any more regular drawing.
1262 */
1263 brw_emit_end_of_pipe_sync(brw, PIPE_CONTROL_RENDER_TARGET_FLUSH);
1264
1265 struct blorp_batch batch;
1266 blorp_batch_init(&brw->blorp, &batch, brw, 0);
1267 blorp_fast_clear(&batch, &surf, isl_format,
1268 level, irb->mt_layer, num_layers,
1269 x0, y0, x1, y1);
1270 blorp_batch_finish(&batch);
1271
1272 brw_emit_end_of_pipe_sync(brw, PIPE_CONTROL_RENDER_TARGET_FLUSH);
1273
1274 /* Now that the fast clear has occurred, put the buffer in
1275 * INTEL_FAST_CLEAR_STATE_CLEAR so that we won't waste time doing
1276 * redundant clears.
1277 */
1278 intel_miptree_set_aux_state(brw, irb->mt, irb->mt_level,
1279 irb->mt_layer, num_layers,
1280 ISL_AUX_STATE_CLEAR);
1281 } else {
1282 DBG("%s (slow) to mt %p level %d layer %d+%d\n", __FUNCTION__,
1283 irb->mt, irb->mt_level, irb->mt_layer, num_layers);
1284
1285 enum isl_aux_usage aux_usage =
1286 intel_miptree_render_aux_usage(brw, irb->mt, isl_format,
1287 false, false);
1288 intel_miptree_prepare_render(brw, irb->mt, level, irb->mt_layer,
1289 num_layers, aux_usage);
1290
1291 struct isl_surf isl_tmp[2];
1292 struct blorp_surf surf;
1293 blorp_surf_for_miptree(brw, &surf, irb->mt, aux_usage, true,
1294 &level, irb->mt_layer, num_layers, isl_tmp);
1295
1296 union isl_color_value clear_color;
1297 memcpy(clear_color.f32, ctx->Color.ClearColor.f, sizeof(float) * 4);
1298
1299 struct blorp_batch batch;
1300 blorp_batch_init(&brw->blorp, &batch, brw, 0);
1301 blorp_clear(&batch, &surf, isl_format, ISL_SWIZZLE_IDENTITY,
1302 level, irb->mt_layer, num_layers,
1303 x0, y0, x1, y1,
1304 clear_color, color_write_disable);
1305 blorp_batch_finish(&batch);
1306
1307 intel_miptree_finish_render(brw, irb->mt, level, irb->mt_layer,
1308 num_layers, aux_usage);
1309 }
1310
1311 return;
1312 }
1313
1314 void
brw_blorp_clear_color(struct brw_context * brw,struct gl_framebuffer * fb,GLbitfield mask,bool partial_clear,bool encode_srgb)1315 brw_blorp_clear_color(struct brw_context *brw, struct gl_framebuffer *fb,
1316 GLbitfield mask, bool partial_clear, bool encode_srgb)
1317 {
1318 for (unsigned buf = 0; buf < fb->_NumColorDrawBuffers; buf++) {
1319 struct gl_renderbuffer *rb = fb->_ColorDrawBuffers[buf];
1320 struct intel_renderbuffer *irb = intel_renderbuffer(rb);
1321
1322 /* Only clear the buffers present in the provided mask */
1323 if (((1 << fb->_ColorDrawBufferIndexes[buf]) & mask) == 0)
1324 continue;
1325
1326 /* If this is an ES2 context or GL_ARB_ES2_compatibility is supported,
1327 * the framebuffer can be complete with some attachments missing. In
1328 * this case the _ColorDrawBuffers pointer will be NULL.
1329 */
1330 if (rb == NULL)
1331 continue;
1332
1333 do_single_blorp_clear(brw, fb, rb, buf, partial_clear, encode_srgb);
1334 irb->need_downsample = true;
1335 }
1336
1337 return;
1338 }
1339
1340 void
brw_blorp_clear_depth_stencil(struct brw_context * brw,struct gl_framebuffer * fb,GLbitfield mask,bool partial_clear)1341 brw_blorp_clear_depth_stencil(struct brw_context *brw,
1342 struct gl_framebuffer *fb,
1343 GLbitfield mask, bool partial_clear)
1344 {
1345 const struct gl_context *ctx = &brw->ctx;
1346 struct gl_renderbuffer *depth_rb =
1347 fb->Attachment[BUFFER_DEPTH].Renderbuffer;
1348 struct gl_renderbuffer *stencil_rb =
1349 fb->Attachment[BUFFER_STENCIL].Renderbuffer;
1350
1351 if (!depth_rb || ctx->Depth.Mask == GL_FALSE)
1352 mask &= ~BUFFER_BIT_DEPTH;
1353
1354 if (!stencil_rb || (ctx->Stencil.WriteMask[0] & 0xff) == 0)
1355 mask &= ~BUFFER_BIT_STENCIL;
1356
1357 if (!(mask & (BUFFER_BITS_DEPTH_STENCIL)))
1358 return;
1359
1360 uint32_t x0, x1, y0, y1, rb_name, rb_height;
1361 if (depth_rb) {
1362 rb_name = depth_rb->Name;
1363 rb_height = depth_rb->Height;
1364 if (stencil_rb) {
1365 assert(depth_rb->Width == stencil_rb->Width);
1366 assert(depth_rb->Height == stencil_rb->Height);
1367 }
1368 } else {
1369 assert(stencil_rb);
1370 rb_name = stencil_rb->Name;
1371 rb_height = stencil_rb->Height;
1372 }
1373
1374 x0 = fb->_Xmin;
1375 x1 = fb->_Xmax;
1376 if (rb_name != 0) {
1377 y0 = fb->_Ymin;
1378 y1 = fb->_Ymax;
1379 } else {
1380 y0 = rb_height - fb->_Ymax;
1381 y1 = rb_height - fb->_Ymin;
1382 }
1383
1384 /* If the clear region is empty, just return. */
1385 if (x0 == x1 || y0 == y1)
1386 return;
1387
1388 uint32_t level, start_layer, num_layers;
1389 struct isl_surf isl_tmp[4];
1390 struct blorp_surf depth_surf, stencil_surf;
1391
1392 struct intel_mipmap_tree *depth_mt = NULL;
1393 if (mask & BUFFER_BIT_DEPTH) {
1394 struct intel_renderbuffer *irb = intel_renderbuffer(depth_rb);
1395 depth_mt = find_miptree(GL_DEPTH_BUFFER_BIT, irb);
1396
1397 level = irb->mt_level;
1398 start_layer = irb->mt_layer;
1399 num_layers = fb->MaxNumLayers ? irb->layer_count : 1;
1400
1401 intel_miptree_prepare_depth(brw, depth_mt, level,
1402 start_layer, num_layers);
1403
1404 unsigned depth_level = level;
1405 blorp_surf_for_miptree(brw, &depth_surf, depth_mt, depth_mt->aux_usage,
1406 true, &depth_level, start_layer, num_layers,
1407 &isl_tmp[0]);
1408 assert(depth_level == level);
1409 }
1410
1411 uint8_t stencil_mask = 0;
1412 struct intel_mipmap_tree *stencil_mt = NULL;
1413 if (mask & BUFFER_BIT_STENCIL) {
1414 struct intel_renderbuffer *irb = intel_renderbuffer(stencil_rb);
1415 stencil_mt = find_miptree(GL_STENCIL_BUFFER_BIT, irb);
1416
1417 if (mask & BUFFER_BIT_DEPTH) {
1418 assert(level == irb->mt_level);
1419 assert(start_layer == irb->mt_layer);
1420 assert(num_layers == fb->MaxNumLayers ? irb->layer_count : 1);
1421 } else {
1422 level = irb->mt_level;
1423 start_layer = irb->mt_layer;
1424 num_layers = fb->MaxNumLayers ? irb->layer_count : 1;
1425 }
1426
1427 stencil_mask = ctx->Stencil.WriteMask[0] & 0xff;
1428
1429 intel_miptree_prepare_access(brw, stencil_mt, level, 1,
1430 start_layer, num_layers,
1431 ISL_AUX_USAGE_NONE, false);
1432
1433 unsigned stencil_level = level;
1434 blorp_surf_for_miptree(brw, &stencil_surf, stencil_mt,
1435 ISL_AUX_USAGE_NONE, true,
1436 &stencil_level, start_layer, num_layers,
1437 &isl_tmp[2]);
1438 }
1439
1440 assert((mask & BUFFER_BIT_DEPTH) || stencil_mask);
1441
1442 struct blorp_batch batch;
1443 blorp_batch_init(&brw->blorp, &batch, brw, 0);
1444 blorp_clear_depth_stencil(&batch, &depth_surf, &stencil_surf,
1445 level, start_layer, num_layers,
1446 x0, y0, x1, y1,
1447 (mask & BUFFER_BIT_DEPTH), ctx->Depth.Clear,
1448 stencil_mask, ctx->Stencil.Clear);
1449 blorp_batch_finish(&batch);
1450
1451 if (mask & BUFFER_BIT_DEPTH) {
1452 intel_miptree_finish_depth(brw, depth_mt, level,
1453 start_layer, num_layers, true);
1454 }
1455
1456 if (stencil_mask) {
1457 intel_miptree_finish_write(brw, stencil_mt, level,
1458 start_layer, num_layers,
1459 ISL_AUX_USAGE_NONE);
1460 }
1461 }
1462
1463 void
brw_blorp_resolve_color(struct brw_context * brw,struct intel_mipmap_tree * mt,unsigned level,unsigned layer,enum blorp_fast_clear_op resolve_op)1464 brw_blorp_resolve_color(struct brw_context *brw, struct intel_mipmap_tree *mt,
1465 unsigned level, unsigned layer,
1466 enum blorp_fast_clear_op resolve_op)
1467 {
1468 DBG("%s to mt %p level %u layer %u\n", __FUNCTION__, mt, level, layer);
1469
1470 const mesa_format format = _mesa_get_srgb_format_linear(mt->format);
1471
1472 struct isl_surf isl_tmp[1];
1473 struct blorp_surf surf;
1474 blorp_surf_for_miptree(brw, &surf, mt, mt->aux_usage, true,
1475 &level, layer, 1 /* num_layers */,
1476 isl_tmp);
1477
1478 /* Ivybrigde PRM Vol 2, Part 1, "11.7 MCS Buffer for Render Target(s)":
1479 *
1480 * "Any transition from any value in {Clear, Render, Resolve} to a
1481 * different value in {Clear, Render, Resolve} requires end of pipe
1482 * synchronization."
1483 *
1484 * In other words, fast clear ops are not properly synchronized with
1485 * other drawing. We need to use a PIPE_CONTROL to ensure that the
1486 * contents of the previous draw hit the render target before we resolve
1487 * and again afterwards to ensure that the resolve is complete before we
1488 * do any more regular drawing.
1489 */
1490 brw_emit_end_of_pipe_sync(brw, PIPE_CONTROL_RENDER_TARGET_FLUSH);
1491
1492
1493 struct blorp_batch batch;
1494 blorp_batch_init(&brw->blorp, &batch, brw, 0);
1495 blorp_ccs_resolve(&batch, &surf, level, layer, 1,
1496 brw_blorp_to_isl_format(brw, format, true),
1497 resolve_op);
1498 blorp_batch_finish(&batch);
1499
1500 /* See comment above */
1501 brw_emit_end_of_pipe_sync(brw, PIPE_CONTROL_RENDER_TARGET_FLUSH);
1502 }
1503
1504 void
brw_blorp_mcs_partial_resolve(struct brw_context * brw,struct intel_mipmap_tree * mt,uint32_t start_layer,uint32_t num_layers)1505 brw_blorp_mcs_partial_resolve(struct brw_context *brw,
1506 struct intel_mipmap_tree *mt,
1507 uint32_t start_layer, uint32_t num_layers)
1508 {
1509 DBG("%s to mt %p layers %u-%u\n", __FUNCTION__, mt,
1510 start_layer, start_layer + num_layers - 1);
1511
1512 assert(mt->aux_usage == ISL_AUX_USAGE_MCS);
1513
1514 const mesa_format format = _mesa_get_srgb_format_linear(mt->format);
1515 enum isl_format isl_format = brw_blorp_to_isl_format(brw, format, true);
1516
1517 struct isl_surf isl_tmp[1];
1518 struct blorp_surf surf;
1519 uint32_t level = 0;
1520 blorp_surf_for_miptree(brw, &surf, mt, ISL_AUX_USAGE_MCS, true,
1521 &level, start_layer, num_layers, isl_tmp);
1522
1523 struct blorp_batch batch;
1524 blorp_batch_init(&brw->blorp, &batch, brw, 0);
1525 blorp_mcs_partial_resolve(&batch, &surf, isl_format,
1526 start_layer, num_layers);
1527 blorp_batch_finish(&batch);
1528 }
1529
1530 /**
1531 * Perform a HiZ or depth resolve operation.
1532 *
1533 * For an overview of HiZ ops, see the following sections of the Sandy Bridge
1534 * PRM, Volume 1, Part 2:
1535 * - 7.5.3.1 Depth Buffer Clear
1536 * - 7.5.3.2 Depth Buffer Resolve
1537 * - 7.5.3.3 Hierarchical Depth Buffer Resolve
1538 */
1539 void
intel_hiz_exec(struct brw_context * brw,struct intel_mipmap_tree * mt,unsigned int level,unsigned int start_layer,unsigned int num_layers,enum blorp_hiz_op op)1540 intel_hiz_exec(struct brw_context *brw, struct intel_mipmap_tree *mt,
1541 unsigned int level, unsigned int start_layer,
1542 unsigned int num_layers, enum blorp_hiz_op op)
1543 {
1544 assert(intel_miptree_level_has_hiz(mt, level));
1545 assert(op != BLORP_HIZ_OP_NONE);
1546 const struct gen_device_info *devinfo = &brw->screen->devinfo;
1547 const char *opname = NULL;
1548
1549 switch (op) {
1550 case BLORP_HIZ_OP_DEPTH_RESOLVE:
1551 opname = "depth resolve";
1552 break;
1553 case BLORP_HIZ_OP_HIZ_RESOLVE:
1554 opname = "hiz ambiguate";
1555 break;
1556 case BLORP_HIZ_OP_DEPTH_CLEAR:
1557 opname = "depth clear";
1558 break;
1559 case BLORP_HIZ_OP_NONE:
1560 opname = "noop?";
1561 break;
1562 }
1563
1564 DBG("%s %s to mt %p level %d layers %d-%d\n",
1565 __func__, opname, mt, level, start_layer, start_layer + num_layers - 1);
1566
1567 /* The following stalls and flushes are only documented to be required for
1568 * HiZ clear operations. However, they also seem to be required for
1569 * resolve operations.
1570 */
1571 if (devinfo->gen == 6) {
1572 /* From the Sandy Bridge PRM, volume 2 part 1, page 313:
1573 *
1574 * "If other rendering operations have preceded this clear, a
1575 * PIPE_CONTROL with write cache flush enabled and Z-inhibit
1576 * disabled must be issued before the rectangle primitive used for
1577 * the depth buffer clear operation.
1578 */
1579 brw_emit_pipe_control_flush(brw,
1580 PIPE_CONTROL_RENDER_TARGET_FLUSH |
1581 PIPE_CONTROL_DEPTH_CACHE_FLUSH |
1582 PIPE_CONTROL_CS_STALL);
1583 } else if (devinfo->gen >= 7) {
1584 /*
1585 * From the Ivybridge PRM, volume 2, "Depth Buffer Clear":
1586 *
1587 * If other rendering operations have preceded this clear, a
1588 * PIPE_CONTROL with depth cache flush enabled, Depth Stall bit
1589 * enabled must be issued before the rectangle primitive used for
1590 * the depth buffer clear operation.
1591 *
1592 * Same applies for Gen8 and Gen9.
1593 *
1594 * In addition, from the Ivybridge PRM, volume 2, 1.10.4.1
1595 * PIPE_CONTROL, Depth Cache Flush Enable:
1596 *
1597 * This bit must not be set when Depth Stall Enable bit is set in
1598 * this packet.
1599 *
1600 * This is confirmed to hold for real, HSW gets immediate gpu hangs.
1601 *
1602 * Therefore issue two pipe control flushes, one for cache flush and
1603 * another for depth stall.
1604 */
1605 brw_emit_pipe_control_flush(brw,
1606 PIPE_CONTROL_DEPTH_CACHE_FLUSH |
1607 PIPE_CONTROL_CS_STALL);
1608
1609 brw_emit_pipe_control_flush(brw, PIPE_CONTROL_DEPTH_STALL);
1610 }
1611
1612 assert(mt->aux_usage == ISL_AUX_USAGE_HIZ && mt->hiz_buf);
1613
1614 struct isl_surf isl_tmp[2];
1615 struct blorp_surf surf;
1616 blorp_surf_for_miptree(brw, &surf, mt, ISL_AUX_USAGE_HIZ, true,
1617 &level, start_layer, num_layers, isl_tmp);
1618
1619 struct blorp_batch batch;
1620 blorp_batch_init(&brw->blorp, &batch, brw, 0);
1621 blorp_hiz_op(&batch, &surf, level, start_layer, num_layers, op);
1622 blorp_batch_finish(&batch);
1623
1624 /* The following stalls and flushes are only documented to be required for
1625 * HiZ clear operations. However, they also seem to be required for
1626 * resolve operations.
1627 */
1628 if (devinfo->gen == 6) {
1629 /* From the Sandy Bridge PRM, volume 2 part 1, page 314:
1630 *
1631 * "DevSNB, DevSNB-B{W/A}]: Depth buffer clear pass must be
1632 * followed by a PIPE_CONTROL command with DEPTH_STALL bit set
1633 * and Then followed by Depth FLUSH'
1634 */
1635 brw_emit_pipe_control_flush(brw,
1636 PIPE_CONTROL_DEPTH_STALL);
1637
1638 brw_emit_pipe_control_flush(brw,
1639 PIPE_CONTROL_DEPTH_CACHE_FLUSH |
1640 PIPE_CONTROL_CS_STALL);
1641 } else if (devinfo->gen >= 8) {
1642 /*
1643 * From the Broadwell PRM, volume 7, "Depth Buffer Clear":
1644 *
1645 * "Depth buffer clear pass using any of the methods (WM_STATE,
1646 * 3DSTATE_WM or 3DSTATE_WM_HZ_OP) must be followed by a
1647 * PIPE_CONTROL command with DEPTH_STALL bit and Depth FLUSH bits
1648 * "set" before starting to render. DepthStall and DepthFlush are
1649 * not needed between consecutive depth clear passes nor is it
1650 * required if the depth clear pass was done with
1651 * 'full_surf_clear' bit set in the 3DSTATE_WM_HZ_OP."
1652 *
1653 * TODO: Such as the spec says, this could be conditional.
1654 */
1655 brw_emit_pipe_control_flush(brw,
1656 PIPE_CONTROL_DEPTH_CACHE_FLUSH |
1657 PIPE_CONTROL_DEPTH_STALL);
1658
1659 }
1660 }
1661