1 /*
2 * Copyright 2006 VMware, Inc.
3 * All Rights Reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sublicense, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
12 *
13 * The above copyright notice and this permission notice (including the
14 * next paragraph) shall be included in all copies or substantial portions
15 * of the Software.
16 *
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
18 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
20 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
21 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
22 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
23 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
24 */
25
26 #include <GL/gl.h>
27 #include <GL/internal/dri_interface.h>
28 #include <drm_fourcc.h>
29
30 #include "intel_batchbuffer.h"
31 #include "intel_image.h"
32 #include "intel_mipmap_tree.h"
33 #include "intel_tex.h"
34 #include "intel_blit.h"
35 #include "intel_fbo.h"
36
37 #include "brw_blorp.h"
38 #include "brw_context.h"
39 #include "brw_state.h"
40
41 #include "main/enums.h"
42 #include "main/fbobject.h"
43 #include "main/formats.h"
44 #include "main/glformats.h"
45 #include "main/texcompress_etc.h"
46 #include "main/teximage.h"
47 #include "main/streaming-load-memcpy.h"
48 #include "x86/common_x86_asm.h"
49
50 #define FILE_DEBUG_FLAG DEBUG_MIPTREE
51
52 static void *intel_miptree_map_raw(struct brw_context *brw,
53 struct intel_mipmap_tree *mt,
54 GLbitfield mode);
55
56 static void intel_miptree_unmap_raw(struct intel_mipmap_tree *mt);
57
58 static bool
59 intel_miptree_alloc_aux(struct brw_context *brw,
60 struct intel_mipmap_tree *mt);
61
62 static bool
intel_miptree_supports_mcs(struct brw_context * brw,const struct intel_mipmap_tree * mt)63 intel_miptree_supports_mcs(struct brw_context *brw,
64 const struct intel_mipmap_tree *mt)
65 {
66 const struct gen_device_info *devinfo = &brw->screen->devinfo;
67
68 /* MCS compression only applies to multisampled miptrees */
69 if (mt->surf.samples <= 1)
70 return false;
71
72 /* Prior to Gen7, all MSAA surfaces used IMS layout. */
73 if (devinfo->gen < 7)
74 return false;
75
76 /* See isl_surf_get_mcs_surf for details. */
77 if (mt->surf.samples == 16 && mt->surf.logical_level0_px.width > 8192)
78 return false;
79
80 /* In Gen7, IMS layout is only used for depth and stencil buffers. */
81 switch (_mesa_get_format_base_format(mt->format)) {
82 case GL_DEPTH_COMPONENT:
83 case GL_STENCIL_INDEX:
84 case GL_DEPTH_STENCIL:
85 return false;
86 default:
87 /* From the Ivy Bridge PRM, Vol4 Part1 p77 ("MCS Enable"):
88 *
89 * This field must be set to 0 for all SINT MSRTs when all RT channels
90 * are not written
91 *
92 * In practice this means that we have to disable MCS for all signed
93 * integer MSAA buffers. The alternative, to disable MCS only when one
94 * of the render target channels is disabled, is impractical because it
95 * would require converting between CMS and UMS MSAA layouts on the fly,
96 * which is expensive.
97 */
98 if (devinfo->gen == 7 && _mesa_get_format_datatype(mt->format) == GL_INT) {
99 return false;
100 } else {
101 return true;
102 }
103 }
104 }
105
106 static bool
intel_tiling_supports_ccs(const struct brw_context * brw,enum isl_tiling tiling)107 intel_tiling_supports_ccs(const struct brw_context *brw,
108 enum isl_tiling tiling)
109 {
110 const struct gen_device_info *devinfo = &brw->screen->devinfo;
111
112 /* From the Ivy Bridge PRM, Vol2 Part1 11.7 "MCS Buffer for Render
113 * Target(s)", beneath the "Fast Color Clear" bullet (p326):
114 *
115 * - Support is limited to tiled render targets.
116 *
117 * Gen9 changes the restriction to Y-tile only.
118 */
119 if (devinfo->gen >= 9)
120 return tiling == ISL_TILING_Y0;
121 else if (devinfo->gen >= 7)
122 return tiling != ISL_TILING_LINEAR;
123 else
124 return false;
125 }
126
127 /**
128 * For a single-sampled render target ("non-MSRT"), determine if an MCS buffer
129 * can be used. This doesn't (and should not) inspect any of the properties of
130 * the miptree's BO.
131 *
132 * From the Ivy Bridge PRM, Vol2 Part1 11.7 "MCS Buffer for Render Target(s)",
133 * beneath the "Fast Color Clear" bullet (p326):
134 *
135 * - Support is for non-mip-mapped and non-array surface types only.
136 *
137 * And then later, on p327:
138 *
139 * - MCS buffer for non-MSRT is supported only for RT formats 32bpp,
140 * 64bpp, and 128bpp.
141 *
142 * From the Skylake documentation, it is made clear that X-tiling is no longer
143 * supported:
144 *
145 * - MCS and Lossless compression is supported for TiledY/TileYs/TileYf
146 * non-MSRTs only.
147 */
148 static bool
intel_miptree_supports_ccs(struct brw_context * brw,const struct intel_mipmap_tree * mt)149 intel_miptree_supports_ccs(struct brw_context *brw,
150 const struct intel_mipmap_tree *mt)
151 {
152 const struct gen_device_info *devinfo = &brw->screen->devinfo;
153
154 /* MCS support does not exist prior to Gen7 */
155 if (devinfo->gen < 7)
156 return false;
157
158 /* This function applies only to non-multisampled render targets. */
159 if (mt->surf.samples > 1)
160 return false;
161
162 /* MCS is only supported for color buffers */
163 switch (_mesa_get_format_base_format(mt->format)) {
164 case GL_DEPTH_COMPONENT:
165 case GL_DEPTH_STENCIL:
166 case GL_STENCIL_INDEX:
167 return false;
168 }
169
170 if (mt->cpp != 4 && mt->cpp != 8 && mt->cpp != 16)
171 return false;
172
173 const bool mip_mapped = mt->first_level != 0 || mt->last_level != 0;
174 const bool arrayed = mt->surf.logical_level0_px.array_len > 1 ||
175 mt->surf.logical_level0_px.depth > 1;
176
177 if (arrayed) {
178 /* Multisample surfaces with the CMS layout are not layered surfaces,
179 * yet still have physical_depth0 > 1. Assert that we don't
180 * accidentally reject a multisampled surface here. We should have
181 * rejected it earlier by explicitly checking the sample count.
182 */
183 assert(mt->surf.samples == 1);
184 }
185
186 /* Handle the hardware restrictions...
187 *
188 * All GENs have the following restriction: "MCS buffer for non-MSRT is
189 * supported only for RT formats 32bpp, 64bpp, and 128bpp."
190 *
191 * From the HSW PRM Volume 7: 3D-Media-GPGPU, page 652: (Color Clear of
192 * Non-MultiSampler Render Target Restrictions) Support is for
193 * non-mip-mapped and non-array surface types only.
194 *
195 * From the BDW PRM Volume 7: 3D-Media-GPGPU, page 649: (Color Clear of
196 * Non-MultiSampler Render Target Restriction). Mip-mapped and arrayed
197 * surfaces are supported with MCS buffer layout with these alignments in
198 * the RT space: Horizontal Alignment = 256 and Vertical Alignment = 128.
199 *
200 * From the SKL PRM Volume 7: 3D-Media-GPGPU, page 632: (Color Clear of
201 * Non-MultiSampler Render Target Restriction). Mip-mapped and arrayed
202 * surfaces are supported with MCS buffer layout with these alignments in
203 * the RT space: Horizontal Alignment = 128 and Vertical Alignment = 64.
204 */
205 if (devinfo->gen < 8 && (mip_mapped || arrayed))
206 return false;
207
208 /* There's no point in using an MCS buffer if the surface isn't in a
209 * renderable format.
210 */
211 if (!brw->mesa_format_supports_render[mt->format])
212 return false;
213
214 return true;
215 }
216
217 static bool
intel_tiling_supports_hiz(const struct brw_context * brw,enum isl_tiling tiling)218 intel_tiling_supports_hiz(const struct brw_context *brw,
219 enum isl_tiling tiling)
220 {
221 const struct gen_device_info *devinfo = &brw->screen->devinfo;
222
223 if (devinfo->gen < 6)
224 return false;
225
226 return tiling == ISL_TILING_Y0;
227 }
228
229 static bool
intel_miptree_supports_hiz(const struct brw_context * brw,const struct intel_mipmap_tree * mt)230 intel_miptree_supports_hiz(const struct brw_context *brw,
231 const struct intel_mipmap_tree *mt)
232 {
233 if (!brw->has_hiz)
234 return false;
235
236 switch (mt->format) {
237 case MESA_FORMAT_Z_FLOAT32:
238 case MESA_FORMAT_Z32_FLOAT_S8X24_UINT:
239 case MESA_FORMAT_Z24_UNORM_X8_UINT:
240 case MESA_FORMAT_Z24_UNORM_S8_UINT:
241 case MESA_FORMAT_Z_UNORM16:
242 return true;
243 default:
244 return false;
245 }
246 }
247
248 /**
249 * Return true if the format that will be used to access the miptree is
250 * CCS_E-compatible with the miptree's linear/non-sRGB format.
251 *
252 * Why use the linear format? Well, although the miptree may be specified with
253 * an sRGB format, the usage of that color space/format can be toggled. Since
254 * our HW tends to support more linear formats than sRGB ones, we use this
255 * format variant for check for CCS_E compatibility.
256 */
257 static bool
format_ccs_e_compat_with_miptree(const struct gen_device_info * devinfo,const struct intel_mipmap_tree * mt,enum isl_format access_format)258 format_ccs_e_compat_with_miptree(const struct gen_device_info *devinfo,
259 const struct intel_mipmap_tree *mt,
260 enum isl_format access_format)
261 {
262 assert(mt->aux_usage == ISL_AUX_USAGE_CCS_E);
263
264 mesa_format linear_format = _mesa_get_srgb_format_linear(mt->format);
265 enum isl_format isl_format = brw_isl_format_for_mesa_format(linear_format);
266 return isl_formats_are_ccs_e_compatible(devinfo, isl_format, access_format);
267 }
268
269 static bool
intel_miptree_supports_ccs_e(struct brw_context * brw,const struct intel_mipmap_tree * mt)270 intel_miptree_supports_ccs_e(struct brw_context *brw,
271 const struct intel_mipmap_tree *mt)
272 {
273 const struct gen_device_info *devinfo = &brw->screen->devinfo;
274
275 if (devinfo->gen < 9)
276 return false;
277
278 /* For now compression is only enabled for integer formats even though
279 * there exist supported floating point formats also. This is a heuristic
280 * decision based on current public benchmarks. In none of the cases these
281 * formats provided any improvement but a few cases were seen to regress.
282 * Hence these are left to to be enabled in the future when they are known
283 * to improve things.
284 */
285 if (_mesa_get_format_datatype(mt->format) == GL_FLOAT)
286 return false;
287
288 if (!intel_miptree_supports_ccs(brw, mt))
289 return false;
290
291 /* Many window system buffers are sRGB even if they are never rendered as
292 * sRGB. For those, we want CCS_E for when sRGBEncode is false. When the
293 * surface is used as sRGB, we fall back to CCS_D.
294 */
295 mesa_format linear_format = _mesa_get_srgb_format_linear(mt->format);
296 enum isl_format isl_format = brw_isl_format_for_mesa_format(linear_format);
297 return isl_format_supports_ccs_e(&brw->screen->devinfo, isl_format);
298 }
299
300 /**
301 * Determine depth format corresponding to a depth+stencil format,
302 * for separate stencil.
303 */
304 mesa_format
intel_depth_format_for_depthstencil_format(mesa_format format)305 intel_depth_format_for_depthstencil_format(mesa_format format) {
306 switch (format) {
307 case MESA_FORMAT_Z24_UNORM_S8_UINT:
308 return MESA_FORMAT_Z24_UNORM_X8_UINT;
309 case MESA_FORMAT_Z32_FLOAT_S8X24_UINT:
310 return MESA_FORMAT_Z_FLOAT32;
311 default:
312 return format;
313 }
314 }
315
316 static bool
create_mapping_table(GLenum target,unsigned first_level,unsigned last_level,unsigned depth0,struct intel_mipmap_level * table)317 create_mapping_table(GLenum target, unsigned first_level, unsigned last_level,
318 unsigned depth0, struct intel_mipmap_level *table)
319 {
320 for (unsigned level = first_level; level <= last_level; level++) {
321 const unsigned d =
322 target == GL_TEXTURE_3D ? minify(depth0, level) : depth0;
323
324 table[level].slice = calloc(d, sizeof(*table[0].slice));
325 if (!table[level].slice)
326 goto unwind;
327 }
328
329 return true;
330
331 unwind:
332 for (unsigned level = first_level; level <= last_level; level++)
333 free(table[level].slice);
334
335 return false;
336 }
337
338 static bool
needs_separate_stencil(const struct brw_context * brw,struct intel_mipmap_tree * mt,mesa_format format)339 needs_separate_stencil(const struct brw_context *brw,
340 struct intel_mipmap_tree *mt,
341 mesa_format format)
342 {
343 const struct gen_device_info *devinfo = &brw->screen->devinfo;
344
345 if (_mesa_get_format_base_format(format) != GL_DEPTH_STENCIL)
346 return false;
347
348 if (devinfo->must_use_separate_stencil)
349 return true;
350
351 return brw->has_separate_stencil &&
352 intel_miptree_supports_hiz(brw, mt);
353 }
354
355 /**
356 * Choose the aux usage for this miptree. This function must be called fairly
357 * late in the miptree create process after we have a tiling.
358 */
359 static void
intel_miptree_choose_aux_usage(struct brw_context * brw,struct intel_mipmap_tree * mt)360 intel_miptree_choose_aux_usage(struct brw_context *brw,
361 struct intel_mipmap_tree *mt)
362 {
363 assert(mt->aux_usage == ISL_AUX_USAGE_NONE);
364
365 if (intel_miptree_supports_mcs(brw, mt)) {
366 assert(mt->surf.msaa_layout == ISL_MSAA_LAYOUT_ARRAY);
367 mt->aux_usage = ISL_AUX_USAGE_MCS;
368 } else if (intel_tiling_supports_ccs(brw, mt->surf.tiling) &&
369 intel_miptree_supports_ccs(brw, mt)) {
370 if (!unlikely(INTEL_DEBUG & DEBUG_NO_RBC) &&
371 intel_miptree_supports_ccs_e(brw, mt)) {
372 mt->aux_usage = ISL_AUX_USAGE_CCS_E;
373 } else {
374 mt->aux_usage = ISL_AUX_USAGE_CCS_D;
375 }
376 } else if (intel_tiling_supports_hiz(brw, mt->surf.tiling) &&
377 intel_miptree_supports_hiz(brw, mt)) {
378 mt->aux_usage = ISL_AUX_USAGE_HIZ;
379 }
380
381 /* We can do fast-clear on all auxiliary surface types that are
382 * allocated through the normal texture creation paths.
383 */
384 if (mt->aux_usage != ISL_AUX_USAGE_NONE)
385 mt->supports_fast_clear = true;
386 }
387
388
389 /**
390 * Choose an appropriate uncompressed format for a requested
391 * compressed format, if unsupported.
392 */
393 mesa_format
intel_lower_compressed_format(struct brw_context * brw,mesa_format format)394 intel_lower_compressed_format(struct brw_context *brw, mesa_format format)
395 {
396 const struct gen_device_info *devinfo = &brw->screen->devinfo;
397
398 /* No need to lower ETC formats on these platforms,
399 * they are supported natively.
400 */
401 if (devinfo->gen >= 8 || devinfo->is_baytrail)
402 return format;
403
404 switch (format) {
405 case MESA_FORMAT_ETC1_RGB8:
406 return MESA_FORMAT_R8G8B8X8_UNORM;
407 case MESA_FORMAT_ETC2_RGB8:
408 return MESA_FORMAT_R8G8B8X8_UNORM;
409 case MESA_FORMAT_ETC2_SRGB8:
410 case MESA_FORMAT_ETC2_SRGB8_ALPHA8_EAC:
411 case MESA_FORMAT_ETC2_SRGB8_PUNCHTHROUGH_ALPHA1:
412 return MESA_FORMAT_B8G8R8A8_SRGB;
413 case MESA_FORMAT_ETC2_RGBA8_EAC:
414 case MESA_FORMAT_ETC2_RGB8_PUNCHTHROUGH_ALPHA1:
415 return MESA_FORMAT_R8G8B8A8_UNORM;
416 case MESA_FORMAT_ETC2_R11_EAC:
417 return MESA_FORMAT_R_UNORM16;
418 case MESA_FORMAT_ETC2_SIGNED_R11_EAC:
419 return MESA_FORMAT_R_SNORM16;
420 case MESA_FORMAT_ETC2_RG11_EAC:
421 return MESA_FORMAT_R16G16_UNORM;
422 case MESA_FORMAT_ETC2_SIGNED_RG11_EAC:
423 return MESA_FORMAT_R16G16_SNORM;
424 default:
425 /* Non ETC1 / ETC2 format */
426 return format;
427 }
428 }
429
430 unsigned
brw_get_num_logical_layers(const struct intel_mipmap_tree * mt,unsigned level)431 brw_get_num_logical_layers(const struct intel_mipmap_tree *mt, unsigned level)
432 {
433 if (mt->surf.dim == ISL_SURF_DIM_3D)
434 return minify(mt->surf.logical_level0_px.depth, level);
435 else
436 return mt->surf.logical_level0_px.array_len;
437 }
438
439 UNUSED static unsigned
get_num_phys_layers(const struct isl_surf * surf,unsigned level)440 get_num_phys_layers(const struct isl_surf *surf, unsigned level)
441 {
442 /* In case of physical dimensions one needs to consider also the layout.
443 * See isl_calc_phys_level0_extent_sa().
444 */
445 if (surf->dim != ISL_SURF_DIM_3D)
446 return surf->phys_level0_sa.array_len;
447
448 if (surf->dim_layout == ISL_DIM_LAYOUT_GEN4_2D)
449 return minify(surf->phys_level0_sa.array_len, level);
450
451 return minify(surf->phys_level0_sa.depth, level);
452 }
453
454 /** \brief Assert that the level and layer are valid for the miptree. */
455 void
intel_miptree_check_level_layer(const struct intel_mipmap_tree * mt,uint32_t level,uint32_t layer)456 intel_miptree_check_level_layer(const struct intel_mipmap_tree *mt,
457 uint32_t level,
458 uint32_t layer)
459 {
460 (void) mt;
461 (void) level;
462 (void) layer;
463
464 assert(level >= mt->first_level);
465 assert(level <= mt->last_level);
466 assert(layer < get_num_phys_layers(&mt->surf, level));
467 }
468
469 static enum isl_aux_state **
create_aux_state_map(struct intel_mipmap_tree * mt,enum isl_aux_state initial)470 create_aux_state_map(struct intel_mipmap_tree *mt,
471 enum isl_aux_state initial)
472 {
473 const uint32_t levels = mt->last_level + 1;
474
475 uint32_t total_slices = 0;
476 for (uint32_t level = 0; level < levels; level++)
477 total_slices += brw_get_num_logical_layers(mt, level);
478
479 const size_t per_level_array_size = levels * sizeof(enum isl_aux_state *);
480
481 /* We're going to allocate a single chunk of data for both the per-level
482 * reference array and the arrays of aux_state. This makes cleanup
483 * significantly easier.
484 */
485 const size_t total_size = per_level_array_size +
486 total_slices * sizeof(enum isl_aux_state);
487 void *data = malloc(total_size);
488 if (data == NULL)
489 return NULL;
490
491 enum isl_aux_state **per_level_arr = data;
492 enum isl_aux_state *s = data + per_level_array_size;
493 for (uint32_t level = 0; level < levels; level++) {
494 per_level_arr[level] = s;
495 const unsigned level_layers = brw_get_num_logical_layers(mt, level);
496 for (uint32_t a = 0; a < level_layers; a++)
497 *(s++) = initial;
498 }
499 assert((void *)s == data + total_size);
500
501 return per_level_arr;
502 }
503
504 static void
free_aux_state_map(enum isl_aux_state ** state)505 free_aux_state_map(enum isl_aux_state **state)
506 {
507 free(state);
508 }
509
510 static bool
need_to_retile_as_linear(struct brw_context * brw,unsigned row_pitch,enum isl_tiling tiling,unsigned samples)511 need_to_retile_as_linear(struct brw_context *brw, unsigned row_pitch,
512 enum isl_tiling tiling, unsigned samples)
513 {
514 if (samples > 1)
515 return false;
516
517 if (tiling == ISL_TILING_LINEAR)
518 return false;
519
520 /* If the width is much smaller than a tile, don't bother tiling. */
521 if (row_pitch < 64)
522 return true;
523
524 if (ALIGN(row_pitch, 512) >= 32768) {
525 perf_debug("row pitch %u too large to blit, falling back to untiled",
526 row_pitch);
527 return true;
528 }
529
530 return false;
531 }
532
533 static bool
need_to_retile_as_x(const struct brw_context * brw,uint64_t size,enum isl_tiling tiling)534 need_to_retile_as_x(const struct brw_context *brw, uint64_t size,
535 enum isl_tiling tiling)
536 {
537 const struct gen_device_info *devinfo = &brw->screen->devinfo;
538
539 /* If the BO is too large to fit in the aperture, we need to use the
540 * BLT engine to support it. Prior to Sandybridge, the BLT paths can't
541 * handle Y-tiling, so we need to fall back to X.
542 */
543 if (devinfo->gen < 6 && size >= brw->max_gtt_map_object_size &&
544 tiling == ISL_TILING_Y0)
545 return true;
546
547 return false;
548 }
549
550 static struct intel_mipmap_tree *
make_surface(struct brw_context * brw,GLenum target,mesa_format format,unsigned first_level,unsigned last_level,unsigned width0,unsigned height0,unsigned depth0,unsigned num_samples,isl_tiling_flags_t tiling_flags,isl_surf_usage_flags_t isl_usage_flags,uint32_t alloc_flags,unsigned row_pitch,struct brw_bo * bo)551 make_surface(struct brw_context *brw, GLenum target, mesa_format format,
552 unsigned first_level, unsigned last_level,
553 unsigned width0, unsigned height0, unsigned depth0,
554 unsigned num_samples, isl_tiling_flags_t tiling_flags,
555 isl_surf_usage_flags_t isl_usage_flags, uint32_t alloc_flags,
556 unsigned row_pitch, struct brw_bo *bo)
557 {
558 struct intel_mipmap_tree *mt = calloc(sizeof(*mt), 1);
559 if (!mt)
560 return NULL;
561
562 if (!create_mapping_table(target, first_level, last_level, depth0,
563 mt->level)) {
564 free(mt);
565 return NULL;
566 }
567
568 mt->refcount = 1;
569
570 if (target == GL_TEXTURE_CUBE_MAP ||
571 target == GL_TEXTURE_CUBE_MAP_ARRAY)
572 isl_usage_flags |= ISL_SURF_USAGE_CUBE_BIT;
573
574 DBG("%s: %s %s %ux %u:%u:%u %d..%d <-- %p\n",
575 __func__,
576 _mesa_enum_to_string(target),
577 _mesa_get_format_name(format),
578 num_samples, width0, height0, depth0,
579 first_level, last_level, mt);
580
581 struct isl_surf_init_info init_info = {
582 .dim = get_isl_surf_dim(target),
583 .format = translate_tex_format(brw, format, false),
584 .width = width0,
585 .height = height0,
586 .depth = target == GL_TEXTURE_3D ? depth0 : 1,
587 .levels = last_level - first_level + 1,
588 .array_len = target == GL_TEXTURE_3D ? 1 : depth0,
589 .samples = num_samples,
590 .row_pitch = row_pitch,
591 .usage = isl_usage_flags,
592 .tiling_flags = tiling_flags,
593 };
594
595 if (!isl_surf_init_s(&brw->isl_dev, &mt->surf, &init_info))
596 goto fail;
597
598 /* Depth surfaces are always Y-tiled and stencil is always W-tiled, although
599 * on gen7 platforms we also need to create Y-tiled copies of stencil for
600 * texturing since the hardware can't sample from W-tiled surfaces. For
601 * everything else, check for corner cases needing special treatment.
602 */
603 bool is_depth_stencil =
604 mt->surf.usage & (ISL_SURF_USAGE_STENCIL_BIT | ISL_SURF_USAGE_DEPTH_BIT);
605 if (!is_depth_stencil) {
606 if (need_to_retile_as_linear(brw, mt->surf.row_pitch,
607 mt->surf.tiling, mt->surf.samples)) {
608 init_info.tiling_flags = 1u << ISL_TILING_LINEAR;
609 if (!isl_surf_init_s(&brw->isl_dev, &mt->surf, &init_info))
610 goto fail;
611 } else if (need_to_retile_as_x(brw, mt->surf.size, mt->surf.tiling)) {
612 init_info.tiling_flags = 1u << ISL_TILING_X;
613 if (!isl_surf_init_s(&brw->isl_dev, &mt->surf, &init_info))
614 goto fail;
615 }
616 }
617
618 /* In case of linear the buffer gets padded by fixed 64 bytes and therefore
619 * the size may not be multiple of row_pitch.
620 * See isl_apply_surface_padding().
621 */
622 if (mt->surf.tiling != ISL_TILING_LINEAR)
623 assert(mt->surf.size % mt->surf.row_pitch == 0);
624
625 if (!bo) {
626 mt->bo = brw_bo_alloc_tiled(brw->bufmgr, "isl-miptree",
627 mt->surf.size,
628 isl_tiling_to_i915_tiling(
629 mt->surf.tiling),
630 mt->surf.row_pitch, alloc_flags);
631 if (!mt->bo)
632 goto fail;
633 } else {
634 mt->bo = bo;
635 }
636
637 mt->first_level = first_level;
638 mt->last_level = last_level;
639 mt->target = target;
640 mt->format = format;
641 mt->aux_state = NULL;
642 mt->cpp = isl_format_get_layout(mt->surf.format)->bpb / 8;
643 mt->compressed = _mesa_is_format_compressed(format);
644 mt->drm_modifier = DRM_FORMAT_MOD_INVALID;
645
646 return mt;
647
648 fail:
649 intel_miptree_release(&mt);
650 return NULL;
651 }
652
653 static bool
make_separate_stencil_surface(struct brw_context * brw,struct intel_mipmap_tree * mt)654 make_separate_stencil_surface(struct brw_context *brw,
655 struct intel_mipmap_tree *mt)
656 {
657 mt->stencil_mt = make_surface(brw, mt->target, MESA_FORMAT_S_UINT8,
658 0, mt->surf.levels - 1,
659 mt->surf.logical_level0_px.width,
660 mt->surf.logical_level0_px.height,
661 mt->surf.dim == ISL_SURF_DIM_3D ?
662 mt->surf.logical_level0_px.depth :
663 mt->surf.logical_level0_px.array_len,
664 mt->surf.samples, ISL_TILING_W_BIT,
665 ISL_SURF_USAGE_STENCIL_BIT |
666 ISL_SURF_USAGE_TEXTURE_BIT,
667 BO_ALLOC_BUSY, 0, NULL);
668
669 if (!mt->stencil_mt)
670 return false;
671
672 mt->stencil_mt->r8stencil_needs_update = true;
673
674 return true;
675 }
676
677 static struct intel_mipmap_tree *
miptree_create(struct brw_context * brw,GLenum target,mesa_format format,GLuint first_level,GLuint last_level,GLuint width0,GLuint height0,GLuint depth0,GLuint num_samples,enum intel_miptree_create_flags flags)678 miptree_create(struct brw_context *brw,
679 GLenum target,
680 mesa_format format,
681 GLuint first_level,
682 GLuint last_level,
683 GLuint width0,
684 GLuint height0,
685 GLuint depth0,
686 GLuint num_samples,
687 enum intel_miptree_create_flags flags)
688 {
689 const struct gen_device_info *devinfo = &brw->screen->devinfo;
690
691 if (format == MESA_FORMAT_S_UINT8)
692 return make_surface(brw, target, format, first_level, last_level,
693 width0, height0, depth0, num_samples,
694 ISL_TILING_W_BIT,
695 ISL_SURF_USAGE_STENCIL_BIT |
696 ISL_SURF_USAGE_TEXTURE_BIT,
697 BO_ALLOC_BUSY,
698 0,
699 NULL);
700
701 const GLenum base_format = _mesa_get_format_base_format(format);
702 if ((base_format == GL_DEPTH_COMPONENT ||
703 base_format == GL_DEPTH_STENCIL) &&
704 !(flags & MIPTREE_CREATE_LINEAR)) {
705 /* Fix up the Z miptree format for how we're splitting out separate
706 * stencil. Gen7 expects there to be no stencil bits in its depth buffer.
707 */
708 const mesa_format depth_only_format =
709 intel_depth_format_for_depthstencil_format(format);
710 struct intel_mipmap_tree *mt = make_surface(
711 brw, target, devinfo->gen >= 6 ? depth_only_format : format,
712 first_level, last_level,
713 width0, height0, depth0, num_samples, ISL_TILING_Y0_BIT,
714 ISL_SURF_USAGE_DEPTH_BIT | ISL_SURF_USAGE_TEXTURE_BIT,
715 BO_ALLOC_BUSY, 0, NULL);
716
717 if (needs_separate_stencil(brw, mt, format) &&
718 !make_separate_stencil_surface(brw, mt)) {
719 intel_miptree_release(&mt);
720 return NULL;
721 }
722
723 if (!(flags & MIPTREE_CREATE_NO_AUX))
724 intel_miptree_choose_aux_usage(brw, mt);
725
726 return mt;
727 }
728
729 mesa_format tex_format = format;
730 mesa_format etc_format = MESA_FORMAT_NONE;
731 uint32_t alloc_flags = 0;
732
733 format = intel_lower_compressed_format(brw, format);
734
735 etc_format = (format != tex_format) ? tex_format : MESA_FORMAT_NONE;
736
737 if (flags & MIPTREE_CREATE_BUSY)
738 alloc_flags |= BO_ALLOC_BUSY;
739
740 isl_tiling_flags_t tiling_flags = (flags & MIPTREE_CREATE_LINEAR) ?
741 ISL_TILING_LINEAR_BIT : ISL_TILING_ANY_MASK;
742
743 /* TODO: This used to be because there wasn't BLORP to handle Y-tiling. */
744 if (devinfo->gen < 6)
745 tiling_flags &= ~ISL_TILING_Y0_BIT;
746
747 struct intel_mipmap_tree *mt = make_surface(
748 brw, target, format,
749 first_level, last_level,
750 width0, height0, depth0,
751 num_samples, tiling_flags,
752 ISL_SURF_USAGE_RENDER_TARGET_BIT |
753 ISL_SURF_USAGE_TEXTURE_BIT,
754 alloc_flags, 0, NULL);
755 if (!mt)
756 return NULL;
757
758 mt->etc_format = etc_format;
759
760 if (!(flags & MIPTREE_CREATE_NO_AUX))
761 intel_miptree_choose_aux_usage(brw, mt);
762
763 return mt;
764 }
765
766 struct intel_mipmap_tree *
intel_miptree_create(struct brw_context * brw,GLenum target,mesa_format format,GLuint first_level,GLuint last_level,GLuint width0,GLuint height0,GLuint depth0,GLuint num_samples,enum intel_miptree_create_flags flags)767 intel_miptree_create(struct brw_context *brw,
768 GLenum target,
769 mesa_format format,
770 GLuint first_level,
771 GLuint last_level,
772 GLuint width0,
773 GLuint height0,
774 GLuint depth0,
775 GLuint num_samples,
776 enum intel_miptree_create_flags flags)
777 {
778 assert(num_samples > 0);
779
780 struct intel_mipmap_tree *mt = miptree_create(
781 brw, target, format,
782 first_level, last_level,
783 width0, height0, depth0, num_samples,
784 flags);
785 if (!mt)
786 return NULL;
787
788 mt->offset = 0;
789
790 if (!intel_miptree_alloc_aux(brw, mt)) {
791 intel_miptree_release(&mt);
792 return NULL;
793 }
794
795 return mt;
796 }
797
798 struct intel_mipmap_tree *
intel_miptree_create_for_bo(struct brw_context * brw,struct brw_bo * bo,mesa_format format,uint32_t offset,uint32_t width,uint32_t height,uint32_t depth,int pitch,enum isl_tiling tiling,enum intel_miptree_create_flags flags)799 intel_miptree_create_for_bo(struct brw_context *brw,
800 struct brw_bo *bo,
801 mesa_format format,
802 uint32_t offset,
803 uint32_t width,
804 uint32_t height,
805 uint32_t depth,
806 int pitch,
807 enum isl_tiling tiling,
808 enum intel_miptree_create_flags flags)
809 {
810 const struct gen_device_info *devinfo = &brw->screen->devinfo;
811 struct intel_mipmap_tree *mt;
812 const GLenum target = depth > 1 ? GL_TEXTURE_2D_ARRAY : GL_TEXTURE_2D;
813 const GLenum base_format = _mesa_get_format_base_format(format);
814
815 if ((base_format == GL_DEPTH_COMPONENT ||
816 base_format == GL_DEPTH_STENCIL)) {
817 const mesa_format depth_only_format =
818 intel_depth_format_for_depthstencil_format(format);
819 mt = make_surface(brw, target,
820 devinfo->gen >= 6 ? depth_only_format : format,
821 0, 0, width, height, depth, 1, ISL_TILING_Y0_BIT,
822 ISL_SURF_USAGE_DEPTH_BIT | ISL_SURF_USAGE_TEXTURE_BIT,
823 BO_ALLOC_BUSY, pitch, bo);
824 if (!mt)
825 return NULL;
826
827 brw_bo_reference(bo);
828
829 if (!(flags & MIPTREE_CREATE_NO_AUX))
830 intel_miptree_choose_aux_usage(brw, mt);
831
832 return mt;
833 } else if (format == MESA_FORMAT_S_UINT8) {
834 mt = make_surface(brw, target, MESA_FORMAT_S_UINT8,
835 0, 0, width, height, depth, 1,
836 ISL_TILING_W_BIT,
837 ISL_SURF_USAGE_STENCIL_BIT |
838 ISL_SURF_USAGE_TEXTURE_BIT,
839 BO_ALLOC_BUSY, pitch, bo);
840 if (!mt)
841 return NULL;
842
843 assert(bo->size >= mt->surf.size);
844
845 brw_bo_reference(bo);
846 return mt;
847 }
848
849 /* Nothing will be able to use this miptree with the BO if the offset isn't
850 * aligned.
851 */
852 if (tiling != ISL_TILING_LINEAR)
853 assert(offset % 4096 == 0);
854
855 /* miptrees can't handle negative pitch. If you need flipping of images,
856 * that's outside of the scope of the mt.
857 */
858 assert(pitch >= 0);
859
860 /* The BO already has a tiling format and we shouldn't confuse the lower
861 * layers by making it try to find a tiling format again.
862 */
863 assert((flags & MIPTREE_CREATE_LINEAR) == 0);
864
865 mt = make_surface(brw, target, format,
866 0, 0, width, height, depth, 1,
867 1lu << tiling,
868 ISL_SURF_USAGE_RENDER_TARGET_BIT |
869 ISL_SURF_USAGE_TEXTURE_BIT,
870 0, pitch, bo);
871 if (!mt)
872 return NULL;
873
874 brw_bo_reference(bo);
875 mt->bo = bo;
876 mt->offset = offset;
877
878 if (!(flags & MIPTREE_CREATE_NO_AUX)) {
879 intel_miptree_choose_aux_usage(brw, mt);
880
881 if (!intel_miptree_alloc_aux(brw, mt)) {
882 intel_miptree_release(&mt);
883 return NULL;
884 }
885 }
886
887 return mt;
888 }
889
890 static struct intel_mipmap_tree *
miptree_create_for_planar_image(struct brw_context * brw,__DRIimage * image,GLenum target,enum isl_tiling tiling)891 miptree_create_for_planar_image(struct brw_context *brw,
892 __DRIimage *image, GLenum target,
893 enum isl_tiling tiling)
894 {
895 const struct intel_image_format *f = image->planar_format;
896 struct intel_mipmap_tree *planar_mt = NULL;
897
898 for (int i = 0; i < f->nplanes; i++) {
899 const int index = f->planes[i].buffer_index;
900 const uint32_t dri_format = f->planes[i].dri_format;
901 const mesa_format format = driImageFormatToGLFormat(dri_format);
902 const uint32_t width = image->width >> f->planes[i].width_shift;
903 const uint32_t height = image->height >> f->planes[i].height_shift;
904
905 /* Disable creation of the texture's aux buffers because the driver
906 * exposes no EGL API to manage them. That is, there is no API for
907 * resolving the aux buffer's content to the main buffer nor for
908 * invalidating the aux buffer's content.
909 */
910 struct intel_mipmap_tree *mt =
911 intel_miptree_create_for_bo(brw, image->bo, format,
912 image->offsets[index],
913 width, height, 1,
914 image->strides[index],
915 tiling,
916 MIPTREE_CREATE_NO_AUX);
917 if (mt == NULL)
918 return NULL;
919
920 mt->target = target;
921
922 if (i == 0)
923 planar_mt = mt;
924 else
925 planar_mt->plane[i - 1] = mt;
926 }
927
928 planar_mt->drm_modifier = image->modifier;
929
930 return planar_mt;
931 }
932
933 static bool
create_ccs_buf_for_image(struct brw_context * brw,__DRIimage * image,struct intel_mipmap_tree * mt,enum isl_aux_state initial_state)934 create_ccs_buf_for_image(struct brw_context *brw,
935 __DRIimage *image,
936 struct intel_mipmap_tree *mt,
937 enum isl_aux_state initial_state)
938 {
939 struct isl_surf temp_ccs_surf;
940
941 /* CCS is only supported for very simple miptrees */
942 assert(image->aux_offset != 0 && image->aux_pitch != 0);
943 assert(image->tile_x == 0 && image->tile_y == 0);
944 assert(mt->surf.samples == 1);
945 assert(mt->surf.levels == 1);
946 assert(mt->surf.logical_level0_px.depth == 1);
947 assert(mt->surf.logical_level0_px.array_len == 1);
948 assert(mt->first_level == 0);
949 assert(mt->last_level == 0);
950
951 /* We shouldn't already have a CCS */
952 assert(!mt->mcs_buf);
953
954 if (!isl_surf_get_ccs_surf(&brw->isl_dev, &mt->surf, &temp_ccs_surf,
955 image->aux_pitch))
956 return false;
957
958 assert(image->aux_offset < image->bo->size);
959 assert(temp_ccs_surf.size <= image->bo->size - image->aux_offset);
960
961 mt->mcs_buf = calloc(sizeof(*mt->mcs_buf), 1);
962 if (mt->mcs_buf == NULL)
963 return false;
964
965 mt->aux_state = create_aux_state_map(mt, initial_state);
966 if (!mt->aux_state) {
967 free(mt->mcs_buf);
968 mt->mcs_buf = NULL;
969 return false;
970 }
971
972 mt->mcs_buf->bo = image->bo;
973 brw_bo_reference(image->bo);
974
975 mt->mcs_buf->offset = image->aux_offset;
976 mt->mcs_buf->size = image->bo->size - image->aux_offset;
977 mt->mcs_buf->pitch = image->aux_pitch;
978 mt->mcs_buf->qpitch = 0;
979 mt->mcs_buf->surf = temp_ccs_surf;
980
981 return true;
982 }
983
984 struct intel_mipmap_tree *
intel_miptree_create_for_dri_image(struct brw_context * brw,__DRIimage * image,GLenum target,mesa_format format,bool is_winsys_image)985 intel_miptree_create_for_dri_image(struct brw_context *brw,
986 __DRIimage *image, GLenum target,
987 mesa_format format,
988 bool is_winsys_image)
989 {
990 uint32_t bo_tiling, bo_swizzle;
991 brw_bo_get_tiling(image->bo, &bo_tiling, &bo_swizzle);
992
993 const struct isl_drm_modifier_info *mod_info =
994 isl_drm_modifier_get_info(image->modifier);
995
996 const enum isl_tiling tiling =
997 mod_info ? mod_info->tiling : isl_tiling_from_i915_tiling(bo_tiling);
998
999 if (image->planar_format && image->planar_format->nplanes > 1)
1000 return miptree_create_for_planar_image(brw, image, target, tiling);
1001
1002 if (image->planar_format)
1003 assert(image->planar_format->planes[0].dri_format == image->dri_format);
1004
1005 if (!brw->ctx.TextureFormatSupported[format]) {
1006 /* The texture storage paths in core Mesa detect if the driver does not
1007 * support the user-requested format, and then searches for a
1008 * fallback format. The DRIimage code bypasses core Mesa, though. So we
1009 * do the fallbacks here for important formats.
1010 *
1011 * We must support DRM_FOURCC_XBGR8888 textures because the Android
1012 * framework produces HAL_PIXEL_FORMAT_RGBX8888 winsys surfaces, which
1013 * the Chrome OS compositor consumes as dma_buf EGLImages.
1014 */
1015 format = _mesa_format_fallback_rgbx_to_rgba(format);
1016 }
1017
1018 if (!brw->ctx.TextureFormatSupported[format])
1019 return NULL;
1020
1021 enum intel_miptree_create_flags mt_create_flags = 0;
1022
1023 /* If this image comes in from a window system, we have different
1024 * requirements than if it comes in via an EGL import operation. Window
1025 * system images can use any form of auxiliary compression we wish because
1026 * they get "flushed" before being handed off to the window system and we
1027 * have the opportunity to do resolves. Non window-system images, on the
1028 * other hand, have no resolve point so we can't have aux without a
1029 * modifier.
1030 */
1031 if (!is_winsys_image)
1032 mt_create_flags |= MIPTREE_CREATE_NO_AUX;
1033
1034 /* If we have a modifier which specifies aux, don't create one yet */
1035 if (mod_info && mod_info->aux_usage != ISL_AUX_USAGE_NONE)
1036 mt_create_flags |= MIPTREE_CREATE_NO_AUX;
1037
1038 /* Disable creation of the texture's aux buffers because the driver exposes
1039 * no EGL API to manage them. That is, there is no API for resolving the aux
1040 * buffer's content to the main buffer nor for invalidating the aux buffer's
1041 * content.
1042 */
1043 struct intel_mipmap_tree *mt =
1044 intel_miptree_create_for_bo(brw, image->bo, format,
1045 image->offset, image->width, image->height, 1,
1046 image->pitch, tiling, mt_create_flags);
1047 if (mt == NULL)
1048 return NULL;
1049
1050 mt->target = target;
1051 mt->level[0].level_x = image->tile_x;
1052 mt->level[0].level_y = image->tile_y;
1053 mt->drm_modifier = image->modifier;
1054
1055 /* From "OES_EGL_image" error reporting. We report GL_INVALID_OPERATION
1056 * for EGL images from non-tile aligned sufaces in gen4 hw and earlier which has
1057 * trouble resolving back to destination image due to alignment issues.
1058 */
1059 const struct gen_device_info *devinfo = &brw->screen->devinfo;
1060 if (!devinfo->has_surface_tile_offset) {
1061 uint32_t draw_x, draw_y;
1062 intel_miptree_get_tile_offsets(mt, 0, 0, &draw_x, &draw_y);
1063
1064 if (draw_x != 0 || draw_y != 0) {
1065 _mesa_error(&brw->ctx, GL_INVALID_OPERATION, __func__);
1066 intel_miptree_release(&mt);
1067 return NULL;
1068 }
1069 }
1070
1071 if (mod_info && mod_info->aux_usage != ISL_AUX_USAGE_NONE) {
1072 assert(mod_info->aux_usage == ISL_AUX_USAGE_CCS_E);
1073
1074 mt->aux_usage = mod_info->aux_usage;
1075 /* If we are a window system buffer, then we can support fast-clears
1076 * even if the modifier doesn't support them by doing a partial resolve
1077 * as part of the flush operation.
1078 */
1079 mt->supports_fast_clear =
1080 is_winsys_image || mod_info->supports_clear_color;
1081
1082 /* We don't know the actual state of the surface when we get it but we
1083 * can make a pretty good guess based on the modifier. What we do know
1084 * for sure is that it isn't in the AUX_INVALID state, so we just assume
1085 * a worst case of compression.
1086 */
1087 enum isl_aux_state initial_state =
1088 isl_drm_modifier_get_default_aux_state(image->modifier);
1089
1090 if (!create_ccs_buf_for_image(brw, image, mt, initial_state)) {
1091 intel_miptree_release(&mt);
1092 return NULL;
1093 }
1094 }
1095
1096 /* Don't assume coherency for imported EGLimages. We don't know what
1097 * external clients are going to do with it. They may scan it out.
1098 */
1099 image->bo->cache_coherent = false;
1100
1101 return mt;
1102 }
1103
1104 /**
1105 * For a singlesample renderbuffer, this simply wraps the given BO with a
1106 * miptree.
1107 *
1108 * For a multisample renderbuffer, this wraps the window system's
1109 * (singlesample) BO with a singlesample miptree attached to the
1110 * intel_renderbuffer, then creates a multisample miptree attached to irb->mt
1111 * that will contain the actual rendering (which is lazily resolved to
1112 * irb->singlesample_mt).
1113 */
1114 bool
intel_update_winsys_renderbuffer_miptree(struct brw_context * intel,struct intel_renderbuffer * irb,struct intel_mipmap_tree * singlesample_mt,uint32_t width,uint32_t height,uint32_t pitch)1115 intel_update_winsys_renderbuffer_miptree(struct brw_context *intel,
1116 struct intel_renderbuffer *irb,
1117 struct intel_mipmap_tree *singlesample_mt,
1118 uint32_t width, uint32_t height,
1119 uint32_t pitch)
1120 {
1121 struct intel_mipmap_tree *multisample_mt = NULL;
1122 struct gl_renderbuffer *rb = &irb->Base.Base;
1123 mesa_format format = rb->Format;
1124 const unsigned num_samples = MAX2(rb->NumSamples, 1);
1125
1126 /* Only the front and back buffers, which are color buffers, are allocated
1127 * through the image loader.
1128 */
1129 assert(_mesa_get_format_base_format(format) == GL_RGB ||
1130 _mesa_get_format_base_format(format) == GL_RGBA);
1131
1132 assert(singlesample_mt);
1133
1134 if (num_samples == 1) {
1135 intel_miptree_release(&irb->mt);
1136 irb->mt = singlesample_mt;
1137
1138 assert(!irb->singlesample_mt);
1139 } else {
1140 intel_miptree_release(&irb->singlesample_mt);
1141 irb->singlesample_mt = singlesample_mt;
1142
1143 if (!irb->mt ||
1144 irb->mt->surf.logical_level0_px.width != width ||
1145 irb->mt->surf.logical_level0_px.height != height) {
1146 multisample_mt = intel_miptree_create_for_renderbuffer(intel,
1147 format,
1148 width,
1149 height,
1150 num_samples);
1151 if (!multisample_mt)
1152 goto fail;
1153
1154 irb->need_downsample = false;
1155 intel_miptree_release(&irb->mt);
1156 irb->mt = multisample_mt;
1157 }
1158 }
1159 return true;
1160
1161 fail:
1162 intel_miptree_release(&irb->mt);
1163 return false;
1164 }
1165
1166 struct intel_mipmap_tree*
intel_miptree_create_for_renderbuffer(struct brw_context * brw,mesa_format format,uint32_t width,uint32_t height,uint32_t num_samples)1167 intel_miptree_create_for_renderbuffer(struct brw_context *brw,
1168 mesa_format format,
1169 uint32_t width,
1170 uint32_t height,
1171 uint32_t num_samples)
1172 {
1173 struct intel_mipmap_tree *mt;
1174 uint32_t depth = 1;
1175 GLenum target = num_samples > 1 ? GL_TEXTURE_2D_MULTISAMPLE : GL_TEXTURE_2D;
1176
1177 mt = intel_miptree_create(brw, target, format, 0, 0,
1178 width, height, depth, num_samples,
1179 MIPTREE_CREATE_BUSY);
1180 if (!mt)
1181 goto fail;
1182
1183 return mt;
1184
1185 fail:
1186 intel_miptree_release(&mt);
1187 return NULL;
1188 }
1189
1190 void
intel_miptree_reference(struct intel_mipmap_tree ** dst,struct intel_mipmap_tree * src)1191 intel_miptree_reference(struct intel_mipmap_tree **dst,
1192 struct intel_mipmap_tree *src)
1193 {
1194 if (*dst == src)
1195 return;
1196
1197 intel_miptree_release(dst);
1198
1199 if (src) {
1200 src->refcount++;
1201 DBG("%s %p refcount now %d\n", __func__, src, src->refcount);
1202 }
1203
1204 *dst = src;
1205 }
1206
1207 static void
intel_miptree_aux_buffer_free(struct intel_miptree_aux_buffer * aux_buf)1208 intel_miptree_aux_buffer_free(struct intel_miptree_aux_buffer *aux_buf)
1209 {
1210 if (aux_buf == NULL)
1211 return;
1212
1213 brw_bo_unreference(aux_buf->bo);
1214
1215 free(aux_buf);
1216 }
1217
1218 void
intel_miptree_release(struct intel_mipmap_tree ** mt)1219 intel_miptree_release(struct intel_mipmap_tree **mt)
1220 {
1221 if (!*mt)
1222 return;
1223
1224 DBG("%s %p refcount will be %d\n", __func__, *mt, (*mt)->refcount - 1);
1225 if (--(*mt)->refcount <= 0) {
1226 GLuint i;
1227
1228 DBG("%s deleting %p\n", __func__, *mt);
1229
1230 brw_bo_unreference((*mt)->bo);
1231 intel_miptree_release(&(*mt)->stencil_mt);
1232 intel_miptree_release(&(*mt)->r8stencil_mt);
1233 intel_miptree_aux_buffer_free((*mt)->hiz_buf);
1234 intel_miptree_aux_buffer_free((*mt)->mcs_buf);
1235 free_aux_state_map((*mt)->aux_state);
1236
1237 intel_miptree_release(&(*mt)->plane[0]);
1238 intel_miptree_release(&(*mt)->plane[1]);
1239
1240 for (i = 0; i < MAX_TEXTURE_LEVELS; i++) {
1241 free((*mt)->level[i].slice);
1242 }
1243
1244 free(*mt);
1245 }
1246 *mt = NULL;
1247 }
1248
1249
1250 void
intel_get_image_dims(struct gl_texture_image * image,int * width,int * height,int * depth)1251 intel_get_image_dims(struct gl_texture_image *image,
1252 int *width, int *height, int *depth)
1253 {
1254 switch (image->TexObject->Target) {
1255 case GL_TEXTURE_1D_ARRAY:
1256 /* For a 1D Array texture the OpenGL API will treat the image height as
1257 * the number of array slices. For Intel hardware, we treat the 1D array
1258 * as a 2D Array with a height of 1. So, here we want to swap image
1259 * height and depth.
1260 */
1261 assert(image->Depth == 1);
1262 *width = image->Width;
1263 *height = 1;
1264 *depth = image->Height;
1265 break;
1266 case GL_TEXTURE_CUBE_MAP:
1267 /* For Cube maps, the mesa/main api layer gives us a depth of 1 even
1268 * though we really have 6 slices.
1269 */
1270 assert(image->Depth == 1);
1271 *width = image->Width;
1272 *height = image->Height;
1273 *depth = 6;
1274 break;
1275 default:
1276 *width = image->Width;
1277 *height = image->Height;
1278 *depth = image->Depth;
1279 break;
1280 }
1281 }
1282
1283 /**
1284 * Can the image be pulled into a unified mipmap tree? This mirrors
1285 * the completeness test in a lot of ways.
1286 *
1287 * Not sure whether I want to pass gl_texture_image here.
1288 */
1289 bool
intel_miptree_match_image(struct intel_mipmap_tree * mt,struct gl_texture_image * image)1290 intel_miptree_match_image(struct intel_mipmap_tree *mt,
1291 struct gl_texture_image *image)
1292 {
1293 struct intel_texture_image *intelImage = intel_texture_image(image);
1294 GLuint level = intelImage->base.Base.Level;
1295 int width, height, depth;
1296
1297 /* glTexImage* choose the texture object based on the target passed in, and
1298 * objects can't change targets over their lifetimes, so this should be
1299 * true.
1300 */
1301 assert(image->TexObject->Target == mt->target);
1302
1303 mesa_format mt_format = mt->format;
1304 if (mt->format == MESA_FORMAT_Z24_UNORM_X8_UINT && mt->stencil_mt)
1305 mt_format = MESA_FORMAT_Z24_UNORM_S8_UINT;
1306 if (mt->format == MESA_FORMAT_Z_FLOAT32 && mt->stencil_mt)
1307 mt_format = MESA_FORMAT_Z32_FLOAT_S8X24_UINT;
1308 if (mt->etc_format != MESA_FORMAT_NONE)
1309 mt_format = mt->etc_format;
1310
1311 if (image->TexFormat != mt_format)
1312 return false;
1313
1314 intel_get_image_dims(image, &width, &height, &depth);
1315
1316 if (mt->target == GL_TEXTURE_CUBE_MAP)
1317 depth = 6;
1318
1319 if (level >= mt->surf.levels)
1320 return false;
1321
1322 const unsigned level_depth =
1323 mt->surf.dim == ISL_SURF_DIM_3D ?
1324 minify(mt->surf.logical_level0_px.depth, level) :
1325 mt->surf.logical_level0_px.array_len;
1326
1327 return width == minify(mt->surf.logical_level0_px.width, level) &&
1328 height == minify(mt->surf.logical_level0_px.height, level) &&
1329 depth == level_depth &&
1330 MAX2(image->NumSamples, 1) == mt->surf.samples;
1331 }
1332
1333 void
intel_miptree_get_image_offset(const struct intel_mipmap_tree * mt,GLuint level,GLuint slice,GLuint * x,GLuint * y)1334 intel_miptree_get_image_offset(const struct intel_mipmap_tree *mt,
1335 GLuint level, GLuint slice,
1336 GLuint *x, GLuint *y)
1337 {
1338 if (level == 0 && slice == 0) {
1339 *x = mt->level[0].level_x;
1340 *y = mt->level[0].level_y;
1341 return;
1342 }
1343
1344 uint32_t x_offset_sa, y_offset_sa;
1345
1346 /* Miptree itself can have an offset only if it represents a single
1347 * slice in an imported buffer object.
1348 * See intel_miptree_create_for_dri_image().
1349 */
1350 assert(mt->level[0].level_x == 0);
1351 assert(mt->level[0].level_y == 0);
1352
1353 /* Given level is relative to level zero while the miptree may be
1354 * represent just a subset of all levels starting from 'first_level'.
1355 */
1356 assert(level >= mt->first_level);
1357 level -= mt->first_level;
1358
1359 const unsigned z = mt->surf.dim == ISL_SURF_DIM_3D ? slice : 0;
1360 slice = mt->surf.dim == ISL_SURF_DIM_3D ? 0 : slice;
1361 isl_surf_get_image_offset_el(&mt->surf, level, slice, z,
1362 &x_offset_sa, &y_offset_sa);
1363
1364 *x = x_offset_sa;
1365 *y = y_offset_sa;
1366 }
1367
1368
1369 /**
1370 * This function computes the tile_w (in bytes) and tile_h (in rows) of
1371 * different tiling patterns. If the BO is untiled, tile_w is set to cpp
1372 * and tile_h is set to 1.
1373 */
1374 void
intel_get_tile_dims(enum isl_tiling tiling,uint32_t cpp,uint32_t * tile_w,uint32_t * tile_h)1375 intel_get_tile_dims(enum isl_tiling tiling, uint32_t cpp,
1376 uint32_t *tile_w, uint32_t *tile_h)
1377 {
1378 switch (tiling) {
1379 case ISL_TILING_X:
1380 *tile_w = 512;
1381 *tile_h = 8;
1382 break;
1383 case ISL_TILING_Y0:
1384 *tile_w = 128;
1385 *tile_h = 32;
1386 break;
1387 case ISL_TILING_LINEAR:
1388 *tile_w = cpp;
1389 *tile_h = 1;
1390 break;
1391 default:
1392 unreachable("not reached");
1393 }
1394 }
1395
1396
1397 /**
1398 * This function computes masks that may be used to select the bits of the X
1399 * and Y coordinates that indicate the offset within a tile. If the BO is
1400 * untiled, the masks are set to 0.
1401 */
1402 void
intel_get_tile_masks(enum isl_tiling tiling,uint32_t cpp,uint32_t * mask_x,uint32_t * mask_y)1403 intel_get_tile_masks(enum isl_tiling tiling, uint32_t cpp,
1404 uint32_t *mask_x, uint32_t *mask_y)
1405 {
1406 uint32_t tile_w_bytes, tile_h;
1407
1408 intel_get_tile_dims(tiling, cpp, &tile_w_bytes, &tile_h);
1409
1410 *mask_x = tile_w_bytes / cpp - 1;
1411 *mask_y = tile_h - 1;
1412 }
1413
1414 /**
1415 * Compute the offset (in bytes) from the start of the BO to the given x
1416 * and y coordinate. For tiled BOs, caller must ensure that x and y are
1417 * multiples of the tile size.
1418 */
1419 uint32_t
intel_miptree_get_aligned_offset(const struct intel_mipmap_tree * mt,uint32_t x,uint32_t y)1420 intel_miptree_get_aligned_offset(const struct intel_mipmap_tree *mt,
1421 uint32_t x, uint32_t y)
1422 {
1423 int cpp = mt->cpp;
1424 uint32_t pitch = mt->surf.row_pitch;
1425
1426 switch (mt->surf.tiling) {
1427 default:
1428 unreachable("not reached");
1429 case ISL_TILING_LINEAR:
1430 return y * pitch + x * cpp;
1431 case ISL_TILING_X:
1432 assert((x % (512 / cpp)) == 0);
1433 assert((y % 8) == 0);
1434 return y * pitch + x / (512 / cpp) * 4096;
1435 case ISL_TILING_Y0:
1436 assert((x % (128 / cpp)) == 0);
1437 assert((y % 32) == 0);
1438 return y * pitch + x / (128 / cpp) * 4096;
1439 }
1440 }
1441
1442 /**
1443 * Rendering with tiled buffers requires that the base address of the buffer
1444 * be aligned to a page boundary. For renderbuffers, and sometimes with
1445 * textures, we may want the surface to point at a texture image level that
1446 * isn't at a page boundary.
1447 *
1448 * This function returns an appropriately-aligned base offset
1449 * according to the tiling restrictions, plus any required x/y offset
1450 * from there.
1451 */
1452 uint32_t
intel_miptree_get_tile_offsets(const struct intel_mipmap_tree * mt,GLuint level,GLuint slice,uint32_t * tile_x,uint32_t * tile_y)1453 intel_miptree_get_tile_offsets(const struct intel_mipmap_tree *mt,
1454 GLuint level, GLuint slice,
1455 uint32_t *tile_x,
1456 uint32_t *tile_y)
1457 {
1458 uint32_t x, y;
1459 uint32_t mask_x, mask_y;
1460
1461 intel_get_tile_masks(mt->surf.tiling, mt->cpp, &mask_x, &mask_y);
1462 intel_miptree_get_image_offset(mt, level, slice, &x, &y);
1463
1464 *tile_x = x & mask_x;
1465 *tile_y = y & mask_y;
1466
1467 return intel_miptree_get_aligned_offset(mt, x & ~mask_x, y & ~mask_y);
1468 }
1469
1470 static void
intel_miptree_copy_slice_sw(struct brw_context * brw,struct intel_mipmap_tree * src_mt,unsigned src_level,unsigned src_layer,struct intel_mipmap_tree * dst_mt,unsigned dst_level,unsigned dst_layer,unsigned width,unsigned height)1471 intel_miptree_copy_slice_sw(struct brw_context *brw,
1472 struct intel_mipmap_tree *src_mt,
1473 unsigned src_level, unsigned src_layer,
1474 struct intel_mipmap_tree *dst_mt,
1475 unsigned dst_level, unsigned dst_layer,
1476 unsigned width, unsigned height)
1477 {
1478 void *src, *dst;
1479 ptrdiff_t src_stride, dst_stride;
1480 const unsigned cpp = (isl_format_get_layout(dst_mt->surf.format)->bpb / 8);
1481
1482 intel_miptree_map(brw, src_mt,
1483 src_level, src_layer,
1484 0, 0,
1485 width, height,
1486 GL_MAP_READ_BIT | BRW_MAP_DIRECT_BIT,
1487 &src, &src_stride);
1488
1489 intel_miptree_map(brw, dst_mt,
1490 dst_level, dst_layer,
1491 0, 0,
1492 width, height,
1493 GL_MAP_WRITE_BIT | GL_MAP_INVALIDATE_RANGE_BIT |
1494 BRW_MAP_DIRECT_BIT,
1495 &dst, &dst_stride);
1496
1497 DBG("sw blit %s mt %p %p/%"PRIdPTR" -> %s mt %p %p/%"PRIdPTR" (%dx%d)\n",
1498 _mesa_get_format_name(src_mt->format),
1499 src_mt, src, src_stride,
1500 _mesa_get_format_name(dst_mt->format),
1501 dst_mt, dst, dst_stride,
1502 width, height);
1503
1504 int row_size = cpp * width;
1505 if (src_stride == row_size &&
1506 dst_stride == row_size) {
1507 memcpy(dst, src, row_size * height);
1508 } else {
1509 for (int i = 0; i < height; i++) {
1510 memcpy(dst, src, row_size);
1511 dst += dst_stride;
1512 src += src_stride;
1513 }
1514 }
1515
1516 intel_miptree_unmap(brw, dst_mt, dst_level, dst_layer);
1517 intel_miptree_unmap(brw, src_mt, src_level, src_layer);
1518
1519 /* Don't forget to copy the stencil data over, too. We could have skipped
1520 * passing BRW_MAP_DIRECT_BIT, but that would have meant intel_miptree_map
1521 * shuffling the two data sources in/out of temporary storage instead of
1522 * the direct mapping we get this way.
1523 */
1524 if (dst_mt->stencil_mt) {
1525 assert(src_mt->stencil_mt);
1526 intel_miptree_copy_slice_sw(brw,
1527 src_mt->stencil_mt, src_level, src_layer,
1528 dst_mt->stencil_mt, dst_level, dst_layer,
1529 width, height);
1530 }
1531 }
1532
1533 void
intel_miptree_copy_slice(struct brw_context * brw,struct intel_mipmap_tree * src_mt,unsigned src_level,unsigned src_layer,struct intel_mipmap_tree * dst_mt,unsigned dst_level,unsigned dst_layer)1534 intel_miptree_copy_slice(struct brw_context *brw,
1535 struct intel_mipmap_tree *src_mt,
1536 unsigned src_level, unsigned src_layer,
1537 struct intel_mipmap_tree *dst_mt,
1538 unsigned dst_level, unsigned dst_layer)
1539
1540 {
1541 mesa_format format = src_mt->format;
1542 unsigned width = minify(src_mt->surf.phys_level0_sa.width,
1543 src_level - src_mt->first_level);
1544 unsigned height = minify(src_mt->surf.phys_level0_sa.height,
1545 src_level - src_mt->first_level);
1546
1547 assert(src_layer < get_num_phys_layers(&src_mt->surf,
1548 src_level - src_mt->first_level));
1549
1550 assert(src_mt->format == dst_mt->format);
1551
1552 if (dst_mt->compressed) {
1553 unsigned int i, j;
1554 _mesa_get_format_block_size(dst_mt->format, &i, &j);
1555 height = ALIGN_NPOT(height, j) / j;
1556 width = ALIGN_NPOT(width, i) / i;
1557 }
1558
1559 /* If it's a packed depth/stencil buffer with separate stencil, the blit
1560 * below won't apply since we can't do the depth's Y tiling or the
1561 * stencil's W tiling in the blitter.
1562 */
1563 if (src_mt->stencil_mt) {
1564 intel_miptree_copy_slice_sw(brw,
1565 src_mt, src_level, src_layer,
1566 dst_mt, dst_level, dst_layer,
1567 width, height);
1568 return;
1569 }
1570
1571 uint32_t dst_x, dst_y, src_x, src_y;
1572 intel_miptree_get_image_offset(dst_mt, dst_level, dst_layer,
1573 &dst_x, &dst_y);
1574 intel_miptree_get_image_offset(src_mt, src_level, src_layer,
1575 &src_x, &src_y);
1576
1577 DBG("validate blit mt %s %p %d,%d/%d -> mt %s %p %d,%d/%d (%dx%d)\n",
1578 _mesa_get_format_name(src_mt->format),
1579 src_mt, src_x, src_y, src_mt->surf.row_pitch,
1580 _mesa_get_format_name(dst_mt->format),
1581 dst_mt, dst_x, dst_y, dst_mt->surf.row_pitch,
1582 width, height);
1583
1584 if (!intel_miptree_blit(brw,
1585 src_mt, src_level, src_layer, 0, 0, false,
1586 dst_mt, dst_level, dst_layer, 0, 0, false,
1587 width, height, GL_COPY)) {
1588 perf_debug("miptree validate blit for %s failed\n",
1589 _mesa_get_format_name(format));
1590
1591 intel_miptree_copy_slice_sw(brw,
1592 src_mt, src_level, src_layer,
1593 dst_mt, dst_level, dst_layer,
1594 width, height);
1595 }
1596 }
1597
1598 /**
1599 * Copies the image's current data to the given miptree, and associates that
1600 * miptree with the image.
1601 */
1602 void
intel_miptree_copy_teximage(struct brw_context * brw,struct intel_texture_image * intelImage,struct intel_mipmap_tree * dst_mt)1603 intel_miptree_copy_teximage(struct brw_context *brw,
1604 struct intel_texture_image *intelImage,
1605 struct intel_mipmap_tree *dst_mt)
1606 {
1607 struct intel_mipmap_tree *src_mt = intelImage->mt;
1608 struct intel_texture_object *intel_obj =
1609 intel_texture_object(intelImage->base.Base.TexObject);
1610 int level = intelImage->base.Base.Level;
1611 const unsigned face = intelImage->base.Base.Face;
1612 unsigned start_layer, end_layer;
1613
1614 if (intel_obj->base.Target == GL_TEXTURE_1D_ARRAY) {
1615 assert(face == 0);
1616 assert(intelImage->base.Base.Height);
1617 start_layer = 0;
1618 end_layer = intelImage->base.Base.Height - 1;
1619 } else if (face > 0) {
1620 start_layer = face;
1621 end_layer = face;
1622 } else {
1623 assert(intelImage->base.Base.Depth);
1624 start_layer = 0;
1625 end_layer = intelImage->base.Base.Depth - 1;
1626 }
1627
1628 for (unsigned i = start_layer; i <= end_layer; i++) {
1629 intel_miptree_copy_slice(brw,
1630 src_mt, level, i,
1631 dst_mt, level, i);
1632 }
1633
1634 intel_miptree_reference(&intelImage->mt, dst_mt);
1635 intel_obj->needs_validate = true;
1636 }
1637
1638 static void
intel_miptree_init_mcs(struct brw_context * brw,struct intel_mipmap_tree * mt,int init_value)1639 intel_miptree_init_mcs(struct brw_context *brw,
1640 struct intel_mipmap_tree *mt,
1641 int init_value)
1642 {
1643 assert(mt->mcs_buf != NULL);
1644
1645 /* From the Ivy Bridge PRM, Vol 2 Part 1 p326:
1646 *
1647 * When MCS buffer is enabled and bound to MSRT, it is required that it
1648 * is cleared prior to any rendering.
1649 *
1650 * Since we don't use the MCS buffer for any purpose other than rendering,
1651 * it makes sense to just clear it immediately upon allocation.
1652 *
1653 * Note: the clear value for MCS buffers is all 1's, so we memset to 0xff.
1654 */
1655 void *map = brw_bo_map(brw, mt->mcs_buf->bo, MAP_WRITE);
1656 if (unlikely(map == NULL)) {
1657 fprintf(stderr, "Failed to map mcs buffer into GTT\n");
1658 brw_bo_unreference(mt->mcs_buf->bo);
1659 free(mt->mcs_buf);
1660 return;
1661 }
1662 void *data = map;
1663 memset(data, init_value, mt->mcs_buf->size);
1664 brw_bo_unmap(mt->mcs_buf->bo);
1665 }
1666
1667 static struct intel_miptree_aux_buffer *
intel_alloc_aux_buffer(struct brw_context * brw,const char * name,const struct isl_surf * aux_surf,uint32_t alloc_flags,struct intel_mipmap_tree * mt)1668 intel_alloc_aux_buffer(struct brw_context *brw,
1669 const char *name,
1670 const struct isl_surf *aux_surf,
1671 uint32_t alloc_flags,
1672 struct intel_mipmap_tree *mt)
1673 {
1674 struct intel_miptree_aux_buffer *buf = calloc(sizeof(*buf), 1);
1675 if (!buf)
1676 return false;
1677
1678 buf->size = aux_surf->size;
1679 buf->pitch = aux_surf->row_pitch;
1680 buf->qpitch = isl_surf_get_array_pitch_sa_rows(aux_surf);
1681
1682 /* ISL has stricter set of alignment rules then the drm allocator.
1683 * Therefore one can pass the ISL dimensions in terms of bytes instead of
1684 * trying to recalculate based on different format block sizes.
1685 */
1686 buf->bo = brw_bo_alloc_tiled(brw->bufmgr, name, buf->size,
1687 I915_TILING_Y, buf->pitch, alloc_flags);
1688 if (!buf->bo) {
1689 free(buf);
1690 return NULL;
1691 }
1692
1693 buf->surf = *aux_surf;
1694
1695 return buf;
1696 }
1697
1698 static bool
intel_miptree_alloc_mcs(struct brw_context * brw,struct intel_mipmap_tree * mt,GLuint num_samples)1699 intel_miptree_alloc_mcs(struct brw_context *brw,
1700 struct intel_mipmap_tree *mt,
1701 GLuint num_samples)
1702 {
1703 assert(brw->screen->devinfo.gen >= 7); /* MCS only used on Gen7+ */
1704 assert(mt->mcs_buf == NULL);
1705 assert(mt->aux_usage == ISL_AUX_USAGE_MCS);
1706
1707 /* Multisampled miptrees are only supported for single level. */
1708 assert(mt->first_level == 0);
1709 enum isl_aux_state **aux_state =
1710 create_aux_state_map(mt, ISL_AUX_STATE_CLEAR);
1711 if (!aux_state)
1712 return false;
1713
1714 struct isl_surf temp_mcs_surf;
1715
1716 MAYBE_UNUSED bool ok =
1717 isl_surf_get_mcs_surf(&brw->isl_dev, &mt->surf, &temp_mcs_surf);
1718 assert(ok);
1719
1720 /* Buffer needs to be initialised requiring the buffer to be immediately
1721 * mapped to cpu space for writing. Therefore do not use the gpu access
1722 * flag which can cause an unnecessary delay if the backing pages happened
1723 * to be just used by the GPU.
1724 */
1725 const uint32_t alloc_flags = 0;
1726 mt->mcs_buf = intel_alloc_aux_buffer(brw, "mcs-miptree",
1727 &temp_mcs_surf, alloc_flags, mt);
1728 if (!mt->mcs_buf) {
1729 free(aux_state);
1730 return false;
1731 }
1732
1733 mt->aux_state = aux_state;
1734
1735 intel_miptree_init_mcs(brw, mt, 0xFF);
1736
1737 return true;
1738 }
1739
1740 bool
intel_miptree_alloc_ccs(struct brw_context * brw,struct intel_mipmap_tree * mt)1741 intel_miptree_alloc_ccs(struct brw_context *brw,
1742 struct intel_mipmap_tree *mt)
1743 {
1744 assert(mt->mcs_buf == NULL);
1745 assert(mt->aux_usage == ISL_AUX_USAGE_CCS_E ||
1746 mt->aux_usage == ISL_AUX_USAGE_CCS_D);
1747
1748 struct isl_surf temp_ccs_surf;
1749
1750 if (!isl_surf_get_ccs_surf(&brw->isl_dev, &mt->surf, &temp_ccs_surf, 0))
1751 return false;
1752
1753 assert(temp_ccs_surf.size &&
1754 (temp_ccs_surf.size % temp_ccs_surf.row_pitch == 0));
1755
1756 enum isl_aux_state **aux_state =
1757 create_aux_state_map(mt, ISL_AUX_STATE_PASS_THROUGH);
1758 if (!aux_state)
1759 return false;
1760
1761 /* When CCS_E is used, we need to ensure that the CCS starts off in a valid
1762 * state. From the Sky Lake PRM, "MCS Buffer for Render Target(s)":
1763 *
1764 * "If Software wants to enable Color Compression without Fast clear,
1765 * Software needs to initialize MCS with zeros."
1766 *
1767 * A CCS value of 0 indicates that the corresponding block is in the
1768 * pass-through state which is what we want.
1769 *
1770 * For CCS_D, on the other hand, we don't care as we're about to perform a
1771 * fast-clear operation. In that case, being hot in caches more useful.
1772 */
1773 const uint32_t alloc_flags = mt->aux_usage == ISL_AUX_USAGE_CCS_E ?
1774 BO_ALLOC_ZEROED : BO_ALLOC_BUSY;
1775 mt->mcs_buf = intel_alloc_aux_buffer(brw, "ccs-miptree",
1776 &temp_ccs_surf, alloc_flags, mt);
1777 if (!mt->mcs_buf) {
1778 free(aux_state);
1779 return false;
1780 }
1781
1782 mt->aux_state = aux_state;
1783
1784 return true;
1785 }
1786
1787 /**
1788 * Helper for intel_miptree_alloc_hiz() that sets
1789 * \c mt->level[level].has_hiz. Return true if and only if
1790 * \c has_hiz was set.
1791 */
1792 static bool
intel_miptree_level_enable_hiz(struct brw_context * brw,struct intel_mipmap_tree * mt,uint32_t level)1793 intel_miptree_level_enable_hiz(struct brw_context *brw,
1794 struct intel_mipmap_tree *mt,
1795 uint32_t level)
1796 {
1797 const struct gen_device_info *devinfo = &brw->screen->devinfo;
1798
1799 assert(mt->hiz_buf);
1800 assert(mt->surf.size > 0);
1801
1802 if (devinfo->gen >= 8 || devinfo->is_haswell) {
1803 uint32_t width = minify(mt->surf.phys_level0_sa.width, level);
1804 uint32_t height = minify(mt->surf.phys_level0_sa.height, level);
1805
1806 /* Disable HiZ for LOD > 0 unless the width is 8 aligned
1807 * and the height is 4 aligned. This allows our HiZ support
1808 * to fulfill Haswell restrictions for HiZ ops. For LOD == 0,
1809 * we can grow the width & height to allow the HiZ op to
1810 * force the proper size alignments.
1811 */
1812 if (level > 0 && ((width & 7) || (height & 3))) {
1813 DBG("mt %p level %d: HiZ DISABLED\n", mt, level);
1814 return false;
1815 }
1816 }
1817
1818 DBG("mt %p level %d: HiZ enabled\n", mt, level);
1819 mt->level[level].has_hiz = true;
1820 return true;
1821 }
1822
1823 bool
intel_miptree_alloc_hiz(struct brw_context * brw,struct intel_mipmap_tree * mt)1824 intel_miptree_alloc_hiz(struct brw_context *brw,
1825 struct intel_mipmap_tree *mt)
1826 {
1827 assert(mt->hiz_buf == NULL);
1828 assert(mt->aux_usage == ISL_AUX_USAGE_HIZ);
1829
1830 enum isl_aux_state **aux_state =
1831 create_aux_state_map(mt, ISL_AUX_STATE_AUX_INVALID);
1832 if (!aux_state)
1833 return false;
1834
1835 struct isl_surf temp_hiz_surf;
1836
1837 MAYBE_UNUSED bool ok =
1838 isl_surf_get_hiz_surf(&brw->isl_dev, &mt->surf, &temp_hiz_surf);
1839 assert(ok);
1840
1841 const uint32_t alloc_flags = BO_ALLOC_BUSY;
1842 mt->hiz_buf = intel_alloc_aux_buffer(brw, "hiz-miptree",
1843 &temp_hiz_surf, alloc_flags, mt);
1844
1845 if (!mt->hiz_buf) {
1846 free(aux_state);
1847 return false;
1848 }
1849
1850 for (unsigned level = mt->first_level; level <= mt->last_level; ++level)
1851 intel_miptree_level_enable_hiz(brw, mt, level);
1852
1853 mt->aux_state = aux_state;
1854
1855 return true;
1856 }
1857
1858
1859 /**
1860 * Allocate the initial aux surface for a miptree based on mt->aux_usage
1861 *
1862 * Since MCS, HiZ, and CCS_E can compress more than just clear color, we
1863 * create the auxiliary surfaces up-front. CCS_D, on the other hand, can only
1864 * compress clear color so we wait until an actual fast-clear to allocate it.
1865 */
1866 static bool
intel_miptree_alloc_aux(struct brw_context * brw,struct intel_mipmap_tree * mt)1867 intel_miptree_alloc_aux(struct brw_context *brw,
1868 struct intel_mipmap_tree *mt)
1869 {
1870 switch (mt->aux_usage) {
1871 case ISL_AUX_USAGE_NONE:
1872 return true;
1873
1874 case ISL_AUX_USAGE_HIZ:
1875 assert(!_mesa_is_format_color_format(mt->format));
1876 if (!intel_miptree_alloc_hiz(brw, mt))
1877 return false;
1878 return true;
1879
1880 case ISL_AUX_USAGE_MCS:
1881 assert(_mesa_is_format_color_format(mt->format));
1882 assert(mt->surf.samples > 1);
1883 if (!intel_miptree_alloc_mcs(brw, mt, mt->surf.samples))
1884 return false;
1885 return true;
1886
1887 case ISL_AUX_USAGE_CCS_D:
1888 /* Since CCS_D can only compress clear color so we wait until an actual
1889 * fast-clear to allocate it.
1890 */
1891 return true;
1892
1893 case ISL_AUX_USAGE_CCS_E:
1894 assert(_mesa_is_format_color_format(mt->format));
1895 assert(mt->surf.samples == 1);
1896 if (!intel_miptree_alloc_ccs(brw, mt))
1897 return false;
1898 return true;
1899 }
1900
1901 unreachable("Invalid aux usage");
1902 }
1903
1904
1905 /**
1906 * Can the miptree sample using the hiz buffer?
1907 */
1908 bool
intel_miptree_sample_with_hiz(struct brw_context * brw,struct intel_mipmap_tree * mt)1909 intel_miptree_sample_with_hiz(struct brw_context *brw,
1910 struct intel_mipmap_tree *mt)
1911 {
1912 const struct gen_device_info *devinfo = &brw->screen->devinfo;
1913
1914 /* It's unclear how well supported sampling from the hiz buffer is on GEN8,
1915 * so keep things conservative for now and never enable it unless we're SKL+.
1916 */
1917 if (devinfo->gen < 9) {
1918 return false;
1919 }
1920
1921 if (!mt->hiz_buf) {
1922 return false;
1923 }
1924
1925 /* It seems the hardware won't fallback to the depth buffer if some of the
1926 * mipmap levels aren't available in the HiZ buffer. So we need all levels
1927 * of the texture to be HiZ enabled.
1928 */
1929 for (unsigned level = 0; level < mt->surf.levels; ++level) {
1930 if (!intel_miptree_level_has_hiz(mt, level))
1931 return false;
1932 }
1933
1934 /* If compressed multisampling is enabled, then we use it for the auxiliary
1935 * buffer instead.
1936 *
1937 * From the BDW PRM (Volume 2d: Command Reference: Structures
1938 * RENDER_SURFACE_STATE.AuxiliarySurfaceMode):
1939 *
1940 * "If this field is set to AUX_HIZ, Number of Multisamples must be
1941 * MULTISAMPLECOUNT_1, and Surface Type cannot be SURFTYPE_3D.
1942 *
1943 * There is no such blurb for 1D textures, but there is sufficient evidence
1944 * that this is broken on SKL+.
1945 */
1946 return (mt->surf.samples == 1 &&
1947 mt->target != GL_TEXTURE_3D &&
1948 mt->target != GL_TEXTURE_1D /* gen9+ restriction */);
1949 }
1950
1951 /**
1952 * Does the miptree slice have hiz enabled?
1953 */
1954 bool
intel_miptree_level_has_hiz(const struct intel_mipmap_tree * mt,uint32_t level)1955 intel_miptree_level_has_hiz(const struct intel_mipmap_tree *mt, uint32_t level)
1956 {
1957 intel_miptree_check_level_layer(mt, level, 0);
1958 return mt->level[level].has_hiz;
1959 }
1960
1961 static inline uint32_t
miptree_level_range_length(const struct intel_mipmap_tree * mt,uint32_t start_level,uint32_t num_levels)1962 miptree_level_range_length(const struct intel_mipmap_tree *mt,
1963 uint32_t start_level, uint32_t num_levels)
1964 {
1965 assert(start_level >= mt->first_level);
1966 assert(start_level <= mt->last_level);
1967
1968 if (num_levels == INTEL_REMAINING_LAYERS)
1969 num_levels = mt->last_level - start_level + 1;
1970 /* Check for overflow */
1971 assert(start_level + num_levels >= start_level);
1972 assert(start_level + num_levels <= mt->last_level + 1);
1973
1974 return num_levels;
1975 }
1976
1977 static inline uint32_t
miptree_layer_range_length(const struct intel_mipmap_tree * mt,uint32_t level,uint32_t start_layer,uint32_t num_layers)1978 miptree_layer_range_length(const struct intel_mipmap_tree *mt, uint32_t level,
1979 uint32_t start_layer, uint32_t num_layers)
1980 {
1981 assert(level <= mt->last_level);
1982
1983 const uint32_t total_num_layers = brw_get_num_logical_layers(mt, level);
1984 assert(start_layer < total_num_layers);
1985 if (num_layers == INTEL_REMAINING_LAYERS)
1986 num_layers = total_num_layers - start_layer;
1987 /* Check for overflow */
1988 assert(start_layer + num_layers >= start_layer);
1989 assert(start_layer + num_layers <= total_num_layers);
1990
1991 return num_layers;
1992 }
1993
1994 bool
intel_miptree_has_color_unresolved(const struct intel_mipmap_tree * mt,unsigned start_level,unsigned num_levels,unsigned start_layer,unsigned num_layers)1995 intel_miptree_has_color_unresolved(const struct intel_mipmap_tree *mt,
1996 unsigned start_level, unsigned num_levels,
1997 unsigned start_layer, unsigned num_layers)
1998 {
1999 assert(_mesa_is_format_color_format(mt->format));
2000
2001 if (!mt->mcs_buf)
2002 return false;
2003
2004 /* Clamp the level range to fit the miptree */
2005 num_levels = miptree_level_range_length(mt, start_level, num_levels);
2006
2007 for (uint32_t l = 0; l < num_levels; l++) {
2008 const uint32_t level = start_level + l;
2009 const uint32_t level_layers =
2010 miptree_layer_range_length(mt, level, start_layer, num_layers);
2011 for (unsigned a = 0; a < level_layers; a++) {
2012 enum isl_aux_state aux_state =
2013 intel_miptree_get_aux_state(mt, level, start_layer + a);
2014 assert(aux_state != ISL_AUX_STATE_AUX_INVALID);
2015 if (aux_state != ISL_AUX_STATE_PASS_THROUGH)
2016 return true;
2017 }
2018 }
2019
2020 return false;
2021 }
2022
2023 static void
intel_miptree_check_color_resolve(const struct brw_context * brw,const struct intel_mipmap_tree * mt,unsigned level,unsigned layer)2024 intel_miptree_check_color_resolve(const struct brw_context *brw,
2025 const struct intel_mipmap_tree *mt,
2026 unsigned level, unsigned layer)
2027 {
2028 if (!mt->mcs_buf)
2029 return;
2030
2031 /* Fast color clear is supported for mipmapped surfaces only on Gen8+. */
2032 assert(brw->screen->devinfo.gen >= 8 ||
2033 (level == 0 && mt->first_level == 0 && mt->last_level == 0));
2034
2035 /* Compression of arrayed msaa surfaces is supported. */
2036 if (mt->surf.samples > 1)
2037 return;
2038
2039 /* Fast color clear is supported for non-msaa arrays only on Gen8+. */
2040 assert(brw->screen->devinfo.gen >= 8 ||
2041 (layer == 0 &&
2042 mt->surf.logical_level0_px.depth == 1 &&
2043 mt->surf.logical_level0_px.array_len == 1));
2044
2045 (void)level;
2046 (void)layer;
2047 }
2048
2049 static enum blorp_fast_clear_op
get_ccs_d_resolve_op(enum isl_aux_state aux_state,enum isl_aux_usage aux_usage,bool fast_clear_supported)2050 get_ccs_d_resolve_op(enum isl_aux_state aux_state,
2051 enum isl_aux_usage aux_usage,
2052 bool fast_clear_supported)
2053 {
2054 assert(aux_usage == ISL_AUX_USAGE_NONE || aux_usage == ISL_AUX_USAGE_CCS_D);
2055
2056 const bool ccs_supported = aux_usage == ISL_AUX_USAGE_CCS_D;
2057
2058 assert(ccs_supported == fast_clear_supported);
2059
2060 switch (aux_state) {
2061 case ISL_AUX_STATE_CLEAR:
2062 case ISL_AUX_STATE_PARTIAL_CLEAR:
2063 if (!ccs_supported)
2064 return BLORP_FAST_CLEAR_OP_RESOLVE_FULL;
2065 else
2066 return BLORP_FAST_CLEAR_OP_NONE;
2067
2068 case ISL_AUX_STATE_PASS_THROUGH:
2069 return BLORP_FAST_CLEAR_OP_NONE;
2070
2071 case ISL_AUX_STATE_RESOLVED:
2072 case ISL_AUX_STATE_AUX_INVALID:
2073 case ISL_AUX_STATE_COMPRESSED_CLEAR:
2074 case ISL_AUX_STATE_COMPRESSED_NO_CLEAR:
2075 break;
2076 }
2077
2078 unreachable("Invalid aux state for CCS_D");
2079 }
2080
2081 static enum blorp_fast_clear_op
get_ccs_e_resolve_op(enum isl_aux_state aux_state,enum isl_aux_usage aux_usage,bool fast_clear_supported)2082 get_ccs_e_resolve_op(enum isl_aux_state aux_state,
2083 enum isl_aux_usage aux_usage,
2084 bool fast_clear_supported)
2085 {
2086 /* CCS_E surfaces can be accessed as CCS_D if we're careful. */
2087 assert(aux_usage == ISL_AUX_USAGE_NONE ||
2088 aux_usage == ISL_AUX_USAGE_CCS_D ||
2089 aux_usage == ISL_AUX_USAGE_CCS_E);
2090
2091 if (aux_usage == ISL_AUX_USAGE_CCS_D)
2092 assert(fast_clear_supported);
2093
2094 switch (aux_state) {
2095 case ISL_AUX_STATE_CLEAR:
2096 case ISL_AUX_STATE_PARTIAL_CLEAR:
2097 if (fast_clear_supported)
2098 return BLORP_FAST_CLEAR_OP_NONE;
2099 else if (aux_usage == ISL_AUX_USAGE_CCS_E)
2100 return BLORP_FAST_CLEAR_OP_RESOLVE_PARTIAL;
2101 else
2102 return BLORP_FAST_CLEAR_OP_RESOLVE_FULL;
2103
2104 case ISL_AUX_STATE_COMPRESSED_CLEAR:
2105 if (aux_usage != ISL_AUX_USAGE_CCS_E)
2106 return BLORP_FAST_CLEAR_OP_RESOLVE_FULL;
2107 else if (!fast_clear_supported)
2108 return BLORP_FAST_CLEAR_OP_RESOLVE_PARTIAL;
2109 else
2110 return BLORP_FAST_CLEAR_OP_NONE;
2111
2112 case ISL_AUX_STATE_COMPRESSED_NO_CLEAR:
2113 if (aux_usage != ISL_AUX_USAGE_CCS_E)
2114 return BLORP_FAST_CLEAR_OP_RESOLVE_FULL;
2115 else
2116 return BLORP_FAST_CLEAR_OP_NONE;
2117
2118 case ISL_AUX_STATE_PASS_THROUGH:
2119 return BLORP_FAST_CLEAR_OP_NONE;
2120
2121 case ISL_AUX_STATE_RESOLVED:
2122 case ISL_AUX_STATE_AUX_INVALID:
2123 break;
2124 }
2125
2126 unreachable("Invalid aux state for CCS_E");
2127 }
2128
2129 static void
intel_miptree_prepare_ccs_access(struct brw_context * brw,struct intel_mipmap_tree * mt,uint32_t level,uint32_t layer,enum isl_aux_usage aux_usage,bool fast_clear_supported)2130 intel_miptree_prepare_ccs_access(struct brw_context *brw,
2131 struct intel_mipmap_tree *mt,
2132 uint32_t level, uint32_t layer,
2133 enum isl_aux_usage aux_usage,
2134 bool fast_clear_supported)
2135 {
2136 enum isl_aux_state aux_state = intel_miptree_get_aux_state(mt, level, layer);
2137
2138 enum blorp_fast_clear_op resolve_op;
2139 if (mt->aux_usage == ISL_AUX_USAGE_CCS_E) {
2140 resolve_op = get_ccs_e_resolve_op(aux_state, aux_usage,
2141 fast_clear_supported);
2142 } else {
2143 assert(mt->aux_usage == ISL_AUX_USAGE_CCS_D);
2144 resolve_op = get_ccs_d_resolve_op(aux_state, aux_usage,
2145 fast_clear_supported);
2146 }
2147
2148 if (resolve_op != BLORP_FAST_CLEAR_OP_NONE) {
2149 intel_miptree_check_color_resolve(brw, mt, level, layer);
2150 brw_blorp_resolve_color(brw, mt, level, layer, resolve_op);
2151
2152 switch (resolve_op) {
2153 case BLORP_FAST_CLEAR_OP_RESOLVE_FULL:
2154 /* The CCS full resolve operation destroys the CCS and sets it to the
2155 * pass-through state. (You can also think of this as being both a
2156 * resolve and an ambiguate in one operation.)
2157 */
2158 intel_miptree_set_aux_state(brw, mt, level, layer, 1,
2159 ISL_AUX_STATE_PASS_THROUGH);
2160 break;
2161
2162 case BLORP_FAST_CLEAR_OP_RESOLVE_PARTIAL:
2163 intel_miptree_set_aux_state(brw, mt, level, layer, 1,
2164 ISL_AUX_STATE_COMPRESSED_NO_CLEAR);
2165 break;
2166
2167 default:
2168 unreachable("Invalid resolve op");
2169 }
2170 }
2171 }
2172
2173 static void
intel_miptree_finish_ccs_write(struct brw_context * brw,struct intel_mipmap_tree * mt,uint32_t level,uint32_t layer,enum isl_aux_usage aux_usage)2174 intel_miptree_finish_ccs_write(struct brw_context *brw,
2175 struct intel_mipmap_tree *mt,
2176 uint32_t level, uint32_t layer,
2177 enum isl_aux_usage aux_usage)
2178 {
2179 assert(aux_usage == ISL_AUX_USAGE_NONE ||
2180 aux_usage == ISL_AUX_USAGE_CCS_D ||
2181 aux_usage == ISL_AUX_USAGE_CCS_E);
2182
2183 enum isl_aux_state aux_state = intel_miptree_get_aux_state(mt, level, layer);
2184
2185 if (mt->aux_usage == ISL_AUX_USAGE_CCS_E) {
2186 switch (aux_state) {
2187 case ISL_AUX_STATE_CLEAR:
2188 case ISL_AUX_STATE_PARTIAL_CLEAR:
2189 assert(aux_usage == ISL_AUX_USAGE_CCS_E ||
2190 aux_usage == ISL_AUX_USAGE_CCS_D);
2191
2192 if (aux_usage == ISL_AUX_USAGE_CCS_E) {
2193 intel_miptree_set_aux_state(brw, mt, level, layer, 1,
2194 ISL_AUX_STATE_COMPRESSED_CLEAR);
2195 } else if (aux_state != ISL_AUX_STATE_PARTIAL_CLEAR) {
2196 intel_miptree_set_aux_state(brw, mt, level, layer, 1,
2197 ISL_AUX_STATE_PARTIAL_CLEAR);
2198 }
2199 break;
2200
2201 case ISL_AUX_STATE_COMPRESSED_CLEAR:
2202 case ISL_AUX_STATE_COMPRESSED_NO_CLEAR:
2203 assert(aux_usage == ISL_AUX_USAGE_CCS_E);
2204 break; /* Nothing to do */
2205
2206 case ISL_AUX_STATE_PASS_THROUGH:
2207 if (aux_usage == ISL_AUX_USAGE_CCS_E) {
2208 intel_miptree_set_aux_state(brw, mt, level, layer, 1,
2209 ISL_AUX_STATE_COMPRESSED_NO_CLEAR);
2210 } else {
2211 /* Nothing to do */
2212 }
2213 break;
2214
2215 case ISL_AUX_STATE_RESOLVED:
2216 case ISL_AUX_STATE_AUX_INVALID:
2217 unreachable("Invalid aux state for CCS_E");
2218 }
2219 } else {
2220 assert(mt->aux_usage == ISL_AUX_USAGE_CCS_D);
2221 /* CCS_D is a bit simpler */
2222 switch (aux_state) {
2223 case ISL_AUX_STATE_CLEAR:
2224 assert(aux_usage == ISL_AUX_USAGE_CCS_D);
2225 intel_miptree_set_aux_state(brw, mt, level, layer, 1,
2226 ISL_AUX_STATE_PARTIAL_CLEAR);
2227 break;
2228
2229 case ISL_AUX_STATE_PARTIAL_CLEAR:
2230 assert(aux_usage == ISL_AUX_USAGE_CCS_D);
2231 break; /* Nothing to do */
2232
2233 case ISL_AUX_STATE_PASS_THROUGH:
2234 /* Nothing to do */
2235 break;
2236
2237 case ISL_AUX_STATE_COMPRESSED_CLEAR:
2238 case ISL_AUX_STATE_COMPRESSED_NO_CLEAR:
2239 case ISL_AUX_STATE_RESOLVED:
2240 case ISL_AUX_STATE_AUX_INVALID:
2241 unreachable("Invalid aux state for CCS_D");
2242 }
2243 }
2244 }
2245
2246 static void
intel_miptree_prepare_mcs_access(struct brw_context * brw,struct intel_mipmap_tree * mt,uint32_t layer,enum isl_aux_usage aux_usage,bool fast_clear_supported)2247 intel_miptree_prepare_mcs_access(struct brw_context *brw,
2248 struct intel_mipmap_tree *mt,
2249 uint32_t layer,
2250 enum isl_aux_usage aux_usage,
2251 bool fast_clear_supported)
2252 {
2253 assert(aux_usage == ISL_AUX_USAGE_MCS);
2254
2255 switch (intel_miptree_get_aux_state(mt, 0, layer)) {
2256 case ISL_AUX_STATE_CLEAR:
2257 case ISL_AUX_STATE_COMPRESSED_CLEAR:
2258 if (!fast_clear_supported) {
2259 brw_blorp_mcs_partial_resolve(brw, mt, layer, 1);
2260 intel_miptree_set_aux_state(brw, mt, 0, layer, 1,
2261 ISL_AUX_STATE_COMPRESSED_NO_CLEAR);
2262 }
2263 break;
2264
2265 case ISL_AUX_STATE_COMPRESSED_NO_CLEAR:
2266 break; /* Nothing to do */
2267
2268 case ISL_AUX_STATE_RESOLVED:
2269 case ISL_AUX_STATE_PASS_THROUGH:
2270 case ISL_AUX_STATE_AUX_INVALID:
2271 case ISL_AUX_STATE_PARTIAL_CLEAR:
2272 unreachable("Invalid aux state for MCS");
2273 }
2274 }
2275
2276 static void
intel_miptree_finish_mcs_write(struct brw_context * brw,struct intel_mipmap_tree * mt,uint32_t layer,enum isl_aux_usage aux_usage)2277 intel_miptree_finish_mcs_write(struct brw_context *brw,
2278 struct intel_mipmap_tree *mt,
2279 uint32_t layer,
2280 enum isl_aux_usage aux_usage)
2281 {
2282 assert(aux_usage == ISL_AUX_USAGE_MCS);
2283
2284 switch (intel_miptree_get_aux_state(mt, 0, layer)) {
2285 case ISL_AUX_STATE_CLEAR:
2286 intel_miptree_set_aux_state(brw, mt, 0, layer, 1,
2287 ISL_AUX_STATE_COMPRESSED_CLEAR);
2288 break;
2289
2290 case ISL_AUX_STATE_COMPRESSED_CLEAR:
2291 case ISL_AUX_STATE_COMPRESSED_NO_CLEAR:
2292 break; /* Nothing to do */
2293
2294 case ISL_AUX_STATE_RESOLVED:
2295 case ISL_AUX_STATE_PASS_THROUGH:
2296 case ISL_AUX_STATE_AUX_INVALID:
2297 case ISL_AUX_STATE_PARTIAL_CLEAR:
2298 unreachable("Invalid aux state for MCS");
2299 }
2300 }
2301
2302 static void
intel_miptree_prepare_hiz_access(struct brw_context * brw,struct intel_mipmap_tree * mt,uint32_t level,uint32_t layer,enum isl_aux_usage aux_usage,bool fast_clear_supported)2303 intel_miptree_prepare_hiz_access(struct brw_context *brw,
2304 struct intel_mipmap_tree *mt,
2305 uint32_t level, uint32_t layer,
2306 enum isl_aux_usage aux_usage,
2307 bool fast_clear_supported)
2308 {
2309 assert(aux_usage == ISL_AUX_USAGE_NONE || aux_usage == ISL_AUX_USAGE_HIZ);
2310
2311 enum blorp_hiz_op hiz_op = BLORP_HIZ_OP_NONE;
2312 switch (intel_miptree_get_aux_state(mt, level, layer)) {
2313 case ISL_AUX_STATE_CLEAR:
2314 case ISL_AUX_STATE_COMPRESSED_CLEAR:
2315 if (aux_usage != ISL_AUX_USAGE_HIZ || !fast_clear_supported)
2316 hiz_op = BLORP_HIZ_OP_DEPTH_RESOLVE;
2317 break;
2318
2319 case ISL_AUX_STATE_COMPRESSED_NO_CLEAR:
2320 if (aux_usage != ISL_AUX_USAGE_HIZ)
2321 hiz_op = BLORP_HIZ_OP_DEPTH_RESOLVE;
2322 break;
2323
2324 case ISL_AUX_STATE_PASS_THROUGH:
2325 case ISL_AUX_STATE_RESOLVED:
2326 break;
2327
2328 case ISL_AUX_STATE_AUX_INVALID:
2329 if (aux_usage == ISL_AUX_USAGE_HIZ)
2330 hiz_op = BLORP_HIZ_OP_HIZ_RESOLVE;
2331 break;
2332
2333 case ISL_AUX_STATE_PARTIAL_CLEAR:
2334 unreachable("Invalid HiZ state");
2335 }
2336
2337 if (hiz_op != BLORP_HIZ_OP_NONE) {
2338 intel_hiz_exec(brw, mt, level, layer, 1, hiz_op);
2339
2340 switch (hiz_op) {
2341 case BLORP_HIZ_OP_DEPTH_RESOLVE:
2342 intel_miptree_set_aux_state(brw, mt, level, layer, 1,
2343 ISL_AUX_STATE_RESOLVED);
2344 break;
2345
2346 case BLORP_HIZ_OP_HIZ_RESOLVE:
2347 /* The HiZ resolve operation is actually an ambiguate */
2348 intel_miptree_set_aux_state(brw, mt, level, layer, 1,
2349 ISL_AUX_STATE_PASS_THROUGH);
2350 break;
2351
2352 default:
2353 unreachable("Invalid HiZ op");
2354 }
2355 }
2356 }
2357
2358 static void
intel_miptree_finish_hiz_write(struct brw_context * brw,struct intel_mipmap_tree * mt,uint32_t level,uint32_t layer,enum isl_aux_usage aux_usage)2359 intel_miptree_finish_hiz_write(struct brw_context *brw,
2360 struct intel_mipmap_tree *mt,
2361 uint32_t level, uint32_t layer,
2362 enum isl_aux_usage aux_usage)
2363 {
2364 assert(aux_usage == ISL_AUX_USAGE_NONE || aux_usage == ISL_AUX_USAGE_HIZ);
2365
2366 switch (intel_miptree_get_aux_state(mt, level, layer)) {
2367 case ISL_AUX_STATE_CLEAR:
2368 assert(aux_usage == ISL_AUX_USAGE_HIZ);
2369 intel_miptree_set_aux_state(brw, mt, level, layer, 1,
2370 ISL_AUX_STATE_COMPRESSED_CLEAR);
2371 break;
2372
2373 case ISL_AUX_STATE_COMPRESSED_NO_CLEAR:
2374 case ISL_AUX_STATE_COMPRESSED_CLEAR:
2375 assert(aux_usage == ISL_AUX_USAGE_HIZ);
2376 break; /* Nothing to do */
2377
2378 case ISL_AUX_STATE_RESOLVED:
2379 if (aux_usage == ISL_AUX_USAGE_HIZ) {
2380 intel_miptree_set_aux_state(brw, mt, level, layer, 1,
2381 ISL_AUX_STATE_COMPRESSED_NO_CLEAR);
2382 } else {
2383 intel_miptree_set_aux_state(brw, mt, level, layer, 1,
2384 ISL_AUX_STATE_AUX_INVALID);
2385 }
2386 break;
2387
2388 case ISL_AUX_STATE_PASS_THROUGH:
2389 if (aux_usage == ISL_AUX_USAGE_HIZ) {
2390 intel_miptree_set_aux_state(brw, mt, level, layer, 1,
2391 ISL_AUX_STATE_COMPRESSED_NO_CLEAR);
2392 }
2393 break;
2394
2395 case ISL_AUX_STATE_AUX_INVALID:
2396 assert(aux_usage != ISL_AUX_USAGE_HIZ);
2397 break;
2398
2399 case ISL_AUX_STATE_PARTIAL_CLEAR:
2400 unreachable("Invalid HiZ state");
2401 }
2402 }
2403
2404 void
intel_miptree_prepare_access(struct brw_context * brw,struct intel_mipmap_tree * mt,uint32_t start_level,uint32_t num_levels,uint32_t start_layer,uint32_t num_layers,enum isl_aux_usage aux_usage,bool fast_clear_supported)2405 intel_miptree_prepare_access(struct brw_context *brw,
2406 struct intel_mipmap_tree *mt,
2407 uint32_t start_level, uint32_t num_levels,
2408 uint32_t start_layer, uint32_t num_layers,
2409 enum isl_aux_usage aux_usage,
2410 bool fast_clear_supported)
2411 {
2412 num_levels = miptree_level_range_length(mt, start_level, num_levels);
2413
2414 switch (mt->aux_usage) {
2415 case ISL_AUX_USAGE_NONE:
2416 /* Nothing to do */
2417 break;
2418
2419 case ISL_AUX_USAGE_MCS:
2420 assert(mt->mcs_buf);
2421 assert(start_level == 0 && num_levels == 1);
2422 const uint32_t level_layers =
2423 miptree_layer_range_length(mt, 0, start_layer, num_layers);
2424 for (uint32_t a = 0; a < level_layers; a++) {
2425 intel_miptree_prepare_mcs_access(brw, mt, start_layer + a,
2426 aux_usage, fast_clear_supported);
2427 }
2428 break;
2429
2430 case ISL_AUX_USAGE_CCS_D:
2431 case ISL_AUX_USAGE_CCS_E:
2432 if (!mt->mcs_buf)
2433 return;
2434
2435 for (uint32_t l = 0; l < num_levels; l++) {
2436 const uint32_t level = start_level + l;
2437 const uint32_t level_layers =
2438 miptree_layer_range_length(mt, level, start_layer, num_layers);
2439 for (uint32_t a = 0; a < level_layers; a++) {
2440 intel_miptree_prepare_ccs_access(brw, mt, level,
2441 start_layer + a,
2442 aux_usage, fast_clear_supported);
2443 }
2444 }
2445 break;
2446
2447 case ISL_AUX_USAGE_HIZ:
2448 assert(mt->hiz_buf);
2449 for (uint32_t l = 0; l < num_levels; l++) {
2450 const uint32_t level = start_level + l;
2451 if (!intel_miptree_level_has_hiz(mt, level))
2452 continue;
2453
2454 const uint32_t level_layers =
2455 miptree_layer_range_length(mt, level, start_layer, num_layers);
2456 for (uint32_t a = 0; a < level_layers; a++) {
2457 intel_miptree_prepare_hiz_access(brw, mt, level, start_layer + a,
2458 aux_usage, fast_clear_supported);
2459 }
2460 }
2461 break;
2462
2463 default:
2464 unreachable("Invalid aux usage");
2465 }
2466 }
2467
2468 void
intel_miptree_finish_write(struct brw_context * brw,struct intel_mipmap_tree * mt,uint32_t level,uint32_t start_layer,uint32_t num_layers,enum isl_aux_usage aux_usage)2469 intel_miptree_finish_write(struct brw_context *brw,
2470 struct intel_mipmap_tree *mt, uint32_t level,
2471 uint32_t start_layer, uint32_t num_layers,
2472 enum isl_aux_usage aux_usage)
2473 {
2474 num_layers = miptree_layer_range_length(mt, level, start_layer, num_layers);
2475
2476 switch (mt->aux_usage) {
2477 case ISL_AUX_USAGE_NONE:
2478 /* Nothing to do */
2479 break;
2480
2481 case ISL_AUX_USAGE_MCS:
2482 assert(mt->mcs_buf);
2483 for (uint32_t a = 0; a < num_layers; a++) {
2484 intel_miptree_finish_mcs_write(brw, mt, start_layer + a,
2485 aux_usage);
2486 }
2487 break;
2488
2489 case ISL_AUX_USAGE_CCS_D:
2490 case ISL_AUX_USAGE_CCS_E:
2491 if (!mt->mcs_buf)
2492 return;
2493
2494 for (uint32_t a = 0; a < num_layers; a++) {
2495 intel_miptree_finish_ccs_write(brw, mt, level, start_layer + a,
2496 aux_usage);
2497 }
2498 break;
2499
2500 case ISL_AUX_USAGE_HIZ:
2501 if (!intel_miptree_level_has_hiz(mt, level))
2502 return;
2503
2504 for (uint32_t a = 0; a < num_layers; a++) {
2505 intel_miptree_finish_hiz_write(brw, mt, level, start_layer + a,
2506 aux_usage);
2507 }
2508 break;
2509
2510 default:
2511 unreachable("Invavlid aux usage");
2512 }
2513 }
2514
2515 enum isl_aux_state
intel_miptree_get_aux_state(const struct intel_mipmap_tree * mt,uint32_t level,uint32_t layer)2516 intel_miptree_get_aux_state(const struct intel_mipmap_tree *mt,
2517 uint32_t level, uint32_t layer)
2518 {
2519 intel_miptree_check_level_layer(mt, level, layer);
2520
2521 if (_mesa_is_format_color_format(mt->format)) {
2522 assert(mt->mcs_buf != NULL);
2523 assert(mt->surf.samples == 1 ||
2524 mt->surf.msaa_layout == ISL_MSAA_LAYOUT_ARRAY);
2525 } else if (mt->format == MESA_FORMAT_S_UINT8) {
2526 unreachable("Cannot get aux state for stencil");
2527 } else {
2528 assert(intel_miptree_level_has_hiz(mt, level));
2529 }
2530
2531 return mt->aux_state[level][layer];
2532 }
2533
2534 void
intel_miptree_set_aux_state(struct brw_context * brw,struct intel_mipmap_tree * mt,uint32_t level,uint32_t start_layer,uint32_t num_layers,enum isl_aux_state aux_state)2535 intel_miptree_set_aux_state(struct brw_context *brw,
2536 struct intel_mipmap_tree *mt, uint32_t level,
2537 uint32_t start_layer, uint32_t num_layers,
2538 enum isl_aux_state aux_state)
2539 {
2540 num_layers = miptree_layer_range_length(mt, level, start_layer, num_layers);
2541
2542 if (_mesa_is_format_color_format(mt->format)) {
2543 assert(mt->mcs_buf != NULL);
2544 assert(mt->surf.samples == 1 ||
2545 mt->surf.msaa_layout == ISL_MSAA_LAYOUT_ARRAY);
2546 } else if (mt->format == MESA_FORMAT_S_UINT8) {
2547 unreachable("Cannot get aux state for stencil");
2548 } else {
2549 assert(intel_miptree_level_has_hiz(mt, level));
2550 }
2551
2552 for (unsigned a = 0; a < num_layers; a++) {
2553 if (mt->aux_state[level][start_layer + a] != aux_state) {
2554 mt->aux_state[level][start_layer + a] = aux_state;
2555 brw->ctx.NewDriverState |= BRW_NEW_AUX_STATE;
2556 }
2557 }
2558 }
2559
2560 /* On Gen9 color buffers may be compressed by the hardware (lossless
2561 * compression). There are, however, format restrictions and care needs to be
2562 * taken that the sampler engine is capable for re-interpreting a buffer with
2563 * format different the buffer was originally written with.
2564 *
2565 * For example, SRGB formats are not compressible and the sampler engine isn't
2566 * capable of treating RGBA_UNORM as SRGB_ALPHA. In such a case the underlying
2567 * color buffer needs to be resolved so that the sampling surface can be
2568 * sampled as non-compressed (i.e., without the auxiliary MCS buffer being
2569 * set).
2570 */
2571 static bool
can_texture_with_ccs(struct brw_context * brw,struct intel_mipmap_tree * mt,enum isl_format view_format)2572 can_texture_with_ccs(struct brw_context *brw,
2573 struct intel_mipmap_tree *mt,
2574 enum isl_format view_format)
2575 {
2576 if (mt->aux_usage != ISL_AUX_USAGE_CCS_E)
2577 return false;
2578
2579 if (!format_ccs_e_compat_with_miptree(&brw->screen->devinfo,
2580 mt, view_format)) {
2581 perf_debug("Incompatible sampling format (%s) for rbc (%s)\n",
2582 isl_format_get_layout(view_format)->name,
2583 _mesa_get_format_name(mt->format));
2584 return false;
2585 }
2586
2587 return true;
2588 }
2589
2590 enum isl_aux_usage
intel_miptree_texture_aux_usage(struct brw_context * brw,struct intel_mipmap_tree * mt,enum isl_format view_format)2591 intel_miptree_texture_aux_usage(struct brw_context *brw,
2592 struct intel_mipmap_tree *mt,
2593 enum isl_format view_format)
2594 {
2595 switch (mt->aux_usage) {
2596 case ISL_AUX_USAGE_HIZ:
2597 if (intel_miptree_sample_with_hiz(brw, mt))
2598 return ISL_AUX_USAGE_HIZ;
2599 break;
2600
2601 case ISL_AUX_USAGE_MCS:
2602 return ISL_AUX_USAGE_MCS;
2603
2604 case ISL_AUX_USAGE_CCS_D:
2605 case ISL_AUX_USAGE_CCS_E:
2606 if (!mt->mcs_buf) {
2607 assert(mt->aux_usage == ISL_AUX_USAGE_CCS_D);
2608 return ISL_AUX_USAGE_NONE;
2609 }
2610
2611 /* If we don't have any unresolved color, report an aux usage of
2612 * ISL_AUX_USAGE_NONE. This way, texturing won't even look at the
2613 * aux surface and we can save some bandwidth.
2614 */
2615 if (!intel_miptree_has_color_unresolved(mt, 0, INTEL_REMAINING_LEVELS,
2616 0, INTEL_REMAINING_LAYERS))
2617 return ISL_AUX_USAGE_NONE;
2618
2619 if (can_texture_with_ccs(brw, mt, view_format))
2620 return ISL_AUX_USAGE_CCS_E;
2621 break;
2622
2623 default:
2624 break;
2625 }
2626
2627 return ISL_AUX_USAGE_NONE;
2628 }
2629
2630 static bool
isl_formats_are_fast_clear_compatible(enum isl_format a,enum isl_format b)2631 isl_formats_are_fast_clear_compatible(enum isl_format a, enum isl_format b)
2632 {
2633 /* On gen8 and earlier, the hardware was only capable of handling 0/1 clear
2634 * values so sRGB curve application was a no-op for all fast-clearable
2635 * formats.
2636 *
2637 * On gen9+, the hardware supports arbitrary clear values. For sRGB clear
2638 * values, the hardware interprets the floats, not as what would be
2639 * returned from the sampler (or written by the shader), but as being
2640 * between format conversion and sRGB curve application. This means that
2641 * we can switch between sRGB and UNORM without having to whack the clear
2642 * color.
2643 */
2644 return isl_format_srgb_to_linear(a) == isl_format_srgb_to_linear(b);
2645 }
2646
2647 void
intel_miptree_prepare_texture(struct brw_context * brw,struct intel_mipmap_tree * mt,enum isl_format view_format,uint32_t start_level,uint32_t num_levels,uint32_t start_layer,uint32_t num_layers,bool disable_aux)2648 intel_miptree_prepare_texture(struct brw_context *brw,
2649 struct intel_mipmap_tree *mt,
2650 enum isl_format view_format,
2651 uint32_t start_level, uint32_t num_levels,
2652 uint32_t start_layer, uint32_t num_layers,
2653 bool disable_aux)
2654 {
2655 enum isl_aux_usage aux_usage = disable_aux ? ISL_AUX_USAGE_NONE :
2656 intel_miptree_texture_aux_usage(brw, mt, view_format);
2657 bool clear_supported = aux_usage != ISL_AUX_USAGE_NONE;
2658
2659 /* Clear color is specified as ints or floats and the conversion is done by
2660 * the sampler. If we have a texture view, we would have to perform the
2661 * clear color conversion manually. Just disable clear color.
2662 */
2663 if (!isl_formats_are_fast_clear_compatible(mt->surf.format, view_format))
2664 clear_supported = false;
2665
2666 intel_miptree_prepare_access(brw, mt, start_level, num_levels,
2667 start_layer, num_layers,
2668 aux_usage, clear_supported);
2669 }
2670
2671 void
intel_miptree_prepare_image(struct brw_context * brw,struct intel_mipmap_tree * mt)2672 intel_miptree_prepare_image(struct brw_context *brw,
2673 struct intel_mipmap_tree *mt)
2674 {
2675 /* The data port doesn't understand any compression */
2676 intel_miptree_prepare_access(brw, mt, 0, INTEL_REMAINING_LEVELS,
2677 0, INTEL_REMAINING_LAYERS,
2678 ISL_AUX_USAGE_NONE, false);
2679 }
2680
2681 enum isl_aux_usage
intel_miptree_render_aux_usage(struct brw_context * brw,struct intel_mipmap_tree * mt,enum isl_format render_format,bool blend_enabled,bool draw_aux_disabled)2682 intel_miptree_render_aux_usage(struct brw_context *brw,
2683 struct intel_mipmap_tree *mt,
2684 enum isl_format render_format,
2685 bool blend_enabled,
2686 bool draw_aux_disabled)
2687 {
2688 struct gen_device_info *devinfo = &brw->screen->devinfo;
2689
2690 if (draw_aux_disabled)
2691 return ISL_AUX_USAGE_NONE;
2692
2693 switch (mt->aux_usage) {
2694 case ISL_AUX_USAGE_MCS:
2695 assert(mt->mcs_buf);
2696 return ISL_AUX_USAGE_MCS;
2697
2698 case ISL_AUX_USAGE_CCS_D:
2699 case ISL_AUX_USAGE_CCS_E:
2700 if (!mt->mcs_buf) {
2701 assert(mt->aux_usage == ISL_AUX_USAGE_CCS_D);
2702 return ISL_AUX_USAGE_NONE;
2703 }
2704
2705 /* gen9 hardware technically supports non-0/1 clear colors with sRGB
2706 * formats. However, there are issues with blending where it doesn't
2707 * properly apply the sRGB curve to the clear color when blending.
2708 */
2709 if (devinfo->gen == 9 && blend_enabled &&
2710 isl_format_is_srgb(render_format) &&
2711 !isl_color_value_is_zero_one(mt->fast_clear_color, render_format))
2712 return ISL_AUX_USAGE_NONE;
2713
2714 if (mt->aux_usage == ISL_AUX_USAGE_CCS_E &&
2715 format_ccs_e_compat_with_miptree(&brw->screen->devinfo,
2716 mt, render_format))
2717 return ISL_AUX_USAGE_CCS_E;
2718
2719 /* Otherwise, we have to fall back to CCS_D */
2720 return ISL_AUX_USAGE_CCS_D;
2721
2722 default:
2723 return ISL_AUX_USAGE_NONE;
2724 }
2725 }
2726
2727 void
intel_miptree_prepare_render(struct brw_context * brw,struct intel_mipmap_tree * mt,uint32_t level,uint32_t start_layer,uint32_t layer_count,enum isl_aux_usage aux_usage)2728 intel_miptree_prepare_render(struct brw_context *brw,
2729 struct intel_mipmap_tree *mt, uint32_t level,
2730 uint32_t start_layer, uint32_t layer_count,
2731 enum isl_aux_usage aux_usage)
2732 {
2733 intel_miptree_prepare_access(brw, mt, level, 1, start_layer, layer_count,
2734 aux_usage, aux_usage != ISL_AUX_USAGE_NONE);
2735 }
2736
2737 void
intel_miptree_finish_render(struct brw_context * brw,struct intel_mipmap_tree * mt,uint32_t level,uint32_t start_layer,uint32_t layer_count,enum isl_aux_usage aux_usage)2738 intel_miptree_finish_render(struct brw_context *brw,
2739 struct intel_mipmap_tree *mt, uint32_t level,
2740 uint32_t start_layer, uint32_t layer_count,
2741 enum isl_aux_usage aux_usage)
2742 {
2743 assert(_mesa_is_format_color_format(mt->format));
2744
2745 intel_miptree_finish_write(brw, mt, level, start_layer, layer_count,
2746 aux_usage);
2747 }
2748
2749 void
intel_miptree_prepare_depth(struct brw_context * brw,struct intel_mipmap_tree * mt,uint32_t level,uint32_t start_layer,uint32_t layer_count)2750 intel_miptree_prepare_depth(struct brw_context *brw,
2751 struct intel_mipmap_tree *mt, uint32_t level,
2752 uint32_t start_layer, uint32_t layer_count)
2753 {
2754 intel_miptree_prepare_access(brw, mt, level, 1, start_layer, layer_count,
2755 mt->aux_usage, mt->hiz_buf != NULL);
2756 }
2757
2758 void
intel_miptree_finish_depth(struct brw_context * brw,struct intel_mipmap_tree * mt,uint32_t level,uint32_t start_layer,uint32_t layer_count,bool depth_written)2759 intel_miptree_finish_depth(struct brw_context *brw,
2760 struct intel_mipmap_tree *mt, uint32_t level,
2761 uint32_t start_layer, uint32_t layer_count,
2762 bool depth_written)
2763 {
2764 if (depth_written) {
2765 intel_miptree_finish_write(brw, mt, level, start_layer, layer_count,
2766 mt->hiz_buf != NULL);
2767 }
2768 }
2769
2770 void
intel_miptree_prepare_external(struct brw_context * brw,struct intel_mipmap_tree * mt)2771 intel_miptree_prepare_external(struct brw_context *brw,
2772 struct intel_mipmap_tree *mt)
2773 {
2774 enum isl_aux_usage aux_usage = ISL_AUX_USAGE_NONE;
2775 bool supports_fast_clear = false;
2776
2777 const struct isl_drm_modifier_info *mod_info =
2778 isl_drm_modifier_get_info(mt->drm_modifier);
2779
2780 if (mod_info && mod_info->aux_usage != ISL_AUX_USAGE_NONE) {
2781 /* CCS_E is the only supported aux for external images and it's only
2782 * supported on very simple images.
2783 */
2784 assert(mod_info->aux_usage == ISL_AUX_USAGE_CCS_E);
2785 assert(_mesa_is_format_color_format(mt->format));
2786 assert(mt->first_level == 0 && mt->last_level == 0);
2787 assert(mt->surf.logical_level0_px.depth == 1);
2788 assert(mt->surf.logical_level0_px.array_len == 1);
2789 assert(mt->surf.samples == 1);
2790 assert(mt->mcs_buf != NULL);
2791
2792 aux_usage = mod_info->aux_usage;
2793 supports_fast_clear = mod_info->supports_clear_color;
2794 }
2795
2796 intel_miptree_prepare_access(brw, mt, 0, INTEL_REMAINING_LEVELS,
2797 0, INTEL_REMAINING_LAYERS,
2798 aux_usage, supports_fast_clear);
2799 }
2800
2801 /**
2802 * Make it possible to share the BO backing the given miptree with another
2803 * process or another miptree.
2804 *
2805 * Fast color clears are unsafe with shared buffers, so we need to resolve and
2806 * then discard the MCS buffer, if present. We also set the no_ccs flag to
2807 * ensure that no MCS buffer gets allocated in the future.
2808 *
2809 * HiZ is similarly unsafe with shared buffers.
2810 */
2811 void
intel_miptree_make_shareable(struct brw_context * brw,struct intel_mipmap_tree * mt)2812 intel_miptree_make_shareable(struct brw_context *brw,
2813 struct intel_mipmap_tree *mt)
2814 {
2815 /* MCS buffers are also used for multisample buffers, but we can't resolve
2816 * away a multisample MCS buffer because it's an integral part of how the
2817 * pixel data is stored. Fortunately this code path should never be
2818 * reached for multisample buffers.
2819 */
2820 assert(mt->surf.msaa_layout == ISL_MSAA_LAYOUT_NONE ||
2821 mt->surf.samples == 1);
2822
2823 intel_miptree_prepare_access(brw, mt, 0, INTEL_REMAINING_LEVELS,
2824 0, INTEL_REMAINING_LAYERS,
2825 ISL_AUX_USAGE_NONE, false);
2826
2827 if (mt->mcs_buf) {
2828 brw_bo_unreference(mt->mcs_buf->bo);
2829 free(mt->mcs_buf);
2830 mt->mcs_buf = NULL;
2831
2832 /* Any pending MCS/CCS operations are no longer needed. Trying to
2833 * execute any will likely crash due to the missing aux buffer. So let's
2834 * delete all pending ops.
2835 */
2836 free(mt->aux_state);
2837 mt->aux_state = NULL;
2838 brw->ctx.NewDriverState |= BRW_NEW_AUX_STATE;
2839 }
2840
2841 if (mt->hiz_buf) {
2842 intel_miptree_aux_buffer_free(mt->hiz_buf);
2843 mt->hiz_buf = NULL;
2844
2845 for (uint32_t l = mt->first_level; l <= mt->last_level; ++l) {
2846 mt->level[l].has_hiz = false;
2847 }
2848
2849 /* Any pending HiZ operations are no longer needed. Trying to execute
2850 * any will likely crash due to the missing aux buffer. So let's delete
2851 * all pending ops.
2852 */
2853 free(mt->aux_state);
2854 mt->aux_state = NULL;
2855 brw->ctx.NewDriverState |= BRW_NEW_AUX_STATE;
2856 }
2857
2858 mt->aux_usage = ISL_AUX_USAGE_NONE;
2859 mt->supports_fast_clear = false;
2860 }
2861
2862
2863 /**
2864 * \brief Get pointer offset into stencil buffer.
2865 *
2866 * The stencil buffer is W tiled. Since the GTT is incapable of W fencing, we
2867 * must decode the tile's layout in software.
2868 *
2869 * See
2870 * - PRM, 2011 Sandy Bridge, Volume 1, Part 2, Section 4.5.2.1 W-Major Tile
2871 * Format.
2872 * - PRM, 2011 Sandy Bridge, Volume 1, Part 2, Section 4.5.3 Tiling Algorithm
2873 *
2874 * Even though the returned offset is always positive, the return type is
2875 * signed due to
2876 * commit e8b1c6d6f55f5be3bef25084fdd8b6127517e137
2877 * mesa: Fix return type of _mesa_get_format_bytes() (#37351)
2878 */
2879 static intptr_t
intel_offset_S8(uint32_t stride,uint32_t x,uint32_t y,bool swizzled)2880 intel_offset_S8(uint32_t stride, uint32_t x, uint32_t y, bool swizzled)
2881 {
2882 uint32_t tile_size = 4096;
2883 uint32_t tile_width = 64;
2884 uint32_t tile_height = 64;
2885 uint32_t row_size = 64 * stride / 2; /* Two rows are interleaved. */
2886
2887 uint32_t tile_x = x / tile_width;
2888 uint32_t tile_y = y / tile_height;
2889
2890 /* The byte's address relative to the tile's base addres. */
2891 uint32_t byte_x = x % tile_width;
2892 uint32_t byte_y = y % tile_height;
2893
2894 uintptr_t u = tile_y * row_size
2895 + tile_x * tile_size
2896 + 512 * (byte_x / 8)
2897 + 64 * (byte_y / 8)
2898 + 32 * ((byte_y / 4) % 2)
2899 + 16 * ((byte_x / 4) % 2)
2900 + 8 * ((byte_y / 2) % 2)
2901 + 4 * ((byte_x / 2) % 2)
2902 + 2 * (byte_y % 2)
2903 + 1 * (byte_x % 2);
2904
2905 if (swizzled) {
2906 /* adjust for bit6 swizzling */
2907 if (((byte_x / 8) % 2) == 1) {
2908 if (((byte_y / 8) % 2) == 0) {
2909 u += 64;
2910 } else {
2911 u -= 64;
2912 }
2913 }
2914 }
2915
2916 return u;
2917 }
2918
2919 void
intel_miptree_updownsample(struct brw_context * brw,struct intel_mipmap_tree * src,struct intel_mipmap_tree * dst)2920 intel_miptree_updownsample(struct brw_context *brw,
2921 struct intel_mipmap_tree *src,
2922 struct intel_mipmap_tree *dst)
2923 {
2924 unsigned src_w = src->surf.logical_level0_px.width;
2925 unsigned src_h = src->surf.logical_level0_px.height;
2926 unsigned dst_w = dst->surf.logical_level0_px.width;
2927 unsigned dst_h = dst->surf.logical_level0_px.height;
2928
2929 brw_blorp_blit_miptrees(brw,
2930 src, 0 /* level */, 0 /* layer */,
2931 src->format, SWIZZLE_XYZW,
2932 dst, 0 /* level */, 0 /* layer */, dst->format,
2933 0, 0, src_w, src_h,
2934 0, 0, dst_w, dst_h,
2935 GL_NEAREST, false, false /*mirror x, y*/,
2936 false, false);
2937
2938 if (src->stencil_mt) {
2939 src_w = src->stencil_mt->surf.logical_level0_px.width;
2940 src_h = src->stencil_mt->surf.logical_level0_px.height;
2941 dst_w = dst->stencil_mt->surf.logical_level0_px.width;
2942 dst_h = dst->stencil_mt->surf.logical_level0_px.height;
2943
2944 brw_blorp_blit_miptrees(brw,
2945 src->stencil_mt, 0 /* level */, 0 /* layer */,
2946 src->stencil_mt->format, SWIZZLE_XYZW,
2947 dst->stencil_mt, 0 /* level */, 0 /* layer */,
2948 dst->stencil_mt->format,
2949 0, 0, src_w, src_h,
2950 0, 0, dst_w, dst_h,
2951 GL_NEAREST, false, false /*mirror x, y*/,
2952 false, false /* decode/encode srgb */);
2953 }
2954 }
2955
2956 void
intel_update_r8stencil(struct brw_context * brw,struct intel_mipmap_tree * mt)2957 intel_update_r8stencil(struct brw_context *brw,
2958 struct intel_mipmap_tree *mt)
2959 {
2960 const struct gen_device_info *devinfo = &brw->screen->devinfo;
2961
2962 assert(devinfo->gen >= 7);
2963 struct intel_mipmap_tree *src =
2964 mt->format == MESA_FORMAT_S_UINT8 ? mt : mt->stencil_mt;
2965 if (!src || devinfo->gen >= 8 || !src->r8stencil_needs_update)
2966 return;
2967
2968 assert(src->surf.size > 0);
2969
2970 if (!mt->r8stencil_mt) {
2971 assert(devinfo->gen > 6); /* Handle MIPTREE_LAYOUT_GEN6_HIZ_STENCIL */
2972 mt->r8stencil_mt = make_surface(
2973 brw,
2974 src->target,
2975 MESA_FORMAT_R_UINT8,
2976 src->first_level, src->last_level,
2977 src->surf.logical_level0_px.width,
2978 src->surf.logical_level0_px.height,
2979 src->surf.dim == ISL_SURF_DIM_3D ?
2980 src->surf.logical_level0_px.depth :
2981 src->surf.logical_level0_px.array_len,
2982 src->surf.samples,
2983 ISL_TILING_Y0_BIT,
2984 ISL_SURF_USAGE_TEXTURE_BIT,
2985 BO_ALLOC_BUSY, 0, NULL);
2986 assert(mt->r8stencil_mt);
2987 }
2988
2989 struct intel_mipmap_tree *dst = mt->r8stencil_mt;
2990
2991 for (int level = src->first_level; level <= src->last_level; level++) {
2992 const unsigned depth = src->surf.dim == ISL_SURF_DIM_3D ?
2993 minify(src->surf.phys_level0_sa.depth, level) :
2994 src->surf.phys_level0_sa.array_len;
2995
2996 for (unsigned layer = 0; layer < depth; layer++) {
2997 brw_blorp_copy_miptrees(brw,
2998 src, level, layer,
2999 dst, level, layer,
3000 0, 0, 0, 0,
3001 minify(src->surf.logical_level0_px.width,
3002 level),
3003 minify(src->surf.logical_level0_px.height,
3004 level));
3005 }
3006 }
3007
3008 brw_cache_flush_for_read(brw, dst->bo);
3009 src->r8stencil_needs_update = false;
3010 }
3011
3012 static void *
intel_miptree_map_raw(struct brw_context * brw,struct intel_mipmap_tree * mt,GLbitfield mode)3013 intel_miptree_map_raw(struct brw_context *brw,
3014 struct intel_mipmap_tree *mt,
3015 GLbitfield mode)
3016 {
3017 struct brw_bo *bo = mt->bo;
3018
3019 if (brw_batch_references(&brw->batch, bo))
3020 intel_batchbuffer_flush(brw);
3021
3022 return brw_bo_map(brw, bo, mode);
3023 }
3024
3025 static void
intel_miptree_unmap_raw(struct intel_mipmap_tree * mt)3026 intel_miptree_unmap_raw(struct intel_mipmap_tree *mt)
3027 {
3028 brw_bo_unmap(mt->bo);
3029 }
3030
3031 static void
intel_miptree_map_gtt(struct brw_context * brw,struct intel_mipmap_tree * mt,struct intel_miptree_map * map,unsigned int level,unsigned int slice)3032 intel_miptree_map_gtt(struct brw_context *brw,
3033 struct intel_mipmap_tree *mt,
3034 struct intel_miptree_map *map,
3035 unsigned int level, unsigned int slice)
3036 {
3037 unsigned int bw, bh;
3038 void *base;
3039 unsigned int image_x, image_y;
3040 intptr_t x = map->x;
3041 intptr_t y = map->y;
3042
3043 /* For compressed formats, the stride is the number of bytes per
3044 * row of blocks. intel_miptree_get_image_offset() already does
3045 * the divide.
3046 */
3047 _mesa_get_format_block_size(mt->format, &bw, &bh);
3048 assert(y % bh == 0);
3049 assert(x % bw == 0);
3050 y /= bh;
3051 x /= bw;
3052
3053 base = intel_miptree_map_raw(brw, mt, map->mode);
3054
3055 if (base == NULL)
3056 map->ptr = NULL;
3057 else {
3058 base += mt->offset;
3059
3060 /* Note that in the case of cube maps, the caller must have passed the
3061 * slice number referencing the face.
3062 */
3063 intel_miptree_get_image_offset(mt, level, slice, &image_x, &image_y);
3064 x += image_x;
3065 y += image_y;
3066
3067 map->stride = mt->surf.row_pitch;
3068 map->ptr = base + y * map->stride + x * mt->cpp;
3069 }
3070
3071 DBG("%s: %d,%d %dx%d from mt %p (%s) "
3072 "%"PRIiPTR",%"PRIiPTR" = %p/%d\n", __func__,
3073 map->x, map->y, map->w, map->h,
3074 mt, _mesa_get_format_name(mt->format),
3075 x, y, map->ptr, map->stride);
3076 }
3077
3078 static void
intel_miptree_unmap_gtt(struct intel_mipmap_tree * mt)3079 intel_miptree_unmap_gtt(struct intel_mipmap_tree *mt)
3080 {
3081 intel_miptree_unmap_raw(mt);
3082 }
3083
3084 static void
intel_miptree_map_blit(struct brw_context * brw,struct intel_mipmap_tree * mt,struct intel_miptree_map * map,unsigned int level,unsigned int slice)3085 intel_miptree_map_blit(struct brw_context *brw,
3086 struct intel_mipmap_tree *mt,
3087 struct intel_miptree_map *map,
3088 unsigned int level, unsigned int slice)
3089 {
3090 map->linear_mt = intel_miptree_create(brw, GL_TEXTURE_2D, mt->format,
3091 /* first_level */ 0,
3092 /* last_level */ 0,
3093 map->w, map->h, 1,
3094 /* samples */ 1,
3095 MIPTREE_CREATE_LINEAR);
3096
3097 if (!map->linear_mt) {
3098 fprintf(stderr, "Failed to allocate blit temporary\n");
3099 goto fail;
3100 }
3101 map->stride = map->linear_mt->surf.row_pitch;
3102
3103 /* One of either READ_BIT or WRITE_BIT or both is set. READ_BIT implies no
3104 * INVALIDATE_RANGE_BIT. WRITE_BIT needs the original values read in unless
3105 * invalidate is set, since we'll be writing the whole rectangle from our
3106 * temporary buffer back out.
3107 */
3108 if (!(map->mode & GL_MAP_INVALIDATE_RANGE_BIT)) {
3109 if (!intel_miptree_copy(brw,
3110 mt, level, slice, map->x, map->y,
3111 map->linear_mt, 0, 0, 0, 0,
3112 map->w, map->h)) {
3113 fprintf(stderr, "Failed to blit\n");
3114 goto fail;
3115 }
3116 }
3117
3118 map->ptr = intel_miptree_map_raw(brw, map->linear_mt, map->mode);
3119
3120 DBG("%s: %d,%d %dx%d from mt %p (%s) %d,%d = %p/%d\n", __func__,
3121 map->x, map->y, map->w, map->h,
3122 mt, _mesa_get_format_name(mt->format),
3123 level, slice, map->ptr, map->stride);
3124
3125 return;
3126
3127 fail:
3128 intel_miptree_release(&map->linear_mt);
3129 map->ptr = NULL;
3130 map->stride = 0;
3131 }
3132
3133 static void
intel_miptree_unmap_blit(struct brw_context * brw,struct intel_mipmap_tree * mt,struct intel_miptree_map * map,unsigned int level,unsigned int slice)3134 intel_miptree_unmap_blit(struct brw_context *brw,
3135 struct intel_mipmap_tree *mt,
3136 struct intel_miptree_map *map,
3137 unsigned int level,
3138 unsigned int slice)
3139 {
3140 struct gl_context *ctx = &brw->ctx;
3141
3142 intel_miptree_unmap_raw(map->linear_mt);
3143
3144 if (map->mode & GL_MAP_WRITE_BIT) {
3145 bool ok = intel_miptree_copy(brw,
3146 map->linear_mt, 0, 0, 0, 0,
3147 mt, level, slice, map->x, map->y,
3148 map->w, map->h);
3149 WARN_ONCE(!ok, "Failed to blit from linear temporary mapping");
3150 }
3151
3152 intel_miptree_release(&map->linear_mt);
3153 }
3154
3155 /**
3156 * "Map" a buffer by copying it to an untiled temporary using MOVNTDQA.
3157 */
3158 #if defined(USE_SSE41)
3159 static void
intel_miptree_map_movntdqa(struct brw_context * brw,struct intel_mipmap_tree * mt,struct intel_miptree_map * map,unsigned int level,unsigned int slice)3160 intel_miptree_map_movntdqa(struct brw_context *brw,
3161 struct intel_mipmap_tree *mt,
3162 struct intel_miptree_map *map,
3163 unsigned int level, unsigned int slice)
3164 {
3165 assert(map->mode & GL_MAP_READ_BIT);
3166 assert(!(map->mode & GL_MAP_WRITE_BIT));
3167
3168 DBG("%s: %d,%d %dx%d from mt %p (%s) %d,%d = %p/%d\n", __func__,
3169 map->x, map->y, map->w, map->h,
3170 mt, _mesa_get_format_name(mt->format),
3171 level, slice, map->ptr, map->stride);
3172
3173 /* Map the original image */
3174 uint32_t image_x;
3175 uint32_t image_y;
3176 intel_miptree_get_image_offset(mt, level, slice, &image_x, &image_y);
3177 image_x += map->x;
3178 image_y += map->y;
3179
3180 void *src = intel_miptree_map_raw(brw, mt, map->mode);
3181 if (!src)
3182 return;
3183
3184 src += mt->offset;
3185
3186 src += image_y * mt->surf.row_pitch;
3187 src += image_x * mt->cpp;
3188
3189 /* Due to the pixel offsets for the particular image being mapped, our
3190 * src pointer may not be 16-byte aligned. However, if the pitch is
3191 * divisible by 16, then the amount by which it's misaligned will remain
3192 * consistent from row to row.
3193 */
3194 assert((mt->surf.row_pitch % 16) == 0);
3195 const int misalignment = ((uintptr_t) src) & 15;
3196
3197 /* Create an untiled temporary buffer for the mapping. */
3198 const unsigned width_bytes = _mesa_format_row_stride(mt->format, map->w);
3199
3200 map->stride = ALIGN(misalignment + width_bytes, 16);
3201
3202 map->buffer = _mesa_align_malloc(map->stride * map->h, 16);
3203 /* Offset the destination so it has the same misalignment as src. */
3204 map->ptr = map->buffer + misalignment;
3205
3206 assert((((uintptr_t) map->ptr) & 15) == misalignment);
3207
3208 for (uint32_t y = 0; y < map->h; y++) {
3209 void *dst_ptr = map->ptr + y * map->stride;
3210 void *src_ptr = src + y * mt->surf.row_pitch;
3211
3212 _mesa_streaming_load_memcpy(dst_ptr, src_ptr, width_bytes);
3213 }
3214
3215 intel_miptree_unmap_raw(mt);
3216 }
3217
3218 static void
intel_miptree_unmap_movntdqa(struct brw_context * brw,struct intel_mipmap_tree * mt,struct intel_miptree_map * map,unsigned int level,unsigned int slice)3219 intel_miptree_unmap_movntdqa(struct brw_context *brw,
3220 struct intel_mipmap_tree *mt,
3221 struct intel_miptree_map *map,
3222 unsigned int level,
3223 unsigned int slice)
3224 {
3225 _mesa_align_free(map->buffer);
3226 map->buffer = NULL;
3227 map->ptr = NULL;
3228 }
3229 #endif
3230
3231 static void
intel_miptree_map_s8(struct brw_context * brw,struct intel_mipmap_tree * mt,struct intel_miptree_map * map,unsigned int level,unsigned int slice)3232 intel_miptree_map_s8(struct brw_context *brw,
3233 struct intel_mipmap_tree *mt,
3234 struct intel_miptree_map *map,
3235 unsigned int level, unsigned int slice)
3236 {
3237 map->stride = map->w;
3238 map->buffer = map->ptr = malloc(map->stride * map->h);
3239 if (!map->buffer)
3240 return;
3241
3242 /* One of either READ_BIT or WRITE_BIT or both is set. READ_BIT implies no
3243 * INVALIDATE_RANGE_BIT. WRITE_BIT needs the original values read in unless
3244 * invalidate is set, since we'll be writing the whole rectangle from our
3245 * temporary buffer back out.
3246 */
3247 if (!(map->mode & GL_MAP_INVALIDATE_RANGE_BIT)) {
3248 uint8_t *untiled_s8_map = map->ptr;
3249 uint8_t *tiled_s8_map = intel_miptree_map_raw(brw, mt, GL_MAP_READ_BIT);
3250 unsigned int image_x, image_y;
3251
3252 intel_miptree_get_image_offset(mt, level, slice, &image_x, &image_y);
3253
3254 for (uint32_t y = 0; y < map->h; y++) {
3255 for (uint32_t x = 0; x < map->w; x++) {
3256 ptrdiff_t offset = intel_offset_S8(mt->surf.row_pitch,
3257 x + image_x + map->x,
3258 y + image_y + map->y,
3259 brw->has_swizzling);
3260 untiled_s8_map[y * map->w + x] = tiled_s8_map[offset];
3261 }
3262 }
3263
3264 intel_miptree_unmap_raw(mt);
3265
3266 DBG("%s: %d,%d %dx%d from mt %p %d,%d = %p/%d\n", __func__,
3267 map->x, map->y, map->w, map->h,
3268 mt, map->x + image_x, map->y + image_y, map->ptr, map->stride);
3269 } else {
3270 DBG("%s: %d,%d %dx%d from mt %p = %p/%d\n", __func__,
3271 map->x, map->y, map->w, map->h,
3272 mt, map->ptr, map->stride);
3273 }
3274 }
3275
3276 static void
intel_miptree_unmap_s8(struct brw_context * brw,struct intel_mipmap_tree * mt,struct intel_miptree_map * map,unsigned int level,unsigned int slice)3277 intel_miptree_unmap_s8(struct brw_context *brw,
3278 struct intel_mipmap_tree *mt,
3279 struct intel_miptree_map *map,
3280 unsigned int level,
3281 unsigned int slice)
3282 {
3283 if (map->mode & GL_MAP_WRITE_BIT) {
3284 unsigned int image_x, image_y;
3285 uint8_t *untiled_s8_map = map->ptr;
3286 uint8_t *tiled_s8_map = intel_miptree_map_raw(brw, mt, GL_MAP_WRITE_BIT);
3287
3288 intel_miptree_get_image_offset(mt, level, slice, &image_x, &image_y);
3289
3290 for (uint32_t y = 0; y < map->h; y++) {
3291 for (uint32_t x = 0; x < map->w; x++) {
3292 ptrdiff_t offset = intel_offset_S8(mt->surf.row_pitch,
3293 image_x + x + map->x,
3294 image_y + y + map->y,
3295 brw->has_swizzling);
3296 tiled_s8_map[offset] = untiled_s8_map[y * map->w + x];
3297 }
3298 }
3299
3300 intel_miptree_unmap_raw(mt);
3301 }
3302
3303 free(map->buffer);
3304 }
3305
3306 static void
intel_miptree_map_etc(struct brw_context * brw,struct intel_mipmap_tree * mt,struct intel_miptree_map * map,unsigned int level,unsigned int slice)3307 intel_miptree_map_etc(struct brw_context *brw,
3308 struct intel_mipmap_tree *mt,
3309 struct intel_miptree_map *map,
3310 unsigned int level,
3311 unsigned int slice)
3312 {
3313 assert(mt->etc_format != MESA_FORMAT_NONE);
3314 if (mt->etc_format == MESA_FORMAT_ETC1_RGB8) {
3315 assert(mt->format == MESA_FORMAT_R8G8B8X8_UNORM);
3316 }
3317
3318 assert(map->mode & GL_MAP_WRITE_BIT);
3319 assert(map->mode & GL_MAP_INVALIDATE_RANGE_BIT);
3320
3321 map->stride = _mesa_format_row_stride(mt->etc_format, map->w);
3322 map->buffer = malloc(_mesa_format_image_size(mt->etc_format,
3323 map->w, map->h, 1));
3324 map->ptr = map->buffer;
3325 }
3326
3327 static void
intel_miptree_unmap_etc(struct brw_context * brw,struct intel_mipmap_tree * mt,struct intel_miptree_map * map,unsigned int level,unsigned int slice)3328 intel_miptree_unmap_etc(struct brw_context *brw,
3329 struct intel_mipmap_tree *mt,
3330 struct intel_miptree_map *map,
3331 unsigned int level,
3332 unsigned int slice)
3333 {
3334 uint32_t image_x;
3335 uint32_t image_y;
3336 intel_miptree_get_image_offset(mt, level, slice, &image_x, &image_y);
3337
3338 image_x += map->x;
3339 image_y += map->y;
3340
3341 uint8_t *dst = intel_miptree_map_raw(brw, mt, GL_MAP_WRITE_BIT)
3342 + image_y * mt->surf.row_pitch
3343 + image_x * mt->cpp;
3344
3345 if (mt->etc_format == MESA_FORMAT_ETC1_RGB8)
3346 _mesa_etc1_unpack_rgba8888(dst, mt->surf.row_pitch,
3347 map->ptr, map->stride,
3348 map->w, map->h);
3349 else
3350 _mesa_unpack_etc2_format(dst, mt->surf.row_pitch,
3351 map->ptr, map->stride,
3352 map->w, map->h, mt->etc_format);
3353
3354 intel_miptree_unmap_raw(mt);
3355 free(map->buffer);
3356 }
3357
3358 /**
3359 * Mapping function for packed depth/stencil miptrees backed by real separate
3360 * miptrees for depth and stencil.
3361 *
3362 * On gen7, and to support HiZ pre-gen7, we have to have the stencil buffer
3363 * separate from the depth buffer. Yet at the GL API level, we have to expose
3364 * packed depth/stencil textures and FBO attachments, and Mesa core expects to
3365 * be able to map that memory for texture storage and glReadPixels-type
3366 * operations. We give Mesa core that access by mallocing a temporary and
3367 * copying the data between the actual backing store and the temporary.
3368 */
3369 static void
intel_miptree_map_depthstencil(struct brw_context * brw,struct intel_mipmap_tree * mt,struct intel_miptree_map * map,unsigned int level,unsigned int slice)3370 intel_miptree_map_depthstencil(struct brw_context *brw,
3371 struct intel_mipmap_tree *mt,
3372 struct intel_miptree_map *map,
3373 unsigned int level, unsigned int slice)
3374 {
3375 struct intel_mipmap_tree *z_mt = mt;
3376 struct intel_mipmap_tree *s_mt = mt->stencil_mt;
3377 bool map_z32f_x24s8 = mt->format == MESA_FORMAT_Z_FLOAT32;
3378 int packed_bpp = map_z32f_x24s8 ? 8 : 4;
3379
3380 map->stride = map->w * packed_bpp;
3381 map->buffer = map->ptr = malloc(map->stride * map->h);
3382 if (!map->buffer)
3383 return;
3384
3385 /* One of either READ_BIT or WRITE_BIT or both is set. READ_BIT implies no
3386 * INVALIDATE_RANGE_BIT. WRITE_BIT needs the original values read in unless
3387 * invalidate is set, since we'll be writing the whole rectangle from our
3388 * temporary buffer back out.
3389 */
3390 if (!(map->mode & GL_MAP_INVALIDATE_RANGE_BIT)) {
3391 uint32_t *packed_map = map->ptr;
3392 uint8_t *s_map = intel_miptree_map_raw(brw, s_mt, GL_MAP_READ_BIT);
3393 uint32_t *z_map = intel_miptree_map_raw(brw, z_mt, GL_MAP_READ_BIT);
3394 unsigned int s_image_x, s_image_y;
3395 unsigned int z_image_x, z_image_y;
3396
3397 intel_miptree_get_image_offset(s_mt, level, slice,
3398 &s_image_x, &s_image_y);
3399 intel_miptree_get_image_offset(z_mt, level, slice,
3400 &z_image_x, &z_image_y);
3401
3402 for (uint32_t y = 0; y < map->h; y++) {
3403 for (uint32_t x = 0; x < map->w; x++) {
3404 int map_x = map->x + x, map_y = map->y + y;
3405 ptrdiff_t s_offset = intel_offset_S8(s_mt->surf.row_pitch,
3406 map_x + s_image_x,
3407 map_y + s_image_y,
3408 brw->has_swizzling);
3409 ptrdiff_t z_offset = ((map_y + z_image_y) *
3410 (z_mt->surf.row_pitch / 4) +
3411 (map_x + z_image_x));
3412 uint8_t s = s_map[s_offset];
3413 uint32_t z = z_map[z_offset];
3414
3415 if (map_z32f_x24s8) {
3416 packed_map[(y * map->w + x) * 2 + 0] = z;
3417 packed_map[(y * map->w + x) * 2 + 1] = s;
3418 } else {
3419 packed_map[y * map->w + x] = (s << 24) | (z & 0x00ffffff);
3420 }
3421 }
3422 }
3423
3424 intel_miptree_unmap_raw(s_mt);
3425 intel_miptree_unmap_raw(z_mt);
3426
3427 DBG("%s: %d,%d %dx%d from z mt %p %d,%d, s mt %p %d,%d = %p/%d\n",
3428 __func__,
3429 map->x, map->y, map->w, map->h,
3430 z_mt, map->x + z_image_x, map->y + z_image_y,
3431 s_mt, map->x + s_image_x, map->y + s_image_y,
3432 map->ptr, map->stride);
3433 } else {
3434 DBG("%s: %d,%d %dx%d from mt %p = %p/%d\n", __func__,
3435 map->x, map->y, map->w, map->h,
3436 mt, map->ptr, map->stride);
3437 }
3438 }
3439
3440 static void
intel_miptree_unmap_depthstencil(struct brw_context * brw,struct intel_mipmap_tree * mt,struct intel_miptree_map * map,unsigned int level,unsigned int slice)3441 intel_miptree_unmap_depthstencil(struct brw_context *brw,
3442 struct intel_mipmap_tree *mt,
3443 struct intel_miptree_map *map,
3444 unsigned int level,
3445 unsigned int slice)
3446 {
3447 struct intel_mipmap_tree *z_mt = mt;
3448 struct intel_mipmap_tree *s_mt = mt->stencil_mt;
3449 bool map_z32f_x24s8 = mt->format == MESA_FORMAT_Z_FLOAT32;
3450
3451 if (map->mode & GL_MAP_WRITE_BIT) {
3452 uint32_t *packed_map = map->ptr;
3453 uint8_t *s_map = intel_miptree_map_raw(brw, s_mt, GL_MAP_WRITE_BIT);
3454 uint32_t *z_map = intel_miptree_map_raw(brw, z_mt, GL_MAP_WRITE_BIT);
3455 unsigned int s_image_x, s_image_y;
3456 unsigned int z_image_x, z_image_y;
3457
3458 intel_miptree_get_image_offset(s_mt, level, slice,
3459 &s_image_x, &s_image_y);
3460 intel_miptree_get_image_offset(z_mt, level, slice,
3461 &z_image_x, &z_image_y);
3462
3463 for (uint32_t y = 0; y < map->h; y++) {
3464 for (uint32_t x = 0; x < map->w; x++) {
3465 ptrdiff_t s_offset = intel_offset_S8(s_mt->surf.row_pitch,
3466 x + s_image_x + map->x,
3467 y + s_image_y + map->y,
3468 brw->has_swizzling);
3469 ptrdiff_t z_offset = ((y + z_image_y + map->y) *
3470 (z_mt->surf.row_pitch / 4) +
3471 (x + z_image_x + map->x));
3472
3473 if (map_z32f_x24s8) {
3474 z_map[z_offset] = packed_map[(y * map->w + x) * 2 + 0];
3475 s_map[s_offset] = packed_map[(y * map->w + x) * 2 + 1];
3476 } else {
3477 uint32_t packed = packed_map[y * map->w + x];
3478 s_map[s_offset] = packed >> 24;
3479 z_map[z_offset] = packed;
3480 }
3481 }
3482 }
3483
3484 intel_miptree_unmap_raw(s_mt);
3485 intel_miptree_unmap_raw(z_mt);
3486
3487 DBG("%s: %d,%d %dx%d from z mt %p (%s) %d,%d, s mt %p %d,%d = %p/%d\n",
3488 __func__,
3489 map->x, map->y, map->w, map->h,
3490 z_mt, _mesa_get_format_name(z_mt->format),
3491 map->x + z_image_x, map->y + z_image_y,
3492 s_mt, map->x + s_image_x, map->y + s_image_y,
3493 map->ptr, map->stride);
3494 }
3495
3496 free(map->buffer);
3497 }
3498
3499 /**
3500 * Create and attach a map to the miptree at (level, slice). Return the
3501 * attached map.
3502 */
3503 static struct intel_miptree_map*
intel_miptree_attach_map(struct intel_mipmap_tree * mt,unsigned int level,unsigned int slice,unsigned int x,unsigned int y,unsigned int w,unsigned int h,GLbitfield mode)3504 intel_miptree_attach_map(struct intel_mipmap_tree *mt,
3505 unsigned int level,
3506 unsigned int slice,
3507 unsigned int x,
3508 unsigned int y,
3509 unsigned int w,
3510 unsigned int h,
3511 GLbitfield mode)
3512 {
3513 struct intel_miptree_map *map = calloc(1, sizeof(*map));
3514
3515 if (!map)
3516 return NULL;
3517
3518 assert(mt->level[level].slice[slice].map == NULL);
3519 mt->level[level].slice[slice].map = map;
3520
3521 map->mode = mode;
3522 map->x = x;
3523 map->y = y;
3524 map->w = w;
3525 map->h = h;
3526
3527 return map;
3528 }
3529
3530 /**
3531 * Release the map at (level, slice).
3532 */
3533 static void
intel_miptree_release_map(struct intel_mipmap_tree * mt,unsigned int level,unsigned int slice)3534 intel_miptree_release_map(struct intel_mipmap_tree *mt,
3535 unsigned int level,
3536 unsigned int slice)
3537 {
3538 struct intel_miptree_map **map;
3539
3540 map = &mt->level[level].slice[slice].map;
3541 free(*map);
3542 *map = NULL;
3543 }
3544
3545 static bool
can_blit_slice(struct intel_mipmap_tree * mt,unsigned int level,unsigned int slice)3546 can_blit_slice(struct intel_mipmap_tree *mt,
3547 unsigned int level, unsigned int slice)
3548 {
3549 /* See intel_miptree_blit() for details on the 32k pitch limit. */
3550 if (mt->surf.row_pitch >= 32768)
3551 return false;
3552
3553 return true;
3554 }
3555
3556 static bool
use_intel_mipree_map_blit(struct brw_context * brw,struct intel_mipmap_tree * mt,GLbitfield mode,unsigned int level,unsigned int slice)3557 use_intel_mipree_map_blit(struct brw_context *brw,
3558 struct intel_mipmap_tree *mt,
3559 GLbitfield mode,
3560 unsigned int level,
3561 unsigned int slice)
3562 {
3563 const struct gen_device_info *devinfo = &brw->screen->devinfo;
3564
3565 if (devinfo->has_llc &&
3566 /* It's probably not worth swapping to the blit ring because of
3567 * all the overhead involved.
3568 */
3569 !(mode & GL_MAP_WRITE_BIT) &&
3570 !mt->compressed &&
3571 (mt->surf.tiling == ISL_TILING_X ||
3572 /* Prior to Sandybridge, the blitter can't handle Y tiling */
3573 (devinfo->gen >= 6 && mt->surf.tiling == ISL_TILING_Y0) ||
3574 /* Fast copy blit on skl+ supports all tiling formats. */
3575 devinfo->gen >= 9) &&
3576 can_blit_slice(mt, level, slice))
3577 return true;
3578
3579 if (mt->surf.tiling != ISL_TILING_LINEAR &&
3580 mt->bo->size >= brw->max_gtt_map_object_size) {
3581 assert(can_blit_slice(mt, level, slice));
3582 return true;
3583 }
3584
3585 return false;
3586 }
3587
3588 /**
3589 * Parameter \a out_stride has type ptrdiff_t not because the buffer stride may
3590 * exceed 32 bits but to diminish the likelihood subtle bugs in pointer
3591 * arithmetic overflow.
3592 *
3593 * If you call this function and use \a out_stride, then you're doing pointer
3594 * arithmetic on \a out_ptr. The type of \a out_stride doesn't prevent all
3595 * bugs. The caller must still take care to avoid 32-bit overflow errors in
3596 * all arithmetic expressions that contain buffer offsets and pixel sizes,
3597 * which usually have type uint32_t or GLuint.
3598 */
3599 void
intel_miptree_map(struct brw_context * brw,struct intel_mipmap_tree * mt,unsigned int level,unsigned int slice,unsigned int x,unsigned int y,unsigned int w,unsigned int h,GLbitfield mode,void ** out_ptr,ptrdiff_t * out_stride)3600 intel_miptree_map(struct brw_context *brw,
3601 struct intel_mipmap_tree *mt,
3602 unsigned int level,
3603 unsigned int slice,
3604 unsigned int x,
3605 unsigned int y,
3606 unsigned int w,
3607 unsigned int h,
3608 GLbitfield mode,
3609 void **out_ptr,
3610 ptrdiff_t *out_stride)
3611 {
3612 struct intel_miptree_map *map;
3613
3614 assert(mt->surf.samples == 1);
3615
3616 map = intel_miptree_attach_map(mt, level, slice, x, y, w, h, mode);
3617 if (!map){
3618 *out_ptr = NULL;
3619 *out_stride = 0;
3620 return;
3621 }
3622
3623 intel_miptree_access_raw(brw, mt, level, slice,
3624 map->mode & GL_MAP_WRITE_BIT);
3625
3626 if (mt->format == MESA_FORMAT_S_UINT8) {
3627 intel_miptree_map_s8(brw, mt, map, level, slice);
3628 } else if (mt->etc_format != MESA_FORMAT_NONE &&
3629 !(mode & BRW_MAP_DIRECT_BIT)) {
3630 intel_miptree_map_etc(brw, mt, map, level, slice);
3631 } else if (mt->stencil_mt && !(mode & BRW_MAP_DIRECT_BIT)) {
3632 intel_miptree_map_depthstencil(brw, mt, map, level, slice);
3633 } else if (use_intel_mipree_map_blit(brw, mt, mode, level, slice)) {
3634 intel_miptree_map_blit(brw, mt, map, level, slice);
3635 #if defined(USE_SSE41)
3636 } else if (!(mode & GL_MAP_WRITE_BIT) &&
3637 !mt->compressed && cpu_has_sse4_1 &&
3638 (mt->surf.row_pitch % 16 == 0)) {
3639 intel_miptree_map_movntdqa(brw, mt, map, level, slice);
3640 #endif
3641 } else {
3642 intel_miptree_map_gtt(brw, mt, map, level, slice);
3643 }
3644
3645 *out_ptr = map->ptr;
3646 *out_stride = map->stride;
3647
3648 if (map->ptr == NULL)
3649 intel_miptree_release_map(mt, level, slice);
3650 }
3651
3652 void
intel_miptree_unmap(struct brw_context * brw,struct intel_mipmap_tree * mt,unsigned int level,unsigned int slice)3653 intel_miptree_unmap(struct brw_context *brw,
3654 struct intel_mipmap_tree *mt,
3655 unsigned int level,
3656 unsigned int slice)
3657 {
3658 struct intel_miptree_map *map = mt->level[level].slice[slice].map;
3659
3660 assert(mt->surf.samples == 1);
3661
3662 if (!map)
3663 return;
3664
3665 DBG("%s: mt %p (%s) level %d slice %d\n", __func__,
3666 mt, _mesa_get_format_name(mt->format), level, slice);
3667
3668 if (mt->format == MESA_FORMAT_S_UINT8) {
3669 intel_miptree_unmap_s8(brw, mt, map, level, slice);
3670 } else if (mt->etc_format != MESA_FORMAT_NONE &&
3671 !(map->mode & BRW_MAP_DIRECT_BIT)) {
3672 intel_miptree_unmap_etc(brw, mt, map, level, slice);
3673 } else if (mt->stencil_mt && !(map->mode & BRW_MAP_DIRECT_BIT)) {
3674 intel_miptree_unmap_depthstencil(brw, mt, map, level, slice);
3675 } else if (map->linear_mt) {
3676 intel_miptree_unmap_blit(brw, mt, map, level, slice);
3677 #if defined(USE_SSE41)
3678 } else if (map->buffer && cpu_has_sse4_1) {
3679 intel_miptree_unmap_movntdqa(brw, mt, map, level, slice);
3680 #endif
3681 } else {
3682 intel_miptree_unmap_gtt(mt);
3683 }
3684
3685 intel_miptree_release_map(mt, level, slice);
3686 }
3687
3688 enum isl_surf_dim
get_isl_surf_dim(GLenum target)3689 get_isl_surf_dim(GLenum target)
3690 {
3691 switch (target) {
3692 case GL_TEXTURE_1D:
3693 case GL_TEXTURE_1D_ARRAY:
3694 return ISL_SURF_DIM_1D;
3695
3696 case GL_TEXTURE_2D:
3697 case GL_TEXTURE_2D_ARRAY:
3698 case GL_TEXTURE_RECTANGLE:
3699 case GL_TEXTURE_CUBE_MAP:
3700 case GL_TEXTURE_CUBE_MAP_ARRAY:
3701 case GL_TEXTURE_2D_MULTISAMPLE:
3702 case GL_TEXTURE_2D_MULTISAMPLE_ARRAY:
3703 case GL_TEXTURE_EXTERNAL_OES:
3704 return ISL_SURF_DIM_2D;
3705
3706 case GL_TEXTURE_3D:
3707 return ISL_SURF_DIM_3D;
3708 }
3709
3710 unreachable("Invalid texture target");
3711 }
3712
3713 enum isl_dim_layout
get_isl_dim_layout(const struct gen_device_info * devinfo,enum isl_tiling tiling,GLenum target)3714 get_isl_dim_layout(const struct gen_device_info *devinfo,
3715 enum isl_tiling tiling, GLenum target)
3716 {
3717 switch (target) {
3718 case GL_TEXTURE_1D:
3719 case GL_TEXTURE_1D_ARRAY:
3720 return (devinfo->gen >= 9 && tiling == ISL_TILING_LINEAR ?
3721 ISL_DIM_LAYOUT_GEN9_1D : ISL_DIM_LAYOUT_GEN4_2D);
3722
3723 case GL_TEXTURE_2D:
3724 case GL_TEXTURE_2D_ARRAY:
3725 case GL_TEXTURE_RECTANGLE:
3726 case GL_TEXTURE_2D_MULTISAMPLE:
3727 case GL_TEXTURE_2D_MULTISAMPLE_ARRAY:
3728 case GL_TEXTURE_EXTERNAL_OES:
3729 return ISL_DIM_LAYOUT_GEN4_2D;
3730
3731 case GL_TEXTURE_CUBE_MAP:
3732 case GL_TEXTURE_CUBE_MAP_ARRAY:
3733 return (devinfo->gen == 4 ? ISL_DIM_LAYOUT_GEN4_3D :
3734 ISL_DIM_LAYOUT_GEN4_2D);
3735
3736 case GL_TEXTURE_3D:
3737 return (devinfo->gen >= 9 ?
3738 ISL_DIM_LAYOUT_GEN4_2D : ISL_DIM_LAYOUT_GEN4_3D);
3739 }
3740
3741 unreachable("Invalid texture target");
3742 }
3743
3744 enum isl_aux_usage
intel_miptree_get_aux_isl_usage(const struct brw_context * brw,const struct intel_mipmap_tree * mt)3745 intel_miptree_get_aux_isl_usage(const struct brw_context *brw,
3746 const struct intel_mipmap_tree *mt)
3747 {
3748 if (mt->hiz_buf)
3749 return ISL_AUX_USAGE_HIZ;
3750
3751 if (!mt->mcs_buf)
3752 return ISL_AUX_USAGE_NONE;
3753
3754 return mt->aux_usage;
3755 }
3756