android-16.0.0_r2/s

/*
 * Copyright (C) 2021 Collabora, Ltd.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 * Authors:
 *   Alyssa Rosenzweig <alyssa.rosenzweig@collabora.com>
 *   Boris Brezillon <boris.brezillon@collabora.com>
 */

#include "util/macros.h"

#include "genxml/gen_macros.h"

#include "pan_desc.h"
#include "pan_encoder.h"
#include "pan_props.h"
#include "pan_texture.h"

static unsigned
mod_to_block_fmt(uint64_t mod)
{
   switch (mod) {
   case DRM_FORMAT_MOD_LINEAR:
      return MALI_BLOCK_FORMAT_LINEAR;
   case DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED:
      return MALI_BLOCK_FORMAT_TILED_U_INTERLEAVED;
   default:
#if PAN_ARCH >= 5
      if (drm_is_afbc(mod) && !(mod & AFBC_FORMAT_MOD_TILED))
         return MALI_BLOCK_FORMAT_AFBC;
#endif
#if PAN_ARCH >= 7
      if (drm_is_afbc(mod) && (mod & AFBC_FORMAT_MOD_TILED))
         return MALI_BLOCK_FORMAT_AFBC_TILED;
#endif
#if PAN_ARCH >= 10
      if (drm_is_afrc(mod))
         return 0; /* Reserved field for AFRC state */
#endif

      unreachable("Unsupported modifer");
   }
}

static enum mali_msaa
mali_sampling_mode(const struct pan_image_view *view)
{
   unsigned nr_samples = pan_image_view_get_nr_samples(view);

   if (nr_samples > 1) {
      ASSERTED const struct pan_image *first_plane =
         pan_image_view_get_first_plane(view);

      assert(view->nr_samples == nr_samples);
      assert(first_plane->layout.slices[0].surface_stride != 0);
      return MALI_MSAA_LAYERED;
   }

   if (view->nr_samples > nr_samples) {
      assert(nr_samples == 1);
      return MALI_MSAA_AVERAGE;
   }

   assert(view->nr_samples == nr_samples);
   assert(view->nr_samples == 1);

   return MALI_MSAA_SINGLE;
}

static bool
renderblock_fits_in_single_pass(const struct pan_image_view *view,
                                unsigned tile_size)
{
   const struct pan_image *first_plane = pan_image_view_get_first_plane(view);
   uint64_t mod = first_plane->layout.modifier;

   if (!drm_is_afbc(mod))
      return tile_size >= 16 * 16;

   struct pan_block_size renderblk_sz = panfrost_afbc_renderblock_size(mod);
   return tile_size >= renderblk_sz.width * renderblk_sz.height;
}

int
GENX(pan_select_crc_rt)(const struct pan_fb_info *fb, unsigned tile_size)
{
   /* Disable CRC when the tile size is smaller than 16x16. In the hardware,
    * CRC tiles are the same size as the tiles of the framebuffer. However,
    * our code only handles 16x16 tiles. Therefore under the current
    * implementation, we must disable CRC when 16x16 tiles are not used.
    *
    * This may hurt performance. However, smaller tile sizes are rare, and
    * CRCs are more expensive at smaller tile sizes, reducing the benefit.
    * Restricting CRC to 16x16 should work in practice.
    */
   if (tile_size < 16 * 16)
      return -1;

#if PAN_ARCH <= 6
   if (fb->rt_count == 1 && fb->rts[0].view && !fb->rts[0].discard &&
       pan_image_view_has_crc(fb->rts[0].view))
      return 0;

   return -1;
#else
   bool best_rt_valid = false;
   int best_rt = -1;

   for (unsigned i = 0; i < fb->rt_count; i++) {
      if (!fb->rts[i].view || fb->rts[i].discard ||
          !pan_image_view_has_crc(fb->rts[i].view))
         continue;

      if (!renderblock_fits_in_single_pass(fb->rts[i].view, tile_size))
         continue;

      bool valid = *(fb->rts[i].crc_valid);
      bool full = !fb->extent.minx && !fb->extent.miny &&
                  fb->extent.maxx == (fb->width - 1) &&
                  fb->extent.maxy == (fb->height - 1);
      if (!full && !valid)
         continue;

      if (best_rt < 0 || (valid && !best_rt_valid)) {
         best_rt = i;
         best_rt_valid = valid;
      }

      if (valid)
         break;
   }

   return best_rt;
#endif
}

static enum mali_zs_format
translate_zs_format(enum pipe_format in)
{
   switch (in) {
   case PIPE_FORMAT_Z16_UNORM:
      return MALI_ZS_FORMAT_D16;
   case PIPE_FORMAT_Z24_UNORM_S8_UINT:
      return MALI_ZS_FORMAT_D24S8;
   case PIPE_FORMAT_Z24X8_UNORM:
      return MALI_ZS_FORMAT_D24X8;
   case PIPE_FORMAT_Z32_FLOAT:
      return MALI_ZS_FORMAT_D32;
#if PAN_ARCH <= 7
   case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT:
      return MALI_ZS_FORMAT_D32_S8X24;
#endif
   default:
      unreachable("Unsupported depth/stencil format.");
   }
}

#if PAN_ARCH >= 5
static enum mali_s_format
translate_s_format(enum pipe_format in)
{
   switch (in) {
   case PIPE_FORMAT_S8_UINT:
      return MALI_S_FORMAT_S8;
   case PIPE_FORMAT_Z24_UNORM_S8_UINT:
   case PIPE_FORMAT_X24S8_UINT:
      return MALI_S_FORMAT_X24S8;

#if PAN_ARCH <= 7
   case PIPE_FORMAT_S8_UINT_Z24_UNORM:
   case PIPE_FORMAT_S8X24_UINT:
      return MALI_S_FORMAT_S8X24;
   case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT:
   case PIPE_FORMAT_X32_S8X24_UINT:
      return MALI_S_FORMAT_X32_S8X24;
#endif

   default:
      unreachable("Unsupported stencil format.");
   }
}

static void
pan_prepare_s(const struct pan_fb_info *fb, unsigned layer_idx,
              struct MALI_ZS_CRC_EXTENSION *ext)
{
   const struct pan_image_view *s = fb->zs.view.s;

   if (!s)
      return;

   const struct pan_image *image = pan_image_view_get_s_plane(s);
   unsigned level = s->first_level;

   ext->s_msaa = mali_sampling_mode(s);

   struct pan_surface surf;
   pan_iview_get_surface(s, 0, layer_idx, 0, &surf);

   assert(image->layout.modifier ==
             DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED ||
          image->layout.modifier == DRM_FORMAT_MOD_LINEAR);
   ext->s_writeback_base = surf.data;
   ext->s_writeback_row_stride = image->layout.slices[level].row_stride;
   ext->s_writeback_surface_stride =
      (pan_image_view_get_nr_samples(s) > 1)
         ? image->layout.slices[level].surface_stride
         : 0;
   ext->s_block_format = mod_to_block_fmt(image->layout.modifier);
   ext->s_write_format = translate_s_format(s->format);
}

static void
pan_prepare_zs(const struct pan_fb_info *fb, unsigned layer_idx,
               struct MALI_ZS_CRC_EXTENSION *ext)
{
   const struct pan_image_view *zs = fb->zs.view.zs;

   if (!zs)
      return;

   const struct pan_image *image = pan_image_view_get_zs_plane(zs);
   unsigned level = zs->first_level;

   ext->zs_msaa = mali_sampling_mode(zs);

   struct pan_surface surf;
   pan_iview_get_surface(zs, 0, layer_idx, 0, &surf);
   UNUSED const struct pan_image_slice_layout *slice =
      &image->layout.slices[level];

   if (drm_is_afbc(image->layout.modifier)) {
#if PAN_ARCH >= 9
      ext->zs_writeback_base = surf.afbc.header;
      ext->zs_writeback_row_stride = slice->row_stride;
      /* TODO: surface stride? */
      ext->zs_afbc_body_offset = surf.afbc.body - surf.afbc.header;

      /* TODO: stencil AFBC? */
#else
#if PAN_ARCH >= 6
      ext->zs_afbc_row_stride =
         pan_afbc_stride_blocks(image->layout.modifier, slice->row_stride);
#else
      ext->zs_block_format = MALI_BLOCK_FORMAT_AFBC;
      ext->zs_afbc_body_size = 0x1000;
      ext->zs_afbc_chunk_size = 9;
      ext->zs_afbc_sparse = true;
#endif

      ext->zs_afbc_header = surf.afbc.header;
      ext->zs_afbc_body = surf.afbc.body;
#endif
   } else {
      assert(image->layout.modifier ==
                DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED ||
             image->layout.modifier == DRM_FORMAT_MOD_LINEAR);

      /* TODO: Z32F(S8) support, which is always linear */

      ext->zs_writeback_base = surf.data;
      ext->zs_writeback_row_stride = image->layout.slices[level].row_stride;
      ext->zs_writeback_surface_stride =
         (pan_image_view_get_nr_samples(zs) > 1)
            ? image->layout.slices[level].surface_stride
            : 0;
   }

   ext->zs_block_format = mod_to_block_fmt(image->layout.modifier);
   ext->zs_write_format = translate_zs_format(zs->format);
   if (ext->zs_write_format == MALI_ZS_FORMAT_D24S8)
      ext->s_writeback_base = ext->zs_writeback_base;
}

static void
pan_prepare_crc(const struct pan_fb_info *fb, int rt_crc,
                struct MALI_ZS_CRC_EXTENSION *ext)
{
   if (rt_crc < 0)
      return;

   assert(rt_crc < fb->rt_count);

   const struct pan_image_view *rt = fb->rts[rt_crc].view;
   const struct pan_image *image = pan_image_view_get_color_plane(rt);
   const struct pan_image_slice_layout *slice =
      &image->layout.slices[rt->first_level];

   ext->crc_base =
      image->data.base + image->data.offset + slice->crc.offset;
   ext->crc_row_stride = slice->crc.stride;

#if PAN_ARCH >= 7
   ext->crc_render_target = rt_crc;

   if (fb->rts[rt_crc].clear) {
      uint32_t clear_val = fb->rts[rt_crc].clear_value[0];
      ext->crc_clear_color = clear_val | 0xc000000000000000 |
                             (((uint64_t)clear_val & 0xffff) << 32);
   }
#endif
}

static void
pan_emit_zs_crc_ext(const struct pan_fb_info *fb, unsigned layer_idx,
                    int rt_crc, struct mali_zs_crc_extension_packed *zs_crc_ext)
{
   pan_pack(zs_crc_ext, ZS_CRC_EXTENSION, cfg) {
      pan_prepare_crc(fb, rt_crc, &cfg);
      cfg.zs_clean_pixel_write_enable = fb->zs.clear.z || fb->zs.clear.s;
      pan_prepare_zs(fb, layer_idx, &cfg);
      pan_prepare_s(fb, layer_idx, &cfg);
   }
}

/* Measure format as it appears in the tile buffer */

static unsigned
pan_bytes_per_pixel_tib(enum pipe_format format)
{
   const struct pan_blendable_format *bf =
     GENX(panfrost_blendable_format_from_pipe_format)(format);

   if (bf->internal) {
      /* Blendable formats are always 32-bits in the tile buffer,
       * extra bits are used as padding or to dither */
      return 4;
   } else {
      /* Non-blendable formats are raw, rounded up to the nearest
       * power-of-two size */
      unsigned bytes = util_format_get_blocksize(format);
      return util_next_power_of_two(bytes);
   }
}

static unsigned
pan_cbuf_bytes_per_pixel(const struct pan_fb_info *fb)
{
   unsigned sum = 0;

   for (int cb = 0; cb < fb->rt_count; ++cb) {
      const struct pan_image_view *rt = fb->rts[cb].view;

      if (!rt)
         continue;

      sum += pan_bytes_per_pixel_tib(rt->format) * rt->nr_samples;
   }

   return sum;
}

/*
 * Select the largest tile size that fits within the tilebuffer budget.
 * Formally, maximize (pixels per tile) such that it is a power of two and
 *
 *      (bytes per pixel) (pixels per tile) <= (max bytes per tile)
 *
 * A bit of algebra gives the following formula.
 *
 * Calculate the color buffer allocation size as well.
 */
void
GENX(pan_select_tile_size)(struct pan_fb_info *fb)
{
   unsigned bytes_per_pixel;

   assert(util_is_power_of_two_nonzero(fb->tile_buf_budget));
   assert(fb->tile_buf_budget >= 1024);

   bytes_per_pixel = pan_cbuf_bytes_per_pixel(fb);
   fb->tile_size = fb->tile_buf_budget >> util_logbase2_ceil(bytes_per_pixel);

   /* Clamp tile size to hardware limits */
   fb->tile_size =
      MIN2(fb->tile_size, panfrost_max_effective_tile_size(PAN_ARCH));
   assert(fb->tile_size >= 4 * 4);

   /* Colour buffer allocations must be 1K aligned. */
   fb->cbuf_allocation = ALIGN_POT(bytes_per_pixel * fb->tile_size, 1024);
   assert(fb->cbuf_allocation <= fb->tile_buf_budget && "tile too big");
}

static enum mali_color_format
pan_mfbd_raw_format(unsigned bits)
{
   /* clang-format off */
   switch (bits) {
   case    8: return MALI_COLOR_FORMAT_RAW8;
   case   16: return MALI_COLOR_FORMAT_RAW16;
   case   24: return MALI_COLOR_FORMAT_RAW24;
   case   32: return MALI_COLOR_FORMAT_RAW32;
   case   48: return MALI_COLOR_FORMAT_RAW48;
   case   64: return MALI_COLOR_FORMAT_RAW64;
   case   96: return MALI_COLOR_FORMAT_RAW96;
   case  128: return MALI_COLOR_FORMAT_RAW128;
   case  192: return MALI_COLOR_FORMAT_RAW192;
   case  256: return MALI_COLOR_FORMAT_RAW256;
   case  384: return MALI_COLOR_FORMAT_RAW384;
   case  512: return MALI_COLOR_FORMAT_RAW512;
   case  768: return MALI_COLOR_FORMAT_RAW768;
   case 1024: return MALI_COLOR_FORMAT_RAW1024;
   case 1536: return MALI_COLOR_FORMAT_RAW1536;
   case 2048: return MALI_COLOR_FORMAT_RAW2048;
   default: unreachable("invalid raw bpp");
   }
   /* clang-format on */
}

static void
pan_rt_init_format(const struct pan_image_view *rt,
                   struct MALI_RENDER_TARGET *cfg)
{
   /* Explode details on the format */

   const struct util_format_description *desc =
      util_format_description(rt->format);

   /* The swizzle for rendering is inverted from texturing */

   unsigned char swizzle[4] = {
      PIPE_SWIZZLE_X,
      PIPE_SWIZZLE_Y,
      PIPE_SWIZZLE_Z,
      PIPE_SWIZZLE_W,
   };

   /* Fill in accordingly, defaulting to 8-bit UNORM */

   if (desc->colorspace == UTIL_FORMAT_COLORSPACE_SRGB)
      cfg->srgb = true;

   struct pan_blendable_format fmt =
      *GENX(panfrost_blendable_format_from_pipe_format)(rt->format);
   enum mali_color_format writeback_format;

   if (fmt.internal) {
      cfg->internal_format = fmt.internal;
      writeback_format = fmt.writeback;
      panfrost_invert_swizzle(desc->swizzle, swizzle);
   } else {
      /* Construct RAW internal/writeback, where internal is
       * specified logarithmically (round to next power-of-two).
       * Offset specified from RAW8, where 8 = 2^3 */

      unsigned bits = desc->block.bits;
      assert(bits >= 8 && bits <= 128);
      unsigned offset = util_logbase2_ceil(bits) - 3;
      assert(offset <= 4);

      cfg->internal_format = MALI_COLOR_BUFFER_INTERNAL_FORMAT_RAW8 + offset;
      writeback_format = pan_mfbd_raw_format(bits);
   }

#if PAN_ARCH >= 10
   const struct pan_image *image = pan_image_view_get_color_plane(rt);

   if (drm_is_afrc(image->layout.modifier))
      cfg->afrc.writeback_format = writeback_format;
   else
      cfg->writeback_format = writeback_format;
#else
   cfg->writeback_format = writeback_format;
#endif

   cfg->swizzle = panfrost_translate_swizzle_4(swizzle);
}

static void
pan_prepare_rt(const struct pan_fb_info *fb, unsigned layer_idx,
               unsigned rt_idx, unsigned cbuf_offset,
               struct MALI_RENDER_TARGET *cfg)
{
   cfg->clean_pixel_write_enable = fb->rts[rt_idx].clear;
   cfg->internal_buffer_offset = cbuf_offset;
   if (fb->rts[rt_idx].clear) {
      cfg->clear.color_0 = fb->rts[rt_idx].clear_value[0];
      cfg->clear.color_1 = fb->rts[rt_idx].clear_value[1];
      cfg->clear.color_2 = fb->rts[rt_idx].clear_value[2];
      cfg->clear.color_3 = fb->rts[rt_idx].clear_value[3];
   }

   const struct pan_image_view *rt = fb->rts[rt_idx].view;
   if (!rt || fb->rts[rt_idx].discard) {
      cfg->internal_format = MALI_COLOR_BUFFER_INTERNAL_FORMAT_R8G8B8A8;
      cfg->internal_buffer_offset = cbuf_offset;
#if PAN_ARCH >= 7
      cfg->writeback_block_format = MALI_BLOCK_FORMAT_TILED_U_INTERLEAVED;
      cfg->dithering_enable = true;
#endif
      return;
   }

   const struct pan_image *image = pan_image_view_get_color_plane(rt);

   if (!drm_is_afrc(image->layout.modifier))
      cfg->write_enable = true;

   cfg->dithering_enable = true;

   const struct pan_image *first_plane = pan_image_view_get_first_plane(rt);
   unsigned level = rt->first_level;
   ASSERTED unsigned layer_count = rt->dim == MALI_TEXTURE_DIMENSION_3D
                                      ? first_plane->layout.depth
                                      : rt->last_layer - rt->first_layer + 1;

   assert(rt->last_level == rt->first_level);
   assert(layer_idx < layer_count);

   int row_stride = image->layout.slices[level].row_stride;

   /* Only set layer_stride for layered MSAA rendering  */

   unsigned layer_stride = (pan_image_view_get_nr_samples(rt) > 1)
                              ? image->layout.slices[level].surface_stride
                              : 0;

   cfg->writeback_msaa = mali_sampling_mode(rt);

   pan_rt_init_format(rt, cfg);

   cfg->writeback_block_format = mod_to_block_fmt(image->layout.modifier);

   struct pan_surface surf;
   pan_iview_get_surface(rt, 0, layer_idx, 0, &surf);

   if (drm_is_afbc(image->layout.modifier)) {
#if PAN_ARCH >= 9
      if (image->layout.modifier & AFBC_FORMAT_MOD_YTR)
         cfg->afbc.yuv_transform = true;

      cfg->afbc.wide_block = panfrost_afbc_is_wide(image->layout.modifier);
      cfg->afbc.split_block =
         (image->layout.modifier & AFBC_FORMAT_MOD_SPLIT);
      cfg->afbc.header = surf.afbc.header;
      cfg->afbc.body_offset = surf.afbc.body - surf.afbc.header;
      assert(surf.afbc.body >= surf.afbc.header);

      cfg->afbc.compression_mode = GENX(pan_afbc_compression_mode)(rt->format);
      cfg->afbc.row_stride = row_stride;
#else
      const struct pan_image_slice_layout *slice = &image->layout.slices[level];

#if PAN_ARCH >= 6
      cfg->afbc.row_stride =
         pan_afbc_stride_blocks(image->layout.modifier, slice->row_stride);
      cfg->afbc.afbc_wide_block_enable =
         panfrost_afbc_is_wide(image->layout.modifier);
      cfg->afbc.afbc_split_block_enable =
         (image->layout.modifier & AFBC_FORMAT_MOD_SPLIT);
#else
      cfg->afbc.chunk_size = 9;
      cfg->afbc.sparse = true;
      cfg->afbc.body_size = slice->afbc.body_size;
#endif

      cfg->afbc.header = surf.afbc.header;
      cfg->afbc.body = surf.afbc.body;

      if (image->layout.modifier & AFBC_FORMAT_MOD_YTR)
         cfg->afbc.yuv_transform_enable = true;
#endif
#if PAN_ARCH >= 10
   } else if (drm_is_afrc(image->layout.modifier)) {
      struct pan_afrc_format_info finfo =
         panfrost_afrc_get_format_info(image->layout.format);

      cfg->writeback_mode = MALI_WRITEBACK_MODE_AFRC_RGB;
      cfg->afrc.block_size =
         GENX(pan_afrc_block_size)(image->layout.modifier, 0);
      cfg->afrc.format =
         GENX(pan_afrc_format)(finfo, image->layout.modifier, 0);

      cfg->rgb.base = surf.data;
      cfg->rgb.row_stride = row_stride;
      cfg->rgb.surface_stride = layer_stride;
#endif
   } else {
      assert(image->layout.modifier == DRM_FORMAT_MOD_LINEAR ||
             image->layout.modifier ==
                DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED);
      cfg->rgb.base = surf.data;
      cfg->rgb.row_stride = row_stride;
      cfg->rgb.surface_stride = layer_stride;
   }
}
#endif

void
GENX(pan_emit_tls)(const struct pan_tls_info *info,
                   struct mali_local_storage_packed *out)
{
   pan_pack(out, LOCAL_STORAGE, cfg) {
      if (info->tls.size) {
         unsigned shift = panfrost_get_stack_shift(info->tls.size);

         cfg.tls_size = shift;
#if PAN_ARCH >= 9
         /* For now, always use packed TLS addressing. This is
          * better for the cache and requires no fix up code in
          * the shader. We may need to revisit this someday for
          * OpenCL generic pointer support.
          */
         cfg.tls_address_mode = MALI_ADDRESS_MODE_PACKED;

         assert((info->tls.ptr & 4095) == 0);
         cfg.tls_base_pointer = info->tls.ptr >> 8;
#else
         cfg.tls_base_pointer = info->tls.ptr;
#endif
      }

      if (info->wls.size) {
         assert(!(info->wls.ptr & 4095));
         assert((info->wls.ptr & 0xffffffff00000000ULL) ==
                ((info->wls.ptr + info->wls.size - 1) & 0xffffffff00000000ULL));
         cfg.wls_base_pointer = info->wls.ptr;
         unsigned wls_size = pan_wls_adjust_size(info->wls.size);
         cfg.wls_instances = info->wls.instances;
         cfg.wls_size_scale = util_logbase2(wls_size) + 1;
      } else {
         cfg.wls_instances = MALI_LOCAL_STORAGE_NO_WORKGROUP_MEM;
      }
   }
}

#if PAN_ARCH <= 5
static void
pan_emit_midgard_tiler(const struct pan_fb_info *fb,
                       const struct pan_tiler_context *tiler_ctx,
                       struct mali_tiler_context_packed *out)
{
   bool hierarchy = !tiler_ctx->midgard.no_hierarchical_tiling;

   assert(tiler_ctx->midgard.polygon_list);

   pan_pack(out, TILER_CONTEXT, cfg) {
      unsigned header_size;

      if (tiler_ctx->midgard.disable) {
         cfg.hierarchy_mask =
            hierarchy ? MALI_MIDGARD_TILER_DISABLED : MALI_MIDGARD_TILER_USER;
         header_size = MALI_MIDGARD_TILER_MINIMUM_HEADER_SIZE;
         cfg.polygon_list_size = header_size + (hierarchy ? 0 : 4);
         cfg.heap_start = tiler_ctx->midgard.polygon_list;
         cfg.heap_end = tiler_ctx->midgard.polygon_list;
      } else {
         cfg.hierarchy_mask = panfrost_choose_hierarchy_mask(
            fb->width, fb->height, tiler_ctx->midgard.vertex_count, hierarchy);
         header_size = panfrost_tiler_header_size(
            fb->width, fb->height, cfg.hierarchy_mask, hierarchy);
         cfg.polygon_list_size = panfrost_tiler_full_size(
            fb->width, fb->height, cfg.hierarchy_mask, hierarchy);
         cfg.heap_start = tiler_ctx->midgard.heap.start;
         cfg.heap_end = cfg.heap_start + tiler_ctx->midgard.heap.size;
      }

      cfg.polygon_list = tiler_ctx->midgard.polygon_list;
      cfg.polygon_list_body = cfg.polygon_list + header_size;
   }
}
#endif

#if PAN_ARCH >= 5
static void
pan_emit_rt(const struct pan_fb_info *fb, unsigned layer_idx, unsigned idx,
            unsigned cbuf_offset, struct mali_render_target_packed *out)
{
   pan_pack(out, RENDER_TARGET, cfg) {
      pan_prepare_rt(fb, layer_idx, idx, cbuf_offset, &cfg);
   }
}

#if PAN_ARCH >= 6
/* All Bifrost and Valhall GPUs are affected by issue TSIX-2033:
 *
 *      Forcing clean_tile_writes breaks INTERSECT readbacks
 *
 * To workaround, use the frame shader mode ALWAYS instead of INTERSECT if
 * clean tile writes is forced. Since INTERSECT is a hint that the hardware may
 * ignore, this cannot affect correctness, only performance */

static enum mali_pre_post_frame_shader_mode
pan_fix_frame_shader_mode(enum mali_pre_post_frame_shader_mode mode,
                          bool force_clean_tile)
{
   if (force_clean_tile && mode == MALI_PRE_POST_FRAME_SHADER_MODE_INTERSECT)
      return MALI_PRE_POST_FRAME_SHADER_MODE_ALWAYS;
   else
      return mode;
}

/* Regardless of clean_tile_write_enable, the hardware writes clean tiles if
 * the effective tile size differs from the superblock size of any enabled AFBC
 * render target. Check this condition. */

static bool
pan_force_clean_write_on(const struct pan_image *image, unsigned tile_size)
{
   if (!image)
      return false;

   if (!drm_is_afbc(image->layout.modifier))
      return false;

   struct pan_block_size renderblk_sz =
      panfrost_afbc_renderblock_size(image->layout.modifier);

   assert(renderblk_sz.width >= 16 && renderblk_sz.height >= 16);
   assert(tile_size <= panfrost_max_effective_tile_size(PAN_ARCH));

   return tile_size != renderblk_sz.width * renderblk_sz.height;
}

static bool
pan_force_clean_write(const struct pan_fb_info *fb, unsigned tile_size)
{
   /* Maximum tile size */
   assert(tile_size <= panfrost_max_effective_tile_size(PAN_ARCH));

   for (unsigned i = 0; i < fb->rt_count; ++i) {
      if (!fb->rts[i].view || fb->rts[i].discard)
         continue;

      const struct pan_image *img =
         pan_image_view_get_color_plane(fb->rts[i].view);

      if (pan_force_clean_write_on(img, tile_size))
         return true;
   }

   if (fb->zs.view.zs && !fb->zs.discard.z &&
       pan_force_clean_write_on(pan_image_view_get_zs_plane(fb->zs.view.zs),
                                tile_size))
      return true;

   if (fb->zs.view.s && !fb->zs.discard.s &&
       pan_force_clean_write_on(pan_image_view_get_s_plane(fb->zs.view.s),
                                tile_size))
      return true;

   return false;
}

#endif

unsigned
GENX(pan_emit_fbd)(const struct pan_fb_info *fb, unsigned layer_idx,
                   const struct pan_tls_info *tls,
                   const struct pan_tiler_context *tiler_ctx, void *out)
{
   void *fbd = out;
   void *rtd = out + pan_size(FRAMEBUFFER);

#if PAN_ARCH <= 5
   GENX(pan_emit_tls)(tls, pan_section_ptr(fbd, FRAMEBUFFER, LOCAL_STORAGE));
#endif

   int crc_rt = GENX(pan_select_crc_rt)(fb, fb->tile_size);
   bool has_zs_crc_ext = (fb->zs.view.zs || fb->zs.view.s || crc_rt >= 0);

   pan_section_pack(fbd, FRAMEBUFFER, PARAMETERS, cfg) {
#if PAN_ARCH >= 6
      bool force_clean_write = pan_force_clean_write(fb, fb->tile_size);

      cfg.sample_locations = fb->sample_positions;
      cfg.pre_frame_0 = pan_fix_frame_shader_mode(fb->bifrost.pre_post.modes[0],
                                                  force_clean_write);
      cfg.pre_frame_1 = pan_fix_frame_shader_mode(fb->bifrost.pre_post.modes[1],
                                                  force_clean_write);
      cfg.post_frame = pan_fix_frame_shader_mode(fb->bifrost.pre_post.modes[2],
                                                 force_clean_write);
#if PAN_ARCH <= 7
      /* On Bifrost, the layer_id is passed through a push_uniform, which forces
       * us to have one pre/post DCD array per layer. */
      cfg.frame_shader_dcds =
         fb->bifrost.pre_post.dcds.gpu + (layer_idx * 3 * pan_size(DRAW));
#else
      /* On Valhall, layer_id is passed through the framebuffer frame_arg, which
       * is preloaded in r62, so we can use the same pre/post DCD array for all
       * layers. */
      cfg.frame_shader_dcds = fb->bifrost.pre_post.dcds.gpu;
#endif
      cfg.tiler =
         PAN_ARCH >= 9 ? tiler_ctx->valhall.desc : tiler_ctx->bifrost.desc;
#endif
      cfg.width = fb->width;
      cfg.height = fb->height;
      cfg.bound_max_x = fb->width - 1;
      cfg.bound_max_y = fb->height - 1;

      cfg.effective_tile_size = fb->tile_size;
      cfg.tie_break_rule = MALI_TIE_BREAK_RULE_MINUS_180_IN_0_OUT;
      cfg.render_target_count = MAX2(fb->rt_count, 1);

      /* Default to 24 bit depth if there's no surface. */
      cfg.z_internal_format =
         fb->zs.view.zs ? panfrost_get_z_internal_format(fb->zs.view.zs->format)
                        : MALI_Z_INTERNAL_FORMAT_D24;

      cfg.z_clear = fb->zs.clear_value.depth;
      cfg.s_clear = fb->zs.clear_value.stencil;
      cfg.color_buffer_allocation = fb->cbuf_allocation;

      /* The force_samples setting dictates the sample-count that is used
       * for rasterization, and works like D3D11's ForcedSampleCount feature:
       *
       * - If force_samples == 0: Let nr_samples dictate sample count
       * - If force_samples == 1: force single-sampled rasterization
       * - If force_samples >= 1: force multi-sampled rasterization
       *
       * This can be used to read SYSTEM_VALUE_SAMPLE_MASK_IN from the
       * fragment shader, even when performing single-sampled rendering.
       */
      if (!fb->force_samples) {
         cfg.sample_count = fb->nr_samples;
         cfg.sample_pattern = pan_sample_pattern(fb->nr_samples);
      } else if (fb->force_samples == 1) {
         cfg.sample_count = fb->nr_samples;
         cfg.sample_pattern = pan_sample_pattern(1);
      } else {
         cfg.sample_count = 1;
         cfg.sample_pattern = pan_sample_pattern(fb->force_samples);
      }

      cfg.z_write_enable = (fb->zs.view.zs && !fb->zs.discard.z);
      cfg.s_write_enable = (fb->zs.view.s && !fb->zs.discard.s);
      cfg.has_zs_crc_extension = has_zs_crc_ext;

      if (crc_rt >= 0) {
         bool *valid = fb->rts[crc_rt].crc_valid;
         bool full = !fb->extent.minx && !fb->extent.miny &&
                     fb->extent.maxx == (fb->width - 1) &&
                     fb->extent.maxy == (fb->height - 1);
         bool clean_tile_write = fb->rts[crc_rt].clear;

#if PAN_ARCH >= 6
         clean_tile_write |= pan_force_clean_write_on(
            pan_image_view_get_color_plane(fb->rts[crc_rt].view),
            fb->tile_size);
#endif

         /* If the CRC was valid it stays valid, if it wasn't, we must ensure
          * the render operation covers the full frame, and clean tiles are
          * pushed to memory. */
         bool new_valid = *valid | (full && clean_tile_write);

         cfg.crc_read_enable = *valid;

         /* If the data is currently invalid, still write CRC
          * data if we are doing a full write, so that it is
          * valid for next time. */
         cfg.crc_write_enable = new_valid;

         *valid = new_valid;
      }

#if PAN_ARCH >= 9
      cfg.point_sprite_coord_origin_max_y = fb->sprite_coord_origin;
      cfg.first_provoking_vertex = fb->first_provoking_vertex;

      /* internal_layer_index is used to select the right primitive list in the
       * tiler context, and frame_arg is the value that's passed to the fragment
       * shader through r62-r63, which we use to pass gl_Layer. Since the
       * layer_idx only takes 8-bits, we might use the extra 56-bits we have
       * in frame_argument to pass other information to the fragment shader at
       * some point. */
      assert(layer_idx >= tiler_ctx->valhall.layer_offset);
      cfg.internal_layer_index = layer_idx - tiler_ctx->valhall.layer_offset;
      cfg.frame_argument = layer_idx;
#endif
   }

#if PAN_ARCH >= 6
   pan_section_pack(fbd, FRAMEBUFFER, PADDING, padding)
      ;
#else
   pan_emit_midgard_tiler(fb, tiler_ctx,
                          pan_section_ptr(fbd, FRAMEBUFFER, TILER));

   /* All weights set to 0, nothing to do here */
   pan_section_pack(fbd, FRAMEBUFFER, TILER_WEIGHTS, w)
      ;
#endif

   if (has_zs_crc_ext) {
      struct mali_zs_crc_extension_packed *zs_crc_ext =
         out + pan_size(FRAMEBUFFER);

      pan_emit_zs_crc_ext(fb, layer_idx, crc_rt, zs_crc_ext);
      rtd += pan_size(ZS_CRC_EXTENSION);
   }

   unsigned rt_count = MAX2(fb->rt_count, 1);
   unsigned cbuf_offset = 0;
   for (unsigned i = 0; i < rt_count; i++) {
      pan_emit_rt(fb, layer_idx, i, cbuf_offset, rtd);
      rtd += pan_size(RENDER_TARGET);
      if (!fb->rts[i].view)
         continue;

      cbuf_offset += pan_bytes_per_pixel_tib(fb->rts[i].view->format) *
                     fb->tile_size * pan_image_view_get_nr_samples(fb->rts[i].view);

      if (i != crc_rt)
         *(fb->rts[i].crc_valid) = false;
   }

   struct mali_framebuffer_pointer_packed tag;
   pan_pack(&tag, FRAMEBUFFER_POINTER, cfg) {
      cfg.zs_crc_extension_present = has_zs_crc_ext;
      cfg.render_target_count = MAX2(fb->rt_count, 1);
   }
   return tag.opaque[0];
}
#else /* PAN_ARCH == 4 */
static enum mali_color_format
pan_sfbd_raw_format(unsigned bits)
{
   /* clang-format off */
   switch (bits) {
   case   16: return MALI_COLOR_FORMAT_1_16B_CHANNEL;
   case   32: return MALI_COLOR_FORMAT_1_32B_CHANNEL;
   case   48: return MALI_COLOR_FORMAT_3_16B_CHANNELS;
   case   64: return MALI_COLOR_FORMAT_2_32B_CHANNELS;
   case   96: return MALI_COLOR_FORMAT_3_32B_CHANNELS;
   case  128: return MALI_COLOR_FORMAT_4_32B_CHANNELS;
   default: unreachable("invalid raw bpp");
   }
   /* clang-format on */
}

void
GENX(pan_select_tile_size)(struct pan_fb_info *fb)
{
   /* Tile size and color buffer allocation are not configurable on gen 4 */
   fb->tile_size = 16 * 16;
}

unsigned
GENX(pan_emit_fbd)(const struct pan_fb_info *fb, unsigned layer_idx,
                   const struct pan_tls_info *tls,
                   const struct pan_tiler_context *tiler_ctx, void *fbd)
{
   assert(fb->rt_count <= 1);

   GENX(pan_emit_tls)(tls, pan_section_ptr(fbd, FRAMEBUFFER, LOCAL_STORAGE));
   pan_section_pack(fbd, FRAMEBUFFER, PARAMETERS, cfg) {
      cfg.bound_max_x = fb->width - 1;
      cfg.bound_max_y = fb->height - 1;
      cfg.dithering_enable = true;
      cfg.clean_pixel_write_enable = true;
      cfg.tie_break_rule = MALI_TIE_BREAK_RULE_MINUS_180_IN_0_OUT;
      if (fb->rts[0].clear) {
         cfg.clear_color_0 = fb->rts[0].clear_value[0];
         cfg.clear_color_1 = fb->rts[0].clear_value[1];
         cfg.clear_color_2 = fb->rts[0].clear_value[2];
         cfg.clear_color_3 = fb->rts[0].clear_value[3];
      }

      if (fb->zs.clear.z)
         cfg.z_clear = fb->zs.clear_value.depth;

      if (fb->zs.clear.s)
         cfg.s_clear = fb->zs.clear_value.stencil;

      if (fb->rt_count && fb->rts[0].view) {
         const struct pan_image_view *rt = fb->rts[0].view;
         const struct pan_image *image = pan_image_view_get_color_plane(rt);

         const struct util_format_description *desc =
            util_format_description(rt->format);

         /* The swizzle for rendering is inverted from texturing */
         unsigned char swizzle[4];
         panfrost_invert_swizzle(desc->swizzle, swizzle);
         cfg.swizzle = panfrost_translate_swizzle_4(swizzle);

         struct pan_blendable_format fmt =
            *GENX(panfrost_blendable_format_from_pipe_format)(rt->format);

         if (fmt.internal) {
            cfg.internal_format = fmt.internal;
            cfg.color_writeback_format = fmt.writeback;
         } else {
            /* Construct RAW internal/writeback */
            unsigned bits = desc->block.bits;

            cfg.internal_format = MALI_COLOR_BUFFER_INTERNAL_FORMAT_RAW_VALUE;
            cfg.color_writeback_format = pan_sfbd_raw_format(bits);
         }

         unsigned level = rt->first_level;
         struct pan_surface surf;

         pan_iview_get_surface(rt, 0, 0, 0, &surf);

         cfg.color_write_enable = !fb->rts[0].discard;
         cfg.color_writeback.base = surf.data;
         cfg.color_writeback.row_stride =
            image->layout.slices[level].row_stride;

         cfg.color_block_format = mod_to_block_fmt(image->layout.modifier);
         assert(cfg.color_block_format == MALI_BLOCK_FORMAT_LINEAR ||
                cfg.color_block_format ==
                   MALI_BLOCK_FORMAT_TILED_U_INTERLEAVED);

         if (pan_image_view_has_crc(rt)) {
            const struct pan_image_slice_layout *slice =
               &image->layout.slices[level];

            cfg.crc_buffer.row_stride = slice->crc.stride;
            cfg.crc_buffer.base =
               image->data.base + image->data.offset + slice->crc.offset;
         }
      }

      if (fb->zs.view.zs) {
         const struct pan_image_view *zs = fb->zs.view.zs;
         const struct pan_image *image = pan_image_view_get_zs_plane(zs);
         unsigned level = zs->first_level;
         struct pan_surface surf;

         pan_iview_get_surface(zs, 0, 0, 0, &surf);

         cfg.zs_write_enable = !fb->zs.discard.z;
         cfg.zs_writeback.base = surf.data;
         cfg.zs_writeback.row_stride = image->layout.slices[level].row_stride;
         cfg.zs_block_format = mod_to_block_fmt(image->layout.modifier);
         assert(cfg.zs_block_format == MALI_BLOCK_FORMAT_LINEAR ||
                cfg.zs_block_format == MALI_BLOCK_FORMAT_TILED_U_INTERLEAVED);

         cfg.zs_format = translate_zs_format(zs->format);
      }

      cfg.sample_count = fb->nr_samples;

      if (fb->rt_count)
         cfg.msaa = mali_sampling_mode(fb->rts[0].view);
   }

   pan_emit_midgard_tiler(fb, tiler_ctx,
                          pan_section_ptr(fbd, FRAMEBUFFER, TILER));

   /* All weights set to 0, nothing to do here */
   pan_section_pack(fbd, FRAMEBUFFER, TILER_WEIGHTS, w)
      ;

   pan_section_pack(fbd, FRAMEBUFFER, PADDING_1, padding)
      ;
   pan_section_pack(fbd, FRAMEBUFFER, PADDING_2, padding)
      ;
   return 0;
}
#endif

#if PAN_ARCH <= 9
void
GENX(pan_emit_fragment_job_payload)(const struct pan_fb_info *fb, uint64_t fbd,
                                    void *out)
{
   pan_section_pack(out, FRAGMENT_JOB, PAYLOAD, payload) {
      payload.bound_min_x = fb->extent.minx >> MALI_TILE_SHIFT;
      payload.bound_min_y = fb->extent.miny >> MALI_TILE_SHIFT;
      payload.bound_max_x = fb->extent.maxx >> MALI_TILE_SHIFT;
      payload.bound_max_y = fb->extent.maxy >> MALI_TILE_SHIFT;
      payload.framebuffer = fbd;

#if PAN_ARCH >= 5
      if (fb->tile_map.base) {
         payload.has_tile_enable_map = true;
         payload.tile_enable_map = fb->tile_map.base;
         payload.tile_enable_map_row_stride = fb->tile_map.stride;
      }
#endif
   }
}
#endif