xref: /external/mesa3d/src/freedreno/vulkan/tu_clear_blit.cc

/*
 * Copyright 2019-2020 Valve Corporation
 * SPDX-License-Identifier: MIT
 *
 * Authors:
 *    Jonathan Marek <jonathan@marek.ca>
 */

#include "tu_clear_blit.h"

#include "ir3/ir3_nir.h"

#include "util/format_r11g11b10f.h"
#include "util/format_rgb9e5.h"
#include "util/format_srgb.h"
#include "util/half_float.h"
#include "compiler/nir/nir_builder.h"

#include "tu_buffer.h"
#include "tu_cmd_buffer.h"
#include "tu_cs.h"
#include "tu_formats.h"
#include "tu_image.h"
#include "tu_tracepoints.h"
#include "tu_lrz.h"

#include "common/freedreno_gpu_event.h"
#include "common/freedreno_lrz.h"

static const VkOffset2D blt_no_coord = { ~0, ~0 };

static uint32_t
tu_pack_float32_for_unorm(float val, int bits)
{
   return _mesa_lroundevenf(CLAMP(val, 0.0f, 1.0f) * (float) ((1 << bits) - 1));
}

/* r2d_ = BLIT_OP_SCALE operations */

static enum a6xx_2d_ifmt
format_to_ifmt(enum pipe_format format)
{
   if (format == PIPE_FORMAT_Z24_UNORM_S8_UINT ||
       format == PIPE_FORMAT_Z24X8_UNORM)
      return R2D_UNORM8;

   /* get_component_bits doesn't work with depth/stencil formats: */
   if (format == PIPE_FORMAT_Z16_UNORM || format == PIPE_FORMAT_Z32_FLOAT)
      return R2D_FLOAT32;
   if (format == PIPE_FORMAT_S8_UINT)
      return R2D_INT8;
   if (format == PIPE_FORMAT_A8_UNORM)
      return R2D_UNORM8;

   /* use the size of the red channel to find the corresponding "ifmt" */
   bool is_int = util_format_is_pure_integer(format);
   switch (util_format_get_component_bits(format, UTIL_FORMAT_COLORSPACE_RGB, PIPE_SWIZZLE_X)) {
   case 4: case 5: case 8:
      return is_int ? R2D_INT8 : R2D_UNORM8;
   case 10: case 11:
      return is_int ? R2D_INT16 : R2D_FLOAT16;
   case 16:
      if (util_format_is_float(format))
         return R2D_FLOAT16;
      return is_int ? R2D_INT16 : R2D_FLOAT32;
   case 32:
      return is_int ? R2D_INT32 : R2D_FLOAT32;
    default:
      unreachable("bad format");
   }
}

template <chip CHIP>
static struct tu_native_format
blit_format_texture(enum pipe_format format, enum a6xx_tile_mode tile_mode, bool is_mutable, bool gmem)
{
   struct tu_native_format fmt = tu6_format_texture(format, tile_mode, is_mutable);

   switch (format) {
   case PIPE_FORMAT_Z24X8_UNORM:
   case PIPE_FORMAT_Z24_UNORM_S8_UINT:
      /* Similar to in fdl6_view_init, we want to use
       * FMT6_Z24_UNORM_S8_UINT_AS_R8G8B8A8 or FMT6_8_8_8_8_UNORM for blit
       * src.  Since this is called when there is no image and thus no ubwc,
       * we can always use FMT6_8_8_8_8_UNORM.
       *
       * Note (A7XX): Since it's erroneous to use FMT6_8_8_8_8_UNORM for a GMEM
       * image (see blit_base_format), we use FMT6_Z24_UNORM_S8_UINT_AS_R8G8B8A8
       * instead.
       */
      fmt.fmt = CHIP >= A7XX && gmem ? FMT6_Z24_UNORM_S8_UINT_AS_R8G8B8A8 : FMT6_8_8_8_8_UNORM;
      break;
   default:
      break;
   }

   return fmt;
}

static struct tu_native_format
blit_format_color(enum pipe_format format, enum a6xx_tile_mode tile_mode)
{
   struct tu_native_format fmt = tu6_format_color(format, tile_mode, false);

   switch (format) {
   case PIPE_FORMAT_Z24X8_UNORM:
   case PIPE_FORMAT_Z24_UNORM_S8_UINT:
      /* similar to blit_format_texture but for blit dst */
      fmt.fmt = FMT6_8_8_8_8_UNORM;
      break;
   default:
      break;
   }

   return fmt;
}

template <chip CHIP>
static enum a6xx_format
blit_base_format(enum pipe_format format, bool ubwc, bool gmem)
{
   if (CHIP >= A7XX && gmem)
      /* A7XX requires D24S8 in GMEM to always be treated as
       * FMT6_Z24_UNORM_S8_UINT_AS_R8G8B8A8 regardless of if the image
       * is UBWC-compatible. Using FMT6_8_8_8_8_UNORM instead will result
       * in misrendering around the edges of the destination image.
       */
      ubwc = true;

   if (ubwc) {
      switch (format) {
      case PIPE_FORMAT_Z24X8_UNORM:
      case PIPE_FORMAT_Z24_UNORM_S8_UINT:
         /* use the ubwc-compatible FMT6_Z24_UNORM_S8_UINT_AS_R8G8B8A8 */
         return FMT6_Z24_UNORM_S8_UINT_AS_R8G8B8A8;
      default:
         break;
      }
   }

   /* note: tu6_format_color doesn't care about tiling for .fmt field */
   return blit_format_color(format, TILE6_LINEAR).fmt;
}

static void
r2d_coords(struct tu_cmd_buffer *cmd,
           struct tu_cs *cs,
           const VkOffset2D dst,
           const VkOffset2D src,
           const VkExtent2D extent)
{
   tu_cs_emit_regs(cs,
      A6XX_GRAS_2D_DST_TL(.x = dst.x,                    .y = dst.y),
      A6XX_GRAS_2D_DST_BR(.x = dst.x + extent.width - 1, .y = dst.y + extent.height - 1));

   if (src.x == blt_no_coord.x)
      return;

   tu_cs_emit_regs(cs,
                   A6XX_GRAS_2D_SRC_TL_X(src.x),
                   A6XX_GRAS_2D_SRC_BR_X(src.x + extent.width - 1),
                   A6XX_GRAS_2D_SRC_TL_Y(src.y),
                   A6XX_GRAS_2D_SRC_BR_Y(src.y + extent.height - 1));
}

static void
r2d_clear_value(struct tu_cmd_buffer *cmd,
                struct tu_cs *cs,
                enum pipe_format format,
                const VkClearValue *val)
{
   uint32_t clear_value[4] = {};

   switch (format) {
   case PIPE_FORMAT_Z24_UNORM_S8_UINT:
   case PIPE_FORMAT_Z24X8_UNORM:
      /* cleared as r8g8b8a8_unorm using special format */
      clear_value[0] = tu_pack_float32_for_unorm(val->depthStencil.depth, 24);
      clear_value[1] = clear_value[0] >> 8;
      clear_value[2] = clear_value[0] >> 16;
      clear_value[3] = val->depthStencil.stencil;
      break;
   case PIPE_FORMAT_Z16_UNORM:
   case PIPE_FORMAT_Z32_FLOAT:
      /* R2D_FLOAT32 */
      clear_value[0] = fui(val->depthStencil.depth);
      break;
   case PIPE_FORMAT_S8_UINT:
      clear_value[0] = val->depthStencil.stencil;
      break;
   case PIPE_FORMAT_R9G9B9E5_FLOAT:
      /* cleared as UINT32 */
      clear_value[0] = float3_to_rgb9e5(val->color.float32);
      break;
   default:
      assert(!util_format_is_depth_or_stencil(format));
      const struct util_format_description *desc = util_format_description(format);
      enum a6xx_2d_ifmt ifmt = format_to_ifmt(format);

      assert(desc->layout == UTIL_FORMAT_LAYOUT_PLAIN ||
             format == PIPE_FORMAT_R11G11B10_FLOAT);

      for (unsigned i = 0; i < 4; i++) {
         if (desc->swizzle[i] > PIPE_SWIZZLE_W)
            continue;

         const struct util_format_channel_description *ch =
            &desc->channel[desc->swizzle[i]];
         if (ifmt == R2D_UNORM8) {
            float linear = val->color.float32[i];
            if (desc->colorspace == UTIL_FORMAT_COLORSPACE_SRGB && i < 3)
               linear = util_format_linear_to_srgb_float(val->color.float32[i]);

            if (ch->type == UTIL_FORMAT_TYPE_SIGNED)
               clear_value[i] = _mesa_lroundevenf(CLAMP(linear, -1.0f, 1.0f) * 127.0f);
            else
               clear_value[i] = tu_pack_float32_for_unorm(linear, 8);
         } else if (ifmt == R2D_FLOAT16) {
            clear_value[i] = _mesa_float_to_half(val->color.float32[i]);
         } else {
            assert(ifmt == R2D_FLOAT32 || ifmt == R2D_INT32 ||
                   ifmt == R2D_INT16 || ifmt == R2D_INT8);
            clear_value[i] = val->color.uint32[i];
         }
      }
      break;
   }

   tu_cs_emit_pkt4(cs, REG_A6XX_RB_2D_SRC_SOLID_C0, 4);
   tu_cs_emit_array(cs, clear_value, 4);
}

static void
fixup_src_format(enum pipe_format *src_format, enum pipe_format dst_format,
                 enum a6xx_format *fmt)
{
   /* When blitting S8 -> D24S8 or vice versa, we have to override S8, which
    * is normally R8_UINT for sampling/blitting purposes, to a unorm format.
    * We also have to move stencil, which is normally in the .w channel, into
    * the right channel. Reintepreting the S8 texture as A8_UNORM solves both
    * problems, and avoids using a swap, which seems to sometimes not work
    * with a D24S8 source, or a texture swizzle which is only supported with
    * the 3d path. Sometimes this blit happens on already-constructed
    * fdl6_view's, e.g. for sysmem resolves, so this has to happen as a fixup.
    */
   if (*src_format == PIPE_FORMAT_S8_UINT &&
       (dst_format == PIPE_FORMAT_Z24_UNORM_S8_UINT ||
        dst_format == PIPE_FORMAT_Z24_UNORM_S8_UINT_AS_R8G8B8A8)) {
      *fmt = FMT6_A8_UNORM;
      *src_format = PIPE_FORMAT_A8_UNORM;
   }
}

static void
fixup_dst_format(enum pipe_format src_format, enum pipe_format *dst_format,
                 enum a6xx_format *fmt)
{
   if (*dst_format == PIPE_FORMAT_S8_UINT &&
       (src_format == PIPE_FORMAT_Z24_UNORM_S8_UINT ||
        src_format == PIPE_FORMAT_Z24_UNORM_S8_UINT_AS_R8G8B8A8)) {
      *dst_format = PIPE_FORMAT_A8_UNORM;
      *fmt = FMT6_A8_UNORM;
   }
}

template <chip CHIP>
static void
r2d_src(struct tu_cmd_buffer *cmd,
        struct tu_cs *cs,
        const struct fdl6_view *iview,
        uint32_t layer,
        VkFilter filter,
        enum pipe_format dst_format)
{
   uint32_t src_info = iview->SP_PS_2D_SRC_INFO;
   if (filter != VK_FILTER_NEAREST)
      src_info |= A6XX_SP_PS_2D_SRC_INFO_FILTER;

   enum a6xx_format fmt = (enum a6xx_format)(
      src_info & A6XX_SP_PS_2D_SRC_INFO_COLOR_FORMAT__MASK);
   enum pipe_format src_format = iview->format;
   fixup_src_format(&src_format, dst_format, &fmt);

   src_info =
      (src_info & ~A6XX_SP_PS_2D_SRC_INFO_COLOR_FORMAT__MASK) |
      A6XX_SP_PS_2D_SRC_INFO_COLOR_FORMAT(fmt);

   tu_cs_emit_pkt4(cs, SP_PS_2D_SRC_INFO(CHIP,).reg, 5);
   tu_cs_emit(cs, src_info);
   tu_cs_emit(cs, iview->SP_PS_2D_SRC_SIZE);
   tu_cs_image_ref_2d<CHIP>(cs, iview, layer, true);

   tu_cs_emit_pkt4(cs, __SP_PS_2D_SRC_FLAGS<CHIP>({}).reg, 3);
   tu_cs_image_flag_ref(cs, iview, layer);
}

template <chip CHIP>
static void
r2d_src_depth(struct tu_cmd_buffer *cmd,
                struct tu_cs *cs,
                const struct tu_image_view *iview,
                uint32_t layer,
                VkFilter filter)
{
   tu_cs_emit_pkt4(cs, SP_PS_2D_SRC_INFO(CHIP).reg, 5);
   tu_cs_emit(cs, tu_image_view_depth(iview, SP_PS_2D_SRC_INFO));
   tu_cs_emit(cs, iview->view.SP_PS_2D_SRC_SIZE);
   tu_cs_emit_qw(cs, iview->depth_base_addr + iview->depth_layer_size * layer);
   /* SP_PS_2D_SRC_PITCH has shifted pitch field */
   tu_cs_emit(cs, SP_PS_2D_SRC_PITCH(CHIP, .pitch = iview->depth_pitch).value);

   tu_cs_emit_pkt4(cs, __SP_PS_2D_SRC_FLAGS<CHIP>({}).reg, 3);
   tu_cs_image_flag_ref(cs, &iview->view, layer);
}

template <chip CHIP>
static void
r2d_src_stencil(struct tu_cmd_buffer *cmd,
                struct tu_cs *cs,
                const struct tu_image_view *iview,
                uint32_t layer,
                VkFilter filter)
{
   tu_cs_emit_pkt4(cs, SP_PS_2D_SRC_INFO(CHIP,).reg, 5);
   tu_cs_emit(cs, tu_image_view_stencil(iview, SP_PS_2D_SRC_INFO) & ~A6XX_SP_PS_2D_SRC_INFO_FLAGS);
   tu_cs_emit(cs, iview->view.SP_PS_2D_SRC_SIZE);
   tu_cs_emit_qw(cs, iview->stencil_base_addr + iview->stencil_layer_size * layer);
   tu_cs_emit(cs, SP_PS_2D_SRC_PITCH(CHIP, .pitch = iview->stencil_pitch).value);
}

template <chip CHIP>
static void
r2d_src_buffer(struct tu_cmd_buffer *cmd,
               struct tu_cs *cs,
               enum pipe_format format,
               uint64_t va, uint32_t pitch,
               uint32_t width, uint32_t height,
               enum pipe_format dst_format)
{
   struct tu_native_format fmt = blit_format_texture<CHIP>(format, TILE6_LINEAR, false, false);
   enum a6xx_format color_format = fmt.fmt;
   fixup_src_format(&format, dst_format, &color_format);

   tu_cs_emit_regs(cs,
                   SP_PS_2D_SRC_INFO(CHIP,
                      .color_format = color_format,
                      .color_swap = fmt.swap,
                      .srgb = util_format_is_srgb(format),
                      .unk20 = 1,
                      .unk22 = 1),
                   SP_PS_2D_SRC_SIZE(CHIP, .width = width, .height = height),
                   SP_PS_2D_SRC(CHIP, .qword = va),
                   SP_PS_2D_SRC_PITCH(CHIP, .pitch = pitch));
}

template <chip CHIP>
static void
r2d_src_buffer_unaligned(struct tu_cmd_buffer *cmd,
                         struct tu_cs *cs,
                         enum pipe_format format,
                         uint64_t va,
                         uint32_t pitch,
                         uint32_t width,
                         uint32_t height,
                         enum pipe_format dst_format)
{
   /* This functionality is only allowed on A7XX, this assertion statically
    * disallows calling this function on prior generations by mistake.
    */
   static_assert(CHIP >= A7XX);

   struct tu_native_format fmt =
      blit_format_texture<CHIP>(format, TILE6_LINEAR, false, false);
   enum a6xx_format color_format = fmt.fmt;
   fixup_src_format(&format, dst_format, &color_format);

   uint32_t offset_texels = ((va & 0x3f) / util_format_get_blocksize(format));
   va &= ~0x3f;
   tu_cs_emit_regs(cs,
                   A7XX_TPL1_2D_SRC_CNTL(.raw_copy = false,
                                         .start_offset_texels = offset_texels,
                                         .type = A6XX_TEX_IMG_BUFFER));

   tu_cs_emit_regs(cs,
                   SP_PS_2D_SRC_INFO(CHIP, .color_format = color_format,
                                     .color_swap = fmt.swap,
                                     .srgb = util_format_is_srgb(format),
                                     .unk20 = 1, .unk22 = 1),
                   SP_PS_2D_SRC_SIZE(CHIP, .width = width, .height = height),
                   SP_PS_2D_SRC(CHIP, .qword = va),
                   SP_PS_2D_SRC_PITCH(CHIP, .pitch = pitch));
}

template <chip CHIP>
static void
r2d_dst(struct tu_cs *cs, const struct fdl6_view *iview, uint32_t layer,
        enum pipe_format src_format)
{
   uint32_t dst_info = iview->RB_2D_DST_INFO;
   enum a6xx_format fmt =
      (enum a6xx_format)(dst_info & A6XX_RB_2D_DST_INFO_COLOR_FORMAT__MASK);
   enum pipe_format dst_format = iview->format;
   fixup_dst_format(src_format, &dst_format, &fmt);

   dst_info =
         (dst_info & ~A6XX_RB_2D_DST_INFO_COLOR_FORMAT__MASK) | fmt;
   tu_cs_emit_pkt4(cs, REG_A6XX_RB_2D_DST_INFO, 4);
   tu_cs_emit(cs, dst_info);
   tu_cs_image_ref_2d<CHIP>(cs, iview, layer, false);

   tu_cs_emit_pkt4(cs, REG_A6XX_RB_2D_DST_FLAGS, 3);
   tu_cs_image_flag_ref(cs, iview, layer);
}

static void
r2d_dst_depth(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t layer)
{
   tu_cs_emit_pkt4(cs, REG_A6XX_RB_2D_DST_INFO, 4);
   tu_cs_emit(cs, tu_image_view_depth(iview, RB_2D_DST_INFO));
   tu_cs_emit_qw(cs, iview->depth_base_addr + iview->depth_layer_size * layer);
   tu_cs_emit(cs, A6XX_RB_2D_DST_PITCH(iview->depth_pitch).value);

   tu_cs_emit_pkt4(cs, REG_A6XX_RB_2D_DST_FLAGS, 3);
   tu_cs_image_flag_ref(cs, &iview->view, layer);
}

static void
r2d_dst_stencil(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t layer)
{
   tu_cs_emit_pkt4(cs, REG_A6XX_RB_2D_DST_INFO, 4);
   tu_cs_emit(cs, tu_image_view_stencil(iview, RB_2D_DST_INFO) & ~A6XX_RB_2D_DST_INFO_FLAGS);
   tu_cs_emit_qw(cs, iview->stencil_base_addr + iview->stencil_layer_size * layer);
   tu_cs_emit(cs, A6XX_RB_2D_DST_PITCH(iview->stencil_pitch).value);
}

static void
r2d_dst_buffer(struct tu_cs *cs, enum pipe_format format, uint64_t va, uint32_t pitch,
               enum pipe_format src_format)
{
   struct tu_native_format fmt = blit_format_color(format, TILE6_LINEAR);
   enum a6xx_format color_fmt = fmt.fmt;
   fixup_dst_format(src_format, &format, &color_fmt);
   fmt.fmt = color_fmt;

   tu_cs_emit_regs(cs,
                   A6XX_RB_2D_DST_INFO(
                      .color_format = fmt.fmt,
                      .color_swap = fmt.swap,
                      .srgb = util_format_is_srgb(format)),
                   A6XX_RB_2D_DST(.qword = va),
                   A6XX_RB_2D_DST_PITCH(pitch));
}

template <chip CHIP>
static void
r2d_setup_common(struct tu_cmd_buffer *cmd,
                 struct tu_cs *cs,
                 enum pipe_format src_format,
                 enum pipe_format dst_format,
                 VkImageAspectFlags aspect_mask,
                 unsigned blit_param,
                 bool clear,
                 bool ubwc,
                 bool scissor)
{
   if (!cmd->state.pass && cmd->device->dbg_renderpass_stomp_cs) {
      tu_cs_emit_call(cs, cmd->device->dbg_renderpass_stomp_cs);
   }

   enum a6xx_format fmt = blit_base_format<CHIP>(dst_format, ubwc, false);
   fixup_dst_format(src_format, &dst_format, &fmt);
   enum a6xx_2d_ifmt ifmt = format_to_ifmt(dst_format);

   uint32_t unknown_8c01 = 0;

   /* note: the only format with partial clearing is D24S8 */
   if (dst_format == PIPE_FORMAT_Z24_UNORM_S8_UINT) {
      /* preserve stencil channel */
      if (aspect_mask == VK_IMAGE_ASPECT_DEPTH_BIT)
         unknown_8c01 = 0x08000041;
      /* preserve depth channels */
      if (aspect_mask == VK_IMAGE_ASPECT_STENCIL_BIT)
         unknown_8c01 = 0x00084001;
   }

   tu_cs_emit_pkt4(cs, REG_A6XX_RB_2D_UNKNOWN_8C01, 1);
   tu_cs_emit(cs, unknown_8c01);    // TODO: seem to be always 0 on A7XX

   uint32_t blit_cntl = A6XX_RB_2D_BLIT_CNTL(
         .rotate = (enum a6xx_rotation) blit_param,
         .solid_color = clear,
         .color_format = fmt,
         .scissor = scissor,
         .d24s8 = fmt == FMT6_Z24_UNORM_S8_UINT_AS_R8G8B8A8 && !clear,
         .mask = 0xf,
         .ifmt = util_format_is_srgb(dst_format) ? R2D_UNORM8_SRGB : ifmt,
      ).value;

   tu_cs_emit_pkt4(cs, REG_A6XX_RB_2D_BLIT_CNTL, 1);
   tu_cs_emit(cs, blit_cntl);

   tu_cs_emit_pkt4(cs, REG_A6XX_GRAS_2D_BLIT_CNTL, 1);
   tu_cs_emit(cs, blit_cntl);

   if (CHIP > A6XX) {
      tu_cs_emit_regs(cs, A7XX_TPL1_2D_SRC_CNTL(.raw_copy = false,
                                                .start_offset_texels = 0,
                                                .type = A6XX_TEX_2D));
   }

   if (fmt == FMT6_10_10_10_2_UNORM_DEST)
      fmt = FMT6_16_16_16_16_FLOAT;

   tu_cs_emit_regs(cs, SP_2D_DST_FORMAT(CHIP,
         .sint = util_format_is_pure_sint(dst_format),
         .uint = util_format_is_pure_uint(dst_format),
         .color_format = fmt,
         .srgb = util_format_is_srgb(dst_format),
         .mask = 0xf));
}

template <chip CHIP>
static void
r2d_setup(struct tu_cmd_buffer *cmd,
          struct tu_cs *cs,
          enum pipe_format src_format,
          enum pipe_format dst_format,
          VkImageAspectFlags aspect_mask,
          unsigned blit_param,
          bool clear,
          bool ubwc,
          VkSampleCountFlagBits samples)
{
   assert(samples == VK_SAMPLE_COUNT_1_BIT);

   if (!cmd->state.pass) {
      tu_emit_cache_flush_ccu<CHIP>(cmd, cs, TU_CMD_CCU_SYSMEM);
   }

   r2d_setup_common<CHIP>(cmd, cs, src_format, dst_format, aspect_mask, blit_param, clear, ubwc, false);
}

static void
r2d_teardown(struct tu_cmd_buffer *cmd,
             struct tu_cs *cs)
{
   /* nothing to do here */
}

static void
r2d_run(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
{
   if (cmd->device->physical_device->info->a6xx.magic.RB_DBG_ECO_CNTL_blit !=
       cmd->device->physical_device->info->a6xx.magic.RB_DBG_ECO_CNTL) {
      /* This a non-context register, so we have to WFI before changing. */
      tu_cs_emit_wfi(cs);
      tu_cs_emit_write_reg(
         cs, REG_A6XX_RB_DBG_ECO_CNTL,
         cmd->device->physical_device->info->a6xx.magic.RB_DBG_ECO_CNTL_blit);
   }

   tu_cs_emit_pkt7(cs, CP_BLIT, 1);
   tu_cs_emit(cs, CP_BLIT_0_OP(BLIT_OP_SCALE));

   if (cmd->device->physical_device->info->a6xx.magic.RB_DBG_ECO_CNTL_blit !=
       cmd->device->physical_device->info->a6xx.magic.RB_DBG_ECO_CNTL) {
      tu_cs_emit_wfi(cs);
      tu_cs_emit_write_reg(
         cs, REG_A6XX_RB_DBG_ECO_CNTL,
         cmd->device->physical_device->info->a6xx.magic.RB_DBG_ECO_CNTL);
   }
}

/* r3d_ = shader path operations */

static nir_def *
load_const(nir_builder *b, unsigned base, unsigned components)
{
   return nir_load_const_ir3(b, components, 32, nir_imm_int(b, 0),
                             .base = base);
}

static nir_shader *
build_blit_vs_shader(void)
{
   nir_builder _b =
      nir_builder_init_simple_shader(MESA_SHADER_VERTEX, NULL, "blit vs");
   nir_builder *b = &_b;
   b->shader->info.internal = true;

   nir_variable *out_pos =
      nir_variable_create(b->shader, nir_var_shader_out, glsl_vec4_type(),
                          "gl_Position");
   out_pos->data.location = VARYING_SLOT_POS;

   nir_def *vert0_pos = load_const(b, 0, 2);
   nir_def *vert1_pos = load_const(b, 4, 2);
   nir_def *vertex = nir_load_vertex_id(b);

   nir_def *pos = nir_bcsel(b, nir_i2b(b, vertex), vert1_pos, vert0_pos);
   pos = nir_vec4(b, nir_channel(b, pos, 0),
                     nir_channel(b, pos, 1),
                     nir_imm_float(b, 0.0),
                     nir_imm_float(b, 1.0));

   nir_store_var(b, out_pos, pos, 0xf);

   nir_variable *out_coords =
      nir_variable_create(b->shader, nir_var_shader_out, glsl_vec_type(3),
                          "coords");
   out_coords->data.location = VARYING_SLOT_VAR0;

   nir_def *vert0_coords = load_const(b, 2, 2);
   nir_def *vert1_coords = load_const(b, 6, 2);

   /* Only used with "z scale" blit path which uses a 3d texture */
   nir_def *z_coord = load_const(b, 16, 1);

   nir_def *coords = nir_bcsel(b, nir_i2b(b, vertex), vert1_coords, vert0_coords);
   coords = nir_vec3(b, nir_channel(b, coords, 0), nir_channel(b, coords, 1),
                     z_coord);

   nir_store_var(b, out_coords, coords, 0x7);

   return b->shader;
}

static nir_shader *
build_clear_vs_shader(void)
{
   nir_builder _b =
      nir_builder_init_simple_shader(MESA_SHADER_VERTEX, NULL, "blit vs");
   nir_builder *b = &_b;
   b->shader->info.internal = true;

   nir_variable *out_pos =
      nir_variable_create(b->shader, nir_var_shader_out, glsl_vec4_type(),
                          "gl_Position");
   out_pos->data.location = VARYING_SLOT_POS;

   nir_def *vert0_pos = load_const(b, 0, 2);
   nir_def *vert1_pos = load_const(b, 4, 2);
   /* c0.z is used to clear depth */
   nir_def *depth = load_const(b, 2, 1);
   nir_def *vertex = nir_load_vertex_id(b);

   nir_def *pos = nir_bcsel(b, nir_i2b(b, vertex), vert1_pos, vert0_pos);
   pos = nir_vec4(b, nir_channel(b, pos, 0),
                     nir_channel(b, pos, 1),
                     depth, nir_imm_float(b, 1.0));

   nir_store_var(b, out_pos, pos, 0xf);

   nir_variable *out_layer =
      nir_variable_create(b->shader, nir_var_shader_out, glsl_uint_type(),
                          "gl_Layer");
   out_layer->data.location = VARYING_SLOT_LAYER;
   nir_def *layer = load_const(b, 3, 1);
   nir_store_var(b, out_layer, layer, 1);

   return b->shader;
}

static nir_shader *
build_blit_fs_shader(bool zscale)
{
   nir_builder _b =
      nir_builder_init_simple_shader(MESA_SHADER_FRAGMENT, NULL,
                                     zscale ? "zscale blit fs" : "blit fs");
   nir_builder *b = &_b;
   b->shader->info.internal = true;

   nir_variable *out_color =
      nir_variable_create(b->shader, nir_var_shader_out, glsl_vec4_type(),
                          "color0");
   out_color->data.location = FRAG_RESULT_DATA0;

   unsigned coord_components = zscale ? 3 : 2;
   nir_variable *in_coords =
      nir_variable_create(b->shader, nir_var_shader_in,
                          glsl_vec_type(coord_components),
                          "coords");
   in_coords->data.location = VARYING_SLOT_VAR0;

   nir_tex_instr *tex = nir_tex_instr_create(b->shader, 1);
   /* Note: since we're just copying data, we rely on the HW ignoring the
    * dest_type.
    */
   tex->dest_type = nir_type_int32;
   tex->is_array = false;
   tex->is_shadow = false;
   tex->sampler_dim = zscale ? GLSL_SAMPLER_DIM_3D : GLSL_SAMPLER_DIM_2D;

   tex->texture_index = 0;
   tex->sampler_index = 0;

   b->shader->info.num_textures = 1;
   BITSET_SET(b->shader->info.textures_used, 0);

   tex->src[0] = nir_tex_src_for_ssa(nir_tex_src_coord,
                                     nir_load_var(b, in_coords));
   tex->coord_components = coord_components;

   nir_def_init(&tex->instr, &tex->def, 4, 32);
   nir_builder_instr_insert(b, &tex->instr);

   nir_store_var(b, out_color, &tex->def, 0xf);

   return b->shader;
}

/* We can only read multisample textures via txf_ms, so we need a separate
 * variant for them.
 */
static nir_shader *
build_ms_copy_fs_shader(bool half_float)
{
   nir_builder _b =
      nir_builder_init_simple_shader(MESA_SHADER_FRAGMENT, NULL,
                                     "multisample copy fs");
   nir_builder *b = &_b;
   b->shader->info.internal = true;

   nir_variable *out_color =
      nir_variable_create(b->shader, nir_var_shader_out,
                          half_float ? glsl_f16vec_type(4) : glsl_vec4_type(),
                          "color0");
   out_color->data.location = FRAG_RESULT_DATA0;

   nir_variable *in_coords =
      nir_variable_create(b->shader, nir_var_shader_in,
                          glsl_vec_type(2),
                          "coords");
   in_coords->data.location = VARYING_SLOT_VAR0;

   nir_tex_instr *tex = nir_tex_instr_create(b->shader, 2);

   tex->op = nir_texop_txf_ms;

   /* Note: since we're just copying data, we rely on the HW ignoring the
    * dest_type.
    */
   tex->dest_type = half_float ? nir_type_float16 : nir_type_int32;
   tex->is_array = false;
   tex->is_shadow = false;
   tex->sampler_dim = GLSL_SAMPLER_DIM_MS;

   tex->texture_index = 0;
   tex->sampler_index = 0;

   b->shader->info.num_textures = 1;
   BITSET_SET(b->shader->info.textures_used, 0);
   BITSET_SET(b->shader->info.textures_used_by_txf, 0);

   nir_def *coord = nir_f2i32(b, nir_load_var(b, in_coords));

   tex->src[0] = nir_tex_src_for_ssa(nir_tex_src_coord, coord);
   tex->coord_components = 2;

   tex->src[1] = nir_tex_src_for_ssa(nir_tex_src_ms_index,
                                     nir_load_sample_id(b));

   nir_def_init(&tex->instr, &tex->def, 4, half_float ? 16 : 32);
   nir_builder_instr_insert(b, &tex->instr);

   nir_store_var(b, out_color, &tex->def, 0xf);

   return b->shader;
}

static nir_shader *
build_clear_fs_shader(unsigned mrts)
{
   nir_builder _b =
      nir_builder_init_simple_shader(MESA_SHADER_FRAGMENT, NULL,
                                     "mrt%u clear fs", mrts);
   nir_builder *b = &_b;
   b->shader->info.internal = true;

   for (unsigned i = 0; i < mrts; i++) {
      nir_variable *out_color =
         nir_variable_create(b->shader, nir_var_shader_out, glsl_vec4_type(),
                             "color");
      out_color->data.location = FRAG_RESULT_DATA0 + i;

      nir_def *color = load_const(b, 4 * i, 4);
      nir_store_var(b, out_color, color, 0xf);
   }

   return b->shader;
}

static void
compile_shader(struct tu_device *dev, struct nir_shader *nir,
               unsigned consts, unsigned *offset, enum global_shader idx)
{
   nir->options = ir3_get_compiler_options(dev->compiler);

   nir_assign_io_var_locations(nir, nir_var_shader_in, &nir->num_inputs, nir->info.stage);
   nir_assign_io_var_locations(nir, nir_var_shader_out, &nir->num_outputs, nir->info.stage);

   struct ir3_const_allocations const_allocs = {};
   if (consts > 0)
      ir3_const_alloc(&const_allocs, IR3_CONST_ALLOC_UBO_RANGES, align(consts, 8), 1);

   const struct ir3_shader_options options = {
      .api_wavesize = IR3_SINGLE_OR_DOUBLE,
      .real_wavesize = IR3_SINGLE_OR_DOUBLE,
      .const_allocs = const_allocs,
   };

   ir3_finalize_nir(dev->compiler, &options.nir_options, nir);

   struct ir3_shader *sh =
      ir3_shader_from_nir(dev->compiler, nir, &options, NULL);

   struct ir3_shader_key key = {};
   bool created;
   struct ir3_shader_variant *so =
      ir3_shader_get_variant(sh, &key, false, false, &created);

   struct tu6_global *global = dev->global_bo_map;

   assert(*offset + so->info.sizedwords <= ARRAY_SIZE(global->shaders));
   dev->global_shaders[idx] = sh;
   dev->global_shader_variants[idx] = so;
   memcpy(&global->shaders[*offset], so->bin,
          sizeof(uint32_t) * so->info.sizedwords);
   dev->global_shader_va[idx] = dev->global_bo->iova +
      offsetof_arr(struct tu6_global, shaders, *offset);
   *offset += align(so->info.sizedwords, 32);
}

void
tu_init_clear_blit_shaders(struct tu_device *dev)
{
   unsigned offset = 0;
   compile_shader(dev, build_blit_vs_shader(), 3, &offset, GLOBAL_SH_VS_BLIT);
   compile_shader(dev, build_clear_vs_shader(), 2, &offset, GLOBAL_SH_VS_CLEAR);
   compile_shader(dev, build_blit_fs_shader(false), 0, &offset, GLOBAL_SH_FS_BLIT);
   compile_shader(dev, build_blit_fs_shader(true), 0, &offset, GLOBAL_SH_FS_BLIT_ZSCALE);
   compile_shader(dev, build_ms_copy_fs_shader(false), 0, &offset, GLOBAL_SH_FS_COPY_MS);
   compile_shader(dev, build_ms_copy_fs_shader(true), 0, &offset, GLOBAL_SH_FS_COPY_MS_HALF);

   for (uint32_t num_rts = 0; num_rts <= MAX_RTS; num_rts++) {
      compile_shader(dev, build_clear_fs_shader(num_rts), num_rts, &offset,
                     (enum global_shader) (GLOBAL_SH_FS_CLEAR0 + num_rts));
   }
}

void
tu_destroy_clear_blit_shaders(struct tu_device *dev)
{
   for (unsigned i = 0; i < GLOBAL_SH_COUNT; i++) {
      if (dev->global_shaders[i])
         ir3_shader_destroy(dev->global_shaders[i]);
   }
}

enum r3d_type {
   R3D_CLEAR,
   R3D_BLIT,
   R3D_COPY_HALF,
};

template <chip CHIP>
static void
r3d_common(struct tu_cmd_buffer *cmd, struct tu_cs *cs, enum r3d_type type,
           uint32_t rts_mask, bool z_scale, VkSampleCountFlagBits samples)
{
   enum global_shader vs_id =
      type == R3D_CLEAR ? GLOBAL_SH_VS_CLEAR : GLOBAL_SH_VS_BLIT;

   struct ir3_shader_variant *vs = cmd->device->global_shader_variants[vs_id];
   uint64_t vs_iova = cmd->device->global_shader_va[vs_id];

   enum global_shader fs_id = GLOBAL_SH_FS_BLIT;

   if (z_scale) {
      fs_id = GLOBAL_SH_FS_BLIT_ZSCALE;
   } else if (type == R3D_COPY_HALF) {
      /* Avoid canonicalizing NaNs due to implicit conversions in the shader.
       *
       * TODO: Add a half-float blit shader that uses texture() but with half
       * registers to avoid NaN canonicaliztion for the single-sampled case.
       */
      fs_id = GLOBAL_SH_FS_COPY_MS_HALF;
   } else if (samples != VK_SAMPLE_COUNT_1_BIT) {
      fs_id = GLOBAL_SH_FS_COPY_MS;
   }

   unsigned num_rts = util_bitcount(rts_mask);
   if (type == R3D_CLEAR)
      fs_id = (enum global_shader) (GLOBAL_SH_FS_CLEAR0 + num_rts);

   struct ir3_shader_variant *fs = cmd->device->global_shader_variants[fs_id];
   uint64_t fs_iova = cmd->device->global_shader_va[fs_id];

   tu_cs_emit_regs(cs, HLSQ_INVALIDATE_CMD(CHIP,
         .vs_state = true,
         .hs_state = true,
         .ds_state = true,
         .gs_state = true,
         .fs_state = true,
         .gfx_ibo = true,
         .gfx_shared_const = true,
         .cs_bindless = CHIP == A6XX ? 0x1f : 0xff,
         .gfx_bindless = CHIP == A6XX ? 0x1f : 0xff,));

   tu6_emit_xs_config<CHIP>(cs, MESA_SHADER_VERTEX, vs);
   tu6_emit_xs_config<CHIP>(cs, MESA_SHADER_TESS_CTRL, NULL);
   tu6_emit_xs_config<CHIP>(cs, MESA_SHADER_TESS_EVAL, NULL);
   tu6_emit_xs_config<CHIP>(cs, MESA_SHADER_GEOMETRY, NULL);
   tu6_emit_xs_config<CHIP>(cs, MESA_SHADER_FRAGMENT, fs);

   struct tu_pvtmem_config pvtmem = {};
   tu6_emit_xs(cs, MESA_SHADER_VERTEX, vs, &pvtmem, vs_iova);
   tu6_emit_xs(cs, MESA_SHADER_FRAGMENT, fs, &pvtmem, fs_iova);

   tu_cs_emit_regs(cs, A6XX_PC_PRIMITIVE_CNTL_0());
   if (CHIP == A7XX) {
      tu_cs_emit_regs(cs, A7XX_VPC_PRIMITIVE_CNTL_0());
   }

   tu6_emit_vpc<CHIP>(cs, vs, NULL, NULL, NULL, fs);

   if (CHIP >= A7XX) {
      tu_cs_emit_regs(cs, A6XX_GRAS_UNKNOWN_8110(0x2));

      tu_cs_emit_regs(cs, A7XX_HLSQ_FS_UNKNOWN_A9AA(.consts_load_disable = false));
   }

   /* REPL_MODE for varying with RECTLIST (2 vertices only) */
   tu_cs_emit_regs(cs, A6XX_VPC_VARYING_INTERP_MODE(0, 0));
   tu_cs_emit_regs(cs, A6XX_VPC_VARYING_PS_REPL_MODE(0, 2 << 2 | 1 << 0));

   tu6_emit_vs<CHIP>(cs, vs, 0);
   tu6_emit_hs<CHIP>(cs, NULL);
   tu6_emit_ds<CHIP>(cs, NULL);
   tu6_emit_gs<CHIP>(cs, NULL);
   tu6_emit_fs<CHIP>(cs, fs);

   tu_cs_emit_regs(cs,
                   A6XX_GRAS_CL_CNTL(
                      .clip_disable = 1,
                      .vp_clip_code_ignore = 1,
                      .vp_xform_disable = 1,
                      .persp_division_disable = 1,));
   tu_cs_emit_regs(cs, A6XX_GRAS_SU_CNTL()); // XXX msaa enable?

   tu_cs_emit_regs(cs, PC_RASTER_CNTL(CHIP));
   if (CHIP == A6XX) {
      tu_cs_emit_regs(cs, A6XX_VPC_UNKNOWN_9107());
   } else {
      tu_cs_emit_regs(cs, A7XX_PC_RASTER_CNTL_V2());
   }

   tu_cs_emit_regs(cs,
                   A6XX_GRAS_SC_VIEWPORT_SCISSOR_TL(0, .x = 0, .y = 0),
                   A6XX_GRAS_SC_VIEWPORT_SCISSOR_BR(0, .x = 0x7fff, .y = 0x7fff));
   tu_cs_emit_regs(cs,
                   A6XX_GRAS_SC_SCREEN_SCISSOR_TL(0, .x = 0, .y = 0),
                   A6XX_GRAS_SC_SCREEN_SCISSOR_BR(0, .x = 0x7fff, .y = 0x7fff));

   tu_cs_emit_regs(cs,
                   A6XX_VFD_INDEX_OFFSET(),
                   A6XX_VFD_INSTANCE_START_OFFSET());

   if (rts_mask) {
      unsigned rts_count = util_last_bit(rts_mask);
      tu_cs_emit_pkt4(cs, REG_A6XX_SP_FS_OUTPUT_REG(0), rts_count);
      unsigned rt = 0;
      for (unsigned i = 0; i < rts_count; i++) {
         unsigned regid = 0;
         if (rts_mask & (1u << i))
            regid = ir3_find_output_regid(fs, FRAG_RESULT_DATA0 + rt++);
         tu_cs_emit(cs, A6XX_SP_FS_OUTPUT_REG_REGID(regid) |
                        COND(regid & HALF_REG_ID,
                             A6XX_SP_FS_OUTPUT_REG_HALF_PRECISION));
      }
   }

   tu6_emit_msaa(cs, samples, false);
}

static void
tu6_emit_blit_consts_load(struct tu_cmd_buffer *cmd,
                          struct tu_cs *cs,
                          uint32_t opcode,
                          enum a6xx_state_block block,
                          uint32_t offset,
                          const void *consts,
                          uint32_t size_vec4)
{
   assert(offset % cmd->device->compiler->const_upload_unit == 0);

   struct tu_cs_memory mem = {};
   VkResult result = tu_cs_alloc(&cmd->sub_cs, size_vec4, 4, &mem);
   if (result != VK_SUCCESS) {
      vk_command_buffer_set_error(&cmd->vk, result);
      return;
   }

   memcpy(mem.map, consts, size_vec4 * 4 * sizeof(uint32_t));

   tu_cs_emit_pkt7(cs, opcode, 3);
   tu_cs_emit(cs, CP_LOAD_STATE6_0_DST_OFF(offset) |
                  CP_LOAD_STATE6_0_STATE_TYPE(ST6_CONSTANTS) |
                  CP_LOAD_STATE6_0_STATE_SRC(SS6_INDIRECT) |
                  CP_LOAD_STATE6_0_STATE_BLOCK(block) |
                  CP_LOAD_STATE6_0_NUM_UNIT(size_vec4));
   tu_cs_emit_qw(cs, mem.iova);
}

static void
r3d_coords_raw(struct tu_cmd_buffer *cmd, struct tu_cs *cs, const float *coords)
{
   tu6_emit_blit_consts_load(cmd, cs, CP_LOAD_STATE6_GEOM, SB6_VS_SHADER, 0, coords, 2);
}

/* z coordinate for "z scale" blit path which uses a 3d texture */
static void
r3d_coord_z(struct tu_cmd_buffer *cmd, struct tu_cs *cs, float z)
{
   const uint32_t coord[] = {
      fui(z),
      0,
      0,
      0,
   };

   tu6_emit_blit_consts_load(cmd, cs, CP_LOAD_STATE6_GEOM, SB6_VS_SHADER, 4, coord, 1);
}

static void
r3d_coords(struct tu_cmd_buffer *cmd,
           struct tu_cs *cs,
           const VkOffset2D dst,
           const VkOffset2D src,
           const VkExtent2D extent)
{
   const bool no_src = src.x != blt_no_coord.x;
   int32_t src_x1 = no_src ? src.x : 0;
   int32_t src_y1 = no_src ? src.y : 0;

   const float coords[] = {
      dst.x,
      dst.y,
      src_x1,
      src_y1,
      dst.x + extent.width,
      dst.y + extent.height,
      src_x1 + extent.width,
      src_y1 + extent.height,
   };
   r3d_coords_raw(cmd, cs, coords);
}

static void
r3d_clear_value(struct tu_cmd_buffer *cmd, struct tu_cs *cs, enum pipe_format format, const VkClearValue *val)
{
   uint32_t coords[4] = {};

   switch (format) {
   case PIPE_FORMAT_Z24X8_UNORM:
   case PIPE_FORMAT_Z24_UNORM_S8_UINT: {
      /* cleared as r8g8b8a8_unorm using special format */
      uint32_t tmp = tu_pack_float32_for_unorm(val->depthStencil.depth, 24);
      coords[0] = fui((tmp & 0xff) / 255.0f);
      coords[1] = fui((tmp >> 8 & 0xff) / 255.0f);
      coords[2] = fui((tmp >> 16 & 0xff) / 255.0f);
      coords[3] = fui((val->depthStencil.stencil & 0xff) / 255.0f);
   } break;
   case PIPE_FORMAT_Z16_UNORM:
   case PIPE_FORMAT_Z32_FLOAT:
      coords[0] = fui(val->depthStencil.depth);
      coords[1] = 0;
      coords[2] = 0;
      coords[3] = 0;
      break;
   case PIPE_FORMAT_S8_UINT:
      coords[0] = val->depthStencil.stencil & 0xff;
      coords[1] = 0;
      coords[2] = 0;
      coords[3] = 0;
      break;
   default:
      /* as color formats use clear value as-is */
      assert(!util_format_is_depth_or_stencil(format));
      memcpy(coords, val->color.uint32, 4 * sizeof(uint32_t));
      break;
   }

   tu6_emit_blit_consts_load(cmd, cs, CP_LOAD_STATE6_FRAG, SB6_FS_SHADER, 0, coords, 1);
}

static void
r3d_src_common(struct tu_cmd_buffer *cmd,
               struct tu_cs *cs,
               const uint32_t *tex_const,
               uint32_t offset_base,
               uint32_t offset_ubwc,
               VkFilter filter)
{
   struct tu_cs_memory texture = { };
   VkResult result = tu_cs_alloc(&cmd->sub_cs,
                                 2, /* allocate space for a sampler too */
                                 A6XX_TEX_CONST_DWORDS, &texture);
   if (result != VK_SUCCESS) {
      vk_command_buffer_set_error(&cmd->vk, result);
      return;
   }

   memcpy(texture.map, tex_const, A6XX_TEX_CONST_DWORDS * 4);

   /* patch addresses for layer offset */
   *(uint64_t*) (texture.map + 4) += offset_base;
   uint64_t ubwc_addr = (texture.map[7] | (uint64_t) texture.map[8] << 32) + offset_ubwc;
   texture.map[7] = ubwc_addr;
   texture.map[8] = ubwc_addr >> 32;

   texture.map[A6XX_TEX_CONST_DWORDS + 0] =
      A6XX_TEX_SAMP_0_XY_MAG(tu6_tex_filter(filter, false)) |
      A6XX_TEX_SAMP_0_XY_MIN(tu6_tex_filter(filter, false)) |
      A6XX_TEX_SAMP_0_WRAP_S(A6XX_TEX_CLAMP_TO_EDGE) |
      A6XX_TEX_SAMP_0_WRAP_T(A6XX_TEX_CLAMP_TO_EDGE) |
      A6XX_TEX_SAMP_0_WRAP_R(A6XX_TEX_CLAMP_TO_EDGE) |
      0x60000; /* XXX used by blob, doesn't seem necessary */
   texture.map[A6XX_TEX_CONST_DWORDS + 1] =
      A6XX_TEX_SAMP_1_UNNORM_COORDS |
      A6XX_TEX_SAMP_1_MIPFILTER_LINEAR_FAR;
   texture.map[A6XX_TEX_CONST_DWORDS + 2] = 0;
   texture.map[A6XX_TEX_CONST_DWORDS + 3] = 0;

   tu_cs_emit_pkt7(cs, CP_LOAD_STATE6_FRAG, 3);
   tu_cs_emit(cs, CP_LOAD_STATE6_0_DST_OFF(0) |
               CP_LOAD_STATE6_0_STATE_TYPE(ST6_SHADER) |
               CP_LOAD_STATE6_0_STATE_SRC(SS6_INDIRECT) |
               CP_LOAD_STATE6_0_STATE_BLOCK(SB6_FS_TEX) |
               CP_LOAD_STATE6_0_NUM_UNIT(1));
   tu_cs_emit_qw(cs, texture.iova + A6XX_TEX_CONST_DWORDS * 4);

   tu_cs_emit_regs(cs, A6XX_SP_FS_TEX_SAMP(.qword = texture.iova + A6XX_TEX_CONST_DWORDS * 4));

   tu_cs_emit_pkt7(cs, CP_LOAD_STATE6_FRAG, 3);
   tu_cs_emit(cs, CP_LOAD_STATE6_0_DST_OFF(0) |
      CP_LOAD_STATE6_0_STATE_TYPE(ST6_CONSTANTS) |
      CP_LOAD_STATE6_0_STATE_SRC(SS6_INDIRECT) |
      CP_LOAD_STATE6_0_STATE_BLOCK(SB6_FS_TEX) |
      CP_LOAD_STATE6_0_NUM_UNIT(1));
   tu_cs_emit_qw(cs, texture.iova);

   tu_cs_emit_regs(cs, A6XX_SP_FS_TEX_CONST(.qword = texture.iova));
   tu_cs_emit_regs(cs, A6XX_SP_FS_TEX_COUNT(1));
}

static void
r3d_src(struct tu_cmd_buffer *cmd,
        struct tu_cs *cs,
        const struct fdl6_view *iview,
        uint32_t layer,
        VkFilter filter,
        enum pipe_format dst_format)
{
   uint32_t desc[A6XX_TEX_CONST_DWORDS];
   memcpy(desc, iview->descriptor, sizeof(desc));

   enum a6xx_format fmt = (enum a6xx_format)(
      (desc[0] & A6XX_TEX_CONST_0_FMT__MASK) >> A6XX_TEX_CONST_0_FMT__SHIFT);
   enum pipe_format src_format = iview->format;
   fixup_src_format(&src_format, dst_format, &fmt);
   desc[0] = (desc[0] & ~A6XX_TEX_CONST_0_FMT__MASK) |
      A6XX_TEX_CONST_0_FMT(fmt);

   r3d_src_common(cmd, cs, desc,
                  iview->layer_size * layer,
                  iview->ubwc_layer_size * layer,
                  filter);
}

template <chip CHIP>
static void
r3d_src_buffer(struct tu_cmd_buffer *cmd,
               struct tu_cs *cs,
               enum pipe_format format,
               uint64_t va, uint32_t pitch,
               uint32_t width, uint32_t height,
               enum pipe_format dst_format)
{
   uint32_t desc[A6XX_TEX_CONST_DWORDS];

   struct tu_native_format fmt = blit_format_texture<CHIP>(format, TILE6_LINEAR, false, false);
   enum a6xx_format color_format = fmt.fmt;
   fixup_src_format(&format, dst_format, &color_format);

   desc[0] =
      COND(util_format_is_srgb(format), A6XX_TEX_CONST_0_SRGB) |
      A6XX_TEX_CONST_0_FMT(color_format) |
      A6XX_TEX_CONST_0_SWAP(fmt.swap) |
      A6XX_TEX_CONST_0_SWIZ_X(A6XX_TEX_X) |
      A6XX_TEX_CONST_0_SWIZ_Y(A6XX_TEX_Y) |
      A6XX_TEX_CONST_0_SWIZ_Z(A6XX_TEX_Z) |
      A6XX_TEX_CONST_0_SWIZ_W(A6XX_TEX_W);
   desc[1] = A6XX_TEX_CONST_1_WIDTH(width) | A6XX_TEX_CONST_1_HEIGHT(height);
   desc[2] =
      A6XX_TEX_CONST_2_PITCH(pitch) |
      A6XX_TEX_CONST_2_TYPE(A6XX_TEX_2D);
   desc[3] = 0;
   desc[4] = va;
   desc[5] = va >> 32;
   for (uint32_t i = 6; i < A6XX_TEX_CONST_DWORDS; i++)
      desc[i] = 0;

   r3d_src_common(cmd, cs, desc, 0, 0, VK_FILTER_NEAREST);
}

static void
r3d_src_depth(struct tu_cmd_buffer *cmd,
              struct tu_cs *cs,
              const struct tu_image_view *iview,
              uint32_t layer)
{
   uint32_t desc[A6XX_TEX_CONST_DWORDS];

   memcpy(desc, iview->view.descriptor, sizeof(desc));
   uint64_t va = iview->depth_base_addr;

   desc[0] &= ~(A6XX_TEX_CONST_0_FMT__MASK |
                A6XX_TEX_CONST_0_SWIZ_X__MASK | A6XX_TEX_CONST_0_SWIZ_Y__MASK |
                A6XX_TEX_CONST_0_SWIZ_Z__MASK | A6XX_TEX_CONST_0_SWIZ_W__MASK |
                A6XX_TEX_CONST_0_SWAP__MASK);
   desc[0] |= A6XX_TEX_CONST_0_FMT(FMT6_32_FLOAT) |
              A6XX_TEX_CONST_0_SWIZ_X(A6XX_TEX_X) |
              A6XX_TEX_CONST_0_SWIZ_Y(A6XX_TEX_Y) |
              A6XX_TEX_CONST_0_SWIZ_Z(A6XX_TEX_Z) |
              A6XX_TEX_CONST_0_SWIZ_W(A6XX_TEX_W);
   desc[2] =
      A6XX_TEX_CONST_2_PITCH(iview->depth_pitch) |
      A6XX_TEX_CONST_2_TYPE(A6XX_TEX_2D);
   desc[3] = A6XX_TEX_CONST_3_ARRAY_PITCH(iview->depth_layer_size) |
      (iview->view.descriptor[3] & ~A6XX_TEX_CONST_3_ARRAY_PITCH__MASK);
   desc[4] = va;
   desc[5] = va >> 32;

   r3d_src_common(cmd, cs, desc,
                  iview->depth_layer_size * layer,
                  iview->view.ubwc_layer_size * layer,
                  VK_FILTER_NEAREST);
}

static void
r3d_src_stencil(struct tu_cmd_buffer *cmd,
                struct tu_cs *cs,
                const struct tu_image_view *iview,
                uint32_t layer)
{
   uint32_t desc[A6XX_TEX_CONST_DWORDS];

   memcpy(desc, iview->view.descriptor, sizeof(desc));
   uint64_t va = iview->stencil_base_addr;

   desc[0] &= ~(A6XX_TEX_CONST_0_FMT__MASK |
                A6XX_TEX_CONST_0_SWIZ_X__MASK | A6XX_TEX_CONST_0_SWIZ_Y__MASK |
                A6XX_TEX_CONST_0_SWIZ_Z__MASK | A6XX_TEX_CONST_0_SWIZ_W__MASK |
                A6XX_TEX_CONST_0_SWAP__MASK);
   desc[0] |= A6XX_TEX_CONST_0_FMT(FMT6_8_UINT) |
              A6XX_TEX_CONST_0_SWIZ_X(A6XX_TEX_X) |
              A6XX_TEX_CONST_0_SWIZ_Y(A6XX_TEX_Y) |
              A6XX_TEX_CONST_0_SWIZ_Z(A6XX_TEX_Z) |
              A6XX_TEX_CONST_0_SWIZ_W(A6XX_TEX_W);
   desc[2] =
      A6XX_TEX_CONST_2_PITCH(iview->stencil_pitch) |
      A6XX_TEX_CONST_2_TYPE(A6XX_TEX_2D);
   desc[3] = A6XX_TEX_CONST_3_ARRAY_PITCH(iview->stencil_layer_size);
   desc[4] = va;
   desc[5] = va >> 32;
   for (unsigned i = 6; i < A6XX_TEX_CONST_DWORDS; i++)
      desc[i] = 0;

   r3d_src_common(cmd, cs, desc, iview->stencil_layer_size * layer, 0,
                  VK_FILTER_NEAREST);
}

static void
r3d_src_gmem_load(struct tu_cmd_buffer *cmd,
                  struct tu_cs *cs,
                  const struct tu_image_view *iview,
                  uint32_t layer)
{
   uint32_t desc[A6XX_TEX_CONST_DWORDS];

   memcpy(desc, iview->view.descriptor, sizeof(desc));

   /* Fixup D24 formats because we always load both depth and stencil. */
   enum pipe_format format = iview->view.format;
   if (format == PIPE_FORMAT_X24S8_UINT ||
       format == PIPE_FORMAT_Z24X8_UNORM ||
       format == PIPE_FORMAT_Z24_UNORM_S8_UINT) {
      desc[0] &= ~A6XX_TEX_CONST_0_FMT__MASK;
      if (iview->view.ubwc_enabled)
         desc[0] |= A6XX_TEX_CONST_0_FMT(FMT6_Z24_UNORM_S8_UINT_AS_R8G8B8A8);
      else
         desc[0] |= A6XX_TEX_CONST_0_FMT(FMT6_8_8_8_8_UNORM);
   }

   /* When loading/storing GMEM we always load the full image and don't do any
    * swizzling or swapping, that's done in the draw when reading/writing
    * GMEM, so we need to fixup the swizzle and swap.
    */
   desc[0] &= ~(A6XX_TEX_CONST_0_SWIZ_X__MASK | A6XX_TEX_CONST_0_SWIZ_Y__MASK |
                A6XX_TEX_CONST_0_SWIZ_Z__MASK | A6XX_TEX_CONST_0_SWIZ_W__MASK |
                A6XX_TEX_CONST_0_SWAP__MASK);
   desc[0] |= A6XX_TEX_CONST_0_SWIZ_X(A6XX_TEX_X) |
              A6XX_TEX_CONST_0_SWIZ_Y(A6XX_TEX_Y) |
              A6XX_TEX_CONST_0_SWIZ_Z(A6XX_TEX_Z) |
              A6XX_TEX_CONST_0_SWIZ_W(A6XX_TEX_W);

   r3d_src_common(cmd, cs, desc,
                  iview->view.layer_size * layer,
                  iview->view.ubwc_layer_size * layer,
                  VK_FILTER_NEAREST);
}

template <chip CHIP>
static void
r3d_src_gmem(struct tu_cmd_buffer *cmd,
             struct tu_cs *cs,
             const struct tu_image_view *iview,
             enum pipe_format format,
             enum pipe_format dst_format,
             uint32_t gmem_offset,
             uint32_t cpp)
{
   uint32_t desc[A6XX_TEX_CONST_DWORDS];
   memcpy(desc, iview->view.descriptor, sizeof(desc));

   enum a6xx_format fmt = blit_format_texture<CHIP>(format, TILE6_LINEAR, false, true).fmt;
   fixup_src_format(&format, dst_format, &fmt);

   /* patch the format so that depth/stencil get the right format and swizzle */
   desc[0] &= ~(A6XX_TEX_CONST_0_FMT__MASK |
                A6XX_TEX_CONST_0_SWIZ_X__MASK | A6XX_TEX_CONST_0_SWIZ_Y__MASK |
                A6XX_TEX_CONST_0_SWIZ_Z__MASK | A6XX_TEX_CONST_0_SWIZ_W__MASK);
   desc[0] |= A6XX_TEX_CONST_0_FMT(fmt) |
               A6XX_TEX_CONST_0_SWIZ_X(A6XX_TEX_X) |
               A6XX_TEX_CONST_0_SWIZ_Y(A6XX_TEX_Y) |
               A6XX_TEX_CONST_0_SWIZ_Z(A6XX_TEX_Z) |
               A6XX_TEX_CONST_0_SWIZ_W(A6XX_TEX_W);

   /* patched for gmem */
   desc[0] &= ~(A6XX_TEX_CONST_0_SWAP__MASK | A6XX_TEX_CONST_0_TILE_MODE__MASK);
   desc[0] |= A6XX_TEX_CONST_0_TILE_MODE(TILE6_2);
   desc[2] =
      A6XX_TEX_CONST_2_TYPE(A6XX_TEX_2D) |
      A6XX_TEX_CONST_2_PITCH(cmd->state.tiling->tile0.width * cpp);
   desc[3] = 0;
   desc[4] = cmd->device->physical_device->gmem_base + gmem_offset;
   desc[5] = A6XX_TEX_CONST_5_DEPTH(1);
   for (unsigned i = 6; i < A6XX_TEX_CONST_DWORDS; i++)
      desc[i] = 0;

   r3d_src_common(cmd, cs, desc, 0, 0, VK_FILTER_NEAREST);
}

template <chip CHIP>
static void
r3d_dst(struct tu_cs *cs, const struct fdl6_view *iview, uint32_t layer,
        enum pipe_format src_format)
{
   uint32_t mrt_buf_info = iview->RB_MRT_BUF_INFO;

   enum a6xx_format fmt = (enum a6xx_format)(
      mrt_buf_info & A6XX_RB_MRT_BUF_INFO_COLOR_FORMAT__MASK);
   enum pipe_format dst_format = iview->format;
   fixup_dst_format(src_format, &dst_format, &fmt);
   mrt_buf_info =
      (mrt_buf_info & ~A6XX_RB_MRT_BUF_INFO_COLOR_FORMAT__MASK) |
      A6XX_RB_MRT_BUF_INFO_COLOR_FORMAT(fmt);

   tu_cs_emit_regs(cs,
      RB_MRT_BUF_INFO(CHIP, 0, .dword = mrt_buf_info),
      A6XX_RB_MRT_PITCH(0, iview->pitch),
      A6XX_RB_MRT_ARRAY_PITCH(0, iview->layer_size),
      A6XX_RB_MRT_BASE(0, .qword = tu_layer_address(iview, layer)),
      A6XX_RB_MRT_BASE_GMEM(0),
   );

   tu_cs_emit_pkt4(cs, REG_A6XX_RB_MRT_FLAG_BUFFER(0), 3);
   tu_cs_image_flag_ref(cs, iview, layer);

   /* Use color format from RB_MRT_BUF_INFO. This register is relevant for
    * FMT6_NV12_Y.
    */
   tu_cs_emit_regs(cs, A6XX_GRAS_LRZ_MRT_BUF_INFO_0(.color_format = fmt));

   tu_cs_emit_regs(cs, RB_RENDER_CNTL(CHIP, .flag_mrts = iview->ubwc_enabled));
   tu_cs_emit_regs(cs, A7XX_GRAS_SU_RENDER_CNTL());
}

template <chip CHIP>
static void
r3d_dst_depth(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t layer)
{
   tu_cs_emit_regs(cs,
      RB_MRT_BUF_INFO(CHIP, 0, .dword = tu_image_view_depth(iview, RB_MRT_BUF_INFO)),
      A6XX_RB_MRT_PITCH(0, iview->depth_pitch),
      A6XX_RB_MRT_ARRAY_PITCH(0, iview->depth_layer_size),
      A6XX_RB_MRT_BASE(0, .qword = iview->depth_base_addr + iview->depth_layer_size * layer),
      A6XX_RB_MRT_BASE_GMEM(0),
   );

   tu_cs_emit_pkt4(cs, REG_A6XX_RB_MRT_FLAG_BUFFER(0), 3);
   tu_cs_image_flag_ref(cs, &iview->view, layer);

   tu_cs_emit_regs(cs, RB_RENDER_CNTL(CHIP, .flag_mrts = iview->view.ubwc_enabled));
   tu_cs_emit_regs(cs, A7XX_GRAS_SU_RENDER_CNTL());
}

template <chip CHIP>
static void
r3d_dst_stencil(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t layer)
{
   tu_cs_emit_regs(cs,
      RB_MRT_BUF_INFO(CHIP, 0, .dword = tu_image_view_stencil(iview, RB_MRT_BUF_INFO)),
      A6XX_RB_MRT_PITCH(0, iview->stencil_pitch),
      A6XX_RB_MRT_ARRAY_PITCH(0, iview->stencil_layer_size),
      A6XX_RB_MRT_BASE(0, .qword = iview->stencil_base_addr + iview->stencil_layer_size * layer),
      A6XX_RB_MRT_BASE_GMEM(0),
   );

   tu_cs_emit_regs(cs, RB_RENDER_CNTL(CHIP));
   tu_cs_emit_regs(cs, A7XX_GRAS_SU_RENDER_CNTL());
}

template <chip CHIP>
static void
r3d_dst_buffer(struct tu_cs *cs, enum pipe_format format, uint64_t va, uint32_t pitch,
               enum pipe_format src_format)
{
   struct tu_native_format fmt = blit_format_color(format, TILE6_LINEAR);

   enum a6xx_format color_fmt = fmt.fmt;
   fixup_dst_format(src_format, &format, &color_fmt);

   tu_cs_emit_regs(cs,
                   RB_MRT_BUF_INFO(CHIP, 0, .color_format = color_fmt, .color_swap = fmt.swap),
                   A6XX_RB_MRT_PITCH(0, pitch),
                   A6XX_RB_MRT_ARRAY_PITCH(0, 0),
                   A6XX_RB_MRT_BASE(0, .qword = va),
                   A6XX_RB_MRT_BASE_GMEM(0, 0));

   tu_cs_emit_regs(cs, RB_RENDER_CNTL(CHIP));
   tu_cs_emit_regs(cs, A7XX_GRAS_SU_RENDER_CNTL());
}

template <chip CHIP>
static void
r3d_dst_gmem(struct tu_cmd_buffer *cmd, struct tu_cs *cs,
             const struct tu_image_view *iview,
             const struct tu_render_pass_attachment *att,
             bool separate_stencil, unsigned layer)
{
   unsigned RB_MRT_BUF_INFO;
   unsigned gmem_offset;

   if (att->format == VK_FORMAT_D32_SFLOAT_S8_UINT) {
      if (!separate_stencil) {
         RB_MRT_BUF_INFO = tu_image_view_depth(iview, RB_MRT_BUF_INFO);
         gmem_offset = tu_attachment_gmem_offset(cmd, att, layer);
      } else {
         RB_MRT_BUF_INFO = tu_image_view_stencil(iview, RB_MRT_BUF_INFO);
         gmem_offset = tu_attachment_gmem_offset_stencil(cmd, att, layer);
      }
   } else {
      RB_MRT_BUF_INFO = iview->view.RB_MRT_BUF_INFO;
      gmem_offset = tu_attachment_gmem_offset(cmd, att, layer);
   }

   tu_cs_emit_regs(cs,
                   RB_MRT_BUF_INFO(CHIP, 0, .dword = RB_MRT_BUF_INFO),
                   A6XX_RB_MRT_PITCH(0, 0),
                   A6XX_RB_MRT_ARRAY_PITCH(0, 0),
                   A6XX_RB_MRT_BASE(0, 0),
                   A6XX_RB_MRT_BASE_GMEM(0, gmem_offset));

   enum a6xx_format color_format =
      (enum a6xx_format)(RB_MRT_BUF_INFO & A6XX_RB_MRT_BUF_INFO_COLOR_FORMAT__MASK);
   tu_cs_emit_regs(cs,
                   A6XX_GRAS_LRZ_MRT_BUF_INFO_0(.color_format = color_format));

   tu_cs_emit_regs(cs, RB_RENDER_CNTL(CHIP));
   tu_cs_emit_regs(cs, A7XX_GRAS_SU_RENDER_CNTL());
}

static uint8_t
aspect_write_mask(enum pipe_format format, VkImageAspectFlags aspect_mask)
{
   uint8_t mask = 0xf;
   assert(aspect_mask);
   /* note: the only format with partial writing is D24S8,
    * clear/blit uses the _AS_R8G8B8A8 format to access it
    */
   if (format == PIPE_FORMAT_Z24_UNORM_S8_UINT) {
      if (aspect_mask == VK_IMAGE_ASPECT_DEPTH_BIT)
         mask = 0x7;
      if (aspect_mask == VK_IMAGE_ASPECT_STENCIL_BIT)
         mask = 0x8;
   }
   return mask;
}

static uint8_t
aspect_write_mask_generic_clear(enum pipe_format format, VkImageAspectFlags aspect_mask)
{
   uint8_t mask = 0xf;
   assert(aspect_mask);
   /* note: the only format with partial writing is D24S8 */
   if (format == PIPE_FORMAT_Z24_UNORM_S8_UINT) {
      if (aspect_mask == VK_IMAGE_ASPECT_DEPTH_BIT)
         mask = 0x1;
      if (aspect_mask == VK_IMAGE_ASPECT_STENCIL_BIT)
         mask = 0x2;
      if (aspect_mask == (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT))
         mask = 0x3;
   }
   return mask;
}

enum r3d_blit_param {
   R3D_Z_SCALE = 1 << 0,
   R3D_DST_GMEM = 1 << 1,
   R3D_COPY = 1 << 2,
};

template <chip CHIP>
static void
r3d_setup(struct tu_cmd_buffer *cmd,
          struct tu_cs *cs,
          enum pipe_format src_format,
          enum pipe_format dst_format,
          VkImageAspectFlags aspect_mask,
          unsigned blit_param,
          bool clear,
          bool ubwc,
          VkSampleCountFlagBits samples)
{
   if (!cmd->state.pass && cmd->device->dbg_renderpass_stomp_cs) {
      tu_cs_emit_call(cs, cmd->device->dbg_renderpass_stomp_cs);
   }

   enum a6xx_format fmt = blit_base_format<CHIP>(dst_format, ubwc, false);
   fixup_dst_format(src_format, &dst_format, &fmt);

   if (!cmd->state.pass) {
      tu_emit_cache_flush_ccu<CHIP>(cmd, cs, TU_CMD_CCU_SYSMEM);
      tu6_emit_window_scissor(cs, 0, 0, 0x3fff, 0x3fff);
   }

   if (!(blit_param & R3D_DST_GMEM)) {
      if (CHIP == A6XX) {
         tu_cs_emit_regs(cs, A6XX_GRAS_BIN_CONTROL(.buffers_location = BUFFERS_IN_SYSMEM));
      } else {
         tu_cs_emit_regs(cs, A6XX_GRAS_BIN_CONTROL());
      }

      tu_cs_emit_regs(cs, RB_BIN_CONTROL(CHIP, .buffers_location = BUFFERS_IN_SYSMEM));

      if (CHIP >= A7XX) {
         tu_cs_emit_regs(cs, A7XX_RB_UNKNOWN_8812(0x3ff));
         tu_cs_emit_regs(cs,
            A7XX_RB_UNKNOWN_8E06(cmd->device->physical_device->info->a6xx.magic.RB_UNKNOWN_8E06));
      }
   }

   enum r3d_type type;
   if (clear) {
      type = R3D_CLEAR;
   } else if ((blit_param & R3D_COPY) && tu_pipe_format_is_float16(src_format)) {
      /* Avoid canonicalizing NaNs in copies by using the special half-float
       * path that uses half regs.
       */
      type = R3D_COPY_HALF;
   } else {
      type = R3D_BLIT;
   }

   r3d_common<CHIP>(cmd, cs, type, 1, blit_param & R3D_Z_SCALE, samples);

   tu_cs_emit_regs(cs, A6XX_SP_FS_OUTPUT_CNTL1(.mrt = 1));
   tu_cs_emit_regs(cs, A6XX_RB_FS_OUTPUT_CNTL1(.mrt = 1));
   tu_cs_emit_regs(cs, A6XX_SP_BLEND_CNTL());
   tu_cs_emit_regs(cs, A6XX_RB_BLEND_CNTL(.sample_mask = 0xffff));

   tu_cs_emit_regs(cs, A6XX_RB_DEPTH_PLANE_CNTL());
   tu_cs_emit_regs(cs, A6XX_RB_DEPTH_CNTL());
   tu_cs_emit_regs(cs, A6XX_GRAS_SU_DEPTH_CNTL());
   tu_cs_emit_regs(cs, A6XX_GRAS_SU_DEPTH_PLANE_CNTL());
   tu_cs_emit_regs(cs, A6XX_RB_STENCIL_CONTROL());
   tu_cs_emit_regs(cs, A6XX_GRAS_SU_STENCIL_CNTL());
   tu_cs_emit_regs(cs, A6XX_RB_STENCILMASK());
   tu_cs_emit_regs(cs, A6XX_RB_STENCILWRMASK());
   tu_cs_emit_regs(cs, A6XX_RB_STENCILREF());

   tu_cs_emit_regs(cs, A6XX_SP_FS_MRT_REG(0,
                        .color_format = fmt,
                        .color_sint = util_format_is_pure_sint(dst_format),
                        .color_uint = util_format_is_pure_uint(dst_format)));

   tu_cs_emit_regs(cs, A6XX_RB_MRT_CONTROL(0,
      .component_enable = aspect_write_mask(dst_format, aspect_mask)));
   tu_cs_emit_regs(cs, A6XX_RB_SRGB_CNTL(util_format_is_srgb(dst_format)));
   tu_cs_emit_regs(cs, A6XX_SP_SRGB_CNTL(util_format_is_srgb(dst_format)));

   tu_cs_emit_regs(cs, A6XX_GRAS_LRZ_CNTL(0));
   tu_cs_emit_regs(cs, A6XX_RB_LRZ_CNTL(0));

   if (CHIP >= A7XX) {
      tu_cs_emit_regs(cs, A7XX_GRAS_LRZ_DEPTH_BUFFER_INFO());

      tu_cs_emit_regs(cs, A6XX_RB_FSR_CONFIG());
      tu_cs_emit_regs(cs, A7XX_SP_FSR_CONFIG());
      tu_cs_emit_regs(cs, A7XX_GRAS_FSR_CONFIG());
   }

   tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_SC_CNTL,
                        A6XX_GRAS_SC_CNTL_CCUSINGLECACHELINESIZE(2));

   /* Disable sample counting in order to not affect occlusion query. */
   tu_cs_emit_regs(cs, A6XX_RB_SAMPLE_COUNT_CONTROL(.disable = true));

   tu_cs_emit_regs(cs, A6XX_RB_DITHER_CNTL());
   if (CHIP >= A7XX) {
      tu_cs_emit_regs(cs, A7XX_SP_DITHER_CNTL());
   }

   if (cmd->state.prim_generated_query_running_before_rp) {
      tu_emit_event_write<CHIP>(cmd, cs, FD_STOP_PRIMITIVE_CTRS);
   }

   if (cmd->state.predication_active) {
      tu_cs_emit_pkt7(cs, CP_DRAW_PRED_ENABLE_LOCAL, 1);
      tu_cs_emit(cs, 0);
   }
}

static void
r3d_run(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
{
   tu_cs_emit_pkt7(cs, CP_DRAW_INDX_OFFSET, 3);
   tu_cs_emit(cs, CP_DRAW_INDX_OFFSET_0_PRIM_TYPE(DI_PT_RECTLIST) |
                  CP_DRAW_INDX_OFFSET_0_SOURCE_SELECT(DI_SRC_SEL_AUTO_INDEX) |
                  CP_DRAW_INDX_OFFSET_0_VIS_CULL(IGNORE_VISIBILITY));
   tu_cs_emit(cs, 1); /* instance count */
   tu_cs_emit(cs, 2); /* vertex count */
}

static void
r3d_run_vis(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
{
   tu_cs_emit_pkt7(cs, CP_DRAW_INDX_OFFSET, 3);
   tu_cs_emit(cs, CP_DRAW_INDX_OFFSET_0_PRIM_TYPE(DI_PT_RECTLIST) |
                  CP_DRAW_INDX_OFFSET_0_SOURCE_SELECT(DI_SRC_SEL_AUTO_INDEX) |
                  CP_DRAW_INDX_OFFSET_0_VIS_CULL(USE_VISIBILITY));
   tu_cs_emit(cs, 1); /* instance count */
   tu_cs_emit(cs, 2); /* vertex count */
}

template <chip CHIP>
static void
r3d_teardown(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
{
   if (cmd->state.predication_active) {
      tu_cs_emit_pkt7(cs, CP_DRAW_PRED_ENABLE_LOCAL, 1);
      tu_cs_emit(cs, 1);
   }

   /* Re-enable sample counting. */
   tu_cs_emit_regs(cs, A6XX_RB_SAMPLE_COUNT_CONTROL(.disable = false));

   if (cmd->state.prim_generated_query_running_before_rp) {
      tu_emit_event_write<CHIP>(cmd, cs, FD_START_PRIMITIVE_CTRS);
   }
}

/* blit ops - common interface for 2d/shader paths */

struct blit_ops {
   void (*coords)(struct tu_cmd_buffer *cmd,
                  struct tu_cs *cs,
                  const VkOffset2D dst,
                  const VkOffset2D src,
                  const VkExtent2D extent);
   void (*clear_value)(struct tu_cmd_buffer *cmd,
                       struct tu_cs *cs,
                       enum pipe_format format,
                       const VkClearValue *val);
   void (*src)(
        struct tu_cmd_buffer *cmd,
        struct tu_cs *cs,
        const struct fdl6_view *iview,
        uint32_t layer,
        VkFilter filter,
        enum pipe_format dst_format);
   void (*src_buffer)(struct tu_cmd_buffer *cmd, struct tu_cs *cs,
                      enum pipe_format format,
                      uint64_t va, uint32_t pitch,
                      uint32_t width, uint32_t height,
                      enum pipe_format dst_format);
   void (*dst)(struct tu_cs *cs, const struct fdl6_view *iview, uint32_t layer,
               enum pipe_format src_format);
   void (*dst_depth)(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t layer);
   void (*dst_stencil)(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t layer);
   void (*dst_buffer)(struct tu_cs *cs, enum pipe_format format, uint64_t va, uint32_t pitch,
                      enum pipe_format src_format);
   void (*setup)(struct tu_cmd_buffer *cmd,
                 struct tu_cs *cs,
                 enum pipe_format src_format,
                 enum pipe_format dst_format,
                 VkImageAspectFlags aspect_mask,
                 unsigned blit_param, /* CmdBlitImage: rotation in 2D path and z scaling in 3D path */
                 bool clear,
                 bool ubwc,
                 VkSampleCountFlagBits samples);
   void (*run)(struct tu_cmd_buffer *cmd, struct tu_cs *cs);
   void (*teardown)(struct tu_cmd_buffer *cmd,
                    struct tu_cs *cs);
};

template <chip CHIP>
static const struct blit_ops r2d_ops = {
   .coords = r2d_coords,
   .clear_value = r2d_clear_value,
   .src = r2d_src<CHIP>,
   .src_buffer = r2d_src_buffer<CHIP>,
   .dst = r2d_dst<CHIP>,
   .dst_depth = r2d_dst_depth,
   .dst_stencil = r2d_dst_stencil,
   .dst_buffer = r2d_dst_buffer,
   .setup = r2d_setup<CHIP>,
   .run = r2d_run,
   .teardown = r2d_teardown,
};

template <chip CHIP>
static const struct blit_ops r3d_ops = {
   .coords = r3d_coords,
   .clear_value = r3d_clear_value,
   .src = r3d_src,
   .src_buffer = r3d_src_buffer<CHIP>,
   .dst = r3d_dst<CHIP>,
   .dst_depth = r3d_dst_depth<CHIP>,
   .dst_stencil = r3d_dst_stencil<CHIP>,
   .dst_buffer = r3d_dst_buffer<CHIP>,
   .setup = r3d_setup<CHIP>,
   .run = r3d_run,
   .teardown = r3d_teardown<CHIP>,
};

/* passthrough set coords from 3D extents */
static void
coords(const struct blit_ops *ops,
       struct tu_cmd_buffer *cmd,
       struct tu_cs *cs,
       const VkOffset3D dst,
       const VkOffset3D src,
       const VkExtent3D extent)
{
   ops->coords(cmd, cs, (VkOffset2D) {dst.x, dst.y}, (VkOffset2D) {src.x, src.y},
               (VkExtent2D) {extent.width, extent.height});
}

/* Decides the VK format to treat our data as for a memcpy-style blit. We have
 * to be a bit careful because we have to pick a format with matching UBWC
 * compression behavior, so no just returning R8_UINT/R16_UINT/R32_UINT for
 * everything.
 */
static enum pipe_format
copy_format(VkFormat vk_format, VkImageAspectFlags aspect_mask)
{
   if (vk_format_is_compressed(vk_format)) {
      switch (vk_format_get_blocksize(vk_format)) {
      case 1: return PIPE_FORMAT_R8_UINT;
      case 2: return PIPE_FORMAT_R16_UINT;
      case 4: return PIPE_FORMAT_R32_UINT;
      case 8: return PIPE_FORMAT_R32G32_UINT;
      case 16:return PIPE_FORMAT_R32G32B32A32_UINT;
      default:
         unreachable("unhandled format size");
      }
   }

   enum pipe_format format = vk_format_to_pipe_format(vk_format);

   /* For SNORM formats, copy them as the equivalent UNORM format.  If we treat
    * them as snorm then the 0x80 (-1.0 snorm8) value will get clamped to 0x81
    * (also -1.0), when we're supposed to be memcpying the bits. See
    * https://gitlab.khronos.org/Tracker/vk-gl-cts/-/issues/2917 for discussion.
    */
   format = util_format_snorm_to_unorm(format);

   if (vk_format == VK_FORMAT_E5B9G9R9_UFLOAT_PACK32)
      return PIPE_FORMAT_R32_UINT;

   /* For VK_FORMAT_D32_SFLOAT_S8_UINT and YCbCr formats use our existing helpers */
   if (vk_format == VK_FORMAT_D32_SFLOAT_S8_UINT ||
       vk_format_get_ycbcr_info(vk_format))
      return tu_aspects_to_plane(vk_format, aspect_mask);

   /* Otherwise, simply return the pipe_format */
   return format;
}

static void
pack_blit_event_clear_value(const VkClearValue *val, enum pipe_format format, uint32_t clear_value[4])
{
   switch (format) {
   case PIPE_FORMAT_Z24X8_UNORM:
   case PIPE_FORMAT_Z24_UNORM_S8_UINT:
      clear_value[0] = tu_pack_float32_for_unorm(val->depthStencil.depth, 24) |
                       val->depthStencil.stencil << 24;
      return;
   case PIPE_FORMAT_Z16_UNORM:
      clear_value[0] = tu_pack_float32_for_unorm(val->depthStencil.depth, 16);
      return;
   case PIPE_FORMAT_Z32_FLOAT:
      clear_value[0] = fui(val->depthStencil.depth);
      return;
   case PIPE_FORMAT_S8_UINT:
      clear_value[0] = val->depthStencil.stencil;
      return;
   default:
      break;
   }

   float tmp[4];
   memcpy(tmp, val->color.float32, 4 * sizeof(float));
   if (util_format_is_srgb(format)) {
      for (int i = 0; i < 3; i++)
         tmp[i] = util_format_linear_to_srgb_float(tmp[i]);
   }

#define PACK_F(type) util_format_##type##_pack_rgba_float \
   ( (uint8_t*) &clear_value[0], 0, tmp, 0, 1, 1)
   switch (util_format_get_component_bits(format, UTIL_FORMAT_COLORSPACE_RGB, PIPE_SWIZZLE_X)) {
   case 4:
      PACK_F(r4g4b4a4_unorm);
      break;
   case 5:
      if (util_format_get_component_bits(format, UTIL_FORMAT_COLORSPACE_RGB, PIPE_SWIZZLE_Y) == 6)
         PACK_F(r5g6b5_unorm);
      else
         PACK_F(r5g5b5a1_unorm);
      break;
   case 8:
      if (util_format_is_snorm(format))
         PACK_F(r8g8b8a8_snorm);
      else if (util_format_is_unorm(format))
         PACK_F(r8g8b8a8_unorm);
      else
         pack_int8(clear_value, val->color.uint32);
      break;
   case 10:
      if (util_format_is_pure_integer(format))
         pack_int10_2(clear_value, val->color.uint32);
      else
         PACK_F(r10g10b10a2_unorm);
      break;
   case 11:
      clear_value[0] = float3_to_r11g11b10f(val->color.float32);
      break;
   case 16:
      if (util_format_is_snorm(format))
         PACK_F(r16g16b16a16_snorm);
      else if (util_format_is_unorm(format))
         PACK_F(r16g16b16a16_unorm);
      else if (util_format_is_float(format))
         PACK_F(r16g16b16a16_float);
      else
         pack_int16(clear_value, val->color.uint32);
      break;
   case 32:
      memcpy(clear_value, val->color.float32, 4 * sizeof(float));
      break;
   case 0:
      assert(format == PIPE_FORMAT_A8_UNORM);
      PACK_F(a8_unorm);
      break;
   default:
      unreachable("unexpected channel size");
   }
#undef PACK_F
}

static void
event_blit_setup(struct tu_cs *cs,
                 uint32_t buffer_id,
                 const struct tu_render_pass_attachment *att,
                 enum a6xx_blit_event_type blit_event_type,
                 uint32_t clear_mask)
{
   tu_cs_emit_regs(
      cs, A6XX_RB_BLIT_GMEM_MSAA_CNTL(tu_msaa_samples(att->samples)));

   tu_cs_emit_pkt4(cs, REG_A6XX_RB_UNKNOWN_88D0, 1);
   tu_cs_emit(cs, 0);

   tu_cs_emit_regs(
      cs,
      A6XX_RB_BLIT_INFO(.type = blit_event_type,
                        .sample_0 =
                           vk_format_is_int(att->format) ||
                           vk_format_is_depth_or_stencil(att->format),
                        .depth = vk_format_is_depth_or_stencil(att->format),
                        .clear_mask = clear_mask,
                        .buffer_id = buffer_id));
}

struct event_blit_dst_view {
   const struct tu_image *image;
   const struct fdl6_view *view;

   uint32_t layer;

   uint64_t depth_addr;
   uint32_t depth_pitch;

   uint64_t stencil_addr;
   uint32_t stencil_pitch;
};

static event_blit_dst_view
blt_view_from_tu_view(const struct tu_image_view *iview,
                      uint32_t layer)
{
   struct event_blit_dst_view blt_view;
   blt_view.image = iview->image;
   blt_view.view = &iview->view;
   blt_view.layer = layer;

   if (iview->image->vk.format == VK_FORMAT_D32_SFLOAT_S8_UINT) {
      blt_view.depth_addr =
         iview->depth_base_addr + iview->depth_layer_size * layer;
      blt_view.depth_pitch = iview->depth_pitch;

      blt_view.stencil_addr =
         iview->stencil_base_addr + iview->stencil_layer_size * layer;
      blt_view.stencil_pitch = iview->stencil_pitch;
   }
   return blt_view;
}

template <chip CHIP>
static void
event_blit_run(struct tu_cmd_buffer *cmd,
               struct tu_cs *cs,
               const struct tu_render_pass_attachment *att,
               const event_blit_dst_view *blt_view,
               bool separate_stencil)
{
   tu_cs_emit_pkt4(cs, REG_A6XX_RB_BLIT_DST_INFO, 4);
   if (blt_view->image->vk.format == VK_FORMAT_D32_SFLOAT_S8_UINT) {
      if (!separate_stencil) {
         tu_cs_emit(cs, tu_fdl_view_depth(blt_view->view, RB_BLIT_DST_INFO));
         tu_cs_emit_qw(cs, blt_view->depth_addr);
         tu_cs_emit(cs, A6XX_RB_2D_DST_PITCH(blt_view->depth_pitch).value);

         tu_cs_emit_pkt4(cs, REG_A6XX_RB_BLIT_FLAG_DST, 3);
         tu_cs_image_flag_ref(cs, blt_view->view, blt_view->layer);
      } else {
         tu_cs_emit(cs, tu_fdl_view_stencil(blt_view->view, RB_BLIT_DST_INFO) &
                           ~A6XX_RB_BLIT_DST_INFO_FLAGS);
         tu_cs_emit_qw(cs, blt_view->stencil_addr);
         tu_cs_emit(cs, A6XX_RB_BLIT_DST_PITCH(blt_view->stencil_pitch).value);
      }
   } else {
      tu_cs_emit(cs, blt_view->view->RB_BLIT_DST_INFO);
      tu_cs_image_ref_2d<CHIP>(cs, blt_view->view, blt_view->layer, false);

      tu_cs_emit_pkt4(cs, REG_A6XX_RB_BLIT_FLAG_DST, 3);
      tu_cs_image_flag_ref(cs, blt_view->view, blt_view->layer);
   }

   if (att) {
      if (att->format == VK_FORMAT_D32_SFLOAT_S8_UINT && separate_stencil) {
         tu_cs_emit_regs(
            cs, A6XX_RB_BLIT_BASE_GMEM(tu_attachment_gmem_offset_stencil(
                   cmd, att, blt_view->layer)));
      } else {
         tu_cs_emit_regs(cs, A6XX_RB_BLIT_BASE_GMEM(tu_attachment_gmem_offset(
                                cmd, att, blt_view->layer)));
      }
   }

   tu_emit_event_write<CHIP>(cmd, cs, FD_BLIT);
}

static void
tu7_generic_layer_clear(struct tu_cmd_buffer *cmd,
                        struct tu_cs *cs,
                        uint32_t buffer_id,
                        enum pipe_format format,
                        uint8_t clear_mask,
                        bool separate_stencil,
                        uint32_t layer,
                        const VkClearValue *value,
                        uint32_t a)
{
   const struct tu_render_pass_attachment *att =
      &cmd->state.pass->attachments[a];
   const struct tu_image_view *iview = cmd->state.attachments[a];

   uint32_t clear_vals[4] = {};
   pack_blit_event_clear_value(value, format, clear_vals);

   tu_cs_emit_pkt4(cs, REG_A6XX_RB_BLIT_CLEAR_COLOR_DW0, 4);
   tu_cs_emit_array(cs, clear_vals, 4);

   event_blit_dst_view blt_view = blt_view_from_tu_view(iview, layer);

   event_blit_setup(cs, buffer_id, att, BLIT_EVENT_CLEAR, clear_mask);
   event_blit_run<A7XX>(cmd, cs, att, &blt_view, separate_stencil);
}



/* Copies/fills/updates for buffers are happening through CCU but need
 * additional synchronization when write range is not aligned to 64 bytes.
 * Because dst buffer access uses either R8_UNORM or R32_UINT and they are not
 * coherent between each other in CCU since format seem to be a part of a
 * cache key.
 *
 * See: https://gitlab.khronos.org/vulkan/vulkan/-/issues/3306
 *
 * The synchronization with writes from UCHE (e.g. with SSBO stores) are
 * solved by the fact that UCHE has byte level dirtiness tracking and that CCU
 * flush would happen always before UCHE flush for such case (e.g. both
 * renderpass and dispatch would flush pending CCU write).
 *
 * Additionally see:
 * https://gitlab.khronos.org/vulkan/vulkan/-/issues/3398#note_400111
 */
template <chip CHIP>
static void
handle_buffer_unaligned_store(struct tu_cmd_buffer *cmd,
                              uint64_t dst_va,
                              uint64_t size,
                              bool *unaligned_store)
{
   if (*unaligned_store)
      return;

   if ((dst_va & 63) || (size & 63)) {
      tu_flush_for_access(&cmd->state.cache, TU_ACCESS_NONE,
                          TU_ACCESS_CCU_COLOR_INCOHERENT_WRITE);
      /* Wait for invalidations to land. */
      cmd->state.cache.flush_bits |= TU_CMD_FLAG_WAIT_FOR_IDLE;
      tu_emit_cache_flush<CHIP>(cmd);
      *unaligned_store = true;
   }
}

template <chip CHIP>
static void
after_buffer_unaligned_buffer_store(struct tu_cmd_buffer *cmd,
                                    bool unaligned_store)
{
   if (unaligned_store) {
      tu_flush_for_access(&cmd->state.cache,
                          TU_ACCESS_CCU_COLOR_INCOHERENT_WRITE,
                          TU_ACCESS_NONE);
   }
}

template <chip CHIP>
void
tu6_clear_lrz(struct tu_cmd_buffer *cmd,
              struct tu_cs *cs,
              struct tu_image *image,
              const VkClearValue *value)
{
   const struct blit_ops *ops = &r2d_ops<CHIP>;

   /* It is assumed that LRZ cache is invalidated at this point for
    * the writes here to become visible to LRZ.
    *
    * LRZ writes are going through UCHE cache, flush UCHE before changing
    * LRZ via CCU. Don't need to invalidate CCU since we are presumably
    * writing whole cache lines we assume to be 64 bytes.
    */
   tu_emit_event_write<CHIP>(cmd, &cmd->cs, FD_CACHE_CLEAN);

   ops->setup(cmd, cs, PIPE_FORMAT_Z16_UNORM, PIPE_FORMAT_Z16_UNORM,
              VK_IMAGE_ASPECT_DEPTH_BIT, 0, true, false,
              VK_SAMPLE_COUNT_1_BIT);
   ops->clear_value(cmd, cs, PIPE_FORMAT_Z16_UNORM, value);
   ops->dst_buffer(cs, PIPE_FORMAT_Z16_UNORM,
                   image->iova + image->lrz_offset,
                   image->lrz_pitch * 2, PIPE_FORMAT_Z16_UNORM);
   ops->coords(cmd, cs, (VkOffset2D) {}, blt_no_coord,
               (VkExtent2D) { image->lrz_pitch, image->lrz_height });
   ops->run(cmd, cs);
   ops->teardown(cmd, cs);

   /* Clearing writes via CCU color in the PS stage, and LRZ is read via
    * UCHE in the earlier GRAS stage.
    */
   cmd->state.cache.flush_bits |=
      TU_CMD_FLAG_CCU_CLEAN_COLOR | TU_CMD_FLAG_CACHE_INVALIDATE |
      TU_CMD_FLAG_WAIT_FOR_IDLE;
}
TU_GENX(tu6_clear_lrz);

template <chip CHIP>
void
tu6_dirty_lrz_fc(struct tu_cmd_buffer *cmd,
                 struct tu_cs *cs,
                 struct tu_image *image)
{
   const struct blit_ops *ops = &r2d_ops<CHIP>;
   VkClearValue clear = {};
   clear.color.uint32[0] = 0xffffffff;

   using LRZFC = fd_lrzfc_layout<CHIP>;
   uint64_t lrz_fc_iova = image->iova + image->lrz_fc_offset;
   ops->setup(cmd, cs, PIPE_FORMAT_R32_UINT, PIPE_FORMAT_R32_UINT,
              VK_IMAGE_ASPECT_COLOR_BIT, 0, true, false,
              VK_SAMPLE_COUNT_1_BIT);
   ops->clear_value(cmd, cs, PIPE_FORMAT_R32_UINT, &clear);
   ops->dst_buffer(cs, PIPE_FORMAT_R32_UINT,
                   lrz_fc_iova + offsetof(LRZFC, fc1),
                   sizeof(LRZFC::fc1),
                   PIPE_FORMAT_R32_UINT);
   ops->coords(cmd, cs, (VkOffset2D) {}, blt_no_coord, (VkExtent2D) {
      sizeof(LRZFC::fc1) / sizeof(uint32_t), 1
   });
   ops->run(cmd, cs);
   if constexpr (LRZFC::HAS_BIDIR) {
      ops->dst_buffer(cs, PIPE_FORMAT_R32_UINT,
                      lrz_fc_iova + offsetof(LRZFC, fc2),
                      sizeof(LRZFC::fc2),
                      PIPE_FORMAT_R32_UINT);
      ops->coords(cmd, cs, (VkOffset2D) {}, blt_no_coord, (VkExtent2D) {
         sizeof(LRZFC::fc2) / sizeof(uint32_t), 1
      });
      ops->run(cmd, cs);
   }
   ops->teardown(cmd, cs);
}
TU_GENX(tu6_dirty_lrz_fc);

template<chip CHIP>
static void
tu_image_view_copy_blit(struct fdl6_view *iview,
                        struct tu_image *image,
                        enum pipe_format format,
                        const VkImageSubresourceLayers *subres,
                        uint32_t layer,
                        bool z_scale)
{
   VkImageAspectFlags aspect_mask = subres->aspectMask;

   /* always use the AS_R8G8B8A8 format for these */
   if (format == PIPE_FORMAT_Z24_UNORM_S8_UINT ||
       format == PIPE_FORMAT_Z24X8_UNORM) {
      aspect_mask = VK_IMAGE_ASPECT_COLOR_BIT;
   }

   const struct fdl_layout *layout =
      &image->layout[tu6_plane_index(image->vk.format, aspect_mask)];

   const struct fdl_view_args args = {
      .chip = CHIP,
      .iova = image->iova,
      .base_miplevel = subres->mipLevel,
      .level_count = 1,
      .base_array_layer = subres->baseArrayLayer + layer,
      .layer_count = 1,
      .swiz = {
         PIPE_SWIZZLE_X, PIPE_SWIZZLE_Y, PIPE_SWIZZLE_Z, PIPE_SWIZZLE_W
      },
      .format = tu_format_for_aspect(format, aspect_mask),
      .type = z_scale ? FDL_VIEW_TYPE_3D : FDL_VIEW_TYPE_2D,
   };
   fdl6_view_init(iview, &layout, &args, false);
}

template<chip CHIP>
static void
tu_image_view_copy(struct fdl6_view *iview,
                   struct tu_image *image,
                   enum pipe_format format,
                   const VkImageSubresourceLayers *subres,
                   uint32_t layer)
{
   tu_image_view_copy_blit<CHIP>(iview, image, format, subres, layer, false);
}

template<chip CHIP>
static void
tu_image_view_blit(struct fdl6_view *iview,
                   struct tu_image *image,
                   const VkImageSubresourceLayers *subres,
                   uint32_t layer)
{
   enum pipe_format format = tu_aspects_to_plane(image->vk.format, subres->aspectMask);
   tu_image_view_copy_blit<CHIP>(iview, image, format, subres, layer, false);
}

template <chip CHIP>
static void
tu6_blit_image(struct tu_cmd_buffer *cmd,
               struct tu_image *src_image,
               struct tu_image *dst_image,
               const VkImageBlit2 *info,
               VkFilter filter)
{
   const struct blit_ops *ops = &r2d_ops<CHIP>;
   struct tu_cs *cs = &cmd->cs;
   bool z_scale = false;
   uint32_t layers = info->dstOffsets[1].z - info->dstOffsets[0].z;

   /* 2D blit can't do rotation mirroring from just coordinates */
   static const enum a6xx_rotation rotate[2][2] = {
      {ROTATE_0, ROTATE_HFLIP},
      {ROTATE_VFLIP, ROTATE_180},
   };

   bool mirror_x = (info->srcOffsets[1].x < info->srcOffsets[0].x) !=
                   (info->dstOffsets[1].x < info->dstOffsets[0].x);
   bool mirror_y = (info->srcOffsets[1].y < info->srcOffsets[0].y) !=
                   (info->dstOffsets[1].y < info->dstOffsets[0].y);

   int32_t src0_z = info->srcOffsets[0].z;
   int32_t src1_z = info->srcOffsets[1].z;

   if ((info->srcOffsets[1].z - info->srcOffsets[0].z !=
        info->dstOffsets[1].z - info->dstOffsets[0].z) ||
       info->srcOffsets[1].z < info->srcOffsets[0].z) {
      z_scale = true;
   }

   if (info->dstOffsets[1].z < info->dstOffsets[0].z) {
      layers = info->dstOffsets[0].z - info->dstOffsets[1].z;
      src0_z = info->srcOffsets[1].z;
      src1_z = info->srcOffsets[0].z;
   }

   if (vk_image_subresource_layer_count(&dst_image->vk, &info->dstSubresource) > 1) {
      assert(layers <= 1);
      layers = vk_image_subresource_layer_count(&dst_image->vk,
                                                &info->dstSubresource);
   }

   /* BC1_RGB_* formats need to have their last components overriden with 1
    * when sampling, which is normally handled with the texture descriptor
    * swizzle. The 2d path can't handle that, so use the 3d path.
    *
    * TODO: we could use RB_2D_BLIT_CNTL::MASK to make these formats work with
    * the 2d path.
    */

   unsigned blit_param = rotate[mirror_y][mirror_x];
   if (dst_image->layout[0].nr_samples > 1 ||
       src_image->vk.format == VK_FORMAT_BC1_RGB_UNORM_BLOCK ||
       src_image->vk.format == VK_FORMAT_BC1_RGB_SRGB_BLOCK ||
       filter == VK_FILTER_CUBIC_EXT ||
       z_scale) {
      ops = &r3d_ops<CHIP>;
      blit_param = z_scale ? R3D_Z_SCALE : 0;
   }

   /* use the right format in setup() for D32_S8 */
   enum pipe_format src_format = tu_aspects_to_plane(
      src_image->vk.format, info->srcSubresource.aspectMask);
   enum pipe_format dst_format = tu_aspects_to_plane(
      dst_image->vk.format, info->dstSubresource.aspectMask);
   trace_start_blit(&cmd->trace, cs,
                  ops == &r3d_ops<CHIP>,
                  src_image->vk.format,
                  dst_image->vk.format,
                  layers);

   ops->setup(cmd, cs, src_format, dst_format, info->dstSubresource.aspectMask,
              blit_param, false, dst_image->layout[0].ubwc,
              (VkSampleCountFlagBits) dst_image->layout[0].nr_samples);

   if (ops == &r3d_ops<CHIP>) {
      const float coords[] = { info->dstOffsets[0].x, info->dstOffsets[0].y,
                               info->srcOffsets[0].x, info->srcOffsets[0].y,
                               info->dstOffsets[1].x, info->dstOffsets[1].y,
                               info->srcOffsets[1].x, info->srcOffsets[1].y };
      r3d_coords_raw(cmd, cs, coords);
   } else {
      tu_cs_emit_regs(cs,
         A6XX_GRAS_2D_DST_TL(.x = MIN2(info->dstOffsets[0].x, info->dstOffsets[1].x),
                             .y = MIN2(info->dstOffsets[0].y, info->dstOffsets[1].y)),
         A6XX_GRAS_2D_DST_BR(.x = MAX2(info->dstOffsets[0].x, info->dstOffsets[1].x) - 1,
                             .y = MAX2(info->dstOffsets[0].y, info->dstOffsets[1].y) - 1));
      tu_cs_emit_regs(cs,
         A6XX_GRAS_2D_SRC_TL_X(MIN2(info->srcOffsets[0].x, info->srcOffsets[1].x)),
         A6XX_GRAS_2D_SRC_BR_X(MAX2(info->srcOffsets[0].x, info->srcOffsets[1].x) - 1),
         A6XX_GRAS_2D_SRC_TL_Y(MIN2(info->srcOffsets[0].y, info->srcOffsets[1].y)),
         A6XX_GRAS_2D_SRC_BR_Y(MAX2(info->srcOffsets[0].y, info->srcOffsets[1].y) - 1));
   }

   struct fdl6_view dst, src;
   tu_image_view_blit<CHIP>(
      &dst, dst_image, &info->dstSubresource,
      MIN2(info->dstOffsets[0].z, info->dstOffsets[1].z));

   if (z_scale) {
      tu_image_view_copy_blit<CHIP>(&src, src_image, src_format,
                                    &info->srcSubresource, 0, true);
      ops->src(cmd, cs, &src, 0, filter, dst_format);
   } else {
      tu_image_view_blit<CHIP>(&src, src_image, &info->srcSubresource, info->srcOffsets[0].z);
   }

   for (uint32_t i = 0; i < layers; i++) {
      if (z_scale) {
         float t = ((float) i + 0.5f) / (float) layers;
         r3d_coord_z(cmd, cs, t * (src1_z - src0_z) + src0_z);
      } else {
         ops->src(cmd, cs, &src, i, filter, dst_format);
      }
      ops->dst(cs, &dst, i, src_format);
      ops->run(cmd, cs);
   }

   ops->teardown(cmd, cs);

   trace_end_blit(&cmd->trace, cs);
}

template <chip CHIP>
VKAPI_ATTR void VKAPI_CALL
tu_CmdBlitImage2(VkCommandBuffer commandBuffer,
                 const VkBlitImageInfo2 *pBlitImageInfo)

{
   VK_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
   VK_FROM_HANDLE(tu_image, src_image, pBlitImageInfo->srcImage);
   VK_FROM_HANDLE(tu_image, dst_image, pBlitImageInfo->dstImage);

   for (uint32_t i = 0; i < pBlitImageInfo->regionCount; ++i) {
      /* can't blit both depth and stencil at once with D32_S8
       * TODO: more advanced 3D blit path to support it instead?
       */
      if (src_image->vk.format == VK_FORMAT_D32_SFLOAT_S8_UINT ||
          dst_image->vk.format == VK_FORMAT_D32_SFLOAT_S8_UINT) {
         VkImageBlit2 region = pBlitImageInfo->pRegions[i];
         u_foreach_bit(b, region.dstSubresource.aspectMask) {
            region.srcSubresource.aspectMask = BIT(b);
            region.dstSubresource.aspectMask = BIT(b);
            tu6_blit_image<CHIP>(cmd, src_image, dst_image, &region, pBlitImageInfo->filter);
         }
         continue;
      }
      tu6_blit_image<CHIP>(cmd, src_image, dst_image, pBlitImageInfo->pRegions + i,
                     pBlitImageInfo->filter);
   }

   if (dst_image->lrz_height) {
      tu_disable_lrz<CHIP>(cmd, &cmd->cs, dst_image);
   }
}
TU_GENX(tu_CmdBlitImage2);

static void
copy_compressed(VkFormat format,
                VkOffset3D *offset,
                VkExtent3D *extent,
                uint32_t *width,
                uint32_t *height)
{
   if (!vk_format_is_compressed(format))
      return;

   uint32_t block_width = vk_format_get_blockwidth(format);
   uint32_t block_height = vk_format_get_blockheight(format);

   offset->x /= block_width;
   offset->y /= block_height;

   if (extent) {
      extent->width = DIV_ROUND_UP(extent->width, block_width);
      extent->height = DIV_ROUND_UP(extent->height, block_height);
   }
   if (width)
      *width = DIV_ROUND_UP(*width, block_width);
   if (height)
      *height = DIV_ROUND_UP(*height, block_height);
}

template <chip CHIP>
static void
tu_copy_buffer_to_image(struct tu_cmd_buffer *cmd,
                        struct tu_buffer *src_buffer,
                        struct tu_image *dst_image,
                        const VkBufferImageCopy2 *info)
{
   struct tu_cs *cs = &cmd->cs;
   uint32_t layers = MAX2(info->imageExtent.depth,
                          vk_image_subresource_layer_count(&dst_image->vk,
                                                           &info->imageSubresource));
   enum pipe_format src_format =
      copy_format(dst_image->vk.format, info->imageSubresource.aspectMask);
   enum pipe_format dst_format =
      copy_format(dst_image->vk.format, info->imageSubresource.aspectMask);
   const struct blit_ops *ops = &r2d_ops<CHIP>;

   /* special case for buffer to stencil */
   if (dst_image->vk.format == VK_FORMAT_D24_UNORM_S8_UINT &&
       info->imageSubresource.aspectMask == VK_IMAGE_ASPECT_STENCIL_BIT) {
      src_format = PIPE_FORMAT_S8_UINT;
   }

   /* note: could use "R8_UNORM" when no UBWC */
   bool has_unaligned = CHIP >= A7XX; /* If unaligned buffer copies are supported. */
   unsigned blit_param = 0;
   if (src_format == PIPE_FORMAT_Y8_UNORM ||
       tu_pipe_format_is_float16(src_format)) {
      ops = &r3d_ops<CHIP>;
      blit_param = R3D_COPY;
      has_unaligned = false;
   }

   VkOffset3D offset = info->imageOffset;
   VkExtent3D extent = info->imageExtent;
   uint32_t src_width = info->bufferRowLength ?: extent.width;
   uint32_t src_height = info->bufferImageHeight ?: extent.height;

   copy_compressed(dst_image->vk.format, &offset, &extent, &src_width, &src_height);

   uint32_t pitch = src_width * util_format_get_blocksize(src_format);
   uint32_t layer_size = src_height * pitch;

   ops->setup(cmd, cs, src_format, dst_format,
              info->imageSubresource.aspectMask, blit_param, false, dst_image->layout[0].ubwc,
              (VkSampleCountFlagBits) dst_image->layout[0].nr_samples);

   struct fdl6_view dst;
   tu_image_view_copy<CHIP>(&dst, dst_image, dst_format,
                            &info->imageSubresource, offset.z);

   for (uint32_t i = 0; i < layers; i++) {
      ops->dst(cs, &dst, i, src_format);

      uint64_t src_va = src_buffer->iova + info->bufferOffset + layer_size * i;
      bool unaligned = (src_va & 63) || (pitch & 63);
      if (!has_unaligned && unaligned) {
         for (uint32_t y = 0; y < extent.height; y++) {
            uint32_t x = (src_va & 63) / util_format_get_blocksize(src_format);
            ops->src_buffer(cmd, cs, src_format, src_va & ~63, pitch,
                            x + extent.width, 1, dst_format);
            ops->coords(cmd, cs, (VkOffset2D) {offset.x, offset.y + y},  (VkOffset2D) {x},
                        (VkExtent2D) {extent.width, 1});
            ops->run(cmd, cs);
            src_va += pitch;
         }
      } else {
         if constexpr (CHIP >= A7XX) {
            /* Necessary to not trigger static assertion from A6XX variant. */
            if (has_unaligned) {
               r2d_src_buffer_unaligned<CHIP>(cmd, cs, src_format, src_va,
                                              pitch, extent.width,
                                              extent.height, dst_format);
            } else {
               ops->src_buffer(cmd, cs, src_format, src_va, pitch,
                               extent.width, extent.height, dst_format);
            }
         } else {
            ops->src_buffer(cmd, cs, src_format, src_va, pitch, extent.width,
                            extent.height, dst_format);
         }
         coords(ops, cmd, cs, offset, (VkOffset3D) {}, extent);
         ops->run(cmd, cs);
      }
   }

   ops->teardown(cmd, cs);
}

template <chip CHIP>
VKAPI_ATTR void VKAPI_CALL
tu_CmdCopyBufferToImage2(VkCommandBuffer commandBuffer,
                         const VkCopyBufferToImageInfo2 *pCopyBufferToImageInfo)
{
   VK_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
   VK_FROM_HANDLE(tu_image, dst_image, pCopyBufferToImageInfo->dstImage);
   VK_FROM_HANDLE(tu_buffer, src_buffer, pCopyBufferToImageInfo->srcBuffer);

   for (unsigned i = 0; i < pCopyBufferToImageInfo->regionCount; ++i)
      tu_copy_buffer_to_image<CHIP>(cmd, src_buffer, dst_image,
                              pCopyBufferToImageInfo->pRegions + i);

   if (dst_image->lrz_height) {
      tu_disable_lrz<CHIP>(cmd, &cmd->cs, dst_image);
   }
}
TU_GENX(tu_CmdCopyBufferToImage2);

static void
tu_copy_memory_to_image(struct tu_device *device,
                        struct tu_image *dst_image,
                        const VkMemoryToImageCopyEXT *info,
                        bool copy_memcpy)
{
   unsigned plane = tu6_plane_index(dst_image->vk.format,
                                    info->imageSubresource.aspectMask);
   const struct fdl_layout *layout = &dst_image->layout[plane];

   VkOffset3D offset = info->imageOffset;
   VkExtent3D extent = info->imageExtent;
   uint32_t src_width = info->memoryRowLength ?: extent.width;
   uint32_t src_height = info->memoryImageHeight ?: extent.height;

   copy_compressed(dst_image->vk.format, &offset, &extent, &src_width, &src_height);

   uint32_t src_pitch = src_width * layout->cpp;

   unsigned start_layer = (dst_image->vk.image_type == VK_IMAGE_TYPE_3D) ?
      offset.z : info->imageSubresource.baseArrayLayer;
   uint32_t layers = MAX2(extent.depth,
                          vk_image_subresource_layer_count(&dst_image->vk,
                                                           &info->imageSubresource));

   uint32_t image_offset =
      fdl_surface_offset(layout,
                         info->imageSubresource.mipLevel,
                         start_layer);

   uint32_t dst_layer_stride =
      fdl_layer_stride(layout, info->imageSubresource.mipLevel);
   uint32_t dst_layer_size =
      layout->slices[info->imageSubresource.mipLevel].size0;
   uint32_t src_layer_stride =
      copy_memcpy ? dst_layer_size :
      (src_width * src_height * layout->cpp);
   bool tiled =
      fdl_tile_mode(layout, info->imageSubresource.mipLevel) != 0;

   const char *src = (const char *) info->pHostPointer;
   char *dst = (char *) dst_image->map + image_offset;
   for (unsigned layer = 0; layer < layers; layer++,
        src += src_layer_stride, dst += dst_layer_stride) {
      if (copy_memcpy) {
         memcpy(dst, src, src_layer_stride);
      } else if (!tiled) {
         uint32_t dst_pitch = fdl_pitch(layout,
                                        info->imageSubresource.mipLevel);
         for (unsigned y = 0; y < extent.height; y++) {
            memcpy(dst + dst_pitch * (y + offset.y) + offset.x * layout->cpp,
                   src + src_pitch * y,
                   extent.width * layout->cpp);
         }
      } else {
         fdl6_memcpy_linear_to_tiled(offset.x, offset.y,
                                     extent.width, extent.height,
                                     dst, src, layout,
                                     info->imageSubresource.mipLevel,
                                     src_pitch,
                                     &device->physical_device->ubwc_config);
      }

      if (dst_image->bo->cached_non_coherent) {
         tu_bo_sync_cache(device, dst_image->bo,
                          dst_image->bo_offset + image_offset,
                          dst_layer_size, TU_MEM_SYNC_CACHE_TO_GPU);
      }
   }
}

VKAPI_ATTR VkResult VKAPI_CALL
tu_CopyMemoryToImageEXT(VkDevice _device,
                        const VkCopyMemoryToImageInfoEXT *info)
{
   VK_FROM_HANDLE(tu_device, device, _device);
   VK_FROM_HANDLE(tu_image, dst_image, info->dstImage);

   for (unsigned i = 0; i < info->regionCount; i++) {
      tu_copy_memory_to_image(device, dst_image, &info->pRegions[i],
                              info->flags & VK_HOST_IMAGE_COPY_MEMCPY_EXT);
   }

   if (dst_image->lrz_height) {
      TU_CALLX(device, tu_disable_lrz_cpu)(device, dst_image);
   }

   return VK_SUCCESS;
}

template <chip CHIP>
static void
tu_copy_image_to_buffer(struct tu_cmd_buffer *cmd,
                        struct tu_image *src_image,
                        struct tu_buffer *dst_buffer,
                        const VkBufferImageCopy2 *info,
                        bool *unaligned_store)
{
   struct tu_cs *cs = &cmd->cs;
   uint32_t layers = MAX2(info->imageExtent.depth,
                          vk_image_subresource_layer_count(&src_image->vk,
                                                           &info->imageSubresource));
   enum pipe_format dst_format =
      copy_format(src_image->vk.format, info->imageSubresource.aspectMask);
   enum pipe_format src_format =
      copy_format(src_image->vk.format, info->imageSubresource.aspectMask);
   const struct blit_ops *ops = &r2d_ops<CHIP>;

   if (src_image->vk.format == VK_FORMAT_D24_UNORM_S8_UINT &&
       info->imageSubresource.aspectMask == VK_IMAGE_ASPECT_STENCIL_BIT) {
      dst_format = PIPE_FORMAT_S8_UINT;
   }

   /* note: could use "R8_UNORM" when no UBWC */
   unsigned blit_param = 0;
   if (dst_format == PIPE_FORMAT_Y8_UNORM ||
       tu_pipe_format_is_float16(src_format)) {
      ops = &r3d_ops<CHIP>;
      blit_param = R3D_COPY;
   }

   VkOffset3D offset = info->imageOffset;
   VkExtent3D extent = info->imageExtent;
   uint32_t dst_width = info->bufferRowLength ?: extent.width;
   uint32_t dst_height = info->bufferImageHeight ?: extent.height;

   copy_compressed(src_image->vk.format, &offset, &extent, &dst_width, &dst_height);

   uint32_t pitch = dst_width * util_format_get_blocksize(dst_format);
   uint32_t layer_size = pitch * dst_height;

   handle_buffer_unaligned_store<CHIP>(cmd,
                                       dst_buffer->iova + info->bufferOffset,
                                       layer_size * layers, unaligned_store);

   ops->setup(cmd, cs, src_format, dst_format, VK_IMAGE_ASPECT_COLOR_BIT, blit_param, false, false,
              VK_SAMPLE_COUNT_1_BIT);

   struct fdl6_view src;
   tu_image_view_copy<CHIP>(&src, src_image, src_format,
                            &info->imageSubresource, offset.z);

   for (uint32_t i = 0; i < layers; i++) {
      ops->src(cmd, cs, &src, i, VK_FILTER_NEAREST, dst_format);

      uint64_t dst_va = dst_buffer->iova + info->bufferOffset + layer_size * i;
      if ((dst_va & 63) || (pitch & 63)) {
         for (uint32_t y = 0; y < extent.height; y++) {
            uint32_t x = (dst_va & 63) / util_format_get_blocksize(dst_format);
            ops->dst_buffer(cs, dst_format, dst_va & ~63, 0, src_format);
            ops->coords(cmd, cs, (VkOffset2D) {x}, (VkOffset2D) {offset.x, offset.y + y},
                        (VkExtent2D) {extent.width, 1});
            ops->run(cmd, cs);
            dst_va += pitch;
         }
      } else {
         ops->dst_buffer(cs, dst_format, dst_va, pitch, src_format);
         coords(ops, cmd, cs, (VkOffset3D) {0, 0}, offset, extent);
         ops->run(cmd, cs);
      }
   }

   ops->teardown(cmd, cs);
}

template <chip CHIP>
VKAPI_ATTR void VKAPI_CALL
tu_CmdCopyImageToBuffer2(VkCommandBuffer commandBuffer,
                         const VkCopyImageToBufferInfo2 *pCopyImageToBufferInfo)
{
   VK_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
   VK_FROM_HANDLE(tu_image, src_image, pCopyImageToBufferInfo->srcImage);
   VK_FROM_HANDLE(tu_buffer, dst_buffer, pCopyImageToBufferInfo->dstBuffer);

   bool unaligned_store = false;
   for (unsigned i = 0; i < pCopyImageToBufferInfo->regionCount; ++i)
      tu_copy_image_to_buffer<CHIP>(cmd, src_image, dst_buffer,
                              pCopyImageToBufferInfo->pRegions + i,
                              &unaligned_store);

   after_buffer_unaligned_buffer_store<CHIP>(cmd, unaligned_store);
}
TU_GENX(tu_CmdCopyImageToBuffer2);

static void
tu_copy_image_to_memory(struct tu_device *device,
                        struct tu_image *src_image,
                        const VkImageToMemoryCopyEXT *info,
                        bool copy_memcpy)
{
   unsigned plane = tu6_plane_index(src_image->vk.format,
                                    info->imageSubresource.aspectMask);
   const struct fdl_layout *layout = &src_image->layout[plane];

   VkOffset3D offset = info->imageOffset;
   VkExtent3D extent = info->imageExtent;
   uint32_t dst_width = info->memoryRowLength ?: extent.width;
   uint32_t dst_height = info->memoryImageHeight ?: extent.height;

   copy_compressed(src_image->vk.format, &offset, &extent, &dst_width, &dst_height);

   uint32_t dst_pitch = dst_width * layout->cpp;

   unsigned start_layer = (src_image->vk.image_type == VK_IMAGE_TYPE_3D) ?
      offset.z : info->imageSubresource.baseArrayLayer;
   uint32_t layers = MAX2(extent.depth,
                          vk_image_subresource_layer_count(&src_image->vk,
                                                           &info->imageSubresource));

   uint32_t image_offset =
      fdl_surface_offset(layout,
                         info->imageSubresource.mipLevel,
                         start_layer);

   uint32_t src_layer_stride =
      fdl_layer_stride(layout, info->imageSubresource.mipLevel);
   uint32_t src_layer_size =
      layout->slices[info->imageSubresource.mipLevel].size0;
   uint32_t dst_layer_stride =
      copy_memcpy ? src_layer_size : (dst_width * dst_height * layout->cpp);
   bool tiled =
      fdl_tile_mode(layout, info->imageSubresource.mipLevel) != 0;

   const char *src = (const char *) src_image->map + image_offset;
   char *dst = (char *) info->pHostPointer;
   for (unsigned layer = 0; layer < layers; layer++,
        src += src_layer_stride, dst += dst_layer_stride) {
      if (src_image->bo->cached_non_coherent) {
         tu_bo_sync_cache(device, src_image->bo,
                          src_image->bo_offset + image_offset,
                          src_layer_size, TU_MEM_SYNC_CACHE_FROM_GPU);
      }

      if (copy_memcpy) {
         memcpy(dst, src, dst_layer_stride);
      } else if (!tiled) {
         uint32_t src_pitch = fdl_pitch(layout,
                                        info->imageSubresource.mipLevel);
         for (unsigned y = 0; y < extent.height; y++) {
            memcpy(dst + dst_pitch * y,
                   src + src_pitch * (y + offset.y) + offset.x * layout->cpp,
                   extent.width * layout->cpp);
         }
      } else {
         fdl6_memcpy_tiled_to_linear(offset.x, offset.y,
                                     extent.width, extent.height,
                                     dst, src, layout,
                                     info->imageSubresource.mipLevel,
                                     dst_pitch,
                                     &device->physical_device->ubwc_config);
      }
   }
}

VKAPI_ATTR VkResult VKAPI_CALL
tu_CopyImageToMemoryEXT(VkDevice _device,
                        const VkCopyImageToMemoryInfoEXT *info)
{
   VK_FROM_HANDLE(tu_device, device, _device);
   VK_FROM_HANDLE(tu_image, image, info->srcImage);

   for (unsigned i = 0; i < info->regionCount; i++) {
      tu_copy_image_to_memory(device, image, &info->pRegions[i],
                              info->flags & VK_HOST_IMAGE_COPY_MEMCPY_EXT);
   }

   return VK_SUCCESS;
}


/* Tiled formats don't support swapping, which means that we can't support
 * formats that require a non-WZYX swap like B8G8R8A8 natively. Also, some
 * formats like B5G5R5A1 have a separate linear-only format when sampling.
 * Currently we fake support for tiled swapped formats and use the unswapped
 * format instead, but this means that reinterpreting copies to and from
 * swapped formats can't be performed correctly unless we can swizzle the
 * components by reinterpreting the other image as the "correct" swapped
 * format, i.e. only when the other image is linear.
 */

template <chip CHIP>
static bool
is_swapped_format(enum pipe_format format, bool is_mutable)
{
   struct tu_native_format linear = blit_format_texture<CHIP>(format, TILE6_LINEAR, is_mutable, false);
   struct tu_native_format tiled = blit_format_texture<CHIP>(format, TILE6_3, is_mutable, false);
   return linear.fmt != tiled.fmt || linear.swap != tiled.swap;
}

/* R8G8_* formats have a different tiling layout than other cpp=2 formats, and
 * therefore R8G8 images can't be reinterpreted as non-R8G8 images (and vice
 * versa). This should mirror the logic in fdl6_layout.
 */
static bool
image_is_r8g8(struct tu_image *image)
{
   return image->layout[0].cpp == 2 &&
      vk_format_get_nr_components(image->vk.format) == 2;
}

template <chip CHIP>
static void
tu_copy_image_to_image(struct tu_cmd_buffer *cmd,
                       struct tu_image *src_image,
                       struct tu_image *dst_image,
                       const VkImageCopy2 *info)
{
   const struct blit_ops *ops = &r2d_ops<CHIP>;
   struct tu_cs *cs = &cmd->cs;

   if (dst_image->layout[0].nr_samples > 1)
      ops = &r3d_ops<CHIP>;

   enum pipe_format format = PIPE_FORMAT_NONE;
   VkOffset3D src_offset = info->srcOffset;
   VkOffset3D dst_offset = info->dstOffset;
   VkExtent3D extent = info->extent;
   uint32_t layers_to_copy = MAX2(info->extent.depth,
                                  vk_image_subresource_layer_count(&src_image->vk,
                                                                   &info->srcSubresource));

   /* From the Vulkan 1.2.140 spec, section 19.3 "Copying Data Between
    * Images":
    *
    *    When copying between compressed and uncompressed formats the extent
    *    members represent the texel dimensions of the source image and not
    *    the destination. When copying from a compressed image to an
    *    uncompressed image the image texel dimensions written to the
    *    uncompressed image will be source extent divided by the compressed
    *    texel block dimensions. When copying from an uncompressed image to a
    *    compressed image the image texel dimensions written to the compressed
    *    image will be the source extent multiplied by the compressed texel
    *    block dimensions.
    *
    * This means we only have to adjust the extent if the source image is
    * compressed.
    */
   copy_compressed(src_image->vk.format, &src_offset, &extent, NULL, NULL);
   copy_compressed(dst_image->vk.format, &dst_offset, NULL, NULL, NULL);

   enum pipe_format dst_format = copy_format(dst_image->vk.format, info->dstSubresource.aspectMask);
   enum pipe_format src_format = copy_format(src_image->vk.format, info->srcSubresource.aspectMask);

   /* note: could use "R8_UNORM" when no UBWC */
   unsigned blit_param = 0;
   if (dst_format == PIPE_FORMAT_Y8_UNORM ||
       src_format == PIPE_FORMAT_Y8_UNORM ||
       tu_pipe_format_is_float16(src_format) ||
       tu_pipe_format_is_float16(dst_format)) {
      ops = &r3d_ops<CHIP>;
      blit_param = R3D_COPY;
   }

   bool use_staging_blit = false;

   if (src_format == dst_format) {
      /* Images that share a format can always be copied directly because it's
       * the same as a blit.
       */
      format = src_format;
   } else if (!src_image->layout[0].tile_mode) {
      /* If an image is linear, we can always safely reinterpret it with the
       * other image's format and then do a regular blit.
       */
      format = dst_format;
   } else if (!dst_image->layout[0].tile_mode) {
      format = src_format;
   } else if (image_is_r8g8(src_image) != image_is_r8g8(dst_image)) {
      /* We can't currently copy r8g8 images to/from other cpp=2 images,
       * due to the different tile layout.
       */
      use_staging_blit = true;
   } else if (is_swapped_format<CHIP>(src_format,
                                      src_image->layout[0].is_mutable) ||
              is_swapped_format<CHIP>(dst_format,
                                      src_image->layout[0].is_mutable)) {
      /* If either format has a non-identity swap, then we can't copy
       * to/from it.
       */
      use_staging_blit = true;
   } else if (!src_image->layout[0].ubwc || src_image->layout[0].is_mutable) {
      format = dst_format;
   } else if (!dst_image->layout[0].ubwc || src_image->layout[0].is_mutable) {
      format = src_format;
   } else {
      /* Both formats use UBWC and so neither can be reinterpreted.
       * TODO: We could do an in-place decompression of the dst instead.
       */
      perf_debug(cmd->device, "TODO: Do in-place UBWC decompression for UBWC->UBWC blits");
      use_staging_blit = true;
   }

   struct fdl6_view dst, src;

   if (use_staging_blit) {
      tu_image_view_copy<CHIP>(&dst, dst_image, dst_format, &info->dstSubresource, dst_offset.z);
      tu_image_view_copy<CHIP>(&src, src_image, src_format, &info->srcSubresource, src_offset.z);

      struct fdl_layout staging_layout = { 0 };
      VkOffset3D staging_offset = { 0 };

      staging_layout.tile_mode = TILE6_LINEAR;
      staging_layout.ubwc = false;

      uint32_t layer_count =
         vk_image_subresource_layer_count(&src_image->vk,
                                          &info->srcSubresource);
      fdl6_layout(&staging_layout,
                  &cmd->device->physical_device->dev_info,
                  src_format,
                  src_image->layout[0].nr_samples,
                  extent.width,
                  extent.height,
                  extent.depth,
                  1,
                  layer_count,
                  extent.depth > 1,
                  false,
                  NULL);

      struct tu_bo *staging_bo;
      VkResult result = tu_get_scratch_bo(cmd->device,
                                          staging_layout.size,
                                          &staging_bo);
      if (result != VK_SUCCESS) {
         vk_command_buffer_set_error(&cmd->vk, result);
         return;
      }

      struct fdl6_view staging;
      const struct fdl_layout *staging_layout_ptr = &staging_layout;
      const struct fdl_view_args copy_to_args = {
         .chip = CHIP,
         .iova = staging_bo->iova,
         .base_miplevel = 0,
         .level_count = 1,
         .base_array_layer = 0,
         .layer_count = layer_count,
         .swiz = { PIPE_SWIZZLE_X, PIPE_SWIZZLE_Y, PIPE_SWIZZLE_Z, PIPE_SWIZZLE_W },
         .format = tu_format_for_aspect(src_format, VK_IMAGE_ASPECT_COLOR_BIT),
         .type = FDL_VIEW_TYPE_2D,
      };
      fdl6_view_init(&staging, &staging_layout_ptr, &copy_to_args, false);

      ops->setup(cmd, cs, src_format, src_format, VK_IMAGE_ASPECT_COLOR_BIT, blit_param, false, false,
                 (VkSampleCountFlagBits) dst_image->layout[0].nr_samples);
      coords(ops, cmd, cs, staging_offset, src_offset, extent);

      for (uint32_t i = 0; i < layers_to_copy; i++) {
         ops->src(cmd, cs, &src, i, VK_FILTER_NEAREST, src_format);
         ops->dst(cs, &staging, i, src_format);
         ops->run(cmd, cs);
      }

      /* When executed by the user there has to be a pipeline barrier here,
       * but since we're doing it manually we'll have to flush ourselves.
       */
      tu_emit_event_write<CHIP>(cmd, cs, FD_CCU_CLEAN_COLOR);
      tu_emit_event_write<CHIP>(cmd, cs, FD_CACHE_INVALIDATE);
      tu_cs_emit_wfi(cs);

      const struct fdl_view_args copy_from_args = {
         .chip = CHIP,
         .iova = staging_bo->iova,
         .base_miplevel = 0,
         .level_count = 1,
         .base_array_layer = 0,
         .layer_count = layer_count,
         .swiz = { PIPE_SWIZZLE_X, PIPE_SWIZZLE_Y, PIPE_SWIZZLE_Z, PIPE_SWIZZLE_W },
         .format = tu_format_for_aspect(dst_format, VK_IMAGE_ASPECT_COLOR_BIT),
         .type = FDL_VIEW_TYPE_2D,
      };
      fdl6_view_init(&staging, &staging_layout_ptr, &copy_from_args, false);

      ops->setup(cmd, cs, dst_format, dst_format, info->dstSubresource.aspectMask,
                 blit_param, false, dst_image->layout[0].ubwc,
                 (VkSampleCountFlagBits) dst_image->layout[0].nr_samples);
      coords(ops, cmd, cs, dst_offset, staging_offset, extent);

      for (uint32_t i = 0; i < layers_to_copy; i++) {
         ops->src(cmd, cs, &staging, i, VK_FILTER_NEAREST, dst_format);
         ops->dst(cs, &dst, i, dst_format);
         ops->run(cmd, cs);
      }
   } else {
      tu_image_view_copy<CHIP>(&dst, dst_image, format, &info->dstSubresource, dst_offset.z);
      tu_image_view_copy<CHIP>(&src, src_image, format, &info->srcSubresource, src_offset.z);

      ops->setup(cmd, cs, format, format, info->dstSubresource.aspectMask,
                 blit_param, false, dst_image->layout[0].ubwc,
                 (VkSampleCountFlagBits) dst_image->layout[0].nr_samples);
      coords(ops, cmd, cs, dst_offset, src_offset, extent);

      for (uint32_t i = 0; i < layers_to_copy; i++) {
         ops->src(cmd, cs, &src, i, VK_FILTER_NEAREST, format);
         ops->dst(cs, &dst, i, format);
         ops->run(cmd, cs);
      }
   }

   ops->teardown(cmd, cs);
}

template <chip CHIP>
VKAPI_ATTR void VKAPI_CALL
tu_CmdCopyImage2(VkCommandBuffer commandBuffer,
                 const VkCopyImageInfo2 *pCopyImageInfo)
{
   VK_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
   VK_FROM_HANDLE(tu_image, src_image, pCopyImageInfo->srcImage);
   VK_FROM_HANDLE(tu_image, dst_image, pCopyImageInfo->dstImage);

   for (uint32_t i = 0; i < pCopyImageInfo->regionCount; ++i) {
      if (src_image->vk.format == VK_FORMAT_D32_SFLOAT_S8_UINT) {
         VkImageCopy2 info = pCopyImageInfo->pRegions[i];
         u_foreach_bit(b, info.dstSubresource.aspectMask) {
            info.srcSubresource.aspectMask = BIT(b);
            info.dstSubresource.aspectMask = BIT(b);
            tu_copy_image_to_image<CHIP>(cmd, src_image, dst_image, &info);
         }
         continue;
      }

      tu_copy_image_to_image<CHIP>(cmd, src_image, dst_image,
                             pCopyImageInfo->pRegions + i);
   }

   if (dst_image->lrz_height) {
      tu_disable_lrz<CHIP>(cmd, &cmd->cs, dst_image);
   }
}
TU_GENX(tu_CmdCopyImage2);

static void
tu_copy_image_to_image_cpu(struct tu_device *device,
                           struct tu_image *src_image,
                           struct tu_image *dst_image,
                           const VkImageCopy2 *info,
                           bool copy_memcpy)
{
   unsigned src_plane = tu6_plane_index(src_image->vk.format,
                                        info->srcSubresource.aspectMask);
   unsigned dst_plane = tu6_plane_index(dst_image->vk.format,
                                        info->dstSubresource.aspectMask);

   const struct fdl_layout *src_layout = &src_image->layout[src_plane];
   const struct fdl_layout *dst_layout = &dst_image->layout[dst_plane];

   VkOffset3D src_offset = info->srcOffset;
   VkOffset3D dst_offset = info->dstOffset;
   VkExtent3D extent = info->extent;
   uint32_t layers_to_copy = MAX2(info->extent.depth,
                                  vk_image_subresource_layer_count(&src_image->vk,
                                                                   &info->srcSubresource));

   /* See comment above. */
   copy_compressed(src_image->vk.format, &src_offset, &extent, NULL, NULL);
   copy_compressed(dst_image->vk.format, &dst_offset, NULL, NULL, NULL);

   unsigned src_start_layer = (src_image->vk.image_type == VK_IMAGE_TYPE_3D) ?
      src_offset.z : info->srcSubresource.baseArrayLayer;
   unsigned dst_start_layer = (dst_image->vk.image_type == VK_IMAGE_TYPE_3D) ?
      dst_offset.z : info->dstSubresource.baseArrayLayer;

   uint32_t src_layer_stride =
      fdl_layer_stride(src_layout, info->srcSubresource.mipLevel);
   uint32_t src_layer_size =
      src_layout->slices[info->srcSubresource.mipLevel].size0;
   uint32_t dst_layer_stride =
      fdl_layer_stride(dst_layout, info->dstSubresource.mipLevel);
   uint32_t dst_layer_size =
      dst_layout->slices[info->dstSubresource.mipLevel].size0;

   uint32_t src_image_offset =
      fdl_surface_offset(src_layout,
                         info->srcSubresource.mipLevel,
                         src_start_layer);
   uint32_t dst_image_offset =
      fdl_surface_offset(dst_layout,
                         info->dstSubresource.mipLevel,
                         dst_start_layer);

   bool src_tiled =
      fdl_tile_mode(src_layout, info->srcSubresource.mipLevel) != 0;
   bool dst_tiled =
      fdl_tile_mode(dst_layout, info->dstSubresource.mipLevel) != 0;

   const char *src = (const char *) src_image->map + src_image_offset;
   char *dst = (char *) dst_image->map + dst_image_offset;
   for (unsigned layer = 0; layer < layers_to_copy; layer++,
        src += src_layer_stride, dst += dst_layer_stride) {
      if (src_image->bo->cached_non_coherent) {
         tu_bo_sync_cache(device, src_image->bo,
                          src_image->bo_offset + src_image_offset,
                          src_layer_size, TU_MEM_SYNC_CACHE_FROM_GPU);
      }

      uint32_t src_pitch = fdl_pitch(src_layout,
                                     info->srcSubresource.mipLevel);
      uint32_t dst_pitch = fdl_pitch(dst_layout,
                                     info->dstSubresource.mipLevel);

      if (copy_memcpy) {
         assert(src_layer_size == dst_layer_size);
         memcpy(dst, src, src_layer_size);
      } else if (!src_tiled && !dst_tiled) {
         for (unsigned y = 0; y < extent.height; y++) {
            memcpy(dst + dst_pitch * (y + dst_offset.y) + dst_offset.x * dst_layout->cpp,
                   src + src_pitch * (y + src_offset.y) + src_offset.x * src_layout->cpp,
                   extent.width * src_layout->cpp);
         }
      } else if (!src_tiled) {
         fdl6_memcpy_linear_to_tiled(dst_offset.x, dst_offset.y,
                                     extent.width, extent.height,
                                     dst,
                                     src + src_pitch * src_offset.y + src_offset.x * src_layout->cpp,
                                     dst_layout,
                                     info->dstSubresource.mipLevel,
                                     src_pitch,
                                     &device->physical_device->ubwc_config);
      } else if (!dst_tiled) {
         fdl6_memcpy_tiled_to_linear(src_offset.x, src_offset.y,
                                     extent.width, extent.height,
                                     dst + dst_pitch * dst_offset.y + dst_offset.x * dst_layout->cpp,
                                     src,
                                     src_layout,
                                     info->dstSubresource.mipLevel,
                                     dst_pitch,
                                     &device->physical_device->ubwc_config);
      } else {
         /* Work tile-by-tile, holding the unswizzled tile in a temporary
          * buffer.
          */
         char temp_tile[256];

         uint32_t block_width, block_height;
         fdl6_get_ubwc_blockwidth(src_layout, &block_width, &block_height);

         uint32_t temp_pitch = block_width * src_layout->cpp;

         for (unsigned by = src_offset.y / block_height;
              by * block_height < src_offset.y + extent.height; by++) {
            uint32_t src_y_start = MAX2(src_offset.y, by * block_height);
            uint32_t dst_y_start = src_y_start - src_offset.y + dst_offset.y;
            uint32_t height =
               MIN2((by + 1) * block_height, src_offset.y + extent.height) -
               src_y_start;
            for (unsigned bx = src_offset.x / block_width;
                 bx * block_width < src_offset.x + extent.width; bx++) {
               uint32_t src_x_start = MAX2(src_offset.x, bx * block_width);
               uint32_t dst_x_start = src_x_start - src_offset.x + dst_offset.x;
               uint32_t width =
                  MIN2((bx + 1) * block_width, src_offset.x + extent.width) -
                  src_x_start;

               fdl6_memcpy_tiled_to_linear(src_x_start, src_y_start,
                                           width, height,
                                           temp_tile, src, src_layout,
                                           info->srcSubresource.mipLevel,
                                           temp_pitch,
                                           &device->physical_device->ubwc_config);
               fdl6_memcpy_linear_to_tiled(dst_x_start, dst_y_start,
                                           width, height,
                                           dst, temp_tile, dst_layout,
                                           info->dstSubresource.mipLevel,
                                           temp_pitch,
                                           &device->physical_device->ubwc_config);
            }
         }
      }

      if (dst_image->bo->cached_non_coherent) {
         tu_bo_sync_cache(device, dst_image->bo,
                          dst_image->bo_offset + dst_image_offset,
                          dst_layer_size, TU_MEM_SYNC_CACHE_TO_GPU);
      }
   }
}

VKAPI_ATTR VkResult VKAPI_CALL
tu_CopyImageToImageEXT(VkDevice _device,
                       const VkCopyImageToImageInfoEXT *pCopyImageToImageInfo)
{
   VK_FROM_HANDLE(tu_device, device, _device);
   VK_FROM_HANDLE(tu_image, src_image, pCopyImageToImageInfo->srcImage);
   VK_FROM_HANDLE(tu_image, dst_image, pCopyImageToImageInfo->dstImage);
   bool copy_memcpy = pCopyImageToImageInfo->flags &
      VK_HOST_IMAGE_COPY_MEMCPY_EXT;

   for (uint32_t i = 0; i < pCopyImageToImageInfo->regionCount; ++i) {
      if (src_image->vk.format == VK_FORMAT_D32_SFLOAT_S8_UINT) {
         VkImageCopy2 info = pCopyImageToImageInfo->pRegions[i];
         u_foreach_bit(b, info.dstSubresource.aspectMask) {
            info.srcSubresource.aspectMask = BIT(b);
            info.dstSubresource.aspectMask = BIT(b);
            tu_copy_image_to_image_cpu(device, src_image, dst_image, &info,
                                       copy_memcpy);
         }
         continue;
      }

      tu_copy_image_to_image_cpu(device, src_image, dst_image,
                                 pCopyImageToImageInfo->pRegions + i,
                                 copy_memcpy);
   }

   if (dst_image->lrz_height) {
      TU_CALLX(device, tu_disable_lrz_cpu)(device, dst_image);
   }

   return VK_SUCCESS;
}

VKAPI_ATTR VkResult VKAPI_CALL
tu_TransitionImageLayoutEXT(VkDevice device,
                            uint32_t transitionCount,
                            const VkHostImageLayoutTransitionInfoEXT *transitions)
{
   /* We don't do anything with layouts so this should be a no-op */
   return VK_SUCCESS;
}

template <chip CHIP>
static void
copy_buffer(struct tu_cmd_buffer *cmd,
            uint64_t dst_va,
            uint64_t src_va,
            uint64_t size,
            uint32_t block_size,
            bool *unaligned_store)
{
   const struct blit_ops *ops = &r2d_ops<CHIP>;
   struct tu_cs *cs = &cmd->cs;
   enum pipe_format format = block_size == 4 ? PIPE_FORMAT_R32_UINT : PIPE_FORMAT_R8_UNORM;
   uint64_t blocks = size / block_size;

   handle_buffer_unaligned_store<CHIP>(cmd, dst_va, size, unaligned_store);

   ops->setup(cmd, cs, format, format, VK_IMAGE_ASPECT_COLOR_BIT, 0, false, false,
              VK_SAMPLE_COUNT_1_BIT);

   while (blocks) {
      uint32_t src_x = (src_va & 63) / block_size;
      uint32_t dst_x = (dst_va & 63) / block_size;
      uint32_t width = MIN2(MIN2(blocks, 0x4000 - src_x), 0x4000 - dst_x);

      ops->src_buffer(cmd, cs, format, src_va & ~63, 0, src_x + width, 1, format);
      ops->dst_buffer(     cs, format, dst_va & ~63, 0, format);
      ops->coords(cmd, cs, (VkOffset2D) {dst_x}, (VkOffset2D) {src_x}, (VkExtent2D) {width, 1});
      ops->run(cmd, cs);

      src_va += width * block_size;
      dst_va += width * block_size;
      blocks -= width;
   }

   ops->teardown(cmd, cs);
}

template <chip CHIP>
VKAPI_ATTR void VKAPI_CALL
tu_CmdCopyBuffer2(VkCommandBuffer commandBuffer,
                  const VkCopyBufferInfo2 *pCopyBufferInfo)
{
   VK_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
   VK_FROM_HANDLE(tu_buffer, src_buffer, pCopyBufferInfo->srcBuffer);
   VK_FROM_HANDLE(tu_buffer, dst_buffer, pCopyBufferInfo->dstBuffer);

   bool unaligned_store = false;
   for (unsigned i = 0; i < pCopyBufferInfo->regionCount; ++i) {
      const VkBufferCopy2 *region = &pCopyBufferInfo->pRegions[i];
      copy_buffer<CHIP>(cmd,
                  dst_buffer->iova + region->dstOffset,
                  src_buffer->iova + region->srcOffset,
                  region->size, 1, &unaligned_store);
   }

   after_buffer_unaligned_buffer_store<CHIP>(cmd, unaligned_store);
}
TU_GENX(tu_CmdCopyBuffer2);

template <chip CHIP>
VKAPI_ATTR void VKAPI_CALL
tu_CmdUpdateBuffer(VkCommandBuffer commandBuffer,
                   VkBuffer dstBuffer,
                   VkDeviceSize dstOffset,
                   VkDeviceSize dataSize,
                   const void *pData)
{
   VK_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
   VK_FROM_HANDLE(tu_buffer, buffer, dstBuffer);

   struct tu_cs_memory tmp;
   VkResult result = tu_cs_alloc(&cmd->sub_cs, DIV_ROUND_UP(dataSize, 64), 64 / 4, &tmp);
   if (result != VK_SUCCESS) {
      vk_command_buffer_set_error(&cmd->vk, result);
      return;
   }

   bool unaligned_store = false;
   memcpy(tmp.map, pData, dataSize);
   copy_buffer<CHIP>(cmd, buffer->iova + dstOffset, tmp.iova, dataSize, 4, &unaligned_store);

   after_buffer_unaligned_buffer_store<CHIP>(cmd, unaligned_store);
}
TU_GENX(tu_CmdUpdateBuffer);

template <chip CHIP>
VKAPI_ATTR void VKAPI_CALL
tu_CmdFillBuffer(VkCommandBuffer commandBuffer,
                 VkBuffer dstBuffer,
                 VkDeviceSize dstOffset,
                 VkDeviceSize fillSize,
                 uint32_t data)
{
   VK_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
   VK_FROM_HANDLE(tu_buffer, buffer, dstBuffer);
   const struct blit_ops *ops = &r2d_ops<CHIP>;
   struct tu_cs *cs = &cmd->cs;

   fillSize = vk_buffer_range(&buffer->vk, dstOffset, fillSize);

   uint64_t dst_va = buffer->iova + dstOffset;
   uint32_t blocks = fillSize / 4;

   bool unaligned_store = false;
   handle_buffer_unaligned_store<CHIP>(cmd, dst_va, fillSize, &unaligned_store);

   ops->setup(cmd, cs, PIPE_FORMAT_R32_UINT, PIPE_FORMAT_R32_UINT,
              VK_IMAGE_ASPECT_COLOR_BIT, 0, true, false,
              VK_SAMPLE_COUNT_1_BIT);

   VkClearValue clear_val = {};
   clear_val.color.uint32[0] = data;
   ops->clear_value(cmd, cs, PIPE_FORMAT_R32_UINT, &clear_val);

   while (blocks) {
      uint32_t dst_x = (dst_va & 63) / 4;
      uint32_t width = MIN2(blocks, 0x4000 - dst_x);

      ops->dst_buffer(cs, PIPE_FORMAT_R32_UINT, dst_va & ~63, 0, PIPE_FORMAT_R32_UINT);
      ops->coords(cmd, cs, (VkOffset2D) {dst_x}, blt_no_coord, (VkExtent2D) {width, 1});
      ops->run(cmd, cs);

      dst_va += width * 4;
      blocks -= width;
   }

   ops->teardown(cmd, cs);

   after_buffer_unaligned_buffer_store<CHIP>(cmd, unaligned_store);
}
TU_GENX(tu_CmdFillBuffer);

template <chip CHIP>
VKAPI_ATTR void VKAPI_CALL
tu_CmdResolveImage2(VkCommandBuffer commandBuffer,
                    const VkResolveImageInfo2 *pResolveImageInfo)
{
   VK_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
   VK_FROM_HANDLE(tu_image, src_image, pResolveImageInfo->srcImage);
   VK_FROM_HANDLE(tu_image, dst_image, pResolveImageInfo->dstImage);
   const struct blit_ops *ops = &r2d_ops<CHIP>;
   struct tu_cs *cs = &cmd->cs;

   enum pipe_format src_format =
      vk_format_to_pipe_format(src_image->vk.format);
   enum pipe_format dst_format =
      vk_format_to_pipe_format(dst_image->vk.format);
   ops->setup(cmd, cs, src_format, dst_format,
              VK_IMAGE_ASPECT_COLOR_BIT, 0, false, dst_image->layout[0].ubwc,
              VK_SAMPLE_COUNT_1_BIT);

   for (uint32_t i = 0; i < pResolveImageInfo->regionCount; ++i) {
      const VkImageResolve2 *info = &pResolveImageInfo->pRegions[i];
      uint32_t layers = MAX2(info->extent.depth,
                             vk_image_subresource_layer_count(&dst_image->vk,
                                                              &info->dstSubresource));

      /* TODO: aspect masks possible ? */

      coords(ops, cmd, cs, info->dstOffset, info->srcOffset, info->extent);

      struct fdl6_view dst, src;
      tu_image_view_blit<CHIP>(&dst, dst_image, &info->dstSubresource, info->dstOffset.z);
      tu_image_view_blit<CHIP>(&src, src_image, &info->srcSubresource, info->srcOffset.z);

      for (uint32_t i = 0; i < layers; i++) {
         ops->src(cmd, cs, &src, i, VK_FILTER_NEAREST, dst_format);
         ops->dst(cs, &dst, i, src_format);
         ops->run(cmd, cs);
      }
   }

   ops->teardown(cmd, cs);
}
TU_GENX(tu_CmdResolveImage2);

#define for_each_layer(layer, layer_mask, layers) \
   for (uint32_t layer = 0; \
        layer < ((layer_mask) ? (util_logbase2(layer_mask) + 1) : layers); \
        layer++) \
      if (!layer_mask || (layer_mask & BIT(layer)))

template <chip CHIP>
static void
resolve_sysmem(struct tu_cmd_buffer *cmd,
               struct tu_cs *cs,
               VkFormat vk_src_format,
               VkFormat vk_dst_format,
               const struct tu_image_view *src,
               const struct tu_image_view *dst,
               uint32_t layer_mask,
               uint32_t layers,
               const VkRect2D *rect,
               bool src_separate_ds,
               bool dst_separate_ds)
{
   const struct blit_ops *ops = &r2d_ops<CHIP>;

   trace_start_sysmem_resolve(&cmd->trace, cs, vk_dst_format);

   enum pipe_format src_format = vk_format_to_pipe_format(vk_src_format);
   enum pipe_format dst_format = vk_format_to_pipe_format(vk_dst_format);

   ops->setup(cmd, cs, src_format, dst_format,
              VK_IMAGE_ASPECT_COLOR_BIT, 0, false, dst->view.ubwc_enabled,
              VK_SAMPLE_COUNT_1_BIT);
   ops->coords(cmd, cs, rect->offset, rect->offset, rect->extent);

   for_each_layer(i, layer_mask, layers) {
      if (src_separate_ds) {
         if (vk_src_format == VK_FORMAT_D32_SFLOAT || vk_dst_format == VK_FORMAT_D32_SFLOAT) {
            r2d_src_depth<CHIP>(cmd, cs, src, i, VK_FILTER_NEAREST);
         } else {
            r2d_src_stencil<CHIP>(cmd, cs, src, i, VK_FILTER_NEAREST);
         }
      } else {
         ops->src(cmd, cs, &src->view, i, VK_FILTER_NEAREST, dst_format);
      }

      if (dst_separate_ds) {
         if (vk_dst_format == VK_FORMAT_D32_SFLOAT) {
            ops->dst_depth(cs, dst, i);
         } else {
            ops->dst_stencil(cs, dst, i);
         }
      } else {
         ops->dst(cs, &dst->view, i, src_format);
      }

      ops->run(cmd, cs);
   }

   ops->teardown(cmd, cs);

   trace_end_sysmem_resolve(&cmd->trace, cs);
}

template <chip CHIP>
void
tu_resolve_sysmem(struct tu_cmd_buffer *cmd,
                  struct tu_cs *cs,
                  const struct tu_image_view *src,
                  const struct tu_image_view *dst,
                  uint32_t layer_mask,
                  uint32_t layers,
                  const VkRect2D *rect)
{
   assert(src->image->vk.format == dst->image->vk.format ||
          (vk_format_is_depth_or_stencil(src->image->vk.format) &&
           vk_format_is_depth_or_stencil(dst->image->vk.format)));

   bool src_separate_ds = src->image->vk.format == VK_FORMAT_D32_SFLOAT_S8_UINT;
   bool dst_separate_ds = dst->image->vk.format == VK_FORMAT_D32_SFLOAT_S8_UINT;

   if (dst_separate_ds) {
      resolve_sysmem<CHIP>(cmd, cs, VK_FORMAT_D32_SFLOAT, VK_FORMAT_D32_SFLOAT,
                     src, dst, layer_mask, layers, rect,
                     src_separate_ds, dst_separate_ds);
      resolve_sysmem<CHIP>(cmd, cs, VK_FORMAT_S8_UINT, VK_FORMAT_S8_UINT,
                     src, dst, layer_mask, layers, rect,
                     src_separate_ds, dst_separate_ds);
   } else {
      resolve_sysmem<CHIP>(cmd, cs, src->image->vk.format, dst->image->vk.format,
                     src, dst, layer_mask, layers, rect,
                     src_separate_ds, dst_separate_ds);
   }
}
TU_GENX(tu_resolve_sysmem);

enum tu_resolve_group_buffer_type {
   TU_RESOLVE_GROUP_COLOR_BUFFER,
   TU_RESOLVE_GROUP_DEPTH_BUFFER,
   TU_RESOLVE_GROUP_STENCIL_BUFFER,
};

template <chip CHIP>
static uint32_t
tu_resolve_group_include_buffer(struct tu_resolve_group *resolve_group,
                                enum tu_resolve_group_buffer_type buffer_type)
{
   /* Resolve groups are not usable on a6xx, so no pending resolve is
    * established. The default value of 0 is returned as the buffer ID.
    */
   if (CHIP == A6XX)
      return 0;

   resolve_group->pending_resolves = true;

   if (buffer_type == TU_RESOLVE_GROUP_DEPTH_BUFFER)
      return 0x8;
   if (buffer_type == TU_RESOLVE_GROUP_STENCIL_BUFFER)
      return 0x9;

   const uint32_t max_color_buffers = 8;
   uint32_t buffer_id = resolve_group->color_buffer_id++;
   return buffer_id % max_color_buffers;
}

template <chip CHIP>
static uint32_t
tu_resolve_group_include_buffer_for_format(struct tu_resolve_group *resolve_group,
                                           VkFormat format)
{
   enum tu_resolve_group_buffer_type buffer_type = TU_RESOLVE_GROUP_COLOR_BUFFER;

   /* D24_UNORM_S8_UINT should be assigned the depth buffer type, regardless of
    * whether depth, stencil or both are being resolved.
    */
   if (format == VK_FORMAT_D24_UNORM_S8_UINT)
      buffer_type = TU_RESOLVE_GROUP_DEPTH_BUFFER;

   return tu_resolve_group_include_buffer<CHIP>(resolve_group, buffer_type);
}

template <chip CHIP>
void
tu_emit_resolve_group(struct tu_cmd_buffer *cmd,
                          struct tu_cs *cs,
                          struct tu_resolve_group *resolve_group)
{
   /* Resolve groups are not usable on A6XX, so that template instantiation
    * should behave as a no-op.
    */
   if (CHIP == A6XX || !resolve_group->pending_resolves)
      return;

   resolve_group->color_buffer_id = 0;
   resolve_group->pending_resolves = false;

   tu_emit_raw_event_write<CHIP>(cmd, cs, CCU_END_RESOLVE_GROUP, false);
}
TU_GENX(tu_emit_resolve_group);

template <chip CHIP>
static void
clear_image_cp_blit(struct tu_cmd_buffer *cmd,
                    struct tu_image *image,
                    const VkClearValue *clear_value,
                    const VkImageSubresourceRange *range,
                    VkImageAspectFlags aspect_mask)
{
   uint32_t level_count = vk_image_subresource_level_count(&image->vk, range);
   uint32_t layer_count = vk_image_subresource_layer_count(&image->vk, range);
   struct tu_cs *cs = &cmd->cs;
   enum pipe_format format;
   if (image->vk.format == VK_FORMAT_E5B9G9R9_UFLOAT_PACK32) {
      format = PIPE_FORMAT_R32_UINT;
   } else {
      format = tu_aspects_to_plane(image->vk.format, aspect_mask);
   }

   if (image->layout[0].depth0 > 1) {
      assert(layer_count == 1);
      assert(range->baseArrayLayer == 0);
   }

   const struct blit_ops *ops = image->layout[0].nr_samples > 1 ? &r3d_ops<CHIP> : &r2d_ops<CHIP>;

   ops->setup(cmd, cs, format, format, aspect_mask, 0, true, image->layout[0].ubwc,
              (VkSampleCountFlagBits) image->layout[0].nr_samples);
   if (image->vk.format == VK_FORMAT_E5B9G9R9_UFLOAT_PACK32)
      ops->clear_value(cmd, cs, PIPE_FORMAT_R9G9B9E5_FLOAT, clear_value);
   else
      ops->clear_value(cmd, cs, format, clear_value);

   for (unsigned j = 0; j < level_count; j++) {
      if (image->layout[0].depth0 > 1)
         layer_count = u_minify(image->layout[0].depth0, range->baseMipLevel + j);

      ops->coords(cmd, cs, (VkOffset2D) {}, blt_no_coord, (VkExtent2D) {
                     u_minify(image->layout[0].width0, range->baseMipLevel + j),
                     u_minify(image->layout[0].height0, range->baseMipLevel + j)
                  });

      struct fdl6_view dst;
      const VkImageSubresourceLayers subresource = {
         .aspectMask = aspect_mask,
         .mipLevel = range->baseMipLevel + j,
         .baseArrayLayer = range->baseArrayLayer,
         .layerCount = 1,
      };
      tu_image_view_copy_blit<CHIP>(&dst, image, format, &subresource, 0, false);

      for (uint32_t i = 0; i < layer_count; i++) {
         ops->dst(cs, &dst, i, format);
         ops->run(cmd, cs);
      }
   }

   ops->teardown(cmd, cs);
}

static void
clear_image_event_blit(struct tu_cmd_buffer *cmd,
                       struct tu_image *image,
                       uint32_t buffer_id,
                       const VkClearValue *clear_value,
                       const VkImageSubresourceRange *range,
                       VkImageAspectFlags aspect_mask)
{
   uint32_t level_count = vk_image_subresource_level_count(&image->vk, range);
   uint32_t layer_count = vk_image_subresource_layer_count(&image->vk, range);
   VkFormat vk_format = image->vk.format;
   if (vk_format == VK_FORMAT_D32_SFLOAT_S8_UINT) {
      if (aspect_mask == VK_IMAGE_ASPECT_STENCIL_BIT)
         vk_format = VK_FORMAT_S8_UINT;
      else
         vk_format = VK_FORMAT_D32_SFLOAT;
   }

   enum pipe_format format = vk_format_to_pipe_format(vk_format);

   if (image->layout[0].depth0 > 1) {
      assert(layer_count == 1);
      assert(range->baseArrayLayer == 0);
   }

   struct tu_cs *cs = &cmd->cs;

   tu_cs_emit_regs(cs,
                   A7XX_RB_BLIT_CLEAR_MODE(.clear_mode = CLEAR_MODE_SYSMEM));

   tu_cs_emit_pkt4(cs, REG_A6XX_RB_UNKNOWN_88D0, 1);
   tu_cs_emit(cs, 0);

   tu_cs_emit_regs(
      cs, A6XX_RB_BLIT_INFO(
                .type = BLIT_EVENT_CLEAR,
                .sample_0 = vk_format_is_int(vk_format) ||
                            vk_format_is_depth_or_stencil(vk_format),
                .depth = vk_format_is_depth_or_stencil(vk_format),
                .clear_mask = aspect_write_mask_generic_clear(format, aspect_mask),
                .buffer_id = buffer_id));

   uint32_t clear_vals[4] = {};
   pack_blit_event_clear_value(clear_value, format, clear_vals);
   tu_cs_emit_pkt4(cs, REG_A6XX_RB_BLIT_CLEAR_COLOR_DW0, 4);
   tu_cs_emit_array(cs, clear_vals, 4);

   for (unsigned level = 0; level < level_count; level++) {
      if (image->layout[0].depth0 > 1)
         layer_count =
            u_minify(image->layout[0].depth0, range->baseMipLevel + level);

      uint32_t width =
         u_minify(image->layout[0].width0, range->baseMipLevel + level);
      uint32_t height =
         u_minify(image->layout[0].height0, range->baseMipLevel + level);
      tu_cs_emit_regs(
         cs, A6XX_RB_BLIT_SCISSOR_TL(.x = 0, .y = 0),
         A6XX_RB_BLIT_SCISSOR_BR(.x = width - 1, .y = height - 1));

      struct fdl6_view dst;
      const VkImageSubresourceLayers subresource = {
         .aspectMask = aspect_mask,
         .mipLevel = range->baseMipLevel + level,
         .baseArrayLayer = range->baseArrayLayer,
         .layerCount = 1,
      };
      tu_image_view_copy_blit<A7XX>(&dst, image, format, &subresource, 0, false);

      for (uint32_t layer = 0; layer < layer_count; layer++) {

         struct event_blit_dst_view blt_view = {
            .image = image,
            .view = &dst,
            .layer = layer,
         };

         if (image->vk.format == VK_FORMAT_D32_SFLOAT_S8_UINT) {
            uint32_t real_level = range->baseMipLevel + level;
            uint32_t real_layer = range->baseArrayLayer + layer;
            if (aspect_mask == VK_IMAGE_ASPECT_DEPTH_BIT) {
               struct fdl_layout *layout = &image->layout[0];
               blt_view.depth_addr =
                  image->iova +
                  fdl_surface_offset(layout, real_level, real_layer);
               blt_view.depth_pitch = fdl_pitch(layout, real_level);
            } else {
               struct fdl_layout *layout = &image->layout[1];
               blt_view.stencil_addr =
                  image->iova +
                  fdl_surface_offset(layout, real_level, real_layer);
               blt_view.stencil_pitch = fdl_pitch(layout, real_level);
            }
         }

         event_blit_run<A7XX>(cmd, cs, NULL, &blt_view,
                              aspect_mask == VK_IMAGE_ASPECT_STENCIL_BIT);
      }
   }
}

static bool
use_generic_clear_for_image_clear(struct tu_cmd_buffer *cmd,
                                  struct tu_image *image)
{
   const struct fd_dev_info *info = cmd->device->physical_device->info;
   return info->a7xx.has_generic_clear &&
          /* A7XX supports R9G9B9E5_FLOAT as color attachment and supports
           * generic clears for it. A7XX TODO: allow R9G9B9E5_FLOAT
           * attachments.
           */
          image->vk.format != VK_FORMAT_E5B9G9R9_UFLOAT_PACK32 &&
          /* Clearing VK_FORMAT_R8G8_* with fast-clear value, certain
           * dimensions (e.g. 960x540), and having GMEM renderpass afterwards
           * may lead to a GPU fault on A7XX.
           */
          !(info->a7xx.r8g8_faulty_fast_clear_quirk && image_is_r8g8(image));
}

template <chip CHIP>
static void
clear_image(struct tu_cmd_buffer *cmd,
            struct tu_image *image,
            uint32_t buffer_id,
            const VkClearValue *clear_value,
            const VkImageSubresourceRange *range,
            VkImageAspectFlags aspect_mask)
{
   if (use_generic_clear_for_image_clear(cmd, image)) {
      clear_image_event_blit(cmd, image, buffer_id, clear_value, range, aspect_mask);
   } else {
      clear_image_cp_blit<CHIP>(cmd, image, clear_value, range, aspect_mask);
   }
}

template <chip CHIP>
VKAPI_ATTR void VKAPI_CALL
tu_CmdClearColorImage(VkCommandBuffer commandBuffer,
                      VkImage image_h,
                      VkImageLayout imageLayout,
                      const VkClearColorValue *pColor,
                      uint32_t rangeCount,
                      const VkImageSubresourceRange *pRanges)
{
   VK_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
   VK_FROM_HANDLE(tu_image, image, image_h);

   bool use_generic_clear = use_generic_clear_for_image_clear(cmd, image);
   if (use_generic_clear) {
      /* Generic clear doesn't go through CCU (or other caches). */
      cmd->state.cache.flush_bits |=
         TU_CMD_FLAG_CCU_INVALIDATE_COLOR | TU_CMD_FLAG_WAIT_FOR_IDLE;
      tu_emit_cache_flush<CHIP>(cmd);
   }

   struct tu_resolve_group resolve_group = {};

   for (unsigned i = 0; i < rangeCount; i++) {
      uint32_t buffer_id = tu_resolve_group_include_buffer<CHIP>(&resolve_group, TU_RESOLVE_GROUP_COLOR_BUFFER);
      clear_image<CHIP>(cmd, image, buffer_id, (const VkClearValue*) pColor, pRanges + i, VK_IMAGE_ASPECT_COLOR_BIT);
   }

   tu_emit_resolve_group<CHIP>(cmd, &cmd->cs, &resolve_group);
   if (use_generic_clear) {
      /* This will emit CCU_RESOLVE_CLEAN which will ensure any future resolves
       * proceed only after the just-emitted generic clears are complete.
       */
      cmd->state.cache.flush_bits |= TU_CMD_FLAG_BLIT_CACHE_CLEAN;
      tu_emit_cache_flush<CHIP>(cmd);
   }
}
TU_GENX(tu_CmdClearColorImage);

template <chip CHIP>
VKAPI_ATTR void VKAPI_CALL
tu_CmdClearDepthStencilImage(VkCommandBuffer commandBuffer,
                             VkImage image_h,
                             VkImageLayout imageLayout,
                             const VkClearDepthStencilValue *pDepthStencil,
                             uint32_t rangeCount,
                             const VkImageSubresourceRange *pRanges)
{
   VK_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
   VK_FROM_HANDLE(tu_image, image, image_h);

   bool use_generic_clear = use_generic_clear_for_image_clear(cmd, image);
   if (use_generic_clear) {
      /* Generic clear doesn't go through CCU (or other caches). */
      cmd->state.cache.flush_bits |= TU_CMD_FLAG_CCU_INVALIDATE_COLOR |
                                     TU_CMD_FLAG_CCU_INVALIDATE_DEPTH |
                                     TU_CMD_FLAG_WAIT_FOR_IDLE;
      tu_emit_cache_flush<CHIP>(cmd);
   }

   struct tu_resolve_group resolve_group = {};

   for (unsigned i = 0; i < rangeCount; i++) {
      const VkImageSubresourceRange *range = &pRanges[i];

      if (image->vk.format == VK_FORMAT_D32_SFLOAT_S8_UINT) {
         /* can't clear both depth and stencil at once, split up the aspect mask */
         u_foreach_bit(b, range->aspectMask) {
            uint32_t buffer_id = 0;
            if (BIT(b) == VK_IMAGE_ASPECT_DEPTH_BIT)
               buffer_id = tu_resolve_group_include_buffer<CHIP>(&resolve_group, TU_RESOLVE_GROUP_DEPTH_BUFFER);
            if (BIT(b) == VK_IMAGE_ASPECT_STENCIL_BIT)
               buffer_id = tu_resolve_group_include_buffer<CHIP>(&resolve_group, TU_RESOLVE_GROUP_STENCIL_BUFFER);

            clear_image<CHIP>(cmd, image, buffer_id, (const VkClearValue*) pDepthStencil, range, BIT(b));
         }
         continue;
      }

      uint32_t buffer_id = tu_resolve_group_include_buffer_for_format<CHIP>(&resolve_group, image->vk.format);
      clear_image<CHIP>(cmd, image, buffer_id, (const VkClearValue*) pDepthStencil, range, range->aspectMask);
   }

   tu_emit_resolve_group<CHIP>(cmd, &cmd->cs, &resolve_group);
   if (use_generic_clear) {
      /* This will emit CCU_RESOLVE_CLEAN which will ensure any future resolves
       * proceed only after the just-emitted generic clears are complete.
       */
      cmd->state.cache.flush_bits |= TU_CMD_FLAG_BLIT_CACHE_CLEAN;
      tu_emit_cache_flush<CHIP>(cmd);
   }

   tu_lrz_clear_depth_image<CHIP>(cmd, image, pDepthStencil, rangeCount, pRanges);
}
TU_GENX(tu_CmdClearDepthStencilImage);

/* CmdClearAttachments uses the original color attachment index instead of the
 * remapped index used by the shader, and our MRTs use the remapped
 * indices, so we have to remap them. We should always be able to find a
 * shader attachment thanks to this VU:
 *
 *    VUID-vkCmdClearAttachments-colorAttachment-09503
 *    "The colorAttachment member of each element of pAttachments must not
 *    identify a color attachment that is currently mapped to
 *    VK_ATTACHMENT_UNUSED in commandBuffer via
 *    VkRenderingAttachmentLocationInfoKHR"
 */
static unsigned
remap_attachment(struct tu_cmd_buffer *cmd, unsigned a)
{
   unsigned i = cmd->vk.dynamic_graphics_state.cal.color_map[a];
   assert(i != MESA_VK_ATTACHMENT_UNUSED &&
          "app violates VUID-vkCmdClearAttachments-colorAttachment-09503");
   return i;
}

template <chip CHIP>
static void
tu_clear_sysmem_attachments(struct tu_cmd_buffer *cmd,
                            uint32_t attachment_count,
                            const VkClearAttachment *attachments,
                            uint32_t rect_count,
                            const VkClearRect *rects)
{
   /* the shader path here is special, it avoids changing MRT/etc state */
   const struct tu_subpass *subpass = cmd->state.subpass;
   const uint32_t mrt_count = subpass->color_count;
   struct tu_cs *cs = &cmd->draw_cs;
   uint32_t clear_value[MAX_RTS][4];
   float z_clear_val = 0.0f;
   uint8_t s_clear_val = 0;
   uint32_t clear_rts = 0, clear_components = 0;
   bool z_clear = false;
   bool s_clear = false;

   trace_start_sysmem_clear_all(&cmd->trace, cs, mrt_count, rect_count);

   for (uint32_t i = 0; i < attachment_count; i++) {
      uint32_t a;
      if (attachments[i].aspectMask & VK_IMAGE_ASPECT_COLOR_BIT) {
         uint32_t c = attachments[i].colorAttachment;
         a = subpass->color_attachments[c].attachment;
         if (a == VK_ATTACHMENT_UNUSED)
            continue;

         uint32_t remapped = remap_attachment(cmd, c);
         clear_rts |= 1 << remapped;
         clear_components |= 0xf << (remapped * 4);
         memcpy(clear_value[remapped], &attachments[i].clearValue, 4 * sizeof(uint32_t));
      } else {
         a = subpass->depth_stencil_attachment.attachment;
         if (a == VK_ATTACHMENT_UNUSED)
            continue;

         if (attachments[i].aspectMask & VK_IMAGE_ASPECT_DEPTH_BIT) {
            z_clear = true;
            z_clear_val = attachments[i].clearValue.depthStencil.depth;
         }

         if (attachments[i].aspectMask & VK_IMAGE_ASPECT_STENCIL_BIT) {
            s_clear = true;
            s_clear_val = attachments[i].clearValue.depthStencil.stencil & 0xff;
         }
      }
   }

   /* We may not know the multisample count if there are no attachments, so
    * just bail early to avoid corner cases later.
    */
   if (clear_rts == 0 && !z_clear && !s_clear)
      return;

   /* disable all draw states so they don't interfere
    * TODO: use and re-use draw states
    * we have to disable draw states individually to preserve
    * input attachment states, because a secondary command buffer
    * won't be able to restore them
    */
   tu_cs_emit_pkt7(cs, CP_SET_DRAW_STATE, 3 * (TU_DRAW_STATE_COUNT - 2));
   for (uint32_t i = 0; i < TU_DRAW_STATE_COUNT; i++) {
      if (i == TU_DRAW_STATE_INPUT_ATTACHMENTS_GMEM ||
          i == TU_DRAW_STATE_INPUT_ATTACHMENTS_SYSMEM)
         continue;
      tu_cs_emit(cs, CP_SET_DRAW_STATE__0_GROUP_ID(i) |
                     CP_SET_DRAW_STATE__0_DISABLE);
      tu_cs_emit_qw(cs, 0);
   }
   cmd->state.dirty |= TU_CMD_DIRTY_DRAW_STATE;

   tu_cs_emit_pkt4(cs, REG_A6XX_SP_FS_OUTPUT_CNTL0, 2);
   tu_cs_emit(cs, A6XX_SP_FS_OUTPUT_CNTL0_DEPTH_REGID(0xfc) |
                  A6XX_SP_FS_OUTPUT_CNTL0_SAMPMASK_REGID(0xfc) |
                  0xfc000000);
   tu_cs_emit(cs, A6XX_SP_FS_OUTPUT_CNTL1_MRT(mrt_count));

   r3d_common<CHIP>(cmd, cs, R3D_CLEAR, clear_rts, false, cmd->state.subpass->samples);

   /* Disable sample counting in order to not affect occlusion query. */
   tu_cs_emit_regs(cs, A6XX_RB_SAMPLE_COUNT_CONTROL(.disable = true));

   if (cmd->state.prim_generated_query_running_before_rp) {
      tu_emit_event_write<CHIP>(cmd, cs, FD_STOP_PRIMITIVE_CTRS);
   }

   tu_cs_emit_regs(cs,
                   A6XX_SP_FS_RENDER_COMPONENTS(.dword = clear_components));
   tu_cs_emit_regs(cs,
                   A6XX_RB_RENDER_COMPONENTS(.dword = clear_components));

   tu_cs_emit_regs(cs,
                   A6XX_RB_FS_OUTPUT_CNTL1(.mrt = mrt_count));

   tu_cs_emit_regs(cs, A6XX_SP_BLEND_CNTL());
   tu_cs_emit_regs(cs, A6XX_RB_BLEND_CNTL(.independent_blend = 1, .sample_mask = 0xffff));
   for (uint32_t i = 0; i < mrt_count; i++) {
      tu_cs_emit_regs(cs, A6XX_RB_MRT_CONTROL(i,
            .component_enable = COND(clear_rts & (1 << i), 0xf)));
   }

   tu_cs_emit_regs(cs, A6XX_GRAS_LRZ_CNTL(0));
   tu_cs_emit_regs(cs, A6XX_RB_LRZ_CNTL(0));

   tu_cs_emit_regs(cs, A6XX_RB_DEPTH_PLANE_CNTL());
   tu_cs_emit_regs(cs, A6XX_RB_DEPTH_CNTL(
         .z_test_enable = z_clear,
         .z_write_enable = z_clear,
         .zfunc = FUNC_ALWAYS));
   tu_cs_emit_regs(cs, A6XX_GRAS_SU_DEPTH_CNTL(z_clear));
   tu_cs_emit_regs(cs, A6XX_GRAS_SU_DEPTH_PLANE_CNTL());
   tu_cs_emit_regs(cs, A6XX_RB_STENCIL_CONTROL(
         .stencil_enable = s_clear,
         .func = FUNC_ALWAYS,
         .zpass = STENCIL_REPLACE));
   tu_cs_emit_regs(cs, A6XX_GRAS_SU_STENCIL_CNTL(s_clear));
   tu_cs_emit_regs(cs, A6XX_RB_STENCILMASK(.mask = 0xff));
   tu_cs_emit_regs(cs, A6XX_RB_STENCILWRMASK(.wrmask = 0xff));
   tu_cs_emit_regs(cs, A6XX_RB_STENCILREF(.ref = s_clear_val));

   tu_cs_emit_regs(cs, A6XX_GRAS_SC_CNTL(.ccusinglecachelinesize = 2));

   unsigned num_rts = util_bitcount(clear_rts);
   uint32_t packed_clear_value[MAX_RTS][4];

   uint32_t idx = 0;
   u_foreach_bit(b, clear_rts) {
      memcpy(&packed_clear_value[idx], &clear_value[b], 4 * sizeof(uint32_t));
      idx++;
   }

   if (num_rts > 0)
      tu6_emit_blit_consts_load(cmd, cs, CP_LOAD_STATE6_FRAG, SB6_FS_SHADER,
                                0, packed_clear_value, num_rts);

   for (uint32_t i = 0; i < rect_count; i++) {
      /* This should be true because of this valid usage for
       * vkCmdClearAttachments:
       *
       *    "If the render pass instance this is recorded in uses multiview,
       *    then baseArrayLayer must be zero and layerCount must be one"
       */
      assert(!subpass->multiview_mask || rects[i].baseArrayLayer == 0);

      /* a630 doesn't support multiview masks, which means that we can't use
       * the normal multiview path without potentially recompiling a shader
       * on-demand or using a more complicated variant that takes the mask as
       * a const. Just use the layered path instead, since it shouldn't be
       * much worse.
       */
      for_each_layer(layer, subpass->multiview_mask, rects[i].layerCount)
      {
         const float coords[] = {
            rects[i].rect.offset.x,
            rects[i].rect.offset.y,
            z_clear_val,
            uif(rects[i].baseArrayLayer + layer),
            rects[i].rect.offset.x + rects[i].rect.extent.width,
            rects[i].rect.offset.y + rects[i].rect.extent.height,
            z_clear_val,
            1.0f,
         };

         r3d_coords_raw(cmd, cs, coords);
         r3d_run_vis(cmd, cs);
      }
   }

   /* Re-enable sample counting. */
   tu_cs_emit_regs(cs, A6XX_RB_SAMPLE_COUNT_CONTROL(.disable = false));

   if (cmd->state.prim_generated_query_running_before_rp) {
      tu_emit_event_write<CHIP>(cmd, cs, FD_START_PRIMITIVE_CTRS);
   }

   trace_end_sysmem_clear_all(&cmd->trace, cs);
}

template <chip CHIP>
static void
clear_gmem_attachment(struct tu_cmd_buffer *cmd,
                      struct tu_cs *cs,
                      uint32_t buffer_id,
                      enum pipe_format format,
                      uint8_t clear_mask,
                      uint32_t gmem_offset,
                      const VkClearValue *value)
{
   tu_cs_emit_pkt4(cs, REG_A6XX_RB_BLIT_DST_INFO, 1);
   tu_cs_emit(cs, A6XX_RB_BLIT_DST_INFO_COLOR_FORMAT(
            blit_base_format<CHIP>(format, false, true)));

   tu_cs_emit_regs(cs, A6XX_RB_BLIT_INFO(.type = BLIT_EVENT_CLEAR,
                                         .clear_mask = clear_mask,
                                         .buffer_id = buffer_id));

   tu_cs_emit_pkt4(cs, REG_A6XX_RB_BLIT_BASE_GMEM, 1);
   tu_cs_emit(cs, gmem_offset);

   tu_cs_emit_pkt4(cs, REG_A6XX_RB_UNKNOWN_88D0, 1);
   tu_cs_emit(cs, 0);

   uint32_t clear_vals[4] = {};
   pack_blit_event_clear_value(value, format, clear_vals);

   tu_cs_emit_pkt4(cs, REG_A6XX_RB_BLIT_CLEAR_COLOR_DW0, 4);
   tu_cs_emit_array(cs, clear_vals, 4);

   tu_emit_event_write<CHIP>(cmd, cs, FD_BLIT);
}

template <chip CHIP>
static void
tu_emit_clear_gmem_attachment(struct tu_cmd_buffer *cmd,
                              struct tu_cs *cs,
                              struct tu_resolve_group *resolve_group,
                              uint32_t attachment,
                              uint32_t base_layer,
                              uint32_t layers,
                              uint32_t layer_mask,
                              VkImageAspectFlags mask,
                              const VkClearValue *value)
{
   const struct tu_render_pass_attachment *att =
      &cmd->state.pass->attachments[attachment];

   trace_start_gmem_clear(&cmd->trace, cs, att->format, att->samples);

   tu_cs_emit_regs(cs,
                   A6XX_RB_BLIT_GMEM_MSAA_CNTL(tu_msaa_samples(att->samples)));

   enum pipe_format format = vk_format_to_pipe_format(att->format);
   for_each_layer(i, layer_mask, layers) {
      uint32_t layer = i + base_layer;
      if (att->format == VK_FORMAT_D32_SFLOAT_S8_UINT) {
         if (mask & VK_IMAGE_ASPECT_DEPTH_BIT) {
            uint32_t buffer_id = tu_resolve_group_include_buffer<CHIP>(resolve_group, TU_RESOLVE_GROUP_DEPTH_BUFFER);
            clear_gmem_attachment<CHIP>(cmd, cs, buffer_id, PIPE_FORMAT_Z32_FLOAT, 0xf,
                                  tu_attachment_gmem_offset(cmd, att, layer), value);
         }
         if (mask & VK_IMAGE_ASPECT_STENCIL_BIT) {
            uint32_t buffer_id = tu_resolve_group_include_buffer<CHIP>(resolve_group, TU_RESOLVE_GROUP_STENCIL_BUFFER);
            clear_gmem_attachment<CHIP>(cmd, cs, buffer_id, PIPE_FORMAT_S8_UINT, 0xf,
                                  tu_attachment_gmem_offset_stencil(cmd, att, layer), value);
         }
      } else {
         uint32_t buffer_id = tu_resolve_group_include_buffer_for_format<CHIP>(resolve_group, att->format);
         clear_gmem_attachment<CHIP>(cmd, cs, buffer_id, format, aspect_write_mask(format, mask),
                               tu_attachment_gmem_offset(cmd, att, layer), value);
      }
   }

   tu_flush_for_access(&cmd->state.renderpass_cache, TU_ACCESS_BLIT_WRITE_GMEM, TU_ACCESS_NONE);

   trace_end_gmem_clear(&cmd->trace, cs);
}

template <chip CHIP>
static void
tu_clear_gmem_attachments(struct tu_cmd_buffer *cmd,
                          uint32_t attachment_count,
                          const VkClearAttachment *attachments,
                          uint32_t rect_count,
                          const VkClearRect *rects)
{
   const struct tu_subpass *subpass = cmd->state.subpass;
   struct tu_cs *cs = &cmd->draw_cs;

   if (rect_count > 1)
      perf_debug(cmd->device, "TODO: Swap tu_clear_gmem_attachments() loop for smaller command stream");

   struct tu_resolve_group resolve_group = {};

   for (unsigned i = 0; i < rect_count; i++) {
      unsigned x1 = rects[i].rect.offset.x;
      unsigned y1 = rects[i].rect.offset.y;
      unsigned x2 = x1 + rects[i].rect.extent.width - 1;
      unsigned y2 = y1 + rects[i].rect.extent.height - 1;

      tu_cs_emit_pkt4(cs, REG_A6XX_RB_BLIT_SCISSOR_TL, 2);
      tu_cs_emit(cs, A6XX_RB_BLIT_SCISSOR_TL_X(x1) | A6XX_RB_BLIT_SCISSOR_TL_Y(y1));
      tu_cs_emit(cs, A6XX_RB_BLIT_SCISSOR_BR_X(x2) | A6XX_RB_BLIT_SCISSOR_BR_Y(y2));

      for (unsigned j = 0; j < attachment_count; j++) {
         uint32_t a;
         if (attachments[j].aspectMask & VK_IMAGE_ASPECT_COLOR_BIT)
            a = subpass->color_attachments[attachments[j].colorAttachment].attachment;
         else
            a = subpass->depth_stencil_attachment.attachment;

         if (a == VK_ATTACHMENT_UNUSED)
               continue;

         tu_emit_clear_gmem_attachment<CHIP>(cmd, cs, &resolve_group, a,
                                       rects[i].baseArrayLayer,
                                       rects[i].layerCount,
                                       subpass->multiview_mask,
                                       attachments[j].aspectMask,
                                       &attachments[j].clearValue);
      }
   }

   tu_emit_resolve_group<CHIP>(cmd, cs, &resolve_group);
}

template <chip CHIP>
static void
tu_clear_attachments(struct tu_cmd_buffer *cmd,
                     uint32_t attachmentCount,
                     const VkClearAttachment *pAttachments,
                     uint32_t rectCount,
                     const VkClearRect *pRects)
{
   struct tu_cs *cs = &cmd->draw_cs;

   /* sysmem path behaves like a draw, note we don't have a way of using different
    * flushes for sysmem/gmem, so this needs to be outside of the cond_exec
    */
   tu_emit_cache_flush_renderpass<CHIP>(cmd);

   /* vkCmdClearAttachments is supposed to respect the predicate if active. The
    * easiest way to do this is to always use the 3d path, which always works
    * even with GMEM because it's just a simple draw using the existing
    * attachment state.
    *
    * Similarly, we also use the 3D path when in a secondary command buffer that
    * doesn't know the GMEM layout that will be chosen by the primary.
    */
   if (cmd->state.predication_active || cmd->state.gmem_layout == TU_GMEM_LAYOUT_COUNT) {
      tu_clear_sysmem_attachments<CHIP>(cmd, attachmentCount, pAttachments, rectCount, pRects);
      return;
   }

   /* If we could skip tile load/stores based on any draws intersecting them at
    * binning time, then emit the clear as a 3D draw so that it contributes to
    * that visibility.
   */
   const struct tu_subpass *subpass = cmd->state.subpass;
   for (uint32_t i = 0; i < attachmentCount; i++) {
      uint32_t a;
      if (pAttachments[i].aspectMask & VK_IMAGE_ASPECT_COLOR_BIT) {
         uint32_t c = pAttachments[i].colorAttachment;
         a = subpass->color_attachments[c].attachment;
      } else {
         a = subpass->depth_stencil_attachment.attachment;
      }
      if (a != VK_ATTACHMENT_UNUSED) {
         const struct tu_render_pass_attachment *att = &cmd->state.pass->attachments[a];
         if (att->cond_load_allowed || att->cond_store_allowed) {
            tu_clear_sysmem_attachments<CHIP>(cmd, attachmentCount, pAttachments, rectCount, pRects);
            return;
         }
      }
   }

   /* Otherwise, emit 2D blits for gmem rendering. */
   tu_cond_exec_start(cs, CP_COND_EXEC_0_RENDER_MODE_GMEM);
   tu_clear_gmem_attachments<CHIP>(cmd, attachmentCount, pAttachments, rectCount, pRects);
   tu_cond_exec_end(cs);

   tu_cond_exec_start(cs, CP_COND_EXEC_0_RENDER_MODE_SYSMEM);
   tu_clear_sysmem_attachments<CHIP>(cmd, attachmentCount, pAttachments, rectCount, pRects);
   tu_cond_exec_end(cs);
}

static void
tu7_clear_attachment_generic_single_rect(
   struct tu_cmd_buffer *cmd,
   struct tu_cs *cs,
   struct tu_resolve_group *resolve_group,
   const struct tu_render_pass_attachment *att,
   const VkClearAttachment *clear_att,
   uint32_t a,
   const VkClearRect *rect)
{
   const struct tu_subpass *subpass = cmd->state.subpass;
   unsigned x1 = rect->rect.offset.x;
   unsigned y1 = rect->rect.offset.y;
   unsigned x2 = x1 + rect->rect.extent.width - 1;
   unsigned y2 = y1 + rect->rect.extent.height - 1;

   tu_cs_emit_pkt4(cs, REG_A6XX_RB_BLIT_SCISSOR_TL, 2);
   tu_cs_emit(cs,
              A6XX_RB_BLIT_SCISSOR_TL_X(x1) | A6XX_RB_BLIT_SCISSOR_TL_Y(y1));
   tu_cs_emit(cs,
              A6XX_RB_BLIT_SCISSOR_BR_X(x2) | A6XX_RB_BLIT_SCISSOR_BR_Y(y2));

   auto value = &clear_att->clearValue;

   enum pipe_format format = vk_format_to_pipe_format(att->format);
   for_each_layer(i, subpass->multiview_mask, rect->layerCount) {
      uint32_t layer = i + rect->baseArrayLayer;
      uint32_t mask =
         aspect_write_mask_generic_clear(format, clear_att->aspectMask);

      if (att->format == VK_FORMAT_D32_SFLOAT_S8_UINT) {
         if (clear_att->aspectMask & VK_IMAGE_ASPECT_DEPTH_BIT) {
            uint32_t buffer_id = tu_resolve_group_include_buffer<A7XX>(resolve_group, TU_RESOLVE_GROUP_DEPTH_BUFFER);
            tu7_generic_layer_clear(cmd, cs, buffer_id, PIPE_FORMAT_Z32_FLOAT, mask,
                                    false, layer, value, a);
         }
         if (clear_att->aspectMask & VK_IMAGE_ASPECT_STENCIL_BIT) {
            uint32_t buffer_id = tu_resolve_group_include_buffer<A7XX>(resolve_group, TU_RESOLVE_GROUP_STENCIL_BUFFER);
            tu7_generic_layer_clear(cmd, cs, buffer_id, PIPE_FORMAT_S8_UINT, mask, true,
                                    layer, value, a);
         }
      } else {
         uint32_t buffer_id = tu_resolve_group_include_buffer_for_format<A7XX>(resolve_group, att->format);
         tu7_generic_layer_clear(cmd, cs, buffer_id, format, mask, false, layer, value, a);
      }
   }
}

static void
tu_clear_attachments_generic(struct tu_cmd_buffer *cmd,
                             uint32_t attachmentCount,
                             const VkClearAttachment *pAttachments,
                             uint32_t rectCount,
                             const VkClearRect *pRects)
{
   struct tu_cs *cs = &cmd->draw_cs;

   uint32_t clear_aspects = 0;
   for (uint32_t i = 0; i < attachmentCount; i++) {
      clear_aspects |= pAttachments[i].aspectMask;
   }

   /* Generic clear doesn't go through CCU (or other caches),
    * so we have to flush (clean+invalidate) corresponding caches.
    */
   tu_cond_exec_start(cs, CP_COND_EXEC_0_RENDER_MODE_SYSMEM);
   if (clear_aspects & VK_IMAGE_ASPECT_COLOR_BIT) {
      tu_cs_emit_pkt7(cs, CP_EVENT_WRITE7, 1);
      tu_cs_emit(cs, CP_EVENT_WRITE7_0(.event = CCU_FLUSH_COLOR).value);
   }
   if (clear_aspects & (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT)) {
      tu_cs_emit_pkt7(cs, CP_EVENT_WRITE7, 1);
      tu_cs_emit(cs, CP_EVENT_WRITE7_0(.event = CCU_FLUSH_DEPTH).value);
   }
   tu_cs_emit_wfi(cs);
   tu_cond_exec_end(cs);

   struct tu_resolve_group resolve_group = {};

   const struct tu_subpass *subpass = cmd->state.subpass;
   for (uint32_t i = 0; i < attachmentCount; i++) {
      uint32_t a;
      if (pAttachments[i].aspectMask & VK_IMAGE_ASPECT_COLOR_BIT) {
         uint32_t c = pAttachments[i].colorAttachment;
         a = subpass->color_attachments[c].attachment;
      } else {
         a = subpass->depth_stencil_attachment.attachment;
      }
      if (a != VK_ATTACHMENT_UNUSED) {
         const struct tu_render_pass_attachment *att = &cmd->state.pass->attachments[a];
         const struct tu_image_view *iview = cmd->state.attachments[a];
         trace_start_generic_clear(&cmd->trace, cs, att->format,
                                   iview->view.ubwc_enabled, att->samples);
         for (unsigned j = 0; j < rectCount; j++) {
            tu7_clear_attachment_generic_single_rect(
               cmd, cs, &resolve_group, att, &pAttachments[i], a, &pRects[j]);
         }
         trace_end_generic_clear(&cmd->trace, cs);
      }
   }

   tu_emit_resolve_group<A7XX>(cmd, cs, &resolve_group);
}

template <chip CHIP>
VKAPI_ATTR void VKAPI_CALL
tu_CmdClearAttachments(VkCommandBuffer commandBuffer,
                       uint32_t attachmentCount,
                       const VkClearAttachment *pAttachments,
                       uint32_t rectCount,
                       const VkClearRect *pRects)
{
   VK_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);

   for (uint32_t j = 0; j < attachmentCount; j++) {
      if ((pAttachments[j].aspectMask & VK_IMAGE_ASPECT_DEPTH_BIT) == 0)
         continue;

      tu_lrz_disable_during_renderpass<CHIP>(cmd);
   }

   if (cmd->device->physical_device->info->a7xx.has_generic_clear &&
       /* Both having predication and not knowing layout could be solved
        * by cs patching, which is exactly what prop driver is doing.
        * We don't implement it because we don't expect a reasonable impact.
        */
       !(cmd->state.predication_active ||
         cmd->state.gmem_layout == TU_GMEM_LAYOUT_COUNT)) {
      tu_clear_attachments_generic(cmd, attachmentCount, pAttachments, rectCount, pRects);
   } else {
      tu_clear_attachments<CHIP>(cmd, attachmentCount, pAttachments,
                                 rectCount, pRects);
   }
}
TU_GENX(tu_CmdClearAttachments);

template <chip CHIP>
static void
clear_sysmem_attachment(struct tu_cmd_buffer *cmd,
                        struct tu_cs *cs,
                        VkFormat vk_format,
                        VkImageAspectFlags clear_mask,
                        uint32_t a,
                        bool separate_ds)
{
   enum pipe_format format = vk_format_to_pipe_format(vk_format);
   const struct tu_framebuffer *fb = cmd->state.framebuffer;
   const struct tu_image_view *iview = cmd->state.attachments[a];
   const uint32_t clear_views = cmd->state.pass->attachments[a].clear_views;
   const struct blit_ops *ops = &r2d_ops<CHIP>;
   const VkClearValue *value = &cmd->state.clear_values[a];
   if (cmd->state.pass->attachments[a].samples > 1)
      ops = &r3d_ops<CHIP>;

   trace_start_sysmem_clear(&cmd->trace, cs, vk_format, ops == &r3d_ops<CHIP>,
                            cmd->state.pass->attachments[a].samples);

   ops->setup(cmd, cs, format, format, clear_mask, 0, true, iview->view.ubwc_enabled,
              cmd->state.pass->attachments[a].samples);
   ops->coords(cmd, cs, cmd->state.render_area.offset, (VkOffset2D) {},
               cmd->state.render_area.extent);
   ops->clear_value(cmd, cs, format, value);

   for_each_layer(i, clear_views, fb->layers) {
      if (separate_ds) {
         if (vk_format == VK_FORMAT_D32_SFLOAT) {
            ops->dst_depth(cs, iview, i);
         } else {
            ops->dst_stencil(cs, iview, i);
         }
      } else {
         ops->dst(cs, &iview->view, i, format);
      }
      ops->run(cmd, cs);
   }

   ops->teardown(cmd, cs);

   trace_end_sysmem_clear(&cmd->trace, cs);
}

template <chip CHIP>
void
tu_clear_sysmem_attachment(struct tu_cmd_buffer *cmd,
                           struct tu_cs *cs,
                           uint32_t a)
{
   const struct tu_render_pass_attachment *attachment =
      &cmd->state.pass->attachments[a];

   if (!attachment->clear_mask)
      return;

   if (attachment->format == VK_FORMAT_D32_SFLOAT_S8_UINT) {
      if (attachment->clear_mask & VK_IMAGE_ASPECT_DEPTH_BIT) {
         clear_sysmem_attachment<CHIP>(cmd, cs, VK_FORMAT_D32_SFLOAT, VK_IMAGE_ASPECT_COLOR_BIT,
                                 a, true);
      }
      if (attachment->clear_mask & VK_IMAGE_ASPECT_STENCIL_BIT) {
         clear_sysmem_attachment<CHIP>(cmd, cs, VK_FORMAT_S8_UINT, VK_IMAGE_ASPECT_COLOR_BIT,
                                 a, true);
      }
   } else {
      clear_sysmem_attachment<CHIP>(cmd, cs, attachment->format, attachment->clear_mask,
                              a, false);
   }

   /* The spec doesn't explicitly say, but presumably the initial renderpass
    * clear is considered part of the renderpass, and therefore barriers
    * aren't required inside the subpass/renderpass.  Therefore we need to
    * flush CCU color into CCU depth here, just like with
    * vkCmdClearAttachments(). Note that because this only happens at the
    * beginning of a renderpass, and renderpass writes are considered
    * "incoherent", we shouldn't have to worry about syncing depth into color
    * beforehand as depth should already be flushed.
    */
   if (vk_format_is_depth_or_stencil(attachment->format)) {
      tu_emit_event_write<CHIP>(cmd, cs, FD_CCU_CLEAN_COLOR);
      tu_emit_event_write<CHIP>(cmd, cs, FD_CCU_CLEAN_DEPTH);
      tu_emit_event_write<CHIP>(cmd, cs, FD_CCU_INVALIDATE_DEPTH);
   } else {
      tu_emit_event_write<CHIP>(cmd, cs, FD_CCU_CLEAN_COLOR);
      tu_emit_event_write<CHIP>(cmd, cs, FD_CCU_INVALIDATE_COLOR);
   }

   tu_cs_emit_wfi(cs);
}
TU_GENX(tu_clear_sysmem_attachment);

template <chip CHIP>
void
tu_clear_gmem_attachment(struct tu_cmd_buffer *cmd,
                         struct tu_cs *cs,
                         struct tu_resolve_group *resolve_group,
                         uint32_t a)
{
   const struct tu_render_pass_attachment *attachment =
      &cmd->state.pass->attachments[a];

   if (!attachment->clear_mask)
      return;

   tu_emit_clear_gmem_attachment<CHIP>(cmd, cs, resolve_group, a, 0,
                                 cmd->state.framebuffer->layers,
                                 attachment->clear_views,
                                 attachment->clear_mask,
                                 &cmd->state.clear_values[a]);
}
TU_GENX(tu_clear_gmem_attachment);

void
tu7_generic_clear_attachment(struct tu_cmd_buffer *cmd,
                             struct tu_cs *cs,
                             struct tu_resolve_group *resolve_group,
                             uint32_t a)
{
   const struct tu_render_pass_attachment *att =
      &cmd->state.pass->attachments[a];
   const VkClearValue *value = &cmd->state.clear_values[a];
   const struct tu_image_view *iview = cmd->state.attachments[a];

   trace_start_generic_clear(&cmd->trace, cs, att->format,
                             iview->view.ubwc_enabled, att->samples);

   enum pipe_format format = vk_format_to_pipe_format(att->format);
   for_each_layer(i, att->clear_views, cmd->state.framebuffer->layers) {
      uint32_t layer = i + 0;
      uint32_t mask =
         aspect_write_mask_generic_clear(format, att->clear_mask);
      if (att->format == VK_FORMAT_D32_SFLOAT_S8_UINT) {
         if (att->clear_mask & VK_IMAGE_ASPECT_DEPTH_BIT) {
            uint32_t buffer_id = tu_resolve_group_include_buffer<A7XX>(resolve_group, TU_RESOLVE_GROUP_DEPTH_BUFFER);
            tu7_generic_layer_clear(cmd, cs, buffer_id, PIPE_FORMAT_Z32_FLOAT, mask,
                                    false, layer, value, a);
         }
         if (att->clear_mask & VK_IMAGE_ASPECT_STENCIL_BIT) {
            uint32_t buffer_id = tu_resolve_group_include_buffer<A7XX>(resolve_group, TU_RESOLVE_GROUP_STENCIL_BUFFER);
            tu7_generic_layer_clear(cmd, cs, buffer_id, PIPE_FORMAT_S8_UINT, mask, true,
                                    layer, value, a);
         }
      } else {
         uint32_t buffer_id = tu_resolve_group_include_buffer_for_format<A7XX>(resolve_group, att->format);
         tu7_generic_layer_clear(cmd, cs, buffer_id, format, mask, false, layer, value, a);
      }
   }

   tu_flush_for_access(&cmd->state.renderpass_cache,
                       TU_ACCESS_BLIT_WRITE_GMEM, TU_ACCESS_NONE);

   trace_end_generic_clear(&cmd->trace, cs);
}

template <chip CHIP>
static void
tu_emit_blit(struct tu_cmd_buffer *cmd,
             struct tu_cs *cs,
             struct tu_resolve_group *resolve_group,
             const struct tu_image_view *iview,
             const struct tu_render_pass_attachment *attachment,
             const VkClearValue *clear_value,
             enum a6xx_blit_event_type blit_event_type,
             bool separate_stencil)
{
   assert(blit_event_type != BLIT_EVENT_CLEAR);
   uint32_t clear_mask = 0;

   /* BLIT_EVENT_STORE_AND_CLEAR would presumably swallow the
    * BLIT_EVENT_CLEAR at the start of a renderpass, and be more efficient.
    */
   if (blit_event_type == BLIT_EVENT_STORE && clear_value &&
       attachment->clear_mask &&
       use_generic_clear_for_image_clear(cmd, iview->image)) {
      blit_event_type = BLIT_EVENT_STORE_AND_CLEAR;

      enum pipe_format format = vk_format_to_pipe_format(attachment->format);
      VkImageAspectFlags aspect_mask = attachment->clear_mask;
      if (format == PIPE_FORMAT_Z24_UNORM_S8_UINT) {
         if (separate_stencil)
            aspect_mask = VK_IMAGE_ASPECT_STENCIL_BIT;
         else
            aspect_mask = VK_IMAGE_ASPECT_DEPTH_BIT;
      }
      if (format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT) {
         if (separate_stencil)
            format = PIPE_FORMAT_S8_UINT;
         else
            format = PIPE_FORMAT_Z32_FLOAT;
      }

      clear_mask = aspect_write_mask_generic_clear(format, aspect_mask);

      uint32_t clear_vals[4] = {};
      pack_blit_event_clear_value(clear_value, format, clear_vals);

      tu_cs_emit_pkt4(cs, REG_A6XX_RB_BLIT_CLEAR_COLOR_DW0, 4);
      tu_cs_emit_array(cs, clear_vals, 4);
   }

   enum tu_resolve_group_buffer_type buffer_type = TU_RESOLVE_GROUP_COLOR_BUFFER;
   if (attachment->format == VK_FORMAT_D32_SFLOAT_S8_UINT) {
      if (!separate_stencil)
         buffer_type = TU_RESOLVE_GROUP_DEPTH_BUFFER;
      else
         buffer_type = TU_RESOLVE_GROUP_STENCIL_BUFFER;
   } else if (attachment->format == VK_FORMAT_D24_UNORM_S8_UINT) {
      buffer_type = TU_RESOLVE_GROUP_DEPTH_BUFFER;
   }

   uint32_t buffer_id = tu_resolve_group_include_buffer<CHIP>(resolve_group, buffer_type);
   event_blit_setup(cs, buffer_id, attachment, blit_event_type, clear_mask);

   for_each_layer(i, attachment->clear_views, cmd->state.framebuffer->layers) {
      event_blit_dst_view blt_view = blt_view_from_tu_view(iview, i);
      event_blit_run<CHIP>(cmd, cs, attachment, &blt_view, separate_stencil);
   }

   tu_flush_for_access(&cmd->state.cache, TU_ACCESS_BLIT_WRITE_GMEM,
                       TU_ACCESS_NONE);
}

static bool
blit_can_resolve(VkFormat format)
{
   const struct util_format_description *desc = vk_format_description(format);

   /* blit event can only do resolve for simple cases:
    * averaging samples as unsigned integers or choosing only one sample
    * Note this is allowed for SRGB formats, but results differ from 2D draw resolve
    */
   if (vk_format_is_snorm(format))
      return false;

   /* can't do formats with larger channel sizes
    * note: this includes all float formats
    * note2: single channel integer formats seem OK
    */
   if (desc->channel[0].size > 10 && vk_format_is_color(format))
      return false;

   switch (format) {
   /* for unknown reasons blit event can't msaa resolve these formats when tiled
    * likely related to these formats having different layout from other cpp=2 formats
    */
   case VK_FORMAT_R8G8_UNORM:
   case VK_FORMAT_R8G8_UINT:
   case VK_FORMAT_R8G8_SINT:
   case VK_FORMAT_R8G8_SRGB:
      return false;
   default:
      break;
   }

   return true;
}

struct apply_load_coords_state {
   unsigned view;
};

static void
fdm_apply_load_coords(struct tu_cmd_buffer *cmd,
                      struct tu_cs *cs,
                      void *data,
                      VkRect2D bin,
                      unsigned views,
                      VkExtent2D *frag_areas)
{
   const struct apply_load_coords_state *state =
      (const struct apply_load_coords_state *)data;
   assert(state->view < views);
   VkExtent2D frag_area = frag_areas[state->view];

   assert(bin.extent.width % frag_area.width == 0);
   assert(bin.extent.height % frag_area.height == 0);
   uint32_t scaled_width = bin.extent.width / frag_area.width;
   uint32_t scaled_height = bin.extent.height / frag_area.height;

   const float coords[] = {
      bin.offset.x,                    bin.offset.y,
      bin.offset.x,                    bin.offset.y,
      bin.offset.x + scaled_width,     bin.offset.y + scaled_height,
      bin.offset.x + bin.extent.width, bin.offset.y + bin.extent.height,
   };
   r3d_coords_raw(cmd, cs, coords);
}

template <chip CHIP>
static void
load_3d_blit(struct tu_cmd_buffer *cmd,
             struct tu_cs *cs,
             const struct tu_image_view *iview,
             const struct tu_render_pass_attachment *att,
             bool separate_stencil)
{
   const struct tu_framebuffer *fb = cmd->state.framebuffer;
   enum pipe_format format = iview->view.format;
   if (iview->image->vk.format == VK_FORMAT_D32_SFLOAT_S8_UINT) {
      if (separate_stencil)
         format = PIPE_FORMAT_S8_UINT;
      else
         format = PIPE_FORMAT_Z32_FLOAT;
   }
   r3d_setup<CHIP>(cmd, cs, format, format, VK_IMAGE_ASPECT_COLOR_BIT,
                   R3D_DST_GMEM, false, iview->view.ubwc_enabled,
                   iview->image->vk.samples);

   if (!cmd->state.pass->has_fdm) {
      r3d_coords(cmd, cs, (VkOffset2D) { 0, 0 }, (VkOffset2D) { 0, 0 },
                 (VkExtent2D) { fb->width, fb->height });
   }

   /* Normal loads read directly from system memory, so we have to invalidate
    * UCHE in case it contains stale data.
    */
   tu_emit_event_write<CHIP>(cmd, cs, FD_CACHE_INVALIDATE);

   /* Wait for CACHE_INVALIDATE to land */
   tu_cs_emit_wfi(cs);

   for_each_layer(i, att->clear_views, cmd->state.framebuffer->layers) {
      if (cmd->state.pass->has_fdm) {
         struct apply_load_coords_state state = {
            .view = att->clear_views ? i : 0,
         };
         tu_create_fdm_bin_patchpoint(cmd, cs, 4, fdm_apply_load_coords, state);
      }

      r3d_dst_gmem<CHIP>(cmd, cs, iview, att, separate_stencil, i);

      if (iview->image->vk.format == VK_FORMAT_D32_SFLOAT_S8_UINT) {
         if (separate_stencil)
            r3d_src_stencil(cmd, cs, iview, i);
         else
            r3d_src_depth(cmd, cs, iview, i);
      } else {
         r3d_src_gmem_load(cmd, cs, iview, i);
      }

      r3d_run(cmd, cs);
   }

   r3d_teardown<CHIP>(cmd, cs);

   /* It seems we need to WFI here for depth/stencil because color writes here
    * aren't synchronized with depth/stencil writes.
    *
    * Note: the blob also uses a WFI for color attachments but this hasn't
    * been seen to be necessary.
    */
   if (vk_format_is_depth_or_stencil(att->format))
      tu_cs_emit_wfi(cs);
}

static void
tu_begin_load_store_cond_exec(struct tu_cmd_buffer *cmd,
                              struct tu_cs *cs, bool load)
{
   tu_cond_exec_start(cs, CP_COND_REG_EXEC_0_MODE(PRED_TEST));

   if (!TU_DEBUG(LOG_SKIP_GMEM_OPS))
      return;

   uint64_t result_iova;
   if (load)
      result_iova = global_iova(cmd, dbg_gmem_taken_loads);
   else
      result_iova = global_iova(cmd, dbg_gmem_taken_stores);

   tu_cs_emit_pkt7(cs, CP_MEM_TO_MEM, 7);
   tu_cs_emit(cs, CP_MEM_TO_MEM_0_NEG_B);
   tu_cs_emit_qw(cs, result_iova);
   tu_cs_emit_qw(cs, result_iova);
   tu_cs_emit_qw(cs, global_iova(cmd, dbg_one));
}

static void
tu_end_load_store_cond_exec(struct tu_cmd_buffer *cmd,
                            struct tu_cs *cs, bool load)
{
   tu_cond_exec_end(cs);

   if (!TU_DEBUG(LOG_SKIP_GMEM_OPS))
      return;

   uint64_t result_iova;
   if (load)
      result_iova = global_iova(cmd, dbg_gmem_total_loads);
   else
      result_iova = global_iova(cmd, dbg_gmem_total_stores);

   tu_cs_emit_pkt7(cs, CP_MEM_TO_MEM, 7);
   tu_cs_emit(cs, CP_MEM_TO_MEM_0_NEG_B);
   tu_cs_emit_qw(cs, result_iova);
   tu_cs_emit_qw(cs, result_iova);
   tu_cs_emit_qw(cs, global_iova(cmd, dbg_one));
}

template <chip CHIP>
void
tu_load_gmem_attachment(struct tu_cmd_buffer *cmd,
                        struct tu_cs *cs,
                        struct tu_resolve_group *resolve_group,
                        uint32_t a,
                        bool cond_exec_allowed,
                        bool force_load)
{
   const struct tu_image_view *iview = cmd->state.attachments[a];
   const struct tu_render_pass_attachment *attachment =
      &cmd->state.pass->attachments[a];

   bool load_common = attachment->load || force_load;
   bool load_stencil =
      attachment->load_stencil ||
      (attachment->format == VK_FORMAT_D32_SFLOAT_S8_UINT && force_load);

   if (!load_common && !load_stencil)
      return;

   trace_start_gmem_load(&cmd->trace, cs, attachment->format, force_load);

   /* If attachment will be cleared by vkCmdClearAttachments - it is likely
    * that it would be partially cleared, and since it is done by 2d blit
    * it doesn't produce geometry, so we have to unconditionally load.
    *
    * To simplify conditions treat partially cleared separate DS as fully
    * cleared and don't emit cond_exec.
    */
   bool cond_exec = cond_exec_allowed && attachment->cond_load_allowed;
   if (cond_exec)
      tu_begin_load_store_cond_exec(cmd, cs, true);

   if (TU_DEBUG(3D_LOAD) ||
       cmd->state.pass->has_fdm) {
      if (load_common || load_stencil)
         tu_disable_draw_states(cmd, cs);

      if (load_common)
         load_3d_blit<CHIP>(cmd, cs, iview, attachment, false);

      if (load_stencil)
         load_3d_blit<CHIP>(cmd, cs, iview, attachment, true);
   } else {
      if (load_common)
         tu_emit_blit<CHIP>(cmd, cs, resolve_group, iview, attachment, NULL, BLIT_EVENT_LOAD, false);

      if (load_stencil)
         tu_emit_blit<CHIP>(cmd, cs, resolve_group, iview, attachment, NULL, BLIT_EVENT_LOAD, true);
   }

   if (cond_exec)
      tu_end_load_store_cond_exec(cmd, cs, true);

   trace_end_gmem_load(&cmd->trace, cs);
}
TU_GENX(tu_load_gmem_attachment);

template <chip CHIP>
static void
store_cp_blit(struct tu_cmd_buffer *cmd,
              struct tu_cs *cs,
              const struct tu_image_view *iview,
              uint32_t samples,
              bool separate_stencil,
              enum pipe_format src_format,
              enum pipe_format dst_format,
              uint32_t layer,
              uint32_t gmem_offset,
              uint32_t cpp)
{
   r2d_setup_common<CHIP>(cmd, cs, src_format, dst_format,
                          VK_IMAGE_ASPECT_COLOR_BIT, 0, false,
                          iview->view.ubwc_enabled, true);

   if (iview->image->vk.format == VK_FORMAT_D32_SFLOAT_S8_UINT) {
      if (!separate_stencil) {
         r2d_dst_depth(cs, iview, layer);
      } else {
         r2d_dst_stencil(cs, iview, layer);
      }
   } else {
      r2d_dst<CHIP>(cs, &iview->view, layer, src_format);
   }

   enum a6xx_format fmt = blit_format_texture<CHIP>(src_format, TILE6_2, false, true).fmt;
   fixup_src_format(&src_format, dst_format, &fmt);

   tu_cs_emit_regs(cs,
                   SP_PS_2D_SRC_INFO(CHIP,
                      .color_format = fmt,
                      .tile_mode = TILE6_2,
                      .color_swap = WZYX,
                      .srgb = util_format_is_srgb(src_format),
                      .samples = tu_msaa_samples(samples),
                      .samples_average = !util_format_is_pure_integer(dst_format) &&
                                         !util_format_is_depth_or_stencil(dst_format),
                      .unk20 = 1,
                      .unk22 = 1),
                   SP_PS_2D_SRC_SIZE(CHIP, .width = iview->vk.extent.width, .height = iview->vk.extent.height),
                   SP_PS_2D_SRC(CHIP, .qword = cmd->device->physical_device->gmem_base + gmem_offset),
                   SP_PS_2D_SRC_PITCH(CHIP, .pitch = cmd->state.tiling->tile0.width * cpp));

   /* sync GMEM writes with CACHE. */
   tu_emit_event_write<CHIP>(cmd, cs, FD_CACHE_INVALIDATE);
   if (CHIP >= A7XX)
      /* On A7XX, we need to wait for any CP_EVENT_WRITE::BLIT operations
       * arising from GMEM load/clears to land before we can continue.
       */
      tu_emit_event_write<CHIP>(cmd, cs, FD_CCU_CLEAN_BLIT_CACHE);

   /* Wait for cache event to land */
   tu_cs_emit_wfi(cs);

   r2d_run(cmd, cs);

   /* CP_BLIT writes to the CCU, unlike CP_EVENT_WRITE::BLIT which writes to
    * sysmem, and we generally assume that GMEM renderpasses leave their
    * results in sysmem, so we need to flush manually here.
    */
   tu_emit_event_write<CHIP>(cmd, cs, FD_CCU_CLEAN_COLOR);
}

template <chip CHIP>
static void
store_3d_blit(struct tu_cmd_buffer *cmd,
              struct tu_cs *cs,
              const struct tu_image_view *iview,
              VkSampleCountFlagBits dst_samples,
              bool separate_stencil,
              enum pipe_format src_format,
              enum pipe_format dst_format,
              const VkRect2D *render_area,
              uint32_t layer,
              uint32_t gmem_offset,
              uint32_t cpp)
{
   /* RB_BIN_CONTROL/GRAS_BIN_CONTROL are normally only set once and they
    * aren't set until we know whether we're HW binning or not, and we want to
    * avoid a dependence on that here to be able to store attachments before
    * the end of the renderpass in the future. Use the scratch space to
    * save/restore them dynamically.
    */
   tu_cs_emit_pkt7(cs, CP_REG_TO_SCRATCH, 1);
   tu_cs_emit(cs, CP_REG_TO_SCRATCH_0_REG(REG_A6XX_RB_BIN_CONTROL) |
                  CP_REG_TO_SCRATCH_0_SCRATCH(0) |
                  CP_REG_TO_SCRATCH_0_CNT(1 - 1));
   if (CHIP >= A7XX) {
      tu_cs_emit_pkt7(cs, CP_REG_TO_SCRATCH, 1);
      tu_cs_emit(cs, CP_REG_TO_SCRATCH_0_REG(REG_A7XX_RB_UNKNOWN_8812) |
                     CP_REG_TO_SCRATCH_0_SCRATCH(1) |
                     CP_REG_TO_SCRATCH_0_CNT(1 - 1));
   }

   r3d_setup<CHIP>(cmd, cs, src_format, dst_format, VK_IMAGE_ASPECT_COLOR_BIT,
                   0, false, iview->view.ubwc_enabled, dst_samples);

   r3d_coords(cmd, cs, render_area->offset, render_area->offset, render_area->extent);

   if (iview->image->vk.format == VK_FORMAT_D32_SFLOAT_S8_UINT) {
      if (!separate_stencil) {
         r3d_dst_depth<CHIP>(cs, iview, layer);
      } else {
         r3d_dst_stencil<CHIP>(cs, iview, layer);
      }
   } else {
      r3d_dst<CHIP>(cs, &iview->view, layer, src_format);
   }

   r3d_src_gmem<CHIP>(cmd, cs, iview, src_format, dst_format, gmem_offset, cpp);

   /* sync GMEM writes with CACHE. */
   tu_emit_event_write<CHIP>(cmd, cs, FD_CACHE_INVALIDATE);

   /* Wait for CACHE_INVALIDATE to land */
   tu_cs_emit_wfi(cs);

   r3d_run(cmd, cs);

   r3d_teardown<CHIP>(cmd, cs);

   /* Draws write to the CCU, unlike CP_EVENT_WRITE::BLIT which writes to
    * sysmem, and we generally assume that GMEM renderpasses leave their
    * results in sysmem, so we need to flush manually here. The 3d blit path
    * writes to depth images as a color RT, so there's no need to flush depth.
    */
   tu_emit_event_write<CHIP>(cmd, cs, FD_CCU_CLEAN_COLOR);

   /* Restore RB_BIN_CONTROL/GRAS_BIN_CONTROL saved above. */
   tu_cs_emit_pkt7(cs, CP_SCRATCH_TO_REG, 1);
   tu_cs_emit(cs, CP_SCRATCH_TO_REG_0_REG(REG_A6XX_RB_BIN_CONTROL) |
                  CP_SCRATCH_TO_REG_0_SCRATCH(0) |
                  CP_SCRATCH_TO_REG_0_CNT(1 - 1));

   tu_cs_emit_pkt7(cs, CP_SCRATCH_TO_REG, 1);
   tu_cs_emit(cs, CP_SCRATCH_TO_REG_0_REG(REG_A6XX_GRAS_BIN_CONTROL) |
                  CP_SCRATCH_TO_REG_0_SCRATCH(0) |
                  CP_SCRATCH_TO_REG_0_CNT(1 - 1));

   if (CHIP >= A7XX) {
      tu_cs_emit_pkt7(cs, CP_SCRATCH_TO_REG, 1);
      tu_cs_emit(cs, CP_SCRATCH_TO_REG_0_REG(REG_A7XX_RB_UNKNOWN_8812) |
                        CP_SCRATCH_TO_REG_0_SCRATCH(1) |
                        CP_SCRATCH_TO_REG_0_CNT(1 - 1));
   }
}

static bool
tu_attachment_store_unaligned(struct tu_cmd_buffer *cmd, uint32_t a)
{
   struct tu_physical_device *phys_dev = cmd->device->physical_device;
   const struct tu_image_view *iview = cmd->state.attachments[a];
   const VkRect2D *render_area = &cmd->state.render_area;

   /* Unaligned store is incredibly rare in CTS, we have to force it to test. */
   if (TU_DEBUG(UNALIGNED_STORE))
      return true;

   /* We always use the unaligned store path when scaling rendering. */
   if (cmd->state.pass->has_fdm)
      return true;

   uint32_t x1 = render_area->offset.x;
   uint32_t y1 = render_area->offset.y;
   uint32_t x2 = x1 + render_area->extent.width;
   uint32_t y2 = y1 + render_area->extent.height;
   /* x2/y2 can be unaligned if equal to the size of the image, since it will
    * write into padding space. The one exception is linear levels which don't
    * have the required y padding in the layout (except for the last level)
    */
   bool need_y2_align =
      y2 != iview->view.height || iview->view.need_y2_align;

   return (x1 % phys_dev->info->gmem_align_w ||
           (x2 % phys_dev->info->gmem_align_w && x2 != iview->view.width) ||
           y1 % phys_dev->info->gmem_align_h ||
           (y2 % phys_dev->info->gmem_align_h && need_y2_align));
}

/* Choose the GMEM layout (use the CCU space or not) based on whether the
 * current attachments will need.  This has to happen at vkBeginRenderPass()
 * time because tu_attachment_store_unaligned() looks at the image views, which
 * are only available at that point.  This should match the logic for the
 * !use_fast_path case in tu_store_gmem_attachment().
 */
void
tu_choose_gmem_layout(struct tu_cmd_buffer *cmd)
{
   cmd->state.gmem_layout = TU_GMEM_LAYOUT_FULL;

   for (unsigned i = 0; i < cmd->state.pass->attachment_count; i++) {
      if (!cmd->state.attachments[i])
         continue;

      struct tu_render_pass_attachment *att =
         &cmd->state.pass->attachments[i];
      if ((att->store || att->store_stencil) &&
          tu_attachment_store_unaligned(cmd, i))
         cmd->state.gmem_layout = TU_GMEM_LAYOUT_AVOID_CCU;
      if (att->store && att->format == VK_FORMAT_S8_UINT)
         /* We cannot pick out S8 from D24S8/D32S8, so we conservatively disable
          * blit events for the S8_UINT format.
          */
         cmd->state.gmem_layout = TU_GMEM_LAYOUT_AVOID_CCU;
      if (att->will_be_resolved && !blit_can_resolve(att->format))
         cmd->state.gmem_layout = TU_GMEM_LAYOUT_AVOID_CCU;
   }

   cmd->state.tiling = &cmd->state.framebuffer->tiling[cmd->state.gmem_layout];
}

struct apply_store_coords_state {
   unsigned view;
};

static void
fdm_apply_store_coords(struct tu_cmd_buffer *cmd,
                       struct tu_cs *cs,
                       void *data,
                       VkRect2D bin,
                       unsigned views,
                       VkExtent2D *frag_areas)
{
   const struct apply_store_coords_state *state =
      (const struct apply_store_coords_state *)data;
   assert(state->view < views);
   VkExtent2D frag_area = frag_areas[state->view];

   /* The bin width/height must be a multiple of the frag_area to make sure
    * that the scaling happens correctly. This means there may be some
    * destination pixels jut out of the framebuffer, but they should be
    * clipped by the render area.
    */
   assert(bin.extent.width % frag_area.width == 0);
   assert(bin.extent.height % frag_area.height == 0);
   uint32_t scaled_width = bin.extent.width / frag_area.width;
   uint32_t scaled_height = bin.extent.height / frag_area.height;

   tu_cs_emit_regs(cs,
      A6XX_GRAS_2D_DST_TL(.x = bin.offset.x,
                          .y = bin.offset.y),
      A6XX_GRAS_2D_DST_BR(.x = bin.offset.x + bin.extent.width - 1,
                          .y = bin.offset.y + bin.extent.height - 1));
   tu_cs_emit_regs(cs,
                   A6XX_GRAS_2D_SRC_TL_X(bin.offset.x),
                   A6XX_GRAS_2D_SRC_BR_X(bin.offset.x + scaled_width - 1),
                   A6XX_GRAS_2D_SRC_TL_Y(bin.offset.y),
                   A6XX_GRAS_2D_SRC_BR_Y(bin.offset.y + scaled_height - 1));
}

template <chip CHIP>
void
tu_store_gmem_attachment(struct tu_cmd_buffer *cmd,
                         struct tu_cs *cs,
                         struct tu_resolve_group *resolve_group,
                         uint32_t a,
                         uint32_t gmem_a,
                         uint32_t layers,
                         uint32_t layer_mask,
                         bool cond_exec_allowed)
{
   const VkRect2D *render_area = &cmd->state.render_area;
   struct tu_render_pass_attachment *dst = &cmd->state.pass->attachments[a];
   const struct tu_image_view *iview = cmd->state.attachments[a];
   struct tu_render_pass_attachment *src = &cmd->state.pass->attachments[gmem_a];
   const VkClearValue *clear_value = &cmd->state.clear_values[gmem_a];
   bool resolve = a != gmem_a;
   if (resolve)
      clear_value = NULL;

   if (!dst->store && !dst->store_stencil)
      return;

   bool unaligned = tu_attachment_store_unaligned(cmd, a);

   /* D32_SFLOAT_S8_UINT is quite special format: it has two planes,
    * one for depth and other for stencil. When resolving a MSAA
    * D32_SFLOAT_S8_UINT to S8_UINT, we need to take that into account.
    */
   bool resolve_d32s8_s8 =
      src->format == VK_FORMAT_D32_SFLOAT_S8_UINT &&
      dst->format == VK_FORMAT_S8_UINT;

   /* The fast path doesn't support picking out the last component of a D24S8
    * texture reinterpreted as RGBA8_UNORM.
    */
   bool resolve_d24s8_s8 =
      src->format == VK_FORMAT_D24_UNORM_S8_UINT &&
      dst->format == VK_FORMAT_S8_UINT;

   bool store_common = dst->store && !resolve_d32s8_s8;
   bool store_separate_stencil = dst->store_stencil || resolve_d32s8_s8;

   bool use_fast_path = !unaligned && !resolve_d24s8_s8 &&
                        (a == gmem_a || blit_can_resolve(dst->format));

   trace_start_gmem_store(&cmd->trace, cs, dst->format, use_fast_path, unaligned);

   /* Unconditional store should happen only if attachment was cleared,
    * which could have happened either by load_op or via vkCmdClearAttachments.
    */
   bool cond_exec = cond_exec_allowed && src->cond_store_allowed;
   if (cond_exec) {
      tu_begin_load_store_cond_exec(cmd, cs, false);
   }

   /* use fast path when render area is aligned, except for unsupported resolve cases */
   if (use_fast_path) {
      if (store_common)
         tu_emit_blit<CHIP>(cmd, cs, resolve_group, iview, src, clear_value, BLIT_EVENT_STORE, false);
      if (store_separate_stencil)
         tu_emit_blit<CHIP>(cmd, cs, resolve_group, iview, src, clear_value, BLIT_EVENT_STORE, true);

      if (cond_exec) {
         tu_end_load_store_cond_exec(cmd, cs, false);
      }

      trace_end_gmem_store(&cmd->trace, cs);
      return;
   }

   assert(cmd->state.gmem_layout == TU_GMEM_LAYOUT_AVOID_CCU);

   enum pipe_format src_format = vk_format_to_pipe_format(src->format);
   if (src_format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT)
      src_format = PIPE_FORMAT_Z32_FLOAT;

   enum pipe_format dst_format = vk_format_to_pipe_format(dst->format);
   if (dst_format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT)
      dst_format = PIPE_FORMAT_Z32_FLOAT;

   if (dst->samples > 1) {
      /* If we hit this path, we have to disable draw states after every tile
       * instead of once at the end of the renderpass, so that they aren't
       * executed when calling CP_DRAW.
       *
       * TODO: store a flag somewhere so we don't do this more than once and
       * don't do it after the renderpass when this happens.
       */
      if (store_common || store_separate_stencil)
         tu_disable_draw_states(cmd, cs);

      for_each_layer(i, layer_mask, layers) {
         if (store_common) {
            store_3d_blit<CHIP>(cmd, cs, iview, dst->samples, false, src_format,
                          dst_format, render_area, i, tu_attachment_gmem_offset(cmd, src, i), src->cpp);
         }
         if (store_separate_stencil) {
            store_3d_blit<CHIP>(cmd, cs, iview, dst->samples, true, PIPE_FORMAT_S8_UINT,
                          PIPE_FORMAT_S8_UINT, render_area, i,
                          tu_attachment_gmem_offset_stencil(cmd, src, i), src->samples);
         }
      }
   } else {
      if (!cmd->state.pass->has_fdm) {
         r2d_coords(cmd, cs, render_area->offset, render_area->offset,
                    render_area->extent);
      } else {
         /* Usually GRAS_2D_RESOLVE_CNTL_* clips the destination to the bin
          * area and the coordinates span the entire render area, but for
          * FDM we need to scale the coordinates so we need to take the
          * opposite aproach, specifying the exact bin size in the destination
          * coordinates and using GRAS_2D_RESOLVE_CNTL_* to clip to the render
          * area.
          */
         tu_cs_emit_regs(cs,
                         A6XX_GRAS_2D_RESOLVE_CNTL_1(.x = render_area->offset.x,
                                                     .y = render_area->offset.y,),
                         A6XX_GRAS_2D_RESOLVE_CNTL_2(.x = render_area->offset.x + render_area->extent.width - 1,
                                                     .y = render_area->offset.y + render_area->extent.height - 1,));
      }

      for_each_layer (i, layer_mask, layers) {
         if (cmd->state.pass->has_fdm) {
            unsigned view = layer_mask ? i : 0;
            struct apply_store_coords_state state = {
               .view = view,
            };
            tu_create_fdm_bin_patchpoint(cmd, cs, 8, fdm_apply_store_coords,
                                         state);
         }
         if (store_common) {
            store_cp_blit<CHIP>(cmd, cs, iview, src->samples, false, src_format,
                          dst_format, i, tu_attachment_gmem_offset(cmd, src, i), src->cpp);
         }
         if (store_separate_stencil) {
            store_cp_blit<CHIP>(cmd, cs, iview, src->samples, true, PIPE_FORMAT_S8_UINT,
                          PIPE_FORMAT_S8_UINT, i, tu_attachment_gmem_offset_stencil(cmd, src, i), src->samples);
         }
      }
   }

   if (cond_exec) {
      tu_end_load_store_cond_exec(cmd, cs, false);
   }

   trace_end_gmem_store(&cmd->trace, cs);
}
TU_GENX(tu_store_gmem_attachment);