drivers/etnaviv/etnaviv_transfer.c

/*
 * Copyright (c) 2012-2015 Etnaviv Project
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sub license,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial portions
 * of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 *
 * Authors:
 *    Wladimir J. van der Laan <laanwj@gmail.com>
 */

#include "etnaviv_transfer.h"
#include "etnaviv_clear_blit.h"
#include "etnaviv_context.h"
#include "etnaviv_debug.h"
#include "etnaviv_etc2.h"
#include "etnaviv_screen.h"

#include "pipe/p_defines.h"
#include "util/format/u_formats.h"
#include "pipe/p_screen.h"
#include "pipe/p_state.h"
#include "util/format/u_format.h"
#include "util/u_inlines.h"
#include "util/u_memory.h"
#include "util/u_surface.h"
#include "util/u_transfer.h"

#include "hw/common_3d.xml.h"

#include "drm-uapi/drm_fourcc.h"

#define ETNA_PIPE_MAP_DISCARD_LEVEL   (PIPE_MAP_DRV_PRV << 0)

/* Compute offset into a 1D/2D/3D buffer of a certain box.
 * This box must be aligned to the block width and height of the
 * underlying format. */
static inline size_t
etna_compute_offset(enum pipe_format format, const struct pipe_box *box,
                    size_t stride, size_t layer_stride)
{
   return box->z * layer_stride +
          box->y / util_format_get_blockheight(format) * stride +
          box->x / util_format_get_blockwidth(format) *
             util_format_get_blocksize(format);
}

static void etna_patch_data(void *buffer, const struct pipe_transfer *ptrans)
{
   struct pipe_resource *prsc = ptrans->resource;
   struct etna_resource *rsc = etna_resource(prsc);
   struct etna_resource_level *level = &rsc->levels[ptrans->level];

   if (likely(!etna_etc2_needs_patching(prsc)))
      return;

   if (level->patched)
      return;

   /* do have the offsets of blocks to patch? */
   if (!level->patch_offsets) {
      level->patch_offsets = CALLOC_STRUCT(util_dynarray);

      etna_etc2_calculate_blocks(buffer, ptrans->stride,
                                         ptrans->box.width, ptrans->box.height,
                                         prsc->format, level->patch_offsets);
   }

   etna_etc2_patch(buffer, level->patch_offsets);

   level->patched = true;
}

static void etna_unpatch_data(void *buffer, const struct pipe_transfer *ptrans)
{
   struct pipe_resource *prsc = ptrans->resource;
   struct etna_resource *rsc = etna_resource(prsc);
   struct etna_resource_level *level = &rsc->levels[ptrans->level];

   if (!level->patched)
      return;

   etna_etc2_patch(buffer, level->patch_offsets);

   level->patched = false;
}

static void
etna_transfer_unmap(struct pipe_context *pctx, struct pipe_transfer *ptrans)
{
   struct etna_context *ctx = etna_context(pctx);
   struct etna_transfer *trans = etna_transfer(ptrans);
   struct etna_resource *rsc = etna_resource(ptrans->resource);
   struct etna_resource_level *res_level = &rsc->levels[ptrans->level];

   if (rsc->texture && !etna_resource_newer(rsc, etna_resource(rsc->texture)))
      rsc = etna_resource(rsc->texture); /* switch to using the texture resource */

   /*
    * Temporary resources are always pulled into the CPU domain, must push them
    * back into GPU domain before the RS execs the blit to the base resource.
    */
   if (trans->rsc)
      etna_bo_cpu_fini(etna_resource(trans->rsc)->bo);

   if (ptrans->usage & PIPE_MAP_WRITE) {
      if (etna_resource_level_needs_flush(res_level)) {
         if (ptrans->usage & ETNA_PIPE_MAP_DISCARD_LEVEL)
            etna_resource_level_mark_flushed(res_level);
         else
            etna_copy_resource(pctx, &rsc->base, &rsc->base, ptrans->level, ptrans->level);
      }

      if (trans->rsc) {
         /* We have a temporary resource due to either tile status or
          * tiling format. Write back the updated buffer contents.
          */
         etna_copy_resource_box(pctx, ptrans->resource, trans->rsc,
                                ptrans->level, 0, &ptrans->box);
      } else if (trans->staging) {
         /* map buffer object */
         if (rsc->layout == ETNA_LAYOUT_TILED) {
            for (unsigned z = 0; z < ptrans->box.depth; z++) {
               etna_texture_tile(
                  trans->mapped + (ptrans->box.z + z) * res_level->layer_stride,
                  trans->staging + z * ptrans->layer_stride,
                  ptrans->box.x, ptrans->box.y,
                  res_level->stride, ptrans->box.width, ptrans->box.height,
                  ptrans->stride, util_format_get_blocksize(rsc->base.format));
            }
         } else if (rsc->layout == ETNA_LAYOUT_LINEAR) {
            util_copy_box(trans->mapped, rsc->base.format, res_level->stride,
                          res_level->layer_stride, ptrans->box.x,
                          ptrans->box.y, ptrans->box.z, ptrans->box.width,
                          ptrans->box.height, ptrans->box.depth,
                          trans->staging, ptrans->stride,
                          ptrans->layer_stride, 0, 0, 0 /* src x,y,z */);
         } else {
            BUG("unsupported tiling %i", rsc->layout);
         }
      }

      if (ptrans->resource->target == PIPE_BUFFER)
         util_range_add(&rsc->base, &rsc->valid_buffer_range,
                        ptrans->box.x, ptrans->box.x + ptrans->box.width);

      etna_resource_level_ts_mark_invalid(res_level);
      etna_resource_level_mark_changed(res_level);

      if (rsc->base.bind & PIPE_BIND_SAMPLER_VIEW)
         ctx->dirty |= ETNA_DIRTY_TEXTURE_CACHES;
      if (rsc->base.bind & PIPE_BIND_CONSTANT_BUFFER)
         ctx->dirty |= ETNA_DIRTY_SHADER_CACHES;
   }

   /* We need to have the patched data ready for the GPU. */
   etna_patch_data(trans->mapped, ptrans);

   /*
    * Transfers without a temporary are only pulled into the CPU domain if they
    * are not mapped unsynchronized. If they are, must push them back into GPU
    * domain after CPU access is finished.
    */
   if (!trans->rsc && !(ptrans->usage & PIPE_MAP_UNSYNCHRONIZED))
      etna_bo_cpu_fini(rsc->bo);

   FREE(trans->staging);
   pipe_resource_reference(&trans->rsc, NULL);
   pipe_resource_reference(&ptrans->resource, NULL);
   slab_free(&ctx->transfer_pool, trans);
}

static void *
etna_transfer_map(struct pipe_context *pctx, struct pipe_resource *prsc,
                  unsigned level,
                  unsigned usage,
                  const struct pipe_box *box,
                  struct pipe_transfer **out_transfer)
{
   struct etna_context *ctx = etna_context(pctx);
   struct etna_screen *screen = ctx->screen;
   struct etna_resource *rsc = etna_resource(prsc);
   struct etna_resource_level *res_level = &rsc->levels[level];
   struct etna_transfer *trans;
   struct pipe_transfer *ptrans;
   enum pipe_format format = prsc->format;

   trans = slab_zalloc(&ctx->transfer_pool);
   if (!trans)
      return NULL;

   assert(level <= prsc->last_level);

   /*
    * Upgrade to UNSYNCHRONIZED if target is PIPE_BUFFER and range is uninitialized.
    */
   if ((usage & PIPE_MAP_WRITE) &&
       (prsc->target == PIPE_BUFFER) &&
       !util_ranges_intersect(&rsc->valid_buffer_range,
                              box->x,
                              box->x + box->width)) {
      usage |= PIPE_MAP_UNSYNCHRONIZED;
   }

   /* Upgrade DISCARD_RANGE to WHOLE_RESOURCE if the whole resource is
    * being mapped. If we add buffer reallocation to avoid CPU/GPU sync this
    * check needs to be extended to coherent mappings and shared resources.
    */
   if ((usage & PIPE_MAP_DISCARD_RANGE) &&
       !(usage & PIPE_MAP_UNSYNCHRONIZED) &&
       !(prsc->flags & PIPE_RESOURCE_FLAG_MAP_PERSISTENT) &&
       prsc->last_level == 0 &&
       prsc->width0 == box->width &&
       prsc->height0 == box->height &&
       prsc->depth0 == box->depth &&
       prsc->array_size == 1) {
      usage |= PIPE_MAP_DISCARD_WHOLE_RESOURCE;
   }

   if ((usage & PIPE_MAP_DISCARD_WHOLE_RESOURCE) ||
       ((usage & PIPE_MAP_DISCARD_RANGE) &&
        util_texrange_covers_whole_level(prsc, level, box->x, box->y, box->z,
                                         box->width, box->height, box->depth)))
      usage |= ETNA_PIPE_MAP_DISCARD_LEVEL;


   ptrans = &trans->base;
   pipe_resource_reference(&ptrans->resource, prsc);
   ptrans->level = level;
   ptrans->usage = usage;
   ptrans->box = *box;

   /* This one is a little tricky: if we have a separate render resource, which
    * is newer than the base resource we want the transfer to target this one,
    * to get the most up-to-date content, but only if we don't have a texture
    * target of the same age, as transfering in/out of the texture target is
    * generally preferred for the reasons listed below */
   if (rsc->render && etna_resource_newer(etna_resource(rsc->render), rsc) &&
       (!rsc->texture || etna_resource_newer(etna_resource(rsc->render),
                                             etna_resource(rsc->texture)))) {
      rsc = etna_resource(rsc->render);
   }

   if (rsc->texture && !etna_resource_newer(rsc, etna_resource(rsc->texture))) {
      /* We have a texture resource which is the same age or newer than the
       * render resource. Use the texture resource, which avoids bouncing
       * pixels between the two resources, and we can de-tile it in s/w. */
      rsc = etna_resource(rsc->texture);
   } else if (etna_resource_level_ts_valid(res_level) ||
              (rsc->layout != ETNA_LAYOUT_LINEAR &&
               etna_resource_hw_tileable(screen->specs.use_blt, prsc) &&
               /* HALIGN 4 resources are incompatible with the resolve engine,
                * so fall back to using software to detile this resource. */
               rsc->halign != TEXTURE_HALIGN_FOUR)) {

      /* If the resource level has valid tile status, we copy the transfer
       * region to a staging resource using the BLT or RS engine, which will
       * resolve the TS information. We also do this for tiled resources without
       * TS, but only if the dedicated blit engines can handle them. */

      if (usage & PIPE_MAP_DIRECTLY) {
         slab_free(&ctx->transfer_pool, trans);
         BUG("unsupported map flags %#x with tile status/tiled layout", usage);
         return NULL;
      }

      struct pipe_resource templ = *prsc;
      templ.last_level = 0;
      templ.width0 = res_level->width;
      templ.height0 = res_level->height;
      templ.nr_samples = 0;
      templ.bind = PIPE_BIND_RENDER_TARGET;

      trans->rsc = etna_resource_alloc(pctx->screen, ETNA_LAYOUT_LINEAR,
                                       DRM_FORMAT_MOD_LINEAR, &templ);
      if (!trans->rsc) {
         slab_free(&ctx->transfer_pool, trans);
         return NULL;
      }

      if (!screen->specs.use_blt) {
         /* Need to align the transfer region to satisfy RS restrictions, as we
          * really want to hit the RS blit path here.
          */
         unsigned w_align, h_align;

         if (rsc->layout & ETNA_LAYOUT_BIT_SUPER) {
            w_align = 64;
            h_align = 64 * ctx->screen->specs.pixel_pipes;
         } else {
            w_align = ETNA_RS_WIDTH_MASK + 1;
            h_align = ETNA_RS_HEIGHT_MASK + 1;
         }

         ptrans->box.width += ptrans->box.x & (w_align - 1);
         ptrans->box.x = ptrans->box.x & ~(w_align - 1);
         ptrans->box.width = align(ptrans->box.width, (ETNA_RS_WIDTH_MASK + 1));
         ptrans->box.height += ptrans->box.y & (h_align - 1);
         ptrans->box.y = ptrans->box.y & ~(h_align - 1);
         ptrans->box.height = align(ptrans->box.height, ETNA_RS_HEIGHT_MASK + 1);
      }

      if ((usage & PIPE_MAP_READ) || !(usage & ETNA_PIPE_MAP_DISCARD_LEVEL))
         etna_copy_resource_box(pctx, trans->rsc, &rsc->base, 0, level, &ptrans->box);

      /* Switch to using the temporary resource instead */
      rsc = etna_resource(trans->rsc);
      res_level = &rsc->levels[0];
   }

   /* XXX we don't handle PIPE_MAP_FLUSH_EXPLICIT; this flag can be ignored
    * when mapping in-place,
    * but when not in place we need to fire off the copy operation in
    * transfer_flush_region (currently
    * a no-op) instead of unmap. Need to handle this to support
    * ARB_map_buffer_range extension at least.
    */

   /*
    * Pull resources into the CPU domain. Only skipped for unsynchronized
    * transfers without a temporary resource.
    */
   if (trans->rsc || !(usage & PIPE_MAP_UNSYNCHRONIZED)) {
      enum etna_resource_status status = etna_resource_status(ctx, rsc);
      uint32_t prep_flags = 0;

      /*
       * Always flush if we have the temporary resource and have a copy to this
       * outstanding. Otherwise infer flush requirement from resource access and
       * current GPU usage (reads must wait for GPU writes, writes must have
       * exclusive access to the buffer).
       */
      if ((trans->rsc && (status & ETNA_PENDING_WRITE)) ||
          (!trans->rsc &&
           (((usage & PIPE_MAP_READ) && (status & ETNA_PENDING_WRITE)) ||
           ((usage & PIPE_MAP_WRITE) && status)))) {
         etna_flush(pctx, NULL, 0, true);
      }

      if (usage & PIPE_MAP_READ)
         prep_flags |= DRM_ETNA_PREP_READ;
      if (usage & PIPE_MAP_WRITE)
         prep_flags |= DRM_ETNA_PREP_WRITE;

      /*
       * The ETC2 patching operates in-place on the resource, so the resource will
       * get written even on read-only transfers. This blocks the GPU to sample
       * from this resource.
       */
      if ((usage & PIPE_MAP_READ) && etna_etc2_needs_patching(prsc))
         prep_flags |= DRM_ETNA_PREP_WRITE;

      if (etna_bo_cpu_prep(rsc->bo, prep_flags))
         goto fail_prep;
   }

   /* map buffer object */
   trans->mapped = etna_bo_map(rsc->bo);
   if (!trans->mapped)
      goto fail;

   *out_transfer = ptrans;

   if (rsc->layout == ETNA_LAYOUT_LINEAR) {
      ptrans->stride = res_level->stride;
      ptrans->layer_stride = res_level->layer_stride;

      trans->mapped += res_level->offset +
             etna_compute_offset(prsc->format, box, res_level->stride,
                                 res_level->layer_stride);

      /* We need to have the unpatched data ready for the gfx stack. */
      if (usage & PIPE_MAP_READ)
         etna_unpatch_data(trans->mapped, ptrans);

      return trans->mapped;
   } else {
      unsigned divSizeX = util_format_get_blockwidth(format);
      unsigned divSizeY = util_format_get_blockheight(format);

      /* No direct mappings of tiled, since we need to manually
       * tile/untile.
       */
      if (usage & PIPE_MAP_DIRECTLY)
         goto fail;

      trans->mapped += res_level->offset;
      ptrans->stride = align(box->width, divSizeX) * util_format_get_blocksize(format); /* row stride in bytes */
      ptrans->layer_stride = align(box->height, divSizeY) * ptrans->stride;
      size_t size = ptrans->layer_stride * box->depth;

      trans->staging = MALLOC(size);
      if (!trans->staging)
         goto fail;

      if (usage & PIPE_MAP_READ) {
         if (rsc->layout == ETNA_LAYOUT_TILED) {
            for (unsigned z = 0; z < ptrans->box.depth; z++) {
               etna_texture_untile(trans->staging + z * ptrans->layer_stride,
                                   trans->mapped + (ptrans->box.z + z) * res_level->layer_stride,
                                   ptrans->box.x, ptrans->box.y, res_level->stride,
                                   ptrans->box.width, ptrans->box.height, ptrans->stride,
                                   util_format_get_blocksize(rsc->base.format));
            }
         } else if (rsc->layout == ETNA_LAYOUT_LINEAR) {
            util_copy_box(trans->staging, rsc->base.format, ptrans->stride,
                          ptrans->layer_stride, 0, 0, 0, /* dst x,y,z */
                          ptrans->box.width, ptrans->box.height,
                          ptrans->box.depth, trans->mapped, res_level->stride,
                          res_level->layer_stride, ptrans->box.x,
                          ptrans->box.y, ptrans->box.z);
         } else {
            /* TODO supertiling */
            BUG("unsupported tiling %i for reading", rsc->layout);
         }
      }

      return trans->staging;
   }

fail:
   etna_bo_cpu_fini(rsc->bo);
fail_prep:
   etna_transfer_unmap(pctx, ptrans);
   return NULL;
}

static void
etna_transfer_flush_region(struct pipe_context *pctx,
                           struct pipe_transfer *ptrans,
                           const struct pipe_box *box)
{
   struct etna_resource *rsc = etna_resource(ptrans->resource);

   if (ptrans->resource->target == PIPE_BUFFER)
      util_range_add(&rsc->base,
                     &rsc->valid_buffer_range,
                     ptrans->box.x + box->x,
                     ptrans->box.x + box->x + box->width);
}

void
etna_transfer_init(struct pipe_context *pctx)
{
   pctx->buffer_map = etna_transfer_map;
   pctx->texture_map = etna_transfer_map;
   pctx->transfer_flush_region = etna_transfer_flush_region;
   pctx->buffer_unmap = etna_transfer_unmap;
   pctx->texture_unmap = etna_transfer_unmap;
   pctx->buffer_subdata = u_default_buffer_subdata;
   pctx->texture_subdata = u_default_texture_subdata;
}