xref: /third_party/mesa3d/src/mesa/drivers/dri/i965/brw_pipe_control.c

/*
 * Copyright © 2010 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

#include "brw_context.h"
#include "brw_defines.h"
#include "brw_state.h"
#include "brw_batch.h"
#include "brw_fbo.h"

/**
 * Emit a PIPE_CONTROL with various flushing flags.
 *
 * The caller is responsible for deciding what flags are appropriate for the
 * given generation.
 */
void
brw_emit_pipe_control_flush(struct brw_context *brw, uint32_t flags)
{
   const struct intel_device_info *devinfo = &brw->screen->devinfo;

   if (devinfo->ver >= 6 &&
       (flags & PIPE_CONTROL_CACHE_FLUSH_BITS) &&
       (flags & PIPE_CONTROL_CACHE_INVALIDATE_BITS)) {
      /* A pipe control command with flush and invalidate bits set
       * simultaneously is an inherently racy operation on Gfx6+ if the
       * contents of the flushed caches were intended to become visible from
       * any of the invalidated caches.  Split it in two PIPE_CONTROLs, the
       * first one should stall the pipeline to make sure that the flushed R/W
       * caches are coherent with memory once the specified R/O caches are
       * invalidated.  On pre-Gfx6 hardware the (implicit) R/O cache
       * invalidation seems to happen at the bottom of the pipeline together
       * with any write cache flush, so this shouldn't be a concern.  In order
       * to ensure a full stall, we do an end-of-pipe sync.
       */
      brw_emit_end_of_pipe_sync(brw, (flags & PIPE_CONTROL_CACHE_FLUSH_BITS));
      flags &= ~(PIPE_CONTROL_CACHE_FLUSH_BITS | PIPE_CONTROL_CS_STALL);
   }

   brw->vtbl.emit_raw_pipe_control(brw, flags, NULL, 0, 0);
}

/**
 * Emit a PIPE_CONTROL that writes to a buffer object.
 *
 * \p flags should contain one of the following items:
 *  - PIPE_CONTROL_WRITE_IMMEDIATE
 *  - PIPE_CONTROL_WRITE_TIMESTAMP
 *  - PIPE_CONTROL_WRITE_DEPTH_COUNT
 */
void
brw_emit_pipe_control_write(struct brw_context *brw, uint32_t flags,
                            struct brw_bo *bo, uint32_t offset,
                            uint64_t imm)
{
   brw->vtbl.emit_raw_pipe_control(brw, flags, bo, offset, imm);
}

/**
 * Restriction [DevSNB, DevIVB]:
 *
 * Prior to changing Depth/Stencil Buffer state (i.e. any combination of
 * 3DSTATE_DEPTH_BUFFER, 3DSTATE_CLEAR_PARAMS, 3DSTATE_STENCIL_BUFFER,
 * 3DSTATE_HIER_DEPTH_BUFFER) SW must first issue a pipelined depth stall
 * (PIPE_CONTROL with Depth Stall bit set), followed by a pipelined depth
 * cache flush (PIPE_CONTROL with Depth Flush Bit set), followed by
 * another pipelined depth stall (PIPE_CONTROL with Depth Stall bit set),
 * unless SW can otherwise guarantee that the pipeline from WM onwards is
 * already flushed (e.g., via a preceding MI_FLUSH).
 */
void
brw_emit_depth_stall_flushes(struct brw_context *brw)
{
   const struct intel_device_info *devinfo = &brw->screen->devinfo;

   assert(devinfo->ver >= 6);

   /* Starting on BDW, these pipe controls are unnecessary.
    *
    *   WM HW will internally manage the draining pipe and flushing of the caches
    *   when this command is issued. The PIPE_CONTROL restrictions are removed.
    */
   if (devinfo->ver >= 8)
      return;

   brw_emit_pipe_control_flush(brw, PIPE_CONTROL_DEPTH_STALL);
   brw_emit_pipe_control_flush(brw, PIPE_CONTROL_DEPTH_CACHE_FLUSH);
   brw_emit_pipe_control_flush(brw, PIPE_CONTROL_DEPTH_STALL);
}

/**
 * From the Ivybridge PRM, Volume 2 Part 1, Section 3.2 (VS Stage Input):
 * "A PIPE_CONTROL with Post-Sync Operation set to 1h and a depth
 *  stall needs to be sent just prior to any 3DSTATE_VS, 3DSTATE_URB_VS,
 *  3DSTATE_CONSTANT_VS, 3DSTATE_BINDING_TABLE_POINTER_VS,
 *  3DSTATE_SAMPLER_STATE_POINTER_VS command.  Only one PIPE_CONTROL needs
 *  to be sent before any combination of VS associated 3DSTATE."
 */
void
gfx7_emit_vs_workaround_flush(struct brw_context *brw)
{
   ASSERTED const struct intel_device_info *devinfo = &brw->screen->devinfo;

   assert(devinfo->ver == 7);
   brw_emit_pipe_control_write(brw,
                               PIPE_CONTROL_WRITE_IMMEDIATE
                               | PIPE_CONTROL_DEPTH_STALL,
                               brw->workaround_bo,
                               brw->workaround_bo_offset, 0);
}

/**
 * From the PRM, Volume 2a:
 *
 *    "Indirect State Pointers Disable
 *
 *    At the completion of the post-sync operation associated with this pipe
 *    control packet, the indirect state pointers in the hardware are
 *    considered invalid; the indirect pointers are not saved in the context.
 *    If any new indirect state commands are executed in the command stream
 *    while the pipe control is pending, the new indirect state commands are
 *    preserved.
 *
 *    [DevIVB+]: Using Invalidate State Pointer (ISP) only inhibits context
 *    restoring of Push Constant (3DSTATE_CONSTANT_*) commands. Push Constant
 *    commands are only considered as Indirect State Pointers. Once ISP is
 *    issued in a context, SW must initialize by programming push constant
 *    commands for all the shaders (at least to zero length) before attempting
 *    any rendering operation for the same context."
 *
 * 3DSTATE_CONSTANT_* packets are restored during a context restore,
 * even though they point to a BO that has been already unreferenced at
 * the end of the previous batch buffer. This has been fine so far since
 * we are protected by these scratch page (every address not covered by
 * a BO should be pointing to the scratch page). But on CNL, it is
 * causing a GPU hang during context restore at the 3DSTATE_CONSTANT_*
 * instruction.
 *
 * The flag "Indirect State Pointers Disable" in PIPE_CONTROL tells the
 * hardware to ignore previous 3DSTATE_CONSTANT_* packets during a
 * context restore, so the mentioned hang doesn't happen. However,
 * software must program push constant commands for all stages prior to
 * rendering anything, so we flag them as dirty.
 *
 * Finally, we also make sure to stall at pixel scoreboard to make sure the
 * constants have been loaded into the EUs prior to disable the push constants
 * so that it doesn't hang a previous 3DPRIMITIVE.
 */
void
gfx7_emit_isp_disable(struct brw_context *brw)
{
   brw->vtbl.emit_raw_pipe_control(brw,
                                   PIPE_CONTROL_STALL_AT_SCOREBOARD |
                                   PIPE_CONTROL_CS_STALL,
                                   NULL, 0, 0);
   brw->vtbl.emit_raw_pipe_control(brw,
                                   PIPE_CONTROL_INDIRECT_STATE_POINTERS_DISABLE |
                                   PIPE_CONTROL_CS_STALL,
                                   NULL, 0, 0);

   brw->vs.base.push_constants_dirty = true;
   brw->tcs.base.push_constants_dirty = true;
   brw->tes.base.push_constants_dirty = true;
   brw->gs.base.push_constants_dirty = true;
   brw->wm.base.push_constants_dirty = true;
}

/**
 * Emit a PIPE_CONTROL command for gfx7 with the CS Stall bit set.
 */
void
gfx7_emit_cs_stall_flush(struct brw_context *brw)
{
   brw_emit_pipe_control_write(brw,
                               PIPE_CONTROL_CS_STALL
                               | PIPE_CONTROL_WRITE_IMMEDIATE,
                               brw->workaround_bo,
                               brw->workaround_bo_offset, 0);
}

/**
 * Emits a PIPE_CONTROL with a non-zero post-sync operation, for
 * implementing two workarounds on gfx6.  From section 1.4.7.1
 * "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1:
 *
 * [DevSNB-C+{W/A}] Before any depth stall flush (including those
 * produced by non-pipelined state commands), software needs to first
 * send a PIPE_CONTROL with no bits set except Post-Sync Operation !=
 * 0.
 *
 * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable
 * =1, a PIPE_CONTROL with any non-zero post-sync-op is required.
 *
 * And the workaround for these two requires this workaround first:
 *
 * [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent
 * BEFORE the pipe-control with a post-sync op and no write-cache
 * flushes.
 *
 * And this last workaround is tricky because of the requirements on
 * that bit.  From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM
 * volume 2 part 1:
 *
 *     "1 of the following must also be set:
 *      - Render Target Cache Flush Enable ([12] of DW1)
 *      - Depth Cache Flush Enable ([0] of DW1)
 *      - Stall at Pixel Scoreboard ([1] of DW1)
 *      - Depth Stall ([13] of DW1)
 *      - Post-Sync Operation ([13] of DW1)
 *      - Notify Enable ([8] of DW1)"
 *
 * The cache flushes require the workaround flush that triggered this
 * one, so we can't use it.  Depth stall would trigger the same.
 * Post-sync nonzero is what triggered this second workaround, so we
 * can't use that one either.  Notify enable is IRQs, which aren't
 * really our business.  That leaves only stall at scoreboard.
 */
void
brw_emit_post_sync_nonzero_flush(struct brw_context *brw)
{
   brw_emit_pipe_control_flush(brw,
                               PIPE_CONTROL_CS_STALL |
                               PIPE_CONTROL_STALL_AT_SCOREBOARD);

   brw_emit_pipe_control_write(brw, PIPE_CONTROL_WRITE_IMMEDIATE,
                               brw->workaround_bo,
                               brw->workaround_bo_offset, 0);
}

/*
 * From Sandybridge PRM, volume 2, "1.7.2 End-of-Pipe Synchronization":
 *
 *  Write synchronization is a special case of end-of-pipe
 *  synchronization that requires that the render cache and/or depth
 *  related caches are flushed to memory, where the data will become
 *  globally visible. This type of synchronization is required prior to
 *  SW (CPU) actually reading the result data from memory, or initiating
 *  an operation that will use as a read surface (such as a texture
 *  surface) a previous render target and/or depth/stencil buffer
 *
 *
 * From Haswell PRM, volume 2, part 1, "End-of-Pipe Synchronization":
 *
 *  Exercising the write cache flush bits (Render Target Cache Flush
 *  Enable, Depth Cache Flush Enable, DC Flush) in PIPE_CONTROL only
 *  ensures the write caches are flushed and doesn't guarantee the data
 *  is globally visible.
 *
 *  SW can track the completion of the end-of-pipe-synchronization by
 *  using "Notify Enable" and "PostSync Operation - Write Immediate
 *  Data" in the PIPE_CONTROL command.
 */
void
brw_emit_end_of_pipe_sync(struct brw_context *brw, uint32_t flags)
{
   const struct intel_device_info *devinfo = &brw->screen->devinfo;

   if (devinfo->ver >= 6) {
      /* From Sandybridge PRM, volume 2, "1.7.3.1 Writing a Value to Memory":
       *
       *    "The most common action to perform upon reaching a synchronization
       *    point is to write a value out to memory. An immediate value
       *    (included with the synchronization command) may be written."
       *
       *
       * From Broadwell PRM, volume 7, "End-of-Pipe Synchronization":
       *
       *    "In case the data flushed out by the render engine is to be read
       *    back in to the render engine in coherent manner, then the render
       *    engine has to wait for the fence completion before accessing the
       *    flushed data. This can be achieved by following means on various
       *    products: PIPE_CONTROL command with CS Stall and the required
       *    write caches flushed with Post-Sync-Operation as Write Immediate
       *    Data.
       *
       *    Example:
       *       - Workload-1 (3D/GPGPU/MEDIA)
       *       - PIPE_CONTROL (CS Stall, Post-Sync-Operation Write Immediate
       *         Data, Required Write Cache Flush bits set)
       *       - Workload-2 (Can use the data produce or output by Workload-1)
       */
      brw_emit_pipe_control_write(brw,
                                  flags | PIPE_CONTROL_CS_STALL |
                                  PIPE_CONTROL_WRITE_IMMEDIATE,
                                  brw->workaround_bo,
                                  brw->workaround_bo_offset, 0);

      if (devinfo->is_haswell) {
         /* Haswell needs addition work-arounds:
          *
          * From Haswell PRM, volume 2, part 1, "End-of-Pipe Synchronization":
          *
          *    Option 1:
          *    PIPE_CONTROL command with the CS Stall and the required write
          *    caches flushed with Post-SyncOperation as Write Immediate Data
          *    followed by eight dummy MI_STORE_DATA_IMM (write to scratch
          *    spce) commands.
          *
          *    Example:
          *       - Workload-1
          *       - PIPE_CONTROL (CS Stall, Post-Sync-Operation Write
          *         Immediate Data, Required Write Cache Flush bits set)
          *       - MI_STORE_DATA_IMM (8 times) (Dummy data, Scratch Address)
          *       - Workload-2 (Can use the data produce or output by
          *         Workload-1)
          *
          * Unfortunately, both the PRMs and the internal docs are a bit
          * out-of-date in this regard.  What the windows driver does (and
          * this appears to actually work) is to emit a register read from the
          * memory address written by the pipe control above.
          *
          * What register we load into doesn't matter.  We choose an indirect
          * rendering register because we know it always exists and it's one
          * of the first registers the command parser allows us to write.  If
          * you don't have command parser support in your kernel (pre-4.2),
          * this will get turned into MI_NOOP and you won't get the
          * workaround.  Unfortunately, there's just not much we can do in
          * that case.  This register is perfectly safe to write since we
          * always re-load all of the indirect draw registers right before
          * 3DPRIMITIVE when needed anyway.
          */
         brw_load_register_mem(brw, GFX7_3DPRIM_START_INSTANCE,
                               brw->workaround_bo, brw->workaround_bo_offset);
      }
   } else {
      /* On gfx4-5, a regular pipe control seems to suffice. */
      brw_emit_pipe_control_flush(brw, flags);
   }
}

/* Emit a pipelined flush to either flush render and texture cache for
 * reading from a FBO-drawn texture, or flush so that frontbuffer
 * render appears on the screen in DRI1.
 *
 * This is also used for the always_flush_cache driconf debug option.
 */
void
brw_emit_mi_flush(struct brw_context *brw)
{
   const struct intel_device_info *devinfo = &brw->screen->devinfo;

   int flags = PIPE_CONTROL_RENDER_TARGET_FLUSH;
   if (devinfo->ver >= 6) {
      flags |= PIPE_CONTROL_INSTRUCTION_INVALIDATE |
               PIPE_CONTROL_CONST_CACHE_INVALIDATE |
               PIPE_CONTROL_DATA_CACHE_FLUSH |
               PIPE_CONTROL_DEPTH_CACHE_FLUSH |
               PIPE_CONTROL_VF_CACHE_INVALIDATE |
               PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
               PIPE_CONTROL_CS_STALL;
   }
   brw_emit_pipe_control_flush(brw, flags);
}

static bool
init_identifier_bo(struct brw_context *brw)
{
   void *bo_map;

   if (!can_do_exec_capture(brw->screen))
      return true;

   bo_map = brw_bo_map(NULL, brw->workaround_bo, MAP_READ | MAP_WRITE);
   if (!bo_map)
      return false;

   brw->workaround_bo->kflags |= EXEC_OBJECT_CAPTURE;
   brw->workaround_bo_offset =
      ALIGN(intel_debug_write_identifiers(bo_map, 4096, "i965") + 8, 8);

   brw_bo_unmap(brw->workaround_bo);

   return true;
}

int
brw_init_pipe_control(struct brw_context *brw,
                      const struct intel_device_info *devinfo)
{
   switch (devinfo->ver) {
   case 11:
      brw->vtbl.emit_raw_pipe_control = gfx11_emit_raw_pipe_control;
      break;
   case 9:
      brw->vtbl.emit_raw_pipe_control = gfx9_emit_raw_pipe_control;
      break;
   case 8:
      brw->vtbl.emit_raw_pipe_control = gfx8_emit_raw_pipe_control;
      break;
   case 7:
      brw->vtbl.emit_raw_pipe_control =
         devinfo->is_haswell ? gfx75_emit_raw_pipe_control
                             : gfx7_emit_raw_pipe_control;
      break;
   case 6:
      brw->vtbl.emit_raw_pipe_control = gfx6_emit_raw_pipe_control;
      break;
   case 5:
      brw->vtbl.emit_raw_pipe_control = gfx5_emit_raw_pipe_control;
      break;
   case 4:
      brw->vtbl.emit_raw_pipe_control =
         devinfo->is_g4x ? gfx45_emit_raw_pipe_control
                         : gfx4_emit_raw_pipe_control;
      break;
   default:
      unreachable("Unhandled Gen.");
   }

   if (devinfo->ver < 6)
      return 0;

   /* We can't just use brw_state_batch to get a chunk of space for
    * the gfx6 workaround because it involves actually writing to
    * the buffer, and the kernel doesn't let us write to the batch.
    */
   brw->workaround_bo = brw_bo_alloc(brw->bufmgr, "workaround", 4096,
                                     BRW_MEMZONE_OTHER);
   if (brw->workaround_bo == NULL)
      return -ENOMEM;

   if (!init_identifier_bo(brw))
      return -ENOMEM; /* Couldn't map workaround_bo?? */

   brw->workaround_bo_offset = 0;
   brw->pipe_controls_since_last_cs_stall = 0;

   return 0;
}

void
brw_fini_pipe_control(struct brw_context *brw)
{
   brw_bo_unreference(brw->workaround_bo);
}