panfrost/compiler/bi_helper_invocations.c

/*
 * Copyright (C) 2019-2021 Collabora, Ltd.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 * Authors (Collabora):
 *    Alyssa Rosenzweig <alyssa.rosenzweig@collabora.com>
 */

#include "compiler.h"

/* Bifrost texture operations have a `skip` bit, instructing helper invocations
 * to skip execution. Each clause has a `terminate_discarded_threads` bit,
 * which will terminate helper invocations.
 *
 * The terminate bit should be set on the last clause requiring helper
 * invocations. Without control flow, that's the last source-order instruction;
 * with control flow, there may be multiple such instructions (with ifs) or no
 * such instruction (with loops).
 *
 * The skip bit should be set unless the value of this instruction is required
 * by a future instruction requiring helper invocations. Consider:
 *
 *      0 = texture ...
 *      1 = fmul 0, #10
 *      2 = dfdx 1
 *      store 2
 *
 * Since the derivative calculation 2 requires helper invocations, the value 1
 * must be calculated by helper invocations, and since it depends on 0, 0 must
 * be calculated by helpers. Hence the texture op does NOT have the skip bit
 * set, and the clause containing the derivative has the terminate bit set.
 *
 * Calculating the terminate bit occurs by forward dataflow analysis to
 * determine which blocks require helper invocations. A block requires
 * invocations in if any of its instructions use helper invocations, or if it
 * depends on a block that requires invocation. With that analysis, the
 * terminate bit is set on the last instruction using invocations within any
 * block that does *not* require invocations out.
 *
 * Likewise, calculating the execute bit requires backward dataflow analysis
 * with union as the join operation and the generating set being the union of
 * sources of instructions writing executed values. The skip bit is the inverse
 * of the execute bit.
 */

static bool
bi_has_skip_bit(enum bi_opcode op)
{
   switch (op) {
   case BI_OPCODE_TEX_SINGLE:
   case BI_OPCODE_TEXC:
   case BI_OPCODE_TEXC_DUAL:
   case BI_OPCODE_TEXS_2D_F16:
   case BI_OPCODE_TEXS_2D_F32:
   case BI_OPCODE_TEXS_CUBE_F16:
   case BI_OPCODE_TEXS_CUBE_F32:
   case BI_OPCODE_VAR_TEX_F16:
   case BI_OPCODE_VAR_TEX_F32:
      return true;
   default:
      return false;
   }
}

/* Does a given instruction require helper threads to be active (because it
 * reads from other subgroup lanes)? This only applies to fragment shaders.
 * Other shader stages do not have a notion of helper threads. */

bool
bi_instr_uses_helpers(bi_instr *I)
{
   switch (I->op) {
   case BI_OPCODE_TEXC:
   case BI_OPCODE_TEXC_DUAL:
   case BI_OPCODE_TEXS_2D_F16:
   case BI_OPCODE_TEXS_2D_F32:
   case BI_OPCODE_TEXS_CUBE_F16:
   case BI_OPCODE_TEXS_CUBE_F32:
   case BI_OPCODE_VAR_TEX_F16:
   case BI_OPCODE_VAR_TEX_F32:
      return !I->lod_mode; /* set for zero, clear for computed */
   case BI_OPCODE_TEX_SINGLE:
      return (I->va_lod_mode == BI_VA_LOD_MODE_COMPUTED_LOD) ||
             (I->va_lod_mode == BI_VA_LOD_MODE_COMPUTED_BIAS);
   case BI_OPCODE_CLPER_I32:
   case BI_OPCODE_CLPER_OLD_I32:
      /* Fragment shaders require helpers to implement derivatives.
       * Other shader stages don't have helpers at all */
      return true;
   default:
      return false;
   }
}

/* Does a block use helpers directly */
static bool
bi_block_uses_helpers(bi_block *block)
{
   bi_foreach_instr_in_block(block, I) {
      if (bi_instr_uses_helpers(I))
         return true;
   }

   return false;
}

bool
bi_block_terminates_helpers(bi_block *block)
{
   /* Can't terminate if a successor needs helpers */
   bi_foreach_successor(block, succ) {
      if (succ->pass_flags & 1)
         return false;
   }

   /* Otherwise we terminate */
   return true;
}

/*
 * Propagate the pass flag up the control flow graph by performing depth-first
 * search on the directed control flow graph.
 */
static void
bi_propagate_pass_flag(bi_block *block)
{
   block->pass_flags = 1;

   bi_foreach_predecessor(block, pred) {
      if ((*pred)->pass_flags == 0)
         bi_propagate_pass_flag(*pred);
   }
}

void
bi_analyze_helper_terminate(bi_context *ctx)
{
   /* Other shader stages do not have a notion of helper threads, so we
    * can skip the analysis. Don't run for blend shaders, either, since
    * they run in the context of another shader that we don't see. */
   if (ctx->stage != MESA_SHADER_FRAGMENT || ctx->inputs->is_blend)
      return;

   /* Clear flags */
   bi_foreach_block(ctx, block)
      block->pass_flags = 0;

   /* For each block, check if it uses helpers and propagate that fact if
    * so. We walk in reverse order to minimize the number of blocks tested:
    * if the (unique) last block uses helpers, only that block is tested.
    */
   bi_foreach_block_rev(ctx, block) {
      if (block->pass_flags == 0 && bi_block_uses_helpers(block))
         bi_propagate_pass_flag(block);
   }
}

void
bi_mark_clauses_td(bi_context *ctx)
{
   if (ctx->stage != MESA_SHADER_FRAGMENT || ctx->inputs->is_blend)
      return;

   /* Finally, mark clauses requiring helpers */
   bi_foreach_block(ctx, block) {
      /* At the end, there are helpers iff we don't terminate */
      bool helpers = !bi_block_terminates_helpers(block);

      bi_foreach_clause_in_block_rev(block, clause) {
         bi_foreach_instr_in_clause_rev(block, clause, I) {
            helpers |= bi_instr_uses_helpers(I);
         }

         clause->td = !helpers;
      }
   }
}

static bool
bi_helper_block_update(BITSET_WORD *deps, bi_block *block)
{
   bool progress = false;

   bi_foreach_instr_in_block_rev(block, I) {
      /* If a destination is required by helper invocation... */
      bi_foreach_dest(I, d) {
         if (!BITSET_TEST(deps, I->dest[d].value))
            continue;

         /* ...so are the sources */
         bi_foreach_ssa_src(I, s) {
            progress |= !BITSET_TEST(deps, I->src[s].value);
            BITSET_SET(deps, I->src[s].value);
         }

         break;
      }
   }

   return progress;
}

void
bi_analyze_helper_requirements(bi_context *ctx)
{
   BITSET_WORD *deps = calloc(sizeof(BITSET_WORD), ctx->ssa_alloc);

   /* Initialize with the sources of instructions consuming
    * derivatives */

   bi_foreach_instr_global(ctx, I) {
      if (!bi_instr_uses_helpers(I))
         continue;

      bi_foreach_ssa_src(I, s)
         BITSET_SET(deps, I->src[s].value);
   }

   /* Propagate that up */
   u_worklist worklist;
   bi_worklist_init(ctx, &worklist);

   bi_foreach_block(ctx, block) {
      bi_worklist_push_tail(&worklist, block);
   }

   while (!u_worklist_is_empty(&worklist)) {
      bi_block *blk = bi_worklist_pop_tail(&worklist);

      if (bi_helper_block_update(deps, blk)) {
         bi_foreach_predecessor(blk, pred)
            bi_worklist_push_head(&worklist, *pred);
      }
   }

   u_worklist_fini(&worklist);

   /* Set the execute bits */

   bi_foreach_instr_global(ctx, I) {
      if (!bi_has_skip_bit(I->op))
         continue;

      bool exec = false;

      bi_foreach_dest(I, d)
         exec |= BITSET_TEST(deps, I->dest[d].value);

      I->skip = !exec;
   }

   free(deps);
}