freedreno/ir3/ir3_sched.c

/*
 * Copyright (C) 2014 Rob Clark <robclark@freedesktop.org>
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 * Authors:
 *    Rob Clark <robclark@freedesktop.org>
 */

#include "util/dag.h"
#include "util/u_math.h"

#include "ir3.h"
#include "ir3_compiler.h"

#ifdef DEBUG
#define SCHED_DEBUG (ir3_shader_debug & IR3_DBG_SCHEDMSGS)
#else
#define SCHED_DEBUG 0
#endif
#define d(fmt, ...)                                                            \
   do {                                                                        \
      if (SCHED_DEBUG) {                                                       \
         mesa_logi("SCHED: " fmt, ##__VA_ARGS__);                              \
      }                                                                        \
   } while (0)

#define di(instr, fmt, ...)                                                    \
   do {                                                                        \
      if (SCHED_DEBUG) {                                                       \
         struct log_stream *stream = mesa_log_streami();                       \
         mesa_log_stream_printf(stream, "SCHED: " fmt ": ", ##__VA_ARGS__);    \
         ir3_print_instr_stream(stream, instr);                                \
         mesa_log_stream_destroy(stream);                                      \
      }                                                                        \
   } while (0)

/*
 * Instruction Scheduling:
 *
 * A block-level pre-RA scheduler, which works by creating a DAG of
 * instruction dependencies, and heuristically picking a DAG head
 * (instruction with no unscheduled dependencies).
 *
 * Where possible, it tries to pick instructions that avoid nop delay
 * slots, but it will prefer to pick instructions that reduce (or do
 * not increase) the number of live values.
 *
 * If the only possible choices are instructions that increase the
 * number of live values, it will try to pick the one with the earliest
 * consumer (based on pre-sched program order).
 *
 * There are a few special cases that need to be handled, since sched
 * is currently independent of register allocation.  Usages of address
 * register (a0.x) or predicate register (p0.x) must be serialized.  Ie.
 * if you have two pairs of instructions that write the same special
 * register and then read it, then those pairs cannot be interleaved.
 * To solve this, when we are in such a scheduling "critical section",
 * and we encounter a conflicting write to a special register, we try
 * to schedule any remaining instructions that use that value first.
 *
 * TODO we can detect too-large live_values here.. would be a good place
 * to "spill" cheap things, like move from uniform/immed.  (Constructing
 * list of ssa def consumers before sched pass would make this easier.
 * Also, in general it is general it might be best not to re-use load_immed
 * across blocks.
 *
 * TODO we can use (abs)/(neg) src modifiers in a lot of cases to reduce
 * the # of immediates in play (or at least that would help with
 * dEQP-GLES31.functional.ubo.random.all_per_block_buffers.*).. probably
 * do this in a nir pass that inserts fneg/etc?  The cp pass should fold
 * these into src modifiers..
 */

struct ir3_sched_ctx {
   struct ir3_block *block; /* the current block */
   struct dag *dag;

   struct list_head unscheduled_list; /* unscheduled instructions */
   struct ir3_instruction *scheduled; /* last scheduled instr */
   struct ir3_instruction *addr0;     /* current a0.x user, if any */
   struct ir3_instruction *addr1;     /* current a1.x user, if any */
   struct ir3_instruction *pred;      /* current p0.x user, if any */

   struct ir3_instruction *split; /* most-recently-split a0/a1/p0 producer */

   int remaining_kills;
   int remaining_tex;

   bool error;

   unsigned ip;

   int sy_delay;
   int ss_delay;

   /* We order the scheduled (sy)/(ss) producers, and keep track of the
    * index of the last waited on instruction, so we can know which
    * instructions are still outstanding (and therefore would require us to
    * wait for all outstanding instructions before scheduling a use).
    */
   int sy_index, first_outstanding_sy_index;
   int ss_index, first_outstanding_ss_index;
};

struct ir3_sched_node {
   struct dag_node dag; /* must be first for util_dynarray_foreach */
   struct ir3_instruction *instr;

   unsigned delay;
   unsigned max_delay;

   unsigned sy_index;
   unsigned ss_index;

   /* For ready instructions, the earliest possible ip that it could be
    * scheduled.
    */
   unsigned earliest_ip;

   /* For instructions that are a meta:collect src, once we schedule
    * the first src of the collect, the entire vecN is live (at least
    * from the PoV of the first RA pass.. the 2nd scalar pass can fill
    * in some of the gaps, but often not all).  So we want to help out
    * RA, and realize that as soon as we schedule the first collect
    * src, there is no penalty to schedule the remainder (ie. they
    * don't make additional values live).  In fact we'd prefer to
    * schedule the rest ASAP to minimize the live range of the vecN.
    *
    * For instructions that are the src of a collect, we track the
    * corresponding collect, and mark them as partially live as soon
    * as any one of the src's is scheduled.
    */
   struct ir3_instruction *collect;
   bool partially_live;

   /* Is this instruction a direct or indirect dependency for a kill?
    * If so, we should prioritize it when possible
    */
   bool kill_path;

   /* This node represents a shader output.  A semi-common pattern in
    * shaders is something along the lines of:
    *
    *    fragcolor.w = 1.0
    *
    * Which we'd prefer to schedule as late as possible, since it
    * produces a live value that is never killed/consumed.  So detect
    * outputs up-front, and avoid scheduling them unless the reduce
    * register pressure (or at least are neutral)
    */
   bool output;
};

#define foreach_sched_node(__n, __list)                                        \
   list_for_each_entry (struct ir3_sched_node, __n, __list, dag.link)

static void sched_node_init(struct ir3_sched_ctx *ctx,
                            struct ir3_instruction *instr);
static void sched_node_add_dep(struct ir3_instruction *instr,
                               struct ir3_instruction *src, int i);

static bool
is_scheduled(struct ir3_instruction *instr)
{
   return !!(instr->flags & IR3_INSTR_MARK);
}

/* check_src_cond() passing a ir3_sched_ctx. */
static bool
sched_check_src_cond(struct ir3_instruction *instr,
                     bool (*cond)(struct ir3_instruction *,
                                  struct ir3_sched_ctx *),
                     struct ir3_sched_ctx *ctx)
{
   foreach_ssa_src (src, instr) {
      /* meta:split/collect aren't real instructions, the thing that
       * we actually care about is *their* srcs
       */
      if ((src->opc == OPC_META_SPLIT) || (src->opc == OPC_META_COLLECT)) {
         if (sched_check_src_cond(src, cond, ctx))
            return true;
      } else {
         if (cond(src, ctx))
            return true;
      }
   }

   return false;
}

/* Is this a sy producer that hasn't been waited on yet? */

static bool
is_outstanding_sy(struct ir3_instruction *instr, struct ir3_sched_ctx *ctx)
{
   if (!is_sy_producer(instr))
      return false;

   /* The sched node is only valid within the same block, we cannot
    * really say anything about src's from other blocks
    */
   if (instr->block != ctx->block)
      return true;

   struct ir3_sched_node *n = instr->data;
   return n->sy_index >= ctx->first_outstanding_sy_index;
}

static bool
is_outstanding_ss(struct ir3_instruction *instr, struct ir3_sched_ctx *ctx)
{
   if (!is_ss_producer(instr))
      return false;

   /* The sched node is only valid within the same block, we cannot
    * really say anything about src's from other blocks
    */
   if (instr->block != ctx->block)
      return true;

   struct ir3_sched_node *n = instr->data;
   return n->ss_index >= ctx->first_outstanding_ss_index;
}

static unsigned
cycle_count(struct ir3_instruction *instr)
{
   if (instr->opc == OPC_META_COLLECT) {
      /* Assume that only immed/const sources produce moves */
      unsigned n = 0;
      foreach_src (src, instr) {
         if (src->flags & (IR3_REG_IMMED | IR3_REG_CONST))
            n++;
      }
      return n;
   } else if (is_meta(instr)) {
      return 0;
   } else {
      return 1;
   }
}

static void
schedule(struct ir3_sched_ctx *ctx, struct ir3_instruction *instr)
{
   assert(ctx->block == instr->block);

   /* remove from depth list:
    */
   list_delinit(&instr->node);

   if (writes_addr0(instr)) {
      assert(ctx->addr0 == NULL);
      ctx->addr0 = instr;
   }

   if (writes_addr1(instr)) {
      assert(ctx->addr1 == NULL);
      ctx->addr1 = instr;
   }

   if (writes_pred(instr)) {
      assert(ctx->pred == NULL);
      ctx->pred = instr;
   }

   instr->flags |= IR3_INSTR_MARK;

   di(instr, "schedule");

   list_addtail(&instr->node, &instr->block->instr_list);
   ctx->scheduled = instr;

   if (is_kill_or_demote(instr)) {
      assert(ctx->remaining_kills > 0);
      ctx->remaining_kills--;
   }

   struct ir3_sched_node *n = instr->data;

   /* If this instruction is a meta:collect src, mark the remaining
    * collect srcs as partially live.
    */
   if (n->collect) {
      foreach_ssa_src (src, n->collect) {
         if (src->block != instr->block)
            continue;
         struct ir3_sched_node *sn = src->data;
         sn->partially_live = true;
      }
   }

   bool counts_for_delay = is_alu(instr) || is_flow(instr);

   /* TODO: switch to "cycles". For now try to match ir3_delay. */
   unsigned delay_cycles = counts_for_delay ? 1 + instr->repeat : 0;

   /* We insert any nop's needed to get to earliest_ip, then advance
    * delay_cycles by scheduling the instruction.
    */
   ctx->ip = MAX2(ctx->ip, n->earliest_ip) + delay_cycles;

   util_dynarray_foreach (&n->dag.edges, struct dag_edge, edge) {
      unsigned delay = (unsigned)(uintptr_t)edge->data;
      struct ir3_sched_node *child =
         container_of(edge->child, struct ir3_sched_node, dag);
      child->earliest_ip = MAX2(child->earliest_ip, ctx->ip + delay);
   }

   dag_prune_head(ctx->dag, &n->dag);

   unsigned cycles = cycle_count(instr);

   if (is_ss_producer(instr)) {
      ctx->ss_delay = soft_ss_delay(instr);
      n->ss_index = ctx->ss_index++;
   } else if (!is_meta(instr) &&
              sched_check_src_cond(instr, is_outstanding_ss, ctx)) {
      ctx->ss_delay = 0;
      ctx->first_outstanding_ss_index = ctx->ss_index;
   } else if (ctx->ss_delay > 0) {
      ctx->ss_delay -= MIN2(cycles, ctx->ss_delay);
   }

   if (is_sy_producer(instr)) {
      /* NOTE that this isn't an attempt to hide texture fetch latency,
       * but an attempt to hide the cost of switching to another warp.
       * If we can, we'd like to try to schedule another texture fetch
       * before scheduling something that would sync.
       */
      ctx->sy_delay = soft_sy_delay(instr, ctx->block->shader);
      assert(ctx->remaining_tex > 0);
      ctx->remaining_tex--;
      n->sy_index = ctx->sy_index++;
   } else if (!is_meta(instr) &&
              sched_check_src_cond(instr, is_outstanding_sy, ctx)) {
      ctx->sy_delay = 0;
      ctx->first_outstanding_sy_index = ctx->sy_index;
   } else if (ctx->sy_delay > 0) {
      ctx->sy_delay -= MIN2(cycles, ctx->sy_delay);
   }

}

struct ir3_sched_notes {
   /* there is at least one kill which could be scheduled, except
    * for unscheduled bary.f's:
    */
   bool blocked_kill;
   /* there is at least one instruction that could be scheduled,
    * except for conflicting address/predicate register usage:
    */
   bool addr0_conflict, addr1_conflict, pred_conflict;
};

static bool
should_skip(struct ir3_sched_ctx *ctx, struct ir3_instruction *instr)
{
   if (ctx->remaining_kills && (is_tex(instr) || is_mem(instr))) {
      /* avoid texture/memory access if we have unscheduled kills
       * that could make the expensive operation unnecessary.  By
       * definition, if there are remaining kills, and this instr
       * is not a dependency of a kill, there are other instructions
       * that we can choose from.
       */
      struct ir3_sched_node *n = instr->data;
      if (!n->kill_path)
         return true;
   }

   return false;
}

/* could an instruction be scheduled if specified ssa src was scheduled? */
static bool
could_sched(struct ir3_sched_ctx *ctx,
            struct ir3_instruction *instr, struct ir3_instruction *src)
{
   foreach_ssa_src (other_src, instr) {
      /* if dependency not scheduled, we aren't ready yet: */
      if ((src != other_src) && !is_scheduled(other_src)) {
         return false;
      }
   }

   /* Instructions not in the current block can never be scheduled.
    */
   if (instr->block != src->block)
      return false;

   return !should_skip(ctx, instr);
}

/* Check if instruction is ok to schedule.  Make sure it is not blocked
 * by use of addr/predicate register, etc.
 */
static bool
check_instr(struct ir3_sched_ctx *ctx, struct ir3_sched_notes *notes,
            struct ir3_instruction *instr)
{
   assert(!is_scheduled(instr));

   if (instr == ctx->split) {
      /* Don't schedule instructions created by splitting a a0.x/a1.x/p0.x
       * write until another "normal" instruction has been scheduled.
       */
      return false;
   }

   if (should_skip(ctx, instr))
       return false;

   /* For instructions that write address register we need to
    * make sure there is at least one instruction that uses the
    * addr value which is otherwise ready.
    *
    * NOTE if any instructions use pred register and have other
    * src args, we would need to do the same for writes_pred()..
    */
   if (writes_addr0(instr)) {
      struct ir3 *ir = instr->block->shader;
      bool ready = false;
      for (unsigned i = 0; (i < ir->a0_users_count) && !ready; i++) {
         struct ir3_instruction *indirect = ir->a0_users[i];
         if (!indirect)
            continue;
         if (indirect->address->def != instr->dsts[0])
            continue;
         ready = could_sched(ctx, indirect, instr);
      }

      /* nothing could be scheduled, so keep looking: */
      if (!ready)
         return false;
   }

   if (writes_addr1(instr)) {
      struct ir3 *ir = instr->block->shader;
      bool ready = false;
      for (unsigned i = 0; (i < ir->a1_users_count) && !ready; i++) {
         struct ir3_instruction *indirect = ir->a1_users[i];
         if (!indirect)
            continue;
         if (indirect->address->def != instr->dsts[0])
            continue;
         ready = could_sched(ctx, indirect, instr);
      }

      /* nothing could be scheduled, so keep looking: */
      if (!ready)
         return false;
   }

   /* if this is a write to address/predicate register, and that
    * register is currently in use, we need to defer until it is
    * free:
    */
   if (writes_addr0(instr) && ctx->addr0) {
      assert(ctx->addr0 != instr);
      notes->addr0_conflict = true;
      return false;
   }

   if (writes_addr1(instr) && ctx->addr1) {
      assert(ctx->addr1 != instr);
      notes->addr1_conflict = true;
      return false;
   }

   if (writes_pred(instr) && ctx->pred) {
      assert(ctx->pred != instr);
      notes->pred_conflict = true;
      return false;
   }

   /* if the instruction is a kill, we need to ensure *every*
    * bary.f is scheduled.  The hw seems unhappy if the thread
    * gets killed before the end-input (ei) flag is hit.
    *
    * We could do this by adding each bary.f instruction as
    * virtual ssa src for the kill instruction.  But we have
    * fixed length instr->srcs[].
    *
    * TODO we could handle this by false-deps now, probably.
    */
   if (is_kill_or_demote(instr)) {
      struct ir3 *ir = instr->block->shader;

      for (unsigned i = 0; i < ir->baryfs_count; i++) {
         struct ir3_instruction *baryf = ir->baryfs[i];
         if (baryf->flags & IR3_INSTR_UNUSED)
            continue;
         if (!is_scheduled(baryf)) {
            notes->blocked_kill = true;
            return false;
         }
      }
   }

   return true;
}

/* Find the instr->ip of the closest use of an instruction, in
 * pre-sched order.  This isn't going to be the same as post-sched
 * order, but it is a reasonable approximation to limit scheduling
 * instructions *too* early.  This is mostly to prevent bad behavior
 * in cases where we have a large number of possible instructions
 * to choose, to avoid creating too much parallelism (ie. blowing
 * up register pressure)
 *
 * See
 * dEQP-GLES31.functional.atomic_counter.layout.reverse_offset.inc_dec.8_counters_5_calls_1_thread
 */
static int
nearest_use(struct ir3_instruction *instr)
{
   unsigned nearest = ~0;
   foreach_ssa_use (use, instr)
      if (!is_scheduled(use))
         nearest = MIN2(nearest, use->ip);

   /* slight hack.. this heuristic tends to push bary.f's to later
    * in the shader, closer to their uses.  But we actually would
    * prefer to get these scheduled earlier, to unlock varying
    * storage for more VS jobs:
    */
   if (is_input(instr))
      nearest /= 2;

   return nearest;
}

static bool
is_only_nonscheduled_use(struct ir3_instruction *instr,
                         struct ir3_instruction *use)
{
   foreach_ssa_use (other_use, instr) {
      if (other_use != use && !is_scheduled(other_use))
         return false;
   }

   return true;
}

static unsigned
new_regs(struct ir3_instruction *instr)
{
   unsigned regs = 0;

   foreach_dst (dst, instr) {
      if (!is_dest_gpr(dst))
         continue;
      regs += reg_elems(dst);
   }

   return regs;
}

/* find net change to live values if instruction were scheduled: */
static int
live_effect(struct ir3_instruction *instr)
{
   struct ir3_sched_node *n = instr->data;
   int new_live =
      (n->partially_live || !instr->uses || instr->uses->entries == 0)
         ? 0
         : new_regs(instr);
   int freed_live = 0;

   /* if we schedule something that causes a vecN to be live,
    * then count all it's other components too:
    */
   if (n->collect)
      new_live *= n->collect->srcs_count;

   foreach_ssa_src_n (src, n, instr) {
      if (__is_false_dep(instr, n))
         continue;

      if (instr->block != src->block)
         continue;

      if (is_only_nonscheduled_use(src, instr))
         freed_live += new_regs(src);
   }

   return new_live - freed_live;
}

/* Determine if this is an instruction that we'd prefer not to schedule
 * yet, in order to avoid an (ss)/(sy) sync.  This is limited by the
 * ss_delay/sy_delay counters, ie. the more cycles it has been since
 * the last SFU/tex, the less costly a sync would be, and the number of
 * outstanding SFU/tex instructions to prevent a blowup in register pressure.
 */
static bool
should_defer(struct ir3_sched_ctx *ctx, struct ir3_instruction *instr)
{
   if (ctx->ss_delay) {
      if (sched_check_src_cond(instr, is_outstanding_ss, ctx))
         return true;
   }

   /* We mostly just want to try to schedule another texture fetch
    * before scheduling something that would (sy) sync, so we can
    * limit this rule to cases where there are remaining texture
    * fetches
    */
   if (ctx->sy_delay && ctx->remaining_tex) {
      if (sched_check_src_cond(instr, is_outstanding_sy, ctx))
         return true;
   }

   /* Avoid scheduling too many outstanding texture or sfu instructions at
    * once by deferring further tex/SFU instructions. This both prevents
    * stalls when the queue of texture/sfu instructions becomes too large,
    * and prevents unacceptably large increases in register pressure from too
    * many outstanding texture instructions.
    */
   if (ctx->sy_index - ctx->first_outstanding_sy_index >= 8 && is_sy_producer(instr))
      return true;

   if (ctx->ss_index - ctx->first_outstanding_ss_index >= 8 && is_ss_producer(instr))
      return true;

   return false;
}

static struct ir3_sched_node *choose_instr_inc(struct ir3_sched_ctx *ctx,
                                               struct ir3_sched_notes *notes,
                                               bool defer, bool avoid_output);

enum choose_instr_dec_rank {
   DEC_NEUTRAL,
   DEC_NEUTRAL_READY,
   DEC_FREED,
   DEC_FREED_READY,
};

static const char *
dec_rank_name(enum choose_instr_dec_rank rank)
{
   switch (rank) {
   case DEC_NEUTRAL:
      return "neutral";
   case DEC_NEUTRAL_READY:
      return "neutral+ready";
   case DEC_FREED:
      return "freed";
   case DEC_FREED_READY:
      return "freed+ready";
   default:
      return NULL;
   }
}

static unsigned
node_delay(struct ir3_sched_ctx *ctx, struct ir3_sched_node *n)
{
   return MAX2(n->earliest_ip, ctx->ip) - ctx->ip;
}

/**
 * Chooses an instruction to schedule using the Goodman/Hsu (1988) CSR (Code
 * Scheduling for Register pressure) heuristic.
 *
 * Only handles the case of choosing instructions that reduce register pressure
 * or are even.
 */
static struct ir3_sched_node *
choose_instr_dec(struct ir3_sched_ctx *ctx, struct ir3_sched_notes *notes,
                 bool defer)
{
   const char *mode = defer ? "-d" : "";
   struct ir3_sched_node *chosen = NULL;
   enum choose_instr_dec_rank chosen_rank = DEC_NEUTRAL;

   foreach_sched_node (n, &ctx->dag->heads) {
      if (defer && should_defer(ctx, n->instr))
         continue;

      unsigned d = node_delay(ctx, n);

      int live = live_effect(n->instr);
      if (live > 0)
         continue;

      if (!check_instr(ctx, notes, n->instr))
         continue;

      enum choose_instr_dec_rank rank;
      if (live < 0) {
         /* Prioritize instrs which free up regs and can be scheduled with no
          * delay.
          */
         if (d == 0)
            rank = DEC_FREED_READY;
         else
            rank = DEC_FREED;
      } else {
         /* Contra the paper, pick a leader with no effect on used regs.  This
          * may open up new opportunities, as otherwise a single-operand instr
          * consuming a value will tend to block finding freeing that value.
          * This had a massive effect on reducing spilling on V3D.
          *
          * XXX: Should this prioritize ready?
          */
         if (d == 0)
            rank = DEC_NEUTRAL_READY;
         else
            rank = DEC_NEUTRAL;
      }

      /* Prefer higher-ranked instructions, or in the case of a rank tie, the
       * highest latency-to-end-of-program instruction.
       */
      if (!chosen || rank > chosen_rank ||
          (rank == chosen_rank && chosen->max_delay < n->max_delay)) {
         chosen = n;
         chosen_rank = rank;
      }
   }

   if (chosen) {
      di(chosen->instr, "dec%s: chose (%s)", mode, dec_rank_name(chosen_rank));
      return chosen;
   }

   return choose_instr_inc(ctx, notes, defer, true);
}

enum choose_instr_inc_rank {
   INC_DISTANCE,
   INC_DISTANCE_READY,
};

static const char *
inc_rank_name(enum choose_instr_inc_rank rank)
{
   switch (rank) {
   case INC_DISTANCE:
      return "distance";
   case INC_DISTANCE_READY:
      return "distance+ready";
   default:
      return NULL;
   }
}

/**
 * When we can't choose an instruction that reduces register pressure or
 * is neutral, we end up here to try and pick the least bad option.
 */
static struct ir3_sched_node *
choose_instr_inc(struct ir3_sched_ctx *ctx, struct ir3_sched_notes *notes,
                 bool defer, bool avoid_output)
{
   const char *mode = defer ? "-d" : "";
   struct ir3_sched_node *chosen = NULL;
   enum choose_instr_inc_rank chosen_rank = INC_DISTANCE;

   /*
    * From hear on out, we are picking something that increases
    * register pressure.  So try to pick something which will
    * be consumed soon:
    */
   unsigned chosen_distance = 0;

   /* Pick the max delay of the remaining ready set. */
   foreach_sched_node (n, &ctx->dag->heads) {
      if (avoid_output && n->output)
         continue;

      if (defer && should_defer(ctx, n->instr))
         continue;

      if (!check_instr(ctx, notes, n->instr))
         continue;

      unsigned d = node_delay(ctx, n);

      enum choose_instr_inc_rank rank;
      if (d == 0)
         rank = INC_DISTANCE_READY;
      else
         rank = INC_DISTANCE;

      unsigned distance = nearest_use(n->instr);

      if (!chosen || rank > chosen_rank ||
          (rank == chosen_rank && distance < chosen_distance)) {
         chosen = n;
         chosen_distance = distance;
         chosen_rank = rank;
      }
   }

   if (chosen) {
      di(chosen->instr, "inc%s: chose (%s)", mode, inc_rank_name(chosen_rank));
      return chosen;
   }

   return NULL;
}

/* Handles instruction selections for instructions we want to prioritize
 * even if csp/csr would not pick them.
 */
static struct ir3_sched_node *
choose_instr_prio(struct ir3_sched_ctx *ctx, struct ir3_sched_notes *notes)
{
   struct ir3_sched_node *chosen = NULL;

   foreach_sched_node (n, &ctx->dag->heads) {
      /*
       * - phi nodes and inputs must be scheduled first
       * - split should be scheduled first, so that the vector value is
       *   killed as soon as possible. RA cannot split up the vector and
       *   reuse components that have been killed until it's been killed.
       * - collect, on the other hand, should be treated as a "normal"
       *   instruction, and may add to register pressure if its sources are
       *   part of another vector or immediates.
       */
      if (!is_meta(n->instr) || n->instr->opc == OPC_META_COLLECT)
         continue;

      if (!chosen || (chosen->max_delay < n->max_delay))
         chosen = n;
   }

   if (chosen) {
      di(chosen->instr, "prio: chose (meta)");
      return chosen;
   }

   return NULL;
}

static void
dump_state(struct ir3_sched_ctx *ctx)
{
   if (!SCHED_DEBUG)
      return;

   foreach_sched_node (n, &ctx->dag->heads) {
      di(n->instr, "maxdel=%3d le=%d del=%u ", n->max_delay,
         live_effect(n->instr), node_delay(ctx, n));

      util_dynarray_foreach (&n->dag.edges, struct dag_edge, edge) {
         struct ir3_sched_node *child = (struct ir3_sched_node *)edge->child;

         di(child->instr, " -> (%d parents) ", child->dag.parent_count);
      }
   }
}

/* find instruction to schedule: */
static struct ir3_instruction *
choose_instr(struct ir3_sched_ctx *ctx, struct ir3_sched_notes *notes)
{
   struct ir3_sched_node *chosen;

   dump_state(ctx);

   chosen = choose_instr_prio(ctx, notes);
   if (chosen)
      return chosen->instr;

   chosen = choose_instr_dec(ctx, notes, true);
   if (chosen)
      return chosen->instr;

   chosen = choose_instr_dec(ctx, notes, false);
   if (chosen)
      return chosen->instr;

   chosen = choose_instr_inc(ctx, notes, false, false);
   if (chosen)
      return chosen->instr;

   return NULL;
}

static struct ir3_instruction *
split_instr(struct ir3_sched_ctx *ctx, struct ir3_instruction *orig_instr)
{
   struct ir3_instruction *new_instr = ir3_instr_clone(orig_instr);
   di(new_instr, "split instruction");
   sched_node_init(ctx, new_instr);
   return new_instr;
}

/* "spill" the address registers by remapping any unscheduled
 * instructions which depend on the current address register
 * to a clone of the instruction which wrote the address reg.
 */
static struct ir3_instruction *
split_addr(struct ir3_sched_ctx *ctx, struct ir3_instruction **addr,
           struct ir3_instruction **users, unsigned users_count)
{
   struct ir3_instruction *new_addr = NULL;
   unsigned i;

   assert(*addr);

   for (i = 0; i < users_count; i++) {
      struct ir3_instruction *indirect = users[i];

      if (!indirect)
         continue;

      /* skip instructions already scheduled: */
      if (is_scheduled(indirect))
         continue;

      /* remap remaining instructions using current addr
       * to new addr:
       */
      if (indirect->address->def == (*addr)->dsts[0]) {
         if (!new_addr) {
            new_addr = split_instr(ctx, *addr);
            /* original addr is scheduled, but new one isn't: */
            new_addr->flags &= ~IR3_INSTR_MARK;
         }
         indirect->address->def = new_addr->dsts[0];
         /* don't need to remove old dag edge since old addr is
          * already scheduled:
          */
         sched_node_add_dep(indirect, new_addr, 0);
         di(indirect, "new address");
      }
   }

   /* all remaining indirects remapped to new addr: */
   *addr = NULL;

   return new_addr;
}

/* "spill" the predicate register by remapping any unscheduled
 * instructions which depend on the current predicate register
 * to a clone of the instruction which wrote the address reg.
 */
static struct ir3_instruction *
split_pred(struct ir3_sched_ctx *ctx)
{
   struct ir3 *ir;
   struct ir3_instruction *new_pred = NULL;
   unsigned i;

   assert(ctx->pred);

   ir = ctx->pred->block->shader;

   for (i = 0; i < ir->predicates_count; i++) {
      struct ir3_instruction *predicated = ir->predicates[i];

      if (!predicated)
         continue;

      /* skip instructions already scheduled: */
      if (is_scheduled(predicated))
         continue;

      /* remap remaining instructions using current pred
       * to new pred:
       *
       * TODO is there ever a case when pred isn't first
       * (and only) src?
       */
      if (ssa(predicated->srcs[0]) == ctx->pred) {
         if (!new_pred) {
            new_pred = split_instr(ctx, ctx->pred);
            /* original pred is scheduled, but new one isn't: */
            new_pred->flags &= ~IR3_INSTR_MARK;
         }
         predicated->srcs[0]->def->instr = new_pred;
         /* don't need to remove old dag edge since old pred is
          * already scheduled:
          */
         sched_node_add_dep(predicated, new_pred, 0);
         di(predicated, "new predicate");
      }
   }

   if (ctx->block->condition == ctx->pred) {
      if (!new_pred) {
         new_pred = split_instr(ctx, ctx->pred);
         /* original pred is scheduled, but new one isn't: */
         new_pred->flags &= ~IR3_INSTR_MARK;
      }
      ctx->block->condition = new_pred;
      d("new branch condition");
   }

   /* all remaining predicated remapped to new pred: */
   ctx->pred = NULL;

   return new_pred;
}

static void
sched_node_init(struct ir3_sched_ctx *ctx, struct ir3_instruction *instr)
{
   struct ir3_sched_node *n = rzalloc(ctx->dag, struct ir3_sched_node);

   dag_init_node(ctx->dag, &n->dag);

   n->instr = instr;
   instr->data = n;
}

static void
sched_node_add_dep(struct ir3_instruction *instr, struct ir3_instruction *src,
                   int i)
{
   /* don't consider dependencies in other blocks: */
   if (src->block != instr->block)
      return;

   /* we could have false-dep's that end up unused: */
   if (src->flags & IR3_INSTR_UNUSED) {
      assert(__is_false_dep(instr, i));
      return;
   }

   struct ir3_sched_node *n = instr->data;
   struct ir3_sched_node *sn = src->data;

   /* If src is consumed by a collect, track that to realize that once
    * any of the collect srcs are live, we should hurry up and schedule
    * the rest.
    */
   if (instr->opc == OPC_META_COLLECT)
      sn->collect = instr;

   unsigned d_soft = ir3_delayslots(src, instr, i, true);
   unsigned d = ir3_delayslots(src, instr, i, false);

   /* delays from (ss) and (sy) are considered separately and more accurately in
    * the scheduling heuristic, so ignore it when calculating the ip of
    * instructions, but do consider it when prioritizing which instructions to
    * schedule.
    */
   dag_add_edge_max_data(&sn->dag, &n->dag, (uintptr_t)d);

   n->delay = MAX2(n->delay, d_soft);
}

static void
mark_kill_path(struct ir3_instruction *instr)
{
   struct ir3_sched_node *n = instr->data;

   if (n->kill_path) {
      return;
   }

   n->kill_path = true;

   foreach_ssa_src (src, instr) {
      if (src->block != instr->block)
         continue;
      mark_kill_path(src);
   }
}

/* Is it an output? */
static bool
is_output_collect(struct ir3_instruction *instr)
{
   if (instr->opc != OPC_META_COLLECT)
      return false;

   foreach_ssa_use (use, instr) {
      if (use->opc != OPC_END && use->opc != OPC_CHMASK)
         return false;
   }

   return true;
}

/* Is it's only use as output? */
static bool
is_output_only(struct ir3_instruction *instr)
{
   foreach_ssa_use (use, instr)
      if (!is_output_collect(use))
         return false;

   return true;
}

static void
sched_node_add_deps(struct ir3_instruction *instr)
{
   /* There's nothing to do for phi nodes, since they always go first. And
    * phi nodes can reference sources later in the same block, so handling
    * sources is not only unnecessary but could cause problems.
    */
   if (instr->opc == OPC_META_PHI)
      return;

   /* Since foreach_ssa_src() already handles false-dep's we can construct
    * the DAG easily in a single pass.
    */
   foreach_ssa_src_n (src, i, instr) {
      sched_node_add_dep(instr, src, i);
   }

   /* NOTE that all inputs must be scheduled before a kill, so
    * mark these to be prioritized as well:
    */
   if (is_kill_or_demote(instr) || is_input(instr)) {
      mark_kill_path(instr);
   }

   if (is_output_only(instr)) {
      struct ir3_sched_node *n = instr->data;
      n->output = true;
   }
}

static void
sched_dag_max_delay_cb(struct dag_node *node, void *state)
{
   struct ir3_sched_node *n = (struct ir3_sched_node *)node;
   uint32_t max_delay = 0;

   util_dynarray_foreach (&n->dag.edges, struct dag_edge, edge) {
      struct ir3_sched_node *child = (struct ir3_sched_node *)edge->child;
      max_delay = MAX2(child->max_delay, max_delay);
   }

   n->max_delay = MAX2(n->max_delay, max_delay + n->delay);
}

static void
sched_dag_init(struct ir3_sched_ctx *ctx)
{
   ctx->dag = dag_create(ctx);

   foreach_instr (instr, &ctx->unscheduled_list)
      sched_node_init(ctx, instr);

   foreach_instr (instr, &ctx->unscheduled_list)
      sched_node_add_deps(instr);

   dag_traverse_bottom_up(ctx->dag, sched_dag_max_delay_cb, NULL);
}

static void
sched_dag_destroy(struct ir3_sched_ctx *ctx)
{
   ralloc_free(ctx->dag);
   ctx->dag = NULL;
}

static void
sched_block(struct ir3_sched_ctx *ctx, struct ir3_block *block)
{
   ctx->block = block;

   /* addr/pred writes are per-block: */
   ctx->addr0 = NULL;
   ctx->addr1 = NULL;
   ctx->pred = NULL;
   ctx->sy_delay = 0;
   ctx->ss_delay = 0;
   ctx->sy_index = ctx->first_outstanding_sy_index = 0;
   ctx->ss_index = ctx->first_outstanding_ss_index = 0;

   /* move all instructions to the unscheduled list, and
    * empty the block's instruction list (to which we will
    * be inserting).
    */
   list_replace(&block->instr_list, &ctx->unscheduled_list);
   list_inithead(&block->instr_list);

   sched_dag_init(ctx);

   ctx->remaining_kills = 0;
   ctx->remaining_tex = 0;
   foreach_instr_safe (instr, &ctx->unscheduled_list) {
      if (is_kill_or_demote(instr))
         ctx->remaining_kills++;
      if (is_sy_producer(instr))
         ctx->remaining_tex++;
   }

   /* First schedule all meta:input and meta:phi instructions, followed by
    * tex-prefetch.  We want all of the instructions that load values into
    * registers before the shader starts to go before any other instructions.
    * But in particular we want inputs to come before prefetches.  This is
    * because a FS's bary_ij input may not actually be live in the shader,
    * but it should not be scheduled on top of any other input (but can be
    * overwritten by a tex prefetch)
    *
    * Note: Because the first block cannot have predecessors, meta:input and
    * meta:phi cannot exist in the same block.
    */
   foreach_instr_safe (instr, &ctx->unscheduled_list)
      if (instr->opc == OPC_META_INPUT || instr->opc == OPC_META_PHI)
         schedule(ctx, instr);

   foreach_instr_safe (instr, &ctx->unscheduled_list)
      if (instr->opc == OPC_META_TEX_PREFETCH)
         schedule(ctx, instr);

   while (!list_is_empty(&ctx->unscheduled_list)) {
      struct ir3_sched_notes notes = {0};
      struct ir3_instruction *instr;

      instr = choose_instr(ctx, &notes);
      if (instr) {
         unsigned delay = node_delay(ctx, instr->data);
         d("delay=%u", delay);

         assert(delay <= 6);

         schedule(ctx, instr);

         /* Since we've scheduled a "real" instruction, we can now
          * schedule any split instruction created by the scheduler again.
          */
         ctx->split = NULL;
      } else {
         struct ir3_instruction *new_instr = NULL;
         struct ir3 *ir = block->shader;

         /* nothing available to schedule.. if we are blocked on
          * address/predicate register conflict, then break the
          * deadlock by cloning the instruction that wrote that
          * reg:
          */
         if (notes.addr0_conflict) {
            new_instr =
               split_addr(ctx, &ctx->addr0, ir->a0_users, ir->a0_users_count);
         } else if (notes.addr1_conflict) {
            new_instr =
               split_addr(ctx, &ctx->addr1, ir->a1_users, ir->a1_users_count);
         } else if (notes.pred_conflict) {
            new_instr = split_pred(ctx);
         } else {
            d("unscheduled_list:");
            foreach_instr (instr, &ctx->unscheduled_list)
               di(instr, "unscheduled: ");
            assert(0);
            ctx->error = true;
            return;
         }

         if (new_instr) {
            list_delinit(&new_instr->node);
            list_addtail(&new_instr->node, &ctx->unscheduled_list);
         }

         /* If we produced a new instruction, do not schedule it next to
          * guarantee progress.
          */
         ctx->split = new_instr;
      }
   }

   sched_dag_destroy(ctx);
}

int
ir3_sched(struct ir3 *ir)
{
   struct ir3_sched_ctx *ctx = rzalloc(NULL, struct ir3_sched_ctx);

   foreach_block (block, &ir->block_list) {
      foreach_instr (instr, &block->instr_list) {
         instr->data = NULL;
      }
   }

   ir3_count_instructions(ir);
   ir3_clear_mark(ir);
   ir3_find_ssa_uses(ir, ctx, false);

   foreach_block (block, &ir->block_list) {
      sched_block(ctx, block);
   }

   int ret = ctx->error ? -1 : 0;

   ralloc_free(ctx);

   return ret;
}

static unsigned
get_array_id(struct ir3_instruction *instr)
{
   /* The expectation is that there is only a single array
    * src or dst, ir3_cp should enforce this.
    */

   foreach_dst (dst, instr)
      if (dst->flags & IR3_REG_ARRAY)
         return dst->array.id;
   foreach_src (src, instr)
      if (src->flags & IR3_REG_ARRAY)
         return src->array.id;

   unreachable("this was unexpected");
}

/* does instruction 'prior' need to be scheduled before 'instr'? */
static bool
depends_on(struct ir3_instruction *instr, struct ir3_instruction *prior)
{
   /* TODO for dependencies that are related to a specific object, ie
    * a specific SSBO/image/array, we could relax this constraint to
    * make accesses to unrelated objects not depend on each other (at
    * least as long as not declared coherent)
    */
   if (((instr->barrier_class & IR3_BARRIER_EVERYTHING) &&
        prior->barrier_class) ||
       ((prior->barrier_class & IR3_BARRIER_EVERYTHING) &&
        instr->barrier_class))
      return true;

   if (instr->barrier_class & prior->barrier_conflict) {
      if (!(instr->barrier_class &
            ~(IR3_BARRIER_ARRAY_R | IR3_BARRIER_ARRAY_W))) {
         /* if only array barrier, then we can further limit false-deps
          * by considering the array-id, ie reads/writes to different
          * arrays do not depend on each other (no aliasing)
          */
         if (get_array_id(instr) != get_array_id(prior)) {
            return false;
         }
      }

      return true;
   }

   return false;
}

static void
add_barrier_deps(struct ir3_block *block, struct ir3_instruction *instr)
{
   struct list_head *prev = instr->node.prev;
   struct list_head *next = instr->node.next;

   /* add dependencies on previous instructions that must be scheduled
    * prior to the current instruction
    */
   while (prev != &block->instr_list) {
      struct ir3_instruction *pi =
         list_entry(prev, struct ir3_instruction, node);

      prev = prev->prev;

      if (is_meta(pi))
         continue;

      if (instr->barrier_class == pi->barrier_class) {
         ir3_instr_add_dep(instr, pi);
         break;
      }

      if (depends_on(instr, pi))
         ir3_instr_add_dep(instr, pi);
   }

   /* add dependencies on this instruction to following instructions
    * that must be scheduled after the current instruction:
    */
   while (next != &block->instr_list) {
      struct ir3_instruction *ni =
         list_entry(next, struct ir3_instruction, node);

      next = next->next;

      if (is_meta(ni))
         continue;

      if (instr->barrier_class == ni->barrier_class) {
         ir3_instr_add_dep(ni, instr);
         break;
      }

      if (depends_on(ni, instr))
         ir3_instr_add_dep(ni, instr);
   }
}

/* before scheduling a block, we need to add any necessary false-dependencies
 * to ensure that:
 *
 *  (1) barriers are scheduled in the right order wrt instructions related
 *      to the barrier
 *
 *  (2) reads that come before a write actually get scheduled before the
 *      write
 */
bool
ir3_sched_add_deps(struct ir3 *ir)
{
   bool progress = false;

   foreach_block (block, &ir->block_list) {
      foreach_instr (instr, &block->instr_list) {
         if (instr->barrier_class) {
            add_barrier_deps(block, instr);
            progress = true;
         }
      }
   }

   return progress;
}