intel/compiler/brw_lower_subgroup_ops.cpp

/*
 * Copyright 2024 Intel Corporation
 * SPDX-License-Identifier: MIT
 */

#include <stdint.h>
#include "util/half_float.h"

#include "brw_fs.h"
#include "brw_builder.h"

using namespace brw;

struct brw_reduction_info {
   brw_reg             identity;
   enum opcode         op;
   brw_conditional_mod cond_mod;
};

static brw_reduction_info
brw_get_reduction_info(brw_reduce_op red_op, brw_reg_type type)
{
   struct brw_reduction_info info;

   info.op = BRW_OPCODE_SEL;
   info.cond_mod = BRW_CONDITIONAL_NONE;

   switch (red_op) {
   case BRW_REDUCE_OP_ADD: info.op = BRW_OPCODE_ADD;           break;
   case BRW_REDUCE_OP_MUL: info.op = BRW_OPCODE_MUL;           break;
   case BRW_REDUCE_OP_AND: info.op = BRW_OPCODE_AND;           break;
   case BRW_REDUCE_OP_OR:  info.op = BRW_OPCODE_OR;            break;
   case BRW_REDUCE_OP_XOR: info.op = BRW_OPCODE_XOR;           break;
   case BRW_REDUCE_OP_MIN: info.cond_mod = BRW_CONDITIONAL_L;  break;
   case BRW_REDUCE_OP_MAX: info.cond_mod = BRW_CONDITIONAL_GE; break;
   default:
      unreachable("invalid reduce op");
   }

   switch (red_op) {
   case BRW_REDUCE_OP_ADD:
   case BRW_REDUCE_OP_XOR:
   case BRW_REDUCE_OP_OR:
      info.identity = retype(brw_imm_u64(0), type);
      return info;
   case BRW_REDUCE_OP_AND:
      info.identity = retype(brw_imm_u64(~0ull), type);
      return info;
   default:
      /* Continue below. */
      break;
   }

   brw_reg id;
   const unsigned size = brw_type_size_bytes(type);

   switch (red_op) {
   case BRW_REDUCE_OP_MUL: {
      if (brw_type_is_int(type)) {
         id = size < 4  ? brw_imm_uw(1) :
              size == 4 ? brw_imm_ud(1) :
                          brw_imm_u64(1);
      } else {
         assert(brw_type_is_float(type));
         id = size == 2 ? brw_imm_uw(_mesa_float_to_half(1.0)) :
              size == 4 ? brw_imm_f(1.0) :
                          brw_imm_df(1.0);
      }
      break;
   }

   case BRW_REDUCE_OP_MIN: {
      if (brw_type_is_uint(type)) {
         id = brw_imm_u64(~0ull);
      } else if (brw_type_is_sint(type)) {
         id = size == 1 ? brw_imm_w(INT8_MAX) :
              size == 2 ? brw_imm_w(INT16_MAX) :
              size == 4 ? brw_imm_d(INT32_MAX) :
                          brw_imm_q(INT64_MAX);
      } else {
         assert(brw_type_is_float(type));
         id = size == 2 ? brw_imm_uw(_mesa_float_to_half(INFINITY)) :
              size == 4 ? brw_imm_f(INFINITY) :
                          brw_imm_df(INFINITY);
      }
      break;
   }

   case BRW_REDUCE_OP_MAX: {
      if (brw_type_is_uint(type)) {
         id = brw_imm_u64(0);
      } else if (brw_type_is_sint(type)) {
         id = size == 1 ? brw_imm_w(INT8_MIN) :
              size == 2 ? brw_imm_w(INT16_MIN) :
              size == 4 ? brw_imm_d(INT32_MIN) :
                          brw_imm_q(INT64_MIN);
      } else {
         assert(brw_type_is_float(type));
         id = size == 2 ? brw_imm_uw(_mesa_float_to_half(-INFINITY)) :
              size == 4 ? brw_imm_f(-INFINITY) :
                          brw_imm_df(-INFINITY);
      }
      break;
   }

   default:
      unreachable("invalid reduce op");
   }

   /* For some cases above (e.g. all bits zeros, all bits ones, first bit one)
    * either the size or the signedness was ignored, so adjust the final type
    * now.
    *
    * B/UB types can't have immediates, so used W/UW above and here.
    */
   if      (type == BRW_TYPE_UB) type = BRW_TYPE_UW;
   else if (type == BRW_TYPE_B)  type = BRW_TYPE_W;

   info.identity = retype(id, type);

   return info;
}

static void
brw_emit_scan_step(const brw_builder &bld, enum opcode opcode, brw_conditional_mod mod,
                   const brw_reg &tmp,
                   unsigned left_offset, unsigned left_stride,
                   unsigned right_offset, unsigned right_stride)
{
   brw_reg left, right;
   left = horiz_stride(horiz_offset(tmp, left_offset), left_stride);
   right = horiz_stride(horiz_offset(tmp, right_offset), right_stride);
   if ((tmp.type == BRW_TYPE_Q || tmp.type == BRW_TYPE_UQ) &&
       (!bld.shader->devinfo->has_64bit_int || bld.shader->devinfo->ver >= 20)) {
      switch (opcode) {
      case BRW_OPCODE_MUL:
         /* This will get lowered by integer MUL lowering */
         set_condmod(mod, bld.emit(opcode, right, left, right));
         break;

      case BRW_OPCODE_SEL: {
         /* In order for the comparisons to work out right, we need our
          * comparisons to be strict.
          */
         assert(mod == BRW_CONDITIONAL_L || mod == BRW_CONDITIONAL_GE);
         if (mod == BRW_CONDITIONAL_GE)
            mod = BRW_CONDITIONAL_G;

         /* We treat the bottom 32 bits as unsigned regardless of
          * whether or not the integer as a whole is signed.
          */
         brw_reg right_low = subscript(right, BRW_TYPE_UD, 0);
         brw_reg left_low = subscript(left, BRW_TYPE_UD, 0);

         /* The upper bits get the same sign as the 64-bit type */
         brw_reg_type type32 = brw_type_with_size(tmp.type, 32);
         brw_reg right_high = subscript(right, type32, 1);
         brw_reg left_high = subscript(left, type32, 1);

         /* Build up our comparison:
          *
          *   l_hi < r_hi || (l_hi == r_hi && l_low < r_low)
          */
         bld.CMP(bld.null_reg_ud(), retype(left_low, BRW_TYPE_UD),
                            retype(right_low, BRW_TYPE_UD), mod);
         set_predicate(BRW_PREDICATE_NORMAL,
                       bld.CMP(bld.null_reg_ud(), left_high, right_high,
                           BRW_CONDITIONAL_EQ));
         set_predicate_inv(BRW_PREDICATE_NORMAL, true,
                           bld.CMP(bld.null_reg_ud(), left_high, right_high, mod));

         /* We could use selects here or we could use predicated MOVs
          * because the destination and second source (if it were a SEL)
          * are the same.
          */
         set_predicate(BRW_PREDICATE_NORMAL, bld.MOV(right_low, left_low));
         set_predicate(BRW_PREDICATE_NORMAL, bld.MOV(right_high, left_high));
         break;
      }

      default:
         unreachable("Unsupported 64-bit scan op");
      }
   } else {
      set_condmod(mod, bld.emit(opcode, right, left, right));
   }
}

static void
brw_emit_scan(const brw_builder &bld, enum opcode opcode, const brw_reg &tmp,
              unsigned cluster_size, brw_conditional_mod mod)
{
   unsigned dispatch_width = bld.dispatch_width();
   assert(dispatch_width >= 8);

   /* The instruction splitting code isn't advanced enough to split
    * these so we need to handle that ourselves.
    */
   if (dispatch_width * brw_type_size_bytes(tmp.type) > 2 * REG_SIZE) {
      const unsigned half_width = dispatch_width / 2;
      const brw_builder ubld = bld.exec_all().group(half_width, 0);
      brw_reg left = tmp;
      brw_reg right = horiz_offset(tmp, half_width);
      brw_emit_scan(ubld, opcode, left, cluster_size, mod);
      brw_emit_scan(ubld, opcode, right, cluster_size, mod);
      if (cluster_size > half_width) {
         brw_emit_scan_step(ubld, opcode, mod, tmp,
                            half_width - 1, 0, half_width, 1);
      }
      return;
   }

   if (cluster_size > 1) {
      const brw_builder ubld = bld.exec_all().group(dispatch_width / 2, 0);
      brw_emit_scan_step(ubld, opcode, mod, tmp, 0, 2, 1, 2);
   }

   if (cluster_size > 2) {
      if (brw_type_size_bytes(tmp.type) <= 4) {
         const brw_builder ubld =
            bld.exec_all().group(dispatch_width / 4, 0);
         brw_emit_scan_step(ubld, opcode, mod, tmp, 1, 4, 2, 4);
         brw_emit_scan_step(ubld, opcode, mod, tmp, 1, 4, 3, 4);
      } else {
         /* For 64-bit types, we have to do things differently because
          * the code above would land us with destination strides that
          * the hardware can't handle.  Fortunately, we'll only be
          * 8-wide in that case and it's the same number of
          * instructions.
          */
         const brw_builder ubld = bld.exec_all().group(2, 0);
         for (unsigned i = 0; i < dispatch_width; i += 4)
            brw_emit_scan_step(ubld, opcode, mod, tmp, i + 1, 0, i + 2, 1);
      }
   }

   for (unsigned i = 4;
        i < MIN2(cluster_size, dispatch_width);
        i *= 2) {
      const brw_builder ubld = bld.exec_all().group(i, 0);
      brw_emit_scan_step(ubld, opcode, mod, tmp, i - 1, 0, i, 1);

      if (dispatch_width > i * 2)
         brw_emit_scan_step(ubld, opcode, mod, tmp, i * 3 - 1, 0, i * 3, 1);

      if (dispatch_width > i * 4) {
         brw_emit_scan_step(ubld, opcode, mod, tmp, i * 5 - 1, 0, i * 5, 1);
         brw_emit_scan_step(ubld, opcode, mod, tmp, i * 7 - 1, 0, i * 7, 1);
      }
   }
}

static bool
brw_lower_reduce(fs_visitor &s, bblock_t *block, fs_inst *inst)
{
   const brw_builder bld(&s, block, inst);

   assert(inst->dst.type == inst->src[0].type);
   brw_reg dst = inst->dst;
   brw_reg src = inst->src[0];

   assert(inst->src[1].file == IMM);
   enum brw_reduce_op op = (enum brw_reduce_op)inst->src[1].ud;

   assert(inst->src[2].file == IMM);
   unsigned cluster_size = inst->src[2].ud;

   assert(cluster_size > 0);
   assert(cluster_size <= s.dispatch_width);

   struct brw_reduction_info info = brw_get_reduction_info(op, src.type);

   /* Set up a register for all of our scratching around and initialize it
    * to reduction operation's identity value.
    */
   brw_reg scan = bld.vgrf(src.type);
   bld.exec_all().emit(SHADER_OPCODE_SEL_EXEC, scan, src, info.identity);

   brw_emit_scan(bld, info.op, scan, cluster_size, info.cond_mod);

   if (cluster_size * brw_type_size_bytes(src.type) >= REG_SIZE * 2) {
      /* In this case, CLUSTER_BROADCAST instruction isn't needed because
       * the distance between clusters is at least 2 GRFs.  In this case,
       * we don't need the weird striding of the CLUSTER_BROADCAST
       * instruction and can just do regular MOVs.
       */
      assert((cluster_size * brw_type_size_bytes(src.type)) % (REG_SIZE * 2) == 0);
      const unsigned groups =
         (s.dispatch_width * brw_type_size_bytes(src.type)) / (REG_SIZE * 2);
      const unsigned group_size = s.dispatch_width / groups;
      for (unsigned i = 0; i < groups; i++) {
         const unsigned cluster = (i * group_size) / cluster_size;
         const unsigned comp = cluster * cluster_size + (cluster_size - 1);
         bld.group(group_size, i).MOV(horiz_offset(dst, i * group_size),
                                      component(scan, comp));
      }
   } else {
      bld.emit(SHADER_OPCODE_CLUSTER_BROADCAST, dst, scan,
               brw_imm_ud(cluster_size - 1), brw_imm_ud(cluster_size));
   }
   inst->remove(block);
   return true;
}

static bool
brw_lower_scan(fs_visitor &s, bblock_t *block, fs_inst *inst)
{
   const brw_builder bld(&s, block, inst);

   assert(inst->dst.type == inst->src[0].type);
   brw_reg dst = inst->dst;
   brw_reg src = inst->src[0];

   assert(inst->src[1].file == IMM);
   enum brw_reduce_op op = (enum brw_reduce_op)inst->src[1].ud;

   struct brw_reduction_info info = brw_get_reduction_info(op, src.type);

   /* Set up a register for all of our scratching around and initialize it
    * to reduction operation's identity value.
    */
   brw_reg scan = bld.vgrf(src.type);
   const brw_builder ubld = bld.exec_all();
   ubld.emit(SHADER_OPCODE_SEL_EXEC, scan, src, info.identity);

   if (inst->opcode == SHADER_OPCODE_EXCLUSIVE_SCAN) {
      /* Exclusive scan is a bit harder because we have to do an annoying
       * shift of the contents before we can begin.  To make things worse,
       * we can't do this with a normal stride; we have to use indirects.
       */
      brw_reg shifted = bld.vgrf(src.type);
      brw_reg idx = bld.vgrf(BRW_TYPE_UW);

      /* Set the saturate modifier in the offset index to ensure it's
       * normalized within the expected range without negative values,
       * since the situation can cause us to read past the end of the
       * register file leading to hangs on Xe3.
       */
      set_saturate(true, ubld.ADD(idx, bld.LOAD_SUBGROUP_INVOCATION(),
                                  brw_imm_w(-1)));
      ubld.emit(SHADER_OPCODE_SHUFFLE, shifted, scan, idx);
      ubld.group(1, 0).MOV(horiz_offset(shifted, 0), info.identity);
      scan = shifted;
   }

   brw_emit_scan(bld, info.op, scan, s.dispatch_width, info.cond_mod);

   bld.MOV(dst, scan);

   inst->remove(block);
   return true;
}

static brw_reg
brw_fill_flag(const brw_builder &bld, unsigned v)
{
   const brw_builder ubld1 = bld.exec_all().group(1, 0);
   brw_reg flag = brw_flag_reg(0, 0);

   if (bld.shader->dispatch_width == 32) {
      /* For SIMD32, we use a UD type so we fill both f0.0 and f0.1. */
      flag = retype(flag, BRW_TYPE_UD);
      ubld1.MOV(flag, brw_imm_ud(v));
   } else {
      ubld1.MOV(flag, brw_imm_uw(v & 0xFFFF));
   }

   return flag;
}

static void
brw_lower_dispatch_width_vote(const brw_builder &bld, enum opcode opcode, brw_reg dst, brw_reg src)
{
   const intel_device_info *devinfo = bld.shader->devinfo;
   const unsigned dispatch_width = bld.shader->dispatch_width;

   assert(opcode == SHADER_OPCODE_VOTE_ANY ||
          opcode == SHADER_OPCODE_VOTE_ALL ||
          opcode == SHADER_OPCODE_VOTE_EQUAL);

   const bool any   = opcode == SHADER_OPCODE_VOTE_ANY;
   const bool equal = opcode == SHADER_OPCODE_VOTE_EQUAL;

   const brw_reg ref = equal ? bld.emit_uniformize(src) : brw_imm_d(0);

   /* The any/all predicates do not consider channel enables. To prevent
    * dead channels from affecting the result, we initialize the flag with
    * with the identity value for the logical operation.
    */
   brw_fill_flag(bld, any ? 0 : 0xFFFFFFFF);
   bld.CMP(bld.null_reg_d(), src, ref, equal ? BRW_CONDITIONAL_Z
                                             : BRW_CONDITIONAL_NZ);

   /* For some reason, the any/all predicates don't work properly with
    * SIMD32.  In particular, it appears that a SEL with a QtrCtrl of 2H
    * doesn't read the correct subset of the flag register and you end up
    * getting garbage in the second half.  Work around this by using a pair
    * of 1-wide MOVs and scattering the result.
    *
    * TODO: Check if we still need this for newer platforms.
    */
   const brw_builder ubld = devinfo->ver >= 20 ? bld.exec_all()
                                               : bld.exec_all().group(1, 0);
   brw_reg res1 = ubld.MOV(brw_imm_d(0));

   enum brw_predicate pred;
   if (any) {
      pred = devinfo->ver >= 20   ? XE2_PREDICATE_ANY :
             dispatch_width == 8  ? BRW_PREDICATE_ALIGN1_ANY8H :
             dispatch_width == 16 ? BRW_PREDICATE_ALIGN1_ANY16H :
                                    BRW_PREDICATE_ALIGN1_ANY32H;
   } else {
      pred = devinfo->ver >= 20   ? XE2_PREDICATE_ALL :
             dispatch_width == 8  ? BRW_PREDICATE_ALIGN1_ALL8H :
             dispatch_width == 16 ? BRW_PREDICATE_ALIGN1_ALL16H :
                                    BRW_PREDICATE_ALIGN1_ALL32H;
   }
   set_predicate(pred, ubld.MOV(res1, brw_imm_d(-1)));

   bld.MOV(retype(dst, BRW_TYPE_D), component(res1, 0));
}

static void
brw_lower_quad_vote_gfx9(const brw_builder &bld, enum opcode opcode, brw_reg dst, brw_reg src)
{
   assert(opcode == SHADER_OPCODE_VOTE_ANY || opcode == SHADER_OPCODE_VOTE_ALL);
   const bool any = opcode == SHADER_OPCODE_VOTE_ANY;

   /* The any/all predicates do not consider channel enables. To prevent
    * dead channels from affecting the result, we initialize the flag with
    * with the identity value for the logical operation.
    */
   brw_fill_flag(bld, any ? 0 : 0xFFFFFFFF);
   bld.CMP(bld.null_reg_ud(), src, brw_imm_ud(0u), BRW_CONDITIONAL_NZ);
   bld.exec_all().MOV(retype(dst, BRW_TYPE_UD), brw_imm_ud(0));

   /* Before Xe2, we can use specialized predicates. */
   const enum brw_predicate pred = any ? BRW_PREDICATE_ALIGN1_ANY4H
                                       : BRW_PREDICATE_ALIGN1_ALL4H;

   fs_inst *mov = bld.MOV(retype(dst, BRW_TYPE_D), brw_imm_d(-1));
   set_predicate(pred, mov);
}

static void
brw_lower_quad_vote_gfx20(const brw_builder &bld, enum opcode opcode, brw_reg dst, brw_reg src)
{
   assert(opcode == SHADER_OPCODE_VOTE_ANY || opcode == SHADER_OPCODE_VOTE_ALL);
   const bool any = opcode == SHADER_OPCODE_VOTE_ANY;

   /* This code is going to manipulate the results of flag mask, so clear it to
    * avoid any residual value from disabled channels.
    */
   brw_reg flag = brw_fill_flag(bld, 0);

   /* Mask of invocations where condition is true, note that mask is
    * replicated to each invocation.
    */
   bld.CMP(bld.null_reg_ud(), src, brw_imm_ud(0u), BRW_CONDITIONAL_NZ);
   brw_reg cond_mask = bld.vgrf(BRW_TYPE_UD);
   bld.MOV(cond_mask, flag);

   /* Mask of invocations in the quad, each invocation will get
    * all the bits set for their quad, i.e. invocations 0-3 will have
    * 0b...1111, invocations 4-7 will have 0b...11110000 and so on.
    */
   brw_reg invoc_ud = bld.vgrf(BRW_TYPE_UD);
   bld.MOV(invoc_ud, bld.LOAD_SUBGROUP_INVOCATION());
   brw_reg quad_mask =
      bld.SHL(brw_imm_ud(0xF), bld.AND(invoc_ud, brw_imm_ud(0xFFFFFFFC)));

   /* An invocation will have bits set for each quad that passes the
    * condition.  This is uniform among each quad.
    */
   brw_reg tmp = bld.AND(cond_mask, quad_mask);

   if (any) {
      bld.CMP(retype(dst, BRW_TYPE_UD), tmp, brw_imm_ud(0), BRW_CONDITIONAL_NZ);
   } else {
      /* Filter out quad_mask to include only active channels. */
      brw_reg active = bld.vgrf(BRW_TYPE_UD);
      bld.exec_all().emit(SHADER_OPCODE_LOAD_LIVE_CHANNELS, active);
      bld.MOV(active, brw_reg(component(active, 0)));
      bld.AND(quad_mask, quad_mask, active);

      bld.CMP(retype(dst, BRW_TYPE_UD), tmp, quad_mask, BRW_CONDITIONAL_Z);
   }
}

static bool
brw_lower_vote(fs_visitor &s, bblock_t *block, fs_inst *inst)
{
   const brw_builder bld(&s, block, inst);

   brw_reg dst = inst->dst;
   brw_reg src = inst->src[0];

   unsigned cluster_size;
   if (inst->sources > 1) {
      assert(inst->src[1].file == IMM);
      cluster_size = inst->src[1].ud;
   } else {
      cluster_size = s.dispatch_width;
   }

   if (cluster_size == s.dispatch_width) {
      brw_lower_dispatch_width_vote(bld, inst->opcode, dst, src);
   } else {
      assert(cluster_size == 4);
      if (s.devinfo->ver < 20)
         brw_lower_quad_vote_gfx9(bld, inst->opcode, dst, src);
      else
         brw_lower_quad_vote_gfx20(bld, inst->opcode, dst, src);
   }

   inst->remove(block);
   return true;
}

static bool
brw_lower_ballot(fs_visitor &s, bblock_t *block, fs_inst *inst)
{
   const brw_builder bld(&s, block, inst);

   brw_reg value = retype(inst->src[0], BRW_TYPE_UD);
   brw_reg dst = inst->dst;

   const brw_builder xbld = dst.is_scalar ? bld.scalar_group() : bld;

   if (value.file == IMM) {
      /* Implement a fast-path for ballot(true). */
      if (!value.is_zero()) {
         brw_reg tmp = bld.vgrf(BRW_TYPE_UD);
         bld.exec_all().emit(SHADER_OPCODE_LOAD_LIVE_CHANNELS, tmp);
         xbld.MOV(dst, brw_reg(component(tmp, 0)));
      } else {
         brw_reg zero = retype(brw_imm_uq(0), dst.type);
         xbld.MOV(dst, zero);
      }
   } else {
      brw_reg flag = brw_fill_flag(bld, 0);
      bld.CMP(bld.null_reg_ud(), value, brw_imm_ud(0u), BRW_CONDITIONAL_NZ);
      xbld.MOV(dst, flag);
   }

   inst->remove(block);
   return true;
}

static bool
brw_lower_quad_swap(fs_visitor &s, bblock_t *block, fs_inst *inst)
{
   const brw_builder bld(&s, block, inst);

   assert(inst->dst.type == inst->src[0].type);
   brw_reg dst = inst->dst;
   brw_reg value = inst->src[0];

   assert(inst->src[1].file == IMM);
   enum brw_swap_direction dir = (enum brw_swap_direction)inst->src[1].ud;

   switch (dir) {
   case BRW_SWAP_HORIZONTAL: {
      const brw_reg tmp = bld.vgrf(value.type);

      const brw_builder ubld = bld.exec_all().group(s.dispatch_width / 2, 0);

      const brw_reg src_left = horiz_stride(value, 2);
      const brw_reg src_right = horiz_stride(horiz_offset(value, 1), 2);
      const brw_reg tmp_left = horiz_stride(tmp, 2);
      const brw_reg tmp_right = horiz_stride(horiz_offset(tmp, 1), 2);

      ubld.MOV(tmp_left, src_right);
      ubld.MOV(tmp_right, src_left);

      bld.MOV(retype(dst, value.type), tmp);
      break;
   }
   case BRW_SWAP_VERTICAL:
   case BRW_SWAP_DIAGONAL: {
      if (brw_type_size_bits(value.type) == 32) {
         /* For 32-bit, we can use a SIMD4x2 instruction to do this easily */
         const unsigned swizzle = dir == BRW_SWAP_VERTICAL ? BRW_SWIZZLE4(2,3,0,1)
                                                           : BRW_SWIZZLE4(3,2,1,0);
         const brw_reg tmp = bld.vgrf(value.type);
         const brw_builder ubld = bld.exec_all();
         ubld.emit(SHADER_OPCODE_QUAD_SWIZZLE, tmp, value, brw_imm_ud(swizzle));
         bld.MOV(dst, tmp);
      } else {
         /* For larger data types, we have to either emit dispatch_width many
          * MOVs or else fall back to doing indirects.
          */
         const unsigned xor_mask = dir == BRW_SWAP_VERTICAL ? 0x2 : 0x3;
         brw_reg idx = bld.vgrf(BRW_TYPE_W);
         bld.XOR(idx, bld.LOAD_SUBGROUP_INVOCATION(), brw_imm_w(xor_mask));
         bld.emit(SHADER_OPCODE_SHUFFLE, dst, value, idx);
      }
      break;
   }
   }

   inst->remove(block);
   return true;
}

static bool
brw_lower_read_from_live_channel(fs_visitor &s, bblock_t *block, fs_inst *inst)
{
   const brw_builder bld(&s, block, inst);

   assert(inst->sources == 1);
   assert(inst->dst.type == inst->src[0].type);
   brw_reg dst = inst->dst;
   brw_reg value = inst->src[0];

   bld.MOV(dst, bld.emit_uniformize(value));

   inst->remove(block);
   return true;
}

static bool
brw_lower_read_from_channel(fs_visitor &s, bblock_t *block, fs_inst *inst)
{
   const brw_builder bld(&s, block, inst);

   assert(inst->sources == 2);
   assert(inst->dst.type == inst->src[0].type);

   brw_reg dst = inst->dst;
   brw_reg value = inst->src[0];
   brw_reg index = retype(inst->src[1], BRW_TYPE_UD);

   /* When for some reason the subgroup_size picked by NIR is larger than
    * the dispatch size picked by the backend (this could happen in RT,
    * FS), bound the invocation to the dispatch size.
    */
   const unsigned dispatch_width_mask = s.dispatch_width - 1;

   if (index.file == IMM) {
      /* Always apply mask here since it is cheap. */
      bld.MOV(dst, component(value, index.ud & dispatch_width_mask));
   } else {
      if (s.api_subgroup_size == 0 || s.dispatch_width < s.api_subgroup_size)
         index = bld.AND(index, brw_imm_ud(dispatch_width_mask));

      brw_reg tmp = bld.BROADCAST(value, bld.emit_uniformize(index));
      bld.MOV(dst, tmp);
   }

   inst->remove(block);
   return true;
}

bool
brw_lower_subgroup_ops(fs_visitor &s)
{
   bool progress = false;

   foreach_block_and_inst_safe(block, fs_inst, inst, s.cfg) {
      switch (inst->opcode) {
      case SHADER_OPCODE_REDUCE:
         progress |= brw_lower_reduce(s, block, inst);
         break;

      case SHADER_OPCODE_INCLUSIVE_SCAN:
      case SHADER_OPCODE_EXCLUSIVE_SCAN:
         progress |= brw_lower_scan(s, block, inst);
         break;

      case SHADER_OPCODE_VOTE_ANY:
      case SHADER_OPCODE_VOTE_ALL:
      case SHADER_OPCODE_VOTE_EQUAL:
         progress |= brw_lower_vote(s, block, inst);
         break;

      case SHADER_OPCODE_BALLOT:
         progress |= brw_lower_ballot(s, block, inst);
         break;

      case SHADER_OPCODE_QUAD_SWAP:
         progress |= brw_lower_quad_swap(s, block, inst);
         break;

      case SHADER_OPCODE_READ_FROM_LIVE_CHANNEL:
         progress |= brw_lower_read_from_live_channel(s, block, inst);
         break;

      case SHADER_OPCODE_READ_FROM_CHANNEL:
         progress |= brw_lower_read_from_channel(s, block, inst);
         break;

      default:
         /* Nothing to do. */
         break;
      }
   }

   if (progress)
      s.invalidate_analysis(DEPENDENCY_INSTRUCTIONS | DEPENDENCY_VARIABLES);

   return progress;
}