intel/compiler/brw_lower_regioning.cpp

/*
 * Copyright © 2018 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

#include "brw_fs.h"
#include "brw_cfg.h"
#include "brw_builder.h"

using namespace brw;

namespace {
   /* From the SKL PRM Vol 2a, "Move":
    *
    * "A mov with the same source and destination type, no source modifier,
    *  and no saturation is a raw move. A packed byte destination region (B
    *  or UB type with HorzStride == 1 and ExecSize > 1) can only be written
    *  using raw move."
    */
   bool
   is_byte_raw_mov(const fs_inst *inst)
   {
      return brw_type_size_bytes(inst->dst.type) == 1 &&
             inst->opcode == BRW_OPCODE_MOV &&
             inst->src[0].type == inst->dst.type &&
             !inst->saturate &&
             !inst->src[0].negate &&
             !inst->src[0].abs;
   }

   /*
    * Return an acceptable byte stride for the specified source of an
    * instruction affected by a regioning restriction.
    */
   unsigned
   required_src_byte_stride(const intel_device_info *devinfo, const fs_inst *inst,
                            unsigned i)
   {
      if (has_dst_aligned_region_restriction(devinfo, inst)) {
         return MAX2(brw_type_size_bytes(inst->dst.type),
                     byte_stride(inst->dst));

      } else if (has_subdword_integer_region_restriction(devinfo, inst,
                                                         &inst->src[i], 1)) {
         /* Use a stride of 32bits if possible, since that will guarantee that
          * the copy emitted to lower this region won't be affected by the
          * sub-dword integer region restrictions.  This may not be possible
          * for the second source of an instruction if we're required to use
          * packed data due to Wa_16012383669.
          */
         return (i == 1 ? brw_type_size_bytes(inst->src[i].type) : 4);

      } else {
         return byte_stride(inst->src[i]);
      }
   }

   /*
    * Return an acceptable byte sub-register offset for the specified source
    * of an instruction affected by a regioning restriction.
    */
   unsigned
   required_src_byte_offset(const intel_device_info *devinfo, const fs_inst *inst,
                            unsigned i)
   {
      if (has_dst_aligned_region_restriction(devinfo, inst)) {
         return reg_offset(inst->dst) % (reg_unit(devinfo) * REG_SIZE);

      } else if (has_subdword_integer_region_restriction(devinfo, inst,
                                                         &inst->src[i], 1)) {
         const unsigned dst_byte_stride =
            MAX2(byte_stride(inst->dst), brw_type_size_bytes(inst->dst.type));
         const unsigned src_byte_stride = required_src_byte_stride(devinfo, inst, i);
         const unsigned dst_byte_offset =
            reg_offset(inst->dst) % (reg_unit(devinfo) * REG_SIZE);
         const unsigned src_byte_offset =
            reg_offset(inst->src[i]) % (reg_unit(devinfo) * REG_SIZE);

         if (src_byte_stride > brw_type_size_bytes(inst->src[i].type)) {
            assert(src_byte_stride >= dst_byte_stride);
            /* The source is affected by the Xe2+ sub-dword integer regioning
             * restrictions.  For the case of source 0 BSpec#56640 specifies a
             * number of equations relating the source and destination
             * sub-register numbers in all cases where a source stride of
             * 32bits is allowed.  These equations have the form:
             *
             *   k * Dst.SubReg % m = Src.SubReg / l
             *
             * For some constants k, l and m different for each combination of
             * source and destination types and strides.  The expression in
             * the return statement below computes a valid source offset by
             * inverting the equation like:
             *
             *   Src.SubReg = l * k * (Dst.SubReg % m)
             *
             * and then scaling by the element type sizes in order to get an
             * expression in terms of byte offsets instead of sub-register
             * numbers.  It can be easily verified that in all cases listed on
             * the hardware spec where the source has a well-defined uniform
             * stride the product l*k is equal to the ratio between the source
             * and destination strides.
             */
            const unsigned m = 64 * dst_byte_stride / src_byte_stride;
            return dst_byte_offset % m * src_byte_stride / dst_byte_stride;
         } else {
            assert(src_byte_stride == brw_type_size_bytes(inst->src[i].type));
            /* A packed source is required, likely due to the stricter
             * requirements of the second source region.  The source being
             * packed guarantees that the region of the original instruction
             * will be valid, but the copy may break the regioning
             * restrictions.  Do our best to try to prevent that from
             * happening by making sure the offset of the temporary matches
             * the original source based on the same equation above -- However
             * that may not be sufficient if the source had a stride larger
             * than 32bits, lowering the copy recursively may be necessary.
             */
            return src_byte_offset * src_byte_stride / byte_stride(inst->src[i]);
         }

      } else {
         return reg_offset(inst->src[i]) % (reg_unit(devinfo) * REG_SIZE);
      }
   }

   /*
    * Return an acceptable byte stride for the destination of an instruction
    * that requires it to have some particular alignment.
    */
   unsigned
   required_dst_byte_stride(const fs_inst *inst)
   {
      if (inst->dst.is_accumulator()) {
         /* If the destination is an accumulator, insist that we leave the
          * stride alone.  We cannot "fix" accumulator destinations by writing
          * to a temporary and emitting a MOV into the original destination.
          * For multiply instructions (our one use of the accumulator), the
          * MUL writes the full 66 bits of the accumulator whereas the MOV we
          * would emit only writes 33 bits and leaves the top 33 bits
          * undefined.
          *
          * It's safe to just require the original stride here because the
          * lowering pass will detect the mismatch in has_invalid_src_region
          * and fix the sources of the multiply instead of the destination.
          */
         return inst->dst.hstride * brw_type_size_bytes(inst->dst.type);
      } else if (brw_type_size_bytes(inst->dst.type) < get_exec_type_size(inst) &&
          !is_byte_raw_mov(inst)) {
         return get_exec_type_size(inst);
      } else {
         /* Calculate the maximum byte stride and the minimum/maximum type
          * size across all source and destination operands we are required to
          * lower.
          */
         unsigned max_stride = inst->dst.stride * brw_type_size_bytes(inst->dst.type);
         unsigned min_size = brw_type_size_bytes(inst->dst.type);
         unsigned max_size = brw_type_size_bytes(inst->dst.type);

         for (unsigned i = 0; i < inst->sources; i++) {
            if (!is_uniform(inst->src[i]) && !inst->is_control_source(i)) {
               const unsigned size = brw_type_size_bytes(inst->src[i].type);
               max_stride = MAX2(max_stride, inst->src[i].stride * size);
               min_size = MIN2(min_size, size);
               max_size = MAX2(max_size, size);
            }
         }

         /* All operands involved in lowering need to fit in the calculated
          * stride.
          */
         assert(max_size <= 4 * min_size);

         /* Attempt to use the largest byte stride among all present operands,
          * but never exceed a stride of 4 since that would lead to illegal
          * destination regions during lowering.
          */
         return MIN2(max_stride, 4 * min_size);
      }
   }

   /*
    * Return an acceptable byte sub-register offset for the destination of an
    * instruction that requires it to be aligned to the sub-register offset of
    * the sources.
    */
   unsigned
   required_dst_byte_offset(const intel_device_info *devinfo, const fs_inst *inst)
   {
      for (unsigned i = 0; i < inst->sources; i++) {
         if (!is_uniform(inst->src[i]) && !inst->is_control_source(i))
            if (reg_offset(inst->src[i]) % (reg_unit(devinfo) * REG_SIZE) !=
                reg_offset(inst->dst) % (reg_unit(devinfo) * REG_SIZE))
               return 0;
      }

      return reg_offset(inst->dst) % (reg_unit(devinfo) * REG_SIZE);
   }

   /*
    * Return the closest legal execution type for an instruction on
    * the specified platform.
    */
   brw_reg_type
   required_exec_type(const intel_device_info *devinfo, const fs_inst *inst)
   {
      const brw_reg_type t = get_exec_type(inst);
      const bool has_64bit = brw_type_is_float(t) ?
         devinfo->has_64bit_float : devinfo->has_64bit_int;

      switch (inst->opcode) {
      case SHADER_OPCODE_SHUFFLE:
         /* IVB has an issue (which we found empirically) where it reads
          * two address register components per channel for indirectly
          * addressed 64-bit sources.
          *
          * From the Cherryview PRM Vol 7. "Register Region Restrictions":
          *
          *    "When source or destination datatype is 64b or operation is
          *    integer DWord multiply, indirect addressing must not be
          *    used."
          *
          * Work around both of the above and handle platforms that
          * don't support 64-bit types at all.
          */
         if ((!devinfo->has_64bit_int ||
              intel_device_info_is_9lp(devinfo) ||
              devinfo->ver >= 20) && brw_type_size_bytes(t) > 4)
            return BRW_TYPE_UD;
         else if (has_dst_aligned_region_restriction(devinfo, inst))
            return brw_int_type(brw_type_size_bytes(t), false);
         else
            return t;

      case SHADER_OPCODE_SEL_EXEC:
         if ((!has_64bit || devinfo->has_64bit_float_via_math_pipe) &&
             brw_type_size_bytes(t) > 4)
            return BRW_TYPE_UD;
         else
            return t;

      case SHADER_OPCODE_QUAD_SWIZZLE:
         if (has_dst_aligned_region_restriction(devinfo, inst))
            return brw_int_type(brw_type_size_bytes(t), false);
         else
            return t;

      case SHADER_OPCODE_CLUSTER_BROADCAST:
         /* From the Cherryview PRM Vol 7. "Register Region Restrictions":
          *
          *    "When source or destination datatype is 64b or operation is
          *    integer DWord multiply, indirect addressing must not be
          *    used."
          *
          * For MTL (verx10 == 125), float64 is supported, but int64 is not.
          * Therefore we need to lower cluster broadcast using 32-bit int ops.
          *
          * For gfx12.5+ platforms that support int64, the register regions
          * used by cluster broadcast aren't supported by the 64-bit pipeline.
          *
          * Work around the above and handle platforms that don't
          * support 64-bit types at all.
          */
         if ((!has_64bit || devinfo->verx10 >= 125 ||
              intel_device_info_is_9lp(devinfo) ||
              devinfo->ver >= 20) && brw_type_size_bytes(t) > 4)
            return BRW_TYPE_UD;
         else
            return brw_int_type(brw_type_size_bytes(t), false);

      default:
         return t;
      }
   }

   /*
    * Return whether the instruction has an unsupported channel bit layout
    * specified for the i-th source region.
    */
   bool
   has_invalid_src_region(const intel_device_info *devinfo, const fs_inst *inst,
                          unsigned i)
   {
      /* Wa_22016140776:
       *
       *    Scalar broadcast on HF math (packed or unpacked) must not be used.
       *    Compiler must use a mov instruction to expand the scalar value to
       *    a vector before using in a HF (packed or unpacked) math operation.
       */
      if (inst->is_math() && intel_needs_workaround(devinfo, 22016140776) &&
          is_uniform(inst->src[i]) && inst->src[i].type == BRW_TYPE_HF) {
         return true;
      }

      if (is_send(inst) || inst->is_control_source(i) ||
          inst->opcode == BRW_OPCODE_DPAS) {
         return false;
      }

      const unsigned dst_byte_offset = reg_offset(inst->dst) % (reg_unit(devinfo) * REG_SIZE);
      const unsigned src_byte_offset = reg_offset(inst->src[i]) % (reg_unit(devinfo) * REG_SIZE);

      return (has_dst_aligned_region_restriction(devinfo, inst) &&
              !is_uniform(inst->src[i]) &&
              (byte_stride(inst->src[i]) != required_src_byte_stride(devinfo, inst, i) ||
               src_byte_offset != dst_byte_offset)) ||
             (has_subdword_integer_region_restriction(devinfo, inst) &&
              (byte_stride(inst->src[i]) != required_src_byte_stride(devinfo, inst, i) ||
               src_byte_offset != required_src_byte_offset(devinfo, inst, i)));
   }

   /*
    * Return whether the instruction has an unsupported channel bit layout
    * specified for the destination region.
    */
   bool
   has_invalid_dst_region(const intel_device_info *devinfo,
                          const fs_inst *inst)
   {
      if (is_send(inst)) {
         return false;
      } else {
         const brw_reg_type exec_type = get_exec_type(inst);
         const unsigned dst_byte_offset = reg_offset(inst->dst) % (reg_unit(devinfo) * REG_SIZE);
         const bool is_narrowing_conversion = !is_byte_raw_mov(inst) &&
            brw_type_size_bytes(inst->dst.type) < brw_type_size_bytes(exec_type);

         return (has_dst_aligned_region_restriction(devinfo, inst) &&
                 (required_dst_byte_stride(inst) != byte_stride(inst->dst) ||
                  required_dst_byte_offset(devinfo, inst) != dst_byte_offset)) ||
                (is_narrowing_conversion &&
                 required_dst_byte_stride(inst) != byte_stride(inst->dst));
      }
   }

   /**
    * Return a non-zero value if the execution type of the instruction is
    * unsupported.  The destination and sources matching the returned mask
    * will be bit-cast to an integer type of appropriate size, lowering any
    * source or destination modifiers into separate MOV instructions.
    */
   unsigned
   has_invalid_exec_type(const intel_device_info *devinfo, const fs_inst *inst)
   {
      if (required_exec_type(devinfo, inst) != get_exec_type(inst)) {
         switch (inst->opcode) {
         case SHADER_OPCODE_SHUFFLE:
         case SHADER_OPCODE_QUAD_SWIZZLE:
         case SHADER_OPCODE_CLUSTER_BROADCAST:
         case SHADER_OPCODE_BROADCAST:
         case SHADER_OPCODE_MOV_INDIRECT:
            return 0x1;

         case SHADER_OPCODE_SEL_EXEC:
            return 0x3;

         default:
            unreachable("Unknown invalid execution type source mask.");
         }
      } else {
         return 0;
      }
   }

   /**
    * Return whether the instruction has an unsupported type conversion
    * that must be handled by expanding the source operand.
    */
   bool
   has_invalid_src_conversion(const intel_device_info *devinfo,
                              const fs_inst *inst)
   {
      /* Scalar byte to float conversion is not allowed on DG2+ */
      return devinfo->verx10 >= 125 &&
             inst->opcode == BRW_OPCODE_MOV &&
             brw_type_is_float(inst->dst.type) &&
             brw_type_size_bits(inst->src[0].type) == 8 &&
             is_uniform(inst->src[0]);
   }

   /*
    * Return whether the instruction has unsupported source modifiers
    * specified for the i-th source region.
    */
   bool
   has_invalid_src_modifiers(const intel_device_info *devinfo,
                             const fs_inst *inst, unsigned i)
   {
      return (!inst->can_do_source_mods(devinfo) &&
              (inst->src[i].negate || inst->src[i].abs)) ||
             ((has_invalid_exec_type(devinfo, inst) & (1u << i)) &&
              (inst->src[i].negate || inst->src[i].abs ||
               inst->src[i].type != get_exec_type(inst))) ||
             has_invalid_src_conversion(devinfo, inst);
   }

   /*
    * Return whether the instruction has an unsupported type conversion
    * specified for the destination.
    */
   bool
   has_invalid_conversion(const intel_device_info *devinfo, const fs_inst *inst)
   {
      switch (inst->opcode) {
      case BRW_OPCODE_MOV:
         return false;
      case BRW_OPCODE_SEL:
         return inst->dst.type != get_exec_type(inst);
      default:
         /* FIXME: We assume the opcodes not explicitly mentioned before just
          * work fine with arbitrary conversions, unless they need to be
          * bit-cast.
          */
         return has_invalid_exec_type(devinfo, inst) &&
                inst->dst.type != get_exec_type(inst);
      }
   }

   /**
    * Return whether the instruction has unsupported destination modifiers.
    */
   bool
   has_invalid_dst_modifiers(const intel_device_info *devinfo, const fs_inst *inst)
   {
      return (has_invalid_exec_type(devinfo, inst) &&
              (inst->saturate || inst->conditional_mod)) ||
             has_invalid_conversion(devinfo, inst);
   }

   /**
    * Return whether the instruction has non-standard semantics for the
    * conditional mod which don't cause the flag register to be updated with
    * the comparison result.
    */
   bool
   has_inconsistent_cmod(const fs_inst *inst)
   {
      return inst->opcode == BRW_OPCODE_SEL ||
             inst->opcode == BRW_OPCODE_CSEL ||
             inst->opcode == BRW_OPCODE_IF ||
             inst->opcode == BRW_OPCODE_WHILE;
   }

   bool
   lower_instruction(fs_visitor *v, bblock_t *block, fs_inst *inst);
}

/**
 * Remove any modifiers from the \p i-th source region of the instruction,
 * including negate, abs and any implicit type conversion to the execution
 * type.  Instead any source modifiers will be implemented as a separate
 * MOV instruction prior to the original instruction.
 */
bool
brw_lower_src_modifiers(fs_visitor &s, bblock_t *block, fs_inst *inst, unsigned i)
{
   assert(inst->components_read(i) == 1);
   assert(s.devinfo->has_integer_dword_mul ||
          inst->opcode != BRW_OPCODE_MUL ||
          brw_type_is_float(get_exec_type(inst)) ||
          MIN2(brw_type_size_bytes(inst->src[0].type), brw_type_size_bytes(inst->src[1].type)) >= 4 ||
          brw_type_size_bytes(inst->src[i].type) == get_exec_type_size(inst));

   const brw_builder ibld(&s, block, inst);
   const brw_reg tmp = ibld.vgrf(get_exec_type(inst));

   lower_instruction(&s, block, ibld.MOV(tmp, inst->src[i]));
   inst->src[i] = tmp;

   return true;
}

namespace {
   /**
    * Remove any modifiers from the destination region of the instruction,
    * including saturate, conditional mod and any implicit type conversion
    * from the execution type.  Instead any destination modifiers will be
    * implemented as a separate MOV instruction after the original
    * instruction.
    */
   bool
   lower_dst_modifiers(fs_visitor *v, bblock_t *block, fs_inst *inst)
   {
      const brw_builder ibld(v, block, inst);
      const brw_reg_type type = get_exec_type(inst);
      /* Not strictly necessary, but if possible use a temporary with the same
       * channel alignment as the current destination in order to avoid
       * violating the restrictions enforced later on by lower_src_region()
       * and lower_dst_region(), which would introduce additional copy
       * instructions into the program unnecessarily.
       */
      const unsigned stride =
         brw_type_size_bytes(inst->dst.type) * inst->dst.stride <= brw_type_size_bytes(type) ? 1 :
         brw_type_size_bytes(inst->dst.type) * inst->dst.stride / brw_type_size_bytes(type);
      brw_reg tmp = ibld.vgrf(type, stride);
      ibld.UNDEF(tmp);
      tmp = horiz_stride(tmp, stride);

      /* Emit a MOV taking care of all the destination modifiers. */
      fs_inst *mov = ibld.at(block, inst->next).MOV(inst->dst, tmp);
      mov->saturate = inst->saturate;
      if (!has_inconsistent_cmod(inst))
         mov->conditional_mod = inst->conditional_mod;
      if (inst->opcode != BRW_OPCODE_SEL) {
         mov->predicate = inst->predicate;
         mov->predicate_inverse = inst->predicate_inverse;
      }
      mov->flag_subreg = inst->flag_subreg;
      lower_instruction(v, block, mov);

      /* Point the original instruction at the temporary, and clean up any
       * destination modifiers.
       */
      assert(inst->size_written == inst->dst.component_size(inst->exec_size));
      inst->dst = tmp;
      inst->size_written = inst->dst.component_size(inst->exec_size);
      inst->saturate = false;
      if (!has_inconsistent_cmod(inst))
         inst->conditional_mod = BRW_CONDITIONAL_NONE;

      assert(!inst->flags_written(v->devinfo) || !mov->predicate);
      return true;
   }

   /**
    * Remove any non-trivial shuffling of data from the \p i-th source region
    * of the instruction.  Instead implement the region as a series of integer
    * copies into a temporary with the same channel layout as the destination.
    */
   bool
   lower_src_region(fs_visitor *v, bblock_t *block, fs_inst *inst, unsigned i)
   {
      assert(inst->components_read(i) == 1);
      const intel_device_info *devinfo = v->devinfo;
      const brw_builder ibld(v, block, inst);
      const unsigned stride = required_src_byte_stride(devinfo, inst, i) /
                              brw_type_size_bytes(inst->src[i].type);
      assert(stride > 0);
      /* Calculate the size of the temporary allocation manually instead of
       * relying on the builder, since we may have to add some amount of
       * padding mandated by the hardware for Xe2+ instructions with sub-dword
       * integer regions.
       */
      const unsigned size =
         DIV_ROUND_UP(required_src_byte_offset(v->devinfo, inst, i) +
                      inst->exec_size * stride *
                      brw_type_size_bytes(inst->src[i].type),
                      reg_unit(devinfo) * REG_SIZE) * reg_unit(devinfo);
      brw_reg tmp = brw_vgrf(v->alloc.allocate(size), inst->src[i].type);
      ibld.UNDEF(tmp);
      tmp = byte_offset(horiz_stride(tmp, stride),
                        required_src_byte_offset(devinfo, inst, i));

      /* Emit a series of 32-bit integer copies with any source modifiers
       * cleaned up (because their semantics are dependent on the type).
       */
      const brw_reg_type raw_type = brw_int_type(MIN2(brw_type_size_bytes(tmp.type), 4),
                                                 false);
      const unsigned n = brw_type_size_bytes(tmp.type) / brw_type_size_bytes(raw_type);
      brw_reg raw_src = inst->src[i];
      raw_src.negate = false;
      raw_src.abs = false;

      for (unsigned j = 0; j < n; j++) {
	fs_inst *jnst = ibld.MOV(subscript(tmp, raw_type, j),
				 subscript(raw_src, raw_type, j));
	if (has_subdword_integer_region_restriction(devinfo, jnst)) {
           /* The copy isn't guaranteed to comply with all subdword integer
            * regioning restrictions in some cases.  Lower it recursively.
            */
	   lower_instruction(v, block, jnst);
        }
      }

      /* Point the original instruction at the temporary, making sure to keep
       * any source modifiers in the instruction.
       */
      brw_reg lower_src = tmp;
      lower_src.negate = inst->src[i].negate;
      lower_src.abs = inst->src[i].abs;
      inst->src[i] = lower_src;

      return true;
   }

   /**
    * Remove any non-trivial shuffling of data from the destination region of
    * the instruction.  Instead implement the region as a series of integer
    * copies from a temporary with a channel layout compatible with the
    * sources.
    */
   bool
   lower_dst_region(fs_visitor *v, bblock_t *block, fs_inst *inst)
   {
      /* We cannot replace the result of an integer multiply which writes the
       * accumulator because MUL+MACH pairs act on the accumulator as a 66-bit
       * value whereas the MOV will act on only 32 or 33 bits of the
       * accumulator.
       */
      assert(inst->opcode != BRW_OPCODE_MUL || !inst->dst.is_accumulator() ||
             brw_type_is_float(inst->dst.type));

      const brw_builder ibld(v, block, inst);
      const unsigned stride = required_dst_byte_stride(inst) /
                              brw_type_size_bytes(inst->dst.type);
      assert(stride > 0);
      brw_reg tmp = ibld.vgrf(inst->dst.type, stride);
      ibld.UNDEF(tmp);
      tmp = horiz_stride(tmp, stride);

      if (!inst->dst.is_null()) {
         /* Emit a series of 32-bit integer copies from the temporary into the
          * original destination.
          */
         const brw_reg_type raw_type =
            brw_int_type(MIN2(brw_type_size_bytes(tmp.type), 4), false);

         const unsigned n =
            brw_type_size_bytes(tmp.type) / brw_type_size_bytes(raw_type);

         if (inst->predicate && inst->opcode != BRW_OPCODE_SEL) {
            /* Note that in general we cannot simply predicate the copies on
             * the same flag register as the original instruction, since it
             * may have been overwritten by the instruction itself.  Instead
             * initialize the temporary with the previous contents of the
             * destination register.
             */
            for (unsigned j = 0; j < n; j++)
               ibld.MOV(subscript(tmp, raw_type, j),
                        subscript(inst->dst, raw_type, j));
         }

         for (unsigned j = 0; j < n; j++) {
            fs_inst *jnst = ibld.at(block, inst->next).MOV(subscript(inst->dst, raw_type, j),
                                                           subscript(tmp, raw_type, j));
            if (has_subdword_integer_region_restriction(v->devinfo, jnst)) {
               /* The copy isn't guaranteed to comply with all subdword integer
                * regioning restrictions in some cases.  Lower it recursively.
                */
               lower_instruction(v, block, jnst);
            }
         }

         /* If the destination was an accumulator, after lowering it will be a
          * GRF. Clear writes_accumulator for the instruction.
          */
         if (inst->dst.is_accumulator())
            inst->writes_accumulator = false;
      }

      /* Point the original instruction at the temporary, making sure to keep
       * any destination modifiers in the instruction.
       */
      assert(inst->size_written == inst->dst.component_size(inst->exec_size));
      inst->dst = tmp;
      inst->size_written = inst->dst.component_size(inst->exec_size);

      return true;
   }

   /**
    * Change sources and destination of the instruction to an
    * appropriate legal type, splitting the instruction into multiple
    * ones of smaller execution type if necessary, to be used in cases
    * where the execution type of an instruction is unsupported.
    */
   bool
   lower_exec_type(fs_visitor *v, bblock_t *block, fs_inst *inst)
   {
      assert(inst->dst.type == get_exec_type(inst));
      const unsigned mask = has_invalid_exec_type(v->devinfo, inst);
      const brw_reg_type raw_type = required_exec_type(v->devinfo, inst);
      const unsigned n = get_exec_type_size(inst) / brw_type_size_bytes(raw_type);
      const brw_builder ibld(v, block, inst);

      brw_reg tmp = ibld.vgrf(inst->dst.type, inst->dst.stride);
      ibld.UNDEF(tmp);
      tmp = horiz_stride(tmp, inst->dst.stride);

      for (unsigned j = 0; j < n; j++) {
         fs_inst sub_inst = *inst;

         for (unsigned i = 0; i < inst->sources; i++) {
            if (mask & (1u << i)) {
               assert(inst->src[i].type == inst->dst.type);
               sub_inst.src[i] = subscript(inst->src[i], raw_type, j);
            }
         }

         sub_inst.dst = subscript(tmp, raw_type, j);

         assert(sub_inst.size_written == sub_inst.dst.component_size(sub_inst.exec_size));
         assert(!sub_inst.flags_written(v->devinfo) && !sub_inst.saturate);
         ibld.emit(sub_inst);

         fs_inst *mov = ibld.MOV(subscript(inst->dst, raw_type, j),
                                 subscript(tmp, raw_type, j));
         if (inst->opcode != BRW_OPCODE_SEL) {
            mov->predicate = inst->predicate;
            mov->predicate_inverse = inst->predicate_inverse;
         }
         lower_instruction(v, block, mov);
      }

      inst->remove(block);

      return true;
   }

   /**
    * Fast-path for very specific kinds of invalid regions.
    *
    * Gfx12.5+ does not allow moves of B or UB sources to floating-point
    * destinations. This restriction can be resolved more efficiently than by
    * the general lowering in lower_src_modifiers or lower_src_region.
    */
   void
   lower_src_conversion(fs_visitor *v, bblock_t *block, fs_inst *inst)
   {
      const intel_device_info *devinfo = v->devinfo;
      const brw_builder ibld = brw_builder(v, block, inst).scalar_group();

      /* We only handle scalar conversions from small types for now. */
      assert(is_uniform(inst->src[0]));

      brw_reg tmp = ibld.vgrf(brw_type_with_size(inst->src[0].type, 32));
      fs_inst *mov = ibld.MOV(tmp, inst->src[0]);

      inst->src[0] = component(tmp, 0);

      /* Assert that neither the added MOV nor the original instruction will need
       * any additional lowering.
       */
      assert(!has_invalid_src_region(devinfo, mov, 0));
      assert(!has_invalid_src_modifiers(devinfo, mov, 0));
      assert(!has_invalid_dst_region(devinfo, mov));

      assert(!has_invalid_src_region(devinfo, inst, 0));
      assert(!has_invalid_src_modifiers(devinfo, inst, 0));
   }

   /**
    * Legalize the source and destination regioning controls of the specified
    * instruction.
    */
   bool
   lower_instruction(fs_visitor *v, bblock_t *block, fs_inst *inst)
   {
      const intel_device_info *devinfo = v->devinfo;
      bool progress = false;

      /* BROADCAST is special. It's destination region is a bit of a lie, and
       * it gets lower in brw_eu_emit. For the purposes of region
       * restrictions, let's assume that the final code emission will do the
       * right thing. Doing a bunch of shuffling here is only going to make a
       * mess of things.
       */
      if (inst->opcode == SHADER_OPCODE_BROADCAST)
         return false;

      if (has_invalid_dst_modifiers(devinfo, inst))
         progress |= lower_dst_modifiers(v, block, inst);

      if (has_invalid_dst_region(devinfo, inst))
         progress |= lower_dst_region(v, block, inst);

      if (has_invalid_src_conversion(devinfo, inst)) {
         lower_src_conversion(v, block, inst);
         progress = true;
      }

      for (unsigned i = 0; i < inst->sources; i++) {
         if (has_invalid_src_modifiers(devinfo, inst, i))
            progress |= brw_lower_src_modifiers(*v, block, inst, i);

         if (has_invalid_src_region(devinfo, inst, i))
            progress |= lower_src_region(v, block, inst, i);
      }

      if (has_invalid_exec_type(devinfo, inst))
         progress |= lower_exec_type(v, block, inst);

      return progress;
   }
}

bool
brw_lower_regioning(fs_visitor &s)
{
   bool progress = false;

   foreach_block_and_inst_safe(block, fs_inst, inst, s.cfg)
      progress |= lower_instruction(&s, block, inst);

   if (progress)
      s.invalidate_analysis(DEPENDENCY_INSTRUCTIONS | DEPENDENCY_VARIABLES);

   return progress;
}