source/opt/upgrade_memory_model.cpp

// Copyright (c) 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "upgrade_memory_model.h"

#include <utility>

#include "source/opt/ir_builder.h"
#include "source/opt/ir_context.h"
#include "source/spirv_constant.h"
#include "source/util/make_unique.h"
#include "source/util/string_utils.h"

namespace spvtools {
namespace opt {

Pass::Status UpgradeMemoryModel::Process() {
  // TODO: This pass needs changes to support cooperative matrices.
  if (context()->get_feature_mgr()->HasCapability(
          spv::Capability::CooperativeMatrixNV)) {
    return Pass::Status::SuccessWithoutChange;
  }

  // Only update Logical GLSL450 to Logical VulkanKHR.
  Instruction* memory_model = get_module()->GetMemoryModel();
  if (memory_model->GetSingleWordInOperand(0u) !=
          uint32_t(spv::AddressingModel::Logical) ||
      memory_model->GetSingleWordInOperand(1u) !=
          uint32_t(spv::MemoryModel::GLSL450)) {
    return Pass::Status::SuccessWithoutChange;
  }

  UpgradeMemoryModelInstruction();
  UpgradeInstructions();
  CleanupDecorations();
  UpgradeBarriers();
  UpgradeMemoryScope();

  return Pass::Status::SuccessWithChange;
}

void UpgradeMemoryModel::UpgradeMemoryModelInstruction() {
  // Overall changes necessary:
  // 1. Add the OpExtension.
  // 2. Add the OpCapability.
  // 3. Modify the memory model.
  Instruction* memory_model = get_module()->GetMemoryModel();
  context()->AddCapability(MakeUnique<Instruction>(
      context(), spv::Op::OpCapability, 0, 0,
      std::initializer_list<Operand>{
          {SPV_OPERAND_TYPE_CAPABILITY,
           {uint32_t(spv::Capability::VulkanMemoryModelKHR)}}}));
  const std::string extension = "SPV_KHR_vulkan_memory_model";
  std::vector<uint32_t> words = spvtools::utils::MakeVector(extension);
  context()->AddExtension(
      MakeUnique<Instruction>(context(), spv::Op::OpExtension, 0, 0,
                              std::initializer_list<Operand>{
                                  {SPV_OPERAND_TYPE_LITERAL_STRING, words}}));
  memory_model->SetInOperand(1u, {uint32_t(spv::MemoryModel::VulkanKHR)});
}

void UpgradeMemoryModel::UpgradeInstructions() {
  // Coherent and Volatile decorations are deprecated. Remove them and replace
  // with flags on the memory/image operations. The decorations can occur on
  // OpVariable, OpFunctionParameter (of pointer type) and OpStructType (member
  // decoration). Trace from the decoration target(s) to the final memory/image
  // instructions. Additionally, Workgroup storage class variables and function
  // parameters are implicitly coherent in GLSL450.

  // Upgrade modf and frexp first since they generate new stores.
  // In SPIR-V 1.4 or later, normalize OpCopyMemory* access operands.
  for (auto& func : *get_module()) {
    func.ForEachInst([this](Instruction* inst) {
      if (inst->opcode() == spv::Op::OpExtInst) {
        auto ext_inst = inst->GetSingleWordInOperand(1u);
        if (ext_inst == GLSLstd450Modf || ext_inst == GLSLstd450Frexp) {
          auto import =
              get_def_use_mgr()->GetDef(inst->GetSingleWordInOperand(0u));
          if (import->GetInOperand(0u).AsString() == "GLSL.std.450") {
            UpgradeExtInst(inst);
          }
        }
      } else if (get_module()->version() >= SPV_SPIRV_VERSION_WORD(1, 4)) {
        if (inst->opcode() == spv::Op::OpCopyMemory ||
            inst->opcode() == spv::Op::OpCopyMemorySized) {
          uint32_t start_operand =
              inst->opcode() == spv::Op::OpCopyMemory ? 2u : 3u;
          if (inst->NumInOperands() > start_operand) {
            auto num_access_words = MemoryAccessNumWords(
                inst->GetSingleWordInOperand(start_operand));
            if ((num_access_words + start_operand) == inst->NumInOperands()) {
              // There is a single memory access operand. Duplicate it to have a
              // separate operand for both source and target.
              for (uint32_t i = 0; i < num_access_words; ++i) {
                auto operand = inst->GetInOperand(start_operand + i);
                inst->AddOperand(std::move(operand));
              }
            }
          } else {
            // Add two memory access operands.
            inst->AddOperand({SPV_OPERAND_TYPE_MEMORY_ACCESS,
                              {uint32_t(spv::MemoryAccessMask::MaskNone)}});
            inst->AddOperand({SPV_OPERAND_TYPE_MEMORY_ACCESS,
                              {uint32_t(spv::MemoryAccessMask::MaskNone)}});
          }
        }
      }
    });
  }

  UpgradeMemoryAndImages();
  UpgradeAtomics();
}

void UpgradeMemoryModel::UpgradeMemoryAndImages() {
  for (auto& func : *get_module()) {
    func.ForEachInst([this](Instruction* inst) {
      bool is_coherent = false;
      bool is_volatile = false;
      bool src_coherent = false;
      bool src_volatile = false;
      bool dst_coherent = false;
      bool dst_volatile = false;
      uint32_t start_operand = 0u;
      spv::Scope scope = spv::Scope::QueueFamilyKHR;
      spv::Scope src_scope = spv::Scope::QueueFamilyKHR;
      spv::Scope dst_scope = spv::Scope::QueueFamilyKHR;
      switch (inst->opcode()) {
        case spv::Op::OpLoad:
        case spv::Op::OpStore:
          std::tie(is_coherent, is_volatile, scope) =
              GetInstructionAttributes(inst->GetSingleWordInOperand(0u));
          break;
        case spv::Op::OpImageRead:
        case spv::Op::OpImageSparseRead:
        case spv::Op::OpImageWrite:
          std::tie(is_coherent, is_volatile, scope) =
              GetInstructionAttributes(inst->GetSingleWordInOperand(0u));
          break;
        case spv::Op::OpCopyMemory:
        case spv::Op::OpCopyMemorySized:
          std::tie(dst_coherent, dst_volatile, dst_scope) =
              GetInstructionAttributes(inst->GetSingleWordInOperand(0u));
          std::tie(src_coherent, src_volatile, src_scope) =
              GetInstructionAttributes(inst->GetSingleWordInOperand(1u));
          break;
        default:
          break;
      }

      switch (inst->opcode()) {
        case spv::Op::OpLoad:
          UpgradeFlags(inst, 1u, is_coherent, is_volatile, kVisibility,
                       kMemory);
          break;
        case spv::Op::OpStore:
          UpgradeFlags(inst, 2u, is_coherent, is_volatile, kAvailability,
                       kMemory);
          break;
        case spv::Op::OpCopyMemory:
        case spv::Op::OpCopyMemorySized:
          start_operand = inst->opcode() == spv::Op::OpCopyMemory ? 2u : 3u;
          if (get_module()->version() >= SPV_SPIRV_VERSION_WORD(1, 4)) {
            // There are guaranteed to be two memory access operands at this
            // point so treat source and target separately.
            uint32_t num_access_words = MemoryAccessNumWords(
                inst->GetSingleWordInOperand(start_operand));
            UpgradeFlags(inst, start_operand, dst_coherent, dst_volatile,
                         kAvailability, kMemory);
            UpgradeFlags(inst, start_operand + num_access_words, src_coherent,
                         src_volatile, kVisibility, kMemory);
          } else {
            UpgradeFlags(inst, start_operand, dst_coherent, dst_volatile,
                         kAvailability, kMemory);
            UpgradeFlags(inst, start_operand, src_coherent, src_volatile,
                         kVisibility, kMemory);
          }
          break;
        case spv::Op::OpImageRead:
        case spv::Op::OpImageSparseRead:
          UpgradeFlags(inst, 2u, is_coherent, is_volatile, kVisibility, kImage);
          break;
        case spv::Op::OpImageWrite:
          UpgradeFlags(inst, 3u, is_coherent, is_volatile, kAvailability,
                       kImage);
          break;
        default:
          break;
      }

      // |is_coherent| is never used for the same instructions as
      // |src_coherent| and |dst_coherent|.
      if (is_coherent) {
        inst->AddOperand(
            {SPV_OPERAND_TYPE_SCOPE_ID, {GetScopeConstant(scope)}});
      }
      if (get_module()->version() >= SPV_SPIRV_VERSION_WORD(1, 4)) {
        // There are two memory access operands. The first is for the target and
        // the second is for the source.
        if (dst_coherent || src_coherent) {
          start_operand = inst->opcode() == spv::Op::OpCopyMemory ? 2u : 3u;
          std::vector<Operand> new_operands;
          uint32_t num_access_words =
              MemoryAccessNumWords(inst->GetSingleWordInOperand(start_operand));
          // The flags were already updated so subtract if we're adding a
          // scope.
          if (dst_coherent) --num_access_words;
          for (uint32_t i = 0; i < start_operand + num_access_words; ++i) {
            new_operands.push_back(inst->GetInOperand(i));
          }
          // Add the target scope if necessary.
          if (dst_coherent) {
            new_operands.push_back(
                {SPV_OPERAND_TYPE_SCOPE_ID, {GetScopeConstant(dst_scope)}});
          }
          // Copy the remaining current operands.
          for (uint32_t i = start_operand + num_access_words;
               i < inst->NumInOperands(); ++i) {
            new_operands.push_back(inst->GetInOperand(i));
          }
          // Add the source scope if necessary.
          if (src_coherent) {
            new_operands.push_back(
                {SPV_OPERAND_TYPE_SCOPE_ID, {GetScopeConstant(src_scope)}});
          }
          inst->SetInOperands(std::move(new_operands));
        }
      } else {
        // According to SPV_KHR_vulkan_memory_model, if both available and
        // visible flags are used the first scope operand is for availability
        // (writes) and the second is for visibility (reads).
        if (dst_coherent) {
          inst->AddOperand(
              {SPV_OPERAND_TYPE_SCOPE_ID, {GetScopeConstant(dst_scope)}});
        }
        if (src_coherent) {
          inst->AddOperand(
              {SPV_OPERAND_TYPE_SCOPE_ID, {GetScopeConstant(src_scope)}});
        }
      }
    });
  }
}

void UpgradeMemoryModel::UpgradeAtomics() {
  for (auto& func : *get_module()) {
    func.ForEachInst([this](Instruction* inst) {
      if (spvOpcodeIsAtomicOp(inst->opcode())) {
        bool unused_coherent = false;
        bool is_volatile = false;
        spv::Scope unused_scope = spv::Scope::QueueFamilyKHR;
        std::tie(unused_coherent, is_volatile, unused_scope) =
            GetInstructionAttributes(inst->GetSingleWordInOperand(0));

        UpgradeSemantics(inst, 2u, is_volatile);
        if (inst->opcode() == spv::Op::OpAtomicCompareExchange ||
            inst->opcode() == spv::Op::OpAtomicCompareExchangeWeak) {
          UpgradeSemantics(inst, 3u, is_volatile);
        }
      }
    });
  }
}

void UpgradeMemoryModel::UpgradeSemantics(Instruction* inst,
                                          uint32_t in_operand,
                                          bool is_volatile) {
  if (!is_volatile) return;

  uint32_t semantics_id = inst->GetSingleWordInOperand(in_operand);
  const analysis::Constant* constant =
      context()->get_constant_mgr()->FindDeclaredConstant(semantics_id);
  const analysis::Integer* type = constant->type()->AsInteger();
  assert(type && type->width() == 32);
  uint32_t value = 0;
  if (type->IsSigned()) {
    value = static_cast<uint32_t>(constant->GetS32());
  } else {
    value = constant->GetU32();
  }

  value |= uint32_t(spv::MemorySemanticsMask::Volatile);
  auto new_constant = context()->get_constant_mgr()->GetConstant(type, {value});
  auto new_semantics =
      context()->get_constant_mgr()->GetDefiningInstruction(new_constant);
  inst->SetInOperand(in_operand, {new_semantics->result_id()});
}

std::tuple<bool, bool, spv::Scope> UpgradeMemoryModel::GetInstructionAttributes(
    uint32_t id) {
  // |id| is a pointer used in a memory/image instruction. Need to determine if
  // that pointer points to volatile or coherent memory. Workgroup storage
  // class is implicitly coherent and cannot be decorated with volatile, so
  // short circuit that case.
  Instruction* inst = context()->get_def_use_mgr()->GetDef(id);
  analysis::Type* type = context()->get_type_mgr()->GetType(inst->type_id());
  if (type->AsPointer() &&
      type->AsPointer()->storage_class() == spv::StorageClass::Workgroup) {
    return std::make_tuple(true, false, spv::Scope::Workgroup);
  }

  bool is_coherent = false;
  bool is_volatile = false;
  std::unordered_set<uint32_t> visited;
  std::tie(is_coherent, is_volatile) =
      TraceInstruction(context()->get_def_use_mgr()->GetDef(id),
                       std::vector<uint32_t>(), &visited);

  return std::make_tuple(is_coherent, is_volatile, spv::Scope::QueueFamilyKHR);
}

std::pair<bool, bool> UpgradeMemoryModel::TraceInstruction(
    Instruction* inst, std::vector<uint32_t> indices,
    std::unordered_set<uint32_t>* visited) {
  auto iter = cache_.find(std::make_pair(inst->result_id(), indices));
  if (iter != cache_.end()) {
    return iter->second;
  }

  if (!visited->insert(inst->result_id()).second) {
    return std::make_pair(false, false);
  }

  // Initialize the cache before |indices| is (potentially) modified.
  auto& cached_result = cache_[std::make_pair(inst->result_id(), indices)];
  cached_result.first = false;
  cached_result.second = false;

  bool is_coherent = false;
  bool is_volatile = false;
  switch (inst->opcode()) {
    case spv::Op::OpVariable:
    case spv::Op::OpFunctionParameter:
      is_coherent |= HasDecoration(inst, 0, spv::Decoration::Coherent);
      is_volatile |= HasDecoration(inst, 0, spv::Decoration::Volatile);
      if (!is_coherent || !is_volatile) {
        bool type_coherent = false;
        bool type_volatile = false;
        std::tie(type_coherent, type_volatile) =
            CheckType(inst->type_id(), indices);
        is_coherent |= type_coherent;
        is_volatile |= type_volatile;
      }
      break;
    case spv::Op::OpAccessChain:
    case spv::Op::OpInBoundsAccessChain:
      // Store indices in reverse order.
      for (uint32_t i = inst->NumInOperands() - 1; i > 0; --i) {
        indices.push_back(inst->GetSingleWordInOperand(i));
      }
      break;
    case spv::Op::OpPtrAccessChain:
      // Store indices in reverse order. Skip the |Element| operand.
      for (uint32_t i = inst->NumInOperands() - 1; i > 1; --i) {
        indices.push_back(inst->GetSingleWordInOperand(i));
      }
      break;
    default:
      break;
  }

  // No point searching further.
  if (is_coherent && is_volatile) {
    cached_result.first = true;
    cached_result.second = true;
    return std::make_pair(true, true);
  }

  // Variables and function parameters are sources. Continue searching until we
  // reach them.
  if (inst->opcode() != spv::Op::OpVariable &&
      inst->opcode() != spv::Op::OpFunctionParameter) {
    inst->ForEachInId([this, &is_coherent, &is_volatile, &indices,
                       &visited](const uint32_t* id_ptr) {
      Instruction* op_inst = context()->get_def_use_mgr()->GetDef(*id_ptr);
      const analysis::Type* type =
          context()->get_type_mgr()->GetType(op_inst->type_id());
      if (type &&
          (type->AsPointer() || type->AsImage() || type->AsSampledImage())) {
        bool operand_coherent = false;
        bool operand_volatile = false;
        std::tie(operand_coherent, operand_volatile) =
            TraceInstruction(op_inst, indices, visited);
        is_coherent |= operand_coherent;
        is_volatile |= operand_volatile;
      }
    });
  }

  cached_result.first = is_coherent;
  cached_result.second = is_volatile;
  return std::make_pair(is_coherent, is_volatile);
}

std::pair<bool, bool> UpgradeMemoryModel::CheckType(
    uint32_t type_id, const std::vector<uint32_t>& indices) {
  bool is_coherent = false;
  bool is_volatile = false;
  Instruction* type_inst = context()->get_def_use_mgr()->GetDef(type_id);
  assert(type_inst->opcode() == spv::Op::OpTypePointer);
  Instruction* element_inst = context()->get_def_use_mgr()->GetDef(
      type_inst->GetSingleWordInOperand(1u));
  for (int i = (int)indices.size() - 1; i >= 0; --i) {
    if (is_coherent && is_volatile) break;

    if (element_inst->opcode() == spv::Op::OpTypePointer) {
      element_inst = context()->get_def_use_mgr()->GetDef(
          element_inst->GetSingleWordInOperand(1u));
    } else if (element_inst->opcode() == spv::Op::OpTypeStruct) {
      uint32_t index = indices.at(i);
      Instruction* index_inst = context()->get_def_use_mgr()->GetDef(index);
      assert(index_inst->opcode() == spv::Op::OpConstant);
      uint64_t value = GetIndexValue(index_inst);
      is_coherent |= HasDecoration(element_inst, static_cast<uint32_t>(value),
                                   spv::Decoration::Coherent);
      is_volatile |= HasDecoration(element_inst, static_cast<uint32_t>(value),
                                   spv::Decoration::Volatile);
      element_inst = context()->get_def_use_mgr()->GetDef(
          element_inst->GetSingleWordInOperand(static_cast<uint32_t>(value)));
    } else {
      assert(spvOpcodeIsComposite(element_inst->opcode()));
      element_inst = context()->get_def_use_mgr()->GetDef(
          element_inst->GetSingleWordInOperand(0u));
    }
  }

  if (!is_coherent || !is_volatile) {
    bool remaining_coherent = false;
    bool remaining_volatile = false;
    std::tie(remaining_coherent, remaining_volatile) =
        CheckAllTypes(element_inst);
    is_coherent |= remaining_coherent;
    is_volatile |= remaining_volatile;
  }

  return std::make_pair(is_coherent, is_volatile);
}

std::pair<bool, bool> UpgradeMemoryModel::CheckAllTypes(
    const Instruction* inst) {
  std::unordered_set<const Instruction*> visited;
  std::vector<const Instruction*> stack;
  stack.push_back(inst);

  bool is_coherent = false;
  bool is_volatile = false;
  while (!stack.empty()) {
    const Instruction* def = stack.back();
    stack.pop_back();

    if (!visited.insert(def).second) continue;

    if (def->opcode() == spv::Op::OpTypeStruct) {
      // Any member decorated with coherent and/or volatile is enough to have
      // the related operation be flagged as coherent and/or volatile.
      is_coherent |= HasDecoration(def, std::numeric_limits<uint32_t>::max(),
                                   spv::Decoration::Coherent);
      is_volatile |= HasDecoration(def, std::numeric_limits<uint32_t>::max(),
                                   spv::Decoration::Volatile);
      if (is_coherent && is_volatile)
        return std::make_pair(is_coherent, is_volatile);

      // Check the subtypes.
      for (uint32_t i = 0; i < def->NumInOperands(); ++i) {
        stack.push_back(context()->get_def_use_mgr()->GetDef(
            def->GetSingleWordInOperand(i)));
      }
    } else if (spvOpcodeIsComposite(def->opcode())) {
      stack.push_back(context()->get_def_use_mgr()->GetDef(
          def->GetSingleWordInOperand(0u)));
    } else if (def->opcode() == spv::Op::OpTypePointer) {
      stack.push_back(context()->get_def_use_mgr()->GetDef(
          def->GetSingleWordInOperand(1u)));
    }
  }

  return std::make_pair(is_coherent, is_volatile);
}

uint64_t UpgradeMemoryModel::GetIndexValue(Instruction* index_inst) {
  const analysis::Constant* index_constant =
      context()->get_constant_mgr()->GetConstantFromInst(index_inst);
  assert(index_constant->AsIntConstant());
  if (index_constant->type()->AsInteger()->IsSigned()) {
    if (index_constant->type()->AsInteger()->width() == 32) {
      return index_constant->GetS32();
    } else {
      return index_constant->GetS64();
    }
  } else {
    if (index_constant->type()->AsInteger()->width() == 32) {
      return index_constant->GetU32();
    } else {
      return index_constant->GetU64();
    }
  }
}

bool UpgradeMemoryModel::HasDecoration(const Instruction* inst, uint32_t value,
                                       spv::Decoration decoration) {
  // If the iteration was terminated early then an appropriate decoration was
  // found.
  return !context()->get_decoration_mgr()->WhileEachDecoration(
      inst->result_id(), (uint32_t)decoration, [value](const Instruction& i) {
        if (i.opcode() == spv::Op::OpDecorate ||
            i.opcode() == spv::Op::OpDecorateId) {
          return false;
        } else if (i.opcode() == spv::Op::OpMemberDecorate) {
          if (value == i.GetSingleWordInOperand(1u) ||
              value == std::numeric_limits<uint32_t>::max())
            return false;
        }

        return true;
      });
}

void UpgradeMemoryModel::UpgradeFlags(Instruction* inst, uint32_t in_operand,
                                      bool is_coherent, bool is_volatile,
                                      OperationType operation_type,
                                      InstructionType inst_type) {
  if (!is_coherent && !is_volatile) return;

  uint32_t flags = 0;
  if (inst->NumInOperands() > in_operand) {
    flags |= inst->GetSingleWordInOperand(in_operand);
  }
  if (is_coherent) {
    if (inst_type == kMemory) {
      flags |= uint32_t(spv::MemoryAccessMask::NonPrivatePointerKHR);
      if (operation_type == kVisibility) {
        flags |= uint32_t(spv::MemoryAccessMask::MakePointerVisibleKHR);
      } else {
        flags |= uint32_t(spv::MemoryAccessMask::MakePointerAvailableKHR);
      }
    } else {
      flags |= uint32_t(spv::ImageOperandsMask::NonPrivateTexelKHR);
      if (operation_type == kVisibility) {
        flags |= uint32_t(spv::ImageOperandsMask::MakeTexelVisibleKHR);
      } else {
        flags |= uint32_t(spv::ImageOperandsMask::MakeTexelAvailableKHR);
      }
    }
  }

  if (is_volatile) {
    if (inst_type == kMemory) {
      flags |= uint32_t(spv::MemoryAccessMask::Volatile);
    } else {
      flags |= uint32_t(spv::ImageOperandsMask::VolatileTexelKHR);
    }
  }

  if (inst->NumInOperands() > in_operand) {
    inst->SetInOperand(in_operand, {flags});
  } else if (inst_type == kMemory) {
    inst->AddOperand({SPV_OPERAND_TYPE_OPTIONAL_MEMORY_ACCESS, {flags}});
  } else {
    inst->AddOperand({SPV_OPERAND_TYPE_OPTIONAL_IMAGE, {flags}});
  }
}

uint32_t UpgradeMemoryModel::GetScopeConstant(spv::Scope scope) {
  analysis::Integer int_ty(32, false);
  uint32_t int_id = context()->get_type_mgr()->GetTypeInstruction(&int_ty);
  const analysis::Constant* constant =
      context()->get_constant_mgr()->GetConstant(
          context()->get_type_mgr()->GetType(int_id),
          {static_cast<uint32_t>(scope)});
  return context()
      ->get_constant_mgr()
      ->GetDefiningInstruction(constant)
      ->result_id();
}

void UpgradeMemoryModel::CleanupDecorations() {
  // All of the volatile and coherent decorations have been dealt with, so now
  // we can just remove them.
  get_module()->ForEachInst([this](Instruction* inst) {
    if (inst->result_id() != 0) {
      context()->get_decoration_mgr()->RemoveDecorationsFrom(
          inst->result_id(), [](const Instruction& dec) {
            switch (dec.opcode()) {
              case spv::Op::OpDecorate:
              case spv::Op::OpDecorateId:
                if (spv::Decoration(dec.GetSingleWordInOperand(1u)) ==
                        spv::Decoration::Coherent ||
                    spv::Decoration(dec.GetSingleWordInOperand(1u)) ==
                        spv::Decoration::Volatile)
                  return true;
                break;
              case spv::Op::OpMemberDecorate:
                if (spv::Decoration(dec.GetSingleWordInOperand(2u)) ==
                        spv::Decoration::Coherent ||
                    spv::Decoration(dec.GetSingleWordInOperand(2u)) ==
                        spv::Decoration::Volatile)
                  return true;
                break;
              default:
                break;
            }
            return false;
          });
    }
  });
}

void UpgradeMemoryModel::UpgradeBarriers() {
  std::vector<Instruction*> barriers;
  // Collects all the control barriers in |function|. Returns true if the
  // function operates on the Output storage class.
  ProcessFunction CollectBarriers = [this, &barriers](Function* function) {
    bool operates_on_output = false;
    for (auto& block : *function) {
      block.ForEachInst([this, &barriers,
                         &operates_on_output](Instruction* inst) {
        if (inst->opcode() == spv::Op::OpControlBarrier) {
          barriers.push_back(inst);
        } else if (!operates_on_output) {
          // This instruction operates on output storage class if it is a
          // pointer to output type or any input operand is a pointer to output
          // type.
          analysis::Type* type =
              context()->get_type_mgr()->GetType(inst->type_id());
          if (type && type->AsPointer() &&
              type->AsPointer()->storage_class() == spv::StorageClass::Output) {
            operates_on_output = true;
            return;
          }
          inst->ForEachInId([this, &operates_on_output](uint32_t* id_ptr) {
            Instruction* op_inst =
                context()->get_def_use_mgr()->GetDef(*id_ptr);
            analysis::Type* op_type =
                context()->get_type_mgr()->GetType(op_inst->type_id());
            if (op_type && op_type->AsPointer() &&
                op_type->AsPointer()->storage_class() ==
                    spv::StorageClass::Output)
              operates_on_output = true;
          });
        }
      });
    }
    return operates_on_output;
  };

  std::queue<uint32_t> roots;
  for (auto& e : get_module()->entry_points())
    if (spv::ExecutionModel(e.GetSingleWordInOperand(0u)) ==
        spv::ExecutionModel::TessellationControl) {
      roots.push(e.GetSingleWordInOperand(1u));
      if (context()->ProcessCallTreeFromRoots(CollectBarriers, &roots)) {
        for (auto barrier : barriers) {
          // Add OutputMemoryKHR to the semantics of the barriers.
          uint32_t semantics_id = barrier->GetSingleWordInOperand(2u);
          Instruction* semantics_inst =
              context()->get_def_use_mgr()->GetDef(semantics_id);
          analysis::Type* semantics_type =
              context()->get_type_mgr()->GetType(semantics_inst->type_id());
          uint64_t semantics_value = GetIndexValue(semantics_inst);
          const analysis::Constant* constant =
              context()->get_constant_mgr()->GetConstant(
                  semantics_type,
                  {static_cast<uint32_t>(semantics_value) |
                   uint32_t(spv::MemorySemanticsMask::OutputMemoryKHR)});
          barrier->SetInOperand(2u, {context()
                                         ->get_constant_mgr()
                                         ->GetDefiningInstruction(constant)
                                         ->result_id()});
        }
      }
      barriers.clear();
    }
}

void UpgradeMemoryModel::UpgradeMemoryScope() {
  get_module()->ForEachInst([this](Instruction* inst) {
    // Don't need to handle all the operations that take a scope.
    // * Group operations can only be subgroup
    // * Non-uniform can only be workgroup or subgroup
    // * Named barriers are not supported by Vulkan
    // * Workgroup ops (e.g. async_copy) have at most workgroup scope.
    if (spvOpcodeIsAtomicOp(inst->opcode())) {
      if (IsDeviceScope(inst->GetSingleWordInOperand(1))) {
        inst->SetInOperand(1, {GetScopeConstant(spv::Scope::QueueFamilyKHR)});
      }
    } else if (inst->opcode() == spv::Op::OpControlBarrier) {
      if (IsDeviceScope(inst->GetSingleWordInOperand(1))) {
        inst->SetInOperand(1, {GetScopeConstant(spv::Scope::QueueFamilyKHR)});
      }
    } else if (inst->opcode() == spv::Op::OpMemoryBarrier) {
      if (IsDeviceScope(inst->GetSingleWordInOperand(0))) {
        inst->SetInOperand(0, {GetScopeConstant(spv::Scope::QueueFamilyKHR)});
      }
    }
  });
}

bool UpgradeMemoryModel::IsDeviceScope(uint32_t scope_id) {
  const analysis::Constant* constant =
      context()->get_constant_mgr()->FindDeclaredConstant(scope_id);
  assert(constant && "Memory scope must be a constant");

  const analysis::Integer* type = constant->type()->AsInteger();
  assert(type);
  assert(type->width() == 32 || type->width() == 64);
  if (type->width() == 32) {
    if (type->IsSigned())
      return static_cast<spv::Scope>(constant->GetS32()) == spv::Scope::Device;
    else
      return static_cast<spv::Scope>(constant->GetU32()) == spv::Scope::Device;
  } else {
    if (type->IsSigned())
      return static_cast<spv::Scope>(constant->GetS64()) == spv::Scope::Device;
    else
      return static_cast<spv::Scope>(constant->GetU64()) == spv::Scope::Device;
  }

  assert(false);
  return false;
}

void UpgradeMemoryModel::UpgradeExtInst(Instruction* ext_inst) {
  const bool is_modf = ext_inst->GetSingleWordInOperand(1u) == GLSLstd450Modf;
  auto ptr_id = ext_inst->GetSingleWordInOperand(3u);
  auto ptr_type_id = get_def_use_mgr()->GetDef(ptr_id)->type_id();
  auto pointee_type_id =
      get_def_use_mgr()->GetDef(ptr_type_id)->GetSingleWordInOperand(1u);
  auto element_type_id = ext_inst->type_id();
  std::vector<const analysis::Type*> element_types(2);
  element_types[0] = context()->get_type_mgr()->GetType(element_type_id);
  element_types[1] = context()->get_type_mgr()->GetType(pointee_type_id);
  analysis::Struct struct_type(element_types);
  uint32_t struct_id =
      context()->get_type_mgr()->GetTypeInstruction(&struct_type);
  // Change the operation
  GLSLstd450 new_op = is_modf ? GLSLstd450ModfStruct : GLSLstd450FrexpStruct;
  ext_inst->SetOperand(3u, {static_cast<uint32_t>(new_op)});
  // Remove the pointer argument
  ext_inst->RemoveOperand(5u);
  // Set the type id to the new struct.
  ext_inst->SetResultType(struct_id);

  // The result is now a struct of the original result. The zero'th element is
  // old result and should replace the old result. The one'th element needs to
  // be stored via a new instruction.
  auto where = ext_inst->NextNode();
  InstructionBuilder builder(
      context(), where,
      IRContext::kAnalysisDefUse | IRContext::kAnalysisInstrToBlockMapping);
  auto extract_0 =
      builder.AddCompositeExtract(element_type_id, ext_inst->result_id(), {0});
  context()->ReplaceAllUsesWith(ext_inst->result_id(), extract_0->result_id());
  // The extract's input was just changed to itself, so fix that.
  extract_0->SetInOperand(0u, {ext_inst->result_id()});
  auto extract_1 =
      builder.AddCompositeExtract(pointee_type_id, ext_inst->result_id(), {1});
  builder.AddStore(ptr_id, extract_1->result_id());
}

uint32_t UpgradeMemoryModel::MemoryAccessNumWords(uint32_t mask) {
  uint32_t result = 1;
  if (mask & uint32_t(spv::MemoryAccessMask::Aligned)) ++result;
  if (mask & uint32_t(spv::MemoryAccessMask::MakePointerAvailableKHR)) ++result;
  if (mask & uint32_t(spv::MemoryAccessMask::MakePointerVisibleKHR)) ++result;
  return result;
}

}  // namespace opt
}  // namespace spvtools