src/wasm/wasm-code-specialization.cc

// Copyright 2017 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "src/wasm/wasm-code-specialization.h"

#include "src/assembler-inl.h"
#include "src/objects-inl.h"
#include "src/source-position-table.h"
#include "src/wasm/decoder.h"
#include "src/wasm/wasm-module.h"
#include "src/wasm/wasm-opcodes.h"

using namespace v8::internal;
using namespace v8::internal::wasm;

namespace {

int ExtractDirectCallIndex(wasm::Decoder& decoder, const byte* pc) {
  DCHECK_EQ(static_cast<int>(kExprCallFunction), static_cast<int>(*pc));
  decoder.Reset(pc + 1, pc + 6);
  uint32_t call_idx = decoder.consume_u32v("call index");
  DCHECK(decoder.ok());
  DCHECK_GE(kMaxInt, call_idx);
  return static_cast<int>(call_idx);
}

int AdvanceSourcePositionTableIterator(SourcePositionTableIterator& iterator,
                                       size_t offset_l) {
  DCHECK_GE(kMaxInt, offset_l);
  int offset = static_cast<int>(offset_l);
  DCHECK(!iterator.done());
  int byte_pos;
  do {
    byte_pos = iterator.source_position().ScriptOffset();
    iterator.Advance();
  } while (!iterator.done() && iterator.code_offset() <= offset);
  return byte_pos;
}

class PatchDirectCallsHelper {
 public:
  PatchDirectCallsHelper(WasmInstanceObject* instance, Code* code)
      : source_pos_it(code->source_position_table()),
        decoder(nullptr, nullptr) {
    FixedArray* deopt_data = code->deoptimization_data();
    DCHECK_EQ(2, deopt_data->length());
    WasmCompiledModule* comp_mod = instance->compiled_module();
    int func_index = Smi::cast(deopt_data->get(1))->value();
    func_bytes = comp_mod->module_bytes()->GetChars() +
                 comp_mod->module()->functions[func_index].code_start_offset;
  }

  SourcePositionTableIterator source_pos_it;
  Decoder decoder;
  const byte* func_bytes;
};

}  // namespace

CodeSpecialization::CodeSpecialization(Isolate* isolate, Zone* zone)
    : objects_to_relocate(isolate->heap(), ZoneAllocationPolicy(zone)) {}

CodeSpecialization::~CodeSpecialization() {}

void CodeSpecialization::RelocateMemoryReferences(Address old_start,
                                                  uint32_t old_size,
                                                  Address new_start,
                                                  uint32_t new_size) {
  DCHECK(old_mem_start == nullptr && old_mem_size == 0 &&
         new_mem_start == nullptr && new_mem_size == 0);
  DCHECK(old_start != new_start || old_size != new_size);
  old_mem_start = old_start;
  old_mem_size = old_size;
  new_mem_start = new_start;
  new_mem_size = new_size;
}

void CodeSpecialization::RelocateGlobals(Address old_start, Address new_start) {
  DCHECK(old_globals_start == 0 && new_globals_start == 0);
  DCHECK(old_start != 0 || new_start != 0);
  old_globals_start = old_start;
  new_globals_start = new_start;
}

void CodeSpecialization::PatchTableSize(uint32_t old_size, uint32_t new_size) {
  DCHECK(old_function_table_size == 0 && new_function_table_size == 0);
  DCHECK(old_size != 0 || new_size != 0);
  old_function_table_size = old_size;
  new_function_table_size = new_size;
}

void CodeSpecialization::RelocateDirectCalls(
    Handle<WasmInstanceObject> instance) {
  DCHECK(relocate_direct_calls_instance.is_null());
  DCHECK(!instance.is_null());
  relocate_direct_calls_instance = instance;
}

void CodeSpecialization::RelocateObject(Handle<Object> old_obj,
                                        Handle<Object> new_obj) {
  DCHECK(!old_obj.is_null() && !new_obj.is_null());
  has_objects_to_relocate = true;
  objects_to_relocate.Set(*old_obj, new_obj);
}

bool CodeSpecialization::ApplyToWholeInstance(
    WasmInstanceObject* instance, ICacheFlushMode icache_flush_mode) {
  DisallowHeapAllocation no_gc;
  WasmCompiledModule* compiled_module = instance->compiled_module();
  FixedArray* code_table = compiled_module->ptr_to_code_table();
  WasmModule* module = compiled_module->module();
  std::vector<WasmFunction>* wasm_functions =
      &compiled_module->module()->functions;
  DCHECK_EQ(wasm_functions->size() +
                compiled_module->module()->num_exported_functions,
            code_table->length());

  bool changed = false;
  int func_index = module->num_imported_functions;

  // Patch all wasm functions.
  for (int num_wasm_functions = static_cast<int>(wasm_functions->size());
       func_index < num_wasm_functions; ++func_index) {
    Code* wasm_function = Code::cast(code_table->get(func_index));
    changed |= ApplyToWasmCode(wasm_function, icache_flush_mode);
  }

  // Patch all exported functions.
  for (auto exp : module->export_table) {
    if (exp.kind != kExternalFunction) continue;
    Code* export_wrapper = Code::cast(code_table->get(func_index));
    DCHECK_EQ(Code::JS_TO_WASM_FUNCTION, export_wrapper->kind());
    // There must be exactly one call to WASM_FUNCTION or WASM_TO_JS_FUNCTION.
    int num_wasm_calls = 0;
    for (RelocIterator it(export_wrapper,
                          RelocInfo::ModeMask(RelocInfo::CODE_TARGET));
         !it.done(); it.next()) {
      DCHECK(RelocInfo::IsCodeTarget(it.rinfo()->rmode()));
      Code* code = Code::GetCodeFromTargetAddress(it.rinfo()->target_address());
      // Ignore calls to other builtins like ToNumber.
      if (code->kind() != Code::WASM_FUNCTION &&
          code->kind() != Code::WASM_TO_JS_FUNCTION &&
          code->builtin_index() != Builtins::kIllegal)
        continue;
      ++num_wasm_calls;
      Code* new_code = Code::cast(code_table->get(exp.index));
      DCHECK(new_code->kind() == Code::WASM_FUNCTION ||
             new_code->kind() == Code::WASM_TO_JS_FUNCTION);
      it.rinfo()->set_target_address(new_code->instruction_start(),
                                     UPDATE_WRITE_BARRIER, SKIP_ICACHE_FLUSH);
      changed = true;
    }
    DCHECK_EQ(1, num_wasm_calls);
    func_index++;
  }
  DCHECK_EQ(code_table->length(), func_index);
  return changed;
}

bool CodeSpecialization::ApplyToWasmCode(Code* code,
                                         ICacheFlushMode icache_flush_mode) {
  DisallowHeapAllocation no_gc;
  DCHECK_EQ(Code::WASM_FUNCTION, code->kind());

  bool reloc_mem_addr = old_mem_start != new_mem_start;
  bool reloc_mem_size = old_mem_size != new_mem_size;
  bool reloc_globals = old_globals_start || new_globals_start;
  bool patch_table_size = old_function_table_size || new_function_table_size;
  bool reloc_direct_calls = !relocate_direct_calls_instance.is_null();
  bool reloc_objects = has_objects_to_relocate;

  int reloc_mode = 0;
  auto add_mode = [&reloc_mode](bool cond, RelocInfo::Mode mode) {
    if (cond) reloc_mode |= RelocInfo::ModeMask(mode);
  };
  add_mode(reloc_mem_addr, RelocInfo::WASM_MEMORY_REFERENCE);
  add_mode(reloc_mem_size, RelocInfo::WASM_MEMORY_SIZE_REFERENCE);
  add_mode(reloc_globals, RelocInfo::WASM_GLOBAL_REFERENCE);
  add_mode(patch_table_size, RelocInfo::WASM_FUNCTION_TABLE_SIZE_REFERENCE);
  add_mode(reloc_direct_calls, RelocInfo::CODE_TARGET);
  add_mode(reloc_objects, RelocInfo::EMBEDDED_OBJECT);

  std::unique_ptr<PatchDirectCallsHelper> patch_direct_calls_helper;
  bool changed = false;

  for (RelocIterator it(code, reloc_mode); !it.done(); it.next()) {
    RelocInfo::Mode mode = it.rinfo()->rmode();
    switch (mode) {
      case RelocInfo::WASM_MEMORY_REFERENCE:
        DCHECK(reloc_mem_addr);
        it.rinfo()->update_wasm_memory_reference(old_mem_start, new_mem_start,
                                                 icache_flush_mode);
        changed = true;
        break;
      case RelocInfo::WASM_MEMORY_SIZE_REFERENCE:
        DCHECK(reloc_mem_size);
        it.rinfo()->update_wasm_memory_size(old_mem_size, new_mem_size,
                                            icache_flush_mode);
        changed = true;
        break;
      case RelocInfo::WASM_GLOBAL_REFERENCE:
        DCHECK(reloc_globals);
        it.rinfo()->update_wasm_global_reference(
            old_globals_start, new_globals_start, icache_flush_mode);
        changed = true;
        break;
      case RelocInfo::CODE_TARGET: {
        DCHECK(reloc_direct_calls);
        Code* old_code =
            Code::GetCodeFromTargetAddress(it.rinfo()->target_address());
        // Skip everything which is not a wasm call (stack checks, traps, ...).
        if (old_code->kind() != Code::WASM_FUNCTION &&
            old_code->kind() != Code::WASM_TO_JS_FUNCTION &&
            old_code->builtin_index() != Builtins::kIllegal)
          continue;
        // Iterate simultaneously over the relocation information and the source
        // position table. For each call in the reloc info, move the source
        // position iterator forward to that position to find the byte offset of
        // the respective call. Then extract the call index from the module wire
        // bytes to find the new compiled function.
        size_t offset = it.rinfo()->pc() - code->instruction_start();
        if (!patch_direct_calls_helper) {
          patch_direct_calls_helper.reset(new PatchDirectCallsHelper(
              *relocate_direct_calls_instance, code));
        }
        int byte_pos = AdvanceSourcePositionTableIterator(
            patch_direct_calls_helper->source_pos_it, offset);
        int called_func_index = ExtractDirectCallIndex(
            patch_direct_calls_helper->decoder,
            patch_direct_calls_helper->func_bytes + byte_pos);
        FixedArray* code_table =
            relocate_direct_calls_instance->compiled_module()
                ->ptr_to_code_table();
        Code* new_code = Code::cast(code_table->get(called_func_index));
        it.rinfo()->set_target_address(new_code->instruction_start(),
                                       UPDATE_WRITE_BARRIER, icache_flush_mode);
        changed = true;
      } break;
      case RelocInfo::EMBEDDED_OBJECT: {
        DCHECK(reloc_objects);
        Object* old = it.rinfo()->target_object();
        Handle<Object>* new_obj = objects_to_relocate.Find(old);
        if (new_obj) {
          it.rinfo()->set_target_object(**new_obj, UPDATE_WRITE_BARRIER,
                                        icache_flush_mode);
          changed = true;
        }
      } break;
      case RelocInfo::WASM_FUNCTION_TABLE_SIZE_REFERENCE:
        DCHECK(patch_table_size);
        it.rinfo()->update_wasm_function_table_size_reference(
            old_function_table_size, new_function_table_size,
            icache_flush_mode);
        changed = true;
        break;
      default:
        UNREACHABLE();
    }
  }

  return changed;
}