1 // Copyright 2018 the V8 project authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #ifndef V8_WASM_JUMP_TABLE_ASSEMBLER_H_ 6 #define V8_WASM_JUMP_TABLE_ASSEMBLER_H_ 7 8 #include "src/codegen/macro-assembler.h" 9 10 namespace v8 { 11 namespace internal { 12 namespace wasm { 13 14 // The jump table is the central dispatch point for all (direct and indirect) 15 // invocations in WebAssembly. It holds one slot per function in a module, with 16 // each slot containing a dispatch to the currently published {WasmCode} that 17 // corresponds to the function. 18 // 19 // Additionally to this main jump table, there exist special jump tables for 20 // other purposes: 21 // - the far stub table contains one entry per wasm runtime stub (see 22 // {WasmCode::RuntimeStubId}, which jumps to the corresponding embedded 23 // builtin, plus (if not the full address space can be reached via the jump 24 // table) one entry per wasm function. 25 // - the lazy compile table contains one entry per wasm function which jumps to 26 // the common {WasmCompileLazy} builtin and passes the function index that was 27 // invoked. 28 // 29 // The main jump table is split into lines of fixed size, with lines laid out 30 // consecutively within the executable memory of the {NativeModule}. The slots 31 // in turn are consecutive within a line, but do not cross line boundaries. 32 // 33 // +- L1 -------------------+ +- L2 -------------------+ +- L3 ... 34 // | S1 | S2 | ... | Sn | x | | S1 | S2 | ... | Sn | x | | S1 ... 35 // +------------------------+ +------------------------+ +---- ... 36 // 37 // The above illustrates jump table lines {Li} containing slots {Si} with each 38 // line containing {n} slots and some padding {x} for alignment purposes. 39 // Other jump tables are just consecutive. 40 // 41 // The main jump table will be patched concurrently while other threads execute 42 // it. The code at the new target might also have been emitted concurrently, so 43 // we need to ensure that there is proper synchronization between code emission, 44 // jump table patching and code execution. 45 // On Intel platforms, this all works out of the box because there is cache 46 // coherency between i-cache and d-cache. 47 // On ARM, it is safe because the i-cache flush after code emission executes an 48 // "ic ivau" (Instruction Cache line Invalidate by Virtual Address to Point of 49 // Unification), which broadcasts to all cores. A core which sees the jump table 50 // update thus also sees the new code. Since the other core does not explicitly 51 // execute an "isb" (Instruction Synchronization Barrier), it might still 52 // execute the old code afterwards, which is no problem, since that code remains 53 // available until it is garbage collected. Garbage collection itself is a 54 // synchronization barrier though. 55 class V8_EXPORT_PRIVATE JumpTableAssembler : public MacroAssembler { 56 public: 57 // Translate an offset into the continuous jump table to a jump table index. SlotOffsetToIndex(uint32_t slot_offset)58 static uint32_t SlotOffsetToIndex(uint32_t slot_offset) { 59 uint32_t line_index = slot_offset / kJumpTableLineSize; 60 uint32_t line_offset = slot_offset % kJumpTableLineSize; 61 DCHECK_EQ(0, line_offset % kJumpTableSlotSize); 62 return line_index * kJumpTableSlotsPerLine + 63 line_offset / kJumpTableSlotSize; 64 } 65 66 // Translate a jump table index to an offset into the continuous jump table. JumpSlotIndexToOffset(uint32_t slot_index)67 static uint32_t JumpSlotIndexToOffset(uint32_t slot_index) { 68 uint32_t line_index = slot_index / kJumpTableSlotsPerLine; 69 uint32_t line_offset = 70 (slot_index % kJumpTableSlotsPerLine) * kJumpTableSlotSize; 71 return line_index * kJumpTableLineSize + line_offset; 72 } 73 74 // Determine the size of a jump table containing the given number of slots. SizeForNumberOfSlots(uint32_t slot_count)75 static constexpr uint32_t SizeForNumberOfSlots(uint32_t slot_count) { 76 return ((slot_count + kJumpTableSlotsPerLine - 1) / 77 kJumpTableSlotsPerLine) * 78 kJumpTableLineSize; 79 } 80 81 // Translate a far jump table index to an offset into the table. FarJumpSlotIndexToOffset(uint32_t slot_index)82 static uint32_t FarJumpSlotIndexToOffset(uint32_t slot_index) { 83 return slot_index * kFarJumpTableSlotSize; 84 } 85 86 // Translate a far jump table offset to the index into the table. FarJumpSlotOffsetToIndex(uint32_t offset)87 static uint32_t FarJumpSlotOffsetToIndex(uint32_t offset) { 88 DCHECK_EQ(0, offset % kFarJumpTableSlotSize); 89 return offset / kFarJumpTableSlotSize; 90 } 91 92 // Determine the size of a far jump table containing the given number of 93 // slots. SizeForNumberOfFarJumpSlots(int num_runtime_slots,int num_function_slots)94 static constexpr uint32_t SizeForNumberOfFarJumpSlots( 95 int num_runtime_slots, int num_function_slots) { 96 int num_entries = num_runtime_slots + num_function_slots; 97 return num_entries * kFarJumpTableSlotSize; 98 } 99 100 // Translate a slot index to an offset into the lazy compile table. LazyCompileSlotIndexToOffset(uint32_t slot_index)101 static uint32_t LazyCompileSlotIndexToOffset(uint32_t slot_index) { 102 return slot_index * kLazyCompileTableSlotSize; 103 } 104 105 // Determine the size of a lazy compile table. SizeForNumberOfLazyFunctions(uint32_t slot_count)106 static constexpr uint32_t SizeForNumberOfLazyFunctions(uint32_t slot_count) { 107 return slot_count * kLazyCompileTableSlotSize; 108 } 109 GenerateLazyCompileTable(Address base,uint32_t num_slots,uint32_t num_imported_functions,Address wasm_compile_lazy_target)110 static void GenerateLazyCompileTable(Address base, uint32_t num_slots, 111 uint32_t num_imported_functions, 112 Address wasm_compile_lazy_target) { 113 uint32_t lazy_compile_table_size = num_slots * kLazyCompileTableSlotSize; 114 // Assume enough space, so the Assembler does not try to grow the buffer. 115 JumpTableAssembler jtasm(base, lazy_compile_table_size + 256); 116 for (uint32_t slot_index = 0; slot_index < num_slots; ++slot_index) { 117 DCHECK_EQ(slot_index * kLazyCompileTableSlotSize, jtasm.pc_offset()); 118 jtasm.EmitLazyCompileJumpSlot(slot_index + num_imported_functions, 119 wasm_compile_lazy_target); 120 } 121 DCHECK_EQ(lazy_compile_table_size, jtasm.pc_offset()); 122 FlushInstructionCache(base, lazy_compile_table_size); 123 } 124 GenerateFarJumpTable(Address base,Address * stub_targets,int num_runtime_slots,int num_function_slots)125 static void GenerateFarJumpTable(Address base, Address* stub_targets, 126 int num_runtime_slots, 127 int num_function_slots) { 128 uint32_t table_size = 129 SizeForNumberOfFarJumpSlots(num_runtime_slots, num_function_slots); 130 // Assume enough space, so the Assembler does not try to grow the buffer. 131 JumpTableAssembler jtasm(base, table_size + 256); 132 int offset = 0; 133 for (int index = 0; index < num_runtime_slots + num_function_slots; 134 ++index) { 135 DCHECK_EQ(offset, FarJumpSlotIndexToOffset(index)); 136 // Functions slots initially jump to themselves. They are patched before 137 // being used. 138 Address target = 139 index < num_runtime_slots ? stub_targets[index] : base + offset; 140 jtasm.EmitFarJumpSlot(target); 141 offset += kFarJumpTableSlotSize; 142 DCHECK_EQ(offset, jtasm.pc_offset()); 143 } 144 FlushInstructionCache(base, table_size); 145 } 146 PatchJumpTableSlot(Address jump_table_slot,Address far_jump_table_slot,Address target)147 static void PatchJumpTableSlot(Address jump_table_slot, 148 Address far_jump_table_slot, Address target) { 149 // First, try to patch the jump table slot. 150 JumpTableAssembler jtasm(jump_table_slot); 151 if (!jtasm.EmitJumpSlot(target)) { 152 // If that fails, we need to patch the far jump table slot, and then 153 // update the jump table slot to jump to this far jump table slot. 154 DCHECK_NE(kNullAddress, far_jump_table_slot); 155 JumpTableAssembler::PatchFarJumpSlot(far_jump_table_slot, target); 156 CHECK(jtasm.EmitJumpSlot(far_jump_table_slot)); 157 } 158 jtasm.NopBytes(kJumpTableSlotSize - jtasm.pc_offset()); 159 FlushInstructionCache(jump_table_slot, kJumpTableSlotSize); 160 } 161 162 private: 163 // Instantiate a {JumpTableAssembler} for patching. 164 explicit JumpTableAssembler(Address slot_addr, int size = 256) MacroAssembler(nullptr,JumpTableAssemblerOptions (),CodeObjectRequired::kNo,ExternalAssemblerBuffer (reinterpret_cast<uint8_t * > (slot_addr),size))165 : MacroAssembler(nullptr, JumpTableAssemblerOptions(), 166 CodeObjectRequired::kNo, 167 ExternalAssemblerBuffer( 168 reinterpret_cast<uint8_t*>(slot_addr), size)) {} 169 170 // To allow concurrent patching of the jump table entries, we need to ensure 171 // that the instruction containing the call target does not cross cache-line 172 // boundaries. The jump table line size has been chosen to satisfy this. 173 #if V8_TARGET_ARCH_X64 174 static constexpr int kJumpTableLineSize = 64; 175 static constexpr int kJumpTableSlotSize = 5; 176 static constexpr int kFarJumpTableSlotSize = 16; 177 static constexpr int kLazyCompileTableSlotSize = 10; 178 #elif V8_TARGET_ARCH_IA32 179 static constexpr int kJumpTableLineSize = 64; 180 static constexpr int kJumpTableSlotSize = 5; 181 static constexpr int kFarJumpTableSlotSize = 5; 182 static constexpr int kLazyCompileTableSlotSize = 10; 183 #elif V8_TARGET_ARCH_ARM 184 static constexpr int kJumpTableLineSize = 3 * kInstrSize; 185 static constexpr int kJumpTableSlotSize = 3 * kInstrSize; 186 static constexpr int kFarJumpTableSlotSize = 2 * kInstrSize; 187 static constexpr int kLazyCompileTableSlotSize = 5 * kInstrSize; 188 #elif V8_TARGET_ARCH_ARM64 && V8_ENABLE_CONTROL_FLOW_INTEGRITY 189 static constexpr int kJumpTableLineSize = 2 * kInstrSize; 190 static constexpr int kJumpTableSlotSize = 2 * kInstrSize; 191 static constexpr int kFarJumpTableSlotSize = 6 * kInstrSize; 192 static constexpr int kLazyCompileTableSlotSize = 4 * kInstrSize; 193 #elif V8_TARGET_ARCH_ARM64 && !V8_ENABLE_CONTROL_FLOW_INTEGRITY 194 static constexpr int kJumpTableLineSize = 1 * kInstrSize; 195 static constexpr int kJumpTableSlotSize = 1 * kInstrSize; 196 static constexpr int kFarJumpTableSlotSize = 4 * kInstrSize; 197 static constexpr int kLazyCompileTableSlotSize = 3 * kInstrSize; 198 #elif V8_TARGET_ARCH_S390X 199 static constexpr int kJumpTableLineSize = 128; 200 static constexpr int kJumpTableSlotSize = 14; 201 static constexpr int kFarJumpTableSlotSize = 14; 202 static constexpr int kLazyCompileTableSlotSize = 20; 203 #elif V8_TARGET_ARCH_PPC64 204 static constexpr int kJumpTableLineSize = 64; 205 static constexpr int kJumpTableSlotSize = 7 * kInstrSize; 206 static constexpr int kFarJumpTableSlotSize = 7 * kInstrSize; 207 static constexpr int kLazyCompileTableSlotSize = 12 * kInstrSize; 208 #elif V8_TARGET_ARCH_MIPS 209 static constexpr int kJumpTableLineSize = 8 * kInstrSize; 210 static constexpr int kJumpTableSlotSize = 8 * kInstrSize; 211 static constexpr int kFarJumpTableSlotSize = 4 * kInstrSize; 212 static constexpr int kLazyCompileTableSlotSize = 6 * kInstrSize; 213 #elif V8_TARGET_ARCH_MIPS64 214 static constexpr int kJumpTableLineSize = 8 * kInstrSize; 215 static constexpr int kJumpTableSlotSize = 8 * kInstrSize; 216 static constexpr int kFarJumpTableSlotSize = 6 * kInstrSize; 217 static constexpr int kLazyCompileTableSlotSize = 8 * kInstrSize; 218 #else 219 #error Unknown architecture. 220 #endif 221 222 static constexpr int kJumpTableSlotsPerLine = 223 kJumpTableLineSize / kJumpTableSlotSize; 224 STATIC_ASSERT(kJumpTableSlotsPerLine >= 1); 225 226 // {JumpTableAssembler} is never used during snapshot generation, and its code 227 // must be independent of the code range of any isolate anyway. Just ensure 228 // that no relocation information is recorded, there is no buffer to store it 229 // since it is instantiated in patching mode in existing code directly. JumpTableAssemblerOptions()230 static AssemblerOptions JumpTableAssemblerOptions() { 231 AssemblerOptions options; 232 options.disable_reloc_info_for_patching = true; 233 return options; 234 } 235 236 void EmitLazyCompileJumpSlot(uint32_t func_index, 237 Address lazy_compile_target); 238 239 // Returns {true} if the jump fits in the jump table slot, {false} otherwise. 240 bool EmitJumpSlot(Address target); 241 242 // Initially emit a far jump slot. 243 void EmitFarJumpSlot(Address target); 244 245 // Patch an existing far jump slot, and make sure that this updated eventually 246 // becomes available to all execution units that might execute this code. 247 static void PatchFarJumpSlot(Address slot, Address target); 248 249 void NopBytes(int bytes); 250 }; 251 252 } // namespace wasm 253 } // namespace internal 254 } // namespace v8 255 256 #endif // V8_WASM_JUMP_TABLE_ASSEMBLER_H_ 257