1 // Copyright 2017 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "src/wasm/baseline/liftoff-compiler.h"
6
7 #include "src/base/enum-set.h"
8 #include "src/base/optional.h"
9 #include "src/base/platform/wrappers.h"
10 #include "src/codegen/assembler-inl.h"
11 // TODO(clemensb): Remove dependences on compiler stuff.
12 #include "src/codegen/external-reference.h"
13 #include "src/codegen/interface-descriptors-inl.h"
14 #include "src/codegen/machine-type.h"
15 #include "src/codegen/macro-assembler-inl.h"
16 #include "src/compiler/linkage.h"
17 #include "src/compiler/wasm-compiler.h"
18 #include "src/logging/counters.h"
19 #include "src/logging/log.h"
20 #include "src/objects/smi.h"
21 #include "src/tracing/trace-event.h"
22 #include "src/utils/ostreams.h"
23 #include "src/utils/utils.h"
24 #include "src/wasm/baseline/liftoff-assembler.h"
25 #include "src/wasm/baseline/liftoff-register.h"
26 #include "src/wasm/function-body-decoder-impl.h"
27 #include "src/wasm/function-compiler.h"
28 #include "src/wasm/memory-tracing.h"
29 #include "src/wasm/object-access.h"
30 #include "src/wasm/simd-shuffle.h"
31 #include "src/wasm/wasm-debug.h"
32 #include "src/wasm/wasm-engine.h"
33 #include "src/wasm/wasm-linkage.h"
34 #include "src/wasm/wasm-objects.h"
35 #include "src/wasm/wasm-opcodes-inl.h"
36
37 namespace v8 {
38 namespace internal {
39 namespace wasm {
40
41 constexpr auto kRegister = LiftoffAssembler::VarState::kRegister;
42 constexpr auto kIntConst = LiftoffAssembler::VarState::kIntConst;
43 constexpr auto kStack = LiftoffAssembler::VarState::kStack;
44
45 namespace {
46
47 #define __ asm_.
48
49 #define TRACE(...) \
50 do { \
51 if (FLAG_trace_liftoff) PrintF("[liftoff] " __VA_ARGS__); \
52 } while (false)
53
54 #define WASM_INSTANCE_OBJECT_FIELD_OFFSET(name) \
55 ObjectAccess::ToTagged(WasmInstanceObject::k##name##Offset)
56
57 template <int expected_size, int actual_size>
58 struct assert_field_size {
59 static_assert(expected_size == actual_size,
60 "field in WasmInstance does not have the expected size");
61 static constexpr int size = actual_size;
62 };
63
64 #define WASM_INSTANCE_OBJECT_FIELD_SIZE(name) \
65 FIELD_SIZE(WasmInstanceObject::k##name##Offset)
66
67 #define LOAD_INSTANCE_FIELD(dst, name, load_size, pinned) \
68 __ LoadFromInstance(dst, LoadInstanceIntoRegister(pinned, dst), \
69 WASM_INSTANCE_OBJECT_FIELD_OFFSET(name), \
70 assert_field_size<WASM_INSTANCE_OBJECT_FIELD_SIZE(name), \
71 load_size>::size);
72
73 #define LOAD_TAGGED_PTR_INSTANCE_FIELD(dst, name, pinned) \
74 static_assert(WASM_INSTANCE_OBJECT_FIELD_SIZE(name) == kTaggedSize, \
75 "field in WasmInstance does not have the expected size"); \
76 __ LoadTaggedPointerFromInstance(dst, LoadInstanceIntoRegister(pinned, dst), \
77 WASM_INSTANCE_OBJECT_FIELD_OFFSET(name));
78
79 #ifdef V8_CODE_COMMENTS
80 #define CODE_COMMENT(str) \
81 do { \
82 __ RecordComment(str); \
83 } while (false)
84 #else
85 #define CODE_COMMENT(str) ((void)0)
86 #endif
87
88 constexpr LoadType::LoadTypeValue kPointerLoadType =
89 kSystemPointerSize == 8 ? LoadType::kI64Load : LoadType::kI32Load;
90
91 constexpr ValueKind kPointerKind = LiftoffAssembler::kPointerKind;
92 constexpr ValueKind kSmiKind = LiftoffAssembler::kSmiKind;
93 constexpr ValueKind kTaggedKind = LiftoffAssembler::kTaggedKind;
94
95 // Used to construct fixed-size signatures: MakeSig::Returns(...).Params(...);
96 using MakeSig = FixedSizeSignature<ValueKind>;
97
98 #if V8_TARGET_ARCH_ARM64
99 // On ARM64, the Assembler keeps track of pointers to Labels to resolve
100 // branches to distant targets. Moving labels would confuse the Assembler,
101 // thus store the label on the heap and keep a unique_ptr.
102 class MovableLabel {
103 public:
104 MOVE_ONLY_NO_DEFAULT_CONSTRUCTOR(MovableLabel);
MovableLabel()105 MovableLabel() : label_(new Label()) {}
106
get()107 Label* get() { return label_.get(); }
108
109 private:
110 std::unique_ptr<Label> label_;
111 };
112 #else
113 // On all other platforms, just store the Label directly.
114 class MovableLabel {
115 public:
116 MOVE_ONLY_WITH_DEFAULT_CONSTRUCTORS(MovableLabel);
117
get()118 Label* get() { return &label_; }
119
120 private:
121 Label label_;
122 };
123 #endif
124
GetLoweredCallDescriptor(Zone * zone,compiler::CallDescriptor * call_desc)125 compiler::CallDescriptor* GetLoweredCallDescriptor(
126 Zone* zone, compiler::CallDescriptor* call_desc) {
127 return kSystemPointerSize == 4
128 ? compiler::GetI32WasmCallDescriptor(zone, call_desc)
129 : call_desc;
130 }
131
GetGpParamRegisters()132 constexpr LiftoffRegList GetGpParamRegisters() {
133 LiftoffRegList registers;
134 for (auto reg : kGpParamRegisters) registers.set(reg);
135 return registers;
136 }
137
GetCompareCondition(WasmOpcode opcode)138 constexpr LiftoffCondition GetCompareCondition(WasmOpcode opcode) {
139 switch (opcode) {
140 case kExprI32Eq:
141 return kEqual;
142 case kExprI32Ne:
143 return kUnequal;
144 case kExprI32LtS:
145 return kSignedLessThan;
146 case kExprI32LtU:
147 return kUnsignedLessThan;
148 case kExprI32GtS:
149 return kSignedGreaterThan;
150 case kExprI32GtU:
151 return kUnsignedGreaterThan;
152 case kExprI32LeS:
153 return kSignedLessEqual;
154 case kExprI32LeU:
155 return kUnsignedLessEqual;
156 case kExprI32GeS:
157 return kSignedGreaterEqual;
158 case kExprI32GeU:
159 return kUnsignedGreaterEqual;
160 default:
161 UNREACHABLE();
162 }
163 }
164
165 // Builds a {DebugSideTable}.
166 class DebugSideTableBuilder {
167 using Entry = DebugSideTable::Entry;
168 using Value = Entry::Value;
169
170 public:
171 enum AssumeSpilling {
172 // All register values will be spilled before the pc covered by the debug
173 // side table entry. Register slots will be marked as stack slots in the
174 // generated debug side table entry.
175 kAssumeSpilling,
176 // Register slots will be written out as they are.
177 kAllowRegisters,
178 // Register slots cannot appear since we already spilled.
179 kDidSpill
180 };
181
182 class EntryBuilder {
183 public:
EntryBuilder(int pc_offset,int stack_height,std::vector<Value> changed_values)184 explicit EntryBuilder(int pc_offset, int stack_height,
185 std::vector<Value> changed_values)
186 : pc_offset_(pc_offset),
187 stack_height_(stack_height),
188 changed_values_(std::move(changed_values)) {}
189
ToTableEntry()190 Entry ToTableEntry() {
191 return Entry{pc_offset_, stack_height_, std::move(changed_values_)};
192 }
193
MinimizeBasedOnPreviousStack(const std::vector<Value> & last_values)194 void MinimizeBasedOnPreviousStack(const std::vector<Value>& last_values) {
195 auto dst = changed_values_.begin();
196 auto end = changed_values_.end();
197 for (auto src = dst; src != end; ++src) {
198 if (src->index < static_cast<int>(last_values.size()) &&
199 *src == last_values[src->index]) {
200 continue;
201 }
202 if (dst != src) *dst = *src;
203 ++dst;
204 }
205 changed_values_.erase(dst, end);
206 }
207
pc_offset() const208 int pc_offset() const { return pc_offset_; }
set_pc_offset(int new_pc_offset)209 void set_pc_offset(int new_pc_offset) { pc_offset_ = new_pc_offset; }
210
211 private:
212 int pc_offset_;
213 int stack_height_;
214 std::vector<Value> changed_values_;
215 };
216
217 // Adds a new entry in regular code.
NewEntry(int pc_offset,base::Vector<DebugSideTable::Entry::Value> values)218 void NewEntry(int pc_offset,
219 base::Vector<DebugSideTable::Entry::Value> values) {
220 entries_.emplace_back(pc_offset, static_cast<int>(values.size()),
221 GetChangedStackValues(last_values_, values));
222 }
223
224 // Adds a new entry for OOL code, and returns a pointer to a builder for
225 // modifying that entry.
NewOOLEntry(base::Vector<DebugSideTable::Entry::Value> values)226 EntryBuilder* NewOOLEntry(base::Vector<DebugSideTable::Entry::Value> values) {
227 constexpr int kNoPcOffsetYet = -1;
228 ool_entries_.emplace_back(kNoPcOffsetYet, static_cast<int>(values.size()),
229 GetChangedStackValues(last_ool_values_, values));
230 return &ool_entries_.back();
231 }
232
SetNumLocals(int num_locals)233 void SetNumLocals(int num_locals) {
234 DCHECK_EQ(-1, num_locals_);
235 DCHECK_LE(0, num_locals);
236 num_locals_ = num_locals;
237 }
238
GenerateDebugSideTable()239 std::unique_ptr<DebugSideTable> GenerateDebugSideTable() {
240 DCHECK_LE(0, num_locals_);
241
242 // Connect {entries_} and {ool_entries_} by removing redundant stack
243 // information from the first {ool_entries_} entry (based on
244 // {last_values_}).
245 if (!entries_.empty() && !ool_entries_.empty()) {
246 ool_entries_.front().MinimizeBasedOnPreviousStack(last_values_);
247 }
248
249 std::vector<Entry> entries;
250 entries.reserve(entries_.size() + ool_entries_.size());
251 for (auto& entry : entries_) entries.push_back(entry.ToTableEntry());
252 for (auto& entry : ool_entries_) entries.push_back(entry.ToTableEntry());
253 DCHECK(std::is_sorted(
254 entries.begin(), entries.end(),
255 [](Entry& a, Entry& b) { return a.pc_offset() < b.pc_offset(); }));
256 return std::make_unique<DebugSideTable>(num_locals_, std::move(entries));
257 }
258
259 private:
GetChangedStackValues(std::vector<Value> & last_values,base::Vector<DebugSideTable::Entry::Value> values)260 static std::vector<Value> GetChangedStackValues(
261 std::vector<Value>& last_values,
262 base::Vector<DebugSideTable::Entry::Value> values) {
263 std::vector<Value> changed_values;
264 int old_stack_size = static_cast<int>(last_values.size());
265 last_values.resize(values.size());
266
267 int index = 0;
268 for (const auto& value : values) {
269 if (index >= old_stack_size || last_values[index] != value) {
270 changed_values.push_back(value);
271 last_values[index] = value;
272 }
273 ++index;
274 }
275 return changed_values;
276 }
277
278 int num_locals_ = -1;
279 // Keep a snapshot of the stack of the last entry, to generate a delta to the
280 // next entry.
281 std::vector<Value> last_values_;
282 std::vector<EntryBuilder> entries_;
283 // Keep OOL code entries separate so we can do proper delta-encoding (more
284 // entries might be added between the existing {entries_} and the
285 // {ool_entries_}). Store the entries in a list so the pointer is not
286 // invalidated by adding more entries.
287 std::vector<Value> last_ool_values_;
288 std::list<EntryBuilder> ool_entries_;
289 };
290
CheckBailoutAllowed(LiftoffBailoutReason reason,const char * detail,const CompilationEnv * env)291 void CheckBailoutAllowed(LiftoffBailoutReason reason, const char* detail,
292 const CompilationEnv* env) {
293 // Decode errors are ok.
294 if (reason == kDecodeError) return;
295
296 // --liftoff-only ensures that tests actually exercise the Liftoff path
297 // without bailing out. We also fail for missing CPU support, to avoid
298 // running any TurboFan code under --liftoff-only.
299 if (FLAG_liftoff_only) {
300 FATAL("--liftoff-only: treating bailout as fatal error. Cause: %s", detail);
301 }
302
303 // Missing CPU features are generally OK, except with --liftoff-only.
304 if (reason == kMissingCPUFeature) return;
305
306 // If --enable-testing-opcode-in-wasm is set, we are expected to bailout with
307 // "testing opcode".
308 if (FLAG_enable_testing_opcode_in_wasm &&
309 strcmp(detail, "testing opcode") == 0) {
310 return;
311 }
312
313 // Some externally maintained architectures don't fully implement Liftoff yet.
314 #if V8_TARGET_ARCH_MIPS || V8_TARGET_ARCH_MIPS64 || V8_TARGET_ARCH_S390X || \
315 V8_TARGET_ARCH_PPC || V8_TARGET_ARCH_PPC64 || V8_TARGET_ARCH_LOONG64
316 return;
317 #endif
318
319 #if V8_TARGET_ARCH_ARM
320 // Allow bailout for missing ARMv7 support.
321 if (!CpuFeatures::IsSupported(ARMv7) && reason == kUnsupportedArchitecture) {
322 return;
323 }
324 #endif
325
326 #define LIST_FEATURE(name, ...) kFeature_##name,
327 constexpr WasmFeatures kExperimentalFeatures{
328 FOREACH_WASM_EXPERIMENTAL_FEATURE_FLAG(LIST_FEATURE)};
329 #undef LIST_FEATURE
330
331 // Bailout is allowed if any experimental feature is enabled.
332 if (env->enabled_features.contains_any(kExperimentalFeatures)) return;
333
334 // Otherwise, bailout is not allowed.
335 FATAL("Liftoff bailout should not happen. Cause: %s\n", detail);
336 }
337
338 class LiftoffCompiler {
339 public:
340 // TODO(clemensb): Make this a template parameter.
341 static constexpr Decoder::ValidateFlag validate = Decoder::kBooleanValidation;
342
343 using Value = ValueBase<validate>;
344
345 struct ElseState {
346 MovableLabel label;
347 LiftoffAssembler::CacheState state;
348 };
349
350 struct TryInfo {
351 TryInfo() = default;
352 LiftoffAssembler::CacheState catch_state;
353 Label catch_label;
354 bool catch_reached = false;
355 bool in_handler = false;
356 };
357
358 struct Control : public ControlBase<Value, validate> {
359 std::unique_ptr<ElseState> else_state;
360 LiftoffAssembler::CacheState label_state;
361 MovableLabel label;
362 std::unique_ptr<TryInfo> try_info;
363 // Number of exceptions on the stack below this control.
364 int num_exceptions = 0;
365
366 MOVE_ONLY_NO_DEFAULT_CONSTRUCTOR(Control);
367
368 template <typename... Args>
Controlv8::internal::wasm::__anon3bd82aa60111::LiftoffCompiler::Control369 explicit Control(Args&&... args) V8_NOEXCEPT
370 : ControlBase(std::forward<Args>(args)...) {}
371 };
372
373 using FullDecoder = WasmFullDecoder<validate, LiftoffCompiler>;
374 using ValueKindSig = LiftoffAssembler::ValueKindSig;
375
376 class MostlySmallValueKindSig : public Signature<ValueKind> {
377 public:
MostlySmallValueKindSig(Zone * zone,const FunctionSig * sig)378 MostlySmallValueKindSig(Zone* zone, const FunctionSig* sig)
379 : Signature<ValueKind>(sig->return_count(), sig->parameter_count(),
380 MakeKinds(inline_storage_, zone, sig)) {}
381
382 private:
383 static constexpr size_t kInlineStorage = 8;
384
MakeKinds(ValueKind * storage,Zone * zone,const FunctionSig * sig)385 static ValueKind* MakeKinds(ValueKind* storage, Zone* zone,
386 const FunctionSig* sig) {
387 const size_t size = sig->parameter_count() + sig->return_count();
388 if (V8_UNLIKELY(size > kInlineStorage)) {
389 storage = zone->NewArray<ValueKind>(size);
390 }
391 std::transform(sig->all().begin(), sig->all().end(), storage,
392 [](ValueType type) { return type.kind(); });
393 return storage;
394 }
395
396 ValueKind inline_storage_[kInlineStorage];
397 };
398
399 // For debugging, we need to spill registers before a trap or a stack check to
400 // be able to inspect them.
401 struct SpilledRegistersForInspection : public ZoneObject {
402 struct Entry {
403 int offset;
404 LiftoffRegister reg;
405 ValueKind kind;
406 };
407 ZoneVector<Entry> entries;
408
SpilledRegistersForInspectionv8::internal::wasm::__anon3bd82aa60111::LiftoffCompiler::SpilledRegistersForInspection409 explicit SpilledRegistersForInspection(Zone* zone) : entries(zone) {}
410 };
411
412 struct OutOfLineSafepointInfo {
413 ZoneVector<int> slots;
414 LiftoffRegList spills;
415
OutOfLineSafepointInfov8::internal::wasm::__anon3bd82aa60111::LiftoffCompiler::OutOfLineSafepointInfo416 explicit OutOfLineSafepointInfo(Zone* zone) : slots(zone) {}
417 };
418
419 struct OutOfLineCode {
420 MovableLabel label;
421 MovableLabel continuation;
422 WasmCode::RuntimeStubId stub;
423 WasmCodePosition position;
424 LiftoffRegList regs_to_save;
425 Register cached_instance;
426 OutOfLineSafepointInfo* safepoint_info;
427 uint32_t pc; // for trap handler.
428 // These two pointers will only be used for debug code:
429 SpilledRegistersForInspection* spilled_registers;
430 DebugSideTableBuilder::EntryBuilder* debug_sidetable_entry_builder;
431
432 // Named constructors:
Trapv8::internal::wasm::__anon3bd82aa60111::LiftoffCompiler::OutOfLineCode433 static OutOfLineCode Trap(
434 WasmCode::RuntimeStubId s, WasmCodePosition pos,
435 SpilledRegistersForInspection* spilled_registers,
436 OutOfLineSafepointInfo* safepoint_info, uint32_t pc,
437 DebugSideTableBuilder::EntryBuilder* debug_sidetable_entry_builder) {
438 DCHECK_LT(0, pos);
439 return {
440 {}, // label
441 {}, // continuation
442 s, // stub
443 pos, // position
444 {}, // regs_to_save
445 no_reg, // cached_instance
446 safepoint_info, // safepoint_info
447 pc, // pc
448 spilled_registers, // spilled_registers
449 debug_sidetable_entry_builder // debug_side_table_entry_builder
450 };
451 }
StackCheckv8::internal::wasm::__anon3bd82aa60111::LiftoffCompiler::OutOfLineCode452 static OutOfLineCode StackCheck(
453 WasmCodePosition pos, LiftoffRegList regs_to_save,
454 Register cached_instance, SpilledRegistersForInspection* spilled_regs,
455 OutOfLineSafepointInfo* safepoint_info,
456 DebugSideTableBuilder::EntryBuilder* debug_sidetable_entry_builder) {
457 return {
458 {}, // label
459 {}, // continuation
460 WasmCode::kWasmStackGuard, // stub
461 pos, // position
462 regs_to_save, // regs_to_save
463 cached_instance, // cached_instance
464 safepoint_info, // safepoint_info
465 0, // pc
466 spilled_regs, // spilled_registers
467 debug_sidetable_entry_builder // debug_side_table_entry_builder
468 };
469 }
TierupCheckv8::internal::wasm::__anon3bd82aa60111::LiftoffCompiler::OutOfLineCode470 static OutOfLineCode TierupCheck(
471 WasmCodePosition pos, LiftoffRegList regs_to_save,
472 Register cached_instance, SpilledRegistersForInspection* spilled_regs,
473 OutOfLineSafepointInfo* safepoint_info,
474 DebugSideTableBuilder::EntryBuilder* debug_sidetable_entry_builder) {
475 return {
476 {}, // label
477 {}, // continuation,
478 WasmCode::kWasmTriggerTierUp, // stub
479 pos, // position
480 regs_to_save, // regs_to_save
481 cached_instance, // cached_instance
482 safepoint_info, // safepoint_info
483 0, // pc
484 spilled_regs, // spilled_registers
485 debug_sidetable_entry_builder // debug_side_table_entry_builder
486 };
487 }
488 };
489
LiftoffCompiler(compiler::CallDescriptor * call_descriptor,CompilationEnv * env,Zone * compilation_zone,std::unique_ptr<AssemblerBuffer> buffer,DebugSideTableBuilder * debug_sidetable_builder,ForDebugging for_debugging,int func_index,base::Vector<const int> breakpoints={},int dead_breakpoint=0,int32_t * max_steps=nullptr,int32_t * nondeterminism=nullptr)490 LiftoffCompiler(compiler::CallDescriptor* call_descriptor,
491 CompilationEnv* env, Zone* compilation_zone,
492 std::unique_ptr<AssemblerBuffer> buffer,
493 DebugSideTableBuilder* debug_sidetable_builder,
494 ForDebugging for_debugging, int func_index,
495 base::Vector<const int> breakpoints = {},
496 int dead_breakpoint = 0, int32_t* max_steps = nullptr,
497 int32_t* nondeterminism = nullptr)
498 : asm_(std::move(buffer)),
499 descriptor_(
500 GetLoweredCallDescriptor(compilation_zone, call_descriptor)),
501 env_(env),
502 debug_sidetable_builder_(debug_sidetable_builder),
503 for_debugging_(for_debugging),
504 func_index_(func_index),
505 out_of_line_code_(compilation_zone),
506 source_position_table_builder_(compilation_zone),
507 protected_instructions_(compilation_zone),
508 compilation_zone_(compilation_zone),
509 safepoint_table_builder_(compilation_zone_),
510 next_breakpoint_ptr_(breakpoints.begin()),
511 next_breakpoint_end_(breakpoints.end()),
512 dead_breakpoint_(dead_breakpoint),
513 handlers_(compilation_zone),
514 max_steps_(max_steps),
515 nondeterminism_(nondeterminism) {
516 if (breakpoints.empty()) {
517 next_breakpoint_ptr_ = next_breakpoint_end_ = nullptr;
518 }
519 }
520
did_bailout() const521 bool did_bailout() const { return bailout_reason_ != kSuccess; }
bailout_reason() const522 LiftoffBailoutReason bailout_reason() const { return bailout_reason_; }
523
GetCode(CodeDesc * desc)524 void GetCode(CodeDesc* desc) {
525 asm_.GetCode(nullptr, desc, &safepoint_table_builder_,
526 handler_table_offset_);
527 }
528
ReleaseBuffer()529 std::unique_ptr<AssemblerBuffer> ReleaseBuffer() {
530 return asm_.ReleaseBuffer();
531 }
532
GetSourcePositionTable()533 base::OwnedVector<uint8_t> GetSourcePositionTable() {
534 return source_position_table_builder_.ToSourcePositionTableVector();
535 }
536
GetProtectedInstructionsData() const537 base::OwnedVector<uint8_t> GetProtectedInstructionsData() const {
538 return base::OwnedVector<uint8_t>::Of(base::Vector<const uint8_t>::cast(
539 base::VectorOf(protected_instructions_)));
540 }
541
GetTotalFrameSlotCountForGC() const542 uint32_t GetTotalFrameSlotCountForGC() const {
543 return __ GetTotalFrameSlotCountForGC();
544 }
545
GetFeedbackVectorSlots() const546 int GetFeedbackVectorSlots() const {
547 // The number of instructions is capped by max function size.
548 STATIC_ASSERT(kV8MaxWasmFunctionSize < std::numeric_limits<int>::max());
549 return static_cast<int>(num_call_instructions_) * 2;
550 }
551
unsupported(FullDecoder * decoder,LiftoffBailoutReason reason,const char * detail)552 void unsupported(FullDecoder* decoder, LiftoffBailoutReason reason,
553 const char* detail) {
554 DCHECK_NE(kSuccess, reason);
555 if (did_bailout()) return;
556 bailout_reason_ = reason;
557 TRACE("unsupported: %s\n", detail);
558 decoder->errorf(decoder->pc_offset(), "unsupported liftoff operation: %s",
559 detail);
560 UnuseLabels(decoder);
561 CheckBailoutAllowed(reason, detail, env_);
562 }
563
DidAssemblerBailout(FullDecoder * decoder)564 bool DidAssemblerBailout(FullDecoder* decoder) {
565 if (decoder->failed() || !__ did_bailout()) return false;
566 unsupported(decoder, __ bailout_reason(), __ bailout_detail());
567 return true;
568 }
569
CheckSupportedType(FullDecoder * decoder,ValueKind kind,const char * context)570 V8_INLINE bool CheckSupportedType(FullDecoder* decoder, ValueKind kind,
571 const char* context) {
572 if (V8_LIKELY(supported_types_.contains(kind))) return true;
573 return MaybeBailoutForUnsupportedType(decoder, kind, context);
574 }
575
MaybeBailoutForUnsupportedType(FullDecoder * decoder,ValueKind kind,const char * context)576 V8_NOINLINE bool MaybeBailoutForUnsupportedType(FullDecoder* decoder,
577 ValueKind kind,
578 const char* context) {
579 DCHECK(!supported_types_.contains(kind));
580
581 // Lazily update {supported_types_}; then check again.
582 if (CpuFeatures::SupportsWasmSimd128()) supported_types_.Add(kS128);
583 if (supported_types_.contains(kind)) return true;
584
585 LiftoffBailoutReason bailout_reason;
586 switch (kind) {
587 case kS128:
588 bailout_reason = kMissingCPUFeature;
589 break;
590 case kRef:
591 case kOptRef:
592 case kRtt:
593 case kI8:
594 case kI16:
595 bailout_reason = kGC;
596 break;
597 default:
598 UNREACHABLE();
599 }
600 base::EmbeddedVector<char, 128> buffer;
601 SNPrintF(buffer, "%s %s", name(kind), context);
602 unsupported(decoder, bailout_reason, buffer.begin());
603 return false;
604 }
605
UnuseLabels(FullDecoder * decoder)606 void UnuseLabels(FullDecoder* decoder) {
607 #ifdef DEBUG
608 auto Unuse = [](Label* label) {
609 label->Unuse();
610 label->UnuseNear();
611 };
612 // Unuse all labels now, otherwise their destructor will fire a DCHECK error
613 // if they where referenced before.
614 uint32_t control_depth = decoder ? decoder->control_depth() : 0;
615 for (uint32_t i = 0; i < control_depth; ++i) {
616 Control* c = decoder->control_at(i);
617 Unuse(c->label.get());
618 if (c->else_state) Unuse(c->else_state->label.get());
619 if (c->try_info != nullptr) Unuse(&c->try_info->catch_label);
620 }
621 for (auto& ool : out_of_line_code_) Unuse(ool.label.get());
622 #endif
623 }
624
StartFunction(FullDecoder * decoder)625 void StartFunction(FullDecoder* decoder) {
626 if (FLAG_trace_liftoff && !FLAG_trace_wasm_decoder) {
627 StdoutStream{} << "hint: add --trace-wasm-decoder to also see the wasm "
628 "instructions being decoded\n";
629 }
630 int num_locals = decoder->num_locals();
631 __ set_num_locals(num_locals);
632 for (int i = 0; i < num_locals; ++i) {
633 ValueKind kind = decoder->local_type(i).kind();
634 __ set_local_kind(i, kind);
635 }
636 }
637
RegsUnusedByParams()638 constexpr static LiftoffRegList RegsUnusedByParams() {
639 LiftoffRegList regs = kGpCacheRegList;
640 for (auto reg : kGpParamRegisters) {
641 regs.clear(reg);
642 }
643 return regs;
644 }
645
646 // Returns the number of inputs processed (1 or 2).
ProcessParameter(ValueKind kind,uint32_t input_idx)647 uint32_t ProcessParameter(ValueKind kind, uint32_t input_idx) {
648 const bool needs_pair = needs_gp_reg_pair(kind);
649 const ValueKind reg_kind = needs_pair ? kI32 : kind;
650 const RegClass rc = reg_class_for(reg_kind);
651
652 auto LoadToReg = [this, reg_kind, rc](compiler::LinkageLocation location,
653 LiftoffRegList pinned) {
654 if (location.IsRegister()) {
655 DCHECK(!location.IsAnyRegister());
656 return LiftoffRegister::from_external_code(rc, reg_kind,
657 location.AsRegister());
658 }
659 DCHECK(location.IsCallerFrameSlot());
660 // For reference type parameters we have to use registers that were not
661 // used for parameters because some reference type stack parameters may
662 // get processed before some value type register parameters.
663 static constexpr auto kRegsUnusedByParams = RegsUnusedByParams();
664 LiftoffRegister reg = is_reference(reg_kind)
665 ? __ GetUnusedRegister(kRegsUnusedByParams)
666 : __ GetUnusedRegister(rc, pinned);
667 __ LoadCallerFrameSlot(reg, -location.AsCallerFrameSlot(), reg_kind);
668 return reg;
669 };
670
671 LiftoffRegister reg =
672 LoadToReg(descriptor_->GetInputLocation(input_idx), {});
673 if (needs_pair) {
674 LiftoffRegister reg2 = LoadToReg(
675 descriptor_->GetInputLocation(input_idx + 1), LiftoffRegList{reg});
676 reg = LiftoffRegister::ForPair(reg.gp(), reg2.gp());
677 }
678 __ PushRegister(kind, reg);
679
680 return needs_pair ? 2 : 1;
681 }
682
StackCheck(FullDecoder * decoder,WasmCodePosition position)683 void StackCheck(FullDecoder* decoder, WasmCodePosition position) {
684 CODE_COMMENT("stack check");
685 if (!FLAG_wasm_stack_checks || !env_->runtime_exception_support) return;
686
687 // Loading the limit address can change the stack state, hence do this
688 // before storing information about registers.
689 Register limit_address = __ GetUnusedRegister(kGpReg, {}).gp();
690 LOAD_INSTANCE_FIELD(limit_address, StackLimitAddress, kSystemPointerSize,
691 {});
692
693 LiftoffRegList regs_to_save = __ cache_state()->used_registers;
694 // The cached instance will be reloaded separately.
695 if (__ cache_state()->cached_instance != no_reg) {
696 DCHECK(regs_to_save.has(__ cache_state()->cached_instance));
697 regs_to_save.clear(__ cache_state()->cached_instance);
698 }
699 SpilledRegistersForInspection* spilled_regs = nullptr;
700
701 OutOfLineSafepointInfo* safepoint_info =
702 compilation_zone_->New<OutOfLineSafepointInfo>(compilation_zone_);
703 __ cache_state()->GetTaggedSlotsForOOLCode(
704 &safepoint_info->slots, &safepoint_info->spills,
705 for_debugging_
706 ? LiftoffAssembler::CacheState::SpillLocation::kStackSlots
707 : LiftoffAssembler::CacheState::SpillLocation::kTopOfStack);
708 if (V8_UNLIKELY(for_debugging_)) {
709 // When debugging, we do not just push all registers to the stack, but we
710 // spill them to their proper stack locations such that we can inspect
711 // them.
712 // The only exception is the cached memory start, which we just push
713 // before the stack check and pop afterwards.
714 regs_to_save = {};
715 if (__ cache_state()->cached_mem_start != no_reg) {
716 regs_to_save.set(__ cache_state()->cached_mem_start);
717 }
718 spilled_regs = GetSpilledRegistersForInspection();
719 }
720 out_of_line_code_.push_back(OutOfLineCode::StackCheck(
721 position, regs_to_save, __ cache_state()->cached_instance, spilled_regs,
722 safepoint_info, RegisterOOLDebugSideTableEntry(decoder)));
723 OutOfLineCode& ool = out_of_line_code_.back();
724 __ StackCheck(ool.label.get(), limit_address);
725 __ bind(ool.continuation.get());
726 }
727
TierupCheck(FullDecoder * decoder,WasmCodePosition position,int budget_used)728 void TierupCheck(FullDecoder* decoder, WasmCodePosition position,
729 int budget_used) {
730 // We should always decrement the budget, and we don't expect integer
731 // overflows in the budget calculation.
732 DCHECK_LE(1, budget_used);
733
734 if (for_debugging_ != kNoDebugging) return;
735 CODE_COMMENT("tierup check");
736 // We never want to blow the entire budget at once.
737 const int kMax = FLAG_wasm_tiering_budget / 4;
738 if (budget_used > kMax) budget_used = kMax;
739
740 LiftoffRegister budget_reg = __ GetUnusedRegister(kGpReg, {});
741 __ Fill(budget_reg, liftoff::kTierupBudgetOffset, ValueKind::kI32);
742 LiftoffRegList regs_to_save = __ cache_state()->used_registers;
743 // The cached instance will be reloaded separately.
744 if (__ cache_state()->cached_instance != no_reg) {
745 DCHECK(regs_to_save.has(__ cache_state()->cached_instance));
746 regs_to_save.clear(__ cache_state()->cached_instance);
747 }
748 SpilledRegistersForInspection* spilled_regs = nullptr;
749
750 OutOfLineSafepointInfo* safepoint_info =
751 compilation_zone_->New<OutOfLineSafepointInfo>(compilation_zone_);
752 __ cache_state()->GetTaggedSlotsForOOLCode(
753 &safepoint_info->slots, &safepoint_info->spills,
754 LiftoffAssembler::CacheState::SpillLocation::kTopOfStack);
755 out_of_line_code_.push_back(OutOfLineCode::TierupCheck(
756 position, regs_to_save, __ cache_state()->cached_instance, spilled_regs,
757 safepoint_info, RegisterOOLDebugSideTableEntry(decoder)));
758 OutOfLineCode& ool = out_of_line_code_.back();
759 __ emit_i32_subi_jump_negative(budget_reg.gp(), budget_used,
760 ool.label.get());
761 __ Spill(liftoff::kTierupBudgetOffset, budget_reg, ValueKind::kI32);
762 __ bind(ool.continuation.get());
763 }
764
SpillLocalsInitially(FullDecoder * decoder,uint32_t num_params)765 bool SpillLocalsInitially(FullDecoder* decoder, uint32_t num_params) {
766 int actual_locals = __ num_locals() - num_params;
767 DCHECK_LE(0, actual_locals);
768 constexpr int kNumCacheRegisters = kLiftoffAssemblerGpCacheRegs.Count();
769 // If we have many locals, we put them on the stack initially. This avoids
770 // having to spill them on merge points. Use of these initial values should
771 // be rare anyway.
772 if (actual_locals > kNumCacheRegisters / 2) return true;
773 // If there are locals which are not i32 or i64, we also spill all locals,
774 // because other types cannot be initialized to constants.
775 for (uint32_t param_idx = num_params; param_idx < __ num_locals();
776 ++param_idx) {
777 ValueKind kind = __ local_kind(param_idx);
778 if (kind != kI32 && kind != kI64) return true;
779 }
780 return false;
781 }
782
TraceFunctionEntry(FullDecoder * decoder)783 void TraceFunctionEntry(FullDecoder* decoder) {
784 CODE_COMMENT("trace function entry");
785 __ SpillAllRegisters();
786 source_position_table_builder_.AddPosition(
787 __ pc_offset(), SourcePosition(decoder->position()), false);
788 __ CallRuntimeStub(WasmCode::kWasmTraceEnter);
789 DefineSafepoint();
790 }
791
dynamic_tiering()792 bool dynamic_tiering() {
793 return env_->dynamic_tiering == DynamicTiering::kEnabled &&
794 for_debugging_ == kNoDebugging &&
795 (FLAG_wasm_tier_up_filter == -1 ||
796 FLAG_wasm_tier_up_filter == func_index_);
797 }
798
StartFunctionBody(FullDecoder * decoder,Control * block)799 void StartFunctionBody(FullDecoder* decoder, Control* block) {
800 for (uint32_t i = 0; i < __ num_locals(); ++i) {
801 if (!CheckSupportedType(decoder, __ local_kind(i), "param")) return;
802 }
803
804 // Parameter 0 is the instance parameter.
805 uint32_t num_params =
806 static_cast<uint32_t>(decoder->sig_->parameter_count());
807
808 __ CodeEntry();
809
810 __ EnterFrame(StackFrame::WASM);
811 __ set_has_frame(true);
812 pc_offset_stack_frame_construction_ = __ PrepareStackFrame();
813 // {PrepareStackFrame} is the first platform-specific assembler method.
814 // If this failed, we can bail out immediately, avoiding runtime overhead
815 // and potential failures because of other unimplemented methods.
816 // A platform implementing {PrepareStackFrame} must ensure that we can
817 // finish compilation without errors even if we hit unimplemented
818 // LiftoffAssembler methods.
819 if (DidAssemblerBailout(decoder)) return;
820
821 // Input 0 is the call target, the instance is at 1.
822 constexpr int kInstanceParameterIndex = 1;
823 // Check that {kWasmInstanceRegister} matches our call descriptor.
824 DCHECK_EQ(kWasmInstanceRegister,
825 Register::from_code(
826 descriptor_->GetInputLocation(kInstanceParameterIndex)
827 .AsRegister()));
828 __ cache_state()->SetInstanceCacheRegister(kWasmInstanceRegister);
829 // Load the feedback vector and cache it in a stack slot.
830 constexpr LiftoffRegList kGpParamRegisters = GetGpParamRegisters();
831 if (FLAG_wasm_speculative_inlining) {
832 CODE_COMMENT("load feedback vector");
833 int declared_func_index =
834 func_index_ - env_->module->num_imported_functions;
835 DCHECK_GE(declared_func_index, 0);
836 LiftoffRegList pinned = kGpParamRegisters;
837 LiftoffRegister tmp = pinned.set(__ GetUnusedRegister(kGpReg, pinned));
838 __ LoadTaggedPointerFromInstance(
839 tmp.gp(), kWasmInstanceRegister,
840 WASM_INSTANCE_OBJECT_FIELD_OFFSET(FeedbackVectors));
841 __ LoadTaggedPointer(tmp.gp(), tmp.gp(), no_reg,
842 wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(
843 declared_func_index),
844 pinned);
845 __ Spill(liftoff::kFeedbackVectorOffset, tmp, kPointerKind);
846 }
847 if (dynamic_tiering()) {
848 CODE_COMMENT("load tier up budget");
849 LiftoffRegList pinned = kGpParamRegisters;
850 LiftoffRegister tmp = pinned.set(__ GetUnusedRegister(kGpReg, pinned));
851 LOAD_INSTANCE_FIELD(tmp.gp(), TieringBudgetArray, kSystemPointerSize,
852 pinned);
853 uint32_t offset =
854 kInt32Size * declared_function_index(env_->module, func_index_);
855 __ Load(tmp, tmp.gp(), no_reg, offset, LoadType::kI32Load, pinned);
856 __ Spill(liftoff::kTierupBudgetOffset, tmp, ValueKind::kI32);
857 }
858 if (for_debugging_) __ ResetOSRTarget();
859
860 // Process parameters.
861 if (num_params) CODE_COMMENT("process parameters");
862 // Input 0 is the code target, 1 is the instance. First parameter at 2.
863 uint32_t input_idx = kInstanceParameterIndex + 1;
864 for (uint32_t param_idx = 0; param_idx < num_params; ++param_idx) {
865 input_idx += ProcessParameter(__ local_kind(param_idx), input_idx);
866 }
867 int params_size = __ TopSpillOffset();
868 DCHECK_EQ(input_idx, descriptor_->InputCount());
869
870 // Initialize locals beyond parameters.
871 if (num_params < __ num_locals()) CODE_COMMENT("init locals");
872 if (SpillLocalsInitially(decoder, num_params)) {
873 bool has_refs = false;
874 for (uint32_t param_idx = num_params; param_idx < __ num_locals();
875 ++param_idx) {
876 ValueKind kind = __ local_kind(param_idx);
877 has_refs |= is_reference(kind);
878 __ PushStack(kind);
879 }
880 int spill_size = __ TopSpillOffset() - params_size;
881 __ FillStackSlotsWithZero(params_size, spill_size);
882
883 // Initialize all reference type locals with ref.null.
884 if (has_refs) {
885 Register null_ref_reg = __ GetUnusedRegister(kGpReg, {}).gp();
886 LoadNullValue(null_ref_reg, {});
887 for (uint32_t local_index = num_params; local_index < __ num_locals();
888 ++local_index) {
889 ValueKind kind = __ local_kind(local_index);
890 if (is_reference(kind)) {
891 __ Spill(__ cache_state()->stack_state[local_index].offset(),
892 LiftoffRegister(null_ref_reg), kind);
893 }
894 }
895 }
896 } else {
897 for (uint32_t param_idx = num_params; param_idx < __ num_locals();
898 ++param_idx) {
899 ValueKind kind = __ local_kind(param_idx);
900 // Anything which is not i32 or i64 requires spilling.
901 DCHECK(kind == kI32 || kind == kI64);
902 __ PushConstant(kind, int32_t{0});
903 }
904 }
905
906 DCHECK_EQ(__ num_locals(), __ cache_state()->stack_height());
907
908 if (V8_UNLIKELY(debug_sidetable_builder_)) {
909 debug_sidetable_builder_->SetNumLocals(__ num_locals());
910 }
911
912 // The function-prologue stack check is associated with position 0, which
913 // is never a position of any instruction in the function.
914 StackCheck(decoder, 0);
915
916 if (FLAG_trace_wasm) TraceFunctionEntry(decoder);
917 }
918
GenerateOutOfLineCode(OutOfLineCode * ool)919 void GenerateOutOfLineCode(OutOfLineCode* ool) {
920 CODE_COMMENT(
921 (std::string("OOL: ") + GetRuntimeStubName(ool->stub)).c_str());
922 __ bind(ool->label.get());
923 const bool is_stack_check = ool->stub == WasmCode::kWasmStackGuard;
924 const bool is_tierup = ool->stub == WasmCode::kWasmTriggerTierUp;
925
926 // Only memory OOB traps need a {pc}, but not unconditionally. Static OOB
927 // accesses do not need protected instruction information, hence they also
928 // do not set {pc}.
929 DCHECK_IMPLIES(ool->stub != WasmCode::kThrowWasmTrapMemOutOfBounds,
930 ool->pc == 0);
931
932 if (env_->bounds_checks == kTrapHandler && ool->pc != 0) {
933 uint32_t pc = static_cast<uint32_t>(__ pc_offset());
934 DCHECK_EQ(pc, __ pc_offset());
935 protected_instructions_.emplace_back(
936 trap_handler::ProtectedInstructionData{ool->pc, pc});
937 }
938
939 if (!env_->runtime_exception_support) {
940 // We cannot test calls to the runtime in cctest/test-run-wasm.
941 // Therefore we emit a call to C here instead of a call to the runtime.
942 // In this mode, we never generate stack checks.
943 DCHECK(!is_stack_check);
944 __ CallTrapCallbackForTesting();
945 __ LeaveFrame(StackFrame::WASM);
946 __ DropStackSlotsAndRet(
947 static_cast<uint32_t>(descriptor_->ParameterSlotCount()));
948 return;
949 }
950
951 if (!ool->regs_to_save.is_empty()) {
952 __ PushRegisters(ool->regs_to_save);
953 }
954 if (V8_UNLIKELY(ool->spilled_registers != nullptr)) {
955 for (auto& entry : ool->spilled_registers->entries) {
956 // We should not push and spill the same register.
957 DCHECK(!ool->regs_to_save.has(entry.reg));
958 __ Spill(entry.offset, entry.reg, entry.kind);
959 }
960 }
961
962 source_position_table_builder_.AddPosition(
963 __ pc_offset(), SourcePosition(ool->position), true);
964 __ CallRuntimeStub(ool->stub);
965 auto safepoint = safepoint_table_builder_.DefineSafepoint(&asm_);
966
967 if (ool->safepoint_info) {
968 for (auto index : ool->safepoint_info->slots) {
969 safepoint.DefineTaggedStackSlot(index);
970 }
971
972 int total_frame_size = __ GetTotalFrameSize();
973 LiftoffRegList gp_regs = ool->regs_to_save & kGpCacheRegList;
974 // {total_frame_size} is the highest offset from the FP that is used to
975 // store a value. The offset of the first spill slot should therefore be
976 // {(total_frame_size / kSystemPointerSize) + 1}. However, spill slots
977 // don't start at offset '0' but at offset '-1' (or
978 // {-kSystemPointerSize}). Therefore we have to add another '+ 1' to the
979 // index of the first spill slot.
980 int index = (total_frame_size / kSystemPointerSize) + 2;
981
982 __ RecordSpillsInSafepoint(safepoint, gp_regs,
983 ool->safepoint_info->spills, index);
984 }
985 if (is_tierup) {
986 // Reset the budget.
987 __ Spill(liftoff::kTierupBudgetOffset,
988 WasmValue(FLAG_wasm_tiering_budget));
989 }
990
991 DCHECK_EQ(!debug_sidetable_builder_, !ool->debug_sidetable_entry_builder);
992 if (V8_UNLIKELY(ool->debug_sidetable_entry_builder)) {
993 ool->debug_sidetable_entry_builder->set_pc_offset(__ pc_offset());
994 }
995 DCHECK_EQ(ool->continuation.get()->is_bound(), is_stack_check || is_tierup);
996 if (is_stack_check) {
997 MaybeOSR();
998 }
999 if (!ool->regs_to_save.is_empty()) __ PopRegisters(ool->regs_to_save);
1000 if (is_stack_check || is_tierup) {
1001 if (V8_UNLIKELY(ool->spilled_registers != nullptr)) {
1002 DCHECK(for_debugging_);
1003 for (auto& entry : ool->spilled_registers->entries) {
1004 __ Fill(entry.reg, entry.offset, entry.kind);
1005 }
1006 }
1007 if (ool->cached_instance != no_reg) {
1008 __ LoadInstanceFromFrame(ool->cached_instance);
1009 }
1010 __ emit_jump(ool->continuation.get());
1011 } else {
1012 __ AssertUnreachable(AbortReason::kUnexpectedReturnFromWasmTrap);
1013 }
1014 }
1015
FinishFunction(FullDecoder * decoder)1016 void FinishFunction(FullDecoder* decoder) {
1017 if (DidAssemblerBailout(decoder)) return;
1018 __ AlignFrameSize();
1019 #if DEBUG
1020 int frame_size = __ GetTotalFrameSize();
1021 #endif
1022 for (OutOfLineCode& ool : out_of_line_code_) {
1023 GenerateOutOfLineCode(&ool);
1024 }
1025 DCHECK_EQ(frame_size, __ GetTotalFrameSize());
1026 __ PatchPrepareStackFrame(pc_offset_stack_frame_construction_,
1027 &safepoint_table_builder_);
1028 __ FinishCode();
1029 safepoint_table_builder_.Emit(&asm_, __ GetTotalFrameSlotCountForGC());
1030 // Emit the handler table.
1031 if (!handlers_.empty()) {
1032 handler_table_offset_ = HandlerTable::EmitReturnTableStart(&asm_);
1033 for (auto& handler : handlers_) {
1034 HandlerTable::EmitReturnEntry(&asm_, handler.pc_offset,
1035 handler.handler.get()->pos());
1036 }
1037 }
1038 __ MaybeEmitOutOfLineConstantPool();
1039 // The previous calls may have also generated a bailout.
1040 DidAssemblerBailout(decoder);
1041 DCHECK_EQ(num_exceptions_, 0);
1042 }
1043
OnFirstError(FullDecoder * decoder)1044 void OnFirstError(FullDecoder* decoder) {
1045 if (!did_bailout()) bailout_reason_ = kDecodeError;
1046 UnuseLabels(decoder);
1047 asm_.AbortCompilation();
1048 }
1049
EmitDebuggingInfo(FullDecoder * decoder,WasmOpcode opcode)1050 V8_NOINLINE void EmitDebuggingInfo(FullDecoder* decoder, WasmOpcode opcode) {
1051 DCHECK(for_debugging_);
1052 if (!WasmOpcodes::IsBreakable(opcode)) return;
1053 bool has_breakpoint = false;
1054 if (next_breakpoint_ptr_) {
1055 if (*next_breakpoint_ptr_ == 0) {
1056 // A single breakpoint at offset 0 indicates stepping.
1057 DCHECK_EQ(next_breakpoint_ptr_ + 1, next_breakpoint_end_);
1058 has_breakpoint = true;
1059 } else {
1060 while (next_breakpoint_ptr_ != next_breakpoint_end_ &&
1061 *next_breakpoint_ptr_ < decoder->position()) {
1062 // Skip unreachable breakpoints.
1063 ++next_breakpoint_ptr_;
1064 }
1065 if (next_breakpoint_ptr_ == next_breakpoint_end_) {
1066 next_breakpoint_ptr_ = next_breakpoint_end_ = nullptr;
1067 } else if (*next_breakpoint_ptr_ == decoder->position()) {
1068 has_breakpoint = true;
1069 }
1070 }
1071 }
1072 if (has_breakpoint) {
1073 CODE_COMMENT("breakpoint");
1074 EmitBreakpoint(decoder);
1075 // Once we emitted an unconditional breakpoint, we don't need to check
1076 // function entry breaks any more.
1077 did_function_entry_break_checks_ = true;
1078 } else if (!did_function_entry_break_checks_) {
1079 did_function_entry_break_checks_ = true;
1080 CODE_COMMENT("check function entry break");
1081 Label do_break;
1082 Label no_break;
1083 Register flag = __ GetUnusedRegister(kGpReg, {}).gp();
1084
1085 // Check the "hook on function call" flag. If set, trigger a break.
1086 LOAD_INSTANCE_FIELD(flag, HookOnFunctionCallAddress, kSystemPointerSize,
1087 {});
1088 __ Load(LiftoffRegister{flag}, flag, no_reg, 0, LoadType::kI32Load8U, {});
1089 __ emit_cond_jump(kNotEqualZero, &do_break, kI32, flag);
1090
1091 // Check if we should stop on "script entry".
1092 LOAD_INSTANCE_FIELD(flag, BreakOnEntry, kUInt8Size, {});
1093 __ emit_cond_jump(kEqualZero, &no_break, kI32, flag);
1094
1095 __ bind(&do_break);
1096 EmitBreakpoint(decoder);
1097 __ bind(&no_break);
1098 } else if (dead_breakpoint_ == decoder->position()) {
1099 DCHECK(!next_breakpoint_ptr_ ||
1100 *next_breakpoint_ptr_ != dead_breakpoint_);
1101 // The top frame is paused at this position, but the breakpoint was
1102 // removed. Adding a dead breakpoint here ensures that the source
1103 // position exists, and that the offset to the return address is the
1104 // same as in the old code.
1105 CODE_COMMENT("dead breakpoint");
1106 Label cont;
1107 __ emit_jump(&cont);
1108 EmitBreakpoint(decoder);
1109 __ bind(&cont);
1110 }
1111 if (V8_UNLIKELY(max_steps_ != nullptr)) {
1112 CODE_COMMENT("check max steps");
1113 LiftoffRegList pinned;
1114 LiftoffRegister max_steps = __ GetUnusedRegister(kGpReg, {});
1115 pinned.set(max_steps);
1116 LiftoffRegister max_steps_addr = __ GetUnusedRegister(kGpReg, pinned);
1117 pinned.set(max_steps_addr);
1118 __ LoadConstant(
1119 max_steps_addr,
1120 WasmValue::ForUintPtr(reinterpret_cast<uintptr_t>(max_steps_)));
1121 __ Load(max_steps, max_steps_addr.gp(), no_reg, 0, LoadType::kI32Load,
1122 pinned);
1123 Label cont;
1124 __ emit_i32_cond_jumpi(kUnequal, &cont, max_steps.gp(), 0);
1125 // Abort.
1126 Trap(decoder, kTrapUnreachable);
1127 __ bind(&cont);
1128 __ emit_i32_subi(max_steps.gp(), max_steps.gp(), 1);
1129 __ Store(max_steps_addr.gp(), no_reg, 0, max_steps, StoreType::kI32Store,
1130 pinned);
1131 }
1132 }
1133
NextInstruction(FullDecoder * decoder,WasmOpcode opcode)1134 void NextInstruction(FullDecoder* decoder, WasmOpcode opcode) {
1135 // Add a single check, so that the fast path can be inlined while
1136 // {EmitDebuggingInfo} stays outlined.
1137 if (V8_UNLIKELY(for_debugging_)) EmitDebuggingInfo(decoder, opcode);
1138 TraceCacheState(decoder);
1139 SLOW_DCHECK(__ ValidateCacheState());
1140 CODE_COMMENT(WasmOpcodes::OpcodeName(
1141 WasmOpcodes::IsPrefixOpcode(opcode)
1142 ? decoder->read_prefixed_opcode<Decoder::kFullValidation>(
1143 decoder->pc())
1144 : opcode));
1145 }
1146
EmitBreakpoint(FullDecoder * decoder)1147 void EmitBreakpoint(FullDecoder* decoder) {
1148 DCHECK(for_debugging_);
1149 source_position_table_builder_.AddPosition(
1150 __ pc_offset(), SourcePosition(decoder->position()), true);
1151 __ CallRuntimeStub(WasmCode::kWasmDebugBreak);
1152 DefineSafepointWithCalleeSavedRegisters();
1153 RegisterDebugSideTableEntry(decoder,
1154 DebugSideTableBuilder::kAllowRegisters);
1155 MaybeOSR();
1156 }
1157
PushControl(Control * block)1158 void PushControl(Control* block) {
1159 // The Liftoff stack includes implicit exception refs stored for catch
1160 // blocks, so that they can be rethrown.
1161 block->num_exceptions = num_exceptions_;
1162 }
1163
Block(FullDecoder * decoder,Control * block)1164 void Block(FullDecoder* decoder, Control* block) { PushControl(block); }
1165
Loop(FullDecoder * decoder,Control * loop)1166 void Loop(FullDecoder* decoder, Control* loop) {
1167 // Before entering a loop, spill all locals to the stack, in order to free
1168 // the cache registers, and to avoid unnecessarily reloading stack values
1169 // into registers at branches.
1170 // TODO(clemensb): Come up with a better strategy here, involving
1171 // pre-analysis of the function.
1172 __ SpillLocals();
1173
1174 __ PrepareLoopArgs(loop->start_merge.arity);
1175
1176 // Loop labels bind at the beginning of the block.
1177 __ bind(loop->label.get());
1178
1179 // Save the current cache state for the merge when jumping to this loop.
1180 loop->label_state.Split(*__ cache_state());
1181
1182 PushControl(loop);
1183
1184 if (!dynamic_tiering()) {
1185 // When the budget-based tiering mechanism is enabled, use that to
1186 // check for interrupt requests; otherwise execute a stack check in the
1187 // loop header.
1188 StackCheck(decoder, decoder->position());
1189 }
1190 }
1191
Try(FullDecoder * decoder,Control * block)1192 void Try(FullDecoder* decoder, Control* block) {
1193 block->try_info = std::make_unique<TryInfo>();
1194 PushControl(block);
1195 }
1196
1197 // Load the property in {kReturnRegister0}.
GetExceptionProperty(LiftoffAssembler::VarState & exception,RootIndex root_index)1198 LiftoffRegister GetExceptionProperty(LiftoffAssembler::VarState& exception,
1199 RootIndex root_index) {
1200 DCHECK(root_index == RootIndex::kwasm_exception_tag_symbol ||
1201 root_index == RootIndex::kwasm_exception_values_symbol);
1202
1203 LiftoffRegList pinned;
1204 LiftoffRegister tag_symbol_reg =
1205 pinned.set(__ GetUnusedRegister(kGpReg, pinned));
1206 LoadExceptionSymbol(tag_symbol_reg.gp(), pinned, root_index);
1207 LiftoffRegister context_reg =
1208 pinned.set(__ GetUnusedRegister(kGpReg, pinned));
1209 LOAD_TAGGED_PTR_INSTANCE_FIELD(context_reg.gp(), NativeContext, pinned);
1210
1211 LiftoffAssembler::VarState tag_symbol(kPointerKind, tag_symbol_reg, 0);
1212 LiftoffAssembler::VarState context(kPointerKind, context_reg, 0);
1213
1214 CallRuntimeStub(WasmCode::kWasmGetOwnProperty,
1215 MakeSig::Returns(kPointerKind)
1216 .Params(kPointerKind, kPointerKind, kPointerKind),
1217 {exception, tag_symbol, context}, kNoSourcePosition);
1218
1219 return LiftoffRegister(kReturnRegister0);
1220 }
1221
CatchException(FullDecoder * decoder,const TagIndexImmediate<validate> & imm,Control * block,base::Vector<Value> values)1222 void CatchException(FullDecoder* decoder,
1223 const TagIndexImmediate<validate>& imm, Control* block,
1224 base::Vector<Value> values) {
1225 DCHECK(block->is_try_catch());
1226 __ emit_jump(block->label.get());
1227
1228 // The catch block is unreachable if no possible throws in the try block
1229 // exist. We only build a landing pad if some node in the try block can
1230 // (possibly) throw. Otherwise the catch environments remain empty.
1231 if (!block->try_info->catch_reached) {
1232 block->reachability = kSpecOnlyReachable;
1233 return;
1234 }
1235
1236 // This is the last use of this label. Re-use the field for the label of the
1237 // next catch block, and jump there if the tag does not match.
1238 __ bind(&block->try_info->catch_label);
1239 new (&block->try_info->catch_label) Label();
1240
1241 __ cache_state()->Split(block->try_info->catch_state);
1242
1243 CODE_COMMENT("load caught exception tag");
1244 DCHECK_EQ(__ cache_state()->stack_state.back().kind(), kRef);
1245 LiftoffRegister caught_tag =
1246 GetExceptionProperty(__ cache_state()->stack_state.back(),
1247 RootIndex::kwasm_exception_tag_symbol);
1248 LiftoffRegList pinned;
1249 pinned.set(caught_tag);
1250
1251 CODE_COMMENT("load expected exception tag");
1252 Register imm_tag = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
1253 LOAD_TAGGED_PTR_INSTANCE_FIELD(imm_tag, TagsTable, pinned);
1254 __ LoadTaggedPointer(
1255 imm_tag, imm_tag, no_reg,
1256 wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(imm.index), {});
1257
1258 CODE_COMMENT("compare tags");
1259 Label caught;
1260 __ emit_cond_jump(kEqual, &caught, kI32, imm_tag, caught_tag.gp());
1261 // The tags don't match, merge the current state into the catch state and
1262 // jump to the next handler.
1263 __ MergeFullStackWith(block->try_info->catch_state, *__ cache_state());
1264 __ emit_jump(&block->try_info->catch_label);
1265
1266 __ bind(&caught);
1267 if (!block->try_info->in_handler) {
1268 block->try_info->in_handler = true;
1269 num_exceptions_++;
1270 }
1271 GetExceptionValues(decoder, __ cache_state()->stack_state.back(), imm.tag);
1272 }
1273
Rethrow(FullDecoder * decoder,const LiftoffAssembler::VarState & exception)1274 void Rethrow(FullDecoder* decoder,
1275 const LiftoffAssembler::VarState& exception) {
1276 DCHECK_EQ(exception.kind(), kRef);
1277 CallRuntimeStub(WasmCode::kWasmRethrow, MakeSig::Params(kPointerKind),
1278 {exception}, decoder->position());
1279 }
1280
Delegate(FullDecoder * decoder,uint32_t depth,Control * block)1281 void Delegate(FullDecoder* decoder, uint32_t depth, Control* block) {
1282 DCHECK_EQ(block, decoder->control_at(0));
1283 Control* target = decoder->control_at(depth);
1284 DCHECK(block->is_incomplete_try());
1285 __ bind(&block->try_info->catch_label);
1286 if (block->try_info->catch_reached) {
1287 __ cache_state()->Steal(block->try_info->catch_state);
1288 if (depth == decoder->control_depth() - 1) {
1289 // Delegate to the caller, do not emit a landing pad.
1290 Rethrow(decoder, __ cache_state()->stack_state.back());
1291 MaybeOSR();
1292 } else {
1293 DCHECK(target->is_incomplete_try());
1294 if (!target->try_info->catch_reached) {
1295 target->try_info->catch_state.InitMerge(
1296 *__ cache_state(), __ num_locals(), 1,
1297 target->stack_depth + target->num_exceptions);
1298 target->try_info->catch_reached = true;
1299 }
1300 __ MergeStackWith(target->try_info->catch_state, 1,
1301 LiftoffAssembler::kForwardJump);
1302 __ emit_jump(&target->try_info->catch_label);
1303 }
1304 }
1305 }
1306
Rethrow(FullDecoder * decoder,Control * try_block)1307 void Rethrow(FullDecoder* decoder, Control* try_block) {
1308 int index = try_block->try_info->catch_state.stack_height() - 1;
1309 auto& exception = __ cache_state()->stack_state[index];
1310 Rethrow(decoder, exception);
1311 int pc_offset = __ pc_offset();
1312 MaybeOSR();
1313 EmitLandingPad(decoder, pc_offset);
1314 }
1315
CatchAll(FullDecoder * decoder,Control * block)1316 void CatchAll(FullDecoder* decoder, Control* block) {
1317 DCHECK(block->is_try_catchall() || block->is_try_catch());
1318 DCHECK_EQ(decoder->control_at(0), block);
1319
1320 // The catch block is unreachable if no possible throws in the try block
1321 // exist. We only build a landing pad if some node in the try block can
1322 // (possibly) throw. Otherwise the catch environments remain empty.
1323 if (!block->try_info->catch_reached) {
1324 decoder->SetSucceedingCodeDynamicallyUnreachable();
1325 return;
1326 }
1327
1328 __ bind(&block->try_info->catch_label);
1329 __ cache_state()->Steal(block->try_info->catch_state);
1330 if (!block->try_info->in_handler) {
1331 block->try_info->in_handler = true;
1332 num_exceptions_++;
1333 }
1334 }
1335
JumpIfFalse(FullDecoder * decoder,Label * false_dst)1336 void JumpIfFalse(FullDecoder* decoder, Label* false_dst) {
1337 LiftoffCondition cond =
1338 test_and_reset_outstanding_op(kExprI32Eqz) ? kNotEqualZero : kEqualZero;
1339
1340 if (!has_outstanding_op()) {
1341 // Unary comparison.
1342 Register value = __ PopToRegister().gp();
1343 __ emit_cond_jump(cond, false_dst, kI32, value);
1344 return;
1345 }
1346
1347 // Binary comparison of i32 values.
1348 cond = Negate(GetCompareCondition(outstanding_op_));
1349 outstanding_op_ = kNoOutstandingOp;
1350 LiftoffAssembler::VarState rhs_slot = __ cache_state()->stack_state.back();
1351 if (rhs_slot.is_const()) {
1352 // Compare to a constant.
1353 int32_t rhs_imm = rhs_slot.i32_const();
1354 __ cache_state()->stack_state.pop_back();
1355 Register lhs = __ PopToRegister().gp();
1356 __ emit_i32_cond_jumpi(cond, false_dst, lhs, rhs_imm);
1357 return;
1358 }
1359
1360 Register rhs = __ PopToRegister().gp();
1361 LiftoffAssembler::VarState lhs_slot = __ cache_state()->stack_state.back();
1362 if (lhs_slot.is_const()) {
1363 // Compare a constant to an arbitrary value.
1364 int32_t lhs_imm = lhs_slot.i32_const();
1365 __ cache_state()->stack_state.pop_back();
1366 // Flip the condition, because {lhs} and {rhs} are swapped.
1367 __ emit_i32_cond_jumpi(Flip(cond), false_dst, rhs, lhs_imm);
1368 return;
1369 }
1370
1371 // Compare two arbitrary values.
1372 Register lhs = __ PopToRegister(LiftoffRegList{rhs}).gp();
1373 __ emit_cond_jump(cond, false_dst, kI32, lhs, rhs);
1374 }
1375
If(FullDecoder * decoder,const Value & cond,Control * if_block)1376 void If(FullDecoder* decoder, const Value& cond, Control* if_block) {
1377 DCHECK_EQ(if_block, decoder->control_at(0));
1378 DCHECK(if_block->is_if());
1379
1380 // Allocate the else state.
1381 if_block->else_state = std::make_unique<ElseState>();
1382
1383 // Test the condition on the value stack, jump to else if zero.
1384 JumpIfFalse(decoder, if_block->else_state->label.get());
1385
1386 // Store the state (after popping the value) for executing the else branch.
1387 if_block->else_state->state.Split(*__ cache_state());
1388
1389 PushControl(if_block);
1390 }
1391
FallThruTo(FullDecoder * decoder,Control * c)1392 void FallThruTo(FullDecoder* decoder, Control* c) {
1393 if (!c->end_merge.reached) {
1394 c->label_state.InitMerge(*__ cache_state(), __ num_locals(),
1395 c->end_merge.arity,
1396 c->stack_depth + c->num_exceptions);
1397 }
1398 DCHECK(!c->is_try_catchall());
1399 if (c->is_try_catch()) {
1400 // Drop the implicit exception ref if any. There may be none if this is a
1401 // catch-less try block.
1402 __ MergeStackWith(c->label_state, c->br_merge()->arity,
1403 LiftoffAssembler::kForwardJump);
1404 } else {
1405 __ MergeFullStackWith(c->label_state, *__ cache_state());
1406 }
1407 __ emit_jump(c->label.get());
1408 TraceCacheState(decoder);
1409 }
1410
FinishOneArmedIf(FullDecoder * decoder,Control * c)1411 void FinishOneArmedIf(FullDecoder* decoder, Control* c) {
1412 DCHECK(c->is_onearmed_if());
1413 if (c->end_merge.reached) {
1414 // Someone already merged to the end of the if. Merge both arms into that.
1415 if (c->reachable()) {
1416 // Merge the if state into the end state.
1417 __ MergeFullStackWith(c->label_state, *__ cache_state());
1418 __ emit_jump(c->label.get());
1419 }
1420 // Merge the else state into the end state. Set this state as the current
1421 // state first so helper functions know which registers are in use.
1422 __ bind(c->else_state->label.get());
1423 __ cache_state()->Steal(c->else_state->state);
1424 __ MergeFullStackWith(c->label_state, *__ cache_state());
1425 __ cache_state()->Steal(c->label_state);
1426 } else if (c->reachable()) {
1427 // No merge yet at the end of the if, but we need to create a merge for
1428 // the both arms of this if. Thus init the merge point from the else
1429 // state, then merge the if state into that.
1430 DCHECK_EQ(c->start_merge.arity, c->end_merge.arity);
1431 c->label_state.InitMerge(c->else_state->state, __ num_locals(),
1432 c->start_merge.arity,
1433 c->stack_depth + c->num_exceptions);
1434 __ MergeFullStackWith(c->label_state, *__ cache_state());
1435 __ emit_jump(c->label.get());
1436 // Merge the else state into the end state. Set this state as the current
1437 // state first so helper functions know which registers are in use.
1438 __ bind(c->else_state->label.get());
1439 __ cache_state()->Steal(c->else_state->state);
1440 __ MergeFullStackWith(c->label_state, *__ cache_state());
1441 __ cache_state()->Steal(c->label_state);
1442 } else {
1443 // No merge needed, just continue with the else state.
1444 __ bind(c->else_state->label.get());
1445 __ cache_state()->Steal(c->else_state->state);
1446 }
1447 }
1448
FinishTry(FullDecoder * decoder,Control * c)1449 void FinishTry(FullDecoder* decoder, Control* c) {
1450 DCHECK(c->is_try_catch() || c->is_try_catchall());
1451 if (!c->end_merge.reached) {
1452 if (c->try_info->catch_reached) {
1453 // Drop the implicit exception ref.
1454 __ DropValue(__ num_locals() + c->stack_depth + c->num_exceptions);
1455 }
1456 // Else we did not enter the catch state, continue with the current state.
1457 } else {
1458 if (c->reachable()) {
1459 __ MergeStackWith(c->label_state, c->br_merge()->arity,
1460 LiftoffAssembler::kForwardJump);
1461 }
1462 __ cache_state()->Steal(c->label_state);
1463 }
1464 if (c->try_info->catch_reached) {
1465 num_exceptions_--;
1466 }
1467 }
1468
PopControl(FullDecoder * decoder,Control * c)1469 void PopControl(FullDecoder* decoder, Control* c) {
1470 if (c->is_loop()) return; // A loop just falls through.
1471 if (c->is_onearmed_if()) {
1472 // Special handling for one-armed ifs.
1473 FinishOneArmedIf(decoder, c);
1474 } else if (c->is_try_catch() || c->is_try_catchall()) {
1475 FinishTry(decoder, c);
1476 } else if (c->end_merge.reached) {
1477 // There is a merge already. Merge our state into that, then continue with
1478 // that state.
1479 if (c->reachable()) {
1480 __ MergeFullStackWith(c->label_state, *__ cache_state());
1481 }
1482 __ cache_state()->Steal(c->label_state);
1483 } else {
1484 // No merge, just continue with our current state.
1485 }
1486
1487 if (!c->label.get()->is_bound()) __ bind(c->label.get());
1488 }
1489
GenerateCCall(const LiftoffRegister * result_regs,const ValueKindSig * sig,ValueKind out_argument_kind,const LiftoffRegister * arg_regs,ExternalReference ext_ref)1490 void GenerateCCall(const LiftoffRegister* result_regs,
1491 const ValueKindSig* sig, ValueKind out_argument_kind,
1492 const LiftoffRegister* arg_regs,
1493 ExternalReference ext_ref) {
1494 // Before making a call, spill all cache registers.
1495 __ SpillAllRegisters();
1496
1497 // Store arguments on our stack, then align the stack for calling to C.
1498 int param_bytes = 0;
1499 for (ValueKind param_kind : sig->parameters()) {
1500 param_bytes += value_kind_size(param_kind);
1501 }
1502 int out_arg_bytes =
1503 out_argument_kind == kVoid ? 0 : value_kind_size(out_argument_kind);
1504 int stack_bytes = std::max(param_bytes, out_arg_bytes);
1505 __ CallC(sig, arg_regs, result_regs, out_argument_kind, stack_bytes,
1506 ext_ref);
1507 }
1508
1509 template <typename EmitFn, typename... Args>
1510 typename std::enable_if<!std::is_member_function_pointer<EmitFn>::value>::type
CallEmitFn(EmitFn fn,Args...args)1511 CallEmitFn(EmitFn fn, Args... args) {
1512 fn(args...);
1513 }
1514
1515 template <typename EmitFn, typename... Args>
1516 typename std::enable_if<std::is_member_function_pointer<EmitFn>::value>::type
CallEmitFn(EmitFn fn,Args...args)1517 CallEmitFn(EmitFn fn, Args... args) {
1518 (asm_.*fn)(ConvertAssemblerArg(args)...);
1519 }
1520
1521 // Wrap a {LiftoffRegister} with implicit conversions to {Register} and
1522 // {DoubleRegister}.
1523 struct AssemblerRegisterConverter {
1524 LiftoffRegister reg;
operator LiftoffRegisterv8::internal::wasm::__anon3bd82aa60111::LiftoffCompiler::AssemblerRegisterConverter1525 operator LiftoffRegister() { return reg; }
operator Registerv8::internal::wasm::__anon3bd82aa60111::LiftoffCompiler::AssemblerRegisterConverter1526 operator Register() { return reg.gp(); }
operator DoubleRegisterv8::internal::wasm::__anon3bd82aa60111::LiftoffCompiler::AssemblerRegisterConverter1527 operator DoubleRegister() { return reg.fp(); }
1528 };
1529
1530 // Convert {LiftoffRegister} to {AssemblerRegisterConverter}, other types stay
1531 // unchanged.
1532 template <typename T>
1533 typename std::conditional<std::is_same<LiftoffRegister, T>::value,
1534 AssemblerRegisterConverter, T>::type
ConvertAssemblerArg(T t)1535 ConvertAssemblerArg(T t) {
1536 return {t};
1537 }
1538
1539 template <typename EmitFn, typename ArgType>
1540 struct EmitFnWithFirstArg {
1541 EmitFn fn;
1542 ArgType first_arg;
1543 };
1544
1545 template <typename EmitFn, typename ArgType>
BindFirst(EmitFn fn,ArgType arg)1546 EmitFnWithFirstArg<EmitFn, ArgType> BindFirst(EmitFn fn, ArgType arg) {
1547 return {fn, arg};
1548 }
1549
1550 template <typename EmitFn, typename T, typename... Args>
CallEmitFn(EmitFnWithFirstArg<EmitFn,T> bound_fn,Args...args)1551 void CallEmitFn(EmitFnWithFirstArg<EmitFn, T> bound_fn, Args... args) {
1552 CallEmitFn(bound_fn.fn, bound_fn.first_arg, ConvertAssemblerArg(args)...);
1553 }
1554
1555 template <ValueKind src_kind, ValueKind result_kind,
1556 ValueKind result_lane_kind = kVoid, class EmitFn>
EmitUnOp(EmitFn fn)1557 void EmitUnOp(EmitFn fn) {
1558 constexpr RegClass src_rc = reg_class_for(src_kind);
1559 constexpr RegClass result_rc = reg_class_for(result_kind);
1560 LiftoffRegister src = __ PopToRegister();
1561 LiftoffRegister dst = src_rc == result_rc
1562 ? __ GetUnusedRegister(result_rc, {src}, {})
1563 : __ GetUnusedRegister(result_rc, {});
1564 CallEmitFn(fn, dst, src);
1565 if (V8_UNLIKELY(nondeterminism_)) {
1566 LiftoffRegList pinned = {dst};
1567 if (result_kind == ValueKind::kF32 || result_kind == ValueKind::kF64) {
1568 CheckNan(dst, pinned, result_kind);
1569 } else if (result_kind == ValueKind::kS128 &&
1570 (result_lane_kind == kF32 || result_lane_kind == kF64)) {
1571 CheckS128Nan(dst, pinned, result_lane_kind);
1572 }
1573 }
1574 __ PushRegister(result_kind, dst);
1575 }
1576
1577 template <ValueKind kind>
EmitFloatUnOpWithCFallback(bool (LiftoffAssembler::* emit_fn)(DoubleRegister,DoubleRegister),ExternalReference (* fallback_fn)())1578 void EmitFloatUnOpWithCFallback(
1579 bool (LiftoffAssembler::*emit_fn)(DoubleRegister, DoubleRegister),
1580 ExternalReference (*fallback_fn)()) {
1581 auto emit_with_c_fallback = [=](LiftoffRegister dst, LiftoffRegister src) {
1582 if ((asm_.*emit_fn)(dst.fp(), src.fp())) return;
1583 ExternalReference ext_ref = fallback_fn();
1584 auto sig = MakeSig::Params(kind);
1585 GenerateCCall(&dst, &sig, kind, &src, ext_ref);
1586 };
1587 EmitUnOp<kind, kind>(emit_with_c_fallback);
1588 }
1589
1590 enum TypeConversionTrapping : bool { kCanTrap = true, kNoTrap = false };
1591 template <ValueKind dst_kind, ValueKind src_kind,
1592 TypeConversionTrapping can_trap>
EmitTypeConversion(FullDecoder * decoder,WasmOpcode opcode,ExternalReference (* fallback_fn)())1593 void EmitTypeConversion(FullDecoder* decoder, WasmOpcode opcode,
1594 ExternalReference (*fallback_fn)()) {
1595 static constexpr RegClass src_rc = reg_class_for(src_kind);
1596 static constexpr RegClass dst_rc = reg_class_for(dst_kind);
1597 LiftoffRegister src = __ PopToRegister();
1598 LiftoffRegister dst = src_rc == dst_rc
1599 ? __ GetUnusedRegister(dst_rc, {src}, {})
1600 : __ GetUnusedRegister(dst_rc, {});
1601 Label* trap =
1602 can_trap ? AddOutOfLineTrap(
1603 decoder, WasmCode::kThrowWasmTrapFloatUnrepresentable)
1604 : nullptr;
1605 if (!__ emit_type_conversion(opcode, dst, src, trap)) {
1606 DCHECK_NOT_NULL(fallback_fn);
1607 ExternalReference ext_ref = fallback_fn();
1608 if (can_trap) {
1609 // External references for potentially trapping conversions return int.
1610 auto sig = MakeSig::Returns(kI32).Params(src_kind);
1611 LiftoffRegister ret_reg =
1612 __ GetUnusedRegister(kGpReg, LiftoffRegList{dst});
1613 LiftoffRegister dst_regs[] = {ret_reg, dst};
1614 GenerateCCall(dst_regs, &sig, dst_kind, &src, ext_ref);
1615 __ emit_cond_jump(kEqual, trap, kI32, ret_reg.gp());
1616 } else {
1617 ValueKind sig_kinds[] = {src_kind};
1618 ValueKindSig sig(0, 1, sig_kinds);
1619 GenerateCCall(&dst, &sig, dst_kind, &src, ext_ref);
1620 }
1621 }
1622 __ PushRegister(dst_kind, dst);
1623 }
1624
UnOp(FullDecoder * decoder,WasmOpcode opcode,const Value & value,Value * result)1625 void UnOp(FullDecoder* decoder, WasmOpcode opcode, const Value& value,
1626 Value* result) {
1627 #define CASE_I32_UNOP(opcode, fn) \
1628 case kExpr##opcode: \
1629 return EmitUnOp<kI32, kI32>(&LiftoffAssembler::emit_##fn);
1630 #define CASE_I64_UNOP(opcode, fn) \
1631 case kExpr##opcode: \
1632 return EmitUnOp<kI64, kI64>(&LiftoffAssembler::emit_##fn);
1633 #define CASE_FLOAT_UNOP(opcode, kind, fn) \
1634 case kExpr##opcode: \
1635 return EmitUnOp<k##kind, k##kind>(&LiftoffAssembler::emit_##fn);
1636 #define CASE_FLOAT_UNOP_WITH_CFALLBACK(opcode, kind, fn) \
1637 case kExpr##opcode: \
1638 return EmitFloatUnOpWithCFallback<k##kind>(&LiftoffAssembler::emit_##fn, \
1639 &ExternalReference::wasm_##fn);
1640 #define CASE_TYPE_CONVERSION(opcode, dst_kind, src_kind, ext_ref, can_trap) \
1641 case kExpr##opcode: \
1642 return EmitTypeConversion<k##dst_kind, k##src_kind, can_trap>( \
1643 decoder, kExpr##opcode, ext_ref);
1644 switch (opcode) {
1645 CASE_I32_UNOP(I32Clz, i32_clz)
1646 CASE_I32_UNOP(I32Ctz, i32_ctz)
1647 CASE_FLOAT_UNOP(F32Abs, F32, f32_abs)
1648 CASE_FLOAT_UNOP(F32Neg, F32, f32_neg)
1649 CASE_FLOAT_UNOP_WITH_CFALLBACK(F32Ceil, F32, f32_ceil)
1650 CASE_FLOAT_UNOP_WITH_CFALLBACK(F32Floor, F32, f32_floor)
1651 CASE_FLOAT_UNOP_WITH_CFALLBACK(F32Trunc, F32, f32_trunc)
1652 CASE_FLOAT_UNOP_WITH_CFALLBACK(F32NearestInt, F32, f32_nearest_int)
1653 CASE_FLOAT_UNOP(F32Sqrt, F32, f32_sqrt)
1654 CASE_FLOAT_UNOP(F64Abs, F64, f64_abs)
1655 CASE_FLOAT_UNOP(F64Neg, F64, f64_neg)
1656 CASE_FLOAT_UNOP_WITH_CFALLBACK(F64Ceil, F64, f64_ceil)
1657 CASE_FLOAT_UNOP_WITH_CFALLBACK(F64Floor, F64, f64_floor)
1658 CASE_FLOAT_UNOP_WITH_CFALLBACK(F64Trunc, F64, f64_trunc)
1659 CASE_FLOAT_UNOP_WITH_CFALLBACK(F64NearestInt, F64, f64_nearest_int)
1660 CASE_FLOAT_UNOP(F64Sqrt, F64, f64_sqrt)
1661 CASE_TYPE_CONVERSION(I32ConvertI64, I32, I64, nullptr, kNoTrap)
1662 CASE_TYPE_CONVERSION(I32SConvertF32, I32, F32, nullptr, kCanTrap)
1663 CASE_TYPE_CONVERSION(I32UConvertF32, I32, F32, nullptr, kCanTrap)
1664 CASE_TYPE_CONVERSION(I32SConvertF64, I32, F64, nullptr, kCanTrap)
1665 CASE_TYPE_CONVERSION(I32UConvertF64, I32, F64, nullptr, kCanTrap)
1666 CASE_TYPE_CONVERSION(I32ReinterpretF32, I32, F32, nullptr, kNoTrap)
1667 CASE_TYPE_CONVERSION(I64SConvertI32, I64, I32, nullptr, kNoTrap)
1668 CASE_TYPE_CONVERSION(I64UConvertI32, I64, I32, nullptr, kNoTrap)
1669 CASE_TYPE_CONVERSION(I64SConvertF32, I64, F32,
1670 &ExternalReference::wasm_float32_to_int64, kCanTrap)
1671 CASE_TYPE_CONVERSION(I64UConvertF32, I64, F32,
1672 &ExternalReference::wasm_float32_to_uint64, kCanTrap)
1673 CASE_TYPE_CONVERSION(I64SConvertF64, I64, F64,
1674 &ExternalReference::wasm_float64_to_int64, kCanTrap)
1675 CASE_TYPE_CONVERSION(I64UConvertF64, I64, F64,
1676 &ExternalReference::wasm_float64_to_uint64, kCanTrap)
1677 CASE_TYPE_CONVERSION(I64ReinterpretF64, I64, F64, nullptr, kNoTrap)
1678 CASE_TYPE_CONVERSION(F32SConvertI32, F32, I32, nullptr, kNoTrap)
1679 CASE_TYPE_CONVERSION(F32UConvertI32, F32, I32, nullptr, kNoTrap)
1680 CASE_TYPE_CONVERSION(F32SConvertI64, F32, I64,
1681 &ExternalReference::wasm_int64_to_float32, kNoTrap)
1682 CASE_TYPE_CONVERSION(F32UConvertI64, F32, I64,
1683 &ExternalReference::wasm_uint64_to_float32, kNoTrap)
1684 CASE_TYPE_CONVERSION(F32ConvertF64, F32, F64, nullptr, kNoTrap)
1685 CASE_TYPE_CONVERSION(F32ReinterpretI32, F32, I32, nullptr, kNoTrap)
1686 CASE_TYPE_CONVERSION(F64SConvertI32, F64, I32, nullptr, kNoTrap)
1687 CASE_TYPE_CONVERSION(F64UConvertI32, F64, I32, nullptr, kNoTrap)
1688 CASE_TYPE_CONVERSION(F64SConvertI64, F64, I64,
1689 &ExternalReference::wasm_int64_to_float64, kNoTrap)
1690 CASE_TYPE_CONVERSION(F64UConvertI64, F64, I64,
1691 &ExternalReference::wasm_uint64_to_float64, kNoTrap)
1692 CASE_TYPE_CONVERSION(F64ConvertF32, F64, F32, nullptr, kNoTrap)
1693 CASE_TYPE_CONVERSION(F64ReinterpretI64, F64, I64, nullptr, kNoTrap)
1694 CASE_I32_UNOP(I32SExtendI8, i32_signextend_i8)
1695 CASE_I32_UNOP(I32SExtendI16, i32_signextend_i16)
1696 CASE_I64_UNOP(I64SExtendI8, i64_signextend_i8)
1697 CASE_I64_UNOP(I64SExtendI16, i64_signextend_i16)
1698 CASE_I64_UNOP(I64SExtendI32, i64_signextend_i32)
1699 CASE_I64_UNOP(I64Clz, i64_clz)
1700 CASE_I64_UNOP(I64Ctz, i64_ctz)
1701 CASE_TYPE_CONVERSION(I32SConvertSatF32, I32, F32, nullptr, kNoTrap)
1702 CASE_TYPE_CONVERSION(I32UConvertSatF32, I32, F32, nullptr, kNoTrap)
1703 CASE_TYPE_CONVERSION(I32SConvertSatF64, I32, F64, nullptr, kNoTrap)
1704 CASE_TYPE_CONVERSION(I32UConvertSatF64, I32, F64, nullptr, kNoTrap)
1705 CASE_TYPE_CONVERSION(I64SConvertSatF32, I64, F32,
1706 &ExternalReference::wasm_float32_to_int64_sat,
1707 kNoTrap)
1708 CASE_TYPE_CONVERSION(I64UConvertSatF32, I64, F32,
1709 &ExternalReference::wasm_float32_to_uint64_sat,
1710 kNoTrap)
1711 CASE_TYPE_CONVERSION(I64SConvertSatF64, I64, F64,
1712 &ExternalReference::wasm_float64_to_int64_sat,
1713 kNoTrap)
1714 CASE_TYPE_CONVERSION(I64UConvertSatF64, I64, F64,
1715 &ExternalReference::wasm_float64_to_uint64_sat,
1716 kNoTrap)
1717 case kExprI32Eqz:
1718 DCHECK(decoder->lookahead(0, kExprI32Eqz));
1719 if ((decoder->lookahead(1, kExprBrIf) ||
1720 decoder->lookahead(1, kExprIf)) &&
1721 !for_debugging_) {
1722 DCHECK(!has_outstanding_op());
1723 outstanding_op_ = kExprI32Eqz;
1724 break;
1725 }
1726 return EmitUnOp<kI32, kI32>(&LiftoffAssembler::emit_i32_eqz);
1727 case kExprI64Eqz:
1728 return EmitUnOp<kI64, kI32>(&LiftoffAssembler::emit_i64_eqz);
1729 case kExprI32Popcnt:
1730 return EmitUnOp<kI32, kI32>(
1731 [=](LiftoffRegister dst, LiftoffRegister src) {
1732 if (__ emit_i32_popcnt(dst.gp(), src.gp())) return;
1733 auto sig = MakeSig::Returns(kI32).Params(kI32);
1734 GenerateCCall(&dst, &sig, kVoid, &src,
1735 ExternalReference::wasm_word32_popcnt());
1736 });
1737 case kExprI64Popcnt:
1738 return EmitUnOp<kI64, kI64>(
1739 [=](LiftoffRegister dst, LiftoffRegister src) {
1740 if (__ emit_i64_popcnt(dst, src)) return;
1741 // The c function returns i32. We will zero-extend later.
1742 auto sig = MakeSig::Returns(kI32).Params(kI64);
1743 LiftoffRegister c_call_dst = kNeedI64RegPair ? dst.low() : dst;
1744 GenerateCCall(&c_call_dst, &sig, kVoid, &src,
1745 ExternalReference::wasm_word64_popcnt());
1746 // Now zero-extend the result to i64.
1747 __ emit_type_conversion(kExprI64UConvertI32, dst, c_call_dst,
1748 nullptr);
1749 });
1750 case kExprRefIsNull:
1751 // We abuse ref.as_non_null, which isn't otherwise used in this switch, as
1752 // a sentinel for the negation of ref.is_null.
1753 case kExprRefAsNonNull: {
1754 LiftoffRegList pinned;
1755 LiftoffRegister ref = pinned.set(__ PopToRegister());
1756 LiftoffRegister null = __ GetUnusedRegister(kGpReg, pinned);
1757 LoadNullValue(null.gp(), pinned);
1758 // Prefer to overwrite one of the input registers with the result
1759 // of the comparison.
1760 LiftoffRegister dst = __ GetUnusedRegister(kGpReg, {ref, null}, {});
1761 __ emit_ptrsize_set_cond(opcode == kExprRefIsNull ? kEqual : kUnequal,
1762 dst.gp(), ref, null);
1763 __ PushRegister(kI32, dst);
1764 return;
1765 }
1766 default:
1767 UNREACHABLE();
1768 }
1769 #undef CASE_I32_UNOP
1770 #undef CASE_I64_UNOP
1771 #undef CASE_FLOAT_UNOP
1772 #undef CASE_FLOAT_UNOP_WITH_CFALLBACK
1773 #undef CASE_TYPE_CONVERSION
1774 }
1775
1776 template <ValueKind src_kind, ValueKind result_kind, typename EmitFn,
1777 typename EmitFnImm>
EmitBinOpImm(EmitFn fn,EmitFnImm fnImm)1778 void EmitBinOpImm(EmitFn fn, EmitFnImm fnImm) {
1779 static constexpr RegClass src_rc = reg_class_for(src_kind);
1780 static constexpr RegClass result_rc = reg_class_for(result_kind);
1781
1782 LiftoffAssembler::VarState rhs_slot = __ cache_state()->stack_state.back();
1783 // Check if the RHS is an immediate.
1784 if (rhs_slot.is_const()) {
1785 __ cache_state()->stack_state.pop_back();
1786 int32_t imm = rhs_slot.i32_const();
1787
1788 LiftoffRegister lhs = __ PopToRegister();
1789 // Either reuse {lhs} for {dst}, or choose a register (pair) which does
1790 // not overlap, for easier code generation.
1791 LiftoffRegList pinned = {lhs};
1792 LiftoffRegister dst = src_rc == result_rc
1793 ? __ GetUnusedRegister(result_rc, {lhs}, pinned)
1794 : __ GetUnusedRegister(result_rc, pinned);
1795
1796 CallEmitFn(fnImm, dst, lhs, imm);
1797 static_assert(result_kind != kF32 && result_kind != kF64,
1798 "Unhandled nondeterminism for fuzzing.");
1799 __ PushRegister(result_kind, dst);
1800 } else {
1801 // The RHS was not an immediate.
1802 EmitBinOp<src_kind, result_kind>(fn);
1803 }
1804 }
1805
1806 template <ValueKind src_kind, ValueKind result_kind,
1807 bool swap_lhs_rhs = false, ValueKind result_lane_kind = kVoid,
1808 typename EmitFn>
EmitBinOp(EmitFn fn)1809 void EmitBinOp(EmitFn fn) {
1810 static constexpr RegClass src_rc = reg_class_for(src_kind);
1811 static constexpr RegClass result_rc = reg_class_for(result_kind);
1812 LiftoffRegister rhs = __ PopToRegister();
1813 LiftoffRegister lhs = __ PopToRegister(LiftoffRegList{rhs});
1814 LiftoffRegister dst = src_rc == result_rc
1815 ? __ GetUnusedRegister(result_rc, {lhs, rhs}, {})
1816 : __ GetUnusedRegister(result_rc, {});
1817
1818 if (swap_lhs_rhs) std::swap(lhs, rhs);
1819
1820 CallEmitFn(fn, dst, lhs, rhs);
1821 if (V8_UNLIKELY(nondeterminism_)) {
1822 LiftoffRegList pinned = {dst};
1823 if (result_kind == ValueKind::kF32 || result_kind == ValueKind::kF64) {
1824 CheckNan(dst, pinned, result_kind);
1825 } else if (result_kind == ValueKind::kS128 &&
1826 (result_lane_kind == kF32 || result_lane_kind == kF64)) {
1827 CheckS128Nan(dst, pinned, result_lane_kind);
1828 }
1829 }
1830 __ PushRegister(result_kind, dst);
1831 }
1832
EmitDivOrRem64CCall(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs,ExternalReference ext_ref,Label * trap_by_zero,Label * trap_unrepresentable=nullptr)1833 void EmitDivOrRem64CCall(LiftoffRegister dst, LiftoffRegister lhs,
1834 LiftoffRegister rhs, ExternalReference ext_ref,
1835 Label* trap_by_zero,
1836 Label* trap_unrepresentable = nullptr) {
1837 // Cannot emit native instructions, build C call.
1838 LiftoffRegister ret = __ GetUnusedRegister(kGpReg, LiftoffRegList{dst});
1839 LiftoffRegister tmp =
1840 __ GetUnusedRegister(kGpReg, LiftoffRegList{dst, ret});
1841 LiftoffRegister arg_regs[] = {lhs, rhs};
1842 LiftoffRegister result_regs[] = {ret, dst};
1843 auto sig = MakeSig::Returns(kI32).Params(kI64, kI64);
1844 GenerateCCall(result_regs, &sig, kI64, arg_regs, ext_ref);
1845 __ LoadConstant(tmp, WasmValue(int32_t{0}));
1846 __ emit_cond_jump(kEqual, trap_by_zero, kI32, ret.gp(), tmp.gp());
1847 if (trap_unrepresentable) {
1848 __ LoadConstant(tmp, WasmValue(int32_t{-1}));
1849 __ emit_cond_jump(kEqual, trap_unrepresentable, kI32, ret.gp(), tmp.gp());
1850 }
1851 }
1852
1853 template <WasmOpcode opcode>
EmitI32CmpOp(FullDecoder * decoder)1854 void EmitI32CmpOp(FullDecoder* decoder) {
1855 DCHECK(decoder->lookahead(0, opcode));
1856 if ((decoder->lookahead(1, kExprBrIf) || decoder->lookahead(1, kExprIf)) &&
1857 !for_debugging_) {
1858 DCHECK(!has_outstanding_op());
1859 outstanding_op_ = opcode;
1860 return;
1861 }
1862 return EmitBinOp<kI32, kI32>(BindFirst(&LiftoffAssembler::emit_i32_set_cond,
1863 GetCompareCondition(opcode)));
1864 }
1865
BinOp(FullDecoder * decoder,WasmOpcode opcode,const Value & lhs,const Value & rhs,Value * result)1866 void BinOp(FullDecoder* decoder, WasmOpcode opcode, const Value& lhs,
1867 const Value& rhs, Value* result) {
1868 #define CASE_I64_SHIFTOP(opcode, fn) \
1869 case kExpr##opcode: \
1870 return EmitBinOpImm<kI64, kI64>( \
1871 [=](LiftoffRegister dst, LiftoffRegister src, \
1872 LiftoffRegister amount) { \
1873 __ emit_##fn(dst, src, \
1874 amount.is_gp_pair() ? amount.low_gp() : amount.gp()); \
1875 }, \
1876 &LiftoffAssembler::emit_##fn##i);
1877 #define CASE_CCALL_BINOP(opcode, kind, ext_ref_fn) \
1878 case kExpr##opcode: \
1879 return EmitBinOp<k##kind, k##kind>( \
1880 [=](LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs) { \
1881 LiftoffRegister args[] = {lhs, rhs}; \
1882 auto ext_ref = ExternalReference::ext_ref_fn(); \
1883 ValueKind sig_kinds[] = {k##kind, k##kind, k##kind}; \
1884 const bool out_via_stack = k##kind == kI64; \
1885 ValueKindSig sig(out_via_stack ? 0 : 1, 2, sig_kinds); \
1886 ValueKind out_arg_kind = out_via_stack ? kI64 : kVoid; \
1887 GenerateCCall(&dst, &sig, out_arg_kind, args, ext_ref); \
1888 });
1889 switch (opcode) {
1890 case kExprI32Add:
1891 return EmitBinOpImm<kI32, kI32>(&LiftoffAssembler::emit_i32_add,
1892 &LiftoffAssembler::emit_i32_addi);
1893 case kExprI32Sub:
1894 return EmitBinOp<kI32, kI32>(&LiftoffAssembler::emit_i32_sub);
1895 case kExprI32Mul:
1896 return EmitBinOp<kI32, kI32>(&LiftoffAssembler::emit_i32_mul);
1897 case kExprI32And:
1898 return EmitBinOpImm<kI32, kI32>(&LiftoffAssembler::emit_i32_and,
1899 &LiftoffAssembler::emit_i32_andi);
1900 case kExprI32Ior:
1901 return EmitBinOpImm<kI32, kI32>(&LiftoffAssembler::emit_i32_or,
1902 &LiftoffAssembler::emit_i32_ori);
1903 case kExprI32Xor:
1904 return EmitBinOpImm<kI32, kI32>(&LiftoffAssembler::emit_i32_xor,
1905 &LiftoffAssembler::emit_i32_xori);
1906 case kExprI32Eq:
1907 return EmitI32CmpOp<kExprI32Eq>(decoder);
1908 case kExprI32Ne:
1909 return EmitI32CmpOp<kExprI32Ne>(decoder);
1910 case kExprI32LtS:
1911 return EmitI32CmpOp<kExprI32LtS>(decoder);
1912 case kExprI32LtU:
1913 return EmitI32CmpOp<kExprI32LtU>(decoder);
1914 case kExprI32GtS:
1915 return EmitI32CmpOp<kExprI32GtS>(decoder);
1916 case kExprI32GtU:
1917 return EmitI32CmpOp<kExprI32GtU>(decoder);
1918 case kExprI32LeS:
1919 return EmitI32CmpOp<kExprI32LeS>(decoder);
1920 case kExprI32LeU:
1921 return EmitI32CmpOp<kExprI32LeU>(decoder);
1922 case kExprI32GeS:
1923 return EmitI32CmpOp<kExprI32GeS>(decoder);
1924 case kExprI32GeU:
1925 return EmitI32CmpOp<kExprI32GeU>(decoder);
1926 case kExprI64Add:
1927 return EmitBinOpImm<kI64, kI64>(&LiftoffAssembler::emit_i64_add,
1928 &LiftoffAssembler::emit_i64_addi);
1929 case kExprI64Sub:
1930 return EmitBinOp<kI64, kI64>(&LiftoffAssembler::emit_i64_sub);
1931 case kExprI64Mul:
1932 return EmitBinOp<kI64, kI64>(&LiftoffAssembler::emit_i64_mul);
1933 case kExprI64And:
1934 return EmitBinOpImm<kI64, kI64>(&LiftoffAssembler::emit_i64_and,
1935 &LiftoffAssembler::emit_i64_andi);
1936 case kExprI64Ior:
1937 return EmitBinOpImm<kI64, kI64>(&LiftoffAssembler::emit_i64_or,
1938 &LiftoffAssembler::emit_i64_ori);
1939 case kExprI64Xor:
1940 return EmitBinOpImm<kI64, kI64>(&LiftoffAssembler::emit_i64_xor,
1941 &LiftoffAssembler::emit_i64_xori);
1942 case kExprI64Eq:
1943 return EmitBinOp<kI64, kI32>(
1944 BindFirst(&LiftoffAssembler::emit_i64_set_cond, kEqual));
1945 case kExprI64Ne:
1946 return EmitBinOp<kI64, kI32>(
1947 BindFirst(&LiftoffAssembler::emit_i64_set_cond, kUnequal));
1948 case kExprI64LtS:
1949 return EmitBinOp<kI64, kI32>(
1950 BindFirst(&LiftoffAssembler::emit_i64_set_cond, kSignedLessThan));
1951 case kExprI64LtU:
1952 return EmitBinOp<kI64, kI32>(
1953 BindFirst(&LiftoffAssembler::emit_i64_set_cond, kUnsignedLessThan));
1954 case kExprI64GtS:
1955 return EmitBinOp<kI64, kI32>(BindFirst(
1956 &LiftoffAssembler::emit_i64_set_cond, kSignedGreaterThan));
1957 case kExprI64GtU:
1958 return EmitBinOp<kI64, kI32>(BindFirst(
1959 &LiftoffAssembler::emit_i64_set_cond, kUnsignedGreaterThan));
1960 case kExprI64LeS:
1961 return EmitBinOp<kI64, kI32>(
1962 BindFirst(&LiftoffAssembler::emit_i64_set_cond, kSignedLessEqual));
1963 case kExprI64LeU:
1964 return EmitBinOp<kI64, kI32>(BindFirst(
1965 &LiftoffAssembler::emit_i64_set_cond, kUnsignedLessEqual));
1966 case kExprI64GeS:
1967 return EmitBinOp<kI64, kI32>(BindFirst(
1968 &LiftoffAssembler::emit_i64_set_cond, kSignedGreaterEqual));
1969 case kExprI64GeU:
1970 return EmitBinOp<kI64, kI32>(BindFirst(
1971 &LiftoffAssembler::emit_i64_set_cond, kUnsignedGreaterEqual));
1972 case kExprF32Eq:
1973 return EmitBinOp<kF32, kI32>(
1974 BindFirst(&LiftoffAssembler::emit_f32_set_cond, kEqual));
1975 case kExprF32Ne:
1976 return EmitBinOp<kF32, kI32>(
1977 BindFirst(&LiftoffAssembler::emit_f32_set_cond, kUnequal));
1978 case kExprF32Lt:
1979 return EmitBinOp<kF32, kI32>(
1980 BindFirst(&LiftoffAssembler::emit_f32_set_cond, kUnsignedLessThan));
1981 case kExprF32Gt:
1982 return EmitBinOp<kF32, kI32>(BindFirst(
1983 &LiftoffAssembler::emit_f32_set_cond, kUnsignedGreaterThan));
1984 case kExprF32Le:
1985 return EmitBinOp<kF32, kI32>(BindFirst(
1986 &LiftoffAssembler::emit_f32_set_cond, kUnsignedLessEqual));
1987 case kExprF32Ge:
1988 return EmitBinOp<kF32, kI32>(BindFirst(
1989 &LiftoffAssembler::emit_f32_set_cond, kUnsignedGreaterEqual));
1990 case kExprF64Eq:
1991 return EmitBinOp<kF64, kI32>(
1992 BindFirst(&LiftoffAssembler::emit_f64_set_cond, kEqual));
1993 case kExprF64Ne:
1994 return EmitBinOp<kF64, kI32>(
1995 BindFirst(&LiftoffAssembler::emit_f64_set_cond, kUnequal));
1996 case kExprF64Lt:
1997 return EmitBinOp<kF64, kI32>(
1998 BindFirst(&LiftoffAssembler::emit_f64_set_cond, kUnsignedLessThan));
1999 case kExprF64Gt:
2000 return EmitBinOp<kF64, kI32>(BindFirst(
2001 &LiftoffAssembler::emit_f64_set_cond, kUnsignedGreaterThan));
2002 case kExprF64Le:
2003 return EmitBinOp<kF64, kI32>(BindFirst(
2004 &LiftoffAssembler::emit_f64_set_cond, kUnsignedLessEqual));
2005 case kExprF64Ge:
2006 return EmitBinOp<kF64, kI32>(BindFirst(
2007 &LiftoffAssembler::emit_f64_set_cond, kUnsignedGreaterEqual));
2008 case kExprI32Shl:
2009 return EmitBinOpImm<kI32, kI32>(&LiftoffAssembler::emit_i32_shl,
2010 &LiftoffAssembler::emit_i32_shli);
2011 case kExprI32ShrS:
2012 return EmitBinOpImm<kI32, kI32>(&LiftoffAssembler::emit_i32_sar,
2013 &LiftoffAssembler::emit_i32_sari);
2014 case kExprI32ShrU:
2015 return EmitBinOpImm<kI32, kI32>(&LiftoffAssembler::emit_i32_shr,
2016 &LiftoffAssembler::emit_i32_shri);
2017 CASE_CCALL_BINOP(I32Rol, I32, wasm_word32_rol)
2018 CASE_CCALL_BINOP(I32Ror, I32, wasm_word32_ror)
2019 CASE_I64_SHIFTOP(I64Shl, i64_shl)
2020 CASE_I64_SHIFTOP(I64ShrS, i64_sar)
2021 CASE_I64_SHIFTOP(I64ShrU, i64_shr)
2022 CASE_CCALL_BINOP(I64Rol, I64, wasm_word64_rol)
2023 CASE_CCALL_BINOP(I64Ror, I64, wasm_word64_ror)
2024 case kExprF32Add:
2025 return EmitBinOp<kF32, kF32>(&LiftoffAssembler::emit_f32_add);
2026 case kExprF32Sub:
2027 return EmitBinOp<kF32, kF32>(&LiftoffAssembler::emit_f32_sub);
2028 case kExprF32Mul:
2029 return EmitBinOp<kF32, kF32>(&LiftoffAssembler::emit_f32_mul);
2030 case kExprF32Div:
2031 return EmitBinOp<kF32, kF32>(&LiftoffAssembler::emit_f32_div);
2032 case kExprF32Min:
2033 return EmitBinOp<kF32, kF32>(&LiftoffAssembler::emit_f32_min);
2034 case kExprF32Max:
2035 return EmitBinOp<kF32, kF32>(&LiftoffAssembler::emit_f32_max);
2036 case kExprF32CopySign:
2037 return EmitBinOp<kF32, kF32>(&LiftoffAssembler::emit_f32_copysign);
2038 case kExprF64Add:
2039 return EmitBinOp<kF64, kF64>(&LiftoffAssembler::emit_f64_add);
2040 case kExprF64Sub:
2041 return EmitBinOp<kF64, kF64>(&LiftoffAssembler::emit_f64_sub);
2042 case kExprF64Mul:
2043 return EmitBinOp<kF64, kF64>(&LiftoffAssembler::emit_f64_mul);
2044 case kExprF64Div:
2045 return EmitBinOp<kF64, kF64>(&LiftoffAssembler::emit_f64_div);
2046 case kExprF64Min:
2047 return EmitBinOp<kF64, kF64>(&LiftoffAssembler::emit_f64_min);
2048 case kExprF64Max:
2049 return EmitBinOp<kF64, kF64>(&LiftoffAssembler::emit_f64_max);
2050 case kExprF64CopySign:
2051 return EmitBinOp<kF64, kF64>(&LiftoffAssembler::emit_f64_copysign);
2052 case kExprI32DivS:
2053 return EmitBinOp<kI32, kI32>([this, decoder](LiftoffRegister dst,
2054 LiftoffRegister lhs,
2055 LiftoffRegister rhs) {
2056 AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapDivByZero);
2057 // Adding the second trap might invalidate the pointer returned for
2058 // the first one, thus get both pointers afterwards.
2059 AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapDivUnrepresentable);
2060 Label* div_by_zero = out_of_line_code_.end()[-2].label.get();
2061 Label* div_unrepresentable = out_of_line_code_.end()[-1].label.get();
2062 __ emit_i32_divs(dst.gp(), lhs.gp(), rhs.gp(), div_by_zero,
2063 div_unrepresentable);
2064 });
2065 case kExprI32DivU:
2066 return EmitBinOp<kI32, kI32>([this, decoder](LiftoffRegister dst,
2067 LiftoffRegister lhs,
2068 LiftoffRegister rhs) {
2069 Label* div_by_zero =
2070 AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapDivByZero);
2071 __ emit_i32_divu(dst.gp(), lhs.gp(), rhs.gp(), div_by_zero);
2072 });
2073 case kExprI32RemS:
2074 return EmitBinOp<kI32, kI32>([this, decoder](LiftoffRegister dst,
2075 LiftoffRegister lhs,
2076 LiftoffRegister rhs) {
2077 Label* rem_by_zero =
2078 AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapRemByZero);
2079 __ emit_i32_rems(dst.gp(), lhs.gp(), rhs.gp(), rem_by_zero);
2080 });
2081 case kExprI32RemU:
2082 return EmitBinOp<kI32, kI32>([this, decoder](LiftoffRegister dst,
2083 LiftoffRegister lhs,
2084 LiftoffRegister rhs) {
2085 Label* rem_by_zero =
2086 AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapRemByZero);
2087 __ emit_i32_remu(dst.gp(), lhs.gp(), rhs.gp(), rem_by_zero);
2088 });
2089 case kExprI64DivS:
2090 return EmitBinOp<kI64, kI64>([this, decoder](LiftoffRegister dst,
2091 LiftoffRegister lhs,
2092 LiftoffRegister rhs) {
2093 AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapDivByZero);
2094 // Adding the second trap might invalidate the pointer returned for
2095 // the first one, thus get both pointers afterwards.
2096 AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapDivUnrepresentable);
2097 Label* div_by_zero = out_of_line_code_.end()[-2].label.get();
2098 Label* div_unrepresentable = out_of_line_code_.end()[-1].label.get();
2099 if (!__ emit_i64_divs(dst, lhs, rhs, div_by_zero,
2100 div_unrepresentable)) {
2101 ExternalReference ext_ref = ExternalReference::wasm_int64_div();
2102 EmitDivOrRem64CCall(dst, lhs, rhs, ext_ref, div_by_zero,
2103 div_unrepresentable);
2104 }
2105 });
2106 case kExprI64DivU:
2107 return EmitBinOp<kI64, kI64>([this, decoder](LiftoffRegister dst,
2108 LiftoffRegister lhs,
2109 LiftoffRegister rhs) {
2110 Label* div_by_zero =
2111 AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapDivByZero);
2112 if (!__ emit_i64_divu(dst, lhs, rhs, div_by_zero)) {
2113 ExternalReference ext_ref = ExternalReference::wasm_uint64_div();
2114 EmitDivOrRem64CCall(dst, lhs, rhs, ext_ref, div_by_zero);
2115 }
2116 });
2117 case kExprI64RemS:
2118 return EmitBinOp<kI64, kI64>([this, decoder](LiftoffRegister dst,
2119 LiftoffRegister lhs,
2120 LiftoffRegister rhs) {
2121 Label* rem_by_zero =
2122 AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapRemByZero);
2123 if (!__ emit_i64_rems(dst, lhs, rhs, rem_by_zero)) {
2124 ExternalReference ext_ref = ExternalReference::wasm_int64_mod();
2125 EmitDivOrRem64CCall(dst, lhs, rhs, ext_ref, rem_by_zero);
2126 }
2127 });
2128 case kExprI64RemU:
2129 return EmitBinOp<kI64, kI64>([this, decoder](LiftoffRegister dst,
2130 LiftoffRegister lhs,
2131 LiftoffRegister rhs) {
2132 Label* rem_by_zero =
2133 AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapRemByZero);
2134 if (!__ emit_i64_remu(dst, lhs, rhs, rem_by_zero)) {
2135 ExternalReference ext_ref = ExternalReference::wasm_uint64_mod();
2136 EmitDivOrRem64CCall(dst, lhs, rhs, ext_ref, rem_by_zero);
2137 }
2138 });
2139 case kExprRefEq: {
2140 return EmitBinOp<kOptRef, kI32>(
2141 BindFirst(&LiftoffAssembler::emit_ptrsize_set_cond, kEqual));
2142 }
2143
2144 default:
2145 UNREACHABLE();
2146 }
2147 #undef CASE_I64_SHIFTOP
2148 #undef CASE_CCALL_BINOP
2149 }
2150
I32Const(FullDecoder * decoder,Value * result,int32_t value)2151 void I32Const(FullDecoder* decoder, Value* result, int32_t value) {
2152 __ PushConstant(kI32, value);
2153 }
2154
I64Const(FullDecoder * decoder,Value * result,int64_t value)2155 void I64Const(FullDecoder* decoder, Value* result, int64_t value) {
2156 // The {VarState} stores constant values as int32_t, thus we only store
2157 // 64-bit constants in this field if it fits in an int32_t. Larger values
2158 // cannot be used as immediate value anyway, so we can also just put them in
2159 // a register immediately.
2160 int32_t value_i32 = static_cast<int32_t>(value);
2161 if (value_i32 == value) {
2162 __ PushConstant(kI64, value_i32);
2163 } else {
2164 LiftoffRegister reg = __ GetUnusedRegister(reg_class_for(kI64), {});
2165 __ LoadConstant(reg, WasmValue(value));
2166 __ PushRegister(kI64, reg);
2167 }
2168 }
2169
F32Const(FullDecoder * decoder,Value * result,float value)2170 void F32Const(FullDecoder* decoder, Value* result, float value) {
2171 LiftoffRegister reg = __ GetUnusedRegister(kFpReg, {});
2172 __ LoadConstant(reg, WasmValue(value));
2173 __ PushRegister(kF32, reg);
2174 }
2175
F64Const(FullDecoder * decoder,Value * result,double value)2176 void F64Const(FullDecoder* decoder, Value* result, double value) {
2177 LiftoffRegister reg = __ GetUnusedRegister(kFpReg, {});
2178 __ LoadConstant(reg, WasmValue(value));
2179 __ PushRegister(kF64, reg);
2180 }
2181
RefNull(FullDecoder * decoder,ValueType type,Value *)2182 void RefNull(FullDecoder* decoder, ValueType type, Value*) {
2183 LiftoffRegister null = __ GetUnusedRegister(kGpReg, {});
2184 LoadNullValue(null.gp(), {});
2185 __ PushRegister(type.kind(), null);
2186 }
2187
RefFunc(FullDecoder * decoder,uint32_t function_index,Value * result)2188 void RefFunc(FullDecoder* decoder, uint32_t function_index, Value* result) {
2189 LiftoffRegister func_index_reg = __ GetUnusedRegister(kGpReg, {});
2190 __ LoadConstant(func_index_reg, WasmValue(function_index));
2191 LiftoffAssembler::VarState func_index_var(kI32, func_index_reg, 0);
2192 CallRuntimeStub(WasmCode::kWasmRefFunc, MakeSig::Returns(kRef).Params(kI32),
2193 {func_index_var}, decoder->position());
2194 __ PushRegister(kRef, LiftoffRegister(kReturnRegister0));
2195 }
2196
RefAsNonNull(FullDecoder * decoder,const Value & arg,Value * result)2197 void RefAsNonNull(FullDecoder* decoder, const Value& arg, Value* result) {
2198 LiftoffRegList pinned;
2199 LiftoffRegister obj = pinned.set(__ PopToRegister(pinned));
2200 MaybeEmitNullCheck(decoder, obj.gp(), pinned, arg.type);
2201 __ PushRegister(kRef, obj);
2202 }
2203
Drop(FullDecoder * decoder)2204 void Drop(FullDecoder* decoder) { __ DropValues(1); }
2205
TraceFunctionExit(FullDecoder * decoder)2206 void TraceFunctionExit(FullDecoder* decoder) {
2207 CODE_COMMENT("trace function exit");
2208 // Before making the runtime call, spill all cache registers.
2209 __ SpillAllRegisters();
2210 LiftoffRegList pinned;
2211 // Get a register to hold the stack slot for the return value.
2212 LiftoffRegister info = pinned.set(__ GetUnusedRegister(kGpReg, pinned));
2213 __ AllocateStackSlot(info.gp(), sizeof(int64_t));
2214
2215 // Store the return value if there is exactly one. Multiple return values
2216 // are not handled yet.
2217 size_t num_returns = decoder->sig_->return_count();
2218 if (num_returns == 1) {
2219 ValueKind return_kind = decoder->sig_->GetReturn(0).kind();
2220 LiftoffRegister return_reg =
2221 __ LoadToRegister(__ cache_state()->stack_state.back(), pinned);
2222 if (is_reference(return_kind)) {
2223 __ StoreTaggedPointer(info.gp(), no_reg, 0, return_reg, pinned);
2224 } else {
2225 __ Store(info.gp(), no_reg, 0, return_reg,
2226 StoreType::ForValueKind(return_kind), pinned);
2227 }
2228 }
2229 // Put the parameter in its place.
2230 WasmTraceExitDescriptor descriptor;
2231 DCHECK_EQ(0, descriptor.GetStackParameterCount());
2232 DCHECK_EQ(1, descriptor.GetRegisterParameterCount());
2233 Register param_reg = descriptor.GetRegisterParameter(0);
2234 if (info.gp() != param_reg) {
2235 __ Move(param_reg, info.gp(), kPointerKind);
2236 }
2237
2238 source_position_table_builder_.AddPosition(
2239 __ pc_offset(), SourcePosition(decoder->position()), false);
2240 __ CallRuntimeStub(WasmCode::kWasmTraceExit);
2241 DefineSafepoint();
2242
2243 __ DeallocateStackSlot(sizeof(int64_t));
2244 }
2245
TierupCheckOnExit(FullDecoder * decoder)2246 void TierupCheckOnExit(FullDecoder* decoder) {
2247 if (!dynamic_tiering()) return;
2248 TierupCheck(decoder, decoder->position(), __ pc_offset());
2249 CODE_COMMENT("update tiering budget");
2250 LiftoffRegList pinned;
2251 LiftoffRegister budget = pinned.set(__ GetUnusedRegister(kGpReg, pinned));
2252 LiftoffRegister array = pinned.set(__ GetUnusedRegister(kGpReg, pinned));
2253 LOAD_INSTANCE_FIELD(array.gp(), TieringBudgetArray, kSystemPointerSize,
2254 pinned);
2255 uint32_t offset =
2256 kInt32Size * declared_function_index(env_->module, func_index_);
2257 __ Fill(budget, liftoff::kTierupBudgetOffset, ValueKind::kI32);
2258 __ Store(array.gp(), no_reg, offset, budget, StoreType::kI32Store, pinned);
2259 }
2260
DoReturn(FullDecoder * decoder,uint32_t)2261 void DoReturn(FullDecoder* decoder, uint32_t /* drop_values */) {
2262 if (FLAG_trace_wasm) TraceFunctionExit(decoder);
2263 TierupCheckOnExit(decoder);
2264 size_t num_returns = decoder->sig_->return_count();
2265 if (num_returns > 0) __ MoveToReturnLocations(decoder->sig_, descriptor_);
2266 __ LeaveFrame(StackFrame::WASM);
2267 __ DropStackSlotsAndRet(
2268 static_cast<uint32_t>(descriptor_->ParameterSlotCount()));
2269 }
2270
LocalGet(FullDecoder * decoder,Value * result,const IndexImmediate<validate> & imm)2271 void LocalGet(FullDecoder* decoder, Value* result,
2272 const IndexImmediate<validate>& imm) {
2273 auto local_slot = __ cache_state()->stack_state[imm.index];
2274 __ cache_state()->stack_state.emplace_back(
2275 local_slot.kind(), __ NextSpillOffset(local_slot.kind()));
2276 auto* slot = &__ cache_state()->stack_state.back();
2277 if (local_slot.is_reg()) {
2278 __ cache_state()->inc_used(local_slot.reg());
2279 slot->MakeRegister(local_slot.reg());
2280 } else if (local_slot.is_const()) {
2281 slot->MakeConstant(local_slot.i32_const());
2282 } else {
2283 DCHECK(local_slot.is_stack());
2284 auto rc = reg_class_for(local_slot.kind());
2285 LiftoffRegister reg = __ GetUnusedRegister(rc, {});
2286 __ cache_state()->inc_used(reg);
2287 slot->MakeRegister(reg);
2288 __ Fill(reg, local_slot.offset(), local_slot.kind());
2289 }
2290 }
2291
LocalSetFromStackSlot(LiftoffAssembler::VarState * dst_slot,uint32_t local_index)2292 void LocalSetFromStackSlot(LiftoffAssembler::VarState* dst_slot,
2293 uint32_t local_index) {
2294 auto& state = *__ cache_state();
2295 auto& src_slot = state.stack_state.back();
2296 ValueKind kind = dst_slot->kind();
2297 if (dst_slot->is_reg()) {
2298 LiftoffRegister slot_reg = dst_slot->reg();
2299 if (state.get_use_count(slot_reg) == 1) {
2300 __ Fill(dst_slot->reg(), src_slot.offset(), kind);
2301 return;
2302 }
2303 state.dec_used(slot_reg);
2304 dst_slot->MakeStack();
2305 }
2306 DCHECK(CheckCompatibleStackSlotTypes(kind, __ local_kind(local_index)));
2307 RegClass rc = reg_class_for(kind);
2308 LiftoffRegister dst_reg = __ GetUnusedRegister(rc, {});
2309 __ Fill(dst_reg, src_slot.offset(), kind);
2310 *dst_slot = LiftoffAssembler::VarState(kind, dst_reg, dst_slot->offset());
2311 __ cache_state()->inc_used(dst_reg);
2312 }
2313
LocalSet(uint32_t local_index,bool is_tee)2314 void LocalSet(uint32_t local_index, bool is_tee) {
2315 auto& state = *__ cache_state();
2316 auto& source_slot = state.stack_state.back();
2317 auto& target_slot = state.stack_state[local_index];
2318 switch (source_slot.loc()) {
2319 case kRegister:
2320 if (target_slot.is_reg()) state.dec_used(target_slot.reg());
2321 target_slot.Copy(source_slot);
2322 if (is_tee) state.inc_used(target_slot.reg());
2323 break;
2324 case kIntConst:
2325 if (target_slot.is_reg()) state.dec_used(target_slot.reg());
2326 target_slot.Copy(source_slot);
2327 break;
2328 case kStack:
2329 LocalSetFromStackSlot(&target_slot, local_index);
2330 break;
2331 }
2332 if (!is_tee) __ cache_state()->stack_state.pop_back();
2333 }
2334
LocalSet(FullDecoder * decoder,const Value & value,const IndexImmediate<validate> & imm)2335 void LocalSet(FullDecoder* decoder, const Value& value,
2336 const IndexImmediate<validate>& imm) {
2337 LocalSet(imm.index, false);
2338 }
2339
LocalTee(FullDecoder * decoder,const Value & value,Value * result,const IndexImmediate<validate> & imm)2340 void LocalTee(FullDecoder* decoder, const Value& value, Value* result,
2341 const IndexImmediate<validate>& imm) {
2342 LocalSet(imm.index, true);
2343 }
2344
AllocateLocals(FullDecoder * decoder,base::Vector<Value> local_values)2345 void AllocateLocals(FullDecoder* decoder, base::Vector<Value> local_values) {
2346 // TODO(7748): Introduce typed functions bailout reason
2347 unsupported(decoder, kGC, "let");
2348 }
2349
DeallocateLocals(FullDecoder * decoder,uint32_t count)2350 void DeallocateLocals(FullDecoder* decoder, uint32_t count) {
2351 // TODO(7748): Introduce typed functions bailout reason
2352 unsupported(decoder, kGC, "let");
2353 }
2354
GetGlobalBaseAndOffset(const WasmGlobal * global,LiftoffRegList * pinned,uint32_t * offset)2355 Register GetGlobalBaseAndOffset(const WasmGlobal* global,
2356 LiftoffRegList* pinned, uint32_t* offset) {
2357 Register addr = pinned->set(__ GetUnusedRegister(kGpReg, {})).gp();
2358 if (global->mutability && global->imported) {
2359 LOAD_INSTANCE_FIELD(addr, ImportedMutableGlobals, kSystemPointerSize,
2360 *pinned);
2361 __ Load(LiftoffRegister(addr), addr, no_reg,
2362 global->index * sizeof(Address), kPointerLoadType, *pinned);
2363 *offset = 0;
2364 } else {
2365 LOAD_INSTANCE_FIELD(addr, GlobalsStart, kSystemPointerSize, *pinned);
2366 *offset = global->offset;
2367 }
2368 return addr;
2369 }
2370
GetBaseAndOffsetForImportedMutableExternRefGlobal(const WasmGlobal * global,LiftoffRegList * pinned,Register * base,Register * offset)2371 void GetBaseAndOffsetForImportedMutableExternRefGlobal(
2372 const WasmGlobal* global, LiftoffRegList* pinned, Register* base,
2373 Register* offset) {
2374 Register globals_buffer =
2375 pinned->set(__ GetUnusedRegister(kGpReg, *pinned)).gp();
2376 LOAD_TAGGED_PTR_INSTANCE_FIELD(globals_buffer,
2377 ImportedMutableGlobalsBuffers, *pinned);
2378 *base = globals_buffer;
2379 __ LoadTaggedPointer(
2380 *base, globals_buffer, no_reg,
2381 wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(global->offset),
2382 *pinned);
2383
2384 // For the offset we need the index of the global in the buffer, and
2385 // then calculate the actual offset from the index. Load the index from
2386 // the ImportedMutableGlobals array of the instance.
2387 Register imported_mutable_globals =
2388 pinned->set(__ GetUnusedRegister(kGpReg, *pinned)).gp();
2389
2390 LOAD_INSTANCE_FIELD(imported_mutable_globals, ImportedMutableGlobals,
2391 kSystemPointerSize, *pinned);
2392 *offset = imported_mutable_globals;
2393 __ Load(LiftoffRegister(*offset), imported_mutable_globals, no_reg,
2394 global->index * sizeof(Address),
2395 kSystemPointerSize == 4 ? LoadType::kI32Load : LoadType::kI64Load,
2396 *pinned);
2397 __ emit_i32_shli(*offset, *offset, kTaggedSizeLog2);
2398 __ emit_i32_addi(*offset, *offset,
2399 wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(0));
2400 }
2401
GlobalGet(FullDecoder * decoder,Value * result,const GlobalIndexImmediate<validate> & imm)2402 void GlobalGet(FullDecoder* decoder, Value* result,
2403 const GlobalIndexImmediate<validate>& imm) {
2404 const auto* global = &env_->module->globals[imm.index];
2405 ValueKind kind = global->type.kind();
2406 if (!CheckSupportedType(decoder, kind, "global")) {
2407 return;
2408 }
2409
2410 if (is_reference(kind)) {
2411 if (global->mutability && global->imported) {
2412 LiftoffRegList pinned;
2413 Register base = no_reg;
2414 Register offset = no_reg;
2415 GetBaseAndOffsetForImportedMutableExternRefGlobal(global, &pinned,
2416 &base, &offset);
2417 __ LoadTaggedPointer(base, base, offset, 0, pinned);
2418 __ PushRegister(kind, LiftoffRegister(base));
2419 return;
2420 }
2421
2422 LiftoffRegList pinned;
2423 Register globals_buffer =
2424 pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
2425 LOAD_TAGGED_PTR_INSTANCE_FIELD(globals_buffer, TaggedGlobalsBuffer,
2426 pinned);
2427 Register value = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
2428 __ LoadTaggedPointer(value, globals_buffer, no_reg,
2429 wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(
2430 imm.global->offset),
2431 pinned);
2432 __ PushRegister(kind, LiftoffRegister(value));
2433 return;
2434 }
2435 LiftoffRegList pinned;
2436 uint32_t offset = 0;
2437 Register addr = GetGlobalBaseAndOffset(global, &pinned, &offset);
2438 LiftoffRegister value =
2439 pinned.set(__ GetUnusedRegister(reg_class_for(kind), pinned));
2440 LoadType type = LoadType::ForValueKind(kind);
2441 __ Load(value, addr, no_reg, offset, type, pinned, nullptr, false);
2442 __ PushRegister(kind, value);
2443 }
2444
GlobalSet(FullDecoder * decoder,const Value &,const GlobalIndexImmediate<validate> & imm)2445 void GlobalSet(FullDecoder* decoder, const Value&,
2446 const GlobalIndexImmediate<validate>& imm) {
2447 auto* global = &env_->module->globals[imm.index];
2448 ValueKind kind = global->type.kind();
2449 if (!CheckSupportedType(decoder, kind, "global")) {
2450 return;
2451 }
2452
2453 if (is_reference(kind)) {
2454 if (global->mutability && global->imported) {
2455 LiftoffRegList pinned;
2456 LiftoffRegister value = pinned.set(__ PopToRegister(pinned));
2457 Register base = no_reg;
2458 Register offset = no_reg;
2459 GetBaseAndOffsetForImportedMutableExternRefGlobal(global, &pinned,
2460 &base, &offset);
2461 __ StoreTaggedPointer(base, offset, 0, value, pinned);
2462 return;
2463 }
2464
2465 LiftoffRegList pinned;
2466 Register globals_buffer =
2467 pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
2468 LOAD_TAGGED_PTR_INSTANCE_FIELD(globals_buffer, TaggedGlobalsBuffer,
2469 pinned);
2470 LiftoffRegister value = pinned.set(__ PopToRegister(pinned));
2471 __ StoreTaggedPointer(globals_buffer, no_reg,
2472 wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(
2473 imm.global->offset),
2474 value, pinned);
2475 return;
2476 }
2477 LiftoffRegList pinned;
2478 uint32_t offset = 0;
2479 Register addr = GetGlobalBaseAndOffset(global, &pinned, &offset);
2480 LiftoffRegister reg = pinned.set(__ PopToRegister(pinned));
2481 StoreType type = StoreType::ForValueKind(kind);
2482 __ Store(addr, no_reg, offset, reg, type, {}, nullptr, false);
2483 }
2484
TableGet(FullDecoder * decoder,const Value &,Value *,const IndexImmediate<validate> & imm)2485 void TableGet(FullDecoder* decoder, const Value&, Value*,
2486 const IndexImmediate<validate>& imm) {
2487 LiftoffRegList pinned;
2488
2489 LiftoffRegister table_index_reg =
2490 pinned.set(__ GetUnusedRegister(kGpReg, pinned));
2491 __ LoadConstant(table_index_reg, WasmValue(imm.index));
2492 LiftoffAssembler::VarState table_index(kPointerKind, table_index_reg, 0);
2493
2494 LiftoffAssembler::VarState index = __ cache_state()->stack_state.back();
2495
2496 ValueKind result_kind = env_->module->tables[imm.index].type.kind();
2497 CallRuntimeStub(WasmCode::kWasmTableGet,
2498 MakeSig::Returns(result_kind).Params(kI32, kI32),
2499 {table_index, index}, decoder->position());
2500
2501 // Pop parameters from the value stack.
2502 __ cache_state()->stack_state.pop_back(1);
2503
2504 RegisterDebugSideTableEntry(decoder, DebugSideTableBuilder::kDidSpill);
2505
2506 __ PushRegister(result_kind, LiftoffRegister(kReturnRegister0));
2507 }
2508
TableSet(FullDecoder * decoder,const Value &,const Value &,const IndexImmediate<validate> & imm)2509 void TableSet(FullDecoder* decoder, const Value&, const Value&,
2510 const IndexImmediate<validate>& imm) {
2511 LiftoffRegList pinned;
2512
2513 LiftoffRegister table_index_reg =
2514 pinned.set(__ GetUnusedRegister(kGpReg, pinned));
2515 __ LoadConstant(table_index_reg, WasmValue(imm.index));
2516 LiftoffAssembler::VarState table_index(kPointerKind, table_index_reg, 0);
2517
2518 LiftoffAssembler::VarState value = __ cache_state()->stack_state.end()[-1];
2519 LiftoffAssembler::VarState index = __ cache_state()->stack_state.end()[-2];
2520
2521 ValueKind table_kind = env_->module->tables[imm.index].type.kind();
2522
2523 CallRuntimeStub(WasmCode::kWasmTableSet,
2524 MakeSig::Params(kI32, kI32, table_kind),
2525 {table_index, index, value}, decoder->position());
2526
2527 // Pop parameters from the value stack.
2528 __ cache_state()->stack_state.pop_back(2);
2529
2530 RegisterDebugSideTableEntry(decoder, DebugSideTableBuilder::kDidSpill);
2531 }
2532
GetRuntimeStubIdForTrapReason(TrapReason reason)2533 WasmCode::RuntimeStubId GetRuntimeStubIdForTrapReason(TrapReason reason) {
2534 switch (reason) {
2535 #define RUNTIME_STUB_FOR_TRAP(trap_reason) \
2536 case k##trap_reason: \
2537 return WasmCode::kThrowWasm##trap_reason;
2538
2539 FOREACH_WASM_TRAPREASON(RUNTIME_STUB_FOR_TRAP)
2540 #undef RUNTIME_STUB_FOR_TRAP
2541 default:
2542 UNREACHABLE();
2543 }
2544 }
2545
Trap(FullDecoder * decoder,TrapReason reason)2546 void Trap(FullDecoder* decoder, TrapReason reason) {
2547 Label* trap_label =
2548 AddOutOfLineTrap(decoder, GetRuntimeStubIdForTrapReason(reason));
2549 __ emit_jump(trap_label);
2550 __ AssertUnreachable(AbortReason::kUnexpectedReturnFromWasmTrap);
2551 }
2552
AssertNull(FullDecoder * decoder,const Value & arg,Value * result)2553 void AssertNull(FullDecoder* decoder, const Value& arg, Value* result) {
2554 LiftoffRegList pinned;
2555 LiftoffRegister obj = pinned.set(__ PopToRegister(pinned));
2556 Label* trap_label =
2557 AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapIllegalCast);
2558 LiftoffRegister null = __ GetUnusedRegister(kGpReg, pinned);
2559 LoadNullValue(null.gp(), pinned);
2560 __ emit_cond_jump(kUnequal, trap_label, kOptRef, obj.gp(), null.gp());
2561 __ PushRegister(kOptRef, obj);
2562 }
2563
NopForTestingUnsupportedInLiftoff(FullDecoder * decoder)2564 void NopForTestingUnsupportedInLiftoff(FullDecoder* decoder) {
2565 unsupported(decoder, kOtherReason, "testing opcode");
2566 }
2567
Select(FullDecoder * decoder,const Value & cond,const Value & fval,const Value & tval,Value * result)2568 void Select(FullDecoder* decoder, const Value& cond, const Value& fval,
2569 const Value& tval, Value* result) {
2570 LiftoffRegList pinned;
2571 Register condition = pinned.set(__ PopToRegister()).gp();
2572 ValueKind kind = __ cache_state()->stack_state.end()[-1].kind();
2573 DCHECK(CheckCompatibleStackSlotTypes(
2574 kind, __ cache_state()->stack_state.end()[-2].kind()));
2575 LiftoffRegister false_value = pinned.set(__ PopToRegister(pinned));
2576 LiftoffRegister true_value = __ PopToRegister(pinned);
2577 LiftoffRegister dst = __ GetUnusedRegister(true_value.reg_class(),
2578 {true_value, false_value}, {});
2579 if (!__ emit_select(dst, condition, true_value, false_value, kind)) {
2580 // Emit generic code (using branches) instead.
2581 Label cont;
2582 Label case_false;
2583 __ emit_cond_jump(kEqual, &case_false, kI32, condition);
2584 if (dst != true_value) __ Move(dst, true_value, kind);
2585 __ emit_jump(&cont);
2586
2587 __ bind(&case_false);
2588 if (dst != false_value) __ Move(dst, false_value, kind);
2589 __ bind(&cont);
2590 }
2591 __ PushRegister(kind, dst);
2592 }
2593
BrImpl(FullDecoder * decoder,Control * target)2594 void BrImpl(FullDecoder* decoder, Control* target) {
2595 if (dynamic_tiering()) {
2596 if (target->is_loop()) {
2597 DCHECK(target->label.get()->is_bound());
2598 int jump_distance = __ pc_offset() - target->label.get()->pos();
2599 // For now we just add one as the cost for the tier up check. We might
2600 // want to revisit this when tuning tiering budgets later.
2601 const int kTierUpCheckCost = 1;
2602 TierupCheck(decoder, decoder->position(),
2603 jump_distance + kTierUpCheckCost);
2604 } else {
2605 // To estimate time spent in this function more accurately, we could
2606 // increment the tiering budget on forward jumps. However, we don't
2607 // know the jump distance yet; using a blanket value has been tried
2608 // and found to not make a difference.
2609 }
2610 }
2611 if (!target->br_merge()->reached) {
2612 target->label_state.InitMerge(
2613 *__ cache_state(), __ num_locals(), target->br_merge()->arity,
2614 target->stack_depth + target->num_exceptions);
2615 }
2616 __ MergeStackWith(target->label_state, target->br_merge()->arity,
2617 target->is_loop() ? LiftoffAssembler::kBackwardJump
2618 : LiftoffAssembler::kForwardJump);
2619 __ jmp(target->label.get());
2620 }
2621
BrOrRet(FullDecoder * decoder,uint32_t depth,uint32_t)2622 void BrOrRet(FullDecoder* decoder, uint32_t depth,
2623 uint32_t /* drop_values */) {
2624 BrOrRetImpl(decoder, depth);
2625 }
2626
BrOrRetImpl(FullDecoder * decoder,uint32_t depth)2627 void BrOrRetImpl(FullDecoder* decoder, uint32_t depth) {
2628 if (depth == decoder->control_depth() - 1) {
2629 DoReturn(decoder, 0);
2630 } else {
2631 BrImpl(decoder, decoder->control_at(depth));
2632 }
2633 }
2634
BrIf(FullDecoder * decoder,const Value &,uint32_t depth)2635 void BrIf(FullDecoder* decoder, const Value& /* cond */, uint32_t depth) {
2636 // Before branching, materialize all constants. This avoids repeatedly
2637 // materializing them for each conditional branch.
2638 // TODO(clemensb): Do the same for br_table.
2639 if (depth != decoder->control_depth() - 1) {
2640 __ MaterializeMergedConstants(
2641 decoder->control_at(depth)->br_merge()->arity);
2642 }
2643
2644 Label cont_false;
2645
2646 // Test the condition on the value stack, jump to {cont_false} if zero.
2647 JumpIfFalse(decoder, &cont_false);
2648
2649 // As a quickfix for https://crbug.com/1314184 we store the cache state
2650 // before calling {BrOrRetImpl} under dynamic tiering, because the tier up
2651 // check modifies the cache state (GetUnusedRegister,
2652 // LoadInstanceIntoRegister).
2653 // TODO(wasm): This causes significant overhead during compilation; try to
2654 // avoid this, maybe by passing in scratch registers.
2655 if (dynamic_tiering()) {
2656 LiftoffAssembler::CacheState old_cache_state;
2657 old_cache_state.Split(*__ cache_state());
2658 BrOrRetImpl(decoder, depth);
2659 __ cache_state()->Steal(old_cache_state);
2660 } else {
2661 BrOrRetImpl(decoder, depth);
2662 }
2663
2664 __ bind(&cont_false);
2665 }
2666
2667 // Generate a branch table case, potentially reusing previously generated
2668 // stack transfer code.
GenerateBrCase(FullDecoder * decoder,uint32_t br_depth,std::map<uint32_t,MovableLabel> * br_targets)2669 void GenerateBrCase(FullDecoder* decoder, uint32_t br_depth,
2670 std::map<uint32_t, MovableLabel>* br_targets) {
2671 MovableLabel& label = (*br_targets)[br_depth];
2672 if (label.get()->is_bound()) {
2673 __ jmp(label.get());
2674 } else {
2675 __ bind(label.get());
2676 BrOrRet(decoder, br_depth, 0);
2677 }
2678 }
2679
2680 // Generate a branch table for input in [min, max).
2681 // TODO(wasm): Generate a real branch table (like TF TableSwitch).
GenerateBrTable(FullDecoder * decoder,LiftoffRegister tmp,LiftoffRegister value,uint32_t min,uint32_t max,BranchTableIterator<validate> * table_iterator,std::map<uint32_t,MovableLabel> * br_targets)2682 void GenerateBrTable(FullDecoder* decoder, LiftoffRegister tmp,
2683 LiftoffRegister value, uint32_t min, uint32_t max,
2684 BranchTableIterator<validate>* table_iterator,
2685 std::map<uint32_t, MovableLabel>* br_targets) {
2686 DCHECK_LT(min, max);
2687 // Check base case.
2688 if (max == min + 1) {
2689 DCHECK_EQ(min, table_iterator->cur_index());
2690 GenerateBrCase(decoder, table_iterator->next(), br_targets);
2691 return;
2692 }
2693
2694 uint32_t split = min + (max - min) / 2;
2695 Label upper_half;
2696 __ LoadConstant(tmp, WasmValue(split));
2697 __ emit_cond_jump(kUnsignedGreaterEqual, &upper_half, kI32, value.gp(),
2698 tmp.gp());
2699 // Emit br table for lower half:
2700 GenerateBrTable(decoder, tmp, value, min, split, table_iterator,
2701 br_targets);
2702 __ bind(&upper_half);
2703 // table_iterator will trigger a DCHECK if we don't stop decoding now.
2704 if (did_bailout()) return;
2705 // Emit br table for upper half:
2706 GenerateBrTable(decoder, tmp, value, split, max, table_iterator,
2707 br_targets);
2708 }
2709
BrTable(FullDecoder * decoder,const BranchTableImmediate<validate> & imm,const Value & key)2710 void BrTable(FullDecoder* decoder, const BranchTableImmediate<validate>& imm,
2711 const Value& key) {
2712 LiftoffRegList pinned;
2713 LiftoffRegister value = pinned.set(__ PopToRegister());
2714 BranchTableIterator<validate> table_iterator(decoder, imm);
2715 std::map<uint32_t, MovableLabel> br_targets;
2716
2717 if (imm.table_count > 0) {
2718 LiftoffRegister tmp = __ GetUnusedRegister(kGpReg, pinned);
2719 __ LoadConstant(tmp, WasmValue(uint32_t{imm.table_count}));
2720 Label case_default;
2721 __ emit_cond_jump(kUnsignedGreaterEqual, &case_default, kI32, value.gp(),
2722 tmp.gp());
2723
2724 GenerateBrTable(decoder, tmp, value, 0, imm.table_count, &table_iterator,
2725 &br_targets);
2726
2727 __ bind(&case_default);
2728 // table_iterator will trigger a DCHECK if we don't stop decoding now.
2729 if (did_bailout()) return;
2730 }
2731
2732 // Generate the default case.
2733 GenerateBrCase(decoder, table_iterator.next(), &br_targets);
2734 DCHECK(!table_iterator.has_next());
2735 }
2736
Else(FullDecoder * decoder,Control * c)2737 void Else(FullDecoder* decoder, Control* c) {
2738 if (c->reachable()) {
2739 if (!c->end_merge.reached) {
2740 c->label_state.InitMerge(*__ cache_state(), __ num_locals(),
2741 c->end_merge.arity,
2742 c->stack_depth + c->num_exceptions);
2743 }
2744 __ MergeFullStackWith(c->label_state, *__ cache_state());
2745 __ emit_jump(c->label.get());
2746 }
2747 __ bind(c->else_state->label.get());
2748 __ cache_state()->Steal(c->else_state->state);
2749 }
2750
GetSpilledRegistersForInspection()2751 SpilledRegistersForInspection* GetSpilledRegistersForInspection() {
2752 DCHECK(for_debugging_);
2753 // If we are generating debugging code, we really need to spill all
2754 // registers to make them inspectable when stopping at the trap.
2755 auto* spilled = compilation_zone_->New<SpilledRegistersForInspection>(
2756 compilation_zone_);
2757 for (uint32_t i = 0, e = __ cache_state()->stack_height(); i < e; ++i) {
2758 auto& slot = __ cache_state()->stack_state[i];
2759 if (!slot.is_reg()) continue;
2760 spilled->entries.push_back(SpilledRegistersForInspection::Entry{
2761 slot.offset(), slot.reg(), slot.kind()});
2762 __ RecordUsedSpillOffset(slot.offset());
2763 }
2764 return spilled;
2765 }
2766
AddOutOfLineTrap(FullDecoder * decoder,WasmCode::RuntimeStubId stub,uint32_t pc=0)2767 Label* AddOutOfLineTrap(FullDecoder* decoder, WasmCode::RuntimeStubId stub,
2768 uint32_t pc = 0) {
2769 // Only memory OOB traps need a {pc}.
2770 DCHECK_IMPLIES(stub != WasmCode::kThrowWasmTrapMemOutOfBounds, pc == 0);
2771 DCHECK(FLAG_wasm_bounds_checks);
2772 OutOfLineSafepointInfo* safepoint_info = nullptr;
2773 if (V8_UNLIKELY(for_debugging_)) {
2774 // Execution does not return after a trap. Therefore we don't have to
2775 // define a safepoint for traps that would preserve references on the
2776 // stack. However, if this is debug code, then we have to preserve the
2777 // references so that they can be inspected.
2778 safepoint_info =
2779 compilation_zone_->New<OutOfLineSafepointInfo>(compilation_zone_);
2780 __ cache_state()->GetTaggedSlotsForOOLCode(
2781 &safepoint_info->slots, &safepoint_info->spills,
2782 LiftoffAssembler::CacheState::SpillLocation::kStackSlots);
2783 }
2784 out_of_line_code_.push_back(OutOfLineCode::Trap(
2785 stub, decoder->position(),
2786 V8_UNLIKELY(for_debugging_) ? GetSpilledRegistersForInspection()
2787 : nullptr,
2788 safepoint_info, pc, RegisterOOLDebugSideTableEntry(decoder)));
2789 return out_of_line_code_.back().label.get();
2790 }
2791
2792 enum ForceCheck : bool { kDoForceCheck = true, kDontForceCheck = false };
2793
2794 // Returns {no_reg} if the memory access is statically known to be out of
2795 // bounds (a jump to the trap was generated then); return the GP {index}
2796 // register otherwise (holding the ptrsized index).
BoundsCheckMem(FullDecoder * decoder,uint32_t access_size,uint64_t offset,LiftoffRegister index,LiftoffRegList pinned,ForceCheck force_check)2797 Register BoundsCheckMem(FullDecoder* decoder, uint32_t access_size,
2798 uint64_t offset, LiftoffRegister index,
2799 LiftoffRegList pinned, ForceCheck force_check) {
2800 const bool statically_oob =
2801 !base::IsInBounds<uintptr_t>(offset, access_size,
2802 env_->max_memory_size);
2803
2804 // After bounds checking, we know that the index must be ptrsize, hence only
2805 // look at the lower word on 32-bit systems (the high word is bounds-checked
2806 // further down).
2807 Register index_ptrsize =
2808 kNeedI64RegPair && index.is_gp_pair() ? index.low_gp() : index.gp();
2809
2810 // Without bounds checks (testing only), just return the ptrsize index.
2811 if (V8_UNLIKELY(env_->bounds_checks == kNoBoundsChecks)) {
2812 return index_ptrsize;
2813 }
2814
2815 // Early return for trap handler.
2816 DCHECK_IMPLIES(env_->module->is_memory64,
2817 env_->bounds_checks == kExplicitBoundsChecks);
2818 if (!force_check && !statically_oob &&
2819 env_->bounds_checks == kTrapHandler) {
2820 // With trap handlers we should not have a register pair as input (we
2821 // would only return the lower half).
2822 DCHECK(index.is_gp());
2823 return index_ptrsize;
2824 }
2825
2826 CODE_COMMENT("bounds check memory");
2827
2828 // Set {pc} of the OOL code to {0} to avoid generation of protected
2829 // instruction information (see {GenerateOutOfLineCode}.
2830 Label* trap_label =
2831 AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapMemOutOfBounds, 0);
2832
2833 if (V8_UNLIKELY(statically_oob)) {
2834 __ emit_jump(trap_label);
2835 decoder->SetSucceedingCodeDynamicallyUnreachable();
2836 return no_reg;
2837 }
2838
2839 // Convert the index to ptrsize, bounds-checking the high word on 32-bit
2840 // systems for memory64.
2841 if (!env_->module->is_memory64) {
2842 __ emit_u32_to_uintptr(index_ptrsize, index_ptrsize);
2843 } else if (kSystemPointerSize == kInt32Size) {
2844 DCHECK_GE(kMaxUInt32, env_->max_memory_size);
2845 __ emit_cond_jump(kNotEqualZero, trap_label, kI32, index.high_gp());
2846 }
2847
2848 uintptr_t end_offset = offset + access_size - 1u;
2849
2850 pinned.set(index_ptrsize);
2851 LiftoffRegister end_offset_reg =
2852 pinned.set(__ GetUnusedRegister(kGpReg, pinned));
2853 LiftoffRegister mem_size = __ GetUnusedRegister(kGpReg, pinned);
2854 LOAD_INSTANCE_FIELD(mem_size.gp(), MemorySize, kSystemPointerSize, pinned);
2855
2856 __ LoadConstant(end_offset_reg, WasmValue::ForUintPtr(end_offset));
2857
2858 // If the end offset is larger than the smallest memory, dynamically check
2859 // the end offset against the actual memory size, which is not known at
2860 // compile time. Otherwise, only one check is required (see below).
2861 if (end_offset > env_->min_memory_size) {
2862 __ emit_cond_jump(kUnsignedGreaterEqual, trap_label, kPointerKind,
2863 end_offset_reg.gp(), mem_size.gp());
2864 }
2865
2866 // Just reuse the end_offset register for computing the effective size
2867 // (which is >= 0 because of the check above).
2868 LiftoffRegister effective_size_reg = end_offset_reg;
2869 __ emit_ptrsize_sub(effective_size_reg.gp(), mem_size.gp(),
2870 end_offset_reg.gp());
2871
2872 __ emit_cond_jump(kUnsignedGreaterEqual, trap_label, kPointerKind,
2873 index_ptrsize, effective_size_reg.gp());
2874 return index_ptrsize;
2875 }
2876
AlignmentCheckMem(FullDecoder * decoder,uint32_t access_size,uintptr_t offset,Register index,LiftoffRegList pinned)2877 void AlignmentCheckMem(FullDecoder* decoder, uint32_t access_size,
2878 uintptr_t offset, Register index,
2879 LiftoffRegList pinned) {
2880 CODE_COMMENT("alignment check");
2881 Label* trap_label =
2882 AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapUnalignedAccess, 0);
2883 Register address = __ GetUnusedRegister(kGpReg, pinned).gp();
2884
2885 const uint32_t align_mask = access_size - 1;
2886 if ((offset & align_mask) == 0) {
2887 // If {offset} is aligned, we can produce faster code.
2888
2889 // TODO(ahaas): On Intel, the "test" instruction implicitly computes the
2890 // AND of two operands. We could introduce a new variant of
2891 // {emit_cond_jump} to use the "test" instruction without the "and" here.
2892 // Then we can also avoid using the temp register here.
2893 __ emit_i32_andi(address, index, align_mask);
2894 __ emit_cond_jump(kUnequal, trap_label, kI32, address);
2895 } else {
2896 // For alignment checks we only look at the lower 32-bits in {offset}.
2897 __ emit_i32_addi(address, index, static_cast<uint32_t>(offset));
2898 __ emit_i32_andi(address, address, align_mask);
2899 __ emit_cond_jump(kUnequal, trap_label, kI32, address);
2900 }
2901 }
2902
TraceMemoryOperation(bool is_store,MachineRepresentation rep,Register index,uintptr_t offset,WasmCodePosition position)2903 void TraceMemoryOperation(bool is_store, MachineRepresentation rep,
2904 Register index, uintptr_t offset,
2905 WasmCodePosition position) {
2906 // Before making the runtime call, spill all cache registers.
2907 __ SpillAllRegisters();
2908
2909 LiftoffRegList pinned;
2910 if (index != no_reg) pinned.set(index);
2911 // Get one register for computing the effective offset (offset + index).
2912 LiftoffRegister effective_offset =
2913 pinned.set(__ GetUnusedRegister(kGpReg, pinned));
2914 DCHECK_GE(kMaxUInt32, offset);
2915 __ LoadConstant(effective_offset, WasmValue(static_cast<uint32_t>(offset)));
2916 if (index != no_reg) {
2917 // TODO(clemensb): Do a 64-bit addition here if memory64 is used.
2918 __ emit_i32_add(effective_offset.gp(), effective_offset.gp(), index);
2919 }
2920
2921 // Get a register to hold the stack slot for MemoryTracingInfo.
2922 LiftoffRegister info = pinned.set(__ GetUnusedRegister(kGpReg, pinned));
2923 // Allocate stack slot for MemoryTracingInfo.
2924 __ AllocateStackSlot(info.gp(), sizeof(MemoryTracingInfo));
2925
2926 // Reuse the {effective_offset} register for all information to be stored in
2927 // the MemoryTracingInfo struct.
2928 LiftoffRegister data = effective_offset;
2929
2930 // Now store all information into the MemoryTracingInfo struct.
2931 if (kSystemPointerSize == 8) {
2932 // Zero-extend the effective offset to u64.
2933 CHECK(__ emit_type_conversion(kExprI64UConvertI32, data, effective_offset,
2934 nullptr));
2935 }
2936 __ Store(
2937 info.gp(), no_reg, offsetof(MemoryTracingInfo, offset), data,
2938 kSystemPointerSize == 8 ? StoreType::kI64Store : StoreType::kI32Store,
2939 pinned);
2940 __ LoadConstant(data, WasmValue(is_store ? 1 : 0));
2941 __ Store(info.gp(), no_reg, offsetof(MemoryTracingInfo, is_store), data,
2942 StoreType::kI32Store8, pinned);
2943 __ LoadConstant(data, WasmValue(static_cast<int>(rep)));
2944 __ Store(info.gp(), no_reg, offsetof(MemoryTracingInfo, mem_rep), data,
2945 StoreType::kI32Store8, pinned);
2946
2947 WasmTraceMemoryDescriptor descriptor;
2948 DCHECK_EQ(0, descriptor.GetStackParameterCount());
2949 DCHECK_EQ(1, descriptor.GetRegisterParameterCount());
2950 Register param_reg = descriptor.GetRegisterParameter(0);
2951 if (info.gp() != param_reg) {
2952 __ Move(param_reg, info.gp(), kPointerKind);
2953 }
2954
2955 source_position_table_builder_.AddPosition(__ pc_offset(),
2956 SourcePosition(position), false);
2957 __ CallRuntimeStub(WasmCode::kWasmTraceMemory);
2958 DefineSafepoint();
2959
2960 __ DeallocateStackSlot(sizeof(MemoryTracingInfo));
2961 }
2962
IndexStaticallyInBounds(const LiftoffAssembler::VarState & index_slot,int access_size,uintptr_t * offset)2963 bool IndexStaticallyInBounds(const LiftoffAssembler::VarState& index_slot,
2964 int access_size, uintptr_t* offset) {
2965 if (!index_slot.is_const()) return false;
2966
2967 // Potentially zero extend index (which is a 32-bit constant).
2968 const uintptr_t index = static_cast<uint32_t>(index_slot.i32_const());
2969 const uintptr_t effective_offset = index + *offset;
2970
2971 if (effective_offset < index // overflow
2972 || !base::IsInBounds<uintptr_t>(effective_offset, access_size,
2973 env_->min_memory_size)) {
2974 return false;
2975 }
2976
2977 *offset = effective_offset;
2978 return true;
2979 }
2980
GetMemoryStart(LiftoffRegList pinned)2981 Register GetMemoryStart(LiftoffRegList pinned) {
2982 Register memory_start = __ cache_state()->cached_mem_start;
2983 if (memory_start == no_reg) {
2984 memory_start = __ GetUnusedRegister(kGpReg, pinned).gp();
2985 LOAD_INSTANCE_FIELD(memory_start, MemoryStart, kSystemPointerSize,
2986 pinned);
2987 #ifdef V8_SANDBOXED_POINTERS
2988 __ DecodeSandboxedPointer(memory_start);
2989 #endif
2990 __ cache_state()->SetMemStartCacheRegister(memory_start);
2991 }
2992 return memory_start;
2993 }
2994
LoadMem(FullDecoder * decoder,LoadType type,const MemoryAccessImmediate<validate> & imm,const Value & index_val,Value * result)2995 void LoadMem(FullDecoder* decoder, LoadType type,
2996 const MemoryAccessImmediate<validate>& imm,
2997 const Value& index_val, Value* result) {
2998 ValueKind kind = type.value_type().kind();
2999 RegClass rc = reg_class_for(kind);
3000 if (!CheckSupportedType(decoder, kind, "load")) return;
3001
3002 uintptr_t offset = imm.offset;
3003 Register index = no_reg;
3004
3005 // Only look at the slot, do not pop it yet (will happen in PopToRegister
3006 // below, if this is not a statically-in-bounds index).
3007 auto& index_slot = __ cache_state()->stack_state.back();
3008 bool i64_offset = index_val.type == kWasmI64;
3009 if (IndexStaticallyInBounds(index_slot, type.size(), &offset)) {
3010 __ cache_state()->stack_state.pop_back();
3011 CODE_COMMENT("load from memory (constant offset)");
3012 LiftoffRegList pinned;
3013 Register mem = pinned.set(GetMemoryStart(pinned));
3014 LiftoffRegister value = pinned.set(__ GetUnusedRegister(rc, pinned));
3015 __ Load(value, mem, no_reg, offset, type, pinned, nullptr, true,
3016 i64_offset);
3017 __ PushRegister(kind, value);
3018 } else {
3019 LiftoffRegister full_index = __ PopToRegister();
3020 index = BoundsCheckMem(decoder, type.size(), offset, full_index, {},
3021 kDontForceCheck);
3022 if (index == no_reg) return;
3023
3024 CODE_COMMENT("load from memory");
3025 LiftoffRegList pinned = {index};
3026
3027 // Load the memory start address only now to reduce register pressure
3028 // (important on ia32).
3029 Register mem = pinned.set(GetMemoryStart(pinned));
3030 LiftoffRegister value = pinned.set(__ GetUnusedRegister(rc, pinned));
3031
3032 uint32_t protected_load_pc = 0;
3033 __ Load(value, mem, index, offset, type, pinned, &protected_load_pc, true,
3034 i64_offset);
3035 if (env_->bounds_checks == kTrapHandler) {
3036 AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapMemOutOfBounds,
3037 protected_load_pc);
3038 }
3039 __ PushRegister(kind, value);
3040 }
3041
3042 if (V8_UNLIKELY(FLAG_trace_wasm_memory)) {
3043 TraceMemoryOperation(false, type.mem_type().representation(), index,
3044 offset, decoder->position());
3045 }
3046 }
3047
LoadTransform(FullDecoder * decoder,LoadType type,LoadTransformationKind transform,const MemoryAccessImmediate<validate> & imm,const Value & index_val,Value * result)3048 void LoadTransform(FullDecoder* decoder, LoadType type,
3049 LoadTransformationKind transform,
3050 const MemoryAccessImmediate<validate>& imm,
3051 const Value& index_val, Value* result) {
3052 // LoadTransform requires SIMD support, so check for it here. If
3053 // unsupported, bailout and let TurboFan lower the code.
3054 if (!CheckSupportedType(decoder, kS128, "LoadTransform")) {
3055 return;
3056 }
3057
3058 LiftoffRegister full_index = __ PopToRegister();
3059 // For load splats and load zero, LoadType is the size of the load, and for
3060 // load extends, LoadType is the size of the lane, and it always loads 8
3061 // bytes.
3062 uint32_t access_size =
3063 transform == LoadTransformationKind::kExtend ? 8 : type.size();
3064 Register index = BoundsCheckMem(decoder, access_size, imm.offset,
3065 full_index, {}, kDontForceCheck);
3066 if (index == no_reg) return;
3067
3068 uintptr_t offset = imm.offset;
3069 LiftoffRegList pinned = {index};
3070 CODE_COMMENT("load with transformation");
3071 Register addr = GetMemoryStart(pinned);
3072 LiftoffRegister value = __ GetUnusedRegister(reg_class_for(kS128), {});
3073 uint32_t protected_load_pc = 0;
3074 __ LoadTransform(value, addr, index, offset, type, transform,
3075 &protected_load_pc);
3076
3077 if (env_->bounds_checks == kTrapHandler) {
3078 AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapMemOutOfBounds,
3079 protected_load_pc);
3080 }
3081 __ PushRegister(kS128, value);
3082
3083 if (V8_UNLIKELY(FLAG_trace_wasm_memory)) {
3084 // Again load extend is different.
3085 MachineRepresentation mem_rep =
3086 transform == LoadTransformationKind::kExtend
3087 ? MachineRepresentation::kWord64
3088 : type.mem_type().representation();
3089 TraceMemoryOperation(false, mem_rep, index, offset, decoder->position());
3090 }
3091 }
3092
LoadLane(FullDecoder * decoder,LoadType type,const Value & _value,const Value & _index,const MemoryAccessImmediate<validate> & imm,const uint8_t laneidx,Value * _result)3093 void LoadLane(FullDecoder* decoder, LoadType type, const Value& _value,
3094 const Value& _index, const MemoryAccessImmediate<validate>& imm,
3095 const uint8_t laneidx, Value* _result) {
3096 if (!CheckSupportedType(decoder, kS128, "LoadLane")) {
3097 return;
3098 }
3099
3100 LiftoffRegList pinned;
3101 LiftoffRegister value = pinned.set(__ PopToRegister());
3102 LiftoffRegister full_index = __ PopToRegister();
3103 Register index = BoundsCheckMem(decoder, type.size(), imm.offset,
3104 full_index, pinned, kDontForceCheck);
3105 if (index == no_reg) return;
3106
3107 uintptr_t offset = imm.offset;
3108 pinned.set(index);
3109 CODE_COMMENT("load lane");
3110 Register addr = GetMemoryStart(pinned);
3111 LiftoffRegister result = __ GetUnusedRegister(reg_class_for(kS128), {});
3112 uint32_t protected_load_pc = 0;
3113
3114 __ LoadLane(result, value, addr, index, offset, type, laneidx,
3115 &protected_load_pc);
3116 if (env_->bounds_checks == kTrapHandler) {
3117 AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapMemOutOfBounds,
3118 protected_load_pc);
3119 }
3120
3121 __ PushRegister(kS128, result);
3122
3123 if (V8_UNLIKELY(FLAG_trace_wasm_memory)) {
3124 TraceMemoryOperation(false, type.mem_type().representation(), index,
3125 offset, decoder->position());
3126 }
3127 }
3128
StoreMem(FullDecoder * decoder,StoreType type,const MemoryAccessImmediate<validate> & imm,const Value & index_val,const Value & value_val)3129 void StoreMem(FullDecoder* decoder, StoreType type,
3130 const MemoryAccessImmediate<validate>& imm,
3131 const Value& index_val, const Value& value_val) {
3132 ValueKind kind = type.value_type().kind();
3133 if (!CheckSupportedType(decoder, kind, "store")) return;
3134
3135 LiftoffRegList pinned;
3136 LiftoffRegister value = pinned.set(__ PopToRegister());
3137
3138 uintptr_t offset = imm.offset;
3139 Register index = no_reg;
3140
3141 auto& index_slot = __ cache_state()->stack_state.back();
3142 if (IndexStaticallyInBounds(index_slot, type.size(), &offset)) {
3143 __ cache_state()->stack_state.pop_back();
3144 CODE_COMMENT("store to memory (constant offset)");
3145 Register mem = pinned.set(GetMemoryStart(pinned));
3146 __ Store(mem, no_reg, offset, value, type, pinned, nullptr, true);
3147 } else {
3148 LiftoffRegister full_index = __ PopToRegister(pinned);
3149 index = BoundsCheckMem(decoder, type.size(), imm.offset, full_index,
3150 pinned, kDontForceCheck);
3151 if (index == no_reg) return;
3152
3153 pinned.set(index);
3154 CODE_COMMENT("store to memory");
3155 uint32_t protected_store_pc = 0;
3156 // Load the memory start address only now to reduce register pressure
3157 // (important on ia32).
3158 Register mem = pinned.set(GetMemoryStart(pinned));
3159 LiftoffRegList outer_pinned;
3160 if (V8_UNLIKELY(FLAG_trace_wasm_memory)) outer_pinned.set(index);
3161 __ Store(mem, index, offset, value, type, outer_pinned,
3162 &protected_store_pc, true);
3163 if (env_->bounds_checks == kTrapHandler) {
3164 AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapMemOutOfBounds,
3165 protected_store_pc);
3166 }
3167 }
3168
3169 if (V8_UNLIKELY(FLAG_trace_wasm_memory)) {
3170 TraceMemoryOperation(true, type.mem_rep(), index, offset,
3171 decoder->position());
3172 }
3173 }
3174
StoreLane(FullDecoder * decoder,StoreType type,const MemoryAccessImmediate<validate> & imm,const Value & _index,const Value & _value,const uint8_t lane)3175 void StoreLane(FullDecoder* decoder, StoreType type,
3176 const MemoryAccessImmediate<validate>& imm,
3177 const Value& _index, const Value& _value, const uint8_t lane) {
3178 if (!CheckSupportedType(decoder, kS128, "StoreLane")) return;
3179 LiftoffRegList pinned;
3180 LiftoffRegister value = pinned.set(__ PopToRegister());
3181 LiftoffRegister full_index = __ PopToRegister(pinned);
3182 Register index = BoundsCheckMem(decoder, type.size(), imm.offset,
3183 full_index, pinned, kDontForceCheck);
3184 if (index == no_reg) return;
3185
3186 uintptr_t offset = imm.offset;
3187 pinned.set(index);
3188 CODE_COMMENT("store lane to memory");
3189 Register addr = pinned.set(GetMemoryStart(pinned));
3190 uint32_t protected_store_pc = 0;
3191 __ StoreLane(addr, index, offset, value, type, lane, &protected_store_pc);
3192 if (env_->bounds_checks == kTrapHandler) {
3193 AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapMemOutOfBounds,
3194 protected_store_pc);
3195 }
3196 if (V8_UNLIKELY(FLAG_trace_wasm_memory)) {
3197 TraceMemoryOperation(true, type.mem_rep(), index, offset,
3198 decoder->position());
3199 }
3200 }
3201
CurrentMemoryPages(FullDecoder *,Value *)3202 void CurrentMemoryPages(FullDecoder* /* decoder */, Value* /* result */) {
3203 Register mem_size = __ GetUnusedRegister(kGpReg, {}).gp();
3204 LOAD_INSTANCE_FIELD(mem_size, MemorySize, kSystemPointerSize, {});
3205 __ emit_ptrsize_shri(mem_size, mem_size, kWasmPageSizeLog2);
3206 LiftoffRegister result{mem_size};
3207 if (env_->module->is_memory64 && kNeedI64RegPair) {
3208 LiftoffRegister high_word =
3209 __ GetUnusedRegister(kGpReg, LiftoffRegList{mem_size});
3210 // The high word is always 0 on 32-bit systems.
3211 __ LoadConstant(high_word, WasmValue{uint32_t{0}});
3212 result = LiftoffRegister::ForPair(mem_size, high_word.gp());
3213 }
3214 __ PushRegister(env_->module->is_memory64 ? kI64 : kI32, result);
3215 }
3216
MemoryGrow(FullDecoder * decoder,const Value & value,Value * result_val)3217 void MemoryGrow(FullDecoder* decoder, const Value& value, Value* result_val) {
3218 // Pop the input, then spill all cache registers to make the runtime call.
3219 LiftoffRegList pinned;
3220 LiftoffRegister input = pinned.set(__ PopToRegister());
3221 __ SpillAllRegisters();
3222
3223 LiftoffRegister result = pinned.set(__ GetUnusedRegister(kGpReg, pinned));
3224
3225 Label done;
3226
3227 if (env_->module->is_memory64) {
3228 // If the high word is not 0, this will always fail (would grow by
3229 // >=256TB). The int32_t value will be sign-extended below.
3230 __ LoadConstant(result, WasmValue(int32_t{-1}));
3231 if (kNeedI64RegPair) {
3232 __ emit_cond_jump(kUnequal /* neq */, &done, kI32, input.high_gp());
3233 input = input.low();
3234 } else {
3235 LiftoffRegister high_word = __ GetUnusedRegister(kGpReg, pinned);
3236 __ emit_i64_shri(high_word, input, 32);
3237 __ emit_cond_jump(kUnequal /* neq */, &done, kI32, high_word.gp());
3238 }
3239 }
3240
3241 WasmMemoryGrowDescriptor descriptor;
3242 DCHECK_EQ(0, descriptor.GetStackParameterCount());
3243 DCHECK_EQ(1, descriptor.GetRegisterParameterCount());
3244 DCHECK_EQ(machine_type(kI32), descriptor.GetParameterType(0));
3245
3246 Register param_reg = descriptor.GetRegisterParameter(0);
3247 if (input.gp() != param_reg) __ Move(param_reg, input.gp(), kI32);
3248
3249 __ CallRuntimeStub(WasmCode::kWasmMemoryGrow);
3250 DefineSafepoint();
3251 RegisterDebugSideTableEntry(decoder, DebugSideTableBuilder::kDidSpill);
3252
3253 if (kReturnRegister0 != result.gp()) {
3254 __ Move(result.gp(), kReturnRegister0, kI32);
3255 }
3256
3257 __ bind(&done);
3258
3259 if (env_->module->is_memory64) {
3260 LiftoffRegister result64 = result;
3261 if (kNeedI64RegPair) result64 = __ GetUnusedRegister(kGpRegPair, pinned);
3262 __ emit_type_conversion(kExprI64SConvertI32, result64, result, nullptr);
3263 __ PushRegister(kI64, result64);
3264 } else {
3265 __ PushRegister(kI32, result);
3266 }
3267 }
3268
3269 base::OwnedVector<DebugSideTable::Entry::Value>
GetCurrentDebugSideTableEntries(FullDecoder * decoder,DebugSideTableBuilder::AssumeSpilling assume_spilling)3270 GetCurrentDebugSideTableEntries(
3271 FullDecoder* decoder,
3272 DebugSideTableBuilder::AssumeSpilling assume_spilling) {
3273 auto& stack_state = __ cache_state()->stack_state;
3274 auto values =
3275 base::OwnedVector<DebugSideTable::Entry::Value>::NewForOverwrite(
3276 stack_state.size());
3277
3278 // For function calls, the decoder still has the arguments on the stack, but
3279 // Liftoff already popped them. Hence {decoder->stack_size()} can be bigger
3280 // than expected. Just ignore that and use the lower part only.
3281 DCHECK_LE(stack_state.size() - num_exceptions_,
3282 decoder->num_locals() + decoder->stack_size());
3283 int index = 0;
3284 int decoder_stack_index = decoder->stack_size();
3285 // Iterate the operand stack control block by control block, so that we can
3286 // handle the implicit exception value for try blocks.
3287 for (int j = decoder->control_depth() - 1; j >= 0; j--) {
3288 Control* control = decoder->control_at(j);
3289 Control* next_control = j > 0 ? decoder->control_at(j - 1) : nullptr;
3290 int end_index = next_control
3291 ? next_control->stack_depth + __ num_locals() +
3292 next_control->num_exceptions
3293 : __ cache_state()->stack_height();
3294 bool exception = control->is_try_catch() || control->is_try_catchall();
3295 for (; index < end_index; ++index) {
3296 auto& slot = stack_state[index];
3297 auto& value = values[index];
3298 value.index = index;
3299 ValueType type =
3300 index < static_cast<int>(__ num_locals())
3301 ? decoder->local_type(index)
3302 : exception ? ValueType::Ref(HeapType::kAny, kNonNullable)
3303 : decoder->stack_value(decoder_stack_index--)->type;
3304 DCHECK(CheckCompatibleStackSlotTypes(slot.kind(), type.kind()));
3305 value.type = type;
3306 switch (slot.loc()) {
3307 case kIntConst:
3308 value.storage = DebugSideTable::Entry::kConstant;
3309 value.i32_const = slot.i32_const();
3310 break;
3311 case kRegister:
3312 DCHECK_NE(DebugSideTableBuilder::kDidSpill, assume_spilling);
3313 if (assume_spilling == DebugSideTableBuilder::kAllowRegisters) {
3314 value.storage = DebugSideTable::Entry::kRegister;
3315 value.reg_code = slot.reg().liftoff_code();
3316 break;
3317 }
3318 DCHECK_EQ(DebugSideTableBuilder::kAssumeSpilling, assume_spilling);
3319 V8_FALLTHROUGH;
3320 case kStack:
3321 value.storage = DebugSideTable::Entry::kStack;
3322 value.stack_offset = slot.offset();
3323 break;
3324 }
3325 exception = false;
3326 }
3327 }
3328 DCHECK_EQ(values.size(), index);
3329 return values;
3330 }
3331
RegisterDebugSideTableEntry(FullDecoder * decoder,DebugSideTableBuilder::AssumeSpilling assume_spilling)3332 void RegisterDebugSideTableEntry(
3333 FullDecoder* decoder,
3334 DebugSideTableBuilder::AssumeSpilling assume_spilling) {
3335 if (V8_LIKELY(!debug_sidetable_builder_)) return;
3336 debug_sidetable_builder_->NewEntry(
3337 __ pc_offset(),
3338 GetCurrentDebugSideTableEntries(decoder, assume_spilling).as_vector());
3339 }
3340
RegisterOOLDebugSideTableEntry(FullDecoder * decoder)3341 DebugSideTableBuilder::EntryBuilder* RegisterOOLDebugSideTableEntry(
3342 FullDecoder* decoder) {
3343 if (V8_LIKELY(!debug_sidetable_builder_)) return nullptr;
3344 return debug_sidetable_builder_->NewOOLEntry(
3345 GetCurrentDebugSideTableEntries(decoder,
3346 DebugSideTableBuilder::kAssumeSpilling)
3347 .as_vector());
3348 }
3349
3350 enum TailCall : bool { kTailCall = true, kNoTailCall = false };
3351
CallDirect(FullDecoder * decoder,const CallFunctionImmediate<validate> & imm,const Value args[],Value[])3352 void CallDirect(FullDecoder* decoder,
3353 const CallFunctionImmediate<validate>& imm,
3354 const Value args[], Value[]) {
3355 CallDirect(decoder, imm, args, nullptr, kNoTailCall);
3356 }
3357
CallIndirect(FullDecoder * decoder,const Value & index_val,const CallIndirectImmediate<validate> & imm,const Value args[],Value returns[])3358 void CallIndirect(FullDecoder* decoder, const Value& index_val,
3359 const CallIndirectImmediate<validate>& imm,
3360 const Value args[], Value returns[]) {
3361 CallIndirect(decoder, index_val, imm, kNoTailCall);
3362 }
3363
CallRef(FullDecoder * decoder,const Value & func_ref,const FunctionSig * sig,uint32_t sig_index,const Value args[],Value returns[])3364 void CallRef(FullDecoder* decoder, const Value& func_ref,
3365 const FunctionSig* sig, uint32_t sig_index, const Value args[],
3366 Value returns[]) {
3367 CallRef(decoder, func_ref.type, sig, kNoTailCall);
3368 }
3369
ReturnCall(FullDecoder * decoder,const CallFunctionImmediate<validate> & imm,const Value args[])3370 void ReturnCall(FullDecoder* decoder,
3371 const CallFunctionImmediate<validate>& imm,
3372 const Value args[]) {
3373 TierupCheckOnExit(decoder);
3374 CallDirect(decoder, imm, args, nullptr, kTailCall);
3375 }
3376
ReturnCallIndirect(FullDecoder * decoder,const Value & index_val,const CallIndirectImmediate<validate> & imm,const Value args[])3377 void ReturnCallIndirect(FullDecoder* decoder, const Value& index_val,
3378 const CallIndirectImmediate<validate>& imm,
3379 const Value args[]) {
3380 TierupCheckOnExit(decoder);
3381 CallIndirect(decoder, index_val, imm, kTailCall);
3382 }
3383
ReturnCallRef(FullDecoder * decoder,const Value & func_ref,const FunctionSig * sig,uint32_t sig_index,const Value args[])3384 void ReturnCallRef(FullDecoder* decoder, const Value& func_ref,
3385 const FunctionSig* sig, uint32_t sig_index,
3386 const Value args[]) {
3387 TierupCheckOnExit(decoder);
3388 CallRef(decoder, func_ref.type, sig, kTailCall);
3389 }
3390
BrOnNull(FullDecoder * decoder,const Value & ref_object,uint32_t depth,bool pass_null_along_branch,Value *)3391 void BrOnNull(FullDecoder* decoder, const Value& ref_object, uint32_t depth,
3392 bool pass_null_along_branch,
3393 Value* /* result_on_fallthrough */) {
3394 // Before branching, materialize all constants. This avoids repeatedly
3395 // materializing them for each conditional branch.
3396 if (depth != decoder->control_depth() - 1) {
3397 __ MaterializeMergedConstants(
3398 decoder->control_at(depth)->br_merge()->arity);
3399 }
3400
3401 Label cont_false;
3402 LiftoffRegList pinned;
3403 LiftoffRegister ref = pinned.set(__ PopToRegister(pinned));
3404 Register null = __ GetUnusedRegister(kGpReg, pinned).gp();
3405 LoadNullValue(null, pinned);
3406 __ emit_cond_jump(kUnequal, &cont_false, ref_object.type.kind(), ref.gp(),
3407 null);
3408 if (pass_null_along_branch) LoadNullValue(null, pinned);
3409 BrOrRet(decoder, depth, 0);
3410 __ bind(&cont_false);
3411 __ PushRegister(kRef, ref);
3412 }
3413
BrOnNonNull(FullDecoder * decoder,const Value & ref_object,uint32_t depth)3414 void BrOnNonNull(FullDecoder* decoder, const Value& ref_object,
3415 uint32_t depth) {
3416 // Before branching, materialize all constants. This avoids repeatedly
3417 // materializing them for each conditional branch.
3418 if (depth != decoder->control_depth() - 1) {
3419 __ MaterializeMergedConstants(
3420 decoder->control_at(depth)->br_merge()->arity);
3421 }
3422
3423 Label cont_false;
3424 LiftoffRegList pinned;
3425 LiftoffRegister ref = pinned.set(__ PopToRegister(pinned));
3426 // Put the reference back onto the stack for the branch.
3427 __ PushRegister(kRef, ref);
3428
3429 Register null = __ GetUnusedRegister(kGpReg, pinned).gp();
3430 LoadNullValue(null, pinned);
3431 __ emit_cond_jump(kEqual, &cont_false, ref_object.type.kind(), ref.gp(),
3432 null);
3433
3434 BrOrRet(decoder, depth, 0);
3435 // Drop the reference if we are not branching.
3436 __ DropValues(1);
3437 __ bind(&cont_false);
3438 }
3439
3440 template <ValueKind src_kind, ValueKind result_kind,
3441 ValueKind result_lane_kind = kVoid, typename EmitFn>
EmitTerOp(EmitFn fn)3442 void EmitTerOp(EmitFn fn) {
3443 static constexpr RegClass src_rc = reg_class_for(src_kind);
3444 static constexpr RegClass result_rc = reg_class_for(result_kind);
3445 LiftoffRegister src3 = __ PopToRegister();
3446 LiftoffRegister src2 = __ PopToRegister(LiftoffRegList{src3});
3447 LiftoffRegister src1 = __ PopToRegister(LiftoffRegList{src3, src2});
3448 // Reusing src1 and src2 will complicate codegen for select for some
3449 // backend, so we allow only reusing src3 (the mask), and pin src1 and src2.
3450 LiftoffRegister dst = src_rc == result_rc
3451 ? __ GetUnusedRegister(result_rc, {src3},
3452 LiftoffRegList{src1, src2})
3453 : __ GetUnusedRegister(result_rc, {});
3454 CallEmitFn(fn, dst, src1, src2, src3);
3455 if (V8_UNLIKELY(nondeterminism_)) {
3456 LiftoffRegList pinned = {dst};
3457 if (result_kind == ValueKind::kF32 || result_kind == ValueKind::kF64) {
3458 CheckNan(dst, pinned, result_kind);
3459 } else if (result_kind == ValueKind::kS128 &&
3460 (result_lane_kind == kF32 || result_lane_kind == kF64)) {
3461 CheckS128Nan(dst, LiftoffRegList{src1, src2, src3, dst},
3462 result_lane_kind);
3463 }
3464 }
3465 __ PushRegister(result_kind, dst);
3466 }
3467
3468 template <typename EmitFn, typename EmitFnImm>
EmitSimdShiftOp(EmitFn fn,EmitFnImm fnImm)3469 void EmitSimdShiftOp(EmitFn fn, EmitFnImm fnImm) {
3470 static constexpr RegClass result_rc = reg_class_for(kS128);
3471
3472 LiftoffAssembler::VarState rhs_slot = __ cache_state()->stack_state.back();
3473 // Check if the RHS is an immediate.
3474 if (rhs_slot.is_const()) {
3475 __ cache_state()->stack_state.pop_back();
3476 int32_t imm = rhs_slot.i32_const();
3477
3478 LiftoffRegister operand = __ PopToRegister();
3479 LiftoffRegister dst = __ GetUnusedRegister(result_rc, {operand}, {});
3480
3481 CallEmitFn(fnImm, dst, operand, imm);
3482 __ PushRegister(kS128, dst);
3483 } else {
3484 LiftoffRegister count = __ PopToRegister();
3485 LiftoffRegister operand = __ PopToRegister();
3486 LiftoffRegister dst = __ GetUnusedRegister(result_rc, {operand}, {});
3487
3488 CallEmitFn(fn, dst, operand, count);
3489 __ PushRegister(kS128, dst);
3490 }
3491 }
3492
3493 template <ValueKind result_lane_kind>
EmitSimdFloatRoundingOpWithCFallback(bool (LiftoffAssembler::* emit_fn)(LiftoffRegister,LiftoffRegister),ExternalReference (* ext_ref)())3494 void EmitSimdFloatRoundingOpWithCFallback(
3495 bool (LiftoffAssembler::*emit_fn)(LiftoffRegister, LiftoffRegister),
3496 ExternalReference (*ext_ref)()) {
3497 static constexpr RegClass rc = reg_class_for(kS128);
3498 LiftoffRegister src = __ PopToRegister();
3499 LiftoffRegister dst = __ GetUnusedRegister(rc, {src}, {});
3500 if (!(asm_.*emit_fn)(dst, src)) {
3501 // Return v128 via stack for ARM.
3502 auto sig_v_s = MakeSig::Params(kS128);
3503 GenerateCCall(&dst, &sig_v_s, kS128, &src, ext_ref());
3504 }
3505 if (V8_UNLIKELY(nondeterminism_)) {
3506 LiftoffRegList pinned = {dst};
3507 CheckS128Nan(dst, pinned, result_lane_kind);
3508 }
3509 __ PushRegister(kS128, dst);
3510 }
3511
SimdOp(FullDecoder * decoder,WasmOpcode opcode,base::Vector<Value> args,Value * result)3512 void SimdOp(FullDecoder* decoder, WasmOpcode opcode, base::Vector<Value> args,
3513 Value* result) {
3514 if (!CpuFeatures::SupportsWasmSimd128()) {
3515 return unsupported(decoder, kSimd, "simd");
3516 }
3517 switch (opcode) {
3518 case wasm::kExprI8x16Swizzle:
3519 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i8x16_swizzle);
3520 case wasm::kExprI8x16Popcnt:
3521 return EmitUnOp<kS128, kS128>(&LiftoffAssembler::emit_i8x16_popcnt);
3522 case wasm::kExprI8x16Splat:
3523 return EmitUnOp<kI32, kS128>(&LiftoffAssembler::emit_i8x16_splat);
3524 case wasm::kExprI16x8Splat:
3525 return EmitUnOp<kI32, kS128>(&LiftoffAssembler::emit_i16x8_splat);
3526 case wasm::kExprI32x4Splat:
3527 return EmitUnOp<kI32, kS128>(&LiftoffAssembler::emit_i32x4_splat);
3528 case wasm::kExprI64x2Splat:
3529 return EmitUnOp<kI64, kS128>(&LiftoffAssembler::emit_i64x2_splat);
3530 case wasm::kExprF32x4Splat:
3531 return EmitUnOp<kF32, kS128, kF32>(&LiftoffAssembler::emit_f32x4_splat);
3532 case wasm::kExprF64x2Splat:
3533 return EmitUnOp<kF64, kS128, kF64>(&LiftoffAssembler::emit_f64x2_splat);
3534 case wasm::kExprI8x16Eq:
3535 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i8x16_eq);
3536 case wasm::kExprI8x16Ne:
3537 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i8x16_ne);
3538 case wasm::kExprI8x16LtS:
3539 return EmitBinOp<kS128, kS128, true>(
3540 &LiftoffAssembler::emit_i8x16_gt_s);
3541 case wasm::kExprI8x16LtU:
3542 return EmitBinOp<kS128, kS128, true>(
3543 &LiftoffAssembler::emit_i8x16_gt_u);
3544 case wasm::kExprI8x16GtS:
3545 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i8x16_gt_s);
3546 case wasm::kExprI8x16GtU:
3547 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i8x16_gt_u);
3548 case wasm::kExprI8x16LeS:
3549 return EmitBinOp<kS128, kS128, true>(
3550 &LiftoffAssembler::emit_i8x16_ge_s);
3551 case wasm::kExprI8x16LeU:
3552 return EmitBinOp<kS128, kS128, true>(
3553 &LiftoffAssembler::emit_i8x16_ge_u);
3554 case wasm::kExprI8x16GeS:
3555 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i8x16_ge_s);
3556 case wasm::kExprI8x16GeU:
3557 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i8x16_ge_u);
3558 case wasm::kExprI16x8Eq:
3559 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i16x8_eq);
3560 case wasm::kExprI16x8Ne:
3561 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i16x8_ne);
3562 case wasm::kExprI16x8LtS:
3563 return EmitBinOp<kS128, kS128, true>(
3564 &LiftoffAssembler::emit_i16x8_gt_s);
3565 case wasm::kExprI16x8LtU:
3566 return EmitBinOp<kS128, kS128, true>(
3567 &LiftoffAssembler::emit_i16x8_gt_u);
3568 case wasm::kExprI16x8GtS:
3569 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i16x8_gt_s);
3570 case wasm::kExprI16x8GtU:
3571 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i16x8_gt_u);
3572 case wasm::kExprI16x8LeS:
3573 return EmitBinOp<kS128, kS128, true>(
3574 &LiftoffAssembler::emit_i16x8_ge_s);
3575 case wasm::kExprI16x8LeU:
3576 return EmitBinOp<kS128, kS128, true>(
3577 &LiftoffAssembler::emit_i16x8_ge_u);
3578 case wasm::kExprI16x8GeS:
3579 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i16x8_ge_s);
3580 case wasm::kExprI16x8GeU:
3581 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i16x8_ge_u);
3582 case wasm::kExprI32x4Eq:
3583 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i32x4_eq);
3584 case wasm::kExprI32x4Ne:
3585 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i32x4_ne);
3586 case wasm::kExprI32x4LtS:
3587 return EmitBinOp<kS128, kS128, true>(
3588 &LiftoffAssembler::emit_i32x4_gt_s);
3589 case wasm::kExprI32x4LtU:
3590 return EmitBinOp<kS128, kS128, true>(
3591 &LiftoffAssembler::emit_i32x4_gt_u);
3592 case wasm::kExprI32x4GtS:
3593 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i32x4_gt_s);
3594 case wasm::kExprI32x4GtU:
3595 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i32x4_gt_u);
3596 case wasm::kExprI32x4LeS:
3597 return EmitBinOp<kS128, kS128, true>(
3598 &LiftoffAssembler::emit_i32x4_ge_s);
3599 case wasm::kExprI32x4LeU:
3600 return EmitBinOp<kS128, kS128, true>(
3601 &LiftoffAssembler::emit_i32x4_ge_u);
3602 case wasm::kExprI32x4GeS:
3603 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i32x4_ge_s);
3604 case wasm::kExprI32x4GeU:
3605 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i32x4_ge_u);
3606 case wasm::kExprI64x2Eq:
3607 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i64x2_eq);
3608 case wasm::kExprI64x2Ne:
3609 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i64x2_ne);
3610 case wasm::kExprI64x2LtS:
3611 return EmitBinOp<kS128, kS128, true>(
3612 &LiftoffAssembler::emit_i64x2_gt_s);
3613 case wasm::kExprI64x2GtS:
3614 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i64x2_gt_s);
3615 case wasm::kExprI64x2LeS:
3616 return EmitBinOp<kS128, kS128, true>(
3617 &LiftoffAssembler::emit_i64x2_ge_s);
3618 case wasm::kExprI64x2GeS:
3619 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i64x2_ge_s);
3620 case wasm::kExprF32x4Eq:
3621 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_f32x4_eq);
3622 case wasm::kExprF32x4Ne:
3623 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_f32x4_ne);
3624 case wasm::kExprF32x4Lt:
3625 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_f32x4_lt);
3626 case wasm::kExprF32x4Gt:
3627 return EmitBinOp<kS128, kS128, true>(&LiftoffAssembler::emit_f32x4_lt);
3628 case wasm::kExprF32x4Le:
3629 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_f32x4_le);
3630 case wasm::kExprF32x4Ge:
3631 return EmitBinOp<kS128, kS128, true>(&LiftoffAssembler::emit_f32x4_le);
3632 case wasm::kExprF64x2Eq:
3633 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_f64x2_eq);
3634 case wasm::kExprF64x2Ne:
3635 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_f64x2_ne);
3636 case wasm::kExprF64x2Lt:
3637 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_f64x2_lt);
3638 case wasm::kExprF64x2Gt:
3639 return EmitBinOp<kS128, kS128, true>(&LiftoffAssembler::emit_f64x2_lt);
3640 case wasm::kExprF64x2Le:
3641 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_f64x2_le);
3642 case wasm::kExprF64x2Ge:
3643 return EmitBinOp<kS128, kS128, true>(&LiftoffAssembler::emit_f64x2_le);
3644 case wasm::kExprS128Not:
3645 return EmitUnOp<kS128, kS128>(&LiftoffAssembler::emit_s128_not);
3646 case wasm::kExprS128And:
3647 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_s128_and);
3648 case wasm::kExprS128Or:
3649 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_s128_or);
3650 case wasm::kExprS128Xor:
3651 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_s128_xor);
3652 case wasm::kExprS128Select:
3653 return EmitTerOp<kS128, kS128>(&LiftoffAssembler::emit_s128_select);
3654 case wasm::kExprI8x16Neg:
3655 return EmitUnOp<kS128, kS128>(&LiftoffAssembler::emit_i8x16_neg);
3656 case wasm::kExprV128AnyTrue:
3657 return EmitUnOp<kS128, kI32>(&LiftoffAssembler::emit_v128_anytrue);
3658 case wasm::kExprI8x16AllTrue:
3659 return EmitUnOp<kS128, kI32>(&LiftoffAssembler::emit_i8x16_alltrue);
3660 case wasm::kExprI8x16BitMask:
3661 return EmitUnOp<kS128, kI32>(&LiftoffAssembler::emit_i8x16_bitmask);
3662 case wasm::kExprI8x16Shl:
3663 return EmitSimdShiftOp(&LiftoffAssembler::emit_i8x16_shl,
3664 &LiftoffAssembler::emit_i8x16_shli);
3665 case wasm::kExprI8x16ShrS:
3666 return EmitSimdShiftOp(&LiftoffAssembler::emit_i8x16_shr_s,
3667 &LiftoffAssembler::emit_i8x16_shri_s);
3668 case wasm::kExprI8x16ShrU:
3669 return EmitSimdShiftOp(&LiftoffAssembler::emit_i8x16_shr_u,
3670 &LiftoffAssembler::emit_i8x16_shri_u);
3671 case wasm::kExprI8x16Add:
3672 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i8x16_add);
3673 case wasm::kExprI8x16AddSatS:
3674 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i8x16_add_sat_s);
3675 case wasm::kExprI8x16AddSatU:
3676 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i8x16_add_sat_u);
3677 case wasm::kExprI8x16Sub:
3678 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i8x16_sub);
3679 case wasm::kExprI8x16SubSatS:
3680 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i8x16_sub_sat_s);
3681 case wasm::kExprI8x16SubSatU:
3682 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i8x16_sub_sat_u);
3683 case wasm::kExprI8x16MinS:
3684 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i8x16_min_s);
3685 case wasm::kExprI8x16MinU:
3686 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i8x16_min_u);
3687 case wasm::kExprI8x16MaxS:
3688 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i8x16_max_s);
3689 case wasm::kExprI8x16MaxU:
3690 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i8x16_max_u);
3691 case wasm::kExprI16x8Neg:
3692 return EmitUnOp<kS128, kS128>(&LiftoffAssembler::emit_i16x8_neg);
3693 case wasm::kExprI16x8AllTrue:
3694 return EmitUnOp<kS128, kI32>(&LiftoffAssembler::emit_i16x8_alltrue);
3695 case wasm::kExprI16x8BitMask:
3696 return EmitUnOp<kS128, kI32>(&LiftoffAssembler::emit_i16x8_bitmask);
3697 case wasm::kExprI16x8Shl:
3698 return EmitSimdShiftOp(&LiftoffAssembler::emit_i16x8_shl,
3699 &LiftoffAssembler::emit_i16x8_shli);
3700 case wasm::kExprI16x8ShrS:
3701 return EmitSimdShiftOp(&LiftoffAssembler::emit_i16x8_shr_s,
3702 &LiftoffAssembler::emit_i16x8_shri_s);
3703 case wasm::kExprI16x8ShrU:
3704 return EmitSimdShiftOp(&LiftoffAssembler::emit_i16x8_shr_u,
3705 &LiftoffAssembler::emit_i16x8_shri_u);
3706 case wasm::kExprI16x8Add:
3707 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i16x8_add);
3708 case wasm::kExprI16x8AddSatS:
3709 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i16x8_add_sat_s);
3710 case wasm::kExprI16x8AddSatU:
3711 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i16x8_add_sat_u);
3712 case wasm::kExprI16x8Sub:
3713 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i16x8_sub);
3714 case wasm::kExprI16x8SubSatS:
3715 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i16x8_sub_sat_s);
3716 case wasm::kExprI16x8SubSatU:
3717 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i16x8_sub_sat_u);
3718 case wasm::kExprI16x8Mul:
3719 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i16x8_mul);
3720 case wasm::kExprI16x8MinS:
3721 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i16x8_min_s);
3722 case wasm::kExprI16x8MinU:
3723 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i16x8_min_u);
3724 case wasm::kExprI16x8MaxS:
3725 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i16x8_max_s);
3726 case wasm::kExprI16x8MaxU:
3727 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i16x8_max_u);
3728 case wasm::kExprI16x8ExtAddPairwiseI8x16S:
3729 return EmitUnOp<kS128, kS128>(
3730 &LiftoffAssembler::emit_i16x8_extadd_pairwise_i8x16_s);
3731 case wasm::kExprI16x8ExtAddPairwiseI8x16U:
3732 return EmitUnOp<kS128, kS128>(
3733 &LiftoffAssembler::emit_i16x8_extadd_pairwise_i8x16_u);
3734 case wasm::kExprI16x8ExtMulLowI8x16S:
3735 return EmitBinOp<kS128, kS128>(
3736 &LiftoffAssembler::emit_i16x8_extmul_low_i8x16_s);
3737 case wasm::kExprI16x8ExtMulLowI8x16U:
3738 return EmitBinOp<kS128, kS128>(
3739 &LiftoffAssembler::emit_i16x8_extmul_low_i8x16_u);
3740 case wasm::kExprI16x8ExtMulHighI8x16S:
3741 return EmitBinOp<kS128, kS128>(
3742 &LiftoffAssembler::emit_i16x8_extmul_high_i8x16_s);
3743 case wasm::kExprI16x8ExtMulHighI8x16U:
3744 return EmitBinOp<kS128, kS128>(
3745 &LiftoffAssembler::emit_i16x8_extmul_high_i8x16_u);
3746 case wasm::kExprI16x8Q15MulRSatS:
3747 return EmitBinOp<kS128, kS128>(
3748 &LiftoffAssembler::emit_i16x8_q15mulr_sat_s);
3749 case wasm::kExprI32x4Neg:
3750 return EmitUnOp<kS128, kS128>(&LiftoffAssembler::emit_i32x4_neg);
3751 case wasm::kExprI32x4AllTrue:
3752 return EmitUnOp<kS128, kI32>(&LiftoffAssembler::emit_i32x4_alltrue);
3753 case wasm::kExprI32x4BitMask:
3754 return EmitUnOp<kS128, kI32>(&LiftoffAssembler::emit_i32x4_bitmask);
3755 case wasm::kExprI32x4Shl:
3756 return EmitSimdShiftOp(&LiftoffAssembler::emit_i32x4_shl,
3757 &LiftoffAssembler::emit_i32x4_shli);
3758 case wasm::kExprI32x4ShrS:
3759 return EmitSimdShiftOp(&LiftoffAssembler::emit_i32x4_shr_s,
3760 &LiftoffAssembler::emit_i32x4_shri_s);
3761 case wasm::kExprI32x4ShrU:
3762 return EmitSimdShiftOp(&LiftoffAssembler::emit_i32x4_shr_u,
3763 &LiftoffAssembler::emit_i32x4_shri_u);
3764 case wasm::kExprI32x4Add:
3765 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i32x4_add);
3766 case wasm::kExprI32x4Sub:
3767 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i32x4_sub);
3768 case wasm::kExprI32x4Mul:
3769 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i32x4_mul);
3770 case wasm::kExprI32x4MinS:
3771 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i32x4_min_s);
3772 case wasm::kExprI32x4MinU:
3773 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i32x4_min_u);
3774 case wasm::kExprI32x4MaxS:
3775 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i32x4_max_s);
3776 case wasm::kExprI32x4MaxU:
3777 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i32x4_max_u);
3778 case wasm::kExprI32x4DotI16x8S:
3779 return EmitBinOp<kS128, kS128>(
3780 &LiftoffAssembler::emit_i32x4_dot_i16x8_s);
3781 case wasm::kExprI32x4ExtAddPairwiseI16x8S:
3782 return EmitUnOp<kS128, kS128>(
3783 &LiftoffAssembler::emit_i32x4_extadd_pairwise_i16x8_s);
3784 case wasm::kExprI32x4ExtAddPairwiseI16x8U:
3785 return EmitUnOp<kS128, kS128>(
3786 &LiftoffAssembler::emit_i32x4_extadd_pairwise_i16x8_u);
3787 case wasm::kExprI32x4ExtMulLowI16x8S:
3788 return EmitBinOp<kS128, kS128>(
3789 &LiftoffAssembler::emit_i32x4_extmul_low_i16x8_s);
3790 case wasm::kExprI32x4ExtMulLowI16x8U:
3791 return EmitBinOp<kS128, kS128>(
3792 &LiftoffAssembler::emit_i32x4_extmul_low_i16x8_u);
3793 case wasm::kExprI32x4ExtMulHighI16x8S:
3794 return EmitBinOp<kS128, kS128>(
3795 &LiftoffAssembler::emit_i32x4_extmul_high_i16x8_s);
3796 case wasm::kExprI32x4ExtMulHighI16x8U:
3797 return EmitBinOp<kS128, kS128>(
3798 &LiftoffAssembler::emit_i32x4_extmul_high_i16x8_u);
3799 case wasm::kExprI64x2Neg:
3800 return EmitUnOp<kS128, kS128>(&LiftoffAssembler::emit_i64x2_neg);
3801 case wasm::kExprI64x2AllTrue:
3802 return EmitUnOp<kS128, kI32>(&LiftoffAssembler::emit_i64x2_alltrue);
3803 case wasm::kExprI64x2Shl:
3804 return EmitSimdShiftOp(&LiftoffAssembler::emit_i64x2_shl,
3805 &LiftoffAssembler::emit_i64x2_shli);
3806 case wasm::kExprI64x2ShrS:
3807 return EmitSimdShiftOp(&LiftoffAssembler::emit_i64x2_shr_s,
3808 &LiftoffAssembler::emit_i64x2_shri_s);
3809 case wasm::kExprI64x2ShrU:
3810 return EmitSimdShiftOp(&LiftoffAssembler::emit_i64x2_shr_u,
3811 &LiftoffAssembler::emit_i64x2_shri_u);
3812 case wasm::kExprI64x2Add:
3813 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i64x2_add);
3814 case wasm::kExprI64x2Sub:
3815 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i64x2_sub);
3816 case wasm::kExprI64x2Mul:
3817 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i64x2_mul);
3818 case wasm::kExprI64x2ExtMulLowI32x4S:
3819 return EmitBinOp<kS128, kS128>(
3820 &LiftoffAssembler::emit_i64x2_extmul_low_i32x4_s);
3821 case wasm::kExprI64x2ExtMulLowI32x4U:
3822 return EmitBinOp<kS128, kS128>(
3823 &LiftoffAssembler::emit_i64x2_extmul_low_i32x4_u);
3824 case wasm::kExprI64x2ExtMulHighI32x4S:
3825 return EmitBinOp<kS128, kS128>(
3826 &LiftoffAssembler::emit_i64x2_extmul_high_i32x4_s);
3827 case wasm::kExprI64x2ExtMulHighI32x4U:
3828 return EmitBinOp<kS128, kS128>(
3829 &LiftoffAssembler::emit_i64x2_extmul_high_i32x4_u);
3830 case wasm::kExprI64x2BitMask:
3831 return EmitUnOp<kS128, kI32>(&LiftoffAssembler::emit_i64x2_bitmask);
3832 case wasm::kExprI64x2SConvertI32x4Low:
3833 return EmitUnOp<kS128, kS128>(
3834 &LiftoffAssembler::emit_i64x2_sconvert_i32x4_low);
3835 case wasm::kExprI64x2SConvertI32x4High:
3836 return EmitUnOp<kS128, kS128>(
3837 &LiftoffAssembler::emit_i64x2_sconvert_i32x4_high);
3838 case wasm::kExprI64x2UConvertI32x4Low:
3839 return EmitUnOp<kS128, kS128>(
3840 &LiftoffAssembler::emit_i64x2_uconvert_i32x4_low);
3841 case wasm::kExprI64x2UConvertI32x4High:
3842 return EmitUnOp<kS128, kS128>(
3843 &LiftoffAssembler::emit_i64x2_uconvert_i32x4_high);
3844 case wasm::kExprF32x4Abs:
3845 return EmitUnOp<kS128, kS128, kF32>(&LiftoffAssembler::emit_f32x4_abs);
3846 case wasm::kExprF32x4Neg:
3847 return EmitUnOp<kS128, kS128, kF32>(&LiftoffAssembler::emit_f32x4_neg);
3848 case wasm::kExprF32x4Sqrt:
3849 return EmitUnOp<kS128, kS128, kF32>(&LiftoffAssembler::emit_f32x4_sqrt);
3850 case wasm::kExprF32x4Ceil:
3851 return EmitSimdFloatRoundingOpWithCFallback<kF32>(
3852 &LiftoffAssembler::emit_f32x4_ceil,
3853 &ExternalReference::wasm_f32x4_ceil);
3854 case wasm::kExprF32x4Floor:
3855 return EmitSimdFloatRoundingOpWithCFallback<kF32>(
3856 &LiftoffAssembler::emit_f32x4_floor,
3857 ExternalReference::wasm_f32x4_floor);
3858 case wasm::kExprF32x4Trunc:
3859 return EmitSimdFloatRoundingOpWithCFallback<kF32>(
3860 &LiftoffAssembler::emit_f32x4_trunc,
3861 ExternalReference::wasm_f32x4_trunc);
3862 case wasm::kExprF32x4NearestInt:
3863 return EmitSimdFloatRoundingOpWithCFallback<kF32>(
3864 &LiftoffAssembler::emit_f32x4_nearest_int,
3865 ExternalReference::wasm_f32x4_nearest_int);
3866 case wasm::kExprF32x4Add:
3867 return EmitBinOp<kS128, kS128, false, kF32>(
3868 &LiftoffAssembler::emit_f32x4_add);
3869 case wasm::kExprF32x4Sub:
3870 return EmitBinOp<kS128, kS128, false, kF32>(
3871 &LiftoffAssembler::emit_f32x4_sub);
3872 case wasm::kExprF32x4Mul:
3873 return EmitBinOp<kS128, kS128, false, kF32>(
3874 &LiftoffAssembler::emit_f32x4_mul);
3875 case wasm::kExprF32x4Div:
3876 return EmitBinOp<kS128, kS128, false, kF32>(
3877 &LiftoffAssembler::emit_f32x4_div);
3878 case wasm::kExprF32x4Min:
3879 return EmitBinOp<kS128, kS128, false, kF32>(
3880 &LiftoffAssembler::emit_f32x4_min);
3881 case wasm::kExprF32x4Max:
3882 return EmitBinOp<kS128, kS128, false, kF32>(
3883 &LiftoffAssembler::emit_f32x4_max);
3884 case wasm::kExprF32x4Pmin:
3885 return EmitBinOp<kS128, kS128, false, kF32>(
3886 &LiftoffAssembler::emit_f32x4_pmin);
3887 case wasm::kExprF32x4Pmax:
3888 return EmitBinOp<kS128, kS128, false, kF32>(
3889 &LiftoffAssembler::emit_f32x4_pmax);
3890 case wasm::kExprF64x2Abs:
3891 return EmitUnOp<kS128, kS128, kF64>(&LiftoffAssembler::emit_f64x2_abs);
3892 case wasm::kExprF64x2Neg:
3893 return EmitUnOp<kS128, kS128, kF64>(&LiftoffAssembler::emit_f64x2_neg);
3894 case wasm::kExprF64x2Sqrt:
3895 return EmitUnOp<kS128, kS128, kF64>(&LiftoffAssembler::emit_f64x2_sqrt);
3896 case wasm::kExprF64x2Ceil:
3897 return EmitSimdFloatRoundingOpWithCFallback<kF64>(
3898 &LiftoffAssembler::emit_f64x2_ceil,
3899 &ExternalReference::wasm_f64x2_ceil);
3900 case wasm::kExprF64x2Floor:
3901 return EmitSimdFloatRoundingOpWithCFallback<kF64>(
3902 &LiftoffAssembler::emit_f64x2_floor,
3903 ExternalReference::wasm_f64x2_floor);
3904 case wasm::kExprF64x2Trunc:
3905 return EmitSimdFloatRoundingOpWithCFallback<kF64>(
3906 &LiftoffAssembler::emit_f64x2_trunc,
3907 ExternalReference::wasm_f64x2_trunc);
3908 case wasm::kExprF64x2NearestInt:
3909 return EmitSimdFloatRoundingOpWithCFallback<kF64>(
3910 &LiftoffAssembler::emit_f64x2_nearest_int,
3911 ExternalReference::wasm_f64x2_nearest_int);
3912 case wasm::kExprF64x2Add:
3913 return EmitBinOp<kS128, kS128, false, kF64>(
3914 &LiftoffAssembler::emit_f64x2_add);
3915 case wasm::kExprF64x2Sub:
3916 return EmitBinOp<kS128, kS128, false, kF64>(
3917 &LiftoffAssembler::emit_f64x2_sub);
3918 case wasm::kExprF64x2Mul:
3919 return EmitBinOp<kS128, kS128, false, kF64>(
3920 &LiftoffAssembler::emit_f64x2_mul);
3921 case wasm::kExprF64x2Div:
3922 return EmitBinOp<kS128, kS128, false, kF64>(
3923 &LiftoffAssembler::emit_f64x2_div);
3924 case wasm::kExprF64x2Min:
3925 return EmitBinOp<kS128, kS128, false, kF64>(
3926 &LiftoffAssembler::emit_f64x2_min);
3927 case wasm::kExprF64x2Max:
3928 return EmitBinOp<kS128, kS128, false, kF64>(
3929 &LiftoffAssembler::emit_f64x2_max);
3930 case wasm::kExprF64x2Pmin:
3931 return EmitBinOp<kS128, kS128, false, kF64>(
3932 &LiftoffAssembler::emit_f64x2_pmin);
3933 case wasm::kExprF64x2Pmax:
3934 return EmitBinOp<kS128, kS128, false, kF64>(
3935 &LiftoffAssembler::emit_f64x2_pmax);
3936 case wasm::kExprI32x4SConvertF32x4:
3937 return EmitUnOp<kS128, kS128, kF32>(
3938 &LiftoffAssembler::emit_i32x4_sconvert_f32x4);
3939 case wasm::kExprI32x4UConvertF32x4:
3940 return EmitUnOp<kS128, kS128, kF32>(
3941 &LiftoffAssembler::emit_i32x4_uconvert_f32x4);
3942 case wasm::kExprF32x4SConvertI32x4:
3943 return EmitUnOp<kS128, kS128, kF32>(
3944 &LiftoffAssembler::emit_f32x4_sconvert_i32x4);
3945 case wasm::kExprF32x4UConvertI32x4:
3946 return EmitUnOp<kS128, kS128, kF32>(
3947 &LiftoffAssembler::emit_f32x4_uconvert_i32x4);
3948 case wasm::kExprI8x16SConvertI16x8:
3949 return EmitBinOp<kS128, kS128>(
3950 &LiftoffAssembler::emit_i8x16_sconvert_i16x8);
3951 case wasm::kExprI8x16UConvertI16x8:
3952 return EmitBinOp<kS128, kS128>(
3953 &LiftoffAssembler::emit_i8x16_uconvert_i16x8);
3954 case wasm::kExprI16x8SConvertI32x4:
3955 return EmitBinOp<kS128, kS128>(
3956 &LiftoffAssembler::emit_i16x8_sconvert_i32x4);
3957 case wasm::kExprI16x8UConvertI32x4:
3958 return EmitBinOp<kS128, kS128>(
3959 &LiftoffAssembler::emit_i16x8_uconvert_i32x4);
3960 case wasm::kExprI16x8SConvertI8x16Low:
3961 return EmitUnOp<kS128, kS128>(
3962 &LiftoffAssembler::emit_i16x8_sconvert_i8x16_low);
3963 case wasm::kExprI16x8SConvertI8x16High:
3964 return EmitUnOp<kS128, kS128>(
3965 &LiftoffAssembler::emit_i16x8_sconvert_i8x16_high);
3966 case wasm::kExprI16x8UConvertI8x16Low:
3967 return EmitUnOp<kS128, kS128>(
3968 &LiftoffAssembler::emit_i16x8_uconvert_i8x16_low);
3969 case wasm::kExprI16x8UConvertI8x16High:
3970 return EmitUnOp<kS128, kS128>(
3971 &LiftoffAssembler::emit_i16x8_uconvert_i8x16_high);
3972 case wasm::kExprI32x4SConvertI16x8Low:
3973 return EmitUnOp<kS128, kS128>(
3974 &LiftoffAssembler::emit_i32x4_sconvert_i16x8_low);
3975 case wasm::kExprI32x4SConvertI16x8High:
3976 return EmitUnOp<kS128, kS128>(
3977 &LiftoffAssembler::emit_i32x4_sconvert_i16x8_high);
3978 case wasm::kExprI32x4UConvertI16x8Low:
3979 return EmitUnOp<kS128, kS128>(
3980 &LiftoffAssembler::emit_i32x4_uconvert_i16x8_low);
3981 case wasm::kExprI32x4UConvertI16x8High:
3982 return EmitUnOp<kS128, kS128>(
3983 &LiftoffAssembler::emit_i32x4_uconvert_i16x8_high);
3984 case wasm::kExprS128AndNot:
3985 return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_s128_and_not);
3986 case wasm::kExprI8x16RoundingAverageU:
3987 return EmitBinOp<kS128, kS128>(
3988 &LiftoffAssembler::emit_i8x16_rounding_average_u);
3989 case wasm::kExprI16x8RoundingAverageU:
3990 return EmitBinOp<kS128, kS128>(
3991 &LiftoffAssembler::emit_i16x8_rounding_average_u);
3992 case wasm::kExprI8x16Abs:
3993 return EmitUnOp<kS128, kS128>(&LiftoffAssembler::emit_i8x16_abs);
3994 case wasm::kExprI16x8Abs:
3995 return EmitUnOp<kS128, kS128>(&LiftoffAssembler::emit_i16x8_abs);
3996 case wasm::kExprI32x4Abs:
3997 return EmitUnOp<kS128, kS128>(&LiftoffAssembler::emit_i32x4_abs);
3998 case wasm::kExprI64x2Abs:
3999 return EmitUnOp<kS128, kS128>(&LiftoffAssembler::emit_i64x2_abs);
4000 case wasm::kExprF64x2ConvertLowI32x4S:
4001 return EmitUnOp<kS128, kS128, kF64>(
4002 &LiftoffAssembler::emit_f64x2_convert_low_i32x4_s);
4003 case wasm::kExprF64x2ConvertLowI32x4U:
4004 return EmitUnOp<kS128, kS128, kF64>(
4005 &LiftoffAssembler::emit_f64x2_convert_low_i32x4_u);
4006 case wasm::kExprF64x2PromoteLowF32x4:
4007 return EmitUnOp<kS128, kS128, kF64>(
4008 &LiftoffAssembler::emit_f64x2_promote_low_f32x4);
4009 case wasm::kExprF32x4DemoteF64x2Zero:
4010 return EmitUnOp<kS128, kS128, kF32>(
4011 &LiftoffAssembler::emit_f32x4_demote_f64x2_zero);
4012 case wasm::kExprI32x4TruncSatF64x2SZero:
4013 return EmitUnOp<kS128, kS128>(
4014 &LiftoffAssembler::emit_i32x4_trunc_sat_f64x2_s_zero);
4015 case wasm::kExprI32x4TruncSatF64x2UZero:
4016 return EmitUnOp<kS128, kS128>(
4017 &LiftoffAssembler::emit_i32x4_trunc_sat_f64x2_u_zero);
4018 default:
4019 unsupported(decoder, kSimd, "simd");
4020 }
4021 }
4022
4023 template <ValueKind src_kind, ValueKind result_kind, typename EmitFn>
EmitSimdExtractLaneOp(EmitFn fn,const SimdLaneImmediate<validate> & imm)4024 void EmitSimdExtractLaneOp(EmitFn fn,
4025 const SimdLaneImmediate<validate>& imm) {
4026 static constexpr RegClass src_rc = reg_class_for(src_kind);
4027 static constexpr RegClass result_rc = reg_class_for(result_kind);
4028 LiftoffRegister lhs = __ PopToRegister();
4029 LiftoffRegister dst = src_rc == result_rc
4030 ? __ GetUnusedRegister(result_rc, {lhs}, {})
4031 : __ GetUnusedRegister(result_rc, {});
4032 fn(dst, lhs, imm.lane);
4033 __ PushRegister(result_kind, dst);
4034 }
4035
4036 template <ValueKind src2_kind, typename EmitFn>
EmitSimdReplaceLaneOp(EmitFn fn,const SimdLaneImmediate<validate> & imm)4037 void EmitSimdReplaceLaneOp(EmitFn fn,
4038 const SimdLaneImmediate<validate>& imm) {
4039 static constexpr RegClass src1_rc = reg_class_for(kS128);
4040 static constexpr RegClass src2_rc = reg_class_for(src2_kind);
4041 static constexpr RegClass result_rc = reg_class_for(kS128);
4042 // On backends which need fp pair, src1_rc and result_rc end up being
4043 // kFpRegPair, which is != kFpReg, but we still want to pin src2 when it is
4044 // kFpReg, since it can overlap with those pairs.
4045 static constexpr bool pin_src2 = kNeedS128RegPair && src2_rc == kFpReg;
4046
4047 // Does not work for arm
4048 LiftoffRegister src2 = __ PopToRegister();
4049 LiftoffRegister src1 = (src1_rc == src2_rc || pin_src2)
4050 ? __ PopToRegister(LiftoffRegList{src2})
4051 : __
4052 PopToRegister();
4053 LiftoffRegister dst =
4054 (src2_rc == result_rc || pin_src2)
4055 ? __ GetUnusedRegister(result_rc, {src1}, LiftoffRegList{src2})
4056 : __ GetUnusedRegister(result_rc, {src1}, {});
4057 fn(dst, src1, src2, imm.lane);
4058 __ PushRegister(kS128, dst);
4059 }
4060
SimdLaneOp(FullDecoder * decoder,WasmOpcode opcode,const SimdLaneImmediate<validate> & imm,const base::Vector<Value> inputs,Value * result)4061 void SimdLaneOp(FullDecoder* decoder, WasmOpcode opcode,
4062 const SimdLaneImmediate<validate>& imm,
4063 const base::Vector<Value> inputs, Value* result) {
4064 if (!CpuFeatures::SupportsWasmSimd128()) {
4065 return unsupported(decoder, kSimd, "simd");
4066 }
4067 switch (opcode) {
4068 #define CASE_SIMD_EXTRACT_LANE_OP(opcode, kind, fn) \
4069 case wasm::kExpr##opcode: \
4070 EmitSimdExtractLaneOp<kS128, k##kind>( \
4071 [=](LiftoffRegister dst, LiftoffRegister lhs, uint8_t imm_lane_idx) { \
4072 __ emit_##fn(dst, lhs, imm_lane_idx); \
4073 }, \
4074 imm); \
4075 break;
4076 CASE_SIMD_EXTRACT_LANE_OP(I8x16ExtractLaneS, I32, i8x16_extract_lane_s)
4077 CASE_SIMD_EXTRACT_LANE_OP(I8x16ExtractLaneU, I32, i8x16_extract_lane_u)
4078 CASE_SIMD_EXTRACT_LANE_OP(I16x8ExtractLaneS, I32, i16x8_extract_lane_s)
4079 CASE_SIMD_EXTRACT_LANE_OP(I16x8ExtractLaneU, I32, i16x8_extract_lane_u)
4080 CASE_SIMD_EXTRACT_LANE_OP(I32x4ExtractLane, I32, i32x4_extract_lane)
4081 CASE_SIMD_EXTRACT_LANE_OP(I64x2ExtractLane, I64, i64x2_extract_lane)
4082 CASE_SIMD_EXTRACT_LANE_OP(F32x4ExtractLane, F32, f32x4_extract_lane)
4083 CASE_SIMD_EXTRACT_LANE_OP(F64x2ExtractLane, F64, f64x2_extract_lane)
4084 #undef CASE_SIMD_EXTRACT_LANE_OP
4085 #define CASE_SIMD_REPLACE_LANE_OP(opcode, kind, fn) \
4086 case wasm::kExpr##opcode: \
4087 EmitSimdReplaceLaneOp<k##kind>( \
4088 [=](LiftoffRegister dst, LiftoffRegister src1, LiftoffRegister src2, \
4089 uint8_t imm_lane_idx) { \
4090 __ emit_##fn(dst, src1, src2, imm_lane_idx); \
4091 }, \
4092 imm); \
4093 break;
4094 CASE_SIMD_REPLACE_LANE_OP(I8x16ReplaceLane, I32, i8x16_replace_lane)
4095 CASE_SIMD_REPLACE_LANE_OP(I16x8ReplaceLane, I32, i16x8_replace_lane)
4096 CASE_SIMD_REPLACE_LANE_OP(I32x4ReplaceLane, I32, i32x4_replace_lane)
4097 CASE_SIMD_REPLACE_LANE_OP(I64x2ReplaceLane, I64, i64x2_replace_lane)
4098 CASE_SIMD_REPLACE_LANE_OP(F32x4ReplaceLane, F32, f32x4_replace_lane)
4099 CASE_SIMD_REPLACE_LANE_OP(F64x2ReplaceLane, F64, f64x2_replace_lane)
4100 #undef CASE_SIMD_REPLACE_LANE_OP
4101 default:
4102 unsupported(decoder, kSimd, "simd");
4103 }
4104 }
4105
S128Const(FullDecoder * decoder,const Simd128Immediate<validate> & imm,Value * result)4106 void S128Const(FullDecoder* decoder, const Simd128Immediate<validate>& imm,
4107 Value* result) {
4108 if (!CpuFeatures::SupportsWasmSimd128()) {
4109 return unsupported(decoder, kSimd, "simd");
4110 }
4111 constexpr RegClass result_rc = reg_class_for(kS128);
4112 LiftoffRegister dst = __ GetUnusedRegister(result_rc, {});
4113 bool all_zeroes = std::all_of(std::begin(imm.value), std::end(imm.value),
4114 [](uint8_t v) { return v == 0; });
4115 bool all_ones = std::all_of(std::begin(imm.value), std::end(imm.value),
4116 [](uint8_t v) { return v == 0xff; });
4117 if (all_zeroes) {
4118 __ LiftoffAssembler::emit_s128_xor(dst, dst, dst);
4119 } else if (all_ones) {
4120 // Any SIMD eq will work, i32x4 is efficient on all archs.
4121 __ LiftoffAssembler::emit_i32x4_eq(dst, dst, dst);
4122 } else {
4123 __ LiftoffAssembler::emit_s128_const(dst, imm.value);
4124 }
4125 __ PushRegister(kS128, dst);
4126 }
4127
Simd8x16ShuffleOp(FullDecoder * decoder,const Simd128Immediate<validate> & imm,const Value & input0,const Value & input1,Value * result)4128 void Simd8x16ShuffleOp(FullDecoder* decoder,
4129 const Simd128Immediate<validate>& imm,
4130 const Value& input0, const Value& input1,
4131 Value* result) {
4132 if (!CpuFeatures::SupportsWasmSimd128()) {
4133 return unsupported(decoder, kSimd, "simd");
4134 }
4135 static constexpr RegClass result_rc = reg_class_for(kS128);
4136 LiftoffRegister rhs = __ PopToRegister();
4137 LiftoffRegister lhs = __ PopToRegister(LiftoffRegList{rhs});
4138 LiftoffRegister dst = __ GetUnusedRegister(result_rc, {lhs, rhs}, {});
4139
4140 uint8_t shuffle[kSimd128Size];
4141 memcpy(shuffle, imm.value, sizeof(shuffle));
4142 bool is_swizzle;
4143 bool needs_swap;
4144 wasm::SimdShuffle::CanonicalizeShuffle(lhs == rhs, shuffle, &needs_swap,
4145 &is_swizzle);
4146 if (needs_swap) {
4147 std::swap(lhs, rhs);
4148 }
4149 __ LiftoffAssembler::emit_i8x16_shuffle(dst, lhs, rhs, shuffle, is_swizzle);
4150 __ PushRegister(kS128, dst);
4151 }
4152
ToSmi(Register reg)4153 void ToSmi(Register reg) {
4154 if (COMPRESS_POINTERS_BOOL || kSystemPointerSize == 4) {
4155 __ emit_i32_shli(reg, reg, kSmiShiftSize + kSmiTagSize);
4156 } else {
4157 __ emit_i64_shli(LiftoffRegister{reg}, LiftoffRegister{reg},
4158 kSmiShiftSize + kSmiTagSize);
4159 }
4160 }
4161
Store32BitExceptionValue(Register values_array,int * index_in_array,Register value,LiftoffRegList pinned)4162 void Store32BitExceptionValue(Register values_array, int* index_in_array,
4163 Register value, LiftoffRegList pinned) {
4164 LiftoffRegister tmp_reg = __ GetUnusedRegister(kGpReg, pinned);
4165 // Get the lower half word into tmp_reg and extend to a Smi.
4166 --*index_in_array;
4167 __ emit_i32_andi(tmp_reg.gp(), value, 0xffff);
4168 ToSmi(tmp_reg.gp());
4169 __ StoreTaggedPointer(
4170 values_array, no_reg,
4171 wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(*index_in_array),
4172 tmp_reg, pinned, LiftoffAssembler::kSkipWriteBarrier);
4173
4174 // Get the upper half word into tmp_reg and extend to a Smi.
4175 --*index_in_array;
4176 __ emit_i32_shri(tmp_reg.gp(), value, 16);
4177 ToSmi(tmp_reg.gp());
4178 __ StoreTaggedPointer(
4179 values_array, no_reg,
4180 wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(*index_in_array),
4181 tmp_reg, pinned, LiftoffAssembler::kSkipWriteBarrier);
4182 }
4183
Store64BitExceptionValue(Register values_array,int * index_in_array,LiftoffRegister value,LiftoffRegList pinned)4184 void Store64BitExceptionValue(Register values_array, int* index_in_array,
4185 LiftoffRegister value, LiftoffRegList pinned) {
4186 if (kNeedI64RegPair) {
4187 Store32BitExceptionValue(values_array, index_in_array, value.low_gp(),
4188 pinned);
4189 Store32BitExceptionValue(values_array, index_in_array, value.high_gp(),
4190 pinned);
4191 } else {
4192 Store32BitExceptionValue(values_array, index_in_array, value.gp(),
4193 pinned);
4194 __ emit_i64_shri(value, value, 32);
4195 Store32BitExceptionValue(values_array, index_in_array, value.gp(),
4196 pinned);
4197 }
4198 }
4199
Load16BitExceptionValue(LiftoffRegister dst,LiftoffRegister values_array,uint32_t * index,LiftoffRegList pinned)4200 void Load16BitExceptionValue(LiftoffRegister dst,
4201 LiftoffRegister values_array, uint32_t* index,
4202 LiftoffRegList pinned) {
4203 __ LoadSmiAsInt32(
4204 dst, values_array.gp(),
4205 wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(*index), pinned);
4206 (*index)++;
4207 }
4208
Load32BitExceptionValue(Register dst,LiftoffRegister values_array,uint32_t * index,LiftoffRegList pinned)4209 void Load32BitExceptionValue(Register dst, LiftoffRegister values_array,
4210 uint32_t* index, LiftoffRegList pinned) {
4211 LiftoffRegister upper = pinned.set(__ GetUnusedRegister(kGpReg, pinned));
4212 Load16BitExceptionValue(upper, values_array, index, pinned);
4213 __ emit_i32_shli(upper.gp(), upper.gp(), 16);
4214 Load16BitExceptionValue(LiftoffRegister(dst), values_array, index, pinned);
4215 __ emit_i32_or(dst, upper.gp(), dst);
4216 }
4217
Load64BitExceptionValue(LiftoffRegister dst,LiftoffRegister values_array,uint32_t * index,LiftoffRegList pinned)4218 void Load64BitExceptionValue(LiftoffRegister dst,
4219 LiftoffRegister values_array, uint32_t* index,
4220 LiftoffRegList pinned) {
4221 if (kNeedI64RegPair) {
4222 Load32BitExceptionValue(dst.high_gp(), values_array, index, pinned);
4223 Load32BitExceptionValue(dst.low_gp(), values_array, index, pinned);
4224 } else {
4225 Load16BitExceptionValue(dst, values_array, index, pinned);
4226 __ emit_i64_shli(dst, dst, 48);
4227 LiftoffRegister tmp_reg =
4228 pinned.set(__ GetUnusedRegister(kGpReg, pinned));
4229 Load16BitExceptionValue(tmp_reg, values_array, index, pinned);
4230 __ emit_i64_shli(tmp_reg, tmp_reg, 32);
4231 __ emit_i64_or(dst, tmp_reg, dst);
4232 Load16BitExceptionValue(tmp_reg, values_array, index, pinned);
4233 __ emit_i64_shli(tmp_reg, tmp_reg, 16);
4234 __ emit_i64_or(dst, tmp_reg, dst);
4235 Load16BitExceptionValue(tmp_reg, values_array, index, pinned);
4236 __ emit_i64_or(dst, tmp_reg, dst);
4237 }
4238 }
4239
StoreExceptionValue(ValueType type,Register values_array,int * index_in_array,LiftoffRegList pinned)4240 void StoreExceptionValue(ValueType type, Register values_array,
4241 int* index_in_array, LiftoffRegList pinned) {
4242 LiftoffRegister value = pinned.set(__ PopToRegister(pinned));
4243 switch (type.kind()) {
4244 case kI32:
4245 Store32BitExceptionValue(values_array, index_in_array, value.gp(),
4246 pinned);
4247 break;
4248 case kF32: {
4249 LiftoffRegister gp_reg =
4250 pinned.set(__ GetUnusedRegister(kGpReg, pinned));
4251 __ emit_type_conversion(kExprI32ReinterpretF32, gp_reg, value, nullptr);
4252 Store32BitExceptionValue(values_array, index_in_array, gp_reg.gp(),
4253 pinned);
4254 break;
4255 }
4256 case kI64:
4257 Store64BitExceptionValue(values_array, index_in_array, value, pinned);
4258 break;
4259 case kF64: {
4260 LiftoffRegister tmp_reg =
4261 pinned.set(__ GetUnusedRegister(reg_class_for(kI64), pinned));
4262 __ emit_type_conversion(kExprI64ReinterpretF64, tmp_reg, value,
4263 nullptr);
4264 Store64BitExceptionValue(values_array, index_in_array, tmp_reg, pinned);
4265 break;
4266 }
4267 case kS128: {
4268 LiftoffRegister tmp_reg =
4269 pinned.set(__ GetUnusedRegister(kGpReg, pinned));
4270 for (int i : {3, 2, 1, 0}) {
4271 __ emit_i32x4_extract_lane(tmp_reg, value, i);
4272 Store32BitExceptionValue(values_array, index_in_array, tmp_reg.gp(),
4273 pinned);
4274 }
4275 break;
4276 }
4277 case wasm::kRef:
4278 case wasm::kOptRef:
4279 case wasm::kRtt: {
4280 --(*index_in_array);
4281 __ StoreTaggedPointer(
4282 values_array, no_reg,
4283 wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(
4284 *index_in_array),
4285 value, pinned);
4286 break;
4287 }
4288 case wasm::kI8:
4289 case wasm::kI16:
4290 case wasm::kVoid:
4291 case wasm::kBottom:
4292 UNREACHABLE();
4293 }
4294 }
4295
LoadExceptionValue(ValueKind kind,LiftoffRegister values_array,uint32_t * index,LiftoffRegList pinned)4296 void LoadExceptionValue(ValueKind kind, LiftoffRegister values_array,
4297 uint32_t* index, LiftoffRegList pinned) {
4298 RegClass rc = reg_class_for(kind);
4299 LiftoffRegister value = pinned.set(__ GetUnusedRegister(rc, pinned));
4300 switch (kind) {
4301 case kI32:
4302 Load32BitExceptionValue(value.gp(), values_array, index, pinned);
4303 break;
4304 case kF32: {
4305 LiftoffRegister tmp_reg =
4306 pinned.set(__ GetUnusedRegister(kGpReg, pinned));
4307 Load32BitExceptionValue(tmp_reg.gp(), values_array, index, pinned);
4308 __ emit_type_conversion(kExprF32ReinterpretI32, value, tmp_reg,
4309 nullptr);
4310 break;
4311 }
4312 case kI64:
4313 Load64BitExceptionValue(value, values_array, index, pinned);
4314 break;
4315 case kF64: {
4316 RegClass rc_i64 = reg_class_for(kI64);
4317 LiftoffRegister tmp_reg =
4318 pinned.set(__ GetUnusedRegister(rc_i64, pinned));
4319 Load64BitExceptionValue(tmp_reg, values_array, index, pinned);
4320 __ emit_type_conversion(kExprF64ReinterpretI64, value, tmp_reg,
4321 nullptr);
4322 break;
4323 }
4324 case kS128: {
4325 LiftoffRegister tmp_reg =
4326 pinned.set(__ GetUnusedRegister(kGpReg, pinned));
4327 Load32BitExceptionValue(tmp_reg.gp(), values_array, index, pinned);
4328 __ emit_i32x4_splat(value, tmp_reg);
4329 for (int lane : {1, 2, 3}) {
4330 Load32BitExceptionValue(tmp_reg.gp(), values_array, index, pinned);
4331 __ emit_i32x4_replace_lane(value, value, tmp_reg, lane);
4332 }
4333 break;
4334 }
4335 case wasm::kRef:
4336 case wasm::kOptRef:
4337 case wasm::kRtt: {
4338 __ LoadTaggedPointer(
4339 value.gp(), values_array.gp(), no_reg,
4340 wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(*index),
4341 pinned);
4342 (*index)++;
4343 break;
4344 }
4345 case wasm::kI8:
4346 case wasm::kI16:
4347 case wasm::kVoid:
4348 case wasm::kBottom:
4349 UNREACHABLE();
4350 }
4351 __ PushRegister(kind, value);
4352 }
4353
GetExceptionValues(FullDecoder * decoder,LiftoffAssembler::VarState & exception_var,const WasmTag * tag)4354 void GetExceptionValues(FullDecoder* decoder,
4355 LiftoffAssembler::VarState& exception_var,
4356 const WasmTag* tag) {
4357 LiftoffRegList pinned;
4358 CODE_COMMENT("get exception values");
4359 LiftoffRegister values_array = GetExceptionProperty(
4360 exception_var, RootIndex::kwasm_exception_values_symbol);
4361 pinned.set(values_array);
4362 uint32_t index = 0;
4363 const WasmTagSig* sig = tag->sig;
4364 for (ValueType param : sig->parameters()) {
4365 LoadExceptionValue(param.kind(), values_array, &index, pinned);
4366 }
4367 DCHECK_EQ(index, WasmExceptionPackage::GetEncodedSize(tag));
4368 }
4369
EmitLandingPad(FullDecoder * decoder,int handler_offset)4370 void EmitLandingPad(FullDecoder* decoder, int handler_offset) {
4371 if (decoder->current_catch() == -1) return;
4372 MovableLabel handler;
4373
4374 // If we return from the throwing code normally, just skip over the handler.
4375 Label skip_handler;
4376 __ emit_jump(&skip_handler);
4377
4378 // Handler: merge into the catch state, and jump to the catch body.
4379 CODE_COMMENT("-- landing pad --");
4380 __ bind(handler.get());
4381 __ ExceptionHandler();
4382 __ PushException();
4383 handlers_.push_back({std::move(handler), handler_offset});
4384 Control* current_try =
4385 decoder->control_at(decoder->control_depth_of_current_catch());
4386 DCHECK_NOT_NULL(current_try->try_info);
4387 if (!current_try->try_info->catch_reached) {
4388 current_try->try_info->catch_state.InitMerge(
4389 *__ cache_state(), __ num_locals(), 1,
4390 current_try->stack_depth + current_try->num_exceptions);
4391 current_try->try_info->catch_reached = true;
4392 }
4393 __ MergeStackWith(current_try->try_info->catch_state, 1,
4394 LiftoffAssembler::kForwardJump);
4395 __ emit_jump(¤t_try->try_info->catch_label);
4396
4397 __ bind(&skip_handler);
4398 // Drop the exception.
4399 __ DropValues(1);
4400 }
4401
Throw(FullDecoder * decoder,const TagIndexImmediate<validate> & imm,const base::Vector<Value> &)4402 void Throw(FullDecoder* decoder, const TagIndexImmediate<validate>& imm,
4403 const base::Vector<Value>& /* args */) {
4404 LiftoffRegList pinned;
4405
4406 // Load the encoded size in a register for the builtin call.
4407 int encoded_size = WasmExceptionPackage::GetEncodedSize(imm.tag);
4408 LiftoffRegister encoded_size_reg =
4409 pinned.set(__ GetUnusedRegister(kGpReg, pinned));
4410 __ LoadConstant(encoded_size_reg, WasmValue(encoded_size));
4411
4412 // Call the WasmAllocateFixedArray builtin to create the values array.
4413 CallRuntimeStub(WasmCode::kWasmAllocateFixedArray,
4414 MakeSig::Returns(kPointerKind).Params(kPointerKind),
4415 {LiftoffAssembler::VarState{
4416 kSmiKind, LiftoffRegister{encoded_size_reg}, 0}},
4417 decoder->position());
4418 MaybeOSR();
4419
4420 // The FixedArray for the exception values is now in the first gp return
4421 // register.
4422 LiftoffRegister values_array{kReturnRegister0};
4423 pinned.set(values_array);
4424
4425 // Now store the exception values in the FixedArray. Do this from last to
4426 // first value, such that we can just pop them from the value stack.
4427 CODE_COMMENT("fill values array");
4428 int index = encoded_size;
4429 auto* sig = imm.tag->sig;
4430 for (size_t param_idx = sig->parameter_count(); param_idx > 0;
4431 --param_idx) {
4432 ValueType type = sig->GetParam(param_idx - 1);
4433 StoreExceptionValue(type, values_array.gp(), &index, pinned);
4434 }
4435 DCHECK_EQ(0, index);
4436
4437 // Load the exception tag.
4438 CODE_COMMENT("load exception tag");
4439 LiftoffRegister exception_tag =
4440 pinned.set(__ GetUnusedRegister(kGpReg, pinned));
4441 LOAD_TAGGED_PTR_INSTANCE_FIELD(exception_tag.gp(), TagsTable, pinned);
4442 __ LoadTaggedPointer(
4443 exception_tag.gp(), exception_tag.gp(), no_reg,
4444 wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(imm.index), {});
4445
4446 // Finally, call WasmThrow.
4447 CallRuntimeStub(WasmCode::kWasmThrow,
4448 MakeSig::Params(kPointerKind, kPointerKind),
4449 {LiftoffAssembler::VarState{kPointerKind, exception_tag, 0},
4450 LiftoffAssembler::VarState{kPointerKind, values_array, 0}},
4451 decoder->position());
4452
4453 int pc_offset = __ pc_offset();
4454 MaybeOSR();
4455 EmitLandingPad(decoder, pc_offset);
4456 }
4457
AtomicStoreMem(FullDecoder * decoder,StoreType type,const MemoryAccessImmediate<validate> & imm)4458 void AtomicStoreMem(FullDecoder* decoder, StoreType type,
4459 const MemoryAccessImmediate<validate>& imm) {
4460 LiftoffRegList pinned;
4461 LiftoffRegister value = pinned.set(__ PopToRegister());
4462 LiftoffRegister full_index = __ PopToRegister(pinned);
4463 Register index = BoundsCheckMem(decoder, type.size(), imm.offset,
4464 full_index, pinned, kDoForceCheck);
4465 if (index == no_reg) return;
4466
4467 pinned.set(index);
4468 AlignmentCheckMem(decoder, type.size(), imm.offset, index, pinned);
4469 uintptr_t offset = imm.offset;
4470 CODE_COMMENT("atomic store to memory");
4471 Register addr = pinned.set(GetMemoryStart(pinned));
4472 LiftoffRegList outer_pinned;
4473 if (V8_UNLIKELY(FLAG_trace_wasm_memory)) outer_pinned.set(index);
4474 __ AtomicStore(addr, index, offset, value, type, outer_pinned);
4475 if (V8_UNLIKELY(FLAG_trace_wasm_memory)) {
4476 TraceMemoryOperation(true, type.mem_rep(), index, offset,
4477 decoder->position());
4478 }
4479 }
4480
AtomicLoadMem(FullDecoder * decoder,LoadType type,const MemoryAccessImmediate<validate> & imm)4481 void AtomicLoadMem(FullDecoder* decoder, LoadType type,
4482 const MemoryAccessImmediate<validate>& imm) {
4483 ValueKind kind = type.value_type().kind();
4484 LiftoffRegister full_index = __ PopToRegister();
4485 Register index = BoundsCheckMem(decoder, type.size(), imm.offset,
4486 full_index, {}, kDoForceCheck);
4487 if (index == no_reg) return;
4488
4489 LiftoffRegList pinned = {index};
4490 AlignmentCheckMem(decoder, type.size(), imm.offset, index, pinned);
4491 uintptr_t offset = imm.offset;
4492 CODE_COMMENT("atomic load from memory");
4493 Register addr = pinned.set(GetMemoryStart(pinned));
4494 RegClass rc = reg_class_for(kind);
4495 LiftoffRegister value = pinned.set(__ GetUnusedRegister(rc, pinned));
4496 __ AtomicLoad(value, addr, index, offset, type, pinned);
4497 __ PushRegister(kind, value);
4498
4499 if (V8_UNLIKELY(FLAG_trace_wasm_memory)) {
4500 TraceMemoryOperation(false, type.mem_type().representation(), index,
4501 offset, decoder->position());
4502 }
4503 }
4504
AtomicBinop(FullDecoder * decoder,StoreType type,const MemoryAccessImmediate<validate> & imm,void (LiftoffAssembler::* emit_fn)(Register,Register,uintptr_t,LiftoffRegister,LiftoffRegister,StoreType))4505 void AtomicBinop(FullDecoder* decoder, StoreType type,
4506 const MemoryAccessImmediate<validate>& imm,
4507 void (LiftoffAssembler::*emit_fn)(Register, Register,
4508 uintptr_t, LiftoffRegister,
4509 LiftoffRegister,
4510 StoreType)) {
4511 ValueKind result_kind = type.value_type().kind();
4512 LiftoffRegList pinned;
4513 LiftoffRegister value = pinned.set(__ PopToRegister());
4514 #ifdef V8_TARGET_ARCH_IA32
4515 // We have to reuse the value register as the result register so that we
4516 // don't run out of registers on ia32. For this we use the value register as
4517 // the result register if it has no other uses. Otherwise we allocate a new
4518 // register and let go of the value register to get spilled.
4519 LiftoffRegister result = value;
4520 if (__ cache_state()->is_used(value)) {
4521 result = pinned.set(__ GetUnusedRegister(value.reg_class(), pinned));
4522 __ Move(result, value, result_kind);
4523 pinned.clear(value);
4524 value = result;
4525 }
4526 #else
4527 LiftoffRegister result =
4528 pinned.set(__ GetUnusedRegister(value.reg_class(), pinned));
4529 #endif
4530 LiftoffRegister full_index = __ PopToRegister(pinned);
4531 Register index = BoundsCheckMem(decoder, type.size(), imm.offset,
4532 full_index, pinned, kDoForceCheck);
4533 if (index == no_reg) return;
4534
4535 pinned.set(index);
4536 AlignmentCheckMem(decoder, type.size(), imm.offset, index, pinned);
4537
4538 CODE_COMMENT("atomic binop");
4539 uintptr_t offset = imm.offset;
4540 Register addr = pinned.set(GetMemoryStart(pinned));
4541
4542 (asm_.*emit_fn)(addr, index, offset, value, result, type);
4543 __ PushRegister(result_kind, result);
4544 }
4545
AtomicCompareExchange(FullDecoder * decoder,StoreType type,const MemoryAccessImmediate<validate> & imm)4546 void AtomicCompareExchange(FullDecoder* decoder, StoreType type,
4547 const MemoryAccessImmediate<validate>& imm) {
4548 #ifdef V8_TARGET_ARCH_IA32
4549 // On ia32 we don't have enough registers to first pop all the values off
4550 // the stack and then start with the code generation. Instead we do the
4551 // complete address calculation first, so that the address only needs a
4552 // single register. Afterwards we load all remaining values into the
4553 // other registers.
4554 LiftoffRegister full_index = __ PeekToRegister(2, {});
4555 Register index = BoundsCheckMem(decoder, type.size(), imm.offset,
4556 full_index, {}, kDoForceCheck);
4557 if (index == no_reg) return;
4558 LiftoffRegList pinned = {index};
4559 AlignmentCheckMem(decoder, type.size(), imm.offset, index, pinned);
4560
4561 uintptr_t offset = imm.offset;
4562 Register addr = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
4563 LOAD_INSTANCE_FIELD(addr, MemoryStart, kSystemPointerSize, pinned);
4564 #ifdef V8_SANDBOXED_POINTERS
4565 __ DecodeSandboxedPointer(addr);
4566 #endif
4567 __ emit_i32_add(addr, addr, index);
4568 pinned.clear(LiftoffRegister(index));
4569 LiftoffRegister new_value = pinned.set(__ PopToRegister(pinned));
4570 LiftoffRegister expected = pinned.set(__ PopToRegister(pinned));
4571
4572 // Pop the index from the stack.
4573 __ DropValues(1);
4574
4575 LiftoffRegister result = expected;
4576 if (__ cache_state()->is_used(result)) __ SpillRegister(result);
4577
4578 // We already added the index to addr, so we can just pass no_reg to the
4579 // assembler now.
4580 __ AtomicCompareExchange(addr, no_reg, offset, expected, new_value, result,
4581 type);
4582 __ PushRegister(type.value_type().kind(), result);
4583 return;
4584 #else
4585 ValueKind result_kind = type.value_type().kind();
4586 LiftoffRegList pinned;
4587 LiftoffRegister new_value = pinned.set(__ PopToRegister());
4588 LiftoffRegister expected = pinned.set(__ PopToRegister(pinned));
4589 LiftoffRegister full_index = __ PopToRegister(pinned);
4590 Register index = BoundsCheckMem(decoder, type.size(), imm.offset,
4591 full_index, pinned, kDoForceCheck);
4592 if (index == no_reg) return;
4593 pinned.set(index);
4594 AlignmentCheckMem(decoder, type.size(), imm.offset, index, pinned);
4595
4596 uintptr_t offset = imm.offset;
4597 Register addr = pinned.set(GetMemoryStart(pinned));
4598 LiftoffRegister result =
4599 pinned.set(__ GetUnusedRegister(reg_class_for(result_kind), pinned));
4600
4601 __ AtomicCompareExchange(addr, index, offset, expected, new_value, result,
4602 type);
4603 __ PushRegister(result_kind, result);
4604 #endif
4605 }
4606
CallRuntimeStub(WasmCode::RuntimeStubId stub_id,const ValueKindSig & sig,std::initializer_list<LiftoffAssembler::VarState> params,int position)4607 void CallRuntimeStub(WasmCode::RuntimeStubId stub_id, const ValueKindSig& sig,
4608 std::initializer_list<LiftoffAssembler::VarState> params,
4609 int position) {
4610 CODE_COMMENT(
4611 (std::string{"call builtin: "} + GetRuntimeStubName(stub_id)).c_str());
4612 auto interface_descriptor = Builtins::CallInterfaceDescriptorFor(
4613 RuntimeStubIdToBuiltinName(stub_id));
4614 auto* call_descriptor = compiler::Linkage::GetStubCallDescriptor(
4615 compilation_zone_, // zone
4616 interface_descriptor, // descriptor
4617 interface_descriptor.GetStackParameterCount(), // stack parameter count
4618 compiler::CallDescriptor::kNoFlags, // flags
4619 compiler::Operator::kNoProperties, // properties
4620 StubCallMode::kCallWasmRuntimeStub); // stub call mode
4621
4622 __ PrepareBuiltinCall(&sig, call_descriptor, params);
4623 if (position != kNoSourcePosition) {
4624 source_position_table_builder_.AddPosition(
4625 __ pc_offset(), SourcePosition(position), true);
4626 }
4627 __ CallRuntimeStub(stub_id);
4628 DefineSafepoint();
4629 }
4630
AtomicWait(FullDecoder * decoder,ValueKind kind,const MemoryAccessImmediate<validate> & imm)4631 void AtomicWait(FullDecoder* decoder, ValueKind kind,
4632 const MemoryAccessImmediate<validate>& imm) {
4633 LiftoffRegister full_index = __ PeekToRegister(2, {});
4634 Register index_reg =
4635 BoundsCheckMem(decoder, value_kind_size(kind), imm.offset, full_index,
4636 {}, kDoForceCheck);
4637 if (index_reg == no_reg) return;
4638 LiftoffRegList pinned = {index_reg};
4639 AlignmentCheckMem(decoder, value_kind_size(kind), imm.offset, index_reg,
4640 pinned);
4641
4642 uintptr_t offset = imm.offset;
4643 Register index_plus_offset =
4644 __ cache_state()->is_used(LiftoffRegister(index_reg))
4645 ? pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp()
4646 : index_reg;
4647 // TODO(clemensb): Skip this if memory is 64 bit.
4648 __ emit_ptrsize_zeroextend_i32(index_plus_offset, index_reg);
4649 if (offset) {
4650 __ emit_ptrsize_addi(index_plus_offset, index_plus_offset, offset);
4651 }
4652
4653 LiftoffAssembler::VarState timeout =
4654 __ cache_state()->stack_state.end()[-1];
4655 LiftoffAssembler::VarState expected_value =
4656 __ cache_state()->stack_state.end()[-2];
4657 LiftoffAssembler::VarState index = __ cache_state()->stack_state.end()[-3];
4658
4659 // We have to set the correct register for the index.
4660 index.MakeRegister(LiftoffRegister(index_plus_offset));
4661
4662 static constexpr WasmCode::RuntimeStubId kTargets[2][2]{
4663 // 64 bit systems (kNeedI64RegPair == false):
4664 {WasmCode::kWasmI64AtomicWait64, WasmCode::kWasmI32AtomicWait64},
4665 // 32 bit systems (kNeedI64RegPair == true):
4666 {WasmCode::kWasmI64AtomicWait32, WasmCode::kWasmI32AtomicWait32}};
4667 auto target = kTargets[kNeedI64RegPair][kind == kI32];
4668
4669 CallRuntimeStub(target, MakeSig::Params(kPointerKind, kind, kI64),
4670 {index, expected_value, timeout}, decoder->position());
4671 // Pop parameters from the value stack.
4672 __ DropValues(3);
4673
4674 RegisterDebugSideTableEntry(decoder, DebugSideTableBuilder::kDidSpill);
4675
4676 __ PushRegister(kI32, LiftoffRegister(kReturnRegister0));
4677 }
4678
AtomicNotify(FullDecoder * decoder,const MemoryAccessImmediate<validate> & imm)4679 void AtomicNotify(FullDecoder* decoder,
4680 const MemoryAccessImmediate<validate>& imm) {
4681 LiftoffRegister full_index = __ PeekToRegister(1, {});
4682 Register index_reg = BoundsCheckMem(decoder, kInt32Size, imm.offset,
4683 full_index, {}, kDoForceCheck);
4684 if (index_reg == no_reg) return;
4685 LiftoffRegList pinned = {index_reg};
4686 AlignmentCheckMem(decoder, kInt32Size, imm.offset, index_reg, pinned);
4687
4688 uintptr_t offset = imm.offset;
4689 Register index_plus_offset =
4690 __ cache_state()->is_used(LiftoffRegister(index_reg))
4691 ? pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp()
4692 : index_reg;
4693 // TODO(clemensb): Skip this if memory is 64 bit.
4694 __ emit_ptrsize_zeroextend_i32(index_plus_offset, index_reg);
4695 if (offset) {
4696 __ emit_ptrsize_addi(index_plus_offset, index_plus_offset, offset);
4697 }
4698
4699 LiftoffAssembler::VarState count = __ cache_state()->stack_state.end()[-1];
4700 LiftoffAssembler::VarState index = __ cache_state()->stack_state.end()[-2];
4701 index.MakeRegister(LiftoffRegister(index_plus_offset));
4702
4703 CallRuntimeStub(WasmCode::kWasmAtomicNotify,
4704 MakeSig::Returns(kI32).Params(kPointerKind, kI32),
4705 {index, count}, decoder->position());
4706 // Pop parameters from the value stack.
4707 __ DropValues(2);
4708
4709 RegisterDebugSideTableEntry(decoder, DebugSideTableBuilder::kDidSpill);
4710
4711 __ PushRegister(kI32, LiftoffRegister(kReturnRegister0));
4712 }
4713
4714 #define ATOMIC_STORE_LIST(V) \
4715 V(I32AtomicStore, kI32Store) \
4716 V(I64AtomicStore, kI64Store) \
4717 V(I32AtomicStore8U, kI32Store8) \
4718 V(I32AtomicStore16U, kI32Store16) \
4719 V(I64AtomicStore8U, kI64Store8) \
4720 V(I64AtomicStore16U, kI64Store16) \
4721 V(I64AtomicStore32U, kI64Store32)
4722
4723 #define ATOMIC_LOAD_LIST(V) \
4724 V(I32AtomicLoad, kI32Load) \
4725 V(I64AtomicLoad, kI64Load) \
4726 V(I32AtomicLoad8U, kI32Load8U) \
4727 V(I32AtomicLoad16U, kI32Load16U) \
4728 V(I64AtomicLoad8U, kI64Load8U) \
4729 V(I64AtomicLoad16U, kI64Load16U) \
4730 V(I64AtomicLoad32U, kI64Load32U)
4731
4732 #define ATOMIC_BINOP_INSTRUCTION_LIST(V) \
4733 V(Add, I32AtomicAdd, kI32Store) \
4734 V(Add, I64AtomicAdd, kI64Store) \
4735 V(Add, I32AtomicAdd8U, kI32Store8) \
4736 V(Add, I32AtomicAdd16U, kI32Store16) \
4737 V(Add, I64AtomicAdd8U, kI64Store8) \
4738 V(Add, I64AtomicAdd16U, kI64Store16) \
4739 V(Add, I64AtomicAdd32U, kI64Store32) \
4740 V(Sub, I32AtomicSub, kI32Store) \
4741 V(Sub, I64AtomicSub, kI64Store) \
4742 V(Sub, I32AtomicSub8U, kI32Store8) \
4743 V(Sub, I32AtomicSub16U, kI32Store16) \
4744 V(Sub, I64AtomicSub8U, kI64Store8) \
4745 V(Sub, I64AtomicSub16U, kI64Store16) \
4746 V(Sub, I64AtomicSub32U, kI64Store32) \
4747 V(And, I32AtomicAnd, kI32Store) \
4748 V(And, I64AtomicAnd, kI64Store) \
4749 V(And, I32AtomicAnd8U, kI32Store8) \
4750 V(And, I32AtomicAnd16U, kI32Store16) \
4751 V(And, I64AtomicAnd8U, kI64Store8) \
4752 V(And, I64AtomicAnd16U, kI64Store16) \
4753 V(And, I64AtomicAnd32U, kI64Store32) \
4754 V(Or, I32AtomicOr, kI32Store) \
4755 V(Or, I64AtomicOr, kI64Store) \
4756 V(Or, I32AtomicOr8U, kI32Store8) \
4757 V(Or, I32AtomicOr16U, kI32Store16) \
4758 V(Or, I64AtomicOr8U, kI64Store8) \
4759 V(Or, I64AtomicOr16U, kI64Store16) \
4760 V(Or, I64AtomicOr32U, kI64Store32) \
4761 V(Xor, I32AtomicXor, kI32Store) \
4762 V(Xor, I64AtomicXor, kI64Store) \
4763 V(Xor, I32AtomicXor8U, kI32Store8) \
4764 V(Xor, I32AtomicXor16U, kI32Store16) \
4765 V(Xor, I64AtomicXor8U, kI64Store8) \
4766 V(Xor, I64AtomicXor16U, kI64Store16) \
4767 V(Xor, I64AtomicXor32U, kI64Store32) \
4768 V(Exchange, I32AtomicExchange, kI32Store) \
4769 V(Exchange, I64AtomicExchange, kI64Store) \
4770 V(Exchange, I32AtomicExchange8U, kI32Store8) \
4771 V(Exchange, I32AtomicExchange16U, kI32Store16) \
4772 V(Exchange, I64AtomicExchange8U, kI64Store8) \
4773 V(Exchange, I64AtomicExchange16U, kI64Store16) \
4774 V(Exchange, I64AtomicExchange32U, kI64Store32)
4775
4776 #define ATOMIC_COMPARE_EXCHANGE_LIST(V) \
4777 V(I32AtomicCompareExchange, kI32Store) \
4778 V(I64AtomicCompareExchange, kI64Store) \
4779 V(I32AtomicCompareExchange8U, kI32Store8) \
4780 V(I32AtomicCompareExchange16U, kI32Store16) \
4781 V(I64AtomicCompareExchange8U, kI64Store8) \
4782 V(I64AtomicCompareExchange16U, kI64Store16) \
4783 V(I64AtomicCompareExchange32U, kI64Store32)
4784
AtomicOp(FullDecoder * decoder,WasmOpcode opcode,base::Vector<Value> args,const MemoryAccessImmediate<validate> & imm,Value * result)4785 void AtomicOp(FullDecoder* decoder, WasmOpcode opcode,
4786 base::Vector<Value> args,
4787 const MemoryAccessImmediate<validate>& imm, Value* result) {
4788 switch (opcode) {
4789 #define ATOMIC_STORE_OP(name, type) \
4790 case wasm::kExpr##name: \
4791 AtomicStoreMem(decoder, StoreType::type, imm); \
4792 break;
4793
4794 ATOMIC_STORE_LIST(ATOMIC_STORE_OP)
4795 #undef ATOMIC_STORE_OP
4796
4797 #define ATOMIC_LOAD_OP(name, type) \
4798 case wasm::kExpr##name: \
4799 AtomicLoadMem(decoder, LoadType::type, imm); \
4800 break;
4801
4802 ATOMIC_LOAD_LIST(ATOMIC_LOAD_OP)
4803 #undef ATOMIC_LOAD_OP
4804
4805 #define ATOMIC_BINOP_OP(op, name, type) \
4806 case wasm::kExpr##name: \
4807 AtomicBinop(decoder, StoreType::type, imm, &LiftoffAssembler::Atomic##op); \
4808 break;
4809
4810 ATOMIC_BINOP_INSTRUCTION_LIST(ATOMIC_BINOP_OP)
4811 #undef ATOMIC_BINOP_OP
4812
4813 #define ATOMIC_COMPARE_EXCHANGE_OP(name, type) \
4814 case wasm::kExpr##name: \
4815 AtomicCompareExchange(decoder, StoreType::type, imm); \
4816 break;
4817
4818 ATOMIC_COMPARE_EXCHANGE_LIST(ATOMIC_COMPARE_EXCHANGE_OP)
4819 #undef ATOMIC_COMPARE_EXCHANGE_OP
4820
4821 case kExprI32AtomicWait:
4822 AtomicWait(decoder, kI32, imm);
4823 break;
4824 case kExprI64AtomicWait:
4825 AtomicWait(decoder, kI64, imm);
4826 break;
4827 case kExprAtomicNotify:
4828 AtomicNotify(decoder, imm);
4829 break;
4830 default:
4831 unsupported(decoder, kAtomics, "atomicop");
4832 }
4833 }
4834
4835 #undef ATOMIC_STORE_LIST
4836 #undef ATOMIC_LOAD_LIST
4837 #undef ATOMIC_BINOP_INSTRUCTION_LIST
4838 #undef ATOMIC_COMPARE_EXCHANGE_LIST
4839
AtomicFence(FullDecoder * decoder)4840 void AtomicFence(FullDecoder* decoder) { __ AtomicFence(); }
4841
4842 // Pop a memtype (i32 or i64 depending on {WasmModule::is_memory64}) to a
4843 // register, updating {*high_word} to contain the ORed combination of all
4844 // popped high words. Returns the ptrsized register holding the popped value.
PopMemTypeToRegister(FullDecoder * decoder,Register * high_word,LiftoffRegList * pinned)4845 LiftoffRegister PopMemTypeToRegister(FullDecoder* decoder,
4846 Register* high_word,
4847 LiftoffRegList* pinned) {
4848 LiftoffRegister reg = __ PopToRegister(*pinned);
4849 LiftoffRegister intptr_reg = reg;
4850 // For memory32 on 64-bit hosts, zero-extend.
4851 if (kSystemPointerSize == kInt64Size && !env_->module->is_memory64) {
4852 // Only overwrite {reg} if it's not used otherwise.
4853 if (pinned->has(reg) || __ cache_state()->is_used(reg)) {
4854 intptr_reg = __ GetUnusedRegister(kGpReg, *pinned);
4855 }
4856 __ emit_u32_to_uintptr(intptr_reg.gp(), reg.gp());
4857 }
4858 // For memory32 or memory64 on 64-bit, we are done here.
4859 if (kSystemPointerSize == kInt64Size || !env_->module->is_memory64) {
4860 pinned->set(intptr_reg);
4861 return intptr_reg;
4862 }
4863
4864 // For memory64 on 32-bit systems, combine all high words for a zero-check
4865 // and only use the low words afterwards. This keeps the register pressure
4866 // managable.
4867 DCHECK_GE(kMaxUInt32, env_->max_memory_size);
4868 pinned->set(reg.low());
4869 if (*high_word == no_reg) {
4870 // Choose a register to hold the (combination of) high word(s). It cannot
4871 // be one of the pinned registers, and it cannot be used in the value
4872 // stack.
4873 *high_word =
4874 pinned->has(reg.high())
4875 ? __ GetUnusedRegister(kGpReg, *pinned).gp()
4876 : __ GetUnusedRegister(kGpReg, {reg.high()}, *pinned).gp();
4877 pinned->set(*high_word);
4878 if (*high_word != reg.high_gp()) {
4879 __ Move(*high_word, reg.high_gp(), kI32);
4880 }
4881 } else if (*high_word != reg.high_gp()) {
4882 // Combine the new high word into existing high words.
4883 __ emit_i32_or(*high_word, *high_word, reg.high_gp());
4884 }
4885 return reg.low();
4886 }
4887
MemoryInit(FullDecoder * decoder,const MemoryInitImmediate<validate> & imm,const Value &,const Value &,const Value &)4888 void MemoryInit(FullDecoder* decoder,
4889 const MemoryInitImmediate<validate>& imm, const Value&,
4890 const Value&, const Value&) {
4891 Register mem_offsets_high_word = no_reg;
4892 LiftoffRegList pinned;
4893 LiftoffRegister size = pinned.set(__ PopToRegister());
4894 LiftoffRegister src = pinned.set(__ PopToRegister(pinned));
4895 LiftoffRegister dst =
4896 PopMemTypeToRegister(decoder, &mem_offsets_high_word, &pinned);
4897
4898 Register instance = __ cache_state()->cached_instance;
4899 if (instance == no_reg) {
4900 instance = __ GetUnusedRegister(kGpReg, pinned).gp();
4901 __ LoadInstanceFromFrame(instance);
4902 }
4903 pinned.set(instance);
4904
4905 // Only allocate the OOB code now, so the state of the stack is reflected
4906 // correctly.
4907 Label* trap_label =
4908 AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapMemOutOfBounds);
4909 if (mem_offsets_high_word != no_reg) {
4910 // If any high word has bits set, jump to the OOB trap.
4911 __ emit_cond_jump(kNotEqualZero, trap_label, kI32, mem_offsets_high_word);
4912 pinned.clear(mem_offsets_high_word);
4913 }
4914
4915 LiftoffRegister segment_index =
4916 pinned.set(__ GetUnusedRegister(kGpReg, pinned));
4917 __ LoadConstant(segment_index, WasmValue(imm.data_segment.index));
4918
4919 auto sig = MakeSig::Returns(kI32).Params(kPointerKind, kPointerKind, kI32,
4920 kI32, kI32);
4921 LiftoffRegister args[] = {LiftoffRegister(instance), dst, src,
4922 segment_index, size};
4923 // We don't need the instance anymore after the call. We can use the
4924 // register for the result.
4925 LiftoffRegister result(instance);
4926 GenerateCCall(&result, &sig, kVoid, args,
4927 ExternalReference::wasm_memory_init());
4928 __ emit_cond_jump(kEqual, trap_label, kI32, result.gp());
4929 }
4930
DataDrop(FullDecoder * decoder,const IndexImmediate<validate> & imm)4931 void DataDrop(FullDecoder* decoder, const IndexImmediate<validate>& imm) {
4932 LiftoffRegList pinned;
4933
4934 Register seg_size_array =
4935 pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
4936 LOAD_INSTANCE_FIELD(seg_size_array, DataSegmentSizes, kSystemPointerSize,
4937 pinned);
4938
4939 LiftoffRegister seg_index =
4940 pinned.set(__ GetUnusedRegister(kGpReg, pinned));
4941 // Scale the seg_index for the array access.
4942 __ LoadConstant(seg_index,
4943 WasmValue(imm.index << value_kind_size_log2(kI32)));
4944
4945 // Set the length of the segment to '0' to drop it.
4946 LiftoffRegister null_reg = pinned.set(__ GetUnusedRegister(kGpReg, pinned));
4947 __ LoadConstant(null_reg, WasmValue(0));
4948 __ Store(seg_size_array, seg_index.gp(), 0, null_reg, StoreType::kI32Store,
4949 pinned);
4950 }
4951
MemoryCopy(FullDecoder * decoder,const MemoryCopyImmediate<validate> & imm,const Value &,const Value &,const Value &)4952 void MemoryCopy(FullDecoder* decoder,
4953 const MemoryCopyImmediate<validate>& imm, const Value&,
4954 const Value&, const Value&) {
4955 Register mem_offsets_high_word = no_reg;
4956 LiftoffRegList pinned;
4957 LiftoffRegister size = pinned.set(
4958 PopMemTypeToRegister(decoder, &mem_offsets_high_word, &pinned));
4959 LiftoffRegister src = pinned.set(
4960 PopMemTypeToRegister(decoder, &mem_offsets_high_word, &pinned));
4961 LiftoffRegister dst = pinned.set(
4962 PopMemTypeToRegister(decoder, &mem_offsets_high_word, &pinned));
4963
4964 Register instance = __ cache_state()->cached_instance;
4965 if (instance == no_reg) {
4966 instance = __ GetUnusedRegister(kGpReg, pinned).gp();
4967 __ LoadInstanceFromFrame(instance);
4968 }
4969
4970 // Only allocate the OOB code now, so the state of the stack is reflected
4971 // correctly.
4972 Label* trap_label =
4973 AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapMemOutOfBounds);
4974 if (mem_offsets_high_word != no_reg) {
4975 // If any high word has bits set, jump to the OOB trap.
4976 __ emit_cond_jump(kNotEqualZero, trap_label, kI32, mem_offsets_high_word);
4977 }
4978
4979 auto sig = MakeSig::Returns(kI32).Params(kPointerKind, kPointerKind,
4980 kPointerKind, kPointerKind);
4981 LiftoffRegister args[] = {LiftoffRegister(instance), dst, src, size};
4982 // We don't need the instance anymore after the call. We can use the
4983 // register for the result.
4984 LiftoffRegister result(instance);
4985 GenerateCCall(&result, &sig, kVoid, args,
4986 ExternalReference::wasm_memory_copy());
4987 __ emit_cond_jump(kEqual, trap_label, kI32, result.gp());
4988 }
4989
MemoryFill(FullDecoder * decoder,const MemoryIndexImmediate<validate> & imm,const Value &,const Value &,const Value &)4990 void MemoryFill(FullDecoder* decoder,
4991 const MemoryIndexImmediate<validate>& imm, const Value&,
4992 const Value&, const Value&) {
4993 Register mem_offsets_high_word = no_reg;
4994 LiftoffRegList pinned;
4995 LiftoffRegister size = pinned.set(
4996 PopMemTypeToRegister(decoder, &mem_offsets_high_word, &pinned));
4997 LiftoffRegister value = pinned.set(__ PopToRegister(pinned));
4998 LiftoffRegister dst = pinned.set(
4999 PopMemTypeToRegister(decoder, &mem_offsets_high_word, &pinned));
5000
5001 Register instance = __ cache_state()->cached_instance;
5002 if (instance == no_reg) {
5003 instance = __ GetUnusedRegister(kGpReg, pinned).gp();
5004 __ LoadInstanceFromFrame(instance);
5005 }
5006
5007 // Only allocate the OOB code now, so the state of the stack is reflected
5008 // correctly.
5009 Label* trap_label =
5010 AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapMemOutOfBounds);
5011 if (mem_offsets_high_word != no_reg) {
5012 // If any high word has bits set, jump to the OOB trap.
5013 __ emit_cond_jump(kNotEqualZero, trap_label, kI32, mem_offsets_high_word);
5014 }
5015
5016 auto sig = MakeSig::Returns(kI32).Params(kPointerKind, kPointerKind, kI32,
5017 kPointerKind);
5018 LiftoffRegister args[] = {LiftoffRegister(instance), dst, value, size};
5019 // We don't need the instance anymore after the call. We can use the
5020 // register for the result.
5021 LiftoffRegister result(instance);
5022 GenerateCCall(&result, &sig, kVoid, args,
5023 ExternalReference::wasm_memory_fill());
5024 __ emit_cond_jump(kEqual, trap_label, kI32, result.gp());
5025 }
5026
LoadSmi(LiftoffRegister reg,int value)5027 void LoadSmi(LiftoffRegister reg, int value) {
5028 Address smi_value = Smi::FromInt(value).ptr();
5029 using smi_type = std::conditional_t<kSmiKind == kI32, int32_t, int64_t>;
5030 __ LoadConstant(reg, WasmValue{static_cast<smi_type>(smi_value)});
5031 }
5032
TableInit(FullDecoder * decoder,const TableInitImmediate<validate> & imm,base::Vector<Value> args)5033 void TableInit(FullDecoder* decoder, const TableInitImmediate<validate>& imm,
5034 base::Vector<Value> args) {
5035 LiftoffRegList pinned;
5036 LiftoffRegister table_index_reg =
5037 pinned.set(__ GetUnusedRegister(kGpReg, pinned));
5038
5039 LoadSmi(table_index_reg, imm.table.index);
5040 LiftoffAssembler::VarState table_index(kPointerKind, table_index_reg, 0);
5041
5042 LiftoffRegister segment_index_reg =
5043 pinned.set(__ GetUnusedRegister(kGpReg, pinned));
5044 LoadSmi(segment_index_reg, imm.element_segment.index);
5045 LiftoffAssembler::VarState segment_index(kPointerKind, segment_index_reg,
5046 0);
5047
5048 LiftoffAssembler::VarState size = __ cache_state()->stack_state.end()[-1];
5049 LiftoffAssembler::VarState src = __ cache_state()->stack_state.end()[-2];
5050 LiftoffAssembler::VarState dst = __ cache_state()->stack_state.end()[-3];
5051
5052 CallRuntimeStub(WasmCode::kWasmTableInit,
5053 MakeSig::Params(kI32, kI32, kI32, kSmiKind, kSmiKind),
5054 {dst, src, size, table_index, segment_index},
5055 decoder->position());
5056
5057 // Pop parameters from the value stack.
5058 __ cache_state()->stack_state.pop_back(3);
5059
5060 RegisterDebugSideTableEntry(decoder, DebugSideTableBuilder::kDidSpill);
5061 }
5062
ElemDrop(FullDecoder * decoder,const IndexImmediate<validate> & imm)5063 void ElemDrop(FullDecoder* decoder, const IndexImmediate<validate>& imm) {
5064 LiftoffRegList pinned;
5065 Register dropped_elem_segments =
5066 pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
5067 LOAD_INSTANCE_FIELD(dropped_elem_segments, DroppedElemSegments,
5068 kSystemPointerSize, pinned);
5069
5070 LiftoffRegister seg_index =
5071 pinned.set(__ GetUnusedRegister(kGpReg, pinned));
5072 __ LoadConstant(seg_index, WasmValue(imm.index));
5073
5074 // Mark the segment as dropped by setting its value in the dropped
5075 // segments list to 1.
5076 LiftoffRegister one_reg = pinned.set(__ GetUnusedRegister(kGpReg, pinned));
5077 __ LoadConstant(one_reg, WasmValue(1));
5078 __ Store(dropped_elem_segments, seg_index.gp(), 0, one_reg,
5079 StoreType::kI32Store8, pinned);
5080 }
5081
TableCopy(FullDecoder * decoder,const TableCopyImmediate<validate> & imm,base::Vector<Value> args)5082 void TableCopy(FullDecoder* decoder, const TableCopyImmediate<validate>& imm,
5083 base::Vector<Value> args) {
5084 LiftoffRegList pinned;
5085
5086 LiftoffRegister table_dst_index_reg =
5087 pinned.set(__ GetUnusedRegister(kGpReg, pinned));
5088 LoadSmi(table_dst_index_reg, imm.table_dst.index);
5089 LiftoffAssembler::VarState table_dst_index(kPointerKind,
5090 table_dst_index_reg, 0);
5091
5092 LiftoffRegister table_src_index_reg =
5093 pinned.set(__ GetUnusedRegister(kGpReg, pinned));
5094 LoadSmi(table_src_index_reg, imm.table_src.index);
5095 LiftoffAssembler::VarState table_src_index(kPointerKind,
5096 table_src_index_reg, 0);
5097
5098 LiftoffAssembler::VarState size = __ cache_state()->stack_state.end()[-1];
5099 LiftoffAssembler::VarState src = __ cache_state()->stack_state.end()[-2];
5100 LiftoffAssembler::VarState dst = __ cache_state()->stack_state.end()[-3];
5101
5102 CallRuntimeStub(WasmCode::kWasmTableCopy,
5103 MakeSig::Params(kI32, kI32, kI32, kSmiKind, kSmiKind),
5104 {dst, src, size, table_dst_index, table_src_index},
5105 decoder->position());
5106
5107 // Pop parameters from the value stack.
5108 __ cache_state()->stack_state.pop_back(3);
5109
5110 RegisterDebugSideTableEntry(decoder, DebugSideTableBuilder::kDidSpill);
5111 }
5112
TableGrow(FullDecoder * decoder,const IndexImmediate<validate> & imm,const Value &,const Value &,Value * result)5113 void TableGrow(FullDecoder* decoder, const IndexImmediate<validate>& imm,
5114 const Value&, const Value&, Value* result) {
5115 LiftoffRegList pinned;
5116
5117 LiftoffRegister table_index_reg =
5118 pinned.set(__ GetUnusedRegister(kGpReg, pinned));
5119 LoadSmi(table_index_reg, imm.index);
5120 LiftoffAssembler::VarState table_index(kPointerKind, table_index_reg, 0);
5121
5122 LiftoffAssembler::VarState delta = __ cache_state()->stack_state.end()[-1];
5123 LiftoffAssembler::VarState value = __ cache_state()->stack_state.end()[-2];
5124
5125 CallRuntimeStub(
5126 WasmCode::kWasmTableGrow,
5127 MakeSig::Returns(kSmiKind).Params(kSmiKind, kI32, kTaggedKind),
5128 {table_index, delta, value}, decoder->position());
5129
5130 // Pop parameters from the value stack.
5131 __ cache_state()->stack_state.pop_back(2);
5132
5133 RegisterDebugSideTableEntry(decoder, DebugSideTableBuilder::kDidSpill);
5134 __ SmiToInt32(kReturnRegister0);
5135 __ PushRegister(kI32, LiftoffRegister(kReturnRegister0));
5136 }
5137
TableSize(FullDecoder * decoder,const IndexImmediate<validate> & imm,Value *)5138 void TableSize(FullDecoder* decoder, const IndexImmediate<validate>& imm,
5139 Value*) {
5140 // We have to look up instance->tables[table_index].length.
5141
5142 LiftoffRegList pinned;
5143 // Get the number of calls array address.
5144 Register tables = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
5145 LOAD_TAGGED_PTR_INSTANCE_FIELD(tables, Tables, pinned);
5146
5147 Register table = tables;
5148 __ LoadTaggedPointer(
5149 table, tables, no_reg,
5150 ObjectAccess::ElementOffsetInTaggedFixedArray(imm.index), pinned);
5151
5152 int length_field_size = WasmTableObject::kCurrentLengthOffsetEnd -
5153 WasmTableObject::kCurrentLengthOffset + 1;
5154
5155 Register result = table;
5156 __ Load(LiftoffRegister(result), table, no_reg,
5157 wasm::ObjectAccess::ToTagged(WasmTableObject::kCurrentLengthOffset),
5158 length_field_size == 4 ? LoadType::kI32Load : LoadType::kI64Load,
5159 pinned);
5160
5161 __ SmiUntag(result);
5162 __ PushRegister(kI32, LiftoffRegister(result));
5163 }
5164
TableFill(FullDecoder * decoder,const IndexImmediate<validate> & imm,const Value &,const Value &,const Value &)5165 void TableFill(FullDecoder* decoder, const IndexImmediate<validate>& imm,
5166 const Value&, const Value&, const Value&) {
5167 LiftoffRegList pinned;
5168
5169 LiftoffRegister table_index_reg =
5170 pinned.set(__ GetUnusedRegister(kGpReg, pinned));
5171 LoadSmi(table_index_reg, imm.index);
5172 LiftoffAssembler::VarState table_index(kPointerKind, table_index_reg, 0);
5173
5174 LiftoffAssembler::VarState count = __ cache_state()->stack_state.end()[-1];
5175 LiftoffAssembler::VarState value = __ cache_state()->stack_state.end()[-2];
5176 LiftoffAssembler::VarState start = __ cache_state()->stack_state.end()[-3];
5177
5178 CallRuntimeStub(WasmCode::kWasmTableFill,
5179 MakeSig::Params(kSmiKind, kI32, kI32, kTaggedKind),
5180 {table_index, start, count, value}, decoder->position());
5181
5182 // Pop parameters from the value stack.
5183 __ cache_state()->stack_state.pop_back(3);
5184
5185 RegisterDebugSideTableEntry(decoder, DebugSideTableBuilder::kDidSpill);
5186 }
5187
StructNew(FullDecoder * decoder,const StructIndexImmediate<validate> & imm,const Value & rtt,bool initial_values_on_stack)5188 void StructNew(FullDecoder* decoder,
5189 const StructIndexImmediate<validate>& imm, const Value& rtt,
5190 bool initial_values_on_stack) {
5191 LiftoffAssembler::VarState rtt_value =
5192 __ cache_state()->stack_state.end()[-1];
5193 CallRuntimeStub(WasmCode::kWasmAllocateStructWithRtt,
5194 MakeSig::Returns(kRef).Params(rtt.type.kind()), {rtt_value},
5195 decoder->position());
5196 // Drop the RTT.
5197 __ cache_state()->stack_state.pop_back(1);
5198
5199 LiftoffRegister obj(kReturnRegister0);
5200 LiftoffRegList pinned = {obj};
5201 for (uint32_t i = imm.struct_type->field_count(); i > 0;) {
5202 i--;
5203 int offset = StructFieldOffset(imm.struct_type, i);
5204 ValueKind field_kind = imm.struct_type->field(i).kind();
5205 LiftoffRegister value = initial_values_on_stack
5206 ? pinned.set(__ PopToRegister(pinned))
5207 : pinned.set(__ GetUnusedRegister(
5208 reg_class_for(field_kind), pinned));
5209 if (!initial_values_on_stack) {
5210 if (!CheckSupportedType(decoder, field_kind, "default value")) return;
5211 SetDefaultValue(value, field_kind, pinned);
5212 }
5213 StoreObjectField(obj.gp(), no_reg, offset, value, pinned, field_kind);
5214 pinned.clear(value);
5215 }
5216 // If this assert fails then initialization of padding field might be
5217 // necessary.
5218 static_assert(Heap::kMinObjectSizeInTaggedWords == 2 &&
5219 WasmStruct::kHeaderSize == 2 * kTaggedSize,
5220 "empty struct might require initialization of padding field");
5221 __ PushRegister(kRef, obj);
5222 }
5223
StructNewWithRtt(FullDecoder * decoder,const StructIndexImmediate<validate> & imm,const Value & rtt,const Value args[],Value * result)5224 void StructNewWithRtt(FullDecoder* decoder,
5225 const StructIndexImmediate<validate>& imm,
5226 const Value& rtt, const Value args[], Value* result) {
5227 StructNew(decoder, imm, rtt, true);
5228 }
5229
StructNewDefault(FullDecoder * decoder,const StructIndexImmediate<validate> & imm,const Value & rtt,Value * result)5230 void StructNewDefault(FullDecoder* decoder,
5231 const StructIndexImmediate<validate>& imm,
5232 const Value& rtt, Value* result) {
5233 StructNew(decoder, imm, rtt, false);
5234 }
5235
StructGet(FullDecoder * decoder,const Value & struct_obj,const FieldImmediate<validate> & field,bool is_signed,Value * result)5236 void StructGet(FullDecoder* decoder, const Value& struct_obj,
5237 const FieldImmediate<validate>& field, bool is_signed,
5238 Value* result) {
5239 const StructType* struct_type = field.struct_imm.struct_type;
5240 ValueKind field_kind = struct_type->field(field.field_imm.index).kind();
5241 if (!CheckSupportedType(decoder, field_kind, "field load")) return;
5242 int offset = StructFieldOffset(struct_type, field.field_imm.index);
5243 LiftoffRegList pinned;
5244 LiftoffRegister obj = pinned.set(__ PopToRegister(pinned));
5245 MaybeEmitNullCheck(decoder, obj.gp(), pinned, struct_obj.type);
5246 LiftoffRegister value =
5247 __ GetUnusedRegister(reg_class_for(field_kind), pinned);
5248 LoadObjectField(value, obj.gp(), no_reg, offset, field_kind, is_signed,
5249 pinned);
5250 __ PushRegister(unpacked(field_kind), value);
5251 }
5252
StructSet(FullDecoder * decoder,const Value & struct_obj,const FieldImmediate<validate> & field,const Value & field_value)5253 void StructSet(FullDecoder* decoder, const Value& struct_obj,
5254 const FieldImmediate<validate>& field,
5255 const Value& field_value) {
5256 const StructType* struct_type = field.struct_imm.struct_type;
5257 ValueKind field_kind = struct_type->field(field.field_imm.index).kind();
5258 int offset = StructFieldOffset(struct_type, field.field_imm.index);
5259 LiftoffRegList pinned;
5260 LiftoffRegister value = pinned.set(__ PopToRegister(pinned));
5261 LiftoffRegister obj = pinned.set(__ PopToRegister(pinned));
5262 MaybeEmitNullCheck(decoder, obj.gp(), pinned, struct_obj.type);
5263 StoreObjectField(obj.gp(), no_reg, offset, value, pinned, field_kind);
5264 }
5265
ArrayNew(FullDecoder * decoder,const ArrayIndexImmediate<validate> & imm,ValueKind rtt_kind,bool initial_value_on_stack)5266 void ArrayNew(FullDecoder* decoder, const ArrayIndexImmediate<validate>& imm,
5267 ValueKind rtt_kind, bool initial_value_on_stack) {
5268 // Max length check.
5269 {
5270 LiftoffRegister length =
5271 __ LoadToRegister(__ cache_state()->stack_state.end()[-2], {});
5272 Label* trap_label =
5273 AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapArrayTooLarge);
5274 __ emit_i32_cond_jumpi(kUnsignedGreaterThan, trap_label, length.gp(),
5275 WasmArray::MaxLength(imm.array_type));
5276 }
5277 ValueKind elem_kind = imm.array_type->element_type().kind();
5278 int elem_size = value_kind_size(elem_kind);
5279 // Allocate the array.
5280 {
5281 LiftoffRegister elem_size_reg = __ GetUnusedRegister(kGpReg, {});
5282 LiftoffAssembler::VarState rtt_var =
5283 __ cache_state()->stack_state.end()[-1];
5284 LiftoffAssembler::VarState length_var =
5285 __ cache_state()->stack_state.end()[-2];
5286 __ LoadConstant(elem_size_reg, WasmValue(elem_size));
5287 LiftoffAssembler::VarState elem_size_var(kI32, elem_size_reg, 0);
5288
5289 WasmCode::RuntimeStubId stub_id =
5290 initial_value_on_stack
5291 ? WasmCode::kWasmAllocateArray_Uninitialized
5292 : is_reference(elem_kind) ? WasmCode::kWasmAllocateArray_InitNull
5293 : WasmCode::kWasmAllocateArray_InitZero;
5294 CallRuntimeStub(
5295 stub_id, MakeSig::Returns(kRef).Params(rtt_kind, kI32, kI32),
5296 {rtt_var, length_var, elem_size_var}, decoder->position());
5297 // Drop the RTT.
5298 __ cache_state()->stack_state.pop_back(1);
5299 }
5300
5301 LiftoffRegister obj(kReturnRegister0);
5302 if (initial_value_on_stack) {
5303 LiftoffRegList pinned = {obj};
5304 LiftoffRegister length = pinned.set(__ PopToModifiableRegister(pinned));
5305 LiftoffRegister value = pinned.set(__ PopToRegister(pinned));
5306
5307 // Initialize the array's elements.
5308 LiftoffRegister offset = pinned.set(__ GetUnusedRegister(kGpReg, pinned));
5309 __ LoadConstant(
5310 offset,
5311 WasmValue(wasm::ObjectAccess::ToTagged(WasmArray::kHeaderSize)));
5312 LiftoffRegister end_offset = length;
5313 if (value_kind_size_log2(elem_kind) != 0) {
5314 __ emit_i32_shli(end_offset.gp(), length.gp(),
5315 value_kind_size_log2(elem_kind));
5316 }
5317 __ emit_i32_add(end_offset.gp(), end_offset.gp(), offset.gp());
5318 Label loop, done;
5319 __ bind(&loop);
5320 __ emit_cond_jump(kUnsignedGreaterEqual, &done, kI32, offset.gp(),
5321 end_offset.gp());
5322 StoreObjectField(obj.gp(), offset.gp(), 0, value, pinned, elem_kind);
5323 __ emit_i32_addi(offset.gp(), offset.gp(), elem_size);
5324 __ emit_jump(&loop);
5325
5326 __ bind(&done);
5327 } else {
5328 if (!CheckSupportedType(decoder, elem_kind, "default value")) return;
5329 // Drop the length.
5330 __ cache_state()->stack_state.pop_back(1);
5331 }
5332 __ PushRegister(kRef, obj);
5333 }
5334
ArrayNewWithRtt(FullDecoder * decoder,const ArrayIndexImmediate<validate> & imm,const Value & length_value,const Value & initial_value,const Value & rtt,Value * result)5335 void ArrayNewWithRtt(FullDecoder* decoder,
5336 const ArrayIndexImmediate<validate>& imm,
5337 const Value& length_value, const Value& initial_value,
5338 const Value& rtt, Value* result) {
5339 ArrayNew(decoder, imm, rtt.type.kind(), true);
5340 }
5341
ArrayNewDefault(FullDecoder * decoder,const ArrayIndexImmediate<validate> & imm,const Value & length,const Value & rtt,Value * result)5342 void ArrayNewDefault(FullDecoder* decoder,
5343 const ArrayIndexImmediate<validate>& imm,
5344 const Value& length, const Value& rtt, Value* result) {
5345 ArrayNew(decoder, imm, rtt.type.kind(), false);
5346 }
5347
ArrayGet(FullDecoder * decoder,const Value & array_obj,const ArrayIndexImmediate<validate> & imm,const Value & index_val,bool is_signed,Value * result)5348 void ArrayGet(FullDecoder* decoder, const Value& array_obj,
5349 const ArrayIndexImmediate<validate>& imm,
5350 const Value& index_val, bool is_signed, Value* result) {
5351 LiftoffRegList pinned;
5352 LiftoffRegister index = pinned.set(__ PopToModifiableRegister(pinned));
5353 LiftoffRegister array = pinned.set(__ PopToRegister(pinned));
5354 MaybeEmitNullCheck(decoder, array.gp(), pinned, array_obj.type);
5355 BoundsCheckArray(decoder, array, index, pinned);
5356 ValueKind elem_kind = imm.array_type->element_type().kind();
5357 if (!CheckSupportedType(decoder, elem_kind, "array load")) return;
5358 int elem_size_shift = value_kind_size_log2(elem_kind);
5359 if (elem_size_shift != 0) {
5360 __ emit_i32_shli(index.gp(), index.gp(), elem_size_shift);
5361 }
5362 LiftoffRegister value =
5363 __ GetUnusedRegister(reg_class_for(elem_kind), pinned);
5364 LoadObjectField(value, array.gp(), index.gp(),
5365 wasm::ObjectAccess::ToTagged(WasmArray::kHeaderSize),
5366 elem_kind, is_signed, pinned);
5367 __ PushRegister(unpacked(elem_kind), value);
5368 }
5369
ArraySet(FullDecoder * decoder,const Value & array_obj,const ArrayIndexImmediate<validate> & imm,const Value & index_val,const Value & value_val)5370 void ArraySet(FullDecoder* decoder, const Value& array_obj,
5371 const ArrayIndexImmediate<validate>& imm,
5372 const Value& index_val, const Value& value_val) {
5373 LiftoffRegList pinned;
5374 LiftoffRegister value = pinned.set(__ PopToRegister(pinned));
5375 DCHECK_EQ(reg_class_for(imm.array_type->element_type().kind()),
5376 value.reg_class());
5377 LiftoffRegister index = pinned.set(__ PopToModifiableRegister(pinned));
5378 LiftoffRegister array = pinned.set(__ PopToRegister(pinned));
5379 MaybeEmitNullCheck(decoder, array.gp(), pinned, array_obj.type);
5380 BoundsCheckArray(decoder, array, index, pinned);
5381 ValueKind elem_kind = imm.array_type->element_type().kind();
5382 int elem_size_shift = value_kind_size_log2(elem_kind);
5383 if (elem_size_shift != 0) {
5384 __ emit_i32_shli(index.gp(), index.gp(), elem_size_shift);
5385 }
5386 StoreObjectField(array.gp(), index.gp(),
5387 wasm::ObjectAccess::ToTagged(WasmArray::kHeaderSize),
5388 value, pinned, elem_kind);
5389 }
5390
ArrayLen(FullDecoder * decoder,const Value & array_obj,Value * result)5391 void ArrayLen(FullDecoder* decoder, const Value& array_obj, Value* result) {
5392 LiftoffRegList pinned;
5393 LiftoffRegister obj = pinned.set(__ PopToRegister(pinned));
5394 MaybeEmitNullCheck(decoder, obj.gp(), pinned, array_obj.type);
5395 LiftoffRegister len = __ GetUnusedRegister(kGpReg, pinned);
5396 int kLengthOffset = wasm::ObjectAccess::ToTagged(WasmArray::kLengthOffset);
5397 LoadObjectField(len, obj.gp(), no_reg, kLengthOffset, kI32, false, pinned);
5398 __ PushRegister(kI32, len);
5399 }
5400
ArrayCopy(FullDecoder * decoder,const Value & dst,const Value & dst_index,const Value & src,const Value & src_index,const Value & length)5401 void ArrayCopy(FullDecoder* decoder, const Value& dst, const Value& dst_index,
5402 const Value& src, const Value& src_index,
5403 const Value& length) {
5404 // TODO(7748): Unify implementation with TF: Implement this with
5405 // GenerateCCall. Remove runtime function and builtin in wasm.tq.
5406 CallRuntimeStub(FLAG_experimental_wasm_skip_bounds_checks
5407 ? WasmCode::kWasmArrayCopy
5408 : WasmCode::kWasmArrayCopyWithChecks,
5409 MakeSig::Params(kI32, kI32, kI32, kOptRef, kOptRef),
5410 // Builtin parameter order:
5411 // [dst_index, src_index, length, dst, src].
5412 {__ cache_state()->stack_state.end()[-4],
5413 __ cache_state()->stack_state.end()[-2],
5414 __ cache_state()->stack_state.end()[-1],
5415 __ cache_state()->stack_state.end()[-5],
5416 __ cache_state()->stack_state.end()[-3]},
5417 decoder->position());
5418 __ cache_state()->stack_state.pop_back(5);
5419 }
5420
ArrayInit(FullDecoder * decoder,const ArrayIndexImmediate<validate> & imm,const base::Vector<Value> & elements,const Value & rtt,Value * result)5421 void ArrayInit(FullDecoder* decoder, const ArrayIndexImmediate<validate>& imm,
5422 const base::Vector<Value>& elements, const Value& rtt,
5423 Value* result) {
5424 ValueKind rtt_kind = rtt.type.kind();
5425 ValueKind elem_kind = imm.array_type->element_type().kind();
5426 // Allocate the array.
5427 {
5428 LiftoffRegList pinned;
5429 LiftoffRegister elem_size_reg =
5430 pinned.set(__ GetUnusedRegister(kGpReg, pinned));
5431
5432 __ LoadConstant(elem_size_reg, WasmValue(value_kind_size(elem_kind)));
5433 LiftoffAssembler::VarState elem_size_var(kI32, elem_size_reg, 0);
5434
5435 LiftoffRegister length_reg =
5436 pinned.set(__ GetUnusedRegister(kGpReg, pinned));
5437 __ LoadConstant(length_reg,
5438 WasmValue(static_cast<int32_t>(elements.size())));
5439 LiftoffAssembler::VarState length_var(kI32, length_reg, 0);
5440
5441 LiftoffAssembler::VarState rtt_var =
5442 __ cache_state()->stack_state.end()[-1];
5443
5444 CallRuntimeStub(WasmCode::kWasmAllocateArray_Uninitialized,
5445 MakeSig::Returns(kRef).Params(rtt_kind, kI32, kI32),
5446 {rtt_var, length_var, elem_size_var},
5447 decoder->position());
5448 // Drop the RTT.
5449 __ DropValues(1);
5450 }
5451
5452 // Initialize the array with stack arguments.
5453 LiftoffRegister array(kReturnRegister0);
5454 if (!CheckSupportedType(decoder, elem_kind, "array.init")) return;
5455 for (int i = static_cast<int>(elements.size()) - 1; i >= 0; i--) {
5456 LiftoffRegList pinned = {array};
5457 LiftoffRegister element = pinned.set(__ PopToRegister(pinned));
5458 LiftoffRegister offset_reg =
5459 pinned.set(__ GetUnusedRegister(kGpReg, pinned));
5460 __ LoadConstant(offset_reg,
5461 WasmValue(i << value_kind_size_log2(elem_kind)));
5462 StoreObjectField(array.gp(), offset_reg.gp(),
5463 wasm::ObjectAccess::ToTagged(WasmArray::kHeaderSize),
5464 element, pinned, elem_kind);
5465 }
5466
5467 // Push the array onto the stack.
5468 __ PushRegister(kRef, array);
5469 }
5470
ArrayInitFromData(FullDecoder * decoder,const ArrayIndexImmediate<validate> & array_imm,const IndexImmediate<validate> & data_segment,const Value &,const Value &,const Value &,Value *)5471 void ArrayInitFromData(FullDecoder* decoder,
5472 const ArrayIndexImmediate<validate>& array_imm,
5473 const IndexImmediate<validate>& data_segment,
5474 const Value& /* offset */, const Value& /* length */,
5475 const Value& /* rtt */, Value* /* result */) {
5476 LiftoffRegList pinned;
5477 LiftoffRegister data_segment_reg =
5478 pinned.set(__ GetUnusedRegister(kGpReg, pinned));
5479 __ LoadConstant(data_segment_reg,
5480 WasmValue(static_cast<int32_t>(data_segment.index)));
5481 LiftoffAssembler::VarState data_segment_var(kI32, data_segment_reg, 0);
5482
5483 CallRuntimeStub(WasmCode::kWasmArrayInitFromData,
5484 MakeSig::Returns(kRef).Params(kI32, kI32, kI32, kRtt),
5485 {
5486 data_segment_var,
5487 __ cache_state()->stack_state.end()[-3], // offset
5488 __ cache_state()->stack_state.end()[-2], // length
5489 __ cache_state()->stack_state.end()[-1] // rtt
5490 },
5491 decoder->position());
5492
5493 LiftoffRegister result(kReturnRegister0);
5494 // Reuse the data segment register for error handling.
5495 LiftoffRegister error_smi = data_segment_reg;
5496 LoadSmi(error_smi, kArrayInitFromDataArrayTooLargeErrorCode);
5497 Label* trap_label_array_too_large =
5498 AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapArrayTooLarge);
5499 __ emit_cond_jump(kEqual, trap_label_array_too_large, kRef, result.gp(),
5500 error_smi.gp());
5501 LoadSmi(error_smi, kArrayInitFromDataSegmentOutOfBoundsErrorCode);
5502 Label* trap_label_segment_out_of_bounds = AddOutOfLineTrap(
5503 decoder, WasmCode::kThrowWasmTrapDataSegmentOutOfBounds);
5504 __ emit_cond_jump(kEqual, trap_label_segment_out_of_bounds, kRef,
5505 result.gp(), error_smi.gp());
5506
5507 __ PushRegister(kRef, result);
5508 }
5509
5510 // 1 bit Smi tag, 31 bits Smi shift, 1 bit i31ref high-bit truncation.
5511 constexpr static int kI31To32BitSmiShift = 33;
5512
I31New(FullDecoder * decoder,const Value & input,Value * result)5513 void I31New(FullDecoder* decoder, const Value& input, Value* result) {
5514 LiftoffRegister src = __ PopToRegister();
5515 LiftoffRegister dst = __ GetUnusedRegister(kGpReg, {src}, {});
5516 if (SmiValuesAre31Bits()) {
5517 STATIC_ASSERT(kSmiTag == 0);
5518 __ emit_i32_shli(dst.gp(), src.gp(), kSmiTagSize);
5519 } else {
5520 DCHECK(SmiValuesAre32Bits());
5521 __ emit_i64_shli(dst, src, kI31To32BitSmiShift);
5522 }
5523 __ PushRegister(kRef, dst);
5524 }
5525
I31GetS(FullDecoder * decoder,const Value & input,Value * result)5526 void I31GetS(FullDecoder* decoder, const Value& input, Value* result) {
5527 LiftoffRegister src = __ PopToRegister();
5528 LiftoffRegister dst = __ GetUnusedRegister(kGpReg, {src}, {});
5529 if (SmiValuesAre31Bits()) {
5530 __ emit_i32_sari(dst.gp(), src.gp(), kSmiTagSize);
5531 } else {
5532 DCHECK(SmiValuesAre32Bits());
5533 __ emit_i64_sari(dst, src, kI31To32BitSmiShift);
5534 }
5535 __ PushRegister(kI32, dst);
5536 }
5537
I31GetU(FullDecoder * decoder,const Value & input,Value * result)5538 void I31GetU(FullDecoder* decoder, const Value& input, Value* result) {
5539 LiftoffRegister src = __ PopToRegister();
5540 LiftoffRegister dst = __ GetUnusedRegister(kGpReg, {src}, {});
5541 if (SmiValuesAre31Bits()) {
5542 __ emit_i32_shri(dst.gp(), src.gp(), kSmiTagSize);
5543 } else {
5544 DCHECK(SmiValuesAre32Bits());
5545 __ emit_i64_shri(dst, src, kI31To32BitSmiShift);
5546 }
5547 __ PushRegister(kI32, dst);
5548 }
5549
RttCanon(FullDecoder * decoder,uint32_t type_index,Value * result)5550 void RttCanon(FullDecoder* decoder, uint32_t type_index, Value* result) {
5551 LiftoffRegister rtt = __ GetUnusedRegister(kGpReg, {});
5552 LOAD_TAGGED_PTR_INSTANCE_FIELD(rtt.gp(), ManagedObjectMaps, {});
5553 __ LoadTaggedPointer(
5554 rtt.gp(), rtt.gp(), no_reg,
5555 wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(type_index), {});
5556 __ PushRegister(kRtt, rtt);
5557 }
5558
5559 enum NullSucceeds : bool { // --
5560 kNullSucceeds = true,
5561 kNullFails = false
5562 };
5563
5564 // Falls through on match (=successful type check).
5565 // Returns the register containing the object.
SubtypeCheck(FullDecoder * decoder,const Value & obj,const Value & rtt,Label * no_match,NullSucceeds null_succeeds,LiftoffRegList pinned={},Register opt_scratch=no_reg)5566 LiftoffRegister SubtypeCheck(FullDecoder* decoder, const Value& obj,
5567 const Value& rtt, Label* no_match,
5568 NullSucceeds null_succeeds,
5569 LiftoffRegList pinned = {},
5570 Register opt_scratch = no_reg) {
5571 Label match;
5572 LiftoffRegister rtt_reg = pinned.set(__ PopToRegister(pinned));
5573 LiftoffRegister obj_reg = pinned.set(__ PopToRegister(pinned));
5574
5575 // Reserve all temporary registers up front, so that the cache state
5576 // tracking doesn't get confused by the following conditional jumps.
5577 LiftoffRegister tmp1 =
5578 opt_scratch != no_reg
5579 ? LiftoffRegister(opt_scratch)
5580 : pinned.set(__ GetUnusedRegister(kGpReg, pinned));
5581 LiftoffRegister tmp2 = pinned.set(__ GetUnusedRegister(kGpReg, pinned));
5582 if (obj.type.is_nullable()) {
5583 LoadNullValue(tmp1.gp(), pinned);
5584 __ emit_cond_jump(kEqual, null_succeeds ? &match : no_match,
5585 obj.type.kind(), obj_reg.gp(), tmp1.gp());
5586 }
5587
5588 __ LoadMap(tmp1.gp(), obj_reg.gp());
5589 // {tmp1} now holds the object's map.
5590
5591 // Check for rtt equality, and if not, check if the rtt is a struct/array
5592 // rtt.
5593 __ emit_cond_jump(kEqual, &match, rtt.type.kind(), tmp1.gp(), rtt_reg.gp());
5594
5595 // Constant-time subtyping check: load exactly one candidate RTT from the
5596 // supertypes list.
5597 // Step 1: load the WasmTypeInfo into {tmp1}.
5598 constexpr int kTypeInfoOffset = wasm::ObjectAccess::ToTagged(
5599 Map::kConstructorOrBackPointerOrNativeContextOffset);
5600 __ LoadTaggedPointer(tmp1.gp(), tmp1.gp(), no_reg, kTypeInfoOffset, pinned);
5601 // Step 2: load the supertypes list into {tmp1}.
5602 constexpr int kSuperTypesOffset =
5603 wasm::ObjectAccess::ToTagged(WasmTypeInfo::kSupertypesOffset);
5604 __ LoadTaggedPointer(tmp1.gp(), tmp1.gp(), no_reg, kSuperTypesOffset,
5605 pinned);
5606 // Step 3: check the list's length if needed.
5607 uint32_t rtt_depth =
5608 GetSubtypingDepth(decoder->module_, rtt.type.ref_index());
5609 if (rtt_depth >= kMinimumSupertypeArraySize) {
5610 LiftoffRegister list_length = tmp2;
5611 __ LoadFixedArrayLengthAsInt32(list_length, tmp1.gp(), pinned);
5612 __ emit_i32_cond_jumpi(kUnsignedLessEqual, no_match, list_length.gp(),
5613 rtt_depth);
5614 }
5615 // Step 4: load the candidate list slot into {tmp1}, and compare it.
5616 __ LoadTaggedPointer(
5617 tmp1.gp(), tmp1.gp(), no_reg,
5618 wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(rtt_depth), pinned);
5619 __ emit_cond_jump(kUnequal, no_match, rtt.type.kind(), tmp1.gp(),
5620 rtt_reg.gp());
5621
5622 // Fall through to {match}.
5623 __ bind(&match);
5624 return obj_reg;
5625 }
5626
RefTest(FullDecoder * decoder,const Value & obj,const Value & rtt,Value *)5627 void RefTest(FullDecoder* decoder, const Value& obj, const Value& rtt,
5628 Value* /* result_val */) {
5629 Label return_false, done;
5630 LiftoffRegList pinned;
5631 LiftoffRegister result = pinned.set(__ GetUnusedRegister(kGpReg, {}));
5632
5633 SubtypeCheck(decoder, obj, rtt, &return_false, kNullFails, pinned,
5634 result.gp());
5635
5636 __ LoadConstant(result, WasmValue(1));
5637 // TODO(jkummerow): Emit near jumps on platforms where it's more efficient.
5638 __ emit_jump(&done);
5639
5640 __ bind(&return_false);
5641 __ LoadConstant(result, WasmValue(0));
5642 __ bind(&done);
5643 __ PushRegister(kI32, result);
5644 }
5645
RefCast(FullDecoder * decoder,const Value & obj,const Value & rtt,Value * result)5646 void RefCast(FullDecoder* decoder, const Value& obj, const Value& rtt,
5647 Value* result) {
5648 if (FLAG_experimental_wasm_assume_ref_cast_succeeds) {
5649 // Just drop the rtt.
5650 __ DropValues(1);
5651 } else {
5652 Label* trap_label =
5653 AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapIllegalCast);
5654 LiftoffRegister obj_reg =
5655 SubtypeCheck(decoder, obj, rtt, trap_label, kNullSucceeds);
5656 __ PushRegister(obj.type.kind(), obj_reg);
5657 }
5658 }
5659
BrOnCast(FullDecoder * decoder,const Value & obj,const Value & rtt,Value *,uint32_t depth)5660 void BrOnCast(FullDecoder* decoder, const Value& obj, const Value& rtt,
5661 Value* /* result_on_branch */, uint32_t depth) {
5662 // Before branching, materialize all constants. This avoids repeatedly
5663 // materializing them for each conditional branch.
5664 if (depth != decoder->control_depth() - 1) {
5665 __ MaterializeMergedConstants(
5666 decoder->control_at(depth)->br_merge()->arity);
5667 }
5668
5669 Label cont_false;
5670 LiftoffRegister obj_reg =
5671 SubtypeCheck(decoder, obj, rtt, &cont_false, kNullFails);
5672
5673 __ PushRegister(rtt.type.is_bottom() ? kBottom : obj.type.kind(), obj_reg);
5674 BrOrRet(decoder, depth, 0);
5675
5676 __ bind(&cont_false);
5677 // Drop the branch's value, restore original value.
5678 Drop(decoder);
5679 __ PushRegister(obj.type.kind(), obj_reg);
5680 }
5681
BrOnCastFail(FullDecoder * decoder,const Value & obj,const Value & rtt,Value *,uint32_t depth)5682 void BrOnCastFail(FullDecoder* decoder, const Value& obj, const Value& rtt,
5683 Value* /* result_on_fallthrough */, uint32_t depth) {
5684 // Before branching, materialize all constants. This avoids repeatedly
5685 // materializing them for each conditional branch.
5686 if (depth != decoder->control_depth() - 1) {
5687 __ MaterializeMergedConstants(
5688 decoder->control_at(depth)->br_merge()->arity);
5689 }
5690
5691 Label cont_branch, fallthrough;
5692 LiftoffRegister obj_reg =
5693 SubtypeCheck(decoder, obj, rtt, &cont_branch, kNullFails);
5694 __ PushRegister(obj.type.kind(), obj_reg);
5695 __ emit_jump(&fallthrough);
5696
5697 __ bind(&cont_branch);
5698 BrOrRet(decoder, depth, 0);
5699
5700 __ bind(&fallthrough);
5701 }
5702
5703 // Abstract type checkers. They all return the object register and fall
5704 // through to match.
DataCheck(const Value & obj,Label * no_match,LiftoffRegList pinned,Register opt_scratch)5705 LiftoffRegister DataCheck(const Value& obj, Label* no_match,
5706 LiftoffRegList pinned, Register opt_scratch) {
5707 TypeCheckRegisters registers =
5708 TypeCheckPrelude(obj, no_match, pinned, opt_scratch);
5709 EmitDataRefCheck(registers.map_reg.gp(), no_match, registers.tmp_reg,
5710 pinned);
5711 return registers.obj_reg;
5712 }
5713
ArrayCheck(const Value & obj,Label * no_match,LiftoffRegList pinned,Register opt_scratch)5714 LiftoffRegister ArrayCheck(const Value& obj, Label* no_match,
5715 LiftoffRegList pinned, Register opt_scratch) {
5716 TypeCheckRegisters registers =
5717 TypeCheckPrelude(obj, no_match, pinned, opt_scratch);
5718 __ Load(registers.map_reg, registers.map_reg.gp(), no_reg,
5719 wasm::ObjectAccess::ToTagged(Map::kInstanceTypeOffset),
5720 LoadType::kI32Load16U, pinned);
5721 __ emit_i32_cond_jumpi(kUnequal, no_match, registers.map_reg.gp(),
5722 WASM_ARRAY_TYPE);
5723 return registers.obj_reg;
5724 }
5725
FuncCheck(const Value & obj,Label * no_match,LiftoffRegList pinned,Register opt_scratch)5726 LiftoffRegister FuncCheck(const Value& obj, Label* no_match,
5727 LiftoffRegList pinned, Register opt_scratch) {
5728 TypeCheckRegisters registers =
5729 TypeCheckPrelude(obj, no_match, pinned, opt_scratch);
5730 __ Load(registers.map_reg, registers.map_reg.gp(), no_reg,
5731 wasm::ObjectAccess::ToTagged(Map::kInstanceTypeOffset),
5732 LoadType::kI32Load16U, pinned);
5733 __ emit_i32_cond_jumpi(kUnequal, no_match, registers.map_reg.gp(),
5734 WASM_INTERNAL_FUNCTION_TYPE);
5735 return registers.obj_reg;
5736 }
5737
I31Check(const Value & object,Label * no_match,LiftoffRegList pinned,Register opt_scratch)5738 LiftoffRegister I31Check(const Value& object, Label* no_match,
5739 LiftoffRegList pinned, Register opt_scratch) {
5740 LiftoffRegister obj_reg = pinned.set(__ PopToRegister(pinned));
5741
5742 __ emit_smi_check(obj_reg.gp(), no_match, LiftoffAssembler::kJumpOnNotSmi);
5743
5744 return obj_reg;
5745 }
5746
5747 using TypeChecker = LiftoffRegister (LiftoffCompiler::*)(
5748 const Value& obj, Label* no_match, LiftoffRegList pinned,
5749 Register opt_scratch);
5750
5751 template <TypeChecker type_checker>
AbstractTypeCheck(const Value & object)5752 void AbstractTypeCheck(const Value& object) {
5753 Label match, no_match, done;
5754 LiftoffRegList pinned;
5755 LiftoffRegister result = pinned.set(__ GetUnusedRegister(kGpReg, {}));
5756
5757 (this->*type_checker)(object, &no_match, pinned, result.gp());
5758
5759 __ bind(&match);
5760 __ LoadConstant(result, WasmValue(1));
5761 // TODO(jkummerow): Emit near jumps on platforms where it's more efficient.
5762 __ emit_jump(&done);
5763
5764 __ bind(&no_match);
5765 __ LoadConstant(result, WasmValue(0));
5766 __ bind(&done);
5767 __ PushRegister(kI32, result);
5768 }
5769
RefIsData(FullDecoder *,const Value & object,Value *)5770 void RefIsData(FullDecoder* /* decoder */, const Value& object,
5771 Value* /* result_val */) {
5772 return AbstractTypeCheck<&LiftoffCompiler::DataCheck>(object);
5773 }
5774
RefIsFunc(FullDecoder *,const Value & object,Value *)5775 void RefIsFunc(FullDecoder* /* decoder */, const Value& object,
5776 Value* /* result_val */) {
5777 return AbstractTypeCheck<&LiftoffCompiler::FuncCheck>(object);
5778 }
5779
RefIsArray(FullDecoder *,const Value & object,Value *)5780 void RefIsArray(FullDecoder* /* decoder */, const Value& object,
5781 Value* /* result_val */) {
5782 return AbstractTypeCheck<&LiftoffCompiler::ArrayCheck>(object);
5783 }
5784
RefIsI31(FullDecoder * decoder,const Value & object,Value *)5785 void RefIsI31(FullDecoder* decoder, const Value& object,
5786 Value* /* result */) {
5787 return AbstractTypeCheck<&LiftoffCompiler::I31Check>(object);
5788 }
5789
5790 template <TypeChecker type_checker>
AbstractTypeCast(const Value & object,FullDecoder * decoder,ValueKind result_kind)5791 void AbstractTypeCast(const Value& object, FullDecoder* decoder,
5792 ValueKind result_kind) {
5793 Label* trap_label =
5794 AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapIllegalCast);
5795 Label match;
5796 LiftoffRegister obj_reg =
5797 (this->*type_checker)(object, trap_label, {}, no_reg);
5798 __ bind(&match);
5799 __ PushRegister(result_kind, obj_reg);
5800 }
5801
RefAsData(FullDecoder * decoder,const Value & object,Value *)5802 void RefAsData(FullDecoder* decoder, const Value& object,
5803 Value* /* result */) {
5804 return AbstractTypeCast<&LiftoffCompiler::DataCheck>(object, decoder, kRef);
5805 }
5806
RefAsFunc(FullDecoder * decoder,const Value & object,Value *)5807 void RefAsFunc(FullDecoder* decoder, const Value& object,
5808 Value* /* result */) {
5809 return AbstractTypeCast<&LiftoffCompiler::FuncCheck>(object, decoder, kRef);
5810 }
5811
RefAsI31(FullDecoder * decoder,const Value & object,Value * result)5812 void RefAsI31(FullDecoder* decoder, const Value& object, Value* result) {
5813 return AbstractTypeCast<&LiftoffCompiler::I31Check>(object, decoder, kRef);
5814 }
5815
RefAsArray(FullDecoder * decoder,const Value & object,Value * result)5816 void RefAsArray(FullDecoder* decoder, const Value& object, Value* result) {
5817 return AbstractTypeCast<&LiftoffCompiler::ArrayCheck>(object, decoder,
5818 kRef);
5819 }
5820
5821 template <TypeChecker type_checker>
BrOnAbstractType(const Value & object,FullDecoder * decoder,uint32_t br_depth)5822 void BrOnAbstractType(const Value& object, FullDecoder* decoder,
5823 uint32_t br_depth) {
5824 // Before branching, materialize all constants. This avoids repeatedly
5825 // materializing them for each conditional branch.
5826 if (br_depth != decoder->control_depth() - 1) {
5827 __ MaterializeMergedConstants(
5828 decoder->control_at(br_depth)->br_merge()->arity);
5829 }
5830
5831 Label no_match;
5832 LiftoffRegister obj_reg =
5833 (this->*type_checker)(object, &no_match, {}, no_reg);
5834
5835 __ PushRegister(kRef, obj_reg);
5836 BrOrRet(decoder, br_depth, 0);
5837
5838 __ bind(&no_match);
5839 }
5840
5841 template <TypeChecker type_checker>
BrOnNonAbstractType(const Value & object,FullDecoder * decoder,uint32_t br_depth)5842 void BrOnNonAbstractType(const Value& object, FullDecoder* decoder,
5843 uint32_t br_depth) {
5844 // Before branching, materialize all constants. This avoids repeatedly
5845 // materializing them for each conditional branch.
5846 if (br_depth != decoder->control_depth() - 1) {
5847 __ MaterializeMergedConstants(
5848 decoder->control_at(br_depth)->br_merge()->arity);
5849 }
5850
5851 Label no_match, end;
5852 LiftoffRegister obj_reg =
5853 (this->*type_checker)(object, &no_match, {}, no_reg);
5854 __ PushRegister(kRef, obj_reg);
5855 __ emit_jump(&end);
5856
5857 __ bind(&no_match);
5858 BrOrRet(decoder, br_depth, 0);
5859
5860 __ bind(&end);
5861 }
5862
BrOnData(FullDecoder * decoder,const Value & object,Value *,uint32_t br_depth)5863 void BrOnData(FullDecoder* decoder, const Value& object,
5864 Value* /* value_on_branch */, uint32_t br_depth) {
5865 return BrOnAbstractType<&LiftoffCompiler::DataCheck>(object, decoder,
5866 br_depth);
5867 }
5868
BrOnFunc(FullDecoder * decoder,const Value & object,Value *,uint32_t br_depth)5869 void BrOnFunc(FullDecoder* decoder, const Value& object,
5870 Value* /* value_on_branch */, uint32_t br_depth) {
5871 return BrOnAbstractType<&LiftoffCompiler::FuncCheck>(object, decoder,
5872 br_depth);
5873 }
5874
BrOnI31(FullDecoder * decoder,const Value & object,Value *,uint32_t br_depth)5875 void BrOnI31(FullDecoder* decoder, const Value& object,
5876 Value* /* value_on_branch */, uint32_t br_depth) {
5877 return BrOnAbstractType<&LiftoffCompiler::I31Check>(object, decoder,
5878 br_depth);
5879 }
5880
BrOnArray(FullDecoder * decoder,const Value & object,Value *,uint32_t br_depth)5881 void BrOnArray(FullDecoder* decoder, const Value& object,
5882 Value* /* value_on_branch */, uint32_t br_depth) {
5883 return BrOnAbstractType<&LiftoffCompiler::ArrayCheck>(object, decoder,
5884 br_depth);
5885 }
5886
BrOnNonData(FullDecoder * decoder,const Value & object,Value *,uint32_t br_depth)5887 void BrOnNonData(FullDecoder* decoder, const Value& object,
5888 Value* /* value_on_branch */, uint32_t br_depth) {
5889 return BrOnNonAbstractType<&LiftoffCompiler::DataCheck>(object, decoder,
5890 br_depth);
5891 }
5892
BrOnNonFunc(FullDecoder * decoder,const Value & object,Value *,uint32_t br_depth)5893 void BrOnNonFunc(FullDecoder* decoder, const Value& object,
5894 Value* /* value_on_branch */, uint32_t br_depth) {
5895 return BrOnNonAbstractType<&LiftoffCompiler::FuncCheck>(object, decoder,
5896 br_depth);
5897 }
5898
BrOnNonI31(FullDecoder * decoder,const Value & object,Value *,uint32_t br_depth)5899 void BrOnNonI31(FullDecoder* decoder, const Value& object,
5900 Value* /* value_on_branch */, uint32_t br_depth) {
5901 return BrOnNonAbstractType<&LiftoffCompiler::I31Check>(object, decoder,
5902 br_depth);
5903 }
5904
BrOnNonArray(FullDecoder * decoder,const Value & object,Value *,uint32_t br_depth)5905 void BrOnNonArray(FullDecoder* decoder, const Value& object,
5906 Value* /* value_on_branch */, uint32_t br_depth) {
5907 return BrOnNonAbstractType<&LiftoffCompiler::ArrayCheck>(object, decoder,
5908 br_depth);
5909 }
5910
Forward(FullDecoder * decoder,const Value & from,Value * to)5911 void Forward(FullDecoder* decoder, const Value& from, Value* to) {
5912 // Nothing to do here.
5913 }
5914
5915 private:
CallDirect(FullDecoder * decoder,const CallFunctionImmediate<validate> & imm,const Value args[],Value returns[],TailCall tail_call)5916 void CallDirect(FullDecoder* decoder,
5917 const CallFunctionImmediate<validate>& imm,
5918 const Value args[], Value returns[], TailCall tail_call) {
5919 MostlySmallValueKindSig sig(compilation_zone_, imm.sig);
5920 for (ValueKind ret : sig.returns()) {
5921 if (!CheckSupportedType(decoder, ret, "return")) return;
5922 }
5923
5924 auto call_descriptor =
5925 compiler::GetWasmCallDescriptor(compilation_zone_, imm.sig);
5926 call_descriptor =
5927 GetLoweredCallDescriptor(compilation_zone_, call_descriptor);
5928
5929 // One slot would be enough for call_direct, but would make index
5930 // computations much more complicated.
5931 uintptr_t vector_slot = num_call_instructions_ * 2;
5932 if (FLAG_wasm_speculative_inlining) {
5933 base::MutexGuard mutex_guard(&decoder->module_->type_feedback.mutex);
5934 decoder->module_->type_feedback.feedback_for_function[func_index_]
5935 .positions[decoder->position()] =
5936 static_cast<int>(num_call_instructions_);
5937 num_call_instructions_++;
5938 }
5939
5940 if (imm.index < env_->module->num_imported_functions) {
5941 // A direct call to an imported function.
5942 LiftoffRegList pinned;
5943 Register tmp = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
5944 Register target = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
5945
5946 Register imported_targets = tmp;
5947 LOAD_INSTANCE_FIELD(imported_targets, ImportedFunctionTargets,
5948 kSystemPointerSize, pinned);
5949 __ Load(LiftoffRegister(target), imported_targets, no_reg,
5950 imm.index * sizeof(Address), kPointerLoadType, pinned);
5951
5952 Register imported_function_refs = tmp;
5953 LOAD_TAGGED_PTR_INSTANCE_FIELD(imported_function_refs,
5954 ImportedFunctionRefs, pinned);
5955 Register imported_function_ref = tmp;
5956 __ LoadTaggedPointer(
5957 imported_function_ref, imported_function_refs, no_reg,
5958 ObjectAccess::ElementOffsetInTaggedFixedArray(imm.index), pinned);
5959
5960 Register* explicit_instance = &imported_function_ref;
5961 __ PrepareCall(&sig, call_descriptor, &target, explicit_instance);
5962 if (tail_call) {
5963 __ PrepareTailCall(
5964 static_cast<int>(call_descriptor->ParameterSlotCount()),
5965 static_cast<int>(
5966 call_descriptor->GetStackParameterDelta(descriptor_)));
5967 __ TailCallIndirect(target);
5968 } else {
5969 source_position_table_builder_.AddPosition(
5970 __ pc_offset(), SourcePosition(decoder->position()), true);
5971 __ CallIndirect(&sig, call_descriptor, target);
5972 FinishCall(decoder, &sig, call_descriptor);
5973 }
5974 } else {
5975 // Inlining direct calls isn't speculative, but existence of the
5976 // feedback vector currently depends on this flag.
5977 if (FLAG_wasm_speculative_inlining) {
5978 LiftoffRegister vector = __ GetUnusedRegister(kGpReg, {});
5979 __ Fill(vector, liftoff::kFeedbackVectorOffset, kPointerKind);
5980 __ IncrementSmi(vector,
5981 wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(
5982 static_cast<int>(vector_slot)));
5983 }
5984 // A direct call within this module just gets the current instance.
5985 __ PrepareCall(&sig, call_descriptor);
5986 // Just encode the function index. This will be patched at instantiation.
5987 Address addr = static_cast<Address>(imm.index);
5988 if (tail_call) {
5989 DCHECK(descriptor_->CanTailCall(call_descriptor));
5990 __ PrepareTailCall(
5991 static_cast<int>(call_descriptor->ParameterSlotCount()),
5992 static_cast<int>(
5993 call_descriptor->GetStackParameterDelta(descriptor_)));
5994 __ TailCallNativeWasmCode(addr);
5995 } else {
5996 source_position_table_builder_.AddPosition(
5997 __ pc_offset(), SourcePosition(decoder->position()), true);
5998 __ CallNativeWasmCode(addr);
5999 FinishCall(decoder, &sig, call_descriptor);
6000 }
6001 }
6002 }
6003
CallIndirect(FullDecoder * decoder,const Value & index_val,const CallIndirectImmediate<validate> & imm,TailCall tail_call)6004 void CallIndirect(FullDecoder* decoder, const Value& index_val,
6005 const CallIndirectImmediate<validate>& imm,
6006 TailCall tail_call) {
6007 MostlySmallValueKindSig sig(compilation_zone_, imm.sig);
6008 for (ValueKind ret : sig.returns()) {
6009 if (!CheckSupportedType(decoder, ret, "return")) return;
6010 }
6011
6012 // Pop the index. We'll modify the register's contents later.
6013 Register index = __ PopToModifiableRegister().gp();
6014
6015 LiftoffRegList pinned = {index};
6016 // Get three temporary registers.
6017 Register table = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
6018 Register tmp_const = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
6019 Register scratch = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
6020 Register indirect_function_table = no_reg;
6021 if (imm.table_imm.index != 0) {
6022 Register indirect_function_tables =
6023 pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
6024 LOAD_TAGGED_PTR_INSTANCE_FIELD(indirect_function_tables,
6025 IndirectFunctionTables, pinned);
6026
6027 indirect_function_table = indirect_function_tables;
6028 __ LoadTaggedPointer(
6029 indirect_function_table, indirect_function_tables, no_reg,
6030 ObjectAccess::ElementOffsetInTaggedFixedArray(imm.table_imm.index),
6031 pinned);
6032 }
6033
6034 // Bounds check against the table size.
6035 Label* invalid_func_label =
6036 AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapTableOutOfBounds);
6037
6038 uint32_t canonical_sig_num =
6039 env_->module->canonicalized_type_ids[imm.sig_imm.index];
6040 DCHECK_GE(canonical_sig_num, 0);
6041 DCHECK_GE(kMaxInt, canonical_sig_num);
6042
6043 // Compare against table size stored in
6044 // {instance->indirect_function_table_size}.
6045 if (imm.table_imm.index == 0) {
6046 LOAD_INSTANCE_FIELD(tmp_const, IndirectFunctionTableSize, kUInt32Size,
6047 pinned);
6048 } else {
6049 __ Load(
6050 LiftoffRegister(tmp_const), indirect_function_table, no_reg,
6051 wasm::ObjectAccess::ToTagged(WasmIndirectFunctionTable::kSizeOffset),
6052 LoadType::kI32Load, pinned);
6053 }
6054 __ emit_cond_jump(kUnsignedGreaterEqual, invalid_func_label, kI32, index,
6055 tmp_const);
6056
6057 CODE_COMMENT("Check indirect call signature");
6058 // Load the signature from {instance->ift_sig_ids[key]}
6059 if (imm.table_imm.index == 0) {
6060 LOAD_INSTANCE_FIELD(table, IndirectFunctionTableSigIds,
6061 kSystemPointerSize, pinned);
6062 } else {
6063 __ Load(LiftoffRegister(table), indirect_function_table, no_reg,
6064 wasm::ObjectAccess::ToTagged(
6065 WasmIndirectFunctionTable::kSigIdsOffset),
6066 kPointerLoadType, pinned);
6067 }
6068 // Shift {index} by 2 (multiply by 4) to represent kInt32Size items.
6069 STATIC_ASSERT((1 << 2) == kInt32Size);
6070 __ emit_i32_shli(index, index, 2);
6071 __ Load(LiftoffRegister(scratch), table, index, 0, LoadType::kI32Load,
6072 pinned);
6073
6074 // Compare against expected signature.
6075 __ LoadConstant(LiftoffRegister(tmp_const), WasmValue(canonical_sig_num));
6076
6077 Label* sig_mismatch_label =
6078 AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapFuncSigMismatch);
6079 __ emit_cond_jump(kUnequal, sig_mismatch_label, kPointerKind, scratch,
6080 tmp_const);
6081
6082 // At this point {index} has already been multiplied by 4.
6083 CODE_COMMENT("Execute indirect call");
6084 if (kTaggedSize != kInt32Size) {
6085 DCHECK_EQ(kTaggedSize, kInt32Size * 2);
6086 // Multiply {index} by another 2 to represent kTaggedSize items.
6087 __ emit_i32_add(index, index, index);
6088 }
6089 // At this point {index} has already been multiplied by kTaggedSize.
6090
6091 // Load the instance from {instance->ift_instances[key]}
6092 if (imm.table_imm.index == 0) {
6093 LOAD_TAGGED_PTR_INSTANCE_FIELD(table, IndirectFunctionTableRefs, pinned);
6094 } else {
6095 __ LoadTaggedPointer(
6096 table, indirect_function_table, no_reg,
6097 wasm::ObjectAccess::ToTagged(WasmIndirectFunctionTable::kRefsOffset),
6098 pinned);
6099 }
6100 __ LoadTaggedPointer(tmp_const, table, index,
6101 ObjectAccess::ElementOffsetInTaggedFixedArray(0),
6102 pinned);
6103
6104 if (kTaggedSize != kSystemPointerSize) {
6105 DCHECK_EQ(kSystemPointerSize, kTaggedSize * 2);
6106 // Multiply {index} by another 2 to represent kSystemPointerSize items.
6107 __ emit_i32_add(index, index, index);
6108 }
6109 // At this point {index} has already been multiplied by kSystemPointerSize.
6110
6111 Register* explicit_instance = &tmp_const;
6112
6113 // Load the target from {instance->ift_targets[key]}
6114 if (imm.table_imm.index == 0) {
6115 LOAD_INSTANCE_FIELD(table, IndirectFunctionTableTargets,
6116 kSystemPointerSize, pinned);
6117 } else {
6118 __ Load(LiftoffRegister(table), indirect_function_table, no_reg,
6119 wasm::ObjectAccess::ToTagged(
6120 WasmIndirectFunctionTable::kTargetsOffset),
6121 kPointerLoadType, pinned);
6122 }
6123 __ Load(LiftoffRegister(scratch), table, index, 0, kPointerLoadType,
6124 pinned);
6125
6126 auto call_descriptor =
6127 compiler::GetWasmCallDescriptor(compilation_zone_, imm.sig);
6128 call_descriptor =
6129 GetLoweredCallDescriptor(compilation_zone_, call_descriptor);
6130
6131 Register target = scratch;
6132 __ PrepareCall(&sig, call_descriptor, &target, explicit_instance);
6133 if (tail_call) {
6134 __ PrepareTailCall(
6135 static_cast<int>(call_descriptor->ParameterSlotCount()),
6136 static_cast<int>(
6137 call_descriptor->GetStackParameterDelta(descriptor_)));
6138 __ TailCallIndirect(target);
6139 } else {
6140 source_position_table_builder_.AddPosition(
6141 __ pc_offset(), SourcePosition(decoder->position()), true);
6142 __ CallIndirect(&sig, call_descriptor, target);
6143
6144 FinishCall(decoder, &sig, call_descriptor);
6145 }
6146 }
6147
CallRef(FullDecoder * decoder,ValueType func_ref_type,const FunctionSig * type_sig,TailCall tail_call)6148 void CallRef(FullDecoder* decoder, ValueType func_ref_type,
6149 const FunctionSig* type_sig, TailCall tail_call) {
6150 MostlySmallValueKindSig sig(compilation_zone_, type_sig);
6151 for (ValueKind ret : sig.returns()) {
6152 if (!CheckSupportedType(decoder, ret, "return")) return;
6153 }
6154 compiler::CallDescriptor* call_descriptor =
6155 compiler::GetWasmCallDescriptor(compilation_zone_, type_sig);
6156 call_descriptor =
6157 GetLoweredCallDescriptor(compilation_zone_, call_descriptor);
6158
6159 Register target_reg = no_reg, instance_reg = no_reg;
6160
6161 if (FLAG_wasm_speculative_inlining) {
6162 ValueKind kIntPtrKind = kPointerKind;
6163
6164 LiftoffRegList pinned;
6165 LiftoffRegister vector = pinned.set(__ GetUnusedRegister(kGpReg, pinned));
6166 LiftoffAssembler::VarState funcref =
6167 __ cache_state()->stack_state.end()[-1];
6168 if (funcref.is_reg()) pinned.set(funcref.reg());
6169 __ Fill(vector, liftoff::kFeedbackVectorOffset, kPointerKind);
6170 LiftoffAssembler::VarState vector_var(kPointerKind, vector, 0);
6171 LiftoffRegister index = pinned.set(__ GetUnusedRegister(kGpReg, pinned));
6172 uintptr_t vector_slot = num_call_instructions_ * 2;
6173 {
6174 base::MutexGuard mutex_guard(&decoder->module_->type_feedback.mutex);
6175 decoder->module_->type_feedback.feedback_for_function[func_index_]
6176 .positions[decoder->position()] =
6177 static_cast<int>(num_call_instructions_);
6178 }
6179 num_call_instructions_++;
6180 __ LoadConstant(index, WasmValue::ForUintPtr(vector_slot));
6181 LiftoffAssembler::VarState index_var(kIntPtrKind, index, 0);
6182
6183 // CallRefIC(vector: FixedArray, index: intptr,
6184 // funcref: WasmInternalFunction)
6185 CallRuntimeStub(WasmCode::kCallRefIC,
6186 MakeSig::Returns(kPointerKind, kPointerKind)
6187 .Params(kPointerKind, kIntPtrKind, kPointerKind),
6188 {vector_var, index_var, funcref}, decoder->position());
6189
6190 __ cache_state()->stack_state.pop_back(1); // Drop funcref.
6191 target_reg = LiftoffRegister(kReturnRegister0).gp();
6192 instance_reg = LiftoffRegister(kReturnRegister1).gp();
6193
6194 } else { // FLAG_wasm_speculative_inlining
6195 // Non-feedback-collecting version.
6196 // Executing a write barrier needs temp registers; doing this on a
6197 // conditional branch confuses the LiftoffAssembler's register management.
6198 // Spill everything up front to work around that.
6199 __ SpillAllRegisters();
6200
6201 // We limit ourselves to four registers:
6202 // (1) func_data, initially reused for func_ref.
6203 // (2) instance, initially used as temp.
6204 // (3) target, initially used as temp.
6205 // (4) temp.
6206 LiftoffRegList pinned;
6207 LiftoffRegister func_ref = pinned.set(__ PopToModifiableRegister(pinned));
6208 MaybeEmitNullCheck(decoder, func_ref.gp(), pinned, func_ref_type);
6209 LiftoffRegister instance =
6210 pinned.set(__ GetUnusedRegister(kGpReg, pinned));
6211 LiftoffRegister target = pinned.set(__ GetUnusedRegister(kGpReg, pinned));
6212 LiftoffRegister temp = pinned.set(__ GetUnusedRegister(kGpReg, pinned));
6213
6214 // Load "ref" (instance or WasmApiFunctionRef) and target.
6215 __ LoadTaggedPointer(
6216 instance.gp(), func_ref.gp(), no_reg,
6217 wasm::ObjectAccess::ToTagged(WasmInternalFunction::kRefOffset),
6218 pinned);
6219
6220 #ifdef V8_SANDBOXED_EXTERNAL_POINTERS
6221 LOAD_INSTANCE_FIELD(temp.gp(), IsolateRoot, kSystemPointerSize, pinned);
6222 __ LoadExternalPointer(target.gp(), func_ref.gp(),
6223 WasmInternalFunction::kForeignAddressOffset,
6224 kForeignForeignAddressTag, temp.gp());
6225 #else
6226 __ Load(target, func_ref.gp(), no_reg,
6227 wasm::ObjectAccess::ToTagged(
6228 WasmInternalFunction::kForeignAddressOffset),
6229 kPointerLoadType, pinned);
6230 #endif
6231
6232 Label perform_call;
6233
6234 LiftoffRegister null_address = temp;
6235 __ LoadConstant(null_address, WasmValue::ForUintPtr(0));
6236 __ emit_cond_jump(kUnequal, &perform_call, kRef, target.gp(),
6237 null_address.gp());
6238 // The cached target can only be null for WasmJSFunctions.
6239 __ LoadTaggedPointer(
6240 target.gp(), func_ref.gp(), no_reg,
6241 wasm::ObjectAccess::ToTagged(WasmInternalFunction::kCodeOffset),
6242 pinned);
6243 #ifdef V8_EXTERNAL_CODE_SPACE
6244 __ LoadCodeDataContainerEntry(target.gp(), target.gp());
6245 #else
6246 __ emit_ptrsize_addi(target.gp(), target.gp(),
6247 wasm::ObjectAccess::ToTagged(Code::kHeaderSize));
6248 #endif
6249 // Fall through to {perform_call}.
6250
6251 __ bind(&perform_call);
6252 // Now the call target is in {target}, and the right instance object
6253 // is in {instance}.
6254 target_reg = target.gp();
6255 instance_reg = instance.gp();
6256 } // FLAG_wasm_speculative_inlining
6257
6258 __ PrepareCall(&sig, call_descriptor, &target_reg, &instance_reg);
6259 if (tail_call) {
6260 __ PrepareTailCall(
6261 static_cast<int>(call_descriptor->ParameterSlotCount()),
6262 static_cast<int>(
6263 call_descriptor->GetStackParameterDelta(descriptor_)));
6264 __ TailCallIndirect(target_reg);
6265 } else {
6266 source_position_table_builder_.AddPosition(
6267 __ pc_offset(), SourcePosition(decoder->position()), true);
6268 __ CallIndirect(&sig, call_descriptor, target_reg);
6269
6270 FinishCall(decoder, &sig, call_descriptor);
6271 }
6272 }
6273
LoadNullValue(Register null,LiftoffRegList pinned)6274 void LoadNullValue(Register null, LiftoffRegList pinned) {
6275 LOAD_INSTANCE_FIELD(null, IsolateRoot, kSystemPointerSize, pinned);
6276 __ LoadFullPointer(null, null,
6277 IsolateData::root_slot_offset(RootIndex::kNullValue));
6278 }
6279
LoadExceptionSymbol(Register dst,LiftoffRegList pinned,RootIndex root_index)6280 void LoadExceptionSymbol(Register dst, LiftoffRegList pinned,
6281 RootIndex root_index) {
6282 LOAD_INSTANCE_FIELD(dst, IsolateRoot, kSystemPointerSize, pinned);
6283 uint32_t offset_imm = IsolateData::root_slot_offset(root_index);
6284 __ LoadFullPointer(dst, dst, offset_imm);
6285 }
6286
MaybeEmitNullCheck(FullDecoder * decoder,Register object,LiftoffRegList pinned,ValueType type)6287 void MaybeEmitNullCheck(FullDecoder* decoder, Register object,
6288 LiftoffRegList pinned, ValueType type) {
6289 if (FLAG_experimental_wasm_skip_null_checks || !type.is_nullable()) return;
6290 Label* trap_label =
6291 AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapNullDereference);
6292 LiftoffRegister null = __ GetUnusedRegister(kGpReg, pinned);
6293 LoadNullValue(null.gp(), pinned);
6294 __ emit_cond_jump(LiftoffCondition::kEqual, trap_label, kOptRef, object,
6295 null.gp());
6296 }
6297
BoundsCheckArray(FullDecoder * decoder,LiftoffRegister array,LiftoffRegister index,LiftoffRegList pinned)6298 void BoundsCheckArray(FullDecoder* decoder, LiftoffRegister array,
6299 LiftoffRegister index, LiftoffRegList pinned) {
6300 if (V8_UNLIKELY(FLAG_experimental_wasm_skip_bounds_checks)) return;
6301 Label* trap_label =
6302 AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapArrayOutOfBounds);
6303 LiftoffRegister length = __ GetUnusedRegister(kGpReg, pinned);
6304 constexpr int kLengthOffset =
6305 wasm::ObjectAccess::ToTagged(WasmArray::kLengthOffset);
6306 __ Load(length, array.gp(), no_reg, kLengthOffset, LoadType::kI32Load,
6307 pinned);
6308 __ emit_cond_jump(LiftoffCondition::kUnsignedGreaterEqual, trap_label, kI32,
6309 index.gp(), length.gp());
6310 }
6311
StructFieldOffset(const StructType * struct_type,int field_index)6312 int StructFieldOffset(const StructType* struct_type, int field_index) {
6313 return wasm::ObjectAccess::ToTagged(WasmStruct::kHeaderSize +
6314 struct_type->field_offset(field_index));
6315 }
6316
LoadObjectField(LiftoffRegister dst,Register src,Register offset_reg,int offset,ValueKind kind,bool is_signed,LiftoffRegList pinned)6317 void LoadObjectField(LiftoffRegister dst, Register src, Register offset_reg,
6318 int offset, ValueKind kind, bool is_signed,
6319 LiftoffRegList pinned) {
6320 if (is_reference(kind)) {
6321 __ LoadTaggedPointer(dst.gp(), src, offset_reg, offset, pinned);
6322 } else {
6323 // Primitive kind.
6324 LoadType load_type = LoadType::ForValueKind(kind, is_signed);
6325 __ Load(dst, src, offset_reg, offset, load_type, pinned);
6326 }
6327 }
6328
StoreObjectField(Register obj,Register offset_reg,int offset,LiftoffRegister value,LiftoffRegList pinned,ValueKind kind)6329 void StoreObjectField(Register obj, Register offset_reg, int offset,
6330 LiftoffRegister value, LiftoffRegList pinned,
6331 ValueKind kind) {
6332 if (is_reference(kind)) {
6333 __ StoreTaggedPointer(obj, offset_reg, offset, value, pinned);
6334 } else {
6335 // Primitive kind.
6336 StoreType store_type = StoreType::ForValueKind(kind);
6337 __ Store(obj, offset_reg, offset, value, store_type, pinned);
6338 }
6339 }
6340
SetDefaultValue(LiftoffRegister reg,ValueKind kind,LiftoffRegList pinned)6341 void SetDefaultValue(LiftoffRegister reg, ValueKind kind,
6342 LiftoffRegList pinned) {
6343 DCHECK(is_defaultable(kind));
6344 switch (kind) {
6345 case kI8:
6346 case kI16:
6347 case kI32:
6348 return __ LoadConstant(reg, WasmValue(int32_t{0}));
6349 case kI64:
6350 return __ LoadConstant(reg, WasmValue(int64_t{0}));
6351 case kF32:
6352 return __ LoadConstant(reg, WasmValue(float{0.0}));
6353 case kF64:
6354 return __ LoadConstant(reg, WasmValue(double{0.0}));
6355 case kS128:
6356 DCHECK(CpuFeatures::SupportsWasmSimd128());
6357 return __ emit_s128_xor(reg, reg, reg);
6358 case kOptRef:
6359 return LoadNullValue(reg.gp(), pinned);
6360 case kRtt:
6361 case kVoid:
6362 case kBottom:
6363 case kRef:
6364 UNREACHABLE();
6365 }
6366 }
6367
6368 struct TypeCheckRegisters {
6369 LiftoffRegister obj_reg, map_reg, tmp_reg;
6370 };
6371
TypeCheckPrelude(const Value & obj,Label * no_match,LiftoffRegList pinned,Register opt_scratch)6372 TypeCheckRegisters TypeCheckPrelude(const Value& obj, Label* no_match,
6373 LiftoffRegList pinned,
6374 Register opt_scratch) {
6375 LiftoffRegister obj_reg = pinned.set(__ PopToRegister(pinned));
6376
6377 // Reserve all temporary registers up front, so that the cache state
6378 // tracking doesn't get confused by the following conditional jumps.
6379 LiftoffRegister map_reg =
6380 opt_scratch != no_reg
6381 ? LiftoffRegister(opt_scratch)
6382 : pinned.set(__ GetUnusedRegister(kGpReg, pinned));
6383 LiftoffRegister tmp_reg = pinned.set(__ GetUnusedRegister(kGpReg, pinned));
6384
6385 if (obj.type.is_nullable()) {
6386 LoadNullValue(map_reg.gp(), pinned);
6387 __ emit_cond_jump(kEqual, no_match, kOptRef, obj_reg.gp(), map_reg.gp());
6388 }
6389
6390 __ emit_smi_check(obj_reg.gp(), no_match, LiftoffAssembler::kJumpOnSmi);
6391
6392 __ LoadMap(map_reg.gp(), obj_reg.gp());
6393
6394 return {obj_reg, map_reg, tmp_reg};
6395 }
6396
EmitDataRefCheck(Register map,Label * not_data_ref,LiftoffRegister tmp,LiftoffRegList pinned)6397 void EmitDataRefCheck(Register map, Label* not_data_ref, LiftoffRegister tmp,
6398 LiftoffRegList pinned) {
6399 constexpr int kInstanceTypeOffset =
6400 wasm::ObjectAccess::ToTagged(Map::kInstanceTypeOffset);
6401 __ Load(tmp, map, no_reg, kInstanceTypeOffset, LoadType::kI32Load16U,
6402 pinned);
6403 // We're going to test a range of WasmObject instance types with a single
6404 // unsigned comparison.
6405 __ emit_i32_subi(tmp.gp(), tmp.gp(), FIRST_WASM_OBJECT_TYPE);
6406 __ emit_i32_cond_jumpi(kUnsignedGreaterThan, not_data_ref, tmp.gp(),
6407 LAST_WASM_OBJECT_TYPE - FIRST_WASM_OBJECT_TYPE);
6408 }
6409
MaybeOSR()6410 void MaybeOSR() {
6411 if (V8_UNLIKELY(for_debugging_)) {
6412 __ MaybeOSR();
6413 }
6414 }
6415
FinishCall(FullDecoder * decoder,ValueKindSig * sig,compiler::CallDescriptor * call_descriptor)6416 void FinishCall(FullDecoder* decoder, ValueKindSig* sig,
6417 compiler::CallDescriptor* call_descriptor) {
6418 DefineSafepoint();
6419 RegisterDebugSideTableEntry(decoder, DebugSideTableBuilder::kDidSpill);
6420 int pc_offset = __ pc_offset();
6421 MaybeOSR();
6422 EmitLandingPad(decoder, pc_offset);
6423 __ FinishCall(sig, call_descriptor);
6424 }
6425
CheckNan(LiftoffRegister src,LiftoffRegList pinned,ValueKind kind)6426 void CheckNan(LiftoffRegister src, LiftoffRegList pinned, ValueKind kind) {
6427 DCHECK(kind == ValueKind::kF32 || kind == ValueKind::kF64);
6428 auto nondeterminism_addr = __ GetUnusedRegister(kGpReg, pinned);
6429 __ LoadConstant(
6430 nondeterminism_addr,
6431 WasmValue::ForUintPtr(reinterpret_cast<uintptr_t>(nondeterminism_)));
6432 __ emit_set_if_nan(nondeterminism_addr.gp(), src.fp(), kind);
6433 }
6434
CheckS128Nan(LiftoffRegister dst,LiftoffRegList pinned,ValueKind lane_kind)6435 void CheckS128Nan(LiftoffRegister dst, LiftoffRegList pinned,
6436 ValueKind lane_kind) {
6437 RegClass rc = reg_class_for(kS128);
6438 LiftoffRegister tmp_gp = pinned.set(__ GetUnusedRegister(kGpReg, pinned));
6439 LiftoffRegister tmp_s128 = pinned.set(__ GetUnusedRegister(rc, pinned));
6440 LiftoffRegister nondeterminism_addr =
6441 pinned.set(__ GetUnusedRegister(kGpReg, pinned));
6442 __ LoadConstant(
6443 nondeterminism_addr,
6444 WasmValue::ForUintPtr(reinterpret_cast<uintptr_t>(nondeterminism_)));
6445 __ emit_s128_set_if_nan(nondeterminism_addr.gp(), dst, tmp_gp.gp(),
6446 tmp_s128, lane_kind);
6447 }
6448
has_outstanding_op() const6449 bool has_outstanding_op() const {
6450 return outstanding_op_ != kNoOutstandingOp;
6451 }
6452
test_and_reset_outstanding_op(WasmOpcode opcode)6453 bool test_and_reset_outstanding_op(WasmOpcode opcode) {
6454 DCHECK_NE(kNoOutstandingOp, opcode);
6455 if (outstanding_op_ != opcode) return false;
6456 outstanding_op_ = kNoOutstandingOp;
6457 return true;
6458 }
6459
TraceCacheState(FullDecoder * decoder) const6460 void TraceCacheState(FullDecoder* decoder) const {
6461 if (!FLAG_trace_liftoff) return;
6462 StdoutStream os;
6463 for (int control_depth = decoder->control_depth() - 1; control_depth >= -1;
6464 --control_depth) {
6465 auto* cache_state =
6466 control_depth == -1 ? __ cache_state()
6467 : &decoder->control_at(control_depth)
6468 ->label_state;
6469 os << PrintCollection(cache_state->stack_state);
6470 if (control_depth != -1) PrintF("; ");
6471 }
6472 os << "\n";
6473 }
6474
DefineSafepoint()6475 void DefineSafepoint() {
6476 auto safepoint = safepoint_table_builder_.DefineSafepoint(&asm_);
6477 __ cache_state()->DefineSafepoint(safepoint);
6478 }
6479
DefineSafepointWithCalleeSavedRegisters()6480 void DefineSafepointWithCalleeSavedRegisters() {
6481 auto safepoint = safepoint_table_builder_.DefineSafepoint(&asm_);
6482 __ cache_state()->DefineSafepointWithCalleeSavedRegisters(safepoint);
6483 }
6484
LoadInstanceIntoRegister(LiftoffRegList pinned,Register fallback)6485 Register LoadInstanceIntoRegister(LiftoffRegList pinned, Register fallback) {
6486 Register instance = __ cache_state()->cached_instance;
6487 if (instance == no_reg) {
6488 instance = __ cache_state()->TrySetCachedInstanceRegister(
6489 pinned | LiftoffRegList{fallback});
6490 if (instance == no_reg) instance = fallback;
6491 __ LoadInstanceFromFrame(instance);
6492 }
6493 return instance;
6494 }
6495
6496 static constexpr WasmOpcode kNoOutstandingOp = kExprUnreachable;
6497 static constexpr base::EnumSet<ValueKind> kUnconditionallySupported{
6498 // MVP:
6499 kI32, kI64, kF32, kF64,
6500 // Extern ref:
6501 kRef, kOptRef, kRtt, kI8, kI16};
6502
6503 LiftoffAssembler asm_;
6504
6505 // Used for merging code generation of subsequent operations (via look-ahead).
6506 // Set by the first opcode, reset by the second.
6507 WasmOpcode outstanding_op_ = kNoOutstandingOp;
6508
6509 // {supported_types_} is updated in {MaybeBailoutForUnsupportedType}.
6510 base::EnumSet<ValueKind> supported_types_ = kUnconditionallySupported;
6511 compiler::CallDescriptor* const descriptor_;
6512 CompilationEnv* const env_;
6513 DebugSideTableBuilder* const debug_sidetable_builder_;
6514 const ForDebugging for_debugging_;
6515 LiftoffBailoutReason bailout_reason_ = kSuccess;
6516 const int func_index_;
6517 ZoneVector<OutOfLineCode> out_of_line_code_;
6518 SourcePositionTableBuilder source_position_table_builder_;
6519 ZoneVector<trap_handler::ProtectedInstructionData> protected_instructions_;
6520 // Zone used to store information during compilation. The result will be
6521 // stored independently, such that this zone can die together with the
6522 // LiftoffCompiler after compilation.
6523 Zone* compilation_zone_;
6524 SafepointTableBuilder safepoint_table_builder_;
6525 // The pc offset of the instructions to reserve the stack frame. Needed to
6526 // patch the actually needed stack size in the end.
6527 uint32_t pc_offset_stack_frame_construction_ = 0;
6528 // For emitting breakpoint, we store a pointer to the position of the next
6529 // breakpoint, and a pointer after the list of breakpoints as end marker.
6530 // A single breakpoint at offset 0 indicates that we should prepare the
6531 // function for stepping by flooding it with breakpoints.
6532 const int* next_breakpoint_ptr_ = nullptr;
6533 const int* next_breakpoint_end_ = nullptr;
6534
6535 // Introduce a dead breakpoint to ensure that the calculation of the return
6536 // address in OSR is correct.
6537 int dead_breakpoint_ = 0;
6538
6539 // Remember whether the did function-entry break checks (for "hook on function
6540 // call" and "break on entry" a.k.a. instrumentation breakpoint). This happens
6541 // at the first breakable opcode in the function (if compiling for debugging).
6542 bool did_function_entry_break_checks_ = false;
6543
6544 struct HandlerInfo {
6545 MovableLabel handler;
6546 int pc_offset;
6547 };
6548
6549 ZoneVector<HandlerInfo> handlers_;
6550 int handler_table_offset_ = Assembler::kNoHandlerTable;
6551
6552 // Current number of exception refs on the stack.
6553 int num_exceptions_ = 0;
6554
6555 // Number of feedback-collecting call instructions encountered. While
6556 // compiling, also index of the next such instruction. Used for indexing type
6557 // feedback.
6558 uintptr_t num_call_instructions_ = 0;
6559
6560 int32_t* max_steps_;
6561 int32_t* nondeterminism_;
6562
6563 DISALLOW_IMPLICIT_CONSTRUCTORS(LiftoffCompiler);
6564 };
6565
6566 // static
6567 constexpr WasmOpcode LiftoffCompiler::kNoOutstandingOp;
6568 // static
6569 constexpr base::EnumSet<ValueKind> LiftoffCompiler::kUnconditionallySupported;
6570
6571 } // namespace
6572
ExecuteLiftoffCompilation(CompilationEnv * env,const FunctionBody & func_body,int func_index,ForDebugging for_debugging,const LiftoffOptions & compiler_options)6573 WasmCompilationResult ExecuteLiftoffCompilation(
6574 CompilationEnv* env, const FunctionBody& func_body, int func_index,
6575 ForDebugging for_debugging, const LiftoffOptions& compiler_options) {
6576 base::TimeTicks start_time;
6577 if (V8_UNLIKELY(FLAG_trace_wasm_compilation_times)) {
6578 start_time = base::TimeTicks::Now();
6579 }
6580 int func_body_size = static_cast<int>(func_body.end - func_body.start);
6581 TRACE_EVENT2(TRACE_DISABLED_BY_DEFAULT("v8.wasm.detailed"),
6582 "wasm.CompileBaseline", "funcIndex", func_index, "bodySize",
6583 func_body_size);
6584
6585 Zone zone(GetWasmEngine()->allocator(), "LiftoffCompilationZone");
6586 auto call_descriptor = compiler::GetWasmCallDescriptor(&zone, func_body.sig);
6587 size_t code_size_estimate =
6588 WasmCodeManager::EstimateLiftoffCodeSize(func_body_size);
6589 // Allocate the initial buffer a bit bigger to avoid reallocation during code
6590 // generation. Overflows when casting to int are fine, as we will allocate at
6591 // least {AssemblerBase::kMinimalBufferSize} anyway, so in the worst case we
6592 // have to grow more often.
6593 int initial_buffer_size = static_cast<int>(128 + code_size_estimate * 4 / 3);
6594 std::unique_ptr<DebugSideTableBuilder> debug_sidetable_builder;
6595 if (compiler_options.debug_sidetable) {
6596 debug_sidetable_builder = std::make_unique<DebugSideTableBuilder>();
6597 }
6598 DCHECK_IMPLIES(compiler_options.max_steps, for_debugging == kForDebugging);
6599 WasmFeatures unused_detected_features;
6600 WasmFullDecoder<Decoder::kBooleanValidation, LiftoffCompiler> decoder(
6601 &zone, env->module, env->enabled_features,
6602 compiler_options.detected_features ? compiler_options.detected_features
6603 : &unused_detected_features,
6604 func_body, call_descriptor, env, &zone,
6605 NewAssemblerBuffer(initial_buffer_size), debug_sidetable_builder.get(),
6606 for_debugging, func_index, compiler_options.breakpoints,
6607 compiler_options.dead_breakpoint, compiler_options.max_steps,
6608 compiler_options.nondeterminism);
6609 decoder.Decode();
6610 LiftoffCompiler* compiler = &decoder.interface();
6611 if (decoder.failed()) compiler->OnFirstError(&decoder);
6612
6613 if (auto* counters = compiler_options.counters) {
6614 // Check that the histogram for the bailout reasons has the correct size.
6615 DCHECK_EQ(0, counters->liftoff_bailout_reasons()->min());
6616 DCHECK_EQ(kNumBailoutReasons - 1,
6617 counters->liftoff_bailout_reasons()->max());
6618 DCHECK_EQ(kNumBailoutReasons,
6619 counters->liftoff_bailout_reasons()->num_buckets());
6620 // Register the bailout reason (can also be {kSuccess}).
6621 counters->liftoff_bailout_reasons()->AddSample(
6622 static_cast<int>(compiler->bailout_reason()));
6623 }
6624
6625 if (compiler->did_bailout()) return WasmCompilationResult{};
6626
6627 WasmCompilationResult result;
6628 compiler->GetCode(&result.code_desc);
6629 result.instr_buffer = compiler->ReleaseBuffer();
6630 result.source_positions = compiler->GetSourcePositionTable();
6631 result.protected_instructions_data = compiler->GetProtectedInstructionsData();
6632 result.frame_slot_count = compiler->GetTotalFrameSlotCountForGC();
6633 auto* lowered_call_desc = GetLoweredCallDescriptor(&zone, call_descriptor);
6634 result.tagged_parameter_slots = lowered_call_desc->GetTaggedParameterSlots();
6635 result.func_index = func_index;
6636 result.result_tier = ExecutionTier::kLiftoff;
6637 result.for_debugging = for_debugging;
6638 if (auto* debug_sidetable = compiler_options.debug_sidetable) {
6639 *debug_sidetable = debug_sidetable_builder->GenerateDebugSideTable();
6640 }
6641 result.feedback_vector_slots = compiler->GetFeedbackVectorSlots();
6642
6643 if (V8_UNLIKELY(FLAG_trace_wasm_compilation_times)) {
6644 base::TimeDelta time = base::TimeTicks::Now() - start_time;
6645 int codesize = result.code_desc.body_size();
6646 StdoutStream{} << "Compiled function "
6647 << reinterpret_cast<const void*>(env->module) << "#"
6648 << func_index << " using Liftoff, took "
6649 << time.InMilliseconds() << " ms and "
6650 << zone.allocation_size() << " bytes; bodysize "
6651 << func_body_size << " codesize " << codesize << std::endl;
6652 }
6653
6654 DCHECK(result.succeeded());
6655 return result;
6656 }
6657
GenerateLiftoffDebugSideTable(const WasmCode * code)6658 std::unique_ptr<DebugSideTable> GenerateLiftoffDebugSideTable(
6659 const WasmCode* code) {
6660 auto* native_module = code->native_module();
6661 auto* function = &native_module->module()->functions[code->index()];
6662 ModuleWireBytes wire_bytes{native_module->wire_bytes()};
6663 base::Vector<const byte> function_bytes =
6664 wire_bytes.GetFunctionBytes(function);
6665 CompilationEnv env = native_module->CreateCompilationEnv();
6666 FunctionBody func_body{function->sig, 0, function_bytes.begin(),
6667 function_bytes.end()};
6668
6669 Zone zone(GetWasmEngine()->allocator(), "LiftoffDebugSideTableZone");
6670 auto call_descriptor = compiler::GetWasmCallDescriptor(&zone, function->sig);
6671 DebugSideTableBuilder debug_sidetable_builder;
6672 WasmFeatures detected;
6673 constexpr int kSteppingBreakpoints[] = {0};
6674 DCHECK(code->for_debugging() == kForDebugging ||
6675 code->for_debugging() == kForStepping);
6676 base::Vector<const int> breakpoints =
6677 code->for_debugging() == kForStepping
6678 ? base::ArrayVector(kSteppingBreakpoints)
6679 : base::Vector<const int>{};
6680 WasmFullDecoder<Decoder::kBooleanValidation, LiftoffCompiler> decoder(
6681 &zone, native_module->module(), env.enabled_features, &detected,
6682 func_body, call_descriptor, &env, &zone,
6683 NewAssemblerBuffer(AssemblerBase::kDefaultBufferSize),
6684 &debug_sidetable_builder, code->for_debugging(), code->index(),
6685 breakpoints);
6686 decoder.Decode();
6687 DCHECK(decoder.ok());
6688 DCHECK(!decoder.interface().did_bailout());
6689 return debug_sidetable_builder.GenerateDebugSideTable();
6690 }
6691
6692 #undef __
6693 #undef TRACE
6694 #undef WASM_INSTANCE_OBJECT_FIELD_OFFSET
6695 #undef WASM_INSTANCE_OBJECT_FIELD_SIZE
6696 #undef LOAD_INSTANCE_FIELD
6697 #undef LOAD_TAGGED_PTR_INSTANCE_FIELD
6698 #undef CODE_COMMENT
6699
6700 } // namespace wasm
6701 } // namespace internal
6702 } // namespace v8
6703