• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // Copyright 2017 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #include "src/wasm/baseline/liftoff-compiler.h"
6 
7 #include "src/base/enum-set.h"
8 #include "src/base/optional.h"
9 #include "src/base/platform/wrappers.h"
10 #include "src/codegen/assembler-inl.h"
11 // TODO(clemensb): Remove dependences on compiler stuff.
12 #include "src/codegen/external-reference.h"
13 #include "src/codegen/interface-descriptors-inl.h"
14 #include "src/codegen/machine-type.h"
15 #include "src/codegen/macro-assembler-inl.h"
16 #include "src/compiler/linkage.h"
17 #include "src/compiler/wasm-compiler.h"
18 #include "src/logging/counters.h"
19 #include "src/logging/log.h"
20 #include "src/objects/smi.h"
21 #include "src/tracing/trace-event.h"
22 #include "src/utils/ostreams.h"
23 #include "src/utils/utils.h"
24 #include "src/wasm/baseline/liftoff-assembler.h"
25 #include "src/wasm/baseline/liftoff-register.h"
26 #include "src/wasm/function-body-decoder-impl.h"
27 #include "src/wasm/function-compiler.h"
28 #include "src/wasm/memory-tracing.h"
29 #include "src/wasm/object-access.h"
30 #include "src/wasm/simd-shuffle.h"
31 #include "src/wasm/wasm-debug.h"
32 #include "src/wasm/wasm-engine.h"
33 #include "src/wasm/wasm-linkage.h"
34 #include "src/wasm/wasm-objects.h"
35 #include "src/wasm/wasm-opcodes-inl.h"
36 
37 namespace v8 {
38 namespace internal {
39 namespace wasm {
40 
41 constexpr auto kRegister = LiftoffAssembler::VarState::kRegister;
42 constexpr auto kIntConst = LiftoffAssembler::VarState::kIntConst;
43 constexpr auto kStack = LiftoffAssembler::VarState::kStack;
44 
45 namespace {
46 
47 #define __ asm_.
48 
49 #define TRACE(...)                                            \
50   do {                                                        \
51     if (FLAG_trace_liftoff) PrintF("[liftoff] " __VA_ARGS__); \
52   } while (false)
53 
54 #define WASM_INSTANCE_OBJECT_FIELD_OFFSET(name) \
55   ObjectAccess::ToTagged(WasmInstanceObject::k##name##Offset)
56 
57 template <int expected_size, int actual_size>
58 struct assert_field_size {
59   static_assert(expected_size == actual_size,
60                 "field in WasmInstance does not have the expected size");
61   static constexpr int size = actual_size;
62 };
63 
64 #define WASM_INSTANCE_OBJECT_FIELD_SIZE(name) \
65   FIELD_SIZE(WasmInstanceObject::k##name##Offset)
66 
67 #define LOAD_INSTANCE_FIELD(dst, name, load_size, pinned)                      \
68   __ LoadFromInstance(dst, LoadInstanceIntoRegister(pinned, dst),              \
69                       WASM_INSTANCE_OBJECT_FIELD_OFFSET(name),                 \
70                       assert_field_size<WASM_INSTANCE_OBJECT_FIELD_SIZE(name), \
71                                         load_size>::size);
72 
73 #define LOAD_TAGGED_PTR_INSTANCE_FIELD(dst, name, pinned)                      \
74   static_assert(WASM_INSTANCE_OBJECT_FIELD_SIZE(name) == kTaggedSize,          \
75                 "field in WasmInstance does not have the expected size");      \
76   __ LoadTaggedPointerFromInstance(dst, LoadInstanceIntoRegister(pinned, dst), \
77                                    WASM_INSTANCE_OBJECT_FIELD_OFFSET(name));
78 
79 #ifdef V8_CODE_COMMENTS
80 #define CODE_COMMENT(str)  \
81   do {                     \
82     __ RecordComment(str); \
83   } while (false)
84 #else
85 #define CODE_COMMENT(str) ((void)0)
86 #endif
87 
88 constexpr LoadType::LoadTypeValue kPointerLoadType =
89     kSystemPointerSize == 8 ? LoadType::kI64Load : LoadType::kI32Load;
90 
91 constexpr ValueKind kPointerKind = LiftoffAssembler::kPointerKind;
92 constexpr ValueKind kSmiKind = LiftoffAssembler::kSmiKind;
93 constexpr ValueKind kTaggedKind = LiftoffAssembler::kTaggedKind;
94 
95 // Used to construct fixed-size signatures: MakeSig::Returns(...).Params(...);
96 using MakeSig = FixedSizeSignature<ValueKind>;
97 
98 #if V8_TARGET_ARCH_ARM64
99 // On ARM64, the Assembler keeps track of pointers to Labels to resolve
100 // branches to distant targets. Moving labels would confuse the Assembler,
101 // thus store the label on the heap and keep a unique_ptr.
102 class MovableLabel {
103  public:
104   MOVE_ONLY_NO_DEFAULT_CONSTRUCTOR(MovableLabel);
MovableLabel()105   MovableLabel() : label_(new Label()) {}
106 
get()107   Label* get() { return label_.get(); }
108 
109  private:
110   std::unique_ptr<Label> label_;
111 };
112 #else
113 // On all other platforms, just store the Label directly.
114 class MovableLabel {
115  public:
116   MOVE_ONLY_WITH_DEFAULT_CONSTRUCTORS(MovableLabel);
117 
get()118   Label* get() { return &label_; }
119 
120  private:
121   Label label_;
122 };
123 #endif
124 
GetLoweredCallDescriptor(Zone * zone,compiler::CallDescriptor * call_desc)125 compiler::CallDescriptor* GetLoweredCallDescriptor(
126     Zone* zone, compiler::CallDescriptor* call_desc) {
127   return kSystemPointerSize == 4
128              ? compiler::GetI32WasmCallDescriptor(zone, call_desc)
129              : call_desc;
130 }
131 
GetGpParamRegisters()132 constexpr LiftoffRegList GetGpParamRegisters() {
133   LiftoffRegList registers;
134   for (auto reg : kGpParamRegisters) registers.set(reg);
135   return registers;
136 }
137 
GetCompareCondition(WasmOpcode opcode)138 constexpr LiftoffCondition GetCompareCondition(WasmOpcode opcode) {
139   switch (opcode) {
140     case kExprI32Eq:
141       return kEqual;
142     case kExprI32Ne:
143       return kUnequal;
144     case kExprI32LtS:
145       return kSignedLessThan;
146     case kExprI32LtU:
147       return kUnsignedLessThan;
148     case kExprI32GtS:
149       return kSignedGreaterThan;
150     case kExprI32GtU:
151       return kUnsignedGreaterThan;
152     case kExprI32LeS:
153       return kSignedLessEqual;
154     case kExprI32LeU:
155       return kUnsignedLessEqual;
156     case kExprI32GeS:
157       return kSignedGreaterEqual;
158     case kExprI32GeU:
159       return kUnsignedGreaterEqual;
160     default:
161       UNREACHABLE();
162   }
163 }
164 
165 // Builds a {DebugSideTable}.
166 class DebugSideTableBuilder {
167   using Entry = DebugSideTable::Entry;
168   using Value = Entry::Value;
169 
170  public:
171   enum AssumeSpilling {
172     // All register values will be spilled before the pc covered by the debug
173     // side table entry. Register slots will be marked as stack slots in the
174     // generated debug side table entry.
175     kAssumeSpilling,
176     // Register slots will be written out as they are.
177     kAllowRegisters,
178     // Register slots cannot appear since we already spilled.
179     kDidSpill
180   };
181 
182   class EntryBuilder {
183    public:
EntryBuilder(int pc_offset,int stack_height,std::vector<Value> changed_values)184     explicit EntryBuilder(int pc_offset, int stack_height,
185                           std::vector<Value> changed_values)
186         : pc_offset_(pc_offset),
187           stack_height_(stack_height),
188           changed_values_(std::move(changed_values)) {}
189 
ToTableEntry()190     Entry ToTableEntry() {
191       return Entry{pc_offset_, stack_height_, std::move(changed_values_)};
192     }
193 
MinimizeBasedOnPreviousStack(const std::vector<Value> & last_values)194     void MinimizeBasedOnPreviousStack(const std::vector<Value>& last_values) {
195       auto dst = changed_values_.begin();
196       auto end = changed_values_.end();
197       for (auto src = dst; src != end; ++src) {
198         if (src->index < static_cast<int>(last_values.size()) &&
199             *src == last_values[src->index]) {
200           continue;
201         }
202         if (dst != src) *dst = *src;
203         ++dst;
204       }
205       changed_values_.erase(dst, end);
206     }
207 
pc_offset() const208     int pc_offset() const { return pc_offset_; }
set_pc_offset(int new_pc_offset)209     void set_pc_offset(int new_pc_offset) { pc_offset_ = new_pc_offset; }
210 
211    private:
212     int pc_offset_;
213     int stack_height_;
214     std::vector<Value> changed_values_;
215   };
216 
217   // Adds a new entry in regular code.
NewEntry(int pc_offset,base::Vector<DebugSideTable::Entry::Value> values)218   void NewEntry(int pc_offset,
219                 base::Vector<DebugSideTable::Entry::Value> values) {
220     entries_.emplace_back(pc_offset, static_cast<int>(values.size()),
221                           GetChangedStackValues(last_values_, values));
222   }
223 
224   // Adds a new entry for OOL code, and returns a pointer to a builder for
225   // modifying that entry.
NewOOLEntry(base::Vector<DebugSideTable::Entry::Value> values)226   EntryBuilder* NewOOLEntry(base::Vector<DebugSideTable::Entry::Value> values) {
227     constexpr int kNoPcOffsetYet = -1;
228     ool_entries_.emplace_back(kNoPcOffsetYet, static_cast<int>(values.size()),
229                               GetChangedStackValues(last_ool_values_, values));
230     return &ool_entries_.back();
231   }
232 
SetNumLocals(int num_locals)233   void SetNumLocals(int num_locals) {
234     DCHECK_EQ(-1, num_locals_);
235     DCHECK_LE(0, num_locals);
236     num_locals_ = num_locals;
237   }
238 
GenerateDebugSideTable()239   std::unique_ptr<DebugSideTable> GenerateDebugSideTable() {
240     DCHECK_LE(0, num_locals_);
241 
242     // Connect {entries_} and {ool_entries_} by removing redundant stack
243     // information from the first {ool_entries_} entry (based on
244     // {last_values_}).
245     if (!entries_.empty() && !ool_entries_.empty()) {
246       ool_entries_.front().MinimizeBasedOnPreviousStack(last_values_);
247     }
248 
249     std::vector<Entry> entries;
250     entries.reserve(entries_.size() + ool_entries_.size());
251     for (auto& entry : entries_) entries.push_back(entry.ToTableEntry());
252     for (auto& entry : ool_entries_) entries.push_back(entry.ToTableEntry());
253     DCHECK(std::is_sorted(
254         entries.begin(), entries.end(),
255         [](Entry& a, Entry& b) { return a.pc_offset() < b.pc_offset(); }));
256     return std::make_unique<DebugSideTable>(num_locals_, std::move(entries));
257   }
258 
259  private:
GetChangedStackValues(std::vector<Value> & last_values,base::Vector<DebugSideTable::Entry::Value> values)260   static std::vector<Value> GetChangedStackValues(
261       std::vector<Value>& last_values,
262       base::Vector<DebugSideTable::Entry::Value> values) {
263     std::vector<Value> changed_values;
264     int old_stack_size = static_cast<int>(last_values.size());
265     last_values.resize(values.size());
266 
267     int index = 0;
268     for (const auto& value : values) {
269       if (index >= old_stack_size || last_values[index] != value) {
270         changed_values.push_back(value);
271         last_values[index] = value;
272       }
273       ++index;
274     }
275     return changed_values;
276   }
277 
278   int num_locals_ = -1;
279   // Keep a snapshot of the stack of the last entry, to generate a delta to the
280   // next entry.
281   std::vector<Value> last_values_;
282   std::vector<EntryBuilder> entries_;
283   // Keep OOL code entries separate so we can do proper delta-encoding (more
284   // entries might be added between the existing {entries_} and the
285   // {ool_entries_}). Store the entries in a list so the pointer is not
286   // invalidated by adding more entries.
287   std::vector<Value> last_ool_values_;
288   std::list<EntryBuilder> ool_entries_;
289 };
290 
CheckBailoutAllowed(LiftoffBailoutReason reason,const char * detail,const CompilationEnv * env)291 void CheckBailoutAllowed(LiftoffBailoutReason reason, const char* detail,
292                          const CompilationEnv* env) {
293   // Decode errors are ok.
294   if (reason == kDecodeError) return;
295 
296   // --liftoff-only ensures that tests actually exercise the Liftoff path
297   // without bailing out. We also fail for missing CPU support, to avoid
298   // running any TurboFan code under --liftoff-only.
299   if (FLAG_liftoff_only) {
300     FATAL("--liftoff-only: treating bailout as fatal error. Cause: %s", detail);
301   }
302 
303   // Missing CPU features are generally OK, except with --liftoff-only.
304   if (reason == kMissingCPUFeature) return;
305 
306   // If --enable-testing-opcode-in-wasm is set, we are expected to bailout with
307   // "testing opcode".
308   if (FLAG_enable_testing_opcode_in_wasm &&
309       strcmp(detail, "testing opcode") == 0) {
310     return;
311   }
312 
313   // Some externally maintained architectures don't fully implement Liftoff yet.
314 #if V8_TARGET_ARCH_MIPS || V8_TARGET_ARCH_MIPS64 || V8_TARGET_ARCH_S390X || \
315     V8_TARGET_ARCH_PPC || V8_TARGET_ARCH_PPC64 || V8_TARGET_ARCH_LOONG64
316   return;
317 #endif
318 
319 #if V8_TARGET_ARCH_ARM
320   // Allow bailout for missing ARMv7 support.
321   if (!CpuFeatures::IsSupported(ARMv7) && reason == kUnsupportedArchitecture) {
322     return;
323   }
324 #endif
325 
326 #define LIST_FEATURE(name, ...) kFeature_##name,
327   constexpr WasmFeatures kExperimentalFeatures{
328       FOREACH_WASM_EXPERIMENTAL_FEATURE_FLAG(LIST_FEATURE)};
329 #undef LIST_FEATURE
330 
331   // Bailout is allowed if any experimental feature is enabled.
332   if (env->enabled_features.contains_any(kExperimentalFeatures)) return;
333 
334   // Otherwise, bailout is not allowed.
335   FATAL("Liftoff bailout should not happen. Cause: %s\n", detail);
336 }
337 
338 class LiftoffCompiler {
339  public:
340   // TODO(clemensb): Make this a template parameter.
341   static constexpr Decoder::ValidateFlag validate = Decoder::kBooleanValidation;
342 
343   using Value = ValueBase<validate>;
344 
345   struct ElseState {
346     MovableLabel label;
347     LiftoffAssembler::CacheState state;
348   };
349 
350   struct TryInfo {
351     TryInfo() = default;
352     LiftoffAssembler::CacheState catch_state;
353     Label catch_label;
354     bool catch_reached = false;
355     bool in_handler = false;
356   };
357 
358   struct Control : public ControlBase<Value, validate> {
359     std::unique_ptr<ElseState> else_state;
360     LiftoffAssembler::CacheState label_state;
361     MovableLabel label;
362     std::unique_ptr<TryInfo> try_info;
363     // Number of exceptions on the stack below this control.
364     int num_exceptions = 0;
365 
366     MOVE_ONLY_NO_DEFAULT_CONSTRUCTOR(Control);
367 
368     template <typename... Args>
Controlv8::internal::wasm::__anon3bd82aa60111::LiftoffCompiler::Control369     explicit Control(Args&&... args) V8_NOEXCEPT
370         : ControlBase(std::forward<Args>(args)...) {}
371   };
372 
373   using FullDecoder = WasmFullDecoder<validate, LiftoffCompiler>;
374   using ValueKindSig = LiftoffAssembler::ValueKindSig;
375 
376   class MostlySmallValueKindSig : public Signature<ValueKind> {
377    public:
MostlySmallValueKindSig(Zone * zone,const FunctionSig * sig)378     MostlySmallValueKindSig(Zone* zone, const FunctionSig* sig)
379         : Signature<ValueKind>(sig->return_count(), sig->parameter_count(),
380                                MakeKinds(inline_storage_, zone, sig)) {}
381 
382    private:
383     static constexpr size_t kInlineStorage = 8;
384 
MakeKinds(ValueKind * storage,Zone * zone,const FunctionSig * sig)385     static ValueKind* MakeKinds(ValueKind* storage, Zone* zone,
386                                 const FunctionSig* sig) {
387       const size_t size = sig->parameter_count() + sig->return_count();
388       if (V8_UNLIKELY(size > kInlineStorage)) {
389         storage = zone->NewArray<ValueKind>(size);
390       }
391       std::transform(sig->all().begin(), sig->all().end(), storage,
392                      [](ValueType type) { return type.kind(); });
393       return storage;
394     }
395 
396     ValueKind inline_storage_[kInlineStorage];
397   };
398 
399   // For debugging, we need to spill registers before a trap or a stack check to
400   // be able to inspect them.
401   struct SpilledRegistersForInspection : public ZoneObject {
402     struct Entry {
403       int offset;
404       LiftoffRegister reg;
405       ValueKind kind;
406     };
407     ZoneVector<Entry> entries;
408 
SpilledRegistersForInspectionv8::internal::wasm::__anon3bd82aa60111::LiftoffCompiler::SpilledRegistersForInspection409     explicit SpilledRegistersForInspection(Zone* zone) : entries(zone) {}
410   };
411 
412   struct OutOfLineSafepointInfo {
413     ZoneVector<int> slots;
414     LiftoffRegList spills;
415 
OutOfLineSafepointInfov8::internal::wasm::__anon3bd82aa60111::LiftoffCompiler::OutOfLineSafepointInfo416     explicit OutOfLineSafepointInfo(Zone* zone) : slots(zone) {}
417   };
418 
419   struct OutOfLineCode {
420     MovableLabel label;
421     MovableLabel continuation;
422     WasmCode::RuntimeStubId stub;
423     WasmCodePosition position;
424     LiftoffRegList regs_to_save;
425     Register cached_instance;
426     OutOfLineSafepointInfo* safepoint_info;
427     uint32_t pc;  // for trap handler.
428     // These two pointers will only be used for debug code:
429     SpilledRegistersForInspection* spilled_registers;
430     DebugSideTableBuilder::EntryBuilder* debug_sidetable_entry_builder;
431 
432     // Named constructors:
Trapv8::internal::wasm::__anon3bd82aa60111::LiftoffCompiler::OutOfLineCode433     static OutOfLineCode Trap(
434         WasmCode::RuntimeStubId s, WasmCodePosition pos,
435         SpilledRegistersForInspection* spilled_registers,
436         OutOfLineSafepointInfo* safepoint_info, uint32_t pc,
437         DebugSideTableBuilder::EntryBuilder* debug_sidetable_entry_builder) {
438       DCHECK_LT(0, pos);
439       return {
440           {},                            // label
441           {},                            // continuation
442           s,                             // stub
443           pos,                           // position
444           {},                            // regs_to_save
445           no_reg,                        // cached_instance
446           safepoint_info,                // safepoint_info
447           pc,                            // pc
448           spilled_registers,             // spilled_registers
449           debug_sidetable_entry_builder  // debug_side_table_entry_builder
450       };
451     }
StackCheckv8::internal::wasm::__anon3bd82aa60111::LiftoffCompiler::OutOfLineCode452     static OutOfLineCode StackCheck(
453         WasmCodePosition pos, LiftoffRegList regs_to_save,
454         Register cached_instance, SpilledRegistersForInspection* spilled_regs,
455         OutOfLineSafepointInfo* safepoint_info,
456         DebugSideTableBuilder::EntryBuilder* debug_sidetable_entry_builder) {
457       return {
458           {},                            // label
459           {},                            // continuation
460           WasmCode::kWasmStackGuard,     // stub
461           pos,                           // position
462           regs_to_save,                  // regs_to_save
463           cached_instance,               // cached_instance
464           safepoint_info,                // safepoint_info
465           0,                             // pc
466           spilled_regs,                  // spilled_registers
467           debug_sidetable_entry_builder  // debug_side_table_entry_builder
468       };
469     }
TierupCheckv8::internal::wasm::__anon3bd82aa60111::LiftoffCompiler::OutOfLineCode470     static OutOfLineCode TierupCheck(
471         WasmCodePosition pos, LiftoffRegList regs_to_save,
472         Register cached_instance, SpilledRegistersForInspection* spilled_regs,
473         OutOfLineSafepointInfo* safepoint_info,
474         DebugSideTableBuilder::EntryBuilder* debug_sidetable_entry_builder) {
475       return {
476           {},                            // label
477           {},                            // continuation,
478           WasmCode::kWasmTriggerTierUp,  // stub
479           pos,                           // position
480           regs_to_save,                  // regs_to_save
481           cached_instance,               // cached_instance
482           safepoint_info,                // safepoint_info
483           0,                             // pc
484           spilled_regs,                  // spilled_registers
485           debug_sidetable_entry_builder  // debug_side_table_entry_builder
486       };
487     }
488   };
489 
LiftoffCompiler(compiler::CallDescriptor * call_descriptor,CompilationEnv * env,Zone * compilation_zone,std::unique_ptr<AssemblerBuffer> buffer,DebugSideTableBuilder * debug_sidetable_builder,ForDebugging for_debugging,int func_index,base::Vector<const int> breakpoints={},int dead_breakpoint=0,int32_t * max_steps=nullptr,int32_t * nondeterminism=nullptr)490   LiftoffCompiler(compiler::CallDescriptor* call_descriptor,
491                   CompilationEnv* env, Zone* compilation_zone,
492                   std::unique_ptr<AssemblerBuffer> buffer,
493                   DebugSideTableBuilder* debug_sidetable_builder,
494                   ForDebugging for_debugging, int func_index,
495                   base::Vector<const int> breakpoints = {},
496                   int dead_breakpoint = 0, int32_t* max_steps = nullptr,
497                   int32_t* nondeterminism = nullptr)
498       : asm_(std::move(buffer)),
499         descriptor_(
500             GetLoweredCallDescriptor(compilation_zone, call_descriptor)),
501         env_(env),
502         debug_sidetable_builder_(debug_sidetable_builder),
503         for_debugging_(for_debugging),
504         func_index_(func_index),
505         out_of_line_code_(compilation_zone),
506         source_position_table_builder_(compilation_zone),
507         protected_instructions_(compilation_zone),
508         compilation_zone_(compilation_zone),
509         safepoint_table_builder_(compilation_zone_),
510         next_breakpoint_ptr_(breakpoints.begin()),
511         next_breakpoint_end_(breakpoints.end()),
512         dead_breakpoint_(dead_breakpoint),
513         handlers_(compilation_zone),
514         max_steps_(max_steps),
515         nondeterminism_(nondeterminism) {
516     if (breakpoints.empty()) {
517       next_breakpoint_ptr_ = next_breakpoint_end_ = nullptr;
518     }
519   }
520 
did_bailout() const521   bool did_bailout() const { return bailout_reason_ != kSuccess; }
bailout_reason() const522   LiftoffBailoutReason bailout_reason() const { return bailout_reason_; }
523 
GetCode(CodeDesc * desc)524   void GetCode(CodeDesc* desc) {
525     asm_.GetCode(nullptr, desc, &safepoint_table_builder_,
526                  handler_table_offset_);
527   }
528 
ReleaseBuffer()529   std::unique_ptr<AssemblerBuffer> ReleaseBuffer() {
530     return asm_.ReleaseBuffer();
531   }
532 
GetSourcePositionTable()533   base::OwnedVector<uint8_t> GetSourcePositionTable() {
534     return source_position_table_builder_.ToSourcePositionTableVector();
535   }
536 
GetProtectedInstructionsData() const537   base::OwnedVector<uint8_t> GetProtectedInstructionsData() const {
538     return base::OwnedVector<uint8_t>::Of(base::Vector<const uint8_t>::cast(
539         base::VectorOf(protected_instructions_)));
540   }
541 
GetTotalFrameSlotCountForGC() const542   uint32_t GetTotalFrameSlotCountForGC() const {
543     return __ GetTotalFrameSlotCountForGC();
544   }
545 
GetFeedbackVectorSlots() const546   int GetFeedbackVectorSlots() const {
547     // The number of instructions is capped by max function size.
548     STATIC_ASSERT(kV8MaxWasmFunctionSize < std::numeric_limits<int>::max());
549     return static_cast<int>(num_call_instructions_) * 2;
550   }
551 
unsupported(FullDecoder * decoder,LiftoffBailoutReason reason,const char * detail)552   void unsupported(FullDecoder* decoder, LiftoffBailoutReason reason,
553                    const char* detail) {
554     DCHECK_NE(kSuccess, reason);
555     if (did_bailout()) return;
556     bailout_reason_ = reason;
557     TRACE("unsupported: %s\n", detail);
558     decoder->errorf(decoder->pc_offset(), "unsupported liftoff operation: %s",
559                     detail);
560     UnuseLabels(decoder);
561     CheckBailoutAllowed(reason, detail, env_);
562   }
563 
DidAssemblerBailout(FullDecoder * decoder)564   bool DidAssemblerBailout(FullDecoder* decoder) {
565     if (decoder->failed() || !__ did_bailout()) return false;
566     unsupported(decoder, __ bailout_reason(), __ bailout_detail());
567     return true;
568   }
569 
CheckSupportedType(FullDecoder * decoder,ValueKind kind,const char * context)570   V8_INLINE bool CheckSupportedType(FullDecoder* decoder, ValueKind kind,
571                                     const char* context) {
572     if (V8_LIKELY(supported_types_.contains(kind))) return true;
573     return MaybeBailoutForUnsupportedType(decoder, kind, context);
574   }
575 
MaybeBailoutForUnsupportedType(FullDecoder * decoder,ValueKind kind,const char * context)576   V8_NOINLINE bool MaybeBailoutForUnsupportedType(FullDecoder* decoder,
577                                                   ValueKind kind,
578                                                   const char* context) {
579     DCHECK(!supported_types_.contains(kind));
580 
581     // Lazily update {supported_types_}; then check again.
582     if (CpuFeatures::SupportsWasmSimd128()) supported_types_.Add(kS128);
583     if (supported_types_.contains(kind)) return true;
584 
585     LiftoffBailoutReason bailout_reason;
586     switch (kind) {
587       case kS128:
588         bailout_reason = kMissingCPUFeature;
589         break;
590       case kRef:
591       case kOptRef:
592       case kRtt:
593       case kI8:
594       case kI16:
595         bailout_reason = kGC;
596         break;
597       default:
598         UNREACHABLE();
599     }
600     base::EmbeddedVector<char, 128> buffer;
601     SNPrintF(buffer, "%s %s", name(kind), context);
602     unsupported(decoder, bailout_reason, buffer.begin());
603     return false;
604   }
605 
UnuseLabels(FullDecoder * decoder)606   void UnuseLabels(FullDecoder* decoder) {
607 #ifdef DEBUG
608     auto Unuse = [](Label* label) {
609       label->Unuse();
610       label->UnuseNear();
611     };
612     // Unuse all labels now, otherwise their destructor will fire a DCHECK error
613     // if they where referenced before.
614     uint32_t control_depth = decoder ? decoder->control_depth() : 0;
615     for (uint32_t i = 0; i < control_depth; ++i) {
616       Control* c = decoder->control_at(i);
617       Unuse(c->label.get());
618       if (c->else_state) Unuse(c->else_state->label.get());
619       if (c->try_info != nullptr) Unuse(&c->try_info->catch_label);
620     }
621     for (auto& ool : out_of_line_code_) Unuse(ool.label.get());
622 #endif
623   }
624 
StartFunction(FullDecoder * decoder)625   void StartFunction(FullDecoder* decoder) {
626     if (FLAG_trace_liftoff && !FLAG_trace_wasm_decoder) {
627       StdoutStream{} << "hint: add --trace-wasm-decoder to also see the wasm "
628                         "instructions being decoded\n";
629     }
630     int num_locals = decoder->num_locals();
631     __ set_num_locals(num_locals);
632     for (int i = 0; i < num_locals; ++i) {
633       ValueKind kind = decoder->local_type(i).kind();
634       __ set_local_kind(i, kind);
635     }
636   }
637 
RegsUnusedByParams()638   constexpr static LiftoffRegList RegsUnusedByParams() {
639     LiftoffRegList regs = kGpCacheRegList;
640     for (auto reg : kGpParamRegisters) {
641       regs.clear(reg);
642     }
643     return regs;
644   }
645 
646   // Returns the number of inputs processed (1 or 2).
ProcessParameter(ValueKind kind,uint32_t input_idx)647   uint32_t ProcessParameter(ValueKind kind, uint32_t input_idx) {
648     const bool needs_pair = needs_gp_reg_pair(kind);
649     const ValueKind reg_kind = needs_pair ? kI32 : kind;
650     const RegClass rc = reg_class_for(reg_kind);
651 
652     auto LoadToReg = [this, reg_kind, rc](compiler::LinkageLocation location,
653                                           LiftoffRegList pinned) {
654       if (location.IsRegister()) {
655         DCHECK(!location.IsAnyRegister());
656         return LiftoffRegister::from_external_code(rc, reg_kind,
657                                                    location.AsRegister());
658       }
659       DCHECK(location.IsCallerFrameSlot());
660       // For reference type parameters we have to use registers that were not
661       // used for parameters because some reference type stack parameters may
662       // get processed before some value type register parameters.
663       static constexpr auto kRegsUnusedByParams = RegsUnusedByParams();
664       LiftoffRegister reg = is_reference(reg_kind)
665                                 ? __ GetUnusedRegister(kRegsUnusedByParams)
666                                 : __ GetUnusedRegister(rc, pinned);
667       __ LoadCallerFrameSlot(reg, -location.AsCallerFrameSlot(), reg_kind);
668       return reg;
669     };
670 
671     LiftoffRegister reg =
672         LoadToReg(descriptor_->GetInputLocation(input_idx), {});
673     if (needs_pair) {
674       LiftoffRegister reg2 = LoadToReg(
675           descriptor_->GetInputLocation(input_idx + 1), LiftoffRegList{reg});
676       reg = LiftoffRegister::ForPair(reg.gp(), reg2.gp());
677     }
678     __ PushRegister(kind, reg);
679 
680     return needs_pair ? 2 : 1;
681   }
682 
StackCheck(FullDecoder * decoder,WasmCodePosition position)683   void StackCheck(FullDecoder* decoder, WasmCodePosition position) {
684     CODE_COMMENT("stack check");
685     if (!FLAG_wasm_stack_checks || !env_->runtime_exception_support) return;
686 
687     // Loading the limit address can change the stack state, hence do this
688     // before storing information about registers.
689     Register limit_address = __ GetUnusedRegister(kGpReg, {}).gp();
690     LOAD_INSTANCE_FIELD(limit_address, StackLimitAddress, kSystemPointerSize,
691                         {});
692 
693     LiftoffRegList regs_to_save = __ cache_state()->used_registers;
694     // The cached instance will be reloaded separately.
695     if (__ cache_state()->cached_instance != no_reg) {
696       DCHECK(regs_to_save.has(__ cache_state()->cached_instance));
697       regs_to_save.clear(__ cache_state()->cached_instance);
698     }
699     SpilledRegistersForInspection* spilled_regs = nullptr;
700 
701     OutOfLineSafepointInfo* safepoint_info =
702         compilation_zone_->New<OutOfLineSafepointInfo>(compilation_zone_);
703     __ cache_state()->GetTaggedSlotsForOOLCode(
704         &safepoint_info->slots, &safepoint_info->spills,
705         for_debugging_
706             ? LiftoffAssembler::CacheState::SpillLocation::kStackSlots
707             : LiftoffAssembler::CacheState::SpillLocation::kTopOfStack);
708     if (V8_UNLIKELY(for_debugging_)) {
709       // When debugging, we do not just push all registers to the stack, but we
710       // spill them to their proper stack locations such that we can inspect
711       // them.
712       // The only exception is the cached memory start, which we just push
713       // before the stack check and pop afterwards.
714       regs_to_save = {};
715       if (__ cache_state()->cached_mem_start != no_reg) {
716         regs_to_save.set(__ cache_state()->cached_mem_start);
717       }
718       spilled_regs = GetSpilledRegistersForInspection();
719     }
720     out_of_line_code_.push_back(OutOfLineCode::StackCheck(
721         position, regs_to_save, __ cache_state()->cached_instance, spilled_regs,
722         safepoint_info, RegisterOOLDebugSideTableEntry(decoder)));
723     OutOfLineCode& ool = out_of_line_code_.back();
724     __ StackCheck(ool.label.get(), limit_address);
725     __ bind(ool.continuation.get());
726   }
727 
TierupCheck(FullDecoder * decoder,WasmCodePosition position,int budget_used)728   void TierupCheck(FullDecoder* decoder, WasmCodePosition position,
729                    int budget_used) {
730     // We should always decrement the budget, and we don't expect integer
731     // overflows in the budget calculation.
732     DCHECK_LE(1, budget_used);
733 
734     if (for_debugging_ != kNoDebugging) return;
735     CODE_COMMENT("tierup check");
736     // We never want to blow the entire budget at once.
737     const int kMax = FLAG_wasm_tiering_budget / 4;
738     if (budget_used > kMax) budget_used = kMax;
739 
740     LiftoffRegister budget_reg = __ GetUnusedRegister(kGpReg, {});
741     __ Fill(budget_reg, liftoff::kTierupBudgetOffset, ValueKind::kI32);
742     LiftoffRegList regs_to_save = __ cache_state()->used_registers;
743     // The cached instance will be reloaded separately.
744     if (__ cache_state()->cached_instance != no_reg) {
745       DCHECK(regs_to_save.has(__ cache_state()->cached_instance));
746       regs_to_save.clear(__ cache_state()->cached_instance);
747     }
748     SpilledRegistersForInspection* spilled_regs = nullptr;
749 
750     OutOfLineSafepointInfo* safepoint_info =
751         compilation_zone_->New<OutOfLineSafepointInfo>(compilation_zone_);
752     __ cache_state()->GetTaggedSlotsForOOLCode(
753         &safepoint_info->slots, &safepoint_info->spills,
754         LiftoffAssembler::CacheState::SpillLocation::kTopOfStack);
755     out_of_line_code_.push_back(OutOfLineCode::TierupCheck(
756         position, regs_to_save, __ cache_state()->cached_instance, spilled_regs,
757         safepoint_info, RegisterOOLDebugSideTableEntry(decoder)));
758     OutOfLineCode& ool = out_of_line_code_.back();
759     __ emit_i32_subi_jump_negative(budget_reg.gp(), budget_used,
760                                    ool.label.get());
761     __ Spill(liftoff::kTierupBudgetOffset, budget_reg, ValueKind::kI32);
762     __ bind(ool.continuation.get());
763   }
764 
SpillLocalsInitially(FullDecoder * decoder,uint32_t num_params)765   bool SpillLocalsInitially(FullDecoder* decoder, uint32_t num_params) {
766     int actual_locals = __ num_locals() - num_params;
767     DCHECK_LE(0, actual_locals);
768     constexpr int kNumCacheRegisters = kLiftoffAssemblerGpCacheRegs.Count();
769     // If we have many locals, we put them on the stack initially. This avoids
770     // having to spill them on merge points. Use of these initial values should
771     // be rare anyway.
772     if (actual_locals > kNumCacheRegisters / 2) return true;
773     // If there are locals which are not i32 or i64, we also spill all locals,
774     // because other types cannot be initialized to constants.
775     for (uint32_t param_idx = num_params; param_idx < __ num_locals();
776          ++param_idx) {
777       ValueKind kind = __ local_kind(param_idx);
778       if (kind != kI32 && kind != kI64) return true;
779     }
780     return false;
781   }
782 
TraceFunctionEntry(FullDecoder * decoder)783   void TraceFunctionEntry(FullDecoder* decoder) {
784     CODE_COMMENT("trace function entry");
785     __ SpillAllRegisters();
786     source_position_table_builder_.AddPosition(
787         __ pc_offset(), SourcePosition(decoder->position()), false);
788     __ CallRuntimeStub(WasmCode::kWasmTraceEnter);
789     DefineSafepoint();
790   }
791 
dynamic_tiering()792   bool dynamic_tiering() {
793     return env_->dynamic_tiering == DynamicTiering::kEnabled &&
794            for_debugging_ == kNoDebugging &&
795            (FLAG_wasm_tier_up_filter == -1 ||
796             FLAG_wasm_tier_up_filter == func_index_);
797   }
798 
StartFunctionBody(FullDecoder * decoder,Control * block)799   void StartFunctionBody(FullDecoder* decoder, Control* block) {
800     for (uint32_t i = 0; i < __ num_locals(); ++i) {
801       if (!CheckSupportedType(decoder, __ local_kind(i), "param")) return;
802     }
803 
804     // Parameter 0 is the instance parameter.
805     uint32_t num_params =
806         static_cast<uint32_t>(decoder->sig_->parameter_count());
807 
808     __ CodeEntry();
809 
810     __ EnterFrame(StackFrame::WASM);
811     __ set_has_frame(true);
812     pc_offset_stack_frame_construction_ = __ PrepareStackFrame();
813     // {PrepareStackFrame} is the first platform-specific assembler method.
814     // If this failed, we can bail out immediately, avoiding runtime overhead
815     // and potential failures because of other unimplemented methods.
816     // A platform implementing {PrepareStackFrame} must ensure that we can
817     // finish compilation without errors even if we hit unimplemented
818     // LiftoffAssembler methods.
819     if (DidAssemblerBailout(decoder)) return;
820 
821     // Input 0 is the call target, the instance is at 1.
822     constexpr int kInstanceParameterIndex = 1;
823     // Check that {kWasmInstanceRegister} matches our call descriptor.
824     DCHECK_EQ(kWasmInstanceRegister,
825               Register::from_code(
826                   descriptor_->GetInputLocation(kInstanceParameterIndex)
827                       .AsRegister()));
828     __ cache_state()->SetInstanceCacheRegister(kWasmInstanceRegister);
829     // Load the feedback vector and cache it in a stack slot.
830     constexpr LiftoffRegList kGpParamRegisters = GetGpParamRegisters();
831     if (FLAG_wasm_speculative_inlining) {
832       CODE_COMMENT("load feedback vector");
833       int declared_func_index =
834           func_index_ - env_->module->num_imported_functions;
835       DCHECK_GE(declared_func_index, 0);
836       LiftoffRegList pinned = kGpParamRegisters;
837       LiftoffRegister tmp = pinned.set(__ GetUnusedRegister(kGpReg, pinned));
838       __ LoadTaggedPointerFromInstance(
839           tmp.gp(), kWasmInstanceRegister,
840           WASM_INSTANCE_OBJECT_FIELD_OFFSET(FeedbackVectors));
841       __ LoadTaggedPointer(tmp.gp(), tmp.gp(), no_reg,
842                            wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(
843                                declared_func_index),
844                            pinned);
845       __ Spill(liftoff::kFeedbackVectorOffset, tmp, kPointerKind);
846     }
847     if (dynamic_tiering()) {
848       CODE_COMMENT("load tier up budget");
849       LiftoffRegList pinned = kGpParamRegisters;
850       LiftoffRegister tmp = pinned.set(__ GetUnusedRegister(kGpReg, pinned));
851       LOAD_INSTANCE_FIELD(tmp.gp(), TieringBudgetArray, kSystemPointerSize,
852                           pinned);
853       uint32_t offset =
854           kInt32Size * declared_function_index(env_->module, func_index_);
855       __ Load(tmp, tmp.gp(), no_reg, offset, LoadType::kI32Load, pinned);
856       __ Spill(liftoff::kTierupBudgetOffset, tmp, ValueKind::kI32);
857     }
858     if (for_debugging_) __ ResetOSRTarget();
859 
860     // Process parameters.
861     if (num_params) CODE_COMMENT("process parameters");
862     // Input 0 is the code target, 1 is the instance. First parameter at 2.
863     uint32_t input_idx = kInstanceParameterIndex + 1;
864     for (uint32_t param_idx = 0; param_idx < num_params; ++param_idx) {
865       input_idx += ProcessParameter(__ local_kind(param_idx), input_idx);
866     }
867     int params_size = __ TopSpillOffset();
868     DCHECK_EQ(input_idx, descriptor_->InputCount());
869 
870     // Initialize locals beyond parameters.
871     if (num_params < __ num_locals()) CODE_COMMENT("init locals");
872     if (SpillLocalsInitially(decoder, num_params)) {
873       bool has_refs = false;
874       for (uint32_t param_idx = num_params; param_idx < __ num_locals();
875            ++param_idx) {
876         ValueKind kind = __ local_kind(param_idx);
877         has_refs |= is_reference(kind);
878         __ PushStack(kind);
879       }
880       int spill_size = __ TopSpillOffset() - params_size;
881       __ FillStackSlotsWithZero(params_size, spill_size);
882 
883       // Initialize all reference type locals with ref.null.
884       if (has_refs) {
885         Register null_ref_reg = __ GetUnusedRegister(kGpReg, {}).gp();
886         LoadNullValue(null_ref_reg, {});
887         for (uint32_t local_index = num_params; local_index < __ num_locals();
888              ++local_index) {
889           ValueKind kind = __ local_kind(local_index);
890           if (is_reference(kind)) {
891             __ Spill(__ cache_state()->stack_state[local_index].offset(),
892                      LiftoffRegister(null_ref_reg), kind);
893           }
894         }
895       }
896     } else {
897       for (uint32_t param_idx = num_params; param_idx < __ num_locals();
898            ++param_idx) {
899         ValueKind kind = __ local_kind(param_idx);
900         // Anything which is not i32 or i64 requires spilling.
901         DCHECK(kind == kI32 || kind == kI64);
902         __ PushConstant(kind, int32_t{0});
903       }
904     }
905 
906     DCHECK_EQ(__ num_locals(), __ cache_state()->stack_height());
907 
908     if (V8_UNLIKELY(debug_sidetable_builder_)) {
909       debug_sidetable_builder_->SetNumLocals(__ num_locals());
910     }
911 
912     // The function-prologue stack check is associated with position 0, which
913     // is never a position of any instruction in the function.
914     StackCheck(decoder, 0);
915 
916     if (FLAG_trace_wasm) TraceFunctionEntry(decoder);
917   }
918 
GenerateOutOfLineCode(OutOfLineCode * ool)919   void GenerateOutOfLineCode(OutOfLineCode* ool) {
920     CODE_COMMENT(
921         (std::string("OOL: ") + GetRuntimeStubName(ool->stub)).c_str());
922     __ bind(ool->label.get());
923     const bool is_stack_check = ool->stub == WasmCode::kWasmStackGuard;
924     const bool is_tierup = ool->stub == WasmCode::kWasmTriggerTierUp;
925 
926     // Only memory OOB traps need a {pc}, but not unconditionally. Static OOB
927     // accesses do not need protected instruction information, hence they also
928     // do not set {pc}.
929     DCHECK_IMPLIES(ool->stub != WasmCode::kThrowWasmTrapMemOutOfBounds,
930                    ool->pc == 0);
931 
932     if (env_->bounds_checks == kTrapHandler && ool->pc != 0) {
933       uint32_t pc = static_cast<uint32_t>(__ pc_offset());
934       DCHECK_EQ(pc, __ pc_offset());
935       protected_instructions_.emplace_back(
936           trap_handler::ProtectedInstructionData{ool->pc, pc});
937     }
938 
939     if (!env_->runtime_exception_support) {
940       // We cannot test calls to the runtime in cctest/test-run-wasm.
941       // Therefore we emit a call to C here instead of a call to the runtime.
942       // In this mode, we never generate stack checks.
943       DCHECK(!is_stack_check);
944       __ CallTrapCallbackForTesting();
945       __ LeaveFrame(StackFrame::WASM);
946       __ DropStackSlotsAndRet(
947           static_cast<uint32_t>(descriptor_->ParameterSlotCount()));
948       return;
949     }
950 
951     if (!ool->regs_to_save.is_empty()) {
952       __ PushRegisters(ool->regs_to_save);
953     }
954     if (V8_UNLIKELY(ool->spilled_registers != nullptr)) {
955       for (auto& entry : ool->spilled_registers->entries) {
956         // We should not push and spill the same register.
957         DCHECK(!ool->regs_to_save.has(entry.reg));
958         __ Spill(entry.offset, entry.reg, entry.kind);
959       }
960     }
961 
962     source_position_table_builder_.AddPosition(
963         __ pc_offset(), SourcePosition(ool->position), true);
964     __ CallRuntimeStub(ool->stub);
965     auto safepoint = safepoint_table_builder_.DefineSafepoint(&asm_);
966 
967     if (ool->safepoint_info) {
968       for (auto index : ool->safepoint_info->slots) {
969         safepoint.DefineTaggedStackSlot(index);
970       }
971 
972       int total_frame_size = __ GetTotalFrameSize();
973       LiftoffRegList gp_regs = ool->regs_to_save & kGpCacheRegList;
974       // {total_frame_size} is the highest offset from the FP that is used to
975       // store a value. The offset of the first spill slot should therefore be
976       // {(total_frame_size / kSystemPointerSize) + 1}. However, spill slots
977       // don't start at offset '0' but at offset '-1' (or
978       // {-kSystemPointerSize}). Therefore we have to add another '+ 1' to the
979       // index of the first spill slot.
980       int index = (total_frame_size / kSystemPointerSize) + 2;
981 
982       __ RecordSpillsInSafepoint(safepoint, gp_regs,
983                                  ool->safepoint_info->spills, index);
984     }
985     if (is_tierup) {
986       // Reset the budget.
987       __ Spill(liftoff::kTierupBudgetOffset,
988                WasmValue(FLAG_wasm_tiering_budget));
989     }
990 
991     DCHECK_EQ(!debug_sidetable_builder_, !ool->debug_sidetable_entry_builder);
992     if (V8_UNLIKELY(ool->debug_sidetable_entry_builder)) {
993       ool->debug_sidetable_entry_builder->set_pc_offset(__ pc_offset());
994     }
995     DCHECK_EQ(ool->continuation.get()->is_bound(), is_stack_check || is_tierup);
996     if (is_stack_check) {
997       MaybeOSR();
998     }
999     if (!ool->regs_to_save.is_empty()) __ PopRegisters(ool->regs_to_save);
1000     if (is_stack_check || is_tierup) {
1001       if (V8_UNLIKELY(ool->spilled_registers != nullptr)) {
1002         DCHECK(for_debugging_);
1003         for (auto& entry : ool->spilled_registers->entries) {
1004           __ Fill(entry.reg, entry.offset, entry.kind);
1005         }
1006       }
1007       if (ool->cached_instance != no_reg) {
1008         __ LoadInstanceFromFrame(ool->cached_instance);
1009       }
1010       __ emit_jump(ool->continuation.get());
1011     } else {
1012       __ AssertUnreachable(AbortReason::kUnexpectedReturnFromWasmTrap);
1013     }
1014   }
1015 
FinishFunction(FullDecoder * decoder)1016   void FinishFunction(FullDecoder* decoder) {
1017     if (DidAssemblerBailout(decoder)) return;
1018     __ AlignFrameSize();
1019 #if DEBUG
1020     int frame_size = __ GetTotalFrameSize();
1021 #endif
1022     for (OutOfLineCode& ool : out_of_line_code_) {
1023       GenerateOutOfLineCode(&ool);
1024     }
1025     DCHECK_EQ(frame_size, __ GetTotalFrameSize());
1026     __ PatchPrepareStackFrame(pc_offset_stack_frame_construction_,
1027                               &safepoint_table_builder_);
1028     __ FinishCode();
1029     safepoint_table_builder_.Emit(&asm_, __ GetTotalFrameSlotCountForGC());
1030     // Emit the handler table.
1031     if (!handlers_.empty()) {
1032       handler_table_offset_ = HandlerTable::EmitReturnTableStart(&asm_);
1033       for (auto& handler : handlers_) {
1034         HandlerTable::EmitReturnEntry(&asm_, handler.pc_offset,
1035                                       handler.handler.get()->pos());
1036       }
1037     }
1038     __ MaybeEmitOutOfLineConstantPool();
1039     // The previous calls may have also generated a bailout.
1040     DidAssemblerBailout(decoder);
1041     DCHECK_EQ(num_exceptions_, 0);
1042   }
1043 
OnFirstError(FullDecoder * decoder)1044   void OnFirstError(FullDecoder* decoder) {
1045     if (!did_bailout()) bailout_reason_ = kDecodeError;
1046     UnuseLabels(decoder);
1047     asm_.AbortCompilation();
1048   }
1049 
EmitDebuggingInfo(FullDecoder * decoder,WasmOpcode opcode)1050   V8_NOINLINE void EmitDebuggingInfo(FullDecoder* decoder, WasmOpcode opcode) {
1051     DCHECK(for_debugging_);
1052     if (!WasmOpcodes::IsBreakable(opcode)) return;
1053     bool has_breakpoint = false;
1054     if (next_breakpoint_ptr_) {
1055       if (*next_breakpoint_ptr_ == 0) {
1056         // A single breakpoint at offset 0 indicates stepping.
1057         DCHECK_EQ(next_breakpoint_ptr_ + 1, next_breakpoint_end_);
1058         has_breakpoint = true;
1059       } else {
1060         while (next_breakpoint_ptr_ != next_breakpoint_end_ &&
1061                *next_breakpoint_ptr_ < decoder->position()) {
1062           // Skip unreachable breakpoints.
1063           ++next_breakpoint_ptr_;
1064         }
1065         if (next_breakpoint_ptr_ == next_breakpoint_end_) {
1066           next_breakpoint_ptr_ = next_breakpoint_end_ = nullptr;
1067         } else if (*next_breakpoint_ptr_ == decoder->position()) {
1068           has_breakpoint = true;
1069         }
1070       }
1071     }
1072     if (has_breakpoint) {
1073       CODE_COMMENT("breakpoint");
1074       EmitBreakpoint(decoder);
1075       // Once we emitted an unconditional breakpoint, we don't need to check
1076       // function entry breaks any more.
1077       did_function_entry_break_checks_ = true;
1078     } else if (!did_function_entry_break_checks_) {
1079       did_function_entry_break_checks_ = true;
1080       CODE_COMMENT("check function entry break");
1081       Label do_break;
1082       Label no_break;
1083       Register flag = __ GetUnusedRegister(kGpReg, {}).gp();
1084 
1085       // Check the "hook on function call" flag. If set, trigger a break.
1086       LOAD_INSTANCE_FIELD(flag, HookOnFunctionCallAddress, kSystemPointerSize,
1087                           {});
1088       __ Load(LiftoffRegister{flag}, flag, no_reg, 0, LoadType::kI32Load8U, {});
1089       __ emit_cond_jump(kNotEqualZero, &do_break, kI32, flag);
1090 
1091       // Check if we should stop on "script entry".
1092       LOAD_INSTANCE_FIELD(flag, BreakOnEntry, kUInt8Size, {});
1093       __ emit_cond_jump(kEqualZero, &no_break, kI32, flag);
1094 
1095       __ bind(&do_break);
1096       EmitBreakpoint(decoder);
1097       __ bind(&no_break);
1098     } else if (dead_breakpoint_ == decoder->position()) {
1099       DCHECK(!next_breakpoint_ptr_ ||
1100              *next_breakpoint_ptr_ != dead_breakpoint_);
1101       // The top frame is paused at this position, but the breakpoint was
1102       // removed. Adding a dead breakpoint here ensures that the source
1103       // position exists, and that the offset to the return address is the
1104       // same as in the old code.
1105       CODE_COMMENT("dead breakpoint");
1106       Label cont;
1107       __ emit_jump(&cont);
1108       EmitBreakpoint(decoder);
1109       __ bind(&cont);
1110     }
1111     if (V8_UNLIKELY(max_steps_ != nullptr)) {
1112       CODE_COMMENT("check max steps");
1113       LiftoffRegList pinned;
1114       LiftoffRegister max_steps = __ GetUnusedRegister(kGpReg, {});
1115       pinned.set(max_steps);
1116       LiftoffRegister max_steps_addr = __ GetUnusedRegister(kGpReg, pinned);
1117       pinned.set(max_steps_addr);
1118       __ LoadConstant(
1119           max_steps_addr,
1120           WasmValue::ForUintPtr(reinterpret_cast<uintptr_t>(max_steps_)));
1121       __ Load(max_steps, max_steps_addr.gp(), no_reg, 0, LoadType::kI32Load,
1122               pinned);
1123       Label cont;
1124       __ emit_i32_cond_jumpi(kUnequal, &cont, max_steps.gp(), 0);
1125       // Abort.
1126       Trap(decoder, kTrapUnreachable);
1127       __ bind(&cont);
1128       __ emit_i32_subi(max_steps.gp(), max_steps.gp(), 1);
1129       __ Store(max_steps_addr.gp(), no_reg, 0, max_steps, StoreType::kI32Store,
1130                pinned);
1131     }
1132   }
1133 
NextInstruction(FullDecoder * decoder,WasmOpcode opcode)1134   void NextInstruction(FullDecoder* decoder, WasmOpcode opcode) {
1135     // Add a single check, so that the fast path can be inlined while
1136     // {EmitDebuggingInfo} stays outlined.
1137     if (V8_UNLIKELY(for_debugging_)) EmitDebuggingInfo(decoder, opcode);
1138     TraceCacheState(decoder);
1139     SLOW_DCHECK(__ ValidateCacheState());
1140     CODE_COMMENT(WasmOpcodes::OpcodeName(
1141         WasmOpcodes::IsPrefixOpcode(opcode)
1142             ? decoder->read_prefixed_opcode<Decoder::kFullValidation>(
1143                   decoder->pc())
1144             : opcode));
1145   }
1146 
EmitBreakpoint(FullDecoder * decoder)1147   void EmitBreakpoint(FullDecoder* decoder) {
1148     DCHECK(for_debugging_);
1149     source_position_table_builder_.AddPosition(
1150         __ pc_offset(), SourcePosition(decoder->position()), true);
1151     __ CallRuntimeStub(WasmCode::kWasmDebugBreak);
1152     DefineSafepointWithCalleeSavedRegisters();
1153     RegisterDebugSideTableEntry(decoder,
1154                                 DebugSideTableBuilder::kAllowRegisters);
1155     MaybeOSR();
1156   }
1157 
PushControl(Control * block)1158   void PushControl(Control* block) {
1159     // The Liftoff stack includes implicit exception refs stored for catch
1160     // blocks, so that they can be rethrown.
1161     block->num_exceptions = num_exceptions_;
1162   }
1163 
Block(FullDecoder * decoder,Control * block)1164   void Block(FullDecoder* decoder, Control* block) { PushControl(block); }
1165 
Loop(FullDecoder * decoder,Control * loop)1166   void Loop(FullDecoder* decoder, Control* loop) {
1167     // Before entering a loop, spill all locals to the stack, in order to free
1168     // the cache registers, and to avoid unnecessarily reloading stack values
1169     // into registers at branches.
1170     // TODO(clemensb): Come up with a better strategy here, involving
1171     // pre-analysis of the function.
1172     __ SpillLocals();
1173 
1174     __ PrepareLoopArgs(loop->start_merge.arity);
1175 
1176     // Loop labels bind at the beginning of the block.
1177     __ bind(loop->label.get());
1178 
1179     // Save the current cache state for the merge when jumping to this loop.
1180     loop->label_state.Split(*__ cache_state());
1181 
1182     PushControl(loop);
1183 
1184     if (!dynamic_tiering()) {
1185       // When the budget-based tiering mechanism is enabled, use that to
1186       // check for interrupt requests; otherwise execute a stack check in the
1187       // loop header.
1188       StackCheck(decoder, decoder->position());
1189     }
1190   }
1191 
Try(FullDecoder * decoder,Control * block)1192   void Try(FullDecoder* decoder, Control* block) {
1193     block->try_info = std::make_unique<TryInfo>();
1194     PushControl(block);
1195   }
1196 
1197   // Load the property in {kReturnRegister0}.
GetExceptionProperty(LiftoffAssembler::VarState & exception,RootIndex root_index)1198   LiftoffRegister GetExceptionProperty(LiftoffAssembler::VarState& exception,
1199                                        RootIndex root_index) {
1200     DCHECK(root_index == RootIndex::kwasm_exception_tag_symbol ||
1201            root_index == RootIndex::kwasm_exception_values_symbol);
1202 
1203     LiftoffRegList pinned;
1204     LiftoffRegister tag_symbol_reg =
1205         pinned.set(__ GetUnusedRegister(kGpReg, pinned));
1206     LoadExceptionSymbol(tag_symbol_reg.gp(), pinned, root_index);
1207     LiftoffRegister context_reg =
1208         pinned.set(__ GetUnusedRegister(kGpReg, pinned));
1209     LOAD_TAGGED_PTR_INSTANCE_FIELD(context_reg.gp(), NativeContext, pinned);
1210 
1211     LiftoffAssembler::VarState tag_symbol(kPointerKind, tag_symbol_reg, 0);
1212     LiftoffAssembler::VarState context(kPointerKind, context_reg, 0);
1213 
1214     CallRuntimeStub(WasmCode::kWasmGetOwnProperty,
1215                     MakeSig::Returns(kPointerKind)
1216                         .Params(kPointerKind, kPointerKind, kPointerKind),
1217                     {exception, tag_symbol, context}, kNoSourcePosition);
1218 
1219     return LiftoffRegister(kReturnRegister0);
1220   }
1221 
CatchException(FullDecoder * decoder,const TagIndexImmediate<validate> & imm,Control * block,base::Vector<Value> values)1222   void CatchException(FullDecoder* decoder,
1223                       const TagIndexImmediate<validate>& imm, Control* block,
1224                       base::Vector<Value> values) {
1225     DCHECK(block->is_try_catch());
1226     __ emit_jump(block->label.get());
1227 
1228     // The catch block is unreachable if no possible throws in the try block
1229     // exist. We only build a landing pad if some node in the try block can
1230     // (possibly) throw. Otherwise the catch environments remain empty.
1231     if (!block->try_info->catch_reached) {
1232       block->reachability = kSpecOnlyReachable;
1233       return;
1234     }
1235 
1236     // This is the last use of this label. Re-use the field for the label of the
1237     // next catch block, and jump there if the tag does not match.
1238     __ bind(&block->try_info->catch_label);
1239     new (&block->try_info->catch_label) Label();
1240 
1241     __ cache_state()->Split(block->try_info->catch_state);
1242 
1243     CODE_COMMENT("load caught exception tag");
1244     DCHECK_EQ(__ cache_state()->stack_state.back().kind(), kRef);
1245     LiftoffRegister caught_tag =
1246         GetExceptionProperty(__ cache_state()->stack_state.back(),
1247                              RootIndex::kwasm_exception_tag_symbol);
1248     LiftoffRegList pinned;
1249     pinned.set(caught_tag);
1250 
1251     CODE_COMMENT("load expected exception tag");
1252     Register imm_tag = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
1253     LOAD_TAGGED_PTR_INSTANCE_FIELD(imm_tag, TagsTable, pinned);
1254     __ LoadTaggedPointer(
1255         imm_tag, imm_tag, no_reg,
1256         wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(imm.index), {});
1257 
1258     CODE_COMMENT("compare tags");
1259     Label caught;
1260     __ emit_cond_jump(kEqual, &caught, kI32, imm_tag, caught_tag.gp());
1261     // The tags don't match, merge the current state into the catch state and
1262     // jump to the next handler.
1263     __ MergeFullStackWith(block->try_info->catch_state, *__ cache_state());
1264     __ emit_jump(&block->try_info->catch_label);
1265 
1266     __ bind(&caught);
1267     if (!block->try_info->in_handler) {
1268       block->try_info->in_handler = true;
1269       num_exceptions_++;
1270     }
1271     GetExceptionValues(decoder, __ cache_state()->stack_state.back(), imm.tag);
1272   }
1273 
Rethrow(FullDecoder * decoder,const LiftoffAssembler::VarState & exception)1274   void Rethrow(FullDecoder* decoder,
1275                const LiftoffAssembler::VarState& exception) {
1276     DCHECK_EQ(exception.kind(), kRef);
1277     CallRuntimeStub(WasmCode::kWasmRethrow, MakeSig::Params(kPointerKind),
1278                     {exception}, decoder->position());
1279   }
1280 
Delegate(FullDecoder * decoder,uint32_t depth,Control * block)1281   void Delegate(FullDecoder* decoder, uint32_t depth, Control* block) {
1282     DCHECK_EQ(block, decoder->control_at(0));
1283     Control* target = decoder->control_at(depth);
1284     DCHECK(block->is_incomplete_try());
1285     __ bind(&block->try_info->catch_label);
1286     if (block->try_info->catch_reached) {
1287       __ cache_state()->Steal(block->try_info->catch_state);
1288       if (depth == decoder->control_depth() - 1) {
1289         // Delegate to the caller, do not emit a landing pad.
1290         Rethrow(decoder, __ cache_state()->stack_state.back());
1291         MaybeOSR();
1292       } else {
1293         DCHECK(target->is_incomplete_try());
1294         if (!target->try_info->catch_reached) {
1295           target->try_info->catch_state.InitMerge(
1296               *__ cache_state(), __ num_locals(), 1,
1297               target->stack_depth + target->num_exceptions);
1298           target->try_info->catch_reached = true;
1299         }
1300         __ MergeStackWith(target->try_info->catch_state, 1,
1301                           LiftoffAssembler::kForwardJump);
1302         __ emit_jump(&target->try_info->catch_label);
1303       }
1304     }
1305   }
1306 
Rethrow(FullDecoder * decoder,Control * try_block)1307   void Rethrow(FullDecoder* decoder, Control* try_block) {
1308     int index = try_block->try_info->catch_state.stack_height() - 1;
1309     auto& exception = __ cache_state()->stack_state[index];
1310     Rethrow(decoder, exception);
1311     int pc_offset = __ pc_offset();
1312     MaybeOSR();
1313     EmitLandingPad(decoder, pc_offset);
1314   }
1315 
CatchAll(FullDecoder * decoder,Control * block)1316   void CatchAll(FullDecoder* decoder, Control* block) {
1317     DCHECK(block->is_try_catchall() || block->is_try_catch());
1318     DCHECK_EQ(decoder->control_at(0), block);
1319 
1320     // The catch block is unreachable if no possible throws in the try block
1321     // exist. We only build a landing pad if some node in the try block can
1322     // (possibly) throw. Otherwise the catch environments remain empty.
1323     if (!block->try_info->catch_reached) {
1324       decoder->SetSucceedingCodeDynamicallyUnreachable();
1325       return;
1326     }
1327 
1328     __ bind(&block->try_info->catch_label);
1329     __ cache_state()->Steal(block->try_info->catch_state);
1330     if (!block->try_info->in_handler) {
1331       block->try_info->in_handler = true;
1332       num_exceptions_++;
1333     }
1334   }
1335 
JumpIfFalse(FullDecoder * decoder,Label * false_dst)1336   void JumpIfFalse(FullDecoder* decoder, Label* false_dst) {
1337     LiftoffCondition cond =
1338         test_and_reset_outstanding_op(kExprI32Eqz) ? kNotEqualZero : kEqualZero;
1339 
1340     if (!has_outstanding_op()) {
1341       // Unary comparison.
1342       Register value = __ PopToRegister().gp();
1343       __ emit_cond_jump(cond, false_dst, kI32, value);
1344       return;
1345     }
1346 
1347     // Binary comparison of i32 values.
1348     cond = Negate(GetCompareCondition(outstanding_op_));
1349     outstanding_op_ = kNoOutstandingOp;
1350     LiftoffAssembler::VarState rhs_slot = __ cache_state()->stack_state.back();
1351     if (rhs_slot.is_const()) {
1352       // Compare to a constant.
1353       int32_t rhs_imm = rhs_slot.i32_const();
1354       __ cache_state()->stack_state.pop_back();
1355       Register lhs = __ PopToRegister().gp();
1356       __ emit_i32_cond_jumpi(cond, false_dst, lhs, rhs_imm);
1357       return;
1358     }
1359 
1360     Register rhs = __ PopToRegister().gp();
1361     LiftoffAssembler::VarState lhs_slot = __ cache_state()->stack_state.back();
1362     if (lhs_slot.is_const()) {
1363       // Compare a constant to an arbitrary value.
1364       int32_t lhs_imm = lhs_slot.i32_const();
1365       __ cache_state()->stack_state.pop_back();
1366       // Flip the condition, because {lhs} and {rhs} are swapped.
1367       __ emit_i32_cond_jumpi(Flip(cond), false_dst, rhs, lhs_imm);
1368       return;
1369     }
1370 
1371     // Compare two arbitrary values.
1372     Register lhs = __ PopToRegister(LiftoffRegList{rhs}).gp();
1373     __ emit_cond_jump(cond, false_dst, kI32, lhs, rhs);
1374   }
1375 
If(FullDecoder * decoder,const Value & cond,Control * if_block)1376   void If(FullDecoder* decoder, const Value& cond, Control* if_block) {
1377     DCHECK_EQ(if_block, decoder->control_at(0));
1378     DCHECK(if_block->is_if());
1379 
1380     // Allocate the else state.
1381     if_block->else_state = std::make_unique<ElseState>();
1382 
1383     // Test the condition on the value stack, jump to else if zero.
1384     JumpIfFalse(decoder, if_block->else_state->label.get());
1385 
1386     // Store the state (after popping the value) for executing the else branch.
1387     if_block->else_state->state.Split(*__ cache_state());
1388 
1389     PushControl(if_block);
1390   }
1391 
FallThruTo(FullDecoder * decoder,Control * c)1392   void FallThruTo(FullDecoder* decoder, Control* c) {
1393     if (!c->end_merge.reached) {
1394       c->label_state.InitMerge(*__ cache_state(), __ num_locals(),
1395                                c->end_merge.arity,
1396                                c->stack_depth + c->num_exceptions);
1397     }
1398     DCHECK(!c->is_try_catchall());
1399     if (c->is_try_catch()) {
1400       // Drop the implicit exception ref if any. There may be none if this is a
1401       // catch-less try block.
1402       __ MergeStackWith(c->label_state, c->br_merge()->arity,
1403                         LiftoffAssembler::kForwardJump);
1404     } else {
1405       __ MergeFullStackWith(c->label_state, *__ cache_state());
1406     }
1407     __ emit_jump(c->label.get());
1408     TraceCacheState(decoder);
1409   }
1410 
FinishOneArmedIf(FullDecoder * decoder,Control * c)1411   void FinishOneArmedIf(FullDecoder* decoder, Control* c) {
1412     DCHECK(c->is_onearmed_if());
1413     if (c->end_merge.reached) {
1414       // Someone already merged to the end of the if. Merge both arms into that.
1415       if (c->reachable()) {
1416         // Merge the if state into the end state.
1417         __ MergeFullStackWith(c->label_state, *__ cache_state());
1418         __ emit_jump(c->label.get());
1419       }
1420       // Merge the else state into the end state. Set this state as the current
1421       // state first so helper functions know which registers are in use.
1422       __ bind(c->else_state->label.get());
1423       __ cache_state()->Steal(c->else_state->state);
1424       __ MergeFullStackWith(c->label_state, *__ cache_state());
1425       __ cache_state()->Steal(c->label_state);
1426     } else if (c->reachable()) {
1427       // No merge yet at the end of the if, but we need to create a merge for
1428       // the both arms of this if. Thus init the merge point from the else
1429       // state, then merge the if state into that.
1430       DCHECK_EQ(c->start_merge.arity, c->end_merge.arity);
1431       c->label_state.InitMerge(c->else_state->state, __ num_locals(),
1432                                c->start_merge.arity,
1433                                c->stack_depth + c->num_exceptions);
1434       __ MergeFullStackWith(c->label_state, *__ cache_state());
1435       __ emit_jump(c->label.get());
1436       // Merge the else state into the end state. Set this state as the current
1437       // state first so helper functions know which registers are in use.
1438       __ bind(c->else_state->label.get());
1439       __ cache_state()->Steal(c->else_state->state);
1440       __ MergeFullStackWith(c->label_state, *__ cache_state());
1441       __ cache_state()->Steal(c->label_state);
1442     } else {
1443       // No merge needed, just continue with the else state.
1444       __ bind(c->else_state->label.get());
1445       __ cache_state()->Steal(c->else_state->state);
1446     }
1447   }
1448 
FinishTry(FullDecoder * decoder,Control * c)1449   void FinishTry(FullDecoder* decoder, Control* c) {
1450     DCHECK(c->is_try_catch() || c->is_try_catchall());
1451     if (!c->end_merge.reached) {
1452       if (c->try_info->catch_reached) {
1453         // Drop the implicit exception ref.
1454         __ DropValue(__ num_locals() + c->stack_depth + c->num_exceptions);
1455       }
1456       // Else we did not enter the catch state, continue with the current state.
1457     } else {
1458       if (c->reachable()) {
1459         __ MergeStackWith(c->label_state, c->br_merge()->arity,
1460                           LiftoffAssembler::kForwardJump);
1461       }
1462       __ cache_state()->Steal(c->label_state);
1463     }
1464     if (c->try_info->catch_reached) {
1465       num_exceptions_--;
1466     }
1467   }
1468 
PopControl(FullDecoder * decoder,Control * c)1469   void PopControl(FullDecoder* decoder, Control* c) {
1470     if (c->is_loop()) return;  // A loop just falls through.
1471     if (c->is_onearmed_if()) {
1472       // Special handling for one-armed ifs.
1473       FinishOneArmedIf(decoder, c);
1474     } else if (c->is_try_catch() || c->is_try_catchall()) {
1475       FinishTry(decoder, c);
1476     } else if (c->end_merge.reached) {
1477       // There is a merge already. Merge our state into that, then continue with
1478       // that state.
1479       if (c->reachable()) {
1480         __ MergeFullStackWith(c->label_state, *__ cache_state());
1481       }
1482       __ cache_state()->Steal(c->label_state);
1483     } else {
1484       // No merge, just continue with our current state.
1485     }
1486 
1487     if (!c->label.get()->is_bound()) __ bind(c->label.get());
1488   }
1489 
GenerateCCall(const LiftoffRegister * result_regs,const ValueKindSig * sig,ValueKind out_argument_kind,const LiftoffRegister * arg_regs,ExternalReference ext_ref)1490   void GenerateCCall(const LiftoffRegister* result_regs,
1491                      const ValueKindSig* sig, ValueKind out_argument_kind,
1492                      const LiftoffRegister* arg_regs,
1493                      ExternalReference ext_ref) {
1494     // Before making a call, spill all cache registers.
1495     __ SpillAllRegisters();
1496 
1497     // Store arguments on our stack, then align the stack for calling to C.
1498     int param_bytes = 0;
1499     for (ValueKind param_kind : sig->parameters()) {
1500       param_bytes += value_kind_size(param_kind);
1501     }
1502     int out_arg_bytes =
1503         out_argument_kind == kVoid ? 0 : value_kind_size(out_argument_kind);
1504     int stack_bytes = std::max(param_bytes, out_arg_bytes);
1505     __ CallC(sig, arg_regs, result_regs, out_argument_kind, stack_bytes,
1506              ext_ref);
1507   }
1508 
1509   template <typename EmitFn, typename... Args>
1510   typename std::enable_if<!std::is_member_function_pointer<EmitFn>::value>::type
CallEmitFn(EmitFn fn,Args...args)1511   CallEmitFn(EmitFn fn, Args... args) {
1512     fn(args...);
1513   }
1514 
1515   template <typename EmitFn, typename... Args>
1516   typename std::enable_if<std::is_member_function_pointer<EmitFn>::value>::type
CallEmitFn(EmitFn fn,Args...args)1517   CallEmitFn(EmitFn fn, Args... args) {
1518     (asm_.*fn)(ConvertAssemblerArg(args)...);
1519   }
1520 
1521   // Wrap a {LiftoffRegister} with implicit conversions to {Register} and
1522   // {DoubleRegister}.
1523   struct AssemblerRegisterConverter {
1524     LiftoffRegister reg;
operator LiftoffRegisterv8::internal::wasm::__anon3bd82aa60111::LiftoffCompiler::AssemblerRegisterConverter1525     operator LiftoffRegister() { return reg; }
operator Registerv8::internal::wasm::__anon3bd82aa60111::LiftoffCompiler::AssemblerRegisterConverter1526     operator Register() { return reg.gp(); }
operator DoubleRegisterv8::internal::wasm::__anon3bd82aa60111::LiftoffCompiler::AssemblerRegisterConverter1527     operator DoubleRegister() { return reg.fp(); }
1528   };
1529 
1530   // Convert {LiftoffRegister} to {AssemblerRegisterConverter}, other types stay
1531   // unchanged.
1532   template <typename T>
1533   typename std::conditional<std::is_same<LiftoffRegister, T>::value,
1534                             AssemblerRegisterConverter, T>::type
ConvertAssemblerArg(T t)1535   ConvertAssemblerArg(T t) {
1536     return {t};
1537   }
1538 
1539   template <typename EmitFn, typename ArgType>
1540   struct EmitFnWithFirstArg {
1541     EmitFn fn;
1542     ArgType first_arg;
1543   };
1544 
1545   template <typename EmitFn, typename ArgType>
BindFirst(EmitFn fn,ArgType arg)1546   EmitFnWithFirstArg<EmitFn, ArgType> BindFirst(EmitFn fn, ArgType arg) {
1547     return {fn, arg};
1548   }
1549 
1550   template <typename EmitFn, typename T, typename... Args>
CallEmitFn(EmitFnWithFirstArg<EmitFn,T> bound_fn,Args...args)1551   void CallEmitFn(EmitFnWithFirstArg<EmitFn, T> bound_fn, Args... args) {
1552     CallEmitFn(bound_fn.fn, bound_fn.first_arg, ConvertAssemblerArg(args)...);
1553   }
1554 
1555   template <ValueKind src_kind, ValueKind result_kind,
1556             ValueKind result_lane_kind = kVoid, class EmitFn>
EmitUnOp(EmitFn fn)1557   void EmitUnOp(EmitFn fn) {
1558     constexpr RegClass src_rc = reg_class_for(src_kind);
1559     constexpr RegClass result_rc = reg_class_for(result_kind);
1560     LiftoffRegister src = __ PopToRegister();
1561     LiftoffRegister dst = src_rc == result_rc
1562                               ? __ GetUnusedRegister(result_rc, {src}, {})
1563                               : __ GetUnusedRegister(result_rc, {});
1564     CallEmitFn(fn, dst, src);
1565     if (V8_UNLIKELY(nondeterminism_)) {
1566       LiftoffRegList pinned = {dst};
1567       if (result_kind == ValueKind::kF32 || result_kind == ValueKind::kF64) {
1568         CheckNan(dst, pinned, result_kind);
1569       } else if (result_kind == ValueKind::kS128 &&
1570                  (result_lane_kind == kF32 || result_lane_kind == kF64)) {
1571         CheckS128Nan(dst, pinned, result_lane_kind);
1572       }
1573     }
1574     __ PushRegister(result_kind, dst);
1575   }
1576 
1577   template <ValueKind kind>
EmitFloatUnOpWithCFallback(bool (LiftoffAssembler::* emit_fn)(DoubleRegister,DoubleRegister),ExternalReference (* fallback_fn)())1578   void EmitFloatUnOpWithCFallback(
1579       bool (LiftoffAssembler::*emit_fn)(DoubleRegister, DoubleRegister),
1580       ExternalReference (*fallback_fn)()) {
1581     auto emit_with_c_fallback = [=](LiftoffRegister dst, LiftoffRegister src) {
1582       if ((asm_.*emit_fn)(dst.fp(), src.fp())) return;
1583       ExternalReference ext_ref = fallback_fn();
1584       auto sig = MakeSig::Params(kind);
1585       GenerateCCall(&dst, &sig, kind, &src, ext_ref);
1586     };
1587     EmitUnOp<kind, kind>(emit_with_c_fallback);
1588   }
1589 
1590   enum TypeConversionTrapping : bool { kCanTrap = true, kNoTrap = false };
1591   template <ValueKind dst_kind, ValueKind src_kind,
1592             TypeConversionTrapping can_trap>
EmitTypeConversion(FullDecoder * decoder,WasmOpcode opcode,ExternalReference (* fallback_fn)())1593   void EmitTypeConversion(FullDecoder* decoder, WasmOpcode opcode,
1594                           ExternalReference (*fallback_fn)()) {
1595     static constexpr RegClass src_rc = reg_class_for(src_kind);
1596     static constexpr RegClass dst_rc = reg_class_for(dst_kind);
1597     LiftoffRegister src = __ PopToRegister();
1598     LiftoffRegister dst = src_rc == dst_rc
1599                               ? __ GetUnusedRegister(dst_rc, {src}, {})
1600                               : __ GetUnusedRegister(dst_rc, {});
1601     Label* trap =
1602         can_trap ? AddOutOfLineTrap(
1603                        decoder, WasmCode::kThrowWasmTrapFloatUnrepresentable)
1604                  : nullptr;
1605     if (!__ emit_type_conversion(opcode, dst, src, trap)) {
1606       DCHECK_NOT_NULL(fallback_fn);
1607       ExternalReference ext_ref = fallback_fn();
1608       if (can_trap) {
1609         // External references for potentially trapping conversions return int.
1610         auto sig = MakeSig::Returns(kI32).Params(src_kind);
1611         LiftoffRegister ret_reg =
1612             __ GetUnusedRegister(kGpReg, LiftoffRegList{dst});
1613         LiftoffRegister dst_regs[] = {ret_reg, dst};
1614         GenerateCCall(dst_regs, &sig, dst_kind, &src, ext_ref);
1615         __ emit_cond_jump(kEqual, trap, kI32, ret_reg.gp());
1616       } else {
1617         ValueKind sig_kinds[] = {src_kind};
1618         ValueKindSig sig(0, 1, sig_kinds);
1619         GenerateCCall(&dst, &sig, dst_kind, &src, ext_ref);
1620       }
1621     }
1622     __ PushRegister(dst_kind, dst);
1623   }
1624 
UnOp(FullDecoder * decoder,WasmOpcode opcode,const Value & value,Value * result)1625   void UnOp(FullDecoder* decoder, WasmOpcode opcode, const Value& value,
1626             Value* result) {
1627 #define CASE_I32_UNOP(opcode, fn) \
1628   case kExpr##opcode:             \
1629     return EmitUnOp<kI32, kI32>(&LiftoffAssembler::emit_##fn);
1630 #define CASE_I64_UNOP(opcode, fn) \
1631   case kExpr##opcode:             \
1632     return EmitUnOp<kI64, kI64>(&LiftoffAssembler::emit_##fn);
1633 #define CASE_FLOAT_UNOP(opcode, kind, fn) \
1634   case kExpr##opcode:                     \
1635     return EmitUnOp<k##kind, k##kind>(&LiftoffAssembler::emit_##fn);
1636 #define CASE_FLOAT_UNOP_WITH_CFALLBACK(opcode, kind, fn)                     \
1637   case kExpr##opcode:                                                        \
1638     return EmitFloatUnOpWithCFallback<k##kind>(&LiftoffAssembler::emit_##fn, \
1639                                                &ExternalReference::wasm_##fn);
1640 #define CASE_TYPE_CONVERSION(opcode, dst_kind, src_kind, ext_ref, can_trap) \
1641   case kExpr##opcode:                                                       \
1642     return EmitTypeConversion<k##dst_kind, k##src_kind, can_trap>(          \
1643         decoder, kExpr##opcode, ext_ref);
1644     switch (opcode) {
1645       CASE_I32_UNOP(I32Clz, i32_clz)
1646       CASE_I32_UNOP(I32Ctz, i32_ctz)
1647       CASE_FLOAT_UNOP(F32Abs, F32, f32_abs)
1648       CASE_FLOAT_UNOP(F32Neg, F32, f32_neg)
1649       CASE_FLOAT_UNOP_WITH_CFALLBACK(F32Ceil, F32, f32_ceil)
1650       CASE_FLOAT_UNOP_WITH_CFALLBACK(F32Floor, F32, f32_floor)
1651       CASE_FLOAT_UNOP_WITH_CFALLBACK(F32Trunc, F32, f32_trunc)
1652       CASE_FLOAT_UNOP_WITH_CFALLBACK(F32NearestInt, F32, f32_nearest_int)
1653       CASE_FLOAT_UNOP(F32Sqrt, F32, f32_sqrt)
1654       CASE_FLOAT_UNOP(F64Abs, F64, f64_abs)
1655       CASE_FLOAT_UNOP(F64Neg, F64, f64_neg)
1656       CASE_FLOAT_UNOP_WITH_CFALLBACK(F64Ceil, F64, f64_ceil)
1657       CASE_FLOAT_UNOP_WITH_CFALLBACK(F64Floor, F64, f64_floor)
1658       CASE_FLOAT_UNOP_WITH_CFALLBACK(F64Trunc, F64, f64_trunc)
1659       CASE_FLOAT_UNOP_WITH_CFALLBACK(F64NearestInt, F64, f64_nearest_int)
1660       CASE_FLOAT_UNOP(F64Sqrt, F64, f64_sqrt)
1661       CASE_TYPE_CONVERSION(I32ConvertI64, I32, I64, nullptr, kNoTrap)
1662       CASE_TYPE_CONVERSION(I32SConvertF32, I32, F32, nullptr, kCanTrap)
1663       CASE_TYPE_CONVERSION(I32UConvertF32, I32, F32, nullptr, kCanTrap)
1664       CASE_TYPE_CONVERSION(I32SConvertF64, I32, F64, nullptr, kCanTrap)
1665       CASE_TYPE_CONVERSION(I32UConvertF64, I32, F64, nullptr, kCanTrap)
1666       CASE_TYPE_CONVERSION(I32ReinterpretF32, I32, F32, nullptr, kNoTrap)
1667       CASE_TYPE_CONVERSION(I64SConvertI32, I64, I32, nullptr, kNoTrap)
1668       CASE_TYPE_CONVERSION(I64UConvertI32, I64, I32, nullptr, kNoTrap)
1669       CASE_TYPE_CONVERSION(I64SConvertF32, I64, F32,
1670                            &ExternalReference::wasm_float32_to_int64, kCanTrap)
1671       CASE_TYPE_CONVERSION(I64UConvertF32, I64, F32,
1672                            &ExternalReference::wasm_float32_to_uint64, kCanTrap)
1673       CASE_TYPE_CONVERSION(I64SConvertF64, I64, F64,
1674                            &ExternalReference::wasm_float64_to_int64, kCanTrap)
1675       CASE_TYPE_CONVERSION(I64UConvertF64, I64, F64,
1676                            &ExternalReference::wasm_float64_to_uint64, kCanTrap)
1677       CASE_TYPE_CONVERSION(I64ReinterpretF64, I64, F64, nullptr, kNoTrap)
1678       CASE_TYPE_CONVERSION(F32SConvertI32, F32, I32, nullptr, kNoTrap)
1679       CASE_TYPE_CONVERSION(F32UConvertI32, F32, I32, nullptr, kNoTrap)
1680       CASE_TYPE_CONVERSION(F32SConvertI64, F32, I64,
1681                            &ExternalReference::wasm_int64_to_float32, kNoTrap)
1682       CASE_TYPE_CONVERSION(F32UConvertI64, F32, I64,
1683                            &ExternalReference::wasm_uint64_to_float32, kNoTrap)
1684       CASE_TYPE_CONVERSION(F32ConvertF64, F32, F64, nullptr, kNoTrap)
1685       CASE_TYPE_CONVERSION(F32ReinterpretI32, F32, I32, nullptr, kNoTrap)
1686       CASE_TYPE_CONVERSION(F64SConvertI32, F64, I32, nullptr, kNoTrap)
1687       CASE_TYPE_CONVERSION(F64UConvertI32, F64, I32, nullptr, kNoTrap)
1688       CASE_TYPE_CONVERSION(F64SConvertI64, F64, I64,
1689                            &ExternalReference::wasm_int64_to_float64, kNoTrap)
1690       CASE_TYPE_CONVERSION(F64UConvertI64, F64, I64,
1691                            &ExternalReference::wasm_uint64_to_float64, kNoTrap)
1692       CASE_TYPE_CONVERSION(F64ConvertF32, F64, F32, nullptr, kNoTrap)
1693       CASE_TYPE_CONVERSION(F64ReinterpretI64, F64, I64, nullptr, kNoTrap)
1694       CASE_I32_UNOP(I32SExtendI8, i32_signextend_i8)
1695       CASE_I32_UNOP(I32SExtendI16, i32_signextend_i16)
1696       CASE_I64_UNOP(I64SExtendI8, i64_signextend_i8)
1697       CASE_I64_UNOP(I64SExtendI16, i64_signextend_i16)
1698       CASE_I64_UNOP(I64SExtendI32, i64_signextend_i32)
1699       CASE_I64_UNOP(I64Clz, i64_clz)
1700       CASE_I64_UNOP(I64Ctz, i64_ctz)
1701       CASE_TYPE_CONVERSION(I32SConvertSatF32, I32, F32, nullptr, kNoTrap)
1702       CASE_TYPE_CONVERSION(I32UConvertSatF32, I32, F32, nullptr, kNoTrap)
1703       CASE_TYPE_CONVERSION(I32SConvertSatF64, I32, F64, nullptr, kNoTrap)
1704       CASE_TYPE_CONVERSION(I32UConvertSatF64, I32, F64, nullptr, kNoTrap)
1705       CASE_TYPE_CONVERSION(I64SConvertSatF32, I64, F32,
1706                            &ExternalReference::wasm_float32_to_int64_sat,
1707                            kNoTrap)
1708       CASE_TYPE_CONVERSION(I64UConvertSatF32, I64, F32,
1709                            &ExternalReference::wasm_float32_to_uint64_sat,
1710                            kNoTrap)
1711       CASE_TYPE_CONVERSION(I64SConvertSatF64, I64, F64,
1712                            &ExternalReference::wasm_float64_to_int64_sat,
1713                            kNoTrap)
1714       CASE_TYPE_CONVERSION(I64UConvertSatF64, I64, F64,
1715                            &ExternalReference::wasm_float64_to_uint64_sat,
1716                            kNoTrap)
1717       case kExprI32Eqz:
1718         DCHECK(decoder->lookahead(0, kExprI32Eqz));
1719         if ((decoder->lookahead(1, kExprBrIf) ||
1720              decoder->lookahead(1, kExprIf)) &&
1721             !for_debugging_) {
1722           DCHECK(!has_outstanding_op());
1723           outstanding_op_ = kExprI32Eqz;
1724           break;
1725         }
1726         return EmitUnOp<kI32, kI32>(&LiftoffAssembler::emit_i32_eqz);
1727       case kExprI64Eqz:
1728         return EmitUnOp<kI64, kI32>(&LiftoffAssembler::emit_i64_eqz);
1729       case kExprI32Popcnt:
1730         return EmitUnOp<kI32, kI32>(
1731             [=](LiftoffRegister dst, LiftoffRegister src) {
1732               if (__ emit_i32_popcnt(dst.gp(), src.gp())) return;
1733               auto sig = MakeSig::Returns(kI32).Params(kI32);
1734               GenerateCCall(&dst, &sig, kVoid, &src,
1735                             ExternalReference::wasm_word32_popcnt());
1736             });
1737       case kExprI64Popcnt:
1738         return EmitUnOp<kI64, kI64>(
1739             [=](LiftoffRegister dst, LiftoffRegister src) {
1740               if (__ emit_i64_popcnt(dst, src)) return;
1741               // The c function returns i32. We will zero-extend later.
1742               auto sig = MakeSig::Returns(kI32).Params(kI64);
1743               LiftoffRegister c_call_dst = kNeedI64RegPair ? dst.low() : dst;
1744               GenerateCCall(&c_call_dst, &sig, kVoid, &src,
1745                             ExternalReference::wasm_word64_popcnt());
1746               // Now zero-extend the result to i64.
1747               __ emit_type_conversion(kExprI64UConvertI32, dst, c_call_dst,
1748                                       nullptr);
1749             });
1750       case kExprRefIsNull:
1751       // We abuse ref.as_non_null, which isn't otherwise used in this switch, as
1752       // a sentinel for the negation of ref.is_null.
1753       case kExprRefAsNonNull: {
1754         LiftoffRegList pinned;
1755         LiftoffRegister ref = pinned.set(__ PopToRegister());
1756         LiftoffRegister null = __ GetUnusedRegister(kGpReg, pinned);
1757         LoadNullValue(null.gp(), pinned);
1758         // Prefer to overwrite one of the input registers with the result
1759         // of the comparison.
1760         LiftoffRegister dst = __ GetUnusedRegister(kGpReg, {ref, null}, {});
1761         __ emit_ptrsize_set_cond(opcode == kExprRefIsNull ? kEqual : kUnequal,
1762                                  dst.gp(), ref, null);
1763         __ PushRegister(kI32, dst);
1764         return;
1765       }
1766       default:
1767         UNREACHABLE();
1768     }
1769 #undef CASE_I32_UNOP
1770 #undef CASE_I64_UNOP
1771 #undef CASE_FLOAT_UNOP
1772 #undef CASE_FLOAT_UNOP_WITH_CFALLBACK
1773 #undef CASE_TYPE_CONVERSION
1774   }
1775 
1776   template <ValueKind src_kind, ValueKind result_kind, typename EmitFn,
1777             typename EmitFnImm>
EmitBinOpImm(EmitFn fn,EmitFnImm fnImm)1778   void EmitBinOpImm(EmitFn fn, EmitFnImm fnImm) {
1779     static constexpr RegClass src_rc = reg_class_for(src_kind);
1780     static constexpr RegClass result_rc = reg_class_for(result_kind);
1781 
1782     LiftoffAssembler::VarState rhs_slot = __ cache_state()->stack_state.back();
1783     // Check if the RHS is an immediate.
1784     if (rhs_slot.is_const()) {
1785       __ cache_state()->stack_state.pop_back();
1786       int32_t imm = rhs_slot.i32_const();
1787 
1788       LiftoffRegister lhs = __ PopToRegister();
1789       // Either reuse {lhs} for {dst}, or choose a register (pair) which does
1790       // not overlap, for easier code generation.
1791       LiftoffRegList pinned = {lhs};
1792       LiftoffRegister dst = src_rc == result_rc
1793                                 ? __ GetUnusedRegister(result_rc, {lhs}, pinned)
1794                                 : __ GetUnusedRegister(result_rc, pinned);
1795 
1796       CallEmitFn(fnImm, dst, lhs, imm);
1797       static_assert(result_kind != kF32 && result_kind != kF64,
1798                     "Unhandled nondeterminism for fuzzing.");
1799       __ PushRegister(result_kind, dst);
1800     } else {
1801       // The RHS was not an immediate.
1802       EmitBinOp<src_kind, result_kind>(fn);
1803     }
1804   }
1805 
1806   template <ValueKind src_kind, ValueKind result_kind,
1807             bool swap_lhs_rhs = false, ValueKind result_lane_kind = kVoid,
1808             typename EmitFn>
EmitBinOp(EmitFn fn)1809   void EmitBinOp(EmitFn fn) {
1810     static constexpr RegClass src_rc = reg_class_for(src_kind);
1811     static constexpr RegClass result_rc = reg_class_for(result_kind);
1812     LiftoffRegister rhs = __ PopToRegister();
1813     LiftoffRegister lhs = __ PopToRegister(LiftoffRegList{rhs});
1814     LiftoffRegister dst = src_rc == result_rc
1815                               ? __ GetUnusedRegister(result_rc, {lhs, rhs}, {})
1816                               : __ GetUnusedRegister(result_rc, {});
1817 
1818     if (swap_lhs_rhs) std::swap(lhs, rhs);
1819 
1820     CallEmitFn(fn, dst, lhs, rhs);
1821     if (V8_UNLIKELY(nondeterminism_)) {
1822       LiftoffRegList pinned = {dst};
1823       if (result_kind == ValueKind::kF32 || result_kind == ValueKind::kF64) {
1824         CheckNan(dst, pinned, result_kind);
1825       } else if (result_kind == ValueKind::kS128 &&
1826                  (result_lane_kind == kF32 || result_lane_kind == kF64)) {
1827         CheckS128Nan(dst, pinned, result_lane_kind);
1828       }
1829     }
1830     __ PushRegister(result_kind, dst);
1831   }
1832 
EmitDivOrRem64CCall(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs,ExternalReference ext_ref,Label * trap_by_zero,Label * trap_unrepresentable=nullptr)1833   void EmitDivOrRem64CCall(LiftoffRegister dst, LiftoffRegister lhs,
1834                            LiftoffRegister rhs, ExternalReference ext_ref,
1835                            Label* trap_by_zero,
1836                            Label* trap_unrepresentable = nullptr) {
1837     // Cannot emit native instructions, build C call.
1838     LiftoffRegister ret = __ GetUnusedRegister(kGpReg, LiftoffRegList{dst});
1839     LiftoffRegister tmp =
1840         __ GetUnusedRegister(kGpReg, LiftoffRegList{dst, ret});
1841     LiftoffRegister arg_regs[] = {lhs, rhs};
1842     LiftoffRegister result_regs[] = {ret, dst};
1843     auto sig = MakeSig::Returns(kI32).Params(kI64, kI64);
1844     GenerateCCall(result_regs, &sig, kI64, arg_regs, ext_ref);
1845     __ LoadConstant(tmp, WasmValue(int32_t{0}));
1846     __ emit_cond_jump(kEqual, trap_by_zero, kI32, ret.gp(), tmp.gp());
1847     if (trap_unrepresentable) {
1848       __ LoadConstant(tmp, WasmValue(int32_t{-1}));
1849       __ emit_cond_jump(kEqual, trap_unrepresentable, kI32, ret.gp(), tmp.gp());
1850     }
1851   }
1852 
1853   template <WasmOpcode opcode>
EmitI32CmpOp(FullDecoder * decoder)1854   void EmitI32CmpOp(FullDecoder* decoder) {
1855     DCHECK(decoder->lookahead(0, opcode));
1856     if ((decoder->lookahead(1, kExprBrIf) || decoder->lookahead(1, kExprIf)) &&
1857         !for_debugging_) {
1858       DCHECK(!has_outstanding_op());
1859       outstanding_op_ = opcode;
1860       return;
1861     }
1862     return EmitBinOp<kI32, kI32>(BindFirst(&LiftoffAssembler::emit_i32_set_cond,
1863                                            GetCompareCondition(opcode)));
1864   }
1865 
BinOp(FullDecoder * decoder,WasmOpcode opcode,const Value & lhs,const Value & rhs,Value * result)1866   void BinOp(FullDecoder* decoder, WasmOpcode opcode, const Value& lhs,
1867              const Value& rhs, Value* result) {
1868 #define CASE_I64_SHIFTOP(opcode, fn)                                         \
1869   case kExpr##opcode:                                                        \
1870     return EmitBinOpImm<kI64, kI64>(                                         \
1871         [=](LiftoffRegister dst, LiftoffRegister src,                        \
1872             LiftoffRegister amount) {                                        \
1873           __ emit_##fn(dst, src,                                             \
1874                        amount.is_gp_pair() ? amount.low_gp() : amount.gp()); \
1875         },                                                                   \
1876         &LiftoffAssembler::emit_##fn##i);
1877 #define CASE_CCALL_BINOP(opcode, kind, ext_ref_fn)                           \
1878   case kExpr##opcode:                                                        \
1879     return EmitBinOp<k##kind, k##kind>(                                      \
1880         [=](LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs) { \
1881           LiftoffRegister args[] = {lhs, rhs};                               \
1882           auto ext_ref = ExternalReference::ext_ref_fn();                    \
1883           ValueKind sig_kinds[] = {k##kind, k##kind, k##kind};               \
1884           const bool out_via_stack = k##kind == kI64;                        \
1885           ValueKindSig sig(out_via_stack ? 0 : 1, 2, sig_kinds);             \
1886           ValueKind out_arg_kind = out_via_stack ? kI64 : kVoid;             \
1887           GenerateCCall(&dst, &sig, out_arg_kind, args, ext_ref);            \
1888         });
1889     switch (opcode) {
1890       case kExprI32Add:
1891         return EmitBinOpImm<kI32, kI32>(&LiftoffAssembler::emit_i32_add,
1892                                         &LiftoffAssembler::emit_i32_addi);
1893       case kExprI32Sub:
1894         return EmitBinOp<kI32, kI32>(&LiftoffAssembler::emit_i32_sub);
1895       case kExprI32Mul:
1896         return EmitBinOp<kI32, kI32>(&LiftoffAssembler::emit_i32_mul);
1897       case kExprI32And:
1898         return EmitBinOpImm<kI32, kI32>(&LiftoffAssembler::emit_i32_and,
1899                                         &LiftoffAssembler::emit_i32_andi);
1900       case kExprI32Ior:
1901         return EmitBinOpImm<kI32, kI32>(&LiftoffAssembler::emit_i32_or,
1902                                         &LiftoffAssembler::emit_i32_ori);
1903       case kExprI32Xor:
1904         return EmitBinOpImm<kI32, kI32>(&LiftoffAssembler::emit_i32_xor,
1905                                         &LiftoffAssembler::emit_i32_xori);
1906       case kExprI32Eq:
1907         return EmitI32CmpOp<kExprI32Eq>(decoder);
1908       case kExprI32Ne:
1909         return EmitI32CmpOp<kExprI32Ne>(decoder);
1910       case kExprI32LtS:
1911         return EmitI32CmpOp<kExprI32LtS>(decoder);
1912       case kExprI32LtU:
1913         return EmitI32CmpOp<kExprI32LtU>(decoder);
1914       case kExprI32GtS:
1915         return EmitI32CmpOp<kExprI32GtS>(decoder);
1916       case kExprI32GtU:
1917         return EmitI32CmpOp<kExprI32GtU>(decoder);
1918       case kExprI32LeS:
1919         return EmitI32CmpOp<kExprI32LeS>(decoder);
1920       case kExprI32LeU:
1921         return EmitI32CmpOp<kExprI32LeU>(decoder);
1922       case kExprI32GeS:
1923         return EmitI32CmpOp<kExprI32GeS>(decoder);
1924       case kExprI32GeU:
1925         return EmitI32CmpOp<kExprI32GeU>(decoder);
1926       case kExprI64Add:
1927         return EmitBinOpImm<kI64, kI64>(&LiftoffAssembler::emit_i64_add,
1928                                         &LiftoffAssembler::emit_i64_addi);
1929       case kExprI64Sub:
1930         return EmitBinOp<kI64, kI64>(&LiftoffAssembler::emit_i64_sub);
1931       case kExprI64Mul:
1932         return EmitBinOp<kI64, kI64>(&LiftoffAssembler::emit_i64_mul);
1933       case kExprI64And:
1934         return EmitBinOpImm<kI64, kI64>(&LiftoffAssembler::emit_i64_and,
1935                                         &LiftoffAssembler::emit_i64_andi);
1936       case kExprI64Ior:
1937         return EmitBinOpImm<kI64, kI64>(&LiftoffAssembler::emit_i64_or,
1938                                         &LiftoffAssembler::emit_i64_ori);
1939       case kExprI64Xor:
1940         return EmitBinOpImm<kI64, kI64>(&LiftoffAssembler::emit_i64_xor,
1941                                         &LiftoffAssembler::emit_i64_xori);
1942       case kExprI64Eq:
1943         return EmitBinOp<kI64, kI32>(
1944             BindFirst(&LiftoffAssembler::emit_i64_set_cond, kEqual));
1945       case kExprI64Ne:
1946         return EmitBinOp<kI64, kI32>(
1947             BindFirst(&LiftoffAssembler::emit_i64_set_cond, kUnequal));
1948       case kExprI64LtS:
1949         return EmitBinOp<kI64, kI32>(
1950             BindFirst(&LiftoffAssembler::emit_i64_set_cond, kSignedLessThan));
1951       case kExprI64LtU:
1952         return EmitBinOp<kI64, kI32>(
1953             BindFirst(&LiftoffAssembler::emit_i64_set_cond, kUnsignedLessThan));
1954       case kExprI64GtS:
1955         return EmitBinOp<kI64, kI32>(BindFirst(
1956             &LiftoffAssembler::emit_i64_set_cond, kSignedGreaterThan));
1957       case kExprI64GtU:
1958         return EmitBinOp<kI64, kI32>(BindFirst(
1959             &LiftoffAssembler::emit_i64_set_cond, kUnsignedGreaterThan));
1960       case kExprI64LeS:
1961         return EmitBinOp<kI64, kI32>(
1962             BindFirst(&LiftoffAssembler::emit_i64_set_cond, kSignedLessEqual));
1963       case kExprI64LeU:
1964         return EmitBinOp<kI64, kI32>(BindFirst(
1965             &LiftoffAssembler::emit_i64_set_cond, kUnsignedLessEqual));
1966       case kExprI64GeS:
1967         return EmitBinOp<kI64, kI32>(BindFirst(
1968             &LiftoffAssembler::emit_i64_set_cond, kSignedGreaterEqual));
1969       case kExprI64GeU:
1970         return EmitBinOp<kI64, kI32>(BindFirst(
1971             &LiftoffAssembler::emit_i64_set_cond, kUnsignedGreaterEqual));
1972       case kExprF32Eq:
1973         return EmitBinOp<kF32, kI32>(
1974             BindFirst(&LiftoffAssembler::emit_f32_set_cond, kEqual));
1975       case kExprF32Ne:
1976         return EmitBinOp<kF32, kI32>(
1977             BindFirst(&LiftoffAssembler::emit_f32_set_cond, kUnequal));
1978       case kExprF32Lt:
1979         return EmitBinOp<kF32, kI32>(
1980             BindFirst(&LiftoffAssembler::emit_f32_set_cond, kUnsignedLessThan));
1981       case kExprF32Gt:
1982         return EmitBinOp<kF32, kI32>(BindFirst(
1983             &LiftoffAssembler::emit_f32_set_cond, kUnsignedGreaterThan));
1984       case kExprF32Le:
1985         return EmitBinOp<kF32, kI32>(BindFirst(
1986             &LiftoffAssembler::emit_f32_set_cond, kUnsignedLessEqual));
1987       case kExprF32Ge:
1988         return EmitBinOp<kF32, kI32>(BindFirst(
1989             &LiftoffAssembler::emit_f32_set_cond, kUnsignedGreaterEqual));
1990       case kExprF64Eq:
1991         return EmitBinOp<kF64, kI32>(
1992             BindFirst(&LiftoffAssembler::emit_f64_set_cond, kEqual));
1993       case kExprF64Ne:
1994         return EmitBinOp<kF64, kI32>(
1995             BindFirst(&LiftoffAssembler::emit_f64_set_cond, kUnequal));
1996       case kExprF64Lt:
1997         return EmitBinOp<kF64, kI32>(
1998             BindFirst(&LiftoffAssembler::emit_f64_set_cond, kUnsignedLessThan));
1999       case kExprF64Gt:
2000         return EmitBinOp<kF64, kI32>(BindFirst(
2001             &LiftoffAssembler::emit_f64_set_cond, kUnsignedGreaterThan));
2002       case kExprF64Le:
2003         return EmitBinOp<kF64, kI32>(BindFirst(
2004             &LiftoffAssembler::emit_f64_set_cond, kUnsignedLessEqual));
2005       case kExprF64Ge:
2006         return EmitBinOp<kF64, kI32>(BindFirst(
2007             &LiftoffAssembler::emit_f64_set_cond, kUnsignedGreaterEqual));
2008       case kExprI32Shl:
2009         return EmitBinOpImm<kI32, kI32>(&LiftoffAssembler::emit_i32_shl,
2010                                         &LiftoffAssembler::emit_i32_shli);
2011       case kExprI32ShrS:
2012         return EmitBinOpImm<kI32, kI32>(&LiftoffAssembler::emit_i32_sar,
2013                                         &LiftoffAssembler::emit_i32_sari);
2014       case kExprI32ShrU:
2015         return EmitBinOpImm<kI32, kI32>(&LiftoffAssembler::emit_i32_shr,
2016                                         &LiftoffAssembler::emit_i32_shri);
2017         CASE_CCALL_BINOP(I32Rol, I32, wasm_word32_rol)
2018         CASE_CCALL_BINOP(I32Ror, I32, wasm_word32_ror)
2019         CASE_I64_SHIFTOP(I64Shl, i64_shl)
2020         CASE_I64_SHIFTOP(I64ShrS, i64_sar)
2021         CASE_I64_SHIFTOP(I64ShrU, i64_shr)
2022         CASE_CCALL_BINOP(I64Rol, I64, wasm_word64_rol)
2023         CASE_CCALL_BINOP(I64Ror, I64, wasm_word64_ror)
2024       case kExprF32Add:
2025         return EmitBinOp<kF32, kF32>(&LiftoffAssembler::emit_f32_add);
2026       case kExprF32Sub:
2027         return EmitBinOp<kF32, kF32>(&LiftoffAssembler::emit_f32_sub);
2028       case kExprF32Mul:
2029         return EmitBinOp<kF32, kF32>(&LiftoffAssembler::emit_f32_mul);
2030       case kExprF32Div:
2031         return EmitBinOp<kF32, kF32>(&LiftoffAssembler::emit_f32_div);
2032       case kExprF32Min:
2033         return EmitBinOp<kF32, kF32>(&LiftoffAssembler::emit_f32_min);
2034       case kExprF32Max:
2035         return EmitBinOp<kF32, kF32>(&LiftoffAssembler::emit_f32_max);
2036       case kExprF32CopySign:
2037         return EmitBinOp<kF32, kF32>(&LiftoffAssembler::emit_f32_copysign);
2038       case kExprF64Add:
2039         return EmitBinOp<kF64, kF64>(&LiftoffAssembler::emit_f64_add);
2040       case kExprF64Sub:
2041         return EmitBinOp<kF64, kF64>(&LiftoffAssembler::emit_f64_sub);
2042       case kExprF64Mul:
2043         return EmitBinOp<kF64, kF64>(&LiftoffAssembler::emit_f64_mul);
2044       case kExprF64Div:
2045         return EmitBinOp<kF64, kF64>(&LiftoffAssembler::emit_f64_div);
2046       case kExprF64Min:
2047         return EmitBinOp<kF64, kF64>(&LiftoffAssembler::emit_f64_min);
2048       case kExprF64Max:
2049         return EmitBinOp<kF64, kF64>(&LiftoffAssembler::emit_f64_max);
2050       case kExprF64CopySign:
2051         return EmitBinOp<kF64, kF64>(&LiftoffAssembler::emit_f64_copysign);
2052       case kExprI32DivS:
2053         return EmitBinOp<kI32, kI32>([this, decoder](LiftoffRegister dst,
2054                                                      LiftoffRegister lhs,
2055                                                      LiftoffRegister rhs) {
2056           AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapDivByZero);
2057           // Adding the second trap might invalidate the pointer returned for
2058           // the first one, thus get both pointers afterwards.
2059           AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapDivUnrepresentable);
2060           Label* div_by_zero = out_of_line_code_.end()[-2].label.get();
2061           Label* div_unrepresentable = out_of_line_code_.end()[-1].label.get();
2062           __ emit_i32_divs(dst.gp(), lhs.gp(), rhs.gp(), div_by_zero,
2063                            div_unrepresentable);
2064         });
2065       case kExprI32DivU:
2066         return EmitBinOp<kI32, kI32>([this, decoder](LiftoffRegister dst,
2067                                                      LiftoffRegister lhs,
2068                                                      LiftoffRegister rhs) {
2069           Label* div_by_zero =
2070               AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapDivByZero);
2071           __ emit_i32_divu(dst.gp(), lhs.gp(), rhs.gp(), div_by_zero);
2072         });
2073       case kExprI32RemS:
2074         return EmitBinOp<kI32, kI32>([this, decoder](LiftoffRegister dst,
2075                                                      LiftoffRegister lhs,
2076                                                      LiftoffRegister rhs) {
2077           Label* rem_by_zero =
2078               AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapRemByZero);
2079           __ emit_i32_rems(dst.gp(), lhs.gp(), rhs.gp(), rem_by_zero);
2080         });
2081       case kExprI32RemU:
2082         return EmitBinOp<kI32, kI32>([this, decoder](LiftoffRegister dst,
2083                                                      LiftoffRegister lhs,
2084                                                      LiftoffRegister rhs) {
2085           Label* rem_by_zero =
2086               AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapRemByZero);
2087           __ emit_i32_remu(dst.gp(), lhs.gp(), rhs.gp(), rem_by_zero);
2088         });
2089       case kExprI64DivS:
2090         return EmitBinOp<kI64, kI64>([this, decoder](LiftoffRegister dst,
2091                                                      LiftoffRegister lhs,
2092                                                      LiftoffRegister rhs) {
2093           AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapDivByZero);
2094           // Adding the second trap might invalidate the pointer returned for
2095           // the first one, thus get both pointers afterwards.
2096           AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapDivUnrepresentable);
2097           Label* div_by_zero = out_of_line_code_.end()[-2].label.get();
2098           Label* div_unrepresentable = out_of_line_code_.end()[-1].label.get();
2099           if (!__ emit_i64_divs(dst, lhs, rhs, div_by_zero,
2100                                 div_unrepresentable)) {
2101             ExternalReference ext_ref = ExternalReference::wasm_int64_div();
2102             EmitDivOrRem64CCall(dst, lhs, rhs, ext_ref, div_by_zero,
2103                                 div_unrepresentable);
2104           }
2105         });
2106       case kExprI64DivU:
2107         return EmitBinOp<kI64, kI64>([this, decoder](LiftoffRegister dst,
2108                                                      LiftoffRegister lhs,
2109                                                      LiftoffRegister rhs) {
2110           Label* div_by_zero =
2111               AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapDivByZero);
2112           if (!__ emit_i64_divu(dst, lhs, rhs, div_by_zero)) {
2113             ExternalReference ext_ref = ExternalReference::wasm_uint64_div();
2114             EmitDivOrRem64CCall(dst, lhs, rhs, ext_ref, div_by_zero);
2115           }
2116         });
2117       case kExprI64RemS:
2118         return EmitBinOp<kI64, kI64>([this, decoder](LiftoffRegister dst,
2119                                                      LiftoffRegister lhs,
2120                                                      LiftoffRegister rhs) {
2121           Label* rem_by_zero =
2122               AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapRemByZero);
2123           if (!__ emit_i64_rems(dst, lhs, rhs, rem_by_zero)) {
2124             ExternalReference ext_ref = ExternalReference::wasm_int64_mod();
2125             EmitDivOrRem64CCall(dst, lhs, rhs, ext_ref, rem_by_zero);
2126           }
2127         });
2128       case kExprI64RemU:
2129         return EmitBinOp<kI64, kI64>([this, decoder](LiftoffRegister dst,
2130                                                      LiftoffRegister lhs,
2131                                                      LiftoffRegister rhs) {
2132           Label* rem_by_zero =
2133               AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapRemByZero);
2134           if (!__ emit_i64_remu(dst, lhs, rhs, rem_by_zero)) {
2135             ExternalReference ext_ref = ExternalReference::wasm_uint64_mod();
2136             EmitDivOrRem64CCall(dst, lhs, rhs, ext_ref, rem_by_zero);
2137           }
2138         });
2139       case kExprRefEq: {
2140         return EmitBinOp<kOptRef, kI32>(
2141             BindFirst(&LiftoffAssembler::emit_ptrsize_set_cond, kEqual));
2142       }
2143 
2144       default:
2145         UNREACHABLE();
2146     }
2147 #undef CASE_I64_SHIFTOP
2148 #undef CASE_CCALL_BINOP
2149   }
2150 
I32Const(FullDecoder * decoder,Value * result,int32_t value)2151   void I32Const(FullDecoder* decoder, Value* result, int32_t value) {
2152     __ PushConstant(kI32, value);
2153   }
2154 
I64Const(FullDecoder * decoder,Value * result,int64_t value)2155   void I64Const(FullDecoder* decoder, Value* result, int64_t value) {
2156     // The {VarState} stores constant values as int32_t, thus we only store
2157     // 64-bit constants in this field if it fits in an int32_t. Larger values
2158     // cannot be used as immediate value anyway, so we can also just put them in
2159     // a register immediately.
2160     int32_t value_i32 = static_cast<int32_t>(value);
2161     if (value_i32 == value) {
2162       __ PushConstant(kI64, value_i32);
2163     } else {
2164       LiftoffRegister reg = __ GetUnusedRegister(reg_class_for(kI64), {});
2165       __ LoadConstant(reg, WasmValue(value));
2166       __ PushRegister(kI64, reg);
2167     }
2168   }
2169 
F32Const(FullDecoder * decoder,Value * result,float value)2170   void F32Const(FullDecoder* decoder, Value* result, float value) {
2171     LiftoffRegister reg = __ GetUnusedRegister(kFpReg, {});
2172     __ LoadConstant(reg, WasmValue(value));
2173     __ PushRegister(kF32, reg);
2174   }
2175 
F64Const(FullDecoder * decoder,Value * result,double value)2176   void F64Const(FullDecoder* decoder, Value* result, double value) {
2177     LiftoffRegister reg = __ GetUnusedRegister(kFpReg, {});
2178     __ LoadConstant(reg, WasmValue(value));
2179     __ PushRegister(kF64, reg);
2180   }
2181 
RefNull(FullDecoder * decoder,ValueType type,Value *)2182   void RefNull(FullDecoder* decoder, ValueType type, Value*) {
2183     LiftoffRegister null = __ GetUnusedRegister(kGpReg, {});
2184     LoadNullValue(null.gp(), {});
2185     __ PushRegister(type.kind(), null);
2186   }
2187 
RefFunc(FullDecoder * decoder,uint32_t function_index,Value * result)2188   void RefFunc(FullDecoder* decoder, uint32_t function_index, Value* result) {
2189     LiftoffRegister func_index_reg = __ GetUnusedRegister(kGpReg, {});
2190     __ LoadConstant(func_index_reg, WasmValue(function_index));
2191     LiftoffAssembler::VarState func_index_var(kI32, func_index_reg, 0);
2192     CallRuntimeStub(WasmCode::kWasmRefFunc, MakeSig::Returns(kRef).Params(kI32),
2193                     {func_index_var}, decoder->position());
2194     __ PushRegister(kRef, LiftoffRegister(kReturnRegister0));
2195   }
2196 
RefAsNonNull(FullDecoder * decoder,const Value & arg,Value * result)2197   void RefAsNonNull(FullDecoder* decoder, const Value& arg, Value* result) {
2198     LiftoffRegList pinned;
2199     LiftoffRegister obj = pinned.set(__ PopToRegister(pinned));
2200     MaybeEmitNullCheck(decoder, obj.gp(), pinned, arg.type);
2201     __ PushRegister(kRef, obj);
2202   }
2203 
Drop(FullDecoder * decoder)2204   void Drop(FullDecoder* decoder) { __ DropValues(1); }
2205 
TraceFunctionExit(FullDecoder * decoder)2206   void TraceFunctionExit(FullDecoder* decoder) {
2207     CODE_COMMENT("trace function exit");
2208     // Before making the runtime call, spill all cache registers.
2209     __ SpillAllRegisters();
2210     LiftoffRegList pinned;
2211     // Get a register to hold the stack slot for the return value.
2212     LiftoffRegister info = pinned.set(__ GetUnusedRegister(kGpReg, pinned));
2213     __ AllocateStackSlot(info.gp(), sizeof(int64_t));
2214 
2215     // Store the return value if there is exactly one. Multiple return values
2216     // are not handled yet.
2217     size_t num_returns = decoder->sig_->return_count();
2218     if (num_returns == 1) {
2219       ValueKind return_kind = decoder->sig_->GetReturn(0).kind();
2220       LiftoffRegister return_reg =
2221           __ LoadToRegister(__ cache_state()->stack_state.back(), pinned);
2222       if (is_reference(return_kind)) {
2223         __ StoreTaggedPointer(info.gp(), no_reg, 0, return_reg, pinned);
2224       } else {
2225         __ Store(info.gp(), no_reg, 0, return_reg,
2226                  StoreType::ForValueKind(return_kind), pinned);
2227       }
2228     }
2229     // Put the parameter in its place.
2230     WasmTraceExitDescriptor descriptor;
2231     DCHECK_EQ(0, descriptor.GetStackParameterCount());
2232     DCHECK_EQ(1, descriptor.GetRegisterParameterCount());
2233     Register param_reg = descriptor.GetRegisterParameter(0);
2234     if (info.gp() != param_reg) {
2235       __ Move(param_reg, info.gp(), kPointerKind);
2236     }
2237 
2238     source_position_table_builder_.AddPosition(
2239         __ pc_offset(), SourcePosition(decoder->position()), false);
2240     __ CallRuntimeStub(WasmCode::kWasmTraceExit);
2241     DefineSafepoint();
2242 
2243     __ DeallocateStackSlot(sizeof(int64_t));
2244   }
2245 
TierupCheckOnExit(FullDecoder * decoder)2246   void TierupCheckOnExit(FullDecoder* decoder) {
2247     if (!dynamic_tiering()) return;
2248     TierupCheck(decoder, decoder->position(), __ pc_offset());
2249     CODE_COMMENT("update tiering budget");
2250     LiftoffRegList pinned;
2251     LiftoffRegister budget = pinned.set(__ GetUnusedRegister(kGpReg, pinned));
2252     LiftoffRegister array = pinned.set(__ GetUnusedRegister(kGpReg, pinned));
2253     LOAD_INSTANCE_FIELD(array.gp(), TieringBudgetArray, kSystemPointerSize,
2254                         pinned);
2255     uint32_t offset =
2256         kInt32Size * declared_function_index(env_->module, func_index_);
2257     __ Fill(budget, liftoff::kTierupBudgetOffset, ValueKind::kI32);
2258     __ Store(array.gp(), no_reg, offset, budget, StoreType::kI32Store, pinned);
2259   }
2260 
DoReturn(FullDecoder * decoder,uint32_t)2261   void DoReturn(FullDecoder* decoder, uint32_t /* drop_values */) {
2262     if (FLAG_trace_wasm) TraceFunctionExit(decoder);
2263     TierupCheckOnExit(decoder);
2264     size_t num_returns = decoder->sig_->return_count();
2265     if (num_returns > 0) __ MoveToReturnLocations(decoder->sig_, descriptor_);
2266     __ LeaveFrame(StackFrame::WASM);
2267     __ DropStackSlotsAndRet(
2268         static_cast<uint32_t>(descriptor_->ParameterSlotCount()));
2269   }
2270 
LocalGet(FullDecoder * decoder,Value * result,const IndexImmediate<validate> & imm)2271   void LocalGet(FullDecoder* decoder, Value* result,
2272                 const IndexImmediate<validate>& imm) {
2273     auto local_slot = __ cache_state()->stack_state[imm.index];
2274     __ cache_state()->stack_state.emplace_back(
2275         local_slot.kind(), __ NextSpillOffset(local_slot.kind()));
2276     auto* slot = &__ cache_state()->stack_state.back();
2277     if (local_slot.is_reg()) {
2278       __ cache_state()->inc_used(local_slot.reg());
2279       slot->MakeRegister(local_slot.reg());
2280     } else if (local_slot.is_const()) {
2281       slot->MakeConstant(local_slot.i32_const());
2282     } else {
2283       DCHECK(local_slot.is_stack());
2284       auto rc = reg_class_for(local_slot.kind());
2285       LiftoffRegister reg = __ GetUnusedRegister(rc, {});
2286       __ cache_state()->inc_used(reg);
2287       slot->MakeRegister(reg);
2288       __ Fill(reg, local_slot.offset(), local_slot.kind());
2289     }
2290   }
2291 
LocalSetFromStackSlot(LiftoffAssembler::VarState * dst_slot,uint32_t local_index)2292   void LocalSetFromStackSlot(LiftoffAssembler::VarState* dst_slot,
2293                              uint32_t local_index) {
2294     auto& state = *__ cache_state();
2295     auto& src_slot = state.stack_state.back();
2296     ValueKind kind = dst_slot->kind();
2297     if (dst_slot->is_reg()) {
2298       LiftoffRegister slot_reg = dst_slot->reg();
2299       if (state.get_use_count(slot_reg) == 1) {
2300         __ Fill(dst_slot->reg(), src_slot.offset(), kind);
2301         return;
2302       }
2303       state.dec_used(slot_reg);
2304       dst_slot->MakeStack();
2305     }
2306     DCHECK(CheckCompatibleStackSlotTypes(kind, __ local_kind(local_index)));
2307     RegClass rc = reg_class_for(kind);
2308     LiftoffRegister dst_reg = __ GetUnusedRegister(rc, {});
2309     __ Fill(dst_reg, src_slot.offset(), kind);
2310     *dst_slot = LiftoffAssembler::VarState(kind, dst_reg, dst_slot->offset());
2311     __ cache_state()->inc_used(dst_reg);
2312   }
2313 
LocalSet(uint32_t local_index,bool is_tee)2314   void LocalSet(uint32_t local_index, bool is_tee) {
2315     auto& state = *__ cache_state();
2316     auto& source_slot = state.stack_state.back();
2317     auto& target_slot = state.stack_state[local_index];
2318     switch (source_slot.loc()) {
2319       case kRegister:
2320         if (target_slot.is_reg()) state.dec_used(target_slot.reg());
2321         target_slot.Copy(source_slot);
2322         if (is_tee) state.inc_used(target_slot.reg());
2323         break;
2324       case kIntConst:
2325         if (target_slot.is_reg()) state.dec_used(target_slot.reg());
2326         target_slot.Copy(source_slot);
2327         break;
2328       case kStack:
2329         LocalSetFromStackSlot(&target_slot, local_index);
2330         break;
2331     }
2332     if (!is_tee) __ cache_state()->stack_state.pop_back();
2333   }
2334 
LocalSet(FullDecoder * decoder,const Value & value,const IndexImmediate<validate> & imm)2335   void LocalSet(FullDecoder* decoder, const Value& value,
2336                 const IndexImmediate<validate>& imm) {
2337     LocalSet(imm.index, false);
2338   }
2339 
LocalTee(FullDecoder * decoder,const Value & value,Value * result,const IndexImmediate<validate> & imm)2340   void LocalTee(FullDecoder* decoder, const Value& value, Value* result,
2341                 const IndexImmediate<validate>& imm) {
2342     LocalSet(imm.index, true);
2343   }
2344 
AllocateLocals(FullDecoder * decoder,base::Vector<Value> local_values)2345   void AllocateLocals(FullDecoder* decoder, base::Vector<Value> local_values) {
2346     // TODO(7748): Introduce typed functions bailout reason
2347     unsupported(decoder, kGC, "let");
2348   }
2349 
DeallocateLocals(FullDecoder * decoder,uint32_t count)2350   void DeallocateLocals(FullDecoder* decoder, uint32_t count) {
2351     // TODO(7748): Introduce typed functions bailout reason
2352     unsupported(decoder, kGC, "let");
2353   }
2354 
GetGlobalBaseAndOffset(const WasmGlobal * global,LiftoffRegList * pinned,uint32_t * offset)2355   Register GetGlobalBaseAndOffset(const WasmGlobal* global,
2356                                   LiftoffRegList* pinned, uint32_t* offset) {
2357     Register addr = pinned->set(__ GetUnusedRegister(kGpReg, {})).gp();
2358     if (global->mutability && global->imported) {
2359       LOAD_INSTANCE_FIELD(addr, ImportedMutableGlobals, kSystemPointerSize,
2360                           *pinned);
2361       __ Load(LiftoffRegister(addr), addr, no_reg,
2362               global->index * sizeof(Address), kPointerLoadType, *pinned);
2363       *offset = 0;
2364     } else {
2365       LOAD_INSTANCE_FIELD(addr, GlobalsStart, kSystemPointerSize, *pinned);
2366       *offset = global->offset;
2367     }
2368     return addr;
2369   }
2370 
GetBaseAndOffsetForImportedMutableExternRefGlobal(const WasmGlobal * global,LiftoffRegList * pinned,Register * base,Register * offset)2371   void GetBaseAndOffsetForImportedMutableExternRefGlobal(
2372       const WasmGlobal* global, LiftoffRegList* pinned, Register* base,
2373       Register* offset) {
2374     Register globals_buffer =
2375         pinned->set(__ GetUnusedRegister(kGpReg, *pinned)).gp();
2376     LOAD_TAGGED_PTR_INSTANCE_FIELD(globals_buffer,
2377                                    ImportedMutableGlobalsBuffers, *pinned);
2378     *base = globals_buffer;
2379     __ LoadTaggedPointer(
2380         *base, globals_buffer, no_reg,
2381         wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(global->offset),
2382         *pinned);
2383 
2384     // For the offset we need the index of the global in the buffer, and
2385     // then calculate the actual offset from the index. Load the index from
2386     // the ImportedMutableGlobals array of the instance.
2387     Register imported_mutable_globals =
2388         pinned->set(__ GetUnusedRegister(kGpReg, *pinned)).gp();
2389 
2390     LOAD_INSTANCE_FIELD(imported_mutable_globals, ImportedMutableGlobals,
2391                         kSystemPointerSize, *pinned);
2392     *offset = imported_mutable_globals;
2393     __ Load(LiftoffRegister(*offset), imported_mutable_globals, no_reg,
2394             global->index * sizeof(Address),
2395             kSystemPointerSize == 4 ? LoadType::kI32Load : LoadType::kI64Load,
2396             *pinned);
2397     __ emit_i32_shli(*offset, *offset, kTaggedSizeLog2);
2398     __ emit_i32_addi(*offset, *offset,
2399                      wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(0));
2400   }
2401 
GlobalGet(FullDecoder * decoder,Value * result,const GlobalIndexImmediate<validate> & imm)2402   void GlobalGet(FullDecoder* decoder, Value* result,
2403                  const GlobalIndexImmediate<validate>& imm) {
2404     const auto* global = &env_->module->globals[imm.index];
2405     ValueKind kind = global->type.kind();
2406     if (!CheckSupportedType(decoder, kind, "global")) {
2407       return;
2408     }
2409 
2410     if (is_reference(kind)) {
2411       if (global->mutability && global->imported) {
2412         LiftoffRegList pinned;
2413         Register base = no_reg;
2414         Register offset = no_reg;
2415         GetBaseAndOffsetForImportedMutableExternRefGlobal(global, &pinned,
2416                                                           &base, &offset);
2417         __ LoadTaggedPointer(base, base, offset, 0, pinned);
2418         __ PushRegister(kind, LiftoffRegister(base));
2419         return;
2420       }
2421 
2422       LiftoffRegList pinned;
2423       Register globals_buffer =
2424           pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
2425       LOAD_TAGGED_PTR_INSTANCE_FIELD(globals_buffer, TaggedGlobalsBuffer,
2426                                      pinned);
2427       Register value = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
2428       __ LoadTaggedPointer(value, globals_buffer, no_reg,
2429                            wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(
2430                                imm.global->offset),
2431                            pinned);
2432       __ PushRegister(kind, LiftoffRegister(value));
2433       return;
2434     }
2435     LiftoffRegList pinned;
2436     uint32_t offset = 0;
2437     Register addr = GetGlobalBaseAndOffset(global, &pinned, &offset);
2438     LiftoffRegister value =
2439         pinned.set(__ GetUnusedRegister(reg_class_for(kind), pinned));
2440     LoadType type = LoadType::ForValueKind(kind);
2441     __ Load(value, addr, no_reg, offset, type, pinned, nullptr, false);
2442     __ PushRegister(kind, value);
2443   }
2444 
GlobalSet(FullDecoder * decoder,const Value &,const GlobalIndexImmediate<validate> & imm)2445   void GlobalSet(FullDecoder* decoder, const Value&,
2446                  const GlobalIndexImmediate<validate>& imm) {
2447     auto* global = &env_->module->globals[imm.index];
2448     ValueKind kind = global->type.kind();
2449     if (!CheckSupportedType(decoder, kind, "global")) {
2450       return;
2451     }
2452 
2453     if (is_reference(kind)) {
2454       if (global->mutability && global->imported) {
2455         LiftoffRegList pinned;
2456         LiftoffRegister value = pinned.set(__ PopToRegister(pinned));
2457         Register base = no_reg;
2458         Register offset = no_reg;
2459         GetBaseAndOffsetForImportedMutableExternRefGlobal(global, &pinned,
2460                                                           &base, &offset);
2461         __ StoreTaggedPointer(base, offset, 0, value, pinned);
2462         return;
2463       }
2464 
2465       LiftoffRegList pinned;
2466       Register globals_buffer =
2467           pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
2468       LOAD_TAGGED_PTR_INSTANCE_FIELD(globals_buffer, TaggedGlobalsBuffer,
2469                                      pinned);
2470       LiftoffRegister value = pinned.set(__ PopToRegister(pinned));
2471       __ StoreTaggedPointer(globals_buffer, no_reg,
2472                             wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(
2473                                 imm.global->offset),
2474                             value, pinned);
2475       return;
2476     }
2477     LiftoffRegList pinned;
2478     uint32_t offset = 0;
2479     Register addr = GetGlobalBaseAndOffset(global, &pinned, &offset);
2480     LiftoffRegister reg = pinned.set(__ PopToRegister(pinned));
2481     StoreType type = StoreType::ForValueKind(kind);
2482     __ Store(addr, no_reg, offset, reg, type, {}, nullptr, false);
2483   }
2484 
TableGet(FullDecoder * decoder,const Value &,Value *,const IndexImmediate<validate> & imm)2485   void TableGet(FullDecoder* decoder, const Value&, Value*,
2486                 const IndexImmediate<validate>& imm) {
2487     LiftoffRegList pinned;
2488 
2489     LiftoffRegister table_index_reg =
2490         pinned.set(__ GetUnusedRegister(kGpReg, pinned));
2491     __ LoadConstant(table_index_reg, WasmValue(imm.index));
2492     LiftoffAssembler::VarState table_index(kPointerKind, table_index_reg, 0);
2493 
2494     LiftoffAssembler::VarState index = __ cache_state()->stack_state.back();
2495 
2496     ValueKind result_kind = env_->module->tables[imm.index].type.kind();
2497     CallRuntimeStub(WasmCode::kWasmTableGet,
2498                     MakeSig::Returns(result_kind).Params(kI32, kI32),
2499                     {table_index, index}, decoder->position());
2500 
2501     // Pop parameters from the value stack.
2502     __ cache_state()->stack_state.pop_back(1);
2503 
2504     RegisterDebugSideTableEntry(decoder, DebugSideTableBuilder::kDidSpill);
2505 
2506     __ PushRegister(result_kind, LiftoffRegister(kReturnRegister0));
2507   }
2508 
TableSet(FullDecoder * decoder,const Value &,const Value &,const IndexImmediate<validate> & imm)2509   void TableSet(FullDecoder* decoder, const Value&, const Value&,
2510                 const IndexImmediate<validate>& imm) {
2511     LiftoffRegList pinned;
2512 
2513     LiftoffRegister table_index_reg =
2514         pinned.set(__ GetUnusedRegister(kGpReg, pinned));
2515     __ LoadConstant(table_index_reg, WasmValue(imm.index));
2516     LiftoffAssembler::VarState table_index(kPointerKind, table_index_reg, 0);
2517 
2518     LiftoffAssembler::VarState value = __ cache_state()->stack_state.end()[-1];
2519     LiftoffAssembler::VarState index = __ cache_state()->stack_state.end()[-2];
2520 
2521     ValueKind table_kind = env_->module->tables[imm.index].type.kind();
2522 
2523     CallRuntimeStub(WasmCode::kWasmTableSet,
2524                     MakeSig::Params(kI32, kI32, table_kind),
2525                     {table_index, index, value}, decoder->position());
2526 
2527     // Pop parameters from the value stack.
2528     __ cache_state()->stack_state.pop_back(2);
2529 
2530     RegisterDebugSideTableEntry(decoder, DebugSideTableBuilder::kDidSpill);
2531   }
2532 
GetRuntimeStubIdForTrapReason(TrapReason reason)2533   WasmCode::RuntimeStubId GetRuntimeStubIdForTrapReason(TrapReason reason) {
2534     switch (reason) {
2535 #define RUNTIME_STUB_FOR_TRAP(trap_reason) \
2536   case k##trap_reason:                     \
2537     return WasmCode::kThrowWasm##trap_reason;
2538 
2539       FOREACH_WASM_TRAPREASON(RUNTIME_STUB_FOR_TRAP)
2540 #undef RUNTIME_STUB_FOR_TRAP
2541       default:
2542         UNREACHABLE();
2543     }
2544   }
2545 
Trap(FullDecoder * decoder,TrapReason reason)2546   void Trap(FullDecoder* decoder, TrapReason reason) {
2547     Label* trap_label =
2548         AddOutOfLineTrap(decoder, GetRuntimeStubIdForTrapReason(reason));
2549     __ emit_jump(trap_label);
2550     __ AssertUnreachable(AbortReason::kUnexpectedReturnFromWasmTrap);
2551   }
2552 
AssertNull(FullDecoder * decoder,const Value & arg,Value * result)2553   void AssertNull(FullDecoder* decoder, const Value& arg, Value* result) {
2554     LiftoffRegList pinned;
2555     LiftoffRegister obj = pinned.set(__ PopToRegister(pinned));
2556     Label* trap_label =
2557         AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapIllegalCast);
2558     LiftoffRegister null = __ GetUnusedRegister(kGpReg, pinned);
2559     LoadNullValue(null.gp(), pinned);
2560     __ emit_cond_jump(kUnequal, trap_label, kOptRef, obj.gp(), null.gp());
2561     __ PushRegister(kOptRef, obj);
2562   }
2563 
NopForTestingUnsupportedInLiftoff(FullDecoder * decoder)2564   void NopForTestingUnsupportedInLiftoff(FullDecoder* decoder) {
2565     unsupported(decoder, kOtherReason, "testing opcode");
2566   }
2567 
Select(FullDecoder * decoder,const Value & cond,const Value & fval,const Value & tval,Value * result)2568   void Select(FullDecoder* decoder, const Value& cond, const Value& fval,
2569               const Value& tval, Value* result) {
2570     LiftoffRegList pinned;
2571     Register condition = pinned.set(__ PopToRegister()).gp();
2572     ValueKind kind = __ cache_state()->stack_state.end()[-1].kind();
2573     DCHECK(CheckCompatibleStackSlotTypes(
2574         kind, __ cache_state()->stack_state.end()[-2].kind()));
2575     LiftoffRegister false_value = pinned.set(__ PopToRegister(pinned));
2576     LiftoffRegister true_value = __ PopToRegister(pinned);
2577     LiftoffRegister dst = __ GetUnusedRegister(true_value.reg_class(),
2578                                                {true_value, false_value}, {});
2579     if (!__ emit_select(dst, condition, true_value, false_value, kind)) {
2580       // Emit generic code (using branches) instead.
2581       Label cont;
2582       Label case_false;
2583       __ emit_cond_jump(kEqual, &case_false, kI32, condition);
2584       if (dst != true_value) __ Move(dst, true_value, kind);
2585       __ emit_jump(&cont);
2586 
2587       __ bind(&case_false);
2588       if (dst != false_value) __ Move(dst, false_value, kind);
2589       __ bind(&cont);
2590     }
2591     __ PushRegister(kind, dst);
2592   }
2593 
BrImpl(FullDecoder * decoder,Control * target)2594   void BrImpl(FullDecoder* decoder, Control* target) {
2595     if (dynamic_tiering()) {
2596       if (target->is_loop()) {
2597         DCHECK(target->label.get()->is_bound());
2598         int jump_distance = __ pc_offset() - target->label.get()->pos();
2599         // For now we just add one as the cost for the tier up check. We might
2600         // want to revisit this when tuning tiering budgets later.
2601         const int kTierUpCheckCost = 1;
2602         TierupCheck(decoder, decoder->position(),
2603                     jump_distance + kTierUpCheckCost);
2604       } else {
2605         // To estimate time spent in this function more accurately, we could
2606         // increment the tiering budget on forward jumps. However, we don't
2607         // know the jump distance yet; using a blanket value has been tried
2608         // and found to not make a difference.
2609       }
2610     }
2611     if (!target->br_merge()->reached) {
2612       target->label_state.InitMerge(
2613           *__ cache_state(), __ num_locals(), target->br_merge()->arity,
2614           target->stack_depth + target->num_exceptions);
2615     }
2616     __ MergeStackWith(target->label_state, target->br_merge()->arity,
2617                       target->is_loop() ? LiftoffAssembler::kBackwardJump
2618                                         : LiftoffAssembler::kForwardJump);
2619     __ jmp(target->label.get());
2620   }
2621 
BrOrRet(FullDecoder * decoder,uint32_t depth,uint32_t)2622   void BrOrRet(FullDecoder* decoder, uint32_t depth,
2623                uint32_t /* drop_values */) {
2624     BrOrRetImpl(decoder, depth);
2625   }
2626 
BrOrRetImpl(FullDecoder * decoder,uint32_t depth)2627   void BrOrRetImpl(FullDecoder* decoder, uint32_t depth) {
2628     if (depth == decoder->control_depth() - 1) {
2629       DoReturn(decoder, 0);
2630     } else {
2631       BrImpl(decoder, decoder->control_at(depth));
2632     }
2633   }
2634 
BrIf(FullDecoder * decoder,const Value &,uint32_t depth)2635   void BrIf(FullDecoder* decoder, const Value& /* cond */, uint32_t depth) {
2636     // Before branching, materialize all constants. This avoids repeatedly
2637     // materializing them for each conditional branch.
2638     // TODO(clemensb): Do the same for br_table.
2639     if (depth != decoder->control_depth() - 1) {
2640       __ MaterializeMergedConstants(
2641           decoder->control_at(depth)->br_merge()->arity);
2642     }
2643 
2644     Label cont_false;
2645 
2646     // Test the condition on the value stack, jump to {cont_false} if zero.
2647     JumpIfFalse(decoder, &cont_false);
2648 
2649     // As a quickfix for https://crbug.com/1314184 we store the cache state
2650     // before calling {BrOrRetImpl} under dynamic tiering, because the tier up
2651     // check modifies the cache state (GetUnusedRegister,
2652     // LoadInstanceIntoRegister).
2653     // TODO(wasm): This causes significant overhead during compilation; try to
2654     // avoid this, maybe by passing in scratch registers.
2655     if (dynamic_tiering()) {
2656       LiftoffAssembler::CacheState old_cache_state;
2657       old_cache_state.Split(*__ cache_state());
2658       BrOrRetImpl(decoder, depth);
2659       __ cache_state()->Steal(old_cache_state);
2660     } else {
2661       BrOrRetImpl(decoder, depth);
2662     }
2663 
2664     __ bind(&cont_false);
2665   }
2666 
2667   // Generate a branch table case, potentially reusing previously generated
2668   // stack transfer code.
GenerateBrCase(FullDecoder * decoder,uint32_t br_depth,std::map<uint32_t,MovableLabel> * br_targets)2669   void GenerateBrCase(FullDecoder* decoder, uint32_t br_depth,
2670                       std::map<uint32_t, MovableLabel>* br_targets) {
2671     MovableLabel& label = (*br_targets)[br_depth];
2672     if (label.get()->is_bound()) {
2673       __ jmp(label.get());
2674     } else {
2675       __ bind(label.get());
2676       BrOrRet(decoder, br_depth, 0);
2677     }
2678   }
2679 
2680   // Generate a branch table for input in [min, max).
2681   // TODO(wasm): Generate a real branch table (like TF TableSwitch).
GenerateBrTable(FullDecoder * decoder,LiftoffRegister tmp,LiftoffRegister value,uint32_t min,uint32_t max,BranchTableIterator<validate> * table_iterator,std::map<uint32_t,MovableLabel> * br_targets)2682   void GenerateBrTable(FullDecoder* decoder, LiftoffRegister tmp,
2683                        LiftoffRegister value, uint32_t min, uint32_t max,
2684                        BranchTableIterator<validate>* table_iterator,
2685                        std::map<uint32_t, MovableLabel>* br_targets) {
2686     DCHECK_LT(min, max);
2687     // Check base case.
2688     if (max == min + 1) {
2689       DCHECK_EQ(min, table_iterator->cur_index());
2690       GenerateBrCase(decoder, table_iterator->next(), br_targets);
2691       return;
2692     }
2693 
2694     uint32_t split = min + (max - min) / 2;
2695     Label upper_half;
2696     __ LoadConstant(tmp, WasmValue(split));
2697     __ emit_cond_jump(kUnsignedGreaterEqual, &upper_half, kI32, value.gp(),
2698                       tmp.gp());
2699     // Emit br table for lower half:
2700     GenerateBrTable(decoder, tmp, value, min, split, table_iterator,
2701                     br_targets);
2702     __ bind(&upper_half);
2703     // table_iterator will trigger a DCHECK if we don't stop decoding now.
2704     if (did_bailout()) return;
2705     // Emit br table for upper half:
2706     GenerateBrTable(decoder, tmp, value, split, max, table_iterator,
2707                     br_targets);
2708   }
2709 
BrTable(FullDecoder * decoder,const BranchTableImmediate<validate> & imm,const Value & key)2710   void BrTable(FullDecoder* decoder, const BranchTableImmediate<validate>& imm,
2711                const Value& key) {
2712     LiftoffRegList pinned;
2713     LiftoffRegister value = pinned.set(__ PopToRegister());
2714     BranchTableIterator<validate> table_iterator(decoder, imm);
2715     std::map<uint32_t, MovableLabel> br_targets;
2716 
2717     if (imm.table_count > 0) {
2718       LiftoffRegister tmp = __ GetUnusedRegister(kGpReg, pinned);
2719       __ LoadConstant(tmp, WasmValue(uint32_t{imm.table_count}));
2720       Label case_default;
2721       __ emit_cond_jump(kUnsignedGreaterEqual, &case_default, kI32, value.gp(),
2722                         tmp.gp());
2723 
2724       GenerateBrTable(decoder, tmp, value, 0, imm.table_count, &table_iterator,
2725                       &br_targets);
2726 
2727       __ bind(&case_default);
2728       // table_iterator will trigger a DCHECK if we don't stop decoding now.
2729       if (did_bailout()) return;
2730     }
2731 
2732     // Generate the default case.
2733     GenerateBrCase(decoder, table_iterator.next(), &br_targets);
2734     DCHECK(!table_iterator.has_next());
2735   }
2736 
Else(FullDecoder * decoder,Control * c)2737   void Else(FullDecoder* decoder, Control* c) {
2738     if (c->reachable()) {
2739       if (!c->end_merge.reached) {
2740         c->label_state.InitMerge(*__ cache_state(), __ num_locals(),
2741                                  c->end_merge.arity,
2742                                  c->stack_depth + c->num_exceptions);
2743       }
2744       __ MergeFullStackWith(c->label_state, *__ cache_state());
2745       __ emit_jump(c->label.get());
2746     }
2747     __ bind(c->else_state->label.get());
2748     __ cache_state()->Steal(c->else_state->state);
2749   }
2750 
GetSpilledRegistersForInspection()2751   SpilledRegistersForInspection* GetSpilledRegistersForInspection() {
2752     DCHECK(for_debugging_);
2753     // If we are generating debugging code, we really need to spill all
2754     // registers to make them inspectable when stopping at the trap.
2755     auto* spilled = compilation_zone_->New<SpilledRegistersForInspection>(
2756         compilation_zone_);
2757     for (uint32_t i = 0, e = __ cache_state()->stack_height(); i < e; ++i) {
2758       auto& slot = __ cache_state()->stack_state[i];
2759       if (!slot.is_reg()) continue;
2760       spilled->entries.push_back(SpilledRegistersForInspection::Entry{
2761           slot.offset(), slot.reg(), slot.kind()});
2762       __ RecordUsedSpillOffset(slot.offset());
2763     }
2764     return spilled;
2765   }
2766 
AddOutOfLineTrap(FullDecoder * decoder,WasmCode::RuntimeStubId stub,uint32_t pc=0)2767   Label* AddOutOfLineTrap(FullDecoder* decoder, WasmCode::RuntimeStubId stub,
2768                           uint32_t pc = 0) {
2769     // Only memory OOB traps need a {pc}.
2770     DCHECK_IMPLIES(stub != WasmCode::kThrowWasmTrapMemOutOfBounds, pc == 0);
2771     DCHECK(FLAG_wasm_bounds_checks);
2772     OutOfLineSafepointInfo* safepoint_info = nullptr;
2773     if (V8_UNLIKELY(for_debugging_)) {
2774       // Execution does not return after a trap. Therefore we don't have to
2775       // define a safepoint for traps that would preserve references on the
2776       // stack. However, if this is debug code, then we have to preserve the
2777       // references so that they can be inspected.
2778       safepoint_info =
2779           compilation_zone_->New<OutOfLineSafepointInfo>(compilation_zone_);
2780       __ cache_state()->GetTaggedSlotsForOOLCode(
2781           &safepoint_info->slots, &safepoint_info->spills,
2782           LiftoffAssembler::CacheState::SpillLocation::kStackSlots);
2783     }
2784     out_of_line_code_.push_back(OutOfLineCode::Trap(
2785         stub, decoder->position(),
2786         V8_UNLIKELY(for_debugging_) ? GetSpilledRegistersForInspection()
2787                                     : nullptr,
2788         safepoint_info, pc, RegisterOOLDebugSideTableEntry(decoder)));
2789     return out_of_line_code_.back().label.get();
2790   }
2791 
2792   enum ForceCheck : bool { kDoForceCheck = true, kDontForceCheck = false };
2793 
2794   // Returns {no_reg} if the memory access is statically known to be out of
2795   // bounds (a jump to the trap was generated then); return the GP {index}
2796   // register otherwise (holding the ptrsized index).
BoundsCheckMem(FullDecoder * decoder,uint32_t access_size,uint64_t offset,LiftoffRegister index,LiftoffRegList pinned,ForceCheck force_check)2797   Register BoundsCheckMem(FullDecoder* decoder, uint32_t access_size,
2798                           uint64_t offset, LiftoffRegister index,
2799                           LiftoffRegList pinned, ForceCheck force_check) {
2800     const bool statically_oob =
2801         !base::IsInBounds<uintptr_t>(offset, access_size,
2802                                      env_->max_memory_size);
2803 
2804     // After bounds checking, we know that the index must be ptrsize, hence only
2805     // look at the lower word on 32-bit systems (the high word is bounds-checked
2806     // further down).
2807     Register index_ptrsize =
2808         kNeedI64RegPair && index.is_gp_pair() ? index.low_gp() : index.gp();
2809 
2810     // Without bounds checks (testing only), just return the ptrsize index.
2811     if (V8_UNLIKELY(env_->bounds_checks == kNoBoundsChecks)) {
2812       return index_ptrsize;
2813     }
2814 
2815     // Early return for trap handler.
2816     DCHECK_IMPLIES(env_->module->is_memory64,
2817                    env_->bounds_checks == kExplicitBoundsChecks);
2818     if (!force_check && !statically_oob &&
2819         env_->bounds_checks == kTrapHandler) {
2820       // With trap handlers we should not have a register pair as input (we
2821       // would only return the lower half).
2822       DCHECK(index.is_gp());
2823       return index_ptrsize;
2824     }
2825 
2826     CODE_COMMENT("bounds check memory");
2827 
2828     // Set {pc} of the OOL code to {0} to avoid generation of protected
2829     // instruction information (see {GenerateOutOfLineCode}.
2830     Label* trap_label =
2831         AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapMemOutOfBounds, 0);
2832 
2833     if (V8_UNLIKELY(statically_oob)) {
2834       __ emit_jump(trap_label);
2835       decoder->SetSucceedingCodeDynamicallyUnreachable();
2836       return no_reg;
2837     }
2838 
2839     // Convert the index to ptrsize, bounds-checking the high word on 32-bit
2840     // systems for memory64.
2841     if (!env_->module->is_memory64) {
2842       __ emit_u32_to_uintptr(index_ptrsize, index_ptrsize);
2843     } else if (kSystemPointerSize == kInt32Size) {
2844       DCHECK_GE(kMaxUInt32, env_->max_memory_size);
2845       __ emit_cond_jump(kNotEqualZero, trap_label, kI32, index.high_gp());
2846     }
2847 
2848     uintptr_t end_offset = offset + access_size - 1u;
2849 
2850     pinned.set(index_ptrsize);
2851     LiftoffRegister end_offset_reg =
2852         pinned.set(__ GetUnusedRegister(kGpReg, pinned));
2853     LiftoffRegister mem_size = __ GetUnusedRegister(kGpReg, pinned);
2854     LOAD_INSTANCE_FIELD(mem_size.gp(), MemorySize, kSystemPointerSize, pinned);
2855 
2856     __ LoadConstant(end_offset_reg, WasmValue::ForUintPtr(end_offset));
2857 
2858     // If the end offset is larger than the smallest memory, dynamically check
2859     // the end offset against the actual memory size, which is not known at
2860     // compile time. Otherwise, only one check is required (see below).
2861     if (end_offset > env_->min_memory_size) {
2862       __ emit_cond_jump(kUnsignedGreaterEqual, trap_label, kPointerKind,
2863                         end_offset_reg.gp(), mem_size.gp());
2864     }
2865 
2866     // Just reuse the end_offset register for computing the effective size
2867     // (which is >= 0 because of the check above).
2868     LiftoffRegister effective_size_reg = end_offset_reg;
2869     __ emit_ptrsize_sub(effective_size_reg.gp(), mem_size.gp(),
2870                         end_offset_reg.gp());
2871 
2872     __ emit_cond_jump(kUnsignedGreaterEqual, trap_label, kPointerKind,
2873                       index_ptrsize, effective_size_reg.gp());
2874     return index_ptrsize;
2875   }
2876 
AlignmentCheckMem(FullDecoder * decoder,uint32_t access_size,uintptr_t offset,Register index,LiftoffRegList pinned)2877   void AlignmentCheckMem(FullDecoder* decoder, uint32_t access_size,
2878                          uintptr_t offset, Register index,
2879                          LiftoffRegList pinned) {
2880     CODE_COMMENT("alignment check");
2881     Label* trap_label =
2882         AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapUnalignedAccess, 0);
2883     Register address = __ GetUnusedRegister(kGpReg, pinned).gp();
2884 
2885     const uint32_t align_mask = access_size - 1;
2886     if ((offset & align_mask) == 0) {
2887       // If {offset} is aligned, we can produce faster code.
2888 
2889       // TODO(ahaas): On Intel, the "test" instruction implicitly computes the
2890       // AND of two operands. We could introduce a new variant of
2891       // {emit_cond_jump} to use the "test" instruction without the "and" here.
2892       // Then we can also avoid using the temp register here.
2893       __ emit_i32_andi(address, index, align_mask);
2894       __ emit_cond_jump(kUnequal, trap_label, kI32, address);
2895     } else {
2896       // For alignment checks we only look at the lower 32-bits in {offset}.
2897       __ emit_i32_addi(address, index, static_cast<uint32_t>(offset));
2898       __ emit_i32_andi(address, address, align_mask);
2899       __ emit_cond_jump(kUnequal, trap_label, kI32, address);
2900     }
2901   }
2902 
TraceMemoryOperation(bool is_store,MachineRepresentation rep,Register index,uintptr_t offset,WasmCodePosition position)2903   void TraceMemoryOperation(bool is_store, MachineRepresentation rep,
2904                             Register index, uintptr_t offset,
2905                             WasmCodePosition position) {
2906     // Before making the runtime call, spill all cache registers.
2907     __ SpillAllRegisters();
2908 
2909     LiftoffRegList pinned;
2910     if (index != no_reg) pinned.set(index);
2911     // Get one register for computing the effective offset (offset + index).
2912     LiftoffRegister effective_offset =
2913         pinned.set(__ GetUnusedRegister(kGpReg, pinned));
2914     DCHECK_GE(kMaxUInt32, offset);
2915     __ LoadConstant(effective_offset, WasmValue(static_cast<uint32_t>(offset)));
2916     if (index != no_reg) {
2917       // TODO(clemensb): Do a 64-bit addition here if memory64 is used.
2918       __ emit_i32_add(effective_offset.gp(), effective_offset.gp(), index);
2919     }
2920 
2921     // Get a register to hold the stack slot for MemoryTracingInfo.
2922     LiftoffRegister info = pinned.set(__ GetUnusedRegister(kGpReg, pinned));
2923     // Allocate stack slot for MemoryTracingInfo.
2924     __ AllocateStackSlot(info.gp(), sizeof(MemoryTracingInfo));
2925 
2926     // Reuse the {effective_offset} register for all information to be stored in
2927     // the MemoryTracingInfo struct.
2928     LiftoffRegister data = effective_offset;
2929 
2930     // Now store all information into the MemoryTracingInfo struct.
2931     if (kSystemPointerSize == 8) {
2932       // Zero-extend the effective offset to u64.
2933       CHECK(__ emit_type_conversion(kExprI64UConvertI32, data, effective_offset,
2934                                     nullptr));
2935     }
2936     __ Store(
2937         info.gp(), no_reg, offsetof(MemoryTracingInfo, offset), data,
2938         kSystemPointerSize == 8 ? StoreType::kI64Store : StoreType::kI32Store,
2939         pinned);
2940     __ LoadConstant(data, WasmValue(is_store ? 1 : 0));
2941     __ Store(info.gp(), no_reg, offsetof(MemoryTracingInfo, is_store), data,
2942              StoreType::kI32Store8, pinned);
2943     __ LoadConstant(data, WasmValue(static_cast<int>(rep)));
2944     __ Store(info.gp(), no_reg, offsetof(MemoryTracingInfo, mem_rep), data,
2945              StoreType::kI32Store8, pinned);
2946 
2947     WasmTraceMemoryDescriptor descriptor;
2948     DCHECK_EQ(0, descriptor.GetStackParameterCount());
2949     DCHECK_EQ(1, descriptor.GetRegisterParameterCount());
2950     Register param_reg = descriptor.GetRegisterParameter(0);
2951     if (info.gp() != param_reg) {
2952       __ Move(param_reg, info.gp(), kPointerKind);
2953     }
2954 
2955     source_position_table_builder_.AddPosition(__ pc_offset(),
2956                                                SourcePosition(position), false);
2957     __ CallRuntimeStub(WasmCode::kWasmTraceMemory);
2958     DefineSafepoint();
2959 
2960     __ DeallocateStackSlot(sizeof(MemoryTracingInfo));
2961   }
2962 
IndexStaticallyInBounds(const LiftoffAssembler::VarState & index_slot,int access_size,uintptr_t * offset)2963   bool IndexStaticallyInBounds(const LiftoffAssembler::VarState& index_slot,
2964                                int access_size, uintptr_t* offset) {
2965     if (!index_slot.is_const()) return false;
2966 
2967     // Potentially zero extend index (which is a 32-bit constant).
2968     const uintptr_t index = static_cast<uint32_t>(index_slot.i32_const());
2969     const uintptr_t effective_offset = index + *offset;
2970 
2971     if (effective_offset < index  // overflow
2972         || !base::IsInBounds<uintptr_t>(effective_offset, access_size,
2973                                         env_->min_memory_size)) {
2974       return false;
2975     }
2976 
2977     *offset = effective_offset;
2978     return true;
2979   }
2980 
GetMemoryStart(LiftoffRegList pinned)2981   Register GetMemoryStart(LiftoffRegList pinned) {
2982     Register memory_start = __ cache_state()->cached_mem_start;
2983     if (memory_start == no_reg) {
2984       memory_start = __ GetUnusedRegister(kGpReg, pinned).gp();
2985       LOAD_INSTANCE_FIELD(memory_start, MemoryStart, kSystemPointerSize,
2986                           pinned);
2987 #ifdef V8_SANDBOXED_POINTERS
2988       __ DecodeSandboxedPointer(memory_start);
2989 #endif
2990       __ cache_state()->SetMemStartCacheRegister(memory_start);
2991     }
2992     return memory_start;
2993   }
2994 
LoadMem(FullDecoder * decoder,LoadType type,const MemoryAccessImmediate<validate> & imm,const Value & index_val,Value * result)2995   void LoadMem(FullDecoder* decoder, LoadType type,
2996                const MemoryAccessImmediate<validate>& imm,
2997                const Value& index_val, Value* result) {
2998     ValueKind kind = type.value_type().kind();
2999     RegClass rc = reg_class_for(kind);
3000     if (!CheckSupportedType(decoder, kind, "load")) return;
3001 
3002     uintptr_t offset = imm.offset;
3003     Register index = no_reg;
3004 
3005     // Only look at the slot, do not pop it yet (will happen in PopToRegister
3006     // below, if this is not a statically-in-bounds index).
3007     auto& index_slot = __ cache_state()->stack_state.back();
3008     bool i64_offset = index_val.type == kWasmI64;
3009     if (IndexStaticallyInBounds(index_slot, type.size(), &offset)) {
3010       __ cache_state()->stack_state.pop_back();
3011       CODE_COMMENT("load from memory (constant offset)");
3012       LiftoffRegList pinned;
3013       Register mem = pinned.set(GetMemoryStart(pinned));
3014       LiftoffRegister value = pinned.set(__ GetUnusedRegister(rc, pinned));
3015       __ Load(value, mem, no_reg, offset, type, pinned, nullptr, true,
3016               i64_offset);
3017       __ PushRegister(kind, value);
3018     } else {
3019       LiftoffRegister full_index = __ PopToRegister();
3020       index = BoundsCheckMem(decoder, type.size(), offset, full_index, {},
3021                              kDontForceCheck);
3022       if (index == no_reg) return;
3023 
3024       CODE_COMMENT("load from memory");
3025       LiftoffRegList pinned = {index};
3026 
3027       // Load the memory start address only now to reduce register pressure
3028       // (important on ia32).
3029       Register mem = pinned.set(GetMemoryStart(pinned));
3030       LiftoffRegister value = pinned.set(__ GetUnusedRegister(rc, pinned));
3031 
3032       uint32_t protected_load_pc = 0;
3033       __ Load(value, mem, index, offset, type, pinned, &protected_load_pc, true,
3034               i64_offset);
3035       if (env_->bounds_checks == kTrapHandler) {
3036         AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapMemOutOfBounds,
3037                          protected_load_pc);
3038       }
3039       __ PushRegister(kind, value);
3040     }
3041 
3042     if (V8_UNLIKELY(FLAG_trace_wasm_memory)) {
3043       TraceMemoryOperation(false, type.mem_type().representation(), index,
3044                            offset, decoder->position());
3045     }
3046   }
3047 
LoadTransform(FullDecoder * decoder,LoadType type,LoadTransformationKind transform,const MemoryAccessImmediate<validate> & imm,const Value & index_val,Value * result)3048   void LoadTransform(FullDecoder* decoder, LoadType type,
3049                      LoadTransformationKind transform,
3050                      const MemoryAccessImmediate<validate>& imm,
3051                      const Value& index_val, Value* result) {
3052     // LoadTransform requires SIMD support, so check for it here. If
3053     // unsupported, bailout and let TurboFan lower the code.
3054     if (!CheckSupportedType(decoder, kS128, "LoadTransform")) {
3055       return;
3056     }
3057 
3058     LiftoffRegister full_index = __ PopToRegister();
3059     // For load splats and load zero, LoadType is the size of the load, and for
3060     // load extends, LoadType is the size of the lane, and it always loads 8
3061     // bytes.
3062     uint32_t access_size =
3063         transform == LoadTransformationKind::kExtend ? 8 : type.size();
3064     Register index = BoundsCheckMem(decoder, access_size, imm.offset,
3065                                     full_index, {}, kDontForceCheck);
3066     if (index == no_reg) return;
3067 
3068     uintptr_t offset = imm.offset;
3069     LiftoffRegList pinned = {index};
3070     CODE_COMMENT("load with transformation");
3071     Register addr = GetMemoryStart(pinned);
3072     LiftoffRegister value = __ GetUnusedRegister(reg_class_for(kS128), {});
3073     uint32_t protected_load_pc = 0;
3074     __ LoadTransform(value, addr, index, offset, type, transform,
3075                      &protected_load_pc);
3076 
3077     if (env_->bounds_checks == kTrapHandler) {
3078       AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapMemOutOfBounds,
3079                        protected_load_pc);
3080     }
3081     __ PushRegister(kS128, value);
3082 
3083     if (V8_UNLIKELY(FLAG_trace_wasm_memory)) {
3084       // Again load extend is different.
3085       MachineRepresentation mem_rep =
3086           transform == LoadTransformationKind::kExtend
3087               ? MachineRepresentation::kWord64
3088               : type.mem_type().representation();
3089       TraceMemoryOperation(false, mem_rep, index, offset, decoder->position());
3090     }
3091   }
3092 
LoadLane(FullDecoder * decoder,LoadType type,const Value & _value,const Value & _index,const MemoryAccessImmediate<validate> & imm,const uint8_t laneidx,Value * _result)3093   void LoadLane(FullDecoder* decoder, LoadType type, const Value& _value,
3094                 const Value& _index, const MemoryAccessImmediate<validate>& imm,
3095                 const uint8_t laneidx, Value* _result) {
3096     if (!CheckSupportedType(decoder, kS128, "LoadLane")) {
3097       return;
3098     }
3099 
3100     LiftoffRegList pinned;
3101     LiftoffRegister value = pinned.set(__ PopToRegister());
3102     LiftoffRegister full_index = __ PopToRegister();
3103     Register index = BoundsCheckMem(decoder, type.size(), imm.offset,
3104                                     full_index, pinned, kDontForceCheck);
3105     if (index == no_reg) return;
3106 
3107     uintptr_t offset = imm.offset;
3108     pinned.set(index);
3109     CODE_COMMENT("load lane");
3110     Register addr = GetMemoryStart(pinned);
3111     LiftoffRegister result = __ GetUnusedRegister(reg_class_for(kS128), {});
3112     uint32_t protected_load_pc = 0;
3113 
3114     __ LoadLane(result, value, addr, index, offset, type, laneidx,
3115                 &protected_load_pc);
3116     if (env_->bounds_checks == kTrapHandler) {
3117       AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapMemOutOfBounds,
3118                        protected_load_pc);
3119     }
3120 
3121     __ PushRegister(kS128, result);
3122 
3123     if (V8_UNLIKELY(FLAG_trace_wasm_memory)) {
3124       TraceMemoryOperation(false, type.mem_type().representation(), index,
3125                            offset, decoder->position());
3126     }
3127   }
3128 
StoreMem(FullDecoder * decoder,StoreType type,const MemoryAccessImmediate<validate> & imm,const Value & index_val,const Value & value_val)3129   void StoreMem(FullDecoder* decoder, StoreType type,
3130                 const MemoryAccessImmediate<validate>& imm,
3131                 const Value& index_val, const Value& value_val) {
3132     ValueKind kind = type.value_type().kind();
3133     if (!CheckSupportedType(decoder, kind, "store")) return;
3134 
3135     LiftoffRegList pinned;
3136     LiftoffRegister value = pinned.set(__ PopToRegister());
3137 
3138     uintptr_t offset = imm.offset;
3139     Register index = no_reg;
3140 
3141     auto& index_slot = __ cache_state()->stack_state.back();
3142     if (IndexStaticallyInBounds(index_slot, type.size(), &offset)) {
3143       __ cache_state()->stack_state.pop_back();
3144       CODE_COMMENT("store to memory (constant offset)");
3145       Register mem = pinned.set(GetMemoryStart(pinned));
3146       __ Store(mem, no_reg, offset, value, type, pinned, nullptr, true);
3147     } else {
3148       LiftoffRegister full_index = __ PopToRegister(pinned);
3149       index = BoundsCheckMem(decoder, type.size(), imm.offset, full_index,
3150                              pinned, kDontForceCheck);
3151       if (index == no_reg) return;
3152 
3153       pinned.set(index);
3154       CODE_COMMENT("store to memory");
3155       uint32_t protected_store_pc = 0;
3156       // Load the memory start address only now to reduce register pressure
3157       // (important on ia32).
3158       Register mem = pinned.set(GetMemoryStart(pinned));
3159       LiftoffRegList outer_pinned;
3160       if (V8_UNLIKELY(FLAG_trace_wasm_memory)) outer_pinned.set(index);
3161       __ Store(mem, index, offset, value, type, outer_pinned,
3162                &protected_store_pc, true);
3163       if (env_->bounds_checks == kTrapHandler) {
3164         AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapMemOutOfBounds,
3165                          protected_store_pc);
3166       }
3167     }
3168 
3169     if (V8_UNLIKELY(FLAG_trace_wasm_memory)) {
3170       TraceMemoryOperation(true, type.mem_rep(), index, offset,
3171                            decoder->position());
3172     }
3173   }
3174 
StoreLane(FullDecoder * decoder,StoreType type,const MemoryAccessImmediate<validate> & imm,const Value & _index,const Value & _value,const uint8_t lane)3175   void StoreLane(FullDecoder* decoder, StoreType type,
3176                  const MemoryAccessImmediate<validate>& imm,
3177                  const Value& _index, const Value& _value, const uint8_t lane) {
3178     if (!CheckSupportedType(decoder, kS128, "StoreLane")) return;
3179     LiftoffRegList pinned;
3180     LiftoffRegister value = pinned.set(__ PopToRegister());
3181     LiftoffRegister full_index = __ PopToRegister(pinned);
3182     Register index = BoundsCheckMem(decoder, type.size(), imm.offset,
3183                                     full_index, pinned, kDontForceCheck);
3184     if (index == no_reg) return;
3185 
3186     uintptr_t offset = imm.offset;
3187     pinned.set(index);
3188     CODE_COMMENT("store lane to memory");
3189     Register addr = pinned.set(GetMemoryStart(pinned));
3190     uint32_t protected_store_pc = 0;
3191     __ StoreLane(addr, index, offset, value, type, lane, &protected_store_pc);
3192     if (env_->bounds_checks == kTrapHandler) {
3193       AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapMemOutOfBounds,
3194                        protected_store_pc);
3195     }
3196     if (V8_UNLIKELY(FLAG_trace_wasm_memory)) {
3197       TraceMemoryOperation(true, type.mem_rep(), index, offset,
3198                            decoder->position());
3199     }
3200   }
3201 
CurrentMemoryPages(FullDecoder *,Value *)3202   void CurrentMemoryPages(FullDecoder* /* decoder */, Value* /* result */) {
3203     Register mem_size = __ GetUnusedRegister(kGpReg, {}).gp();
3204     LOAD_INSTANCE_FIELD(mem_size, MemorySize, kSystemPointerSize, {});
3205     __ emit_ptrsize_shri(mem_size, mem_size, kWasmPageSizeLog2);
3206     LiftoffRegister result{mem_size};
3207     if (env_->module->is_memory64 && kNeedI64RegPair) {
3208       LiftoffRegister high_word =
3209           __ GetUnusedRegister(kGpReg, LiftoffRegList{mem_size});
3210       // The high word is always 0 on 32-bit systems.
3211       __ LoadConstant(high_word, WasmValue{uint32_t{0}});
3212       result = LiftoffRegister::ForPair(mem_size, high_word.gp());
3213     }
3214     __ PushRegister(env_->module->is_memory64 ? kI64 : kI32, result);
3215   }
3216 
MemoryGrow(FullDecoder * decoder,const Value & value,Value * result_val)3217   void MemoryGrow(FullDecoder* decoder, const Value& value, Value* result_val) {
3218     // Pop the input, then spill all cache registers to make the runtime call.
3219     LiftoffRegList pinned;
3220     LiftoffRegister input = pinned.set(__ PopToRegister());
3221     __ SpillAllRegisters();
3222 
3223     LiftoffRegister result = pinned.set(__ GetUnusedRegister(kGpReg, pinned));
3224 
3225     Label done;
3226 
3227     if (env_->module->is_memory64) {
3228       // If the high word is not 0, this will always fail (would grow by
3229       // >=256TB). The int32_t value will be sign-extended below.
3230       __ LoadConstant(result, WasmValue(int32_t{-1}));
3231       if (kNeedI64RegPair) {
3232         __ emit_cond_jump(kUnequal /* neq */, &done, kI32, input.high_gp());
3233         input = input.low();
3234       } else {
3235         LiftoffRegister high_word = __ GetUnusedRegister(kGpReg, pinned);
3236         __ emit_i64_shri(high_word, input, 32);
3237         __ emit_cond_jump(kUnequal /* neq */, &done, kI32, high_word.gp());
3238       }
3239     }
3240 
3241     WasmMemoryGrowDescriptor descriptor;
3242     DCHECK_EQ(0, descriptor.GetStackParameterCount());
3243     DCHECK_EQ(1, descriptor.GetRegisterParameterCount());
3244     DCHECK_EQ(machine_type(kI32), descriptor.GetParameterType(0));
3245 
3246     Register param_reg = descriptor.GetRegisterParameter(0);
3247     if (input.gp() != param_reg) __ Move(param_reg, input.gp(), kI32);
3248 
3249     __ CallRuntimeStub(WasmCode::kWasmMemoryGrow);
3250     DefineSafepoint();
3251     RegisterDebugSideTableEntry(decoder, DebugSideTableBuilder::kDidSpill);
3252 
3253     if (kReturnRegister0 != result.gp()) {
3254       __ Move(result.gp(), kReturnRegister0, kI32);
3255     }
3256 
3257     __ bind(&done);
3258 
3259     if (env_->module->is_memory64) {
3260       LiftoffRegister result64 = result;
3261       if (kNeedI64RegPair) result64 = __ GetUnusedRegister(kGpRegPair, pinned);
3262       __ emit_type_conversion(kExprI64SConvertI32, result64, result, nullptr);
3263       __ PushRegister(kI64, result64);
3264     } else {
3265       __ PushRegister(kI32, result);
3266     }
3267   }
3268 
3269   base::OwnedVector<DebugSideTable::Entry::Value>
GetCurrentDebugSideTableEntries(FullDecoder * decoder,DebugSideTableBuilder::AssumeSpilling assume_spilling)3270   GetCurrentDebugSideTableEntries(
3271       FullDecoder* decoder,
3272       DebugSideTableBuilder::AssumeSpilling assume_spilling) {
3273     auto& stack_state = __ cache_state()->stack_state;
3274     auto values =
3275         base::OwnedVector<DebugSideTable::Entry::Value>::NewForOverwrite(
3276             stack_state.size());
3277 
3278     // For function calls, the decoder still has the arguments on the stack, but
3279     // Liftoff already popped them. Hence {decoder->stack_size()} can be bigger
3280     // than expected. Just ignore that and use the lower part only.
3281     DCHECK_LE(stack_state.size() - num_exceptions_,
3282               decoder->num_locals() + decoder->stack_size());
3283     int index = 0;
3284     int decoder_stack_index = decoder->stack_size();
3285     // Iterate the operand stack control block by control block, so that we can
3286     // handle the implicit exception value for try blocks.
3287     for (int j = decoder->control_depth() - 1; j >= 0; j--) {
3288       Control* control = decoder->control_at(j);
3289       Control* next_control = j > 0 ? decoder->control_at(j - 1) : nullptr;
3290       int end_index = next_control
3291                           ? next_control->stack_depth + __ num_locals() +
3292                                 next_control->num_exceptions
3293                           : __ cache_state()->stack_height();
3294       bool exception = control->is_try_catch() || control->is_try_catchall();
3295       for (; index < end_index; ++index) {
3296         auto& slot = stack_state[index];
3297         auto& value = values[index];
3298         value.index = index;
3299         ValueType type =
3300             index < static_cast<int>(__ num_locals())
3301                 ? decoder->local_type(index)
3302                 : exception ? ValueType::Ref(HeapType::kAny, kNonNullable)
3303                             : decoder->stack_value(decoder_stack_index--)->type;
3304         DCHECK(CheckCompatibleStackSlotTypes(slot.kind(), type.kind()));
3305         value.type = type;
3306         switch (slot.loc()) {
3307           case kIntConst:
3308             value.storage = DebugSideTable::Entry::kConstant;
3309             value.i32_const = slot.i32_const();
3310             break;
3311           case kRegister:
3312             DCHECK_NE(DebugSideTableBuilder::kDidSpill, assume_spilling);
3313             if (assume_spilling == DebugSideTableBuilder::kAllowRegisters) {
3314               value.storage = DebugSideTable::Entry::kRegister;
3315               value.reg_code = slot.reg().liftoff_code();
3316               break;
3317             }
3318             DCHECK_EQ(DebugSideTableBuilder::kAssumeSpilling, assume_spilling);
3319             V8_FALLTHROUGH;
3320           case kStack:
3321             value.storage = DebugSideTable::Entry::kStack;
3322             value.stack_offset = slot.offset();
3323             break;
3324         }
3325         exception = false;
3326       }
3327     }
3328     DCHECK_EQ(values.size(), index);
3329     return values;
3330   }
3331 
RegisterDebugSideTableEntry(FullDecoder * decoder,DebugSideTableBuilder::AssumeSpilling assume_spilling)3332   void RegisterDebugSideTableEntry(
3333       FullDecoder* decoder,
3334       DebugSideTableBuilder::AssumeSpilling assume_spilling) {
3335     if (V8_LIKELY(!debug_sidetable_builder_)) return;
3336     debug_sidetable_builder_->NewEntry(
3337         __ pc_offset(),
3338         GetCurrentDebugSideTableEntries(decoder, assume_spilling).as_vector());
3339   }
3340 
RegisterOOLDebugSideTableEntry(FullDecoder * decoder)3341   DebugSideTableBuilder::EntryBuilder* RegisterOOLDebugSideTableEntry(
3342       FullDecoder* decoder) {
3343     if (V8_LIKELY(!debug_sidetable_builder_)) return nullptr;
3344     return debug_sidetable_builder_->NewOOLEntry(
3345         GetCurrentDebugSideTableEntries(decoder,
3346                                         DebugSideTableBuilder::kAssumeSpilling)
3347             .as_vector());
3348   }
3349 
3350   enum TailCall : bool { kTailCall = true, kNoTailCall = false };
3351 
CallDirect(FullDecoder * decoder,const CallFunctionImmediate<validate> & imm,const Value args[],Value[])3352   void CallDirect(FullDecoder* decoder,
3353                   const CallFunctionImmediate<validate>& imm,
3354                   const Value args[], Value[]) {
3355     CallDirect(decoder, imm, args, nullptr, kNoTailCall);
3356   }
3357 
CallIndirect(FullDecoder * decoder,const Value & index_val,const CallIndirectImmediate<validate> & imm,const Value args[],Value returns[])3358   void CallIndirect(FullDecoder* decoder, const Value& index_val,
3359                     const CallIndirectImmediate<validate>& imm,
3360                     const Value args[], Value returns[]) {
3361     CallIndirect(decoder, index_val, imm, kNoTailCall);
3362   }
3363 
CallRef(FullDecoder * decoder,const Value & func_ref,const FunctionSig * sig,uint32_t sig_index,const Value args[],Value returns[])3364   void CallRef(FullDecoder* decoder, const Value& func_ref,
3365                const FunctionSig* sig, uint32_t sig_index, const Value args[],
3366                Value returns[]) {
3367     CallRef(decoder, func_ref.type, sig, kNoTailCall);
3368   }
3369 
ReturnCall(FullDecoder * decoder,const CallFunctionImmediate<validate> & imm,const Value args[])3370   void ReturnCall(FullDecoder* decoder,
3371                   const CallFunctionImmediate<validate>& imm,
3372                   const Value args[]) {
3373     TierupCheckOnExit(decoder);
3374     CallDirect(decoder, imm, args, nullptr, kTailCall);
3375   }
3376 
ReturnCallIndirect(FullDecoder * decoder,const Value & index_val,const CallIndirectImmediate<validate> & imm,const Value args[])3377   void ReturnCallIndirect(FullDecoder* decoder, const Value& index_val,
3378                           const CallIndirectImmediate<validate>& imm,
3379                           const Value args[]) {
3380     TierupCheckOnExit(decoder);
3381     CallIndirect(decoder, index_val, imm, kTailCall);
3382   }
3383 
ReturnCallRef(FullDecoder * decoder,const Value & func_ref,const FunctionSig * sig,uint32_t sig_index,const Value args[])3384   void ReturnCallRef(FullDecoder* decoder, const Value& func_ref,
3385                      const FunctionSig* sig, uint32_t sig_index,
3386                      const Value args[]) {
3387     TierupCheckOnExit(decoder);
3388     CallRef(decoder, func_ref.type, sig, kTailCall);
3389   }
3390 
BrOnNull(FullDecoder * decoder,const Value & ref_object,uint32_t depth,bool pass_null_along_branch,Value *)3391   void BrOnNull(FullDecoder* decoder, const Value& ref_object, uint32_t depth,
3392                 bool pass_null_along_branch,
3393                 Value* /* result_on_fallthrough */) {
3394     // Before branching, materialize all constants. This avoids repeatedly
3395     // materializing them for each conditional branch.
3396     if (depth != decoder->control_depth() - 1) {
3397       __ MaterializeMergedConstants(
3398           decoder->control_at(depth)->br_merge()->arity);
3399     }
3400 
3401     Label cont_false;
3402     LiftoffRegList pinned;
3403     LiftoffRegister ref = pinned.set(__ PopToRegister(pinned));
3404     Register null = __ GetUnusedRegister(kGpReg, pinned).gp();
3405     LoadNullValue(null, pinned);
3406     __ emit_cond_jump(kUnequal, &cont_false, ref_object.type.kind(), ref.gp(),
3407                       null);
3408     if (pass_null_along_branch) LoadNullValue(null, pinned);
3409     BrOrRet(decoder, depth, 0);
3410     __ bind(&cont_false);
3411     __ PushRegister(kRef, ref);
3412   }
3413 
BrOnNonNull(FullDecoder * decoder,const Value & ref_object,uint32_t depth)3414   void BrOnNonNull(FullDecoder* decoder, const Value& ref_object,
3415                    uint32_t depth) {
3416     // Before branching, materialize all constants. This avoids repeatedly
3417     // materializing them for each conditional branch.
3418     if (depth != decoder->control_depth() - 1) {
3419       __ MaterializeMergedConstants(
3420           decoder->control_at(depth)->br_merge()->arity);
3421     }
3422 
3423     Label cont_false;
3424     LiftoffRegList pinned;
3425     LiftoffRegister ref = pinned.set(__ PopToRegister(pinned));
3426     // Put the reference back onto the stack for the branch.
3427     __ PushRegister(kRef, ref);
3428 
3429     Register null = __ GetUnusedRegister(kGpReg, pinned).gp();
3430     LoadNullValue(null, pinned);
3431     __ emit_cond_jump(kEqual, &cont_false, ref_object.type.kind(), ref.gp(),
3432                       null);
3433 
3434     BrOrRet(decoder, depth, 0);
3435     // Drop the reference if we are not branching.
3436     __ DropValues(1);
3437     __ bind(&cont_false);
3438   }
3439 
3440   template <ValueKind src_kind, ValueKind result_kind,
3441             ValueKind result_lane_kind = kVoid, typename EmitFn>
EmitTerOp(EmitFn fn)3442   void EmitTerOp(EmitFn fn) {
3443     static constexpr RegClass src_rc = reg_class_for(src_kind);
3444     static constexpr RegClass result_rc = reg_class_for(result_kind);
3445     LiftoffRegister src3 = __ PopToRegister();
3446     LiftoffRegister src2 = __ PopToRegister(LiftoffRegList{src3});
3447     LiftoffRegister src1 = __ PopToRegister(LiftoffRegList{src3, src2});
3448     // Reusing src1 and src2 will complicate codegen for select for some
3449     // backend, so we allow only reusing src3 (the mask), and pin src1 and src2.
3450     LiftoffRegister dst = src_rc == result_rc
3451                               ? __ GetUnusedRegister(result_rc, {src3},
3452                                                      LiftoffRegList{src1, src2})
3453                               : __ GetUnusedRegister(result_rc, {});
3454     CallEmitFn(fn, dst, src1, src2, src3);
3455     if (V8_UNLIKELY(nondeterminism_)) {
3456       LiftoffRegList pinned = {dst};
3457       if (result_kind == ValueKind::kF32 || result_kind == ValueKind::kF64) {
3458         CheckNan(dst, pinned, result_kind);
3459       } else if (result_kind == ValueKind::kS128 &&
3460                  (result_lane_kind == kF32 || result_lane_kind == kF64)) {
3461         CheckS128Nan(dst, LiftoffRegList{src1, src2, src3, dst},
3462                      result_lane_kind);
3463       }
3464     }
3465     __ PushRegister(result_kind, dst);
3466   }
3467 
3468   template <typename EmitFn, typename EmitFnImm>
EmitSimdShiftOp(EmitFn fn,EmitFnImm fnImm)3469   void EmitSimdShiftOp(EmitFn fn, EmitFnImm fnImm) {
3470     static constexpr RegClass result_rc = reg_class_for(kS128);
3471 
3472     LiftoffAssembler::VarState rhs_slot = __ cache_state()->stack_state.back();
3473     // Check if the RHS is an immediate.
3474     if (rhs_slot.is_const()) {
3475       __ cache_state()->stack_state.pop_back();
3476       int32_t imm = rhs_slot.i32_const();
3477 
3478       LiftoffRegister operand = __ PopToRegister();
3479       LiftoffRegister dst = __ GetUnusedRegister(result_rc, {operand}, {});
3480 
3481       CallEmitFn(fnImm, dst, operand, imm);
3482       __ PushRegister(kS128, dst);
3483     } else {
3484       LiftoffRegister count = __ PopToRegister();
3485       LiftoffRegister operand = __ PopToRegister();
3486       LiftoffRegister dst = __ GetUnusedRegister(result_rc, {operand}, {});
3487 
3488       CallEmitFn(fn, dst, operand, count);
3489       __ PushRegister(kS128, dst);
3490     }
3491   }
3492 
3493   template <ValueKind result_lane_kind>
EmitSimdFloatRoundingOpWithCFallback(bool (LiftoffAssembler::* emit_fn)(LiftoffRegister,LiftoffRegister),ExternalReference (* ext_ref)())3494   void EmitSimdFloatRoundingOpWithCFallback(
3495       bool (LiftoffAssembler::*emit_fn)(LiftoffRegister, LiftoffRegister),
3496       ExternalReference (*ext_ref)()) {
3497     static constexpr RegClass rc = reg_class_for(kS128);
3498     LiftoffRegister src = __ PopToRegister();
3499     LiftoffRegister dst = __ GetUnusedRegister(rc, {src}, {});
3500     if (!(asm_.*emit_fn)(dst, src)) {
3501       // Return v128 via stack for ARM.
3502       auto sig_v_s = MakeSig::Params(kS128);
3503       GenerateCCall(&dst, &sig_v_s, kS128, &src, ext_ref());
3504     }
3505     if (V8_UNLIKELY(nondeterminism_)) {
3506       LiftoffRegList pinned = {dst};
3507       CheckS128Nan(dst, pinned, result_lane_kind);
3508     }
3509     __ PushRegister(kS128, dst);
3510   }
3511 
SimdOp(FullDecoder * decoder,WasmOpcode opcode,base::Vector<Value> args,Value * result)3512   void SimdOp(FullDecoder* decoder, WasmOpcode opcode, base::Vector<Value> args,
3513               Value* result) {
3514     if (!CpuFeatures::SupportsWasmSimd128()) {
3515       return unsupported(decoder, kSimd, "simd");
3516     }
3517     switch (opcode) {
3518       case wasm::kExprI8x16Swizzle:
3519         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i8x16_swizzle);
3520       case wasm::kExprI8x16Popcnt:
3521         return EmitUnOp<kS128, kS128>(&LiftoffAssembler::emit_i8x16_popcnt);
3522       case wasm::kExprI8x16Splat:
3523         return EmitUnOp<kI32, kS128>(&LiftoffAssembler::emit_i8x16_splat);
3524       case wasm::kExprI16x8Splat:
3525         return EmitUnOp<kI32, kS128>(&LiftoffAssembler::emit_i16x8_splat);
3526       case wasm::kExprI32x4Splat:
3527         return EmitUnOp<kI32, kS128>(&LiftoffAssembler::emit_i32x4_splat);
3528       case wasm::kExprI64x2Splat:
3529         return EmitUnOp<kI64, kS128>(&LiftoffAssembler::emit_i64x2_splat);
3530       case wasm::kExprF32x4Splat:
3531         return EmitUnOp<kF32, kS128, kF32>(&LiftoffAssembler::emit_f32x4_splat);
3532       case wasm::kExprF64x2Splat:
3533         return EmitUnOp<kF64, kS128, kF64>(&LiftoffAssembler::emit_f64x2_splat);
3534       case wasm::kExprI8x16Eq:
3535         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i8x16_eq);
3536       case wasm::kExprI8x16Ne:
3537         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i8x16_ne);
3538       case wasm::kExprI8x16LtS:
3539         return EmitBinOp<kS128, kS128, true>(
3540             &LiftoffAssembler::emit_i8x16_gt_s);
3541       case wasm::kExprI8x16LtU:
3542         return EmitBinOp<kS128, kS128, true>(
3543             &LiftoffAssembler::emit_i8x16_gt_u);
3544       case wasm::kExprI8x16GtS:
3545         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i8x16_gt_s);
3546       case wasm::kExprI8x16GtU:
3547         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i8x16_gt_u);
3548       case wasm::kExprI8x16LeS:
3549         return EmitBinOp<kS128, kS128, true>(
3550             &LiftoffAssembler::emit_i8x16_ge_s);
3551       case wasm::kExprI8x16LeU:
3552         return EmitBinOp<kS128, kS128, true>(
3553             &LiftoffAssembler::emit_i8x16_ge_u);
3554       case wasm::kExprI8x16GeS:
3555         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i8x16_ge_s);
3556       case wasm::kExprI8x16GeU:
3557         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i8x16_ge_u);
3558       case wasm::kExprI16x8Eq:
3559         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i16x8_eq);
3560       case wasm::kExprI16x8Ne:
3561         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i16x8_ne);
3562       case wasm::kExprI16x8LtS:
3563         return EmitBinOp<kS128, kS128, true>(
3564             &LiftoffAssembler::emit_i16x8_gt_s);
3565       case wasm::kExprI16x8LtU:
3566         return EmitBinOp<kS128, kS128, true>(
3567             &LiftoffAssembler::emit_i16x8_gt_u);
3568       case wasm::kExprI16x8GtS:
3569         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i16x8_gt_s);
3570       case wasm::kExprI16x8GtU:
3571         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i16x8_gt_u);
3572       case wasm::kExprI16x8LeS:
3573         return EmitBinOp<kS128, kS128, true>(
3574             &LiftoffAssembler::emit_i16x8_ge_s);
3575       case wasm::kExprI16x8LeU:
3576         return EmitBinOp<kS128, kS128, true>(
3577             &LiftoffAssembler::emit_i16x8_ge_u);
3578       case wasm::kExprI16x8GeS:
3579         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i16x8_ge_s);
3580       case wasm::kExprI16x8GeU:
3581         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i16x8_ge_u);
3582       case wasm::kExprI32x4Eq:
3583         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i32x4_eq);
3584       case wasm::kExprI32x4Ne:
3585         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i32x4_ne);
3586       case wasm::kExprI32x4LtS:
3587         return EmitBinOp<kS128, kS128, true>(
3588             &LiftoffAssembler::emit_i32x4_gt_s);
3589       case wasm::kExprI32x4LtU:
3590         return EmitBinOp<kS128, kS128, true>(
3591             &LiftoffAssembler::emit_i32x4_gt_u);
3592       case wasm::kExprI32x4GtS:
3593         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i32x4_gt_s);
3594       case wasm::kExprI32x4GtU:
3595         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i32x4_gt_u);
3596       case wasm::kExprI32x4LeS:
3597         return EmitBinOp<kS128, kS128, true>(
3598             &LiftoffAssembler::emit_i32x4_ge_s);
3599       case wasm::kExprI32x4LeU:
3600         return EmitBinOp<kS128, kS128, true>(
3601             &LiftoffAssembler::emit_i32x4_ge_u);
3602       case wasm::kExprI32x4GeS:
3603         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i32x4_ge_s);
3604       case wasm::kExprI32x4GeU:
3605         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i32x4_ge_u);
3606       case wasm::kExprI64x2Eq:
3607         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i64x2_eq);
3608       case wasm::kExprI64x2Ne:
3609         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i64x2_ne);
3610       case wasm::kExprI64x2LtS:
3611         return EmitBinOp<kS128, kS128, true>(
3612             &LiftoffAssembler::emit_i64x2_gt_s);
3613       case wasm::kExprI64x2GtS:
3614         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i64x2_gt_s);
3615       case wasm::kExprI64x2LeS:
3616         return EmitBinOp<kS128, kS128, true>(
3617             &LiftoffAssembler::emit_i64x2_ge_s);
3618       case wasm::kExprI64x2GeS:
3619         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i64x2_ge_s);
3620       case wasm::kExprF32x4Eq:
3621         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_f32x4_eq);
3622       case wasm::kExprF32x4Ne:
3623         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_f32x4_ne);
3624       case wasm::kExprF32x4Lt:
3625         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_f32x4_lt);
3626       case wasm::kExprF32x4Gt:
3627         return EmitBinOp<kS128, kS128, true>(&LiftoffAssembler::emit_f32x4_lt);
3628       case wasm::kExprF32x4Le:
3629         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_f32x4_le);
3630       case wasm::kExprF32x4Ge:
3631         return EmitBinOp<kS128, kS128, true>(&LiftoffAssembler::emit_f32x4_le);
3632       case wasm::kExprF64x2Eq:
3633         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_f64x2_eq);
3634       case wasm::kExprF64x2Ne:
3635         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_f64x2_ne);
3636       case wasm::kExprF64x2Lt:
3637         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_f64x2_lt);
3638       case wasm::kExprF64x2Gt:
3639         return EmitBinOp<kS128, kS128, true>(&LiftoffAssembler::emit_f64x2_lt);
3640       case wasm::kExprF64x2Le:
3641         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_f64x2_le);
3642       case wasm::kExprF64x2Ge:
3643         return EmitBinOp<kS128, kS128, true>(&LiftoffAssembler::emit_f64x2_le);
3644       case wasm::kExprS128Not:
3645         return EmitUnOp<kS128, kS128>(&LiftoffAssembler::emit_s128_not);
3646       case wasm::kExprS128And:
3647         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_s128_and);
3648       case wasm::kExprS128Or:
3649         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_s128_or);
3650       case wasm::kExprS128Xor:
3651         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_s128_xor);
3652       case wasm::kExprS128Select:
3653         return EmitTerOp<kS128, kS128>(&LiftoffAssembler::emit_s128_select);
3654       case wasm::kExprI8x16Neg:
3655         return EmitUnOp<kS128, kS128>(&LiftoffAssembler::emit_i8x16_neg);
3656       case wasm::kExprV128AnyTrue:
3657         return EmitUnOp<kS128, kI32>(&LiftoffAssembler::emit_v128_anytrue);
3658       case wasm::kExprI8x16AllTrue:
3659         return EmitUnOp<kS128, kI32>(&LiftoffAssembler::emit_i8x16_alltrue);
3660       case wasm::kExprI8x16BitMask:
3661         return EmitUnOp<kS128, kI32>(&LiftoffAssembler::emit_i8x16_bitmask);
3662       case wasm::kExprI8x16Shl:
3663         return EmitSimdShiftOp(&LiftoffAssembler::emit_i8x16_shl,
3664                                &LiftoffAssembler::emit_i8x16_shli);
3665       case wasm::kExprI8x16ShrS:
3666         return EmitSimdShiftOp(&LiftoffAssembler::emit_i8x16_shr_s,
3667                                &LiftoffAssembler::emit_i8x16_shri_s);
3668       case wasm::kExprI8x16ShrU:
3669         return EmitSimdShiftOp(&LiftoffAssembler::emit_i8x16_shr_u,
3670                                &LiftoffAssembler::emit_i8x16_shri_u);
3671       case wasm::kExprI8x16Add:
3672         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i8x16_add);
3673       case wasm::kExprI8x16AddSatS:
3674         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i8x16_add_sat_s);
3675       case wasm::kExprI8x16AddSatU:
3676         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i8x16_add_sat_u);
3677       case wasm::kExprI8x16Sub:
3678         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i8x16_sub);
3679       case wasm::kExprI8x16SubSatS:
3680         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i8x16_sub_sat_s);
3681       case wasm::kExprI8x16SubSatU:
3682         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i8x16_sub_sat_u);
3683       case wasm::kExprI8x16MinS:
3684         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i8x16_min_s);
3685       case wasm::kExprI8x16MinU:
3686         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i8x16_min_u);
3687       case wasm::kExprI8x16MaxS:
3688         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i8x16_max_s);
3689       case wasm::kExprI8x16MaxU:
3690         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i8x16_max_u);
3691       case wasm::kExprI16x8Neg:
3692         return EmitUnOp<kS128, kS128>(&LiftoffAssembler::emit_i16x8_neg);
3693       case wasm::kExprI16x8AllTrue:
3694         return EmitUnOp<kS128, kI32>(&LiftoffAssembler::emit_i16x8_alltrue);
3695       case wasm::kExprI16x8BitMask:
3696         return EmitUnOp<kS128, kI32>(&LiftoffAssembler::emit_i16x8_bitmask);
3697       case wasm::kExprI16x8Shl:
3698         return EmitSimdShiftOp(&LiftoffAssembler::emit_i16x8_shl,
3699                                &LiftoffAssembler::emit_i16x8_shli);
3700       case wasm::kExprI16x8ShrS:
3701         return EmitSimdShiftOp(&LiftoffAssembler::emit_i16x8_shr_s,
3702                                &LiftoffAssembler::emit_i16x8_shri_s);
3703       case wasm::kExprI16x8ShrU:
3704         return EmitSimdShiftOp(&LiftoffAssembler::emit_i16x8_shr_u,
3705                                &LiftoffAssembler::emit_i16x8_shri_u);
3706       case wasm::kExprI16x8Add:
3707         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i16x8_add);
3708       case wasm::kExprI16x8AddSatS:
3709         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i16x8_add_sat_s);
3710       case wasm::kExprI16x8AddSatU:
3711         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i16x8_add_sat_u);
3712       case wasm::kExprI16x8Sub:
3713         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i16x8_sub);
3714       case wasm::kExprI16x8SubSatS:
3715         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i16x8_sub_sat_s);
3716       case wasm::kExprI16x8SubSatU:
3717         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i16x8_sub_sat_u);
3718       case wasm::kExprI16x8Mul:
3719         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i16x8_mul);
3720       case wasm::kExprI16x8MinS:
3721         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i16x8_min_s);
3722       case wasm::kExprI16x8MinU:
3723         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i16x8_min_u);
3724       case wasm::kExprI16x8MaxS:
3725         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i16x8_max_s);
3726       case wasm::kExprI16x8MaxU:
3727         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i16x8_max_u);
3728       case wasm::kExprI16x8ExtAddPairwiseI8x16S:
3729         return EmitUnOp<kS128, kS128>(
3730             &LiftoffAssembler::emit_i16x8_extadd_pairwise_i8x16_s);
3731       case wasm::kExprI16x8ExtAddPairwiseI8x16U:
3732         return EmitUnOp<kS128, kS128>(
3733             &LiftoffAssembler::emit_i16x8_extadd_pairwise_i8x16_u);
3734       case wasm::kExprI16x8ExtMulLowI8x16S:
3735         return EmitBinOp<kS128, kS128>(
3736             &LiftoffAssembler::emit_i16x8_extmul_low_i8x16_s);
3737       case wasm::kExprI16x8ExtMulLowI8x16U:
3738         return EmitBinOp<kS128, kS128>(
3739             &LiftoffAssembler::emit_i16x8_extmul_low_i8x16_u);
3740       case wasm::kExprI16x8ExtMulHighI8x16S:
3741         return EmitBinOp<kS128, kS128>(
3742             &LiftoffAssembler::emit_i16x8_extmul_high_i8x16_s);
3743       case wasm::kExprI16x8ExtMulHighI8x16U:
3744         return EmitBinOp<kS128, kS128>(
3745             &LiftoffAssembler::emit_i16x8_extmul_high_i8x16_u);
3746       case wasm::kExprI16x8Q15MulRSatS:
3747         return EmitBinOp<kS128, kS128>(
3748             &LiftoffAssembler::emit_i16x8_q15mulr_sat_s);
3749       case wasm::kExprI32x4Neg:
3750         return EmitUnOp<kS128, kS128>(&LiftoffAssembler::emit_i32x4_neg);
3751       case wasm::kExprI32x4AllTrue:
3752         return EmitUnOp<kS128, kI32>(&LiftoffAssembler::emit_i32x4_alltrue);
3753       case wasm::kExprI32x4BitMask:
3754         return EmitUnOp<kS128, kI32>(&LiftoffAssembler::emit_i32x4_bitmask);
3755       case wasm::kExprI32x4Shl:
3756         return EmitSimdShiftOp(&LiftoffAssembler::emit_i32x4_shl,
3757                                &LiftoffAssembler::emit_i32x4_shli);
3758       case wasm::kExprI32x4ShrS:
3759         return EmitSimdShiftOp(&LiftoffAssembler::emit_i32x4_shr_s,
3760                                &LiftoffAssembler::emit_i32x4_shri_s);
3761       case wasm::kExprI32x4ShrU:
3762         return EmitSimdShiftOp(&LiftoffAssembler::emit_i32x4_shr_u,
3763                                &LiftoffAssembler::emit_i32x4_shri_u);
3764       case wasm::kExprI32x4Add:
3765         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i32x4_add);
3766       case wasm::kExprI32x4Sub:
3767         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i32x4_sub);
3768       case wasm::kExprI32x4Mul:
3769         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i32x4_mul);
3770       case wasm::kExprI32x4MinS:
3771         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i32x4_min_s);
3772       case wasm::kExprI32x4MinU:
3773         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i32x4_min_u);
3774       case wasm::kExprI32x4MaxS:
3775         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i32x4_max_s);
3776       case wasm::kExprI32x4MaxU:
3777         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i32x4_max_u);
3778       case wasm::kExprI32x4DotI16x8S:
3779         return EmitBinOp<kS128, kS128>(
3780             &LiftoffAssembler::emit_i32x4_dot_i16x8_s);
3781       case wasm::kExprI32x4ExtAddPairwiseI16x8S:
3782         return EmitUnOp<kS128, kS128>(
3783             &LiftoffAssembler::emit_i32x4_extadd_pairwise_i16x8_s);
3784       case wasm::kExprI32x4ExtAddPairwiseI16x8U:
3785         return EmitUnOp<kS128, kS128>(
3786             &LiftoffAssembler::emit_i32x4_extadd_pairwise_i16x8_u);
3787       case wasm::kExprI32x4ExtMulLowI16x8S:
3788         return EmitBinOp<kS128, kS128>(
3789             &LiftoffAssembler::emit_i32x4_extmul_low_i16x8_s);
3790       case wasm::kExprI32x4ExtMulLowI16x8U:
3791         return EmitBinOp<kS128, kS128>(
3792             &LiftoffAssembler::emit_i32x4_extmul_low_i16x8_u);
3793       case wasm::kExprI32x4ExtMulHighI16x8S:
3794         return EmitBinOp<kS128, kS128>(
3795             &LiftoffAssembler::emit_i32x4_extmul_high_i16x8_s);
3796       case wasm::kExprI32x4ExtMulHighI16x8U:
3797         return EmitBinOp<kS128, kS128>(
3798             &LiftoffAssembler::emit_i32x4_extmul_high_i16x8_u);
3799       case wasm::kExprI64x2Neg:
3800         return EmitUnOp<kS128, kS128>(&LiftoffAssembler::emit_i64x2_neg);
3801       case wasm::kExprI64x2AllTrue:
3802         return EmitUnOp<kS128, kI32>(&LiftoffAssembler::emit_i64x2_alltrue);
3803       case wasm::kExprI64x2Shl:
3804         return EmitSimdShiftOp(&LiftoffAssembler::emit_i64x2_shl,
3805                                &LiftoffAssembler::emit_i64x2_shli);
3806       case wasm::kExprI64x2ShrS:
3807         return EmitSimdShiftOp(&LiftoffAssembler::emit_i64x2_shr_s,
3808                                &LiftoffAssembler::emit_i64x2_shri_s);
3809       case wasm::kExprI64x2ShrU:
3810         return EmitSimdShiftOp(&LiftoffAssembler::emit_i64x2_shr_u,
3811                                &LiftoffAssembler::emit_i64x2_shri_u);
3812       case wasm::kExprI64x2Add:
3813         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i64x2_add);
3814       case wasm::kExprI64x2Sub:
3815         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i64x2_sub);
3816       case wasm::kExprI64x2Mul:
3817         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i64x2_mul);
3818       case wasm::kExprI64x2ExtMulLowI32x4S:
3819         return EmitBinOp<kS128, kS128>(
3820             &LiftoffAssembler::emit_i64x2_extmul_low_i32x4_s);
3821       case wasm::kExprI64x2ExtMulLowI32x4U:
3822         return EmitBinOp<kS128, kS128>(
3823             &LiftoffAssembler::emit_i64x2_extmul_low_i32x4_u);
3824       case wasm::kExprI64x2ExtMulHighI32x4S:
3825         return EmitBinOp<kS128, kS128>(
3826             &LiftoffAssembler::emit_i64x2_extmul_high_i32x4_s);
3827       case wasm::kExprI64x2ExtMulHighI32x4U:
3828         return EmitBinOp<kS128, kS128>(
3829             &LiftoffAssembler::emit_i64x2_extmul_high_i32x4_u);
3830       case wasm::kExprI64x2BitMask:
3831         return EmitUnOp<kS128, kI32>(&LiftoffAssembler::emit_i64x2_bitmask);
3832       case wasm::kExprI64x2SConvertI32x4Low:
3833         return EmitUnOp<kS128, kS128>(
3834             &LiftoffAssembler::emit_i64x2_sconvert_i32x4_low);
3835       case wasm::kExprI64x2SConvertI32x4High:
3836         return EmitUnOp<kS128, kS128>(
3837             &LiftoffAssembler::emit_i64x2_sconvert_i32x4_high);
3838       case wasm::kExprI64x2UConvertI32x4Low:
3839         return EmitUnOp<kS128, kS128>(
3840             &LiftoffAssembler::emit_i64x2_uconvert_i32x4_low);
3841       case wasm::kExprI64x2UConvertI32x4High:
3842         return EmitUnOp<kS128, kS128>(
3843             &LiftoffAssembler::emit_i64x2_uconvert_i32x4_high);
3844       case wasm::kExprF32x4Abs:
3845         return EmitUnOp<kS128, kS128, kF32>(&LiftoffAssembler::emit_f32x4_abs);
3846       case wasm::kExprF32x4Neg:
3847         return EmitUnOp<kS128, kS128, kF32>(&LiftoffAssembler::emit_f32x4_neg);
3848       case wasm::kExprF32x4Sqrt:
3849         return EmitUnOp<kS128, kS128, kF32>(&LiftoffAssembler::emit_f32x4_sqrt);
3850       case wasm::kExprF32x4Ceil:
3851         return EmitSimdFloatRoundingOpWithCFallback<kF32>(
3852             &LiftoffAssembler::emit_f32x4_ceil,
3853             &ExternalReference::wasm_f32x4_ceil);
3854       case wasm::kExprF32x4Floor:
3855         return EmitSimdFloatRoundingOpWithCFallback<kF32>(
3856             &LiftoffAssembler::emit_f32x4_floor,
3857             ExternalReference::wasm_f32x4_floor);
3858       case wasm::kExprF32x4Trunc:
3859         return EmitSimdFloatRoundingOpWithCFallback<kF32>(
3860             &LiftoffAssembler::emit_f32x4_trunc,
3861             ExternalReference::wasm_f32x4_trunc);
3862       case wasm::kExprF32x4NearestInt:
3863         return EmitSimdFloatRoundingOpWithCFallback<kF32>(
3864             &LiftoffAssembler::emit_f32x4_nearest_int,
3865             ExternalReference::wasm_f32x4_nearest_int);
3866       case wasm::kExprF32x4Add:
3867         return EmitBinOp<kS128, kS128, false, kF32>(
3868             &LiftoffAssembler::emit_f32x4_add);
3869       case wasm::kExprF32x4Sub:
3870         return EmitBinOp<kS128, kS128, false, kF32>(
3871             &LiftoffAssembler::emit_f32x4_sub);
3872       case wasm::kExprF32x4Mul:
3873         return EmitBinOp<kS128, kS128, false, kF32>(
3874             &LiftoffAssembler::emit_f32x4_mul);
3875       case wasm::kExprF32x4Div:
3876         return EmitBinOp<kS128, kS128, false, kF32>(
3877             &LiftoffAssembler::emit_f32x4_div);
3878       case wasm::kExprF32x4Min:
3879         return EmitBinOp<kS128, kS128, false, kF32>(
3880             &LiftoffAssembler::emit_f32x4_min);
3881       case wasm::kExprF32x4Max:
3882         return EmitBinOp<kS128, kS128, false, kF32>(
3883             &LiftoffAssembler::emit_f32x4_max);
3884       case wasm::kExprF32x4Pmin:
3885         return EmitBinOp<kS128, kS128, false, kF32>(
3886             &LiftoffAssembler::emit_f32x4_pmin);
3887       case wasm::kExprF32x4Pmax:
3888         return EmitBinOp<kS128, kS128, false, kF32>(
3889             &LiftoffAssembler::emit_f32x4_pmax);
3890       case wasm::kExprF64x2Abs:
3891         return EmitUnOp<kS128, kS128, kF64>(&LiftoffAssembler::emit_f64x2_abs);
3892       case wasm::kExprF64x2Neg:
3893         return EmitUnOp<kS128, kS128, kF64>(&LiftoffAssembler::emit_f64x2_neg);
3894       case wasm::kExprF64x2Sqrt:
3895         return EmitUnOp<kS128, kS128, kF64>(&LiftoffAssembler::emit_f64x2_sqrt);
3896       case wasm::kExprF64x2Ceil:
3897         return EmitSimdFloatRoundingOpWithCFallback<kF64>(
3898             &LiftoffAssembler::emit_f64x2_ceil,
3899             &ExternalReference::wasm_f64x2_ceil);
3900       case wasm::kExprF64x2Floor:
3901         return EmitSimdFloatRoundingOpWithCFallback<kF64>(
3902             &LiftoffAssembler::emit_f64x2_floor,
3903             ExternalReference::wasm_f64x2_floor);
3904       case wasm::kExprF64x2Trunc:
3905         return EmitSimdFloatRoundingOpWithCFallback<kF64>(
3906             &LiftoffAssembler::emit_f64x2_trunc,
3907             ExternalReference::wasm_f64x2_trunc);
3908       case wasm::kExprF64x2NearestInt:
3909         return EmitSimdFloatRoundingOpWithCFallback<kF64>(
3910             &LiftoffAssembler::emit_f64x2_nearest_int,
3911             ExternalReference::wasm_f64x2_nearest_int);
3912       case wasm::kExprF64x2Add:
3913         return EmitBinOp<kS128, kS128, false, kF64>(
3914             &LiftoffAssembler::emit_f64x2_add);
3915       case wasm::kExprF64x2Sub:
3916         return EmitBinOp<kS128, kS128, false, kF64>(
3917             &LiftoffAssembler::emit_f64x2_sub);
3918       case wasm::kExprF64x2Mul:
3919         return EmitBinOp<kS128, kS128, false, kF64>(
3920             &LiftoffAssembler::emit_f64x2_mul);
3921       case wasm::kExprF64x2Div:
3922         return EmitBinOp<kS128, kS128, false, kF64>(
3923             &LiftoffAssembler::emit_f64x2_div);
3924       case wasm::kExprF64x2Min:
3925         return EmitBinOp<kS128, kS128, false, kF64>(
3926             &LiftoffAssembler::emit_f64x2_min);
3927       case wasm::kExprF64x2Max:
3928         return EmitBinOp<kS128, kS128, false, kF64>(
3929             &LiftoffAssembler::emit_f64x2_max);
3930       case wasm::kExprF64x2Pmin:
3931         return EmitBinOp<kS128, kS128, false, kF64>(
3932             &LiftoffAssembler::emit_f64x2_pmin);
3933       case wasm::kExprF64x2Pmax:
3934         return EmitBinOp<kS128, kS128, false, kF64>(
3935             &LiftoffAssembler::emit_f64x2_pmax);
3936       case wasm::kExprI32x4SConvertF32x4:
3937         return EmitUnOp<kS128, kS128, kF32>(
3938             &LiftoffAssembler::emit_i32x4_sconvert_f32x4);
3939       case wasm::kExprI32x4UConvertF32x4:
3940         return EmitUnOp<kS128, kS128, kF32>(
3941             &LiftoffAssembler::emit_i32x4_uconvert_f32x4);
3942       case wasm::kExprF32x4SConvertI32x4:
3943         return EmitUnOp<kS128, kS128, kF32>(
3944             &LiftoffAssembler::emit_f32x4_sconvert_i32x4);
3945       case wasm::kExprF32x4UConvertI32x4:
3946         return EmitUnOp<kS128, kS128, kF32>(
3947             &LiftoffAssembler::emit_f32x4_uconvert_i32x4);
3948       case wasm::kExprI8x16SConvertI16x8:
3949         return EmitBinOp<kS128, kS128>(
3950             &LiftoffAssembler::emit_i8x16_sconvert_i16x8);
3951       case wasm::kExprI8x16UConvertI16x8:
3952         return EmitBinOp<kS128, kS128>(
3953             &LiftoffAssembler::emit_i8x16_uconvert_i16x8);
3954       case wasm::kExprI16x8SConvertI32x4:
3955         return EmitBinOp<kS128, kS128>(
3956             &LiftoffAssembler::emit_i16x8_sconvert_i32x4);
3957       case wasm::kExprI16x8UConvertI32x4:
3958         return EmitBinOp<kS128, kS128>(
3959             &LiftoffAssembler::emit_i16x8_uconvert_i32x4);
3960       case wasm::kExprI16x8SConvertI8x16Low:
3961         return EmitUnOp<kS128, kS128>(
3962             &LiftoffAssembler::emit_i16x8_sconvert_i8x16_low);
3963       case wasm::kExprI16x8SConvertI8x16High:
3964         return EmitUnOp<kS128, kS128>(
3965             &LiftoffAssembler::emit_i16x8_sconvert_i8x16_high);
3966       case wasm::kExprI16x8UConvertI8x16Low:
3967         return EmitUnOp<kS128, kS128>(
3968             &LiftoffAssembler::emit_i16x8_uconvert_i8x16_low);
3969       case wasm::kExprI16x8UConvertI8x16High:
3970         return EmitUnOp<kS128, kS128>(
3971             &LiftoffAssembler::emit_i16x8_uconvert_i8x16_high);
3972       case wasm::kExprI32x4SConvertI16x8Low:
3973         return EmitUnOp<kS128, kS128>(
3974             &LiftoffAssembler::emit_i32x4_sconvert_i16x8_low);
3975       case wasm::kExprI32x4SConvertI16x8High:
3976         return EmitUnOp<kS128, kS128>(
3977             &LiftoffAssembler::emit_i32x4_sconvert_i16x8_high);
3978       case wasm::kExprI32x4UConvertI16x8Low:
3979         return EmitUnOp<kS128, kS128>(
3980             &LiftoffAssembler::emit_i32x4_uconvert_i16x8_low);
3981       case wasm::kExprI32x4UConvertI16x8High:
3982         return EmitUnOp<kS128, kS128>(
3983             &LiftoffAssembler::emit_i32x4_uconvert_i16x8_high);
3984       case wasm::kExprS128AndNot:
3985         return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_s128_and_not);
3986       case wasm::kExprI8x16RoundingAverageU:
3987         return EmitBinOp<kS128, kS128>(
3988             &LiftoffAssembler::emit_i8x16_rounding_average_u);
3989       case wasm::kExprI16x8RoundingAverageU:
3990         return EmitBinOp<kS128, kS128>(
3991             &LiftoffAssembler::emit_i16x8_rounding_average_u);
3992       case wasm::kExprI8x16Abs:
3993         return EmitUnOp<kS128, kS128>(&LiftoffAssembler::emit_i8x16_abs);
3994       case wasm::kExprI16x8Abs:
3995         return EmitUnOp<kS128, kS128>(&LiftoffAssembler::emit_i16x8_abs);
3996       case wasm::kExprI32x4Abs:
3997         return EmitUnOp<kS128, kS128>(&LiftoffAssembler::emit_i32x4_abs);
3998       case wasm::kExprI64x2Abs:
3999         return EmitUnOp<kS128, kS128>(&LiftoffAssembler::emit_i64x2_abs);
4000       case wasm::kExprF64x2ConvertLowI32x4S:
4001         return EmitUnOp<kS128, kS128, kF64>(
4002             &LiftoffAssembler::emit_f64x2_convert_low_i32x4_s);
4003       case wasm::kExprF64x2ConvertLowI32x4U:
4004         return EmitUnOp<kS128, kS128, kF64>(
4005             &LiftoffAssembler::emit_f64x2_convert_low_i32x4_u);
4006       case wasm::kExprF64x2PromoteLowF32x4:
4007         return EmitUnOp<kS128, kS128, kF64>(
4008             &LiftoffAssembler::emit_f64x2_promote_low_f32x4);
4009       case wasm::kExprF32x4DemoteF64x2Zero:
4010         return EmitUnOp<kS128, kS128, kF32>(
4011             &LiftoffAssembler::emit_f32x4_demote_f64x2_zero);
4012       case wasm::kExprI32x4TruncSatF64x2SZero:
4013         return EmitUnOp<kS128, kS128>(
4014             &LiftoffAssembler::emit_i32x4_trunc_sat_f64x2_s_zero);
4015       case wasm::kExprI32x4TruncSatF64x2UZero:
4016         return EmitUnOp<kS128, kS128>(
4017             &LiftoffAssembler::emit_i32x4_trunc_sat_f64x2_u_zero);
4018       default:
4019         unsupported(decoder, kSimd, "simd");
4020     }
4021   }
4022 
4023   template <ValueKind src_kind, ValueKind result_kind, typename EmitFn>
EmitSimdExtractLaneOp(EmitFn fn,const SimdLaneImmediate<validate> & imm)4024   void EmitSimdExtractLaneOp(EmitFn fn,
4025                              const SimdLaneImmediate<validate>& imm) {
4026     static constexpr RegClass src_rc = reg_class_for(src_kind);
4027     static constexpr RegClass result_rc = reg_class_for(result_kind);
4028     LiftoffRegister lhs = __ PopToRegister();
4029     LiftoffRegister dst = src_rc == result_rc
4030                               ? __ GetUnusedRegister(result_rc, {lhs}, {})
4031                               : __ GetUnusedRegister(result_rc, {});
4032     fn(dst, lhs, imm.lane);
4033     __ PushRegister(result_kind, dst);
4034   }
4035 
4036   template <ValueKind src2_kind, typename EmitFn>
EmitSimdReplaceLaneOp(EmitFn fn,const SimdLaneImmediate<validate> & imm)4037   void EmitSimdReplaceLaneOp(EmitFn fn,
4038                              const SimdLaneImmediate<validate>& imm) {
4039     static constexpr RegClass src1_rc = reg_class_for(kS128);
4040     static constexpr RegClass src2_rc = reg_class_for(src2_kind);
4041     static constexpr RegClass result_rc = reg_class_for(kS128);
4042     // On backends which need fp pair, src1_rc and result_rc end up being
4043     // kFpRegPair, which is != kFpReg, but we still want to pin src2 when it is
4044     // kFpReg, since it can overlap with those pairs.
4045     static constexpr bool pin_src2 = kNeedS128RegPair && src2_rc == kFpReg;
4046 
4047     // Does not work for arm
4048     LiftoffRegister src2 = __ PopToRegister();
4049     LiftoffRegister src1 = (src1_rc == src2_rc || pin_src2)
4050                                ? __ PopToRegister(LiftoffRegList{src2})
4051                                : __
4052                                  PopToRegister();
4053     LiftoffRegister dst =
4054         (src2_rc == result_rc || pin_src2)
4055             ? __ GetUnusedRegister(result_rc, {src1}, LiftoffRegList{src2})
4056             : __ GetUnusedRegister(result_rc, {src1}, {});
4057     fn(dst, src1, src2, imm.lane);
4058     __ PushRegister(kS128, dst);
4059   }
4060 
SimdLaneOp(FullDecoder * decoder,WasmOpcode opcode,const SimdLaneImmediate<validate> & imm,const base::Vector<Value> inputs,Value * result)4061   void SimdLaneOp(FullDecoder* decoder, WasmOpcode opcode,
4062                   const SimdLaneImmediate<validate>& imm,
4063                   const base::Vector<Value> inputs, Value* result) {
4064     if (!CpuFeatures::SupportsWasmSimd128()) {
4065       return unsupported(decoder, kSimd, "simd");
4066     }
4067     switch (opcode) {
4068 #define CASE_SIMD_EXTRACT_LANE_OP(opcode, kind, fn)                           \
4069   case wasm::kExpr##opcode:                                                   \
4070     EmitSimdExtractLaneOp<kS128, k##kind>(                                    \
4071         [=](LiftoffRegister dst, LiftoffRegister lhs, uint8_t imm_lane_idx) { \
4072           __ emit_##fn(dst, lhs, imm_lane_idx);                               \
4073         },                                                                    \
4074         imm);                                                                 \
4075     break;
4076       CASE_SIMD_EXTRACT_LANE_OP(I8x16ExtractLaneS, I32, i8x16_extract_lane_s)
4077       CASE_SIMD_EXTRACT_LANE_OP(I8x16ExtractLaneU, I32, i8x16_extract_lane_u)
4078       CASE_SIMD_EXTRACT_LANE_OP(I16x8ExtractLaneS, I32, i16x8_extract_lane_s)
4079       CASE_SIMD_EXTRACT_LANE_OP(I16x8ExtractLaneU, I32, i16x8_extract_lane_u)
4080       CASE_SIMD_EXTRACT_LANE_OP(I32x4ExtractLane, I32, i32x4_extract_lane)
4081       CASE_SIMD_EXTRACT_LANE_OP(I64x2ExtractLane, I64, i64x2_extract_lane)
4082       CASE_SIMD_EXTRACT_LANE_OP(F32x4ExtractLane, F32, f32x4_extract_lane)
4083       CASE_SIMD_EXTRACT_LANE_OP(F64x2ExtractLane, F64, f64x2_extract_lane)
4084 #undef CASE_SIMD_EXTRACT_LANE_OP
4085 #define CASE_SIMD_REPLACE_LANE_OP(opcode, kind, fn)                          \
4086   case wasm::kExpr##opcode:                                                  \
4087     EmitSimdReplaceLaneOp<k##kind>(                                          \
4088         [=](LiftoffRegister dst, LiftoffRegister src1, LiftoffRegister src2, \
4089             uint8_t imm_lane_idx) {                                          \
4090           __ emit_##fn(dst, src1, src2, imm_lane_idx);                       \
4091         },                                                                   \
4092         imm);                                                                \
4093     break;
4094       CASE_SIMD_REPLACE_LANE_OP(I8x16ReplaceLane, I32, i8x16_replace_lane)
4095       CASE_SIMD_REPLACE_LANE_OP(I16x8ReplaceLane, I32, i16x8_replace_lane)
4096       CASE_SIMD_REPLACE_LANE_OP(I32x4ReplaceLane, I32, i32x4_replace_lane)
4097       CASE_SIMD_REPLACE_LANE_OP(I64x2ReplaceLane, I64, i64x2_replace_lane)
4098       CASE_SIMD_REPLACE_LANE_OP(F32x4ReplaceLane, F32, f32x4_replace_lane)
4099       CASE_SIMD_REPLACE_LANE_OP(F64x2ReplaceLane, F64, f64x2_replace_lane)
4100 #undef CASE_SIMD_REPLACE_LANE_OP
4101       default:
4102         unsupported(decoder, kSimd, "simd");
4103     }
4104   }
4105 
S128Const(FullDecoder * decoder,const Simd128Immediate<validate> & imm,Value * result)4106   void S128Const(FullDecoder* decoder, const Simd128Immediate<validate>& imm,
4107                  Value* result) {
4108     if (!CpuFeatures::SupportsWasmSimd128()) {
4109       return unsupported(decoder, kSimd, "simd");
4110     }
4111     constexpr RegClass result_rc = reg_class_for(kS128);
4112     LiftoffRegister dst = __ GetUnusedRegister(result_rc, {});
4113     bool all_zeroes = std::all_of(std::begin(imm.value), std::end(imm.value),
4114                                   [](uint8_t v) { return v == 0; });
4115     bool all_ones = std::all_of(std::begin(imm.value), std::end(imm.value),
4116                                 [](uint8_t v) { return v == 0xff; });
4117     if (all_zeroes) {
4118       __ LiftoffAssembler::emit_s128_xor(dst, dst, dst);
4119     } else if (all_ones) {
4120       // Any SIMD eq will work, i32x4 is efficient on all archs.
4121       __ LiftoffAssembler::emit_i32x4_eq(dst, dst, dst);
4122     } else {
4123       __ LiftoffAssembler::emit_s128_const(dst, imm.value);
4124     }
4125     __ PushRegister(kS128, dst);
4126   }
4127 
Simd8x16ShuffleOp(FullDecoder * decoder,const Simd128Immediate<validate> & imm,const Value & input0,const Value & input1,Value * result)4128   void Simd8x16ShuffleOp(FullDecoder* decoder,
4129                          const Simd128Immediate<validate>& imm,
4130                          const Value& input0, const Value& input1,
4131                          Value* result) {
4132     if (!CpuFeatures::SupportsWasmSimd128()) {
4133       return unsupported(decoder, kSimd, "simd");
4134     }
4135     static constexpr RegClass result_rc = reg_class_for(kS128);
4136     LiftoffRegister rhs = __ PopToRegister();
4137     LiftoffRegister lhs = __ PopToRegister(LiftoffRegList{rhs});
4138     LiftoffRegister dst = __ GetUnusedRegister(result_rc, {lhs, rhs}, {});
4139 
4140     uint8_t shuffle[kSimd128Size];
4141     memcpy(shuffle, imm.value, sizeof(shuffle));
4142     bool is_swizzle;
4143     bool needs_swap;
4144     wasm::SimdShuffle::CanonicalizeShuffle(lhs == rhs, shuffle, &needs_swap,
4145                                            &is_swizzle);
4146     if (needs_swap) {
4147       std::swap(lhs, rhs);
4148     }
4149     __ LiftoffAssembler::emit_i8x16_shuffle(dst, lhs, rhs, shuffle, is_swizzle);
4150     __ PushRegister(kS128, dst);
4151   }
4152 
ToSmi(Register reg)4153   void ToSmi(Register reg) {
4154     if (COMPRESS_POINTERS_BOOL || kSystemPointerSize == 4) {
4155       __ emit_i32_shli(reg, reg, kSmiShiftSize + kSmiTagSize);
4156     } else {
4157       __ emit_i64_shli(LiftoffRegister{reg}, LiftoffRegister{reg},
4158                        kSmiShiftSize + kSmiTagSize);
4159     }
4160   }
4161 
Store32BitExceptionValue(Register values_array,int * index_in_array,Register value,LiftoffRegList pinned)4162   void Store32BitExceptionValue(Register values_array, int* index_in_array,
4163                                 Register value, LiftoffRegList pinned) {
4164     LiftoffRegister tmp_reg = __ GetUnusedRegister(kGpReg, pinned);
4165     // Get the lower half word into tmp_reg and extend to a Smi.
4166     --*index_in_array;
4167     __ emit_i32_andi(tmp_reg.gp(), value, 0xffff);
4168     ToSmi(tmp_reg.gp());
4169     __ StoreTaggedPointer(
4170         values_array, no_reg,
4171         wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(*index_in_array),
4172         tmp_reg, pinned, LiftoffAssembler::kSkipWriteBarrier);
4173 
4174     // Get the upper half word into tmp_reg and extend to a Smi.
4175     --*index_in_array;
4176     __ emit_i32_shri(tmp_reg.gp(), value, 16);
4177     ToSmi(tmp_reg.gp());
4178     __ StoreTaggedPointer(
4179         values_array, no_reg,
4180         wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(*index_in_array),
4181         tmp_reg, pinned, LiftoffAssembler::kSkipWriteBarrier);
4182   }
4183 
Store64BitExceptionValue(Register values_array,int * index_in_array,LiftoffRegister value,LiftoffRegList pinned)4184   void Store64BitExceptionValue(Register values_array, int* index_in_array,
4185                                 LiftoffRegister value, LiftoffRegList pinned) {
4186     if (kNeedI64RegPair) {
4187       Store32BitExceptionValue(values_array, index_in_array, value.low_gp(),
4188                                pinned);
4189       Store32BitExceptionValue(values_array, index_in_array, value.high_gp(),
4190                                pinned);
4191     } else {
4192       Store32BitExceptionValue(values_array, index_in_array, value.gp(),
4193                                pinned);
4194       __ emit_i64_shri(value, value, 32);
4195       Store32BitExceptionValue(values_array, index_in_array, value.gp(),
4196                                pinned);
4197     }
4198   }
4199 
Load16BitExceptionValue(LiftoffRegister dst,LiftoffRegister values_array,uint32_t * index,LiftoffRegList pinned)4200   void Load16BitExceptionValue(LiftoffRegister dst,
4201                                LiftoffRegister values_array, uint32_t* index,
4202                                LiftoffRegList pinned) {
4203     __ LoadSmiAsInt32(
4204         dst, values_array.gp(),
4205         wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(*index), pinned);
4206     (*index)++;
4207   }
4208 
Load32BitExceptionValue(Register dst,LiftoffRegister values_array,uint32_t * index,LiftoffRegList pinned)4209   void Load32BitExceptionValue(Register dst, LiftoffRegister values_array,
4210                                uint32_t* index, LiftoffRegList pinned) {
4211     LiftoffRegister upper = pinned.set(__ GetUnusedRegister(kGpReg, pinned));
4212     Load16BitExceptionValue(upper, values_array, index, pinned);
4213     __ emit_i32_shli(upper.gp(), upper.gp(), 16);
4214     Load16BitExceptionValue(LiftoffRegister(dst), values_array, index, pinned);
4215     __ emit_i32_or(dst, upper.gp(), dst);
4216   }
4217 
Load64BitExceptionValue(LiftoffRegister dst,LiftoffRegister values_array,uint32_t * index,LiftoffRegList pinned)4218   void Load64BitExceptionValue(LiftoffRegister dst,
4219                                LiftoffRegister values_array, uint32_t* index,
4220                                LiftoffRegList pinned) {
4221     if (kNeedI64RegPair) {
4222       Load32BitExceptionValue(dst.high_gp(), values_array, index, pinned);
4223       Load32BitExceptionValue(dst.low_gp(), values_array, index, pinned);
4224     } else {
4225       Load16BitExceptionValue(dst, values_array, index, pinned);
4226       __ emit_i64_shli(dst, dst, 48);
4227       LiftoffRegister tmp_reg =
4228           pinned.set(__ GetUnusedRegister(kGpReg, pinned));
4229       Load16BitExceptionValue(tmp_reg, values_array, index, pinned);
4230       __ emit_i64_shli(tmp_reg, tmp_reg, 32);
4231       __ emit_i64_or(dst, tmp_reg, dst);
4232       Load16BitExceptionValue(tmp_reg, values_array, index, pinned);
4233       __ emit_i64_shli(tmp_reg, tmp_reg, 16);
4234       __ emit_i64_or(dst, tmp_reg, dst);
4235       Load16BitExceptionValue(tmp_reg, values_array, index, pinned);
4236       __ emit_i64_or(dst, tmp_reg, dst);
4237     }
4238   }
4239 
StoreExceptionValue(ValueType type,Register values_array,int * index_in_array,LiftoffRegList pinned)4240   void StoreExceptionValue(ValueType type, Register values_array,
4241                            int* index_in_array, LiftoffRegList pinned) {
4242     LiftoffRegister value = pinned.set(__ PopToRegister(pinned));
4243     switch (type.kind()) {
4244       case kI32:
4245         Store32BitExceptionValue(values_array, index_in_array, value.gp(),
4246                                  pinned);
4247         break;
4248       case kF32: {
4249         LiftoffRegister gp_reg =
4250             pinned.set(__ GetUnusedRegister(kGpReg, pinned));
4251         __ emit_type_conversion(kExprI32ReinterpretF32, gp_reg, value, nullptr);
4252         Store32BitExceptionValue(values_array, index_in_array, gp_reg.gp(),
4253                                  pinned);
4254         break;
4255       }
4256       case kI64:
4257         Store64BitExceptionValue(values_array, index_in_array, value, pinned);
4258         break;
4259       case kF64: {
4260         LiftoffRegister tmp_reg =
4261             pinned.set(__ GetUnusedRegister(reg_class_for(kI64), pinned));
4262         __ emit_type_conversion(kExprI64ReinterpretF64, tmp_reg, value,
4263                                 nullptr);
4264         Store64BitExceptionValue(values_array, index_in_array, tmp_reg, pinned);
4265         break;
4266       }
4267       case kS128: {
4268         LiftoffRegister tmp_reg =
4269             pinned.set(__ GetUnusedRegister(kGpReg, pinned));
4270         for (int i : {3, 2, 1, 0}) {
4271           __ emit_i32x4_extract_lane(tmp_reg, value, i);
4272           Store32BitExceptionValue(values_array, index_in_array, tmp_reg.gp(),
4273                                    pinned);
4274         }
4275         break;
4276       }
4277       case wasm::kRef:
4278       case wasm::kOptRef:
4279       case wasm::kRtt: {
4280         --(*index_in_array);
4281         __ StoreTaggedPointer(
4282             values_array, no_reg,
4283             wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(
4284                 *index_in_array),
4285             value, pinned);
4286         break;
4287       }
4288       case wasm::kI8:
4289       case wasm::kI16:
4290       case wasm::kVoid:
4291       case wasm::kBottom:
4292         UNREACHABLE();
4293     }
4294   }
4295 
LoadExceptionValue(ValueKind kind,LiftoffRegister values_array,uint32_t * index,LiftoffRegList pinned)4296   void LoadExceptionValue(ValueKind kind, LiftoffRegister values_array,
4297                           uint32_t* index, LiftoffRegList pinned) {
4298     RegClass rc = reg_class_for(kind);
4299     LiftoffRegister value = pinned.set(__ GetUnusedRegister(rc, pinned));
4300     switch (kind) {
4301       case kI32:
4302         Load32BitExceptionValue(value.gp(), values_array, index, pinned);
4303         break;
4304       case kF32: {
4305         LiftoffRegister tmp_reg =
4306             pinned.set(__ GetUnusedRegister(kGpReg, pinned));
4307         Load32BitExceptionValue(tmp_reg.gp(), values_array, index, pinned);
4308         __ emit_type_conversion(kExprF32ReinterpretI32, value, tmp_reg,
4309                                 nullptr);
4310         break;
4311       }
4312       case kI64:
4313         Load64BitExceptionValue(value, values_array, index, pinned);
4314         break;
4315       case kF64: {
4316         RegClass rc_i64 = reg_class_for(kI64);
4317         LiftoffRegister tmp_reg =
4318             pinned.set(__ GetUnusedRegister(rc_i64, pinned));
4319         Load64BitExceptionValue(tmp_reg, values_array, index, pinned);
4320         __ emit_type_conversion(kExprF64ReinterpretI64, value, tmp_reg,
4321                                 nullptr);
4322         break;
4323       }
4324       case kS128: {
4325         LiftoffRegister tmp_reg =
4326             pinned.set(__ GetUnusedRegister(kGpReg, pinned));
4327         Load32BitExceptionValue(tmp_reg.gp(), values_array, index, pinned);
4328         __ emit_i32x4_splat(value, tmp_reg);
4329         for (int lane : {1, 2, 3}) {
4330           Load32BitExceptionValue(tmp_reg.gp(), values_array, index, pinned);
4331           __ emit_i32x4_replace_lane(value, value, tmp_reg, lane);
4332         }
4333         break;
4334       }
4335       case wasm::kRef:
4336       case wasm::kOptRef:
4337       case wasm::kRtt: {
4338         __ LoadTaggedPointer(
4339             value.gp(), values_array.gp(), no_reg,
4340             wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(*index),
4341             pinned);
4342         (*index)++;
4343         break;
4344       }
4345       case wasm::kI8:
4346       case wasm::kI16:
4347       case wasm::kVoid:
4348       case wasm::kBottom:
4349         UNREACHABLE();
4350     }
4351     __ PushRegister(kind, value);
4352   }
4353 
GetExceptionValues(FullDecoder * decoder,LiftoffAssembler::VarState & exception_var,const WasmTag * tag)4354   void GetExceptionValues(FullDecoder* decoder,
4355                           LiftoffAssembler::VarState& exception_var,
4356                           const WasmTag* tag) {
4357     LiftoffRegList pinned;
4358     CODE_COMMENT("get exception values");
4359     LiftoffRegister values_array = GetExceptionProperty(
4360         exception_var, RootIndex::kwasm_exception_values_symbol);
4361     pinned.set(values_array);
4362     uint32_t index = 0;
4363     const WasmTagSig* sig = tag->sig;
4364     for (ValueType param : sig->parameters()) {
4365       LoadExceptionValue(param.kind(), values_array, &index, pinned);
4366     }
4367     DCHECK_EQ(index, WasmExceptionPackage::GetEncodedSize(tag));
4368   }
4369 
EmitLandingPad(FullDecoder * decoder,int handler_offset)4370   void EmitLandingPad(FullDecoder* decoder, int handler_offset) {
4371     if (decoder->current_catch() == -1) return;
4372     MovableLabel handler;
4373 
4374     // If we return from the throwing code normally, just skip over the handler.
4375     Label skip_handler;
4376     __ emit_jump(&skip_handler);
4377 
4378     // Handler: merge into the catch state, and jump to the catch body.
4379     CODE_COMMENT("-- landing pad --");
4380     __ bind(handler.get());
4381     __ ExceptionHandler();
4382     __ PushException();
4383     handlers_.push_back({std::move(handler), handler_offset});
4384     Control* current_try =
4385         decoder->control_at(decoder->control_depth_of_current_catch());
4386     DCHECK_NOT_NULL(current_try->try_info);
4387     if (!current_try->try_info->catch_reached) {
4388       current_try->try_info->catch_state.InitMerge(
4389           *__ cache_state(), __ num_locals(), 1,
4390           current_try->stack_depth + current_try->num_exceptions);
4391       current_try->try_info->catch_reached = true;
4392     }
4393     __ MergeStackWith(current_try->try_info->catch_state, 1,
4394                       LiftoffAssembler::kForwardJump);
4395     __ emit_jump(&current_try->try_info->catch_label);
4396 
4397     __ bind(&skip_handler);
4398     // Drop the exception.
4399     __ DropValues(1);
4400   }
4401 
Throw(FullDecoder * decoder,const TagIndexImmediate<validate> & imm,const base::Vector<Value> &)4402   void Throw(FullDecoder* decoder, const TagIndexImmediate<validate>& imm,
4403              const base::Vector<Value>& /* args */) {
4404     LiftoffRegList pinned;
4405 
4406     // Load the encoded size in a register for the builtin call.
4407     int encoded_size = WasmExceptionPackage::GetEncodedSize(imm.tag);
4408     LiftoffRegister encoded_size_reg =
4409         pinned.set(__ GetUnusedRegister(kGpReg, pinned));
4410     __ LoadConstant(encoded_size_reg, WasmValue(encoded_size));
4411 
4412     // Call the WasmAllocateFixedArray builtin to create the values array.
4413     CallRuntimeStub(WasmCode::kWasmAllocateFixedArray,
4414                     MakeSig::Returns(kPointerKind).Params(kPointerKind),
4415                     {LiftoffAssembler::VarState{
4416                         kSmiKind, LiftoffRegister{encoded_size_reg}, 0}},
4417                     decoder->position());
4418     MaybeOSR();
4419 
4420     // The FixedArray for the exception values is now in the first gp return
4421     // register.
4422     LiftoffRegister values_array{kReturnRegister0};
4423     pinned.set(values_array);
4424 
4425     // Now store the exception values in the FixedArray. Do this from last to
4426     // first value, such that we can just pop them from the value stack.
4427     CODE_COMMENT("fill values array");
4428     int index = encoded_size;
4429     auto* sig = imm.tag->sig;
4430     for (size_t param_idx = sig->parameter_count(); param_idx > 0;
4431          --param_idx) {
4432       ValueType type = sig->GetParam(param_idx - 1);
4433       StoreExceptionValue(type, values_array.gp(), &index, pinned);
4434     }
4435     DCHECK_EQ(0, index);
4436 
4437     // Load the exception tag.
4438     CODE_COMMENT("load exception tag");
4439     LiftoffRegister exception_tag =
4440         pinned.set(__ GetUnusedRegister(kGpReg, pinned));
4441     LOAD_TAGGED_PTR_INSTANCE_FIELD(exception_tag.gp(), TagsTable, pinned);
4442     __ LoadTaggedPointer(
4443         exception_tag.gp(), exception_tag.gp(), no_reg,
4444         wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(imm.index), {});
4445 
4446     // Finally, call WasmThrow.
4447     CallRuntimeStub(WasmCode::kWasmThrow,
4448                     MakeSig::Params(kPointerKind, kPointerKind),
4449                     {LiftoffAssembler::VarState{kPointerKind, exception_tag, 0},
4450                      LiftoffAssembler::VarState{kPointerKind, values_array, 0}},
4451                     decoder->position());
4452 
4453     int pc_offset = __ pc_offset();
4454     MaybeOSR();
4455     EmitLandingPad(decoder, pc_offset);
4456   }
4457 
AtomicStoreMem(FullDecoder * decoder,StoreType type,const MemoryAccessImmediate<validate> & imm)4458   void AtomicStoreMem(FullDecoder* decoder, StoreType type,
4459                       const MemoryAccessImmediate<validate>& imm) {
4460     LiftoffRegList pinned;
4461     LiftoffRegister value = pinned.set(__ PopToRegister());
4462     LiftoffRegister full_index = __ PopToRegister(pinned);
4463     Register index = BoundsCheckMem(decoder, type.size(), imm.offset,
4464                                     full_index, pinned, kDoForceCheck);
4465     if (index == no_reg) return;
4466 
4467     pinned.set(index);
4468     AlignmentCheckMem(decoder, type.size(), imm.offset, index, pinned);
4469     uintptr_t offset = imm.offset;
4470     CODE_COMMENT("atomic store to memory");
4471     Register addr = pinned.set(GetMemoryStart(pinned));
4472     LiftoffRegList outer_pinned;
4473     if (V8_UNLIKELY(FLAG_trace_wasm_memory)) outer_pinned.set(index);
4474     __ AtomicStore(addr, index, offset, value, type, outer_pinned);
4475     if (V8_UNLIKELY(FLAG_trace_wasm_memory)) {
4476       TraceMemoryOperation(true, type.mem_rep(), index, offset,
4477                            decoder->position());
4478     }
4479   }
4480 
AtomicLoadMem(FullDecoder * decoder,LoadType type,const MemoryAccessImmediate<validate> & imm)4481   void AtomicLoadMem(FullDecoder* decoder, LoadType type,
4482                      const MemoryAccessImmediate<validate>& imm) {
4483     ValueKind kind = type.value_type().kind();
4484     LiftoffRegister full_index = __ PopToRegister();
4485     Register index = BoundsCheckMem(decoder, type.size(), imm.offset,
4486                                     full_index, {}, kDoForceCheck);
4487     if (index == no_reg) return;
4488 
4489     LiftoffRegList pinned = {index};
4490     AlignmentCheckMem(decoder, type.size(), imm.offset, index, pinned);
4491     uintptr_t offset = imm.offset;
4492     CODE_COMMENT("atomic load from memory");
4493     Register addr = pinned.set(GetMemoryStart(pinned));
4494     RegClass rc = reg_class_for(kind);
4495     LiftoffRegister value = pinned.set(__ GetUnusedRegister(rc, pinned));
4496     __ AtomicLoad(value, addr, index, offset, type, pinned);
4497     __ PushRegister(kind, value);
4498 
4499     if (V8_UNLIKELY(FLAG_trace_wasm_memory)) {
4500       TraceMemoryOperation(false, type.mem_type().representation(), index,
4501                            offset, decoder->position());
4502     }
4503   }
4504 
AtomicBinop(FullDecoder * decoder,StoreType type,const MemoryAccessImmediate<validate> & imm,void (LiftoffAssembler::* emit_fn)(Register,Register,uintptr_t,LiftoffRegister,LiftoffRegister,StoreType))4505   void AtomicBinop(FullDecoder* decoder, StoreType type,
4506                    const MemoryAccessImmediate<validate>& imm,
4507                    void (LiftoffAssembler::*emit_fn)(Register, Register,
4508                                                      uintptr_t, LiftoffRegister,
4509                                                      LiftoffRegister,
4510                                                      StoreType)) {
4511     ValueKind result_kind = type.value_type().kind();
4512     LiftoffRegList pinned;
4513     LiftoffRegister value = pinned.set(__ PopToRegister());
4514 #ifdef V8_TARGET_ARCH_IA32
4515     // We have to reuse the value register as the result register so that we
4516     // don't run out of registers on ia32. For this we use the value register as
4517     // the result register if it has no other uses. Otherwise we allocate a new
4518     // register and let go of the value register to get spilled.
4519     LiftoffRegister result = value;
4520     if (__ cache_state()->is_used(value)) {
4521       result = pinned.set(__ GetUnusedRegister(value.reg_class(), pinned));
4522       __ Move(result, value, result_kind);
4523       pinned.clear(value);
4524       value = result;
4525     }
4526 #else
4527     LiftoffRegister result =
4528         pinned.set(__ GetUnusedRegister(value.reg_class(), pinned));
4529 #endif
4530     LiftoffRegister full_index = __ PopToRegister(pinned);
4531     Register index = BoundsCheckMem(decoder, type.size(), imm.offset,
4532                                     full_index, pinned, kDoForceCheck);
4533     if (index == no_reg) return;
4534 
4535     pinned.set(index);
4536     AlignmentCheckMem(decoder, type.size(), imm.offset, index, pinned);
4537 
4538     CODE_COMMENT("atomic binop");
4539     uintptr_t offset = imm.offset;
4540     Register addr = pinned.set(GetMemoryStart(pinned));
4541 
4542     (asm_.*emit_fn)(addr, index, offset, value, result, type);
4543     __ PushRegister(result_kind, result);
4544   }
4545 
AtomicCompareExchange(FullDecoder * decoder,StoreType type,const MemoryAccessImmediate<validate> & imm)4546   void AtomicCompareExchange(FullDecoder* decoder, StoreType type,
4547                              const MemoryAccessImmediate<validate>& imm) {
4548 #ifdef V8_TARGET_ARCH_IA32
4549     // On ia32 we don't have enough registers to first pop all the values off
4550     // the stack and then start with the code generation. Instead we do the
4551     // complete address calculation first, so that the address only needs a
4552     // single register. Afterwards we load all remaining values into the
4553     // other registers.
4554     LiftoffRegister full_index = __ PeekToRegister(2, {});
4555     Register index = BoundsCheckMem(decoder, type.size(), imm.offset,
4556                                     full_index, {}, kDoForceCheck);
4557     if (index == no_reg) return;
4558     LiftoffRegList pinned = {index};
4559     AlignmentCheckMem(decoder, type.size(), imm.offset, index, pinned);
4560 
4561     uintptr_t offset = imm.offset;
4562     Register addr = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
4563     LOAD_INSTANCE_FIELD(addr, MemoryStart, kSystemPointerSize, pinned);
4564 #ifdef V8_SANDBOXED_POINTERS
4565     __ DecodeSandboxedPointer(addr);
4566 #endif
4567     __ emit_i32_add(addr, addr, index);
4568     pinned.clear(LiftoffRegister(index));
4569     LiftoffRegister new_value = pinned.set(__ PopToRegister(pinned));
4570     LiftoffRegister expected = pinned.set(__ PopToRegister(pinned));
4571 
4572     // Pop the index from the stack.
4573     __ DropValues(1);
4574 
4575     LiftoffRegister result = expected;
4576     if (__ cache_state()->is_used(result)) __ SpillRegister(result);
4577 
4578     // We already added the index to addr, so we can just pass no_reg to the
4579     // assembler now.
4580     __ AtomicCompareExchange(addr, no_reg, offset, expected, new_value, result,
4581                              type);
4582     __ PushRegister(type.value_type().kind(), result);
4583     return;
4584 #else
4585     ValueKind result_kind = type.value_type().kind();
4586     LiftoffRegList pinned;
4587     LiftoffRegister new_value = pinned.set(__ PopToRegister());
4588     LiftoffRegister expected = pinned.set(__ PopToRegister(pinned));
4589     LiftoffRegister full_index = __ PopToRegister(pinned);
4590     Register index = BoundsCheckMem(decoder, type.size(), imm.offset,
4591                                     full_index, pinned, kDoForceCheck);
4592     if (index == no_reg) return;
4593     pinned.set(index);
4594     AlignmentCheckMem(decoder, type.size(), imm.offset, index, pinned);
4595 
4596     uintptr_t offset = imm.offset;
4597     Register addr = pinned.set(GetMemoryStart(pinned));
4598     LiftoffRegister result =
4599         pinned.set(__ GetUnusedRegister(reg_class_for(result_kind), pinned));
4600 
4601     __ AtomicCompareExchange(addr, index, offset, expected, new_value, result,
4602                              type);
4603     __ PushRegister(result_kind, result);
4604 #endif
4605   }
4606 
CallRuntimeStub(WasmCode::RuntimeStubId stub_id,const ValueKindSig & sig,std::initializer_list<LiftoffAssembler::VarState> params,int position)4607   void CallRuntimeStub(WasmCode::RuntimeStubId stub_id, const ValueKindSig& sig,
4608                        std::initializer_list<LiftoffAssembler::VarState> params,
4609                        int position) {
4610     CODE_COMMENT(
4611         (std::string{"call builtin: "} + GetRuntimeStubName(stub_id)).c_str());
4612     auto interface_descriptor = Builtins::CallInterfaceDescriptorFor(
4613         RuntimeStubIdToBuiltinName(stub_id));
4614     auto* call_descriptor = compiler::Linkage::GetStubCallDescriptor(
4615         compilation_zone_,                              // zone
4616         interface_descriptor,                           // descriptor
4617         interface_descriptor.GetStackParameterCount(),  // stack parameter count
4618         compiler::CallDescriptor::kNoFlags,             // flags
4619         compiler::Operator::kNoProperties,              // properties
4620         StubCallMode::kCallWasmRuntimeStub);            // stub call mode
4621 
4622     __ PrepareBuiltinCall(&sig, call_descriptor, params);
4623     if (position != kNoSourcePosition) {
4624       source_position_table_builder_.AddPosition(
4625           __ pc_offset(), SourcePosition(position), true);
4626     }
4627     __ CallRuntimeStub(stub_id);
4628     DefineSafepoint();
4629   }
4630 
AtomicWait(FullDecoder * decoder,ValueKind kind,const MemoryAccessImmediate<validate> & imm)4631   void AtomicWait(FullDecoder* decoder, ValueKind kind,
4632                   const MemoryAccessImmediate<validate>& imm) {
4633     LiftoffRegister full_index = __ PeekToRegister(2, {});
4634     Register index_reg =
4635         BoundsCheckMem(decoder, value_kind_size(kind), imm.offset, full_index,
4636                        {}, kDoForceCheck);
4637     if (index_reg == no_reg) return;
4638     LiftoffRegList pinned = {index_reg};
4639     AlignmentCheckMem(decoder, value_kind_size(kind), imm.offset, index_reg,
4640                       pinned);
4641 
4642     uintptr_t offset = imm.offset;
4643     Register index_plus_offset =
4644         __ cache_state()->is_used(LiftoffRegister(index_reg))
4645             ? pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp()
4646             : index_reg;
4647     // TODO(clemensb): Skip this if memory is 64 bit.
4648     __ emit_ptrsize_zeroextend_i32(index_plus_offset, index_reg);
4649     if (offset) {
4650       __ emit_ptrsize_addi(index_plus_offset, index_plus_offset, offset);
4651     }
4652 
4653     LiftoffAssembler::VarState timeout =
4654         __ cache_state()->stack_state.end()[-1];
4655     LiftoffAssembler::VarState expected_value =
4656         __ cache_state()->stack_state.end()[-2];
4657     LiftoffAssembler::VarState index = __ cache_state()->stack_state.end()[-3];
4658 
4659     // We have to set the correct register for the index.
4660     index.MakeRegister(LiftoffRegister(index_plus_offset));
4661 
4662     static constexpr WasmCode::RuntimeStubId kTargets[2][2]{
4663         // 64 bit systems (kNeedI64RegPair == false):
4664         {WasmCode::kWasmI64AtomicWait64, WasmCode::kWasmI32AtomicWait64},
4665         // 32 bit systems (kNeedI64RegPair == true):
4666         {WasmCode::kWasmI64AtomicWait32, WasmCode::kWasmI32AtomicWait32}};
4667     auto target = kTargets[kNeedI64RegPair][kind == kI32];
4668 
4669     CallRuntimeStub(target, MakeSig::Params(kPointerKind, kind, kI64),
4670                     {index, expected_value, timeout}, decoder->position());
4671     // Pop parameters from the value stack.
4672     __ DropValues(3);
4673 
4674     RegisterDebugSideTableEntry(decoder, DebugSideTableBuilder::kDidSpill);
4675 
4676     __ PushRegister(kI32, LiftoffRegister(kReturnRegister0));
4677   }
4678 
AtomicNotify(FullDecoder * decoder,const MemoryAccessImmediate<validate> & imm)4679   void AtomicNotify(FullDecoder* decoder,
4680                     const MemoryAccessImmediate<validate>& imm) {
4681     LiftoffRegister full_index = __ PeekToRegister(1, {});
4682     Register index_reg = BoundsCheckMem(decoder, kInt32Size, imm.offset,
4683                                         full_index, {}, kDoForceCheck);
4684     if (index_reg == no_reg) return;
4685     LiftoffRegList pinned = {index_reg};
4686     AlignmentCheckMem(decoder, kInt32Size, imm.offset, index_reg, pinned);
4687 
4688     uintptr_t offset = imm.offset;
4689     Register index_plus_offset =
4690         __ cache_state()->is_used(LiftoffRegister(index_reg))
4691             ? pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp()
4692             : index_reg;
4693     // TODO(clemensb): Skip this if memory is 64 bit.
4694     __ emit_ptrsize_zeroextend_i32(index_plus_offset, index_reg);
4695     if (offset) {
4696       __ emit_ptrsize_addi(index_plus_offset, index_plus_offset, offset);
4697     }
4698 
4699     LiftoffAssembler::VarState count = __ cache_state()->stack_state.end()[-1];
4700     LiftoffAssembler::VarState index = __ cache_state()->stack_state.end()[-2];
4701     index.MakeRegister(LiftoffRegister(index_plus_offset));
4702 
4703     CallRuntimeStub(WasmCode::kWasmAtomicNotify,
4704                     MakeSig::Returns(kI32).Params(kPointerKind, kI32),
4705                     {index, count}, decoder->position());
4706     // Pop parameters from the value stack.
4707     __ DropValues(2);
4708 
4709     RegisterDebugSideTableEntry(decoder, DebugSideTableBuilder::kDidSpill);
4710 
4711     __ PushRegister(kI32, LiftoffRegister(kReturnRegister0));
4712   }
4713 
4714 #define ATOMIC_STORE_LIST(V)        \
4715   V(I32AtomicStore, kI32Store)      \
4716   V(I64AtomicStore, kI64Store)      \
4717   V(I32AtomicStore8U, kI32Store8)   \
4718   V(I32AtomicStore16U, kI32Store16) \
4719   V(I64AtomicStore8U, kI64Store8)   \
4720   V(I64AtomicStore16U, kI64Store16) \
4721   V(I64AtomicStore32U, kI64Store32)
4722 
4723 #define ATOMIC_LOAD_LIST(V)        \
4724   V(I32AtomicLoad, kI32Load)       \
4725   V(I64AtomicLoad, kI64Load)       \
4726   V(I32AtomicLoad8U, kI32Load8U)   \
4727   V(I32AtomicLoad16U, kI32Load16U) \
4728   V(I64AtomicLoad8U, kI64Load8U)   \
4729   V(I64AtomicLoad16U, kI64Load16U) \
4730   V(I64AtomicLoad32U, kI64Load32U)
4731 
4732 #define ATOMIC_BINOP_INSTRUCTION_LIST(V)         \
4733   V(Add, I32AtomicAdd, kI32Store)                \
4734   V(Add, I64AtomicAdd, kI64Store)                \
4735   V(Add, I32AtomicAdd8U, kI32Store8)             \
4736   V(Add, I32AtomicAdd16U, kI32Store16)           \
4737   V(Add, I64AtomicAdd8U, kI64Store8)             \
4738   V(Add, I64AtomicAdd16U, kI64Store16)           \
4739   V(Add, I64AtomicAdd32U, kI64Store32)           \
4740   V(Sub, I32AtomicSub, kI32Store)                \
4741   V(Sub, I64AtomicSub, kI64Store)                \
4742   V(Sub, I32AtomicSub8U, kI32Store8)             \
4743   V(Sub, I32AtomicSub16U, kI32Store16)           \
4744   V(Sub, I64AtomicSub8U, kI64Store8)             \
4745   V(Sub, I64AtomicSub16U, kI64Store16)           \
4746   V(Sub, I64AtomicSub32U, kI64Store32)           \
4747   V(And, I32AtomicAnd, kI32Store)                \
4748   V(And, I64AtomicAnd, kI64Store)                \
4749   V(And, I32AtomicAnd8U, kI32Store8)             \
4750   V(And, I32AtomicAnd16U, kI32Store16)           \
4751   V(And, I64AtomicAnd8U, kI64Store8)             \
4752   V(And, I64AtomicAnd16U, kI64Store16)           \
4753   V(And, I64AtomicAnd32U, kI64Store32)           \
4754   V(Or, I32AtomicOr, kI32Store)                  \
4755   V(Or, I64AtomicOr, kI64Store)                  \
4756   V(Or, I32AtomicOr8U, kI32Store8)               \
4757   V(Or, I32AtomicOr16U, kI32Store16)             \
4758   V(Or, I64AtomicOr8U, kI64Store8)               \
4759   V(Or, I64AtomicOr16U, kI64Store16)             \
4760   V(Or, I64AtomicOr32U, kI64Store32)             \
4761   V(Xor, I32AtomicXor, kI32Store)                \
4762   V(Xor, I64AtomicXor, kI64Store)                \
4763   V(Xor, I32AtomicXor8U, kI32Store8)             \
4764   V(Xor, I32AtomicXor16U, kI32Store16)           \
4765   V(Xor, I64AtomicXor8U, kI64Store8)             \
4766   V(Xor, I64AtomicXor16U, kI64Store16)           \
4767   V(Xor, I64AtomicXor32U, kI64Store32)           \
4768   V(Exchange, I32AtomicExchange, kI32Store)      \
4769   V(Exchange, I64AtomicExchange, kI64Store)      \
4770   V(Exchange, I32AtomicExchange8U, kI32Store8)   \
4771   V(Exchange, I32AtomicExchange16U, kI32Store16) \
4772   V(Exchange, I64AtomicExchange8U, kI64Store8)   \
4773   V(Exchange, I64AtomicExchange16U, kI64Store16) \
4774   V(Exchange, I64AtomicExchange32U, kI64Store32)
4775 
4776 #define ATOMIC_COMPARE_EXCHANGE_LIST(V)       \
4777   V(I32AtomicCompareExchange, kI32Store)      \
4778   V(I64AtomicCompareExchange, kI64Store)      \
4779   V(I32AtomicCompareExchange8U, kI32Store8)   \
4780   V(I32AtomicCompareExchange16U, kI32Store16) \
4781   V(I64AtomicCompareExchange8U, kI64Store8)   \
4782   V(I64AtomicCompareExchange16U, kI64Store16) \
4783   V(I64AtomicCompareExchange32U, kI64Store32)
4784 
AtomicOp(FullDecoder * decoder,WasmOpcode opcode,base::Vector<Value> args,const MemoryAccessImmediate<validate> & imm,Value * result)4785   void AtomicOp(FullDecoder* decoder, WasmOpcode opcode,
4786                 base::Vector<Value> args,
4787                 const MemoryAccessImmediate<validate>& imm, Value* result) {
4788     switch (opcode) {
4789 #define ATOMIC_STORE_OP(name, type)                \
4790   case wasm::kExpr##name:                          \
4791     AtomicStoreMem(decoder, StoreType::type, imm); \
4792     break;
4793 
4794       ATOMIC_STORE_LIST(ATOMIC_STORE_OP)
4795 #undef ATOMIC_STORE_OP
4796 
4797 #define ATOMIC_LOAD_OP(name, type)               \
4798   case wasm::kExpr##name:                        \
4799     AtomicLoadMem(decoder, LoadType::type, imm); \
4800     break;
4801 
4802       ATOMIC_LOAD_LIST(ATOMIC_LOAD_OP)
4803 #undef ATOMIC_LOAD_OP
4804 
4805 #define ATOMIC_BINOP_OP(op, name, type)                                        \
4806   case wasm::kExpr##name:                                                      \
4807     AtomicBinop(decoder, StoreType::type, imm, &LiftoffAssembler::Atomic##op); \
4808     break;
4809 
4810       ATOMIC_BINOP_INSTRUCTION_LIST(ATOMIC_BINOP_OP)
4811 #undef ATOMIC_BINOP_OP
4812 
4813 #define ATOMIC_COMPARE_EXCHANGE_OP(name, type)            \
4814   case wasm::kExpr##name:                                 \
4815     AtomicCompareExchange(decoder, StoreType::type, imm); \
4816     break;
4817 
4818       ATOMIC_COMPARE_EXCHANGE_LIST(ATOMIC_COMPARE_EXCHANGE_OP)
4819 #undef ATOMIC_COMPARE_EXCHANGE_OP
4820 
4821       case kExprI32AtomicWait:
4822         AtomicWait(decoder, kI32, imm);
4823         break;
4824       case kExprI64AtomicWait:
4825         AtomicWait(decoder, kI64, imm);
4826         break;
4827       case kExprAtomicNotify:
4828         AtomicNotify(decoder, imm);
4829         break;
4830       default:
4831         unsupported(decoder, kAtomics, "atomicop");
4832     }
4833   }
4834 
4835 #undef ATOMIC_STORE_LIST
4836 #undef ATOMIC_LOAD_LIST
4837 #undef ATOMIC_BINOP_INSTRUCTION_LIST
4838 #undef ATOMIC_COMPARE_EXCHANGE_LIST
4839 
AtomicFence(FullDecoder * decoder)4840   void AtomicFence(FullDecoder* decoder) { __ AtomicFence(); }
4841 
4842   // Pop a memtype (i32 or i64 depending on {WasmModule::is_memory64}) to a
4843   // register, updating {*high_word} to contain the ORed combination of all
4844   // popped high words. Returns the ptrsized register holding the popped value.
PopMemTypeToRegister(FullDecoder * decoder,Register * high_word,LiftoffRegList * pinned)4845   LiftoffRegister PopMemTypeToRegister(FullDecoder* decoder,
4846                                        Register* high_word,
4847                                        LiftoffRegList* pinned) {
4848     LiftoffRegister reg = __ PopToRegister(*pinned);
4849     LiftoffRegister intptr_reg = reg;
4850     // For memory32 on 64-bit hosts, zero-extend.
4851     if (kSystemPointerSize == kInt64Size && !env_->module->is_memory64) {
4852       // Only overwrite {reg} if it's not used otherwise.
4853       if (pinned->has(reg) || __ cache_state()->is_used(reg)) {
4854         intptr_reg = __ GetUnusedRegister(kGpReg, *pinned);
4855       }
4856       __ emit_u32_to_uintptr(intptr_reg.gp(), reg.gp());
4857     }
4858     // For memory32 or memory64 on 64-bit, we are done here.
4859     if (kSystemPointerSize == kInt64Size || !env_->module->is_memory64) {
4860       pinned->set(intptr_reg);
4861       return intptr_reg;
4862     }
4863 
4864     // For memory64 on 32-bit systems, combine all high words for a zero-check
4865     // and only use the low words afterwards. This keeps the register pressure
4866     // managable.
4867     DCHECK_GE(kMaxUInt32, env_->max_memory_size);
4868     pinned->set(reg.low());
4869     if (*high_word == no_reg) {
4870       // Choose a register to hold the (combination of) high word(s). It cannot
4871       // be one of the pinned registers, and it cannot be used in the value
4872       // stack.
4873       *high_word =
4874           pinned->has(reg.high())
4875               ? __ GetUnusedRegister(kGpReg, *pinned).gp()
4876               : __ GetUnusedRegister(kGpReg, {reg.high()}, *pinned).gp();
4877       pinned->set(*high_word);
4878       if (*high_word != reg.high_gp()) {
4879         __ Move(*high_word, reg.high_gp(), kI32);
4880       }
4881     } else if (*high_word != reg.high_gp()) {
4882       // Combine the new high word into existing high words.
4883       __ emit_i32_or(*high_word, *high_word, reg.high_gp());
4884     }
4885     return reg.low();
4886   }
4887 
MemoryInit(FullDecoder * decoder,const MemoryInitImmediate<validate> & imm,const Value &,const Value &,const Value &)4888   void MemoryInit(FullDecoder* decoder,
4889                   const MemoryInitImmediate<validate>& imm, const Value&,
4890                   const Value&, const Value&) {
4891     Register mem_offsets_high_word = no_reg;
4892     LiftoffRegList pinned;
4893     LiftoffRegister size = pinned.set(__ PopToRegister());
4894     LiftoffRegister src = pinned.set(__ PopToRegister(pinned));
4895     LiftoffRegister dst =
4896         PopMemTypeToRegister(decoder, &mem_offsets_high_word, &pinned);
4897 
4898     Register instance = __ cache_state()->cached_instance;
4899     if (instance == no_reg) {
4900       instance = __ GetUnusedRegister(kGpReg, pinned).gp();
4901       __ LoadInstanceFromFrame(instance);
4902     }
4903     pinned.set(instance);
4904 
4905     // Only allocate the OOB code now, so the state of the stack is reflected
4906     // correctly.
4907     Label* trap_label =
4908         AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapMemOutOfBounds);
4909     if (mem_offsets_high_word != no_reg) {
4910       // If any high word has bits set, jump to the OOB trap.
4911       __ emit_cond_jump(kNotEqualZero, trap_label, kI32, mem_offsets_high_word);
4912       pinned.clear(mem_offsets_high_word);
4913     }
4914 
4915     LiftoffRegister segment_index =
4916         pinned.set(__ GetUnusedRegister(kGpReg, pinned));
4917     __ LoadConstant(segment_index, WasmValue(imm.data_segment.index));
4918 
4919     auto sig = MakeSig::Returns(kI32).Params(kPointerKind, kPointerKind, kI32,
4920                                              kI32, kI32);
4921     LiftoffRegister args[] = {LiftoffRegister(instance), dst, src,
4922                               segment_index, size};
4923     // We don't need the instance anymore after the call. We can use the
4924     // register for the result.
4925     LiftoffRegister result(instance);
4926     GenerateCCall(&result, &sig, kVoid, args,
4927                   ExternalReference::wasm_memory_init());
4928     __ emit_cond_jump(kEqual, trap_label, kI32, result.gp());
4929   }
4930 
DataDrop(FullDecoder * decoder,const IndexImmediate<validate> & imm)4931   void DataDrop(FullDecoder* decoder, const IndexImmediate<validate>& imm) {
4932     LiftoffRegList pinned;
4933 
4934     Register seg_size_array =
4935         pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
4936     LOAD_INSTANCE_FIELD(seg_size_array, DataSegmentSizes, kSystemPointerSize,
4937                         pinned);
4938 
4939     LiftoffRegister seg_index =
4940         pinned.set(__ GetUnusedRegister(kGpReg, pinned));
4941     // Scale the seg_index for the array access.
4942     __ LoadConstant(seg_index,
4943                     WasmValue(imm.index << value_kind_size_log2(kI32)));
4944 
4945     // Set the length of the segment to '0' to drop it.
4946     LiftoffRegister null_reg = pinned.set(__ GetUnusedRegister(kGpReg, pinned));
4947     __ LoadConstant(null_reg, WasmValue(0));
4948     __ Store(seg_size_array, seg_index.gp(), 0, null_reg, StoreType::kI32Store,
4949              pinned);
4950   }
4951 
MemoryCopy(FullDecoder * decoder,const MemoryCopyImmediate<validate> & imm,const Value &,const Value &,const Value &)4952   void MemoryCopy(FullDecoder* decoder,
4953                   const MemoryCopyImmediate<validate>& imm, const Value&,
4954                   const Value&, const Value&) {
4955     Register mem_offsets_high_word = no_reg;
4956     LiftoffRegList pinned;
4957     LiftoffRegister size = pinned.set(
4958         PopMemTypeToRegister(decoder, &mem_offsets_high_word, &pinned));
4959     LiftoffRegister src = pinned.set(
4960         PopMemTypeToRegister(decoder, &mem_offsets_high_word, &pinned));
4961     LiftoffRegister dst = pinned.set(
4962         PopMemTypeToRegister(decoder, &mem_offsets_high_word, &pinned));
4963 
4964     Register instance = __ cache_state()->cached_instance;
4965     if (instance == no_reg) {
4966       instance = __ GetUnusedRegister(kGpReg, pinned).gp();
4967       __ LoadInstanceFromFrame(instance);
4968     }
4969 
4970     // Only allocate the OOB code now, so the state of the stack is reflected
4971     // correctly.
4972     Label* trap_label =
4973         AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapMemOutOfBounds);
4974     if (mem_offsets_high_word != no_reg) {
4975       // If any high word has bits set, jump to the OOB trap.
4976       __ emit_cond_jump(kNotEqualZero, trap_label, kI32, mem_offsets_high_word);
4977     }
4978 
4979     auto sig = MakeSig::Returns(kI32).Params(kPointerKind, kPointerKind,
4980                                              kPointerKind, kPointerKind);
4981     LiftoffRegister args[] = {LiftoffRegister(instance), dst, src, size};
4982     // We don't need the instance anymore after the call. We can use the
4983     // register for the result.
4984     LiftoffRegister result(instance);
4985     GenerateCCall(&result, &sig, kVoid, args,
4986                   ExternalReference::wasm_memory_copy());
4987     __ emit_cond_jump(kEqual, trap_label, kI32, result.gp());
4988   }
4989 
MemoryFill(FullDecoder * decoder,const MemoryIndexImmediate<validate> & imm,const Value &,const Value &,const Value &)4990   void MemoryFill(FullDecoder* decoder,
4991                   const MemoryIndexImmediate<validate>& imm, const Value&,
4992                   const Value&, const Value&) {
4993     Register mem_offsets_high_word = no_reg;
4994     LiftoffRegList pinned;
4995     LiftoffRegister size = pinned.set(
4996         PopMemTypeToRegister(decoder, &mem_offsets_high_word, &pinned));
4997     LiftoffRegister value = pinned.set(__ PopToRegister(pinned));
4998     LiftoffRegister dst = pinned.set(
4999         PopMemTypeToRegister(decoder, &mem_offsets_high_word, &pinned));
5000 
5001     Register instance = __ cache_state()->cached_instance;
5002     if (instance == no_reg) {
5003       instance = __ GetUnusedRegister(kGpReg, pinned).gp();
5004       __ LoadInstanceFromFrame(instance);
5005     }
5006 
5007     // Only allocate the OOB code now, so the state of the stack is reflected
5008     // correctly.
5009     Label* trap_label =
5010         AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapMemOutOfBounds);
5011     if (mem_offsets_high_word != no_reg) {
5012       // If any high word has bits set, jump to the OOB trap.
5013       __ emit_cond_jump(kNotEqualZero, trap_label, kI32, mem_offsets_high_word);
5014     }
5015 
5016     auto sig = MakeSig::Returns(kI32).Params(kPointerKind, kPointerKind, kI32,
5017                                              kPointerKind);
5018     LiftoffRegister args[] = {LiftoffRegister(instance), dst, value, size};
5019     // We don't need the instance anymore after the call. We can use the
5020     // register for the result.
5021     LiftoffRegister result(instance);
5022     GenerateCCall(&result, &sig, kVoid, args,
5023                   ExternalReference::wasm_memory_fill());
5024     __ emit_cond_jump(kEqual, trap_label, kI32, result.gp());
5025   }
5026 
LoadSmi(LiftoffRegister reg,int value)5027   void LoadSmi(LiftoffRegister reg, int value) {
5028     Address smi_value = Smi::FromInt(value).ptr();
5029     using smi_type = std::conditional_t<kSmiKind == kI32, int32_t, int64_t>;
5030     __ LoadConstant(reg, WasmValue{static_cast<smi_type>(smi_value)});
5031   }
5032 
TableInit(FullDecoder * decoder,const TableInitImmediate<validate> & imm,base::Vector<Value> args)5033   void TableInit(FullDecoder* decoder, const TableInitImmediate<validate>& imm,
5034                  base::Vector<Value> args) {
5035     LiftoffRegList pinned;
5036     LiftoffRegister table_index_reg =
5037         pinned.set(__ GetUnusedRegister(kGpReg, pinned));
5038 
5039     LoadSmi(table_index_reg, imm.table.index);
5040     LiftoffAssembler::VarState table_index(kPointerKind, table_index_reg, 0);
5041 
5042     LiftoffRegister segment_index_reg =
5043         pinned.set(__ GetUnusedRegister(kGpReg, pinned));
5044     LoadSmi(segment_index_reg, imm.element_segment.index);
5045     LiftoffAssembler::VarState segment_index(kPointerKind, segment_index_reg,
5046                                              0);
5047 
5048     LiftoffAssembler::VarState size = __ cache_state()->stack_state.end()[-1];
5049     LiftoffAssembler::VarState src = __ cache_state()->stack_state.end()[-2];
5050     LiftoffAssembler::VarState dst = __ cache_state()->stack_state.end()[-3];
5051 
5052     CallRuntimeStub(WasmCode::kWasmTableInit,
5053                     MakeSig::Params(kI32, kI32, kI32, kSmiKind, kSmiKind),
5054                     {dst, src, size, table_index, segment_index},
5055                     decoder->position());
5056 
5057     // Pop parameters from the value stack.
5058     __ cache_state()->stack_state.pop_back(3);
5059 
5060     RegisterDebugSideTableEntry(decoder, DebugSideTableBuilder::kDidSpill);
5061   }
5062 
ElemDrop(FullDecoder * decoder,const IndexImmediate<validate> & imm)5063   void ElemDrop(FullDecoder* decoder, const IndexImmediate<validate>& imm) {
5064     LiftoffRegList pinned;
5065     Register dropped_elem_segments =
5066         pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
5067     LOAD_INSTANCE_FIELD(dropped_elem_segments, DroppedElemSegments,
5068                         kSystemPointerSize, pinned);
5069 
5070     LiftoffRegister seg_index =
5071         pinned.set(__ GetUnusedRegister(kGpReg, pinned));
5072     __ LoadConstant(seg_index, WasmValue(imm.index));
5073 
5074     // Mark the segment as dropped by setting its value in the dropped
5075     // segments list to 1.
5076     LiftoffRegister one_reg = pinned.set(__ GetUnusedRegister(kGpReg, pinned));
5077     __ LoadConstant(one_reg, WasmValue(1));
5078     __ Store(dropped_elem_segments, seg_index.gp(), 0, one_reg,
5079              StoreType::kI32Store8, pinned);
5080   }
5081 
TableCopy(FullDecoder * decoder,const TableCopyImmediate<validate> & imm,base::Vector<Value> args)5082   void TableCopy(FullDecoder* decoder, const TableCopyImmediate<validate>& imm,
5083                  base::Vector<Value> args) {
5084     LiftoffRegList pinned;
5085 
5086     LiftoffRegister table_dst_index_reg =
5087         pinned.set(__ GetUnusedRegister(kGpReg, pinned));
5088     LoadSmi(table_dst_index_reg, imm.table_dst.index);
5089     LiftoffAssembler::VarState table_dst_index(kPointerKind,
5090                                                table_dst_index_reg, 0);
5091 
5092     LiftoffRegister table_src_index_reg =
5093         pinned.set(__ GetUnusedRegister(kGpReg, pinned));
5094     LoadSmi(table_src_index_reg, imm.table_src.index);
5095     LiftoffAssembler::VarState table_src_index(kPointerKind,
5096                                                table_src_index_reg, 0);
5097 
5098     LiftoffAssembler::VarState size = __ cache_state()->stack_state.end()[-1];
5099     LiftoffAssembler::VarState src = __ cache_state()->stack_state.end()[-2];
5100     LiftoffAssembler::VarState dst = __ cache_state()->stack_state.end()[-3];
5101 
5102     CallRuntimeStub(WasmCode::kWasmTableCopy,
5103                     MakeSig::Params(kI32, kI32, kI32, kSmiKind, kSmiKind),
5104                     {dst, src, size, table_dst_index, table_src_index},
5105                     decoder->position());
5106 
5107     // Pop parameters from the value stack.
5108     __ cache_state()->stack_state.pop_back(3);
5109 
5110     RegisterDebugSideTableEntry(decoder, DebugSideTableBuilder::kDidSpill);
5111   }
5112 
TableGrow(FullDecoder * decoder,const IndexImmediate<validate> & imm,const Value &,const Value &,Value * result)5113   void TableGrow(FullDecoder* decoder, const IndexImmediate<validate>& imm,
5114                  const Value&, const Value&, Value* result) {
5115     LiftoffRegList pinned;
5116 
5117     LiftoffRegister table_index_reg =
5118         pinned.set(__ GetUnusedRegister(kGpReg, pinned));
5119     LoadSmi(table_index_reg, imm.index);
5120     LiftoffAssembler::VarState table_index(kPointerKind, table_index_reg, 0);
5121 
5122     LiftoffAssembler::VarState delta = __ cache_state()->stack_state.end()[-1];
5123     LiftoffAssembler::VarState value = __ cache_state()->stack_state.end()[-2];
5124 
5125     CallRuntimeStub(
5126         WasmCode::kWasmTableGrow,
5127         MakeSig::Returns(kSmiKind).Params(kSmiKind, kI32, kTaggedKind),
5128         {table_index, delta, value}, decoder->position());
5129 
5130     // Pop parameters from the value stack.
5131     __ cache_state()->stack_state.pop_back(2);
5132 
5133     RegisterDebugSideTableEntry(decoder, DebugSideTableBuilder::kDidSpill);
5134     __ SmiToInt32(kReturnRegister0);
5135     __ PushRegister(kI32, LiftoffRegister(kReturnRegister0));
5136   }
5137 
TableSize(FullDecoder * decoder,const IndexImmediate<validate> & imm,Value *)5138   void TableSize(FullDecoder* decoder, const IndexImmediate<validate>& imm,
5139                  Value*) {
5140     // We have to look up instance->tables[table_index].length.
5141 
5142     LiftoffRegList pinned;
5143     // Get the number of calls array address.
5144     Register tables = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
5145     LOAD_TAGGED_PTR_INSTANCE_FIELD(tables, Tables, pinned);
5146 
5147     Register table = tables;
5148     __ LoadTaggedPointer(
5149         table, tables, no_reg,
5150         ObjectAccess::ElementOffsetInTaggedFixedArray(imm.index), pinned);
5151 
5152     int length_field_size = WasmTableObject::kCurrentLengthOffsetEnd -
5153                             WasmTableObject::kCurrentLengthOffset + 1;
5154 
5155     Register result = table;
5156     __ Load(LiftoffRegister(result), table, no_reg,
5157             wasm::ObjectAccess::ToTagged(WasmTableObject::kCurrentLengthOffset),
5158             length_field_size == 4 ? LoadType::kI32Load : LoadType::kI64Load,
5159             pinned);
5160 
5161     __ SmiUntag(result);
5162     __ PushRegister(kI32, LiftoffRegister(result));
5163   }
5164 
TableFill(FullDecoder * decoder,const IndexImmediate<validate> & imm,const Value &,const Value &,const Value &)5165   void TableFill(FullDecoder* decoder, const IndexImmediate<validate>& imm,
5166                  const Value&, const Value&, const Value&) {
5167     LiftoffRegList pinned;
5168 
5169     LiftoffRegister table_index_reg =
5170         pinned.set(__ GetUnusedRegister(kGpReg, pinned));
5171     LoadSmi(table_index_reg, imm.index);
5172     LiftoffAssembler::VarState table_index(kPointerKind, table_index_reg, 0);
5173 
5174     LiftoffAssembler::VarState count = __ cache_state()->stack_state.end()[-1];
5175     LiftoffAssembler::VarState value = __ cache_state()->stack_state.end()[-2];
5176     LiftoffAssembler::VarState start = __ cache_state()->stack_state.end()[-3];
5177 
5178     CallRuntimeStub(WasmCode::kWasmTableFill,
5179                     MakeSig::Params(kSmiKind, kI32, kI32, kTaggedKind),
5180                     {table_index, start, count, value}, decoder->position());
5181 
5182     // Pop parameters from the value stack.
5183     __ cache_state()->stack_state.pop_back(3);
5184 
5185     RegisterDebugSideTableEntry(decoder, DebugSideTableBuilder::kDidSpill);
5186   }
5187 
StructNew(FullDecoder * decoder,const StructIndexImmediate<validate> & imm,const Value & rtt,bool initial_values_on_stack)5188   void StructNew(FullDecoder* decoder,
5189                  const StructIndexImmediate<validate>& imm, const Value& rtt,
5190                  bool initial_values_on_stack) {
5191     LiftoffAssembler::VarState rtt_value =
5192         __ cache_state()->stack_state.end()[-1];
5193     CallRuntimeStub(WasmCode::kWasmAllocateStructWithRtt,
5194                     MakeSig::Returns(kRef).Params(rtt.type.kind()), {rtt_value},
5195                     decoder->position());
5196     // Drop the RTT.
5197     __ cache_state()->stack_state.pop_back(1);
5198 
5199     LiftoffRegister obj(kReturnRegister0);
5200     LiftoffRegList pinned = {obj};
5201     for (uint32_t i = imm.struct_type->field_count(); i > 0;) {
5202       i--;
5203       int offset = StructFieldOffset(imm.struct_type, i);
5204       ValueKind field_kind = imm.struct_type->field(i).kind();
5205       LiftoffRegister value = initial_values_on_stack
5206                                   ? pinned.set(__ PopToRegister(pinned))
5207                                   : pinned.set(__ GetUnusedRegister(
5208                                         reg_class_for(field_kind), pinned));
5209       if (!initial_values_on_stack) {
5210         if (!CheckSupportedType(decoder, field_kind, "default value")) return;
5211         SetDefaultValue(value, field_kind, pinned);
5212       }
5213       StoreObjectField(obj.gp(), no_reg, offset, value, pinned, field_kind);
5214       pinned.clear(value);
5215     }
5216     // If this assert fails then initialization of padding field might be
5217     // necessary.
5218     static_assert(Heap::kMinObjectSizeInTaggedWords == 2 &&
5219                       WasmStruct::kHeaderSize == 2 * kTaggedSize,
5220                   "empty struct might require initialization of padding field");
5221     __ PushRegister(kRef, obj);
5222   }
5223 
StructNewWithRtt(FullDecoder * decoder,const StructIndexImmediate<validate> & imm,const Value & rtt,const Value args[],Value * result)5224   void StructNewWithRtt(FullDecoder* decoder,
5225                         const StructIndexImmediate<validate>& imm,
5226                         const Value& rtt, const Value args[], Value* result) {
5227     StructNew(decoder, imm, rtt, true);
5228   }
5229 
StructNewDefault(FullDecoder * decoder,const StructIndexImmediate<validate> & imm,const Value & rtt,Value * result)5230   void StructNewDefault(FullDecoder* decoder,
5231                         const StructIndexImmediate<validate>& imm,
5232                         const Value& rtt, Value* result) {
5233     StructNew(decoder, imm, rtt, false);
5234   }
5235 
StructGet(FullDecoder * decoder,const Value & struct_obj,const FieldImmediate<validate> & field,bool is_signed,Value * result)5236   void StructGet(FullDecoder* decoder, const Value& struct_obj,
5237                  const FieldImmediate<validate>& field, bool is_signed,
5238                  Value* result) {
5239     const StructType* struct_type = field.struct_imm.struct_type;
5240     ValueKind field_kind = struct_type->field(field.field_imm.index).kind();
5241     if (!CheckSupportedType(decoder, field_kind, "field load")) return;
5242     int offset = StructFieldOffset(struct_type, field.field_imm.index);
5243     LiftoffRegList pinned;
5244     LiftoffRegister obj = pinned.set(__ PopToRegister(pinned));
5245     MaybeEmitNullCheck(decoder, obj.gp(), pinned, struct_obj.type);
5246     LiftoffRegister value =
5247         __ GetUnusedRegister(reg_class_for(field_kind), pinned);
5248     LoadObjectField(value, obj.gp(), no_reg, offset, field_kind, is_signed,
5249                     pinned);
5250     __ PushRegister(unpacked(field_kind), value);
5251   }
5252 
StructSet(FullDecoder * decoder,const Value & struct_obj,const FieldImmediate<validate> & field,const Value & field_value)5253   void StructSet(FullDecoder* decoder, const Value& struct_obj,
5254                  const FieldImmediate<validate>& field,
5255                  const Value& field_value) {
5256     const StructType* struct_type = field.struct_imm.struct_type;
5257     ValueKind field_kind = struct_type->field(field.field_imm.index).kind();
5258     int offset = StructFieldOffset(struct_type, field.field_imm.index);
5259     LiftoffRegList pinned;
5260     LiftoffRegister value = pinned.set(__ PopToRegister(pinned));
5261     LiftoffRegister obj = pinned.set(__ PopToRegister(pinned));
5262     MaybeEmitNullCheck(decoder, obj.gp(), pinned, struct_obj.type);
5263     StoreObjectField(obj.gp(), no_reg, offset, value, pinned, field_kind);
5264   }
5265 
ArrayNew(FullDecoder * decoder,const ArrayIndexImmediate<validate> & imm,ValueKind rtt_kind,bool initial_value_on_stack)5266   void ArrayNew(FullDecoder* decoder, const ArrayIndexImmediate<validate>& imm,
5267                 ValueKind rtt_kind, bool initial_value_on_stack) {
5268     // Max length check.
5269     {
5270       LiftoffRegister length =
5271           __ LoadToRegister(__ cache_state()->stack_state.end()[-2], {});
5272       Label* trap_label =
5273           AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapArrayTooLarge);
5274       __ emit_i32_cond_jumpi(kUnsignedGreaterThan, trap_label, length.gp(),
5275                              WasmArray::MaxLength(imm.array_type));
5276     }
5277     ValueKind elem_kind = imm.array_type->element_type().kind();
5278     int elem_size = value_kind_size(elem_kind);
5279     // Allocate the array.
5280     {
5281       LiftoffRegister elem_size_reg = __ GetUnusedRegister(kGpReg, {});
5282       LiftoffAssembler::VarState rtt_var =
5283           __ cache_state()->stack_state.end()[-1];
5284       LiftoffAssembler::VarState length_var =
5285           __ cache_state()->stack_state.end()[-2];
5286       __ LoadConstant(elem_size_reg, WasmValue(elem_size));
5287       LiftoffAssembler::VarState elem_size_var(kI32, elem_size_reg, 0);
5288 
5289       WasmCode::RuntimeStubId stub_id =
5290           initial_value_on_stack
5291               ? WasmCode::kWasmAllocateArray_Uninitialized
5292               : is_reference(elem_kind) ? WasmCode::kWasmAllocateArray_InitNull
5293                                         : WasmCode::kWasmAllocateArray_InitZero;
5294       CallRuntimeStub(
5295           stub_id, MakeSig::Returns(kRef).Params(rtt_kind, kI32, kI32),
5296           {rtt_var, length_var, elem_size_var}, decoder->position());
5297       // Drop the RTT.
5298       __ cache_state()->stack_state.pop_back(1);
5299     }
5300 
5301     LiftoffRegister obj(kReturnRegister0);
5302     if (initial_value_on_stack) {
5303       LiftoffRegList pinned = {obj};
5304       LiftoffRegister length = pinned.set(__ PopToModifiableRegister(pinned));
5305       LiftoffRegister value = pinned.set(__ PopToRegister(pinned));
5306 
5307       // Initialize the array's elements.
5308       LiftoffRegister offset = pinned.set(__ GetUnusedRegister(kGpReg, pinned));
5309       __ LoadConstant(
5310           offset,
5311           WasmValue(wasm::ObjectAccess::ToTagged(WasmArray::kHeaderSize)));
5312       LiftoffRegister end_offset = length;
5313       if (value_kind_size_log2(elem_kind) != 0) {
5314         __ emit_i32_shli(end_offset.gp(), length.gp(),
5315                          value_kind_size_log2(elem_kind));
5316       }
5317       __ emit_i32_add(end_offset.gp(), end_offset.gp(), offset.gp());
5318       Label loop, done;
5319       __ bind(&loop);
5320       __ emit_cond_jump(kUnsignedGreaterEqual, &done, kI32, offset.gp(),
5321                         end_offset.gp());
5322       StoreObjectField(obj.gp(), offset.gp(), 0, value, pinned, elem_kind);
5323       __ emit_i32_addi(offset.gp(), offset.gp(), elem_size);
5324       __ emit_jump(&loop);
5325 
5326       __ bind(&done);
5327     } else {
5328       if (!CheckSupportedType(decoder, elem_kind, "default value")) return;
5329       // Drop the length.
5330       __ cache_state()->stack_state.pop_back(1);
5331     }
5332     __ PushRegister(kRef, obj);
5333   }
5334 
ArrayNewWithRtt(FullDecoder * decoder,const ArrayIndexImmediate<validate> & imm,const Value & length_value,const Value & initial_value,const Value & rtt,Value * result)5335   void ArrayNewWithRtt(FullDecoder* decoder,
5336                        const ArrayIndexImmediate<validate>& imm,
5337                        const Value& length_value, const Value& initial_value,
5338                        const Value& rtt, Value* result) {
5339     ArrayNew(decoder, imm, rtt.type.kind(), true);
5340   }
5341 
ArrayNewDefault(FullDecoder * decoder,const ArrayIndexImmediate<validate> & imm,const Value & length,const Value & rtt,Value * result)5342   void ArrayNewDefault(FullDecoder* decoder,
5343                        const ArrayIndexImmediate<validate>& imm,
5344                        const Value& length, const Value& rtt, Value* result) {
5345     ArrayNew(decoder, imm, rtt.type.kind(), false);
5346   }
5347 
ArrayGet(FullDecoder * decoder,const Value & array_obj,const ArrayIndexImmediate<validate> & imm,const Value & index_val,bool is_signed,Value * result)5348   void ArrayGet(FullDecoder* decoder, const Value& array_obj,
5349                 const ArrayIndexImmediate<validate>& imm,
5350                 const Value& index_val, bool is_signed, Value* result) {
5351     LiftoffRegList pinned;
5352     LiftoffRegister index = pinned.set(__ PopToModifiableRegister(pinned));
5353     LiftoffRegister array = pinned.set(__ PopToRegister(pinned));
5354     MaybeEmitNullCheck(decoder, array.gp(), pinned, array_obj.type);
5355     BoundsCheckArray(decoder, array, index, pinned);
5356     ValueKind elem_kind = imm.array_type->element_type().kind();
5357     if (!CheckSupportedType(decoder, elem_kind, "array load")) return;
5358     int elem_size_shift = value_kind_size_log2(elem_kind);
5359     if (elem_size_shift != 0) {
5360       __ emit_i32_shli(index.gp(), index.gp(), elem_size_shift);
5361     }
5362     LiftoffRegister value =
5363         __ GetUnusedRegister(reg_class_for(elem_kind), pinned);
5364     LoadObjectField(value, array.gp(), index.gp(),
5365                     wasm::ObjectAccess::ToTagged(WasmArray::kHeaderSize),
5366                     elem_kind, is_signed, pinned);
5367     __ PushRegister(unpacked(elem_kind), value);
5368   }
5369 
ArraySet(FullDecoder * decoder,const Value & array_obj,const ArrayIndexImmediate<validate> & imm,const Value & index_val,const Value & value_val)5370   void ArraySet(FullDecoder* decoder, const Value& array_obj,
5371                 const ArrayIndexImmediate<validate>& imm,
5372                 const Value& index_val, const Value& value_val) {
5373     LiftoffRegList pinned;
5374     LiftoffRegister value = pinned.set(__ PopToRegister(pinned));
5375     DCHECK_EQ(reg_class_for(imm.array_type->element_type().kind()),
5376               value.reg_class());
5377     LiftoffRegister index = pinned.set(__ PopToModifiableRegister(pinned));
5378     LiftoffRegister array = pinned.set(__ PopToRegister(pinned));
5379     MaybeEmitNullCheck(decoder, array.gp(), pinned, array_obj.type);
5380     BoundsCheckArray(decoder, array, index, pinned);
5381     ValueKind elem_kind = imm.array_type->element_type().kind();
5382     int elem_size_shift = value_kind_size_log2(elem_kind);
5383     if (elem_size_shift != 0) {
5384       __ emit_i32_shli(index.gp(), index.gp(), elem_size_shift);
5385     }
5386     StoreObjectField(array.gp(), index.gp(),
5387                      wasm::ObjectAccess::ToTagged(WasmArray::kHeaderSize),
5388                      value, pinned, elem_kind);
5389   }
5390 
ArrayLen(FullDecoder * decoder,const Value & array_obj,Value * result)5391   void ArrayLen(FullDecoder* decoder, const Value& array_obj, Value* result) {
5392     LiftoffRegList pinned;
5393     LiftoffRegister obj = pinned.set(__ PopToRegister(pinned));
5394     MaybeEmitNullCheck(decoder, obj.gp(), pinned, array_obj.type);
5395     LiftoffRegister len = __ GetUnusedRegister(kGpReg, pinned);
5396     int kLengthOffset = wasm::ObjectAccess::ToTagged(WasmArray::kLengthOffset);
5397     LoadObjectField(len, obj.gp(), no_reg, kLengthOffset, kI32, false, pinned);
5398     __ PushRegister(kI32, len);
5399   }
5400 
ArrayCopy(FullDecoder * decoder,const Value & dst,const Value & dst_index,const Value & src,const Value & src_index,const Value & length)5401   void ArrayCopy(FullDecoder* decoder, const Value& dst, const Value& dst_index,
5402                  const Value& src, const Value& src_index,
5403                  const Value& length) {
5404     // TODO(7748): Unify implementation with TF: Implement this with
5405     // GenerateCCall. Remove runtime function and builtin in wasm.tq.
5406     CallRuntimeStub(FLAG_experimental_wasm_skip_bounds_checks
5407                         ? WasmCode::kWasmArrayCopy
5408                         : WasmCode::kWasmArrayCopyWithChecks,
5409                     MakeSig::Params(kI32, kI32, kI32, kOptRef, kOptRef),
5410                     // Builtin parameter order:
5411                     // [dst_index, src_index, length, dst, src].
5412                     {__ cache_state()->stack_state.end()[-4],
5413                      __ cache_state()->stack_state.end()[-2],
5414                      __ cache_state()->stack_state.end()[-1],
5415                      __ cache_state()->stack_state.end()[-5],
5416                      __ cache_state()->stack_state.end()[-3]},
5417                     decoder->position());
5418     __ cache_state()->stack_state.pop_back(5);
5419   }
5420 
ArrayInit(FullDecoder * decoder,const ArrayIndexImmediate<validate> & imm,const base::Vector<Value> & elements,const Value & rtt,Value * result)5421   void ArrayInit(FullDecoder* decoder, const ArrayIndexImmediate<validate>& imm,
5422                  const base::Vector<Value>& elements, const Value& rtt,
5423                  Value* result) {
5424     ValueKind rtt_kind = rtt.type.kind();
5425     ValueKind elem_kind = imm.array_type->element_type().kind();
5426     // Allocate the array.
5427     {
5428       LiftoffRegList pinned;
5429       LiftoffRegister elem_size_reg =
5430           pinned.set(__ GetUnusedRegister(kGpReg, pinned));
5431 
5432       __ LoadConstant(elem_size_reg, WasmValue(value_kind_size(elem_kind)));
5433       LiftoffAssembler::VarState elem_size_var(kI32, elem_size_reg, 0);
5434 
5435       LiftoffRegister length_reg =
5436           pinned.set(__ GetUnusedRegister(kGpReg, pinned));
5437       __ LoadConstant(length_reg,
5438                       WasmValue(static_cast<int32_t>(elements.size())));
5439       LiftoffAssembler::VarState length_var(kI32, length_reg, 0);
5440 
5441       LiftoffAssembler::VarState rtt_var =
5442           __ cache_state()->stack_state.end()[-1];
5443 
5444       CallRuntimeStub(WasmCode::kWasmAllocateArray_Uninitialized,
5445                       MakeSig::Returns(kRef).Params(rtt_kind, kI32, kI32),
5446                       {rtt_var, length_var, elem_size_var},
5447                       decoder->position());
5448       // Drop the RTT.
5449       __ DropValues(1);
5450     }
5451 
5452     // Initialize the array with stack arguments.
5453     LiftoffRegister array(kReturnRegister0);
5454     if (!CheckSupportedType(decoder, elem_kind, "array.init")) return;
5455     for (int i = static_cast<int>(elements.size()) - 1; i >= 0; i--) {
5456       LiftoffRegList pinned = {array};
5457       LiftoffRegister element = pinned.set(__ PopToRegister(pinned));
5458       LiftoffRegister offset_reg =
5459           pinned.set(__ GetUnusedRegister(kGpReg, pinned));
5460       __ LoadConstant(offset_reg,
5461                       WasmValue(i << value_kind_size_log2(elem_kind)));
5462       StoreObjectField(array.gp(), offset_reg.gp(),
5463                        wasm::ObjectAccess::ToTagged(WasmArray::kHeaderSize),
5464                        element, pinned, elem_kind);
5465     }
5466 
5467     // Push the array onto the stack.
5468     __ PushRegister(kRef, array);
5469   }
5470 
ArrayInitFromData(FullDecoder * decoder,const ArrayIndexImmediate<validate> & array_imm,const IndexImmediate<validate> & data_segment,const Value &,const Value &,const Value &,Value *)5471   void ArrayInitFromData(FullDecoder* decoder,
5472                          const ArrayIndexImmediate<validate>& array_imm,
5473                          const IndexImmediate<validate>& data_segment,
5474                          const Value& /* offset */, const Value& /* length */,
5475                          const Value& /* rtt */, Value* /* result */) {
5476     LiftoffRegList pinned;
5477     LiftoffRegister data_segment_reg =
5478         pinned.set(__ GetUnusedRegister(kGpReg, pinned));
5479     __ LoadConstant(data_segment_reg,
5480                     WasmValue(static_cast<int32_t>(data_segment.index)));
5481     LiftoffAssembler::VarState data_segment_var(kI32, data_segment_reg, 0);
5482 
5483     CallRuntimeStub(WasmCode::kWasmArrayInitFromData,
5484                     MakeSig::Returns(kRef).Params(kI32, kI32, kI32, kRtt),
5485                     {
5486                         data_segment_var,
5487                         __ cache_state()->stack_state.end()[-3],  // offset
5488                         __ cache_state()->stack_state.end()[-2],  // length
5489                         __ cache_state()->stack_state.end()[-1]   // rtt
5490                     },
5491                     decoder->position());
5492 
5493     LiftoffRegister result(kReturnRegister0);
5494     // Reuse the data segment register for error handling.
5495     LiftoffRegister error_smi = data_segment_reg;
5496     LoadSmi(error_smi, kArrayInitFromDataArrayTooLargeErrorCode);
5497     Label* trap_label_array_too_large =
5498         AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapArrayTooLarge);
5499     __ emit_cond_jump(kEqual, trap_label_array_too_large, kRef, result.gp(),
5500                       error_smi.gp());
5501     LoadSmi(error_smi, kArrayInitFromDataSegmentOutOfBoundsErrorCode);
5502     Label* trap_label_segment_out_of_bounds = AddOutOfLineTrap(
5503         decoder, WasmCode::kThrowWasmTrapDataSegmentOutOfBounds);
5504     __ emit_cond_jump(kEqual, trap_label_segment_out_of_bounds, kRef,
5505                       result.gp(), error_smi.gp());
5506 
5507     __ PushRegister(kRef, result);
5508   }
5509 
5510   // 1 bit Smi tag, 31 bits Smi shift, 1 bit i31ref high-bit truncation.
5511   constexpr static int kI31To32BitSmiShift = 33;
5512 
I31New(FullDecoder * decoder,const Value & input,Value * result)5513   void I31New(FullDecoder* decoder, const Value& input, Value* result) {
5514     LiftoffRegister src = __ PopToRegister();
5515     LiftoffRegister dst = __ GetUnusedRegister(kGpReg, {src}, {});
5516     if (SmiValuesAre31Bits()) {
5517       STATIC_ASSERT(kSmiTag == 0);
5518       __ emit_i32_shli(dst.gp(), src.gp(), kSmiTagSize);
5519     } else {
5520       DCHECK(SmiValuesAre32Bits());
5521       __ emit_i64_shli(dst, src, kI31To32BitSmiShift);
5522     }
5523     __ PushRegister(kRef, dst);
5524   }
5525 
I31GetS(FullDecoder * decoder,const Value & input,Value * result)5526   void I31GetS(FullDecoder* decoder, const Value& input, Value* result) {
5527     LiftoffRegister src = __ PopToRegister();
5528     LiftoffRegister dst = __ GetUnusedRegister(kGpReg, {src}, {});
5529     if (SmiValuesAre31Bits()) {
5530       __ emit_i32_sari(dst.gp(), src.gp(), kSmiTagSize);
5531     } else {
5532       DCHECK(SmiValuesAre32Bits());
5533       __ emit_i64_sari(dst, src, kI31To32BitSmiShift);
5534     }
5535     __ PushRegister(kI32, dst);
5536   }
5537 
I31GetU(FullDecoder * decoder,const Value & input,Value * result)5538   void I31GetU(FullDecoder* decoder, const Value& input, Value* result) {
5539     LiftoffRegister src = __ PopToRegister();
5540     LiftoffRegister dst = __ GetUnusedRegister(kGpReg, {src}, {});
5541     if (SmiValuesAre31Bits()) {
5542       __ emit_i32_shri(dst.gp(), src.gp(), kSmiTagSize);
5543     } else {
5544       DCHECK(SmiValuesAre32Bits());
5545       __ emit_i64_shri(dst, src, kI31To32BitSmiShift);
5546     }
5547     __ PushRegister(kI32, dst);
5548   }
5549 
RttCanon(FullDecoder * decoder,uint32_t type_index,Value * result)5550   void RttCanon(FullDecoder* decoder, uint32_t type_index, Value* result) {
5551     LiftoffRegister rtt = __ GetUnusedRegister(kGpReg, {});
5552     LOAD_TAGGED_PTR_INSTANCE_FIELD(rtt.gp(), ManagedObjectMaps, {});
5553     __ LoadTaggedPointer(
5554         rtt.gp(), rtt.gp(), no_reg,
5555         wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(type_index), {});
5556     __ PushRegister(kRtt, rtt);
5557   }
5558 
5559   enum NullSucceeds : bool {  // --
5560     kNullSucceeds = true,
5561     kNullFails = false
5562   };
5563 
5564   // Falls through on match (=successful type check).
5565   // Returns the register containing the object.
SubtypeCheck(FullDecoder * decoder,const Value & obj,const Value & rtt,Label * no_match,NullSucceeds null_succeeds,LiftoffRegList pinned={},Register opt_scratch=no_reg)5566   LiftoffRegister SubtypeCheck(FullDecoder* decoder, const Value& obj,
5567                                const Value& rtt, Label* no_match,
5568                                NullSucceeds null_succeeds,
5569                                LiftoffRegList pinned = {},
5570                                Register opt_scratch = no_reg) {
5571     Label match;
5572     LiftoffRegister rtt_reg = pinned.set(__ PopToRegister(pinned));
5573     LiftoffRegister obj_reg = pinned.set(__ PopToRegister(pinned));
5574 
5575     // Reserve all temporary registers up front, so that the cache state
5576     // tracking doesn't get confused by the following conditional jumps.
5577     LiftoffRegister tmp1 =
5578         opt_scratch != no_reg
5579             ? LiftoffRegister(opt_scratch)
5580             : pinned.set(__ GetUnusedRegister(kGpReg, pinned));
5581     LiftoffRegister tmp2 = pinned.set(__ GetUnusedRegister(kGpReg, pinned));
5582     if (obj.type.is_nullable()) {
5583       LoadNullValue(tmp1.gp(), pinned);
5584       __ emit_cond_jump(kEqual, null_succeeds ? &match : no_match,
5585                         obj.type.kind(), obj_reg.gp(), tmp1.gp());
5586     }
5587 
5588     __ LoadMap(tmp1.gp(), obj_reg.gp());
5589     // {tmp1} now holds the object's map.
5590 
5591     // Check for rtt equality, and if not, check if the rtt is a struct/array
5592     // rtt.
5593     __ emit_cond_jump(kEqual, &match, rtt.type.kind(), tmp1.gp(), rtt_reg.gp());
5594 
5595     // Constant-time subtyping check: load exactly one candidate RTT from the
5596     // supertypes list.
5597     // Step 1: load the WasmTypeInfo into {tmp1}.
5598     constexpr int kTypeInfoOffset = wasm::ObjectAccess::ToTagged(
5599         Map::kConstructorOrBackPointerOrNativeContextOffset);
5600     __ LoadTaggedPointer(tmp1.gp(), tmp1.gp(), no_reg, kTypeInfoOffset, pinned);
5601     // Step 2: load the supertypes list into {tmp1}.
5602     constexpr int kSuperTypesOffset =
5603         wasm::ObjectAccess::ToTagged(WasmTypeInfo::kSupertypesOffset);
5604     __ LoadTaggedPointer(tmp1.gp(), tmp1.gp(), no_reg, kSuperTypesOffset,
5605                          pinned);
5606     // Step 3: check the list's length if needed.
5607     uint32_t rtt_depth =
5608         GetSubtypingDepth(decoder->module_, rtt.type.ref_index());
5609     if (rtt_depth >= kMinimumSupertypeArraySize) {
5610       LiftoffRegister list_length = tmp2;
5611       __ LoadFixedArrayLengthAsInt32(list_length, tmp1.gp(), pinned);
5612       __ emit_i32_cond_jumpi(kUnsignedLessEqual, no_match, list_length.gp(),
5613                              rtt_depth);
5614     }
5615     // Step 4: load the candidate list slot into {tmp1}, and compare it.
5616     __ LoadTaggedPointer(
5617         tmp1.gp(), tmp1.gp(), no_reg,
5618         wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(rtt_depth), pinned);
5619     __ emit_cond_jump(kUnequal, no_match, rtt.type.kind(), tmp1.gp(),
5620                       rtt_reg.gp());
5621 
5622     // Fall through to {match}.
5623     __ bind(&match);
5624     return obj_reg;
5625   }
5626 
RefTest(FullDecoder * decoder,const Value & obj,const Value & rtt,Value *)5627   void RefTest(FullDecoder* decoder, const Value& obj, const Value& rtt,
5628                Value* /* result_val */) {
5629     Label return_false, done;
5630     LiftoffRegList pinned;
5631     LiftoffRegister result = pinned.set(__ GetUnusedRegister(kGpReg, {}));
5632 
5633     SubtypeCheck(decoder, obj, rtt, &return_false, kNullFails, pinned,
5634                  result.gp());
5635 
5636     __ LoadConstant(result, WasmValue(1));
5637     // TODO(jkummerow): Emit near jumps on platforms where it's more efficient.
5638     __ emit_jump(&done);
5639 
5640     __ bind(&return_false);
5641     __ LoadConstant(result, WasmValue(0));
5642     __ bind(&done);
5643     __ PushRegister(kI32, result);
5644   }
5645 
RefCast(FullDecoder * decoder,const Value & obj,const Value & rtt,Value * result)5646   void RefCast(FullDecoder* decoder, const Value& obj, const Value& rtt,
5647                Value* result) {
5648     if (FLAG_experimental_wasm_assume_ref_cast_succeeds) {
5649       // Just drop the rtt.
5650       __ DropValues(1);
5651     } else {
5652       Label* trap_label =
5653           AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapIllegalCast);
5654       LiftoffRegister obj_reg =
5655           SubtypeCheck(decoder, obj, rtt, trap_label, kNullSucceeds);
5656       __ PushRegister(obj.type.kind(), obj_reg);
5657     }
5658   }
5659 
BrOnCast(FullDecoder * decoder,const Value & obj,const Value & rtt,Value *,uint32_t depth)5660   void BrOnCast(FullDecoder* decoder, const Value& obj, const Value& rtt,
5661                 Value* /* result_on_branch */, uint32_t depth) {
5662     // Before branching, materialize all constants. This avoids repeatedly
5663     // materializing them for each conditional branch.
5664     if (depth != decoder->control_depth() - 1) {
5665       __ MaterializeMergedConstants(
5666           decoder->control_at(depth)->br_merge()->arity);
5667     }
5668 
5669     Label cont_false;
5670     LiftoffRegister obj_reg =
5671         SubtypeCheck(decoder, obj, rtt, &cont_false, kNullFails);
5672 
5673     __ PushRegister(rtt.type.is_bottom() ? kBottom : obj.type.kind(), obj_reg);
5674     BrOrRet(decoder, depth, 0);
5675 
5676     __ bind(&cont_false);
5677     // Drop the branch's value, restore original value.
5678     Drop(decoder);
5679     __ PushRegister(obj.type.kind(), obj_reg);
5680   }
5681 
BrOnCastFail(FullDecoder * decoder,const Value & obj,const Value & rtt,Value *,uint32_t depth)5682   void BrOnCastFail(FullDecoder* decoder, const Value& obj, const Value& rtt,
5683                     Value* /* result_on_fallthrough */, uint32_t depth) {
5684     // Before branching, materialize all constants. This avoids repeatedly
5685     // materializing them for each conditional branch.
5686     if (depth != decoder->control_depth() - 1) {
5687       __ MaterializeMergedConstants(
5688           decoder->control_at(depth)->br_merge()->arity);
5689     }
5690 
5691     Label cont_branch, fallthrough;
5692     LiftoffRegister obj_reg =
5693         SubtypeCheck(decoder, obj, rtt, &cont_branch, kNullFails);
5694     __ PushRegister(obj.type.kind(), obj_reg);
5695     __ emit_jump(&fallthrough);
5696 
5697     __ bind(&cont_branch);
5698     BrOrRet(decoder, depth, 0);
5699 
5700     __ bind(&fallthrough);
5701   }
5702 
5703   // Abstract type checkers. They all return the object register and fall
5704   // through to match.
DataCheck(const Value & obj,Label * no_match,LiftoffRegList pinned,Register opt_scratch)5705   LiftoffRegister DataCheck(const Value& obj, Label* no_match,
5706                             LiftoffRegList pinned, Register opt_scratch) {
5707     TypeCheckRegisters registers =
5708         TypeCheckPrelude(obj, no_match, pinned, opt_scratch);
5709     EmitDataRefCheck(registers.map_reg.gp(), no_match, registers.tmp_reg,
5710                      pinned);
5711     return registers.obj_reg;
5712   }
5713 
ArrayCheck(const Value & obj,Label * no_match,LiftoffRegList pinned,Register opt_scratch)5714   LiftoffRegister ArrayCheck(const Value& obj, Label* no_match,
5715                              LiftoffRegList pinned, Register opt_scratch) {
5716     TypeCheckRegisters registers =
5717         TypeCheckPrelude(obj, no_match, pinned, opt_scratch);
5718     __ Load(registers.map_reg, registers.map_reg.gp(), no_reg,
5719             wasm::ObjectAccess::ToTagged(Map::kInstanceTypeOffset),
5720             LoadType::kI32Load16U, pinned);
5721     __ emit_i32_cond_jumpi(kUnequal, no_match, registers.map_reg.gp(),
5722                            WASM_ARRAY_TYPE);
5723     return registers.obj_reg;
5724   }
5725 
FuncCheck(const Value & obj,Label * no_match,LiftoffRegList pinned,Register opt_scratch)5726   LiftoffRegister FuncCheck(const Value& obj, Label* no_match,
5727                             LiftoffRegList pinned, Register opt_scratch) {
5728     TypeCheckRegisters registers =
5729         TypeCheckPrelude(obj, no_match, pinned, opt_scratch);
5730     __ Load(registers.map_reg, registers.map_reg.gp(), no_reg,
5731             wasm::ObjectAccess::ToTagged(Map::kInstanceTypeOffset),
5732             LoadType::kI32Load16U, pinned);
5733     __ emit_i32_cond_jumpi(kUnequal, no_match, registers.map_reg.gp(),
5734                            WASM_INTERNAL_FUNCTION_TYPE);
5735     return registers.obj_reg;
5736   }
5737 
I31Check(const Value & object,Label * no_match,LiftoffRegList pinned,Register opt_scratch)5738   LiftoffRegister I31Check(const Value& object, Label* no_match,
5739                            LiftoffRegList pinned, Register opt_scratch) {
5740     LiftoffRegister obj_reg = pinned.set(__ PopToRegister(pinned));
5741 
5742     __ emit_smi_check(obj_reg.gp(), no_match, LiftoffAssembler::kJumpOnNotSmi);
5743 
5744     return obj_reg;
5745   }
5746 
5747   using TypeChecker = LiftoffRegister (LiftoffCompiler::*)(
5748       const Value& obj, Label* no_match, LiftoffRegList pinned,
5749       Register opt_scratch);
5750 
5751   template <TypeChecker type_checker>
AbstractTypeCheck(const Value & object)5752   void AbstractTypeCheck(const Value& object) {
5753     Label match, no_match, done;
5754     LiftoffRegList pinned;
5755     LiftoffRegister result = pinned.set(__ GetUnusedRegister(kGpReg, {}));
5756 
5757     (this->*type_checker)(object, &no_match, pinned, result.gp());
5758 
5759     __ bind(&match);
5760     __ LoadConstant(result, WasmValue(1));
5761     // TODO(jkummerow): Emit near jumps on platforms where it's more efficient.
5762     __ emit_jump(&done);
5763 
5764     __ bind(&no_match);
5765     __ LoadConstant(result, WasmValue(0));
5766     __ bind(&done);
5767     __ PushRegister(kI32, result);
5768   }
5769 
RefIsData(FullDecoder *,const Value & object,Value *)5770   void RefIsData(FullDecoder* /* decoder */, const Value& object,
5771                  Value* /* result_val */) {
5772     return AbstractTypeCheck<&LiftoffCompiler::DataCheck>(object);
5773   }
5774 
RefIsFunc(FullDecoder *,const Value & object,Value *)5775   void RefIsFunc(FullDecoder* /* decoder */, const Value& object,
5776                  Value* /* result_val */) {
5777     return AbstractTypeCheck<&LiftoffCompiler::FuncCheck>(object);
5778   }
5779 
RefIsArray(FullDecoder *,const Value & object,Value *)5780   void RefIsArray(FullDecoder* /* decoder */, const Value& object,
5781                   Value* /* result_val */) {
5782     return AbstractTypeCheck<&LiftoffCompiler::ArrayCheck>(object);
5783   }
5784 
RefIsI31(FullDecoder * decoder,const Value & object,Value *)5785   void RefIsI31(FullDecoder* decoder, const Value& object,
5786                 Value* /* result */) {
5787     return AbstractTypeCheck<&LiftoffCompiler::I31Check>(object);
5788   }
5789 
5790   template <TypeChecker type_checker>
AbstractTypeCast(const Value & object,FullDecoder * decoder,ValueKind result_kind)5791   void AbstractTypeCast(const Value& object, FullDecoder* decoder,
5792                         ValueKind result_kind) {
5793     Label* trap_label =
5794         AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapIllegalCast);
5795     Label match;
5796     LiftoffRegister obj_reg =
5797         (this->*type_checker)(object, trap_label, {}, no_reg);
5798     __ bind(&match);
5799     __ PushRegister(result_kind, obj_reg);
5800   }
5801 
RefAsData(FullDecoder * decoder,const Value & object,Value *)5802   void RefAsData(FullDecoder* decoder, const Value& object,
5803                  Value* /* result */) {
5804     return AbstractTypeCast<&LiftoffCompiler::DataCheck>(object, decoder, kRef);
5805   }
5806 
RefAsFunc(FullDecoder * decoder,const Value & object,Value *)5807   void RefAsFunc(FullDecoder* decoder, const Value& object,
5808                  Value* /* result */) {
5809     return AbstractTypeCast<&LiftoffCompiler::FuncCheck>(object, decoder, kRef);
5810   }
5811 
RefAsI31(FullDecoder * decoder,const Value & object,Value * result)5812   void RefAsI31(FullDecoder* decoder, const Value& object, Value* result) {
5813     return AbstractTypeCast<&LiftoffCompiler::I31Check>(object, decoder, kRef);
5814   }
5815 
RefAsArray(FullDecoder * decoder,const Value & object,Value * result)5816   void RefAsArray(FullDecoder* decoder, const Value& object, Value* result) {
5817     return AbstractTypeCast<&LiftoffCompiler::ArrayCheck>(object, decoder,
5818                                                           kRef);
5819   }
5820 
5821   template <TypeChecker type_checker>
BrOnAbstractType(const Value & object,FullDecoder * decoder,uint32_t br_depth)5822   void BrOnAbstractType(const Value& object, FullDecoder* decoder,
5823                         uint32_t br_depth) {
5824     // Before branching, materialize all constants. This avoids repeatedly
5825     // materializing them for each conditional branch.
5826     if (br_depth != decoder->control_depth() - 1) {
5827       __ MaterializeMergedConstants(
5828           decoder->control_at(br_depth)->br_merge()->arity);
5829     }
5830 
5831     Label no_match;
5832     LiftoffRegister obj_reg =
5833         (this->*type_checker)(object, &no_match, {}, no_reg);
5834 
5835     __ PushRegister(kRef, obj_reg);
5836     BrOrRet(decoder, br_depth, 0);
5837 
5838     __ bind(&no_match);
5839   }
5840 
5841   template <TypeChecker type_checker>
BrOnNonAbstractType(const Value & object,FullDecoder * decoder,uint32_t br_depth)5842   void BrOnNonAbstractType(const Value& object, FullDecoder* decoder,
5843                            uint32_t br_depth) {
5844     // Before branching, materialize all constants. This avoids repeatedly
5845     // materializing them for each conditional branch.
5846     if (br_depth != decoder->control_depth() - 1) {
5847       __ MaterializeMergedConstants(
5848           decoder->control_at(br_depth)->br_merge()->arity);
5849     }
5850 
5851     Label no_match, end;
5852     LiftoffRegister obj_reg =
5853         (this->*type_checker)(object, &no_match, {}, no_reg);
5854     __ PushRegister(kRef, obj_reg);
5855     __ emit_jump(&end);
5856 
5857     __ bind(&no_match);
5858     BrOrRet(decoder, br_depth, 0);
5859 
5860     __ bind(&end);
5861   }
5862 
BrOnData(FullDecoder * decoder,const Value & object,Value *,uint32_t br_depth)5863   void BrOnData(FullDecoder* decoder, const Value& object,
5864                 Value* /* value_on_branch */, uint32_t br_depth) {
5865     return BrOnAbstractType<&LiftoffCompiler::DataCheck>(object, decoder,
5866                                                          br_depth);
5867   }
5868 
BrOnFunc(FullDecoder * decoder,const Value & object,Value *,uint32_t br_depth)5869   void BrOnFunc(FullDecoder* decoder, const Value& object,
5870                 Value* /* value_on_branch */, uint32_t br_depth) {
5871     return BrOnAbstractType<&LiftoffCompiler::FuncCheck>(object, decoder,
5872                                                          br_depth);
5873   }
5874 
BrOnI31(FullDecoder * decoder,const Value & object,Value *,uint32_t br_depth)5875   void BrOnI31(FullDecoder* decoder, const Value& object,
5876                Value* /* value_on_branch */, uint32_t br_depth) {
5877     return BrOnAbstractType<&LiftoffCompiler::I31Check>(object, decoder,
5878                                                         br_depth);
5879   }
5880 
BrOnArray(FullDecoder * decoder,const Value & object,Value *,uint32_t br_depth)5881   void BrOnArray(FullDecoder* decoder, const Value& object,
5882                  Value* /* value_on_branch */, uint32_t br_depth) {
5883     return BrOnAbstractType<&LiftoffCompiler::ArrayCheck>(object, decoder,
5884                                                           br_depth);
5885   }
5886 
BrOnNonData(FullDecoder * decoder,const Value & object,Value *,uint32_t br_depth)5887   void BrOnNonData(FullDecoder* decoder, const Value& object,
5888                    Value* /* value_on_branch */, uint32_t br_depth) {
5889     return BrOnNonAbstractType<&LiftoffCompiler::DataCheck>(object, decoder,
5890                                                             br_depth);
5891   }
5892 
BrOnNonFunc(FullDecoder * decoder,const Value & object,Value *,uint32_t br_depth)5893   void BrOnNonFunc(FullDecoder* decoder, const Value& object,
5894                    Value* /* value_on_branch */, uint32_t br_depth) {
5895     return BrOnNonAbstractType<&LiftoffCompiler::FuncCheck>(object, decoder,
5896                                                             br_depth);
5897   }
5898 
BrOnNonI31(FullDecoder * decoder,const Value & object,Value *,uint32_t br_depth)5899   void BrOnNonI31(FullDecoder* decoder, const Value& object,
5900                   Value* /* value_on_branch */, uint32_t br_depth) {
5901     return BrOnNonAbstractType<&LiftoffCompiler::I31Check>(object, decoder,
5902                                                            br_depth);
5903   }
5904 
BrOnNonArray(FullDecoder * decoder,const Value & object,Value *,uint32_t br_depth)5905   void BrOnNonArray(FullDecoder* decoder, const Value& object,
5906                     Value* /* value_on_branch */, uint32_t br_depth) {
5907     return BrOnNonAbstractType<&LiftoffCompiler::ArrayCheck>(object, decoder,
5908                                                              br_depth);
5909   }
5910 
Forward(FullDecoder * decoder,const Value & from,Value * to)5911   void Forward(FullDecoder* decoder, const Value& from, Value* to) {
5912     // Nothing to do here.
5913   }
5914 
5915  private:
CallDirect(FullDecoder * decoder,const CallFunctionImmediate<validate> & imm,const Value args[],Value returns[],TailCall tail_call)5916   void CallDirect(FullDecoder* decoder,
5917                   const CallFunctionImmediate<validate>& imm,
5918                   const Value args[], Value returns[], TailCall tail_call) {
5919     MostlySmallValueKindSig sig(compilation_zone_, imm.sig);
5920     for (ValueKind ret : sig.returns()) {
5921       if (!CheckSupportedType(decoder, ret, "return")) return;
5922     }
5923 
5924     auto call_descriptor =
5925         compiler::GetWasmCallDescriptor(compilation_zone_, imm.sig);
5926     call_descriptor =
5927         GetLoweredCallDescriptor(compilation_zone_, call_descriptor);
5928 
5929     // One slot would be enough for call_direct, but would make index
5930     // computations much more complicated.
5931     uintptr_t vector_slot = num_call_instructions_ * 2;
5932     if (FLAG_wasm_speculative_inlining) {
5933       base::MutexGuard mutex_guard(&decoder->module_->type_feedback.mutex);
5934       decoder->module_->type_feedback.feedback_for_function[func_index_]
5935           .positions[decoder->position()] =
5936           static_cast<int>(num_call_instructions_);
5937       num_call_instructions_++;
5938     }
5939 
5940     if (imm.index < env_->module->num_imported_functions) {
5941       // A direct call to an imported function.
5942       LiftoffRegList pinned;
5943       Register tmp = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
5944       Register target = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
5945 
5946       Register imported_targets = tmp;
5947       LOAD_INSTANCE_FIELD(imported_targets, ImportedFunctionTargets,
5948                           kSystemPointerSize, pinned);
5949       __ Load(LiftoffRegister(target), imported_targets, no_reg,
5950               imm.index * sizeof(Address), kPointerLoadType, pinned);
5951 
5952       Register imported_function_refs = tmp;
5953       LOAD_TAGGED_PTR_INSTANCE_FIELD(imported_function_refs,
5954                                      ImportedFunctionRefs, pinned);
5955       Register imported_function_ref = tmp;
5956       __ LoadTaggedPointer(
5957           imported_function_ref, imported_function_refs, no_reg,
5958           ObjectAccess::ElementOffsetInTaggedFixedArray(imm.index), pinned);
5959 
5960       Register* explicit_instance = &imported_function_ref;
5961       __ PrepareCall(&sig, call_descriptor, &target, explicit_instance);
5962       if (tail_call) {
5963         __ PrepareTailCall(
5964             static_cast<int>(call_descriptor->ParameterSlotCount()),
5965             static_cast<int>(
5966                 call_descriptor->GetStackParameterDelta(descriptor_)));
5967         __ TailCallIndirect(target);
5968       } else {
5969         source_position_table_builder_.AddPosition(
5970             __ pc_offset(), SourcePosition(decoder->position()), true);
5971         __ CallIndirect(&sig, call_descriptor, target);
5972         FinishCall(decoder, &sig, call_descriptor);
5973       }
5974     } else {
5975       // Inlining direct calls isn't speculative, but existence of the
5976       // feedback vector currently depends on this flag.
5977       if (FLAG_wasm_speculative_inlining) {
5978         LiftoffRegister vector = __ GetUnusedRegister(kGpReg, {});
5979         __ Fill(vector, liftoff::kFeedbackVectorOffset, kPointerKind);
5980         __ IncrementSmi(vector,
5981                         wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(
5982                             static_cast<int>(vector_slot)));
5983       }
5984       // A direct call within this module just gets the current instance.
5985       __ PrepareCall(&sig, call_descriptor);
5986       // Just encode the function index. This will be patched at instantiation.
5987       Address addr = static_cast<Address>(imm.index);
5988       if (tail_call) {
5989         DCHECK(descriptor_->CanTailCall(call_descriptor));
5990         __ PrepareTailCall(
5991             static_cast<int>(call_descriptor->ParameterSlotCount()),
5992             static_cast<int>(
5993                 call_descriptor->GetStackParameterDelta(descriptor_)));
5994         __ TailCallNativeWasmCode(addr);
5995       } else {
5996         source_position_table_builder_.AddPosition(
5997             __ pc_offset(), SourcePosition(decoder->position()), true);
5998         __ CallNativeWasmCode(addr);
5999         FinishCall(decoder, &sig, call_descriptor);
6000       }
6001     }
6002   }
6003 
CallIndirect(FullDecoder * decoder,const Value & index_val,const CallIndirectImmediate<validate> & imm,TailCall tail_call)6004   void CallIndirect(FullDecoder* decoder, const Value& index_val,
6005                     const CallIndirectImmediate<validate>& imm,
6006                     TailCall tail_call) {
6007     MostlySmallValueKindSig sig(compilation_zone_, imm.sig);
6008     for (ValueKind ret : sig.returns()) {
6009       if (!CheckSupportedType(decoder, ret, "return")) return;
6010     }
6011 
6012     // Pop the index. We'll modify the register's contents later.
6013     Register index = __ PopToModifiableRegister().gp();
6014 
6015     LiftoffRegList pinned = {index};
6016     // Get three temporary registers.
6017     Register table = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
6018     Register tmp_const = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
6019     Register scratch = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
6020     Register indirect_function_table = no_reg;
6021     if (imm.table_imm.index != 0) {
6022       Register indirect_function_tables =
6023           pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
6024       LOAD_TAGGED_PTR_INSTANCE_FIELD(indirect_function_tables,
6025                                      IndirectFunctionTables, pinned);
6026 
6027       indirect_function_table = indirect_function_tables;
6028       __ LoadTaggedPointer(
6029           indirect_function_table, indirect_function_tables, no_reg,
6030           ObjectAccess::ElementOffsetInTaggedFixedArray(imm.table_imm.index),
6031           pinned);
6032     }
6033 
6034     // Bounds check against the table size.
6035     Label* invalid_func_label =
6036         AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapTableOutOfBounds);
6037 
6038     uint32_t canonical_sig_num =
6039         env_->module->canonicalized_type_ids[imm.sig_imm.index];
6040     DCHECK_GE(canonical_sig_num, 0);
6041     DCHECK_GE(kMaxInt, canonical_sig_num);
6042 
6043     // Compare against table size stored in
6044     // {instance->indirect_function_table_size}.
6045     if (imm.table_imm.index == 0) {
6046       LOAD_INSTANCE_FIELD(tmp_const, IndirectFunctionTableSize, kUInt32Size,
6047                           pinned);
6048     } else {
6049       __ Load(
6050           LiftoffRegister(tmp_const), indirect_function_table, no_reg,
6051           wasm::ObjectAccess::ToTagged(WasmIndirectFunctionTable::kSizeOffset),
6052           LoadType::kI32Load, pinned);
6053     }
6054     __ emit_cond_jump(kUnsignedGreaterEqual, invalid_func_label, kI32, index,
6055                       tmp_const);
6056 
6057     CODE_COMMENT("Check indirect call signature");
6058     // Load the signature from {instance->ift_sig_ids[key]}
6059     if (imm.table_imm.index == 0) {
6060       LOAD_INSTANCE_FIELD(table, IndirectFunctionTableSigIds,
6061                           kSystemPointerSize, pinned);
6062     } else {
6063       __ Load(LiftoffRegister(table), indirect_function_table, no_reg,
6064               wasm::ObjectAccess::ToTagged(
6065                   WasmIndirectFunctionTable::kSigIdsOffset),
6066               kPointerLoadType, pinned);
6067     }
6068     // Shift {index} by 2 (multiply by 4) to represent kInt32Size items.
6069     STATIC_ASSERT((1 << 2) == kInt32Size);
6070     __ emit_i32_shli(index, index, 2);
6071     __ Load(LiftoffRegister(scratch), table, index, 0, LoadType::kI32Load,
6072             pinned);
6073 
6074     // Compare against expected signature.
6075     __ LoadConstant(LiftoffRegister(tmp_const), WasmValue(canonical_sig_num));
6076 
6077     Label* sig_mismatch_label =
6078         AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapFuncSigMismatch);
6079     __ emit_cond_jump(kUnequal, sig_mismatch_label, kPointerKind, scratch,
6080                       tmp_const);
6081 
6082     // At this point {index} has already been multiplied by 4.
6083     CODE_COMMENT("Execute indirect call");
6084     if (kTaggedSize != kInt32Size) {
6085       DCHECK_EQ(kTaggedSize, kInt32Size * 2);
6086       // Multiply {index} by another 2 to represent kTaggedSize items.
6087       __ emit_i32_add(index, index, index);
6088     }
6089     // At this point {index} has already been multiplied by kTaggedSize.
6090 
6091     // Load the instance from {instance->ift_instances[key]}
6092     if (imm.table_imm.index == 0) {
6093       LOAD_TAGGED_PTR_INSTANCE_FIELD(table, IndirectFunctionTableRefs, pinned);
6094     } else {
6095       __ LoadTaggedPointer(
6096           table, indirect_function_table, no_reg,
6097           wasm::ObjectAccess::ToTagged(WasmIndirectFunctionTable::kRefsOffset),
6098           pinned);
6099     }
6100     __ LoadTaggedPointer(tmp_const, table, index,
6101                          ObjectAccess::ElementOffsetInTaggedFixedArray(0),
6102                          pinned);
6103 
6104     if (kTaggedSize != kSystemPointerSize) {
6105       DCHECK_EQ(kSystemPointerSize, kTaggedSize * 2);
6106       // Multiply {index} by another 2 to represent kSystemPointerSize items.
6107       __ emit_i32_add(index, index, index);
6108     }
6109     // At this point {index} has already been multiplied by kSystemPointerSize.
6110 
6111     Register* explicit_instance = &tmp_const;
6112 
6113     // Load the target from {instance->ift_targets[key]}
6114     if (imm.table_imm.index == 0) {
6115       LOAD_INSTANCE_FIELD(table, IndirectFunctionTableTargets,
6116                           kSystemPointerSize, pinned);
6117     } else {
6118       __ Load(LiftoffRegister(table), indirect_function_table, no_reg,
6119               wasm::ObjectAccess::ToTagged(
6120                   WasmIndirectFunctionTable::kTargetsOffset),
6121               kPointerLoadType, pinned);
6122     }
6123     __ Load(LiftoffRegister(scratch), table, index, 0, kPointerLoadType,
6124             pinned);
6125 
6126     auto call_descriptor =
6127         compiler::GetWasmCallDescriptor(compilation_zone_, imm.sig);
6128     call_descriptor =
6129         GetLoweredCallDescriptor(compilation_zone_, call_descriptor);
6130 
6131     Register target = scratch;
6132     __ PrepareCall(&sig, call_descriptor, &target, explicit_instance);
6133     if (tail_call) {
6134       __ PrepareTailCall(
6135           static_cast<int>(call_descriptor->ParameterSlotCount()),
6136           static_cast<int>(
6137               call_descriptor->GetStackParameterDelta(descriptor_)));
6138       __ TailCallIndirect(target);
6139     } else {
6140       source_position_table_builder_.AddPosition(
6141           __ pc_offset(), SourcePosition(decoder->position()), true);
6142       __ CallIndirect(&sig, call_descriptor, target);
6143 
6144       FinishCall(decoder, &sig, call_descriptor);
6145     }
6146   }
6147 
CallRef(FullDecoder * decoder,ValueType func_ref_type,const FunctionSig * type_sig,TailCall tail_call)6148   void CallRef(FullDecoder* decoder, ValueType func_ref_type,
6149                const FunctionSig* type_sig, TailCall tail_call) {
6150     MostlySmallValueKindSig sig(compilation_zone_, type_sig);
6151     for (ValueKind ret : sig.returns()) {
6152       if (!CheckSupportedType(decoder, ret, "return")) return;
6153     }
6154     compiler::CallDescriptor* call_descriptor =
6155         compiler::GetWasmCallDescriptor(compilation_zone_, type_sig);
6156     call_descriptor =
6157         GetLoweredCallDescriptor(compilation_zone_, call_descriptor);
6158 
6159     Register target_reg = no_reg, instance_reg = no_reg;
6160 
6161     if (FLAG_wasm_speculative_inlining) {
6162       ValueKind kIntPtrKind = kPointerKind;
6163 
6164       LiftoffRegList pinned;
6165       LiftoffRegister vector = pinned.set(__ GetUnusedRegister(kGpReg, pinned));
6166       LiftoffAssembler::VarState funcref =
6167           __ cache_state()->stack_state.end()[-1];
6168       if (funcref.is_reg()) pinned.set(funcref.reg());
6169       __ Fill(vector, liftoff::kFeedbackVectorOffset, kPointerKind);
6170       LiftoffAssembler::VarState vector_var(kPointerKind, vector, 0);
6171       LiftoffRegister index = pinned.set(__ GetUnusedRegister(kGpReg, pinned));
6172       uintptr_t vector_slot = num_call_instructions_ * 2;
6173       {
6174         base::MutexGuard mutex_guard(&decoder->module_->type_feedback.mutex);
6175         decoder->module_->type_feedback.feedback_for_function[func_index_]
6176             .positions[decoder->position()] =
6177             static_cast<int>(num_call_instructions_);
6178       }
6179       num_call_instructions_++;
6180       __ LoadConstant(index, WasmValue::ForUintPtr(vector_slot));
6181       LiftoffAssembler::VarState index_var(kIntPtrKind, index, 0);
6182 
6183       // CallRefIC(vector: FixedArray, index: intptr,
6184       //           funcref: WasmInternalFunction)
6185       CallRuntimeStub(WasmCode::kCallRefIC,
6186                       MakeSig::Returns(kPointerKind, kPointerKind)
6187                           .Params(kPointerKind, kIntPtrKind, kPointerKind),
6188                       {vector_var, index_var, funcref}, decoder->position());
6189 
6190       __ cache_state()->stack_state.pop_back(1);  // Drop funcref.
6191       target_reg = LiftoffRegister(kReturnRegister0).gp();
6192       instance_reg = LiftoffRegister(kReturnRegister1).gp();
6193 
6194     } else {  // FLAG_wasm_speculative_inlining
6195       // Non-feedback-collecting version.
6196       // Executing a write barrier needs temp registers; doing this on a
6197       // conditional branch confuses the LiftoffAssembler's register management.
6198       // Spill everything up front to work around that.
6199       __ SpillAllRegisters();
6200 
6201       // We limit ourselves to four registers:
6202       // (1) func_data, initially reused for func_ref.
6203       // (2) instance, initially used as temp.
6204       // (3) target, initially used as temp.
6205       // (4) temp.
6206       LiftoffRegList pinned;
6207       LiftoffRegister func_ref = pinned.set(__ PopToModifiableRegister(pinned));
6208       MaybeEmitNullCheck(decoder, func_ref.gp(), pinned, func_ref_type);
6209       LiftoffRegister instance =
6210           pinned.set(__ GetUnusedRegister(kGpReg, pinned));
6211       LiftoffRegister target = pinned.set(__ GetUnusedRegister(kGpReg, pinned));
6212       LiftoffRegister temp = pinned.set(__ GetUnusedRegister(kGpReg, pinned));
6213 
6214       // Load "ref" (instance or WasmApiFunctionRef) and target.
6215       __ LoadTaggedPointer(
6216           instance.gp(), func_ref.gp(), no_reg,
6217           wasm::ObjectAccess::ToTagged(WasmInternalFunction::kRefOffset),
6218           pinned);
6219 
6220 #ifdef V8_SANDBOXED_EXTERNAL_POINTERS
6221       LOAD_INSTANCE_FIELD(temp.gp(), IsolateRoot, kSystemPointerSize, pinned);
6222       __ LoadExternalPointer(target.gp(), func_ref.gp(),
6223                              WasmInternalFunction::kForeignAddressOffset,
6224                              kForeignForeignAddressTag, temp.gp());
6225 #else
6226       __ Load(target, func_ref.gp(), no_reg,
6227               wasm::ObjectAccess::ToTagged(
6228                   WasmInternalFunction::kForeignAddressOffset),
6229               kPointerLoadType, pinned);
6230 #endif
6231 
6232       Label perform_call;
6233 
6234       LiftoffRegister null_address = temp;
6235       __ LoadConstant(null_address, WasmValue::ForUintPtr(0));
6236       __ emit_cond_jump(kUnequal, &perform_call, kRef, target.gp(),
6237                         null_address.gp());
6238       // The cached target can only be null for WasmJSFunctions.
6239       __ LoadTaggedPointer(
6240           target.gp(), func_ref.gp(), no_reg,
6241           wasm::ObjectAccess::ToTagged(WasmInternalFunction::kCodeOffset),
6242           pinned);
6243 #ifdef V8_EXTERNAL_CODE_SPACE
6244       __ LoadCodeDataContainerEntry(target.gp(), target.gp());
6245 #else
6246       __ emit_ptrsize_addi(target.gp(), target.gp(),
6247                            wasm::ObjectAccess::ToTagged(Code::kHeaderSize));
6248 #endif
6249       // Fall through to {perform_call}.
6250 
6251       __ bind(&perform_call);
6252       // Now the call target is in {target}, and the right instance object
6253       // is in {instance}.
6254       target_reg = target.gp();
6255       instance_reg = instance.gp();
6256     }  // FLAG_wasm_speculative_inlining
6257 
6258     __ PrepareCall(&sig, call_descriptor, &target_reg, &instance_reg);
6259     if (tail_call) {
6260       __ PrepareTailCall(
6261           static_cast<int>(call_descriptor->ParameterSlotCount()),
6262           static_cast<int>(
6263               call_descriptor->GetStackParameterDelta(descriptor_)));
6264       __ TailCallIndirect(target_reg);
6265     } else {
6266       source_position_table_builder_.AddPosition(
6267           __ pc_offset(), SourcePosition(decoder->position()), true);
6268       __ CallIndirect(&sig, call_descriptor, target_reg);
6269 
6270       FinishCall(decoder, &sig, call_descriptor);
6271     }
6272   }
6273 
LoadNullValue(Register null,LiftoffRegList pinned)6274   void LoadNullValue(Register null, LiftoffRegList pinned) {
6275     LOAD_INSTANCE_FIELD(null, IsolateRoot, kSystemPointerSize, pinned);
6276     __ LoadFullPointer(null, null,
6277                        IsolateData::root_slot_offset(RootIndex::kNullValue));
6278   }
6279 
LoadExceptionSymbol(Register dst,LiftoffRegList pinned,RootIndex root_index)6280   void LoadExceptionSymbol(Register dst, LiftoffRegList pinned,
6281                            RootIndex root_index) {
6282     LOAD_INSTANCE_FIELD(dst, IsolateRoot, kSystemPointerSize, pinned);
6283     uint32_t offset_imm = IsolateData::root_slot_offset(root_index);
6284     __ LoadFullPointer(dst, dst, offset_imm);
6285   }
6286 
MaybeEmitNullCheck(FullDecoder * decoder,Register object,LiftoffRegList pinned,ValueType type)6287   void MaybeEmitNullCheck(FullDecoder* decoder, Register object,
6288                           LiftoffRegList pinned, ValueType type) {
6289     if (FLAG_experimental_wasm_skip_null_checks || !type.is_nullable()) return;
6290     Label* trap_label =
6291         AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapNullDereference);
6292     LiftoffRegister null = __ GetUnusedRegister(kGpReg, pinned);
6293     LoadNullValue(null.gp(), pinned);
6294     __ emit_cond_jump(LiftoffCondition::kEqual, trap_label, kOptRef, object,
6295                       null.gp());
6296   }
6297 
BoundsCheckArray(FullDecoder * decoder,LiftoffRegister array,LiftoffRegister index,LiftoffRegList pinned)6298   void BoundsCheckArray(FullDecoder* decoder, LiftoffRegister array,
6299                         LiftoffRegister index, LiftoffRegList pinned) {
6300     if (V8_UNLIKELY(FLAG_experimental_wasm_skip_bounds_checks)) return;
6301     Label* trap_label =
6302         AddOutOfLineTrap(decoder, WasmCode::kThrowWasmTrapArrayOutOfBounds);
6303     LiftoffRegister length = __ GetUnusedRegister(kGpReg, pinned);
6304     constexpr int kLengthOffset =
6305         wasm::ObjectAccess::ToTagged(WasmArray::kLengthOffset);
6306     __ Load(length, array.gp(), no_reg, kLengthOffset, LoadType::kI32Load,
6307             pinned);
6308     __ emit_cond_jump(LiftoffCondition::kUnsignedGreaterEqual, trap_label, kI32,
6309                       index.gp(), length.gp());
6310   }
6311 
StructFieldOffset(const StructType * struct_type,int field_index)6312   int StructFieldOffset(const StructType* struct_type, int field_index) {
6313     return wasm::ObjectAccess::ToTagged(WasmStruct::kHeaderSize +
6314                                         struct_type->field_offset(field_index));
6315   }
6316 
LoadObjectField(LiftoffRegister dst,Register src,Register offset_reg,int offset,ValueKind kind,bool is_signed,LiftoffRegList pinned)6317   void LoadObjectField(LiftoffRegister dst, Register src, Register offset_reg,
6318                        int offset, ValueKind kind, bool is_signed,
6319                        LiftoffRegList pinned) {
6320     if (is_reference(kind)) {
6321       __ LoadTaggedPointer(dst.gp(), src, offset_reg, offset, pinned);
6322     } else {
6323       // Primitive kind.
6324       LoadType load_type = LoadType::ForValueKind(kind, is_signed);
6325       __ Load(dst, src, offset_reg, offset, load_type, pinned);
6326     }
6327   }
6328 
StoreObjectField(Register obj,Register offset_reg,int offset,LiftoffRegister value,LiftoffRegList pinned,ValueKind kind)6329   void StoreObjectField(Register obj, Register offset_reg, int offset,
6330                         LiftoffRegister value, LiftoffRegList pinned,
6331                         ValueKind kind) {
6332     if (is_reference(kind)) {
6333       __ StoreTaggedPointer(obj, offset_reg, offset, value, pinned);
6334     } else {
6335       // Primitive kind.
6336       StoreType store_type = StoreType::ForValueKind(kind);
6337       __ Store(obj, offset_reg, offset, value, store_type, pinned);
6338     }
6339   }
6340 
SetDefaultValue(LiftoffRegister reg,ValueKind kind,LiftoffRegList pinned)6341   void SetDefaultValue(LiftoffRegister reg, ValueKind kind,
6342                        LiftoffRegList pinned) {
6343     DCHECK(is_defaultable(kind));
6344     switch (kind) {
6345       case kI8:
6346       case kI16:
6347       case kI32:
6348         return __ LoadConstant(reg, WasmValue(int32_t{0}));
6349       case kI64:
6350         return __ LoadConstant(reg, WasmValue(int64_t{0}));
6351       case kF32:
6352         return __ LoadConstant(reg, WasmValue(float{0.0}));
6353       case kF64:
6354         return __ LoadConstant(reg, WasmValue(double{0.0}));
6355       case kS128:
6356         DCHECK(CpuFeatures::SupportsWasmSimd128());
6357         return __ emit_s128_xor(reg, reg, reg);
6358       case kOptRef:
6359         return LoadNullValue(reg.gp(), pinned);
6360       case kRtt:
6361       case kVoid:
6362       case kBottom:
6363       case kRef:
6364         UNREACHABLE();
6365     }
6366   }
6367 
6368   struct TypeCheckRegisters {
6369     LiftoffRegister obj_reg, map_reg, tmp_reg;
6370   };
6371 
TypeCheckPrelude(const Value & obj,Label * no_match,LiftoffRegList pinned,Register opt_scratch)6372   TypeCheckRegisters TypeCheckPrelude(const Value& obj, Label* no_match,
6373                                       LiftoffRegList pinned,
6374                                       Register opt_scratch) {
6375     LiftoffRegister obj_reg = pinned.set(__ PopToRegister(pinned));
6376 
6377     // Reserve all temporary registers up front, so that the cache state
6378     // tracking doesn't get confused by the following conditional jumps.
6379     LiftoffRegister map_reg =
6380         opt_scratch != no_reg
6381             ? LiftoffRegister(opt_scratch)
6382             : pinned.set(__ GetUnusedRegister(kGpReg, pinned));
6383     LiftoffRegister tmp_reg = pinned.set(__ GetUnusedRegister(kGpReg, pinned));
6384 
6385     if (obj.type.is_nullable()) {
6386       LoadNullValue(map_reg.gp(), pinned);
6387       __ emit_cond_jump(kEqual, no_match, kOptRef, obj_reg.gp(), map_reg.gp());
6388     }
6389 
6390     __ emit_smi_check(obj_reg.gp(), no_match, LiftoffAssembler::kJumpOnSmi);
6391 
6392     __ LoadMap(map_reg.gp(), obj_reg.gp());
6393 
6394     return {obj_reg, map_reg, tmp_reg};
6395   }
6396 
EmitDataRefCheck(Register map,Label * not_data_ref,LiftoffRegister tmp,LiftoffRegList pinned)6397   void EmitDataRefCheck(Register map, Label* not_data_ref, LiftoffRegister tmp,
6398                         LiftoffRegList pinned) {
6399     constexpr int kInstanceTypeOffset =
6400         wasm::ObjectAccess::ToTagged(Map::kInstanceTypeOffset);
6401     __ Load(tmp, map, no_reg, kInstanceTypeOffset, LoadType::kI32Load16U,
6402             pinned);
6403     // We're going to test a range of WasmObject instance types with a single
6404     // unsigned comparison.
6405     __ emit_i32_subi(tmp.gp(), tmp.gp(), FIRST_WASM_OBJECT_TYPE);
6406     __ emit_i32_cond_jumpi(kUnsignedGreaterThan, not_data_ref, tmp.gp(),
6407                            LAST_WASM_OBJECT_TYPE - FIRST_WASM_OBJECT_TYPE);
6408   }
6409 
MaybeOSR()6410   void MaybeOSR() {
6411     if (V8_UNLIKELY(for_debugging_)) {
6412       __ MaybeOSR();
6413     }
6414   }
6415 
FinishCall(FullDecoder * decoder,ValueKindSig * sig,compiler::CallDescriptor * call_descriptor)6416   void FinishCall(FullDecoder* decoder, ValueKindSig* sig,
6417                   compiler::CallDescriptor* call_descriptor) {
6418     DefineSafepoint();
6419     RegisterDebugSideTableEntry(decoder, DebugSideTableBuilder::kDidSpill);
6420     int pc_offset = __ pc_offset();
6421     MaybeOSR();
6422     EmitLandingPad(decoder, pc_offset);
6423     __ FinishCall(sig, call_descriptor);
6424   }
6425 
CheckNan(LiftoffRegister src,LiftoffRegList pinned,ValueKind kind)6426   void CheckNan(LiftoffRegister src, LiftoffRegList pinned, ValueKind kind) {
6427     DCHECK(kind == ValueKind::kF32 || kind == ValueKind::kF64);
6428     auto nondeterminism_addr = __ GetUnusedRegister(kGpReg, pinned);
6429     __ LoadConstant(
6430         nondeterminism_addr,
6431         WasmValue::ForUintPtr(reinterpret_cast<uintptr_t>(nondeterminism_)));
6432     __ emit_set_if_nan(nondeterminism_addr.gp(), src.fp(), kind);
6433   }
6434 
CheckS128Nan(LiftoffRegister dst,LiftoffRegList pinned,ValueKind lane_kind)6435   void CheckS128Nan(LiftoffRegister dst, LiftoffRegList pinned,
6436                     ValueKind lane_kind) {
6437     RegClass rc = reg_class_for(kS128);
6438     LiftoffRegister tmp_gp = pinned.set(__ GetUnusedRegister(kGpReg, pinned));
6439     LiftoffRegister tmp_s128 = pinned.set(__ GetUnusedRegister(rc, pinned));
6440     LiftoffRegister nondeterminism_addr =
6441         pinned.set(__ GetUnusedRegister(kGpReg, pinned));
6442     __ LoadConstant(
6443         nondeterminism_addr,
6444         WasmValue::ForUintPtr(reinterpret_cast<uintptr_t>(nondeterminism_)));
6445     __ emit_s128_set_if_nan(nondeterminism_addr.gp(), dst, tmp_gp.gp(),
6446                             tmp_s128, lane_kind);
6447   }
6448 
has_outstanding_op() const6449   bool has_outstanding_op() const {
6450     return outstanding_op_ != kNoOutstandingOp;
6451   }
6452 
test_and_reset_outstanding_op(WasmOpcode opcode)6453   bool test_and_reset_outstanding_op(WasmOpcode opcode) {
6454     DCHECK_NE(kNoOutstandingOp, opcode);
6455     if (outstanding_op_ != opcode) return false;
6456     outstanding_op_ = kNoOutstandingOp;
6457     return true;
6458   }
6459 
TraceCacheState(FullDecoder * decoder) const6460   void TraceCacheState(FullDecoder* decoder) const {
6461     if (!FLAG_trace_liftoff) return;
6462     StdoutStream os;
6463     for (int control_depth = decoder->control_depth() - 1; control_depth >= -1;
6464          --control_depth) {
6465       auto* cache_state =
6466           control_depth == -1 ? __ cache_state()
6467                               : &decoder->control_at(control_depth)
6468                                      ->label_state;
6469       os << PrintCollection(cache_state->stack_state);
6470       if (control_depth != -1) PrintF("; ");
6471     }
6472     os << "\n";
6473   }
6474 
DefineSafepoint()6475   void DefineSafepoint() {
6476     auto safepoint = safepoint_table_builder_.DefineSafepoint(&asm_);
6477     __ cache_state()->DefineSafepoint(safepoint);
6478   }
6479 
DefineSafepointWithCalleeSavedRegisters()6480   void DefineSafepointWithCalleeSavedRegisters() {
6481     auto safepoint = safepoint_table_builder_.DefineSafepoint(&asm_);
6482     __ cache_state()->DefineSafepointWithCalleeSavedRegisters(safepoint);
6483   }
6484 
LoadInstanceIntoRegister(LiftoffRegList pinned,Register fallback)6485   Register LoadInstanceIntoRegister(LiftoffRegList pinned, Register fallback) {
6486     Register instance = __ cache_state()->cached_instance;
6487     if (instance == no_reg) {
6488       instance = __ cache_state()->TrySetCachedInstanceRegister(
6489           pinned | LiftoffRegList{fallback});
6490       if (instance == no_reg) instance = fallback;
6491       __ LoadInstanceFromFrame(instance);
6492     }
6493     return instance;
6494   }
6495 
6496   static constexpr WasmOpcode kNoOutstandingOp = kExprUnreachable;
6497   static constexpr base::EnumSet<ValueKind> kUnconditionallySupported{
6498       // MVP:
6499       kI32, kI64, kF32, kF64,
6500       // Extern ref:
6501       kRef, kOptRef, kRtt, kI8, kI16};
6502 
6503   LiftoffAssembler asm_;
6504 
6505   // Used for merging code generation of subsequent operations (via look-ahead).
6506   // Set by the first opcode, reset by the second.
6507   WasmOpcode outstanding_op_ = kNoOutstandingOp;
6508 
6509   // {supported_types_} is updated in {MaybeBailoutForUnsupportedType}.
6510   base::EnumSet<ValueKind> supported_types_ = kUnconditionallySupported;
6511   compiler::CallDescriptor* const descriptor_;
6512   CompilationEnv* const env_;
6513   DebugSideTableBuilder* const debug_sidetable_builder_;
6514   const ForDebugging for_debugging_;
6515   LiftoffBailoutReason bailout_reason_ = kSuccess;
6516   const int func_index_;
6517   ZoneVector<OutOfLineCode> out_of_line_code_;
6518   SourcePositionTableBuilder source_position_table_builder_;
6519   ZoneVector<trap_handler::ProtectedInstructionData> protected_instructions_;
6520   // Zone used to store information during compilation. The result will be
6521   // stored independently, such that this zone can die together with the
6522   // LiftoffCompiler after compilation.
6523   Zone* compilation_zone_;
6524   SafepointTableBuilder safepoint_table_builder_;
6525   // The pc offset of the instructions to reserve the stack frame. Needed to
6526   // patch the actually needed stack size in the end.
6527   uint32_t pc_offset_stack_frame_construction_ = 0;
6528   // For emitting breakpoint, we store a pointer to the position of the next
6529   // breakpoint, and a pointer after the list of breakpoints as end marker.
6530   // A single breakpoint at offset 0 indicates that we should prepare the
6531   // function for stepping by flooding it with breakpoints.
6532   const int* next_breakpoint_ptr_ = nullptr;
6533   const int* next_breakpoint_end_ = nullptr;
6534 
6535   // Introduce a dead breakpoint to ensure that the calculation of the return
6536   // address in OSR is correct.
6537   int dead_breakpoint_ = 0;
6538 
6539   // Remember whether the did function-entry break checks (for "hook on function
6540   // call" and "break on entry" a.k.a. instrumentation breakpoint). This happens
6541   // at the first breakable opcode in the function (if compiling for debugging).
6542   bool did_function_entry_break_checks_ = false;
6543 
6544   struct HandlerInfo {
6545     MovableLabel handler;
6546     int pc_offset;
6547   };
6548 
6549   ZoneVector<HandlerInfo> handlers_;
6550   int handler_table_offset_ = Assembler::kNoHandlerTable;
6551 
6552   // Current number of exception refs on the stack.
6553   int num_exceptions_ = 0;
6554 
6555   // Number of feedback-collecting call instructions encountered. While
6556   // compiling, also index of the next such instruction. Used for indexing type
6557   // feedback.
6558   uintptr_t num_call_instructions_ = 0;
6559 
6560   int32_t* max_steps_;
6561   int32_t* nondeterminism_;
6562 
6563   DISALLOW_IMPLICIT_CONSTRUCTORS(LiftoffCompiler);
6564 };
6565 
6566 // static
6567 constexpr WasmOpcode LiftoffCompiler::kNoOutstandingOp;
6568 // static
6569 constexpr base::EnumSet<ValueKind> LiftoffCompiler::kUnconditionallySupported;
6570 
6571 }  // namespace
6572 
ExecuteLiftoffCompilation(CompilationEnv * env,const FunctionBody & func_body,int func_index,ForDebugging for_debugging,const LiftoffOptions & compiler_options)6573 WasmCompilationResult ExecuteLiftoffCompilation(
6574     CompilationEnv* env, const FunctionBody& func_body, int func_index,
6575     ForDebugging for_debugging, const LiftoffOptions& compiler_options) {
6576   base::TimeTicks start_time;
6577   if (V8_UNLIKELY(FLAG_trace_wasm_compilation_times)) {
6578     start_time = base::TimeTicks::Now();
6579   }
6580   int func_body_size = static_cast<int>(func_body.end - func_body.start);
6581   TRACE_EVENT2(TRACE_DISABLED_BY_DEFAULT("v8.wasm.detailed"),
6582                "wasm.CompileBaseline", "funcIndex", func_index, "bodySize",
6583                func_body_size);
6584 
6585   Zone zone(GetWasmEngine()->allocator(), "LiftoffCompilationZone");
6586   auto call_descriptor = compiler::GetWasmCallDescriptor(&zone, func_body.sig);
6587   size_t code_size_estimate =
6588       WasmCodeManager::EstimateLiftoffCodeSize(func_body_size);
6589   // Allocate the initial buffer a bit bigger to avoid reallocation during code
6590   // generation. Overflows when casting to int are fine, as we will allocate at
6591   // least {AssemblerBase::kMinimalBufferSize} anyway, so in the worst case we
6592   // have to grow more often.
6593   int initial_buffer_size = static_cast<int>(128 + code_size_estimate * 4 / 3);
6594   std::unique_ptr<DebugSideTableBuilder> debug_sidetable_builder;
6595   if (compiler_options.debug_sidetable) {
6596     debug_sidetable_builder = std::make_unique<DebugSideTableBuilder>();
6597   }
6598   DCHECK_IMPLIES(compiler_options.max_steps, for_debugging == kForDebugging);
6599   WasmFeatures unused_detected_features;
6600   WasmFullDecoder<Decoder::kBooleanValidation, LiftoffCompiler> decoder(
6601       &zone, env->module, env->enabled_features,
6602       compiler_options.detected_features ? compiler_options.detected_features
6603                                          : &unused_detected_features,
6604       func_body, call_descriptor, env, &zone,
6605       NewAssemblerBuffer(initial_buffer_size), debug_sidetable_builder.get(),
6606       for_debugging, func_index, compiler_options.breakpoints,
6607       compiler_options.dead_breakpoint, compiler_options.max_steps,
6608       compiler_options.nondeterminism);
6609   decoder.Decode();
6610   LiftoffCompiler* compiler = &decoder.interface();
6611   if (decoder.failed()) compiler->OnFirstError(&decoder);
6612 
6613   if (auto* counters = compiler_options.counters) {
6614     // Check that the histogram for the bailout reasons has the correct size.
6615     DCHECK_EQ(0, counters->liftoff_bailout_reasons()->min());
6616     DCHECK_EQ(kNumBailoutReasons - 1,
6617               counters->liftoff_bailout_reasons()->max());
6618     DCHECK_EQ(kNumBailoutReasons,
6619               counters->liftoff_bailout_reasons()->num_buckets());
6620     // Register the bailout reason (can also be {kSuccess}).
6621     counters->liftoff_bailout_reasons()->AddSample(
6622         static_cast<int>(compiler->bailout_reason()));
6623   }
6624 
6625   if (compiler->did_bailout()) return WasmCompilationResult{};
6626 
6627   WasmCompilationResult result;
6628   compiler->GetCode(&result.code_desc);
6629   result.instr_buffer = compiler->ReleaseBuffer();
6630   result.source_positions = compiler->GetSourcePositionTable();
6631   result.protected_instructions_data = compiler->GetProtectedInstructionsData();
6632   result.frame_slot_count = compiler->GetTotalFrameSlotCountForGC();
6633   auto* lowered_call_desc = GetLoweredCallDescriptor(&zone, call_descriptor);
6634   result.tagged_parameter_slots = lowered_call_desc->GetTaggedParameterSlots();
6635   result.func_index = func_index;
6636   result.result_tier = ExecutionTier::kLiftoff;
6637   result.for_debugging = for_debugging;
6638   if (auto* debug_sidetable = compiler_options.debug_sidetable) {
6639     *debug_sidetable = debug_sidetable_builder->GenerateDebugSideTable();
6640   }
6641   result.feedback_vector_slots = compiler->GetFeedbackVectorSlots();
6642 
6643   if (V8_UNLIKELY(FLAG_trace_wasm_compilation_times)) {
6644     base::TimeDelta time = base::TimeTicks::Now() - start_time;
6645     int codesize = result.code_desc.body_size();
6646     StdoutStream{} << "Compiled function "
6647                    << reinterpret_cast<const void*>(env->module) << "#"
6648                    << func_index << " using Liftoff, took "
6649                    << time.InMilliseconds() << " ms and "
6650                    << zone.allocation_size() << " bytes; bodysize "
6651                    << func_body_size << " codesize " << codesize << std::endl;
6652   }
6653 
6654   DCHECK(result.succeeded());
6655   return result;
6656 }
6657 
GenerateLiftoffDebugSideTable(const WasmCode * code)6658 std::unique_ptr<DebugSideTable> GenerateLiftoffDebugSideTable(
6659     const WasmCode* code) {
6660   auto* native_module = code->native_module();
6661   auto* function = &native_module->module()->functions[code->index()];
6662   ModuleWireBytes wire_bytes{native_module->wire_bytes()};
6663   base::Vector<const byte> function_bytes =
6664       wire_bytes.GetFunctionBytes(function);
6665   CompilationEnv env = native_module->CreateCompilationEnv();
6666   FunctionBody func_body{function->sig, 0, function_bytes.begin(),
6667                          function_bytes.end()};
6668 
6669   Zone zone(GetWasmEngine()->allocator(), "LiftoffDebugSideTableZone");
6670   auto call_descriptor = compiler::GetWasmCallDescriptor(&zone, function->sig);
6671   DebugSideTableBuilder debug_sidetable_builder;
6672   WasmFeatures detected;
6673   constexpr int kSteppingBreakpoints[] = {0};
6674   DCHECK(code->for_debugging() == kForDebugging ||
6675          code->for_debugging() == kForStepping);
6676   base::Vector<const int> breakpoints =
6677       code->for_debugging() == kForStepping
6678           ? base::ArrayVector(kSteppingBreakpoints)
6679           : base::Vector<const int>{};
6680   WasmFullDecoder<Decoder::kBooleanValidation, LiftoffCompiler> decoder(
6681       &zone, native_module->module(), env.enabled_features, &detected,
6682       func_body, call_descriptor, &env, &zone,
6683       NewAssemblerBuffer(AssemblerBase::kDefaultBufferSize),
6684       &debug_sidetable_builder, code->for_debugging(), code->index(),
6685       breakpoints);
6686   decoder.Decode();
6687   DCHECK(decoder.ok());
6688   DCHECK(!decoder.interface().did_bailout());
6689   return debug_sidetable_builder.GenerateDebugSideTable();
6690 }
6691 
6692 #undef __
6693 #undef TRACE
6694 #undef WASM_INSTANCE_OBJECT_FIELD_OFFSET
6695 #undef WASM_INSTANCE_OBJECT_FIELD_SIZE
6696 #undef LOAD_INSTANCE_FIELD
6697 #undef LOAD_TAGGED_PTR_INSTANCE_FIELD
6698 #undef CODE_COMMENT
6699 
6700 }  // namespace wasm
6701 }  // namespace internal
6702 }  // namespace v8
6703