• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // Copyright 2015 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #include "src/compiler/wasm-compiler.h"
6 
7 #include <memory>
8 
9 #include "src/assembler-inl.h"
10 #include "src/assembler.h"
11 #include "src/base/optional.h"
12 #include "src/base/platform/elapsed-timer.h"
13 #include "src/base/platform/platform.h"
14 #include "src/base/v8-fallthrough.h"
15 #include "src/builtins/builtins.h"
16 #include "src/code-factory.h"
17 #include "src/compiler.h"
18 #include "src/compiler/access-builder.h"
19 #include "src/compiler/code-generator.h"
20 #include "src/compiler/common-operator.h"
21 #include "src/compiler/compiler-source-position-table.h"
22 #include "src/compiler/diamond.h"
23 #include "src/compiler/graph-visualizer.h"
24 #include "src/compiler/graph.h"
25 #include "src/compiler/instruction-selector.h"
26 #include "src/compiler/int64-lowering.h"
27 #include "src/compiler/js-graph.h"
28 #include "src/compiler/js-operator.h"
29 #include "src/compiler/linkage.h"
30 #include "src/compiler/machine-operator.h"
31 #include "src/compiler/node-matchers.h"
32 #include "src/compiler/node-origin-table.h"
33 #include "src/compiler/pipeline.h"
34 #include "src/compiler/simd-scalar-lowering.h"
35 #include "src/compiler/zone-stats.h"
36 #include "src/heap/factory.h"
37 #include "src/isolate-inl.h"
38 #include "src/log-inl.h"
39 #include "src/optimized-compilation-info.h"
40 #include "src/tracing/trace-event.h"
41 #include "src/trap-handler/trap-handler.h"
42 #include "src/wasm/function-body-decoder.h"
43 #include "src/wasm/function-compiler.h"
44 #include "src/wasm/jump-table-assembler.h"
45 #include "src/wasm/memory-tracing.h"
46 #include "src/wasm/wasm-code-manager.h"
47 #include "src/wasm/wasm-limits.h"
48 #include "src/wasm/wasm-linkage.h"
49 #include "src/wasm/wasm-module.h"
50 #include "src/wasm/wasm-objects-inl.h"
51 #include "src/wasm/wasm-opcodes.h"
52 #include "src/wasm/wasm-text.h"
53 
54 namespace v8 {
55 namespace internal {
56 namespace compiler {
57 
58 // TODO(titzer): pull WASM_64 up to a common header.
59 #if !V8_TARGET_ARCH_32_BIT || V8_TARGET_ARCH_X64
60 #define WASM_64 1
61 #else
62 #define WASM_64 0
63 #endif
64 
65 #define FATAL_UNSUPPORTED_OPCODE(opcode)        \
66   FATAL("Unsupported opcode 0x%x:%s", (opcode), \
67         wasm::WasmOpcodes::OpcodeName(opcode));
68 
69 #define WASM_INSTANCE_OBJECT_OFFSET(name) \
70   (WasmInstanceObject::k##name##Offset - kHeapObjectTag)
71 
72 #define LOAD_INSTANCE_FIELD(name, type)                                      \
73   SetEffect(graph()->NewNode(                                                \
74       mcgraph()->machine()->Load(type), instance_node_.get(),                \
75       mcgraph()->Int32Constant(WASM_INSTANCE_OBJECT_OFFSET(name)), Effect(), \
76       Control()))
77 
78 #define LOAD_FIXED_ARRAY_SLOT(array_node, index)                            \
79   SetEffect(graph()->NewNode(                                               \
80       mcgraph()->machine()->Load(MachineType::TaggedPointer()), array_node, \
81       mcgraph()->Int32Constant(FixedArrayOffsetMinusTag(index)), Effect(),  \
82       Control()))
83 
FixedArrayOffsetMinusTag(uint32_t index)84 int FixedArrayOffsetMinusTag(uint32_t index) {
85   auto access = AccessBuilder::ForFixedArraySlot(index);
86   return access.offset - access.tag();
87 }
88 
89 namespace {
90 
91 constexpr uint32_t kBytesPerExceptionValuesArrayElement = 2;
92 
MergeControlToEnd(MachineGraph * mcgraph,Node * node)93 void MergeControlToEnd(MachineGraph* mcgraph, Node* node) {
94   Graph* g = mcgraph->graph();
95   if (g->end()) {
96     NodeProperties::MergeControlToEnd(g, mcgraph->common(), node);
97   } else {
98     g->SetEnd(g->NewNode(mcgraph->common()->End(1), node));
99   }
100 }
101 
ContainsSimd(wasm::FunctionSig * sig)102 bool ContainsSimd(wasm::FunctionSig* sig) {
103   for (auto type : sig->all()) {
104     if (type == wasm::kWasmS128) return true;
105   }
106   return false;
107 }
108 
ContainsInt64(wasm::FunctionSig * sig)109 bool ContainsInt64(wasm::FunctionSig* sig) {
110   for (auto type : sig->all()) {
111     if (type == wasm::kWasmI64) return true;
112   }
113   return false;
114 }
115 }  // namespace
116 
WasmGraphBuilder(wasm::ModuleEnv * env,Zone * zone,MachineGraph * mcgraph,wasm::FunctionSig * sig,compiler::SourcePositionTable * source_position_table)117 WasmGraphBuilder::WasmGraphBuilder(
118     wasm::ModuleEnv* env, Zone* zone, MachineGraph* mcgraph,
119     wasm::FunctionSig* sig,
120     compiler::SourcePositionTable* source_position_table)
121     : zone_(zone),
122       mcgraph_(mcgraph),
123       env_(env),
124       cur_buffer_(def_buffer_),
125       cur_bufsize_(kDefaultBufferSize),
126       has_simd_(ContainsSimd(sig)),
127       untrusted_code_mitigations_(FLAG_untrusted_code_mitigations),
128       sig_(sig),
129       source_position_table_(source_position_table) {
130   DCHECK_IMPLIES(use_trap_handler(), trap_handler::IsTrapHandlerEnabled());
131   DCHECK_NOT_NULL(mcgraph_);
132 }
133 
Error()134 Node* WasmGraphBuilder::Error() { return mcgraph()->Dead(); }
135 
Start(unsigned params)136 Node* WasmGraphBuilder::Start(unsigned params) {
137   Node* start = graph()->NewNode(mcgraph()->common()->Start(params));
138   graph()->SetStart(start);
139   return start;
140 }
141 
Param(unsigned index)142 Node* WasmGraphBuilder::Param(unsigned index) {
143   return graph()->NewNode(mcgraph()->common()->Parameter(index),
144                           graph()->start());
145 }
146 
Loop(Node * entry)147 Node* WasmGraphBuilder::Loop(Node* entry) {
148   return graph()->NewNode(mcgraph()->common()->Loop(1), entry);
149 }
150 
Terminate(Node * effect,Node * control)151 Node* WasmGraphBuilder::Terminate(Node* effect, Node* control) {
152   Node* terminate =
153       graph()->NewNode(mcgraph()->common()->Terminate(), effect, control);
154   MergeControlToEnd(mcgraph(), terminate);
155   return terminate;
156 }
157 
IsPhiWithMerge(Node * phi,Node * merge)158 bool WasmGraphBuilder::IsPhiWithMerge(Node* phi, Node* merge) {
159   return phi && IrOpcode::IsPhiOpcode(phi->opcode()) &&
160          NodeProperties::GetControlInput(phi) == merge;
161 }
162 
ThrowsException(Node * node,Node ** if_success,Node ** if_exception)163 bool WasmGraphBuilder::ThrowsException(Node* node, Node** if_success,
164                                        Node** if_exception) {
165   if (node->op()->HasProperty(compiler::Operator::kNoThrow)) {
166     return false;
167   }
168 
169   *if_success = graph()->NewNode(mcgraph()->common()->IfSuccess(), node);
170   *if_exception =
171       graph()->NewNode(mcgraph()->common()->IfException(), node, node);
172 
173   return true;
174 }
175 
AppendToMerge(Node * merge,Node * from)176 void WasmGraphBuilder::AppendToMerge(Node* merge, Node* from) {
177   DCHECK(IrOpcode::IsMergeOpcode(merge->opcode()));
178   merge->AppendInput(mcgraph()->zone(), from);
179   int new_size = merge->InputCount();
180   NodeProperties::ChangeOp(
181       merge, mcgraph()->common()->ResizeMergeOrPhi(merge->op(), new_size));
182 }
183 
AppendToPhi(Node * phi,Node * from)184 void WasmGraphBuilder::AppendToPhi(Node* phi, Node* from) {
185   DCHECK(IrOpcode::IsPhiOpcode(phi->opcode()));
186   int new_size = phi->InputCount();
187   phi->InsertInput(mcgraph()->zone(), phi->InputCount() - 1, from);
188   NodeProperties::ChangeOp(
189       phi, mcgraph()->common()->ResizeMergeOrPhi(phi->op(), new_size));
190 }
191 
Merge(unsigned count,Node ** controls)192 Node* WasmGraphBuilder::Merge(unsigned count, Node** controls) {
193   return graph()->NewNode(mcgraph()->common()->Merge(count), count, controls);
194 }
195 
Phi(wasm::ValueType type,unsigned count,Node ** vals,Node * control)196 Node* WasmGraphBuilder::Phi(wasm::ValueType type, unsigned count, Node** vals,
197                             Node* control) {
198   DCHECK(IrOpcode::IsMergeOpcode(control->opcode()));
199   Node** buf = Realloc(vals, count, count + 1);
200   buf[count] = control;
201   return graph()->NewNode(
202       mcgraph()->common()->Phi(wasm::ValueTypes::MachineRepresentationFor(type),
203                                count),
204       count + 1, buf);
205 }
206 
EffectPhi(unsigned count,Node ** effects,Node * control)207 Node* WasmGraphBuilder::EffectPhi(unsigned count, Node** effects,
208                                   Node* control) {
209   DCHECK(IrOpcode::IsMergeOpcode(control->opcode()));
210   Node** buf = Realloc(effects, count, count + 1);
211   buf[count] = control;
212   return graph()->NewNode(mcgraph()->common()->EffectPhi(count), count + 1,
213                           buf);
214 }
215 
RefNull()216 Node* WasmGraphBuilder::RefNull() {
217   return LOAD_INSTANCE_FIELD(NullValue, MachineType::TaggedPointer());
218 }
219 
NoContextConstant()220 Node* WasmGraphBuilder::NoContextConstant() {
221   // TODO(titzer): avoiding a dependency on JSGraph here. Refactor.
222   return mcgraph()->IntPtrConstant(0);
223 }
224 
Uint32Constant(uint32_t value)225 Node* WasmGraphBuilder::Uint32Constant(uint32_t value) {
226   return mcgraph()->Uint32Constant(value);
227 }
228 
Int32Constant(int32_t value)229 Node* WasmGraphBuilder::Int32Constant(int32_t value) {
230   return mcgraph()->Int32Constant(value);
231 }
232 
Int64Constant(int64_t value)233 Node* WasmGraphBuilder::Int64Constant(int64_t value) {
234   return mcgraph()->Int64Constant(value);
235 }
236 
IntPtrConstant(intptr_t value)237 Node* WasmGraphBuilder::IntPtrConstant(intptr_t value) {
238   return mcgraph()->IntPtrConstant(value);
239 }
240 
StackCheck(wasm::WasmCodePosition position,Node ** effect,Node ** control)241 void WasmGraphBuilder::StackCheck(wasm::WasmCodePosition position,
242                                   Node** effect, Node** control) {
243   DCHECK_NOT_NULL(env_);  // Wrappers don't get stack checks.
244   if (FLAG_wasm_no_stack_checks || !env_->runtime_exception_support) {
245     return;
246   }
247   if (effect == nullptr) effect = effect_;
248   if (control == nullptr) control = control_;
249 
250   // This instruction sequence is matched in the instruction selector to
251   // load the stack pointer directly on some platforms. Hence, when modifying
252   // please also fix WasmStackCheckMatcher in node-matchers.h
253 
254   Node* limit_address = graph()->NewNode(
255       mcgraph()->machine()->Load(MachineType::Pointer()), instance_node_.get(),
256       mcgraph()->Int32Constant(WASM_INSTANCE_OBJECT_OFFSET(StackLimitAddress)),
257       *effect, *control);
258   Node* limit = graph()->NewNode(
259       mcgraph()->machine()->Load(MachineType::Pointer()), limit_address,
260       mcgraph()->IntPtrConstant(0), limit_address, *control);
261   *effect = limit;
262   Node* pointer = graph()->NewNode(mcgraph()->machine()->LoadStackPointer());
263 
264   Node* check =
265       graph()->NewNode(mcgraph()->machine()->UintLessThan(), limit, pointer);
266 
267   Diamond stack_check(graph(), mcgraph()->common(), check, BranchHint::kTrue);
268   stack_check.Chain(*control);
269 
270   if (stack_check_call_operator_ == nullptr) {
271     // Build and cache the stack check call operator and the constant
272     // representing the stack check code.
273     auto call_descriptor = Linkage::GetStubCallDescriptor(
274         mcgraph()->zone(),                    // zone
275         NoContextDescriptor{},                // descriptor
276         0,                                    // stack parameter count
277         CallDescriptor::kNoFlags,             // flags
278         Operator::kNoProperties,              // properties
279         StubCallMode::kCallWasmRuntimeStub);  // stub call mode
280     // A direct call to a wasm runtime stub defined in this module.
281     // Just encode the stub index. This will be patched at relocation.
282     stack_check_code_node_.set(mcgraph()->RelocatableIntPtrConstant(
283         wasm::WasmCode::kWasmStackGuard, RelocInfo::WASM_STUB_CALL));
284     stack_check_call_operator_ = mcgraph()->common()->Call(call_descriptor);
285   }
286 
287   Node* call = graph()->NewNode(stack_check_call_operator_.get(),
288                                 stack_check_code_node_.get(), *effect,
289                                 stack_check.if_false);
290 
291   SetSourcePosition(call, position);
292 
293   Node* ephi = stack_check.EffectPhi(*effect, call);
294 
295   *control = stack_check.merge;
296   *effect = ephi;
297 }
298 
PatchInStackCheckIfNeeded()299 void WasmGraphBuilder::PatchInStackCheckIfNeeded() {
300   if (!needs_stack_check_) return;
301 
302   Node* start = graph()->start();
303   // Place a stack check which uses a dummy node as control and effect.
304   Node* dummy = graph()->NewNode(mcgraph()->common()->Dead());
305   Node* control = dummy;
306   Node* effect = dummy;
307   // The function-prologue stack check is associated with position 0, which
308   // is never a position of any instruction in the function.
309   StackCheck(0, &effect, &control);
310 
311   // In testing, no steck checks were emitted. Nothing to rewire then.
312   if (effect == dummy) return;
313 
314   // Now patch all control uses of {start} to use {control} and all effect uses
315   // to use {effect} instead. Then rewire the dummy node to use start instead.
316   NodeProperties::ReplaceUses(start, start, effect, control);
317   NodeProperties::ReplaceUses(dummy, nullptr, start, start);
318 }
319 
Binop(wasm::WasmOpcode opcode,Node * left,Node * right,wasm::WasmCodePosition position)320 Node* WasmGraphBuilder::Binop(wasm::WasmOpcode opcode, Node* left, Node* right,
321                               wasm::WasmCodePosition position) {
322   const Operator* op;
323   MachineOperatorBuilder* m = mcgraph()->machine();
324   switch (opcode) {
325     case wasm::kExprI32Add:
326       op = m->Int32Add();
327       break;
328     case wasm::kExprI32Sub:
329       op = m->Int32Sub();
330       break;
331     case wasm::kExprI32Mul:
332       op = m->Int32Mul();
333       break;
334     case wasm::kExprI32DivS:
335       return BuildI32DivS(left, right, position);
336     case wasm::kExprI32DivU:
337       return BuildI32DivU(left, right, position);
338     case wasm::kExprI32RemS:
339       return BuildI32RemS(left, right, position);
340     case wasm::kExprI32RemU:
341       return BuildI32RemU(left, right, position);
342     case wasm::kExprI32And:
343       op = m->Word32And();
344       break;
345     case wasm::kExprI32Ior:
346       op = m->Word32Or();
347       break;
348     case wasm::kExprI32Xor:
349       op = m->Word32Xor();
350       break;
351     case wasm::kExprI32Shl:
352       op = m->Word32Shl();
353       right = MaskShiftCount32(right);
354       break;
355     case wasm::kExprI32ShrU:
356       op = m->Word32Shr();
357       right = MaskShiftCount32(right);
358       break;
359     case wasm::kExprI32ShrS:
360       op = m->Word32Sar();
361       right = MaskShiftCount32(right);
362       break;
363     case wasm::kExprI32Ror:
364       op = m->Word32Ror();
365       right = MaskShiftCount32(right);
366       break;
367     case wasm::kExprI32Rol:
368       right = MaskShiftCount32(right);
369       return BuildI32Rol(left, right);
370     case wasm::kExprI32Eq:
371       op = m->Word32Equal();
372       break;
373     case wasm::kExprI32Ne:
374       return Invert(Binop(wasm::kExprI32Eq, left, right));
375     case wasm::kExprI32LtS:
376       op = m->Int32LessThan();
377       break;
378     case wasm::kExprI32LeS:
379       op = m->Int32LessThanOrEqual();
380       break;
381     case wasm::kExprI32LtU:
382       op = m->Uint32LessThan();
383       break;
384     case wasm::kExprI32LeU:
385       op = m->Uint32LessThanOrEqual();
386       break;
387     case wasm::kExprI32GtS:
388       op = m->Int32LessThan();
389       std::swap(left, right);
390       break;
391     case wasm::kExprI32GeS:
392       op = m->Int32LessThanOrEqual();
393       std::swap(left, right);
394       break;
395     case wasm::kExprI32GtU:
396       op = m->Uint32LessThan();
397       std::swap(left, right);
398       break;
399     case wasm::kExprI32GeU:
400       op = m->Uint32LessThanOrEqual();
401       std::swap(left, right);
402       break;
403     case wasm::kExprI64And:
404       op = m->Word64And();
405       break;
406     case wasm::kExprI64Add:
407       op = m->Int64Add();
408       break;
409     case wasm::kExprI64Sub:
410       op = m->Int64Sub();
411       break;
412     case wasm::kExprI64Mul:
413       op = m->Int64Mul();
414       break;
415     case wasm::kExprI64DivS:
416       return BuildI64DivS(left, right, position);
417     case wasm::kExprI64DivU:
418       return BuildI64DivU(left, right, position);
419     case wasm::kExprI64RemS:
420       return BuildI64RemS(left, right, position);
421     case wasm::kExprI64RemU:
422       return BuildI64RemU(left, right, position);
423     case wasm::kExprI64Ior:
424       op = m->Word64Or();
425       break;
426     case wasm::kExprI64Xor:
427       op = m->Word64Xor();
428       break;
429     case wasm::kExprI64Shl:
430       op = m->Word64Shl();
431       right = MaskShiftCount64(right);
432       break;
433     case wasm::kExprI64ShrU:
434       op = m->Word64Shr();
435       right = MaskShiftCount64(right);
436       break;
437     case wasm::kExprI64ShrS:
438       op = m->Word64Sar();
439       right = MaskShiftCount64(right);
440       break;
441     case wasm::kExprI64Eq:
442       op = m->Word64Equal();
443       break;
444     case wasm::kExprI64Ne:
445       return Invert(Binop(wasm::kExprI64Eq, left, right));
446     case wasm::kExprI64LtS:
447       op = m->Int64LessThan();
448       break;
449     case wasm::kExprI64LeS:
450       op = m->Int64LessThanOrEqual();
451       break;
452     case wasm::kExprI64LtU:
453       op = m->Uint64LessThan();
454       break;
455     case wasm::kExprI64LeU:
456       op = m->Uint64LessThanOrEqual();
457       break;
458     case wasm::kExprI64GtS:
459       op = m->Int64LessThan();
460       std::swap(left, right);
461       break;
462     case wasm::kExprI64GeS:
463       op = m->Int64LessThanOrEqual();
464       std::swap(left, right);
465       break;
466     case wasm::kExprI64GtU:
467       op = m->Uint64LessThan();
468       std::swap(left, right);
469       break;
470     case wasm::kExprI64GeU:
471       op = m->Uint64LessThanOrEqual();
472       std::swap(left, right);
473       break;
474     case wasm::kExprI64Ror:
475       op = m->Word64Ror();
476       right = MaskShiftCount64(right);
477       break;
478     case wasm::kExprI64Rol:
479       return BuildI64Rol(left, right);
480     case wasm::kExprF32CopySign:
481       return BuildF32CopySign(left, right);
482     case wasm::kExprF64CopySign:
483       return BuildF64CopySign(left, right);
484     case wasm::kExprF32Add:
485       op = m->Float32Add();
486       break;
487     case wasm::kExprF32Sub:
488       op = m->Float32Sub();
489       break;
490     case wasm::kExprF32Mul:
491       op = m->Float32Mul();
492       break;
493     case wasm::kExprF32Div:
494       op = m->Float32Div();
495       break;
496     case wasm::kExprF32Eq:
497       op = m->Float32Equal();
498       break;
499     case wasm::kExprF32Ne:
500       return Invert(Binop(wasm::kExprF32Eq, left, right));
501     case wasm::kExprF32Lt:
502       op = m->Float32LessThan();
503       break;
504     case wasm::kExprF32Ge:
505       op = m->Float32LessThanOrEqual();
506       std::swap(left, right);
507       break;
508     case wasm::kExprF32Gt:
509       op = m->Float32LessThan();
510       std::swap(left, right);
511       break;
512     case wasm::kExprF32Le:
513       op = m->Float32LessThanOrEqual();
514       break;
515     case wasm::kExprF64Add:
516       op = m->Float64Add();
517       break;
518     case wasm::kExprF64Sub:
519       op = m->Float64Sub();
520       break;
521     case wasm::kExprF64Mul:
522       op = m->Float64Mul();
523       break;
524     case wasm::kExprF64Div:
525       op = m->Float64Div();
526       break;
527     case wasm::kExprF64Eq:
528       op = m->Float64Equal();
529       break;
530     case wasm::kExprF64Ne:
531       return Invert(Binop(wasm::kExprF64Eq, left, right));
532     case wasm::kExprF64Lt:
533       op = m->Float64LessThan();
534       break;
535     case wasm::kExprF64Le:
536       op = m->Float64LessThanOrEqual();
537       break;
538     case wasm::kExprF64Gt:
539       op = m->Float64LessThan();
540       std::swap(left, right);
541       break;
542     case wasm::kExprF64Ge:
543       op = m->Float64LessThanOrEqual();
544       std::swap(left, right);
545       break;
546     case wasm::kExprF32Min:
547       op = m->Float32Min();
548       break;
549     case wasm::kExprF64Min:
550       op = m->Float64Min();
551       break;
552     case wasm::kExprF32Max:
553       op = m->Float32Max();
554       break;
555     case wasm::kExprF64Max:
556       op = m->Float64Max();
557       break;
558     case wasm::kExprF64Pow:
559       return BuildF64Pow(left, right);
560     case wasm::kExprF64Atan2:
561       op = m->Float64Atan2();
562       break;
563     case wasm::kExprF64Mod:
564       return BuildF64Mod(left, right);
565     case wasm::kExprI32AsmjsDivS:
566       return BuildI32AsmjsDivS(left, right);
567     case wasm::kExprI32AsmjsDivU:
568       return BuildI32AsmjsDivU(left, right);
569     case wasm::kExprI32AsmjsRemS:
570       return BuildI32AsmjsRemS(left, right);
571     case wasm::kExprI32AsmjsRemU:
572       return BuildI32AsmjsRemU(left, right);
573     case wasm::kExprI32AsmjsStoreMem8:
574       return BuildAsmjsStoreMem(MachineType::Int8(), left, right);
575     case wasm::kExprI32AsmjsStoreMem16:
576       return BuildAsmjsStoreMem(MachineType::Int16(), left, right);
577     case wasm::kExprI32AsmjsStoreMem:
578       return BuildAsmjsStoreMem(MachineType::Int32(), left, right);
579     case wasm::kExprF32AsmjsStoreMem:
580       return BuildAsmjsStoreMem(MachineType::Float32(), left, right);
581     case wasm::kExprF64AsmjsStoreMem:
582       return BuildAsmjsStoreMem(MachineType::Float64(), left, right);
583     default:
584       FATAL_UNSUPPORTED_OPCODE(opcode);
585   }
586   return graph()->NewNode(op, left, right);
587 }
588 
Unop(wasm::WasmOpcode opcode,Node * input,wasm::WasmCodePosition position)589 Node* WasmGraphBuilder::Unop(wasm::WasmOpcode opcode, Node* input,
590                              wasm::WasmCodePosition position) {
591   const Operator* op;
592   MachineOperatorBuilder* m = mcgraph()->machine();
593   switch (opcode) {
594     case wasm::kExprI32Eqz:
595       op = m->Word32Equal();
596       return graph()->NewNode(op, input, mcgraph()->Int32Constant(0));
597     case wasm::kExprF32Abs:
598       op = m->Float32Abs();
599       break;
600     case wasm::kExprF32Neg: {
601       op = m->Float32Neg();
602       break;
603     }
604     case wasm::kExprF32Sqrt:
605       op = m->Float32Sqrt();
606       break;
607     case wasm::kExprF64Abs:
608       op = m->Float64Abs();
609       break;
610     case wasm::kExprF64Neg: {
611       op = m->Float64Neg();
612       break;
613     }
614     case wasm::kExprF64Sqrt:
615       op = m->Float64Sqrt();
616       break;
617     case wasm::kExprI32SConvertF32:
618     case wasm::kExprI32UConvertF32:
619     case wasm::kExprI32SConvertF64:
620     case wasm::kExprI32UConvertF64:
621     case wasm::kExprI32SConvertSatF64:
622     case wasm::kExprI32UConvertSatF64:
623     case wasm::kExprI32SConvertSatF32:
624     case wasm::kExprI32UConvertSatF32:
625       return BuildIntConvertFloat(input, position, opcode);
626     case wasm::kExprI32AsmjsSConvertF64:
627       return BuildI32AsmjsSConvertF64(input);
628     case wasm::kExprI32AsmjsUConvertF64:
629       return BuildI32AsmjsUConvertF64(input);
630     case wasm::kExprF32ConvertF64:
631       op = m->TruncateFloat64ToFloat32();
632       break;
633     case wasm::kExprF64SConvertI32:
634       op = m->ChangeInt32ToFloat64();
635       break;
636     case wasm::kExprF64UConvertI32:
637       op = m->ChangeUint32ToFloat64();
638       break;
639     case wasm::kExprF32SConvertI32:
640       op = m->RoundInt32ToFloat32();
641       break;
642     case wasm::kExprF32UConvertI32:
643       op = m->RoundUint32ToFloat32();
644       break;
645     case wasm::kExprI32AsmjsSConvertF32:
646       return BuildI32AsmjsSConvertF32(input);
647     case wasm::kExprI32AsmjsUConvertF32:
648       return BuildI32AsmjsUConvertF32(input);
649     case wasm::kExprF64ConvertF32:
650       op = m->ChangeFloat32ToFloat64();
651       break;
652     case wasm::kExprF32ReinterpretI32:
653       op = m->BitcastInt32ToFloat32();
654       break;
655     case wasm::kExprI32ReinterpretF32:
656       op = m->BitcastFloat32ToInt32();
657       break;
658     case wasm::kExprI32Clz:
659       op = m->Word32Clz();
660       break;
661     case wasm::kExprI32Ctz: {
662       if (m->Word32Ctz().IsSupported()) {
663         op = m->Word32Ctz().op();
664         break;
665       } else if (m->Word32ReverseBits().IsSupported()) {
666         Node* reversed = graph()->NewNode(m->Word32ReverseBits().op(), input);
667         Node* result = graph()->NewNode(m->Word32Clz(), reversed);
668         return result;
669       } else {
670         return BuildI32Ctz(input);
671       }
672     }
673     case wasm::kExprI32Popcnt: {
674       if (m->Word32Popcnt().IsSupported()) {
675         op = m->Word32Popcnt().op();
676         break;
677       } else {
678         return BuildI32Popcnt(input);
679       }
680     }
681     case wasm::kExprF32Floor: {
682       if (!m->Float32RoundDown().IsSupported()) return BuildF32Floor(input);
683       op = m->Float32RoundDown().op();
684       break;
685     }
686     case wasm::kExprF32Ceil: {
687       if (!m->Float32RoundUp().IsSupported()) return BuildF32Ceil(input);
688       op = m->Float32RoundUp().op();
689       break;
690     }
691     case wasm::kExprF32Trunc: {
692       if (!m->Float32RoundTruncate().IsSupported()) return BuildF32Trunc(input);
693       op = m->Float32RoundTruncate().op();
694       break;
695     }
696     case wasm::kExprF32NearestInt: {
697       if (!m->Float32RoundTiesEven().IsSupported())
698         return BuildF32NearestInt(input);
699       op = m->Float32RoundTiesEven().op();
700       break;
701     }
702     case wasm::kExprF64Floor: {
703       if (!m->Float64RoundDown().IsSupported()) return BuildF64Floor(input);
704       op = m->Float64RoundDown().op();
705       break;
706     }
707     case wasm::kExprF64Ceil: {
708       if (!m->Float64RoundUp().IsSupported()) return BuildF64Ceil(input);
709       op = m->Float64RoundUp().op();
710       break;
711     }
712     case wasm::kExprF64Trunc: {
713       if (!m->Float64RoundTruncate().IsSupported()) return BuildF64Trunc(input);
714       op = m->Float64RoundTruncate().op();
715       break;
716     }
717     case wasm::kExprF64NearestInt: {
718       if (!m->Float64RoundTiesEven().IsSupported())
719         return BuildF64NearestInt(input);
720       op = m->Float64RoundTiesEven().op();
721       break;
722     }
723     case wasm::kExprF64Acos: {
724       return BuildF64Acos(input);
725     }
726     case wasm::kExprF64Asin: {
727       return BuildF64Asin(input);
728     }
729     case wasm::kExprF64Atan:
730       op = m->Float64Atan();
731       break;
732     case wasm::kExprF64Cos: {
733       op = m->Float64Cos();
734       break;
735     }
736     case wasm::kExprF64Sin: {
737       op = m->Float64Sin();
738       break;
739     }
740     case wasm::kExprF64Tan: {
741       op = m->Float64Tan();
742       break;
743     }
744     case wasm::kExprF64Exp: {
745       op = m->Float64Exp();
746       break;
747     }
748     case wasm::kExprF64Log:
749       op = m->Float64Log();
750       break;
751     case wasm::kExprI32ConvertI64:
752       op = m->TruncateInt64ToInt32();
753       break;
754     case wasm::kExprI64SConvertI32:
755       op = m->ChangeInt32ToInt64();
756       break;
757     case wasm::kExprI64UConvertI32:
758       op = m->ChangeUint32ToUint64();
759       break;
760     case wasm::kExprF64ReinterpretI64:
761       op = m->BitcastInt64ToFloat64();
762       break;
763     case wasm::kExprI64ReinterpretF64:
764       op = m->BitcastFloat64ToInt64();
765       break;
766     case wasm::kExprI64Clz:
767       op = m->Word64Clz();
768       break;
769     case wasm::kExprI64Ctz: {
770       OptionalOperator ctz64 = m->Word64Ctz();
771       if (ctz64.IsSupported()) {
772         op = ctz64.op();
773         break;
774       } else if (m->Is32() && m->Word32Ctz().IsSupported()) {
775         op = ctz64.placeholder();
776         break;
777       } else if (m->Word64ReverseBits().IsSupported()) {
778         Node* reversed = graph()->NewNode(m->Word64ReverseBits().op(), input);
779         Node* result = graph()->NewNode(m->Word64Clz(), reversed);
780         return result;
781       } else {
782         return BuildI64Ctz(input);
783       }
784     }
785     case wasm::kExprI64Popcnt: {
786       OptionalOperator popcnt64 = m->Word64Popcnt();
787       if (popcnt64.IsSupported()) {
788         op = popcnt64.op();
789       } else if (m->Is32() && m->Word32Popcnt().IsSupported()) {
790         op = popcnt64.placeholder();
791       } else {
792         return BuildI64Popcnt(input);
793       }
794       break;
795     }
796     case wasm::kExprI64Eqz:
797       op = m->Word64Equal();
798       return graph()->NewNode(op, input, mcgraph()->Int64Constant(0));
799     case wasm::kExprF32SConvertI64:
800       if (m->Is32()) {
801         return BuildF32SConvertI64(input);
802       }
803       op = m->RoundInt64ToFloat32();
804       break;
805     case wasm::kExprF32UConvertI64:
806       if (m->Is32()) {
807         return BuildF32UConvertI64(input);
808       }
809       op = m->RoundUint64ToFloat32();
810       break;
811     case wasm::kExprF64SConvertI64:
812       if (m->Is32()) {
813         return BuildF64SConvertI64(input);
814       }
815       op = m->RoundInt64ToFloat64();
816       break;
817     case wasm::kExprF64UConvertI64:
818       if (m->Is32()) {
819         return BuildF64UConvertI64(input);
820       }
821       op = m->RoundUint64ToFloat64();
822       break;
823     case wasm::kExprI32SExtendI8:
824       op = m->SignExtendWord8ToInt32();
825       break;
826     case wasm::kExprI32SExtendI16:
827       op = m->SignExtendWord16ToInt32();
828       break;
829     case wasm::kExprI64SExtendI8:
830       op = m->SignExtendWord8ToInt64();
831       break;
832     case wasm::kExprI64SExtendI16:
833       op = m->SignExtendWord16ToInt64();
834       break;
835     case wasm::kExprI64SExtendI32:
836       op = m->SignExtendWord32ToInt64();
837       break;
838     case wasm::kExprI64SConvertF32:
839     case wasm::kExprI64UConvertF32:
840     case wasm::kExprI64SConvertF64:
841     case wasm::kExprI64UConvertF64:
842     case wasm::kExprI64SConvertSatF32:
843     case wasm::kExprI64UConvertSatF32:
844     case wasm::kExprI64SConvertSatF64:
845     case wasm::kExprI64UConvertSatF64:
846       return mcgraph()->machine()->Is32()
847                  ? BuildCcallConvertFloat(input, position, opcode)
848                  : BuildIntConvertFloat(input, position, opcode);
849     case wasm::kExprRefIsNull:
850       return graph()->NewNode(m->WordEqual(), input, RefNull());
851     case wasm::kExprI32AsmjsLoadMem8S:
852       return BuildAsmjsLoadMem(MachineType::Int8(), input);
853     case wasm::kExprI32AsmjsLoadMem8U:
854       return BuildAsmjsLoadMem(MachineType::Uint8(), input);
855     case wasm::kExprI32AsmjsLoadMem16S:
856       return BuildAsmjsLoadMem(MachineType::Int16(), input);
857     case wasm::kExprI32AsmjsLoadMem16U:
858       return BuildAsmjsLoadMem(MachineType::Uint16(), input);
859     case wasm::kExprI32AsmjsLoadMem:
860       return BuildAsmjsLoadMem(MachineType::Int32(), input);
861     case wasm::kExprF32AsmjsLoadMem:
862       return BuildAsmjsLoadMem(MachineType::Float32(), input);
863     case wasm::kExprF64AsmjsLoadMem:
864       return BuildAsmjsLoadMem(MachineType::Float64(), input);
865     default:
866       FATAL_UNSUPPORTED_OPCODE(opcode);
867   }
868   return graph()->NewNode(op, input);
869 }
870 
Float32Constant(float value)871 Node* WasmGraphBuilder::Float32Constant(float value) {
872   return mcgraph()->Float32Constant(value);
873 }
874 
Float64Constant(double value)875 Node* WasmGraphBuilder::Float64Constant(double value) {
876   return mcgraph()->Float64Constant(value);
877 }
878 
879 namespace {
Branch(MachineGraph * mcgraph,Node * cond,Node ** true_node,Node ** false_node,Node * control,BranchHint hint)880 Node* Branch(MachineGraph* mcgraph, Node* cond, Node** true_node,
881              Node** false_node, Node* control, BranchHint hint) {
882   DCHECK_NOT_NULL(cond);
883   DCHECK_NOT_NULL(control);
884   Node* branch =
885       mcgraph->graph()->NewNode(mcgraph->common()->Branch(hint), cond, control);
886   *true_node = mcgraph->graph()->NewNode(mcgraph->common()->IfTrue(), branch);
887   *false_node = mcgraph->graph()->NewNode(mcgraph->common()->IfFalse(), branch);
888   return branch;
889 }
890 }  // namespace
891 
BranchNoHint(Node * cond,Node ** true_node,Node ** false_node)892 Node* WasmGraphBuilder::BranchNoHint(Node* cond, Node** true_node,
893                                      Node** false_node) {
894   return Branch(mcgraph(), cond, true_node, false_node, Control(),
895                 BranchHint::kNone);
896 }
897 
BranchExpectTrue(Node * cond,Node ** true_node,Node ** false_node)898 Node* WasmGraphBuilder::BranchExpectTrue(Node* cond, Node** true_node,
899                                          Node** false_node) {
900   return Branch(mcgraph(), cond, true_node, false_node, Control(),
901                 BranchHint::kTrue);
902 }
903 
BranchExpectFalse(Node * cond,Node ** true_node,Node ** false_node)904 Node* WasmGraphBuilder::BranchExpectFalse(Node* cond, Node** true_node,
905                                           Node** false_node) {
906   return Branch(mcgraph(), cond, true_node, false_node, Control(),
907                 BranchHint::kFalse);
908 }
909 
GetTrapIdForTrap(wasm::TrapReason reason)910 TrapId WasmGraphBuilder::GetTrapIdForTrap(wasm::TrapReason reason) {
911   // TODO(wasm): "!env_" should not happen when compiling an actual wasm
912   // function.
913   if (!env_ || !env_->runtime_exception_support) {
914     // We use TrapId::kInvalid as a marker to tell the code generator
915     // to generate a call to a testing c-function instead of a runtime
916     // stub. This code should only be called from a cctest.
917     return TrapId::kInvalid;
918   }
919 
920   switch (reason) {
921 #define TRAPREASON_TO_TRAPID(name)                                             \
922   case wasm::k##name:                                                          \
923     static_assert(                                                             \
924         static_cast<int>(TrapId::k##name) == wasm::WasmCode::kThrowWasm##name, \
925         "trap id mismatch");                                                   \
926     return TrapId::k##name;
927     FOREACH_WASM_TRAPREASON(TRAPREASON_TO_TRAPID)
928 #undef TRAPREASON_TO_TRAPID
929     default:
930       UNREACHABLE();
931   }
932 }
933 
TrapIfTrue(wasm::TrapReason reason,Node * cond,wasm::WasmCodePosition position)934 Node* WasmGraphBuilder::TrapIfTrue(wasm::TrapReason reason, Node* cond,
935                                    wasm::WasmCodePosition position) {
936   TrapId trap_id = GetTrapIdForTrap(reason);
937   Node* node = SetControl(graph()->NewNode(mcgraph()->common()->TrapIf(trap_id),
938                                            cond, Effect(), Control()));
939   SetSourcePosition(node, position);
940   return node;
941 }
942 
TrapIfFalse(wasm::TrapReason reason,Node * cond,wasm::WasmCodePosition position)943 Node* WasmGraphBuilder::TrapIfFalse(wasm::TrapReason reason, Node* cond,
944                                     wasm::WasmCodePosition position) {
945   TrapId trap_id = GetTrapIdForTrap(reason);
946   Node* node = SetControl(graph()->NewNode(
947       mcgraph()->common()->TrapUnless(trap_id), cond, Effect(), Control()));
948   SetSourcePosition(node, position);
949   return node;
950 }
951 
952 // Add a check that traps if {node} is equal to {val}.
TrapIfEq32(wasm::TrapReason reason,Node * node,int32_t val,wasm::WasmCodePosition position)953 Node* WasmGraphBuilder::TrapIfEq32(wasm::TrapReason reason, Node* node,
954                                    int32_t val,
955                                    wasm::WasmCodePosition position) {
956   Int32Matcher m(node);
957   if (m.HasValue() && !m.Is(val)) return graph()->start();
958   if (val == 0) {
959     return TrapIfFalse(reason, node, position);
960   } else {
961     return TrapIfTrue(reason,
962                       graph()->NewNode(mcgraph()->machine()->Word32Equal(),
963                                        node, mcgraph()->Int32Constant(val)),
964                       position);
965   }
966 }
967 
968 // Add a check that traps if {node} is zero.
ZeroCheck32(wasm::TrapReason reason,Node * node,wasm::WasmCodePosition position)969 Node* WasmGraphBuilder::ZeroCheck32(wasm::TrapReason reason, Node* node,
970                                     wasm::WasmCodePosition position) {
971   return TrapIfEq32(reason, node, 0, position);
972 }
973 
974 // Add a check that traps if {node} is equal to {val}.
TrapIfEq64(wasm::TrapReason reason,Node * node,int64_t val,wasm::WasmCodePosition position)975 Node* WasmGraphBuilder::TrapIfEq64(wasm::TrapReason reason, Node* node,
976                                    int64_t val,
977                                    wasm::WasmCodePosition position) {
978   Int64Matcher m(node);
979   if (m.HasValue() && !m.Is(val)) return graph()->start();
980   return TrapIfTrue(reason,
981                     graph()->NewNode(mcgraph()->machine()->Word64Equal(), node,
982                                      mcgraph()->Int64Constant(val)),
983                     position);
984 }
985 
986 // Add a check that traps if {node} is zero.
ZeroCheck64(wasm::TrapReason reason,Node * node,wasm::WasmCodePosition position)987 Node* WasmGraphBuilder::ZeroCheck64(wasm::TrapReason reason, Node* node,
988                                     wasm::WasmCodePosition position) {
989   return TrapIfEq64(reason, node, 0, position);
990 }
991 
Switch(unsigned count,Node * key)992 Node* WasmGraphBuilder::Switch(unsigned count, Node* key) {
993   return graph()->NewNode(mcgraph()->common()->Switch(count), key, Control());
994 }
995 
IfValue(int32_t value,Node * sw)996 Node* WasmGraphBuilder::IfValue(int32_t value, Node* sw) {
997   DCHECK_EQ(IrOpcode::kSwitch, sw->opcode());
998   return graph()->NewNode(mcgraph()->common()->IfValue(value), sw);
999 }
1000 
IfDefault(Node * sw)1001 Node* WasmGraphBuilder::IfDefault(Node* sw) {
1002   DCHECK_EQ(IrOpcode::kSwitch, sw->opcode());
1003   return graph()->NewNode(mcgraph()->common()->IfDefault(), sw);
1004 }
1005 
Return(unsigned count,Node ** vals)1006 Node* WasmGraphBuilder::Return(unsigned count, Node** vals) {
1007   static const int kStackAllocatedNodeBufferSize = 8;
1008   Node* stack_buffer[kStackAllocatedNodeBufferSize];
1009   std::vector<Node*> heap_buffer;
1010 
1011   Node** buf = stack_buffer;
1012   if (count + 3 > kStackAllocatedNodeBufferSize) {
1013     heap_buffer.resize(count + 3);
1014     buf = heap_buffer.data();
1015   }
1016 
1017   buf[0] = mcgraph()->Int32Constant(0);
1018   memcpy(buf + 1, vals, sizeof(void*) * count);
1019   buf[count + 1] = Effect();
1020   buf[count + 2] = Control();
1021   Node* ret =
1022       graph()->NewNode(mcgraph()->common()->Return(count), count + 3, buf);
1023 
1024   MergeControlToEnd(mcgraph(), ret);
1025   return ret;
1026 }
1027 
ReturnVoid()1028 Node* WasmGraphBuilder::ReturnVoid() { return Return(0, nullptr); }
1029 
Unreachable(wasm::WasmCodePosition position)1030 Node* WasmGraphBuilder::Unreachable(wasm::WasmCodePosition position) {
1031   TrapIfFalse(wasm::TrapReason::kTrapUnreachable, Int32Constant(0), position);
1032   ReturnVoid();
1033   return nullptr;
1034 }
1035 
MaskShiftCount32(Node * node)1036 Node* WasmGraphBuilder::MaskShiftCount32(Node* node) {
1037   static const int32_t kMask32 = 0x1F;
1038   if (!mcgraph()->machine()->Word32ShiftIsSafe()) {
1039     // Shifts by constants are so common we pattern-match them here.
1040     Int32Matcher match(node);
1041     if (match.HasValue()) {
1042       int32_t masked = (match.Value() & kMask32);
1043       if (match.Value() != masked) node = mcgraph()->Int32Constant(masked);
1044     } else {
1045       node = graph()->NewNode(mcgraph()->machine()->Word32And(), node,
1046                               mcgraph()->Int32Constant(kMask32));
1047     }
1048   }
1049   return node;
1050 }
1051 
MaskShiftCount64(Node * node)1052 Node* WasmGraphBuilder::MaskShiftCount64(Node* node) {
1053   static const int64_t kMask64 = 0x3F;
1054   if (!mcgraph()->machine()->Word32ShiftIsSafe()) {
1055     // Shifts by constants are so common we pattern-match them here.
1056     Int64Matcher match(node);
1057     if (match.HasValue()) {
1058       int64_t masked = (match.Value() & kMask64);
1059       if (match.Value() != masked) node = mcgraph()->Int64Constant(masked);
1060     } else {
1061       node = graph()->NewNode(mcgraph()->machine()->Word64And(), node,
1062                               mcgraph()->Int64Constant(kMask64));
1063     }
1064   }
1065   return node;
1066 }
1067 
ReverseBytesSupported(MachineOperatorBuilder * m,size_t size_in_bytes)1068 static bool ReverseBytesSupported(MachineOperatorBuilder* m,
1069                                   size_t size_in_bytes) {
1070   switch (size_in_bytes) {
1071     case 4:
1072     case 16:
1073       return true;
1074     case 8:
1075       return m->Is64();
1076     default:
1077       break;
1078   }
1079   return false;
1080 }
1081 
BuildChangeEndiannessStore(Node * node,MachineRepresentation mem_rep,wasm::ValueType wasmtype)1082 Node* WasmGraphBuilder::BuildChangeEndiannessStore(
1083     Node* node, MachineRepresentation mem_rep, wasm::ValueType wasmtype) {
1084   Node* result;
1085   Node* value = node;
1086   MachineOperatorBuilder* m = mcgraph()->machine();
1087   int valueSizeInBytes = wasm::ValueTypes::ElementSizeInBytes(wasmtype);
1088   int valueSizeInBits = 8 * valueSizeInBytes;
1089   bool isFloat = false;
1090 
1091   switch (wasmtype) {
1092     case wasm::kWasmF64:
1093       value = graph()->NewNode(m->BitcastFloat64ToInt64(), node);
1094       isFloat = true;
1095       V8_FALLTHROUGH;
1096     case wasm::kWasmI64:
1097       result = mcgraph()->Int64Constant(0);
1098       break;
1099     case wasm::kWasmF32:
1100       value = graph()->NewNode(m->BitcastFloat32ToInt32(), node);
1101       isFloat = true;
1102       V8_FALLTHROUGH;
1103     case wasm::kWasmI32:
1104       result = mcgraph()->Int32Constant(0);
1105       break;
1106     case wasm::kWasmS128:
1107       DCHECK(ReverseBytesSupported(m, valueSizeInBytes));
1108       break;
1109     default:
1110       UNREACHABLE();
1111       break;
1112   }
1113 
1114   if (mem_rep == MachineRepresentation::kWord8) {
1115     // No need to change endianness for byte size, return original node
1116     return node;
1117   }
1118   if (wasmtype == wasm::kWasmI64 && mem_rep < MachineRepresentation::kWord64) {
1119     // In case we store lower part of WasmI64 expression, we can truncate
1120     // upper 32bits
1121     value = graph()->NewNode(m->TruncateInt64ToInt32(), value);
1122     valueSizeInBytes = wasm::ValueTypes::ElementSizeInBytes(wasm::kWasmI32);
1123     valueSizeInBits = 8 * valueSizeInBytes;
1124     if (mem_rep == MachineRepresentation::kWord16) {
1125       value =
1126           graph()->NewNode(m->Word32Shl(), value, mcgraph()->Int32Constant(16));
1127     }
1128   } else if (wasmtype == wasm::kWasmI32 &&
1129              mem_rep == MachineRepresentation::kWord16) {
1130     value =
1131         graph()->NewNode(m->Word32Shl(), value, mcgraph()->Int32Constant(16));
1132   }
1133 
1134   int i;
1135   uint32_t shiftCount;
1136 
1137   if (ReverseBytesSupported(m, valueSizeInBytes)) {
1138     switch (valueSizeInBytes) {
1139       case 4:
1140         result = graph()->NewNode(m->Word32ReverseBytes(), value);
1141         break;
1142       case 8:
1143         result = graph()->NewNode(m->Word64ReverseBytes(), value);
1144         break;
1145       case 16: {
1146         Node* byte_reversed_lanes[4];
1147         for (int lane = 0; lane < 4; lane++) {
1148           byte_reversed_lanes[lane] = graph()->NewNode(
1149               m->Word32ReverseBytes(),
1150               graph()->NewNode(mcgraph()->machine()->I32x4ExtractLane(lane),
1151                                value));
1152         }
1153 
1154         // This is making a copy of the value.
1155         result =
1156             graph()->NewNode(mcgraph()->machine()->S128And(), value, value);
1157 
1158         for (int lane = 0; lane < 4; lane++) {
1159           result =
1160               graph()->NewNode(mcgraph()->machine()->I32x4ReplaceLane(3 - lane),
1161                                result, byte_reversed_lanes[lane]);
1162         }
1163 
1164         break;
1165       }
1166       default:
1167         UNREACHABLE();
1168         break;
1169     }
1170   } else {
1171     for (i = 0, shiftCount = valueSizeInBits - 8; i < valueSizeInBits / 2;
1172          i += 8, shiftCount -= 16) {
1173       Node* shiftLower;
1174       Node* shiftHigher;
1175       Node* lowerByte;
1176       Node* higherByte;
1177 
1178       DCHECK_LT(0, shiftCount);
1179       DCHECK_EQ(0, (shiftCount + 8) % 16);
1180 
1181       if (valueSizeInBits > 32) {
1182         shiftLower = graph()->NewNode(m->Word64Shl(), value,
1183                                       mcgraph()->Int64Constant(shiftCount));
1184         shiftHigher = graph()->NewNode(m->Word64Shr(), value,
1185                                        mcgraph()->Int64Constant(shiftCount));
1186         lowerByte = graph()->NewNode(
1187             m->Word64And(), shiftLower,
1188             mcgraph()->Int64Constant(static_cast<uint64_t>(0xFF)
1189                                      << (valueSizeInBits - 8 - i)));
1190         higherByte = graph()->NewNode(
1191             m->Word64And(), shiftHigher,
1192             mcgraph()->Int64Constant(static_cast<uint64_t>(0xFF) << i));
1193         result = graph()->NewNode(m->Word64Or(), result, lowerByte);
1194         result = graph()->NewNode(m->Word64Or(), result, higherByte);
1195       } else {
1196         shiftLower = graph()->NewNode(m->Word32Shl(), value,
1197                                       mcgraph()->Int32Constant(shiftCount));
1198         shiftHigher = graph()->NewNode(m->Word32Shr(), value,
1199                                        mcgraph()->Int32Constant(shiftCount));
1200         lowerByte = graph()->NewNode(
1201             m->Word32And(), shiftLower,
1202             mcgraph()->Int32Constant(static_cast<uint32_t>(0xFF)
1203                                      << (valueSizeInBits - 8 - i)));
1204         higherByte = graph()->NewNode(
1205             m->Word32And(), shiftHigher,
1206             mcgraph()->Int32Constant(static_cast<uint32_t>(0xFF) << i));
1207         result = graph()->NewNode(m->Word32Or(), result, lowerByte);
1208         result = graph()->NewNode(m->Word32Or(), result, higherByte);
1209       }
1210     }
1211   }
1212 
1213   if (isFloat) {
1214     switch (wasmtype) {
1215       case wasm::kWasmF64:
1216         result = graph()->NewNode(m->BitcastInt64ToFloat64(), result);
1217         break;
1218       case wasm::kWasmF32:
1219         result = graph()->NewNode(m->BitcastInt32ToFloat32(), result);
1220         break;
1221       default:
1222         UNREACHABLE();
1223         break;
1224     }
1225   }
1226 
1227   return result;
1228 }
1229 
BuildChangeEndiannessLoad(Node * node,MachineType memtype,wasm::ValueType wasmtype)1230 Node* WasmGraphBuilder::BuildChangeEndiannessLoad(Node* node,
1231                                                   MachineType memtype,
1232                                                   wasm::ValueType wasmtype) {
1233   Node* result;
1234   Node* value = node;
1235   MachineOperatorBuilder* m = mcgraph()->machine();
1236   int valueSizeInBytes = ElementSizeInBytes(memtype.representation());
1237   int valueSizeInBits = 8 * valueSizeInBytes;
1238   bool isFloat = false;
1239 
1240   switch (memtype.representation()) {
1241     case MachineRepresentation::kFloat64:
1242       value = graph()->NewNode(m->BitcastFloat64ToInt64(), node);
1243       isFloat = true;
1244       V8_FALLTHROUGH;
1245     case MachineRepresentation::kWord64:
1246       result = mcgraph()->Int64Constant(0);
1247       break;
1248     case MachineRepresentation::kFloat32:
1249       value = graph()->NewNode(m->BitcastFloat32ToInt32(), node);
1250       isFloat = true;
1251       V8_FALLTHROUGH;
1252     case MachineRepresentation::kWord32:
1253     case MachineRepresentation::kWord16:
1254       result = mcgraph()->Int32Constant(0);
1255       break;
1256     case MachineRepresentation::kWord8:
1257       // No need to change endianness for byte size, return original node
1258       return node;
1259       break;
1260     case MachineRepresentation::kSimd128:
1261       DCHECK(ReverseBytesSupported(m, valueSizeInBytes));
1262       break;
1263     default:
1264       UNREACHABLE();
1265       break;
1266   }
1267 
1268   int i;
1269   uint32_t shiftCount;
1270 
1271   if (ReverseBytesSupported(m, valueSizeInBytes < 4 ? 4 : valueSizeInBytes)) {
1272     switch (valueSizeInBytes) {
1273       case 2:
1274         result =
1275             graph()->NewNode(m->Word32ReverseBytes(),
1276                              graph()->NewNode(m->Word32Shl(), value,
1277                                               mcgraph()->Int32Constant(16)));
1278         break;
1279       case 4:
1280         result = graph()->NewNode(m->Word32ReverseBytes(), value);
1281         break;
1282       case 8:
1283         result = graph()->NewNode(m->Word64ReverseBytes(), value);
1284         break;
1285       case 16: {
1286         Node* byte_reversed_lanes[4];
1287         for (int lane = 0; lane < 4; lane++) {
1288           byte_reversed_lanes[lane] = graph()->NewNode(
1289               m->Word32ReverseBytes(),
1290               graph()->NewNode(mcgraph()->machine()->I32x4ExtractLane(lane),
1291                                value));
1292         }
1293 
1294         // This is making a copy of the value.
1295         result =
1296             graph()->NewNode(mcgraph()->machine()->S128And(), value, value);
1297 
1298         for (int lane = 0; lane < 4; lane++) {
1299           result =
1300               graph()->NewNode(mcgraph()->machine()->I32x4ReplaceLane(3 - lane),
1301                                result, byte_reversed_lanes[lane]);
1302         }
1303 
1304         break;
1305       }
1306       default:
1307         UNREACHABLE();
1308     }
1309   } else {
1310     for (i = 0, shiftCount = valueSizeInBits - 8; i < valueSizeInBits / 2;
1311          i += 8, shiftCount -= 16) {
1312       Node* shiftLower;
1313       Node* shiftHigher;
1314       Node* lowerByte;
1315       Node* higherByte;
1316 
1317       DCHECK_LT(0, shiftCount);
1318       DCHECK_EQ(0, (shiftCount + 8) % 16);
1319 
1320       if (valueSizeInBits > 32) {
1321         shiftLower = graph()->NewNode(m->Word64Shl(), value,
1322                                       mcgraph()->Int64Constant(shiftCount));
1323         shiftHigher = graph()->NewNode(m->Word64Shr(), value,
1324                                        mcgraph()->Int64Constant(shiftCount));
1325         lowerByte = graph()->NewNode(
1326             m->Word64And(), shiftLower,
1327             mcgraph()->Int64Constant(static_cast<uint64_t>(0xFF)
1328                                      << (valueSizeInBits - 8 - i)));
1329         higherByte = graph()->NewNode(
1330             m->Word64And(), shiftHigher,
1331             mcgraph()->Int64Constant(static_cast<uint64_t>(0xFF) << i));
1332         result = graph()->NewNode(m->Word64Or(), result, lowerByte);
1333         result = graph()->NewNode(m->Word64Or(), result, higherByte);
1334       } else {
1335         shiftLower = graph()->NewNode(m->Word32Shl(), value,
1336                                       mcgraph()->Int32Constant(shiftCount));
1337         shiftHigher = graph()->NewNode(m->Word32Shr(), value,
1338                                        mcgraph()->Int32Constant(shiftCount));
1339         lowerByte = graph()->NewNode(
1340             m->Word32And(), shiftLower,
1341             mcgraph()->Int32Constant(static_cast<uint32_t>(0xFF)
1342                                      << (valueSizeInBits - 8 - i)));
1343         higherByte = graph()->NewNode(
1344             m->Word32And(), shiftHigher,
1345             mcgraph()->Int32Constant(static_cast<uint32_t>(0xFF) << i));
1346         result = graph()->NewNode(m->Word32Or(), result, lowerByte);
1347         result = graph()->NewNode(m->Word32Or(), result, higherByte);
1348       }
1349     }
1350   }
1351 
1352   if (isFloat) {
1353     switch (memtype.representation()) {
1354       case MachineRepresentation::kFloat64:
1355         result = graph()->NewNode(m->BitcastInt64ToFloat64(), result);
1356         break;
1357       case MachineRepresentation::kFloat32:
1358         result = graph()->NewNode(m->BitcastInt32ToFloat32(), result);
1359         break;
1360       default:
1361         UNREACHABLE();
1362         break;
1363     }
1364   }
1365 
1366   // We need to sign extend the value
1367   if (memtype.IsSigned()) {
1368     DCHECK(!isFloat);
1369     if (valueSizeInBits < 32) {
1370       Node* shiftBitCount;
1371       // Perform sign extension using following trick
1372       // result = (x << machine_width - type_width) >> (machine_width -
1373       // type_width)
1374       if (wasmtype == wasm::kWasmI64) {
1375         shiftBitCount = mcgraph()->Int32Constant(64 - valueSizeInBits);
1376         result = graph()->NewNode(
1377             m->Word64Sar(),
1378             graph()->NewNode(m->Word64Shl(),
1379                              graph()->NewNode(m->ChangeInt32ToInt64(), result),
1380                              shiftBitCount),
1381             shiftBitCount);
1382       } else if (wasmtype == wasm::kWasmI32) {
1383         shiftBitCount = mcgraph()->Int32Constant(32 - valueSizeInBits);
1384         result = graph()->NewNode(
1385             m->Word32Sar(),
1386             graph()->NewNode(m->Word32Shl(), result, shiftBitCount),
1387             shiftBitCount);
1388       }
1389     }
1390   }
1391 
1392   return result;
1393 }
1394 
BuildF32CopySign(Node * left,Node * right)1395 Node* WasmGraphBuilder::BuildF32CopySign(Node* left, Node* right) {
1396   Node* result = Unop(
1397       wasm::kExprF32ReinterpretI32,
1398       Binop(wasm::kExprI32Ior,
1399             Binop(wasm::kExprI32And, Unop(wasm::kExprI32ReinterpretF32, left),
1400                   mcgraph()->Int32Constant(0x7FFFFFFF)),
1401             Binop(wasm::kExprI32And, Unop(wasm::kExprI32ReinterpretF32, right),
1402                   mcgraph()->Int32Constant(0x80000000))));
1403 
1404   return result;
1405 }
1406 
BuildF64CopySign(Node * left,Node * right)1407 Node* WasmGraphBuilder::BuildF64CopySign(Node* left, Node* right) {
1408 #if WASM_64
1409   Node* result = Unop(
1410       wasm::kExprF64ReinterpretI64,
1411       Binop(wasm::kExprI64Ior,
1412             Binop(wasm::kExprI64And, Unop(wasm::kExprI64ReinterpretF64, left),
1413                   mcgraph()->Int64Constant(0x7FFFFFFFFFFFFFFF)),
1414             Binop(wasm::kExprI64And, Unop(wasm::kExprI64ReinterpretF64, right),
1415                   mcgraph()->Int64Constant(0x8000000000000000))));
1416 
1417   return result;
1418 #else
1419   MachineOperatorBuilder* m = mcgraph()->machine();
1420 
1421   Node* high_word_left = graph()->NewNode(m->Float64ExtractHighWord32(), left);
1422   Node* high_word_right =
1423       graph()->NewNode(m->Float64ExtractHighWord32(), right);
1424 
1425   Node* new_high_word = Binop(wasm::kExprI32Ior,
1426                               Binop(wasm::kExprI32And, high_word_left,
1427                                     mcgraph()->Int32Constant(0x7FFFFFFF)),
1428                               Binop(wasm::kExprI32And, high_word_right,
1429                                     mcgraph()->Int32Constant(0x80000000)));
1430 
1431   return graph()->NewNode(m->Float64InsertHighWord32(), left, new_high_word);
1432 #endif
1433 }
1434 
1435 namespace {
1436 
IntConvertType(wasm::WasmOpcode opcode)1437 MachineType IntConvertType(wasm::WasmOpcode opcode) {
1438   switch (opcode) {
1439     case wasm::kExprI32SConvertF32:
1440     case wasm::kExprI32SConvertF64:
1441     case wasm::kExprI32SConvertSatF32:
1442     case wasm::kExprI32SConvertSatF64:
1443       return MachineType::Int32();
1444     case wasm::kExprI32UConvertF32:
1445     case wasm::kExprI32UConvertF64:
1446     case wasm::kExprI32UConvertSatF32:
1447     case wasm::kExprI32UConvertSatF64:
1448       return MachineType::Uint32();
1449     case wasm::kExprI64SConvertF32:
1450     case wasm::kExprI64SConvertF64:
1451     case wasm::kExprI64SConvertSatF32:
1452     case wasm::kExprI64SConvertSatF64:
1453       return MachineType::Int64();
1454     case wasm::kExprI64UConvertF32:
1455     case wasm::kExprI64UConvertF64:
1456     case wasm::kExprI64UConvertSatF32:
1457     case wasm::kExprI64UConvertSatF64:
1458       return MachineType::Uint64();
1459     default:
1460       UNREACHABLE();
1461   }
1462 }
1463 
FloatConvertType(wasm::WasmOpcode opcode)1464 MachineType FloatConvertType(wasm::WasmOpcode opcode) {
1465   switch (opcode) {
1466     case wasm::kExprI32SConvertF32:
1467     case wasm::kExprI32UConvertF32:
1468     case wasm::kExprI32SConvertSatF32:
1469     case wasm::kExprI64SConvertF32:
1470     case wasm::kExprI64UConvertF32:
1471     case wasm::kExprI32UConvertSatF32:
1472     case wasm::kExprI64SConvertSatF32:
1473     case wasm::kExprI64UConvertSatF32:
1474       return MachineType::Float32();
1475     case wasm::kExprI32SConvertF64:
1476     case wasm::kExprI32UConvertF64:
1477     case wasm::kExprI64SConvertF64:
1478     case wasm::kExprI64UConvertF64:
1479     case wasm::kExprI32SConvertSatF64:
1480     case wasm::kExprI32UConvertSatF64:
1481     case wasm::kExprI64SConvertSatF64:
1482     case wasm::kExprI64UConvertSatF64:
1483       return MachineType::Float64();
1484     default:
1485       UNREACHABLE();
1486   }
1487 }
1488 
ConvertOp(WasmGraphBuilder * builder,wasm::WasmOpcode opcode)1489 const Operator* ConvertOp(WasmGraphBuilder* builder, wasm::WasmOpcode opcode) {
1490   switch (opcode) {
1491     case wasm::kExprI32SConvertF32:
1492     case wasm::kExprI32SConvertSatF32:
1493       return builder->mcgraph()->machine()->TruncateFloat32ToInt32();
1494     case wasm::kExprI32UConvertF32:
1495     case wasm::kExprI32UConvertSatF32:
1496       return builder->mcgraph()->machine()->TruncateFloat32ToUint32();
1497     case wasm::kExprI32SConvertF64:
1498     case wasm::kExprI32SConvertSatF64:
1499       return builder->mcgraph()->machine()->ChangeFloat64ToInt32();
1500     case wasm::kExprI32UConvertF64:
1501     case wasm::kExprI32UConvertSatF64:
1502       return builder->mcgraph()->machine()->TruncateFloat64ToUint32();
1503     case wasm::kExprI64SConvertF32:
1504     case wasm::kExprI64SConvertSatF32:
1505       return builder->mcgraph()->machine()->TryTruncateFloat32ToInt64();
1506     case wasm::kExprI64UConvertF32:
1507     case wasm::kExprI64UConvertSatF32:
1508       return builder->mcgraph()->machine()->TryTruncateFloat32ToUint64();
1509     case wasm::kExprI64SConvertF64:
1510     case wasm::kExprI64SConvertSatF64:
1511       return builder->mcgraph()->machine()->TryTruncateFloat64ToInt64();
1512     case wasm::kExprI64UConvertF64:
1513     case wasm::kExprI64UConvertSatF64:
1514       return builder->mcgraph()->machine()->TryTruncateFloat64ToUint64();
1515     default:
1516       UNREACHABLE();
1517   }
1518 }
1519 
ConvertBackOp(wasm::WasmOpcode opcode)1520 wasm::WasmOpcode ConvertBackOp(wasm::WasmOpcode opcode) {
1521   switch (opcode) {
1522     case wasm::kExprI32SConvertF32:
1523     case wasm::kExprI32SConvertSatF32:
1524       return wasm::kExprF32SConvertI32;
1525     case wasm::kExprI32UConvertF32:
1526     case wasm::kExprI32UConvertSatF32:
1527       return wasm::kExprF32UConvertI32;
1528     case wasm::kExprI32SConvertF64:
1529     case wasm::kExprI32SConvertSatF64:
1530       return wasm::kExprF64SConvertI32;
1531     case wasm::kExprI32UConvertF64:
1532     case wasm::kExprI32UConvertSatF64:
1533       return wasm::kExprF64UConvertI32;
1534     default:
1535       UNREACHABLE();
1536   }
1537 }
1538 
IsTrappingConvertOp(wasm::WasmOpcode opcode)1539 bool IsTrappingConvertOp(wasm::WasmOpcode opcode) {
1540   switch (opcode) {
1541     case wasm::kExprI32SConvertF32:
1542     case wasm::kExprI32UConvertF32:
1543     case wasm::kExprI32SConvertF64:
1544     case wasm::kExprI32UConvertF64:
1545     case wasm::kExprI64SConvertF32:
1546     case wasm::kExprI64UConvertF32:
1547     case wasm::kExprI64SConvertF64:
1548     case wasm::kExprI64UConvertF64:
1549       return true;
1550     case wasm::kExprI32SConvertSatF64:
1551     case wasm::kExprI32UConvertSatF64:
1552     case wasm::kExprI32SConvertSatF32:
1553     case wasm::kExprI32UConvertSatF32:
1554     case wasm::kExprI64SConvertSatF32:
1555     case wasm::kExprI64UConvertSatF32:
1556     case wasm::kExprI64SConvertSatF64:
1557     case wasm::kExprI64UConvertSatF64:
1558       return false;
1559     default:
1560       UNREACHABLE();
1561   }
1562 }
1563 
Zero(WasmGraphBuilder * builder,const MachineType & ty)1564 Node* Zero(WasmGraphBuilder* builder, const MachineType& ty) {
1565   switch (ty.representation()) {
1566     case MachineRepresentation::kWord32:
1567       return builder->Int32Constant(0);
1568     case MachineRepresentation::kWord64:
1569       return builder->Int64Constant(0);
1570     case MachineRepresentation::kFloat32:
1571       return builder->Float32Constant(0.0);
1572     case MachineRepresentation::kFloat64:
1573       return builder->Float64Constant(0.0);
1574     default:
1575       UNREACHABLE();
1576   }
1577 }
1578 
Min(WasmGraphBuilder * builder,const MachineType & ty)1579 Node* Min(WasmGraphBuilder* builder, const MachineType& ty) {
1580   switch (ty.semantic()) {
1581     case MachineSemantic::kInt32:
1582       return builder->Int32Constant(std::numeric_limits<int32_t>::min());
1583     case MachineSemantic::kUint32:
1584       return builder->Int32Constant(std::numeric_limits<uint32_t>::min());
1585     case MachineSemantic::kInt64:
1586       return builder->Int64Constant(std::numeric_limits<int64_t>::min());
1587     case MachineSemantic::kUint64:
1588       return builder->Int64Constant(std::numeric_limits<uint64_t>::min());
1589     default:
1590       UNREACHABLE();
1591   }
1592 }
1593 
Max(WasmGraphBuilder * builder,const MachineType & ty)1594 Node* Max(WasmGraphBuilder* builder, const MachineType& ty) {
1595   switch (ty.semantic()) {
1596     case MachineSemantic::kInt32:
1597       return builder->Int32Constant(std::numeric_limits<int32_t>::max());
1598     case MachineSemantic::kUint32:
1599       return builder->Int32Constant(std::numeric_limits<uint32_t>::max());
1600     case MachineSemantic::kInt64:
1601       return builder->Int64Constant(std::numeric_limits<int64_t>::max());
1602     case MachineSemantic::kUint64:
1603       return builder->Int64Constant(std::numeric_limits<uint64_t>::max());
1604     default:
1605       UNREACHABLE();
1606   }
1607 }
1608 
TruncOp(const MachineType & ty)1609 wasm::WasmOpcode TruncOp(const MachineType& ty) {
1610   switch (ty.representation()) {
1611     case MachineRepresentation::kFloat32:
1612       return wasm::kExprF32Trunc;
1613     case MachineRepresentation::kFloat64:
1614       return wasm::kExprF64Trunc;
1615     default:
1616       UNREACHABLE();
1617   }
1618 }
1619 
NeOp(const MachineType & ty)1620 wasm::WasmOpcode NeOp(const MachineType& ty) {
1621   switch (ty.representation()) {
1622     case MachineRepresentation::kFloat32:
1623       return wasm::kExprF32Ne;
1624     case MachineRepresentation::kFloat64:
1625       return wasm::kExprF64Ne;
1626     default:
1627       UNREACHABLE();
1628   }
1629 }
1630 
LtOp(const MachineType & ty)1631 wasm::WasmOpcode LtOp(const MachineType& ty) {
1632   switch (ty.representation()) {
1633     case MachineRepresentation::kFloat32:
1634       return wasm::kExprF32Lt;
1635     case MachineRepresentation::kFloat64:
1636       return wasm::kExprF64Lt;
1637     default:
1638       UNREACHABLE();
1639   }
1640 }
1641 
ConvertTrapTest(WasmGraphBuilder * builder,wasm::WasmOpcode opcode,const MachineType & int_ty,const MachineType & float_ty,Node * trunc,Node * converted_value)1642 Node* ConvertTrapTest(WasmGraphBuilder* builder, wasm::WasmOpcode opcode,
1643                       const MachineType& int_ty, const MachineType& float_ty,
1644                       Node* trunc, Node* converted_value) {
1645   if (int_ty.representation() == MachineRepresentation::kWord32) {
1646     Node* check = builder->Unop(ConvertBackOp(opcode), converted_value);
1647     return builder->Binop(NeOp(float_ty), trunc, check);
1648   }
1649   return builder->graph()->NewNode(builder->mcgraph()->common()->Projection(1),
1650                                    trunc, builder->graph()->start());
1651 }
1652 
ConvertSaturateTest(WasmGraphBuilder * builder,wasm::WasmOpcode opcode,const MachineType & int_ty,const MachineType & float_ty,Node * trunc,Node * converted_value)1653 Node* ConvertSaturateTest(WasmGraphBuilder* builder, wasm::WasmOpcode opcode,
1654                           const MachineType& int_ty,
1655                           const MachineType& float_ty, Node* trunc,
1656                           Node* converted_value) {
1657   Node* test = ConvertTrapTest(builder, opcode, int_ty, float_ty, trunc,
1658                                converted_value);
1659   if (int_ty.representation() == MachineRepresentation::kWord64) {
1660     test = builder->Binop(wasm::kExprI64Eq, test, builder->Int64Constant(0));
1661   }
1662   return test;
1663 }
1664 
1665 }  // namespace
1666 
BuildIntConvertFloat(Node * input,wasm::WasmCodePosition position,wasm::WasmOpcode opcode)1667 Node* WasmGraphBuilder::BuildIntConvertFloat(Node* input,
1668                                              wasm::WasmCodePosition position,
1669                                              wasm::WasmOpcode opcode) {
1670   const MachineType int_ty = IntConvertType(opcode);
1671   const MachineType float_ty = FloatConvertType(opcode);
1672   const Operator* conv_op = ConvertOp(this, opcode);
1673   Node* trunc = nullptr;
1674   Node* converted_value = nullptr;
1675   const bool is_int32 =
1676       int_ty.representation() == MachineRepresentation::kWord32;
1677   if (is_int32) {
1678     trunc = Unop(TruncOp(float_ty), input);
1679     converted_value = graph()->NewNode(conv_op, trunc);
1680   } else {
1681     trunc = graph()->NewNode(conv_op, input);
1682     converted_value = graph()->NewNode(mcgraph()->common()->Projection(0),
1683                                        trunc, graph()->start());
1684   }
1685   if (IsTrappingConvertOp(opcode)) {
1686     Node* test =
1687         ConvertTrapTest(this, opcode, int_ty, float_ty, trunc, converted_value);
1688     if (is_int32) {
1689       TrapIfTrue(wasm::kTrapFloatUnrepresentable, test, position);
1690     } else {
1691       ZeroCheck64(wasm::kTrapFloatUnrepresentable, test, position);
1692     }
1693     return converted_value;
1694   }
1695   Node* test = ConvertSaturateTest(this, opcode, int_ty, float_ty, trunc,
1696                                    converted_value);
1697   Diamond tl_d(graph(), mcgraph()->common(), test, BranchHint::kFalse);
1698   tl_d.Chain(Control());
1699   Node* nan_test = Binop(NeOp(float_ty), input, input);
1700   Diamond nan_d(graph(), mcgraph()->common(), nan_test, BranchHint::kFalse);
1701   nan_d.Nest(tl_d, true);
1702   Node* neg_test = Binop(LtOp(float_ty), input, Zero(this, float_ty));
1703   Diamond sat_d(graph(), mcgraph()->common(), neg_test, BranchHint::kNone);
1704   sat_d.Nest(nan_d, false);
1705   Node* sat_val =
1706       sat_d.Phi(int_ty.representation(), Min(this, int_ty), Max(this, int_ty));
1707   Node* nan_val =
1708       nan_d.Phi(int_ty.representation(), Zero(this, int_ty), sat_val);
1709   return tl_d.Phi(int_ty.representation(), nan_val, converted_value);
1710 }
1711 
BuildI32AsmjsSConvertF32(Node * input)1712 Node* WasmGraphBuilder::BuildI32AsmjsSConvertF32(Node* input) {
1713   MachineOperatorBuilder* m = mcgraph()->machine();
1714   // asm.js must use the wacky JS semantics.
1715   input = graph()->NewNode(m->ChangeFloat32ToFloat64(), input);
1716   return graph()->NewNode(m->TruncateFloat64ToWord32(), input);
1717 }
1718 
BuildI32AsmjsSConvertF64(Node * input)1719 Node* WasmGraphBuilder::BuildI32AsmjsSConvertF64(Node* input) {
1720   MachineOperatorBuilder* m = mcgraph()->machine();
1721   // asm.js must use the wacky JS semantics.
1722   return graph()->NewNode(m->TruncateFloat64ToWord32(), input);
1723 }
1724 
BuildI32AsmjsUConvertF32(Node * input)1725 Node* WasmGraphBuilder::BuildI32AsmjsUConvertF32(Node* input) {
1726   MachineOperatorBuilder* m = mcgraph()->machine();
1727   // asm.js must use the wacky JS semantics.
1728   input = graph()->NewNode(m->ChangeFloat32ToFloat64(), input);
1729   return graph()->NewNode(m->TruncateFloat64ToWord32(), input);
1730 }
1731 
BuildI32AsmjsUConvertF64(Node * input)1732 Node* WasmGraphBuilder::BuildI32AsmjsUConvertF64(Node* input) {
1733   MachineOperatorBuilder* m = mcgraph()->machine();
1734   // asm.js must use the wacky JS semantics.
1735   return graph()->NewNode(m->TruncateFloat64ToWord32(), input);
1736 }
1737 
BuildBitCountingCall(Node * input,ExternalReference ref,MachineRepresentation input_type)1738 Node* WasmGraphBuilder::BuildBitCountingCall(Node* input, ExternalReference ref,
1739                                              MachineRepresentation input_type) {
1740   Node* stack_slot_param =
1741       graph()->NewNode(mcgraph()->machine()->StackSlot(input_type));
1742 
1743   const Operator* store_op = mcgraph()->machine()->Store(
1744       StoreRepresentation(input_type, kNoWriteBarrier));
1745   SetEffect(graph()->NewNode(store_op, stack_slot_param,
1746                              mcgraph()->Int32Constant(0), input, Effect(),
1747                              Control()));
1748 
1749   MachineType sig_types[] = {MachineType::Int32(), MachineType::Pointer()};
1750   MachineSignature sig(1, 1, sig_types);
1751 
1752   Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(ref));
1753 
1754   return BuildCCall(&sig, function, stack_slot_param);
1755 }
1756 
BuildI32Ctz(Node * input)1757 Node* WasmGraphBuilder::BuildI32Ctz(Node* input) {
1758   return BuildBitCountingCall(input, ExternalReference::wasm_word32_ctz(),
1759                               MachineRepresentation::kWord32);
1760 }
1761 
BuildI64Ctz(Node * input)1762 Node* WasmGraphBuilder::BuildI64Ctz(Node* input) {
1763   return Unop(wasm::kExprI64UConvertI32,
1764               BuildBitCountingCall(input, ExternalReference::wasm_word64_ctz(),
1765                                    MachineRepresentation::kWord64));
1766 }
1767 
BuildI32Popcnt(Node * input)1768 Node* WasmGraphBuilder::BuildI32Popcnt(Node* input) {
1769   return BuildBitCountingCall(input, ExternalReference::wasm_word32_popcnt(),
1770                               MachineRepresentation::kWord32);
1771 }
1772 
BuildI64Popcnt(Node * input)1773 Node* WasmGraphBuilder::BuildI64Popcnt(Node* input) {
1774   return Unop(
1775       wasm::kExprI64UConvertI32,
1776       BuildBitCountingCall(input, ExternalReference::wasm_word64_popcnt(),
1777                            MachineRepresentation::kWord64));
1778 }
1779 
BuildF32Trunc(Node * input)1780 Node* WasmGraphBuilder::BuildF32Trunc(Node* input) {
1781   MachineType type = MachineType::Float32();
1782   ExternalReference ref = ExternalReference::wasm_f32_trunc();
1783 
1784   return BuildCFuncInstruction(ref, type, input);
1785 }
1786 
BuildF32Floor(Node * input)1787 Node* WasmGraphBuilder::BuildF32Floor(Node* input) {
1788   MachineType type = MachineType::Float32();
1789   ExternalReference ref = ExternalReference::wasm_f32_floor();
1790   return BuildCFuncInstruction(ref, type, input);
1791 }
1792 
BuildF32Ceil(Node * input)1793 Node* WasmGraphBuilder::BuildF32Ceil(Node* input) {
1794   MachineType type = MachineType::Float32();
1795   ExternalReference ref = ExternalReference::wasm_f32_ceil();
1796   return BuildCFuncInstruction(ref, type, input);
1797 }
1798 
BuildF32NearestInt(Node * input)1799 Node* WasmGraphBuilder::BuildF32NearestInt(Node* input) {
1800   MachineType type = MachineType::Float32();
1801   ExternalReference ref = ExternalReference::wasm_f32_nearest_int();
1802   return BuildCFuncInstruction(ref, type, input);
1803 }
1804 
BuildF64Trunc(Node * input)1805 Node* WasmGraphBuilder::BuildF64Trunc(Node* input) {
1806   MachineType type = MachineType::Float64();
1807   ExternalReference ref = ExternalReference::wasm_f64_trunc();
1808   return BuildCFuncInstruction(ref, type, input);
1809 }
1810 
BuildF64Floor(Node * input)1811 Node* WasmGraphBuilder::BuildF64Floor(Node* input) {
1812   MachineType type = MachineType::Float64();
1813   ExternalReference ref = ExternalReference::wasm_f64_floor();
1814   return BuildCFuncInstruction(ref, type, input);
1815 }
1816 
BuildF64Ceil(Node * input)1817 Node* WasmGraphBuilder::BuildF64Ceil(Node* input) {
1818   MachineType type = MachineType::Float64();
1819   ExternalReference ref = ExternalReference::wasm_f64_ceil();
1820   return BuildCFuncInstruction(ref, type, input);
1821 }
1822 
BuildF64NearestInt(Node * input)1823 Node* WasmGraphBuilder::BuildF64NearestInt(Node* input) {
1824   MachineType type = MachineType::Float64();
1825   ExternalReference ref = ExternalReference::wasm_f64_nearest_int();
1826   return BuildCFuncInstruction(ref, type, input);
1827 }
1828 
BuildF64Acos(Node * input)1829 Node* WasmGraphBuilder::BuildF64Acos(Node* input) {
1830   MachineType type = MachineType::Float64();
1831   ExternalReference ref = ExternalReference::f64_acos_wrapper_function();
1832   return BuildCFuncInstruction(ref, type, input);
1833 }
1834 
BuildF64Asin(Node * input)1835 Node* WasmGraphBuilder::BuildF64Asin(Node* input) {
1836   MachineType type = MachineType::Float64();
1837   ExternalReference ref = ExternalReference::f64_asin_wrapper_function();
1838   return BuildCFuncInstruction(ref, type, input);
1839 }
1840 
BuildF64Pow(Node * left,Node * right)1841 Node* WasmGraphBuilder::BuildF64Pow(Node* left, Node* right) {
1842   MachineType type = MachineType::Float64();
1843   ExternalReference ref = ExternalReference::wasm_float64_pow();
1844   return BuildCFuncInstruction(ref, type, left, right);
1845 }
1846 
BuildF64Mod(Node * left,Node * right)1847 Node* WasmGraphBuilder::BuildF64Mod(Node* left, Node* right) {
1848   MachineType type = MachineType::Float64();
1849   ExternalReference ref = ExternalReference::f64_mod_wrapper_function();
1850   return BuildCFuncInstruction(ref, type, left, right);
1851 }
1852 
BuildCFuncInstruction(ExternalReference ref,MachineType type,Node * input0,Node * input1)1853 Node* WasmGraphBuilder::BuildCFuncInstruction(ExternalReference ref,
1854                                               MachineType type, Node* input0,
1855                                               Node* input1) {
1856   // We do truncation by calling a C function which calculates the result.
1857   // The input is passed to the C function as a byte buffer holding the two
1858   // input doubles. We reserve this byte buffer as a stack slot, store the
1859   // parameters in this buffer slots, pass a pointer to the buffer to the C
1860   // function, and after calling the C function we collect the return value from
1861   // the buffer.
1862 
1863   const int type_size = ElementSizeInBytes(type.representation());
1864   const int stack_slot_bytes = (input1 == nullptr ? 1 : 2) * type_size;
1865   Node* stack_slot =
1866       graph()->NewNode(mcgraph()->machine()->StackSlot(stack_slot_bytes));
1867 
1868   const Operator* store_op = mcgraph()->machine()->Store(
1869       StoreRepresentation(type.representation(), kNoWriteBarrier));
1870   SetEffect(graph()->NewNode(store_op, stack_slot, mcgraph()->Int32Constant(0),
1871                              input0, Effect(), Control()));
1872 
1873   Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(ref));
1874 
1875   if (input1 != nullptr) {
1876     SetEffect(graph()->NewNode(store_op, stack_slot,
1877                                mcgraph()->Int32Constant(type_size), input1,
1878                                Effect(), Control()));
1879   }
1880 
1881   MachineType sig_types[] = {MachineType::Pointer()};
1882   MachineSignature sig(0, 1, sig_types);
1883   BuildCCall(&sig, function, stack_slot);
1884 
1885   return SetEffect(graph()->NewNode(mcgraph()->machine()->Load(type),
1886                                     stack_slot, mcgraph()->Int32Constant(0),
1887                                     Effect(), Control()));
1888 }
1889 
BuildF32SConvertI64(Node * input)1890 Node* WasmGraphBuilder::BuildF32SConvertI64(Node* input) {
1891   // TODO(titzer/bradnelson): Check handlng of asm.js case.
1892   return BuildIntToFloatConversionInstruction(
1893       input, ExternalReference::wasm_int64_to_float32(),
1894       MachineRepresentation::kWord64, MachineType::Float32());
1895 }
BuildF32UConvertI64(Node * input)1896 Node* WasmGraphBuilder::BuildF32UConvertI64(Node* input) {
1897   // TODO(titzer/bradnelson): Check handlng of asm.js case.
1898   return BuildIntToFloatConversionInstruction(
1899       input, ExternalReference::wasm_uint64_to_float32(),
1900       MachineRepresentation::kWord64, MachineType::Float32());
1901 }
BuildF64SConvertI64(Node * input)1902 Node* WasmGraphBuilder::BuildF64SConvertI64(Node* input) {
1903   return BuildIntToFloatConversionInstruction(
1904       input, ExternalReference::wasm_int64_to_float64(),
1905       MachineRepresentation::kWord64, MachineType::Float64());
1906 }
BuildF64UConvertI64(Node * input)1907 Node* WasmGraphBuilder::BuildF64UConvertI64(Node* input) {
1908   return BuildIntToFloatConversionInstruction(
1909       input, ExternalReference::wasm_uint64_to_float64(),
1910       MachineRepresentation::kWord64, MachineType::Float64());
1911 }
1912 
BuildIntToFloatConversionInstruction(Node * input,ExternalReference ref,MachineRepresentation parameter_representation,const MachineType result_type)1913 Node* WasmGraphBuilder::BuildIntToFloatConversionInstruction(
1914     Node* input, ExternalReference ref,
1915     MachineRepresentation parameter_representation,
1916     const MachineType result_type) {
1917   int stack_slot_size =
1918       std::max(ElementSizeInBytes(parameter_representation),
1919                ElementSizeInBytes(result_type.representation()));
1920   Node* stack_slot =
1921       graph()->NewNode(mcgraph()->machine()->StackSlot(stack_slot_size));
1922   const Operator* store_op = mcgraph()->machine()->Store(
1923       StoreRepresentation(parameter_representation, kNoWriteBarrier));
1924   SetEffect(graph()->NewNode(store_op, stack_slot, mcgraph()->Int32Constant(0),
1925                              input, Effect(), Control()));
1926   MachineType sig_types[] = {MachineType::Pointer()};
1927   MachineSignature sig(0, 1, sig_types);
1928   Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(ref));
1929   BuildCCall(&sig, function, stack_slot);
1930   return SetEffect(graph()->NewNode(mcgraph()->machine()->Load(result_type),
1931                                     stack_slot, mcgraph()->Int32Constant(0),
1932                                     Effect(), Control()));
1933 }
1934 
1935 namespace {
1936 
convert_ccall_ref(WasmGraphBuilder * builder,wasm::WasmOpcode opcode)1937 ExternalReference convert_ccall_ref(WasmGraphBuilder* builder,
1938                                     wasm::WasmOpcode opcode) {
1939   switch (opcode) {
1940     case wasm::kExprI64SConvertF32:
1941     case wasm::kExprI64SConvertSatF32:
1942       return ExternalReference::wasm_float32_to_int64();
1943     case wasm::kExprI64UConvertF32:
1944     case wasm::kExprI64UConvertSatF32:
1945       return ExternalReference::wasm_float32_to_uint64();
1946     case wasm::kExprI64SConvertF64:
1947     case wasm::kExprI64SConvertSatF64:
1948       return ExternalReference::wasm_float64_to_int64();
1949     case wasm::kExprI64UConvertF64:
1950     case wasm::kExprI64UConvertSatF64:
1951       return ExternalReference::wasm_float64_to_uint64();
1952     default:
1953       UNREACHABLE();
1954   }
1955 }
1956 
1957 }  // namespace
1958 
BuildCcallConvertFloat(Node * input,wasm::WasmCodePosition position,wasm::WasmOpcode opcode)1959 Node* WasmGraphBuilder::BuildCcallConvertFloat(Node* input,
1960                                                wasm::WasmCodePosition position,
1961                                                wasm::WasmOpcode opcode) {
1962   const MachineType int_ty = IntConvertType(opcode);
1963   const MachineType float_ty = FloatConvertType(opcode);
1964   ExternalReference call_ref = convert_ccall_ref(this, opcode);
1965   int stack_slot_size = std::max(ElementSizeInBytes(int_ty.representation()),
1966                                  ElementSizeInBytes(float_ty.representation()));
1967   Node* stack_slot =
1968       graph()->NewNode(mcgraph()->machine()->StackSlot(stack_slot_size));
1969   const Operator* store_op = mcgraph()->machine()->Store(
1970       StoreRepresentation(float_ty.representation(), kNoWriteBarrier));
1971   SetEffect(graph()->NewNode(store_op, stack_slot, Int32Constant(0), input,
1972                              Effect(), Control()));
1973   MachineType sig_types[] = {MachineType::Int32(), MachineType::Pointer()};
1974   MachineSignature sig(1, 1, sig_types);
1975   Node* function =
1976       graph()->NewNode(mcgraph()->common()->ExternalConstant(call_ref));
1977   Node* overflow = BuildCCall(&sig, function, stack_slot);
1978   if (IsTrappingConvertOp(opcode)) {
1979     ZeroCheck32(wasm::kTrapFloatUnrepresentable, overflow, position);
1980     return SetEffect(graph()->NewNode(mcgraph()->machine()->Load(int_ty),
1981                                       stack_slot, Int32Constant(0), Effect(),
1982                                       Control()));
1983   }
1984   Node* test = Binop(wasm::kExprI32Eq, overflow, Int32Constant(0), position);
1985   Diamond tl_d(graph(), mcgraph()->common(), test, BranchHint::kFalse);
1986   tl_d.Chain(Control());
1987   Node* nan_test = Binop(NeOp(float_ty), input, input);
1988   Diamond nan_d(graph(), mcgraph()->common(), nan_test, BranchHint::kFalse);
1989   nan_d.Nest(tl_d, true);
1990   Node* neg_test = Binop(LtOp(float_ty), input, Zero(this, float_ty));
1991   Diamond sat_d(graph(), mcgraph()->common(), neg_test, BranchHint::kNone);
1992   sat_d.Nest(nan_d, false);
1993   Node* sat_val =
1994       sat_d.Phi(int_ty.representation(), Min(this, int_ty), Max(this, int_ty));
1995   Node* load =
1996       SetEffect(graph()->NewNode(mcgraph()->machine()->Load(int_ty), stack_slot,
1997                                  Int32Constant(0), Effect(), Control()));
1998   Node* nan_val =
1999       nan_d.Phi(int_ty.representation(), Zero(this, int_ty), sat_val);
2000   return tl_d.Phi(int_ty.representation(), nan_val, load);
2001 }
2002 
GrowMemory(Node * input)2003 Node* WasmGraphBuilder::GrowMemory(Node* input) {
2004   SetNeedsStackCheck();
2005 
2006   WasmGrowMemoryDescriptor interface_descriptor;
2007   auto call_descriptor = Linkage::GetStubCallDescriptor(
2008       mcgraph()->zone(),                              // zone
2009       interface_descriptor,                           // descriptor
2010       interface_descriptor.GetStackParameterCount(),  // stack parameter count
2011       CallDescriptor::kNoFlags,                       // flags
2012       Operator::kNoProperties,                        // properties
2013       StubCallMode::kCallWasmRuntimeStub);            // stub call mode
2014   // A direct call to a wasm runtime stub defined in this module.
2015   // Just encode the stub index. This will be patched at relocation.
2016   Node* call_target = mcgraph()->RelocatableIntPtrConstant(
2017       wasm::WasmCode::kWasmGrowMemory, RelocInfo::WASM_STUB_CALL);
2018   return SetEffect(
2019       SetControl(graph()->NewNode(mcgraph()->common()->Call(call_descriptor),
2020                                   call_target, input, Effect(), Control())));
2021 }
2022 
GetExceptionEncodedSize(const wasm::WasmException * exception) const2023 uint32_t WasmGraphBuilder::GetExceptionEncodedSize(
2024     const wasm::WasmException* exception) const {
2025   const wasm::WasmExceptionSig* sig = exception->sig;
2026   uint32_t encoded_size = 0;
2027   for (size_t i = 0; i < sig->parameter_count(); ++i) {
2028     size_t byte_size = static_cast<size_t>(
2029         wasm::ValueTypes::ElementSizeInBytes(sig->GetParam(i)));
2030     DCHECK_EQ(byte_size % kBytesPerExceptionValuesArrayElement, 0);
2031     DCHECK_LE(1, byte_size / kBytesPerExceptionValuesArrayElement);
2032     encoded_size += byte_size / kBytesPerExceptionValuesArrayElement;
2033   }
2034   return encoded_size;
2035 }
2036 
Throw(uint32_t tag,const wasm::WasmException * exception,const Vector<Node * > values)2037 Node* WasmGraphBuilder::Throw(uint32_t tag,
2038                               const wasm::WasmException* exception,
2039                               const Vector<Node*> values) {
2040   SetNeedsStackCheck();
2041   uint32_t encoded_size = GetExceptionEncodedSize(exception);
2042   Node* create_parameters[] = {
2043       BuildChangeUint31ToSmi(ConvertExceptionTagToRuntimeId(tag)),
2044       BuildChangeUint31ToSmi(Uint32Constant(encoded_size))};
2045   BuildCallToRuntime(Runtime::kWasmThrowCreate, create_parameters,
2046                      arraysize(create_parameters));
2047   uint32_t index = 0;
2048   const wasm::WasmExceptionSig* sig = exception->sig;
2049   MachineOperatorBuilder* m = mcgraph()->machine();
2050   for (size_t i = 0; i < sig->parameter_count(); ++i) {
2051     Node* value = values[i];
2052     switch (sig->GetParam(i)) {
2053       case wasm::kWasmF32:
2054         value = graph()->NewNode(m->BitcastFloat32ToInt32(), value);
2055         V8_FALLTHROUGH;
2056       case wasm::kWasmI32:
2057         BuildEncodeException32BitValue(&index, value);
2058         break;
2059       case wasm::kWasmF64:
2060         value = graph()->NewNode(m->BitcastFloat64ToInt64(), value);
2061         V8_FALLTHROUGH;
2062       case wasm::kWasmI64: {
2063         Node* upper32 = graph()->NewNode(
2064             m->TruncateInt64ToInt32(),
2065             Binop(wasm::kExprI64ShrU, value, Int64Constant(32)));
2066         BuildEncodeException32BitValue(&index, upper32);
2067         Node* lower32 = graph()->NewNode(m->TruncateInt64ToInt32(), value);
2068         BuildEncodeException32BitValue(&index, lower32);
2069         break;
2070       }
2071       default:
2072         UNREACHABLE();
2073     }
2074   }
2075   DCHECK_EQ(encoded_size, index);
2076   return BuildCallToRuntime(Runtime::kWasmThrow, nullptr, 0);
2077 }
2078 
BuildEncodeException32BitValue(uint32_t * index,Node * value)2079 void WasmGraphBuilder::BuildEncodeException32BitValue(uint32_t* index,
2080                                                       Node* value) {
2081   MachineOperatorBuilder* machine = mcgraph()->machine();
2082   Node* upper_parameters[] = {
2083       BuildChangeUint31ToSmi(Int32Constant(*index)),
2084       BuildChangeUint31ToSmi(
2085           graph()->NewNode(machine->Word32Shr(), value, Int32Constant(16))),
2086   };
2087   BuildCallToRuntime(Runtime::kWasmExceptionSetElement, upper_parameters,
2088                      arraysize(upper_parameters));
2089   ++(*index);
2090   Node* lower_parameters[] = {
2091       BuildChangeUint31ToSmi(Int32Constant(*index)),
2092       BuildChangeUint31ToSmi(graph()->NewNode(machine->Word32And(), value,
2093                                               Int32Constant(0xFFFFu))),
2094   };
2095   BuildCallToRuntime(Runtime::kWasmExceptionSetElement, lower_parameters,
2096                      arraysize(lower_parameters));
2097   ++(*index);
2098 }
2099 
BuildDecodeException32BitValue(Node * const * values,uint32_t * index)2100 Node* WasmGraphBuilder::BuildDecodeException32BitValue(Node* const* values,
2101                                                        uint32_t* index) {
2102   MachineOperatorBuilder* machine = mcgraph()->machine();
2103   Node* upper = BuildChangeSmiToInt32(values[*index]);
2104   (*index)++;
2105   upper = graph()->NewNode(machine->Word32Shl(), upper, Int32Constant(16));
2106   Node* lower = BuildChangeSmiToInt32(values[*index]);
2107   (*index)++;
2108   Node* value = graph()->NewNode(machine->Word32Or(), upper, lower);
2109   return value;
2110 }
2111 
Rethrow()2112 Node* WasmGraphBuilder::Rethrow() {
2113   SetNeedsStackCheck();
2114   Node* result = BuildCallToRuntime(Runtime::kWasmThrow, nullptr, 0);
2115   return result;
2116 }
2117 
ConvertExceptionTagToRuntimeId(uint32_t tag)2118 Node* WasmGraphBuilder::ConvertExceptionTagToRuntimeId(uint32_t tag) {
2119   // TODO(kschimpf): Handle exceptions from different modules, when they are
2120   // linked at runtime.
2121   return Uint32Constant(tag);
2122 }
2123 
GetExceptionRuntimeId()2124 Node* WasmGraphBuilder::GetExceptionRuntimeId() {
2125   SetNeedsStackCheck();
2126   return BuildChangeSmiToInt32(
2127       BuildCallToRuntime(Runtime::kWasmGetExceptionRuntimeId, nullptr, 0));
2128 }
2129 
GetExceptionValues(const wasm::WasmException * except_decl)2130 Node** WasmGraphBuilder::GetExceptionValues(
2131     const wasm::WasmException* except_decl) {
2132   // TODO(kschimpf): We need to move this code to the function-body-decoder.cc
2133   // in order to build landing-pad (exception) edges in case the runtime
2134   // call causes an exception.
2135 
2136   // Start by getting the encoded values from the exception.
2137   uint32_t encoded_size = GetExceptionEncodedSize(except_decl);
2138   Node** values = Buffer(encoded_size);
2139   for (uint32_t i = 0; i < encoded_size; ++i) {
2140     Node* parameters[] = {BuildChangeUint31ToSmi(Uint32Constant(i))};
2141     values[i] = BuildCallToRuntime(Runtime::kWasmExceptionGetElement,
2142                                    parameters, arraysize(parameters));
2143   }
2144 
2145   // Now convert the leading entries to the corresponding parameter values.
2146   uint32_t index = 0;
2147   const wasm::WasmExceptionSig* sig = except_decl->sig;
2148   for (size_t i = 0; i < sig->parameter_count(); ++i) {
2149     Node* value = BuildDecodeException32BitValue(values, &index);
2150     switch (wasm::ValueType type = sig->GetParam(i)) {
2151       case wasm::kWasmF32: {
2152         value = Unop(wasm::kExprF32ReinterpretI32, value);
2153         break;
2154       }
2155       case wasm::kWasmI32:
2156         break;
2157       case wasm::kWasmF64:
2158       case wasm::kWasmI64: {
2159         Node* upper =
2160             Binop(wasm::kExprI64Shl, Unop(wasm::kExprI64UConvertI32, value),
2161                   Int64Constant(32));
2162         Node* lower = Unop(wasm::kExprI64UConvertI32,
2163                            BuildDecodeException32BitValue(values, &index));
2164         value = Binop(wasm::kExprI64Ior, upper, lower);
2165         if (type == wasm::kWasmF64) {
2166           value = Unop(wasm::kExprF64ReinterpretI64, value);
2167         }
2168         break;
2169       }
2170       default:
2171         UNREACHABLE();
2172     }
2173     values[i] = value;
2174   }
2175   DCHECK_EQ(index, encoded_size);
2176   return values;
2177 }
2178 
BuildI32DivS(Node * left,Node * right,wasm::WasmCodePosition position)2179 Node* WasmGraphBuilder::BuildI32DivS(Node* left, Node* right,
2180                                      wasm::WasmCodePosition position) {
2181   MachineOperatorBuilder* m = mcgraph()->machine();
2182   ZeroCheck32(wasm::kTrapDivByZero, right, position);
2183   Node* before = Control();
2184   Node* denom_is_m1;
2185   Node* denom_is_not_m1;
2186   BranchExpectFalse(
2187       graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(-1)),
2188       &denom_is_m1, &denom_is_not_m1);
2189   SetControl(denom_is_m1);
2190   TrapIfEq32(wasm::kTrapDivUnrepresentable, left, kMinInt, position);
2191   if (Control() != denom_is_m1) {
2192     SetControl(graph()->NewNode(mcgraph()->common()->Merge(2), denom_is_not_m1,
2193                                 Control()));
2194   } else {
2195     SetControl(before);
2196   }
2197   return graph()->NewNode(m->Int32Div(), left, right, Control());
2198 }
2199 
BuildI32RemS(Node * left,Node * right,wasm::WasmCodePosition position)2200 Node* WasmGraphBuilder::BuildI32RemS(Node* left, Node* right,
2201                                      wasm::WasmCodePosition position) {
2202   MachineOperatorBuilder* m = mcgraph()->machine();
2203 
2204   ZeroCheck32(wasm::kTrapRemByZero, right, position);
2205 
2206   Diamond d(
2207       graph(), mcgraph()->common(),
2208       graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(-1)),
2209       BranchHint::kFalse);
2210   d.Chain(Control());
2211 
2212   return d.Phi(MachineRepresentation::kWord32, mcgraph()->Int32Constant(0),
2213                graph()->NewNode(m->Int32Mod(), left, right, d.if_false));
2214 }
2215 
BuildI32DivU(Node * left,Node * right,wasm::WasmCodePosition position)2216 Node* WasmGraphBuilder::BuildI32DivU(Node* left, Node* right,
2217                                      wasm::WasmCodePosition position) {
2218   MachineOperatorBuilder* m = mcgraph()->machine();
2219   return graph()->NewNode(m->Uint32Div(), left, right,
2220                           ZeroCheck32(wasm::kTrapDivByZero, right, position));
2221 }
2222 
BuildI32RemU(Node * left,Node * right,wasm::WasmCodePosition position)2223 Node* WasmGraphBuilder::BuildI32RemU(Node* left, Node* right,
2224                                      wasm::WasmCodePosition position) {
2225   MachineOperatorBuilder* m = mcgraph()->machine();
2226   return graph()->NewNode(m->Uint32Mod(), left, right,
2227                           ZeroCheck32(wasm::kTrapRemByZero, right, position));
2228 }
2229 
BuildI32AsmjsDivS(Node * left,Node * right)2230 Node* WasmGraphBuilder::BuildI32AsmjsDivS(Node* left, Node* right) {
2231   MachineOperatorBuilder* m = mcgraph()->machine();
2232 
2233   Int32Matcher mr(right);
2234   if (mr.HasValue()) {
2235     if (mr.Value() == 0) {
2236       return mcgraph()->Int32Constant(0);
2237     } else if (mr.Value() == -1) {
2238       // The result is the negation of the left input.
2239       return graph()->NewNode(m->Int32Sub(), mcgraph()->Int32Constant(0), left);
2240     }
2241     return graph()->NewNode(m->Int32Div(), left, right, Control());
2242   }
2243 
2244   // asm.js semantics return 0 on divide or mod by zero.
2245   if (m->Int32DivIsSafe()) {
2246     // The hardware instruction does the right thing (e.g. arm).
2247     return graph()->NewNode(m->Int32Div(), left, right, graph()->start());
2248   }
2249 
2250   // Check denominator for zero.
2251   Diamond z(
2252       graph(), mcgraph()->common(),
2253       graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(0)),
2254       BranchHint::kFalse);
2255 
2256   // Check numerator for -1. (avoid minint / -1 case).
2257   Diamond n(
2258       graph(), mcgraph()->common(),
2259       graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(-1)),
2260       BranchHint::kFalse);
2261 
2262   Node* div = graph()->NewNode(m->Int32Div(), left, right, z.if_false);
2263   Node* neg =
2264       graph()->NewNode(m->Int32Sub(), mcgraph()->Int32Constant(0), left);
2265 
2266   return n.Phi(
2267       MachineRepresentation::kWord32, neg,
2268       z.Phi(MachineRepresentation::kWord32, mcgraph()->Int32Constant(0), div));
2269 }
2270 
BuildI32AsmjsRemS(Node * left,Node * right)2271 Node* WasmGraphBuilder::BuildI32AsmjsRemS(Node* left, Node* right) {
2272   CommonOperatorBuilder* c = mcgraph()->common();
2273   MachineOperatorBuilder* m = mcgraph()->machine();
2274   Node* const zero = mcgraph()->Int32Constant(0);
2275 
2276   Int32Matcher mr(right);
2277   if (mr.HasValue()) {
2278     if (mr.Value() == 0 || mr.Value() == -1) {
2279       return zero;
2280     }
2281     return graph()->NewNode(m->Int32Mod(), left, right, Control());
2282   }
2283 
2284   // General case for signed integer modulus, with optimization for (unknown)
2285   // power of 2 right hand side.
2286   //
2287   //   if 0 < right then
2288   //     msk = right - 1
2289   //     if right & msk != 0 then
2290   //       left % right
2291   //     else
2292   //       if left < 0 then
2293   //         -(-left & msk)
2294   //       else
2295   //         left & msk
2296   //   else
2297   //     if right < -1 then
2298   //       left % right
2299   //     else
2300   //       zero
2301   //
2302   // Note: We do not use the Diamond helper class here, because it really hurts
2303   // readability with nested diamonds.
2304   Node* const minus_one = mcgraph()->Int32Constant(-1);
2305 
2306   const Operator* const merge_op = c->Merge(2);
2307   const Operator* const phi_op = c->Phi(MachineRepresentation::kWord32, 2);
2308 
2309   Node* check0 = graph()->NewNode(m->Int32LessThan(), zero, right);
2310   Node* branch0 =
2311       graph()->NewNode(c->Branch(BranchHint::kTrue), check0, graph()->start());
2312 
2313   Node* if_true0 = graph()->NewNode(c->IfTrue(), branch0);
2314   Node* true0;
2315   {
2316     Node* msk = graph()->NewNode(m->Int32Add(), right, minus_one);
2317 
2318     Node* check1 = graph()->NewNode(m->Word32And(), right, msk);
2319     Node* branch1 = graph()->NewNode(c->Branch(), check1, if_true0);
2320 
2321     Node* if_true1 = graph()->NewNode(c->IfTrue(), branch1);
2322     Node* true1 = graph()->NewNode(m->Int32Mod(), left, right, if_true1);
2323 
2324     Node* if_false1 = graph()->NewNode(c->IfFalse(), branch1);
2325     Node* false1;
2326     {
2327       Node* check2 = graph()->NewNode(m->Int32LessThan(), left, zero);
2328       Node* branch2 =
2329           graph()->NewNode(c->Branch(BranchHint::kFalse), check2, if_false1);
2330 
2331       Node* if_true2 = graph()->NewNode(c->IfTrue(), branch2);
2332       Node* true2 = graph()->NewNode(
2333           m->Int32Sub(), zero,
2334           graph()->NewNode(m->Word32And(),
2335                            graph()->NewNode(m->Int32Sub(), zero, left), msk));
2336 
2337       Node* if_false2 = graph()->NewNode(c->IfFalse(), branch2);
2338       Node* false2 = graph()->NewNode(m->Word32And(), left, msk);
2339 
2340       if_false1 = graph()->NewNode(merge_op, if_true2, if_false2);
2341       false1 = graph()->NewNode(phi_op, true2, false2, if_false1);
2342     }
2343 
2344     if_true0 = graph()->NewNode(merge_op, if_true1, if_false1);
2345     true0 = graph()->NewNode(phi_op, true1, false1, if_true0);
2346   }
2347 
2348   Node* if_false0 = graph()->NewNode(c->IfFalse(), branch0);
2349   Node* false0;
2350   {
2351     Node* check1 = graph()->NewNode(m->Int32LessThan(), right, minus_one);
2352     Node* branch1 =
2353         graph()->NewNode(c->Branch(BranchHint::kTrue), check1, if_false0);
2354 
2355     Node* if_true1 = graph()->NewNode(c->IfTrue(), branch1);
2356     Node* true1 = graph()->NewNode(m->Int32Mod(), left, right, if_true1);
2357 
2358     Node* if_false1 = graph()->NewNode(c->IfFalse(), branch1);
2359     Node* false1 = zero;
2360 
2361     if_false0 = graph()->NewNode(merge_op, if_true1, if_false1);
2362     false0 = graph()->NewNode(phi_op, true1, false1, if_false0);
2363   }
2364 
2365   Node* merge0 = graph()->NewNode(merge_op, if_true0, if_false0);
2366   return graph()->NewNode(phi_op, true0, false0, merge0);
2367 }
2368 
BuildI32AsmjsDivU(Node * left,Node * right)2369 Node* WasmGraphBuilder::BuildI32AsmjsDivU(Node* left, Node* right) {
2370   MachineOperatorBuilder* m = mcgraph()->machine();
2371   // asm.js semantics return 0 on divide or mod by zero.
2372   if (m->Uint32DivIsSafe()) {
2373     // The hardware instruction does the right thing (e.g. arm).
2374     return graph()->NewNode(m->Uint32Div(), left, right, graph()->start());
2375   }
2376 
2377   // Explicit check for x % 0.
2378   Diamond z(
2379       graph(), mcgraph()->common(),
2380       graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(0)),
2381       BranchHint::kFalse);
2382 
2383   return z.Phi(MachineRepresentation::kWord32, mcgraph()->Int32Constant(0),
2384                graph()->NewNode(mcgraph()->machine()->Uint32Div(), left, right,
2385                                 z.if_false));
2386 }
2387 
BuildI32AsmjsRemU(Node * left,Node * right)2388 Node* WasmGraphBuilder::BuildI32AsmjsRemU(Node* left, Node* right) {
2389   MachineOperatorBuilder* m = mcgraph()->machine();
2390   // asm.js semantics return 0 on divide or mod by zero.
2391   // Explicit check for x % 0.
2392   Diamond z(
2393       graph(), mcgraph()->common(),
2394       graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(0)),
2395       BranchHint::kFalse);
2396 
2397   Node* rem = graph()->NewNode(mcgraph()->machine()->Uint32Mod(), left, right,
2398                                z.if_false);
2399   return z.Phi(MachineRepresentation::kWord32, mcgraph()->Int32Constant(0),
2400                rem);
2401 }
2402 
BuildI64DivS(Node * left,Node * right,wasm::WasmCodePosition position)2403 Node* WasmGraphBuilder::BuildI64DivS(Node* left, Node* right,
2404                                      wasm::WasmCodePosition position) {
2405   if (mcgraph()->machine()->Is32()) {
2406     return BuildDiv64Call(left, right, ExternalReference::wasm_int64_div(),
2407                           MachineType::Int64(), wasm::kTrapDivByZero, position);
2408   }
2409   ZeroCheck64(wasm::kTrapDivByZero, right, position);
2410   Node* before = Control();
2411   Node* denom_is_m1;
2412   Node* denom_is_not_m1;
2413   BranchExpectFalse(graph()->NewNode(mcgraph()->machine()->Word64Equal(), right,
2414                                      mcgraph()->Int64Constant(-1)),
2415                     &denom_is_m1, &denom_is_not_m1);
2416   SetControl(denom_is_m1);
2417   TrapIfEq64(wasm::kTrapDivUnrepresentable, left,
2418              std::numeric_limits<int64_t>::min(), position);
2419   if (Control() != denom_is_m1) {
2420     SetControl(graph()->NewNode(mcgraph()->common()->Merge(2), denom_is_not_m1,
2421                                 Control()));
2422   } else {
2423     SetControl(before);
2424   }
2425   return graph()->NewNode(mcgraph()->machine()->Int64Div(), left, right,
2426                           Control());
2427 }
2428 
BuildI64RemS(Node * left,Node * right,wasm::WasmCodePosition position)2429 Node* WasmGraphBuilder::BuildI64RemS(Node* left, Node* right,
2430                                      wasm::WasmCodePosition position) {
2431   if (mcgraph()->machine()->Is32()) {
2432     return BuildDiv64Call(left, right, ExternalReference::wasm_int64_mod(),
2433                           MachineType::Int64(), wasm::kTrapRemByZero, position);
2434   }
2435   ZeroCheck64(wasm::kTrapRemByZero, right, position);
2436   Diamond d(mcgraph()->graph(), mcgraph()->common(),
2437             graph()->NewNode(mcgraph()->machine()->Word64Equal(), right,
2438                              mcgraph()->Int64Constant(-1)));
2439 
2440   d.Chain(Control());
2441 
2442   Node* rem = graph()->NewNode(mcgraph()->machine()->Int64Mod(), left, right,
2443                                d.if_false);
2444 
2445   return d.Phi(MachineRepresentation::kWord64, mcgraph()->Int64Constant(0),
2446                rem);
2447 }
2448 
BuildI64DivU(Node * left,Node * right,wasm::WasmCodePosition position)2449 Node* WasmGraphBuilder::BuildI64DivU(Node* left, Node* right,
2450                                      wasm::WasmCodePosition position) {
2451   if (mcgraph()->machine()->Is32()) {
2452     return BuildDiv64Call(left, right, ExternalReference::wasm_uint64_div(),
2453                           MachineType::Int64(), wasm::kTrapDivByZero, position);
2454   }
2455   return graph()->NewNode(mcgraph()->machine()->Uint64Div(), left, right,
2456                           ZeroCheck64(wasm::kTrapDivByZero, right, position));
2457 }
BuildI64RemU(Node * left,Node * right,wasm::WasmCodePosition position)2458 Node* WasmGraphBuilder::BuildI64RemU(Node* left, Node* right,
2459                                      wasm::WasmCodePosition position) {
2460   if (mcgraph()->machine()->Is32()) {
2461     return BuildDiv64Call(left, right, ExternalReference::wasm_uint64_mod(),
2462                           MachineType::Int64(), wasm::kTrapRemByZero, position);
2463   }
2464   return graph()->NewNode(mcgraph()->machine()->Uint64Mod(), left, right,
2465                           ZeroCheck64(wasm::kTrapRemByZero, right, position));
2466 }
2467 
BuildDiv64Call(Node * left,Node * right,ExternalReference ref,MachineType result_type,wasm::TrapReason trap_zero,wasm::WasmCodePosition position)2468 Node* WasmGraphBuilder::BuildDiv64Call(Node* left, Node* right,
2469                                        ExternalReference ref,
2470                                        MachineType result_type,
2471                                        wasm::TrapReason trap_zero,
2472                                        wasm::WasmCodePosition position) {
2473   Node* stack_slot =
2474       graph()->NewNode(mcgraph()->machine()->StackSlot(2 * sizeof(double)));
2475 
2476   const Operator* store_op = mcgraph()->machine()->Store(
2477       StoreRepresentation(MachineRepresentation::kWord64, kNoWriteBarrier));
2478   SetEffect(graph()->NewNode(store_op, stack_slot, mcgraph()->Int32Constant(0),
2479                              left, Effect(), Control()));
2480   SetEffect(graph()->NewNode(store_op, stack_slot,
2481                              mcgraph()->Int32Constant(sizeof(double)), right,
2482                              Effect(), Control()));
2483 
2484   MachineType sig_types[] = {MachineType::Int32(), MachineType::Pointer()};
2485   MachineSignature sig(1, 1, sig_types);
2486 
2487   Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(ref));
2488   Node* call = BuildCCall(&sig, function, stack_slot);
2489 
2490   ZeroCheck32(trap_zero, call, position);
2491   TrapIfEq32(wasm::kTrapDivUnrepresentable, call, -1, position);
2492   return SetEffect(graph()->NewNode(mcgraph()->machine()->Load(result_type),
2493                                     stack_slot, mcgraph()->Int32Constant(0),
2494                                     Effect(), Control()));
2495 }
2496 
2497 template <typename... Args>
BuildCCall(MachineSignature * sig,Node * function,Args...args)2498 Node* WasmGraphBuilder::BuildCCall(MachineSignature* sig, Node* function,
2499                                    Args... args) {
2500   DCHECK_LE(sig->return_count(), 1);
2501   DCHECK_EQ(sizeof...(args), sig->parameter_count());
2502   Node* const call_args[] = {function, args..., Effect(), Control()};
2503 
2504   auto call_descriptor =
2505       Linkage::GetSimplifiedCDescriptor(mcgraph()->zone(), sig);
2506 
2507   const Operator* op = mcgraph()->common()->Call(call_descriptor);
2508   return SetEffect(graph()->NewNode(op, arraysize(call_args), call_args));
2509 }
2510 
BuildWasmCall(wasm::FunctionSig * sig,Node ** args,Node *** rets,wasm::WasmCodePosition position,Node * instance_node,UseRetpoline use_retpoline)2511 Node* WasmGraphBuilder::BuildWasmCall(wasm::FunctionSig* sig, Node** args,
2512                                       Node*** rets,
2513                                       wasm::WasmCodePosition position,
2514                                       Node* instance_node,
2515                                       UseRetpoline use_retpoline) {
2516   if (instance_node == nullptr) {
2517     DCHECK_NOT_NULL(instance_node_);
2518     instance_node = instance_node_.get();
2519   }
2520   SetNeedsStackCheck();
2521   const size_t params = sig->parameter_count();
2522   const size_t extra = 3;  // instance_node, effect, and control.
2523   const size_t count = 1 + params + extra;
2524 
2525   // Reallocate the buffer to make space for extra inputs.
2526   args = Realloc(args, 1 + params, count);
2527 
2528   // Make room for the instance_node parameter at index 1, just after code.
2529   memmove(&args[2], &args[1], params * sizeof(Node*));
2530   args[1] = instance_node;
2531 
2532   // Add effect and control inputs.
2533   args[params + 2] = Effect();
2534   args[params + 3] = Control();
2535 
2536   auto call_descriptor =
2537       GetWasmCallDescriptor(mcgraph()->zone(), sig, use_retpoline);
2538   const Operator* op = mcgraph()->common()->Call(call_descriptor);
2539   Node* call = SetEffect(graph()->NewNode(op, static_cast<int>(count), args));
2540   DCHECK(position == wasm::kNoCodePosition || position > 0);
2541   if (position > 0) SetSourcePosition(call, position);
2542 
2543   size_t ret_count = sig->return_count();
2544   if (ret_count == 0) return call;  // No return value.
2545 
2546   *rets = Buffer(ret_count);
2547   if (ret_count == 1) {
2548     // Only a single return value.
2549     (*rets)[0] = call;
2550   } else {
2551     // Create projections for all return values.
2552     for (size_t i = 0; i < ret_count; i++) {
2553       (*rets)[i] = graph()->NewNode(mcgraph()->common()->Projection(i), call,
2554                                     graph()->start());
2555     }
2556   }
2557   return call;
2558 }
2559 
BuildImportWasmCall(wasm::FunctionSig * sig,Node ** args,Node *** rets,wasm::WasmCodePosition position,int func_index)2560 Node* WasmGraphBuilder::BuildImportWasmCall(wasm::FunctionSig* sig, Node** args,
2561                                             Node*** rets,
2562                                             wasm::WasmCodePosition position,
2563                                             int func_index) {
2564   // Load the instance from the imported_instances array at a known offset.
2565   Node* imported_instances = LOAD_INSTANCE_FIELD(ImportedFunctionInstances,
2566                                                  MachineType::TaggedPointer());
2567   Node* instance_node = LOAD_FIXED_ARRAY_SLOT(imported_instances, func_index);
2568 
2569   // Load the target from the imported_targets array at a known offset.
2570   Node* imported_targets =
2571       LOAD_INSTANCE_FIELD(ImportedFunctionTargets, MachineType::Pointer());
2572   Node* target_node = SetEffect(graph()->NewNode(
2573       mcgraph()->machine()->Load(MachineType::Pointer()), imported_targets,
2574       mcgraph()->Int32Constant(func_index * kPointerSize), Effect(),
2575       Control()));
2576   args[0] = target_node;
2577   return BuildWasmCall(sig, args, rets, position, instance_node,
2578                        untrusted_code_mitigations_ ? kRetpoline : kNoRetpoline);
2579 }
2580 
BuildImportWasmCall(wasm::FunctionSig * sig,Node ** args,Node *** rets,wasm::WasmCodePosition position,Node * func_index)2581 Node* WasmGraphBuilder::BuildImportWasmCall(wasm::FunctionSig* sig, Node** args,
2582                                             Node*** rets,
2583                                             wasm::WasmCodePosition position,
2584                                             Node* func_index) {
2585   // Load the instance from the imported_instances array.
2586   Node* imported_instances = LOAD_INSTANCE_FIELD(ImportedFunctionInstances,
2587                                                  MachineType::TaggedPointer());
2588   // Access fixed array at {header_size - tag + func_index * kPointerSize}.
2589   Node* imported_instances_data =
2590       graph()->NewNode(mcgraph()->machine()->IntAdd(), imported_instances,
2591                        mcgraph()->IntPtrConstant(FixedArrayOffsetMinusTag(0)));
2592   Node* func_index_times_pointersize = graph()->NewNode(
2593       mcgraph()->machine()->IntMul(), Uint32ToUintptr(func_index),
2594       mcgraph()->Int32Constant(kPointerSize));
2595   Node* instance_node = SetEffect(
2596       graph()->NewNode(mcgraph()->machine()->Load(MachineType::TaggedPointer()),
2597                        imported_instances_data, func_index_times_pointersize,
2598                        Effect(), Control()));
2599 
2600   // Load the target from the imported_targets array at the offset of
2601   // {func_index}.
2602   Node* imported_targets =
2603       LOAD_INSTANCE_FIELD(ImportedFunctionTargets, MachineType::Pointer());
2604   Node* target_node = SetEffect(graph()->NewNode(
2605       mcgraph()->machine()->Load(MachineType::Pointer()), imported_targets,
2606       func_index_times_pointersize, Effect(), Control()));
2607   args[0] = target_node;
2608   return BuildWasmCall(sig, args, rets, position, instance_node,
2609                        untrusted_code_mitigations_ ? kRetpoline : kNoRetpoline);
2610 }
2611 
CallDirect(uint32_t index,Node ** args,Node *** rets,wasm::WasmCodePosition position)2612 Node* WasmGraphBuilder::CallDirect(uint32_t index, Node** args, Node*** rets,
2613                                    wasm::WasmCodePosition position) {
2614   DCHECK_NULL(args[0]);
2615   wasm::FunctionSig* sig = env_->module->functions[index].sig;
2616 
2617   if (env_ && index < env_->module->num_imported_functions) {
2618     // Call to an imported function.
2619     return BuildImportWasmCall(sig, args, rets, position, index);
2620   }
2621 
2622   // A direct call to a wasm function defined in this module.
2623   // Just encode the function index. This will be patched at instantiation.
2624   Address code = static_cast<Address>(index);
2625   args[0] = mcgraph()->RelocatableIntPtrConstant(code, RelocInfo::WASM_CALL);
2626 
2627   return BuildWasmCall(sig, args, rets, position, nullptr, kNoRetpoline);
2628 }
2629 
CallIndirect(uint32_t sig_index,Node ** args,Node *** rets,wasm::WasmCodePosition position)2630 Node* WasmGraphBuilder::CallIndirect(uint32_t sig_index, Node** args,
2631                                      Node*** rets,
2632                                      wasm::WasmCodePosition position) {
2633   DCHECK_NOT_NULL(args[0]);
2634   DCHECK_NOT_NULL(env_);
2635 
2636   // Assume only one table for now.
2637   wasm::FunctionSig* sig = env_->module->signatures[sig_index];
2638 
2639   Node* ift_size =
2640       LOAD_INSTANCE_FIELD(IndirectFunctionTableSize, MachineType::Uint32());
2641 
2642   MachineOperatorBuilder* machine = mcgraph()->machine();
2643   Node* key = args[0];
2644 
2645   // Bounds check against the table size.
2646   Node* in_bounds = graph()->NewNode(machine->Uint32LessThan(), key, ift_size);
2647   TrapIfFalse(wasm::kTrapFuncInvalid, in_bounds, position);
2648 
2649   // Mask the key to prevent SSCA.
2650   if (untrusted_code_mitigations_) {
2651     // mask = ((key - size) & ~key) >> 31
2652     Node* neg_key =
2653         graph()->NewNode(machine->Word32Xor(), key, Int32Constant(-1));
2654     Node* masked_diff = graph()->NewNode(
2655         machine->Word32And(),
2656         graph()->NewNode(machine->Int32Sub(), key, ift_size), neg_key);
2657     Node* mask =
2658         graph()->NewNode(machine->Word32Sar(), masked_diff, Int32Constant(31));
2659     key = graph()->NewNode(machine->Word32And(), key, mask);
2660   }
2661 
2662   // Load signature from the table and check.
2663   Node* ift_sig_ids =
2664       LOAD_INSTANCE_FIELD(IndirectFunctionTableSigIds, MachineType::Pointer());
2665 
2666   int32_t expected_sig_id = env_->module->signature_ids[sig_index];
2667   Node* scaled_key = Uint32ToUintptr(
2668       graph()->NewNode(machine->Word32Shl(), key, Int32Constant(2)));
2669 
2670   Node* loaded_sig =
2671       SetEffect(graph()->NewNode(machine->Load(MachineType::Int32()),
2672                                  ift_sig_ids, scaled_key, Effect(), Control()));
2673   Node* sig_match = graph()->NewNode(machine->WordEqual(), loaded_sig,
2674                                      Int32Constant(expected_sig_id));
2675 
2676   TrapIfFalse(wasm::kTrapFuncSigMismatch, sig_match, position);
2677 
2678   Node* ift_targets =
2679       LOAD_INSTANCE_FIELD(IndirectFunctionTableTargets, MachineType::Pointer());
2680   Node* ift_instances = LOAD_INSTANCE_FIELD(IndirectFunctionTableInstances,
2681                                             MachineType::TaggedPointer());
2682 
2683   scaled_key = graph()->NewNode(machine->Word32Shl(), key,
2684                                 Int32Constant(kPointerSizeLog2));
2685 
2686   Node* target =
2687       SetEffect(graph()->NewNode(machine->Load(MachineType::Pointer()),
2688                                  ift_targets, scaled_key, Effect(), Control()));
2689 
2690   auto access = AccessBuilder::ForFixedArrayElement();
2691   Node* target_instance = SetEffect(graph()->NewNode(
2692       machine->Load(MachineType::TaggedPointer()),
2693       graph()->NewNode(machine->IntAdd(), ift_instances, scaled_key),
2694       Int32Constant(access.header_size - access.tag()), Effect(), Control()));
2695 
2696   args[0] = target;
2697 
2698   return BuildWasmCall(sig, args, rets, position, target_instance,
2699                        untrusted_code_mitigations_ ? kRetpoline : kNoRetpoline);
2700 }
2701 
BuildI32Rol(Node * left,Node * right)2702 Node* WasmGraphBuilder::BuildI32Rol(Node* left, Node* right) {
2703   // Implement Rol by Ror since TurboFan does not have Rol opcode.
2704   // TODO(weiliang): support Word32Rol opcode in TurboFan.
2705   Int32Matcher m(right);
2706   if (m.HasValue()) {
2707     return Binop(wasm::kExprI32Ror, left,
2708                  mcgraph()->Int32Constant(32 - m.Value()));
2709   } else {
2710     return Binop(wasm::kExprI32Ror, left,
2711                  Binop(wasm::kExprI32Sub, mcgraph()->Int32Constant(32), right));
2712   }
2713 }
2714 
BuildI64Rol(Node * left,Node * right)2715 Node* WasmGraphBuilder::BuildI64Rol(Node* left, Node* right) {
2716   // Implement Rol by Ror since TurboFan does not have Rol opcode.
2717   // TODO(weiliang): support Word64Rol opcode in TurboFan.
2718   Int64Matcher m(right);
2719   if (m.HasValue()) {
2720     return Binop(wasm::kExprI64Ror, left,
2721                  mcgraph()->Int64Constant(64 - m.Value()));
2722   } else {
2723     return Binop(wasm::kExprI64Ror, left,
2724                  Binop(wasm::kExprI64Sub, mcgraph()->Int64Constant(64), right));
2725   }
2726 }
2727 
Invert(Node * node)2728 Node* WasmGraphBuilder::Invert(Node* node) {
2729   return Unop(wasm::kExprI32Eqz, node);
2730 }
2731 
CanCover(Node * value,IrOpcode::Value opcode)2732 bool CanCover(Node* value, IrOpcode::Value opcode) {
2733   if (value->opcode() != opcode) return false;
2734   bool first = true;
2735   for (Edge const edge : value->use_edges()) {
2736     if (NodeProperties::IsControlEdge(edge)) continue;
2737     if (NodeProperties::IsEffectEdge(edge)) continue;
2738     DCHECK(NodeProperties::IsValueEdge(edge));
2739     if (!first) return false;
2740     first = false;
2741   }
2742   return true;
2743 }
2744 
BuildChangeInt32ToIntPtr(Node * value)2745 Node* WasmGraphBuilder::BuildChangeInt32ToIntPtr(Node* value) {
2746   if (mcgraph()->machine()->Is64()) {
2747     value = graph()->NewNode(mcgraph()->machine()->ChangeInt32ToInt64(), value);
2748   }
2749   return value;
2750 }
2751 
BuildChangeInt32ToSmi(Node * value)2752 Node* WasmGraphBuilder::BuildChangeInt32ToSmi(Node* value) {
2753   value = BuildChangeInt32ToIntPtr(value);
2754   return graph()->NewNode(mcgraph()->machine()->WordShl(), value,
2755                           BuildSmiShiftBitsConstant());
2756 }
2757 
BuildChangeUint31ToSmi(Node * value)2758 Node* WasmGraphBuilder::BuildChangeUint31ToSmi(Node* value) {
2759   return graph()->NewNode(mcgraph()->machine()->WordShl(),
2760                           Uint32ToUintptr(value), BuildSmiShiftBitsConstant());
2761 }
2762 
BuildSmiShiftBitsConstant()2763 Node* WasmGraphBuilder::BuildSmiShiftBitsConstant() {
2764   return mcgraph()->IntPtrConstant(kSmiShiftSize + kSmiTagSize);
2765 }
2766 
BuildChangeSmiToInt32(Node * value)2767 Node* WasmGraphBuilder::BuildChangeSmiToInt32(Node* value) {
2768   value = graph()->NewNode(mcgraph()->machine()->WordSar(), value,
2769                            BuildSmiShiftBitsConstant());
2770   if (mcgraph()->machine()->Is64()) {
2771     value =
2772         graph()->NewNode(mcgraph()->machine()->TruncateInt64ToInt32(), value);
2773   }
2774   return value;
2775 }
2776 
InitInstanceCache(WasmInstanceCacheNodes * instance_cache)2777 void WasmGraphBuilder::InitInstanceCache(
2778     WasmInstanceCacheNodes* instance_cache) {
2779   DCHECK_NOT_NULL(instance_node_);
2780 
2781   // Load the memory start.
2782   instance_cache->mem_start = SetEffect(graph()->NewNode(
2783       mcgraph()->machine()->Load(MachineType::UintPtr()), instance_node_.get(),
2784       mcgraph()->Int32Constant(WASM_INSTANCE_OBJECT_OFFSET(MemoryStart)),
2785       Effect(), Control()));
2786 
2787   // Load the memory size.
2788   instance_cache->mem_size = SetEffect(graph()->NewNode(
2789       mcgraph()->machine()->Load(MachineType::UintPtr()), instance_node_.get(),
2790       mcgraph()->Int32Constant(WASM_INSTANCE_OBJECT_OFFSET(MemorySize)),
2791       Effect(), Control()));
2792 
2793   if (untrusted_code_mitigations_) {
2794     // Load the memory mask.
2795     instance_cache->mem_mask = SetEffect(graph()->NewNode(
2796         mcgraph()->machine()->Load(MachineType::UintPtr()),
2797         instance_node_.get(),
2798         mcgraph()->Int32Constant(WASM_INSTANCE_OBJECT_OFFSET(MemoryMask)),
2799         Effect(), Control()));
2800   } else {
2801     // Explicitly set to nullptr to ensure a SEGV when we try to use it.
2802     instance_cache->mem_mask = nullptr;
2803   }
2804 }
2805 
PrepareInstanceCacheForLoop(WasmInstanceCacheNodes * instance_cache,Node * control)2806 void WasmGraphBuilder::PrepareInstanceCacheForLoop(
2807     WasmInstanceCacheNodes* instance_cache, Node* control) {
2808 #define INTRODUCE_PHI(field, rep)                                            \
2809   instance_cache->field = graph()->NewNode(mcgraph()->common()->Phi(rep, 1), \
2810                                            instance_cache->field, control);
2811 
2812   INTRODUCE_PHI(mem_start, MachineType::PointerRepresentation());
2813   INTRODUCE_PHI(mem_size, MachineRepresentation::kWord32);
2814   if (untrusted_code_mitigations_) {
2815     INTRODUCE_PHI(mem_mask, MachineRepresentation::kWord32);
2816   }
2817 
2818 #undef INTRODUCE_PHI
2819 }
2820 
NewInstanceCacheMerge(WasmInstanceCacheNodes * to,WasmInstanceCacheNodes * from,Node * merge)2821 void WasmGraphBuilder::NewInstanceCacheMerge(WasmInstanceCacheNodes* to,
2822                                              WasmInstanceCacheNodes* from,
2823                                              Node* merge) {
2824 #define INTRODUCE_PHI(field, rep)                                            \
2825   if (to->field != from->field) {                                            \
2826     Node* vals[] = {to->field, from->field, merge};                          \
2827     to->field = graph()->NewNode(mcgraph()->common()->Phi(rep, 2), 3, vals); \
2828   }
2829 
2830   INTRODUCE_PHI(mem_start, MachineType::PointerRepresentation());
2831   INTRODUCE_PHI(mem_size, MachineRepresentation::kWord32);
2832   if (untrusted_code_mitigations_) {
2833     INTRODUCE_PHI(mem_mask, MachineRepresentation::kWord32);
2834   }
2835 
2836 #undef INTRODUCE_PHI
2837 }
2838 
MergeInstanceCacheInto(WasmInstanceCacheNodes * to,WasmInstanceCacheNodes * from,Node * merge)2839 void WasmGraphBuilder::MergeInstanceCacheInto(WasmInstanceCacheNodes* to,
2840                                               WasmInstanceCacheNodes* from,
2841                                               Node* merge) {
2842   to->mem_size = CreateOrMergeIntoPhi(MachineType::PointerRepresentation(),
2843                                       merge, to->mem_size, from->mem_size);
2844   to->mem_start = CreateOrMergeIntoPhi(MachineType::PointerRepresentation(),
2845                                        merge, to->mem_start, from->mem_start);
2846   if (untrusted_code_mitigations_) {
2847     to->mem_mask = CreateOrMergeIntoPhi(MachineType::PointerRepresentation(),
2848                                         merge, to->mem_mask, from->mem_mask);
2849   }
2850 }
2851 
CreateOrMergeIntoPhi(MachineRepresentation rep,Node * merge,Node * tnode,Node * fnode)2852 Node* WasmGraphBuilder::CreateOrMergeIntoPhi(MachineRepresentation rep,
2853                                              Node* merge, Node* tnode,
2854                                              Node* fnode) {
2855   if (IsPhiWithMerge(tnode, merge)) {
2856     AppendToPhi(tnode, fnode);
2857   } else if (tnode != fnode) {
2858     uint32_t count = merge->InputCount();
2859     // + 1 for the merge node.
2860     Node** vals = Buffer(count + 1);
2861     for (uint32_t j = 0; j < count - 1; j++) vals[j] = tnode;
2862     vals[count - 1] = fnode;
2863     vals[count] = merge;
2864     return graph()->NewNode(mcgraph()->common()->Phi(rep, count), count + 1,
2865                             vals);
2866   }
2867   return tnode;
2868 }
2869 
CreateOrMergeIntoEffectPhi(Node * merge,Node * tnode,Node * fnode)2870 Node* WasmGraphBuilder::CreateOrMergeIntoEffectPhi(Node* merge, Node* tnode,
2871                                                    Node* fnode) {
2872   if (IsPhiWithMerge(tnode, merge)) {
2873     AppendToPhi(tnode, fnode);
2874   } else if (tnode != fnode) {
2875     uint32_t count = merge->InputCount();
2876     Node** effects = Buffer(count);
2877     for (uint32_t j = 0; j < count - 1; j++) {
2878       effects[j] = tnode;
2879     }
2880     effects[count - 1] = fnode;
2881     tnode = EffectPhi(count, effects, merge);
2882   }
2883   return tnode;
2884 }
2885 
GetGlobalBaseAndOffset(MachineType mem_type,const wasm::WasmGlobal & global,Node ** base_node,Node ** offset_node)2886 void WasmGraphBuilder::GetGlobalBaseAndOffset(MachineType mem_type,
2887                                               const wasm::WasmGlobal& global,
2888                                               Node** base_node,
2889                                               Node** offset_node) {
2890   DCHECK_NOT_NULL(instance_node_);
2891   if (global.mutability && global.imported) {
2892     if (imported_mutable_globals_ == nullptr) {
2893       // Load imported_mutable_globals_ from the instance object at runtime.
2894       imported_mutable_globals_ = graph()->NewNode(
2895           mcgraph()->machine()->Load(MachineType::UintPtr()),
2896           instance_node_.get(),
2897           mcgraph()->Int32Constant(
2898               WASM_INSTANCE_OBJECT_OFFSET(ImportedMutableGlobals)),
2899           graph()->start(), graph()->start());
2900     }
2901     *base_node = SetEffect(graph()->NewNode(
2902         mcgraph()->machine()->Load(MachineType::UintPtr()),
2903         imported_mutable_globals_.get(),
2904         mcgraph()->Int32Constant(global.index * sizeof(Address)), Effect(),
2905         Control()));
2906     *offset_node = mcgraph()->Int32Constant(0);
2907   } else {
2908     if (globals_start_ == nullptr) {
2909       // Load globals_start from the instance object at runtime.
2910       // TODO(wasm): we currently generate only one load of the {globals_start}
2911       // start per graph, which means it can be placed anywhere by the
2912       // scheduler. This is legal because the globals_start should never change.
2913       // However, in some cases (e.g. if the instance object is already in a
2914       // register), it is slightly more efficient to reload this value from the
2915       // instance object. Since this depends on register allocation, it is not
2916       // possible to express in the graph, and would essentially constitute a
2917       // "mem2reg" optimization in TurboFan.
2918       globals_start_ = graph()->NewNode(
2919           mcgraph()->machine()->Load(MachineType::UintPtr()),
2920           instance_node_.get(),
2921           mcgraph()->Int32Constant(WASM_INSTANCE_OBJECT_OFFSET(GlobalsStart)),
2922           graph()->start(), graph()->start());
2923     }
2924     *base_node = globals_start_.get();
2925     *offset_node = mcgraph()->Int32Constant(global.offset);
2926 
2927     if (mem_type == MachineType::Simd128() && global.offset != 0) {
2928       // TODO(titzer,bbudge): code generation for SIMD memory offsets is broken.
2929       *base_node = graph()->NewNode(mcgraph()->machine()->IntAdd(), *base_node,
2930                                     *offset_node);
2931       *offset_node = mcgraph()->Int32Constant(0);
2932     }
2933   }
2934 }
2935 
MemBuffer(uint32_t offset)2936 Node* WasmGraphBuilder::MemBuffer(uint32_t offset) {
2937   DCHECK_NOT_NULL(instance_cache_);
2938   Node* mem_start = instance_cache_->mem_start;
2939   DCHECK_NOT_NULL(mem_start);
2940   if (offset == 0) return mem_start;
2941   return graph()->NewNode(mcgraph()->machine()->IntAdd(), mem_start,
2942                           mcgraph()->IntPtrConstant(offset));
2943 }
2944 
CurrentMemoryPages()2945 Node* WasmGraphBuilder::CurrentMemoryPages() {
2946   // CurrentMemoryPages can not be called from asm.js.
2947   DCHECK_EQ(wasm::kWasmOrigin, env_->module->origin);
2948   DCHECK_NOT_NULL(instance_cache_);
2949   Node* mem_size = instance_cache_->mem_size;
2950   DCHECK_NOT_NULL(mem_size);
2951   Node* result =
2952       graph()->NewNode(mcgraph()->machine()->WordShr(), mem_size,
2953                        mcgraph()->Int32Constant(wasm::kWasmPageSizeLog2));
2954   if (mcgraph()->machine()->Is64()) {
2955     result =
2956         graph()->NewNode(mcgraph()->machine()->TruncateInt64ToInt32(), result);
2957   }
2958   return result;
2959 }
2960 
2961 // Only call this function for code which is not reused across instantiations,
2962 // as we do not patch the embedded js_context.
BuildCallToRuntimeWithContext(Runtime::FunctionId f,Node * js_context,Node ** parameters,int parameter_count)2963 Node* WasmGraphBuilder::BuildCallToRuntimeWithContext(Runtime::FunctionId f,
2964                                                       Node* js_context,
2965                                                       Node** parameters,
2966                                                       int parameter_count) {
2967   const Runtime::Function* fun = Runtime::FunctionForId(f);
2968   auto call_descriptor = Linkage::GetRuntimeCallDescriptor(
2969       mcgraph()->zone(), f, fun->nargs, Operator::kNoProperties,
2970       CallDescriptor::kNoFlags);
2971   // The CEntryStub is loaded from the instance_node so that generated code is
2972   // Isolate independent. At the moment this is only done for CEntryStub(1).
2973   DCHECK_EQ(1, fun->result_size);
2974   Node* centry_stub =
2975       LOAD_INSTANCE_FIELD(CEntryStub, MachineType::TaggedPointer());
2976   // At the moment we only allow 4 parameters. If more parameters are needed,
2977   // increase this constant accordingly.
2978   static const int kMaxParams = 4;
2979   DCHECK_GE(kMaxParams, parameter_count);
2980   Node* inputs[kMaxParams + 6];
2981   int count = 0;
2982   inputs[count++] = centry_stub;
2983   for (int i = 0; i < parameter_count; i++) {
2984     inputs[count++] = parameters[i];
2985   }
2986   inputs[count++] =
2987       mcgraph()->ExternalConstant(ExternalReference::Create(f));  // ref
2988   inputs[count++] = mcgraph()->Int32Constant(fun->nargs);         // arity
2989   inputs[count++] = js_context;                                   // js_context
2990   inputs[count++] = Effect();
2991   inputs[count++] = Control();
2992 
2993   return SetEffect(mcgraph()->graph()->NewNode(
2994       mcgraph()->common()->Call(call_descriptor), count, inputs));
2995 }
2996 
BuildCallToRuntime(Runtime::FunctionId f,Node ** parameters,int parameter_count)2997 Node* WasmGraphBuilder::BuildCallToRuntime(Runtime::FunctionId f,
2998                                            Node** parameters,
2999                                            int parameter_count) {
3000   return BuildCallToRuntimeWithContext(f, NoContextConstant(), parameters,
3001                                        parameter_count);
3002 }
3003 
GetGlobal(uint32_t index)3004 Node* WasmGraphBuilder::GetGlobal(uint32_t index) {
3005   MachineType mem_type =
3006       wasm::ValueTypes::MachineTypeFor(env_->module->globals[index].type);
3007   Node* base = nullptr;
3008   Node* offset = nullptr;
3009   GetGlobalBaseAndOffset(mem_type, env_->module->globals[index], &base,
3010                          &offset);
3011   Node* load = SetEffect(graph()->NewNode(mcgraph()->machine()->Load(mem_type),
3012                                           base, offset, Effect(), Control()));
3013 #if defined(V8_TARGET_BIG_ENDIAN)
3014   load = BuildChangeEndiannessLoad(load, mem_type,
3015                                    env_->module->globals[index].type);
3016 #endif
3017   return load;
3018 }
3019 
SetGlobal(uint32_t index,Node * val)3020 Node* WasmGraphBuilder::SetGlobal(uint32_t index, Node* val) {
3021   MachineType mem_type =
3022       wasm::ValueTypes::MachineTypeFor(env_->module->globals[index].type);
3023   Node* base = nullptr;
3024   Node* offset = nullptr;
3025   GetGlobalBaseAndOffset(mem_type, env_->module->globals[index], &base,
3026                          &offset);
3027   const Operator* op = mcgraph()->machine()->Store(
3028       StoreRepresentation(mem_type.representation(), kNoWriteBarrier));
3029 #if defined(V8_TARGET_BIG_ENDIAN)
3030   val = BuildChangeEndiannessStore(val, mem_type.representation(),
3031                                    env_->module->globals[index].type);
3032 #endif
3033   return SetEffect(
3034       graph()->NewNode(op, base, offset, val, Effect(), Control()));
3035 }
3036 
CheckBoundsAndAlignment(uint8_t access_size,Node * index,uint32_t offset,wasm::WasmCodePosition position)3037 Node* WasmGraphBuilder::CheckBoundsAndAlignment(
3038     uint8_t access_size, Node* index, uint32_t offset,
3039     wasm::WasmCodePosition position) {
3040   // Atomic operations access the memory, need to be bound checked till
3041   // TrapHandlers are enabled on atomic operations
3042   index =
3043       BoundsCheckMem(access_size, index, offset, position, kNeedsBoundsCheck);
3044   Node* effective_address =
3045       graph()->NewNode(mcgraph()->machine()->IntAdd(), MemBuffer(offset),
3046                        Uint32ToUintptr(index));
3047   // Unlike regular memory accesses, unaligned memory accesses for atomic
3048   // operations should trap
3049   // Access sizes are in powers of two, calculate mod without using division
3050   Node* cond =
3051       graph()->NewNode(mcgraph()->machine()->WordAnd(), effective_address,
3052                        IntPtrConstant(access_size - 1));
3053   TrapIfFalse(wasm::kTrapUnalignedAccess,
3054               graph()->NewNode(mcgraph()->machine()->Word32Equal(), cond,
3055                                mcgraph()->Int32Constant(0)),
3056               position);
3057   return index;
3058 }
3059 
BoundsCheckMem(uint8_t access_size,Node * index,uint32_t offset,wasm::WasmCodePosition position,EnforceBoundsCheck enforce_check)3060 Node* WasmGraphBuilder::BoundsCheckMem(uint8_t access_size, Node* index,
3061                                        uint32_t offset,
3062                                        wasm::WasmCodePosition position,
3063                                        EnforceBoundsCheck enforce_check) {
3064   DCHECK_LE(1, access_size);
3065   index = Uint32ToUintptr(index);
3066   if (FLAG_wasm_no_bounds_checks) return index;
3067 
3068   if (use_trap_handler() && enforce_check == kCanOmitBoundsCheck) {
3069     return index;
3070   }
3071 
3072   const bool statically_oob = access_size > env_->max_memory_size ||
3073                               offset > env_->max_memory_size - access_size;
3074   if (statically_oob) {
3075     // The access will be out of bounds, even for the largest memory.
3076     TrapIfEq32(wasm::kTrapMemOutOfBounds, Int32Constant(0), 0, position);
3077     return mcgraph()->IntPtrConstant(0);
3078   }
3079   uint64_t end_offset = uint64_t{offset} + access_size - 1u;
3080   Node* end_offset_node = IntPtrConstant(end_offset);
3081 
3082   // The accessed memory is [index + offset, index + end_offset].
3083   // Check that the last read byte (at {index + end_offset}) is in bounds.
3084   // 1) Check that {end_offset < mem_size}. This also ensures that we can safely
3085   //    compute {effective_size} as {mem_size - end_offset)}.
3086   //    {effective_size} is >= 1 if condition 1) holds.
3087   // 2) Check that {index + end_offset < mem_size} by
3088   //    - computing {effective_size} as {mem_size - end_offset} and
3089   //    - checking that {index < effective_size}.
3090 
3091   auto m = mcgraph()->machine();
3092   Node* mem_size = instance_cache_->mem_size;
3093   if (end_offset >= env_->min_memory_size) {
3094     // The end offset is larger than the smallest memory.
3095     // Dynamically check the end offset against the dynamic memory size.
3096     Node* cond = graph()->NewNode(m->UintLessThan(), end_offset_node, mem_size);
3097     TrapIfFalse(wasm::kTrapMemOutOfBounds, cond, position);
3098   } else {
3099     // The end offset is smaller than the smallest memory, so only one check is
3100     // required. Check to see if the index is also a constant.
3101     UintPtrMatcher match(index);
3102     if (match.HasValue()) {
3103       uintptr_t index_val = match.Value();
3104       if (index_val < env_->min_memory_size - end_offset) {
3105         // The input index is a constant and everything is statically within
3106         // bounds of the smallest possible memory.
3107         return index;
3108       }
3109     }
3110   }
3111 
3112   // This produces a positive number, since {end_offset < min_size <= mem_size}.
3113   Node* effective_size =
3114       graph()->NewNode(m->IntSub(), mem_size, end_offset_node);
3115 
3116   // Introduce the actual bounds check.
3117   Node* cond = graph()->NewNode(m->UintLessThan(), index, effective_size);
3118   TrapIfFalse(wasm::kTrapMemOutOfBounds, cond, position);
3119 
3120   if (untrusted_code_mitigations_) {
3121     // In the fallthrough case, condition the index with the memory mask.
3122     Node* mem_mask = instance_cache_->mem_mask;
3123     DCHECK_NOT_NULL(mem_mask);
3124     index = graph()->NewNode(m->WordAnd(), index, mem_mask);
3125   }
3126   return index;
3127 }
3128 
GetSafeLoadOperator(int offset,wasm::ValueType type)3129 const Operator* WasmGraphBuilder::GetSafeLoadOperator(int offset,
3130                                                       wasm::ValueType type) {
3131   int alignment = offset % (wasm::ValueTypes::ElementSizeInBytes(type));
3132   MachineType mach_type = wasm::ValueTypes::MachineTypeFor(type);
3133   if (alignment == 0 || mcgraph()->machine()->UnalignedLoadSupported(
3134                             wasm::ValueTypes::MachineRepresentationFor(type))) {
3135     return mcgraph()->machine()->Load(mach_type);
3136   }
3137   return mcgraph()->machine()->UnalignedLoad(mach_type);
3138 }
3139 
GetSafeStoreOperator(int offset,wasm::ValueType type)3140 const Operator* WasmGraphBuilder::GetSafeStoreOperator(int offset,
3141                                                        wasm::ValueType type) {
3142   int alignment = offset % (wasm::ValueTypes::ElementSizeInBytes(type));
3143   MachineRepresentation rep = wasm::ValueTypes::MachineRepresentationFor(type);
3144   if (alignment == 0 || mcgraph()->machine()->UnalignedStoreSupported(rep)) {
3145     StoreRepresentation store_rep(rep, WriteBarrierKind::kNoWriteBarrier);
3146     return mcgraph()->machine()->Store(store_rep);
3147   }
3148   UnalignedStoreRepresentation store_rep(rep);
3149   return mcgraph()->machine()->UnalignedStore(store_rep);
3150 }
3151 
TraceMemoryOperation(bool is_store,MachineRepresentation rep,Node * index,uint32_t offset,wasm::WasmCodePosition position)3152 Node* WasmGraphBuilder::TraceMemoryOperation(bool is_store,
3153                                              MachineRepresentation rep,
3154                                              Node* index, uint32_t offset,
3155                                              wasm::WasmCodePosition position) {
3156   int kAlign = 4;  // Ensure that the LSB is 0, such that this looks like a Smi.
3157   Node* info = graph()->NewNode(
3158       mcgraph()->machine()->StackSlot(sizeof(wasm::MemoryTracingInfo), kAlign));
3159 
3160   Node* address = graph()->NewNode(mcgraph()->machine()->Int32Add(),
3161                                    Int32Constant(offset), index);
3162   auto store = [&](int offset, MachineRepresentation rep, Node* data) {
3163     SetEffect(graph()->NewNode(
3164         mcgraph()->machine()->Store(StoreRepresentation(rep, kNoWriteBarrier)),
3165         info, mcgraph()->Int32Constant(offset), data, Effect(), Control()));
3166   };
3167   // Store address, is_store, and mem_rep.
3168   store(offsetof(wasm::MemoryTracingInfo, address),
3169         MachineRepresentation::kWord32, address);
3170   store(offsetof(wasm::MemoryTracingInfo, is_store),
3171         MachineRepresentation::kWord8,
3172         mcgraph()->Int32Constant(is_store ? 1 : 0));
3173   store(offsetof(wasm::MemoryTracingInfo, mem_rep),
3174         MachineRepresentation::kWord8,
3175         mcgraph()->Int32Constant(static_cast<int>(rep)));
3176 
3177   Node* call = BuildCallToRuntime(Runtime::kWasmTraceMemory, &info, 1);
3178   SetSourcePosition(call, position);
3179   return call;
3180 }
3181 
LoadMem(wasm::ValueType type,MachineType memtype,Node * index,uint32_t offset,uint32_t alignment,wasm::WasmCodePosition position)3182 Node* WasmGraphBuilder::LoadMem(wasm::ValueType type, MachineType memtype,
3183                                 Node* index, uint32_t offset,
3184                                 uint32_t alignment,
3185                                 wasm::WasmCodePosition position) {
3186   Node* load;
3187 
3188   // Wasm semantics throw on OOB. Introduce explicit bounds check and
3189   // conditioning when not using the trap handler.
3190   index = BoundsCheckMem(wasm::ValueTypes::MemSize(memtype), index, offset,
3191                          position, kCanOmitBoundsCheck);
3192 
3193   if (memtype.representation() == MachineRepresentation::kWord8 ||
3194       mcgraph()->machine()->UnalignedLoadSupported(memtype.representation())) {
3195     if (use_trap_handler()) {
3196       load = graph()->NewNode(mcgraph()->machine()->ProtectedLoad(memtype),
3197                               MemBuffer(offset), index, Effect(), Control());
3198       SetSourcePosition(load, position);
3199     } else {
3200       load = graph()->NewNode(mcgraph()->machine()->Load(memtype),
3201                               MemBuffer(offset), index, Effect(), Control());
3202     }
3203   } else {
3204     // TODO(eholk): Support unaligned loads with trap handlers.
3205     DCHECK(!use_trap_handler());
3206     load = graph()->NewNode(mcgraph()->machine()->UnalignedLoad(memtype),
3207                             MemBuffer(offset), index, Effect(), Control());
3208   }
3209 
3210   SetEffect(load);
3211 
3212 #if defined(V8_TARGET_BIG_ENDIAN)
3213   load = BuildChangeEndiannessLoad(load, memtype, type);
3214 #endif
3215 
3216   if (type == wasm::kWasmI64 &&
3217       ElementSizeInBytes(memtype.representation()) < 8) {
3218     // TODO(titzer): TF zeroes the upper bits of 64-bit loads for subword sizes.
3219     if (memtype.IsSigned()) {
3220       // sign extend
3221       load = graph()->NewNode(mcgraph()->machine()->ChangeInt32ToInt64(), load);
3222     } else {
3223       // zero extend
3224       load =
3225           graph()->NewNode(mcgraph()->machine()->ChangeUint32ToUint64(), load);
3226     }
3227   }
3228 
3229   if (FLAG_wasm_trace_memory) {
3230     TraceMemoryOperation(false, memtype.representation(), index, offset,
3231                          position);
3232   }
3233 
3234   return load;
3235 }
3236 
StoreMem(MachineRepresentation mem_rep,Node * index,uint32_t offset,uint32_t alignment,Node * val,wasm::WasmCodePosition position,wasm::ValueType type)3237 Node* WasmGraphBuilder::StoreMem(MachineRepresentation mem_rep, Node* index,
3238                                  uint32_t offset, uint32_t alignment, Node* val,
3239                                  wasm::WasmCodePosition position,
3240                                  wasm::ValueType type) {
3241   Node* store;
3242 
3243   index = BoundsCheckMem(i::ElementSizeInBytes(mem_rep), index, offset,
3244                          position, kCanOmitBoundsCheck);
3245 
3246 #if defined(V8_TARGET_BIG_ENDIAN)
3247   val = BuildChangeEndiannessStore(val, mem_rep, type);
3248 #endif
3249 
3250   if (mem_rep == MachineRepresentation::kWord8 ||
3251       mcgraph()->machine()->UnalignedStoreSupported(mem_rep)) {
3252     if (use_trap_handler()) {
3253       store =
3254           graph()->NewNode(mcgraph()->machine()->ProtectedStore(mem_rep),
3255                            MemBuffer(offset), index, val, Effect(), Control());
3256       SetSourcePosition(store, position);
3257     } else {
3258       StoreRepresentation rep(mem_rep, kNoWriteBarrier);
3259       store =
3260           graph()->NewNode(mcgraph()->machine()->Store(rep), MemBuffer(offset),
3261                            index, val, Effect(), Control());
3262     }
3263   } else {
3264     // TODO(eholk): Support unaligned stores with trap handlers.
3265     DCHECK(!use_trap_handler());
3266     UnalignedStoreRepresentation rep(mem_rep);
3267     store =
3268         graph()->NewNode(mcgraph()->machine()->UnalignedStore(rep),
3269                          MemBuffer(offset), index, val, Effect(), Control());
3270   }
3271 
3272   SetEffect(store);
3273 
3274   if (FLAG_wasm_trace_memory) {
3275     TraceMemoryOperation(true, mem_rep, index, offset, position);
3276   }
3277 
3278   return store;
3279 }
3280 
3281 namespace {
GetAsmJsOOBValue(MachineRepresentation rep,MachineGraph * mcgraph)3282 Node* GetAsmJsOOBValue(MachineRepresentation rep, MachineGraph* mcgraph) {
3283   switch (rep) {
3284     case MachineRepresentation::kWord8:
3285     case MachineRepresentation::kWord16:
3286     case MachineRepresentation::kWord32:
3287       return mcgraph->Int32Constant(0);
3288     case MachineRepresentation::kWord64:
3289       return mcgraph->Int64Constant(0);
3290     case MachineRepresentation::kFloat32:
3291       return mcgraph->Float32Constant(std::numeric_limits<float>::quiet_NaN());
3292     case MachineRepresentation::kFloat64:
3293       return mcgraph->Float64Constant(std::numeric_limits<double>::quiet_NaN());
3294     default:
3295       UNREACHABLE();
3296   }
3297 }
3298 }  // namespace
3299 
BuildAsmjsLoadMem(MachineType type,Node * index)3300 Node* WasmGraphBuilder::BuildAsmjsLoadMem(MachineType type, Node* index) {
3301   DCHECK_NOT_NULL(instance_cache_);
3302   Node* mem_start = instance_cache_->mem_start;
3303   Node* mem_size = instance_cache_->mem_size;
3304   DCHECK_NOT_NULL(mem_start);
3305   DCHECK_NOT_NULL(mem_size);
3306 
3307   // Asm.js semantics are defined in terms of typed arrays, hence OOB
3308   // reads return {undefined} coerced to the result type (0 for integers, NaN
3309   // for float and double).
3310   // Note that we check against the memory size ignoring the size of the
3311   // stored value, which is conservative if misaligned. Technically, asm.js
3312   // should never have misaligned accesses.
3313   index = Uint32ToUintptr(index);
3314   Diamond bounds_check(
3315       graph(), mcgraph()->common(),
3316       graph()->NewNode(mcgraph()->machine()->UintLessThan(), index, mem_size),
3317       BranchHint::kTrue);
3318   bounds_check.Chain(Control());
3319 
3320   if (untrusted_code_mitigations_) {
3321     // Condition the index with the memory mask.
3322     Node* mem_mask = instance_cache_->mem_mask;
3323     DCHECK_NOT_NULL(mem_mask);
3324     index = graph()->NewNode(mcgraph()->machine()->WordAnd(), index, mem_mask);
3325   }
3326 
3327   Node* load = graph()->NewNode(mcgraph()->machine()->Load(type), mem_start,
3328                                 index, Effect(), bounds_check.if_true);
3329   SetEffect(bounds_check.EffectPhi(load, Effect()));
3330   SetControl(bounds_check.merge);
3331   return bounds_check.Phi(type.representation(), load,
3332                           GetAsmJsOOBValue(type.representation(), mcgraph()));
3333 }
3334 
Uint32ToUintptr(Node * node)3335 Node* WasmGraphBuilder::Uint32ToUintptr(Node* node) {
3336   if (mcgraph()->machine()->Is32()) return node;
3337   // Fold instances of ChangeUint32ToUint64(IntConstant) directly.
3338   Uint32Matcher matcher(node);
3339   if (matcher.HasValue()) {
3340     uintptr_t value = matcher.Value();
3341     return mcgraph()->IntPtrConstant(bit_cast<intptr_t>(value));
3342   }
3343   return graph()->NewNode(mcgraph()->machine()->ChangeUint32ToUint64(), node);
3344 }
3345 
BuildAsmjsStoreMem(MachineType type,Node * index,Node * val)3346 Node* WasmGraphBuilder::BuildAsmjsStoreMem(MachineType type, Node* index,
3347                                            Node* val) {
3348   DCHECK_NOT_NULL(instance_cache_);
3349   Node* mem_start = instance_cache_->mem_start;
3350   Node* mem_size = instance_cache_->mem_size;
3351   DCHECK_NOT_NULL(mem_start);
3352   DCHECK_NOT_NULL(mem_size);
3353 
3354   // Asm.js semantics are to ignore OOB writes.
3355   // Note that we check against the memory size ignoring the size of the
3356   // stored value, which is conservative if misaligned. Technically, asm.js
3357   // should never have misaligned accesses.
3358   Diamond bounds_check(
3359       graph(), mcgraph()->common(),
3360       graph()->NewNode(mcgraph()->machine()->Uint32LessThan(), index, mem_size),
3361       BranchHint::kTrue);
3362   bounds_check.Chain(Control());
3363 
3364   if (untrusted_code_mitigations_) {
3365     // Condition the index with the memory mask.
3366     Node* mem_mask = instance_cache_->mem_mask;
3367     DCHECK_NOT_NULL(mem_mask);
3368     index =
3369         graph()->NewNode(mcgraph()->machine()->Word32And(), index, mem_mask);
3370   }
3371 
3372   index = Uint32ToUintptr(index);
3373   const Operator* store_op = mcgraph()->machine()->Store(StoreRepresentation(
3374       type.representation(), WriteBarrierKind::kNoWriteBarrier));
3375   Node* store = graph()->NewNode(store_op, mem_start, index, val, Effect(),
3376                                  bounds_check.if_true);
3377   SetEffect(bounds_check.EffectPhi(store, Effect()));
3378   SetControl(bounds_check.merge);
3379   return val;
3380 }
3381 
PrintDebugName(Node * node)3382 void WasmGraphBuilder::PrintDebugName(Node* node) {
3383   PrintF("#%d:%s", node->id(), node->op()->mnemonic());
3384 }
3385 
graph()3386 Graph* WasmGraphBuilder::graph() { return mcgraph()->graph(); }
3387 
3388 namespace {
CreateMachineSignature(Zone * zone,wasm::FunctionSig * sig)3389 Signature<MachineRepresentation>* CreateMachineSignature(
3390     Zone* zone, wasm::FunctionSig* sig) {
3391   Signature<MachineRepresentation>::Builder builder(zone, sig->return_count(),
3392                                                     sig->parameter_count());
3393   for (auto ret : sig->returns()) {
3394     builder.AddReturn(wasm::ValueTypes::MachineRepresentationFor(ret));
3395   }
3396 
3397   for (auto param : sig->parameters()) {
3398     builder.AddParam(wasm::ValueTypes::MachineRepresentationFor(param));
3399   }
3400   return builder.Build();
3401 }
3402 }  // namespace
3403 
LowerInt64()3404 void WasmGraphBuilder::LowerInt64() {
3405   if (mcgraph()->machine()->Is64()) return;
3406   Int64Lowering r(mcgraph()->graph(), mcgraph()->machine(), mcgraph()->common(),
3407                   mcgraph()->zone(),
3408                   CreateMachineSignature(mcgraph()->zone(), sig_));
3409   r.LowerGraph();
3410 }
3411 
SimdScalarLoweringForTesting()3412 void WasmGraphBuilder::SimdScalarLoweringForTesting() {
3413   SimdScalarLowering(mcgraph(), CreateMachineSignature(mcgraph()->zone(), sig_))
3414       .LowerGraph();
3415 }
3416 
SetSourcePosition(Node * node,wasm::WasmCodePosition position)3417 void WasmGraphBuilder::SetSourcePosition(Node* node,
3418                                          wasm::WasmCodePosition position) {
3419   DCHECK_NE(position, wasm::kNoCodePosition);
3420   if (source_position_table_)
3421     source_position_table_->SetSourcePosition(node, SourcePosition(position));
3422 }
3423 
S128Zero()3424 Node* WasmGraphBuilder::S128Zero() {
3425   has_simd_ = true;
3426   return graph()->NewNode(mcgraph()->machine()->S128Zero());
3427 }
3428 
SimdOp(wasm::WasmOpcode opcode,Node * const * inputs)3429 Node* WasmGraphBuilder::SimdOp(wasm::WasmOpcode opcode, Node* const* inputs) {
3430   has_simd_ = true;
3431   switch (opcode) {
3432     case wasm::kExprF32x4Splat:
3433       return graph()->NewNode(mcgraph()->machine()->F32x4Splat(), inputs[0]);
3434     case wasm::kExprF32x4SConvertI32x4:
3435       return graph()->NewNode(mcgraph()->machine()->F32x4SConvertI32x4(),
3436                               inputs[0]);
3437     case wasm::kExprF32x4UConvertI32x4:
3438       return graph()->NewNode(mcgraph()->machine()->F32x4UConvertI32x4(),
3439                               inputs[0]);
3440     case wasm::kExprF32x4Abs:
3441       return graph()->NewNode(mcgraph()->machine()->F32x4Abs(), inputs[0]);
3442     case wasm::kExprF32x4Neg:
3443       return graph()->NewNode(mcgraph()->machine()->F32x4Neg(), inputs[0]);
3444     case wasm::kExprF32x4RecipApprox:
3445       return graph()->NewNode(mcgraph()->machine()->F32x4RecipApprox(),
3446                               inputs[0]);
3447     case wasm::kExprF32x4RecipSqrtApprox:
3448       return graph()->NewNode(mcgraph()->machine()->F32x4RecipSqrtApprox(),
3449                               inputs[0]);
3450     case wasm::kExprF32x4Add:
3451       return graph()->NewNode(mcgraph()->machine()->F32x4Add(), inputs[0],
3452                               inputs[1]);
3453     case wasm::kExprF32x4AddHoriz:
3454       return graph()->NewNode(mcgraph()->machine()->F32x4AddHoriz(), inputs[0],
3455                               inputs[1]);
3456     case wasm::kExprF32x4Sub:
3457       return graph()->NewNode(mcgraph()->machine()->F32x4Sub(), inputs[0],
3458                               inputs[1]);
3459     case wasm::kExprF32x4Mul:
3460       return graph()->NewNode(mcgraph()->machine()->F32x4Mul(), inputs[0],
3461                               inputs[1]);
3462     case wasm::kExprF32x4Min:
3463       return graph()->NewNode(mcgraph()->machine()->F32x4Min(), inputs[0],
3464                               inputs[1]);
3465     case wasm::kExprF32x4Max:
3466       return graph()->NewNode(mcgraph()->machine()->F32x4Max(), inputs[0],
3467                               inputs[1]);
3468     case wasm::kExprF32x4Eq:
3469       return graph()->NewNode(mcgraph()->machine()->F32x4Eq(), inputs[0],
3470                               inputs[1]);
3471     case wasm::kExprF32x4Ne:
3472       return graph()->NewNode(mcgraph()->machine()->F32x4Ne(), inputs[0],
3473                               inputs[1]);
3474     case wasm::kExprF32x4Lt:
3475       return graph()->NewNode(mcgraph()->machine()->F32x4Lt(), inputs[0],
3476                               inputs[1]);
3477     case wasm::kExprF32x4Le:
3478       return graph()->NewNode(mcgraph()->machine()->F32x4Le(), inputs[0],
3479                               inputs[1]);
3480     case wasm::kExprF32x4Gt:
3481       return graph()->NewNode(mcgraph()->machine()->F32x4Lt(), inputs[1],
3482                               inputs[0]);
3483     case wasm::kExprF32x4Ge:
3484       return graph()->NewNode(mcgraph()->machine()->F32x4Le(), inputs[1],
3485                               inputs[0]);
3486     case wasm::kExprI32x4Splat:
3487       return graph()->NewNode(mcgraph()->machine()->I32x4Splat(), inputs[0]);
3488     case wasm::kExprI32x4SConvertF32x4:
3489       return graph()->NewNode(mcgraph()->machine()->I32x4SConvertF32x4(),
3490                               inputs[0]);
3491     case wasm::kExprI32x4UConvertF32x4:
3492       return graph()->NewNode(mcgraph()->machine()->I32x4UConvertF32x4(),
3493                               inputs[0]);
3494     case wasm::kExprI32x4SConvertI16x8Low:
3495       return graph()->NewNode(mcgraph()->machine()->I32x4SConvertI16x8Low(),
3496                               inputs[0]);
3497     case wasm::kExprI32x4SConvertI16x8High:
3498       return graph()->NewNode(mcgraph()->machine()->I32x4SConvertI16x8High(),
3499                               inputs[0]);
3500     case wasm::kExprI32x4Neg:
3501       return graph()->NewNode(mcgraph()->machine()->I32x4Neg(), inputs[0]);
3502     case wasm::kExprI32x4Add:
3503       return graph()->NewNode(mcgraph()->machine()->I32x4Add(), inputs[0],
3504                               inputs[1]);
3505     case wasm::kExprI32x4AddHoriz:
3506       return graph()->NewNode(mcgraph()->machine()->I32x4AddHoriz(), inputs[0],
3507                               inputs[1]);
3508     case wasm::kExprI32x4Sub:
3509       return graph()->NewNode(mcgraph()->machine()->I32x4Sub(), inputs[0],
3510                               inputs[1]);
3511     case wasm::kExprI32x4Mul:
3512       return graph()->NewNode(mcgraph()->machine()->I32x4Mul(), inputs[0],
3513                               inputs[1]);
3514     case wasm::kExprI32x4MinS:
3515       return graph()->NewNode(mcgraph()->machine()->I32x4MinS(), inputs[0],
3516                               inputs[1]);
3517     case wasm::kExprI32x4MaxS:
3518       return graph()->NewNode(mcgraph()->machine()->I32x4MaxS(), inputs[0],
3519                               inputs[1]);
3520     case wasm::kExprI32x4Eq:
3521       return graph()->NewNode(mcgraph()->machine()->I32x4Eq(), inputs[0],
3522                               inputs[1]);
3523     case wasm::kExprI32x4Ne:
3524       return graph()->NewNode(mcgraph()->machine()->I32x4Ne(), inputs[0],
3525                               inputs[1]);
3526     case wasm::kExprI32x4LtS:
3527       return graph()->NewNode(mcgraph()->machine()->I32x4GtS(), inputs[1],
3528                               inputs[0]);
3529     case wasm::kExprI32x4LeS:
3530       return graph()->NewNode(mcgraph()->machine()->I32x4GeS(), inputs[1],
3531                               inputs[0]);
3532     case wasm::kExprI32x4GtS:
3533       return graph()->NewNode(mcgraph()->machine()->I32x4GtS(), inputs[0],
3534                               inputs[1]);
3535     case wasm::kExprI32x4GeS:
3536       return graph()->NewNode(mcgraph()->machine()->I32x4GeS(), inputs[0],
3537                               inputs[1]);
3538     case wasm::kExprI32x4UConvertI16x8Low:
3539       return graph()->NewNode(mcgraph()->machine()->I32x4UConvertI16x8Low(),
3540                               inputs[0]);
3541     case wasm::kExprI32x4UConvertI16x8High:
3542       return graph()->NewNode(mcgraph()->machine()->I32x4UConvertI16x8High(),
3543                               inputs[0]);
3544     case wasm::kExprI32x4MinU:
3545       return graph()->NewNode(mcgraph()->machine()->I32x4MinU(), inputs[0],
3546                               inputs[1]);
3547     case wasm::kExprI32x4MaxU:
3548       return graph()->NewNode(mcgraph()->machine()->I32x4MaxU(), inputs[0],
3549                               inputs[1]);
3550     case wasm::kExprI32x4LtU:
3551       return graph()->NewNode(mcgraph()->machine()->I32x4GtU(), inputs[1],
3552                               inputs[0]);
3553     case wasm::kExprI32x4LeU:
3554       return graph()->NewNode(mcgraph()->machine()->I32x4GeU(), inputs[1],
3555                               inputs[0]);
3556     case wasm::kExprI32x4GtU:
3557       return graph()->NewNode(mcgraph()->machine()->I32x4GtU(), inputs[0],
3558                               inputs[1]);
3559     case wasm::kExprI32x4GeU:
3560       return graph()->NewNode(mcgraph()->machine()->I32x4GeU(), inputs[0],
3561                               inputs[1]);
3562     case wasm::kExprI16x8Splat:
3563       return graph()->NewNode(mcgraph()->machine()->I16x8Splat(), inputs[0]);
3564     case wasm::kExprI16x8SConvertI8x16Low:
3565       return graph()->NewNode(mcgraph()->machine()->I16x8SConvertI8x16Low(),
3566                               inputs[0]);
3567     case wasm::kExprI16x8SConvertI8x16High:
3568       return graph()->NewNode(mcgraph()->machine()->I16x8SConvertI8x16High(),
3569                               inputs[0]);
3570     case wasm::kExprI16x8Neg:
3571       return graph()->NewNode(mcgraph()->machine()->I16x8Neg(), inputs[0]);
3572     case wasm::kExprI16x8SConvertI32x4:
3573       return graph()->NewNode(mcgraph()->machine()->I16x8SConvertI32x4(),
3574                               inputs[0], inputs[1]);
3575     case wasm::kExprI16x8Add:
3576       return graph()->NewNode(mcgraph()->machine()->I16x8Add(), inputs[0],
3577                               inputs[1]);
3578     case wasm::kExprI16x8AddSaturateS:
3579       return graph()->NewNode(mcgraph()->machine()->I16x8AddSaturateS(),
3580                               inputs[0], inputs[1]);
3581     case wasm::kExprI16x8AddHoriz:
3582       return graph()->NewNode(mcgraph()->machine()->I16x8AddHoriz(), inputs[0],
3583                               inputs[1]);
3584     case wasm::kExprI16x8Sub:
3585       return graph()->NewNode(mcgraph()->machine()->I16x8Sub(), inputs[0],
3586                               inputs[1]);
3587     case wasm::kExprI16x8SubSaturateS:
3588       return graph()->NewNode(mcgraph()->machine()->I16x8SubSaturateS(),
3589                               inputs[0], inputs[1]);
3590     case wasm::kExprI16x8Mul:
3591       return graph()->NewNode(mcgraph()->machine()->I16x8Mul(), inputs[0],
3592                               inputs[1]);
3593     case wasm::kExprI16x8MinS:
3594       return graph()->NewNode(mcgraph()->machine()->I16x8MinS(), inputs[0],
3595                               inputs[1]);
3596     case wasm::kExprI16x8MaxS:
3597       return graph()->NewNode(mcgraph()->machine()->I16x8MaxS(), inputs[0],
3598                               inputs[1]);
3599     case wasm::kExprI16x8Eq:
3600       return graph()->NewNode(mcgraph()->machine()->I16x8Eq(), inputs[0],
3601                               inputs[1]);
3602     case wasm::kExprI16x8Ne:
3603       return graph()->NewNode(mcgraph()->machine()->I16x8Ne(), inputs[0],
3604                               inputs[1]);
3605     case wasm::kExprI16x8LtS:
3606       return graph()->NewNode(mcgraph()->machine()->I16x8GtS(), inputs[1],
3607                               inputs[0]);
3608     case wasm::kExprI16x8LeS:
3609       return graph()->NewNode(mcgraph()->machine()->I16x8GeS(), inputs[1],
3610                               inputs[0]);
3611     case wasm::kExprI16x8GtS:
3612       return graph()->NewNode(mcgraph()->machine()->I16x8GtS(), inputs[0],
3613                               inputs[1]);
3614     case wasm::kExprI16x8GeS:
3615       return graph()->NewNode(mcgraph()->machine()->I16x8GeS(), inputs[0],
3616                               inputs[1]);
3617     case wasm::kExprI16x8UConvertI8x16Low:
3618       return graph()->NewNode(mcgraph()->machine()->I16x8UConvertI8x16Low(),
3619                               inputs[0]);
3620     case wasm::kExprI16x8UConvertI8x16High:
3621       return graph()->NewNode(mcgraph()->machine()->I16x8UConvertI8x16High(),
3622                               inputs[0]);
3623     case wasm::kExprI16x8UConvertI32x4:
3624       return graph()->NewNode(mcgraph()->machine()->I16x8UConvertI32x4(),
3625                               inputs[0], inputs[1]);
3626     case wasm::kExprI16x8AddSaturateU:
3627       return graph()->NewNode(mcgraph()->machine()->I16x8AddSaturateU(),
3628                               inputs[0], inputs[1]);
3629     case wasm::kExprI16x8SubSaturateU:
3630       return graph()->NewNode(mcgraph()->machine()->I16x8SubSaturateU(),
3631                               inputs[0], inputs[1]);
3632     case wasm::kExprI16x8MinU:
3633       return graph()->NewNode(mcgraph()->machine()->I16x8MinU(), inputs[0],
3634                               inputs[1]);
3635     case wasm::kExprI16x8MaxU:
3636       return graph()->NewNode(mcgraph()->machine()->I16x8MaxU(), inputs[0],
3637                               inputs[1]);
3638     case wasm::kExprI16x8LtU:
3639       return graph()->NewNode(mcgraph()->machine()->I16x8GtU(), inputs[1],
3640                               inputs[0]);
3641     case wasm::kExprI16x8LeU:
3642       return graph()->NewNode(mcgraph()->machine()->I16x8GeU(), inputs[1],
3643                               inputs[0]);
3644     case wasm::kExprI16x8GtU:
3645       return graph()->NewNode(mcgraph()->machine()->I16x8GtU(), inputs[0],
3646                               inputs[1]);
3647     case wasm::kExprI16x8GeU:
3648       return graph()->NewNode(mcgraph()->machine()->I16x8GeU(), inputs[0],
3649                               inputs[1]);
3650     case wasm::kExprI8x16Splat:
3651       return graph()->NewNode(mcgraph()->machine()->I8x16Splat(), inputs[0]);
3652     case wasm::kExprI8x16Neg:
3653       return graph()->NewNode(mcgraph()->machine()->I8x16Neg(), inputs[0]);
3654     case wasm::kExprI8x16SConvertI16x8:
3655       return graph()->NewNode(mcgraph()->machine()->I8x16SConvertI16x8(),
3656                               inputs[0], inputs[1]);
3657     case wasm::kExprI8x16Add:
3658       return graph()->NewNode(mcgraph()->machine()->I8x16Add(), inputs[0],
3659                               inputs[1]);
3660     case wasm::kExprI8x16AddSaturateS:
3661       return graph()->NewNode(mcgraph()->machine()->I8x16AddSaturateS(),
3662                               inputs[0], inputs[1]);
3663     case wasm::kExprI8x16Sub:
3664       return graph()->NewNode(mcgraph()->machine()->I8x16Sub(), inputs[0],
3665                               inputs[1]);
3666     case wasm::kExprI8x16SubSaturateS:
3667       return graph()->NewNode(mcgraph()->machine()->I8x16SubSaturateS(),
3668                               inputs[0], inputs[1]);
3669     case wasm::kExprI8x16Mul:
3670       return graph()->NewNode(mcgraph()->machine()->I8x16Mul(), inputs[0],
3671                               inputs[1]);
3672     case wasm::kExprI8x16MinS:
3673       return graph()->NewNode(mcgraph()->machine()->I8x16MinS(), inputs[0],
3674                               inputs[1]);
3675     case wasm::kExprI8x16MaxS:
3676       return graph()->NewNode(mcgraph()->machine()->I8x16MaxS(), inputs[0],
3677                               inputs[1]);
3678     case wasm::kExprI8x16Eq:
3679       return graph()->NewNode(mcgraph()->machine()->I8x16Eq(), inputs[0],
3680                               inputs[1]);
3681     case wasm::kExprI8x16Ne:
3682       return graph()->NewNode(mcgraph()->machine()->I8x16Ne(), inputs[0],
3683                               inputs[1]);
3684     case wasm::kExprI8x16LtS:
3685       return graph()->NewNode(mcgraph()->machine()->I8x16GtS(), inputs[1],
3686                               inputs[0]);
3687     case wasm::kExprI8x16LeS:
3688       return graph()->NewNode(mcgraph()->machine()->I8x16GeS(), inputs[1],
3689                               inputs[0]);
3690     case wasm::kExprI8x16GtS:
3691       return graph()->NewNode(mcgraph()->machine()->I8x16GtS(), inputs[0],
3692                               inputs[1]);
3693     case wasm::kExprI8x16GeS:
3694       return graph()->NewNode(mcgraph()->machine()->I8x16GeS(), inputs[0],
3695                               inputs[1]);
3696     case wasm::kExprI8x16UConvertI16x8:
3697       return graph()->NewNode(mcgraph()->machine()->I8x16UConvertI16x8(),
3698                               inputs[0], inputs[1]);
3699     case wasm::kExprI8x16AddSaturateU:
3700       return graph()->NewNode(mcgraph()->machine()->I8x16AddSaturateU(),
3701                               inputs[0], inputs[1]);
3702     case wasm::kExprI8x16SubSaturateU:
3703       return graph()->NewNode(mcgraph()->machine()->I8x16SubSaturateU(),
3704                               inputs[0], inputs[1]);
3705     case wasm::kExprI8x16MinU:
3706       return graph()->NewNode(mcgraph()->machine()->I8x16MinU(), inputs[0],
3707                               inputs[1]);
3708     case wasm::kExprI8x16MaxU:
3709       return graph()->NewNode(mcgraph()->machine()->I8x16MaxU(), inputs[0],
3710                               inputs[1]);
3711     case wasm::kExprI8x16LtU:
3712       return graph()->NewNode(mcgraph()->machine()->I8x16GtU(), inputs[1],
3713                               inputs[0]);
3714     case wasm::kExprI8x16LeU:
3715       return graph()->NewNode(mcgraph()->machine()->I8x16GeU(), inputs[1],
3716                               inputs[0]);
3717     case wasm::kExprI8x16GtU:
3718       return graph()->NewNode(mcgraph()->machine()->I8x16GtU(), inputs[0],
3719                               inputs[1]);
3720     case wasm::kExprI8x16GeU:
3721       return graph()->NewNode(mcgraph()->machine()->I8x16GeU(), inputs[0],
3722                               inputs[1]);
3723     case wasm::kExprS128And:
3724       return graph()->NewNode(mcgraph()->machine()->S128And(), inputs[0],
3725                               inputs[1]);
3726     case wasm::kExprS128Or:
3727       return graph()->NewNode(mcgraph()->machine()->S128Or(), inputs[0],
3728                               inputs[1]);
3729     case wasm::kExprS128Xor:
3730       return graph()->NewNode(mcgraph()->machine()->S128Xor(), inputs[0],
3731                               inputs[1]);
3732     case wasm::kExprS128Not:
3733       return graph()->NewNode(mcgraph()->machine()->S128Not(), inputs[0]);
3734     case wasm::kExprS128Select:
3735       return graph()->NewNode(mcgraph()->machine()->S128Select(), inputs[0],
3736                               inputs[1], inputs[2]);
3737     case wasm::kExprS1x4AnyTrue:
3738       return graph()->NewNode(mcgraph()->machine()->S1x4AnyTrue(), inputs[0]);
3739     case wasm::kExprS1x4AllTrue:
3740       return graph()->NewNode(mcgraph()->machine()->S1x4AllTrue(), inputs[0]);
3741     case wasm::kExprS1x8AnyTrue:
3742       return graph()->NewNode(mcgraph()->machine()->S1x8AnyTrue(), inputs[0]);
3743     case wasm::kExprS1x8AllTrue:
3744       return graph()->NewNode(mcgraph()->machine()->S1x8AllTrue(), inputs[0]);
3745     case wasm::kExprS1x16AnyTrue:
3746       return graph()->NewNode(mcgraph()->machine()->S1x16AnyTrue(), inputs[0]);
3747     case wasm::kExprS1x16AllTrue:
3748       return graph()->NewNode(mcgraph()->machine()->S1x16AllTrue(), inputs[0]);
3749     default:
3750       FATAL_UNSUPPORTED_OPCODE(opcode);
3751   }
3752 }
3753 
SimdLaneOp(wasm::WasmOpcode opcode,uint8_t lane,Node * const * inputs)3754 Node* WasmGraphBuilder::SimdLaneOp(wasm::WasmOpcode opcode, uint8_t lane,
3755                                    Node* const* inputs) {
3756   has_simd_ = true;
3757   switch (opcode) {
3758     case wasm::kExprF32x4ExtractLane:
3759       return graph()->NewNode(mcgraph()->machine()->F32x4ExtractLane(lane),
3760                               inputs[0]);
3761     case wasm::kExprF32x4ReplaceLane:
3762       return graph()->NewNode(mcgraph()->machine()->F32x4ReplaceLane(lane),
3763                               inputs[0], inputs[1]);
3764     case wasm::kExprI32x4ExtractLane:
3765       return graph()->NewNode(mcgraph()->machine()->I32x4ExtractLane(lane),
3766                               inputs[0]);
3767     case wasm::kExprI32x4ReplaceLane:
3768       return graph()->NewNode(mcgraph()->machine()->I32x4ReplaceLane(lane),
3769                               inputs[0], inputs[1]);
3770     case wasm::kExprI16x8ExtractLane:
3771       return graph()->NewNode(mcgraph()->machine()->I16x8ExtractLane(lane),
3772                               inputs[0]);
3773     case wasm::kExprI16x8ReplaceLane:
3774       return graph()->NewNode(mcgraph()->machine()->I16x8ReplaceLane(lane),
3775                               inputs[0], inputs[1]);
3776     case wasm::kExprI8x16ExtractLane:
3777       return graph()->NewNode(mcgraph()->machine()->I8x16ExtractLane(lane),
3778                               inputs[0]);
3779     case wasm::kExprI8x16ReplaceLane:
3780       return graph()->NewNode(mcgraph()->machine()->I8x16ReplaceLane(lane),
3781                               inputs[0], inputs[1]);
3782     default:
3783       FATAL_UNSUPPORTED_OPCODE(opcode);
3784   }
3785 }
3786 
SimdShiftOp(wasm::WasmOpcode opcode,uint8_t shift,Node * const * inputs)3787 Node* WasmGraphBuilder::SimdShiftOp(wasm::WasmOpcode opcode, uint8_t shift,
3788                                     Node* const* inputs) {
3789   has_simd_ = true;
3790   switch (opcode) {
3791     case wasm::kExprI32x4Shl:
3792       return graph()->NewNode(mcgraph()->machine()->I32x4Shl(shift), inputs[0]);
3793     case wasm::kExprI32x4ShrS:
3794       return graph()->NewNode(mcgraph()->machine()->I32x4ShrS(shift),
3795                               inputs[0]);
3796     case wasm::kExprI32x4ShrU:
3797       return graph()->NewNode(mcgraph()->machine()->I32x4ShrU(shift),
3798                               inputs[0]);
3799     case wasm::kExprI16x8Shl:
3800       return graph()->NewNode(mcgraph()->machine()->I16x8Shl(shift), inputs[0]);
3801     case wasm::kExprI16x8ShrS:
3802       return graph()->NewNode(mcgraph()->machine()->I16x8ShrS(shift),
3803                               inputs[0]);
3804     case wasm::kExprI16x8ShrU:
3805       return graph()->NewNode(mcgraph()->machine()->I16x8ShrU(shift),
3806                               inputs[0]);
3807     case wasm::kExprI8x16Shl:
3808       return graph()->NewNode(mcgraph()->machine()->I8x16Shl(shift), inputs[0]);
3809     case wasm::kExprI8x16ShrS:
3810       return graph()->NewNode(mcgraph()->machine()->I8x16ShrS(shift),
3811                               inputs[0]);
3812     case wasm::kExprI8x16ShrU:
3813       return graph()->NewNode(mcgraph()->machine()->I8x16ShrU(shift),
3814                               inputs[0]);
3815     default:
3816       FATAL_UNSUPPORTED_OPCODE(opcode);
3817   }
3818 }
3819 
Simd8x16ShuffleOp(const uint8_t shuffle[16],Node * const * inputs)3820 Node* WasmGraphBuilder::Simd8x16ShuffleOp(const uint8_t shuffle[16],
3821                                           Node* const* inputs) {
3822   has_simd_ = true;
3823   return graph()->NewNode(mcgraph()->machine()->S8x16Shuffle(shuffle),
3824                           inputs[0], inputs[1]);
3825 }
3826 
3827 #define ATOMIC_BINOP_LIST(V)                        \
3828   V(I32AtomicAdd, Add, Uint32, Word32)              \
3829   V(I64AtomicAdd, Add, Uint64, Word64)              \
3830   V(I32AtomicAdd8U, Add, Uint8, Word32)             \
3831   V(I32AtomicAdd16U, Add, Uint16, Word32)           \
3832   V(I64AtomicAdd8U, Add, Uint8, Word64)             \
3833   V(I64AtomicAdd16U, Add, Uint16, Word64)           \
3834   V(I64AtomicAdd32U, Add, Uint32, Word64)           \
3835   V(I32AtomicSub, Sub, Uint32, Word32)              \
3836   V(I64AtomicSub, Sub, Uint64, Word64)              \
3837   V(I32AtomicSub8U, Sub, Uint8, Word32)             \
3838   V(I32AtomicSub16U, Sub, Uint16, Word32)           \
3839   V(I64AtomicSub8U, Sub, Uint8, Word64)             \
3840   V(I64AtomicSub16U, Sub, Uint16, Word64)           \
3841   V(I64AtomicSub32U, Sub, Uint32, Word64)           \
3842   V(I32AtomicAnd, And, Uint32, Word32)              \
3843   V(I64AtomicAnd, And, Uint64, Word64)              \
3844   V(I32AtomicAnd8U, And, Uint8, Word32)             \
3845   V(I64AtomicAnd16U, And, Uint16, Word64)           \
3846   V(I32AtomicAnd16U, And, Uint16, Word32)           \
3847   V(I64AtomicAnd8U, And, Uint8, Word64)             \
3848   V(I64AtomicAnd32U, And, Uint32, Word64)           \
3849   V(I32AtomicOr, Or, Uint32, Word32)                \
3850   V(I64AtomicOr, Or, Uint64, Word64)                \
3851   V(I32AtomicOr8U, Or, Uint8, Word32)               \
3852   V(I32AtomicOr16U, Or, Uint16, Word32)             \
3853   V(I64AtomicOr8U, Or, Uint8, Word64)               \
3854   V(I64AtomicOr16U, Or, Uint16, Word64)             \
3855   V(I64AtomicOr32U, Or, Uint32, Word64)             \
3856   V(I32AtomicXor, Xor, Uint32, Word32)              \
3857   V(I64AtomicXor, Xor, Uint64, Word64)              \
3858   V(I32AtomicXor8U, Xor, Uint8, Word32)             \
3859   V(I32AtomicXor16U, Xor, Uint16, Word32)           \
3860   V(I64AtomicXor8U, Xor, Uint8, Word64)             \
3861   V(I64AtomicXor16U, Xor, Uint16, Word64)           \
3862   V(I64AtomicXor32U, Xor, Uint32, Word64)           \
3863   V(I32AtomicExchange, Exchange, Uint32, Word32)    \
3864   V(I64AtomicExchange, Exchange, Uint64, Word64)    \
3865   V(I32AtomicExchange8U, Exchange, Uint8, Word32)   \
3866   V(I32AtomicExchange16U, Exchange, Uint16, Word32) \
3867   V(I64AtomicExchange8U, Exchange, Uint8, Word64)   \
3868   V(I64AtomicExchange16U, Exchange, Uint16, Word64) \
3869   V(I64AtomicExchange32U, Exchange, Uint32, Word64)
3870 
3871 #define ATOMIC_CMP_EXCHG_LIST(V)                 \
3872   V(I32AtomicCompareExchange, Uint32, Word32)    \
3873   V(I64AtomicCompareExchange, Uint64, Word64)    \
3874   V(I32AtomicCompareExchange8U, Uint8, Word32)   \
3875   V(I32AtomicCompareExchange16U, Uint16, Word32) \
3876   V(I64AtomicCompareExchange8U, Uint8, Word64)   \
3877   V(I64AtomicCompareExchange16U, Uint16, Word64) \
3878   V(I64AtomicCompareExchange32U, Uint32, Word64)
3879 
3880 #define ATOMIC_LOAD_LIST(V)           \
3881   V(I32AtomicLoad, Uint32, Word32)    \
3882   V(I64AtomicLoad, Uint64, Word64)    \
3883   V(I32AtomicLoad8U, Uint8, Word32)   \
3884   V(I32AtomicLoad16U, Uint16, Word32) \
3885   V(I64AtomicLoad8U, Uint8, Word64)   \
3886   V(I64AtomicLoad16U, Uint16, Word64) \
3887   V(I64AtomicLoad32U, Uint32, Word64)
3888 
3889 #define ATOMIC_STORE_LIST(V)                    \
3890   V(I32AtomicStore, Uint32, kWord32, Word32)    \
3891   V(I64AtomicStore, Uint64, kWord64, Word64)    \
3892   V(I32AtomicStore8U, Uint8, kWord8, Word32)    \
3893   V(I32AtomicStore16U, Uint16, kWord16, Word32) \
3894   V(I64AtomicStore8U, Uint8, kWord8, Word64)    \
3895   V(I64AtomicStore16U, Uint16, kWord16, Word64) \
3896   V(I64AtomicStore32U, Uint32, kWord32, Word64)
3897 
AtomicOp(wasm::WasmOpcode opcode,Node * const * inputs,uint32_t alignment,uint32_t offset,wasm::WasmCodePosition position)3898 Node* WasmGraphBuilder::AtomicOp(wasm::WasmOpcode opcode, Node* const* inputs,
3899                                  uint32_t alignment, uint32_t offset,
3900                                  wasm::WasmCodePosition position) {
3901   Node* node;
3902   switch (opcode) {
3903 #define BUILD_ATOMIC_BINOP(Name, Operation, Type, Prefix)                     \
3904   case wasm::kExpr##Name: {                                                   \
3905     Node* index = CheckBoundsAndAlignment(                                    \
3906         wasm::ValueTypes::MemSize(MachineType::Type()), inputs[0], offset,    \
3907         position);                                                            \
3908     node = graph()->NewNode(                                                  \
3909         mcgraph()->machine()->Prefix##Atomic##Operation(MachineType::Type()), \
3910         MemBuffer(offset), index, inputs[1], Effect(), Control());            \
3911     break;                                                                    \
3912   }
3913     ATOMIC_BINOP_LIST(BUILD_ATOMIC_BINOP)
3914 #undef BUILD_ATOMIC_BINOP
3915 
3916 #define BUILD_ATOMIC_CMP_EXCHG(Name, Type, Prefix)                            \
3917   case wasm::kExpr##Name: {                                                   \
3918     Node* index = CheckBoundsAndAlignment(                                    \
3919         wasm::ValueTypes::MemSize(MachineType::Type()), inputs[0], offset,    \
3920         position);                                                            \
3921     node = graph()->NewNode(                                                  \
3922         mcgraph()->machine()->Prefix##AtomicCompareExchange(                  \
3923             MachineType::Type()),                                             \
3924         MemBuffer(offset), index, inputs[1], inputs[2], Effect(), Control()); \
3925     break;                                                                    \
3926   }
3927     ATOMIC_CMP_EXCHG_LIST(BUILD_ATOMIC_CMP_EXCHG)
3928 #undef BUILD_ATOMIC_CMP_EXCHG
3929 
3930 #define BUILD_ATOMIC_LOAD_OP(Name, Type, Prefix)                           \
3931   case wasm::kExpr##Name: {                                                \
3932     Node* index = CheckBoundsAndAlignment(                                 \
3933         wasm::ValueTypes::MemSize(MachineType::Type()), inputs[0], offset, \
3934         position);                                                         \
3935     node = graph()->NewNode(                                               \
3936         mcgraph()->machine()->Prefix##AtomicLoad(MachineType::Type()),     \
3937         MemBuffer(offset), index, Effect(), Control());                    \
3938     break;                                                                 \
3939   }
3940     ATOMIC_LOAD_LIST(BUILD_ATOMIC_LOAD_OP)
3941 #undef BUILD_ATOMIC_LOAD_OP
3942 
3943 #define BUILD_ATOMIC_STORE_OP(Name, Type, Rep, Prefix)                         \
3944   case wasm::kExpr##Name: {                                                    \
3945     Node* index = CheckBoundsAndAlignment(                                     \
3946         wasm::ValueTypes::MemSize(MachineType::Type()), inputs[0], offset,     \
3947         position);                                                             \
3948     node = graph()->NewNode(                                                   \
3949         mcgraph()->machine()->Prefix##AtomicStore(MachineRepresentation::Rep), \
3950         MemBuffer(offset), index, inputs[1], Effect(), Control());             \
3951     break;                                                                     \
3952   }
3953     ATOMIC_STORE_LIST(BUILD_ATOMIC_STORE_OP)
3954 #undef BUILD_ATOMIC_STORE_OP
3955     default:
3956       FATAL_UNSUPPORTED_OPCODE(opcode);
3957   }
3958   return SetEffect(node);
3959 }
3960 
3961 #undef ATOMIC_BINOP_LIST
3962 #undef ATOMIC_CMP_EXCHG_LIST
3963 #undef ATOMIC_LOAD_LIST
3964 #undef ATOMIC_STORE_LIST
3965 
3966 class WasmDecorator final : public GraphDecorator {
3967  public:
WasmDecorator(NodeOriginTable * origins,wasm::Decoder * decoder)3968   explicit WasmDecorator(NodeOriginTable* origins, wasm::Decoder* decoder)
3969       : origins_(origins), decoder_(decoder) {}
3970 
Decorate(Node * node)3971   void Decorate(Node* node) final {
3972     origins_->SetNodeOrigin(
3973         node, NodeOrigin("wasm graph creation", "n/a",
3974                          NodeOrigin::kWasmBytecode, decoder_->position()));
3975   }
3976 
3977  private:
3978   compiler::NodeOriginTable* origins_;
3979   wasm::Decoder* decoder_;
3980 };
3981 
AddBytecodePositionDecorator(NodeOriginTable * node_origins,wasm::Decoder * decoder)3982 void WasmGraphBuilder::AddBytecodePositionDecorator(
3983     NodeOriginTable* node_origins, wasm::Decoder* decoder) {
3984   DCHECK_NULL(decorator_);
3985   decorator_ = new (graph()->zone()) WasmDecorator(node_origins, decoder);
3986   graph()->AddDecorator(decorator_);
3987 }
3988 
RemoveBytecodePositionDecorator()3989 void WasmGraphBuilder::RemoveBytecodePositionDecorator() {
3990   DCHECK_NOT_NULL(decorator_);
3991   graph()->RemoveDecorator(decorator_);
3992   decorator_ = nullptr;
3993 }
3994 
3995 namespace {
must_record_function_compilation(Isolate * isolate)3996 bool must_record_function_compilation(Isolate* isolate) {
3997   return isolate->logger()->is_listening_to_code_events() ||
3998          isolate->is_profiling();
3999 }
4000 
4001 PRINTF_FORMAT(4, 5)
RecordFunctionCompilation(CodeEventListener::LogEventsAndTags tag,Isolate * isolate,Handle<Code> code,const char * format,...)4002 void RecordFunctionCompilation(CodeEventListener::LogEventsAndTags tag,
4003                                Isolate* isolate, Handle<Code> code,
4004                                const char* format, ...) {
4005   DCHECK(must_record_function_compilation(isolate));
4006 
4007   ScopedVector<char> buffer(128);
4008   va_list arguments;
4009   va_start(arguments, format);
4010   int len = VSNPrintF(buffer, format, arguments);
4011   CHECK_LT(0, len);
4012   va_end(arguments);
4013   Handle<String> name_str =
4014       isolate->factory()->NewStringFromAsciiChecked(buffer.start());
4015   PROFILE(isolate, CodeCreateEvent(tag, AbstractCode::cast(*code), *name_str));
4016 }
4017 
4018 class WasmWrapperGraphBuilder : public WasmGraphBuilder {
4019  public:
WasmWrapperGraphBuilder(Zone * zone,wasm::ModuleEnv * env,JSGraph * jsgraph,wasm::FunctionSig * sig,compiler::SourcePositionTable * spt,StubCallMode stub_mode)4020   WasmWrapperGraphBuilder(Zone* zone, wasm::ModuleEnv* env, JSGraph* jsgraph,
4021                           wasm::FunctionSig* sig,
4022                           compiler::SourcePositionTable* spt,
4023                           StubCallMode stub_mode)
4024       : WasmGraphBuilder(env, zone, jsgraph, sig, spt),
4025         isolate_(jsgraph->isolate()),
4026         jsgraph_(jsgraph),
4027         stub_mode_(stub_mode) {}
4028 
BuildAllocateHeapNumberWithValue(Node * value,Node * control)4029   Node* BuildAllocateHeapNumberWithValue(Node* value, Node* control) {
4030     MachineOperatorBuilder* machine = mcgraph()->machine();
4031     CommonOperatorBuilder* common = mcgraph()->common();
4032     Node* target = (stub_mode_ == StubCallMode::kCallWasmRuntimeStub)
4033                        ? mcgraph()->RelocatableIntPtrConstant(
4034                              wasm::WasmCode::kWasmAllocateHeapNumber,
4035                              RelocInfo::WASM_STUB_CALL)
4036                        : jsgraph()->HeapConstant(
4037                              BUILTIN_CODE(isolate_, AllocateHeapNumber));
4038     if (!allocate_heap_number_operator_.is_set()) {
4039       auto call_descriptor = Linkage::GetStubCallDescriptor(
4040           mcgraph()->zone(), AllocateHeapNumberDescriptor(), 0,
4041           CallDescriptor::kNoFlags, Operator::kNoThrow, stub_mode_);
4042       allocate_heap_number_operator_.set(common->Call(call_descriptor));
4043     }
4044     Node* heap_number = graph()->NewNode(allocate_heap_number_operator_.get(),
4045                                          target, Effect(), control);
4046     SetEffect(
4047         graph()->NewNode(machine->Store(StoreRepresentation(
4048                              MachineRepresentation::kFloat64, kNoWriteBarrier)),
4049                          heap_number, BuildHeapNumberValueIndexConstant(),
4050                          value, heap_number, control));
4051     return heap_number;
4052   }
4053 
BuildChangeSmiToFloat64(Node * value)4054   Node* BuildChangeSmiToFloat64(Node* value) {
4055     return graph()->NewNode(mcgraph()->machine()->ChangeInt32ToFloat64(),
4056                             BuildChangeSmiToInt32(value));
4057   }
4058 
BuildTestHeapObject(Node * value)4059   Node* BuildTestHeapObject(Node* value) {
4060     return graph()->NewNode(mcgraph()->machine()->WordAnd(), value,
4061                             mcgraph()->IntPtrConstant(kHeapObjectTag));
4062   }
4063 
BuildLoadHeapNumberValue(Node * value)4064   Node* BuildLoadHeapNumberValue(Node* value) {
4065     return SetEffect(graph()->NewNode(
4066         mcgraph()->machine()->Load(MachineType::Float64()), value,
4067         BuildHeapNumberValueIndexConstant(), Effect(), Control()));
4068   }
4069 
BuildHeapNumberValueIndexConstant()4070   Node* BuildHeapNumberValueIndexConstant() {
4071     return mcgraph()->IntPtrConstant(HeapNumber::kValueOffset - kHeapObjectTag);
4072   }
4073 
BuildChangeInt32ToTagged(Node * value)4074   Node* BuildChangeInt32ToTagged(Node* value) {
4075     MachineOperatorBuilder* machine = mcgraph()->machine();
4076     CommonOperatorBuilder* common = mcgraph()->common();
4077 
4078     if (SmiValuesAre32Bits()) {
4079       return BuildChangeInt32ToSmi(value);
4080     }
4081     DCHECK(SmiValuesAre31Bits());
4082 
4083     Node* effect = Effect();
4084     Node* control = Control();
4085     Node* add = graph()->NewNode(machine->Int32AddWithOverflow(), value, value,
4086                                  graph()->start());
4087 
4088     Node* ovf = graph()->NewNode(common->Projection(1), add, graph()->start());
4089     Node* branch =
4090         graph()->NewNode(common->Branch(BranchHint::kFalse), ovf, control);
4091 
4092     Node* if_true = graph()->NewNode(common->IfTrue(), branch);
4093     Node* vtrue = BuildAllocateHeapNumberWithValue(
4094         graph()->NewNode(machine->ChangeInt32ToFloat64(), value), if_true);
4095     Node* etrue = Effect();
4096 
4097     Node* if_false = graph()->NewNode(common->IfFalse(), branch);
4098     Node* vfalse = graph()->NewNode(common->Projection(0), add, if_false);
4099     vfalse = BuildChangeInt32ToIntPtr(vfalse);
4100 
4101     Node* merge =
4102         SetControl(graph()->NewNode(common->Merge(2), if_true, if_false));
4103     SetEffect(graph()->NewNode(common->EffectPhi(2), etrue, effect, merge));
4104     return graph()->NewNode(common->Phi(MachineRepresentation::kTagged, 2),
4105                             vtrue, vfalse, merge);
4106   }
4107 
BuildChangeFloat64ToTagged(Node * value)4108   Node* BuildChangeFloat64ToTagged(Node* value) {
4109     MachineOperatorBuilder* machine = mcgraph()->machine();
4110     CommonOperatorBuilder* common = mcgraph()->common();
4111 
4112     // Check several conditions:
4113     //  i32?
4114     //  ├─ true: zero?
4115     //  │        ├─ true: negative?
4116     //  │        │        ├─ true: box
4117     //  │        │        └─ false: potentially Smi
4118     //  │        └─ false: potentially Smi
4119     //  └─ false: box
4120     // For potential Smi values, depending on whether Smis are 31 or 32 bit, we
4121     // still need to check whether the value fits in a Smi.
4122 
4123     Node* effect = Effect();
4124     Node* control = Control();
4125     Node* value32 = graph()->NewNode(machine->RoundFloat64ToInt32(), value);
4126     Node* check_i32 = graph()->NewNode(
4127         machine->Float64Equal(), value,
4128         graph()->NewNode(machine->ChangeInt32ToFloat64(), value32));
4129     Node* branch_i32 = graph()->NewNode(common->Branch(), check_i32, control);
4130 
4131     Node* if_i32 = graph()->NewNode(common->IfTrue(), branch_i32);
4132     Node* if_not_i32 = graph()->NewNode(common->IfFalse(), branch_i32);
4133 
4134     // We only need to check for -0 if the {value} can potentially contain -0.
4135     Node* check_zero = graph()->NewNode(machine->Word32Equal(), value32,
4136                                         mcgraph()->Int32Constant(0));
4137     Node* branch_zero = graph()->NewNode(common->Branch(BranchHint::kFalse),
4138                                          check_zero, if_i32);
4139 
4140     Node* if_zero = graph()->NewNode(common->IfTrue(), branch_zero);
4141     Node* if_not_zero = graph()->NewNode(common->IfFalse(), branch_zero);
4142 
4143     // In case of 0, we need to check the high bits for the IEEE -0 pattern.
4144     Node* check_negative = graph()->NewNode(
4145         machine->Int32LessThan(),
4146         graph()->NewNode(machine->Float64ExtractHighWord32(), value),
4147         mcgraph()->Int32Constant(0));
4148     Node* branch_negative = graph()->NewNode(common->Branch(BranchHint::kFalse),
4149                                              check_negative, if_zero);
4150 
4151     Node* if_negative = graph()->NewNode(common->IfTrue(), branch_negative);
4152     Node* if_not_negative =
4153         graph()->NewNode(common->IfFalse(), branch_negative);
4154 
4155     // We need to create a box for negative 0.
4156     Node* if_smi =
4157         graph()->NewNode(common->Merge(2), if_not_zero, if_not_negative);
4158     Node* if_box = graph()->NewNode(common->Merge(2), if_not_i32, if_negative);
4159 
4160     // On 64-bit machines we can just wrap the 32-bit integer in a smi, for
4161     // 32-bit machines we need to deal with potential overflow and fallback to
4162     // boxing.
4163     Node* vsmi;
4164     if (SmiValuesAre32Bits()) {
4165       vsmi = BuildChangeInt32ToSmi(value32);
4166     } else {
4167       DCHECK(SmiValuesAre31Bits());
4168       Node* smi_tag = graph()->NewNode(machine->Int32AddWithOverflow(), value32,
4169                                        value32, if_smi);
4170 
4171       Node* check_ovf =
4172           graph()->NewNode(common->Projection(1), smi_tag, if_smi);
4173       Node* branch_ovf = graph()->NewNode(common->Branch(BranchHint::kFalse),
4174                                           check_ovf, if_smi);
4175 
4176       Node* if_ovf = graph()->NewNode(common->IfTrue(), branch_ovf);
4177       if_box = graph()->NewNode(common->Merge(2), if_ovf, if_box);
4178 
4179       if_smi = graph()->NewNode(common->IfFalse(), branch_ovf);
4180       vsmi = graph()->NewNode(common->Projection(0), smi_tag, if_smi);
4181       vsmi = BuildChangeInt32ToIntPtr(vsmi);
4182     }
4183 
4184     // Allocate the box for the {value}.
4185     Node* vbox = BuildAllocateHeapNumberWithValue(value, if_box);
4186     Node* ebox = Effect();
4187 
4188     Node* merge =
4189         SetControl(graph()->NewNode(common->Merge(2), if_smi, if_box));
4190     SetEffect(graph()->NewNode(common->EffectPhi(2), effect, ebox, merge));
4191     return graph()->NewNode(common->Phi(MachineRepresentation::kTagged, 2),
4192                             vsmi, vbox, merge);
4193   }
4194 
AddArgumentNodes(Node ** args,int pos,int param_count,wasm::FunctionSig * sig)4195   int AddArgumentNodes(Node** args, int pos, int param_count,
4196                        wasm::FunctionSig* sig) {
4197     // Convert wasm numbers to JS values.
4198     for (int i = 0; i < param_count; ++i) {
4199       Node* param =
4200           Param(i + 1);  // Start from index 1 to drop the instance_node.
4201       args[pos++] = ToJS(param, sig->GetParam(i));
4202     }
4203     return pos;
4204   }
4205 
BuildJavaScriptToNumber(Node * node,Node * js_context)4206   Node* BuildJavaScriptToNumber(Node* node, Node* js_context) {
4207     auto call_descriptor = Linkage::GetStubCallDescriptor(
4208         mcgraph()->zone(), TypeConversionDescriptor{}, 0,
4209         CallDescriptor::kNoFlags, Operator::kNoProperties, stub_mode_);
4210     Node* stub_code =
4211         (stub_mode_ == StubCallMode::kCallWasmRuntimeStub)
4212             ? mcgraph()->RelocatableIntPtrConstant(
4213                   wasm::WasmCode::kWasmToNumber, RelocInfo::WASM_STUB_CALL)
4214             : jsgraph()->HeapConstant(BUILTIN_CODE(isolate_, ToNumber));
4215 
4216     Node* result = SetEffect(
4217         graph()->NewNode(mcgraph()->common()->Call(call_descriptor), stub_code,
4218                          node, js_context, Effect(), Control()));
4219 
4220     SetSourcePosition(result, 1);
4221 
4222     return result;
4223   }
4224 
BuildChangeTaggedToFloat64(Node * value)4225   Node* BuildChangeTaggedToFloat64(Node* value) {
4226     MachineOperatorBuilder* machine = mcgraph()->machine();
4227     CommonOperatorBuilder* common = mcgraph()->common();
4228 
4229     // Implement the following decision tree:
4230     //  heap object?
4231     //  ├─ true: undefined?
4232     //  │        ├─ true: f64 const
4233     //  │        └─ false: load heap number value
4234     //  └─ false: smi to float64
4235 
4236     Node* check_heap_object = BuildTestHeapObject(value);
4237     Diamond is_heap_object(graph(), common, check_heap_object,
4238                            BranchHint::kFalse);
4239     is_heap_object.Chain(Control());
4240 
4241     SetControl(is_heap_object.if_true);
4242     Node* orig_effect = Effect();
4243 
4244     Node* undefined_node =
4245         LOAD_INSTANCE_FIELD(UndefinedValue, MachineType::TaggedPointer());
4246     Node* check_undefined =
4247         graph()->NewNode(machine->WordEqual(), value, undefined_node);
4248     Node* effect_tagged = Effect();
4249 
4250     Diamond is_undefined(graph(), common, check_undefined, BranchHint::kFalse);
4251     is_undefined.Nest(is_heap_object, true);
4252 
4253     SetControl(is_undefined.if_false);
4254     Node* vheap_number = BuildLoadHeapNumberValue(value);
4255     Node* effect_undefined = Effect();
4256 
4257     SetControl(is_undefined.merge);
4258     Node* vundefined =
4259         mcgraph()->Float64Constant(std::numeric_limits<double>::quiet_NaN());
4260     Node* vtagged = is_undefined.Phi(MachineRepresentation::kFloat64,
4261                                      vundefined, vheap_number);
4262 
4263     effect_tagged = is_undefined.EffectPhi(effect_tagged, effect_undefined);
4264 
4265     // If input is Smi: just convert to float64.
4266     Node* vfrom_smi = BuildChangeSmiToFloat64(value);
4267 
4268     SetControl(is_heap_object.merge);
4269     SetEffect(is_heap_object.EffectPhi(effect_tagged, orig_effect));
4270     return is_heap_object.Phi(MachineRepresentation::kFloat64, vtagged,
4271                               vfrom_smi);
4272   }
4273 
ToJS(Node * node,wasm::ValueType type)4274   Node* ToJS(Node* node, wasm::ValueType type) {
4275     switch (type) {
4276       case wasm::kWasmI32:
4277         return BuildChangeInt32ToTagged(node);
4278       case wasm::kWasmS128:
4279       case wasm::kWasmI64:
4280         UNREACHABLE();
4281       case wasm::kWasmF32:
4282         node = graph()->NewNode(mcgraph()->machine()->ChangeFloat32ToFloat64(),
4283                                 node);
4284         return BuildChangeFloat64ToTagged(node);
4285       case wasm::kWasmF64:
4286         return BuildChangeFloat64ToTagged(node);
4287       case wasm::kWasmAnyRef:
4288         return node;
4289       default:
4290         UNREACHABLE();
4291     }
4292   }
4293 
FromJS(Node * node,Node * js_context,wasm::ValueType type)4294   Node* FromJS(Node* node, Node* js_context, wasm::ValueType type) {
4295     DCHECK_NE(wasm::kWasmStmt, type);
4296 
4297     // The parameter is of type AnyRef, we take it as is.
4298     if (type == wasm::kWasmAnyRef) {
4299       return node;
4300     }
4301 
4302     // Do a JavaScript ToNumber.
4303     Node* num = BuildJavaScriptToNumber(node, js_context);
4304 
4305     // Change representation.
4306     num = BuildChangeTaggedToFloat64(num);
4307 
4308     switch (type) {
4309       case wasm::kWasmI32: {
4310         num = graph()->NewNode(mcgraph()->machine()->TruncateFloat64ToWord32(),
4311                                num);
4312         break;
4313       }
4314       case wasm::kWasmS128:
4315       case wasm::kWasmI64:
4316         UNREACHABLE();
4317       case wasm::kWasmF32:
4318         num = graph()->NewNode(mcgraph()->machine()->TruncateFloat64ToFloat32(),
4319                                num);
4320         break;
4321       case wasm::kWasmF64:
4322         break;
4323       default:
4324         UNREACHABLE();
4325     }
4326     return num;
4327   }
4328 
BuildModifyThreadInWasmFlag(bool new_value)4329   void BuildModifyThreadInWasmFlag(bool new_value) {
4330     if (!trap_handler::IsTrapHandlerEnabled()) return;
4331     Node* thread_in_wasm_flag_address_address =
4332         graph()->NewNode(mcgraph()->common()->ExternalConstant(
4333             ExternalReference::wasm_thread_in_wasm_flag_address_address(
4334                 isolate_)));
4335     Node* thread_in_wasm_flag_address = SetEffect(graph()->NewNode(
4336         mcgraph()->machine()->Load(LoadRepresentation(MachineType::Pointer())),
4337         thread_in_wasm_flag_address_address, mcgraph()->Int32Constant(0),
4338         Effect(), Control()));
4339     SetEffect(graph()->NewNode(
4340         mcgraph()->machine()->Store(StoreRepresentation(
4341             MachineRepresentation::kWord32, kNoWriteBarrier)),
4342         thread_in_wasm_flag_address, mcgraph()->Int32Constant(0),
4343         mcgraph()->Int32Constant(new_value ? 1 : 0), Effect(), Control()));
4344   }
4345 
BuildLoadFunctionDataFromExportedFunction(Node * closure)4346   Node* BuildLoadFunctionDataFromExportedFunction(Node* closure) {
4347     Node* shared = SetEffect(graph()->NewNode(
4348         jsgraph()->machine()->Load(MachineType::AnyTagged()), closure,
4349         jsgraph()->Int32Constant(JSFunction::kSharedFunctionInfoOffset -
4350                                  kHeapObjectTag),
4351         Effect(), Control()));
4352     return SetEffect(graph()->NewNode(
4353         jsgraph()->machine()->Load(MachineType::AnyTagged()), shared,
4354         jsgraph()->Int32Constant(SharedFunctionInfo::kFunctionDataOffset -
4355                                  kHeapObjectTag),
4356         Effect(), Control()));
4357   }
4358 
BuildLoadInstanceFromExportedFunctionData(Node * function_data)4359   Node* BuildLoadInstanceFromExportedFunctionData(Node* function_data) {
4360     return SetEffect(graph()->NewNode(
4361         jsgraph()->machine()->Load(MachineType::AnyTagged()), function_data,
4362         jsgraph()->Int32Constant(WasmExportedFunctionData::kInstanceOffset -
4363                                  kHeapObjectTag),
4364         Effect(), Control()));
4365   }
4366 
BuildLoadFunctionIndexFromExportedFunctionData(Node * function_data)4367   Node* BuildLoadFunctionIndexFromExportedFunctionData(Node* function_data) {
4368     Node* function_index_smi = SetEffect(graph()->NewNode(
4369         jsgraph()->machine()->Load(MachineType::AnyTagged()), function_data,
4370         jsgraph()->Int32Constant(
4371             WasmExportedFunctionData::kFunctionIndexOffset - kHeapObjectTag),
4372         Effect(), Control()));
4373     Node* function_index = BuildChangeSmiToInt32(function_index_smi);
4374     return function_index;
4375   }
4376 
BuildLoadJumpTableOffsetFromExportedFunctionData(Node * function_data)4377   Node* BuildLoadJumpTableOffsetFromExportedFunctionData(Node* function_data) {
4378     Node* jump_table_offset_smi = SetEffect(graph()->NewNode(
4379         jsgraph()->machine()->Load(MachineType::AnyTagged()), function_data,
4380         jsgraph()->Int32Constant(
4381             WasmExportedFunctionData::kJumpTableOffsetOffset - kHeapObjectTag),
4382         Effect(), Control()));
4383     Node* jump_table_offset = BuildChangeSmiToInt32(jump_table_offset_smi);
4384     return jump_table_offset;
4385   }
4386 
BuildJSToWasmWrapper(bool is_import)4387   void BuildJSToWasmWrapper(bool is_import) {
4388     const int wasm_count = static_cast<int>(sig_->parameter_count());
4389 
4390     // Build the start and the JS parameter nodes.
4391     SetEffect(SetControl(Start(wasm_count + 5)));
4392 
4393     // Create the js_closure and js_context parameters.
4394     Node* js_closure =
4395         graph()->NewNode(jsgraph()->common()->Parameter(
4396                              Linkage::kJSCallClosureParamIndex, "%closure"),
4397                          graph()->start());
4398     Node* js_context = graph()->NewNode(
4399         mcgraph()->common()->Parameter(
4400             Linkage::GetJSCallContextParamIndex(wasm_count + 1), "%context"),
4401         graph()->start());
4402 
4403     // Create the instance_node node to pass as parameter. It is loaded from
4404     // an actual reference to an instance or a placeholder reference,
4405     // called {WasmExportedFunction} via the {WasmExportedFunctionData}
4406     // structure.
4407     Node* function_data = BuildLoadFunctionDataFromExportedFunction(js_closure);
4408     instance_node_.set(
4409         BuildLoadInstanceFromExportedFunctionData(function_data));
4410 
4411     if (!wasm::IsJSCompatibleSignature(sig_)) {
4412       // Throw a TypeError. Use the js_context of the calling javascript
4413       // function (passed as a parameter), such that the generated code is
4414       // js_context independent.
4415       BuildCallToRuntimeWithContext(Runtime::kWasmThrowTypeError, js_context,
4416                                     nullptr, 0);
4417       Return(jsgraph()->SmiConstant(0));
4418       return;
4419     }
4420 
4421     const int args_count = wasm_count + 1;  // +1 for wasm_code.
4422     Node** args = Buffer(args_count);
4423     Node** rets;
4424 
4425     // Convert JS parameters to wasm numbers.
4426     for (int i = 0; i < wasm_count; ++i) {
4427       Node* param = Param(i + 1);
4428       Node* wasm_param = FromJS(param, js_context, sig_->GetParam(i));
4429       args[i + 1] = wasm_param;
4430     }
4431 
4432     // Set the ThreadInWasm flag before we do the actual call.
4433     BuildModifyThreadInWasmFlag(true);
4434 
4435     if (is_import) {
4436       // Call to an imported function.
4437       // Load function index from {WasmExportedFunctionData}.
4438       Node* function_index =
4439           BuildLoadFunctionIndexFromExportedFunctionData(function_data);
4440       BuildImportWasmCall(sig_, args, &rets, wasm::kNoCodePosition,
4441                           function_index);
4442     } else {
4443       // Call to a wasm function defined in this module.
4444       // The call target is the jump table slot for that function.
4445       Node* jump_table_start =
4446           LOAD_INSTANCE_FIELD(JumpTableStart, MachineType::Pointer());
4447       Node* jump_table_offset =
4448           BuildLoadJumpTableOffsetFromExportedFunctionData(function_data);
4449       Node* jump_table_slot = graph()->NewNode(
4450           mcgraph()->machine()->IntAdd(), jump_table_start, jump_table_offset);
4451       args[0] = jump_table_slot;
4452 
4453       BuildWasmCall(sig_, args, &rets, wasm::kNoCodePosition, nullptr,
4454                     kNoRetpoline);
4455     }
4456 
4457     // Clear the ThreadInWasm flag.
4458     BuildModifyThreadInWasmFlag(false);
4459 
4460     Node* jsval = sig_->return_count() == 0 ? jsgraph()->UndefinedConstant()
4461                                             : ToJS(rets[0], sig_->GetReturn());
4462     Return(jsval);
4463   }
4464 
BuildWasmToJSWrapper(Handle<JSReceiver> target,int index)4465   bool BuildWasmToJSWrapper(Handle<JSReceiver> target, int index) {
4466     DCHECK(target->IsCallable());
4467 
4468     int wasm_count = static_cast<int>(sig_->parameter_count());
4469 
4470     // Build the start and the parameter nodes.
4471     SetEffect(SetControl(Start(wasm_count + 3)));
4472 
4473     // Create the instance_node from the passed parameter.
4474     instance_node_.set(Param(wasm::kWasmInstanceParameterIndex));
4475 
4476     Node* callables_node = LOAD_INSTANCE_FIELD(ImportedFunctionCallables,
4477                                                MachineType::TaggedPointer());
4478     Node* callable_node = LOAD_FIXED_ARRAY_SLOT(callables_node, index);
4479     Node* undefined_node =
4480         LOAD_INSTANCE_FIELD(UndefinedValue, MachineType::TaggedPointer());
4481     Node* native_context =
4482         LOAD_INSTANCE_FIELD(NativeContext, MachineType::TaggedPointer());
4483 
4484     if (!wasm::IsJSCompatibleSignature(sig_)) {
4485       // Throw a TypeError.
4486       BuildCallToRuntimeWithContext(Runtime::kWasmThrowTypeError,
4487                                     native_context, nullptr, 0);
4488       // We don't need to return a value here, as the runtime call will not
4489       // return anyway (the c entry stub will trigger stack unwinding).
4490       ReturnVoid();
4491       return false;
4492     }
4493 
4494     CallDescriptor* call_descriptor;
4495     Node** args = Buffer(wasm_count + 9);
4496     Node* call = nullptr;
4497 
4498     BuildModifyThreadInWasmFlag(false);
4499 
4500     if (target->IsJSFunction()) {
4501       Handle<JSFunction> function = Handle<JSFunction>::cast(target);
4502       FieldAccess field_access = AccessBuilder::ForJSFunctionContext();
4503       Node* function_context = SetEffect(graph()->NewNode(
4504           mcgraph()->machine()->Load(MachineType::TaggedPointer()),
4505           callable_node,
4506           mcgraph()->Int32Constant(field_access.offset - field_access.tag()),
4507           Effect(), Control()));
4508 
4509       if (!IsClassConstructor(function->shared()->kind())) {
4510         if (function->shared()->internal_formal_parameter_count() ==
4511             wasm_count) {
4512           int pos = 0;
4513           args[pos++] = callable_node;  // target callable.
4514           // Receiver.
4515           if (is_sloppy(function->shared()->language_mode()) &&
4516               !function->shared()->native()) {
4517             Node* global_proxy = LOAD_FIXED_ARRAY_SLOT(
4518                 native_context, Context::GLOBAL_PROXY_INDEX);
4519             args[pos++] = global_proxy;
4520           } else {
4521             args[pos++] = undefined_node;
4522           }
4523 
4524           call_descriptor = Linkage::GetJSCallDescriptor(
4525               graph()->zone(), false, wasm_count + 1, CallDescriptor::kNoFlags);
4526 
4527           // Convert wasm numbers to JS values.
4528           pos = AddArgumentNodes(args, pos, wasm_count, sig_);
4529 
4530           args[pos++] = undefined_node;                        // new target
4531           args[pos++] = mcgraph()->Int32Constant(wasm_count);  // argument count
4532           args[pos++] = function_context;
4533           args[pos++] = Effect();
4534           args[pos++] = Control();
4535 
4536           call = graph()->NewNode(mcgraph()->common()->Call(call_descriptor),
4537                                   pos, args);
4538         } else if (function->shared()->internal_formal_parameter_count() >= 0) {
4539           int pos = 0;
4540           args[pos++] = mcgraph()->RelocatableIntPtrConstant(
4541               wasm::WasmCode::kWasmArgumentsAdaptor, RelocInfo::WASM_STUB_CALL);
4542           args[pos++] = callable_node;   // target callable
4543           args[pos++] = undefined_node;  // new target
4544           args[pos++] = mcgraph()->Int32Constant(wasm_count);  // argument count
4545           args[pos++] = mcgraph()->Int32Constant(
4546               function->shared()->internal_formal_parameter_count());
4547           // Receiver.
4548           if (is_sloppy(function->shared()->language_mode()) &&
4549               !function->shared()->native()) {
4550             Node* global_proxy = LOAD_FIXED_ARRAY_SLOT(
4551                 native_context, Context::GLOBAL_PROXY_INDEX);
4552             args[pos++] = global_proxy;
4553           } else {
4554             args[pos++] = undefined_node;
4555           }
4556 
4557           call_descriptor = Linkage::GetStubCallDescriptor(
4558               mcgraph()->zone(), ArgumentAdaptorDescriptor{}, 1 + wasm_count,
4559               CallDescriptor::kNoFlags, Operator::kNoProperties,
4560               StubCallMode::kCallWasmRuntimeStub);
4561 
4562           // Convert wasm numbers to JS values.
4563           pos = AddArgumentNodes(args, pos, wasm_count, sig_);
4564           args[pos++] = function_context;
4565           args[pos++] = Effect();
4566           args[pos++] = Control();
4567           call = graph()->NewNode(mcgraph()->common()->Call(call_descriptor),
4568                                   pos, args);
4569         }
4570       }
4571     }
4572 
4573     // We cannot call the target directly, we have to use the Call builtin.
4574     if (!call) {
4575       int pos = 0;
4576       args[pos++] = mcgraph()->RelocatableIntPtrConstant(
4577           wasm::WasmCode::kWasmCallJavaScript, RelocInfo::WASM_STUB_CALL);
4578       args[pos++] = callable_node;
4579       args[pos++] = mcgraph()->Int32Constant(wasm_count);  // argument count
4580       args[pos++] = undefined_node;                        // receiver
4581 
4582       call_descriptor = Linkage::GetStubCallDescriptor(
4583           graph()->zone(), CallTrampolineDescriptor{}, wasm_count + 1,
4584           CallDescriptor::kNoFlags, Operator::kNoProperties,
4585           StubCallMode::kCallWasmRuntimeStub);
4586 
4587       // Convert wasm numbers to JS values.
4588       pos = AddArgumentNodes(args, pos, wasm_count, sig_);
4589 
4590       // The native_context is sufficient here, because all kind of callables
4591       // which depend on the context provide their own context. The context here
4592       // is only needed if the target is a constructor to throw a TypeError, if
4593       // the target is a native function, or if the target is a callable
4594       // JSObject, which can only be constructed by the runtime.
4595       args[pos++] = native_context;
4596       args[pos++] = Effect();
4597       args[pos++] = Control();
4598 
4599       call = graph()->NewNode(mcgraph()->common()->Call(call_descriptor), pos,
4600                               args);
4601     }
4602 
4603     SetEffect(call);
4604     SetSourcePosition(call, 0);
4605 
4606     // Convert the return value back.
4607     Node* val = sig_->return_count() == 0
4608                     ? mcgraph()->Int32Constant(0)
4609                     : FromJS(call, native_context, sig_->GetReturn());
4610 
4611     BuildModifyThreadInWasmFlag(true);
4612 
4613     Return(val);
4614     return true;
4615   }
4616 
BuildWasmInterpreterEntry(uint32_t func_index)4617   void BuildWasmInterpreterEntry(uint32_t func_index) {
4618     int param_count = static_cast<int>(sig_->parameter_count());
4619 
4620     // Build the start and the parameter nodes.
4621     SetEffect(SetControl(Start(param_count + 3)));
4622 
4623     // Create the instance_node from the passed parameter.
4624     instance_node_.set(Param(wasm::kWasmInstanceParameterIndex));
4625 
4626     // Compute size for the argument buffer.
4627     int args_size_bytes = 0;
4628     for (wasm::ValueType type : sig_->parameters()) {
4629       args_size_bytes += wasm::ValueTypes::ElementSizeInBytes(type);
4630     }
4631 
4632     // The return value is also passed via this buffer:
4633     DCHECK_GE(wasm::kV8MaxWasmFunctionReturns, sig_->return_count());
4634     // TODO(wasm): Handle multi-value returns.
4635     DCHECK_EQ(1, wasm::kV8MaxWasmFunctionReturns);
4636     int return_size_bytes =
4637         sig_->return_count() == 0
4638             ? 0
4639             : wasm::ValueTypes::ElementSizeInBytes(sig_->GetReturn());
4640 
4641     // Get a stack slot for the arguments.
4642     Node* arg_buffer =
4643         args_size_bytes == 0 && return_size_bytes == 0
4644             ? mcgraph()->IntPtrConstant(0)
4645             : graph()->NewNode(mcgraph()->machine()->StackSlot(
4646                   std::max(args_size_bytes, return_size_bytes), 8));
4647 
4648     // Now store all our arguments to the buffer.
4649     int offset = 0;
4650 
4651     for (int i = 0; i < param_count; ++i) {
4652       wasm::ValueType type = sig_->GetParam(i);
4653       // Start from the parameter with index 1 to drop the instance_node.
4654       SetEffect(graph()->NewNode(GetSafeStoreOperator(offset, type), arg_buffer,
4655                                  Int32Constant(offset), Param(i + 1), Effect(),
4656                                  Control()));
4657       offset += wasm::ValueTypes::ElementSizeInBytes(type);
4658     }
4659     DCHECK_EQ(args_size_bytes, offset);
4660 
4661     // We are passing the raw arg_buffer here. To the GC and other parts, it
4662     // looks like a Smi (lowest bit not set). In the runtime function however,
4663     // don't call Smi::value on it, but just cast it to a byte pointer.
4664     Node* parameters[] = {
4665         jsgraph()->SmiConstant(func_index), arg_buffer,
4666     };
4667     BuildCallToRuntime(Runtime::kWasmRunInterpreter, parameters,
4668                        arraysize(parameters));
4669 
4670     // Read back the return value.
4671     if (sig_->return_count() == 0) {
4672       Return(Int32Constant(0));
4673     } else {
4674       // TODO(wasm): Implement multi-return.
4675       DCHECK_EQ(1, sig_->return_count());
4676       MachineType load_rep =
4677           wasm::ValueTypes::MachineTypeFor(sig_->GetReturn());
4678       Node* val = SetEffect(
4679           graph()->NewNode(mcgraph()->machine()->Load(load_rep), arg_buffer,
4680                            Int32Constant(0), Effect(), Control()));
4681       Return(val);
4682     }
4683 
4684     if (ContainsInt64(sig_)) LowerInt64();
4685   }
4686 
BuildCWasmEntry()4687   void BuildCWasmEntry() {
4688     // Build the start and the JS parameter nodes.
4689     SetEffect(SetControl(Start(CWasmEntryParameters::kNumParameters + 5)));
4690 
4691     // Create parameter nodes (offset by 1 for the receiver parameter).
4692     Node* foreign_code_obj = Param(CWasmEntryParameters::kCodeObject + 1);
4693     MachineOperatorBuilder* machine = mcgraph()->machine();
4694     Node* code_obj = graph()->NewNode(
4695         machine->Load(MachineType::Pointer()), foreign_code_obj,
4696         Int32Constant(Foreign::kForeignAddressOffset - kHeapObjectTag),
4697         Effect(), Control());
4698     Node* instance_node = Param(CWasmEntryParameters::kWasmInstance + 1);
4699     Node* arg_buffer = Param(CWasmEntryParameters::kArgumentsBuffer + 1);
4700 
4701     int wasm_arg_count = static_cast<int>(sig_->parameter_count());
4702     int arg_count = wasm_arg_count + 4;  // code, instance_node, control, effect
4703     Node** args = Buffer(arg_count);
4704 
4705     int pos = 0;
4706     args[pos++] = code_obj;
4707     args[pos++] = instance_node;
4708 
4709     int offset = 0;
4710     for (wasm::ValueType type : sig_->parameters()) {
4711       Node* arg_load = SetEffect(
4712           graph()->NewNode(GetSafeLoadOperator(offset, type), arg_buffer,
4713                            Int32Constant(offset), Effect(), Control()));
4714       args[pos++] = arg_load;
4715       offset += wasm::ValueTypes::ElementSizeInBytes(type);
4716     }
4717 
4718     args[pos++] = Effect();
4719     args[pos++] = Control();
4720     DCHECK_EQ(arg_count, pos);
4721 
4722     // Call the wasm code.
4723     auto call_descriptor = GetWasmCallDescriptor(mcgraph()->zone(), sig_);
4724 
4725     Node* call = SetEffect(graph()->NewNode(
4726         mcgraph()->common()->Call(call_descriptor), arg_count, args));
4727 
4728     // Store the return value.
4729     DCHECK_GE(1, sig_->return_count());
4730     if (sig_->return_count() == 1) {
4731       StoreRepresentation store_rep(
4732           wasm::ValueTypes::MachineRepresentationFor(sig_->GetReturn()),
4733           kNoWriteBarrier);
4734       SetEffect(graph()->NewNode(mcgraph()->machine()->Store(store_rep),
4735                                  arg_buffer, Int32Constant(0), call, Effect(),
4736                                  Control()));
4737     }
4738     Return(jsgraph()->SmiConstant(0));
4739 
4740     if (mcgraph()->machine()->Is32() && ContainsInt64(sig_)) {
4741       MachineRepresentation sig_reps[] = {
4742           MachineRepresentation::kWord32,  // return value
4743           MachineRepresentation::kTagged,  // receiver
4744           MachineRepresentation::kTagged,  // arg0 (code)
4745           MachineRepresentation::kTagged   // arg1 (buffer)
4746       };
4747       Signature<MachineRepresentation> c_entry_sig(1, 2, sig_reps);
4748       Int64Lowering r(mcgraph()->graph(), mcgraph()->machine(),
4749                       mcgraph()->common(), mcgraph()->zone(), &c_entry_sig);
4750       r.LowerGraph();
4751     }
4752   }
4753 
jsgraph()4754   JSGraph* jsgraph() { return jsgraph_; }
4755 
4756  private:
4757   Isolate* const isolate_;
4758   JSGraph* jsgraph_;
4759   StubCallMode stub_mode_;
4760   SetOncePointer<const Operator> allocate_heap_number_operator_;
4761 };
4762 }  // namespace
4763 
CompileJSToWasmWrapper(Isolate * isolate,const wasm::NativeModule * native_module,wasm::FunctionSig * sig,bool is_import,wasm::UseTrapHandler use_trap_handler)4764 MaybeHandle<Code> CompileJSToWasmWrapper(
4765     Isolate* isolate, const wasm::NativeModule* native_module,
4766     wasm::FunctionSig* sig, bool is_import,
4767     wasm::UseTrapHandler use_trap_handler) {
4768   TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm"),
4769                "CompileJSToWasmWrapper");
4770   const wasm::WasmModule* module = native_module->module();
4771 
4772   //----------------------------------------------------------------------------
4773   // Create the Graph.
4774   //----------------------------------------------------------------------------
4775   Zone zone(isolate->allocator(), ZONE_NAME);
4776   Graph graph(&zone);
4777   CommonOperatorBuilder common(&zone);
4778   MachineOperatorBuilder machine(
4779       &zone, MachineType::PointerRepresentation(),
4780       InstructionSelector::SupportedMachineOperatorFlags(),
4781       InstructionSelector::AlignmentRequirements());
4782   JSGraph jsgraph(isolate, &graph, &common, nullptr, nullptr, &machine);
4783 
4784   Node* control = nullptr;
4785   Node* effect = nullptr;
4786 
4787   wasm::ModuleEnv env(module, use_trap_handler, wasm::kRuntimeExceptionSupport);
4788   WasmWrapperGraphBuilder builder(&zone, &env, &jsgraph, sig, nullptr,
4789                                   StubCallMode::kCallOnHeapBuiltin);
4790   builder.set_control_ptr(&control);
4791   builder.set_effect_ptr(&effect);
4792   builder.BuildJSToWasmWrapper(is_import);
4793 
4794   //----------------------------------------------------------------------------
4795   // Run the compilation pipeline.
4796   //----------------------------------------------------------------------------
4797 #ifdef DEBUG
4798   EmbeddedVector<char, 32> func_name;
4799   static unsigned id = 0;
4800   func_name.Truncate(SNPrintF(func_name, "js-to-wasm#%d", id++));
4801 #else
4802   Vector<const char> func_name = CStrVector("js-to-wasm");
4803 #endif
4804 
4805   OptimizedCompilationInfo info(func_name, &zone, Code::JS_TO_WASM_FUNCTION);
4806 
4807   if (info.trace_turbo_graph_enabled()) {  // Simple textual RPO.
4808     StdoutStream{} << "-- Graph after change lowering -- " << std::endl
4809                    << AsRPO(graph);
4810   }
4811 
4812   // Schedule and compile to machine code.
4813   int params = static_cast<int>(sig->parameter_count());
4814   CallDescriptor* incoming = Linkage::GetJSCallDescriptor(
4815       &zone, false, params + 1, CallDescriptor::kNoFlags);
4816 
4817   MaybeHandle<Code> maybe_code = Pipeline::GenerateCodeForTesting(
4818       &info, isolate, incoming, &graph, WasmAssemblerOptions());
4819   Handle<Code> code;
4820   if (!maybe_code.ToHandle(&code)) {
4821     return maybe_code;
4822   }
4823 #ifdef ENABLE_DISASSEMBLER
4824   if (FLAG_print_opt_code) {
4825     CodeTracer::Scope tracing_scope(isolate->GetCodeTracer());
4826     OFStream os(tracing_scope.file());
4827     code->Disassemble(func_name.start(), os);
4828   }
4829 #endif
4830 
4831   if (must_record_function_compilation(isolate)) {
4832     RecordFunctionCompilation(CodeEventListener::STUB_TAG, isolate, code,
4833                               "%.*s", func_name.length(), func_name.start());
4834   }
4835 
4836   return code;
4837 }
4838 
CompileWasmToJSWrapper(Isolate * isolate,Handle<JSReceiver> target,wasm::FunctionSig * sig,uint32_t index,wasm::ModuleOrigin origin,wasm::UseTrapHandler use_trap_handler)4839 MaybeHandle<Code> CompileWasmToJSWrapper(
4840     Isolate* isolate, Handle<JSReceiver> target, wasm::FunctionSig* sig,
4841     uint32_t index, wasm::ModuleOrigin origin,
4842     wasm::UseTrapHandler use_trap_handler) {
4843   TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm"),
4844                "CompileWasmToJSWrapper");
4845   //----------------------------------------------------------------------------
4846   // Create the Graph
4847   //----------------------------------------------------------------------------
4848   Zone zone(isolate->allocator(), ZONE_NAME);
4849   Graph graph(&zone);
4850   CommonOperatorBuilder common(&zone);
4851   MachineOperatorBuilder machine(
4852       &zone, MachineType::PointerRepresentation(),
4853       InstructionSelector::SupportedMachineOperatorFlags(),
4854       InstructionSelector::AlignmentRequirements());
4855   JSGraph jsgraph(isolate, &graph, &common, nullptr, nullptr, &machine);
4856 
4857   Node* control = nullptr;
4858   Node* effect = nullptr;
4859 
4860   SourcePositionTable* source_position_table =
4861       origin == wasm::kAsmJsOrigin ? new (&zone) SourcePositionTable(&graph)
4862                                    : nullptr;
4863 
4864   wasm::ModuleEnv env(nullptr, use_trap_handler,
4865                       wasm::kRuntimeExceptionSupport);
4866 
4867   WasmWrapperGraphBuilder builder(&zone, &env, &jsgraph, sig,
4868                                   source_position_table,
4869                                   StubCallMode::kCallWasmRuntimeStub);
4870   builder.set_control_ptr(&control);
4871   builder.set_effect_ptr(&effect);
4872   builder.BuildWasmToJSWrapper(target, index);
4873 
4874 #ifdef DEBUG
4875   EmbeddedVector<char, 32> func_name;
4876   static unsigned id = 0;
4877   func_name.Truncate(SNPrintF(func_name, "wasm-to-js#%d", id++));
4878 #else
4879   Vector<const char> func_name = CStrVector("wasm-to-js");
4880 #endif
4881 
4882   OptimizedCompilationInfo info(func_name, &zone, Code::WASM_TO_JS_FUNCTION);
4883 
4884   if (info.trace_turbo_graph_enabled()) {  // Simple textual RPO.
4885     StdoutStream{} << "-- Graph after change lowering -- " << std::endl
4886                    << AsRPO(graph);
4887   }
4888 
4889   // Schedule and compile to machine code.
4890   CallDescriptor* incoming = GetWasmCallDescriptor(&zone, sig);
4891   if (machine.Is32()) {
4892     incoming = GetI32WasmCallDescriptor(&zone, incoming);
4893   }
4894   MaybeHandle<Code> maybe_code = Pipeline::GenerateCodeForTesting(
4895       &info, isolate, incoming, &graph, AssemblerOptions::Default(isolate),
4896       nullptr, source_position_table);
4897   Handle<Code> code;
4898   if (!maybe_code.ToHandle(&code)) {
4899     return maybe_code;
4900   }
4901 #ifdef ENABLE_DISASSEMBLER
4902   if (FLAG_print_opt_code) {
4903     CodeTracer::Scope tracing_scope(isolate->GetCodeTracer());
4904     OFStream os(tracing_scope.file());
4905     code->Disassemble(func_name.start(), os);
4906   }
4907 #endif
4908 
4909   if (must_record_function_compilation(isolate)) {
4910     RecordFunctionCompilation(CodeEventListener::STUB_TAG, isolate, code,
4911                               "%.*s", func_name.length(), func_name.start());
4912   }
4913 
4914   return code;
4915 }
4916 
CompileWasmInterpreterEntry(Isolate * isolate,uint32_t func_index,wasm::FunctionSig * sig)4917 MaybeHandle<Code> CompileWasmInterpreterEntry(Isolate* isolate,
4918                                               uint32_t func_index,
4919                                               wasm::FunctionSig* sig) {
4920   //----------------------------------------------------------------------------
4921   // Create the Graph
4922   //----------------------------------------------------------------------------
4923   Zone zone(isolate->allocator(), ZONE_NAME);
4924   Graph graph(&zone);
4925   CommonOperatorBuilder common(&zone);
4926   MachineOperatorBuilder machine(
4927       &zone, MachineType::PointerRepresentation(),
4928       InstructionSelector::SupportedMachineOperatorFlags(),
4929       InstructionSelector::AlignmentRequirements());
4930   JSGraph jsgraph(isolate, &graph, &common, nullptr, nullptr, &machine);
4931 
4932   Node* control = nullptr;
4933   Node* effect = nullptr;
4934 
4935   WasmWrapperGraphBuilder builder(&zone, nullptr, &jsgraph, sig, nullptr,
4936                                   StubCallMode::kCallWasmRuntimeStub);
4937   builder.set_control_ptr(&control);
4938   builder.set_effect_ptr(&effect);
4939   builder.BuildWasmInterpreterEntry(func_index);
4940 
4941   // Schedule and compile to machine code.
4942   CallDescriptor* incoming = GetWasmCallDescriptor(&zone, sig);
4943   if (machine.Is32()) {
4944     incoming = GetI32WasmCallDescriptor(&zone, incoming);
4945   }
4946 #ifdef DEBUG
4947   EmbeddedVector<char, 32> func_name;
4948   func_name.Truncate(
4949       SNPrintF(func_name, "wasm-interpreter-entry#%d", func_index));
4950 #else
4951   Vector<const char> func_name = CStrVector("wasm-interpreter-entry");
4952 #endif
4953 
4954   OptimizedCompilationInfo info(func_name, &zone, Code::WASM_INTERPRETER_ENTRY);
4955 
4956   if (info.trace_turbo_graph_enabled()) {  // Simple textual RPO.
4957     StdoutStream{} << "-- Wasm interpreter entry graph -- " << std::endl
4958                    << AsRPO(graph);
4959   }
4960 
4961   MaybeHandle<Code> maybe_code = Pipeline::GenerateCodeForTesting(
4962       &info, isolate, incoming, &graph, AssemblerOptions::Default(isolate),
4963       nullptr);
4964   Handle<Code> code;
4965   if (!maybe_code.ToHandle(&code)) {
4966     return maybe_code;
4967   }
4968 #ifdef ENABLE_DISASSEMBLER
4969   if (FLAG_print_opt_code) {
4970     CodeTracer::Scope tracing_scope(isolate->GetCodeTracer());
4971     OFStream os(tracing_scope.file());
4972     code->Disassemble(func_name.start(), os);
4973   }
4974 #endif
4975 
4976   if (must_record_function_compilation(isolate)) {
4977     RecordFunctionCompilation(CodeEventListener::STUB_TAG, isolate, code,
4978                               "%.*s", func_name.length(), func_name.start());
4979   }
4980 
4981   return maybe_code;
4982 }
4983 
CompileCWasmEntry(Isolate * isolate,wasm::FunctionSig * sig)4984 MaybeHandle<Code> CompileCWasmEntry(Isolate* isolate, wasm::FunctionSig* sig) {
4985   Zone zone(isolate->allocator(), ZONE_NAME);
4986   Graph graph(&zone);
4987   CommonOperatorBuilder common(&zone);
4988   MachineOperatorBuilder machine(
4989       &zone, MachineType::PointerRepresentation(),
4990       InstructionSelector::SupportedMachineOperatorFlags(),
4991       InstructionSelector::AlignmentRequirements());
4992   JSGraph jsgraph(isolate, &graph, &common, nullptr, nullptr, &machine);
4993 
4994   Node* control = nullptr;
4995   Node* effect = nullptr;
4996 
4997   WasmWrapperGraphBuilder builder(&zone, nullptr, &jsgraph, sig, nullptr,
4998                                   StubCallMode::kCallOnHeapBuiltin);
4999   builder.set_control_ptr(&control);
5000   builder.set_effect_ptr(&effect);
5001   builder.BuildCWasmEntry();
5002 
5003   // Schedule and compile to machine code.
5004   CallDescriptor* incoming = Linkage::GetJSCallDescriptor(
5005       &zone, false, CWasmEntryParameters::kNumParameters + 1,
5006       CallDescriptor::kNoFlags);
5007 
5008   // Build a name in the form "c-wasm-entry:<params>:<returns>".
5009   static constexpr size_t kMaxNameLen = 128;
5010   char debug_name[kMaxNameLen] = "c-wasm-entry:";
5011   size_t name_len = strlen(debug_name);
5012   auto append_name_char = [&](char c) {
5013     if (name_len + 1 < kMaxNameLen) debug_name[name_len++] = c;
5014   };
5015   for (wasm::ValueType t : sig->parameters()) {
5016     append_name_char(wasm::ValueTypes::ShortNameOf(t));
5017   }
5018   append_name_char(':');
5019   for (wasm::ValueType t : sig->returns()) {
5020     append_name_char(wasm::ValueTypes::ShortNameOf(t));
5021   }
5022   debug_name[name_len] = '\0';
5023   Vector<const char> debug_name_vec(debug_name, name_len);
5024 
5025   OptimizedCompilationInfo info(debug_name_vec, &zone, Code::C_WASM_ENTRY);
5026 
5027   if (info.trace_turbo_graph_enabled()) {  // Simple textual RPO.
5028     StdoutStream{} << "-- C Wasm entry graph -- " << std::endl << AsRPO(graph);
5029   }
5030 
5031   MaybeHandle<Code> maybe_code = Pipeline::GenerateCodeForTesting(
5032       &info, isolate, incoming, &graph, AssemblerOptions::Default(isolate));
5033   Handle<Code> code;
5034   if (!maybe_code.ToHandle(&code)) {
5035     return maybe_code;
5036   }
5037 #ifdef ENABLE_DISASSEMBLER
5038   if (FLAG_print_opt_code) {
5039     CodeTracer::Scope tracing_scope(isolate->GetCodeTracer());
5040     OFStream os(tracing_scope.file());
5041     code->Disassemble(debug_name, os);
5042   }
5043 #endif
5044 
5045   return code;
5046 }
5047 
TurbofanWasmCompilationUnit(wasm::WasmCompilationUnit * wasm_unit)5048 TurbofanWasmCompilationUnit::TurbofanWasmCompilationUnit(
5049     wasm::WasmCompilationUnit* wasm_unit)
5050     : wasm_unit_(wasm_unit) {}
5051 
5052 // Clears unique_ptrs, but (part of) the type is forward declared in the header.
5053 TurbofanWasmCompilationUnit::~TurbofanWasmCompilationUnit() = default;
5054 
BuildGraphForWasmFunction(wasm::WasmFeatures * detected,double * decode_ms,MachineGraph * mcgraph,NodeOriginTable * node_origins)5055 SourcePositionTable* TurbofanWasmCompilationUnit::BuildGraphForWasmFunction(
5056     wasm::WasmFeatures* detected, double* decode_ms, MachineGraph* mcgraph,
5057     NodeOriginTable* node_origins) {
5058   base::ElapsedTimer decode_timer;
5059   if (FLAG_trace_wasm_decode_time) {
5060     decode_timer.Start();
5061   }
5062 
5063   // Create a TF graph during decoding.
5064   SourcePositionTable* source_position_table =
5065       new (mcgraph->zone()) SourcePositionTable(mcgraph->graph());
5066   WasmGraphBuilder builder(wasm_unit_->env_, mcgraph->zone(), mcgraph,
5067                            wasm_unit_->func_body_.sig, source_position_table);
5068   graph_construction_result_ = wasm::BuildTFGraph(
5069       wasm_unit_->wasm_engine_->allocator(),
5070       wasm_unit_->native_module_->enabled_features(), wasm_unit_->env_->module,
5071       &builder, detected, wasm_unit_->func_body_, node_origins);
5072   if (graph_construction_result_.failed()) {
5073     if (FLAG_trace_wasm_compiler) {
5074       StdoutStream{} << "Compilation failed: "
5075                      << graph_construction_result_.error_msg() << std::endl;
5076     }
5077     return nullptr;
5078   }
5079 
5080   builder.LowerInt64();
5081 
5082   if (builder.has_simd() &&
5083       (!CpuFeatures::SupportsWasmSimd128() || wasm_unit_->env_->lower_simd)) {
5084     SimdScalarLowering(
5085         mcgraph,
5086         CreateMachineSignature(mcgraph->zone(), wasm_unit_->func_body_.sig))
5087         .LowerGraph();
5088   }
5089 
5090   if (wasm_unit_->func_index_ >= FLAG_trace_wasm_ast_start &&
5091       wasm_unit_->func_index_ < FLAG_trace_wasm_ast_end) {
5092     PrintRawWasmCode(wasm_unit_->wasm_engine_->allocator(),
5093                      wasm_unit_->func_body_, wasm_unit_->env_->module,
5094                      wasm::kPrintLocals);
5095   }
5096   if (FLAG_trace_wasm_decode_time) {
5097     *decode_ms = decode_timer.Elapsed().InMillisecondsF();
5098   }
5099   return source_position_table;
5100 }
5101 
5102 namespace {
GetDebugName(Zone * zone,wasm::WasmName name,int index)5103 Vector<const char> GetDebugName(Zone* zone, wasm::WasmName name, int index) {
5104   if (!name.is_empty()) {
5105     return name;
5106   }
5107 #ifdef DEBUG
5108   constexpr int kBufferLength = 15;
5109 
5110   EmbeddedVector<char, kBufferLength> name_vector;
5111   int name_len = SNPrintF(name_vector, "wasm#%d", index);
5112   DCHECK(name_len > 0 && name_len < name_vector.length());
5113 
5114   char* index_name = zone->NewArray<char>(name_len);
5115   memcpy(index_name, name_vector.start(), name_len);
5116   return Vector<const char>(index_name, name_len);
5117 #else
5118   return {};
5119 #endif
5120 }
5121 
5122 }  // namespace
5123 
ExecuteCompilation(wasm::WasmFeatures * detected)5124 void TurbofanWasmCompilationUnit::ExecuteCompilation(
5125     wasm::WasmFeatures* detected) {
5126   TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm"),
5127                "ExecuteTurbofanCompilation");
5128   double decode_ms = 0;
5129   size_t node_count = 0;
5130 
5131   // Scope for the {graph_zone}.
5132   {
5133     Zone graph_zone(wasm_unit_->wasm_engine_->allocator(), ZONE_NAME);
5134     MachineGraph* mcgraph = new (&graph_zone)
5135         MachineGraph(new (&graph_zone) Graph(&graph_zone),
5136                      new (&graph_zone) CommonOperatorBuilder(&graph_zone),
5137                      new (&graph_zone) MachineOperatorBuilder(
5138                          &graph_zone, MachineType::PointerRepresentation(),
5139                          InstructionSelector::SupportedMachineOperatorFlags(),
5140                          InstructionSelector::AlignmentRequirements()));
5141 
5142     Zone compilation_zone(wasm_unit_->wasm_engine_->allocator(), ZONE_NAME);
5143 
5144     OptimizedCompilationInfo info(
5145         GetDebugName(&compilation_zone, wasm_unit_->func_name_,
5146                      wasm_unit_->func_index_),
5147         &compilation_zone, Code::WASM_FUNCTION);
5148     if (wasm_unit_->env_->runtime_exception_support) {
5149       info.SetWasmRuntimeExceptionSupport();
5150     }
5151 
5152     NodeOriginTable* node_origins = info.trace_turbo_json_enabled()
5153                                         ? new (&graph_zone)
5154                                               NodeOriginTable(mcgraph->graph())
5155                                         : nullptr;
5156     SourcePositionTable* source_positions =
5157         BuildGraphForWasmFunction(detected, &decode_ms, mcgraph, node_origins);
5158 
5159     if (graph_construction_result_.failed()) {
5160       ok_ = false;
5161       return;
5162     }
5163 
5164     if (node_origins) {
5165       node_origins->AddDecorator();
5166     }
5167 
5168     base::ElapsedTimer pipeline_timer;
5169     if (FLAG_trace_wasm_decode_time) {
5170       node_count = mcgraph->graph()->NodeCount();
5171       pipeline_timer.Start();
5172     }
5173 
5174     // Run the compiler pipeline to generate machine code.
5175     auto call_descriptor =
5176         GetWasmCallDescriptor(&compilation_zone, wasm_unit_->func_body_.sig);
5177     if (mcgraph->machine()->Is32()) {
5178       call_descriptor =
5179           GetI32WasmCallDescriptor(&compilation_zone, call_descriptor);
5180     }
5181 
5182     std::unique_ptr<OptimizedCompilationJob> job(
5183         Pipeline::NewWasmCompilationJob(
5184             &info, wasm_unit_->wasm_engine_, mcgraph, call_descriptor,
5185             source_positions, node_origins, wasm_unit_->func_body_,
5186             const_cast<wasm::WasmModule*>(wasm_unit_->env_->module),
5187             wasm_unit_->native_module_, wasm_unit_->func_index_,
5188             wasm_unit_->env_->module->origin));
5189     ok_ = job->ExecuteJob() == CompilationJob::SUCCEEDED;
5190     // TODO(bradnelson): Improve histogram handling of size_t.
5191     wasm_unit_->counters_->wasm_compile_function_peak_memory_bytes()->AddSample(
5192         static_cast<int>(mcgraph->graph()->zone()->allocation_size()));
5193 
5194     if (FLAG_trace_wasm_decode_time) {
5195       double pipeline_ms = pipeline_timer.Elapsed().InMillisecondsF();
5196       PrintF(
5197           "wasm-compilation phase 1 ok: %u bytes, %0.3f ms decode, %zu nodes, "
5198           "%0.3f ms pipeline\n",
5199           static_cast<unsigned>(wasm_unit_->func_body_.end -
5200                                 wasm_unit_->func_body_.start),
5201           decode_ms, node_count, pipeline_ms);
5202     }
5203     if (ok_) wasm_code_ = info.wasm_code();
5204   }
5205   if (ok_) wasm_unit_->native_module()->PublishCode(wasm_code_);
5206 }
5207 
FinishCompilation(wasm::ErrorThrower * thrower)5208 wasm::WasmCode* TurbofanWasmCompilationUnit::FinishCompilation(
5209     wasm::ErrorThrower* thrower) {
5210   if (!ok_) {
5211     if (graph_construction_result_.failed()) {
5212       // Add the function as another context for the exception.
5213       EmbeddedVector<char, 128> message;
5214       if (wasm_unit_->func_name_.start() == nullptr) {
5215         SNPrintF(message, "Compiling wasm function #%d failed",
5216                  wasm_unit_->func_index_);
5217       } else {
5218         wasm::TruncatedUserString<> trunc_name(wasm_unit_->func_name_);
5219         SNPrintF(message, "Compiling wasm function #%d:%.*s failed",
5220                  wasm_unit_->func_index_, trunc_name.length(),
5221                  trunc_name.start());
5222       }
5223       thrower->CompileFailed(message.start(), graph_construction_result_);
5224     }
5225 
5226     return nullptr;
5227   }
5228   return wasm_code_;
5229 }
5230 
5231 namespace {
5232 // Helper for allocating either an GP or FP reg, or the next stack slot.
5233 class LinkageLocationAllocator {
5234  public:
5235   template <size_t kNumGpRegs, size_t kNumFpRegs>
LinkageLocationAllocator(const Register (& gp)[kNumGpRegs],const DoubleRegister (& fp)[kNumFpRegs])5236   constexpr LinkageLocationAllocator(const Register (&gp)[kNumGpRegs],
5237                                      const DoubleRegister (&fp)[kNumFpRegs])
5238       : allocator_(wasm::LinkageAllocator(gp, fp)) {}
5239 
Next(MachineRepresentation rep)5240   LinkageLocation Next(MachineRepresentation rep) {
5241     MachineType type = MachineType::TypeForRepresentation(rep);
5242     if (IsFloatingPoint(rep)) {
5243       if (allocator_.CanAllocateFP(rep)) {
5244         int reg_code = allocator_.NextFpReg(rep);
5245         return LinkageLocation::ForRegister(reg_code, type);
5246       }
5247     } else if (allocator_.CanAllocateGP()) {
5248       int reg_code = allocator_.NextGpReg();
5249       return LinkageLocation::ForRegister(reg_code, type);
5250     }
5251     // Cannot use register; use stack slot.
5252     int index = -1 - allocator_.NextStackSlot(rep);
5253     return LinkageLocation::ForCallerFrameSlot(index, type);
5254   }
5255 
SetStackOffset(int offset)5256   void SetStackOffset(int offset) { allocator_.SetStackOffset(offset); }
NumStackSlots() const5257   int NumStackSlots() const { return allocator_.NumStackSlots(); }
5258 
5259  private:
5260   wasm::LinkageAllocator allocator_;
5261 };
5262 }  // namespace
5263 
5264 // General code uses the above configuration data.
GetWasmCallDescriptor(Zone * zone,wasm::FunctionSig * fsig,WasmGraphBuilder::UseRetpoline use_retpoline)5265 CallDescriptor* GetWasmCallDescriptor(
5266     Zone* zone, wasm::FunctionSig* fsig,
5267     WasmGraphBuilder::UseRetpoline use_retpoline) {
5268   // The '+ 1' here is to accomodate the instance object as first parameter.
5269   LocationSignature::Builder locations(zone, fsig->return_count(),
5270                                        fsig->parameter_count() + 1);
5271 
5272   // Add register and/or stack parameter(s).
5273   LinkageLocationAllocator params(wasm::kGpParamRegisters,
5274                                   wasm::kFpParamRegisters);
5275 
5276   // The instance object.
5277   locations.AddParam(params.Next(MachineRepresentation::kTaggedPointer));
5278 
5279   const int parameter_count = static_cast<int>(fsig->parameter_count());
5280   for (int i = 0; i < parameter_count; i++) {
5281     MachineRepresentation param =
5282         wasm::ValueTypes::MachineRepresentationFor(fsig->GetParam(i));
5283     auto l = params.Next(param);
5284     locations.AddParam(l);
5285   }
5286 
5287   // Add return location(s).
5288   LinkageLocationAllocator rets(wasm::kGpReturnRegisters,
5289                                 wasm::kFpReturnRegisters);
5290 
5291   int parameter_slots = params.NumStackSlots();
5292   if (kPadArguments) parameter_slots = RoundUp(parameter_slots, 2);
5293 
5294   rets.SetStackOffset(parameter_slots);
5295 
5296   const int return_count = static_cast<int>(locations.return_count_);
5297   for (int i = 0; i < return_count; i++) {
5298     MachineRepresentation ret =
5299         wasm::ValueTypes::MachineRepresentationFor(fsig->GetReturn(i));
5300     auto l = rets.Next(ret);
5301     locations.AddReturn(l);
5302   }
5303 
5304   const RegList kCalleeSaveRegisters = 0;
5305   const RegList kCalleeSaveFPRegisters = 0;
5306 
5307   // The target for wasm calls is always a code object.
5308   MachineType target_type = MachineType::Pointer();
5309   LinkageLocation target_loc = LinkageLocation::ForAnyRegister(target_type);
5310 
5311   CallDescriptor::Kind kind = CallDescriptor::kCallWasmFunction;
5312 
5313   CallDescriptor::Flags flags =
5314       use_retpoline ? CallDescriptor::kRetpoline : CallDescriptor::kNoFlags;
5315   return new (zone) CallDescriptor(             // --
5316       kind,                                     // kind
5317       target_type,                              // target MachineType
5318       target_loc,                               // target location
5319       locations.Build(),                        // location_sig
5320       parameter_slots,                          // stack_parameter_count
5321       compiler::Operator::kNoProperties,        // properties
5322       kCalleeSaveRegisters,                     // callee-saved registers
5323       kCalleeSaveFPRegisters,                   // callee-saved fp regs
5324       flags,                                    // flags
5325       "wasm-call",                              // debug name
5326       0,                                        // allocatable registers
5327       rets.NumStackSlots() - parameter_slots);  // stack_return_count
5328 }
5329 
5330 namespace {
ReplaceTypeInCallDescriptorWith(Zone * zone,CallDescriptor * call_descriptor,size_t num_replacements,MachineType input_type,MachineRepresentation output_type)5331 CallDescriptor* ReplaceTypeInCallDescriptorWith(
5332     Zone* zone, CallDescriptor* call_descriptor, size_t num_replacements,
5333     MachineType input_type, MachineRepresentation output_type) {
5334   size_t parameter_count = call_descriptor->ParameterCount();
5335   size_t return_count = call_descriptor->ReturnCount();
5336   for (size_t i = 0; i < call_descriptor->ParameterCount(); i++) {
5337     if (call_descriptor->GetParameterType(i) == input_type) {
5338       parameter_count += num_replacements - 1;
5339     }
5340   }
5341   for (size_t i = 0; i < call_descriptor->ReturnCount(); i++) {
5342     if (call_descriptor->GetReturnType(i) == input_type) {
5343       return_count += num_replacements - 1;
5344     }
5345   }
5346   if (parameter_count == call_descriptor->ParameterCount() &&
5347       return_count == call_descriptor->ReturnCount()) {
5348     return call_descriptor;
5349   }
5350 
5351   LocationSignature::Builder locations(zone, return_count, parameter_count);
5352 
5353   LinkageLocationAllocator params(wasm::kGpParamRegisters,
5354                                   wasm::kFpParamRegisters);
5355   for (size_t i = 0; i < call_descriptor->ParameterCount(); i++) {
5356     if (call_descriptor->GetParameterType(i) == input_type) {
5357       for (size_t j = 0; j < num_replacements; j++) {
5358         locations.AddParam(params.Next(output_type));
5359       }
5360     } else {
5361       locations.AddParam(
5362           params.Next(call_descriptor->GetParameterType(i).representation()));
5363     }
5364   }
5365 
5366   LinkageLocationAllocator rets(wasm::kGpReturnRegisters,
5367                                 wasm::kFpReturnRegisters);
5368   rets.SetStackOffset(params.NumStackSlots());
5369   for (size_t i = 0; i < call_descriptor->ReturnCount(); i++) {
5370     if (call_descriptor->GetReturnType(i) == input_type) {
5371       for (size_t j = 0; j < num_replacements; j++) {
5372         locations.AddReturn(rets.Next(output_type));
5373       }
5374     } else {
5375       locations.AddReturn(
5376           rets.Next(call_descriptor->GetReturnType(i).representation()));
5377     }
5378   }
5379 
5380   return new (zone) CallDescriptor(                    // --
5381       call_descriptor->kind(),                         // kind
5382       call_descriptor->GetInputType(0),                // target MachineType
5383       call_descriptor->GetInputLocation(0),            // target location
5384       locations.Build(),                               // location_sig
5385       params.NumStackSlots(),                          // stack_parameter_count
5386       call_descriptor->properties(),                   // properties
5387       call_descriptor->CalleeSavedRegisters(),         // callee-saved registers
5388       call_descriptor->CalleeSavedFPRegisters(),       // callee-saved fp regs
5389       call_descriptor->flags(),                        // flags
5390       call_descriptor->debug_name(),                   // debug name
5391       call_descriptor->AllocatableRegisters(),         // allocatable registers
5392       rets.NumStackSlots() - params.NumStackSlots());  // stack_return_count
5393 }
5394 }  // namespace
5395 
GetI32WasmCallDescriptor(Zone * zone,CallDescriptor * call_descriptor)5396 CallDescriptor* GetI32WasmCallDescriptor(Zone* zone,
5397                                          CallDescriptor* call_descriptor) {
5398   return ReplaceTypeInCallDescriptorWith(zone, call_descriptor, 2,
5399                                          MachineType::Int64(),
5400                                          MachineRepresentation::kWord32);
5401 }
5402 
GetI32WasmCallDescriptorForSimd(Zone * zone,CallDescriptor * call_descriptor)5403 CallDescriptor* GetI32WasmCallDescriptorForSimd(
5404     Zone* zone, CallDescriptor* call_descriptor) {
5405   return ReplaceTypeInCallDescriptorWith(zone, call_descriptor, 4,
5406                                          MachineType::Simd128(),
5407                                          MachineRepresentation::kWord32);
5408 }
5409 
WasmAssemblerOptions()5410 AssemblerOptions WasmAssemblerOptions() {
5411   AssemblerOptions options;
5412   options.record_reloc_info_for_serialization = true;
5413   options.enable_root_array_delta_access = false;
5414   return options;
5415 }
5416 
5417 #undef WASM_64
5418 #undef FATAL_UNSUPPORTED_OPCODE
5419 #undef WASM_INSTANCE_OBJECT_OFFSET
5420 #undef LOAD_INSTANCE_FIELD
5421 #undef LOAD_FIXED_ARRAY_SLOT
5422 
5423 }  // namespace compiler
5424 }  // namespace internal
5425 }  // namespace v8
5426