1 // Copyright 2015 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "src/compiler/wasm-compiler.h"
6
7 #include <memory>
8
9 #include "src/assembler-inl.h"
10 #include "src/assembler.h"
11 #include "src/base/optional.h"
12 #include "src/base/platform/elapsed-timer.h"
13 #include "src/base/platform/platform.h"
14 #include "src/base/v8-fallthrough.h"
15 #include "src/builtins/builtins.h"
16 #include "src/code-factory.h"
17 #include "src/compiler.h"
18 #include "src/compiler/access-builder.h"
19 #include "src/compiler/code-generator.h"
20 #include "src/compiler/common-operator.h"
21 #include "src/compiler/compiler-source-position-table.h"
22 #include "src/compiler/diamond.h"
23 #include "src/compiler/graph-visualizer.h"
24 #include "src/compiler/graph.h"
25 #include "src/compiler/instruction-selector.h"
26 #include "src/compiler/int64-lowering.h"
27 #include "src/compiler/js-graph.h"
28 #include "src/compiler/js-operator.h"
29 #include "src/compiler/linkage.h"
30 #include "src/compiler/machine-operator.h"
31 #include "src/compiler/node-matchers.h"
32 #include "src/compiler/node-origin-table.h"
33 #include "src/compiler/pipeline.h"
34 #include "src/compiler/simd-scalar-lowering.h"
35 #include "src/compiler/zone-stats.h"
36 #include "src/heap/factory.h"
37 #include "src/isolate-inl.h"
38 #include "src/log-inl.h"
39 #include "src/optimized-compilation-info.h"
40 #include "src/tracing/trace-event.h"
41 #include "src/trap-handler/trap-handler.h"
42 #include "src/wasm/function-body-decoder.h"
43 #include "src/wasm/function-compiler.h"
44 #include "src/wasm/jump-table-assembler.h"
45 #include "src/wasm/memory-tracing.h"
46 #include "src/wasm/wasm-code-manager.h"
47 #include "src/wasm/wasm-limits.h"
48 #include "src/wasm/wasm-linkage.h"
49 #include "src/wasm/wasm-module.h"
50 #include "src/wasm/wasm-objects-inl.h"
51 #include "src/wasm/wasm-opcodes.h"
52 #include "src/wasm/wasm-text.h"
53
54 namespace v8 {
55 namespace internal {
56 namespace compiler {
57
58 // TODO(titzer): pull WASM_64 up to a common header.
59 #if !V8_TARGET_ARCH_32_BIT || V8_TARGET_ARCH_X64
60 #define WASM_64 1
61 #else
62 #define WASM_64 0
63 #endif
64
65 #define FATAL_UNSUPPORTED_OPCODE(opcode) \
66 FATAL("Unsupported opcode 0x%x:%s", (opcode), \
67 wasm::WasmOpcodes::OpcodeName(opcode));
68
69 #define WASM_INSTANCE_OBJECT_OFFSET(name) \
70 (WasmInstanceObject::k##name##Offset - kHeapObjectTag)
71
72 #define LOAD_INSTANCE_FIELD(name, type) \
73 SetEffect(graph()->NewNode( \
74 mcgraph()->machine()->Load(type), instance_node_.get(), \
75 mcgraph()->Int32Constant(WASM_INSTANCE_OBJECT_OFFSET(name)), Effect(), \
76 Control()))
77
78 #define LOAD_FIXED_ARRAY_SLOT(array_node, index) \
79 SetEffect(graph()->NewNode( \
80 mcgraph()->machine()->Load(MachineType::TaggedPointer()), array_node, \
81 mcgraph()->Int32Constant(FixedArrayOffsetMinusTag(index)), Effect(), \
82 Control()))
83
FixedArrayOffsetMinusTag(uint32_t index)84 int FixedArrayOffsetMinusTag(uint32_t index) {
85 auto access = AccessBuilder::ForFixedArraySlot(index);
86 return access.offset - access.tag();
87 }
88
89 namespace {
90
91 constexpr uint32_t kBytesPerExceptionValuesArrayElement = 2;
92
MergeControlToEnd(MachineGraph * mcgraph,Node * node)93 void MergeControlToEnd(MachineGraph* mcgraph, Node* node) {
94 Graph* g = mcgraph->graph();
95 if (g->end()) {
96 NodeProperties::MergeControlToEnd(g, mcgraph->common(), node);
97 } else {
98 g->SetEnd(g->NewNode(mcgraph->common()->End(1), node));
99 }
100 }
101
ContainsSimd(wasm::FunctionSig * sig)102 bool ContainsSimd(wasm::FunctionSig* sig) {
103 for (auto type : sig->all()) {
104 if (type == wasm::kWasmS128) return true;
105 }
106 return false;
107 }
108
ContainsInt64(wasm::FunctionSig * sig)109 bool ContainsInt64(wasm::FunctionSig* sig) {
110 for (auto type : sig->all()) {
111 if (type == wasm::kWasmI64) return true;
112 }
113 return false;
114 }
115 } // namespace
116
WasmGraphBuilder(wasm::ModuleEnv * env,Zone * zone,MachineGraph * mcgraph,wasm::FunctionSig * sig,compiler::SourcePositionTable * source_position_table)117 WasmGraphBuilder::WasmGraphBuilder(
118 wasm::ModuleEnv* env, Zone* zone, MachineGraph* mcgraph,
119 wasm::FunctionSig* sig,
120 compiler::SourcePositionTable* source_position_table)
121 : zone_(zone),
122 mcgraph_(mcgraph),
123 env_(env),
124 cur_buffer_(def_buffer_),
125 cur_bufsize_(kDefaultBufferSize),
126 has_simd_(ContainsSimd(sig)),
127 untrusted_code_mitigations_(FLAG_untrusted_code_mitigations),
128 sig_(sig),
129 source_position_table_(source_position_table) {
130 DCHECK_IMPLIES(use_trap_handler(), trap_handler::IsTrapHandlerEnabled());
131 DCHECK_NOT_NULL(mcgraph_);
132 }
133
Error()134 Node* WasmGraphBuilder::Error() { return mcgraph()->Dead(); }
135
Start(unsigned params)136 Node* WasmGraphBuilder::Start(unsigned params) {
137 Node* start = graph()->NewNode(mcgraph()->common()->Start(params));
138 graph()->SetStart(start);
139 return start;
140 }
141
Param(unsigned index)142 Node* WasmGraphBuilder::Param(unsigned index) {
143 return graph()->NewNode(mcgraph()->common()->Parameter(index),
144 graph()->start());
145 }
146
Loop(Node * entry)147 Node* WasmGraphBuilder::Loop(Node* entry) {
148 return graph()->NewNode(mcgraph()->common()->Loop(1), entry);
149 }
150
Terminate(Node * effect,Node * control)151 Node* WasmGraphBuilder::Terminate(Node* effect, Node* control) {
152 Node* terminate =
153 graph()->NewNode(mcgraph()->common()->Terminate(), effect, control);
154 MergeControlToEnd(mcgraph(), terminate);
155 return terminate;
156 }
157
IsPhiWithMerge(Node * phi,Node * merge)158 bool WasmGraphBuilder::IsPhiWithMerge(Node* phi, Node* merge) {
159 return phi && IrOpcode::IsPhiOpcode(phi->opcode()) &&
160 NodeProperties::GetControlInput(phi) == merge;
161 }
162
ThrowsException(Node * node,Node ** if_success,Node ** if_exception)163 bool WasmGraphBuilder::ThrowsException(Node* node, Node** if_success,
164 Node** if_exception) {
165 if (node->op()->HasProperty(compiler::Operator::kNoThrow)) {
166 return false;
167 }
168
169 *if_success = graph()->NewNode(mcgraph()->common()->IfSuccess(), node);
170 *if_exception =
171 graph()->NewNode(mcgraph()->common()->IfException(), node, node);
172
173 return true;
174 }
175
AppendToMerge(Node * merge,Node * from)176 void WasmGraphBuilder::AppendToMerge(Node* merge, Node* from) {
177 DCHECK(IrOpcode::IsMergeOpcode(merge->opcode()));
178 merge->AppendInput(mcgraph()->zone(), from);
179 int new_size = merge->InputCount();
180 NodeProperties::ChangeOp(
181 merge, mcgraph()->common()->ResizeMergeOrPhi(merge->op(), new_size));
182 }
183
AppendToPhi(Node * phi,Node * from)184 void WasmGraphBuilder::AppendToPhi(Node* phi, Node* from) {
185 DCHECK(IrOpcode::IsPhiOpcode(phi->opcode()));
186 int new_size = phi->InputCount();
187 phi->InsertInput(mcgraph()->zone(), phi->InputCount() - 1, from);
188 NodeProperties::ChangeOp(
189 phi, mcgraph()->common()->ResizeMergeOrPhi(phi->op(), new_size));
190 }
191
Merge(unsigned count,Node ** controls)192 Node* WasmGraphBuilder::Merge(unsigned count, Node** controls) {
193 return graph()->NewNode(mcgraph()->common()->Merge(count), count, controls);
194 }
195
Phi(wasm::ValueType type,unsigned count,Node ** vals,Node * control)196 Node* WasmGraphBuilder::Phi(wasm::ValueType type, unsigned count, Node** vals,
197 Node* control) {
198 DCHECK(IrOpcode::IsMergeOpcode(control->opcode()));
199 Node** buf = Realloc(vals, count, count + 1);
200 buf[count] = control;
201 return graph()->NewNode(
202 mcgraph()->common()->Phi(wasm::ValueTypes::MachineRepresentationFor(type),
203 count),
204 count + 1, buf);
205 }
206
EffectPhi(unsigned count,Node ** effects,Node * control)207 Node* WasmGraphBuilder::EffectPhi(unsigned count, Node** effects,
208 Node* control) {
209 DCHECK(IrOpcode::IsMergeOpcode(control->opcode()));
210 Node** buf = Realloc(effects, count, count + 1);
211 buf[count] = control;
212 return graph()->NewNode(mcgraph()->common()->EffectPhi(count), count + 1,
213 buf);
214 }
215
RefNull()216 Node* WasmGraphBuilder::RefNull() {
217 return LOAD_INSTANCE_FIELD(NullValue, MachineType::TaggedPointer());
218 }
219
NoContextConstant()220 Node* WasmGraphBuilder::NoContextConstant() {
221 // TODO(titzer): avoiding a dependency on JSGraph here. Refactor.
222 return mcgraph()->IntPtrConstant(0);
223 }
224
Uint32Constant(uint32_t value)225 Node* WasmGraphBuilder::Uint32Constant(uint32_t value) {
226 return mcgraph()->Uint32Constant(value);
227 }
228
Int32Constant(int32_t value)229 Node* WasmGraphBuilder::Int32Constant(int32_t value) {
230 return mcgraph()->Int32Constant(value);
231 }
232
Int64Constant(int64_t value)233 Node* WasmGraphBuilder::Int64Constant(int64_t value) {
234 return mcgraph()->Int64Constant(value);
235 }
236
IntPtrConstant(intptr_t value)237 Node* WasmGraphBuilder::IntPtrConstant(intptr_t value) {
238 return mcgraph()->IntPtrConstant(value);
239 }
240
StackCheck(wasm::WasmCodePosition position,Node ** effect,Node ** control)241 void WasmGraphBuilder::StackCheck(wasm::WasmCodePosition position,
242 Node** effect, Node** control) {
243 DCHECK_NOT_NULL(env_); // Wrappers don't get stack checks.
244 if (FLAG_wasm_no_stack_checks || !env_->runtime_exception_support) {
245 return;
246 }
247 if (effect == nullptr) effect = effect_;
248 if (control == nullptr) control = control_;
249
250 // This instruction sequence is matched in the instruction selector to
251 // load the stack pointer directly on some platforms. Hence, when modifying
252 // please also fix WasmStackCheckMatcher in node-matchers.h
253
254 Node* limit_address = graph()->NewNode(
255 mcgraph()->machine()->Load(MachineType::Pointer()), instance_node_.get(),
256 mcgraph()->Int32Constant(WASM_INSTANCE_OBJECT_OFFSET(StackLimitAddress)),
257 *effect, *control);
258 Node* limit = graph()->NewNode(
259 mcgraph()->machine()->Load(MachineType::Pointer()), limit_address,
260 mcgraph()->IntPtrConstant(0), limit_address, *control);
261 *effect = limit;
262 Node* pointer = graph()->NewNode(mcgraph()->machine()->LoadStackPointer());
263
264 Node* check =
265 graph()->NewNode(mcgraph()->machine()->UintLessThan(), limit, pointer);
266
267 Diamond stack_check(graph(), mcgraph()->common(), check, BranchHint::kTrue);
268 stack_check.Chain(*control);
269
270 if (stack_check_call_operator_ == nullptr) {
271 // Build and cache the stack check call operator and the constant
272 // representing the stack check code.
273 auto call_descriptor = Linkage::GetStubCallDescriptor(
274 mcgraph()->zone(), // zone
275 NoContextDescriptor{}, // descriptor
276 0, // stack parameter count
277 CallDescriptor::kNoFlags, // flags
278 Operator::kNoProperties, // properties
279 StubCallMode::kCallWasmRuntimeStub); // stub call mode
280 // A direct call to a wasm runtime stub defined in this module.
281 // Just encode the stub index. This will be patched at relocation.
282 stack_check_code_node_.set(mcgraph()->RelocatableIntPtrConstant(
283 wasm::WasmCode::kWasmStackGuard, RelocInfo::WASM_STUB_CALL));
284 stack_check_call_operator_ = mcgraph()->common()->Call(call_descriptor);
285 }
286
287 Node* call = graph()->NewNode(stack_check_call_operator_.get(),
288 stack_check_code_node_.get(), *effect,
289 stack_check.if_false);
290
291 SetSourcePosition(call, position);
292
293 Node* ephi = stack_check.EffectPhi(*effect, call);
294
295 *control = stack_check.merge;
296 *effect = ephi;
297 }
298
PatchInStackCheckIfNeeded()299 void WasmGraphBuilder::PatchInStackCheckIfNeeded() {
300 if (!needs_stack_check_) return;
301
302 Node* start = graph()->start();
303 // Place a stack check which uses a dummy node as control and effect.
304 Node* dummy = graph()->NewNode(mcgraph()->common()->Dead());
305 Node* control = dummy;
306 Node* effect = dummy;
307 // The function-prologue stack check is associated with position 0, which
308 // is never a position of any instruction in the function.
309 StackCheck(0, &effect, &control);
310
311 // In testing, no steck checks were emitted. Nothing to rewire then.
312 if (effect == dummy) return;
313
314 // Now patch all control uses of {start} to use {control} and all effect uses
315 // to use {effect} instead. Then rewire the dummy node to use start instead.
316 NodeProperties::ReplaceUses(start, start, effect, control);
317 NodeProperties::ReplaceUses(dummy, nullptr, start, start);
318 }
319
Binop(wasm::WasmOpcode opcode,Node * left,Node * right,wasm::WasmCodePosition position)320 Node* WasmGraphBuilder::Binop(wasm::WasmOpcode opcode, Node* left, Node* right,
321 wasm::WasmCodePosition position) {
322 const Operator* op;
323 MachineOperatorBuilder* m = mcgraph()->machine();
324 switch (opcode) {
325 case wasm::kExprI32Add:
326 op = m->Int32Add();
327 break;
328 case wasm::kExprI32Sub:
329 op = m->Int32Sub();
330 break;
331 case wasm::kExprI32Mul:
332 op = m->Int32Mul();
333 break;
334 case wasm::kExprI32DivS:
335 return BuildI32DivS(left, right, position);
336 case wasm::kExprI32DivU:
337 return BuildI32DivU(left, right, position);
338 case wasm::kExprI32RemS:
339 return BuildI32RemS(left, right, position);
340 case wasm::kExprI32RemU:
341 return BuildI32RemU(left, right, position);
342 case wasm::kExprI32And:
343 op = m->Word32And();
344 break;
345 case wasm::kExprI32Ior:
346 op = m->Word32Or();
347 break;
348 case wasm::kExprI32Xor:
349 op = m->Word32Xor();
350 break;
351 case wasm::kExprI32Shl:
352 op = m->Word32Shl();
353 right = MaskShiftCount32(right);
354 break;
355 case wasm::kExprI32ShrU:
356 op = m->Word32Shr();
357 right = MaskShiftCount32(right);
358 break;
359 case wasm::kExprI32ShrS:
360 op = m->Word32Sar();
361 right = MaskShiftCount32(right);
362 break;
363 case wasm::kExprI32Ror:
364 op = m->Word32Ror();
365 right = MaskShiftCount32(right);
366 break;
367 case wasm::kExprI32Rol:
368 right = MaskShiftCount32(right);
369 return BuildI32Rol(left, right);
370 case wasm::kExprI32Eq:
371 op = m->Word32Equal();
372 break;
373 case wasm::kExprI32Ne:
374 return Invert(Binop(wasm::kExprI32Eq, left, right));
375 case wasm::kExprI32LtS:
376 op = m->Int32LessThan();
377 break;
378 case wasm::kExprI32LeS:
379 op = m->Int32LessThanOrEqual();
380 break;
381 case wasm::kExprI32LtU:
382 op = m->Uint32LessThan();
383 break;
384 case wasm::kExprI32LeU:
385 op = m->Uint32LessThanOrEqual();
386 break;
387 case wasm::kExprI32GtS:
388 op = m->Int32LessThan();
389 std::swap(left, right);
390 break;
391 case wasm::kExprI32GeS:
392 op = m->Int32LessThanOrEqual();
393 std::swap(left, right);
394 break;
395 case wasm::kExprI32GtU:
396 op = m->Uint32LessThan();
397 std::swap(left, right);
398 break;
399 case wasm::kExprI32GeU:
400 op = m->Uint32LessThanOrEqual();
401 std::swap(left, right);
402 break;
403 case wasm::kExprI64And:
404 op = m->Word64And();
405 break;
406 case wasm::kExprI64Add:
407 op = m->Int64Add();
408 break;
409 case wasm::kExprI64Sub:
410 op = m->Int64Sub();
411 break;
412 case wasm::kExprI64Mul:
413 op = m->Int64Mul();
414 break;
415 case wasm::kExprI64DivS:
416 return BuildI64DivS(left, right, position);
417 case wasm::kExprI64DivU:
418 return BuildI64DivU(left, right, position);
419 case wasm::kExprI64RemS:
420 return BuildI64RemS(left, right, position);
421 case wasm::kExprI64RemU:
422 return BuildI64RemU(left, right, position);
423 case wasm::kExprI64Ior:
424 op = m->Word64Or();
425 break;
426 case wasm::kExprI64Xor:
427 op = m->Word64Xor();
428 break;
429 case wasm::kExprI64Shl:
430 op = m->Word64Shl();
431 right = MaskShiftCount64(right);
432 break;
433 case wasm::kExprI64ShrU:
434 op = m->Word64Shr();
435 right = MaskShiftCount64(right);
436 break;
437 case wasm::kExprI64ShrS:
438 op = m->Word64Sar();
439 right = MaskShiftCount64(right);
440 break;
441 case wasm::kExprI64Eq:
442 op = m->Word64Equal();
443 break;
444 case wasm::kExprI64Ne:
445 return Invert(Binop(wasm::kExprI64Eq, left, right));
446 case wasm::kExprI64LtS:
447 op = m->Int64LessThan();
448 break;
449 case wasm::kExprI64LeS:
450 op = m->Int64LessThanOrEqual();
451 break;
452 case wasm::kExprI64LtU:
453 op = m->Uint64LessThan();
454 break;
455 case wasm::kExprI64LeU:
456 op = m->Uint64LessThanOrEqual();
457 break;
458 case wasm::kExprI64GtS:
459 op = m->Int64LessThan();
460 std::swap(left, right);
461 break;
462 case wasm::kExprI64GeS:
463 op = m->Int64LessThanOrEqual();
464 std::swap(left, right);
465 break;
466 case wasm::kExprI64GtU:
467 op = m->Uint64LessThan();
468 std::swap(left, right);
469 break;
470 case wasm::kExprI64GeU:
471 op = m->Uint64LessThanOrEqual();
472 std::swap(left, right);
473 break;
474 case wasm::kExprI64Ror:
475 op = m->Word64Ror();
476 right = MaskShiftCount64(right);
477 break;
478 case wasm::kExprI64Rol:
479 return BuildI64Rol(left, right);
480 case wasm::kExprF32CopySign:
481 return BuildF32CopySign(left, right);
482 case wasm::kExprF64CopySign:
483 return BuildF64CopySign(left, right);
484 case wasm::kExprF32Add:
485 op = m->Float32Add();
486 break;
487 case wasm::kExprF32Sub:
488 op = m->Float32Sub();
489 break;
490 case wasm::kExprF32Mul:
491 op = m->Float32Mul();
492 break;
493 case wasm::kExprF32Div:
494 op = m->Float32Div();
495 break;
496 case wasm::kExprF32Eq:
497 op = m->Float32Equal();
498 break;
499 case wasm::kExprF32Ne:
500 return Invert(Binop(wasm::kExprF32Eq, left, right));
501 case wasm::kExprF32Lt:
502 op = m->Float32LessThan();
503 break;
504 case wasm::kExprF32Ge:
505 op = m->Float32LessThanOrEqual();
506 std::swap(left, right);
507 break;
508 case wasm::kExprF32Gt:
509 op = m->Float32LessThan();
510 std::swap(left, right);
511 break;
512 case wasm::kExprF32Le:
513 op = m->Float32LessThanOrEqual();
514 break;
515 case wasm::kExprF64Add:
516 op = m->Float64Add();
517 break;
518 case wasm::kExprF64Sub:
519 op = m->Float64Sub();
520 break;
521 case wasm::kExprF64Mul:
522 op = m->Float64Mul();
523 break;
524 case wasm::kExprF64Div:
525 op = m->Float64Div();
526 break;
527 case wasm::kExprF64Eq:
528 op = m->Float64Equal();
529 break;
530 case wasm::kExprF64Ne:
531 return Invert(Binop(wasm::kExprF64Eq, left, right));
532 case wasm::kExprF64Lt:
533 op = m->Float64LessThan();
534 break;
535 case wasm::kExprF64Le:
536 op = m->Float64LessThanOrEqual();
537 break;
538 case wasm::kExprF64Gt:
539 op = m->Float64LessThan();
540 std::swap(left, right);
541 break;
542 case wasm::kExprF64Ge:
543 op = m->Float64LessThanOrEqual();
544 std::swap(left, right);
545 break;
546 case wasm::kExprF32Min:
547 op = m->Float32Min();
548 break;
549 case wasm::kExprF64Min:
550 op = m->Float64Min();
551 break;
552 case wasm::kExprF32Max:
553 op = m->Float32Max();
554 break;
555 case wasm::kExprF64Max:
556 op = m->Float64Max();
557 break;
558 case wasm::kExprF64Pow:
559 return BuildF64Pow(left, right);
560 case wasm::kExprF64Atan2:
561 op = m->Float64Atan2();
562 break;
563 case wasm::kExprF64Mod:
564 return BuildF64Mod(left, right);
565 case wasm::kExprI32AsmjsDivS:
566 return BuildI32AsmjsDivS(left, right);
567 case wasm::kExprI32AsmjsDivU:
568 return BuildI32AsmjsDivU(left, right);
569 case wasm::kExprI32AsmjsRemS:
570 return BuildI32AsmjsRemS(left, right);
571 case wasm::kExprI32AsmjsRemU:
572 return BuildI32AsmjsRemU(left, right);
573 case wasm::kExprI32AsmjsStoreMem8:
574 return BuildAsmjsStoreMem(MachineType::Int8(), left, right);
575 case wasm::kExprI32AsmjsStoreMem16:
576 return BuildAsmjsStoreMem(MachineType::Int16(), left, right);
577 case wasm::kExprI32AsmjsStoreMem:
578 return BuildAsmjsStoreMem(MachineType::Int32(), left, right);
579 case wasm::kExprF32AsmjsStoreMem:
580 return BuildAsmjsStoreMem(MachineType::Float32(), left, right);
581 case wasm::kExprF64AsmjsStoreMem:
582 return BuildAsmjsStoreMem(MachineType::Float64(), left, right);
583 default:
584 FATAL_UNSUPPORTED_OPCODE(opcode);
585 }
586 return graph()->NewNode(op, left, right);
587 }
588
Unop(wasm::WasmOpcode opcode,Node * input,wasm::WasmCodePosition position)589 Node* WasmGraphBuilder::Unop(wasm::WasmOpcode opcode, Node* input,
590 wasm::WasmCodePosition position) {
591 const Operator* op;
592 MachineOperatorBuilder* m = mcgraph()->machine();
593 switch (opcode) {
594 case wasm::kExprI32Eqz:
595 op = m->Word32Equal();
596 return graph()->NewNode(op, input, mcgraph()->Int32Constant(0));
597 case wasm::kExprF32Abs:
598 op = m->Float32Abs();
599 break;
600 case wasm::kExprF32Neg: {
601 op = m->Float32Neg();
602 break;
603 }
604 case wasm::kExprF32Sqrt:
605 op = m->Float32Sqrt();
606 break;
607 case wasm::kExprF64Abs:
608 op = m->Float64Abs();
609 break;
610 case wasm::kExprF64Neg: {
611 op = m->Float64Neg();
612 break;
613 }
614 case wasm::kExprF64Sqrt:
615 op = m->Float64Sqrt();
616 break;
617 case wasm::kExprI32SConvertF32:
618 case wasm::kExprI32UConvertF32:
619 case wasm::kExprI32SConvertF64:
620 case wasm::kExprI32UConvertF64:
621 case wasm::kExprI32SConvertSatF64:
622 case wasm::kExprI32UConvertSatF64:
623 case wasm::kExprI32SConvertSatF32:
624 case wasm::kExprI32UConvertSatF32:
625 return BuildIntConvertFloat(input, position, opcode);
626 case wasm::kExprI32AsmjsSConvertF64:
627 return BuildI32AsmjsSConvertF64(input);
628 case wasm::kExprI32AsmjsUConvertF64:
629 return BuildI32AsmjsUConvertF64(input);
630 case wasm::kExprF32ConvertF64:
631 op = m->TruncateFloat64ToFloat32();
632 break;
633 case wasm::kExprF64SConvertI32:
634 op = m->ChangeInt32ToFloat64();
635 break;
636 case wasm::kExprF64UConvertI32:
637 op = m->ChangeUint32ToFloat64();
638 break;
639 case wasm::kExprF32SConvertI32:
640 op = m->RoundInt32ToFloat32();
641 break;
642 case wasm::kExprF32UConvertI32:
643 op = m->RoundUint32ToFloat32();
644 break;
645 case wasm::kExprI32AsmjsSConvertF32:
646 return BuildI32AsmjsSConvertF32(input);
647 case wasm::kExprI32AsmjsUConvertF32:
648 return BuildI32AsmjsUConvertF32(input);
649 case wasm::kExprF64ConvertF32:
650 op = m->ChangeFloat32ToFloat64();
651 break;
652 case wasm::kExprF32ReinterpretI32:
653 op = m->BitcastInt32ToFloat32();
654 break;
655 case wasm::kExprI32ReinterpretF32:
656 op = m->BitcastFloat32ToInt32();
657 break;
658 case wasm::kExprI32Clz:
659 op = m->Word32Clz();
660 break;
661 case wasm::kExprI32Ctz: {
662 if (m->Word32Ctz().IsSupported()) {
663 op = m->Word32Ctz().op();
664 break;
665 } else if (m->Word32ReverseBits().IsSupported()) {
666 Node* reversed = graph()->NewNode(m->Word32ReverseBits().op(), input);
667 Node* result = graph()->NewNode(m->Word32Clz(), reversed);
668 return result;
669 } else {
670 return BuildI32Ctz(input);
671 }
672 }
673 case wasm::kExprI32Popcnt: {
674 if (m->Word32Popcnt().IsSupported()) {
675 op = m->Word32Popcnt().op();
676 break;
677 } else {
678 return BuildI32Popcnt(input);
679 }
680 }
681 case wasm::kExprF32Floor: {
682 if (!m->Float32RoundDown().IsSupported()) return BuildF32Floor(input);
683 op = m->Float32RoundDown().op();
684 break;
685 }
686 case wasm::kExprF32Ceil: {
687 if (!m->Float32RoundUp().IsSupported()) return BuildF32Ceil(input);
688 op = m->Float32RoundUp().op();
689 break;
690 }
691 case wasm::kExprF32Trunc: {
692 if (!m->Float32RoundTruncate().IsSupported()) return BuildF32Trunc(input);
693 op = m->Float32RoundTruncate().op();
694 break;
695 }
696 case wasm::kExprF32NearestInt: {
697 if (!m->Float32RoundTiesEven().IsSupported())
698 return BuildF32NearestInt(input);
699 op = m->Float32RoundTiesEven().op();
700 break;
701 }
702 case wasm::kExprF64Floor: {
703 if (!m->Float64RoundDown().IsSupported()) return BuildF64Floor(input);
704 op = m->Float64RoundDown().op();
705 break;
706 }
707 case wasm::kExprF64Ceil: {
708 if (!m->Float64RoundUp().IsSupported()) return BuildF64Ceil(input);
709 op = m->Float64RoundUp().op();
710 break;
711 }
712 case wasm::kExprF64Trunc: {
713 if (!m->Float64RoundTruncate().IsSupported()) return BuildF64Trunc(input);
714 op = m->Float64RoundTruncate().op();
715 break;
716 }
717 case wasm::kExprF64NearestInt: {
718 if (!m->Float64RoundTiesEven().IsSupported())
719 return BuildF64NearestInt(input);
720 op = m->Float64RoundTiesEven().op();
721 break;
722 }
723 case wasm::kExprF64Acos: {
724 return BuildF64Acos(input);
725 }
726 case wasm::kExprF64Asin: {
727 return BuildF64Asin(input);
728 }
729 case wasm::kExprF64Atan:
730 op = m->Float64Atan();
731 break;
732 case wasm::kExprF64Cos: {
733 op = m->Float64Cos();
734 break;
735 }
736 case wasm::kExprF64Sin: {
737 op = m->Float64Sin();
738 break;
739 }
740 case wasm::kExprF64Tan: {
741 op = m->Float64Tan();
742 break;
743 }
744 case wasm::kExprF64Exp: {
745 op = m->Float64Exp();
746 break;
747 }
748 case wasm::kExprF64Log:
749 op = m->Float64Log();
750 break;
751 case wasm::kExprI32ConvertI64:
752 op = m->TruncateInt64ToInt32();
753 break;
754 case wasm::kExprI64SConvertI32:
755 op = m->ChangeInt32ToInt64();
756 break;
757 case wasm::kExprI64UConvertI32:
758 op = m->ChangeUint32ToUint64();
759 break;
760 case wasm::kExprF64ReinterpretI64:
761 op = m->BitcastInt64ToFloat64();
762 break;
763 case wasm::kExprI64ReinterpretF64:
764 op = m->BitcastFloat64ToInt64();
765 break;
766 case wasm::kExprI64Clz:
767 op = m->Word64Clz();
768 break;
769 case wasm::kExprI64Ctz: {
770 OptionalOperator ctz64 = m->Word64Ctz();
771 if (ctz64.IsSupported()) {
772 op = ctz64.op();
773 break;
774 } else if (m->Is32() && m->Word32Ctz().IsSupported()) {
775 op = ctz64.placeholder();
776 break;
777 } else if (m->Word64ReverseBits().IsSupported()) {
778 Node* reversed = graph()->NewNode(m->Word64ReverseBits().op(), input);
779 Node* result = graph()->NewNode(m->Word64Clz(), reversed);
780 return result;
781 } else {
782 return BuildI64Ctz(input);
783 }
784 }
785 case wasm::kExprI64Popcnt: {
786 OptionalOperator popcnt64 = m->Word64Popcnt();
787 if (popcnt64.IsSupported()) {
788 op = popcnt64.op();
789 } else if (m->Is32() && m->Word32Popcnt().IsSupported()) {
790 op = popcnt64.placeholder();
791 } else {
792 return BuildI64Popcnt(input);
793 }
794 break;
795 }
796 case wasm::kExprI64Eqz:
797 op = m->Word64Equal();
798 return graph()->NewNode(op, input, mcgraph()->Int64Constant(0));
799 case wasm::kExprF32SConvertI64:
800 if (m->Is32()) {
801 return BuildF32SConvertI64(input);
802 }
803 op = m->RoundInt64ToFloat32();
804 break;
805 case wasm::kExprF32UConvertI64:
806 if (m->Is32()) {
807 return BuildF32UConvertI64(input);
808 }
809 op = m->RoundUint64ToFloat32();
810 break;
811 case wasm::kExprF64SConvertI64:
812 if (m->Is32()) {
813 return BuildF64SConvertI64(input);
814 }
815 op = m->RoundInt64ToFloat64();
816 break;
817 case wasm::kExprF64UConvertI64:
818 if (m->Is32()) {
819 return BuildF64UConvertI64(input);
820 }
821 op = m->RoundUint64ToFloat64();
822 break;
823 case wasm::kExprI32SExtendI8:
824 op = m->SignExtendWord8ToInt32();
825 break;
826 case wasm::kExprI32SExtendI16:
827 op = m->SignExtendWord16ToInt32();
828 break;
829 case wasm::kExprI64SExtendI8:
830 op = m->SignExtendWord8ToInt64();
831 break;
832 case wasm::kExprI64SExtendI16:
833 op = m->SignExtendWord16ToInt64();
834 break;
835 case wasm::kExprI64SExtendI32:
836 op = m->SignExtendWord32ToInt64();
837 break;
838 case wasm::kExprI64SConvertF32:
839 case wasm::kExprI64UConvertF32:
840 case wasm::kExprI64SConvertF64:
841 case wasm::kExprI64UConvertF64:
842 case wasm::kExprI64SConvertSatF32:
843 case wasm::kExprI64UConvertSatF32:
844 case wasm::kExprI64SConvertSatF64:
845 case wasm::kExprI64UConvertSatF64:
846 return mcgraph()->machine()->Is32()
847 ? BuildCcallConvertFloat(input, position, opcode)
848 : BuildIntConvertFloat(input, position, opcode);
849 case wasm::kExprRefIsNull:
850 return graph()->NewNode(m->WordEqual(), input, RefNull());
851 case wasm::kExprI32AsmjsLoadMem8S:
852 return BuildAsmjsLoadMem(MachineType::Int8(), input);
853 case wasm::kExprI32AsmjsLoadMem8U:
854 return BuildAsmjsLoadMem(MachineType::Uint8(), input);
855 case wasm::kExprI32AsmjsLoadMem16S:
856 return BuildAsmjsLoadMem(MachineType::Int16(), input);
857 case wasm::kExprI32AsmjsLoadMem16U:
858 return BuildAsmjsLoadMem(MachineType::Uint16(), input);
859 case wasm::kExprI32AsmjsLoadMem:
860 return BuildAsmjsLoadMem(MachineType::Int32(), input);
861 case wasm::kExprF32AsmjsLoadMem:
862 return BuildAsmjsLoadMem(MachineType::Float32(), input);
863 case wasm::kExprF64AsmjsLoadMem:
864 return BuildAsmjsLoadMem(MachineType::Float64(), input);
865 default:
866 FATAL_UNSUPPORTED_OPCODE(opcode);
867 }
868 return graph()->NewNode(op, input);
869 }
870
Float32Constant(float value)871 Node* WasmGraphBuilder::Float32Constant(float value) {
872 return mcgraph()->Float32Constant(value);
873 }
874
Float64Constant(double value)875 Node* WasmGraphBuilder::Float64Constant(double value) {
876 return mcgraph()->Float64Constant(value);
877 }
878
879 namespace {
Branch(MachineGraph * mcgraph,Node * cond,Node ** true_node,Node ** false_node,Node * control,BranchHint hint)880 Node* Branch(MachineGraph* mcgraph, Node* cond, Node** true_node,
881 Node** false_node, Node* control, BranchHint hint) {
882 DCHECK_NOT_NULL(cond);
883 DCHECK_NOT_NULL(control);
884 Node* branch =
885 mcgraph->graph()->NewNode(mcgraph->common()->Branch(hint), cond, control);
886 *true_node = mcgraph->graph()->NewNode(mcgraph->common()->IfTrue(), branch);
887 *false_node = mcgraph->graph()->NewNode(mcgraph->common()->IfFalse(), branch);
888 return branch;
889 }
890 } // namespace
891
BranchNoHint(Node * cond,Node ** true_node,Node ** false_node)892 Node* WasmGraphBuilder::BranchNoHint(Node* cond, Node** true_node,
893 Node** false_node) {
894 return Branch(mcgraph(), cond, true_node, false_node, Control(),
895 BranchHint::kNone);
896 }
897
BranchExpectTrue(Node * cond,Node ** true_node,Node ** false_node)898 Node* WasmGraphBuilder::BranchExpectTrue(Node* cond, Node** true_node,
899 Node** false_node) {
900 return Branch(mcgraph(), cond, true_node, false_node, Control(),
901 BranchHint::kTrue);
902 }
903
BranchExpectFalse(Node * cond,Node ** true_node,Node ** false_node)904 Node* WasmGraphBuilder::BranchExpectFalse(Node* cond, Node** true_node,
905 Node** false_node) {
906 return Branch(mcgraph(), cond, true_node, false_node, Control(),
907 BranchHint::kFalse);
908 }
909
GetTrapIdForTrap(wasm::TrapReason reason)910 TrapId WasmGraphBuilder::GetTrapIdForTrap(wasm::TrapReason reason) {
911 // TODO(wasm): "!env_" should not happen when compiling an actual wasm
912 // function.
913 if (!env_ || !env_->runtime_exception_support) {
914 // We use TrapId::kInvalid as a marker to tell the code generator
915 // to generate a call to a testing c-function instead of a runtime
916 // stub. This code should only be called from a cctest.
917 return TrapId::kInvalid;
918 }
919
920 switch (reason) {
921 #define TRAPREASON_TO_TRAPID(name) \
922 case wasm::k##name: \
923 static_assert( \
924 static_cast<int>(TrapId::k##name) == wasm::WasmCode::kThrowWasm##name, \
925 "trap id mismatch"); \
926 return TrapId::k##name;
927 FOREACH_WASM_TRAPREASON(TRAPREASON_TO_TRAPID)
928 #undef TRAPREASON_TO_TRAPID
929 default:
930 UNREACHABLE();
931 }
932 }
933
TrapIfTrue(wasm::TrapReason reason,Node * cond,wasm::WasmCodePosition position)934 Node* WasmGraphBuilder::TrapIfTrue(wasm::TrapReason reason, Node* cond,
935 wasm::WasmCodePosition position) {
936 TrapId trap_id = GetTrapIdForTrap(reason);
937 Node* node = SetControl(graph()->NewNode(mcgraph()->common()->TrapIf(trap_id),
938 cond, Effect(), Control()));
939 SetSourcePosition(node, position);
940 return node;
941 }
942
TrapIfFalse(wasm::TrapReason reason,Node * cond,wasm::WasmCodePosition position)943 Node* WasmGraphBuilder::TrapIfFalse(wasm::TrapReason reason, Node* cond,
944 wasm::WasmCodePosition position) {
945 TrapId trap_id = GetTrapIdForTrap(reason);
946 Node* node = SetControl(graph()->NewNode(
947 mcgraph()->common()->TrapUnless(trap_id), cond, Effect(), Control()));
948 SetSourcePosition(node, position);
949 return node;
950 }
951
952 // Add a check that traps if {node} is equal to {val}.
TrapIfEq32(wasm::TrapReason reason,Node * node,int32_t val,wasm::WasmCodePosition position)953 Node* WasmGraphBuilder::TrapIfEq32(wasm::TrapReason reason, Node* node,
954 int32_t val,
955 wasm::WasmCodePosition position) {
956 Int32Matcher m(node);
957 if (m.HasValue() && !m.Is(val)) return graph()->start();
958 if (val == 0) {
959 return TrapIfFalse(reason, node, position);
960 } else {
961 return TrapIfTrue(reason,
962 graph()->NewNode(mcgraph()->machine()->Word32Equal(),
963 node, mcgraph()->Int32Constant(val)),
964 position);
965 }
966 }
967
968 // Add a check that traps if {node} is zero.
ZeroCheck32(wasm::TrapReason reason,Node * node,wasm::WasmCodePosition position)969 Node* WasmGraphBuilder::ZeroCheck32(wasm::TrapReason reason, Node* node,
970 wasm::WasmCodePosition position) {
971 return TrapIfEq32(reason, node, 0, position);
972 }
973
974 // Add a check that traps if {node} is equal to {val}.
TrapIfEq64(wasm::TrapReason reason,Node * node,int64_t val,wasm::WasmCodePosition position)975 Node* WasmGraphBuilder::TrapIfEq64(wasm::TrapReason reason, Node* node,
976 int64_t val,
977 wasm::WasmCodePosition position) {
978 Int64Matcher m(node);
979 if (m.HasValue() && !m.Is(val)) return graph()->start();
980 return TrapIfTrue(reason,
981 graph()->NewNode(mcgraph()->machine()->Word64Equal(), node,
982 mcgraph()->Int64Constant(val)),
983 position);
984 }
985
986 // Add a check that traps if {node} is zero.
ZeroCheck64(wasm::TrapReason reason,Node * node,wasm::WasmCodePosition position)987 Node* WasmGraphBuilder::ZeroCheck64(wasm::TrapReason reason, Node* node,
988 wasm::WasmCodePosition position) {
989 return TrapIfEq64(reason, node, 0, position);
990 }
991
Switch(unsigned count,Node * key)992 Node* WasmGraphBuilder::Switch(unsigned count, Node* key) {
993 return graph()->NewNode(mcgraph()->common()->Switch(count), key, Control());
994 }
995
IfValue(int32_t value,Node * sw)996 Node* WasmGraphBuilder::IfValue(int32_t value, Node* sw) {
997 DCHECK_EQ(IrOpcode::kSwitch, sw->opcode());
998 return graph()->NewNode(mcgraph()->common()->IfValue(value), sw);
999 }
1000
IfDefault(Node * sw)1001 Node* WasmGraphBuilder::IfDefault(Node* sw) {
1002 DCHECK_EQ(IrOpcode::kSwitch, sw->opcode());
1003 return graph()->NewNode(mcgraph()->common()->IfDefault(), sw);
1004 }
1005
Return(unsigned count,Node ** vals)1006 Node* WasmGraphBuilder::Return(unsigned count, Node** vals) {
1007 static const int kStackAllocatedNodeBufferSize = 8;
1008 Node* stack_buffer[kStackAllocatedNodeBufferSize];
1009 std::vector<Node*> heap_buffer;
1010
1011 Node** buf = stack_buffer;
1012 if (count + 3 > kStackAllocatedNodeBufferSize) {
1013 heap_buffer.resize(count + 3);
1014 buf = heap_buffer.data();
1015 }
1016
1017 buf[0] = mcgraph()->Int32Constant(0);
1018 memcpy(buf + 1, vals, sizeof(void*) * count);
1019 buf[count + 1] = Effect();
1020 buf[count + 2] = Control();
1021 Node* ret =
1022 graph()->NewNode(mcgraph()->common()->Return(count), count + 3, buf);
1023
1024 MergeControlToEnd(mcgraph(), ret);
1025 return ret;
1026 }
1027
ReturnVoid()1028 Node* WasmGraphBuilder::ReturnVoid() { return Return(0, nullptr); }
1029
Unreachable(wasm::WasmCodePosition position)1030 Node* WasmGraphBuilder::Unreachable(wasm::WasmCodePosition position) {
1031 TrapIfFalse(wasm::TrapReason::kTrapUnreachable, Int32Constant(0), position);
1032 ReturnVoid();
1033 return nullptr;
1034 }
1035
MaskShiftCount32(Node * node)1036 Node* WasmGraphBuilder::MaskShiftCount32(Node* node) {
1037 static const int32_t kMask32 = 0x1F;
1038 if (!mcgraph()->machine()->Word32ShiftIsSafe()) {
1039 // Shifts by constants are so common we pattern-match them here.
1040 Int32Matcher match(node);
1041 if (match.HasValue()) {
1042 int32_t masked = (match.Value() & kMask32);
1043 if (match.Value() != masked) node = mcgraph()->Int32Constant(masked);
1044 } else {
1045 node = graph()->NewNode(mcgraph()->machine()->Word32And(), node,
1046 mcgraph()->Int32Constant(kMask32));
1047 }
1048 }
1049 return node;
1050 }
1051
MaskShiftCount64(Node * node)1052 Node* WasmGraphBuilder::MaskShiftCount64(Node* node) {
1053 static const int64_t kMask64 = 0x3F;
1054 if (!mcgraph()->machine()->Word32ShiftIsSafe()) {
1055 // Shifts by constants are so common we pattern-match them here.
1056 Int64Matcher match(node);
1057 if (match.HasValue()) {
1058 int64_t masked = (match.Value() & kMask64);
1059 if (match.Value() != masked) node = mcgraph()->Int64Constant(masked);
1060 } else {
1061 node = graph()->NewNode(mcgraph()->machine()->Word64And(), node,
1062 mcgraph()->Int64Constant(kMask64));
1063 }
1064 }
1065 return node;
1066 }
1067
ReverseBytesSupported(MachineOperatorBuilder * m,size_t size_in_bytes)1068 static bool ReverseBytesSupported(MachineOperatorBuilder* m,
1069 size_t size_in_bytes) {
1070 switch (size_in_bytes) {
1071 case 4:
1072 case 16:
1073 return true;
1074 case 8:
1075 return m->Is64();
1076 default:
1077 break;
1078 }
1079 return false;
1080 }
1081
BuildChangeEndiannessStore(Node * node,MachineRepresentation mem_rep,wasm::ValueType wasmtype)1082 Node* WasmGraphBuilder::BuildChangeEndiannessStore(
1083 Node* node, MachineRepresentation mem_rep, wasm::ValueType wasmtype) {
1084 Node* result;
1085 Node* value = node;
1086 MachineOperatorBuilder* m = mcgraph()->machine();
1087 int valueSizeInBytes = wasm::ValueTypes::ElementSizeInBytes(wasmtype);
1088 int valueSizeInBits = 8 * valueSizeInBytes;
1089 bool isFloat = false;
1090
1091 switch (wasmtype) {
1092 case wasm::kWasmF64:
1093 value = graph()->NewNode(m->BitcastFloat64ToInt64(), node);
1094 isFloat = true;
1095 V8_FALLTHROUGH;
1096 case wasm::kWasmI64:
1097 result = mcgraph()->Int64Constant(0);
1098 break;
1099 case wasm::kWasmF32:
1100 value = graph()->NewNode(m->BitcastFloat32ToInt32(), node);
1101 isFloat = true;
1102 V8_FALLTHROUGH;
1103 case wasm::kWasmI32:
1104 result = mcgraph()->Int32Constant(0);
1105 break;
1106 case wasm::kWasmS128:
1107 DCHECK(ReverseBytesSupported(m, valueSizeInBytes));
1108 break;
1109 default:
1110 UNREACHABLE();
1111 break;
1112 }
1113
1114 if (mem_rep == MachineRepresentation::kWord8) {
1115 // No need to change endianness for byte size, return original node
1116 return node;
1117 }
1118 if (wasmtype == wasm::kWasmI64 && mem_rep < MachineRepresentation::kWord64) {
1119 // In case we store lower part of WasmI64 expression, we can truncate
1120 // upper 32bits
1121 value = graph()->NewNode(m->TruncateInt64ToInt32(), value);
1122 valueSizeInBytes = wasm::ValueTypes::ElementSizeInBytes(wasm::kWasmI32);
1123 valueSizeInBits = 8 * valueSizeInBytes;
1124 if (mem_rep == MachineRepresentation::kWord16) {
1125 value =
1126 graph()->NewNode(m->Word32Shl(), value, mcgraph()->Int32Constant(16));
1127 }
1128 } else if (wasmtype == wasm::kWasmI32 &&
1129 mem_rep == MachineRepresentation::kWord16) {
1130 value =
1131 graph()->NewNode(m->Word32Shl(), value, mcgraph()->Int32Constant(16));
1132 }
1133
1134 int i;
1135 uint32_t shiftCount;
1136
1137 if (ReverseBytesSupported(m, valueSizeInBytes)) {
1138 switch (valueSizeInBytes) {
1139 case 4:
1140 result = graph()->NewNode(m->Word32ReverseBytes(), value);
1141 break;
1142 case 8:
1143 result = graph()->NewNode(m->Word64ReverseBytes(), value);
1144 break;
1145 case 16: {
1146 Node* byte_reversed_lanes[4];
1147 for (int lane = 0; lane < 4; lane++) {
1148 byte_reversed_lanes[lane] = graph()->NewNode(
1149 m->Word32ReverseBytes(),
1150 graph()->NewNode(mcgraph()->machine()->I32x4ExtractLane(lane),
1151 value));
1152 }
1153
1154 // This is making a copy of the value.
1155 result =
1156 graph()->NewNode(mcgraph()->machine()->S128And(), value, value);
1157
1158 for (int lane = 0; lane < 4; lane++) {
1159 result =
1160 graph()->NewNode(mcgraph()->machine()->I32x4ReplaceLane(3 - lane),
1161 result, byte_reversed_lanes[lane]);
1162 }
1163
1164 break;
1165 }
1166 default:
1167 UNREACHABLE();
1168 break;
1169 }
1170 } else {
1171 for (i = 0, shiftCount = valueSizeInBits - 8; i < valueSizeInBits / 2;
1172 i += 8, shiftCount -= 16) {
1173 Node* shiftLower;
1174 Node* shiftHigher;
1175 Node* lowerByte;
1176 Node* higherByte;
1177
1178 DCHECK_LT(0, shiftCount);
1179 DCHECK_EQ(0, (shiftCount + 8) % 16);
1180
1181 if (valueSizeInBits > 32) {
1182 shiftLower = graph()->NewNode(m->Word64Shl(), value,
1183 mcgraph()->Int64Constant(shiftCount));
1184 shiftHigher = graph()->NewNode(m->Word64Shr(), value,
1185 mcgraph()->Int64Constant(shiftCount));
1186 lowerByte = graph()->NewNode(
1187 m->Word64And(), shiftLower,
1188 mcgraph()->Int64Constant(static_cast<uint64_t>(0xFF)
1189 << (valueSizeInBits - 8 - i)));
1190 higherByte = graph()->NewNode(
1191 m->Word64And(), shiftHigher,
1192 mcgraph()->Int64Constant(static_cast<uint64_t>(0xFF) << i));
1193 result = graph()->NewNode(m->Word64Or(), result, lowerByte);
1194 result = graph()->NewNode(m->Word64Or(), result, higherByte);
1195 } else {
1196 shiftLower = graph()->NewNode(m->Word32Shl(), value,
1197 mcgraph()->Int32Constant(shiftCount));
1198 shiftHigher = graph()->NewNode(m->Word32Shr(), value,
1199 mcgraph()->Int32Constant(shiftCount));
1200 lowerByte = graph()->NewNode(
1201 m->Word32And(), shiftLower,
1202 mcgraph()->Int32Constant(static_cast<uint32_t>(0xFF)
1203 << (valueSizeInBits - 8 - i)));
1204 higherByte = graph()->NewNode(
1205 m->Word32And(), shiftHigher,
1206 mcgraph()->Int32Constant(static_cast<uint32_t>(0xFF) << i));
1207 result = graph()->NewNode(m->Word32Or(), result, lowerByte);
1208 result = graph()->NewNode(m->Word32Or(), result, higherByte);
1209 }
1210 }
1211 }
1212
1213 if (isFloat) {
1214 switch (wasmtype) {
1215 case wasm::kWasmF64:
1216 result = graph()->NewNode(m->BitcastInt64ToFloat64(), result);
1217 break;
1218 case wasm::kWasmF32:
1219 result = graph()->NewNode(m->BitcastInt32ToFloat32(), result);
1220 break;
1221 default:
1222 UNREACHABLE();
1223 break;
1224 }
1225 }
1226
1227 return result;
1228 }
1229
BuildChangeEndiannessLoad(Node * node,MachineType memtype,wasm::ValueType wasmtype)1230 Node* WasmGraphBuilder::BuildChangeEndiannessLoad(Node* node,
1231 MachineType memtype,
1232 wasm::ValueType wasmtype) {
1233 Node* result;
1234 Node* value = node;
1235 MachineOperatorBuilder* m = mcgraph()->machine();
1236 int valueSizeInBytes = ElementSizeInBytes(memtype.representation());
1237 int valueSizeInBits = 8 * valueSizeInBytes;
1238 bool isFloat = false;
1239
1240 switch (memtype.representation()) {
1241 case MachineRepresentation::kFloat64:
1242 value = graph()->NewNode(m->BitcastFloat64ToInt64(), node);
1243 isFloat = true;
1244 V8_FALLTHROUGH;
1245 case MachineRepresentation::kWord64:
1246 result = mcgraph()->Int64Constant(0);
1247 break;
1248 case MachineRepresentation::kFloat32:
1249 value = graph()->NewNode(m->BitcastFloat32ToInt32(), node);
1250 isFloat = true;
1251 V8_FALLTHROUGH;
1252 case MachineRepresentation::kWord32:
1253 case MachineRepresentation::kWord16:
1254 result = mcgraph()->Int32Constant(0);
1255 break;
1256 case MachineRepresentation::kWord8:
1257 // No need to change endianness for byte size, return original node
1258 return node;
1259 break;
1260 case MachineRepresentation::kSimd128:
1261 DCHECK(ReverseBytesSupported(m, valueSizeInBytes));
1262 break;
1263 default:
1264 UNREACHABLE();
1265 break;
1266 }
1267
1268 int i;
1269 uint32_t shiftCount;
1270
1271 if (ReverseBytesSupported(m, valueSizeInBytes < 4 ? 4 : valueSizeInBytes)) {
1272 switch (valueSizeInBytes) {
1273 case 2:
1274 result =
1275 graph()->NewNode(m->Word32ReverseBytes(),
1276 graph()->NewNode(m->Word32Shl(), value,
1277 mcgraph()->Int32Constant(16)));
1278 break;
1279 case 4:
1280 result = graph()->NewNode(m->Word32ReverseBytes(), value);
1281 break;
1282 case 8:
1283 result = graph()->NewNode(m->Word64ReverseBytes(), value);
1284 break;
1285 case 16: {
1286 Node* byte_reversed_lanes[4];
1287 for (int lane = 0; lane < 4; lane++) {
1288 byte_reversed_lanes[lane] = graph()->NewNode(
1289 m->Word32ReverseBytes(),
1290 graph()->NewNode(mcgraph()->machine()->I32x4ExtractLane(lane),
1291 value));
1292 }
1293
1294 // This is making a copy of the value.
1295 result =
1296 graph()->NewNode(mcgraph()->machine()->S128And(), value, value);
1297
1298 for (int lane = 0; lane < 4; lane++) {
1299 result =
1300 graph()->NewNode(mcgraph()->machine()->I32x4ReplaceLane(3 - lane),
1301 result, byte_reversed_lanes[lane]);
1302 }
1303
1304 break;
1305 }
1306 default:
1307 UNREACHABLE();
1308 }
1309 } else {
1310 for (i = 0, shiftCount = valueSizeInBits - 8; i < valueSizeInBits / 2;
1311 i += 8, shiftCount -= 16) {
1312 Node* shiftLower;
1313 Node* shiftHigher;
1314 Node* lowerByte;
1315 Node* higherByte;
1316
1317 DCHECK_LT(0, shiftCount);
1318 DCHECK_EQ(0, (shiftCount + 8) % 16);
1319
1320 if (valueSizeInBits > 32) {
1321 shiftLower = graph()->NewNode(m->Word64Shl(), value,
1322 mcgraph()->Int64Constant(shiftCount));
1323 shiftHigher = graph()->NewNode(m->Word64Shr(), value,
1324 mcgraph()->Int64Constant(shiftCount));
1325 lowerByte = graph()->NewNode(
1326 m->Word64And(), shiftLower,
1327 mcgraph()->Int64Constant(static_cast<uint64_t>(0xFF)
1328 << (valueSizeInBits - 8 - i)));
1329 higherByte = graph()->NewNode(
1330 m->Word64And(), shiftHigher,
1331 mcgraph()->Int64Constant(static_cast<uint64_t>(0xFF) << i));
1332 result = graph()->NewNode(m->Word64Or(), result, lowerByte);
1333 result = graph()->NewNode(m->Word64Or(), result, higherByte);
1334 } else {
1335 shiftLower = graph()->NewNode(m->Word32Shl(), value,
1336 mcgraph()->Int32Constant(shiftCount));
1337 shiftHigher = graph()->NewNode(m->Word32Shr(), value,
1338 mcgraph()->Int32Constant(shiftCount));
1339 lowerByte = graph()->NewNode(
1340 m->Word32And(), shiftLower,
1341 mcgraph()->Int32Constant(static_cast<uint32_t>(0xFF)
1342 << (valueSizeInBits - 8 - i)));
1343 higherByte = graph()->NewNode(
1344 m->Word32And(), shiftHigher,
1345 mcgraph()->Int32Constant(static_cast<uint32_t>(0xFF) << i));
1346 result = graph()->NewNode(m->Word32Or(), result, lowerByte);
1347 result = graph()->NewNode(m->Word32Or(), result, higherByte);
1348 }
1349 }
1350 }
1351
1352 if (isFloat) {
1353 switch (memtype.representation()) {
1354 case MachineRepresentation::kFloat64:
1355 result = graph()->NewNode(m->BitcastInt64ToFloat64(), result);
1356 break;
1357 case MachineRepresentation::kFloat32:
1358 result = graph()->NewNode(m->BitcastInt32ToFloat32(), result);
1359 break;
1360 default:
1361 UNREACHABLE();
1362 break;
1363 }
1364 }
1365
1366 // We need to sign extend the value
1367 if (memtype.IsSigned()) {
1368 DCHECK(!isFloat);
1369 if (valueSizeInBits < 32) {
1370 Node* shiftBitCount;
1371 // Perform sign extension using following trick
1372 // result = (x << machine_width - type_width) >> (machine_width -
1373 // type_width)
1374 if (wasmtype == wasm::kWasmI64) {
1375 shiftBitCount = mcgraph()->Int32Constant(64 - valueSizeInBits);
1376 result = graph()->NewNode(
1377 m->Word64Sar(),
1378 graph()->NewNode(m->Word64Shl(),
1379 graph()->NewNode(m->ChangeInt32ToInt64(), result),
1380 shiftBitCount),
1381 shiftBitCount);
1382 } else if (wasmtype == wasm::kWasmI32) {
1383 shiftBitCount = mcgraph()->Int32Constant(32 - valueSizeInBits);
1384 result = graph()->NewNode(
1385 m->Word32Sar(),
1386 graph()->NewNode(m->Word32Shl(), result, shiftBitCount),
1387 shiftBitCount);
1388 }
1389 }
1390 }
1391
1392 return result;
1393 }
1394
BuildF32CopySign(Node * left,Node * right)1395 Node* WasmGraphBuilder::BuildF32CopySign(Node* left, Node* right) {
1396 Node* result = Unop(
1397 wasm::kExprF32ReinterpretI32,
1398 Binop(wasm::kExprI32Ior,
1399 Binop(wasm::kExprI32And, Unop(wasm::kExprI32ReinterpretF32, left),
1400 mcgraph()->Int32Constant(0x7FFFFFFF)),
1401 Binop(wasm::kExprI32And, Unop(wasm::kExprI32ReinterpretF32, right),
1402 mcgraph()->Int32Constant(0x80000000))));
1403
1404 return result;
1405 }
1406
BuildF64CopySign(Node * left,Node * right)1407 Node* WasmGraphBuilder::BuildF64CopySign(Node* left, Node* right) {
1408 #if WASM_64
1409 Node* result = Unop(
1410 wasm::kExprF64ReinterpretI64,
1411 Binop(wasm::kExprI64Ior,
1412 Binop(wasm::kExprI64And, Unop(wasm::kExprI64ReinterpretF64, left),
1413 mcgraph()->Int64Constant(0x7FFFFFFFFFFFFFFF)),
1414 Binop(wasm::kExprI64And, Unop(wasm::kExprI64ReinterpretF64, right),
1415 mcgraph()->Int64Constant(0x8000000000000000))));
1416
1417 return result;
1418 #else
1419 MachineOperatorBuilder* m = mcgraph()->machine();
1420
1421 Node* high_word_left = graph()->NewNode(m->Float64ExtractHighWord32(), left);
1422 Node* high_word_right =
1423 graph()->NewNode(m->Float64ExtractHighWord32(), right);
1424
1425 Node* new_high_word = Binop(wasm::kExprI32Ior,
1426 Binop(wasm::kExprI32And, high_word_left,
1427 mcgraph()->Int32Constant(0x7FFFFFFF)),
1428 Binop(wasm::kExprI32And, high_word_right,
1429 mcgraph()->Int32Constant(0x80000000)));
1430
1431 return graph()->NewNode(m->Float64InsertHighWord32(), left, new_high_word);
1432 #endif
1433 }
1434
1435 namespace {
1436
IntConvertType(wasm::WasmOpcode opcode)1437 MachineType IntConvertType(wasm::WasmOpcode opcode) {
1438 switch (opcode) {
1439 case wasm::kExprI32SConvertF32:
1440 case wasm::kExprI32SConvertF64:
1441 case wasm::kExprI32SConvertSatF32:
1442 case wasm::kExprI32SConvertSatF64:
1443 return MachineType::Int32();
1444 case wasm::kExprI32UConvertF32:
1445 case wasm::kExprI32UConvertF64:
1446 case wasm::kExprI32UConvertSatF32:
1447 case wasm::kExprI32UConvertSatF64:
1448 return MachineType::Uint32();
1449 case wasm::kExprI64SConvertF32:
1450 case wasm::kExprI64SConvertF64:
1451 case wasm::kExprI64SConvertSatF32:
1452 case wasm::kExprI64SConvertSatF64:
1453 return MachineType::Int64();
1454 case wasm::kExprI64UConvertF32:
1455 case wasm::kExprI64UConvertF64:
1456 case wasm::kExprI64UConvertSatF32:
1457 case wasm::kExprI64UConvertSatF64:
1458 return MachineType::Uint64();
1459 default:
1460 UNREACHABLE();
1461 }
1462 }
1463
FloatConvertType(wasm::WasmOpcode opcode)1464 MachineType FloatConvertType(wasm::WasmOpcode opcode) {
1465 switch (opcode) {
1466 case wasm::kExprI32SConvertF32:
1467 case wasm::kExprI32UConvertF32:
1468 case wasm::kExprI32SConvertSatF32:
1469 case wasm::kExprI64SConvertF32:
1470 case wasm::kExprI64UConvertF32:
1471 case wasm::kExprI32UConvertSatF32:
1472 case wasm::kExprI64SConvertSatF32:
1473 case wasm::kExprI64UConvertSatF32:
1474 return MachineType::Float32();
1475 case wasm::kExprI32SConvertF64:
1476 case wasm::kExprI32UConvertF64:
1477 case wasm::kExprI64SConvertF64:
1478 case wasm::kExprI64UConvertF64:
1479 case wasm::kExprI32SConvertSatF64:
1480 case wasm::kExprI32UConvertSatF64:
1481 case wasm::kExprI64SConvertSatF64:
1482 case wasm::kExprI64UConvertSatF64:
1483 return MachineType::Float64();
1484 default:
1485 UNREACHABLE();
1486 }
1487 }
1488
ConvertOp(WasmGraphBuilder * builder,wasm::WasmOpcode opcode)1489 const Operator* ConvertOp(WasmGraphBuilder* builder, wasm::WasmOpcode opcode) {
1490 switch (opcode) {
1491 case wasm::kExprI32SConvertF32:
1492 case wasm::kExprI32SConvertSatF32:
1493 return builder->mcgraph()->machine()->TruncateFloat32ToInt32();
1494 case wasm::kExprI32UConvertF32:
1495 case wasm::kExprI32UConvertSatF32:
1496 return builder->mcgraph()->machine()->TruncateFloat32ToUint32();
1497 case wasm::kExprI32SConvertF64:
1498 case wasm::kExprI32SConvertSatF64:
1499 return builder->mcgraph()->machine()->ChangeFloat64ToInt32();
1500 case wasm::kExprI32UConvertF64:
1501 case wasm::kExprI32UConvertSatF64:
1502 return builder->mcgraph()->machine()->TruncateFloat64ToUint32();
1503 case wasm::kExprI64SConvertF32:
1504 case wasm::kExprI64SConvertSatF32:
1505 return builder->mcgraph()->machine()->TryTruncateFloat32ToInt64();
1506 case wasm::kExprI64UConvertF32:
1507 case wasm::kExprI64UConvertSatF32:
1508 return builder->mcgraph()->machine()->TryTruncateFloat32ToUint64();
1509 case wasm::kExprI64SConvertF64:
1510 case wasm::kExprI64SConvertSatF64:
1511 return builder->mcgraph()->machine()->TryTruncateFloat64ToInt64();
1512 case wasm::kExprI64UConvertF64:
1513 case wasm::kExprI64UConvertSatF64:
1514 return builder->mcgraph()->machine()->TryTruncateFloat64ToUint64();
1515 default:
1516 UNREACHABLE();
1517 }
1518 }
1519
ConvertBackOp(wasm::WasmOpcode opcode)1520 wasm::WasmOpcode ConvertBackOp(wasm::WasmOpcode opcode) {
1521 switch (opcode) {
1522 case wasm::kExprI32SConvertF32:
1523 case wasm::kExprI32SConvertSatF32:
1524 return wasm::kExprF32SConvertI32;
1525 case wasm::kExprI32UConvertF32:
1526 case wasm::kExprI32UConvertSatF32:
1527 return wasm::kExprF32UConvertI32;
1528 case wasm::kExprI32SConvertF64:
1529 case wasm::kExprI32SConvertSatF64:
1530 return wasm::kExprF64SConvertI32;
1531 case wasm::kExprI32UConvertF64:
1532 case wasm::kExprI32UConvertSatF64:
1533 return wasm::kExprF64UConvertI32;
1534 default:
1535 UNREACHABLE();
1536 }
1537 }
1538
IsTrappingConvertOp(wasm::WasmOpcode opcode)1539 bool IsTrappingConvertOp(wasm::WasmOpcode opcode) {
1540 switch (opcode) {
1541 case wasm::kExprI32SConvertF32:
1542 case wasm::kExprI32UConvertF32:
1543 case wasm::kExprI32SConvertF64:
1544 case wasm::kExprI32UConvertF64:
1545 case wasm::kExprI64SConvertF32:
1546 case wasm::kExprI64UConvertF32:
1547 case wasm::kExprI64SConvertF64:
1548 case wasm::kExprI64UConvertF64:
1549 return true;
1550 case wasm::kExprI32SConvertSatF64:
1551 case wasm::kExprI32UConvertSatF64:
1552 case wasm::kExprI32SConvertSatF32:
1553 case wasm::kExprI32UConvertSatF32:
1554 case wasm::kExprI64SConvertSatF32:
1555 case wasm::kExprI64UConvertSatF32:
1556 case wasm::kExprI64SConvertSatF64:
1557 case wasm::kExprI64UConvertSatF64:
1558 return false;
1559 default:
1560 UNREACHABLE();
1561 }
1562 }
1563
Zero(WasmGraphBuilder * builder,const MachineType & ty)1564 Node* Zero(WasmGraphBuilder* builder, const MachineType& ty) {
1565 switch (ty.representation()) {
1566 case MachineRepresentation::kWord32:
1567 return builder->Int32Constant(0);
1568 case MachineRepresentation::kWord64:
1569 return builder->Int64Constant(0);
1570 case MachineRepresentation::kFloat32:
1571 return builder->Float32Constant(0.0);
1572 case MachineRepresentation::kFloat64:
1573 return builder->Float64Constant(0.0);
1574 default:
1575 UNREACHABLE();
1576 }
1577 }
1578
Min(WasmGraphBuilder * builder,const MachineType & ty)1579 Node* Min(WasmGraphBuilder* builder, const MachineType& ty) {
1580 switch (ty.semantic()) {
1581 case MachineSemantic::kInt32:
1582 return builder->Int32Constant(std::numeric_limits<int32_t>::min());
1583 case MachineSemantic::kUint32:
1584 return builder->Int32Constant(std::numeric_limits<uint32_t>::min());
1585 case MachineSemantic::kInt64:
1586 return builder->Int64Constant(std::numeric_limits<int64_t>::min());
1587 case MachineSemantic::kUint64:
1588 return builder->Int64Constant(std::numeric_limits<uint64_t>::min());
1589 default:
1590 UNREACHABLE();
1591 }
1592 }
1593
Max(WasmGraphBuilder * builder,const MachineType & ty)1594 Node* Max(WasmGraphBuilder* builder, const MachineType& ty) {
1595 switch (ty.semantic()) {
1596 case MachineSemantic::kInt32:
1597 return builder->Int32Constant(std::numeric_limits<int32_t>::max());
1598 case MachineSemantic::kUint32:
1599 return builder->Int32Constant(std::numeric_limits<uint32_t>::max());
1600 case MachineSemantic::kInt64:
1601 return builder->Int64Constant(std::numeric_limits<int64_t>::max());
1602 case MachineSemantic::kUint64:
1603 return builder->Int64Constant(std::numeric_limits<uint64_t>::max());
1604 default:
1605 UNREACHABLE();
1606 }
1607 }
1608
TruncOp(const MachineType & ty)1609 wasm::WasmOpcode TruncOp(const MachineType& ty) {
1610 switch (ty.representation()) {
1611 case MachineRepresentation::kFloat32:
1612 return wasm::kExprF32Trunc;
1613 case MachineRepresentation::kFloat64:
1614 return wasm::kExprF64Trunc;
1615 default:
1616 UNREACHABLE();
1617 }
1618 }
1619
NeOp(const MachineType & ty)1620 wasm::WasmOpcode NeOp(const MachineType& ty) {
1621 switch (ty.representation()) {
1622 case MachineRepresentation::kFloat32:
1623 return wasm::kExprF32Ne;
1624 case MachineRepresentation::kFloat64:
1625 return wasm::kExprF64Ne;
1626 default:
1627 UNREACHABLE();
1628 }
1629 }
1630
LtOp(const MachineType & ty)1631 wasm::WasmOpcode LtOp(const MachineType& ty) {
1632 switch (ty.representation()) {
1633 case MachineRepresentation::kFloat32:
1634 return wasm::kExprF32Lt;
1635 case MachineRepresentation::kFloat64:
1636 return wasm::kExprF64Lt;
1637 default:
1638 UNREACHABLE();
1639 }
1640 }
1641
ConvertTrapTest(WasmGraphBuilder * builder,wasm::WasmOpcode opcode,const MachineType & int_ty,const MachineType & float_ty,Node * trunc,Node * converted_value)1642 Node* ConvertTrapTest(WasmGraphBuilder* builder, wasm::WasmOpcode opcode,
1643 const MachineType& int_ty, const MachineType& float_ty,
1644 Node* trunc, Node* converted_value) {
1645 if (int_ty.representation() == MachineRepresentation::kWord32) {
1646 Node* check = builder->Unop(ConvertBackOp(opcode), converted_value);
1647 return builder->Binop(NeOp(float_ty), trunc, check);
1648 }
1649 return builder->graph()->NewNode(builder->mcgraph()->common()->Projection(1),
1650 trunc, builder->graph()->start());
1651 }
1652
ConvertSaturateTest(WasmGraphBuilder * builder,wasm::WasmOpcode opcode,const MachineType & int_ty,const MachineType & float_ty,Node * trunc,Node * converted_value)1653 Node* ConvertSaturateTest(WasmGraphBuilder* builder, wasm::WasmOpcode opcode,
1654 const MachineType& int_ty,
1655 const MachineType& float_ty, Node* trunc,
1656 Node* converted_value) {
1657 Node* test = ConvertTrapTest(builder, opcode, int_ty, float_ty, trunc,
1658 converted_value);
1659 if (int_ty.representation() == MachineRepresentation::kWord64) {
1660 test = builder->Binop(wasm::kExprI64Eq, test, builder->Int64Constant(0));
1661 }
1662 return test;
1663 }
1664
1665 } // namespace
1666
BuildIntConvertFloat(Node * input,wasm::WasmCodePosition position,wasm::WasmOpcode opcode)1667 Node* WasmGraphBuilder::BuildIntConvertFloat(Node* input,
1668 wasm::WasmCodePosition position,
1669 wasm::WasmOpcode opcode) {
1670 const MachineType int_ty = IntConvertType(opcode);
1671 const MachineType float_ty = FloatConvertType(opcode);
1672 const Operator* conv_op = ConvertOp(this, opcode);
1673 Node* trunc = nullptr;
1674 Node* converted_value = nullptr;
1675 const bool is_int32 =
1676 int_ty.representation() == MachineRepresentation::kWord32;
1677 if (is_int32) {
1678 trunc = Unop(TruncOp(float_ty), input);
1679 converted_value = graph()->NewNode(conv_op, trunc);
1680 } else {
1681 trunc = graph()->NewNode(conv_op, input);
1682 converted_value = graph()->NewNode(mcgraph()->common()->Projection(0),
1683 trunc, graph()->start());
1684 }
1685 if (IsTrappingConvertOp(opcode)) {
1686 Node* test =
1687 ConvertTrapTest(this, opcode, int_ty, float_ty, trunc, converted_value);
1688 if (is_int32) {
1689 TrapIfTrue(wasm::kTrapFloatUnrepresentable, test, position);
1690 } else {
1691 ZeroCheck64(wasm::kTrapFloatUnrepresentable, test, position);
1692 }
1693 return converted_value;
1694 }
1695 Node* test = ConvertSaturateTest(this, opcode, int_ty, float_ty, trunc,
1696 converted_value);
1697 Diamond tl_d(graph(), mcgraph()->common(), test, BranchHint::kFalse);
1698 tl_d.Chain(Control());
1699 Node* nan_test = Binop(NeOp(float_ty), input, input);
1700 Diamond nan_d(graph(), mcgraph()->common(), nan_test, BranchHint::kFalse);
1701 nan_d.Nest(tl_d, true);
1702 Node* neg_test = Binop(LtOp(float_ty), input, Zero(this, float_ty));
1703 Diamond sat_d(graph(), mcgraph()->common(), neg_test, BranchHint::kNone);
1704 sat_d.Nest(nan_d, false);
1705 Node* sat_val =
1706 sat_d.Phi(int_ty.representation(), Min(this, int_ty), Max(this, int_ty));
1707 Node* nan_val =
1708 nan_d.Phi(int_ty.representation(), Zero(this, int_ty), sat_val);
1709 return tl_d.Phi(int_ty.representation(), nan_val, converted_value);
1710 }
1711
BuildI32AsmjsSConvertF32(Node * input)1712 Node* WasmGraphBuilder::BuildI32AsmjsSConvertF32(Node* input) {
1713 MachineOperatorBuilder* m = mcgraph()->machine();
1714 // asm.js must use the wacky JS semantics.
1715 input = graph()->NewNode(m->ChangeFloat32ToFloat64(), input);
1716 return graph()->NewNode(m->TruncateFloat64ToWord32(), input);
1717 }
1718
BuildI32AsmjsSConvertF64(Node * input)1719 Node* WasmGraphBuilder::BuildI32AsmjsSConvertF64(Node* input) {
1720 MachineOperatorBuilder* m = mcgraph()->machine();
1721 // asm.js must use the wacky JS semantics.
1722 return graph()->NewNode(m->TruncateFloat64ToWord32(), input);
1723 }
1724
BuildI32AsmjsUConvertF32(Node * input)1725 Node* WasmGraphBuilder::BuildI32AsmjsUConvertF32(Node* input) {
1726 MachineOperatorBuilder* m = mcgraph()->machine();
1727 // asm.js must use the wacky JS semantics.
1728 input = graph()->NewNode(m->ChangeFloat32ToFloat64(), input);
1729 return graph()->NewNode(m->TruncateFloat64ToWord32(), input);
1730 }
1731
BuildI32AsmjsUConvertF64(Node * input)1732 Node* WasmGraphBuilder::BuildI32AsmjsUConvertF64(Node* input) {
1733 MachineOperatorBuilder* m = mcgraph()->machine();
1734 // asm.js must use the wacky JS semantics.
1735 return graph()->NewNode(m->TruncateFloat64ToWord32(), input);
1736 }
1737
BuildBitCountingCall(Node * input,ExternalReference ref,MachineRepresentation input_type)1738 Node* WasmGraphBuilder::BuildBitCountingCall(Node* input, ExternalReference ref,
1739 MachineRepresentation input_type) {
1740 Node* stack_slot_param =
1741 graph()->NewNode(mcgraph()->machine()->StackSlot(input_type));
1742
1743 const Operator* store_op = mcgraph()->machine()->Store(
1744 StoreRepresentation(input_type, kNoWriteBarrier));
1745 SetEffect(graph()->NewNode(store_op, stack_slot_param,
1746 mcgraph()->Int32Constant(0), input, Effect(),
1747 Control()));
1748
1749 MachineType sig_types[] = {MachineType::Int32(), MachineType::Pointer()};
1750 MachineSignature sig(1, 1, sig_types);
1751
1752 Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(ref));
1753
1754 return BuildCCall(&sig, function, stack_slot_param);
1755 }
1756
BuildI32Ctz(Node * input)1757 Node* WasmGraphBuilder::BuildI32Ctz(Node* input) {
1758 return BuildBitCountingCall(input, ExternalReference::wasm_word32_ctz(),
1759 MachineRepresentation::kWord32);
1760 }
1761
BuildI64Ctz(Node * input)1762 Node* WasmGraphBuilder::BuildI64Ctz(Node* input) {
1763 return Unop(wasm::kExprI64UConvertI32,
1764 BuildBitCountingCall(input, ExternalReference::wasm_word64_ctz(),
1765 MachineRepresentation::kWord64));
1766 }
1767
BuildI32Popcnt(Node * input)1768 Node* WasmGraphBuilder::BuildI32Popcnt(Node* input) {
1769 return BuildBitCountingCall(input, ExternalReference::wasm_word32_popcnt(),
1770 MachineRepresentation::kWord32);
1771 }
1772
BuildI64Popcnt(Node * input)1773 Node* WasmGraphBuilder::BuildI64Popcnt(Node* input) {
1774 return Unop(
1775 wasm::kExprI64UConvertI32,
1776 BuildBitCountingCall(input, ExternalReference::wasm_word64_popcnt(),
1777 MachineRepresentation::kWord64));
1778 }
1779
BuildF32Trunc(Node * input)1780 Node* WasmGraphBuilder::BuildF32Trunc(Node* input) {
1781 MachineType type = MachineType::Float32();
1782 ExternalReference ref = ExternalReference::wasm_f32_trunc();
1783
1784 return BuildCFuncInstruction(ref, type, input);
1785 }
1786
BuildF32Floor(Node * input)1787 Node* WasmGraphBuilder::BuildF32Floor(Node* input) {
1788 MachineType type = MachineType::Float32();
1789 ExternalReference ref = ExternalReference::wasm_f32_floor();
1790 return BuildCFuncInstruction(ref, type, input);
1791 }
1792
BuildF32Ceil(Node * input)1793 Node* WasmGraphBuilder::BuildF32Ceil(Node* input) {
1794 MachineType type = MachineType::Float32();
1795 ExternalReference ref = ExternalReference::wasm_f32_ceil();
1796 return BuildCFuncInstruction(ref, type, input);
1797 }
1798
BuildF32NearestInt(Node * input)1799 Node* WasmGraphBuilder::BuildF32NearestInt(Node* input) {
1800 MachineType type = MachineType::Float32();
1801 ExternalReference ref = ExternalReference::wasm_f32_nearest_int();
1802 return BuildCFuncInstruction(ref, type, input);
1803 }
1804
BuildF64Trunc(Node * input)1805 Node* WasmGraphBuilder::BuildF64Trunc(Node* input) {
1806 MachineType type = MachineType::Float64();
1807 ExternalReference ref = ExternalReference::wasm_f64_trunc();
1808 return BuildCFuncInstruction(ref, type, input);
1809 }
1810
BuildF64Floor(Node * input)1811 Node* WasmGraphBuilder::BuildF64Floor(Node* input) {
1812 MachineType type = MachineType::Float64();
1813 ExternalReference ref = ExternalReference::wasm_f64_floor();
1814 return BuildCFuncInstruction(ref, type, input);
1815 }
1816
BuildF64Ceil(Node * input)1817 Node* WasmGraphBuilder::BuildF64Ceil(Node* input) {
1818 MachineType type = MachineType::Float64();
1819 ExternalReference ref = ExternalReference::wasm_f64_ceil();
1820 return BuildCFuncInstruction(ref, type, input);
1821 }
1822
BuildF64NearestInt(Node * input)1823 Node* WasmGraphBuilder::BuildF64NearestInt(Node* input) {
1824 MachineType type = MachineType::Float64();
1825 ExternalReference ref = ExternalReference::wasm_f64_nearest_int();
1826 return BuildCFuncInstruction(ref, type, input);
1827 }
1828
BuildF64Acos(Node * input)1829 Node* WasmGraphBuilder::BuildF64Acos(Node* input) {
1830 MachineType type = MachineType::Float64();
1831 ExternalReference ref = ExternalReference::f64_acos_wrapper_function();
1832 return BuildCFuncInstruction(ref, type, input);
1833 }
1834
BuildF64Asin(Node * input)1835 Node* WasmGraphBuilder::BuildF64Asin(Node* input) {
1836 MachineType type = MachineType::Float64();
1837 ExternalReference ref = ExternalReference::f64_asin_wrapper_function();
1838 return BuildCFuncInstruction(ref, type, input);
1839 }
1840
BuildF64Pow(Node * left,Node * right)1841 Node* WasmGraphBuilder::BuildF64Pow(Node* left, Node* right) {
1842 MachineType type = MachineType::Float64();
1843 ExternalReference ref = ExternalReference::wasm_float64_pow();
1844 return BuildCFuncInstruction(ref, type, left, right);
1845 }
1846
BuildF64Mod(Node * left,Node * right)1847 Node* WasmGraphBuilder::BuildF64Mod(Node* left, Node* right) {
1848 MachineType type = MachineType::Float64();
1849 ExternalReference ref = ExternalReference::f64_mod_wrapper_function();
1850 return BuildCFuncInstruction(ref, type, left, right);
1851 }
1852
BuildCFuncInstruction(ExternalReference ref,MachineType type,Node * input0,Node * input1)1853 Node* WasmGraphBuilder::BuildCFuncInstruction(ExternalReference ref,
1854 MachineType type, Node* input0,
1855 Node* input1) {
1856 // We do truncation by calling a C function which calculates the result.
1857 // The input is passed to the C function as a byte buffer holding the two
1858 // input doubles. We reserve this byte buffer as a stack slot, store the
1859 // parameters in this buffer slots, pass a pointer to the buffer to the C
1860 // function, and after calling the C function we collect the return value from
1861 // the buffer.
1862
1863 const int type_size = ElementSizeInBytes(type.representation());
1864 const int stack_slot_bytes = (input1 == nullptr ? 1 : 2) * type_size;
1865 Node* stack_slot =
1866 graph()->NewNode(mcgraph()->machine()->StackSlot(stack_slot_bytes));
1867
1868 const Operator* store_op = mcgraph()->machine()->Store(
1869 StoreRepresentation(type.representation(), kNoWriteBarrier));
1870 SetEffect(graph()->NewNode(store_op, stack_slot, mcgraph()->Int32Constant(0),
1871 input0, Effect(), Control()));
1872
1873 Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(ref));
1874
1875 if (input1 != nullptr) {
1876 SetEffect(graph()->NewNode(store_op, stack_slot,
1877 mcgraph()->Int32Constant(type_size), input1,
1878 Effect(), Control()));
1879 }
1880
1881 MachineType sig_types[] = {MachineType::Pointer()};
1882 MachineSignature sig(0, 1, sig_types);
1883 BuildCCall(&sig, function, stack_slot);
1884
1885 return SetEffect(graph()->NewNode(mcgraph()->machine()->Load(type),
1886 stack_slot, mcgraph()->Int32Constant(0),
1887 Effect(), Control()));
1888 }
1889
BuildF32SConvertI64(Node * input)1890 Node* WasmGraphBuilder::BuildF32SConvertI64(Node* input) {
1891 // TODO(titzer/bradnelson): Check handlng of asm.js case.
1892 return BuildIntToFloatConversionInstruction(
1893 input, ExternalReference::wasm_int64_to_float32(),
1894 MachineRepresentation::kWord64, MachineType::Float32());
1895 }
BuildF32UConvertI64(Node * input)1896 Node* WasmGraphBuilder::BuildF32UConvertI64(Node* input) {
1897 // TODO(titzer/bradnelson): Check handlng of asm.js case.
1898 return BuildIntToFloatConversionInstruction(
1899 input, ExternalReference::wasm_uint64_to_float32(),
1900 MachineRepresentation::kWord64, MachineType::Float32());
1901 }
BuildF64SConvertI64(Node * input)1902 Node* WasmGraphBuilder::BuildF64SConvertI64(Node* input) {
1903 return BuildIntToFloatConversionInstruction(
1904 input, ExternalReference::wasm_int64_to_float64(),
1905 MachineRepresentation::kWord64, MachineType::Float64());
1906 }
BuildF64UConvertI64(Node * input)1907 Node* WasmGraphBuilder::BuildF64UConvertI64(Node* input) {
1908 return BuildIntToFloatConversionInstruction(
1909 input, ExternalReference::wasm_uint64_to_float64(),
1910 MachineRepresentation::kWord64, MachineType::Float64());
1911 }
1912
BuildIntToFloatConversionInstruction(Node * input,ExternalReference ref,MachineRepresentation parameter_representation,const MachineType result_type)1913 Node* WasmGraphBuilder::BuildIntToFloatConversionInstruction(
1914 Node* input, ExternalReference ref,
1915 MachineRepresentation parameter_representation,
1916 const MachineType result_type) {
1917 int stack_slot_size =
1918 std::max(ElementSizeInBytes(parameter_representation),
1919 ElementSizeInBytes(result_type.representation()));
1920 Node* stack_slot =
1921 graph()->NewNode(mcgraph()->machine()->StackSlot(stack_slot_size));
1922 const Operator* store_op = mcgraph()->machine()->Store(
1923 StoreRepresentation(parameter_representation, kNoWriteBarrier));
1924 SetEffect(graph()->NewNode(store_op, stack_slot, mcgraph()->Int32Constant(0),
1925 input, Effect(), Control()));
1926 MachineType sig_types[] = {MachineType::Pointer()};
1927 MachineSignature sig(0, 1, sig_types);
1928 Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(ref));
1929 BuildCCall(&sig, function, stack_slot);
1930 return SetEffect(graph()->NewNode(mcgraph()->machine()->Load(result_type),
1931 stack_slot, mcgraph()->Int32Constant(0),
1932 Effect(), Control()));
1933 }
1934
1935 namespace {
1936
convert_ccall_ref(WasmGraphBuilder * builder,wasm::WasmOpcode opcode)1937 ExternalReference convert_ccall_ref(WasmGraphBuilder* builder,
1938 wasm::WasmOpcode opcode) {
1939 switch (opcode) {
1940 case wasm::kExprI64SConvertF32:
1941 case wasm::kExprI64SConvertSatF32:
1942 return ExternalReference::wasm_float32_to_int64();
1943 case wasm::kExprI64UConvertF32:
1944 case wasm::kExprI64UConvertSatF32:
1945 return ExternalReference::wasm_float32_to_uint64();
1946 case wasm::kExprI64SConvertF64:
1947 case wasm::kExprI64SConvertSatF64:
1948 return ExternalReference::wasm_float64_to_int64();
1949 case wasm::kExprI64UConvertF64:
1950 case wasm::kExprI64UConvertSatF64:
1951 return ExternalReference::wasm_float64_to_uint64();
1952 default:
1953 UNREACHABLE();
1954 }
1955 }
1956
1957 } // namespace
1958
BuildCcallConvertFloat(Node * input,wasm::WasmCodePosition position,wasm::WasmOpcode opcode)1959 Node* WasmGraphBuilder::BuildCcallConvertFloat(Node* input,
1960 wasm::WasmCodePosition position,
1961 wasm::WasmOpcode opcode) {
1962 const MachineType int_ty = IntConvertType(opcode);
1963 const MachineType float_ty = FloatConvertType(opcode);
1964 ExternalReference call_ref = convert_ccall_ref(this, opcode);
1965 int stack_slot_size = std::max(ElementSizeInBytes(int_ty.representation()),
1966 ElementSizeInBytes(float_ty.representation()));
1967 Node* stack_slot =
1968 graph()->NewNode(mcgraph()->machine()->StackSlot(stack_slot_size));
1969 const Operator* store_op = mcgraph()->machine()->Store(
1970 StoreRepresentation(float_ty.representation(), kNoWriteBarrier));
1971 SetEffect(graph()->NewNode(store_op, stack_slot, Int32Constant(0), input,
1972 Effect(), Control()));
1973 MachineType sig_types[] = {MachineType::Int32(), MachineType::Pointer()};
1974 MachineSignature sig(1, 1, sig_types);
1975 Node* function =
1976 graph()->NewNode(mcgraph()->common()->ExternalConstant(call_ref));
1977 Node* overflow = BuildCCall(&sig, function, stack_slot);
1978 if (IsTrappingConvertOp(opcode)) {
1979 ZeroCheck32(wasm::kTrapFloatUnrepresentable, overflow, position);
1980 return SetEffect(graph()->NewNode(mcgraph()->machine()->Load(int_ty),
1981 stack_slot, Int32Constant(0), Effect(),
1982 Control()));
1983 }
1984 Node* test = Binop(wasm::kExprI32Eq, overflow, Int32Constant(0), position);
1985 Diamond tl_d(graph(), mcgraph()->common(), test, BranchHint::kFalse);
1986 tl_d.Chain(Control());
1987 Node* nan_test = Binop(NeOp(float_ty), input, input);
1988 Diamond nan_d(graph(), mcgraph()->common(), nan_test, BranchHint::kFalse);
1989 nan_d.Nest(tl_d, true);
1990 Node* neg_test = Binop(LtOp(float_ty), input, Zero(this, float_ty));
1991 Diamond sat_d(graph(), mcgraph()->common(), neg_test, BranchHint::kNone);
1992 sat_d.Nest(nan_d, false);
1993 Node* sat_val =
1994 sat_d.Phi(int_ty.representation(), Min(this, int_ty), Max(this, int_ty));
1995 Node* load =
1996 SetEffect(graph()->NewNode(mcgraph()->machine()->Load(int_ty), stack_slot,
1997 Int32Constant(0), Effect(), Control()));
1998 Node* nan_val =
1999 nan_d.Phi(int_ty.representation(), Zero(this, int_ty), sat_val);
2000 return tl_d.Phi(int_ty.representation(), nan_val, load);
2001 }
2002
GrowMemory(Node * input)2003 Node* WasmGraphBuilder::GrowMemory(Node* input) {
2004 SetNeedsStackCheck();
2005
2006 WasmGrowMemoryDescriptor interface_descriptor;
2007 auto call_descriptor = Linkage::GetStubCallDescriptor(
2008 mcgraph()->zone(), // zone
2009 interface_descriptor, // descriptor
2010 interface_descriptor.GetStackParameterCount(), // stack parameter count
2011 CallDescriptor::kNoFlags, // flags
2012 Operator::kNoProperties, // properties
2013 StubCallMode::kCallWasmRuntimeStub); // stub call mode
2014 // A direct call to a wasm runtime stub defined in this module.
2015 // Just encode the stub index. This will be patched at relocation.
2016 Node* call_target = mcgraph()->RelocatableIntPtrConstant(
2017 wasm::WasmCode::kWasmGrowMemory, RelocInfo::WASM_STUB_CALL);
2018 return SetEffect(
2019 SetControl(graph()->NewNode(mcgraph()->common()->Call(call_descriptor),
2020 call_target, input, Effect(), Control())));
2021 }
2022
GetExceptionEncodedSize(const wasm::WasmException * exception) const2023 uint32_t WasmGraphBuilder::GetExceptionEncodedSize(
2024 const wasm::WasmException* exception) const {
2025 const wasm::WasmExceptionSig* sig = exception->sig;
2026 uint32_t encoded_size = 0;
2027 for (size_t i = 0; i < sig->parameter_count(); ++i) {
2028 size_t byte_size = static_cast<size_t>(
2029 wasm::ValueTypes::ElementSizeInBytes(sig->GetParam(i)));
2030 DCHECK_EQ(byte_size % kBytesPerExceptionValuesArrayElement, 0);
2031 DCHECK_LE(1, byte_size / kBytesPerExceptionValuesArrayElement);
2032 encoded_size += byte_size / kBytesPerExceptionValuesArrayElement;
2033 }
2034 return encoded_size;
2035 }
2036
Throw(uint32_t tag,const wasm::WasmException * exception,const Vector<Node * > values)2037 Node* WasmGraphBuilder::Throw(uint32_t tag,
2038 const wasm::WasmException* exception,
2039 const Vector<Node*> values) {
2040 SetNeedsStackCheck();
2041 uint32_t encoded_size = GetExceptionEncodedSize(exception);
2042 Node* create_parameters[] = {
2043 BuildChangeUint31ToSmi(ConvertExceptionTagToRuntimeId(tag)),
2044 BuildChangeUint31ToSmi(Uint32Constant(encoded_size))};
2045 BuildCallToRuntime(Runtime::kWasmThrowCreate, create_parameters,
2046 arraysize(create_parameters));
2047 uint32_t index = 0;
2048 const wasm::WasmExceptionSig* sig = exception->sig;
2049 MachineOperatorBuilder* m = mcgraph()->machine();
2050 for (size_t i = 0; i < sig->parameter_count(); ++i) {
2051 Node* value = values[i];
2052 switch (sig->GetParam(i)) {
2053 case wasm::kWasmF32:
2054 value = graph()->NewNode(m->BitcastFloat32ToInt32(), value);
2055 V8_FALLTHROUGH;
2056 case wasm::kWasmI32:
2057 BuildEncodeException32BitValue(&index, value);
2058 break;
2059 case wasm::kWasmF64:
2060 value = graph()->NewNode(m->BitcastFloat64ToInt64(), value);
2061 V8_FALLTHROUGH;
2062 case wasm::kWasmI64: {
2063 Node* upper32 = graph()->NewNode(
2064 m->TruncateInt64ToInt32(),
2065 Binop(wasm::kExprI64ShrU, value, Int64Constant(32)));
2066 BuildEncodeException32BitValue(&index, upper32);
2067 Node* lower32 = graph()->NewNode(m->TruncateInt64ToInt32(), value);
2068 BuildEncodeException32BitValue(&index, lower32);
2069 break;
2070 }
2071 default:
2072 UNREACHABLE();
2073 }
2074 }
2075 DCHECK_EQ(encoded_size, index);
2076 return BuildCallToRuntime(Runtime::kWasmThrow, nullptr, 0);
2077 }
2078
BuildEncodeException32BitValue(uint32_t * index,Node * value)2079 void WasmGraphBuilder::BuildEncodeException32BitValue(uint32_t* index,
2080 Node* value) {
2081 MachineOperatorBuilder* machine = mcgraph()->machine();
2082 Node* upper_parameters[] = {
2083 BuildChangeUint31ToSmi(Int32Constant(*index)),
2084 BuildChangeUint31ToSmi(
2085 graph()->NewNode(machine->Word32Shr(), value, Int32Constant(16))),
2086 };
2087 BuildCallToRuntime(Runtime::kWasmExceptionSetElement, upper_parameters,
2088 arraysize(upper_parameters));
2089 ++(*index);
2090 Node* lower_parameters[] = {
2091 BuildChangeUint31ToSmi(Int32Constant(*index)),
2092 BuildChangeUint31ToSmi(graph()->NewNode(machine->Word32And(), value,
2093 Int32Constant(0xFFFFu))),
2094 };
2095 BuildCallToRuntime(Runtime::kWasmExceptionSetElement, lower_parameters,
2096 arraysize(lower_parameters));
2097 ++(*index);
2098 }
2099
BuildDecodeException32BitValue(Node * const * values,uint32_t * index)2100 Node* WasmGraphBuilder::BuildDecodeException32BitValue(Node* const* values,
2101 uint32_t* index) {
2102 MachineOperatorBuilder* machine = mcgraph()->machine();
2103 Node* upper = BuildChangeSmiToInt32(values[*index]);
2104 (*index)++;
2105 upper = graph()->NewNode(machine->Word32Shl(), upper, Int32Constant(16));
2106 Node* lower = BuildChangeSmiToInt32(values[*index]);
2107 (*index)++;
2108 Node* value = graph()->NewNode(machine->Word32Or(), upper, lower);
2109 return value;
2110 }
2111
Rethrow()2112 Node* WasmGraphBuilder::Rethrow() {
2113 SetNeedsStackCheck();
2114 Node* result = BuildCallToRuntime(Runtime::kWasmThrow, nullptr, 0);
2115 return result;
2116 }
2117
ConvertExceptionTagToRuntimeId(uint32_t tag)2118 Node* WasmGraphBuilder::ConvertExceptionTagToRuntimeId(uint32_t tag) {
2119 // TODO(kschimpf): Handle exceptions from different modules, when they are
2120 // linked at runtime.
2121 return Uint32Constant(tag);
2122 }
2123
GetExceptionRuntimeId()2124 Node* WasmGraphBuilder::GetExceptionRuntimeId() {
2125 SetNeedsStackCheck();
2126 return BuildChangeSmiToInt32(
2127 BuildCallToRuntime(Runtime::kWasmGetExceptionRuntimeId, nullptr, 0));
2128 }
2129
GetExceptionValues(const wasm::WasmException * except_decl)2130 Node** WasmGraphBuilder::GetExceptionValues(
2131 const wasm::WasmException* except_decl) {
2132 // TODO(kschimpf): We need to move this code to the function-body-decoder.cc
2133 // in order to build landing-pad (exception) edges in case the runtime
2134 // call causes an exception.
2135
2136 // Start by getting the encoded values from the exception.
2137 uint32_t encoded_size = GetExceptionEncodedSize(except_decl);
2138 Node** values = Buffer(encoded_size);
2139 for (uint32_t i = 0; i < encoded_size; ++i) {
2140 Node* parameters[] = {BuildChangeUint31ToSmi(Uint32Constant(i))};
2141 values[i] = BuildCallToRuntime(Runtime::kWasmExceptionGetElement,
2142 parameters, arraysize(parameters));
2143 }
2144
2145 // Now convert the leading entries to the corresponding parameter values.
2146 uint32_t index = 0;
2147 const wasm::WasmExceptionSig* sig = except_decl->sig;
2148 for (size_t i = 0; i < sig->parameter_count(); ++i) {
2149 Node* value = BuildDecodeException32BitValue(values, &index);
2150 switch (wasm::ValueType type = sig->GetParam(i)) {
2151 case wasm::kWasmF32: {
2152 value = Unop(wasm::kExprF32ReinterpretI32, value);
2153 break;
2154 }
2155 case wasm::kWasmI32:
2156 break;
2157 case wasm::kWasmF64:
2158 case wasm::kWasmI64: {
2159 Node* upper =
2160 Binop(wasm::kExprI64Shl, Unop(wasm::kExprI64UConvertI32, value),
2161 Int64Constant(32));
2162 Node* lower = Unop(wasm::kExprI64UConvertI32,
2163 BuildDecodeException32BitValue(values, &index));
2164 value = Binop(wasm::kExprI64Ior, upper, lower);
2165 if (type == wasm::kWasmF64) {
2166 value = Unop(wasm::kExprF64ReinterpretI64, value);
2167 }
2168 break;
2169 }
2170 default:
2171 UNREACHABLE();
2172 }
2173 values[i] = value;
2174 }
2175 DCHECK_EQ(index, encoded_size);
2176 return values;
2177 }
2178
BuildI32DivS(Node * left,Node * right,wasm::WasmCodePosition position)2179 Node* WasmGraphBuilder::BuildI32DivS(Node* left, Node* right,
2180 wasm::WasmCodePosition position) {
2181 MachineOperatorBuilder* m = mcgraph()->machine();
2182 ZeroCheck32(wasm::kTrapDivByZero, right, position);
2183 Node* before = Control();
2184 Node* denom_is_m1;
2185 Node* denom_is_not_m1;
2186 BranchExpectFalse(
2187 graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(-1)),
2188 &denom_is_m1, &denom_is_not_m1);
2189 SetControl(denom_is_m1);
2190 TrapIfEq32(wasm::kTrapDivUnrepresentable, left, kMinInt, position);
2191 if (Control() != denom_is_m1) {
2192 SetControl(graph()->NewNode(mcgraph()->common()->Merge(2), denom_is_not_m1,
2193 Control()));
2194 } else {
2195 SetControl(before);
2196 }
2197 return graph()->NewNode(m->Int32Div(), left, right, Control());
2198 }
2199
BuildI32RemS(Node * left,Node * right,wasm::WasmCodePosition position)2200 Node* WasmGraphBuilder::BuildI32RemS(Node* left, Node* right,
2201 wasm::WasmCodePosition position) {
2202 MachineOperatorBuilder* m = mcgraph()->machine();
2203
2204 ZeroCheck32(wasm::kTrapRemByZero, right, position);
2205
2206 Diamond d(
2207 graph(), mcgraph()->common(),
2208 graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(-1)),
2209 BranchHint::kFalse);
2210 d.Chain(Control());
2211
2212 return d.Phi(MachineRepresentation::kWord32, mcgraph()->Int32Constant(0),
2213 graph()->NewNode(m->Int32Mod(), left, right, d.if_false));
2214 }
2215
BuildI32DivU(Node * left,Node * right,wasm::WasmCodePosition position)2216 Node* WasmGraphBuilder::BuildI32DivU(Node* left, Node* right,
2217 wasm::WasmCodePosition position) {
2218 MachineOperatorBuilder* m = mcgraph()->machine();
2219 return graph()->NewNode(m->Uint32Div(), left, right,
2220 ZeroCheck32(wasm::kTrapDivByZero, right, position));
2221 }
2222
BuildI32RemU(Node * left,Node * right,wasm::WasmCodePosition position)2223 Node* WasmGraphBuilder::BuildI32RemU(Node* left, Node* right,
2224 wasm::WasmCodePosition position) {
2225 MachineOperatorBuilder* m = mcgraph()->machine();
2226 return graph()->NewNode(m->Uint32Mod(), left, right,
2227 ZeroCheck32(wasm::kTrapRemByZero, right, position));
2228 }
2229
BuildI32AsmjsDivS(Node * left,Node * right)2230 Node* WasmGraphBuilder::BuildI32AsmjsDivS(Node* left, Node* right) {
2231 MachineOperatorBuilder* m = mcgraph()->machine();
2232
2233 Int32Matcher mr(right);
2234 if (mr.HasValue()) {
2235 if (mr.Value() == 0) {
2236 return mcgraph()->Int32Constant(0);
2237 } else if (mr.Value() == -1) {
2238 // The result is the negation of the left input.
2239 return graph()->NewNode(m->Int32Sub(), mcgraph()->Int32Constant(0), left);
2240 }
2241 return graph()->NewNode(m->Int32Div(), left, right, Control());
2242 }
2243
2244 // asm.js semantics return 0 on divide or mod by zero.
2245 if (m->Int32DivIsSafe()) {
2246 // The hardware instruction does the right thing (e.g. arm).
2247 return graph()->NewNode(m->Int32Div(), left, right, graph()->start());
2248 }
2249
2250 // Check denominator for zero.
2251 Diamond z(
2252 graph(), mcgraph()->common(),
2253 graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(0)),
2254 BranchHint::kFalse);
2255
2256 // Check numerator for -1. (avoid minint / -1 case).
2257 Diamond n(
2258 graph(), mcgraph()->common(),
2259 graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(-1)),
2260 BranchHint::kFalse);
2261
2262 Node* div = graph()->NewNode(m->Int32Div(), left, right, z.if_false);
2263 Node* neg =
2264 graph()->NewNode(m->Int32Sub(), mcgraph()->Int32Constant(0), left);
2265
2266 return n.Phi(
2267 MachineRepresentation::kWord32, neg,
2268 z.Phi(MachineRepresentation::kWord32, mcgraph()->Int32Constant(0), div));
2269 }
2270
BuildI32AsmjsRemS(Node * left,Node * right)2271 Node* WasmGraphBuilder::BuildI32AsmjsRemS(Node* left, Node* right) {
2272 CommonOperatorBuilder* c = mcgraph()->common();
2273 MachineOperatorBuilder* m = mcgraph()->machine();
2274 Node* const zero = mcgraph()->Int32Constant(0);
2275
2276 Int32Matcher mr(right);
2277 if (mr.HasValue()) {
2278 if (mr.Value() == 0 || mr.Value() == -1) {
2279 return zero;
2280 }
2281 return graph()->NewNode(m->Int32Mod(), left, right, Control());
2282 }
2283
2284 // General case for signed integer modulus, with optimization for (unknown)
2285 // power of 2 right hand side.
2286 //
2287 // if 0 < right then
2288 // msk = right - 1
2289 // if right & msk != 0 then
2290 // left % right
2291 // else
2292 // if left < 0 then
2293 // -(-left & msk)
2294 // else
2295 // left & msk
2296 // else
2297 // if right < -1 then
2298 // left % right
2299 // else
2300 // zero
2301 //
2302 // Note: We do not use the Diamond helper class here, because it really hurts
2303 // readability with nested diamonds.
2304 Node* const minus_one = mcgraph()->Int32Constant(-1);
2305
2306 const Operator* const merge_op = c->Merge(2);
2307 const Operator* const phi_op = c->Phi(MachineRepresentation::kWord32, 2);
2308
2309 Node* check0 = graph()->NewNode(m->Int32LessThan(), zero, right);
2310 Node* branch0 =
2311 graph()->NewNode(c->Branch(BranchHint::kTrue), check0, graph()->start());
2312
2313 Node* if_true0 = graph()->NewNode(c->IfTrue(), branch0);
2314 Node* true0;
2315 {
2316 Node* msk = graph()->NewNode(m->Int32Add(), right, minus_one);
2317
2318 Node* check1 = graph()->NewNode(m->Word32And(), right, msk);
2319 Node* branch1 = graph()->NewNode(c->Branch(), check1, if_true0);
2320
2321 Node* if_true1 = graph()->NewNode(c->IfTrue(), branch1);
2322 Node* true1 = graph()->NewNode(m->Int32Mod(), left, right, if_true1);
2323
2324 Node* if_false1 = graph()->NewNode(c->IfFalse(), branch1);
2325 Node* false1;
2326 {
2327 Node* check2 = graph()->NewNode(m->Int32LessThan(), left, zero);
2328 Node* branch2 =
2329 graph()->NewNode(c->Branch(BranchHint::kFalse), check2, if_false1);
2330
2331 Node* if_true2 = graph()->NewNode(c->IfTrue(), branch2);
2332 Node* true2 = graph()->NewNode(
2333 m->Int32Sub(), zero,
2334 graph()->NewNode(m->Word32And(),
2335 graph()->NewNode(m->Int32Sub(), zero, left), msk));
2336
2337 Node* if_false2 = graph()->NewNode(c->IfFalse(), branch2);
2338 Node* false2 = graph()->NewNode(m->Word32And(), left, msk);
2339
2340 if_false1 = graph()->NewNode(merge_op, if_true2, if_false2);
2341 false1 = graph()->NewNode(phi_op, true2, false2, if_false1);
2342 }
2343
2344 if_true0 = graph()->NewNode(merge_op, if_true1, if_false1);
2345 true0 = graph()->NewNode(phi_op, true1, false1, if_true0);
2346 }
2347
2348 Node* if_false0 = graph()->NewNode(c->IfFalse(), branch0);
2349 Node* false0;
2350 {
2351 Node* check1 = graph()->NewNode(m->Int32LessThan(), right, minus_one);
2352 Node* branch1 =
2353 graph()->NewNode(c->Branch(BranchHint::kTrue), check1, if_false0);
2354
2355 Node* if_true1 = graph()->NewNode(c->IfTrue(), branch1);
2356 Node* true1 = graph()->NewNode(m->Int32Mod(), left, right, if_true1);
2357
2358 Node* if_false1 = graph()->NewNode(c->IfFalse(), branch1);
2359 Node* false1 = zero;
2360
2361 if_false0 = graph()->NewNode(merge_op, if_true1, if_false1);
2362 false0 = graph()->NewNode(phi_op, true1, false1, if_false0);
2363 }
2364
2365 Node* merge0 = graph()->NewNode(merge_op, if_true0, if_false0);
2366 return graph()->NewNode(phi_op, true0, false0, merge0);
2367 }
2368
BuildI32AsmjsDivU(Node * left,Node * right)2369 Node* WasmGraphBuilder::BuildI32AsmjsDivU(Node* left, Node* right) {
2370 MachineOperatorBuilder* m = mcgraph()->machine();
2371 // asm.js semantics return 0 on divide or mod by zero.
2372 if (m->Uint32DivIsSafe()) {
2373 // The hardware instruction does the right thing (e.g. arm).
2374 return graph()->NewNode(m->Uint32Div(), left, right, graph()->start());
2375 }
2376
2377 // Explicit check for x % 0.
2378 Diamond z(
2379 graph(), mcgraph()->common(),
2380 graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(0)),
2381 BranchHint::kFalse);
2382
2383 return z.Phi(MachineRepresentation::kWord32, mcgraph()->Int32Constant(0),
2384 graph()->NewNode(mcgraph()->machine()->Uint32Div(), left, right,
2385 z.if_false));
2386 }
2387
BuildI32AsmjsRemU(Node * left,Node * right)2388 Node* WasmGraphBuilder::BuildI32AsmjsRemU(Node* left, Node* right) {
2389 MachineOperatorBuilder* m = mcgraph()->machine();
2390 // asm.js semantics return 0 on divide or mod by zero.
2391 // Explicit check for x % 0.
2392 Diamond z(
2393 graph(), mcgraph()->common(),
2394 graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(0)),
2395 BranchHint::kFalse);
2396
2397 Node* rem = graph()->NewNode(mcgraph()->machine()->Uint32Mod(), left, right,
2398 z.if_false);
2399 return z.Phi(MachineRepresentation::kWord32, mcgraph()->Int32Constant(0),
2400 rem);
2401 }
2402
BuildI64DivS(Node * left,Node * right,wasm::WasmCodePosition position)2403 Node* WasmGraphBuilder::BuildI64DivS(Node* left, Node* right,
2404 wasm::WasmCodePosition position) {
2405 if (mcgraph()->machine()->Is32()) {
2406 return BuildDiv64Call(left, right, ExternalReference::wasm_int64_div(),
2407 MachineType::Int64(), wasm::kTrapDivByZero, position);
2408 }
2409 ZeroCheck64(wasm::kTrapDivByZero, right, position);
2410 Node* before = Control();
2411 Node* denom_is_m1;
2412 Node* denom_is_not_m1;
2413 BranchExpectFalse(graph()->NewNode(mcgraph()->machine()->Word64Equal(), right,
2414 mcgraph()->Int64Constant(-1)),
2415 &denom_is_m1, &denom_is_not_m1);
2416 SetControl(denom_is_m1);
2417 TrapIfEq64(wasm::kTrapDivUnrepresentable, left,
2418 std::numeric_limits<int64_t>::min(), position);
2419 if (Control() != denom_is_m1) {
2420 SetControl(graph()->NewNode(mcgraph()->common()->Merge(2), denom_is_not_m1,
2421 Control()));
2422 } else {
2423 SetControl(before);
2424 }
2425 return graph()->NewNode(mcgraph()->machine()->Int64Div(), left, right,
2426 Control());
2427 }
2428
BuildI64RemS(Node * left,Node * right,wasm::WasmCodePosition position)2429 Node* WasmGraphBuilder::BuildI64RemS(Node* left, Node* right,
2430 wasm::WasmCodePosition position) {
2431 if (mcgraph()->machine()->Is32()) {
2432 return BuildDiv64Call(left, right, ExternalReference::wasm_int64_mod(),
2433 MachineType::Int64(), wasm::kTrapRemByZero, position);
2434 }
2435 ZeroCheck64(wasm::kTrapRemByZero, right, position);
2436 Diamond d(mcgraph()->graph(), mcgraph()->common(),
2437 graph()->NewNode(mcgraph()->machine()->Word64Equal(), right,
2438 mcgraph()->Int64Constant(-1)));
2439
2440 d.Chain(Control());
2441
2442 Node* rem = graph()->NewNode(mcgraph()->machine()->Int64Mod(), left, right,
2443 d.if_false);
2444
2445 return d.Phi(MachineRepresentation::kWord64, mcgraph()->Int64Constant(0),
2446 rem);
2447 }
2448
BuildI64DivU(Node * left,Node * right,wasm::WasmCodePosition position)2449 Node* WasmGraphBuilder::BuildI64DivU(Node* left, Node* right,
2450 wasm::WasmCodePosition position) {
2451 if (mcgraph()->machine()->Is32()) {
2452 return BuildDiv64Call(left, right, ExternalReference::wasm_uint64_div(),
2453 MachineType::Int64(), wasm::kTrapDivByZero, position);
2454 }
2455 return graph()->NewNode(mcgraph()->machine()->Uint64Div(), left, right,
2456 ZeroCheck64(wasm::kTrapDivByZero, right, position));
2457 }
BuildI64RemU(Node * left,Node * right,wasm::WasmCodePosition position)2458 Node* WasmGraphBuilder::BuildI64RemU(Node* left, Node* right,
2459 wasm::WasmCodePosition position) {
2460 if (mcgraph()->machine()->Is32()) {
2461 return BuildDiv64Call(left, right, ExternalReference::wasm_uint64_mod(),
2462 MachineType::Int64(), wasm::kTrapRemByZero, position);
2463 }
2464 return graph()->NewNode(mcgraph()->machine()->Uint64Mod(), left, right,
2465 ZeroCheck64(wasm::kTrapRemByZero, right, position));
2466 }
2467
BuildDiv64Call(Node * left,Node * right,ExternalReference ref,MachineType result_type,wasm::TrapReason trap_zero,wasm::WasmCodePosition position)2468 Node* WasmGraphBuilder::BuildDiv64Call(Node* left, Node* right,
2469 ExternalReference ref,
2470 MachineType result_type,
2471 wasm::TrapReason trap_zero,
2472 wasm::WasmCodePosition position) {
2473 Node* stack_slot =
2474 graph()->NewNode(mcgraph()->machine()->StackSlot(2 * sizeof(double)));
2475
2476 const Operator* store_op = mcgraph()->machine()->Store(
2477 StoreRepresentation(MachineRepresentation::kWord64, kNoWriteBarrier));
2478 SetEffect(graph()->NewNode(store_op, stack_slot, mcgraph()->Int32Constant(0),
2479 left, Effect(), Control()));
2480 SetEffect(graph()->NewNode(store_op, stack_slot,
2481 mcgraph()->Int32Constant(sizeof(double)), right,
2482 Effect(), Control()));
2483
2484 MachineType sig_types[] = {MachineType::Int32(), MachineType::Pointer()};
2485 MachineSignature sig(1, 1, sig_types);
2486
2487 Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(ref));
2488 Node* call = BuildCCall(&sig, function, stack_slot);
2489
2490 ZeroCheck32(trap_zero, call, position);
2491 TrapIfEq32(wasm::kTrapDivUnrepresentable, call, -1, position);
2492 return SetEffect(graph()->NewNode(mcgraph()->machine()->Load(result_type),
2493 stack_slot, mcgraph()->Int32Constant(0),
2494 Effect(), Control()));
2495 }
2496
2497 template <typename... Args>
BuildCCall(MachineSignature * sig,Node * function,Args...args)2498 Node* WasmGraphBuilder::BuildCCall(MachineSignature* sig, Node* function,
2499 Args... args) {
2500 DCHECK_LE(sig->return_count(), 1);
2501 DCHECK_EQ(sizeof...(args), sig->parameter_count());
2502 Node* const call_args[] = {function, args..., Effect(), Control()};
2503
2504 auto call_descriptor =
2505 Linkage::GetSimplifiedCDescriptor(mcgraph()->zone(), sig);
2506
2507 const Operator* op = mcgraph()->common()->Call(call_descriptor);
2508 return SetEffect(graph()->NewNode(op, arraysize(call_args), call_args));
2509 }
2510
BuildWasmCall(wasm::FunctionSig * sig,Node ** args,Node *** rets,wasm::WasmCodePosition position,Node * instance_node,UseRetpoline use_retpoline)2511 Node* WasmGraphBuilder::BuildWasmCall(wasm::FunctionSig* sig, Node** args,
2512 Node*** rets,
2513 wasm::WasmCodePosition position,
2514 Node* instance_node,
2515 UseRetpoline use_retpoline) {
2516 if (instance_node == nullptr) {
2517 DCHECK_NOT_NULL(instance_node_);
2518 instance_node = instance_node_.get();
2519 }
2520 SetNeedsStackCheck();
2521 const size_t params = sig->parameter_count();
2522 const size_t extra = 3; // instance_node, effect, and control.
2523 const size_t count = 1 + params + extra;
2524
2525 // Reallocate the buffer to make space for extra inputs.
2526 args = Realloc(args, 1 + params, count);
2527
2528 // Make room for the instance_node parameter at index 1, just after code.
2529 memmove(&args[2], &args[1], params * sizeof(Node*));
2530 args[1] = instance_node;
2531
2532 // Add effect and control inputs.
2533 args[params + 2] = Effect();
2534 args[params + 3] = Control();
2535
2536 auto call_descriptor =
2537 GetWasmCallDescriptor(mcgraph()->zone(), sig, use_retpoline);
2538 const Operator* op = mcgraph()->common()->Call(call_descriptor);
2539 Node* call = SetEffect(graph()->NewNode(op, static_cast<int>(count), args));
2540 DCHECK(position == wasm::kNoCodePosition || position > 0);
2541 if (position > 0) SetSourcePosition(call, position);
2542
2543 size_t ret_count = sig->return_count();
2544 if (ret_count == 0) return call; // No return value.
2545
2546 *rets = Buffer(ret_count);
2547 if (ret_count == 1) {
2548 // Only a single return value.
2549 (*rets)[0] = call;
2550 } else {
2551 // Create projections for all return values.
2552 for (size_t i = 0; i < ret_count; i++) {
2553 (*rets)[i] = graph()->NewNode(mcgraph()->common()->Projection(i), call,
2554 graph()->start());
2555 }
2556 }
2557 return call;
2558 }
2559
BuildImportWasmCall(wasm::FunctionSig * sig,Node ** args,Node *** rets,wasm::WasmCodePosition position,int func_index)2560 Node* WasmGraphBuilder::BuildImportWasmCall(wasm::FunctionSig* sig, Node** args,
2561 Node*** rets,
2562 wasm::WasmCodePosition position,
2563 int func_index) {
2564 // Load the instance from the imported_instances array at a known offset.
2565 Node* imported_instances = LOAD_INSTANCE_FIELD(ImportedFunctionInstances,
2566 MachineType::TaggedPointer());
2567 Node* instance_node = LOAD_FIXED_ARRAY_SLOT(imported_instances, func_index);
2568
2569 // Load the target from the imported_targets array at a known offset.
2570 Node* imported_targets =
2571 LOAD_INSTANCE_FIELD(ImportedFunctionTargets, MachineType::Pointer());
2572 Node* target_node = SetEffect(graph()->NewNode(
2573 mcgraph()->machine()->Load(MachineType::Pointer()), imported_targets,
2574 mcgraph()->Int32Constant(func_index * kPointerSize), Effect(),
2575 Control()));
2576 args[0] = target_node;
2577 return BuildWasmCall(sig, args, rets, position, instance_node,
2578 untrusted_code_mitigations_ ? kRetpoline : kNoRetpoline);
2579 }
2580
BuildImportWasmCall(wasm::FunctionSig * sig,Node ** args,Node *** rets,wasm::WasmCodePosition position,Node * func_index)2581 Node* WasmGraphBuilder::BuildImportWasmCall(wasm::FunctionSig* sig, Node** args,
2582 Node*** rets,
2583 wasm::WasmCodePosition position,
2584 Node* func_index) {
2585 // Load the instance from the imported_instances array.
2586 Node* imported_instances = LOAD_INSTANCE_FIELD(ImportedFunctionInstances,
2587 MachineType::TaggedPointer());
2588 // Access fixed array at {header_size - tag + func_index * kPointerSize}.
2589 Node* imported_instances_data =
2590 graph()->NewNode(mcgraph()->machine()->IntAdd(), imported_instances,
2591 mcgraph()->IntPtrConstant(FixedArrayOffsetMinusTag(0)));
2592 Node* func_index_times_pointersize = graph()->NewNode(
2593 mcgraph()->machine()->IntMul(), Uint32ToUintptr(func_index),
2594 mcgraph()->Int32Constant(kPointerSize));
2595 Node* instance_node = SetEffect(
2596 graph()->NewNode(mcgraph()->machine()->Load(MachineType::TaggedPointer()),
2597 imported_instances_data, func_index_times_pointersize,
2598 Effect(), Control()));
2599
2600 // Load the target from the imported_targets array at the offset of
2601 // {func_index}.
2602 Node* imported_targets =
2603 LOAD_INSTANCE_FIELD(ImportedFunctionTargets, MachineType::Pointer());
2604 Node* target_node = SetEffect(graph()->NewNode(
2605 mcgraph()->machine()->Load(MachineType::Pointer()), imported_targets,
2606 func_index_times_pointersize, Effect(), Control()));
2607 args[0] = target_node;
2608 return BuildWasmCall(sig, args, rets, position, instance_node,
2609 untrusted_code_mitigations_ ? kRetpoline : kNoRetpoline);
2610 }
2611
CallDirect(uint32_t index,Node ** args,Node *** rets,wasm::WasmCodePosition position)2612 Node* WasmGraphBuilder::CallDirect(uint32_t index, Node** args, Node*** rets,
2613 wasm::WasmCodePosition position) {
2614 DCHECK_NULL(args[0]);
2615 wasm::FunctionSig* sig = env_->module->functions[index].sig;
2616
2617 if (env_ && index < env_->module->num_imported_functions) {
2618 // Call to an imported function.
2619 return BuildImportWasmCall(sig, args, rets, position, index);
2620 }
2621
2622 // A direct call to a wasm function defined in this module.
2623 // Just encode the function index. This will be patched at instantiation.
2624 Address code = static_cast<Address>(index);
2625 args[0] = mcgraph()->RelocatableIntPtrConstant(code, RelocInfo::WASM_CALL);
2626
2627 return BuildWasmCall(sig, args, rets, position, nullptr, kNoRetpoline);
2628 }
2629
CallIndirect(uint32_t sig_index,Node ** args,Node *** rets,wasm::WasmCodePosition position)2630 Node* WasmGraphBuilder::CallIndirect(uint32_t sig_index, Node** args,
2631 Node*** rets,
2632 wasm::WasmCodePosition position) {
2633 DCHECK_NOT_NULL(args[0]);
2634 DCHECK_NOT_NULL(env_);
2635
2636 // Assume only one table for now.
2637 wasm::FunctionSig* sig = env_->module->signatures[sig_index];
2638
2639 Node* ift_size =
2640 LOAD_INSTANCE_FIELD(IndirectFunctionTableSize, MachineType::Uint32());
2641
2642 MachineOperatorBuilder* machine = mcgraph()->machine();
2643 Node* key = args[0];
2644
2645 // Bounds check against the table size.
2646 Node* in_bounds = graph()->NewNode(machine->Uint32LessThan(), key, ift_size);
2647 TrapIfFalse(wasm::kTrapFuncInvalid, in_bounds, position);
2648
2649 // Mask the key to prevent SSCA.
2650 if (untrusted_code_mitigations_) {
2651 // mask = ((key - size) & ~key) >> 31
2652 Node* neg_key =
2653 graph()->NewNode(machine->Word32Xor(), key, Int32Constant(-1));
2654 Node* masked_diff = graph()->NewNode(
2655 machine->Word32And(),
2656 graph()->NewNode(machine->Int32Sub(), key, ift_size), neg_key);
2657 Node* mask =
2658 graph()->NewNode(machine->Word32Sar(), masked_diff, Int32Constant(31));
2659 key = graph()->NewNode(machine->Word32And(), key, mask);
2660 }
2661
2662 // Load signature from the table and check.
2663 Node* ift_sig_ids =
2664 LOAD_INSTANCE_FIELD(IndirectFunctionTableSigIds, MachineType::Pointer());
2665
2666 int32_t expected_sig_id = env_->module->signature_ids[sig_index];
2667 Node* scaled_key = Uint32ToUintptr(
2668 graph()->NewNode(machine->Word32Shl(), key, Int32Constant(2)));
2669
2670 Node* loaded_sig =
2671 SetEffect(graph()->NewNode(machine->Load(MachineType::Int32()),
2672 ift_sig_ids, scaled_key, Effect(), Control()));
2673 Node* sig_match = graph()->NewNode(machine->WordEqual(), loaded_sig,
2674 Int32Constant(expected_sig_id));
2675
2676 TrapIfFalse(wasm::kTrapFuncSigMismatch, sig_match, position);
2677
2678 Node* ift_targets =
2679 LOAD_INSTANCE_FIELD(IndirectFunctionTableTargets, MachineType::Pointer());
2680 Node* ift_instances = LOAD_INSTANCE_FIELD(IndirectFunctionTableInstances,
2681 MachineType::TaggedPointer());
2682
2683 scaled_key = graph()->NewNode(machine->Word32Shl(), key,
2684 Int32Constant(kPointerSizeLog2));
2685
2686 Node* target =
2687 SetEffect(graph()->NewNode(machine->Load(MachineType::Pointer()),
2688 ift_targets, scaled_key, Effect(), Control()));
2689
2690 auto access = AccessBuilder::ForFixedArrayElement();
2691 Node* target_instance = SetEffect(graph()->NewNode(
2692 machine->Load(MachineType::TaggedPointer()),
2693 graph()->NewNode(machine->IntAdd(), ift_instances, scaled_key),
2694 Int32Constant(access.header_size - access.tag()), Effect(), Control()));
2695
2696 args[0] = target;
2697
2698 return BuildWasmCall(sig, args, rets, position, target_instance,
2699 untrusted_code_mitigations_ ? kRetpoline : kNoRetpoline);
2700 }
2701
BuildI32Rol(Node * left,Node * right)2702 Node* WasmGraphBuilder::BuildI32Rol(Node* left, Node* right) {
2703 // Implement Rol by Ror since TurboFan does not have Rol opcode.
2704 // TODO(weiliang): support Word32Rol opcode in TurboFan.
2705 Int32Matcher m(right);
2706 if (m.HasValue()) {
2707 return Binop(wasm::kExprI32Ror, left,
2708 mcgraph()->Int32Constant(32 - m.Value()));
2709 } else {
2710 return Binop(wasm::kExprI32Ror, left,
2711 Binop(wasm::kExprI32Sub, mcgraph()->Int32Constant(32), right));
2712 }
2713 }
2714
BuildI64Rol(Node * left,Node * right)2715 Node* WasmGraphBuilder::BuildI64Rol(Node* left, Node* right) {
2716 // Implement Rol by Ror since TurboFan does not have Rol opcode.
2717 // TODO(weiliang): support Word64Rol opcode in TurboFan.
2718 Int64Matcher m(right);
2719 if (m.HasValue()) {
2720 return Binop(wasm::kExprI64Ror, left,
2721 mcgraph()->Int64Constant(64 - m.Value()));
2722 } else {
2723 return Binop(wasm::kExprI64Ror, left,
2724 Binop(wasm::kExprI64Sub, mcgraph()->Int64Constant(64), right));
2725 }
2726 }
2727
Invert(Node * node)2728 Node* WasmGraphBuilder::Invert(Node* node) {
2729 return Unop(wasm::kExprI32Eqz, node);
2730 }
2731
CanCover(Node * value,IrOpcode::Value opcode)2732 bool CanCover(Node* value, IrOpcode::Value opcode) {
2733 if (value->opcode() != opcode) return false;
2734 bool first = true;
2735 for (Edge const edge : value->use_edges()) {
2736 if (NodeProperties::IsControlEdge(edge)) continue;
2737 if (NodeProperties::IsEffectEdge(edge)) continue;
2738 DCHECK(NodeProperties::IsValueEdge(edge));
2739 if (!first) return false;
2740 first = false;
2741 }
2742 return true;
2743 }
2744
BuildChangeInt32ToIntPtr(Node * value)2745 Node* WasmGraphBuilder::BuildChangeInt32ToIntPtr(Node* value) {
2746 if (mcgraph()->machine()->Is64()) {
2747 value = graph()->NewNode(mcgraph()->machine()->ChangeInt32ToInt64(), value);
2748 }
2749 return value;
2750 }
2751
BuildChangeInt32ToSmi(Node * value)2752 Node* WasmGraphBuilder::BuildChangeInt32ToSmi(Node* value) {
2753 value = BuildChangeInt32ToIntPtr(value);
2754 return graph()->NewNode(mcgraph()->machine()->WordShl(), value,
2755 BuildSmiShiftBitsConstant());
2756 }
2757
BuildChangeUint31ToSmi(Node * value)2758 Node* WasmGraphBuilder::BuildChangeUint31ToSmi(Node* value) {
2759 return graph()->NewNode(mcgraph()->machine()->WordShl(),
2760 Uint32ToUintptr(value), BuildSmiShiftBitsConstant());
2761 }
2762
BuildSmiShiftBitsConstant()2763 Node* WasmGraphBuilder::BuildSmiShiftBitsConstant() {
2764 return mcgraph()->IntPtrConstant(kSmiShiftSize + kSmiTagSize);
2765 }
2766
BuildChangeSmiToInt32(Node * value)2767 Node* WasmGraphBuilder::BuildChangeSmiToInt32(Node* value) {
2768 value = graph()->NewNode(mcgraph()->machine()->WordSar(), value,
2769 BuildSmiShiftBitsConstant());
2770 if (mcgraph()->machine()->Is64()) {
2771 value =
2772 graph()->NewNode(mcgraph()->machine()->TruncateInt64ToInt32(), value);
2773 }
2774 return value;
2775 }
2776
InitInstanceCache(WasmInstanceCacheNodes * instance_cache)2777 void WasmGraphBuilder::InitInstanceCache(
2778 WasmInstanceCacheNodes* instance_cache) {
2779 DCHECK_NOT_NULL(instance_node_);
2780
2781 // Load the memory start.
2782 instance_cache->mem_start = SetEffect(graph()->NewNode(
2783 mcgraph()->machine()->Load(MachineType::UintPtr()), instance_node_.get(),
2784 mcgraph()->Int32Constant(WASM_INSTANCE_OBJECT_OFFSET(MemoryStart)),
2785 Effect(), Control()));
2786
2787 // Load the memory size.
2788 instance_cache->mem_size = SetEffect(graph()->NewNode(
2789 mcgraph()->machine()->Load(MachineType::UintPtr()), instance_node_.get(),
2790 mcgraph()->Int32Constant(WASM_INSTANCE_OBJECT_OFFSET(MemorySize)),
2791 Effect(), Control()));
2792
2793 if (untrusted_code_mitigations_) {
2794 // Load the memory mask.
2795 instance_cache->mem_mask = SetEffect(graph()->NewNode(
2796 mcgraph()->machine()->Load(MachineType::UintPtr()),
2797 instance_node_.get(),
2798 mcgraph()->Int32Constant(WASM_INSTANCE_OBJECT_OFFSET(MemoryMask)),
2799 Effect(), Control()));
2800 } else {
2801 // Explicitly set to nullptr to ensure a SEGV when we try to use it.
2802 instance_cache->mem_mask = nullptr;
2803 }
2804 }
2805
PrepareInstanceCacheForLoop(WasmInstanceCacheNodes * instance_cache,Node * control)2806 void WasmGraphBuilder::PrepareInstanceCacheForLoop(
2807 WasmInstanceCacheNodes* instance_cache, Node* control) {
2808 #define INTRODUCE_PHI(field, rep) \
2809 instance_cache->field = graph()->NewNode(mcgraph()->common()->Phi(rep, 1), \
2810 instance_cache->field, control);
2811
2812 INTRODUCE_PHI(mem_start, MachineType::PointerRepresentation());
2813 INTRODUCE_PHI(mem_size, MachineRepresentation::kWord32);
2814 if (untrusted_code_mitigations_) {
2815 INTRODUCE_PHI(mem_mask, MachineRepresentation::kWord32);
2816 }
2817
2818 #undef INTRODUCE_PHI
2819 }
2820
NewInstanceCacheMerge(WasmInstanceCacheNodes * to,WasmInstanceCacheNodes * from,Node * merge)2821 void WasmGraphBuilder::NewInstanceCacheMerge(WasmInstanceCacheNodes* to,
2822 WasmInstanceCacheNodes* from,
2823 Node* merge) {
2824 #define INTRODUCE_PHI(field, rep) \
2825 if (to->field != from->field) { \
2826 Node* vals[] = {to->field, from->field, merge}; \
2827 to->field = graph()->NewNode(mcgraph()->common()->Phi(rep, 2), 3, vals); \
2828 }
2829
2830 INTRODUCE_PHI(mem_start, MachineType::PointerRepresentation());
2831 INTRODUCE_PHI(mem_size, MachineRepresentation::kWord32);
2832 if (untrusted_code_mitigations_) {
2833 INTRODUCE_PHI(mem_mask, MachineRepresentation::kWord32);
2834 }
2835
2836 #undef INTRODUCE_PHI
2837 }
2838
MergeInstanceCacheInto(WasmInstanceCacheNodes * to,WasmInstanceCacheNodes * from,Node * merge)2839 void WasmGraphBuilder::MergeInstanceCacheInto(WasmInstanceCacheNodes* to,
2840 WasmInstanceCacheNodes* from,
2841 Node* merge) {
2842 to->mem_size = CreateOrMergeIntoPhi(MachineType::PointerRepresentation(),
2843 merge, to->mem_size, from->mem_size);
2844 to->mem_start = CreateOrMergeIntoPhi(MachineType::PointerRepresentation(),
2845 merge, to->mem_start, from->mem_start);
2846 if (untrusted_code_mitigations_) {
2847 to->mem_mask = CreateOrMergeIntoPhi(MachineType::PointerRepresentation(),
2848 merge, to->mem_mask, from->mem_mask);
2849 }
2850 }
2851
CreateOrMergeIntoPhi(MachineRepresentation rep,Node * merge,Node * tnode,Node * fnode)2852 Node* WasmGraphBuilder::CreateOrMergeIntoPhi(MachineRepresentation rep,
2853 Node* merge, Node* tnode,
2854 Node* fnode) {
2855 if (IsPhiWithMerge(tnode, merge)) {
2856 AppendToPhi(tnode, fnode);
2857 } else if (tnode != fnode) {
2858 uint32_t count = merge->InputCount();
2859 // + 1 for the merge node.
2860 Node** vals = Buffer(count + 1);
2861 for (uint32_t j = 0; j < count - 1; j++) vals[j] = tnode;
2862 vals[count - 1] = fnode;
2863 vals[count] = merge;
2864 return graph()->NewNode(mcgraph()->common()->Phi(rep, count), count + 1,
2865 vals);
2866 }
2867 return tnode;
2868 }
2869
CreateOrMergeIntoEffectPhi(Node * merge,Node * tnode,Node * fnode)2870 Node* WasmGraphBuilder::CreateOrMergeIntoEffectPhi(Node* merge, Node* tnode,
2871 Node* fnode) {
2872 if (IsPhiWithMerge(tnode, merge)) {
2873 AppendToPhi(tnode, fnode);
2874 } else if (tnode != fnode) {
2875 uint32_t count = merge->InputCount();
2876 Node** effects = Buffer(count);
2877 for (uint32_t j = 0; j < count - 1; j++) {
2878 effects[j] = tnode;
2879 }
2880 effects[count - 1] = fnode;
2881 tnode = EffectPhi(count, effects, merge);
2882 }
2883 return tnode;
2884 }
2885
GetGlobalBaseAndOffset(MachineType mem_type,const wasm::WasmGlobal & global,Node ** base_node,Node ** offset_node)2886 void WasmGraphBuilder::GetGlobalBaseAndOffset(MachineType mem_type,
2887 const wasm::WasmGlobal& global,
2888 Node** base_node,
2889 Node** offset_node) {
2890 DCHECK_NOT_NULL(instance_node_);
2891 if (global.mutability && global.imported) {
2892 if (imported_mutable_globals_ == nullptr) {
2893 // Load imported_mutable_globals_ from the instance object at runtime.
2894 imported_mutable_globals_ = graph()->NewNode(
2895 mcgraph()->machine()->Load(MachineType::UintPtr()),
2896 instance_node_.get(),
2897 mcgraph()->Int32Constant(
2898 WASM_INSTANCE_OBJECT_OFFSET(ImportedMutableGlobals)),
2899 graph()->start(), graph()->start());
2900 }
2901 *base_node = SetEffect(graph()->NewNode(
2902 mcgraph()->machine()->Load(MachineType::UintPtr()),
2903 imported_mutable_globals_.get(),
2904 mcgraph()->Int32Constant(global.index * sizeof(Address)), Effect(),
2905 Control()));
2906 *offset_node = mcgraph()->Int32Constant(0);
2907 } else {
2908 if (globals_start_ == nullptr) {
2909 // Load globals_start from the instance object at runtime.
2910 // TODO(wasm): we currently generate only one load of the {globals_start}
2911 // start per graph, which means it can be placed anywhere by the
2912 // scheduler. This is legal because the globals_start should never change.
2913 // However, in some cases (e.g. if the instance object is already in a
2914 // register), it is slightly more efficient to reload this value from the
2915 // instance object. Since this depends on register allocation, it is not
2916 // possible to express in the graph, and would essentially constitute a
2917 // "mem2reg" optimization in TurboFan.
2918 globals_start_ = graph()->NewNode(
2919 mcgraph()->machine()->Load(MachineType::UintPtr()),
2920 instance_node_.get(),
2921 mcgraph()->Int32Constant(WASM_INSTANCE_OBJECT_OFFSET(GlobalsStart)),
2922 graph()->start(), graph()->start());
2923 }
2924 *base_node = globals_start_.get();
2925 *offset_node = mcgraph()->Int32Constant(global.offset);
2926
2927 if (mem_type == MachineType::Simd128() && global.offset != 0) {
2928 // TODO(titzer,bbudge): code generation for SIMD memory offsets is broken.
2929 *base_node = graph()->NewNode(mcgraph()->machine()->IntAdd(), *base_node,
2930 *offset_node);
2931 *offset_node = mcgraph()->Int32Constant(0);
2932 }
2933 }
2934 }
2935
MemBuffer(uint32_t offset)2936 Node* WasmGraphBuilder::MemBuffer(uint32_t offset) {
2937 DCHECK_NOT_NULL(instance_cache_);
2938 Node* mem_start = instance_cache_->mem_start;
2939 DCHECK_NOT_NULL(mem_start);
2940 if (offset == 0) return mem_start;
2941 return graph()->NewNode(mcgraph()->machine()->IntAdd(), mem_start,
2942 mcgraph()->IntPtrConstant(offset));
2943 }
2944
CurrentMemoryPages()2945 Node* WasmGraphBuilder::CurrentMemoryPages() {
2946 // CurrentMemoryPages can not be called from asm.js.
2947 DCHECK_EQ(wasm::kWasmOrigin, env_->module->origin);
2948 DCHECK_NOT_NULL(instance_cache_);
2949 Node* mem_size = instance_cache_->mem_size;
2950 DCHECK_NOT_NULL(mem_size);
2951 Node* result =
2952 graph()->NewNode(mcgraph()->machine()->WordShr(), mem_size,
2953 mcgraph()->Int32Constant(wasm::kWasmPageSizeLog2));
2954 if (mcgraph()->machine()->Is64()) {
2955 result =
2956 graph()->NewNode(mcgraph()->machine()->TruncateInt64ToInt32(), result);
2957 }
2958 return result;
2959 }
2960
2961 // Only call this function for code which is not reused across instantiations,
2962 // as we do not patch the embedded js_context.
BuildCallToRuntimeWithContext(Runtime::FunctionId f,Node * js_context,Node ** parameters,int parameter_count)2963 Node* WasmGraphBuilder::BuildCallToRuntimeWithContext(Runtime::FunctionId f,
2964 Node* js_context,
2965 Node** parameters,
2966 int parameter_count) {
2967 const Runtime::Function* fun = Runtime::FunctionForId(f);
2968 auto call_descriptor = Linkage::GetRuntimeCallDescriptor(
2969 mcgraph()->zone(), f, fun->nargs, Operator::kNoProperties,
2970 CallDescriptor::kNoFlags);
2971 // The CEntryStub is loaded from the instance_node so that generated code is
2972 // Isolate independent. At the moment this is only done for CEntryStub(1).
2973 DCHECK_EQ(1, fun->result_size);
2974 Node* centry_stub =
2975 LOAD_INSTANCE_FIELD(CEntryStub, MachineType::TaggedPointer());
2976 // At the moment we only allow 4 parameters. If more parameters are needed,
2977 // increase this constant accordingly.
2978 static const int kMaxParams = 4;
2979 DCHECK_GE(kMaxParams, parameter_count);
2980 Node* inputs[kMaxParams + 6];
2981 int count = 0;
2982 inputs[count++] = centry_stub;
2983 for (int i = 0; i < parameter_count; i++) {
2984 inputs[count++] = parameters[i];
2985 }
2986 inputs[count++] =
2987 mcgraph()->ExternalConstant(ExternalReference::Create(f)); // ref
2988 inputs[count++] = mcgraph()->Int32Constant(fun->nargs); // arity
2989 inputs[count++] = js_context; // js_context
2990 inputs[count++] = Effect();
2991 inputs[count++] = Control();
2992
2993 return SetEffect(mcgraph()->graph()->NewNode(
2994 mcgraph()->common()->Call(call_descriptor), count, inputs));
2995 }
2996
BuildCallToRuntime(Runtime::FunctionId f,Node ** parameters,int parameter_count)2997 Node* WasmGraphBuilder::BuildCallToRuntime(Runtime::FunctionId f,
2998 Node** parameters,
2999 int parameter_count) {
3000 return BuildCallToRuntimeWithContext(f, NoContextConstant(), parameters,
3001 parameter_count);
3002 }
3003
GetGlobal(uint32_t index)3004 Node* WasmGraphBuilder::GetGlobal(uint32_t index) {
3005 MachineType mem_type =
3006 wasm::ValueTypes::MachineTypeFor(env_->module->globals[index].type);
3007 Node* base = nullptr;
3008 Node* offset = nullptr;
3009 GetGlobalBaseAndOffset(mem_type, env_->module->globals[index], &base,
3010 &offset);
3011 Node* load = SetEffect(graph()->NewNode(mcgraph()->machine()->Load(mem_type),
3012 base, offset, Effect(), Control()));
3013 #if defined(V8_TARGET_BIG_ENDIAN)
3014 load = BuildChangeEndiannessLoad(load, mem_type,
3015 env_->module->globals[index].type);
3016 #endif
3017 return load;
3018 }
3019
SetGlobal(uint32_t index,Node * val)3020 Node* WasmGraphBuilder::SetGlobal(uint32_t index, Node* val) {
3021 MachineType mem_type =
3022 wasm::ValueTypes::MachineTypeFor(env_->module->globals[index].type);
3023 Node* base = nullptr;
3024 Node* offset = nullptr;
3025 GetGlobalBaseAndOffset(mem_type, env_->module->globals[index], &base,
3026 &offset);
3027 const Operator* op = mcgraph()->machine()->Store(
3028 StoreRepresentation(mem_type.representation(), kNoWriteBarrier));
3029 #if defined(V8_TARGET_BIG_ENDIAN)
3030 val = BuildChangeEndiannessStore(val, mem_type.representation(),
3031 env_->module->globals[index].type);
3032 #endif
3033 return SetEffect(
3034 graph()->NewNode(op, base, offset, val, Effect(), Control()));
3035 }
3036
CheckBoundsAndAlignment(uint8_t access_size,Node * index,uint32_t offset,wasm::WasmCodePosition position)3037 Node* WasmGraphBuilder::CheckBoundsAndAlignment(
3038 uint8_t access_size, Node* index, uint32_t offset,
3039 wasm::WasmCodePosition position) {
3040 // Atomic operations access the memory, need to be bound checked till
3041 // TrapHandlers are enabled on atomic operations
3042 index =
3043 BoundsCheckMem(access_size, index, offset, position, kNeedsBoundsCheck);
3044 Node* effective_address =
3045 graph()->NewNode(mcgraph()->machine()->IntAdd(), MemBuffer(offset),
3046 Uint32ToUintptr(index));
3047 // Unlike regular memory accesses, unaligned memory accesses for atomic
3048 // operations should trap
3049 // Access sizes are in powers of two, calculate mod without using division
3050 Node* cond =
3051 graph()->NewNode(mcgraph()->machine()->WordAnd(), effective_address,
3052 IntPtrConstant(access_size - 1));
3053 TrapIfFalse(wasm::kTrapUnalignedAccess,
3054 graph()->NewNode(mcgraph()->machine()->Word32Equal(), cond,
3055 mcgraph()->Int32Constant(0)),
3056 position);
3057 return index;
3058 }
3059
BoundsCheckMem(uint8_t access_size,Node * index,uint32_t offset,wasm::WasmCodePosition position,EnforceBoundsCheck enforce_check)3060 Node* WasmGraphBuilder::BoundsCheckMem(uint8_t access_size, Node* index,
3061 uint32_t offset,
3062 wasm::WasmCodePosition position,
3063 EnforceBoundsCheck enforce_check) {
3064 DCHECK_LE(1, access_size);
3065 index = Uint32ToUintptr(index);
3066 if (FLAG_wasm_no_bounds_checks) return index;
3067
3068 if (use_trap_handler() && enforce_check == kCanOmitBoundsCheck) {
3069 return index;
3070 }
3071
3072 const bool statically_oob = access_size > env_->max_memory_size ||
3073 offset > env_->max_memory_size - access_size;
3074 if (statically_oob) {
3075 // The access will be out of bounds, even for the largest memory.
3076 TrapIfEq32(wasm::kTrapMemOutOfBounds, Int32Constant(0), 0, position);
3077 return mcgraph()->IntPtrConstant(0);
3078 }
3079 uint64_t end_offset = uint64_t{offset} + access_size - 1u;
3080 Node* end_offset_node = IntPtrConstant(end_offset);
3081
3082 // The accessed memory is [index + offset, index + end_offset].
3083 // Check that the last read byte (at {index + end_offset}) is in bounds.
3084 // 1) Check that {end_offset < mem_size}. This also ensures that we can safely
3085 // compute {effective_size} as {mem_size - end_offset)}.
3086 // {effective_size} is >= 1 if condition 1) holds.
3087 // 2) Check that {index + end_offset < mem_size} by
3088 // - computing {effective_size} as {mem_size - end_offset} and
3089 // - checking that {index < effective_size}.
3090
3091 auto m = mcgraph()->machine();
3092 Node* mem_size = instance_cache_->mem_size;
3093 if (end_offset >= env_->min_memory_size) {
3094 // The end offset is larger than the smallest memory.
3095 // Dynamically check the end offset against the dynamic memory size.
3096 Node* cond = graph()->NewNode(m->UintLessThan(), end_offset_node, mem_size);
3097 TrapIfFalse(wasm::kTrapMemOutOfBounds, cond, position);
3098 } else {
3099 // The end offset is smaller than the smallest memory, so only one check is
3100 // required. Check to see if the index is also a constant.
3101 UintPtrMatcher match(index);
3102 if (match.HasValue()) {
3103 uintptr_t index_val = match.Value();
3104 if (index_val < env_->min_memory_size - end_offset) {
3105 // The input index is a constant and everything is statically within
3106 // bounds of the smallest possible memory.
3107 return index;
3108 }
3109 }
3110 }
3111
3112 // This produces a positive number, since {end_offset < min_size <= mem_size}.
3113 Node* effective_size =
3114 graph()->NewNode(m->IntSub(), mem_size, end_offset_node);
3115
3116 // Introduce the actual bounds check.
3117 Node* cond = graph()->NewNode(m->UintLessThan(), index, effective_size);
3118 TrapIfFalse(wasm::kTrapMemOutOfBounds, cond, position);
3119
3120 if (untrusted_code_mitigations_) {
3121 // In the fallthrough case, condition the index with the memory mask.
3122 Node* mem_mask = instance_cache_->mem_mask;
3123 DCHECK_NOT_NULL(mem_mask);
3124 index = graph()->NewNode(m->WordAnd(), index, mem_mask);
3125 }
3126 return index;
3127 }
3128
GetSafeLoadOperator(int offset,wasm::ValueType type)3129 const Operator* WasmGraphBuilder::GetSafeLoadOperator(int offset,
3130 wasm::ValueType type) {
3131 int alignment = offset % (wasm::ValueTypes::ElementSizeInBytes(type));
3132 MachineType mach_type = wasm::ValueTypes::MachineTypeFor(type);
3133 if (alignment == 0 || mcgraph()->machine()->UnalignedLoadSupported(
3134 wasm::ValueTypes::MachineRepresentationFor(type))) {
3135 return mcgraph()->machine()->Load(mach_type);
3136 }
3137 return mcgraph()->machine()->UnalignedLoad(mach_type);
3138 }
3139
GetSafeStoreOperator(int offset,wasm::ValueType type)3140 const Operator* WasmGraphBuilder::GetSafeStoreOperator(int offset,
3141 wasm::ValueType type) {
3142 int alignment = offset % (wasm::ValueTypes::ElementSizeInBytes(type));
3143 MachineRepresentation rep = wasm::ValueTypes::MachineRepresentationFor(type);
3144 if (alignment == 0 || mcgraph()->machine()->UnalignedStoreSupported(rep)) {
3145 StoreRepresentation store_rep(rep, WriteBarrierKind::kNoWriteBarrier);
3146 return mcgraph()->machine()->Store(store_rep);
3147 }
3148 UnalignedStoreRepresentation store_rep(rep);
3149 return mcgraph()->machine()->UnalignedStore(store_rep);
3150 }
3151
TraceMemoryOperation(bool is_store,MachineRepresentation rep,Node * index,uint32_t offset,wasm::WasmCodePosition position)3152 Node* WasmGraphBuilder::TraceMemoryOperation(bool is_store,
3153 MachineRepresentation rep,
3154 Node* index, uint32_t offset,
3155 wasm::WasmCodePosition position) {
3156 int kAlign = 4; // Ensure that the LSB is 0, such that this looks like a Smi.
3157 Node* info = graph()->NewNode(
3158 mcgraph()->machine()->StackSlot(sizeof(wasm::MemoryTracingInfo), kAlign));
3159
3160 Node* address = graph()->NewNode(mcgraph()->machine()->Int32Add(),
3161 Int32Constant(offset), index);
3162 auto store = [&](int offset, MachineRepresentation rep, Node* data) {
3163 SetEffect(graph()->NewNode(
3164 mcgraph()->machine()->Store(StoreRepresentation(rep, kNoWriteBarrier)),
3165 info, mcgraph()->Int32Constant(offset), data, Effect(), Control()));
3166 };
3167 // Store address, is_store, and mem_rep.
3168 store(offsetof(wasm::MemoryTracingInfo, address),
3169 MachineRepresentation::kWord32, address);
3170 store(offsetof(wasm::MemoryTracingInfo, is_store),
3171 MachineRepresentation::kWord8,
3172 mcgraph()->Int32Constant(is_store ? 1 : 0));
3173 store(offsetof(wasm::MemoryTracingInfo, mem_rep),
3174 MachineRepresentation::kWord8,
3175 mcgraph()->Int32Constant(static_cast<int>(rep)));
3176
3177 Node* call = BuildCallToRuntime(Runtime::kWasmTraceMemory, &info, 1);
3178 SetSourcePosition(call, position);
3179 return call;
3180 }
3181
LoadMem(wasm::ValueType type,MachineType memtype,Node * index,uint32_t offset,uint32_t alignment,wasm::WasmCodePosition position)3182 Node* WasmGraphBuilder::LoadMem(wasm::ValueType type, MachineType memtype,
3183 Node* index, uint32_t offset,
3184 uint32_t alignment,
3185 wasm::WasmCodePosition position) {
3186 Node* load;
3187
3188 // Wasm semantics throw on OOB. Introduce explicit bounds check and
3189 // conditioning when not using the trap handler.
3190 index = BoundsCheckMem(wasm::ValueTypes::MemSize(memtype), index, offset,
3191 position, kCanOmitBoundsCheck);
3192
3193 if (memtype.representation() == MachineRepresentation::kWord8 ||
3194 mcgraph()->machine()->UnalignedLoadSupported(memtype.representation())) {
3195 if (use_trap_handler()) {
3196 load = graph()->NewNode(mcgraph()->machine()->ProtectedLoad(memtype),
3197 MemBuffer(offset), index, Effect(), Control());
3198 SetSourcePosition(load, position);
3199 } else {
3200 load = graph()->NewNode(mcgraph()->machine()->Load(memtype),
3201 MemBuffer(offset), index, Effect(), Control());
3202 }
3203 } else {
3204 // TODO(eholk): Support unaligned loads with trap handlers.
3205 DCHECK(!use_trap_handler());
3206 load = graph()->NewNode(mcgraph()->machine()->UnalignedLoad(memtype),
3207 MemBuffer(offset), index, Effect(), Control());
3208 }
3209
3210 SetEffect(load);
3211
3212 #if defined(V8_TARGET_BIG_ENDIAN)
3213 load = BuildChangeEndiannessLoad(load, memtype, type);
3214 #endif
3215
3216 if (type == wasm::kWasmI64 &&
3217 ElementSizeInBytes(memtype.representation()) < 8) {
3218 // TODO(titzer): TF zeroes the upper bits of 64-bit loads for subword sizes.
3219 if (memtype.IsSigned()) {
3220 // sign extend
3221 load = graph()->NewNode(mcgraph()->machine()->ChangeInt32ToInt64(), load);
3222 } else {
3223 // zero extend
3224 load =
3225 graph()->NewNode(mcgraph()->machine()->ChangeUint32ToUint64(), load);
3226 }
3227 }
3228
3229 if (FLAG_wasm_trace_memory) {
3230 TraceMemoryOperation(false, memtype.representation(), index, offset,
3231 position);
3232 }
3233
3234 return load;
3235 }
3236
StoreMem(MachineRepresentation mem_rep,Node * index,uint32_t offset,uint32_t alignment,Node * val,wasm::WasmCodePosition position,wasm::ValueType type)3237 Node* WasmGraphBuilder::StoreMem(MachineRepresentation mem_rep, Node* index,
3238 uint32_t offset, uint32_t alignment, Node* val,
3239 wasm::WasmCodePosition position,
3240 wasm::ValueType type) {
3241 Node* store;
3242
3243 index = BoundsCheckMem(i::ElementSizeInBytes(mem_rep), index, offset,
3244 position, kCanOmitBoundsCheck);
3245
3246 #if defined(V8_TARGET_BIG_ENDIAN)
3247 val = BuildChangeEndiannessStore(val, mem_rep, type);
3248 #endif
3249
3250 if (mem_rep == MachineRepresentation::kWord8 ||
3251 mcgraph()->machine()->UnalignedStoreSupported(mem_rep)) {
3252 if (use_trap_handler()) {
3253 store =
3254 graph()->NewNode(mcgraph()->machine()->ProtectedStore(mem_rep),
3255 MemBuffer(offset), index, val, Effect(), Control());
3256 SetSourcePosition(store, position);
3257 } else {
3258 StoreRepresentation rep(mem_rep, kNoWriteBarrier);
3259 store =
3260 graph()->NewNode(mcgraph()->machine()->Store(rep), MemBuffer(offset),
3261 index, val, Effect(), Control());
3262 }
3263 } else {
3264 // TODO(eholk): Support unaligned stores with trap handlers.
3265 DCHECK(!use_trap_handler());
3266 UnalignedStoreRepresentation rep(mem_rep);
3267 store =
3268 graph()->NewNode(mcgraph()->machine()->UnalignedStore(rep),
3269 MemBuffer(offset), index, val, Effect(), Control());
3270 }
3271
3272 SetEffect(store);
3273
3274 if (FLAG_wasm_trace_memory) {
3275 TraceMemoryOperation(true, mem_rep, index, offset, position);
3276 }
3277
3278 return store;
3279 }
3280
3281 namespace {
GetAsmJsOOBValue(MachineRepresentation rep,MachineGraph * mcgraph)3282 Node* GetAsmJsOOBValue(MachineRepresentation rep, MachineGraph* mcgraph) {
3283 switch (rep) {
3284 case MachineRepresentation::kWord8:
3285 case MachineRepresentation::kWord16:
3286 case MachineRepresentation::kWord32:
3287 return mcgraph->Int32Constant(0);
3288 case MachineRepresentation::kWord64:
3289 return mcgraph->Int64Constant(0);
3290 case MachineRepresentation::kFloat32:
3291 return mcgraph->Float32Constant(std::numeric_limits<float>::quiet_NaN());
3292 case MachineRepresentation::kFloat64:
3293 return mcgraph->Float64Constant(std::numeric_limits<double>::quiet_NaN());
3294 default:
3295 UNREACHABLE();
3296 }
3297 }
3298 } // namespace
3299
BuildAsmjsLoadMem(MachineType type,Node * index)3300 Node* WasmGraphBuilder::BuildAsmjsLoadMem(MachineType type, Node* index) {
3301 DCHECK_NOT_NULL(instance_cache_);
3302 Node* mem_start = instance_cache_->mem_start;
3303 Node* mem_size = instance_cache_->mem_size;
3304 DCHECK_NOT_NULL(mem_start);
3305 DCHECK_NOT_NULL(mem_size);
3306
3307 // Asm.js semantics are defined in terms of typed arrays, hence OOB
3308 // reads return {undefined} coerced to the result type (0 for integers, NaN
3309 // for float and double).
3310 // Note that we check against the memory size ignoring the size of the
3311 // stored value, which is conservative if misaligned. Technically, asm.js
3312 // should never have misaligned accesses.
3313 index = Uint32ToUintptr(index);
3314 Diamond bounds_check(
3315 graph(), mcgraph()->common(),
3316 graph()->NewNode(mcgraph()->machine()->UintLessThan(), index, mem_size),
3317 BranchHint::kTrue);
3318 bounds_check.Chain(Control());
3319
3320 if (untrusted_code_mitigations_) {
3321 // Condition the index with the memory mask.
3322 Node* mem_mask = instance_cache_->mem_mask;
3323 DCHECK_NOT_NULL(mem_mask);
3324 index = graph()->NewNode(mcgraph()->machine()->WordAnd(), index, mem_mask);
3325 }
3326
3327 Node* load = graph()->NewNode(mcgraph()->machine()->Load(type), mem_start,
3328 index, Effect(), bounds_check.if_true);
3329 SetEffect(bounds_check.EffectPhi(load, Effect()));
3330 SetControl(bounds_check.merge);
3331 return bounds_check.Phi(type.representation(), load,
3332 GetAsmJsOOBValue(type.representation(), mcgraph()));
3333 }
3334
Uint32ToUintptr(Node * node)3335 Node* WasmGraphBuilder::Uint32ToUintptr(Node* node) {
3336 if (mcgraph()->machine()->Is32()) return node;
3337 // Fold instances of ChangeUint32ToUint64(IntConstant) directly.
3338 Uint32Matcher matcher(node);
3339 if (matcher.HasValue()) {
3340 uintptr_t value = matcher.Value();
3341 return mcgraph()->IntPtrConstant(bit_cast<intptr_t>(value));
3342 }
3343 return graph()->NewNode(mcgraph()->machine()->ChangeUint32ToUint64(), node);
3344 }
3345
BuildAsmjsStoreMem(MachineType type,Node * index,Node * val)3346 Node* WasmGraphBuilder::BuildAsmjsStoreMem(MachineType type, Node* index,
3347 Node* val) {
3348 DCHECK_NOT_NULL(instance_cache_);
3349 Node* mem_start = instance_cache_->mem_start;
3350 Node* mem_size = instance_cache_->mem_size;
3351 DCHECK_NOT_NULL(mem_start);
3352 DCHECK_NOT_NULL(mem_size);
3353
3354 // Asm.js semantics are to ignore OOB writes.
3355 // Note that we check against the memory size ignoring the size of the
3356 // stored value, which is conservative if misaligned. Technically, asm.js
3357 // should never have misaligned accesses.
3358 Diamond bounds_check(
3359 graph(), mcgraph()->common(),
3360 graph()->NewNode(mcgraph()->machine()->Uint32LessThan(), index, mem_size),
3361 BranchHint::kTrue);
3362 bounds_check.Chain(Control());
3363
3364 if (untrusted_code_mitigations_) {
3365 // Condition the index with the memory mask.
3366 Node* mem_mask = instance_cache_->mem_mask;
3367 DCHECK_NOT_NULL(mem_mask);
3368 index =
3369 graph()->NewNode(mcgraph()->machine()->Word32And(), index, mem_mask);
3370 }
3371
3372 index = Uint32ToUintptr(index);
3373 const Operator* store_op = mcgraph()->machine()->Store(StoreRepresentation(
3374 type.representation(), WriteBarrierKind::kNoWriteBarrier));
3375 Node* store = graph()->NewNode(store_op, mem_start, index, val, Effect(),
3376 bounds_check.if_true);
3377 SetEffect(bounds_check.EffectPhi(store, Effect()));
3378 SetControl(bounds_check.merge);
3379 return val;
3380 }
3381
PrintDebugName(Node * node)3382 void WasmGraphBuilder::PrintDebugName(Node* node) {
3383 PrintF("#%d:%s", node->id(), node->op()->mnemonic());
3384 }
3385
graph()3386 Graph* WasmGraphBuilder::graph() { return mcgraph()->graph(); }
3387
3388 namespace {
CreateMachineSignature(Zone * zone,wasm::FunctionSig * sig)3389 Signature<MachineRepresentation>* CreateMachineSignature(
3390 Zone* zone, wasm::FunctionSig* sig) {
3391 Signature<MachineRepresentation>::Builder builder(zone, sig->return_count(),
3392 sig->parameter_count());
3393 for (auto ret : sig->returns()) {
3394 builder.AddReturn(wasm::ValueTypes::MachineRepresentationFor(ret));
3395 }
3396
3397 for (auto param : sig->parameters()) {
3398 builder.AddParam(wasm::ValueTypes::MachineRepresentationFor(param));
3399 }
3400 return builder.Build();
3401 }
3402 } // namespace
3403
LowerInt64()3404 void WasmGraphBuilder::LowerInt64() {
3405 if (mcgraph()->machine()->Is64()) return;
3406 Int64Lowering r(mcgraph()->graph(), mcgraph()->machine(), mcgraph()->common(),
3407 mcgraph()->zone(),
3408 CreateMachineSignature(mcgraph()->zone(), sig_));
3409 r.LowerGraph();
3410 }
3411
SimdScalarLoweringForTesting()3412 void WasmGraphBuilder::SimdScalarLoweringForTesting() {
3413 SimdScalarLowering(mcgraph(), CreateMachineSignature(mcgraph()->zone(), sig_))
3414 .LowerGraph();
3415 }
3416
SetSourcePosition(Node * node,wasm::WasmCodePosition position)3417 void WasmGraphBuilder::SetSourcePosition(Node* node,
3418 wasm::WasmCodePosition position) {
3419 DCHECK_NE(position, wasm::kNoCodePosition);
3420 if (source_position_table_)
3421 source_position_table_->SetSourcePosition(node, SourcePosition(position));
3422 }
3423
S128Zero()3424 Node* WasmGraphBuilder::S128Zero() {
3425 has_simd_ = true;
3426 return graph()->NewNode(mcgraph()->machine()->S128Zero());
3427 }
3428
SimdOp(wasm::WasmOpcode opcode,Node * const * inputs)3429 Node* WasmGraphBuilder::SimdOp(wasm::WasmOpcode opcode, Node* const* inputs) {
3430 has_simd_ = true;
3431 switch (opcode) {
3432 case wasm::kExprF32x4Splat:
3433 return graph()->NewNode(mcgraph()->machine()->F32x4Splat(), inputs[0]);
3434 case wasm::kExprF32x4SConvertI32x4:
3435 return graph()->NewNode(mcgraph()->machine()->F32x4SConvertI32x4(),
3436 inputs[0]);
3437 case wasm::kExprF32x4UConvertI32x4:
3438 return graph()->NewNode(mcgraph()->machine()->F32x4UConvertI32x4(),
3439 inputs[0]);
3440 case wasm::kExprF32x4Abs:
3441 return graph()->NewNode(mcgraph()->machine()->F32x4Abs(), inputs[0]);
3442 case wasm::kExprF32x4Neg:
3443 return graph()->NewNode(mcgraph()->machine()->F32x4Neg(), inputs[0]);
3444 case wasm::kExprF32x4RecipApprox:
3445 return graph()->NewNode(mcgraph()->machine()->F32x4RecipApprox(),
3446 inputs[0]);
3447 case wasm::kExprF32x4RecipSqrtApprox:
3448 return graph()->NewNode(mcgraph()->machine()->F32x4RecipSqrtApprox(),
3449 inputs[0]);
3450 case wasm::kExprF32x4Add:
3451 return graph()->NewNode(mcgraph()->machine()->F32x4Add(), inputs[0],
3452 inputs[1]);
3453 case wasm::kExprF32x4AddHoriz:
3454 return graph()->NewNode(mcgraph()->machine()->F32x4AddHoriz(), inputs[0],
3455 inputs[1]);
3456 case wasm::kExprF32x4Sub:
3457 return graph()->NewNode(mcgraph()->machine()->F32x4Sub(), inputs[0],
3458 inputs[1]);
3459 case wasm::kExprF32x4Mul:
3460 return graph()->NewNode(mcgraph()->machine()->F32x4Mul(), inputs[0],
3461 inputs[1]);
3462 case wasm::kExprF32x4Min:
3463 return graph()->NewNode(mcgraph()->machine()->F32x4Min(), inputs[0],
3464 inputs[1]);
3465 case wasm::kExprF32x4Max:
3466 return graph()->NewNode(mcgraph()->machine()->F32x4Max(), inputs[0],
3467 inputs[1]);
3468 case wasm::kExprF32x4Eq:
3469 return graph()->NewNode(mcgraph()->machine()->F32x4Eq(), inputs[0],
3470 inputs[1]);
3471 case wasm::kExprF32x4Ne:
3472 return graph()->NewNode(mcgraph()->machine()->F32x4Ne(), inputs[0],
3473 inputs[1]);
3474 case wasm::kExprF32x4Lt:
3475 return graph()->NewNode(mcgraph()->machine()->F32x4Lt(), inputs[0],
3476 inputs[1]);
3477 case wasm::kExprF32x4Le:
3478 return graph()->NewNode(mcgraph()->machine()->F32x4Le(), inputs[0],
3479 inputs[1]);
3480 case wasm::kExprF32x4Gt:
3481 return graph()->NewNode(mcgraph()->machine()->F32x4Lt(), inputs[1],
3482 inputs[0]);
3483 case wasm::kExprF32x4Ge:
3484 return graph()->NewNode(mcgraph()->machine()->F32x4Le(), inputs[1],
3485 inputs[0]);
3486 case wasm::kExprI32x4Splat:
3487 return graph()->NewNode(mcgraph()->machine()->I32x4Splat(), inputs[0]);
3488 case wasm::kExprI32x4SConvertF32x4:
3489 return graph()->NewNode(mcgraph()->machine()->I32x4SConvertF32x4(),
3490 inputs[0]);
3491 case wasm::kExprI32x4UConvertF32x4:
3492 return graph()->NewNode(mcgraph()->machine()->I32x4UConvertF32x4(),
3493 inputs[0]);
3494 case wasm::kExprI32x4SConvertI16x8Low:
3495 return graph()->NewNode(mcgraph()->machine()->I32x4SConvertI16x8Low(),
3496 inputs[0]);
3497 case wasm::kExprI32x4SConvertI16x8High:
3498 return graph()->NewNode(mcgraph()->machine()->I32x4SConvertI16x8High(),
3499 inputs[0]);
3500 case wasm::kExprI32x4Neg:
3501 return graph()->NewNode(mcgraph()->machine()->I32x4Neg(), inputs[0]);
3502 case wasm::kExprI32x4Add:
3503 return graph()->NewNode(mcgraph()->machine()->I32x4Add(), inputs[0],
3504 inputs[1]);
3505 case wasm::kExprI32x4AddHoriz:
3506 return graph()->NewNode(mcgraph()->machine()->I32x4AddHoriz(), inputs[0],
3507 inputs[1]);
3508 case wasm::kExprI32x4Sub:
3509 return graph()->NewNode(mcgraph()->machine()->I32x4Sub(), inputs[0],
3510 inputs[1]);
3511 case wasm::kExprI32x4Mul:
3512 return graph()->NewNode(mcgraph()->machine()->I32x4Mul(), inputs[0],
3513 inputs[1]);
3514 case wasm::kExprI32x4MinS:
3515 return graph()->NewNode(mcgraph()->machine()->I32x4MinS(), inputs[0],
3516 inputs[1]);
3517 case wasm::kExprI32x4MaxS:
3518 return graph()->NewNode(mcgraph()->machine()->I32x4MaxS(), inputs[0],
3519 inputs[1]);
3520 case wasm::kExprI32x4Eq:
3521 return graph()->NewNode(mcgraph()->machine()->I32x4Eq(), inputs[0],
3522 inputs[1]);
3523 case wasm::kExprI32x4Ne:
3524 return graph()->NewNode(mcgraph()->machine()->I32x4Ne(), inputs[0],
3525 inputs[1]);
3526 case wasm::kExprI32x4LtS:
3527 return graph()->NewNode(mcgraph()->machine()->I32x4GtS(), inputs[1],
3528 inputs[0]);
3529 case wasm::kExprI32x4LeS:
3530 return graph()->NewNode(mcgraph()->machine()->I32x4GeS(), inputs[1],
3531 inputs[0]);
3532 case wasm::kExprI32x4GtS:
3533 return graph()->NewNode(mcgraph()->machine()->I32x4GtS(), inputs[0],
3534 inputs[1]);
3535 case wasm::kExprI32x4GeS:
3536 return graph()->NewNode(mcgraph()->machine()->I32x4GeS(), inputs[0],
3537 inputs[1]);
3538 case wasm::kExprI32x4UConvertI16x8Low:
3539 return graph()->NewNode(mcgraph()->machine()->I32x4UConvertI16x8Low(),
3540 inputs[0]);
3541 case wasm::kExprI32x4UConvertI16x8High:
3542 return graph()->NewNode(mcgraph()->machine()->I32x4UConvertI16x8High(),
3543 inputs[0]);
3544 case wasm::kExprI32x4MinU:
3545 return graph()->NewNode(mcgraph()->machine()->I32x4MinU(), inputs[0],
3546 inputs[1]);
3547 case wasm::kExprI32x4MaxU:
3548 return graph()->NewNode(mcgraph()->machine()->I32x4MaxU(), inputs[0],
3549 inputs[1]);
3550 case wasm::kExprI32x4LtU:
3551 return graph()->NewNode(mcgraph()->machine()->I32x4GtU(), inputs[1],
3552 inputs[0]);
3553 case wasm::kExprI32x4LeU:
3554 return graph()->NewNode(mcgraph()->machine()->I32x4GeU(), inputs[1],
3555 inputs[0]);
3556 case wasm::kExprI32x4GtU:
3557 return graph()->NewNode(mcgraph()->machine()->I32x4GtU(), inputs[0],
3558 inputs[1]);
3559 case wasm::kExprI32x4GeU:
3560 return graph()->NewNode(mcgraph()->machine()->I32x4GeU(), inputs[0],
3561 inputs[1]);
3562 case wasm::kExprI16x8Splat:
3563 return graph()->NewNode(mcgraph()->machine()->I16x8Splat(), inputs[0]);
3564 case wasm::kExprI16x8SConvertI8x16Low:
3565 return graph()->NewNode(mcgraph()->machine()->I16x8SConvertI8x16Low(),
3566 inputs[0]);
3567 case wasm::kExprI16x8SConvertI8x16High:
3568 return graph()->NewNode(mcgraph()->machine()->I16x8SConvertI8x16High(),
3569 inputs[0]);
3570 case wasm::kExprI16x8Neg:
3571 return graph()->NewNode(mcgraph()->machine()->I16x8Neg(), inputs[0]);
3572 case wasm::kExprI16x8SConvertI32x4:
3573 return graph()->NewNode(mcgraph()->machine()->I16x8SConvertI32x4(),
3574 inputs[0], inputs[1]);
3575 case wasm::kExprI16x8Add:
3576 return graph()->NewNode(mcgraph()->machine()->I16x8Add(), inputs[0],
3577 inputs[1]);
3578 case wasm::kExprI16x8AddSaturateS:
3579 return graph()->NewNode(mcgraph()->machine()->I16x8AddSaturateS(),
3580 inputs[0], inputs[1]);
3581 case wasm::kExprI16x8AddHoriz:
3582 return graph()->NewNode(mcgraph()->machine()->I16x8AddHoriz(), inputs[0],
3583 inputs[1]);
3584 case wasm::kExprI16x8Sub:
3585 return graph()->NewNode(mcgraph()->machine()->I16x8Sub(), inputs[0],
3586 inputs[1]);
3587 case wasm::kExprI16x8SubSaturateS:
3588 return graph()->NewNode(mcgraph()->machine()->I16x8SubSaturateS(),
3589 inputs[0], inputs[1]);
3590 case wasm::kExprI16x8Mul:
3591 return graph()->NewNode(mcgraph()->machine()->I16x8Mul(), inputs[0],
3592 inputs[1]);
3593 case wasm::kExprI16x8MinS:
3594 return graph()->NewNode(mcgraph()->machine()->I16x8MinS(), inputs[0],
3595 inputs[1]);
3596 case wasm::kExprI16x8MaxS:
3597 return graph()->NewNode(mcgraph()->machine()->I16x8MaxS(), inputs[0],
3598 inputs[1]);
3599 case wasm::kExprI16x8Eq:
3600 return graph()->NewNode(mcgraph()->machine()->I16x8Eq(), inputs[0],
3601 inputs[1]);
3602 case wasm::kExprI16x8Ne:
3603 return graph()->NewNode(mcgraph()->machine()->I16x8Ne(), inputs[0],
3604 inputs[1]);
3605 case wasm::kExprI16x8LtS:
3606 return graph()->NewNode(mcgraph()->machine()->I16x8GtS(), inputs[1],
3607 inputs[0]);
3608 case wasm::kExprI16x8LeS:
3609 return graph()->NewNode(mcgraph()->machine()->I16x8GeS(), inputs[1],
3610 inputs[0]);
3611 case wasm::kExprI16x8GtS:
3612 return graph()->NewNode(mcgraph()->machine()->I16x8GtS(), inputs[0],
3613 inputs[1]);
3614 case wasm::kExprI16x8GeS:
3615 return graph()->NewNode(mcgraph()->machine()->I16x8GeS(), inputs[0],
3616 inputs[1]);
3617 case wasm::kExprI16x8UConvertI8x16Low:
3618 return graph()->NewNode(mcgraph()->machine()->I16x8UConvertI8x16Low(),
3619 inputs[0]);
3620 case wasm::kExprI16x8UConvertI8x16High:
3621 return graph()->NewNode(mcgraph()->machine()->I16x8UConvertI8x16High(),
3622 inputs[0]);
3623 case wasm::kExprI16x8UConvertI32x4:
3624 return graph()->NewNode(mcgraph()->machine()->I16x8UConvertI32x4(),
3625 inputs[0], inputs[1]);
3626 case wasm::kExprI16x8AddSaturateU:
3627 return graph()->NewNode(mcgraph()->machine()->I16x8AddSaturateU(),
3628 inputs[0], inputs[1]);
3629 case wasm::kExprI16x8SubSaturateU:
3630 return graph()->NewNode(mcgraph()->machine()->I16x8SubSaturateU(),
3631 inputs[0], inputs[1]);
3632 case wasm::kExprI16x8MinU:
3633 return graph()->NewNode(mcgraph()->machine()->I16x8MinU(), inputs[0],
3634 inputs[1]);
3635 case wasm::kExprI16x8MaxU:
3636 return graph()->NewNode(mcgraph()->machine()->I16x8MaxU(), inputs[0],
3637 inputs[1]);
3638 case wasm::kExprI16x8LtU:
3639 return graph()->NewNode(mcgraph()->machine()->I16x8GtU(), inputs[1],
3640 inputs[0]);
3641 case wasm::kExprI16x8LeU:
3642 return graph()->NewNode(mcgraph()->machine()->I16x8GeU(), inputs[1],
3643 inputs[0]);
3644 case wasm::kExprI16x8GtU:
3645 return graph()->NewNode(mcgraph()->machine()->I16x8GtU(), inputs[0],
3646 inputs[1]);
3647 case wasm::kExprI16x8GeU:
3648 return graph()->NewNode(mcgraph()->machine()->I16x8GeU(), inputs[0],
3649 inputs[1]);
3650 case wasm::kExprI8x16Splat:
3651 return graph()->NewNode(mcgraph()->machine()->I8x16Splat(), inputs[0]);
3652 case wasm::kExprI8x16Neg:
3653 return graph()->NewNode(mcgraph()->machine()->I8x16Neg(), inputs[0]);
3654 case wasm::kExprI8x16SConvertI16x8:
3655 return graph()->NewNode(mcgraph()->machine()->I8x16SConvertI16x8(),
3656 inputs[0], inputs[1]);
3657 case wasm::kExprI8x16Add:
3658 return graph()->NewNode(mcgraph()->machine()->I8x16Add(), inputs[0],
3659 inputs[1]);
3660 case wasm::kExprI8x16AddSaturateS:
3661 return graph()->NewNode(mcgraph()->machine()->I8x16AddSaturateS(),
3662 inputs[0], inputs[1]);
3663 case wasm::kExprI8x16Sub:
3664 return graph()->NewNode(mcgraph()->machine()->I8x16Sub(), inputs[0],
3665 inputs[1]);
3666 case wasm::kExprI8x16SubSaturateS:
3667 return graph()->NewNode(mcgraph()->machine()->I8x16SubSaturateS(),
3668 inputs[0], inputs[1]);
3669 case wasm::kExprI8x16Mul:
3670 return graph()->NewNode(mcgraph()->machine()->I8x16Mul(), inputs[0],
3671 inputs[1]);
3672 case wasm::kExprI8x16MinS:
3673 return graph()->NewNode(mcgraph()->machine()->I8x16MinS(), inputs[0],
3674 inputs[1]);
3675 case wasm::kExprI8x16MaxS:
3676 return graph()->NewNode(mcgraph()->machine()->I8x16MaxS(), inputs[0],
3677 inputs[1]);
3678 case wasm::kExprI8x16Eq:
3679 return graph()->NewNode(mcgraph()->machine()->I8x16Eq(), inputs[0],
3680 inputs[1]);
3681 case wasm::kExprI8x16Ne:
3682 return graph()->NewNode(mcgraph()->machine()->I8x16Ne(), inputs[0],
3683 inputs[1]);
3684 case wasm::kExprI8x16LtS:
3685 return graph()->NewNode(mcgraph()->machine()->I8x16GtS(), inputs[1],
3686 inputs[0]);
3687 case wasm::kExprI8x16LeS:
3688 return graph()->NewNode(mcgraph()->machine()->I8x16GeS(), inputs[1],
3689 inputs[0]);
3690 case wasm::kExprI8x16GtS:
3691 return graph()->NewNode(mcgraph()->machine()->I8x16GtS(), inputs[0],
3692 inputs[1]);
3693 case wasm::kExprI8x16GeS:
3694 return graph()->NewNode(mcgraph()->machine()->I8x16GeS(), inputs[0],
3695 inputs[1]);
3696 case wasm::kExprI8x16UConvertI16x8:
3697 return graph()->NewNode(mcgraph()->machine()->I8x16UConvertI16x8(),
3698 inputs[0], inputs[1]);
3699 case wasm::kExprI8x16AddSaturateU:
3700 return graph()->NewNode(mcgraph()->machine()->I8x16AddSaturateU(),
3701 inputs[0], inputs[1]);
3702 case wasm::kExprI8x16SubSaturateU:
3703 return graph()->NewNode(mcgraph()->machine()->I8x16SubSaturateU(),
3704 inputs[0], inputs[1]);
3705 case wasm::kExprI8x16MinU:
3706 return graph()->NewNode(mcgraph()->machine()->I8x16MinU(), inputs[0],
3707 inputs[1]);
3708 case wasm::kExprI8x16MaxU:
3709 return graph()->NewNode(mcgraph()->machine()->I8x16MaxU(), inputs[0],
3710 inputs[1]);
3711 case wasm::kExprI8x16LtU:
3712 return graph()->NewNode(mcgraph()->machine()->I8x16GtU(), inputs[1],
3713 inputs[0]);
3714 case wasm::kExprI8x16LeU:
3715 return graph()->NewNode(mcgraph()->machine()->I8x16GeU(), inputs[1],
3716 inputs[0]);
3717 case wasm::kExprI8x16GtU:
3718 return graph()->NewNode(mcgraph()->machine()->I8x16GtU(), inputs[0],
3719 inputs[1]);
3720 case wasm::kExprI8x16GeU:
3721 return graph()->NewNode(mcgraph()->machine()->I8x16GeU(), inputs[0],
3722 inputs[1]);
3723 case wasm::kExprS128And:
3724 return graph()->NewNode(mcgraph()->machine()->S128And(), inputs[0],
3725 inputs[1]);
3726 case wasm::kExprS128Or:
3727 return graph()->NewNode(mcgraph()->machine()->S128Or(), inputs[0],
3728 inputs[1]);
3729 case wasm::kExprS128Xor:
3730 return graph()->NewNode(mcgraph()->machine()->S128Xor(), inputs[0],
3731 inputs[1]);
3732 case wasm::kExprS128Not:
3733 return graph()->NewNode(mcgraph()->machine()->S128Not(), inputs[0]);
3734 case wasm::kExprS128Select:
3735 return graph()->NewNode(mcgraph()->machine()->S128Select(), inputs[0],
3736 inputs[1], inputs[2]);
3737 case wasm::kExprS1x4AnyTrue:
3738 return graph()->NewNode(mcgraph()->machine()->S1x4AnyTrue(), inputs[0]);
3739 case wasm::kExprS1x4AllTrue:
3740 return graph()->NewNode(mcgraph()->machine()->S1x4AllTrue(), inputs[0]);
3741 case wasm::kExprS1x8AnyTrue:
3742 return graph()->NewNode(mcgraph()->machine()->S1x8AnyTrue(), inputs[0]);
3743 case wasm::kExprS1x8AllTrue:
3744 return graph()->NewNode(mcgraph()->machine()->S1x8AllTrue(), inputs[0]);
3745 case wasm::kExprS1x16AnyTrue:
3746 return graph()->NewNode(mcgraph()->machine()->S1x16AnyTrue(), inputs[0]);
3747 case wasm::kExprS1x16AllTrue:
3748 return graph()->NewNode(mcgraph()->machine()->S1x16AllTrue(), inputs[0]);
3749 default:
3750 FATAL_UNSUPPORTED_OPCODE(opcode);
3751 }
3752 }
3753
SimdLaneOp(wasm::WasmOpcode opcode,uint8_t lane,Node * const * inputs)3754 Node* WasmGraphBuilder::SimdLaneOp(wasm::WasmOpcode opcode, uint8_t lane,
3755 Node* const* inputs) {
3756 has_simd_ = true;
3757 switch (opcode) {
3758 case wasm::kExprF32x4ExtractLane:
3759 return graph()->NewNode(mcgraph()->machine()->F32x4ExtractLane(lane),
3760 inputs[0]);
3761 case wasm::kExprF32x4ReplaceLane:
3762 return graph()->NewNode(mcgraph()->machine()->F32x4ReplaceLane(lane),
3763 inputs[0], inputs[1]);
3764 case wasm::kExprI32x4ExtractLane:
3765 return graph()->NewNode(mcgraph()->machine()->I32x4ExtractLane(lane),
3766 inputs[0]);
3767 case wasm::kExprI32x4ReplaceLane:
3768 return graph()->NewNode(mcgraph()->machine()->I32x4ReplaceLane(lane),
3769 inputs[0], inputs[1]);
3770 case wasm::kExprI16x8ExtractLane:
3771 return graph()->NewNode(mcgraph()->machine()->I16x8ExtractLane(lane),
3772 inputs[0]);
3773 case wasm::kExprI16x8ReplaceLane:
3774 return graph()->NewNode(mcgraph()->machine()->I16x8ReplaceLane(lane),
3775 inputs[0], inputs[1]);
3776 case wasm::kExprI8x16ExtractLane:
3777 return graph()->NewNode(mcgraph()->machine()->I8x16ExtractLane(lane),
3778 inputs[0]);
3779 case wasm::kExprI8x16ReplaceLane:
3780 return graph()->NewNode(mcgraph()->machine()->I8x16ReplaceLane(lane),
3781 inputs[0], inputs[1]);
3782 default:
3783 FATAL_UNSUPPORTED_OPCODE(opcode);
3784 }
3785 }
3786
SimdShiftOp(wasm::WasmOpcode opcode,uint8_t shift,Node * const * inputs)3787 Node* WasmGraphBuilder::SimdShiftOp(wasm::WasmOpcode opcode, uint8_t shift,
3788 Node* const* inputs) {
3789 has_simd_ = true;
3790 switch (opcode) {
3791 case wasm::kExprI32x4Shl:
3792 return graph()->NewNode(mcgraph()->machine()->I32x4Shl(shift), inputs[0]);
3793 case wasm::kExprI32x4ShrS:
3794 return graph()->NewNode(mcgraph()->machine()->I32x4ShrS(shift),
3795 inputs[0]);
3796 case wasm::kExprI32x4ShrU:
3797 return graph()->NewNode(mcgraph()->machine()->I32x4ShrU(shift),
3798 inputs[0]);
3799 case wasm::kExprI16x8Shl:
3800 return graph()->NewNode(mcgraph()->machine()->I16x8Shl(shift), inputs[0]);
3801 case wasm::kExprI16x8ShrS:
3802 return graph()->NewNode(mcgraph()->machine()->I16x8ShrS(shift),
3803 inputs[0]);
3804 case wasm::kExprI16x8ShrU:
3805 return graph()->NewNode(mcgraph()->machine()->I16x8ShrU(shift),
3806 inputs[0]);
3807 case wasm::kExprI8x16Shl:
3808 return graph()->NewNode(mcgraph()->machine()->I8x16Shl(shift), inputs[0]);
3809 case wasm::kExprI8x16ShrS:
3810 return graph()->NewNode(mcgraph()->machine()->I8x16ShrS(shift),
3811 inputs[0]);
3812 case wasm::kExprI8x16ShrU:
3813 return graph()->NewNode(mcgraph()->machine()->I8x16ShrU(shift),
3814 inputs[0]);
3815 default:
3816 FATAL_UNSUPPORTED_OPCODE(opcode);
3817 }
3818 }
3819
Simd8x16ShuffleOp(const uint8_t shuffle[16],Node * const * inputs)3820 Node* WasmGraphBuilder::Simd8x16ShuffleOp(const uint8_t shuffle[16],
3821 Node* const* inputs) {
3822 has_simd_ = true;
3823 return graph()->NewNode(mcgraph()->machine()->S8x16Shuffle(shuffle),
3824 inputs[0], inputs[1]);
3825 }
3826
3827 #define ATOMIC_BINOP_LIST(V) \
3828 V(I32AtomicAdd, Add, Uint32, Word32) \
3829 V(I64AtomicAdd, Add, Uint64, Word64) \
3830 V(I32AtomicAdd8U, Add, Uint8, Word32) \
3831 V(I32AtomicAdd16U, Add, Uint16, Word32) \
3832 V(I64AtomicAdd8U, Add, Uint8, Word64) \
3833 V(I64AtomicAdd16U, Add, Uint16, Word64) \
3834 V(I64AtomicAdd32U, Add, Uint32, Word64) \
3835 V(I32AtomicSub, Sub, Uint32, Word32) \
3836 V(I64AtomicSub, Sub, Uint64, Word64) \
3837 V(I32AtomicSub8U, Sub, Uint8, Word32) \
3838 V(I32AtomicSub16U, Sub, Uint16, Word32) \
3839 V(I64AtomicSub8U, Sub, Uint8, Word64) \
3840 V(I64AtomicSub16U, Sub, Uint16, Word64) \
3841 V(I64AtomicSub32U, Sub, Uint32, Word64) \
3842 V(I32AtomicAnd, And, Uint32, Word32) \
3843 V(I64AtomicAnd, And, Uint64, Word64) \
3844 V(I32AtomicAnd8U, And, Uint8, Word32) \
3845 V(I64AtomicAnd16U, And, Uint16, Word64) \
3846 V(I32AtomicAnd16U, And, Uint16, Word32) \
3847 V(I64AtomicAnd8U, And, Uint8, Word64) \
3848 V(I64AtomicAnd32U, And, Uint32, Word64) \
3849 V(I32AtomicOr, Or, Uint32, Word32) \
3850 V(I64AtomicOr, Or, Uint64, Word64) \
3851 V(I32AtomicOr8U, Or, Uint8, Word32) \
3852 V(I32AtomicOr16U, Or, Uint16, Word32) \
3853 V(I64AtomicOr8U, Or, Uint8, Word64) \
3854 V(I64AtomicOr16U, Or, Uint16, Word64) \
3855 V(I64AtomicOr32U, Or, Uint32, Word64) \
3856 V(I32AtomicXor, Xor, Uint32, Word32) \
3857 V(I64AtomicXor, Xor, Uint64, Word64) \
3858 V(I32AtomicXor8U, Xor, Uint8, Word32) \
3859 V(I32AtomicXor16U, Xor, Uint16, Word32) \
3860 V(I64AtomicXor8U, Xor, Uint8, Word64) \
3861 V(I64AtomicXor16U, Xor, Uint16, Word64) \
3862 V(I64AtomicXor32U, Xor, Uint32, Word64) \
3863 V(I32AtomicExchange, Exchange, Uint32, Word32) \
3864 V(I64AtomicExchange, Exchange, Uint64, Word64) \
3865 V(I32AtomicExchange8U, Exchange, Uint8, Word32) \
3866 V(I32AtomicExchange16U, Exchange, Uint16, Word32) \
3867 V(I64AtomicExchange8U, Exchange, Uint8, Word64) \
3868 V(I64AtomicExchange16U, Exchange, Uint16, Word64) \
3869 V(I64AtomicExchange32U, Exchange, Uint32, Word64)
3870
3871 #define ATOMIC_CMP_EXCHG_LIST(V) \
3872 V(I32AtomicCompareExchange, Uint32, Word32) \
3873 V(I64AtomicCompareExchange, Uint64, Word64) \
3874 V(I32AtomicCompareExchange8U, Uint8, Word32) \
3875 V(I32AtomicCompareExchange16U, Uint16, Word32) \
3876 V(I64AtomicCompareExchange8U, Uint8, Word64) \
3877 V(I64AtomicCompareExchange16U, Uint16, Word64) \
3878 V(I64AtomicCompareExchange32U, Uint32, Word64)
3879
3880 #define ATOMIC_LOAD_LIST(V) \
3881 V(I32AtomicLoad, Uint32, Word32) \
3882 V(I64AtomicLoad, Uint64, Word64) \
3883 V(I32AtomicLoad8U, Uint8, Word32) \
3884 V(I32AtomicLoad16U, Uint16, Word32) \
3885 V(I64AtomicLoad8U, Uint8, Word64) \
3886 V(I64AtomicLoad16U, Uint16, Word64) \
3887 V(I64AtomicLoad32U, Uint32, Word64)
3888
3889 #define ATOMIC_STORE_LIST(V) \
3890 V(I32AtomicStore, Uint32, kWord32, Word32) \
3891 V(I64AtomicStore, Uint64, kWord64, Word64) \
3892 V(I32AtomicStore8U, Uint8, kWord8, Word32) \
3893 V(I32AtomicStore16U, Uint16, kWord16, Word32) \
3894 V(I64AtomicStore8U, Uint8, kWord8, Word64) \
3895 V(I64AtomicStore16U, Uint16, kWord16, Word64) \
3896 V(I64AtomicStore32U, Uint32, kWord32, Word64)
3897
AtomicOp(wasm::WasmOpcode opcode,Node * const * inputs,uint32_t alignment,uint32_t offset,wasm::WasmCodePosition position)3898 Node* WasmGraphBuilder::AtomicOp(wasm::WasmOpcode opcode, Node* const* inputs,
3899 uint32_t alignment, uint32_t offset,
3900 wasm::WasmCodePosition position) {
3901 Node* node;
3902 switch (opcode) {
3903 #define BUILD_ATOMIC_BINOP(Name, Operation, Type, Prefix) \
3904 case wasm::kExpr##Name: { \
3905 Node* index = CheckBoundsAndAlignment( \
3906 wasm::ValueTypes::MemSize(MachineType::Type()), inputs[0], offset, \
3907 position); \
3908 node = graph()->NewNode( \
3909 mcgraph()->machine()->Prefix##Atomic##Operation(MachineType::Type()), \
3910 MemBuffer(offset), index, inputs[1], Effect(), Control()); \
3911 break; \
3912 }
3913 ATOMIC_BINOP_LIST(BUILD_ATOMIC_BINOP)
3914 #undef BUILD_ATOMIC_BINOP
3915
3916 #define BUILD_ATOMIC_CMP_EXCHG(Name, Type, Prefix) \
3917 case wasm::kExpr##Name: { \
3918 Node* index = CheckBoundsAndAlignment( \
3919 wasm::ValueTypes::MemSize(MachineType::Type()), inputs[0], offset, \
3920 position); \
3921 node = graph()->NewNode( \
3922 mcgraph()->machine()->Prefix##AtomicCompareExchange( \
3923 MachineType::Type()), \
3924 MemBuffer(offset), index, inputs[1], inputs[2], Effect(), Control()); \
3925 break; \
3926 }
3927 ATOMIC_CMP_EXCHG_LIST(BUILD_ATOMIC_CMP_EXCHG)
3928 #undef BUILD_ATOMIC_CMP_EXCHG
3929
3930 #define BUILD_ATOMIC_LOAD_OP(Name, Type, Prefix) \
3931 case wasm::kExpr##Name: { \
3932 Node* index = CheckBoundsAndAlignment( \
3933 wasm::ValueTypes::MemSize(MachineType::Type()), inputs[0], offset, \
3934 position); \
3935 node = graph()->NewNode( \
3936 mcgraph()->machine()->Prefix##AtomicLoad(MachineType::Type()), \
3937 MemBuffer(offset), index, Effect(), Control()); \
3938 break; \
3939 }
3940 ATOMIC_LOAD_LIST(BUILD_ATOMIC_LOAD_OP)
3941 #undef BUILD_ATOMIC_LOAD_OP
3942
3943 #define BUILD_ATOMIC_STORE_OP(Name, Type, Rep, Prefix) \
3944 case wasm::kExpr##Name: { \
3945 Node* index = CheckBoundsAndAlignment( \
3946 wasm::ValueTypes::MemSize(MachineType::Type()), inputs[0], offset, \
3947 position); \
3948 node = graph()->NewNode( \
3949 mcgraph()->machine()->Prefix##AtomicStore(MachineRepresentation::Rep), \
3950 MemBuffer(offset), index, inputs[1], Effect(), Control()); \
3951 break; \
3952 }
3953 ATOMIC_STORE_LIST(BUILD_ATOMIC_STORE_OP)
3954 #undef BUILD_ATOMIC_STORE_OP
3955 default:
3956 FATAL_UNSUPPORTED_OPCODE(opcode);
3957 }
3958 return SetEffect(node);
3959 }
3960
3961 #undef ATOMIC_BINOP_LIST
3962 #undef ATOMIC_CMP_EXCHG_LIST
3963 #undef ATOMIC_LOAD_LIST
3964 #undef ATOMIC_STORE_LIST
3965
3966 class WasmDecorator final : public GraphDecorator {
3967 public:
WasmDecorator(NodeOriginTable * origins,wasm::Decoder * decoder)3968 explicit WasmDecorator(NodeOriginTable* origins, wasm::Decoder* decoder)
3969 : origins_(origins), decoder_(decoder) {}
3970
Decorate(Node * node)3971 void Decorate(Node* node) final {
3972 origins_->SetNodeOrigin(
3973 node, NodeOrigin("wasm graph creation", "n/a",
3974 NodeOrigin::kWasmBytecode, decoder_->position()));
3975 }
3976
3977 private:
3978 compiler::NodeOriginTable* origins_;
3979 wasm::Decoder* decoder_;
3980 };
3981
AddBytecodePositionDecorator(NodeOriginTable * node_origins,wasm::Decoder * decoder)3982 void WasmGraphBuilder::AddBytecodePositionDecorator(
3983 NodeOriginTable* node_origins, wasm::Decoder* decoder) {
3984 DCHECK_NULL(decorator_);
3985 decorator_ = new (graph()->zone()) WasmDecorator(node_origins, decoder);
3986 graph()->AddDecorator(decorator_);
3987 }
3988
RemoveBytecodePositionDecorator()3989 void WasmGraphBuilder::RemoveBytecodePositionDecorator() {
3990 DCHECK_NOT_NULL(decorator_);
3991 graph()->RemoveDecorator(decorator_);
3992 decorator_ = nullptr;
3993 }
3994
3995 namespace {
must_record_function_compilation(Isolate * isolate)3996 bool must_record_function_compilation(Isolate* isolate) {
3997 return isolate->logger()->is_listening_to_code_events() ||
3998 isolate->is_profiling();
3999 }
4000
4001 PRINTF_FORMAT(4, 5)
RecordFunctionCompilation(CodeEventListener::LogEventsAndTags tag,Isolate * isolate,Handle<Code> code,const char * format,...)4002 void RecordFunctionCompilation(CodeEventListener::LogEventsAndTags tag,
4003 Isolate* isolate, Handle<Code> code,
4004 const char* format, ...) {
4005 DCHECK(must_record_function_compilation(isolate));
4006
4007 ScopedVector<char> buffer(128);
4008 va_list arguments;
4009 va_start(arguments, format);
4010 int len = VSNPrintF(buffer, format, arguments);
4011 CHECK_LT(0, len);
4012 va_end(arguments);
4013 Handle<String> name_str =
4014 isolate->factory()->NewStringFromAsciiChecked(buffer.start());
4015 PROFILE(isolate, CodeCreateEvent(tag, AbstractCode::cast(*code), *name_str));
4016 }
4017
4018 class WasmWrapperGraphBuilder : public WasmGraphBuilder {
4019 public:
WasmWrapperGraphBuilder(Zone * zone,wasm::ModuleEnv * env,JSGraph * jsgraph,wasm::FunctionSig * sig,compiler::SourcePositionTable * spt,StubCallMode stub_mode)4020 WasmWrapperGraphBuilder(Zone* zone, wasm::ModuleEnv* env, JSGraph* jsgraph,
4021 wasm::FunctionSig* sig,
4022 compiler::SourcePositionTable* spt,
4023 StubCallMode stub_mode)
4024 : WasmGraphBuilder(env, zone, jsgraph, sig, spt),
4025 isolate_(jsgraph->isolate()),
4026 jsgraph_(jsgraph),
4027 stub_mode_(stub_mode) {}
4028
BuildAllocateHeapNumberWithValue(Node * value,Node * control)4029 Node* BuildAllocateHeapNumberWithValue(Node* value, Node* control) {
4030 MachineOperatorBuilder* machine = mcgraph()->machine();
4031 CommonOperatorBuilder* common = mcgraph()->common();
4032 Node* target = (stub_mode_ == StubCallMode::kCallWasmRuntimeStub)
4033 ? mcgraph()->RelocatableIntPtrConstant(
4034 wasm::WasmCode::kWasmAllocateHeapNumber,
4035 RelocInfo::WASM_STUB_CALL)
4036 : jsgraph()->HeapConstant(
4037 BUILTIN_CODE(isolate_, AllocateHeapNumber));
4038 if (!allocate_heap_number_operator_.is_set()) {
4039 auto call_descriptor = Linkage::GetStubCallDescriptor(
4040 mcgraph()->zone(), AllocateHeapNumberDescriptor(), 0,
4041 CallDescriptor::kNoFlags, Operator::kNoThrow, stub_mode_);
4042 allocate_heap_number_operator_.set(common->Call(call_descriptor));
4043 }
4044 Node* heap_number = graph()->NewNode(allocate_heap_number_operator_.get(),
4045 target, Effect(), control);
4046 SetEffect(
4047 graph()->NewNode(machine->Store(StoreRepresentation(
4048 MachineRepresentation::kFloat64, kNoWriteBarrier)),
4049 heap_number, BuildHeapNumberValueIndexConstant(),
4050 value, heap_number, control));
4051 return heap_number;
4052 }
4053
BuildChangeSmiToFloat64(Node * value)4054 Node* BuildChangeSmiToFloat64(Node* value) {
4055 return graph()->NewNode(mcgraph()->machine()->ChangeInt32ToFloat64(),
4056 BuildChangeSmiToInt32(value));
4057 }
4058
BuildTestHeapObject(Node * value)4059 Node* BuildTestHeapObject(Node* value) {
4060 return graph()->NewNode(mcgraph()->machine()->WordAnd(), value,
4061 mcgraph()->IntPtrConstant(kHeapObjectTag));
4062 }
4063
BuildLoadHeapNumberValue(Node * value)4064 Node* BuildLoadHeapNumberValue(Node* value) {
4065 return SetEffect(graph()->NewNode(
4066 mcgraph()->machine()->Load(MachineType::Float64()), value,
4067 BuildHeapNumberValueIndexConstant(), Effect(), Control()));
4068 }
4069
BuildHeapNumberValueIndexConstant()4070 Node* BuildHeapNumberValueIndexConstant() {
4071 return mcgraph()->IntPtrConstant(HeapNumber::kValueOffset - kHeapObjectTag);
4072 }
4073
BuildChangeInt32ToTagged(Node * value)4074 Node* BuildChangeInt32ToTagged(Node* value) {
4075 MachineOperatorBuilder* machine = mcgraph()->machine();
4076 CommonOperatorBuilder* common = mcgraph()->common();
4077
4078 if (SmiValuesAre32Bits()) {
4079 return BuildChangeInt32ToSmi(value);
4080 }
4081 DCHECK(SmiValuesAre31Bits());
4082
4083 Node* effect = Effect();
4084 Node* control = Control();
4085 Node* add = graph()->NewNode(machine->Int32AddWithOverflow(), value, value,
4086 graph()->start());
4087
4088 Node* ovf = graph()->NewNode(common->Projection(1), add, graph()->start());
4089 Node* branch =
4090 graph()->NewNode(common->Branch(BranchHint::kFalse), ovf, control);
4091
4092 Node* if_true = graph()->NewNode(common->IfTrue(), branch);
4093 Node* vtrue = BuildAllocateHeapNumberWithValue(
4094 graph()->NewNode(machine->ChangeInt32ToFloat64(), value), if_true);
4095 Node* etrue = Effect();
4096
4097 Node* if_false = graph()->NewNode(common->IfFalse(), branch);
4098 Node* vfalse = graph()->NewNode(common->Projection(0), add, if_false);
4099 vfalse = BuildChangeInt32ToIntPtr(vfalse);
4100
4101 Node* merge =
4102 SetControl(graph()->NewNode(common->Merge(2), if_true, if_false));
4103 SetEffect(graph()->NewNode(common->EffectPhi(2), etrue, effect, merge));
4104 return graph()->NewNode(common->Phi(MachineRepresentation::kTagged, 2),
4105 vtrue, vfalse, merge);
4106 }
4107
BuildChangeFloat64ToTagged(Node * value)4108 Node* BuildChangeFloat64ToTagged(Node* value) {
4109 MachineOperatorBuilder* machine = mcgraph()->machine();
4110 CommonOperatorBuilder* common = mcgraph()->common();
4111
4112 // Check several conditions:
4113 // i32?
4114 // ├─ true: zero?
4115 // │ ├─ true: negative?
4116 // │ │ ├─ true: box
4117 // │ │ └─ false: potentially Smi
4118 // │ └─ false: potentially Smi
4119 // └─ false: box
4120 // For potential Smi values, depending on whether Smis are 31 or 32 bit, we
4121 // still need to check whether the value fits in a Smi.
4122
4123 Node* effect = Effect();
4124 Node* control = Control();
4125 Node* value32 = graph()->NewNode(machine->RoundFloat64ToInt32(), value);
4126 Node* check_i32 = graph()->NewNode(
4127 machine->Float64Equal(), value,
4128 graph()->NewNode(machine->ChangeInt32ToFloat64(), value32));
4129 Node* branch_i32 = graph()->NewNode(common->Branch(), check_i32, control);
4130
4131 Node* if_i32 = graph()->NewNode(common->IfTrue(), branch_i32);
4132 Node* if_not_i32 = graph()->NewNode(common->IfFalse(), branch_i32);
4133
4134 // We only need to check for -0 if the {value} can potentially contain -0.
4135 Node* check_zero = graph()->NewNode(machine->Word32Equal(), value32,
4136 mcgraph()->Int32Constant(0));
4137 Node* branch_zero = graph()->NewNode(common->Branch(BranchHint::kFalse),
4138 check_zero, if_i32);
4139
4140 Node* if_zero = graph()->NewNode(common->IfTrue(), branch_zero);
4141 Node* if_not_zero = graph()->NewNode(common->IfFalse(), branch_zero);
4142
4143 // In case of 0, we need to check the high bits for the IEEE -0 pattern.
4144 Node* check_negative = graph()->NewNode(
4145 machine->Int32LessThan(),
4146 graph()->NewNode(machine->Float64ExtractHighWord32(), value),
4147 mcgraph()->Int32Constant(0));
4148 Node* branch_negative = graph()->NewNode(common->Branch(BranchHint::kFalse),
4149 check_negative, if_zero);
4150
4151 Node* if_negative = graph()->NewNode(common->IfTrue(), branch_negative);
4152 Node* if_not_negative =
4153 graph()->NewNode(common->IfFalse(), branch_negative);
4154
4155 // We need to create a box for negative 0.
4156 Node* if_smi =
4157 graph()->NewNode(common->Merge(2), if_not_zero, if_not_negative);
4158 Node* if_box = graph()->NewNode(common->Merge(2), if_not_i32, if_negative);
4159
4160 // On 64-bit machines we can just wrap the 32-bit integer in a smi, for
4161 // 32-bit machines we need to deal with potential overflow and fallback to
4162 // boxing.
4163 Node* vsmi;
4164 if (SmiValuesAre32Bits()) {
4165 vsmi = BuildChangeInt32ToSmi(value32);
4166 } else {
4167 DCHECK(SmiValuesAre31Bits());
4168 Node* smi_tag = graph()->NewNode(machine->Int32AddWithOverflow(), value32,
4169 value32, if_smi);
4170
4171 Node* check_ovf =
4172 graph()->NewNode(common->Projection(1), smi_tag, if_smi);
4173 Node* branch_ovf = graph()->NewNode(common->Branch(BranchHint::kFalse),
4174 check_ovf, if_smi);
4175
4176 Node* if_ovf = graph()->NewNode(common->IfTrue(), branch_ovf);
4177 if_box = graph()->NewNode(common->Merge(2), if_ovf, if_box);
4178
4179 if_smi = graph()->NewNode(common->IfFalse(), branch_ovf);
4180 vsmi = graph()->NewNode(common->Projection(0), smi_tag, if_smi);
4181 vsmi = BuildChangeInt32ToIntPtr(vsmi);
4182 }
4183
4184 // Allocate the box for the {value}.
4185 Node* vbox = BuildAllocateHeapNumberWithValue(value, if_box);
4186 Node* ebox = Effect();
4187
4188 Node* merge =
4189 SetControl(graph()->NewNode(common->Merge(2), if_smi, if_box));
4190 SetEffect(graph()->NewNode(common->EffectPhi(2), effect, ebox, merge));
4191 return graph()->NewNode(common->Phi(MachineRepresentation::kTagged, 2),
4192 vsmi, vbox, merge);
4193 }
4194
AddArgumentNodes(Node ** args,int pos,int param_count,wasm::FunctionSig * sig)4195 int AddArgumentNodes(Node** args, int pos, int param_count,
4196 wasm::FunctionSig* sig) {
4197 // Convert wasm numbers to JS values.
4198 for (int i = 0; i < param_count; ++i) {
4199 Node* param =
4200 Param(i + 1); // Start from index 1 to drop the instance_node.
4201 args[pos++] = ToJS(param, sig->GetParam(i));
4202 }
4203 return pos;
4204 }
4205
BuildJavaScriptToNumber(Node * node,Node * js_context)4206 Node* BuildJavaScriptToNumber(Node* node, Node* js_context) {
4207 auto call_descriptor = Linkage::GetStubCallDescriptor(
4208 mcgraph()->zone(), TypeConversionDescriptor{}, 0,
4209 CallDescriptor::kNoFlags, Operator::kNoProperties, stub_mode_);
4210 Node* stub_code =
4211 (stub_mode_ == StubCallMode::kCallWasmRuntimeStub)
4212 ? mcgraph()->RelocatableIntPtrConstant(
4213 wasm::WasmCode::kWasmToNumber, RelocInfo::WASM_STUB_CALL)
4214 : jsgraph()->HeapConstant(BUILTIN_CODE(isolate_, ToNumber));
4215
4216 Node* result = SetEffect(
4217 graph()->NewNode(mcgraph()->common()->Call(call_descriptor), stub_code,
4218 node, js_context, Effect(), Control()));
4219
4220 SetSourcePosition(result, 1);
4221
4222 return result;
4223 }
4224
BuildChangeTaggedToFloat64(Node * value)4225 Node* BuildChangeTaggedToFloat64(Node* value) {
4226 MachineOperatorBuilder* machine = mcgraph()->machine();
4227 CommonOperatorBuilder* common = mcgraph()->common();
4228
4229 // Implement the following decision tree:
4230 // heap object?
4231 // ├─ true: undefined?
4232 // │ ├─ true: f64 const
4233 // │ └─ false: load heap number value
4234 // └─ false: smi to float64
4235
4236 Node* check_heap_object = BuildTestHeapObject(value);
4237 Diamond is_heap_object(graph(), common, check_heap_object,
4238 BranchHint::kFalse);
4239 is_heap_object.Chain(Control());
4240
4241 SetControl(is_heap_object.if_true);
4242 Node* orig_effect = Effect();
4243
4244 Node* undefined_node =
4245 LOAD_INSTANCE_FIELD(UndefinedValue, MachineType::TaggedPointer());
4246 Node* check_undefined =
4247 graph()->NewNode(machine->WordEqual(), value, undefined_node);
4248 Node* effect_tagged = Effect();
4249
4250 Diamond is_undefined(graph(), common, check_undefined, BranchHint::kFalse);
4251 is_undefined.Nest(is_heap_object, true);
4252
4253 SetControl(is_undefined.if_false);
4254 Node* vheap_number = BuildLoadHeapNumberValue(value);
4255 Node* effect_undefined = Effect();
4256
4257 SetControl(is_undefined.merge);
4258 Node* vundefined =
4259 mcgraph()->Float64Constant(std::numeric_limits<double>::quiet_NaN());
4260 Node* vtagged = is_undefined.Phi(MachineRepresentation::kFloat64,
4261 vundefined, vheap_number);
4262
4263 effect_tagged = is_undefined.EffectPhi(effect_tagged, effect_undefined);
4264
4265 // If input is Smi: just convert to float64.
4266 Node* vfrom_smi = BuildChangeSmiToFloat64(value);
4267
4268 SetControl(is_heap_object.merge);
4269 SetEffect(is_heap_object.EffectPhi(effect_tagged, orig_effect));
4270 return is_heap_object.Phi(MachineRepresentation::kFloat64, vtagged,
4271 vfrom_smi);
4272 }
4273
ToJS(Node * node,wasm::ValueType type)4274 Node* ToJS(Node* node, wasm::ValueType type) {
4275 switch (type) {
4276 case wasm::kWasmI32:
4277 return BuildChangeInt32ToTagged(node);
4278 case wasm::kWasmS128:
4279 case wasm::kWasmI64:
4280 UNREACHABLE();
4281 case wasm::kWasmF32:
4282 node = graph()->NewNode(mcgraph()->machine()->ChangeFloat32ToFloat64(),
4283 node);
4284 return BuildChangeFloat64ToTagged(node);
4285 case wasm::kWasmF64:
4286 return BuildChangeFloat64ToTagged(node);
4287 case wasm::kWasmAnyRef:
4288 return node;
4289 default:
4290 UNREACHABLE();
4291 }
4292 }
4293
FromJS(Node * node,Node * js_context,wasm::ValueType type)4294 Node* FromJS(Node* node, Node* js_context, wasm::ValueType type) {
4295 DCHECK_NE(wasm::kWasmStmt, type);
4296
4297 // The parameter is of type AnyRef, we take it as is.
4298 if (type == wasm::kWasmAnyRef) {
4299 return node;
4300 }
4301
4302 // Do a JavaScript ToNumber.
4303 Node* num = BuildJavaScriptToNumber(node, js_context);
4304
4305 // Change representation.
4306 num = BuildChangeTaggedToFloat64(num);
4307
4308 switch (type) {
4309 case wasm::kWasmI32: {
4310 num = graph()->NewNode(mcgraph()->machine()->TruncateFloat64ToWord32(),
4311 num);
4312 break;
4313 }
4314 case wasm::kWasmS128:
4315 case wasm::kWasmI64:
4316 UNREACHABLE();
4317 case wasm::kWasmF32:
4318 num = graph()->NewNode(mcgraph()->machine()->TruncateFloat64ToFloat32(),
4319 num);
4320 break;
4321 case wasm::kWasmF64:
4322 break;
4323 default:
4324 UNREACHABLE();
4325 }
4326 return num;
4327 }
4328
BuildModifyThreadInWasmFlag(bool new_value)4329 void BuildModifyThreadInWasmFlag(bool new_value) {
4330 if (!trap_handler::IsTrapHandlerEnabled()) return;
4331 Node* thread_in_wasm_flag_address_address =
4332 graph()->NewNode(mcgraph()->common()->ExternalConstant(
4333 ExternalReference::wasm_thread_in_wasm_flag_address_address(
4334 isolate_)));
4335 Node* thread_in_wasm_flag_address = SetEffect(graph()->NewNode(
4336 mcgraph()->machine()->Load(LoadRepresentation(MachineType::Pointer())),
4337 thread_in_wasm_flag_address_address, mcgraph()->Int32Constant(0),
4338 Effect(), Control()));
4339 SetEffect(graph()->NewNode(
4340 mcgraph()->machine()->Store(StoreRepresentation(
4341 MachineRepresentation::kWord32, kNoWriteBarrier)),
4342 thread_in_wasm_flag_address, mcgraph()->Int32Constant(0),
4343 mcgraph()->Int32Constant(new_value ? 1 : 0), Effect(), Control()));
4344 }
4345
BuildLoadFunctionDataFromExportedFunction(Node * closure)4346 Node* BuildLoadFunctionDataFromExportedFunction(Node* closure) {
4347 Node* shared = SetEffect(graph()->NewNode(
4348 jsgraph()->machine()->Load(MachineType::AnyTagged()), closure,
4349 jsgraph()->Int32Constant(JSFunction::kSharedFunctionInfoOffset -
4350 kHeapObjectTag),
4351 Effect(), Control()));
4352 return SetEffect(graph()->NewNode(
4353 jsgraph()->machine()->Load(MachineType::AnyTagged()), shared,
4354 jsgraph()->Int32Constant(SharedFunctionInfo::kFunctionDataOffset -
4355 kHeapObjectTag),
4356 Effect(), Control()));
4357 }
4358
BuildLoadInstanceFromExportedFunctionData(Node * function_data)4359 Node* BuildLoadInstanceFromExportedFunctionData(Node* function_data) {
4360 return SetEffect(graph()->NewNode(
4361 jsgraph()->machine()->Load(MachineType::AnyTagged()), function_data,
4362 jsgraph()->Int32Constant(WasmExportedFunctionData::kInstanceOffset -
4363 kHeapObjectTag),
4364 Effect(), Control()));
4365 }
4366
BuildLoadFunctionIndexFromExportedFunctionData(Node * function_data)4367 Node* BuildLoadFunctionIndexFromExportedFunctionData(Node* function_data) {
4368 Node* function_index_smi = SetEffect(graph()->NewNode(
4369 jsgraph()->machine()->Load(MachineType::AnyTagged()), function_data,
4370 jsgraph()->Int32Constant(
4371 WasmExportedFunctionData::kFunctionIndexOffset - kHeapObjectTag),
4372 Effect(), Control()));
4373 Node* function_index = BuildChangeSmiToInt32(function_index_smi);
4374 return function_index;
4375 }
4376
BuildLoadJumpTableOffsetFromExportedFunctionData(Node * function_data)4377 Node* BuildLoadJumpTableOffsetFromExportedFunctionData(Node* function_data) {
4378 Node* jump_table_offset_smi = SetEffect(graph()->NewNode(
4379 jsgraph()->machine()->Load(MachineType::AnyTagged()), function_data,
4380 jsgraph()->Int32Constant(
4381 WasmExportedFunctionData::kJumpTableOffsetOffset - kHeapObjectTag),
4382 Effect(), Control()));
4383 Node* jump_table_offset = BuildChangeSmiToInt32(jump_table_offset_smi);
4384 return jump_table_offset;
4385 }
4386
BuildJSToWasmWrapper(bool is_import)4387 void BuildJSToWasmWrapper(bool is_import) {
4388 const int wasm_count = static_cast<int>(sig_->parameter_count());
4389
4390 // Build the start and the JS parameter nodes.
4391 SetEffect(SetControl(Start(wasm_count + 5)));
4392
4393 // Create the js_closure and js_context parameters.
4394 Node* js_closure =
4395 graph()->NewNode(jsgraph()->common()->Parameter(
4396 Linkage::kJSCallClosureParamIndex, "%closure"),
4397 graph()->start());
4398 Node* js_context = graph()->NewNode(
4399 mcgraph()->common()->Parameter(
4400 Linkage::GetJSCallContextParamIndex(wasm_count + 1), "%context"),
4401 graph()->start());
4402
4403 // Create the instance_node node to pass as parameter. It is loaded from
4404 // an actual reference to an instance or a placeholder reference,
4405 // called {WasmExportedFunction} via the {WasmExportedFunctionData}
4406 // structure.
4407 Node* function_data = BuildLoadFunctionDataFromExportedFunction(js_closure);
4408 instance_node_.set(
4409 BuildLoadInstanceFromExportedFunctionData(function_data));
4410
4411 if (!wasm::IsJSCompatibleSignature(sig_)) {
4412 // Throw a TypeError. Use the js_context of the calling javascript
4413 // function (passed as a parameter), such that the generated code is
4414 // js_context independent.
4415 BuildCallToRuntimeWithContext(Runtime::kWasmThrowTypeError, js_context,
4416 nullptr, 0);
4417 Return(jsgraph()->SmiConstant(0));
4418 return;
4419 }
4420
4421 const int args_count = wasm_count + 1; // +1 for wasm_code.
4422 Node** args = Buffer(args_count);
4423 Node** rets;
4424
4425 // Convert JS parameters to wasm numbers.
4426 for (int i = 0; i < wasm_count; ++i) {
4427 Node* param = Param(i + 1);
4428 Node* wasm_param = FromJS(param, js_context, sig_->GetParam(i));
4429 args[i + 1] = wasm_param;
4430 }
4431
4432 // Set the ThreadInWasm flag before we do the actual call.
4433 BuildModifyThreadInWasmFlag(true);
4434
4435 if (is_import) {
4436 // Call to an imported function.
4437 // Load function index from {WasmExportedFunctionData}.
4438 Node* function_index =
4439 BuildLoadFunctionIndexFromExportedFunctionData(function_data);
4440 BuildImportWasmCall(sig_, args, &rets, wasm::kNoCodePosition,
4441 function_index);
4442 } else {
4443 // Call to a wasm function defined in this module.
4444 // The call target is the jump table slot for that function.
4445 Node* jump_table_start =
4446 LOAD_INSTANCE_FIELD(JumpTableStart, MachineType::Pointer());
4447 Node* jump_table_offset =
4448 BuildLoadJumpTableOffsetFromExportedFunctionData(function_data);
4449 Node* jump_table_slot = graph()->NewNode(
4450 mcgraph()->machine()->IntAdd(), jump_table_start, jump_table_offset);
4451 args[0] = jump_table_slot;
4452
4453 BuildWasmCall(sig_, args, &rets, wasm::kNoCodePosition, nullptr,
4454 kNoRetpoline);
4455 }
4456
4457 // Clear the ThreadInWasm flag.
4458 BuildModifyThreadInWasmFlag(false);
4459
4460 Node* jsval = sig_->return_count() == 0 ? jsgraph()->UndefinedConstant()
4461 : ToJS(rets[0], sig_->GetReturn());
4462 Return(jsval);
4463 }
4464
BuildWasmToJSWrapper(Handle<JSReceiver> target,int index)4465 bool BuildWasmToJSWrapper(Handle<JSReceiver> target, int index) {
4466 DCHECK(target->IsCallable());
4467
4468 int wasm_count = static_cast<int>(sig_->parameter_count());
4469
4470 // Build the start and the parameter nodes.
4471 SetEffect(SetControl(Start(wasm_count + 3)));
4472
4473 // Create the instance_node from the passed parameter.
4474 instance_node_.set(Param(wasm::kWasmInstanceParameterIndex));
4475
4476 Node* callables_node = LOAD_INSTANCE_FIELD(ImportedFunctionCallables,
4477 MachineType::TaggedPointer());
4478 Node* callable_node = LOAD_FIXED_ARRAY_SLOT(callables_node, index);
4479 Node* undefined_node =
4480 LOAD_INSTANCE_FIELD(UndefinedValue, MachineType::TaggedPointer());
4481 Node* native_context =
4482 LOAD_INSTANCE_FIELD(NativeContext, MachineType::TaggedPointer());
4483
4484 if (!wasm::IsJSCompatibleSignature(sig_)) {
4485 // Throw a TypeError.
4486 BuildCallToRuntimeWithContext(Runtime::kWasmThrowTypeError,
4487 native_context, nullptr, 0);
4488 // We don't need to return a value here, as the runtime call will not
4489 // return anyway (the c entry stub will trigger stack unwinding).
4490 ReturnVoid();
4491 return false;
4492 }
4493
4494 CallDescriptor* call_descriptor;
4495 Node** args = Buffer(wasm_count + 9);
4496 Node* call = nullptr;
4497
4498 BuildModifyThreadInWasmFlag(false);
4499
4500 if (target->IsJSFunction()) {
4501 Handle<JSFunction> function = Handle<JSFunction>::cast(target);
4502 FieldAccess field_access = AccessBuilder::ForJSFunctionContext();
4503 Node* function_context = SetEffect(graph()->NewNode(
4504 mcgraph()->machine()->Load(MachineType::TaggedPointer()),
4505 callable_node,
4506 mcgraph()->Int32Constant(field_access.offset - field_access.tag()),
4507 Effect(), Control()));
4508
4509 if (!IsClassConstructor(function->shared()->kind())) {
4510 if (function->shared()->internal_formal_parameter_count() ==
4511 wasm_count) {
4512 int pos = 0;
4513 args[pos++] = callable_node; // target callable.
4514 // Receiver.
4515 if (is_sloppy(function->shared()->language_mode()) &&
4516 !function->shared()->native()) {
4517 Node* global_proxy = LOAD_FIXED_ARRAY_SLOT(
4518 native_context, Context::GLOBAL_PROXY_INDEX);
4519 args[pos++] = global_proxy;
4520 } else {
4521 args[pos++] = undefined_node;
4522 }
4523
4524 call_descriptor = Linkage::GetJSCallDescriptor(
4525 graph()->zone(), false, wasm_count + 1, CallDescriptor::kNoFlags);
4526
4527 // Convert wasm numbers to JS values.
4528 pos = AddArgumentNodes(args, pos, wasm_count, sig_);
4529
4530 args[pos++] = undefined_node; // new target
4531 args[pos++] = mcgraph()->Int32Constant(wasm_count); // argument count
4532 args[pos++] = function_context;
4533 args[pos++] = Effect();
4534 args[pos++] = Control();
4535
4536 call = graph()->NewNode(mcgraph()->common()->Call(call_descriptor),
4537 pos, args);
4538 } else if (function->shared()->internal_formal_parameter_count() >= 0) {
4539 int pos = 0;
4540 args[pos++] = mcgraph()->RelocatableIntPtrConstant(
4541 wasm::WasmCode::kWasmArgumentsAdaptor, RelocInfo::WASM_STUB_CALL);
4542 args[pos++] = callable_node; // target callable
4543 args[pos++] = undefined_node; // new target
4544 args[pos++] = mcgraph()->Int32Constant(wasm_count); // argument count
4545 args[pos++] = mcgraph()->Int32Constant(
4546 function->shared()->internal_formal_parameter_count());
4547 // Receiver.
4548 if (is_sloppy(function->shared()->language_mode()) &&
4549 !function->shared()->native()) {
4550 Node* global_proxy = LOAD_FIXED_ARRAY_SLOT(
4551 native_context, Context::GLOBAL_PROXY_INDEX);
4552 args[pos++] = global_proxy;
4553 } else {
4554 args[pos++] = undefined_node;
4555 }
4556
4557 call_descriptor = Linkage::GetStubCallDescriptor(
4558 mcgraph()->zone(), ArgumentAdaptorDescriptor{}, 1 + wasm_count,
4559 CallDescriptor::kNoFlags, Operator::kNoProperties,
4560 StubCallMode::kCallWasmRuntimeStub);
4561
4562 // Convert wasm numbers to JS values.
4563 pos = AddArgumentNodes(args, pos, wasm_count, sig_);
4564 args[pos++] = function_context;
4565 args[pos++] = Effect();
4566 args[pos++] = Control();
4567 call = graph()->NewNode(mcgraph()->common()->Call(call_descriptor),
4568 pos, args);
4569 }
4570 }
4571 }
4572
4573 // We cannot call the target directly, we have to use the Call builtin.
4574 if (!call) {
4575 int pos = 0;
4576 args[pos++] = mcgraph()->RelocatableIntPtrConstant(
4577 wasm::WasmCode::kWasmCallJavaScript, RelocInfo::WASM_STUB_CALL);
4578 args[pos++] = callable_node;
4579 args[pos++] = mcgraph()->Int32Constant(wasm_count); // argument count
4580 args[pos++] = undefined_node; // receiver
4581
4582 call_descriptor = Linkage::GetStubCallDescriptor(
4583 graph()->zone(), CallTrampolineDescriptor{}, wasm_count + 1,
4584 CallDescriptor::kNoFlags, Operator::kNoProperties,
4585 StubCallMode::kCallWasmRuntimeStub);
4586
4587 // Convert wasm numbers to JS values.
4588 pos = AddArgumentNodes(args, pos, wasm_count, sig_);
4589
4590 // The native_context is sufficient here, because all kind of callables
4591 // which depend on the context provide their own context. The context here
4592 // is only needed if the target is a constructor to throw a TypeError, if
4593 // the target is a native function, or if the target is a callable
4594 // JSObject, which can only be constructed by the runtime.
4595 args[pos++] = native_context;
4596 args[pos++] = Effect();
4597 args[pos++] = Control();
4598
4599 call = graph()->NewNode(mcgraph()->common()->Call(call_descriptor), pos,
4600 args);
4601 }
4602
4603 SetEffect(call);
4604 SetSourcePosition(call, 0);
4605
4606 // Convert the return value back.
4607 Node* val = sig_->return_count() == 0
4608 ? mcgraph()->Int32Constant(0)
4609 : FromJS(call, native_context, sig_->GetReturn());
4610
4611 BuildModifyThreadInWasmFlag(true);
4612
4613 Return(val);
4614 return true;
4615 }
4616
BuildWasmInterpreterEntry(uint32_t func_index)4617 void BuildWasmInterpreterEntry(uint32_t func_index) {
4618 int param_count = static_cast<int>(sig_->parameter_count());
4619
4620 // Build the start and the parameter nodes.
4621 SetEffect(SetControl(Start(param_count + 3)));
4622
4623 // Create the instance_node from the passed parameter.
4624 instance_node_.set(Param(wasm::kWasmInstanceParameterIndex));
4625
4626 // Compute size for the argument buffer.
4627 int args_size_bytes = 0;
4628 for (wasm::ValueType type : sig_->parameters()) {
4629 args_size_bytes += wasm::ValueTypes::ElementSizeInBytes(type);
4630 }
4631
4632 // The return value is also passed via this buffer:
4633 DCHECK_GE(wasm::kV8MaxWasmFunctionReturns, sig_->return_count());
4634 // TODO(wasm): Handle multi-value returns.
4635 DCHECK_EQ(1, wasm::kV8MaxWasmFunctionReturns);
4636 int return_size_bytes =
4637 sig_->return_count() == 0
4638 ? 0
4639 : wasm::ValueTypes::ElementSizeInBytes(sig_->GetReturn());
4640
4641 // Get a stack slot for the arguments.
4642 Node* arg_buffer =
4643 args_size_bytes == 0 && return_size_bytes == 0
4644 ? mcgraph()->IntPtrConstant(0)
4645 : graph()->NewNode(mcgraph()->machine()->StackSlot(
4646 std::max(args_size_bytes, return_size_bytes), 8));
4647
4648 // Now store all our arguments to the buffer.
4649 int offset = 0;
4650
4651 for (int i = 0; i < param_count; ++i) {
4652 wasm::ValueType type = sig_->GetParam(i);
4653 // Start from the parameter with index 1 to drop the instance_node.
4654 SetEffect(graph()->NewNode(GetSafeStoreOperator(offset, type), arg_buffer,
4655 Int32Constant(offset), Param(i + 1), Effect(),
4656 Control()));
4657 offset += wasm::ValueTypes::ElementSizeInBytes(type);
4658 }
4659 DCHECK_EQ(args_size_bytes, offset);
4660
4661 // We are passing the raw arg_buffer here. To the GC and other parts, it
4662 // looks like a Smi (lowest bit not set). In the runtime function however,
4663 // don't call Smi::value on it, but just cast it to a byte pointer.
4664 Node* parameters[] = {
4665 jsgraph()->SmiConstant(func_index), arg_buffer,
4666 };
4667 BuildCallToRuntime(Runtime::kWasmRunInterpreter, parameters,
4668 arraysize(parameters));
4669
4670 // Read back the return value.
4671 if (sig_->return_count() == 0) {
4672 Return(Int32Constant(0));
4673 } else {
4674 // TODO(wasm): Implement multi-return.
4675 DCHECK_EQ(1, sig_->return_count());
4676 MachineType load_rep =
4677 wasm::ValueTypes::MachineTypeFor(sig_->GetReturn());
4678 Node* val = SetEffect(
4679 graph()->NewNode(mcgraph()->machine()->Load(load_rep), arg_buffer,
4680 Int32Constant(0), Effect(), Control()));
4681 Return(val);
4682 }
4683
4684 if (ContainsInt64(sig_)) LowerInt64();
4685 }
4686
BuildCWasmEntry()4687 void BuildCWasmEntry() {
4688 // Build the start and the JS parameter nodes.
4689 SetEffect(SetControl(Start(CWasmEntryParameters::kNumParameters + 5)));
4690
4691 // Create parameter nodes (offset by 1 for the receiver parameter).
4692 Node* foreign_code_obj = Param(CWasmEntryParameters::kCodeObject + 1);
4693 MachineOperatorBuilder* machine = mcgraph()->machine();
4694 Node* code_obj = graph()->NewNode(
4695 machine->Load(MachineType::Pointer()), foreign_code_obj,
4696 Int32Constant(Foreign::kForeignAddressOffset - kHeapObjectTag),
4697 Effect(), Control());
4698 Node* instance_node = Param(CWasmEntryParameters::kWasmInstance + 1);
4699 Node* arg_buffer = Param(CWasmEntryParameters::kArgumentsBuffer + 1);
4700
4701 int wasm_arg_count = static_cast<int>(sig_->parameter_count());
4702 int arg_count = wasm_arg_count + 4; // code, instance_node, control, effect
4703 Node** args = Buffer(arg_count);
4704
4705 int pos = 0;
4706 args[pos++] = code_obj;
4707 args[pos++] = instance_node;
4708
4709 int offset = 0;
4710 for (wasm::ValueType type : sig_->parameters()) {
4711 Node* arg_load = SetEffect(
4712 graph()->NewNode(GetSafeLoadOperator(offset, type), arg_buffer,
4713 Int32Constant(offset), Effect(), Control()));
4714 args[pos++] = arg_load;
4715 offset += wasm::ValueTypes::ElementSizeInBytes(type);
4716 }
4717
4718 args[pos++] = Effect();
4719 args[pos++] = Control();
4720 DCHECK_EQ(arg_count, pos);
4721
4722 // Call the wasm code.
4723 auto call_descriptor = GetWasmCallDescriptor(mcgraph()->zone(), sig_);
4724
4725 Node* call = SetEffect(graph()->NewNode(
4726 mcgraph()->common()->Call(call_descriptor), arg_count, args));
4727
4728 // Store the return value.
4729 DCHECK_GE(1, sig_->return_count());
4730 if (sig_->return_count() == 1) {
4731 StoreRepresentation store_rep(
4732 wasm::ValueTypes::MachineRepresentationFor(sig_->GetReturn()),
4733 kNoWriteBarrier);
4734 SetEffect(graph()->NewNode(mcgraph()->machine()->Store(store_rep),
4735 arg_buffer, Int32Constant(0), call, Effect(),
4736 Control()));
4737 }
4738 Return(jsgraph()->SmiConstant(0));
4739
4740 if (mcgraph()->machine()->Is32() && ContainsInt64(sig_)) {
4741 MachineRepresentation sig_reps[] = {
4742 MachineRepresentation::kWord32, // return value
4743 MachineRepresentation::kTagged, // receiver
4744 MachineRepresentation::kTagged, // arg0 (code)
4745 MachineRepresentation::kTagged // arg1 (buffer)
4746 };
4747 Signature<MachineRepresentation> c_entry_sig(1, 2, sig_reps);
4748 Int64Lowering r(mcgraph()->graph(), mcgraph()->machine(),
4749 mcgraph()->common(), mcgraph()->zone(), &c_entry_sig);
4750 r.LowerGraph();
4751 }
4752 }
4753
jsgraph()4754 JSGraph* jsgraph() { return jsgraph_; }
4755
4756 private:
4757 Isolate* const isolate_;
4758 JSGraph* jsgraph_;
4759 StubCallMode stub_mode_;
4760 SetOncePointer<const Operator> allocate_heap_number_operator_;
4761 };
4762 } // namespace
4763
CompileJSToWasmWrapper(Isolate * isolate,const wasm::NativeModule * native_module,wasm::FunctionSig * sig,bool is_import,wasm::UseTrapHandler use_trap_handler)4764 MaybeHandle<Code> CompileJSToWasmWrapper(
4765 Isolate* isolate, const wasm::NativeModule* native_module,
4766 wasm::FunctionSig* sig, bool is_import,
4767 wasm::UseTrapHandler use_trap_handler) {
4768 TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm"),
4769 "CompileJSToWasmWrapper");
4770 const wasm::WasmModule* module = native_module->module();
4771
4772 //----------------------------------------------------------------------------
4773 // Create the Graph.
4774 //----------------------------------------------------------------------------
4775 Zone zone(isolate->allocator(), ZONE_NAME);
4776 Graph graph(&zone);
4777 CommonOperatorBuilder common(&zone);
4778 MachineOperatorBuilder machine(
4779 &zone, MachineType::PointerRepresentation(),
4780 InstructionSelector::SupportedMachineOperatorFlags(),
4781 InstructionSelector::AlignmentRequirements());
4782 JSGraph jsgraph(isolate, &graph, &common, nullptr, nullptr, &machine);
4783
4784 Node* control = nullptr;
4785 Node* effect = nullptr;
4786
4787 wasm::ModuleEnv env(module, use_trap_handler, wasm::kRuntimeExceptionSupport);
4788 WasmWrapperGraphBuilder builder(&zone, &env, &jsgraph, sig, nullptr,
4789 StubCallMode::kCallOnHeapBuiltin);
4790 builder.set_control_ptr(&control);
4791 builder.set_effect_ptr(&effect);
4792 builder.BuildJSToWasmWrapper(is_import);
4793
4794 //----------------------------------------------------------------------------
4795 // Run the compilation pipeline.
4796 //----------------------------------------------------------------------------
4797 #ifdef DEBUG
4798 EmbeddedVector<char, 32> func_name;
4799 static unsigned id = 0;
4800 func_name.Truncate(SNPrintF(func_name, "js-to-wasm#%d", id++));
4801 #else
4802 Vector<const char> func_name = CStrVector("js-to-wasm");
4803 #endif
4804
4805 OptimizedCompilationInfo info(func_name, &zone, Code::JS_TO_WASM_FUNCTION);
4806
4807 if (info.trace_turbo_graph_enabled()) { // Simple textual RPO.
4808 StdoutStream{} << "-- Graph after change lowering -- " << std::endl
4809 << AsRPO(graph);
4810 }
4811
4812 // Schedule and compile to machine code.
4813 int params = static_cast<int>(sig->parameter_count());
4814 CallDescriptor* incoming = Linkage::GetJSCallDescriptor(
4815 &zone, false, params + 1, CallDescriptor::kNoFlags);
4816
4817 MaybeHandle<Code> maybe_code = Pipeline::GenerateCodeForTesting(
4818 &info, isolate, incoming, &graph, WasmAssemblerOptions());
4819 Handle<Code> code;
4820 if (!maybe_code.ToHandle(&code)) {
4821 return maybe_code;
4822 }
4823 #ifdef ENABLE_DISASSEMBLER
4824 if (FLAG_print_opt_code) {
4825 CodeTracer::Scope tracing_scope(isolate->GetCodeTracer());
4826 OFStream os(tracing_scope.file());
4827 code->Disassemble(func_name.start(), os);
4828 }
4829 #endif
4830
4831 if (must_record_function_compilation(isolate)) {
4832 RecordFunctionCompilation(CodeEventListener::STUB_TAG, isolate, code,
4833 "%.*s", func_name.length(), func_name.start());
4834 }
4835
4836 return code;
4837 }
4838
CompileWasmToJSWrapper(Isolate * isolate,Handle<JSReceiver> target,wasm::FunctionSig * sig,uint32_t index,wasm::ModuleOrigin origin,wasm::UseTrapHandler use_trap_handler)4839 MaybeHandle<Code> CompileWasmToJSWrapper(
4840 Isolate* isolate, Handle<JSReceiver> target, wasm::FunctionSig* sig,
4841 uint32_t index, wasm::ModuleOrigin origin,
4842 wasm::UseTrapHandler use_trap_handler) {
4843 TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm"),
4844 "CompileWasmToJSWrapper");
4845 //----------------------------------------------------------------------------
4846 // Create the Graph
4847 //----------------------------------------------------------------------------
4848 Zone zone(isolate->allocator(), ZONE_NAME);
4849 Graph graph(&zone);
4850 CommonOperatorBuilder common(&zone);
4851 MachineOperatorBuilder machine(
4852 &zone, MachineType::PointerRepresentation(),
4853 InstructionSelector::SupportedMachineOperatorFlags(),
4854 InstructionSelector::AlignmentRequirements());
4855 JSGraph jsgraph(isolate, &graph, &common, nullptr, nullptr, &machine);
4856
4857 Node* control = nullptr;
4858 Node* effect = nullptr;
4859
4860 SourcePositionTable* source_position_table =
4861 origin == wasm::kAsmJsOrigin ? new (&zone) SourcePositionTable(&graph)
4862 : nullptr;
4863
4864 wasm::ModuleEnv env(nullptr, use_trap_handler,
4865 wasm::kRuntimeExceptionSupport);
4866
4867 WasmWrapperGraphBuilder builder(&zone, &env, &jsgraph, sig,
4868 source_position_table,
4869 StubCallMode::kCallWasmRuntimeStub);
4870 builder.set_control_ptr(&control);
4871 builder.set_effect_ptr(&effect);
4872 builder.BuildWasmToJSWrapper(target, index);
4873
4874 #ifdef DEBUG
4875 EmbeddedVector<char, 32> func_name;
4876 static unsigned id = 0;
4877 func_name.Truncate(SNPrintF(func_name, "wasm-to-js#%d", id++));
4878 #else
4879 Vector<const char> func_name = CStrVector("wasm-to-js");
4880 #endif
4881
4882 OptimizedCompilationInfo info(func_name, &zone, Code::WASM_TO_JS_FUNCTION);
4883
4884 if (info.trace_turbo_graph_enabled()) { // Simple textual RPO.
4885 StdoutStream{} << "-- Graph after change lowering -- " << std::endl
4886 << AsRPO(graph);
4887 }
4888
4889 // Schedule and compile to machine code.
4890 CallDescriptor* incoming = GetWasmCallDescriptor(&zone, sig);
4891 if (machine.Is32()) {
4892 incoming = GetI32WasmCallDescriptor(&zone, incoming);
4893 }
4894 MaybeHandle<Code> maybe_code = Pipeline::GenerateCodeForTesting(
4895 &info, isolate, incoming, &graph, AssemblerOptions::Default(isolate),
4896 nullptr, source_position_table);
4897 Handle<Code> code;
4898 if (!maybe_code.ToHandle(&code)) {
4899 return maybe_code;
4900 }
4901 #ifdef ENABLE_DISASSEMBLER
4902 if (FLAG_print_opt_code) {
4903 CodeTracer::Scope tracing_scope(isolate->GetCodeTracer());
4904 OFStream os(tracing_scope.file());
4905 code->Disassemble(func_name.start(), os);
4906 }
4907 #endif
4908
4909 if (must_record_function_compilation(isolate)) {
4910 RecordFunctionCompilation(CodeEventListener::STUB_TAG, isolate, code,
4911 "%.*s", func_name.length(), func_name.start());
4912 }
4913
4914 return code;
4915 }
4916
CompileWasmInterpreterEntry(Isolate * isolate,uint32_t func_index,wasm::FunctionSig * sig)4917 MaybeHandle<Code> CompileWasmInterpreterEntry(Isolate* isolate,
4918 uint32_t func_index,
4919 wasm::FunctionSig* sig) {
4920 //----------------------------------------------------------------------------
4921 // Create the Graph
4922 //----------------------------------------------------------------------------
4923 Zone zone(isolate->allocator(), ZONE_NAME);
4924 Graph graph(&zone);
4925 CommonOperatorBuilder common(&zone);
4926 MachineOperatorBuilder machine(
4927 &zone, MachineType::PointerRepresentation(),
4928 InstructionSelector::SupportedMachineOperatorFlags(),
4929 InstructionSelector::AlignmentRequirements());
4930 JSGraph jsgraph(isolate, &graph, &common, nullptr, nullptr, &machine);
4931
4932 Node* control = nullptr;
4933 Node* effect = nullptr;
4934
4935 WasmWrapperGraphBuilder builder(&zone, nullptr, &jsgraph, sig, nullptr,
4936 StubCallMode::kCallWasmRuntimeStub);
4937 builder.set_control_ptr(&control);
4938 builder.set_effect_ptr(&effect);
4939 builder.BuildWasmInterpreterEntry(func_index);
4940
4941 // Schedule and compile to machine code.
4942 CallDescriptor* incoming = GetWasmCallDescriptor(&zone, sig);
4943 if (machine.Is32()) {
4944 incoming = GetI32WasmCallDescriptor(&zone, incoming);
4945 }
4946 #ifdef DEBUG
4947 EmbeddedVector<char, 32> func_name;
4948 func_name.Truncate(
4949 SNPrintF(func_name, "wasm-interpreter-entry#%d", func_index));
4950 #else
4951 Vector<const char> func_name = CStrVector("wasm-interpreter-entry");
4952 #endif
4953
4954 OptimizedCompilationInfo info(func_name, &zone, Code::WASM_INTERPRETER_ENTRY);
4955
4956 if (info.trace_turbo_graph_enabled()) { // Simple textual RPO.
4957 StdoutStream{} << "-- Wasm interpreter entry graph -- " << std::endl
4958 << AsRPO(graph);
4959 }
4960
4961 MaybeHandle<Code> maybe_code = Pipeline::GenerateCodeForTesting(
4962 &info, isolate, incoming, &graph, AssemblerOptions::Default(isolate),
4963 nullptr);
4964 Handle<Code> code;
4965 if (!maybe_code.ToHandle(&code)) {
4966 return maybe_code;
4967 }
4968 #ifdef ENABLE_DISASSEMBLER
4969 if (FLAG_print_opt_code) {
4970 CodeTracer::Scope tracing_scope(isolate->GetCodeTracer());
4971 OFStream os(tracing_scope.file());
4972 code->Disassemble(func_name.start(), os);
4973 }
4974 #endif
4975
4976 if (must_record_function_compilation(isolate)) {
4977 RecordFunctionCompilation(CodeEventListener::STUB_TAG, isolate, code,
4978 "%.*s", func_name.length(), func_name.start());
4979 }
4980
4981 return maybe_code;
4982 }
4983
CompileCWasmEntry(Isolate * isolate,wasm::FunctionSig * sig)4984 MaybeHandle<Code> CompileCWasmEntry(Isolate* isolate, wasm::FunctionSig* sig) {
4985 Zone zone(isolate->allocator(), ZONE_NAME);
4986 Graph graph(&zone);
4987 CommonOperatorBuilder common(&zone);
4988 MachineOperatorBuilder machine(
4989 &zone, MachineType::PointerRepresentation(),
4990 InstructionSelector::SupportedMachineOperatorFlags(),
4991 InstructionSelector::AlignmentRequirements());
4992 JSGraph jsgraph(isolate, &graph, &common, nullptr, nullptr, &machine);
4993
4994 Node* control = nullptr;
4995 Node* effect = nullptr;
4996
4997 WasmWrapperGraphBuilder builder(&zone, nullptr, &jsgraph, sig, nullptr,
4998 StubCallMode::kCallOnHeapBuiltin);
4999 builder.set_control_ptr(&control);
5000 builder.set_effect_ptr(&effect);
5001 builder.BuildCWasmEntry();
5002
5003 // Schedule and compile to machine code.
5004 CallDescriptor* incoming = Linkage::GetJSCallDescriptor(
5005 &zone, false, CWasmEntryParameters::kNumParameters + 1,
5006 CallDescriptor::kNoFlags);
5007
5008 // Build a name in the form "c-wasm-entry:<params>:<returns>".
5009 static constexpr size_t kMaxNameLen = 128;
5010 char debug_name[kMaxNameLen] = "c-wasm-entry:";
5011 size_t name_len = strlen(debug_name);
5012 auto append_name_char = [&](char c) {
5013 if (name_len + 1 < kMaxNameLen) debug_name[name_len++] = c;
5014 };
5015 for (wasm::ValueType t : sig->parameters()) {
5016 append_name_char(wasm::ValueTypes::ShortNameOf(t));
5017 }
5018 append_name_char(':');
5019 for (wasm::ValueType t : sig->returns()) {
5020 append_name_char(wasm::ValueTypes::ShortNameOf(t));
5021 }
5022 debug_name[name_len] = '\0';
5023 Vector<const char> debug_name_vec(debug_name, name_len);
5024
5025 OptimizedCompilationInfo info(debug_name_vec, &zone, Code::C_WASM_ENTRY);
5026
5027 if (info.trace_turbo_graph_enabled()) { // Simple textual RPO.
5028 StdoutStream{} << "-- C Wasm entry graph -- " << std::endl << AsRPO(graph);
5029 }
5030
5031 MaybeHandle<Code> maybe_code = Pipeline::GenerateCodeForTesting(
5032 &info, isolate, incoming, &graph, AssemblerOptions::Default(isolate));
5033 Handle<Code> code;
5034 if (!maybe_code.ToHandle(&code)) {
5035 return maybe_code;
5036 }
5037 #ifdef ENABLE_DISASSEMBLER
5038 if (FLAG_print_opt_code) {
5039 CodeTracer::Scope tracing_scope(isolate->GetCodeTracer());
5040 OFStream os(tracing_scope.file());
5041 code->Disassemble(debug_name, os);
5042 }
5043 #endif
5044
5045 return code;
5046 }
5047
TurbofanWasmCompilationUnit(wasm::WasmCompilationUnit * wasm_unit)5048 TurbofanWasmCompilationUnit::TurbofanWasmCompilationUnit(
5049 wasm::WasmCompilationUnit* wasm_unit)
5050 : wasm_unit_(wasm_unit) {}
5051
5052 // Clears unique_ptrs, but (part of) the type is forward declared in the header.
5053 TurbofanWasmCompilationUnit::~TurbofanWasmCompilationUnit() = default;
5054
BuildGraphForWasmFunction(wasm::WasmFeatures * detected,double * decode_ms,MachineGraph * mcgraph,NodeOriginTable * node_origins)5055 SourcePositionTable* TurbofanWasmCompilationUnit::BuildGraphForWasmFunction(
5056 wasm::WasmFeatures* detected, double* decode_ms, MachineGraph* mcgraph,
5057 NodeOriginTable* node_origins) {
5058 base::ElapsedTimer decode_timer;
5059 if (FLAG_trace_wasm_decode_time) {
5060 decode_timer.Start();
5061 }
5062
5063 // Create a TF graph during decoding.
5064 SourcePositionTable* source_position_table =
5065 new (mcgraph->zone()) SourcePositionTable(mcgraph->graph());
5066 WasmGraphBuilder builder(wasm_unit_->env_, mcgraph->zone(), mcgraph,
5067 wasm_unit_->func_body_.sig, source_position_table);
5068 graph_construction_result_ = wasm::BuildTFGraph(
5069 wasm_unit_->wasm_engine_->allocator(),
5070 wasm_unit_->native_module_->enabled_features(), wasm_unit_->env_->module,
5071 &builder, detected, wasm_unit_->func_body_, node_origins);
5072 if (graph_construction_result_.failed()) {
5073 if (FLAG_trace_wasm_compiler) {
5074 StdoutStream{} << "Compilation failed: "
5075 << graph_construction_result_.error_msg() << std::endl;
5076 }
5077 return nullptr;
5078 }
5079
5080 builder.LowerInt64();
5081
5082 if (builder.has_simd() &&
5083 (!CpuFeatures::SupportsWasmSimd128() || wasm_unit_->env_->lower_simd)) {
5084 SimdScalarLowering(
5085 mcgraph,
5086 CreateMachineSignature(mcgraph->zone(), wasm_unit_->func_body_.sig))
5087 .LowerGraph();
5088 }
5089
5090 if (wasm_unit_->func_index_ >= FLAG_trace_wasm_ast_start &&
5091 wasm_unit_->func_index_ < FLAG_trace_wasm_ast_end) {
5092 PrintRawWasmCode(wasm_unit_->wasm_engine_->allocator(),
5093 wasm_unit_->func_body_, wasm_unit_->env_->module,
5094 wasm::kPrintLocals);
5095 }
5096 if (FLAG_trace_wasm_decode_time) {
5097 *decode_ms = decode_timer.Elapsed().InMillisecondsF();
5098 }
5099 return source_position_table;
5100 }
5101
5102 namespace {
GetDebugName(Zone * zone,wasm::WasmName name,int index)5103 Vector<const char> GetDebugName(Zone* zone, wasm::WasmName name, int index) {
5104 if (!name.is_empty()) {
5105 return name;
5106 }
5107 #ifdef DEBUG
5108 constexpr int kBufferLength = 15;
5109
5110 EmbeddedVector<char, kBufferLength> name_vector;
5111 int name_len = SNPrintF(name_vector, "wasm#%d", index);
5112 DCHECK(name_len > 0 && name_len < name_vector.length());
5113
5114 char* index_name = zone->NewArray<char>(name_len);
5115 memcpy(index_name, name_vector.start(), name_len);
5116 return Vector<const char>(index_name, name_len);
5117 #else
5118 return {};
5119 #endif
5120 }
5121
5122 } // namespace
5123
ExecuteCompilation(wasm::WasmFeatures * detected)5124 void TurbofanWasmCompilationUnit::ExecuteCompilation(
5125 wasm::WasmFeatures* detected) {
5126 TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm"),
5127 "ExecuteTurbofanCompilation");
5128 double decode_ms = 0;
5129 size_t node_count = 0;
5130
5131 // Scope for the {graph_zone}.
5132 {
5133 Zone graph_zone(wasm_unit_->wasm_engine_->allocator(), ZONE_NAME);
5134 MachineGraph* mcgraph = new (&graph_zone)
5135 MachineGraph(new (&graph_zone) Graph(&graph_zone),
5136 new (&graph_zone) CommonOperatorBuilder(&graph_zone),
5137 new (&graph_zone) MachineOperatorBuilder(
5138 &graph_zone, MachineType::PointerRepresentation(),
5139 InstructionSelector::SupportedMachineOperatorFlags(),
5140 InstructionSelector::AlignmentRequirements()));
5141
5142 Zone compilation_zone(wasm_unit_->wasm_engine_->allocator(), ZONE_NAME);
5143
5144 OptimizedCompilationInfo info(
5145 GetDebugName(&compilation_zone, wasm_unit_->func_name_,
5146 wasm_unit_->func_index_),
5147 &compilation_zone, Code::WASM_FUNCTION);
5148 if (wasm_unit_->env_->runtime_exception_support) {
5149 info.SetWasmRuntimeExceptionSupport();
5150 }
5151
5152 NodeOriginTable* node_origins = info.trace_turbo_json_enabled()
5153 ? new (&graph_zone)
5154 NodeOriginTable(mcgraph->graph())
5155 : nullptr;
5156 SourcePositionTable* source_positions =
5157 BuildGraphForWasmFunction(detected, &decode_ms, mcgraph, node_origins);
5158
5159 if (graph_construction_result_.failed()) {
5160 ok_ = false;
5161 return;
5162 }
5163
5164 if (node_origins) {
5165 node_origins->AddDecorator();
5166 }
5167
5168 base::ElapsedTimer pipeline_timer;
5169 if (FLAG_trace_wasm_decode_time) {
5170 node_count = mcgraph->graph()->NodeCount();
5171 pipeline_timer.Start();
5172 }
5173
5174 // Run the compiler pipeline to generate machine code.
5175 auto call_descriptor =
5176 GetWasmCallDescriptor(&compilation_zone, wasm_unit_->func_body_.sig);
5177 if (mcgraph->machine()->Is32()) {
5178 call_descriptor =
5179 GetI32WasmCallDescriptor(&compilation_zone, call_descriptor);
5180 }
5181
5182 std::unique_ptr<OptimizedCompilationJob> job(
5183 Pipeline::NewWasmCompilationJob(
5184 &info, wasm_unit_->wasm_engine_, mcgraph, call_descriptor,
5185 source_positions, node_origins, wasm_unit_->func_body_,
5186 const_cast<wasm::WasmModule*>(wasm_unit_->env_->module),
5187 wasm_unit_->native_module_, wasm_unit_->func_index_,
5188 wasm_unit_->env_->module->origin));
5189 ok_ = job->ExecuteJob() == CompilationJob::SUCCEEDED;
5190 // TODO(bradnelson): Improve histogram handling of size_t.
5191 wasm_unit_->counters_->wasm_compile_function_peak_memory_bytes()->AddSample(
5192 static_cast<int>(mcgraph->graph()->zone()->allocation_size()));
5193
5194 if (FLAG_trace_wasm_decode_time) {
5195 double pipeline_ms = pipeline_timer.Elapsed().InMillisecondsF();
5196 PrintF(
5197 "wasm-compilation phase 1 ok: %u bytes, %0.3f ms decode, %zu nodes, "
5198 "%0.3f ms pipeline\n",
5199 static_cast<unsigned>(wasm_unit_->func_body_.end -
5200 wasm_unit_->func_body_.start),
5201 decode_ms, node_count, pipeline_ms);
5202 }
5203 if (ok_) wasm_code_ = info.wasm_code();
5204 }
5205 if (ok_) wasm_unit_->native_module()->PublishCode(wasm_code_);
5206 }
5207
FinishCompilation(wasm::ErrorThrower * thrower)5208 wasm::WasmCode* TurbofanWasmCompilationUnit::FinishCompilation(
5209 wasm::ErrorThrower* thrower) {
5210 if (!ok_) {
5211 if (graph_construction_result_.failed()) {
5212 // Add the function as another context for the exception.
5213 EmbeddedVector<char, 128> message;
5214 if (wasm_unit_->func_name_.start() == nullptr) {
5215 SNPrintF(message, "Compiling wasm function #%d failed",
5216 wasm_unit_->func_index_);
5217 } else {
5218 wasm::TruncatedUserString<> trunc_name(wasm_unit_->func_name_);
5219 SNPrintF(message, "Compiling wasm function #%d:%.*s failed",
5220 wasm_unit_->func_index_, trunc_name.length(),
5221 trunc_name.start());
5222 }
5223 thrower->CompileFailed(message.start(), graph_construction_result_);
5224 }
5225
5226 return nullptr;
5227 }
5228 return wasm_code_;
5229 }
5230
5231 namespace {
5232 // Helper for allocating either an GP or FP reg, or the next stack slot.
5233 class LinkageLocationAllocator {
5234 public:
5235 template <size_t kNumGpRegs, size_t kNumFpRegs>
LinkageLocationAllocator(const Register (& gp)[kNumGpRegs],const DoubleRegister (& fp)[kNumFpRegs])5236 constexpr LinkageLocationAllocator(const Register (&gp)[kNumGpRegs],
5237 const DoubleRegister (&fp)[kNumFpRegs])
5238 : allocator_(wasm::LinkageAllocator(gp, fp)) {}
5239
Next(MachineRepresentation rep)5240 LinkageLocation Next(MachineRepresentation rep) {
5241 MachineType type = MachineType::TypeForRepresentation(rep);
5242 if (IsFloatingPoint(rep)) {
5243 if (allocator_.CanAllocateFP(rep)) {
5244 int reg_code = allocator_.NextFpReg(rep);
5245 return LinkageLocation::ForRegister(reg_code, type);
5246 }
5247 } else if (allocator_.CanAllocateGP()) {
5248 int reg_code = allocator_.NextGpReg();
5249 return LinkageLocation::ForRegister(reg_code, type);
5250 }
5251 // Cannot use register; use stack slot.
5252 int index = -1 - allocator_.NextStackSlot(rep);
5253 return LinkageLocation::ForCallerFrameSlot(index, type);
5254 }
5255
SetStackOffset(int offset)5256 void SetStackOffset(int offset) { allocator_.SetStackOffset(offset); }
NumStackSlots() const5257 int NumStackSlots() const { return allocator_.NumStackSlots(); }
5258
5259 private:
5260 wasm::LinkageAllocator allocator_;
5261 };
5262 } // namespace
5263
5264 // General code uses the above configuration data.
GetWasmCallDescriptor(Zone * zone,wasm::FunctionSig * fsig,WasmGraphBuilder::UseRetpoline use_retpoline)5265 CallDescriptor* GetWasmCallDescriptor(
5266 Zone* zone, wasm::FunctionSig* fsig,
5267 WasmGraphBuilder::UseRetpoline use_retpoline) {
5268 // The '+ 1' here is to accomodate the instance object as first parameter.
5269 LocationSignature::Builder locations(zone, fsig->return_count(),
5270 fsig->parameter_count() + 1);
5271
5272 // Add register and/or stack parameter(s).
5273 LinkageLocationAllocator params(wasm::kGpParamRegisters,
5274 wasm::kFpParamRegisters);
5275
5276 // The instance object.
5277 locations.AddParam(params.Next(MachineRepresentation::kTaggedPointer));
5278
5279 const int parameter_count = static_cast<int>(fsig->parameter_count());
5280 for (int i = 0; i < parameter_count; i++) {
5281 MachineRepresentation param =
5282 wasm::ValueTypes::MachineRepresentationFor(fsig->GetParam(i));
5283 auto l = params.Next(param);
5284 locations.AddParam(l);
5285 }
5286
5287 // Add return location(s).
5288 LinkageLocationAllocator rets(wasm::kGpReturnRegisters,
5289 wasm::kFpReturnRegisters);
5290
5291 int parameter_slots = params.NumStackSlots();
5292 if (kPadArguments) parameter_slots = RoundUp(parameter_slots, 2);
5293
5294 rets.SetStackOffset(parameter_slots);
5295
5296 const int return_count = static_cast<int>(locations.return_count_);
5297 for (int i = 0; i < return_count; i++) {
5298 MachineRepresentation ret =
5299 wasm::ValueTypes::MachineRepresentationFor(fsig->GetReturn(i));
5300 auto l = rets.Next(ret);
5301 locations.AddReturn(l);
5302 }
5303
5304 const RegList kCalleeSaveRegisters = 0;
5305 const RegList kCalleeSaveFPRegisters = 0;
5306
5307 // The target for wasm calls is always a code object.
5308 MachineType target_type = MachineType::Pointer();
5309 LinkageLocation target_loc = LinkageLocation::ForAnyRegister(target_type);
5310
5311 CallDescriptor::Kind kind = CallDescriptor::kCallWasmFunction;
5312
5313 CallDescriptor::Flags flags =
5314 use_retpoline ? CallDescriptor::kRetpoline : CallDescriptor::kNoFlags;
5315 return new (zone) CallDescriptor( // --
5316 kind, // kind
5317 target_type, // target MachineType
5318 target_loc, // target location
5319 locations.Build(), // location_sig
5320 parameter_slots, // stack_parameter_count
5321 compiler::Operator::kNoProperties, // properties
5322 kCalleeSaveRegisters, // callee-saved registers
5323 kCalleeSaveFPRegisters, // callee-saved fp regs
5324 flags, // flags
5325 "wasm-call", // debug name
5326 0, // allocatable registers
5327 rets.NumStackSlots() - parameter_slots); // stack_return_count
5328 }
5329
5330 namespace {
ReplaceTypeInCallDescriptorWith(Zone * zone,CallDescriptor * call_descriptor,size_t num_replacements,MachineType input_type,MachineRepresentation output_type)5331 CallDescriptor* ReplaceTypeInCallDescriptorWith(
5332 Zone* zone, CallDescriptor* call_descriptor, size_t num_replacements,
5333 MachineType input_type, MachineRepresentation output_type) {
5334 size_t parameter_count = call_descriptor->ParameterCount();
5335 size_t return_count = call_descriptor->ReturnCount();
5336 for (size_t i = 0; i < call_descriptor->ParameterCount(); i++) {
5337 if (call_descriptor->GetParameterType(i) == input_type) {
5338 parameter_count += num_replacements - 1;
5339 }
5340 }
5341 for (size_t i = 0; i < call_descriptor->ReturnCount(); i++) {
5342 if (call_descriptor->GetReturnType(i) == input_type) {
5343 return_count += num_replacements - 1;
5344 }
5345 }
5346 if (parameter_count == call_descriptor->ParameterCount() &&
5347 return_count == call_descriptor->ReturnCount()) {
5348 return call_descriptor;
5349 }
5350
5351 LocationSignature::Builder locations(zone, return_count, parameter_count);
5352
5353 LinkageLocationAllocator params(wasm::kGpParamRegisters,
5354 wasm::kFpParamRegisters);
5355 for (size_t i = 0; i < call_descriptor->ParameterCount(); i++) {
5356 if (call_descriptor->GetParameterType(i) == input_type) {
5357 for (size_t j = 0; j < num_replacements; j++) {
5358 locations.AddParam(params.Next(output_type));
5359 }
5360 } else {
5361 locations.AddParam(
5362 params.Next(call_descriptor->GetParameterType(i).representation()));
5363 }
5364 }
5365
5366 LinkageLocationAllocator rets(wasm::kGpReturnRegisters,
5367 wasm::kFpReturnRegisters);
5368 rets.SetStackOffset(params.NumStackSlots());
5369 for (size_t i = 0; i < call_descriptor->ReturnCount(); i++) {
5370 if (call_descriptor->GetReturnType(i) == input_type) {
5371 for (size_t j = 0; j < num_replacements; j++) {
5372 locations.AddReturn(rets.Next(output_type));
5373 }
5374 } else {
5375 locations.AddReturn(
5376 rets.Next(call_descriptor->GetReturnType(i).representation()));
5377 }
5378 }
5379
5380 return new (zone) CallDescriptor( // --
5381 call_descriptor->kind(), // kind
5382 call_descriptor->GetInputType(0), // target MachineType
5383 call_descriptor->GetInputLocation(0), // target location
5384 locations.Build(), // location_sig
5385 params.NumStackSlots(), // stack_parameter_count
5386 call_descriptor->properties(), // properties
5387 call_descriptor->CalleeSavedRegisters(), // callee-saved registers
5388 call_descriptor->CalleeSavedFPRegisters(), // callee-saved fp regs
5389 call_descriptor->flags(), // flags
5390 call_descriptor->debug_name(), // debug name
5391 call_descriptor->AllocatableRegisters(), // allocatable registers
5392 rets.NumStackSlots() - params.NumStackSlots()); // stack_return_count
5393 }
5394 } // namespace
5395
GetI32WasmCallDescriptor(Zone * zone,CallDescriptor * call_descriptor)5396 CallDescriptor* GetI32WasmCallDescriptor(Zone* zone,
5397 CallDescriptor* call_descriptor) {
5398 return ReplaceTypeInCallDescriptorWith(zone, call_descriptor, 2,
5399 MachineType::Int64(),
5400 MachineRepresentation::kWord32);
5401 }
5402
GetI32WasmCallDescriptorForSimd(Zone * zone,CallDescriptor * call_descriptor)5403 CallDescriptor* GetI32WasmCallDescriptorForSimd(
5404 Zone* zone, CallDescriptor* call_descriptor) {
5405 return ReplaceTypeInCallDescriptorWith(zone, call_descriptor, 4,
5406 MachineType::Simd128(),
5407 MachineRepresentation::kWord32);
5408 }
5409
WasmAssemblerOptions()5410 AssemblerOptions WasmAssemblerOptions() {
5411 AssemblerOptions options;
5412 options.record_reloc_info_for_serialization = true;
5413 options.enable_root_array_delta_access = false;
5414 return options;
5415 }
5416
5417 #undef WASM_64
5418 #undef FATAL_UNSUPPORTED_OPCODE
5419 #undef WASM_INSTANCE_OBJECT_OFFSET
5420 #undef LOAD_INSTANCE_FIELD
5421 #undef LOAD_FIXED_ARRAY_SLOT
5422
5423 } // namespace compiler
5424 } // namespace internal
5425 } // namespace v8
5426