• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // Copyright 2014 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #include "src/compiler/instruction-selector.h"
6 
7 #include <limits>
8 
9 #include "src/assembler-inl.h"
10 #include "src/base/adapters.h"
11 #include "src/compiler/compiler-source-position-table.h"
12 #include "src/compiler/instruction-selector-impl.h"
13 #include "src/compiler/node-matchers.h"
14 #include "src/compiler/pipeline.h"
15 #include "src/compiler/schedule.h"
16 #include "src/compiler/state-values-utils.h"
17 #include "src/deoptimizer.h"
18 
19 namespace v8 {
20 namespace internal {
21 namespace compiler {
22 
InstructionSelector(Zone * zone,size_t node_count,Linkage * linkage,InstructionSequence * sequence,Schedule * schedule,SourcePositionTable * source_positions,Frame * frame,EnableSwitchJumpTable enable_switch_jump_table,SourcePositionMode source_position_mode,Features features,EnableScheduling enable_scheduling,EnableRootsRelativeAddressing enable_roots_relative_addressing,PoisoningMitigationLevel poisoning_level,EnableTraceTurboJson trace_turbo)23 InstructionSelector::InstructionSelector(
24     Zone* zone, size_t node_count, Linkage* linkage,
25     InstructionSequence* sequence, Schedule* schedule,
26     SourcePositionTable* source_positions, Frame* frame,
27     EnableSwitchJumpTable enable_switch_jump_table,
28     SourcePositionMode source_position_mode, Features features,
29     EnableScheduling enable_scheduling,
30     EnableRootsRelativeAddressing enable_roots_relative_addressing,
31     PoisoningMitigationLevel poisoning_level, EnableTraceTurboJson trace_turbo)
32     : zone_(zone),
33       linkage_(linkage),
34       sequence_(sequence),
35       source_positions_(source_positions),
36       source_position_mode_(source_position_mode),
37       features_(features),
38       schedule_(schedule),
39       current_block_(nullptr),
40       instructions_(zone),
41       continuation_inputs_(sequence->zone()),
42       continuation_outputs_(sequence->zone()),
43       defined_(node_count, false, zone),
44       used_(node_count, false, zone),
45       effect_level_(node_count, 0, zone),
46       virtual_registers_(node_count,
47                          InstructionOperand::kInvalidVirtualRegister, zone),
48       virtual_register_rename_(zone),
49       scheduler_(nullptr),
50       enable_scheduling_(enable_scheduling),
51       enable_roots_relative_addressing_(enable_roots_relative_addressing),
52       enable_switch_jump_table_(enable_switch_jump_table),
53       poisoning_level_(poisoning_level),
54       frame_(frame),
55       instruction_selection_failed_(false),
56       instr_origins_(sequence->zone()),
57       trace_turbo_(trace_turbo) {
58   instructions_.reserve(node_count);
59   continuation_inputs_.reserve(5);
60   continuation_outputs_.reserve(2);
61 
62   if (trace_turbo_ == kEnableTraceTurboJson) {
63     instr_origins_.assign(node_count, {-1, 0});
64   }
65 }
66 
SelectInstructions()67 bool InstructionSelector::SelectInstructions() {
68   // Mark the inputs of all phis in loop headers as used.
69   BasicBlockVector* blocks = schedule()->rpo_order();
70   for (auto const block : *blocks) {
71     if (!block->IsLoopHeader()) continue;
72     DCHECK_LE(2u, block->PredecessorCount());
73     for (Node* const phi : *block) {
74       if (phi->opcode() != IrOpcode::kPhi) continue;
75 
76       // Mark all inputs as used.
77       for (Node* const input : phi->inputs()) {
78         MarkAsUsed(input);
79       }
80     }
81   }
82 
83   // Visit each basic block in post order.
84   for (auto i = blocks->rbegin(); i != blocks->rend(); ++i) {
85     VisitBlock(*i);
86     if (instruction_selection_failed()) return false;
87   }
88 
89   // Schedule the selected instructions.
90   if (UseInstructionScheduling()) {
91     scheduler_ = new (zone()) InstructionScheduler(zone(), sequence());
92   }
93 
94   for (auto const block : *blocks) {
95     InstructionBlock* instruction_block =
96         sequence()->InstructionBlockAt(RpoNumber::FromInt(block->rpo_number()));
97     for (size_t i = 0; i < instruction_block->phis().size(); i++) {
98       UpdateRenamesInPhi(instruction_block->PhiAt(i));
99     }
100     size_t end = instruction_block->code_end();
101     size_t start = instruction_block->code_start();
102     DCHECK_LE(end, start);
103     StartBlock(RpoNumber::FromInt(block->rpo_number()));
104     if (end != start) {
105       while (start-- > end + 1) {
106         UpdateRenames(instructions_[start]);
107         AddInstruction(instructions_[start]);
108       }
109       UpdateRenames(instructions_[end]);
110       AddTerminator(instructions_[end]);
111     }
112     EndBlock(RpoNumber::FromInt(block->rpo_number()));
113   }
114 #if DEBUG
115   sequence()->ValidateSSA();
116 #endif
117   return true;
118 }
119 
StartBlock(RpoNumber rpo)120 void InstructionSelector::StartBlock(RpoNumber rpo) {
121   if (UseInstructionScheduling()) {
122     DCHECK_NOT_NULL(scheduler_);
123     scheduler_->StartBlock(rpo);
124   } else {
125     sequence()->StartBlock(rpo);
126   }
127 }
128 
129 
EndBlock(RpoNumber rpo)130 void InstructionSelector::EndBlock(RpoNumber rpo) {
131   if (UseInstructionScheduling()) {
132     DCHECK_NOT_NULL(scheduler_);
133     scheduler_->EndBlock(rpo);
134   } else {
135     sequence()->EndBlock(rpo);
136   }
137 }
138 
AddTerminator(Instruction * instr)139 void InstructionSelector::AddTerminator(Instruction* instr) {
140   if (UseInstructionScheduling()) {
141     DCHECK_NOT_NULL(scheduler_);
142     scheduler_->AddTerminator(instr);
143   } else {
144     sequence()->AddInstruction(instr);
145   }
146 }
147 
AddInstruction(Instruction * instr)148 void InstructionSelector::AddInstruction(Instruction* instr) {
149   if (UseInstructionScheduling()) {
150     DCHECK_NOT_NULL(scheduler_);
151     scheduler_->AddInstruction(instr);
152   } else {
153     sequence()->AddInstruction(instr);
154   }
155 }
156 
Emit(InstructionCode opcode,InstructionOperand output,size_t temp_count,InstructionOperand * temps)157 Instruction* InstructionSelector::Emit(InstructionCode opcode,
158                                        InstructionOperand output,
159                                        size_t temp_count,
160                                        InstructionOperand* temps) {
161   size_t output_count = output.IsInvalid() ? 0 : 1;
162   return Emit(opcode, output_count, &output, 0, nullptr, temp_count, temps);
163 }
164 
165 
Emit(InstructionCode opcode,InstructionOperand output,InstructionOperand a,size_t temp_count,InstructionOperand * temps)166 Instruction* InstructionSelector::Emit(InstructionCode opcode,
167                                        InstructionOperand output,
168                                        InstructionOperand a, size_t temp_count,
169                                        InstructionOperand* temps) {
170   size_t output_count = output.IsInvalid() ? 0 : 1;
171   return Emit(opcode, output_count, &output, 1, &a, temp_count, temps);
172 }
173 
174 
Emit(InstructionCode opcode,InstructionOperand output,InstructionOperand a,InstructionOperand b,size_t temp_count,InstructionOperand * temps)175 Instruction* InstructionSelector::Emit(InstructionCode opcode,
176                                        InstructionOperand output,
177                                        InstructionOperand a,
178                                        InstructionOperand b, size_t temp_count,
179                                        InstructionOperand* temps) {
180   size_t output_count = output.IsInvalid() ? 0 : 1;
181   InstructionOperand inputs[] = {a, b};
182   size_t input_count = arraysize(inputs);
183   return Emit(opcode, output_count, &output, input_count, inputs, temp_count,
184               temps);
185 }
186 
187 
Emit(InstructionCode opcode,InstructionOperand output,InstructionOperand a,InstructionOperand b,InstructionOperand c,size_t temp_count,InstructionOperand * temps)188 Instruction* InstructionSelector::Emit(InstructionCode opcode,
189                                        InstructionOperand output,
190                                        InstructionOperand a,
191                                        InstructionOperand b,
192                                        InstructionOperand c, size_t temp_count,
193                                        InstructionOperand* temps) {
194   size_t output_count = output.IsInvalid() ? 0 : 1;
195   InstructionOperand inputs[] = {a, b, c};
196   size_t input_count = arraysize(inputs);
197   return Emit(opcode, output_count, &output, input_count, inputs, temp_count,
198               temps);
199 }
200 
201 
Emit(InstructionCode opcode,InstructionOperand output,InstructionOperand a,InstructionOperand b,InstructionOperand c,InstructionOperand d,size_t temp_count,InstructionOperand * temps)202 Instruction* InstructionSelector::Emit(
203     InstructionCode opcode, InstructionOperand output, InstructionOperand a,
204     InstructionOperand b, InstructionOperand c, InstructionOperand d,
205     size_t temp_count, InstructionOperand* temps) {
206   size_t output_count = output.IsInvalid() ? 0 : 1;
207   InstructionOperand inputs[] = {a, b, c, d};
208   size_t input_count = arraysize(inputs);
209   return Emit(opcode, output_count, &output, input_count, inputs, temp_count,
210               temps);
211 }
212 
213 
Emit(InstructionCode opcode,InstructionOperand output,InstructionOperand a,InstructionOperand b,InstructionOperand c,InstructionOperand d,InstructionOperand e,size_t temp_count,InstructionOperand * temps)214 Instruction* InstructionSelector::Emit(
215     InstructionCode opcode, InstructionOperand output, InstructionOperand a,
216     InstructionOperand b, InstructionOperand c, InstructionOperand d,
217     InstructionOperand e, size_t temp_count, InstructionOperand* temps) {
218   size_t output_count = output.IsInvalid() ? 0 : 1;
219   InstructionOperand inputs[] = {a, b, c, d, e};
220   size_t input_count = arraysize(inputs);
221   return Emit(opcode, output_count, &output, input_count, inputs, temp_count,
222               temps);
223 }
224 
225 
Emit(InstructionCode opcode,InstructionOperand output,InstructionOperand a,InstructionOperand b,InstructionOperand c,InstructionOperand d,InstructionOperand e,InstructionOperand f,size_t temp_count,InstructionOperand * temps)226 Instruction* InstructionSelector::Emit(
227     InstructionCode opcode, InstructionOperand output, InstructionOperand a,
228     InstructionOperand b, InstructionOperand c, InstructionOperand d,
229     InstructionOperand e, InstructionOperand f, size_t temp_count,
230     InstructionOperand* temps) {
231   size_t output_count = output.IsInvalid() ? 0 : 1;
232   InstructionOperand inputs[] = {a, b, c, d, e, f};
233   size_t input_count = arraysize(inputs);
234   return Emit(opcode, output_count, &output, input_count, inputs, temp_count,
235               temps);
236 }
237 
238 
Emit(InstructionCode opcode,size_t output_count,InstructionOperand * outputs,size_t input_count,InstructionOperand * inputs,size_t temp_count,InstructionOperand * temps)239 Instruction* InstructionSelector::Emit(
240     InstructionCode opcode, size_t output_count, InstructionOperand* outputs,
241     size_t input_count, InstructionOperand* inputs, size_t temp_count,
242     InstructionOperand* temps) {
243   if (output_count >= Instruction::kMaxOutputCount ||
244       input_count >= Instruction::kMaxInputCount ||
245       temp_count >= Instruction::kMaxTempCount) {
246     set_instruction_selection_failed();
247     return nullptr;
248   }
249 
250   Instruction* instr =
251       Instruction::New(instruction_zone(), opcode, output_count, outputs,
252                        input_count, inputs, temp_count, temps);
253   return Emit(instr);
254 }
255 
256 
Emit(Instruction * instr)257 Instruction* InstructionSelector::Emit(Instruction* instr) {
258   instructions_.push_back(instr);
259   return instr;
260 }
261 
262 
CanCover(Node * user,Node * node) const263 bool InstructionSelector::CanCover(Node* user, Node* node) const {
264   // 1. Both {user} and {node} must be in the same basic block.
265   if (schedule()->block(node) != schedule()->block(user)) {
266     return false;
267   }
268   // 2. Pure {node}s must be owned by the {user}.
269   if (node->op()->HasProperty(Operator::kPure)) {
270     return node->OwnedBy(user);
271   }
272   // 3. Impure {node}s must match the effect level of {user}.
273   if (GetEffectLevel(node) != GetEffectLevel(user)) {
274     return false;
275   }
276   // 4. Only {node} must have value edges pointing to {user}.
277   for (Edge const edge : node->use_edges()) {
278     if (edge.from() != user && NodeProperties::IsValueEdge(edge)) {
279       return false;
280     }
281   }
282   return true;
283 }
284 
IsOnlyUserOfNodeInSameBlock(Node * user,Node * node) const285 bool InstructionSelector::IsOnlyUserOfNodeInSameBlock(Node* user,
286                                                       Node* node) const {
287   BasicBlock* bb_user = schedule()->block(user);
288   BasicBlock* bb_node = schedule()->block(node);
289   if (bb_user != bb_node) return false;
290   for (Edge const edge : node->use_edges()) {
291     Node* from = edge.from();
292     if ((from != user) && (schedule()->block(from) == bb_user)) {
293       return false;
294     }
295   }
296   return true;
297 }
298 
UpdateRenames(Instruction * instruction)299 void InstructionSelector::UpdateRenames(Instruction* instruction) {
300   for (size_t i = 0; i < instruction->InputCount(); i++) {
301     TryRename(instruction->InputAt(i));
302   }
303 }
304 
UpdateRenamesInPhi(PhiInstruction * phi)305 void InstructionSelector::UpdateRenamesInPhi(PhiInstruction* phi) {
306   for (size_t i = 0; i < phi->operands().size(); i++) {
307     int vreg = phi->operands()[i];
308     int renamed = GetRename(vreg);
309     if (vreg != renamed) {
310       phi->RenameInput(i, renamed);
311     }
312   }
313 }
314 
GetRename(int virtual_register)315 int InstructionSelector::GetRename(int virtual_register) {
316   int rename = virtual_register;
317   while (true) {
318     if (static_cast<size_t>(rename) >= virtual_register_rename_.size()) break;
319     int next = virtual_register_rename_[rename];
320     if (next == InstructionOperand::kInvalidVirtualRegister) {
321       break;
322     }
323     rename = next;
324   }
325   return rename;
326 }
327 
TryRename(InstructionOperand * op)328 void InstructionSelector::TryRename(InstructionOperand* op) {
329   if (!op->IsUnallocated()) return;
330   UnallocatedOperand* unalloc = UnallocatedOperand::cast(op);
331   int vreg = unalloc->virtual_register();
332   int rename = GetRename(vreg);
333   if (rename != vreg) {
334     *unalloc = UnallocatedOperand(*unalloc, rename);
335   }
336 }
337 
SetRename(const Node * node,const Node * rename)338 void InstructionSelector::SetRename(const Node* node, const Node* rename) {
339   int vreg = GetVirtualRegister(node);
340   if (static_cast<size_t>(vreg) >= virtual_register_rename_.size()) {
341     int invalid = InstructionOperand::kInvalidVirtualRegister;
342     virtual_register_rename_.resize(vreg + 1, invalid);
343   }
344   virtual_register_rename_[vreg] = GetVirtualRegister(rename);
345 }
346 
GetVirtualRegister(const Node * node)347 int InstructionSelector::GetVirtualRegister(const Node* node) {
348   DCHECK_NOT_NULL(node);
349   size_t const id = node->id();
350   DCHECK_LT(id, virtual_registers_.size());
351   int virtual_register = virtual_registers_[id];
352   if (virtual_register == InstructionOperand::kInvalidVirtualRegister) {
353     virtual_register = sequence()->NextVirtualRegister();
354     virtual_registers_[id] = virtual_register;
355   }
356   return virtual_register;
357 }
358 
359 
GetVirtualRegistersForTesting() const360 const std::map<NodeId, int> InstructionSelector::GetVirtualRegistersForTesting()
361     const {
362   std::map<NodeId, int> virtual_registers;
363   for (size_t n = 0; n < virtual_registers_.size(); ++n) {
364     if (virtual_registers_[n] != InstructionOperand::kInvalidVirtualRegister) {
365       NodeId const id = static_cast<NodeId>(n);
366       virtual_registers.insert(std::make_pair(id, virtual_registers_[n]));
367     }
368   }
369   return virtual_registers;
370 }
371 
372 
IsDefined(Node * node) const373 bool InstructionSelector::IsDefined(Node* node) const {
374   DCHECK_NOT_NULL(node);
375   size_t const id = node->id();
376   DCHECK_LT(id, defined_.size());
377   return defined_[id];
378 }
379 
380 
MarkAsDefined(Node * node)381 void InstructionSelector::MarkAsDefined(Node* node) {
382   DCHECK_NOT_NULL(node);
383   size_t const id = node->id();
384   DCHECK_LT(id, defined_.size());
385   defined_[id] = true;
386 }
387 
388 
IsUsed(Node * node) const389 bool InstructionSelector::IsUsed(Node* node) const {
390   DCHECK_NOT_NULL(node);
391   // TODO(bmeurer): This is a terrible monster hack, but we have to make sure
392   // that the Retain is actually emitted, otherwise the GC will mess up.
393   if (node->opcode() == IrOpcode::kRetain) return true;
394   if (!node->op()->HasProperty(Operator::kEliminatable)) return true;
395   size_t const id = node->id();
396   DCHECK_LT(id, used_.size());
397   return used_[id];
398 }
399 
400 
MarkAsUsed(Node * node)401 void InstructionSelector::MarkAsUsed(Node* node) {
402   DCHECK_NOT_NULL(node);
403   size_t const id = node->id();
404   DCHECK_LT(id, used_.size());
405   used_[id] = true;
406 }
407 
GetEffectLevel(Node * node) const408 int InstructionSelector::GetEffectLevel(Node* node) const {
409   DCHECK_NOT_NULL(node);
410   size_t const id = node->id();
411   DCHECK_LT(id, effect_level_.size());
412   return effect_level_[id];
413 }
414 
SetEffectLevel(Node * node,int effect_level)415 void InstructionSelector::SetEffectLevel(Node* node, int effect_level) {
416   DCHECK_NOT_NULL(node);
417   size_t const id = node->id();
418   DCHECK_LT(id, effect_level_.size());
419   effect_level_[id] = effect_level;
420 }
421 
CanAddressRelativeToRootsRegister() const422 bool InstructionSelector::CanAddressRelativeToRootsRegister() const {
423   return enable_roots_relative_addressing_ == kEnableRootsRelativeAddressing &&
424          CanUseRootsRegister();
425 }
426 
CanUseRootsRegister() const427 bool InstructionSelector::CanUseRootsRegister() const {
428   return linkage()->GetIncomingDescriptor()->flags() &
429          CallDescriptor::kCanUseRoots;
430 }
431 
MarkAsRepresentation(MachineRepresentation rep,const InstructionOperand & op)432 void InstructionSelector::MarkAsRepresentation(MachineRepresentation rep,
433                                                const InstructionOperand& op) {
434   UnallocatedOperand unalloc = UnallocatedOperand::cast(op);
435   sequence()->MarkAsRepresentation(rep, unalloc.virtual_register());
436 }
437 
438 
MarkAsRepresentation(MachineRepresentation rep,Node * node)439 void InstructionSelector::MarkAsRepresentation(MachineRepresentation rep,
440                                                Node* node) {
441   sequence()->MarkAsRepresentation(rep, GetVirtualRegister(node));
442 }
443 
444 namespace {
445 
OperandForDeopt(Isolate * isolate,OperandGenerator * g,Node * input,FrameStateInputKind kind,MachineRepresentation rep)446 InstructionOperand OperandForDeopt(Isolate* isolate, OperandGenerator* g,
447                                    Node* input, FrameStateInputKind kind,
448                                    MachineRepresentation rep) {
449   if (rep == MachineRepresentation::kNone) {
450     return g->TempImmediate(FrameStateDescriptor::kImpossibleValue);
451   }
452 
453   switch (input->opcode()) {
454     case IrOpcode::kInt32Constant:
455     case IrOpcode::kInt64Constant:
456     case IrOpcode::kNumberConstant:
457     case IrOpcode::kFloat32Constant:
458     case IrOpcode::kFloat64Constant:
459       return g->UseImmediate(input);
460     case IrOpcode::kHeapConstant: {
461       if (!CanBeTaggedPointer(rep)) {
462         // If we have inconsistent static and dynamic types, e.g. if we
463         // smi-check a string, we can get here with a heap object that
464         // says it is a smi. In that case, we return an invalid instruction
465         // operand, which will be interpreted as an optimized-out value.
466 
467         // TODO(jarin) Ideally, we should turn the current instruction
468         // into an abort (we should never execute it).
469         return InstructionOperand();
470       }
471 
472       Handle<HeapObject> constant = HeapConstantOf(input->op());
473       Heap::RootListIndex root_index;
474       if (isolate->heap()->IsRootHandle(constant, &root_index) &&
475           root_index == Heap::kOptimizedOutRootIndex) {
476         // For an optimized-out object we return an invalid instruction
477         // operand, so that we take the fast path for optimized-out values.
478         return InstructionOperand();
479       }
480 
481       return g->UseImmediate(input);
482     }
483     case IrOpcode::kArgumentsElementsState:
484     case IrOpcode::kArgumentsLengthState:
485     case IrOpcode::kObjectState:
486     case IrOpcode::kTypedObjectState:
487       UNREACHABLE();
488       break;
489     default:
490       switch (kind) {
491         case FrameStateInputKind::kStackSlot:
492           return g->UseUniqueSlot(input);
493         case FrameStateInputKind::kAny:
494           // Currently deopts "wrap" other operations, so the deopt's inputs
495           // are potentially needed until the end of the deoptimising code.
496           return g->UseAnyAtEnd(input);
497       }
498   }
499   UNREACHABLE();
500 }
501 
502 }  // namespace
503 
504 class StateObjectDeduplicator {
505  public:
StateObjectDeduplicator(Zone * zone)506   explicit StateObjectDeduplicator(Zone* zone) : objects_(zone) {}
507   static const size_t kNotDuplicated = SIZE_MAX;
508 
GetObjectId(Node * node)509   size_t GetObjectId(Node* node) {
510     DCHECK(node->opcode() == IrOpcode::kTypedObjectState ||
511            node->opcode() == IrOpcode::kObjectId ||
512            node->opcode() == IrOpcode::kArgumentsElementsState);
513     for (size_t i = 0; i < objects_.size(); ++i) {
514       if (objects_[i] == node) return i;
515       // ObjectId nodes are the Turbofan way to express objects with the same
516       // identity in the deopt info. So they should always be mapped to
517       // previously appearing TypedObjectState nodes.
518       if (HasObjectId(objects_[i]) && HasObjectId(node) &&
519           ObjectIdOf(objects_[i]->op()) == ObjectIdOf(node->op())) {
520         return i;
521       }
522     }
523     DCHECK(node->opcode() == IrOpcode::kTypedObjectState ||
524            node->opcode() == IrOpcode::kArgumentsElementsState);
525     return kNotDuplicated;
526   }
527 
InsertObject(Node * node)528   size_t InsertObject(Node* node) {
529     DCHECK(node->opcode() == IrOpcode::kTypedObjectState ||
530            node->opcode() == IrOpcode::kObjectId ||
531            node->opcode() == IrOpcode::kArgumentsElementsState);
532     size_t id = objects_.size();
533     objects_.push_back(node);
534     return id;
535   }
536 
537  private:
HasObjectId(Node * node)538   static bool HasObjectId(Node* node) {
539     return node->opcode() == IrOpcode::kTypedObjectState ||
540            node->opcode() == IrOpcode::kObjectId;
541   }
542 
543   ZoneVector<Node*> objects_;
544 };
545 
546 // Returns the number of instruction operands added to inputs.
AddOperandToStateValueDescriptor(StateValueList * values,InstructionOperandVector * inputs,OperandGenerator * g,StateObjectDeduplicator * deduplicator,Node * input,MachineType type,FrameStateInputKind kind,Zone * zone)547 size_t InstructionSelector::AddOperandToStateValueDescriptor(
548     StateValueList* values, InstructionOperandVector* inputs,
549     OperandGenerator* g, StateObjectDeduplicator* deduplicator, Node* input,
550     MachineType type, FrameStateInputKind kind, Zone* zone) {
551   if (input == nullptr) {
552     values->PushOptimizedOut();
553     return 0;
554   }
555 
556   switch (input->opcode()) {
557     case IrOpcode::kArgumentsElementsState: {
558       values->PushArgumentsElements(ArgumentsStateTypeOf(input->op()));
559       // The elements backing store of an arguments object participates in the
560       // duplicate object counting, but can itself never appear duplicated.
561       DCHECK_EQ(StateObjectDeduplicator::kNotDuplicated,
562                 deduplicator->GetObjectId(input));
563       deduplicator->InsertObject(input);
564       return 0;
565     }
566     case IrOpcode::kArgumentsLengthState: {
567       values->PushArgumentsLength(ArgumentsStateTypeOf(input->op()));
568       return 0;
569     }
570     case IrOpcode::kObjectState: {
571       UNREACHABLE();
572     }
573     case IrOpcode::kTypedObjectState:
574     case IrOpcode::kObjectId: {
575       size_t id = deduplicator->GetObjectId(input);
576       if (id == StateObjectDeduplicator::kNotDuplicated) {
577         DCHECK_EQ(IrOpcode::kTypedObjectState, input->opcode());
578         size_t entries = 0;
579         id = deduplicator->InsertObject(input);
580         StateValueList* nested = values->PushRecursiveField(zone, id);
581         int const input_count = input->op()->ValueInputCount();
582         ZoneVector<MachineType> const* types = MachineTypesOf(input->op());
583         for (int i = 0; i < input_count; ++i) {
584           entries += AddOperandToStateValueDescriptor(
585               nested, inputs, g, deduplicator, input->InputAt(i), types->at(i),
586               kind, zone);
587         }
588         return entries;
589       } else {
590         // Deoptimizer counts duplicate objects for the running id, so we have
591         // to push the input again.
592         deduplicator->InsertObject(input);
593         values->PushDuplicate(id);
594         return 0;
595       }
596     }
597     default: {
598       InstructionOperand op =
599           OperandForDeopt(isolate(), g, input, kind, type.representation());
600       if (op.kind() == InstructionOperand::INVALID) {
601         // Invalid operand means the value is impossible or optimized-out.
602         values->PushOptimizedOut();
603         return 0;
604       } else {
605         inputs->push_back(op);
606         values->PushPlain(type);
607         return 1;
608       }
609     }
610   }
611 }
612 
613 
614 // Returns the number of instruction operands added to inputs.
AddInputsToFrameStateDescriptor(FrameStateDescriptor * descriptor,Node * state,OperandGenerator * g,StateObjectDeduplicator * deduplicator,InstructionOperandVector * inputs,FrameStateInputKind kind,Zone * zone)615 size_t InstructionSelector::AddInputsToFrameStateDescriptor(
616     FrameStateDescriptor* descriptor, Node* state, OperandGenerator* g,
617     StateObjectDeduplicator* deduplicator, InstructionOperandVector* inputs,
618     FrameStateInputKind kind, Zone* zone) {
619   DCHECK_EQ(IrOpcode::kFrameState, state->op()->opcode());
620 
621   size_t entries = 0;
622   size_t initial_size = inputs->size();
623   USE(initial_size);  // initial_size is only used for debug.
624 
625   if (descriptor->outer_state()) {
626     entries += AddInputsToFrameStateDescriptor(
627         descriptor->outer_state(), state->InputAt(kFrameStateOuterStateInput),
628         g, deduplicator, inputs, kind, zone);
629   }
630 
631   Node* parameters = state->InputAt(kFrameStateParametersInput);
632   Node* locals = state->InputAt(kFrameStateLocalsInput);
633   Node* stack = state->InputAt(kFrameStateStackInput);
634   Node* context = state->InputAt(kFrameStateContextInput);
635   Node* function = state->InputAt(kFrameStateFunctionInput);
636 
637   DCHECK_EQ(descriptor->parameters_count(),
638             StateValuesAccess(parameters).size());
639   DCHECK_EQ(descriptor->locals_count(), StateValuesAccess(locals).size());
640   DCHECK_EQ(descriptor->stack_count(), StateValuesAccess(stack).size());
641 
642   StateValueList* values_descriptor = descriptor->GetStateValueDescriptors();
643 
644   DCHECK_EQ(values_descriptor->size(), 0u);
645   values_descriptor->ReserveSize(descriptor->GetSize());
646 
647   entries += AddOperandToStateValueDescriptor(
648       values_descriptor, inputs, g, deduplicator, function,
649       MachineType::AnyTagged(), FrameStateInputKind::kStackSlot, zone);
650   for (StateValuesAccess::TypedNode input_node :
651        StateValuesAccess(parameters)) {
652     entries += AddOperandToStateValueDescriptor(values_descriptor, inputs, g,
653                                                 deduplicator, input_node.node,
654                                                 input_node.type, kind, zone);
655   }
656   if (descriptor->HasContext()) {
657     entries += AddOperandToStateValueDescriptor(
658         values_descriptor, inputs, g, deduplicator, context,
659         MachineType::AnyTagged(), FrameStateInputKind::kStackSlot, zone);
660   }
661   for (StateValuesAccess::TypedNode input_node : StateValuesAccess(locals)) {
662     entries += AddOperandToStateValueDescriptor(values_descriptor, inputs, g,
663                                                 deduplicator, input_node.node,
664                                                 input_node.type, kind, zone);
665   }
666   for (StateValuesAccess::TypedNode input_node : StateValuesAccess(stack)) {
667     entries += AddOperandToStateValueDescriptor(values_descriptor, inputs, g,
668                                                 deduplicator, input_node.node,
669                                                 input_node.type, kind, zone);
670   }
671   DCHECK_EQ(initial_size + entries, inputs->size());
672   return entries;
673 }
674 
EmitWithContinuation(InstructionCode opcode,FlagsContinuation * cont)675 Instruction* InstructionSelector::EmitWithContinuation(
676     InstructionCode opcode, FlagsContinuation* cont) {
677   return EmitWithContinuation(opcode, 0, nullptr, 0, nullptr, cont);
678 }
679 
EmitWithContinuation(InstructionCode opcode,InstructionOperand a,FlagsContinuation * cont)680 Instruction* InstructionSelector::EmitWithContinuation(
681     InstructionCode opcode, InstructionOperand a, FlagsContinuation* cont) {
682   return EmitWithContinuation(opcode, 0, nullptr, 1, &a, cont);
683 }
684 
EmitWithContinuation(InstructionCode opcode,InstructionOperand a,InstructionOperand b,FlagsContinuation * cont)685 Instruction* InstructionSelector::EmitWithContinuation(
686     InstructionCode opcode, InstructionOperand a, InstructionOperand b,
687     FlagsContinuation* cont) {
688   InstructionOperand inputs[] = {a, b};
689   return EmitWithContinuation(opcode, 0, nullptr, arraysize(inputs), inputs,
690                               cont);
691 }
692 
EmitWithContinuation(InstructionCode opcode,InstructionOperand a,InstructionOperand b,InstructionOperand c,FlagsContinuation * cont)693 Instruction* InstructionSelector::EmitWithContinuation(
694     InstructionCode opcode, InstructionOperand a, InstructionOperand b,
695     InstructionOperand c, FlagsContinuation* cont) {
696   InstructionOperand inputs[] = {a, b, c};
697   return EmitWithContinuation(opcode, 0, nullptr, arraysize(inputs), inputs,
698                               cont);
699 }
700 
EmitWithContinuation(InstructionCode opcode,size_t output_count,InstructionOperand * outputs,size_t input_count,InstructionOperand * inputs,FlagsContinuation * cont)701 Instruction* InstructionSelector::EmitWithContinuation(
702     InstructionCode opcode, size_t output_count, InstructionOperand* outputs,
703     size_t input_count, InstructionOperand* inputs, FlagsContinuation* cont) {
704   OperandGenerator g(this);
705 
706   opcode = cont->Encode(opcode);
707 
708   continuation_inputs_.resize(0);
709   for (size_t i = 0; i < input_count; i++) {
710     continuation_inputs_.push_back(inputs[i]);
711   }
712 
713   continuation_outputs_.resize(0);
714   for (size_t i = 0; i < output_count; i++) {
715     continuation_outputs_.push_back(outputs[i]);
716   }
717 
718   if (cont->IsBranch()) {
719     continuation_inputs_.push_back(g.Label(cont->true_block()));
720     continuation_inputs_.push_back(g.Label(cont->false_block()));
721   } else if (cont->IsDeoptimize()) {
722     opcode |= MiscField::encode(static_cast<int>(input_count));
723     AppendDeoptimizeArguments(&continuation_inputs_, cont->kind(),
724                               cont->reason(), cont->feedback(),
725                               cont->frame_state());
726   } else if (cont->IsSet()) {
727     continuation_outputs_.push_back(g.DefineAsRegister(cont->result()));
728   } else if (cont->IsTrap()) {
729     int trap_id = static_cast<int>(cont->trap_id());
730     continuation_inputs_.push_back(g.UseImmediate(trap_id));
731   } else {
732     DCHECK(cont->IsNone());
733   }
734 
735   size_t const emit_inputs_size = continuation_inputs_.size();
736   auto* emit_inputs =
737       emit_inputs_size ? &continuation_inputs_.front() : nullptr;
738   size_t const emit_outputs_size = continuation_outputs_.size();
739   auto* emit_outputs =
740       emit_outputs_size ? &continuation_outputs_.front() : nullptr;
741   return Emit(opcode, emit_outputs_size, emit_outputs, emit_inputs_size,
742               emit_inputs, 0, nullptr);
743 }
744 
AppendDeoptimizeArguments(InstructionOperandVector * args,DeoptimizeKind kind,DeoptimizeReason reason,VectorSlotPair const & feedback,Node * frame_state)745 void InstructionSelector::AppendDeoptimizeArguments(
746     InstructionOperandVector* args, DeoptimizeKind kind,
747     DeoptimizeReason reason, VectorSlotPair const& feedback,
748     Node* frame_state) {
749   OperandGenerator g(this);
750   FrameStateDescriptor* const descriptor = GetFrameStateDescriptor(frame_state);
751   DCHECK_NE(DeoptimizeKind::kLazy, kind);
752   int const state_id =
753       sequence()->AddDeoptimizationEntry(descriptor, kind, reason, feedback);
754   args->push_back(g.TempImmediate(state_id));
755   StateObjectDeduplicator deduplicator(instruction_zone());
756   AddInputsToFrameStateDescriptor(descriptor, frame_state, &g, &deduplicator,
757                                   args, FrameStateInputKind::kAny,
758                                   instruction_zone());
759 }
760 
EmitDeoptimize(InstructionCode opcode,size_t output_count,InstructionOperand * outputs,size_t input_count,InstructionOperand * inputs,DeoptimizeKind kind,DeoptimizeReason reason,VectorSlotPair const & feedback,Node * frame_state)761 Instruction* InstructionSelector::EmitDeoptimize(
762     InstructionCode opcode, size_t output_count, InstructionOperand* outputs,
763     size_t input_count, InstructionOperand* inputs, DeoptimizeKind kind,
764     DeoptimizeReason reason, VectorSlotPair const& feedback,
765     Node* frame_state) {
766   InstructionOperandVector args(instruction_zone());
767   for (size_t i = 0; i < input_count; ++i) {
768     args.push_back(inputs[i]);
769   }
770   opcode |= MiscField::encode(static_cast<int>(input_count));
771   AppendDeoptimizeArguments(&args, kind, reason, feedback, frame_state);
772   return Emit(opcode, output_count, outputs, args.size(), &args.front(), 0,
773               nullptr);
774 }
775 
776 // An internal helper class for generating the operands to calls.
777 // TODO(bmeurer): Get rid of the CallBuffer business and make
778 // InstructionSelector::VisitCall platform independent instead.
779 struct CallBuffer {
CallBufferv8::internal::compiler::CallBuffer780   CallBuffer(Zone* zone, const CallDescriptor* call_descriptor,
781              FrameStateDescriptor* frame_state)
782       : descriptor(call_descriptor),
783         frame_state_descriptor(frame_state),
784         output_nodes(zone),
785         outputs(zone),
786         instruction_args(zone),
787         pushed_nodes(zone) {
788     output_nodes.reserve(call_descriptor->ReturnCount());
789     outputs.reserve(call_descriptor->ReturnCount());
790     pushed_nodes.reserve(input_count());
791     instruction_args.reserve(input_count() + frame_state_value_count());
792   }
793 
794 
795   const CallDescriptor* descriptor;
796   FrameStateDescriptor* frame_state_descriptor;
797   ZoneVector<PushParameter> output_nodes;
798   InstructionOperandVector outputs;
799   InstructionOperandVector instruction_args;
800   ZoneVector<PushParameter> pushed_nodes;
801 
input_countv8::internal::compiler::CallBuffer802   size_t input_count() const { return descriptor->InputCount(); }
803 
frame_state_countv8::internal::compiler::CallBuffer804   size_t frame_state_count() const { return descriptor->FrameStateCount(); }
805 
frame_state_value_countv8::internal::compiler::CallBuffer806   size_t frame_state_value_count() const {
807     return (frame_state_descriptor == nullptr)
808                ? 0
809                : (frame_state_descriptor->GetTotalSize() +
810                   1);  // Include deopt id.
811   }
812 };
813 
814 
815 // TODO(bmeurer): Get rid of the CallBuffer business and make
816 // InstructionSelector::VisitCall platform independent instead.
InitializeCallBuffer(Node * call,CallBuffer * buffer,CallBufferFlags flags,bool is_tail_call,int stack_param_delta)817 void InstructionSelector::InitializeCallBuffer(Node* call, CallBuffer* buffer,
818                                                CallBufferFlags flags,
819                                                bool is_tail_call,
820                                                int stack_param_delta) {
821   OperandGenerator g(this);
822   size_t ret_count = buffer->descriptor->ReturnCount();
823   DCHECK_LE(call->op()->ValueOutputCount(), ret_count);
824   DCHECK_EQ(
825       call->op()->ValueInputCount(),
826       static_cast<int>(buffer->input_count() + buffer->frame_state_count()));
827 
828   if (ret_count > 0) {
829     // Collect the projections that represent multiple outputs from this call.
830     if (ret_count == 1) {
831       PushParameter result = {call, buffer->descriptor->GetReturnLocation(0)};
832       buffer->output_nodes.push_back(result);
833     } else {
834       buffer->output_nodes.resize(ret_count);
835       int stack_count = 0;
836       for (size_t i = 0; i < ret_count; ++i) {
837         LinkageLocation location = buffer->descriptor->GetReturnLocation(i);
838         buffer->output_nodes[i] = PushParameter(nullptr, location);
839         if (location.IsCallerFrameSlot()) {
840           stack_count += location.GetSizeInPointers();
841         }
842       }
843       for (Edge const edge : call->use_edges()) {
844         if (!NodeProperties::IsValueEdge(edge)) continue;
845         Node* node = edge.from();
846         DCHECK_EQ(IrOpcode::kProjection, node->opcode());
847         size_t const index = ProjectionIndexOf(node->op());
848 
849         DCHECK_LT(index, buffer->output_nodes.size());
850         DCHECK(!buffer->output_nodes[index].node);
851         buffer->output_nodes[index].node = node;
852       }
853       frame_->EnsureReturnSlots(stack_count);
854     }
855 
856     // Filter out the outputs that aren't live because no projection uses them.
857     size_t outputs_needed_by_framestate =
858         buffer->frame_state_descriptor == nullptr
859             ? 0
860             : buffer->frame_state_descriptor->state_combine()
861                   .ConsumedOutputCount();
862     for (size_t i = 0; i < buffer->output_nodes.size(); i++) {
863       bool output_is_live = buffer->output_nodes[i].node != nullptr ||
864                             i < outputs_needed_by_framestate;
865       if (output_is_live) {
866         LinkageLocation location = buffer->output_nodes[i].location;
867         MachineRepresentation rep = location.GetType().representation();
868 
869         Node* output = buffer->output_nodes[i].node;
870         InstructionOperand op = output == nullptr
871                                     ? g.TempLocation(location)
872                                     : g.DefineAsLocation(output, location);
873         MarkAsRepresentation(rep, op);
874 
875         if (!UnallocatedOperand::cast(op).HasFixedSlotPolicy()) {
876           buffer->outputs.push_back(op);
877           buffer->output_nodes[i].node = nullptr;
878         }
879       }
880     }
881   }
882 
883   // The first argument is always the callee code.
884   Node* callee = call->InputAt(0);
885   bool call_code_immediate = (flags & kCallCodeImmediate) != 0;
886   bool call_address_immediate = (flags & kCallAddressImmediate) != 0;
887   bool call_use_fixed_target_reg = (flags & kCallFixedTargetRegister) != 0;
888   switch (buffer->descriptor->kind()) {
889     case CallDescriptor::kCallCodeObject:
890       // TODO(jgruber, v8:7449): The below is a hack to support tail-calls from
891       // JS-linkage callers with a register code target. The problem is that the
892       // code target register may be clobbered before the final jmp by
893       // AssemblePopArgumentsAdaptorFrame. As a more permanent fix we could
894       // entirely remove support for tail-calls from JS-linkage callers.
895       buffer->instruction_args.push_back(
896           (call_code_immediate && callee->opcode() == IrOpcode::kHeapConstant)
897               ? g.UseImmediate(callee)
898               : call_use_fixed_target_reg
899                     ? g.UseFixed(callee, kJavaScriptCallCodeStartRegister)
900                     : is_tail_call ? g.UseUniqueRegister(callee)
901                                    : g.UseRegister(callee));
902       break;
903     case CallDescriptor::kCallAddress:
904       buffer->instruction_args.push_back(
905           (call_address_immediate &&
906            callee->opcode() == IrOpcode::kExternalConstant)
907               ? g.UseImmediate(callee)
908               : call_use_fixed_target_reg
909                     ? g.UseFixed(callee, kJavaScriptCallCodeStartRegister)
910                     : g.UseRegister(callee));
911       break;
912     case CallDescriptor::kCallWasmFunction:
913       buffer->instruction_args.push_back(
914           (call_address_immediate &&
915            (callee->opcode() == IrOpcode::kRelocatableInt64Constant ||
916             callee->opcode() == IrOpcode::kRelocatableInt32Constant))
917               ? g.UseImmediate(callee)
918               : call_use_fixed_target_reg
919                     ? g.UseFixed(callee, kJavaScriptCallCodeStartRegister)
920                     : g.UseRegister(callee));
921       break;
922     case CallDescriptor::kCallJSFunction:
923       buffer->instruction_args.push_back(
924           g.UseLocation(callee, buffer->descriptor->GetInputLocation(0)));
925       break;
926   }
927   DCHECK_EQ(1u, buffer->instruction_args.size());
928 
929   // Argument 1 is used for poison-alias index (encoded in a word-sized
930   // immediate. This an index of the operand that aliases with poison register
931   // or -1 if there is no aliasing.
932   buffer->instruction_args.push_back(g.TempImmediate(-1));
933   const size_t poison_alias_index = 1;
934   DCHECK_EQ(buffer->instruction_args.size() - 1, poison_alias_index);
935 
936   // If the call needs a frame state, we insert the state information as
937   // follows (n is the number of value inputs to the frame state):
938   // arg 2               : deoptimization id.
939   // arg 3 - arg (n + 2) : value inputs to the frame state.
940   size_t frame_state_entries = 0;
941   USE(frame_state_entries);  // frame_state_entries is only used for debug.
942   if (buffer->frame_state_descriptor != nullptr) {
943     Node* frame_state =
944         call->InputAt(static_cast<int>(buffer->descriptor->InputCount()));
945 
946     // If it was a syntactic tail call we need to drop the current frame and
947     // all the frames on top of it that are either an arguments adaptor frame
948     // or a tail caller frame.
949     if (is_tail_call) {
950       frame_state = NodeProperties::GetFrameStateInput(frame_state);
951       buffer->frame_state_descriptor =
952           buffer->frame_state_descriptor->outer_state();
953       while (buffer->frame_state_descriptor != nullptr &&
954              buffer->frame_state_descriptor->type() ==
955                  FrameStateType::kArgumentsAdaptor) {
956         frame_state = NodeProperties::GetFrameStateInput(frame_state);
957         buffer->frame_state_descriptor =
958             buffer->frame_state_descriptor->outer_state();
959       }
960     }
961 
962     int const state_id = sequence()->AddDeoptimizationEntry(
963         buffer->frame_state_descriptor, DeoptimizeKind::kLazy,
964         DeoptimizeReason::kUnknown, VectorSlotPair());
965     buffer->instruction_args.push_back(g.TempImmediate(state_id));
966 
967     StateObjectDeduplicator deduplicator(instruction_zone());
968 
969     frame_state_entries =
970         1 + AddInputsToFrameStateDescriptor(
971                 buffer->frame_state_descriptor, frame_state, &g, &deduplicator,
972                 &buffer->instruction_args, FrameStateInputKind::kStackSlot,
973                 instruction_zone());
974 
975     DCHECK_EQ(2 + frame_state_entries, buffer->instruction_args.size());
976   }
977 
978   size_t input_count = static_cast<size_t>(buffer->input_count());
979 
980   // Split the arguments into pushed_nodes and instruction_args. Pushed
981   // arguments require an explicit push instruction before the call and do
982   // not appear as arguments to the call. Everything else ends up
983   // as an InstructionOperand argument to the call.
984   auto iter(call->inputs().begin());
985   size_t pushed_count = 0;
986   bool call_tail = (flags & kCallTail) != 0;
987   for (size_t index = 0; index < input_count; ++iter, ++index) {
988     DCHECK(iter != call->inputs().end());
989     DCHECK_NE(IrOpcode::kFrameState, (*iter)->op()->opcode());
990     if (index == 0) continue;  // The first argument (callee) is already done.
991 
992     LinkageLocation location = buffer->descriptor->GetInputLocation(index);
993     if (call_tail) {
994       location = LinkageLocation::ConvertToTailCallerLocation(
995           location, stack_param_delta);
996     }
997     InstructionOperand op = g.UseLocation(*iter, location);
998     UnallocatedOperand unallocated = UnallocatedOperand::cast(op);
999     if (unallocated.HasFixedSlotPolicy() && !call_tail) {
1000       int stack_index = -unallocated.fixed_slot_index() - 1;
1001       if (static_cast<size_t>(stack_index) >= buffer->pushed_nodes.size()) {
1002         buffer->pushed_nodes.resize(stack_index + 1);
1003       }
1004       PushParameter param = {*iter, location};
1005       buffer->pushed_nodes[stack_index] = param;
1006       pushed_count++;
1007     } else {
1008       // If we do load poisoning and the linkage uses the poisoning register,
1009       // then we request the input in memory location, and during code
1010       // generation, we move the input to the register.
1011       if (poisoning_level_ != PoisoningMitigationLevel::kDontPoison &&
1012           unallocated.HasFixedRegisterPolicy()) {
1013         int reg = unallocated.fixed_register_index();
1014         if (reg == kSpeculationPoisonRegister.code()) {
1015           buffer->instruction_args[poison_alias_index] = g.TempImmediate(
1016               static_cast<int32_t>(buffer->instruction_args.size()));
1017           op = g.UseRegisterOrSlotOrConstant(*iter);
1018         }
1019       }
1020       buffer->instruction_args.push_back(op);
1021     }
1022   }
1023   DCHECK_EQ(input_count, buffer->instruction_args.size() + pushed_count -
1024                              frame_state_entries - 1);
1025   if (V8_TARGET_ARCH_STORES_RETURN_ADDRESS_ON_STACK && call_tail &&
1026       stack_param_delta != 0) {
1027     // For tail calls that change the size of their parameter list and keep
1028     // their return address on the stack, move the return address to just above
1029     // the parameters.
1030     LinkageLocation saved_return_location =
1031         LinkageLocation::ForSavedCallerReturnAddress();
1032     InstructionOperand return_address =
1033         g.UsePointerLocation(LinkageLocation::ConvertToTailCallerLocation(
1034                                  saved_return_location, stack_param_delta),
1035                              saved_return_location);
1036     buffer->instruction_args.push_back(return_address);
1037   }
1038 }
1039 
IsSourcePositionUsed(Node * node)1040 bool InstructionSelector::IsSourcePositionUsed(Node* node) {
1041   return (source_position_mode_ == kAllSourcePositions ||
1042           node->opcode() == IrOpcode::kCall ||
1043           node->opcode() == IrOpcode::kCallWithCallerSavedRegisters ||
1044           node->opcode() == IrOpcode::kTrapIf ||
1045           node->opcode() == IrOpcode::kTrapUnless ||
1046           node->opcode() == IrOpcode::kProtectedLoad ||
1047           node->opcode() == IrOpcode::kProtectedStore);
1048 }
1049 
VisitBlock(BasicBlock * block)1050 void InstructionSelector::VisitBlock(BasicBlock* block) {
1051   DCHECK(!current_block_);
1052   current_block_ = block;
1053   auto current_num_instructions = [&] {
1054     DCHECK_GE(kMaxInt, instructions_.size());
1055     return static_cast<int>(instructions_.size());
1056   };
1057   int current_block_end = current_num_instructions();
1058 
1059   int effect_level = 0;
1060   for (Node* const node : *block) {
1061     SetEffectLevel(node, effect_level);
1062     if (node->opcode() == IrOpcode::kStore ||
1063         node->opcode() == IrOpcode::kUnalignedStore ||
1064         node->opcode() == IrOpcode::kCall ||
1065         node->opcode() == IrOpcode::kCallWithCallerSavedRegisters ||
1066         node->opcode() == IrOpcode::kProtectedLoad ||
1067         node->opcode() == IrOpcode::kProtectedStore) {
1068       ++effect_level;
1069     }
1070   }
1071 
1072   // We visit the control first, then the nodes in the block, so the block's
1073   // control input should be on the same effect level as the last node.
1074   if (block->control_input() != nullptr) {
1075     SetEffectLevel(block->control_input(), effect_level);
1076   }
1077 
1078   auto FinishEmittedInstructions = [&](Node* node, int instruction_start) {
1079     if (instruction_selection_failed()) return false;
1080     if (current_num_instructions() == instruction_start) return true;
1081     std::reverse(instructions_.begin() + instruction_start,
1082                  instructions_.end());
1083     if (!node) return true;
1084     SourcePosition source_position = source_positions_->GetSourcePosition(node);
1085     if (source_position.IsKnown() && IsSourcePositionUsed(node)) {
1086       sequence()->SetSourcePosition(instructions_[instruction_start],
1087                                     source_position);
1088     }
1089     return true;
1090   };
1091 
1092   // Generate code for the block control "top down", but schedule the code
1093   // "bottom up".
1094   VisitControl(block);
1095   if (!FinishEmittedInstructions(block->control_input(), current_block_end))
1096     return;
1097 
1098   // Visit code in reverse control flow order, because architecture-specific
1099   // matching may cover more than one node at a time.
1100   for (auto node : base::Reversed(*block)) {
1101     int current_node_end = current_num_instructions();
1102     // Skip nodes that are unused or already defined.
1103     if (IsUsed(node) && !IsDefined(node)) {
1104       // Generate code for this node "top down", but schedule the code "bottom
1105       // up".
1106       VisitNode(node);
1107       if (!FinishEmittedInstructions(node, current_node_end)) return;
1108     }
1109     if (trace_turbo_ == kEnableTraceTurboJson) {
1110       instr_origins_[node->id()] = {current_num_instructions(),
1111                                     current_node_end};
1112     }
1113   }
1114 
1115   // We're done with the block.
1116   InstructionBlock* instruction_block =
1117       sequence()->InstructionBlockAt(RpoNumber::FromInt(block->rpo_number()));
1118   if (current_num_instructions() == current_block_end) {
1119     // Avoid empty block: insert a {kArchNop} instruction.
1120     Emit(Instruction::New(sequence()->zone(), kArchNop));
1121   }
1122   instruction_block->set_code_start(current_num_instructions());
1123   instruction_block->set_code_end(current_block_end);
1124   current_block_ = nullptr;
1125 }
1126 
1127 
VisitControl(BasicBlock * block)1128 void InstructionSelector::VisitControl(BasicBlock* block) {
1129 #ifdef DEBUG
1130   // SSA deconstruction requires targets of branches not to have phis.
1131   // Edge split form guarantees this property, but is more strict.
1132   if (block->SuccessorCount() > 1) {
1133     for (BasicBlock* const successor : block->successors()) {
1134       for (Node* const node : *successor) {
1135         if (IrOpcode::IsPhiOpcode(node->opcode())) {
1136           std::ostringstream str;
1137           str << "You might have specified merged variables for a label with "
1138               << "only one predecessor." << std::endl
1139               << "# Current Block: " << *successor << std::endl
1140               << "#          Node: " << *node;
1141           FATAL("%s", str.str().c_str());
1142         }
1143       }
1144     }
1145   }
1146 #endif
1147 
1148   Node* input = block->control_input();
1149   int instruction_end = static_cast<int>(instructions_.size());
1150   switch (block->control()) {
1151     case BasicBlock::kGoto:
1152       VisitGoto(block->SuccessorAt(0));
1153       break;
1154     case BasicBlock::kCall: {
1155       DCHECK_EQ(IrOpcode::kCall, input->opcode());
1156       BasicBlock* success = block->SuccessorAt(0);
1157       BasicBlock* exception = block->SuccessorAt(1);
1158       VisitCall(input, exception);
1159       VisitGoto(success);
1160       break;
1161     }
1162     case BasicBlock::kTailCall: {
1163       DCHECK_EQ(IrOpcode::kTailCall, input->opcode());
1164       VisitTailCall(input);
1165       break;
1166     }
1167     case BasicBlock::kBranch: {
1168       DCHECK_EQ(IrOpcode::kBranch, input->opcode());
1169       BasicBlock* tbranch = block->SuccessorAt(0);
1170       BasicBlock* fbranch = block->SuccessorAt(1);
1171       if (tbranch == fbranch) {
1172         VisitGoto(tbranch);
1173       } else {
1174         VisitBranch(input, tbranch, fbranch);
1175       }
1176       break;
1177     }
1178     case BasicBlock::kSwitch: {
1179       DCHECK_EQ(IrOpcode::kSwitch, input->opcode());
1180       // Last successor must be {IfDefault}.
1181       BasicBlock* default_branch = block->successors().back();
1182       DCHECK_EQ(IrOpcode::kIfDefault, default_branch->front()->opcode());
1183       // All other successors must be {IfValue}s.
1184       int32_t min_value = std::numeric_limits<int32_t>::max();
1185       int32_t max_value = std::numeric_limits<int32_t>::min();
1186       size_t case_count = block->SuccessorCount() - 1;
1187       ZoneVector<CaseInfo> cases(case_count, zone());
1188       for (size_t i = 0; i < case_count; ++i) {
1189         BasicBlock* branch = block->SuccessorAt(i);
1190         const IfValueParameters& p = IfValueParametersOf(branch->front()->op());
1191         cases[i] = CaseInfo{p.value(), p.comparison_order(), branch};
1192         if (min_value > p.value()) min_value = p.value();
1193         if (max_value < p.value()) max_value = p.value();
1194       }
1195       SwitchInfo sw(cases, min_value, max_value, default_branch);
1196       VisitSwitch(input, sw);
1197       break;
1198     }
1199     case BasicBlock::kReturn: {
1200       DCHECK_EQ(IrOpcode::kReturn, input->opcode());
1201       VisitReturn(input);
1202       break;
1203     }
1204     case BasicBlock::kDeoptimize: {
1205       DeoptimizeParameters p = DeoptimizeParametersOf(input->op());
1206       Node* value = input->InputAt(0);
1207       VisitDeoptimize(p.kind(), p.reason(), p.feedback(), value);
1208       break;
1209     }
1210     case BasicBlock::kThrow:
1211       DCHECK_EQ(IrOpcode::kThrow, input->opcode());
1212       VisitThrow(input);
1213       break;
1214     case BasicBlock::kNone: {
1215       // Exit block doesn't have control.
1216       DCHECK_NULL(input);
1217       break;
1218     }
1219     default:
1220       UNREACHABLE();
1221       break;
1222   }
1223   if (trace_turbo_ == kEnableTraceTurboJson && input) {
1224     int instruction_start = static_cast<int>(instructions_.size());
1225     instr_origins_[input->id()] = {instruction_start, instruction_end};
1226   }
1227 }
1228 
MarkPairProjectionsAsWord32(Node * node)1229 void InstructionSelector::MarkPairProjectionsAsWord32(Node* node) {
1230   Node* projection0 = NodeProperties::FindProjection(node, 0);
1231   if (projection0) {
1232     MarkAsWord32(projection0);
1233   }
1234   Node* projection1 = NodeProperties::FindProjection(node, 1);
1235   if (projection1) {
1236     MarkAsWord32(projection1);
1237   }
1238 }
1239 
VisitNode(Node * node)1240 void InstructionSelector::VisitNode(Node* node) {
1241   DCHECK_NOT_NULL(schedule()->block(node));  // should only use scheduled nodes.
1242   switch (node->opcode()) {
1243     case IrOpcode::kStart:
1244     case IrOpcode::kLoop:
1245     case IrOpcode::kEnd:
1246     case IrOpcode::kBranch:
1247     case IrOpcode::kIfTrue:
1248     case IrOpcode::kIfFalse:
1249     case IrOpcode::kIfSuccess:
1250     case IrOpcode::kSwitch:
1251     case IrOpcode::kIfValue:
1252     case IrOpcode::kIfDefault:
1253     case IrOpcode::kEffectPhi:
1254     case IrOpcode::kMerge:
1255     case IrOpcode::kTerminate:
1256     case IrOpcode::kBeginRegion:
1257       // No code needed for these graph artifacts.
1258       return;
1259     case IrOpcode::kIfException:
1260       return MarkAsReference(node), VisitIfException(node);
1261     case IrOpcode::kFinishRegion:
1262       return MarkAsReference(node), VisitFinishRegion(node);
1263     case IrOpcode::kParameter: {
1264       MachineType type =
1265           linkage()->GetParameterType(ParameterIndexOf(node->op()));
1266       MarkAsRepresentation(type.representation(), node);
1267       return VisitParameter(node);
1268     }
1269     case IrOpcode::kOsrValue:
1270       return MarkAsReference(node), VisitOsrValue(node);
1271     case IrOpcode::kPhi: {
1272       MachineRepresentation rep = PhiRepresentationOf(node->op());
1273       if (rep == MachineRepresentation::kNone) return;
1274       MarkAsRepresentation(rep, node);
1275       return VisitPhi(node);
1276     }
1277     case IrOpcode::kProjection:
1278       return VisitProjection(node);
1279     case IrOpcode::kInt32Constant:
1280     case IrOpcode::kInt64Constant:
1281     case IrOpcode::kExternalConstant:
1282     case IrOpcode::kRelocatableInt32Constant:
1283     case IrOpcode::kRelocatableInt64Constant:
1284       return VisitConstant(node);
1285     case IrOpcode::kFloat32Constant:
1286       return MarkAsFloat32(node), VisitConstant(node);
1287     case IrOpcode::kFloat64Constant:
1288       return MarkAsFloat64(node), VisitConstant(node);
1289     case IrOpcode::kHeapConstant:
1290       return MarkAsReference(node), VisitConstant(node);
1291     case IrOpcode::kNumberConstant: {
1292       double value = OpParameter<double>(node->op());
1293       if (!IsSmiDouble(value)) MarkAsReference(node);
1294       return VisitConstant(node);
1295     }
1296     case IrOpcode::kCall:
1297       return VisitCall(node);
1298     case IrOpcode::kCallWithCallerSavedRegisters:
1299       return VisitCallWithCallerSavedRegisters(node);
1300     case IrOpcode::kDeoptimizeIf:
1301       return VisitDeoptimizeIf(node);
1302     case IrOpcode::kDeoptimizeUnless:
1303       return VisitDeoptimizeUnless(node);
1304     case IrOpcode::kTrapIf:
1305       return VisitTrapIf(node, TrapIdOf(node->op()));
1306     case IrOpcode::kTrapUnless:
1307       return VisitTrapUnless(node, TrapIdOf(node->op()));
1308     case IrOpcode::kFrameState:
1309     case IrOpcode::kStateValues:
1310     case IrOpcode::kObjectState:
1311       return;
1312     case IrOpcode::kDebugAbort:
1313       VisitDebugAbort(node);
1314       return;
1315     case IrOpcode::kDebugBreak:
1316       VisitDebugBreak(node);
1317       return;
1318     case IrOpcode::kUnreachable:
1319       VisitUnreachable(node);
1320       return;
1321     case IrOpcode::kDeadValue:
1322       VisitDeadValue(node);
1323       return;
1324     case IrOpcode::kComment:
1325       VisitComment(node);
1326       return;
1327     case IrOpcode::kRetain:
1328       VisitRetain(node);
1329       return;
1330     case IrOpcode::kLoad: {
1331       LoadRepresentation type = LoadRepresentationOf(node->op());
1332       MarkAsRepresentation(type.representation(), node);
1333       return VisitLoad(node);
1334     }
1335     case IrOpcode::kPoisonedLoad: {
1336       LoadRepresentation type = LoadRepresentationOf(node->op());
1337       MarkAsRepresentation(type.representation(), node);
1338       return VisitPoisonedLoad(node);
1339     }
1340     case IrOpcode::kStore:
1341       return VisitStore(node);
1342     case IrOpcode::kProtectedStore:
1343       return VisitProtectedStore(node);
1344     case IrOpcode::kWord32And:
1345       return MarkAsWord32(node), VisitWord32And(node);
1346     case IrOpcode::kWord32Or:
1347       return MarkAsWord32(node), VisitWord32Or(node);
1348     case IrOpcode::kWord32Xor:
1349       return MarkAsWord32(node), VisitWord32Xor(node);
1350     case IrOpcode::kWord32Shl:
1351       return MarkAsWord32(node), VisitWord32Shl(node);
1352     case IrOpcode::kWord32Shr:
1353       return MarkAsWord32(node), VisitWord32Shr(node);
1354     case IrOpcode::kWord32Sar:
1355       return MarkAsWord32(node), VisitWord32Sar(node);
1356     case IrOpcode::kWord32Ror:
1357       return MarkAsWord32(node), VisitWord32Ror(node);
1358     case IrOpcode::kWord32Equal:
1359       return VisitWord32Equal(node);
1360     case IrOpcode::kWord32Clz:
1361       return MarkAsWord32(node), VisitWord32Clz(node);
1362     case IrOpcode::kWord32Ctz:
1363       return MarkAsWord32(node), VisitWord32Ctz(node);
1364     case IrOpcode::kWord32ReverseBits:
1365       return MarkAsWord32(node), VisitWord32ReverseBits(node);
1366     case IrOpcode::kWord32ReverseBytes:
1367       return MarkAsWord32(node), VisitWord32ReverseBytes(node);
1368     case IrOpcode::kInt32AbsWithOverflow:
1369       return MarkAsWord32(node), VisitInt32AbsWithOverflow(node);
1370     case IrOpcode::kWord32Popcnt:
1371       return MarkAsWord32(node), VisitWord32Popcnt(node);
1372     case IrOpcode::kWord64Popcnt:
1373       return MarkAsWord32(node), VisitWord64Popcnt(node);
1374     case IrOpcode::kWord64And:
1375       return MarkAsWord64(node), VisitWord64And(node);
1376     case IrOpcode::kWord64Or:
1377       return MarkAsWord64(node), VisitWord64Or(node);
1378     case IrOpcode::kWord64Xor:
1379       return MarkAsWord64(node), VisitWord64Xor(node);
1380     case IrOpcode::kWord64Shl:
1381       return MarkAsWord64(node), VisitWord64Shl(node);
1382     case IrOpcode::kWord64Shr:
1383       return MarkAsWord64(node), VisitWord64Shr(node);
1384     case IrOpcode::kWord64Sar:
1385       return MarkAsWord64(node), VisitWord64Sar(node);
1386     case IrOpcode::kWord64Ror:
1387       return MarkAsWord64(node), VisitWord64Ror(node);
1388     case IrOpcode::kWord64Clz:
1389       return MarkAsWord64(node), VisitWord64Clz(node);
1390     case IrOpcode::kWord64Ctz:
1391       return MarkAsWord64(node), VisitWord64Ctz(node);
1392     case IrOpcode::kWord64ReverseBits:
1393       return MarkAsWord64(node), VisitWord64ReverseBits(node);
1394     case IrOpcode::kWord64ReverseBytes:
1395       return MarkAsWord64(node), VisitWord64ReverseBytes(node);
1396     case IrOpcode::kInt64AbsWithOverflow:
1397       return MarkAsWord64(node), VisitInt64AbsWithOverflow(node);
1398     case IrOpcode::kWord64Equal:
1399       return VisitWord64Equal(node);
1400     case IrOpcode::kInt32Add:
1401       return MarkAsWord32(node), VisitInt32Add(node);
1402     case IrOpcode::kInt32AddWithOverflow:
1403       return MarkAsWord32(node), VisitInt32AddWithOverflow(node);
1404     case IrOpcode::kInt32Sub:
1405       return MarkAsWord32(node), VisitInt32Sub(node);
1406     case IrOpcode::kInt32SubWithOverflow:
1407       return VisitInt32SubWithOverflow(node);
1408     case IrOpcode::kInt32Mul:
1409       return MarkAsWord32(node), VisitInt32Mul(node);
1410     case IrOpcode::kInt32MulWithOverflow:
1411       return MarkAsWord32(node), VisitInt32MulWithOverflow(node);
1412     case IrOpcode::kInt32MulHigh:
1413       return VisitInt32MulHigh(node);
1414     case IrOpcode::kInt32Div:
1415       return MarkAsWord32(node), VisitInt32Div(node);
1416     case IrOpcode::kInt32Mod:
1417       return MarkAsWord32(node), VisitInt32Mod(node);
1418     case IrOpcode::kInt32LessThan:
1419       return VisitInt32LessThan(node);
1420     case IrOpcode::kInt32LessThanOrEqual:
1421       return VisitInt32LessThanOrEqual(node);
1422     case IrOpcode::kUint32Div:
1423       return MarkAsWord32(node), VisitUint32Div(node);
1424     case IrOpcode::kUint32LessThan:
1425       return VisitUint32LessThan(node);
1426     case IrOpcode::kUint32LessThanOrEqual:
1427       return VisitUint32LessThanOrEqual(node);
1428     case IrOpcode::kUint32Mod:
1429       return MarkAsWord32(node), VisitUint32Mod(node);
1430     case IrOpcode::kUint32MulHigh:
1431       return VisitUint32MulHigh(node);
1432     case IrOpcode::kInt64Add:
1433       return MarkAsWord64(node), VisitInt64Add(node);
1434     case IrOpcode::kInt64AddWithOverflow:
1435       return MarkAsWord64(node), VisitInt64AddWithOverflow(node);
1436     case IrOpcode::kInt64Sub:
1437       return MarkAsWord64(node), VisitInt64Sub(node);
1438     case IrOpcode::kInt64SubWithOverflow:
1439       return MarkAsWord64(node), VisitInt64SubWithOverflow(node);
1440     case IrOpcode::kInt64Mul:
1441       return MarkAsWord64(node), VisitInt64Mul(node);
1442     case IrOpcode::kInt64Div:
1443       return MarkAsWord64(node), VisitInt64Div(node);
1444     case IrOpcode::kInt64Mod:
1445       return MarkAsWord64(node), VisitInt64Mod(node);
1446     case IrOpcode::kInt64LessThan:
1447       return VisitInt64LessThan(node);
1448     case IrOpcode::kInt64LessThanOrEqual:
1449       return VisitInt64LessThanOrEqual(node);
1450     case IrOpcode::kUint64Div:
1451       return MarkAsWord64(node), VisitUint64Div(node);
1452     case IrOpcode::kUint64LessThan:
1453       return VisitUint64LessThan(node);
1454     case IrOpcode::kUint64LessThanOrEqual:
1455       return VisitUint64LessThanOrEqual(node);
1456     case IrOpcode::kUint64Mod:
1457       return MarkAsWord64(node), VisitUint64Mod(node);
1458     case IrOpcode::kBitcastTaggedToWord:
1459       return MarkAsRepresentation(MachineType::PointerRepresentation(), node),
1460              VisitBitcastTaggedToWord(node);
1461     case IrOpcode::kBitcastWordToTagged:
1462       return MarkAsReference(node), VisitBitcastWordToTagged(node);
1463     case IrOpcode::kBitcastWordToTaggedSigned:
1464       return MarkAsRepresentation(MachineRepresentation::kTaggedSigned, node),
1465              EmitIdentity(node);
1466     case IrOpcode::kChangeFloat32ToFloat64:
1467       return MarkAsFloat64(node), VisitChangeFloat32ToFloat64(node);
1468     case IrOpcode::kChangeInt32ToFloat64:
1469       return MarkAsFloat64(node), VisitChangeInt32ToFloat64(node);
1470     case IrOpcode::kChangeUint32ToFloat64:
1471       return MarkAsFloat64(node), VisitChangeUint32ToFloat64(node);
1472     case IrOpcode::kChangeFloat64ToInt32:
1473       return MarkAsWord32(node), VisitChangeFloat64ToInt32(node);
1474     case IrOpcode::kChangeFloat64ToUint32:
1475       return MarkAsWord32(node), VisitChangeFloat64ToUint32(node);
1476     case IrOpcode::kChangeFloat64ToUint64:
1477       return MarkAsWord64(node), VisitChangeFloat64ToUint64(node);
1478     case IrOpcode::kFloat64SilenceNaN:
1479       MarkAsFloat64(node);
1480       if (CanProduceSignalingNaN(node->InputAt(0))) {
1481         return VisitFloat64SilenceNaN(node);
1482       } else {
1483         return EmitIdentity(node);
1484       }
1485     case IrOpcode::kTruncateFloat64ToUint32:
1486       return MarkAsWord32(node), VisitTruncateFloat64ToUint32(node);
1487     case IrOpcode::kTruncateFloat32ToInt32:
1488       return MarkAsWord32(node), VisitTruncateFloat32ToInt32(node);
1489     case IrOpcode::kTruncateFloat32ToUint32:
1490       return MarkAsWord32(node), VisitTruncateFloat32ToUint32(node);
1491     case IrOpcode::kTryTruncateFloat32ToInt64:
1492       return MarkAsWord64(node), VisitTryTruncateFloat32ToInt64(node);
1493     case IrOpcode::kTryTruncateFloat64ToInt64:
1494       return MarkAsWord64(node), VisitTryTruncateFloat64ToInt64(node);
1495     case IrOpcode::kTryTruncateFloat32ToUint64:
1496       return MarkAsWord64(node), VisitTryTruncateFloat32ToUint64(node);
1497     case IrOpcode::kTryTruncateFloat64ToUint64:
1498       return MarkAsWord64(node), VisitTryTruncateFloat64ToUint64(node);
1499     case IrOpcode::kChangeInt32ToInt64:
1500       return MarkAsWord64(node), VisitChangeInt32ToInt64(node);
1501     case IrOpcode::kChangeUint32ToUint64:
1502       return MarkAsWord64(node), VisitChangeUint32ToUint64(node);
1503     case IrOpcode::kTruncateFloat64ToFloat32:
1504       return MarkAsFloat32(node), VisitTruncateFloat64ToFloat32(node);
1505     case IrOpcode::kTruncateFloat64ToWord32:
1506       return MarkAsWord32(node), VisitTruncateFloat64ToWord32(node);
1507     case IrOpcode::kTruncateInt64ToInt32:
1508       return MarkAsWord32(node), VisitTruncateInt64ToInt32(node);
1509     case IrOpcode::kRoundFloat64ToInt32:
1510       return MarkAsWord32(node), VisitRoundFloat64ToInt32(node);
1511     case IrOpcode::kRoundInt64ToFloat32:
1512       return MarkAsFloat32(node), VisitRoundInt64ToFloat32(node);
1513     case IrOpcode::kRoundInt32ToFloat32:
1514       return MarkAsFloat32(node), VisitRoundInt32ToFloat32(node);
1515     case IrOpcode::kRoundInt64ToFloat64:
1516       return MarkAsFloat64(node), VisitRoundInt64ToFloat64(node);
1517     case IrOpcode::kBitcastFloat32ToInt32:
1518       return MarkAsWord32(node), VisitBitcastFloat32ToInt32(node);
1519     case IrOpcode::kRoundUint32ToFloat32:
1520       return MarkAsFloat32(node), VisitRoundUint32ToFloat32(node);
1521     case IrOpcode::kRoundUint64ToFloat32:
1522       return MarkAsFloat64(node), VisitRoundUint64ToFloat32(node);
1523     case IrOpcode::kRoundUint64ToFloat64:
1524       return MarkAsFloat64(node), VisitRoundUint64ToFloat64(node);
1525     case IrOpcode::kBitcastFloat64ToInt64:
1526       return MarkAsWord64(node), VisitBitcastFloat64ToInt64(node);
1527     case IrOpcode::kBitcastInt32ToFloat32:
1528       return MarkAsFloat32(node), VisitBitcastInt32ToFloat32(node);
1529     case IrOpcode::kBitcastInt64ToFloat64:
1530       return MarkAsFloat64(node), VisitBitcastInt64ToFloat64(node);
1531     case IrOpcode::kFloat32Add:
1532       return MarkAsFloat32(node), VisitFloat32Add(node);
1533     case IrOpcode::kFloat32Sub:
1534       return MarkAsFloat32(node), VisitFloat32Sub(node);
1535     case IrOpcode::kFloat32Neg:
1536       return MarkAsFloat32(node), VisitFloat32Neg(node);
1537     case IrOpcode::kFloat32Mul:
1538       return MarkAsFloat32(node), VisitFloat32Mul(node);
1539     case IrOpcode::kFloat32Div:
1540       return MarkAsFloat32(node), VisitFloat32Div(node);
1541     case IrOpcode::kFloat32Abs:
1542       return MarkAsFloat32(node), VisitFloat32Abs(node);
1543     case IrOpcode::kFloat32Sqrt:
1544       return MarkAsFloat32(node), VisitFloat32Sqrt(node);
1545     case IrOpcode::kFloat32Equal:
1546       return VisitFloat32Equal(node);
1547     case IrOpcode::kFloat32LessThan:
1548       return VisitFloat32LessThan(node);
1549     case IrOpcode::kFloat32LessThanOrEqual:
1550       return VisitFloat32LessThanOrEqual(node);
1551     case IrOpcode::kFloat32Max:
1552       return MarkAsFloat32(node), VisitFloat32Max(node);
1553     case IrOpcode::kFloat32Min:
1554       return MarkAsFloat32(node), VisitFloat32Min(node);
1555     case IrOpcode::kFloat64Add:
1556       return MarkAsFloat64(node), VisitFloat64Add(node);
1557     case IrOpcode::kFloat64Sub:
1558       return MarkAsFloat64(node), VisitFloat64Sub(node);
1559     case IrOpcode::kFloat64Neg:
1560       return MarkAsFloat64(node), VisitFloat64Neg(node);
1561     case IrOpcode::kFloat64Mul:
1562       return MarkAsFloat64(node), VisitFloat64Mul(node);
1563     case IrOpcode::kFloat64Div:
1564       return MarkAsFloat64(node), VisitFloat64Div(node);
1565     case IrOpcode::kFloat64Mod:
1566       return MarkAsFloat64(node), VisitFloat64Mod(node);
1567     case IrOpcode::kFloat64Min:
1568       return MarkAsFloat64(node), VisitFloat64Min(node);
1569     case IrOpcode::kFloat64Max:
1570       return MarkAsFloat64(node), VisitFloat64Max(node);
1571     case IrOpcode::kFloat64Abs:
1572       return MarkAsFloat64(node), VisitFloat64Abs(node);
1573     case IrOpcode::kFloat64Acos:
1574       return MarkAsFloat64(node), VisitFloat64Acos(node);
1575     case IrOpcode::kFloat64Acosh:
1576       return MarkAsFloat64(node), VisitFloat64Acosh(node);
1577     case IrOpcode::kFloat64Asin:
1578       return MarkAsFloat64(node), VisitFloat64Asin(node);
1579     case IrOpcode::kFloat64Asinh:
1580       return MarkAsFloat64(node), VisitFloat64Asinh(node);
1581     case IrOpcode::kFloat64Atan:
1582       return MarkAsFloat64(node), VisitFloat64Atan(node);
1583     case IrOpcode::kFloat64Atanh:
1584       return MarkAsFloat64(node), VisitFloat64Atanh(node);
1585     case IrOpcode::kFloat64Atan2:
1586       return MarkAsFloat64(node), VisitFloat64Atan2(node);
1587     case IrOpcode::kFloat64Cbrt:
1588       return MarkAsFloat64(node), VisitFloat64Cbrt(node);
1589     case IrOpcode::kFloat64Cos:
1590       return MarkAsFloat64(node), VisitFloat64Cos(node);
1591     case IrOpcode::kFloat64Cosh:
1592       return MarkAsFloat64(node), VisitFloat64Cosh(node);
1593     case IrOpcode::kFloat64Exp:
1594       return MarkAsFloat64(node), VisitFloat64Exp(node);
1595     case IrOpcode::kFloat64Expm1:
1596       return MarkAsFloat64(node), VisitFloat64Expm1(node);
1597     case IrOpcode::kFloat64Log:
1598       return MarkAsFloat64(node), VisitFloat64Log(node);
1599     case IrOpcode::kFloat64Log1p:
1600       return MarkAsFloat64(node), VisitFloat64Log1p(node);
1601     case IrOpcode::kFloat64Log10:
1602       return MarkAsFloat64(node), VisitFloat64Log10(node);
1603     case IrOpcode::kFloat64Log2:
1604       return MarkAsFloat64(node), VisitFloat64Log2(node);
1605     case IrOpcode::kFloat64Pow:
1606       return MarkAsFloat64(node), VisitFloat64Pow(node);
1607     case IrOpcode::kFloat64Sin:
1608       return MarkAsFloat64(node), VisitFloat64Sin(node);
1609     case IrOpcode::kFloat64Sinh:
1610       return MarkAsFloat64(node), VisitFloat64Sinh(node);
1611     case IrOpcode::kFloat64Sqrt:
1612       return MarkAsFloat64(node), VisitFloat64Sqrt(node);
1613     case IrOpcode::kFloat64Tan:
1614       return MarkAsFloat64(node), VisitFloat64Tan(node);
1615     case IrOpcode::kFloat64Tanh:
1616       return MarkAsFloat64(node), VisitFloat64Tanh(node);
1617     case IrOpcode::kFloat64Equal:
1618       return VisitFloat64Equal(node);
1619     case IrOpcode::kFloat64LessThan:
1620       return VisitFloat64LessThan(node);
1621     case IrOpcode::kFloat64LessThanOrEqual:
1622       return VisitFloat64LessThanOrEqual(node);
1623     case IrOpcode::kFloat32RoundDown:
1624       return MarkAsFloat32(node), VisitFloat32RoundDown(node);
1625     case IrOpcode::kFloat64RoundDown:
1626       return MarkAsFloat64(node), VisitFloat64RoundDown(node);
1627     case IrOpcode::kFloat32RoundUp:
1628       return MarkAsFloat32(node), VisitFloat32RoundUp(node);
1629     case IrOpcode::kFloat64RoundUp:
1630       return MarkAsFloat64(node), VisitFloat64RoundUp(node);
1631     case IrOpcode::kFloat32RoundTruncate:
1632       return MarkAsFloat32(node), VisitFloat32RoundTruncate(node);
1633     case IrOpcode::kFloat64RoundTruncate:
1634       return MarkAsFloat64(node), VisitFloat64RoundTruncate(node);
1635     case IrOpcode::kFloat64RoundTiesAway:
1636       return MarkAsFloat64(node), VisitFloat64RoundTiesAway(node);
1637     case IrOpcode::kFloat32RoundTiesEven:
1638       return MarkAsFloat32(node), VisitFloat32RoundTiesEven(node);
1639     case IrOpcode::kFloat64RoundTiesEven:
1640       return MarkAsFloat64(node), VisitFloat64RoundTiesEven(node);
1641     case IrOpcode::kFloat64ExtractLowWord32:
1642       return MarkAsWord32(node), VisitFloat64ExtractLowWord32(node);
1643     case IrOpcode::kFloat64ExtractHighWord32:
1644       return MarkAsWord32(node), VisitFloat64ExtractHighWord32(node);
1645     case IrOpcode::kFloat64InsertLowWord32:
1646       return MarkAsFloat64(node), VisitFloat64InsertLowWord32(node);
1647     case IrOpcode::kFloat64InsertHighWord32:
1648       return MarkAsFloat64(node), VisitFloat64InsertHighWord32(node);
1649     case IrOpcode::kTaggedPoisonOnSpeculation:
1650       return MarkAsReference(node), VisitTaggedPoisonOnSpeculation(node);
1651     case IrOpcode::kWord32PoisonOnSpeculation:
1652       return MarkAsWord32(node), VisitWord32PoisonOnSpeculation(node);
1653     case IrOpcode::kWord64PoisonOnSpeculation:
1654       return MarkAsWord64(node), VisitWord64PoisonOnSpeculation(node);
1655     case IrOpcode::kStackSlot:
1656       return VisitStackSlot(node);
1657     case IrOpcode::kLoadStackPointer:
1658       return VisitLoadStackPointer(node);
1659     case IrOpcode::kLoadFramePointer:
1660       return VisitLoadFramePointer(node);
1661     case IrOpcode::kLoadParentFramePointer:
1662       return VisitLoadParentFramePointer(node);
1663     case IrOpcode::kUnalignedLoad: {
1664       LoadRepresentation type = LoadRepresentationOf(node->op());
1665       MarkAsRepresentation(type.representation(), node);
1666       return VisitUnalignedLoad(node);
1667     }
1668     case IrOpcode::kUnalignedStore:
1669       return VisitUnalignedStore(node);
1670     case IrOpcode::kInt32PairAdd:
1671       MarkAsWord32(node);
1672       MarkPairProjectionsAsWord32(node);
1673       return VisitInt32PairAdd(node);
1674     case IrOpcode::kInt32PairSub:
1675       MarkAsWord32(node);
1676       MarkPairProjectionsAsWord32(node);
1677       return VisitInt32PairSub(node);
1678     case IrOpcode::kInt32PairMul:
1679       MarkAsWord32(node);
1680       MarkPairProjectionsAsWord32(node);
1681       return VisitInt32PairMul(node);
1682     case IrOpcode::kWord32PairShl:
1683       MarkAsWord32(node);
1684       MarkPairProjectionsAsWord32(node);
1685       return VisitWord32PairShl(node);
1686     case IrOpcode::kWord32PairShr:
1687       MarkAsWord32(node);
1688       MarkPairProjectionsAsWord32(node);
1689       return VisitWord32PairShr(node);
1690     case IrOpcode::kWord32PairSar:
1691       MarkAsWord32(node);
1692       MarkPairProjectionsAsWord32(node);
1693       return VisitWord32PairSar(node);
1694     case IrOpcode::kWord32AtomicLoad: {
1695       LoadRepresentation type = LoadRepresentationOf(node->op());
1696       MarkAsRepresentation(type.representation(), node);
1697       return VisitWord32AtomicLoad(node);
1698     }
1699     case IrOpcode::kWord64AtomicLoad: {
1700       LoadRepresentation type = LoadRepresentationOf(node->op());
1701       MarkAsRepresentation(type.representation(), node);
1702       return VisitWord64AtomicLoad(node);
1703     }
1704     case IrOpcode::kWord32AtomicStore:
1705       return VisitWord32AtomicStore(node);
1706     case IrOpcode::kWord64AtomicStore:
1707       return VisitWord64AtomicStore(node);
1708     case IrOpcode::kWord32AtomicPairStore:
1709       return VisitWord32AtomicPairStore(node);
1710     case IrOpcode::kWord32AtomicPairLoad: {
1711       MarkAsWord32(node);
1712       MarkPairProjectionsAsWord32(node);
1713       return VisitWord32AtomicPairLoad(node);
1714     }
1715 #define ATOMIC_CASE(name, rep)                         \
1716   case IrOpcode::k##rep##Atomic##name: {               \
1717     MachineType type = AtomicOpType(node->op());       \
1718     MarkAsRepresentation(type.representation(), node); \
1719     return Visit##rep##Atomic##name(node);             \
1720   }
1721       ATOMIC_CASE(Add, Word32)
1722       ATOMIC_CASE(Add, Word64)
1723       ATOMIC_CASE(Sub, Word32)
1724       ATOMIC_CASE(Sub, Word64)
1725       ATOMIC_CASE(And, Word32)
1726       ATOMIC_CASE(And, Word64)
1727       ATOMIC_CASE(Or, Word32)
1728       ATOMIC_CASE(Or, Word64)
1729       ATOMIC_CASE(Xor, Word32)
1730       ATOMIC_CASE(Xor, Word64)
1731       ATOMIC_CASE(Exchange, Word32)
1732       ATOMIC_CASE(Exchange, Word64)
1733       ATOMIC_CASE(CompareExchange, Word32)
1734       ATOMIC_CASE(CompareExchange, Word64)
1735 #undef ATOMIC_CASE
1736 #define ATOMIC_CASE(name)                     \
1737   case IrOpcode::kWord32AtomicPair##name: {   \
1738     MarkAsWord32(node);                       \
1739     MarkPairProjectionsAsWord32(node);        \
1740     return VisitWord32AtomicPair##name(node); \
1741   }
1742       ATOMIC_CASE(Add)
1743       ATOMIC_CASE(Sub)
1744       ATOMIC_CASE(And)
1745       ATOMIC_CASE(Or)
1746       ATOMIC_CASE(Xor)
1747       ATOMIC_CASE(Exchange)
1748       ATOMIC_CASE(CompareExchange)
1749 #undef ATOMIC_CASE
1750 #define ATOMIC_CASE(name)                              \
1751   case IrOpcode::kWord64AtomicNarrow##name: {          \
1752     MachineType type = AtomicOpType(node->op());       \
1753     MarkAsRepresentation(type.representation(), node); \
1754     MarkPairProjectionsAsWord32(node);                 \
1755     return VisitWord64AtomicNarrow##name(node);        \
1756   }
1757       ATOMIC_CASE(Add)
1758       ATOMIC_CASE(Sub)
1759       ATOMIC_CASE(And)
1760       ATOMIC_CASE(Or)
1761       ATOMIC_CASE(Xor)
1762       ATOMIC_CASE(Exchange)
1763       ATOMIC_CASE(CompareExchange)
1764 #undef ATOMIC_CASE
1765     case IrOpcode::kSpeculationFence:
1766       return VisitSpeculationFence(node);
1767     case IrOpcode::kProtectedLoad: {
1768       LoadRepresentation type = LoadRepresentationOf(node->op());
1769       MarkAsRepresentation(type.representation(), node);
1770       return VisitProtectedLoad(node);
1771     }
1772     case IrOpcode::kSignExtendWord8ToInt32:
1773       return MarkAsWord32(node), VisitSignExtendWord8ToInt32(node);
1774     case IrOpcode::kSignExtendWord16ToInt32:
1775       return MarkAsWord32(node), VisitSignExtendWord16ToInt32(node);
1776     case IrOpcode::kSignExtendWord8ToInt64:
1777       return MarkAsWord64(node), VisitSignExtendWord8ToInt64(node);
1778     case IrOpcode::kSignExtendWord16ToInt64:
1779       return MarkAsWord64(node), VisitSignExtendWord16ToInt64(node);
1780     case IrOpcode::kSignExtendWord32ToInt64:
1781       return MarkAsWord64(node), VisitSignExtendWord32ToInt64(node);
1782     case IrOpcode::kUnsafePointerAdd:
1783       MarkAsRepresentation(MachineType::PointerRepresentation(), node);
1784       return VisitUnsafePointerAdd(node);
1785     case IrOpcode::kF32x4Splat:
1786       return MarkAsSimd128(node), VisitF32x4Splat(node);
1787     case IrOpcode::kF32x4ExtractLane:
1788       return MarkAsFloat32(node), VisitF32x4ExtractLane(node);
1789     case IrOpcode::kF32x4ReplaceLane:
1790       return MarkAsSimd128(node), VisitF32x4ReplaceLane(node);
1791     case IrOpcode::kF32x4SConvertI32x4:
1792       return MarkAsSimd128(node), VisitF32x4SConvertI32x4(node);
1793     case IrOpcode::kF32x4UConvertI32x4:
1794       return MarkAsSimd128(node), VisitF32x4UConvertI32x4(node);
1795     case IrOpcode::kF32x4Abs:
1796       return MarkAsSimd128(node), VisitF32x4Abs(node);
1797     case IrOpcode::kF32x4Neg:
1798       return MarkAsSimd128(node), VisitF32x4Neg(node);
1799     case IrOpcode::kF32x4RecipApprox:
1800       return MarkAsSimd128(node), VisitF32x4RecipApprox(node);
1801     case IrOpcode::kF32x4RecipSqrtApprox:
1802       return MarkAsSimd128(node), VisitF32x4RecipSqrtApprox(node);
1803     case IrOpcode::kF32x4Add:
1804       return MarkAsSimd128(node), VisitF32x4Add(node);
1805     case IrOpcode::kF32x4AddHoriz:
1806       return MarkAsSimd128(node), VisitF32x4AddHoriz(node);
1807     case IrOpcode::kF32x4Sub:
1808       return MarkAsSimd128(node), VisitF32x4Sub(node);
1809     case IrOpcode::kF32x4Mul:
1810       return MarkAsSimd128(node), VisitF32x4Mul(node);
1811     case IrOpcode::kF32x4Min:
1812       return MarkAsSimd128(node), VisitF32x4Min(node);
1813     case IrOpcode::kF32x4Max:
1814       return MarkAsSimd128(node), VisitF32x4Max(node);
1815     case IrOpcode::kF32x4Eq:
1816       return MarkAsSimd128(node), VisitF32x4Eq(node);
1817     case IrOpcode::kF32x4Ne:
1818       return MarkAsSimd128(node), VisitF32x4Ne(node);
1819     case IrOpcode::kF32x4Lt:
1820       return MarkAsSimd128(node), VisitF32x4Lt(node);
1821     case IrOpcode::kF32x4Le:
1822       return MarkAsSimd128(node), VisitF32x4Le(node);
1823     case IrOpcode::kI32x4Splat:
1824       return MarkAsSimd128(node), VisitI32x4Splat(node);
1825     case IrOpcode::kI32x4ExtractLane:
1826       return MarkAsWord32(node), VisitI32x4ExtractLane(node);
1827     case IrOpcode::kI32x4ReplaceLane:
1828       return MarkAsSimd128(node), VisitI32x4ReplaceLane(node);
1829     case IrOpcode::kI32x4SConvertF32x4:
1830       return MarkAsSimd128(node), VisitI32x4SConvertF32x4(node);
1831     case IrOpcode::kI32x4SConvertI16x8Low:
1832       return MarkAsSimd128(node), VisitI32x4SConvertI16x8Low(node);
1833     case IrOpcode::kI32x4SConvertI16x8High:
1834       return MarkAsSimd128(node), VisitI32x4SConvertI16x8High(node);
1835     case IrOpcode::kI32x4Neg:
1836       return MarkAsSimd128(node), VisitI32x4Neg(node);
1837     case IrOpcode::kI32x4Shl:
1838       return MarkAsSimd128(node), VisitI32x4Shl(node);
1839     case IrOpcode::kI32x4ShrS:
1840       return MarkAsSimd128(node), VisitI32x4ShrS(node);
1841     case IrOpcode::kI32x4Add:
1842       return MarkAsSimd128(node), VisitI32x4Add(node);
1843     case IrOpcode::kI32x4AddHoriz:
1844       return MarkAsSimd128(node), VisitI32x4AddHoriz(node);
1845     case IrOpcode::kI32x4Sub:
1846       return MarkAsSimd128(node), VisitI32x4Sub(node);
1847     case IrOpcode::kI32x4Mul:
1848       return MarkAsSimd128(node), VisitI32x4Mul(node);
1849     case IrOpcode::kI32x4MinS:
1850       return MarkAsSimd128(node), VisitI32x4MinS(node);
1851     case IrOpcode::kI32x4MaxS:
1852       return MarkAsSimd128(node), VisitI32x4MaxS(node);
1853     case IrOpcode::kI32x4Eq:
1854       return MarkAsSimd128(node), VisitI32x4Eq(node);
1855     case IrOpcode::kI32x4Ne:
1856       return MarkAsSimd128(node), VisitI32x4Ne(node);
1857     case IrOpcode::kI32x4GtS:
1858       return MarkAsSimd128(node), VisitI32x4GtS(node);
1859     case IrOpcode::kI32x4GeS:
1860       return MarkAsSimd128(node), VisitI32x4GeS(node);
1861     case IrOpcode::kI32x4UConvertF32x4:
1862       return MarkAsSimd128(node), VisitI32x4UConvertF32x4(node);
1863     case IrOpcode::kI32x4UConvertI16x8Low:
1864       return MarkAsSimd128(node), VisitI32x4UConvertI16x8Low(node);
1865     case IrOpcode::kI32x4UConvertI16x8High:
1866       return MarkAsSimd128(node), VisitI32x4UConvertI16x8High(node);
1867     case IrOpcode::kI32x4ShrU:
1868       return MarkAsSimd128(node), VisitI32x4ShrU(node);
1869     case IrOpcode::kI32x4MinU:
1870       return MarkAsSimd128(node), VisitI32x4MinU(node);
1871     case IrOpcode::kI32x4MaxU:
1872       return MarkAsSimd128(node), VisitI32x4MaxU(node);
1873     case IrOpcode::kI32x4GtU:
1874       return MarkAsSimd128(node), VisitI32x4GtU(node);
1875     case IrOpcode::kI32x4GeU:
1876       return MarkAsSimd128(node), VisitI32x4GeU(node);
1877     case IrOpcode::kI16x8Splat:
1878       return MarkAsSimd128(node), VisitI16x8Splat(node);
1879     case IrOpcode::kI16x8ExtractLane:
1880       return MarkAsWord32(node), VisitI16x8ExtractLane(node);
1881     case IrOpcode::kI16x8ReplaceLane:
1882       return MarkAsSimd128(node), VisitI16x8ReplaceLane(node);
1883     case IrOpcode::kI16x8SConvertI8x16Low:
1884       return MarkAsSimd128(node), VisitI16x8SConvertI8x16Low(node);
1885     case IrOpcode::kI16x8SConvertI8x16High:
1886       return MarkAsSimd128(node), VisitI16x8SConvertI8x16High(node);
1887     case IrOpcode::kI16x8Neg:
1888       return MarkAsSimd128(node), VisitI16x8Neg(node);
1889     case IrOpcode::kI16x8Shl:
1890       return MarkAsSimd128(node), VisitI16x8Shl(node);
1891     case IrOpcode::kI16x8ShrS:
1892       return MarkAsSimd128(node), VisitI16x8ShrS(node);
1893     case IrOpcode::kI16x8SConvertI32x4:
1894       return MarkAsSimd128(node), VisitI16x8SConvertI32x4(node);
1895     case IrOpcode::kI16x8Add:
1896       return MarkAsSimd128(node), VisitI16x8Add(node);
1897     case IrOpcode::kI16x8AddSaturateS:
1898       return MarkAsSimd128(node), VisitI16x8AddSaturateS(node);
1899     case IrOpcode::kI16x8AddHoriz:
1900       return MarkAsSimd128(node), VisitI16x8AddHoriz(node);
1901     case IrOpcode::kI16x8Sub:
1902       return MarkAsSimd128(node), VisitI16x8Sub(node);
1903     case IrOpcode::kI16x8SubSaturateS:
1904       return MarkAsSimd128(node), VisitI16x8SubSaturateS(node);
1905     case IrOpcode::kI16x8Mul:
1906       return MarkAsSimd128(node), VisitI16x8Mul(node);
1907     case IrOpcode::kI16x8MinS:
1908       return MarkAsSimd128(node), VisitI16x8MinS(node);
1909     case IrOpcode::kI16x8MaxS:
1910       return MarkAsSimd128(node), VisitI16x8MaxS(node);
1911     case IrOpcode::kI16x8Eq:
1912       return MarkAsSimd128(node), VisitI16x8Eq(node);
1913     case IrOpcode::kI16x8Ne:
1914       return MarkAsSimd128(node), VisitI16x8Ne(node);
1915     case IrOpcode::kI16x8GtS:
1916       return MarkAsSimd128(node), VisitI16x8GtS(node);
1917     case IrOpcode::kI16x8GeS:
1918       return MarkAsSimd128(node), VisitI16x8GeS(node);
1919     case IrOpcode::kI16x8UConvertI8x16Low:
1920       return MarkAsSimd128(node), VisitI16x8UConvertI8x16Low(node);
1921     case IrOpcode::kI16x8UConvertI8x16High:
1922       return MarkAsSimd128(node), VisitI16x8UConvertI8x16High(node);
1923     case IrOpcode::kI16x8ShrU:
1924       return MarkAsSimd128(node), VisitI16x8ShrU(node);
1925     case IrOpcode::kI16x8UConvertI32x4:
1926       return MarkAsSimd128(node), VisitI16x8UConvertI32x4(node);
1927     case IrOpcode::kI16x8AddSaturateU:
1928       return MarkAsSimd128(node), VisitI16x8AddSaturateU(node);
1929     case IrOpcode::kI16x8SubSaturateU:
1930       return MarkAsSimd128(node), VisitI16x8SubSaturateU(node);
1931     case IrOpcode::kI16x8MinU:
1932       return MarkAsSimd128(node), VisitI16x8MinU(node);
1933     case IrOpcode::kI16x8MaxU:
1934       return MarkAsSimd128(node), VisitI16x8MaxU(node);
1935     case IrOpcode::kI16x8GtU:
1936       return MarkAsSimd128(node), VisitI16x8GtU(node);
1937     case IrOpcode::kI16x8GeU:
1938       return MarkAsSimd128(node), VisitI16x8GeU(node);
1939     case IrOpcode::kI8x16Splat:
1940       return MarkAsSimd128(node), VisitI8x16Splat(node);
1941     case IrOpcode::kI8x16ExtractLane:
1942       return MarkAsWord32(node), VisitI8x16ExtractLane(node);
1943     case IrOpcode::kI8x16ReplaceLane:
1944       return MarkAsSimd128(node), VisitI8x16ReplaceLane(node);
1945     case IrOpcode::kI8x16Neg:
1946       return MarkAsSimd128(node), VisitI8x16Neg(node);
1947     case IrOpcode::kI8x16Shl:
1948       return MarkAsSimd128(node), VisitI8x16Shl(node);
1949     case IrOpcode::kI8x16ShrS:
1950       return MarkAsSimd128(node), VisitI8x16ShrS(node);
1951     case IrOpcode::kI8x16SConvertI16x8:
1952       return MarkAsSimd128(node), VisitI8x16SConvertI16x8(node);
1953     case IrOpcode::kI8x16Add:
1954       return MarkAsSimd128(node), VisitI8x16Add(node);
1955     case IrOpcode::kI8x16AddSaturateS:
1956       return MarkAsSimd128(node), VisitI8x16AddSaturateS(node);
1957     case IrOpcode::kI8x16Sub:
1958       return MarkAsSimd128(node), VisitI8x16Sub(node);
1959     case IrOpcode::kI8x16SubSaturateS:
1960       return MarkAsSimd128(node), VisitI8x16SubSaturateS(node);
1961     case IrOpcode::kI8x16Mul:
1962       return MarkAsSimd128(node), VisitI8x16Mul(node);
1963     case IrOpcode::kI8x16MinS:
1964       return MarkAsSimd128(node), VisitI8x16MinS(node);
1965     case IrOpcode::kI8x16MaxS:
1966       return MarkAsSimd128(node), VisitI8x16MaxS(node);
1967     case IrOpcode::kI8x16Eq:
1968       return MarkAsSimd128(node), VisitI8x16Eq(node);
1969     case IrOpcode::kI8x16Ne:
1970       return MarkAsSimd128(node), VisitI8x16Ne(node);
1971     case IrOpcode::kI8x16GtS:
1972       return MarkAsSimd128(node), VisitI8x16GtS(node);
1973     case IrOpcode::kI8x16GeS:
1974       return MarkAsSimd128(node), VisitI8x16GeS(node);
1975     case IrOpcode::kI8x16ShrU:
1976       return MarkAsSimd128(node), VisitI8x16ShrU(node);
1977     case IrOpcode::kI8x16UConvertI16x8:
1978       return MarkAsSimd128(node), VisitI8x16UConvertI16x8(node);
1979     case IrOpcode::kI8x16AddSaturateU:
1980       return MarkAsSimd128(node), VisitI8x16AddSaturateU(node);
1981     case IrOpcode::kI8x16SubSaturateU:
1982       return MarkAsSimd128(node), VisitI8x16SubSaturateU(node);
1983     case IrOpcode::kI8x16MinU:
1984       return MarkAsSimd128(node), VisitI8x16MinU(node);
1985     case IrOpcode::kI8x16MaxU:
1986       return MarkAsSimd128(node), VisitI8x16MaxU(node);
1987     case IrOpcode::kI8x16GtU:
1988       return MarkAsSimd128(node), VisitI8x16GtU(node);
1989     case IrOpcode::kI8x16GeU:
1990       return MarkAsSimd128(node), VisitI16x8GeU(node);
1991     case IrOpcode::kS128Zero:
1992       return MarkAsSimd128(node), VisitS128Zero(node);
1993     case IrOpcode::kS128And:
1994       return MarkAsSimd128(node), VisitS128And(node);
1995     case IrOpcode::kS128Or:
1996       return MarkAsSimd128(node), VisitS128Or(node);
1997     case IrOpcode::kS128Xor:
1998       return MarkAsSimd128(node), VisitS128Xor(node);
1999     case IrOpcode::kS128Not:
2000       return MarkAsSimd128(node), VisitS128Not(node);
2001     case IrOpcode::kS128Select:
2002       return MarkAsSimd128(node), VisitS128Select(node);
2003     case IrOpcode::kS8x16Shuffle:
2004       return MarkAsSimd128(node), VisitS8x16Shuffle(node);
2005     case IrOpcode::kS1x4AnyTrue:
2006       return MarkAsWord32(node), VisitS1x4AnyTrue(node);
2007     case IrOpcode::kS1x4AllTrue:
2008       return MarkAsWord32(node), VisitS1x4AllTrue(node);
2009     case IrOpcode::kS1x8AnyTrue:
2010       return MarkAsWord32(node), VisitS1x8AnyTrue(node);
2011     case IrOpcode::kS1x8AllTrue:
2012       return MarkAsWord32(node), VisitS1x8AllTrue(node);
2013     case IrOpcode::kS1x16AnyTrue:
2014       return MarkAsWord32(node), VisitS1x16AnyTrue(node);
2015     case IrOpcode::kS1x16AllTrue:
2016       return MarkAsWord32(node), VisitS1x16AllTrue(node);
2017     default:
2018       FATAL("Unexpected operator #%d:%s @ node #%d", node->opcode(),
2019             node->op()->mnemonic(), node->id());
2020       break;
2021   }
2022 }
2023 
EmitWordPoisonOnSpeculation(Node * node)2024 void InstructionSelector::EmitWordPoisonOnSpeculation(Node* node) {
2025   if (poisoning_level_ != PoisoningMitigationLevel::kDontPoison) {
2026     OperandGenerator g(this);
2027     Node* input_node = NodeProperties::GetValueInput(node, 0);
2028     InstructionOperand input = g.UseRegister(input_node);
2029     InstructionOperand output = g.DefineSameAsFirst(node);
2030     Emit(kArchWordPoisonOnSpeculation, output, input);
2031   } else {
2032     EmitIdentity(node);
2033   }
2034 }
2035 
VisitWord32PoisonOnSpeculation(Node * node)2036 void InstructionSelector::VisitWord32PoisonOnSpeculation(Node* node) {
2037   EmitWordPoisonOnSpeculation(node);
2038 }
2039 
VisitWord64PoisonOnSpeculation(Node * node)2040 void InstructionSelector::VisitWord64PoisonOnSpeculation(Node* node) {
2041   EmitWordPoisonOnSpeculation(node);
2042 }
2043 
VisitTaggedPoisonOnSpeculation(Node * node)2044 void InstructionSelector::VisitTaggedPoisonOnSpeculation(Node* node) {
2045   EmitWordPoisonOnSpeculation(node);
2046 }
2047 
VisitLoadStackPointer(Node * node)2048 void InstructionSelector::VisitLoadStackPointer(Node* node) {
2049   OperandGenerator g(this);
2050   Emit(kArchStackPointer, g.DefineAsRegister(node));
2051 }
2052 
VisitLoadFramePointer(Node * node)2053 void InstructionSelector::VisitLoadFramePointer(Node* node) {
2054   OperandGenerator g(this);
2055   Emit(kArchFramePointer, g.DefineAsRegister(node));
2056 }
2057 
VisitLoadParentFramePointer(Node * node)2058 void InstructionSelector::VisitLoadParentFramePointer(Node* node) {
2059   OperandGenerator g(this);
2060   Emit(kArchParentFramePointer, g.DefineAsRegister(node));
2061 }
2062 
VisitFloat64Acos(Node * node)2063 void InstructionSelector::VisitFloat64Acos(Node* node) {
2064   VisitFloat64Ieee754Unop(node, kIeee754Float64Acos);
2065 }
2066 
VisitFloat64Acosh(Node * node)2067 void InstructionSelector::VisitFloat64Acosh(Node* node) {
2068   VisitFloat64Ieee754Unop(node, kIeee754Float64Acosh);
2069 }
2070 
VisitFloat64Asin(Node * node)2071 void InstructionSelector::VisitFloat64Asin(Node* node) {
2072   VisitFloat64Ieee754Unop(node, kIeee754Float64Asin);
2073 }
2074 
VisitFloat64Asinh(Node * node)2075 void InstructionSelector::VisitFloat64Asinh(Node* node) {
2076   VisitFloat64Ieee754Unop(node, kIeee754Float64Asinh);
2077 }
2078 
VisitFloat64Atan(Node * node)2079 void InstructionSelector::VisitFloat64Atan(Node* node) {
2080   VisitFloat64Ieee754Unop(node, kIeee754Float64Atan);
2081 }
2082 
VisitFloat64Atanh(Node * node)2083 void InstructionSelector::VisitFloat64Atanh(Node* node) {
2084   VisitFloat64Ieee754Unop(node, kIeee754Float64Atanh);
2085 }
2086 
VisitFloat64Atan2(Node * node)2087 void InstructionSelector::VisitFloat64Atan2(Node* node) {
2088   VisitFloat64Ieee754Binop(node, kIeee754Float64Atan2);
2089 }
2090 
VisitFloat64Cbrt(Node * node)2091 void InstructionSelector::VisitFloat64Cbrt(Node* node) {
2092   VisitFloat64Ieee754Unop(node, kIeee754Float64Cbrt);
2093 }
2094 
VisitFloat64Cos(Node * node)2095 void InstructionSelector::VisitFloat64Cos(Node* node) {
2096   VisitFloat64Ieee754Unop(node, kIeee754Float64Cos);
2097 }
2098 
VisitFloat64Cosh(Node * node)2099 void InstructionSelector::VisitFloat64Cosh(Node* node) {
2100   VisitFloat64Ieee754Unop(node, kIeee754Float64Cosh);
2101 }
2102 
VisitFloat64Exp(Node * node)2103 void InstructionSelector::VisitFloat64Exp(Node* node) {
2104   VisitFloat64Ieee754Unop(node, kIeee754Float64Exp);
2105 }
2106 
VisitFloat64Expm1(Node * node)2107 void InstructionSelector::VisitFloat64Expm1(Node* node) {
2108   VisitFloat64Ieee754Unop(node, kIeee754Float64Expm1);
2109 }
2110 
VisitFloat64Log(Node * node)2111 void InstructionSelector::VisitFloat64Log(Node* node) {
2112   VisitFloat64Ieee754Unop(node, kIeee754Float64Log);
2113 }
2114 
VisitFloat64Log1p(Node * node)2115 void InstructionSelector::VisitFloat64Log1p(Node* node) {
2116   VisitFloat64Ieee754Unop(node, kIeee754Float64Log1p);
2117 }
2118 
VisitFloat64Log2(Node * node)2119 void InstructionSelector::VisitFloat64Log2(Node* node) {
2120   VisitFloat64Ieee754Unop(node, kIeee754Float64Log2);
2121 }
2122 
VisitFloat64Log10(Node * node)2123 void InstructionSelector::VisitFloat64Log10(Node* node) {
2124   VisitFloat64Ieee754Unop(node, kIeee754Float64Log10);
2125 }
2126 
VisitFloat64Pow(Node * node)2127 void InstructionSelector::VisitFloat64Pow(Node* node) {
2128   VisitFloat64Ieee754Binop(node, kIeee754Float64Pow);
2129 }
2130 
VisitFloat64Sin(Node * node)2131 void InstructionSelector::VisitFloat64Sin(Node* node) {
2132   VisitFloat64Ieee754Unop(node, kIeee754Float64Sin);
2133 }
2134 
VisitFloat64Sinh(Node * node)2135 void InstructionSelector::VisitFloat64Sinh(Node* node) {
2136   VisitFloat64Ieee754Unop(node, kIeee754Float64Sinh);
2137 }
2138 
VisitFloat64Tan(Node * node)2139 void InstructionSelector::VisitFloat64Tan(Node* node) {
2140   VisitFloat64Ieee754Unop(node, kIeee754Float64Tan);
2141 }
2142 
VisitFloat64Tanh(Node * node)2143 void InstructionSelector::VisitFloat64Tanh(Node* node) {
2144   VisitFloat64Ieee754Unop(node, kIeee754Float64Tanh);
2145 }
2146 
EmitTableSwitch(const SwitchInfo & sw,InstructionOperand & index_operand)2147 void InstructionSelector::EmitTableSwitch(const SwitchInfo& sw,
2148                                           InstructionOperand& index_operand) {
2149   OperandGenerator g(this);
2150   size_t input_count = 2 + sw.value_range();
2151   DCHECK_LE(sw.value_range(), std::numeric_limits<size_t>::max() - 2);
2152   auto* inputs = zone()->NewArray<InstructionOperand>(input_count);
2153   inputs[0] = index_operand;
2154   InstructionOperand default_operand = g.Label(sw.default_branch());
2155   std::fill(&inputs[1], &inputs[input_count], default_operand);
2156   for (const CaseInfo& c : sw.CasesUnsorted()) {
2157     size_t value = c.value - sw.min_value();
2158     DCHECK_LE(0u, value);
2159     DCHECK_LT(value + 2, input_count);
2160     inputs[value + 2] = g.Label(c.branch);
2161   }
2162   Emit(kArchTableSwitch, 0, nullptr, input_count, inputs, 0, nullptr);
2163 }
2164 
2165 
EmitLookupSwitch(const SwitchInfo & sw,InstructionOperand & value_operand)2166 void InstructionSelector::EmitLookupSwitch(const SwitchInfo& sw,
2167                                            InstructionOperand& value_operand) {
2168   OperandGenerator g(this);
2169   std::vector<CaseInfo> cases = sw.CasesSortedByOriginalOrder();
2170   size_t input_count = 2 + sw.case_count() * 2;
2171   DCHECK_LE(sw.case_count(), (std::numeric_limits<size_t>::max() - 2) / 2);
2172   auto* inputs = zone()->NewArray<InstructionOperand>(input_count);
2173   inputs[0] = value_operand;
2174   inputs[1] = g.Label(sw.default_branch());
2175   for (size_t index = 0; index < cases.size(); ++index) {
2176     const CaseInfo& c = cases[index];
2177     inputs[index * 2 + 2 + 0] = g.TempImmediate(c.value);
2178     inputs[index * 2 + 2 + 1] = g.Label(c.branch);
2179   }
2180   Emit(kArchLookupSwitch, 0, nullptr, input_count, inputs, 0, nullptr);
2181 }
2182 
EmitBinarySearchSwitch(const SwitchInfo & sw,InstructionOperand & value_operand)2183 void InstructionSelector::EmitBinarySearchSwitch(
2184     const SwitchInfo& sw, InstructionOperand& value_operand) {
2185   OperandGenerator g(this);
2186   size_t input_count = 2 + sw.case_count() * 2;
2187   DCHECK_LE(sw.case_count(), (std::numeric_limits<size_t>::max() - 2) / 2);
2188   auto* inputs = zone()->NewArray<InstructionOperand>(input_count);
2189   inputs[0] = value_operand;
2190   inputs[1] = g.Label(sw.default_branch());
2191   std::vector<CaseInfo> cases = sw.CasesSortedByValue();
2192   std::stable_sort(cases.begin(), cases.end(),
2193                    [](CaseInfo a, CaseInfo b) { return a.value < b.value; });
2194   for (size_t index = 0; index < cases.size(); ++index) {
2195     const CaseInfo& c = cases[index];
2196     inputs[index * 2 + 2 + 0] = g.TempImmediate(c.value);
2197     inputs[index * 2 + 2 + 1] = g.Label(c.branch);
2198   }
2199   Emit(kArchBinarySearchSwitch, 0, nullptr, input_count, inputs, 0, nullptr);
2200 }
2201 
VisitBitcastTaggedToWord(Node * node)2202 void InstructionSelector::VisitBitcastTaggedToWord(Node* node) {
2203   EmitIdentity(node);
2204 }
2205 
VisitBitcastWordToTagged(Node * node)2206 void InstructionSelector::VisitBitcastWordToTagged(Node* node) {
2207   OperandGenerator g(this);
2208   Emit(kArchNop, g.DefineSameAsFirst(node), g.Use(node->InputAt(0)));
2209 }
2210 
2211 // 32 bit targets do not implement the following instructions.
2212 #if V8_TARGET_ARCH_32_BIT
2213 
VisitWord64And(Node * node)2214 void InstructionSelector::VisitWord64And(Node* node) { UNIMPLEMENTED(); }
2215 
2216 
VisitWord64Or(Node * node)2217 void InstructionSelector::VisitWord64Or(Node* node) { UNIMPLEMENTED(); }
2218 
2219 
VisitWord64Xor(Node * node)2220 void InstructionSelector::VisitWord64Xor(Node* node) { UNIMPLEMENTED(); }
2221 
2222 
VisitWord64Shl(Node * node)2223 void InstructionSelector::VisitWord64Shl(Node* node) { UNIMPLEMENTED(); }
2224 
2225 
VisitWord64Shr(Node * node)2226 void InstructionSelector::VisitWord64Shr(Node* node) { UNIMPLEMENTED(); }
2227 
2228 
VisitWord64Sar(Node * node)2229 void InstructionSelector::VisitWord64Sar(Node* node) { UNIMPLEMENTED(); }
2230 
2231 
VisitWord64Ror(Node * node)2232 void InstructionSelector::VisitWord64Ror(Node* node) { UNIMPLEMENTED(); }
2233 
2234 
VisitWord64Clz(Node * node)2235 void InstructionSelector::VisitWord64Clz(Node* node) { UNIMPLEMENTED(); }
2236 
2237 
VisitWord64Ctz(Node * node)2238 void InstructionSelector::VisitWord64Ctz(Node* node) { UNIMPLEMENTED(); }
2239 
2240 
VisitWord64ReverseBits(Node * node)2241 void InstructionSelector::VisitWord64ReverseBits(Node* node) {
2242   UNIMPLEMENTED();
2243 }
2244 
2245 
VisitWord64Popcnt(Node * node)2246 void InstructionSelector::VisitWord64Popcnt(Node* node) { UNIMPLEMENTED(); }
2247 
2248 
VisitWord64Equal(Node * node)2249 void InstructionSelector::VisitWord64Equal(Node* node) { UNIMPLEMENTED(); }
2250 
2251 
VisitInt64Add(Node * node)2252 void InstructionSelector::VisitInt64Add(Node* node) { UNIMPLEMENTED(); }
2253 
2254 
VisitInt64AddWithOverflow(Node * node)2255 void InstructionSelector::VisitInt64AddWithOverflow(Node* node) {
2256   UNIMPLEMENTED();
2257 }
2258 
2259 
VisitInt64Sub(Node * node)2260 void InstructionSelector::VisitInt64Sub(Node* node) { UNIMPLEMENTED(); }
2261 
2262 
VisitInt64SubWithOverflow(Node * node)2263 void InstructionSelector::VisitInt64SubWithOverflow(Node* node) {
2264   UNIMPLEMENTED();
2265 }
2266 
VisitInt64Mul(Node * node)2267 void InstructionSelector::VisitInt64Mul(Node* node) { UNIMPLEMENTED(); }
2268 
2269 
VisitInt64Div(Node * node)2270 void InstructionSelector::VisitInt64Div(Node* node) { UNIMPLEMENTED(); }
2271 
2272 
VisitInt64LessThan(Node * node)2273 void InstructionSelector::VisitInt64LessThan(Node* node) { UNIMPLEMENTED(); }
2274 
2275 
VisitInt64LessThanOrEqual(Node * node)2276 void InstructionSelector::VisitInt64LessThanOrEqual(Node* node) {
2277   UNIMPLEMENTED();
2278 }
2279 
2280 
VisitUint64Div(Node * node)2281 void InstructionSelector::VisitUint64Div(Node* node) { UNIMPLEMENTED(); }
2282 
2283 
VisitInt64Mod(Node * node)2284 void InstructionSelector::VisitInt64Mod(Node* node) { UNIMPLEMENTED(); }
2285 
2286 
VisitUint64LessThan(Node * node)2287 void InstructionSelector::VisitUint64LessThan(Node* node) { UNIMPLEMENTED(); }
2288 
2289 
VisitUint64LessThanOrEqual(Node * node)2290 void InstructionSelector::VisitUint64LessThanOrEqual(Node* node) {
2291   UNIMPLEMENTED();
2292 }
2293 
2294 
VisitUint64Mod(Node * node)2295 void InstructionSelector::VisitUint64Mod(Node* node) { UNIMPLEMENTED(); }
2296 
2297 
VisitChangeInt32ToInt64(Node * node)2298 void InstructionSelector::VisitChangeInt32ToInt64(Node* node) {
2299   UNIMPLEMENTED();
2300 }
2301 
2302 
VisitChangeUint32ToUint64(Node * node)2303 void InstructionSelector::VisitChangeUint32ToUint64(Node* node) {
2304   UNIMPLEMENTED();
2305 }
2306 
VisitChangeFloat64ToUint64(Node * node)2307 void InstructionSelector::VisitChangeFloat64ToUint64(Node* node) {
2308   UNIMPLEMENTED();
2309 }
2310 
VisitTryTruncateFloat32ToInt64(Node * node)2311 void InstructionSelector::VisitTryTruncateFloat32ToInt64(Node* node) {
2312   UNIMPLEMENTED();
2313 }
2314 
2315 
VisitTryTruncateFloat64ToInt64(Node * node)2316 void InstructionSelector::VisitTryTruncateFloat64ToInt64(Node* node) {
2317   UNIMPLEMENTED();
2318 }
2319 
2320 
VisitTryTruncateFloat32ToUint64(Node * node)2321 void InstructionSelector::VisitTryTruncateFloat32ToUint64(Node* node) {
2322   UNIMPLEMENTED();
2323 }
2324 
2325 
VisitTryTruncateFloat64ToUint64(Node * node)2326 void InstructionSelector::VisitTryTruncateFloat64ToUint64(Node* node) {
2327   UNIMPLEMENTED();
2328 }
2329 
2330 
VisitTruncateInt64ToInt32(Node * node)2331 void InstructionSelector::VisitTruncateInt64ToInt32(Node* node) {
2332   UNIMPLEMENTED();
2333 }
2334 
2335 
VisitRoundInt64ToFloat32(Node * node)2336 void InstructionSelector::VisitRoundInt64ToFloat32(Node* node) {
2337   UNIMPLEMENTED();
2338 }
2339 
2340 
VisitRoundInt64ToFloat64(Node * node)2341 void InstructionSelector::VisitRoundInt64ToFloat64(Node* node) {
2342   UNIMPLEMENTED();
2343 }
2344 
2345 
VisitRoundUint64ToFloat32(Node * node)2346 void InstructionSelector::VisitRoundUint64ToFloat32(Node* node) {
2347   UNIMPLEMENTED();
2348 }
2349 
2350 
VisitRoundUint64ToFloat64(Node * node)2351 void InstructionSelector::VisitRoundUint64ToFloat64(Node* node) {
2352   UNIMPLEMENTED();
2353 }
2354 
VisitBitcastFloat64ToInt64(Node * node)2355 void InstructionSelector::VisitBitcastFloat64ToInt64(Node* node) {
2356   UNIMPLEMENTED();
2357 }
2358 
2359 
VisitBitcastInt64ToFloat64(Node * node)2360 void InstructionSelector::VisitBitcastInt64ToFloat64(Node* node) {
2361   UNIMPLEMENTED();
2362 }
2363 
VisitSignExtendWord8ToInt64(Node * node)2364 void InstructionSelector::VisitSignExtendWord8ToInt64(Node* node) {
2365   UNIMPLEMENTED();
2366 }
2367 
VisitSignExtendWord16ToInt64(Node * node)2368 void InstructionSelector::VisitSignExtendWord16ToInt64(Node* node) {
2369   UNIMPLEMENTED();
2370 }
2371 
VisitSignExtendWord32ToInt64(Node * node)2372 void InstructionSelector::VisitSignExtendWord32ToInt64(Node* node) {
2373   UNIMPLEMENTED();
2374 }
2375 #endif  // V8_TARGET_ARCH_32_BIT
2376 
2377 // 64 bit targets do not implement the following instructions.
2378 #if V8_TARGET_ARCH_64_BIT
VisitInt32PairAdd(Node * node)2379 void InstructionSelector::VisitInt32PairAdd(Node* node) { UNIMPLEMENTED(); }
2380 
VisitInt32PairSub(Node * node)2381 void InstructionSelector::VisitInt32PairSub(Node* node) { UNIMPLEMENTED(); }
2382 
VisitInt32PairMul(Node * node)2383 void InstructionSelector::VisitInt32PairMul(Node* node) { UNIMPLEMENTED(); }
2384 
VisitWord32PairShl(Node * node)2385 void InstructionSelector::VisitWord32PairShl(Node* node) { UNIMPLEMENTED(); }
2386 
VisitWord32PairShr(Node * node)2387 void InstructionSelector::VisitWord32PairShr(Node* node) { UNIMPLEMENTED(); }
2388 
VisitWord32PairSar(Node * node)2389 void InstructionSelector::VisitWord32PairSar(Node* node) { UNIMPLEMENTED(); }
2390 #endif  // V8_TARGET_ARCH_64_BIT
2391 
2392 #if !V8_TARGET_ARCH_IA32 && !V8_TARGET_ARCH_ARM
VisitWord32AtomicPairLoad(Node * node)2393 void InstructionSelector::VisitWord32AtomicPairLoad(Node* node) {
2394   UNIMPLEMENTED();
2395 }
2396 
VisitWord32AtomicPairStore(Node * node)2397 void InstructionSelector::VisitWord32AtomicPairStore(Node* node) {
2398   UNIMPLEMENTED();
2399 }
2400 
VisitWord32AtomicPairAdd(Node * node)2401 void InstructionSelector::VisitWord32AtomicPairAdd(Node* node) {
2402   UNIMPLEMENTED();
2403 }
2404 
VisitWord32AtomicPairSub(Node * node)2405 void InstructionSelector::VisitWord32AtomicPairSub(Node* node) {
2406   UNIMPLEMENTED();
2407 }
2408 
VisitWord32AtomicPairAnd(Node * node)2409 void InstructionSelector::VisitWord32AtomicPairAnd(Node* node) {
2410   UNIMPLEMENTED();
2411 }
2412 
VisitWord32AtomicPairOr(Node * node)2413 void InstructionSelector::VisitWord32AtomicPairOr(Node* node) {
2414   UNIMPLEMENTED();
2415 }
2416 
VisitWord32AtomicPairXor(Node * node)2417 void InstructionSelector::VisitWord32AtomicPairXor(Node* node) {
2418   UNIMPLEMENTED();
2419 }
2420 
VisitWord32AtomicPairExchange(Node * node)2421 void InstructionSelector::VisitWord32AtomicPairExchange(Node* node) {
2422   UNIMPLEMENTED();
2423 }
2424 
VisitWord32AtomicPairCompareExchange(Node * node)2425 void InstructionSelector::VisitWord32AtomicPairCompareExchange(Node* node) {
2426   UNIMPLEMENTED();
2427 }
2428 
VisitWord64AtomicNarrowAdd(Node * node)2429 void InstructionSelector::VisitWord64AtomicNarrowAdd(Node* node) {
2430   UNIMPLEMENTED();
2431 }
2432 
VisitWord64AtomicNarrowSub(Node * node)2433 void InstructionSelector::VisitWord64AtomicNarrowSub(Node* node) {
2434   UNIMPLEMENTED();
2435 }
2436 
VisitWord64AtomicNarrowAnd(Node * node)2437 void InstructionSelector::VisitWord64AtomicNarrowAnd(Node* node) {
2438   UNIMPLEMENTED();
2439 }
2440 
VisitWord64AtomicNarrowOr(Node * node)2441 void InstructionSelector::VisitWord64AtomicNarrowOr(Node* node) {
2442   UNIMPLEMENTED();
2443 }
2444 
VisitWord64AtomicNarrowXor(Node * node)2445 void InstructionSelector::VisitWord64AtomicNarrowXor(Node* node) {
2446   UNIMPLEMENTED();
2447 }
2448 
VisitWord64AtomicNarrowExchange(Node * node)2449 void InstructionSelector::VisitWord64AtomicNarrowExchange(Node* node) {
2450   UNIMPLEMENTED();
2451 }
2452 
VisitWord64AtomicNarrowCompareExchange(Node * node)2453 void InstructionSelector::VisitWord64AtomicNarrowCompareExchange(Node* node) {
2454   UNIMPLEMENTED();
2455 }
2456 #endif  // !V8_TARGET_ARCH_IA32 && !V8_TARGET_ARCH_ARM
2457 
2458 #if !V8_TARGET_ARCH_ARM && !V8_TARGET_ARCH_ARM64 && !V8_TARGET_ARCH_MIPS && \
2459     !V8_TARGET_ARCH_MIPS64 && !V8_TARGET_ARCH_IA32
VisitF32x4SConvertI32x4(Node * node)2460 void InstructionSelector::VisitF32x4SConvertI32x4(Node* node) {
2461   UNIMPLEMENTED();
2462 }
2463 
VisitF32x4UConvertI32x4(Node * node)2464 void InstructionSelector::VisitF32x4UConvertI32x4(Node* node) {
2465   UNIMPLEMENTED();
2466 }
2467 #endif  // !V8_TARGET_ARCH_ARM && !V8_TARGET_ARCH_ARM64 && !V8_TARGET_ARCH_MIPS
2468         // && !V8_TARGET_ARCH_MIPS64 && !V8_TARGET_ARCH_IA32
2469 
2470 #if !V8_TARGET_ARCH_X64 && !V8_TARGET_ARCH_ARM64
VisitWord64AtomicLoad(Node * node)2471 void InstructionSelector::VisitWord64AtomicLoad(Node* node) { UNIMPLEMENTED(); }
2472 
VisitWord64AtomicStore(Node * node)2473 void InstructionSelector::VisitWord64AtomicStore(Node* node) {
2474   UNIMPLEMENTED();
2475 }
2476 
VisitWord64AtomicAdd(Node * node)2477 void InstructionSelector::VisitWord64AtomicAdd(Node* node) { UNIMPLEMENTED(); }
2478 
VisitWord64AtomicSub(Node * node)2479 void InstructionSelector::VisitWord64AtomicSub(Node* node) { UNIMPLEMENTED(); }
2480 
VisitWord64AtomicAnd(Node * node)2481 void InstructionSelector::VisitWord64AtomicAnd(Node* node) { UNIMPLEMENTED(); }
2482 
VisitWord64AtomicOr(Node * node)2483 void InstructionSelector::VisitWord64AtomicOr(Node* node) { UNIMPLEMENTED(); }
2484 
VisitWord64AtomicXor(Node * node)2485 void InstructionSelector::VisitWord64AtomicXor(Node* node) { UNIMPLEMENTED(); }
2486 
VisitWord64AtomicExchange(Node * node)2487 void InstructionSelector::VisitWord64AtomicExchange(Node* node) {
2488   UNIMPLEMENTED();
2489 }
2490 
VisitWord64AtomicCompareExchange(Node * node)2491 void InstructionSelector::VisitWord64AtomicCompareExchange(Node* node) {
2492   UNIMPLEMENTED();
2493 }
2494 #endif  // !V8_TARGET_ARCH_X64 && !V8_TARGET_ARCH_ARM64
2495 
2496 #if !V8_TARGET_ARCH_ARM && !V8_TARGET_ARCH_ARM64 && !V8_TARGET_ARCH_MIPS && \
2497     !V8_TARGET_ARCH_MIPS64 && !V8_TARGET_ARCH_IA32
VisitI32x4SConvertF32x4(Node * node)2498 void InstructionSelector::VisitI32x4SConvertF32x4(Node* node) {
2499   UNIMPLEMENTED();
2500 }
2501 
VisitI32x4UConvertF32x4(Node * node)2502 void InstructionSelector::VisitI32x4UConvertF32x4(Node* node) {
2503   UNIMPLEMENTED();
2504 }
2505 
VisitI32x4SConvertI16x8Low(Node * node)2506 void InstructionSelector::VisitI32x4SConvertI16x8Low(Node* node) {
2507   UNIMPLEMENTED();
2508 }
2509 
VisitI32x4SConvertI16x8High(Node * node)2510 void InstructionSelector::VisitI32x4SConvertI16x8High(Node* node) {
2511   UNIMPLEMENTED();
2512 }
2513 
VisitI32x4UConvertI16x8Low(Node * node)2514 void InstructionSelector::VisitI32x4UConvertI16x8Low(Node* node) {
2515   UNIMPLEMENTED();
2516 }
2517 
VisitI32x4UConvertI16x8High(Node * node)2518 void InstructionSelector::VisitI32x4UConvertI16x8High(Node* node) {
2519   UNIMPLEMENTED();
2520 }
2521 
VisitI16x8SConvertI8x16Low(Node * node)2522 void InstructionSelector::VisitI16x8SConvertI8x16Low(Node* node) {
2523   UNIMPLEMENTED();
2524 }
2525 
VisitI16x8SConvertI8x16High(Node * node)2526 void InstructionSelector::VisitI16x8SConvertI8x16High(Node* node) {
2527   UNIMPLEMENTED();
2528 }
2529 
VisitI16x8UConvertI8x16Low(Node * node)2530 void InstructionSelector::VisitI16x8UConvertI8x16Low(Node* node) {
2531   UNIMPLEMENTED();
2532 }
2533 
VisitI16x8UConvertI8x16High(Node * node)2534 void InstructionSelector::VisitI16x8UConvertI8x16High(Node* node) {
2535   UNIMPLEMENTED();
2536 }
2537 
VisitI16x8SConvertI32x4(Node * node)2538 void InstructionSelector::VisitI16x8SConvertI32x4(Node* node) {
2539   UNIMPLEMENTED();
2540 }
VisitI16x8UConvertI32x4(Node * node)2541 void InstructionSelector::VisitI16x8UConvertI32x4(Node* node) {
2542   UNIMPLEMENTED();
2543 }
2544 
VisitI8x16SConvertI16x8(Node * node)2545 void InstructionSelector::VisitI8x16SConvertI16x8(Node* node) {
2546   UNIMPLEMENTED();
2547 }
2548 
VisitI8x16UConvertI16x8(Node * node)2549 void InstructionSelector::VisitI8x16UConvertI16x8(Node* node) {
2550   UNIMPLEMENTED();
2551 }
2552 
VisitI8x16Shl(Node * node)2553 void InstructionSelector::VisitI8x16Shl(Node* node) { UNIMPLEMENTED(); }
2554 
VisitI8x16ShrS(Node * node)2555 void InstructionSelector::VisitI8x16ShrS(Node* node) { UNIMPLEMENTED(); }
2556 
VisitI8x16ShrU(Node * node)2557 void InstructionSelector::VisitI8x16ShrU(Node* node) { UNIMPLEMENTED(); }
2558 
VisitI8x16Mul(Node * node)2559 void InstructionSelector::VisitI8x16Mul(Node* node) { UNIMPLEMENTED(); }
2560 
VisitS8x16Shuffle(Node * node)2561 void InstructionSelector::VisitS8x16Shuffle(Node* node) { UNIMPLEMENTED(); }
2562 
VisitS1x4AnyTrue(Node * node)2563 void InstructionSelector::VisitS1x4AnyTrue(Node* node) { UNIMPLEMENTED(); }
2564 
VisitS1x4AllTrue(Node * node)2565 void InstructionSelector::VisitS1x4AllTrue(Node* node) { UNIMPLEMENTED(); }
2566 
VisitS1x8AnyTrue(Node * node)2567 void InstructionSelector::VisitS1x8AnyTrue(Node* node) { UNIMPLEMENTED(); }
2568 
VisitS1x8AllTrue(Node * node)2569 void InstructionSelector::VisitS1x8AllTrue(Node* node) { UNIMPLEMENTED(); }
2570 
VisitS1x16AnyTrue(Node * node)2571 void InstructionSelector::VisitS1x16AnyTrue(Node* node) { UNIMPLEMENTED(); }
2572 
VisitS1x16AllTrue(Node * node)2573 void InstructionSelector::VisitS1x16AllTrue(Node* node) { UNIMPLEMENTED(); }
2574 #endif  // !V8_TARGET_ARCH_ARM && !V8_TARGET_ARCH_ARM64 && !V8_TARGET_ARCH_MIPS
2575         // && !V8_TARGET_ARCH_MIPS64 && !V8_TARGET_ARCH_IA32
2576 
VisitFinishRegion(Node * node)2577 void InstructionSelector::VisitFinishRegion(Node* node) { EmitIdentity(node); }
2578 
VisitParameter(Node * node)2579 void InstructionSelector::VisitParameter(Node* node) {
2580   OperandGenerator g(this);
2581   int index = ParameterIndexOf(node->op());
2582   InstructionOperand op =
2583       linkage()->ParameterHasSecondaryLocation(index)
2584           ? g.DefineAsDualLocation(
2585                 node, linkage()->GetParameterLocation(index),
2586                 linkage()->GetParameterSecondaryLocation(index))
2587           : g.DefineAsLocation(node, linkage()->GetParameterLocation(index));
2588 
2589   Emit(kArchNop, op);
2590 }
2591 
2592 namespace {
ExceptionLocation()2593 LinkageLocation ExceptionLocation() {
2594   return LinkageLocation::ForRegister(kReturnRegister0.code(),
2595                                       MachineType::IntPtr());
2596 }
2597 }
2598 
VisitIfException(Node * node)2599 void InstructionSelector::VisitIfException(Node* node) {
2600   OperandGenerator g(this);
2601   DCHECK_EQ(IrOpcode::kCall, node->InputAt(1)->opcode());
2602   Emit(kArchNop, g.DefineAsLocation(node, ExceptionLocation()));
2603 }
2604 
2605 
VisitOsrValue(Node * node)2606 void InstructionSelector::VisitOsrValue(Node* node) {
2607   OperandGenerator g(this);
2608   int index = OsrValueIndexOf(node->op());
2609   Emit(kArchNop,
2610        g.DefineAsLocation(node, linkage()->GetOsrValueLocation(index)));
2611 }
2612 
2613 
VisitPhi(Node * node)2614 void InstructionSelector::VisitPhi(Node* node) {
2615   const int input_count = node->op()->ValueInputCount();
2616   DCHECK_EQ(input_count, current_block_->PredecessorCount());
2617   PhiInstruction* phi = new (instruction_zone())
2618       PhiInstruction(instruction_zone(), GetVirtualRegister(node),
2619                      static_cast<size_t>(input_count));
2620   sequence()
2621       ->InstructionBlockAt(RpoNumber::FromInt(current_block_->rpo_number()))
2622       ->AddPhi(phi);
2623   for (int i = 0; i < input_count; ++i) {
2624     Node* const input = node->InputAt(i);
2625     MarkAsUsed(input);
2626     phi->SetInput(static_cast<size_t>(i), GetVirtualRegister(input));
2627   }
2628 }
2629 
2630 
VisitProjection(Node * node)2631 void InstructionSelector::VisitProjection(Node* node) {
2632   OperandGenerator g(this);
2633   Node* value = node->InputAt(0);
2634   switch (value->opcode()) {
2635     case IrOpcode::kInt32AddWithOverflow:
2636     case IrOpcode::kInt32SubWithOverflow:
2637     case IrOpcode::kInt32MulWithOverflow:
2638     case IrOpcode::kInt64AddWithOverflow:
2639     case IrOpcode::kInt64SubWithOverflow:
2640     case IrOpcode::kTryTruncateFloat32ToInt64:
2641     case IrOpcode::kTryTruncateFloat64ToInt64:
2642     case IrOpcode::kTryTruncateFloat32ToUint64:
2643     case IrOpcode::kTryTruncateFloat64ToUint64:
2644     case IrOpcode::kInt32PairAdd:
2645     case IrOpcode::kInt32PairSub:
2646     case IrOpcode::kInt32PairMul:
2647     case IrOpcode::kWord32PairShl:
2648     case IrOpcode::kWord32PairShr:
2649     case IrOpcode::kWord32PairSar:
2650     case IrOpcode::kInt32AbsWithOverflow:
2651     case IrOpcode::kInt64AbsWithOverflow:
2652       if (ProjectionIndexOf(node->op()) == 0u) {
2653         Emit(kArchNop, g.DefineSameAsFirst(node), g.Use(value));
2654       } else {
2655         DCHECK_EQ(1u, ProjectionIndexOf(node->op()));
2656         MarkAsUsed(value);
2657       }
2658       break;
2659     default:
2660       break;
2661   }
2662 }
2663 
2664 
VisitConstant(Node * node)2665 void InstructionSelector::VisitConstant(Node* node) {
2666   // We must emit a NOP here because every live range needs a defining
2667   // instruction in the register allocator.
2668   OperandGenerator g(this);
2669   Emit(kArchNop, g.DefineAsConstant(node));
2670 }
2671 
2672 
VisitCall(Node * node,BasicBlock * handler)2673 void InstructionSelector::VisitCall(Node* node, BasicBlock* handler) {
2674   OperandGenerator g(this);
2675   auto call_descriptor = CallDescriptorOf(node->op());
2676 
2677   FrameStateDescriptor* frame_state_descriptor = nullptr;
2678   if (call_descriptor->NeedsFrameState()) {
2679     frame_state_descriptor = GetFrameStateDescriptor(
2680         node->InputAt(static_cast<int>(call_descriptor->InputCount())));
2681   }
2682 
2683   CallBuffer buffer(zone(), call_descriptor, frame_state_descriptor);
2684 
2685   // Compute InstructionOperands for inputs and outputs.
2686   // TODO(turbofan): on some architectures it's probably better to use
2687   // the code object in a register if there are multiple uses of it.
2688   // Improve constant pool and the heuristics in the register allocator
2689   // for where to emit constants.
2690   CallBufferFlags call_buffer_flags(kCallCodeImmediate | kCallAddressImmediate);
2691   InitializeCallBuffer(node, &buffer, call_buffer_flags, false);
2692 
2693   EmitPrepareArguments(&(buffer.pushed_nodes), call_descriptor, node);
2694 
2695   // Pass label of exception handler block.
2696   CallDescriptor::Flags flags = call_descriptor->flags();
2697   if (handler) {
2698     DCHECK_EQ(IrOpcode::kIfException, handler->front()->opcode());
2699     flags |= CallDescriptor::kHasExceptionHandler;
2700     buffer.instruction_args.push_back(g.Label(handler));
2701   }
2702 
2703   // Select the appropriate opcode based on the call type.
2704   InstructionCode opcode = kArchNop;
2705   switch (call_descriptor->kind()) {
2706     case CallDescriptor::kCallAddress:
2707       opcode = kArchCallCFunction | MiscField::encode(static_cast<int>(
2708                                         call_descriptor->ParameterCount()));
2709       break;
2710     case CallDescriptor::kCallCodeObject:
2711       opcode = kArchCallCodeObject | MiscField::encode(flags);
2712       break;
2713     case CallDescriptor::kCallJSFunction:
2714       opcode = kArchCallJSFunction | MiscField::encode(flags);
2715       break;
2716     case CallDescriptor::kCallWasmFunction:
2717       opcode = kArchCallWasmFunction | MiscField::encode(flags);
2718       break;
2719   }
2720 
2721   // Emit the call instruction.
2722   size_t const output_count = buffer.outputs.size();
2723   auto* outputs = output_count ? &buffer.outputs.front() : nullptr;
2724   Instruction* call_instr =
2725       Emit(opcode, output_count, outputs, buffer.instruction_args.size(),
2726            &buffer.instruction_args.front());
2727   if (instruction_selection_failed()) return;
2728   call_instr->MarkAsCall();
2729 
2730   EmitPrepareResults(&(buffer.output_nodes), call_descriptor, node);
2731 }
2732 
VisitCallWithCallerSavedRegisters(Node * node,BasicBlock * handler)2733 void InstructionSelector::VisitCallWithCallerSavedRegisters(
2734     Node* node, BasicBlock* handler) {
2735   OperandGenerator g(this);
2736   const auto fp_mode = CallDescriptorOf(node->op())->get_save_fp_mode();
2737   Emit(kArchSaveCallerRegisters | MiscField::encode(static_cast<int>(fp_mode)),
2738        g.NoOutput());
2739   VisitCall(node, handler);
2740   Emit(kArchRestoreCallerRegisters |
2741            MiscField::encode(static_cast<int>(fp_mode)),
2742        g.NoOutput());
2743 }
2744 
VisitTailCall(Node * node)2745 void InstructionSelector::VisitTailCall(Node* node) {
2746   OperandGenerator g(this);
2747   auto call_descriptor = CallDescriptorOf(node->op());
2748 
2749   CallDescriptor* caller = linkage()->GetIncomingDescriptor();
2750   DCHECK(caller->CanTailCall(node));
2751   const CallDescriptor* callee = CallDescriptorOf(node->op());
2752   int stack_param_delta = callee->GetStackParameterDelta(caller);
2753   CallBuffer buffer(zone(), call_descriptor, nullptr);
2754 
2755   // Compute InstructionOperands for inputs and outputs.
2756   CallBufferFlags flags(kCallCodeImmediate | kCallTail);
2757   if (IsTailCallAddressImmediate()) {
2758     flags |= kCallAddressImmediate;
2759   }
2760   if (callee->flags() & CallDescriptor::kFixedTargetRegister) {
2761     flags |= kCallFixedTargetRegister;
2762   }
2763   InitializeCallBuffer(node, &buffer, flags, true, stack_param_delta);
2764 
2765   // Select the appropriate opcode based on the call type.
2766   InstructionCode opcode;
2767   InstructionOperandVector temps(zone());
2768   if (linkage()->GetIncomingDescriptor()->IsJSFunctionCall()) {
2769     switch (call_descriptor->kind()) {
2770       case CallDescriptor::kCallCodeObject:
2771         opcode = kArchTailCallCodeObjectFromJSFunction;
2772         break;
2773       default:
2774         UNREACHABLE();
2775         return;
2776     }
2777     int temps_count = GetTempsCountForTailCallFromJSFunction();
2778     for (int i = 0; i < temps_count; i++) {
2779       temps.push_back(g.TempRegister());
2780     }
2781   } else {
2782     switch (call_descriptor->kind()) {
2783       case CallDescriptor::kCallCodeObject:
2784         opcode = kArchTailCallCodeObject;
2785         break;
2786       case CallDescriptor::kCallAddress:
2787         opcode = kArchTailCallAddress;
2788         break;
2789       case CallDescriptor::kCallWasmFunction:
2790         opcode = kArchTailCallWasm;
2791         break;
2792       default:
2793         UNREACHABLE();
2794         return;
2795     }
2796   }
2797   opcode |= MiscField::encode(call_descriptor->flags());
2798 
2799   Emit(kArchPrepareTailCall, g.NoOutput());
2800 
2801   // Add an immediate operand that represents the first slot that is unused
2802   // with respect to the stack pointer that has been updated for the tail call
2803   // instruction. This is used by backends that need to pad arguments for stack
2804   // alignment, in order to store an optional slot of padding above the
2805   // arguments.
2806   int optional_padding_slot = callee->GetFirstUnusedStackSlot();
2807   buffer.instruction_args.push_back(g.TempImmediate(optional_padding_slot));
2808 
2809   int first_unused_stack_slot =
2810       (V8_TARGET_ARCH_STORES_RETURN_ADDRESS_ON_STACK ? 1 : 0) +
2811       stack_param_delta;
2812   buffer.instruction_args.push_back(g.TempImmediate(first_unused_stack_slot));
2813 
2814   // Emit the tailcall instruction.
2815   Emit(opcode, 0, nullptr, buffer.instruction_args.size(),
2816        &buffer.instruction_args.front(), temps.size(),
2817        temps.empty() ? nullptr : &temps.front());
2818 }
2819 
2820 
VisitGoto(BasicBlock * target)2821 void InstructionSelector::VisitGoto(BasicBlock* target) {
2822   // jump to the next block.
2823   OperandGenerator g(this);
2824   Emit(kArchJmp, g.NoOutput(), g.Label(target));
2825 }
2826 
VisitReturn(Node * ret)2827 void InstructionSelector::VisitReturn(Node* ret) {
2828   OperandGenerator g(this);
2829   const int input_count = linkage()->GetIncomingDescriptor()->ReturnCount() == 0
2830                               ? 1
2831                               : ret->op()->ValueInputCount();
2832   DCHECK_GE(input_count, 1);
2833   auto value_locations = zone()->NewArray<InstructionOperand>(input_count);
2834   Node* pop_count = ret->InputAt(0);
2835   value_locations[0] = (pop_count->opcode() == IrOpcode::kInt32Constant ||
2836                         pop_count->opcode() == IrOpcode::kInt64Constant)
2837                            ? g.UseImmediate(pop_count)
2838                            : g.UseRegister(pop_count);
2839   for (int i = 1; i < input_count; ++i) {
2840     value_locations[i] =
2841         g.UseLocation(ret->InputAt(i), linkage()->GetReturnLocation(i - 1));
2842   }
2843   Emit(kArchRet, 0, nullptr, input_count, value_locations);
2844 }
2845 
VisitBranch(Node * branch,BasicBlock * tbranch,BasicBlock * fbranch)2846 void InstructionSelector::VisitBranch(Node* branch, BasicBlock* tbranch,
2847                                       BasicBlock* fbranch) {
2848   if (NeedsPoisoning(IsSafetyCheckOf(branch->op()))) {
2849     FlagsContinuation cont =
2850         FlagsContinuation::ForBranchAndPoison(kNotEqual, tbranch, fbranch);
2851     VisitWordCompareZero(branch, branch->InputAt(0), &cont);
2852   } else {
2853     FlagsContinuation cont =
2854         FlagsContinuation::ForBranch(kNotEqual, tbranch, fbranch);
2855     VisitWordCompareZero(branch, branch->InputAt(0), &cont);
2856   }
2857 }
2858 
VisitDeoptimizeIf(Node * node)2859 void InstructionSelector::VisitDeoptimizeIf(Node* node) {
2860   DeoptimizeParameters p = DeoptimizeParametersOf(node->op());
2861   if (NeedsPoisoning(p.is_safety_check())) {
2862     FlagsContinuation cont = FlagsContinuation::ForDeoptimizeAndPoison(
2863         kNotEqual, p.kind(), p.reason(), p.feedback(), node->InputAt(1));
2864     VisitWordCompareZero(node, node->InputAt(0), &cont);
2865   } else {
2866     FlagsContinuation cont = FlagsContinuation::ForDeoptimize(
2867         kNotEqual, p.kind(), p.reason(), p.feedback(), node->InputAt(1));
2868     VisitWordCompareZero(node, node->InputAt(0), &cont);
2869   }
2870 }
2871 
VisitDeoptimizeUnless(Node * node)2872 void InstructionSelector::VisitDeoptimizeUnless(Node* node) {
2873   DeoptimizeParameters p = DeoptimizeParametersOf(node->op());
2874   if (NeedsPoisoning(p.is_safety_check())) {
2875     FlagsContinuation cont = FlagsContinuation::ForDeoptimizeAndPoison(
2876         kEqual, p.kind(), p.reason(), p.feedback(), node->InputAt(1));
2877     VisitWordCompareZero(node, node->InputAt(0), &cont);
2878   } else {
2879     FlagsContinuation cont = FlagsContinuation::ForDeoptimize(
2880         kEqual, p.kind(), p.reason(), p.feedback(), node->InputAt(1));
2881     VisitWordCompareZero(node, node->InputAt(0), &cont);
2882   }
2883 }
2884 
VisitTrapIf(Node * node,TrapId trap_id)2885 void InstructionSelector::VisitTrapIf(Node* node, TrapId trap_id) {
2886   FlagsContinuation cont =
2887       FlagsContinuation::ForTrap(kNotEqual, trap_id, node->InputAt(1));
2888   VisitWordCompareZero(node, node->InputAt(0), &cont);
2889 }
2890 
VisitTrapUnless(Node * node,TrapId trap_id)2891 void InstructionSelector::VisitTrapUnless(Node* node, TrapId trap_id) {
2892   FlagsContinuation cont =
2893       FlagsContinuation::ForTrap(kEqual, trap_id, node->InputAt(1));
2894   VisitWordCompareZero(node, node->InputAt(0), &cont);
2895 }
2896 
EmitIdentity(Node * node)2897 void InstructionSelector::EmitIdentity(Node* node) {
2898   OperandGenerator g(this);
2899   MarkAsUsed(node->InputAt(0));
2900   SetRename(node, node->InputAt(0));
2901 }
2902 
VisitDeoptimize(DeoptimizeKind kind,DeoptimizeReason reason,VectorSlotPair const & feedback,Node * value)2903 void InstructionSelector::VisitDeoptimize(DeoptimizeKind kind,
2904                                           DeoptimizeReason reason,
2905                                           VectorSlotPair const& feedback,
2906                                           Node* value) {
2907   EmitDeoptimize(kArchDeoptimize, 0, nullptr, 0, nullptr, kind, reason,
2908                  feedback, value);
2909 }
2910 
VisitThrow(Node * node)2911 void InstructionSelector::VisitThrow(Node* node) {
2912   OperandGenerator g(this);
2913   Emit(kArchThrowTerminator, g.NoOutput());
2914 }
2915 
VisitDebugBreak(Node * node)2916 void InstructionSelector::VisitDebugBreak(Node* node) {
2917   OperandGenerator g(this);
2918   Emit(kArchDebugBreak, g.NoOutput());
2919 }
2920 
VisitUnreachable(Node * node)2921 void InstructionSelector::VisitUnreachable(Node* node) {
2922   OperandGenerator g(this);
2923   Emit(kArchDebugBreak, g.NoOutput());
2924 }
2925 
VisitDeadValue(Node * node)2926 void InstructionSelector::VisitDeadValue(Node* node) {
2927   OperandGenerator g(this);
2928   MarkAsRepresentation(DeadValueRepresentationOf(node->op()), node);
2929   Emit(kArchDebugBreak, g.DefineAsConstant(node));
2930 }
2931 
VisitComment(Node * node)2932 void InstructionSelector::VisitComment(Node* node) {
2933   OperandGenerator g(this);
2934   InstructionOperand operand(g.UseImmediate(node));
2935   Emit(kArchComment, 0, nullptr, 1, &operand);
2936 }
2937 
VisitUnsafePointerAdd(Node * node)2938 void InstructionSelector::VisitUnsafePointerAdd(Node* node) {
2939 #if V8_TARGET_ARCH_64_BIT
2940   VisitInt64Add(node);
2941 #else   // V8_TARGET_ARCH_64_BIT
2942   VisitInt32Add(node);
2943 #endif  // V8_TARGET_ARCH_64_BIT
2944 }
2945 
VisitRetain(Node * node)2946 void InstructionSelector::VisitRetain(Node* node) {
2947   OperandGenerator g(this);
2948   Emit(kArchNop, g.NoOutput(), g.UseAny(node->InputAt(0)));
2949 }
2950 
CanProduceSignalingNaN(Node * node)2951 bool InstructionSelector::CanProduceSignalingNaN(Node* node) {
2952   // TODO(jarin) Improve the heuristic here.
2953   if (node->opcode() == IrOpcode::kFloat64Add ||
2954       node->opcode() == IrOpcode::kFloat64Sub ||
2955       node->opcode() == IrOpcode::kFloat64Mul) {
2956     return false;
2957   }
2958   return true;
2959 }
2960 
GetFrameStateDescriptor(Node * state)2961 FrameStateDescriptor* InstructionSelector::GetFrameStateDescriptor(
2962     Node* state) {
2963   DCHECK_EQ(IrOpcode::kFrameState, state->opcode());
2964   DCHECK_EQ(kFrameStateInputCount, state->InputCount());
2965   FrameStateInfo state_info = FrameStateInfoOf(state->op());
2966 
2967   int parameters = static_cast<int>(
2968       StateValuesAccess(state->InputAt(kFrameStateParametersInput)).size());
2969   int locals = static_cast<int>(
2970       StateValuesAccess(state->InputAt(kFrameStateLocalsInput)).size());
2971   int stack = static_cast<int>(
2972       StateValuesAccess(state->InputAt(kFrameStateStackInput)).size());
2973 
2974   DCHECK_EQ(parameters, state_info.parameter_count());
2975   DCHECK_EQ(locals, state_info.local_count());
2976 
2977   FrameStateDescriptor* outer_state = nullptr;
2978   Node* outer_node = state->InputAt(kFrameStateOuterStateInput);
2979   if (outer_node->opcode() == IrOpcode::kFrameState) {
2980     outer_state = GetFrameStateDescriptor(outer_node);
2981   }
2982 
2983   return new (instruction_zone()) FrameStateDescriptor(
2984       instruction_zone(), state_info.type(), state_info.bailout_id(),
2985       state_info.state_combine(), parameters, locals, stack,
2986       state_info.shared_info(), outer_state);
2987 }
2988 
2989 // static
CanonicalizeShuffle(bool inputs_equal,uint8_t * shuffle,bool * needs_swap,bool * is_swizzle)2990 void InstructionSelector::CanonicalizeShuffle(bool inputs_equal,
2991                                               uint8_t* shuffle,
2992                                               bool* needs_swap,
2993                                               bool* is_swizzle) {
2994   *needs_swap = false;
2995   // Inputs equal, then it's a swizzle.
2996   if (inputs_equal) {
2997     *is_swizzle = true;
2998   } else {
2999     // Inputs are distinct; check that both are required.
3000     bool src0_is_used = false;
3001     bool src1_is_used = false;
3002     for (int i = 0; i < kSimd128Size; ++i) {
3003       if (shuffle[i] < kSimd128Size) {
3004         src0_is_used = true;
3005       } else {
3006         src1_is_used = true;
3007       }
3008     }
3009     if (src0_is_used && !src1_is_used) {
3010       *is_swizzle = true;
3011     } else if (src1_is_used && !src0_is_used) {
3012       *needs_swap = true;
3013       *is_swizzle = true;
3014     } else {
3015       *is_swizzle = false;
3016       // Canonicalize general 2 input shuffles so that the first input lanes are
3017       // encountered first. This makes architectural shuffle pattern matching
3018       // easier, since we only need to consider 1 input ordering instead of 2.
3019       if (shuffle[0] >= kSimd128Size) {
3020         // The second operand is used first. Swap inputs and adjust the shuffle.
3021         *needs_swap = true;
3022         for (int i = 0; i < kSimd128Size; ++i) {
3023           shuffle[i] ^= kSimd128Size;
3024         }
3025       }
3026     }
3027   }
3028   if (*is_swizzle) {
3029     for (int i = 0; i < kSimd128Size; ++i) shuffle[i] &= kSimd128Size - 1;
3030   }
3031 }
3032 
CanonicalizeShuffle(Node * node,uint8_t * shuffle,bool * is_swizzle)3033 void InstructionSelector::CanonicalizeShuffle(Node* node, uint8_t* shuffle,
3034                                               bool* is_swizzle) {
3035   // Get raw shuffle indices.
3036   memcpy(shuffle, OpParameter<uint8_t*>(node->op()), kSimd128Size);
3037   bool needs_swap;
3038   bool inputs_equal = GetVirtualRegister(node->InputAt(0)) ==
3039                       GetVirtualRegister(node->InputAt(1));
3040   CanonicalizeShuffle(inputs_equal, shuffle, &needs_swap, is_swizzle);
3041   if (needs_swap) {
3042     SwapShuffleInputs(node);
3043   }
3044   // Duplicate the first input; for some shuffles on some architectures, it's
3045   // easiest to implement a swizzle as a shuffle so it might be used.
3046   if (*is_swizzle) {
3047     node->ReplaceInput(1, node->InputAt(0));
3048   }
3049 }
3050 
3051 // static
SwapShuffleInputs(Node * node)3052 void InstructionSelector::SwapShuffleInputs(Node* node) {
3053   Node* input0 = node->InputAt(0);
3054   Node* input1 = node->InputAt(1);
3055   node->ReplaceInput(0, input1);
3056   node->ReplaceInput(1, input0);
3057 }
3058 
3059 // static
TryMatchIdentity(const uint8_t * shuffle)3060 bool InstructionSelector::TryMatchIdentity(const uint8_t* shuffle) {
3061   for (int i = 0; i < kSimd128Size; ++i) {
3062     if (shuffle[i] != i) return false;
3063   }
3064   return true;
3065 }
3066 
3067 // static
TryMatch32x4Shuffle(const uint8_t * shuffle,uint8_t * shuffle32x4)3068 bool InstructionSelector::TryMatch32x4Shuffle(const uint8_t* shuffle,
3069                                               uint8_t* shuffle32x4) {
3070   for (int i = 0; i < 4; ++i) {
3071     if (shuffle[i * 4] % 4 != 0) return false;
3072     for (int j = 1; j < 4; ++j) {
3073       if (shuffle[i * 4 + j] - shuffle[i * 4 + j - 1] != 1) return false;
3074     }
3075     shuffle32x4[i] = shuffle[i * 4] / 4;
3076   }
3077   return true;
3078 }
3079 
3080 // static
TryMatch16x8Shuffle(const uint8_t * shuffle,uint8_t * shuffle16x8)3081 bool InstructionSelector::TryMatch16x8Shuffle(const uint8_t* shuffle,
3082                                               uint8_t* shuffle16x8) {
3083   for (int i = 0; i < 8; ++i) {
3084     if (shuffle[i * 2] % 2 != 0) return false;
3085     for (int j = 1; j < 2; ++j) {
3086       if (shuffle[i * 2 + j] - shuffle[i * 2 + j - 1] != 1) return false;
3087     }
3088     shuffle16x8[i] = shuffle[i * 2] / 2;
3089   }
3090   return true;
3091 }
3092 
3093 // static
TryMatchConcat(const uint8_t * shuffle,uint8_t * offset)3094 bool InstructionSelector::TryMatchConcat(const uint8_t* shuffle,
3095                                          uint8_t* offset) {
3096   // Don't match the identity shuffle (e.g. [0 1 2 ... 15]).
3097   uint8_t start = shuffle[0];
3098   if (start == 0) return false;
3099   DCHECK_GT(kSimd128Size, start);  // The shuffle should be canonicalized.
3100   // A concatenation is a series of consecutive indices, with at most one jump
3101   // in the middle from the last lane to the first.
3102   for (int i = 1; i < kSimd128Size; ++i) {
3103     if ((shuffle[i]) != ((shuffle[i - 1] + 1))) {
3104       if (shuffle[i - 1] != 15) return false;
3105       if (shuffle[i] % kSimd128Size != 0) return false;
3106     }
3107   }
3108   *offset = start;
3109   return true;
3110 }
3111 
3112 // static
TryMatchBlend(const uint8_t * shuffle)3113 bool InstructionSelector::TryMatchBlend(const uint8_t* shuffle) {
3114   for (int i = 0; i < 16; ++i) {
3115     if ((shuffle[i] & 0xF) != i) return false;
3116   }
3117   return true;
3118 }
3119 
3120 // static
Pack4Lanes(const uint8_t * shuffle)3121 int32_t InstructionSelector::Pack4Lanes(const uint8_t* shuffle) {
3122   int32_t result = 0;
3123   for (int i = 3; i >= 0; --i) {
3124     result <<= 8;
3125     result |= shuffle[i];
3126   }
3127   return result;
3128 }
3129 
NeedsPoisoning(IsSafetyCheck safety_check) const3130 bool InstructionSelector::NeedsPoisoning(IsSafetyCheck safety_check) const {
3131   switch (poisoning_level_) {
3132     case PoisoningMitigationLevel::kDontPoison:
3133       return false;
3134     case PoisoningMitigationLevel::kPoisonAll:
3135       return safety_check != IsSafetyCheck::kNoSafetyCheck;
3136     case PoisoningMitigationLevel::kPoisonCriticalOnly:
3137       return safety_check == IsSafetyCheck::kCriticalSafetyCheck;
3138   }
3139   UNREACHABLE();
3140 }
3141 
3142 }  // namespace compiler
3143 }  // namespace internal
3144 }  // namespace v8
3145