1 // Copyright 2016 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "src/compiler/simd-scalar-lowering.h"
6
7 #include "src/compiler/diamond.h"
8 #include "src/compiler/linkage.h"
9 #include "src/compiler/node-matchers.h"
10 #include "src/compiler/node-properties.h"
11 #include "src/compiler/node.h"
12 #include "src/compiler/wasm-compiler.h"
13
14 namespace v8 {
15 namespace internal {
16 namespace compiler {
17
18 namespace {
19 static const int kNumLanes32 = 4;
20 static const int kNumLanes16 = 8;
21 static const int kNumLanes8 = 16;
22 static const int32_t kMask16 = 0xFFFF;
23 static const int32_t kMask8 = 0xFF;
24 static const int32_t kShift16 = 16;
25 static const int32_t kShift8 = 24;
26 } // anonymous
27
SimdScalarLowering(MachineGraph * mcgraph,Signature<MachineRepresentation> * signature)28 SimdScalarLowering::SimdScalarLowering(
29 MachineGraph* mcgraph, Signature<MachineRepresentation>* signature)
30 : mcgraph_(mcgraph),
31 state_(mcgraph->graph(), 3),
32 stack_(mcgraph_->zone()),
33 replacements_(nullptr),
34 signature_(signature),
35 placeholder_(graph()->NewNode(common()->Parameter(-2, "placeholder"),
36 graph()->start())),
37 parameter_count_after_lowering_(-1) {
38 DCHECK_NOT_NULL(graph());
39 DCHECK_NOT_NULL(graph()->end());
40 replacements_ = zone()->NewArray<Replacement>(graph()->NodeCount());
41 memset(replacements_, 0, sizeof(Replacement) * graph()->NodeCount());
42 }
43
LowerGraph()44 void SimdScalarLowering::LowerGraph() {
45 stack_.push_back({graph()->end(), 0});
46 state_.Set(graph()->end(), State::kOnStack);
47 replacements_[graph()->end()->id()].type = SimdType::kInt32x4;
48
49 while (!stack_.empty()) {
50 NodeState& top = stack_.back();
51 if (top.input_index == top.node->InputCount()) {
52 // All inputs of top have already been lowered, now lower top.
53 stack_.pop_back();
54 state_.Set(top.node, State::kVisited);
55 LowerNode(top.node);
56 } else {
57 // Push the next input onto the stack.
58 Node* input = top.node->InputAt(top.input_index++);
59 if (state_.Get(input) == State::kUnvisited) {
60 SetLoweredType(input, top.node);
61 if (input->opcode() == IrOpcode::kPhi) {
62 // To break cycles with phi nodes we push phis on a separate stack so
63 // that they are processed after all other nodes.
64 PreparePhiReplacement(input);
65 stack_.push_front({input, 0});
66 } else if (input->opcode() == IrOpcode::kEffectPhi ||
67 input->opcode() == IrOpcode::kLoop) {
68 stack_.push_front({input, 0});
69 } else {
70 stack_.push_back({input, 0});
71 }
72 state_.Set(input, State::kOnStack);
73 }
74 }
75 }
76 }
77
78 #define FOREACH_INT32X4_OPCODE(V) \
79 V(I32x4Splat) \
80 V(I32x4ExtractLane) \
81 V(I32x4ReplaceLane) \
82 V(I32x4SConvertF32x4) \
83 V(I32x4UConvertF32x4) \
84 V(I32x4SConvertI16x8Low) \
85 V(I32x4SConvertI16x8High) \
86 V(I32x4Neg) \
87 V(I32x4Shl) \
88 V(I32x4ShrS) \
89 V(I32x4Add) \
90 V(I32x4AddHoriz) \
91 V(I32x4Sub) \
92 V(I32x4Mul) \
93 V(I32x4MinS) \
94 V(I32x4MaxS) \
95 V(I32x4ShrU) \
96 V(I32x4MinU) \
97 V(I32x4MaxU) \
98 V(I32x4Eq) \
99 V(I32x4Ne) \
100 V(I32x4LtS) \
101 V(I32x4LeS) \
102 V(I32x4GtS) \
103 V(I32x4GeS) \
104 V(I32x4UConvertI16x8Low) \
105 V(I32x4UConvertI16x8High) \
106 V(I32x4LtU) \
107 V(I32x4LeU) \
108 V(I32x4GtU) \
109 V(I32x4GeU) \
110 V(S128And) \
111 V(S128Or) \
112 V(S128Xor) \
113 V(S128Not) \
114 V(S1x4AnyTrue) \
115 V(S1x4AllTrue) \
116 V(S1x8AnyTrue) \
117 V(S1x8AllTrue) \
118 V(S1x16AnyTrue) \
119 V(S1x16AllTrue)
120
121 #define FOREACH_FLOAT32X4_OPCODE(V) \
122 V(F32x4Splat) \
123 V(F32x4ExtractLane) \
124 V(F32x4ReplaceLane) \
125 V(F32x4SConvertI32x4) \
126 V(F32x4UConvertI32x4) \
127 V(F32x4Abs) \
128 V(F32x4Neg) \
129 V(F32x4RecipApprox) \
130 V(F32x4RecipSqrtApprox) \
131 V(F32x4Add) \
132 V(F32x4AddHoriz) \
133 V(F32x4Sub) \
134 V(F32x4Mul) \
135 V(F32x4Min) \
136 V(F32x4Max)
137
138 #define FOREACH_FLOAT32X4_TO_INT32X4OPCODE(V) \
139 V(F32x4Eq) \
140 V(F32x4Ne) \
141 V(F32x4Lt) \
142 V(F32x4Le) \
143 V(F32x4Gt) \
144 V(F32x4Ge)
145
146 #define FOREACH_INT16X8_OPCODE(V) \
147 V(I16x8Splat) \
148 V(I16x8ExtractLane) \
149 V(I16x8ReplaceLane) \
150 V(I16x8SConvertI8x16Low) \
151 V(I16x8SConvertI8x16High) \
152 V(I16x8Neg) \
153 V(I16x8Shl) \
154 V(I16x8ShrS) \
155 V(I16x8SConvertI32x4) \
156 V(I16x8Add) \
157 V(I16x8AddSaturateS) \
158 V(I16x8AddHoriz) \
159 V(I16x8Sub) \
160 V(I16x8SubSaturateS) \
161 V(I16x8Mul) \
162 V(I16x8MinS) \
163 V(I16x8MaxS) \
164 V(I16x8UConvertI8x16Low) \
165 V(I16x8UConvertI8x16High) \
166 V(I16x8ShrU) \
167 V(I16x8UConvertI32x4) \
168 V(I16x8AddSaturateU) \
169 V(I16x8SubSaturateU) \
170 V(I16x8MinU) \
171 V(I16x8MaxU) \
172 V(I16x8Eq) \
173 V(I16x8Ne) \
174 V(I16x8LtS) \
175 V(I16x8LeS) \
176 V(I16x8LtU) \
177 V(I16x8LeU)
178
179 #define FOREACH_INT8X16_OPCODE(V) \
180 V(I8x16Splat) \
181 V(I8x16ExtractLane) \
182 V(I8x16ReplaceLane) \
183 V(I8x16SConvertI16x8) \
184 V(I8x16Neg) \
185 V(I8x16Shl) \
186 V(I8x16ShrS) \
187 V(I8x16Add) \
188 V(I8x16AddSaturateS) \
189 V(I8x16Sub) \
190 V(I8x16SubSaturateS) \
191 V(I8x16Mul) \
192 V(I8x16MinS) \
193 V(I8x16MaxS) \
194 V(I8x16ShrU) \
195 V(I8x16UConvertI16x8) \
196 V(I8x16AddSaturateU) \
197 V(I8x16SubSaturateU) \
198 V(I8x16MinU) \
199 V(I8x16MaxU) \
200 V(I8x16Eq) \
201 V(I8x16Ne) \
202 V(I8x16LtS) \
203 V(I8x16LeS) \
204 V(I8x16LtU) \
205 V(I8x16LeU) \
206 V(S8x16Shuffle)
207
MachineTypeFrom(SimdType simdType)208 MachineType SimdScalarLowering::MachineTypeFrom(SimdType simdType) {
209 switch (simdType) {
210 case SimdType::kFloat32x4:
211 return MachineType::Float32();
212 case SimdType::kInt32x4:
213 return MachineType::Int32();
214 case SimdType::kInt16x8:
215 return MachineType::Int16();
216 case SimdType::kInt8x16:
217 return MachineType::Int8();
218 }
219 return MachineType::None();
220 }
221
SetLoweredType(Node * node,Node * output)222 void SimdScalarLowering::SetLoweredType(Node* node, Node* output) {
223 switch (node->opcode()) {
224 #define CASE_STMT(name) case IrOpcode::k##name:
225 FOREACH_INT32X4_OPCODE(CASE_STMT)
226 case IrOpcode::kReturn:
227 case IrOpcode::kParameter:
228 case IrOpcode::kCall: {
229 replacements_[node->id()].type = SimdType::kInt32x4;
230 break;
231 }
232 FOREACH_FLOAT32X4_OPCODE(CASE_STMT) {
233 replacements_[node->id()].type = SimdType::kFloat32x4;
234 break;
235 }
236 FOREACH_FLOAT32X4_TO_INT32X4OPCODE(CASE_STMT) {
237 replacements_[node->id()].type = SimdType::kInt32x4;
238 break;
239 }
240 FOREACH_INT16X8_OPCODE(CASE_STMT) {
241 replacements_[node->id()].type = SimdType::kInt16x8;
242 break;
243 }
244 FOREACH_INT8X16_OPCODE(CASE_STMT) {
245 replacements_[node->id()].type = SimdType::kInt8x16;
246 break;
247 }
248 default: {
249 switch (output->opcode()) {
250 case IrOpcode::kF32x4SConvertI32x4:
251 case IrOpcode::kF32x4UConvertI32x4:
252 case IrOpcode::kI16x8SConvertI32x4:
253 case IrOpcode::kI16x8UConvertI32x4: {
254 replacements_[node->id()].type = SimdType::kInt32x4;
255 break;
256 }
257 case IrOpcode::kI8x16SConvertI16x8:
258 case IrOpcode::kI8x16UConvertI16x8:
259 case IrOpcode::kI32x4SConvertI16x8Low:
260 case IrOpcode::kI32x4SConvertI16x8High:
261 case IrOpcode::kI32x4UConvertI16x8Low:
262 case IrOpcode::kI32x4UConvertI16x8High: {
263 replacements_[node->id()].type = SimdType::kInt16x8;
264 break;
265 }
266 case IrOpcode::kI16x8SConvertI8x16Low:
267 case IrOpcode::kI16x8SConvertI8x16High:
268 case IrOpcode::kI16x8UConvertI8x16Low:
269 case IrOpcode::kI16x8UConvertI8x16High: {
270 replacements_[node->id()].type = SimdType::kInt8x16;
271 break;
272 }
273 FOREACH_FLOAT32X4_TO_INT32X4OPCODE(CASE_STMT)
274 case IrOpcode::kI32x4SConvertF32x4:
275 case IrOpcode::kI32x4UConvertF32x4: {
276 replacements_[node->id()].type = SimdType::kFloat32x4;
277 break;
278 }
279 case IrOpcode::kS128Select: {
280 replacements_[node->id()].type = SimdType::kInt32x4;
281 break;
282 }
283 default: {
284 replacements_[node->id()].type = replacements_[output->id()].type;
285 }
286 }
287 }
288 #undef CASE_STMT
289 }
290 }
291
GetParameterIndexAfterLoweringSimd128(Signature<MachineRepresentation> * signature,int old_index)292 static int GetParameterIndexAfterLoweringSimd128(
293 Signature<MachineRepresentation>* signature, int old_index) {
294 // In function calls, the simd128 types are passed as 4 Int32 types. The
295 // parameters are typecast to the types as needed for various operations.
296 int result = old_index;
297 for (int i = 0; i < old_index; ++i) {
298 if (signature->GetParam(i) == MachineRepresentation::kSimd128) {
299 result += 3;
300 }
301 }
302 return result;
303 }
304
GetParameterCountAfterLowering()305 int SimdScalarLowering::GetParameterCountAfterLowering() {
306 if (parameter_count_after_lowering_ == -1) {
307 // GetParameterIndexAfterLoweringSimd128(parameter_count) returns the
308 // parameter count after lowering.
309 parameter_count_after_lowering_ = GetParameterIndexAfterLoweringSimd128(
310 signature(), static_cast<int>(signature()->parameter_count()));
311 }
312 return parameter_count_after_lowering_;
313 }
314
GetReturnCountAfterLoweringSimd128(Signature<MachineRepresentation> * signature)315 static int GetReturnCountAfterLoweringSimd128(
316 Signature<MachineRepresentation>* signature) {
317 int result = static_cast<int>(signature->return_count());
318 for (int i = 0; i < static_cast<int>(signature->return_count()); ++i) {
319 if (signature->GetReturn(i) == MachineRepresentation::kSimd128) {
320 result += 3;
321 }
322 }
323 return result;
324 }
325
NumLanes(SimdType type)326 int SimdScalarLowering::NumLanes(SimdType type) {
327 int num_lanes = 0;
328 if (type == SimdType::kFloat32x4 || type == SimdType::kInt32x4) {
329 num_lanes = kNumLanes32;
330 } else if (type == SimdType::kInt16x8) {
331 num_lanes = kNumLanes16;
332 } else if (type == SimdType::kInt8x16) {
333 num_lanes = kNumLanes8;
334 } else {
335 UNREACHABLE();
336 }
337 return num_lanes;
338 }
339
340 constexpr int SimdScalarLowering::kLaneOffsets[];
341
GetIndexNodes(Node * index,Node ** new_indices,SimdType type)342 void SimdScalarLowering::GetIndexNodes(Node* index, Node** new_indices,
343 SimdType type) {
344 int num_lanes = NumLanes(type);
345 int lane_width = kSimd128Size / num_lanes;
346 int laneIndex = kLaneOffsets[0] / lane_width;
347 new_indices[laneIndex] = index;
348 for (int i = 1; i < num_lanes; ++i) {
349 laneIndex = kLaneOffsets[i * lane_width] / lane_width;
350 new_indices[laneIndex] = graph()->NewNode(
351 machine()->Int32Add(), index,
352 graph()->NewNode(
353 common()->Int32Constant(static_cast<int>(i) * lane_width)));
354 }
355 }
356
LowerLoadOp(Node * node,SimdType type)357 void SimdScalarLowering::LowerLoadOp(Node* node, SimdType type) {
358 MachineRepresentation rep = LoadRepresentationOf(node->op()).representation();
359 const Operator* load_op;
360 switch (node->opcode()) {
361 case IrOpcode::kLoad:
362 load_op = machine()->Load(MachineTypeFrom(type));
363 break;
364 case IrOpcode::kUnalignedLoad:
365 load_op = machine()->UnalignedLoad(MachineTypeFrom(type));
366 break;
367 case IrOpcode::kProtectedLoad:
368 load_op = machine()->ProtectedLoad(MachineTypeFrom(type));
369 break;
370 default:
371 UNREACHABLE();
372 }
373 if (rep == MachineRepresentation::kSimd128) {
374 Node* base = node->InputAt(0);
375 Node* index = node->InputAt(1);
376 int num_lanes = NumLanes(type);
377 Node** indices = zone()->NewArray<Node*>(num_lanes);
378 GetIndexNodes(index, indices, type);
379 Node** rep_nodes = zone()->NewArray<Node*>(num_lanes);
380 rep_nodes[0] = node;
381 rep_nodes[0]->ReplaceInput(1, indices[0]);
382 NodeProperties::ChangeOp(rep_nodes[0], load_op);
383 if (node->InputCount() > 2) {
384 DCHECK_LT(3, node->InputCount());
385 Node* effect_input = node->InputAt(2);
386 Node* control_input = node->InputAt(3);
387 for (int i = num_lanes - 1; i > 0; --i) {
388 rep_nodes[i] = graph()->NewNode(load_op, base, indices[i], effect_input,
389 control_input);
390 effect_input = rep_nodes[i];
391 }
392 rep_nodes[0]->ReplaceInput(2, rep_nodes[1]);
393 } else {
394 for (int i = 1; i < num_lanes; ++i) {
395 rep_nodes[i] = graph()->NewNode(load_op, base, indices[i]);
396 }
397 }
398 ReplaceNode(node, rep_nodes, num_lanes);
399 } else {
400 DefaultLowering(node);
401 }
402 }
403
LowerStoreOp(Node * node)404 void SimdScalarLowering::LowerStoreOp(Node* node) {
405 // For store operation, use replacement type of its input instead of the
406 // one of its effected node.
407 DCHECK_LT(2, node->InputCount());
408 SimdType rep_type = ReplacementType(node->InputAt(2));
409 replacements_[node->id()].type = rep_type;
410 const Operator* store_op;
411 MachineRepresentation rep;
412 switch (node->opcode()) {
413 case IrOpcode::kStore: {
414 rep = StoreRepresentationOf(node->op()).representation();
415 WriteBarrierKind write_barrier_kind =
416 StoreRepresentationOf(node->op()).write_barrier_kind();
417 store_op = machine()->Store(StoreRepresentation(
418 MachineTypeFrom(rep_type).representation(), write_barrier_kind));
419 break;
420 }
421 case IrOpcode::kUnalignedStore: {
422 rep = UnalignedStoreRepresentationOf(node->op());
423 store_op =
424 machine()->UnalignedStore(MachineTypeFrom(rep_type).representation());
425 break;
426 }
427 case IrOpcode::kProtectedStore: {
428 rep = StoreRepresentationOf(node->op()).representation();
429 store_op =
430 machine()->ProtectedStore(MachineTypeFrom(rep_type).representation());
431 break;
432 }
433 default:
434 UNREACHABLE();
435 }
436 if (rep == MachineRepresentation::kSimd128) {
437 Node* base = node->InputAt(0);
438 Node* index = node->InputAt(1);
439 int num_lanes = NumLanes(rep_type);
440 Node** indices = zone()->NewArray<Node*>(num_lanes);
441 GetIndexNodes(index, indices, rep_type);
442 Node* value = node->InputAt(2);
443 DCHECK(HasReplacement(1, value));
444 Node** rep_nodes = zone()->NewArray<Node*>(num_lanes);
445 rep_nodes[0] = node;
446 Node** rep_inputs = GetReplacementsWithType(value, rep_type);
447 rep_nodes[0]->ReplaceInput(2, rep_inputs[0]);
448 rep_nodes[0]->ReplaceInput(1, indices[0]);
449 NodeProperties::ChangeOp(node, store_op);
450 if (node->InputCount() > 3) {
451 DCHECK_LT(4, node->InputCount());
452 Node* effect_input = node->InputAt(3);
453 Node* control_input = node->InputAt(4);
454 for (int i = num_lanes - 1; i > 0; --i) {
455 rep_nodes[i] =
456 graph()->NewNode(store_op, base, indices[i], rep_inputs[i],
457 effect_input, control_input);
458 effect_input = rep_nodes[i];
459 }
460 rep_nodes[0]->ReplaceInput(3, rep_nodes[1]);
461 } else {
462 for (int i = 1; i < num_lanes; ++i) {
463 rep_nodes[i] =
464 graph()->NewNode(store_op, base, indices[i], rep_inputs[i]);
465 }
466 }
467 ReplaceNode(node, rep_nodes, num_lanes);
468 } else {
469 DefaultLowering(node);
470 }
471 }
472
LowerBinaryOp(Node * node,SimdType input_rep_type,const Operator * op,bool not_horizontal)473 void SimdScalarLowering::LowerBinaryOp(Node* node, SimdType input_rep_type,
474 const Operator* op,
475 bool not_horizontal) {
476 DCHECK_EQ(2, node->InputCount());
477 Node** rep_left = GetReplacementsWithType(node->InputAt(0), input_rep_type);
478 Node** rep_right = GetReplacementsWithType(node->InputAt(1), input_rep_type);
479 int num_lanes = NumLanes(input_rep_type);
480 Node** rep_node = zone()->NewArray<Node*>(num_lanes);
481 if (not_horizontal) {
482 for (int i = 0; i < num_lanes; ++i) {
483 rep_node[i] = graph()->NewNode(op, rep_left[i], rep_right[i]);
484 }
485 } else {
486 for (int i = 0; i < num_lanes / 2; ++i) {
487 rep_node[i] = graph()->NewNode(op, rep_left[i * 2], rep_left[i * 2 + 1]);
488 rep_node[i + num_lanes / 2] =
489 graph()->NewNode(op, rep_right[i * 2], rep_right[i * 2 + 1]);
490 }
491 }
492 ReplaceNode(node, rep_node, num_lanes);
493 }
494
LowerCompareOp(Node * node,SimdType input_rep_type,const Operator * op,bool invert_inputs)495 void SimdScalarLowering::LowerCompareOp(Node* node, SimdType input_rep_type,
496 const Operator* op,
497 bool invert_inputs) {
498 DCHECK_EQ(2, node->InputCount());
499 Node** rep_left = GetReplacementsWithType(node->InputAt(0), input_rep_type);
500 Node** rep_right = GetReplacementsWithType(node->InputAt(1), input_rep_type);
501 int num_lanes = NumLanes(input_rep_type);
502 Node** rep_node = zone()->NewArray<Node*>(num_lanes);
503 for (int i = 0; i < num_lanes; ++i) {
504 Node* cmp_result = nullptr;
505 if (invert_inputs) {
506 cmp_result = graph()->NewNode(op, rep_right[i], rep_left[i]);
507 } else {
508 cmp_result = graph()->NewNode(op, rep_left[i], rep_right[i]);
509 }
510 Diamond d_cmp(graph(), common(),
511 graph()->NewNode(machine()->Word32Equal(), cmp_result,
512 mcgraph_->Int32Constant(0)));
513 MachineRepresentation rep =
514 (input_rep_type == SimdType::kFloat32x4)
515 ? MachineRepresentation::kWord32
516 : MachineTypeFrom(input_rep_type).representation();
517 rep_node[i] =
518 d_cmp.Phi(rep, mcgraph_->Int32Constant(0), mcgraph_->Int32Constant(-1));
519 }
520 ReplaceNode(node, rep_node, num_lanes);
521 }
522
FixUpperBits(Node * input,int32_t shift)523 Node* SimdScalarLowering::FixUpperBits(Node* input, int32_t shift) {
524 return graph()->NewNode(machine()->Word32Sar(),
525 graph()->NewNode(machine()->Word32Shl(), input,
526 mcgraph_->Int32Constant(shift)),
527 mcgraph_->Int32Constant(shift));
528 }
529
LowerBinaryOpForSmallInt(Node * node,SimdType input_rep_type,const Operator * op,bool not_horizontal)530 void SimdScalarLowering::LowerBinaryOpForSmallInt(Node* node,
531 SimdType input_rep_type,
532 const Operator* op,
533 bool not_horizontal) {
534 DCHECK_EQ(2, node->InputCount());
535 DCHECK(input_rep_type == SimdType::kInt16x8 ||
536 input_rep_type == SimdType::kInt8x16);
537 Node** rep_left = GetReplacementsWithType(node->InputAt(0), input_rep_type);
538 Node** rep_right = GetReplacementsWithType(node->InputAt(1), input_rep_type);
539 int num_lanes = NumLanes(input_rep_type);
540 Node** rep_node = zone()->NewArray<Node*>(num_lanes);
541 int32_t shift_val =
542 (input_rep_type == SimdType::kInt16x8) ? kShift16 : kShift8;
543 if (not_horizontal) {
544 for (int i = 0; i < num_lanes; ++i) {
545 rep_node[i] = FixUpperBits(
546 graph()->NewNode(op, rep_left[i], rep_right[i]), shift_val);
547 }
548 } else {
549 for (int i = 0; i < num_lanes / 2; ++i) {
550 rep_node[i] = FixUpperBits(
551 graph()->NewNode(op, rep_left[i * 2], rep_left[i * 2 + 1]),
552 shift_val);
553 rep_node[i + num_lanes / 2] = FixUpperBits(
554 graph()->NewNode(op, rep_right[i * 2], rep_right[i * 2 + 1]),
555 shift_val);
556 }
557 }
558 ReplaceNode(node, rep_node, num_lanes);
559 }
560
Mask(Node * input,int32_t mask)561 Node* SimdScalarLowering::Mask(Node* input, int32_t mask) {
562 return graph()->NewNode(machine()->Word32And(), input,
563 mcgraph_->Int32Constant(mask));
564 }
565
LowerSaturateBinaryOp(Node * node,SimdType input_rep_type,const Operator * op,bool is_signed)566 void SimdScalarLowering::LowerSaturateBinaryOp(Node* node,
567 SimdType input_rep_type,
568 const Operator* op,
569 bool is_signed) {
570 DCHECK_EQ(2, node->InputCount());
571 DCHECK(input_rep_type == SimdType::kInt16x8 ||
572 input_rep_type == SimdType::kInt8x16);
573 Node** rep_left = GetReplacementsWithType(node->InputAt(0), input_rep_type);
574 Node** rep_right = GetReplacementsWithType(node->InputAt(1), input_rep_type);
575 int32_t min = 0;
576 int32_t max = 0;
577 int32_t mask = 0;
578 int32_t shift_val = 0;
579 MachineRepresentation phi_rep;
580 if (input_rep_type == SimdType::kInt16x8) {
581 if (is_signed) {
582 min = std::numeric_limits<int16_t>::min();
583 max = std::numeric_limits<int16_t>::max();
584 } else {
585 min = std::numeric_limits<uint16_t>::min();
586 max = std::numeric_limits<uint16_t>::max();
587 }
588 mask = kMask16;
589 shift_val = kShift16;
590 phi_rep = MachineRepresentation::kWord16;
591 } else {
592 if (is_signed) {
593 min = std::numeric_limits<int8_t>::min();
594 max = std::numeric_limits<int8_t>::max();
595 } else {
596 min = std::numeric_limits<uint8_t>::min();
597 max = std::numeric_limits<uint8_t>::max();
598 }
599 mask = kMask8;
600 shift_val = kShift8;
601 phi_rep = MachineRepresentation::kWord8;
602 }
603 int num_lanes = NumLanes(input_rep_type);
604 Node** rep_node = zone()->NewArray<Node*>(num_lanes);
605 for (int i = 0; i < num_lanes; ++i) {
606 Node* op_result = nullptr;
607 Node* left = is_signed ? rep_left[i] : Mask(rep_left[i], mask);
608 Node* right = is_signed ? rep_right[i] : Mask(rep_right[i], mask);
609 op_result = graph()->NewNode(op, left, right);
610 Diamond d_min(graph(), common(),
611 graph()->NewNode(machine()->Int32LessThan(), op_result,
612 mcgraph_->Int32Constant(min)));
613 rep_node[i] = d_min.Phi(phi_rep, mcgraph_->Int32Constant(min), op_result);
614 Diamond d_max(graph(), common(),
615 graph()->NewNode(machine()->Int32LessThan(),
616 mcgraph_->Int32Constant(max), rep_node[i]));
617 rep_node[i] = d_max.Phi(phi_rep, mcgraph_->Int32Constant(max), rep_node[i]);
618 rep_node[i] =
619 is_signed ? rep_node[i] : FixUpperBits(rep_node[i], shift_val);
620 }
621 ReplaceNode(node, rep_node, num_lanes);
622 }
623
LowerUnaryOp(Node * node,SimdType input_rep_type,const Operator * op)624 void SimdScalarLowering::LowerUnaryOp(Node* node, SimdType input_rep_type,
625 const Operator* op) {
626 DCHECK_EQ(1, node->InputCount());
627 Node** rep = GetReplacementsWithType(node->InputAt(0), input_rep_type);
628 int num_lanes = NumLanes(input_rep_type);
629 Node** rep_node = zone()->NewArray<Node*>(num_lanes);
630 for (int i = 0; i < num_lanes; ++i) {
631 rep_node[i] = graph()->NewNode(op, rep[i]);
632 }
633 ReplaceNode(node, rep_node, num_lanes);
634 }
635
LowerIntMinMax(Node * node,const Operator * op,bool is_max,SimdType type)636 void SimdScalarLowering::LowerIntMinMax(Node* node, const Operator* op,
637 bool is_max, SimdType type) {
638 DCHECK_EQ(2, node->InputCount());
639 Node** rep_left = GetReplacementsWithType(node->InputAt(0), type);
640 Node** rep_right = GetReplacementsWithType(node->InputAt(1), type);
641 int num_lanes = NumLanes(type);
642 Node** rep_node = zone()->NewArray<Node*>(num_lanes);
643 MachineRepresentation rep = MachineRepresentation::kNone;
644 if (type == SimdType::kInt32x4) {
645 rep = MachineRepresentation::kWord32;
646 } else if (type == SimdType::kInt16x8) {
647 rep = MachineRepresentation::kWord16;
648 } else if (type == SimdType::kInt8x16) {
649 rep = MachineRepresentation::kWord8;
650 } else {
651 UNREACHABLE();
652 }
653 for (int i = 0; i < num_lanes; ++i) {
654 Diamond d(graph(), common(),
655 graph()->NewNode(op, rep_left[i], rep_right[i]));
656 if (is_max) {
657 rep_node[i] = d.Phi(rep, rep_right[i], rep_left[i]);
658 } else {
659 rep_node[i] = d.Phi(rep, rep_left[i], rep_right[i]);
660 }
661 }
662 ReplaceNode(node, rep_node, num_lanes);
663 }
664
BuildF64Trunc(Node * input)665 Node* SimdScalarLowering::BuildF64Trunc(Node* input) {
666 if (machine()->Float64RoundTruncate().IsSupported()) {
667 return graph()->NewNode(machine()->Float64RoundTruncate().op(), input);
668 } else {
669 ExternalReference ref = ExternalReference::wasm_f64_trunc();
670 Node* stack_slot =
671 graph()->NewNode(machine()->StackSlot(MachineRepresentation::kFloat64));
672 const Operator* store_op = machine()->Store(
673 StoreRepresentation(MachineRepresentation::kFloat64, kNoWriteBarrier));
674 Node* effect =
675 graph()->NewNode(store_op, stack_slot, mcgraph_->Int32Constant(0),
676 input, graph()->start(), graph()->start());
677 Node* function = graph()->NewNode(common()->ExternalConstant(ref));
678 Node** args = zone()->NewArray<Node*>(4);
679 args[0] = function;
680 args[1] = stack_slot;
681 args[2] = effect;
682 args[3] = graph()->start();
683 Signature<MachineType>::Builder sig_builder(zone(), 0, 1);
684 sig_builder.AddParam(MachineType::Pointer());
685 auto call_descriptor =
686 Linkage::GetSimplifiedCDescriptor(zone(), sig_builder.Build());
687 Node* call = graph()->NewNode(common()->Call(call_descriptor), 4, args);
688 return graph()->NewNode(machine()->Load(LoadRepresentation::Float64()),
689 stack_slot, mcgraph_->Int32Constant(0), call,
690 graph()->start());
691 }
692 }
693
LowerConvertFromFloat(Node * node,bool is_signed)694 void SimdScalarLowering::LowerConvertFromFloat(Node* node, bool is_signed) {
695 DCHECK_EQ(1, node->InputCount());
696 Node** rep = GetReplacementsWithType(node->InputAt(0), SimdType::kFloat32x4);
697 Node* rep_node[kNumLanes32];
698 Node* double_zero = graph()->NewNode(common()->Float64Constant(0.0));
699 Node* min = graph()->NewNode(
700 common()->Float64Constant(static_cast<double>(is_signed ? kMinInt : 0)));
701 Node* max = graph()->NewNode(common()->Float64Constant(
702 static_cast<double>(is_signed ? kMaxInt : 0xFFFFFFFFu)));
703 for (int i = 0; i < kNumLanes32; ++i) {
704 Node* double_rep =
705 graph()->NewNode(machine()->ChangeFloat32ToFloat64(), rep[i]);
706 Diamond nan_d(graph(), common(), graph()->NewNode(machine()->Float64Equal(),
707 double_rep, double_rep));
708 Node* temp =
709 nan_d.Phi(MachineRepresentation::kFloat64, double_rep, double_zero);
710 Diamond min_d(graph(), common(),
711 graph()->NewNode(machine()->Float64LessThan(), temp, min));
712 temp = min_d.Phi(MachineRepresentation::kFloat64, min, temp);
713 Diamond max_d(graph(), common(),
714 graph()->NewNode(machine()->Float64LessThan(), max, temp));
715 temp = max_d.Phi(MachineRepresentation::kFloat64, max, temp);
716 Node* trunc = BuildF64Trunc(temp);
717 if (is_signed) {
718 rep_node[i] = graph()->NewNode(machine()->ChangeFloat64ToInt32(), trunc);
719 } else {
720 rep_node[i] =
721 graph()->NewNode(machine()->TruncateFloat64ToUint32(), trunc);
722 }
723 }
724 ReplaceNode(node, rep_node, kNumLanes32);
725 }
726
LowerConvertFromInt(Node * node,SimdType input_rep_type,SimdType output_rep_type,bool is_signed,int start_index)727 void SimdScalarLowering::LowerConvertFromInt(Node* node,
728 SimdType input_rep_type,
729 SimdType output_rep_type,
730 bool is_signed, int start_index) {
731 DCHECK_EQ(1, node->InputCount());
732 Node** rep = GetReplacementsWithType(node->InputAt(0), input_rep_type);
733
734 int32_t mask = 0;
735 if (input_rep_type == SimdType::kInt16x8) {
736 DCHECK_EQ(output_rep_type, SimdType::kInt32x4);
737 mask = kMask16;
738 } else {
739 DCHECK_EQ(output_rep_type, SimdType::kInt16x8);
740 DCHECK_EQ(input_rep_type, SimdType::kInt8x16);
741 mask = kMask8;
742 }
743
744 int num_lanes = NumLanes(output_rep_type);
745 Node** rep_node = zone()->NewArray<Node*>(num_lanes);
746 for (int i = 0; i < num_lanes; ++i) {
747 rep_node[i] =
748 is_signed ? rep[i + start_index] : Mask(rep[i + start_index], mask);
749 }
750
751 ReplaceNode(node, rep_node, num_lanes);
752 }
753
LowerPack(Node * node,SimdType input_rep_type,SimdType output_rep_type,bool is_signed)754 void SimdScalarLowering::LowerPack(Node* node, SimdType input_rep_type,
755 SimdType output_rep_type, bool is_signed) {
756 DCHECK_EQ(2, node->InputCount());
757 Node** rep_left = GetReplacementsWithType(node->InputAt(0), input_rep_type);
758 Node** rep_right = GetReplacementsWithType(node->InputAt(1), input_rep_type);
759 const Operator* less_op =
760 is_signed ? machine()->Int32LessThan() : machine()->Uint32LessThan();
761 Node* min = nullptr;
762 Node* max = nullptr;
763 int32_t shift_val = 0;
764 MachineRepresentation phi_rep;
765 if (output_rep_type == SimdType::kInt16x8) {
766 DCHECK(input_rep_type == SimdType::kInt32x4);
767 if (is_signed) {
768 min = mcgraph_->Int32Constant(std::numeric_limits<int16_t>::min());
769 max = mcgraph_->Int32Constant(std::numeric_limits<int16_t>::max());
770 } else {
771 max = mcgraph_->Uint32Constant(std::numeric_limits<uint16_t>::max());
772 shift_val = kShift16;
773 }
774 phi_rep = MachineRepresentation::kWord16;
775 } else {
776 DCHECK(output_rep_type == SimdType::kInt8x16 &&
777 input_rep_type == SimdType::kInt16x8);
778 if (is_signed) {
779 min = mcgraph_->Int32Constant(std::numeric_limits<int8_t>::min());
780 max = mcgraph_->Int32Constant(std::numeric_limits<int8_t>::max());
781 } else {
782 max = mcgraph_->Uint32Constant(std::numeric_limits<uint8_t>::max());
783 shift_val = kShift8;
784 }
785 phi_rep = MachineRepresentation::kWord8;
786 }
787 int num_lanes = NumLanes(output_rep_type);
788 Node** rep_node = zone()->NewArray<Node*>(num_lanes);
789 for (int i = 0; i < num_lanes; ++i) {
790 Node* input = nullptr;
791 if (i < num_lanes / 2)
792 input = rep_left[i];
793 else
794 input = rep_right[i - num_lanes / 2];
795 if (is_signed) {
796 Diamond d_min(graph(), common(), graph()->NewNode(less_op, input, min));
797 input = d_min.Phi(phi_rep, min, input);
798 }
799 Diamond d_max(graph(), common(), graph()->NewNode(less_op, max, input));
800 rep_node[i] = d_max.Phi(phi_rep, max, input);
801 rep_node[i] =
802 is_signed ? rep_node[i] : FixUpperBits(rep_node[i], shift_val);
803 }
804 ReplaceNode(node, rep_node, num_lanes);
805 }
806
LowerShiftOp(Node * node,SimdType type)807 void SimdScalarLowering::LowerShiftOp(Node* node, SimdType type) {
808 DCHECK_EQ(1, node->InputCount());
809 int32_t shift_amount = OpParameter<int32_t>(node->op());
810 Node* shift_node = graph()->NewNode(common()->Int32Constant(shift_amount));
811 Node** rep = GetReplacementsWithType(node->InputAt(0), type);
812 int num_lanes = NumLanes(type);
813 Node** rep_node = zone()->NewArray<Node*>(num_lanes);
814 for (int i = 0; i < num_lanes; ++i) {
815 rep_node[i] = rep[i];
816 switch (node->opcode()) {
817 case IrOpcode::kI8x16ShrU:
818 rep_node[i] = Mask(rep_node[i], kMask8);
819 rep_node[i] =
820 graph()->NewNode(machine()->Word32Shr(), rep_node[i], shift_node);
821 break;
822 case IrOpcode::kI16x8ShrU:
823 rep_node[i] = Mask(rep_node[i], kMask16);
824 V8_FALLTHROUGH;
825 case IrOpcode::kI32x4ShrU:
826 rep_node[i] =
827 graph()->NewNode(machine()->Word32Shr(), rep_node[i], shift_node);
828 break;
829 case IrOpcode::kI32x4Shl:
830 rep_node[i] =
831 graph()->NewNode(machine()->Word32Shl(), rep_node[i], shift_node);
832 break;
833 case IrOpcode::kI16x8Shl:
834 rep_node[i] =
835 graph()->NewNode(machine()->Word32Shl(), rep_node[i], shift_node);
836 rep_node[i] = FixUpperBits(rep_node[i], kShift16);
837 break;
838 case IrOpcode::kI8x16Shl:
839 rep_node[i] =
840 graph()->NewNode(machine()->Word32Shl(), rep_node[i], shift_node);
841 rep_node[i] = FixUpperBits(rep_node[i], kShift8);
842 break;
843 case IrOpcode::kI32x4ShrS:
844 case IrOpcode::kI16x8ShrS:
845 case IrOpcode::kI8x16ShrS:
846 rep_node[i] =
847 graph()->NewNode(machine()->Word32Sar(), rep_node[i], shift_node);
848 break;
849 default:
850 UNREACHABLE();
851 }
852 }
853 ReplaceNode(node, rep_node, num_lanes);
854 }
855
LowerNotEqual(Node * node,SimdType input_rep_type,const Operator * op)856 void SimdScalarLowering::LowerNotEqual(Node* node, SimdType input_rep_type,
857 const Operator* op) {
858 DCHECK_EQ(2, node->InputCount());
859 Node** rep_left = GetReplacementsWithType(node->InputAt(0), input_rep_type);
860 Node** rep_right = GetReplacementsWithType(node->InputAt(1), input_rep_type);
861 int num_lanes = NumLanes(input_rep_type);
862 Node** rep_node = zone()->NewArray<Node*>(num_lanes);
863 for (int i = 0; i < num_lanes; ++i) {
864 Diamond d(graph(), common(),
865 graph()->NewNode(op, rep_left[i], rep_right[i]));
866 MachineRepresentation rep =
867 (input_rep_type == SimdType::kFloat32x4)
868 ? MachineRepresentation::kWord32
869 : MachineTypeFrom(input_rep_type).representation();
870 rep_node[i] =
871 d.Phi(rep, mcgraph_->Int32Constant(0), mcgraph_->Int32Constant(-1));
872 }
873 ReplaceNode(node, rep_node, num_lanes);
874 }
875
LowerNode(Node * node)876 void SimdScalarLowering::LowerNode(Node* node) {
877 SimdType rep_type = ReplacementType(node);
878 int num_lanes = NumLanes(rep_type);
879 switch (node->opcode()) {
880 case IrOpcode::kStart: {
881 int parameter_count = GetParameterCountAfterLowering();
882 // Only exchange the node if the parameter count actually changed.
883 if (parameter_count != static_cast<int>(signature()->parameter_count())) {
884 int delta =
885 parameter_count - static_cast<int>(signature()->parameter_count());
886 int new_output_count = node->op()->ValueOutputCount() + delta;
887 NodeProperties::ChangeOp(node, common()->Start(new_output_count));
888 }
889 break;
890 }
891 case IrOpcode::kParameter: {
892 DCHECK_EQ(1, node->InputCount());
893 // Only exchange the node if the parameter count actually changed. We do
894 // not even have to do the default lowering because the the start node,
895 // the only input of a parameter node, only changes if the parameter count
896 // changes.
897 if (GetParameterCountAfterLowering() !=
898 static_cast<int>(signature()->parameter_count())) {
899 int old_index = ParameterIndexOf(node->op());
900 int new_index =
901 GetParameterIndexAfterLoweringSimd128(signature(), old_index);
902 if (old_index == new_index) {
903 NodeProperties::ChangeOp(node, common()->Parameter(new_index));
904
905 Node* new_node[kNumLanes32];
906 for (int i = 0; i < kNumLanes32; ++i) {
907 new_node[i] = nullptr;
908 }
909 new_node[0] = node;
910 if (signature()->GetParam(old_index) ==
911 MachineRepresentation::kSimd128) {
912 for (int i = 1; i < kNumLanes32; ++i) {
913 new_node[i] = graph()->NewNode(common()->Parameter(new_index + i),
914 graph()->start());
915 }
916 }
917 ReplaceNode(node, new_node, kNumLanes32);
918 }
919 }
920 break;
921 }
922 case IrOpcode::kLoad:
923 case IrOpcode::kUnalignedLoad:
924 case IrOpcode::kProtectedLoad: {
925 LowerLoadOp(node, rep_type);
926 break;
927 }
928 case IrOpcode::kStore:
929 case IrOpcode::kUnalignedStore:
930 case IrOpcode::kProtectedStore: {
931 LowerStoreOp(node);
932 break;
933 }
934 case IrOpcode::kReturn: {
935 DefaultLowering(node);
936 int new_return_count = GetReturnCountAfterLoweringSimd128(signature());
937 if (static_cast<int>(signature()->return_count()) != new_return_count) {
938 NodeProperties::ChangeOp(node, common()->Return(new_return_count));
939 }
940 break;
941 }
942 case IrOpcode::kCall: {
943 // TODO(turbofan): Make wasm code const-correct wrt. CallDescriptor.
944 auto call_descriptor =
945 const_cast<CallDescriptor*>(CallDescriptorOf(node->op()));
946 if (DefaultLowering(node) ||
947 (call_descriptor->ReturnCount() == 1 &&
948 call_descriptor->GetReturnType(0) == MachineType::Simd128())) {
949 // We have to adjust the call descriptor.
950 const Operator* op = common()->Call(
951 GetI32WasmCallDescriptorForSimd(zone(), call_descriptor));
952 NodeProperties::ChangeOp(node, op);
953 }
954 if (call_descriptor->ReturnCount() == 1 &&
955 call_descriptor->GetReturnType(0) == MachineType::Simd128()) {
956 // We access the additional return values through projections.
957 Node* rep_node[kNumLanes32];
958 for (int i = 0; i < kNumLanes32; ++i) {
959 rep_node[i] =
960 graph()->NewNode(common()->Projection(i), node, graph()->start());
961 }
962 ReplaceNode(node, rep_node, kNumLanes32);
963 }
964 break;
965 }
966 case IrOpcode::kPhi: {
967 MachineRepresentation rep = PhiRepresentationOf(node->op());
968 if (rep == MachineRepresentation::kSimd128) {
969 // The replacement nodes have already been created, we only have to
970 // replace placeholder nodes.
971 Node** rep_node = GetReplacements(node);
972 for (int i = 0; i < node->op()->ValueInputCount(); ++i) {
973 Node** rep_input =
974 GetReplacementsWithType(node->InputAt(i), rep_type);
975 for (int j = 0; j < num_lanes; j++) {
976 rep_node[j]->ReplaceInput(i, rep_input[j]);
977 }
978 }
979 } else {
980 DefaultLowering(node);
981 }
982 break;
983 }
984 #define I32X4_BINOP_CASE(opcode, instruction) \
985 case IrOpcode::opcode: { \
986 LowerBinaryOp(node, rep_type, machine()->instruction()); \
987 break; \
988 }
989 I32X4_BINOP_CASE(kI32x4Add, Int32Add)
990 I32X4_BINOP_CASE(kI32x4Sub, Int32Sub)
991 I32X4_BINOP_CASE(kI32x4Mul, Int32Mul)
992 I32X4_BINOP_CASE(kS128And, Word32And)
993 I32X4_BINOP_CASE(kS128Or, Word32Or)
994 I32X4_BINOP_CASE(kS128Xor, Word32Xor)
995 #undef I32X4_BINOP_CASE
996 case IrOpcode::kI32x4AddHoriz: {
997 LowerBinaryOp(node, rep_type, machine()->Int32Add(), false);
998 break;
999 }
1000 case IrOpcode::kI16x8AddHoriz: {
1001 LowerBinaryOpForSmallInt(node, rep_type, machine()->Int32Add(), false);
1002 break;
1003 }
1004 case IrOpcode::kI16x8Add:
1005 case IrOpcode::kI8x16Add: {
1006 LowerBinaryOpForSmallInt(node, rep_type, machine()->Int32Add());
1007 break;
1008 }
1009 case IrOpcode::kI16x8Sub:
1010 case IrOpcode::kI8x16Sub: {
1011 LowerBinaryOpForSmallInt(node, rep_type, machine()->Int32Sub());
1012 break;
1013 }
1014 case IrOpcode::kI16x8Mul:
1015 case IrOpcode::kI8x16Mul: {
1016 LowerBinaryOpForSmallInt(node, rep_type, machine()->Int32Mul());
1017 break;
1018 }
1019 case IrOpcode::kI16x8AddSaturateS:
1020 case IrOpcode::kI8x16AddSaturateS: {
1021 LowerSaturateBinaryOp(node, rep_type, machine()->Int32Add(), true);
1022 break;
1023 }
1024 case IrOpcode::kI16x8SubSaturateS:
1025 case IrOpcode::kI8x16SubSaturateS: {
1026 LowerSaturateBinaryOp(node, rep_type, machine()->Int32Sub(), true);
1027 break;
1028 }
1029 case IrOpcode::kI16x8AddSaturateU:
1030 case IrOpcode::kI8x16AddSaturateU: {
1031 LowerSaturateBinaryOp(node, rep_type, machine()->Int32Add(), false);
1032 break;
1033 }
1034 case IrOpcode::kI16x8SubSaturateU:
1035 case IrOpcode::kI8x16SubSaturateU: {
1036 LowerSaturateBinaryOp(node, rep_type, machine()->Int32Sub(), false);
1037 break;
1038 }
1039 case IrOpcode::kI32x4MaxS:
1040 case IrOpcode::kI16x8MaxS:
1041 case IrOpcode::kI8x16MaxS: {
1042 LowerIntMinMax(node, machine()->Int32LessThan(), true, rep_type);
1043 break;
1044 }
1045 case IrOpcode::kI32x4MinS:
1046 case IrOpcode::kI16x8MinS:
1047 case IrOpcode::kI8x16MinS: {
1048 LowerIntMinMax(node, machine()->Int32LessThan(), false, rep_type);
1049 break;
1050 }
1051 case IrOpcode::kI32x4MaxU:
1052 case IrOpcode::kI16x8MaxU:
1053 case IrOpcode::kI8x16MaxU: {
1054 LowerIntMinMax(node, machine()->Uint32LessThan(), true, rep_type);
1055 break;
1056 }
1057 case IrOpcode::kI32x4MinU:
1058 case IrOpcode::kI16x8MinU:
1059 case IrOpcode::kI8x16MinU: {
1060 LowerIntMinMax(node, machine()->Uint32LessThan(), false, rep_type);
1061 break;
1062 }
1063 case IrOpcode::kI32x4Neg:
1064 case IrOpcode::kI16x8Neg:
1065 case IrOpcode::kI8x16Neg: {
1066 DCHECK_EQ(1, node->InputCount());
1067 Node** rep = GetReplacementsWithType(node->InputAt(0), rep_type);
1068 int num_lanes = NumLanes(rep_type);
1069 Node** rep_node = zone()->NewArray<Node*>(num_lanes);
1070 Node* zero = graph()->NewNode(common()->Int32Constant(0));
1071 for (int i = 0; i < num_lanes; ++i) {
1072 rep_node[i] = graph()->NewNode(machine()->Int32Sub(), zero, rep[i]);
1073 if (node->opcode() == IrOpcode::kI16x8Neg) {
1074 rep_node[i] = FixUpperBits(rep_node[i], kShift16);
1075 } else if (node->opcode() == IrOpcode::kI8x16Neg) {
1076 rep_node[i] = FixUpperBits(rep_node[i], kShift8);
1077 }
1078 }
1079 ReplaceNode(node, rep_node, num_lanes);
1080 break;
1081 }
1082 case IrOpcode::kS128Not: {
1083 DCHECK_EQ(1, node->InputCount());
1084 Node** rep = GetReplacementsWithType(node->InputAt(0), rep_type);
1085 Node* rep_node[kNumLanes32];
1086 Node* mask = graph()->NewNode(common()->Int32Constant(0xFFFFFFFF));
1087 for (int i = 0; i < kNumLanes32; ++i) {
1088 rep_node[i] = graph()->NewNode(machine()->Word32Xor(), rep[i], mask);
1089 }
1090 ReplaceNode(node, rep_node, kNumLanes32);
1091 break;
1092 }
1093 case IrOpcode::kI32x4SConvertF32x4: {
1094 LowerConvertFromFloat(node, true);
1095 break;
1096 }
1097 case IrOpcode::kI32x4UConvertF32x4: {
1098 LowerConvertFromFloat(node, false);
1099 break;
1100 }
1101 case IrOpcode::kI32x4SConvertI16x8Low: {
1102 LowerConvertFromInt(node, SimdType::kInt16x8, SimdType::kInt32x4, true,
1103 0);
1104 break;
1105 }
1106 case IrOpcode::kI32x4SConvertI16x8High: {
1107 LowerConvertFromInt(node, SimdType::kInt16x8, SimdType::kInt32x4, true,
1108 4);
1109 break;
1110 }
1111 case IrOpcode::kI32x4UConvertI16x8Low: {
1112 LowerConvertFromInt(node, SimdType::kInt16x8, SimdType::kInt32x4, false,
1113 0);
1114 break;
1115 }
1116 case IrOpcode::kI32x4UConvertI16x8High: {
1117 LowerConvertFromInt(node, SimdType::kInt16x8, SimdType::kInt32x4, false,
1118 4);
1119 break;
1120 }
1121 case IrOpcode::kI16x8SConvertI8x16Low: {
1122 LowerConvertFromInt(node, SimdType::kInt8x16, SimdType::kInt16x8, true,
1123 0);
1124 break;
1125 }
1126 case IrOpcode::kI16x8SConvertI8x16High: {
1127 LowerConvertFromInt(node, SimdType::kInt8x16, SimdType::kInt16x8, true,
1128 8);
1129 break;
1130 }
1131 case IrOpcode::kI16x8UConvertI8x16Low: {
1132 LowerConvertFromInt(node, SimdType::kInt8x16, SimdType::kInt16x8, false,
1133 0);
1134 break;
1135 }
1136 case IrOpcode::kI16x8UConvertI8x16High: {
1137 LowerConvertFromInt(node, SimdType::kInt8x16, SimdType::kInt16x8, false,
1138 8);
1139 break;
1140 }
1141 case IrOpcode::kI16x8SConvertI32x4: {
1142 LowerPack(node, SimdType::kInt32x4, SimdType::kInt16x8, true);
1143 break;
1144 }
1145 case IrOpcode::kI16x8UConvertI32x4: {
1146 LowerPack(node, SimdType::kInt32x4, SimdType::kInt16x8, false);
1147 break;
1148 }
1149 case IrOpcode::kI8x16SConvertI16x8: {
1150 LowerPack(node, SimdType::kInt16x8, SimdType::kInt8x16, true);
1151 break;
1152 }
1153 case IrOpcode::kI8x16UConvertI16x8: {
1154 LowerPack(node, SimdType::kInt16x8, SimdType::kInt8x16, false);
1155 break;
1156 }
1157 case IrOpcode::kI32x4Shl:
1158 case IrOpcode::kI16x8Shl:
1159 case IrOpcode::kI8x16Shl:
1160 case IrOpcode::kI32x4ShrS:
1161 case IrOpcode::kI16x8ShrS:
1162 case IrOpcode::kI8x16ShrS:
1163 case IrOpcode::kI32x4ShrU:
1164 case IrOpcode::kI16x8ShrU:
1165 case IrOpcode::kI8x16ShrU: {
1166 LowerShiftOp(node, rep_type);
1167 break;
1168 }
1169 case IrOpcode::kF32x4AddHoriz: {
1170 LowerBinaryOp(node, rep_type, machine()->Float32Add(), false);
1171 break;
1172 }
1173 #define F32X4_BINOP_CASE(name) \
1174 case IrOpcode::kF32x4##name: { \
1175 LowerBinaryOp(node, rep_type, machine()->Float32##name()); \
1176 break; \
1177 }
1178 F32X4_BINOP_CASE(Add)
1179 F32X4_BINOP_CASE(Sub)
1180 F32X4_BINOP_CASE(Mul)
1181 F32X4_BINOP_CASE(Min)
1182 F32X4_BINOP_CASE(Max)
1183 #undef F32X4_BINOP_CASE
1184 #define F32X4_UNOP_CASE(name) \
1185 case IrOpcode::kF32x4##name: { \
1186 LowerUnaryOp(node, rep_type, machine()->Float32##name()); \
1187 break; \
1188 }
1189 F32X4_UNOP_CASE(Abs)
1190 F32X4_UNOP_CASE(Neg)
1191 #undef F32x4_UNOP_CASE
1192 case IrOpcode::kF32x4RecipApprox:
1193 case IrOpcode::kF32x4RecipSqrtApprox: {
1194 DCHECK_EQ(1, node->InputCount());
1195 Node** rep = GetReplacementsWithType(node->InputAt(0), rep_type);
1196 Node** rep_node = zone()->NewArray<Node*>(num_lanes);
1197 Node* float_one = graph()->NewNode(common()->Float32Constant(1.0));
1198 for (int i = 0; i < num_lanes; ++i) {
1199 Node* tmp = rep[i];
1200 if (node->opcode() == IrOpcode::kF32x4RecipSqrtApprox) {
1201 tmp = graph()->NewNode(machine()->Float32Sqrt(), rep[i]);
1202 }
1203 rep_node[i] = graph()->NewNode(machine()->Float32Div(), float_one, tmp);
1204 }
1205 ReplaceNode(node, rep_node, num_lanes);
1206 break;
1207 }
1208 case IrOpcode::kF32x4SConvertI32x4: {
1209 LowerUnaryOp(node, SimdType::kInt32x4, machine()->RoundInt32ToFloat32());
1210 break;
1211 }
1212 case IrOpcode::kF32x4UConvertI32x4: {
1213 LowerUnaryOp(node, SimdType::kInt32x4, machine()->RoundUint32ToFloat32());
1214 break;
1215 }
1216 case IrOpcode::kI32x4Splat:
1217 case IrOpcode::kF32x4Splat:
1218 case IrOpcode::kI16x8Splat:
1219 case IrOpcode::kI8x16Splat: {
1220 Node** rep_node = zone()->NewArray<Node*>(num_lanes);
1221 for (int i = 0; i < num_lanes; ++i) {
1222 if (HasReplacement(0, node->InputAt(0))) {
1223 rep_node[i] = GetReplacements(node->InputAt(0))[0];
1224 } else {
1225 rep_node[i] = node->InputAt(0);
1226 }
1227 }
1228 ReplaceNode(node, rep_node, num_lanes);
1229 break;
1230 }
1231 case IrOpcode::kI32x4ExtractLane:
1232 case IrOpcode::kF32x4ExtractLane:
1233 case IrOpcode::kI16x8ExtractLane:
1234 case IrOpcode::kI8x16ExtractLane: {
1235 int32_t lane = OpParameter<int32_t>(node->op());
1236 Node** rep_node = zone()->NewArray<Node*>(num_lanes);
1237 rep_node[0] = GetReplacementsWithType(node->InputAt(0), rep_type)[lane];
1238 for (int i = 1; i < num_lanes; ++i) {
1239 rep_node[i] = nullptr;
1240 }
1241 ReplaceNode(node, rep_node, num_lanes);
1242 break;
1243 }
1244 case IrOpcode::kI32x4ReplaceLane:
1245 case IrOpcode::kF32x4ReplaceLane:
1246 case IrOpcode::kI16x8ReplaceLane:
1247 case IrOpcode::kI8x16ReplaceLane: {
1248 DCHECK_EQ(2, node->InputCount());
1249 Node* repNode = node->InputAt(1);
1250 int32_t lane = OpParameter<int32_t>(node->op());
1251 Node** old_rep_node = GetReplacementsWithType(node->InputAt(0), rep_type);
1252 Node** rep_node = zone()->NewArray<Node*>(num_lanes);
1253 for (int i = 0; i < num_lanes; ++i) {
1254 rep_node[i] = old_rep_node[i];
1255 }
1256 if (HasReplacement(0, repNode)) {
1257 rep_node[lane] = GetReplacements(repNode)[0];
1258 } else {
1259 rep_node[lane] = repNode;
1260 }
1261 ReplaceNode(node, rep_node, num_lanes);
1262 break;
1263 }
1264 #define COMPARISON_CASE(type, simd_op, lowering_op, invert) \
1265 case IrOpcode::simd_op: { \
1266 LowerCompareOp(node, SimdType::k##type, machine()->lowering_op(), invert); \
1267 break; \
1268 }
1269 COMPARISON_CASE(Float32x4, kF32x4Eq, Float32Equal, false)
1270 COMPARISON_CASE(Float32x4, kF32x4Lt, Float32LessThan, false)
1271 COMPARISON_CASE(Float32x4, kF32x4Le, Float32LessThanOrEqual, false)
1272 COMPARISON_CASE(Float32x4, kF32x4Gt, Float32LessThan, true)
1273 COMPARISON_CASE(Float32x4, kF32x4Ge, Float32LessThanOrEqual, true)
1274 COMPARISON_CASE(Int32x4, kI32x4Eq, Word32Equal, false)
1275 COMPARISON_CASE(Int32x4, kI32x4LtS, Int32LessThan, false)
1276 COMPARISON_CASE(Int32x4, kI32x4LeS, Int32LessThanOrEqual, false)
1277 COMPARISON_CASE(Int32x4, kI32x4GtS, Int32LessThan, true)
1278 COMPARISON_CASE(Int32x4, kI32x4GeS, Int32LessThanOrEqual, true)
1279 COMPARISON_CASE(Int32x4, kI32x4LtU, Uint32LessThan, false)
1280 COMPARISON_CASE(Int32x4, kI32x4LeU, Uint32LessThanOrEqual, false)
1281 COMPARISON_CASE(Int32x4, kI32x4GtU, Uint32LessThan, true)
1282 COMPARISON_CASE(Int32x4, kI32x4GeU, Uint32LessThanOrEqual, true)
1283 COMPARISON_CASE(Int16x8, kI16x8Eq, Word32Equal, false)
1284 COMPARISON_CASE(Int16x8, kI16x8LtS, Int32LessThan, false)
1285 COMPARISON_CASE(Int16x8, kI16x8LeS, Int32LessThanOrEqual, false)
1286 COMPARISON_CASE(Int16x8, kI16x8GtS, Int32LessThan, true)
1287 COMPARISON_CASE(Int16x8, kI16x8GeS, Int32LessThanOrEqual, true)
1288 COMPARISON_CASE(Int16x8, kI16x8LtU, Uint32LessThan, false)
1289 COMPARISON_CASE(Int16x8, kI16x8LeU, Uint32LessThanOrEqual, false)
1290 COMPARISON_CASE(Int16x8, kI16x8GtU, Uint32LessThan, true)
1291 COMPARISON_CASE(Int16x8, kI16x8GeU, Uint32LessThanOrEqual, true)
1292 COMPARISON_CASE(Int8x16, kI8x16Eq, Word32Equal, false)
1293 COMPARISON_CASE(Int8x16, kI8x16LtS, Int32LessThan, false)
1294 COMPARISON_CASE(Int8x16, kI8x16LeS, Int32LessThanOrEqual, false)
1295 COMPARISON_CASE(Int8x16, kI8x16GtS, Int32LessThan, true)
1296 COMPARISON_CASE(Int8x16, kI8x16GeS, Int32LessThanOrEqual, true)
1297 COMPARISON_CASE(Int8x16, kI8x16LtU, Uint32LessThan, false)
1298 COMPARISON_CASE(Int8x16, kI8x16LeU, Uint32LessThanOrEqual, false)
1299 COMPARISON_CASE(Int8x16, kI8x16GtU, Uint32LessThan, true)
1300 COMPARISON_CASE(Int8x16, kI8x16GeU, Uint32LessThanOrEqual, true)
1301 #undef COMPARISON_CASE
1302 case IrOpcode::kF32x4Ne: {
1303 LowerNotEqual(node, SimdType::kFloat32x4, machine()->Float32Equal());
1304 break;
1305 }
1306 case IrOpcode::kI32x4Ne: {
1307 LowerNotEqual(node, SimdType::kInt32x4, machine()->Word32Equal());
1308 break;
1309 }
1310 case IrOpcode::kI16x8Ne: {
1311 LowerNotEqual(node, SimdType::kInt16x8, machine()->Word32Equal());
1312 break;
1313 }
1314 case IrOpcode::kI8x16Ne: {
1315 LowerNotEqual(node, SimdType::kInt8x16, machine()->Word32Equal());
1316 break;
1317 }
1318 case IrOpcode::kS128Select: {
1319 DCHECK_EQ(3, node->InputCount());
1320 DCHECK(ReplacementType(node->InputAt(0)) == SimdType::kInt32x4 ||
1321 ReplacementType(node->InputAt(0)) == SimdType::kInt16x8 ||
1322 ReplacementType(node->InputAt(0)) == SimdType::kInt8x16);
1323 Node** boolean_input = GetReplacements(node->InputAt(0));
1324 Node** rep_left = GetReplacementsWithType(node->InputAt(1), rep_type);
1325 Node** rep_right = GetReplacementsWithType(node->InputAt(2), rep_type);
1326 Node** rep_node = zone()->NewArray<Node*>(num_lanes);
1327 for (int i = 0; i < num_lanes; ++i) {
1328 Node* tmp1 =
1329 graph()->NewNode(machine()->Word32Xor(), rep_left[i], rep_right[i]);
1330 Node* tmp2 =
1331 graph()->NewNode(machine()->Word32And(), boolean_input[i], tmp1);
1332 rep_node[i] =
1333 graph()->NewNode(machine()->Word32Xor(), rep_right[i], tmp2);
1334 }
1335 ReplaceNode(node, rep_node, num_lanes);
1336 break;
1337 }
1338 case IrOpcode::kS8x16Shuffle: {
1339 DCHECK_EQ(2, node->InputCount());
1340 const uint8_t* shuffle = OpParameter<uint8_t*>(node->op());
1341 Node** rep_left = GetReplacementsWithType(node->InputAt(0), rep_type);
1342 Node** rep_right = GetReplacementsWithType(node->InputAt(1), rep_type);
1343 Node** rep_node = zone()->NewArray<Node*>(16);
1344 for (int i = 0; i < 16; i++) {
1345 int lane = shuffle[i];
1346 rep_node[i] = lane < 16 ? rep_left[lane] : rep_right[lane - 16];
1347 }
1348 ReplaceNode(node, rep_node, 16);
1349 break;
1350 }
1351 case IrOpcode::kS1x4AnyTrue:
1352 case IrOpcode::kS1x4AllTrue:
1353 case IrOpcode::kS1x8AnyTrue:
1354 case IrOpcode::kS1x8AllTrue:
1355 case IrOpcode::kS1x16AnyTrue:
1356 case IrOpcode::kS1x16AllTrue: {
1357 DCHECK_EQ(1, node->InputCount());
1358 SimdType input_rep_type = ReplacementType(node->InputAt(0));
1359 int input_num_lanes = NumLanes(input_rep_type);
1360 Node** rep = GetReplacements(node->InputAt(0));
1361 Node** rep_node = zone()->NewArray<Node*>(num_lanes);
1362 Node* true_node = mcgraph_->Int32Constant(-1);
1363 Node* false_node = mcgraph_->Int32Constant(0);
1364 Node* tmp_result = false_node;
1365 if (node->opcode() == IrOpcode::kS1x4AllTrue ||
1366 node->opcode() == IrOpcode::kS1x8AllTrue ||
1367 node->opcode() == IrOpcode::kS1x16AllTrue) {
1368 tmp_result = true_node;
1369 }
1370 for (int i = 0; i < input_num_lanes; ++i) {
1371 Diamond is_false(
1372 graph(), common(),
1373 graph()->NewNode(machine()->Word32Equal(), rep[i], false_node));
1374 if (node->opcode() == IrOpcode::kS1x4AllTrue ||
1375 node->opcode() == IrOpcode::kS1x8AllTrue ||
1376 node->opcode() == IrOpcode::kS1x16AllTrue) {
1377 tmp_result = is_false.Phi(MachineRepresentation::kWord32, false_node,
1378 tmp_result);
1379 } else {
1380 tmp_result = is_false.Phi(MachineRepresentation::kWord32, tmp_result,
1381 true_node);
1382 }
1383 }
1384 rep_node[0] = tmp_result;
1385 for (int i = 1; i < num_lanes; ++i) {
1386 rep_node[i] = nullptr;
1387 }
1388 ReplaceNode(node, rep_node, num_lanes);
1389 break;
1390 }
1391 default: { DefaultLowering(node); }
1392 }
1393 }
1394
DefaultLowering(Node * node)1395 bool SimdScalarLowering::DefaultLowering(Node* node) {
1396 bool something_changed = false;
1397 for (int i = NodeProperties::PastValueIndex(node) - 1; i >= 0; i--) {
1398 Node* input = node->InputAt(i);
1399 if (HasReplacement(0, input)) {
1400 something_changed = true;
1401 node->ReplaceInput(i, GetReplacements(input)[0]);
1402 }
1403 if (HasReplacement(1, input)) {
1404 something_changed = true;
1405 for (int j = 1; j < ReplacementCount(input); ++j) {
1406 node->InsertInput(zone(), i + j, GetReplacements(input)[j]);
1407 }
1408 }
1409 }
1410 return something_changed;
1411 }
1412
ReplaceNode(Node * old,Node ** new_nodes,int count)1413 void SimdScalarLowering::ReplaceNode(Node* old, Node** new_nodes, int count) {
1414 replacements_[old->id()].node = zone()->NewArray<Node*>(count);
1415 for (int i = 0; i < count; ++i) {
1416 replacements_[old->id()].node[i] = new_nodes[i];
1417 }
1418 replacements_[old->id()].num_replacements = count;
1419 }
1420
HasReplacement(size_t index,Node * node)1421 bool SimdScalarLowering::HasReplacement(size_t index, Node* node) {
1422 return replacements_[node->id()].node != nullptr &&
1423 replacements_[node->id()].node[index] != nullptr;
1424 }
1425
ReplacementType(Node * node)1426 SimdScalarLowering::SimdType SimdScalarLowering::ReplacementType(Node* node) {
1427 return replacements_[node->id()].type;
1428 }
1429
GetReplacements(Node * node)1430 Node** SimdScalarLowering::GetReplacements(Node* node) {
1431 Node** result = replacements_[node->id()].node;
1432 DCHECK(result);
1433 return result;
1434 }
1435
ReplacementCount(Node * node)1436 int SimdScalarLowering::ReplacementCount(Node* node) {
1437 return replacements_[node->id()].num_replacements;
1438 }
1439
Int32ToFloat32(Node ** replacements,Node ** result)1440 void SimdScalarLowering::Int32ToFloat32(Node** replacements, Node** result) {
1441 for (int i = 0; i < kNumLanes32; ++i) {
1442 if (replacements[i] != nullptr) {
1443 result[i] =
1444 graph()->NewNode(machine()->BitcastInt32ToFloat32(), replacements[i]);
1445 } else {
1446 result[i] = nullptr;
1447 }
1448 }
1449 }
1450
Float32ToInt32(Node ** replacements,Node ** result)1451 void SimdScalarLowering::Float32ToInt32(Node** replacements, Node** result) {
1452 for (int i = 0; i < kNumLanes32; ++i) {
1453 if (replacements[i] != nullptr) {
1454 result[i] =
1455 graph()->NewNode(machine()->BitcastFloat32ToInt32(), replacements[i]);
1456 } else {
1457 result[i] = nullptr;
1458 }
1459 }
1460 }
1461
1462 template <typename T>
Int32ToSmallerInt(Node ** replacements,Node ** result)1463 void SimdScalarLowering::Int32ToSmallerInt(Node** replacements, Node** result) {
1464 const int num_ints = sizeof(int32_t) / sizeof(T);
1465 const int bit_size = sizeof(T) * 8;
1466 const Operator* sign_extend;
1467 switch (sizeof(T)) {
1468 case 1:
1469 sign_extend = machine()->SignExtendWord8ToInt32();
1470 break;
1471 case 2:
1472 sign_extend = machine()->SignExtendWord16ToInt32();
1473 break;
1474 default:
1475 UNREACHABLE();
1476 }
1477
1478 for (int i = 0; i < kNumLanes32; i++) {
1479 if (replacements[i] != nullptr) {
1480 for (int j = 0; j < num_ints; j++) {
1481 result[num_ints * i + j] = graph()->NewNode(
1482 sign_extend,
1483 graph()->NewNode(machine()->Word32Sar(), replacements[i],
1484 mcgraph_->Int32Constant(j * bit_size)));
1485 }
1486 } else {
1487 for (int j = 0; j < num_ints; j++) {
1488 result[num_ints * i + j] = nullptr;
1489 }
1490 }
1491 }
1492 }
1493
1494 template <typename T>
SmallerIntToInt32(Node ** replacements,Node ** result)1495 void SimdScalarLowering::SmallerIntToInt32(Node** replacements, Node** result) {
1496 const int num_ints = sizeof(int32_t) / sizeof(T);
1497 const int bit_size = sizeof(T) * 8;
1498 const int bit_mask = (1 << bit_size) - 1;
1499
1500 for (int i = 0; i < kNumLanes32; ++i) {
1501 result[i] = mcgraph_->Int32Constant(0);
1502 for (int j = 0; j < num_ints; j++) {
1503 if (replacements[num_ints * i + j] != nullptr) {
1504 Node* clean_bits = graph()->NewNode(machine()->Word32And(),
1505 replacements[num_ints * i + j],
1506 mcgraph_->Int32Constant(bit_mask));
1507 Node* shift = graph()->NewNode(machine()->Word32Shl(), clean_bits,
1508 mcgraph_->Int32Constant(j * bit_size));
1509 result[i] = graph()->NewNode(machine()->Word32Or(), result[i], shift);
1510 }
1511 }
1512 }
1513 }
1514
GetReplacementsWithType(Node * node,SimdType type)1515 Node** SimdScalarLowering::GetReplacementsWithType(Node* node, SimdType type) {
1516 Node** replacements = GetReplacements(node);
1517 if (ReplacementType(node) == type) {
1518 return GetReplacements(node);
1519 }
1520 int num_lanes = NumLanes(type);
1521 Node** result = zone()->NewArray<Node*>(num_lanes);
1522 if (type == SimdType::kInt32x4) {
1523 if (ReplacementType(node) == SimdType::kFloat32x4) {
1524 Float32ToInt32(replacements, result);
1525 } else if (ReplacementType(node) == SimdType::kInt16x8) {
1526 SmallerIntToInt32<int16_t>(replacements, result);
1527 } else if (ReplacementType(node) == SimdType::kInt8x16) {
1528 SmallerIntToInt32<int8_t>(replacements, result);
1529 } else {
1530 UNREACHABLE();
1531 }
1532 } else if (type == SimdType::kFloat32x4) {
1533 if (ReplacementType(node) == SimdType::kInt32x4) {
1534 Int32ToFloat32(replacements, result);
1535 } else if (ReplacementType(node) == SimdType::kInt16x8) {
1536 UNIMPLEMENTED();
1537 } else {
1538 UNREACHABLE();
1539 }
1540 } else if (type == SimdType::kInt16x8) {
1541 if (ReplacementType(node) == SimdType::kInt32x4) {
1542 Int32ToSmallerInt<int16_t>(replacements, result);
1543 } else if (ReplacementType(node) == SimdType::kFloat32x4) {
1544 UNIMPLEMENTED();
1545 } else {
1546 UNREACHABLE();
1547 }
1548 } else if (type == SimdType::kInt8x16) {
1549 if (ReplacementType(node) == SimdType::kInt32x4) {
1550 Int32ToSmallerInt<int8_t>(replacements, result);
1551 } else {
1552 UNIMPLEMENTED();
1553 }
1554 } else {
1555 UNREACHABLE();
1556 }
1557 return result;
1558 }
1559
PreparePhiReplacement(Node * phi)1560 void SimdScalarLowering::PreparePhiReplacement(Node* phi) {
1561 MachineRepresentation rep = PhiRepresentationOf(phi->op());
1562 if (rep == MachineRepresentation::kSimd128) {
1563 // We have to create the replacements for a phi node before we actually
1564 // lower the phi to break potential cycles in the graph. The replacements of
1565 // input nodes do not exist yet, so we use a placeholder node to pass the
1566 // graph verifier.
1567 int value_count = phi->op()->ValueInputCount();
1568 SimdType type = ReplacementType(phi);
1569 int num_lanes = NumLanes(type);
1570 Node*** inputs_rep = zone()->NewArray<Node**>(num_lanes);
1571 for (int i = 0; i < num_lanes; ++i) {
1572 inputs_rep[i] = zone()->NewArray<Node*>(value_count + 1);
1573 inputs_rep[i][value_count] = NodeProperties::GetControlInput(phi, 0);
1574 }
1575 for (int i = 0; i < value_count; ++i) {
1576 for (int j = 0; j < num_lanes; ++j) {
1577 inputs_rep[j][i] = placeholder_;
1578 }
1579 }
1580 Node** rep_nodes = zone()->NewArray<Node*>(num_lanes);
1581 for (int i = 0; i < num_lanes; ++i) {
1582 rep_nodes[i] = graph()->NewNode(
1583 common()->Phi(MachineTypeFrom(type).representation(), value_count),
1584 value_count + 1, inputs_rep[i], false);
1585 }
1586 ReplaceNode(phi, rep_nodes, num_lanes);
1587 }
1588 }
1589 } // namespace compiler
1590 } // namespace internal
1591 } // namespace v8
1592