1 // Copyright 2014 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "src/compiler/int64-lowering.h"
6
7 #include "src/compiler/common-operator.h"
8 #include "src/compiler/diamond.h"
9 #include "src/compiler/graph.h"
10 #include "src/compiler/linkage.h"
11 #include "src/compiler/machine-operator.h"
12 #include "src/compiler/node-matchers.h"
13 #include "src/compiler/node-properties.h"
14 #include "src/compiler/node.h"
15 #include "src/compiler/wasm-compiler.h"
16 // TODO(wasm): Remove this include.
17 #include "src/wasm/wasm-linkage.h"
18 #include "src/zone/zone.h"
19
20 namespace v8 {
21 namespace internal {
22 namespace compiler {
23
Int64Lowering(Graph * graph,MachineOperatorBuilder * machine,CommonOperatorBuilder * common,Zone * zone,Signature<MachineRepresentation> * signature)24 Int64Lowering::Int64Lowering(Graph* graph, MachineOperatorBuilder* machine,
25 CommonOperatorBuilder* common, Zone* zone,
26 Signature<MachineRepresentation>* signature)
27 : zone_(zone),
28 graph_(graph),
29 machine_(machine),
30 common_(common),
31 state_(graph, 3),
32 stack_(zone),
33 replacements_(nullptr),
34 signature_(signature),
35 placeholder_(graph->NewNode(common->Parameter(-2, "placeholder"),
36 graph->start())) {
37 DCHECK_NOT_NULL(graph);
38 DCHECK_NOT_NULL(graph->end());
39 replacements_ = zone->NewArray<Replacement>(graph->NodeCount());
40 memset(replacements_, 0, sizeof(Replacement) * graph->NodeCount());
41 }
42
LowerGraph()43 void Int64Lowering::LowerGraph() {
44 if (!machine()->Is32()) {
45 return;
46 }
47 stack_.push_back({graph()->end(), 0});
48 state_.Set(graph()->end(), State::kOnStack);
49
50 while (!stack_.empty()) {
51 NodeState& top = stack_.back();
52 if (top.input_index == top.node->InputCount()) {
53 // All inputs of top have already been lowered, now lower top.
54 stack_.pop_back();
55 state_.Set(top.node, State::kVisited);
56 LowerNode(top.node);
57 } else {
58 // Push the next input onto the stack.
59 Node* input = top.node->InputAt(top.input_index++);
60 if (state_.Get(input) == State::kUnvisited) {
61 if (input->opcode() == IrOpcode::kPhi) {
62 // To break cycles with phi nodes we push phis on a separate stack so
63 // that they are processed after all other nodes.
64 PreparePhiReplacement(input);
65 stack_.push_front({input, 0});
66 } else if (input->opcode() == IrOpcode::kEffectPhi ||
67 input->opcode() == IrOpcode::kLoop) {
68 stack_.push_front({input, 0});
69 } else {
70 stack_.push_back({input, 0});
71 }
72 state_.Set(input, State::kOnStack);
73 }
74 }
75 }
76 }
77
78 namespace {
79
GetReturnIndexAfterLowering(CallDescriptor * call_descriptor,int old_index)80 int GetReturnIndexAfterLowering(CallDescriptor* call_descriptor,
81 int old_index) {
82 int result = old_index;
83 for (int i = 0; i < old_index; i++) {
84 if (call_descriptor->GetReturnType(i).representation() ==
85 MachineRepresentation::kWord64) {
86 result++;
87 }
88 }
89 return result;
90 }
91
GetReturnCountAfterLowering(CallDescriptor * call_descriptor)92 int GetReturnCountAfterLowering(CallDescriptor* call_descriptor) {
93 return GetReturnIndexAfterLowering(
94 call_descriptor, static_cast<int>(call_descriptor->ReturnCount()));
95 }
96
GetParameterIndexAfterLowering(Signature<MachineRepresentation> * signature,int old_index)97 int GetParameterIndexAfterLowering(
98 Signature<MachineRepresentation>* signature, int old_index) {
99 int result = old_index;
100 for (int i = 0; i < old_index; i++) {
101 if (signature->GetParam(i) == MachineRepresentation::kWord64) {
102 result++;
103 }
104 }
105 return result;
106 }
107
GetReturnCountAfterLowering(Signature<MachineRepresentation> * signature)108 int GetReturnCountAfterLowering(Signature<MachineRepresentation>* signature) {
109 int result = static_cast<int>(signature->return_count());
110 for (int i = 0; i < static_cast<int>(signature->return_count()); i++) {
111 if (signature->GetReturn(i) == MachineRepresentation::kWord64) {
112 result++;
113 }
114 }
115 return result;
116 }
117
118 } // namespace
119
LowerWord64AtomicBinop(Node * node,const Operator * op)120 void Int64Lowering::LowerWord64AtomicBinop(Node* node, const Operator* op) {
121 DCHECK_EQ(5, node->InputCount());
122 Node* value = node->InputAt(2);
123 node->ReplaceInput(2, GetReplacementLow(value));
124 node->InsertInput(zone(), 3, GetReplacementHigh(value));
125 NodeProperties::ChangeOp(node, op);
126 ReplaceNodeWithProjections(node);
127 }
128
LowerWord64AtomicNarrowOp(Node * node,const Operator * op)129 void Int64Lowering::LowerWord64AtomicNarrowOp(Node* node, const Operator* op) {
130 DefaultLowering(node, true);
131 NodeProperties::ChangeOp(node, op);
132 ReplaceNodeWithProjections(node);
133 }
134
135 // static
GetParameterCountAfterLowering(Signature<MachineRepresentation> * signature)136 int Int64Lowering::GetParameterCountAfterLowering(
137 Signature<MachineRepresentation>* signature) {
138 // GetParameterIndexAfterLowering(parameter_count) returns the parameter count
139 // after lowering.
140 return GetParameterIndexAfterLowering(
141 signature, static_cast<int>(signature->parameter_count()));
142 }
143
GetIndexNodes(Node * index,Node * & index_low,Node * & index_high)144 void Int64Lowering::GetIndexNodes(Node* index, Node*& index_low,
145 Node*& index_high) {
146 if (HasReplacementLow(index)) {
147 index = GetReplacementLow(index);
148 }
149 #if defined(V8_TARGET_LITTLE_ENDIAN)
150 index_low = index;
151 index_high = graph()->NewNode(machine()->Int32Add(), index,
152 graph()->NewNode(common()->Int32Constant(4)));
153 #elif defined(V8_TARGET_BIG_ENDIAN)
154 index_low = graph()->NewNode(machine()->Int32Add(), index,
155 graph()->NewNode(common()->Int32Constant(4)));
156 index_high = index;
157 #endif
158 }
159
LowerNode(Node * node)160 void Int64Lowering::LowerNode(Node* node) {
161 switch (node->opcode()) {
162 case IrOpcode::kInt64Constant: {
163 int64_t value = OpParameter<int64_t>(node->op());
164 Node* low_node = graph()->NewNode(
165 common()->Int32Constant(static_cast<int32_t>(value & 0xFFFFFFFF)));
166 Node* high_node = graph()->NewNode(
167 common()->Int32Constant(static_cast<int32_t>(value >> 32)));
168 ReplaceNode(node, low_node, high_node);
169 break;
170 }
171 case IrOpcode::kLoad:
172 case IrOpcode::kUnalignedLoad: {
173 MachineRepresentation rep;
174 if (node->opcode() == IrOpcode::kLoad) {
175 rep = LoadRepresentationOf(node->op()).representation();
176 } else {
177 DCHECK_EQ(IrOpcode::kUnalignedLoad, node->opcode());
178 rep = LoadRepresentationOf(node->op()).representation();
179 }
180
181 if (rep == MachineRepresentation::kWord64) {
182 Node* base = node->InputAt(0);
183 Node* index = node->InputAt(1);
184 Node* index_low;
185 Node* index_high;
186 GetIndexNodes(index, index_low, index_high);
187 const Operator* load_op;
188
189 if (node->opcode() == IrOpcode::kLoad) {
190 load_op = machine()->Load(MachineType::Int32());
191 } else {
192 DCHECK_EQ(IrOpcode::kUnalignedLoad, node->opcode());
193 load_op = machine()->UnalignedLoad(MachineType::Int32());
194 }
195
196 Node* high_node;
197 if (node->InputCount() > 2) {
198 Node* effect_high = node->InputAt(2);
199 Node* control_high = node->InputAt(3);
200 high_node = graph()->NewNode(load_op, base, index_high, effect_high,
201 control_high);
202 // change the effect change from old_node --> old_effect to
203 // old_node --> high_node --> old_effect.
204 node->ReplaceInput(2, high_node);
205 } else {
206 high_node = graph()->NewNode(load_op, base, index_high);
207 }
208 node->ReplaceInput(1, index_low);
209 NodeProperties::ChangeOp(node, load_op);
210 ReplaceNode(node, node, high_node);
211 } else {
212 DefaultLowering(node);
213 }
214 break;
215 }
216 case IrOpcode::kStore:
217 case IrOpcode::kUnalignedStore: {
218 MachineRepresentation rep;
219 if (node->opcode() == IrOpcode::kStore) {
220 rep = StoreRepresentationOf(node->op()).representation();
221 } else {
222 DCHECK_EQ(IrOpcode::kUnalignedStore, node->opcode());
223 rep = UnalignedStoreRepresentationOf(node->op());
224 }
225
226 if (rep == MachineRepresentation::kWord64) {
227 // We change the original store node to store the low word, and create
228 // a new store node to store the high word. The effect and control edges
229 // are copied from the original store to the new store node, the effect
230 // edge of the original store is redirected to the new store.
231 Node* base = node->InputAt(0);
232 Node* index = node->InputAt(1);
233 Node* index_low;
234 Node* index_high;
235 GetIndexNodes(index, index_low, index_high);
236 Node* value = node->InputAt(2);
237 DCHECK(HasReplacementLow(value));
238 DCHECK(HasReplacementHigh(value));
239
240 const Operator* store_op;
241 if (node->opcode() == IrOpcode::kStore) {
242 WriteBarrierKind write_barrier_kind =
243 StoreRepresentationOf(node->op()).write_barrier_kind();
244 store_op = machine()->Store(StoreRepresentation(
245 MachineRepresentation::kWord32, write_barrier_kind));
246 } else {
247 DCHECK_EQ(IrOpcode::kUnalignedStore, node->opcode());
248 store_op = machine()->UnalignedStore(MachineRepresentation::kWord32);
249 }
250
251 Node* high_node;
252 if (node->InputCount() > 3) {
253 Node* effect_high = node->InputAt(3);
254 Node* control_high = node->InputAt(4);
255 high_node = graph()->NewNode(store_op, base, index_high,
256 GetReplacementHigh(value), effect_high,
257 control_high);
258 node->ReplaceInput(3, high_node);
259
260 } else {
261 high_node = graph()->NewNode(store_op, base, index_high,
262 GetReplacementHigh(value));
263 }
264
265 node->ReplaceInput(1, index_low);
266 node->ReplaceInput(2, GetReplacementLow(value));
267 NodeProperties::ChangeOp(node, store_op);
268 ReplaceNode(node, node, high_node);
269 } else {
270 DefaultLowering(node, true);
271 }
272 break;
273 }
274 case IrOpcode::kStart: {
275 int parameter_count = GetParameterCountAfterLowering(signature());
276 // Only exchange the node if the parameter count actually changed.
277 if (parameter_count != static_cast<int>(signature()->parameter_count())) {
278 int delta =
279 parameter_count - static_cast<int>(signature()->parameter_count());
280 int new_output_count = node->op()->ValueOutputCount() + delta;
281 NodeProperties::ChangeOp(node, common()->Start(new_output_count));
282 }
283 break;
284 }
285 case IrOpcode::kParameter: {
286 DCHECK_EQ(1, node->InputCount());
287 // Only exchange the node if the parameter count actually changed. We do
288 // not even have to do the default lowering because the the start node,
289 // the only input of a parameter node, only changes if the parameter count
290 // changes.
291 if (GetParameterCountAfterLowering(signature()) !=
292 static_cast<int>(signature()->parameter_count())) {
293 int old_index = ParameterIndexOf(node->op());
294 // TODO(wasm): Make this part not wasm specific.
295 // Prevent special lowering of the instance parameter.
296 if (old_index == wasm::kWasmInstanceParameterIndex) {
297 DefaultLowering(node);
298 break;
299 }
300 // Adjust old_index to be compliant with the signature.
301 --old_index;
302 int new_index = GetParameterIndexAfterLowering(signature(), old_index);
303 // Adjust new_index to consider the instance parameter.
304 ++new_index;
305 NodeProperties::ChangeOp(node, common()->Parameter(new_index));
306
307 if (signature()->GetParam(old_index) ==
308 MachineRepresentation::kWord64) {
309 Node* high_node = graph()->NewNode(common()->Parameter(new_index + 1),
310 graph()->start());
311 ReplaceNode(node, node, high_node);
312 }
313 }
314 break;
315 }
316 case IrOpcode::kReturn: {
317 int input_count = node->InputCount();
318 DefaultLowering(node);
319 if (input_count != node->InputCount()) {
320 int new_return_count = GetReturnCountAfterLowering(signature());
321 if (static_cast<int>(signature()->return_count()) != new_return_count) {
322 NodeProperties::ChangeOp(node, common()->Return(new_return_count));
323 }
324 }
325 break;
326 }
327 case IrOpcode::kTailCall: {
328 auto call_descriptor =
329 const_cast<CallDescriptor*>(CallDescriptorOf(node->op()));
330 bool returns_require_lowering =
331 GetReturnCountAfterLowering(call_descriptor) !=
332 static_cast<int>(call_descriptor->ReturnCount());
333 if (DefaultLowering(node) || returns_require_lowering) {
334 // Tail calls do not have return values, so adjusting the call
335 // descriptor is enough.
336 auto new_descriptor = GetI32WasmCallDescriptor(zone(), call_descriptor);
337 NodeProperties::ChangeOp(node, common()->TailCall(new_descriptor));
338 }
339 break;
340 }
341 case IrOpcode::kCall: {
342 auto call_descriptor =
343 const_cast<CallDescriptor*>(CallDescriptorOf(node->op()));
344 bool returns_require_lowering =
345 GetReturnCountAfterLowering(call_descriptor) !=
346 static_cast<int>(call_descriptor->ReturnCount());
347 if (DefaultLowering(node) || returns_require_lowering) {
348 // We have to adjust the call descriptor.
349 NodeProperties::ChangeOp(node, common()->Call(GetI32WasmCallDescriptor(
350 zone(), call_descriptor)));
351 }
352 if (returns_require_lowering) {
353 size_t return_arity = call_descriptor->ReturnCount();
354 if (return_arity == 1) {
355 // We access the additional return values through projections.
356 ReplaceNodeWithProjections(node);
357 } else {
358 ZoneVector<Node*> projections(return_arity, zone());
359 NodeProperties::CollectValueProjections(node, projections.data(),
360 return_arity);
361 for (size_t old_index = 0, new_index = 0; old_index < return_arity;
362 ++old_index, ++new_index) {
363 Node* use_node = projections[old_index];
364 DCHECK_EQ(ProjectionIndexOf(use_node->op()), old_index);
365 DCHECK_EQ(GetReturnIndexAfterLowering(call_descriptor,
366 static_cast<int>(old_index)),
367 static_cast<int>(new_index));
368 if (new_index != old_index) {
369 NodeProperties::ChangeOp(
370 use_node, common()->Projection(new_index));
371 }
372 if (call_descriptor->GetReturnType(old_index).representation() ==
373 MachineRepresentation::kWord64) {
374 Node* high_node = graph()->NewNode(
375 common()->Projection(new_index + 1), node,
376 graph()->start());
377 ReplaceNode(use_node, use_node, high_node);
378 ++new_index;
379 }
380 }
381 }
382 }
383 break;
384 }
385 case IrOpcode::kWord64And: {
386 DCHECK_EQ(2, node->InputCount());
387 Node* left = node->InputAt(0);
388 Node* right = node->InputAt(1);
389
390 Node* low_node =
391 graph()->NewNode(machine()->Word32And(), GetReplacementLow(left),
392 GetReplacementLow(right));
393 Node* high_node =
394 graph()->NewNode(machine()->Word32And(), GetReplacementHigh(left),
395 GetReplacementHigh(right));
396 ReplaceNode(node, low_node, high_node);
397 break;
398 }
399 case IrOpcode::kTruncateInt64ToInt32: {
400 DCHECK_EQ(1, node->InputCount());
401 Node* input = node->InputAt(0);
402 ReplaceNode(node, GetReplacementLow(input), nullptr);
403 node->NullAllInputs();
404 break;
405 }
406 case IrOpcode::kInt64Add: {
407 DCHECK_EQ(2, node->InputCount());
408
409 Node* right = node->InputAt(1);
410 node->ReplaceInput(1, GetReplacementLow(right));
411 node->AppendInput(zone(), GetReplacementHigh(right));
412
413 Node* left = node->InputAt(0);
414 node->ReplaceInput(0, GetReplacementLow(left));
415 node->InsertInput(zone(), 1, GetReplacementHigh(left));
416
417 NodeProperties::ChangeOp(node, machine()->Int32PairAdd());
418 // We access the additional return values through projections.
419 ReplaceNodeWithProjections(node);
420 break;
421 }
422 case IrOpcode::kInt64Sub: {
423 DCHECK_EQ(2, node->InputCount());
424
425 Node* right = node->InputAt(1);
426 node->ReplaceInput(1, GetReplacementLow(right));
427 node->AppendInput(zone(), GetReplacementHigh(right));
428
429 Node* left = node->InputAt(0);
430 node->ReplaceInput(0, GetReplacementLow(left));
431 node->InsertInput(zone(), 1, GetReplacementHigh(left));
432
433 NodeProperties::ChangeOp(node, machine()->Int32PairSub());
434 // We access the additional return values through projections.
435 ReplaceNodeWithProjections(node);
436 break;
437 }
438 case IrOpcode::kInt64Mul: {
439 DCHECK_EQ(2, node->InputCount());
440
441 Node* right = node->InputAt(1);
442 node->ReplaceInput(1, GetReplacementLow(right));
443 node->AppendInput(zone(), GetReplacementHigh(right));
444
445 Node* left = node->InputAt(0);
446 node->ReplaceInput(0, GetReplacementLow(left));
447 node->InsertInput(zone(), 1, GetReplacementHigh(left));
448
449 NodeProperties::ChangeOp(node, machine()->Int32PairMul());
450 // We access the additional return values through projections.
451 ReplaceNodeWithProjections(node);
452 break;
453 }
454 case IrOpcode::kWord64Or: {
455 DCHECK_EQ(2, node->InputCount());
456 Node* left = node->InputAt(0);
457 Node* right = node->InputAt(1);
458
459 Node* low_node =
460 graph()->NewNode(machine()->Word32Or(), GetReplacementLow(left),
461 GetReplacementLow(right));
462 Node* high_node =
463 graph()->NewNode(machine()->Word32Or(), GetReplacementHigh(left),
464 GetReplacementHigh(right));
465 ReplaceNode(node, low_node, high_node);
466 break;
467 }
468 case IrOpcode::kWord64Xor: {
469 DCHECK_EQ(2, node->InputCount());
470 Node* left = node->InputAt(0);
471 Node* right = node->InputAt(1);
472
473 Node* low_node =
474 graph()->NewNode(machine()->Word32Xor(), GetReplacementLow(left),
475 GetReplacementLow(right));
476 Node* high_node =
477 graph()->NewNode(machine()->Word32Xor(), GetReplacementHigh(left),
478 GetReplacementHigh(right));
479 ReplaceNode(node, low_node, high_node);
480 break;
481 }
482 case IrOpcode::kWord64Shl: {
483 // TODO(turbofan): if the shift count >= 32, then we can set the low word
484 // of the output to 0 and just calculate the high word.
485 DCHECK_EQ(2, node->InputCount());
486 Node* shift = node->InputAt(1);
487 if (HasReplacementLow(shift)) {
488 // We do not have to care about the high word replacement, because
489 // the shift can only be between 0 and 63 anyways.
490 node->ReplaceInput(1, GetReplacementLow(shift));
491 }
492
493 Node* value = node->InputAt(0);
494 node->ReplaceInput(0, GetReplacementLow(value));
495 node->InsertInput(zone(), 1, GetReplacementHigh(value));
496
497 NodeProperties::ChangeOp(node, machine()->Word32PairShl());
498 // We access the additional return values through projections.
499 ReplaceNodeWithProjections(node);
500 break;
501 }
502 case IrOpcode::kWord64Shr: {
503 // TODO(turbofan): if the shift count >= 32, then we can set the low word
504 // of the output to 0 and just calculate the high word.
505 DCHECK_EQ(2, node->InputCount());
506 Node* shift = node->InputAt(1);
507 if (HasReplacementLow(shift)) {
508 // We do not have to care about the high word replacement, because
509 // the shift can only be between 0 and 63 anyways.
510 node->ReplaceInput(1, GetReplacementLow(shift));
511 }
512
513 Node* value = node->InputAt(0);
514 node->ReplaceInput(0, GetReplacementLow(value));
515 node->InsertInput(zone(), 1, GetReplacementHigh(value));
516
517 NodeProperties::ChangeOp(node, machine()->Word32PairShr());
518 // We access the additional return values through projections.
519 ReplaceNodeWithProjections(node);
520 break;
521 }
522 case IrOpcode::kWord64Sar: {
523 // TODO(turbofan): if the shift count >= 32, then we can set the low word
524 // of the output to 0 and just calculate the high word.
525 DCHECK_EQ(2, node->InputCount());
526 Node* shift = node->InputAt(1);
527 if (HasReplacementLow(shift)) {
528 // We do not have to care about the high word replacement, because
529 // the shift can only be between 0 and 63 anyways.
530 node->ReplaceInput(1, GetReplacementLow(shift));
531 }
532
533 Node* value = node->InputAt(0);
534 node->ReplaceInput(0, GetReplacementLow(value));
535 node->InsertInput(zone(), 1, GetReplacementHigh(value));
536
537 NodeProperties::ChangeOp(node, machine()->Word32PairSar());
538 // We access the additional return values through projections.
539 ReplaceNodeWithProjections(node);
540 break;
541 }
542 case IrOpcode::kWord64Equal: {
543 DCHECK_EQ(2, node->InputCount());
544 Node* left = node->InputAt(0);
545 Node* right = node->InputAt(1);
546
547 // TODO(wasm): Use explicit comparisons and && here?
548 Node* replacement = graph()->NewNode(
549 machine()->Word32Equal(),
550 graph()->NewNode(
551 machine()->Word32Or(),
552 graph()->NewNode(machine()->Word32Xor(), GetReplacementLow(left),
553 GetReplacementLow(right)),
554 graph()->NewNode(machine()->Word32Xor(), GetReplacementHigh(left),
555 GetReplacementHigh(right))),
556 graph()->NewNode(common()->Int32Constant(0)));
557
558 ReplaceNode(node, replacement, nullptr);
559 break;
560 }
561 case IrOpcode::kInt64LessThan: {
562 LowerComparison(node, machine()->Int32LessThan(),
563 machine()->Uint32LessThan());
564 break;
565 }
566 case IrOpcode::kInt64LessThanOrEqual: {
567 LowerComparison(node, machine()->Int32LessThan(),
568 machine()->Uint32LessThanOrEqual());
569 break;
570 }
571 case IrOpcode::kUint64LessThan: {
572 LowerComparison(node, machine()->Uint32LessThan(),
573 machine()->Uint32LessThan());
574 break;
575 }
576 case IrOpcode::kUint64LessThanOrEqual: {
577 LowerComparison(node, machine()->Uint32LessThan(),
578 machine()->Uint32LessThanOrEqual());
579 break;
580 }
581 case IrOpcode::kSignExtendWord32ToInt64:
582 case IrOpcode::kChangeInt32ToInt64: {
583 DCHECK_EQ(1, node->InputCount());
584 Node* input = node->InputAt(0);
585 if (HasReplacementLow(input)) {
586 input = GetReplacementLow(input);
587 }
588 // We use SAR to preserve the sign in the high word.
589 ReplaceNode(
590 node, input,
591 graph()->NewNode(machine()->Word32Sar(), input,
592 graph()->NewNode(common()->Int32Constant(31))));
593 node->NullAllInputs();
594 break;
595 }
596 case IrOpcode::kChangeUint32ToUint64: {
597 DCHECK_EQ(1, node->InputCount());
598 Node* input = node->InputAt(0);
599 if (HasReplacementLow(input)) {
600 input = GetReplacementLow(input);
601 }
602 ReplaceNode(node, input, graph()->NewNode(common()->Int32Constant(0)));
603 node->NullAllInputs();
604 break;
605 }
606 case IrOpcode::kBitcastInt64ToFloat64: {
607 DCHECK_EQ(1, node->InputCount());
608 Node* input = node->InputAt(0);
609 Node* stack_slot = graph()->NewNode(
610 machine()->StackSlot(MachineRepresentation::kWord64));
611
612 Node* store_high_word = graph()->NewNode(
613 machine()->Store(
614 StoreRepresentation(MachineRepresentation::kWord32,
615 WriteBarrierKind::kNoWriteBarrier)),
616 stack_slot,
617 graph()->NewNode(
618 common()->Int32Constant(kInt64UpperHalfMemoryOffset)),
619 GetReplacementHigh(input), graph()->start(), graph()->start());
620
621 Node* store_low_word = graph()->NewNode(
622 machine()->Store(
623 StoreRepresentation(MachineRepresentation::kWord32,
624 WriteBarrierKind::kNoWriteBarrier)),
625 stack_slot,
626 graph()->NewNode(
627 common()->Int32Constant(kInt64LowerHalfMemoryOffset)),
628 GetReplacementLow(input), store_high_word, graph()->start());
629
630 Node* load =
631 graph()->NewNode(machine()->Load(MachineType::Float64()), stack_slot,
632 graph()->NewNode(common()->Int32Constant(0)),
633 store_low_word, graph()->start());
634
635 ReplaceNode(node, load, nullptr);
636 break;
637 }
638 case IrOpcode::kBitcastFloat64ToInt64: {
639 DCHECK_EQ(1, node->InputCount());
640 Node* input = node->InputAt(0);
641 if (HasReplacementLow(input)) {
642 input = GetReplacementLow(input);
643 }
644 Node* stack_slot = graph()->NewNode(
645 machine()->StackSlot(MachineRepresentation::kWord64));
646 Node* store = graph()->NewNode(
647 machine()->Store(
648 StoreRepresentation(MachineRepresentation::kFloat64,
649 WriteBarrierKind::kNoWriteBarrier)),
650 stack_slot, graph()->NewNode(common()->Int32Constant(0)), input,
651 graph()->start(), graph()->start());
652
653 Node* high_node = graph()->NewNode(
654 machine()->Load(MachineType::Int32()), stack_slot,
655 graph()->NewNode(
656 common()->Int32Constant(kInt64UpperHalfMemoryOffset)),
657 store, graph()->start());
658
659 Node* low_node = graph()->NewNode(
660 machine()->Load(MachineType::Int32()), stack_slot,
661 graph()->NewNode(
662 common()->Int32Constant(kInt64LowerHalfMemoryOffset)),
663 store, graph()->start());
664 ReplaceNode(node, low_node, high_node);
665 break;
666 }
667 case IrOpcode::kWord64Ror: {
668 DCHECK_EQ(2, node->InputCount());
669 Node* input = node->InputAt(0);
670 Node* shift = HasReplacementLow(node->InputAt(1))
671 ? GetReplacementLow(node->InputAt(1))
672 : node->InputAt(1);
673 Int32Matcher m(shift);
674 if (m.HasValue()) {
675 // Precondition: 0 <= shift < 64.
676 int32_t shift_value = m.Value() & 0x3F;
677 if (shift_value == 0) {
678 ReplaceNode(node, GetReplacementLow(input),
679 GetReplacementHigh(input));
680 } else if (shift_value == 32) {
681 ReplaceNode(node, GetReplacementHigh(input),
682 GetReplacementLow(input));
683 } else {
684 Node* low_input;
685 Node* high_input;
686 if (shift_value < 32) {
687 low_input = GetReplacementLow(input);
688 high_input = GetReplacementHigh(input);
689 } else {
690 low_input = GetReplacementHigh(input);
691 high_input = GetReplacementLow(input);
692 }
693 int32_t masked_shift_value = shift_value & 0x1F;
694 Node* masked_shift =
695 graph()->NewNode(common()->Int32Constant(masked_shift_value));
696 Node* inv_shift = graph()->NewNode(
697 common()->Int32Constant(32 - masked_shift_value));
698
699 Node* low_node = graph()->NewNode(
700 machine()->Word32Or(),
701 graph()->NewNode(machine()->Word32Shr(), low_input, masked_shift),
702 graph()->NewNode(machine()->Word32Shl(), high_input, inv_shift));
703 Node* high_node = graph()->NewNode(
704 machine()->Word32Or(), graph()->NewNode(machine()->Word32Shr(),
705 high_input, masked_shift),
706 graph()->NewNode(machine()->Word32Shl(), low_input, inv_shift));
707 ReplaceNode(node, low_node, high_node);
708 }
709 } else {
710 Node* safe_shift = shift;
711 if (!machine()->Word32ShiftIsSafe()) {
712 safe_shift =
713 graph()->NewNode(machine()->Word32And(), shift,
714 graph()->NewNode(common()->Int32Constant(0x1F)));
715 }
716
717 // By creating this bit-mask with SAR and SHL we do not have to deal
718 // with shift == 0 as a special case.
719 Node* inv_mask = graph()->NewNode(
720 machine()->Word32Shl(),
721 graph()->NewNode(machine()->Word32Sar(),
722 graph()->NewNode(common()->Int32Constant(
723 std::numeric_limits<int32_t>::min())),
724 safe_shift),
725 graph()->NewNode(common()->Int32Constant(1)));
726
727 Node* bit_mask =
728 graph()->NewNode(machine()->Word32Xor(), inv_mask,
729 graph()->NewNode(common()->Int32Constant(-1)));
730
731 // We have to mask the shift value for this comparison. If
732 // !machine()->Word32ShiftIsSafe() then the masking should already be
733 // part of the graph.
734 Node* masked_shift6 = shift;
735 if (machine()->Word32ShiftIsSafe()) {
736 masked_shift6 =
737 graph()->NewNode(machine()->Word32And(), shift,
738 graph()->NewNode(common()->Int32Constant(0x3F)));
739 }
740
741 Diamond lt32(
742 graph(), common(),
743 graph()->NewNode(machine()->Int32LessThan(), masked_shift6,
744 graph()->NewNode(common()->Int32Constant(32))));
745
746 // The low word and the high word can be swapped either at the input or
747 // at the output. We swap the inputs so that shift does not have to be
748 // kept for so long in a register.
749 Node* input_low =
750 lt32.Phi(MachineRepresentation::kWord32, GetReplacementLow(input),
751 GetReplacementHigh(input));
752 Node* input_high =
753 lt32.Phi(MachineRepresentation::kWord32, GetReplacementHigh(input),
754 GetReplacementLow(input));
755
756 Node* rotate_low =
757 graph()->NewNode(machine()->Word32Ror(), input_low, safe_shift);
758 Node* rotate_high =
759 graph()->NewNode(machine()->Word32Ror(), input_high, safe_shift);
760
761 Node* low_node = graph()->NewNode(
762 machine()->Word32Or(),
763 graph()->NewNode(machine()->Word32And(), rotate_low, bit_mask),
764 graph()->NewNode(machine()->Word32And(), rotate_high, inv_mask));
765
766 Node* high_node = graph()->NewNode(
767 machine()->Word32Or(),
768 graph()->NewNode(machine()->Word32And(), rotate_high, bit_mask),
769 graph()->NewNode(machine()->Word32And(), rotate_low, inv_mask));
770
771 ReplaceNode(node, low_node, high_node);
772 }
773 break;
774 }
775 case IrOpcode::kWord64Clz: {
776 DCHECK_EQ(1, node->InputCount());
777 Node* input = node->InputAt(0);
778 Diamond d(
779 graph(), common(),
780 graph()->NewNode(machine()->Word32Equal(), GetReplacementHigh(input),
781 graph()->NewNode(common()->Int32Constant(0))));
782
783 Node* low_node = d.Phi(
784 MachineRepresentation::kWord32,
785 graph()->NewNode(machine()->Int32Add(),
786 graph()->NewNode(machine()->Word32Clz(),
787 GetReplacementLow(input)),
788 graph()->NewNode(common()->Int32Constant(32))),
789 graph()->NewNode(machine()->Word32Clz(), GetReplacementHigh(input)));
790 ReplaceNode(node, low_node, graph()->NewNode(common()->Int32Constant(0)));
791 break;
792 }
793 case IrOpcode::kWord64Ctz: {
794 DCHECK_EQ(1, node->InputCount());
795 DCHECK(machine()->Word32Ctz().IsSupported());
796 Node* input = node->InputAt(0);
797 Diamond d(
798 graph(), common(),
799 graph()->NewNode(machine()->Word32Equal(), GetReplacementLow(input),
800 graph()->NewNode(common()->Int32Constant(0))));
801 Node* low_node =
802 d.Phi(MachineRepresentation::kWord32,
803 graph()->NewNode(machine()->Int32Add(),
804 graph()->NewNode(machine()->Word32Ctz().op(),
805 GetReplacementHigh(input)),
806 graph()->NewNode(common()->Int32Constant(32))),
807 graph()->NewNode(machine()->Word32Ctz().op(),
808 GetReplacementLow(input)));
809 ReplaceNode(node, low_node, graph()->NewNode(common()->Int32Constant(0)));
810 break;
811 }
812 case IrOpcode::kWord64Popcnt: {
813 DCHECK_EQ(1, node->InputCount());
814 Node* input = node->InputAt(0);
815 // We assume that a Word64Popcnt node only has been created if
816 // Word32Popcnt is actually supported.
817 DCHECK(machine()->Word32Popcnt().IsSupported());
818 ReplaceNode(node, graph()->NewNode(
819 machine()->Int32Add(),
820 graph()->NewNode(machine()->Word32Popcnt().op(),
821 GetReplacementLow(input)),
822 graph()->NewNode(machine()->Word32Popcnt().op(),
823 GetReplacementHigh(input))),
824 graph()->NewNode(common()->Int32Constant(0)));
825 break;
826 }
827 case IrOpcode::kPhi: {
828 MachineRepresentation rep = PhiRepresentationOf(node->op());
829 if (rep == MachineRepresentation::kWord64) {
830 // The replacement nodes have already been created, we only have to
831 // replace placeholder nodes.
832 Node* low_node = GetReplacementLow(node);
833 Node* high_node = GetReplacementHigh(node);
834 for (int i = 0; i < node->op()->ValueInputCount(); i++) {
835 low_node->ReplaceInput(i, GetReplacementLow(node->InputAt(i)));
836 high_node->ReplaceInput(i, GetReplacementHigh(node->InputAt(i)));
837 }
838 } else {
839 DefaultLowering(node);
840 }
841 break;
842 }
843 case IrOpcode::kWord64ReverseBytes: {
844 Node* input = node->InputAt(0);
845 ReplaceNode(node,
846 graph()->NewNode(machine()->Word32ReverseBytes(),
847 GetReplacementHigh(input)),
848 graph()->NewNode(machine()->Word32ReverseBytes(),
849 GetReplacementLow(input)));
850 break;
851 }
852 case IrOpcode::kSignExtendWord8ToInt64: {
853 DCHECK_EQ(1, node->InputCount());
854 Node* input = node->InputAt(0);
855 if (HasReplacementLow(input)) {
856 input = GetReplacementLow(input);
857 }
858 // Sign extend low node to Int32
859 input = graph()->NewNode(machine()->SignExtendWord8ToInt32(), input);
860
861 // We use SAR to preserve the sign in the high word.
862 ReplaceNode(
863 node, input,
864 graph()->NewNode(machine()->Word32Sar(), input,
865 graph()->NewNode(common()->Int32Constant(31))));
866 node->NullAllInputs();
867 break;
868 }
869 case IrOpcode::kSignExtendWord16ToInt64: {
870 DCHECK_EQ(1, node->InputCount());
871 Node* input = node->InputAt(0);
872 if (HasReplacementLow(input)) {
873 input = GetReplacementLow(input);
874 }
875 // Sign extend low node to Int32
876 input = graph()->NewNode(machine()->SignExtendWord16ToInt32(), input);
877
878 // We use SAR to preserve the sign in the high word.
879 ReplaceNode(
880 node, input,
881 graph()->NewNode(machine()->Word32Sar(), input,
882 graph()->NewNode(common()->Int32Constant(31))));
883 node->NullAllInputs();
884 break;
885 }
886 case IrOpcode::kWord64AtomicLoad: {
887 DCHECK_EQ(4, node->InputCount());
888 MachineType type = AtomicOpType(node->op());
889 if (type == MachineType::Uint64()) {
890 NodeProperties::ChangeOp(node, machine()->Word32AtomicPairLoad());
891 ReplaceNodeWithProjections(node);
892 } else {
893 NodeProperties::ChangeOp(node, machine()->Word32AtomicLoad(type));
894 ReplaceNode(node, node, graph()->NewNode(common()->Int32Constant(0)));
895 }
896 break;
897 }
898 case IrOpcode::kWord64AtomicStore: {
899 DCHECK_EQ(5, node->InputCount());
900 MachineRepresentation rep = AtomicStoreRepresentationOf(node->op());
901 if (rep == MachineRepresentation::kWord64) {
902 Node* value = node->InputAt(2);
903 node->ReplaceInput(2, GetReplacementLow(value));
904 node->InsertInput(zone(), 3, GetReplacementHigh(value));
905 NodeProperties::ChangeOp(node, machine()->Word32AtomicPairStore());
906 } else {
907 DefaultLowering(node, true);
908 NodeProperties::ChangeOp(node, machine()->Word32AtomicStore(rep));
909 }
910 break;
911 }
912 #define ATOMIC_CASE(name) \
913 case IrOpcode::kWord64Atomic##name: { \
914 MachineType type = AtomicOpType(node->op()); \
915 if (type == MachineType::Uint64()) { \
916 LowerWord64AtomicBinop(node, machine()->Word32AtomicPair##name()); \
917 } else { \
918 LowerWord64AtomicNarrowOp(node, \
919 machine()->Word64AtomicNarrow##name(type)); \
920 } \
921 break; \
922 }
923 ATOMIC_CASE(Add)
924 ATOMIC_CASE(Sub)
925 ATOMIC_CASE(And)
926 ATOMIC_CASE(Or)
927 ATOMIC_CASE(Xor)
928 ATOMIC_CASE(Exchange)
929 #undef ATOMIC_CASE
930 case IrOpcode::kWord64AtomicCompareExchange: {
931 MachineType type = AtomicOpType(node->op());
932 if (type == MachineType::Uint64()) {
933 Node* old_value = node->InputAt(2);
934 Node* new_value = node->InputAt(3);
935 node->ReplaceInput(2, GetReplacementLow(old_value));
936 node->ReplaceInput(3, GetReplacementHigh(old_value));
937 node->InsertInput(zone(), 4, GetReplacementLow(new_value));
938 node->InsertInput(zone(), 5, GetReplacementHigh(new_value));
939 NodeProperties::ChangeOp(node,
940 machine()->Word32AtomicPairCompareExchange());
941 ReplaceNodeWithProjections(node);
942 } else {
943 LowerWord64AtomicNarrowOp(
944 node, machine()->Word64AtomicNarrowCompareExchange(type));
945 }
946 break;
947 }
948
949 default: { DefaultLowering(node); }
950 }
951 } // NOLINT(readability/fn_size)
952
LowerComparison(Node * node,const Operator * high_word_op,const Operator * low_word_op)953 void Int64Lowering::LowerComparison(Node* node, const Operator* high_word_op,
954 const Operator* low_word_op) {
955 DCHECK_EQ(2, node->InputCount());
956 Node* left = node->InputAt(0);
957 Node* right = node->InputAt(1);
958 Node* replacement = graph()->NewNode(
959 machine()->Word32Or(),
960 graph()->NewNode(high_word_op, GetReplacementHigh(left),
961 GetReplacementHigh(right)),
962 graph()->NewNode(
963 machine()->Word32And(),
964 graph()->NewNode(machine()->Word32Equal(), GetReplacementHigh(left),
965 GetReplacementHigh(right)),
966 graph()->NewNode(low_word_op, GetReplacementLow(left),
967 GetReplacementLow(right))));
968
969 ReplaceNode(node, replacement, nullptr);
970 }
971
DefaultLowering(Node * node,bool low_word_only)972 bool Int64Lowering::DefaultLowering(Node* node, bool low_word_only) {
973 bool something_changed = false;
974 for (int i = NodeProperties::PastValueIndex(node) - 1; i >= 0; i--) {
975 Node* input = node->InputAt(i);
976 if (HasReplacementLow(input)) {
977 something_changed = true;
978 node->ReplaceInput(i, GetReplacementLow(input));
979 }
980 if (!low_word_only && HasReplacementHigh(input)) {
981 something_changed = true;
982 node->InsertInput(zone(), i + 1, GetReplacementHigh(input));
983 }
984 }
985 return something_changed;
986 }
987
ReplaceNode(Node * old,Node * new_low,Node * new_high)988 void Int64Lowering::ReplaceNode(Node* old, Node* new_low, Node* new_high) {
989 // if new_low == nullptr, then also new_high == nullptr.
990 DCHECK(new_low != nullptr || new_high == nullptr);
991 replacements_[old->id()].low = new_low;
992 replacements_[old->id()].high = new_high;
993 }
994
HasReplacementLow(Node * node)995 bool Int64Lowering::HasReplacementLow(Node* node) {
996 return replacements_[node->id()].low != nullptr;
997 }
998
GetReplacementLow(Node * node)999 Node* Int64Lowering::GetReplacementLow(Node* node) {
1000 Node* result = replacements_[node->id()].low;
1001 DCHECK(result);
1002 return result;
1003 }
1004
HasReplacementHigh(Node * node)1005 bool Int64Lowering::HasReplacementHigh(Node* node) {
1006 return replacements_[node->id()].high != nullptr;
1007 }
1008
GetReplacementHigh(Node * node)1009 Node* Int64Lowering::GetReplacementHigh(Node* node) {
1010 Node* result = replacements_[node->id()].high;
1011 DCHECK(result);
1012 return result;
1013 }
1014
PreparePhiReplacement(Node * phi)1015 void Int64Lowering::PreparePhiReplacement(Node* phi) {
1016 MachineRepresentation rep = PhiRepresentationOf(phi->op());
1017 if (rep == MachineRepresentation::kWord64) {
1018 // We have to create the replacements for a phi node before we actually
1019 // lower the phi to break potential cycles in the graph. The replacements of
1020 // input nodes do not exist yet, so we use a placeholder node to pass the
1021 // graph verifier.
1022 int value_count = phi->op()->ValueInputCount();
1023 Node** inputs_low = zone()->NewArray<Node*>(value_count + 1);
1024 Node** inputs_high = zone()->NewArray<Node*>(value_count + 1);
1025 for (int i = 0; i < value_count; i++) {
1026 inputs_low[i] = placeholder_;
1027 inputs_high[i] = placeholder_;
1028 }
1029 inputs_low[value_count] = NodeProperties::GetControlInput(phi, 0);
1030 inputs_high[value_count] = NodeProperties::GetControlInput(phi, 0);
1031 ReplaceNode(phi,
1032 graph()->NewNode(
1033 common()->Phi(MachineRepresentation::kWord32, value_count),
1034 value_count + 1, inputs_low, false),
1035 graph()->NewNode(
1036 common()->Phi(MachineRepresentation::kWord32, value_count),
1037 value_count + 1, inputs_high, false));
1038 }
1039 }
1040
ReplaceNodeWithProjections(Node * node)1041 void Int64Lowering::ReplaceNodeWithProjections(Node* node) {
1042 DCHECK(node != nullptr);
1043 Node* low_node =
1044 graph()->NewNode(common()->Projection(0), node, graph()->start());
1045 Node* high_node =
1046 graph()->NewNode(common()->Projection(1), node, graph()->start());
1047 ReplaceNode(node, low_node, high_node);
1048 }
1049
1050 } // namespace compiler
1051 } // namespace internal
1052 } // namespace v8
1053