1 // Copyright 2014 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include <algorithm>
6
7 #include "src/base/iterator.h"
8 #include "src/base/logging.h"
9 #include "src/base/overflowing-math.h"
10 #include "src/base/platform/wrappers.h"
11 #include "src/codegen/cpu-features.h"
12 #include "src/codegen/machine-type.h"
13 #include "src/compiler/backend/instruction-codes.h"
14 #include "src/compiler/backend/instruction-selector-impl.h"
15 #include "src/compiler/backend/instruction.h"
16 #include "src/compiler/machine-operator.h"
17 #include "src/compiler/node-matchers.h"
18 #include "src/compiler/node-properties.h"
19 #include "src/compiler/opcodes.h"
20 #include "src/roots/roots-inl.h"
21
22 #if V8_ENABLE_WEBASSEMBLY
23 #include "src/wasm/simd-shuffle.h"
24 #endif // V8_ENABLE_WEBASSEMBLY
25
26 namespace v8 {
27 namespace internal {
28 namespace compiler {
29
30 // Adds X64-specific methods for generating operands.
31 class X64OperandGenerator final : public OperandGenerator {
32 public:
X64OperandGenerator(InstructionSelector * selector)33 explicit X64OperandGenerator(InstructionSelector* selector)
34 : OperandGenerator(selector) {}
35
CanBeImmediate(Node * node)36 bool CanBeImmediate(Node* node) {
37 switch (node->opcode()) {
38 case IrOpcode::kInt32Constant:
39 case IrOpcode::kRelocatableInt32Constant: {
40 const int32_t value = OpParameter<int32_t>(node->op());
41 // int32_t min will overflow if displacement mode is
42 // kNegativeDisplacement.
43 return value != std::numeric_limits<int32_t>::min();
44 }
45 case IrOpcode::kInt64Constant: {
46 const int64_t value = OpParameter<int64_t>(node->op());
47 return std::numeric_limits<int32_t>::min() < value &&
48 value <= std::numeric_limits<int32_t>::max();
49 }
50 case IrOpcode::kNumberConstant: {
51 const double value = OpParameter<double>(node->op());
52 return bit_cast<int64_t>(value) == 0;
53 }
54 default:
55 return false;
56 }
57 }
58
GetImmediateIntegerValue(Node * node)59 int32_t GetImmediateIntegerValue(Node* node) {
60 DCHECK(CanBeImmediate(node));
61 if (node->opcode() == IrOpcode::kInt32Constant) {
62 return OpParameter<int32_t>(node->op());
63 }
64 DCHECK_EQ(IrOpcode::kInt64Constant, node->opcode());
65 return static_cast<int32_t>(OpParameter<int64_t>(node->op()));
66 }
67
CanBeMemoryOperand(InstructionCode opcode,Node * node,Node * input,int effect_level)68 bool CanBeMemoryOperand(InstructionCode opcode, Node* node, Node* input,
69 int effect_level) {
70 if ((input->opcode() != IrOpcode::kLoad &&
71 input->opcode() != IrOpcode::kLoadImmutable) ||
72 !selector()->CanCover(node, input)) {
73 return false;
74 }
75 if (effect_level != selector()->GetEffectLevel(input)) {
76 return false;
77 }
78 MachineRepresentation rep =
79 LoadRepresentationOf(input->op()).representation();
80 switch (opcode) {
81 case kX64And:
82 case kX64Or:
83 case kX64Xor:
84 case kX64Add:
85 case kX64Sub:
86 case kX64Push:
87 case kX64Cmp:
88 case kX64Test:
89 // When pointer compression is enabled 64-bit memory operands can't be
90 // used for tagged values.
91 return rep == MachineRepresentation::kWord64 ||
92 (!COMPRESS_POINTERS_BOOL && IsAnyTagged(rep));
93 case kX64And32:
94 case kX64Or32:
95 case kX64Xor32:
96 case kX64Add32:
97 case kX64Sub32:
98 case kX64Cmp32:
99 case kX64Test32:
100 // When pointer compression is enabled 32-bit memory operands can be
101 // used for tagged values.
102 return rep == MachineRepresentation::kWord32 ||
103 (COMPRESS_POINTERS_BOOL &&
104 (IsAnyTagged(rep) || IsAnyCompressed(rep)));
105 case kAVXFloat64Add:
106 case kAVXFloat64Sub:
107 case kAVXFloat64Mul:
108 DCHECK_EQ(MachineRepresentation::kFloat64, rep);
109 return true;
110 case kAVXFloat32Add:
111 case kAVXFloat32Sub:
112 case kAVXFloat32Mul:
113 DCHECK_EQ(MachineRepresentation::kFloat32, rep);
114 return true;
115 case kX64Cmp16:
116 case kX64Test16:
117 return rep == MachineRepresentation::kWord16;
118 case kX64Cmp8:
119 case kX64Test8:
120 return rep == MachineRepresentation::kWord8;
121 default:
122 break;
123 }
124 return false;
125 }
126
GenerateMemoryOperandInputs(Node * index,int scale_exponent,Node * base,Node * displacement,DisplacementMode displacement_mode,InstructionOperand inputs[],size_t * input_count,RegisterUseKind reg_kind=RegisterUseKind::kUseRegister)127 AddressingMode GenerateMemoryOperandInputs(
128 Node* index, int scale_exponent, Node* base, Node* displacement,
129 DisplacementMode displacement_mode, InstructionOperand inputs[],
130 size_t* input_count,
131 RegisterUseKind reg_kind = RegisterUseKind::kUseRegister) {
132 AddressingMode mode = kMode_MRI;
133 if (base != nullptr && (index != nullptr || displacement != nullptr)) {
134 if (base->opcode() == IrOpcode::kInt32Constant &&
135 OpParameter<int32_t>(base->op()) == 0) {
136 base = nullptr;
137 } else if (base->opcode() == IrOpcode::kInt64Constant &&
138 OpParameter<int64_t>(base->op()) == 0) {
139 base = nullptr;
140 }
141 }
142 if (base != nullptr) {
143 inputs[(*input_count)++] = UseRegister(base, reg_kind);
144 if (index != nullptr) {
145 DCHECK(scale_exponent >= 0 && scale_exponent <= 3);
146 inputs[(*input_count)++] = UseRegister(index, reg_kind);
147 if (displacement != nullptr) {
148 inputs[(*input_count)++] = displacement_mode == kNegativeDisplacement
149 ? UseNegatedImmediate(displacement)
150 : UseImmediate(displacement);
151 static const AddressingMode kMRnI_modes[] = {kMode_MR1I, kMode_MR2I,
152 kMode_MR4I, kMode_MR8I};
153 mode = kMRnI_modes[scale_exponent];
154 } else {
155 static const AddressingMode kMRn_modes[] = {kMode_MR1, kMode_MR2,
156 kMode_MR4, kMode_MR8};
157 mode = kMRn_modes[scale_exponent];
158 }
159 } else {
160 if (displacement == nullptr) {
161 mode = kMode_MR;
162 } else {
163 inputs[(*input_count)++] = displacement_mode == kNegativeDisplacement
164 ? UseNegatedImmediate(displacement)
165 : UseImmediate(displacement);
166 mode = kMode_MRI;
167 }
168 }
169 } else {
170 DCHECK(scale_exponent >= 0 && scale_exponent <= 3);
171 if (displacement != nullptr) {
172 if (index == nullptr) {
173 inputs[(*input_count)++] = UseRegister(displacement, reg_kind);
174 mode = kMode_MR;
175 } else {
176 inputs[(*input_count)++] = UseRegister(index, reg_kind);
177 inputs[(*input_count)++] = displacement_mode == kNegativeDisplacement
178 ? UseNegatedImmediate(displacement)
179 : UseImmediate(displacement);
180 static const AddressingMode kMnI_modes[] = {kMode_MRI, kMode_M2I,
181 kMode_M4I, kMode_M8I};
182 mode = kMnI_modes[scale_exponent];
183 }
184 } else {
185 inputs[(*input_count)++] = UseRegister(index, reg_kind);
186 static const AddressingMode kMn_modes[] = {kMode_MR, kMode_MR1,
187 kMode_M4, kMode_M8};
188 mode = kMn_modes[scale_exponent];
189 if (mode == kMode_MR1) {
190 // [%r1 + %r1*1] has a smaller encoding than [%r1*2+0]
191 inputs[(*input_count)++] = UseRegister(index, reg_kind);
192 }
193 }
194 }
195 return mode;
196 }
197
GetEffectiveAddressMemoryOperand(Node * operand,InstructionOperand inputs[],size_t * input_count,RegisterUseKind reg_kind=RegisterUseKind::kUseRegister)198 AddressingMode GetEffectiveAddressMemoryOperand(
199 Node* operand, InstructionOperand inputs[], size_t* input_count,
200 RegisterUseKind reg_kind = RegisterUseKind::kUseRegister) {
201 {
202 LoadMatcher<ExternalReferenceMatcher> m(operand);
203 if (m.index().HasResolvedValue() && m.object().HasResolvedValue() &&
204 selector()->CanAddressRelativeToRootsRegister(
205 m.object().ResolvedValue())) {
206 ptrdiff_t const delta =
207 m.index().ResolvedValue() +
208 TurboAssemblerBase::RootRegisterOffsetForExternalReference(
209 selector()->isolate(), m.object().ResolvedValue());
210 if (is_int32(delta)) {
211 inputs[(*input_count)++] = TempImmediate(static_cast<int32_t>(delta));
212 return kMode_Root;
213 }
214 }
215 }
216 BaseWithIndexAndDisplacement64Matcher m(operand, AddressOption::kAllowAll);
217 DCHECK(m.matches());
218 if (m.displacement() == nullptr || CanBeImmediate(m.displacement())) {
219 return GenerateMemoryOperandInputs(
220 m.index(), m.scale(), m.base(), m.displacement(),
221 m.displacement_mode(), inputs, input_count, reg_kind);
222 } else if (m.base() == nullptr &&
223 m.displacement_mode() == kPositiveDisplacement) {
224 // The displacement cannot be an immediate, but we can use the
225 // displacement as base instead and still benefit from addressing
226 // modes for the scale.
227 return GenerateMemoryOperandInputs(m.index(), m.scale(), m.displacement(),
228 nullptr, m.displacement_mode(), inputs,
229 input_count, reg_kind);
230 } else {
231 inputs[(*input_count)++] = UseRegister(operand->InputAt(0), reg_kind);
232 inputs[(*input_count)++] = UseRegister(operand->InputAt(1), reg_kind);
233 return kMode_MR1;
234 }
235 }
236
GetEffectiveIndexOperand(Node * index,AddressingMode * mode)237 InstructionOperand GetEffectiveIndexOperand(Node* index,
238 AddressingMode* mode) {
239 if (CanBeImmediate(index)) {
240 *mode = kMode_MRI;
241 return UseImmediate(index);
242 } else {
243 *mode = kMode_MR1;
244 return UseUniqueRegister(index);
245 }
246 }
247
CanBeBetterLeftOperand(Node * node) const248 bool CanBeBetterLeftOperand(Node* node) const {
249 return !selector()->IsLive(node);
250 }
251 };
252
253 namespace {
254
GetLoadOpcode(LoadRepresentation load_rep)255 ArchOpcode GetLoadOpcode(LoadRepresentation load_rep) {
256 ArchOpcode opcode;
257 switch (load_rep.representation()) {
258 case MachineRepresentation::kFloat32:
259 opcode = kX64Movss;
260 break;
261 case MachineRepresentation::kFloat64:
262 opcode = kX64Movsd;
263 break;
264 case MachineRepresentation::kBit: // Fall through.
265 case MachineRepresentation::kWord8:
266 opcode = load_rep.IsSigned() ? kX64Movsxbl : kX64Movzxbl;
267 break;
268 case MachineRepresentation::kWord16:
269 opcode = load_rep.IsSigned() ? kX64Movsxwl : kX64Movzxwl;
270 break;
271 case MachineRepresentation::kWord32:
272 opcode = kX64Movl;
273 break;
274 case MachineRepresentation::kCompressedPointer: // Fall through.
275 case MachineRepresentation::kCompressed:
276 #ifdef V8_COMPRESS_POINTERS
277 opcode = kX64Movl;
278 break;
279 #else
280 UNREACHABLE();
281 #endif
282 #ifdef V8_COMPRESS_POINTERS
283 case MachineRepresentation::kTaggedSigned:
284 opcode = kX64MovqDecompressTaggedSigned;
285 break;
286 case MachineRepresentation::kTaggedPointer:
287 opcode = kX64MovqDecompressTaggedPointer;
288 break;
289 case MachineRepresentation::kTagged:
290 opcode = kX64MovqDecompressAnyTagged;
291 break;
292 #else
293 case MachineRepresentation::kTaggedSigned: // Fall through.
294 case MachineRepresentation::kTaggedPointer: // Fall through.
295 case MachineRepresentation::kTagged: // Fall through.
296 #endif
297 case MachineRepresentation::kWord64:
298 opcode = kX64Movq;
299 break;
300 case MachineRepresentation::kSandboxedPointer:
301 opcode = kX64MovqDecodeSandboxedPointer;
302 break;
303 case MachineRepresentation::kSimd128:
304 opcode = kX64Movdqu;
305 break;
306 case MachineRepresentation::kNone: // Fall through.
307 case MachineRepresentation::kMapWord:
308 UNREACHABLE();
309 }
310 return opcode;
311 }
312
GetStoreOpcode(StoreRepresentation store_rep)313 ArchOpcode GetStoreOpcode(StoreRepresentation store_rep) {
314 switch (store_rep.representation()) {
315 case MachineRepresentation::kFloat32:
316 return kX64Movss;
317 case MachineRepresentation::kFloat64:
318 return kX64Movsd;
319 case MachineRepresentation::kBit: // Fall through.
320 case MachineRepresentation::kWord8:
321 return kX64Movb;
322 case MachineRepresentation::kWord16:
323 return kX64Movw;
324 case MachineRepresentation::kWord32:
325 return kX64Movl;
326 case MachineRepresentation::kCompressedPointer: // Fall through.
327 case MachineRepresentation::kCompressed:
328 #ifdef V8_COMPRESS_POINTERS
329 return kX64MovqCompressTagged;
330 #else
331 UNREACHABLE();
332 #endif
333 case MachineRepresentation::kTaggedSigned: // Fall through.
334 case MachineRepresentation::kTaggedPointer: // Fall through.
335 case MachineRepresentation::kTagged:
336 return kX64MovqCompressTagged;
337 case MachineRepresentation::kWord64:
338 return kX64Movq;
339 case MachineRepresentation::kSandboxedPointer:
340 return kX64MovqEncodeSandboxedPointer;
341 case MachineRepresentation::kSimd128:
342 return kX64Movdqu;
343 case MachineRepresentation::kNone: // Fall through.
344 case MachineRepresentation::kMapWord:
345 UNREACHABLE();
346 }
347 UNREACHABLE();
348 }
349
GetSeqCstStoreOpcode(StoreRepresentation store_rep)350 ArchOpcode GetSeqCstStoreOpcode(StoreRepresentation store_rep) {
351 switch (store_rep.representation()) {
352 case MachineRepresentation::kWord8:
353 return kAtomicStoreWord8;
354 case MachineRepresentation::kWord16:
355 return kAtomicStoreWord16;
356 case MachineRepresentation::kWord32:
357 return kAtomicStoreWord32;
358 case MachineRepresentation::kWord64:
359 return kX64Word64AtomicStoreWord64;
360 case MachineRepresentation::kTaggedSigned: // Fall through.
361 case MachineRepresentation::kTaggedPointer: // Fall through.
362 case MachineRepresentation::kTagged:
363 if (COMPRESS_POINTERS_BOOL) return kAtomicStoreWord32;
364 return kX64Word64AtomicStoreWord64;
365 case MachineRepresentation::kCompressedPointer: // Fall through.
366 case MachineRepresentation::kCompressed:
367 CHECK(COMPRESS_POINTERS_BOOL);
368 return kAtomicStoreWord32;
369 default:
370 UNREACHABLE();
371 }
372 }
373
374 } // namespace
375
VisitStackSlot(Node * node)376 void InstructionSelector::VisitStackSlot(Node* node) {
377 StackSlotRepresentation rep = StackSlotRepresentationOf(node->op());
378 int slot = frame_->AllocateSpillSlot(rep.size(), rep.alignment());
379 OperandGenerator g(this);
380
381 Emit(kArchStackSlot, g.DefineAsRegister(node),
382 sequence()->AddImmediate(Constant(slot)), 0, nullptr);
383 }
384
VisitAbortCSADcheck(Node * node)385 void InstructionSelector::VisitAbortCSADcheck(Node* node) {
386 X64OperandGenerator g(this);
387 Emit(kArchAbortCSADcheck, g.NoOutput(), g.UseFixed(node->InputAt(0), rdx));
388 }
389
VisitLoadLane(Node * node)390 void InstructionSelector::VisitLoadLane(Node* node) {
391 LoadLaneParameters params = LoadLaneParametersOf(node->op());
392 InstructionCode opcode = kArchNop;
393 if (params.rep == MachineType::Int8()) {
394 opcode = kX64Pinsrb;
395 } else if (params.rep == MachineType::Int16()) {
396 opcode = kX64Pinsrw;
397 } else if (params.rep == MachineType::Int32()) {
398 opcode = kX64Pinsrd;
399 } else if (params.rep == MachineType::Int64()) {
400 opcode = kX64Pinsrq;
401 } else {
402 UNREACHABLE();
403 }
404
405 X64OperandGenerator g(this);
406 InstructionOperand outputs[] = {g.DefineAsRegister(node)};
407 // Input 0 is value node, 1 is lane idx, and GetEffectiveAddressMemoryOperand
408 // uses up to 3 inputs. This ordering is consistent with other operations that
409 // use the same opcode.
410 InstructionOperand inputs[5];
411 size_t input_count = 0;
412
413 inputs[input_count++] = g.UseRegister(node->InputAt(2));
414 inputs[input_count++] = g.UseImmediate(params.laneidx);
415
416 AddressingMode mode =
417 g.GetEffectiveAddressMemoryOperand(node, inputs, &input_count);
418 opcode |= AddressingModeField::encode(mode);
419
420 DCHECK_GE(5, input_count);
421
422 // x64 supports unaligned loads.
423 DCHECK_NE(params.kind, MemoryAccessKind::kUnaligned);
424 if (params.kind == MemoryAccessKind::kProtected) {
425 opcode |= AccessModeField::encode(kMemoryAccessProtected);
426 }
427 Emit(opcode, 1, outputs, input_count, inputs);
428 }
429
VisitLoadTransform(Node * node)430 void InstructionSelector::VisitLoadTransform(Node* node) {
431 LoadTransformParameters params = LoadTransformParametersOf(node->op());
432 ArchOpcode opcode;
433 switch (params.transformation) {
434 case LoadTransformation::kS128Load8Splat:
435 opcode = kX64S128Load8Splat;
436 break;
437 case LoadTransformation::kS128Load16Splat:
438 opcode = kX64S128Load16Splat;
439 break;
440 case LoadTransformation::kS128Load32Splat:
441 opcode = kX64S128Load32Splat;
442 break;
443 case LoadTransformation::kS128Load64Splat:
444 opcode = kX64S128Load64Splat;
445 break;
446 case LoadTransformation::kS128Load8x8S:
447 opcode = kX64S128Load8x8S;
448 break;
449 case LoadTransformation::kS128Load8x8U:
450 opcode = kX64S128Load8x8U;
451 break;
452 case LoadTransformation::kS128Load16x4S:
453 opcode = kX64S128Load16x4S;
454 break;
455 case LoadTransformation::kS128Load16x4U:
456 opcode = kX64S128Load16x4U;
457 break;
458 case LoadTransformation::kS128Load32x2S:
459 opcode = kX64S128Load32x2S;
460 break;
461 case LoadTransformation::kS128Load32x2U:
462 opcode = kX64S128Load32x2U;
463 break;
464 case LoadTransformation::kS128Load32Zero:
465 opcode = kX64Movss;
466 break;
467 case LoadTransformation::kS128Load64Zero:
468 opcode = kX64Movsd;
469 break;
470 default:
471 UNREACHABLE();
472 }
473 // x64 supports unaligned loads
474 DCHECK_NE(params.kind, MemoryAccessKind::kUnaligned);
475 InstructionCode code = opcode;
476 if (params.kind == MemoryAccessKind::kProtected) {
477 code |= AccessModeField::encode(kMemoryAccessProtected);
478 }
479 VisitLoad(node, node, code);
480 }
481
VisitLoad(Node * node,Node * value,InstructionCode opcode)482 void InstructionSelector::VisitLoad(Node* node, Node* value,
483 InstructionCode opcode) {
484 X64OperandGenerator g(this);
485 #ifdef V8_IS_TSAN
486 // On TSAN builds we require one scratch register. Because of this we also
487 // have to modify the inputs to take into account possible aliasing and use
488 // UseUniqueRegister which is not required for non-TSAN builds.
489 InstructionOperand temps[] = {g.TempRegister()};
490 size_t temp_count = arraysize(temps);
491 auto reg_kind = OperandGenerator::RegisterUseKind::kUseUniqueRegister;
492 #else
493 InstructionOperand* temps = nullptr;
494 size_t temp_count = 0;
495 auto reg_kind = OperandGenerator::RegisterUseKind::kUseRegister;
496 #endif // V8_IS_TSAN
497 InstructionOperand outputs[] = {g.DefineAsRegister(node)};
498 InstructionOperand inputs[3];
499 size_t input_count = 0;
500 AddressingMode mode =
501 g.GetEffectiveAddressMemoryOperand(value, inputs, &input_count, reg_kind);
502 InstructionCode code = opcode | AddressingModeField::encode(mode);
503 if (node->opcode() == IrOpcode::kProtectedLoad) {
504 code |= AccessModeField::encode(kMemoryAccessProtected);
505 }
506 Emit(code, 1, outputs, input_count, inputs, temp_count, temps);
507 }
508
VisitLoad(Node * node)509 void InstructionSelector::VisitLoad(Node* node) {
510 LoadRepresentation load_rep = LoadRepresentationOf(node->op());
511 DCHECK(!load_rep.IsMapWord());
512 VisitLoad(node, node, GetLoadOpcode(load_rep));
513 }
514
VisitProtectedLoad(Node * node)515 void InstructionSelector::VisitProtectedLoad(Node* node) { VisitLoad(node); }
516
517 namespace {
518
519 // Shared routine for Word32/Word64 Atomic Exchange
VisitAtomicExchange(InstructionSelector * selector,Node * node,ArchOpcode opcode,AtomicWidth width)520 void VisitAtomicExchange(InstructionSelector* selector, Node* node,
521 ArchOpcode opcode, AtomicWidth width) {
522 X64OperandGenerator g(selector);
523 Node* base = node->InputAt(0);
524 Node* index = node->InputAt(1);
525 Node* value = node->InputAt(2);
526 AddressingMode addressing_mode;
527 InstructionOperand inputs[] = {
528 g.UseUniqueRegister(value), g.UseUniqueRegister(base),
529 g.GetEffectiveIndexOperand(index, &addressing_mode)};
530 InstructionOperand outputs[] = {g.DefineSameAsFirst(node)};
531 InstructionCode code = opcode | AddressingModeField::encode(addressing_mode) |
532 AtomicWidthField::encode(width);
533 selector->Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs);
534 }
535
VisitStoreCommon(InstructionSelector * selector,Node * node,StoreRepresentation store_rep,base::Optional<AtomicMemoryOrder> atomic_order)536 void VisitStoreCommon(InstructionSelector* selector, Node* node,
537 StoreRepresentation store_rep,
538 base::Optional<AtomicMemoryOrder> atomic_order) {
539 X64OperandGenerator g(selector);
540 Node* base = node->InputAt(0);
541 Node* index = node->InputAt(1);
542 Node* value = node->InputAt(2);
543
544 DCHECK_NE(store_rep.representation(), MachineRepresentation::kMapWord);
545 WriteBarrierKind write_barrier_kind = store_rep.write_barrier_kind();
546 const bool is_seqcst =
547 atomic_order && *atomic_order == AtomicMemoryOrder::kSeqCst;
548
549 if (FLAG_enable_unconditional_write_barriers &&
550 CanBeTaggedOrCompressedPointer(store_rep.representation())) {
551 write_barrier_kind = kFullWriteBarrier;
552 }
553
554 if (write_barrier_kind != kNoWriteBarrier && !FLAG_disable_write_barriers) {
555 DCHECK(CanBeTaggedOrCompressedPointer(store_rep.representation()));
556 AddressingMode addressing_mode;
557 InstructionOperand inputs[] = {
558 g.UseUniqueRegister(base),
559 g.GetEffectiveIndexOperand(index, &addressing_mode),
560 g.UseUniqueRegister(value)};
561 RecordWriteMode record_write_mode =
562 WriteBarrierKindToRecordWriteMode(write_barrier_kind);
563 InstructionOperand temps[] = {g.TempRegister(), g.TempRegister()};
564 InstructionCode code = is_seqcst ? kArchAtomicStoreWithWriteBarrier
565 : kArchStoreWithWriteBarrier;
566 code |= AddressingModeField::encode(addressing_mode);
567 code |= MiscField::encode(static_cast<int>(record_write_mode));
568 selector->Emit(code, 0, nullptr, arraysize(inputs), inputs,
569 arraysize(temps), temps);
570 } else {
571 #ifdef V8_IS_TSAN
572 // On TSAN builds we require two scratch registers. Because of this we also
573 // have to modify the inputs to take into account possible aliasing and use
574 // UseUniqueRegister which is not required for non-TSAN builds.
575 InstructionOperand temps[] = {g.TempRegister(), g.TempRegister()};
576 size_t temp_count = arraysize(temps);
577 auto reg_kind = OperandGenerator::RegisterUseKind::kUseUniqueRegister;
578 #else
579 InstructionOperand* temps = nullptr;
580 size_t temp_count = 0;
581 auto reg_kind = OperandGenerator::RegisterUseKind::kUseRegister;
582 #endif // V8_IS_TSAN
583
584 // Release and non-atomic stores emit MOV and sequentially consistent stores
585 // emit XCHG.
586 // https://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html
587
588 ArchOpcode opcode;
589 AddressingMode addressing_mode;
590 InstructionOperand inputs[4];
591 size_t input_count = 0;
592
593 if (is_seqcst) {
594 // SeqCst stores emit XCHG instead of MOV, so encode the inputs as we
595 // would for XCHG. XCHG can't encode the value as an immediate and has
596 // fewer addressing modes available.
597 inputs[input_count++] = g.UseUniqueRegister(value);
598 inputs[input_count++] = g.UseUniqueRegister(base);
599 inputs[input_count++] =
600 g.GetEffectiveIndexOperand(index, &addressing_mode);
601 opcode = GetSeqCstStoreOpcode(store_rep);
602 } else {
603 if ((ElementSizeLog2Of(store_rep.representation()) <
604 kSystemPointerSizeLog2) &&
605 value->opcode() == IrOpcode::kTruncateInt64ToInt32) {
606 value = value->InputAt(0);
607 }
608
609 addressing_mode = g.GetEffectiveAddressMemoryOperand(
610 node, inputs, &input_count, reg_kind);
611 InstructionOperand value_operand = g.CanBeImmediate(value)
612 ? g.UseImmediate(value)
613 : g.UseRegister(value, reg_kind);
614 inputs[input_count++] = value_operand;
615 opcode = GetStoreOpcode(store_rep);
616 }
617
618 InstructionCode code =
619 opcode | AddressingModeField::encode(addressing_mode);
620 selector->Emit(code, 0, static_cast<InstructionOperand*>(nullptr),
621 input_count, inputs, temp_count, temps);
622 }
623 }
624
625 } // namespace
626
VisitStore(Node * node)627 void InstructionSelector::VisitStore(Node* node) {
628 return VisitStoreCommon(this, node, StoreRepresentationOf(node->op()),
629 base::nullopt);
630 }
631
VisitProtectedStore(Node * node)632 void InstructionSelector::VisitProtectedStore(Node* node) {
633 X64OperandGenerator g(this);
634 Node* value = node->InputAt(2);
635 StoreRepresentation store_rep = StoreRepresentationOf(node->op());
636
637 #ifdef V8_IS_TSAN
638 // On TSAN builds we require two scratch registers. Because of this we also
639 // have to modify the inputs to take into account possible aliasing and use
640 // UseUniqueRegister which is not required for non-TSAN builds.
641 InstructionOperand temps[] = {g.TempRegister(), g.TempRegister()};
642 size_t temp_count = arraysize(temps);
643 auto reg_kind = OperandGenerator::RegisterUseKind::kUseUniqueRegister;
644 #else
645 InstructionOperand* temps = nullptr;
646 size_t temp_count = 0;
647 auto reg_kind = OperandGenerator::RegisterUseKind::kUseRegister;
648 #endif // V8_IS_TSAN
649
650 InstructionOperand inputs[4];
651 size_t input_count = 0;
652 AddressingMode addressing_mode =
653 g.GetEffectiveAddressMemoryOperand(node, inputs, &input_count, reg_kind);
654 InstructionOperand value_operand = g.CanBeImmediate(value)
655 ? g.UseImmediate(value)
656 : g.UseRegister(value, reg_kind);
657 inputs[input_count++] = value_operand;
658 ArchOpcode opcode = GetStoreOpcode(store_rep);
659 InstructionCode code = opcode | AddressingModeField::encode(addressing_mode) |
660 AccessModeField::encode(kMemoryAccessProtected);
661 Emit(code, 0, static_cast<InstructionOperand*>(nullptr), input_count, inputs,
662 temp_count, temps);
663 }
664
665 // Architecture supports unaligned access, therefore VisitLoad is used instead
VisitUnalignedLoad(Node * node)666 void InstructionSelector::VisitUnalignedLoad(Node* node) { UNREACHABLE(); }
667
668 // Architecture supports unaligned access, therefore VisitStore is used instead
VisitUnalignedStore(Node * node)669 void InstructionSelector::VisitUnalignedStore(Node* node) { UNREACHABLE(); }
670
VisitStoreLane(Node * node)671 void InstructionSelector::VisitStoreLane(Node* node) {
672 X64OperandGenerator g(this);
673
674 StoreLaneParameters params = StoreLaneParametersOf(node->op());
675 InstructionCode opcode = kArchNop;
676 if (params.rep == MachineRepresentation::kWord8) {
677 opcode = kX64Pextrb;
678 } else if (params.rep == MachineRepresentation::kWord16) {
679 opcode = kX64Pextrw;
680 } else if (params.rep == MachineRepresentation::kWord32) {
681 opcode = kX64S128Store32Lane;
682 } else if (params.rep == MachineRepresentation::kWord64) {
683 opcode = kX64S128Store64Lane;
684 } else {
685 UNREACHABLE();
686 }
687
688 InstructionOperand inputs[4];
689 size_t input_count = 0;
690 AddressingMode addressing_mode =
691 g.GetEffectiveAddressMemoryOperand(node, inputs, &input_count);
692 opcode |= AddressingModeField::encode(addressing_mode);
693
694 if (params.kind == MemoryAccessKind::kProtected) {
695 opcode |= AccessModeField::encode(kMemoryAccessProtected);
696 }
697
698 InstructionOperand value_operand = g.UseRegister(node->InputAt(2));
699 inputs[input_count++] = value_operand;
700 inputs[input_count++] = g.UseImmediate(params.laneidx);
701 DCHECK_GE(4, input_count);
702 Emit(opcode, 0, nullptr, input_count, inputs);
703 }
704
705 // Shared routine for multiple binary operations.
VisitBinop(InstructionSelector * selector,Node * node,InstructionCode opcode,FlagsContinuation * cont)706 static void VisitBinop(InstructionSelector* selector, Node* node,
707 InstructionCode opcode, FlagsContinuation* cont) {
708 X64OperandGenerator g(selector);
709 Int32BinopMatcher m(node);
710 Node* left = m.left().node();
711 Node* right = m.right().node();
712 InstructionOperand inputs[8];
713 size_t input_count = 0;
714 InstructionOperand outputs[1];
715 size_t output_count = 0;
716
717 // TODO(turbofan): match complex addressing modes.
718 if (left == right) {
719 // If both inputs refer to the same operand, enforce allocating a register
720 // for both of them to ensure that we don't end up generating code like
721 // this:
722 //
723 // mov rax, [rbp-0x10]
724 // add rax, [rbp-0x10]
725 // jo label
726 InstructionOperand const input = g.UseRegister(left);
727 inputs[input_count++] = input;
728 inputs[input_count++] = input;
729 } else if (g.CanBeImmediate(right)) {
730 inputs[input_count++] = g.UseRegister(left);
731 inputs[input_count++] = g.UseImmediate(right);
732 } else {
733 int effect_level = selector->GetEffectLevel(node, cont);
734 if (node->op()->HasProperty(Operator::kCommutative) &&
735 g.CanBeBetterLeftOperand(right) &&
736 (!g.CanBeBetterLeftOperand(left) ||
737 !g.CanBeMemoryOperand(opcode, node, right, effect_level))) {
738 std::swap(left, right);
739 }
740 if (g.CanBeMemoryOperand(opcode, node, right, effect_level)) {
741 inputs[input_count++] = g.UseRegister(left);
742 AddressingMode addressing_mode =
743 g.GetEffectiveAddressMemoryOperand(right, inputs, &input_count);
744 opcode |= AddressingModeField::encode(addressing_mode);
745 } else {
746 inputs[input_count++] = g.UseRegister(left);
747 inputs[input_count++] = g.Use(right);
748 }
749 }
750
751 if (cont->IsBranch()) {
752 inputs[input_count++] = g.Label(cont->true_block());
753 inputs[input_count++] = g.Label(cont->false_block());
754 }
755
756 outputs[output_count++] = g.DefineSameAsFirst(node);
757
758 DCHECK_NE(0u, input_count);
759 DCHECK_EQ(1u, output_count);
760 DCHECK_GE(arraysize(inputs), input_count);
761 DCHECK_GE(arraysize(outputs), output_count);
762
763 selector->EmitWithContinuation(opcode, output_count, outputs, input_count,
764 inputs, cont);
765 }
766
767 // Shared routine for multiple binary operations.
VisitBinop(InstructionSelector * selector,Node * node,InstructionCode opcode)768 static void VisitBinop(InstructionSelector* selector, Node* node,
769 InstructionCode opcode) {
770 FlagsContinuation cont;
771 VisitBinop(selector, node, opcode, &cont);
772 }
773
VisitWord32And(Node * node)774 void InstructionSelector::VisitWord32And(Node* node) {
775 X64OperandGenerator g(this);
776 Uint32BinopMatcher m(node);
777 if (m.right().Is(0xFF)) {
778 Emit(kX64Movzxbl, g.DefineAsRegister(node), g.Use(m.left().node()));
779 } else if (m.right().Is(0xFFFF)) {
780 Emit(kX64Movzxwl, g.DefineAsRegister(node), g.Use(m.left().node()));
781 } else {
782 VisitBinop(this, node, kX64And32);
783 }
784 }
785
VisitWord64And(Node * node)786 void InstructionSelector::VisitWord64And(Node* node) {
787 X64OperandGenerator g(this);
788 Uint64BinopMatcher m(node);
789 if (m.right().Is(0xFF)) {
790 Emit(kX64Movzxbq, g.DefineAsRegister(node), g.Use(m.left().node()));
791 } else if (m.right().Is(0xFFFF)) {
792 Emit(kX64Movzxwq, g.DefineAsRegister(node), g.Use(m.left().node()));
793 } else if (m.right().Is(0xFFFFFFFF)) {
794 Emit(kX64Movl, g.DefineAsRegister(node), g.Use(m.left().node()));
795 } else if (m.right().IsInRange(std::numeric_limits<uint32_t>::min(),
796 std::numeric_limits<uint32_t>::max())) {
797 Emit(kX64And32, g.DefineSameAsFirst(node), g.UseRegister(m.left().node()),
798 g.UseImmediate(static_cast<int32_t>(m.right().ResolvedValue())));
799 } else {
800 VisitBinop(this, node, kX64And);
801 }
802 }
803
VisitWord32Or(Node * node)804 void InstructionSelector::VisitWord32Or(Node* node) {
805 VisitBinop(this, node, kX64Or32);
806 }
807
VisitWord64Or(Node * node)808 void InstructionSelector::VisitWord64Or(Node* node) {
809 VisitBinop(this, node, kX64Or);
810 }
811
VisitWord32Xor(Node * node)812 void InstructionSelector::VisitWord32Xor(Node* node) {
813 X64OperandGenerator g(this);
814 Uint32BinopMatcher m(node);
815 if (m.right().Is(-1)) {
816 Emit(kX64Not32, g.DefineSameAsFirst(node), g.UseRegister(m.left().node()));
817 } else {
818 VisitBinop(this, node, kX64Xor32);
819 }
820 }
821
VisitWord64Xor(Node * node)822 void InstructionSelector::VisitWord64Xor(Node* node) {
823 X64OperandGenerator g(this);
824 Uint64BinopMatcher m(node);
825 if (m.right().Is(-1)) {
826 Emit(kX64Not, g.DefineSameAsFirst(node), g.UseRegister(m.left().node()));
827 } else {
828 VisitBinop(this, node, kX64Xor);
829 }
830 }
831
VisitStackPointerGreaterThan(Node * node,FlagsContinuation * cont)832 void InstructionSelector::VisitStackPointerGreaterThan(
833 Node* node, FlagsContinuation* cont) {
834 StackCheckKind kind = StackCheckKindOf(node->op());
835 InstructionCode opcode =
836 kArchStackPointerGreaterThan | MiscField::encode(static_cast<int>(kind));
837
838 int effect_level = GetEffectLevel(node, cont);
839
840 X64OperandGenerator g(this);
841 Node* const value = node->InputAt(0);
842 if (g.CanBeMemoryOperand(kX64Cmp, node, value, effect_level)) {
843 DCHECK(IrOpcode::kLoad == value->opcode() ||
844 IrOpcode::kLoadImmutable == value->opcode());
845
846 // GetEffectiveAddressMemoryOperand can create at most 3 inputs.
847 static constexpr int kMaxInputCount = 3;
848
849 size_t input_count = 0;
850 InstructionOperand inputs[kMaxInputCount];
851 AddressingMode addressing_mode =
852 g.GetEffectiveAddressMemoryOperand(value, inputs, &input_count);
853 opcode |= AddressingModeField::encode(addressing_mode);
854 DCHECK_LE(input_count, kMaxInputCount);
855
856 EmitWithContinuation(opcode, 0, nullptr, input_count, inputs, cont);
857 } else {
858 EmitWithContinuation(opcode, g.UseRegister(value), cont);
859 }
860 }
861
862 namespace {
863
TryMergeTruncateInt64ToInt32IntoLoad(InstructionSelector * selector,Node * node,Node * load)864 bool TryMergeTruncateInt64ToInt32IntoLoad(InstructionSelector* selector,
865 Node* node, Node* load) {
866 if ((load->opcode() == IrOpcode::kLoad ||
867 load->opcode() == IrOpcode::kLoadImmutable) &&
868 selector->CanCover(node, load)) {
869 LoadRepresentation load_rep = LoadRepresentationOf(load->op());
870 MachineRepresentation rep = load_rep.representation();
871 InstructionCode opcode;
872 switch (rep) {
873 case MachineRepresentation::kBit: // Fall through.
874 case MachineRepresentation::kWord8:
875 opcode = load_rep.IsSigned() ? kX64Movsxbl : kX64Movzxbl;
876 break;
877 case MachineRepresentation::kWord16:
878 opcode = load_rep.IsSigned() ? kX64Movsxwl : kX64Movzxwl;
879 break;
880 case MachineRepresentation::kWord32:
881 case MachineRepresentation::kWord64:
882 case MachineRepresentation::kTaggedSigned:
883 case MachineRepresentation::kTagged:
884 case MachineRepresentation::kCompressed: // Fall through.
885 opcode = kX64Movl;
886 break;
887 default:
888 UNREACHABLE();
889 }
890 X64OperandGenerator g(selector);
891 #ifdef V8_IS_TSAN
892 // On TSAN builds we require one scratch register. Because of this we also
893 // have to modify the inputs to take into account possible aliasing and use
894 // UseUniqueRegister which is not required for non-TSAN builds.
895 InstructionOperand temps[] = {g.TempRegister()};
896 size_t temp_count = arraysize(temps);
897 auto reg_kind = OperandGenerator::RegisterUseKind::kUseUniqueRegister;
898 #else
899 InstructionOperand* temps = nullptr;
900 size_t temp_count = 0;
901 auto reg_kind = OperandGenerator::RegisterUseKind::kUseRegister;
902 #endif // V8_IS_TSAN
903 InstructionOperand outputs[] = {g.DefineAsRegister(node)};
904 size_t input_count = 0;
905 InstructionOperand inputs[3];
906 AddressingMode mode = g.GetEffectiveAddressMemoryOperand(
907 node->InputAt(0), inputs, &input_count, reg_kind);
908 opcode |= AddressingModeField::encode(mode);
909 selector->Emit(opcode, 1, outputs, input_count, inputs, temp_count, temps);
910 return true;
911 }
912 return false;
913 }
914
915 // Shared routine for multiple 32-bit shift operations.
916 // TODO(bmeurer): Merge this with VisitWord64Shift using template magic?
VisitWord32Shift(InstructionSelector * selector,Node * node,ArchOpcode opcode)917 void VisitWord32Shift(InstructionSelector* selector, Node* node,
918 ArchOpcode opcode) {
919 X64OperandGenerator g(selector);
920 Int32BinopMatcher m(node);
921 Node* left = m.left().node();
922 Node* right = m.right().node();
923
924 if (left->opcode() == IrOpcode::kTruncateInt64ToInt32) {
925 left = left->InputAt(0);
926 }
927
928 if (g.CanBeImmediate(right)) {
929 selector->Emit(opcode, g.DefineSameAsFirst(node), g.UseRegister(left),
930 g.UseImmediate(right));
931 } else {
932 selector->Emit(opcode, g.DefineSameAsFirst(node), g.UseRegister(left),
933 g.UseFixed(right, rcx));
934 }
935 }
936
937 // Shared routine for multiple 64-bit shift operations.
938 // TODO(bmeurer): Merge this with VisitWord32Shift using template magic?
VisitWord64Shift(InstructionSelector * selector,Node * node,ArchOpcode opcode)939 void VisitWord64Shift(InstructionSelector* selector, Node* node,
940 ArchOpcode opcode) {
941 X64OperandGenerator g(selector);
942 Int64BinopMatcher m(node);
943 Node* left = m.left().node();
944 Node* right = m.right().node();
945
946 if (g.CanBeImmediate(right)) {
947 selector->Emit(opcode, g.DefineSameAsFirst(node), g.UseRegister(left),
948 g.UseImmediate(right));
949 } else {
950 if (m.right().IsWord64And()) {
951 Int64BinopMatcher mright(right);
952 if (mright.right().Is(0x3F)) {
953 right = mright.left().node();
954 }
955 }
956 selector->Emit(opcode, g.DefineSameAsFirst(node), g.UseRegister(left),
957 g.UseFixed(right, rcx));
958 }
959 }
960
961 // Shared routine for multiple shift operations with continuation.
962 template <typename BinopMatcher, int Bits>
TryVisitWordShift(InstructionSelector * selector,Node * node,ArchOpcode opcode,FlagsContinuation * cont)963 bool TryVisitWordShift(InstructionSelector* selector, Node* node,
964 ArchOpcode opcode, FlagsContinuation* cont) {
965 X64OperandGenerator g(selector);
966 BinopMatcher m(node);
967 Node* left = m.left().node();
968 Node* right = m.right().node();
969
970 // If the shift count is 0, the flags are not affected.
971 if (!g.CanBeImmediate(right) ||
972 (g.GetImmediateIntegerValue(right) & (Bits - 1)) == 0) {
973 return false;
974 }
975 InstructionOperand output = g.DefineSameAsFirst(node);
976 InstructionOperand inputs[2];
977 inputs[0] = g.UseRegister(left);
978 inputs[1] = g.UseImmediate(right);
979 selector->EmitWithContinuation(opcode, 1, &output, 2, inputs, cont);
980 return true;
981 }
982
EmitLea(InstructionSelector * selector,InstructionCode opcode,Node * result,Node * index,int scale,Node * base,Node * displacement,DisplacementMode displacement_mode)983 void EmitLea(InstructionSelector* selector, InstructionCode opcode,
984 Node* result, Node* index, int scale, Node* base,
985 Node* displacement, DisplacementMode displacement_mode) {
986 X64OperandGenerator g(selector);
987
988 InstructionOperand inputs[4];
989 size_t input_count = 0;
990 AddressingMode mode =
991 g.GenerateMemoryOperandInputs(index, scale, base, displacement,
992 displacement_mode, inputs, &input_count);
993
994 DCHECK_NE(0u, input_count);
995 DCHECK_GE(arraysize(inputs), input_count);
996
997 InstructionOperand outputs[1];
998 outputs[0] = g.DefineAsRegister(result);
999
1000 opcode = AddressingModeField::encode(mode) | opcode;
1001
1002 selector->Emit(opcode, 1, outputs, input_count, inputs);
1003 }
1004
1005 } // namespace
1006
VisitWord32Shl(Node * node)1007 void InstructionSelector::VisitWord32Shl(Node* node) {
1008 Int32ScaleMatcher m(node, true);
1009 if (m.matches()) {
1010 Node* index = node->InputAt(0);
1011 Node* base = m.power_of_two_plus_one() ? index : nullptr;
1012 EmitLea(this, kX64Lea32, node, index, m.scale(), base, nullptr,
1013 kPositiveDisplacement);
1014 return;
1015 }
1016 VisitWord32Shift(this, node, kX64Shl32);
1017 }
1018
VisitWord64Shl(Node * node)1019 void InstructionSelector::VisitWord64Shl(Node* node) {
1020 X64OperandGenerator g(this);
1021 Int64ScaleMatcher m(node, true);
1022 if (m.matches()) {
1023 Node* index = node->InputAt(0);
1024 Node* base = m.power_of_two_plus_one() ? index : nullptr;
1025 EmitLea(this, kX64Lea, node, index, m.scale(), base, nullptr,
1026 kPositiveDisplacement);
1027 return;
1028 } else {
1029 Int64BinopMatcher bm(node);
1030 if ((bm.left().IsChangeInt32ToInt64() ||
1031 bm.left().IsChangeUint32ToUint64()) &&
1032 bm.right().IsInRange(32, 63)) {
1033 // There's no need to sign/zero-extend to 64-bit if we shift out the upper
1034 // 32 bits anyway.
1035 Emit(kX64Shl, g.DefineSameAsFirst(node),
1036 g.UseRegister(bm.left().node()->InputAt(0)),
1037 g.UseImmediate(bm.right().node()));
1038 return;
1039 }
1040 }
1041 VisitWord64Shift(this, node, kX64Shl);
1042 }
1043
VisitWord32Shr(Node * node)1044 void InstructionSelector::VisitWord32Shr(Node* node) {
1045 VisitWord32Shift(this, node, kX64Shr32);
1046 }
1047
1048 namespace {
1049
AddDisplacementToAddressingMode(AddressingMode mode)1050 inline AddressingMode AddDisplacementToAddressingMode(AddressingMode mode) {
1051 switch (mode) {
1052 case kMode_MR:
1053 return kMode_MRI;
1054 case kMode_MR1:
1055 return kMode_MR1I;
1056 case kMode_MR2:
1057 return kMode_MR2I;
1058 case kMode_MR4:
1059 return kMode_MR4I;
1060 case kMode_MR8:
1061 return kMode_MR8I;
1062 case kMode_M1:
1063 return kMode_M1I;
1064 case kMode_M2:
1065 return kMode_M2I;
1066 case kMode_M4:
1067 return kMode_M4I;
1068 case kMode_M8:
1069 return kMode_M8I;
1070 case kMode_None:
1071 case kMode_MRI:
1072 case kMode_MR1I:
1073 case kMode_MR2I:
1074 case kMode_MR4I:
1075 case kMode_MR8I:
1076 case kMode_M1I:
1077 case kMode_M2I:
1078 case kMode_M4I:
1079 case kMode_M8I:
1080 case kMode_Root:
1081 UNREACHABLE();
1082 }
1083 UNREACHABLE();
1084 }
1085
TryMatchLoadWord64AndShiftRight(InstructionSelector * selector,Node * node,InstructionCode opcode)1086 bool TryMatchLoadWord64AndShiftRight(InstructionSelector* selector, Node* node,
1087 InstructionCode opcode) {
1088 DCHECK(IrOpcode::kWord64Sar == node->opcode() ||
1089 IrOpcode::kWord64Shr == node->opcode());
1090 X64OperandGenerator g(selector);
1091 Int64BinopMatcher m(node);
1092 if (selector->CanCover(m.node(), m.left().node()) && m.left().IsLoad() &&
1093 m.right().Is(32)) {
1094 DCHECK_EQ(selector->GetEffectLevel(node),
1095 selector->GetEffectLevel(m.left().node()));
1096 // Just load and sign-extend the interesting 4 bytes instead. This happens,
1097 // for example, when we're loading and untagging SMIs.
1098 BaseWithIndexAndDisplacement64Matcher mleft(m.left().node(),
1099 AddressOption::kAllowAll);
1100 if (mleft.matches() && (mleft.displacement() == nullptr ||
1101 g.CanBeImmediate(mleft.displacement()))) {
1102 #ifdef V8_IS_TSAN
1103 // On TSAN builds we require one scratch register. Because of this we also
1104 // have to modify the inputs to take into account possible aliasing and
1105 // use UseUniqueRegister which is not required for non-TSAN builds.
1106 InstructionOperand temps[] = {g.TempRegister()};
1107 size_t temp_count = arraysize(temps);
1108 auto reg_kind = OperandGenerator::RegisterUseKind::kUseUniqueRegister;
1109 #else
1110 InstructionOperand* temps = nullptr;
1111 size_t temp_count = 0;
1112 auto reg_kind = OperandGenerator::RegisterUseKind::kUseRegister;
1113 #endif // V8_IS_TSAN
1114 size_t input_count = 0;
1115 InstructionOperand inputs[3];
1116 AddressingMode mode = g.GetEffectiveAddressMemoryOperand(
1117 m.left().node(), inputs, &input_count, reg_kind);
1118 if (mleft.displacement() == nullptr) {
1119 // Make sure that the addressing mode indicates the presence of an
1120 // immediate displacement. It seems that we never use M1 and M2, but we
1121 // handle them here anyways.
1122 mode = AddDisplacementToAddressingMode(mode);
1123 inputs[input_count++] =
1124 ImmediateOperand(ImmediateOperand::INLINE_INT32, 4);
1125 } else {
1126 // In the case that the base address was zero, the displacement will be
1127 // in a register and replacing it with an immediate is not allowed. This
1128 // usually only happens in dead code anyway.
1129 if (!inputs[input_count - 1].IsImmediate()) return false;
1130 int32_t displacement = g.GetImmediateIntegerValue(mleft.displacement());
1131 inputs[input_count - 1] =
1132 ImmediateOperand(ImmediateOperand::INLINE_INT32, displacement + 4);
1133 }
1134 InstructionOperand outputs[] = {g.DefineAsRegister(node)};
1135 InstructionCode code = opcode | AddressingModeField::encode(mode);
1136 selector->Emit(code, 1, outputs, input_count, inputs, temp_count, temps);
1137 return true;
1138 }
1139 }
1140 return false;
1141 }
1142
1143 } // namespace
1144
VisitWord64Shr(Node * node)1145 void InstructionSelector::VisitWord64Shr(Node* node) {
1146 if (TryMatchLoadWord64AndShiftRight(this, node, kX64Movl)) return;
1147 VisitWord64Shift(this, node, kX64Shr);
1148 }
1149
VisitWord32Sar(Node * node)1150 void InstructionSelector::VisitWord32Sar(Node* node) {
1151 X64OperandGenerator g(this);
1152 Int32BinopMatcher m(node);
1153 if (CanCover(m.node(), m.left().node()) && m.left().IsWord32Shl()) {
1154 Int32BinopMatcher mleft(m.left().node());
1155 if (mleft.right().Is(16) && m.right().Is(16)) {
1156 Emit(kX64Movsxwl, g.DefineAsRegister(node), g.Use(mleft.left().node()));
1157 return;
1158 } else if (mleft.right().Is(24) && m.right().Is(24)) {
1159 Emit(kX64Movsxbl, g.DefineAsRegister(node), g.Use(mleft.left().node()));
1160 return;
1161 }
1162 }
1163 VisitWord32Shift(this, node, kX64Sar32);
1164 }
1165
VisitWord64Sar(Node * node)1166 void InstructionSelector::VisitWord64Sar(Node* node) {
1167 if (TryMatchLoadWord64AndShiftRight(this, node, kX64Movsxlq)) return;
1168 VisitWord64Shift(this, node, kX64Sar);
1169 }
1170
VisitWord32Rol(Node * node)1171 void InstructionSelector::VisitWord32Rol(Node* node) {
1172 VisitWord32Shift(this, node, kX64Rol32);
1173 }
1174
VisitWord64Rol(Node * node)1175 void InstructionSelector::VisitWord64Rol(Node* node) {
1176 VisitWord64Shift(this, node, kX64Rol);
1177 }
1178
VisitWord32Ror(Node * node)1179 void InstructionSelector::VisitWord32Ror(Node* node) {
1180 VisitWord32Shift(this, node, kX64Ror32);
1181 }
1182
VisitWord64Ror(Node * node)1183 void InstructionSelector::VisitWord64Ror(Node* node) {
1184 VisitWord64Shift(this, node, kX64Ror);
1185 }
1186
VisitWord32ReverseBits(Node * node)1187 void InstructionSelector::VisitWord32ReverseBits(Node* node) { UNREACHABLE(); }
1188
VisitWord64ReverseBits(Node * node)1189 void InstructionSelector::VisitWord64ReverseBits(Node* node) { UNREACHABLE(); }
1190
VisitWord64ReverseBytes(Node * node)1191 void InstructionSelector::VisitWord64ReverseBytes(Node* node) {
1192 X64OperandGenerator g(this);
1193 Emit(kX64Bswap, g.DefineSameAsFirst(node), g.UseRegister(node->InputAt(0)));
1194 }
1195
VisitWord32ReverseBytes(Node * node)1196 void InstructionSelector::VisitWord32ReverseBytes(Node* node) {
1197 X64OperandGenerator g(this);
1198 Emit(kX64Bswap32, g.DefineSameAsFirst(node), g.UseRegister(node->InputAt(0)));
1199 }
1200
VisitSimd128ReverseBytes(Node * node)1201 void InstructionSelector::VisitSimd128ReverseBytes(Node* node) {
1202 UNREACHABLE();
1203 }
1204
VisitInt32Add(Node * node)1205 void InstructionSelector::VisitInt32Add(Node* node) {
1206 X64OperandGenerator g(this);
1207
1208 // No need to truncate the values before Int32Add.
1209 DCHECK_EQ(node->InputCount(), 2);
1210 Node* left = node->InputAt(0);
1211 Node* right = node->InputAt(1);
1212 if (left->opcode() == IrOpcode::kTruncateInt64ToInt32) {
1213 node->ReplaceInput(0, left->InputAt(0));
1214 }
1215 if (right->opcode() == IrOpcode::kTruncateInt64ToInt32) {
1216 node->ReplaceInput(1, right->InputAt(0));
1217 }
1218
1219 // Try to match the Add to a leal pattern
1220 BaseWithIndexAndDisplacement32Matcher m(node);
1221 if (m.matches() &&
1222 (m.displacement() == nullptr || g.CanBeImmediate(m.displacement()))) {
1223 EmitLea(this, kX64Lea32, node, m.index(), m.scale(), m.base(),
1224 m.displacement(), m.displacement_mode());
1225 return;
1226 }
1227
1228 // No leal pattern match, use addl
1229 VisitBinop(this, node, kX64Add32);
1230 }
1231
VisitInt64Add(Node * node)1232 void InstructionSelector::VisitInt64Add(Node* node) {
1233 X64OperandGenerator g(this);
1234
1235 // Try to match the Add to a leaq pattern
1236 BaseWithIndexAndDisplacement64Matcher m(node);
1237 if (m.matches() &&
1238 (m.displacement() == nullptr || g.CanBeImmediate(m.displacement()))) {
1239 EmitLea(this, kX64Lea, node, m.index(), m.scale(), m.base(),
1240 m.displacement(), m.displacement_mode());
1241 return;
1242 }
1243
1244 // No leal pattern match, use addq
1245 VisitBinop(this, node, kX64Add);
1246 }
1247
VisitInt64AddWithOverflow(Node * node)1248 void InstructionSelector::VisitInt64AddWithOverflow(Node* node) {
1249 if (Node* ovf = NodeProperties::FindProjection(node, 1)) {
1250 FlagsContinuation cont = FlagsContinuation::ForSet(kOverflow, ovf);
1251 return VisitBinop(this, node, kX64Add, &cont);
1252 }
1253 FlagsContinuation cont;
1254 VisitBinop(this, node, kX64Add, &cont);
1255 }
1256
VisitInt32Sub(Node * node)1257 void InstructionSelector::VisitInt32Sub(Node* node) {
1258 X64OperandGenerator g(this);
1259 DCHECK_EQ(node->InputCount(), 2);
1260 Node* input1 = node->InputAt(0);
1261 Node* input2 = node->InputAt(1);
1262 if (input1->opcode() == IrOpcode::kTruncateInt64ToInt32 &&
1263 g.CanBeImmediate(input2)) {
1264 int32_t imm = g.GetImmediateIntegerValue(input2);
1265 InstructionOperand int64_input = g.UseRegister(input1->InputAt(0));
1266 if (imm == 0) {
1267 // Emit "movl" for subtraction of 0.
1268 Emit(kX64Movl, g.DefineAsRegister(node), int64_input);
1269 } else {
1270 // Omit truncation and turn subtractions of constant values into immediate
1271 // "leal" instructions by negating the value.
1272 Emit(kX64Lea32 | AddressingModeField::encode(kMode_MRI),
1273 g.DefineAsRegister(node), int64_input,
1274 g.TempImmediate(base::NegateWithWraparound(imm)));
1275 }
1276 return;
1277 }
1278
1279 Int32BinopMatcher m(node);
1280 if (m.left().Is(0)) {
1281 Emit(kX64Neg32, g.DefineSameAsFirst(node), g.UseRegister(m.right().node()));
1282 } else if (m.right().Is(0)) {
1283 // {EmitIdentity} reuses the virtual register of the first input
1284 // for the output. This is exactly what we want here.
1285 EmitIdentity(node);
1286 } else if (m.right().HasResolvedValue() &&
1287 g.CanBeImmediate(m.right().node())) {
1288 // Turn subtractions of constant values into immediate "leal" instructions
1289 // by negating the value.
1290 Emit(
1291 kX64Lea32 | AddressingModeField::encode(kMode_MRI),
1292 g.DefineAsRegister(node), g.UseRegister(m.left().node()),
1293 g.TempImmediate(base::NegateWithWraparound(m.right().ResolvedValue())));
1294 } else {
1295 VisitBinop(this, node, kX64Sub32);
1296 }
1297 }
1298
VisitInt64Sub(Node * node)1299 void InstructionSelector::VisitInt64Sub(Node* node) {
1300 X64OperandGenerator g(this);
1301 Int64BinopMatcher m(node);
1302 if (m.left().Is(0)) {
1303 Emit(kX64Neg, g.DefineSameAsFirst(node), g.UseRegister(m.right().node()));
1304 } else {
1305 if (m.right().HasResolvedValue() && g.CanBeImmediate(m.right().node())) {
1306 // Turn subtractions of constant values into immediate "leaq" instructions
1307 // by negating the value.
1308 Emit(kX64Lea | AddressingModeField::encode(kMode_MRI),
1309 g.DefineAsRegister(node), g.UseRegister(m.left().node()),
1310 g.TempImmediate(-static_cast<int32_t>(m.right().ResolvedValue())));
1311 return;
1312 }
1313 VisitBinop(this, node, kX64Sub);
1314 }
1315 }
1316
VisitInt64SubWithOverflow(Node * node)1317 void InstructionSelector::VisitInt64SubWithOverflow(Node* node) {
1318 if (Node* ovf = NodeProperties::FindProjection(node, 1)) {
1319 FlagsContinuation cont = FlagsContinuation::ForSet(kOverflow, ovf);
1320 return VisitBinop(this, node, kX64Sub, &cont);
1321 }
1322 FlagsContinuation cont;
1323 VisitBinop(this, node, kX64Sub, &cont);
1324 }
1325
1326 namespace {
1327
VisitMul(InstructionSelector * selector,Node * node,ArchOpcode opcode)1328 void VisitMul(InstructionSelector* selector, Node* node, ArchOpcode opcode) {
1329 X64OperandGenerator g(selector);
1330 Int32BinopMatcher m(node);
1331 Node* left = m.left().node();
1332 Node* right = m.right().node();
1333 if (g.CanBeImmediate(right)) {
1334 selector->Emit(opcode, g.DefineAsRegister(node), g.Use(left),
1335 g.UseImmediate(right));
1336 } else {
1337 if (g.CanBeBetterLeftOperand(right)) {
1338 std::swap(left, right);
1339 }
1340 selector->Emit(opcode, g.DefineSameAsFirst(node), g.UseRegister(left),
1341 g.Use(right));
1342 }
1343 }
1344
VisitMulHigh(InstructionSelector * selector,Node * node,ArchOpcode opcode)1345 void VisitMulHigh(InstructionSelector* selector, Node* node,
1346 ArchOpcode opcode) {
1347 X64OperandGenerator g(selector);
1348 Node* left = node->InputAt(0);
1349 Node* right = node->InputAt(1);
1350 if (selector->IsLive(left) && !selector->IsLive(right)) {
1351 std::swap(left, right);
1352 }
1353 InstructionOperand temps[] = {g.TempRegister(rax)};
1354 // TODO(turbofan): We use UseUniqueRegister here to improve register
1355 // allocation.
1356 selector->Emit(opcode, g.DefineAsFixed(node, rdx), g.UseFixed(left, rax),
1357 g.UseUniqueRegister(right), arraysize(temps), temps);
1358 }
1359
VisitDiv(InstructionSelector * selector,Node * node,ArchOpcode opcode)1360 void VisitDiv(InstructionSelector* selector, Node* node, ArchOpcode opcode) {
1361 X64OperandGenerator g(selector);
1362 InstructionOperand temps[] = {g.TempRegister(rdx)};
1363 selector->Emit(
1364 opcode, g.DefineAsFixed(node, rax), g.UseFixed(node->InputAt(0), rax),
1365 g.UseUniqueRegister(node->InputAt(1)), arraysize(temps), temps);
1366 }
1367
VisitMod(InstructionSelector * selector,Node * node,ArchOpcode opcode)1368 void VisitMod(InstructionSelector* selector, Node* node, ArchOpcode opcode) {
1369 X64OperandGenerator g(selector);
1370 InstructionOperand temps[] = {g.TempRegister(rax)};
1371 selector->Emit(
1372 opcode, g.DefineAsFixed(node, rdx), g.UseFixed(node->InputAt(0), rax),
1373 g.UseUniqueRegister(node->InputAt(1)), arraysize(temps), temps);
1374 }
1375
1376 } // namespace
1377
VisitInt32Mul(Node * node)1378 void InstructionSelector::VisitInt32Mul(Node* node) {
1379 Int32ScaleMatcher m(node, true);
1380 if (m.matches()) {
1381 Node* index = node->InputAt(0);
1382 Node* base = m.power_of_two_plus_one() ? index : nullptr;
1383 EmitLea(this, kX64Lea32, node, index, m.scale(), base, nullptr,
1384 kPositiveDisplacement);
1385 return;
1386 }
1387 VisitMul(this, node, kX64Imul32);
1388 }
1389
VisitInt32MulWithOverflow(Node * node)1390 void InstructionSelector::VisitInt32MulWithOverflow(Node* node) {
1391 // TODO(mvstanton): Use Int32ScaleMatcher somehow.
1392 if (Node* ovf = NodeProperties::FindProjection(node, 1)) {
1393 FlagsContinuation cont = FlagsContinuation::ForSet(kOverflow, ovf);
1394 return VisitBinop(this, node, kX64Imul32, &cont);
1395 }
1396 FlagsContinuation cont;
1397 VisitBinop(this, node, kX64Imul32, &cont);
1398 }
1399
VisitInt64Mul(Node * node)1400 void InstructionSelector::VisitInt64Mul(Node* node) {
1401 VisitMul(this, node, kX64Imul);
1402 }
1403
VisitInt32MulHigh(Node * node)1404 void InstructionSelector::VisitInt32MulHigh(Node* node) {
1405 VisitMulHigh(this, node, kX64ImulHigh32);
1406 }
1407
VisitInt32Div(Node * node)1408 void InstructionSelector::VisitInt32Div(Node* node) {
1409 VisitDiv(this, node, kX64Idiv32);
1410 }
1411
VisitInt64Div(Node * node)1412 void InstructionSelector::VisitInt64Div(Node* node) {
1413 VisitDiv(this, node, kX64Idiv);
1414 }
1415
VisitUint32Div(Node * node)1416 void InstructionSelector::VisitUint32Div(Node* node) {
1417 VisitDiv(this, node, kX64Udiv32);
1418 }
1419
VisitUint64Div(Node * node)1420 void InstructionSelector::VisitUint64Div(Node* node) {
1421 VisitDiv(this, node, kX64Udiv);
1422 }
1423
VisitInt32Mod(Node * node)1424 void InstructionSelector::VisitInt32Mod(Node* node) {
1425 VisitMod(this, node, kX64Idiv32);
1426 }
1427
VisitInt64Mod(Node * node)1428 void InstructionSelector::VisitInt64Mod(Node* node) {
1429 VisitMod(this, node, kX64Idiv);
1430 }
1431
VisitUint32Mod(Node * node)1432 void InstructionSelector::VisitUint32Mod(Node* node) {
1433 VisitMod(this, node, kX64Udiv32);
1434 }
1435
VisitUint64Mod(Node * node)1436 void InstructionSelector::VisitUint64Mod(Node* node) {
1437 VisitMod(this, node, kX64Udiv);
1438 }
1439
VisitUint32MulHigh(Node * node)1440 void InstructionSelector::VisitUint32MulHigh(Node* node) {
1441 VisitMulHigh(this, node, kX64UmulHigh32);
1442 }
1443
VisitTryTruncateFloat32ToInt64(Node * node)1444 void InstructionSelector::VisitTryTruncateFloat32ToInt64(Node* node) {
1445 X64OperandGenerator g(this);
1446 InstructionOperand inputs[] = {g.UseRegister(node->InputAt(0))};
1447 InstructionOperand outputs[2];
1448 InstructionOperand temps[1];
1449 size_t output_count = 0;
1450 size_t temp_count = 0;
1451 outputs[output_count++] = g.DefineAsRegister(node);
1452
1453 Node* success_output = NodeProperties::FindProjection(node, 1);
1454 if (success_output) {
1455 outputs[output_count++] = g.DefineAsRegister(success_output);
1456 temps[temp_count++] = g.TempSimd128Register();
1457 }
1458
1459 Emit(kSSEFloat32ToInt64, output_count, outputs, 1, inputs, temp_count, temps);
1460 }
1461
VisitTryTruncateFloat64ToInt64(Node * node)1462 void InstructionSelector::VisitTryTruncateFloat64ToInt64(Node* node) {
1463 X64OperandGenerator g(this);
1464 InstructionOperand inputs[] = {g.UseRegister(node->InputAt(0))};
1465 InstructionOperand outputs[2];
1466 InstructionOperand temps[1];
1467 size_t output_count = 0;
1468 size_t temp_count = 0;
1469 outputs[output_count++] = g.DefineAsRegister(node);
1470
1471 Node* success_output = NodeProperties::FindProjection(node, 1);
1472 if (success_output) {
1473 outputs[output_count++] = g.DefineAsRegister(success_output);
1474 temps[temp_count++] = g.TempSimd128Register();
1475 }
1476
1477 Emit(kSSEFloat64ToInt64, output_count, outputs, 1, inputs, temp_count, temps);
1478 }
1479
VisitTryTruncateFloat32ToUint64(Node * node)1480 void InstructionSelector::VisitTryTruncateFloat32ToUint64(Node* node) {
1481 X64OperandGenerator g(this);
1482 InstructionOperand inputs[] = {g.UseRegister(node->InputAt(0))};
1483 InstructionOperand outputs[2];
1484 size_t output_count = 0;
1485 outputs[output_count++] = g.DefineAsRegister(node);
1486
1487 Node* success_output = NodeProperties::FindProjection(node, 1);
1488 if (success_output) {
1489 outputs[output_count++] = g.DefineAsRegister(success_output);
1490 }
1491
1492 Emit(kSSEFloat32ToUint64, output_count, outputs, 1, inputs);
1493 }
1494
VisitTryTruncateFloat64ToUint64(Node * node)1495 void InstructionSelector::VisitTryTruncateFloat64ToUint64(Node* node) {
1496 X64OperandGenerator g(this);
1497 InstructionOperand inputs[] = {g.UseRegister(node->InputAt(0))};
1498 InstructionOperand outputs[2];
1499 size_t output_count = 0;
1500 outputs[output_count++] = g.DefineAsRegister(node);
1501
1502 Node* success_output = NodeProperties::FindProjection(node, 1);
1503 if (success_output) {
1504 outputs[output_count++] = g.DefineAsRegister(success_output);
1505 }
1506
1507 Emit(kSSEFloat64ToUint64, output_count, outputs, 1, inputs);
1508 }
1509
VisitBitcastWord32ToWord64(Node * node)1510 void InstructionSelector::VisitBitcastWord32ToWord64(Node* node) {
1511 DCHECK(SmiValuesAre31Bits());
1512 DCHECK(COMPRESS_POINTERS_BOOL);
1513 EmitIdentity(node);
1514 }
1515
VisitChangeInt32ToInt64(Node * node)1516 void InstructionSelector::VisitChangeInt32ToInt64(Node* node) {
1517 DCHECK_EQ(node->InputCount(), 1);
1518 Node* input = node->InputAt(0);
1519 if (input->opcode() == IrOpcode::kTruncateInt64ToInt32) {
1520 node->ReplaceInput(0, input->InputAt(0));
1521 }
1522
1523 X64OperandGenerator g(this);
1524 Node* const value = node->InputAt(0);
1525 if ((value->opcode() == IrOpcode::kLoad ||
1526 value->opcode() == IrOpcode::kLoadImmutable) &&
1527 CanCover(node, value)) {
1528 LoadRepresentation load_rep = LoadRepresentationOf(value->op());
1529 MachineRepresentation rep = load_rep.representation();
1530 InstructionCode opcode;
1531 switch (rep) {
1532 case MachineRepresentation::kBit: // Fall through.
1533 case MachineRepresentation::kWord8:
1534 opcode = load_rep.IsSigned() ? kX64Movsxbq : kX64Movzxbq;
1535 break;
1536 case MachineRepresentation::kWord16:
1537 opcode = load_rep.IsSigned() ? kX64Movsxwq : kX64Movzxwq;
1538 break;
1539 case MachineRepresentation::kWord32:
1540 // ChangeInt32ToInt64 must interpret its input as a _signed_ 32-bit
1541 // integer, so here we must sign-extend the loaded value in any case.
1542 opcode = kX64Movsxlq;
1543 break;
1544 default:
1545 UNREACHABLE();
1546 }
1547 InstructionOperand outputs[] = {g.DefineAsRegister(node)};
1548 size_t input_count = 0;
1549 InstructionOperand inputs[3];
1550 AddressingMode mode = g.GetEffectiveAddressMemoryOperand(
1551 node->InputAt(0), inputs, &input_count);
1552 opcode |= AddressingModeField::encode(mode);
1553 Emit(opcode, 1, outputs, input_count, inputs);
1554 } else {
1555 Emit(kX64Movsxlq, g.DefineAsRegister(node), g.Use(node->InputAt(0)));
1556 }
1557 }
1558
ZeroExtendsWord32ToWord64NoPhis(Node * node)1559 bool InstructionSelector::ZeroExtendsWord32ToWord64NoPhis(Node* node) {
1560 X64OperandGenerator g(this);
1561 DCHECK_NE(node->opcode(), IrOpcode::kPhi);
1562 switch (node->opcode()) {
1563 case IrOpcode::kWord32And:
1564 case IrOpcode::kWord32Or:
1565 case IrOpcode::kWord32Xor:
1566 case IrOpcode::kWord32Shl:
1567 case IrOpcode::kWord32Shr:
1568 case IrOpcode::kWord32Sar:
1569 case IrOpcode::kWord32Rol:
1570 case IrOpcode::kWord32Ror:
1571 case IrOpcode::kWord32Equal:
1572 case IrOpcode::kInt32Add:
1573 case IrOpcode::kInt32Sub:
1574 case IrOpcode::kInt32Mul:
1575 case IrOpcode::kInt32MulHigh:
1576 case IrOpcode::kInt32Div:
1577 case IrOpcode::kInt32LessThan:
1578 case IrOpcode::kInt32LessThanOrEqual:
1579 case IrOpcode::kInt32Mod:
1580 case IrOpcode::kUint32Div:
1581 case IrOpcode::kUint32LessThan:
1582 case IrOpcode::kUint32LessThanOrEqual:
1583 case IrOpcode::kUint32Mod:
1584 case IrOpcode::kUint32MulHigh:
1585 case IrOpcode::kTruncateInt64ToInt32:
1586 // These 32-bit operations implicitly zero-extend to 64-bit on x64, so the
1587 // zero-extension is a no-op.
1588 return true;
1589 case IrOpcode::kProjection: {
1590 Node* const value = node->InputAt(0);
1591 switch (value->opcode()) {
1592 case IrOpcode::kInt32AddWithOverflow:
1593 case IrOpcode::kInt32SubWithOverflow:
1594 case IrOpcode::kInt32MulWithOverflow:
1595 return true;
1596 default:
1597 return false;
1598 }
1599 }
1600 case IrOpcode::kLoad:
1601 case IrOpcode::kLoadImmutable:
1602 case IrOpcode::kProtectedLoad: {
1603 // The movzxbl/movsxbl/movzxwl/movsxwl/movl operations implicitly
1604 // zero-extend to 64-bit on x64, so the zero-extension is a no-op.
1605 LoadRepresentation load_rep = LoadRepresentationOf(node->op());
1606 switch (load_rep.representation()) {
1607 case MachineRepresentation::kWord8:
1608 case MachineRepresentation::kWord16:
1609 case MachineRepresentation::kWord32:
1610 return true;
1611 default:
1612 return false;
1613 }
1614 }
1615 case IrOpcode::kInt32Constant:
1616 case IrOpcode::kInt64Constant:
1617 // Constants are loaded with movl or movq, or xorl for zero; see
1618 // CodeGenerator::AssembleMove. So any non-negative constant that fits
1619 // in a 32-bit signed integer is zero-extended to 64 bits.
1620 if (g.CanBeImmediate(node)) {
1621 return g.GetImmediateIntegerValue(node) >= 0;
1622 }
1623 return false;
1624 default:
1625 return false;
1626 }
1627 }
1628
VisitChangeUint32ToUint64(Node * node)1629 void InstructionSelector::VisitChangeUint32ToUint64(Node* node) {
1630 X64OperandGenerator g(this);
1631 Node* value = node->InputAt(0);
1632 if (ZeroExtendsWord32ToWord64(value)) {
1633 // These 32-bit operations implicitly zero-extend to 64-bit on x64, so the
1634 // zero-extension is a no-op.
1635 return EmitIdentity(node);
1636 }
1637 Emit(kX64Movl, g.DefineAsRegister(node), g.Use(value));
1638 }
1639
1640 namespace {
1641
VisitRO(InstructionSelector * selector,Node * node,InstructionCode opcode)1642 void VisitRO(InstructionSelector* selector, Node* node,
1643 InstructionCode opcode) {
1644 X64OperandGenerator g(selector);
1645 selector->Emit(opcode, g.DefineAsRegister(node), g.Use(node->InputAt(0)));
1646 }
1647
VisitRR(InstructionSelector * selector,Node * node,InstructionCode opcode)1648 void VisitRR(InstructionSelector* selector, Node* node,
1649 InstructionCode opcode) {
1650 X64OperandGenerator g(selector);
1651 selector->Emit(opcode, g.DefineAsRegister(node),
1652 g.UseRegister(node->InputAt(0)));
1653 }
1654
VisitRRO(InstructionSelector * selector,Node * node,InstructionCode opcode)1655 void VisitRRO(InstructionSelector* selector, Node* node,
1656 InstructionCode opcode) {
1657 X64OperandGenerator g(selector);
1658 selector->Emit(opcode, g.DefineSameAsFirst(node),
1659 g.UseRegister(node->InputAt(0)), g.Use(node->InputAt(1)));
1660 }
1661
VisitFloatBinop(InstructionSelector * selector,Node * node,InstructionCode avx_opcode,InstructionCode sse_opcode)1662 void VisitFloatBinop(InstructionSelector* selector, Node* node,
1663 InstructionCode avx_opcode, InstructionCode sse_opcode) {
1664 X64OperandGenerator g(selector);
1665 Node* left = node->InputAt(0);
1666 Node* right = node->InputAt(1);
1667 InstructionOperand inputs[8];
1668 size_t input_count = 0;
1669 InstructionOperand outputs[1];
1670 size_t output_count = 0;
1671
1672 if (left == right) {
1673 // If both inputs refer to the same operand, enforce allocating a register
1674 // for both of them to ensure that we don't end up generating code like
1675 // this:
1676 //
1677 // movss rax, [rbp-0x10]
1678 // addss rax, [rbp-0x10]
1679 // jo label
1680 InstructionOperand const input = g.UseRegister(left);
1681 inputs[input_count++] = input;
1682 inputs[input_count++] = input;
1683 } else {
1684 int effect_level = selector->GetEffectLevel(node);
1685 if (node->op()->HasProperty(Operator::kCommutative) &&
1686 (g.CanBeBetterLeftOperand(right) ||
1687 g.CanBeMemoryOperand(avx_opcode, node, left, effect_level)) &&
1688 (!g.CanBeBetterLeftOperand(left) ||
1689 !g.CanBeMemoryOperand(avx_opcode, node, right, effect_level))) {
1690 std::swap(left, right);
1691 }
1692 if (g.CanBeMemoryOperand(avx_opcode, node, right, effect_level)) {
1693 inputs[input_count++] = g.UseRegister(left);
1694 AddressingMode addressing_mode =
1695 g.GetEffectiveAddressMemoryOperand(right, inputs, &input_count);
1696 avx_opcode |= AddressingModeField::encode(addressing_mode);
1697 sse_opcode |= AddressingModeField::encode(addressing_mode);
1698 } else {
1699 inputs[input_count++] = g.UseRegister(left);
1700 inputs[input_count++] = g.Use(right);
1701 }
1702 }
1703
1704 DCHECK_NE(0u, input_count);
1705 DCHECK_GE(arraysize(inputs), input_count);
1706
1707 if (selector->IsSupported(AVX)) {
1708 outputs[output_count++] = g.DefineAsRegister(node);
1709 DCHECK_EQ(1u, output_count);
1710 DCHECK_GE(arraysize(outputs), output_count);
1711 selector->Emit(avx_opcode, output_count, outputs, input_count, inputs);
1712 } else {
1713 outputs[output_count++] = g.DefineSameAsFirst(node);
1714 DCHECK_EQ(1u, output_count);
1715 DCHECK_GE(arraysize(outputs), output_count);
1716 selector->Emit(sse_opcode, output_count, outputs, input_count, inputs);
1717 }
1718 }
1719
VisitFloatUnop(InstructionSelector * selector,Node * node,Node * input,ArchOpcode opcode)1720 void VisitFloatUnop(InstructionSelector* selector, Node* node, Node* input,
1721 ArchOpcode opcode) {
1722 X64OperandGenerator g(selector);
1723 if (selector->IsSupported(AVX)) {
1724 selector->Emit(opcode, g.DefineAsRegister(node), g.UseRegister(input));
1725 } else {
1726 selector->Emit(opcode, g.DefineSameAsFirst(node), g.UseRegister(input));
1727 }
1728 }
1729
1730 } // namespace
1731
1732 #define RO_OP_LIST(V) \
1733 V(Word64Clz, kX64Lzcnt) \
1734 V(Word32Clz, kX64Lzcnt32) \
1735 V(Word64Ctz, kX64Tzcnt) \
1736 V(Word32Ctz, kX64Tzcnt32) \
1737 V(Word64Popcnt, kX64Popcnt) \
1738 V(Word32Popcnt, kX64Popcnt32) \
1739 V(Float64Sqrt, kSSEFloat64Sqrt) \
1740 V(Float32Sqrt, kSSEFloat32Sqrt) \
1741 V(ChangeFloat64ToInt32, kSSEFloat64ToInt32) \
1742 V(ChangeFloat64ToInt64, kSSEFloat64ToInt64) \
1743 V(ChangeFloat64ToUint32, kSSEFloat64ToUint32 | MiscField::encode(1)) \
1744 V(TruncateFloat64ToInt64, kSSEFloat64ToInt64) \
1745 V(TruncateFloat64ToUint32, kSSEFloat64ToUint32 | MiscField::encode(0)) \
1746 V(ChangeFloat64ToUint64, kSSEFloat64ToUint64) \
1747 V(TruncateFloat64ToFloat32, kSSEFloat64ToFloat32) \
1748 V(ChangeFloat32ToFloat64, kSSEFloat32ToFloat64) \
1749 V(TruncateFloat32ToInt32, kSSEFloat32ToInt32) \
1750 V(TruncateFloat32ToUint32, kSSEFloat32ToUint32) \
1751 V(ChangeInt32ToFloat64, kSSEInt32ToFloat64) \
1752 V(ChangeInt64ToFloat64, kSSEInt64ToFloat64) \
1753 V(ChangeUint32ToFloat64, kSSEUint32ToFloat64) \
1754 V(RoundFloat64ToInt32, kSSEFloat64ToInt32) \
1755 V(RoundInt32ToFloat32, kSSEInt32ToFloat32) \
1756 V(RoundInt64ToFloat32, kSSEInt64ToFloat32) \
1757 V(RoundUint64ToFloat32, kSSEUint64ToFloat32) \
1758 V(RoundInt64ToFloat64, kSSEInt64ToFloat64) \
1759 V(RoundUint64ToFloat64, kSSEUint64ToFloat64) \
1760 V(RoundUint32ToFloat32, kSSEUint32ToFloat32) \
1761 V(BitcastFloat32ToInt32, kX64BitcastFI) \
1762 V(BitcastFloat64ToInt64, kX64BitcastDL) \
1763 V(BitcastInt32ToFloat32, kX64BitcastIF) \
1764 V(BitcastInt64ToFloat64, kX64BitcastLD) \
1765 V(Float64ExtractLowWord32, kSSEFloat64ExtractLowWord32) \
1766 V(Float64ExtractHighWord32, kSSEFloat64ExtractHighWord32) \
1767 V(SignExtendWord8ToInt32, kX64Movsxbl) \
1768 V(SignExtendWord16ToInt32, kX64Movsxwl) \
1769 V(SignExtendWord8ToInt64, kX64Movsxbq) \
1770 V(SignExtendWord16ToInt64, kX64Movsxwq) \
1771 V(SignExtendWord32ToInt64, kX64Movsxlq)
1772
1773 #define RR_OP_LIST(V) \
1774 V(Float32RoundDown, kSSEFloat32Round | MiscField::encode(kRoundDown)) \
1775 V(Float64RoundDown, kSSEFloat64Round | MiscField::encode(kRoundDown)) \
1776 V(Float32RoundUp, kSSEFloat32Round | MiscField::encode(kRoundUp)) \
1777 V(Float64RoundUp, kSSEFloat64Round | MiscField::encode(kRoundUp)) \
1778 V(Float32RoundTruncate, kSSEFloat32Round | MiscField::encode(kRoundToZero)) \
1779 V(Float64RoundTruncate, kSSEFloat64Round | MiscField::encode(kRoundToZero)) \
1780 V(Float32RoundTiesEven, \
1781 kSSEFloat32Round | MiscField::encode(kRoundToNearest)) \
1782 V(Float64RoundTiesEven, \
1783 kSSEFloat64Round | MiscField::encode(kRoundToNearest)) \
1784 V(F32x4Ceil, kX64F32x4Round | MiscField::encode(kRoundUp)) \
1785 V(F32x4Floor, kX64F32x4Round | MiscField::encode(kRoundDown)) \
1786 V(F32x4Trunc, kX64F32x4Round | MiscField::encode(kRoundToZero)) \
1787 V(F32x4NearestInt, kX64F32x4Round | MiscField::encode(kRoundToNearest)) \
1788 V(F64x2Ceil, kX64F64x2Round | MiscField::encode(kRoundUp)) \
1789 V(F64x2Floor, kX64F64x2Round | MiscField::encode(kRoundDown)) \
1790 V(F64x2Trunc, kX64F64x2Round | MiscField::encode(kRoundToZero)) \
1791 V(F64x2NearestInt, kX64F64x2Round | MiscField::encode(kRoundToNearest))
1792
1793 #define RO_VISITOR(Name, opcode) \
1794 void InstructionSelector::Visit##Name(Node* node) { \
1795 VisitRO(this, node, opcode); \
1796 }
1797 RO_OP_LIST(RO_VISITOR)
1798 #undef RO_VISITOR
1799 #undef RO_OP_LIST
1800
1801 #define RR_VISITOR(Name, opcode) \
1802 void InstructionSelector::Visit##Name(Node* node) { \
1803 VisitRR(this, node, opcode); \
1804 }
RR_OP_LIST(RR_VISITOR)1805 RR_OP_LIST(RR_VISITOR)
1806 #undef RR_VISITOR
1807 #undef RR_OP_LIST
1808
1809 void InstructionSelector::VisitTruncateFloat64ToWord32(Node* node) {
1810 VisitRR(this, node, kArchTruncateDoubleToI);
1811 }
1812
VisitTruncateInt64ToInt32(Node * node)1813 void InstructionSelector::VisitTruncateInt64ToInt32(Node* node) {
1814 // We rely on the fact that TruncateInt64ToInt32 zero extends the
1815 // value (see ZeroExtendsWord32ToWord64). So all code paths here
1816 // have to satisfy that condition.
1817 X64OperandGenerator g(this);
1818 Node* value = node->InputAt(0);
1819 if (CanCover(node, value)) {
1820 switch (value->opcode()) {
1821 case IrOpcode::kWord64Sar:
1822 case IrOpcode::kWord64Shr: {
1823 Int64BinopMatcher m(value);
1824 if (m.right().Is(32)) {
1825 if (CanCover(value, value->InputAt(0)) &&
1826 TryMatchLoadWord64AndShiftRight(this, value, kX64Movl)) {
1827 return EmitIdentity(node);
1828 }
1829 Emit(kX64Shr, g.DefineSameAsFirst(node),
1830 g.UseRegister(m.left().node()), g.TempImmediate(32));
1831 return;
1832 }
1833 break;
1834 }
1835 case IrOpcode::kLoad:
1836 case IrOpcode::kLoadImmutable: {
1837 if (TryMergeTruncateInt64ToInt32IntoLoad(this, node, value)) {
1838 return;
1839 }
1840 break;
1841 }
1842 default:
1843 break;
1844 }
1845 }
1846 Emit(kX64Movl, g.DefineAsRegister(node), g.Use(value));
1847 }
1848
VisitFloat32Add(Node * node)1849 void InstructionSelector::VisitFloat32Add(Node* node) {
1850 VisitFloatBinop(this, node, kAVXFloat32Add, kSSEFloat32Add);
1851 }
1852
VisitFloat32Sub(Node * node)1853 void InstructionSelector::VisitFloat32Sub(Node* node) {
1854 VisitFloatBinop(this, node, kAVXFloat32Sub, kSSEFloat32Sub);
1855 }
1856
VisitFloat32Mul(Node * node)1857 void InstructionSelector::VisitFloat32Mul(Node* node) {
1858 VisitFloatBinop(this, node, kAVXFloat32Mul, kSSEFloat32Mul);
1859 }
1860
VisitFloat32Div(Node * node)1861 void InstructionSelector::VisitFloat32Div(Node* node) {
1862 VisitFloatBinop(this, node, kAVXFloat32Div, kSSEFloat32Div);
1863 }
1864
VisitFloat32Abs(Node * node)1865 void InstructionSelector::VisitFloat32Abs(Node* node) {
1866 VisitFloatUnop(this, node, node->InputAt(0), kX64Float32Abs);
1867 }
1868
VisitFloat32Max(Node * node)1869 void InstructionSelector::VisitFloat32Max(Node* node) {
1870 VisitRRO(this, node, kSSEFloat32Max);
1871 }
1872
VisitFloat32Min(Node * node)1873 void InstructionSelector::VisitFloat32Min(Node* node) {
1874 VisitRRO(this, node, kSSEFloat32Min);
1875 }
1876
VisitFloat64Add(Node * node)1877 void InstructionSelector::VisitFloat64Add(Node* node) {
1878 VisitFloatBinop(this, node, kAVXFloat64Add, kSSEFloat64Add);
1879 }
1880
VisitFloat64Sub(Node * node)1881 void InstructionSelector::VisitFloat64Sub(Node* node) {
1882 VisitFloatBinop(this, node, kAVXFloat64Sub, kSSEFloat64Sub);
1883 }
1884
VisitFloat64Mul(Node * node)1885 void InstructionSelector::VisitFloat64Mul(Node* node) {
1886 VisitFloatBinop(this, node, kAVXFloat64Mul, kSSEFloat64Mul);
1887 }
1888
VisitFloat64Div(Node * node)1889 void InstructionSelector::VisitFloat64Div(Node* node) {
1890 VisitFloatBinop(this, node, kAVXFloat64Div, kSSEFloat64Div);
1891 }
1892
VisitFloat64Mod(Node * node)1893 void InstructionSelector::VisitFloat64Mod(Node* node) {
1894 X64OperandGenerator g(this);
1895 InstructionOperand temps[] = {g.TempRegister(rax)};
1896 Emit(kSSEFloat64Mod, g.DefineSameAsFirst(node),
1897 g.UseRegister(node->InputAt(0)), g.UseRegister(node->InputAt(1)), 1,
1898 temps);
1899 }
1900
VisitFloat64Max(Node * node)1901 void InstructionSelector::VisitFloat64Max(Node* node) {
1902 VisitRRO(this, node, kSSEFloat64Max);
1903 }
1904
VisitFloat64Min(Node * node)1905 void InstructionSelector::VisitFloat64Min(Node* node) {
1906 VisitRRO(this, node, kSSEFloat64Min);
1907 }
1908
VisitFloat64Abs(Node * node)1909 void InstructionSelector::VisitFloat64Abs(Node* node) {
1910 VisitFloatUnop(this, node, node->InputAt(0), kX64Float64Abs);
1911 }
1912
VisitFloat64RoundTiesAway(Node * node)1913 void InstructionSelector::VisitFloat64RoundTiesAway(Node* node) {
1914 UNREACHABLE();
1915 }
1916
VisitFloat32Neg(Node * node)1917 void InstructionSelector::VisitFloat32Neg(Node* node) {
1918 VisitFloatUnop(this, node, node->InputAt(0), kX64Float32Neg);
1919 }
1920
VisitFloat64Neg(Node * node)1921 void InstructionSelector::VisitFloat64Neg(Node* node) {
1922 VisitFloatUnop(this, node, node->InputAt(0), kX64Float64Neg);
1923 }
1924
VisitFloat64Ieee754Binop(Node * node,InstructionCode opcode)1925 void InstructionSelector::VisitFloat64Ieee754Binop(Node* node,
1926 InstructionCode opcode) {
1927 X64OperandGenerator g(this);
1928 Emit(opcode, g.DefineAsFixed(node, xmm0), g.UseFixed(node->InputAt(0), xmm0),
1929 g.UseFixed(node->InputAt(1), xmm1))
1930 ->MarkAsCall();
1931 }
1932
VisitFloat64Ieee754Unop(Node * node,InstructionCode opcode)1933 void InstructionSelector::VisitFloat64Ieee754Unop(Node* node,
1934 InstructionCode opcode) {
1935 X64OperandGenerator g(this);
1936 Emit(opcode, g.DefineAsFixed(node, xmm0), g.UseFixed(node->InputAt(0), xmm0))
1937 ->MarkAsCall();
1938 }
1939
EmitPrepareArguments(ZoneVector<PushParameter> * arguments,const CallDescriptor * call_descriptor,Node * node)1940 void InstructionSelector::EmitPrepareArguments(
1941 ZoneVector<PushParameter>* arguments, const CallDescriptor* call_descriptor,
1942 Node* node) {
1943 X64OperandGenerator g(this);
1944
1945 // Prepare for C function call.
1946 if (call_descriptor->IsCFunctionCall()) {
1947 Emit(kArchPrepareCallCFunction | MiscField::encode(static_cast<int>(
1948 call_descriptor->ParameterCount())),
1949 0, nullptr, 0, nullptr);
1950
1951 // Poke any stack arguments.
1952 for (size_t n = 0; n < arguments->size(); ++n) {
1953 PushParameter input = (*arguments)[n];
1954 if (input.node) {
1955 int slot = static_cast<int>(n);
1956 InstructionOperand value = g.CanBeImmediate(input.node)
1957 ? g.UseImmediate(input.node)
1958 : g.UseRegister(input.node);
1959 Emit(kX64Poke | MiscField::encode(slot), g.NoOutput(), value);
1960 }
1961 }
1962 } else {
1963 // Push any stack arguments.
1964 int effect_level = GetEffectLevel(node);
1965 int stack_decrement = 0;
1966 for (PushParameter input : base::Reversed(*arguments)) {
1967 stack_decrement += kSystemPointerSize;
1968 // Skip holes in the param array. These represent both extra slots for
1969 // multi-slot values and padding slots for alignment.
1970 if (input.node == nullptr) continue;
1971 InstructionOperand decrement = g.UseImmediate(stack_decrement);
1972 stack_decrement = 0;
1973 if (g.CanBeImmediate(input.node)) {
1974 Emit(kX64Push, g.NoOutput(), decrement, g.UseImmediate(input.node));
1975 } else if (IsSupported(INTEL_ATOM) ||
1976 sequence()->IsFP(GetVirtualRegister(input.node))) {
1977 // TODO(titzer): X64Push cannot handle stack->stack double moves
1978 // because there is no way to encode fixed double slots.
1979 Emit(kX64Push, g.NoOutput(), decrement, g.UseRegister(input.node));
1980 } else if (g.CanBeMemoryOperand(kX64Push, node, input.node,
1981 effect_level)) {
1982 InstructionOperand outputs[1];
1983 InstructionOperand inputs[5];
1984 size_t input_count = 0;
1985 inputs[input_count++] = decrement;
1986 AddressingMode mode = g.GetEffectiveAddressMemoryOperand(
1987 input.node, inputs, &input_count);
1988 InstructionCode opcode = kX64Push | AddressingModeField::encode(mode);
1989 Emit(opcode, 0, outputs, input_count, inputs);
1990 } else {
1991 Emit(kX64Push, g.NoOutput(), decrement, g.UseAny(input.node));
1992 }
1993 }
1994 }
1995 }
1996
EmitPrepareResults(ZoneVector<PushParameter> * results,const CallDescriptor * call_descriptor,Node * node)1997 void InstructionSelector::EmitPrepareResults(
1998 ZoneVector<PushParameter>* results, const CallDescriptor* call_descriptor,
1999 Node* node) {
2000 X64OperandGenerator g(this);
2001 for (PushParameter output : *results) {
2002 if (!output.location.IsCallerFrameSlot()) continue;
2003 // Skip any alignment holes in nodes.
2004 if (output.node != nullptr) {
2005 DCHECK(!call_descriptor->IsCFunctionCall());
2006 if (output.location.GetType() == MachineType::Float32()) {
2007 MarkAsFloat32(output.node);
2008 } else if (output.location.GetType() == MachineType::Float64()) {
2009 MarkAsFloat64(output.node);
2010 } else if (output.location.GetType() == MachineType::Simd128()) {
2011 MarkAsSimd128(output.node);
2012 }
2013 InstructionOperand result = g.DefineAsRegister(output.node);
2014 int offset = call_descriptor->GetOffsetToReturns();
2015 int reverse_slot = -output.location.GetLocation() - offset;
2016 InstructionOperand slot = g.UseImmediate(reverse_slot);
2017 Emit(kX64Peek, 1, &result, 1, &slot);
2018 }
2019 }
2020 }
2021
IsTailCallAddressImmediate()2022 bool InstructionSelector::IsTailCallAddressImmediate() { return true; }
2023
2024 namespace {
2025
VisitCompareWithMemoryOperand(InstructionSelector * selector,InstructionCode opcode,Node * left,InstructionOperand right,FlagsContinuation * cont)2026 void VisitCompareWithMemoryOperand(InstructionSelector* selector,
2027 InstructionCode opcode, Node* left,
2028 InstructionOperand right,
2029 FlagsContinuation* cont) {
2030 DCHECK(IrOpcode::kLoad == left->opcode() ||
2031 IrOpcode::kLoadImmutable == left->opcode());
2032 X64OperandGenerator g(selector);
2033 size_t input_count = 0;
2034 InstructionOperand inputs[6];
2035 AddressingMode addressing_mode =
2036 g.GetEffectiveAddressMemoryOperand(left, inputs, &input_count);
2037 opcode |= AddressingModeField::encode(addressing_mode);
2038 inputs[input_count++] = right;
2039 if (cont->IsSelect()) {
2040 if (opcode == kUnorderedEqual) {
2041 cont->Negate();
2042 inputs[input_count++] = g.UseRegister(cont->true_value());
2043 inputs[input_count++] = g.Use(cont->false_value());
2044 } else {
2045 inputs[input_count++] = g.UseRegister(cont->false_value());
2046 inputs[input_count++] = g.Use(cont->true_value());
2047 }
2048 }
2049
2050 selector->EmitWithContinuation(opcode, 0, nullptr, input_count, inputs, cont);
2051 }
2052
2053 // Shared routine for multiple compare operations.
VisitCompare(InstructionSelector * selector,InstructionCode opcode,InstructionOperand left,InstructionOperand right,FlagsContinuation * cont)2054 void VisitCompare(InstructionSelector* selector, InstructionCode opcode,
2055 InstructionOperand left, InstructionOperand right,
2056 FlagsContinuation* cont) {
2057 if (cont->IsSelect()) {
2058 X64OperandGenerator g(selector);
2059 InstructionOperand inputs[4] = {left, right};
2060 if (cont->condition() == kUnorderedEqual) {
2061 cont->Negate();
2062 inputs[2] = g.UseRegister(cont->true_value());
2063 inputs[3] = g.Use(cont->false_value());
2064 } else {
2065 inputs[2] = g.UseRegister(cont->false_value());
2066 inputs[3] = g.Use(cont->true_value());
2067 }
2068 selector->EmitWithContinuation(opcode, 0, nullptr, 4, inputs, cont);
2069 return;
2070 }
2071 selector->EmitWithContinuation(opcode, left, right, cont);
2072 }
2073
2074 // Shared routine for multiple compare operations.
VisitCompare(InstructionSelector * selector,InstructionCode opcode,Node * left,Node * right,FlagsContinuation * cont,bool commutative)2075 void VisitCompare(InstructionSelector* selector, InstructionCode opcode,
2076 Node* left, Node* right, FlagsContinuation* cont,
2077 bool commutative) {
2078 X64OperandGenerator g(selector);
2079 if (commutative && g.CanBeBetterLeftOperand(right)) {
2080 std::swap(left, right);
2081 }
2082 VisitCompare(selector, opcode, g.UseRegister(left), g.Use(right), cont);
2083 }
2084
MachineTypeForNarrow(Node * node,Node * hint_node)2085 MachineType MachineTypeForNarrow(Node* node, Node* hint_node) {
2086 if (hint_node->opcode() == IrOpcode::kLoad ||
2087 hint_node->opcode() == IrOpcode::kLoadImmutable) {
2088 MachineType hint = LoadRepresentationOf(hint_node->op());
2089 if (node->opcode() == IrOpcode::kInt32Constant ||
2090 node->opcode() == IrOpcode::kInt64Constant) {
2091 int64_t constant = node->opcode() == IrOpcode::kInt32Constant
2092 ? OpParameter<int32_t>(node->op())
2093 : OpParameter<int64_t>(node->op());
2094 if (hint == MachineType::Int8()) {
2095 if (constant >= std::numeric_limits<int8_t>::min() &&
2096 constant <= std::numeric_limits<int8_t>::max()) {
2097 return hint;
2098 }
2099 } else if (hint == MachineType::Uint8()) {
2100 if (constant >= std::numeric_limits<uint8_t>::min() &&
2101 constant <= std::numeric_limits<uint8_t>::max()) {
2102 return hint;
2103 }
2104 } else if (hint == MachineType::Int16()) {
2105 if (constant >= std::numeric_limits<int16_t>::min() &&
2106 constant <= std::numeric_limits<int16_t>::max()) {
2107 return hint;
2108 }
2109 } else if (hint == MachineType::Uint16()) {
2110 if (constant >= std::numeric_limits<uint16_t>::min() &&
2111 constant <= std::numeric_limits<uint16_t>::max()) {
2112 return hint;
2113 }
2114 } else if (hint == MachineType::Int32()) {
2115 if (constant >= std::numeric_limits<int32_t>::min() &&
2116 constant <= std::numeric_limits<int32_t>::max()) {
2117 return hint;
2118 }
2119 } else if (hint == MachineType::Uint32()) {
2120 if (constant >= std::numeric_limits<uint32_t>::min() &&
2121 constant <= std::numeric_limits<uint32_t>::max())
2122 return hint;
2123 }
2124 }
2125 }
2126 return node->opcode() == IrOpcode::kLoad ||
2127 node->opcode() == IrOpcode::kLoadImmutable
2128 ? LoadRepresentationOf(node->op())
2129 : MachineType::None();
2130 }
2131
IsIntConstant(Node * node)2132 bool IsIntConstant(Node* node) {
2133 return node->opcode() == IrOpcode::kInt32Constant ||
2134 node->opcode() == IrOpcode::kInt64Constant;
2135 }
2136
IsWordAnd(Node * node)2137 bool IsWordAnd(Node* node) {
2138 return node->opcode() == IrOpcode::kWord32And ||
2139 node->opcode() == IrOpcode::kWord64And;
2140 }
2141
2142 // The result of WordAnd with a positive interger constant in X64 is known to
2143 // be sign(zero)-extended. Comparing this result with another positive interger
2144 // constant can have narrowed operand.
MachineTypeForNarrowWordAnd(Node * and_node,Node * constant_node)2145 MachineType MachineTypeForNarrowWordAnd(Node* and_node, Node* constant_node) {
2146 Node* and_left = and_node->InputAt(0);
2147 Node* and_right = and_node->InputAt(1);
2148 Node* and_constant_node = IsIntConstant(and_right)
2149 ? and_right
2150 : IsIntConstant(and_left) ? and_left : nullptr;
2151
2152 if (and_constant_node != nullptr) {
2153 int64_t and_constant =
2154 and_constant_node->opcode() == IrOpcode::kInt32Constant
2155 ? OpParameter<int32_t>(and_constant_node->op())
2156 : OpParameter<int64_t>(and_constant_node->op());
2157 int64_t cmp_constant = constant_node->opcode() == IrOpcode::kInt32Constant
2158 ? OpParameter<int32_t>(constant_node->op())
2159 : OpParameter<int64_t>(constant_node->op());
2160 if (and_constant >= 0 && cmp_constant >= 0) {
2161 int64_t constant =
2162 and_constant > cmp_constant ? and_constant : cmp_constant;
2163 if (constant <= std::numeric_limits<int8_t>::max()) {
2164 return MachineType::Int8();
2165 } else if (constant <= std::numeric_limits<uint8_t>::max()) {
2166 return MachineType::Uint8();
2167 } else if (constant <= std::numeric_limits<int16_t>::max()) {
2168 return MachineType::Int16();
2169 } else if (constant <= std::numeric_limits<uint16_t>::max()) {
2170 return MachineType::Uint16();
2171 } else if (constant <= std::numeric_limits<int32_t>::max()) {
2172 return MachineType::Int32();
2173 } else if (constant <= std::numeric_limits<uint32_t>::max()) {
2174 return MachineType::Uint32();
2175 }
2176 }
2177 }
2178
2179 return MachineType::None();
2180 }
2181
2182 // Tries to match the size of the given opcode to that of the operands, if
2183 // possible.
TryNarrowOpcodeSize(InstructionCode opcode,Node * left,Node * right,FlagsContinuation * cont)2184 InstructionCode TryNarrowOpcodeSize(InstructionCode opcode, Node* left,
2185 Node* right, FlagsContinuation* cont) {
2186 MachineType left_type = MachineType::None();
2187 MachineType right_type = MachineType::None();
2188 if (IsWordAnd(left) && IsIntConstant(right)) {
2189 left_type = MachineTypeForNarrowWordAnd(left, right);
2190 right_type = left_type;
2191 } else if (IsWordAnd(right) && IsIntConstant(left)) {
2192 right_type = MachineTypeForNarrowWordAnd(right, left);
2193 left_type = right_type;
2194 } else {
2195 // TODO(epertoso): we can probably get some size information out phi nodes.
2196 // If the load representations don't match, both operands will be
2197 // zero/sign-extended to 32bit.
2198 left_type = MachineTypeForNarrow(left, right);
2199 right_type = MachineTypeForNarrow(right, left);
2200 }
2201 if (left_type == right_type) {
2202 switch (left_type.representation()) {
2203 case MachineRepresentation::kBit:
2204 case MachineRepresentation::kWord8: {
2205 if (opcode == kX64Test || opcode == kX64Test32) return kX64Test8;
2206 if (opcode == kX64Cmp || opcode == kX64Cmp32) {
2207 if (left_type.semantic() == MachineSemantic::kUint32) {
2208 cont->OverwriteUnsignedIfSigned();
2209 } else {
2210 CHECK_EQ(MachineSemantic::kInt32, left_type.semantic());
2211 }
2212 return kX64Cmp8;
2213 }
2214 break;
2215 }
2216 case MachineRepresentation::kWord16:
2217 if (opcode == kX64Test || opcode == kX64Test32) return kX64Test16;
2218 if (opcode == kX64Cmp || opcode == kX64Cmp32) {
2219 if (left_type.semantic() == MachineSemantic::kUint32) {
2220 cont->OverwriteUnsignedIfSigned();
2221 } else {
2222 CHECK_EQ(MachineSemantic::kInt32, left_type.semantic());
2223 }
2224 return kX64Cmp16;
2225 }
2226 break;
2227 case MachineRepresentation::kWord32:
2228 if (opcode == kX64Test) return kX64Test32;
2229 if (opcode == kX64Cmp) {
2230 if (left_type.semantic() == MachineSemantic::kUint32) {
2231 cont->OverwriteUnsignedIfSigned();
2232 } else {
2233 CHECK_EQ(MachineSemantic::kInt32, left_type.semantic());
2234 }
2235 return kX64Cmp32;
2236 }
2237 break;
2238 #ifdef V8_COMPRESS_POINTERS
2239 case MachineRepresentation::kTaggedSigned:
2240 case MachineRepresentation::kTaggedPointer:
2241 case MachineRepresentation::kTagged:
2242 // When pointer compression is enabled the lower 32-bits uniquely
2243 // identify tagged value.
2244 if (opcode == kX64Cmp) return kX64Cmp32;
2245 break;
2246 #endif
2247 default:
2248 break;
2249 }
2250 }
2251 return opcode;
2252 }
2253
2254 // Shared routine for multiple word compare operations.
VisitWordCompare(InstructionSelector * selector,Node * node,InstructionCode opcode,FlagsContinuation * cont)2255 void VisitWordCompare(InstructionSelector* selector, Node* node,
2256 InstructionCode opcode, FlagsContinuation* cont) {
2257 X64OperandGenerator g(selector);
2258 Node* left = node->InputAt(0);
2259 Node* right = node->InputAt(1);
2260
2261 // The 32-bit comparisons automatically truncate Word64
2262 // values to Word32 range, no need to do that explicitly.
2263 if (opcode == kX64Cmp32 || opcode == kX64Test32) {
2264 if (left->opcode() == IrOpcode::kTruncateInt64ToInt32) {
2265 left = left->InputAt(0);
2266 }
2267
2268 if (right->opcode() == IrOpcode::kTruncateInt64ToInt32) {
2269 right = right->InputAt(0);
2270 }
2271 }
2272
2273 opcode = TryNarrowOpcodeSize(opcode, left, right, cont);
2274
2275 // If one of the two inputs is an immediate, make sure it's on the right, or
2276 // if one of the two inputs is a memory operand, make sure it's on the left.
2277 int effect_level = selector->GetEffectLevel(node, cont);
2278
2279 if ((!g.CanBeImmediate(right) && g.CanBeImmediate(left)) ||
2280 (g.CanBeMemoryOperand(opcode, node, right, effect_level) &&
2281 !g.CanBeMemoryOperand(opcode, node, left, effect_level))) {
2282 if (!node->op()->HasProperty(Operator::kCommutative)) cont->Commute();
2283 std::swap(left, right);
2284 }
2285
2286 // Match immediates on right side of comparison.
2287 if (g.CanBeImmediate(right)) {
2288 if (g.CanBeMemoryOperand(opcode, node, left, effect_level)) {
2289 return VisitCompareWithMemoryOperand(selector, opcode, left,
2290 g.UseImmediate(right), cont);
2291 }
2292 return VisitCompare(selector, opcode, g.Use(left), g.UseImmediate(right),
2293 cont);
2294 }
2295
2296 // Match memory operands on left side of comparison.
2297 if (g.CanBeMemoryOperand(opcode, node, left, effect_level)) {
2298 return VisitCompareWithMemoryOperand(selector, opcode, left,
2299 g.UseRegister(right), cont);
2300 }
2301
2302 return VisitCompare(selector, opcode, left, right, cont,
2303 node->op()->HasProperty(Operator::kCommutative));
2304 }
2305
VisitWord64EqualImpl(InstructionSelector * selector,Node * node,FlagsContinuation * cont)2306 void VisitWord64EqualImpl(InstructionSelector* selector, Node* node,
2307 FlagsContinuation* cont) {
2308 if (selector->CanUseRootsRegister()) {
2309 X64OperandGenerator g(selector);
2310 const RootsTable& roots_table = selector->isolate()->roots_table();
2311 RootIndex root_index;
2312 HeapObjectBinopMatcher m(node);
2313 if (m.right().HasResolvedValue() &&
2314 roots_table.IsRootHandle(m.right().ResolvedValue(), &root_index)) {
2315 InstructionCode opcode =
2316 kX64Cmp | AddressingModeField::encode(kMode_Root);
2317 return VisitCompare(
2318 selector, opcode,
2319 g.TempImmediate(
2320 TurboAssemblerBase::RootRegisterOffsetForRootIndex(root_index)),
2321 g.UseRegister(m.left().node()), cont);
2322 }
2323 }
2324 VisitWordCompare(selector, node, kX64Cmp, cont);
2325 }
2326
VisitWord32EqualImpl(InstructionSelector * selector,Node * node,FlagsContinuation * cont)2327 void VisitWord32EqualImpl(InstructionSelector* selector, Node* node,
2328 FlagsContinuation* cont) {
2329 if (COMPRESS_POINTERS_BOOL && selector->CanUseRootsRegister()) {
2330 X64OperandGenerator g(selector);
2331 const RootsTable& roots_table = selector->isolate()->roots_table();
2332 RootIndex root_index;
2333 Node* left = nullptr;
2334 Handle<HeapObject> right;
2335 // HeapConstants and CompressedHeapConstants can be treated the same when
2336 // using them as an input to a 32-bit comparison. Check whether either is
2337 // present.
2338 {
2339 CompressedHeapObjectBinopMatcher m(node);
2340 if (m.right().HasResolvedValue()) {
2341 left = m.left().node();
2342 right = m.right().ResolvedValue();
2343 } else {
2344 HeapObjectBinopMatcher m2(node);
2345 if (m2.right().HasResolvedValue()) {
2346 left = m2.left().node();
2347 right = m2.right().ResolvedValue();
2348 }
2349 }
2350 }
2351 if (!right.is_null() && roots_table.IsRootHandle(right, &root_index)) {
2352 DCHECK_NE(left, nullptr);
2353 InstructionCode opcode =
2354 kX64Cmp32 | AddressingModeField::encode(kMode_Root);
2355 return VisitCompare(
2356 selector, opcode,
2357 g.TempImmediate(
2358 TurboAssemblerBase::RootRegisterOffsetForRootIndex(root_index)),
2359 g.UseRegister(left), cont);
2360 }
2361 }
2362 VisitWordCompare(selector, node, kX64Cmp32, cont);
2363 }
2364
2365 // Shared routine for comparison with zero.
VisitCompareZero(InstructionSelector * selector,Node * user,Node * node,InstructionCode opcode,FlagsContinuation * cont)2366 void VisitCompareZero(InstructionSelector* selector, Node* user, Node* node,
2367 InstructionCode opcode, FlagsContinuation* cont) {
2368 X64OperandGenerator g(selector);
2369 if (cont->IsBranch() &&
2370 (cont->condition() == kNotEqual || cont->condition() == kEqual)) {
2371 switch (node->opcode()) {
2372 #define FLAGS_SET_BINOP_LIST(V) \
2373 V(kInt32Add, VisitBinop, kX64Add32) \
2374 V(kInt32Sub, VisitBinop, kX64Sub32) \
2375 V(kWord32And, VisitBinop, kX64And32) \
2376 V(kWord32Or, VisitBinop, kX64Or32) \
2377 V(kInt64Add, VisitBinop, kX64Add) \
2378 V(kInt64Sub, VisitBinop, kX64Sub) \
2379 V(kWord64And, VisitBinop, kX64And) \
2380 V(kWord64Or, VisitBinop, kX64Or)
2381 #define FLAGS_SET_BINOP(opcode, Visit, archOpcode) \
2382 case IrOpcode::opcode: \
2383 if (selector->IsOnlyUserOfNodeInSameBlock(user, node)) { \
2384 return Visit(selector, node, archOpcode, cont); \
2385 } \
2386 break;
2387 FLAGS_SET_BINOP_LIST(FLAGS_SET_BINOP)
2388 #undef FLAGS_SET_BINOP_LIST
2389 #undef FLAGS_SET_BINOP
2390
2391 #define TRY_VISIT_WORD32_SHIFT TryVisitWordShift<Int32BinopMatcher, 32>
2392 #define TRY_VISIT_WORD64_SHIFT TryVisitWordShift<Int64BinopMatcher, 64>
2393 // Skip Word64Sar/Word32Sar since no instruction reduction in most cases.
2394 #define FLAGS_SET_SHIFT_LIST(V) \
2395 V(kWord32Shl, TRY_VISIT_WORD32_SHIFT, kX64Shl32) \
2396 V(kWord32Shr, TRY_VISIT_WORD32_SHIFT, kX64Shr32) \
2397 V(kWord64Shl, TRY_VISIT_WORD64_SHIFT, kX64Shl) \
2398 V(kWord64Shr, TRY_VISIT_WORD64_SHIFT, kX64Shr)
2399 #define FLAGS_SET_SHIFT(opcode, TryVisit, archOpcode) \
2400 case IrOpcode::opcode: \
2401 if (selector->IsOnlyUserOfNodeInSameBlock(user, node)) { \
2402 if (TryVisit(selector, node, archOpcode, cont)) return; \
2403 } \
2404 break;
2405 FLAGS_SET_SHIFT_LIST(FLAGS_SET_SHIFT)
2406 #undef TRY_VISIT_WORD32_SHIFT
2407 #undef TRY_VISIT_WORD64_SHIFT
2408 #undef FLAGS_SET_SHIFT_LIST
2409 #undef FLAGS_SET_SHIFT
2410 default:
2411 break;
2412 }
2413 }
2414 int effect_level = selector->GetEffectLevel(node, cont);
2415 if (node->opcode() == IrOpcode::kLoad ||
2416 node->opcode() == IrOpcode::kLoadImmutable) {
2417 switch (LoadRepresentationOf(node->op()).representation()) {
2418 case MachineRepresentation::kWord8:
2419 if (opcode == kX64Cmp32) {
2420 opcode = kX64Cmp8;
2421 } else if (opcode == kX64Test32) {
2422 opcode = kX64Test8;
2423 }
2424 break;
2425 case MachineRepresentation::kWord16:
2426 if (opcode == kX64Cmp32) {
2427 opcode = kX64Cmp16;
2428 } else if (opcode == kX64Test32) {
2429 opcode = kX64Test16;
2430 }
2431 break;
2432 default:
2433 break;
2434 }
2435 }
2436 if (g.CanBeMemoryOperand(opcode, user, node, effect_level)) {
2437 VisitCompareWithMemoryOperand(selector, opcode, node, g.TempImmediate(0),
2438 cont);
2439 } else {
2440 VisitCompare(selector, opcode, g.Use(node), g.TempImmediate(0), cont);
2441 }
2442 }
2443
2444 // Shared routine for multiple float32 compare operations (inputs commuted).
VisitFloat32Compare(InstructionSelector * selector,Node * node,FlagsContinuation * cont)2445 void VisitFloat32Compare(InstructionSelector* selector, Node* node,
2446 FlagsContinuation* cont) {
2447 Node* const left = node->InputAt(0);
2448 Node* const right = node->InputAt(1);
2449 InstructionCode const opcode =
2450 selector->IsSupported(AVX) ? kAVXFloat32Cmp : kSSEFloat32Cmp;
2451 VisitCompare(selector, opcode, right, left, cont, false);
2452 }
2453
2454 // Shared routine for multiple float64 compare operations (inputs commuted).
VisitFloat64Compare(InstructionSelector * selector,Node * node,FlagsContinuation * cont)2455 void VisitFloat64Compare(InstructionSelector* selector, Node* node,
2456 FlagsContinuation* cont) {
2457 Node* const left = node->InputAt(0);
2458 Node* const right = node->InputAt(1);
2459 InstructionCode const opcode =
2460 selector->IsSupported(AVX) ? kAVXFloat64Cmp : kSSEFloat64Cmp;
2461 VisitCompare(selector, opcode, right, left, cont, false);
2462 }
2463
2464 // Shared routine for Word32/Word64 Atomic Binops
VisitAtomicBinop(InstructionSelector * selector,Node * node,ArchOpcode opcode,AtomicWidth width)2465 void VisitAtomicBinop(InstructionSelector* selector, Node* node,
2466 ArchOpcode opcode, AtomicWidth width) {
2467 X64OperandGenerator g(selector);
2468 Node* base = node->InputAt(0);
2469 Node* index = node->InputAt(1);
2470 Node* value = node->InputAt(2);
2471 AddressingMode addressing_mode;
2472 InstructionOperand inputs[] = {
2473 g.UseUniqueRegister(value), g.UseUniqueRegister(base),
2474 g.GetEffectiveIndexOperand(index, &addressing_mode)};
2475 InstructionOperand outputs[] = {g.DefineAsFixed(node, rax)};
2476 InstructionOperand temps[] = {g.TempRegister()};
2477 InstructionCode code = opcode | AddressingModeField::encode(addressing_mode) |
2478 AtomicWidthField::encode(width);
2479 selector->Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs,
2480 arraysize(temps), temps);
2481 }
2482
2483 // Shared routine for Word32/Word64 Atomic CmpExchg
VisitAtomicCompareExchange(InstructionSelector * selector,Node * node,ArchOpcode opcode,AtomicWidth width)2484 void VisitAtomicCompareExchange(InstructionSelector* selector, Node* node,
2485 ArchOpcode opcode, AtomicWidth width) {
2486 X64OperandGenerator g(selector);
2487 Node* base = node->InputAt(0);
2488 Node* index = node->InputAt(1);
2489 Node* old_value = node->InputAt(2);
2490 Node* new_value = node->InputAt(3);
2491 AddressingMode addressing_mode;
2492 InstructionOperand inputs[] = {
2493 g.UseFixed(old_value, rax), g.UseUniqueRegister(new_value),
2494 g.UseUniqueRegister(base),
2495 g.GetEffectiveIndexOperand(index, &addressing_mode)};
2496 InstructionOperand outputs[] = {g.DefineAsFixed(node, rax)};
2497 InstructionCode code = opcode | AddressingModeField::encode(addressing_mode) |
2498 AtomicWidthField::encode(width);
2499 selector->Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs);
2500 }
2501
2502 } // namespace
2503
2504 // Shared routine for word comparison against zero.
VisitWordCompareZero(Node * user,Node * value,FlagsContinuation * cont)2505 void InstructionSelector::VisitWordCompareZero(Node* user, Node* value,
2506 FlagsContinuation* cont) {
2507 // Try to combine with comparisons against 0 by simply inverting the branch.
2508 while (value->opcode() == IrOpcode::kWord32Equal && CanCover(user, value)) {
2509 Int32BinopMatcher m(value);
2510 if (!m.right().Is(0)) break;
2511
2512 user = value;
2513 value = m.left().node();
2514 cont->Negate();
2515 }
2516
2517 if (CanCover(user, value)) {
2518 switch (value->opcode()) {
2519 case IrOpcode::kWord32Equal:
2520 cont->OverwriteAndNegateIfEqual(kEqual);
2521 return VisitWord32EqualImpl(this, value, cont);
2522 case IrOpcode::kInt32LessThan:
2523 cont->OverwriteAndNegateIfEqual(kSignedLessThan);
2524 return VisitWordCompare(this, value, kX64Cmp32, cont);
2525 case IrOpcode::kInt32LessThanOrEqual:
2526 cont->OverwriteAndNegateIfEqual(kSignedLessThanOrEqual);
2527 return VisitWordCompare(this, value, kX64Cmp32, cont);
2528 case IrOpcode::kUint32LessThan:
2529 cont->OverwriteAndNegateIfEqual(kUnsignedLessThan);
2530 return VisitWordCompare(this, value, kX64Cmp32, cont);
2531 case IrOpcode::kUint32LessThanOrEqual:
2532 cont->OverwriteAndNegateIfEqual(kUnsignedLessThanOrEqual);
2533 return VisitWordCompare(this, value, kX64Cmp32, cont);
2534 case IrOpcode::kWord64Equal: {
2535 cont->OverwriteAndNegateIfEqual(kEqual);
2536 Int64BinopMatcher m(value);
2537 if (m.right().Is(0)) {
2538 // Try to combine the branch with a comparison.
2539 Node* const eq_user = m.node();
2540 Node* const eq_value = m.left().node();
2541 if (CanCover(eq_user, eq_value)) {
2542 switch (eq_value->opcode()) {
2543 case IrOpcode::kInt64Sub:
2544 return VisitWordCompare(this, eq_value, kX64Cmp, cont);
2545 case IrOpcode::kWord64And:
2546 return VisitWordCompare(this, eq_value, kX64Test, cont);
2547 default:
2548 break;
2549 }
2550 }
2551 return VisitCompareZero(this, eq_user, eq_value, kX64Cmp, cont);
2552 }
2553 return VisitWord64EqualImpl(this, value, cont);
2554 }
2555 case IrOpcode::kInt64LessThan:
2556 cont->OverwriteAndNegateIfEqual(kSignedLessThan);
2557 return VisitWordCompare(this, value, kX64Cmp, cont);
2558 case IrOpcode::kInt64LessThanOrEqual:
2559 cont->OverwriteAndNegateIfEqual(kSignedLessThanOrEqual);
2560 return VisitWordCompare(this, value, kX64Cmp, cont);
2561 case IrOpcode::kUint64LessThan:
2562 cont->OverwriteAndNegateIfEqual(kUnsignedLessThan);
2563 return VisitWordCompare(this, value, kX64Cmp, cont);
2564 case IrOpcode::kUint64LessThanOrEqual:
2565 cont->OverwriteAndNegateIfEqual(kUnsignedLessThanOrEqual);
2566 return VisitWordCompare(this, value, kX64Cmp, cont);
2567 case IrOpcode::kFloat32Equal:
2568 cont->OverwriteAndNegateIfEqual(kUnorderedEqual);
2569 return VisitFloat32Compare(this, value, cont);
2570 case IrOpcode::kFloat32LessThan:
2571 cont->OverwriteAndNegateIfEqual(kUnsignedGreaterThan);
2572 return VisitFloat32Compare(this, value, cont);
2573 case IrOpcode::kFloat32LessThanOrEqual:
2574 cont->OverwriteAndNegateIfEqual(kUnsignedGreaterThanOrEqual);
2575 return VisitFloat32Compare(this, value, cont);
2576 case IrOpcode::kFloat64Equal:
2577 cont->OverwriteAndNegateIfEqual(kUnorderedEqual);
2578 return VisitFloat64Compare(this, value, cont);
2579 case IrOpcode::kFloat64LessThan: {
2580 Float64BinopMatcher m(value);
2581 if (m.left().Is(0.0) && m.right().IsFloat64Abs()) {
2582 // This matches the pattern
2583 //
2584 // Float64LessThan(#0.0, Float64Abs(x))
2585 //
2586 // which TurboFan generates for NumberToBoolean in the general case,
2587 // and which evaluates to false if x is 0, -0 or NaN. We can compile
2588 // this to a simple (v)ucomisd using not_equal flags condition, which
2589 // avoids the costly Float64Abs.
2590 cont->OverwriteAndNegateIfEqual(kNotEqual);
2591 InstructionCode const opcode =
2592 IsSupported(AVX) ? kAVXFloat64Cmp : kSSEFloat64Cmp;
2593 return VisitCompare(this, opcode, m.left().node(),
2594 m.right().InputAt(0), cont, false);
2595 }
2596 cont->OverwriteAndNegateIfEqual(kUnsignedGreaterThan);
2597 return VisitFloat64Compare(this, value, cont);
2598 }
2599 case IrOpcode::kFloat64LessThanOrEqual:
2600 cont->OverwriteAndNegateIfEqual(kUnsignedGreaterThanOrEqual);
2601 return VisitFloat64Compare(this, value, cont);
2602 case IrOpcode::kProjection:
2603 // Check if this is the overflow output projection of an
2604 // <Operation>WithOverflow node.
2605 if (ProjectionIndexOf(value->op()) == 1u) {
2606 // We cannot combine the <Operation>WithOverflow with this branch
2607 // unless the 0th projection (the use of the actual value of the
2608 // <Operation> is either nullptr, which means there's no use of the
2609 // actual value, or was already defined, which means it is scheduled
2610 // *AFTER* this branch).
2611 Node* const node = value->InputAt(0);
2612 Node* const result = NodeProperties::FindProjection(node, 0);
2613 if (result == nullptr || IsDefined(result)) {
2614 switch (node->opcode()) {
2615 case IrOpcode::kInt32AddWithOverflow:
2616 cont->OverwriteAndNegateIfEqual(kOverflow);
2617 return VisitBinop(this, node, kX64Add32, cont);
2618 case IrOpcode::kInt32SubWithOverflow:
2619 cont->OverwriteAndNegateIfEqual(kOverflow);
2620 return VisitBinop(this, node, kX64Sub32, cont);
2621 case IrOpcode::kInt32MulWithOverflow:
2622 cont->OverwriteAndNegateIfEqual(kOverflow);
2623 return VisitBinop(this, node, kX64Imul32, cont);
2624 case IrOpcode::kInt64AddWithOverflow:
2625 cont->OverwriteAndNegateIfEqual(kOverflow);
2626 return VisitBinop(this, node, kX64Add, cont);
2627 case IrOpcode::kInt64SubWithOverflow:
2628 cont->OverwriteAndNegateIfEqual(kOverflow);
2629 return VisitBinop(this, node, kX64Sub, cont);
2630 default:
2631 break;
2632 }
2633 }
2634 }
2635 break;
2636 case IrOpcode::kInt32Sub:
2637 return VisitWordCompare(this, value, kX64Cmp32, cont);
2638 case IrOpcode::kWord32And:
2639 return VisitWordCompare(this, value, kX64Test32, cont);
2640 case IrOpcode::kStackPointerGreaterThan:
2641 cont->OverwriteAndNegateIfEqual(kStackPointerGreaterThanCondition);
2642 return VisitStackPointerGreaterThan(value, cont);
2643 default:
2644 break;
2645 }
2646 }
2647
2648 // Branch could not be combined with a compare, emit compare against 0.
2649 VisitCompareZero(this, user, value, kX64Cmp32, cont);
2650 }
2651
VisitSwitch(Node * node,const SwitchInfo & sw)2652 void InstructionSelector::VisitSwitch(Node* node, const SwitchInfo& sw) {
2653 X64OperandGenerator g(this);
2654 InstructionOperand value_operand = g.UseRegister(node->InputAt(0));
2655
2656 // Emit either ArchTableSwitch or ArchBinarySearchSwitch.
2657 if (enable_switch_jump_table_ == kEnableSwitchJumpTable) {
2658 static const size_t kMaxTableSwitchValueRange = 2 << 16;
2659 size_t table_space_cost = 4 + sw.value_range();
2660 size_t table_time_cost = 3;
2661 size_t lookup_space_cost = 3 + 2 * sw.case_count();
2662 size_t lookup_time_cost = sw.case_count();
2663 if (sw.case_count() > 4 &&
2664 table_space_cost + 3 * table_time_cost <=
2665 lookup_space_cost + 3 * lookup_time_cost &&
2666 sw.min_value() > std::numeric_limits<int32_t>::min() &&
2667 sw.value_range() <= kMaxTableSwitchValueRange) {
2668 InstructionOperand index_operand = g.TempRegister();
2669 if (sw.min_value()) {
2670 // The leal automatically zero extends, so result is a valid 64-bit
2671 // index.
2672 Emit(kX64Lea32 | AddressingModeField::encode(kMode_MRI), index_operand,
2673 value_operand, g.TempImmediate(-sw.min_value()));
2674 } else {
2675 // Zero extend, because we use it as 64-bit index into the jump table.
2676 if (ZeroExtendsWord32ToWord64(node->InputAt(0))) {
2677 // Input value has already been zero-extended.
2678 index_operand = value_operand;
2679 } else {
2680 Emit(kX64Movl, index_operand, value_operand);
2681 }
2682 }
2683 // Generate a table lookup.
2684 return EmitTableSwitch(sw, index_operand);
2685 }
2686 }
2687
2688 // Generate a tree of conditional jumps.
2689 return EmitBinarySearchSwitch(sw, value_operand);
2690 }
2691
VisitWord32Equal(Node * const node)2692 void InstructionSelector::VisitWord32Equal(Node* const node) {
2693 Node* user = node;
2694 FlagsContinuation cont = FlagsContinuation::ForSet(kEqual, node);
2695 Int32BinopMatcher m(user);
2696 if (m.right().Is(0)) {
2697 return VisitWordCompareZero(m.node(), m.left().node(), &cont);
2698 }
2699 VisitWord32EqualImpl(this, node, &cont);
2700 }
2701
VisitInt32LessThan(Node * node)2702 void InstructionSelector::VisitInt32LessThan(Node* node) {
2703 FlagsContinuation cont = FlagsContinuation::ForSet(kSignedLessThan, node);
2704 VisitWordCompare(this, node, kX64Cmp32, &cont);
2705 }
2706
VisitInt32LessThanOrEqual(Node * node)2707 void InstructionSelector::VisitInt32LessThanOrEqual(Node* node) {
2708 FlagsContinuation cont =
2709 FlagsContinuation::ForSet(kSignedLessThanOrEqual, node);
2710 VisitWordCompare(this, node, kX64Cmp32, &cont);
2711 }
2712
VisitUint32LessThan(Node * node)2713 void InstructionSelector::VisitUint32LessThan(Node* node) {
2714 FlagsContinuation cont = FlagsContinuation::ForSet(kUnsignedLessThan, node);
2715 VisitWordCompare(this, node, kX64Cmp32, &cont);
2716 }
2717
VisitUint32LessThanOrEqual(Node * node)2718 void InstructionSelector::VisitUint32LessThanOrEqual(Node* node) {
2719 FlagsContinuation cont =
2720 FlagsContinuation::ForSet(kUnsignedLessThanOrEqual, node);
2721 VisitWordCompare(this, node, kX64Cmp32, &cont);
2722 }
2723
VisitWord64Equal(Node * node)2724 void InstructionSelector::VisitWord64Equal(Node* node) {
2725 FlagsContinuation cont = FlagsContinuation::ForSet(kEqual, node);
2726 Int64BinopMatcher m(node);
2727 if (m.right().Is(0)) {
2728 // Try to combine the equality check with a comparison.
2729 Node* const user = m.node();
2730 Node* const value = m.left().node();
2731 if (CanCover(user, value)) {
2732 switch (value->opcode()) {
2733 case IrOpcode::kInt64Sub:
2734 return VisitWordCompare(this, value, kX64Cmp, &cont);
2735 case IrOpcode::kWord64And:
2736 return VisitWordCompare(this, value, kX64Test, &cont);
2737 default:
2738 break;
2739 }
2740 }
2741 }
2742 VisitWord64EqualImpl(this, node, &cont);
2743 }
2744
VisitInt32AddWithOverflow(Node * node)2745 void InstructionSelector::VisitInt32AddWithOverflow(Node* node) {
2746 if (Node* ovf = NodeProperties::FindProjection(node, 1)) {
2747 FlagsContinuation cont = FlagsContinuation::ForSet(kOverflow, ovf);
2748 return VisitBinop(this, node, kX64Add32, &cont);
2749 }
2750 FlagsContinuation cont;
2751 VisitBinop(this, node, kX64Add32, &cont);
2752 }
2753
VisitInt32SubWithOverflow(Node * node)2754 void InstructionSelector::VisitInt32SubWithOverflow(Node* node) {
2755 if (Node* ovf = NodeProperties::FindProjection(node, 1)) {
2756 FlagsContinuation cont = FlagsContinuation::ForSet(kOverflow, ovf);
2757 return VisitBinop(this, node, kX64Sub32, &cont);
2758 }
2759 FlagsContinuation cont;
2760 VisitBinop(this, node, kX64Sub32, &cont);
2761 }
2762
VisitInt64LessThan(Node * node)2763 void InstructionSelector::VisitInt64LessThan(Node* node) {
2764 FlagsContinuation cont = FlagsContinuation::ForSet(kSignedLessThan, node);
2765 VisitWordCompare(this, node, kX64Cmp, &cont);
2766 }
2767
VisitInt64LessThanOrEqual(Node * node)2768 void InstructionSelector::VisitInt64LessThanOrEqual(Node* node) {
2769 FlagsContinuation cont =
2770 FlagsContinuation::ForSet(kSignedLessThanOrEqual, node);
2771 VisitWordCompare(this, node, kX64Cmp, &cont);
2772 }
2773
VisitUint64LessThan(Node * node)2774 void InstructionSelector::VisitUint64LessThan(Node* node) {
2775 FlagsContinuation cont = FlagsContinuation::ForSet(kUnsignedLessThan, node);
2776 VisitWordCompare(this, node, kX64Cmp, &cont);
2777 }
2778
VisitUint64LessThanOrEqual(Node * node)2779 void InstructionSelector::VisitUint64LessThanOrEqual(Node* node) {
2780 FlagsContinuation cont =
2781 FlagsContinuation::ForSet(kUnsignedLessThanOrEqual, node);
2782 VisitWordCompare(this, node, kX64Cmp, &cont);
2783 }
2784
VisitFloat32Equal(Node * node)2785 void InstructionSelector::VisitFloat32Equal(Node* node) {
2786 FlagsContinuation cont = FlagsContinuation::ForSet(kUnorderedEqual, node);
2787 VisitFloat32Compare(this, node, &cont);
2788 }
2789
VisitFloat32LessThan(Node * node)2790 void InstructionSelector::VisitFloat32LessThan(Node* node) {
2791 FlagsContinuation cont =
2792 FlagsContinuation::ForSet(kUnsignedGreaterThan, node);
2793 VisitFloat32Compare(this, node, &cont);
2794 }
2795
VisitFloat32LessThanOrEqual(Node * node)2796 void InstructionSelector::VisitFloat32LessThanOrEqual(Node* node) {
2797 FlagsContinuation cont =
2798 FlagsContinuation::ForSet(kUnsignedGreaterThanOrEqual, node);
2799 VisitFloat32Compare(this, node, &cont);
2800 }
2801
VisitFloat64Equal(Node * node)2802 void InstructionSelector::VisitFloat64Equal(Node* node) {
2803 FlagsContinuation cont = FlagsContinuation::ForSet(kUnorderedEqual, node);
2804 VisitFloat64Compare(this, node, &cont);
2805 }
2806
VisitFloat64LessThan(Node * node)2807 void InstructionSelector::VisitFloat64LessThan(Node* node) {
2808 Float64BinopMatcher m(node);
2809 if (m.left().Is(0.0) && m.right().IsFloat64Abs()) {
2810 // This matches the pattern
2811 //
2812 // Float64LessThan(#0.0, Float64Abs(x))
2813 //
2814 // which TurboFan generates for NumberToBoolean in the general case,
2815 // and which evaluates to false if x is 0, -0 or NaN. We can compile
2816 // this to a simple (v)ucomisd using not_equal flags condition, which
2817 // avoids the costly Float64Abs.
2818 FlagsContinuation cont = FlagsContinuation::ForSet(kNotEqual, node);
2819 InstructionCode const opcode =
2820 IsSupported(AVX) ? kAVXFloat64Cmp : kSSEFloat64Cmp;
2821 return VisitCompare(this, opcode, m.left().node(), m.right().InputAt(0),
2822 &cont, false);
2823 }
2824 FlagsContinuation cont =
2825 FlagsContinuation::ForSet(kUnsignedGreaterThan, node);
2826 VisitFloat64Compare(this, node, &cont);
2827 }
2828
VisitFloat64LessThanOrEqual(Node * node)2829 void InstructionSelector::VisitFloat64LessThanOrEqual(Node* node) {
2830 FlagsContinuation cont =
2831 FlagsContinuation::ForSet(kUnsignedGreaterThanOrEqual, node);
2832 VisitFloat64Compare(this, node, &cont);
2833 }
2834
VisitFloat64InsertLowWord32(Node * node)2835 void InstructionSelector::VisitFloat64InsertLowWord32(Node* node) {
2836 X64OperandGenerator g(this);
2837 Node* left = node->InputAt(0);
2838 Node* right = node->InputAt(1);
2839 Float64Matcher mleft(left);
2840 if (mleft.HasResolvedValue() &&
2841 (bit_cast<uint64_t>(mleft.ResolvedValue()) >> 32) == 0u) {
2842 Emit(kSSEFloat64LoadLowWord32, g.DefineAsRegister(node), g.Use(right));
2843 return;
2844 }
2845 Emit(kSSEFloat64InsertLowWord32, g.DefineSameAsFirst(node),
2846 g.UseRegister(left), g.Use(right));
2847 }
2848
VisitFloat64InsertHighWord32(Node * node)2849 void InstructionSelector::VisitFloat64InsertHighWord32(Node* node) {
2850 X64OperandGenerator g(this);
2851 Node* left = node->InputAt(0);
2852 Node* right = node->InputAt(1);
2853 Emit(kSSEFloat64InsertHighWord32, g.DefineSameAsFirst(node),
2854 g.UseRegister(left), g.Use(right));
2855 }
2856
VisitFloat64SilenceNaN(Node * node)2857 void InstructionSelector::VisitFloat64SilenceNaN(Node* node) {
2858 X64OperandGenerator g(this);
2859 Emit(kSSEFloat64SilenceNaN, g.DefineSameAsFirst(node),
2860 g.UseRegister(node->InputAt(0)));
2861 }
2862
VisitMemoryBarrier(Node * node)2863 void InstructionSelector::VisitMemoryBarrier(Node* node) {
2864 X64OperandGenerator g(this);
2865 Emit(kX64MFence, g.NoOutput());
2866 }
2867
VisitWord32AtomicLoad(Node * node)2868 void InstructionSelector::VisitWord32AtomicLoad(Node* node) {
2869 AtomicLoadParameters atomic_load_params = AtomicLoadParametersOf(node->op());
2870 LoadRepresentation load_rep = atomic_load_params.representation();
2871 DCHECK(IsIntegral(load_rep.representation()) ||
2872 IsAnyTagged(load_rep.representation()) ||
2873 (COMPRESS_POINTERS_BOOL &&
2874 CanBeCompressedPointer(load_rep.representation())));
2875 DCHECK_NE(load_rep.representation(), MachineRepresentation::kWord64);
2876 DCHECK(!load_rep.IsMapWord());
2877 // The memory order is ignored as both acquire and sequentially consistent
2878 // loads can emit MOV.
2879 // https://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html
2880 VisitLoad(node, node, GetLoadOpcode(load_rep));
2881 }
2882
VisitWord64AtomicLoad(Node * node)2883 void InstructionSelector::VisitWord64AtomicLoad(Node* node) {
2884 AtomicLoadParameters atomic_load_params = AtomicLoadParametersOf(node->op());
2885 DCHECK(!atomic_load_params.representation().IsMapWord());
2886 // The memory order is ignored as both acquire and sequentially consistent
2887 // loads can emit MOV.
2888 // https://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html
2889 VisitLoad(node, node, GetLoadOpcode(atomic_load_params.representation()));
2890 }
2891
VisitWord32AtomicStore(Node * node)2892 void InstructionSelector::VisitWord32AtomicStore(Node* node) {
2893 AtomicStoreParameters params = AtomicStoreParametersOf(node->op());
2894 DCHECK_NE(params.representation(), MachineRepresentation::kWord64);
2895 DCHECK_IMPLIES(CanBeTaggedOrCompressedPointer(params.representation()),
2896 kTaggedSize == 4);
2897 VisitStoreCommon(this, node, params.store_representation(), params.order());
2898 }
2899
VisitWord64AtomicStore(Node * node)2900 void InstructionSelector::VisitWord64AtomicStore(Node* node) {
2901 AtomicStoreParameters params = AtomicStoreParametersOf(node->op());
2902 DCHECK_IMPLIES(CanBeTaggedOrCompressedPointer(params.representation()),
2903 kTaggedSize == 8);
2904 VisitStoreCommon(this, node, params.store_representation(), params.order());
2905 }
2906
VisitWord32AtomicExchange(Node * node)2907 void InstructionSelector::VisitWord32AtomicExchange(Node* node) {
2908 MachineType type = AtomicOpType(node->op());
2909 ArchOpcode opcode;
2910 if (type == MachineType::Int8()) {
2911 opcode = kAtomicExchangeInt8;
2912 } else if (type == MachineType::Uint8()) {
2913 opcode = kAtomicExchangeUint8;
2914 } else if (type == MachineType::Int16()) {
2915 opcode = kAtomicExchangeInt16;
2916 } else if (type == MachineType::Uint16()) {
2917 opcode = kAtomicExchangeUint16;
2918 } else if (type == MachineType::Int32() || type == MachineType::Uint32()) {
2919 opcode = kAtomicExchangeWord32;
2920 } else {
2921 UNREACHABLE();
2922 }
2923 VisitAtomicExchange(this, node, opcode, AtomicWidth::kWord32);
2924 }
2925
VisitWord64AtomicExchange(Node * node)2926 void InstructionSelector::VisitWord64AtomicExchange(Node* node) {
2927 MachineType type = AtomicOpType(node->op());
2928 ArchOpcode opcode;
2929 if (type == MachineType::Uint8()) {
2930 opcode = kAtomicExchangeUint8;
2931 } else if (type == MachineType::Uint16()) {
2932 opcode = kAtomicExchangeUint16;
2933 } else if (type == MachineType::Uint32()) {
2934 opcode = kAtomicExchangeWord32;
2935 } else if (type == MachineType::Uint64()) {
2936 opcode = kX64Word64AtomicExchangeUint64;
2937 } else {
2938 UNREACHABLE();
2939 }
2940 VisitAtomicExchange(this, node, opcode, AtomicWidth::kWord64);
2941 }
2942
VisitWord32AtomicCompareExchange(Node * node)2943 void InstructionSelector::VisitWord32AtomicCompareExchange(Node* node) {
2944 MachineType type = AtomicOpType(node->op());
2945 ArchOpcode opcode;
2946 if (type == MachineType::Int8()) {
2947 opcode = kAtomicCompareExchangeInt8;
2948 } else if (type == MachineType::Uint8()) {
2949 opcode = kAtomicCompareExchangeUint8;
2950 } else if (type == MachineType::Int16()) {
2951 opcode = kAtomicCompareExchangeInt16;
2952 } else if (type == MachineType::Uint16()) {
2953 opcode = kAtomicCompareExchangeUint16;
2954 } else if (type == MachineType::Int32() || type == MachineType::Uint32()) {
2955 opcode = kAtomicCompareExchangeWord32;
2956 } else {
2957 UNREACHABLE();
2958 }
2959 VisitAtomicCompareExchange(this, node, opcode, AtomicWidth::kWord32);
2960 }
2961
VisitWord64AtomicCompareExchange(Node * node)2962 void InstructionSelector::VisitWord64AtomicCompareExchange(Node* node) {
2963 MachineType type = AtomicOpType(node->op());
2964 ArchOpcode opcode;
2965 if (type == MachineType::Uint8()) {
2966 opcode = kAtomicCompareExchangeUint8;
2967 } else if (type == MachineType::Uint16()) {
2968 opcode = kAtomicCompareExchangeUint16;
2969 } else if (type == MachineType::Uint32()) {
2970 opcode = kAtomicCompareExchangeWord32;
2971 } else if (type == MachineType::Uint64()) {
2972 opcode = kX64Word64AtomicCompareExchangeUint64;
2973 } else {
2974 UNREACHABLE();
2975 }
2976 VisitAtomicCompareExchange(this, node, opcode, AtomicWidth::kWord64);
2977 }
2978
VisitWord32AtomicBinaryOperation(Node * node,ArchOpcode int8_op,ArchOpcode uint8_op,ArchOpcode int16_op,ArchOpcode uint16_op,ArchOpcode word32_op)2979 void InstructionSelector::VisitWord32AtomicBinaryOperation(
2980 Node* node, ArchOpcode int8_op, ArchOpcode uint8_op, ArchOpcode int16_op,
2981 ArchOpcode uint16_op, ArchOpcode word32_op) {
2982 MachineType type = AtomicOpType(node->op());
2983 ArchOpcode opcode;
2984 if (type == MachineType::Int8()) {
2985 opcode = int8_op;
2986 } else if (type == MachineType::Uint8()) {
2987 opcode = uint8_op;
2988 } else if (type == MachineType::Int16()) {
2989 opcode = int16_op;
2990 } else if (type == MachineType::Uint16()) {
2991 opcode = uint16_op;
2992 } else if (type == MachineType::Int32() || type == MachineType::Uint32()) {
2993 opcode = word32_op;
2994 } else {
2995 UNREACHABLE();
2996 }
2997 VisitAtomicBinop(this, node, opcode, AtomicWidth::kWord32);
2998 }
2999
3000 #define VISIT_ATOMIC_BINOP(op) \
3001 void InstructionSelector::VisitWord32Atomic##op(Node* node) { \
3002 VisitWord32AtomicBinaryOperation( \
3003 node, kAtomic##op##Int8, kAtomic##op##Uint8, kAtomic##op##Int16, \
3004 kAtomic##op##Uint16, kAtomic##op##Word32); \
3005 }
3006 VISIT_ATOMIC_BINOP(Add)
VISIT_ATOMIC_BINOP(Sub)3007 VISIT_ATOMIC_BINOP(Sub)
3008 VISIT_ATOMIC_BINOP(And)
3009 VISIT_ATOMIC_BINOP(Or)
3010 VISIT_ATOMIC_BINOP(Xor)
3011 #undef VISIT_ATOMIC_BINOP
3012
3013 void InstructionSelector::VisitWord64AtomicBinaryOperation(
3014 Node* node, ArchOpcode uint8_op, ArchOpcode uint16_op, ArchOpcode uint32_op,
3015 ArchOpcode word64_op) {
3016 MachineType type = AtomicOpType(node->op());
3017 ArchOpcode opcode;
3018 if (type == MachineType::Uint8()) {
3019 opcode = uint8_op;
3020 } else if (type == MachineType::Uint16()) {
3021 opcode = uint16_op;
3022 } else if (type == MachineType::Uint32()) {
3023 opcode = uint32_op;
3024 } else if (type == MachineType::Uint64()) {
3025 opcode = word64_op;
3026 } else {
3027 UNREACHABLE();
3028 }
3029 VisitAtomicBinop(this, node, opcode, AtomicWidth::kWord64);
3030 }
3031
3032 #define VISIT_ATOMIC_BINOP(op) \
3033 void InstructionSelector::VisitWord64Atomic##op(Node* node) { \
3034 VisitWord64AtomicBinaryOperation(node, kAtomic##op##Uint8, \
3035 kAtomic##op##Uint16, kAtomic##op##Word32, \
3036 kX64Word64Atomic##op##Uint64); \
3037 }
3038 VISIT_ATOMIC_BINOP(Add)
VISIT_ATOMIC_BINOP(Sub)3039 VISIT_ATOMIC_BINOP(Sub)
3040 VISIT_ATOMIC_BINOP(And)
3041 VISIT_ATOMIC_BINOP(Or)
3042 VISIT_ATOMIC_BINOP(Xor)
3043 #undef VISIT_ATOMIC_BINOP
3044
3045 #define SIMD_BINOP_SSE_AVX_LIST(V) \
3046 V(F64x2Add) \
3047 V(F64x2Sub) \
3048 V(F64x2Mul) \
3049 V(F64x2Div) \
3050 V(F64x2Eq) \
3051 V(F64x2Ne) \
3052 V(F64x2Lt) \
3053 V(F64x2Le) \
3054 V(F32x4Add) \
3055 V(F32x4Sub) \
3056 V(F32x4Mul) \
3057 V(F32x4Div) \
3058 V(F32x4Eq) \
3059 V(F32x4Ne) \
3060 V(F32x4Lt) \
3061 V(F32x4Le) \
3062 V(I64x2Add) \
3063 V(I64x2Sub) \
3064 V(I64x2Eq) \
3065 V(I64x2ExtMulLowI32x4S) \
3066 V(I64x2ExtMulHighI32x4S) \
3067 V(I64x2ExtMulLowI32x4U) \
3068 V(I64x2ExtMulHighI32x4U) \
3069 V(I32x4Add) \
3070 V(I32x4Sub) \
3071 V(I32x4Mul) \
3072 V(I32x4MinS) \
3073 V(I32x4MaxS) \
3074 V(I32x4Eq) \
3075 V(I32x4GtS) \
3076 V(I32x4MinU) \
3077 V(I32x4MaxU) \
3078 V(I32x4DotI16x8S) \
3079 V(I32x4ExtMulLowI16x8S) \
3080 V(I32x4ExtMulHighI16x8S) \
3081 V(I32x4ExtMulLowI16x8U) \
3082 V(I32x4ExtMulHighI16x8U) \
3083 V(I16x8SConvertI32x4) \
3084 V(I16x8UConvertI32x4) \
3085 V(I16x8Add) \
3086 V(I16x8AddSatS) \
3087 V(I16x8Sub) \
3088 V(I16x8SubSatS) \
3089 V(I16x8Mul) \
3090 V(I16x8MinS) \
3091 V(I16x8MaxS) \
3092 V(I16x8Eq) \
3093 V(I16x8GtS) \
3094 V(I16x8AddSatU) \
3095 V(I16x8SubSatU) \
3096 V(I16x8MinU) \
3097 V(I16x8MaxU) \
3098 V(I16x8RoundingAverageU) \
3099 V(I16x8ExtMulLowI8x16S) \
3100 V(I16x8ExtMulHighI8x16S) \
3101 V(I16x8ExtMulLowI8x16U) \
3102 V(I16x8ExtMulHighI8x16U) \
3103 V(I16x8Q15MulRSatS) \
3104 V(I8x16SConvertI16x8) \
3105 V(I8x16UConvertI16x8) \
3106 V(I8x16Add) \
3107 V(I8x16AddSatS) \
3108 V(I8x16Sub) \
3109 V(I8x16SubSatS) \
3110 V(I8x16MinS) \
3111 V(I8x16MaxS) \
3112 V(I8x16Eq) \
3113 V(I8x16GtS) \
3114 V(I8x16AddSatU) \
3115 V(I8x16SubSatU) \
3116 V(I8x16MinU) \
3117 V(I8x16MaxU) \
3118 V(I8x16RoundingAverageU) \
3119 V(S128And) \
3120 V(S128Or) \
3121 V(S128Xor)
3122
3123 #define SIMD_BINOP_LIST(V) \
3124 V(F64x2Min) \
3125 V(F64x2Max) \
3126 V(F32x4Min) \
3127 V(F32x4Max) \
3128 V(I64x2Ne) \
3129 V(I32x4Ne) \
3130 V(I32x4GtU) \
3131 V(I32x4GeS) \
3132 V(I32x4GeU) \
3133 V(I16x8Ne) \
3134 V(I16x8GtU) \
3135 V(I16x8GeS) \
3136 V(I16x8GeU) \
3137 V(I8x16Ne) \
3138 V(I8x16GtU) \
3139 V(I8x16GeS) \
3140 V(I8x16GeU)
3141
3142 #define SIMD_UNOP_LIST(V) \
3143 V(F64x2Sqrt) \
3144 V(F64x2ConvertLowI32x4S) \
3145 V(F32x4SConvertI32x4) \
3146 V(F32x4Abs) \
3147 V(F32x4Neg) \
3148 V(F32x4Sqrt) \
3149 V(F32x4RecipApprox) \
3150 V(F32x4RecipSqrtApprox) \
3151 V(F32x4DemoteF64x2Zero) \
3152 V(I64x2BitMask) \
3153 V(I64x2SConvertI32x4Low) \
3154 V(I64x2SConvertI32x4High) \
3155 V(I64x2UConvertI32x4Low) \
3156 V(I64x2UConvertI32x4High) \
3157 V(I32x4SConvertI16x8Low) \
3158 V(I32x4SConvertI16x8High) \
3159 V(I32x4Neg) \
3160 V(I32x4UConvertI16x8Low) \
3161 V(I32x4UConvertI16x8High) \
3162 V(I32x4Abs) \
3163 V(I32x4BitMask) \
3164 V(I16x8SConvertI8x16Low) \
3165 V(I16x8SConvertI8x16High) \
3166 V(I16x8Neg) \
3167 V(I16x8UConvertI8x16Low) \
3168 V(I16x8UConvertI8x16High) \
3169 V(I16x8Abs) \
3170 V(I16x8BitMask) \
3171 V(I8x16Neg) \
3172 V(I8x16Abs) \
3173 V(I8x16BitMask) \
3174 V(I64x2AllTrue) \
3175 V(I32x4AllTrue) \
3176 V(I16x8AllTrue) \
3177 V(I8x16AllTrue) \
3178 V(S128Not)
3179
3180 #define SIMD_SHIFT_OPCODES(V) \
3181 V(I64x2Shl) \
3182 V(I64x2ShrU) \
3183 V(I32x4Shl) \
3184 V(I32x4ShrS) \
3185 V(I32x4ShrU) \
3186 V(I16x8Shl) \
3187 V(I16x8ShrS) \
3188 V(I16x8ShrU)
3189
3190 #define SIMD_NARROW_SHIFT_OPCODES(V) \
3191 V(I8x16Shl) \
3192 V(I8x16ShrS) \
3193 V(I8x16ShrU)
3194
3195 void InstructionSelector::VisitS128Const(Node* node) {
3196 X64OperandGenerator g(this);
3197 static const int kUint32Immediates = kSimd128Size / sizeof(uint32_t);
3198 uint32_t val[kUint32Immediates];
3199 memcpy(val, S128ImmediateParameterOf(node->op()).data(), kSimd128Size);
3200 // If all bytes are zeros or ones, avoid emitting code for generic constants
3201 bool all_zeros = !(val[0] || val[1] || val[2] || val[3]);
3202 bool all_ones = val[0] == UINT32_MAX && val[1] == UINT32_MAX &&
3203 val[2] == UINT32_MAX && val[3] == UINT32_MAX;
3204 InstructionOperand dst = g.DefineAsRegister(node);
3205 if (all_zeros) {
3206 Emit(kX64S128Zero, dst);
3207 } else if (all_ones) {
3208 Emit(kX64S128AllOnes, dst);
3209 } else {
3210 Emit(kX64S128Const, dst, g.UseImmediate(val[0]), g.UseImmediate(val[1]),
3211 g.UseImmediate(val[2]), g.UseImmediate(val[3]));
3212 }
3213 }
3214
VisitS128Zero(Node * node)3215 void InstructionSelector::VisitS128Zero(Node* node) {
3216 X64OperandGenerator g(this);
3217 Emit(kX64S128Zero, g.DefineAsRegister(node));
3218 }
3219
3220 #define SIMD_TYPES_FOR_SPLAT(V) \
3221 V(I64x2) \
3222 V(I32x4) \
3223 V(I16x8) \
3224 V(I8x16)
3225
3226 // Splat with an optimization for const 0.
3227 #define VISIT_SIMD_SPLAT(Type) \
3228 void InstructionSelector::Visit##Type##Splat(Node* node) { \
3229 X64OperandGenerator g(this); \
3230 Node* input = node->InputAt(0); \
3231 if (g.CanBeImmediate(input) && g.GetImmediateIntegerValue(input) == 0) { \
3232 Emit(kX64S128Zero, g.DefineAsRegister(node)); \
3233 } else { \
3234 Emit(kX64##Type##Splat, g.DefineAsRegister(node), g.Use(input)); \
3235 } \
3236 }
SIMD_TYPES_FOR_SPLAT(VISIT_SIMD_SPLAT)3237 SIMD_TYPES_FOR_SPLAT(VISIT_SIMD_SPLAT)
3238 #undef VISIT_SIMD_SPLAT
3239 #undef SIMD_TYPES_FOR_SPLAT
3240
3241 void InstructionSelector::VisitF64x2Splat(Node* node) {
3242 X64OperandGenerator g(this);
3243 Emit(kX64F64x2Splat, g.DefineAsRegister(node), g.Use(node->InputAt(0)));
3244 }
3245
VisitF32x4Splat(Node * node)3246 void InstructionSelector::VisitF32x4Splat(Node* node) {
3247 X64OperandGenerator g(this);
3248 Emit(kX64F32x4Splat, g.DefineAsRegister(node),
3249 g.UseRegister(node->InputAt(0)));
3250 }
3251
3252 #define SIMD_VISIT_EXTRACT_LANE(Type, Sign, Op) \
3253 void InstructionSelector::Visit##Type##ExtractLane##Sign(Node* node) { \
3254 X64OperandGenerator g(this); \
3255 int32_t lane = OpParameter<int32_t>(node->op()); \
3256 Emit(kX64##Op, g.DefineAsRegister(node), g.UseRegister(node->InputAt(0)), \
3257 g.UseImmediate(lane)); \
3258 }
3259 SIMD_VISIT_EXTRACT_LANE(F64x2, , F64x2ExtractLane)
3260 SIMD_VISIT_EXTRACT_LANE(F32x4, , F32x4ExtractLane)
3261 SIMD_VISIT_EXTRACT_LANE(I64x2, , I64x2ExtractLane)
3262 SIMD_VISIT_EXTRACT_LANE(I32x4, , I32x4ExtractLane)
SIMD_VISIT_EXTRACT_LANE(I16x8,S,I16x8ExtractLaneS)3263 SIMD_VISIT_EXTRACT_LANE(I16x8, S, I16x8ExtractLaneS)
3264 SIMD_VISIT_EXTRACT_LANE(I16x8, U, Pextrw)
3265 SIMD_VISIT_EXTRACT_LANE(I8x16, S, I8x16ExtractLaneS)
3266 SIMD_VISIT_EXTRACT_LANE(I8x16, U, Pextrb)
3267 #undef SIMD_VISIT_EXTRACT_LANE
3268
3269 void InstructionSelector::VisitF32x4ReplaceLane(Node* node) {
3270 X64OperandGenerator g(this);
3271 int32_t lane = OpParameter<int32_t>(node->op());
3272 Emit(kX64F32x4ReplaceLane, g.DefineSameAsFirst(node),
3273 g.UseRegister(node->InputAt(0)), g.UseImmediate(lane),
3274 g.Use(node->InputAt(1)));
3275 }
3276
VisitF64x2ReplaceLane(Node * node)3277 void InstructionSelector::VisitF64x2ReplaceLane(Node* node) {
3278 X64OperandGenerator g(this);
3279 int32_t lane = OpParameter<int32_t>(node->op());
3280 // When no-AVX, define dst == src to save a move.
3281 InstructionOperand dst =
3282 IsSupported(AVX) ? g.DefineAsRegister(node) : g.DefineSameAsFirst(node);
3283 Emit(kX64F64x2ReplaceLane, dst, g.UseRegister(node->InputAt(0)),
3284 g.UseImmediate(lane), g.UseRegister(node->InputAt(1)));
3285 }
3286
3287 #define VISIT_SIMD_REPLACE_LANE(TYPE, OPCODE) \
3288 void InstructionSelector::Visit##TYPE##ReplaceLane(Node* node) { \
3289 X64OperandGenerator g(this); \
3290 int32_t lane = OpParameter<int32_t>(node->op()); \
3291 Emit(OPCODE, g.DefineAsRegister(node), g.UseRegister(node->InputAt(0)), \
3292 g.UseImmediate(lane), g.Use(node->InputAt(1))); \
3293 }
3294
3295 #define SIMD_TYPES_FOR_REPLACE_LANE(V) \
3296 V(I64x2, kX64Pinsrq) \
3297 V(I32x4, kX64Pinsrd) \
3298 V(I16x8, kX64Pinsrw) \
3299 V(I8x16, kX64Pinsrb)
3300
3301 SIMD_TYPES_FOR_REPLACE_LANE(VISIT_SIMD_REPLACE_LANE)
3302 #undef SIMD_TYPES_FOR_REPLACE_LANE
3303 #undef VISIT_SIMD_REPLACE_LANE
3304
3305 #define VISIT_SIMD_SHIFT(Opcode) \
3306 void InstructionSelector::Visit##Opcode(Node* node) { \
3307 X64OperandGenerator g(this); \
3308 InstructionOperand dst = IsSupported(AVX) ? g.DefineAsRegister(node) \
3309 : g.DefineSameAsFirst(node); \
3310 if (g.CanBeImmediate(node->InputAt(1))) { \
3311 Emit(kX64##Opcode, dst, g.UseRegister(node->InputAt(0)), \
3312 g.UseImmediate(node->InputAt(1))); \
3313 } else { \
3314 Emit(kX64##Opcode, dst, g.UseRegister(node->InputAt(0)), \
3315 g.UseRegister(node->InputAt(1))); \
3316 } \
3317 }
SIMD_SHIFT_OPCODES(VISIT_SIMD_SHIFT)3318 SIMD_SHIFT_OPCODES(VISIT_SIMD_SHIFT)
3319 #undef VISIT_SIMD_SHIFT
3320 #undef SIMD_SHIFT_OPCODES
3321
3322 #define VISIT_SIMD_NARROW_SHIFT(Opcode) \
3323 void InstructionSelector::Visit##Opcode(Node* node) { \
3324 X64OperandGenerator g(this); \
3325 InstructionOperand output = \
3326 IsSupported(AVX) ? g.UseRegister(node) : g.DefineSameAsFirst(node); \
3327 if (g.CanBeImmediate(node->InputAt(1))) { \
3328 Emit(kX64##Opcode, output, g.UseRegister(node->InputAt(0)), \
3329 g.UseImmediate(node->InputAt(1))); \
3330 } else { \
3331 InstructionOperand temps[] = {g.TempSimd128Register()}; \
3332 Emit(kX64##Opcode, output, g.UseUniqueRegister(node->InputAt(0)), \
3333 g.UseUniqueRegister(node->InputAt(1)), arraysize(temps), temps); \
3334 } \
3335 }
3336 SIMD_NARROW_SHIFT_OPCODES(VISIT_SIMD_NARROW_SHIFT)
3337 #undef VISIT_SIMD_NARROW_SHIFT
3338 #undef SIMD_NARROW_SHIFT_OPCODES
3339
3340 #define VISIT_SIMD_UNOP(Opcode) \
3341 void InstructionSelector::Visit##Opcode(Node* node) { \
3342 X64OperandGenerator g(this); \
3343 Emit(kX64##Opcode, g.DefineAsRegister(node), \
3344 g.UseRegister(node->InputAt(0))); \
3345 }
3346 SIMD_UNOP_LIST(VISIT_SIMD_UNOP)
3347 #undef VISIT_SIMD_UNOP
3348 #undef SIMD_UNOP_LIST
3349
3350 #define VISIT_SIMD_BINOP(Opcode) \
3351 void InstructionSelector::Visit##Opcode(Node* node) { \
3352 X64OperandGenerator g(this); \
3353 Emit(kX64##Opcode, g.DefineSameAsFirst(node), \
3354 g.UseRegister(node->InputAt(0)), g.UseRegister(node->InputAt(1))); \
3355 }
3356 SIMD_BINOP_LIST(VISIT_SIMD_BINOP)
3357 #undef VISIT_SIMD_BINOP
3358 #undef SIMD_BINOP_LIST
3359
3360 #define VISIT_SIMD_BINOP(Opcode) \
3361 void InstructionSelector::Visit##Opcode(Node* node) { \
3362 X64OperandGenerator g(this); \
3363 if (IsSupported(AVX)) { \
3364 Emit(kX64##Opcode, g.DefineAsRegister(node), \
3365 g.UseRegister(node->InputAt(0)), g.UseRegister(node->InputAt(1))); \
3366 } else { \
3367 Emit(kX64##Opcode, g.DefineSameAsFirst(node), \
3368 g.UseRegister(node->InputAt(0)), g.UseRegister(node->InputAt(1))); \
3369 } \
3370 }
3371 SIMD_BINOP_SSE_AVX_LIST(VISIT_SIMD_BINOP)
3372 #undef VISIT_SIMD_BINOP
3373 #undef SIMD_BINOP_SSE_AVX_LIST
3374
3375 void InstructionSelector::VisitV128AnyTrue(Node* node) {
3376 X64OperandGenerator g(this);
3377 Emit(kX64V128AnyTrue, g.DefineAsRegister(node),
3378 g.UseUniqueRegister(node->InputAt(0)));
3379 }
3380
VisitS128Select(Node * node)3381 void InstructionSelector::VisitS128Select(Node* node) {
3382 X64OperandGenerator g(this);
3383 InstructionOperand dst =
3384 IsSupported(AVX) ? g.DefineAsRegister(node) : g.DefineSameAsFirst(node);
3385 Emit(kX64S128Select, dst, g.UseRegister(node->InputAt(0)),
3386 g.UseRegister(node->InputAt(1)), g.UseRegister(node->InputAt(2)));
3387 }
3388
VisitS128AndNot(Node * node)3389 void InstructionSelector::VisitS128AndNot(Node* node) {
3390 X64OperandGenerator g(this);
3391 // andnps a b does ~a & b, but we want a & !b, so flip the input.
3392 Emit(kX64S128AndNot, g.DefineSameAsFirst(node),
3393 g.UseRegister(node->InputAt(1)), g.UseRegister(node->InputAt(0)));
3394 }
3395
VisitF64x2Abs(Node * node)3396 void InstructionSelector::VisitF64x2Abs(Node* node) {
3397 VisitFloatUnop(this, node, node->InputAt(0), kX64F64x2Abs);
3398 }
3399
VisitF64x2Neg(Node * node)3400 void InstructionSelector::VisitF64x2Neg(Node* node) {
3401 VisitFloatUnop(this, node, node->InputAt(0), kX64F64x2Neg);
3402 }
3403
VisitF32x4UConvertI32x4(Node * node)3404 void InstructionSelector::VisitF32x4UConvertI32x4(Node* node) {
3405 X64OperandGenerator g(this);
3406 Emit(kX64F32x4UConvertI32x4, g.DefineSameAsFirst(node),
3407 g.UseRegister(node->InputAt(0)));
3408 }
3409
3410 #define VISIT_SIMD_QFMOP(Opcode) \
3411 void InstructionSelector::Visit##Opcode(Node* node) { \
3412 X64OperandGenerator g(this); \
3413 Emit(kX64##Opcode, g.UseRegister(node), g.UseRegister(node->InputAt(0)), \
3414 g.UseRegister(node->InputAt(1)), g.UseRegister(node->InputAt(2))); \
3415 }
3416 VISIT_SIMD_QFMOP(F64x2Qfma)
VISIT_SIMD_QFMOP(F64x2Qfms)3417 VISIT_SIMD_QFMOP(F64x2Qfms)
3418 VISIT_SIMD_QFMOP(F32x4Qfma)
3419 VISIT_SIMD_QFMOP(F32x4Qfms)
3420 #undef VISIT_SIMD_QFMOP
3421
3422 void InstructionSelector::VisitI64x2Neg(Node* node) {
3423 X64OperandGenerator g(this);
3424 // If AVX unsupported, make sure dst != src to avoid a move.
3425 InstructionOperand operand0 = IsSupported(AVX)
3426 ? g.UseRegister(node->InputAt(0))
3427 : g.UseUnique(node->InputAt(0));
3428 Emit(kX64I64x2Neg, g.DefineAsRegister(node), operand0);
3429 }
3430
VisitI64x2ShrS(Node * node)3431 void InstructionSelector::VisitI64x2ShrS(Node* node) {
3432 X64OperandGenerator g(this);
3433 InstructionOperand dst =
3434 IsSupported(AVX) ? g.DefineAsRegister(node) : g.DefineSameAsFirst(node);
3435
3436 if (g.CanBeImmediate(node->InputAt(1))) {
3437 Emit(kX64I64x2ShrS, dst, g.UseRegister(node->InputAt(0)),
3438 g.UseImmediate(node->InputAt(1)));
3439 } else {
3440 InstructionOperand temps[] = {g.TempSimd128Register()};
3441 Emit(kX64I64x2ShrS, dst, g.UseUniqueRegister(node->InputAt(0)),
3442 g.UseRegister(node->InputAt(1)), arraysize(temps), temps);
3443 }
3444 }
3445
VisitI64x2Mul(Node * node)3446 void InstructionSelector::VisitI64x2Mul(Node* node) {
3447 X64OperandGenerator g(this);
3448 InstructionOperand temps[] = {g.TempSimd128Register()};
3449 Emit(kX64I64x2Mul, g.DefineAsRegister(node),
3450 g.UseUniqueRegister(node->InputAt(0)),
3451 g.UseUniqueRegister(node->InputAt(1)), arraysize(temps), temps);
3452 }
3453
VisitI32x4SConvertF32x4(Node * node)3454 void InstructionSelector::VisitI32x4SConvertF32x4(Node* node) {
3455 X64OperandGenerator g(this);
3456 Emit(kX64I32x4SConvertF32x4,
3457 IsSupported(AVX) ? g.DefineAsRegister(node) : g.DefineSameAsFirst(node),
3458 g.UseRegister(node->InputAt(0)));
3459 }
3460
VisitI32x4UConvertF32x4(Node * node)3461 void InstructionSelector::VisitI32x4UConvertF32x4(Node* node) {
3462 X64OperandGenerator g(this);
3463 InstructionOperand temps[] = {g.TempSimd128Register(),
3464 g.TempSimd128Register()};
3465 Emit(kX64I32x4UConvertF32x4, g.DefineSameAsFirst(node),
3466 g.UseRegister(node->InputAt(0)), arraysize(temps), temps);
3467 }
3468
VisitInt32AbsWithOverflow(Node * node)3469 void InstructionSelector::VisitInt32AbsWithOverflow(Node* node) {
3470 UNREACHABLE();
3471 }
3472
VisitInt64AbsWithOverflow(Node * node)3473 void InstructionSelector::VisitInt64AbsWithOverflow(Node* node) {
3474 UNREACHABLE();
3475 }
3476
3477 #if V8_ENABLE_WEBASSEMBLY
3478 namespace {
3479
3480 // Returns true if shuffle can be decomposed into two 16x4 half shuffles
3481 // followed by a 16x8 blend.
3482 // E.g. [3 2 1 0 15 14 13 12].
TryMatch16x8HalfShuffle(uint8_t * shuffle16x8,uint8_t * blend_mask)3483 bool TryMatch16x8HalfShuffle(uint8_t* shuffle16x8, uint8_t* blend_mask) {
3484 *blend_mask = 0;
3485 for (int i = 0; i < 8; i++) {
3486 if ((shuffle16x8[i] & 0x4) != (i & 0x4)) return false;
3487 *blend_mask |= (shuffle16x8[i] > 7 ? 1 : 0) << i;
3488 }
3489 return true;
3490 }
3491
3492 struct ShuffleEntry {
3493 uint8_t shuffle[kSimd128Size];
3494 ArchOpcode opcode;
3495 bool src0_needs_reg;
3496 bool src1_needs_reg;
3497 // If AVX is supported, this shuffle can use AVX's three-operand encoding, so
3498 // does not require same as first. We conservatively set this to false
3499 // (original behavior), and selectively enable for specific arch shuffles.
3500 bool no_same_as_first_if_avx = false;
3501 };
3502
3503 // Shuffles that map to architecture-specific instruction sequences. These are
3504 // matched very early, so we shouldn't include shuffles that match better in
3505 // later tests, like 32x4 and 16x8 shuffles. In general, these patterns should
3506 // map to either a single instruction, or be finer grained, such as zip/unzip or
3507 // transpose patterns.
3508 static const ShuffleEntry arch_shuffles[] = {
3509 {{0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23},
3510 kX64S64x2UnpackLow,
3511 true,
3512 true,
3513 true},
3514 {{8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31},
3515 kX64S64x2UnpackHigh,
3516 true,
3517 true,
3518 true},
3519 {{0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23},
3520 kX64S32x4UnpackLow,
3521 true,
3522 true,
3523 true},
3524 {{8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31},
3525 kX64S32x4UnpackHigh,
3526 true,
3527 true,
3528 true},
3529 {{0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23},
3530 kX64S16x8UnpackLow,
3531 true,
3532 true,
3533 true},
3534 {{8, 9, 24, 25, 10, 11, 26, 27, 12, 13, 28, 29, 14, 15, 30, 31},
3535 kX64S16x8UnpackHigh,
3536 true,
3537 true,
3538 true},
3539 {{0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23},
3540 kX64S8x16UnpackLow,
3541 true,
3542 true,
3543 true},
3544 {{8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31},
3545 kX64S8x16UnpackHigh,
3546 true,
3547 true,
3548 true},
3549
3550 {{0, 1, 4, 5, 8, 9, 12, 13, 16, 17, 20, 21, 24, 25, 28, 29},
3551 kX64S16x8UnzipLow,
3552 true,
3553 true},
3554 {{2, 3, 6, 7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31},
3555 kX64S16x8UnzipHigh,
3556 true,
3557 true},
3558 {{0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30},
3559 kX64S8x16UnzipLow,
3560 true,
3561 true},
3562 {{1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31},
3563 kX64S8x16UnzipHigh,
3564 true,
3565 true},
3566 {{0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30},
3567 kX64S8x16TransposeLow,
3568 true,
3569 true},
3570 {{1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31},
3571 kX64S8x16TransposeHigh,
3572 true,
3573 true},
3574 {{7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8},
3575 kX64S8x8Reverse,
3576 true,
3577 true},
3578 {{3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12},
3579 kX64S8x4Reverse,
3580 true,
3581 true},
3582 {{1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14},
3583 kX64S8x2Reverse,
3584 true,
3585 true}};
3586
TryMatchArchShuffle(const uint8_t * shuffle,const ShuffleEntry * table,size_t num_entries,bool is_swizzle,const ShuffleEntry ** arch_shuffle)3587 bool TryMatchArchShuffle(const uint8_t* shuffle, const ShuffleEntry* table,
3588 size_t num_entries, bool is_swizzle,
3589 const ShuffleEntry** arch_shuffle) {
3590 uint8_t mask = is_swizzle ? kSimd128Size - 1 : 2 * kSimd128Size - 1;
3591 for (size_t i = 0; i < num_entries; ++i) {
3592 const ShuffleEntry& entry = table[i];
3593 int j = 0;
3594 for (; j < kSimd128Size; ++j) {
3595 if ((entry.shuffle[j] & mask) != (shuffle[j] & mask)) {
3596 break;
3597 }
3598 }
3599 if (j == kSimd128Size) {
3600 *arch_shuffle = &entry;
3601 return true;
3602 }
3603 }
3604 return false;
3605 }
3606
TryMatchShufps(const uint8_t * shuffle32x4)3607 bool TryMatchShufps(const uint8_t* shuffle32x4) {
3608 DCHECK_GT(8, shuffle32x4[2]);
3609 DCHECK_GT(8, shuffle32x4[3]);
3610 // shufps can be used if the first 2 indices select the first input [0-3], and
3611 // the other 2 indices select the second input [4-7].
3612 return shuffle32x4[0] < 4 && shuffle32x4[1] < 4 && shuffle32x4[2] > 3 &&
3613 shuffle32x4[3] > 3;
3614 }
3615
3616 } // namespace
3617
VisitI8x16Shuffle(Node * node)3618 void InstructionSelector::VisitI8x16Shuffle(Node* node) {
3619 uint8_t shuffle[kSimd128Size];
3620 bool is_swizzle;
3621 CanonicalizeShuffle(node, shuffle, &is_swizzle);
3622
3623 int imm_count = 0;
3624 static const int kMaxImms = 6;
3625 uint32_t imms[kMaxImms];
3626 int temp_count = 0;
3627 static const int kMaxTemps = 2;
3628 InstructionOperand temps[kMaxTemps];
3629
3630 X64OperandGenerator g(this);
3631 // Swizzles don't generally need DefineSameAsFirst to avoid a move.
3632 bool no_same_as_first = is_swizzle;
3633 // We generally need UseRegister for input0, Use for input1.
3634 // TODO(v8:9198): We don't have 16-byte alignment for SIMD operands yet, but
3635 // we retain this logic (continue setting these in the various shuffle match
3636 // clauses), but ignore it when selecting registers or slots.
3637 bool src0_needs_reg = true;
3638 bool src1_needs_reg = false;
3639 ArchOpcode opcode = kX64I8x16Shuffle; // general shuffle is the default
3640
3641 uint8_t offset;
3642 uint8_t shuffle32x4[4];
3643 uint8_t shuffle16x8[8];
3644 int index;
3645 const ShuffleEntry* arch_shuffle;
3646 if (wasm::SimdShuffle::TryMatchConcat(shuffle, &offset)) {
3647 if (wasm::SimdShuffle::TryMatch32x4Rotate(shuffle, shuffle32x4,
3648 is_swizzle)) {
3649 uint8_t shuffle_mask = wasm::SimdShuffle::PackShuffle4(shuffle32x4);
3650 opcode = kX64S32x4Rotate;
3651 imms[imm_count++] = shuffle_mask;
3652 } else {
3653 // Swap inputs from the normal order for (v)palignr.
3654 SwapShuffleInputs(node);
3655 is_swizzle = false; // It's simpler to just handle the general case.
3656 no_same_as_first = CpuFeatures::IsSupported(AVX);
3657 // TODO(v8:9608): also see v8:9083
3658 src1_needs_reg = true;
3659 opcode = kX64S8x16Alignr;
3660 // palignr takes a single imm8 offset.
3661 imms[imm_count++] = offset;
3662 }
3663 } else if (TryMatchArchShuffle(shuffle, arch_shuffles,
3664 arraysize(arch_shuffles), is_swizzle,
3665 &arch_shuffle)) {
3666 opcode = arch_shuffle->opcode;
3667 src0_needs_reg = arch_shuffle->src0_needs_reg;
3668 // SSE can't take advantage of both operands in registers and needs
3669 // same-as-first.
3670 src1_needs_reg = arch_shuffle->src1_needs_reg;
3671 no_same_as_first =
3672 IsSupported(AVX) && arch_shuffle->no_same_as_first_if_avx;
3673 } else if (wasm::SimdShuffle::TryMatch32x4Shuffle(shuffle, shuffle32x4)) {
3674 uint8_t shuffle_mask = wasm::SimdShuffle::PackShuffle4(shuffle32x4);
3675 if (is_swizzle) {
3676 if (wasm::SimdShuffle::TryMatchIdentity(shuffle)) {
3677 // Bypass normal shuffle code generation in this case.
3678 EmitIdentity(node);
3679 return;
3680 } else {
3681 // pshufd takes a single imm8 shuffle mask.
3682 opcode = kX64S32x4Swizzle;
3683 no_same_as_first = true;
3684 // TODO(v8:9083): This doesn't strictly require a register, forcing the
3685 // swizzles to always use registers until generation of incorrect memory
3686 // operands can be fixed.
3687 src0_needs_reg = true;
3688 imms[imm_count++] = shuffle_mask;
3689 }
3690 } else {
3691 // 2 operand shuffle
3692 // A blend is more efficient than a general 32x4 shuffle; try it first.
3693 if (wasm::SimdShuffle::TryMatchBlend(shuffle)) {
3694 opcode = kX64S16x8Blend;
3695 uint8_t blend_mask = wasm::SimdShuffle::PackBlend4(shuffle32x4);
3696 imms[imm_count++] = blend_mask;
3697 no_same_as_first = CpuFeatures::IsSupported(AVX);
3698 } else if (TryMatchShufps(shuffle32x4)) {
3699 opcode = kX64Shufps;
3700 uint8_t mask = wasm::SimdShuffle::PackShuffle4(shuffle32x4);
3701 imms[imm_count++] = mask;
3702 src1_needs_reg = true;
3703 no_same_as_first = IsSupported(AVX);
3704 } else {
3705 opcode = kX64S32x4Shuffle;
3706 no_same_as_first = true;
3707 // TODO(v8:9083): src0 and src1 is used by pshufd in codegen, which
3708 // requires memory to be 16-byte aligned, since we cannot guarantee that
3709 // yet, force using a register here.
3710 src0_needs_reg = true;
3711 src1_needs_reg = true;
3712 imms[imm_count++] = shuffle_mask;
3713 uint8_t blend_mask = wasm::SimdShuffle::PackBlend4(shuffle32x4);
3714 imms[imm_count++] = blend_mask;
3715 }
3716 }
3717 } else if (wasm::SimdShuffle::TryMatch16x8Shuffle(shuffle, shuffle16x8)) {
3718 uint8_t blend_mask;
3719 if (wasm::SimdShuffle::TryMatchBlend(shuffle)) {
3720 opcode = kX64S16x8Blend;
3721 blend_mask = wasm::SimdShuffle::PackBlend8(shuffle16x8);
3722 imms[imm_count++] = blend_mask;
3723 no_same_as_first = CpuFeatures::IsSupported(AVX);
3724 } else if (wasm::SimdShuffle::TryMatchSplat<8>(shuffle, &index)) {
3725 opcode = kX64S16x8Dup;
3726 src0_needs_reg = false;
3727 imms[imm_count++] = index;
3728 } else if (TryMatch16x8HalfShuffle(shuffle16x8, &blend_mask)) {
3729 opcode = is_swizzle ? kX64S16x8HalfShuffle1 : kX64S16x8HalfShuffle2;
3730 // Half-shuffles don't need DefineSameAsFirst or UseRegister(src0).
3731 no_same_as_first = true;
3732 src0_needs_reg = false;
3733 uint8_t mask_lo = wasm::SimdShuffle::PackShuffle4(shuffle16x8);
3734 uint8_t mask_hi = wasm::SimdShuffle::PackShuffle4(shuffle16x8 + 4);
3735 imms[imm_count++] = mask_lo;
3736 imms[imm_count++] = mask_hi;
3737 if (!is_swizzle) imms[imm_count++] = blend_mask;
3738 }
3739 } else if (wasm::SimdShuffle::TryMatchSplat<16>(shuffle, &index)) {
3740 opcode = kX64S8x16Dup;
3741 no_same_as_first = false;
3742 src0_needs_reg = true;
3743 imms[imm_count++] = index;
3744 }
3745 if (opcode == kX64I8x16Shuffle) {
3746 // Use same-as-first for general swizzle, but not shuffle.
3747 no_same_as_first = !is_swizzle;
3748 src0_needs_reg = !no_same_as_first;
3749 imms[imm_count++] = wasm::SimdShuffle::Pack4Lanes(shuffle);
3750 imms[imm_count++] = wasm::SimdShuffle::Pack4Lanes(shuffle + 4);
3751 imms[imm_count++] = wasm::SimdShuffle::Pack4Lanes(shuffle + 8);
3752 imms[imm_count++] = wasm::SimdShuffle::Pack4Lanes(shuffle + 12);
3753 temps[temp_count++] = g.TempSimd128Register();
3754 }
3755
3756 // Use DefineAsRegister(node) and Use(src0) if we can without forcing an extra
3757 // move instruction in the CodeGenerator.
3758 Node* input0 = node->InputAt(0);
3759 InstructionOperand dst =
3760 no_same_as_first ? g.DefineAsRegister(node) : g.DefineSameAsFirst(node);
3761 // TODO(v8:9198): Use src0_needs_reg when we have memory alignment for SIMD.
3762 // We only need a unique register for input0 if we use temp registers.
3763 InstructionOperand src0 =
3764 temp_count ? g.UseUniqueRegister(input0) : g.UseRegister(input0);
3765 USE(src0_needs_reg);
3766
3767 int input_count = 0;
3768 InstructionOperand inputs[2 + kMaxImms + kMaxTemps];
3769 inputs[input_count++] = src0;
3770 if (!is_swizzle) {
3771 Node* input1 = node->InputAt(1);
3772 // TODO(v8:9198): Use src1_needs_reg when we have memory alignment for SIMD.
3773 // We only need a unique register for input1 if we use temp registers.
3774 inputs[input_count++] =
3775 temp_count ? g.UseUniqueRegister(input1) : g.UseRegister(input1);
3776 USE(src1_needs_reg);
3777 }
3778 for (int i = 0; i < imm_count; ++i) {
3779 inputs[input_count++] = g.UseImmediate(imms[i]);
3780 }
3781 Emit(opcode, 1, &dst, input_count, inputs, temp_count, temps);
3782 }
3783 #else
VisitI8x16Shuffle(Node * node)3784 void InstructionSelector::VisitI8x16Shuffle(Node* node) { UNREACHABLE(); }
3785 #endif // V8_ENABLE_WEBASSEMBLY
3786
3787 #if V8_ENABLE_WEBASSEMBLY
VisitI8x16Swizzle(Node * node)3788 void InstructionSelector::VisitI8x16Swizzle(Node* node) {
3789 InstructionCode op = kX64I8x16Swizzle;
3790
3791 bool relaxed = OpParameter<bool>(node->op());
3792 if (relaxed) {
3793 op |= MiscField::encode(true);
3794 } else {
3795 auto m = V128ConstMatcher(node->InputAt(1));
3796 if (m.HasResolvedValue()) {
3797 // If the indices vector is a const, check if they are in range, or if the
3798 // top bit is set, then we can avoid the paddusb in the codegen and simply
3799 // emit a pshufb.
3800 auto imms = m.ResolvedValue().immediate();
3801 op |= MiscField::encode(wasm::SimdSwizzle::AllInRangeOrTopBitSet(imms));
3802 }
3803 }
3804
3805 X64OperandGenerator g(this);
3806 Emit(op,
3807 IsSupported(AVX) ? g.DefineAsRegister(node) : g.DefineSameAsFirst(node),
3808 g.UseRegister(node->InputAt(0)), g.UseRegister(node->InputAt(1)));
3809 }
3810
3811 namespace {
3812 // pblendvb is a correct implementation for all the various relaxed lane select,
3813 // see https://github.com/WebAssembly/relaxed-simd/issues/17.
VisitRelaxedLaneSelect(InstructionSelector * selector,Node * node)3814 void VisitRelaxedLaneSelect(InstructionSelector* selector, Node* node) {
3815 X64OperandGenerator g(selector);
3816 // pblendvb copies src2 when mask is set, opposite from Wasm semantics.
3817 // node's inputs are: mask, lhs, rhs (determined in wasm-compiler.cc).
3818 if (selector->IsSupported(AVX)) {
3819 selector->Emit(
3820 kX64Pblendvb, g.DefineAsRegister(node), g.UseRegister(node->InputAt(2)),
3821 g.UseRegister(node->InputAt(1)), g.UseRegister(node->InputAt(0)));
3822 } else {
3823 // SSE4.1 pblendvb requires xmm0 to hold the mask as an implicit operand.
3824 selector->Emit(kX64Pblendvb, g.DefineSameAsFirst(node),
3825 g.UseRegister(node->InputAt(2)),
3826 g.UseRegister(node->InputAt(1)),
3827 g.UseFixed(node->InputAt(0), xmm0));
3828 }
3829 }
3830 } // namespace
3831
VisitI8x16RelaxedLaneSelect(Node * node)3832 void InstructionSelector::VisitI8x16RelaxedLaneSelect(Node* node) {
3833 VisitRelaxedLaneSelect(this, node);
3834 }
VisitI16x8RelaxedLaneSelect(Node * node)3835 void InstructionSelector::VisitI16x8RelaxedLaneSelect(Node* node) {
3836 VisitRelaxedLaneSelect(this, node);
3837 }
VisitI32x4RelaxedLaneSelect(Node * node)3838 void InstructionSelector::VisitI32x4RelaxedLaneSelect(Node* node) {
3839 VisitRelaxedLaneSelect(this, node);
3840 }
VisitI64x2RelaxedLaneSelect(Node * node)3841 void InstructionSelector::VisitI64x2RelaxedLaneSelect(Node* node) {
3842 VisitRelaxedLaneSelect(this, node);
3843 }
3844 #else
VisitI8x16Swizzle(Node * node)3845 void InstructionSelector::VisitI8x16Swizzle(Node* node) { UNREACHABLE(); }
VisitI8x16RelaxedLaneSelect(Node * node)3846 void InstructionSelector::VisitI8x16RelaxedLaneSelect(Node* node) {
3847 UNREACHABLE();
3848 }
VisitI16x8RelaxedLaneSelect(Node * node)3849 void InstructionSelector::VisitI16x8RelaxedLaneSelect(Node* node) {
3850 UNREACHABLE();
3851 }
VisitI32x4RelaxedLaneSelect(Node * node)3852 void InstructionSelector::VisitI32x4RelaxedLaneSelect(Node* node) {
3853 UNREACHABLE();
3854 }
VisitI64x2RelaxedLaneSelect(Node * node)3855 void InstructionSelector::VisitI64x2RelaxedLaneSelect(Node* node) {
3856 UNREACHABLE();
3857 }
3858 #endif // V8_ENABLE_WEBASSEMBLY
3859
3860 namespace {
3861 // Used for pmin/pmax and relaxed min/max.
VisitMinOrMax(InstructionSelector * selector,Node * node,ArchOpcode opcode,bool flip_inputs)3862 void VisitMinOrMax(InstructionSelector* selector, Node* node, ArchOpcode opcode,
3863 bool flip_inputs) {
3864 X64OperandGenerator g(selector);
3865 InstructionOperand dst = selector->IsSupported(AVX)
3866 ? g.DefineAsRegister(node)
3867 : g.DefineSameAsFirst(node);
3868 if (flip_inputs) {
3869 // Due to the way minps/minpd work, we want the dst to be same as the second
3870 // input: b = pmin(a, b) directly maps to minps b a.
3871 selector->Emit(opcode, dst, g.UseRegister(node->InputAt(1)),
3872 g.UseRegister(node->InputAt(0)));
3873 } else {
3874 selector->Emit(opcode, dst, g.UseRegister(node->InputAt(0)),
3875 g.UseRegister(node->InputAt(1)));
3876 }
3877 }
3878 } // namespace
3879
VisitF32x4Pmin(Node * node)3880 void InstructionSelector::VisitF32x4Pmin(Node* node) {
3881 VisitMinOrMax(this, node, kX64Minps, true);
3882 }
3883
VisitF32x4Pmax(Node * node)3884 void InstructionSelector::VisitF32x4Pmax(Node* node) {
3885 VisitMinOrMax(this, node, kX64Maxps, true);
3886 }
3887
VisitF64x2Pmin(Node * node)3888 void InstructionSelector::VisitF64x2Pmin(Node* node) {
3889 VisitMinOrMax(this, node, kX64Minpd, true);
3890 }
3891
VisitF64x2Pmax(Node * node)3892 void InstructionSelector::VisitF64x2Pmax(Node* node) {
3893 VisitMinOrMax(this, node, kX64Maxpd, true);
3894 }
3895
VisitF32x4RelaxedMin(Node * node)3896 void InstructionSelector::VisitF32x4RelaxedMin(Node* node) {
3897 VisitMinOrMax(this, node, kX64Minps, false);
3898 }
3899
VisitF32x4RelaxedMax(Node * node)3900 void InstructionSelector::VisitF32x4RelaxedMax(Node* node) {
3901 VisitMinOrMax(this, node, kX64Maxps, false);
3902 }
3903
VisitF64x2RelaxedMin(Node * node)3904 void InstructionSelector::VisitF64x2RelaxedMin(Node* node) {
3905 VisitMinOrMax(this, node, kX64Minpd, false);
3906 }
3907
VisitF64x2RelaxedMax(Node * node)3908 void InstructionSelector::VisitF64x2RelaxedMax(Node* node) {
3909 VisitMinOrMax(this, node, kX64Maxpd, false);
3910 }
3911
VisitI32x4ExtAddPairwiseI16x8S(Node * node)3912 void InstructionSelector::VisitI32x4ExtAddPairwiseI16x8S(Node* node) {
3913 X64OperandGenerator g(this);
3914 InstructionOperand dst = CpuFeatures::IsSupported(AVX)
3915 ? g.DefineAsRegister(node)
3916 : g.DefineSameAsFirst(node);
3917 Emit(kX64I32x4ExtAddPairwiseI16x8S, dst, g.UseRegister(node->InputAt(0)));
3918 }
3919
VisitI32x4ExtAddPairwiseI16x8U(Node * node)3920 void InstructionSelector::VisitI32x4ExtAddPairwiseI16x8U(Node* node) {
3921 X64OperandGenerator g(this);
3922 InstructionOperand dst = CpuFeatures::IsSupported(AVX)
3923 ? g.DefineAsRegister(node)
3924 : g.DefineSameAsFirst(node);
3925 Emit(kX64I32x4ExtAddPairwiseI16x8U, dst, g.UseRegister(node->InputAt(0)));
3926 }
3927
VisitI16x8ExtAddPairwiseI8x16S(Node * node)3928 void InstructionSelector::VisitI16x8ExtAddPairwiseI8x16S(Node* node) {
3929 X64OperandGenerator g(this);
3930 // Codegen depends on dst != src.
3931 Emit(kX64I16x8ExtAddPairwiseI8x16S, g.DefineAsRegister(node),
3932 g.UseUniqueRegister(node->InputAt(0)));
3933 }
3934
VisitI16x8ExtAddPairwiseI8x16U(Node * node)3935 void InstructionSelector::VisitI16x8ExtAddPairwiseI8x16U(Node* node) {
3936 X64OperandGenerator g(this);
3937 InstructionOperand dst = CpuFeatures::IsSupported(AVX)
3938 ? g.DefineAsRegister(node)
3939 : g.DefineSameAsFirst(node);
3940 Emit(kX64I16x8ExtAddPairwiseI8x16U, dst, g.UseRegister(node->InputAt(0)));
3941 }
3942
VisitI8x16Popcnt(Node * node)3943 void InstructionSelector::VisitI8x16Popcnt(Node* node) {
3944 X64OperandGenerator g(this);
3945 InstructionOperand temps[] = {g.TempSimd128Register()};
3946 Emit(kX64I8x16Popcnt, g.DefineAsRegister(node),
3947 g.UseUniqueRegister(node->InputAt(0)), arraysize(temps), temps);
3948 }
3949
VisitF64x2ConvertLowI32x4U(Node * node)3950 void InstructionSelector::VisitF64x2ConvertLowI32x4U(Node* node) {
3951 X64OperandGenerator g(this);
3952 InstructionOperand dst =
3953 IsSupported(AVX) ? g.DefineAsRegister(node) : g.DefineSameAsFirst(node);
3954 Emit(kX64F64x2ConvertLowI32x4U, dst, g.UseRegister(node->InputAt(0)));
3955 }
3956
VisitI32x4TruncSatF64x2SZero(Node * node)3957 void InstructionSelector::VisitI32x4TruncSatF64x2SZero(Node* node) {
3958 X64OperandGenerator g(this);
3959 if (CpuFeatures::IsSupported(AVX)) {
3960 // Requires dst != src.
3961 Emit(kX64I32x4TruncSatF64x2SZero, g.DefineAsRegister(node),
3962 g.UseUniqueRegister(node->InputAt(0)));
3963 } else {
3964 Emit(kX64I32x4TruncSatF64x2SZero, g.DefineSameAsFirst(node),
3965 g.UseRegister(node->InputAt(0)));
3966 }
3967 }
3968
VisitI32x4TruncSatF64x2UZero(Node * node)3969 void InstructionSelector::VisitI32x4TruncSatF64x2UZero(Node* node) {
3970 X64OperandGenerator g(this);
3971 InstructionOperand dst = CpuFeatures::IsSupported(AVX)
3972 ? g.DefineAsRegister(node)
3973 : g.DefineSameAsFirst(node);
3974 Emit(kX64I32x4TruncSatF64x2UZero, dst, g.UseRegister(node->InputAt(0)));
3975 }
3976
VisitI32x4RelaxedTruncF64x2SZero(Node * node)3977 void InstructionSelector::VisitI32x4RelaxedTruncF64x2SZero(Node* node) {
3978 VisitFloatUnop(this, node, node->InputAt(0), kX64Cvttpd2dq);
3979 }
3980
VisitI32x4RelaxedTruncF64x2UZero(Node * node)3981 void InstructionSelector::VisitI32x4RelaxedTruncF64x2UZero(Node* node) {
3982 VisitFloatUnop(this, node, node->InputAt(0), kX64I32x4TruncF64x2UZero);
3983 }
3984
VisitI32x4RelaxedTruncF32x4S(Node * node)3985 void InstructionSelector::VisitI32x4RelaxedTruncF32x4S(Node* node) {
3986 VisitFloatUnop(this, node, node->InputAt(0), kX64Cvttps2dq);
3987 }
3988
VisitI32x4RelaxedTruncF32x4U(Node * node)3989 void InstructionSelector::VisitI32x4RelaxedTruncF32x4U(Node* node) {
3990 VisitFloatUnop(this, node, node->InputAt(0), kX64I32x4TruncF32x4U);
3991 }
3992
VisitI64x2GtS(Node * node)3993 void InstructionSelector::VisitI64x2GtS(Node* node) {
3994 X64OperandGenerator g(this);
3995 if (CpuFeatures::IsSupported(AVX)) {
3996 Emit(kX64I64x2GtS, g.DefineAsRegister(node),
3997 g.UseRegister(node->InputAt(0)), g.UseRegister(node->InputAt(1)));
3998 } else if (CpuFeatures::IsSupported(SSE4_2)) {
3999 Emit(kX64I64x2GtS, g.DefineSameAsFirst(node),
4000 g.UseRegister(node->InputAt(0)), g.UseRegister(node->InputAt(1)));
4001 } else {
4002 Emit(kX64I64x2GtS, g.DefineAsRegister(node),
4003 g.UseUniqueRegister(node->InputAt(0)),
4004 g.UseUniqueRegister(node->InputAt(1)));
4005 }
4006 }
4007
VisitI64x2GeS(Node * node)4008 void InstructionSelector::VisitI64x2GeS(Node* node) {
4009 X64OperandGenerator g(this);
4010 if (CpuFeatures::IsSupported(AVX)) {
4011 Emit(kX64I64x2GeS, g.DefineAsRegister(node),
4012 g.UseRegister(node->InputAt(0)), g.UseRegister(node->InputAt(1)));
4013 } else if (CpuFeatures::IsSupported(SSE4_2)) {
4014 Emit(kX64I64x2GeS, g.DefineAsRegister(node),
4015 g.UseUniqueRegister(node->InputAt(0)),
4016 g.UseRegister(node->InputAt(1)));
4017 } else {
4018 Emit(kX64I64x2GeS, g.DefineAsRegister(node),
4019 g.UseUniqueRegister(node->InputAt(0)),
4020 g.UseUniqueRegister(node->InputAt(1)));
4021 }
4022 }
4023
VisitI64x2Abs(Node * node)4024 void InstructionSelector::VisitI64x2Abs(Node* node) {
4025 X64OperandGenerator g(this);
4026 if (CpuFeatures::IsSupported(AVX)) {
4027 Emit(kX64I64x2Abs, g.DefineAsRegister(node),
4028 g.UseUniqueRegister(node->InputAt(0)));
4029 } else {
4030 Emit(kX64I64x2Abs, g.DefineSameAsFirst(node),
4031 g.UseRegister(node->InputAt(0)));
4032 }
4033 }
4034
VisitF64x2PromoteLowF32x4(Node * node)4035 void InstructionSelector::VisitF64x2PromoteLowF32x4(Node* node) {
4036 X64OperandGenerator g(this);
4037 InstructionCode code = kX64F64x2PromoteLowF32x4;
4038 Node* input = node->InputAt(0);
4039 LoadTransformMatcher m(input);
4040
4041 if (m.Is(LoadTransformation::kS128Load64Zero) && CanCover(node, input)) {
4042 if (m.ResolvedValue().kind == MemoryAccessKind::kProtected) {
4043 code |= AccessModeField::encode(kMemoryAccessProtected);
4044 }
4045 // LoadTransforms cannot be eliminated, so they are visited even if
4046 // unused. Mark it as defined so that we don't visit it.
4047 MarkAsDefined(input);
4048 VisitLoad(node, input, code);
4049 return;
4050 }
4051
4052 VisitRR(this, node, code);
4053 }
4054
AddOutputToSelectContinuation(OperandGenerator * g,int first_input_index,Node * node)4055 void InstructionSelector::AddOutputToSelectContinuation(OperandGenerator* g,
4056 int first_input_index,
4057 Node* node) {
4058 continuation_outputs_.push_back(
4059 g->DefineSameAsInput(node, first_input_index));
4060 }
4061
4062 // static
4063 MachineOperatorBuilder::Flags
SupportedMachineOperatorFlags()4064 InstructionSelector::SupportedMachineOperatorFlags() {
4065 MachineOperatorBuilder::Flags flags =
4066 MachineOperatorBuilder::kWord32ShiftIsSafe |
4067 MachineOperatorBuilder::kWord32Ctz | MachineOperatorBuilder::kWord64Ctz |
4068 MachineOperatorBuilder::kWord32Rol | MachineOperatorBuilder::kWord64Rol |
4069 MachineOperatorBuilder::kWord32Select |
4070 MachineOperatorBuilder::kWord64Select;
4071 if (CpuFeatures::IsSupported(POPCNT)) {
4072 flags |= MachineOperatorBuilder::kWord32Popcnt |
4073 MachineOperatorBuilder::kWord64Popcnt;
4074 }
4075 if (CpuFeatures::IsSupported(SSE4_1)) {
4076 flags |= MachineOperatorBuilder::kFloat32RoundDown |
4077 MachineOperatorBuilder::kFloat64RoundDown |
4078 MachineOperatorBuilder::kFloat32RoundUp |
4079 MachineOperatorBuilder::kFloat64RoundUp |
4080 MachineOperatorBuilder::kFloat32RoundTruncate |
4081 MachineOperatorBuilder::kFloat64RoundTruncate |
4082 MachineOperatorBuilder::kFloat32RoundTiesEven |
4083 MachineOperatorBuilder::kFloat64RoundTiesEven;
4084 }
4085 return flags;
4086 }
4087
4088 // static
4089 MachineOperatorBuilder::AlignmentRequirements
AlignmentRequirements()4090 InstructionSelector::AlignmentRequirements() {
4091 return MachineOperatorBuilder::AlignmentRequirements::
4092 FullUnalignedAccessSupport();
4093 }
4094
4095 } // namespace compiler
4096 } // namespace internal
4097 } // namespace v8
4098