1 // Copyright 2013 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "src/compiler/code-generator.h"
6
7 #include <limits>
8
9 #include "src/compilation-info.h"
10 #include "src/compiler/code-generator-impl.h"
11 #include "src/compiler/gap-resolver.h"
12 #include "src/compiler/node-matchers.h"
13 #include "src/compiler/osr.h"
14 #include "src/heap/heap-inl.h"
15 #include "src/wasm/wasm-module.h"
16 #include "src/x64/assembler-x64.h"
17 #include "src/x64/macro-assembler-x64.h"
18
19 namespace v8 {
20 namespace internal {
21 namespace compiler {
22
23 #define __ masm()->
24
25 // Adds X64 specific methods for decoding operands.
26 class X64OperandConverter : public InstructionOperandConverter {
27 public:
X64OperandConverter(CodeGenerator * gen,Instruction * instr)28 X64OperandConverter(CodeGenerator* gen, Instruction* instr)
29 : InstructionOperandConverter(gen, instr) {}
30
InputImmediate(size_t index)31 Immediate InputImmediate(size_t index) {
32 return ToImmediate(instr_->InputAt(index));
33 }
34
InputOperand(size_t index,int extra=0)35 Operand InputOperand(size_t index, int extra = 0) {
36 return ToOperand(instr_->InputAt(index), extra);
37 }
38
OutputOperand()39 Operand OutputOperand() { return ToOperand(instr_->Output()); }
40
ToImmediate(InstructionOperand * operand)41 Immediate ToImmediate(InstructionOperand* operand) {
42 Constant constant = ToConstant(operand);
43 if (constant.type() == Constant::kFloat64) {
44 DCHECK_EQ(0, bit_cast<int64_t>(constant.ToFloat64()));
45 return Immediate(0);
46 }
47 if (RelocInfo::IsWasmReference(constant.rmode())) {
48 return Immediate(constant.ToInt32(), constant.rmode());
49 }
50 return Immediate(constant.ToInt32());
51 }
52
ToOperand(InstructionOperand * op,int extra=0)53 Operand ToOperand(InstructionOperand* op, int extra = 0) {
54 DCHECK(op->IsStackSlot() || op->IsFPStackSlot());
55 return SlotToOperand(AllocatedOperand::cast(op)->index(), extra);
56 }
57
SlotToOperand(int slot_index,int extra=0)58 Operand SlotToOperand(int slot_index, int extra = 0) {
59 FrameOffset offset = frame_access_state()->GetFrameOffset(slot_index);
60 return Operand(offset.from_stack_pointer() ? rsp : rbp,
61 offset.offset() + extra);
62 }
63
NextOffset(size_t * offset)64 static size_t NextOffset(size_t* offset) {
65 size_t i = *offset;
66 (*offset)++;
67 return i;
68 }
69
ScaleFor(AddressingMode one,AddressingMode mode)70 static ScaleFactor ScaleFor(AddressingMode one, AddressingMode mode) {
71 STATIC_ASSERT(0 == static_cast<int>(times_1));
72 STATIC_ASSERT(1 == static_cast<int>(times_2));
73 STATIC_ASSERT(2 == static_cast<int>(times_4));
74 STATIC_ASSERT(3 == static_cast<int>(times_8));
75 int scale = static_cast<int>(mode - one);
76 DCHECK(scale >= 0 && scale < 4);
77 return static_cast<ScaleFactor>(scale);
78 }
79
MemoryOperand(size_t * offset)80 Operand MemoryOperand(size_t* offset) {
81 AddressingMode mode = AddressingModeField::decode(instr_->opcode());
82 switch (mode) {
83 case kMode_MR: {
84 Register base = InputRegister(NextOffset(offset));
85 int32_t disp = 0;
86 return Operand(base, disp);
87 }
88 case kMode_MRI: {
89 Register base = InputRegister(NextOffset(offset));
90 int32_t disp = InputInt32(NextOffset(offset));
91 return Operand(base, disp);
92 }
93 case kMode_MR1:
94 case kMode_MR2:
95 case kMode_MR4:
96 case kMode_MR8: {
97 Register base = InputRegister(NextOffset(offset));
98 Register index = InputRegister(NextOffset(offset));
99 ScaleFactor scale = ScaleFor(kMode_MR1, mode);
100 int32_t disp = 0;
101 return Operand(base, index, scale, disp);
102 }
103 case kMode_MR1I:
104 case kMode_MR2I:
105 case kMode_MR4I:
106 case kMode_MR8I: {
107 Register base = InputRegister(NextOffset(offset));
108 Register index = InputRegister(NextOffset(offset));
109 ScaleFactor scale = ScaleFor(kMode_MR1I, mode);
110 int32_t disp = InputInt32(NextOffset(offset));
111 return Operand(base, index, scale, disp);
112 }
113 case kMode_M1: {
114 Register base = InputRegister(NextOffset(offset));
115 int32_t disp = 0;
116 return Operand(base, disp);
117 }
118 case kMode_M2:
119 UNREACHABLE(); // Should use kModeMR with more compact encoding instead
120 return Operand(no_reg, 0);
121 case kMode_M4:
122 case kMode_M8: {
123 Register index = InputRegister(NextOffset(offset));
124 ScaleFactor scale = ScaleFor(kMode_M1, mode);
125 int32_t disp = 0;
126 return Operand(index, scale, disp);
127 }
128 case kMode_M1I:
129 case kMode_M2I:
130 case kMode_M4I:
131 case kMode_M8I: {
132 Register index = InputRegister(NextOffset(offset));
133 ScaleFactor scale = ScaleFor(kMode_M1I, mode);
134 int32_t disp = InputInt32(NextOffset(offset));
135 return Operand(index, scale, disp);
136 }
137 case kMode_Root: {
138 Register base = kRootRegister;
139 int32_t disp = InputInt32(NextOffset(offset));
140 return Operand(base, disp);
141 }
142 case kMode_None:
143 UNREACHABLE();
144 return Operand(no_reg, 0);
145 }
146 UNREACHABLE();
147 return Operand(no_reg, 0);
148 }
149
MemoryOperand(size_t first_input=0)150 Operand MemoryOperand(size_t first_input = 0) {
151 return MemoryOperand(&first_input);
152 }
153 };
154
155
156 namespace {
157
HasImmediateInput(Instruction * instr,size_t index)158 bool HasImmediateInput(Instruction* instr, size_t index) {
159 return instr->InputAt(index)->IsImmediate();
160 }
161
162
163 class OutOfLineLoadZero final : public OutOfLineCode {
164 public:
OutOfLineLoadZero(CodeGenerator * gen,Register result)165 OutOfLineLoadZero(CodeGenerator* gen, Register result)
166 : OutOfLineCode(gen), result_(result) {}
167
Generate()168 void Generate() final { __ xorl(result_, result_); }
169
170 private:
171 Register const result_;
172 };
173
174 class OutOfLineLoadFloat32NaN final : public OutOfLineCode {
175 public:
OutOfLineLoadFloat32NaN(CodeGenerator * gen,XMMRegister result)176 OutOfLineLoadFloat32NaN(CodeGenerator* gen, XMMRegister result)
177 : OutOfLineCode(gen), result_(result) {}
178
Generate()179 void Generate() final {
180 __ Xorps(result_, result_);
181 __ Divss(result_, result_);
182 }
183
184 private:
185 XMMRegister const result_;
186 };
187
188 class OutOfLineLoadFloat64NaN final : public OutOfLineCode {
189 public:
OutOfLineLoadFloat64NaN(CodeGenerator * gen,XMMRegister result)190 OutOfLineLoadFloat64NaN(CodeGenerator* gen, XMMRegister result)
191 : OutOfLineCode(gen), result_(result) {}
192
Generate()193 void Generate() final {
194 __ Xorpd(result_, result_);
195 __ Divsd(result_, result_);
196 }
197
198 private:
199 XMMRegister const result_;
200 };
201
202 class OutOfLineTruncateDoubleToI final : public OutOfLineCode {
203 public:
OutOfLineTruncateDoubleToI(CodeGenerator * gen,Register result,XMMRegister input,UnwindingInfoWriter * unwinding_info_writer)204 OutOfLineTruncateDoubleToI(CodeGenerator* gen, Register result,
205 XMMRegister input,
206 UnwindingInfoWriter* unwinding_info_writer)
207 : OutOfLineCode(gen),
208 result_(result),
209 input_(input),
210 unwinding_info_writer_(unwinding_info_writer) {}
211
Generate()212 void Generate() final {
213 __ subp(rsp, Immediate(kDoubleSize));
214 unwinding_info_writer_->MaybeIncreaseBaseOffsetAt(__ pc_offset(),
215 kDoubleSize);
216 __ Movsd(MemOperand(rsp, 0), input_);
217 __ SlowTruncateToI(result_, rsp, 0);
218 __ addp(rsp, Immediate(kDoubleSize));
219 unwinding_info_writer_->MaybeIncreaseBaseOffsetAt(__ pc_offset(),
220 -kDoubleSize);
221 }
222
223 private:
224 Register const result_;
225 XMMRegister const input_;
226 UnwindingInfoWriter* const unwinding_info_writer_;
227 };
228
229
230 class OutOfLineRecordWrite final : public OutOfLineCode {
231 public:
OutOfLineRecordWrite(CodeGenerator * gen,Register object,Operand operand,Register value,Register scratch0,Register scratch1,RecordWriteMode mode)232 OutOfLineRecordWrite(CodeGenerator* gen, Register object, Operand operand,
233 Register value, Register scratch0, Register scratch1,
234 RecordWriteMode mode)
235 : OutOfLineCode(gen),
236 object_(object),
237 operand_(operand),
238 value_(value),
239 scratch0_(scratch0),
240 scratch1_(scratch1),
241 mode_(mode) {}
242
Generate()243 void Generate() final {
244 if (mode_ > RecordWriteMode::kValueIsPointer) {
245 __ JumpIfSmi(value_, exit());
246 }
247 __ CheckPageFlag(value_, scratch0_,
248 MemoryChunk::kPointersToHereAreInterestingMask, zero,
249 exit());
250 RememberedSetAction const remembered_set_action =
251 mode_ > RecordWriteMode::kValueIsMap ? EMIT_REMEMBERED_SET
252 : OMIT_REMEMBERED_SET;
253 SaveFPRegsMode const save_fp_mode =
254 frame()->DidAllocateDoubleRegisters() ? kSaveFPRegs : kDontSaveFPRegs;
255 RecordWriteStub stub(isolate(), object_, scratch0_, scratch1_,
256 remembered_set_action, save_fp_mode);
257 __ leap(scratch1_, operand_);
258 __ CallStub(&stub);
259 }
260
261 private:
262 Register const object_;
263 Operand const operand_;
264 Register const value_;
265 Register const scratch0_;
266 Register const scratch1_;
267 RecordWriteMode const mode_;
268 };
269
270 class WasmOutOfLineTrap final : public OutOfLineCode {
271 public:
WasmOutOfLineTrap(CodeGenerator * gen,int pc,bool frame_elided,int32_t position,Instruction * instr)272 WasmOutOfLineTrap(CodeGenerator* gen, int pc, bool frame_elided,
273 int32_t position, Instruction* instr)
274 : OutOfLineCode(gen),
275 gen_(gen),
276 pc_(pc),
277 frame_elided_(frame_elided),
278 position_(position),
279 instr_(instr) {}
280
281 // TODO(eholk): Refactor this method to take the code generator as a
282 // parameter.
Generate()283 void Generate() final {
284 __ RecordProtectedInstructionLanding(pc_);
285
286 if (frame_elided_) {
287 __ EnterFrame(StackFrame::WASM_COMPILED);
288 }
289
290 wasm::TrapReason trap_id = wasm::kTrapMemOutOfBounds;
291 int trap_reason = wasm::WasmOpcodes::TrapReasonToMessageId(trap_id);
292 __ Push(Smi::FromInt(trap_reason));
293 __ Push(Smi::FromInt(position_));
294 __ Move(rsi, gen_->isolate()->native_context());
295 __ CallRuntime(Runtime::kThrowWasmError);
296
297 if (instr_->reference_map() != nullptr) {
298 gen_->RecordSafepoint(instr_->reference_map(), Safepoint::kSimple, 0,
299 Safepoint::kNoLazyDeopt);
300 }
301 }
302
303 private:
304 CodeGenerator* gen_;
305 int pc_;
306 bool frame_elided_;
307 int32_t position_;
308 Instruction* instr_;
309 };
310
EmitOOLTrapIfNeeded(Zone * zone,CodeGenerator * codegen,InstructionCode opcode,size_t input_count,X64OperandConverter & i,int pc,Instruction * instr)311 void EmitOOLTrapIfNeeded(Zone* zone, CodeGenerator* codegen,
312 InstructionCode opcode, size_t input_count,
313 X64OperandConverter& i, int pc, Instruction* instr) {
314 const X64MemoryProtection protection =
315 static_cast<X64MemoryProtection>(MiscField::decode(opcode));
316 if (protection == X64MemoryProtection::kProtected) {
317 const bool frame_elided = !codegen->frame_access_state()->has_frame();
318 const int32_t position = i.InputInt32(input_count - 1);
319 new (zone) WasmOutOfLineTrap(codegen, pc, frame_elided, position, instr);
320 }
321 }
322 } // namespace
323
324
325 #define ASSEMBLE_UNOP(asm_instr) \
326 do { \
327 if (instr->Output()->IsRegister()) { \
328 __ asm_instr(i.OutputRegister()); \
329 } else { \
330 __ asm_instr(i.OutputOperand()); \
331 } \
332 } while (0)
333
334
335 #define ASSEMBLE_BINOP(asm_instr) \
336 do { \
337 if (HasImmediateInput(instr, 1)) { \
338 if (instr->InputAt(0)->IsRegister()) { \
339 __ asm_instr(i.InputRegister(0), i.InputImmediate(1)); \
340 } else { \
341 __ asm_instr(i.InputOperand(0), i.InputImmediate(1)); \
342 } \
343 } else { \
344 if (instr->InputAt(1)->IsRegister()) { \
345 __ asm_instr(i.InputRegister(0), i.InputRegister(1)); \
346 } else { \
347 __ asm_instr(i.InputRegister(0), i.InputOperand(1)); \
348 } \
349 } \
350 } while (0)
351
352 #define ASSEMBLE_COMPARE(asm_instr) \
353 do { \
354 if (AddressingModeField::decode(instr->opcode()) != kMode_None) { \
355 size_t index = 0; \
356 Operand left = i.MemoryOperand(&index); \
357 if (HasImmediateInput(instr, index)) { \
358 __ asm_instr(left, i.InputImmediate(index)); \
359 } else { \
360 __ asm_instr(left, i.InputRegister(index)); \
361 } \
362 } else { \
363 if (HasImmediateInput(instr, 1)) { \
364 if (instr->InputAt(0)->IsRegister()) { \
365 __ asm_instr(i.InputRegister(0), i.InputImmediate(1)); \
366 } else { \
367 __ asm_instr(i.InputOperand(0), i.InputImmediate(1)); \
368 } \
369 } else { \
370 if (instr->InputAt(1)->IsRegister()) { \
371 __ asm_instr(i.InputRegister(0), i.InputRegister(1)); \
372 } else { \
373 __ asm_instr(i.InputRegister(0), i.InputOperand(1)); \
374 } \
375 } \
376 } \
377 } while (0)
378
379 #define ASSEMBLE_MULT(asm_instr) \
380 do { \
381 if (HasImmediateInput(instr, 1)) { \
382 if (instr->InputAt(0)->IsRegister()) { \
383 __ asm_instr(i.OutputRegister(), i.InputRegister(0), \
384 i.InputImmediate(1)); \
385 } else { \
386 __ asm_instr(i.OutputRegister(), i.InputOperand(0), \
387 i.InputImmediate(1)); \
388 } \
389 } else { \
390 if (instr->InputAt(1)->IsRegister()) { \
391 __ asm_instr(i.OutputRegister(), i.InputRegister(1)); \
392 } else { \
393 __ asm_instr(i.OutputRegister(), i.InputOperand(1)); \
394 } \
395 } \
396 } while (0)
397
398
399 #define ASSEMBLE_SHIFT(asm_instr, width) \
400 do { \
401 if (HasImmediateInput(instr, 1)) { \
402 if (instr->Output()->IsRegister()) { \
403 __ asm_instr(i.OutputRegister(), Immediate(i.InputInt##width(1))); \
404 } else { \
405 __ asm_instr(i.OutputOperand(), Immediate(i.InputInt##width(1))); \
406 } \
407 } else { \
408 if (instr->Output()->IsRegister()) { \
409 __ asm_instr##_cl(i.OutputRegister()); \
410 } else { \
411 __ asm_instr##_cl(i.OutputOperand()); \
412 } \
413 } \
414 } while (0)
415
416
417 #define ASSEMBLE_MOVX(asm_instr) \
418 do { \
419 if (instr->addressing_mode() != kMode_None) { \
420 __ asm_instr(i.OutputRegister(), i.MemoryOperand()); \
421 } else if (instr->InputAt(0)->IsRegister()) { \
422 __ asm_instr(i.OutputRegister(), i.InputRegister(0)); \
423 } else { \
424 __ asm_instr(i.OutputRegister(), i.InputOperand(0)); \
425 } \
426 } while (0)
427
428 #define ASSEMBLE_SSE_BINOP(asm_instr) \
429 do { \
430 if (instr->InputAt(1)->IsFPRegister()) { \
431 __ asm_instr(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); \
432 } else { \
433 __ asm_instr(i.InputDoubleRegister(0), i.InputOperand(1)); \
434 } \
435 } while (0)
436
437 #define ASSEMBLE_SSE_UNOP(asm_instr) \
438 do { \
439 if (instr->InputAt(0)->IsFPRegister()) { \
440 __ asm_instr(i.OutputDoubleRegister(), i.InputDoubleRegister(0)); \
441 } else { \
442 __ asm_instr(i.OutputDoubleRegister(), i.InputOperand(0)); \
443 } \
444 } while (0)
445
446 #define ASSEMBLE_AVX_BINOP(asm_instr) \
447 do { \
448 CpuFeatureScope avx_scope(masm(), AVX); \
449 if (instr->InputAt(1)->IsFPRegister()) { \
450 __ asm_instr(i.OutputDoubleRegister(), i.InputDoubleRegister(0), \
451 i.InputDoubleRegister(1)); \
452 } else { \
453 __ asm_instr(i.OutputDoubleRegister(), i.InputDoubleRegister(0), \
454 i.InputOperand(1)); \
455 } \
456 } while (0)
457
458 #define ASSEMBLE_CHECKED_LOAD_FLOAT(asm_instr, OutOfLineLoadNaN) \
459 do { \
460 auto result = i.OutputDoubleRegister(); \
461 auto buffer = i.InputRegister(0); \
462 auto index1 = i.InputRegister(1); \
463 auto index2 = i.InputUint32(2); \
464 OutOfLineCode* ool; \
465 if (instr->InputAt(3)->IsRegister()) { \
466 auto length = i.InputRegister(3); \
467 DCHECK_EQ(0u, index2); \
468 __ cmpl(index1, length); \
469 ool = new (zone()) OutOfLineLoadNaN(this, result); \
470 } else { \
471 auto length = i.InputUint32(3); \
472 RelocInfo::Mode rmode = i.ToConstant(instr->InputAt(3)).rmode(); \
473 DCHECK_LE(index2, length); \
474 __ cmpl(index1, Immediate(length - index2, rmode)); \
475 class OutOfLineLoadFloat final : public OutOfLineCode { \
476 public: \
477 OutOfLineLoadFloat(CodeGenerator* gen, XMMRegister result, \
478 Register buffer, Register index1, int32_t index2, \
479 int32_t length, RelocInfo::Mode rmode) \
480 : OutOfLineCode(gen), \
481 result_(result), \
482 buffer_(buffer), \
483 index1_(index1), \
484 index2_(index2), \
485 length_(length), \
486 rmode_(rmode) {} \
487 \
488 void Generate() final { \
489 __ leal(kScratchRegister, Operand(index1_, index2_)); \
490 __ Pcmpeqd(result_, result_); \
491 __ cmpl(kScratchRegister, Immediate(length_, rmode_)); \
492 __ j(above_equal, exit()); \
493 __ asm_instr(result_, \
494 Operand(buffer_, kScratchRegister, times_1, 0)); \
495 } \
496 \
497 private: \
498 XMMRegister const result_; \
499 Register const buffer_; \
500 Register const index1_; \
501 int32_t const index2_; \
502 int32_t const length_; \
503 RelocInfo::Mode rmode_; \
504 }; \
505 ool = new (zone()) OutOfLineLoadFloat(this, result, buffer, index1, \
506 index2, length, rmode); \
507 } \
508 __ j(above_equal, ool->entry()); \
509 __ asm_instr(result, Operand(buffer, index1, times_1, index2)); \
510 __ bind(ool->exit()); \
511 } while (false)
512
513 #define ASSEMBLE_CHECKED_LOAD_INTEGER(asm_instr) \
514 do { \
515 auto result = i.OutputRegister(); \
516 auto buffer = i.InputRegister(0); \
517 auto index1 = i.InputRegister(1); \
518 auto index2 = i.InputUint32(2); \
519 OutOfLineCode* ool; \
520 if (instr->InputAt(3)->IsRegister()) { \
521 auto length = i.InputRegister(3); \
522 DCHECK_EQ(0u, index2); \
523 __ cmpl(index1, length); \
524 ool = new (zone()) OutOfLineLoadZero(this, result); \
525 } else { \
526 auto length = i.InputUint32(3); \
527 RelocInfo::Mode rmode = i.ToConstant(instr->InputAt(3)).rmode(); \
528 DCHECK_LE(index2, length); \
529 __ cmpl(index1, Immediate(length - index2, rmode)); \
530 class OutOfLineLoadInteger final : public OutOfLineCode { \
531 public: \
532 OutOfLineLoadInteger(CodeGenerator* gen, Register result, \
533 Register buffer, Register index1, int32_t index2, \
534 int32_t length, RelocInfo::Mode rmode) \
535 : OutOfLineCode(gen), \
536 result_(result), \
537 buffer_(buffer), \
538 index1_(index1), \
539 index2_(index2), \
540 length_(length), \
541 rmode_(rmode) {} \
542 \
543 void Generate() final { \
544 Label oob; \
545 __ leal(kScratchRegister, Operand(index1_, index2_)); \
546 __ cmpl(kScratchRegister, Immediate(length_, rmode_)); \
547 __ j(above_equal, &oob, Label::kNear); \
548 __ asm_instr(result_, \
549 Operand(buffer_, kScratchRegister, times_1, 0)); \
550 __ jmp(exit()); \
551 __ bind(&oob); \
552 __ xorl(result_, result_); \
553 } \
554 \
555 private: \
556 Register const result_; \
557 Register const buffer_; \
558 Register const index1_; \
559 int32_t const index2_; \
560 int32_t const length_; \
561 RelocInfo::Mode const rmode_; \
562 }; \
563 ool = new (zone()) OutOfLineLoadInteger(this, result, buffer, index1, \
564 index2, length, rmode); \
565 } \
566 __ j(above_equal, ool->entry()); \
567 __ asm_instr(result, Operand(buffer, index1, times_1, index2)); \
568 __ bind(ool->exit()); \
569 } while (false)
570
571 #define ASSEMBLE_CHECKED_STORE_FLOAT(asm_instr) \
572 do { \
573 auto buffer = i.InputRegister(0); \
574 auto index1 = i.InputRegister(1); \
575 auto index2 = i.InputUint32(2); \
576 auto value = i.InputDoubleRegister(4); \
577 if (instr->InputAt(3)->IsRegister()) { \
578 auto length = i.InputRegister(3); \
579 DCHECK_EQ(0u, index2); \
580 Label done; \
581 __ cmpl(index1, length); \
582 __ j(above_equal, &done, Label::kNear); \
583 __ asm_instr(Operand(buffer, index1, times_1, index2), value); \
584 __ bind(&done); \
585 } else { \
586 auto length = i.InputUint32(3); \
587 RelocInfo::Mode rmode = i.ToConstant(instr->InputAt(3)).rmode(); \
588 DCHECK_LE(index2, length); \
589 __ cmpl(index1, Immediate(length - index2, rmode)); \
590 class OutOfLineStoreFloat final : public OutOfLineCode { \
591 public: \
592 OutOfLineStoreFloat(CodeGenerator* gen, Register buffer, \
593 Register index1, int32_t index2, int32_t length, \
594 XMMRegister value, RelocInfo::Mode rmode) \
595 : OutOfLineCode(gen), \
596 buffer_(buffer), \
597 index1_(index1), \
598 index2_(index2), \
599 length_(length), \
600 value_(value), \
601 rmode_(rmode) {} \
602 \
603 void Generate() final { \
604 __ leal(kScratchRegister, Operand(index1_, index2_)); \
605 __ cmpl(kScratchRegister, Immediate(length_, rmode_)); \
606 __ j(above_equal, exit()); \
607 __ asm_instr(Operand(buffer_, kScratchRegister, times_1, 0), \
608 value_); \
609 } \
610 \
611 private: \
612 Register const buffer_; \
613 Register const index1_; \
614 int32_t const index2_; \
615 int32_t const length_; \
616 XMMRegister const value_; \
617 RelocInfo::Mode rmode_; \
618 }; \
619 auto ool = new (zone()) OutOfLineStoreFloat( \
620 this, buffer, index1, index2, length, value, rmode); \
621 __ j(above_equal, ool->entry()); \
622 __ asm_instr(Operand(buffer, index1, times_1, index2), value); \
623 __ bind(ool->exit()); \
624 } \
625 } while (false)
626
627 #define ASSEMBLE_CHECKED_STORE_INTEGER_IMPL(asm_instr, Value) \
628 do { \
629 auto buffer = i.InputRegister(0); \
630 auto index1 = i.InputRegister(1); \
631 auto index2 = i.InputUint32(2); \
632 if (instr->InputAt(3)->IsRegister()) { \
633 auto length = i.InputRegister(3); \
634 DCHECK_EQ(0u, index2); \
635 Label done; \
636 __ cmpl(index1, length); \
637 __ j(above_equal, &done, Label::kNear); \
638 __ asm_instr(Operand(buffer, index1, times_1, index2), value); \
639 __ bind(&done); \
640 } else { \
641 auto length = i.InputUint32(3); \
642 RelocInfo::Mode rmode = i.ToConstant(instr->InputAt(3)).rmode(); \
643 DCHECK_LE(index2, length); \
644 __ cmpl(index1, Immediate(length - index2, rmode)); \
645 class OutOfLineStoreInteger final : public OutOfLineCode { \
646 public: \
647 OutOfLineStoreInteger(CodeGenerator* gen, Register buffer, \
648 Register index1, int32_t index2, int32_t length, \
649 Value value, RelocInfo::Mode rmode) \
650 : OutOfLineCode(gen), \
651 buffer_(buffer), \
652 index1_(index1), \
653 index2_(index2), \
654 length_(length), \
655 value_(value), \
656 rmode_(rmode) {} \
657 \
658 void Generate() final { \
659 __ leal(kScratchRegister, Operand(index1_, index2_)); \
660 __ cmpl(kScratchRegister, Immediate(length_, rmode_)); \
661 __ j(above_equal, exit()); \
662 __ asm_instr(Operand(buffer_, kScratchRegister, times_1, 0), \
663 value_); \
664 } \
665 \
666 private: \
667 Register const buffer_; \
668 Register const index1_; \
669 int32_t const index2_; \
670 int32_t const length_; \
671 Value const value_; \
672 RelocInfo::Mode rmode_; \
673 }; \
674 auto ool = new (zone()) OutOfLineStoreInteger( \
675 this, buffer, index1, index2, length, value, rmode); \
676 __ j(above_equal, ool->entry()); \
677 __ asm_instr(Operand(buffer, index1, times_1, index2), value); \
678 __ bind(ool->exit()); \
679 } \
680 } while (false)
681
682 #define ASSEMBLE_CHECKED_STORE_INTEGER(asm_instr) \
683 do { \
684 if (instr->InputAt(4)->IsRegister()) { \
685 Register value = i.InputRegister(4); \
686 ASSEMBLE_CHECKED_STORE_INTEGER_IMPL(asm_instr, Register); \
687 } else { \
688 Immediate value = i.InputImmediate(4); \
689 ASSEMBLE_CHECKED_STORE_INTEGER_IMPL(asm_instr, Immediate); \
690 } \
691 } while (false)
692
693 #define ASSEMBLE_IEEE754_BINOP(name) \
694 do { \
695 __ PrepareCallCFunction(2); \
696 __ CallCFunction(ExternalReference::ieee754_##name##_function(isolate()), \
697 2); \
698 } while (false)
699
700 #define ASSEMBLE_IEEE754_UNOP(name) \
701 do { \
702 __ PrepareCallCFunction(1); \
703 __ CallCFunction(ExternalReference::ieee754_##name##_function(isolate()), \
704 1); \
705 } while (false)
706
AssembleDeconstructFrame()707 void CodeGenerator::AssembleDeconstructFrame() {
708 unwinding_info_writer_.MarkFrameDeconstructed(__ pc_offset());
709 __ movq(rsp, rbp);
710 __ popq(rbp);
711 }
712
AssemblePrepareTailCall()713 void CodeGenerator::AssemblePrepareTailCall() {
714 if (frame_access_state()->has_frame()) {
715 __ movq(rbp, MemOperand(rbp, 0));
716 }
717 frame_access_state()->SetFrameAccessToSP();
718 }
719
AssemblePopArgumentsAdaptorFrame(Register args_reg,Register scratch1,Register scratch2,Register scratch3)720 void CodeGenerator::AssemblePopArgumentsAdaptorFrame(Register args_reg,
721 Register scratch1,
722 Register scratch2,
723 Register scratch3) {
724 DCHECK(!AreAliased(args_reg, scratch1, scratch2, scratch3));
725 Label done;
726
727 // Check if current frame is an arguments adaptor frame.
728 __ cmpp(Operand(rbp, CommonFrameConstants::kContextOrFrameTypeOffset),
729 Immediate(StackFrame::TypeToMarker(StackFrame::ARGUMENTS_ADAPTOR)));
730 __ j(not_equal, &done, Label::kNear);
731
732 // Load arguments count from current arguments adaptor frame (note, it
733 // does not include receiver).
734 Register caller_args_count_reg = scratch1;
735 __ SmiToInteger32(
736 caller_args_count_reg,
737 Operand(rbp, ArgumentsAdaptorFrameConstants::kLengthOffset));
738
739 ParameterCount callee_args_count(args_reg);
740 __ PrepareForTailCall(callee_args_count, caller_args_count_reg, scratch2,
741 scratch3, ReturnAddressState::kOnStack);
742 __ bind(&done);
743 }
744
745 namespace {
746
AdjustStackPointerForTailCall(MacroAssembler * masm,FrameAccessState * state,int new_slot_above_sp,bool allow_shrinkage=true)747 void AdjustStackPointerForTailCall(MacroAssembler* masm,
748 FrameAccessState* state,
749 int new_slot_above_sp,
750 bool allow_shrinkage = true) {
751 int current_sp_offset = state->GetSPToFPSlotCount() +
752 StandardFrameConstants::kFixedSlotCountAboveFp;
753 int stack_slot_delta = new_slot_above_sp - current_sp_offset;
754 if (stack_slot_delta > 0) {
755 masm->subq(rsp, Immediate(stack_slot_delta * kPointerSize));
756 state->IncreaseSPDelta(stack_slot_delta);
757 } else if (allow_shrinkage && stack_slot_delta < 0) {
758 masm->addq(rsp, Immediate(-stack_slot_delta * kPointerSize));
759 state->IncreaseSPDelta(stack_slot_delta);
760 }
761 }
762
763 } // namespace
764
AssembleTailCallBeforeGap(Instruction * instr,int first_unused_stack_slot)765 void CodeGenerator::AssembleTailCallBeforeGap(Instruction* instr,
766 int first_unused_stack_slot) {
767 CodeGenerator::PushTypeFlags flags(kImmediatePush | kScalarPush);
768 ZoneVector<MoveOperands*> pushes(zone());
769 GetPushCompatibleMoves(instr, flags, &pushes);
770
771 if (!pushes.empty() &&
772 (LocationOperand::cast(pushes.back()->destination()).index() + 1 ==
773 first_unused_stack_slot)) {
774 X64OperandConverter g(this, instr);
775 for (auto move : pushes) {
776 LocationOperand destination_location(
777 LocationOperand::cast(move->destination()));
778 InstructionOperand source(move->source());
779 AdjustStackPointerForTailCall(masm(), frame_access_state(),
780 destination_location.index());
781 if (source.IsStackSlot()) {
782 LocationOperand source_location(LocationOperand::cast(source));
783 __ Push(g.SlotToOperand(source_location.index()));
784 } else if (source.IsRegister()) {
785 LocationOperand source_location(LocationOperand::cast(source));
786 __ Push(source_location.GetRegister());
787 } else if (source.IsImmediate()) {
788 __ Push(Immediate(ImmediateOperand::cast(source).inline_value()));
789 } else {
790 // Pushes of non-scalar data types is not supported.
791 UNIMPLEMENTED();
792 }
793 frame_access_state()->IncreaseSPDelta(1);
794 move->Eliminate();
795 }
796 }
797 AdjustStackPointerForTailCall(masm(), frame_access_state(),
798 first_unused_stack_slot, false);
799 }
800
AssembleTailCallAfterGap(Instruction * instr,int first_unused_stack_slot)801 void CodeGenerator::AssembleTailCallAfterGap(Instruction* instr,
802 int first_unused_stack_slot) {
803 AdjustStackPointerForTailCall(masm(), frame_access_state(),
804 first_unused_stack_slot);
805 }
806
807 // Assembles an instruction after register allocation, producing machine code.
AssembleArchInstruction(Instruction * instr)808 CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
809 Instruction* instr) {
810 X64OperandConverter i(this, instr);
811 InstructionCode opcode = instr->opcode();
812 ArchOpcode arch_opcode = ArchOpcodeField::decode(opcode);
813 switch (arch_opcode) {
814 case kArchCallCodeObject: {
815 EnsureSpaceForLazyDeopt();
816 if (HasImmediateInput(instr, 0)) {
817 Handle<Code> code = Handle<Code>::cast(i.InputHeapObject(0));
818 __ Call(code, RelocInfo::CODE_TARGET);
819 } else {
820 Register reg = i.InputRegister(0);
821 __ addp(reg, Immediate(Code::kHeaderSize - kHeapObjectTag));
822 __ call(reg);
823 }
824 RecordCallPosition(instr);
825 frame_access_state()->ClearSPDelta();
826 break;
827 }
828 case kArchTailCallCodeObjectFromJSFunction:
829 case kArchTailCallCodeObject: {
830 if (arch_opcode == kArchTailCallCodeObjectFromJSFunction) {
831 AssemblePopArgumentsAdaptorFrame(kJavaScriptCallArgCountRegister,
832 i.TempRegister(0), i.TempRegister(1),
833 i.TempRegister(2));
834 }
835 if (HasImmediateInput(instr, 0)) {
836 Handle<Code> code = Handle<Code>::cast(i.InputHeapObject(0));
837 __ jmp(code, RelocInfo::CODE_TARGET);
838 } else {
839 Register reg = i.InputRegister(0);
840 __ addp(reg, Immediate(Code::kHeaderSize - kHeapObjectTag));
841 __ jmp(reg);
842 }
843 unwinding_info_writer_.MarkBlockWillExit();
844 frame_access_state()->ClearSPDelta();
845 frame_access_state()->SetFrameAccessToDefault();
846 break;
847 }
848 case kArchTailCallAddress: {
849 CHECK(!HasImmediateInput(instr, 0));
850 Register reg = i.InputRegister(0);
851 __ jmp(reg);
852 unwinding_info_writer_.MarkBlockWillExit();
853 frame_access_state()->ClearSPDelta();
854 frame_access_state()->SetFrameAccessToDefault();
855 break;
856 }
857 case kArchCallJSFunction: {
858 EnsureSpaceForLazyDeopt();
859 Register func = i.InputRegister(0);
860 if (FLAG_debug_code) {
861 // Check the function's context matches the context argument.
862 __ cmpp(rsi, FieldOperand(func, JSFunction::kContextOffset));
863 __ Assert(equal, kWrongFunctionContext);
864 }
865 __ Call(FieldOperand(func, JSFunction::kCodeEntryOffset));
866 frame_access_state()->ClearSPDelta();
867 RecordCallPosition(instr);
868 break;
869 }
870 case kArchTailCallJSFunctionFromJSFunction: {
871 Register func = i.InputRegister(0);
872 if (FLAG_debug_code) {
873 // Check the function's context matches the context argument.
874 __ cmpp(rsi, FieldOperand(func, JSFunction::kContextOffset));
875 __ Assert(equal, kWrongFunctionContext);
876 }
877 AssemblePopArgumentsAdaptorFrame(kJavaScriptCallArgCountRegister,
878 i.TempRegister(0), i.TempRegister(1),
879 i.TempRegister(2));
880 __ jmp(FieldOperand(func, JSFunction::kCodeEntryOffset));
881 frame_access_state()->ClearSPDelta();
882 frame_access_state()->SetFrameAccessToDefault();
883 break;
884 }
885 case kArchPrepareCallCFunction: {
886 // Frame alignment requires using FP-relative frame addressing.
887 frame_access_state()->SetFrameAccessToFP();
888 int const num_parameters = MiscField::decode(instr->opcode());
889 __ PrepareCallCFunction(num_parameters);
890 break;
891 }
892 case kArchPrepareTailCall:
893 AssemblePrepareTailCall();
894 break;
895 case kArchCallCFunction: {
896 int const num_parameters = MiscField::decode(instr->opcode());
897 if (HasImmediateInput(instr, 0)) {
898 ExternalReference ref = i.InputExternalReference(0);
899 __ CallCFunction(ref, num_parameters);
900 } else {
901 Register func = i.InputRegister(0);
902 __ CallCFunction(func, num_parameters);
903 }
904 frame_access_state()->SetFrameAccessToDefault();
905 frame_access_state()->ClearSPDelta();
906 break;
907 }
908 case kArchJmp:
909 AssembleArchJump(i.InputRpo(0));
910 break;
911 case kArchLookupSwitch:
912 AssembleArchLookupSwitch(instr);
913 break;
914 case kArchTableSwitch:
915 AssembleArchTableSwitch(instr);
916 break;
917 case kArchComment: {
918 Address comment_string = i.InputExternalReference(0).address();
919 __ RecordComment(reinterpret_cast<const char*>(comment_string));
920 break;
921 }
922 case kArchDebugBreak:
923 __ int3();
924 break;
925 case kArchNop:
926 case kArchThrowTerminator:
927 // don't emit code for nops.
928 break;
929 case kArchDeoptimize: {
930 int deopt_state_id =
931 BuildTranslation(instr, -1, 0, OutputFrameStateCombine::Ignore());
932 CodeGenResult result =
933 AssembleDeoptimizerCall(deopt_state_id, current_source_position_);
934 if (result != kSuccess) return result;
935 break;
936 }
937 case kArchRet:
938 AssembleReturn(instr->InputAt(0));
939 break;
940 case kArchStackPointer:
941 __ movq(i.OutputRegister(), rsp);
942 break;
943 case kArchFramePointer:
944 __ movq(i.OutputRegister(), rbp);
945 break;
946 case kArchParentFramePointer:
947 if (frame_access_state()->has_frame()) {
948 __ movq(i.OutputRegister(), Operand(rbp, 0));
949 } else {
950 __ movq(i.OutputRegister(), rbp);
951 }
952 break;
953 case kArchTruncateDoubleToI: {
954 auto result = i.OutputRegister();
955 auto input = i.InputDoubleRegister(0);
956 auto ool = new (zone()) OutOfLineTruncateDoubleToI(
957 this, result, input, &unwinding_info_writer_);
958 // We use Cvttsd2siq instead of Cvttsd2si due to performance reasons. The
959 // use of Cvttsd2siq requires the movl below to avoid sign extension.
960 __ Cvttsd2siq(result, input);
961 __ cmpq(result, Immediate(1));
962 __ j(overflow, ool->entry());
963 __ bind(ool->exit());
964 __ movl(result, result);
965 break;
966 }
967 case kArchStoreWithWriteBarrier: {
968 RecordWriteMode mode =
969 static_cast<RecordWriteMode>(MiscField::decode(instr->opcode()));
970 Register object = i.InputRegister(0);
971 size_t index = 0;
972 Operand operand = i.MemoryOperand(&index);
973 Register value = i.InputRegister(index);
974 Register scratch0 = i.TempRegister(0);
975 Register scratch1 = i.TempRegister(1);
976 auto ool = new (zone()) OutOfLineRecordWrite(this, object, operand, value,
977 scratch0, scratch1, mode);
978 __ movp(operand, value);
979 __ CheckPageFlag(object, scratch0,
980 MemoryChunk::kPointersFromHereAreInterestingMask,
981 not_zero, ool->entry());
982 __ bind(ool->exit());
983 break;
984 }
985 case kArchStackSlot: {
986 FrameOffset offset =
987 frame_access_state()->GetFrameOffset(i.InputInt32(0));
988 Register base;
989 if (offset.from_stack_pointer()) {
990 base = rsp;
991 } else {
992 base = rbp;
993 }
994 __ leaq(i.OutputRegister(), Operand(base, offset.offset()));
995 break;
996 }
997 case kIeee754Float64Acos:
998 ASSEMBLE_IEEE754_UNOP(acos);
999 break;
1000 case kIeee754Float64Acosh:
1001 ASSEMBLE_IEEE754_UNOP(acosh);
1002 break;
1003 case kIeee754Float64Asin:
1004 ASSEMBLE_IEEE754_UNOP(asin);
1005 break;
1006 case kIeee754Float64Asinh:
1007 ASSEMBLE_IEEE754_UNOP(asinh);
1008 break;
1009 case kIeee754Float64Atan:
1010 ASSEMBLE_IEEE754_UNOP(atan);
1011 break;
1012 case kIeee754Float64Atanh:
1013 ASSEMBLE_IEEE754_UNOP(atanh);
1014 break;
1015 case kIeee754Float64Atan2:
1016 ASSEMBLE_IEEE754_BINOP(atan2);
1017 break;
1018 case kIeee754Float64Cbrt:
1019 ASSEMBLE_IEEE754_UNOP(cbrt);
1020 break;
1021 case kIeee754Float64Cos:
1022 ASSEMBLE_IEEE754_UNOP(cos);
1023 break;
1024 case kIeee754Float64Cosh:
1025 ASSEMBLE_IEEE754_UNOP(cosh);
1026 break;
1027 case kIeee754Float64Exp:
1028 ASSEMBLE_IEEE754_UNOP(exp);
1029 break;
1030 case kIeee754Float64Expm1:
1031 ASSEMBLE_IEEE754_UNOP(expm1);
1032 break;
1033 case kIeee754Float64Log:
1034 ASSEMBLE_IEEE754_UNOP(log);
1035 break;
1036 case kIeee754Float64Log1p:
1037 ASSEMBLE_IEEE754_UNOP(log1p);
1038 break;
1039 case kIeee754Float64Log2:
1040 ASSEMBLE_IEEE754_UNOP(log2);
1041 break;
1042 case kIeee754Float64Log10:
1043 ASSEMBLE_IEEE754_UNOP(log10);
1044 break;
1045 case kIeee754Float64Pow: {
1046 // TODO(bmeurer): Improve integration of the stub.
1047 __ Movsd(xmm2, xmm0);
1048 MathPowStub stub(isolate(), MathPowStub::DOUBLE);
1049 __ CallStub(&stub);
1050 __ Movsd(xmm0, xmm3);
1051 break;
1052 }
1053 case kIeee754Float64Sin:
1054 ASSEMBLE_IEEE754_UNOP(sin);
1055 break;
1056 case kIeee754Float64Sinh:
1057 ASSEMBLE_IEEE754_UNOP(sinh);
1058 break;
1059 case kIeee754Float64Tan:
1060 ASSEMBLE_IEEE754_UNOP(tan);
1061 break;
1062 case kIeee754Float64Tanh:
1063 ASSEMBLE_IEEE754_UNOP(tanh);
1064 break;
1065 case kX64Add32:
1066 ASSEMBLE_BINOP(addl);
1067 break;
1068 case kX64Add:
1069 ASSEMBLE_BINOP(addq);
1070 break;
1071 case kX64Sub32:
1072 ASSEMBLE_BINOP(subl);
1073 break;
1074 case kX64Sub:
1075 ASSEMBLE_BINOP(subq);
1076 break;
1077 case kX64And32:
1078 ASSEMBLE_BINOP(andl);
1079 break;
1080 case kX64And:
1081 ASSEMBLE_BINOP(andq);
1082 break;
1083 case kX64Cmp8:
1084 ASSEMBLE_COMPARE(cmpb);
1085 break;
1086 case kX64Cmp16:
1087 ASSEMBLE_COMPARE(cmpw);
1088 break;
1089 case kX64Cmp32:
1090 ASSEMBLE_COMPARE(cmpl);
1091 break;
1092 case kX64Cmp:
1093 ASSEMBLE_COMPARE(cmpq);
1094 break;
1095 case kX64Test8:
1096 ASSEMBLE_COMPARE(testb);
1097 break;
1098 case kX64Test16:
1099 ASSEMBLE_COMPARE(testw);
1100 break;
1101 case kX64Test32:
1102 ASSEMBLE_COMPARE(testl);
1103 break;
1104 case kX64Test:
1105 ASSEMBLE_COMPARE(testq);
1106 break;
1107 case kX64Imul32:
1108 ASSEMBLE_MULT(imull);
1109 break;
1110 case kX64Imul:
1111 ASSEMBLE_MULT(imulq);
1112 break;
1113 case kX64ImulHigh32:
1114 if (instr->InputAt(1)->IsRegister()) {
1115 __ imull(i.InputRegister(1));
1116 } else {
1117 __ imull(i.InputOperand(1));
1118 }
1119 break;
1120 case kX64UmulHigh32:
1121 if (instr->InputAt(1)->IsRegister()) {
1122 __ mull(i.InputRegister(1));
1123 } else {
1124 __ mull(i.InputOperand(1));
1125 }
1126 break;
1127 case kX64Idiv32:
1128 __ cdq();
1129 __ idivl(i.InputRegister(1));
1130 break;
1131 case kX64Idiv:
1132 __ cqo();
1133 __ idivq(i.InputRegister(1));
1134 break;
1135 case kX64Udiv32:
1136 __ xorl(rdx, rdx);
1137 __ divl(i.InputRegister(1));
1138 break;
1139 case kX64Udiv:
1140 __ xorq(rdx, rdx);
1141 __ divq(i.InputRegister(1));
1142 break;
1143 case kX64Not:
1144 ASSEMBLE_UNOP(notq);
1145 break;
1146 case kX64Not32:
1147 ASSEMBLE_UNOP(notl);
1148 break;
1149 case kX64Neg:
1150 ASSEMBLE_UNOP(negq);
1151 break;
1152 case kX64Neg32:
1153 ASSEMBLE_UNOP(negl);
1154 break;
1155 case kX64Or32:
1156 ASSEMBLE_BINOP(orl);
1157 break;
1158 case kX64Or:
1159 ASSEMBLE_BINOP(orq);
1160 break;
1161 case kX64Xor32:
1162 ASSEMBLE_BINOP(xorl);
1163 break;
1164 case kX64Xor:
1165 ASSEMBLE_BINOP(xorq);
1166 break;
1167 case kX64Shl32:
1168 ASSEMBLE_SHIFT(shll, 5);
1169 break;
1170 case kX64Shl:
1171 ASSEMBLE_SHIFT(shlq, 6);
1172 break;
1173 case kX64Shr32:
1174 ASSEMBLE_SHIFT(shrl, 5);
1175 break;
1176 case kX64Shr:
1177 ASSEMBLE_SHIFT(shrq, 6);
1178 break;
1179 case kX64Sar32:
1180 ASSEMBLE_SHIFT(sarl, 5);
1181 break;
1182 case kX64Sar:
1183 ASSEMBLE_SHIFT(sarq, 6);
1184 break;
1185 case kX64Ror32:
1186 ASSEMBLE_SHIFT(rorl, 5);
1187 break;
1188 case kX64Ror:
1189 ASSEMBLE_SHIFT(rorq, 6);
1190 break;
1191 case kX64Lzcnt:
1192 if (instr->InputAt(0)->IsRegister()) {
1193 __ Lzcntq(i.OutputRegister(), i.InputRegister(0));
1194 } else {
1195 __ Lzcntq(i.OutputRegister(), i.InputOperand(0));
1196 }
1197 break;
1198 case kX64Lzcnt32:
1199 if (instr->InputAt(0)->IsRegister()) {
1200 __ Lzcntl(i.OutputRegister(), i.InputRegister(0));
1201 } else {
1202 __ Lzcntl(i.OutputRegister(), i.InputOperand(0));
1203 }
1204 break;
1205 case kX64Tzcnt:
1206 if (instr->InputAt(0)->IsRegister()) {
1207 __ Tzcntq(i.OutputRegister(), i.InputRegister(0));
1208 } else {
1209 __ Tzcntq(i.OutputRegister(), i.InputOperand(0));
1210 }
1211 break;
1212 case kX64Tzcnt32:
1213 if (instr->InputAt(0)->IsRegister()) {
1214 __ Tzcntl(i.OutputRegister(), i.InputRegister(0));
1215 } else {
1216 __ Tzcntl(i.OutputRegister(), i.InputOperand(0));
1217 }
1218 break;
1219 case kX64Popcnt:
1220 if (instr->InputAt(0)->IsRegister()) {
1221 __ Popcntq(i.OutputRegister(), i.InputRegister(0));
1222 } else {
1223 __ Popcntq(i.OutputRegister(), i.InputOperand(0));
1224 }
1225 break;
1226 case kX64Popcnt32:
1227 if (instr->InputAt(0)->IsRegister()) {
1228 __ Popcntl(i.OutputRegister(), i.InputRegister(0));
1229 } else {
1230 __ Popcntl(i.OutputRegister(), i.InputOperand(0));
1231 }
1232 break;
1233 case kSSEFloat32Cmp:
1234 ASSEMBLE_SSE_BINOP(Ucomiss);
1235 break;
1236 case kSSEFloat32Add:
1237 ASSEMBLE_SSE_BINOP(addss);
1238 break;
1239 case kSSEFloat32Sub:
1240 ASSEMBLE_SSE_BINOP(subss);
1241 break;
1242 case kSSEFloat32Mul:
1243 ASSEMBLE_SSE_BINOP(mulss);
1244 break;
1245 case kSSEFloat32Div:
1246 ASSEMBLE_SSE_BINOP(divss);
1247 // Don't delete this mov. It may improve performance on some CPUs,
1248 // when there is a (v)mulss depending on the result.
1249 __ movaps(i.OutputDoubleRegister(), i.OutputDoubleRegister());
1250 break;
1251 case kSSEFloat32Abs: {
1252 // TODO(bmeurer): Use RIP relative 128-bit constants.
1253 __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
1254 __ psrlq(kScratchDoubleReg, 33);
1255 __ andps(i.OutputDoubleRegister(), kScratchDoubleReg);
1256 break;
1257 }
1258 case kSSEFloat32Neg: {
1259 // TODO(bmeurer): Use RIP relative 128-bit constants.
1260 __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
1261 __ psllq(kScratchDoubleReg, 31);
1262 __ xorps(i.OutputDoubleRegister(), kScratchDoubleReg);
1263 break;
1264 }
1265 case kSSEFloat32Sqrt:
1266 ASSEMBLE_SSE_UNOP(sqrtss);
1267 break;
1268 case kSSEFloat32ToFloat64:
1269 ASSEMBLE_SSE_UNOP(Cvtss2sd);
1270 break;
1271 case kSSEFloat32Round: {
1272 CpuFeatureScope sse_scope(masm(), SSE4_1);
1273 RoundingMode const mode =
1274 static_cast<RoundingMode>(MiscField::decode(instr->opcode()));
1275 __ Roundss(i.OutputDoubleRegister(), i.InputDoubleRegister(0), mode);
1276 break;
1277 }
1278 case kSSEFloat32ToInt32:
1279 if (instr->InputAt(0)->IsFPRegister()) {
1280 __ Cvttss2si(i.OutputRegister(), i.InputDoubleRegister(0));
1281 } else {
1282 __ Cvttss2si(i.OutputRegister(), i.InputOperand(0));
1283 }
1284 break;
1285 case kSSEFloat32ToUint32: {
1286 if (instr->InputAt(0)->IsFPRegister()) {
1287 __ Cvttss2siq(i.OutputRegister(), i.InputDoubleRegister(0));
1288 } else {
1289 __ Cvttss2siq(i.OutputRegister(), i.InputOperand(0));
1290 }
1291 break;
1292 }
1293 case kSSEFloat64Cmp:
1294 ASSEMBLE_SSE_BINOP(Ucomisd);
1295 break;
1296 case kSSEFloat64Add:
1297 ASSEMBLE_SSE_BINOP(addsd);
1298 break;
1299 case kSSEFloat64Sub:
1300 ASSEMBLE_SSE_BINOP(subsd);
1301 break;
1302 case kSSEFloat64Mul:
1303 ASSEMBLE_SSE_BINOP(mulsd);
1304 break;
1305 case kSSEFloat64Div:
1306 ASSEMBLE_SSE_BINOP(divsd);
1307 // Don't delete this mov. It may improve performance on some CPUs,
1308 // when there is a (v)mulsd depending on the result.
1309 __ Movapd(i.OutputDoubleRegister(), i.OutputDoubleRegister());
1310 break;
1311 case kSSEFloat64Mod: {
1312 __ subq(rsp, Immediate(kDoubleSize));
1313 unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
1314 kDoubleSize);
1315 // Move values to st(0) and st(1).
1316 __ Movsd(Operand(rsp, 0), i.InputDoubleRegister(1));
1317 __ fld_d(Operand(rsp, 0));
1318 __ Movsd(Operand(rsp, 0), i.InputDoubleRegister(0));
1319 __ fld_d(Operand(rsp, 0));
1320 // Loop while fprem isn't done.
1321 Label mod_loop;
1322 __ bind(&mod_loop);
1323 // This instructions traps on all kinds inputs, but we are assuming the
1324 // floating point control word is set to ignore them all.
1325 __ fprem();
1326 // The following 2 instruction implicitly use rax.
1327 __ fnstsw_ax();
1328 if (CpuFeatures::IsSupported(SAHF)) {
1329 CpuFeatureScope sahf_scope(masm(), SAHF);
1330 __ sahf();
1331 } else {
1332 __ shrl(rax, Immediate(8));
1333 __ andl(rax, Immediate(0xFF));
1334 __ pushq(rax);
1335 unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
1336 kPointerSize);
1337 __ popfq();
1338 unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
1339 -kPointerSize);
1340 }
1341 __ j(parity_even, &mod_loop);
1342 // Move output to stack and clean up.
1343 __ fstp(1);
1344 __ fstp_d(Operand(rsp, 0));
1345 __ Movsd(i.OutputDoubleRegister(), Operand(rsp, 0));
1346 __ addq(rsp, Immediate(kDoubleSize));
1347 unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
1348 -kDoubleSize);
1349 break;
1350 }
1351 case kSSEFloat32Max: {
1352 Label compare_nan, compare_swap, done_compare;
1353 if (instr->InputAt(1)->IsFPRegister()) {
1354 __ Ucomiss(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1355 } else {
1356 __ Ucomiss(i.InputDoubleRegister(0), i.InputOperand(1));
1357 }
1358 auto ool =
1359 new (zone()) OutOfLineLoadFloat32NaN(this, i.OutputDoubleRegister());
1360 __ j(parity_even, ool->entry());
1361 __ j(above, &done_compare, Label::kNear);
1362 __ j(below, &compare_swap, Label::kNear);
1363 __ Movmskps(kScratchRegister, i.InputDoubleRegister(0));
1364 __ testl(kScratchRegister, Immediate(1));
1365 __ j(zero, &done_compare, Label::kNear);
1366 __ bind(&compare_swap);
1367 if (instr->InputAt(1)->IsFPRegister()) {
1368 __ Movss(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1369 } else {
1370 __ Movss(i.InputDoubleRegister(0), i.InputOperand(1));
1371 }
1372 __ bind(&done_compare);
1373 __ bind(ool->exit());
1374 break;
1375 }
1376 case kSSEFloat32Min: {
1377 Label compare_swap, done_compare;
1378 if (instr->InputAt(1)->IsFPRegister()) {
1379 __ Ucomiss(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1380 } else {
1381 __ Ucomiss(i.InputDoubleRegister(0), i.InputOperand(1));
1382 }
1383 auto ool =
1384 new (zone()) OutOfLineLoadFloat32NaN(this, i.OutputDoubleRegister());
1385 __ j(parity_even, ool->entry());
1386 __ j(below, &done_compare, Label::kNear);
1387 __ j(above, &compare_swap, Label::kNear);
1388 if (instr->InputAt(1)->IsFPRegister()) {
1389 __ Movmskps(kScratchRegister, i.InputDoubleRegister(1));
1390 } else {
1391 __ Movss(kScratchDoubleReg, i.InputOperand(1));
1392 __ Movmskps(kScratchRegister, kScratchDoubleReg);
1393 }
1394 __ testl(kScratchRegister, Immediate(1));
1395 __ j(zero, &done_compare, Label::kNear);
1396 __ bind(&compare_swap);
1397 if (instr->InputAt(1)->IsFPRegister()) {
1398 __ Movss(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1399 } else {
1400 __ Movss(i.InputDoubleRegister(0), i.InputOperand(1));
1401 }
1402 __ bind(&done_compare);
1403 __ bind(ool->exit());
1404 break;
1405 }
1406 case kSSEFloat64Max: {
1407 Label compare_nan, compare_swap, done_compare;
1408 if (instr->InputAt(1)->IsFPRegister()) {
1409 __ Ucomisd(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1410 } else {
1411 __ Ucomisd(i.InputDoubleRegister(0), i.InputOperand(1));
1412 }
1413 auto ool =
1414 new (zone()) OutOfLineLoadFloat64NaN(this, i.OutputDoubleRegister());
1415 __ j(parity_even, ool->entry());
1416 __ j(above, &done_compare, Label::kNear);
1417 __ j(below, &compare_swap, Label::kNear);
1418 __ Movmskpd(kScratchRegister, i.InputDoubleRegister(0));
1419 __ testl(kScratchRegister, Immediate(1));
1420 __ j(zero, &done_compare, Label::kNear);
1421 __ bind(&compare_swap);
1422 if (instr->InputAt(1)->IsFPRegister()) {
1423 __ Movsd(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1424 } else {
1425 __ Movsd(i.InputDoubleRegister(0), i.InputOperand(1));
1426 }
1427 __ bind(&done_compare);
1428 __ bind(ool->exit());
1429 break;
1430 }
1431 case kSSEFloat64Min: {
1432 Label compare_swap, done_compare;
1433 if (instr->InputAt(1)->IsFPRegister()) {
1434 __ Ucomisd(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1435 } else {
1436 __ Ucomisd(i.InputDoubleRegister(0), i.InputOperand(1));
1437 }
1438 auto ool =
1439 new (zone()) OutOfLineLoadFloat64NaN(this, i.OutputDoubleRegister());
1440 __ j(parity_even, ool->entry());
1441 __ j(below, &done_compare, Label::kNear);
1442 __ j(above, &compare_swap, Label::kNear);
1443 if (instr->InputAt(1)->IsFPRegister()) {
1444 __ Movmskpd(kScratchRegister, i.InputDoubleRegister(1));
1445 } else {
1446 __ Movsd(kScratchDoubleReg, i.InputOperand(1));
1447 __ Movmskpd(kScratchRegister, kScratchDoubleReg);
1448 }
1449 __ testl(kScratchRegister, Immediate(1));
1450 __ j(zero, &done_compare, Label::kNear);
1451 __ bind(&compare_swap);
1452 if (instr->InputAt(1)->IsFPRegister()) {
1453 __ Movsd(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1454 } else {
1455 __ Movsd(i.InputDoubleRegister(0), i.InputOperand(1));
1456 }
1457 __ bind(&done_compare);
1458 __ bind(ool->exit());
1459 break;
1460 }
1461 case kSSEFloat64Abs: {
1462 // TODO(bmeurer): Use RIP relative 128-bit constants.
1463 __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
1464 __ psrlq(kScratchDoubleReg, 1);
1465 __ andpd(i.OutputDoubleRegister(), kScratchDoubleReg);
1466 break;
1467 }
1468 case kSSEFloat64Neg: {
1469 // TODO(bmeurer): Use RIP relative 128-bit constants.
1470 __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
1471 __ psllq(kScratchDoubleReg, 63);
1472 __ xorpd(i.OutputDoubleRegister(), kScratchDoubleReg);
1473 break;
1474 }
1475 case kSSEFloat64Sqrt:
1476 ASSEMBLE_SSE_UNOP(Sqrtsd);
1477 break;
1478 case kSSEFloat64Round: {
1479 CpuFeatureScope sse_scope(masm(), SSE4_1);
1480 RoundingMode const mode =
1481 static_cast<RoundingMode>(MiscField::decode(instr->opcode()));
1482 __ Roundsd(i.OutputDoubleRegister(), i.InputDoubleRegister(0), mode);
1483 break;
1484 }
1485 case kSSEFloat64ToFloat32:
1486 ASSEMBLE_SSE_UNOP(Cvtsd2ss);
1487 break;
1488 case kSSEFloat64ToInt32:
1489 if (instr->InputAt(0)->IsFPRegister()) {
1490 __ Cvttsd2si(i.OutputRegister(), i.InputDoubleRegister(0));
1491 } else {
1492 __ Cvttsd2si(i.OutputRegister(), i.InputOperand(0));
1493 }
1494 break;
1495 case kSSEFloat64ToUint32: {
1496 if (instr->InputAt(0)->IsFPRegister()) {
1497 __ Cvttsd2siq(i.OutputRegister(), i.InputDoubleRegister(0));
1498 } else {
1499 __ Cvttsd2siq(i.OutputRegister(), i.InputOperand(0));
1500 }
1501 if (MiscField::decode(instr->opcode())) {
1502 __ AssertZeroExtended(i.OutputRegister());
1503 }
1504 break;
1505 }
1506 case kSSEFloat32ToInt64:
1507 if (instr->InputAt(0)->IsFPRegister()) {
1508 __ Cvttss2siq(i.OutputRegister(), i.InputDoubleRegister(0));
1509 } else {
1510 __ Cvttss2siq(i.OutputRegister(), i.InputOperand(0));
1511 }
1512 if (instr->OutputCount() > 1) {
1513 __ Set(i.OutputRegister(1), 1);
1514 Label done;
1515 Label fail;
1516 __ Move(kScratchDoubleReg, static_cast<float>(INT64_MIN));
1517 if (instr->InputAt(0)->IsFPRegister()) {
1518 __ Ucomiss(kScratchDoubleReg, i.InputDoubleRegister(0));
1519 } else {
1520 __ Ucomiss(kScratchDoubleReg, i.InputOperand(0));
1521 }
1522 // If the input is NaN, then the conversion fails.
1523 __ j(parity_even, &fail);
1524 // If the input is INT64_MIN, then the conversion succeeds.
1525 __ j(equal, &done);
1526 __ cmpq(i.OutputRegister(0), Immediate(1));
1527 // If the conversion results in INT64_MIN, but the input was not
1528 // INT64_MIN, then the conversion fails.
1529 __ j(no_overflow, &done);
1530 __ bind(&fail);
1531 __ Set(i.OutputRegister(1), 0);
1532 __ bind(&done);
1533 }
1534 break;
1535 case kSSEFloat64ToInt64:
1536 if (instr->InputAt(0)->IsFPRegister()) {
1537 __ Cvttsd2siq(i.OutputRegister(0), i.InputDoubleRegister(0));
1538 } else {
1539 __ Cvttsd2siq(i.OutputRegister(0), i.InputOperand(0));
1540 }
1541 if (instr->OutputCount() > 1) {
1542 __ Set(i.OutputRegister(1), 1);
1543 Label done;
1544 Label fail;
1545 __ Move(kScratchDoubleReg, static_cast<double>(INT64_MIN));
1546 if (instr->InputAt(0)->IsFPRegister()) {
1547 __ Ucomisd(kScratchDoubleReg, i.InputDoubleRegister(0));
1548 } else {
1549 __ Ucomisd(kScratchDoubleReg, i.InputOperand(0));
1550 }
1551 // If the input is NaN, then the conversion fails.
1552 __ j(parity_even, &fail);
1553 // If the input is INT64_MIN, then the conversion succeeds.
1554 __ j(equal, &done);
1555 __ cmpq(i.OutputRegister(0), Immediate(1));
1556 // If the conversion results in INT64_MIN, but the input was not
1557 // INT64_MIN, then the conversion fails.
1558 __ j(no_overflow, &done);
1559 __ bind(&fail);
1560 __ Set(i.OutputRegister(1), 0);
1561 __ bind(&done);
1562 }
1563 break;
1564 case kSSEFloat32ToUint64: {
1565 Label done;
1566 Label success;
1567 if (instr->OutputCount() > 1) {
1568 __ Set(i.OutputRegister(1), 0);
1569 }
1570 // There does not exist a Float32ToUint64 instruction, so we have to use
1571 // the Float32ToInt64 instruction.
1572 if (instr->InputAt(0)->IsFPRegister()) {
1573 __ Cvttss2siq(i.OutputRegister(), i.InputDoubleRegister(0));
1574 } else {
1575 __ Cvttss2siq(i.OutputRegister(), i.InputOperand(0));
1576 }
1577 // Check if the result of the Float32ToInt64 conversion is positive, we
1578 // are already done.
1579 __ testq(i.OutputRegister(), i.OutputRegister());
1580 __ j(positive, &success);
1581 // The result of the first conversion was negative, which means that the
1582 // input value was not within the positive int64 range. We subtract 2^64
1583 // and convert it again to see if it is within the uint64 range.
1584 __ Move(kScratchDoubleReg, -9223372036854775808.0f);
1585 if (instr->InputAt(0)->IsFPRegister()) {
1586 __ addss(kScratchDoubleReg, i.InputDoubleRegister(0));
1587 } else {
1588 __ addss(kScratchDoubleReg, i.InputOperand(0));
1589 }
1590 __ Cvttss2siq(i.OutputRegister(), kScratchDoubleReg);
1591 __ testq(i.OutputRegister(), i.OutputRegister());
1592 // The only possible negative value here is 0x80000000000000000, which is
1593 // used on x64 to indicate an integer overflow.
1594 __ j(negative, &done);
1595 // The input value is within uint64 range and the second conversion worked
1596 // successfully, but we still have to undo the subtraction we did
1597 // earlier.
1598 __ Set(kScratchRegister, 0x8000000000000000);
1599 __ orq(i.OutputRegister(), kScratchRegister);
1600 __ bind(&success);
1601 if (instr->OutputCount() > 1) {
1602 __ Set(i.OutputRegister(1), 1);
1603 }
1604 __ bind(&done);
1605 break;
1606 }
1607 case kSSEFloat64ToUint64: {
1608 Label done;
1609 Label success;
1610 if (instr->OutputCount() > 1) {
1611 __ Set(i.OutputRegister(1), 0);
1612 }
1613 // There does not exist a Float64ToUint64 instruction, so we have to use
1614 // the Float64ToInt64 instruction.
1615 if (instr->InputAt(0)->IsFPRegister()) {
1616 __ Cvttsd2siq(i.OutputRegister(), i.InputDoubleRegister(0));
1617 } else {
1618 __ Cvttsd2siq(i.OutputRegister(), i.InputOperand(0));
1619 }
1620 // Check if the result of the Float64ToInt64 conversion is positive, we
1621 // are already done.
1622 __ testq(i.OutputRegister(), i.OutputRegister());
1623 __ j(positive, &success);
1624 // The result of the first conversion was negative, which means that the
1625 // input value was not within the positive int64 range. We subtract 2^64
1626 // and convert it again to see if it is within the uint64 range.
1627 __ Move(kScratchDoubleReg, -9223372036854775808.0);
1628 if (instr->InputAt(0)->IsFPRegister()) {
1629 __ addsd(kScratchDoubleReg, i.InputDoubleRegister(0));
1630 } else {
1631 __ addsd(kScratchDoubleReg, i.InputOperand(0));
1632 }
1633 __ Cvttsd2siq(i.OutputRegister(), kScratchDoubleReg);
1634 __ testq(i.OutputRegister(), i.OutputRegister());
1635 // The only possible negative value here is 0x80000000000000000, which is
1636 // used on x64 to indicate an integer overflow.
1637 __ j(negative, &done);
1638 // The input value is within uint64 range and the second conversion worked
1639 // successfully, but we still have to undo the subtraction we did
1640 // earlier.
1641 __ Set(kScratchRegister, 0x8000000000000000);
1642 __ orq(i.OutputRegister(), kScratchRegister);
1643 __ bind(&success);
1644 if (instr->OutputCount() > 1) {
1645 __ Set(i.OutputRegister(1), 1);
1646 }
1647 __ bind(&done);
1648 break;
1649 }
1650 case kSSEInt32ToFloat64:
1651 if (instr->InputAt(0)->IsRegister()) {
1652 __ Cvtlsi2sd(i.OutputDoubleRegister(), i.InputRegister(0));
1653 } else {
1654 __ Cvtlsi2sd(i.OutputDoubleRegister(), i.InputOperand(0));
1655 }
1656 break;
1657 case kSSEInt32ToFloat32:
1658 if (instr->InputAt(0)->IsRegister()) {
1659 __ Cvtlsi2ss(i.OutputDoubleRegister(), i.InputRegister(0));
1660 } else {
1661 __ Cvtlsi2ss(i.OutputDoubleRegister(), i.InputOperand(0));
1662 }
1663 break;
1664 case kSSEInt64ToFloat32:
1665 if (instr->InputAt(0)->IsRegister()) {
1666 __ Cvtqsi2ss(i.OutputDoubleRegister(), i.InputRegister(0));
1667 } else {
1668 __ Cvtqsi2ss(i.OutputDoubleRegister(), i.InputOperand(0));
1669 }
1670 break;
1671 case kSSEInt64ToFloat64:
1672 if (instr->InputAt(0)->IsRegister()) {
1673 __ Cvtqsi2sd(i.OutputDoubleRegister(), i.InputRegister(0));
1674 } else {
1675 __ Cvtqsi2sd(i.OutputDoubleRegister(), i.InputOperand(0));
1676 }
1677 break;
1678 case kSSEUint64ToFloat32:
1679 if (instr->InputAt(0)->IsRegister()) {
1680 __ movq(kScratchRegister, i.InputRegister(0));
1681 } else {
1682 __ movq(kScratchRegister, i.InputOperand(0));
1683 }
1684 __ Cvtqui2ss(i.OutputDoubleRegister(), kScratchRegister,
1685 i.TempRegister(0));
1686 break;
1687 case kSSEUint64ToFloat64:
1688 if (instr->InputAt(0)->IsRegister()) {
1689 __ movq(kScratchRegister, i.InputRegister(0));
1690 } else {
1691 __ movq(kScratchRegister, i.InputOperand(0));
1692 }
1693 __ Cvtqui2sd(i.OutputDoubleRegister(), kScratchRegister,
1694 i.TempRegister(0));
1695 break;
1696 case kSSEUint32ToFloat64:
1697 if (instr->InputAt(0)->IsRegister()) {
1698 __ movl(kScratchRegister, i.InputRegister(0));
1699 } else {
1700 __ movl(kScratchRegister, i.InputOperand(0));
1701 }
1702 __ Cvtqsi2sd(i.OutputDoubleRegister(), kScratchRegister);
1703 break;
1704 case kSSEUint32ToFloat32:
1705 if (instr->InputAt(0)->IsRegister()) {
1706 __ movl(kScratchRegister, i.InputRegister(0));
1707 } else {
1708 __ movl(kScratchRegister, i.InputOperand(0));
1709 }
1710 __ Cvtqsi2ss(i.OutputDoubleRegister(), kScratchRegister);
1711 break;
1712 case kSSEFloat64ExtractLowWord32:
1713 if (instr->InputAt(0)->IsFPStackSlot()) {
1714 __ movl(i.OutputRegister(), i.InputOperand(0));
1715 } else {
1716 __ Movd(i.OutputRegister(), i.InputDoubleRegister(0));
1717 }
1718 break;
1719 case kSSEFloat64ExtractHighWord32:
1720 if (instr->InputAt(0)->IsFPStackSlot()) {
1721 __ movl(i.OutputRegister(), i.InputOperand(0, kDoubleSize / 2));
1722 } else {
1723 __ Pextrd(i.OutputRegister(), i.InputDoubleRegister(0), 1);
1724 }
1725 break;
1726 case kSSEFloat64InsertLowWord32:
1727 if (instr->InputAt(1)->IsRegister()) {
1728 __ Pinsrd(i.OutputDoubleRegister(), i.InputRegister(1), 0);
1729 } else {
1730 __ Pinsrd(i.OutputDoubleRegister(), i.InputOperand(1), 0);
1731 }
1732 break;
1733 case kSSEFloat64InsertHighWord32:
1734 if (instr->InputAt(1)->IsRegister()) {
1735 __ Pinsrd(i.OutputDoubleRegister(), i.InputRegister(1), 1);
1736 } else {
1737 __ Pinsrd(i.OutputDoubleRegister(), i.InputOperand(1), 1);
1738 }
1739 break;
1740 case kSSEFloat64LoadLowWord32:
1741 if (instr->InputAt(0)->IsRegister()) {
1742 __ Movd(i.OutputDoubleRegister(), i.InputRegister(0));
1743 } else {
1744 __ Movd(i.OutputDoubleRegister(), i.InputOperand(0));
1745 }
1746 break;
1747 case kAVXFloat32Cmp: {
1748 CpuFeatureScope avx_scope(masm(), AVX);
1749 if (instr->InputAt(1)->IsFPRegister()) {
1750 __ vucomiss(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1751 } else {
1752 __ vucomiss(i.InputDoubleRegister(0), i.InputOperand(1));
1753 }
1754 break;
1755 }
1756 case kAVXFloat32Add:
1757 ASSEMBLE_AVX_BINOP(vaddss);
1758 break;
1759 case kAVXFloat32Sub:
1760 ASSEMBLE_AVX_BINOP(vsubss);
1761 break;
1762 case kAVXFloat32Mul:
1763 ASSEMBLE_AVX_BINOP(vmulss);
1764 break;
1765 case kAVXFloat32Div:
1766 ASSEMBLE_AVX_BINOP(vdivss);
1767 // Don't delete this mov. It may improve performance on some CPUs,
1768 // when there is a (v)mulss depending on the result.
1769 __ Movaps(i.OutputDoubleRegister(), i.OutputDoubleRegister());
1770 break;
1771 case kAVXFloat64Cmp: {
1772 CpuFeatureScope avx_scope(masm(), AVX);
1773 if (instr->InputAt(1)->IsFPRegister()) {
1774 __ vucomisd(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1775 } else {
1776 __ vucomisd(i.InputDoubleRegister(0), i.InputOperand(1));
1777 }
1778 break;
1779 }
1780 case kAVXFloat64Add:
1781 ASSEMBLE_AVX_BINOP(vaddsd);
1782 break;
1783 case kAVXFloat64Sub:
1784 ASSEMBLE_AVX_BINOP(vsubsd);
1785 break;
1786 case kAVXFloat64Mul:
1787 ASSEMBLE_AVX_BINOP(vmulsd);
1788 break;
1789 case kAVXFloat64Div:
1790 ASSEMBLE_AVX_BINOP(vdivsd);
1791 // Don't delete this mov. It may improve performance on some CPUs,
1792 // when there is a (v)mulsd depending on the result.
1793 __ Movapd(i.OutputDoubleRegister(), i.OutputDoubleRegister());
1794 break;
1795 case kAVXFloat32Abs: {
1796 // TODO(bmeurer): Use RIP relative 128-bit constants.
1797 CpuFeatureScope avx_scope(masm(), AVX);
1798 __ vpcmpeqd(kScratchDoubleReg, kScratchDoubleReg, kScratchDoubleReg);
1799 __ vpsrlq(kScratchDoubleReg, kScratchDoubleReg, 33);
1800 if (instr->InputAt(0)->IsFPRegister()) {
1801 __ vandps(i.OutputDoubleRegister(), kScratchDoubleReg,
1802 i.InputDoubleRegister(0));
1803 } else {
1804 __ vandps(i.OutputDoubleRegister(), kScratchDoubleReg,
1805 i.InputOperand(0));
1806 }
1807 break;
1808 }
1809 case kAVXFloat32Neg: {
1810 // TODO(bmeurer): Use RIP relative 128-bit constants.
1811 CpuFeatureScope avx_scope(masm(), AVX);
1812 __ vpcmpeqd(kScratchDoubleReg, kScratchDoubleReg, kScratchDoubleReg);
1813 __ vpsllq(kScratchDoubleReg, kScratchDoubleReg, 31);
1814 if (instr->InputAt(0)->IsFPRegister()) {
1815 __ vxorps(i.OutputDoubleRegister(), kScratchDoubleReg,
1816 i.InputDoubleRegister(0));
1817 } else {
1818 __ vxorps(i.OutputDoubleRegister(), kScratchDoubleReg,
1819 i.InputOperand(0));
1820 }
1821 break;
1822 }
1823 case kAVXFloat64Abs: {
1824 // TODO(bmeurer): Use RIP relative 128-bit constants.
1825 CpuFeatureScope avx_scope(masm(), AVX);
1826 __ vpcmpeqd(kScratchDoubleReg, kScratchDoubleReg, kScratchDoubleReg);
1827 __ vpsrlq(kScratchDoubleReg, kScratchDoubleReg, 1);
1828 if (instr->InputAt(0)->IsFPRegister()) {
1829 __ vandpd(i.OutputDoubleRegister(), kScratchDoubleReg,
1830 i.InputDoubleRegister(0));
1831 } else {
1832 __ vandpd(i.OutputDoubleRegister(), kScratchDoubleReg,
1833 i.InputOperand(0));
1834 }
1835 break;
1836 }
1837 case kAVXFloat64Neg: {
1838 // TODO(bmeurer): Use RIP relative 128-bit constants.
1839 CpuFeatureScope avx_scope(masm(), AVX);
1840 __ vpcmpeqd(kScratchDoubleReg, kScratchDoubleReg, kScratchDoubleReg);
1841 __ vpsllq(kScratchDoubleReg, kScratchDoubleReg, 63);
1842 if (instr->InputAt(0)->IsFPRegister()) {
1843 __ vxorpd(i.OutputDoubleRegister(), kScratchDoubleReg,
1844 i.InputDoubleRegister(0));
1845 } else {
1846 __ vxorpd(i.OutputDoubleRegister(), kScratchDoubleReg,
1847 i.InputOperand(0));
1848 }
1849 break;
1850 }
1851 case kSSEFloat64SilenceNaN:
1852 __ Xorpd(kScratchDoubleReg, kScratchDoubleReg);
1853 __ Subsd(i.InputDoubleRegister(0), kScratchDoubleReg);
1854 break;
1855 case kX64Movsxbl:
1856 EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i,
1857 __ pc_offset(), instr);
1858 ASSEMBLE_MOVX(movsxbl);
1859 __ AssertZeroExtended(i.OutputRegister());
1860 break;
1861 case kX64Movzxbl:
1862 EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i,
1863 __ pc_offset(), instr);
1864 ASSEMBLE_MOVX(movzxbl);
1865 __ AssertZeroExtended(i.OutputRegister());
1866 break;
1867 case kX64Movsxbq:
1868 EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i,
1869 __ pc_offset(), instr);
1870 ASSEMBLE_MOVX(movsxbq);
1871 break;
1872 case kX64Movzxbq:
1873 EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i,
1874 __ pc_offset(), instr);
1875 ASSEMBLE_MOVX(movzxbq);
1876 __ AssertZeroExtended(i.OutputRegister());
1877 break;
1878 case kX64Movb: {
1879 EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i,
1880 __ pc_offset(), instr);
1881 size_t index = 0;
1882 Operand operand = i.MemoryOperand(&index);
1883 if (HasImmediateInput(instr, index)) {
1884 __ movb(operand, Immediate(i.InputInt8(index)));
1885 } else {
1886 __ movb(operand, i.InputRegister(index));
1887 }
1888 break;
1889 }
1890 case kX64Movsxwl:
1891 EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i,
1892 __ pc_offset(), instr);
1893 ASSEMBLE_MOVX(movsxwl);
1894 __ AssertZeroExtended(i.OutputRegister());
1895 break;
1896 case kX64Movzxwl:
1897 EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i,
1898 __ pc_offset(), instr);
1899 ASSEMBLE_MOVX(movzxwl);
1900 __ AssertZeroExtended(i.OutputRegister());
1901 break;
1902 case kX64Movsxwq:
1903 EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i,
1904 __ pc_offset(), instr);
1905 ASSEMBLE_MOVX(movsxwq);
1906 break;
1907 case kX64Movzxwq:
1908 EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i,
1909 __ pc_offset(), instr);
1910 ASSEMBLE_MOVX(movzxwq);
1911 __ AssertZeroExtended(i.OutputRegister());
1912 break;
1913 case kX64Movw: {
1914 EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i,
1915 __ pc_offset(), instr);
1916 size_t index = 0;
1917 Operand operand = i.MemoryOperand(&index);
1918 if (HasImmediateInput(instr, index)) {
1919 __ movw(operand, Immediate(i.InputInt16(index)));
1920 } else {
1921 __ movw(operand, i.InputRegister(index));
1922 }
1923 break;
1924 }
1925 case kX64Movl:
1926 EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i,
1927 __ pc_offset(), instr);
1928 if (instr->HasOutput()) {
1929 if (instr->addressing_mode() == kMode_None) {
1930 if (instr->InputAt(0)->IsRegister()) {
1931 __ movl(i.OutputRegister(), i.InputRegister(0));
1932 } else {
1933 __ movl(i.OutputRegister(), i.InputOperand(0));
1934 }
1935 } else {
1936 __ movl(i.OutputRegister(), i.MemoryOperand());
1937 }
1938 __ AssertZeroExtended(i.OutputRegister());
1939 } else {
1940 size_t index = 0;
1941 Operand operand = i.MemoryOperand(&index);
1942 if (HasImmediateInput(instr, index)) {
1943 __ movl(operand, i.InputImmediate(index));
1944 } else {
1945 __ movl(operand, i.InputRegister(index));
1946 }
1947 }
1948 break;
1949 case kX64Movsxlq:
1950 EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i,
1951 __ pc_offset(), instr);
1952 ASSEMBLE_MOVX(movsxlq);
1953 break;
1954 case kX64Movq:
1955 EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i,
1956 __ pc_offset(), instr);
1957 if (instr->HasOutput()) {
1958 __ movq(i.OutputRegister(), i.MemoryOperand());
1959 } else {
1960 size_t index = 0;
1961 Operand operand = i.MemoryOperand(&index);
1962 if (HasImmediateInput(instr, index)) {
1963 __ movq(operand, i.InputImmediate(index));
1964 } else {
1965 __ movq(operand, i.InputRegister(index));
1966 }
1967 }
1968 break;
1969 case kX64Movss:
1970 EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i,
1971 __ pc_offset(), instr);
1972 if (instr->HasOutput()) {
1973 __ movss(i.OutputDoubleRegister(), i.MemoryOperand());
1974 } else {
1975 size_t index = 0;
1976 Operand operand = i.MemoryOperand(&index);
1977 __ movss(operand, i.InputDoubleRegister(index));
1978 }
1979 break;
1980 case kX64Movsd:
1981 EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i,
1982 __ pc_offset(), instr);
1983 if (instr->HasOutput()) {
1984 __ Movsd(i.OutputDoubleRegister(), i.MemoryOperand());
1985 } else {
1986 size_t index = 0;
1987 Operand operand = i.MemoryOperand(&index);
1988 __ Movsd(operand, i.InputDoubleRegister(index));
1989 }
1990 break;
1991 case kX64BitcastFI:
1992 if (instr->InputAt(0)->IsFPStackSlot()) {
1993 __ movl(i.OutputRegister(), i.InputOperand(0));
1994 } else {
1995 __ Movd(i.OutputRegister(), i.InputDoubleRegister(0));
1996 }
1997 break;
1998 case kX64BitcastDL:
1999 if (instr->InputAt(0)->IsFPStackSlot()) {
2000 __ movq(i.OutputRegister(), i.InputOperand(0));
2001 } else {
2002 __ Movq(i.OutputRegister(), i.InputDoubleRegister(0));
2003 }
2004 break;
2005 case kX64BitcastIF:
2006 if (instr->InputAt(0)->IsRegister()) {
2007 __ Movd(i.OutputDoubleRegister(), i.InputRegister(0));
2008 } else {
2009 __ movss(i.OutputDoubleRegister(), i.InputOperand(0));
2010 }
2011 break;
2012 case kX64BitcastLD:
2013 if (instr->InputAt(0)->IsRegister()) {
2014 __ Movq(i.OutputDoubleRegister(), i.InputRegister(0));
2015 } else {
2016 __ Movsd(i.OutputDoubleRegister(), i.InputOperand(0));
2017 }
2018 break;
2019 case kX64Lea32: {
2020 AddressingMode mode = AddressingModeField::decode(instr->opcode());
2021 // Shorten "leal" to "addl", "subl" or "shll" if the register allocation
2022 // and addressing mode just happens to work out. The "addl"/"subl" forms
2023 // in these cases are faster based on measurements.
2024 if (i.InputRegister(0).is(i.OutputRegister())) {
2025 if (mode == kMode_MRI) {
2026 int32_t constant_summand = i.InputInt32(1);
2027 if (constant_summand > 0) {
2028 __ addl(i.OutputRegister(), Immediate(constant_summand));
2029 } else if (constant_summand < 0) {
2030 __ subl(i.OutputRegister(), Immediate(-constant_summand));
2031 }
2032 } else if (mode == kMode_MR1) {
2033 if (i.InputRegister(1).is(i.OutputRegister())) {
2034 __ shll(i.OutputRegister(), Immediate(1));
2035 } else {
2036 __ addl(i.OutputRegister(), i.InputRegister(1));
2037 }
2038 } else if (mode == kMode_M2) {
2039 __ shll(i.OutputRegister(), Immediate(1));
2040 } else if (mode == kMode_M4) {
2041 __ shll(i.OutputRegister(), Immediate(2));
2042 } else if (mode == kMode_M8) {
2043 __ shll(i.OutputRegister(), Immediate(3));
2044 } else {
2045 __ leal(i.OutputRegister(), i.MemoryOperand());
2046 }
2047 } else if (mode == kMode_MR1 &&
2048 i.InputRegister(1).is(i.OutputRegister())) {
2049 __ addl(i.OutputRegister(), i.InputRegister(0));
2050 } else {
2051 __ leal(i.OutputRegister(), i.MemoryOperand());
2052 }
2053 __ AssertZeroExtended(i.OutputRegister());
2054 break;
2055 }
2056 case kX64Lea: {
2057 AddressingMode mode = AddressingModeField::decode(instr->opcode());
2058 // Shorten "leaq" to "addq", "subq" or "shlq" if the register allocation
2059 // and addressing mode just happens to work out. The "addq"/"subq" forms
2060 // in these cases are faster based on measurements.
2061 if (i.InputRegister(0).is(i.OutputRegister())) {
2062 if (mode == kMode_MRI) {
2063 int32_t constant_summand = i.InputInt32(1);
2064 if (constant_summand > 0) {
2065 __ addq(i.OutputRegister(), Immediate(constant_summand));
2066 } else if (constant_summand < 0) {
2067 __ subq(i.OutputRegister(), Immediate(-constant_summand));
2068 }
2069 } else if (mode == kMode_MR1) {
2070 if (i.InputRegister(1).is(i.OutputRegister())) {
2071 __ shlq(i.OutputRegister(), Immediate(1));
2072 } else {
2073 __ addq(i.OutputRegister(), i.InputRegister(1));
2074 }
2075 } else if (mode == kMode_M2) {
2076 __ shlq(i.OutputRegister(), Immediate(1));
2077 } else if (mode == kMode_M4) {
2078 __ shlq(i.OutputRegister(), Immediate(2));
2079 } else if (mode == kMode_M8) {
2080 __ shlq(i.OutputRegister(), Immediate(3));
2081 } else {
2082 __ leaq(i.OutputRegister(), i.MemoryOperand());
2083 }
2084 } else if (mode == kMode_MR1 &&
2085 i.InputRegister(1).is(i.OutputRegister())) {
2086 __ addq(i.OutputRegister(), i.InputRegister(0));
2087 } else {
2088 __ leaq(i.OutputRegister(), i.MemoryOperand());
2089 }
2090 break;
2091 }
2092 case kX64Dec32:
2093 __ decl(i.OutputRegister());
2094 break;
2095 case kX64Inc32:
2096 __ incl(i.OutputRegister());
2097 break;
2098 case kX64Push:
2099 if (AddressingModeField::decode(instr->opcode()) != kMode_None) {
2100 size_t index = 0;
2101 Operand operand = i.MemoryOperand(&index);
2102 __ pushq(operand);
2103 frame_access_state()->IncreaseSPDelta(1);
2104 unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
2105 kPointerSize);
2106 } else if (HasImmediateInput(instr, 0)) {
2107 __ pushq(i.InputImmediate(0));
2108 frame_access_state()->IncreaseSPDelta(1);
2109 unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
2110 kPointerSize);
2111 } else if (instr->InputAt(0)->IsRegister()) {
2112 __ pushq(i.InputRegister(0));
2113 frame_access_state()->IncreaseSPDelta(1);
2114 unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
2115 kPointerSize);
2116 } else if (instr->InputAt(0)->IsFPRegister()) {
2117 // TODO(titzer): use another machine instruction?
2118 __ subq(rsp, Immediate(kDoubleSize));
2119 frame_access_state()->IncreaseSPDelta(kDoubleSize / kPointerSize);
2120 unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
2121 kDoubleSize);
2122 __ Movsd(Operand(rsp, 0), i.InputDoubleRegister(0));
2123 } else {
2124 __ pushq(i.InputOperand(0));
2125 frame_access_state()->IncreaseSPDelta(1);
2126 unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
2127 kPointerSize);
2128 }
2129 break;
2130 case kX64Poke: {
2131 int const slot = MiscField::decode(instr->opcode());
2132 if (HasImmediateInput(instr, 0)) {
2133 __ movq(Operand(rsp, slot * kPointerSize), i.InputImmediate(0));
2134 } else {
2135 __ movq(Operand(rsp, slot * kPointerSize), i.InputRegister(0));
2136 }
2137 break;
2138 }
2139 case kX64Xchgb: {
2140 size_t index = 0;
2141 Operand operand = i.MemoryOperand(&index);
2142 __ xchgb(i.InputRegister(index), operand);
2143 break;
2144 }
2145 case kX64Xchgw: {
2146 size_t index = 0;
2147 Operand operand = i.MemoryOperand(&index);
2148 __ xchgw(i.InputRegister(index), operand);
2149 break;
2150 }
2151 case kX64Xchgl: {
2152 size_t index = 0;
2153 Operand operand = i.MemoryOperand(&index);
2154 __ xchgl(i.InputRegister(index), operand);
2155 break;
2156 }
2157 case kX64Int32x4Create: {
2158 CpuFeatureScope sse_scope(masm(), SSE4_1);
2159 XMMRegister dst = i.OutputSimd128Register();
2160 __ Movd(dst, i.InputRegister(0));
2161 __ shufps(dst, dst, 0x0);
2162 break;
2163 }
2164 case kX64Int32x4ExtractLane: {
2165 CpuFeatureScope sse_scope(masm(), SSE4_1);
2166 __ Pextrd(i.OutputRegister(), i.InputSimd128Register(0), i.InputInt8(1));
2167 break;
2168 }
2169 case kX64Int32x4ReplaceLane: {
2170 CpuFeatureScope sse_scope(masm(), SSE4_1);
2171 if (instr->InputAt(2)->IsRegister()) {
2172 __ Pinsrd(i.OutputSimd128Register(), i.InputRegister(2),
2173 i.InputInt8(1));
2174 } else {
2175 __ Pinsrd(i.OutputSimd128Register(), i.InputOperand(2), i.InputInt8(1));
2176 }
2177 break;
2178 }
2179 case kX64Int32x4Add: {
2180 CpuFeatureScope sse_scope(masm(), SSE4_1);
2181 __ paddd(i.OutputSimd128Register(), i.InputSimd128Register(1));
2182 break;
2183 }
2184 case kX64Int32x4Sub: {
2185 CpuFeatureScope sse_scope(masm(), SSE4_1);
2186 __ psubd(i.OutputSimd128Register(), i.InputSimd128Register(1));
2187 break;
2188 }
2189 case kCheckedLoadInt8:
2190 ASSEMBLE_CHECKED_LOAD_INTEGER(movsxbl);
2191 break;
2192 case kCheckedLoadUint8:
2193 ASSEMBLE_CHECKED_LOAD_INTEGER(movzxbl);
2194 break;
2195 case kCheckedLoadInt16:
2196 ASSEMBLE_CHECKED_LOAD_INTEGER(movsxwl);
2197 break;
2198 case kCheckedLoadUint16:
2199 ASSEMBLE_CHECKED_LOAD_INTEGER(movzxwl);
2200 break;
2201 case kCheckedLoadWord32:
2202 ASSEMBLE_CHECKED_LOAD_INTEGER(movl);
2203 break;
2204 case kCheckedLoadWord64:
2205 ASSEMBLE_CHECKED_LOAD_INTEGER(movq);
2206 break;
2207 case kCheckedLoadFloat32:
2208 ASSEMBLE_CHECKED_LOAD_FLOAT(Movss, OutOfLineLoadFloat32NaN);
2209 break;
2210 case kCheckedLoadFloat64:
2211 ASSEMBLE_CHECKED_LOAD_FLOAT(Movsd, OutOfLineLoadFloat64NaN);
2212 break;
2213 case kCheckedStoreWord8:
2214 ASSEMBLE_CHECKED_STORE_INTEGER(movb);
2215 break;
2216 case kCheckedStoreWord16:
2217 ASSEMBLE_CHECKED_STORE_INTEGER(movw);
2218 break;
2219 case kCheckedStoreWord32:
2220 ASSEMBLE_CHECKED_STORE_INTEGER(movl);
2221 break;
2222 case kCheckedStoreWord64:
2223 ASSEMBLE_CHECKED_STORE_INTEGER(movq);
2224 break;
2225 case kCheckedStoreFloat32:
2226 ASSEMBLE_CHECKED_STORE_FLOAT(Movss);
2227 break;
2228 case kCheckedStoreFloat64:
2229 ASSEMBLE_CHECKED_STORE_FLOAT(Movsd);
2230 break;
2231 case kX64StackCheck:
2232 __ CompareRoot(rsp, Heap::kStackLimitRootIndex);
2233 break;
2234 case kAtomicLoadInt8:
2235 case kAtomicLoadUint8:
2236 case kAtomicLoadInt16:
2237 case kAtomicLoadUint16:
2238 case kAtomicLoadWord32:
2239 case kAtomicStoreWord8:
2240 case kAtomicStoreWord16:
2241 case kAtomicStoreWord32:
2242 UNREACHABLE(); // Won't be generated by instruction selector.
2243 break;
2244 }
2245 return kSuccess;
2246 } // NOLINT(readability/fn_size)
2247
2248 namespace {
2249
FlagsConditionToCondition(FlagsCondition condition)2250 Condition FlagsConditionToCondition(FlagsCondition condition) {
2251 switch (condition) {
2252 case kUnorderedEqual:
2253 case kEqual:
2254 return equal;
2255 case kUnorderedNotEqual:
2256 case kNotEqual:
2257 return not_equal;
2258 case kSignedLessThan:
2259 return less;
2260 case kSignedGreaterThanOrEqual:
2261 return greater_equal;
2262 case kSignedLessThanOrEqual:
2263 return less_equal;
2264 case kSignedGreaterThan:
2265 return greater;
2266 case kUnsignedLessThan:
2267 return below;
2268 case kUnsignedGreaterThanOrEqual:
2269 return above_equal;
2270 case kUnsignedLessThanOrEqual:
2271 return below_equal;
2272 case kUnsignedGreaterThan:
2273 return above;
2274 case kOverflow:
2275 return overflow;
2276 case kNotOverflow:
2277 return no_overflow;
2278 default:
2279 break;
2280 }
2281 UNREACHABLE();
2282 return no_condition;
2283 }
2284
2285 } // namespace
2286
2287 // Assembles branches after this instruction.
AssembleArchBranch(Instruction * instr,BranchInfo * branch)2288 void CodeGenerator::AssembleArchBranch(Instruction* instr, BranchInfo* branch) {
2289 Label::Distance flabel_distance =
2290 branch->fallthru ? Label::kNear : Label::kFar;
2291 Label* tlabel = branch->true_label;
2292 Label* flabel = branch->false_label;
2293 if (branch->condition == kUnorderedEqual) {
2294 __ j(parity_even, flabel, flabel_distance);
2295 } else if (branch->condition == kUnorderedNotEqual) {
2296 __ j(parity_even, tlabel);
2297 }
2298 __ j(FlagsConditionToCondition(branch->condition), tlabel);
2299
2300 if (!branch->fallthru) __ jmp(flabel, flabel_distance);
2301 }
2302
2303
AssembleArchJump(RpoNumber target)2304 void CodeGenerator::AssembleArchJump(RpoNumber target) {
2305 if (!IsNextInAssemblyOrder(target)) __ jmp(GetLabel(target));
2306 }
2307
AssembleArchTrap(Instruction * instr,FlagsCondition condition)2308 void CodeGenerator::AssembleArchTrap(Instruction* instr,
2309 FlagsCondition condition) {
2310 class OutOfLineTrap final : public OutOfLineCode {
2311 public:
2312 OutOfLineTrap(CodeGenerator* gen, bool frame_elided, Instruction* instr)
2313 : OutOfLineCode(gen),
2314 frame_elided_(frame_elided),
2315 instr_(instr),
2316 gen_(gen) {}
2317
2318 void Generate() final {
2319 X64OperandConverter i(gen_, instr_);
2320
2321 Builtins::Name trap_id =
2322 static_cast<Builtins::Name>(i.InputInt32(instr_->InputCount() - 1));
2323 bool old_has_frame = __ has_frame();
2324 if (frame_elided_) {
2325 __ set_has_frame(true);
2326 __ EnterFrame(StackFrame::WASM_COMPILED);
2327 }
2328 GenerateCallToTrap(trap_id);
2329 if (frame_elided_) {
2330 __ set_has_frame(old_has_frame);
2331 }
2332 }
2333
2334 private:
2335 void GenerateCallToTrap(Builtins::Name trap_id) {
2336 if (trap_id == Builtins::builtin_count) {
2337 // We cannot test calls to the runtime in cctest/test-run-wasm.
2338 // Therefore we emit a call to C here instead of a call to the runtime.
2339 __ PrepareCallCFunction(0);
2340 __ CallCFunction(
2341 ExternalReference::wasm_call_trap_callback_for_testing(isolate()),
2342 0);
2343 __ LeaveFrame(StackFrame::WASM_COMPILED);
2344 __ Ret();
2345 } else {
2346 gen_->AssembleSourcePosition(instr_);
2347 __ Call(handle(isolate()->builtins()->builtin(trap_id), isolate()),
2348 RelocInfo::CODE_TARGET);
2349 ReferenceMap* reference_map =
2350 new (gen_->zone()) ReferenceMap(gen_->zone());
2351 gen_->RecordSafepoint(reference_map, Safepoint::kSimple, 0,
2352 Safepoint::kNoLazyDeopt);
2353 if (FLAG_debug_code) {
2354 __ ud2();
2355 }
2356 }
2357 }
2358
2359 bool frame_elided_;
2360 Instruction* instr_;
2361 CodeGenerator* gen_;
2362 };
2363 bool frame_elided = !frame_access_state()->has_frame();
2364 auto ool = new (zone()) OutOfLineTrap(this, frame_elided, instr);
2365 Label* tlabel = ool->entry();
2366 Label end;
2367 if (condition == kUnorderedEqual) {
2368 __ j(parity_even, &end);
2369 } else if (condition == kUnorderedNotEqual) {
2370 __ j(parity_even, tlabel);
2371 }
2372 __ j(FlagsConditionToCondition(condition), tlabel);
2373 __ bind(&end);
2374 }
2375
2376 // Assembles boolean materializations after this instruction.
AssembleArchBoolean(Instruction * instr,FlagsCondition condition)2377 void CodeGenerator::AssembleArchBoolean(Instruction* instr,
2378 FlagsCondition condition) {
2379 X64OperandConverter i(this, instr);
2380 Label done;
2381
2382 // Materialize a full 64-bit 1 or 0 value. The result register is always the
2383 // last output of the instruction.
2384 Label check;
2385 DCHECK_NE(0u, instr->OutputCount());
2386 Register reg = i.OutputRegister(instr->OutputCount() - 1);
2387 if (condition == kUnorderedEqual) {
2388 __ j(parity_odd, &check, Label::kNear);
2389 __ movl(reg, Immediate(0));
2390 __ jmp(&done, Label::kNear);
2391 } else if (condition == kUnorderedNotEqual) {
2392 __ j(parity_odd, &check, Label::kNear);
2393 __ movl(reg, Immediate(1));
2394 __ jmp(&done, Label::kNear);
2395 }
2396 __ bind(&check);
2397 __ setcc(FlagsConditionToCondition(condition), reg);
2398 __ movzxbl(reg, reg);
2399 __ bind(&done);
2400 }
2401
2402
AssembleArchLookupSwitch(Instruction * instr)2403 void CodeGenerator::AssembleArchLookupSwitch(Instruction* instr) {
2404 X64OperandConverter i(this, instr);
2405 Register input = i.InputRegister(0);
2406 for (size_t index = 2; index < instr->InputCount(); index += 2) {
2407 __ cmpl(input, Immediate(i.InputInt32(index + 0)));
2408 __ j(equal, GetLabel(i.InputRpo(index + 1)));
2409 }
2410 AssembleArchJump(i.InputRpo(1));
2411 }
2412
2413
AssembleArchTableSwitch(Instruction * instr)2414 void CodeGenerator::AssembleArchTableSwitch(Instruction* instr) {
2415 X64OperandConverter i(this, instr);
2416 Register input = i.InputRegister(0);
2417 int32_t const case_count = static_cast<int32_t>(instr->InputCount() - 2);
2418 Label** cases = zone()->NewArray<Label*>(case_count);
2419 for (int32_t index = 0; index < case_count; ++index) {
2420 cases[index] = GetLabel(i.InputRpo(index + 2));
2421 }
2422 Label* const table = AddJumpTable(cases, case_count);
2423 __ cmpl(input, Immediate(case_count));
2424 __ j(above_equal, GetLabel(i.InputRpo(1)));
2425 __ leaq(kScratchRegister, Operand(table));
2426 __ jmp(Operand(kScratchRegister, input, times_8, 0));
2427 }
2428
AssembleDeoptimizerCall(int deoptimization_id,SourcePosition pos)2429 CodeGenerator::CodeGenResult CodeGenerator::AssembleDeoptimizerCall(
2430 int deoptimization_id, SourcePosition pos) {
2431 DeoptimizeKind deoptimization_kind = GetDeoptimizationKind(deoptimization_id);
2432 DeoptimizeReason deoptimization_reason =
2433 GetDeoptimizationReason(deoptimization_id);
2434 Deoptimizer::BailoutType bailout_type =
2435 deoptimization_kind == DeoptimizeKind::kSoft ? Deoptimizer::SOFT
2436 : Deoptimizer::EAGER;
2437 Address deopt_entry = Deoptimizer::GetDeoptimizationEntry(
2438 isolate(), deoptimization_id, bailout_type);
2439 if (deopt_entry == nullptr) return kTooManyDeoptimizationBailouts;
2440 __ RecordDeoptReason(deoptimization_reason, pos, deoptimization_id);
2441 __ call(deopt_entry, RelocInfo::RUNTIME_ENTRY);
2442 return kSuccess;
2443 }
2444
2445
2446 namespace {
2447
2448 static const int kQuadWordSize = 16;
2449
2450 } // namespace
2451
FinishFrame(Frame * frame)2452 void CodeGenerator::FinishFrame(Frame* frame) {
2453 CallDescriptor* descriptor = linkage()->GetIncomingDescriptor();
2454
2455 const RegList saves_fp = descriptor->CalleeSavedFPRegisters();
2456 if (saves_fp != 0) {
2457 frame->AlignSavedCalleeRegisterSlots();
2458 if (saves_fp != 0) { // Save callee-saved XMM registers.
2459 const uint32_t saves_fp_count = base::bits::CountPopulation32(saves_fp);
2460 frame->AllocateSavedCalleeRegisterSlots(saves_fp_count *
2461 (kQuadWordSize / kPointerSize));
2462 }
2463 }
2464 const RegList saves = descriptor->CalleeSavedRegisters();
2465 if (saves != 0) { // Save callee-saved registers.
2466 int count = 0;
2467 for (int i = Register::kNumRegisters - 1; i >= 0; i--) {
2468 if (((1 << i) & saves)) {
2469 ++count;
2470 }
2471 }
2472 frame->AllocateSavedCalleeRegisterSlots(count);
2473 }
2474 }
2475
AssembleConstructFrame()2476 void CodeGenerator::AssembleConstructFrame() {
2477 CallDescriptor* descriptor = linkage()->GetIncomingDescriptor();
2478 if (frame_access_state()->has_frame()) {
2479 int pc_base = __ pc_offset();
2480
2481 if (descriptor->IsCFunctionCall()) {
2482 __ pushq(rbp);
2483 __ movq(rbp, rsp);
2484 } else if (descriptor->IsJSFunctionCall()) {
2485 __ Prologue(this->info()->GeneratePreagedPrologue());
2486 if (descriptor->PushArgumentCount()) {
2487 __ pushq(kJavaScriptCallArgCountRegister);
2488 }
2489 } else {
2490 __ StubPrologue(info()->GetOutputStackFrameType());
2491 }
2492
2493 if (!descriptor->IsJSFunctionCall() || !info()->GeneratePreagedPrologue()) {
2494 unwinding_info_writer_.MarkFrameConstructed(pc_base);
2495 }
2496 }
2497 int shrink_slots =
2498 frame()->GetTotalFrameSlotCount() - descriptor->CalculateFixedFrameSize();
2499
2500 if (info()->is_osr()) {
2501 // TurboFan OSR-compiled functions cannot be entered directly.
2502 __ Abort(kShouldNotDirectlyEnterOsrFunction);
2503
2504 // Unoptimized code jumps directly to this entrypoint while the unoptimized
2505 // frame is still on the stack. Optimized code uses OSR values directly from
2506 // the unoptimized frame. Thus, all that needs to be done is to allocate the
2507 // remaining stack slots.
2508 if (FLAG_code_comments) __ RecordComment("-- OSR entrypoint --");
2509 osr_pc_offset_ = __ pc_offset();
2510 shrink_slots -= static_cast<int>(OsrHelper(info()).UnoptimizedFrameSlots());
2511 }
2512
2513 const RegList saves_fp = descriptor->CalleeSavedFPRegisters();
2514 if (shrink_slots > 0) {
2515 __ subq(rsp, Immediate(shrink_slots * kPointerSize));
2516 }
2517
2518 if (saves_fp != 0) { // Save callee-saved XMM registers.
2519 const uint32_t saves_fp_count = base::bits::CountPopulation32(saves_fp);
2520 const int stack_size = saves_fp_count * kQuadWordSize;
2521 // Adjust the stack pointer.
2522 __ subp(rsp, Immediate(stack_size));
2523 // Store the registers on the stack.
2524 int slot_idx = 0;
2525 for (int i = 0; i < XMMRegister::kMaxNumRegisters; i++) {
2526 if (!((1 << i) & saves_fp)) continue;
2527 __ movdqu(Operand(rsp, kQuadWordSize * slot_idx),
2528 XMMRegister::from_code(i));
2529 slot_idx++;
2530 }
2531 }
2532
2533 const RegList saves = descriptor->CalleeSavedRegisters();
2534 if (saves != 0) { // Save callee-saved registers.
2535 for (int i = Register::kNumRegisters - 1; i >= 0; i--) {
2536 if (!((1 << i) & saves)) continue;
2537 __ pushq(Register::from_code(i));
2538 }
2539 }
2540 }
2541
AssembleReturn(InstructionOperand * pop)2542 void CodeGenerator::AssembleReturn(InstructionOperand* pop) {
2543 CallDescriptor* descriptor = linkage()->GetIncomingDescriptor();
2544
2545 // Restore registers.
2546 const RegList saves = descriptor->CalleeSavedRegisters();
2547 if (saves != 0) {
2548 for (int i = 0; i < Register::kNumRegisters; i++) {
2549 if (!((1 << i) & saves)) continue;
2550 __ popq(Register::from_code(i));
2551 }
2552 }
2553 const RegList saves_fp = descriptor->CalleeSavedFPRegisters();
2554 if (saves_fp != 0) {
2555 const uint32_t saves_fp_count = base::bits::CountPopulation32(saves_fp);
2556 const int stack_size = saves_fp_count * kQuadWordSize;
2557 // Load the registers from the stack.
2558 int slot_idx = 0;
2559 for (int i = 0; i < XMMRegister::kMaxNumRegisters; i++) {
2560 if (!((1 << i) & saves_fp)) continue;
2561 __ movdqu(XMMRegister::from_code(i),
2562 Operand(rsp, kQuadWordSize * slot_idx));
2563 slot_idx++;
2564 }
2565 // Adjust the stack pointer.
2566 __ addp(rsp, Immediate(stack_size));
2567 }
2568
2569 unwinding_info_writer_.MarkBlockWillExit();
2570
2571 // Might need rcx for scratch if pop_size is too big or if there is a variable
2572 // pop count.
2573 DCHECK_EQ(0u, descriptor->CalleeSavedRegisters() & rcx.bit());
2574 DCHECK_EQ(0u, descriptor->CalleeSavedRegisters() & rdx.bit());
2575 size_t pop_size = descriptor->StackParameterCount() * kPointerSize;
2576 X64OperandConverter g(this, nullptr);
2577 if (descriptor->IsCFunctionCall()) {
2578 AssembleDeconstructFrame();
2579 } else if (frame_access_state()->has_frame()) {
2580 if (pop->IsImmediate() && g.ToConstant(pop).ToInt32() == 0) {
2581 // Canonicalize JSFunction return sites for now.
2582 if (return_label_.is_bound()) {
2583 __ jmp(&return_label_);
2584 return;
2585 } else {
2586 __ bind(&return_label_);
2587 AssembleDeconstructFrame();
2588 }
2589 } else {
2590 AssembleDeconstructFrame();
2591 }
2592 }
2593
2594 if (pop->IsImmediate()) {
2595 DCHECK_EQ(Constant::kInt32, g.ToConstant(pop).type());
2596 pop_size += g.ToConstant(pop).ToInt32() * kPointerSize;
2597 CHECK_LT(pop_size, static_cast<size_t>(std::numeric_limits<int>::max()));
2598 __ Ret(static_cast<int>(pop_size), rcx);
2599 } else {
2600 Register pop_reg = g.ToRegister(pop);
2601 Register scratch_reg = pop_reg.is(rcx) ? rdx : rcx;
2602 __ popq(scratch_reg);
2603 __ leaq(rsp, Operand(rsp, pop_reg, times_8, static_cast<int>(pop_size)));
2604 __ jmp(scratch_reg);
2605 }
2606 }
2607
2608
AssembleMove(InstructionOperand * source,InstructionOperand * destination)2609 void CodeGenerator::AssembleMove(InstructionOperand* source,
2610 InstructionOperand* destination) {
2611 X64OperandConverter g(this, nullptr);
2612 // Dispatch on the source and destination operand kinds. Not all
2613 // combinations are possible.
2614 if (source->IsRegister()) {
2615 DCHECK(destination->IsRegister() || destination->IsStackSlot());
2616 Register src = g.ToRegister(source);
2617 if (destination->IsRegister()) {
2618 __ movq(g.ToRegister(destination), src);
2619 } else {
2620 __ movq(g.ToOperand(destination), src);
2621 }
2622 } else if (source->IsStackSlot()) {
2623 DCHECK(destination->IsRegister() || destination->IsStackSlot());
2624 Operand src = g.ToOperand(source);
2625 if (destination->IsRegister()) {
2626 Register dst = g.ToRegister(destination);
2627 __ movq(dst, src);
2628 } else {
2629 // Spill on demand to use a temporary register for memory-to-memory
2630 // moves.
2631 Register tmp = kScratchRegister;
2632 Operand dst = g.ToOperand(destination);
2633 __ movq(tmp, src);
2634 __ movq(dst, tmp);
2635 }
2636 } else if (source->IsConstant()) {
2637 ConstantOperand* constant_source = ConstantOperand::cast(source);
2638 Constant src = g.ToConstant(constant_source);
2639 if (destination->IsRegister() || destination->IsStackSlot()) {
2640 Register dst = destination->IsRegister() ? g.ToRegister(destination)
2641 : kScratchRegister;
2642 switch (src.type()) {
2643 case Constant::kInt32: {
2644 if (RelocInfo::IsWasmPtrReference(src.rmode())) {
2645 __ movq(dst, src.ToInt64(), src.rmode());
2646 } else {
2647 // TODO(dcarney): don't need scratch in this case.
2648 int32_t value = src.ToInt32();
2649 if (value == 0) {
2650 __ xorl(dst, dst);
2651 } else {
2652 if (RelocInfo::IsWasmSizeReference(src.rmode())) {
2653 __ movl(dst, Immediate(value, src.rmode()));
2654 } else {
2655 __ movl(dst, Immediate(value));
2656 }
2657 }
2658 }
2659 break;
2660 }
2661 case Constant::kInt64:
2662 if (RelocInfo::IsWasmPtrReference(src.rmode())) {
2663 __ movq(dst, src.ToInt64(), src.rmode());
2664 } else {
2665 DCHECK(!RelocInfo::IsWasmSizeReference(src.rmode()));
2666 __ Set(dst, src.ToInt64());
2667 }
2668 break;
2669 case Constant::kFloat32:
2670 __ Move(dst,
2671 isolate()->factory()->NewNumber(src.ToFloat32(), TENURED));
2672 break;
2673 case Constant::kFloat64:
2674 __ Move(dst,
2675 isolate()->factory()->NewNumber(src.ToFloat64(), TENURED));
2676 break;
2677 case Constant::kExternalReference:
2678 __ Move(dst, src.ToExternalReference());
2679 break;
2680 case Constant::kHeapObject: {
2681 Handle<HeapObject> src_object = src.ToHeapObject();
2682 Heap::RootListIndex index;
2683 if (IsMaterializableFromRoot(src_object, &index)) {
2684 __ LoadRoot(dst, index);
2685 } else {
2686 __ Move(dst, src_object);
2687 }
2688 break;
2689 }
2690 case Constant::kRpoNumber:
2691 UNREACHABLE(); // TODO(dcarney): load of labels on x64.
2692 break;
2693 }
2694 if (destination->IsStackSlot()) {
2695 __ movq(g.ToOperand(destination), kScratchRegister);
2696 }
2697 } else if (src.type() == Constant::kFloat32) {
2698 // TODO(turbofan): Can we do better here?
2699 uint32_t src_const = bit_cast<uint32_t>(src.ToFloat32());
2700 if (destination->IsFPRegister()) {
2701 __ Move(g.ToDoubleRegister(destination), src_const);
2702 } else {
2703 DCHECK(destination->IsFPStackSlot());
2704 Operand dst = g.ToOperand(destination);
2705 __ movl(dst, Immediate(src_const));
2706 }
2707 } else {
2708 DCHECK_EQ(Constant::kFloat64, src.type());
2709 uint64_t src_const = bit_cast<uint64_t>(src.ToFloat64());
2710 if (destination->IsFPRegister()) {
2711 __ Move(g.ToDoubleRegister(destination), src_const);
2712 } else {
2713 DCHECK(destination->IsFPStackSlot());
2714 __ movq(kScratchRegister, src_const);
2715 __ movq(g.ToOperand(destination), kScratchRegister);
2716 }
2717 }
2718 } else if (source->IsFPRegister()) {
2719 XMMRegister src = g.ToDoubleRegister(source);
2720 if (destination->IsFPRegister()) {
2721 XMMRegister dst = g.ToDoubleRegister(destination);
2722 __ Movapd(dst, src);
2723 } else {
2724 DCHECK(destination->IsFPStackSlot());
2725 Operand dst = g.ToOperand(destination);
2726 MachineRepresentation rep =
2727 LocationOperand::cast(source)->representation();
2728 if (rep != MachineRepresentation::kSimd128) {
2729 __ Movsd(dst, src);
2730 } else {
2731 __ Movups(dst, src);
2732 }
2733 }
2734 } else if (source->IsFPStackSlot()) {
2735 DCHECK(destination->IsFPRegister() || destination->IsFPStackSlot());
2736 Operand src = g.ToOperand(source);
2737 MachineRepresentation rep = LocationOperand::cast(source)->representation();
2738 if (destination->IsFPRegister()) {
2739 XMMRegister dst = g.ToDoubleRegister(destination);
2740 if (rep != MachineRepresentation::kSimd128) {
2741 __ Movsd(dst, src);
2742 } else {
2743 __ Movups(dst, src);
2744 }
2745 } else {
2746 Operand dst = g.ToOperand(destination);
2747 if (rep != MachineRepresentation::kSimd128) {
2748 __ Movsd(kScratchDoubleReg, src);
2749 __ Movsd(dst, kScratchDoubleReg);
2750 } else {
2751 __ Movups(kScratchDoubleReg, src);
2752 __ Movups(dst, kScratchDoubleReg);
2753 }
2754 }
2755 } else {
2756 UNREACHABLE();
2757 }
2758 }
2759
2760
AssembleSwap(InstructionOperand * source,InstructionOperand * destination)2761 void CodeGenerator::AssembleSwap(InstructionOperand* source,
2762 InstructionOperand* destination) {
2763 X64OperandConverter g(this, nullptr);
2764 // Dispatch on the source and destination operand kinds. Not all
2765 // combinations are possible.
2766 if (source->IsRegister() && destination->IsRegister()) {
2767 // Register-register.
2768 Register src = g.ToRegister(source);
2769 Register dst = g.ToRegister(destination);
2770 __ movq(kScratchRegister, src);
2771 __ movq(src, dst);
2772 __ movq(dst, kScratchRegister);
2773 } else if (source->IsRegister() && destination->IsStackSlot()) {
2774 Register src = g.ToRegister(source);
2775 __ pushq(src);
2776 frame_access_state()->IncreaseSPDelta(1);
2777 unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
2778 kPointerSize);
2779 Operand dst = g.ToOperand(destination);
2780 __ movq(src, dst);
2781 frame_access_state()->IncreaseSPDelta(-1);
2782 dst = g.ToOperand(destination);
2783 __ popq(dst);
2784 unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
2785 -kPointerSize);
2786 } else if ((source->IsStackSlot() && destination->IsStackSlot()) ||
2787 (source->IsFPStackSlot() && destination->IsFPStackSlot())) {
2788 // Memory-memory.
2789 Operand src = g.ToOperand(source);
2790 Operand dst = g.ToOperand(destination);
2791 MachineRepresentation rep = LocationOperand::cast(source)->representation();
2792 if (rep != MachineRepresentation::kSimd128) {
2793 Register tmp = kScratchRegister;
2794 __ movq(tmp, dst);
2795 __ pushq(src);
2796 unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
2797 kPointerSize);
2798 frame_access_state()->IncreaseSPDelta(1);
2799 src = g.ToOperand(source);
2800 __ movq(src, tmp);
2801 frame_access_state()->IncreaseSPDelta(-1);
2802 dst = g.ToOperand(destination);
2803 __ popq(dst);
2804 unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
2805 -kPointerSize);
2806 } else {
2807 // Use the XOR trick to swap without a temporary.
2808 __ Movups(kScratchDoubleReg, src);
2809 __ Xorps(kScratchDoubleReg, dst); // scratch contains src ^ dst.
2810 __ Movups(src, kScratchDoubleReg);
2811 __ Xorps(kScratchDoubleReg, dst); // scratch contains src.
2812 __ Movups(dst, kScratchDoubleReg);
2813 __ Xorps(kScratchDoubleReg, src); // scratch contains dst.
2814 __ Movups(src, kScratchDoubleReg);
2815 }
2816 } else if (source->IsFPRegister() && destination->IsFPRegister()) {
2817 // XMM register-register swap.
2818 XMMRegister src = g.ToDoubleRegister(source);
2819 XMMRegister dst = g.ToDoubleRegister(destination);
2820 __ Movapd(kScratchDoubleReg, src);
2821 __ Movapd(src, dst);
2822 __ Movapd(dst, kScratchDoubleReg);
2823 } else if (source->IsFPRegister() && destination->IsFPStackSlot()) {
2824 // XMM register-memory swap.
2825 XMMRegister src = g.ToDoubleRegister(source);
2826 Operand dst = g.ToOperand(destination);
2827 MachineRepresentation rep = LocationOperand::cast(source)->representation();
2828 if (rep != MachineRepresentation::kSimd128) {
2829 __ Movsd(kScratchDoubleReg, src);
2830 __ Movsd(src, dst);
2831 __ Movsd(dst, kScratchDoubleReg);
2832 } else {
2833 __ Movups(kScratchDoubleReg, src);
2834 __ Movups(src, dst);
2835 __ Movups(dst, kScratchDoubleReg);
2836 }
2837 } else {
2838 // No other combinations are possible.
2839 UNREACHABLE();
2840 }
2841 }
2842
2843
AssembleJumpTable(Label ** targets,size_t target_count)2844 void CodeGenerator::AssembleJumpTable(Label** targets, size_t target_count) {
2845 for (size_t index = 0; index < target_count; ++index) {
2846 __ dq(targets[index]);
2847 }
2848 }
2849
2850
EnsureSpaceForLazyDeopt()2851 void CodeGenerator::EnsureSpaceForLazyDeopt() {
2852 if (!info()->ShouldEnsureSpaceForLazyDeopt()) {
2853 return;
2854 }
2855
2856 int space_needed = Deoptimizer::patch_size();
2857 // Ensure that we have enough space after the previous lazy-bailout
2858 // instruction for patching the code here.
2859 int current_pc = __ pc_offset();
2860 if (current_pc < last_lazy_deopt_pc_ + space_needed) {
2861 int padding_size = last_lazy_deopt_pc_ + space_needed - current_pc;
2862 __ Nop(padding_size);
2863 }
2864 }
2865
2866 #undef __
2867
2868 } // namespace compiler
2869 } // namespace internal
2870 } // namespace v8
2871