1 // Copyright 2013 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "src/compiler/code-generator.h"
6
7 #include "src/compilation-info.h"
8 #include "src/compiler/code-generator-impl.h"
9 #include "src/compiler/gap-resolver.h"
10 #include "src/compiler/node-matchers.h"
11 #include "src/compiler/osr.h"
12 #include "src/frames.h"
13 #include "src/ia32/assembler-ia32.h"
14 #include "src/ia32/frames-ia32.h"
15 #include "src/ia32/macro-assembler-ia32.h"
16
17 namespace v8 {
18 namespace internal {
19 namespace compiler {
20
21 #define __ masm()->
22
23
24 #define kScratchDoubleReg xmm0
25
26
27 // Adds IA-32 specific methods for decoding operands.
28 class IA32OperandConverter : public InstructionOperandConverter {
29 public:
IA32OperandConverter(CodeGenerator * gen,Instruction * instr)30 IA32OperandConverter(CodeGenerator* gen, Instruction* instr)
31 : InstructionOperandConverter(gen, instr) {}
32
InputOperand(size_t index,int extra=0)33 Operand InputOperand(size_t index, int extra = 0) {
34 return ToOperand(instr_->InputAt(index), extra);
35 }
36
InputImmediate(size_t index)37 Immediate InputImmediate(size_t index) {
38 return ToImmediate(instr_->InputAt(index));
39 }
40
OutputOperand()41 Operand OutputOperand() { return ToOperand(instr_->Output()); }
42
ToOperand(InstructionOperand * op,int extra=0)43 Operand ToOperand(InstructionOperand* op, int extra = 0) {
44 if (op->IsRegister()) {
45 DCHECK(extra == 0);
46 return Operand(ToRegister(op));
47 } else if (op->IsFPRegister()) {
48 DCHECK(extra == 0);
49 return Operand(ToDoubleRegister(op));
50 }
51 DCHECK(op->IsStackSlot() || op->IsFPStackSlot());
52 return SlotToOperand(AllocatedOperand::cast(op)->index(), extra);
53 }
54
SlotToOperand(int slot,int extra=0)55 Operand SlotToOperand(int slot, int extra = 0) {
56 FrameOffset offset = frame_access_state()->GetFrameOffset(slot);
57 return Operand(offset.from_stack_pointer() ? esp : ebp,
58 offset.offset() + extra);
59 }
60
HighOperand(InstructionOperand * op)61 Operand HighOperand(InstructionOperand* op) {
62 DCHECK(op->IsFPStackSlot());
63 return ToOperand(op, kPointerSize);
64 }
65
ToImmediate(InstructionOperand * operand)66 Immediate ToImmediate(InstructionOperand* operand) {
67 Constant constant = ToConstant(operand);
68 if (constant.type() == Constant::kInt32 &&
69 RelocInfo::IsWasmReference(constant.rmode())) {
70 return Immediate(reinterpret_cast<Address>(constant.ToInt32()),
71 constant.rmode());
72 }
73 switch (constant.type()) {
74 case Constant::kInt32:
75 return Immediate(constant.ToInt32());
76 case Constant::kFloat32:
77 return Immediate(
78 isolate()->factory()->NewNumber(constant.ToFloat32(), TENURED));
79 case Constant::kFloat64:
80 return Immediate(
81 isolate()->factory()->NewNumber(constant.ToFloat64(), TENURED));
82 case Constant::kExternalReference:
83 return Immediate(constant.ToExternalReference());
84 case Constant::kHeapObject:
85 return Immediate(constant.ToHeapObject());
86 case Constant::kInt64:
87 break;
88 case Constant::kRpoNumber:
89 return Immediate::CodeRelativeOffset(ToLabel(operand));
90 }
91 UNREACHABLE();
92 return Immediate(-1);
93 }
94
NextOffset(size_t * offset)95 static size_t NextOffset(size_t* offset) {
96 size_t i = *offset;
97 (*offset)++;
98 return i;
99 }
100
ScaleFor(AddressingMode one,AddressingMode mode)101 static ScaleFactor ScaleFor(AddressingMode one, AddressingMode mode) {
102 STATIC_ASSERT(0 == static_cast<int>(times_1));
103 STATIC_ASSERT(1 == static_cast<int>(times_2));
104 STATIC_ASSERT(2 == static_cast<int>(times_4));
105 STATIC_ASSERT(3 == static_cast<int>(times_8));
106 int scale = static_cast<int>(mode - one);
107 DCHECK(scale >= 0 && scale < 4);
108 return static_cast<ScaleFactor>(scale);
109 }
110
MemoryOperand(size_t * offset)111 Operand MemoryOperand(size_t* offset) {
112 AddressingMode mode = AddressingModeField::decode(instr_->opcode());
113 switch (mode) {
114 case kMode_MR: {
115 Register base = InputRegister(NextOffset(offset));
116 int32_t disp = 0;
117 return Operand(base, disp);
118 }
119 case kMode_MRI: {
120 Register base = InputRegister(NextOffset(offset));
121 Constant ctant = ToConstant(instr_->InputAt(NextOffset(offset)));
122 return Operand(base, ctant.ToInt32(), ctant.rmode());
123 }
124 case kMode_MR1:
125 case kMode_MR2:
126 case kMode_MR4:
127 case kMode_MR8: {
128 Register base = InputRegister(NextOffset(offset));
129 Register index = InputRegister(NextOffset(offset));
130 ScaleFactor scale = ScaleFor(kMode_MR1, mode);
131 int32_t disp = 0;
132 return Operand(base, index, scale, disp);
133 }
134 case kMode_MR1I:
135 case kMode_MR2I:
136 case kMode_MR4I:
137 case kMode_MR8I: {
138 Register base = InputRegister(NextOffset(offset));
139 Register index = InputRegister(NextOffset(offset));
140 ScaleFactor scale = ScaleFor(kMode_MR1I, mode);
141 Constant ctant = ToConstant(instr_->InputAt(NextOffset(offset)));
142 return Operand(base, index, scale, ctant.ToInt32(), ctant.rmode());
143 }
144 case kMode_M1:
145 case kMode_M2:
146 case kMode_M4:
147 case kMode_M8: {
148 Register index = InputRegister(NextOffset(offset));
149 ScaleFactor scale = ScaleFor(kMode_M1, mode);
150 int32_t disp = 0;
151 return Operand(index, scale, disp);
152 }
153 case kMode_M1I:
154 case kMode_M2I:
155 case kMode_M4I:
156 case kMode_M8I: {
157 Register index = InputRegister(NextOffset(offset));
158 ScaleFactor scale = ScaleFor(kMode_M1I, mode);
159 Constant ctant = ToConstant(instr_->InputAt(NextOffset(offset)));
160 return Operand(index, scale, ctant.ToInt32(), ctant.rmode());
161 }
162 case kMode_MI: {
163 Constant ctant = ToConstant(instr_->InputAt(NextOffset(offset)));
164 return Operand(ctant.ToInt32(), ctant.rmode());
165 }
166 case kMode_None:
167 UNREACHABLE();
168 return Operand(no_reg, 0);
169 }
170 UNREACHABLE();
171 return Operand(no_reg, 0);
172 }
173
MemoryOperand(size_t first_input=0)174 Operand MemoryOperand(size_t first_input = 0) {
175 return MemoryOperand(&first_input);
176 }
177 };
178
179
180 namespace {
181
HasImmediateInput(Instruction * instr,size_t index)182 bool HasImmediateInput(Instruction* instr, size_t index) {
183 return instr->InputAt(index)->IsImmediate();
184 }
185
186 class OutOfLineLoadZero final : public OutOfLineCode {
187 public:
OutOfLineLoadZero(CodeGenerator * gen,Register result)188 OutOfLineLoadZero(CodeGenerator* gen, Register result)
189 : OutOfLineCode(gen), result_(result) {}
190
Generate()191 void Generate() final { __ xor_(result_, result_); }
192
193 private:
194 Register const result_;
195 };
196
197 class OutOfLineLoadFloat32NaN final : public OutOfLineCode {
198 public:
OutOfLineLoadFloat32NaN(CodeGenerator * gen,XMMRegister result)199 OutOfLineLoadFloat32NaN(CodeGenerator* gen, XMMRegister result)
200 : OutOfLineCode(gen), result_(result) {}
201
Generate()202 void Generate() final {
203 __ xorps(result_, result_);
204 __ divss(result_, result_);
205 }
206
207 private:
208 XMMRegister const result_;
209 };
210
211 class OutOfLineLoadFloat64NaN final : public OutOfLineCode {
212 public:
OutOfLineLoadFloat64NaN(CodeGenerator * gen,XMMRegister result)213 OutOfLineLoadFloat64NaN(CodeGenerator* gen, XMMRegister result)
214 : OutOfLineCode(gen), result_(result) {}
215
Generate()216 void Generate() final {
217 __ xorpd(result_, result_);
218 __ divsd(result_, result_);
219 }
220
221 private:
222 XMMRegister const result_;
223 };
224
225 class OutOfLineTruncateDoubleToI final : public OutOfLineCode {
226 public:
OutOfLineTruncateDoubleToI(CodeGenerator * gen,Register result,XMMRegister input)227 OutOfLineTruncateDoubleToI(CodeGenerator* gen, Register result,
228 XMMRegister input)
229 : OutOfLineCode(gen), result_(result), input_(input) {}
230
Generate()231 void Generate() final {
232 __ sub(esp, Immediate(kDoubleSize));
233 __ movsd(MemOperand(esp, 0), input_);
234 __ SlowTruncateToI(result_, esp, 0);
235 __ add(esp, Immediate(kDoubleSize));
236 }
237
238 private:
239 Register const result_;
240 XMMRegister const input_;
241 };
242
243
244 class OutOfLineRecordWrite final : public OutOfLineCode {
245 public:
OutOfLineRecordWrite(CodeGenerator * gen,Register object,Operand operand,Register value,Register scratch0,Register scratch1,RecordWriteMode mode)246 OutOfLineRecordWrite(CodeGenerator* gen, Register object, Operand operand,
247 Register value, Register scratch0, Register scratch1,
248 RecordWriteMode mode)
249 : OutOfLineCode(gen),
250 object_(object),
251 operand_(operand),
252 value_(value),
253 scratch0_(scratch0),
254 scratch1_(scratch1),
255 mode_(mode) {}
256
Generate()257 void Generate() final {
258 if (mode_ > RecordWriteMode::kValueIsPointer) {
259 __ JumpIfSmi(value_, exit());
260 }
261 __ CheckPageFlag(value_, scratch0_,
262 MemoryChunk::kPointersToHereAreInterestingMask, zero,
263 exit());
264 RememberedSetAction const remembered_set_action =
265 mode_ > RecordWriteMode::kValueIsMap ? EMIT_REMEMBERED_SET
266 : OMIT_REMEMBERED_SET;
267 SaveFPRegsMode const save_fp_mode =
268 frame()->DidAllocateDoubleRegisters() ? kSaveFPRegs : kDontSaveFPRegs;
269 RecordWriteStub stub(isolate(), object_, scratch0_, scratch1_,
270 remembered_set_action, save_fp_mode);
271 __ lea(scratch1_, operand_);
272 __ CallStub(&stub);
273 }
274
275 private:
276 Register const object_;
277 Operand const operand_;
278 Register const value_;
279 Register const scratch0_;
280 Register const scratch1_;
281 RecordWriteMode const mode_;
282 };
283
284 } // namespace
285
286 #define ASSEMBLE_CHECKED_LOAD_FLOAT(asm_instr, OutOfLineLoadNaN, \
287 SingleOrDouble) \
288 do { \
289 auto result = i.OutputDoubleRegister(); \
290 if (instr->InputAt(0)->IsRegister()) { \
291 auto offset = i.InputRegister(0); \
292 if (instr->InputAt(1)->IsRegister()) { \
293 __ cmp(offset, i.InputRegister(1)); \
294 } else { \
295 __ cmp(offset, i.InputImmediate(1)); \
296 } \
297 OutOfLineCode* ool = new (zone()) OutOfLineLoadNaN(this, result); \
298 __ j(above_equal, ool->entry()); \
299 __ asm_instr(result, i.MemoryOperand(2)); \
300 __ bind(ool->exit()); \
301 } else { \
302 auto index2 = i.InputInt32(0); \
303 auto length = i.InputInt32(1); \
304 auto index1 = i.InputRegister(2); \
305 RelocInfo::Mode rmode_length = i.ToConstant(instr->InputAt(1)).rmode(); \
306 RelocInfo::Mode rmode_buffer = i.ToConstant(instr->InputAt(3)).rmode(); \
307 DCHECK_LE(index2, length); \
308 __ cmp(index1, Immediate(reinterpret_cast<Address>(length - index2), \
309 rmode_length)); \
310 class OutOfLineLoadFloat final : public OutOfLineCode { \
311 public: \
312 OutOfLineLoadFloat(CodeGenerator* gen, XMMRegister result, \
313 Register buffer, Register index1, int32_t index2, \
314 int32_t length, RelocInfo::Mode rmode_length, \
315 RelocInfo::Mode rmode_buffer) \
316 : OutOfLineCode(gen), \
317 result_(result), \
318 buffer_reg_(buffer), \
319 buffer_int_(0), \
320 index1_(index1), \
321 index2_(index2), \
322 length_(length), \
323 rmode_length_(rmode_length), \
324 rmode_buffer_(rmode_buffer) {} \
325 \
326 OutOfLineLoadFloat(CodeGenerator* gen, XMMRegister result, \
327 int32_t buffer, Register index1, int32_t index2, \
328 int32_t length, RelocInfo::Mode rmode_length, \
329 RelocInfo::Mode rmode_buffer) \
330 : OutOfLineCode(gen), \
331 result_(result), \
332 buffer_reg_({-1}), \
333 buffer_int_(buffer), \
334 index1_(index1), \
335 index2_(index2), \
336 length_(length), \
337 rmode_length_(rmode_length), \
338 rmode_buffer_(rmode_buffer) {} \
339 \
340 void Generate() final { \
341 Label oob; \
342 __ push(index1_); \
343 __ lea(index1_, Operand(index1_, index2_)); \
344 __ cmp(index1_, Immediate(reinterpret_cast<Address>(length_), \
345 rmode_length_)); \
346 __ j(above_equal, &oob, Label::kNear); \
347 if (buffer_reg_.is_valid()) { \
348 __ asm_instr(result_, Operand(buffer_reg_, index1_, times_1, 0)); \
349 } else { \
350 __ asm_instr(result_, \
351 Operand(index1_, buffer_int_, rmode_buffer_)); \
352 } \
353 __ pop(index1_); \
354 __ jmp(exit()); \
355 __ bind(&oob); \
356 __ pop(index1_); \
357 __ xorp##SingleOrDouble(result_, result_); \
358 __ divs##SingleOrDouble(result_, result_); \
359 } \
360 \
361 private: \
362 XMMRegister const result_; \
363 Register const buffer_reg_; \
364 int32_t const buffer_int_; \
365 Register const index1_; \
366 int32_t const index2_; \
367 int32_t const length_; \
368 RelocInfo::Mode rmode_length_; \
369 RelocInfo::Mode rmode_buffer_; \
370 }; \
371 if (instr->InputAt(3)->IsRegister()) { \
372 auto buffer = i.InputRegister(3); \
373 OutOfLineCode* ool = new (zone()) \
374 OutOfLineLoadFloat(this, result, buffer, index1, index2, length, \
375 rmode_length, rmode_buffer); \
376 __ j(above_equal, ool->entry()); \
377 __ asm_instr(result, Operand(buffer, index1, times_1, index2)); \
378 __ bind(ool->exit()); \
379 } else { \
380 auto buffer = i.InputInt32(3); \
381 OutOfLineCode* ool = new (zone()) \
382 OutOfLineLoadFloat(this, result, buffer, index1, index2, length, \
383 rmode_length, rmode_buffer); \
384 __ j(above_equal, ool->entry()); \
385 __ asm_instr(result, Operand(index1, buffer + index2, rmode_buffer)); \
386 __ bind(ool->exit()); \
387 } \
388 } \
389 } while (false)
390
391 #define ASSEMBLE_CHECKED_LOAD_INTEGER(asm_instr) \
392 do { \
393 auto result = i.OutputRegister(); \
394 if (instr->InputAt(0)->IsRegister()) { \
395 auto offset = i.InputRegister(0); \
396 if (instr->InputAt(1)->IsRegister()) { \
397 __ cmp(offset, i.InputRegister(1)); \
398 } else { \
399 __ cmp(offset, i.InputImmediate(1)); \
400 } \
401 OutOfLineCode* ool = new (zone()) OutOfLineLoadZero(this, result); \
402 __ j(above_equal, ool->entry()); \
403 __ asm_instr(result, i.MemoryOperand(2)); \
404 __ bind(ool->exit()); \
405 } else { \
406 auto index2 = i.InputInt32(0); \
407 auto length = i.InputInt32(1); \
408 auto index1 = i.InputRegister(2); \
409 RelocInfo::Mode rmode_length = i.ToConstant(instr->InputAt(1)).rmode(); \
410 RelocInfo::Mode rmode_buffer = i.ToConstant(instr->InputAt(3)).rmode(); \
411 DCHECK_LE(index2, length); \
412 __ cmp(index1, Immediate(reinterpret_cast<Address>(length - index2), \
413 rmode_length)); \
414 class OutOfLineLoadInteger final : public OutOfLineCode { \
415 public: \
416 OutOfLineLoadInteger(CodeGenerator* gen, Register result, \
417 Register buffer, Register index1, int32_t index2, \
418 int32_t length, RelocInfo::Mode rmode_length, \
419 RelocInfo::Mode rmode_buffer) \
420 : OutOfLineCode(gen), \
421 result_(result), \
422 buffer_reg_(buffer), \
423 buffer_int_(0), \
424 index1_(index1), \
425 index2_(index2), \
426 length_(length), \
427 rmode_length_(rmode_length), \
428 rmode_buffer_(rmode_buffer) {} \
429 \
430 OutOfLineLoadInteger(CodeGenerator* gen, Register result, \
431 int32_t buffer, Register index1, int32_t index2, \
432 int32_t length, RelocInfo::Mode rmode_length, \
433 RelocInfo::Mode rmode_buffer) \
434 : OutOfLineCode(gen), \
435 result_(result), \
436 buffer_reg_({-1}), \
437 buffer_int_(buffer), \
438 index1_(index1), \
439 index2_(index2), \
440 length_(length), \
441 rmode_length_(rmode_length), \
442 rmode_buffer_(rmode_buffer) {} \
443 \
444 void Generate() final { \
445 Label oob; \
446 bool need_cache = !result_.is(index1_); \
447 if (need_cache) __ push(index1_); \
448 __ lea(index1_, Operand(index1_, index2_)); \
449 __ cmp(index1_, Immediate(reinterpret_cast<Address>(length_), \
450 rmode_length_)); \
451 __ j(above_equal, &oob, Label::kNear); \
452 if (buffer_reg_.is_valid()) { \
453 __ asm_instr(result_, Operand(buffer_reg_, index1_, times_1, 0)); \
454 } else { \
455 __ asm_instr(result_, \
456 Operand(index1_, buffer_int_, rmode_buffer_)); \
457 } \
458 if (need_cache) __ pop(index1_); \
459 __ jmp(exit()); \
460 __ bind(&oob); \
461 if (need_cache) __ pop(index1_); \
462 __ xor_(result_, result_); \
463 } \
464 \
465 private: \
466 Register const result_; \
467 Register const buffer_reg_; \
468 int32_t const buffer_int_; \
469 Register const index1_; \
470 int32_t const index2_; \
471 int32_t const length_; \
472 RelocInfo::Mode rmode_length_; \
473 RelocInfo::Mode rmode_buffer_; \
474 }; \
475 if (instr->InputAt(3)->IsRegister()) { \
476 auto buffer = i.InputRegister(3); \
477 OutOfLineCode* ool = new (zone()) \
478 OutOfLineLoadInteger(this, result, buffer, index1, index2, length, \
479 rmode_length, rmode_buffer); \
480 __ j(above_equal, ool->entry()); \
481 __ asm_instr(result, Operand(buffer, index1, times_1, index2)); \
482 __ bind(ool->exit()); \
483 } else { \
484 auto buffer = i.InputInt32(3); \
485 OutOfLineCode* ool = new (zone()) \
486 OutOfLineLoadInteger(this, result, buffer, index1, index2, length, \
487 rmode_length, rmode_buffer); \
488 __ j(above_equal, ool->entry()); \
489 __ asm_instr(result, Operand(index1, buffer + index2, rmode_buffer)); \
490 __ bind(ool->exit()); \
491 } \
492 } \
493 } while (false)
494
495 #define ASSEMBLE_CHECKED_STORE_FLOAT(asm_instr) \
496 do { \
497 auto value = i.InputDoubleRegister(2); \
498 if (instr->InputAt(0)->IsRegister()) { \
499 auto offset = i.InputRegister(0); \
500 if (instr->InputAt(1)->IsRegister()) { \
501 __ cmp(offset, i.InputRegister(1)); \
502 } else { \
503 __ cmp(offset, i.InputImmediate(1)); \
504 } \
505 Label done; \
506 __ j(above_equal, &done, Label::kNear); \
507 __ asm_instr(i.MemoryOperand(3), value); \
508 __ bind(&done); \
509 } else { \
510 auto index2 = i.InputInt32(0); \
511 auto length = i.InputInt32(1); \
512 auto index1 = i.InputRegister(3); \
513 RelocInfo::Mode rmode_length = i.ToConstant(instr->InputAt(1)).rmode(); \
514 RelocInfo::Mode rmode_buffer = i.ToConstant(instr->InputAt(4)).rmode(); \
515 DCHECK_LE(index2, length); \
516 __ cmp(index1, Immediate(reinterpret_cast<Address>(length - index2), \
517 rmode_length)); \
518 class OutOfLineStoreFloat final : public OutOfLineCode { \
519 public: \
520 OutOfLineStoreFloat(CodeGenerator* gen, Register buffer, \
521 Register index1, int32_t index2, int32_t length, \
522 XMMRegister value, RelocInfo::Mode rmode_length, \
523 RelocInfo::Mode rmode_buffer) \
524 : OutOfLineCode(gen), \
525 buffer_reg_(buffer), \
526 buffer_int_(0), \
527 index1_(index1), \
528 index2_(index2), \
529 length_(length), \
530 value_(value), \
531 rmode_length_(rmode_length), \
532 rmode_buffer_(rmode_buffer) {} \
533 \
534 OutOfLineStoreFloat(CodeGenerator* gen, int32_t buffer, \
535 Register index1, int32_t index2, int32_t length, \
536 XMMRegister value, RelocInfo::Mode rmode_length, \
537 RelocInfo::Mode rmode_buffer) \
538 : OutOfLineCode(gen), \
539 buffer_reg_({-1}), \
540 buffer_int_(buffer), \
541 index1_(index1), \
542 index2_(index2), \
543 length_(length), \
544 value_(value), \
545 rmode_length_(rmode_length), \
546 rmode_buffer_(rmode_buffer) {} \
547 \
548 void Generate() final { \
549 Label oob; \
550 __ push(index1_); \
551 __ lea(index1_, Operand(index1_, index2_)); \
552 __ cmp(index1_, Immediate(reinterpret_cast<Address>(length_), \
553 rmode_length_)); \
554 __ j(above_equal, &oob, Label::kNear); \
555 if (buffer_reg_.is_valid()) { \
556 __ asm_instr(Operand(buffer_reg_, index1_, times_1, 0), value_); \
557 } else { \
558 __ asm_instr(Operand(index1_, buffer_int_, rmode_buffer_), \
559 value_); \
560 } \
561 __ bind(&oob); \
562 __ pop(index1_); \
563 } \
564 \
565 private: \
566 Register const buffer_reg_; \
567 int32_t const buffer_int_; \
568 Register const index1_; \
569 int32_t const index2_; \
570 int32_t const length_; \
571 XMMRegister const value_; \
572 RelocInfo::Mode rmode_length_; \
573 RelocInfo::Mode rmode_buffer_; \
574 }; \
575 if (instr->InputAt(4)->IsRegister()) { \
576 auto buffer = i.InputRegister(4); \
577 OutOfLineCode* ool = new (zone()) \
578 OutOfLineStoreFloat(this, buffer, index1, index2, length, value, \
579 rmode_length, rmode_buffer); \
580 __ j(above_equal, ool->entry()); \
581 __ asm_instr(Operand(buffer, index1, times_1, index2), value); \
582 __ bind(ool->exit()); \
583 } else { \
584 auto buffer = i.InputInt32(4); \
585 OutOfLineCode* ool = new (zone()) \
586 OutOfLineStoreFloat(this, buffer, index1, index2, length, value, \
587 rmode_length, rmode_buffer); \
588 __ j(above_equal, ool->entry()); \
589 __ asm_instr(Operand(index1, buffer + index2, rmode_buffer), value); \
590 __ bind(ool->exit()); \
591 } \
592 } \
593 } while (false)
594
595 #define ASSEMBLE_CHECKED_STORE_INTEGER_IMPL(asm_instr, Value) \
596 do { \
597 if (instr->InputAt(0)->IsRegister()) { \
598 auto offset = i.InputRegister(0); \
599 if (instr->InputAt(1)->IsRegister()) { \
600 __ cmp(offset, i.InputRegister(1)); \
601 } else { \
602 __ cmp(offset, i.InputImmediate(1)); \
603 } \
604 Label done; \
605 __ j(above_equal, &done, Label::kNear); \
606 __ asm_instr(i.MemoryOperand(3), value); \
607 __ bind(&done); \
608 } else { \
609 auto index2 = i.InputInt32(0); \
610 auto length = i.InputInt32(1); \
611 auto index1 = i.InputRegister(3); \
612 RelocInfo::Mode rmode_length = i.ToConstant(instr->InputAt(1)).rmode(); \
613 RelocInfo::Mode rmode_buffer = i.ToConstant(instr->InputAt(4)).rmode(); \
614 DCHECK_LE(index2, length); \
615 __ cmp(index1, Immediate(reinterpret_cast<Address>(length - index2), \
616 rmode_length)); \
617 class OutOfLineStoreInteger final : public OutOfLineCode { \
618 public: \
619 OutOfLineStoreInteger(CodeGenerator* gen, Register buffer, \
620 Register index1, int32_t index2, int32_t length, \
621 Value value, RelocInfo::Mode rmode_length, \
622 RelocInfo::Mode rmode_buffer) \
623 : OutOfLineCode(gen), \
624 buffer_reg_(buffer), \
625 buffer_int_(0), \
626 index1_(index1), \
627 index2_(index2), \
628 length_(length), \
629 value_(value), \
630 rmode_length_(rmode_length), \
631 rmode_buffer_(rmode_buffer) {} \
632 \
633 OutOfLineStoreInteger(CodeGenerator* gen, int32_t buffer, \
634 Register index1, int32_t index2, int32_t length, \
635 Value value, RelocInfo::Mode rmode_length, \
636 RelocInfo::Mode rmode_buffer) \
637 : OutOfLineCode(gen), \
638 buffer_reg_({-1}), \
639 buffer_int_(buffer), \
640 index1_(index1), \
641 index2_(index2), \
642 length_(length), \
643 value_(value), \
644 rmode_length_(rmode_length), \
645 rmode_buffer_(rmode_buffer) {} \
646 \
647 void Generate() final { \
648 Label oob; \
649 __ push(index1_); \
650 __ lea(index1_, Operand(index1_, index2_)); \
651 __ cmp(index1_, Immediate(reinterpret_cast<Address>(length_), \
652 rmode_length_)); \
653 __ j(above_equal, &oob, Label::kNear); \
654 if (buffer_reg_.is_valid()) { \
655 __ asm_instr(Operand(buffer_reg_, index1_, times_1, 0), value_); \
656 } else { \
657 __ asm_instr(Operand(index1_, buffer_int_, rmode_buffer_), \
658 value_); \
659 } \
660 __ bind(&oob); \
661 __ pop(index1_); \
662 } \
663 \
664 private: \
665 Register const buffer_reg_; \
666 int32_t const buffer_int_; \
667 Register const index1_; \
668 int32_t const index2_; \
669 int32_t const length_; \
670 Value const value_; \
671 RelocInfo::Mode rmode_length_; \
672 RelocInfo::Mode rmode_buffer_; \
673 }; \
674 if (instr->InputAt(4)->IsRegister()) { \
675 auto buffer = i.InputRegister(4); \
676 OutOfLineCode* ool = new (zone()) \
677 OutOfLineStoreInteger(this, buffer, index1, index2, length, value, \
678 rmode_length, rmode_buffer); \
679 __ j(above_equal, ool->entry()); \
680 __ asm_instr(Operand(buffer, index1, times_1, index2), value); \
681 __ bind(ool->exit()); \
682 } else { \
683 auto buffer = i.InputInt32(4); \
684 OutOfLineCode* ool = new (zone()) \
685 OutOfLineStoreInteger(this, buffer, index1, index2, length, value, \
686 rmode_length, rmode_buffer); \
687 __ j(above_equal, ool->entry()); \
688 __ asm_instr(Operand(index1, buffer + index2, rmode_buffer), value); \
689 __ bind(ool->exit()); \
690 } \
691 } \
692 } while (false)
693
694 #define ASSEMBLE_CHECKED_STORE_INTEGER(asm_instr) \
695 do { \
696 if (instr->InputAt(2)->IsRegister()) { \
697 Register value = i.InputRegister(2); \
698 ASSEMBLE_CHECKED_STORE_INTEGER_IMPL(asm_instr, Register); \
699 } else { \
700 Immediate value = i.InputImmediate(2); \
701 ASSEMBLE_CHECKED_STORE_INTEGER_IMPL(asm_instr, Immediate); \
702 } \
703 } while (false)
704
705 #define ASSEMBLE_COMPARE(asm_instr) \
706 do { \
707 if (AddressingModeField::decode(instr->opcode()) != kMode_None) { \
708 size_t index = 0; \
709 Operand left = i.MemoryOperand(&index); \
710 if (HasImmediateInput(instr, index)) { \
711 __ asm_instr(left, i.InputImmediate(index)); \
712 } else { \
713 __ asm_instr(left, i.InputRegister(index)); \
714 } \
715 } else { \
716 if (HasImmediateInput(instr, 1)) { \
717 if (instr->InputAt(0)->IsRegister()) { \
718 __ asm_instr(i.InputRegister(0), i.InputImmediate(1)); \
719 } else { \
720 __ asm_instr(i.InputOperand(0), i.InputImmediate(1)); \
721 } \
722 } else { \
723 if (instr->InputAt(1)->IsRegister()) { \
724 __ asm_instr(i.InputRegister(0), i.InputRegister(1)); \
725 } else { \
726 __ asm_instr(i.InputRegister(0), i.InputOperand(1)); \
727 } \
728 } \
729 } \
730 } while (0)
731
732 #define ASSEMBLE_IEEE754_BINOP(name) \
733 do { \
734 /* Pass two doubles as arguments on the stack. */ \
735 __ PrepareCallCFunction(4, eax); \
736 __ movsd(Operand(esp, 0 * kDoubleSize), i.InputDoubleRegister(0)); \
737 __ movsd(Operand(esp, 1 * kDoubleSize), i.InputDoubleRegister(1)); \
738 __ CallCFunction(ExternalReference::ieee754_##name##_function(isolate()), \
739 4); \
740 /* Return value is in st(0) on ia32. */ \
741 /* Store it into the result register. */ \
742 __ sub(esp, Immediate(kDoubleSize)); \
743 __ fstp_d(Operand(esp, 0)); \
744 __ movsd(i.OutputDoubleRegister(), Operand(esp, 0)); \
745 __ add(esp, Immediate(kDoubleSize)); \
746 } while (false)
747
748 #define ASSEMBLE_IEEE754_UNOP(name) \
749 do { \
750 /* Pass one double as argument on the stack. */ \
751 __ PrepareCallCFunction(2, eax); \
752 __ movsd(Operand(esp, 0 * kDoubleSize), i.InputDoubleRegister(0)); \
753 __ CallCFunction(ExternalReference::ieee754_##name##_function(isolate()), \
754 2); \
755 /* Return value is in st(0) on ia32. */ \
756 /* Store it into the result register. */ \
757 __ sub(esp, Immediate(kDoubleSize)); \
758 __ fstp_d(Operand(esp, 0)); \
759 __ movsd(i.OutputDoubleRegister(), Operand(esp, 0)); \
760 __ add(esp, Immediate(kDoubleSize)); \
761 } while (false)
762
AssembleDeconstructFrame()763 void CodeGenerator::AssembleDeconstructFrame() {
764 __ mov(esp, ebp);
765 __ pop(ebp);
766 }
767
AssemblePrepareTailCall()768 void CodeGenerator::AssemblePrepareTailCall() {
769 if (frame_access_state()->has_frame()) {
770 __ mov(ebp, MemOperand(ebp, 0));
771 }
772 frame_access_state()->SetFrameAccessToSP();
773 }
774
AssemblePopArgumentsAdaptorFrame(Register args_reg,Register,Register,Register)775 void CodeGenerator::AssemblePopArgumentsAdaptorFrame(Register args_reg,
776 Register, Register,
777 Register) {
778 // There are not enough temp registers left on ia32 for a call instruction
779 // so we pick some scratch registers and save/restore them manually here.
780 int scratch_count = 3;
781 Register scratch1 = ebx;
782 Register scratch2 = ecx;
783 Register scratch3 = edx;
784 DCHECK(!AreAliased(args_reg, scratch1, scratch2, scratch3));
785 Label done;
786
787 // Check if current frame is an arguments adaptor frame.
788 __ cmp(Operand(ebp, StandardFrameConstants::kContextOffset),
789 Immediate(StackFrame::TypeToMarker(StackFrame::ARGUMENTS_ADAPTOR)));
790 __ j(not_equal, &done, Label::kNear);
791
792 __ push(scratch1);
793 __ push(scratch2);
794 __ push(scratch3);
795
796 // Load arguments count from current arguments adaptor frame (note, it
797 // does not include receiver).
798 Register caller_args_count_reg = scratch1;
799 __ mov(caller_args_count_reg,
800 Operand(ebp, ArgumentsAdaptorFrameConstants::kLengthOffset));
801 __ SmiUntag(caller_args_count_reg);
802
803 ParameterCount callee_args_count(args_reg);
804 __ PrepareForTailCall(callee_args_count, caller_args_count_reg, scratch2,
805 scratch3, ReturnAddressState::kOnStack, scratch_count);
806 __ pop(scratch3);
807 __ pop(scratch2);
808 __ pop(scratch1);
809
810 __ bind(&done);
811 }
812
813 namespace {
814
AdjustStackPointerForTailCall(MacroAssembler * masm,FrameAccessState * state,int new_slot_above_sp,bool allow_shrinkage=true)815 void AdjustStackPointerForTailCall(MacroAssembler* masm,
816 FrameAccessState* state,
817 int new_slot_above_sp,
818 bool allow_shrinkage = true) {
819 int current_sp_offset = state->GetSPToFPSlotCount() +
820 StandardFrameConstants::kFixedSlotCountAboveFp;
821 int stack_slot_delta = new_slot_above_sp - current_sp_offset;
822 if (stack_slot_delta > 0) {
823 masm->sub(esp, Immediate(stack_slot_delta * kPointerSize));
824 state->IncreaseSPDelta(stack_slot_delta);
825 } else if (allow_shrinkage && stack_slot_delta < 0) {
826 masm->add(esp, Immediate(-stack_slot_delta * kPointerSize));
827 state->IncreaseSPDelta(stack_slot_delta);
828 }
829 }
830
831 } // namespace
832
AssembleTailCallBeforeGap(Instruction * instr,int first_unused_stack_slot)833 void CodeGenerator::AssembleTailCallBeforeGap(Instruction* instr,
834 int first_unused_stack_slot) {
835 CodeGenerator::PushTypeFlags flags(kImmediatePush | kScalarPush);
836 ZoneVector<MoveOperands*> pushes(zone());
837 GetPushCompatibleMoves(instr, flags, &pushes);
838
839 if (!pushes.empty() &&
840 (LocationOperand::cast(pushes.back()->destination()).index() + 1 ==
841 first_unused_stack_slot)) {
842 IA32OperandConverter g(this, instr);
843 for (auto move : pushes) {
844 LocationOperand destination_location(
845 LocationOperand::cast(move->destination()));
846 InstructionOperand source(move->source());
847 AdjustStackPointerForTailCall(masm(), frame_access_state(),
848 destination_location.index());
849 if (source.IsStackSlot()) {
850 LocationOperand source_location(LocationOperand::cast(source));
851 __ push(g.SlotToOperand(source_location.index()));
852 } else if (source.IsRegister()) {
853 LocationOperand source_location(LocationOperand::cast(source));
854 __ push(source_location.GetRegister());
855 } else if (source.IsImmediate()) {
856 __ push(Immediate(ImmediateOperand::cast(source).inline_value()));
857 } else {
858 // Pushes of non-scalar data types is not supported.
859 UNIMPLEMENTED();
860 }
861 frame_access_state()->IncreaseSPDelta(1);
862 move->Eliminate();
863 }
864 }
865 AdjustStackPointerForTailCall(masm(), frame_access_state(),
866 first_unused_stack_slot, false);
867 }
868
AssembleTailCallAfterGap(Instruction * instr,int first_unused_stack_slot)869 void CodeGenerator::AssembleTailCallAfterGap(Instruction* instr,
870 int first_unused_stack_slot) {
871 AdjustStackPointerForTailCall(masm(), frame_access_state(),
872 first_unused_stack_slot);
873 }
874
875 // Assembles an instruction after register allocation, producing machine code.
AssembleArchInstruction(Instruction * instr)876 CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
877 Instruction* instr) {
878 IA32OperandConverter i(this, instr);
879 InstructionCode opcode = instr->opcode();
880 ArchOpcode arch_opcode = ArchOpcodeField::decode(opcode);
881 switch (arch_opcode) {
882 case kArchCallCodeObject: {
883 EnsureSpaceForLazyDeopt();
884 if (HasImmediateInput(instr, 0)) {
885 Handle<Code> code = Handle<Code>::cast(i.InputHeapObject(0));
886 __ call(code, RelocInfo::CODE_TARGET);
887 } else {
888 Register reg = i.InputRegister(0);
889 __ add(reg, Immediate(Code::kHeaderSize - kHeapObjectTag));
890 __ call(reg);
891 }
892 RecordCallPosition(instr);
893 frame_access_state()->ClearSPDelta();
894 break;
895 }
896 case kArchTailCallCodeObjectFromJSFunction:
897 case kArchTailCallCodeObject: {
898 if (arch_opcode == kArchTailCallCodeObjectFromJSFunction) {
899 AssemblePopArgumentsAdaptorFrame(kJavaScriptCallArgCountRegister,
900 no_reg, no_reg, no_reg);
901 }
902 if (HasImmediateInput(instr, 0)) {
903 Handle<Code> code = Handle<Code>::cast(i.InputHeapObject(0));
904 __ jmp(code, RelocInfo::CODE_TARGET);
905 } else {
906 Register reg = i.InputRegister(0);
907 __ add(reg, Immediate(Code::kHeaderSize - kHeapObjectTag));
908 __ jmp(reg);
909 }
910 frame_access_state()->ClearSPDelta();
911 frame_access_state()->SetFrameAccessToDefault();
912 break;
913 }
914 case kArchTailCallAddress: {
915 CHECK(!HasImmediateInput(instr, 0));
916 Register reg = i.InputRegister(0);
917 __ jmp(reg);
918 frame_access_state()->ClearSPDelta();
919 frame_access_state()->SetFrameAccessToDefault();
920 break;
921 }
922 case kArchCallJSFunction: {
923 EnsureSpaceForLazyDeopt();
924 Register func = i.InputRegister(0);
925 if (FLAG_debug_code) {
926 // Check the function's context matches the context argument.
927 __ cmp(esi, FieldOperand(func, JSFunction::kContextOffset));
928 __ Assert(equal, kWrongFunctionContext);
929 }
930 __ call(FieldOperand(func, JSFunction::kCodeEntryOffset));
931 RecordCallPosition(instr);
932 frame_access_state()->ClearSPDelta();
933 break;
934 }
935 case kArchTailCallJSFunctionFromJSFunction: {
936 Register func = i.InputRegister(0);
937 if (FLAG_debug_code) {
938 // Check the function's context matches the context argument.
939 __ cmp(esi, FieldOperand(func, JSFunction::kContextOffset));
940 __ Assert(equal, kWrongFunctionContext);
941 }
942 AssemblePopArgumentsAdaptorFrame(kJavaScriptCallArgCountRegister, no_reg,
943 no_reg, no_reg);
944 __ jmp(FieldOperand(func, JSFunction::kCodeEntryOffset));
945 frame_access_state()->ClearSPDelta();
946 frame_access_state()->SetFrameAccessToDefault();
947 break;
948 }
949 case kArchPrepareCallCFunction: {
950 // Frame alignment requires using FP-relative frame addressing.
951 frame_access_state()->SetFrameAccessToFP();
952 int const num_parameters = MiscField::decode(instr->opcode());
953 __ PrepareCallCFunction(num_parameters, i.TempRegister(0));
954 break;
955 }
956 case kArchPrepareTailCall:
957 AssemblePrepareTailCall();
958 break;
959 case kArchCallCFunction: {
960 int const num_parameters = MiscField::decode(instr->opcode());
961 if (HasImmediateInput(instr, 0)) {
962 ExternalReference ref = i.InputExternalReference(0);
963 __ CallCFunction(ref, num_parameters);
964 } else {
965 Register func = i.InputRegister(0);
966 __ CallCFunction(func, num_parameters);
967 }
968 frame_access_state()->SetFrameAccessToDefault();
969 frame_access_state()->ClearSPDelta();
970 break;
971 }
972 case kArchJmp:
973 AssembleArchJump(i.InputRpo(0));
974 break;
975 case kArchLookupSwitch:
976 AssembleArchLookupSwitch(instr);
977 break;
978 case kArchTableSwitch:
979 AssembleArchTableSwitch(instr);
980 break;
981 case kArchComment: {
982 Address comment_string = i.InputExternalReference(0).address();
983 __ RecordComment(reinterpret_cast<const char*>(comment_string));
984 break;
985 }
986 case kArchDebugBreak:
987 __ int3();
988 break;
989 case kArchNop:
990 case kArchThrowTerminator:
991 // don't emit code for nops.
992 break;
993 case kArchDeoptimize: {
994 int deopt_state_id =
995 BuildTranslation(instr, -1, 0, OutputFrameStateCombine::Ignore());
996 CodeGenResult result =
997 AssembleDeoptimizerCall(deopt_state_id, current_source_position_);
998 if (result != kSuccess) return result;
999 break;
1000 }
1001 case kArchRet:
1002 AssembleReturn(instr->InputAt(0));
1003 break;
1004 case kArchStackPointer:
1005 __ mov(i.OutputRegister(), esp);
1006 break;
1007 case kArchFramePointer:
1008 __ mov(i.OutputRegister(), ebp);
1009 break;
1010 case kArchParentFramePointer:
1011 if (frame_access_state()->has_frame()) {
1012 __ mov(i.OutputRegister(), Operand(ebp, 0));
1013 } else {
1014 __ mov(i.OutputRegister(), ebp);
1015 }
1016 break;
1017 case kArchTruncateDoubleToI: {
1018 auto result = i.OutputRegister();
1019 auto input = i.InputDoubleRegister(0);
1020 auto ool = new (zone()) OutOfLineTruncateDoubleToI(this, result, input);
1021 __ cvttsd2si(result, Operand(input));
1022 __ cmp(result, 1);
1023 __ j(overflow, ool->entry());
1024 __ bind(ool->exit());
1025 break;
1026 }
1027 case kArchStoreWithWriteBarrier: {
1028 RecordWriteMode mode =
1029 static_cast<RecordWriteMode>(MiscField::decode(instr->opcode()));
1030 Register object = i.InputRegister(0);
1031 size_t index = 0;
1032 Operand operand = i.MemoryOperand(&index);
1033 Register value = i.InputRegister(index);
1034 Register scratch0 = i.TempRegister(0);
1035 Register scratch1 = i.TempRegister(1);
1036 auto ool = new (zone()) OutOfLineRecordWrite(this, object, operand, value,
1037 scratch0, scratch1, mode);
1038 __ mov(operand, value);
1039 __ CheckPageFlag(object, scratch0,
1040 MemoryChunk::kPointersFromHereAreInterestingMask,
1041 not_zero, ool->entry());
1042 __ bind(ool->exit());
1043 break;
1044 }
1045 case kArchStackSlot: {
1046 FrameOffset offset =
1047 frame_access_state()->GetFrameOffset(i.InputInt32(0));
1048 Register base;
1049 if (offset.from_stack_pointer()) {
1050 base = esp;
1051 } else {
1052 base = ebp;
1053 }
1054 __ lea(i.OutputRegister(), Operand(base, offset.offset()));
1055 break;
1056 }
1057 case kIeee754Float64Acos:
1058 ASSEMBLE_IEEE754_UNOP(acos);
1059 break;
1060 case kIeee754Float64Acosh:
1061 ASSEMBLE_IEEE754_UNOP(acosh);
1062 break;
1063 case kIeee754Float64Asin:
1064 ASSEMBLE_IEEE754_UNOP(asin);
1065 break;
1066 case kIeee754Float64Asinh:
1067 ASSEMBLE_IEEE754_UNOP(asinh);
1068 break;
1069 case kIeee754Float64Atan:
1070 ASSEMBLE_IEEE754_UNOP(atan);
1071 break;
1072 case kIeee754Float64Atanh:
1073 ASSEMBLE_IEEE754_UNOP(atanh);
1074 break;
1075 case kIeee754Float64Atan2:
1076 ASSEMBLE_IEEE754_BINOP(atan2);
1077 break;
1078 case kIeee754Float64Cbrt:
1079 ASSEMBLE_IEEE754_UNOP(cbrt);
1080 break;
1081 case kIeee754Float64Cos:
1082 ASSEMBLE_IEEE754_UNOP(cos);
1083 break;
1084 case kIeee754Float64Cosh:
1085 ASSEMBLE_IEEE754_UNOP(cosh);
1086 break;
1087 case kIeee754Float64Expm1:
1088 ASSEMBLE_IEEE754_UNOP(expm1);
1089 break;
1090 case kIeee754Float64Exp:
1091 ASSEMBLE_IEEE754_UNOP(exp);
1092 break;
1093 case kIeee754Float64Log:
1094 ASSEMBLE_IEEE754_UNOP(log);
1095 break;
1096 case kIeee754Float64Log1p:
1097 ASSEMBLE_IEEE754_UNOP(log1p);
1098 break;
1099 case kIeee754Float64Log2:
1100 ASSEMBLE_IEEE754_UNOP(log2);
1101 break;
1102 case kIeee754Float64Log10:
1103 ASSEMBLE_IEEE754_UNOP(log10);
1104 break;
1105 case kIeee754Float64Pow: {
1106 // TODO(bmeurer): Improve integration of the stub.
1107 if (!i.InputDoubleRegister(1).is(xmm2)) {
1108 __ movaps(xmm2, i.InputDoubleRegister(0));
1109 __ movaps(xmm1, i.InputDoubleRegister(1));
1110 } else {
1111 __ movaps(xmm0, i.InputDoubleRegister(0));
1112 __ movaps(xmm1, xmm2);
1113 __ movaps(xmm2, xmm0);
1114 }
1115 MathPowStub stub(isolate(), MathPowStub::DOUBLE);
1116 __ CallStub(&stub);
1117 __ movaps(i.OutputDoubleRegister(), xmm3);
1118 break;
1119 }
1120 case kIeee754Float64Sin:
1121 ASSEMBLE_IEEE754_UNOP(sin);
1122 break;
1123 case kIeee754Float64Sinh:
1124 ASSEMBLE_IEEE754_UNOP(sinh);
1125 break;
1126 case kIeee754Float64Tan:
1127 ASSEMBLE_IEEE754_UNOP(tan);
1128 break;
1129 case kIeee754Float64Tanh:
1130 ASSEMBLE_IEEE754_UNOP(tanh);
1131 break;
1132 case kIA32Add:
1133 if (HasImmediateInput(instr, 1)) {
1134 __ add(i.InputOperand(0), i.InputImmediate(1));
1135 } else {
1136 __ add(i.InputRegister(0), i.InputOperand(1));
1137 }
1138 break;
1139 case kIA32And:
1140 if (HasImmediateInput(instr, 1)) {
1141 __ and_(i.InputOperand(0), i.InputImmediate(1));
1142 } else {
1143 __ and_(i.InputRegister(0), i.InputOperand(1));
1144 }
1145 break;
1146 case kIA32Cmp:
1147 ASSEMBLE_COMPARE(cmp);
1148 break;
1149 case kIA32Cmp16:
1150 ASSEMBLE_COMPARE(cmpw);
1151 break;
1152 case kIA32Cmp8:
1153 ASSEMBLE_COMPARE(cmpb);
1154 break;
1155 case kIA32Test:
1156 ASSEMBLE_COMPARE(test);
1157 break;
1158 case kIA32Test16:
1159 ASSEMBLE_COMPARE(test_w);
1160 break;
1161 case kIA32Test8:
1162 ASSEMBLE_COMPARE(test_b);
1163 break;
1164 case kIA32Imul:
1165 if (HasImmediateInput(instr, 1)) {
1166 __ imul(i.OutputRegister(), i.InputOperand(0), i.InputInt32(1));
1167 } else {
1168 __ imul(i.OutputRegister(), i.InputOperand(1));
1169 }
1170 break;
1171 case kIA32ImulHigh:
1172 __ imul(i.InputRegister(1));
1173 break;
1174 case kIA32UmulHigh:
1175 __ mul(i.InputRegister(1));
1176 break;
1177 case kIA32Idiv:
1178 __ cdq();
1179 __ idiv(i.InputOperand(1));
1180 break;
1181 case kIA32Udiv:
1182 __ Move(edx, Immediate(0));
1183 __ div(i.InputOperand(1));
1184 break;
1185 case kIA32Not:
1186 __ not_(i.OutputOperand());
1187 break;
1188 case kIA32Neg:
1189 __ neg(i.OutputOperand());
1190 break;
1191 case kIA32Or:
1192 if (HasImmediateInput(instr, 1)) {
1193 __ or_(i.InputOperand(0), i.InputImmediate(1));
1194 } else {
1195 __ or_(i.InputRegister(0), i.InputOperand(1));
1196 }
1197 break;
1198 case kIA32Xor:
1199 if (HasImmediateInput(instr, 1)) {
1200 __ xor_(i.InputOperand(0), i.InputImmediate(1));
1201 } else {
1202 __ xor_(i.InputRegister(0), i.InputOperand(1));
1203 }
1204 break;
1205 case kIA32Sub:
1206 if (HasImmediateInput(instr, 1)) {
1207 __ sub(i.InputOperand(0), i.InputImmediate(1));
1208 } else {
1209 __ sub(i.InputRegister(0), i.InputOperand(1));
1210 }
1211 break;
1212 case kIA32Shl:
1213 if (HasImmediateInput(instr, 1)) {
1214 __ shl(i.OutputOperand(), i.InputInt5(1));
1215 } else {
1216 __ shl_cl(i.OutputOperand());
1217 }
1218 break;
1219 case kIA32Shr:
1220 if (HasImmediateInput(instr, 1)) {
1221 __ shr(i.OutputOperand(), i.InputInt5(1));
1222 } else {
1223 __ shr_cl(i.OutputOperand());
1224 }
1225 break;
1226 case kIA32Sar:
1227 if (HasImmediateInput(instr, 1)) {
1228 __ sar(i.OutputOperand(), i.InputInt5(1));
1229 } else {
1230 __ sar_cl(i.OutputOperand());
1231 }
1232 break;
1233 case kIA32AddPair: {
1234 // i.OutputRegister(0) == i.InputRegister(0) ... left low word.
1235 // i.InputRegister(1) ... left high word.
1236 // i.InputRegister(2) ... right low word.
1237 // i.InputRegister(3) ... right high word.
1238 bool use_temp = false;
1239 if (i.OutputRegister(0).code() == i.InputRegister(1).code() ||
1240 i.OutputRegister(0).code() == i.InputRegister(3).code()) {
1241 // We cannot write to the output register directly, because it would
1242 // overwrite an input for adc. We have to use the temp register.
1243 use_temp = true;
1244 __ Move(i.TempRegister(0), i.InputRegister(0));
1245 __ add(i.TempRegister(0), i.InputRegister(2));
1246 } else {
1247 __ add(i.OutputRegister(0), i.InputRegister(2));
1248 }
1249 if (i.OutputRegister(1).code() != i.InputRegister(1).code()) {
1250 __ Move(i.OutputRegister(1), i.InputRegister(1));
1251 }
1252 __ adc(i.OutputRegister(1), Operand(i.InputRegister(3)));
1253 if (use_temp) {
1254 __ Move(i.OutputRegister(0), i.TempRegister(0));
1255 }
1256 break;
1257 }
1258 case kIA32SubPair: {
1259 // i.OutputRegister(0) == i.InputRegister(0) ... left low word.
1260 // i.InputRegister(1) ... left high word.
1261 // i.InputRegister(2) ... right low word.
1262 // i.InputRegister(3) ... right high word.
1263 bool use_temp = false;
1264 if (i.OutputRegister(0).code() == i.InputRegister(1).code() ||
1265 i.OutputRegister(0).code() == i.InputRegister(3).code()) {
1266 // We cannot write to the output register directly, because it would
1267 // overwrite an input for adc. We have to use the temp register.
1268 use_temp = true;
1269 __ Move(i.TempRegister(0), i.InputRegister(0));
1270 __ sub(i.TempRegister(0), i.InputRegister(2));
1271 } else {
1272 __ sub(i.OutputRegister(0), i.InputRegister(2));
1273 }
1274 if (i.OutputRegister(1).code() != i.InputRegister(1).code()) {
1275 __ Move(i.OutputRegister(1), i.InputRegister(1));
1276 }
1277 __ sbb(i.OutputRegister(1), Operand(i.InputRegister(3)));
1278 if (use_temp) {
1279 __ Move(i.OutputRegister(0), i.TempRegister(0));
1280 }
1281 break;
1282 }
1283 case kIA32MulPair: {
1284 __ imul(i.OutputRegister(1), i.InputOperand(0));
1285 __ mov(i.TempRegister(0), i.InputOperand(1));
1286 __ imul(i.TempRegister(0), i.InputOperand(2));
1287 __ add(i.OutputRegister(1), i.TempRegister(0));
1288 __ mov(i.OutputRegister(0), i.InputOperand(0));
1289 // Multiplies the low words and stores them in eax and edx.
1290 __ mul(i.InputRegister(2));
1291 __ add(i.OutputRegister(1), i.TempRegister(0));
1292
1293 break;
1294 }
1295 case kIA32ShlPair:
1296 if (HasImmediateInput(instr, 2)) {
1297 __ ShlPair(i.InputRegister(1), i.InputRegister(0), i.InputInt6(2));
1298 } else {
1299 // Shift has been loaded into CL by the register allocator.
1300 __ ShlPair_cl(i.InputRegister(1), i.InputRegister(0));
1301 }
1302 break;
1303 case kIA32ShrPair:
1304 if (HasImmediateInput(instr, 2)) {
1305 __ ShrPair(i.InputRegister(1), i.InputRegister(0), i.InputInt6(2));
1306 } else {
1307 // Shift has been loaded into CL by the register allocator.
1308 __ ShrPair_cl(i.InputRegister(1), i.InputRegister(0));
1309 }
1310 break;
1311 case kIA32SarPair:
1312 if (HasImmediateInput(instr, 2)) {
1313 __ SarPair(i.InputRegister(1), i.InputRegister(0), i.InputInt6(2));
1314 } else {
1315 // Shift has been loaded into CL by the register allocator.
1316 __ SarPair_cl(i.InputRegister(1), i.InputRegister(0));
1317 }
1318 break;
1319 case kIA32Ror:
1320 if (HasImmediateInput(instr, 1)) {
1321 __ ror(i.OutputOperand(), i.InputInt5(1));
1322 } else {
1323 __ ror_cl(i.OutputOperand());
1324 }
1325 break;
1326 case kIA32Lzcnt:
1327 __ Lzcnt(i.OutputRegister(), i.InputOperand(0));
1328 break;
1329 case kIA32Tzcnt:
1330 __ Tzcnt(i.OutputRegister(), i.InputOperand(0));
1331 break;
1332 case kIA32Popcnt:
1333 __ Popcnt(i.OutputRegister(), i.InputOperand(0));
1334 break;
1335 case kSSEFloat32Cmp:
1336 __ ucomiss(i.InputDoubleRegister(0), i.InputOperand(1));
1337 break;
1338 case kSSEFloat32Add:
1339 __ addss(i.InputDoubleRegister(0), i.InputOperand(1));
1340 break;
1341 case kSSEFloat32Sub:
1342 __ subss(i.InputDoubleRegister(0), i.InputOperand(1));
1343 break;
1344 case kSSEFloat32Mul:
1345 __ mulss(i.InputDoubleRegister(0), i.InputOperand(1));
1346 break;
1347 case kSSEFloat32Div:
1348 __ divss(i.InputDoubleRegister(0), i.InputOperand(1));
1349 // Don't delete this mov. It may improve performance on some CPUs,
1350 // when there is a (v)mulss depending on the result.
1351 __ movaps(i.OutputDoubleRegister(), i.OutputDoubleRegister());
1352 break;
1353 case kSSEFloat32Sqrt:
1354 __ sqrtss(i.OutputDoubleRegister(), i.InputOperand(0));
1355 break;
1356 case kSSEFloat32Abs: {
1357 // TODO(bmeurer): Use 128-bit constants.
1358 __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
1359 __ psrlq(kScratchDoubleReg, 33);
1360 __ andps(i.OutputDoubleRegister(), kScratchDoubleReg);
1361 break;
1362 }
1363 case kSSEFloat32Neg: {
1364 // TODO(bmeurer): Use 128-bit constants.
1365 __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
1366 __ psllq(kScratchDoubleReg, 31);
1367 __ xorps(i.OutputDoubleRegister(), kScratchDoubleReg);
1368 break;
1369 }
1370 case kSSEFloat32Round: {
1371 CpuFeatureScope sse_scope(masm(), SSE4_1);
1372 RoundingMode const mode =
1373 static_cast<RoundingMode>(MiscField::decode(instr->opcode()));
1374 __ roundss(i.OutputDoubleRegister(), i.InputDoubleRegister(0), mode);
1375 break;
1376 }
1377 case kSSEFloat64Cmp:
1378 __ ucomisd(i.InputDoubleRegister(0), i.InputOperand(1));
1379 break;
1380 case kSSEFloat64Add:
1381 __ addsd(i.InputDoubleRegister(0), i.InputOperand(1));
1382 break;
1383 case kSSEFloat64Sub:
1384 __ subsd(i.InputDoubleRegister(0), i.InputOperand(1));
1385 break;
1386 case kSSEFloat64Mul:
1387 __ mulsd(i.InputDoubleRegister(0), i.InputOperand(1));
1388 break;
1389 case kSSEFloat64Div:
1390 __ divsd(i.InputDoubleRegister(0), i.InputOperand(1));
1391 // Don't delete this mov. It may improve performance on some CPUs,
1392 // when there is a (v)mulsd depending on the result.
1393 __ movaps(i.OutputDoubleRegister(), i.OutputDoubleRegister());
1394 break;
1395 case kSSEFloat32Max: {
1396 Label compare_nan, compare_swap, done_compare;
1397 if (instr->InputAt(1)->IsFPRegister()) {
1398 __ ucomiss(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1399 } else {
1400 __ ucomiss(i.InputDoubleRegister(0), i.InputOperand(1));
1401 }
1402 auto ool =
1403 new (zone()) OutOfLineLoadFloat32NaN(this, i.OutputDoubleRegister());
1404 __ j(parity_even, ool->entry());
1405 __ j(above, &done_compare, Label::kNear);
1406 __ j(below, &compare_swap, Label::kNear);
1407 __ movmskps(i.TempRegister(0), i.InputDoubleRegister(0));
1408 __ test(i.TempRegister(0), Immediate(1));
1409 __ j(zero, &done_compare, Label::kNear);
1410 __ bind(&compare_swap);
1411 if (instr->InputAt(1)->IsFPRegister()) {
1412 __ movss(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1413 } else {
1414 __ movss(i.InputDoubleRegister(0), i.InputOperand(1));
1415 }
1416 __ bind(&done_compare);
1417 __ bind(ool->exit());
1418 break;
1419 }
1420
1421 case kSSEFloat64Max: {
1422 Label compare_nan, compare_swap, done_compare;
1423 if (instr->InputAt(1)->IsFPRegister()) {
1424 __ ucomisd(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1425 } else {
1426 __ ucomisd(i.InputDoubleRegister(0), i.InputOperand(1));
1427 }
1428 auto ool =
1429 new (zone()) OutOfLineLoadFloat64NaN(this, i.OutputDoubleRegister());
1430 __ j(parity_even, ool->entry());
1431 __ j(above, &done_compare, Label::kNear);
1432 __ j(below, &compare_swap, Label::kNear);
1433 __ movmskpd(i.TempRegister(0), i.InputDoubleRegister(0));
1434 __ test(i.TempRegister(0), Immediate(1));
1435 __ j(zero, &done_compare, Label::kNear);
1436 __ bind(&compare_swap);
1437 if (instr->InputAt(1)->IsFPRegister()) {
1438 __ movsd(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1439 } else {
1440 __ movsd(i.InputDoubleRegister(0), i.InputOperand(1));
1441 }
1442 __ bind(&done_compare);
1443 __ bind(ool->exit());
1444 break;
1445 }
1446 case kSSEFloat32Min: {
1447 Label compare_swap, done_compare;
1448 if (instr->InputAt(1)->IsFPRegister()) {
1449 __ ucomiss(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1450 } else {
1451 __ ucomiss(i.InputDoubleRegister(0), i.InputOperand(1));
1452 }
1453 auto ool =
1454 new (zone()) OutOfLineLoadFloat32NaN(this, i.OutputDoubleRegister());
1455 __ j(parity_even, ool->entry());
1456 __ j(below, &done_compare, Label::kNear);
1457 __ j(above, &compare_swap, Label::kNear);
1458 if (instr->InputAt(1)->IsFPRegister()) {
1459 __ movmskps(i.TempRegister(0), i.InputDoubleRegister(1));
1460 } else {
1461 __ movss(kScratchDoubleReg, i.InputOperand(1));
1462 __ movmskps(i.TempRegister(0), kScratchDoubleReg);
1463 }
1464 __ test(i.TempRegister(0), Immediate(1));
1465 __ j(zero, &done_compare, Label::kNear);
1466 __ bind(&compare_swap);
1467 if (instr->InputAt(1)->IsFPRegister()) {
1468 __ movss(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1469 } else {
1470 __ movss(i.InputDoubleRegister(0), i.InputOperand(1));
1471 }
1472 __ bind(&done_compare);
1473 __ bind(ool->exit());
1474 break;
1475 }
1476 case kSSEFloat64Min: {
1477 Label compare_swap, done_compare;
1478 if (instr->InputAt(1)->IsFPRegister()) {
1479 __ ucomisd(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1480 } else {
1481 __ ucomisd(i.InputDoubleRegister(0), i.InputOperand(1));
1482 }
1483 auto ool =
1484 new (zone()) OutOfLineLoadFloat64NaN(this, i.OutputDoubleRegister());
1485 __ j(parity_even, ool->entry());
1486 __ j(below, &done_compare, Label::kNear);
1487 __ j(above, &compare_swap, Label::kNear);
1488 if (instr->InputAt(1)->IsFPRegister()) {
1489 __ movmskpd(i.TempRegister(0), i.InputDoubleRegister(1));
1490 } else {
1491 __ movsd(kScratchDoubleReg, i.InputOperand(1));
1492 __ movmskpd(i.TempRegister(0), kScratchDoubleReg);
1493 }
1494 __ test(i.TempRegister(0), Immediate(1));
1495 __ j(zero, &done_compare, Label::kNear);
1496 __ bind(&compare_swap);
1497 if (instr->InputAt(1)->IsFPRegister()) {
1498 __ movsd(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1499 } else {
1500 __ movsd(i.InputDoubleRegister(0), i.InputOperand(1));
1501 }
1502 __ bind(&done_compare);
1503 __ bind(ool->exit());
1504 break;
1505 }
1506 case kSSEFloat64Mod: {
1507 // TODO(dcarney): alignment is wrong.
1508 __ sub(esp, Immediate(kDoubleSize));
1509 // Move values to st(0) and st(1).
1510 __ movsd(Operand(esp, 0), i.InputDoubleRegister(1));
1511 __ fld_d(Operand(esp, 0));
1512 __ movsd(Operand(esp, 0), i.InputDoubleRegister(0));
1513 __ fld_d(Operand(esp, 0));
1514 // Loop while fprem isn't done.
1515 Label mod_loop;
1516 __ bind(&mod_loop);
1517 // This instructions traps on all kinds inputs, but we are assuming the
1518 // floating point control word is set to ignore them all.
1519 __ fprem();
1520 // The following 2 instruction implicitly use eax.
1521 __ fnstsw_ax();
1522 __ sahf();
1523 __ j(parity_even, &mod_loop);
1524 // Move output to stack and clean up.
1525 __ fstp(1);
1526 __ fstp_d(Operand(esp, 0));
1527 __ movsd(i.OutputDoubleRegister(), Operand(esp, 0));
1528 __ add(esp, Immediate(kDoubleSize));
1529 break;
1530 }
1531 case kSSEFloat64Abs: {
1532 // TODO(bmeurer): Use 128-bit constants.
1533 __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
1534 __ psrlq(kScratchDoubleReg, 1);
1535 __ andpd(i.OutputDoubleRegister(), kScratchDoubleReg);
1536 break;
1537 }
1538 case kSSEFloat64Neg: {
1539 // TODO(bmeurer): Use 128-bit constants.
1540 __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
1541 __ psllq(kScratchDoubleReg, 63);
1542 __ xorpd(i.OutputDoubleRegister(), kScratchDoubleReg);
1543 break;
1544 }
1545 case kSSEFloat64Sqrt:
1546 __ sqrtsd(i.OutputDoubleRegister(), i.InputOperand(0));
1547 break;
1548 case kSSEFloat64Round: {
1549 CpuFeatureScope sse_scope(masm(), SSE4_1);
1550 RoundingMode const mode =
1551 static_cast<RoundingMode>(MiscField::decode(instr->opcode()));
1552 __ roundsd(i.OutputDoubleRegister(), i.InputDoubleRegister(0), mode);
1553 break;
1554 }
1555 case kSSEFloat32ToFloat64:
1556 __ cvtss2sd(i.OutputDoubleRegister(), i.InputOperand(0));
1557 break;
1558 case kSSEFloat64ToFloat32:
1559 __ cvtsd2ss(i.OutputDoubleRegister(), i.InputOperand(0));
1560 break;
1561 case kSSEFloat32ToInt32:
1562 __ cvttss2si(i.OutputRegister(), i.InputOperand(0));
1563 break;
1564 case kSSEFloat32ToUint32: {
1565 Label success;
1566 __ cvttss2si(i.OutputRegister(), i.InputOperand(0));
1567 __ test(i.OutputRegister(), i.OutputRegister());
1568 __ j(positive, &success);
1569 __ Move(kScratchDoubleReg, static_cast<float>(INT32_MIN));
1570 __ addss(kScratchDoubleReg, i.InputOperand(0));
1571 __ cvttss2si(i.OutputRegister(), kScratchDoubleReg);
1572 __ or_(i.OutputRegister(), Immediate(0x80000000));
1573 __ bind(&success);
1574 break;
1575 }
1576 case kSSEFloat64ToInt32:
1577 __ cvttsd2si(i.OutputRegister(), i.InputOperand(0));
1578 break;
1579 case kSSEFloat64ToUint32: {
1580 __ Move(kScratchDoubleReg, -2147483648.0);
1581 __ addsd(kScratchDoubleReg, i.InputOperand(0));
1582 __ cvttsd2si(i.OutputRegister(), kScratchDoubleReg);
1583 __ add(i.OutputRegister(), Immediate(0x80000000));
1584 break;
1585 }
1586 case kSSEInt32ToFloat32:
1587 __ cvtsi2ss(i.OutputDoubleRegister(), i.InputOperand(0));
1588 break;
1589 case kSSEUint32ToFloat32: {
1590 Register scratch0 = i.TempRegister(0);
1591 Register scratch1 = i.TempRegister(1);
1592 __ mov(scratch0, i.InputOperand(0));
1593 __ Cvtui2ss(i.OutputDoubleRegister(), scratch0, scratch1);
1594 break;
1595 }
1596 case kSSEInt32ToFloat64:
1597 __ cvtsi2sd(i.OutputDoubleRegister(), i.InputOperand(0));
1598 break;
1599 case kSSEUint32ToFloat64:
1600 __ LoadUint32(i.OutputDoubleRegister(), i.InputOperand(0));
1601 break;
1602 case kSSEFloat64ExtractLowWord32:
1603 if (instr->InputAt(0)->IsFPStackSlot()) {
1604 __ mov(i.OutputRegister(), i.InputOperand(0));
1605 } else {
1606 __ movd(i.OutputRegister(), i.InputDoubleRegister(0));
1607 }
1608 break;
1609 case kSSEFloat64ExtractHighWord32:
1610 if (instr->InputAt(0)->IsFPStackSlot()) {
1611 __ mov(i.OutputRegister(), i.InputOperand(0, kDoubleSize / 2));
1612 } else {
1613 __ Pextrd(i.OutputRegister(), i.InputDoubleRegister(0), 1);
1614 }
1615 break;
1616 case kSSEFloat64InsertLowWord32:
1617 __ Pinsrd(i.OutputDoubleRegister(), i.InputOperand(1), 0);
1618 break;
1619 case kSSEFloat64InsertHighWord32:
1620 __ Pinsrd(i.OutputDoubleRegister(), i.InputOperand(1), 1);
1621 break;
1622 case kSSEFloat64LoadLowWord32:
1623 __ movd(i.OutputDoubleRegister(), i.InputOperand(0));
1624 break;
1625 case kAVXFloat32Add: {
1626 CpuFeatureScope avx_scope(masm(), AVX);
1627 __ vaddss(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1628 i.InputOperand(1));
1629 break;
1630 }
1631 case kAVXFloat32Sub: {
1632 CpuFeatureScope avx_scope(masm(), AVX);
1633 __ vsubss(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1634 i.InputOperand(1));
1635 break;
1636 }
1637 case kAVXFloat32Mul: {
1638 CpuFeatureScope avx_scope(masm(), AVX);
1639 __ vmulss(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1640 i.InputOperand(1));
1641 break;
1642 }
1643 case kAVXFloat32Div: {
1644 CpuFeatureScope avx_scope(masm(), AVX);
1645 __ vdivss(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1646 i.InputOperand(1));
1647 // Don't delete this mov. It may improve performance on some CPUs,
1648 // when there is a (v)mulss depending on the result.
1649 __ movaps(i.OutputDoubleRegister(), i.OutputDoubleRegister());
1650 break;
1651 }
1652 case kAVXFloat64Add: {
1653 CpuFeatureScope avx_scope(masm(), AVX);
1654 __ vaddsd(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1655 i.InputOperand(1));
1656 break;
1657 }
1658 case kAVXFloat64Sub: {
1659 CpuFeatureScope avx_scope(masm(), AVX);
1660 __ vsubsd(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1661 i.InputOperand(1));
1662 break;
1663 }
1664 case kAVXFloat64Mul: {
1665 CpuFeatureScope avx_scope(masm(), AVX);
1666 __ vmulsd(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1667 i.InputOperand(1));
1668 break;
1669 }
1670 case kAVXFloat64Div: {
1671 CpuFeatureScope avx_scope(masm(), AVX);
1672 __ vdivsd(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1673 i.InputOperand(1));
1674 // Don't delete this mov. It may improve performance on some CPUs,
1675 // when there is a (v)mulsd depending on the result.
1676 __ movaps(i.OutputDoubleRegister(), i.OutputDoubleRegister());
1677 break;
1678 }
1679 case kAVXFloat32Abs: {
1680 // TODO(bmeurer): Use RIP relative 128-bit constants.
1681 __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
1682 __ psrlq(kScratchDoubleReg, 33);
1683 CpuFeatureScope avx_scope(masm(), AVX);
1684 __ vandps(i.OutputDoubleRegister(), kScratchDoubleReg, i.InputOperand(0));
1685 break;
1686 }
1687 case kAVXFloat32Neg: {
1688 // TODO(bmeurer): Use RIP relative 128-bit constants.
1689 __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
1690 __ psllq(kScratchDoubleReg, 31);
1691 CpuFeatureScope avx_scope(masm(), AVX);
1692 __ vxorps(i.OutputDoubleRegister(), kScratchDoubleReg, i.InputOperand(0));
1693 break;
1694 }
1695 case kAVXFloat64Abs: {
1696 // TODO(bmeurer): Use RIP relative 128-bit constants.
1697 __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
1698 __ psrlq(kScratchDoubleReg, 1);
1699 CpuFeatureScope avx_scope(masm(), AVX);
1700 __ vandpd(i.OutputDoubleRegister(), kScratchDoubleReg, i.InputOperand(0));
1701 break;
1702 }
1703 case kAVXFloat64Neg: {
1704 // TODO(bmeurer): Use RIP relative 128-bit constants.
1705 __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
1706 __ psllq(kScratchDoubleReg, 63);
1707 CpuFeatureScope avx_scope(masm(), AVX);
1708 __ vxorpd(i.OutputDoubleRegister(), kScratchDoubleReg, i.InputOperand(0));
1709 break;
1710 }
1711 case kSSEFloat64SilenceNaN:
1712 __ xorpd(kScratchDoubleReg, kScratchDoubleReg);
1713 __ subsd(i.InputDoubleRegister(0), kScratchDoubleReg);
1714 break;
1715 case kIA32Movsxbl:
1716 __ movsx_b(i.OutputRegister(), i.MemoryOperand());
1717 break;
1718 case kIA32Movzxbl:
1719 __ movzx_b(i.OutputRegister(), i.MemoryOperand());
1720 break;
1721 case kIA32Movb: {
1722 size_t index = 0;
1723 Operand operand = i.MemoryOperand(&index);
1724 if (HasImmediateInput(instr, index)) {
1725 __ mov_b(operand, i.InputInt8(index));
1726 } else {
1727 __ mov_b(operand, i.InputRegister(index));
1728 }
1729 break;
1730 }
1731 case kIA32Movsxwl:
1732 __ movsx_w(i.OutputRegister(), i.MemoryOperand());
1733 break;
1734 case kIA32Movzxwl:
1735 __ movzx_w(i.OutputRegister(), i.MemoryOperand());
1736 break;
1737 case kIA32Movw: {
1738 size_t index = 0;
1739 Operand operand = i.MemoryOperand(&index);
1740 if (HasImmediateInput(instr, index)) {
1741 __ mov_w(operand, i.InputInt16(index));
1742 } else {
1743 __ mov_w(operand, i.InputRegister(index));
1744 }
1745 break;
1746 }
1747 case kIA32Movl:
1748 if (instr->HasOutput()) {
1749 __ mov(i.OutputRegister(), i.MemoryOperand());
1750 } else {
1751 size_t index = 0;
1752 Operand operand = i.MemoryOperand(&index);
1753 if (HasImmediateInput(instr, index)) {
1754 __ mov(operand, i.InputImmediate(index));
1755 } else {
1756 __ mov(operand, i.InputRegister(index));
1757 }
1758 }
1759 break;
1760 case kIA32Movsd:
1761 if (instr->HasOutput()) {
1762 __ movsd(i.OutputDoubleRegister(), i.MemoryOperand());
1763 } else {
1764 size_t index = 0;
1765 Operand operand = i.MemoryOperand(&index);
1766 __ movsd(operand, i.InputDoubleRegister(index));
1767 }
1768 break;
1769 case kIA32Movss:
1770 if (instr->HasOutput()) {
1771 __ movss(i.OutputDoubleRegister(), i.MemoryOperand());
1772 } else {
1773 size_t index = 0;
1774 Operand operand = i.MemoryOperand(&index);
1775 __ movss(operand, i.InputDoubleRegister(index));
1776 }
1777 break;
1778 case kIA32BitcastFI:
1779 if (instr->InputAt(0)->IsFPStackSlot()) {
1780 __ mov(i.OutputRegister(), i.InputOperand(0));
1781 } else {
1782 __ movd(i.OutputRegister(), i.InputDoubleRegister(0));
1783 }
1784 break;
1785 case kIA32BitcastIF:
1786 if (instr->InputAt(0)->IsRegister()) {
1787 __ movd(i.OutputDoubleRegister(), i.InputRegister(0));
1788 } else {
1789 __ movss(i.OutputDoubleRegister(), i.InputOperand(0));
1790 }
1791 break;
1792 case kIA32Lea: {
1793 AddressingMode mode = AddressingModeField::decode(instr->opcode());
1794 // Shorten "leal" to "addl", "subl" or "shll" if the register allocation
1795 // and addressing mode just happens to work out. The "addl"/"subl" forms
1796 // in these cases are faster based on measurements.
1797 if (mode == kMode_MI) {
1798 __ Move(i.OutputRegister(), Immediate(i.InputInt32(0)));
1799 } else if (i.InputRegister(0).is(i.OutputRegister())) {
1800 if (mode == kMode_MRI) {
1801 int32_t constant_summand = i.InputInt32(1);
1802 if (constant_summand > 0) {
1803 __ add(i.OutputRegister(), Immediate(constant_summand));
1804 } else if (constant_summand < 0) {
1805 __ sub(i.OutputRegister(), Immediate(-constant_summand));
1806 }
1807 } else if (mode == kMode_MR1) {
1808 if (i.InputRegister(1).is(i.OutputRegister())) {
1809 __ shl(i.OutputRegister(), 1);
1810 } else {
1811 __ add(i.OutputRegister(), i.InputRegister(1));
1812 }
1813 } else if (mode == kMode_M2) {
1814 __ shl(i.OutputRegister(), 1);
1815 } else if (mode == kMode_M4) {
1816 __ shl(i.OutputRegister(), 2);
1817 } else if (mode == kMode_M8) {
1818 __ shl(i.OutputRegister(), 3);
1819 } else {
1820 __ lea(i.OutputRegister(), i.MemoryOperand());
1821 }
1822 } else if (mode == kMode_MR1 &&
1823 i.InputRegister(1).is(i.OutputRegister())) {
1824 __ add(i.OutputRegister(), i.InputRegister(0));
1825 } else {
1826 __ lea(i.OutputRegister(), i.MemoryOperand());
1827 }
1828 break;
1829 }
1830 case kIA32PushFloat32:
1831 if (instr->InputAt(0)->IsFPRegister()) {
1832 __ sub(esp, Immediate(kFloatSize));
1833 __ movss(Operand(esp, 0), i.InputDoubleRegister(0));
1834 frame_access_state()->IncreaseSPDelta(kFloatSize / kPointerSize);
1835 } else if (HasImmediateInput(instr, 0)) {
1836 __ Move(kScratchDoubleReg, i.InputDouble(0));
1837 __ sub(esp, Immediate(kDoubleSize));
1838 __ movss(Operand(esp, 0), kScratchDoubleReg);
1839 frame_access_state()->IncreaseSPDelta(kDoubleSize / kPointerSize);
1840 } else {
1841 __ movsd(kScratchDoubleReg, i.InputOperand(0));
1842 __ sub(esp, Immediate(kDoubleSize));
1843 __ movss(Operand(esp, 0), kScratchDoubleReg);
1844 frame_access_state()->IncreaseSPDelta(kDoubleSize / kPointerSize);
1845 }
1846 break;
1847 case kIA32PushFloat64:
1848 if (instr->InputAt(0)->IsFPRegister()) {
1849 __ sub(esp, Immediate(kDoubleSize));
1850 __ movsd(Operand(esp, 0), i.InputDoubleRegister(0));
1851 frame_access_state()->IncreaseSPDelta(kDoubleSize / kPointerSize);
1852 } else if (HasImmediateInput(instr, 0)) {
1853 __ Move(kScratchDoubleReg, i.InputDouble(0));
1854 __ sub(esp, Immediate(kDoubleSize));
1855 __ movsd(Operand(esp, 0), kScratchDoubleReg);
1856 frame_access_state()->IncreaseSPDelta(kDoubleSize / kPointerSize);
1857 } else {
1858 __ movsd(kScratchDoubleReg, i.InputOperand(0));
1859 __ sub(esp, Immediate(kDoubleSize));
1860 __ movsd(Operand(esp, 0), kScratchDoubleReg);
1861 frame_access_state()->IncreaseSPDelta(kDoubleSize / kPointerSize);
1862 }
1863 break;
1864 case kIA32Push:
1865 if (AddressingModeField::decode(instr->opcode()) != kMode_None) {
1866 size_t index = 0;
1867 Operand operand = i.MemoryOperand(&index);
1868 __ push(operand);
1869 frame_access_state()->IncreaseSPDelta(kFloatSize / kPointerSize);
1870 } else if (instr->InputAt(0)->IsFPRegister()) {
1871 __ sub(esp, Immediate(kFloatSize));
1872 __ movsd(Operand(esp, 0), i.InputDoubleRegister(0));
1873 frame_access_state()->IncreaseSPDelta(kFloatSize / kPointerSize);
1874 } else if (HasImmediateInput(instr, 0)) {
1875 __ push(i.InputImmediate(0));
1876 frame_access_state()->IncreaseSPDelta(1);
1877 } else {
1878 __ push(i.InputOperand(0));
1879 frame_access_state()->IncreaseSPDelta(1);
1880 }
1881 break;
1882 case kIA32Poke: {
1883 int const slot = MiscField::decode(instr->opcode());
1884 if (HasImmediateInput(instr, 0)) {
1885 __ mov(Operand(esp, slot * kPointerSize), i.InputImmediate(0));
1886 } else {
1887 __ mov(Operand(esp, slot * kPointerSize), i.InputRegister(0));
1888 }
1889 break;
1890 }
1891 case kIA32Xchgb: {
1892 size_t index = 0;
1893 Operand operand = i.MemoryOperand(&index);
1894 __ xchg_b(i.InputRegister(index), operand);
1895 break;
1896 }
1897 case kIA32Xchgw: {
1898 size_t index = 0;
1899 Operand operand = i.MemoryOperand(&index);
1900 __ xchg_w(i.InputRegister(index), operand);
1901 break;
1902 }
1903 case kIA32Xchgl: {
1904 size_t index = 0;
1905 Operand operand = i.MemoryOperand(&index);
1906 __ xchg(i.InputRegister(index), operand);
1907 break;
1908 }
1909 case kCheckedLoadInt8:
1910 ASSEMBLE_CHECKED_LOAD_INTEGER(movsx_b);
1911 break;
1912 case kCheckedLoadUint8:
1913 ASSEMBLE_CHECKED_LOAD_INTEGER(movzx_b);
1914 break;
1915 case kCheckedLoadInt16:
1916 ASSEMBLE_CHECKED_LOAD_INTEGER(movsx_w);
1917 break;
1918 case kCheckedLoadUint16:
1919 ASSEMBLE_CHECKED_LOAD_INTEGER(movzx_w);
1920 break;
1921 case kCheckedLoadWord32:
1922 ASSEMBLE_CHECKED_LOAD_INTEGER(mov);
1923 break;
1924 case kCheckedLoadFloat32:
1925 ASSEMBLE_CHECKED_LOAD_FLOAT(movss, OutOfLineLoadFloat32NaN, s);
1926 break;
1927 case kCheckedLoadFloat64:
1928 ASSEMBLE_CHECKED_LOAD_FLOAT(movsd, OutOfLineLoadFloat64NaN, d);
1929 break;
1930 case kCheckedStoreWord8:
1931 ASSEMBLE_CHECKED_STORE_INTEGER(mov_b);
1932 break;
1933 case kCheckedStoreWord16:
1934 ASSEMBLE_CHECKED_STORE_INTEGER(mov_w);
1935 break;
1936 case kCheckedStoreWord32:
1937 ASSEMBLE_CHECKED_STORE_INTEGER(mov);
1938 break;
1939 case kCheckedStoreFloat32:
1940 ASSEMBLE_CHECKED_STORE_FLOAT(movss);
1941 break;
1942 case kCheckedStoreFloat64:
1943 ASSEMBLE_CHECKED_STORE_FLOAT(movsd);
1944 break;
1945 case kIA32StackCheck: {
1946 ExternalReference const stack_limit =
1947 ExternalReference::address_of_stack_limit(isolate());
1948 __ cmp(esp, Operand::StaticVariable(stack_limit));
1949 break;
1950 }
1951 case kCheckedLoadWord64:
1952 case kCheckedStoreWord64:
1953 UNREACHABLE(); // currently unsupported checked int64 load/store.
1954 break;
1955 case kAtomicLoadInt8:
1956 case kAtomicLoadUint8:
1957 case kAtomicLoadInt16:
1958 case kAtomicLoadUint16:
1959 case kAtomicLoadWord32:
1960 case kAtomicStoreWord8:
1961 case kAtomicStoreWord16:
1962 case kAtomicStoreWord32:
1963 UNREACHABLE(); // Won't be generated by instruction selector.
1964 break;
1965 }
1966 return kSuccess;
1967 } // NOLINT(readability/fn_size)
1968
FlagsConditionToCondition(FlagsCondition condition)1969 static Condition FlagsConditionToCondition(FlagsCondition condition) {
1970 switch (condition) {
1971 case kUnorderedEqual:
1972 case kEqual:
1973 return equal;
1974 break;
1975 case kUnorderedNotEqual:
1976 case kNotEqual:
1977 return not_equal;
1978 break;
1979 case kSignedLessThan:
1980 return less;
1981 break;
1982 case kSignedGreaterThanOrEqual:
1983 return greater_equal;
1984 break;
1985 case kSignedLessThanOrEqual:
1986 return less_equal;
1987 break;
1988 case kSignedGreaterThan:
1989 return greater;
1990 break;
1991 case kUnsignedLessThan:
1992 return below;
1993 break;
1994 case kUnsignedGreaterThanOrEqual:
1995 return above_equal;
1996 break;
1997 case kUnsignedLessThanOrEqual:
1998 return below_equal;
1999 break;
2000 case kUnsignedGreaterThan:
2001 return above;
2002 break;
2003 case kOverflow:
2004 return overflow;
2005 break;
2006 case kNotOverflow:
2007 return no_overflow;
2008 break;
2009 default:
2010 UNREACHABLE();
2011 return no_condition;
2012 break;
2013 }
2014 }
2015
2016 // Assembles a branch after an instruction.
AssembleArchBranch(Instruction * instr,BranchInfo * branch)2017 void CodeGenerator::AssembleArchBranch(Instruction* instr, BranchInfo* branch) {
2018 Label::Distance flabel_distance =
2019 branch->fallthru ? Label::kNear : Label::kFar;
2020 Label* tlabel = branch->true_label;
2021 Label* flabel = branch->false_label;
2022 if (branch->condition == kUnorderedEqual) {
2023 __ j(parity_even, flabel, flabel_distance);
2024 } else if (branch->condition == kUnorderedNotEqual) {
2025 __ j(parity_even, tlabel);
2026 }
2027 __ j(FlagsConditionToCondition(branch->condition), tlabel);
2028
2029 // Add a jump if not falling through to the next block.
2030 if (!branch->fallthru) __ jmp(flabel);
2031 }
2032
2033
AssembleArchJump(RpoNumber target)2034 void CodeGenerator::AssembleArchJump(RpoNumber target) {
2035 if (!IsNextInAssemblyOrder(target)) __ jmp(GetLabel(target));
2036 }
2037
AssembleArchTrap(Instruction * instr,FlagsCondition condition)2038 void CodeGenerator::AssembleArchTrap(Instruction* instr,
2039 FlagsCondition condition) {
2040 class OutOfLineTrap final : public OutOfLineCode {
2041 public:
2042 OutOfLineTrap(CodeGenerator* gen, bool frame_elided, Instruction* instr)
2043 : OutOfLineCode(gen),
2044 frame_elided_(frame_elided),
2045 instr_(instr),
2046 gen_(gen) {}
2047
2048 void Generate() final {
2049 IA32OperandConverter i(gen_, instr_);
2050
2051 Builtins::Name trap_id =
2052 static_cast<Builtins::Name>(i.InputInt32(instr_->InputCount() - 1));
2053 bool old_has_frame = __ has_frame();
2054 if (frame_elided_) {
2055 __ set_has_frame(true);
2056 __ EnterFrame(StackFrame::WASM_COMPILED);
2057 }
2058 GenerateCallToTrap(trap_id);
2059 if (frame_elided_) {
2060 __ set_has_frame(old_has_frame);
2061 }
2062 }
2063
2064 private:
2065 void GenerateCallToTrap(Builtins::Name trap_id) {
2066 if (trap_id == Builtins::builtin_count) {
2067 // We cannot test calls to the runtime in cctest/test-run-wasm.
2068 // Therefore we emit a call to C here instead of a call to the runtime.
2069 __ PrepareCallCFunction(0, esi);
2070 __ CallCFunction(
2071 ExternalReference::wasm_call_trap_callback_for_testing(isolate()),
2072 0);
2073 __ LeaveFrame(StackFrame::WASM_COMPILED);
2074 __ Ret();
2075 } else {
2076 gen_->AssembleSourcePosition(instr_);
2077 __ Call(handle(isolate()->builtins()->builtin(trap_id), isolate()),
2078 RelocInfo::CODE_TARGET);
2079 ReferenceMap* reference_map =
2080 new (gen_->zone()) ReferenceMap(gen_->zone());
2081 gen_->RecordSafepoint(reference_map, Safepoint::kSimple, 0,
2082 Safepoint::kNoLazyDeopt);
2083 if (FLAG_debug_code) {
2084 __ ud2();
2085 }
2086 }
2087 }
2088
2089 bool frame_elided_;
2090 Instruction* instr_;
2091 CodeGenerator* gen_;
2092 };
2093 bool frame_elided = !frame_access_state()->has_frame();
2094 auto ool = new (zone()) OutOfLineTrap(this, frame_elided, instr);
2095 Label* tlabel = ool->entry();
2096 Label end;
2097 if (condition == kUnorderedEqual) {
2098 __ j(parity_even, &end);
2099 } else if (condition == kUnorderedNotEqual) {
2100 __ j(parity_even, tlabel);
2101 }
2102 __ j(FlagsConditionToCondition(condition), tlabel);
2103 __ bind(&end);
2104 }
2105
2106 // Assembles boolean materializations after an instruction.
AssembleArchBoolean(Instruction * instr,FlagsCondition condition)2107 void CodeGenerator::AssembleArchBoolean(Instruction* instr,
2108 FlagsCondition condition) {
2109 IA32OperandConverter i(this, instr);
2110 Label done;
2111
2112 // Materialize a full 32-bit 1 or 0 value. The result register is always the
2113 // last output of the instruction.
2114 Label check;
2115 DCHECK_NE(0u, instr->OutputCount());
2116 Register reg = i.OutputRegister(instr->OutputCount() - 1);
2117 if (condition == kUnorderedEqual) {
2118 __ j(parity_odd, &check, Label::kNear);
2119 __ Move(reg, Immediate(0));
2120 __ jmp(&done, Label::kNear);
2121 } else if (condition == kUnorderedNotEqual) {
2122 __ j(parity_odd, &check, Label::kNear);
2123 __ mov(reg, Immediate(1));
2124 __ jmp(&done, Label::kNear);
2125 }
2126 Condition cc = FlagsConditionToCondition(condition);
2127
2128 __ bind(&check);
2129 if (reg.is_byte_register()) {
2130 // setcc for byte registers (al, bl, cl, dl).
2131 __ setcc(cc, reg);
2132 __ movzx_b(reg, reg);
2133 } else {
2134 // Emit a branch to set a register to either 1 or 0.
2135 Label set;
2136 __ j(cc, &set, Label::kNear);
2137 __ Move(reg, Immediate(0));
2138 __ jmp(&done, Label::kNear);
2139 __ bind(&set);
2140 __ mov(reg, Immediate(1));
2141 }
2142 __ bind(&done);
2143 }
2144
2145
AssembleArchLookupSwitch(Instruction * instr)2146 void CodeGenerator::AssembleArchLookupSwitch(Instruction* instr) {
2147 IA32OperandConverter i(this, instr);
2148 Register input = i.InputRegister(0);
2149 for (size_t index = 2; index < instr->InputCount(); index += 2) {
2150 __ cmp(input, Immediate(i.InputInt32(index + 0)));
2151 __ j(equal, GetLabel(i.InputRpo(index + 1)));
2152 }
2153 AssembleArchJump(i.InputRpo(1));
2154 }
2155
2156
AssembleArchTableSwitch(Instruction * instr)2157 void CodeGenerator::AssembleArchTableSwitch(Instruction* instr) {
2158 IA32OperandConverter i(this, instr);
2159 Register input = i.InputRegister(0);
2160 size_t const case_count = instr->InputCount() - 2;
2161 Label** cases = zone()->NewArray<Label*>(case_count);
2162 for (size_t index = 0; index < case_count; ++index) {
2163 cases[index] = GetLabel(i.InputRpo(index + 2));
2164 }
2165 Label* const table = AddJumpTable(cases, case_count);
2166 __ cmp(input, Immediate(case_count));
2167 __ j(above_equal, GetLabel(i.InputRpo(1)));
2168 __ jmp(Operand::JumpTable(input, times_4, table));
2169 }
2170
AssembleDeoptimizerCall(int deoptimization_id,SourcePosition pos)2171 CodeGenerator::CodeGenResult CodeGenerator::AssembleDeoptimizerCall(
2172 int deoptimization_id, SourcePosition pos) {
2173 DeoptimizeKind deoptimization_kind = GetDeoptimizationKind(deoptimization_id);
2174 DeoptimizeReason deoptimization_reason =
2175 GetDeoptimizationReason(deoptimization_id);
2176 Deoptimizer::BailoutType bailout_type =
2177 deoptimization_kind == DeoptimizeKind::kSoft ? Deoptimizer::SOFT
2178 : Deoptimizer::EAGER;
2179 Address deopt_entry = Deoptimizer::GetDeoptimizationEntry(
2180 isolate(), deoptimization_id, bailout_type);
2181 if (deopt_entry == nullptr) return kTooManyDeoptimizationBailouts;
2182 __ RecordDeoptReason(deoptimization_reason, pos, deoptimization_id);
2183 __ call(deopt_entry, RelocInfo::RUNTIME_ENTRY);
2184 return kSuccess;
2185 }
2186
2187
2188 // The calling convention for JSFunctions on IA32 passes arguments on the
2189 // stack and the JSFunction and context in EDI and ESI, respectively, thus
2190 // the steps of the call look as follows:
2191
2192 // --{ before the call instruction }--------------------------------------------
2193 // | caller frame |
2194 // ^ esp ^ ebp
2195
2196 // --{ push arguments and setup ESI, EDI }--------------------------------------
2197 // | args + receiver | caller frame |
2198 // ^ esp ^ ebp
2199 // [edi = JSFunction, esi = context]
2200
2201 // --{ call [edi + kCodeEntryOffset] }------------------------------------------
2202 // | RET | args + receiver | caller frame |
2203 // ^ esp ^ ebp
2204
2205 // =={ prologue of called function }============================================
2206 // --{ push ebp }---------------------------------------------------------------
2207 // | FP | RET | args + receiver | caller frame |
2208 // ^ esp ^ ebp
2209
2210 // --{ mov ebp, esp }-----------------------------------------------------------
2211 // | FP | RET | args + receiver | caller frame |
2212 // ^ ebp,esp
2213
2214 // --{ push esi }---------------------------------------------------------------
2215 // | CTX | FP | RET | args + receiver | caller frame |
2216 // ^esp ^ ebp
2217
2218 // --{ push edi }---------------------------------------------------------------
2219 // | FNC | CTX | FP | RET | args + receiver | caller frame |
2220 // ^esp ^ ebp
2221
2222 // --{ subi esp, #N }-----------------------------------------------------------
2223 // | callee frame | FNC | CTX | FP | RET | args + receiver | caller frame |
2224 // ^esp ^ ebp
2225
2226 // =={ body of called function }================================================
2227
2228 // =={ epilogue of called function }============================================
2229 // --{ mov esp, ebp }-----------------------------------------------------------
2230 // | FP | RET | args + receiver | caller frame |
2231 // ^ esp,ebp
2232
2233 // --{ pop ebp }-----------------------------------------------------------
2234 // | | RET | args + receiver | caller frame |
2235 // ^ esp ^ ebp
2236
2237 // --{ ret #A+1 }-----------------------------------------------------------
2238 // | | caller frame |
2239 // ^ esp ^ ebp
2240
2241
2242 // Runtime function calls are accomplished by doing a stub call to the
2243 // CEntryStub (a real code object). On IA32 passes arguments on the
2244 // stack, the number of arguments in EAX, the address of the runtime function
2245 // in EBX, and the context in ESI.
2246
2247 // --{ before the call instruction }--------------------------------------------
2248 // | caller frame |
2249 // ^ esp ^ ebp
2250
2251 // --{ push arguments and setup EAX, EBX, and ESI }-----------------------------
2252 // | args + receiver | caller frame |
2253 // ^ esp ^ ebp
2254 // [eax = #args, ebx = runtime function, esi = context]
2255
2256 // --{ call #CEntryStub }-------------------------------------------------------
2257 // | RET | args + receiver | caller frame |
2258 // ^ esp ^ ebp
2259
2260 // =={ body of runtime function }===============================================
2261
2262 // --{ runtime returns }--------------------------------------------------------
2263 // | caller frame |
2264 // ^ esp ^ ebp
2265
2266 // Other custom linkages (e.g. for calling directly into and out of C++) may
2267 // need to save callee-saved registers on the stack, which is done in the
2268 // function prologue of generated code.
2269
2270 // --{ before the call instruction }--------------------------------------------
2271 // | caller frame |
2272 // ^ esp ^ ebp
2273
2274 // --{ set up arguments in registers on stack }---------------------------------
2275 // | args | caller frame |
2276 // ^ esp ^ ebp
2277 // [r0 = arg0, r1 = arg1, ...]
2278
2279 // --{ call code }--------------------------------------------------------------
2280 // | RET | args | caller frame |
2281 // ^ esp ^ ebp
2282
2283 // =={ prologue of called function }============================================
2284 // --{ push ebp }---------------------------------------------------------------
2285 // | FP | RET | args | caller frame |
2286 // ^ esp ^ ebp
2287
2288 // --{ mov ebp, esp }-----------------------------------------------------------
2289 // | FP | RET | args | caller frame |
2290 // ^ ebp,esp
2291
2292 // --{ save registers }---------------------------------------------------------
2293 // | regs | FP | RET | args | caller frame |
2294 // ^ esp ^ ebp
2295
2296 // --{ subi esp, #N }-----------------------------------------------------------
2297 // | callee frame | regs | FP | RET | args | caller frame |
2298 // ^esp ^ ebp
2299
2300 // =={ body of called function }================================================
2301
2302 // =={ epilogue of called function }============================================
2303 // --{ restore registers }------------------------------------------------------
2304 // | regs | FP | RET | args | caller frame |
2305 // ^ esp ^ ebp
2306
2307 // --{ mov esp, ebp }-----------------------------------------------------------
2308 // | FP | RET | args | caller frame |
2309 // ^ esp,ebp
2310
2311 // --{ pop ebp }----------------------------------------------------------------
2312 // | RET | args | caller frame |
2313 // ^ esp ^ ebp
2314
FinishFrame(Frame * frame)2315 void CodeGenerator::FinishFrame(Frame* frame) {
2316 CallDescriptor* descriptor = linkage()->GetIncomingDescriptor();
2317 const RegList saves = descriptor->CalleeSavedRegisters();
2318 if (saves != 0) { // Save callee-saved registers.
2319 DCHECK(!info()->is_osr());
2320 int pushed = 0;
2321 for (int i = Register::kNumRegisters - 1; i >= 0; i--) {
2322 if (!((1 << i) & saves)) continue;
2323 ++pushed;
2324 }
2325 frame->AllocateSavedCalleeRegisterSlots(pushed);
2326 }
2327 }
2328
AssembleConstructFrame()2329 void CodeGenerator::AssembleConstructFrame() {
2330 CallDescriptor* descriptor = linkage()->GetIncomingDescriptor();
2331 if (frame_access_state()->has_frame()) {
2332 if (descriptor->IsCFunctionCall()) {
2333 __ push(ebp);
2334 __ mov(ebp, esp);
2335 } else if (descriptor->IsJSFunctionCall()) {
2336 __ Prologue(this->info()->GeneratePreagedPrologue());
2337 if (descriptor->PushArgumentCount()) {
2338 __ push(kJavaScriptCallArgCountRegister);
2339 }
2340 } else {
2341 __ StubPrologue(info()->GetOutputStackFrameType());
2342 }
2343 }
2344
2345 int shrink_slots =
2346 frame()->GetTotalFrameSlotCount() - descriptor->CalculateFixedFrameSize();
2347
2348 if (info()->is_osr()) {
2349 // TurboFan OSR-compiled functions cannot be entered directly.
2350 __ Abort(kShouldNotDirectlyEnterOsrFunction);
2351
2352 // Unoptimized code jumps directly to this entrypoint while the unoptimized
2353 // frame is still on the stack. Optimized code uses OSR values directly from
2354 // the unoptimized frame. Thus, all that needs to be done is to allocate the
2355 // remaining stack slots.
2356 if (FLAG_code_comments) __ RecordComment("-- OSR entrypoint --");
2357 osr_pc_offset_ = __ pc_offset();
2358 shrink_slots -= OsrHelper(info()).UnoptimizedFrameSlots();
2359 }
2360
2361 const RegList saves = descriptor->CalleeSavedRegisters();
2362 if (shrink_slots > 0) {
2363 __ sub(esp, Immediate(shrink_slots * kPointerSize));
2364 }
2365
2366 if (saves != 0) { // Save callee-saved registers.
2367 DCHECK(!info()->is_osr());
2368 int pushed = 0;
2369 for (int i = Register::kNumRegisters - 1; i >= 0; i--) {
2370 if (!((1 << i) & saves)) continue;
2371 __ push(Register::from_code(i));
2372 ++pushed;
2373 }
2374 }
2375 }
2376
AssembleReturn(InstructionOperand * pop)2377 void CodeGenerator::AssembleReturn(InstructionOperand* pop) {
2378 CallDescriptor* descriptor = linkage()->GetIncomingDescriptor();
2379
2380 const RegList saves = descriptor->CalleeSavedRegisters();
2381 // Restore registers.
2382 if (saves != 0) {
2383 for (int i = 0; i < Register::kNumRegisters; i++) {
2384 if (!((1 << i) & saves)) continue;
2385 __ pop(Register::from_code(i));
2386 }
2387 }
2388
2389 // Might need ecx for scratch if pop_size is too big or if there is a variable
2390 // pop count.
2391 DCHECK_EQ(0u, descriptor->CalleeSavedRegisters() & ecx.bit());
2392 size_t pop_size = descriptor->StackParameterCount() * kPointerSize;
2393 IA32OperandConverter g(this, nullptr);
2394 if (descriptor->IsCFunctionCall()) {
2395 AssembleDeconstructFrame();
2396 } else if (frame_access_state()->has_frame()) {
2397 // Canonicalize JSFunction return sites for now if they always have the same
2398 // number of return args.
2399 if (pop->IsImmediate() && g.ToConstant(pop).ToInt32() == 0) {
2400 if (return_label_.is_bound()) {
2401 __ jmp(&return_label_);
2402 return;
2403 } else {
2404 __ bind(&return_label_);
2405 AssembleDeconstructFrame();
2406 }
2407 } else {
2408 AssembleDeconstructFrame();
2409 }
2410 }
2411 DCHECK_EQ(0u, descriptor->CalleeSavedRegisters() & edx.bit());
2412 DCHECK_EQ(0u, descriptor->CalleeSavedRegisters() & ecx.bit());
2413 if (pop->IsImmediate()) {
2414 DCHECK_EQ(Constant::kInt32, g.ToConstant(pop).type());
2415 pop_size += g.ToConstant(pop).ToInt32() * kPointerSize;
2416 __ Ret(static_cast<int>(pop_size), ecx);
2417 } else {
2418 Register pop_reg = g.ToRegister(pop);
2419 Register scratch_reg = pop_reg.is(ecx) ? edx : ecx;
2420 __ pop(scratch_reg);
2421 __ lea(esp, Operand(esp, pop_reg, times_4, static_cast<int>(pop_size)));
2422 __ jmp(scratch_reg);
2423 }
2424 }
2425
2426
AssembleMove(InstructionOperand * source,InstructionOperand * destination)2427 void CodeGenerator::AssembleMove(InstructionOperand* source,
2428 InstructionOperand* destination) {
2429 IA32OperandConverter g(this, nullptr);
2430 // Dispatch on the source and destination operand kinds. Not all
2431 // combinations are possible.
2432 if (source->IsRegister()) {
2433 DCHECK(destination->IsRegister() || destination->IsStackSlot());
2434 Register src = g.ToRegister(source);
2435 Operand dst = g.ToOperand(destination);
2436 __ mov(dst, src);
2437 } else if (source->IsStackSlot()) {
2438 DCHECK(destination->IsRegister() || destination->IsStackSlot());
2439 Operand src = g.ToOperand(source);
2440 if (destination->IsRegister()) {
2441 Register dst = g.ToRegister(destination);
2442 __ mov(dst, src);
2443 } else {
2444 Operand dst = g.ToOperand(destination);
2445 __ push(src);
2446 __ pop(dst);
2447 }
2448 } else if (source->IsConstant()) {
2449 Constant src_constant = g.ToConstant(source);
2450 if (src_constant.type() == Constant::kHeapObject) {
2451 Handle<HeapObject> src = src_constant.ToHeapObject();
2452 if (destination->IsRegister()) {
2453 Register dst = g.ToRegister(destination);
2454 __ LoadHeapObject(dst, src);
2455 } else {
2456 DCHECK(destination->IsStackSlot());
2457 Operand dst = g.ToOperand(destination);
2458 AllowDeferredHandleDereference embedding_raw_address;
2459 if (isolate()->heap()->InNewSpace(*src)) {
2460 __ PushHeapObject(src);
2461 __ pop(dst);
2462 } else {
2463 __ mov(dst, src);
2464 }
2465 }
2466 } else if (destination->IsRegister()) {
2467 Register dst = g.ToRegister(destination);
2468 __ Move(dst, g.ToImmediate(source));
2469 } else if (destination->IsStackSlot()) {
2470 Operand dst = g.ToOperand(destination);
2471 __ Move(dst, g.ToImmediate(source));
2472 } else if (src_constant.type() == Constant::kFloat32) {
2473 // TODO(turbofan): Can we do better here?
2474 uint32_t src = src_constant.ToFloat32AsInt();
2475 if (destination->IsFPRegister()) {
2476 XMMRegister dst = g.ToDoubleRegister(destination);
2477 __ Move(dst, src);
2478 } else {
2479 DCHECK(destination->IsFPStackSlot());
2480 Operand dst = g.ToOperand(destination);
2481 __ Move(dst, Immediate(src));
2482 }
2483 } else {
2484 DCHECK_EQ(Constant::kFloat64, src_constant.type());
2485 uint64_t src = src_constant.ToFloat64AsInt();
2486 uint32_t lower = static_cast<uint32_t>(src);
2487 uint32_t upper = static_cast<uint32_t>(src >> 32);
2488 if (destination->IsFPRegister()) {
2489 XMMRegister dst = g.ToDoubleRegister(destination);
2490 __ Move(dst, src);
2491 } else {
2492 DCHECK(destination->IsFPStackSlot());
2493 Operand dst0 = g.ToOperand(destination);
2494 Operand dst1 = g.HighOperand(destination);
2495 __ Move(dst0, Immediate(lower));
2496 __ Move(dst1, Immediate(upper));
2497 }
2498 }
2499 } else if (source->IsFPRegister()) {
2500 XMMRegister src = g.ToDoubleRegister(source);
2501 if (destination->IsFPRegister()) {
2502 XMMRegister dst = g.ToDoubleRegister(destination);
2503 __ movaps(dst, src);
2504 } else {
2505 DCHECK(destination->IsFPStackSlot());
2506 Operand dst = g.ToOperand(destination);
2507 MachineRepresentation rep =
2508 LocationOperand::cast(source)->representation();
2509 if (rep == MachineRepresentation::kFloat64) {
2510 __ movsd(dst, src);
2511 } else if (rep == MachineRepresentation::kFloat32) {
2512 __ movss(dst, src);
2513 } else {
2514 DCHECK_EQ(MachineRepresentation::kSimd128, rep);
2515 __ movups(dst, src);
2516 }
2517 }
2518 } else if (source->IsFPStackSlot()) {
2519 DCHECK(destination->IsFPRegister() || destination->IsFPStackSlot());
2520 Operand src = g.ToOperand(source);
2521 MachineRepresentation rep = LocationOperand::cast(source)->representation();
2522 if (destination->IsFPRegister()) {
2523 XMMRegister dst = g.ToDoubleRegister(destination);
2524 if (rep == MachineRepresentation::kFloat64) {
2525 __ movsd(dst, src);
2526 } else if (rep == MachineRepresentation::kFloat32) {
2527 __ movss(dst, src);
2528 } else {
2529 DCHECK_EQ(MachineRepresentation::kSimd128, rep);
2530 __ movups(dst, src);
2531 }
2532 } else {
2533 Operand dst = g.ToOperand(destination);
2534 if (rep == MachineRepresentation::kFloat64) {
2535 __ movsd(kScratchDoubleReg, src);
2536 __ movsd(dst, kScratchDoubleReg);
2537 } else if (rep == MachineRepresentation::kFloat32) {
2538 __ movss(kScratchDoubleReg, src);
2539 __ movss(dst, kScratchDoubleReg);
2540 } else {
2541 DCHECK_EQ(MachineRepresentation::kSimd128, rep);
2542 __ movups(kScratchDoubleReg, src);
2543 __ movups(dst, kScratchDoubleReg);
2544 }
2545 }
2546 } else {
2547 UNREACHABLE();
2548 }
2549 }
2550
2551
AssembleSwap(InstructionOperand * source,InstructionOperand * destination)2552 void CodeGenerator::AssembleSwap(InstructionOperand* source,
2553 InstructionOperand* destination) {
2554 IA32OperandConverter g(this, nullptr);
2555 // Dispatch on the source and destination operand kinds. Not all
2556 // combinations are possible.
2557 if (source->IsRegister() && destination->IsRegister()) {
2558 // Register-register.
2559 Register src = g.ToRegister(source);
2560 Register dst = g.ToRegister(destination);
2561 __ push(src);
2562 __ mov(src, dst);
2563 __ pop(dst);
2564 } else if (source->IsRegister() && destination->IsStackSlot()) {
2565 // Register-memory.
2566 Register src = g.ToRegister(source);
2567 __ push(src);
2568 frame_access_state()->IncreaseSPDelta(1);
2569 Operand dst = g.ToOperand(destination);
2570 __ mov(src, dst);
2571 frame_access_state()->IncreaseSPDelta(-1);
2572 dst = g.ToOperand(destination);
2573 __ pop(dst);
2574 } else if (source->IsStackSlot() && destination->IsStackSlot()) {
2575 // Memory-memory.
2576 Operand dst1 = g.ToOperand(destination);
2577 __ push(dst1);
2578 frame_access_state()->IncreaseSPDelta(1);
2579 Operand src1 = g.ToOperand(source);
2580 __ push(src1);
2581 Operand dst2 = g.ToOperand(destination);
2582 __ pop(dst2);
2583 frame_access_state()->IncreaseSPDelta(-1);
2584 Operand src2 = g.ToOperand(source);
2585 __ pop(src2);
2586 } else if (source->IsFPRegister() && destination->IsFPRegister()) {
2587 // XMM register-register swap.
2588 XMMRegister src = g.ToDoubleRegister(source);
2589 XMMRegister dst = g.ToDoubleRegister(destination);
2590 __ movaps(kScratchDoubleReg, src);
2591 __ movaps(src, dst);
2592 __ movaps(dst, kScratchDoubleReg);
2593 } else if (source->IsFPRegister() && destination->IsFPStackSlot()) {
2594 // XMM register-memory swap.
2595 XMMRegister reg = g.ToDoubleRegister(source);
2596 Operand other = g.ToOperand(destination);
2597 MachineRepresentation rep = LocationOperand::cast(source)->representation();
2598 if (rep == MachineRepresentation::kFloat64) {
2599 __ movsd(kScratchDoubleReg, other);
2600 __ movsd(other, reg);
2601 __ movaps(reg, kScratchDoubleReg);
2602 } else if (rep == MachineRepresentation::kFloat32) {
2603 __ movss(kScratchDoubleReg, other);
2604 __ movss(other, reg);
2605 __ movaps(reg, kScratchDoubleReg);
2606 } else {
2607 DCHECK_EQ(MachineRepresentation::kSimd128, rep);
2608 __ movups(kScratchDoubleReg, other);
2609 __ movups(other, reg);
2610 __ movups(reg, kScratchDoubleReg);
2611 }
2612 } else if (source->IsFPStackSlot() && destination->IsFPStackSlot()) {
2613 // Double-width memory-to-memory.
2614 Operand src0 = g.ToOperand(source);
2615 Operand dst0 = g.ToOperand(destination);
2616 MachineRepresentation rep = LocationOperand::cast(source)->representation();
2617 if (rep == MachineRepresentation::kFloat64) {
2618 Operand src1 = g.HighOperand(source);
2619 Operand dst1 = g.HighOperand(destination);
2620 __ movsd(kScratchDoubleReg, dst0); // Save dst in scratch register.
2621 __ push(src0); // Then use stack to copy src to destination.
2622 __ pop(dst0);
2623 __ push(src1);
2624 __ pop(dst1);
2625 __ movsd(src0, kScratchDoubleReg);
2626 } else if (rep == MachineRepresentation::kFloat32) {
2627 __ movss(kScratchDoubleReg, dst0); // Save dst in scratch register.
2628 __ push(src0); // Then use stack to copy src to destination.
2629 __ pop(dst0);
2630 __ movss(src0, kScratchDoubleReg);
2631 } else {
2632 DCHECK_EQ(MachineRepresentation::kSimd128, rep);
2633 // Use the XOR trick to swap without a temporary.
2634 __ movups(kScratchDoubleReg, src0);
2635 __ xorps(kScratchDoubleReg, dst0); // scratch contains src ^ dst.
2636 __ movups(src0, kScratchDoubleReg);
2637 __ xorps(kScratchDoubleReg, dst0); // scratch contains src.
2638 __ movups(dst0, kScratchDoubleReg);
2639 __ xorps(kScratchDoubleReg, src0); // scratch contains dst.
2640 __ movups(src0, kScratchDoubleReg);
2641 }
2642 } else {
2643 // No other combinations are possible.
2644 UNREACHABLE();
2645 }
2646 }
2647
2648
AssembleJumpTable(Label ** targets,size_t target_count)2649 void CodeGenerator::AssembleJumpTable(Label** targets, size_t target_count) {
2650 for (size_t index = 0; index < target_count; ++index) {
2651 __ dd(targets[index]);
2652 }
2653 }
2654
2655
EnsureSpaceForLazyDeopt()2656 void CodeGenerator::EnsureSpaceForLazyDeopt() {
2657 if (!info()->ShouldEnsureSpaceForLazyDeopt()) {
2658 return;
2659 }
2660
2661 int space_needed = Deoptimizer::patch_size();
2662 // Ensure that we have enough space after the previous lazy-bailout
2663 // instruction for patching the code here.
2664 int current_pc = masm()->pc_offset();
2665 if (current_pc < last_lazy_deopt_pc_ + space_needed) {
2666 int padding_size = last_lazy_deopt_pc_ + space_needed - current_pc;
2667 __ Nop(padding_size);
2668 }
2669 }
2670
2671 #undef __
2672
2673 } // namespace compiler
2674 } // namespace internal
2675 } // namespace v8
2676