1 // Copyright 2014 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "src/compiler/code-generator.h"
6
7 #include "src/arm64/frames-arm64.h"
8 #include "src/arm64/macro-assembler-arm64.h"
9 #include "src/compilation-info.h"
10 #include "src/compiler/code-generator-impl.h"
11 #include "src/compiler/gap-resolver.h"
12 #include "src/compiler/node-matchers.h"
13 #include "src/compiler/osr.h"
14
15 namespace v8 {
16 namespace internal {
17 namespace compiler {
18
19 #define __ masm()->
20
21
22 // Adds Arm64-specific methods to convert InstructionOperands.
23 class Arm64OperandConverter final : public InstructionOperandConverter {
24 public:
Arm64OperandConverter(CodeGenerator * gen,Instruction * instr)25 Arm64OperandConverter(CodeGenerator* gen, Instruction* instr)
26 : InstructionOperandConverter(gen, instr) {}
27
InputFloat32Register(size_t index)28 DoubleRegister InputFloat32Register(size_t index) {
29 return InputDoubleRegister(index).S();
30 }
31
InputFloat64Register(size_t index)32 DoubleRegister InputFloat64Register(size_t index) {
33 return InputDoubleRegister(index);
34 }
35
InputFloat32OrZeroRegister(size_t index)36 CPURegister InputFloat32OrZeroRegister(size_t index) {
37 if (instr_->InputAt(index)->IsImmediate()) {
38 DCHECK(bit_cast<int32_t>(InputFloat32(index)) == 0);
39 return wzr;
40 }
41 DCHECK(instr_->InputAt(index)->IsFPRegister());
42 return InputDoubleRegister(index).S();
43 }
44
InputFloat64OrZeroRegister(size_t index)45 CPURegister InputFloat64OrZeroRegister(size_t index) {
46 if (instr_->InputAt(index)->IsImmediate()) {
47 DCHECK(bit_cast<int64_t>(InputDouble(index)) == 0);
48 return xzr;
49 }
50 DCHECK(instr_->InputAt(index)->IsDoubleRegister());
51 return InputDoubleRegister(index);
52 }
53
OutputCount()54 size_t OutputCount() { return instr_->OutputCount(); }
55
OutputFloat32Register()56 DoubleRegister OutputFloat32Register() { return OutputDoubleRegister().S(); }
57
OutputFloat64Register()58 DoubleRegister OutputFloat64Register() { return OutputDoubleRegister(); }
59
InputRegister32(size_t index)60 Register InputRegister32(size_t index) {
61 return ToRegister(instr_->InputAt(index)).W();
62 }
63
InputOrZeroRegister32(size_t index)64 Register InputOrZeroRegister32(size_t index) {
65 DCHECK(instr_->InputAt(index)->IsRegister() ||
66 (instr_->InputAt(index)->IsImmediate() && (InputInt32(index) == 0)));
67 if (instr_->InputAt(index)->IsImmediate()) {
68 return wzr;
69 }
70 return InputRegister32(index);
71 }
72
InputRegister64(size_t index)73 Register InputRegister64(size_t index) { return InputRegister(index); }
74
InputOrZeroRegister64(size_t index)75 Register InputOrZeroRegister64(size_t index) {
76 DCHECK(instr_->InputAt(index)->IsRegister() ||
77 (instr_->InputAt(index)->IsImmediate() && (InputInt64(index) == 0)));
78 if (instr_->InputAt(index)->IsImmediate()) {
79 return xzr;
80 }
81 return InputRegister64(index);
82 }
83
InputImmediate(size_t index)84 Operand InputImmediate(size_t index) {
85 return ToImmediate(instr_->InputAt(index));
86 }
87
InputOperand(size_t index)88 Operand InputOperand(size_t index) {
89 return ToOperand(instr_->InputAt(index));
90 }
91
InputOperand64(size_t index)92 Operand InputOperand64(size_t index) { return InputOperand(index); }
93
InputOperand32(size_t index)94 Operand InputOperand32(size_t index) {
95 return ToOperand32(instr_->InputAt(index));
96 }
97
OutputRegister64()98 Register OutputRegister64() { return OutputRegister(); }
99
OutputRegister32()100 Register OutputRegister32() { return ToRegister(instr_->Output()).W(); }
101
InputOperand2_32(size_t index)102 Operand InputOperand2_32(size_t index) {
103 switch (AddressingModeField::decode(instr_->opcode())) {
104 case kMode_None:
105 return InputOperand32(index);
106 case kMode_Operand2_R_LSL_I:
107 return Operand(InputRegister32(index), LSL, InputInt5(index + 1));
108 case kMode_Operand2_R_LSR_I:
109 return Operand(InputRegister32(index), LSR, InputInt5(index + 1));
110 case kMode_Operand2_R_ASR_I:
111 return Operand(InputRegister32(index), ASR, InputInt5(index + 1));
112 case kMode_Operand2_R_ROR_I:
113 return Operand(InputRegister32(index), ROR, InputInt5(index + 1));
114 case kMode_Operand2_R_UXTB:
115 return Operand(InputRegister32(index), UXTB);
116 case kMode_Operand2_R_UXTH:
117 return Operand(InputRegister32(index), UXTH);
118 case kMode_Operand2_R_SXTB:
119 return Operand(InputRegister32(index), SXTB);
120 case kMode_Operand2_R_SXTH:
121 return Operand(InputRegister32(index), SXTH);
122 case kMode_Operand2_R_SXTW:
123 return Operand(InputRegister32(index), SXTW);
124 case kMode_MRI:
125 case kMode_MRR:
126 break;
127 }
128 UNREACHABLE();
129 return Operand(-1);
130 }
131
InputOperand2_64(size_t index)132 Operand InputOperand2_64(size_t index) {
133 switch (AddressingModeField::decode(instr_->opcode())) {
134 case kMode_None:
135 return InputOperand64(index);
136 case kMode_Operand2_R_LSL_I:
137 return Operand(InputRegister64(index), LSL, InputInt6(index + 1));
138 case kMode_Operand2_R_LSR_I:
139 return Operand(InputRegister64(index), LSR, InputInt6(index + 1));
140 case kMode_Operand2_R_ASR_I:
141 return Operand(InputRegister64(index), ASR, InputInt6(index + 1));
142 case kMode_Operand2_R_ROR_I:
143 return Operand(InputRegister64(index), ROR, InputInt6(index + 1));
144 case kMode_Operand2_R_UXTB:
145 return Operand(InputRegister64(index), UXTB);
146 case kMode_Operand2_R_UXTH:
147 return Operand(InputRegister64(index), UXTH);
148 case kMode_Operand2_R_SXTB:
149 return Operand(InputRegister64(index), SXTB);
150 case kMode_Operand2_R_SXTH:
151 return Operand(InputRegister64(index), SXTH);
152 case kMode_Operand2_R_SXTW:
153 return Operand(InputRegister64(index), SXTW);
154 case kMode_MRI:
155 case kMode_MRR:
156 break;
157 }
158 UNREACHABLE();
159 return Operand(-1);
160 }
161
MemoryOperand(size_t * first_index)162 MemOperand MemoryOperand(size_t* first_index) {
163 const size_t index = *first_index;
164 switch (AddressingModeField::decode(instr_->opcode())) {
165 case kMode_None:
166 case kMode_Operand2_R_LSR_I:
167 case kMode_Operand2_R_ASR_I:
168 case kMode_Operand2_R_ROR_I:
169 case kMode_Operand2_R_UXTB:
170 case kMode_Operand2_R_UXTH:
171 case kMode_Operand2_R_SXTB:
172 case kMode_Operand2_R_SXTH:
173 case kMode_Operand2_R_SXTW:
174 break;
175 case kMode_Operand2_R_LSL_I:
176 *first_index += 3;
177 return MemOperand(InputRegister(index + 0), InputRegister(index + 1),
178 LSL, InputInt32(index + 2));
179 case kMode_MRI:
180 *first_index += 2;
181 return MemOperand(InputRegister(index + 0), InputInt32(index + 1));
182 case kMode_MRR:
183 *first_index += 2;
184 return MemOperand(InputRegister(index + 0), InputRegister(index + 1));
185 }
186 UNREACHABLE();
187 return MemOperand(no_reg);
188 }
189
MemoryOperand(size_t first_index=0)190 MemOperand MemoryOperand(size_t first_index = 0) {
191 return MemoryOperand(&first_index);
192 }
193
ToOperand(InstructionOperand * op)194 Operand ToOperand(InstructionOperand* op) {
195 if (op->IsRegister()) {
196 return Operand(ToRegister(op));
197 }
198 return ToImmediate(op);
199 }
200
ToOperand32(InstructionOperand * op)201 Operand ToOperand32(InstructionOperand* op) {
202 if (op->IsRegister()) {
203 return Operand(ToRegister(op).W());
204 }
205 return ToImmediate(op);
206 }
207
ToImmediate(InstructionOperand * operand)208 Operand ToImmediate(InstructionOperand* operand) {
209 Constant constant = ToConstant(operand);
210 switch (constant.type()) {
211 case Constant::kInt32:
212 if (RelocInfo::IsWasmSizeReference(constant.rmode())) {
213 return Operand(constant.ToInt32(), constant.rmode());
214 } else {
215 return Operand(constant.ToInt32());
216 }
217 case Constant::kInt64:
218 if (RelocInfo::IsWasmPtrReference(constant.rmode())) {
219 return Operand(constant.ToInt64(), constant.rmode());
220 } else {
221 DCHECK(!RelocInfo::IsWasmSizeReference(constant.rmode()));
222 return Operand(constant.ToInt64());
223 }
224 case Constant::kFloat32:
225 return Operand(
226 isolate()->factory()->NewNumber(constant.ToFloat32(), TENURED));
227 case Constant::kFloat64:
228 return Operand(
229 isolate()->factory()->NewNumber(constant.ToFloat64(), TENURED));
230 case Constant::kExternalReference:
231 return Operand(constant.ToExternalReference());
232 case Constant::kHeapObject:
233 return Operand(constant.ToHeapObject());
234 case Constant::kRpoNumber:
235 UNREACHABLE(); // TODO(dcarney): RPO immediates on arm64.
236 break;
237 }
238 UNREACHABLE();
239 return Operand(-1);
240 }
241
ToMemOperand(InstructionOperand * op,MacroAssembler * masm) const242 MemOperand ToMemOperand(InstructionOperand* op, MacroAssembler* masm) const {
243 DCHECK_NOT_NULL(op);
244 DCHECK(op->IsStackSlot() || op->IsFPStackSlot());
245 return SlotToMemOperand(AllocatedOperand::cast(op)->index(), masm);
246 }
247
SlotToMemOperand(int slot,MacroAssembler * masm) const248 MemOperand SlotToMemOperand(int slot, MacroAssembler* masm) const {
249 FrameOffset offset = frame_access_state()->GetFrameOffset(slot);
250 if (offset.from_frame_pointer()) {
251 int from_sp = offset.offset() + frame_access_state()->GetSPToFPOffset();
252 // Convert FP-offsets to SP-offsets if it results in better code.
253 if (Assembler::IsImmLSUnscaled(from_sp) ||
254 Assembler::IsImmLSScaled(from_sp, LSDoubleWord)) {
255 offset = FrameOffset::FromStackPointer(from_sp);
256 }
257 }
258 return MemOperand(offset.from_stack_pointer() ? masm->StackPointer() : fp,
259 offset.offset());
260 }
261 };
262
263
264 namespace {
265
266 class OutOfLineLoadNaN32 final : public OutOfLineCode {
267 public:
OutOfLineLoadNaN32(CodeGenerator * gen,DoubleRegister result)268 OutOfLineLoadNaN32(CodeGenerator* gen, DoubleRegister result)
269 : OutOfLineCode(gen), result_(result) {}
270
Generate()271 void Generate() final {
272 __ Fmov(result_, std::numeric_limits<float>::quiet_NaN());
273 }
274
275 private:
276 DoubleRegister const result_;
277 };
278
279
280 class OutOfLineLoadNaN64 final : public OutOfLineCode {
281 public:
OutOfLineLoadNaN64(CodeGenerator * gen,DoubleRegister result)282 OutOfLineLoadNaN64(CodeGenerator* gen, DoubleRegister result)
283 : OutOfLineCode(gen), result_(result) {}
284
Generate()285 void Generate() final {
286 __ Fmov(result_, std::numeric_limits<double>::quiet_NaN());
287 }
288
289 private:
290 DoubleRegister const result_;
291 };
292
293
294 class OutOfLineLoadZero final : public OutOfLineCode {
295 public:
OutOfLineLoadZero(CodeGenerator * gen,Register result)296 OutOfLineLoadZero(CodeGenerator* gen, Register result)
297 : OutOfLineCode(gen), result_(result) {}
298
Generate()299 void Generate() final { __ Mov(result_, 0); }
300
301 private:
302 Register const result_;
303 };
304
305
306 class OutOfLineRecordWrite final : public OutOfLineCode {
307 public:
OutOfLineRecordWrite(CodeGenerator * gen,Register object,Operand index,Register value,Register scratch0,Register scratch1,RecordWriteMode mode,UnwindingInfoWriter * unwinding_info_writer)308 OutOfLineRecordWrite(CodeGenerator* gen, Register object, Operand index,
309 Register value, Register scratch0, Register scratch1,
310 RecordWriteMode mode,
311 UnwindingInfoWriter* unwinding_info_writer)
312 : OutOfLineCode(gen),
313 object_(object),
314 index_(index),
315 value_(value),
316 scratch0_(scratch0),
317 scratch1_(scratch1),
318 mode_(mode),
319 must_save_lr_(!gen->frame_access_state()->has_frame()),
320 unwinding_info_writer_(unwinding_info_writer) {}
321
Generate()322 void Generate() final {
323 if (mode_ > RecordWriteMode::kValueIsPointer) {
324 __ JumpIfSmi(value_, exit());
325 }
326 __ CheckPageFlagClear(value_, scratch0_,
327 MemoryChunk::kPointersToHereAreInterestingMask,
328 exit());
329 RememberedSetAction const remembered_set_action =
330 mode_ > RecordWriteMode::kValueIsMap ? EMIT_REMEMBERED_SET
331 : OMIT_REMEMBERED_SET;
332 SaveFPRegsMode const save_fp_mode =
333 frame()->DidAllocateDoubleRegisters() ? kSaveFPRegs : kDontSaveFPRegs;
334 if (must_save_lr_) {
335 // We need to save and restore lr if the frame was elided.
336 __ Push(lr);
337 unwinding_info_writer_->MarkLinkRegisterOnTopOfStack(__ pc_offset(),
338 __ StackPointer());
339 }
340 RecordWriteStub stub(isolate(), object_, scratch0_, scratch1_,
341 remembered_set_action, save_fp_mode);
342 __ Add(scratch1_, object_, index_);
343 __ CallStub(&stub);
344 if (must_save_lr_) {
345 __ Pop(lr);
346 unwinding_info_writer_->MarkPopLinkRegisterFromTopOfStack(__ pc_offset());
347 }
348 }
349
350 private:
351 Register const object_;
352 Operand const index_;
353 Register const value_;
354 Register const scratch0_;
355 Register const scratch1_;
356 RecordWriteMode const mode_;
357 bool must_save_lr_;
358 UnwindingInfoWriter* const unwinding_info_writer_;
359 };
360
361
FlagsConditionToCondition(FlagsCondition condition)362 Condition FlagsConditionToCondition(FlagsCondition condition) {
363 switch (condition) {
364 case kEqual:
365 return eq;
366 case kNotEqual:
367 return ne;
368 case kSignedLessThan:
369 return lt;
370 case kSignedGreaterThanOrEqual:
371 return ge;
372 case kSignedLessThanOrEqual:
373 return le;
374 case kSignedGreaterThan:
375 return gt;
376 case kUnsignedLessThan:
377 return lo;
378 case kUnsignedGreaterThanOrEqual:
379 return hs;
380 case kUnsignedLessThanOrEqual:
381 return ls;
382 case kUnsignedGreaterThan:
383 return hi;
384 case kFloatLessThanOrUnordered:
385 return lt;
386 case kFloatGreaterThanOrEqual:
387 return ge;
388 case kFloatLessThanOrEqual:
389 return ls;
390 case kFloatGreaterThanOrUnordered:
391 return hi;
392 case kFloatLessThan:
393 return lo;
394 case kFloatGreaterThanOrEqualOrUnordered:
395 return hs;
396 case kFloatLessThanOrEqualOrUnordered:
397 return le;
398 case kFloatGreaterThan:
399 return gt;
400 case kOverflow:
401 return vs;
402 case kNotOverflow:
403 return vc;
404 case kUnorderedEqual:
405 case kUnorderedNotEqual:
406 break;
407 case kPositiveOrZero:
408 return pl;
409 case kNegative:
410 return mi;
411 }
412 UNREACHABLE();
413 return nv;
414 }
415
416 } // namespace
417
418 #define ASSEMBLE_BOUNDS_CHECK(offset, length, out_of_bounds) \
419 do { \
420 if (length.IsImmediate() && \
421 base::bits::IsPowerOfTwo64(length.ImmediateValue())) { \
422 __ Tst(offset, ~(length.ImmediateValue() - 1)); \
423 __ B(ne, out_of_bounds); \
424 } else { \
425 __ Cmp(offset, length); \
426 __ B(hs, out_of_bounds); \
427 } \
428 } while (0)
429
430 #define ASSEMBLE_CHECKED_LOAD_FLOAT(width) \
431 do { \
432 auto result = i.OutputFloat##width##Register(); \
433 auto buffer = i.InputRegister(0); \
434 auto offset = i.InputRegister32(1); \
435 auto length = i.InputOperand32(2); \
436 auto ool = new (zone()) OutOfLineLoadNaN##width(this, result); \
437 ASSEMBLE_BOUNDS_CHECK(offset, length, ool->entry()); \
438 __ Ldr(result, MemOperand(buffer, offset, UXTW)); \
439 __ Bind(ool->exit()); \
440 } while (0)
441
442 #define ASSEMBLE_CHECKED_LOAD_INTEGER(asm_instr) \
443 do { \
444 auto result = i.OutputRegister32(); \
445 auto buffer = i.InputRegister(0); \
446 auto offset = i.InputRegister32(1); \
447 auto length = i.InputOperand32(2); \
448 auto ool = new (zone()) OutOfLineLoadZero(this, result); \
449 ASSEMBLE_BOUNDS_CHECK(offset, length, ool->entry()); \
450 __ asm_instr(result, MemOperand(buffer, offset, UXTW)); \
451 __ Bind(ool->exit()); \
452 } while (0)
453
454 #define ASSEMBLE_CHECKED_LOAD_INTEGER_64(asm_instr) \
455 do { \
456 auto result = i.OutputRegister(); \
457 auto buffer = i.InputRegister(0); \
458 auto offset = i.InputRegister32(1); \
459 auto length = i.InputOperand32(2); \
460 auto ool = new (zone()) OutOfLineLoadZero(this, result); \
461 ASSEMBLE_BOUNDS_CHECK(offset, length, ool->entry()); \
462 __ asm_instr(result, MemOperand(buffer, offset, UXTW)); \
463 __ Bind(ool->exit()); \
464 } while (0)
465
466 #define ASSEMBLE_CHECKED_STORE_FLOAT(width) \
467 do { \
468 auto buffer = i.InputRegister(0); \
469 auto offset = i.InputRegister32(1); \
470 auto length = i.InputOperand32(2); \
471 auto value = i.InputFloat##width##OrZeroRegister(3); \
472 Label done; \
473 ASSEMBLE_BOUNDS_CHECK(offset, length, &done); \
474 __ Str(value, MemOperand(buffer, offset, UXTW)); \
475 __ Bind(&done); \
476 } while (0)
477
478 #define ASSEMBLE_CHECKED_STORE_INTEGER(asm_instr) \
479 do { \
480 auto buffer = i.InputRegister(0); \
481 auto offset = i.InputRegister32(1); \
482 auto length = i.InputOperand32(2); \
483 auto value = i.InputOrZeroRegister32(3); \
484 Label done; \
485 ASSEMBLE_BOUNDS_CHECK(offset, length, &done); \
486 __ asm_instr(value, MemOperand(buffer, offset, UXTW)); \
487 __ Bind(&done); \
488 } while (0)
489
490 #define ASSEMBLE_CHECKED_STORE_INTEGER_64(asm_instr) \
491 do { \
492 auto buffer = i.InputRegister(0); \
493 auto offset = i.InputRegister32(1); \
494 auto length = i.InputOperand32(2); \
495 auto value = i.InputOrZeroRegister64(3); \
496 Label done; \
497 ASSEMBLE_BOUNDS_CHECK(offset, length, &done); \
498 __ asm_instr(value, MemOperand(buffer, offset, UXTW)); \
499 __ Bind(&done); \
500 } while (0)
501
502 #define ASSEMBLE_SHIFT(asm_instr, width) \
503 do { \
504 if (instr->InputAt(1)->IsRegister()) { \
505 __ asm_instr(i.OutputRegister##width(), i.InputRegister##width(0), \
506 i.InputRegister##width(1)); \
507 } else { \
508 uint32_t imm = \
509 static_cast<uint32_t>(i.InputOperand##width(1).ImmediateValue()); \
510 __ asm_instr(i.OutputRegister##width(), i.InputRegister##width(0), \
511 imm % (width)); \
512 } \
513 } while (0)
514
515 #define ASSEMBLE_ATOMIC_LOAD_INTEGER(asm_instr) \
516 do { \
517 __ asm_instr(i.OutputRegister(), \
518 MemOperand(i.InputRegister(0), i.InputRegister(1))); \
519 __ Dmb(InnerShareable, BarrierAll); \
520 } while (0)
521
522 #define ASSEMBLE_ATOMIC_STORE_INTEGER(asm_instr) \
523 do { \
524 __ Dmb(InnerShareable, BarrierAll); \
525 __ asm_instr(i.InputRegister(2), \
526 MemOperand(i.InputRegister(0), i.InputRegister(1))); \
527 __ Dmb(InnerShareable, BarrierAll); \
528 } while (0)
529
530 #define ASSEMBLE_IEEE754_BINOP(name) \
531 do { \
532 FrameScope scope(masm(), StackFrame::MANUAL); \
533 __ CallCFunction(ExternalReference::ieee754_##name##_function(isolate()), \
534 0, 2); \
535 } while (0)
536
537 #define ASSEMBLE_IEEE754_UNOP(name) \
538 do { \
539 FrameScope scope(masm(), StackFrame::MANUAL); \
540 __ CallCFunction(ExternalReference::ieee754_##name##_function(isolate()), \
541 0, 1); \
542 } while (0)
543
AssembleDeconstructFrame()544 void CodeGenerator::AssembleDeconstructFrame() {
545 const CallDescriptor* descriptor = linkage()->GetIncomingDescriptor();
546 if (descriptor->IsCFunctionCall() || descriptor->UseNativeStack()) {
547 __ Mov(csp, fp);
548 } else {
549 __ Mov(jssp, fp);
550 }
551 __ Pop(fp, lr);
552
553 unwinding_info_writer_.MarkFrameDeconstructed(__ pc_offset());
554 }
555
AssemblePrepareTailCall()556 void CodeGenerator::AssemblePrepareTailCall() {
557 if (frame_access_state()->has_frame()) {
558 __ Ldr(lr, MemOperand(fp, StandardFrameConstants::kCallerPCOffset));
559 __ Ldr(fp, MemOperand(fp, StandardFrameConstants::kCallerFPOffset));
560 }
561 frame_access_state()->SetFrameAccessToSP();
562 }
563
AssemblePopArgumentsAdaptorFrame(Register args_reg,Register scratch1,Register scratch2,Register scratch3)564 void CodeGenerator::AssemblePopArgumentsAdaptorFrame(Register args_reg,
565 Register scratch1,
566 Register scratch2,
567 Register scratch3) {
568 DCHECK(!AreAliased(args_reg, scratch1, scratch2, scratch3));
569 Label done;
570
571 // Check if current frame is an arguments adaptor frame.
572 __ Ldr(scratch1, MemOperand(fp, StandardFrameConstants::kContextOffset));
573 __ Cmp(scratch1,
574 Operand(StackFrame::TypeToMarker(StackFrame::ARGUMENTS_ADAPTOR)));
575 __ B(ne, &done);
576
577 // Load arguments count from current arguments adaptor frame (note, it
578 // does not include receiver).
579 Register caller_args_count_reg = scratch1;
580 __ Ldr(caller_args_count_reg,
581 MemOperand(fp, ArgumentsAdaptorFrameConstants::kLengthOffset));
582 __ SmiUntag(caller_args_count_reg);
583
584 ParameterCount callee_args_count(args_reg);
585 __ PrepareForTailCall(callee_args_count, caller_args_count_reg, scratch2,
586 scratch3);
587 __ bind(&done);
588 }
589
590 namespace {
591
AdjustStackPointerForTailCall(MacroAssembler * masm,FrameAccessState * state,int new_slot_above_sp,bool allow_shrinkage=true)592 void AdjustStackPointerForTailCall(MacroAssembler* masm,
593 FrameAccessState* state,
594 int new_slot_above_sp,
595 bool allow_shrinkage = true) {
596 int current_sp_offset = state->GetSPToFPSlotCount() +
597 StandardFrameConstants::kFixedSlotCountAboveFp;
598 int stack_slot_delta = new_slot_above_sp - current_sp_offset;
599 if (stack_slot_delta > 0) {
600 masm->Claim(stack_slot_delta);
601 state->IncreaseSPDelta(stack_slot_delta);
602 } else if (allow_shrinkage && stack_slot_delta < 0) {
603 masm->Drop(-stack_slot_delta);
604 state->IncreaseSPDelta(stack_slot_delta);
605 }
606 }
607
608 } // namespace
609
AssembleTailCallBeforeGap(Instruction * instr,int first_unused_stack_slot)610 void CodeGenerator::AssembleTailCallBeforeGap(Instruction* instr,
611 int first_unused_stack_slot) {
612 AdjustStackPointerForTailCall(masm(), frame_access_state(),
613 first_unused_stack_slot, false);
614 }
615
AssembleTailCallAfterGap(Instruction * instr,int first_unused_stack_slot)616 void CodeGenerator::AssembleTailCallAfterGap(Instruction* instr,
617 int first_unused_stack_slot) {
618 AdjustStackPointerForTailCall(masm(), frame_access_state(),
619 first_unused_stack_slot);
620 }
621
622 // Assembles an instruction after register allocation, producing machine code.
AssembleArchInstruction(Instruction * instr)623 CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
624 Instruction* instr) {
625 Arm64OperandConverter i(this, instr);
626 InstructionCode opcode = instr->opcode();
627 ArchOpcode arch_opcode = ArchOpcodeField::decode(opcode);
628 switch (arch_opcode) {
629 case kArchCallCodeObject: {
630 EnsureSpaceForLazyDeopt();
631 if (instr->InputAt(0)->IsImmediate()) {
632 __ Call(Handle<Code>::cast(i.InputHeapObject(0)),
633 RelocInfo::CODE_TARGET);
634 } else {
635 Register target = i.InputRegister(0);
636 __ Add(target, target, Code::kHeaderSize - kHeapObjectTag);
637 __ Call(target);
638 }
639 RecordCallPosition(instr);
640 // TODO(titzer): this is ugly. JSSP should be a caller-save register
641 // in this case, but it is not possible to express in the register
642 // allocator.
643 CallDescriptor::Flags flags(MiscField::decode(opcode));
644 if (flags & CallDescriptor::kRestoreJSSP) {
645 __ Ldr(jssp, MemOperand(csp));
646 __ Mov(csp, jssp);
647 }
648 if (flags & CallDescriptor::kRestoreCSP) {
649 __ Mov(csp, jssp);
650 __ AssertCspAligned();
651 }
652 frame_access_state()->ClearSPDelta();
653 break;
654 }
655 case kArchTailCallCodeObjectFromJSFunction:
656 case kArchTailCallCodeObject: {
657 if (arch_opcode == kArchTailCallCodeObjectFromJSFunction) {
658 AssemblePopArgumentsAdaptorFrame(kJavaScriptCallArgCountRegister,
659 i.TempRegister(0), i.TempRegister(1),
660 i.TempRegister(2));
661 }
662 if (instr->InputAt(0)->IsImmediate()) {
663 __ Jump(Handle<Code>::cast(i.InputHeapObject(0)),
664 RelocInfo::CODE_TARGET);
665 } else {
666 Register target = i.InputRegister(0);
667 __ Add(target, target, Code::kHeaderSize - kHeapObjectTag);
668 __ Jump(target);
669 }
670 unwinding_info_writer_.MarkBlockWillExit();
671 frame_access_state()->ClearSPDelta();
672 frame_access_state()->SetFrameAccessToDefault();
673 break;
674 }
675 case kArchTailCallAddress: {
676 CHECK(!instr->InputAt(0)->IsImmediate());
677 __ Jump(i.InputRegister(0));
678 unwinding_info_writer_.MarkBlockWillExit();
679 frame_access_state()->ClearSPDelta();
680 frame_access_state()->SetFrameAccessToDefault();
681 break;
682 }
683 case kArchCallJSFunction: {
684 EnsureSpaceForLazyDeopt();
685 Register func = i.InputRegister(0);
686 if (FLAG_debug_code) {
687 // Check the function's context matches the context argument.
688 UseScratchRegisterScope scope(masm());
689 Register temp = scope.AcquireX();
690 __ Ldr(temp, FieldMemOperand(func, JSFunction::kContextOffset));
691 __ cmp(cp, temp);
692 __ Assert(eq, kWrongFunctionContext);
693 }
694 __ Ldr(x10, FieldMemOperand(func, JSFunction::kCodeEntryOffset));
695 __ Call(x10);
696 RecordCallPosition(instr);
697 // TODO(titzer): this is ugly. JSSP should be a caller-save register
698 // in this case, but it is not possible to express in the register
699 // allocator.
700 CallDescriptor::Flags flags(MiscField::decode(opcode));
701 if (flags & CallDescriptor::kRestoreJSSP) {
702 __ Ldr(jssp, MemOperand(csp));
703 __ Mov(csp, jssp);
704 }
705 if (flags & CallDescriptor::kRestoreCSP) {
706 __ Mov(csp, jssp);
707 __ AssertCspAligned();
708 }
709 frame_access_state()->ClearSPDelta();
710 break;
711 }
712 case kArchTailCallJSFunctionFromJSFunction: {
713 Register func = i.InputRegister(0);
714 if (FLAG_debug_code) {
715 // Check the function's context matches the context argument.
716 UseScratchRegisterScope scope(masm());
717 Register temp = scope.AcquireX();
718 __ Ldr(temp, FieldMemOperand(func, JSFunction::kContextOffset));
719 __ cmp(cp, temp);
720 __ Assert(eq, kWrongFunctionContext);
721 }
722 AssemblePopArgumentsAdaptorFrame(kJavaScriptCallArgCountRegister,
723 i.TempRegister(0), i.TempRegister(1),
724 i.TempRegister(2));
725 __ Ldr(x10, FieldMemOperand(func, JSFunction::kCodeEntryOffset));
726 __ Jump(x10);
727 frame_access_state()->ClearSPDelta();
728 frame_access_state()->SetFrameAccessToDefault();
729 break;
730 }
731 case kArchPrepareCallCFunction:
732 // We don't need kArchPrepareCallCFunction on arm64 as the instruction
733 // selector already perform a Claim to reserve space on the stack and
734 // guarantee correct alignment of stack pointer.
735 UNREACHABLE();
736 break;
737 case kArchPrepareTailCall:
738 AssemblePrepareTailCall();
739 break;
740 case kArchCallCFunction: {
741 int const num_parameters = MiscField::decode(instr->opcode());
742 if (instr->InputAt(0)->IsImmediate()) {
743 ExternalReference ref = i.InputExternalReference(0);
744 __ CallCFunction(ref, num_parameters, 0);
745 } else {
746 Register func = i.InputRegister(0);
747 __ CallCFunction(func, num_parameters, 0);
748 }
749 // CallCFunction only supports register arguments so we never need to call
750 // frame()->ClearOutgoingParameterSlots() here.
751 DCHECK(frame_access_state()->sp_delta() == 0);
752 break;
753 }
754 case kArchJmp:
755 AssembleArchJump(i.InputRpo(0));
756 break;
757 case kArchTableSwitch:
758 AssembleArchTableSwitch(instr);
759 break;
760 case kArchLookupSwitch:
761 AssembleArchLookupSwitch(instr);
762 break;
763 case kArchDebugBreak:
764 __ Debug("kArchDebugBreak", 0, BREAK);
765 break;
766 case kArchComment: {
767 Address comment_string = i.InputExternalReference(0).address();
768 __ RecordComment(reinterpret_cast<const char*>(comment_string));
769 break;
770 }
771 case kArchNop:
772 case kArchThrowTerminator:
773 // don't emit code for nops.
774 break;
775 case kArchDeoptimize: {
776 int deopt_state_id =
777 BuildTranslation(instr, -1, 0, OutputFrameStateCombine::Ignore());
778 CodeGenResult result =
779 AssembleDeoptimizerCall(deopt_state_id, current_source_position_);
780 if (result != kSuccess) return result;
781 break;
782 }
783 case kArchRet:
784 AssembleReturn(instr->InputAt(0));
785 break;
786 case kArchStackPointer:
787 __ mov(i.OutputRegister(), masm()->StackPointer());
788 break;
789 case kArchFramePointer:
790 __ mov(i.OutputRegister(), fp);
791 break;
792 case kArchParentFramePointer:
793 if (frame_access_state()->has_frame()) {
794 __ ldr(i.OutputRegister(), MemOperand(fp, 0));
795 } else {
796 __ mov(i.OutputRegister(), fp);
797 }
798 break;
799 case kArchTruncateDoubleToI:
800 __ TruncateDoubleToI(i.OutputRegister(), i.InputDoubleRegister(0));
801 break;
802 case kArchStoreWithWriteBarrier: {
803 RecordWriteMode mode =
804 static_cast<RecordWriteMode>(MiscField::decode(instr->opcode()));
805 AddressingMode addressing_mode =
806 AddressingModeField::decode(instr->opcode());
807 Register object = i.InputRegister(0);
808 Operand index(0);
809 if (addressing_mode == kMode_MRI) {
810 index = Operand(i.InputInt64(1));
811 } else {
812 DCHECK_EQ(addressing_mode, kMode_MRR);
813 index = Operand(i.InputRegister(1));
814 }
815 Register value = i.InputRegister(2);
816 Register scratch0 = i.TempRegister(0);
817 Register scratch1 = i.TempRegister(1);
818 auto ool = new (zone())
819 OutOfLineRecordWrite(this, object, index, value, scratch0, scratch1,
820 mode, &unwinding_info_writer_);
821 __ Str(value, MemOperand(object, index));
822 __ CheckPageFlagSet(object, scratch0,
823 MemoryChunk::kPointersFromHereAreInterestingMask,
824 ool->entry());
825 __ Bind(ool->exit());
826 break;
827 }
828 case kArchStackSlot: {
829 FrameOffset offset =
830 frame_access_state()->GetFrameOffset(i.InputInt32(0));
831 Register base;
832 if (offset.from_stack_pointer()) {
833 base = __ StackPointer();
834 } else {
835 base = fp;
836 }
837 __ Add(i.OutputRegister(0), base, Operand(offset.offset()));
838 break;
839 }
840 case kIeee754Float64Acos:
841 ASSEMBLE_IEEE754_UNOP(acos);
842 break;
843 case kIeee754Float64Acosh:
844 ASSEMBLE_IEEE754_UNOP(acosh);
845 break;
846 case kIeee754Float64Asin:
847 ASSEMBLE_IEEE754_UNOP(asin);
848 break;
849 case kIeee754Float64Asinh:
850 ASSEMBLE_IEEE754_UNOP(asinh);
851 break;
852 case kIeee754Float64Atan:
853 ASSEMBLE_IEEE754_UNOP(atan);
854 break;
855 case kIeee754Float64Atanh:
856 ASSEMBLE_IEEE754_UNOP(atanh);
857 break;
858 case kIeee754Float64Atan2:
859 ASSEMBLE_IEEE754_BINOP(atan2);
860 break;
861 case kIeee754Float64Cos:
862 ASSEMBLE_IEEE754_UNOP(cos);
863 break;
864 case kIeee754Float64Cosh:
865 ASSEMBLE_IEEE754_UNOP(cosh);
866 break;
867 case kIeee754Float64Cbrt:
868 ASSEMBLE_IEEE754_UNOP(cbrt);
869 break;
870 case kIeee754Float64Exp:
871 ASSEMBLE_IEEE754_UNOP(exp);
872 break;
873 case kIeee754Float64Expm1:
874 ASSEMBLE_IEEE754_UNOP(expm1);
875 break;
876 case kIeee754Float64Log:
877 ASSEMBLE_IEEE754_UNOP(log);
878 break;
879 case kIeee754Float64Log1p:
880 ASSEMBLE_IEEE754_UNOP(log1p);
881 break;
882 case kIeee754Float64Log2:
883 ASSEMBLE_IEEE754_UNOP(log2);
884 break;
885 case kIeee754Float64Log10:
886 ASSEMBLE_IEEE754_UNOP(log10);
887 break;
888 case kIeee754Float64Pow: {
889 MathPowStub stub(isolate(), MathPowStub::DOUBLE);
890 __ CallStub(&stub);
891 break;
892 }
893 case kIeee754Float64Sin:
894 ASSEMBLE_IEEE754_UNOP(sin);
895 break;
896 case kIeee754Float64Sinh:
897 ASSEMBLE_IEEE754_UNOP(sinh);
898 break;
899 case kIeee754Float64Tan:
900 ASSEMBLE_IEEE754_UNOP(tan);
901 break;
902 case kIeee754Float64Tanh:
903 ASSEMBLE_IEEE754_UNOP(tanh);
904 break;
905 case kArm64Float32RoundDown:
906 __ Frintm(i.OutputFloat32Register(), i.InputFloat32Register(0));
907 break;
908 case kArm64Float64RoundDown:
909 __ Frintm(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
910 break;
911 case kArm64Float32RoundUp:
912 __ Frintp(i.OutputFloat32Register(), i.InputFloat32Register(0));
913 break;
914 case kArm64Float64RoundUp:
915 __ Frintp(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
916 break;
917 case kArm64Float64RoundTiesAway:
918 __ Frinta(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
919 break;
920 case kArm64Float32RoundTruncate:
921 __ Frintz(i.OutputFloat32Register(), i.InputFloat32Register(0));
922 break;
923 case kArm64Float64RoundTruncate:
924 __ Frintz(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
925 break;
926 case kArm64Float32RoundTiesEven:
927 __ Frintn(i.OutputFloat32Register(), i.InputFloat32Register(0));
928 break;
929 case kArm64Float64RoundTiesEven:
930 __ Frintn(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
931 break;
932 case kArm64Add:
933 if (FlagsModeField::decode(opcode) != kFlags_none) {
934 __ Adds(i.OutputRegister(), i.InputOrZeroRegister64(0),
935 i.InputOperand2_64(1));
936 } else {
937 __ Add(i.OutputRegister(), i.InputOrZeroRegister64(0),
938 i.InputOperand2_64(1));
939 }
940 break;
941 case kArm64Add32:
942 if (FlagsModeField::decode(opcode) != kFlags_none) {
943 __ Adds(i.OutputRegister32(), i.InputOrZeroRegister32(0),
944 i.InputOperand2_32(1));
945 } else {
946 __ Add(i.OutputRegister32(), i.InputOrZeroRegister32(0),
947 i.InputOperand2_32(1));
948 }
949 break;
950 case kArm64And:
951 if (FlagsModeField::decode(opcode) != kFlags_none) {
952 // The ands instruction only sets N and Z, so only the following
953 // conditions make sense.
954 DCHECK(FlagsConditionField::decode(opcode) == kEqual ||
955 FlagsConditionField::decode(opcode) == kNotEqual ||
956 FlagsConditionField::decode(opcode) == kPositiveOrZero ||
957 FlagsConditionField::decode(opcode) == kNegative);
958 __ Ands(i.OutputRegister(), i.InputOrZeroRegister64(0),
959 i.InputOperand2_64(1));
960 } else {
961 __ And(i.OutputRegister(), i.InputOrZeroRegister64(0),
962 i.InputOperand2_64(1));
963 }
964 break;
965 case kArm64And32:
966 if (FlagsModeField::decode(opcode) != kFlags_none) {
967 // The ands instruction only sets N and Z, so only the following
968 // conditions make sense.
969 DCHECK(FlagsConditionField::decode(opcode) == kEqual ||
970 FlagsConditionField::decode(opcode) == kNotEqual ||
971 FlagsConditionField::decode(opcode) == kPositiveOrZero ||
972 FlagsConditionField::decode(opcode) == kNegative);
973 __ Ands(i.OutputRegister32(), i.InputOrZeroRegister32(0),
974 i.InputOperand2_32(1));
975 } else {
976 __ And(i.OutputRegister32(), i.InputOrZeroRegister32(0),
977 i.InputOperand2_32(1));
978 }
979 break;
980 case kArm64Bic:
981 __ Bic(i.OutputRegister(), i.InputOrZeroRegister64(0),
982 i.InputOperand2_64(1));
983 break;
984 case kArm64Bic32:
985 __ Bic(i.OutputRegister32(), i.InputOrZeroRegister32(0),
986 i.InputOperand2_32(1));
987 break;
988 case kArm64Mul:
989 __ Mul(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1));
990 break;
991 case kArm64Mul32:
992 __ Mul(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1));
993 break;
994 case kArm64Smull:
995 __ Smull(i.OutputRegister(), i.InputRegister32(0), i.InputRegister32(1));
996 break;
997 case kArm64Umull:
998 __ Umull(i.OutputRegister(), i.InputRegister32(0), i.InputRegister32(1));
999 break;
1000 case kArm64Madd:
1001 __ Madd(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1),
1002 i.InputRegister(2));
1003 break;
1004 case kArm64Madd32:
1005 __ Madd(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1),
1006 i.InputRegister32(2));
1007 break;
1008 case kArm64Msub:
1009 __ Msub(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1),
1010 i.InputRegister(2));
1011 break;
1012 case kArm64Msub32:
1013 __ Msub(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1),
1014 i.InputRegister32(2));
1015 break;
1016 case kArm64Mneg:
1017 __ Mneg(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1));
1018 break;
1019 case kArm64Mneg32:
1020 __ Mneg(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1));
1021 break;
1022 case kArm64Idiv:
1023 __ Sdiv(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1));
1024 break;
1025 case kArm64Idiv32:
1026 __ Sdiv(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1));
1027 break;
1028 case kArm64Udiv:
1029 __ Udiv(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1));
1030 break;
1031 case kArm64Udiv32:
1032 __ Udiv(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1));
1033 break;
1034 case kArm64Imod: {
1035 UseScratchRegisterScope scope(masm());
1036 Register temp = scope.AcquireX();
1037 __ Sdiv(temp, i.InputRegister(0), i.InputRegister(1));
1038 __ Msub(i.OutputRegister(), temp, i.InputRegister(1), i.InputRegister(0));
1039 break;
1040 }
1041 case kArm64Imod32: {
1042 UseScratchRegisterScope scope(masm());
1043 Register temp = scope.AcquireW();
1044 __ Sdiv(temp, i.InputRegister32(0), i.InputRegister32(1));
1045 __ Msub(i.OutputRegister32(), temp, i.InputRegister32(1),
1046 i.InputRegister32(0));
1047 break;
1048 }
1049 case kArm64Umod: {
1050 UseScratchRegisterScope scope(masm());
1051 Register temp = scope.AcquireX();
1052 __ Udiv(temp, i.InputRegister(0), i.InputRegister(1));
1053 __ Msub(i.OutputRegister(), temp, i.InputRegister(1), i.InputRegister(0));
1054 break;
1055 }
1056 case kArm64Umod32: {
1057 UseScratchRegisterScope scope(masm());
1058 Register temp = scope.AcquireW();
1059 __ Udiv(temp, i.InputRegister32(0), i.InputRegister32(1));
1060 __ Msub(i.OutputRegister32(), temp, i.InputRegister32(1),
1061 i.InputRegister32(0));
1062 break;
1063 }
1064 case kArm64Not:
1065 __ Mvn(i.OutputRegister(), i.InputOperand(0));
1066 break;
1067 case kArm64Not32:
1068 __ Mvn(i.OutputRegister32(), i.InputOperand32(0));
1069 break;
1070 case kArm64Or:
1071 __ Orr(i.OutputRegister(), i.InputOrZeroRegister64(0),
1072 i.InputOperand2_64(1));
1073 break;
1074 case kArm64Or32:
1075 __ Orr(i.OutputRegister32(), i.InputOrZeroRegister32(0),
1076 i.InputOperand2_32(1));
1077 break;
1078 case kArm64Orn:
1079 __ Orn(i.OutputRegister(), i.InputOrZeroRegister64(0),
1080 i.InputOperand2_64(1));
1081 break;
1082 case kArm64Orn32:
1083 __ Orn(i.OutputRegister32(), i.InputOrZeroRegister32(0),
1084 i.InputOperand2_32(1));
1085 break;
1086 case kArm64Eor:
1087 __ Eor(i.OutputRegister(), i.InputOrZeroRegister64(0),
1088 i.InputOperand2_64(1));
1089 break;
1090 case kArm64Eor32:
1091 __ Eor(i.OutputRegister32(), i.InputOrZeroRegister32(0),
1092 i.InputOperand2_32(1));
1093 break;
1094 case kArm64Eon:
1095 __ Eon(i.OutputRegister(), i.InputOrZeroRegister64(0),
1096 i.InputOperand2_64(1));
1097 break;
1098 case kArm64Eon32:
1099 __ Eon(i.OutputRegister32(), i.InputOrZeroRegister32(0),
1100 i.InputOperand2_32(1));
1101 break;
1102 case kArm64Sub:
1103 if (FlagsModeField::decode(opcode) != kFlags_none) {
1104 __ Subs(i.OutputRegister(), i.InputOrZeroRegister64(0),
1105 i.InputOperand2_64(1));
1106 } else {
1107 __ Sub(i.OutputRegister(), i.InputOrZeroRegister64(0),
1108 i.InputOperand2_64(1));
1109 }
1110 break;
1111 case kArm64Sub32:
1112 if (FlagsModeField::decode(opcode) != kFlags_none) {
1113 __ Subs(i.OutputRegister32(), i.InputOrZeroRegister32(0),
1114 i.InputOperand2_32(1));
1115 } else {
1116 __ Sub(i.OutputRegister32(), i.InputOrZeroRegister32(0),
1117 i.InputOperand2_32(1));
1118 }
1119 break;
1120 case kArm64Lsl:
1121 ASSEMBLE_SHIFT(Lsl, 64);
1122 break;
1123 case kArm64Lsl32:
1124 ASSEMBLE_SHIFT(Lsl, 32);
1125 break;
1126 case kArm64Lsr:
1127 ASSEMBLE_SHIFT(Lsr, 64);
1128 break;
1129 case kArm64Lsr32:
1130 ASSEMBLE_SHIFT(Lsr, 32);
1131 break;
1132 case kArm64Asr:
1133 ASSEMBLE_SHIFT(Asr, 64);
1134 break;
1135 case kArm64Asr32:
1136 ASSEMBLE_SHIFT(Asr, 32);
1137 break;
1138 case kArm64Ror:
1139 ASSEMBLE_SHIFT(Ror, 64);
1140 break;
1141 case kArm64Ror32:
1142 ASSEMBLE_SHIFT(Ror, 32);
1143 break;
1144 case kArm64Mov32:
1145 __ Mov(i.OutputRegister32(), i.InputRegister32(0));
1146 break;
1147 case kArm64Sxtb32:
1148 __ Sxtb(i.OutputRegister32(), i.InputRegister32(0));
1149 break;
1150 case kArm64Sxth32:
1151 __ Sxth(i.OutputRegister32(), i.InputRegister32(0));
1152 break;
1153 case kArm64Sxtw:
1154 __ Sxtw(i.OutputRegister(), i.InputRegister32(0));
1155 break;
1156 case kArm64Sbfx32:
1157 __ Sbfx(i.OutputRegister32(), i.InputRegister32(0), i.InputInt5(1),
1158 i.InputInt5(2));
1159 break;
1160 case kArm64Ubfx:
1161 __ Ubfx(i.OutputRegister(), i.InputRegister(0), i.InputInt6(1),
1162 i.InputInt6(2));
1163 break;
1164 case kArm64Ubfx32:
1165 __ Ubfx(i.OutputRegister32(), i.InputRegister32(0), i.InputInt5(1),
1166 i.InputInt5(2));
1167 break;
1168 case kArm64Ubfiz32:
1169 __ Ubfiz(i.OutputRegister32(), i.InputRegister32(0), i.InputInt5(1),
1170 i.InputInt5(2));
1171 break;
1172 case kArm64Bfi:
1173 __ Bfi(i.OutputRegister(), i.InputRegister(1), i.InputInt6(2),
1174 i.InputInt6(3));
1175 break;
1176 case kArm64TestAndBranch32:
1177 case kArm64TestAndBranch:
1178 // Pseudo instructions turned into tbz/tbnz in AssembleArchBranch.
1179 break;
1180 case kArm64CompareAndBranch32:
1181 case kArm64CompareAndBranch:
1182 // Pseudo instruction turned into cbz/cbnz in AssembleArchBranch.
1183 break;
1184 case kArm64ClaimCSP: {
1185 int count = RoundUp(i.InputInt32(0), 2);
1186 Register prev = __ StackPointer();
1187 if (prev.Is(jssp)) {
1188 // TODO(titzer): make this a macro-assembler method.
1189 // Align the CSP and store the previous JSSP on the stack.
1190 UseScratchRegisterScope scope(masm());
1191 Register tmp = scope.AcquireX();
1192
1193 int sp_alignment = __ ActivationFrameAlignment();
1194 __ Sub(tmp, jssp, kPointerSize);
1195 __ And(tmp, tmp, Operand(~static_cast<uint64_t>(sp_alignment - 1)));
1196 __ Mov(csp, tmp);
1197 __ Str(jssp, MemOperand(csp));
1198 if (count > 0) {
1199 __ SetStackPointer(csp);
1200 __ Claim(count);
1201 __ SetStackPointer(prev);
1202 }
1203 } else {
1204 __ AssertCspAligned();
1205 if (count > 0) {
1206 __ Claim(count);
1207 frame_access_state()->IncreaseSPDelta(count);
1208 }
1209 }
1210 break;
1211 }
1212 case kArm64ClaimJSSP: {
1213 int count = i.InputInt32(0);
1214 if (csp.Is(__ StackPointer())) {
1215 // No JSSP is set up. Compute it from the CSP.
1216 __ AssertCspAligned();
1217 if (count > 0) {
1218 int even = RoundUp(count, 2);
1219 __ Sub(jssp, csp, count * kPointerSize);
1220 __ Sub(csp, csp, even * kPointerSize); // Must always be aligned.
1221 frame_access_state()->IncreaseSPDelta(even);
1222 } else {
1223 __ Mov(jssp, csp);
1224 }
1225 } else {
1226 // JSSP is the current stack pointer, just use regular Claim().
1227 __ Claim(count);
1228 frame_access_state()->IncreaseSPDelta(count);
1229 }
1230 break;
1231 }
1232 case kArm64PokeCSP: // fall through
1233 case kArm64PokeJSSP: {
1234 Register prev = __ StackPointer();
1235 __ SetStackPointer(arch_opcode == kArm64PokeCSP ? csp : jssp);
1236 Operand operand(i.InputInt32(1) * kPointerSize);
1237 if (instr->InputAt(0)->IsFPRegister()) {
1238 __ Poke(i.InputFloat64Register(0), operand);
1239 } else {
1240 __ Poke(i.InputRegister(0), operand);
1241 }
1242 __ SetStackPointer(prev);
1243 break;
1244 }
1245 case kArm64PokePair: {
1246 int slot = i.InputInt32(2) - 1;
1247 if (instr->InputAt(0)->IsFPRegister()) {
1248 __ PokePair(i.InputFloat64Register(1), i.InputFloat64Register(0),
1249 slot * kPointerSize);
1250 } else {
1251 __ PokePair(i.InputRegister(1), i.InputRegister(0),
1252 slot * kPointerSize);
1253 }
1254 break;
1255 }
1256 case kArm64Clz:
1257 __ Clz(i.OutputRegister64(), i.InputRegister64(0));
1258 break;
1259 case kArm64Clz32:
1260 __ Clz(i.OutputRegister32(), i.InputRegister32(0));
1261 break;
1262 case kArm64Rbit:
1263 __ Rbit(i.OutputRegister64(), i.InputRegister64(0));
1264 break;
1265 case kArm64Rbit32:
1266 __ Rbit(i.OutputRegister32(), i.InputRegister32(0));
1267 break;
1268 case kArm64Cmp:
1269 __ Cmp(i.InputOrZeroRegister64(0), i.InputOperand2_64(1));
1270 break;
1271 case kArm64Cmp32:
1272 __ Cmp(i.InputOrZeroRegister32(0), i.InputOperand2_32(1));
1273 break;
1274 case kArm64Cmn:
1275 __ Cmn(i.InputOrZeroRegister64(0), i.InputOperand2_64(1));
1276 break;
1277 case kArm64Cmn32:
1278 __ Cmn(i.InputOrZeroRegister32(0), i.InputOperand2_32(1));
1279 break;
1280 case kArm64Tst:
1281 __ Tst(i.InputOrZeroRegister64(0), i.InputOperand(1));
1282 break;
1283 case kArm64Tst32:
1284 __ Tst(i.InputOrZeroRegister32(0), i.InputOperand32(1));
1285 break;
1286 case kArm64Float32Cmp:
1287 if (instr->InputAt(1)->IsFPRegister()) {
1288 __ Fcmp(i.InputFloat32Register(0), i.InputFloat32Register(1));
1289 } else {
1290 DCHECK(instr->InputAt(1)->IsImmediate());
1291 // 0.0 is the only immediate supported by fcmp instructions.
1292 DCHECK(i.InputFloat32(1) == 0.0f);
1293 __ Fcmp(i.InputFloat32Register(0), i.InputFloat32(1));
1294 }
1295 break;
1296 case kArm64Float32Add:
1297 __ Fadd(i.OutputFloat32Register(), i.InputFloat32Register(0),
1298 i.InputFloat32Register(1));
1299 break;
1300 case kArm64Float32Sub:
1301 __ Fsub(i.OutputFloat32Register(), i.InputFloat32Register(0),
1302 i.InputFloat32Register(1));
1303 break;
1304 case kArm64Float32Mul:
1305 __ Fmul(i.OutputFloat32Register(), i.InputFloat32Register(0),
1306 i.InputFloat32Register(1));
1307 break;
1308 case kArm64Float32Div:
1309 __ Fdiv(i.OutputFloat32Register(), i.InputFloat32Register(0),
1310 i.InputFloat32Register(1));
1311 break;
1312 case kArm64Float32Abs:
1313 __ Fabs(i.OutputFloat32Register(), i.InputFloat32Register(0));
1314 break;
1315 case kArm64Float32Neg:
1316 __ Fneg(i.OutputFloat32Register(), i.InputFloat32Register(0));
1317 break;
1318 case kArm64Float32Sqrt:
1319 __ Fsqrt(i.OutputFloat32Register(), i.InputFloat32Register(0));
1320 break;
1321 case kArm64Float64Cmp:
1322 if (instr->InputAt(1)->IsFPRegister()) {
1323 __ Fcmp(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1324 } else {
1325 DCHECK(instr->InputAt(1)->IsImmediate());
1326 // 0.0 is the only immediate supported by fcmp instructions.
1327 DCHECK(i.InputDouble(1) == 0.0);
1328 __ Fcmp(i.InputDoubleRegister(0), i.InputDouble(1));
1329 }
1330 break;
1331 case kArm64Float64Add:
1332 __ Fadd(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1333 i.InputDoubleRegister(1));
1334 break;
1335 case kArm64Float64Sub:
1336 __ Fsub(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1337 i.InputDoubleRegister(1));
1338 break;
1339 case kArm64Float64Mul:
1340 __ Fmul(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1341 i.InputDoubleRegister(1));
1342 break;
1343 case kArm64Float64Div:
1344 __ Fdiv(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1345 i.InputDoubleRegister(1));
1346 break;
1347 case kArm64Float64Mod: {
1348 // TODO(dcarney): implement directly. See note in lithium-codegen-arm64.cc
1349 FrameScope scope(masm(), StackFrame::MANUAL);
1350 DCHECK(d0.is(i.InputDoubleRegister(0)));
1351 DCHECK(d1.is(i.InputDoubleRegister(1)));
1352 DCHECK(d0.is(i.OutputDoubleRegister()));
1353 // TODO(dcarney): make sure this saves all relevant registers.
1354 __ CallCFunction(ExternalReference::mod_two_doubles_operation(isolate()),
1355 0, 2);
1356 break;
1357 }
1358 case kArm64Float32Max: {
1359 __ Fmax(i.OutputFloat32Register(), i.InputFloat32Register(0),
1360 i.InputFloat32Register(1));
1361 break;
1362 }
1363 case kArm64Float64Max: {
1364 __ Fmax(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1365 i.InputDoubleRegister(1));
1366 break;
1367 }
1368 case kArm64Float32Min: {
1369 __ Fmin(i.OutputFloat32Register(), i.InputFloat32Register(0),
1370 i.InputFloat32Register(1));
1371 break;
1372 }
1373 case kArm64Float64Min: {
1374 __ Fmin(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1375 i.InputDoubleRegister(1));
1376 break;
1377 }
1378 case kArm64Float64Abs:
1379 __ Fabs(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
1380 break;
1381 case kArm64Float64Neg:
1382 __ Fneg(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
1383 break;
1384 case kArm64Float64Sqrt:
1385 __ Fsqrt(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
1386 break;
1387 case kArm64Float32ToFloat64:
1388 __ Fcvt(i.OutputDoubleRegister(), i.InputDoubleRegister(0).S());
1389 break;
1390 case kArm64Float64ToFloat32:
1391 __ Fcvt(i.OutputDoubleRegister().S(), i.InputDoubleRegister(0));
1392 break;
1393 case kArm64Float32ToInt32:
1394 __ Fcvtzs(i.OutputRegister32(), i.InputFloat32Register(0));
1395 // Avoid INT32_MAX as an overflow indicator and use INT32_MIN instead,
1396 // because INT32_MIN allows easier out-of-bounds detection.
1397 __ Cmn(i.OutputRegister32(), 1);
1398 __ Csinc(i.OutputRegister32(), i.OutputRegister32(), i.OutputRegister32(),
1399 vc);
1400 break;
1401 case kArm64Float64ToInt32:
1402 __ Fcvtzs(i.OutputRegister32(), i.InputDoubleRegister(0));
1403 break;
1404 case kArm64Float32ToUint32:
1405 __ Fcvtzu(i.OutputRegister32(), i.InputFloat32Register(0));
1406 // Avoid UINT32_MAX as an overflow indicator and use 0 instead,
1407 // because 0 allows easier out-of-bounds detection.
1408 __ Cmn(i.OutputRegister32(), 1);
1409 __ Adc(i.OutputRegister32(), i.OutputRegister32(), Operand(0));
1410 break;
1411 case kArm64Float64ToUint32:
1412 __ Fcvtzu(i.OutputRegister32(), i.InputDoubleRegister(0));
1413 break;
1414 case kArm64Float32ToInt64:
1415 __ Fcvtzs(i.OutputRegister64(), i.InputFloat32Register(0));
1416 if (i.OutputCount() > 1) {
1417 __ Mov(i.OutputRegister(1), 1);
1418 Label done;
1419 __ Cmp(i.OutputRegister(0), 1);
1420 __ Ccmp(i.OutputRegister(0), -1, VFlag, vc);
1421 __ Fccmp(i.InputFloat32Register(0), i.InputFloat32Register(0), VFlag,
1422 vc);
1423 __ B(vc, &done);
1424 __ Fcmp(i.InputFloat32Register(0), static_cast<float>(INT64_MIN));
1425 __ Cset(i.OutputRegister(1), eq);
1426 __ Bind(&done);
1427 }
1428 break;
1429 case kArm64Float64ToInt64:
1430 __ Fcvtzs(i.OutputRegister(0), i.InputDoubleRegister(0));
1431 if (i.OutputCount() > 1) {
1432 __ Mov(i.OutputRegister(1), 1);
1433 Label done;
1434 __ Cmp(i.OutputRegister(0), 1);
1435 __ Ccmp(i.OutputRegister(0), -1, VFlag, vc);
1436 __ Fccmp(i.InputDoubleRegister(0), i.InputDoubleRegister(0), VFlag, vc);
1437 __ B(vc, &done);
1438 __ Fcmp(i.InputDoubleRegister(0), static_cast<double>(INT64_MIN));
1439 __ Cset(i.OutputRegister(1), eq);
1440 __ Bind(&done);
1441 }
1442 break;
1443 case kArm64Float32ToUint64:
1444 __ Fcvtzu(i.OutputRegister64(), i.InputFloat32Register(0));
1445 if (i.OutputCount() > 1) {
1446 __ Fcmp(i.InputFloat32Register(0), -1.0);
1447 __ Ccmp(i.OutputRegister(0), -1, ZFlag, gt);
1448 __ Cset(i.OutputRegister(1), ne);
1449 }
1450 break;
1451 case kArm64Float64ToUint64:
1452 __ Fcvtzu(i.OutputRegister64(), i.InputDoubleRegister(0));
1453 if (i.OutputCount() > 1) {
1454 __ Fcmp(i.InputDoubleRegister(0), -1.0);
1455 __ Ccmp(i.OutputRegister(0), -1, ZFlag, gt);
1456 __ Cset(i.OutputRegister(1), ne);
1457 }
1458 break;
1459 case kArm64Int32ToFloat32:
1460 __ Scvtf(i.OutputFloat32Register(), i.InputRegister32(0));
1461 break;
1462 case kArm64Int32ToFloat64:
1463 __ Scvtf(i.OutputDoubleRegister(), i.InputRegister32(0));
1464 break;
1465 case kArm64Int64ToFloat32:
1466 __ Scvtf(i.OutputDoubleRegister().S(), i.InputRegister64(0));
1467 break;
1468 case kArm64Int64ToFloat64:
1469 __ Scvtf(i.OutputDoubleRegister(), i.InputRegister64(0));
1470 break;
1471 case kArm64Uint32ToFloat32:
1472 __ Ucvtf(i.OutputFloat32Register(), i.InputRegister32(0));
1473 break;
1474 case kArm64Uint32ToFloat64:
1475 __ Ucvtf(i.OutputDoubleRegister(), i.InputRegister32(0));
1476 break;
1477 case kArm64Uint64ToFloat32:
1478 __ Ucvtf(i.OutputDoubleRegister().S(), i.InputRegister64(0));
1479 break;
1480 case kArm64Uint64ToFloat64:
1481 __ Ucvtf(i.OutputDoubleRegister(), i.InputRegister64(0));
1482 break;
1483 case kArm64Float64ExtractLowWord32:
1484 __ Fmov(i.OutputRegister32(), i.InputFloat32Register(0));
1485 break;
1486 case kArm64Float64ExtractHighWord32:
1487 // TODO(arm64): This should use MOV (to general) when NEON is supported.
1488 __ Fmov(i.OutputRegister(), i.InputFloat64Register(0));
1489 __ Lsr(i.OutputRegister(), i.OutputRegister(), 32);
1490 break;
1491 case kArm64Float64InsertLowWord32: {
1492 // TODO(arm64): This should use MOV (from general) when NEON is supported.
1493 UseScratchRegisterScope scope(masm());
1494 Register tmp = scope.AcquireX();
1495 __ Fmov(tmp, i.InputFloat64Register(0));
1496 __ Bfi(tmp, i.InputRegister(1), 0, 32);
1497 __ Fmov(i.OutputFloat64Register(), tmp);
1498 break;
1499 }
1500 case kArm64Float64InsertHighWord32: {
1501 // TODO(arm64): This should use MOV (from general) when NEON is supported.
1502 UseScratchRegisterScope scope(masm());
1503 Register tmp = scope.AcquireX();
1504 __ Fmov(tmp.W(), i.InputFloat32Register(0));
1505 __ Bfi(tmp, i.InputRegister(1), 32, 32);
1506 __ Fmov(i.OutputFloat64Register(), tmp);
1507 break;
1508 }
1509 case kArm64Float64MoveU64:
1510 __ Fmov(i.OutputFloat64Register(), i.InputRegister(0));
1511 break;
1512 case kArm64Float64SilenceNaN:
1513 __ CanonicalizeNaN(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
1514 break;
1515 case kArm64U64MoveFloat64:
1516 __ Fmov(i.OutputRegister(), i.InputDoubleRegister(0));
1517 break;
1518 case kArm64Ldrb:
1519 __ Ldrb(i.OutputRegister(), i.MemoryOperand());
1520 break;
1521 case kArm64Ldrsb:
1522 __ Ldrsb(i.OutputRegister(), i.MemoryOperand());
1523 break;
1524 case kArm64Strb:
1525 __ Strb(i.InputOrZeroRegister64(0), i.MemoryOperand(1));
1526 break;
1527 case kArm64Ldrh:
1528 __ Ldrh(i.OutputRegister(), i.MemoryOperand());
1529 break;
1530 case kArm64Ldrsh:
1531 __ Ldrsh(i.OutputRegister(), i.MemoryOperand());
1532 break;
1533 case kArm64Strh:
1534 __ Strh(i.InputOrZeroRegister64(0), i.MemoryOperand(1));
1535 break;
1536 case kArm64Ldrsw:
1537 __ Ldrsw(i.OutputRegister(), i.MemoryOperand());
1538 break;
1539 case kArm64LdrW:
1540 __ Ldr(i.OutputRegister32(), i.MemoryOperand());
1541 break;
1542 case kArm64StrW:
1543 __ Str(i.InputOrZeroRegister32(0), i.MemoryOperand(1));
1544 break;
1545 case kArm64Ldr:
1546 __ Ldr(i.OutputRegister(), i.MemoryOperand());
1547 break;
1548 case kArm64Str:
1549 __ Str(i.InputOrZeroRegister64(0), i.MemoryOperand(1));
1550 break;
1551 case kArm64LdrS:
1552 __ Ldr(i.OutputDoubleRegister().S(), i.MemoryOperand());
1553 break;
1554 case kArm64StrS:
1555 __ Str(i.InputFloat32OrZeroRegister(0), i.MemoryOperand(1));
1556 break;
1557 case kArm64LdrD:
1558 __ Ldr(i.OutputDoubleRegister(), i.MemoryOperand());
1559 break;
1560 case kArm64StrD:
1561 __ Str(i.InputFloat64OrZeroRegister(0), i.MemoryOperand(1));
1562 break;
1563 case kCheckedLoadInt8:
1564 ASSEMBLE_CHECKED_LOAD_INTEGER(Ldrsb);
1565 break;
1566 case kCheckedLoadUint8:
1567 ASSEMBLE_CHECKED_LOAD_INTEGER(Ldrb);
1568 break;
1569 case kCheckedLoadInt16:
1570 ASSEMBLE_CHECKED_LOAD_INTEGER(Ldrsh);
1571 break;
1572 case kCheckedLoadUint16:
1573 ASSEMBLE_CHECKED_LOAD_INTEGER(Ldrh);
1574 break;
1575 case kCheckedLoadWord32:
1576 ASSEMBLE_CHECKED_LOAD_INTEGER(Ldr);
1577 break;
1578 case kCheckedLoadWord64:
1579 ASSEMBLE_CHECKED_LOAD_INTEGER_64(Ldr);
1580 break;
1581 case kCheckedLoadFloat32:
1582 ASSEMBLE_CHECKED_LOAD_FLOAT(32);
1583 break;
1584 case kCheckedLoadFloat64:
1585 ASSEMBLE_CHECKED_LOAD_FLOAT(64);
1586 break;
1587 case kCheckedStoreWord8:
1588 ASSEMBLE_CHECKED_STORE_INTEGER(Strb);
1589 break;
1590 case kCheckedStoreWord16:
1591 ASSEMBLE_CHECKED_STORE_INTEGER(Strh);
1592 break;
1593 case kCheckedStoreWord32:
1594 ASSEMBLE_CHECKED_STORE_INTEGER(Str);
1595 break;
1596 case kCheckedStoreWord64:
1597 ASSEMBLE_CHECKED_STORE_INTEGER_64(Str);
1598 break;
1599 case kCheckedStoreFloat32:
1600 ASSEMBLE_CHECKED_STORE_FLOAT(32);
1601 break;
1602 case kCheckedStoreFloat64:
1603 ASSEMBLE_CHECKED_STORE_FLOAT(64);
1604 break;
1605 case kAtomicLoadInt8:
1606 ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldrsb);
1607 break;
1608 case kAtomicLoadUint8:
1609 ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldrb);
1610 break;
1611 case kAtomicLoadInt16:
1612 ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldrsh);
1613 break;
1614 case kAtomicLoadUint16:
1615 ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldrh);
1616 break;
1617 case kAtomicLoadWord32:
1618 __ Ldr(i.OutputRegister32(),
1619 MemOperand(i.InputRegister(0), i.InputRegister(1)));
1620 __ Dmb(InnerShareable, BarrierAll);
1621 break;
1622 case kAtomicStoreWord8:
1623 ASSEMBLE_ATOMIC_STORE_INTEGER(Strb);
1624 break;
1625 case kAtomicStoreWord16:
1626 ASSEMBLE_ATOMIC_STORE_INTEGER(Strh);
1627 break;
1628 case kAtomicStoreWord32:
1629 __ Dmb(InnerShareable, BarrierAll);
1630 __ Str(i.InputRegister32(2),
1631 MemOperand(i.InputRegister(0), i.InputRegister(1)));
1632 __ Dmb(InnerShareable, BarrierAll);
1633 break;
1634 }
1635 return kSuccess;
1636 } // NOLINT(readability/fn_size)
1637
1638
1639 // Assemble branches after this instruction.
AssembleArchBranch(Instruction * instr,BranchInfo * branch)1640 void CodeGenerator::AssembleArchBranch(Instruction* instr, BranchInfo* branch) {
1641 Arm64OperandConverter i(this, instr);
1642 Label* tlabel = branch->true_label;
1643 Label* flabel = branch->false_label;
1644 FlagsCondition condition = branch->condition;
1645 ArchOpcode opcode = instr->arch_opcode();
1646
1647 if (opcode == kArm64CompareAndBranch32) {
1648 switch (condition) {
1649 case kEqual:
1650 __ Cbz(i.InputRegister32(0), tlabel);
1651 break;
1652 case kNotEqual:
1653 __ Cbnz(i.InputRegister32(0), tlabel);
1654 break;
1655 default:
1656 UNREACHABLE();
1657 }
1658 } else if (opcode == kArm64CompareAndBranch) {
1659 switch (condition) {
1660 case kEqual:
1661 __ Cbz(i.InputRegister64(0), tlabel);
1662 break;
1663 case kNotEqual:
1664 __ Cbnz(i.InputRegister64(0), tlabel);
1665 break;
1666 default:
1667 UNREACHABLE();
1668 }
1669 } else if (opcode == kArm64TestAndBranch32) {
1670 switch (condition) {
1671 case kEqual:
1672 __ Tbz(i.InputRegister32(0), i.InputInt5(1), tlabel);
1673 break;
1674 case kNotEqual:
1675 __ Tbnz(i.InputRegister32(0), i.InputInt5(1), tlabel);
1676 break;
1677 default:
1678 UNREACHABLE();
1679 }
1680 } else if (opcode == kArm64TestAndBranch) {
1681 switch (condition) {
1682 case kEqual:
1683 __ Tbz(i.InputRegister64(0), i.InputInt6(1), tlabel);
1684 break;
1685 case kNotEqual:
1686 __ Tbnz(i.InputRegister64(0), i.InputInt6(1), tlabel);
1687 break;
1688 default:
1689 UNREACHABLE();
1690 }
1691 } else {
1692 Condition cc = FlagsConditionToCondition(condition);
1693 __ B(cc, tlabel);
1694 }
1695 if (!branch->fallthru) __ B(flabel); // no fallthru to flabel.
1696 }
1697
1698
AssembleArchJump(RpoNumber target)1699 void CodeGenerator::AssembleArchJump(RpoNumber target) {
1700 if (!IsNextInAssemblyOrder(target)) __ B(GetLabel(target));
1701 }
1702
AssembleArchTrap(Instruction * instr,FlagsCondition condition)1703 void CodeGenerator::AssembleArchTrap(Instruction* instr,
1704 FlagsCondition condition) {
1705 class OutOfLineTrap final : public OutOfLineCode {
1706 public:
1707 OutOfLineTrap(CodeGenerator* gen, bool frame_elided, Instruction* instr)
1708 : OutOfLineCode(gen),
1709 frame_elided_(frame_elided),
1710 instr_(instr),
1711 gen_(gen) {}
1712 void Generate() final {
1713 Arm64OperandConverter i(gen_, instr_);
1714 Builtins::Name trap_id =
1715 static_cast<Builtins::Name>(i.InputInt32(instr_->InputCount() - 1));
1716 bool old_has_frame = __ has_frame();
1717 if (frame_elided_) {
1718 __ set_has_frame(true);
1719 __ EnterFrame(StackFrame::WASM_COMPILED);
1720 }
1721 GenerateCallToTrap(trap_id);
1722 if (frame_elided_) {
1723 __ set_has_frame(old_has_frame);
1724 }
1725 }
1726
1727 private:
1728 void GenerateCallToTrap(Builtins::Name trap_id) {
1729 if (trap_id == Builtins::builtin_count) {
1730 // We cannot test calls to the runtime in cctest/test-run-wasm.
1731 // Therefore we emit a call to C here instead of a call to the runtime.
1732 __ CallCFunction(
1733 ExternalReference::wasm_call_trap_callback_for_testing(isolate()),
1734 0);
1735 __ LeaveFrame(StackFrame::WASM_COMPILED);
1736 __ Ret();
1737 } else {
1738 DCHECK(csp.Is(__ StackPointer()));
1739 // Initialize the jssp because it is required for the runtime call.
1740 __ Mov(jssp, csp);
1741 gen_->AssembleSourcePosition(instr_);
1742 __ Call(handle(isolate()->builtins()->builtin(trap_id), isolate()),
1743 RelocInfo::CODE_TARGET);
1744 ReferenceMap* reference_map =
1745 new (gen_->zone()) ReferenceMap(gen_->zone());
1746 gen_->RecordSafepoint(reference_map, Safepoint::kSimple, 0,
1747 Safepoint::kNoLazyDeopt);
1748 if (FLAG_debug_code) {
1749 // The trap code should never return.
1750 __ Brk(0);
1751 }
1752 }
1753 }
1754 bool frame_elided_;
1755 Instruction* instr_;
1756 CodeGenerator* gen_;
1757 };
1758 bool frame_elided = !frame_access_state()->has_frame();
1759 auto ool = new (zone()) OutOfLineTrap(this, frame_elided, instr);
1760 Label* tlabel = ool->entry();
1761 Condition cc = FlagsConditionToCondition(condition);
1762 __ B(cc, tlabel);
1763 }
1764
1765 // Assemble boolean materializations after this instruction.
AssembleArchBoolean(Instruction * instr,FlagsCondition condition)1766 void CodeGenerator::AssembleArchBoolean(Instruction* instr,
1767 FlagsCondition condition) {
1768 Arm64OperandConverter i(this, instr);
1769
1770 // Materialize a full 64-bit 1 or 0 value. The result register is always the
1771 // last output of the instruction.
1772 DCHECK_NE(0u, instr->OutputCount());
1773 Register reg = i.OutputRegister(instr->OutputCount() - 1);
1774 Condition cc = FlagsConditionToCondition(condition);
1775 __ Cset(reg, cc);
1776 }
1777
1778
AssembleArchLookupSwitch(Instruction * instr)1779 void CodeGenerator::AssembleArchLookupSwitch(Instruction* instr) {
1780 Arm64OperandConverter i(this, instr);
1781 Register input = i.InputRegister32(0);
1782 for (size_t index = 2; index < instr->InputCount(); index += 2) {
1783 __ Cmp(input, i.InputInt32(index + 0));
1784 __ B(eq, GetLabel(i.InputRpo(index + 1)));
1785 }
1786 AssembleArchJump(i.InputRpo(1));
1787 }
1788
1789
AssembleArchTableSwitch(Instruction * instr)1790 void CodeGenerator::AssembleArchTableSwitch(Instruction* instr) {
1791 Arm64OperandConverter i(this, instr);
1792 UseScratchRegisterScope scope(masm());
1793 Register input = i.InputRegister32(0);
1794 Register temp = scope.AcquireX();
1795 size_t const case_count = instr->InputCount() - 2;
1796 Label table;
1797 __ Cmp(input, case_count);
1798 __ B(hs, GetLabel(i.InputRpo(1)));
1799 __ Adr(temp, &table);
1800 __ Add(temp, temp, Operand(input, UXTW, 2));
1801 __ Br(temp);
1802 __ StartBlockPools();
1803 __ Bind(&table);
1804 for (size_t index = 0; index < case_count; ++index) {
1805 __ B(GetLabel(i.InputRpo(index + 2)));
1806 }
1807 __ EndBlockPools();
1808 }
1809
AssembleDeoptimizerCall(int deoptimization_id,SourcePosition pos)1810 CodeGenerator::CodeGenResult CodeGenerator::AssembleDeoptimizerCall(
1811 int deoptimization_id, SourcePosition pos) {
1812 DeoptimizeKind deoptimization_kind = GetDeoptimizationKind(deoptimization_id);
1813 DeoptimizeReason deoptimization_reason =
1814 GetDeoptimizationReason(deoptimization_id);
1815 Deoptimizer::BailoutType bailout_type =
1816 deoptimization_kind == DeoptimizeKind::kSoft ? Deoptimizer::SOFT
1817 : Deoptimizer::EAGER;
1818 Address deopt_entry = Deoptimizer::GetDeoptimizationEntry(
1819 isolate(), deoptimization_id, bailout_type);
1820 if (deopt_entry == nullptr) return kTooManyDeoptimizationBailouts;
1821 __ RecordDeoptReason(deoptimization_reason, pos, deoptimization_id);
1822 __ Call(deopt_entry, RelocInfo::RUNTIME_ENTRY);
1823 return kSuccess;
1824 }
1825
FinishFrame(Frame * frame)1826 void CodeGenerator::FinishFrame(Frame* frame) {
1827 frame->AlignFrame(16);
1828 CallDescriptor* descriptor = linkage()->GetIncomingDescriptor();
1829
1830 if (descriptor->UseNativeStack() || descriptor->IsCFunctionCall()) {
1831 __ SetStackPointer(csp);
1832 } else {
1833 __ SetStackPointer(jssp);
1834 }
1835
1836 // Save FP registers.
1837 CPURegList saves_fp = CPURegList(CPURegister::kFPRegister, kDRegSizeInBits,
1838 descriptor->CalleeSavedFPRegisters());
1839 int saved_count = saves_fp.Count();
1840 if (saved_count != 0) {
1841 DCHECK(saves_fp.list() == CPURegList::GetCalleeSavedFP().list());
1842 frame->AllocateSavedCalleeRegisterSlots(saved_count *
1843 (kDoubleSize / kPointerSize));
1844 }
1845
1846 CPURegList saves = CPURegList(CPURegister::kRegister, kXRegSizeInBits,
1847 descriptor->CalleeSavedRegisters());
1848 saved_count = saves.Count();
1849 if (saved_count != 0) {
1850 frame->AllocateSavedCalleeRegisterSlots(saved_count);
1851 }
1852 }
1853
AssembleConstructFrame()1854 void CodeGenerator::AssembleConstructFrame() {
1855 CallDescriptor* descriptor = linkage()->GetIncomingDescriptor();
1856 if (descriptor->UseNativeStack()) {
1857 __ AssertCspAligned();
1858 }
1859
1860 int fixed_frame_size = descriptor->CalculateFixedFrameSize();
1861 int shrink_slots =
1862 frame()->GetTotalFrameSlotCount() - descriptor->CalculateFixedFrameSize();
1863
1864 if (frame_access_state()->has_frame()) {
1865 // Link the frame
1866 if (descriptor->IsJSFunctionCall()) {
1867 DCHECK(!descriptor->UseNativeStack());
1868 __ Prologue(this->info()->GeneratePreagedPrologue());
1869 } else {
1870 __ Push(lr, fp);
1871 __ Mov(fp, masm_.StackPointer());
1872 }
1873 if (!info()->GeneratePreagedPrologue()) {
1874 unwinding_info_writer_.MarkFrameConstructed(__ pc_offset());
1875 }
1876
1877 // Create OSR entry if applicable
1878 if (info()->is_osr()) {
1879 // TurboFan OSR-compiled functions cannot be entered directly.
1880 __ Abort(kShouldNotDirectlyEnterOsrFunction);
1881
1882 // Unoptimized code jumps directly to this entrypoint while the
1883 // unoptimized
1884 // frame is still on the stack. Optimized code uses OSR values directly
1885 // from
1886 // the unoptimized frame. Thus, all that needs to be done is to allocate
1887 // the
1888 // remaining stack slots.
1889 if (FLAG_code_comments) __ RecordComment("-- OSR entrypoint --");
1890 osr_pc_offset_ = __ pc_offset();
1891 shrink_slots -= OsrHelper(info()).UnoptimizedFrameSlots();
1892 }
1893 // Build remainder of frame, including accounting for and filling-in
1894 // frame-specific header information, e.g. claiming the extra slot that
1895 // other platforms explicitly push for STUB frames and frames recording
1896 // their argument count.
1897 __ Claim(shrink_slots + (fixed_frame_size & 1));
1898 if (descriptor->PushArgumentCount()) {
1899 __ Str(kJavaScriptCallArgCountRegister,
1900 MemOperand(fp, OptimizedBuiltinFrameConstants::kArgCOffset));
1901 }
1902 bool is_stub_frame =
1903 !descriptor->IsJSFunctionCall() && !descriptor->IsCFunctionCall();
1904 if (is_stub_frame) {
1905 UseScratchRegisterScope temps(masm());
1906 Register temp = temps.AcquireX();
1907 __ Mov(temp, StackFrame::TypeToMarker(info()->GetOutputStackFrameType()));
1908 __ Str(temp, MemOperand(fp, TypedFrameConstants::kFrameTypeOffset));
1909 }
1910 }
1911
1912 // Save FP registers.
1913 CPURegList saves_fp = CPURegList(CPURegister::kFPRegister, kDRegSizeInBits,
1914 descriptor->CalleeSavedFPRegisters());
1915 int saved_count = saves_fp.Count();
1916 if (saved_count != 0) {
1917 DCHECK(saves_fp.list() == CPURegList::GetCalleeSavedFP().list());
1918 __ PushCPURegList(saves_fp);
1919 }
1920 // Save registers.
1921 // TODO(palfia): TF save list is not in sync with
1922 // CPURegList::GetCalleeSaved(): x30 is missing.
1923 // DCHECK(saves.list() == CPURegList::GetCalleeSaved().list());
1924 CPURegList saves = CPURegList(CPURegister::kRegister, kXRegSizeInBits,
1925 descriptor->CalleeSavedRegisters());
1926 saved_count = saves.Count();
1927 if (saved_count != 0) {
1928 __ PushCPURegList(saves);
1929 }
1930 }
1931
AssembleReturn(InstructionOperand * pop)1932 void CodeGenerator::AssembleReturn(InstructionOperand* pop) {
1933 CallDescriptor* descriptor = linkage()->GetIncomingDescriptor();
1934
1935 // Restore registers.
1936 CPURegList saves = CPURegList(CPURegister::kRegister, kXRegSizeInBits,
1937 descriptor->CalleeSavedRegisters());
1938 if (saves.Count() != 0) {
1939 __ PopCPURegList(saves);
1940 }
1941
1942 // Restore fp registers.
1943 CPURegList saves_fp = CPURegList(CPURegister::kFPRegister, kDRegSizeInBits,
1944 descriptor->CalleeSavedFPRegisters());
1945 if (saves_fp.Count() != 0) {
1946 __ PopCPURegList(saves_fp);
1947 }
1948
1949 unwinding_info_writer_.MarkBlockWillExit();
1950
1951 Arm64OperandConverter g(this, nullptr);
1952 int pop_count = static_cast<int>(descriptor->StackParameterCount());
1953 if (descriptor->IsCFunctionCall()) {
1954 AssembleDeconstructFrame();
1955 } else if (frame_access_state()->has_frame()) {
1956 // Canonicalize JSFunction return sites for now unless they have an variable
1957 // number of stack slot pops.
1958 if (pop->IsImmediate() && g.ToConstant(pop).ToInt32() == 0) {
1959 if (return_label_.is_bound()) {
1960 __ B(&return_label_);
1961 return;
1962 } else {
1963 __ Bind(&return_label_);
1964 AssembleDeconstructFrame();
1965 if (descriptor->UseNativeStack()) {
1966 pop_count += (pop_count & 1); // align
1967 }
1968 }
1969 } else {
1970 AssembleDeconstructFrame();
1971 if (descriptor->UseNativeStack()) {
1972 pop_count += (pop_count & 1); // align
1973 }
1974 }
1975 } else if (descriptor->UseNativeStack()) {
1976 pop_count += (pop_count & 1); // align
1977 }
1978
1979 if (pop->IsImmediate()) {
1980 DCHECK_EQ(Constant::kInt32, g.ToConstant(pop).type());
1981 pop_count += g.ToConstant(pop).ToInt32();
1982 __ Drop(pop_count);
1983 } else {
1984 Register pop_reg = g.ToRegister(pop);
1985 __ Add(pop_reg, pop_reg, pop_count);
1986 __ Drop(pop_reg);
1987 }
1988
1989 if (descriptor->UseNativeStack()) {
1990 __ AssertCspAligned();
1991 }
1992 __ Ret();
1993 }
1994
1995
AssembleMove(InstructionOperand * source,InstructionOperand * destination)1996 void CodeGenerator::AssembleMove(InstructionOperand* source,
1997 InstructionOperand* destination) {
1998 Arm64OperandConverter g(this, nullptr);
1999 // Dispatch on the source and destination operand kinds. Not all
2000 // combinations are possible.
2001 if (source->IsRegister()) {
2002 DCHECK(destination->IsRegister() || destination->IsStackSlot());
2003 Register src = g.ToRegister(source);
2004 if (destination->IsRegister()) {
2005 __ Mov(g.ToRegister(destination), src);
2006 } else {
2007 __ Str(src, g.ToMemOperand(destination, masm()));
2008 }
2009 } else if (source->IsStackSlot()) {
2010 MemOperand src = g.ToMemOperand(source, masm());
2011 DCHECK(destination->IsRegister() || destination->IsStackSlot());
2012 if (destination->IsRegister()) {
2013 __ Ldr(g.ToRegister(destination), src);
2014 } else {
2015 UseScratchRegisterScope scope(masm());
2016 Register temp = scope.AcquireX();
2017 __ Ldr(temp, src);
2018 __ Str(temp, g.ToMemOperand(destination, masm()));
2019 }
2020 } else if (source->IsConstant()) {
2021 Constant src = g.ToConstant(ConstantOperand::cast(source));
2022 if (destination->IsRegister() || destination->IsStackSlot()) {
2023 UseScratchRegisterScope scope(masm());
2024 Register dst = destination->IsRegister() ? g.ToRegister(destination)
2025 : scope.AcquireX();
2026 if (src.type() == Constant::kHeapObject) {
2027 Handle<HeapObject> src_object = src.ToHeapObject();
2028 Heap::RootListIndex index;
2029 if (IsMaterializableFromRoot(src_object, &index)) {
2030 __ LoadRoot(dst, index);
2031 } else {
2032 __ LoadObject(dst, src_object);
2033 }
2034 } else {
2035 __ Mov(dst, g.ToImmediate(source));
2036 }
2037 if (destination->IsStackSlot()) {
2038 __ Str(dst, g.ToMemOperand(destination, masm()));
2039 }
2040 } else if (src.type() == Constant::kFloat32) {
2041 if (destination->IsFPRegister()) {
2042 FPRegister dst = g.ToDoubleRegister(destination).S();
2043 __ Fmov(dst, src.ToFloat32());
2044 } else {
2045 DCHECK(destination->IsFPStackSlot());
2046 if (bit_cast<int32_t>(src.ToFloat32()) == 0) {
2047 __ Str(wzr, g.ToMemOperand(destination, masm()));
2048 } else {
2049 UseScratchRegisterScope scope(masm());
2050 FPRegister temp = scope.AcquireS();
2051 __ Fmov(temp, src.ToFloat32());
2052 __ Str(temp, g.ToMemOperand(destination, masm()));
2053 }
2054 }
2055 } else {
2056 DCHECK_EQ(Constant::kFloat64, src.type());
2057 if (destination->IsFPRegister()) {
2058 FPRegister dst = g.ToDoubleRegister(destination);
2059 __ Fmov(dst, src.ToFloat64());
2060 } else {
2061 DCHECK(destination->IsFPStackSlot());
2062 if (bit_cast<int64_t>(src.ToFloat64()) == 0) {
2063 __ Str(xzr, g.ToMemOperand(destination, masm()));
2064 } else {
2065 UseScratchRegisterScope scope(masm());
2066 FPRegister temp = scope.AcquireD();
2067 __ Fmov(temp, src.ToFloat64());
2068 __ Str(temp, g.ToMemOperand(destination, masm()));
2069 }
2070 }
2071 }
2072 } else if (source->IsFPRegister()) {
2073 FPRegister src = g.ToDoubleRegister(source);
2074 if (destination->IsFPRegister()) {
2075 FPRegister dst = g.ToDoubleRegister(destination);
2076 __ Fmov(dst, src);
2077 } else {
2078 DCHECK(destination->IsFPStackSlot());
2079 __ Str(src, g.ToMemOperand(destination, masm()));
2080 }
2081 } else if (source->IsFPStackSlot()) {
2082 DCHECK(destination->IsFPRegister() || destination->IsFPStackSlot());
2083 MemOperand src = g.ToMemOperand(source, masm());
2084 if (destination->IsFPRegister()) {
2085 __ Ldr(g.ToDoubleRegister(destination), src);
2086 } else {
2087 UseScratchRegisterScope scope(masm());
2088 FPRegister temp = scope.AcquireD();
2089 __ Ldr(temp, src);
2090 __ Str(temp, g.ToMemOperand(destination, masm()));
2091 }
2092 } else {
2093 UNREACHABLE();
2094 }
2095 }
2096
2097
AssembleSwap(InstructionOperand * source,InstructionOperand * destination)2098 void CodeGenerator::AssembleSwap(InstructionOperand* source,
2099 InstructionOperand* destination) {
2100 Arm64OperandConverter g(this, nullptr);
2101 // Dispatch on the source and destination operand kinds. Not all
2102 // combinations are possible.
2103 if (source->IsRegister()) {
2104 // Register-register.
2105 UseScratchRegisterScope scope(masm());
2106 Register temp = scope.AcquireX();
2107 Register src = g.ToRegister(source);
2108 if (destination->IsRegister()) {
2109 Register dst = g.ToRegister(destination);
2110 __ Mov(temp, src);
2111 __ Mov(src, dst);
2112 __ Mov(dst, temp);
2113 } else {
2114 DCHECK(destination->IsStackSlot());
2115 MemOperand dst = g.ToMemOperand(destination, masm());
2116 __ Mov(temp, src);
2117 __ Ldr(src, dst);
2118 __ Str(temp, dst);
2119 }
2120 } else if (source->IsStackSlot() || source->IsFPStackSlot()) {
2121 UseScratchRegisterScope scope(masm());
2122 DoubleRegister temp_0 = scope.AcquireD();
2123 DoubleRegister temp_1 = scope.AcquireD();
2124 MemOperand src = g.ToMemOperand(source, masm());
2125 MemOperand dst = g.ToMemOperand(destination, masm());
2126 __ Ldr(temp_0, src);
2127 __ Ldr(temp_1, dst);
2128 __ Str(temp_0, dst);
2129 __ Str(temp_1, src);
2130 } else if (source->IsFPRegister()) {
2131 UseScratchRegisterScope scope(masm());
2132 FPRegister temp = scope.AcquireD();
2133 FPRegister src = g.ToDoubleRegister(source);
2134 if (destination->IsFPRegister()) {
2135 FPRegister dst = g.ToDoubleRegister(destination);
2136 __ Fmov(temp, src);
2137 __ Fmov(src, dst);
2138 __ Fmov(dst, temp);
2139 } else {
2140 DCHECK(destination->IsFPStackSlot());
2141 MemOperand dst = g.ToMemOperand(destination, masm());
2142 __ Fmov(temp, src);
2143 __ Ldr(src, dst);
2144 __ Str(temp, dst);
2145 }
2146 } else {
2147 // No other combinations are possible.
2148 UNREACHABLE();
2149 }
2150 }
2151
2152
AssembleJumpTable(Label ** targets,size_t target_count)2153 void CodeGenerator::AssembleJumpTable(Label** targets, size_t target_count) {
2154 // On 64-bit ARM we emit the jump tables inline.
2155 UNREACHABLE();
2156 }
2157
2158
EnsureSpaceForLazyDeopt()2159 void CodeGenerator::EnsureSpaceForLazyDeopt() {
2160 if (!info()->ShouldEnsureSpaceForLazyDeopt()) {
2161 return;
2162 }
2163
2164 int space_needed = Deoptimizer::patch_size();
2165 // Ensure that we have enough space after the previous lazy-bailout
2166 // instruction for patching the code here.
2167 intptr_t current_pc = masm()->pc_offset();
2168
2169 if (current_pc < (last_lazy_deopt_pc_ + space_needed)) {
2170 intptr_t padding_size = last_lazy_deopt_pc_ + space_needed - current_pc;
2171 DCHECK((padding_size % kInstructionSize) == 0);
2172 InstructionAccurateScope instruction_accurate(
2173 masm(), padding_size / kInstructionSize);
2174
2175 while (padding_size > 0) {
2176 __ nop();
2177 padding_size -= kInstructionSize;
2178 }
2179 }
2180 }
2181
2182 #undef __
2183
2184 } // namespace compiler
2185 } // namespace internal
2186 } // namespace v8
2187