1 // Copyright 2014 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "src/compiler/code-generator.h"
6
7 #include "src/arm64/assembler-arm64-inl.h"
8 #include "src/arm64/macro-assembler-arm64-inl.h"
9 #include "src/compiler/code-generator-impl.h"
10 #include "src/compiler/gap-resolver.h"
11 #include "src/compiler/node-matchers.h"
12 #include "src/compiler/osr.h"
13 #include "src/frame-constants.h"
14 #include "src/heap/heap-inl.h"
15 #include "src/optimized-compilation-info.h"
16 #include "src/wasm/wasm-objects.h"
17
18 namespace v8 {
19 namespace internal {
20 namespace compiler {
21
22 #define __ tasm()->
23
24 // Adds Arm64-specific methods to convert InstructionOperands.
25 class Arm64OperandConverter final : public InstructionOperandConverter {
26 public:
Arm64OperandConverter(CodeGenerator * gen,Instruction * instr)27 Arm64OperandConverter(CodeGenerator* gen, Instruction* instr)
28 : InstructionOperandConverter(gen, instr) {}
29
InputFloat32Register(size_t index)30 DoubleRegister InputFloat32Register(size_t index) {
31 return InputDoubleRegister(index).S();
32 }
33
InputFloat64Register(size_t index)34 DoubleRegister InputFloat64Register(size_t index) {
35 return InputDoubleRegister(index);
36 }
37
InputSimd128Register(size_t index)38 DoubleRegister InputSimd128Register(size_t index) {
39 return InputDoubleRegister(index).Q();
40 }
41
InputFloat32OrZeroRegister(size_t index)42 CPURegister InputFloat32OrZeroRegister(size_t index) {
43 if (instr_->InputAt(index)->IsImmediate()) {
44 DCHECK_EQ(0, bit_cast<int32_t>(InputFloat32(index)));
45 return wzr;
46 }
47 DCHECK(instr_->InputAt(index)->IsFPRegister());
48 return InputDoubleRegister(index).S();
49 }
50
InputFloat64OrZeroRegister(size_t index)51 CPURegister InputFloat64OrZeroRegister(size_t index) {
52 if (instr_->InputAt(index)->IsImmediate()) {
53 DCHECK_EQ(0, bit_cast<int64_t>(InputDouble(index)));
54 return xzr;
55 }
56 DCHECK(instr_->InputAt(index)->IsDoubleRegister());
57 return InputDoubleRegister(index);
58 }
59
OutputCount()60 size_t OutputCount() { return instr_->OutputCount(); }
61
OutputFloat32Register()62 DoubleRegister OutputFloat32Register() { return OutputDoubleRegister().S(); }
63
OutputFloat64Register()64 DoubleRegister OutputFloat64Register() { return OutputDoubleRegister(); }
65
OutputSimd128Register()66 DoubleRegister OutputSimd128Register() { return OutputDoubleRegister().Q(); }
67
InputRegister32(size_t index)68 Register InputRegister32(size_t index) {
69 return ToRegister(instr_->InputAt(index)).W();
70 }
71
InputOrZeroRegister32(size_t index)72 Register InputOrZeroRegister32(size_t index) {
73 DCHECK(instr_->InputAt(index)->IsRegister() ||
74 (instr_->InputAt(index)->IsImmediate() && (InputInt32(index) == 0)));
75 if (instr_->InputAt(index)->IsImmediate()) {
76 return wzr;
77 }
78 return InputRegister32(index);
79 }
80
InputRegister64(size_t index)81 Register InputRegister64(size_t index) { return InputRegister(index); }
82
InputOrZeroRegister64(size_t index)83 Register InputOrZeroRegister64(size_t index) {
84 DCHECK(instr_->InputAt(index)->IsRegister() ||
85 (instr_->InputAt(index)->IsImmediate() && (InputInt64(index) == 0)));
86 if (instr_->InputAt(index)->IsImmediate()) {
87 return xzr;
88 }
89 return InputRegister64(index);
90 }
91
InputOperand(size_t index)92 Operand InputOperand(size_t index) {
93 return ToOperand(instr_->InputAt(index));
94 }
95
InputOperand64(size_t index)96 Operand InputOperand64(size_t index) { return InputOperand(index); }
97
InputOperand32(size_t index)98 Operand InputOperand32(size_t index) {
99 return ToOperand32(instr_->InputAt(index));
100 }
101
OutputRegister64()102 Register OutputRegister64() { return OutputRegister(); }
103
OutputRegister32()104 Register OutputRegister32() { return ToRegister(instr_->Output()).W(); }
105
TempRegister32(size_t index)106 Register TempRegister32(size_t index) {
107 return ToRegister(instr_->TempAt(index)).W();
108 }
109
InputOperand2_32(size_t index)110 Operand InputOperand2_32(size_t index) {
111 switch (AddressingModeField::decode(instr_->opcode())) {
112 case kMode_None:
113 return InputOperand32(index);
114 case kMode_Operand2_R_LSL_I:
115 return Operand(InputRegister32(index), LSL, InputInt5(index + 1));
116 case kMode_Operand2_R_LSR_I:
117 return Operand(InputRegister32(index), LSR, InputInt5(index + 1));
118 case kMode_Operand2_R_ASR_I:
119 return Operand(InputRegister32(index), ASR, InputInt5(index + 1));
120 case kMode_Operand2_R_ROR_I:
121 return Operand(InputRegister32(index), ROR, InputInt5(index + 1));
122 case kMode_Operand2_R_UXTB:
123 return Operand(InputRegister32(index), UXTB);
124 case kMode_Operand2_R_UXTH:
125 return Operand(InputRegister32(index), UXTH);
126 case kMode_Operand2_R_SXTB:
127 return Operand(InputRegister32(index), SXTB);
128 case kMode_Operand2_R_SXTH:
129 return Operand(InputRegister32(index), SXTH);
130 case kMode_Operand2_R_SXTW:
131 return Operand(InputRegister32(index), SXTW);
132 case kMode_MRI:
133 case kMode_MRR:
134 case kMode_Root:
135 break;
136 }
137 UNREACHABLE();
138 }
139
InputOperand2_64(size_t index)140 Operand InputOperand2_64(size_t index) {
141 switch (AddressingModeField::decode(instr_->opcode())) {
142 case kMode_None:
143 return InputOperand64(index);
144 case kMode_Operand2_R_LSL_I:
145 return Operand(InputRegister64(index), LSL, InputInt6(index + 1));
146 case kMode_Operand2_R_LSR_I:
147 return Operand(InputRegister64(index), LSR, InputInt6(index + 1));
148 case kMode_Operand2_R_ASR_I:
149 return Operand(InputRegister64(index), ASR, InputInt6(index + 1));
150 case kMode_Operand2_R_ROR_I:
151 return Operand(InputRegister64(index), ROR, InputInt6(index + 1));
152 case kMode_Operand2_R_UXTB:
153 return Operand(InputRegister64(index), UXTB);
154 case kMode_Operand2_R_UXTH:
155 return Operand(InputRegister64(index), UXTH);
156 case kMode_Operand2_R_SXTB:
157 return Operand(InputRegister64(index), SXTB);
158 case kMode_Operand2_R_SXTH:
159 return Operand(InputRegister64(index), SXTH);
160 case kMode_Operand2_R_SXTW:
161 return Operand(InputRegister64(index), SXTW);
162 case kMode_MRI:
163 case kMode_MRR:
164 case kMode_Root:
165 break;
166 }
167 UNREACHABLE();
168 }
169
MemoryOperand(size_t index=0)170 MemOperand MemoryOperand(size_t index = 0) {
171 switch (AddressingModeField::decode(instr_->opcode())) {
172 case kMode_None:
173 case kMode_Operand2_R_LSR_I:
174 case kMode_Operand2_R_ASR_I:
175 case kMode_Operand2_R_ROR_I:
176 case kMode_Operand2_R_UXTB:
177 case kMode_Operand2_R_UXTH:
178 case kMode_Operand2_R_SXTB:
179 case kMode_Operand2_R_SXTH:
180 case kMode_Operand2_R_SXTW:
181 break;
182 case kMode_Root:
183 return MemOperand(kRootRegister, InputInt64(index));
184 case kMode_Operand2_R_LSL_I:
185 return MemOperand(InputRegister(index + 0), InputRegister(index + 1),
186 LSL, InputInt32(index + 2));
187 case kMode_MRI:
188 return MemOperand(InputRegister(index + 0), InputInt32(index + 1));
189 case kMode_MRR:
190 return MemOperand(InputRegister(index + 0), InputRegister(index + 1));
191 }
192 UNREACHABLE();
193 }
194
ToOperand(InstructionOperand * op)195 Operand ToOperand(InstructionOperand* op) {
196 if (op->IsRegister()) {
197 return Operand(ToRegister(op));
198 }
199 return ToImmediate(op);
200 }
201
ToOperand32(InstructionOperand * op)202 Operand ToOperand32(InstructionOperand* op) {
203 if (op->IsRegister()) {
204 return Operand(ToRegister(op).W());
205 }
206 return ToImmediate(op);
207 }
208
ToImmediate(InstructionOperand * operand)209 Operand ToImmediate(InstructionOperand* operand) {
210 Constant constant = ToConstant(operand);
211 switch (constant.type()) {
212 case Constant::kInt32:
213 return Operand(constant.ToInt32());
214 case Constant::kInt64:
215 if (RelocInfo::IsWasmPtrReference(constant.rmode())) {
216 return Operand(constant.ToInt64(), constant.rmode());
217 } else {
218 return Operand(constant.ToInt64());
219 }
220 case Constant::kFloat32:
221 return Operand(Operand::EmbeddedNumber(constant.ToFloat32()));
222 case Constant::kFloat64:
223 return Operand(Operand::EmbeddedNumber(constant.ToFloat64().value()));
224 case Constant::kExternalReference:
225 return Operand(constant.ToExternalReference());
226 case Constant::kHeapObject:
227 return Operand(constant.ToHeapObject());
228 case Constant::kRpoNumber:
229 UNREACHABLE(); // TODO(dcarney): RPO immediates on arm64.
230 break;
231 }
232 UNREACHABLE();
233 }
234
ToMemOperand(InstructionOperand * op,TurboAssembler * tasm) const235 MemOperand ToMemOperand(InstructionOperand* op, TurboAssembler* tasm) const {
236 DCHECK_NOT_NULL(op);
237 DCHECK(op->IsStackSlot() || op->IsFPStackSlot());
238 return SlotToMemOperand(AllocatedOperand::cast(op)->index(), tasm);
239 }
240
SlotToMemOperand(int slot,TurboAssembler * tasm) const241 MemOperand SlotToMemOperand(int slot, TurboAssembler* tasm) const {
242 FrameOffset offset = frame_access_state()->GetFrameOffset(slot);
243 if (offset.from_frame_pointer()) {
244 int from_sp = offset.offset() + frame_access_state()->GetSPToFPOffset();
245 // Convert FP-offsets to SP-offsets if it results in better code.
246 if (Assembler::IsImmLSUnscaled(from_sp) ||
247 Assembler::IsImmLSScaled(from_sp, 3)) {
248 offset = FrameOffset::FromStackPointer(from_sp);
249 }
250 }
251 return MemOperand(offset.from_stack_pointer() ? sp : fp, offset.offset());
252 }
253 };
254
255
256 namespace {
257
258 class OutOfLineRecordWrite final : public OutOfLineCode {
259 public:
OutOfLineRecordWrite(CodeGenerator * gen,Register object,Operand index,Register value,Register scratch0,Register scratch1,RecordWriteMode mode,UnwindingInfoWriter * unwinding_info_writer)260 OutOfLineRecordWrite(CodeGenerator* gen, Register object, Operand index,
261 Register value, Register scratch0, Register scratch1,
262 RecordWriteMode mode,
263 UnwindingInfoWriter* unwinding_info_writer)
264 : OutOfLineCode(gen),
265 object_(object),
266 index_(index),
267 value_(value),
268 scratch0_(scratch0),
269 scratch1_(scratch1),
270 mode_(mode),
271 must_save_lr_(!gen->frame_access_state()->has_frame()),
272 unwinding_info_writer_(unwinding_info_writer),
273 zone_(gen->zone()) {}
274
Generate()275 void Generate() final {
276 if (mode_ > RecordWriteMode::kValueIsPointer) {
277 __ JumpIfSmi(value_, exit());
278 }
279 __ CheckPageFlagClear(value_, scratch0_,
280 MemoryChunk::kPointersToHereAreInterestingMask,
281 exit());
282 __ Add(scratch1_, object_, index_);
283 RememberedSetAction const remembered_set_action =
284 mode_ > RecordWriteMode::kValueIsMap ? EMIT_REMEMBERED_SET
285 : OMIT_REMEMBERED_SET;
286 SaveFPRegsMode const save_fp_mode =
287 frame()->DidAllocateDoubleRegisters() ? kSaveFPRegs : kDontSaveFPRegs;
288 if (must_save_lr_) {
289 // We need to save and restore lr if the frame was elided.
290 __ Push(lr, padreg);
291 unwinding_info_writer_->MarkLinkRegisterOnTopOfStack(__ pc_offset(), sp);
292 }
293 __ CallRecordWriteStub(object_, scratch1_, remembered_set_action,
294 save_fp_mode);
295 if (must_save_lr_) {
296 __ Pop(padreg, lr);
297 unwinding_info_writer_->MarkPopLinkRegisterFromTopOfStack(__ pc_offset());
298 }
299 }
300
301 private:
302 Register const object_;
303 Operand const index_;
304 Register const value_;
305 Register const scratch0_;
306 Register const scratch1_;
307 RecordWriteMode const mode_;
308 bool must_save_lr_;
309 UnwindingInfoWriter* const unwinding_info_writer_;
310 Zone* zone_;
311 };
312
313
FlagsConditionToCondition(FlagsCondition condition)314 Condition FlagsConditionToCondition(FlagsCondition condition) {
315 switch (condition) {
316 case kEqual:
317 return eq;
318 case kNotEqual:
319 return ne;
320 case kSignedLessThan:
321 return lt;
322 case kSignedGreaterThanOrEqual:
323 return ge;
324 case kSignedLessThanOrEqual:
325 return le;
326 case kSignedGreaterThan:
327 return gt;
328 case kUnsignedLessThan:
329 return lo;
330 case kUnsignedGreaterThanOrEqual:
331 return hs;
332 case kUnsignedLessThanOrEqual:
333 return ls;
334 case kUnsignedGreaterThan:
335 return hi;
336 case kFloatLessThanOrUnordered:
337 return lt;
338 case kFloatGreaterThanOrEqual:
339 return ge;
340 case kFloatLessThanOrEqual:
341 return ls;
342 case kFloatGreaterThanOrUnordered:
343 return hi;
344 case kFloatLessThan:
345 return lo;
346 case kFloatGreaterThanOrEqualOrUnordered:
347 return hs;
348 case kFloatLessThanOrEqualOrUnordered:
349 return le;
350 case kFloatGreaterThan:
351 return gt;
352 case kOverflow:
353 return vs;
354 case kNotOverflow:
355 return vc;
356 case kUnorderedEqual:
357 case kUnorderedNotEqual:
358 break;
359 case kPositiveOrZero:
360 return pl;
361 case kNegative:
362 return mi;
363 }
364 UNREACHABLE();
365 }
366
EmitWordLoadPoisoningIfNeeded(CodeGenerator * codegen,InstructionCode opcode,Instruction * instr,Arm64OperandConverter & i)367 void EmitWordLoadPoisoningIfNeeded(CodeGenerator* codegen,
368 InstructionCode opcode, Instruction* instr,
369 Arm64OperandConverter& i) {
370 const MemoryAccessMode access_mode =
371 static_cast<MemoryAccessMode>(MiscField::decode(opcode));
372 if (access_mode == kMemoryAccessPoisoned) {
373 Register value = i.OutputRegister();
374 Register poison = value.Is64Bits() ? kSpeculationPoisonRegister
375 : kSpeculationPoisonRegister.W();
376 codegen->tasm()->And(value, value, Operand(poison));
377 }
378 }
379
380 } // namespace
381
382 #define ASSEMBLE_SHIFT(asm_instr, width) \
383 do { \
384 if (instr->InputAt(1)->IsRegister()) { \
385 __ asm_instr(i.OutputRegister##width(), i.InputRegister##width(0), \
386 i.InputRegister##width(1)); \
387 } else { \
388 uint32_t imm = \
389 static_cast<uint32_t>(i.InputOperand##width(1).ImmediateValue()); \
390 __ asm_instr(i.OutputRegister##width(), i.InputRegister##width(0), \
391 imm % (width)); \
392 } \
393 } while (0)
394
395 #define ASSEMBLE_ATOMIC_LOAD_INTEGER(asm_instr, reg) \
396 do { \
397 __ Add(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1)); \
398 __ asm_instr(i.Output##reg(), i.TempRegister(0)); \
399 } while (0)
400
401 #define ASSEMBLE_ATOMIC_STORE_INTEGER(asm_instr, reg) \
402 do { \
403 __ Add(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1)); \
404 __ asm_instr(i.Input##reg(2), i.TempRegister(0)); \
405 } while (0)
406
407 #define ASSEMBLE_ATOMIC_EXCHANGE_INTEGER(load_instr, store_instr, reg) \
408 do { \
409 Label exchange; \
410 __ Add(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1)); \
411 __ Bind(&exchange); \
412 __ load_instr(i.Output##reg(), i.TempRegister(0)); \
413 __ store_instr(i.TempRegister32(1), i.Input##reg(2), i.TempRegister(0)); \
414 __ Cbnz(i.TempRegister32(1), &exchange); \
415 } while (0)
416
417 #define ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER(load_instr, store_instr, ext, \
418 reg) \
419 do { \
420 Label compareExchange; \
421 Label exit; \
422 __ Add(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1)); \
423 __ Bind(&compareExchange); \
424 __ load_instr(i.Output##reg(), i.TempRegister(0)); \
425 __ Cmp(i.Output##reg(), Operand(i.Input##reg(2), ext)); \
426 __ B(ne, &exit); \
427 __ store_instr(i.TempRegister32(1), i.Input##reg(3), i.TempRegister(0)); \
428 __ Cbnz(i.TempRegister32(1), &compareExchange); \
429 __ Bind(&exit); \
430 } while (0)
431
432 #define ASSEMBLE_ATOMIC_BINOP(load_instr, store_instr, bin_instr, reg) \
433 do { \
434 Label binop; \
435 __ Add(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1)); \
436 __ Bind(&binop); \
437 __ load_instr(i.Output##reg(), i.TempRegister(0)); \
438 __ bin_instr(i.Temp##reg(1), i.Output##reg(), Operand(i.Input##reg(2))); \
439 __ store_instr(i.TempRegister32(2), i.Temp##reg(1), i.TempRegister(0)); \
440 __ Cbnz(i.TempRegister32(2), &binop); \
441 } while (0)
442
443 #define ASSEMBLE_IEEE754_BINOP(name) \
444 do { \
445 FrameScope scope(tasm(), StackFrame::MANUAL); \
446 __ CallCFunction(ExternalReference::ieee754_##name##_function(), 0, 2); \
447 } while (0)
448
449 #define ASSEMBLE_IEEE754_UNOP(name) \
450 do { \
451 FrameScope scope(tasm(), StackFrame::MANUAL); \
452 __ CallCFunction(ExternalReference::ieee754_##name##_function(), 0, 1); \
453 } while (0)
454
AssembleDeconstructFrame()455 void CodeGenerator::AssembleDeconstructFrame() {
456 __ Mov(sp, fp);
457 __ Pop(fp, lr);
458
459 unwinding_info_writer_.MarkFrameDeconstructed(__ pc_offset());
460 }
461
AssemblePrepareTailCall()462 void CodeGenerator::AssemblePrepareTailCall() {
463 if (frame_access_state()->has_frame()) {
464 __ Ldr(lr, MemOperand(fp, StandardFrameConstants::kCallerPCOffset));
465 __ Ldr(fp, MemOperand(fp, StandardFrameConstants::kCallerFPOffset));
466 }
467 frame_access_state()->SetFrameAccessToSP();
468 }
469
AssemblePopArgumentsAdaptorFrame(Register args_reg,Register scratch1,Register scratch2,Register scratch3)470 void CodeGenerator::AssemblePopArgumentsAdaptorFrame(Register args_reg,
471 Register scratch1,
472 Register scratch2,
473 Register scratch3) {
474 DCHECK(!AreAliased(args_reg, scratch1, scratch2, scratch3));
475 Label done;
476
477 // Check if current frame is an arguments adaptor frame.
478 __ Ldr(scratch1, MemOperand(fp, StandardFrameConstants::kContextOffset));
479 __ Cmp(scratch1,
480 Operand(StackFrame::TypeToMarker(StackFrame::ARGUMENTS_ADAPTOR)));
481 __ B(ne, &done);
482
483 // Load arguments count from current arguments adaptor frame (note, it
484 // does not include receiver).
485 Register caller_args_count_reg = scratch1;
486 __ Ldr(caller_args_count_reg,
487 MemOperand(fp, ArgumentsAdaptorFrameConstants::kLengthOffset));
488 __ SmiUntag(caller_args_count_reg);
489
490 ParameterCount callee_args_count(args_reg);
491 __ PrepareForTailCall(callee_args_count, caller_args_count_reg, scratch2,
492 scratch3);
493 __ bind(&done);
494 }
495
496 namespace {
497
AdjustStackPointerForTailCall(TurboAssembler * tasm,FrameAccessState * state,int new_slot_above_sp,bool allow_shrinkage=true)498 void AdjustStackPointerForTailCall(TurboAssembler* tasm,
499 FrameAccessState* state,
500 int new_slot_above_sp,
501 bool allow_shrinkage = true) {
502 int current_sp_offset = state->GetSPToFPSlotCount() +
503 StandardFrameConstants::kFixedSlotCountAboveFp;
504 int stack_slot_delta = new_slot_above_sp - current_sp_offset;
505 DCHECK_EQ(stack_slot_delta % 2, 0);
506 if (stack_slot_delta > 0) {
507 tasm->Claim(stack_slot_delta);
508 state->IncreaseSPDelta(stack_slot_delta);
509 } else if (allow_shrinkage && stack_slot_delta < 0) {
510 tasm->Drop(-stack_slot_delta);
511 state->IncreaseSPDelta(stack_slot_delta);
512 }
513 }
514
515 } // namespace
516
AssembleTailCallBeforeGap(Instruction * instr,int first_unused_stack_slot)517 void CodeGenerator::AssembleTailCallBeforeGap(Instruction* instr,
518 int first_unused_stack_slot) {
519 AdjustStackPointerForTailCall(tasm(), frame_access_state(),
520 first_unused_stack_slot, false);
521 }
522
AssembleTailCallAfterGap(Instruction * instr,int first_unused_stack_slot)523 void CodeGenerator::AssembleTailCallAfterGap(Instruction* instr,
524 int first_unused_stack_slot) {
525 DCHECK_EQ(first_unused_stack_slot % 2, 0);
526 AdjustStackPointerForTailCall(tasm(), frame_access_state(),
527 first_unused_stack_slot);
528 DCHECK(instr->IsTailCall());
529 InstructionOperandConverter g(this, instr);
530 int optional_padding_slot = g.InputInt32(instr->InputCount() - 2);
531 if (optional_padding_slot % 2) {
532 __ Poke(padreg, optional_padding_slot * kPointerSize);
533 }
534 }
535
536 // Check that {kJavaScriptCallCodeStartRegister} is correct.
AssembleCodeStartRegisterCheck()537 void CodeGenerator::AssembleCodeStartRegisterCheck() {
538 UseScratchRegisterScope temps(tasm());
539 Register scratch = temps.AcquireX();
540 __ ComputeCodeStartAddress(scratch);
541 __ cmp(scratch, kJavaScriptCallCodeStartRegister);
542 __ Assert(eq, AbortReason::kWrongFunctionCodeStart);
543 }
544
545 // Check if the code object is marked for deoptimization. If it is, then it
546 // jumps to the CompileLazyDeoptimizedCode builtin. In order to do this we need
547 // to:
548 // 1. read from memory the word that contains that bit, which can be found in
549 // the flags in the referenced {CodeDataContainer} object;
550 // 2. test kMarkedForDeoptimizationBit in those flags; and
551 // 3. if it is not zero then it jumps to the builtin.
BailoutIfDeoptimized()552 void CodeGenerator::BailoutIfDeoptimized() {
553 UseScratchRegisterScope temps(tasm());
554 Register scratch = temps.AcquireX();
555 int offset = Code::kCodeDataContainerOffset - Code::kHeaderSize;
556 __ Ldr(scratch, MemOperand(kJavaScriptCallCodeStartRegister, offset));
557 __ Ldr(scratch,
558 FieldMemOperand(scratch, CodeDataContainer::kKindSpecificFlagsOffset));
559 Label not_deoptimized;
560 __ Tbz(scratch, Code::kMarkedForDeoptimizationBit, ¬_deoptimized);
561 // Ensure we're not serializing (otherwise we'd need to use an indirection to
562 // access the builtin below).
563 DCHECK(!isolate()->ShouldLoadConstantsFromRootList());
564 Handle<Code> code = isolate()->builtins()->builtin_handle(
565 Builtins::kCompileLazyDeoptimizedCode);
566 __ Jump(code, RelocInfo::CODE_TARGET);
567 __ Bind(¬_deoptimized);
568 }
569
GenerateSpeculationPoisonFromCodeStartRegister()570 void CodeGenerator::GenerateSpeculationPoisonFromCodeStartRegister() {
571 UseScratchRegisterScope temps(tasm());
572 Register scratch = temps.AcquireX();
573
574 // Set a mask which has all bits set in the normal case, but has all
575 // bits cleared if we are speculatively executing the wrong PC.
576 __ ComputeCodeStartAddress(scratch);
577 __ Cmp(kJavaScriptCallCodeStartRegister, scratch);
578 __ Csetm(kSpeculationPoisonRegister, eq);
579 __ Csdb();
580 }
581
AssembleRegisterArgumentPoisoning()582 void CodeGenerator::AssembleRegisterArgumentPoisoning() {
583 UseScratchRegisterScope temps(tasm());
584 Register scratch = temps.AcquireX();
585
586 __ Mov(scratch, sp);
587 __ And(kJSFunctionRegister, kJSFunctionRegister, kSpeculationPoisonRegister);
588 __ And(kContextRegister, kContextRegister, kSpeculationPoisonRegister);
589 __ And(scratch, scratch, kSpeculationPoisonRegister);
590 __ Mov(sp, scratch);
591 }
592
593 // Assembles an instruction after register allocation, producing machine code.
AssembleArchInstruction(Instruction * instr)594 CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
595 Instruction* instr) {
596 Arm64OperandConverter i(this, instr);
597 InstructionCode opcode = instr->opcode();
598 ArchOpcode arch_opcode = ArchOpcodeField::decode(opcode);
599 switch (arch_opcode) {
600 case kArchCallCodeObject: {
601 if (instr->InputAt(0)->IsImmediate()) {
602 __ Call(i.InputCode(0), RelocInfo::CODE_TARGET);
603 } else {
604 Register reg = i.InputRegister(0);
605 DCHECK_IMPLIES(
606 HasCallDescriptorFlag(instr, CallDescriptor::kFixedTargetRegister),
607 reg == kJavaScriptCallCodeStartRegister);
608 __ Add(reg, reg, Code::kHeaderSize - kHeapObjectTag);
609 __ Call(reg);
610 }
611 RecordCallPosition(instr);
612 frame_access_state()->ClearSPDelta();
613 break;
614 }
615 case kArchCallWasmFunction: {
616 if (instr->InputAt(0)->IsImmediate()) {
617 Constant constant = i.ToConstant(instr->InputAt(0));
618 Address wasm_code = static_cast<Address>(constant.ToInt64());
619 __ Call(wasm_code, constant.rmode());
620 } else {
621 Register target = i.InputRegister(0);
622 __ Call(target);
623 }
624 RecordCallPosition(instr);
625 frame_access_state()->ClearSPDelta();
626 break;
627 }
628 case kArchTailCallCodeObjectFromJSFunction:
629 case kArchTailCallCodeObject: {
630 if (arch_opcode == kArchTailCallCodeObjectFromJSFunction) {
631 AssemblePopArgumentsAdaptorFrame(kJavaScriptCallArgCountRegister,
632 i.TempRegister(0), i.TempRegister(1),
633 i.TempRegister(2));
634 }
635 if (instr->InputAt(0)->IsImmediate()) {
636 __ Jump(i.InputCode(0), RelocInfo::CODE_TARGET);
637 } else {
638 Register reg = i.InputRegister(0);
639 DCHECK_IMPLIES(
640 HasCallDescriptorFlag(instr, CallDescriptor::kFixedTargetRegister),
641 reg == kJavaScriptCallCodeStartRegister);
642 __ Add(reg, reg, Code::kHeaderSize - kHeapObjectTag);
643 __ Jump(reg);
644 }
645 unwinding_info_writer_.MarkBlockWillExit();
646 frame_access_state()->ClearSPDelta();
647 frame_access_state()->SetFrameAccessToDefault();
648 break;
649 }
650 case kArchTailCallWasm: {
651 if (instr->InputAt(0)->IsImmediate()) {
652 Constant constant = i.ToConstant(instr->InputAt(0));
653 Address wasm_code = static_cast<Address>(constant.ToInt64());
654 __ Jump(wasm_code, constant.rmode());
655 } else {
656 Register target = i.InputRegister(0);
657 __ Jump(target);
658 }
659 unwinding_info_writer_.MarkBlockWillExit();
660 frame_access_state()->ClearSPDelta();
661 frame_access_state()->SetFrameAccessToDefault();
662 break;
663 }
664 case kArchTailCallAddress: {
665 CHECK(!instr->InputAt(0)->IsImmediate());
666 Register reg = i.InputRegister(0);
667 DCHECK_IMPLIES(
668 HasCallDescriptorFlag(instr, CallDescriptor::kFixedTargetRegister),
669 reg == kJavaScriptCallCodeStartRegister);
670 __ Jump(reg);
671 unwinding_info_writer_.MarkBlockWillExit();
672 frame_access_state()->ClearSPDelta();
673 frame_access_state()->SetFrameAccessToDefault();
674 break;
675 }
676 case kArchCallJSFunction: {
677 Register func = i.InputRegister(0);
678 if (FLAG_debug_code) {
679 // Check the function's context matches the context argument.
680 UseScratchRegisterScope scope(tasm());
681 Register temp = scope.AcquireX();
682 __ Ldr(temp, FieldMemOperand(func, JSFunction::kContextOffset));
683 __ cmp(cp, temp);
684 __ Assert(eq, AbortReason::kWrongFunctionContext);
685 }
686 static_assert(kJavaScriptCallCodeStartRegister == x2, "ABI mismatch");
687 __ Ldr(x2, FieldMemOperand(func, JSFunction::kCodeOffset));
688 __ Add(x2, x2, Operand(Code::kHeaderSize - kHeapObjectTag));
689 __ Call(x2);
690 RecordCallPosition(instr);
691 frame_access_state()->ClearSPDelta();
692 break;
693 }
694 case kArchPrepareCallCFunction:
695 // We don't need kArchPrepareCallCFunction on arm64 as the instruction
696 // selector has already performed a Claim to reserve space on the stack.
697 // Frame alignment is always 16 bytes, and the stack pointer is already
698 // 16-byte aligned, therefore we do not need to align the stack pointer
699 // by an unknown value, and it is safe to continue accessing the frame
700 // via the stack pointer.
701 UNREACHABLE();
702 break;
703 case kArchSaveCallerRegisters: {
704 fp_mode_ =
705 static_cast<SaveFPRegsMode>(MiscField::decode(instr->opcode()));
706 DCHECK(fp_mode_ == kDontSaveFPRegs || fp_mode_ == kSaveFPRegs);
707 // kReturnRegister0 should have been saved before entering the stub.
708 int bytes = __ PushCallerSaved(fp_mode_, kReturnRegister0);
709 DCHECK_EQ(0, bytes % kPointerSize);
710 DCHECK_EQ(0, frame_access_state()->sp_delta());
711 frame_access_state()->IncreaseSPDelta(bytes / kPointerSize);
712 DCHECK(!caller_registers_saved_);
713 caller_registers_saved_ = true;
714 break;
715 }
716 case kArchRestoreCallerRegisters: {
717 DCHECK(fp_mode_ ==
718 static_cast<SaveFPRegsMode>(MiscField::decode(instr->opcode())));
719 DCHECK(fp_mode_ == kDontSaveFPRegs || fp_mode_ == kSaveFPRegs);
720 // Don't overwrite the returned value.
721 int bytes = __ PopCallerSaved(fp_mode_, kReturnRegister0);
722 frame_access_state()->IncreaseSPDelta(-(bytes / kPointerSize));
723 DCHECK_EQ(0, frame_access_state()->sp_delta());
724 DCHECK(caller_registers_saved_);
725 caller_registers_saved_ = false;
726 break;
727 }
728 case kArchPrepareTailCall:
729 AssemblePrepareTailCall();
730 break;
731 case kArchCallCFunction: {
732 int const num_parameters = MiscField::decode(instr->opcode());
733 if (instr->InputAt(0)->IsImmediate()) {
734 ExternalReference ref = i.InputExternalReference(0);
735 __ CallCFunction(ref, num_parameters, 0);
736 } else {
737 Register func = i.InputRegister(0);
738 __ CallCFunction(func, num_parameters, 0);
739 }
740 frame_access_state()->SetFrameAccessToDefault();
741 // Ideally, we should decrement SP delta to match the change of stack
742 // pointer in CallCFunction. However, for certain architectures (e.g.
743 // ARM), there may be more strict alignment requirement, causing old SP
744 // to be saved on the stack. In those cases, we can not calculate the SP
745 // delta statically.
746 frame_access_state()->ClearSPDelta();
747 if (caller_registers_saved_) {
748 // Need to re-sync SP delta introduced in kArchSaveCallerRegisters.
749 // Here, we assume the sequence to be:
750 // kArchSaveCallerRegisters;
751 // kArchCallCFunction;
752 // kArchRestoreCallerRegisters;
753 int bytes =
754 __ RequiredStackSizeForCallerSaved(fp_mode_, kReturnRegister0);
755 frame_access_state()->IncreaseSPDelta(bytes / kPointerSize);
756 }
757 break;
758 }
759 case kArchJmp:
760 AssembleArchJump(i.InputRpo(0));
761 break;
762 case kArchTableSwitch:
763 AssembleArchTableSwitch(instr);
764 break;
765 case kArchBinarySearchSwitch:
766 AssembleArchBinarySearchSwitch(instr);
767 break;
768 case kArchLookupSwitch:
769 AssembleArchLookupSwitch(instr);
770 break;
771 case kArchDebugAbort:
772 DCHECK(i.InputRegister(0).is(x1));
773 if (!frame_access_state()->has_frame()) {
774 // We don't actually want to generate a pile of code for this, so just
775 // claim there is a stack frame, without generating one.
776 FrameScope scope(tasm(), StackFrame::NONE);
777 __ Call(isolate()->builtins()->builtin_handle(Builtins::kAbortJS),
778 RelocInfo::CODE_TARGET);
779 } else {
780 __ Call(isolate()->builtins()->builtin_handle(Builtins::kAbortJS),
781 RelocInfo::CODE_TARGET);
782 }
783 __ Debug("kArchDebugAbort", 0, BREAK);
784 unwinding_info_writer_.MarkBlockWillExit();
785 break;
786 case kArchDebugBreak:
787 __ Debug("kArchDebugBreak", 0, BREAK);
788 break;
789 case kArchComment:
790 __ RecordComment(reinterpret_cast<const char*>(i.InputInt64(0)));
791 break;
792 case kArchThrowTerminator:
793 unwinding_info_writer_.MarkBlockWillExit();
794 break;
795 case kArchNop:
796 // don't emit code for nops.
797 break;
798 case kArchDeoptimize: {
799 int deopt_state_id =
800 BuildTranslation(instr, -1, 0, OutputFrameStateCombine::Ignore());
801 CodeGenResult result =
802 AssembleDeoptimizerCall(deopt_state_id, current_source_position_);
803 if (result != kSuccess) return result;
804 unwinding_info_writer_.MarkBlockWillExit();
805 break;
806 }
807 case kArchRet:
808 AssembleReturn(instr->InputAt(0));
809 break;
810 case kArchStackPointer:
811 __ mov(i.OutputRegister(), sp);
812 break;
813 case kArchFramePointer:
814 __ mov(i.OutputRegister(), fp);
815 break;
816 case kArchParentFramePointer:
817 if (frame_access_state()->has_frame()) {
818 __ ldr(i.OutputRegister(), MemOperand(fp, 0));
819 } else {
820 __ mov(i.OutputRegister(), fp);
821 }
822 break;
823 case kArchTruncateDoubleToI:
824 __ TruncateDoubleToI(isolate(), zone(), i.OutputRegister(),
825 i.InputDoubleRegister(0), DetermineStubCallMode());
826 break;
827 case kArchStoreWithWriteBarrier: {
828 RecordWriteMode mode =
829 static_cast<RecordWriteMode>(MiscField::decode(instr->opcode()));
830 AddressingMode addressing_mode =
831 AddressingModeField::decode(instr->opcode());
832 Register object = i.InputRegister(0);
833 Operand index(0);
834 if (addressing_mode == kMode_MRI) {
835 index = Operand(i.InputInt64(1));
836 } else {
837 DCHECK_EQ(addressing_mode, kMode_MRR);
838 index = Operand(i.InputRegister(1));
839 }
840 Register value = i.InputRegister(2);
841 Register scratch0 = i.TempRegister(0);
842 Register scratch1 = i.TempRegister(1);
843 auto ool = new (zone())
844 OutOfLineRecordWrite(this, object, index, value, scratch0, scratch1,
845 mode, &unwinding_info_writer_);
846 __ Str(value, MemOperand(object, index));
847 __ CheckPageFlagSet(object, scratch0,
848 MemoryChunk::kPointersFromHereAreInterestingMask,
849 ool->entry());
850 __ Bind(ool->exit());
851 break;
852 }
853 case kArchStackSlot: {
854 FrameOffset offset =
855 frame_access_state()->GetFrameOffset(i.InputInt32(0));
856 Register base = offset.from_stack_pointer() ? sp : fp;
857 __ Add(i.OutputRegister(0), base, Operand(offset.offset()));
858 break;
859 }
860 case kIeee754Float64Acos:
861 ASSEMBLE_IEEE754_UNOP(acos);
862 break;
863 case kIeee754Float64Acosh:
864 ASSEMBLE_IEEE754_UNOP(acosh);
865 break;
866 case kIeee754Float64Asin:
867 ASSEMBLE_IEEE754_UNOP(asin);
868 break;
869 case kIeee754Float64Asinh:
870 ASSEMBLE_IEEE754_UNOP(asinh);
871 break;
872 case kIeee754Float64Atan:
873 ASSEMBLE_IEEE754_UNOP(atan);
874 break;
875 case kIeee754Float64Atanh:
876 ASSEMBLE_IEEE754_UNOP(atanh);
877 break;
878 case kIeee754Float64Atan2:
879 ASSEMBLE_IEEE754_BINOP(atan2);
880 break;
881 case kIeee754Float64Cos:
882 ASSEMBLE_IEEE754_UNOP(cos);
883 break;
884 case kIeee754Float64Cosh:
885 ASSEMBLE_IEEE754_UNOP(cosh);
886 break;
887 case kIeee754Float64Cbrt:
888 ASSEMBLE_IEEE754_UNOP(cbrt);
889 break;
890 case kIeee754Float64Exp:
891 ASSEMBLE_IEEE754_UNOP(exp);
892 break;
893 case kIeee754Float64Expm1:
894 ASSEMBLE_IEEE754_UNOP(expm1);
895 break;
896 case kIeee754Float64Log:
897 ASSEMBLE_IEEE754_UNOP(log);
898 break;
899 case kIeee754Float64Log1p:
900 ASSEMBLE_IEEE754_UNOP(log1p);
901 break;
902 case kIeee754Float64Log2:
903 ASSEMBLE_IEEE754_UNOP(log2);
904 break;
905 case kIeee754Float64Log10:
906 ASSEMBLE_IEEE754_UNOP(log10);
907 break;
908 case kIeee754Float64Pow: {
909 __ Call(BUILTIN_CODE(isolate(), MathPowInternal), RelocInfo::CODE_TARGET);
910 break;
911 }
912 case kIeee754Float64Sin:
913 ASSEMBLE_IEEE754_UNOP(sin);
914 break;
915 case kIeee754Float64Sinh:
916 ASSEMBLE_IEEE754_UNOP(sinh);
917 break;
918 case kIeee754Float64Tan:
919 ASSEMBLE_IEEE754_UNOP(tan);
920 break;
921 case kIeee754Float64Tanh:
922 ASSEMBLE_IEEE754_UNOP(tanh);
923 break;
924 case kArm64Float32RoundDown:
925 __ Frintm(i.OutputFloat32Register(), i.InputFloat32Register(0));
926 break;
927 case kArm64Float64RoundDown:
928 __ Frintm(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
929 break;
930 case kArm64Float32RoundUp:
931 __ Frintp(i.OutputFloat32Register(), i.InputFloat32Register(0));
932 break;
933 case kArm64Float64RoundUp:
934 __ Frintp(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
935 break;
936 case kArm64Float64RoundTiesAway:
937 __ Frinta(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
938 break;
939 case kArm64Float32RoundTruncate:
940 __ Frintz(i.OutputFloat32Register(), i.InputFloat32Register(0));
941 break;
942 case kArm64Float64RoundTruncate:
943 __ Frintz(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
944 break;
945 case kArm64Float32RoundTiesEven:
946 __ Frintn(i.OutputFloat32Register(), i.InputFloat32Register(0));
947 break;
948 case kArm64Float64RoundTiesEven:
949 __ Frintn(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
950 break;
951 case kArm64Add:
952 if (FlagsModeField::decode(opcode) != kFlags_none) {
953 __ Adds(i.OutputRegister(), i.InputOrZeroRegister64(0),
954 i.InputOperand2_64(1));
955 } else {
956 __ Add(i.OutputRegister(), i.InputOrZeroRegister64(0),
957 i.InputOperand2_64(1));
958 }
959 break;
960 case kArm64Add32:
961 if (FlagsModeField::decode(opcode) != kFlags_none) {
962 __ Adds(i.OutputRegister32(), i.InputOrZeroRegister32(0),
963 i.InputOperand2_32(1));
964 } else {
965 __ Add(i.OutputRegister32(), i.InputOrZeroRegister32(0),
966 i.InputOperand2_32(1));
967 }
968 break;
969 case kArm64And:
970 if (FlagsModeField::decode(opcode) != kFlags_none) {
971 // The ands instruction only sets N and Z, so only the following
972 // conditions make sense.
973 DCHECK(FlagsConditionField::decode(opcode) == kEqual ||
974 FlagsConditionField::decode(opcode) == kNotEqual ||
975 FlagsConditionField::decode(opcode) == kPositiveOrZero ||
976 FlagsConditionField::decode(opcode) == kNegative);
977 __ Ands(i.OutputRegister(), i.InputOrZeroRegister64(0),
978 i.InputOperand2_64(1));
979 } else {
980 __ And(i.OutputRegister(), i.InputOrZeroRegister64(0),
981 i.InputOperand2_64(1));
982 }
983 break;
984 case kArm64And32:
985 if (FlagsModeField::decode(opcode) != kFlags_none) {
986 // The ands instruction only sets N and Z, so only the following
987 // conditions make sense.
988 DCHECK(FlagsConditionField::decode(opcode) == kEqual ||
989 FlagsConditionField::decode(opcode) == kNotEqual ||
990 FlagsConditionField::decode(opcode) == kPositiveOrZero ||
991 FlagsConditionField::decode(opcode) == kNegative);
992 __ Ands(i.OutputRegister32(), i.InputOrZeroRegister32(0),
993 i.InputOperand2_32(1));
994 } else {
995 __ And(i.OutputRegister32(), i.InputOrZeroRegister32(0),
996 i.InputOperand2_32(1));
997 }
998 break;
999 case kArm64Bic:
1000 __ Bic(i.OutputRegister(), i.InputOrZeroRegister64(0),
1001 i.InputOperand2_64(1));
1002 break;
1003 case kArm64Bic32:
1004 __ Bic(i.OutputRegister32(), i.InputOrZeroRegister32(0),
1005 i.InputOperand2_32(1));
1006 break;
1007 case kArm64Mul:
1008 __ Mul(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1));
1009 break;
1010 case kArm64Mul32:
1011 __ Mul(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1));
1012 break;
1013 case kArm64Smull:
1014 __ Smull(i.OutputRegister(), i.InputRegister32(0), i.InputRegister32(1));
1015 break;
1016 case kArm64Umull:
1017 __ Umull(i.OutputRegister(), i.InputRegister32(0), i.InputRegister32(1));
1018 break;
1019 case kArm64Madd:
1020 __ Madd(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1),
1021 i.InputRegister(2));
1022 break;
1023 case kArm64Madd32:
1024 __ Madd(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1),
1025 i.InputRegister32(2));
1026 break;
1027 case kArm64Msub:
1028 __ Msub(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1),
1029 i.InputRegister(2));
1030 break;
1031 case kArm64Msub32:
1032 __ Msub(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1),
1033 i.InputRegister32(2));
1034 break;
1035 case kArm64Mneg:
1036 __ Mneg(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1));
1037 break;
1038 case kArm64Mneg32:
1039 __ Mneg(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1));
1040 break;
1041 case kArm64Idiv:
1042 __ Sdiv(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1));
1043 break;
1044 case kArm64Idiv32:
1045 __ Sdiv(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1));
1046 break;
1047 case kArm64Udiv:
1048 __ Udiv(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1));
1049 break;
1050 case kArm64Udiv32:
1051 __ Udiv(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1));
1052 break;
1053 case kArm64Imod: {
1054 UseScratchRegisterScope scope(tasm());
1055 Register temp = scope.AcquireX();
1056 __ Sdiv(temp, i.InputRegister(0), i.InputRegister(1));
1057 __ Msub(i.OutputRegister(), temp, i.InputRegister(1), i.InputRegister(0));
1058 break;
1059 }
1060 case kArm64Imod32: {
1061 UseScratchRegisterScope scope(tasm());
1062 Register temp = scope.AcquireW();
1063 __ Sdiv(temp, i.InputRegister32(0), i.InputRegister32(1));
1064 __ Msub(i.OutputRegister32(), temp, i.InputRegister32(1),
1065 i.InputRegister32(0));
1066 break;
1067 }
1068 case kArm64Umod: {
1069 UseScratchRegisterScope scope(tasm());
1070 Register temp = scope.AcquireX();
1071 __ Udiv(temp, i.InputRegister(0), i.InputRegister(1));
1072 __ Msub(i.OutputRegister(), temp, i.InputRegister(1), i.InputRegister(0));
1073 break;
1074 }
1075 case kArm64Umod32: {
1076 UseScratchRegisterScope scope(tasm());
1077 Register temp = scope.AcquireW();
1078 __ Udiv(temp, i.InputRegister32(0), i.InputRegister32(1));
1079 __ Msub(i.OutputRegister32(), temp, i.InputRegister32(1),
1080 i.InputRegister32(0));
1081 break;
1082 }
1083 case kArm64Not:
1084 __ Mvn(i.OutputRegister(), i.InputOperand(0));
1085 break;
1086 case kArm64Not32:
1087 __ Mvn(i.OutputRegister32(), i.InputOperand32(0));
1088 break;
1089 case kArm64Or:
1090 __ Orr(i.OutputRegister(), i.InputOrZeroRegister64(0),
1091 i.InputOperand2_64(1));
1092 break;
1093 case kArm64Or32:
1094 __ Orr(i.OutputRegister32(), i.InputOrZeroRegister32(0),
1095 i.InputOperand2_32(1));
1096 break;
1097 case kArm64Orn:
1098 __ Orn(i.OutputRegister(), i.InputOrZeroRegister64(0),
1099 i.InputOperand2_64(1));
1100 break;
1101 case kArm64Orn32:
1102 __ Orn(i.OutputRegister32(), i.InputOrZeroRegister32(0),
1103 i.InputOperand2_32(1));
1104 break;
1105 case kArm64Eor:
1106 __ Eor(i.OutputRegister(), i.InputOrZeroRegister64(0),
1107 i.InputOperand2_64(1));
1108 break;
1109 case kArm64Eor32:
1110 __ Eor(i.OutputRegister32(), i.InputOrZeroRegister32(0),
1111 i.InputOperand2_32(1));
1112 break;
1113 case kArm64Eon:
1114 __ Eon(i.OutputRegister(), i.InputOrZeroRegister64(0),
1115 i.InputOperand2_64(1));
1116 break;
1117 case kArm64Eon32:
1118 __ Eon(i.OutputRegister32(), i.InputOrZeroRegister32(0),
1119 i.InputOperand2_32(1));
1120 break;
1121 case kArm64Sub:
1122 if (FlagsModeField::decode(opcode) != kFlags_none) {
1123 __ Subs(i.OutputRegister(), i.InputOrZeroRegister64(0),
1124 i.InputOperand2_64(1));
1125 } else {
1126 __ Sub(i.OutputRegister(), i.InputOrZeroRegister64(0),
1127 i.InputOperand2_64(1));
1128 }
1129 break;
1130 case kArm64Sub32:
1131 if (FlagsModeField::decode(opcode) != kFlags_none) {
1132 __ Subs(i.OutputRegister32(), i.InputOrZeroRegister32(0),
1133 i.InputOperand2_32(1));
1134 } else {
1135 __ Sub(i.OutputRegister32(), i.InputOrZeroRegister32(0),
1136 i.InputOperand2_32(1));
1137 }
1138 break;
1139 case kArm64Lsl:
1140 ASSEMBLE_SHIFT(Lsl, 64);
1141 break;
1142 case kArm64Lsl32:
1143 ASSEMBLE_SHIFT(Lsl, 32);
1144 break;
1145 case kArm64Lsr:
1146 ASSEMBLE_SHIFT(Lsr, 64);
1147 break;
1148 case kArm64Lsr32:
1149 ASSEMBLE_SHIFT(Lsr, 32);
1150 break;
1151 case kArm64Asr:
1152 ASSEMBLE_SHIFT(Asr, 64);
1153 break;
1154 case kArm64Asr32:
1155 ASSEMBLE_SHIFT(Asr, 32);
1156 break;
1157 case kArm64Ror:
1158 ASSEMBLE_SHIFT(Ror, 64);
1159 break;
1160 case kArm64Ror32:
1161 ASSEMBLE_SHIFT(Ror, 32);
1162 break;
1163 case kArm64Mov32:
1164 __ Mov(i.OutputRegister32(), i.InputRegister32(0));
1165 break;
1166 case kArm64Sxtb32:
1167 __ Sxtb(i.OutputRegister32(), i.InputRegister32(0));
1168 break;
1169 case kArm64Sxth32:
1170 __ Sxth(i.OutputRegister32(), i.InputRegister32(0));
1171 break;
1172 case kArm64Sxtb:
1173 __ Sxtb(i.OutputRegister(), i.InputRegister32(0));
1174 break;
1175 case kArm64Sxth:
1176 __ Sxth(i.OutputRegister(), i.InputRegister32(0));
1177 break;
1178 case kArm64Sxtw:
1179 __ Sxtw(i.OutputRegister(), i.InputRegister32(0));
1180 break;
1181 case kArm64Sbfx32:
1182 __ Sbfx(i.OutputRegister32(), i.InputRegister32(0), i.InputInt5(1),
1183 i.InputInt5(2));
1184 break;
1185 case kArm64Ubfx:
1186 __ Ubfx(i.OutputRegister(), i.InputRegister(0), i.InputInt6(1),
1187 i.InputInt32(2));
1188 break;
1189 case kArm64Ubfx32:
1190 __ Ubfx(i.OutputRegister32(), i.InputRegister32(0), i.InputInt5(1),
1191 i.InputInt32(2));
1192 break;
1193 case kArm64Ubfiz32:
1194 __ Ubfiz(i.OutputRegister32(), i.InputRegister32(0), i.InputInt5(1),
1195 i.InputInt5(2));
1196 break;
1197 case kArm64Bfi:
1198 __ Bfi(i.OutputRegister(), i.InputRegister(1), i.InputInt6(2),
1199 i.InputInt6(3));
1200 break;
1201 case kArm64TestAndBranch32:
1202 case kArm64TestAndBranch:
1203 // Pseudo instructions turned into tbz/tbnz in AssembleArchBranch.
1204 break;
1205 case kArm64CompareAndBranch32:
1206 case kArm64CompareAndBranch:
1207 // Pseudo instruction handled in AssembleArchBranch.
1208 break;
1209 case kArm64Claim: {
1210 int count = i.InputInt32(0);
1211 DCHECK_EQ(count % 2, 0);
1212 __ AssertSpAligned();
1213 if (count > 0) {
1214 __ Claim(count);
1215 frame_access_state()->IncreaseSPDelta(count);
1216 }
1217 break;
1218 }
1219 case kArm64Poke: {
1220 Operand operand(i.InputInt32(1) * kPointerSize);
1221 if (instr->InputAt(0)->IsSimd128Register()) {
1222 __ Poke(i.InputSimd128Register(0), operand);
1223 } else if (instr->InputAt(0)->IsFPRegister()) {
1224 __ Poke(i.InputFloat64Register(0), operand);
1225 } else {
1226 __ Poke(i.InputOrZeroRegister64(0), operand);
1227 }
1228 break;
1229 }
1230 case kArm64PokePair: {
1231 int slot = i.InputInt32(2) - 1;
1232 if (instr->InputAt(0)->IsFPRegister()) {
1233 __ PokePair(i.InputFloat64Register(1), i.InputFloat64Register(0),
1234 slot * kPointerSize);
1235 } else {
1236 __ PokePair(i.InputRegister(1), i.InputRegister(0),
1237 slot * kPointerSize);
1238 }
1239 break;
1240 }
1241 case kArm64Peek: {
1242 int reverse_slot = i.InputInt32(0);
1243 int offset =
1244 FrameSlotToFPOffset(frame()->GetTotalFrameSlotCount() - reverse_slot);
1245 if (instr->OutputAt(0)->IsFPRegister()) {
1246 LocationOperand* op = LocationOperand::cast(instr->OutputAt(0));
1247 if (op->representation() == MachineRepresentation::kFloat64) {
1248 __ Ldr(i.OutputDoubleRegister(), MemOperand(fp, offset));
1249 } else {
1250 DCHECK_EQ(MachineRepresentation::kFloat32, op->representation());
1251 __ Ldr(i.OutputFloatRegister(), MemOperand(fp, offset));
1252 }
1253 } else {
1254 __ Ldr(i.OutputRegister(), MemOperand(fp, offset));
1255 }
1256 break;
1257 }
1258 case kArm64Clz:
1259 __ Clz(i.OutputRegister64(), i.InputRegister64(0));
1260 break;
1261 case kArm64Clz32:
1262 __ Clz(i.OutputRegister32(), i.InputRegister32(0));
1263 break;
1264 case kArm64Rbit:
1265 __ Rbit(i.OutputRegister64(), i.InputRegister64(0));
1266 break;
1267 case kArm64Rbit32:
1268 __ Rbit(i.OutputRegister32(), i.InputRegister32(0));
1269 break;
1270 case kArm64Rev:
1271 __ Rev(i.OutputRegister64(), i.InputRegister64(0));
1272 break;
1273 case kArm64Rev32:
1274 __ Rev(i.OutputRegister32(), i.InputRegister32(0));
1275 break;
1276 case kArm64Cmp:
1277 __ Cmp(i.InputOrZeroRegister64(0), i.InputOperand2_64(1));
1278 break;
1279 case kArm64Cmp32:
1280 __ Cmp(i.InputOrZeroRegister32(0), i.InputOperand2_32(1));
1281 break;
1282 case kArm64Cmn:
1283 __ Cmn(i.InputOrZeroRegister64(0), i.InputOperand2_64(1));
1284 break;
1285 case kArm64Cmn32:
1286 __ Cmn(i.InputOrZeroRegister32(0), i.InputOperand2_32(1));
1287 break;
1288 case kArm64Tst:
1289 __ Tst(i.InputOrZeroRegister64(0), i.InputOperand2_64(1));
1290 break;
1291 case kArm64Tst32:
1292 __ Tst(i.InputOrZeroRegister32(0), i.InputOperand2_32(1));
1293 break;
1294 case kArm64Float32Cmp:
1295 if (instr->InputAt(1)->IsFPRegister()) {
1296 __ Fcmp(i.InputFloat32Register(0), i.InputFloat32Register(1));
1297 } else {
1298 DCHECK(instr->InputAt(1)->IsImmediate());
1299 // 0.0 is the only immediate supported by fcmp instructions.
1300 DCHECK_EQ(0.0f, i.InputFloat32(1));
1301 __ Fcmp(i.InputFloat32Register(0), i.InputFloat32(1));
1302 }
1303 break;
1304 case kArm64Float32Add:
1305 __ Fadd(i.OutputFloat32Register(), i.InputFloat32Register(0),
1306 i.InputFloat32Register(1));
1307 break;
1308 case kArm64Float32Sub:
1309 __ Fsub(i.OutputFloat32Register(), i.InputFloat32Register(0),
1310 i.InputFloat32Register(1));
1311 break;
1312 case kArm64Float32Mul:
1313 __ Fmul(i.OutputFloat32Register(), i.InputFloat32Register(0),
1314 i.InputFloat32Register(1));
1315 break;
1316 case kArm64Float32Div:
1317 __ Fdiv(i.OutputFloat32Register(), i.InputFloat32Register(0),
1318 i.InputFloat32Register(1));
1319 break;
1320 case kArm64Float32Abs:
1321 __ Fabs(i.OutputFloat32Register(), i.InputFloat32Register(0));
1322 break;
1323 case kArm64Float32Neg:
1324 __ Fneg(i.OutputFloat32Register(), i.InputFloat32Register(0));
1325 break;
1326 case kArm64Float32Sqrt:
1327 __ Fsqrt(i.OutputFloat32Register(), i.InputFloat32Register(0));
1328 break;
1329 case kArm64Float64Cmp:
1330 if (instr->InputAt(1)->IsFPRegister()) {
1331 __ Fcmp(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1332 } else {
1333 DCHECK(instr->InputAt(1)->IsImmediate());
1334 // 0.0 is the only immediate supported by fcmp instructions.
1335 DCHECK_EQ(0.0, i.InputDouble(1));
1336 __ Fcmp(i.InputDoubleRegister(0), i.InputDouble(1));
1337 }
1338 break;
1339 case kArm64Float64Add:
1340 __ Fadd(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1341 i.InputDoubleRegister(1));
1342 break;
1343 case kArm64Float64Sub:
1344 __ Fsub(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1345 i.InputDoubleRegister(1));
1346 break;
1347 case kArm64Float64Mul:
1348 __ Fmul(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1349 i.InputDoubleRegister(1));
1350 break;
1351 case kArm64Float64Div:
1352 __ Fdiv(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1353 i.InputDoubleRegister(1));
1354 break;
1355 case kArm64Float64Mod: {
1356 // TODO(turbofan): implement directly.
1357 FrameScope scope(tasm(), StackFrame::MANUAL);
1358 DCHECK(d0.is(i.InputDoubleRegister(0)));
1359 DCHECK(d1.is(i.InputDoubleRegister(1)));
1360 DCHECK(d0.is(i.OutputDoubleRegister()));
1361 // TODO(turbofan): make sure this saves all relevant registers.
1362 __ CallCFunction(ExternalReference::mod_two_doubles_operation(), 0, 2);
1363 break;
1364 }
1365 case kArm64Float32Max: {
1366 __ Fmax(i.OutputFloat32Register(), i.InputFloat32Register(0),
1367 i.InputFloat32Register(1));
1368 break;
1369 }
1370 case kArm64Float64Max: {
1371 __ Fmax(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1372 i.InputDoubleRegister(1));
1373 break;
1374 }
1375 case kArm64Float32Min: {
1376 __ Fmin(i.OutputFloat32Register(), i.InputFloat32Register(0),
1377 i.InputFloat32Register(1));
1378 break;
1379 }
1380 case kArm64Float64Min: {
1381 __ Fmin(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1382 i.InputDoubleRegister(1));
1383 break;
1384 }
1385 case kArm64Float64Abs:
1386 __ Fabs(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
1387 break;
1388 case kArm64Float64Neg:
1389 __ Fneg(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
1390 break;
1391 case kArm64Float64Sqrt:
1392 __ Fsqrt(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
1393 break;
1394 case kArm64Float32ToFloat64:
1395 __ Fcvt(i.OutputDoubleRegister(), i.InputDoubleRegister(0).S());
1396 break;
1397 case kArm64Float64ToFloat32:
1398 __ Fcvt(i.OutputDoubleRegister().S(), i.InputDoubleRegister(0));
1399 break;
1400 case kArm64Float32ToInt32:
1401 __ Fcvtzs(i.OutputRegister32(), i.InputFloat32Register(0));
1402 // Avoid INT32_MAX as an overflow indicator and use INT32_MIN instead,
1403 // because INT32_MIN allows easier out-of-bounds detection.
1404 __ Cmn(i.OutputRegister32(), 1);
1405 __ Csinc(i.OutputRegister32(), i.OutputRegister32(), i.OutputRegister32(),
1406 vc);
1407 break;
1408 case kArm64Float64ToInt32:
1409 __ Fcvtzs(i.OutputRegister32(), i.InputDoubleRegister(0));
1410 break;
1411 case kArm64Float32ToUint32:
1412 __ Fcvtzu(i.OutputRegister32(), i.InputFloat32Register(0));
1413 // Avoid UINT32_MAX as an overflow indicator and use 0 instead,
1414 // because 0 allows easier out-of-bounds detection.
1415 __ Cmn(i.OutputRegister32(), 1);
1416 __ Adc(i.OutputRegister32(), i.OutputRegister32(), Operand(0));
1417 break;
1418 case kArm64Float64ToUint32:
1419 __ Fcvtzu(i.OutputRegister32(), i.InputDoubleRegister(0));
1420 break;
1421 case kArm64Float32ToInt64:
1422 __ Fcvtzs(i.OutputRegister64(), i.InputFloat32Register(0));
1423 if (i.OutputCount() > 1) {
1424 // Check for inputs below INT64_MIN and NaN.
1425 __ Fcmp(i.InputFloat32Register(0), static_cast<float>(INT64_MIN));
1426 // Check overflow.
1427 // -1 value is used to indicate a possible overflow which will occur
1428 // when subtracting (-1) from the provided INT64_MAX operand.
1429 // OutputRegister(1) is set to 0 if the input was out of range or NaN.
1430 __ Ccmp(i.OutputRegister(0), -1, VFlag, ge);
1431 __ Cset(i.OutputRegister(1), vc);
1432 }
1433 break;
1434 case kArm64Float64ToInt64:
1435 __ Fcvtzs(i.OutputRegister(0), i.InputDoubleRegister(0));
1436 if (i.OutputCount() > 1) {
1437 // See kArm64Float32ToInt64 for a detailed description.
1438 __ Fcmp(i.InputDoubleRegister(0), static_cast<double>(INT64_MIN));
1439 __ Ccmp(i.OutputRegister(0), -1, VFlag, ge);
1440 __ Cset(i.OutputRegister(1), vc);
1441 }
1442 break;
1443 case kArm64Float32ToUint64:
1444 __ Fcvtzu(i.OutputRegister64(), i.InputFloat32Register(0));
1445 if (i.OutputCount() > 1) {
1446 // See kArm64Float32ToInt64 for a detailed description.
1447 __ Fcmp(i.InputFloat32Register(0), -1.0);
1448 __ Ccmp(i.OutputRegister(0), -1, ZFlag, gt);
1449 __ Cset(i.OutputRegister(1), ne);
1450 }
1451 break;
1452 case kArm64Float64ToUint64:
1453 __ Fcvtzu(i.OutputRegister64(), i.InputDoubleRegister(0));
1454 if (i.OutputCount() > 1) {
1455 // See kArm64Float32ToInt64 for a detailed description.
1456 __ Fcmp(i.InputDoubleRegister(0), -1.0);
1457 __ Ccmp(i.OutputRegister(0), -1, ZFlag, gt);
1458 __ Cset(i.OutputRegister(1), ne);
1459 }
1460 break;
1461 case kArm64Int32ToFloat32:
1462 __ Scvtf(i.OutputFloat32Register(), i.InputRegister32(0));
1463 break;
1464 case kArm64Int32ToFloat64:
1465 __ Scvtf(i.OutputDoubleRegister(), i.InputRegister32(0));
1466 break;
1467 case kArm64Int64ToFloat32:
1468 __ Scvtf(i.OutputDoubleRegister().S(), i.InputRegister64(0));
1469 break;
1470 case kArm64Int64ToFloat64:
1471 __ Scvtf(i.OutputDoubleRegister(), i.InputRegister64(0));
1472 break;
1473 case kArm64Uint32ToFloat32:
1474 __ Ucvtf(i.OutputFloat32Register(), i.InputRegister32(0));
1475 break;
1476 case kArm64Uint32ToFloat64:
1477 __ Ucvtf(i.OutputDoubleRegister(), i.InputRegister32(0));
1478 break;
1479 case kArm64Uint64ToFloat32:
1480 __ Ucvtf(i.OutputDoubleRegister().S(), i.InputRegister64(0));
1481 break;
1482 case kArm64Uint64ToFloat64:
1483 __ Ucvtf(i.OutputDoubleRegister(), i.InputRegister64(0));
1484 break;
1485 case kArm64Float64ExtractLowWord32:
1486 __ Fmov(i.OutputRegister32(), i.InputFloat32Register(0));
1487 break;
1488 case kArm64Float64ExtractHighWord32:
1489 __ Umov(i.OutputRegister32(), i.InputFloat64Register(0).V2S(), 1);
1490 break;
1491 case kArm64Float64InsertLowWord32:
1492 DCHECK(i.OutputFloat64Register().Is(i.InputFloat64Register(0)));
1493 __ Ins(i.OutputFloat64Register().V2S(), 0, i.InputRegister32(1));
1494 break;
1495 case kArm64Float64InsertHighWord32:
1496 DCHECK(i.OutputFloat64Register().Is(i.InputFloat64Register(0)));
1497 __ Ins(i.OutputFloat64Register().V2S(), 1, i.InputRegister32(1));
1498 break;
1499 case kArm64Float64MoveU64:
1500 __ Fmov(i.OutputFloat64Register(), i.InputRegister(0));
1501 break;
1502 case kArm64Float64SilenceNaN:
1503 __ CanonicalizeNaN(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
1504 break;
1505 case kArm64U64MoveFloat64:
1506 __ Fmov(i.OutputRegister(), i.InputDoubleRegister(0));
1507 break;
1508 case kArm64Ldrb:
1509 __ Ldrb(i.OutputRegister(), i.MemoryOperand());
1510 EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i);
1511 break;
1512 case kArm64Ldrsb:
1513 __ Ldrsb(i.OutputRegister(), i.MemoryOperand());
1514 EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i);
1515 break;
1516 case kArm64Strb:
1517 __ Strb(i.InputOrZeroRegister64(0), i.MemoryOperand(1));
1518 break;
1519 case kArm64Ldrh:
1520 __ Ldrh(i.OutputRegister(), i.MemoryOperand());
1521 EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i);
1522 break;
1523 case kArm64Ldrsh:
1524 __ Ldrsh(i.OutputRegister(), i.MemoryOperand());
1525 EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i);
1526 break;
1527 case kArm64Strh:
1528 __ Strh(i.InputOrZeroRegister64(0), i.MemoryOperand(1));
1529 break;
1530 case kArm64Ldrsw:
1531 __ Ldrsw(i.OutputRegister(), i.MemoryOperand());
1532 EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i);
1533 break;
1534 case kArm64LdrW:
1535 __ Ldr(i.OutputRegister32(), i.MemoryOperand());
1536 EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i);
1537 break;
1538 case kArm64StrW:
1539 __ Str(i.InputOrZeroRegister32(0), i.MemoryOperand(1));
1540 break;
1541 case kArm64Ldr:
1542 __ Ldr(i.OutputRegister(), i.MemoryOperand());
1543 EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i);
1544 break;
1545 case kArm64Str:
1546 __ Str(i.InputOrZeroRegister64(0), i.MemoryOperand(1));
1547 break;
1548 case kArm64LdrS:
1549 __ Ldr(i.OutputDoubleRegister().S(), i.MemoryOperand());
1550 break;
1551 case kArm64StrS:
1552 __ Str(i.InputFloat32OrZeroRegister(0), i.MemoryOperand(1));
1553 break;
1554 case kArm64LdrD:
1555 __ Ldr(i.OutputDoubleRegister(), i.MemoryOperand());
1556 break;
1557 case kArm64StrD:
1558 __ Str(i.InputFloat64OrZeroRegister(0), i.MemoryOperand(1));
1559 break;
1560 case kArm64LdrQ:
1561 __ Ldr(i.OutputSimd128Register(), i.MemoryOperand());
1562 break;
1563 case kArm64StrQ:
1564 __ Str(i.InputSimd128Register(0), i.MemoryOperand(1));
1565 break;
1566 case kArm64DsbIsb:
1567 __ Dsb(FullSystem, BarrierAll);
1568 __ Isb();
1569 break;
1570 case kArchWordPoisonOnSpeculation:
1571 __ And(i.OutputRegister(0), i.InputRegister(0),
1572 Operand(kSpeculationPoisonRegister));
1573 break;
1574 case kWord32AtomicLoadInt8:
1575 ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldarb, Register32);
1576 __ Sxtb(i.OutputRegister(0), i.OutputRegister(0));
1577 break;
1578 case kWord32AtomicLoadUint8:
1579 case kArm64Word64AtomicLoadUint8:
1580 ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldarb, Register32);
1581 break;
1582 case kWord32AtomicLoadInt16:
1583 ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldarh, Register32);
1584 __ Sxth(i.OutputRegister(0), i.OutputRegister(0));
1585 break;
1586 case kWord32AtomicLoadUint16:
1587 case kArm64Word64AtomicLoadUint16:
1588 ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldarh, Register32);
1589 break;
1590 case kWord32AtomicLoadWord32:
1591 case kArm64Word64AtomicLoadUint32:
1592 ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldar, Register32);
1593 break;
1594 case kArm64Word64AtomicLoadUint64:
1595 ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldar, Register);
1596 break;
1597 case kWord32AtomicStoreWord8:
1598 case kArm64Word64AtomicStoreWord8:
1599 ASSEMBLE_ATOMIC_STORE_INTEGER(Stlrb, Register32);
1600 break;
1601 case kWord32AtomicStoreWord16:
1602 case kArm64Word64AtomicStoreWord16:
1603 ASSEMBLE_ATOMIC_STORE_INTEGER(Stlrh, Register32);
1604 break;
1605 case kWord32AtomicStoreWord32:
1606 case kArm64Word64AtomicStoreWord32:
1607 ASSEMBLE_ATOMIC_STORE_INTEGER(Stlr, Register32);
1608 break;
1609 case kArm64Word64AtomicStoreWord64:
1610 ASSEMBLE_ATOMIC_STORE_INTEGER(Stlr, Register);
1611 break;
1612 case kWord32AtomicExchangeInt8:
1613 ASSEMBLE_ATOMIC_EXCHANGE_INTEGER(ldaxrb, stlxrb, Register32);
1614 __ Sxtb(i.OutputRegister(0), i.OutputRegister(0));
1615 break;
1616 case kWord32AtomicExchangeUint8:
1617 case kArm64Word64AtomicExchangeUint8:
1618 ASSEMBLE_ATOMIC_EXCHANGE_INTEGER(ldaxrb, stlxrb, Register32);
1619 break;
1620 case kWord32AtomicExchangeInt16:
1621 ASSEMBLE_ATOMIC_EXCHANGE_INTEGER(ldaxrh, stlxrh, Register32);
1622 __ Sxth(i.OutputRegister(0), i.OutputRegister(0));
1623 break;
1624 case kWord32AtomicExchangeUint16:
1625 case kArm64Word64AtomicExchangeUint16:
1626 ASSEMBLE_ATOMIC_EXCHANGE_INTEGER(ldaxrh, stlxrh, Register32);
1627 break;
1628 case kWord32AtomicExchangeWord32:
1629 case kArm64Word64AtomicExchangeUint32:
1630 ASSEMBLE_ATOMIC_EXCHANGE_INTEGER(ldaxr, stlxr, Register32);
1631 break;
1632 case kArm64Word64AtomicExchangeUint64:
1633 ASSEMBLE_ATOMIC_EXCHANGE_INTEGER(ldaxr, stlxr, Register);
1634 break;
1635 case kWord32AtomicCompareExchangeInt8:
1636 ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER(ldaxrb, stlxrb, UXTB,
1637 Register32);
1638 __ Sxtb(i.OutputRegister(0), i.OutputRegister(0));
1639 break;
1640 case kWord32AtomicCompareExchangeUint8:
1641 case kArm64Word64AtomicCompareExchangeUint8:
1642 ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER(ldaxrb, stlxrb, UXTB,
1643 Register32);
1644 break;
1645 case kWord32AtomicCompareExchangeInt16:
1646 ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER(ldaxrh, stlxrh, UXTH,
1647 Register32);
1648 __ Sxth(i.OutputRegister(0), i.OutputRegister(0));
1649 break;
1650 case kWord32AtomicCompareExchangeUint16:
1651 case kArm64Word64AtomicCompareExchangeUint16:
1652 ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER(ldaxrh, stlxrh, UXTH,
1653 Register32);
1654 break;
1655 case kWord32AtomicCompareExchangeWord32:
1656 case kArm64Word64AtomicCompareExchangeUint32:
1657 ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER(ldaxr, stlxr, UXTW, Register32);
1658 break;
1659 case kArm64Word64AtomicCompareExchangeUint64:
1660 ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER(ldaxr, stlxr, UXTX, Register);
1661 break;
1662 #define ATOMIC_BINOP_CASE(op, inst) \
1663 case kWord32Atomic##op##Int8: \
1664 ASSEMBLE_ATOMIC_BINOP(ldaxrb, stlxrb, inst, Register32); \
1665 __ Sxtb(i.OutputRegister(0), i.OutputRegister(0)); \
1666 break; \
1667 case kWord32Atomic##op##Uint8: \
1668 case kArm64Word64Atomic##op##Uint8: \
1669 ASSEMBLE_ATOMIC_BINOP(ldaxrb, stlxrb, inst, Register32); \
1670 break; \
1671 case kWord32Atomic##op##Int16: \
1672 ASSEMBLE_ATOMIC_BINOP(ldaxrh, stlxrh, inst, Register32); \
1673 __ Sxth(i.OutputRegister(0), i.OutputRegister(0)); \
1674 break; \
1675 case kWord32Atomic##op##Uint16: \
1676 case kArm64Word64Atomic##op##Uint16: \
1677 ASSEMBLE_ATOMIC_BINOP(ldaxrh, stlxrh, inst, Register32); \
1678 break; \
1679 case kWord32Atomic##op##Word32: \
1680 case kArm64Word64Atomic##op##Uint32: \
1681 ASSEMBLE_ATOMIC_BINOP(ldaxr, stlxr, inst, Register32); \
1682 break; \
1683 case kArm64Word64Atomic##op##Uint64: \
1684 ASSEMBLE_ATOMIC_BINOP(ldaxr, stlxr, inst, Register); \
1685 break;
1686 ATOMIC_BINOP_CASE(Add, Add)
1687 ATOMIC_BINOP_CASE(Sub, Sub)
1688 ATOMIC_BINOP_CASE(And, And)
1689 ATOMIC_BINOP_CASE(Or, Orr)
1690 ATOMIC_BINOP_CASE(Xor, Eor)
1691 #undef ATOMIC_BINOP_CASE
1692 #undef ASSEMBLE_SHIFT
1693 #undef ASSEMBLE_ATOMIC_LOAD_INTEGER
1694 #undef ASSEMBLE_ATOMIC_STORE_INTEGER
1695 #undef ASSEMBLE_ATOMIC_EXCHANGE_INTEGER
1696 #undef ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER
1697 #undef ASSEMBLE_ATOMIC_BINOP
1698 #undef ASSEMBLE_IEEE754_BINOP
1699 #undef ASSEMBLE_IEEE754_UNOP
1700
1701 #define SIMD_UNOP_CASE(Op, Instr, FORMAT) \
1702 case Op: \
1703 __ Instr(i.OutputSimd128Register().V##FORMAT(), \
1704 i.InputSimd128Register(0).V##FORMAT()); \
1705 break;
1706 #define SIMD_WIDENING_UNOP_CASE(Op, Instr, WIDE, NARROW) \
1707 case Op: \
1708 __ Instr(i.OutputSimd128Register().V##WIDE(), \
1709 i.InputSimd128Register(0).V##NARROW()); \
1710 break;
1711 #define SIMD_BINOP_CASE(Op, Instr, FORMAT) \
1712 case Op: \
1713 __ Instr(i.OutputSimd128Register().V##FORMAT(), \
1714 i.InputSimd128Register(0).V##FORMAT(), \
1715 i.InputSimd128Register(1).V##FORMAT()); \
1716 break;
1717
1718 case kArm64F32x4Splat: {
1719 __ Dup(i.OutputSimd128Register().V4S(), i.InputSimd128Register(0).S(), 0);
1720 break;
1721 }
1722 case kArm64F32x4ExtractLane: {
1723 __ Mov(i.OutputSimd128Register().S(), i.InputSimd128Register(0).V4S(),
1724 i.InputInt8(1));
1725 break;
1726 }
1727 case kArm64F32x4ReplaceLane: {
1728 VRegister dst = i.OutputSimd128Register().V4S(),
1729 src1 = i.InputSimd128Register(0).V4S();
1730 if (!dst.is(src1)) {
1731 __ Mov(dst, src1);
1732 }
1733 __ Mov(dst, i.InputInt8(1), i.InputSimd128Register(2).V4S(), 0);
1734 break;
1735 }
1736 SIMD_UNOP_CASE(kArm64F32x4SConvertI32x4, Scvtf, 4S);
1737 SIMD_UNOP_CASE(kArm64F32x4UConvertI32x4, Ucvtf, 4S);
1738 SIMD_UNOP_CASE(kArm64F32x4Abs, Fabs, 4S);
1739 SIMD_UNOP_CASE(kArm64F32x4Neg, Fneg, 4S);
1740 SIMD_UNOP_CASE(kArm64F32x4RecipApprox, Frecpe, 4S);
1741 SIMD_UNOP_CASE(kArm64F32x4RecipSqrtApprox, Frsqrte, 4S);
1742 SIMD_BINOP_CASE(kArm64F32x4Add, Fadd, 4S);
1743 SIMD_BINOP_CASE(kArm64F32x4AddHoriz, Faddp, 4S);
1744 SIMD_BINOP_CASE(kArm64F32x4Sub, Fsub, 4S);
1745 SIMD_BINOP_CASE(kArm64F32x4Mul, Fmul, 4S);
1746 SIMD_BINOP_CASE(kArm64F32x4Min, Fmin, 4S);
1747 SIMD_BINOP_CASE(kArm64F32x4Max, Fmax, 4S);
1748 SIMD_BINOP_CASE(kArm64F32x4Eq, Fcmeq, 4S);
1749 case kArm64F32x4Ne: {
1750 VRegister dst = i.OutputSimd128Register().V4S();
1751 __ Fcmeq(dst, i.InputSimd128Register(0).V4S(),
1752 i.InputSimd128Register(1).V4S());
1753 __ Mvn(dst, dst);
1754 break;
1755 }
1756 case kArm64F32x4Lt: {
1757 __ Fcmgt(i.OutputSimd128Register().V4S(), i.InputSimd128Register(1).V4S(),
1758 i.InputSimd128Register(0).V4S());
1759 break;
1760 }
1761 case kArm64F32x4Le: {
1762 __ Fcmge(i.OutputSimd128Register().V4S(), i.InputSimd128Register(1).V4S(),
1763 i.InputSimd128Register(0).V4S());
1764 break;
1765 }
1766 case kArm64I32x4Splat: {
1767 __ Dup(i.OutputSimd128Register().V4S(), i.InputRegister32(0));
1768 break;
1769 }
1770 case kArm64I32x4ExtractLane: {
1771 __ Mov(i.OutputRegister32(), i.InputSimd128Register(0).V4S(),
1772 i.InputInt8(1));
1773 break;
1774 }
1775 case kArm64I32x4ReplaceLane: {
1776 VRegister dst = i.OutputSimd128Register().V4S(),
1777 src1 = i.InputSimd128Register(0).V4S();
1778 if (!dst.is(src1)) {
1779 __ Mov(dst, src1);
1780 }
1781 __ Mov(dst, i.InputInt8(1), i.InputRegister32(2));
1782 break;
1783 }
1784 SIMD_UNOP_CASE(kArm64I32x4SConvertF32x4, Fcvtzs, 4S);
1785 SIMD_WIDENING_UNOP_CASE(kArm64I32x4SConvertI16x8Low, Sxtl, 4S, 4H);
1786 SIMD_WIDENING_UNOP_CASE(kArm64I32x4SConvertI16x8High, Sxtl2, 4S, 8H);
1787 SIMD_UNOP_CASE(kArm64I32x4Neg, Neg, 4S);
1788 case kArm64I32x4Shl: {
1789 __ Shl(i.OutputSimd128Register().V4S(), i.InputSimd128Register(0).V4S(),
1790 i.InputInt5(1));
1791 break;
1792 }
1793 case kArm64I32x4ShrS: {
1794 __ Sshr(i.OutputSimd128Register().V4S(), i.InputSimd128Register(0).V4S(),
1795 i.InputInt5(1));
1796 break;
1797 }
1798 SIMD_BINOP_CASE(kArm64I32x4Add, Add, 4S);
1799 SIMD_BINOP_CASE(kArm64I32x4AddHoriz, Addp, 4S);
1800 SIMD_BINOP_CASE(kArm64I32x4Sub, Sub, 4S);
1801 SIMD_BINOP_CASE(kArm64I32x4Mul, Mul, 4S);
1802 SIMD_BINOP_CASE(kArm64I32x4MinS, Smin, 4S);
1803 SIMD_BINOP_CASE(kArm64I32x4MaxS, Smax, 4S);
1804 SIMD_BINOP_CASE(kArm64I32x4Eq, Cmeq, 4S);
1805 case kArm64I32x4Ne: {
1806 VRegister dst = i.OutputSimd128Register().V4S();
1807 __ Cmeq(dst, i.InputSimd128Register(0).V4S(),
1808 i.InputSimd128Register(1).V4S());
1809 __ Mvn(dst, dst);
1810 break;
1811 }
1812 SIMD_BINOP_CASE(kArm64I32x4GtS, Cmgt, 4S);
1813 SIMD_BINOP_CASE(kArm64I32x4GeS, Cmge, 4S);
1814 SIMD_UNOP_CASE(kArm64I32x4UConvertF32x4, Fcvtzu, 4S);
1815 SIMD_WIDENING_UNOP_CASE(kArm64I32x4UConvertI16x8Low, Uxtl, 4S, 4H);
1816 SIMD_WIDENING_UNOP_CASE(kArm64I32x4UConvertI16x8High, Uxtl2, 4S, 8H);
1817 case kArm64I32x4ShrU: {
1818 __ Ushr(i.OutputSimd128Register().V4S(), i.InputSimd128Register(0).V4S(),
1819 i.InputInt5(1));
1820 break;
1821 }
1822 SIMD_BINOP_CASE(kArm64I32x4MinU, Umin, 4S);
1823 SIMD_BINOP_CASE(kArm64I32x4MaxU, Umax, 4S);
1824 SIMD_BINOP_CASE(kArm64I32x4GtU, Cmhi, 4S);
1825 SIMD_BINOP_CASE(kArm64I32x4GeU, Cmhs, 4S);
1826 case kArm64I16x8Splat: {
1827 __ Dup(i.OutputSimd128Register().V8H(), i.InputRegister32(0));
1828 break;
1829 }
1830 case kArm64I16x8ExtractLane: {
1831 __ Smov(i.OutputRegister32(), i.InputSimd128Register(0).V8H(),
1832 i.InputInt8(1));
1833 break;
1834 }
1835 case kArm64I16x8ReplaceLane: {
1836 VRegister dst = i.OutputSimd128Register().V8H(),
1837 src1 = i.InputSimd128Register(0).V8H();
1838 if (!dst.is(src1)) {
1839 __ Mov(dst, src1);
1840 }
1841 __ Mov(dst, i.InputInt8(1), i.InputRegister32(2));
1842 break;
1843 }
1844 SIMD_WIDENING_UNOP_CASE(kArm64I16x8SConvertI8x16Low, Sxtl, 8H, 8B);
1845 SIMD_WIDENING_UNOP_CASE(kArm64I16x8SConvertI8x16High, Sxtl2, 8H, 16B);
1846 SIMD_UNOP_CASE(kArm64I16x8Neg, Neg, 8H);
1847 case kArm64I16x8Shl: {
1848 __ Shl(i.OutputSimd128Register().V8H(), i.InputSimd128Register(0).V8H(),
1849 i.InputInt5(1));
1850 break;
1851 }
1852 case kArm64I16x8ShrS: {
1853 __ Sshr(i.OutputSimd128Register().V8H(), i.InputSimd128Register(0).V8H(),
1854 i.InputInt5(1));
1855 break;
1856 }
1857 case kArm64I16x8SConvertI32x4: {
1858 VRegister dst = i.OutputSimd128Register(),
1859 src0 = i.InputSimd128Register(0),
1860 src1 = i.InputSimd128Register(1);
1861 UseScratchRegisterScope scope(tasm());
1862 VRegister temp = scope.AcquireV(kFormat4S);
1863 if (dst.is(src1)) {
1864 __ Mov(temp, src1.V4S());
1865 src1 = temp;
1866 }
1867 __ Sqxtn(dst.V4H(), src0.V4S());
1868 __ Sqxtn2(dst.V8H(), src1.V4S());
1869 break;
1870 }
1871 SIMD_BINOP_CASE(kArm64I16x8Add, Add, 8H);
1872 SIMD_BINOP_CASE(kArm64I16x8AddSaturateS, Sqadd, 8H);
1873 SIMD_BINOP_CASE(kArm64I16x8AddHoriz, Addp, 8H);
1874 SIMD_BINOP_CASE(kArm64I16x8Sub, Sub, 8H);
1875 SIMD_BINOP_CASE(kArm64I16x8SubSaturateS, Sqsub, 8H);
1876 SIMD_BINOP_CASE(kArm64I16x8Mul, Mul, 8H);
1877 SIMD_BINOP_CASE(kArm64I16x8MinS, Smin, 8H);
1878 SIMD_BINOP_CASE(kArm64I16x8MaxS, Smax, 8H);
1879 SIMD_BINOP_CASE(kArm64I16x8Eq, Cmeq, 8H);
1880 case kArm64I16x8Ne: {
1881 VRegister dst = i.OutputSimd128Register().V8H();
1882 __ Cmeq(dst, i.InputSimd128Register(0).V8H(),
1883 i.InputSimd128Register(1).V8H());
1884 __ Mvn(dst, dst);
1885 break;
1886 }
1887 SIMD_BINOP_CASE(kArm64I16x8GtS, Cmgt, 8H);
1888 SIMD_BINOP_CASE(kArm64I16x8GeS, Cmge, 8H);
1889 case kArm64I16x8UConvertI8x16Low: {
1890 __ Uxtl(i.OutputSimd128Register().V8H(), i.InputSimd128Register(0).V8B());
1891 break;
1892 }
1893 case kArm64I16x8UConvertI8x16High: {
1894 __ Uxtl2(i.OutputSimd128Register().V8H(),
1895 i.InputSimd128Register(0).V16B());
1896 break;
1897 }
1898 case kArm64I16x8ShrU: {
1899 __ Ushr(i.OutputSimd128Register().V8H(), i.InputSimd128Register(0).V8H(),
1900 i.InputInt5(1));
1901 break;
1902 }
1903 case kArm64I16x8UConvertI32x4: {
1904 VRegister dst = i.OutputSimd128Register(),
1905 src0 = i.InputSimd128Register(0),
1906 src1 = i.InputSimd128Register(1);
1907 UseScratchRegisterScope scope(tasm());
1908 VRegister temp = scope.AcquireV(kFormat4S);
1909 if (dst.is(src1)) {
1910 __ Mov(temp, src1.V4S());
1911 src1 = temp;
1912 }
1913 __ Uqxtn(dst.V4H(), src0.V4S());
1914 __ Uqxtn2(dst.V8H(), src1.V4S());
1915 break;
1916 }
1917 SIMD_BINOP_CASE(kArm64I16x8AddSaturateU, Uqadd, 8H);
1918 SIMD_BINOP_CASE(kArm64I16x8SubSaturateU, Uqsub, 8H);
1919 SIMD_BINOP_CASE(kArm64I16x8MinU, Umin, 8H);
1920 SIMD_BINOP_CASE(kArm64I16x8MaxU, Umax, 8H);
1921 SIMD_BINOP_CASE(kArm64I16x8GtU, Cmhi, 8H);
1922 SIMD_BINOP_CASE(kArm64I16x8GeU, Cmhs, 8H);
1923 case kArm64I8x16Splat: {
1924 __ Dup(i.OutputSimd128Register().V16B(), i.InputRegister32(0));
1925 break;
1926 }
1927 case kArm64I8x16ExtractLane: {
1928 __ Smov(i.OutputRegister32(), i.InputSimd128Register(0).V16B(),
1929 i.InputInt8(1));
1930 break;
1931 }
1932 case kArm64I8x16ReplaceLane: {
1933 VRegister dst = i.OutputSimd128Register().V16B(),
1934 src1 = i.InputSimd128Register(0).V16B();
1935 if (!dst.is(src1)) {
1936 __ Mov(dst, src1);
1937 }
1938 __ Mov(dst, i.InputInt8(1), i.InputRegister32(2));
1939 break;
1940 }
1941 SIMD_UNOP_CASE(kArm64I8x16Neg, Neg, 16B);
1942 case kArm64I8x16Shl: {
1943 __ Shl(i.OutputSimd128Register().V16B(), i.InputSimd128Register(0).V16B(),
1944 i.InputInt5(1));
1945 break;
1946 }
1947 case kArm64I8x16ShrS: {
1948 __ Sshr(i.OutputSimd128Register().V16B(),
1949 i.InputSimd128Register(0).V16B(), i.InputInt5(1));
1950 break;
1951 }
1952 case kArm64I8x16SConvertI16x8: {
1953 VRegister dst = i.OutputSimd128Register(),
1954 src0 = i.InputSimd128Register(0),
1955 src1 = i.InputSimd128Register(1);
1956 UseScratchRegisterScope scope(tasm());
1957 VRegister temp = scope.AcquireV(kFormat8H);
1958 if (dst.is(src1)) {
1959 __ Mov(temp, src1.V8H());
1960 src1 = temp;
1961 }
1962 __ Sqxtn(dst.V8B(), src0.V8H());
1963 __ Sqxtn2(dst.V16B(), src1.V8H());
1964 break;
1965 }
1966 SIMD_BINOP_CASE(kArm64I8x16Add, Add, 16B);
1967 SIMD_BINOP_CASE(kArm64I8x16AddSaturateS, Sqadd, 16B);
1968 SIMD_BINOP_CASE(kArm64I8x16Sub, Sub, 16B);
1969 SIMD_BINOP_CASE(kArm64I8x16SubSaturateS, Sqsub, 16B);
1970 SIMD_BINOP_CASE(kArm64I8x16Mul, Mul, 16B);
1971 SIMD_BINOP_CASE(kArm64I8x16MinS, Smin, 16B);
1972 SIMD_BINOP_CASE(kArm64I8x16MaxS, Smax, 16B);
1973 SIMD_BINOP_CASE(kArm64I8x16Eq, Cmeq, 16B);
1974 case kArm64I8x16Ne: {
1975 VRegister dst = i.OutputSimd128Register().V16B();
1976 __ Cmeq(dst, i.InputSimd128Register(0).V16B(),
1977 i.InputSimd128Register(1).V16B());
1978 __ Mvn(dst, dst);
1979 break;
1980 }
1981 SIMD_BINOP_CASE(kArm64I8x16GtS, Cmgt, 16B);
1982 SIMD_BINOP_CASE(kArm64I8x16GeS, Cmge, 16B);
1983 case kArm64I8x16ShrU: {
1984 __ Ushr(i.OutputSimd128Register().V16B(),
1985 i.InputSimd128Register(0).V16B(), i.InputInt5(1));
1986 break;
1987 }
1988 case kArm64I8x16UConvertI16x8: {
1989 VRegister dst = i.OutputSimd128Register(),
1990 src0 = i.InputSimd128Register(0),
1991 src1 = i.InputSimd128Register(1);
1992 UseScratchRegisterScope scope(tasm());
1993 VRegister temp = scope.AcquireV(kFormat8H);
1994 if (dst.is(src1)) {
1995 __ Mov(temp, src1.V8H());
1996 src1 = temp;
1997 }
1998 __ Uqxtn(dst.V8B(), src0.V8H());
1999 __ Uqxtn2(dst.V16B(), src1.V8H());
2000 break;
2001 }
2002 SIMD_BINOP_CASE(kArm64I8x16AddSaturateU, Uqadd, 16B);
2003 SIMD_BINOP_CASE(kArm64I8x16SubSaturateU, Uqsub, 16B);
2004 SIMD_BINOP_CASE(kArm64I8x16MinU, Umin, 16B);
2005 SIMD_BINOP_CASE(kArm64I8x16MaxU, Umax, 16B);
2006 SIMD_BINOP_CASE(kArm64I8x16GtU, Cmhi, 16B);
2007 SIMD_BINOP_CASE(kArm64I8x16GeU, Cmhs, 16B);
2008 case kArm64S128Zero: {
2009 __ Movi(i.OutputSimd128Register().V16B(), 0);
2010 break;
2011 }
2012 SIMD_BINOP_CASE(kArm64S128And, And, 16B);
2013 SIMD_BINOP_CASE(kArm64S128Or, Orr, 16B);
2014 SIMD_BINOP_CASE(kArm64S128Xor, Eor, 16B);
2015 SIMD_UNOP_CASE(kArm64S128Not, Mvn, 16B);
2016 case kArm64S128Dup: {
2017 VRegister dst = i.OutputSimd128Register(),
2018 src = i.InputSimd128Register(0);
2019 int lanes = i.InputInt32(1);
2020 int index = i.InputInt32(2);
2021 switch (lanes) {
2022 case 4:
2023 __ Dup(dst.V4S(), src.V4S(), index);
2024 break;
2025 case 8:
2026 __ Dup(dst.V8H(), src.V8H(), index);
2027 break;
2028 case 16:
2029 __ Dup(dst.V16B(), src.V16B(), index);
2030 break;
2031 default:
2032 UNREACHABLE();
2033 break;
2034 }
2035 break;
2036 }
2037 case kArm64S128Select: {
2038 VRegister dst = i.OutputSimd128Register().V16B();
2039 DCHECK(dst.is(i.InputSimd128Register(0).V16B()));
2040 __ Bsl(dst, i.InputSimd128Register(1).V16B(),
2041 i.InputSimd128Register(2).V16B());
2042 break;
2043 }
2044 case kArm64S32x4Shuffle: {
2045 Simd128Register dst = i.OutputSimd128Register().V4S(),
2046 src0 = i.InputSimd128Register(0).V4S(),
2047 src1 = i.InputSimd128Register(1).V4S();
2048 // Check for in-place shuffles.
2049 // If dst == src0 == src1, then the shuffle is unary and we only use src0.
2050 UseScratchRegisterScope scope(tasm());
2051 VRegister temp = scope.AcquireV(kFormat4S);
2052 if (dst.is(src0)) {
2053 __ Mov(temp, src0);
2054 src0 = temp;
2055 } else if (dst.is(src1)) {
2056 __ Mov(temp, src1);
2057 src1 = temp;
2058 }
2059 // Perform shuffle as a vmov per lane.
2060 int32_t shuffle = i.InputInt32(2);
2061 for (int i = 0; i < 4; i++) {
2062 VRegister src = src0;
2063 int lane = shuffle & 0x7;
2064 if (lane >= 4) {
2065 src = src1;
2066 lane &= 0x3;
2067 }
2068 __ Mov(dst, i, src, lane);
2069 shuffle >>= 8;
2070 }
2071 break;
2072 }
2073 SIMD_BINOP_CASE(kArm64S32x4ZipLeft, Zip1, 4S);
2074 SIMD_BINOP_CASE(kArm64S32x4ZipRight, Zip2, 4S);
2075 SIMD_BINOP_CASE(kArm64S32x4UnzipLeft, Uzp1, 4S);
2076 SIMD_BINOP_CASE(kArm64S32x4UnzipRight, Uzp2, 4S);
2077 SIMD_BINOP_CASE(kArm64S32x4TransposeLeft, Trn1, 4S);
2078 SIMD_BINOP_CASE(kArm64S32x4TransposeRight, Trn2, 4S);
2079 SIMD_BINOP_CASE(kArm64S16x8ZipLeft, Zip1, 8H);
2080 SIMD_BINOP_CASE(kArm64S16x8ZipRight, Zip2, 8H);
2081 SIMD_BINOP_CASE(kArm64S16x8UnzipLeft, Uzp1, 8H);
2082 SIMD_BINOP_CASE(kArm64S16x8UnzipRight, Uzp2, 8H);
2083 SIMD_BINOP_CASE(kArm64S16x8TransposeLeft, Trn1, 8H);
2084 SIMD_BINOP_CASE(kArm64S16x8TransposeRight, Trn2, 8H);
2085 SIMD_BINOP_CASE(kArm64S8x16ZipLeft, Zip1, 16B);
2086 SIMD_BINOP_CASE(kArm64S8x16ZipRight, Zip2, 16B);
2087 SIMD_BINOP_CASE(kArm64S8x16UnzipLeft, Uzp1, 16B);
2088 SIMD_BINOP_CASE(kArm64S8x16UnzipRight, Uzp2, 16B);
2089 SIMD_BINOP_CASE(kArm64S8x16TransposeLeft, Trn1, 16B);
2090 SIMD_BINOP_CASE(kArm64S8x16TransposeRight, Trn2, 16B);
2091 case kArm64S8x16Concat: {
2092 __ Ext(i.OutputSimd128Register().V16B(), i.InputSimd128Register(0).V16B(),
2093 i.InputSimd128Register(1).V16B(), i.InputInt4(2));
2094 break;
2095 }
2096 case kArm64S8x16Shuffle: {
2097 Simd128Register dst = i.OutputSimd128Register().V16B(),
2098 src0 = i.InputSimd128Register(0).V16B(),
2099 src1 = i.InputSimd128Register(1).V16B();
2100 // Unary shuffle table is in src0, binary shuffle table is in src0, src1,
2101 // which must be consecutive.
2102 int64_t mask = 0;
2103 if (src0.is(src1)) {
2104 mask = 0x0F0F0F0F;
2105 } else {
2106 mask = 0x1F1F1F1F;
2107 DCHECK(AreConsecutive(src0, src1));
2108 }
2109 int64_t imm1 =
2110 (i.InputInt32(2) & mask) | ((i.InputInt32(3) & mask) << 32);
2111 int64_t imm2 =
2112 (i.InputInt32(4) & mask) | ((i.InputInt32(5) & mask) << 32);
2113 UseScratchRegisterScope scope(tasm());
2114 VRegister temp = scope.AcquireV(kFormat16B);
2115 __ Movi(temp, imm2, imm1);
2116
2117 if (src0.is(src1)) {
2118 __ Tbl(dst, src0, temp.V16B());
2119 } else {
2120 __ Tbl(dst, src0, src1, temp.V16B());
2121 }
2122 break;
2123 }
2124 SIMD_UNOP_CASE(kArm64S32x2Reverse, Rev64, 4S);
2125 SIMD_UNOP_CASE(kArm64S16x4Reverse, Rev64, 8H);
2126 SIMD_UNOP_CASE(kArm64S16x2Reverse, Rev32, 8H);
2127 SIMD_UNOP_CASE(kArm64S8x8Reverse, Rev64, 16B);
2128 SIMD_UNOP_CASE(kArm64S8x4Reverse, Rev32, 16B);
2129 SIMD_UNOP_CASE(kArm64S8x2Reverse, Rev16, 16B);
2130
2131 #define SIMD_REDUCE_OP_CASE(Op, Instr, format, FORMAT) \
2132 case Op: { \
2133 UseScratchRegisterScope scope(tasm()); \
2134 VRegister temp = scope.AcquireV(format); \
2135 __ Instr(temp, i.InputSimd128Register(0).V##FORMAT()); \
2136 __ Umov(i.OutputRegister32(), temp, 0); \
2137 break; \
2138 }
2139 SIMD_REDUCE_OP_CASE(kArm64S1x4AnyTrue, Umaxv, kFormatS, 4S);
2140 SIMD_REDUCE_OP_CASE(kArm64S1x4AllTrue, Uminv, kFormatS, 4S);
2141 SIMD_REDUCE_OP_CASE(kArm64S1x8AnyTrue, Umaxv, kFormatH, 8H);
2142 SIMD_REDUCE_OP_CASE(kArm64S1x8AllTrue, Uminv, kFormatH, 8H);
2143 SIMD_REDUCE_OP_CASE(kArm64S1x16AnyTrue, Umaxv, kFormatB, 16B);
2144 SIMD_REDUCE_OP_CASE(kArm64S1x16AllTrue, Uminv, kFormatB, 16B);
2145 }
2146 return kSuccess;
2147 } // NOLINT(readability/fn_size)
2148
2149 #undef SIMD_UNOP_CASE
2150 #undef SIMD_WIDENING_UNOP_CASE
2151 #undef SIMD_BINOP_CASE
2152 #undef SIMD_REDUCE_OP_CASE
2153
2154 // Assemble branches after this instruction.
AssembleArchBranch(Instruction * instr,BranchInfo * branch)2155 void CodeGenerator::AssembleArchBranch(Instruction* instr, BranchInfo* branch) {
2156 Arm64OperandConverter i(this, instr);
2157 Label* tlabel = branch->true_label;
2158 Label* flabel = branch->false_label;
2159 FlagsCondition condition = branch->condition;
2160 ArchOpcode opcode = instr->arch_opcode();
2161
2162 if (opcode == kArm64CompareAndBranch32) {
2163 DCHECK(FlagsModeField::decode(instr->opcode()) != kFlags_branch_and_poison);
2164 switch (condition) {
2165 case kEqual:
2166 __ Cbz(i.InputRegister32(0), tlabel);
2167 break;
2168 case kNotEqual:
2169 __ Cbnz(i.InputRegister32(0), tlabel);
2170 break;
2171 default:
2172 UNREACHABLE();
2173 }
2174 } else if (opcode == kArm64CompareAndBranch) {
2175 DCHECK(FlagsModeField::decode(instr->opcode()) != kFlags_branch_and_poison);
2176 switch (condition) {
2177 case kEqual:
2178 __ Cbz(i.InputRegister64(0), tlabel);
2179 break;
2180 case kNotEqual:
2181 __ Cbnz(i.InputRegister64(0), tlabel);
2182 break;
2183 default:
2184 UNREACHABLE();
2185 }
2186 } else if (opcode == kArm64TestAndBranch32) {
2187 DCHECK(FlagsModeField::decode(instr->opcode()) != kFlags_branch_and_poison);
2188 switch (condition) {
2189 case kEqual:
2190 __ Tbz(i.InputRegister32(0), i.InputInt5(1), tlabel);
2191 break;
2192 case kNotEqual:
2193 __ Tbnz(i.InputRegister32(0), i.InputInt5(1), tlabel);
2194 break;
2195 default:
2196 UNREACHABLE();
2197 }
2198 } else if (opcode == kArm64TestAndBranch) {
2199 DCHECK(FlagsModeField::decode(instr->opcode()) != kFlags_branch_and_poison);
2200 switch (condition) {
2201 case kEqual:
2202 __ Tbz(i.InputRegister64(0), i.InputInt6(1), tlabel);
2203 break;
2204 case kNotEqual:
2205 __ Tbnz(i.InputRegister64(0), i.InputInt6(1), tlabel);
2206 break;
2207 default:
2208 UNREACHABLE();
2209 }
2210 } else {
2211 Condition cc = FlagsConditionToCondition(condition);
2212 __ B(cc, tlabel);
2213 }
2214 if (!branch->fallthru) __ B(flabel); // no fallthru to flabel.
2215 }
2216
AssembleBranchPoisoning(FlagsCondition condition,Instruction * instr)2217 void CodeGenerator::AssembleBranchPoisoning(FlagsCondition condition,
2218 Instruction* instr) {
2219 // TODO(jarin) Handle float comparisons (kUnordered[Not]Equal).
2220 if (condition == kUnorderedEqual || condition == kUnorderedNotEqual) {
2221 return;
2222 }
2223
2224 condition = NegateFlagsCondition(condition);
2225 __ CmovX(kSpeculationPoisonRegister, xzr,
2226 FlagsConditionToCondition(condition));
2227 __ Csdb();
2228 }
2229
AssembleArchDeoptBranch(Instruction * instr,BranchInfo * branch)2230 void CodeGenerator::AssembleArchDeoptBranch(Instruction* instr,
2231 BranchInfo* branch) {
2232 AssembleArchBranch(instr, branch);
2233 }
2234
AssembleArchJump(RpoNumber target)2235 void CodeGenerator::AssembleArchJump(RpoNumber target) {
2236 if (!IsNextInAssemblyOrder(target)) __ B(GetLabel(target));
2237 }
2238
AssembleArchTrap(Instruction * instr,FlagsCondition condition)2239 void CodeGenerator::AssembleArchTrap(Instruction* instr,
2240 FlagsCondition condition) {
2241 class OutOfLineTrap final : public OutOfLineCode {
2242 public:
2243 OutOfLineTrap(CodeGenerator* gen, Instruction* instr)
2244 : OutOfLineCode(gen), instr_(instr), gen_(gen) {}
2245 void Generate() final {
2246 Arm64OperandConverter i(gen_, instr_);
2247 TrapId trap_id =
2248 static_cast<TrapId>(i.InputInt32(instr_->InputCount() - 1));
2249 GenerateCallToTrap(trap_id);
2250 }
2251
2252 private:
2253 void GenerateCallToTrap(TrapId trap_id) {
2254 if (trap_id == TrapId::kInvalid) {
2255 // We cannot test calls to the runtime in cctest/test-run-wasm.
2256 // Therefore we emit a call to C here instead of a call to the runtime.
2257 __ CallCFunction(
2258 ExternalReference::wasm_call_trap_callback_for_testing(), 0);
2259 __ LeaveFrame(StackFrame::WASM_COMPILED);
2260 auto call_descriptor = gen_->linkage()->GetIncomingDescriptor();
2261 int pop_count =
2262 static_cast<int>(call_descriptor->StackParameterCount());
2263 pop_count += (pop_count & 1); // align
2264 __ Drop(pop_count);
2265 __ Ret();
2266 } else {
2267 gen_->AssembleSourcePosition(instr_);
2268 // A direct call to a wasm runtime stub defined in this module.
2269 // Just encode the stub index. This will be patched at relocation.
2270 __ Call(static_cast<Address>(trap_id), RelocInfo::WASM_STUB_CALL);
2271 ReferenceMap* reference_map =
2272 new (gen_->zone()) ReferenceMap(gen_->zone());
2273 gen_->RecordSafepoint(reference_map, Safepoint::kSimple, 0,
2274 Safepoint::kNoLazyDeopt);
2275 if (FLAG_debug_code) {
2276 // The trap code should never return.
2277 __ Brk(0);
2278 }
2279 }
2280 }
2281 Instruction* instr_;
2282 CodeGenerator* gen_;
2283 };
2284 auto ool = new (zone()) OutOfLineTrap(this, instr);
2285 Label* tlabel = ool->entry();
2286 Condition cc = FlagsConditionToCondition(condition);
2287 __ B(cc, tlabel);
2288 }
2289
2290 // Assemble boolean materializations after this instruction.
AssembleArchBoolean(Instruction * instr,FlagsCondition condition)2291 void CodeGenerator::AssembleArchBoolean(Instruction* instr,
2292 FlagsCondition condition) {
2293 Arm64OperandConverter i(this, instr);
2294
2295 // Materialize a full 64-bit 1 or 0 value. The result register is always the
2296 // last output of the instruction.
2297 DCHECK_NE(0u, instr->OutputCount());
2298 Register reg = i.OutputRegister(instr->OutputCount() - 1);
2299 Condition cc = FlagsConditionToCondition(condition);
2300 __ Cset(reg, cc);
2301 }
2302
AssembleArchBinarySearchSwitch(Instruction * instr)2303 void CodeGenerator::AssembleArchBinarySearchSwitch(Instruction* instr) {
2304 Arm64OperandConverter i(this, instr);
2305 Register input = i.InputRegister32(0);
2306 std::vector<std::pair<int32_t, Label*>> cases;
2307 for (size_t index = 2; index < instr->InputCount(); index += 2) {
2308 cases.push_back({i.InputInt32(index + 0), GetLabel(i.InputRpo(index + 1))});
2309 }
2310 AssembleArchBinarySearchSwitchRange(input, i.InputRpo(1), cases.data(),
2311 cases.data() + cases.size());
2312 }
2313
AssembleArchLookupSwitch(Instruction * instr)2314 void CodeGenerator::AssembleArchLookupSwitch(Instruction* instr) {
2315 Arm64OperandConverter i(this, instr);
2316 Register input = i.InputRegister32(0);
2317 for (size_t index = 2; index < instr->InputCount(); index += 2) {
2318 __ Cmp(input, i.InputInt32(index + 0));
2319 __ B(eq, GetLabel(i.InputRpo(index + 1)));
2320 }
2321 AssembleArchJump(i.InputRpo(1));
2322 }
2323
2324
AssembleArchTableSwitch(Instruction * instr)2325 void CodeGenerator::AssembleArchTableSwitch(Instruction* instr) {
2326 Arm64OperandConverter i(this, instr);
2327 UseScratchRegisterScope scope(tasm());
2328 Register input = i.InputRegister32(0);
2329 Register temp = scope.AcquireX();
2330 size_t const case_count = instr->InputCount() - 2;
2331 Label table;
2332 __ Cmp(input, case_count);
2333 __ B(hs, GetLabel(i.InputRpo(1)));
2334 __ Adr(temp, &table);
2335 __ Add(temp, temp, Operand(input, UXTW, 2));
2336 __ Br(temp);
2337 __ StartBlockPools();
2338 __ Bind(&table);
2339 for (size_t index = 0; index < case_count; ++index) {
2340 __ B(GetLabel(i.InputRpo(index + 2)));
2341 }
2342 __ EndBlockPools();
2343 }
2344
FinishFrame(Frame * frame)2345 void CodeGenerator::FinishFrame(Frame* frame) {
2346 frame->AlignFrame(16);
2347 auto call_descriptor = linkage()->GetIncomingDescriptor();
2348
2349 // Save FP registers.
2350 CPURegList saves_fp = CPURegList(CPURegister::kVRegister, kDRegSizeInBits,
2351 call_descriptor->CalleeSavedFPRegisters());
2352 int saved_count = saves_fp.Count();
2353 if (saved_count != 0) {
2354 DCHECK(saves_fp.list() == CPURegList::GetCalleeSavedV().list());
2355 DCHECK_EQ(saved_count % 2, 0);
2356 frame->AllocateSavedCalleeRegisterSlots(saved_count *
2357 (kDoubleSize / kPointerSize));
2358 }
2359
2360 CPURegList saves = CPURegList(CPURegister::kRegister, kXRegSizeInBits,
2361 call_descriptor->CalleeSavedRegisters());
2362 saved_count = saves.Count();
2363 if (saved_count != 0) {
2364 DCHECK_EQ(saved_count % 2, 0);
2365 frame->AllocateSavedCalleeRegisterSlots(saved_count);
2366 }
2367 }
2368
AssembleConstructFrame()2369 void CodeGenerator::AssembleConstructFrame() {
2370 auto call_descriptor = linkage()->GetIncomingDescriptor();
2371 __ AssertSpAligned();
2372
2373 // The frame has been previously padded in CodeGenerator::FinishFrame().
2374 DCHECK_EQ(frame()->GetTotalFrameSlotCount() % 2, 0);
2375 int shrink_slots = frame()->GetTotalFrameSlotCount() -
2376 call_descriptor->CalculateFixedFrameSize();
2377
2378 CPURegList saves = CPURegList(CPURegister::kRegister, kXRegSizeInBits,
2379 call_descriptor->CalleeSavedRegisters());
2380 CPURegList saves_fp = CPURegList(CPURegister::kVRegister, kDRegSizeInBits,
2381 call_descriptor->CalleeSavedFPRegisters());
2382 // The number of slots for returns has to be even to ensure the correct stack
2383 // alignment.
2384 const int returns = RoundUp(frame()->GetReturnSlotCount(), 2);
2385
2386 if (frame_access_state()->has_frame()) {
2387 // Link the frame
2388 if (call_descriptor->IsJSFunctionCall()) {
2389 __ Prologue();
2390 } else {
2391 __ Push(lr, fp);
2392 __ Mov(fp, sp);
2393 }
2394 unwinding_info_writer_.MarkFrameConstructed(__ pc_offset());
2395
2396 // Create OSR entry if applicable
2397 if (info()->is_osr()) {
2398 // TurboFan OSR-compiled functions cannot be entered directly.
2399 __ Abort(AbortReason::kShouldNotDirectlyEnterOsrFunction);
2400
2401 // Unoptimized code jumps directly to this entrypoint while the
2402 // unoptimized frame is still on the stack. Optimized code uses OSR values
2403 // directly from the unoptimized frame. Thus, all that needs to be done is
2404 // to allocate the remaining stack slots.
2405 if (FLAG_code_comments) __ RecordComment("-- OSR entrypoint --");
2406 osr_pc_offset_ = __ pc_offset();
2407 shrink_slots -= osr_helper()->UnoptimizedFrameSlots();
2408 ResetSpeculationPoison();
2409 }
2410
2411 if (info()->IsWasm() && shrink_slots > 128) {
2412 // For WebAssembly functions with big frames we have to do the stack
2413 // overflow check before we construct the frame. Otherwise we may not
2414 // have enough space on the stack to call the runtime for the stack
2415 // overflow.
2416 Label done;
2417 // If the frame is bigger than the stack, we throw the stack overflow
2418 // exception unconditionally. Thereby we can avoid the integer overflow
2419 // check in the condition code.
2420 if (shrink_slots * kPointerSize < FLAG_stack_size * 1024) {
2421 UseScratchRegisterScope scope(tasm());
2422 Register scratch = scope.AcquireX();
2423 __ Ldr(scratch, FieldMemOperand(
2424 kWasmInstanceRegister,
2425 WasmInstanceObject::kRealStackLimitAddressOffset));
2426 __ Ldr(scratch, MemOperand(scratch));
2427 __ Add(scratch, scratch, shrink_slots * kPointerSize);
2428 __ Cmp(sp, scratch);
2429 __ B(hs, &done);
2430 }
2431
2432 {
2433 // Finish the frame that hasn't been fully built yet.
2434 UseScratchRegisterScope temps(tasm());
2435 __ Claim(2); // Claim extra slots for marker + instance.
2436 Register scratch = temps.AcquireX();
2437 __ Mov(scratch,
2438 StackFrame::TypeToMarker(info()->GetOutputStackFrameType()));
2439 __ Str(scratch, MemOperand(fp, TypedFrameConstants::kFrameTypeOffset));
2440 __ Str(kWasmInstanceRegister,
2441 MemOperand(fp, WasmCompiledFrameConstants::kWasmInstanceOffset));
2442 }
2443 __ Ldr(x2, FieldMemOperand(kWasmInstanceRegister,
2444 WasmInstanceObject::kCEntryStubOffset));
2445 __ Mov(cp, Smi::kZero);
2446 __ CallRuntimeWithCEntry(Runtime::kThrowWasmStackOverflow, x2);
2447 // We come from WebAssembly, there are no references for the GC.
2448 ReferenceMap* reference_map = new (zone()) ReferenceMap(zone());
2449 RecordSafepoint(reference_map, Safepoint::kSimple, 0,
2450 Safepoint::kNoLazyDeopt);
2451 if (FLAG_debug_code) {
2452 __ Brk(0);
2453 }
2454 __ Bind(&done);
2455 }
2456
2457 // Skip callee-saved slots, which are pushed below.
2458 shrink_slots -= saves.Count();
2459 shrink_slots -= saves_fp.Count();
2460 shrink_slots -= returns;
2461
2462 // Build remainder of frame, including accounting for and filling-in
2463 // frame-specific header information, i.e. claiming the extra slot that
2464 // other platforms explicitly push for STUB (code object) frames and frames
2465 // recording their argument count.
2466 switch (call_descriptor->kind()) {
2467 case CallDescriptor::kCallJSFunction:
2468 if (call_descriptor->PushArgumentCount()) {
2469 __ Claim(shrink_slots + 1); // Claim extra slot for argc.
2470 __ Str(kJavaScriptCallArgCountRegister,
2471 MemOperand(fp, OptimizedBuiltinFrameConstants::kArgCOffset));
2472 } else {
2473 __ Claim(shrink_slots);
2474 }
2475 break;
2476 case CallDescriptor::kCallCodeObject: {
2477 UseScratchRegisterScope temps(tasm());
2478 __ Claim(shrink_slots + 1); // Claim extra slot for frame type marker.
2479 Register scratch = temps.AcquireX();
2480 __ Mov(scratch,
2481 StackFrame::TypeToMarker(info()->GetOutputStackFrameType()));
2482 __ Str(scratch, MemOperand(fp, TypedFrameConstants::kFrameTypeOffset));
2483 } break;
2484 case CallDescriptor::kCallWasmFunction: {
2485 UseScratchRegisterScope temps(tasm());
2486 __ Claim(shrink_slots + 2); // Claim extra slots for marker + instance.
2487 Register scratch = temps.AcquireX();
2488 __ Mov(scratch,
2489 StackFrame::TypeToMarker(info()->GetOutputStackFrameType()));
2490 __ Str(scratch, MemOperand(fp, TypedFrameConstants::kFrameTypeOffset));
2491 __ Str(kWasmInstanceRegister,
2492 MemOperand(fp, WasmCompiledFrameConstants::kWasmInstanceOffset));
2493 } break;
2494 case CallDescriptor::kCallAddress:
2495 __ Claim(shrink_slots);
2496 break;
2497 default:
2498 UNREACHABLE();
2499 }
2500 }
2501
2502 // Save FP registers.
2503 DCHECK_IMPLIES(saves_fp.Count() != 0,
2504 saves_fp.list() == CPURegList::GetCalleeSavedV().list());
2505 __ PushCPURegList(saves_fp);
2506
2507 // Save registers.
2508 // TODO(palfia): TF save list is not in sync with
2509 // CPURegList::GetCalleeSaved(): x30 is missing.
2510 // DCHECK(saves.list() == CPURegList::GetCalleeSaved().list());
2511 __ PushCPURegList(saves);
2512
2513 if (returns != 0) {
2514 __ Claim(returns);
2515 }
2516 }
2517
AssembleReturn(InstructionOperand * pop)2518 void CodeGenerator::AssembleReturn(InstructionOperand* pop) {
2519 auto call_descriptor = linkage()->GetIncomingDescriptor();
2520
2521 const int returns = RoundUp(frame()->GetReturnSlotCount(), 2);
2522
2523 if (returns != 0) {
2524 __ Drop(returns);
2525 }
2526
2527 // Restore registers.
2528 CPURegList saves = CPURegList(CPURegister::kRegister, kXRegSizeInBits,
2529 call_descriptor->CalleeSavedRegisters());
2530 __ PopCPURegList(saves);
2531
2532 // Restore fp registers.
2533 CPURegList saves_fp = CPURegList(CPURegister::kVRegister, kDRegSizeInBits,
2534 call_descriptor->CalleeSavedFPRegisters());
2535 __ PopCPURegList(saves_fp);
2536
2537 unwinding_info_writer_.MarkBlockWillExit();
2538
2539 Arm64OperandConverter g(this, nullptr);
2540 int pop_count = static_cast<int>(call_descriptor->StackParameterCount());
2541 if (call_descriptor->IsCFunctionCall()) {
2542 AssembleDeconstructFrame();
2543 } else if (frame_access_state()->has_frame()) {
2544 // Canonicalize JSFunction return sites for now unless they have an variable
2545 // number of stack slot pops.
2546 if (pop->IsImmediate() && g.ToConstant(pop).ToInt32() == 0) {
2547 if (return_label_.is_bound()) {
2548 __ B(&return_label_);
2549 return;
2550 } else {
2551 __ Bind(&return_label_);
2552 AssembleDeconstructFrame();
2553 }
2554 } else {
2555 AssembleDeconstructFrame();
2556 }
2557 }
2558
2559 if (pop->IsImmediate()) {
2560 pop_count += g.ToConstant(pop).ToInt32();
2561 __ DropArguments(pop_count);
2562 } else {
2563 Register pop_reg = g.ToRegister(pop);
2564 __ Add(pop_reg, pop_reg, pop_count);
2565 __ DropArguments(pop_reg);
2566 }
2567
2568 __ AssertSpAligned();
2569 __ Ret();
2570 }
2571
FinishCode()2572 void CodeGenerator::FinishCode() { __ CheckConstPool(true, false); }
2573
AssembleMove(InstructionOperand * source,InstructionOperand * destination)2574 void CodeGenerator::AssembleMove(InstructionOperand* source,
2575 InstructionOperand* destination) {
2576 Arm64OperandConverter g(this, nullptr);
2577 // Helper function to write the given constant to the dst register.
2578 auto MoveConstantToRegister = [&](Register dst, Constant src) {
2579 if (src.type() == Constant::kHeapObject) {
2580 Handle<HeapObject> src_object = src.ToHeapObject();
2581 Heap::RootListIndex index;
2582 if (IsMaterializableFromRoot(src_object, &index)) {
2583 __ LoadRoot(dst, index);
2584 } else {
2585 __ Mov(dst, src_object);
2586 }
2587 } else {
2588 __ Mov(dst, g.ToImmediate(source));
2589 }
2590 };
2591 switch (MoveType::InferMove(source, destination)) {
2592 case MoveType::kRegisterToRegister:
2593 if (source->IsRegister()) {
2594 __ Mov(g.ToRegister(destination), g.ToRegister(source));
2595 } else if (source->IsFloatRegister() || source->IsDoubleRegister()) {
2596 __ Mov(g.ToDoubleRegister(destination), g.ToDoubleRegister(source));
2597 } else {
2598 DCHECK(source->IsSimd128Register());
2599 __ Mov(g.ToDoubleRegister(destination).Q(),
2600 g.ToDoubleRegister(source).Q());
2601 }
2602 return;
2603 case MoveType::kRegisterToStack: {
2604 MemOperand dst = g.ToMemOperand(destination, tasm());
2605 if (source->IsRegister()) {
2606 __ Str(g.ToRegister(source), dst);
2607 } else {
2608 VRegister src = g.ToDoubleRegister(source);
2609 if (source->IsFloatRegister() || source->IsDoubleRegister()) {
2610 __ Str(src, dst);
2611 } else {
2612 DCHECK(source->IsSimd128Register());
2613 __ Str(src.Q(), dst);
2614 }
2615 }
2616 return;
2617 }
2618 case MoveType::kStackToRegister: {
2619 MemOperand src = g.ToMemOperand(source, tasm());
2620 if (destination->IsRegister()) {
2621 __ Ldr(g.ToRegister(destination), src);
2622 } else {
2623 VRegister dst = g.ToDoubleRegister(destination);
2624 if (destination->IsFloatRegister() || destination->IsDoubleRegister()) {
2625 __ Ldr(dst, src);
2626 } else {
2627 DCHECK(destination->IsSimd128Register());
2628 __ Ldr(dst.Q(), src);
2629 }
2630 }
2631 return;
2632 }
2633 case MoveType::kStackToStack: {
2634 MemOperand src = g.ToMemOperand(source, tasm());
2635 MemOperand dst = g.ToMemOperand(destination, tasm());
2636 if (source->IsSimd128StackSlot()) {
2637 UseScratchRegisterScope scope(tasm());
2638 VRegister temp = scope.AcquireQ();
2639 __ Ldr(temp, src);
2640 __ Str(temp, dst);
2641 } else {
2642 UseScratchRegisterScope scope(tasm());
2643 Register temp = scope.AcquireX();
2644 __ Ldr(temp, src);
2645 __ Str(temp, dst);
2646 }
2647 return;
2648 }
2649 case MoveType::kConstantToRegister: {
2650 Constant src = g.ToConstant(source);
2651 if (destination->IsRegister()) {
2652 MoveConstantToRegister(g.ToRegister(destination), src);
2653 } else {
2654 VRegister dst = g.ToDoubleRegister(destination);
2655 if (destination->IsFloatRegister()) {
2656 __ Fmov(dst.S(), src.ToFloat32());
2657 } else {
2658 DCHECK(destination->IsDoubleRegister());
2659 __ Fmov(dst, src.ToFloat64().value());
2660 }
2661 }
2662 return;
2663 }
2664 case MoveType::kConstantToStack: {
2665 Constant src = g.ToConstant(source);
2666 MemOperand dst = g.ToMemOperand(destination, tasm());
2667 if (destination->IsStackSlot()) {
2668 UseScratchRegisterScope scope(tasm());
2669 Register temp = scope.AcquireX();
2670 MoveConstantToRegister(temp, src);
2671 __ Str(temp, dst);
2672 } else if (destination->IsFloatStackSlot()) {
2673 if (bit_cast<int32_t>(src.ToFloat32()) == 0) {
2674 __ Str(wzr, dst);
2675 } else {
2676 UseScratchRegisterScope scope(tasm());
2677 VRegister temp = scope.AcquireS();
2678 __ Fmov(temp, src.ToFloat32());
2679 __ Str(temp, dst);
2680 }
2681 } else {
2682 DCHECK(destination->IsDoubleStackSlot());
2683 if (src.ToFloat64().AsUint64() == 0) {
2684 __ Str(xzr, dst);
2685 } else {
2686 UseScratchRegisterScope scope(tasm());
2687 VRegister temp = scope.AcquireD();
2688 __ Fmov(temp, src.ToFloat64().value());
2689 __ Str(temp, dst);
2690 }
2691 }
2692 return;
2693 }
2694 }
2695 UNREACHABLE();
2696 }
2697
2698
AssembleSwap(InstructionOperand * source,InstructionOperand * destination)2699 void CodeGenerator::AssembleSwap(InstructionOperand* source,
2700 InstructionOperand* destination) {
2701 Arm64OperandConverter g(this, nullptr);
2702 switch (MoveType::InferSwap(source, destination)) {
2703 case MoveType::kRegisterToRegister:
2704 if (source->IsRegister()) {
2705 __ Swap(g.ToRegister(source), g.ToRegister(destination));
2706 } else {
2707 VRegister src = g.ToDoubleRegister(source);
2708 VRegister dst = g.ToDoubleRegister(destination);
2709 if (source->IsFloatRegister() || source->IsDoubleRegister()) {
2710 __ Swap(src, dst);
2711 } else {
2712 DCHECK(source->IsSimd128Register());
2713 __ Swap(src.Q(), dst.Q());
2714 }
2715 }
2716 return;
2717 case MoveType::kRegisterToStack: {
2718 UseScratchRegisterScope scope(tasm());
2719 MemOperand dst = g.ToMemOperand(destination, tasm());
2720 if (source->IsRegister()) {
2721 Register temp = scope.AcquireX();
2722 Register src = g.ToRegister(source);
2723 __ Mov(temp, src);
2724 __ Ldr(src, dst);
2725 __ Str(temp, dst);
2726 } else {
2727 UseScratchRegisterScope scope(tasm());
2728 VRegister src = g.ToDoubleRegister(source);
2729 if (source->IsFloatRegister() || source->IsDoubleRegister()) {
2730 VRegister temp = scope.AcquireD();
2731 __ Mov(temp, src);
2732 __ Ldr(src, dst);
2733 __ Str(temp, dst);
2734 } else {
2735 DCHECK(source->IsSimd128Register());
2736 VRegister temp = scope.AcquireQ();
2737 __ Mov(temp, src.Q());
2738 __ Ldr(src.Q(), dst);
2739 __ Str(temp, dst);
2740 }
2741 }
2742 return;
2743 }
2744 case MoveType::kStackToStack: {
2745 UseScratchRegisterScope scope(tasm());
2746 MemOperand src = g.ToMemOperand(source, tasm());
2747 MemOperand dst = g.ToMemOperand(destination, tasm());
2748 VRegister temp_0 = scope.AcquireD();
2749 VRegister temp_1 = scope.AcquireD();
2750 if (source->IsSimd128StackSlot()) {
2751 __ Ldr(temp_0.Q(), src);
2752 __ Ldr(temp_1.Q(), dst);
2753 __ Str(temp_0.Q(), dst);
2754 __ Str(temp_1.Q(), src);
2755 } else {
2756 __ Ldr(temp_0, src);
2757 __ Ldr(temp_1, dst);
2758 __ Str(temp_0, dst);
2759 __ Str(temp_1, src);
2760 }
2761 return;
2762 }
2763 default:
2764 UNREACHABLE();
2765 break;
2766 }
2767 }
2768
2769
AssembleJumpTable(Label ** targets,size_t target_count)2770 void CodeGenerator::AssembleJumpTable(Label** targets, size_t target_count) {
2771 // On 64-bit ARM we emit the jump tables inline.
2772 UNREACHABLE();
2773 }
2774
2775 #undef __
2776
2777 } // namespace compiler
2778 } // namespace internal
2779 } // namespace v8
2780