1 // Copyright 2014 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "src/codegen/arm64/assembler-arm64-inl.h"
6 #include "src/codegen/arm64/macro-assembler-arm64-inl.h"
7 #include "src/codegen/optimized-compilation-info.h"
8 #include "src/compiler/backend/code-generator-impl.h"
9 #include "src/compiler/backend/code-generator.h"
10 #include "src/compiler/backend/gap-resolver.h"
11 #include "src/compiler/node-matchers.h"
12 #include "src/compiler/osr.h"
13 #include "src/execution/frame-constants.h"
14 #include "src/heap/memory-chunk.h"
15
16 #if V8_ENABLE_WEBASSEMBLY
17 #include "src/wasm/wasm-code-manager.h"
18 #include "src/wasm/wasm-objects.h"
19 #endif // V8_ENABLE_WEBASSEMBLY
20
21 namespace v8 {
22 namespace internal {
23 namespace compiler {
24
25 #define __ tasm()->
26
27 // Adds Arm64-specific methods to convert InstructionOperands.
28 class Arm64OperandConverter final : public InstructionOperandConverter {
29 public:
Arm64OperandConverter(CodeGenerator * gen,Instruction * instr)30 Arm64OperandConverter(CodeGenerator* gen, Instruction* instr)
31 : InstructionOperandConverter(gen, instr) {}
32
InputFloat32Register(size_t index)33 DoubleRegister InputFloat32Register(size_t index) {
34 return InputDoubleRegister(index).S();
35 }
36
InputFloat64Register(size_t index)37 DoubleRegister InputFloat64Register(size_t index) {
38 return InputDoubleRegister(index);
39 }
40
InputSimd128Register(size_t index)41 DoubleRegister InputSimd128Register(size_t index) {
42 return InputDoubleRegister(index).Q();
43 }
44
InputFloat32OrZeroRegister(size_t index)45 CPURegister InputFloat32OrZeroRegister(size_t index) {
46 if (instr_->InputAt(index)->IsImmediate()) {
47 DCHECK_EQ(0, bit_cast<int32_t>(InputFloat32(index)));
48 return wzr;
49 }
50 DCHECK(instr_->InputAt(index)->IsFPRegister());
51 return InputDoubleRegister(index).S();
52 }
53
InputFloat64OrZeroRegister(size_t index)54 CPURegister InputFloat64OrZeroRegister(size_t index) {
55 if (instr_->InputAt(index)->IsImmediate()) {
56 DCHECK_EQ(0, bit_cast<int64_t>(InputDouble(index)));
57 return xzr;
58 }
59 DCHECK(instr_->InputAt(index)->IsDoubleRegister());
60 return InputDoubleRegister(index);
61 }
62
OutputCount()63 size_t OutputCount() { return instr_->OutputCount(); }
64
OutputFloat32Register()65 DoubleRegister OutputFloat32Register() { return OutputDoubleRegister().S(); }
66
OutputFloat64Register()67 DoubleRegister OutputFloat64Register() { return OutputDoubleRegister(); }
68
OutputSimd128Register()69 DoubleRegister OutputSimd128Register() { return OutputDoubleRegister().Q(); }
70
InputRegister32(size_t index)71 Register InputRegister32(size_t index) {
72 return ToRegister(instr_->InputAt(index)).W();
73 }
74
InputOrZeroRegister32(size_t index)75 Register InputOrZeroRegister32(size_t index) {
76 DCHECK(instr_->InputAt(index)->IsRegister() ||
77 (instr_->InputAt(index)->IsImmediate() && (InputInt32(index) == 0)));
78 if (instr_->InputAt(index)->IsImmediate()) {
79 return wzr;
80 }
81 return InputRegister32(index);
82 }
83
InputRegister64(size_t index)84 Register InputRegister64(size_t index) { return InputRegister(index); }
85
InputOrZeroRegister64(size_t index)86 Register InputOrZeroRegister64(size_t index) {
87 DCHECK(instr_->InputAt(index)->IsRegister() ||
88 (instr_->InputAt(index)->IsImmediate() && (InputInt64(index) == 0)));
89 if (instr_->InputAt(index)->IsImmediate()) {
90 return xzr;
91 }
92 return InputRegister64(index);
93 }
94
InputOperand(size_t index)95 Operand InputOperand(size_t index) {
96 return ToOperand(instr_->InputAt(index));
97 }
98
InputOperand64(size_t index)99 Operand InputOperand64(size_t index) { return InputOperand(index); }
100
InputOperand32(size_t index)101 Operand InputOperand32(size_t index) {
102 return ToOperand32(instr_->InputAt(index));
103 }
104
OutputRegister64()105 Register OutputRegister64() { return OutputRegister(); }
106
OutputRegister32()107 Register OutputRegister32() { return OutputRegister().W(); }
108
TempRegister32(size_t index)109 Register TempRegister32(size_t index) {
110 return ToRegister(instr_->TempAt(index)).W();
111 }
112
InputOperand2_32(size_t index)113 Operand InputOperand2_32(size_t index) {
114 switch (AddressingModeField::decode(instr_->opcode())) {
115 case kMode_None:
116 return InputOperand32(index);
117 case kMode_Operand2_R_LSL_I:
118 return Operand(InputRegister32(index), LSL, InputInt5(index + 1));
119 case kMode_Operand2_R_LSR_I:
120 return Operand(InputRegister32(index), LSR, InputInt5(index + 1));
121 case kMode_Operand2_R_ASR_I:
122 return Operand(InputRegister32(index), ASR, InputInt5(index + 1));
123 case kMode_Operand2_R_ROR_I:
124 return Operand(InputRegister32(index), ROR, InputInt5(index + 1));
125 case kMode_Operand2_R_UXTB:
126 return Operand(InputRegister32(index), UXTB);
127 case kMode_Operand2_R_UXTH:
128 return Operand(InputRegister32(index), UXTH);
129 case kMode_Operand2_R_SXTB:
130 return Operand(InputRegister32(index), SXTB);
131 case kMode_Operand2_R_SXTH:
132 return Operand(InputRegister32(index), SXTH);
133 case kMode_Operand2_R_SXTW:
134 return Operand(InputRegister32(index), SXTW);
135 case kMode_MRI:
136 case kMode_MRR:
137 case kMode_Root:
138 break;
139 }
140 UNREACHABLE();
141 }
142
InputOperand2_64(size_t index)143 Operand InputOperand2_64(size_t index) {
144 switch (AddressingModeField::decode(instr_->opcode())) {
145 case kMode_None:
146 return InputOperand64(index);
147 case kMode_Operand2_R_LSL_I:
148 return Operand(InputRegister64(index), LSL, InputInt6(index + 1));
149 case kMode_Operand2_R_LSR_I:
150 return Operand(InputRegister64(index), LSR, InputInt6(index + 1));
151 case kMode_Operand2_R_ASR_I:
152 return Operand(InputRegister64(index), ASR, InputInt6(index + 1));
153 case kMode_Operand2_R_ROR_I:
154 return Operand(InputRegister64(index), ROR, InputInt6(index + 1));
155 case kMode_Operand2_R_UXTB:
156 return Operand(InputRegister64(index), UXTB);
157 case kMode_Operand2_R_UXTH:
158 return Operand(InputRegister64(index), UXTH);
159 case kMode_Operand2_R_SXTB:
160 return Operand(InputRegister64(index), SXTB);
161 case kMode_Operand2_R_SXTH:
162 return Operand(InputRegister64(index), SXTH);
163 case kMode_Operand2_R_SXTW:
164 return Operand(InputRegister64(index), SXTW);
165 case kMode_MRI:
166 case kMode_MRR:
167 case kMode_Root:
168 break;
169 }
170 UNREACHABLE();
171 }
172
MemoryOperand(size_t index=0)173 MemOperand MemoryOperand(size_t index = 0) {
174 switch (AddressingModeField::decode(instr_->opcode())) {
175 case kMode_None:
176 case kMode_Operand2_R_LSR_I:
177 case kMode_Operand2_R_ASR_I:
178 case kMode_Operand2_R_ROR_I:
179 case kMode_Operand2_R_UXTB:
180 case kMode_Operand2_R_UXTH:
181 case kMode_Operand2_R_SXTB:
182 case kMode_Operand2_R_SXTH:
183 case kMode_Operand2_R_SXTW:
184 break;
185 case kMode_Root:
186 return MemOperand(kRootRegister, InputInt64(index));
187 case kMode_Operand2_R_LSL_I:
188 return MemOperand(InputRegister(index + 0), InputRegister(index + 1),
189 LSL, InputInt32(index + 2));
190 case kMode_MRI:
191 return MemOperand(InputRegister(index + 0), InputInt32(index + 1));
192 case kMode_MRR:
193 return MemOperand(InputRegister(index + 0), InputRegister(index + 1));
194 }
195 UNREACHABLE();
196 }
197
ToOperand(InstructionOperand * op)198 Operand ToOperand(InstructionOperand* op) {
199 if (op->IsRegister()) {
200 return Operand(ToRegister(op));
201 }
202 return ToImmediate(op);
203 }
204
ToOperand32(InstructionOperand * op)205 Operand ToOperand32(InstructionOperand* op) {
206 if (op->IsRegister()) {
207 return Operand(ToRegister(op).W());
208 }
209 return ToImmediate(op);
210 }
211
ToImmediate(InstructionOperand * operand)212 Operand ToImmediate(InstructionOperand* operand) {
213 Constant constant = ToConstant(operand);
214 switch (constant.type()) {
215 case Constant::kInt32:
216 return Operand(constant.ToInt32());
217 case Constant::kInt64:
218 #if V8_ENABLE_WEBASSEMBLY
219 if (RelocInfo::IsWasmReference(constant.rmode())) {
220 return Operand(constant.ToInt64(), constant.rmode());
221 }
222 #endif // V8_ENABLE_WEBASSEMBLY
223 return Operand(constant.ToInt64());
224 case Constant::kFloat32:
225 return Operand(Operand::EmbeddedNumber(constant.ToFloat32()));
226 case Constant::kFloat64:
227 return Operand(Operand::EmbeddedNumber(constant.ToFloat64().value()));
228 case Constant::kExternalReference:
229 return Operand(constant.ToExternalReference());
230 case Constant::kCompressedHeapObject: // Fall through.
231 case Constant::kHeapObject:
232 return Operand(constant.ToHeapObject());
233 case Constant::kDelayedStringConstant:
234 return Operand::EmbeddedStringConstant(
235 constant.ToDelayedStringConstant());
236 case Constant::kRpoNumber:
237 UNREACHABLE(); // TODO(dcarney): RPO immediates on arm64.
238 }
239 UNREACHABLE();
240 }
241
ToMemOperand(InstructionOperand * op,TurboAssembler * tasm) const242 MemOperand ToMemOperand(InstructionOperand* op, TurboAssembler* tasm) const {
243 DCHECK_NOT_NULL(op);
244 DCHECK(op->IsStackSlot() || op->IsFPStackSlot());
245 return SlotToMemOperand(AllocatedOperand::cast(op)->index(), tasm);
246 }
247
SlotToMemOperand(int slot,TurboAssembler * tasm) const248 MemOperand SlotToMemOperand(int slot, TurboAssembler* tasm) const {
249 FrameOffset offset = frame_access_state()->GetFrameOffset(slot);
250 if (offset.from_frame_pointer()) {
251 int from_sp = offset.offset() + frame_access_state()->GetSPToFPOffset();
252 // Convert FP-offsets to SP-offsets if it results in better code.
253 if (Assembler::IsImmLSUnscaled(from_sp) ||
254 Assembler::IsImmLSScaled(from_sp, 3)) {
255 offset = FrameOffset::FromStackPointer(from_sp);
256 }
257 }
258 // Access below the stack pointer is not expected in arm64 and is actively
259 // prevented at run time in the simulator.
260 DCHECK_IMPLIES(offset.from_stack_pointer(), offset.offset() >= 0);
261 return MemOperand(offset.from_stack_pointer() ? sp : fp, offset.offset());
262 }
263 };
264
265 namespace {
266
267 class OutOfLineRecordWrite final : public OutOfLineCode {
268 public:
OutOfLineRecordWrite(CodeGenerator * gen,Register object,Operand offset,Register value,RecordWriteMode mode,StubCallMode stub_mode,UnwindingInfoWriter * unwinding_info_writer)269 OutOfLineRecordWrite(CodeGenerator* gen, Register object, Operand offset,
270 Register value, RecordWriteMode mode,
271 StubCallMode stub_mode,
272 UnwindingInfoWriter* unwinding_info_writer)
273 : OutOfLineCode(gen),
274 object_(object),
275 offset_(offset),
276 value_(value),
277 mode_(mode),
278 #if V8_ENABLE_WEBASSEMBLY
279 stub_mode_(stub_mode),
280 #endif // V8_ENABLE_WEBASSEMBLY
281 must_save_lr_(!gen->frame_access_state()->has_frame()),
282 unwinding_info_writer_(unwinding_info_writer),
283 zone_(gen->zone()) {
284 }
285
Generate()286 void Generate() final {
287 if (COMPRESS_POINTERS_BOOL) {
288 __ DecompressTaggedPointer(value_, value_);
289 }
290 __ CheckPageFlag(value_, MemoryChunk::kPointersToHereAreInterestingMask, ne,
291 exit());
292 RememberedSetAction const remembered_set_action =
293 mode_ > RecordWriteMode::kValueIsMap ||
294 FLAG_use_full_record_write_builtin
295 ? RememberedSetAction::kEmit
296 : RememberedSetAction::kOmit;
297 SaveFPRegsMode const save_fp_mode = frame()->DidAllocateDoubleRegisters()
298 ? SaveFPRegsMode::kSave
299 : SaveFPRegsMode::kIgnore;
300 if (must_save_lr_) {
301 // We need to save and restore lr if the frame was elided.
302 __ Push<TurboAssembler::kSignLR>(lr, padreg);
303 unwinding_info_writer_->MarkLinkRegisterOnTopOfStack(__ pc_offset(), sp);
304 }
305 if (mode_ == RecordWriteMode::kValueIsEphemeronKey) {
306 __ CallEphemeronKeyBarrier(object_, offset_, save_fp_mode);
307 #if V8_ENABLE_WEBASSEMBLY
308 } else if (stub_mode_ == StubCallMode::kCallWasmRuntimeStub) {
309 // A direct call to a wasm runtime stub defined in this module.
310 // Just encode the stub index. This will be patched when the code
311 // is added to the native module and copied into wasm code space.
312 __ CallRecordWriteStubSaveRegisters(object_, offset_,
313 remembered_set_action, save_fp_mode,
314 StubCallMode::kCallWasmRuntimeStub);
315 #endif // V8_ENABLE_WEBASSEMBLY
316 } else {
317 __ CallRecordWriteStubSaveRegisters(object_, offset_,
318 remembered_set_action, save_fp_mode);
319 }
320 if (must_save_lr_) {
321 __ Pop<TurboAssembler::kAuthLR>(padreg, lr);
322 unwinding_info_writer_->MarkPopLinkRegisterFromTopOfStack(__ pc_offset());
323 }
324 }
325
326 private:
327 Register const object_;
328 Operand const offset_;
329 Register const value_;
330 RecordWriteMode const mode_;
331 #if V8_ENABLE_WEBASSEMBLY
332 StubCallMode const stub_mode_;
333 #endif // V8_ENABLE_WEBASSEMBLY
334 bool must_save_lr_;
335 UnwindingInfoWriter* const unwinding_info_writer_;
336 Zone* zone_;
337 };
338
FlagsConditionToCondition(FlagsCondition condition)339 Condition FlagsConditionToCondition(FlagsCondition condition) {
340 switch (condition) {
341 case kEqual:
342 return eq;
343 case kNotEqual:
344 return ne;
345 case kSignedLessThan:
346 return lt;
347 case kSignedGreaterThanOrEqual:
348 return ge;
349 case kSignedLessThanOrEqual:
350 return le;
351 case kSignedGreaterThan:
352 return gt;
353 case kUnsignedLessThan:
354 return lo;
355 case kUnsignedGreaterThanOrEqual:
356 return hs;
357 case kUnsignedLessThanOrEqual:
358 return ls;
359 case kUnsignedGreaterThan:
360 return hi;
361 case kFloatLessThanOrUnordered:
362 return lt;
363 case kFloatGreaterThanOrEqual:
364 return ge;
365 case kFloatLessThanOrEqual:
366 return ls;
367 case kFloatGreaterThanOrUnordered:
368 return hi;
369 case kFloatLessThan:
370 return lo;
371 case kFloatGreaterThanOrEqualOrUnordered:
372 return hs;
373 case kFloatLessThanOrEqualOrUnordered:
374 return le;
375 case kFloatGreaterThan:
376 return gt;
377 case kOverflow:
378 return vs;
379 case kNotOverflow:
380 return vc;
381 case kUnorderedEqual:
382 case kUnorderedNotEqual:
383 break;
384 case kPositiveOrZero:
385 return pl;
386 case kNegative:
387 return mi;
388 }
389 UNREACHABLE();
390 }
391
392 #if V8_ENABLE_WEBASSEMBLY
393 class WasmOutOfLineTrap : public OutOfLineCode {
394 public:
WasmOutOfLineTrap(CodeGenerator * gen,Instruction * instr)395 WasmOutOfLineTrap(CodeGenerator* gen, Instruction* instr)
396 : OutOfLineCode(gen), gen_(gen), instr_(instr) {}
Generate()397 void Generate() override {
398 Arm64OperandConverter i(gen_, instr_);
399 TrapId trap_id =
400 static_cast<TrapId>(i.InputInt32(instr_->InputCount() - 1));
401 GenerateCallToTrap(trap_id);
402 }
403
404 protected:
405 CodeGenerator* gen_;
406
GenerateWithTrapId(TrapId trap_id)407 void GenerateWithTrapId(TrapId trap_id) { GenerateCallToTrap(trap_id); }
408
409 private:
GenerateCallToTrap(TrapId trap_id)410 void GenerateCallToTrap(TrapId trap_id) {
411 if (!gen_->wasm_runtime_exception_support()) {
412 // We cannot test calls to the runtime in cctest/test-run-wasm.
413 // Therefore we emit a call to C here instead of a call to the runtime.
414 __ CallCFunction(ExternalReference::wasm_call_trap_callback_for_testing(),
415 0);
416 __ LeaveFrame(StackFrame::WASM);
417 auto call_descriptor = gen_->linkage()->GetIncomingDescriptor();
418 int pop_count = static_cast<int>(call_descriptor->ParameterSlotCount());
419 pop_count += (pop_count & 1); // align
420 __ Drop(pop_count);
421 __ Ret();
422 } else {
423 gen_->AssembleSourcePosition(instr_);
424 // A direct call to a wasm runtime stub defined in this module.
425 // Just encode the stub index. This will be patched when the code
426 // is added to the native module and copied into wasm code space.
427 __ Call(static_cast<Address>(trap_id), RelocInfo::WASM_STUB_CALL);
428 ReferenceMap* reference_map =
429 gen_->zone()->New<ReferenceMap>(gen_->zone());
430 gen_->RecordSafepoint(reference_map);
431 __ AssertUnreachable(AbortReason::kUnexpectedReturnFromWasmTrap);
432 }
433 }
434
435 Instruction* instr_;
436 };
437
438 class WasmProtectedInstructionTrap final : public WasmOutOfLineTrap {
439 public:
WasmProtectedInstructionTrap(CodeGenerator * gen,int pc,Instruction * instr)440 WasmProtectedInstructionTrap(CodeGenerator* gen, int pc, Instruction* instr)
441 : WasmOutOfLineTrap(gen, instr), pc_(pc) {}
442
Generate()443 void Generate() override {
444 DCHECK(FLAG_wasm_bounds_checks && !FLAG_wasm_enforce_bounds_checks);
445 gen_->AddProtectedInstructionLanding(pc_, __ pc_offset());
446 GenerateWithTrapId(TrapId::kTrapMemOutOfBounds);
447 }
448
449 private:
450 int pc_;
451 };
452
EmitOOLTrapIfNeeded(Zone * zone,CodeGenerator * codegen,InstructionCode opcode,Instruction * instr,int pc)453 void EmitOOLTrapIfNeeded(Zone* zone, CodeGenerator* codegen,
454 InstructionCode opcode, Instruction* instr, int pc) {
455 const MemoryAccessMode access_mode = AccessModeField::decode(opcode);
456 if (access_mode == kMemoryAccessProtected) {
457 zone->New<WasmProtectedInstructionTrap>(codegen, pc, instr);
458 }
459 }
460 #else
EmitOOLTrapIfNeeded(Zone * zone,CodeGenerator * codegen,InstructionCode opcode,Instruction * instr,int pc)461 void EmitOOLTrapIfNeeded(Zone* zone, CodeGenerator* codegen,
462 InstructionCode opcode, Instruction* instr, int pc) {
463 DCHECK_NE(kMemoryAccessProtected, AccessModeField::decode(opcode));
464 }
465 #endif // V8_ENABLE_WEBASSEMBLY
466
467 // Handles unary ops that work for float (scalar), double (scalar), or NEON.
468 template <typename Fn>
EmitFpOrNeonUnop(TurboAssembler * tasm,Fn fn,Instruction * instr,Arm64OperandConverter i,VectorFormat scalar,VectorFormat vector)469 void EmitFpOrNeonUnop(TurboAssembler* tasm, Fn fn, Instruction* instr,
470 Arm64OperandConverter i, VectorFormat scalar,
471 VectorFormat vector) {
472 VectorFormat f = instr->InputAt(0)->IsSimd128Register() ? vector : scalar;
473
474 VRegister output = VRegister::Create(i.OutputDoubleRegister().code(), f);
475 VRegister input = VRegister::Create(i.InputDoubleRegister(0).code(), f);
476 (tasm->*fn)(output, input);
477 }
478
479 } // namespace
480
481 #define ASSEMBLE_SHIFT(asm_instr, width) \
482 do { \
483 if (instr->InputAt(1)->IsRegister()) { \
484 __ asm_instr(i.OutputRegister##width(), i.InputRegister##width(0), \
485 i.InputRegister##width(1)); \
486 } else { \
487 uint32_t imm = \
488 static_cast<uint32_t>(i.InputOperand##width(1).ImmediateValue()); \
489 __ asm_instr(i.OutputRegister##width(), i.InputRegister##width(0), \
490 imm % (width)); \
491 } \
492 } while (0)
493
494 #define ASSEMBLE_ATOMIC_LOAD_INTEGER(asm_instr, reg) \
495 do { \
496 __ Add(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1)); \
497 __ asm_instr(i.Output##reg(), i.TempRegister(0)); \
498 } while (0)
499
500 #define ASSEMBLE_ATOMIC_STORE_INTEGER(asm_instr, reg) \
501 do { \
502 __ Add(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1)); \
503 __ asm_instr(i.Input##reg(2), i.TempRegister(0)); \
504 } while (0)
505
506 #define ASSEMBLE_ATOMIC_EXCHANGE_INTEGER(load_instr, store_instr, reg) \
507 do { \
508 Label exchange; \
509 __ Add(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1)); \
510 __ Bind(&exchange); \
511 __ load_instr(i.Output##reg(), i.TempRegister(0)); \
512 __ store_instr(i.TempRegister32(1), i.Input##reg(2), i.TempRegister(0)); \
513 __ Cbnz(i.TempRegister32(1), &exchange); \
514 } while (0)
515
516 #define ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER(load_instr, store_instr, ext, \
517 reg) \
518 do { \
519 Label compareExchange; \
520 Label exit; \
521 __ Add(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1)); \
522 __ Bind(&compareExchange); \
523 __ load_instr(i.Output##reg(), i.TempRegister(0)); \
524 __ Cmp(i.Output##reg(), Operand(i.Input##reg(2), ext)); \
525 __ B(ne, &exit); \
526 __ store_instr(i.TempRegister32(1), i.Input##reg(3), i.TempRegister(0)); \
527 __ Cbnz(i.TempRegister32(1), &compareExchange); \
528 __ Bind(&exit); \
529 } while (0)
530
531 #define ASSEMBLE_ATOMIC_BINOP(load_instr, store_instr, bin_instr, reg) \
532 do { \
533 Label binop; \
534 __ Add(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1)); \
535 __ Bind(&binop); \
536 __ load_instr(i.Output##reg(), i.TempRegister(0)); \
537 __ bin_instr(i.Temp##reg(1), i.Output##reg(), Operand(i.Input##reg(2))); \
538 __ store_instr(i.TempRegister32(2), i.Temp##reg(1), i.TempRegister(0)); \
539 __ Cbnz(i.TempRegister32(2), &binop); \
540 } while (0)
541
542 #define ASSEMBLE_IEEE754_BINOP(name) \
543 do { \
544 FrameScope scope(tasm(), StackFrame::MANUAL); \
545 __ CallCFunction(ExternalReference::ieee754_##name##_function(), 0, 2); \
546 } while (0)
547
548 #define ASSEMBLE_IEEE754_UNOP(name) \
549 do { \
550 FrameScope scope(tasm(), StackFrame::MANUAL); \
551 __ CallCFunction(ExternalReference::ieee754_##name##_function(), 0, 1); \
552 } while (0)
553
554 // If shift value is an immediate, we can call asm_imm, taking the shift value
555 // modulo 2^width. Otherwise, emit code to perform the modulus operation, and
556 // call asm_shl.
557 #define ASSEMBLE_SIMD_SHIFT_LEFT(asm_imm, width, format, asm_shl, gp) \
558 do { \
559 if (instr->InputAt(1)->IsImmediate()) { \
560 __ asm_imm(i.OutputSimd128Register().format(), \
561 i.InputSimd128Register(0).format(), i.InputInt##width(1)); \
562 } else { \
563 UseScratchRegisterScope temps(tasm()); \
564 VRegister tmp = temps.AcquireQ(); \
565 Register shift = temps.Acquire##gp(); \
566 constexpr int mask = (1 << width) - 1; \
567 __ And(shift, i.InputRegister32(1), mask); \
568 __ Dup(tmp.format(), shift); \
569 __ asm_shl(i.OutputSimd128Register().format(), \
570 i.InputSimd128Register(0).format(), tmp.format()); \
571 } \
572 } while (0)
573
574 // If shift value is an immediate, we can call asm_imm, taking the shift value
575 // modulo 2^width. Otherwise, emit code to perform the modulus operation, and
576 // call asm_shl, passing in the negative shift value (treated as right shift).
577 #define ASSEMBLE_SIMD_SHIFT_RIGHT(asm_imm, width, format, asm_shl, gp) \
578 do { \
579 if (instr->InputAt(1)->IsImmediate()) { \
580 __ asm_imm(i.OutputSimd128Register().format(), \
581 i.InputSimd128Register(0).format(), i.InputInt##width(1)); \
582 } else { \
583 UseScratchRegisterScope temps(tasm()); \
584 VRegister tmp = temps.AcquireQ(); \
585 Register shift = temps.Acquire##gp(); \
586 constexpr int mask = (1 << width) - 1; \
587 __ And(shift, i.InputRegister32(1), mask); \
588 __ Dup(tmp.format(), shift); \
589 __ Neg(tmp.format(), tmp.format()); \
590 __ asm_shl(i.OutputSimd128Register().format(), \
591 i.InputSimd128Register(0).format(), tmp.format()); \
592 } \
593 } while (0)
594
AssembleDeconstructFrame()595 void CodeGenerator::AssembleDeconstructFrame() {
596 __ Mov(sp, fp);
597 __ Pop<TurboAssembler::kAuthLR>(fp, lr);
598
599 unwinding_info_writer_.MarkFrameDeconstructed(__ pc_offset());
600 }
601
AssemblePrepareTailCall()602 void CodeGenerator::AssemblePrepareTailCall() {
603 if (frame_access_state()->has_frame()) {
604 __ RestoreFPAndLR();
605 }
606 frame_access_state()->SetFrameAccessToSP();
607 }
608
609 namespace {
610
AdjustStackPointerForTailCall(TurboAssembler * tasm,FrameAccessState * state,int new_slot_above_sp,bool allow_shrinkage=true)611 void AdjustStackPointerForTailCall(TurboAssembler* tasm,
612 FrameAccessState* state,
613 int new_slot_above_sp,
614 bool allow_shrinkage = true) {
615 int current_sp_offset = state->GetSPToFPSlotCount() +
616 StandardFrameConstants::kFixedSlotCountAboveFp;
617 int stack_slot_delta = new_slot_above_sp - current_sp_offset;
618 DCHECK_EQ(stack_slot_delta % 2, 0);
619 if (stack_slot_delta > 0) {
620 tasm->Claim(stack_slot_delta);
621 state->IncreaseSPDelta(stack_slot_delta);
622 } else if (allow_shrinkage && stack_slot_delta < 0) {
623 tasm->Drop(-stack_slot_delta);
624 state->IncreaseSPDelta(stack_slot_delta);
625 }
626 }
627
628 } // namespace
629
AssembleTailCallBeforeGap(Instruction * instr,int first_unused_slot_offset)630 void CodeGenerator::AssembleTailCallBeforeGap(Instruction* instr,
631 int first_unused_slot_offset) {
632 AdjustStackPointerForTailCall(tasm(), frame_access_state(),
633 first_unused_slot_offset, false);
634 }
635
AssembleTailCallAfterGap(Instruction * instr,int first_unused_slot_offset)636 void CodeGenerator::AssembleTailCallAfterGap(Instruction* instr,
637 int first_unused_slot_offset) {
638 DCHECK_EQ(first_unused_slot_offset % 2, 0);
639 AdjustStackPointerForTailCall(tasm(), frame_access_state(),
640 first_unused_slot_offset);
641 DCHECK(instr->IsTailCall());
642 InstructionOperandConverter g(this, instr);
643 int optional_padding_offset = g.InputInt32(instr->InputCount() - 2);
644 if (optional_padding_offset % 2) {
645 __ Poke(padreg, optional_padding_offset * kSystemPointerSize);
646 }
647 }
648
649 // Check that {kJavaScriptCallCodeStartRegister} is correct.
AssembleCodeStartRegisterCheck()650 void CodeGenerator::AssembleCodeStartRegisterCheck() {
651 UseScratchRegisterScope temps(tasm());
652 Register scratch = temps.AcquireX();
653 __ ComputeCodeStartAddress(scratch);
654 __ cmp(scratch, kJavaScriptCallCodeStartRegister);
655 __ Assert(eq, AbortReason::kWrongFunctionCodeStart);
656 }
657
658 // Check if the code object is marked for deoptimization. If it is, then it
659 // jumps to the CompileLazyDeoptimizedCode builtin. In order to do this we need
660 // to:
661 // 1. read from memory the word that contains that bit, which can be found in
662 // the flags in the referenced {CodeDataContainer} object;
663 // 2. test kMarkedForDeoptimizationBit in those flags; and
664 // 3. if it is not zero then it jumps to the builtin.
BailoutIfDeoptimized()665 void CodeGenerator::BailoutIfDeoptimized() {
666 UseScratchRegisterScope temps(tasm());
667 Register scratch = temps.AcquireX();
668 int offset = Code::kCodeDataContainerOffset - Code::kHeaderSize;
669 __ LoadTaggedPointerField(
670 scratch, MemOperand(kJavaScriptCallCodeStartRegister, offset));
671 __ Ldr(scratch.W(),
672 FieldMemOperand(scratch, CodeDataContainer::kKindSpecificFlagsOffset));
673 Label not_deoptimized;
674 __ Tbz(scratch.W(), Code::kMarkedForDeoptimizationBit, ¬_deoptimized);
675 __ Jump(BUILTIN_CODE(isolate(), CompileLazyDeoptimizedCode),
676 RelocInfo::CODE_TARGET);
677 __ Bind(¬_deoptimized);
678 }
679
680 // Assembles an instruction after register allocation, producing machine code.
AssembleArchInstruction(Instruction * instr)681 CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
682 Instruction* instr) {
683 Arm64OperandConverter i(this, instr);
684 InstructionCode opcode = instr->opcode();
685 ArchOpcode arch_opcode = ArchOpcodeField::decode(opcode);
686 switch (arch_opcode) {
687 case kArchCallCodeObject: {
688 if (instr->InputAt(0)->IsImmediate()) {
689 __ Call(i.InputCode(0), RelocInfo::CODE_TARGET);
690 } else {
691 Register reg = i.InputRegister(0);
692 DCHECK_IMPLIES(
693 instr->HasCallDescriptorFlag(CallDescriptor::kFixedTargetRegister),
694 reg == kJavaScriptCallCodeStartRegister);
695 __ CallCodeObject(reg);
696 }
697 RecordCallPosition(instr);
698 frame_access_state()->ClearSPDelta();
699 break;
700 }
701 case kArchCallBuiltinPointer: {
702 DCHECK(!instr->InputAt(0)->IsImmediate());
703 Register builtin_index = i.InputRegister(0);
704 __ CallBuiltinByIndex(builtin_index);
705 RecordCallPosition(instr);
706 frame_access_state()->ClearSPDelta();
707 break;
708 }
709 #if V8_ENABLE_WEBASSEMBLY
710 case kArchCallWasmFunction: {
711 if (instr->InputAt(0)->IsImmediate()) {
712 Constant constant = i.ToConstant(instr->InputAt(0));
713 Address wasm_code = static_cast<Address>(constant.ToInt64());
714 __ Call(wasm_code, constant.rmode());
715 } else {
716 Register target = i.InputRegister(0);
717 __ Call(target);
718 }
719 RecordCallPosition(instr);
720 frame_access_state()->ClearSPDelta();
721 break;
722 }
723 case kArchTailCallWasm: {
724 if (instr->InputAt(0)->IsImmediate()) {
725 Constant constant = i.ToConstant(instr->InputAt(0));
726 Address wasm_code = static_cast<Address>(constant.ToInt64());
727 __ Jump(wasm_code, constant.rmode());
728 } else {
729 Register target = i.InputRegister(0);
730 UseScratchRegisterScope temps(tasm());
731 temps.Exclude(x17);
732 __ Mov(x17, target);
733 __ Jump(x17);
734 }
735 unwinding_info_writer_.MarkBlockWillExit();
736 frame_access_state()->ClearSPDelta();
737 frame_access_state()->SetFrameAccessToDefault();
738 break;
739 }
740 #endif // V8_ENABLE_WEBASSEMBLY
741 case kArchTailCallCodeObject: {
742 if (instr->InputAt(0)->IsImmediate()) {
743 __ Jump(i.InputCode(0), RelocInfo::CODE_TARGET);
744 } else {
745 Register reg = i.InputRegister(0);
746 DCHECK_IMPLIES(
747 instr->HasCallDescriptorFlag(CallDescriptor::kFixedTargetRegister),
748 reg == kJavaScriptCallCodeStartRegister);
749 __ JumpCodeObject(reg);
750 }
751 unwinding_info_writer_.MarkBlockWillExit();
752 frame_access_state()->ClearSPDelta();
753 frame_access_state()->SetFrameAccessToDefault();
754 break;
755 }
756 case kArchTailCallAddress: {
757 CHECK(!instr->InputAt(0)->IsImmediate());
758 Register reg = i.InputRegister(0);
759 DCHECK_IMPLIES(
760 instr->HasCallDescriptorFlag(CallDescriptor::kFixedTargetRegister),
761 reg == kJavaScriptCallCodeStartRegister);
762 UseScratchRegisterScope temps(tasm());
763 temps.Exclude(x17);
764 __ Mov(x17, reg);
765 __ Jump(x17);
766 unwinding_info_writer_.MarkBlockWillExit();
767 frame_access_state()->ClearSPDelta();
768 frame_access_state()->SetFrameAccessToDefault();
769 break;
770 }
771 case kArchCallJSFunction: {
772 Register func = i.InputRegister(0);
773 if (FLAG_debug_code) {
774 // Check the function's context matches the context argument.
775 UseScratchRegisterScope scope(tasm());
776 Register temp = scope.AcquireX();
777 __ LoadTaggedPointerField(
778 temp, FieldMemOperand(func, JSFunction::kContextOffset));
779 __ cmp(cp, temp);
780 __ Assert(eq, AbortReason::kWrongFunctionContext);
781 }
782 static_assert(kJavaScriptCallCodeStartRegister == x2, "ABI mismatch");
783 __ LoadTaggedPointerField(x2,
784 FieldMemOperand(func, JSFunction::kCodeOffset));
785 __ CallCodeTObject(x2);
786 RecordCallPosition(instr);
787 frame_access_state()->ClearSPDelta();
788 break;
789 }
790 case kArchPrepareCallCFunction:
791 // We don't need kArchPrepareCallCFunction on arm64 as the instruction
792 // selector has already performed a Claim to reserve space on the stack.
793 // Frame alignment is always 16 bytes, and the stack pointer is already
794 // 16-byte aligned, therefore we do not need to align the stack pointer
795 // by an unknown value, and it is safe to continue accessing the frame
796 // via the stack pointer.
797 UNREACHABLE();
798 case kArchSaveCallerRegisters: {
799 fp_mode_ =
800 static_cast<SaveFPRegsMode>(MiscField::decode(instr->opcode()));
801 DCHECK(fp_mode_ == SaveFPRegsMode::kIgnore ||
802 fp_mode_ == SaveFPRegsMode::kSave);
803 // kReturnRegister0 should have been saved before entering the stub.
804 int bytes = __ PushCallerSaved(fp_mode_, kReturnRegister0);
805 DCHECK(IsAligned(bytes, kSystemPointerSize));
806 DCHECK_EQ(0, frame_access_state()->sp_delta());
807 frame_access_state()->IncreaseSPDelta(bytes / kSystemPointerSize);
808 DCHECK(!caller_registers_saved_);
809 caller_registers_saved_ = true;
810 break;
811 }
812 case kArchRestoreCallerRegisters: {
813 DCHECK(fp_mode_ ==
814 static_cast<SaveFPRegsMode>(MiscField::decode(instr->opcode())));
815 DCHECK(fp_mode_ == SaveFPRegsMode::kIgnore ||
816 fp_mode_ == SaveFPRegsMode::kSave);
817 // Don't overwrite the returned value.
818 int bytes = __ PopCallerSaved(fp_mode_, kReturnRegister0);
819 frame_access_state()->IncreaseSPDelta(-(bytes / kSystemPointerSize));
820 DCHECK_EQ(0, frame_access_state()->sp_delta());
821 DCHECK(caller_registers_saved_);
822 caller_registers_saved_ = false;
823 break;
824 }
825 case kArchPrepareTailCall:
826 AssemblePrepareTailCall();
827 break;
828 case kArchCallCFunction: {
829 int const num_gp_parameters = ParamField::decode(instr->opcode());
830 int const num_fp_parameters = FPParamField::decode(instr->opcode());
831 Label return_location;
832 #if V8_ENABLE_WEBASSEMBLY
833 if (linkage()->GetIncomingDescriptor()->IsWasmCapiFunction()) {
834 // Put the return address in a stack slot.
835 __ StoreReturnAddressInWasmExitFrame(&return_location);
836 }
837 #endif // V8_ENABLE_WEBASSEMBLY
838
839 if (instr->InputAt(0)->IsImmediate()) {
840 ExternalReference ref = i.InputExternalReference(0);
841 __ CallCFunction(ref, num_gp_parameters, num_fp_parameters);
842 } else {
843 Register func = i.InputRegister(0);
844 __ CallCFunction(func, num_gp_parameters, num_fp_parameters);
845 }
846 __ Bind(&return_location);
847 #if V8_ENABLE_WEBASSEMBLY
848 if (linkage()->GetIncomingDescriptor()->IsWasmCapiFunction()) {
849 RecordSafepoint(instr->reference_map());
850 }
851 #endif // V8_ENABLE_WEBASSEMBLY
852 frame_access_state()->SetFrameAccessToDefault();
853 // Ideally, we should decrement SP delta to match the change of stack
854 // pointer in CallCFunction. However, for certain architectures (e.g.
855 // ARM), there may be more strict alignment requirement, causing old SP
856 // to be saved on the stack. In those cases, we can not calculate the SP
857 // delta statically.
858 frame_access_state()->ClearSPDelta();
859 if (caller_registers_saved_) {
860 // Need to re-sync SP delta introduced in kArchSaveCallerRegisters.
861 // Here, we assume the sequence to be:
862 // kArchSaveCallerRegisters;
863 // kArchCallCFunction;
864 // kArchRestoreCallerRegisters;
865 int bytes =
866 __ RequiredStackSizeForCallerSaved(fp_mode_, kReturnRegister0);
867 frame_access_state()->IncreaseSPDelta(bytes / kSystemPointerSize);
868 }
869 break;
870 }
871 case kArchJmp:
872 AssembleArchJump(i.InputRpo(0));
873 break;
874 case kArchTableSwitch:
875 AssembleArchTableSwitch(instr);
876 break;
877 case kArchBinarySearchSwitch:
878 AssembleArchBinarySearchSwitch(instr);
879 break;
880 case kArchAbortCSADcheck:
881 DCHECK_EQ(i.InputRegister(0), x1);
882 {
883 // We don't actually want to generate a pile of code for this, so just
884 // claim there is a stack frame, without generating one.
885 FrameScope scope(tasm(), StackFrame::NO_FRAME_TYPE);
886 __ Call(BUILTIN_CODE(isolate(), AbortCSADcheck),
887 RelocInfo::CODE_TARGET);
888 }
889 __ Debug("kArchAbortCSADcheck", 0, BREAK);
890 unwinding_info_writer_.MarkBlockWillExit();
891 break;
892 case kArchDebugBreak:
893 __ DebugBreak();
894 break;
895 case kArchComment:
896 __ RecordComment(reinterpret_cast<const char*>(i.InputInt64(0)));
897 break;
898 case kArchThrowTerminator:
899 unwinding_info_writer_.MarkBlockWillExit();
900 break;
901 case kArchNop:
902 // don't emit code for nops.
903 break;
904 case kArchDeoptimize: {
905 DeoptimizationExit* exit =
906 BuildTranslation(instr, -1, 0, 0, OutputFrameStateCombine::Ignore());
907 __ B(exit->label());
908 break;
909 }
910 case kArchRet:
911 AssembleReturn(instr->InputAt(0));
912 break;
913 case kArchFramePointer:
914 __ mov(i.OutputRegister(), fp);
915 break;
916 case kArchParentFramePointer:
917 if (frame_access_state()->has_frame()) {
918 __ ldr(i.OutputRegister(), MemOperand(fp, 0));
919 } else {
920 __ mov(i.OutputRegister(), fp);
921 }
922 break;
923 case kArchStackPointerGreaterThan: {
924 // Potentially apply an offset to the current stack pointer before the
925 // comparison to consider the size difference of an optimized frame versus
926 // the contained unoptimized frames.
927
928 Register lhs_register = sp;
929 uint32_t offset;
930
931 if (ShouldApplyOffsetToStackCheck(instr, &offset)) {
932 lhs_register = i.TempRegister(0);
933 __ Sub(lhs_register, sp, offset);
934 }
935
936 constexpr size_t kValueIndex = 0;
937 DCHECK(instr->InputAt(kValueIndex)->IsRegister());
938 __ Cmp(lhs_register, i.InputRegister(kValueIndex));
939 break;
940 }
941 case kArchStackCheckOffset:
942 __ Move(i.OutputRegister(), Smi::FromInt(GetStackCheckOffset()));
943 break;
944 case kArchTruncateDoubleToI:
945 __ TruncateDoubleToI(isolate(), zone(), i.OutputRegister(),
946 i.InputDoubleRegister(0), DetermineStubCallMode(),
947 frame_access_state()->has_frame()
948 ? kLRHasBeenSaved
949 : kLRHasNotBeenSaved);
950
951 break;
952 case kArchStoreWithWriteBarrier: {
953 RecordWriteMode mode =
954 static_cast<RecordWriteMode>(MiscField::decode(instr->opcode()));
955 AddressingMode addressing_mode =
956 AddressingModeField::decode(instr->opcode());
957 Register object = i.InputRegister(0);
958 Operand offset(0);
959 if (addressing_mode == kMode_MRI) {
960 offset = Operand(i.InputInt64(1));
961 } else {
962 DCHECK_EQ(addressing_mode, kMode_MRR);
963 offset = Operand(i.InputRegister(1));
964 }
965 Register value = i.InputRegister(2);
966
967 if (FLAG_debug_code) {
968 // Checking that |value| is not a cleared weakref: our write barrier
969 // does not support that for now.
970 __ cmp(value, Operand(kClearedWeakHeapObjectLower32));
971 __ Check(ne, AbortReason::kOperandIsCleared);
972 }
973
974 auto ool = zone()->New<OutOfLineRecordWrite>(
975 this, object, offset, value, mode, DetermineStubCallMode(),
976 &unwinding_info_writer_);
977 __ StoreTaggedField(value, MemOperand(object, offset));
978 if (mode > RecordWriteMode::kValueIsPointer) {
979 __ JumpIfSmi(value, ool->exit());
980 }
981 __ CheckPageFlag(object, MemoryChunk::kPointersFromHereAreInterestingMask,
982 eq, ool->entry());
983 __ Bind(ool->exit());
984 break;
985 }
986 case kArchAtomicStoreWithWriteBarrier: {
987 DCHECK_EQ(AddressingModeField::decode(instr->opcode()), kMode_MRR);
988 RecordWriteMode mode =
989 static_cast<RecordWriteMode>(MiscField::decode(instr->opcode()));
990 Register object = i.InputRegister(0);
991 Register offset = i.InputRegister(1);
992 Register value = i.InputRegister(2);
993 auto ool = zone()->New<OutOfLineRecordWrite>(
994 this, object, offset, value, mode, DetermineStubCallMode(),
995 &unwinding_info_writer_);
996 __ AtomicStoreTaggedField(value, object, offset, i.TempRegister(0));
997 if (mode > RecordWriteMode::kValueIsPointer) {
998 __ JumpIfSmi(value, ool->exit());
999 }
1000 __ CheckPageFlag(object, MemoryChunk::kPointersFromHereAreInterestingMask,
1001 eq, ool->entry());
1002 __ Bind(ool->exit());
1003 break;
1004 }
1005 case kArchStackSlot: {
1006 FrameOffset offset =
1007 frame_access_state()->GetFrameOffset(i.InputInt32(0));
1008 Register base = offset.from_stack_pointer() ? sp : fp;
1009 __ Add(i.OutputRegister(0), base, Operand(offset.offset()));
1010 break;
1011 }
1012 case kIeee754Float64Acos:
1013 ASSEMBLE_IEEE754_UNOP(acos);
1014 break;
1015 case kIeee754Float64Acosh:
1016 ASSEMBLE_IEEE754_UNOP(acosh);
1017 break;
1018 case kIeee754Float64Asin:
1019 ASSEMBLE_IEEE754_UNOP(asin);
1020 break;
1021 case kIeee754Float64Asinh:
1022 ASSEMBLE_IEEE754_UNOP(asinh);
1023 break;
1024 case kIeee754Float64Atan:
1025 ASSEMBLE_IEEE754_UNOP(atan);
1026 break;
1027 case kIeee754Float64Atanh:
1028 ASSEMBLE_IEEE754_UNOP(atanh);
1029 break;
1030 case kIeee754Float64Atan2:
1031 ASSEMBLE_IEEE754_BINOP(atan2);
1032 break;
1033 case kIeee754Float64Cos:
1034 ASSEMBLE_IEEE754_UNOP(cos);
1035 break;
1036 case kIeee754Float64Cosh:
1037 ASSEMBLE_IEEE754_UNOP(cosh);
1038 break;
1039 case kIeee754Float64Cbrt:
1040 ASSEMBLE_IEEE754_UNOP(cbrt);
1041 break;
1042 case kIeee754Float64Exp:
1043 ASSEMBLE_IEEE754_UNOP(exp);
1044 break;
1045 case kIeee754Float64Expm1:
1046 ASSEMBLE_IEEE754_UNOP(expm1);
1047 break;
1048 case kIeee754Float64Log:
1049 ASSEMBLE_IEEE754_UNOP(log);
1050 break;
1051 case kIeee754Float64Log1p:
1052 ASSEMBLE_IEEE754_UNOP(log1p);
1053 break;
1054 case kIeee754Float64Log2:
1055 ASSEMBLE_IEEE754_UNOP(log2);
1056 break;
1057 case kIeee754Float64Log10:
1058 ASSEMBLE_IEEE754_UNOP(log10);
1059 break;
1060 case kIeee754Float64Pow:
1061 ASSEMBLE_IEEE754_BINOP(pow);
1062 break;
1063 case kIeee754Float64Sin:
1064 ASSEMBLE_IEEE754_UNOP(sin);
1065 break;
1066 case kIeee754Float64Sinh:
1067 ASSEMBLE_IEEE754_UNOP(sinh);
1068 break;
1069 case kIeee754Float64Tan:
1070 ASSEMBLE_IEEE754_UNOP(tan);
1071 break;
1072 case kIeee754Float64Tanh:
1073 ASSEMBLE_IEEE754_UNOP(tanh);
1074 break;
1075 case kArm64Float32RoundDown:
1076 EmitFpOrNeonUnop(tasm(), &TurboAssembler::Frintm, instr, i, kFormatS,
1077 kFormat4S);
1078 break;
1079 case kArm64Float64RoundDown:
1080 EmitFpOrNeonUnop(tasm(), &TurboAssembler::Frintm, instr, i, kFormatD,
1081 kFormat2D);
1082 break;
1083 case kArm64Float32RoundUp:
1084 EmitFpOrNeonUnop(tasm(), &TurboAssembler::Frintp, instr, i, kFormatS,
1085 kFormat4S);
1086 break;
1087 case kArm64Float64RoundUp:
1088 EmitFpOrNeonUnop(tasm(), &TurboAssembler::Frintp, instr, i, kFormatD,
1089 kFormat2D);
1090 break;
1091 case kArm64Float64RoundTiesAway:
1092 EmitFpOrNeonUnop(tasm(), &TurboAssembler::Frinta, instr, i, kFormatD,
1093 kFormat2D);
1094 break;
1095 case kArm64Float32RoundTruncate:
1096 EmitFpOrNeonUnop(tasm(), &TurboAssembler::Frintz, instr, i, kFormatS,
1097 kFormat4S);
1098 break;
1099 case kArm64Float64RoundTruncate:
1100 EmitFpOrNeonUnop(tasm(), &TurboAssembler::Frintz, instr, i, kFormatD,
1101 kFormat2D);
1102 break;
1103 case kArm64Float32RoundTiesEven:
1104 EmitFpOrNeonUnop(tasm(), &TurboAssembler::Frintn, instr, i, kFormatS,
1105 kFormat4S);
1106 break;
1107 case kArm64Float64RoundTiesEven:
1108 EmitFpOrNeonUnop(tasm(), &TurboAssembler::Frintn, instr, i, kFormatD,
1109 kFormat2D);
1110 break;
1111 case kArm64Add:
1112 if (FlagsModeField::decode(opcode) != kFlags_none) {
1113 __ Adds(i.OutputRegister(), i.InputOrZeroRegister64(0),
1114 i.InputOperand2_64(1));
1115 } else {
1116 __ Add(i.OutputRegister(), i.InputOrZeroRegister64(0),
1117 i.InputOperand2_64(1));
1118 }
1119 break;
1120 case kArm64Add32:
1121 if (FlagsModeField::decode(opcode) != kFlags_none) {
1122 __ Adds(i.OutputRegister32(), i.InputOrZeroRegister32(0),
1123 i.InputOperand2_32(1));
1124 } else {
1125 __ Add(i.OutputRegister32(), i.InputOrZeroRegister32(0),
1126 i.InputOperand2_32(1));
1127 }
1128 break;
1129 case kArm64And:
1130 if (FlagsModeField::decode(opcode) != kFlags_none) {
1131 // The ands instruction only sets N and Z, so only the following
1132 // conditions make sense.
1133 DCHECK(FlagsConditionField::decode(opcode) == kEqual ||
1134 FlagsConditionField::decode(opcode) == kNotEqual ||
1135 FlagsConditionField::decode(opcode) == kPositiveOrZero ||
1136 FlagsConditionField::decode(opcode) == kNegative);
1137 __ Ands(i.OutputRegister(), i.InputOrZeroRegister64(0),
1138 i.InputOperand2_64(1));
1139 } else {
1140 __ And(i.OutputRegister(), i.InputOrZeroRegister64(0),
1141 i.InputOperand2_64(1));
1142 }
1143 break;
1144 case kArm64And32:
1145 if (FlagsModeField::decode(opcode) != kFlags_none) {
1146 // The ands instruction only sets N and Z, so only the following
1147 // conditions make sense.
1148 DCHECK(FlagsConditionField::decode(opcode) == kEqual ||
1149 FlagsConditionField::decode(opcode) == kNotEqual ||
1150 FlagsConditionField::decode(opcode) == kPositiveOrZero ||
1151 FlagsConditionField::decode(opcode) == kNegative);
1152 __ Ands(i.OutputRegister32(), i.InputOrZeroRegister32(0),
1153 i.InputOperand2_32(1));
1154 } else {
1155 __ And(i.OutputRegister32(), i.InputOrZeroRegister32(0),
1156 i.InputOperand2_32(1));
1157 }
1158 break;
1159 case kArm64Bic:
1160 __ Bic(i.OutputRegister(), i.InputOrZeroRegister64(0),
1161 i.InputOperand2_64(1));
1162 break;
1163 case kArm64Bic32:
1164 __ Bic(i.OutputRegister32(), i.InputOrZeroRegister32(0),
1165 i.InputOperand2_32(1));
1166 break;
1167 case kArm64Mul:
1168 __ Mul(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1));
1169 break;
1170 case kArm64Mul32:
1171 __ Mul(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1));
1172 break;
1173 case kArm64Sadalp: {
1174 DCHECK_EQ(i.OutputSimd128Register(), i.InputSimd128Register(0));
1175 VectorFormat dst_f = VectorFormatFillQ(LaneSizeField::decode(opcode));
1176 VectorFormat src_f = VectorFormatHalfWidthDoubleLanes(dst_f);
1177 __ Sadalp(i.OutputSimd128Register().Format(dst_f),
1178 i.InputSimd128Register(1).Format(src_f));
1179 break;
1180 }
1181 case kArm64Saddlp: {
1182 VectorFormat dst_f = VectorFormatFillQ(LaneSizeField::decode(opcode));
1183 VectorFormat src_f = VectorFormatHalfWidthDoubleLanes(dst_f);
1184 __ Saddlp(i.OutputSimd128Register().Format(dst_f),
1185 i.InputSimd128Register(0).Format(src_f));
1186 break;
1187 }
1188 case kArm64Uadalp: {
1189 DCHECK_EQ(i.OutputSimd128Register(), i.InputSimd128Register(0));
1190 VectorFormat dst_f = VectorFormatFillQ(LaneSizeField::decode(opcode));
1191 VectorFormat src_f = VectorFormatHalfWidthDoubleLanes(dst_f);
1192 __ Uadalp(i.OutputSimd128Register().Format(dst_f),
1193 i.InputSimd128Register(1).Format(src_f));
1194 break;
1195 }
1196 case kArm64Uaddlp: {
1197 VectorFormat dst_f = VectorFormatFillQ(LaneSizeField::decode(opcode));
1198 VectorFormat src_f = VectorFormatHalfWidthDoubleLanes(dst_f);
1199 __ Uaddlp(i.OutputSimd128Register().Format(dst_f),
1200 i.InputSimd128Register(0).Format(src_f));
1201 break;
1202 }
1203 case kArm64ISplat: {
1204 VectorFormat f = VectorFormatFillQ(LaneSizeField::decode(opcode));
1205 Register src = LaneSizeField::decode(opcode) == 64 ? i.InputRegister64(0)
1206 : i.InputRegister32(0);
1207 __ Dup(i.OutputSimd128Register().Format(f), src);
1208 break;
1209 }
1210 case kArm64FSplat: {
1211 VectorFormat src_f =
1212 ScalarFormatFromLaneSize(LaneSizeField::decode(opcode));
1213 VectorFormat dst_f = VectorFormatFillQ(src_f);
1214 __ Dup(i.OutputSimd128Register().Format(dst_f),
1215 i.InputSimd128Register(0).Format(src_f), 0);
1216 break;
1217 }
1218 case kArm64Smlal: {
1219 VectorFormat dst_f = VectorFormatFillQ(LaneSizeField::decode(opcode));
1220 VectorFormat src_f = VectorFormatHalfWidth(dst_f);
1221 DCHECK_EQ(i.OutputSimd128Register(), i.InputSimd128Register(0));
1222 __ Smlal(i.OutputSimd128Register().Format(dst_f),
1223 i.InputSimd128Register(1).Format(src_f),
1224 i.InputSimd128Register(2).Format(src_f));
1225 break;
1226 }
1227 case kArm64Smlal2: {
1228 VectorFormat dst_f = VectorFormatFillQ(LaneSizeField::decode(opcode));
1229 VectorFormat src_f = VectorFormatHalfWidthDoubleLanes(dst_f);
1230 DCHECK_EQ(i.OutputSimd128Register(), i.InputSimd128Register(0));
1231 __ Smlal2(i.OutputSimd128Register().Format(dst_f),
1232 i.InputSimd128Register(1).Format(src_f),
1233 i.InputSimd128Register(2).Format(src_f));
1234 break;
1235 }
1236 case kArm64Smull: {
1237 if (instr->InputAt(0)->IsRegister()) {
1238 __ Smull(i.OutputRegister(), i.InputRegister32(0),
1239 i.InputRegister32(1));
1240 } else {
1241 DCHECK(instr->InputAt(0)->IsSimd128Register());
1242 VectorFormat dst_f = VectorFormatFillQ(LaneSizeField::decode(opcode));
1243 VectorFormat src_f = VectorFormatHalfWidth(dst_f);
1244 __ Smull(i.OutputSimd128Register().Format(dst_f),
1245 i.InputSimd128Register(0).Format(src_f),
1246 i.InputSimd128Register(1).Format(src_f));
1247 }
1248 break;
1249 }
1250 case kArm64Smull2: {
1251 VectorFormat dst_f = VectorFormatFillQ(LaneSizeField::decode(opcode));
1252 VectorFormat src_f = VectorFormatHalfWidthDoubleLanes(dst_f);
1253 __ Smull2(i.OutputSimd128Register().Format(dst_f),
1254 i.InputSimd128Register(0).Format(src_f),
1255 i.InputSimd128Register(1).Format(src_f));
1256 break;
1257 }
1258 case kArm64Umlal: {
1259 VectorFormat dst_f = VectorFormatFillQ(LaneSizeField::decode(opcode));
1260 VectorFormat src_f = VectorFormatHalfWidth(dst_f);
1261 DCHECK_EQ(i.OutputSimd128Register(), i.InputSimd128Register(0));
1262 __ Umlal(i.OutputSimd128Register().Format(dst_f),
1263 i.InputSimd128Register(1).Format(src_f),
1264 i.InputSimd128Register(2).Format(src_f));
1265 break;
1266 }
1267 case kArm64Umlal2: {
1268 VectorFormat dst_f = VectorFormatFillQ(LaneSizeField::decode(opcode));
1269 VectorFormat src_f = VectorFormatHalfWidthDoubleLanes(dst_f);
1270 DCHECK_EQ(i.OutputSimd128Register(), i.InputSimd128Register(0));
1271 __ Umlal2(i.OutputSimd128Register().Format(dst_f),
1272 i.InputSimd128Register(1).Format(src_f),
1273 i.InputSimd128Register(2).Format(src_f));
1274 break;
1275 }
1276 case kArm64Umull: {
1277 if (instr->InputAt(0)->IsRegister()) {
1278 __ Umull(i.OutputRegister(), i.InputRegister32(0),
1279 i.InputRegister32(1));
1280 } else {
1281 DCHECK(instr->InputAt(0)->IsSimd128Register());
1282 VectorFormat dst_f = VectorFormatFillQ(LaneSizeField::decode(opcode));
1283 VectorFormat src_f = VectorFormatHalfWidth(dst_f);
1284 __ Umull(i.OutputSimd128Register().Format(dst_f),
1285 i.InputSimd128Register(0).Format(src_f),
1286 i.InputSimd128Register(1).Format(src_f));
1287 }
1288 break;
1289 }
1290 case kArm64Umull2: {
1291 VectorFormat dst_f = VectorFormatFillQ(LaneSizeField::decode(opcode));
1292 VectorFormat src_f = VectorFormatHalfWidthDoubleLanes(dst_f);
1293 __ Umull2(i.OutputSimd128Register().Format(dst_f),
1294 i.InputSimd128Register(0).Format(src_f),
1295 i.InputSimd128Register(1).Format(src_f));
1296 break;
1297 }
1298 case kArm64Madd:
1299 __ Madd(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1),
1300 i.InputRegister(2));
1301 break;
1302 case kArm64Madd32:
1303 __ Madd(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1),
1304 i.InputRegister32(2));
1305 break;
1306 case kArm64Msub:
1307 __ Msub(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1),
1308 i.InputRegister(2));
1309 break;
1310 case kArm64Msub32:
1311 __ Msub(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1),
1312 i.InputRegister32(2));
1313 break;
1314 case kArm64Mneg:
1315 __ Mneg(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1));
1316 break;
1317 case kArm64Mneg32:
1318 __ Mneg(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1));
1319 break;
1320 case kArm64Idiv:
1321 __ Sdiv(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1));
1322 break;
1323 case kArm64Idiv32:
1324 __ Sdiv(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1));
1325 break;
1326 case kArm64Udiv:
1327 __ Udiv(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1));
1328 break;
1329 case kArm64Udiv32:
1330 __ Udiv(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1));
1331 break;
1332 case kArm64Imod: {
1333 UseScratchRegisterScope scope(tasm());
1334 Register temp = scope.AcquireX();
1335 __ Sdiv(temp, i.InputRegister(0), i.InputRegister(1));
1336 __ Msub(i.OutputRegister(), temp, i.InputRegister(1), i.InputRegister(0));
1337 break;
1338 }
1339 case kArm64Imod32: {
1340 UseScratchRegisterScope scope(tasm());
1341 Register temp = scope.AcquireW();
1342 __ Sdiv(temp, i.InputRegister32(0), i.InputRegister32(1));
1343 __ Msub(i.OutputRegister32(), temp, i.InputRegister32(1),
1344 i.InputRegister32(0));
1345 break;
1346 }
1347 case kArm64Umod: {
1348 UseScratchRegisterScope scope(tasm());
1349 Register temp = scope.AcquireX();
1350 __ Udiv(temp, i.InputRegister(0), i.InputRegister(1));
1351 __ Msub(i.OutputRegister(), temp, i.InputRegister(1), i.InputRegister(0));
1352 break;
1353 }
1354 case kArm64Umod32: {
1355 UseScratchRegisterScope scope(tasm());
1356 Register temp = scope.AcquireW();
1357 __ Udiv(temp, i.InputRegister32(0), i.InputRegister32(1));
1358 __ Msub(i.OutputRegister32(), temp, i.InputRegister32(1),
1359 i.InputRegister32(0));
1360 break;
1361 }
1362 case kArm64Not:
1363 __ Mvn(i.OutputRegister(), i.InputOperand(0));
1364 break;
1365 case kArm64Not32:
1366 __ Mvn(i.OutputRegister32(), i.InputOperand32(0));
1367 break;
1368 case kArm64Or:
1369 __ Orr(i.OutputRegister(), i.InputOrZeroRegister64(0),
1370 i.InputOperand2_64(1));
1371 break;
1372 case kArm64Or32:
1373 __ Orr(i.OutputRegister32(), i.InputOrZeroRegister32(0),
1374 i.InputOperand2_32(1));
1375 break;
1376 case kArm64Orn:
1377 __ Orn(i.OutputRegister(), i.InputOrZeroRegister64(0),
1378 i.InputOperand2_64(1));
1379 break;
1380 case kArm64Orn32:
1381 __ Orn(i.OutputRegister32(), i.InputOrZeroRegister32(0),
1382 i.InputOperand2_32(1));
1383 break;
1384 case kArm64Eor:
1385 __ Eor(i.OutputRegister(), i.InputOrZeroRegister64(0),
1386 i.InputOperand2_64(1));
1387 break;
1388 case kArm64Eor32:
1389 __ Eor(i.OutputRegister32(), i.InputOrZeroRegister32(0),
1390 i.InputOperand2_32(1));
1391 break;
1392 case kArm64Eon:
1393 __ Eon(i.OutputRegister(), i.InputOrZeroRegister64(0),
1394 i.InputOperand2_64(1));
1395 break;
1396 case kArm64Eon32:
1397 __ Eon(i.OutputRegister32(), i.InputOrZeroRegister32(0),
1398 i.InputOperand2_32(1));
1399 break;
1400 case kArm64Sub:
1401 if (FlagsModeField::decode(opcode) != kFlags_none) {
1402 __ Subs(i.OutputRegister(), i.InputOrZeroRegister64(0),
1403 i.InputOperand2_64(1));
1404 } else {
1405 __ Sub(i.OutputRegister(), i.InputOrZeroRegister64(0),
1406 i.InputOperand2_64(1));
1407 }
1408 break;
1409 case kArm64Sub32:
1410 if (FlagsModeField::decode(opcode) != kFlags_none) {
1411 __ Subs(i.OutputRegister32(), i.InputOrZeroRegister32(0),
1412 i.InputOperand2_32(1));
1413 } else {
1414 __ Sub(i.OutputRegister32(), i.InputOrZeroRegister32(0),
1415 i.InputOperand2_32(1));
1416 }
1417 break;
1418 case kArm64Lsl:
1419 ASSEMBLE_SHIFT(Lsl, 64);
1420 break;
1421 case kArm64Lsl32:
1422 ASSEMBLE_SHIFT(Lsl, 32);
1423 break;
1424 case kArm64Lsr:
1425 ASSEMBLE_SHIFT(Lsr, 64);
1426 break;
1427 case kArm64Lsr32:
1428 ASSEMBLE_SHIFT(Lsr, 32);
1429 break;
1430 case kArm64Asr:
1431 ASSEMBLE_SHIFT(Asr, 64);
1432 break;
1433 case kArm64Asr32:
1434 ASSEMBLE_SHIFT(Asr, 32);
1435 break;
1436 case kArm64Ror:
1437 ASSEMBLE_SHIFT(Ror, 64);
1438 break;
1439 case kArm64Ror32:
1440 ASSEMBLE_SHIFT(Ror, 32);
1441 break;
1442 case kArm64Mov32:
1443 __ Mov(i.OutputRegister32(), i.InputRegister32(0));
1444 break;
1445 case kArm64Sxtb32:
1446 __ Sxtb(i.OutputRegister32(), i.InputRegister32(0));
1447 break;
1448 case kArm64Sxth32:
1449 __ Sxth(i.OutputRegister32(), i.InputRegister32(0));
1450 break;
1451 case kArm64Sxtb:
1452 __ Sxtb(i.OutputRegister(), i.InputRegister32(0));
1453 break;
1454 case kArm64Sxth:
1455 __ Sxth(i.OutputRegister(), i.InputRegister32(0));
1456 break;
1457 case kArm64Sxtw:
1458 __ Sxtw(i.OutputRegister(), i.InputRegister32(0));
1459 break;
1460 case kArm64Sbfx:
1461 __ Sbfx(i.OutputRegister(), i.InputRegister(0), i.InputInt6(1),
1462 i.InputInt6(2));
1463 break;
1464 case kArm64Sbfx32:
1465 __ Sbfx(i.OutputRegister32(), i.InputRegister32(0), i.InputInt5(1),
1466 i.InputInt5(2));
1467 break;
1468 case kArm64Ubfx:
1469 __ Ubfx(i.OutputRegister(), i.InputRegister(0), i.InputInt6(1),
1470 i.InputInt32(2));
1471 break;
1472 case kArm64Ubfx32:
1473 __ Ubfx(i.OutputRegister32(), i.InputRegister32(0), i.InputInt5(1),
1474 i.InputInt32(2));
1475 break;
1476 case kArm64Ubfiz32:
1477 __ Ubfiz(i.OutputRegister32(), i.InputRegister32(0), i.InputInt5(1),
1478 i.InputInt5(2));
1479 break;
1480 case kArm64Bfi:
1481 __ Bfi(i.OutputRegister(), i.InputRegister(1), i.InputInt6(2),
1482 i.InputInt6(3));
1483 break;
1484 case kArm64TestAndBranch32:
1485 case kArm64TestAndBranch:
1486 // Pseudo instructions turned into tbz/tbnz in AssembleArchBranch.
1487 break;
1488 case kArm64CompareAndBranch32:
1489 case kArm64CompareAndBranch:
1490 // Pseudo instruction handled in AssembleArchBranch.
1491 break;
1492 case kArm64Claim: {
1493 int count = i.InputInt32(0);
1494 DCHECK_EQ(count % 2, 0);
1495 __ AssertSpAligned();
1496 if (count > 0) {
1497 __ Claim(count);
1498 frame_access_state()->IncreaseSPDelta(count);
1499 }
1500 break;
1501 }
1502 case kArm64Poke: {
1503 Operand operand(i.InputInt32(1) * kSystemPointerSize);
1504 if (instr->InputAt(0)->IsSimd128Register()) {
1505 __ Poke(i.InputSimd128Register(0), operand);
1506 } else if (instr->InputAt(0)->IsFPRegister()) {
1507 __ Poke(i.InputFloat64Register(0), operand);
1508 } else {
1509 __ Poke(i.InputOrZeroRegister64(0), operand);
1510 }
1511 break;
1512 }
1513 case kArm64PokePair: {
1514 int slot = i.InputInt32(2) - 1;
1515 if (instr->InputAt(0)->IsFPRegister()) {
1516 __ PokePair(i.InputFloat64Register(1), i.InputFloat64Register(0),
1517 slot * kSystemPointerSize);
1518 } else {
1519 __ PokePair(i.InputRegister(1), i.InputRegister(0),
1520 slot * kSystemPointerSize);
1521 }
1522 break;
1523 }
1524 case kArm64Peek: {
1525 int reverse_slot = i.InputInt32(0);
1526 int offset =
1527 FrameSlotToFPOffset(frame()->GetTotalFrameSlotCount() - reverse_slot);
1528 if (instr->OutputAt(0)->IsFPRegister()) {
1529 LocationOperand* op = LocationOperand::cast(instr->OutputAt(0));
1530 if (op->representation() == MachineRepresentation::kFloat64) {
1531 __ Ldr(i.OutputDoubleRegister(), MemOperand(fp, offset));
1532 } else if (op->representation() == MachineRepresentation::kFloat32) {
1533 __ Ldr(i.OutputFloatRegister(), MemOperand(fp, offset));
1534 } else {
1535 DCHECK_EQ(MachineRepresentation::kSimd128, op->representation());
1536 __ Ldr(i.OutputSimd128Register(), MemOperand(fp, offset));
1537 }
1538 } else {
1539 __ Ldr(i.OutputRegister(), MemOperand(fp, offset));
1540 }
1541 break;
1542 }
1543 case kArm64Clz:
1544 __ Clz(i.OutputRegister64(), i.InputRegister64(0));
1545 break;
1546 case kArm64Clz32:
1547 __ Clz(i.OutputRegister32(), i.InputRegister32(0));
1548 break;
1549 case kArm64Rbit:
1550 __ Rbit(i.OutputRegister64(), i.InputRegister64(0));
1551 break;
1552 case kArm64Rbit32:
1553 __ Rbit(i.OutputRegister32(), i.InputRegister32(0));
1554 break;
1555 case kArm64Rev:
1556 __ Rev(i.OutputRegister64(), i.InputRegister64(0));
1557 break;
1558 case kArm64Rev32:
1559 __ Rev(i.OutputRegister32(), i.InputRegister32(0));
1560 break;
1561 case kArm64Cmp:
1562 __ Cmp(i.InputOrZeroRegister64(0), i.InputOperand2_64(1));
1563 break;
1564 case kArm64Cmp32:
1565 __ Cmp(i.InputOrZeroRegister32(0), i.InputOperand2_32(1));
1566 break;
1567 case kArm64Cmn:
1568 __ Cmn(i.InputOrZeroRegister64(0), i.InputOperand2_64(1));
1569 break;
1570 case kArm64Cmn32:
1571 __ Cmn(i.InputOrZeroRegister32(0), i.InputOperand2_32(1));
1572 break;
1573 case kArm64Cnt32: {
1574 __ PopcntHelper(i.OutputRegister32(), i.InputRegister32(0));
1575 break;
1576 }
1577 case kArm64Cnt64: {
1578 __ PopcntHelper(i.OutputRegister64(), i.InputRegister64(0));
1579 break;
1580 }
1581 case kArm64Cnt: {
1582 VectorFormat f = VectorFormatFillQ(LaneSizeField::decode(opcode));
1583 __ Cnt(i.OutputSimd128Register().Format(f),
1584 i.InputSimd128Register(0).Format(f));
1585 break;
1586 }
1587 case kArm64Tst:
1588 __ Tst(i.InputOrZeroRegister64(0), i.InputOperand2_64(1));
1589 break;
1590 case kArm64Tst32:
1591 __ Tst(i.InputOrZeroRegister32(0), i.InputOperand2_32(1));
1592 break;
1593 case kArm64Float32Cmp:
1594 if (instr->InputAt(1)->IsFPRegister()) {
1595 __ Fcmp(i.InputFloat32Register(0), i.InputFloat32Register(1));
1596 } else {
1597 DCHECK(instr->InputAt(1)->IsImmediate());
1598 // 0.0 is the only immediate supported by fcmp instructions.
1599 DCHECK_EQ(0.0f, i.InputFloat32(1));
1600 __ Fcmp(i.InputFloat32Register(0), i.InputFloat32(1));
1601 }
1602 break;
1603 case kArm64Float32Add:
1604 __ Fadd(i.OutputFloat32Register(), i.InputFloat32Register(0),
1605 i.InputFloat32Register(1));
1606 break;
1607 case kArm64Float32Sub:
1608 __ Fsub(i.OutputFloat32Register(), i.InputFloat32Register(0),
1609 i.InputFloat32Register(1));
1610 break;
1611 case kArm64Float32Mul:
1612 __ Fmul(i.OutputFloat32Register(), i.InputFloat32Register(0),
1613 i.InputFloat32Register(1));
1614 break;
1615 case kArm64Float32Div:
1616 __ Fdiv(i.OutputFloat32Register(), i.InputFloat32Register(0),
1617 i.InputFloat32Register(1));
1618 break;
1619 case kArm64Float32Abs:
1620 __ Fabs(i.OutputFloat32Register(), i.InputFloat32Register(0));
1621 break;
1622 case kArm64Float32Abd:
1623 __ Fabd(i.OutputFloat32Register(), i.InputFloat32Register(0),
1624 i.InputFloat32Register(1));
1625 break;
1626 case kArm64Float32Neg:
1627 __ Fneg(i.OutputFloat32Register(), i.InputFloat32Register(0));
1628 break;
1629 case kArm64Float32Sqrt:
1630 __ Fsqrt(i.OutputFloat32Register(), i.InputFloat32Register(0));
1631 break;
1632 case kArm64Float32Fnmul: {
1633 __ Fnmul(i.OutputFloat32Register(), i.InputFloat32Register(0),
1634 i.InputFloat32Register(1));
1635 break;
1636 }
1637 case kArm64Float64Cmp:
1638 if (instr->InputAt(1)->IsFPRegister()) {
1639 __ Fcmp(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1640 } else {
1641 DCHECK(instr->InputAt(1)->IsImmediate());
1642 // 0.0 is the only immediate supported by fcmp instructions.
1643 DCHECK_EQ(0.0, i.InputDouble(1));
1644 __ Fcmp(i.InputDoubleRegister(0), i.InputDouble(1));
1645 }
1646 break;
1647 case kArm64Float64Add:
1648 __ Fadd(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1649 i.InputDoubleRegister(1));
1650 break;
1651 case kArm64Float64Sub:
1652 __ Fsub(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1653 i.InputDoubleRegister(1));
1654 break;
1655 case kArm64Float64Mul:
1656 __ Fmul(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1657 i.InputDoubleRegister(1));
1658 break;
1659 case kArm64Float64Div:
1660 __ Fdiv(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1661 i.InputDoubleRegister(1));
1662 break;
1663 case kArm64Float64Mod: {
1664 // TODO(turbofan): implement directly.
1665 FrameScope scope(tasm(), StackFrame::MANUAL);
1666 DCHECK_EQ(d0, i.InputDoubleRegister(0));
1667 DCHECK_EQ(d1, i.InputDoubleRegister(1));
1668 DCHECK_EQ(d0, i.OutputDoubleRegister());
1669 // TODO(turbofan): make sure this saves all relevant registers.
1670 __ CallCFunction(ExternalReference::mod_two_doubles_operation(), 0, 2);
1671 break;
1672 }
1673 case kArm64Float32Max: {
1674 __ Fmax(i.OutputFloat32Register(), i.InputFloat32Register(0),
1675 i.InputFloat32Register(1));
1676 break;
1677 }
1678 case kArm64Float64Max: {
1679 __ Fmax(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1680 i.InputDoubleRegister(1));
1681 break;
1682 }
1683 case kArm64Float32Min: {
1684 __ Fmin(i.OutputFloat32Register(), i.InputFloat32Register(0),
1685 i.InputFloat32Register(1));
1686 break;
1687 }
1688 case kArm64Float64Min: {
1689 __ Fmin(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1690 i.InputDoubleRegister(1));
1691 break;
1692 }
1693 case kArm64Float64Abs:
1694 __ Fabs(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
1695 break;
1696 case kArm64Float64Abd:
1697 __ Fabd(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1698 i.InputDoubleRegister(1));
1699 break;
1700 case kArm64Float64Neg:
1701 __ Fneg(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
1702 break;
1703 case kArm64Float64Sqrt:
1704 __ Fsqrt(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
1705 break;
1706 case kArm64Float64Fnmul:
1707 __ Fnmul(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1708 i.InputDoubleRegister(1));
1709 break;
1710 case kArm64Float32ToFloat64:
1711 __ Fcvt(i.OutputDoubleRegister(), i.InputDoubleRegister(0).S());
1712 break;
1713 case kArm64Float64ToFloat32:
1714 __ Fcvt(i.OutputDoubleRegister().S(), i.InputDoubleRegister(0));
1715 break;
1716 case kArm64Float32ToInt32: {
1717 __ Fcvtzs(i.OutputRegister32(), i.InputFloat32Register(0));
1718 bool set_overflow_to_min_i32 = MiscField::decode(instr->opcode());
1719 if (set_overflow_to_min_i32) {
1720 // Avoid INT32_MAX as an overflow indicator and use INT32_MIN instead,
1721 // because INT32_MIN allows easier out-of-bounds detection.
1722 __ Cmn(i.OutputRegister32(), 1);
1723 __ Csinc(i.OutputRegister32(), i.OutputRegister32(),
1724 i.OutputRegister32(), vc);
1725 }
1726 break;
1727 }
1728 case kArm64Float64ToInt32:
1729 __ Fcvtzs(i.OutputRegister32(), i.InputDoubleRegister(0));
1730 break;
1731 case kArm64Float32ToUint32: {
1732 __ Fcvtzu(i.OutputRegister32(), i.InputFloat32Register(0));
1733 bool set_overflow_to_min_u32 = MiscField::decode(instr->opcode());
1734 if (set_overflow_to_min_u32) {
1735 // Avoid UINT32_MAX as an overflow indicator and use 0 instead,
1736 // because 0 allows easier out-of-bounds detection.
1737 __ Cmn(i.OutputRegister32(), 1);
1738 __ Adc(i.OutputRegister32(), i.OutputRegister32(), Operand(0));
1739 }
1740 break;
1741 }
1742 case kArm64Float64ToUint32:
1743 __ Fcvtzu(i.OutputRegister32(), i.InputDoubleRegister(0));
1744 break;
1745 case kArm64Float32ToInt64:
1746 __ Fcvtzs(i.OutputRegister64(), i.InputFloat32Register(0));
1747 if (i.OutputCount() > 1) {
1748 // Check for inputs below INT64_MIN and NaN.
1749 __ Fcmp(i.InputFloat32Register(0), static_cast<float>(INT64_MIN));
1750 // Check overflow.
1751 // -1 value is used to indicate a possible overflow which will occur
1752 // when subtracting (-1) from the provided INT64_MAX operand.
1753 // OutputRegister(1) is set to 0 if the input was out of range or NaN.
1754 __ Ccmp(i.OutputRegister(0), -1, VFlag, ge);
1755 __ Cset(i.OutputRegister(1), vc);
1756 }
1757 break;
1758 case kArm64Float64ToInt64: {
1759 __ Fcvtzs(i.OutputRegister(0), i.InputDoubleRegister(0));
1760 bool set_overflow_to_min_i64 = MiscField::decode(instr->opcode());
1761 DCHECK_IMPLIES(set_overflow_to_min_i64, i.OutputCount() == 1);
1762 if (set_overflow_to_min_i64) {
1763 // Avoid INT64_MAX as an overflow indicator and use INT64_MIN instead,
1764 // because INT64_MIN allows easier out-of-bounds detection.
1765 __ Cmn(i.OutputRegister64(), 1);
1766 __ Csinc(i.OutputRegister64(), i.OutputRegister64(),
1767 i.OutputRegister64(), vc);
1768 } else if (i.OutputCount() > 1) {
1769 // See kArm64Float32ToInt64 for a detailed description.
1770 __ Fcmp(i.InputDoubleRegister(0), static_cast<double>(INT64_MIN));
1771 __ Ccmp(i.OutputRegister(0), -1, VFlag, ge);
1772 __ Cset(i.OutputRegister(1), vc);
1773 }
1774 break;
1775 }
1776 case kArm64Float32ToUint64:
1777 __ Fcvtzu(i.OutputRegister64(), i.InputFloat32Register(0));
1778 if (i.OutputCount() > 1) {
1779 // See kArm64Float32ToInt64 for a detailed description.
1780 __ Fcmp(i.InputFloat32Register(0), -1.0);
1781 __ Ccmp(i.OutputRegister(0), -1, ZFlag, gt);
1782 __ Cset(i.OutputRegister(1), ne);
1783 }
1784 break;
1785 case kArm64Float64ToUint64:
1786 __ Fcvtzu(i.OutputRegister64(), i.InputDoubleRegister(0));
1787 if (i.OutputCount() > 1) {
1788 // See kArm64Float32ToInt64 for a detailed description.
1789 __ Fcmp(i.InputDoubleRegister(0), -1.0);
1790 __ Ccmp(i.OutputRegister(0), -1, ZFlag, gt);
1791 __ Cset(i.OutputRegister(1), ne);
1792 }
1793 break;
1794 case kArm64Int32ToFloat32:
1795 __ Scvtf(i.OutputFloat32Register(), i.InputRegister32(0));
1796 break;
1797 case kArm64Int32ToFloat64:
1798 __ Scvtf(i.OutputDoubleRegister(), i.InputRegister32(0));
1799 break;
1800 case kArm64Int64ToFloat32:
1801 __ Scvtf(i.OutputDoubleRegister().S(), i.InputRegister64(0));
1802 break;
1803 case kArm64Int64ToFloat64:
1804 __ Scvtf(i.OutputDoubleRegister(), i.InputRegister64(0));
1805 break;
1806 case kArm64Uint32ToFloat32:
1807 __ Ucvtf(i.OutputFloat32Register(), i.InputRegister32(0));
1808 break;
1809 case kArm64Uint32ToFloat64:
1810 __ Ucvtf(i.OutputDoubleRegister(), i.InputRegister32(0));
1811 break;
1812 case kArm64Uint64ToFloat32:
1813 __ Ucvtf(i.OutputDoubleRegister().S(), i.InputRegister64(0));
1814 break;
1815 case kArm64Uint64ToFloat64:
1816 __ Ucvtf(i.OutputDoubleRegister(), i.InputRegister64(0));
1817 break;
1818 case kArm64Float64ExtractLowWord32:
1819 __ Fmov(i.OutputRegister32(), i.InputFloat32Register(0));
1820 break;
1821 case kArm64Float64ExtractHighWord32:
1822 __ Umov(i.OutputRegister32(), i.InputFloat64Register(0).V2S(), 1);
1823 break;
1824 case kArm64Float64InsertLowWord32:
1825 DCHECK_EQ(i.OutputFloat64Register(), i.InputFloat64Register(0));
1826 __ Ins(i.OutputFloat64Register().V2S(), 0, i.InputRegister32(1));
1827 break;
1828 case kArm64Float64InsertHighWord32:
1829 DCHECK_EQ(i.OutputFloat64Register(), i.InputFloat64Register(0));
1830 __ Ins(i.OutputFloat64Register().V2S(), 1, i.InputRegister32(1));
1831 break;
1832 case kArm64Float64MoveU64:
1833 __ Fmov(i.OutputFloat64Register(), i.InputRegister(0));
1834 break;
1835 case kArm64Float64SilenceNaN:
1836 __ CanonicalizeNaN(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
1837 break;
1838 case kArm64U64MoveFloat64:
1839 __ Fmov(i.OutputRegister(), i.InputDoubleRegister(0));
1840 break;
1841 case kArm64Ldrb:
1842 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
1843 __ Ldrb(i.OutputRegister(), i.MemoryOperand());
1844 break;
1845 case kArm64Ldrsb:
1846 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
1847 __ Ldrsb(i.OutputRegister(), i.MemoryOperand());
1848 break;
1849 case kArm64LdrsbW:
1850 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
1851 __ Ldrsb(i.OutputRegister32(), i.MemoryOperand());
1852 break;
1853 case kArm64Strb:
1854 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
1855 __ Strb(i.InputOrZeroRegister64(0), i.MemoryOperand(1));
1856 break;
1857 case kArm64Ldrh:
1858 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
1859 __ Ldrh(i.OutputRegister(), i.MemoryOperand());
1860 break;
1861 case kArm64Ldrsh:
1862 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
1863 __ Ldrsh(i.OutputRegister(), i.MemoryOperand());
1864 break;
1865 case kArm64LdrshW:
1866 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
1867 __ Ldrsh(i.OutputRegister32(), i.MemoryOperand());
1868 break;
1869 case kArm64Strh:
1870 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
1871 __ Strh(i.InputOrZeroRegister64(0), i.MemoryOperand(1));
1872 break;
1873 case kArm64Ldrsw:
1874 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
1875 __ Ldrsw(i.OutputRegister(), i.MemoryOperand());
1876 break;
1877 case kArm64LdrW:
1878 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
1879 __ Ldr(i.OutputRegister32(), i.MemoryOperand());
1880 break;
1881 case kArm64StrW:
1882 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
1883 __ Str(i.InputOrZeroRegister32(0), i.MemoryOperand(1));
1884 break;
1885 case kArm64Ldr:
1886 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
1887 __ Ldr(i.OutputRegister(), i.MemoryOperand());
1888 break;
1889 case kArm64LdrDecompressTaggedSigned:
1890 __ DecompressTaggedSigned(i.OutputRegister(), i.MemoryOperand());
1891 break;
1892 case kArm64LdrDecompressTaggedPointer:
1893 __ DecompressTaggedPointer(i.OutputRegister(), i.MemoryOperand());
1894 break;
1895 case kArm64LdrDecompressAnyTagged:
1896 __ DecompressAnyTagged(i.OutputRegister(), i.MemoryOperand());
1897 break;
1898 case kArm64LdarDecompressTaggedSigned:
1899 __ AtomicDecompressTaggedSigned(i.OutputRegister(), i.InputRegister(0),
1900 i.InputRegister(1), i.TempRegister(0));
1901 break;
1902 case kArm64LdarDecompressTaggedPointer:
1903 __ AtomicDecompressTaggedPointer(i.OutputRegister(), i.InputRegister(0),
1904 i.InputRegister(1), i.TempRegister(0));
1905 break;
1906 case kArm64LdarDecompressAnyTagged:
1907 __ AtomicDecompressAnyTagged(i.OutputRegister(), i.InputRegister(0),
1908 i.InputRegister(1), i.TempRegister(0));
1909 break;
1910 case kArm64LdrDecodeSandboxedPointer:
1911 __ LoadSandboxedPointerField(i.OutputRegister(), i.MemoryOperand());
1912 break;
1913 case kArm64Str:
1914 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
1915 __ Str(i.InputOrZeroRegister64(0), i.MemoryOperand(1));
1916 break;
1917 case kArm64StrCompressTagged:
1918 __ StoreTaggedField(i.InputOrZeroRegister64(0), i.MemoryOperand(1));
1919 break;
1920 case kArm64StlrCompressTagged:
1921 // To be consistent with other STLR instructions, the value is stored at
1922 // the 3rd input register instead of the 1st.
1923 __ AtomicStoreTaggedField(i.InputRegister(2), i.InputRegister(0),
1924 i.InputRegister(1), i.TempRegister(0));
1925 break;
1926 case kArm64StrEncodeSandboxedPointer:
1927 __ StoreSandboxedPointerField(i.InputOrZeroRegister64(0),
1928 i.MemoryOperand(1));
1929 break;
1930 case kArm64LdrS:
1931 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
1932 __ Ldr(i.OutputDoubleRegister().S(), i.MemoryOperand());
1933 break;
1934 case kArm64StrS:
1935 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
1936 __ Str(i.InputFloat32OrZeroRegister(0), i.MemoryOperand(1));
1937 break;
1938 case kArm64LdrD:
1939 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
1940 __ Ldr(i.OutputDoubleRegister(), i.MemoryOperand());
1941 break;
1942 case kArm64StrD:
1943 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
1944 __ Str(i.InputFloat64OrZeroRegister(0), i.MemoryOperand(1));
1945 break;
1946 case kArm64LdrQ:
1947 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
1948 __ Ldr(i.OutputSimd128Register(), i.MemoryOperand());
1949 break;
1950 case kArm64StrQ:
1951 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
1952 __ Str(i.InputSimd128Register(0), i.MemoryOperand(1));
1953 break;
1954 case kArm64DmbIsh:
1955 __ Dmb(InnerShareable, BarrierAll);
1956 break;
1957 case kArm64DsbIsb:
1958 __ Dsb(FullSystem, BarrierAll);
1959 __ Isb();
1960 break;
1961 case kAtomicLoadInt8:
1962 DCHECK_EQ(AtomicWidthField::decode(opcode), AtomicWidth::kWord32);
1963 ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldarb, Register32);
1964 __ Sxtb(i.OutputRegister(0), i.OutputRegister(0));
1965 break;
1966 case kAtomicLoadUint8:
1967 ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldarb, Register32);
1968 break;
1969 case kAtomicLoadInt16:
1970 DCHECK_EQ(AtomicWidthField::decode(opcode), AtomicWidth::kWord32);
1971 ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldarh, Register32);
1972 __ Sxth(i.OutputRegister(0), i.OutputRegister(0));
1973 break;
1974 case kAtomicLoadUint16:
1975 ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldarh, Register32);
1976 break;
1977 case kAtomicLoadWord32:
1978 ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldar, Register32);
1979 break;
1980 case kArm64Word64AtomicLoadUint64:
1981 ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldar, Register);
1982 break;
1983 case kAtomicStoreWord8:
1984 ASSEMBLE_ATOMIC_STORE_INTEGER(Stlrb, Register32);
1985 break;
1986 case kAtomicStoreWord16:
1987 ASSEMBLE_ATOMIC_STORE_INTEGER(Stlrh, Register32);
1988 break;
1989 case kAtomicStoreWord32:
1990 ASSEMBLE_ATOMIC_STORE_INTEGER(Stlr, Register32);
1991 break;
1992 case kArm64Word64AtomicStoreWord64:
1993 ASSEMBLE_ATOMIC_STORE_INTEGER(Stlr, Register);
1994 break;
1995 case kAtomicExchangeInt8:
1996 ASSEMBLE_ATOMIC_EXCHANGE_INTEGER(ldaxrb, stlxrb, Register32);
1997 __ Sxtb(i.OutputRegister(0), i.OutputRegister(0));
1998 break;
1999 case kAtomicExchangeUint8:
2000 ASSEMBLE_ATOMIC_EXCHANGE_INTEGER(ldaxrb, stlxrb, Register32);
2001 break;
2002 case kAtomicExchangeInt16:
2003 ASSEMBLE_ATOMIC_EXCHANGE_INTEGER(ldaxrh, stlxrh, Register32);
2004 __ Sxth(i.OutputRegister(0), i.OutputRegister(0));
2005 break;
2006 case kAtomicExchangeUint16:
2007 ASSEMBLE_ATOMIC_EXCHANGE_INTEGER(ldaxrh, stlxrh, Register32);
2008 break;
2009 case kAtomicExchangeWord32:
2010 ASSEMBLE_ATOMIC_EXCHANGE_INTEGER(ldaxr, stlxr, Register32);
2011 break;
2012 case kArm64Word64AtomicExchangeUint64:
2013 ASSEMBLE_ATOMIC_EXCHANGE_INTEGER(ldaxr, stlxr, Register);
2014 break;
2015 case kAtomicCompareExchangeInt8:
2016 ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER(ldaxrb, stlxrb, UXTB,
2017 Register32);
2018 __ Sxtb(i.OutputRegister(0), i.OutputRegister(0));
2019 break;
2020 case kAtomicCompareExchangeUint8:
2021 ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER(ldaxrb, stlxrb, UXTB,
2022 Register32);
2023 break;
2024 case kAtomicCompareExchangeInt16:
2025 ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER(ldaxrh, stlxrh, UXTH,
2026 Register32);
2027 __ Sxth(i.OutputRegister(0), i.OutputRegister(0));
2028 break;
2029 case kAtomicCompareExchangeUint16:
2030 ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER(ldaxrh, stlxrh, UXTH,
2031 Register32);
2032 break;
2033 case kAtomicCompareExchangeWord32:
2034 ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER(ldaxr, stlxr, UXTW, Register32);
2035 break;
2036 case kArm64Word64AtomicCompareExchangeUint64:
2037 ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER(ldaxr, stlxr, UXTX, Register);
2038 break;
2039 #define ATOMIC_BINOP_CASE(op, inst) \
2040 case kAtomic##op##Int8: \
2041 ASSEMBLE_ATOMIC_BINOP(ldaxrb, stlxrb, inst, Register32); \
2042 __ Sxtb(i.OutputRegister(0), i.OutputRegister(0)); \
2043 break; \
2044 case kAtomic##op##Uint8: \
2045 ASSEMBLE_ATOMIC_BINOP(ldaxrb, stlxrb, inst, Register32); \
2046 break; \
2047 case kAtomic##op##Int16: \
2048 ASSEMBLE_ATOMIC_BINOP(ldaxrh, stlxrh, inst, Register32); \
2049 __ Sxth(i.OutputRegister(0), i.OutputRegister(0)); \
2050 break; \
2051 case kAtomic##op##Uint16: \
2052 ASSEMBLE_ATOMIC_BINOP(ldaxrh, stlxrh, inst, Register32); \
2053 break; \
2054 case kAtomic##op##Word32: \
2055 ASSEMBLE_ATOMIC_BINOP(ldaxr, stlxr, inst, Register32); \
2056 break; \
2057 case kArm64Word64Atomic##op##Uint64: \
2058 ASSEMBLE_ATOMIC_BINOP(ldaxr, stlxr, inst, Register); \
2059 break;
2060 ATOMIC_BINOP_CASE(Add, Add)
2061 ATOMIC_BINOP_CASE(Sub, Sub)
2062 ATOMIC_BINOP_CASE(And, And)
2063 ATOMIC_BINOP_CASE(Or, Orr)
2064 ATOMIC_BINOP_CASE(Xor, Eor)
2065 #undef ATOMIC_BINOP_CASE
2066 #undef ASSEMBLE_SHIFT
2067 #undef ASSEMBLE_ATOMIC_LOAD_INTEGER
2068 #undef ASSEMBLE_ATOMIC_STORE_INTEGER
2069 #undef ASSEMBLE_ATOMIC_EXCHANGE_INTEGER
2070 #undef ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER
2071 #undef ASSEMBLE_ATOMIC_BINOP
2072 #undef ASSEMBLE_IEEE754_BINOP
2073 #undef ASSEMBLE_IEEE754_UNOP
2074
2075 #define SIMD_UNOP_CASE(Op, Instr, FORMAT) \
2076 case Op: \
2077 __ Instr(i.OutputSimd128Register().V##FORMAT(), \
2078 i.InputSimd128Register(0).V##FORMAT()); \
2079 break;
2080 #define SIMD_UNOP_LANE_SIZE_CASE(Op, Instr) \
2081 case Op: { \
2082 VectorFormat f = VectorFormatFillQ(LaneSizeField::decode(opcode)); \
2083 __ Instr(i.OutputSimd128Register().Format(f), \
2084 i.InputSimd128Register(0).Format(f)); \
2085 break; \
2086 }
2087 #define SIMD_BINOP_CASE(Op, Instr, FORMAT) \
2088 case Op: \
2089 __ Instr(i.OutputSimd128Register().V##FORMAT(), \
2090 i.InputSimd128Register(0).V##FORMAT(), \
2091 i.InputSimd128Register(1).V##FORMAT()); \
2092 break;
2093 #define SIMD_BINOP_LANE_SIZE_CASE(Op, Instr) \
2094 case Op: { \
2095 VectorFormat f = VectorFormatFillQ(LaneSizeField::decode(opcode)); \
2096 __ Instr(i.OutputSimd128Register().Format(f), \
2097 i.InputSimd128Register(0).Format(f), \
2098 i.InputSimd128Register(1).Format(f)); \
2099 break; \
2100 }
2101 #define SIMD_FCM_L_CASE(Op, ImmOp, RegOp) \
2102 case Op: { \
2103 VectorFormat f = VectorFormatFillQ(LaneSizeField::decode(opcode)); \
2104 if (instr->InputCount() == 1) { \
2105 __ Fcm##ImmOp(i.OutputSimd128Register().Format(f), \
2106 i.InputSimd128Register(0).Format(f), +0.0); \
2107 } else { \
2108 __ Fcm##RegOp(i.OutputSimd128Register().Format(f), \
2109 i.InputSimd128Register(1).Format(f), \
2110 i.InputSimd128Register(0).Format(f)); \
2111 } \
2112 break; \
2113 }
2114 #define SIMD_FCM_G_CASE(Op, ImmOp) \
2115 case Op: { \
2116 VectorFormat f = VectorFormatFillQ(LaneSizeField::decode(opcode)); \
2117 /* Currently Gt/Ge instructions are only used with zero */ \
2118 DCHECK_EQ(instr->InputCount(), 1); \
2119 __ Fcm##ImmOp(i.OutputSimd128Register().Format(f), \
2120 i.InputSimd128Register(0).Format(f), +0.0); \
2121 break; \
2122 }
2123 #define SIMD_CM_L_CASE(Op, ImmOp) \
2124 case Op: { \
2125 VectorFormat f = VectorFormatFillQ(LaneSizeField::decode(opcode)); \
2126 DCHECK_EQ(instr->InputCount(), 1); \
2127 __ Cm##ImmOp(i.OutputSimd128Register().Format(f), \
2128 i.InputSimd128Register(0).Format(f), 0); \
2129 break; \
2130 }
2131 #define SIMD_CM_G_CASE(Op, CmOp) \
2132 case Op: { \
2133 VectorFormat f = VectorFormatFillQ(LaneSizeField::decode(opcode)); \
2134 if (instr->InputCount() == 1) { \
2135 __ Cm##CmOp(i.OutputSimd128Register().Format(f), \
2136 i.InputSimd128Register(0).Format(f), 0); \
2137 } else { \
2138 __ Cm##CmOp(i.OutputSimd128Register().Format(f), \
2139 i.InputSimd128Register(0).Format(f), \
2140 i.InputSimd128Register(1).Format(f)); \
2141 } \
2142 break; \
2143 }
2144 #define SIMD_DESTRUCTIVE_BINOP_CASE(Op, Instr, FORMAT) \
2145 case Op: { \
2146 VRegister dst = i.OutputSimd128Register().V##FORMAT(); \
2147 DCHECK_EQ(dst, i.InputSimd128Register(0).V##FORMAT()); \
2148 __ Instr(dst, i.InputSimd128Register(1).V##FORMAT(), \
2149 i.InputSimd128Register(2).V##FORMAT()); \
2150 break; \
2151 }
2152 #define SIMD_DESTRUCTIVE_BINOP_LANE_SIZE_CASE(Op, Instr) \
2153 case Op: { \
2154 VectorFormat f = VectorFormatFillQ(LaneSizeField::decode(opcode)); \
2155 VRegister dst = i.OutputSimd128Register().Format(f); \
2156 DCHECK_EQ(dst, i.InputSimd128Register(0).Format(f)); \
2157 __ Instr(dst, i.InputSimd128Register(1).Format(f), \
2158 i.InputSimd128Register(2).Format(f)); \
2159 break; \
2160 }
2161 SIMD_BINOP_LANE_SIZE_CASE(kArm64FMin, Fmin);
2162 SIMD_BINOP_LANE_SIZE_CASE(kArm64FMax, Fmax);
2163 SIMD_UNOP_LANE_SIZE_CASE(kArm64FAbs, Fabs);
2164 SIMD_UNOP_LANE_SIZE_CASE(kArm64FSqrt, Fsqrt);
2165 SIMD_BINOP_LANE_SIZE_CASE(kArm64FAdd, Fadd);
2166 SIMD_BINOP_LANE_SIZE_CASE(kArm64FSub, Fsub);
2167 SIMD_BINOP_LANE_SIZE_CASE(kArm64FMul, Fmul);
2168 SIMD_BINOP_LANE_SIZE_CASE(kArm64FDiv, Fdiv);
2169 SIMD_UNOP_LANE_SIZE_CASE(kArm64FNeg, Fneg);
2170 SIMD_UNOP_LANE_SIZE_CASE(kArm64IAbs, Abs);
2171 SIMD_UNOP_LANE_SIZE_CASE(kArm64INeg, Neg);
2172 SIMD_BINOP_LANE_SIZE_CASE(kArm64RoundingAverageU, Urhadd);
2173 SIMD_BINOP_LANE_SIZE_CASE(kArm64IMinS, Smin);
2174 SIMD_BINOP_LANE_SIZE_CASE(kArm64IMaxS, Smax);
2175 SIMD_BINOP_LANE_SIZE_CASE(kArm64IMinU, Umin);
2176 SIMD_BINOP_LANE_SIZE_CASE(kArm64IMaxU, Umax);
2177 SIMD_DESTRUCTIVE_BINOP_LANE_SIZE_CASE(kArm64Mla, Mla);
2178 SIMD_DESTRUCTIVE_BINOP_LANE_SIZE_CASE(kArm64Mls, Mls);
2179 case kArm64Sxtl: {
2180 VectorFormat wide = VectorFormatFillQ(LaneSizeField::decode(opcode));
2181 VectorFormat narrow = VectorFormatHalfWidth(wide);
2182 __ Sxtl(i.OutputSimd128Register().Format(wide),
2183 i.InputSimd128Register(0).Format(narrow));
2184 break;
2185 }
2186 case kArm64Sxtl2: {
2187 VectorFormat wide = VectorFormatFillQ(LaneSizeField::decode(opcode));
2188 VectorFormat narrow = VectorFormatHalfWidthDoubleLanes(wide);
2189 __ Sxtl2(i.OutputSimd128Register().Format(wide),
2190 i.InputSimd128Register(0).Format(narrow));
2191 break;
2192 }
2193 case kArm64Uxtl: {
2194 VectorFormat wide = VectorFormatFillQ(LaneSizeField::decode(opcode));
2195 VectorFormat narrow = VectorFormatHalfWidth(wide);
2196 __ Uxtl(i.OutputSimd128Register().Format(wide),
2197 i.InputSimd128Register(0).Format(narrow));
2198 break;
2199 }
2200 case kArm64Uxtl2: {
2201 VectorFormat wide = VectorFormatFillQ(LaneSizeField::decode(opcode));
2202 VectorFormat narrow = VectorFormatHalfWidthDoubleLanes(wide);
2203 __ Uxtl2(i.OutputSimd128Register().Format(wide),
2204 i.InputSimd128Register(0).Format(narrow));
2205 break;
2206 }
2207 case kArm64F64x2ConvertLowI32x4S: {
2208 VRegister dst = i.OutputSimd128Register().V2D();
2209 __ Sxtl(dst, i.InputSimd128Register(0).V2S());
2210 __ Scvtf(dst, dst);
2211 break;
2212 }
2213 case kArm64F64x2ConvertLowI32x4U: {
2214 VRegister dst = i.OutputSimd128Register().V2D();
2215 __ Uxtl(dst, i.InputSimd128Register(0).V2S());
2216 __ Ucvtf(dst, dst);
2217 break;
2218 }
2219 case kArm64I32x4TruncSatF64x2SZero: {
2220 VRegister dst = i.OutputSimd128Register();
2221 __ Fcvtzs(dst.V2D(), i.InputSimd128Register(0).V2D());
2222 __ Sqxtn(dst.V2S(), dst.V2D());
2223 break;
2224 }
2225 case kArm64I32x4TruncSatF64x2UZero: {
2226 VRegister dst = i.OutputSimd128Register();
2227 __ Fcvtzu(dst.V2D(), i.InputSimd128Register(0).V2D());
2228 __ Uqxtn(dst.V2S(), dst.V2D());
2229 break;
2230 }
2231 case kArm64F32x4DemoteF64x2Zero: {
2232 __ Fcvtn(i.OutputSimd128Register().V2S(),
2233 i.InputSimd128Register(0).V2D());
2234 break;
2235 }
2236 case kArm64F64x2PromoteLowF32x4: {
2237 __ Fcvtl(i.OutputSimd128Register().V2D(),
2238 i.InputSimd128Register(0).V2S());
2239 break;
2240 }
2241 case kArm64FExtractLane: {
2242 VectorFormat dst_f =
2243 ScalarFormatFromLaneSize(LaneSizeField::decode(opcode));
2244 VectorFormat src_f = VectorFormatFillQ(dst_f);
2245 __ Mov(i.OutputSimd128Register().Format(dst_f),
2246 i.InputSimd128Register(0).Format(src_f), i.InputInt8(1));
2247 break;
2248 }
2249 case kArm64FReplaceLane: {
2250 VectorFormat f = VectorFormatFillQ(LaneSizeField::decode(opcode));
2251 VRegister dst = i.OutputSimd128Register().Format(f),
2252 src1 = i.InputSimd128Register(0).Format(f);
2253 if (dst != src1) {
2254 __ Mov(dst, src1);
2255 }
2256 __ Mov(dst, i.InputInt8(1), i.InputSimd128Register(2).Format(f), 0);
2257 break;
2258 }
2259 SIMD_FCM_L_CASE(kArm64FEq, eq, eq);
2260 case kArm64FNe: {
2261 VectorFormat f = VectorFormatFillQ(LaneSizeField::decode(opcode));
2262 VRegister dst = i.OutputSimd128Register().Format(f);
2263 if (instr->InputCount() == 1) {
2264 __ Fcmeq(dst, i.InputSimd128Register(0).Format(f), +0.0);
2265 } else {
2266 __ Fcmeq(dst, i.InputSimd128Register(0).Format(f),
2267 i.InputSimd128Register(1).Format(f));
2268 }
2269 __ Mvn(dst, dst);
2270 break;
2271 }
2272 SIMD_FCM_L_CASE(kArm64FLt, lt, gt);
2273 SIMD_FCM_L_CASE(kArm64FLe, le, ge);
2274 SIMD_FCM_G_CASE(kArm64FGt, gt);
2275 SIMD_FCM_G_CASE(kArm64FGe, ge);
2276 SIMD_DESTRUCTIVE_BINOP_CASE(kArm64F64x2Qfma, Fmla, 2D);
2277 SIMD_DESTRUCTIVE_BINOP_CASE(kArm64F64x2Qfms, Fmls, 2D);
2278 case kArm64F64x2Pmin: {
2279 VRegister dst = i.OutputSimd128Register().V2D();
2280 VRegister lhs = i.InputSimd128Register(0).V2D();
2281 VRegister rhs = i.InputSimd128Register(1).V2D();
2282 // f64x2.pmin(lhs, rhs)
2283 // = v128.bitselect(rhs, lhs, f64x2.lt(rhs,lhs))
2284 // = v128.bitselect(rhs, lhs, f64x2.gt(lhs,rhs))
2285 __ Fcmgt(dst, lhs, rhs);
2286 __ Bsl(dst.V16B(), rhs.V16B(), lhs.V16B());
2287 break;
2288 }
2289 case kArm64F64x2Pmax: {
2290 VRegister dst = i.OutputSimd128Register().V2D();
2291 VRegister lhs = i.InputSimd128Register(0).V2D();
2292 VRegister rhs = i.InputSimd128Register(1).V2D();
2293 // f64x2.pmax(lhs, rhs)
2294 // = v128.bitselect(rhs, lhs, f64x2.gt(rhs, lhs))
2295 __ Fcmgt(dst, rhs, lhs);
2296 __ Bsl(dst.V16B(), rhs.V16B(), lhs.V16B());
2297 break;
2298 }
2299 SIMD_UNOP_CASE(kArm64F32x4SConvertI32x4, Scvtf, 4S);
2300 SIMD_UNOP_CASE(kArm64F32x4UConvertI32x4, Ucvtf, 4S);
2301 SIMD_UNOP_CASE(kArm64F32x4RecipApprox, Frecpe, 4S);
2302 SIMD_UNOP_CASE(kArm64F32x4RecipSqrtApprox, Frsqrte, 4S);
2303 case kArm64FMulElement: {
2304 VectorFormat s_f =
2305 ScalarFormatFromLaneSize(LaneSizeField::decode(opcode));
2306 VectorFormat v_f = VectorFormatFillQ(s_f);
2307 __ Fmul(i.OutputSimd128Register().Format(v_f),
2308 i.InputSimd128Register(0).Format(v_f),
2309 i.InputSimd128Register(1).Format(s_f), i.InputInt8(2));
2310 break;
2311 }
2312 SIMD_DESTRUCTIVE_BINOP_CASE(kArm64F32x4Qfma, Fmla, 4S);
2313 SIMD_DESTRUCTIVE_BINOP_CASE(kArm64F32x4Qfms, Fmls, 4S);
2314 case kArm64F32x4Pmin: {
2315 VRegister dst = i.OutputSimd128Register().V4S();
2316 VRegister lhs = i.InputSimd128Register(0).V4S();
2317 VRegister rhs = i.InputSimd128Register(1).V4S();
2318 // f32x4.pmin(lhs, rhs)
2319 // = v128.bitselect(rhs, lhs, f32x4.lt(rhs, lhs))
2320 // = v128.bitselect(rhs, lhs, f32x4.gt(lhs, rhs))
2321 __ Fcmgt(dst, lhs, rhs);
2322 __ Bsl(dst.V16B(), rhs.V16B(), lhs.V16B());
2323 break;
2324 }
2325 case kArm64F32x4Pmax: {
2326 VRegister dst = i.OutputSimd128Register().V4S();
2327 VRegister lhs = i.InputSimd128Register(0).V4S();
2328 VRegister rhs = i.InputSimd128Register(1).V4S();
2329 // f32x4.pmax(lhs, rhs)
2330 // = v128.bitselect(rhs, lhs, f32x4.gt(rhs, lhs))
2331 __ Fcmgt(dst, rhs, lhs);
2332 __ Bsl(dst.V16B(), rhs.V16B(), lhs.V16B());
2333 break;
2334 }
2335 case kArm64IExtractLane: {
2336 VectorFormat f = VectorFormatFillQ(LaneSizeField::decode(opcode));
2337 Register dst =
2338 f == kFormat2D ? i.OutputRegister64() : i.OutputRegister32();
2339 __ Mov(dst, i.InputSimd128Register(0).Format(f), i.InputInt8(1));
2340 break;
2341 }
2342 case kArm64IReplaceLane: {
2343 VectorFormat f = VectorFormatFillQ(LaneSizeField::decode(opcode));
2344 VRegister dst = i.OutputSimd128Register().Format(f),
2345 src1 = i.InputSimd128Register(0).Format(f);
2346 Register src2 =
2347 f == kFormat2D ? i.InputRegister64(2) : i.InputRegister32(2);
2348 if (dst != src1) {
2349 __ Mov(dst, src1);
2350 }
2351 __ Mov(dst, i.InputInt8(1), src2);
2352 break;
2353 }
2354 case kArm64I64x2Shl: {
2355 ASSEMBLE_SIMD_SHIFT_LEFT(Shl, 6, V2D, Sshl, X);
2356 break;
2357 }
2358 case kArm64I64x2ShrS: {
2359 ASSEMBLE_SIMD_SHIFT_RIGHT(Sshr, 6, V2D, Sshl, X);
2360 break;
2361 }
2362 SIMD_BINOP_LANE_SIZE_CASE(kArm64IAdd, Add);
2363 SIMD_BINOP_LANE_SIZE_CASE(kArm64ISub, Sub);
2364 case kArm64I64x2Mul: {
2365 UseScratchRegisterScope scope(tasm());
2366 VRegister dst = i.OutputSimd128Register();
2367 VRegister src1 = i.InputSimd128Register(0);
2368 VRegister src2 = i.InputSimd128Register(1);
2369 VRegister tmp1 = scope.AcquireSameSizeAs(dst);
2370 VRegister tmp2 = scope.AcquireSameSizeAs(dst);
2371 VRegister tmp3 = i.ToSimd128Register(instr->TempAt(0));
2372
2373 // This 2x64-bit multiplication is performed with several 32-bit
2374 // multiplications.
2375
2376 // 64-bit numbers x and y, can be represented as:
2377 // x = a + 2^32(b)
2378 // y = c + 2^32(d)
2379
2380 // A 64-bit multiplication is:
2381 // x * y = ac + 2^32(ad + bc) + 2^64(bd)
2382 // note: `2^64(bd)` can be ignored, the value is too large to fit in
2383 // 64-bits.
2384
2385 // This sequence implements a 2x64bit multiply, where the registers
2386 // `src1` and `src2` are split up into 32-bit components:
2387 // src1 = |d|c|b|a|
2388 // src2 = |h|g|f|e|
2389 //
2390 // src1 * src2 = |cg + 2^32(ch + dg)|ae + 2^32(af + be)|
2391
2392 // Reverse the 32-bit elements in the 64-bit words.
2393 // tmp2 = |g|h|e|f|
2394 __ Rev64(tmp2.V4S(), src2.V4S());
2395
2396 // Calculate the high half components.
2397 // tmp2 = |dg|ch|be|af|
2398 __ Mul(tmp2.V4S(), tmp2.V4S(), src1.V4S());
2399
2400 // Extract the low half components of src1.
2401 // tmp1 = |c|a|
2402 __ Xtn(tmp1.V2S(), src1.V2D());
2403
2404 // Sum the respective high half components.
2405 // tmp2 = |dg+ch|be+af||dg+ch|be+af|
2406 __ Addp(tmp2.V4S(), tmp2.V4S(), tmp2.V4S());
2407
2408 // Extract the low half components of src2.
2409 // tmp3 = |g|e|
2410 __ Xtn(tmp3.V2S(), src2.V2D());
2411
2412 // Shift the high half components, into the high half.
2413 // dst = |dg+ch << 32|be+af << 32|
2414 __ Shll(dst.V2D(), tmp2.V2S(), 32);
2415
2416 // Multiply the low components together, and accumulate with the high
2417 // half.
2418 // dst = |dst[1] + cg|dst[0] + ae|
2419 __ Umlal(dst.V2D(), tmp3.V2S(), tmp1.V2S());
2420
2421 break;
2422 }
2423 SIMD_CM_G_CASE(kArm64IEq, eq);
2424 case kArm64INe: {
2425 VectorFormat f = VectorFormatFillQ(LaneSizeField::decode(opcode));
2426 VRegister dst = i.OutputSimd128Register().Format(f);
2427 if (instr->InputCount() == 1) {
2428 __ Cmeq(dst, i.InputSimd128Register(0).Format(f), 0);
2429 } else {
2430 __ Cmeq(dst, i.InputSimd128Register(0).Format(f),
2431 i.InputSimd128Register(1).Format(f));
2432 }
2433 __ Mvn(dst, dst);
2434 break;
2435 }
2436 SIMD_CM_L_CASE(kArm64ILtS, lt);
2437 SIMD_CM_L_CASE(kArm64ILeS, le);
2438 SIMD_CM_G_CASE(kArm64IGtS, gt);
2439 SIMD_CM_G_CASE(kArm64IGeS, ge);
2440 case kArm64I64x2ShrU: {
2441 ASSEMBLE_SIMD_SHIFT_RIGHT(Ushr, 6, V2D, Ushl, X);
2442 break;
2443 }
2444 case kArm64I64x2BitMask: {
2445 __ I64x2BitMask(i.OutputRegister32(), i.InputSimd128Register(0));
2446 break;
2447 }
2448 SIMD_UNOP_CASE(kArm64I32x4SConvertF32x4, Fcvtzs, 4S);
2449 case kArm64I32x4Shl: {
2450 ASSEMBLE_SIMD_SHIFT_LEFT(Shl, 5, V4S, Sshl, W);
2451 break;
2452 }
2453 case kArm64I32x4ShrS: {
2454 ASSEMBLE_SIMD_SHIFT_RIGHT(Sshr, 5, V4S, Sshl, W);
2455 break;
2456 }
2457 SIMD_BINOP_CASE(kArm64I32x4Mul, Mul, 4S);
2458 SIMD_UNOP_CASE(kArm64I32x4UConvertF32x4, Fcvtzu, 4S);
2459 case kArm64I32x4ShrU: {
2460 ASSEMBLE_SIMD_SHIFT_RIGHT(Ushr, 5, V4S, Ushl, W);
2461 break;
2462 }
2463 SIMD_BINOP_LANE_SIZE_CASE(kArm64IGtU, Cmhi);
2464 SIMD_BINOP_LANE_SIZE_CASE(kArm64IGeU, Cmhs);
2465 case kArm64I32x4BitMask: {
2466 UseScratchRegisterScope scope(tasm());
2467 Register dst = i.OutputRegister32();
2468 VRegister src = i.InputSimd128Register(0);
2469 VRegister tmp = scope.AcquireQ();
2470 VRegister mask = scope.AcquireQ();
2471
2472 __ Sshr(tmp.V4S(), src.V4S(), 31);
2473 // Set i-th bit of each lane i. When AND with tmp, the lanes that
2474 // are signed will have i-th bit set, unsigned will be 0.
2475 __ Movi(mask.V2D(), 0x0000'0008'0000'0004, 0x0000'0002'0000'0001);
2476 __ And(tmp.V16B(), mask.V16B(), tmp.V16B());
2477 __ Addv(tmp.S(), tmp.V4S());
2478 __ Mov(dst.W(), tmp.V4S(), 0);
2479 break;
2480 }
2481 case kArm64I32x4DotI16x8S: {
2482 UseScratchRegisterScope scope(tasm());
2483 VRegister lhs = i.InputSimd128Register(0);
2484 VRegister rhs = i.InputSimd128Register(1);
2485 VRegister tmp1 = scope.AcquireV(kFormat4S);
2486 VRegister tmp2 = scope.AcquireV(kFormat4S);
2487 __ Smull(tmp1, lhs.V4H(), rhs.V4H());
2488 __ Smull2(tmp2, lhs.V8H(), rhs.V8H());
2489 __ Addp(i.OutputSimd128Register().V4S(), tmp1, tmp2);
2490 break;
2491 }
2492 case kArm64IExtractLaneU: {
2493 VectorFormat f = VectorFormatFillQ(LaneSizeField::decode(opcode));
2494 __ Umov(i.OutputRegister32(), i.InputSimd128Register(0).Format(f),
2495 i.InputInt8(1));
2496 break;
2497 }
2498 case kArm64IExtractLaneS: {
2499 VectorFormat f = VectorFormatFillQ(LaneSizeField::decode(opcode));
2500 __ Smov(i.OutputRegister32(), i.InputSimd128Register(0).Format(f),
2501 i.InputInt8(1));
2502 break;
2503 }
2504 case kArm64I16x8Shl: {
2505 ASSEMBLE_SIMD_SHIFT_LEFT(Shl, 4, V8H, Sshl, W);
2506 break;
2507 }
2508 case kArm64I16x8ShrS: {
2509 ASSEMBLE_SIMD_SHIFT_RIGHT(Sshr, 4, V8H, Sshl, W);
2510 break;
2511 }
2512 case kArm64I16x8SConvertI32x4: {
2513 VRegister dst = i.OutputSimd128Register(),
2514 src0 = i.InputSimd128Register(0),
2515 src1 = i.InputSimd128Register(1);
2516 UseScratchRegisterScope scope(tasm());
2517 VRegister temp = scope.AcquireV(kFormat4S);
2518 if (dst == src1) {
2519 __ Mov(temp, src1.V4S());
2520 src1 = temp;
2521 }
2522 __ Sqxtn(dst.V4H(), src0.V4S());
2523 __ Sqxtn2(dst.V8H(), src1.V4S());
2524 break;
2525 }
2526 SIMD_BINOP_LANE_SIZE_CASE(kArm64IAddSatS, Sqadd);
2527 SIMD_BINOP_LANE_SIZE_CASE(kArm64ISubSatS, Sqsub);
2528 SIMD_BINOP_CASE(kArm64I16x8Mul, Mul, 8H);
2529 case kArm64I16x8ShrU: {
2530 ASSEMBLE_SIMD_SHIFT_RIGHT(Ushr, 4, V8H, Ushl, W);
2531 break;
2532 }
2533 case kArm64I16x8UConvertI32x4: {
2534 VRegister dst = i.OutputSimd128Register(),
2535 src0 = i.InputSimd128Register(0),
2536 src1 = i.InputSimd128Register(1);
2537 UseScratchRegisterScope scope(tasm());
2538 VRegister temp = scope.AcquireV(kFormat4S);
2539 if (dst == src1) {
2540 __ Mov(temp, src1.V4S());
2541 src1 = temp;
2542 }
2543 __ Sqxtun(dst.V4H(), src0.V4S());
2544 __ Sqxtun2(dst.V8H(), src1.V4S());
2545 break;
2546 }
2547 SIMD_BINOP_LANE_SIZE_CASE(kArm64IAddSatU, Uqadd);
2548 SIMD_BINOP_LANE_SIZE_CASE(kArm64ISubSatU, Uqsub);
2549 SIMD_BINOP_CASE(kArm64I16x8Q15MulRSatS, Sqrdmulh, 8H);
2550 case kArm64I16x8BitMask: {
2551 UseScratchRegisterScope scope(tasm());
2552 Register dst = i.OutputRegister32();
2553 VRegister src = i.InputSimd128Register(0);
2554 VRegister tmp = scope.AcquireQ();
2555 VRegister mask = scope.AcquireQ();
2556
2557 __ Sshr(tmp.V8H(), src.V8H(), 15);
2558 // Set i-th bit of each lane i. When AND with tmp, the lanes that
2559 // are signed will have i-th bit set, unsigned will be 0.
2560 __ Movi(mask.V2D(), 0x0080'0040'0020'0010, 0x0008'0004'0002'0001);
2561 __ And(tmp.V16B(), mask.V16B(), tmp.V16B());
2562 __ Addv(tmp.H(), tmp.V8H());
2563 __ Mov(dst.W(), tmp.V8H(), 0);
2564 break;
2565 }
2566 case kArm64I8x16Shl: {
2567 ASSEMBLE_SIMD_SHIFT_LEFT(Shl, 3, V16B, Sshl, W);
2568 break;
2569 }
2570 case kArm64I8x16ShrS: {
2571 ASSEMBLE_SIMD_SHIFT_RIGHT(Sshr, 3, V16B, Sshl, W);
2572 break;
2573 }
2574 case kArm64I8x16SConvertI16x8: {
2575 VRegister dst = i.OutputSimd128Register(),
2576 src0 = i.InputSimd128Register(0),
2577 src1 = i.InputSimd128Register(1);
2578 UseScratchRegisterScope scope(tasm());
2579 VRegister temp = scope.AcquireV(kFormat8H);
2580 if (dst == src1) {
2581 __ Mov(temp, src1.V8H());
2582 src1 = temp;
2583 }
2584 __ Sqxtn(dst.V8B(), src0.V8H());
2585 __ Sqxtn2(dst.V16B(), src1.V8H());
2586 break;
2587 }
2588 case kArm64I8x16ShrU: {
2589 ASSEMBLE_SIMD_SHIFT_RIGHT(Ushr, 3, V16B, Ushl, W);
2590 break;
2591 }
2592 case kArm64I8x16UConvertI16x8: {
2593 VRegister dst = i.OutputSimd128Register(),
2594 src0 = i.InputSimd128Register(0),
2595 src1 = i.InputSimd128Register(1);
2596 UseScratchRegisterScope scope(tasm());
2597 VRegister temp = scope.AcquireV(kFormat8H);
2598 if (dst == src1) {
2599 __ Mov(temp, src1.V8H());
2600 src1 = temp;
2601 }
2602 __ Sqxtun(dst.V8B(), src0.V8H());
2603 __ Sqxtun2(dst.V16B(), src1.V8H());
2604 break;
2605 }
2606 case kArm64I8x16BitMask: {
2607 UseScratchRegisterScope scope(tasm());
2608 Register dst = i.OutputRegister32();
2609 VRegister src = i.InputSimd128Register(0);
2610 VRegister tmp = scope.AcquireQ();
2611 VRegister mask = scope.AcquireQ();
2612
2613 // Set i-th bit of each lane i. When AND with tmp, the lanes that
2614 // are signed will have i-th bit set, unsigned will be 0.
2615 __ Sshr(tmp.V16B(), src.V16B(), 7);
2616 __ Movi(mask.V2D(), 0x8040'2010'0804'0201);
2617 __ And(tmp.V16B(), mask.V16B(), tmp.V16B());
2618 __ Ext(mask.V16B(), tmp.V16B(), tmp.V16B(), 8);
2619 __ Zip1(tmp.V16B(), tmp.V16B(), mask.V16B());
2620 __ Addv(tmp.H(), tmp.V8H());
2621 __ Mov(dst.W(), tmp.V8H(), 0);
2622 break;
2623 }
2624 case kArm64S128Const: {
2625 uint64_t imm1 = make_uint64(i.InputUint32(1), i.InputUint32(0));
2626 uint64_t imm2 = make_uint64(i.InputUint32(3), i.InputUint32(2));
2627 __ Movi(i.OutputSimd128Register().V16B(), imm2, imm1);
2628 break;
2629 }
2630 case kArm64S128Zero: {
2631 VRegister dst = i.OutputSimd128Register().V16B();
2632 __ Eor(dst, dst, dst);
2633 break;
2634 }
2635 SIMD_BINOP_CASE(kArm64S128And, And, 16B);
2636 SIMD_BINOP_CASE(kArm64S128Or, Orr, 16B);
2637 SIMD_BINOP_CASE(kArm64S128Xor, Eor, 16B);
2638 SIMD_UNOP_CASE(kArm64S128Not, Mvn, 16B);
2639 case kArm64S128Dup: {
2640 VRegister dst = i.OutputSimd128Register(),
2641 src = i.InputSimd128Register(0);
2642 int lanes = i.InputInt32(1);
2643 int index = i.InputInt32(2);
2644 switch (lanes) {
2645 case 4:
2646 __ Dup(dst.V4S(), src.V4S(), index);
2647 break;
2648 case 8:
2649 __ Dup(dst.V8H(), src.V8H(), index);
2650 break;
2651 case 16:
2652 __ Dup(dst.V16B(), src.V16B(), index);
2653 break;
2654 default:
2655 UNREACHABLE();
2656 }
2657 break;
2658 }
2659 SIMD_DESTRUCTIVE_BINOP_CASE(kArm64S128Select, Bsl, 16B);
2660 SIMD_BINOP_CASE(kArm64S128AndNot, Bic, 16B);
2661 case kArm64Ssra: {
2662 int8_t laneSize = LaneSizeField::decode(opcode);
2663 VectorFormat f = VectorFormatFillQ(laneSize);
2664 int8_t mask = laneSize - 1;
2665 VRegister dst = i.OutputSimd128Register().Format(f);
2666 DCHECK_EQ(dst, i.InputSimd128Register(0).Format(f));
2667 __ Ssra(dst, i.InputSimd128Register(1).Format(f), i.InputInt8(2) & mask);
2668 break;
2669 }
2670 case kArm64Usra: {
2671 int8_t laneSize = LaneSizeField::decode(opcode);
2672 VectorFormat f = VectorFormatFillQ(laneSize);
2673 int8_t mask = laneSize - 1;
2674 VRegister dst = i.OutputSimd128Register().Format(f);
2675 DCHECK_EQ(dst, i.InputSimd128Register(0).Format(f));
2676 __ Usra(dst, i.InputSimd128Register(1).Format(f), i.InputUint8(2) & mask);
2677 break;
2678 }
2679 case kArm64S32x4Shuffle: {
2680 Simd128Register dst = i.OutputSimd128Register().V4S(),
2681 src0 = i.InputSimd128Register(0).V4S(),
2682 src1 = i.InputSimd128Register(1).V4S();
2683 // Check for in-place shuffles.
2684 // If dst == src0 == src1, then the shuffle is unary and we only use src0.
2685 UseScratchRegisterScope scope(tasm());
2686 VRegister temp = scope.AcquireV(kFormat4S);
2687 if (dst == src0) {
2688 __ Mov(temp, src0);
2689 src0 = temp;
2690 } else if (dst == src1) {
2691 __ Mov(temp, src1);
2692 src1 = temp;
2693 }
2694 // Perform shuffle as a vmov per lane.
2695 int32_t shuffle = i.InputInt32(2);
2696 for (int i = 0; i < 4; i++) {
2697 VRegister src = src0;
2698 int lane = shuffle & 0x7;
2699 if (lane >= 4) {
2700 src = src1;
2701 lane &= 0x3;
2702 }
2703 __ Mov(dst, i, src, lane);
2704 shuffle >>= 8;
2705 }
2706 break;
2707 }
2708 SIMD_BINOP_CASE(kArm64S32x4ZipLeft, Zip1, 4S);
2709 SIMD_BINOP_CASE(kArm64S32x4ZipRight, Zip2, 4S);
2710 SIMD_BINOP_CASE(kArm64S32x4UnzipLeft, Uzp1, 4S);
2711 SIMD_BINOP_CASE(kArm64S32x4UnzipRight, Uzp2, 4S);
2712 SIMD_BINOP_CASE(kArm64S32x4TransposeLeft, Trn1, 4S);
2713 SIMD_BINOP_CASE(kArm64S32x4TransposeRight, Trn2, 4S);
2714 SIMD_BINOP_CASE(kArm64S16x8ZipLeft, Zip1, 8H);
2715 SIMD_BINOP_CASE(kArm64S16x8ZipRight, Zip2, 8H);
2716 SIMD_BINOP_CASE(kArm64S16x8UnzipLeft, Uzp1, 8H);
2717 SIMD_BINOP_CASE(kArm64S16x8UnzipRight, Uzp2, 8H);
2718 SIMD_BINOP_CASE(kArm64S16x8TransposeLeft, Trn1, 8H);
2719 SIMD_BINOP_CASE(kArm64S16x8TransposeRight, Trn2, 8H);
2720 SIMD_BINOP_CASE(kArm64S8x16ZipLeft, Zip1, 16B);
2721 SIMD_BINOP_CASE(kArm64S8x16ZipRight, Zip2, 16B);
2722 SIMD_BINOP_CASE(kArm64S8x16UnzipLeft, Uzp1, 16B);
2723 SIMD_BINOP_CASE(kArm64S8x16UnzipRight, Uzp2, 16B);
2724 SIMD_BINOP_CASE(kArm64S8x16TransposeLeft, Trn1, 16B);
2725 SIMD_BINOP_CASE(kArm64S8x16TransposeRight, Trn2, 16B);
2726 case kArm64S8x16Concat: {
2727 __ Ext(i.OutputSimd128Register().V16B(), i.InputSimd128Register(0).V16B(),
2728 i.InputSimd128Register(1).V16B(), i.InputInt4(2));
2729 break;
2730 }
2731 case kArm64I8x16Swizzle: {
2732 __ Tbl(i.OutputSimd128Register().V16B(), i.InputSimd128Register(0).V16B(),
2733 i.InputSimd128Register(1).V16B());
2734 break;
2735 }
2736 case kArm64I8x16Shuffle: {
2737 Simd128Register dst = i.OutputSimd128Register().V16B(),
2738 src0 = i.InputSimd128Register(0).V16B(),
2739 src1 = i.InputSimd128Register(1).V16B();
2740 // Unary shuffle table is in src0, binary shuffle table is in src0, src1,
2741 // which must be consecutive.
2742 if (src0 != src1) {
2743 DCHECK(AreConsecutive(src0, src1));
2744 }
2745
2746 int64_t imm1 = make_uint64(i.InputInt32(3), i.InputInt32(2));
2747 int64_t imm2 = make_uint64(i.InputInt32(5), i.InputInt32(4));
2748 DCHECK_EQ(0, (imm1 | imm2) & (src0 == src1 ? 0xF0F0F0F0F0F0F0F0
2749 : 0xE0E0E0E0E0E0E0E0));
2750
2751 UseScratchRegisterScope scope(tasm());
2752 VRegister temp = scope.AcquireV(kFormat16B);
2753 __ Movi(temp, imm2, imm1);
2754
2755 if (src0 == src1) {
2756 __ Tbl(dst, src0, temp.V16B());
2757 } else {
2758 __ Tbl(dst, src0, src1, temp.V16B());
2759 }
2760 break;
2761 }
2762 SIMD_UNOP_CASE(kArm64S32x2Reverse, Rev64, 4S);
2763 SIMD_UNOP_CASE(kArm64S16x4Reverse, Rev64, 8H);
2764 SIMD_UNOP_CASE(kArm64S16x2Reverse, Rev32, 8H);
2765 SIMD_UNOP_CASE(kArm64S8x8Reverse, Rev64, 16B);
2766 SIMD_UNOP_CASE(kArm64S8x4Reverse, Rev32, 16B);
2767 SIMD_UNOP_CASE(kArm64S8x2Reverse, Rev16, 16B);
2768 case kArm64LoadSplat: {
2769 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
2770 VectorFormat f = VectorFormatFillQ(LaneSizeField::decode(opcode));
2771 __ ld1r(i.OutputSimd128Register().Format(f), i.MemoryOperand(0));
2772 break;
2773 }
2774 case kArm64LoadLane: {
2775 DCHECK_EQ(i.OutputSimd128Register(), i.InputSimd128Register(0));
2776 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
2777 VectorFormat f = VectorFormatFillQ(LaneSizeField::decode(opcode));
2778 int laneidx = i.InputInt8(1);
2779 __ ld1(i.OutputSimd128Register().Format(f), laneidx, i.MemoryOperand(2));
2780 break;
2781 }
2782 case kArm64StoreLane: {
2783 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
2784 VectorFormat f = VectorFormatFillQ(LaneSizeField::decode(opcode));
2785 int laneidx = i.InputInt8(1);
2786 __ st1(i.InputSimd128Register(0).Format(f), laneidx, i.MemoryOperand(2));
2787 break;
2788 }
2789 case kArm64S128Load8x8S: {
2790 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
2791 __ Ldr(i.OutputSimd128Register().V8B(), i.MemoryOperand(0));
2792 __ Sxtl(i.OutputSimd128Register().V8H(), i.OutputSimd128Register().V8B());
2793 break;
2794 }
2795 case kArm64S128Load8x8U: {
2796 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
2797 __ Ldr(i.OutputSimd128Register().V8B(), i.MemoryOperand(0));
2798 __ Uxtl(i.OutputSimd128Register().V8H(), i.OutputSimd128Register().V8B());
2799 break;
2800 }
2801 case kArm64S128Load16x4S: {
2802 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
2803 __ Ldr(i.OutputSimd128Register().V4H(), i.MemoryOperand(0));
2804 __ Sxtl(i.OutputSimd128Register().V4S(), i.OutputSimd128Register().V4H());
2805 break;
2806 }
2807 case kArm64S128Load16x4U: {
2808 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
2809 __ Ldr(i.OutputSimd128Register().V4H(), i.MemoryOperand(0));
2810 __ Uxtl(i.OutputSimd128Register().V4S(), i.OutputSimd128Register().V4H());
2811 break;
2812 }
2813 case kArm64S128Load32x2S: {
2814 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
2815 __ Ldr(i.OutputSimd128Register().V2S(), i.MemoryOperand(0));
2816 __ Sxtl(i.OutputSimd128Register().V2D(), i.OutputSimd128Register().V2S());
2817 break;
2818 }
2819 case kArm64S128Load32x2U: {
2820 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
2821 __ Ldr(i.OutputSimd128Register().V2S(), i.MemoryOperand(0));
2822 __ Uxtl(i.OutputSimd128Register().V2D(), i.OutputSimd128Register().V2S());
2823 break;
2824 }
2825 case kArm64I64x2AllTrue: {
2826 __ I64x2AllTrue(i.OutputRegister32(), i.InputSimd128Register(0));
2827 break;
2828 }
2829 #define SIMD_REDUCE_OP_CASE(Op, Instr, format, FORMAT) \
2830 case Op: { \
2831 UseScratchRegisterScope scope(tasm()); \
2832 VRegister temp = scope.AcquireV(format); \
2833 __ Instr(temp, i.InputSimd128Register(0).V##FORMAT()); \
2834 __ Umov(i.OutputRegister32(), temp, 0); \
2835 __ Cmp(i.OutputRegister32(), 0); \
2836 __ Cset(i.OutputRegister32(), ne); \
2837 break; \
2838 }
2839 // For AnyTrue, the format does not matter.
2840 SIMD_REDUCE_OP_CASE(kArm64V128AnyTrue, Umaxv, kFormatS, 4S);
2841 SIMD_REDUCE_OP_CASE(kArm64I32x4AllTrue, Uminv, kFormatS, 4S);
2842 SIMD_REDUCE_OP_CASE(kArm64I16x8AllTrue, Uminv, kFormatH, 8H);
2843 SIMD_REDUCE_OP_CASE(kArm64I8x16AllTrue, Uminv, kFormatB, 16B);
2844 }
2845 return kSuccess;
2846 }
2847
2848 #undef SIMD_UNOP_CASE
2849 #undef SIMD_UNOP_LANE_SIZE_CASE
2850 #undef SIMD_BINOP_CASE
2851 #undef SIMD_BINOP_LANE_SIZE_CASE
2852 #undef SIMD_DESTRUCTIVE_BINOP_CASE
2853 #undef SIMD_DESTRUCTIVE_BINOP_LANE_SIZE_CASE
2854 #undef SIMD_REDUCE_OP_CASE
2855 #undef ASSEMBLE_SIMD_SHIFT_LEFT
2856 #undef ASSEMBLE_SIMD_SHIFT_RIGHT
2857
2858 // Assemble branches after this instruction.
AssembleArchBranch(Instruction * instr,BranchInfo * branch)2859 void CodeGenerator::AssembleArchBranch(Instruction* instr, BranchInfo* branch) {
2860 Arm64OperandConverter i(this, instr);
2861 Label* tlabel = branch->true_label;
2862 Label* flabel = branch->false_label;
2863 FlagsCondition condition = branch->condition;
2864 ArchOpcode opcode = instr->arch_opcode();
2865
2866 if (opcode == kArm64CompareAndBranch32) {
2867 switch (condition) {
2868 case kEqual:
2869 __ Cbz(i.InputRegister32(0), tlabel);
2870 break;
2871 case kNotEqual:
2872 __ Cbnz(i.InputRegister32(0), tlabel);
2873 break;
2874 default:
2875 UNREACHABLE();
2876 }
2877 } else if (opcode == kArm64CompareAndBranch) {
2878 switch (condition) {
2879 case kEqual:
2880 __ Cbz(i.InputRegister64(0), tlabel);
2881 break;
2882 case kNotEqual:
2883 __ Cbnz(i.InputRegister64(0), tlabel);
2884 break;
2885 default:
2886 UNREACHABLE();
2887 }
2888 } else if (opcode == kArm64TestAndBranch32) {
2889 switch (condition) {
2890 case kEqual:
2891 __ Tbz(i.InputRegister32(0), i.InputInt5(1), tlabel);
2892 break;
2893 case kNotEqual:
2894 __ Tbnz(i.InputRegister32(0), i.InputInt5(1), tlabel);
2895 break;
2896 default:
2897 UNREACHABLE();
2898 }
2899 } else if (opcode == kArm64TestAndBranch) {
2900 switch (condition) {
2901 case kEqual:
2902 __ Tbz(i.InputRegister64(0), i.InputInt6(1), tlabel);
2903 break;
2904 case kNotEqual:
2905 __ Tbnz(i.InputRegister64(0), i.InputInt6(1), tlabel);
2906 break;
2907 default:
2908 UNREACHABLE();
2909 }
2910 } else {
2911 Condition cc = FlagsConditionToCondition(condition);
2912 __ B(cc, tlabel);
2913 }
2914 if (!branch->fallthru) __ B(flabel); // no fallthru to flabel.
2915 }
2916
AssembleArchDeoptBranch(Instruction * instr,BranchInfo * branch)2917 void CodeGenerator::AssembleArchDeoptBranch(Instruction* instr,
2918 BranchInfo* branch) {
2919 AssembleArchBranch(instr, branch);
2920 }
2921
AssembleArchJumpRegardlessOfAssemblyOrder(RpoNumber target)2922 void CodeGenerator::AssembleArchJumpRegardlessOfAssemblyOrder(
2923 RpoNumber target) {
2924 __ B(GetLabel(target));
2925 }
2926
2927 #if V8_ENABLE_WEBASSEMBLY
AssembleArchTrap(Instruction * instr,FlagsCondition condition)2928 void CodeGenerator::AssembleArchTrap(Instruction* instr,
2929 FlagsCondition condition) {
2930 auto ool = zone()->New<WasmOutOfLineTrap>(this, instr);
2931 Label* tlabel = ool->entry();
2932 Condition cc = FlagsConditionToCondition(condition);
2933 __ B(cc, tlabel);
2934 }
2935 #endif // V8_ENABLE_WEBASSEMBLY
2936
2937 // Assemble boolean materializations after this instruction.
AssembleArchBoolean(Instruction * instr,FlagsCondition condition)2938 void CodeGenerator::AssembleArchBoolean(Instruction* instr,
2939 FlagsCondition condition) {
2940 Arm64OperandConverter i(this, instr);
2941
2942 // Materialize a full 64-bit 1 or 0 value. The result register is always the
2943 // last output of the instruction.
2944 DCHECK_NE(0u, instr->OutputCount());
2945 Register reg = i.OutputRegister(instr->OutputCount() - 1);
2946 Condition cc = FlagsConditionToCondition(condition);
2947 __ Cset(reg, cc);
2948 }
2949
AssembleArchSelect(Instruction * instr,FlagsCondition condition)2950 void CodeGenerator::AssembleArchSelect(Instruction* instr,
2951 FlagsCondition condition) {
2952 Arm64OperandConverter i(this, instr);
2953 MachineRepresentation rep =
2954 LocationOperand::cast(instr->OutputAt(0))->representation();
2955 Condition cc = FlagsConditionToCondition(condition);
2956 // We don't now how many inputs were consumed by the condition, so we have to
2957 // calculate the indices of the last two inputs.
2958 DCHECK_GE(instr->InputCount(), 2);
2959 size_t true_value_index = instr->InputCount() - 2;
2960 size_t false_value_index = instr->InputCount() - 1;
2961 if (rep == MachineRepresentation::kFloat32) {
2962 __ Fcsel(i.OutputFloat32Register(),
2963 i.InputFloat32Register(true_value_index),
2964 i.InputFloat32Register(false_value_index), cc);
2965 } else {
2966 DCHECK_EQ(rep, MachineRepresentation::kFloat64);
2967 __ Fcsel(i.OutputFloat64Register(),
2968 i.InputFloat64Register(true_value_index),
2969 i.InputFloat64Register(false_value_index), cc);
2970 }
2971 }
2972
AssembleArchBinarySearchSwitch(Instruction * instr)2973 void CodeGenerator::AssembleArchBinarySearchSwitch(Instruction* instr) {
2974 Arm64OperandConverter i(this, instr);
2975 Register input = i.InputRegister32(0);
2976 std::vector<std::pair<int32_t, Label*>> cases;
2977 for (size_t index = 2; index < instr->InputCount(); index += 2) {
2978 cases.push_back({i.InputInt32(index + 0), GetLabel(i.InputRpo(index + 1))});
2979 }
2980 AssembleArchBinarySearchSwitchRange(input, i.InputRpo(1), cases.data(),
2981 cases.data() + cases.size());
2982 }
2983
AssembleArchTableSwitch(Instruction * instr)2984 void CodeGenerator::AssembleArchTableSwitch(Instruction* instr) {
2985 Arm64OperandConverter i(this, instr);
2986 UseScratchRegisterScope scope(tasm());
2987 Register input = i.InputRegister32(0);
2988 Register temp = scope.AcquireX();
2989 size_t const case_count = instr->InputCount() - 2;
2990 Label table;
2991 __ Cmp(input, case_count);
2992 __ B(hs, GetLabel(i.InputRpo(1)));
2993 __ Adr(temp, &table);
2994 int entry_size_log2 = 2;
2995 #ifdef V8_ENABLE_CONTROL_FLOW_INTEGRITY
2996 ++entry_size_log2; // Account for BTI.
2997 #endif
2998 __ Add(temp, temp, Operand(input, UXTW, entry_size_log2));
2999 __ Br(temp);
3000 {
3001 TurboAssembler::BlockPoolsScope block_pools(tasm(),
3002 case_count * kInstrSize);
3003 __ Bind(&table);
3004 for (size_t index = 0; index < case_count; ++index) {
3005 __ JumpTarget();
3006 __ B(GetLabel(i.InputRpo(index + 2)));
3007 }
3008 __ JumpTarget();
3009 }
3010 }
3011
FinishFrame(Frame * frame)3012 void CodeGenerator::FinishFrame(Frame* frame) {
3013 auto call_descriptor = linkage()->GetIncomingDescriptor();
3014
3015 // Save FP registers.
3016 CPURegList saves_fp =
3017 CPURegList(kDRegSizeInBits, call_descriptor->CalleeSavedFPRegisters());
3018 int saved_count = saves_fp.Count();
3019 if (saved_count != 0) {
3020 DCHECK(saves_fp.bits() == CPURegList::GetCalleeSavedV().bits());
3021 frame->AllocateSavedCalleeRegisterSlots(saved_count *
3022 (kDoubleSize / kSystemPointerSize));
3023 }
3024
3025 CPURegList saves =
3026 CPURegList(kXRegSizeInBits, call_descriptor->CalleeSavedRegisters());
3027 saved_count = saves.Count();
3028 if (saved_count != 0) {
3029 frame->AllocateSavedCalleeRegisterSlots(saved_count);
3030 }
3031 frame->AlignFrame(16);
3032 }
3033
AssembleConstructFrame()3034 void CodeGenerator::AssembleConstructFrame() {
3035 auto call_descriptor = linkage()->GetIncomingDescriptor();
3036 __ AssertSpAligned();
3037
3038 // The frame has been previously padded in CodeGenerator::FinishFrame().
3039 DCHECK_EQ(frame()->GetTotalFrameSlotCount() % 2, 0);
3040 int required_slots =
3041 frame()->GetTotalFrameSlotCount() - frame()->GetFixedSlotCount();
3042
3043 CPURegList saves =
3044 CPURegList(kXRegSizeInBits, call_descriptor->CalleeSavedRegisters());
3045 DCHECK_EQ(saves.Count() % 2, 0);
3046 CPURegList saves_fp =
3047 CPURegList(kDRegSizeInBits, call_descriptor->CalleeSavedFPRegisters());
3048 DCHECK_EQ(saves_fp.Count() % 2, 0);
3049 // The number of return slots should be even after aligning the Frame.
3050 const int returns = frame()->GetReturnSlotCount();
3051 DCHECK_EQ(returns % 2, 0);
3052
3053 if (frame_access_state()->has_frame()) {
3054 // Link the frame
3055 if (call_descriptor->IsJSFunctionCall()) {
3056 STATIC_ASSERT(InterpreterFrameConstants::kFixedFrameSize % 16 == 8);
3057 DCHECK_EQ(required_slots % 2, 1);
3058 __ Prologue();
3059 // Update required_slots count since we have just claimed one extra slot.
3060 STATIC_ASSERT(TurboAssembler::kExtraSlotClaimedByPrologue == 1);
3061 required_slots -= TurboAssembler::kExtraSlotClaimedByPrologue;
3062 } else {
3063 __ Push<TurboAssembler::kSignLR>(lr, fp);
3064 __ Mov(fp, sp);
3065 }
3066 unwinding_info_writer_.MarkFrameConstructed(__ pc_offset());
3067
3068 // Create OSR entry if applicable
3069 if (info()->is_osr()) {
3070 // TurboFan OSR-compiled functions cannot be entered directly.
3071 __ Abort(AbortReason::kShouldNotDirectlyEnterOsrFunction);
3072
3073 // Unoptimized code jumps directly to this entrypoint while the
3074 // unoptimized frame is still on the stack. Optimized code uses OSR values
3075 // directly from the unoptimized frame. Thus, all that needs to be done is
3076 // to allocate the remaining stack slots.
3077 __ RecordComment("-- OSR entrypoint --");
3078 osr_pc_offset_ = __ pc_offset();
3079 __ CodeEntry();
3080 size_t unoptimized_frame_slots = osr_helper()->UnoptimizedFrameSlots();
3081 DCHECK(call_descriptor->IsJSFunctionCall());
3082 DCHECK_EQ(unoptimized_frame_slots % 2, 1);
3083 // One unoptimized frame slot has already been claimed when the actual
3084 // arguments count was pushed.
3085 required_slots -=
3086 unoptimized_frame_slots - TurboAssembler::kExtraSlotClaimedByPrologue;
3087 }
3088
3089 #if V8_ENABLE_WEBASSEMBLY
3090 if (info()->IsWasm() && required_slots * kSystemPointerSize > 4 * KB) {
3091 // For WebAssembly functions with big frames we have to do the stack
3092 // overflow check before we construct the frame. Otherwise we may not
3093 // have enough space on the stack to call the runtime for the stack
3094 // overflow.
3095 Label done;
3096 // If the frame is bigger than the stack, we throw the stack overflow
3097 // exception unconditionally. Thereby we can avoid the integer overflow
3098 // check in the condition code.
3099 if (required_slots * kSystemPointerSize < FLAG_stack_size * KB) {
3100 UseScratchRegisterScope scope(tasm());
3101 Register scratch = scope.AcquireX();
3102 __ Ldr(scratch, FieldMemOperand(
3103 kWasmInstanceRegister,
3104 WasmInstanceObject::kRealStackLimitAddressOffset));
3105 __ Ldr(scratch, MemOperand(scratch));
3106 __ Add(scratch, scratch, required_slots * kSystemPointerSize);
3107 __ Cmp(sp, scratch);
3108 __ B(hs, &done);
3109 }
3110
3111 {
3112 // Finish the frame that hasn't been fully built yet.
3113 UseScratchRegisterScope temps(tasm());
3114 Register scratch = temps.AcquireX();
3115 __ Mov(scratch,
3116 StackFrame::TypeToMarker(info()->GetOutputStackFrameType()));
3117 __ Push(scratch, kWasmInstanceRegister);
3118 }
3119
3120 __ Call(wasm::WasmCode::kWasmStackOverflow, RelocInfo::WASM_STUB_CALL);
3121 // The call does not return, hence we can ignore any references and just
3122 // define an empty safepoint.
3123 ReferenceMap* reference_map = zone()->New<ReferenceMap>(zone());
3124 RecordSafepoint(reference_map);
3125 if (FLAG_debug_code) __ Brk(0);
3126 __ Bind(&done);
3127 }
3128 #endif // V8_ENABLE_WEBASSEMBLY
3129
3130 // Skip callee-saved slots, which are pushed below.
3131 required_slots -= saves.Count();
3132 required_slots -= saves_fp.Count();
3133 required_slots -= returns;
3134
3135 // Build remainder of frame, including accounting for and filling-in
3136 // frame-specific header information, i.e. claiming the extra slot that
3137 // other platforms explicitly push for STUB (code object) frames and frames
3138 // recording their argument count.
3139 switch (call_descriptor->kind()) {
3140 case CallDescriptor::kCallJSFunction:
3141 __ Claim(required_slots);
3142 break;
3143 case CallDescriptor::kCallCodeObject: {
3144 UseScratchRegisterScope temps(tasm());
3145 Register scratch = temps.AcquireX();
3146 __ Mov(scratch,
3147 StackFrame::TypeToMarker(info()->GetOutputStackFrameType()));
3148 __ Push(scratch, padreg);
3149 // One of the extra slots has just been claimed when pushing the frame
3150 // type marker above. We also know that we have at least one slot to
3151 // claim here, as the typed frame has an odd number of fixed slots, and
3152 // all other parts of the total frame slots are even, leaving
3153 // {required_slots} to be odd.
3154 DCHECK_GE(required_slots, 1);
3155 __ Claim(required_slots - 1);
3156 break;
3157 }
3158 #if V8_ENABLE_WEBASSEMBLY
3159 case CallDescriptor::kCallWasmFunction: {
3160 UseScratchRegisterScope temps(tasm());
3161 Register scratch = temps.AcquireX();
3162 __ Mov(scratch,
3163 StackFrame::TypeToMarker(info()->GetOutputStackFrameType()));
3164 __ Push(scratch, kWasmInstanceRegister);
3165 __ Claim(required_slots);
3166 break;
3167 }
3168 case CallDescriptor::kCallWasmImportWrapper:
3169 case CallDescriptor::kCallWasmCapiFunction: {
3170 UseScratchRegisterScope temps(tasm());
3171 Register scratch = temps.AcquireX();
3172 __ Mov(scratch,
3173 StackFrame::TypeToMarker(info()->GetOutputStackFrameType()));
3174 __ Push(scratch, kWasmInstanceRegister);
3175 int extra_slots =
3176 call_descriptor->kind() == CallDescriptor::kCallWasmImportWrapper
3177 ? 0 // Import wrapper: none.
3178 : 1; // C-API function: PC.
3179 __ Claim(required_slots + extra_slots);
3180 break;
3181 }
3182 #endif // V8_ENABLE_WEBASSEMBLY
3183 case CallDescriptor::kCallAddress:
3184 #if V8_ENABLE_WEBASSEMBLY
3185 if (info()->GetOutputStackFrameType() == StackFrame::C_WASM_ENTRY) {
3186 UseScratchRegisterScope temps(tasm());
3187 Register scratch = temps.AcquireX();
3188 __ Mov(scratch, StackFrame::TypeToMarker(StackFrame::C_WASM_ENTRY));
3189 __ Push(scratch, padreg);
3190 // The additional slot will be used for the saved c_entry_fp.
3191 }
3192 #endif // V8_ENABLE_WEBASSEMBLY
3193 __ Claim(required_slots);
3194 break;
3195 default:
3196 UNREACHABLE();
3197 }
3198 }
3199
3200 // Save FP registers.
3201 DCHECK_IMPLIES(saves_fp.Count() != 0,
3202 saves_fp.bits() == CPURegList::GetCalleeSavedV().bits());
3203 __ PushCPURegList(saves_fp);
3204
3205 // Save registers.
3206 __ PushCPURegList(saves);
3207
3208 if (returns != 0) {
3209 __ Claim(returns);
3210 }
3211 }
3212
AssembleReturn(InstructionOperand * additional_pop_count)3213 void CodeGenerator::AssembleReturn(InstructionOperand* additional_pop_count) {
3214 auto call_descriptor = linkage()->GetIncomingDescriptor();
3215
3216 const int returns = RoundUp(frame()->GetReturnSlotCount(), 2);
3217 if (returns != 0) {
3218 __ Drop(returns);
3219 }
3220
3221 // Restore registers.
3222 CPURegList saves =
3223 CPURegList(kXRegSizeInBits, call_descriptor->CalleeSavedRegisters());
3224 __ PopCPURegList(saves);
3225
3226 // Restore fp registers.
3227 CPURegList saves_fp =
3228 CPURegList(kDRegSizeInBits, call_descriptor->CalleeSavedFPRegisters());
3229 __ PopCPURegList(saves_fp);
3230
3231 unwinding_info_writer_.MarkBlockWillExit();
3232
3233 const int parameter_slots =
3234 static_cast<int>(call_descriptor->ParameterSlotCount());
3235 Arm64OperandConverter g(this, nullptr);
3236
3237 // {aditional_pop_count} is only greater than zero if {parameter_slots = 0}.
3238 // Check RawMachineAssembler::PopAndReturn.
3239 if (parameter_slots != 0) {
3240 if (additional_pop_count->IsImmediate()) {
3241 DCHECK_EQ(g.ToConstant(additional_pop_count).ToInt32(), 0);
3242 } else if (FLAG_debug_code) {
3243 __ cmp(g.ToRegister(additional_pop_count), Operand(0));
3244 __ Assert(eq, AbortReason::kUnexpectedAdditionalPopValue);
3245 }
3246 }
3247
3248 Register argc_reg = x3;
3249 // Functions with JS linkage have at least one parameter (the receiver).
3250 // If {parameter_slots} == 0, it means it is a builtin with
3251 // kDontAdaptArgumentsSentinel, which takes care of JS arguments popping
3252 // itself.
3253 const bool drop_jsargs = parameter_slots != 0 &&
3254 frame_access_state()->has_frame() &&
3255 call_descriptor->IsJSFunctionCall();
3256 if (call_descriptor->IsCFunctionCall()) {
3257 AssembleDeconstructFrame();
3258 } else if (frame_access_state()->has_frame()) {
3259 // Canonicalize JSFunction return sites for now unless they have an variable
3260 // number of stack slot pops.
3261 if (additional_pop_count->IsImmediate() &&
3262 g.ToConstant(additional_pop_count).ToInt32() == 0) {
3263 if (return_label_.is_bound()) {
3264 __ B(&return_label_);
3265 return;
3266 } else {
3267 __ Bind(&return_label_);
3268 }
3269 }
3270 if (drop_jsargs) {
3271 // Get the actual argument count.
3272 DCHECK(!call_descriptor->CalleeSavedRegisters().has(argc_reg));
3273 __ Ldr(argc_reg, MemOperand(fp, StandardFrameConstants::kArgCOffset));
3274 }
3275 AssembleDeconstructFrame();
3276 }
3277
3278 if (drop_jsargs) {
3279 // We must pop all arguments from the stack (including the receiver). This
3280 // number of arguments is given by max(1 + argc_reg, parameter_slots).
3281 Label argc_reg_has_final_count;
3282 DCHECK(!call_descriptor->CalleeSavedRegisters().has(argc_reg));
3283 if (parameter_slots > 1) {
3284 __ Cmp(argc_reg, Operand(parameter_slots));
3285 __ B(&argc_reg_has_final_count, ge);
3286 __ Mov(argc_reg, Operand(parameter_slots));
3287 __ Bind(&argc_reg_has_final_count);
3288 }
3289 __ DropArguments(argc_reg);
3290 } else if (additional_pop_count->IsImmediate()) {
3291 int additional_count = g.ToConstant(additional_pop_count).ToInt32();
3292 __ DropArguments(parameter_slots + additional_count);
3293 } else if (parameter_slots == 0) {
3294 __ DropArguments(g.ToRegister(additional_pop_count));
3295 } else {
3296 // {additional_pop_count} is guaranteed to be zero if {parameter_slots !=
3297 // 0}. Check RawMachineAssembler::PopAndReturn.
3298 __ DropArguments(parameter_slots);
3299 }
3300 __ AssertSpAligned();
3301 __ Ret();
3302 }
3303
FinishCode()3304 void CodeGenerator::FinishCode() { __ ForceConstantPoolEmissionWithoutJump(); }
3305
PrepareForDeoptimizationExits(ZoneDeque<DeoptimizationExit * > * exits)3306 void CodeGenerator::PrepareForDeoptimizationExits(
3307 ZoneDeque<DeoptimizationExit*>* exits) {
3308 __ ForceConstantPoolEmissionWithoutJump();
3309 // We are conservative here, reserving sufficient space for the largest deopt
3310 // kind.
3311 DCHECK_GE(Deoptimizer::kLazyDeoptExitSize, Deoptimizer::kEagerDeoptExitSize);
3312 __ CheckVeneerPool(
3313 false, false,
3314 static_cast<int>(exits->size()) * Deoptimizer::kLazyDeoptExitSize);
3315
3316 // Check which deopt kinds exist in this Code object, to avoid emitting jumps
3317 // to unused entries.
3318 bool saw_deopt_kind[kDeoptimizeKindCount] = {false};
3319 for (auto exit : *exits) {
3320 saw_deopt_kind[static_cast<int>(exit->kind())] = true;
3321 }
3322
3323 // Emit the jumps to deoptimization entries.
3324 UseScratchRegisterScope scope(tasm());
3325 Register scratch = scope.AcquireX();
3326 STATIC_ASSERT(static_cast<int>(kFirstDeoptimizeKind) == 0);
3327 for (int i = 0; i < kDeoptimizeKindCount; i++) {
3328 if (!saw_deopt_kind[i]) continue;
3329 DeoptimizeKind kind = static_cast<DeoptimizeKind>(i);
3330 __ bind(&jump_deoptimization_entry_labels_[i]);
3331 __ LoadEntryFromBuiltin(Deoptimizer::GetDeoptimizationEntry(kind), scratch);
3332 __ Jump(scratch);
3333 }
3334 }
3335
AssembleMove(InstructionOperand * source,InstructionOperand * destination)3336 void CodeGenerator::AssembleMove(InstructionOperand* source,
3337 InstructionOperand* destination) {
3338 Arm64OperandConverter g(this, nullptr);
3339 // Helper function to write the given constant to the dst register.
3340 auto MoveConstantToRegister = [&](Register dst, Constant src) {
3341 if (src.type() == Constant::kHeapObject) {
3342 Handle<HeapObject> src_object = src.ToHeapObject();
3343 RootIndex index;
3344 if (IsMaterializableFromRoot(src_object, &index)) {
3345 __ LoadRoot(dst, index);
3346 } else {
3347 __ Mov(dst, src_object);
3348 }
3349 } else if (src.type() == Constant::kCompressedHeapObject) {
3350 Handle<HeapObject> src_object = src.ToHeapObject();
3351 RootIndex index;
3352 if (IsMaterializableFromRoot(src_object, &index)) {
3353 __ LoadRoot(dst, index);
3354 } else {
3355 // TODO(v8:8977): Even though this mov happens on 32 bits (Note the
3356 // .W()) and we are passing along the RelocInfo, we still haven't made
3357 // the address embedded in the code-stream actually be compressed.
3358 __ Mov(dst.W(),
3359 Immediate(src_object, RelocInfo::COMPRESSED_EMBEDDED_OBJECT));
3360 }
3361 } else {
3362 __ Mov(dst, g.ToImmediate(source));
3363 }
3364 };
3365 switch (MoveType::InferMove(source, destination)) {
3366 case MoveType::kRegisterToRegister:
3367 if (source->IsRegister()) {
3368 __ Mov(g.ToRegister(destination), g.ToRegister(source));
3369 } else if (source->IsFloatRegister() || source->IsDoubleRegister()) {
3370 __ Mov(g.ToDoubleRegister(destination), g.ToDoubleRegister(source));
3371 } else {
3372 DCHECK(source->IsSimd128Register());
3373 __ Mov(g.ToDoubleRegister(destination).Q(),
3374 g.ToDoubleRegister(source).Q());
3375 }
3376 return;
3377 case MoveType::kRegisterToStack: {
3378 MemOperand dst = g.ToMemOperand(destination, tasm());
3379 if (source->IsRegister()) {
3380 __ Str(g.ToRegister(source), dst);
3381 } else {
3382 VRegister src = g.ToDoubleRegister(source);
3383 if (source->IsFloatRegister() || source->IsDoubleRegister()) {
3384 __ Str(src, dst);
3385 } else {
3386 DCHECK(source->IsSimd128Register());
3387 __ Str(src.Q(), dst);
3388 }
3389 }
3390 return;
3391 }
3392 case MoveType::kStackToRegister: {
3393 MemOperand src = g.ToMemOperand(source, tasm());
3394 if (destination->IsRegister()) {
3395 __ Ldr(g.ToRegister(destination), src);
3396 } else {
3397 VRegister dst = g.ToDoubleRegister(destination);
3398 if (destination->IsFloatRegister() || destination->IsDoubleRegister()) {
3399 __ Ldr(dst, src);
3400 } else {
3401 DCHECK(destination->IsSimd128Register());
3402 __ Ldr(dst.Q(), src);
3403 }
3404 }
3405 return;
3406 }
3407 case MoveType::kStackToStack: {
3408 MemOperand src = g.ToMemOperand(source, tasm());
3409 MemOperand dst = g.ToMemOperand(destination, tasm());
3410 if (source->IsSimd128StackSlot()) {
3411 UseScratchRegisterScope scope(tasm());
3412 VRegister temp = scope.AcquireQ();
3413 __ Ldr(temp, src);
3414 __ Str(temp, dst);
3415 } else {
3416 UseScratchRegisterScope scope(tasm());
3417 Register temp = scope.AcquireX();
3418 __ Ldr(temp, src);
3419 __ Str(temp, dst);
3420 }
3421 return;
3422 }
3423 case MoveType::kConstantToRegister: {
3424 Constant src = g.ToConstant(source);
3425 if (destination->IsRegister()) {
3426 MoveConstantToRegister(g.ToRegister(destination), src);
3427 } else {
3428 VRegister dst = g.ToDoubleRegister(destination);
3429 if (destination->IsFloatRegister()) {
3430 __ Fmov(dst.S(), src.ToFloat32());
3431 } else {
3432 DCHECK(destination->IsDoubleRegister());
3433 __ Fmov(dst, src.ToFloat64().value());
3434 }
3435 }
3436 return;
3437 }
3438 case MoveType::kConstantToStack: {
3439 Constant src = g.ToConstant(source);
3440 MemOperand dst = g.ToMemOperand(destination, tasm());
3441 if (destination->IsStackSlot()) {
3442 UseScratchRegisterScope scope(tasm());
3443 Register temp = scope.AcquireX();
3444 MoveConstantToRegister(temp, src);
3445 __ Str(temp, dst);
3446 } else if (destination->IsFloatStackSlot()) {
3447 if (bit_cast<int32_t>(src.ToFloat32()) == 0) {
3448 __ Str(wzr, dst);
3449 } else {
3450 UseScratchRegisterScope scope(tasm());
3451 VRegister temp = scope.AcquireS();
3452 __ Fmov(temp, src.ToFloat32());
3453 __ Str(temp, dst);
3454 }
3455 } else {
3456 DCHECK(destination->IsDoubleStackSlot());
3457 if (src.ToFloat64().AsUint64() == 0) {
3458 __ Str(xzr, dst);
3459 } else {
3460 UseScratchRegisterScope scope(tasm());
3461 VRegister temp = scope.AcquireD();
3462 __ Fmov(temp, src.ToFloat64().value());
3463 __ Str(temp, dst);
3464 }
3465 }
3466 return;
3467 }
3468 }
3469 UNREACHABLE();
3470 }
3471
AssembleSwap(InstructionOperand * source,InstructionOperand * destination)3472 void CodeGenerator::AssembleSwap(InstructionOperand* source,
3473 InstructionOperand* destination) {
3474 Arm64OperandConverter g(this, nullptr);
3475 switch (MoveType::InferSwap(source, destination)) {
3476 case MoveType::kRegisterToRegister:
3477 if (source->IsRegister()) {
3478 __ Swap(g.ToRegister(source), g.ToRegister(destination));
3479 } else {
3480 VRegister src = g.ToDoubleRegister(source);
3481 VRegister dst = g.ToDoubleRegister(destination);
3482 if (source->IsFloatRegister() || source->IsDoubleRegister()) {
3483 __ Swap(src, dst);
3484 } else {
3485 DCHECK(source->IsSimd128Register());
3486 __ Swap(src.Q(), dst.Q());
3487 }
3488 }
3489 return;
3490 case MoveType::kRegisterToStack: {
3491 UseScratchRegisterScope scope(tasm());
3492 MemOperand dst = g.ToMemOperand(destination, tasm());
3493 if (source->IsRegister()) {
3494 Register temp = scope.AcquireX();
3495 Register src = g.ToRegister(source);
3496 __ Mov(temp, src);
3497 __ Ldr(src, dst);
3498 __ Str(temp, dst);
3499 } else {
3500 UseScratchRegisterScope scope(tasm());
3501 VRegister src = g.ToDoubleRegister(source);
3502 if (source->IsFloatRegister() || source->IsDoubleRegister()) {
3503 VRegister temp = scope.AcquireD();
3504 __ Mov(temp, src);
3505 __ Ldr(src, dst);
3506 __ Str(temp, dst);
3507 } else {
3508 DCHECK(source->IsSimd128Register());
3509 VRegister temp = scope.AcquireQ();
3510 __ Mov(temp, src.Q());
3511 __ Ldr(src.Q(), dst);
3512 __ Str(temp, dst);
3513 }
3514 }
3515 return;
3516 }
3517 case MoveType::kStackToStack: {
3518 UseScratchRegisterScope scope(tasm());
3519 MemOperand src = g.ToMemOperand(source, tasm());
3520 MemOperand dst = g.ToMemOperand(destination, tasm());
3521 VRegister temp_0 = scope.AcquireD();
3522 VRegister temp_1 = scope.AcquireD();
3523 if (source->IsSimd128StackSlot()) {
3524 __ Ldr(temp_0.Q(), src);
3525 __ Ldr(temp_1.Q(), dst);
3526 __ Str(temp_0.Q(), dst);
3527 __ Str(temp_1.Q(), src);
3528 } else {
3529 __ Ldr(temp_0, src);
3530 __ Ldr(temp_1, dst);
3531 __ Str(temp_0, dst);
3532 __ Str(temp_1, src);
3533 }
3534 return;
3535 }
3536 default:
3537 UNREACHABLE();
3538 }
3539 }
3540
AssembleJumpTable(Label ** targets,size_t target_count)3541 void CodeGenerator::AssembleJumpTable(Label** targets, size_t target_count) {
3542 // On 64-bit ARM we emit the jump tables inline.
3543 UNREACHABLE();
3544 }
3545
3546 #undef __
3547
3548 } // namespace compiler
3549 } // namespace internal
3550 } // namespace v8
3551