1 // Copyright 2017 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #ifndef V8_WASM_BASELINE_ARM64_LIFTOFF_ASSEMBLER_ARM64_H_
6 #define V8_WASM_BASELINE_ARM64_LIFTOFF_ASSEMBLER_ARM64_H_
7
8 #include "src/wasm/baseline/liftoff-assembler.h"
9
10 #define BAILOUT(reason) bailout("arm64 " reason)
11
12 namespace v8 {
13 namespace internal {
14 namespace wasm {
15
16 namespace liftoff {
17
18 // Liftoff Frames.
19 //
20 // slot Frame
21 // +--------------------+---------------------------
22 // n+4 | optional padding slot to keep the stack 16 byte aligned.
23 // n+3 | parameter n |
24 // ... | ... |
25 // 4 | parameter 1 | or parameter 2
26 // 3 | parameter 0 | or parameter 1
27 // 2 | (result address) | or parameter 0
28 // -----+--------------------+---------------------------
29 // 1 | return addr (lr) |
30 // 0 | previous frame (fp)|
31 // -----+--------------------+ <-- frame ptr (fp)
32 // -1 | 0xa: WASM_COMPILED |
33 // -2 | instance |
34 // -----+--------------------+---------------------------
35 // -3 | slot 0 | ^
36 // -4 | slot 1 | |
37 // | | Frame slots
38 // | | |
39 // | | v
40 // | optional padding slot to keep the stack 16 byte aligned.
41 // -----+--------------------+ <-- stack ptr (sp)
42 //
43
44 constexpr int32_t kInstanceOffset = 2 * kPointerSize;
45 constexpr int32_t kFirstStackSlotOffset = kInstanceOffset + kPointerSize;
46 constexpr int32_t kConstantStackSpace = 0;
47
GetStackSlot(uint32_t index)48 inline MemOperand GetStackSlot(uint32_t index) {
49 int32_t offset =
50 kFirstStackSlotOffset + index * LiftoffAssembler::kStackSlotSize;
51 return MemOperand(fp, -offset);
52 }
53
GetInstanceOperand()54 inline MemOperand GetInstanceOperand() {
55 return MemOperand(fp, -kInstanceOffset);
56 }
57
GetRegFromType(const LiftoffRegister & reg,ValueType type)58 inline CPURegister GetRegFromType(const LiftoffRegister& reg, ValueType type) {
59 switch (type) {
60 case kWasmI32:
61 return reg.gp().W();
62 case kWasmI64:
63 return reg.gp().X();
64 case kWasmF32:
65 return reg.fp().S();
66 case kWasmF64:
67 return reg.fp().D();
68 default:
69 UNREACHABLE();
70 }
71 }
72
PadRegList(RegList list)73 inline CPURegList PadRegList(RegList list) {
74 if ((base::bits::CountPopulation(list) & 1) != 0) list |= padreg.bit();
75 return CPURegList(CPURegister::kRegister, kXRegSizeInBits, list);
76 }
77
PadVRegList(RegList list)78 inline CPURegList PadVRegList(RegList list) {
79 if ((base::bits::CountPopulation(list) & 1) != 0) list |= fp_scratch.bit();
80 return CPURegList(CPURegister::kVRegister, kDRegSizeInBits, list);
81 }
82
AcquireByType(UseScratchRegisterScope * temps,ValueType type)83 inline CPURegister AcquireByType(UseScratchRegisterScope* temps,
84 ValueType type) {
85 switch (type) {
86 case kWasmI32:
87 return temps->AcquireW();
88 case kWasmI64:
89 return temps->AcquireX();
90 case kWasmF32:
91 return temps->AcquireS();
92 case kWasmF64:
93 return temps->AcquireD();
94 default:
95 UNREACHABLE();
96 }
97 }
98
GetMemOp(LiftoffAssembler * assm,UseScratchRegisterScope * temps,Register addr,Register offset,uint32_t offset_imm)99 inline MemOperand GetMemOp(LiftoffAssembler* assm,
100 UseScratchRegisterScope* temps, Register addr,
101 Register offset, uint32_t offset_imm) {
102 // Wasm memory is limited to a size <2GB, so all offsets can be encoded as
103 // immediate value (in 31 bits, interpreted as signed value).
104 // If the offset is bigger, we always trap and this code is not reached.
105 DCHECK(is_uint31(offset_imm));
106 if (offset.IsValid()) {
107 if (offset_imm == 0) return MemOperand(addr.X(), offset.W(), UXTW);
108 Register tmp = temps->AcquireW();
109 assm->Add(tmp, offset.W(), offset_imm);
110 return MemOperand(addr.X(), tmp, UXTW);
111 }
112 return MemOperand(addr.X(), offset_imm);
113 }
114
115 } // namespace liftoff
116
PrepareStackFrame()117 int LiftoffAssembler::PrepareStackFrame() {
118 int offset = pc_offset();
119 InstructionAccurateScope scope(this, 1);
120 sub(sp, sp, 0);
121 return offset;
122 }
123
PatchPrepareStackFrame(int offset,uint32_t stack_slots)124 void LiftoffAssembler::PatchPrepareStackFrame(int offset,
125 uint32_t stack_slots) {
126 static_assert(kStackSlotSize == kXRegSize,
127 "kStackSlotSize must equal kXRegSize");
128 uint32_t bytes = liftoff::kConstantStackSpace + kStackSlotSize * stack_slots;
129 // The stack pointer is required to be quadword aligned.
130 // Misalignment will cause a stack alignment fault.
131 bytes = RoundUp(bytes, kQuadWordSizeInBytes);
132 if (!IsImmAddSub(bytes)) {
133 // Round the stack to a page to try to fit a add/sub immediate.
134 bytes = RoundUp(bytes, 0x1000);
135 if (!IsImmAddSub(bytes)) {
136 // Stack greater than 4M! Because this is a quite improbable case, we
137 // just fallback to Turbofan.
138 BAILOUT("Stack too big");
139 return;
140 }
141 }
142 #ifdef USE_SIMULATOR
143 // When using the simulator, deal with Liftoff which allocates the stack
144 // before checking it.
145 // TODO(arm): Remove this when the stack check mechanism will be updated.
146 if (bytes > KB / 2) {
147 BAILOUT("Stack limited to 512 bytes to avoid a bug in StackCheck");
148 return;
149 }
150 #endif
151 PatchingAssembler patching_assembler(AssemblerOptions{}, buffer_ + offset, 1);
152 patching_assembler.PatchSubSp(bytes);
153 }
154
FinishCode()155 void LiftoffAssembler::FinishCode() { CheckConstPool(true, false); }
156
AbortCompilation()157 void LiftoffAssembler::AbortCompilation() { AbortedCodeGeneration(); }
158
LoadConstant(LiftoffRegister reg,WasmValue value,RelocInfo::Mode rmode)159 void LiftoffAssembler::LoadConstant(LiftoffRegister reg, WasmValue value,
160 RelocInfo::Mode rmode) {
161 switch (value.type()) {
162 case kWasmI32:
163 Mov(reg.gp().W(), Immediate(value.to_i32(), rmode));
164 break;
165 case kWasmI64:
166 Mov(reg.gp().X(), Immediate(value.to_i64(), rmode));
167 break;
168 case kWasmF32:
169 Fmov(reg.fp().S(), value.to_f32_boxed().get_scalar());
170 break;
171 case kWasmF64:
172 Fmov(reg.fp().D(), value.to_f64_boxed().get_scalar());
173 break;
174 default:
175 UNREACHABLE();
176 }
177 }
178
LoadFromInstance(Register dst,uint32_t offset,int size)179 void LiftoffAssembler::LoadFromInstance(Register dst, uint32_t offset,
180 int size) {
181 DCHECK_LE(offset, kMaxInt);
182 Ldr(dst, liftoff::GetInstanceOperand());
183 DCHECK(size == 4 || size == 8);
184 if (size == 4) {
185 Ldr(dst.W(), MemOperand(dst, offset));
186 } else {
187 Ldr(dst, MemOperand(dst, offset));
188 }
189 }
190
SpillInstance(Register instance)191 void LiftoffAssembler::SpillInstance(Register instance) {
192 Str(instance, liftoff::GetInstanceOperand());
193 }
194
FillInstanceInto(Register dst)195 void LiftoffAssembler::FillInstanceInto(Register dst) {
196 Ldr(dst, liftoff::GetInstanceOperand());
197 }
198
Load(LiftoffRegister dst,Register src_addr,Register offset_reg,uint32_t offset_imm,LoadType type,LiftoffRegList pinned,uint32_t * protected_load_pc,bool is_load_mem)199 void LiftoffAssembler::Load(LiftoffRegister dst, Register src_addr,
200 Register offset_reg, uint32_t offset_imm,
201 LoadType type, LiftoffRegList pinned,
202 uint32_t* protected_load_pc, bool is_load_mem) {
203 UseScratchRegisterScope temps(this);
204 MemOperand src_op =
205 liftoff::GetMemOp(this, &temps, src_addr, offset_reg, offset_imm);
206 if (protected_load_pc) *protected_load_pc = pc_offset();
207 switch (type.value()) {
208 case LoadType::kI32Load8U:
209 case LoadType::kI64Load8U:
210 Ldrb(dst.gp().W(), src_op);
211 break;
212 case LoadType::kI32Load8S:
213 Ldrsb(dst.gp().W(), src_op);
214 break;
215 case LoadType::kI64Load8S:
216 Ldrsb(dst.gp().X(), src_op);
217 break;
218 case LoadType::kI32Load16U:
219 case LoadType::kI64Load16U:
220 Ldrh(dst.gp().W(), src_op);
221 break;
222 case LoadType::kI32Load16S:
223 Ldrsh(dst.gp().W(), src_op);
224 break;
225 case LoadType::kI64Load16S:
226 Ldrsh(dst.gp().X(), src_op);
227 break;
228 case LoadType::kI32Load:
229 case LoadType::kI64Load32U:
230 Ldr(dst.gp().W(), src_op);
231 break;
232 case LoadType::kI64Load32S:
233 Ldrsw(dst.gp().X(), src_op);
234 break;
235 case LoadType::kI64Load:
236 Ldr(dst.gp().X(), src_op);
237 break;
238 case LoadType::kF32Load:
239 Ldr(dst.fp().S(), src_op);
240 break;
241 case LoadType::kF64Load:
242 Ldr(dst.fp().D(), src_op);
243 break;
244 default:
245 UNREACHABLE();
246 }
247 }
248
Store(Register dst_addr,Register offset_reg,uint32_t offset_imm,LiftoffRegister src,StoreType type,LiftoffRegList pinned,uint32_t * protected_store_pc,bool is_store_mem)249 void LiftoffAssembler::Store(Register dst_addr, Register offset_reg,
250 uint32_t offset_imm, LiftoffRegister src,
251 StoreType type, LiftoffRegList pinned,
252 uint32_t* protected_store_pc, bool is_store_mem) {
253 UseScratchRegisterScope temps(this);
254 MemOperand dst_op =
255 liftoff::GetMemOp(this, &temps, dst_addr, offset_reg, offset_imm);
256 if (protected_store_pc) *protected_store_pc = pc_offset();
257 switch (type.value()) {
258 case StoreType::kI32Store8:
259 case StoreType::kI64Store8:
260 Strb(src.gp().W(), dst_op);
261 break;
262 case StoreType::kI32Store16:
263 case StoreType::kI64Store16:
264 Strh(src.gp().W(), dst_op);
265 break;
266 case StoreType::kI32Store:
267 case StoreType::kI64Store32:
268 Str(src.gp().W(), dst_op);
269 break;
270 case StoreType::kI64Store:
271 Str(src.gp().X(), dst_op);
272 break;
273 case StoreType::kF32Store:
274 Str(src.fp().S(), dst_op);
275 break;
276 case StoreType::kF64Store:
277 Str(src.fp().D(), dst_op);
278 break;
279 default:
280 UNREACHABLE();
281 }
282 }
283
LoadCallerFrameSlot(LiftoffRegister dst,uint32_t caller_slot_idx,ValueType type)284 void LiftoffAssembler::LoadCallerFrameSlot(LiftoffRegister dst,
285 uint32_t caller_slot_idx,
286 ValueType type) {
287 int32_t offset = (caller_slot_idx + 1) * LiftoffAssembler::kStackSlotSize;
288 Ldr(liftoff::GetRegFromType(dst, type), MemOperand(fp, offset));
289 }
290
MoveStackValue(uint32_t dst_index,uint32_t src_index,ValueType type)291 void LiftoffAssembler::MoveStackValue(uint32_t dst_index, uint32_t src_index,
292 ValueType type) {
293 UseScratchRegisterScope temps(this);
294 CPURegister scratch = liftoff::AcquireByType(&temps, type);
295 Ldr(scratch, liftoff::GetStackSlot(src_index));
296 Str(scratch, liftoff::GetStackSlot(dst_index));
297 }
298
Move(Register dst,Register src,ValueType type)299 void LiftoffAssembler::Move(Register dst, Register src, ValueType type) {
300 if (type == kWasmI32) {
301 Mov(dst.W(), src.W());
302 } else {
303 DCHECK_EQ(kWasmI64, type);
304 Mov(dst.X(), src.X());
305 }
306 }
307
Move(DoubleRegister dst,DoubleRegister src,ValueType type)308 void LiftoffAssembler::Move(DoubleRegister dst, DoubleRegister src,
309 ValueType type) {
310 if (type == kWasmF32) {
311 Fmov(dst.S(), src.S());
312 } else {
313 DCHECK_EQ(kWasmF64, type);
314 Fmov(dst.D(), src.D());
315 }
316 }
317
Spill(uint32_t index,LiftoffRegister reg,ValueType type)318 void LiftoffAssembler::Spill(uint32_t index, LiftoffRegister reg,
319 ValueType type) {
320 RecordUsedSpillSlot(index);
321 MemOperand dst = liftoff::GetStackSlot(index);
322 Str(liftoff::GetRegFromType(reg, type), dst);
323 }
324
Spill(uint32_t index,WasmValue value)325 void LiftoffAssembler::Spill(uint32_t index, WasmValue value) {
326 RecordUsedSpillSlot(index);
327 MemOperand dst = liftoff::GetStackSlot(index);
328 UseScratchRegisterScope temps(this);
329 CPURegister src = CPURegister::no_reg();
330 switch (value.type()) {
331 case kWasmI32:
332 src = temps.AcquireW();
333 Mov(src.W(), value.to_i32());
334 break;
335 case kWasmI64:
336 src = temps.AcquireX();
337 Mov(src.X(), value.to_i64());
338 break;
339 default:
340 // We do not track f32 and f64 constants, hence they are unreachable.
341 UNREACHABLE();
342 }
343 Str(src, dst);
344 }
345
Fill(LiftoffRegister reg,uint32_t index,ValueType type)346 void LiftoffAssembler::Fill(LiftoffRegister reg, uint32_t index,
347 ValueType type) {
348 MemOperand src = liftoff::GetStackSlot(index);
349 Ldr(liftoff::GetRegFromType(reg, type), src);
350 }
351
FillI64Half(Register,uint32_t half_index)352 void LiftoffAssembler::FillI64Half(Register, uint32_t half_index) {
353 UNREACHABLE();
354 }
355
356 #define I32_BINOP(name, instruction) \
357 void LiftoffAssembler::emit_##name(Register dst, Register lhs, \
358 Register rhs) { \
359 instruction(dst.W(), lhs.W(), rhs.W()); \
360 }
361 #define I64_BINOP(name, instruction) \
362 void LiftoffAssembler::emit_##name(LiftoffRegister dst, LiftoffRegister lhs, \
363 LiftoffRegister rhs) { \
364 instruction(dst.gp().X(), lhs.gp().X(), rhs.gp().X()); \
365 }
366 #define FP32_BINOP(name, instruction) \
367 void LiftoffAssembler::emit_##name(DoubleRegister dst, DoubleRegister lhs, \
368 DoubleRegister rhs) { \
369 instruction(dst.S(), lhs.S(), rhs.S()); \
370 }
371 #define FP32_UNOP(name, instruction) \
372 void LiftoffAssembler::emit_##name(DoubleRegister dst, DoubleRegister src) { \
373 instruction(dst.S(), src.S()); \
374 }
375 #define FP64_BINOP(name, instruction) \
376 void LiftoffAssembler::emit_##name(DoubleRegister dst, DoubleRegister lhs, \
377 DoubleRegister rhs) { \
378 instruction(dst.D(), lhs.D(), rhs.D()); \
379 }
380 #define FP64_UNOP(name, instruction) \
381 void LiftoffAssembler::emit_##name(DoubleRegister dst, DoubleRegister src) { \
382 instruction(dst.D(), src.D()); \
383 }
384 #define FP64_UNOP_RETURN_TRUE(name, instruction) \
385 bool LiftoffAssembler::emit_##name(DoubleRegister dst, DoubleRegister src) { \
386 instruction(dst.D(), src.D()); \
387 return true; \
388 }
389 #define I32_SHIFTOP(name, instruction) \
390 void LiftoffAssembler::emit_##name(Register dst, Register src, \
391 Register amount, LiftoffRegList pinned) { \
392 instruction(dst.W(), src.W(), amount.W()); \
393 }
394 #define I64_SHIFTOP(name, instruction) \
395 void LiftoffAssembler::emit_##name(LiftoffRegister dst, LiftoffRegister src, \
396 Register amount, LiftoffRegList pinned) { \
397 instruction(dst.gp().X(), src.gp().X(), amount.X()); \
398 }
399
I32_BINOP(i32_add,Add)400 I32_BINOP(i32_add, Add)
401 I32_BINOP(i32_sub, Sub)
402 I32_BINOP(i32_mul, Mul)
403 I32_BINOP(i32_and, And)
404 I32_BINOP(i32_or, Orr)
405 I32_BINOP(i32_xor, Eor)
406 I32_SHIFTOP(i32_shl, Lsl)
407 I32_SHIFTOP(i32_sar, Asr)
408 I32_SHIFTOP(i32_shr, Lsr)
409 I64_BINOP(i64_add, Add)
410 I64_BINOP(i64_sub, Sub)
411 I64_BINOP(i64_mul, Mul)
412 I64_BINOP(i64_and, And)
413 I64_BINOP(i64_or, Orr)
414 I64_BINOP(i64_xor, Eor)
415 I64_SHIFTOP(i64_shl, Lsl)
416 I64_SHIFTOP(i64_sar, Asr)
417 I64_SHIFTOP(i64_shr, Lsr)
418 FP32_BINOP(f32_add, Fadd)
419 FP32_BINOP(f32_sub, Fsub)
420 FP32_BINOP(f32_mul, Fmul)
421 FP32_BINOP(f32_div, Fdiv)
422 FP32_BINOP(f32_min, Fmin)
423 FP32_BINOP(f32_max, Fmax)
424 FP32_UNOP(f32_abs, Fabs)
425 FP32_UNOP(f32_neg, Fneg)
426 FP32_UNOP(f32_ceil, Frintp)
427 FP32_UNOP(f32_floor, Frintm)
428 FP32_UNOP(f32_trunc, Frintz)
429 FP32_UNOP(f32_nearest_int, Frintn)
430 FP32_UNOP(f32_sqrt, Fsqrt)
431 FP64_BINOP(f64_add, Fadd)
432 FP64_BINOP(f64_sub, Fsub)
433 FP64_BINOP(f64_mul, Fmul)
434 FP64_BINOP(f64_div, Fdiv)
435 FP64_BINOP(f64_min, Fmin)
436 FP64_BINOP(f64_max, Fmax)
437 FP64_UNOP(f64_abs, Fabs)
438 FP64_UNOP(f64_neg, Fneg)
439 FP64_UNOP_RETURN_TRUE(f64_ceil, Frintp)
440 FP64_UNOP_RETURN_TRUE(f64_floor, Frintm)
441 FP64_UNOP_RETURN_TRUE(f64_trunc, Frintz)
442 FP64_UNOP_RETURN_TRUE(f64_nearest_int, Frintn)
443 FP64_UNOP(f64_sqrt, Fsqrt)
444
445 #undef I32_BINOP
446 #undef I64_BINOP
447 #undef FP32_BINOP
448 #undef FP32_UNOP
449 #undef FP64_BINOP
450 #undef FP64_UNOP
451 #undef FP64_UNOP_RETURN_TRUE
452 #undef I32_SHIFTOP
453 #undef I64_SHIFTOP
454
455 bool LiftoffAssembler::emit_i32_clz(Register dst, Register src) {
456 Clz(dst.W(), src.W());
457 return true;
458 }
459
emit_i32_ctz(Register dst,Register src)460 bool LiftoffAssembler::emit_i32_ctz(Register dst, Register src) {
461 Rbit(dst.W(), src.W());
462 Clz(dst.W(), dst.W());
463 return true;
464 }
465
emit_i32_popcnt(Register dst,Register src)466 bool LiftoffAssembler::emit_i32_popcnt(Register dst, Register src) {
467 UseScratchRegisterScope temps(this);
468 VRegister scratch = temps.AcquireV(kFormat8B);
469 Fmov(scratch.S(), src.W());
470 Cnt(scratch, scratch);
471 Addv(scratch.B(), scratch);
472 Fmov(dst.W(), scratch.S());
473 return true;
474 }
475
emit_i32_divs(Register dst,Register lhs,Register rhs,Label * trap_div_by_zero,Label * trap_div_unrepresentable)476 void LiftoffAssembler::emit_i32_divs(Register dst, Register lhs, Register rhs,
477 Label* trap_div_by_zero,
478 Label* trap_div_unrepresentable) {
479 Register dst_w = dst.W();
480 Register lhs_w = lhs.W();
481 Register rhs_w = rhs.W();
482 bool can_use_dst = !dst_w.Aliases(lhs_w) && !dst_w.Aliases(rhs_w);
483 if (can_use_dst) {
484 // Do div early.
485 Sdiv(dst_w, lhs_w, rhs_w);
486 }
487 // Check for division by zero.
488 Cbz(rhs_w, trap_div_by_zero);
489 // Check for kMinInt / -1. This is unrepresentable.
490 Cmp(rhs_w, -1);
491 Ccmp(lhs_w, 1, NoFlag, eq);
492 B(trap_div_unrepresentable, vs);
493 if (!can_use_dst) {
494 // Do div.
495 Sdiv(dst_w, lhs_w, rhs_w);
496 }
497 }
498
emit_i32_divu(Register dst,Register lhs,Register rhs,Label * trap_div_by_zero)499 void LiftoffAssembler::emit_i32_divu(Register dst, Register lhs, Register rhs,
500 Label* trap_div_by_zero) {
501 // Check for division by zero.
502 Cbz(rhs.W(), trap_div_by_zero);
503 // Do div.
504 Udiv(dst.W(), lhs.W(), rhs.W());
505 }
506
emit_i32_rems(Register dst,Register lhs,Register rhs,Label * trap_div_by_zero)507 void LiftoffAssembler::emit_i32_rems(Register dst, Register lhs, Register rhs,
508 Label* trap_div_by_zero) {
509 Register dst_w = dst.W();
510 Register lhs_w = lhs.W();
511 Register rhs_w = rhs.W();
512 // Do early div.
513 // No need to check kMinInt / -1 because the result is kMinInt and then
514 // kMinInt * -1 -> kMinInt. In this case, the Msub result is therefore 0.
515 UseScratchRegisterScope temps(this);
516 Register scratch = temps.AcquireW();
517 Sdiv(scratch, lhs_w, rhs_w);
518 // Check for division by zero.
519 Cbz(rhs_w, trap_div_by_zero);
520 // Compute remainder.
521 Msub(dst_w, scratch, rhs_w, lhs_w);
522 }
523
emit_i32_remu(Register dst,Register lhs,Register rhs,Label * trap_div_by_zero)524 void LiftoffAssembler::emit_i32_remu(Register dst, Register lhs, Register rhs,
525 Label* trap_div_by_zero) {
526 Register dst_w = dst.W();
527 Register lhs_w = lhs.W();
528 Register rhs_w = rhs.W();
529 // Do early div.
530 UseScratchRegisterScope temps(this);
531 Register scratch = temps.AcquireW();
532 Udiv(scratch, lhs_w, rhs_w);
533 // Check for division by zero.
534 Cbz(rhs_w, trap_div_by_zero);
535 // Compute remainder.
536 Msub(dst_w, scratch, rhs_w, lhs_w);
537 }
538
emit_i64_divs(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs,Label * trap_div_by_zero,Label * trap_div_unrepresentable)539 bool LiftoffAssembler::emit_i64_divs(LiftoffRegister dst, LiftoffRegister lhs,
540 LiftoffRegister rhs,
541 Label* trap_div_by_zero,
542 Label* trap_div_unrepresentable) {
543 Register dst_x = dst.gp().X();
544 Register lhs_x = lhs.gp().X();
545 Register rhs_x = rhs.gp().X();
546 bool can_use_dst = !dst_x.Aliases(lhs_x) && !dst_x.Aliases(rhs_x);
547 if (can_use_dst) {
548 // Do div early.
549 Sdiv(dst_x, lhs_x, rhs_x);
550 }
551 // Check for division by zero.
552 Cbz(rhs_x, trap_div_by_zero);
553 // Check for kMinInt / -1. This is unrepresentable.
554 Cmp(rhs_x, -1);
555 Ccmp(lhs_x, 1, NoFlag, eq);
556 B(trap_div_unrepresentable, vs);
557 if (!can_use_dst) {
558 // Do div.
559 Sdiv(dst_x, lhs_x, rhs_x);
560 }
561 return true;
562 }
563
emit_i64_divu(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs,Label * trap_div_by_zero)564 bool LiftoffAssembler::emit_i64_divu(LiftoffRegister dst, LiftoffRegister lhs,
565 LiftoffRegister rhs,
566 Label* trap_div_by_zero) {
567 // Check for division by zero.
568 Cbz(rhs.gp().X(), trap_div_by_zero);
569 // Do div.
570 Udiv(dst.gp().X(), lhs.gp().X(), rhs.gp().X());
571 return true;
572 }
573
emit_i64_rems(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs,Label * trap_div_by_zero)574 bool LiftoffAssembler::emit_i64_rems(LiftoffRegister dst, LiftoffRegister lhs,
575 LiftoffRegister rhs,
576 Label* trap_div_by_zero) {
577 Register dst_x = dst.gp().X();
578 Register lhs_x = lhs.gp().X();
579 Register rhs_x = rhs.gp().X();
580 // Do early div.
581 // No need to check kMinInt / -1 because the result is kMinInt and then
582 // kMinInt * -1 -> kMinInt. In this case, the Msub result is therefore 0.
583 UseScratchRegisterScope temps(this);
584 Register scratch = temps.AcquireX();
585 Sdiv(scratch, lhs_x, rhs_x);
586 // Check for division by zero.
587 Cbz(rhs_x, trap_div_by_zero);
588 // Compute remainder.
589 Msub(dst_x, scratch, rhs_x, lhs_x);
590 return true;
591 }
592
emit_i64_remu(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs,Label * trap_div_by_zero)593 bool LiftoffAssembler::emit_i64_remu(LiftoffRegister dst, LiftoffRegister lhs,
594 LiftoffRegister rhs,
595 Label* trap_div_by_zero) {
596 Register dst_x = dst.gp().X();
597 Register lhs_x = lhs.gp().X();
598 Register rhs_x = rhs.gp().X();
599 // Do early div.
600 UseScratchRegisterScope temps(this);
601 Register scratch = temps.AcquireX();
602 Udiv(scratch, lhs_x, rhs_x);
603 // Check for division by zero.
604 Cbz(rhs_x, trap_div_by_zero);
605 // Compute remainder.
606 Msub(dst_x, scratch, rhs_x, lhs_x);
607 return true;
608 }
609
emit_i32_to_intptr(Register dst,Register src)610 void LiftoffAssembler::emit_i32_to_intptr(Register dst, Register src) {
611 Sxtw(dst, src);
612 }
613
emit_type_conversion(WasmOpcode opcode,LiftoffRegister dst,LiftoffRegister src,Label * trap)614 bool LiftoffAssembler::emit_type_conversion(WasmOpcode opcode,
615 LiftoffRegister dst,
616 LiftoffRegister src, Label* trap) {
617 switch (opcode) {
618 case kExprI32ConvertI64:
619 if (src != dst) Mov(dst.gp().W(), src.gp().W());
620 return true;
621 case kExprI32SConvertF32:
622 Fcvtzs(dst.gp().W(), src.fp().S()); // f32 -> i32 round to zero.
623 // Check underflow and NaN.
624 Fcmp(src.fp().S(), static_cast<float>(INT32_MIN));
625 // Check overflow.
626 Ccmp(dst.gp().W(), -1, VFlag, ge);
627 B(trap, vs);
628 return true;
629 case kExprI32UConvertF32:
630 Fcvtzu(dst.gp().W(), src.fp().S()); // f32 -> i32 round to zero.
631 // Check underflow and NaN.
632 Fcmp(src.fp().S(), -1.0);
633 // Check overflow.
634 Ccmp(dst.gp().W(), -1, ZFlag, gt);
635 B(trap, eq);
636 return true;
637 case kExprI32SConvertF64: {
638 // INT32_MIN and INT32_MAX are valid results, we cannot test the result
639 // to detect the overflows. We could have done two immediate floating
640 // point comparisons but it would have generated two conditional branches.
641 UseScratchRegisterScope temps(this);
642 VRegister fp_ref = temps.AcquireD();
643 VRegister fp_cmp = temps.AcquireD();
644 Fcvtzs(dst.gp().W(), src.fp().D()); // f64 -> i32 round to zero.
645 Frintz(fp_ref, src.fp().D()); // f64 -> f64 round to zero.
646 Scvtf(fp_cmp, dst.gp().W()); // i32 -> f64.
647 // If comparison fails, we have an overflow or a NaN.
648 Fcmp(fp_cmp, fp_ref);
649 B(trap, ne);
650 return true;
651 }
652 case kExprI32UConvertF64: {
653 // INT32_MAX is a valid result, we cannot test the result to detect the
654 // overflows. We could have done two immediate floating point comparisons
655 // but it would have generated two conditional branches.
656 UseScratchRegisterScope temps(this);
657 VRegister fp_ref = temps.AcquireD();
658 VRegister fp_cmp = temps.AcquireD();
659 Fcvtzu(dst.gp().W(), src.fp().D()); // f64 -> i32 round to zero.
660 Frintz(fp_ref, src.fp().D()); // f64 -> f64 round to zero.
661 Ucvtf(fp_cmp, dst.gp().W()); // i32 -> f64.
662 // If comparison fails, we have an overflow or a NaN.
663 Fcmp(fp_cmp, fp_ref);
664 B(trap, ne);
665 return true;
666 }
667 case kExprI32ReinterpretF32:
668 Fmov(dst.gp().W(), src.fp().S());
669 return true;
670 case kExprI64SConvertI32:
671 Sxtw(dst.gp().X(), src.gp().W());
672 return true;
673 case kExprI64SConvertF32:
674 Fcvtzs(dst.gp().X(), src.fp().S()); // f32 -> i64 round to zero.
675 // Check underflow and NaN.
676 Fcmp(src.fp().S(), static_cast<float>(INT64_MIN));
677 // Check overflow.
678 Ccmp(dst.gp().X(), -1, VFlag, ge);
679 B(trap, vs);
680 return true;
681 case kExprI64UConvertF32:
682 Fcvtzu(dst.gp().X(), src.fp().S()); // f32 -> i64 round to zero.
683 // Check underflow and NaN.
684 Fcmp(src.fp().S(), -1.0);
685 // Check overflow.
686 Ccmp(dst.gp().X(), -1, ZFlag, gt);
687 B(trap, eq);
688 return true;
689 case kExprI64SConvertF64:
690 Fcvtzs(dst.gp().X(), src.fp().D()); // f64 -> i64 round to zero.
691 // Check underflow and NaN.
692 Fcmp(src.fp().D(), static_cast<float>(INT64_MIN));
693 // Check overflow.
694 Ccmp(dst.gp().X(), -1, VFlag, ge);
695 B(trap, vs);
696 return true;
697 case kExprI64UConvertF64:
698 Fcvtzu(dst.gp().X(), src.fp().D()); // f64 -> i64 round to zero.
699 // Check underflow and NaN.
700 Fcmp(src.fp().D(), -1.0);
701 // Check overflow.
702 Ccmp(dst.gp().X(), -1, ZFlag, gt);
703 B(trap, eq);
704 return true;
705 case kExprI64UConvertI32:
706 Mov(dst.gp().W(), src.gp().W());
707 return true;
708 case kExprI64ReinterpretF64:
709 Fmov(dst.gp().X(), src.fp().D());
710 return true;
711 case kExprF32SConvertI32:
712 Scvtf(dst.fp().S(), src.gp().W());
713 return true;
714 case kExprF32UConvertI32:
715 Ucvtf(dst.fp().S(), src.gp().W());
716 return true;
717 case kExprF32SConvertI64:
718 Scvtf(dst.fp().S(), src.gp().X());
719 return true;
720 case kExprF32UConvertI64:
721 Ucvtf(dst.fp().S(), src.gp().X());
722 return true;
723 case kExprF32ConvertF64:
724 Fcvt(dst.fp().S(), src.fp().D());
725 return true;
726 case kExprF32ReinterpretI32:
727 Fmov(dst.fp().S(), src.gp().W());
728 return true;
729 case kExprF64SConvertI32:
730 Scvtf(dst.fp().D(), src.gp().W());
731 return true;
732 case kExprF64UConvertI32:
733 Ucvtf(dst.fp().D(), src.gp().W());
734 return true;
735 case kExprF64SConvertI64:
736 Scvtf(dst.fp().D(), src.gp().X());
737 return true;
738 case kExprF64UConvertI64:
739 Ucvtf(dst.fp().D(), src.gp().X());
740 return true;
741 case kExprF64ConvertF32:
742 Fcvt(dst.fp().D(), src.fp().S());
743 return true;
744 case kExprF64ReinterpretI64:
745 Fmov(dst.fp().D(), src.gp().X());
746 return true;
747 default:
748 UNREACHABLE();
749 }
750 }
751
emit_jump(Label * label)752 void LiftoffAssembler::emit_jump(Label* label) { B(label); }
753
emit_jump(Register target)754 void LiftoffAssembler::emit_jump(Register target) { Br(target); }
755
emit_cond_jump(Condition cond,Label * label,ValueType type,Register lhs,Register rhs)756 void LiftoffAssembler::emit_cond_jump(Condition cond, Label* label,
757 ValueType type, Register lhs,
758 Register rhs) {
759 switch (type) {
760 case kWasmI32:
761 if (rhs.IsValid()) {
762 Cmp(lhs.W(), rhs.W());
763 } else {
764 Cmp(lhs.W(), wzr);
765 }
766 break;
767 case kWasmI64:
768 if (rhs.IsValid()) {
769 Cmp(lhs.X(), rhs.X());
770 } else {
771 Cmp(lhs.X(), xzr);
772 }
773 break;
774 default:
775 UNREACHABLE();
776 }
777 B(label, cond);
778 }
779
emit_i32_eqz(Register dst,Register src)780 void LiftoffAssembler::emit_i32_eqz(Register dst, Register src) {
781 Cmp(src.W(), wzr);
782 Cset(dst.W(), eq);
783 }
784
emit_i32_set_cond(Condition cond,Register dst,Register lhs,Register rhs)785 void LiftoffAssembler::emit_i32_set_cond(Condition cond, Register dst,
786 Register lhs, Register rhs) {
787 Cmp(lhs.W(), rhs.W());
788 Cset(dst.W(), cond);
789 }
790
emit_i64_eqz(Register dst,LiftoffRegister src)791 void LiftoffAssembler::emit_i64_eqz(Register dst, LiftoffRegister src) {
792 Cmp(src.gp().X(), xzr);
793 Cset(dst.W(), eq);
794 }
795
emit_i64_set_cond(Condition cond,Register dst,LiftoffRegister lhs,LiftoffRegister rhs)796 void LiftoffAssembler::emit_i64_set_cond(Condition cond, Register dst,
797 LiftoffRegister lhs,
798 LiftoffRegister rhs) {
799 Cmp(lhs.gp().X(), rhs.gp().X());
800 Cset(dst.W(), cond);
801 }
802
emit_f32_set_cond(Condition cond,Register dst,DoubleRegister lhs,DoubleRegister rhs)803 void LiftoffAssembler::emit_f32_set_cond(Condition cond, Register dst,
804 DoubleRegister lhs,
805 DoubleRegister rhs) {
806 Fcmp(lhs.S(), rhs.S());
807 Cset(dst.W(), cond);
808 if (cond != ne) {
809 // If V flag set, at least one of the arguments was a Nan -> false.
810 Csel(dst.W(), wzr, dst.W(), vs);
811 }
812 }
813
emit_f64_set_cond(Condition cond,Register dst,DoubleRegister lhs,DoubleRegister rhs)814 void LiftoffAssembler::emit_f64_set_cond(Condition cond, Register dst,
815 DoubleRegister lhs,
816 DoubleRegister rhs) {
817 Fcmp(lhs.D(), rhs.D());
818 Cset(dst.W(), cond);
819 if (cond != ne) {
820 // If V flag set, at least one of the arguments was a Nan -> false.
821 Csel(dst.W(), wzr, dst.W(), vs);
822 }
823 }
824
StackCheck(Label * ool_code,Register limit_address)825 void LiftoffAssembler::StackCheck(Label* ool_code, Register limit_address) {
826 Ldr(limit_address, MemOperand(limit_address));
827 Cmp(sp, limit_address);
828 B(ool_code, ls);
829 }
830
CallTrapCallbackForTesting()831 void LiftoffAssembler::CallTrapCallbackForTesting() {
832 CallCFunction(ExternalReference::wasm_call_trap_callback_for_testing(), 0);
833 }
834
AssertUnreachable(AbortReason reason)835 void LiftoffAssembler::AssertUnreachable(AbortReason reason) {
836 TurboAssembler::AssertUnreachable(reason);
837 }
838
PushRegisters(LiftoffRegList regs)839 void LiftoffAssembler::PushRegisters(LiftoffRegList regs) {
840 PushCPURegList(liftoff::PadRegList(regs.GetGpList()));
841 PushCPURegList(liftoff::PadVRegList(regs.GetFpList()));
842 }
843
PopRegisters(LiftoffRegList regs)844 void LiftoffAssembler::PopRegisters(LiftoffRegList regs) {
845 PopCPURegList(liftoff::PadVRegList(regs.GetFpList()));
846 PopCPURegList(liftoff::PadRegList(regs.GetGpList()));
847 }
848
DropStackSlotsAndRet(uint32_t num_stack_slots)849 void LiftoffAssembler::DropStackSlotsAndRet(uint32_t num_stack_slots) {
850 DropSlots(num_stack_slots);
851 Ret();
852 }
853
CallC(wasm::FunctionSig * sig,const LiftoffRegister * args,const LiftoffRegister * rets,ValueType out_argument_type,int stack_bytes,ExternalReference ext_ref)854 void LiftoffAssembler::CallC(wasm::FunctionSig* sig,
855 const LiftoffRegister* args,
856 const LiftoffRegister* rets,
857 ValueType out_argument_type, int stack_bytes,
858 ExternalReference ext_ref) {
859 // The stack pointer is required to be quadword aligned.
860 int total_size = RoundUp(stack_bytes, kQuadWordSizeInBytes);
861 // Reserve space in the stack.
862 Claim(total_size, 1);
863
864 int arg_bytes = 0;
865 for (ValueType param_type : sig->parameters()) {
866 Poke(liftoff::GetRegFromType(*args++, param_type), arg_bytes);
867 arg_bytes += ValueTypes::MemSize(param_type);
868 }
869 DCHECK_LE(arg_bytes, stack_bytes);
870
871 // Pass a pointer to the buffer with the arguments to the C function.
872 Mov(x0, sp);
873
874 // Now call the C function.
875 constexpr int kNumCCallArgs = 1;
876 CallCFunction(ext_ref, kNumCCallArgs);
877
878 // Move return value to the right register.
879 const LiftoffRegister* next_result_reg = rets;
880 if (sig->return_count() > 0) {
881 DCHECK_EQ(1, sig->return_count());
882 constexpr Register kReturnReg = x0;
883 if (kReturnReg != next_result_reg->gp()) {
884 Move(*next_result_reg, LiftoffRegister(kReturnReg), sig->GetReturn(0));
885 }
886 ++next_result_reg;
887 }
888
889 // Load potential output value from the buffer on the stack.
890 if (out_argument_type != kWasmStmt) {
891 Peek(liftoff::GetRegFromType(*next_result_reg, out_argument_type), 0);
892 }
893
894 Drop(total_size, 1);
895 }
896
CallNativeWasmCode(Address addr)897 void LiftoffAssembler::CallNativeWasmCode(Address addr) {
898 Call(addr, RelocInfo::WASM_CALL);
899 }
900
CallIndirect(wasm::FunctionSig * sig,compiler::CallDescriptor * call_descriptor,Register target)901 void LiftoffAssembler::CallIndirect(wasm::FunctionSig* sig,
902 compiler::CallDescriptor* call_descriptor,
903 Register target) {
904 // For Arm64, we have more cache registers than wasm parameters. That means
905 // that target will always be in a register.
906 DCHECK(target.IsValid());
907 Call(target);
908 }
909
CallRuntimeStub(WasmCode::RuntimeStubId sid)910 void LiftoffAssembler::CallRuntimeStub(WasmCode::RuntimeStubId sid) {
911 // A direct call to a wasm runtime stub defined in this module.
912 // Just encode the stub index. This will be patched at relocation.
913 Call(static_cast<Address>(sid), RelocInfo::WASM_STUB_CALL);
914 }
915
AllocateStackSlot(Register addr,uint32_t size)916 void LiftoffAssembler::AllocateStackSlot(Register addr, uint32_t size) {
917 // The stack pointer is required to be quadword aligned.
918 size = RoundUp(size, kQuadWordSizeInBytes);
919 Claim(size, 1);
920 Mov(addr, sp);
921 }
922
DeallocateStackSlot(uint32_t size)923 void LiftoffAssembler::DeallocateStackSlot(uint32_t size) {
924 // The stack pointer is required to be quadword aligned.
925 size = RoundUp(size, kQuadWordSizeInBytes);
926 Drop(size, 1);
927 }
928
Construct()929 void LiftoffStackSlots::Construct() {
930 size_t slot_count = slots_.size();
931 // The stack pointer is required to be quadword aligned.
932 asm_->Claim(RoundUp(slot_count, 2));
933 size_t slot_index = 0;
934 for (auto& slot : slots_) {
935 size_t poke_offset = (slot_count - slot_index - 1) * kXRegSize;
936 switch (slot.src_.loc()) {
937 case LiftoffAssembler::VarState::kStack: {
938 UseScratchRegisterScope temps(asm_);
939 CPURegister scratch = liftoff::AcquireByType(&temps, slot.src_.type());
940 asm_->Ldr(scratch, liftoff::GetStackSlot(slot.src_index_));
941 asm_->Poke(scratch, poke_offset);
942 break;
943 }
944 case LiftoffAssembler::VarState::kRegister:
945 asm_->Poke(liftoff::GetRegFromType(slot.src_.reg(), slot.src_.type()),
946 poke_offset);
947 break;
948 case LiftoffAssembler::VarState::KIntConst:
949 DCHECK(slot.src_.type() == kWasmI32 || slot.src_.type() == kWasmI64);
950 if (slot.src_.i32_const() == 0) {
951 Register zero_reg = slot.src_.type() == kWasmI32 ? wzr : xzr;
952 asm_->Poke(zero_reg, poke_offset);
953 } else {
954 UseScratchRegisterScope temps(asm_);
955 Register scratch = slot.src_.type() == kWasmI32 ? temps.AcquireW()
956 : temps.AcquireX();
957 asm_->Mov(scratch, int64_t{slot.src_.i32_const()});
958 asm_->Poke(scratch, poke_offset);
959 }
960 break;
961 }
962 slot_index++;
963 }
964 }
965
966 } // namespace wasm
967 } // namespace internal
968 } // namespace v8
969
970 #undef BAILOUT
971
972 #endif // V8_WASM_BASELINE_ARM64_LIFTOFF_ASSEMBLER_ARM64_H_
973