1 // Copyright 2017 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #ifndef V8_WASM_BASELINE_MIPS64_LIFTOFF_ASSEMBLER_MIPS64_H_
6 #define V8_WASM_BASELINE_MIPS64_LIFTOFF_ASSEMBLER_MIPS64_H_
7
8 #include "src/base/platform/wrappers.h"
9 #include "src/codegen/machine-type.h"
10 #include "src/heap/memory-chunk.h"
11 #include "src/wasm/baseline/liftoff-assembler.h"
12 #include "src/wasm/wasm-objects.h"
13
14 namespace v8 {
15 namespace internal {
16 namespace wasm {
17
18 namespace liftoff {
19
ToCondition(LiftoffCondition liftoff_cond)20 inline constexpr Condition ToCondition(LiftoffCondition liftoff_cond) {
21 switch (liftoff_cond) {
22 case kEqual:
23 return eq;
24 case kUnequal:
25 return ne;
26 case kSignedLessThan:
27 return lt;
28 case kSignedLessEqual:
29 return le;
30 case kSignedGreaterThan:
31 return gt;
32 case kSignedGreaterEqual:
33 return ge;
34 case kUnsignedLessThan:
35 return ult;
36 case kUnsignedLessEqual:
37 return ule;
38 case kUnsignedGreaterThan:
39 return ugt;
40 case kUnsignedGreaterEqual:
41 return uge;
42 }
43 }
44
45 // Liftoff Frames.
46 //
47 // slot Frame
48 // +--------------------+---------------------------
49 // n+4 | optional padding slot to keep the stack 16 byte aligned.
50 // n+3 | parameter n |
51 // ... | ... |
52 // 4 | parameter 1 | or parameter 2
53 // 3 | parameter 0 | or parameter 1
54 // 2 | (result address) | or parameter 0
55 // -----+--------------------+---------------------------
56 // 1 | return addr (ra) |
57 // 0 | previous frame (fp)|
58 // -----+--------------------+ <-- frame ptr (fp)
59 // -1 | StackFrame::WASM |
60 // -2 | instance |
61 // -3 | feedback vector|
62 // -4 | tiering budget |
63 // -----+--------------------+---------------------------
64 // -5 | slot 0 | ^
65 // -6 | slot 1 | |
66 // | | Frame slots
67 // | | |
68 // | | v
69 // | optional padding slot to keep the stack 16 byte aligned.
70 // -----+--------------------+ <-- stack ptr (sp)
71 //
72
73 constexpr int kInstanceOffset = 2 * kSystemPointerSize;
74 constexpr int kFeedbackVectorOffset = 3 * kSystemPointerSize;
75 constexpr int kTierupBudgetOffset = 4 * kSystemPointerSize;
76
GetStackSlot(int offset)77 inline MemOperand GetStackSlot(int offset) { return MemOperand(fp, -offset); }
78
GetInstanceOperand()79 inline MemOperand GetInstanceOperand() { return GetStackSlot(kInstanceOffset); }
80
81 template <typename T>
GetMemOp(LiftoffAssembler * assm,Register addr,Register offset,T offset_imm)82 inline MemOperand GetMemOp(LiftoffAssembler* assm, Register addr,
83 Register offset, T offset_imm) {
84 if (is_int32(offset_imm)) {
85 int32_t offset_imm32 = static_cast<int32_t>(offset_imm);
86 if (offset == no_reg) return MemOperand(addr, offset_imm32);
87 assm->daddu(kScratchReg, addr, offset);
88 return MemOperand(kScratchReg, offset_imm32);
89 }
90 // Offset immediate does not fit in 31 bits.
91 assm->li(kScratchReg, offset_imm);
92 assm->daddu(kScratchReg, kScratchReg, addr);
93 if (offset != no_reg) {
94 assm->daddu(kScratchReg, kScratchReg, offset);
95 }
96 return MemOperand(kScratchReg, 0);
97 }
98
Load(LiftoffAssembler * assm,LiftoffRegister dst,MemOperand src,ValueKind kind)99 inline void Load(LiftoffAssembler* assm, LiftoffRegister dst, MemOperand src,
100 ValueKind kind) {
101 switch (kind) {
102 case kI32:
103 assm->Lw(dst.gp(), src);
104 break;
105 case kI64:
106 case kRef:
107 case kOptRef:
108 case kRtt:
109 assm->Ld(dst.gp(), src);
110 break;
111 case kF32:
112 assm->Lwc1(dst.fp(), src);
113 break;
114 case kF64:
115 assm->Ldc1(dst.fp(), src);
116 break;
117 case kS128:
118 assm->ld_b(dst.fp().toW(), src);
119 break;
120 default:
121 UNREACHABLE();
122 }
123 }
124
Store(LiftoffAssembler * assm,Register base,int32_t offset,LiftoffRegister src,ValueKind kind)125 inline void Store(LiftoffAssembler* assm, Register base, int32_t offset,
126 LiftoffRegister src, ValueKind kind) {
127 MemOperand dst(base, offset);
128 switch (kind) {
129 case kI32:
130 assm->Usw(src.gp(), dst);
131 break;
132 case kI64:
133 case kOptRef:
134 case kRef:
135 case kRtt:
136 assm->Usd(src.gp(), dst);
137 break;
138 case kF32:
139 assm->Uswc1(src.fp(), dst, t8);
140 break;
141 case kF64:
142 assm->Usdc1(src.fp(), dst, t8);
143 break;
144 case kS128:
145 assm->st_b(src.fp().toW(), dst);
146 break;
147 default:
148 UNREACHABLE();
149 }
150 }
151
push(LiftoffAssembler * assm,LiftoffRegister reg,ValueKind kind)152 inline void push(LiftoffAssembler* assm, LiftoffRegister reg, ValueKind kind) {
153 switch (kind) {
154 case kI32:
155 assm->daddiu(sp, sp, -kSystemPointerSize);
156 assm->sw(reg.gp(), MemOperand(sp, 0));
157 break;
158 case kI64:
159 case kOptRef:
160 case kRef:
161 case kRtt:
162 assm->push(reg.gp());
163 break;
164 case kF32:
165 assm->daddiu(sp, sp, -kSystemPointerSize);
166 assm->swc1(reg.fp(), MemOperand(sp, 0));
167 break;
168 case kF64:
169 assm->daddiu(sp, sp, -kSystemPointerSize);
170 assm->Sdc1(reg.fp(), MemOperand(sp, 0));
171 break;
172 case kS128:
173 assm->daddiu(sp, sp, -kSystemPointerSize * 2);
174 assm->st_b(reg.fp().toW(), MemOperand(sp, 0));
175 break;
176 default:
177 UNREACHABLE();
178 }
179 }
180
181 #if defined(V8_TARGET_BIG_ENDIAN)
ChangeEndiannessLoad(LiftoffAssembler * assm,LiftoffRegister dst,LoadType type,LiftoffRegList pinned)182 inline void ChangeEndiannessLoad(LiftoffAssembler* assm, LiftoffRegister dst,
183 LoadType type, LiftoffRegList pinned) {
184 bool is_float = false;
185 LiftoffRegister tmp = dst;
186 switch (type.value()) {
187 case LoadType::kI64Load8U:
188 case LoadType::kI64Load8S:
189 case LoadType::kI32Load8U:
190 case LoadType::kI32Load8S:
191 // No need to change endianness for byte size.
192 return;
193 case LoadType::kF32Load:
194 is_float = true;
195 tmp = assm->GetUnusedRegister(kGpReg, pinned);
196 assm->emit_type_conversion(kExprI32ReinterpretF32, tmp, dst);
197 V8_FALLTHROUGH;
198 case LoadType::kI64Load32U:
199 assm->TurboAssembler::ByteSwapUnsigned(tmp.gp(), tmp.gp(), 4);
200 break;
201 case LoadType::kI32Load:
202 case LoadType::kI64Load32S:
203 assm->TurboAssembler::ByteSwapSigned(tmp.gp(), tmp.gp(), 4);
204 break;
205 case LoadType::kI32Load16S:
206 case LoadType::kI64Load16S:
207 assm->TurboAssembler::ByteSwapSigned(tmp.gp(), tmp.gp(), 2);
208 break;
209 case LoadType::kI32Load16U:
210 case LoadType::kI64Load16U:
211 assm->TurboAssembler::ByteSwapUnsigned(tmp.gp(), tmp.gp(), 2);
212 break;
213 case LoadType::kF64Load:
214 is_float = true;
215 tmp = assm->GetUnusedRegister(kGpReg, pinned);
216 assm->emit_type_conversion(kExprI64ReinterpretF64, tmp, dst);
217 V8_FALLTHROUGH;
218 case LoadType::kI64Load:
219 assm->TurboAssembler::ByteSwapSigned(tmp.gp(), tmp.gp(), 8);
220 break;
221 default:
222 UNREACHABLE();
223 }
224
225 if (is_float) {
226 switch (type.value()) {
227 case LoadType::kF32Load:
228 assm->emit_type_conversion(kExprF32ReinterpretI32, dst, tmp);
229 break;
230 case LoadType::kF64Load:
231 assm->emit_type_conversion(kExprF64ReinterpretI64, dst, tmp);
232 break;
233 default:
234 UNREACHABLE();
235 }
236 }
237 }
238
ChangeEndiannessStore(LiftoffAssembler * assm,LiftoffRegister src,StoreType type,LiftoffRegList pinned)239 inline void ChangeEndiannessStore(LiftoffAssembler* assm, LiftoffRegister src,
240 StoreType type, LiftoffRegList pinned) {
241 bool is_float = false;
242 LiftoffRegister tmp = src;
243 switch (type.value()) {
244 case StoreType::kI64Store8:
245 case StoreType::kI32Store8:
246 // No need to change endianness for byte size.
247 return;
248 case StoreType::kF32Store:
249 is_float = true;
250 tmp = assm->GetUnusedRegister(kGpReg, pinned);
251 assm->emit_type_conversion(kExprI32ReinterpretF32, tmp, src);
252 V8_FALLTHROUGH;
253 case StoreType::kI32Store:
254 assm->TurboAssembler::ByteSwapSigned(tmp.gp(), tmp.gp(), 4);
255 break;
256 case StoreType::kI32Store16:
257 assm->TurboAssembler::ByteSwapSigned(tmp.gp(), tmp.gp(), 2);
258 break;
259 case StoreType::kF64Store:
260 is_float = true;
261 tmp = assm->GetUnusedRegister(kGpReg, pinned);
262 assm->emit_type_conversion(kExprI64ReinterpretF64, tmp, src);
263 V8_FALLTHROUGH;
264 case StoreType::kI64Store:
265 assm->TurboAssembler::ByteSwapSigned(tmp.gp(), tmp.gp(), 8);
266 break;
267 case StoreType::kI64Store32:
268 assm->TurboAssembler::ByteSwapSigned(tmp.gp(), tmp.gp(), 4);
269 break;
270 case StoreType::kI64Store16:
271 assm->TurboAssembler::ByteSwapSigned(tmp.gp(), tmp.gp(), 2);
272 break;
273 default:
274 UNREACHABLE();
275 }
276
277 if (is_float) {
278 switch (type.value()) {
279 case StoreType::kF32Store:
280 assm->emit_type_conversion(kExprF32ReinterpretI32, src, tmp);
281 break;
282 case StoreType::kF64Store:
283 assm->emit_type_conversion(kExprF64ReinterpretI64, src, tmp);
284 break;
285 default:
286 UNREACHABLE();
287 }
288 }
289 }
290 #endif // V8_TARGET_BIG_ENDIAN
291
292 } // namespace liftoff
293
PrepareStackFrame()294 int LiftoffAssembler::PrepareStackFrame() {
295 int offset = pc_offset();
296 // When the frame size is bigger than 4KB, we need seven instructions for
297 // stack checking, so we reserve space for this case.
298 daddiu(sp, sp, 0);
299 nop();
300 nop();
301 nop();
302 nop();
303 nop();
304 nop();
305 return offset;
306 }
307
PrepareTailCall(int num_callee_stack_params,int stack_param_delta)308 void LiftoffAssembler::PrepareTailCall(int num_callee_stack_params,
309 int stack_param_delta) {
310 UseScratchRegisterScope temps(this);
311 Register scratch = temps.Acquire();
312
313 // Push the return address and frame pointer to complete the stack frame.
314 Ld(scratch, MemOperand(fp, 8));
315 Push(scratch);
316 Ld(scratch, MemOperand(fp, 0));
317 Push(scratch);
318
319 // Shift the whole frame upwards.
320 int slot_count = num_callee_stack_params + 2;
321 for (int i = slot_count - 1; i >= 0; --i) {
322 Ld(scratch, MemOperand(sp, i * 8));
323 Sd(scratch, MemOperand(fp, (i - stack_param_delta) * 8));
324 }
325
326 // Set the new stack and frame pointer.
327 daddiu(sp, fp, -stack_param_delta * 8);
328 Pop(ra, fp);
329 }
330
AlignFrameSize()331 void LiftoffAssembler::AlignFrameSize() {}
332
PatchPrepareStackFrame(int offset,SafepointTableBuilder * safepoint_table_builder)333 void LiftoffAssembler::PatchPrepareStackFrame(
334 int offset, SafepointTableBuilder* safepoint_table_builder) {
335 // The frame_size includes the frame marker and the instance slot. Both are
336 // pushed as part of frame construction, so we don't need to allocate memory
337 // for them anymore.
338 int frame_size = GetTotalFrameSize() - 2 * kSystemPointerSize;
339
340 // We can't run out of space, just pass anything big enough to not cause the
341 // assembler to try to grow the buffer.
342 constexpr int kAvailableSpace = 256;
343 TurboAssembler patching_assembler(
344 nullptr, AssemblerOptions{}, CodeObjectRequired::kNo,
345 ExternalAssemblerBuffer(buffer_start_ + offset, kAvailableSpace));
346
347 if (V8_LIKELY(frame_size < 4 * KB)) {
348 // This is the standard case for small frames: just subtract from SP and be
349 // done with it.
350 patching_assembler.Daddu(sp, sp, Operand(-frame_size));
351 return;
352 }
353
354 // The frame size is bigger than 4KB, so we might overflow the available stack
355 // space if we first allocate the frame and then do the stack check (we will
356 // need some remaining stack space for throwing the exception). That's why we
357 // check the available stack space before we allocate the frame. To do this we
358 // replace the {__ Daddu(sp, sp, -frame_size)} with a jump to OOL code that
359 // does this "extended stack check".
360 //
361 // The OOL code can simply be generated here with the normal assembler,
362 // because all other code generation, including OOL code, has already finished
363 // when {PatchPrepareStackFrame} is called. The function prologue then jumps
364 // to the current {pc_offset()} to execute the OOL code for allocating the
365 // large frame.
366 // Emit the unconditional branch in the function prologue (from {offset} to
367 // {pc_offset()}).
368
369 int imm32 = pc_offset() - offset - 3 * kInstrSize;
370 patching_assembler.BranchLong(imm32);
371
372 // If the frame is bigger than the stack, we throw the stack overflow
373 // exception unconditionally. Thereby we can avoid the integer overflow
374 // check in the condition code.
375 RecordComment("OOL: stack check for large frame");
376 Label continuation;
377 if (frame_size < FLAG_stack_size * 1024) {
378 Register stack_limit = kScratchReg;
379 Ld(stack_limit,
380 FieldMemOperand(kWasmInstanceRegister,
381 WasmInstanceObject::kRealStackLimitAddressOffset));
382 Ld(stack_limit, MemOperand(stack_limit));
383 Daddu(stack_limit, stack_limit, Operand(frame_size));
384 Branch(&continuation, uge, sp, Operand(stack_limit));
385 }
386
387 Call(wasm::WasmCode::kWasmStackOverflow, RelocInfo::WASM_STUB_CALL);
388 // The call will not return; just define an empty safepoint.
389 safepoint_table_builder->DefineSafepoint(this);
390 if (FLAG_debug_code) stop();
391
392 bind(&continuation);
393
394 // Now allocate the stack space. Note that this might do more than just
395 // decrementing the SP;
396 Daddu(sp, sp, Operand(-frame_size));
397
398 // Jump back to the start of the function, from {pc_offset()} to
399 // right after the reserved space for the {__ Daddu(sp, sp, -framesize)}
400 // (which is a Branch now).
401 int func_start_offset = offset + 7 * kInstrSize;
402 imm32 = func_start_offset - pc_offset() - 3 * kInstrSize;
403 BranchLong(imm32);
404 }
405
FinishCode()406 void LiftoffAssembler::FinishCode() {}
407
AbortCompilation()408 void LiftoffAssembler::AbortCompilation() {}
409
410 // static
StaticStackFrameSize()411 constexpr int LiftoffAssembler::StaticStackFrameSize() {
412 return liftoff::kTierupBudgetOffset;
413 }
414
SlotSizeForType(ValueKind kind)415 int LiftoffAssembler::SlotSizeForType(ValueKind kind) {
416 switch (kind) {
417 case kS128:
418 return value_kind_size(kind);
419 default:
420 return kStackSlotSize;
421 }
422 }
423
NeedsAlignment(ValueKind kind)424 bool LiftoffAssembler::NeedsAlignment(ValueKind kind) {
425 return kind == kS128 || is_reference(kind);
426 }
427
LoadConstant(LiftoffRegister reg,WasmValue value,RelocInfo::Mode rmode)428 void LiftoffAssembler::LoadConstant(LiftoffRegister reg, WasmValue value,
429 RelocInfo::Mode rmode) {
430 switch (value.type().kind()) {
431 case kI32:
432 TurboAssembler::li(reg.gp(), Operand(value.to_i32(), rmode));
433 break;
434 case kI64:
435 TurboAssembler::li(reg.gp(), Operand(value.to_i64(), rmode));
436 break;
437 case kF32:
438 TurboAssembler::Move(reg.fp(), value.to_f32_boxed().get_bits());
439 break;
440 case kF64:
441 TurboAssembler::Move(reg.fp(), value.to_f64_boxed().get_bits());
442 break;
443 default:
444 UNREACHABLE();
445 }
446 }
447
LoadInstanceFromFrame(Register dst)448 void LiftoffAssembler::LoadInstanceFromFrame(Register dst) {
449 Ld(dst, liftoff::GetInstanceOperand());
450 }
451
LoadFromInstance(Register dst,Register instance,int offset,int size)452 void LiftoffAssembler::LoadFromInstance(Register dst, Register instance,
453 int offset, int size) {
454 DCHECK_LE(0, offset);
455 switch (size) {
456 case 1:
457 Lb(dst, MemOperand(instance, offset));
458 break;
459 case 4:
460 Lw(dst, MemOperand(instance, offset));
461 break;
462 case 8:
463 Ld(dst, MemOperand(instance, offset));
464 break;
465 default:
466 UNIMPLEMENTED();
467 }
468 }
469
LoadTaggedPointerFromInstance(Register dst,Register instance,int32_t offset)470 void LiftoffAssembler::LoadTaggedPointerFromInstance(Register dst,
471 Register instance,
472 int32_t offset) {
473 STATIC_ASSERT(kTaggedSize == kSystemPointerSize);
474 Ld(dst, MemOperand(instance, offset));
475 }
476
SpillInstance(Register instance)477 void LiftoffAssembler::SpillInstance(Register instance) {
478 Sd(instance, liftoff::GetInstanceOperand());
479 }
480
ResetOSRTarget()481 void LiftoffAssembler::ResetOSRTarget() {}
482
LoadTaggedPointer(Register dst,Register src_addr,Register offset_reg,int32_t offset_imm,LiftoffRegList pinned)483 void LiftoffAssembler::LoadTaggedPointer(Register dst, Register src_addr,
484 Register offset_reg,
485 int32_t offset_imm,
486 LiftoffRegList pinned) {
487 STATIC_ASSERT(kTaggedSize == kInt64Size);
488 MemOperand src_op = liftoff::GetMemOp(this, src_addr, offset_reg, offset_imm);
489 Ld(dst, src_op);
490 }
491
LoadFullPointer(Register dst,Register src_addr,int32_t offset_imm)492 void LiftoffAssembler::LoadFullPointer(Register dst, Register src_addr,
493 int32_t offset_imm) {
494 MemOperand src_op = liftoff::GetMemOp(this, src_addr, no_reg, offset_imm);
495 Ld(dst, src_op);
496 }
497
StoreTaggedPointer(Register dst_addr,Register offset_reg,int32_t offset_imm,LiftoffRegister src,LiftoffRegList pinned,SkipWriteBarrier skip_write_barrier)498 void LiftoffAssembler::StoreTaggedPointer(Register dst_addr,
499 Register offset_reg,
500 int32_t offset_imm,
501 LiftoffRegister src,
502 LiftoffRegList pinned,
503 SkipWriteBarrier skip_write_barrier) {
504 STATIC_ASSERT(kTaggedSize == kInt64Size);
505 Register scratch = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp();
506 MemOperand dst_op = liftoff::GetMemOp(this, dst_addr, offset_reg, offset_imm);
507 Sd(src.gp(), dst_op);
508
509 if (skip_write_barrier || FLAG_disable_write_barriers) return;
510
511 Label write_barrier;
512 Label exit;
513 CheckPageFlag(dst_addr, scratch,
514 MemoryChunk::kPointersFromHereAreInterestingMask, ne,
515 &write_barrier);
516 Branch(&exit);
517 bind(&write_barrier);
518 JumpIfSmi(src.gp(), &exit);
519 CheckPageFlag(src.gp(), scratch,
520 MemoryChunk::kPointersToHereAreInterestingMask, eq,
521 &exit);
522 Daddu(scratch, dst_op.rm(), dst_op.offset());
523 CallRecordWriteStubSaveRegisters(
524 dst_addr, scratch, RememberedSetAction::kEmit, SaveFPRegsMode::kSave,
525 StubCallMode::kCallWasmRuntimeStub);
526 bind(&exit);
527 }
528
Load(LiftoffRegister dst,Register src_addr,Register offset_reg,uintptr_t offset_imm,LoadType type,LiftoffRegList pinned,uint32_t * protected_load_pc,bool is_load_mem,bool i64_offset)529 void LiftoffAssembler::Load(LiftoffRegister dst, Register src_addr,
530 Register offset_reg, uintptr_t offset_imm,
531 LoadType type, LiftoffRegList pinned,
532 uint32_t* protected_load_pc, bool is_load_mem,
533 bool i64_offset) {
534 MemOperand src_op = liftoff::GetMemOp(this, src_addr, offset_reg, offset_imm);
535
536 if (protected_load_pc) *protected_load_pc = pc_offset();
537 switch (type.value()) {
538 case LoadType::kI32Load8U:
539 case LoadType::kI64Load8U:
540 Lbu(dst.gp(), src_op);
541 break;
542 case LoadType::kI32Load8S:
543 case LoadType::kI64Load8S:
544 Lb(dst.gp(), src_op);
545 break;
546 case LoadType::kI32Load16U:
547 case LoadType::kI64Load16U:
548 TurboAssembler::Ulhu(dst.gp(), src_op);
549 break;
550 case LoadType::kI32Load16S:
551 case LoadType::kI64Load16S:
552 TurboAssembler::Ulh(dst.gp(), src_op);
553 break;
554 case LoadType::kI64Load32U:
555 TurboAssembler::Ulwu(dst.gp(), src_op);
556 break;
557 case LoadType::kI32Load:
558 case LoadType::kI64Load32S:
559 TurboAssembler::Ulw(dst.gp(), src_op);
560 break;
561 case LoadType::kI64Load:
562 TurboAssembler::Uld(dst.gp(), src_op);
563 break;
564 case LoadType::kF32Load:
565 TurboAssembler::Ulwc1(dst.fp(), src_op, t8);
566 break;
567 case LoadType::kF64Load:
568 TurboAssembler::Uldc1(dst.fp(), src_op, t8);
569 break;
570 case LoadType::kS128Load:
571 TurboAssembler::ld_b(dst.fp().toW(), src_op);
572 break;
573 default:
574 UNREACHABLE();
575 }
576
577 #if defined(V8_TARGET_BIG_ENDIAN)
578 if (is_load_mem) {
579 pinned.set(src_op.rm());
580 liftoff::ChangeEndiannessLoad(this, dst, type, pinned);
581 }
582 #endif
583 }
584
Store(Register dst_addr,Register offset_reg,uintptr_t offset_imm,LiftoffRegister src,StoreType type,LiftoffRegList pinned,uint32_t * protected_store_pc,bool is_store_mem)585 void LiftoffAssembler::Store(Register dst_addr, Register offset_reg,
586 uintptr_t offset_imm, LiftoffRegister src,
587 StoreType type, LiftoffRegList pinned,
588 uint32_t* protected_store_pc, bool is_store_mem) {
589 MemOperand dst_op = liftoff::GetMemOp(this, dst_addr, offset_reg, offset_imm);
590
591 #if defined(V8_TARGET_BIG_ENDIAN)
592 if (is_store_mem) {
593 pinned.set(dst_op.rm());
594 LiftoffRegister tmp = GetUnusedRegister(src.reg_class(), pinned);
595 // Save original value.
596 Move(tmp, src, type.value_type());
597
598 src = tmp;
599 pinned.set(tmp);
600 liftoff::ChangeEndiannessStore(this, src, type, pinned);
601 }
602 #endif
603
604 if (protected_store_pc) *protected_store_pc = pc_offset();
605 switch (type.value()) {
606 case StoreType::kI32Store8:
607 case StoreType::kI64Store8:
608 Sb(src.gp(), dst_op);
609 break;
610 case StoreType::kI32Store16:
611 case StoreType::kI64Store16:
612 TurboAssembler::Ush(src.gp(), dst_op, t8);
613 break;
614 case StoreType::kI32Store:
615 case StoreType::kI64Store32:
616 TurboAssembler::Usw(src.gp(), dst_op);
617 break;
618 case StoreType::kI64Store:
619 TurboAssembler::Usd(src.gp(), dst_op);
620 break;
621 case StoreType::kF32Store:
622 TurboAssembler::Uswc1(src.fp(), dst_op, t8);
623 break;
624 case StoreType::kF64Store:
625 TurboAssembler::Usdc1(src.fp(), dst_op, t8);
626 break;
627 case StoreType::kS128Store:
628 TurboAssembler::st_b(src.fp().toW(), dst_op);
629 break;
630 default:
631 UNREACHABLE();
632 }
633 }
634
AtomicLoad(LiftoffRegister dst,Register src_addr,Register offset_reg,uintptr_t offset_imm,LoadType type,LiftoffRegList pinned)635 void LiftoffAssembler::AtomicLoad(LiftoffRegister dst, Register src_addr,
636 Register offset_reg, uintptr_t offset_imm,
637 LoadType type, LiftoffRegList pinned) {
638 UseScratchRegisterScope temps(this);
639 MemOperand src_op = liftoff::GetMemOp(this, src_addr, offset_reg, offset_imm);
640 switch (type.value()) {
641 case LoadType::kI32Load8U:
642 case LoadType::kI64Load8U: {
643 Lbu(dst.gp(), src_op);
644 sync();
645 return;
646 }
647 case LoadType::kI32Load16U:
648 case LoadType::kI64Load16U: {
649 Lhu(dst.gp(), src_op);
650 sync();
651 return;
652 }
653 case LoadType::kI32Load: {
654 Lw(dst.gp(), src_op);
655 sync();
656 return;
657 }
658 case LoadType::kI64Load32U: {
659 Lwu(dst.gp(), src_op);
660 sync();
661 return;
662 }
663 case LoadType::kI64Load: {
664 Ld(dst.gp(), src_op);
665 sync();
666 return;
667 }
668 default:
669 UNREACHABLE();
670 }
671 }
672
AtomicStore(Register dst_addr,Register offset_reg,uintptr_t offset_imm,LiftoffRegister src,StoreType type,LiftoffRegList pinned)673 void LiftoffAssembler::AtomicStore(Register dst_addr, Register offset_reg,
674 uintptr_t offset_imm, LiftoffRegister src,
675 StoreType type, LiftoffRegList pinned) {
676 UseScratchRegisterScope temps(this);
677 MemOperand dst_op = liftoff::GetMemOp(this, dst_addr, offset_reg, offset_imm);
678 switch (type.value()) {
679 case StoreType::kI64Store8:
680 case StoreType::kI32Store8: {
681 sync();
682 Sb(src.gp(), dst_op);
683 return;
684 }
685 case StoreType::kI64Store16:
686 case StoreType::kI32Store16: {
687 sync();
688 Sh(src.gp(), dst_op);
689 return;
690 }
691 case StoreType::kI64Store32:
692 case StoreType::kI32Store: {
693 sync();
694 Sw(src.gp(), dst_op);
695 return;
696 }
697 case StoreType::kI64Store: {
698 sync();
699 Sd(src.gp(), dst_op);
700 return;
701 }
702 default:
703 UNREACHABLE();
704 }
705 }
706
707 #define ASSEMBLE_ATOMIC_BINOP(load_linked, store_conditional, bin_instr) \
708 do { \
709 Label binop; \
710 sync(); \
711 bind(&binop); \
712 load_linked(result.gp(), MemOperand(temp0, 0)); \
713 bin_instr(temp1, result.gp(), Operand(value.gp())); \
714 store_conditional(temp1, MemOperand(temp0, 0)); \
715 BranchShort(&binop, eq, temp1, Operand(zero_reg)); \
716 sync(); \
717 } while (0)
718
719 #define ASSEMBLE_ATOMIC_BINOP_EXT(load_linked, store_conditional, size, \
720 bin_instr, aligned) \
721 do { \
722 Label binop; \
723 andi(temp3, temp0, aligned); \
724 Dsubu(temp0, temp0, Operand(temp3)); \
725 sll(temp3, temp3, 3); \
726 sync(); \
727 bind(&binop); \
728 load_linked(temp1, MemOperand(temp0, 0)); \
729 ExtractBits(result.gp(), temp1, temp3, size, false); \
730 bin_instr(temp2, result.gp(), value.gp()); \
731 InsertBits(temp1, temp2, temp3, size); \
732 store_conditional(temp1, MemOperand(temp0, 0)); \
733 BranchShort(&binop, eq, temp1, Operand(zero_reg)); \
734 sync(); \
735 } while (0)
736
737 #define ATOMIC_BINOP_CASE(name, inst32, inst64) \
738 void LiftoffAssembler::Atomic##name( \
739 Register dst_addr, Register offset_reg, uintptr_t offset_imm, \
740 LiftoffRegister value, LiftoffRegister result, StoreType type) { \
741 LiftoffRegList pinned = {dst_addr, offset_reg, value, result}; \
742 Register temp0 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); \
743 Register temp1 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); \
744 Register temp2 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); \
745 Register temp3 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); \
746 MemOperand dst_op = \
747 liftoff::GetMemOp(this, dst_addr, offset_reg, offset_imm); \
748 Daddu(temp0, dst_op.rm(), dst_op.offset()); \
749 switch (type.value()) { \
750 case StoreType::kI64Store8: \
751 ASSEMBLE_ATOMIC_BINOP_EXT(Lld, Scd, 8, inst64, 7); \
752 break; \
753 case StoreType::kI32Store8: \
754 ASSEMBLE_ATOMIC_BINOP_EXT(Ll, Sc, 8, inst32, 3); \
755 break; \
756 case StoreType::kI64Store16: \
757 ASSEMBLE_ATOMIC_BINOP_EXT(Lld, Scd, 16, inst64, 7); \
758 break; \
759 case StoreType::kI32Store16: \
760 ASSEMBLE_ATOMIC_BINOP_EXT(Ll, Sc, 16, inst32, 3); \
761 break; \
762 case StoreType::kI64Store32: \
763 ASSEMBLE_ATOMIC_BINOP_EXT(Lld, Scd, 32, inst64, 7); \
764 break; \
765 case StoreType::kI32Store: \
766 ASSEMBLE_ATOMIC_BINOP(Ll, Sc, inst32); \
767 break; \
768 case StoreType::kI64Store: \
769 ASSEMBLE_ATOMIC_BINOP(Lld, Scd, inst64); \
770 break; \
771 default: \
772 UNREACHABLE(); \
773 } \
774 }
775
ATOMIC_BINOP_CASE(Add,Addu,Daddu)776 ATOMIC_BINOP_CASE(Add, Addu, Daddu)
777 ATOMIC_BINOP_CASE(Sub, Subu, Dsubu)
778 ATOMIC_BINOP_CASE(And, And, And)
779 ATOMIC_BINOP_CASE(Or, Or, Or)
780 ATOMIC_BINOP_CASE(Xor, Xor, Xor)
781 #undef ASSEMBLE_ATOMIC_BINOP
782 #undef ASSEMBLE_ATOMIC_BINOP_EXT
783 #undef ATOMIC_BINOP_CASE
784
785 #define ASSEMBLE_ATOMIC_EXCHANGE_INTEGER(load_linked, store_conditional) \
786 do { \
787 Label exchange; \
788 sync(); \
789 bind(&exchange); \
790 load_linked(result.gp(), MemOperand(temp0, 0)); \
791 mov(temp1, value.gp()); \
792 store_conditional(temp1, MemOperand(temp0, 0)); \
793 BranchShort(&exchange, eq, temp1, Operand(zero_reg)); \
794 sync(); \
795 } while (0)
796
797 #define ASSEMBLE_ATOMIC_EXCHANGE_INTEGER_EXT(load_linked, store_conditional, \
798 size, aligned) \
799 do { \
800 Label exchange; \
801 andi(temp1, temp0, aligned); \
802 Dsubu(temp0, temp0, Operand(temp1)); \
803 sll(temp1, temp1, 3); \
804 sync(); \
805 bind(&exchange); \
806 load_linked(temp2, MemOperand(temp0, 0)); \
807 ExtractBits(result.gp(), temp2, temp1, size, false); \
808 InsertBits(temp2, value.gp(), temp1, size); \
809 store_conditional(temp2, MemOperand(temp0, 0)); \
810 BranchShort(&exchange, eq, temp2, Operand(zero_reg)); \
811 sync(); \
812 } while (0)
813
814 void LiftoffAssembler::AtomicExchange(Register dst_addr, Register offset_reg,
815 uintptr_t offset_imm,
816 LiftoffRegister value,
817 LiftoffRegister result, StoreType type) {
818 LiftoffRegList pinned = {dst_addr, offset_reg, value, result};
819 Register temp0 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp();
820 Register temp1 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp();
821 Register temp2 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp();
822 MemOperand dst_op = liftoff::GetMemOp(this, dst_addr, offset_reg, offset_imm);
823 Daddu(temp0, dst_op.rm(), dst_op.offset());
824 switch (type.value()) {
825 case StoreType::kI64Store8:
826 ASSEMBLE_ATOMIC_EXCHANGE_INTEGER_EXT(Lld, Scd, 8, 7);
827 break;
828 case StoreType::kI32Store8:
829 ASSEMBLE_ATOMIC_EXCHANGE_INTEGER_EXT(Ll, Sc, 8, 3);
830 break;
831 case StoreType::kI64Store16:
832 ASSEMBLE_ATOMIC_EXCHANGE_INTEGER_EXT(Lld, Scd, 16, 7);
833 break;
834 case StoreType::kI32Store16:
835 ASSEMBLE_ATOMIC_EXCHANGE_INTEGER_EXT(Ll, Sc, 16, 3);
836 break;
837 case StoreType::kI64Store32:
838 ASSEMBLE_ATOMIC_EXCHANGE_INTEGER_EXT(Lld, Scd, 32, 7);
839 break;
840 case StoreType::kI32Store:
841 ASSEMBLE_ATOMIC_EXCHANGE_INTEGER(Ll, Sc);
842 break;
843 case StoreType::kI64Store:
844 ASSEMBLE_ATOMIC_EXCHANGE_INTEGER(Lld, Scd);
845 break;
846 default:
847 UNREACHABLE();
848 }
849 }
850 #undef ASSEMBLE_ATOMIC_EXCHANGE_INTEGER
851 #undef ASSEMBLE_ATOMIC_EXCHANGE_INTEGER_EXT
852
853 #define ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER(load_linked, \
854 store_conditional) \
855 do { \
856 Label compareExchange; \
857 Label exit; \
858 sync(); \
859 bind(&compareExchange); \
860 load_linked(result.gp(), MemOperand(temp0, 0)); \
861 BranchShort(&exit, ne, expected.gp(), Operand(result.gp())); \
862 mov(temp2, new_value.gp()); \
863 store_conditional(temp2, MemOperand(temp0, 0)); \
864 BranchShort(&compareExchange, eq, temp2, Operand(zero_reg)); \
865 bind(&exit); \
866 sync(); \
867 } while (0)
868
869 #define ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER_EXT( \
870 load_linked, store_conditional, size, aligned) \
871 do { \
872 Label compareExchange; \
873 Label exit; \
874 andi(temp1, temp0, aligned); \
875 Dsubu(temp0, temp0, Operand(temp1)); \
876 sll(temp1, temp1, 3); \
877 sync(); \
878 bind(&compareExchange); \
879 load_linked(temp2, MemOperand(temp0, 0)); \
880 ExtractBits(result.gp(), temp2, temp1, size, false); \
881 ExtractBits(temp2, expected.gp(), zero_reg, size, false); \
882 BranchShort(&exit, ne, temp2, Operand(result.gp())); \
883 InsertBits(temp2, new_value.gp(), temp1, size); \
884 store_conditional(temp2, MemOperand(temp0, 0)); \
885 BranchShort(&compareExchange, eq, temp2, Operand(zero_reg)); \
886 bind(&exit); \
887 sync(); \
888 } while (0)
889
AtomicCompareExchange(Register dst_addr,Register offset_reg,uintptr_t offset_imm,LiftoffRegister expected,LiftoffRegister new_value,LiftoffRegister result,StoreType type)890 void LiftoffAssembler::AtomicCompareExchange(
891 Register dst_addr, Register offset_reg, uintptr_t offset_imm,
892 LiftoffRegister expected, LiftoffRegister new_value, LiftoffRegister result,
893 StoreType type) {
894 LiftoffRegList pinned = {dst_addr, offset_reg, expected, new_value, result};
895 Register temp0 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp();
896 Register temp1 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp();
897 Register temp2 = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp();
898 MemOperand dst_op = liftoff::GetMemOp(this, dst_addr, offset_reg, offset_imm);
899 Daddu(temp0, dst_op.rm(), dst_op.offset());
900 switch (type.value()) {
901 case StoreType::kI64Store8:
902 ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER_EXT(Lld, Scd, 8, 7);
903 break;
904 case StoreType::kI32Store8:
905 ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER_EXT(Ll, Sc, 8, 3);
906 break;
907 case StoreType::kI64Store16:
908 ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER_EXT(Lld, Scd, 16, 7);
909 break;
910 case StoreType::kI32Store16:
911 ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER_EXT(Ll, Sc, 16, 3);
912 break;
913 case StoreType::kI64Store32:
914 ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER_EXT(Lld, Scd, 32, 7);
915 break;
916 case StoreType::kI32Store:
917 ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER(Ll, Sc);
918 break;
919 case StoreType::kI64Store:
920 ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER(Lld, Scd);
921 break;
922 default:
923 UNREACHABLE();
924 }
925 }
926 #undef ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER
927 #undef ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER_EXT
928
AtomicFence()929 void LiftoffAssembler::AtomicFence() { sync(); }
930
LoadCallerFrameSlot(LiftoffRegister dst,uint32_t caller_slot_idx,ValueKind kind)931 void LiftoffAssembler::LoadCallerFrameSlot(LiftoffRegister dst,
932 uint32_t caller_slot_idx,
933 ValueKind kind) {
934 MemOperand src(fp, kSystemPointerSize * (caller_slot_idx + 1));
935 liftoff::Load(this, dst, src, kind);
936 }
937
StoreCallerFrameSlot(LiftoffRegister src,uint32_t caller_slot_idx,ValueKind kind)938 void LiftoffAssembler::StoreCallerFrameSlot(LiftoffRegister src,
939 uint32_t caller_slot_idx,
940 ValueKind kind) {
941 int32_t offset = kSystemPointerSize * (caller_slot_idx + 1);
942 liftoff::Store(this, fp, offset, src, kind);
943 }
944
LoadReturnStackSlot(LiftoffRegister dst,int offset,ValueKind kind)945 void LiftoffAssembler::LoadReturnStackSlot(LiftoffRegister dst, int offset,
946 ValueKind kind) {
947 liftoff::Load(this, dst, MemOperand(sp, offset), kind);
948 }
949
MoveStackValue(uint32_t dst_offset,uint32_t src_offset,ValueKind kind)950 void LiftoffAssembler::MoveStackValue(uint32_t dst_offset, uint32_t src_offset,
951 ValueKind kind) {
952 DCHECK_NE(dst_offset, src_offset);
953 LiftoffRegister reg = GetUnusedRegister(reg_class_for(kind), {});
954 Fill(reg, src_offset, kind);
955 Spill(dst_offset, reg, kind);
956 }
957
Move(Register dst,Register src,ValueKind kind)958 void LiftoffAssembler::Move(Register dst, Register src, ValueKind kind) {
959 DCHECK_NE(dst, src);
960 // TODO(ksreten): Handle different sizes here.
961 TurboAssembler::Move(dst, src);
962 }
963
Move(DoubleRegister dst,DoubleRegister src,ValueKind kind)964 void LiftoffAssembler::Move(DoubleRegister dst, DoubleRegister src,
965 ValueKind kind) {
966 DCHECK_NE(dst, src);
967 if (kind != kS128) {
968 TurboAssembler::Move(dst, src);
969 } else {
970 TurboAssembler::move_v(dst.toW(), src.toW());
971 }
972 }
973
Spill(int offset,LiftoffRegister reg,ValueKind kind)974 void LiftoffAssembler::Spill(int offset, LiftoffRegister reg, ValueKind kind) {
975 RecordUsedSpillOffset(offset);
976 MemOperand dst = liftoff::GetStackSlot(offset);
977 switch (kind) {
978 case kI32:
979 Sw(reg.gp(), dst);
980 break;
981 case kI64:
982 case kRef:
983 case kOptRef:
984 case kRtt:
985 Sd(reg.gp(), dst);
986 break;
987 case kF32:
988 Swc1(reg.fp(), dst);
989 break;
990 case kF64:
991 TurboAssembler::Sdc1(reg.fp(), dst);
992 break;
993 case kS128:
994 TurboAssembler::st_b(reg.fp().toW(), dst);
995 break;
996 default:
997 UNREACHABLE();
998 }
999 }
1000
Spill(int offset,WasmValue value)1001 void LiftoffAssembler::Spill(int offset, WasmValue value) {
1002 RecordUsedSpillOffset(offset);
1003 MemOperand dst = liftoff::GetStackSlot(offset);
1004 switch (value.type().kind()) {
1005 case kI32: {
1006 LiftoffRegister tmp = GetUnusedRegister(kGpReg, {});
1007 TurboAssembler::li(tmp.gp(), Operand(value.to_i32()));
1008 Sw(tmp.gp(), dst);
1009 break;
1010 }
1011 case kI64:
1012 case kRef:
1013 case kOptRef: {
1014 LiftoffRegister tmp = GetUnusedRegister(kGpReg, {});
1015 TurboAssembler::li(tmp.gp(), value.to_i64());
1016 Sd(tmp.gp(), dst);
1017 break;
1018 }
1019 default:
1020 // kWasmF32 and kWasmF64 are unreachable, since those
1021 // constants are not tracked.
1022 UNREACHABLE();
1023 }
1024 }
1025
Fill(LiftoffRegister reg,int offset,ValueKind kind)1026 void LiftoffAssembler::Fill(LiftoffRegister reg, int offset, ValueKind kind) {
1027 MemOperand src = liftoff::GetStackSlot(offset);
1028 switch (kind) {
1029 case kI32:
1030 Lw(reg.gp(), src);
1031 break;
1032 case kI64:
1033 case kRef:
1034 case kOptRef:
1035 Ld(reg.gp(), src);
1036 break;
1037 case kF32:
1038 Lwc1(reg.fp(), src);
1039 break;
1040 case kF64:
1041 TurboAssembler::Ldc1(reg.fp(), src);
1042 break;
1043 case kS128:
1044 TurboAssembler::ld_b(reg.fp().toW(), src);
1045 break;
1046 default:
1047 UNREACHABLE();
1048 }
1049 }
1050
FillI64Half(Register,int offset,RegPairHalf)1051 void LiftoffAssembler::FillI64Half(Register, int offset, RegPairHalf) {
1052 UNREACHABLE();
1053 }
1054
FillStackSlotsWithZero(int start,int size)1055 void LiftoffAssembler::FillStackSlotsWithZero(int start, int size) {
1056 DCHECK_LT(0, size);
1057 RecordUsedSpillOffset(start + size);
1058
1059 if (size <= 12 * kStackSlotSize) {
1060 // Special straight-line code for up to 12 slots. Generates one
1061 // instruction per slot (<= 12 instructions total).
1062 uint32_t remainder = size;
1063 for (; remainder >= kStackSlotSize; remainder -= kStackSlotSize) {
1064 Sd(zero_reg, liftoff::GetStackSlot(start + remainder));
1065 }
1066 DCHECK(remainder == 4 || remainder == 0);
1067 if (remainder) {
1068 Sw(zero_reg, liftoff::GetStackSlot(start + remainder));
1069 }
1070 } else {
1071 // General case for bigger counts (12 instructions).
1072 // Use a0 for start address (inclusive), a1 for end address (exclusive).
1073 Push(a1, a0);
1074 Daddu(a0, fp, Operand(-start - size));
1075 Daddu(a1, fp, Operand(-start));
1076
1077 Label loop;
1078 bind(&loop);
1079 Sd(zero_reg, MemOperand(a0));
1080 daddiu(a0, a0, kSystemPointerSize);
1081 BranchShort(&loop, ne, a0, Operand(a1));
1082
1083 Pop(a1, a0);
1084 }
1085 }
1086
emit_i64_clz(LiftoffRegister dst,LiftoffRegister src)1087 void LiftoffAssembler::emit_i64_clz(LiftoffRegister dst, LiftoffRegister src) {
1088 TurboAssembler::Dclz(dst.gp(), src.gp());
1089 }
1090
emit_i64_ctz(LiftoffRegister dst,LiftoffRegister src)1091 void LiftoffAssembler::emit_i64_ctz(LiftoffRegister dst, LiftoffRegister src) {
1092 TurboAssembler::Dctz(dst.gp(), src.gp());
1093 }
1094
emit_i64_popcnt(LiftoffRegister dst,LiftoffRegister src)1095 bool LiftoffAssembler::emit_i64_popcnt(LiftoffRegister dst,
1096 LiftoffRegister src) {
1097 TurboAssembler::Dpopcnt(dst.gp(), src.gp());
1098 return true;
1099 }
1100
IncrementSmi(LiftoffRegister dst,int offset)1101 void LiftoffAssembler::IncrementSmi(LiftoffRegister dst, int offset) {
1102 UseScratchRegisterScope temps(this);
1103 Register scratch = temps.Acquire();
1104 SmiUntag(scratch, MemOperand(dst.gp(), offset));
1105 Daddu(scratch, scratch, Operand(1));
1106 SmiTag(scratch);
1107 Sd(scratch, MemOperand(dst.gp(), offset));
1108 }
1109
emit_i32_mul(Register dst,Register lhs,Register rhs)1110 void LiftoffAssembler::emit_i32_mul(Register dst, Register lhs, Register rhs) {
1111 TurboAssembler::Mul(dst, lhs, rhs);
1112 }
1113
emit_i32_divs(Register dst,Register lhs,Register rhs,Label * trap_div_by_zero,Label * trap_div_unrepresentable)1114 void LiftoffAssembler::emit_i32_divs(Register dst, Register lhs, Register rhs,
1115 Label* trap_div_by_zero,
1116 Label* trap_div_unrepresentable) {
1117 TurboAssembler::Branch(trap_div_by_zero, eq, rhs, Operand(zero_reg));
1118
1119 // Check if lhs == kMinInt and rhs == -1, since this case is unrepresentable.
1120 TurboAssembler::li(kScratchReg, 1);
1121 TurboAssembler::li(kScratchReg2, 1);
1122 TurboAssembler::LoadZeroOnCondition(kScratchReg, lhs, Operand(kMinInt), eq);
1123 TurboAssembler::LoadZeroOnCondition(kScratchReg2, rhs, Operand(-1), eq);
1124 daddu(kScratchReg, kScratchReg, kScratchReg2);
1125 TurboAssembler::Branch(trap_div_unrepresentable, eq, kScratchReg,
1126 Operand(zero_reg));
1127
1128 TurboAssembler::Div(dst, lhs, rhs);
1129 }
1130
emit_i32_divu(Register dst,Register lhs,Register rhs,Label * trap_div_by_zero)1131 void LiftoffAssembler::emit_i32_divu(Register dst, Register lhs, Register rhs,
1132 Label* trap_div_by_zero) {
1133 TurboAssembler::Branch(trap_div_by_zero, eq, rhs, Operand(zero_reg));
1134 TurboAssembler::Divu(dst, lhs, rhs);
1135 }
1136
emit_i32_rems(Register dst,Register lhs,Register rhs,Label * trap_div_by_zero)1137 void LiftoffAssembler::emit_i32_rems(Register dst, Register lhs, Register rhs,
1138 Label* trap_div_by_zero) {
1139 TurboAssembler::Branch(trap_div_by_zero, eq, rhs, Operand(zero_reg));
1140 TurboAssembler::Mod(dst, lhs, rhs);
1141 }
1142
emit_i32_remu(Register dst,Register lhs,Register rhs,Label * trap_div_by_zero)1143 void LiftoffAssembler::emit_i32_remu(Register dst, Register lhs, Register rhs,
1144 Label* trap_div_by_zero) {
1145 TurboAssembler::Branch(trap_div_by_zero, eq, rhs, Operand(zero_reg));
1146 TurboAssembler::Modu(dst, lhs, rhs);
1147 }
1148
1149 #define I32_BINOP(name, instruction) \
1150 void LiftoffAssembler::emit_i32_##name(Register dst, Register lhs, \
1151 Register rhs) { \
1152 instruction(dst, lhs, rhs); \
1153 }
1154
1155 // clang-format off
I32_BINOP(add,addu)1156 I32_BINOP(add, addu)
1157 I32_BINOP(sub, subu)
1158 I32_BINOP(and, and_)
1159 I32_BINOP(or, or_)
1160 I32_BINOP(xor, xor_)
1161 // clang-format on
1162
1163 #undef I32_BINOP
1164
1165 #define I32_BINOP_I(name, instruction) \
1166 void LiftoffAssembler::emit_i32_##name##i(Register dst, Register lhs, \
1167 int32_t imm) { \
1168 instruction(dst, lhs, Operand(imm)); \
1169 }
1170
1171 // clang-format off
1172 I32_BINOP_I(add, Addu)
1173 I32_BINOP_I(sub, Subu)
1174 I32_BINOP_I(and, And)
1175 I32_BINOP_I(or, Or)
1176 I32_BINOP_I(xor, Xor)
1177 // clang-format on
1178
1179 #undef I32_BINOP_I
1180
1181 void LiftoffAssembler::emit_i32_clz(Register dst, Register src) {
1182 TurboAssembler::Clz(dst, src);
1183 }
1184
emit_i32_ctz(Register dst,Register src)1185 void LiftoffAssembler::emit_i32_ctz(Register dst, Register src) {
1186 TurboAssembler::Ctz(dst, src);
1187 }
1188
emit_i32_popcnt(Register dst,Register src)1189 bool LiftoffAssembler::emit_i32_popcnt(Register dst, Register src) {
1190 TurboAssembler::Popcnt(dst, src);
1191 return true;
1192 }
1193
1194 #define I32_SHIFTOP(name, instruction) \
1195 void LiftoffAssembler::emit_i32_##name(Register dst, Register src, \
1196 Register amount) { \
1197 instruction(dst, src, amount); \
1198 }
1199 #define I32_SHIFTOP_I(name, instruction) \
1200 I32_SHIFTOP(name, instruction##v) \
1201 void LiftoffAssembler::emit_i32_##name##i(Register dst, Register src, \
1202 int amount) { \
1203 instruction(dst, src, amount & 31); \
1204 }
1205
I32_SHIFTOP_I(shl,sll)1206 I32_SHIFTOP_I(shl, sll)
1207 I32_SHIFTOP_I(sar, sra)
1208 I32_SHIFTOP_I(shr, srl)
1209
1210 #undef I32_SHIFTOP
1211 #undef I32_SHIFTOP_I
1212
1213 void LiftoffAssembler::emit_i64_addi(LiftoffRegister dst, LiftoffRegister lhs,
1214 int64_t imm) {
1215 TurboAssembler::Daddu(dst.gp(), lhs.gp(), Operand(imm));
1216 }
1217
emit_i64_mul(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)1218 void LiftoffAssembler::emit_i64_mul(LiftoffRegister dst, LiftoffRegister lhs,
1219 LiftoffRegister rhs) {
1220 TurboAssembler::Dmul(dst.gp(), lhs.gp(), rhs.gp());
1221 }
1222
emit_i64_divs(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs,Label * trap_div_by_zero,Label * trap_div_unrepresentable)1223 bool LiftoffAssembler::emit_i64_divs(LiftoffRegister dst, LiftoffRegister lhs,
1224 LiftoffRegister rhs,
1225 Label* trap_div_by_zero,
1226 Label* trap_div_unrepresentable) {
1227 TurboAssembler::Branch(trap_div_by_zero, eq, rhs.gp(), Operand(zero_reg));
1228
1229 // Check if lhs == MinInt64 and rhs == -1, since this case is unrepresentable.
1230 TurboAssembler::li(kScratchReg, 1);
1231 TurboAssembler::li(kScratchReg2, 1);
1232 TurboAssembler::LoadZeroOnCondition(
1233 kScratchReg, lhs.gp(), Operand(std::numeric_limits<int64_t>::min()), eq);
1234 TurboAssembler::LoadZeroOnCondition(kScratchReg2, rhs.gp(), Operand(-1), eq);
1235 daddu(kScratchReg, kScratchReg, kScratchReg2);
1236 TurboAssembler::Branch(trap_div_unrepresentable, eq, kScratchReg,
1237 Operand(zero_reg));
1238
1239 TurboAssembler::Ddiv(dst.gp(), lhs.gp(), rhs.gp());
1240 return true;
1241 }
1242
emit_i64_divu(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs,Label * trap_div_by_zero)1243 bool LiftoffAssembler::emit_i64_divu(LiftoffRegister dst, LiftoffRegister lhs,
1244 LiftoffRegister rhs,
1245 Label* trap_div_by_zero) {
1246 TurboAssembler::Branch(trap_div_by_zero, eq, rhs.gp(), Operand(zero_reg));
1247 TurboAssembler::Ddivu(dst.gp(), lhs.gp(), rhs.gp());
1248 return true;
1249 }
1250
emit_i64_rems(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs,Label * trap_div_by_zero)1251 bool LiftoffAssembler::emit_i64_rems(LiftoffRegister dst, LiftoffRegister lhs,
1252 LiftoffRegister rhs,
1253 Label* trap_div_by_zero) {
1254 TurboAssembler::Branch(trap_div_by_zero, eq, rhs.gp(), Operand(zero_reg));
1255 TurboAssembler::Dmod(dst.gp(), lhs.gp(), rhs.gp());
1256 return true;
1257 }
1258
emit_i64_remu(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs,Label * trap_div_by_zero)1259 bool LiftoffAssembler::emit_i64_remu(LiftoffRegister dst, LiftoffRegister lhs,
1260 LiftoffRegister rhs,
1261 Label* trap_div_by_zero) {
1262 TurboAssembler::Branch(trap_div_by_zero, eq, rhs.gp(), Operand(zero_reg));
1263 TurboAssembler::Dmodu(dst.gp(), lhs.gp(), rhs.gp());
1264 return true;
1265 }
1266
1267 #define I64_BINOP(name, instruction) \
1268 void LiftoffAssembler::emit_i64_##name( \
1269 LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs) { \
1270 instruction(dst.gp(), lhs.gp(), rhs.gp()); \
1271 }
1272
1273 // clang-format off
I64_BINOP(add,daddu)1274 I64_BINOP(add, daddu)
1275 I64_BINOP(sub, dsubu)
1276 I64_BINOP(and, and_)
1277 I64_BINOP(or, or_)
1278 I64_BINOP(xor, xor_)
1279 // clang-format on
1280
1281 #undef I64_BINOP
1282
1283 #define I64_BINOP_I(name, instruction) \
1284 void LiftoffAssembler::emit_i64_##name##i( \
1285 LiftoffRegister dst, LiftoffRegister lhs, int32_t imm) { \
1286 instruction(dst.gp(), lhs.gp(), Operand(imm)); \
1287 }
1288
1289 // clang-format off
1290 I64_BINOP_I(and, And)
1291 I64_BINOP_I(or, Or)
1292 I64_BINOP_I(xor, Xor)
1293 // clang-format on
1294
1295 #undef I64_BINOP_I
1296
1297 #define I64_SHIFTOP(name, instruction) \
1298 void LiftoffAssembler::emit_i64_##name( \
1299 LiftoffRegister dst, LiftoffRegister src, Register amount) { \
1300 instruction(dst.gp(), src.gp(), amount); \
1301 }
1302 #define I64_SHIFTOP_I(name, instruction) \
1303 I64_SHIFTOP(name, instruction##v) \
1304 void LiftoffAssembler::emit_i64_##name##i(LiftoffRegister dst, \
1305 LiftoffRegister src, int amount) { \
1306 amount &= 63; \
1307 if (amount < 32) \
1308 instruction(dst.gp(), src.gp(), amount); \
1309 else \
1310 instruction##32(dst.gp(), src.gp(), amount - 32); \
1311 }
1312
1313 I64_SHIFTOP_I(shl, dsll)
1314 I64_SHIFTOP_I(sar, dsra)
1315 I64_SHIFTOP_I(shr, dsrl)
1316
1317 #undef I64_SHIFTOP
1318 #undef I64_SHIFTOP_I
1319
1320 void LiftoffAssembler::emit_u32_to_uintptr(Register dst, Register src) {
1321 Dext(dst, src, 0, 32);
1322 }
1323
emit_f32_neg(DoubleRegister dst,DoubleRegister src)1324 void LiftoffAssembler::emit_f32_neg(DoubleRegister dst, DoubleRegister src) {
1325 TurboAssembler::Neg_s(dst, src);
1326 }
1327
emit_f64_neg(DoubleRegister dst,DoubleRegister src)1328 void LiftoffAssembler::emit_f64_neg(DoubleRegister dst, DoubleRegister src) {
1329 TurboAssembler::Neg_d(dst, src);
1330 }
1331
emit_f32_min(DoubleRegister dst,DoubleRegister lhs,DoubleRegister rhs)1332 void LiftoffAssembler::emit_f32_min(DoubleRegister dst, DoubleRegister lhs,
1333 DoubleRegister rhs) {
1334 Label ool, done;
1335 TurboAssembler::Float32Min(dst, lhs, rhs, &ool);
1336 Branch(&done);
1337
1338 bind(&ool);
1339 TurboAssembler::Float32MinOutOfLine(dst, lhs, rhs);
1340 bind(&done);
1341 }
1342
emit_f32_max(DoubleRegister dst,DoubleRegister lhs,DoubleRegister rhs)1343 void LiftoffAssembler::emit_f32_max(DoubleRegister dst, DoubleRegister lhs,
1344 DoubleRegister rhs) {
1345 Label ool, done;
1346 TurboAssembler::Float32Max(dst, lhs, rhs, &ool);
1347 Branch(&done);
1348
1349 bind(&ool);
1350 TurboAssembler::Float32MaxOutOfLine(dst, lhs, rhs);
1351 bind(&done);
1352 }
1353
emit_f32_copysign(DoubleRegister dst,DoubleRegister lhs,DoubleRegister rhs)1354 void LiftoffAssembler::emit_f32_copysign(DoubleRegister dst, DoubleRegister lhs,
1355 DoubleRegister rhs) {
1356 if (CpuFeatures::IsSupported(MIPS_SIMD)) {
1357 DoubleRegister scratch = rhs;
1358 if (dst == rhs) {
1359 scratch = kScratchDoubleReg;
1360 Move_d(scratch, rhs);
1361 }
1362 if (dst != lhs) {
1363 Move_d(dst, lhs);
1364 }
1365 binsli_w(dst.toW(), scratch.toW(), 0);
1366 } else {
1367 UseScratchRegisterScope temps(this);
1368 Register scratch1 = temps.Acquire();
1369 Register scratch2 = temps.Acquire();
1370 mfc1(scratch1, lhs);
1371 mfc1(scratch2, rhs);
1372 srl(scratch2, scratch2, 31);
1373 Ins(scratch1, scratch2, 31, 1);
1374 mtc1(scratch1, dst);
1375 }
1376 }
1377
emit_f64_min(DoubleRegister dst,DoubleRegister lhs,DoubleRegister rhs)1378 void LiftoffAssembler::emit_f64_min(DoubleRegister dst, DoubleRegister lhs,
1379 DoubleRegister rhs) {
1380 Label ool, done;
1381 TurboAssembler::Float64Min(dst, lhs, rhs, &ool);
1382 Branch(&done);
1383
1384 bind(&ool);
1385 TurboAssembler::Float64MinOutOfLine(dst, lhs, rhs);
1386 bind(&done);
1387 }
1388
emit_f64_max(DoubleRegister dst,DoubleRegister lhs,DoubleRegister rhs)1389 void LiftoffAssembler::emit_f64_max(DoubleRegister dst, DoubleRegister lhs,
1390 DoubleRegister rhs) {
1391 Label ool, done;
1392 TurboAssembler::Float64Max(dst, lhs, rhs, &ool);
1393 Branch(&done);
1394
1395 bind(&ool);
1396 TurboAssembler::Float64MaxOutOfLine(dst, lhs, rhs);
1397 bind(&done);
1398 }
1399
emit_f64_copysign(DoubleRegister dst,DoubleRegister lhs,DoubleRegister rhs)1400 void LiftoffAssembler::emit_f64_copysign(DoubleRegister dst, DoubleRegister lhs,
1401 DoubleRegister rhs) {
1402 if (CpuFeatures::IsSupported(MIPS_SIMD)) {
1403 DoubleRegister scratch = rhs;
1404 if (dst == rhs) {
1405 scratch = kScratchDoubleReg;
1406 Move_d(scratch, rhs);
1407 }
1408 if (dst != lhs) {
1409 Move_d(dst, lhs);
1410 }
1411 binsli_d(dst.toW(), scratch.toW(), 0);
1412 } else {
1413 UseScratchRegisterScope temps(this);
1414 Register scratch1 = temps.Acquire();
1415 Register scratch2 = temps.Acquire();
1416 dmfc1(scratch1, lhs);
1417 dmfc1(scratch2, rhs);
1418 dsrl32(scratch2, scratch2, 31);
1419 Dins(scratch1, scratch2, 63, 1);
1420 dmtc1(scratch1, dst);
1421 }
1422 }
1423
1424 #define FP_BINOP(name, instruction) \
1425 void LiftoffAssembler::emit_##name(DoubleRegister dst, DoubleRegister lhs, \
1426 DoubleRegister rhs) { \
1427 instruction(dst, lhs, rhs); \
1428 }
1429 #define FP_UNOP(name, instruction) \
1430 void LiftoffAssembler::emit_##name(DoubleRegister dst, DoubleRegister src) { \
1431 instruction(dst, src); \
1432 }
1433 #define FP_UNOP_RETURN_TRUE(name, instruction) \
1434 bool LiftoffAssembler::emit_##name(DoubleRegister dst, DoubleRegister src) { \
1435 instruction(dst, src); \
1436 return true; \
1437 }
1438
FP_BINOP(f32_add,add_s)1439 FP_BINOP(f32_add, add_s)
1440 FP_BINOP(f32_sub, sub_s)
1441 FP_BINOP(f32_mul, mul_s)
1442 FP_BINOP(f32_div, div_s)
1443 FP_UNOP(f32_abs, abs_s)
1444 FP_UNOP_RETURN_TRUE(f32_ceil, Ceil_s_s)
1445 FP_UNOP_RETURN_TRUE(f32_floor, Floor_s_s)
1446 FP_UNOP_RETURN_TRUE(f32_trunc, Trunc_s_s)
1447 FP_UNOP_RETURN_TRUE(f32_nearest_int, Round_s_s)
1448 FP_UNOP(f32_sqrt, sqrt_s)
1449 FP_BINOP(f64_add, add_d)
1450 FP_BINOP(f64_sub, sub_d)
1451 FP_BINOP(f64_mul, mul_d)
1452 FP_BINOP(f64_div, div_d)
1453 FP_UNOP(f64_abs, abs_d)
1454 FP_UNOP_RETURN_TRUE(f64_ceil, Ceil_d_d)
1455 FP_UNOP_RETURN_TRUE(f64_floor, Floor_d_d)
1456 FP_UNOP_RETURN_TRUE(f64_trunc, Trunc_d_d)
1457 FP_UNOP_RETURN_TRUE(f64_nearest_int, Round_d_d)
1458 FP_UNOP(f64_sqrt, sqrt_d)
1459
1460 #undef FP_BINOP
1461 #undef FP_UNOP
1462 #undef FP_UNOP_RETURN_TRUE
1463
1464 bool LiftoffAssembler::emit_type_conversion(WasmOpcode opcode,
1465 LiftoffRegister dst,
1466 LiftoffRegister src, Label* trap) {
1467 switch (opcode) {
1468 case kExprI32ConvertI64:
1469 TurboAssembler::Ext(dst.gp(), src.gp(), 0, 32);
1470 return true;
1471 case kExprI32SConvertF32: {
1472 LiftoffRegister rounded = GetUnusedRegister(kFpReg, LiftoffRegList{src});
1473 LiftoffRegister converted_back =
1474 GetUnusedRegister(kFpReg, LiftoffRegList{src, rounded});
1475
1476 // Real conversion.
1477 TurboAssembler::Trunc_s_s(rounded.fp(), src.fp());
1478 trunc_w_s(kScratchDoubleReg, rounded.fp());
1479 mfc1(dst.gp(), kScratchDoubleReg);
1480 // Avoid INT32_MAX as an overflow indicator and use INT32_MIN instead,
1481 // because INT32_MIN allows easier out-of-bounds detection.
1482 TurboAssembler::Addu(kScratchReg, dst.gp(), 1);
1483 TurboAssembler::Slt(kScratchReg2, kScratchReg, dst.gp());
1484 TurboAssembler::Movn(dst.gp(), kScratchReg, kScratchReg2);
1485
1486 // Checking if trap.
1487 mtc1(dst.gp(), kScratchDoubleReg);
1488 cvt_s_w(converted_back.fp(), kScratchDoubleReg);
1489 TurboAssembler::CompareF32(EQ, rounded.fp(), converted_back.fp());
1490 TurboAssembler::BranchFalseF(trap);
1491 return true;
1492 }
1493 case kExprI32UConvertF32: {
1494 LiftoffRegister rounded = GetUnusedRegister(kFpReg, LiftoffRegList{src});
1495 LiftoffRegister converted_back =
1496 GetUnusedRegister(kFpReg, LiftoffRegList{src, rounded});
1497
1498 // Real conversion.
1499 TurboAssembler::Trunc_s_s(rounded.fp(), src.fp());
1500 TurboAssembler::Trunc_uw_s(dst.gp(), rounded.fp(), kScratchDoubleReg);
1501 // Avoid UINT32_MAX as an overflow indicator and use 0 instead,
1502 // because 0 allows easier out-of-bounds detection.
1503 TurboAssembler::Addu(kScratchReg, dst.gp(), 1);
1504 TurboAssembler::Movz(dst.gp(), zero_reg, kScratchReg);
1505
1506 // Checking if trap.
1507 TurboAssembler::Cvt_d_uw(converted_back.fp(), dst.gp());
1508 cvt_s_d(converted_back.fp(), converted_back.fp());
1509 TurboAssembler::CompareF32(EQ, rounded.fp(), converted_back.fp());
1510 TurboAssembler::BranchFalseF(trap);
1511 return true;
1512 }
1513 case kExprI32SConvertF64: {
1514 LiftoffRegister rounded = GetUnusedRegister(kFpReg, LiftoffRegList{src});
1515 LiftoffRegister converted_back =
1516 GetUnusedRegister(kFpReg, LiftoffRegList{src, rounded});
1517
1518 // Real conversion.
1519 TurboAssembler::Trunc_d_d(rounded.fp(), src.fp());
1520 trunc_w_d(kScratchDoubleReg, rounded.fp());
1521 mfc1(dst.gp(), kScratchDoubleReg);
1522
1523 // Checking if trap.
1524 cvt_d_w(converted_back.fp(), kScratchDoubleReg);
1525 TurboAssembler::CompareF64(EQ, rounded.fp(), converted_back.fp());
1526 TurboAssembler::BranchFalseF(trap);
1527 return true;
1528 }
1529 case kExprI32UConvertF64: {
1530 LiftoffRegister rounded = GetUnusedRegister(kFpReg, LiftoffRegList{src});
1531 LiftoffRegister converted_back =
1532 GetUnusedRegister(kFpReg, LiftoffRegList{src, rounded});
1533
1534 // Real conversion.
1535 TurboAssembler::Trunc_d_d(rounded.fp(), src.fp());
1536 TurboAssembler::Trunc_uw_d(dst.gp(), rounded.fp(), kScratchDoubleReg);
1537
1538 // Checking if trap.
1539 TurboAssembler::Cvt_d_uw(converted_back.fp(), dst.gp());
1540 TurboAssembler::CompareF64(EQ, rounded.fp(), converted_back.fp());
1541 TurboAssembler::BranchFalseF(trap);
1542 return true;
1543 }
1544 case kExprI32ReinterpretF32:
1545 TurboAssembler::FmoveLow(dst.gp(), src.fp());
1546 return true;
1547 case kExprI64SConvertI32:
1548 sll(dst.gp(), src.gp(), 0);
1549 return true;
1550 case kExprI64UConvertI32:
1551 TurboAssembler::Dext(dst.gp(), src.gp(), 0, 32);
1552 return true;
1553 case kExprI64SConvertF32: {
1554 LiftoffRegister rounded = GetUnusedRegister(kFpReg, LiftoffRegList{src});
1555 LiftoffRegister converted_back =
1556 GetUnusedRegister(kFpReg, LiftoffRegList{src, rounded});
1557
1558 // Real conversion.
1559 TurboAssembler::Trunc_s_s(rounded.fp(), src.fp());
1560 trunc_l_s(kScratchDoubleReg, rounded.fp());
1561 dmfc1(dst.gp(), kScratchDoubleReg);
1562 // Avoid INT64_MAX as an overflow indicator and use INT64_MIN instead,
1563 // because INT64_MIN allows easier out-of-bounds detection.
1564 TurboAssembler::Daddu(kScratchReg, dst.gp(), 1);
1565 TurboAssembler::Slt(kScratchReg2, kScratchReg, dst.gp());
1566 TurboAssembler::Movn(dst.gp(), kScratchReg, kScratchReg2);
1567
1568 // Checking if trap.
1569 dmtc1(dst.gp(), kScratchDoubleReg);
1570 cvt_s_l(converted_back.fp(), kScratchDoubleReg);
1571 TurboAssembler::CompareF32(EQ, rounded.fp(), converted_back.fp());
1572 TurboAssembler::BranchFalseF(trap);
1573 return true;
1574 }
1575 case kExprI64UConvertF32: {
1576 // Real conversion.
1577 TurboAssembler::Trunc_ul_s(dst.gp(), src.fp(), kScratchDoubleReg,
1578 kScratchReg);
1579
1580 // Checking if trap.
1581 TurboAssembler::Branch(trap, eq, kScratchReg, Operand(zero_reg));
1582 return true;
1583 }
1584 case kExprI64SConvertF64: {
1585 LiftoffRegister rounded = GetUnusedRegister(kFpReg, LiftoffRegList{src});
1586 LiftoffRegister converted_back =
1587 GetUnusedRegister(kFpReg, LiftoffRegList{src, rounded});
1588
1589 // Real conversion.
1590 TurboAssembler::Trunc_d_d(rounded.fp(), src.fp());
1591 trunc_l_d(kScratchDoubleReg, rounded.fp());
1592 dmfc1(dst.gp(), kScratchDoubleReg);
1593 // Avoid INT64_MAX as an overflow indicator and use INT64_MIN instead,
1594 // because INT64_MIN allows easier out-of-bounds detection.
1595 TurboAssembler::Daddu(kScratchReg, dst.gp(), 1);
1596 TurboAssembler::Slt(kScratchReg2, kScratchReg, dst.gp());
1597 TurboAssembler::Movn(dst.gp(), kScratchReg, kScratchReg2);
1598
1599 // Checking if trap.
1600 dmtc1(dst.gp(), kScratchDoubleReg);
1601 cvt_d_l(converted_back.fp(), kScratchDoubleReg);
1602 TurboAssembler::CompareF64(EQ, rounded.fp(), converted_back.fp());
1603 TurboAssembler::BranchFalseF(trap);
1604 return true;
1605 }
1606 case kExprI64UConvertF64: {
1607 // Real conversion.
1608 TurboAssembler::Trunc_ul_d(dst.gp(), src.fp(), kScratchDoubleReg,
1609 kScratchReg);
1610
1611 // Checking if trap.
1612 TurboAssembler::Branch(trap, eq, kScratchReg, Operand(zero_reg));
1613 return true;
1614 }
1615 case kExprI64ReinterpretF64:
1616 dmfc1(dst.gp(), src.fp());
1617 return true;
1618 case kExprF32SConvertI32: {
1619 LiftoffRegister scratch = GetUnusedRegister(kFpReg, LiftoffRegList{dst});
1620 mtc1(src.gp(), scratch.fp());
1621 cvt_s_w(dst.fp(), scratch.fp());
1622 return true;
1623 }
1624 case kExprF32UConvertI32:
1625 TurboAssembler::Cvt_s_uw(dst.fp(), src.gp());
1626 return true;
1627 case kExprF32ConvertF64:
1628 cvt_s_d(dst.fp(), src.fp());
1629 return true;
1630 case kExprF32ReinterpretI32:
1631 TurboAssembler::FmoveLow(dst.fp(), src.gp());
1632 return true;
1633 case kExprF64SConvertI32: {
1634 LiftoffRegister scratch = GetUnusedRegister(kFpReg, LiftoffRegList{dst});
1635 mtc1(src.gp(), scratch.fp());
1636 cvt_d_w(dst.fp(), scratch.fp());
1637 return true;
1638 }
1639 case kExprF64UConvertI32:
1640 TurboAssembler::Cvt_d_uw(dst.fp(), src.gp());
1641 return true;
1642 case kExprF64ConvertF32:
1643 cvt_d_s(dst.fp(), src.fp());
1644 return true;
1645 case kExprF64ReinterpretI64:
1646 dmtc1(src.gp(), dst.fp());
1647 return true;
1648 case kExprI32SConvertSatF32: {
1649 // Other arches use round to zero here, so we follow.
1650 if (CpuFeatures::IsSupported(MIPS_SIMD)) {
1651 trunc_w_s(kScratchDoubleReg, src.fp());
1652 mfc1(dst.gp(), kScratchDoubleReg);
1653 } else {
1654 Label done;
1655 mov(dst.gp(), zero_reg);
1656 CompareIsNanF32(src.fp(), src.fp());
1657 BranchTrueShortF(&done);
1658 li(dst.gp(), static_cast<int32_t>(std::numeric_limits<int32_t>::min()));
1659 TurboAssembler::Move(
1660 kScratchDoubleReg,
1661 static_cast<float>(std::numeric_limits<int32_t>::min()));
1662 CompareF32(OLT, src.fp(), kScratchDoubleReg);
1663 BranchTrueShortF(&done);
1664 trunc_w_s(kScratchDoubleReg, src.fp());
1665 mfc1(dst.gp(), kScratchDoubleReg);
1666 bind(&done);
1667 }
1668 return true;
1669 }
1670 case kExprI32UConvertSatF32: {
1671 Label isnan_or_lessthan_or_equal_zero;
1672 mov(dst.gp(), zero_reg);
1673 TurboAssembler::Move(kScratchDoubleReg, static_cast<float>(0.0));
1674 CompareF32(ULE, src.fp(), kScratchDoubleReg);
1675 BranchTrueShortF(&isnan_or_lessthan_or_equal_zero);
1676 Trunc_uw_s(dst.gp(), src.fp(), kScratchDoubleReg);
1677 bind(&isnan_or_lessthan_or_equal_zero);
1678 return true;
1679 }
1680 case kExprI32SConvertSatF64: {
1681 if (CpuFeatures::IsSupported(MIPS_SIMD)) {
1682 trunc_w_d(kScratchDoubleReg, src.fp());
1683 mfc1(dst.gp(), kScratchDoubleReg);
1684 } else {
1685 Label done;
1686 mov(dst.gp(), zero_reg);
1687 CompareIsNanF64(src.fp(), src.fp());
1688 BranchTrueShortF(&done);
1689 li(dst.gp(), static_cast<int32_t>(std::numeric_limits<int32_t>::min()));
1690 TurboAssembler::Move(
1691 kScratchDoubleReg,
1692 static_cast<double>(std::numeric_limits<int32_t>::min()));
1693 CompareF64(OLT, src.fp(), kScratchDoubleReg);
1694 BranchTrueShortF(&done);
1695 trunc_w_d(kScratchDoubleReg, src.fp());
1696 mfc1(dst.gp(), kScratchDoubleReg);
1697 bind(&done);
1698 }
1699 return true;
1700 }
1701 case kExprI32UConvertSatF64: {
1702 Label isnan_or_lessthan_or_equal_zero;
1703 mov(dst.gp(), zero_reg);
1704 TurboAssembler::Move(kScratchDoubleReg, static_cast<double>(0.0));
1705 CompareF64(ULE, src.fp(), kScratchDoubleReg);
1706 BranchTrueShortF(&isnan_or_lessthan_or_equal_zero);
1707 Trunc_uw_d(dst.gp(), src.fp(), kScratchDoubleReg);
1708 bind(&isnan_or_lessthan_or_equal_zero);
1709 return true;
1710 }
1711 case kExprI64SConvertSatF32: {
1712 if (CpuFeatures::IsSupported(MIPS_SIMD)) {
1713 trunc_l_s(kScratchDoubleReg, src.fp());
1714 dmfc1(dst.gp(), kScratchDoubleReg);
1715 } else {
1716 Label done;
1717 mov(dst.gp(), zero_reg);
1718 CompareIsNanF32(src.fp(), src.fp());
1719 BranchTrueShortF(&done);
1720 li(dst.gp(), static_cast<int64_t>(std::numeric_limits<int64_t>::min()));
1721 TurboAssembler::Move(
1722 kScratchDoubleReg,
1723 static_cast<float>(std::numeric_limits<int64_t>::min()));
1724 CompareF32(OLT, src.fp(), kScratchDoubleReg);
1725 BranchTrueShortF(&done);
1726 trunc_l_s(kScratchDoubleReg, src.fp());
1727 dmfc1(dst.gp(), kScratchDoubleReg);
1728 bind(&done);
1729 }
1730 return true;
1731 }
1732 case kExprI64UConvertSatF32: {
1733 Label isnan_or_lessthan_or_equal_zero;
1734 mov(dst.gp(), zero_reg);
1735 TurboAssembler::Move(kScratchDoubleReg, static_cast<float>(0.0));
1736 CompareF32(ULE, src.fp(), kScratchDoubleReg);
1737 BranchTrueShortF(&isnan_or_lessthan_or_equal_zero);
1738 Trunc_ul_s(dst.gp(), src.fp(), kScratchDoubleReg, no_reg);
1739 bind(&isnan_or_lessthan_or_equal_zero);
1740 return true;
1741 }
1742 case kExprI64SConvertSatF64: {
1743 if (CpuFeatures::IsSupported(MIPS_SIMD)) {
1744 trunc_l_d(kScratchDoubleReg, src.fp());
1745 dmfc1(dst.gp(), kScratchDoubleReg);
1746 } else {
1747 Label done;
1748 mov(dst.gp(), zero_reg);
1749 CompareIsNanF64(src.fp(), src.fp());
1750 BranchTrueShortF(&done);
1751 li(dst.gp(), static_cast<int64_t>(std::numeric_limits<int64_t>::min()));
1752 TurboAssembler::Move(
1753 kScratchDoubleReg,
1754 static_cast<double>(std::numeric_limits<int64_t>::min()));
1755 CompareF64(OLT, src.fp(), kScratchDoubleReg);
1756 BranchTrueShortF(&done);
1757 trunc_l_d(kScratchDoubleReg, src.fp());
1758 dmfc1(dst.gp(), kScratchDoubleReg);
1759 bind(&done);
1760 }
1761 return true;
1762 }
1763 case kExprI64UConvertSatF64: {
1764 Label isnan_or_lessthan_or_equal_zero;
1765 mov(dst.gp(), zero_reg);
1766 TurboAssembler::Move(kScratchDoubleReg, static_cast<double>(0.0));
1767 CompareF64(ULE, src.fp(), kScratchDoubleReg);
1768 BranchTrueShortF(&isnan_or_lessthan_or_equal_zero);
1769 Trunc_ul_d(dst.gp(), src.fp(), kScratchDoubleReg, no_reg);
1770 bind(&isnan_or_lessthan_or_equal_zero);
1771 return true;
1772 }
1773 default:
1774 return false;
1775 }
1776 }
1777
emit_i32_signextend_i8(Register dst,Register src)1778 void LiftoffAssembler::emit_i32_signextend_i8(Register dst, Register src) {
1779 seb(dst, src);
1780 }
1781
emit_i32_signextend_i16(Register dst,Register src)1782 void LiftoffAssembler::emit_i32_signextend_i16(Register dst, Register src) {
1783 seh(dst, src);
1784 }
1785
emit_i64_signextend_i8(LiftoffRegister dst,LiftoffRegister src)1786 void LiftoffAssembler::emit_i64_signextend_i8(LiftoffRegister dst,
1787 LiftoffRegister src) {
1788 seb(dst.gp(), src.gp());
1789 }
1790
emit_i64_signextend_i16(LiftoffRegister dst,LiftoffRegister src)1791 void LiftoffAssembler::emit_i64_signextend_i16(LiftoffRegister dst,
1792 LiftoffRegister src) {
1793 seh(dst.gp(), src.gp());
1794 }
1795
emit_i64_signextend_i32(LiftoffRegister dst,LiftoffRegister src)1796 void LiftoffAssembler::emit_i64_signextend_i32(LiftoffRegister dst,
1797 LiftoffRegister src) {
1798 sll(dst.gp(), src.gp(), 0);
1799 }
1800
emit_jump(Label * label)1801 void LiftoffAssembler::emit_jump(Label* label) {
1802 TurboAssembler::Branch(label);
1803 }
1804
emit_jump(Register target)1805 void LiftoffAssembler::emit_jump(Register target) {
1806 TurboAssembler::Jump(target);
1807 }
1808
emit_cond_jump(LiftoffCondition liftoff_cond,Label * label,ValueKind kind,Register lhs,Register rhs)1809 void LiftoffAssembler::emit_cond_jump(LiftoffCondition liftoff_cond,
1810 Label* label, ValueKind kind,
1811 Register lhs, Register rhs) {
1812 Condition cond = liftoff::ToCondition(liftoff_cond);
1813 if (rhs == no_reg) {
1814 DCHECK(kind == kI32 || kind == kI64);
1815 TurboAssembler::Branch(label, cond, lhs, Operand(zero_reg));
1816 } else {
1817 DCHECK((kind == kI32 || kind == kI64) ||
1818 (is_reference(kind) &&
1819 (liftoff_cond == kEqual || liftoff_cond == kUnequal)));
1820 TurboAssembler::Branch(label, cond, lhs, Operand(rhs));
1821 }
1822 }
1823
emit_i32_cond_jumpi(LiftoffCondition liftoff_cond,Label * label,Register lhs,int32_t imm)1824 void LiftoffAssembler::emit_i32_cond_jumpi(LiftoffCondition liftoff_cond,
1825 Label* label, Register lhs,
1826 int32_t imm) {
1827 Condition cond = liftoff::ToCondition(liftoff_cond);
1828 TurboAssembler::Branch(label, cond, lhs, Operand(imm));
1829 }
1830
emit_i32_subi_jump_negative(Register value,int subtrahend,Label * result_negative)1831 void LiftoffAssembler::emit_i32_subi_jump_negative(Register value,
1832 int subtrahend,
1833 Label* result_negative) {
1834 TurboAssembler::Dsubu(value, value, Operand(subtrahend));
1835 TurboAssembler::Branch(result_negative, less, value, Operand(zero_reg));
1836 }
1837
emit_i32_eqz(Register dst,Register src)1838 void LiftoffAssembler::emit_i32_eqz(Register dst, Register src) {
1839 sltiu(dst, src, 1);
1840 }
1841
emit_i32_set_cond(LiftoffCondition liftoff_cond,Register dst,Register lhs,Register rhs)1842 void LiftoffAssembler::emit_i32_set_cond(LiftoffCondition liftoff_cond,
1843 Register dst, Register lhs,
1844 Register rhs) {
1845 Condition cond = liftoff::ToCondition(liftoff_cond);
1846 Register tmp = dst;
1847 if (dst == lhs || dst == rhs) {
1848 tmp = GetUnusedRegister(kGpReg, LiftoffRegList{lhs, rhs}).gp();
1849 }
1850 // Write 1 as result.
1851 TurboAssembler::li(tmp, 1);
1852
1853 // If negative condition is true, write 0 as result.
1854 Condition neg_cond = NegateCondition(cond);
1855 TurboAssembler::LoadZeroOnCondition(tmp, lhs, Operand(rhs), neg_cond);
1856
1857 // If tmp != dst, result will be moved.
1858 TurboAssembler::Move(dst, tmp);
1859 }
1860
emit_i64_eqz(Register dst,LiftoffRegister src)1861 void LiftoffAssembler::emit_i64_eqz(Register dst, LiftoffRegister src) {
1862 sltiu(dst, src.gp(), 1);
1863 }
1864
emit_i64_set_cond(LiftoffCondition liftoff_cond,Register dst,LiftoffRegister lhs,LiftoffRegister rhs)1865 void LiftoffAssembler::emit_i64_set_cond(LiftoffCondition liftoff_cond,
1866 Register dst, LiftoffRegister lhs,
1867 LiftoffRegister rhs) {
1868 Condition cond = liftoff::ToCondition(liftoff_cond);
1869 Register tmp = dst;
1870 if (dst == lhs.gp() || dst == rhs.gp()) {
1871 tmp = GetUnusedRegister(kGpReg, LiftoffRegList{lhs, rhs}).gp();
1872 }
1873 // Write 1 as result.
1874 TurboAssembler::li(tmp, 1);
1875
1876 // If negative condition is true, write 0 as result.
1877 Condition neg_cond = NegateCondition(cond);
1878 TurboAssembler::LoadZeroOnCondition(tmp, lhs.gp(), Operand(rhs.gp()),
1879 neg_cond);
1880
1881 // If tmp != dst, result will be moved.
1882 TurboAssembler::Move(dst, tmp);
1883 }
1884
1885 namespace liftoff {
1886
ConditionToConditionCmpFPU(LiftoffCondition condition,bool * predicate)1887 inline FPUCondition ConditionToConditionCmpFPU(LiftoffCondition condition,
1888 bool* predicate) {
1889 switch (condition) {
1890 case kEqual:
1891 *predicate = true;
1892 return EQ;
1893 case kUnequal:
1894 *predicate = false;
1895 return EQ;
1896 case kUnsignedLessThan:
1897 *predicate = true;
1898 return OLT;
1899 case kUnsignedGreaterEqual:
1900 *predicate = false;
1901 return OLT;
1902 case kUnsignedLessEqual:
1903 *predicate = true;
1904 return OLE;
1905 case kUnsignedGreaterThan:
1906 *predicate = false;
1907 return OLE;
1908 default:
1909 *predicate = true;
1910 break;
1911 }
1912 UNREACHABLE();
1913 }
1914
EmitAnyTrue(LiftoffAssembler * assm,LiftoffRegister dst,LiftoffRegister src)1915 inline void EmitAnyTrue(LiftoffAssembler* assm, LiftoffRegister dst,
1916 LiftoffRegister src) {
1917 Label all_false;
1918 assm->BranchMSA(&all_false, MSA_BRANCH_V, all_zero, src.fp().toW(),
1919 USE_DELAY_SLOT);
1920 assm->li(dst.gp(), 0l);
1921 assm->li(dst.gp(), 1);
1922 assm->bind(&all_false);
1923 }
1924
EmitAllTrue(LiftoffAssembler * assm,LiftoffRegister dst,LiftoffRegister src,MSABranchDF msa_branch_df)1925 inline void EmitAllTrue(LiftoffAssembler* assm, LiftoffRegister dst,
1926 LiftoffRegister src, MSABranchDF msa_branch_df) {
1927 Label all_true;
1928 assm->BranchMSA(&all_true, msa_branch_df, all_not_zero, src.fp().toW(),
1929 USE_DELAY_SLOT);
1930 assm->li(dst.gp(), 1);
1931 assm->li(dst.gp(), 0l);
1932 assm->bind(&all_true);
1933 }
1934
1935 } // namespace liftoff
1936
emit_f32_set_cond(LiftoffCondition liftoff_cond,Register dst,DoubleRegister lhs,DoubleRegister rhs)1937 void LiftoffAssembler::emit_f32_set_cond(LiftoffCondition liftoff_cond,
1938 Register dst, DoubleRegister lhs,
1939 DoubleRegister rhs) {
1940 Condition cond = liftoff::ToCondition(liftoff_cond);
1941 Label not_nan, cont;
1942 TurboAssembler::CompareIsNanF32(lhs, rhs);
1943 TurboAssembler::BranchFalseF(¬_nan);
1944 // If one of the operands is NaN, return 1 for f32.ne, else 0.
1945 if (cond == ne) {
1946 TurboAssembler::li(dst, 1);
1947 } else {
1948 TurboAssembler::Move(dst, zero_reg);
1949 }
1950 TurboAssembler::Branch(&cont);
1951
1952 bind(¬_nan);
1953
1954 TurboAssembler::li(dst, 1);
1955 bool predicate;
1956 FPUCondition fcond =
1957 liftoff::ConditionToConditionCmpFPU(liftoff_cond, &predicate);
1958 TurboAssembler::CompareF32(fcond, lhs, rhs);
1959 if (predicate) {
1960 TurboAssembler::LoadZeroIfNotFPUCondition(dst);
1961 } else {
1962 TurboAssembler::LoadZeroIfFPUCondition(dst);
1963 }
1964
1965 bind(&cont);
1966 }
1967
emit_f64_set_cond(LiftoffCondition liftoff_cond,Register dst,DoubleRegister lhs,DoubleRegister rhs)1968 void LiftoffAssembler::emit_f64_set_cond(LiftoffCondition liftoff_cond,
1969 Register dst, DoubleRegister lhs,
1970 DoubleRegister rhs) {
1971 Condition cond = liftoff::ToCondition(liftoff_cond);
1972 Label not_nan, cont;
1973 TurboAssembler::CompareIsNanF64(lhs, rhs);
1974 TurboAssembler::BranchFalseF(¬_nan);
1975 // If one of the operands is NaN, return 1 for f64.ne, else 0.
1976 if (cond == ne) {
1977 TurboAssembler::li(dst, 1);
1978 } else {
1979 TurboAssembler::Move(dst, zero_reg);
1980 }
1981 TurboAssembler::Branch(&cont);
1982
1983 bind(¬_nan);
1984
1985 TurboAssembler::li(dst, 1);
1986 bool predicate;
1987 FPUCondition fcond =
1988 liftoff::ConditionToConditionCmpFPU(liftoff_cond, &predicate);
1989 TurboAssembler::CompareF64(fcond, lhs, rhs);
1990 if (predicate) {
1991 TurboAssembler::LoadZeroIfNotFPUCondition(dst);
1992 } else {
1993 TurboAssembler::LoadZeroIfFPUCondition(dst);
1994 }
1995
1996 bind(&cont);
1997 }
1998
emit_select(LiftoffRegister dst,Register condition,LiftoffRegister true_value,LiftoffRegister false_value,ValueKind kind)1999 bool LiftoffAssembler::emit_select(LiftoffRegister dst, Register condition,
2000 LiftoffRegister true_value,
2001 LiftoffRegister false_value,
2002 ValueKind kind) {
2003 return false;
2004 }
2005
emit_smi_check(Register obj,Label * target,SmiCheckMode mode)2006 void LiftoffAssembler::emit_smi_check(Register obj, Label* target,
2007 SmiCheckMode mode) {
2008 UseScratchRegisterScope temps(this);
2009 Register scratch = temps.Acquire();
2010 And(scratch, obj, Operand(kSmiTagMask));
2011 Condition condition = mode == kJumpOnSmi ? eq : ne;
2012 Branch(target, condition, scratch, Operand(zero_reg));
2013 }
2014
LoadTransform(LiftoffRegister dst,Register src_addr,Register offset_reg,uintptr_t offset_imm,LoadType type,LoadTransformationKind transform,uint32_t * protected_load_pc)2015 void LiftoffAssembler::LoadTransform(LiftoffRegister dst, Register src_addr,
2016 Register offset_reg, uintptr_t offset_imm,
2017 LoadType type,
2018 LoadTransformationKind transform,
2019 uint32_t* protected_load_pc) {
2020 UseScratchRegisterScope temps(this);
2021 Register scratch = temps.Acquire();
2022 MemOperand src_op =
2023 liftoff::GetMemOp(this, src_addr, offset_reg, offset_imm);
2024 MSARegister dst_msa = dst.fp().toW();
2025 *protected_load_pc = pc_offset();
2026 MachineType memtype = type.mem_type();
2027
2028 if (transform == LoadTransformationKind::kExtend) {
2029 Ld(scratch, src_op);
2030 if (memtype == MachineType::Int8()) {
2031 fill_d(dst_msa, scratch);
2032 clti_s_b(kSimd128ScratchReg, dst_msa, 0);
2033 ilvr_b(dst_msa, kSimd128ScratchReg, dst_msa);
2034 } else if (memtype == MachineType::Uint8()) {
2035 xor_v(kSimd128RegZero, kSimd128RegZero, kSimd128RegZero);
2036 fill_d(dst_msa, scratch);
2037 ilvr_b(dst_msa, kSimd128RegZero, dst_msa);
2038 } else if (memtype == MachineType::Int16()) {
2039 fill_d(dst_msa, scratch);
2040 clti_s_h(kSimd128ScratchReg, dst_msa, 0);
2041 ilvr_h(dst_msa, kSimd128ScratchReg, dst_msa);
2042 } else if (memtype == MachineType::Uint16()) {
2043 xor_v(kSimd128RegZero, kSimd128RegZero, kSimd128RegZero);
2044 fill_d(dst_msa, scratch);
2045 ilvr_h(dst_msa, kSimd128RegZero, dst_msa);
2046 } else if (memtype == MachineType::Int32()) {
2047 fill_d(dst_msa, scratch);
2048 clti_s_w(kSimd128ScratchReg, dst_msa, 0);
2049 ilvr_w(dst_msa, kSimd128ScratchReg, dst_msa);
2050 } else if (memtype == MachineType::Uint32()) {
2051 xor_v(kSimd128RegZero, kSimd128RegZero, kSimd128RegZero);
2052 fill_d(dst_msa, scratch);
2053 ilvr_w(dst_msa, kSimd128RegZero, dst_msa);
2054 }
2055 } else if (transform == LoadTransformationKind::kZeroExtend) {
2056 xor_v(dst_msa, dst_msa, dst_msa);
2057 if (memtype == MachineType::Int32()) {
2058 Lwu(scratch, src_op);
2059 insert_w(dst_msa, 0, scratch);
2060 } else {
2061 DCHECK_EQ(MachineType::Int64(), memtype);
2062 Ld(scratch, src_op);
2063 insert_d(dst_msa, 0, scratch);
2064 }
2065 } else {
2066 DCHECK_EQ(LoadTransformationKind::kSplat, transform);
2067 if (memtype == MachineType::Int8()) {
2068 Lb(scratch, src_op);
2069 fill_b(dst_msa, scratch);
2070 } else if (memtype == MachineType::Int16()) {
2071 Lh(scratch, src_op);
2072 fill_h(dst_msa, scratch);
2073 } else if (memtype == MachineType::Int32()) {
2074 Lw(scratch, src_op);
2075 fill_w(dst_msa, scratch);
2076 } else if (memtype == MachineType::Int64()) {
2077 Ld(scratch, src_op);
2078 fill_d(dst_msa, scratch);
2079 }
2080 }
2081 }
2082
LoadLane(LiftoffRegister dst,LiftoffRegister src,Register addr,Register offset_reg,uintptr_t offset_imm,LoadType type,uint8_t laneidx,uint32_t * protected_load_pc)2083 void LiftoffAssembler::LoadLane(LiftoffRegister dst, LiftoffRegister src,
2084 Register addr, Register offset_reg,
2085 uintptr_t offset_imm, LoadType type,
2086 uint8_t laneidx, uint32_t* protected_load_pc) {
2087 MemOperand src_op = liftoff::GetMemOp(this, addr, offset_reg, offset_imm);
2088 *protected_load_pc = pc_offset();
2089 LoadStoreLaneParams load_params(type.mem_type().representation(), laneidx);
2090 TurboAssembler::LoadLane(load_params.sz, dst.fp().toW(), laneidx, src_op);
2091 }
2092
StoreLane(Register dst,Register offset,uintptr_t offset_imm,LiftoffRegister src,StoreType type,uint8_t lane,uint32_t * protected_store_pc)2093 void LiftoffAssembler::StoreLane(Register dst, Register offset,
2094 uintptr_t offset_imm, LiftoffRegister src,
2095 StoreType type, uint8_t lane,
2096 uint32_t* protected_store_pc) {
2097 MemOperand dst_op = liftoff::GetMemOp(this, dst, offset, offset_imm);
2098 if (protected_store_pc) *protected_store_pc = pc_offset();
2099 LoadStoreLaneParams store_params(type.mem_rep(), lane);
2100 TurboAssembler::StoreLane(store_params.sz, src.fp().toW(), lane, dst_op);
2101 }
2102
emit_i8x16_shuffle(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs,const uint8_t shuffle[16],bool is_swizzle)2103 void LiftoffAssembler::emit_i8x16_shuffle(LiftoffRegister dst,
2104 LiftoffRegister lhs,
2105 LiftoffRegister rhs,
2106 const uint8_t shuffle[16],
2107 bool is_swizzle) {
2108 MSARegister dst_msa = dst.fp().toW();
2109 MSARegister lhs_msa = lhs.fp().toW();
2110 MSARegister rhs_msa = rhs.fp().toW();
2111
2112 uint64_t control_hi = 0;
2113 uint64_t control_low = 0;
2114 for (int i = 7; i >= 0; i--) {
2115 control_hi <<= 8;
2116 control_hi |= shuffle[i + 8];
2117 control_low <<= 8;
2118 control_low |= shuffle[i];
2119 }
2120
2121 if (dst_msa == lhs_msa) {
2122 move_v(kSimd128ScratchReg, lhs_msa);
2123 lhs_msa = kSimd128ScratchReg;
2124 } else if (dst_msa == rhs_msa) {
2125 move_v(kSimd128ScratchReg, rhs_msa);
2126 rhs_msa = kSimd128ScratchReg;
2127 }
2128
2129 li(kScratchReg, control_low);
2130 insert_d(dst_msa, 0, kScratchReg);
2131 li(kScratchReg, control_hi);
2132 insert_d(dst_msa, 1, kScratchReg);
2133 vshf_b(dst_msa, rhs_msa, lhs_msa);
2134 }
2135
emit_i8x16_swizzle(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2136 void LiftoffAssembler::emit_i8x16_swizzle(LiftoffRegister dst,
2137 LiftoffRegister lhs,
2138 LiftoffRegister rhs) {
2139 MSARegister dst_msa = dst.fp().toW();
2140 MSARegister lhs_msa = lhs.fp().toW();
2141 MSARegister rhs_msa = rhs.fp().toW();
2142
2143 if (dst == lhs) {
2144 move_v(kSimd128ScratchReg, lhs_msa);
2145 lhs_msa = kSimd128ScratchReg;
2146 }
2147 xor_v(kSimd128RegZero, kSimd128RegZero, kSimd128RegZero);
2148 move_v(dst_msa, rhs_msa);
2149 vshf_b(dst_msa, kSimd128RegZero, lhs_msa);
2150 }
2151
emit_i8x16_splat(LiftoffRegister dst,LiftoffRegister src)2152 void LiftoffAssembler::emit_i8x16_splat(LiftoffRegister dst,
2153 LiftoffRegister src) {
2154 fill_b(dst.fp().toW(), src.gp());
2155 }
2156
emit_i16x8_splat(LiftoffRegister dst,LiftoffRegister src)2157 void LiftoffAssembler::emit_i16x8_splat(LiftoffRegister dst,
2158 LiftoffRegister src) {
2159 fill_h(dst.fp().toW(), src.gp());
2160 }
2161
emit_i32x4_splat(LiftoffRegister dst,LiftoffRegister src)2162 void LiftoffAssembler::emit_i32x4_splat(LiftoffRegister dst,
2163 LiftoffRegister src) {
2164 fill_w(dst.fp().toW(), src.gp());
2165 }
2166
emit_i64x2_splat(LiftoffRegister dst,LiftoffRegister src)2167 void LiftoffAssembler::emit_i64x2_splat(LiftoffRegister dst,
2168 LiftoffRegister src) {
2169 fill_d(dst.fp().toW(), src.gp());
2170 }
2171
emit_f32x4_splat(LiftoffRegister dst,LiftoffRegister src)2172 void LiftoffAssembler::emit_f32x4_splat(LiftoffRegister dst,
2173 LiftoffRegister src) {
2174 TurboAssembler::FmoveLow(kScratchReg, src.fp());
2175 fill_w(dst.fp().toW(), kScratchReg);
2176 }
2177
emit_f64x2_splat(LiftoffRegister dst,LiftoffRegister src)2178 void LiftoffAssembler::emit_f64x2_splat(LiftoffRegister dst,
2179 LiftoffRegister src) {
2180 TurboAssembler::Move(kScratchReg, src.fp());
2181 fill_d(dst.fp().toW(), kScratchReg);
2182 }
2183
2184 #define SIMD_BINOP(name1, name2, type) \
2185 void LiftoffAssembler::emit_##name1##_extmul_low_##name2( \
2186 LiftoffRegister dst, LiftoffRegister src1, LiftoffRegister src2) { \
2187 TurboAssembler::ExtMulLow(type, dst.fp().toW(), src1.fp().toW(), \
2188 src2.fp().toW()); \
2189 } \
2190 void LiftoffAssembler::emit_##name1##_extmul_high_##name2( \
2191 LiftoffRegister dst, LiftoffRegister src1, LiftoffRegister src2) { \
2192 TurboAssembler::ExtMulHigh(type, dst.fp().toW(), src1.fp().toW(), \
2193 src2.fp().toW()); \
2194 }
2195
SIMD_BINOP(i16x8,i8x16_s,MSAS8)2196 SIMD_BINOP(i16x8, i8x16_s, MSAS8)
2197 SIMD_BINOP(i16x8, i8x16_u, MSAU8)
2198
2199 SIMD_BINOP(i32x4, i16x8_s, MSAS16)
2200 SIMD_BINOP(i32x4, i16x8_u, MSAU16)
2201
2202 SIMD_BINOP(i64x2, i32x4_s, MSAS32)
2203 SIMD_BINOP(i64x2, i32x4_u, MSAU32)
2204
2205 #undef SIMD_BINOP
2206
2207 #define SIMD_BINOP(name1, name2, type) \
2208 void LiftoffAssembler::emit_##name1##_extadd_pairwise_##name2( \
2209 LiftoffRegister dst, LiftoffRegister src) { \
2210 TurboAssembler::ExtAddPairwise(type, dst.fp().toW(), src.fp().toW()); \
2211 }
2212
2213 SIMD_BINOP(i16x8, i8x16_s, MSAS8)
2214 SIMD_BINOP(i16x8, i8x16_u, MSAU8)
2215 SIMD_BINOP(i32x4, i16x8_s, MSAS16)
2216 SIMD_BINOP(i32x4, i16x8_u, MSAU16)
2217 #undef SIMD_BINOP
2218
2219 void LiftoffAssembler::emit_i16x8_q15mulr_sat_s(LiftoffRegister dst,
2220 LiftoffRegister src1,
2221 LiftoffRegister src2) {
2222 mulr_q_h(dst.fp().toW(), src1.fp().toW(), src2.fp().toW());
2223 }
2224
emit_i8x16_eq(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2225 void LiftoffAssembler::emit_i8x16_eq(LiftoffRegister dst, LiftoffRegister lhs,
2226 LiftoffRegister rhs) {
2227 ceq_b(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2228 }
2229
emit_i8x16_ne(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2230 void LiftoffAssembler::emit_i8x16_ne(LiftoffRegister dst, LiftoffRegister lhs,
2231 LiftoffRegister rhs) {
2232 ceq_b(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2233 nor_v(dst.fp().toW(), dst.fp().toW(), dst.fp().toW());
2234 }
2235
emit_i8x16_gt_s(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2236 void LiftoffAssembler::emit_i8x16_gt_s(LiftoffRegister dst, LiftoffRegister lhs,
2237 LiftoffRegister rhs) {
2238 clt_s_b(dst.fp().toW(), rhs.fp().toW(), lhs.fp().toW());
2239 }
2240
emit_i8x16_gt_u(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2241 void LiftoffAssembler::emit_i8x16_gt_u(LiftoffRegister dst, LiftoffRegister lhs,
2242 LiftoffRegister rhs) {
2243 clt_u_b(dst.fp().toW(), rhs.fp().toW(), lhs.fp().toW());
2244 }
2245
emit_i8x16_ge_s(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2246 void LiftoffAssembler::emit_i8x16_ge_s(LiftoffRegister dst, LiftoffRegister lhs,
2247 LiftoffRegister rhs) {
2248 cle_s_b(dst.fp().toW(), rhs.fp().toW(), lhs.fp().toW());
2249 }
2250
emit_i8x16_ge_u(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2251 void LiftoffAssembler::emit_i8x16_ge_u(LiftoffRegister dst, LiftoffRegister lhs,
2252 LiftoffRegister rhs) {
2253 cle_u_b(dst.fp().toW(), rhs.fp().toW(), lhs.fp().toW());
2254 }
2255
emit_i16x8_eq(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2256 void LiftoffAssembler::emit_i16x8_eq(LiftoffRegister dst, LiftoffRegister lhs,
2257 LiftoffRegister rhs) {
2258 ceq_h(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2259 }
2260
emit_i16x8_ne(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2261 void LiftoffAssembler::emit_i16x8_ne(LiftoffRegister dst, LiftoffRegister lhs,
2262 LiftoffRegister rhs) {
2263 ceq_h(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2264 nor_v(dst.fp().toW(), dst.fp().toW(), dst.fp().toW());
2265 }
2266
emit_i16x8_gt_s(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2267 void LiftoffAssembler::emit_i16x8_gt_s(LiftoffRegister dst, LiftoffRegister lhs,
2268 LiftoffRegister rhs) {
2269 clt_s_h(dst.fp().toW(), rhs.fp().toW(), lhs.fp().toW());
2270 }
2271
emit_i16x8_gt_u(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2272 void LiftoffAssembler::emit_i16x8_gt_u(LiftoffRegister dst, LiftoffRegister lhs,
2273 LiftoffRegister rhs) {
2274 clt_u_h(dst.fp().toW(), rhs.fp().toW(), lhs.fp().toW());
2275 }
2276
emit_i16x8_ge_s(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2277 void LiftoffAssembler::emit_i16x8_ge_s(LiftoffRegister dst, LiftoffRegister lhs,
2278 LiftoffRegister rhs) {
2279 cle_s_h(dst.fp().toW(), rhs.fp().toW(), lhs.fp().toW());
2280 }
2281
emit_i16x8_ge_u(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2282 void LiftoffAssembler::emit_i16x8_ge_u(LiftoffRegister dst, LiftoffRegister lhs,
2283 LiftoffRegister rhs) {
2284 cle_u_h(dst.fp().toW(), rhs.fp().toW(), lhs.fp().toW());
2285 }
2286
emit_i32x4_eq(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2287 void LiftoffAssembler::emit_i32x4_eq(LiftoffRegister dst, LiftoffRegister lhs,
2288 LiftoffRegister rhs) {
2289 ceq_w(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2290 }
2291
emit_i32x4_ne(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2292 void LiftoffAssembler::emit_i32x4_ne(LiftoffRegister dst, LiftoffRegister lhs,
2293 LiftoffRegister rhs) {
2294 ceq_w(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2295 nor_v(dst.fp().toW(), dst.fp().toW(), dst.fp().toW());
2296 }
2297
emit_i32x4_gt_s(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2298 void LiftoffAssembler::emit_i32x4_gt_s(LiftoffRegister dst, LiftoffRegister lhs,
2299 LiftoffRegister rhs) {
2300 clt_s_w(dst.fp().toW(), rhs.fp().toW(), lhs.fp().toW());
2301 }
2302
emit_i32x4_gt_u(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2303 void LiftoffAssembler::emit_i32x4_gt_u(LiftoffRegister dst, LiftoffRegister lhs,
2304 LiftoffRegister rhs) {
2305 clt_u_w(dst.fp().toW(), rhs.fp().toW(), lhs.fp().toW());
2306 }
2307
emit_i32x4_ge_s(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2308 void LiftoffAssembler::emit_i32x4_ge_s(LiftoffRegister dst, LiftoffRegister lhs,
2309 LiftoffRegister rhs) {
2310 cle_s_w(dst.fp().toW(), rhs.fp().toW(), lhs.fp().toW());
2311 }
2312
emit_i32x4_ge_u(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2313 void LiftoffAssembler::emit_i32x4_ge_u(LiftoffRegister dst, LiftoffRegister lhs,
2314 LiftoffRegister rhs) {
2315 cle_u_w(dst.fp().toW(), rhs.fp().toW(), lhs.fp().toW());
2316 }
2317
emit_f32x4_eq(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2318 void LiftoffAssembler::emit_f32x4_eq(LiftoffRegister dst, LiftoffRegister lhs,
2319 LiftoffRegister rhs) {
2320 fceq_w(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2321 }
2322
emit_f32x4_ne(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2323 void LiftoffAssembler::emit_f32x4_ne(LiftoffRegister dst, LiftoffRegister lhs,
2324 LiftoffRegister rhs) {
2325 fcune_w(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2326 }
2327
emit_f32x4_lt(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2328 void LiftoffAssembler::emit_f32x4_lt(LiftoffRegister dst, LiftoffRegister lhs,
2329 LiftoffRegister rhs) {
2330 fclt_w(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2331 }
2332
emit_f32x4_le(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2333 void LiftoffAssembler::emit_f32x4_le(LiftoffRegister dst, LiftoffRegister lhs,
2334 LiftoffRegister rhs) {
2335 fcle_w(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2336 }
2337
emit_i64x2_eq(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2338 void LiftoffAssembler::emit_i64x2_eq(LiftoffRegister dst, LiftoffRegister lhs,
2339 LiftoffRegister rhs) {
2340 ceq_d(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2341 }
2342
emit_i64x2_ne(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2343 void LiftoffAssembler::emit_i64x2_ne(LiftoffRegister dst, LiftoffRegister lhs,
2344 LiftoffRegister rhs) {
2345 ceq_d(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2346 nor_v(dst.fp().toW(), dst.fp().toW(), dst.fp().toW());
2347 }
2348
emit_i64x2_abs(LiftoffRegister dst,LiftoffRegister src)2349 void LiftoffAssembler::emit_i64x2_abs(LiftoffRegister dst,
2350 LiftoffRegister src) {
2351 xor_v(kSimd128RegZero, kSimd128RegZero, kSimd128RegZero);
2352 add_a_d(dst.fp().toW(), src.fp().toW(), kSimd128RegZero);
2353 }
2354
emit_f64x2_eq(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2355 void LiftoffAssembler::emit_f64x2_eq(LiftoffRegister dst, LiftoffRegister lhs,
2356 LiftoffRegister rhs) {
2357 fceq_d(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2358 }
2359
emit_f64x2_ne(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2360 void LiftoffAssembler::emit_f64x2_ne(LiftoffRegister dst, LiftoffRegister lhs,
2361 LiftoffRegister rhs) {
2362 fcune_d(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2363 }
2364
emit_f64x2_lt(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2365 void LiftoffAssembler::emit_f64x2_lt(LiftoffRegister dst, LiftoffRegister lhs,
2366 LiftoffRegister rhs) {
2367 fclt_d(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2368 }
2369
emit_f64x2_le(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2370 void LiftoffAssembler::emit_f64x2_le(LiftoffRegister dst, LiftoffRegister lhs,
2371 LiftoffRegister rhs) {
2372 fcle_d(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2373 }
2374
emit_s128_const(LiftoffRegister dst,const uint8_t imms[16])2375 void LiftoffAssembler::emit_s128_const(LiftoffRegister dst,
2376 const uint8_t imms[16]) {
2377 MSARegister dst_msa = dst.fp().toW();
2378 uint64_t vals[2];
2379 memcpy(vals, imms, sizeof(vals));
2380 li(kScratchReg, vals[0]);
2381 insert_d(dst_msa, 0, kScratchReg);
2382 li(kScratchReg, vals[1]);
2383 insert_d(dst_msa, 1, kScratchReg);
2384 }
2385
emit_s128_not(LiftoffRegister dst,LiftoffRegister src)2386 void LiftoffAssembler::emit_s128_not(LiftoffRegister dst, LiftoffRegister src) {
2387 nor_v(dst.fp().toW(), src.fp().toW(), src.fp().toW());
2388 }
2389
emit_s128_and(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2390 void LiftoffAssembler::emit_s128_and(LiftoffRegister dst, LiftoffRegister lhs,
2391 LiftoffRegister rhs) {
2392 and_v(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2393 }
2394
emit_s128_or(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2395 void LiftoffAssembler::emit_s128_or(LiftoffRegister dst, LiftoffRegister lhs,
2396 LiftoffRegister rhs) {
2397 or_v(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2398 }
2399
emit_s128_xor(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2400 void LiftoffAssembler::emit_s128_xor(LiftoffRegister dst, LiftoffRegister lhs,
2401 LiftoffRegister rhs) {
2402 xor_v(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2403 }
2404
emit_s128_and_not(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2405 void LiftoffAssembler::emit_s128_and_not(LiftoffRegister dst,
2406 LiftoffRegister lhs,
2407 LiftoffRegister rhs) {
2408 nor_v(kSimd128ScratchReg, rhs.fp().toW(), rhs.fp().toW());
2409 and_v(dst.fp().toW(), kSimd128ScratchReg, lhs.fp().toW());
2410 }
2411
emit_s128_select(LiftoffRegister dst,LiftoffRegister src1,LiftoffRegister src2,LiftoffRegister mask)2412 void LiftoffAssembler::emit_s128_select(LiftoffRegister dst,
2413 LiftoffRegister src1,
2414 LiftoffRegister src2,
2415 LiftoffRegister mask) {
2416 if (dst == mask) {
2417 bsel_v(dst.fp().toW(), src2.fp().toW(), src1.fp().toW());
2418 } else {
2419 xor_v(kSimd128ScratchReg, src1.fp().toW(), src2.fp().toW());
2420 and_v(kSimd128ScratchReg, kSimd128ScratchReg, mask.fp().toW());
2421 xor_v(dst.fp().toW(), kSimd128ScratchReg, src2.fp().toW());
2422 }
2423 }
2424
emit_i8x16_neg(LiftoffRegister dst,LiftoffRegister src)2425 void LiftoffAssembler::emit_i8x16_neg(LiftoffRegister dst,
2426 LiftoffRegister src) {
2427 xor_v(kSimd128RegZero, kSimd128RegZero, kSimd128RegZero);
2428 subv_b(dst.fp().toW(), kSimd128RegZero, src.fp().toW());
2429 }
2430
emit_v128_anytrue(LiftoffRegister dst,LiftoffRegister src)2431 void LiftoffAssembler::emit_v128_anytrue(LiftoffRegister dst,
2432 LiftoffRegister src) {
2433 liftoff::EmitAnyTrue(this, dst, src);
2434 }
2435
emit_i8x16_alltrue(LiftoffRegister dst,LiftoffRegister src)2436 void LiftoffAssembler::emit_i8x16_alltrue(LiftoffRegister dst,
2437 LiftoffRegister src) {
2438 liftoff::EmitAllTrue(this, dst, src, MSA_BRANCH_B);
2439 }
2440
emit_i8x16_bitmask(LiftoffRegister dst,LiftoffRegister src)2441 void LiftoffAssembler::emit_i8x16_bitmask(LiftoffRegister dst,
2442 LiftoffRegister src) {
2443 MSARegister scratch0 = kSimd128RegZero;
2444 MSARegister scratch1 = kSimd128ScratchReg;
2445 srli_b(scratch0, src.fp().toW(), 7);
2446 srli_h(scratch1, scratch0, 7);
2447 or_v(scratch0, scratch0, scratch1);
2448 srli_w(scratch1, scratch0, 14);
2449 or_v(scratch0, scratch0, scratch1);
2450 srli_d(scratch1, scratch0, 28);
2451 or_v(scratch0, scratch0, scratch1);
2452 shf_w(scratch1, scratch0, 0x0E);
2453 ilvev_b(scratch0, scratch1, scratch0);
2454 copy_u_h(dst.gp(), scratch0, 0);
2455 }
2456
emit_i8x16_shl(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2457 void LiftoffAssembler::emit_i8x16_shl(LiftoffRegister dst, LiftoffRegister lhs,
2458 LiftoffRegister rhs) {
2459 fill_b(kSimd128ScratchReg, rhs.gp());
2460 sll_b(dst.fp().toW(), lhs.fp().toW(), kSimd128ScratchReg);
2461 }
2462
emit_i8x16_shli(LiftoffRegister dst,LiftoffRegister lhs,int32_t rhs)2463 void LiftoffAssembler::emit_i8x16_shli(LiftoffRegister dst, LiftoffRegister lhs,
2464 int32_t rhs) {
2465 slli_b(dst.fp().toW(), lhs.fp().toW(), rhs & 7);
2466 }
2467
emit_i8x16_shr_s(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2468 void LiftoffAssembler::emit_i8x16_shr_s(LiftoffRegister dst,
2469 LiftoffRegister lhs,
2470 LiftoffRegister rhs) {
2471 fill_b(kSimd128ScratchReg, rhs.gp());
2472 sra_b(dst.fp().toW(), lhs.fp().toW(), kSimd128ScratchReg);
2473 }
2474
emit_i8x16_shri_s(LiftoffRegister dst,LiftoffRegister lhs,int32_t rhs)2475 void LiftoffAssembler::emit_i8x16_shri_s(LiftoffRegister dst,
2476 LiftoffRegister lhs, int32_t rhs) {
2477 srai_b(dst.fp().toW(), lhs.fp().toW(), rhs & 7);
2478 }
2479
emit_i8x16_shr_u(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2480 void LiftoffAssembler::emit_i8x16_shr_u(LiftoffRegister dst,
2481 LiftoffRegister lhs,
2482 LiftoffRegister rhs) {
2483 fill_b(kSimd128ScratchReg, rhs.gp());
2484 srl_b(dst.fp().toW(), lhs.fp().toW(), kSimd128ScratchReg);
2485 }
2486
emit_i8x16_shri_u(LiftoffRegister dst,LiftoffRegister lhs,int32_t rhs)2487 void LiftoffAssembler::emit_i8x16_shri_u(LiftoffRegister dst,
2488 LiftoffRegister lhs, int32_t rhs) {
2489 srli_b(dst.fp().toW(), lhs.fp().toW(), rhs & 7);
2490 }
2491
emit_i8x16_add(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2492 void LiftoffAssembler::emit_i8x16_add(LiftoffRegister dst, LiftoffRegister lhs,
2493 LiftoffRegister rhs) {
2494 addv_b(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2495 }
2496
emit_i8x16_add_sat_s(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2497 void LiftoffAssembler::emit_i8x16_add_sat_s(LiftoffRegister dst,
2498 LiftoffRegister lhs,
2499 LiftoffRegister rhs) {
2500 adds_s_b(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2501 }
2502
emit_i8x16_add_sat_u(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2503 void LiftoffAssembler::emit_i8x16_add_sat_u(LiftoffRegister dst,
2504 LiftoffRegister lhs,
2505 LiftoffRegister rhs) {
2506 adds_u_b(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2507 }
2508
emit_i8x16_sub(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2509 void LiftoffAssembler::emit_i8x16_sub(LiftoffRegister dst, LiftoffRegister lhs,
2510 LiftoffRegister rhs) {
2511 subv_b(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2512 }
2513
emit_i8x16_sub_sat_s(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2514 void LiftoffAssembler::emit_i8x16_sub_sat_s(LiftoffRegister dst,
2515 LiftoffRegister lhs,
2516 LiftoffRegister rhs) {
2517 subs_s_b(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2518 }
2519
emit_i8x16_sub_sat_u(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2520 void LiftoffAssembler::emit_i8x16_sub_sat_u(LiftoffRegister dst,
2521 LiftoffRegister lhs,
2522 LiftoffRegister rhs) {
2523 subs_u_b(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2524 }
2525
emit_i8x16_min_s(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2526 void LiftoffAssembler::emit_i8x16_min_s(LiftoffRegister dst,
2527 LiftoffRegister lhs,
2528 LiftoffRegister rhs) {
2529 min_s_b(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2530 }
2531
emit_i8x16_min_u(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2532 void LiftoffAssembler::emit_i8x16_min_u(LiftoffRegister dst,
2533 LiftoffRegister lhs,
2534 LiftoffRegister rhs) {
2535 min_u_b(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2536 }
2537
emit_i8x16_max_s(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2538 void LiftoffAssembler::emit_i8x16_max_s(LiftoffRegister dst,
2539 LiftoffRegister lhs,
2540 LiftoffRegister rhs) {
2541 max_s_b(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2542 }
2543
emit_i8x16_max_u(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2544 void LiftoffAssembler::emit_i8x16_max_u(LiftoffRegister dst,
2545 LiftoffRegister lhs,
2546 LiftoffRegister rhs) {
2547 max_u_b(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2548 }
2549
emit_i8x16_popcnt(LiftoffRegister dst,LiftoffRegister src)2550 void LiftoffAssembler::emit_i8x16_popcnt(LiftoffRegister dst,
2551 LiftoffRegister src) {
2552 pcnt_b(dst.fp().toW(), src.fp().toW());
2553 }
2554
emit_i16x8_neg(LiftoffRegister dst,LiftoffRegister src)2555 void LiftoffAssembler::emit_i16x8_neg(LiftoffRegister dst,
2556 LiftoffRegister src) {
2557 xor_v(kSimd128RegZero, kSimd128RegZero, kSimd128RegZero);
2558 subv_h(dst.fp().toW(), kSimd128RegZero, src.fp().toW());
2559 }
2560
emit_i16x8_alltrue(LiftoffRegister dst,LiftoffRegister src)2561 void LiftoffAssembler::emit_i16x8_alltrue(LiftoffRegister dst,
2562 LiftoffRegister src) {
2563 liftoff::EmitAllTrue(this, dst, src, MSA_BRANCH_H);
2564 }
2565
emit_i16x8_bitmask(LiftoffRegister dst,LiftoffRegister src)2566 void LiftoffAssembler::emit_i16x8_bitmask(LiftoffRegister dst,
2567 LiftoffRegister src) {
2568 MSARegister scratch0 = kSimd128RegZero;
2569 MSARegister scratch1 = kSimd128ScratchReg;
2570 srli_h(scratch0, src.fp().toW(), 15);
2571 srli_w(scratch1, scratch0, 15);
2572 or_v(scratch0, scratch0, scratch1);
2573 srli_d(scratch1, scratch0, 30);
2574 or_v(scratch0, scratch0, scratch1);
2575 shf_w(scratch1, scratch0, 0x0E);
2576 slli_d(scratch1, scratch1, 4);
2577 or_v(scratch0, scratch0, scratch1);
2578 copy_u_b(dst.gp(), scratch0, 0);
2579 }
2580
emit_i16x8_shl(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2581 void LiftoffAssembler::emit_i16x8_shl(LiftoffRegister dst, LiftoffRegister lhs,
2582 LiftoffRegister rhs) {
2583 fill_h(kSimd128ScratchReg, rhs.gp());
2584 sll_h(dst.fp().toW(), lhs.fp().toW(), kSimd128ScratchReg);
2585 }
2586
emit_i16x8_shli(LiftoffRegister dst,LiftoffRegister lhs,int32_t rhs)2587 void LiftoffAssembler::emit_i16x8_shli(LiftoffRegister dst, LiftoffRegister lhs,
2588 int32_t rhs) {
2589 slli_h(dst.fp().toW(), lhs.fp().toW(), rhs & 15);
2590 }
2591
emit_i16x8_shr_s(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2592 void LiftoffAssembler::emit_i16x8_shr_s(LiftoffRegister dst,
2593 LiftoffRegister lhs,
2594 LiftoffRegister rhs) {
2595 fill_h(kSimd128ScratchReg, rhs.gp());
2596 sra_h(dst.fp().toW(), lhs.fp().toW(), kSimd128ScratchReg);
2597 }
2598
emit_i16x8_shri_s(LiftoffRegister dst,LiftoffRegister lhs,int32_t rhs)2599 void LiftoffAssembler::emit_i16x8_shri_s(LiftoffRegister dst,
2600 LiftoffRegister lhs, int32_t rhs) {
2601 srai_h(dst.fp().toW(), lhs.fp().toW(), rhs & 15);
2602 }
2603
emit_i16x8_shr_u(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2604 void LiftoffAssembler::emit_i16x8_shr_u(LiftoffRegister dst,
2605 LiftoffRegister lhs,
2606 LiftoffRegister rhs) {
2607 fill_h(kSimd128ScratchReg, rhs.gp());
2608 srl_h(dst.fp().toW(), lhs.fp().toW(), kSimd128ScratchReg);
2609 }
2610
emit_i16x8_shri_u(LiftoffRegister dst,LiftoffRegister lhs,int32_t rhs)2611 void LiftoffAssembler::emit_i16x8_shri_u(LiftoffRegister dst,
2612 LiftoffRegister lhs, int32_t rhs) {
2613 srli_h(dst.fp().toW(), lhs.fp().toW(), rhs & 15);
2614 }
2615
emit_i16x8_add(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2616 void LiftoffAssembler::emit_i16x8_add(LiftoffRegister dst, LiftoffRegister lhs,
2617 LiftoffRegister rhs) {
2618 addv_h(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2619 }
2620
emit_i16x8_add_sat_s(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2621 void LiftoffAssembler::emit_i16x8_add_sat_s(LiftoffRegister dst,
2622 LiftoffRegister lhs,
2623 LiftoffRegister rhs) {
2624 adds_s_h(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2625 }
2626
emit_i16x8_add_sat_u(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2627 void LiftoffAssembler::emit_i16x8_add_sat_u(LiftoffRegister dst,
2628 LiftoffRegister lhs,
2629 LiftoffRegister rhs) {
2630 adds_u_h(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2631 }
2632
emit_i16x8_sub(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2633 void LiftoffAssembler::emit_i16x8_sub(LiftoffRegister dst, LiftoffRegister lhs,
2634 LiftoffRegister rhs) {
2635 subv_h(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2636 }
2637
emit_i16x8_sub_sat_s(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2638 void LiftoffAssembler::emit_i16x8_sub_sat_s(LiftoffRegister dst,
2639 LiftoffRegister lhs,
2640 LiftoffRegister rhs) {
2641 subs_s_h(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2642 }
2643
emit_i16x8_sub_sat_u(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2644 void LiftoffAssembler::emit_i16x8_sub_sat_u(LiftoffRegister dst,
2645 LiftoffRegister lhs,
2646 LiftoffRegister rhs) {
2647 subs_u_h(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2648 }
2649
emit_i16x8_mul(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2650 void LiftoffAssembler::emit_i16x8_mul(LiftoffRegister dst, LiftoffRegister lhs,
2651 LiftoffRegister rhs) {
2652 mulv_h(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2653 }
2654
emit_i16x8_min_s(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2655 void LiftoffAssembler::emit_i16x8_min_s(LiftoffRegister dst,
2656 LiftoffRegister lhs,
2657 LiftoffRegister rhs) {
2658 min_s_h(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2659 }
2660
emit_i16x8_min_u(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2661 void LiftoffAssembler::emit_i16x8_min_u(LiftoffRegister dst,
2662 LiftoffRegister lhs,
2663 LiftoffRegister rhs) {
2664 min_u_h(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2665 }
2666
emit_i16x8_max_s(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2667 void LiftoffAssembler::emit_i16x8_max_s(LiftoffRegister dst,
2668 LiftoffRegister lhs,
2669 LiftoffRegister rhs) {
2670 max_s_h(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2671 }
2672
emit_i16x8_max_u(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2673 void LiftoffAssembler::emit_i16x8_max_u(LiftoffRegister dst,
2674 LiftoffRegister lhs,
2675 LiftoffRegister rhs) {
2676 max_u_h(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2677 }
2678
emit_i32x4_neg(LiftoffRegister dst,LiftoffRegister src)2679 void LiftoffAssembler::emit_i32x4_neg(LiftoffRegister dst,
2680 LiftoffRegister src) {
2681 xor_v(kSimd128RegZero, kSimd128RegZero, kSimd128RegZero);
2682 subv_w(dst.fp().toW(), kSimd128RegZero, src.fp().toW());
2683 }
2684
emit_i32x4_alltrue(LiftoffRegister dst,LiftoffRegister src)2685 void LiftoffAssembler::emit_i32x4_alltrue(LiftoffRegister dst,
2686 LiftoffRegister src) {
2687 liftoff::EmitAllTrue(this, dst, src, MSA_BRANCH_W);
2688 }
2689
emit_i32x4_bitmask(LiftoffRegister dst,LiftoffRegister src)2690 void LiftoffAssembler::emit_i32x4_bitmask(LiftoffRegister dst,
2691 LiftoffRegister src) {
2692 MSARegister scratch0 = kSimd128RegZero;
2693 MSARegister scratch1 = kSimd128ScratchReg;
2694 srli_w(scratch0, src.fp().toW(), 31);
2695 srli_d(scratch1, scratch0, 31);
2696 or_v(scratch0, scratch0, scratch1);
2697 shf_w(scratch1, scratch0, 0x0E);
2698 slli_d(scratch1, scratch1, 2);
2699 or_v(scratch0, scratch0, scratch1);
2700 copy_u_b(dst.gp(), scratch0, 0);
2701 }
2702
emit_i32x4_shl(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2703 void LiftoffAssembler::emit_i32x4_shl(LiftoffRegister dst, LiftoffRegister lhs,
2704 LiftoffRegister rhs) {
2705 fill_w(kSimd128ScratchReg, rhs.gp());
2706 sll_w(dst.fp().toW(), lhs.fp().toW(), kSimd128ScratchReg);
2707 }
2708
emit_i32x4_shli(LiftoffRegister dst,LiftoffRegister lhs,int32_t rhs)2709 void LiftoffAssembler::emit_i32x4_shli(LiftoffRegister dst, LiftoffRegister lhs,
2710 int32_t rhs) {
2711 slli_w(dst.fp().toW(), lhs.fp().toW(), rhs & 31);
2712 }
2713
emit_i32x4_shr_s(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2714 void LiftoffAssembler::emit_i32x4_shr_s(LiftoffRegister dst,
2715 LiftoffRegister lhs,
2716 LiftoffRegister rhs) {
2717 fill_w(kSimd128ScratchReg, rhs.gp());
2718 sra_w(dst.fp().toW(), lhs.fp().toW(), kSimd128ScratchReg);
2719 }
2720
emit_i32x4_shri_s(LiftoffRegister dst,LiftoffRegister lhs,int32_t rhs)2721 void LiftoffAssembler::emit_i32x4_shri_s(LiftoffRegister dst,
2722 LiftoffRegister lhs, int32_t rhs) {
2723 srai_w(dst.fp().toW(), lhs.fp().toW(), rhs & 31);
2724 }
2725
emit_i32x4_shr_u(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2726 void LiftoffAssembler::emit_i32x4_shr_u(LiftoffRegister dst,
2727 LiftoffRegister lhs,
2728 LiftoffRegister rhs) {
2729 fill_w(kSimd128ScratchReg, rhs.gp());
2730 srl_w(dst.fp().toW(), lhs.fp().toW(), kSimd128ScratchReg);
2731 }
2732
emit_i32x4_shri_u(LiftoffRegister dst,LiftoffRegister lhs,int32_t rhs)2733 void LiftoffAssembler::emit_i32x4_shri_u(LiftoffRegister dst,
2734 LiftoffRegister lhs, int32_t rhs) {
2735 srli_w(dst.fp().toW(), lhs.fp().toW(), rhs & 31);
2736 }
2737
emit_i32x4_add(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2738 void LiftoffAssembler::emit_i32x4_add(LiftoffRegister dst, LiftoffRegister lhs,
2739 LiftoffRegister rhs) {
2740 addv_w(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2741 }
2742
emit_i32x4_sub(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2743 void LiftoffAssembler::emit_i32x4_sub(LiftoffRegister dst, LiftoffRegister lhs,
2744 LiftoffRegister rhs) {
2745 subv_w(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2746 }
2747
emit_i32x4_mul(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2748 void LiftoffAssembler::emit_i32x4_mul(LiftoffRegister dst, LiftoffRegister lhs,
2749 LiftoffRegister rhs) {
2750 mulv_w(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2751 }
2752
emit_i32x4_min_s(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2753 void LiftoffAssembler::emit_i32x4_min_s(LiftoffRegister dst,
2754 LiftoffRegister lhs,
2755 LiftoffRegister rhs) {
2756 min_s_w(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2757 }
2758
emit_i32x4_min_u(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2759 void LiftoffAssembler::emit_i32x4_min_u(LiftoffRegister dst,
2760 LiftoffRegister lhs,
2761 LiftoffRegister rhs) {
2762 min_u_w(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2763 }
2764
emit_i32x4_max_s(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2765 void LiftoffAssembler::emit_i32x4_max_s(LiftoffRegister dst,
2766 LiftoffRegister lhs,
2767 LiftoffRegister rhs) {
2768 max_s_w(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2769 }
2770
emit_i32x4_max_u(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2771 void LiftoffAssembler::emit_i32x4_max_u(LiftoffRegister dst,
2772 LiftoffRegister lhs,
2773 LiftoffRegister rhs) {
2774 max_u_w(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2775 }
2776
emit_i32x4_dot_i16x8_s(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2777 void LiftoffAssembler::emit_i32x4_dot_i16x8_s(LiftoffRegister dst,
2778 LiftoffRegister lhs,
2779 LiftoffRegister rhs) {
2780 dotp_s_w(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2781 }
2782
emit_i64x2_neg(LiftoffRegister dst,LiftoffRegister src)2783 void LiftoffAssembler::emit_i64x2_neg(LiftoffRegister dst,
2784 LiftoffRegister src) {
2785 xor_v(kSimd128RegZero, kSimd128RegZero, kSimd128RegZero);
2786 subv_d(dst.fp().toW(), kSimd128RegZero, src.fp().toW());
2787 }
2788
emit_i64x2_alltrue(LiftoffRegister dst,LiftoffRegister src)2789 void LiftoffAssembler::emit_i64x2_alltrue(LiftoffRegister dst,
2790 LiftoffRegister src) {
2791 liftoff::EmitAllTrue(this, dst, src, MSA_BRANCH_D);
2792 }
2793
emit_i64x2_bitmask(LiftoffRegister dst,LiftoffRegister src)2794 void LiftoffAssembler::emit_i64x2_bitmask(LiftoffRegister dst,
2795 LiftoffRegister src) {
2796 srli_d(kSimd128RegZero, src.fp().toW(), 63);
2797 shf_w(kSimd128ScratchReg, kSimd128RegZero, 0x02);
2798 slli_d(kSimd128ScratchReg, kSimd128ScratchReg, 1);
2799 or_v(kSimd128RegZero, kSimd128RegZero, kSimd128ScratchReg);
2800 copy_u_b(dst.gp(), kSimd128RegZero, 0);
2801 }
2802
emit_i64x2_shl(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2803 void LiftoffAssembler::emit_i64x2_shl(LiftoffRegister dst, LiftoffRegister lhs,
2804 LiftoffRegister rhs) {
2805 fill_d(kSimd128ScratchReg, rhs.gp());
2806 sll_d(dst.fp().toW(), lhs.fp().toW(), kSimd128ScratchReg);
2807 }
2808
emit_i64x2_shli(LiftoffRegister dst,LiftoffRegister lhs,int32_t rhs)2809 void LiftoffAssembler::emit_i64x2_shli(LiftoffRegister dst, LiftoffRegister lhs,
2810 int32_t rhs) {
2811 slli_d(dst.fp().toW(), lhs.fp().toW(), rhs & 63);
2812 }
2813
emit_i64x2_shr_s(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2814 void LiftoffAssembler::emit_i64x2_shr_s(LiftoffRegister dst,
2815 LiftoffRegister lhs,
2816 LiftoffRegister rhs) {
2817 fill_d(kSimd128ScratchReg, rhs.gp());
2818 sra_d(dst.fp().toW(), lhs.fp().toW(), kSimd128ScratchReg);
2819 }
2820
emit_i64x2_shri_s(LiftoffRegister dst,LiftoffRegister lhs,int32_t rhs)2821 void LiftoffAssembler::emit_i64x2_shri_s(LiftoffRegister dst,
2822 LiftoffRegister lhs, int32_t rhs) {
2823 srai_d(dst.fp().toW(), lhs.fp().toW(), rhs & 63);
2824 }
2825
emit_i64x2_shr_u(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2826 void LiftoffAssembler::emit_i64x2_shr_u(LiftoffRegister dst,
2827 LiftoffRegister lhs,
2828 LiftoffRegister rhs) {
2829 fill_d(kSimd128ScratchReg, rhs.gp());
2830 srl_d(dst.fp().toW(), lhs.fp().toW(), kSimd128ScratchReg);
2831 }
2832
emit_i64x2_shri_u(LiftoffRegister dst,LiftoffRegister lhs,int32_t rhs)2833 void LiftoffAssembler::emit_i64x2_shri_u(LiftoffRegister dst,
2834 LiftoffRegister lhs, int32_t rhs) {
2835 srli_d(dst.fp().toW(), lhs.fp().toW(), rhs & 63);
2836 }
2837
emit_i64x2_add(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2838 void LiftoffAssembler::emit_i64x2_add(LiftoffRegister dst, LiftoffRegister lhs,
2839 LiftoffRegister rhs) {
2840 addv_d(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2841 }
2842
emit_i64x2_sub(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2843 void LiftoffAssembler::emit_i64x2_sub(LiftoffRegister dst, LiftoffRegister lhs,
2844 LiftoffRegister rhs) {
2845 subv_d(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2846 }
2847
emit_i64x2_mul(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2848 void LiftoffAssembler::emit_i64x2_mul(LiftoffRegister dst, LiftoffRegister lhs,
2849 LiftoffRegister rhs) {
2850 mulv_d(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2851 }
2852
emit_i64x2_gt_s(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2853 void LiftoffAssembler::emit_i64x2_gt_s(LiftoffRegister dst, LiftoffRegister lhs,
2854 LiftoffRegister rhs) {
2855 clt_s_d(dst.fp().toW(), rhs.fp().toW(), lhs.fp().toW());
2856 }
2857
emit_i64x2_ge_s(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2858 void LiftoffAssembler::emit_i64x2_ge_s(LiftoffRegister dst, LiftoffRegister lhs,
2859 LiftoffRegister rhs) {
2860 cle_s_d(dst.fp().toW(), rhs.fp().toW(), lhs.fp().toW());
2861 }
2862
emit_f32x4_abs(LiftoffRegister dst,LiftoffRegister src)2863 void LiftoffAssembler::emit_f32x4_abs(LiftoffRegister dst,
2864 LiftoffRegister src) {
2865 bclri_w(dst.fp().toW(), src.fp().toW(), 31);
2866 }
2867
emit_f32x4_neg(LiftoffRegister dst,LiftoffRegister src)2868 void LiftoffAssembler::emit_f32x4_neg(LiftoffRegister dst,
2869 LiftoffRegister src) {
2870 bnegi_w(dst.fp().toW(), src.fp().toW(), 31);
2871 }
2872
emit_f32x4_sqrt(LiftoffRegister dst,LiftoffRegister src)2873 void LiftoffAssembler::emit_f32x4_sqrt(LiftoffRegister dst,
2874 LiftoffRegister src) {
2875 fsqrt_w(dst.fp().toW(), src.fp().toW());
2876 }
2877
emit_f32x4_ceil(LiftoffRegister dst,LiftoffRegister src)2878 bool LiftoffAssembler::emit_f32x4_ceil(LiftoffRegister dst,
2879 LiftoffRegister src) {
2880 MSARoundW(dst.fp().toW(), src.fp().toW(), kRoundToPlusInf);
2881 return true;
2882 }
2883
emit_f32x4_floor(LiftoffRegister dst,LiftoffRegister src)2884 bool LiftoffAssembler::emit_f32x4_floor(LiftoffRegister dst,
2885 LiftoffRegister src) {
2886 MSARoundW(dst.fp().toW(), src.fp().toW(), kRoundToMinusInf);
2887 return true;
2888 }
2889
emit_f32x4_trunc(LiftoffRegister dst,LiftoffRegister src)2890 bool LiftoffAssembler::emit_f32x4_trunc(LiftoffRegister dst,
2891 LiftoffRegister src) {
2892 MSARoundW(dst.fp().toW(), src.fp().toW(), kRoundToZero);
2893 return true;
2894 }
2895
emit_f32x4_nearest_int(LiftoffRegister dst,LiftoffRegister src)2896 bool LiftoffAssembler::emit_f32x4_nearest_int(LiftoffRegister dst,
2897 LiftoffRegister src) {
2898 MSARoundW(dst.fp().toW(), src.fp().toW(), kRoundToNearest);
2899 return true;
2900 }
2901
emit_f32x4_add(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2902 void LiftoffAssembler::emit_f32x4_add(LiftoffRegister dst, LiftoffRegister lhs,
2903 LiftoffRegister rhs) {
2904 fadd_w(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2905 }
2906
emit_f32x4_sub(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2907 void LiftoffAssembler::emit_f32x4_sub(LiftoffRegister dst, LiftoffRegister lhs,
2908 LiftoffRegister rhs) {
2909 fsub_w(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2910 }
2911
emit_f32x4_mul(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2912 void LiftoffAssembler::emit_f32x4_mul(LiftoffRegister dst, LiftoffRegister lhs,
2913 LiftoffRegister rhs) {
2914 fmul_w(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2915 }
2916
emit_f32x4_div(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2917 void LiftoffAssembler::emit_f32x4_div(LiftoffRegister dst, LiftoffRegister lhs,
2918 LiftoffRegister rhs) {
2919 fdiv_w(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
2920 }
2921
emit_f32x4_min(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2922 void LiftoffAssembler::emit_f32x4_min(LiftoffRegister dst, LiftoffRegister lhs,
2923 LiftoffRegister rhs) {
2924 MSARegister dst_msa = dst.fp().toW();
2925 MSARegister lhs_msa = lhs.fp().toW();
2926 MSARegister rhs_msa = rhs.fp().toW();
2927 MSARegister scratch0 = kSimd128RegZero;
2928 MSARegister scratch1 = kSimd128ScratchReg;
2929 // If inputs are -0.0. and +0.0, then write -0.0 to scratch1.
2930 // scratch1 = (lhs == rhs) ? (lhs | rhs) : (rhs | rhs).
2931 fseq_w(scratch0, lhs_msa, rhs_msa);
2932 bsel_v(scratch0, rhs_msa, lhs_msa);
2933 or_v(scratch1, scratch0, rhs_msa);
2934 // scratch0 = isNaN(scratch1) ? scratch1: lhs.
2935 fseq_w(scratch0, scratch1, scratch1);
2936 bsel_v(scratch0, scratch1, lhs_msa);
2937 // dst = (scratch1 <= scratch0) ? scratch1 : scratch0.
2938 fsle_w(dst_msa, scratch1, scratch0);
2939 bsel_v(dst_msa, scratch0, scratch1);
2940 // Canonicalize the result.
2941 fmin_w(dst_msa, dst_msa, dst_msa);
2942 }
2943
emit_f32x4_max(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2944 void LiftoffAssembler::emit_f32x4_max(LiftoffRegister dst, LiftoffRegister lhs,
2945 LiftoffRegister rhs) {
2946 MSARegister dst_msa = dst.fp().toW();
2947 MSARegister lhs_msa = lhs.fp().toW();
2948 MSARegister rhs_msa = rhs.fp().toW();
2949 MSARegister scratch0 = kSimd128RegZero;
2950 MSARegister scratch1 = kSimd128ScratchReg;
2951 // If inputs are -0.0. and +0.0, then write +0.0 to scratch1.
2952 // scratch1 = (lhs == rhs) ? (lhs | rhs) : (rhs | rhs).
2953 fseq_w(scratch0, lhs_msa, rhs_msa);
2954 bsel_v(scratch0, rhs_msa, lhs_msa);
2955 and_v(scratch1, scratch0, rhs_msa);
2956 // scratch0 = isNaN(scratch1) ? scratch1: lhs.
2957 fseq_w(scratch0, scratch1, scratch1);
2958 bsel_v(scratch0, scratch1, lhs_msa);
2959 // dst = (scratch0 <= scratch1) ? scratch1 : scratch0.
2960 fsle_w(dst_msa, scratch0, scratch1);
2961 bsel_v(dst_msa, scratch0, scratch1);
2962 // Canonicalize the result.
2963 fmax_w(dst_msa, dst_msa, dst_msa);
2964 }
2965
emit_f32x4_pmin(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2966 void LiftoffAssembler::emit_f32x4_pmin(LiftoffRegister dst, LiftoffRegister lhs,
2967 LiftoffRegister rhs) {
2968 MSARegister dst_msa = dst.fp().toW();
2969 MSARegister lhs_msa = lhs.fp().toW();
2970 MSARegister rhs_msa = rhs.fp().toW();
2971 // dst = rhs < lhs ? rhs : lhs
2972 fclt_w(dst_msa, rhs_msa, lhs_msa);
2973 bsel_v(dst_msa, lhs_msa, rhs_msa);
2974 }
2975
emit_f32x4_pmax(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)2976 void LiftoffAssembler::emit_f32x4_pmax(LiftoffRegister dst, LiftoffRegister lhs,
2977 LiftoffRegister rhs) {
2978 MSARegister dst_msa = dst.fp().toW();
2979 MSARegister lhs_msa = lhs.fp().toW();
2980 MSARegister rhs_msa = rhs.fp().toW();
2981 // dst = lhs < rhs ? rhs : lhs
2982 fclt_w(dst_msa, lhs_msa, rhs_msa);
2983 bsel_v(dst_msa, lhs_msa, rhs_msa);
2984 }
2985
emit_f64x2_abs(LiftoffRegister dst,LiftoffRegister src)2986 void LiftoffAssembler::emit_f64x2_abs(LiftoffRegister dst,
2987 LiftoffRegister src) {
2988 bclri_d(dst.fp().toW(), src.fp().toW(), 63);
2989 }
2990
emit_f64x2_neg(LiftoffRegister dst,LiftoffRegister src)2991 void LiftoffAssembler::emit_f64x2_neg(LiftoffRegister dst,
2992 LiftoffRegister src) {
2993 bnegi_d(dst.fp().toW(), src.fp().toW(), 63);
2994 }
2995
emit_f64x2_sqrt(LiftoffRegister dst,LiftoffRegister src)2996 void LiftoffAssembler::emit_f64x2_sqrt(LiftoffRegister dst,
2997 LiftoffRegister src) {
2998 fsqrt_d(dst.fp().toW(), src.fp().toW());
2999 }
3000
emit_f64x2_ceil(LiftoffRegister dst,LiftoffRegister src)3001 bool LiftoffAssembler::emit_f64x2_ceil(LiftoffRegister dst,
3002 LiftoffRegister src) {
3003 MSARoundD(dst.fp().toW(), src.fp().toW(), kRoundToPlusInf);
3004 return true;
3005 }
3006
emit_f64x2_floor(LiftoffRegister dst,LiftoffRegister src)3007 bool LiftoffAssembler::emit_f64x2_floor(LiftoffRegister dst,
3008 LiftoffRegister src) {
3009 MSARoundD(dst.fp().toW(), src.fp().toW(), kRoundToMinusInf);
3010 return true;
3011 }
3012
emit_f64x2_trunc(LiftoffRegister dst,LiftoffRegister src)3013 bool LiftoffAssembler::emit_f64x2_trunc(LiftoffRegister dst,
3014 LiftoffRegister src) {
3015 MSARoundD(dst.fp().toW(), src.fp().toW(), kRoundToZero);
3016 return true;
3017 }
3018
emit_f64x2_nearest_int(LiftoffRegister dst,LiftoffRegister src)3019 bool LiftoffAssembler::emit_f64x2_nearest_int(LiftoffRegister dst,
3020 LiftoffRegister src) {
3021 MSARoundD(dst.fp().toW(), src.fp().toW(), kRoundToNearest);
3022 return true;
3023 }
3024
emit_f64x2_add(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)3025 void LiftoffAssembler::emit_f64x2_add(LiftoffRegister dst, LiftoffRegister lhs,
3026 LiftoffRegister rhs) {
3027 fadd_d(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
3028 }
3029
emit_f64x2_sub(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)3030 void LiftoffAssembler::emit_f64x2_sub(LiftoffRegister dst, LiftoffRegister lhs,
3031 LiftoffRegister rhs) {
3032 fsub_d(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
3033 }
3034
emit_f64x2_mul(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)3035 void LiftoffAssembler::emit_f64x2_mul(LiftoffRegister dst, LiftoffRegister lhs,
3036 LiftoffRegister rhs) {
3037 fmul_d(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
3038 }
3039
emit_f64x2_div(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)3040 void LiftoffAssembler::emit_f64x2_div(LiftoffRegister dst, LiftoffRegister lhs,
3041 LiftoffRegister rhs) {
3042 fdiv_d(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
3043 }
3044
emit_f64x2_min(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)3045 void LiftoffAssembler::emit_f64x2_min(LiftoffRegister dst, LiftoffRegister lhs,
3046 LiftoffRegister rhs) {
3047 MSARegister dst_msa = dst.fp().toW();
3048 MSARegister lhs_msa = lhs.fp().toW();
3049 MSARegister rhs_msa = rhs.fp().toW();
3050 MSARegister scratch0 = kSimd128RegZero;
3051 MSARegister scratch1 = kSimd128ScratchReg;
3052 // If inputs are -0.0. and +0.0, then write -0.0 to scratch1.
3053 // scratch1 = (lhs == rhs) ? (lhs | rhs) : (rhs | rhs).
3054 fseq_d(scratch0, lhs_msa, rhs_msa);
3055 bsel_v(scratch0, rhs_msa, lhs_msa);
3056 or_v(scratch1, scratch0, rhs_msa);
3057 // scratch0 = isNaN(scratch1) ? scratch1: lhs.
3058 fseq_d(scratch0, scratch1, scratch1);
3059 bsel_v(scratch0, scratch1, lhs_msa);
3060 // dst = (scratch1 <= scratch0) ? scratch1 : scratch0.
3061 fsle_d(dst_msa, scratch1, scratch0);
3062 bsel_v(dst_msa, scratch0, scratch1);
3063 // Canonicalize the result.
3064 fmin_d(dst_msa, dst_msa, dst_msa);
3065 }
3066
emit_f64x2_max(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)3067 void LiftoffAssembler::emit_f64x2_max(LiftoffRegister dst, LiftoffRegister lhs,
3068 LiftoffRegister rhs) {
3069 MSARegister dst_msa = dst.fp().toW();
3070 MSARegister lhs_msa = lhs.fp().toW();
3071 MSARegister rhs_msa = rhs.fp().toW();
3072 MSARegister scratch0 = kSimd128RegZero;
3073 MSARegister scratch1 = kSimd128ScratchReg;
3074 // If inputs are -0.0. and +0.0, then write +0.0 to scratch1.
3075 // scratch1 = (lhs == rhs) ? (lhs | rhs) : (rhs | rhs).
3076 fseq_d(scratch0, lhs_msa, rhs_msa);
3077 bsel_v(scratch0, rhs_msa, lhs_msa);
3078 and_v(scratch1, scratch0, rhs_msa);
3079 // scratch0 = isNaN(scratch1) ? scratch1: lhs.
3080 fseq_d(scratch0, scratch1, scratch1);
3081 bsel_v(scratch0, scratch1, lhs_msa);
3082 // dst = (scratch0 <= scratch1) ? scratch1 : scratch0.
3083 fsle_d(dst_msa, scratch0, scratch1);
3084 bsel_v(dst_msa, scratch0, scratch1);
3085 // Canonicalize the result.
3086 fmax_d(dst_msa, dst_msa, dst_msa);
3087 }
3088
emit_f64x2_pmin(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)3089 void LiftoffAssembler::emit_f64x2_pmin(LiftoffRegister dst, LiftoffRegister lhs,
3090 LiftoffRegister rhs) {
3091 MSARegister dst_msa = dst.fp().toW();
3092 MSARegister lhs_msa = lhs.fp().toW();
3093 MSARegister rhs_msa = rhs.fp().toW();
3094 // dst = rhs < lhs ? rhs : lhs
3095 fclt_d(dst_msa, rhs_msa, lhs_msa);
3096 bsel_v(dst_msa, lhs_msa, rhs_msa);
3097 }
3098
emit_f64x2_pmax(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)3099 void LiftoffAssembler::emit_f64x2_pmax(LiftoffRegister dst, LiftoffRegister lhs,
3100 LiftoffRegister rhs) {
3101 MSARegister dst_msa = dst.fp().toW();
3102 MSARegister lhs_msa = lhs.fp().toW();
3103 MSARegister rhs_msa = rhs.fp().toW();
3104 // dst = lhs < rhs ? rhs : lhs
3105 fclt_d(dst_msa, lhs_msa, rhs_msa);
3106 bsel_v(dst_msa, lhs_msa, rhs_msa);
3107 }
3108
emit_f64x2_convert_low_i32x4_s(LiftoffRegister dst,LiftoffRegister src)3109 void LiftoffAssembler::emit_f64x2_convert_low_i32x4_s(LiftoffRegister dst,
3110 LiftoffRegister src) {
3111 xor_v(kSimd128RegZero, kSimd128RegZero, kSimd128RegZero);
3112 ilvr_w(kSimd128RegZero, kSimd128RegZero, src.fp().toW());
3113 slli_d(kSimd128RegZero, kSimd128RegZero, 32);
3114 srai_d(kSimd128RegZero, kSimd128RegZero, 32);
3115 ffint_s_d(dst.fp().toW(), kSimd128RegZero);
3116 }
3117
emit_f64x2_convert_low_i32x4_u(LiftoffRegister dst,LiftoffRegister src)3118 void LiftoffAssembler::emit_f64x2_convert_low_i32x4_u(LiftoffRegister dst,
3119 LiftoffRegister src) {
3120 xor_v(kSimd128RegZero, kSimd128RegZero, kSimd128RegZero);
3121 ilvr_w(kSimd128RegZero, kSimd128RegZero, src.fp().toW());
3122 ffint_u_d(dst.fp().toW(), kSimd128RegZero);
3123 }
3124
emit_f64x2_promote_low_f32x4(LiftoffRegister dst,LiftoffRegister src)3125 void LiftoffAssembler::emit_f64x2_promote_low_f32x4(LiftoffRegister dst,
3126 LiftoffRegister src) {
3127 fexupr_d(dst.fp().toW(), src.fp().toW());
3128 }
3129
emit_i32x4_sconvert_f32x4(LiftoffRegister dst,LiftoffRegister src)3130 void LiftoffAssembler::emit_i32x4_sconvert_f32x4(LiftoffRegister dst,
3131 LiftoffRegister src) {
3132 ftrunc_s_w(dst.fp().toW(), src.fp().toW());
3133 }
3134
emit_i32x4_uconvert_f32x4(LiftoffRegister dst,LiftoffRegister src)3135 void LiftoffAssembler::emit_i32x4_uconvert_f32x4(LiftoffRegister dst,
3136 LiftoffRegister src) {
3137 ftrunc_u_w(dst.fp().toW(), src.fp().toW());
3138 }
3139
emit_i32x4_trunc_sat_f64x2_s_zero(LiftoffRegister dst,LiftoffRegister src)3140 void LiftoffAssembler::emit_i32x4_trunc_sat_f64x2_s_zero(LiftoffRegister dst,
3141 LiftoffRegister src) {
3142 xor_v(kSimd128RegZero, kSimd128RegZero, kSimd128RegZero);
3143 ftrunc_s_d(kSimd128ScratchReg, src.fp().toW());
3144 sat_s_d(kSimd128ScratchReg, kSimd128ScratchReg, 31);
3145 pckev_w(dst.fp().toW(), kSimd128RegZero, kSimd128ScratchReg);
3146 }
3147
emit_i32x4_trunc_sat_f64x2_u_zero(LiftoffRegister dst,LiftoffRegister src)3148 void LiftoffAssembler::emit_i32x4_trunc_sat_f64x2_u_zero(LiftoffRegister dst,
3149 LiftoffRegister src) {
3150 xor_v(kSimd128RegZero, kSimd128RegZero, kSimd128RegZero);
3151 ftrunc_u_d(kSimd128ScratchReg, src.fp().toW());
3152 sat_u_d(kSimd128ScratchReg, kSimd128ScratchReg, 31);
3153 pckev_w(dst.fp().toW(), kSimd128RegZero, kSimd128ScratchReg);
3154 }
3155
emit_f32x4_sconvert_i32x4(LiftoffRegister dst,LiftoffRegister src)3156 void LiftoffAssembler::emit_f32x4_sconvert_i32x4(LiftoffRegister dst,
3157 LiftoffRegister src) {
3158 ffint_s_w(dst.fp().toW(), src.fp().toW());
3159 }
3160
emit_f32x4_uconvert_i32x4(LiftoffRegister dst,LiftoffRegister src)3161 void LiftoffAssembler::emit_f32x4_uconvert_i32x4(LiftoffRegister dst,
3162 LiftoffRegister src) {
3163 ffint_u_w(dst.fp().toW(), src.fp().toW());
3164 }
3165
emit_f32x4_demote_f64x2_zero(LiftoffRegister dst,LiftoffRegister src)3166 void LiftoffAssembler::emit_f32x4_demote_f64x2_zero(LiftoffRegister dst,
3167 LiftoffRegister src) {
3168 xor_v(kSimd128RegZero, kSimd128RegZero, kSimd128RegZero);
3169 fexdo_w(dst.fp().toW(), kSimd128RegZero, src.fp().toW());
3170 }
3171
emit_i8x16_sconvert_i16x8(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)3172 void LiftoffAssembler::emit_i8x16_sconvert_i16x8(LiftoffRegister dst,
3173 LiftoffRegister lhs,
3174 LiftoffRegister rhs) {
3175 sat_s_h(kSimd128ScratchReg, lhs.fp().toW(), 7);
3176 sat_s_h(dst.fp().toW(), lhs.fp().toW(), 7);
3177 pckev_b(dst.fp().toW(), dst.fp().toW(), kSimd128ScratchReg);
3178 }
3179
emit_i8x16_uconvert_i16x8(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)3180 void LiftoffAssembler::emit_i8x16_uconvert_i16x8(LiftoffRegister dst,
3181 LiftoffRegister lhs,
3182 LiftoffRegister rhs) {
3183 xor_v(kSimd128RegZero, kSimd128RegZero, kSimd128RegZero);
3184 max_s_h(kSimd128ScratchReg, kSimd128RegZero, lhs.fp().toW());
3185 sat_u_h(kSimd128ScratchReg, kSimd128ScratchReg, 7);
3186 max_s_h(dst.fp().toW(), kSimd128RegZero, rhs.fp().toW());
3187 sat_u_h(dst.fp().toW(), dst.fp().toW(), 7);
3188 pckev_b(dst.fp().toW(), dst.fp().toW(), kSimd128ScratchReg);
3189 }
3190
emit_i16x8_sconvert_i32x4(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)3191 void LiftoffAssembler::emit_i16x8_sconvert_i32x4(LiftoffRegister dst,
3192 LiftoffRegister lhs,
3193 LiftoffRegister rhs) {
3194 sat_s_w(kSimd128ScratchReg, lhs.fp().toW(), 15);
3195 sat_s_w(dst.fp().toW(), lhs.fp().toW(), 15);
3196 pckev_h(dst.fp().toW(), dst.fp().toW(), kSimd128ScratchReg);
3197 }
3198
emit_i16x8_uconvert_i32x4(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)3199 void LiftoffAssembler::emit_i16x8_uconvert_i32x4(LiftoffRegister dst,
3200 LiftoffRegister lhs,
3201 LiftoffRegister rhs) {
3202 xor_v(kSimd128RegZero, kSimd128RegZero, kSimd128RegZero);
3203 max_s_w(kSimd128ScratchReg, kSimd128RegZero, lhs.fp().toW());
3204 sat_u_w(kSimd128ScratchReg, kSimd128ScratchReg, 15);
3205 max_s_w(dst.fp().toW(), kSimd128RegZero, rhs.fp().toW());
3206 sat_u_w(dst.fp().toW(), dst.fp().toW(), 15);
3207 pckev_h(dst.fp().toW(), dst.fp().toW(), kSimd128ScratchReg);
3208 }
3209
emit_i16x8_sconvert_i8x16_low(LiftoffRegister dst,LiftoffRegister src)3210 void LiftoffAssembler::emit_i16x8_sconvert_i8x16_low(LiftoffRegister dst,
3211 LiftoffRegister src) {
3212 ilvr_b(kSimd128ScratchReg, src.fp().toW(), src.fp().toW());
3213 slli_h(dst.fp().toW(), kSimd128ScratchReg, 8);
3214 srai_h(dst.fp().toW(), dst.fp().toW(), 8);
3215 }
3216
emit_i16x8_sconvert_i8x16_high(LiftoffRegister dst,LiftoffRegister src)3217 void LiftoffAssembler::emit_i16x8_sconvert_i8x16_high(LiftoffRegister dst,
3218 LiftoffRegister src) {
3219 ilvl_b(kSimd128ScratchReg, src.fp().toW(), src.fp().toW());
3220 slli_h(dst.fp().toW(), kSimd128ScratchReg, 8);
3221 srai_h(dst.fp().toW(), dst.fp().toW(), 8);
3222 }
3223
emit_i16x8_uconvert_i8x16_low(LiftoffRegister dst,LiftoffRegister src)3224 void LiftoffAssembler::emit_i16x8_uconvert_i8x16_low(LiftoffRegister dst,
3225 LiftoffRegister src) {
3226 xor_v(kSimd128RegZero, kSimd128RegZero, kSimd128RegZero);
3227 ilvr_b(dst.fp().toW(), kSimd128RegZero, src.fp().toW());
3228 }
3229
emit_i16x8_uconvert_i8x16_high(LiftoffRegister dst,LiftoffRegister src)3230 void LiftoffAssembler::emit_i16x8_uconvert_i8x16_high(LiftoffRegister dst,
3231 LiftoffRegister src) {
3232 xor_v(kSimd128RegZero, kSimd128RegZero, kSimd128RegZero);
3233 ilvl_b(dst.fp().toW(), kSimd128RegZero, src.fp().toW());
3234 }
3235
emit_i32x4_sconvert_i16x8_low(LiftoffRegister dst,LiftoffRegister src)3236 void LiftoffAssembler::emit_i32x4_sconvert_i16x8_low(LiftoffRegister dst,
3237 LiftoffRegister src) {
3238 ilvr_h(kSimd128ScratchReg, src.fp().toW(), src.fp().toW());
3239 slli_w(dst.fp().toW(), kSimd128ScratchReg, 16);
3240 srai_w(dst.fp().toW(), dst.fp().toW(), 16);
3241 }
3242
emit_i32x4_sconvert_i16x8_high(LiftoffRegister dst,LiftoffRegister src)3243 void LiftoffAssembler::emit_i32x4_sconvert_i16x8_high(LiftoffRegister dst,
3244 LiftoffRegister src) {
3245 ilvl_h(kSimd128ScratchReg, src.fp().toW(), src.fp().toW());
3246 slli_w(dst.fp().toW(), kSimd128ScratchReg, 16);
3247 srai_w(dst.fp().toW(), dst.fp().toW(), 16);
3248 }
3249
emit_i32x4_uconvert_i16x8_low(LiftoffRegister dst,LiftoffRegister src)3250 void LiftoffAssembler::emit_i32x4_uconvert_i16x8_low(LiftoffRegister dst,
3251 LiftoffRegister src) {
3252 xor_v(kSimd128RegZero, kSimd128RegZero, kSimd128RegZero);
3253 ilvr_h(dst.fp().toW(), kSimd128RegZero, src.fp().toW());
3254 }
3255
emit_i32x4_uconvert_i16x8_high(LiftoffRegister dst,LiftoffRegister src)3256 void LiftoffAssembler::emit_i32x4_uconvert_i16x8_high(LiftoffRegister dst,
3257 LiftoffRegister src) {
3258 xor_v(kSimd128RegZero, kSimd128RegZero, kSimd128RegZero);
3259 ilvl_h(dst.fp().toW(), kSimd128RegZero, src.fp().toW());
3260 }
3261
emit_i64x2_sconvert_i32x4_low(LiftoffRegister dst,LiftoffRegister src)3262 void LiftoffAssembler::emit_i64x2_sconvert_i32x4_low(LiftoffRegister dst,
3263 LiftoffRegister src) {
3264 ilvr_w(kSimd128ScratchReg, src.fp().toW(), src.fp().toW());
3265 slli_d(dst.fp().toW(), kSimd128ScratchReg, 32);
3266 srai_d(dst.fp().toW(), dst.fp().toW(), 32);
3267 }
3268
emit_i64x2_sconvert_i32x4_high(LiftoffRegister dst,LiftoffRegister src)3269 void LiftoffAssembler::emit_i64x2_sconvert_i32x4_high(LiftoffRegister dst,
3270 LiftoffRegister src) {
3271 ilvl_w(kSimd128ScratchReg, src.fp().toW(), src.fp().toW());
3272 slli_d(dst.fp().toW(), kSimd128ScratchReg, 32);
3273 srai_d(dst.fp().toW(), dst.fp().toW(), 32);
3274 }
3275
emit_i64x2_uconvert_i32x4_low(LiftoffRegister dst,LiftoffRegister src)3276 void LiftoffAssembler::emit_i64x2_uconvert_i32x4_low(LiftoffRegister dst,
3277 LiftoffRegister src) {
3278 xor_v(kSimd128RegZero, kSimd128RegZero, kSimd128RegZero);
3279 ilvr_w(dst.fp().toW(), kSimd128RegZero, src.fp().toW());
3280 }
3281
emit_i64x2_uconvert_i32x4_high(LiftoffRegister dst,LiftoffRegister src)3282 void LiftoffAssembler::emit_i64x2_uconvert_i32x4_high(LiftoffRegister dst,
3283 LiftoffRegister src) {
3284 xor_v(kSimd128RegZero, kSimd128RegZero, kSimd128RegZero);
3285 ilvl_w(dst.fp().toW(), kSimd128RegZero, src.fp().toW());
3286 }
3287
emit_i8x16_rounding_average_u(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)3288 void LiftoffAssembler::emit_i8x16_rounding_average_u(LiftoffRegister dst,
3289 LiftoffRegister lhs,
3290 LiftoffRegister rhs) {
3291 aver_u_b(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
3292 }
3293
emit_i16x8_rounding_average_u(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)3294 void LiftoffAssembler::emit_i16x8_rounding_average_u(LiftoffRegister dst,
3295 LiftoffRegister lhs,
3296 LiftoffRegister rhs) {
3297 aver_u_h(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW());
3298 }
3299
emit_i8x16_abs(LiftoffRegister dst,LiftoffRegister src)3300 void LiftoffAssembler::emit_i8x16_abs(LiftoffRegister dst,
3301 LiftoffRegister src) {
3302 xor_v(kSimd128RegZero, kSimd128RegZero, kSimd128RegZero);
3303 asub_s_b(dst.fp().toW(), src.fp().toW(), kSimd128RegZero);
3304 }
3305
emit_i16x8_abs(LiftoffRegister dst,LiftoffRegister src)3306 void LiftoffAssembler::emit_i16x8_abs(LiftoffRegister dst,
3307 LiftoffRegister src) {
3308 xor_v(kSimd128RegZero, kSimd128RegZero, kSimd128RegZero);
3309 asub_s_h(dst.fp().toW(), src.fp().toW(), kSimd128RegZero);
3310 }
3311
emit_i32x4_abs(LiftoffRegister dst,LiftoffRegister src)3312 void LiftoffAssembler::emit_i32x4_abs(LiftoffRegister dst,
3313 LiftoffRegister src) {
3314 xor_v(kSimd128RegZero, kSimd128RegZero, kSimd128RegZero);
3315 asub_s_w(dst.fp().toW(), src.fp().toW(), kSimd128RegZero);
3316 }
3317
emit_i8x16_extract_lane_s(LiftoffRegister dst,LiftoffRegister lhs,uint8_t imm_lane_idx)3318 void LiftoffAssembler::emit_i8x16_extract_lane_s(LiftoffRegister dst,
3319 LiftoffRegister lhs,
3320 uint8_t imm_lane_idx) {
3321 copy_s_b(dst.gp(), lhs.fp().toW(), imm_lane_idx);
3322 }
3323
emit_i8x16_extract_lane_u(LiftoffRegister dst,LiftoffRegister lhs,uint8_t imm_lane_idx)3324 void LiftoffAssembler::emit_i8x16_extract_lane_u(LiftoffRegister dst,
3325 LiftoffRegister lhs,
3326 uint8_t imm_lane_idx) {
3327 copy_u_b(dst.gp(), lhs.fp().toW(), imm_lane_idx);
3328 }
3329
emit_i16x8_extract_lane_s(LiftoffRegister dst,LiftoffRegister lhs,uint8_t imm_lane_idx)3330 void LiftoffAssembler::emit_i16x8_extract_lane_s(LiftoffRegister dst,
3331 LiftoffRegister lhs,
3332 uint8_t imm_lane_idx) {
3333 copy_s_h(dst.gp(), lhs.fp().toW(), imm_lane_idx);
3334 }
3335
emit_i16x8_extract_lane_u(LiftoffRegister dst,LiftoffRegister lhs,uint8_t imm_lane_idx)3336 void LiftoffAssembler::emit_i16x8_extract_lane_u(LiftoffRegister dst,
3337 LiftoffRegister lhs,
3338 uint8_t imm_lane_idx) {
3339 copy_u_h(dst.gp(), lhs.fp().toW(), imm_lane_idx);
3340 }
3341
emit_i32x4_extract_lane(LiftoffRegister dst,LiftoffRegister lhs,uint8_t imm_lane_idx)3342 void LiftoffAssembler::emit_i32x4_extract_lane(LiftoffRegister dst,
3343 LiftoffRegister lhs,
3344 uint8_t imm_lane_idx) {
3345 copy_s_w(dst.gp(), lhs.fp().toW(), imm_lane_idx);
3346 }
3347
emit_i64x2_extract_lane(LiftoffRegister dst,LiftoffRegister lhs,uint8_t imm_lane_idx)3348 void LiftoffAssembler::emit_i64x2_extract_lane(LiftoffRegister dst,
3349 LiftoffRegister lhs,
3350 uint8_t imm_lane_idx) {
3351 copy_s_d(dst.gp(), lhs.fp().toW(), imm_lane_idx);
3352 }
3353
emit_f32x4_extract_lane(LiftoffRegister dst,LiftoffRegister lhs,uint8_t imm_lane_idx)3354 void LiftoffAssembler::emit_f32x4_extract_lane(LiftoffRegister dst,
3355 LiftoffRegister lhs,
3356 uint8_t imm_lane_idx) {
3357 copy_u_w(kScratchReg, lhs.fp().toW(), imm_lane_idx);
3358 TurboAssembler::FmoveLow(dst.fp(), kScratchReg);
3359 }
3360
emit_f64x2_extract_lane(LiftoffRegister dst,LiftoffRegister lhs,uint8_t imm_lane_idx)3361 void LiftoffAssembler::emit_f64x2_extract_lane(LiftoffRegister dst,
3362 LiftoffRegister lhs,
3363 uint8_t imm_lane_idx) {
3364 copy_s_d(kScratchReg, lhs.fp().toW(), imm_lane_idx);
3365 TurboAssembler::Move(dst.fp(), kScratchReg);
3366 }
3367
emit_i8x16_replace_lane(LiftoffRegister dst,LiftoffRegister src1,LiftoffRegister src2,uint8_t imm_lane_idx)3368 void LiftoffAssembler::emit_i8x16_replace_lane(LiftoffRegister dst,
3369 LiftoffRegister src1,
3370 LiftoffRegister src2,
3371 uint8_t imm_lane_idx) {
3372 if (dst != src1) {
3373 move_v(dst.fp().toW(), src1.fp().toW());
3374 }
3375 insert_b(dst.fp().toW(), imm_lane_idx, src2.gp());
3376 }
3377
emit_i16x8_replace_lane(LiftoffRegister dst,LiftoffRegister src1,LiftoffRegister src2,uint8_t imm_lane_idx)3378 void LiftoffAssembler::emit_i16x8_replace_lane(LiftoffRegister dst,
3379 LiftoffRegister src1,
3380 LiftoffRegister src2,
3381 uint8_t imm_lane_idx) {
3382 if (dst != src1) {
3383 move_v(dst.fp().toW(), src1.fp().toW());
3384 }
3385 insert_h(dst.fp().toW(), imm_lane_idx, src2.gp());
3386 }
3387
emit_i32x4_replace_lane(LiftoffRegister dst,LiftoffRegister src1,LiftoffRegister src2,uint8_t imm_lane_idx)3388 void LiftoffAssembler::emit_i32x4_replace_lane(LiftoffRegister dst,
3389 LiftoffRegister src1,
3390 LiftoffRegister src2,
3391 uint8_t imm_lane_idx) {
3392 if (dst != src1) {
3393 move_v(dst.fp().toW(), src1.fp().toW());
3394 }
3395 insert_w(dst.fp().toW(), imm_lane_idx, src2.gp());
3396 }
3397
emit_i64x2_replace_lane(LiftoffRegister dst,LiftoffRegister src1,LiftoffRegister src2,uint8_t imm_lane_idx)3398 void LiftoffAssembler::emit_i64x2_replace_lane(LiftoffRegister dst,
3399 LiftoffRegister src1,
3400 LiftoffRegister src2,
3401 uint8_t imm_lane_idx) {
3402 if (dst != src1) {
3403 move_v(dst.fp().toW(), src1.fp().toW());
3404 }
3405 insert_d(dst.fp().toW(), imm_lane_idx, src2.gp());
3406 }
3407
emit_f32x4_replace_lane(LiftoffRegister dst,LiftoffRegister src1,LiftoffRegister src2,uint8_t imm_lane_idx)3408 void LiftoffAssembler::emit_f32x4_replace_lane(LiftoffRegister dst,
3409 LiftoffRegister src1,
3410 LiftoffRegister src2,
3411 uint8_t imm_lane_idx) {
3412 TurboAssembler::FmoveLow(kScratchReg, src2.fp());
3413 if (dst != src1) {
3414 move_v(dst.fp().toW(), src1.fp().toW());
3415 }
3416 insert_w(dst.fp().toW(), imm_lane_idx, kScratchReg);
3417 }
3418
emit_f64x2_replace_lane(LiftoffRegister dst,LiftoffRegister src1,LiftoffRegister src2,uint8_t imm_lane_idx)3419 void LiftoffAssembler::emit_f64x2_replace_lane(LiftoffRegister dst,
3420 LiftoffRegister src1,
3421 LiftoffRegister src2,
3422 uint8_t imm_lane_idx) {
3423 TurboAssembler::Move(kScratchReg, src2.fp());
3424 if (dst != src1) {
3425 move_v(dst.fp().toW(), src1.fp().toW());
3426 }
3427 insert_d(dst.fp().toW(), imm_lane_idx, kScratchReg);
3428 }
3429
StackCheck(Label * ool_code,Register limit_address)3430 void LiftoffAssembler::StackCheck(Label* ool_code, Register limit_address) {
3431 TurboAssembler::Uld(limit_address, MemOperand(limit_address));
3432 TurboAssembler::Branch(ool_code, ule, sp, Operand(limit_address));
3433 }
3434
CallTrapCallbackForTesting()3435 void LiftoffAssembler::CallTrapCallbackForTesting() {
3436 PrepareCallCFunction(0, GetUnusedRegister(kGpReg, {}).gp());
3437 CallCFunction(ExternalReference::wasm_call_trap_callback_for_testing(), 0);
3438 }
3439
AssertUnreachable(AbortReason reason)3440 void LiftoffAssembler::AssertUnreachable(AbortReason reason) {
3441 if (FLAG_debug_code) Abort(reason);
3442 }
3443
PushRegisters(LiftoffRegList regs)3444 void LiftoffAssembler::PushRegisters(LiftoffRegList regs) {
3445 LiftoffRegList gp_regs = regs & kGpCacheRegList;
3446 unsigned num_gp_regs = gp_regs.GetNumRegsSet();
3447 if (num_gp_regs) {
3448 unsigned offset = num_gp_regs * kSystemPointerSize;
3449 daddiu(sp, sp, -offset);
3450 while (!gp_regs.is_empty()) {
3451 LiftoffRegister reg = gp_regs.GetFirstRegSet();
3452 offset -= kSystemPointerSize;
3453 sd(reg.gp(), MemOperand(sp, offset));
3454 gp_regs.clear(reg);
3455 }
3456 DCHECK_EQ(offset, 0);
3457 }
3458 LiftoffRegList fp_regs = regs & kFpCacheRegList;
3459 unsigned num_fp_regs = fp_regs.GetNumRegsSet();
3460 if (num_fp_regs) {
3461 unsigned slot_size = IsEnabled(MIPS_SIMD) ? 16 : 8;
3462 daddiu(sp, sp, -(num_fp_regs * slot_size));
3463 unsigned offset = 0;
3464 while (!fp_regs.is_empty()) {
3465 LiftoffRegister reg = fp_regs.GetFirstRegSet();
3466 if (IsEnabled(MIPS_SIMD)) {
3467 TurboAssembler::st_d(reg.fp().toW(), MemOperand(sp, offset));
3468 } else {
3469 TurboAssembler::Sdc1(reg.fp(), MemOperand(sp, offset));
3470 }
3471 fp_regs.clear(reg);
3472 offset += slot_size;
3473 }
3474 DCHECK_EQ(offset, num_fp_regs * slot_size);
3475 }
3476 }
3477
PopRegisters(LiftoffRegList regs)3478 void LiftoffAssembler::PopRegisters(LiftoffRegList regs) {
3479 LiftoffRegList fp_regs = regs & kFpCacheRegList;
3480 unsigned fp_offset = 0;
3481 while (!fp_regs.is_empty()) {
3482 LiftoffRegister reg = fp_regs.GetFirstRegSet();
3483 if (IsEnabled(MIPS_SIMD)) {
3484 TurboAssembler::ld_d(reg.fp().toW(), MemOperand(sp, fp_offset));
3485 } else {
3486 TurboAssembler::Ldc1(reg.fp(), MemOperand(sp, fp_offset));
3487 }
3488 fp_regs.clear(reg);
3489 fp_offset += (IsEnabled(MIPS_SIMD) ? 16 : 8);
3490 }
3491 if (fp_offset) daddiu(sp, sp, fp_offset);
3492 LiftoffRegList gp_regs = regs & kGpCacheRegList;
3493 unsigned gp_offset = 0;
3494 while (!gp_regs.is_empty()) {
3495 LiftoffRegister reg = gp_regs.GetLastRegSet();
3496 ld(reg.gp(), MemOperand(sp, gp_offset));
3497 gp_regs.clear(reg);
3498 gp_offset += kSystemPointerSize;
3499 }
3500 daddiu(sp, sp, gp_offset);
3501 }
3502
RecordSpillsInSafepoint(SafepointTableBuilder::Safepoint & safepoint,LiftoffRegList all_spills,LiftoffRegList ref_spills,int spill_offset)3503 void LiftoffAssembler::RecordSpillsInSafepoint(
3504 SafepointTableBuilder::Safepoint& safepoint, LiftoffRegList all_spills,
3505 LiftoffRegList ref_spills, int spill_offset) {
3506 int spill_space_size = 0;
3507 while (!all_spills.is_empty()) {
3508 LiftoffRegister reg = all_spills.GetFirstRegSet();
3509 if (ref_spills.has(reg)) {
3510 safepoint.DefineTaggedStackSlot(spill_offset);
3511 }
3512 all_spills.clear(reg);
3513 ++spill_offset;
3514 spill_space_size += kSystemPointerSize;
3515 }
3516 // Record the number of additional spill slots.
3517 RecordOolSpillSpaceSize(spill_space_size);
3518 }
3519
DropStackSlotsAndRet(uint32_t num_stack_slots)3520 void LiftoffAssembler::DropStackSlotsAndRet(uint32_t num_stack_slots) {
3521 DCHECK_LT(num_stack_slots,
3522 (1 << 16) / kSystemPointerSize); // 16 bit immediate
3523 TurboAssembler::DropAndRet(static_cast<int>(num_stack_slots));
3524 }
3525
CallC(const ValueKindSig * sig,const LiftoffRegister * args,const LiftoffRegister * rets,ValueKind out_argument_kind,int stack_bytes,ExternalReference ext_ref)3526 void LiftoffAssembler::CallC(const ValueKindSig* sig,
3527 const LiftoffRegister* args,
3528 const LiftoffRegister* rets,
3529 ValueKind out_argument_kind, int stack_bytes,
3530 ExternalReference ext_ref) {
3531 Daddu(sp, sp, -stack_bytes);
3532
3533 int arg_bytes = 0;
3534 for (ValueKind param_kind : sig->parameters()) {
3535 liftoff::Store(this, sp, arg_bytes, *args++, param_kind);
3536 arg_bytes += value_kind_size(param_kind);
3537 }
3538 DCHECK_LE(arg_bytes, stack_bytes);
3539
3540 // Pass a pointer to the buffer with the arguments to the C function.
3541 // On mips, the first argument is passed in {a0}.
3542 constexpr Register kFirstArgReg = a0;
3543 mov(kFirstArgReg, sp);
3544
3545 // Now call the C function.
3546 constexpr int kNumCCallArgs = 1;
3547 PrepareCallCFunction(kNumCCallArgs, kScratchReg);
3548 CallCFunction(ext_ref, kNumCCallArgs);
3549
3550 // Move return value to the right register.
3551 const LiftoffRegister* next_result_reg = rets;
3552 if (sig->return_count() > 0) {
3553 DCHECK_EQ(1, sig->return_count());
3554 constexpr Register kReturnReg = v0;
3555 if (kReturnReg != next_result_reg->gp()) {
3556 Move(*next_result_reg, LiftoffRegister(kReturnReg), sig->GetReturn(0));
3557 }
3558 ++next_result_reg;
3559 }
3560
3561 // Load potential output value from the buffer on the stack.
3562 if (out_argument_kind != kVoid) {
3563 liftoff::Load(this, *next_result_reg, MemOperand(sp, 0), out_argument_kind);
3564 }
3565
3566 Daddu(sp, sp, stack_bytes);
3567 }
3568
CallNativeWasmCode(Address addr)3569 void LiftoffAssembler::CallNativeWasmCode(Address addr) {
3570 Call(addr, RelocInfo::WASM_CALL);
3571 }
3572
TailCallNativeWasmCode(Address addr)3573 void LiftoffAssembler::TailCallNativeWasmCode(Address addr) {
3574 Jump(addr, RelocInfo::WASM_CALL);
3575 }
3576
CallIndirect(const ValueKindSig * sig,compiler::CallDescriptor * call_descriptor,Register target)3577 void LiftoffAssembler::CallIndirect(const ValueKindSig* sig,
3578 compiler::CallDescriptor* call_descriptor,
3579 Register target) {
3580 if (target == no_reg) {
3581 pop(kScratchReg);
3582 Call(kScratchReg);
3583 } else {
3584 Call(target);
3585 }
3586 }
3587
TailCallIndirect(Register target)3588 void LiftoffAssembler::TailCallIndirect(Register target) {
3589 if (target == no_reg) {
3590 Pop(kScratchReg);
3591 Jump(kScratchReg);
3592 } else {
3593 Jump(target);
3594 }
3595 }
3596
CallRuntimeStub(WasmCode::RuntimeStubId sid)3597 void LiftoffAssembler::CallRuntimeStub(WasmCode::RuntimeStubId sid) {
3598 // A direct call to a wasm runtime stub defined in this module.
3599 // Just encode the stub index. This will be patched at relocation.
3600 Call(static_cast<Address>(sid), RelocInfo::WASM_STUB_CALL);
3601 }
3602
AllocateStackSlot(Register addr,uint32_t size)3603 void LiftoffAssembler::AllocateStackSlot(Register addr, uint32_t size) {
3604 Daddu(sp, sp, -size);
3605 TurboAssembler::Move(addr, sp);
3606 }
3607
DeallocateStackSlot(uint32_t size)3608 void LiftoffAssembler::DeallocateStackSlot(uint32_t size) {
3609 Daddu(sp, sp, size);
3610 }
3611
MaybeOSR()3612 void LiftoffAssembler::MaybeOSR() {}
3613
emit_set_if_nan(Register dst,FPURegister src,ValueKind kind)3614 void LiftoffAssembler::emit_set_if_nan(Register dst, FPURegister src,
3615 ValueKind kind) {
3616 UseScratchRegisterScope temps(this);
3617 Register scratch = temps.Acquire();
3618 Label not_nan;
3619 if (kind == kF32) {
3620 CompareIsNanF32(src, src);
3621 } else {
3622 DCHECK_EQ(kind, kF64);
3623 CompareIsNanF64(src, src);
3624 }
3625 BranchFalseShortF(¬_nan, USE_DELAY_SLOT);
3626 li(scratch, 1);
3627 Sw(dst, MemOperand(dst));
3628 bind(¬_nan);
3629 }
3630
emit_s128_set_if_nan(Register dst,LiftoffRegister src,Register tmp_gp,LiftoffRegister tmp_s128,ValueKind lane_kind)3631 void LiftoffAssembler::emit_s128_set_if_nan(Register dst, LiftoffRegister src,
3632 Register tmp_gp,
3633 LiftoffRegister tmp_s128,
3634 ValueKind lane_kind) {
3635 Label not_nan;
3636 if (lane_kind == kF32) {
3637 fcun_w(tmp_s128.fp().toW(), src.fp().toW(), src.fp().toW());
3638 } else {
3639 DCHECK_EQ(lane_kind, kF64);
3640 fcun_d(tmp_s128.fp().toW(), src.fp().toW(), src.fp().toW());
3641 }
3642 BranchMSA(¬_nan, MSA_BRANCH_V, all_zero, tmp_s128.fp().toW(),
3643 USE_DELAY_SLOT);
3644 li(tmp_gp, 1);
3645 Sw(tmp_gp, MemOperand(dst));
3646 bind(¬_nan);
3647 }
3648
Construct(int param_slots)3649 void LiftoffStackSlots::Construct(int param_slots) {
3650 DCHECK_LT(0, slots_.size());
3651 SortInPushOrder();
3652 int last_stack_slot = param_slots;
3653 for (auto& slot : slots_) {
3654 const int stack_slot = slot.dst_slot_;
3655 int stack_decrement = (last_stack_slot - stack_slot) * kSystemPointerSize;
3656 DCHECK_LT(0, stack_decrement);
3657 last_stack_slot = stack_slot;
3658 const LiftoffAssembler::VarState& src = slot.src_;
3659 switch (src.loc()) {
3660 case LiftoffAssembler::VarState::kStack:
3661 if (src.kind() != kS128) {
3662 asm_->AllocateStackSpace(stack_decrement - kSystemPointerSize);
3663 asm_->Ld(kScratchReg, liftoff::GetStackSlot(slot.src_offset_));
3664 asm_->push(kScratchReg);
3665 } else {
3666 asm_->AllocateStackSpace(stack_decrement - kSimd128Size);
3667 asm_->Ld(kScratchReg, liftoff::GetStackSlot(slot.src_offset_ - 8));
3668 asm_->push(kScratchReg);
3669 asm_->Ld(kScratchReg, liftoff::GetStackSlot(slot.src_offset_));
3670 asm_->push(kScratchReg);
3671 }
3672 break;
3673 case LiftoffAssembler::VarState::kRegister: {
3674 int pushed_bytes = SlotSizeInBytes(slot);
3675 asm_->AllocateStackSpace(stack_decrement - pushed_bytes);
3676 liftoff::push(asm_, src.reg(), src.kind());
3677 break;
3678 }
3679 case LiftoffAssembler::VarState::kIntConst: {
3680 asm_->AllocateStackSpace(stack_decrement - kSystemPointerSize);
3681 asm_->li(kScratchReg, Operand(src.i32_const()));
3682 asm_->push(kScratchReg);
3683 break;
3684 }
3685 }
3686 }
3687 }
3688
3689 } // namespace wasm
3690 } // namespace internal
3691 } // namespace v8
3692
3693 #endif // V8_WASM_BASELINE_MIPS64_LIFTOFF_ASSEMBLER_MIPS64_H_
3694