1 // Copyright 2013 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #ifndef V8_ARM64_ASSEMBLER_ARM64_INL_H_
6 #define V8_ARM64_ASSEMBLER_ARM64_INL_H_
7
8 #include "src/arm64/assembler-arm64.h"
9 #include "src/assembler.h"
10 #include "src/debug/debug.h"
11 #include "src/objects-inl.h"
12
13 namespace v8 {
14 namespace internal {
15
SupportsOptimizer()16 bool CpuFeatures::SupportsOptimizer() { return true; }
17
SupportsWasmSimd128()18 bool CpuFeatures::SupportsWasmSimd128() { return true; }
19
apply(intptr_t delta)20 void RelocInfo::apply(intptr_t delta) {
21 // On arm64 only internal references and immediate branches need extra work.
22 if (RelocInfo::IsInternalReference(rmode_)) {
23 // Absolute code pointer inside code object moves with the code object.
24 intptr_t* p = reinterpret_cast<intptr_t*>(pc_);
25 *p += delta; // Relocate entry.
26 } else {
27 Instruction* instr = reinterpret_cast<Instruction*>(pc_);
28 if (instr->IsBranchAndLink() || instr->IsUnconditionalBranch()) {
29 Address old_target =
30 reinterpret_cast<Address>(instr->ImmPCOffsetTarget());
31 Address new_target = old_target - delta;
32 instr->SetBranchImmTarget(reinterpret_cast<Instruction*>(new_target));
33 }
34 }
35 }
36
37
IsSameSizeAndType(const CPURegister & other)38 inline bool CPURegister::IsSameSizeAndType(const CPURegister& other) const {
39 return (reg_size_ == other.reg_size_) && (reg_type_ == other.reg_type_);
40 }
41
42
IsZero()43 inline bool CPURegister::IsZero() const {
44 DCHECK(IsValid());
45 return IsRegister() && (reg_code_ == kZeroRegCode);
46 }
47
48
IsSP()49 inline bool CPURegister::IsSP() const {
50 DCHECK(IsValid());
51 return IsRegister() && (reg_code_ == kSPRegInternalCode);
52 }
53
54
Combine(const CPURegList & other)55 inline void CPURegList::Combine(const CPURegList& other) {
56 DCHECK(IsValid());
57 DCHECK(other.type() == type_);
58 DCHECK(other.RegisterSizeInBits() == size_);
59 list_ |= other.list();
60 }
61
62
Remove(const CPURegList & other)63 inline void CPURegList::Remove(const CPURegList& other) {
64 DCHECK(IsValid());
65 if (other.type() == type_) {
66 list_ &= ~other.list();
67 }
68 }
69
70
Combine(const CPURegister & other)71 inline void CPURegList::Combine(const CPURegister& other) {
72 DCHECK(other.type() == type_);
73 DCHECK(other.SizeInBits() == size_);
74 Combine(other.code());
75 }
76
77
Remove(const CPURegister & other1,const CPURegister & other2,const CPURegister & other3,const CPURegister & other4)78 inline void CPURegList::Remove(const CPURegister& other1,
79 const CPURegister& other2,
80 const CPURegister& other3,
81 const CPURegister& other4) {
82 if (!other1.IsNone() && (other1.type() == type_)) Remove(other1.code());
83 if (!other2.IsNone() && (other2.type() == type_)) Remove(other2.code());
84 if (!other3.IsNone() && (other3.type() == type_)) Remove(other3.code());
85 if (!other4.IsNone() && (other4.type() == type_)) Remove(other4.code());
86 }
87
88
Combine(int code)89 inline void CPURegList::Combine(int code) {
90 DCHECK(IsValid());
91 DCHECK(CPURegister::Create(code, size_, type_).IsValid());
92 list_ |= (1UL << code);
93 }
94
95
Remove(int code)96 inline void CPURegList::Remove(int code) {
97 DCHECK(IsValid());
98 DCHECK(CPURegister::Create(code, size_, type_).IsValid());
99 list_ &= ~(1UL << code);
100 }
101
102
XRegFromCode(unsigned code)103 inline Register Register::XRegFromCode(unsigned code) {
104 if (code == kSPRegInternalCode) {
105 return sp;
106 } else {
107 DCHECK_LT(code, static_cast<unsigned>(kNumberOfRegisters));
108 return Register::Create(code, kXRegSizeInBits);
109 }
110 }
111
112
WRegFromCode(unsigned code)113 inline Register Register::WRegFromCode(unsigned code) {
114 if (code == kSPRegInternalCode) {
115 return wsp;
116 } else {
117 DCHECK_LT(code, static_cast<unsigned>(kNumberOfRegisters));
118 return Register::Create(code, kWRegSizeInBits);
119 }
120 }
121
BRegFromCode(unsigned code)122 inline VRegister VRegister::BRegFromCode(unsigned code) {
123 DCHECK_LT(code, static_cast<unsigned>(kNumberOfVRegisters));
124 return VRegister::Create(code, kBRegSizeInBits);
125 }
126
HRegFromCode(unsigned code)127 inline VRegister VRegister::HRegFromCode(unsigned code) {
128 DCHECK_LT(code, static_cast<unsigned>(kNumberOfVRegisters));
129 return VRegister::Create(code, kHRegSizeInBits);
130 }
131
SRegFromCode(unsigned code)132 inline VRegister VRegister::SRegFromCode(unsigned code) {
133 DCHECK_LT(code, static_cast<unsigned>(kNumberOfVRegisters));
134 return VRegister::Create(code, kSRegSizeInBits);
135 }
136
DRegFromCode(unsigned code)137 inline VRegister VRegister::DRegFromCode(unsigned code) {
138 DCHECK_LT(code, static_cast<unsigned>(kNumberOfVRegisters));
139 return VRegister::Create(code, kDRegSizeInBits);
140 }
141
QRegFromCode(unsigned code)142 inline VRegister VRegister::QRegFromCode(unsigned code) {
143 DCHECK_LT(code, static_cast<unsigned>(kNumberOfVRegisters));
144 return VRegister::Create(code, kQRegSizeInBits);
145 }
146
VRegFromCode(unsigned code)147 inline VRegister VRegister::VRegFromCode(unsigned code) {
148 DCHECK_LT(code, static_cast<unsigned>(kNumberOfVRegisters));
149 return VRegister::Create(code, kVRegSizeInBits);
150 }
151
W()152 inline Register CPURegister::W() const {
153 DCHECK(IsRegister());
154 return Register::WRegFromCode(reg_code_);
155 }
156
Reg()157 inline Register CPURegister::Reg() const {
158 DCHECK(IsRegister());
159 return Register::Create(reg_code_, reg_size_);
160 }
161
VReg()162 inline VRegister CPURegister::VReg() const {
163 DCHECK(IsVRegister());
164 return VRegister::Create(reg_code_, reg_size_);
165 }
166
X()167 inline Register CPURegister::X() const {
168 DCHECK(IsRegister());
169 return Register::XRegFromCode(reg_code_);
170 }
171
V()172 inline VRegister CPURegister::V() const {
173 DCHECK(IsVRegister());
174 return VRegister::VRegFromCode(reg_code_);
175 }
176
B()177 inline VRegister CPURegister::B() const {
178 DCHECK(IsVRegister());
179 return VRegister::BRegFromCode(reg_code_);
180 }
181
H()182 inline VRegister CPURegister::H() const {
183 DCHECK(IsVRegister());
184 return VRegister::HRegFromCode(reg_code_);
185 }
186
S()187 inline VRegister CPURegister::S() const {
188 DCHECK(IsVRegister());
189 return VRegister::SRegFromCode(reg_code_);
190 }
191
D()192 inline VRegister CPURegister::D() const {
193 DCHECK(IsVRegister());
194 return VRegister::DRegFromCode(reg_code_);
195 }
196
Q()197 inline VRegister CPURegister::Q() const {
198 DCHECK(IsVRegister());
199 return VRegister::QRegFromCode(reg_code_);
200 }
201
202
203 // Immediate.
204 // Default initializer is for int types
205 template<typename T>
206 struct ImmediateInitializer {
207 static const bool kIsIntType = true;
rmode_forImmediateInitializer208 static inline RelocInfo::Mode rmode_for(T) { return RelocInfo::NONE; }
immediate_forImmediateInitializer209 static inline int64_t immediate_for(T t) {
210 STATIC_ASSERT(sizeof(T) <= 8);
211 return t;
212 }
213 };
214
215
216 template<>
217 struct ImmediateInitializer<Smi*> {
218 static const bool kIsIntType = false;
219 static inline RelocInfo::Mode rmode_for(Smi* t) { return RelocInfo::NONE; }
220 static inline int64_t immediate_for(Smi* t) {;
221 return reinterpret_cast<int64_t>(t);
222 }
223 };
224
225
226 template<>
227 struct ImmediateInitializer<ExternalReference> {
228 static const bool kIsIntType = false;
229 static inline RelocInfo::Mode rmode_for(ExternalReference t) {
230 return RelocInfo::EXTERNAL_REFERENCE;
231 }
232 static inline int64_t immediate_for(ExternalReference t) {;
233 return static_cast<int64_t>(t.address());
234 }
235 };
236
237
238 template<typename T>
239 Immediate::Immediate(Handle<T> value) {
240 InitializeHandle(value);
241 }
242
243
244 template<typename T>
245 Immediate::Immediate(T t)
246 : value_(ImmediateInitializer<T>::immediate_for(t)),
247 rmode_(ImmediateInitializer<T>::rmode_for(t)) {}
248
249
250 template<typename T>
251 Immediate::Immediate(T t, RelocInfo::Mode rmode)
252 : value_(ImmediateInitializer<T>::immediate_for(t)),
253 rmode_(rmode) {
254 STATIC_ASSERT(ImmediateInitializer<T>::kIsIntType);
255 }
256
257 // Operand.
258 template<typename T>
259 Operand::Operand(Handle<T> value) : immediate_(value), reg_(NoReg) {}
260
261
262 template<typename T>
263 Operand::Operand(T t) : immediate_(t), reg_(NoReg) {}
264
265
266 template<typename T>
267 Operand::Operand(T t, RelocInfo::Mode rmode)
268 : immediate_(t, rmode),
269 reg_(NoReg) {}
270
271 Operand::Operand(Register reg, Shift shift, unsigned shift_amount)
272 : immediate_(0),
273 reg_(reg),
274 shift_(shift),
275 extend_(NO_EXTEND),
276 shift_amount_(shift_amount) {
277 DCHECK(reg.Is64Bits() || (shift_amount < kWRegSizeInBits));
278 DCHECK(reg.Is32Bits() || (shift_amount < kXRegSizeInBits));
279 DCHECK_IMPLIES(reg.IsSP(), shift_amount == 0);
280 }
281
282
283 Operand::Operand(Register reg, Extend extend, unsigned shift_amount)
284 : immediate_(0),
285 reg_(reg),
286 shift_(NO_SHIFT),
287 extend_(extend),
288 shift_amount_(shift_amount) {
289 DCHECK(reg.IsValid());
290 DCHECK_LE(shift_amount, 4);
291 DCHECK(!reg.IsSP());
292
293 // Extend modes SXTX and UXTX require a 64-bit register.
294 DCHECK(reg.Is64Bits() || ((extend != SXTX) && (extend != UXTX)));
295 }
296
297 bool Operand::IsHeapObjectRequest() const {
298 DCHECK_IMPLIES(heap_object_request_.has_value(), reg_.Is(NoReg));
299 DCHECK_IMPLIES(heap_object_request_.has_value(),
300 immediate_.rmode() == RelocInfo::EMBEDDED_OBJECT ||
301 immediate_.rmode() == RelocInfo::CODE_TARGET);
302 return heap_object_request_.has_value();
303 }
304
305 HeapObjectRequest Operand::heap_object_request() const {
306 DCHECK(IsHeapObjectRequest());
307 return *heap_object_request_;
308 }
309
310 bool Operand::IsImmediate() const {
311 return reg_.Is(NoReg) && !IsHeapObjectRequest();
312 }
313
314
315 bool Operand::IsShiftedRegister() const {
316 return reg_.IsValid() && (shift_ != NO_SHIFT);
317 }
318
319
320 bool Operand::IsExtendedRegister() const {
321 return reg_.IsValid() && (extend_ != NO_EXTEND);
322 }
323
324
325 bool Operand::IsZero() const {
326 if (IsImmediate()) {
327 return ImmediateValue() == 0;
328 } else {
329 return reg().IsZero();
330 }
331 }
332
333
334 Operand Operand::ToExtendedRegister() const {
335 DCHECK(IsShiftedRegister());
336 DCHECK((shift_ == LSL) && (shift_amount_ <= 4));
337 return Operand(reg_, reg_.Is64Bits() ? UXTX : UXTW, shift_amount_);
338 }
339
340 Immediate Operand::immediate_for_heap_object_request() const {
341 DCHECK((heap_object_request().kind() == HeapObjectRequest::kHeapNumber &&
342 immediate_.rmode() == RelocInfo::EMBEDDED_OBJECT) ||
343 (heap_object_request().kind() == HeapObjectRequest::kCodeStub &&
344 immediate_.rmode() == RelocInfo::CODE_TARGET));
345 return immediate_;
346 }
347
348 Immediate Operand::immediate() const {
349 DCHECK(IsImmediate());
350 return immediate_;
351 }
352
353
354 int64_t Operand::ImmediateValue() const {
355 DCHECK(IsImmediate());
356 return immediate_.value();
357 }
358
359 RelocInfo::Mode Operand::ImmediateRMode() const {
360 DCHECK(IsImmediate() || IsHeapObjectRequest());
361 return immediate_.rmode();
362 }
363
364 Register Operand::reg() const {
365 DCHECK(IsShiftedRegister() || IsExtendedRegister());
366 return reg_;
367 }
368
369
370 Shift Operand::shift() const {
371 DCHECK(IsShiftedRegister());
372 return shift_;
373 }
374
375
376 Extend Operand::extend() const {
377 DCHECK(IsExtendedRegister());
378 return extend_;
379 }
380
381
382 unsigned Operand::shift_amount() const {
383 DCHECK(IsShiftedRegister() || IsExtendedRegister());
384 return shift_amount_;
385 }
386
387
388 Operand Operand::UntagSmi(Register smi) {
389 DCHECK(smi.Is64Bits());
390 DCHECK(SmiValuesAre32Bits() || SmiValuesAre31Bits());
391 return Operand(smi, ASR, kSmiShift);
392 }
393
394
395 Operand Operand::UntagSmiAndScale(Register smi, int scale) {
396 DCHECK(smi.Is64Bits());
397 DCHECK((scale >= 0) && (scale <= (64 - kSmiValueSize)));
398 DCHECK(SmiValuesAre32Bits() || SmiValuesAre31Bits());
399 if (scale > kSmiShift) {
400 return Operand(smi, LSL, scale - kSmiShift);
401 } else if (scale < kSmiShift) {
402 return Operand(smi, ASR, kSmiShift - scale);
403 }
404 return Operand(smi);
405 }
406
407
408 MemOperand::MemOperand()
409 : base_(NoReg), regoffset_(NoReg), offset_(0), addrmode_(Offset),
410 shift_(NO_SHIFT), extend_(NO_EXTEND), shift_amount_(0) {
411 }
412
413
414 MemOperand::MemOperand(Register base, int64_t offset, AddrMode addrmode)
415 : base_(base), regoffset_(NoReg), offset_(offset), addrmode_(addrmode),
416 shift_(NO_SHIFT), extend_(NO_EXTEND), shift_amount_(0) {
417 DCHECK(base.Is64Bits() && !base.IsZero());
418 }
419
420
421 MemOperand::MemOperand(Register base,
422 Register regoffset,
423 Extend extend,
424 unsigned shift_amount)
425 : base_(base), regoffset_(regoffset), offset_(0), addrmode_(Offset),
426 shift_(NO_SHIFT), extend_(extend), shift_amount_(shift_amount) {
427 DCHECK(base.Is64Bits() && !base.IsZero());
428 DCHECK(!regoffset.IsSP());
429 DCHECK((extend == UXTW) || (extend == SXTW) || (extend == SXTX));
430
431 // SXTX extend mode requires a 64-bit offset register.
432 DCHECK(regoffset.Is64Bits() || (extend != SXTX));
433 }
434
435
436 MemOperand::MemOperand(Register base,
437 Register regoffset,
438 Shift shift,
439 unsigned shift_amount)
440 : base_(base), regoffset_(regoffset), offset_(0), addrmode_(Offset),
441 shift_(shift), extend_(NO_EXTEND), shift_amount_(shift_amount) {
442 DCHECK(base.Is64Bits() && !base.IsZero());
443 DCHECK(regoffset.Is64Bits() && !regoffset.IsSP());
444 DCHECK(shift == LSL);
445 }
446
447 MemOperand::MemOperand(Register base, const Operand& offset, AddrMode addrmode)
448 : base_(base), regoffset_(NoReg), addrmode_(addrmode) {
449 DCHECK(base.Is64Bits() && !base.IsZero());
450
451 if (offset.IsImmediate()) {
452 offset_ = offset.ImmediateValue();
453 } else if (offset.IsShiftedRegister()) {
454 DCHECK((addrmode == Offset) || (addrmode == PostIndex));
455
456 regoffset_ = offset.reg();
457 shift_ = offset.shift();
458 shift_amount_ = offset.shift_amount();
459
460 extend_ = NO_EXTEND;
461 offset_ = 0;
462
463 // These assertions match those in the shifted-register constructor.
464 DCHECK(regoffset_.Is64Bits() && !regoffset_.IsSP());
465 DCHECK(shift_ == LSL);
466 } else {
467 DCHECK(offset.IsExtendedRegister());
468 DCHECK(addrmode == Offset);
469
470 regoffset_ = offset.reg();
471 extend_ = offset.extend();
472 shift_amount_ = offset.shift_amount();
473
474 shift_ = NO_SHIFT;
475 offset_ = 0;
476
477 // These assertions match those in the extended-register constructor.
478 DCHECK(!regoffset_.IsSP());
479 DCHECK((extend_ == UXTW) || (extend_ == SXTW) || (extend_ == SXTX));
480 DCHECK((regoffset_.Is64Bits() || (extend_ != SXTX)));
481 }
482 }
483
484 bool MemOperand::IsImmediateOffset() const {
485 return (addrmode_ == Offset) && regoffset_.Is(NoReg);
486 }
487
488
489 bool MemOperand::IsRegisterOffset() const {
490 return (addrmode_ == Offset) && !regoffset_.Is(NoReg);
491 }
492
493
494 bool MemOperand::IsPreIndex() const {
495 return addrmode_ == PreIndex;
496 }
497
498
499 bool MemOperand::IsPostIndex() const {
500 return addrmode_ == PostIndex;
501 }
502
503 Operand MemOperand::OffsetAsOperand() const {
504 if (IsImmediateOffset()) {
505 return offset();
506 } else {
507 DCHECK(IsRegisterOffset());
508 if (extend() == NO_EXTEND) {
509 return Operand(regoffset(), shift(), shift_amount());
510 } else {
511 return Operand(regoffset(), extend(), shift_amount());
512 }
513 }
514 }
515
516
517 void Assembler::Unreachable() {
518 #ifdef USE_SIMULATOR
519 debug("UNREACHABLE", __LINE__, BREAK);
520 #else
521 // Crash by branching to 0. lr now points near the fault.
522 Emit(BLR | Rn(xzr));
523 #endif
524 }
525
526
527 Address Assembler::target_pointer_address_at(Address pc) {
528 Instruction* instr = reinterpret_cast<Instruction*>(pc);
529 DCHECK(instr->IsLdrLiteralX());
530 return reinterpret_cast<Address>(instr->ImmPCOffsetTarget());
531 }
532
533
534 // Read/Modify the code target address in the branch/call instruction at pc.
535 Address Assembler::target_address_at(Address pc, Address constant_pool) {
536 Instruction* instr = reinterpret_cast<Instruction*>(pc);
537 if (instr->IsLdrLiteralX()) {
538 return Memory<Address>(target_pointer_address_at(pc));
539 } else {
540 DCHECK(instr->IsBranchAndLink() || instr->IsUnconditionalBranch());
541 return reinterpret_cast<Address>(instr->ImmPCOffsetTarget());
542 }
543 }
544
545 Handle<Code> Assembler::code_target_object_handle_at(Address pc) {
546 Instruction* instr = reinterpret_cast<Instruction*>(pc);
547 if (instr->IsLdrLiteralX()) {
548 return Handle<Code>(reinterpret_cast<Code**>(
549 Assembler::target_address_at(pc, 0 /* unused */)));
550 } else {
551 DCHECK(instr->IsBranchAndLink() || instr->IsUnconditionalBranch());
552 DCHECK_EQ(instr->ImmPCOffset() % kInstrSize, 0);
553 return GetCodeTarget(instr->ImmPCOffset() >> kInstrSizeLog2);
554 }
555 }
556
557 Address Assembler::runtime_entry_at(Address pc) {
558 Instruction* instr = reinterpret_cast<Instruction*>(pc);
559 if (instr->IsLdrLiteralX()) {
560 return Assembler::target_address_at(pc, 0 /* unused */);
561 } else {
562 DCHECK(instr->IsBranchAndLink() || instr->IsUnconditionalBranch());
563 return instr->ImmPCOffset() + options().code_range_start;
564 }
565 }
566
567 Address Assembler::target_address_from_return_address(Address pc) {
568 // Returns the address of the call target from the return address that will
569 // be returned to after a call.
570 // Call sequence on ARM64 is:
571 // ldr ip0, #... @ load from literal pool
572 // blr ip0
573 Address candidate = pc - 2 * kInstrSize;
574 Instruction* instr = reinterpret_cast<Instruction*>(candidate);
575 USE(instr);
576 DCHECK(instr->IsLdrLiteralX());
577 return candidate;
578 }
579
580 int Assembler::deserialization_special_target_size(Address location) {
581 Instruction* instr = reinterpret_cast<Instruction*>(location);
582 if (instr->IsBranchAndLink() || instr->IsUnconditionalBranch()) {
583 return kSpecialTargetSize;
584 } else {
585 DCHECK_EQ(instr->InstructionBits(), 0);
586 return kPointerSize;
587 }
588 }
589
590 void Assembler::deserialization_set_special_target_at(Address location,
591 Code* code,
592 Address target) {
593 Instruction* instr = reinterpret_cast<Instruction*>(location);
594 if (instr->IsBranchAndLink() || instr->IsUnconditionalBranch()) {
595 if (target == 0) {
596 // We are simply wiping the target out for serialization. Set the offset
597 // to zero instead.
598 target = location;
599 }
600 instr->SetBranchImmTarget(reinterpret_cast<Instruction*>(target));
601 Assembler::FlushICache(location, kInstrSize);
602 } else {
603 DCHECK_EQ(instr->InstructionBits(), 0);
604 Memory<Address>(location) = target;
605 // Intuitively, we would think it is necessary to always flush the
606 // instruction cache after patching a target address in the code. However,
607 // in this case, only the constant pool contents change. The instruction
608 // accessing the constant pool remains unchanged, so a flush is not
609 // required.
610 }
611 }
612
613 void Assembler::deserialization_set_target_internal_reference_at(
614 Address pc, Address target, RelocInfo::Mode mode) {
615 Memory<Address>(pc) = target;
616 }
617
618 void Assembler::set_target_address_at(Address pc, Address constant_pool,
619 Address target,
620 ICacheFlushMode icache_flush_mode) {
621 Instruction* instr = reinterpret_cast<Instruction*>(pc);
622 if (instr->IsLdrLiteralX()) {
623 Memory<Address>(target_pointer_address_at(pc)) = target;
624 // Intuitively, we would think it is necessary to always flush the
625 // instruction cache after patching a target address in the code. However,
626 // in this case, only the constant pool contents change. The instruction
627 // accessing the constant pool remains unchanged, so a flush is not
628 // required.
629 } else {
630 DCHECK(instr->IsBranchAndLink() || instr->IsUnconditionalBranch());
631 if (target == 0) {
632 // We are simply wiping the target out for serialization. Set the offset
633 // to zero instead.
634 target = pc;
635 }
636 instr->SetBranchImmTarget(reinterpret_cast<Instruction*>(target));
637 if (icache_flush_mode != SKIP_ICACHE_FLUSH) {
638 Assembler::FlushICache(pc, kInstrSize);
639 }
640 }
641 }
642
643 int RelocInfo::target_address_size() {
644 if (IsCodedSpecially()) {
645 return Assembler::kSpecialTargetSize;
646 } else {
647 DCHECK(reinterpret_cast<Instruction*>(pc_)->IsLdrLiteralX());
648 return kPointerSize;
649 }
650 }
651
652
653 Address RelocInfo::target_address() {
654 DCHECK(IsCodeTarget(rmode_) || IsRuntimeEntry(rmode_) || IsWasmCall(rmode_));
655 return Assembler::target_address_at(pc_, constant_pool_);
656 }
657
658 Address RelocInfo::target_address_address() {
659 DCHECK(IsCodeTarget(rmode_) || IsRuntimeEntry(rmode_) || IsWasmCall(rmode_) ||
660 IsEmbeddedObject(rmode_) || IsExternalReference(rmode_) ||
661 IsOffHeapTarget(rmode_));
662 Instruction* instr = reinterpret_cast<Instruction*>(pc_);
663 // Read the address of the word containing the target_address in an
664 // instruction stream.
665 // The only architecture-independent user of this function is the serializer.
666 // The serializer uses it to find out how many raw bytes of instruction to
667 // output before the next target.
668 // For an instruction like B/BL, where the target bits are mixed into the
669 // instruction bits, the size of the target will be zero, indicating that the
670 // serializer should not step forward in memory after a target is resolved
671 // and written.
672 // For LDR literal instructions, we can skip up to the constant pool entry
673 // address. We make sure that RelocInfo is ordered by the
674 // target_address_address so that we do not skip over any relocatable
675 // instruction sequences.
676 if (instr->IsLdrLiteralX()) {
677 return constant_pool_entry_address();
678 } else {
679 DCHECK(instr->IsBranchAndLink() || instr->IsUnconditionalBranch());
680 return reinterpret_cast<Address>(pc_);
681 }
682 }
683
684
685 Address RelocInfo::constant_pool_entry_address() {
686 DCHECK(IsInConstantPool());
687 return Assembler::target_pointer_address_at(pc_);
688 }
689
690 HeapObject* RelocInfo::target_object() {
691 DCHECK(IsCodeTarget(rmode_) || rmode_ == EMBEDDED_OBJECT);
692 return HeapObject::cast(reinterpret_cast<Object*>(
693 Assembler::target_address_at(pc_, constant_pool_)));
694 }
695
696 Handle<HeapObject> RelocInfo::target_object_handle(Assembler* origin) {
697 if (rmode_ == EMBEDDED_OBJECT) {
698 return Handle<HeapObject>(reinterpret_cast<HeapObject**>(
699 Assembler::target_address_at(pc_, constant_pool_)));
700 } else {
701 DCHECK(IsCodeTarget(rmode_));
702 return origin->code_target_object_handle_at(pc_);
703 }
704 }
705
706 void RelocInfo::set_target_object(Heap* heap, HeapObject* target,
707 WriteBarrierMode write_barrier_mode,
708 ICacheFlushMode icache_flush_mode) {
709 DCHECK(IsCodeTarget(rmode_) || rmode_ == EMBEDDED_OBJECT);
710 Assembler::set_target_address_at(pc_, constant_pool_,
711 reinterpret_cast<Address>(target),
712 icache_flush_mode);
713 if (write_barrier_mode == UPDATE_WRITE_BARRIER && host() != nullptr) {
714 WriteBarrierForCode(host(), this, target);
715 }
716 }
717
718
719 Address RelocInfo::target_external_reference() {
720 DCHECK(rmode_ == EXTERNAL_REFERENCE);
721 return Assembler::target_address_at(pc_, constant_pool_);
722 }
723
724 void RelocInfo::set_target_external_reference(
725 Address target, ICacheFlushMode icache_flush_mode) {
726 DCHECK(rmode_ == RelocInfo::EXTERNAL_REFERENCE);
727 Assembler::set_target_address_at(pc_, constant_pool_, target,
728 icache_flush_mode);
729 }
730
731 Address RelocInfo::target_internal_reference() {
732 DCHECK(rmode_ == INTERNAL_REFERENCE);
733 return Memory<Address>(pc_);
734 }
735
736
737 Address RelocInfo::target_internal_reference_address() {
738 DCHECK(rmode_ == INTERNAL_REFERENCE);
739 return pc_;
740 }
741
742 Address RelocInfo::target_runtime_entry(Assembler* origin) {
743 DCHECK(IsRuntimeEntry(rmode_));
744 return origin->runtime_entry_at(pc_);
745 }
746
747 void RelocInfo::set_target_runtime_entry(Address target,
748 WriteBarrierMode write_barrier_mode,
749 ICacheFlushMode icache_flush_mode) {
750 DCHECK(IsRuntimeEntry(rmode_));
751 if (target_address() != target) {
752 set_target_address(target, write_barrier_mode, icache_flush_mode);
753 }
754 }
755
756 Address RelocInfo::target_off_heap_target() {
757 DCHECK(IsOffHeapTarget(rmode_));
758 return Assembler::target_address_at(pc_, constant_pool_);
759 }
760
761 void RelocInfo::WipeOut() {
762 DCHECK(IsEmbeddedObject(rmode_) || IsCodeTarget(rmode_) ||
763 IsRuntimeEntry(rmode_) || IsExternalReference(rmode_) ||
764 IsInternalReference(rmode_) || IsOffHeapTarget(rmode_));
765 if (IsInternalReference(rmode_)) {
766 Memory<Address>(pc_) = kNullAddress;
767 } else {
768 Assembler::set_target_address_at(pc_, constant_pool_, kNullAddress);
769 }
770 }
771
772 template <typename ObjectVisitor>
773 void RelocInfo::Visit(ObjectVisitor* visitor) {
774 RelocInfo::Mode mode = rmode();
775 if (mode == RelocInfo::EMBEDDED_OBJECT) {
776 visitor->VisitEmbeddedPointer(host(), this);
777 } else if (RelocInfo::IsCodeTargetMode(mode)) {
778 visitor->VisitCodeTarget(host(), this);
779 } else if (mode == RelocInfo::EXTERNAL_REFERENCE) {
780 visitor->VisitExternalReference(host(), this);
781 } else if (mode == RelocInfo::INTERNAL_REFERENCE) {
782 visitor->VisitInternalReference(host(), this);
783 } else if (RelocInfo::IsRuntimeEntry(mode)) {
784 visitor->VisitRuntimeEntry(host(), this);
785 } else if (RelocInfo::IsOffHeapTarget(mode)) {
786 visitor->VisitOffHeapTarget(host(), this);
787 }
788 }
789
790 LoadStoreOp Assembler::LoadOpFor(const CPURegister& rt) {
791 DCHECK(rt.IsValid());
792 if (rt.IsRegister()) {
793 return rt.Is64Bits() ? LDR_x : LDR_w;
794 } else {
795 DCHECK(rt.IsVRegister());
796 switch (rt.SizeInBits()) {
797 case kBRegSizeInBits:
798 return LDR_b;
799 case kHRegSizeInBits:
800 return LDR_h;
801 case kSRegSizeInBits:
802 return LDR_s;
803 case kDRegSizeInBits:
804 return LDR_d;
805 default:
806 DCHECK(rt.IsQ());
807 return LDR_q;
808 }
809 }
810 }
811
812
813 LoadStoreOp Assembler::StoreOpFor(const CPURegister& rt) {
814 DCHECK(rt.IsValid());
815 if (rt.IsRegister()) {
816 return rt.Is64Bits() ? STR_x : STR_w;
817 } else {
818 DCHECK(rt.IsVRegister());
819 switch (rt.SizeInBits()) {
820 case kBRegSizeInBits:
821 return STR_b;
822 case kHRegSizeInBits:
823 return STR_h;
824 case kSRegSizeInBits:
825 return STR_s;
826 case kDRegSizeInBits:
827 return STR_d;
828 default:
829 DCHECK(rt.IsQ());
830 return STR_q;
831 }
832 }
833 }
834
835 LoadStorePairOp Assembler::LoadPairOpFor(const CPURegister& rt,
836 const CPURegister& rt2) {
837 DCHECK_EQ(STP_w | LoadStorePairLBit, LDP_w);
838 return static_cast<LoadStorePairOp>(StorePairOpFor(rt, rt2) |
839 LoadStorePairLBit);
840 }
841
842 LoadStorePairOp Assembler::StorePairOpFor(const CPURegister& rt,
843 const CPURegister& rt2) {
844 DCHECK(AreSameSizeAndType(rt, rt2));
845 USE(rt2);
846 if (rt.IsRegister()) {
847 return rt.Is64Bits() ? STP_x : STP_w;
848 } else {
849 DCHECK(rt.IsVRegister());
850 switch (rt.SizeInBits()) {
851 case kSRegSizeInBits:
852 return STP_s;
853 case kDRegSizeInBits:
854 return STP_d;
855 default:
856 DCHECK(rt.IsQ());
857 return STP_q;
858 }
859 }
860 }
861
862
863 LoadLiteralOp Assembler::LoadLiteralOpFor(const CPURegister& rt) {
864 if (rt.IsRegister()) {
865 return rt.Is64Bits() ? LDR_x_lit : LDR_w_lit;
866 } else {
867 DCHECK(rt.IsVRegister());
868 return rt.Is64Bits() ? LDR_d_lit : LDR_s_lit;
869 }
870 }
871
872
873 int Assembler::LinkAndGetInstructionOffsetTo(Label* label) {
874 DCHECK_EQ(kStartOfLabelLinkChain, 0);
875 int offset = LinkAndGetByteOffsetTo(label);
876 DCHECK(IsAligned(offset, kInstrSize));
877 return offset >> kInstrSizeLog2;
878 }
879
880
881 Instr Assembler::Flags(FlagsUpdate S) {
882 if (S == SetFlags) {
883 return 1 << FlagsUpdate_offset;
884 } else if (S == LeaveFlags) {
885 return 0 << FlagsUpdate_offset;
886 }
887 UNREACHABLE();
888 }
889
890
891 Instr Assembler::Cond(Condition cond) {
892 return cond << Condition_offset;
893 }
894
895
896 Instr Assembler::ImmPCRelAddress(int imm21) {
897 CHECK(is_int21(imm21));
898 Instr imm = static_cast<Instr>(truncate_to_int21(imm21));
899 Instr immhi = (imm >> ImmPCRelLo_width) << ImmPCRelHi_offset;
900 Instr immlo = imm << ImmPCRelLo_offset;
901 return (immhi & ImmPCRelHi_mask) | (immlo & ImmPCRelLo_mask);
902 }
903
904
905 Instr Assembler::ImmUncondBranch(int imm26) {
906 CHECK(is_int26(imm26));
907 return truncate_to_int26(imm26) << ImmUncondBranch_offset;
908 }
909
910
911 Instr Assembler::ImmCondBranch(int imm19) {
912 CHECK(is_int19(imm19));
913 return truncate_to_int19(imm19) << ImmCondBranch_offset;
914 }
915
916
917 Instr Assembler::ImmCmpBranch(int imm19) {
918 CHECK(is_int19(imm19));
919 return truncate_to_int19(imm19) << ImmCmpBranch_offset;
920 }
921
922
923 Instr Assembler::ImmTestBranch(int imm14) {
924 CHECK(is_int14(imm14));
925 return truncate_to_int14(imm14) << ImmTestBranch_offset;
926 }
927
928
929 Instr Assembler::ImmTestBranchBit(unsigned bit_pos) {
930 DCHECK(is_uint6(bit_pos));
931 // Subtract five from the shift offset, as we need bit 5 from bit_pos.
932 unsigned b5 = bit_pos << (ImmTestBranchBit5_offset - 5);
933 unsigned b40 = bit_pos << ImmTestBranchBit40_offset;
934 b5 &= ImmTestBranchBit5_mask;
935 b40 &= ImmTestBranchBit40_mask;
936 return b5 | b40;
937 }
938
939
940 Instr Assembler::SF(Register rd) {
941 return rd.Is64Bits() ? SixtyFourBits : ThirtyTwoBits;
942 }
943
944
945 Instr Assembler::ImmAddSub(int imm) {
946 DCHECK(IsImmAddSub(imm));
947 if (is_uint12(imm)) { // No shift required.
948 imm <<= ImmAddSub_offset;
949 } else {
950 imm = ((imm >> 12) << ImmAddSub_offset) | (1 << ShiftAddSub_offset);
951 }
952 return imm;
953 }
954
955
956 Instr Assembler::ImmS(unsigned imms, unsigned reg_size) {
957 DCHECK(((reg_size == kXRegSizeInBits) && is_uint6(imms)) ||
958 ((reg_size == kWRegSizeInBits) && is_uint5(imms)));
959 USE(reg_size);
960 return imms << ImmS_offset;
961 }
962
963
964 Instr Assembler::ImmR(unsigned immr, unsigned reg_size) {
965 DCHECK(((reg_size == kXRegSizeInBits) && is_uint6(immr)) ||
966 ((reg_size == kWRegSizeInBits) && is_uint5(immr)));
967 USE(reg_size);
968 DCHECK(is_uint6(immr));
969 return immr << ImmR_offset;
970 }
971
972
973 Instr Assembler::ImmSetBits(unsigned imms, unsigned reg_size) {
974 DCHECK((reg_size == kWRegSizeInBits) || (reg_size == kXRegSizeInBits));
975 DCHECK(is_uint6(imms));
976 DCHECK((reg_size == kXRegSizeInBits) || is_uint6(imms + 3));
977 USE(reg_size);
978 return imms << ImmSetBits_offset;
979 }
980
981
982 Instr Assembler::ImmRotate(unsigned immr, unsigned reg_size) {
983 DCHECK((reg_size == kWRegSizeInBits) || (reg_size == kXRegSizeInBits));
984 DCHECK(((reg_size == kXRegSizeInBits) && is_uint6(immr)) ||
985 ((reg_size == kWRegSizeInBits) && is_uint5(immr)));
986 USE(reg_size);
987 return immr << ImmRotate_offset;
988 }
989
990
991 Instr Assembler::ImmLLiteral(int imm19) {
992 CHECK(is_int19(imm19));
993 return truncate_to_int19(imm19) << ImmLLiteral_offset;
994 }
995
996
997 Instr Assembler::BitN(unsigned bitn, unsigned reg_size) {
998 DCHECK((reg_size == kWRegSizeInBits) || (reg_size == kXRegSizeInBits));
999 DCHECK((reg_size == kXRegSizeInBits) || (bitn == 0));
1000 USE(reg_size);
1001 return bitn << BitN_offset;
1002 }
1003
1004
1005 Instr Assembler::ShiftDP(Shift shift) {
1006 DCHECK(shift == LSL || shift == LSR || shift == ASR || shift == ROR);
1007 return shift << ShiftDP_offset;
1008 }
1009
1010
1011 Instr Assembler::ImmDPShift(unsigned amount) {
1012 DCHECK(is_uint6(amount));
1013 return amount << ImmDPShift_offset;
1014 }
1015
1016
1017 Instr Assembler::ExtendMode(Extend extend) {
1018 return extend << ExtendMode_offset;
1019 }
1020
1021
1022 Instr Assembler::ImmExtendShift(unsigned left_shift) {
1023 DCHECK_LE(left_shift, 4);
1024 return left_shift << ImmExtendShift_offset;
1025 }
1026
1027
1028 Instr Assembler::ImmCondCmp(unsigned imm) {
1029 DCHECK(is_uint5(imm));
1030 return imm << ImmCondCmp_offset;
1031 }
1032
1033
1034 Instr Assembler::Nzcv(StatusFlags nzcv) {
1035 return ((nzcv >> Flags_offset) & 0xf) << Nzcv_offset;
1036 }
1037
1038
1039 Instr Assembler::ImmLSUnsigned(int imm12) {
1040 DCHECK(is_uint12(imm12));
1041 return imm12 << ImmLSUnsigned_offset;
1042 }
1043
1044
1045 Instr Assembler::ImmLS(int imm9) {
1046 DCHECK(is_int9(imm9));
1047 return truncate_to_int9(imm9) << ImmLS_offset;
1048 }
1049
1050 Instr Assembler::ImmLSPair(int imm7, unsigned size) {
1051 DCHECK_EQ((imm7 >> size) << size, imm7);
1052 int scaled_imm7 = imm7 >> size;
1053 DCHECK(is_int7(scaled_imm7));
1054 return truncate_to_int7(scaled_imm7) << ImmLSPair_offset;
1055 }
1056
1057
1058 Instr Assembler::ImmShiftLS(unsigned shift_amount) {
1059 DCHECK(is_uint1(shift_amount));
1060 return shift_amount << ImmShiftLS_offset;
1061 }
1062
1063
1064 Instr Assembler::ImmException(int imm16) {
1065 DCHECK(is_uint16(imm16));
1066 return imm16 << ImmException_offset;
1067 }
1068
1069
1070 Instr Assembler::ImmSystemRegister(int imm15) {
1071 DCHECK(is_uint15(imm15));
1072 return imm15 << ImmSystemRegister_offset;
1073 }
1074
1075
1076 Instr Assembler::ImmHint(int imm7) {
1077 DCHECK(is_uint7(imm7));
1078 return imm7 << ImmHint_offset;
1079 }
1080
1081
1082 Instr Assembler::ImmBarrierDomain(int imm2) {
1083 DCHECK(is_uint2(imm2));
1084 return imm2 << ImmBarrierDomain_offset;
1085 }
1086
1087
1088 Instr Assembler::ImmBarrierType(int imm2) {
1089 DCHECK(is_uint2(imm2));
1090 return imm2 << ImmBarrierType_offset;
1091 }
1092
1093 unsigned Assembler::CalcLSDataSize(LoadStoreOp op) {
1094 DCHECK((LSSize_offset + LSSize_width) == (kInstrSize * 8));
1095 unsigned size = static_cast<Instr>(op >> LSSize_offset);
1096 if ((op & LSVector_mask) != 0) {
1097 // Vector register memory operations encode the access size in the "size"
1098 // and "opc" fields.
1099 if ((size == 0) && ((op & LSOpc_mask) >> LSOpc_offset) >= 2) {
1100 size = kQRegSizeLog2;
1101 }
1102 }
1103 return size;
1104 }
1105
1106
1107 Instr Assembler::ImmMoveWide(int imm) {
1108 DCHECK(is_uint16(imm));
1109 return imm << ImmMoveWide_offset;
1110 }
1111
1112
1113 Instr Assembler::ShiftMoveWide(int shift) {
1114 DCHECK(is_uint2(shift));
1115 return shift << ShiftMoveWide_offset;
1116 }
1117
1118 Instr Assembler::FPType(VRegister fd) { return fd.Is64Bits() ? FP64 : FP32; }
1119
1120 Instr Assembler::FPScale(unsigned scale) {
1121 DCHECK(is_uint6(scale));
1122 return scale << FPScale_offset;
1123 }
1124
1125
1126 const Register& Assembler::AppropriateZeroRegFor(const CPURegister& reg) const {
1127 return reg.Is64Bits() ? xzr : wzr;
1128 }
1129
1130
1131 inline void Assembler::CheckBufferSpace() {
1132 DCHECK(pc_ < (buffer_ + buffer_size_));
1133 if (buffer_space() < kGap) {
1134 GrowBuffer();
1135 }
1136 }
1137
1138
1139 inline void Assembler::CheckBuffer() {
1140 CheckBufferSpace();
1141 if (pc_offset() >= next_veneer_pool_check_) {
1142 CheckVeneerPool(false, true);
1143 }
1144 if (pc_offset() >= next_constant_pool_check_) {
1145 CheckConstPool(false, true);
1146 }
1147 }
1148
1149 } // namespace internal
1150 } // namespace v8
1151
1152 #endif // V8_ARM64_ASSEMBLER_ARM64_INL_H_
1153