1 // Copyright 2015, VIXL authors
2 // All rights reserved.
3 //
4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions are met:
6 //
7 // * Redistributions of source code must retain the above copyright notice,
8 // this list of conditions and the following disclaimer.
9 // * Redistributions in binary form must reproduce the above copyright notice,
10 // this list of conditions and the following disclaimer in the documentation
11 // and/or other materials provided with the distribution.
12 // * Neither the name of ARM Limited nor the names of its contributors may be
13 // used to endorse or promote products derived from this software without
14 // specific prior written permission.
15 //
16 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
17 // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
20 // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
22 // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
23 // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24 // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26
27
28 #include "assembler-aarch64.h"
29
30 #include <cmath>
31
32 #include "macro-assembler-aarch64.h"
33
34 namespace vixl {
35 namespace aarch64 {
36
RawLiteral(size_t size,LiteralPool * literal_pool,DeletionPolicy deletion_policy)37 RawLiteral::RawLiteral(size_t size,
38 LiteralPool* literal_pool,
39 DeletionPolicy deletion_policy)
40 : size_(size),
41 offset_(0),
42 low64_(0),
43 high64_(0),
44 literal_pool_(literal_pool),
45 deletion_policy_(deletion_policy) {
46 VIXL_ASSERT((deletion_policy == kManuallyDeleted) || (literal_pool_ != NULL));
47 if (deletion_policy == kDeletedOnPoolDestruction) {
48 literal_pool_->DeleteOnDestruction(this);
49 }
50 }
51
52
Reset()53 void Assembler::Reset() { GetBuffer()->Reset(); }
54
55
bind(Label * label)56 void Assembler::bind(Label* label) {
57 BindToOffset(label, GetBuffer()->GetCursorOffset());
58 }
59
60
BindToOffset(Label * label,ptrdiff_t offset)61 void Assembler::BindToOffset(Label* label, ptrdiff_t offset) {
62 VIXL_ASSERT((offset >= 0) && (offset <= GetBuffer()->GetCursorOffset()));
63 VIXL_ASSERT(offset % kInstructionSize == 0);
64
65 label->Bind(offset);
66
67 for (Label::LabelLinksIterator it(label); !it.Done(); it.Advance()) {
68 Instruction* link =
69 GetBuffer()->GetOffsetAddress<Instruction*>(*it.Current());
70 link->SetImmPCOffsetTarget(GetLabelAddress<Instruction*>(label));
71 }
72 label->ClearAllLinks();
73 }
74
75
76 // A common implementation for the LinkAndGet<Type>OffsetTo helpers.
77 //
78 // The offset is calculated by aligning the PC and label addresses down to a
79 // multiple of 1 << element_shift, then calculating the (scaled) offset between
80 // them. This matches the semantics of adrp, for example.
81 template <int element_shift>
LinkAndGetOffsetTo(Label * label)82 ptrdiff_t Assembler::LinkAndGetOffsetTo(Label* label) {
83 VIXL_STATIC_ASSERT(element_shift < (sizeof(ptrdiff_t) * 8));
84
85 if (label->IsBound()) {
86 uintptr_t pc_offset = GetCursorAddress<uintptr_t>() >> element_shift;
87 uintptr_t label_offset = GetLabelAddress<uintptr_t>(label) >> element_shift;
88 return label_offset - pc_offset;
89 } else {
90 label->AddLink(GetBuffer()->GetCursorOffset());
91 return 0;
92 }
93 }
94
95
LinkAndGetByteOffsetTo(Label * label)96 ptrdiff_t Assembler::LinkAndGetByteOffsetTo(Label* label) {
97 return LinkAndGetOffsetTo<0>(label);
98 }
99
100
LinkAndGetInstructionOffsetTo(Label * label)101 ptrdiff_t Assembler::LinkAndGetInstructionOffsetTo(Label* label) {
102 return LinkAndGetOffsetTo<kInstructionSizeLog2>(label);
103 }
104
105
LinkAndGetPageOffsetTo(Label * label)106 ptrdiff_t Assembler::LinkAndGetPageOffsetTo(Label* label) {
107 return LinkAndGetOffsetTo<kPageSizeLog2>(label);
108 }
109
110
place(RawLiteral * literal)111 void Assembler::place(RawLiteral* literal) {
112 VIXL_ASSERT(!literal->IsPlaced());
113
114 // Patch instructions using this literal.
115 if (literal->IsUsed()) {
116 Instruction* target = GetCursorAddress<Instruction*>();
117 ptrdiff_t offset = literal->GetLastUse();
118 bool done;
119 do {
120 Instruction* ldr = GetBuffer()->GetOffsetAddress<Instruction*>(offset);
121 VIXL_ASSERT(ldr->IsLoadLiteral());
122
123 ptrdiff_t imm19 = ldr->GetImmLLiteral();
124 VIXL_ASSERT(imm19 <= 0);
125 done = (imm19 == 0);
126 offset += imm19 * kLiteralEntrySize;
127
128 ldr->SetImmLLiteral(target);
129 } while (!done);
130 }
131
132 // "bind" the literal.
133 literal->SetOffset(GetCursorOffset());
134 // Copy the data into the pool.
135 switch (literal->GetSize()) {
136 case kSRegSizeInBytes:
137 dc32(literal->GetRawValue32());
138 break;
139 case kDRegSizeInBytes:
140 dc64(literal->GetRawValue64());
141 break;
142 default:
143 VIXL_ASSERT(literal->GetSize() == kQRegSizeInBytes);
144 dc64(literal->GetRawValue128Low64());
145 dc64(literal->GetRawValue128High64());
146 }
147
148 literal->literal_pool_ = NULL;
149 }
150
151
LinkAndGetWordOffsetTo(RawLiteral * literal)152 ptrdiff_t Assembler::LinkAndGetWordOffsetTo(RawLiteral* literal) {
153 VIXL_ASSERT(IsWordAligned(GetCursorOffset()));
154
155 bool register_first_use =
156 (literal->GetLiteralPool() != NULL) && !literal->IsUsed();
157
158 if (literal->IsPlaced()) {
159 // The literal is "behind", the offset will be negative.
160 VIXL_ASSERT((literal->GetOffset() - GetCursorOffset()) <= 0);
161 return (literal->GetOffset() - GetCursorOffset()) >> kLiteralEntrySizeLog2;
162 }
163
164 ptrdiff_t offset = 0;
165 // Link all uses together.
166 if (literal->IsUsed()) {
167 offset =
168 (literal->GetLastUse() - GetCursorOffset()) >> kLiteralEntrySizeLog2;
169 }
170 literal->SetLastUse(GetCursorOffset());
171
172 if (register_first_use) {
173 literal->GetLiteralPool()->AddEntry(literal);
174 }
175
176 return offset;
177 }
178
179
180 // Code generation.
br(const Register & xn)181 void Assembler::br(const Register& xn) {
182 VIXL_ASSERT(xn.Is64Bits());
183 Emit(BR | Rn(xn));
184 }
185
186
blr(const Register & xn)187 void Assembler::blr(const Register& xn) {
188 VIXL_ASSERT(xn.Is64Bits());
189 Emit(BLR | Rn(xn));
190 }
191
192
ret(const Register & xn)193 void Assembler::ret(const Register& xn) {
194 VIXL_ASSERT(xn.Is64Bits());
195 Emit(RET | Rn(xn));
196 }
197
198
braaz(const Register & xn)199 void Assembler::braaz(const Register& xn) {
200 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
201 VIXL_ASSERT(xn.Is64Bits());
202 Emit(BRAAZ | Rn(xn) | Rd_mask);
203 }
204
brabz(const Register & xn)205 void Assembler::brabz(const Register& xn) {
206 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
207 VIXL_ASSERT(xn.Is64Bits());
208 Emit(BRABZ | Rn(xn) | Rd_mask);
209 }
210
blraaz(const Register & xn)211 void Assembler::blraaz(const Register& xn) {
212 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
213 VIXL_ASSERT(xn.Is64Bits());
214 Emit(BLRAAZ | Rn(xn) | Rd_mask);
215 }
216
blrabz(const Register & xn)217 void Assembler::blrabz(const Register& xn) {
218 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
219 VIXL_ASSERT(xn.Is64Bits());
220 Emit(BLRABZ | Rn(xn) | Rd_mask);
221 }
222
retaa()223 void Assembler::retaa() {
224 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
225 Emit(RETAA | Rn_mask | Rd_mask);
226 }
227
retab()228 void Assembler::retab() {
229 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
230 Emit(RETAB | Rn_mask | Rd_mask);
231 }
232
233 // The Arm ARM names the register Xm but encodes it in the Xd bitfield.
braa(const Register & xn,const Register & xm)234 void Assembler::braa(const Register& xn, const Register& xm) {
235 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
236 VIXL_ASSERT(xn.Is64Bits() && xm.Is64Bits());
237 Emit(BRAA | Rn(xn) | RdSP(xm));
238 }
239
brab(const Register & xn,const Register & xm)240 void Assembler::brab(const Register& xn, const Register& xm) {
241 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
242 VIXL_ASSERT(xn.Is64Bits() && xm.Is64Bits());
243 Emit(BRAB | Rn(xn) | RdSP(xm));
244 }
245
blraa(const Register & xn,const Register & xm)246 void Assembler::blraa(const Register& xn, const Register& xm) {
247 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
248 VIXL_ASSERT(xn.Is64Bits() && xm.Is64Bits());
249 Emit(BLRAA | Rn(xn) | RdSP(xm));
250 }
251
blrab(const Register & xn,const Register & xm)252 void Assembler::blrab(const Register& xn, const Register& xm) {
253 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
254 VIXL_ASSERT(xn.Is64Bits() && xm.Is64Bits());
255 Emit(BLRAB | Rn(xn) | RdSP(xm));
256 }
257
258
b(int64_t imm26)259 void Assembler::b(int64_t imm26) { Emit(B | ImmUncondBranch(imm26)); }
260
261
b(int64_t imm19,Condition cond)262 void Assembler::b(int64_t imm19, Condition cond) {
263 Emit(B_cond | ImmCondBranch(imm19) | cond);
264 }
265
266
b(Label * label)267 void Assembler::b(Label* label) {
268 int64_t offset = LinkAndGetInstructionOffsetTo(label);
269 VIXL_ASSERT(Instruction::IsValidImmPCOffset(UncondBranchType, offset));
270 b(static_cast<int>(offset));
271 }
272
273
b(Label * label,Condition cond)274 void Assembler::b(Label* label, Condition cond) {
275 int64_t offset = LinkAndGetInstructionOffsetTo(label);
276 VIXL_ASSERT(Instruction::IsValidImmPCOffset(CondBranchType, offset));
277 b(static_cast<int>(offset), cond);
278 }
279
280
bl(int64_t imm26)281 void Assembler::bl(int64_t imm26) { Emit(BL | ImmUncondBranch(imm26)); }
282
283
bl(Label * label)284 void Assembler::bl(Label* label) {
285 int64_t offset = LinkAndGetInstructionOffsetTo(label);
286 VIXL_ASSERT(Instruction::IsValidImmPCOffset(UncondBranchType, offset));
287 bl(static_cast<int>(offset));
288 }
289
290
cbz(const Register & rt,int64_t imm19)291 void Assembler::cbz(const Register& rt, int64_t imm19) {
292 Emit(SF(rt) | CBZ | ImmCmpBranch(imm19) | Rt(rt));
293 }
294
295
cbz(const Register & rt,Label * label)296 void Assembler::cbz(const Register& rt, Label* label) {
297 int64_t offset = LinkAndGetInstructionOffsetTo(label);
298 VIXL_ASSERT(Instruction::IsValidImmPCOffset(CompareBranchType, offset));
299 cbz(rt, static_cast<int>(offset));
300 }
301
302
cbnz(const Register & rt,int64_t imm19)303 void Assembler::cbnz(const Register& rt, int64_t imm19) {
304 Emit(SF(rt) | CBNZ | ImmCmpBranch(imm19) | Rt(rt));
305 }
306
307
cbnz(const Register & rt,Label * label)308 void Assembler::cbnz(const Register& rt, Label* label) {
309 int64_t offset = LinkAndGetInstructionOffsetTo(label);
310 VIXL_ASSERT(Instruction::IsValidImmPCOffset(CompareBranchType, offset));
311 cbnz(rt, static_cast<int>(offset));
312 }
313
314
NEONTable(const VRegister & vd,const VRegister & vn,const VRegister & vm,NEONTableOp op)315 void Assembler::NEONTable(const VRegister& vd,
316 const VRegister& vn,
317 const VRegister& vm,
318 NEONTableOp op) {
319 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
320 VIXL_ASSERT(vd.Is16B() || vd.Is8B());
321 VIXL_ASSERT(vn.Is16B());
322 VIXL_ASSERT(AreSameFormat(vd, vm));
323 Emit(op | (vd.IsQ() ? NEON_Q : 0) | Rm(vm) | Rn(vn) | Rd(vd));
324 }
325
326
tbl(const VRegister & vd,const VRegister & vn,const VRegister & vm)327 void Assembler::tbl(const VRegister& vd,
328 const VRegister& vn,
329 const VRegister& vm) {
330 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
331 NEONTable(vd, vn, vm, NEON_TBL_1v);
332 }
333
334
tbl(const VRegister & vd,const VRegister & vn,const VRegister & vn2,const VRegister & vm)335 void Assembler::tbl(const VRegister& vd,
336 const VRegister& vn,
337 const VRegister& vn2,
338 const VRegister& vm) {
339 USE(vn2);
340 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
341 VIXL_ASSERT(AreSameFormat(vn, vn2));
342 VIXL_ASSERT(AreConsecutive(vn, vn2));
343 NEONTable(vd, vn, vm, NEON_TBL_2v);
344 }
345
346
tbl(const VRegister & vd,const VRegister & vn,const VRegister & vn2,const VRegister & vn3,const VRegister & vm)347 void Assembler::tbl(const VRegister& vd,
348 const VRegister& vn,
349 const VRegister& vn2,
350 const VRegister& vn3,
351 const VRegister& vm) {
352 USE(vn2, vn3);
353 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
354 VIXL_ASSERT(AreSameFormat(vn, vn2, vn3));
355 VIXL_ASSERT(AreConsecutive(vn, vn2, vn3));
356 NEONTable(vd, vn, vm, NEON_TBL_3v);
357 }
358
359
tbl(const VRegister & vd,const VRegister & vn,const VRegister & vn2,const VRegister & vn3,const VRegister & vn4,const VRegister & vm)360 void Assembler::tbl(const VRegister& vd,
361 const VRegister& vn,
362 const VRegister& vn2,
363 const VRegister& vn3,
364 const VRegister& vn4,
365 const VRegister& vm) {
366 USE(vn2, vn3, vn4);
367 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
368 VIXL_ASSERT(AreSameFormat(vn, vn2, vn3, vn4));
369 VIXL_ASSERT(AreConsecutive(vn, vn2, vn3, vn4));
370 NEONTable(vd, vn, vm, NEON_TBL_4v);
371 }
372
373
tbx(const VRegister & vd,const VRegister & vn,const VRegister & vm)374 void Assembler::tbx(const VRegister& vd,
375 const VRegister& vn,
376 const VRegister& vm) {
377 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
378 NEONTable(vd, vn, vm, NEON_TBX_1v);
379 }
380
381
tbx(const VRegister & vd,const VRegister & vn,const VRegister & vn2,const VRegister & vm)382 void Assembler::tbx(const VRegister& vd,
383 const VRegister& vn,
384 const VRegister& vn2,
385 const VRegister& vm) {
386 USE(vn2);
387 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
388 VIXL_ASSERT(AreSameFormat(vn, vn2));
389 VIXL_ASSERT(AreConsecutive(vn, vn2));
390 NEONTable(vd, vn, vm, NEON_TBX_2v);
391 }
392
393
tbx(const VRegister & vd,const VRegister & vn,const VRegister & vn2,const VRegister & vn3,const VRegister & vm)394 void Assembler::tbx(const VRegister& vd,
395 const VRegister& vn,
396 const VRegister& vn2,
397 const VRegister& vn3,
398 const VRegister& vm) {
399 USE(vn2, vn3);
400 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
401 VIXL_ASSERT(AreSameFormat(vn, vn2, vn3));
402 VIXL_ASSERT(AreConsecutive(vn, vn2, vn3));
403 NEONTable(vd, vn, vm, NEON_TBX_3v);
404 }
405
406
tbx(const VRegister & vd,const VRegister & vn,const VRegister & vn2,const VRegister & vn3,const VRegister & vn4,const VRegister & vm)407 void Assembler::tbx(const VRegister& vd,
408 const VRegister& vn,
409 const VRegister& vn2,
410 const VRegister& vn3,
411 const VRegister& vn4,
412 const VRegister& vm) {
413 USE(vn2, vn3, vn4);
414 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
415 VIXL_ASSERT(AreSameFormat(vn, vn2, vn3, vn4));
416 VIXL_ASSERT(AreConsecutive(vn, vn2, vn3, vn4));
417 NEONTable(vd, vn, vm, NEON_TBX_4v);
418 }
419
420
tbz(const Register & rt,unsigned bit_pos,int64_t imm14)421 void Assembler::tbz(const Register& rt, unsigned bit_pos, int64_t imm14) {
422 VIXL_ASSERT(rt.Is64Bits() || (rt.Is32Bits() && (bit_pos < kWRegSize)));
423 Emit(TBZ | ImmTestBranchBit(bit_pos) | ImmTestBranch(imm14) | Rt(rt));
424 }
425
426
tbz(const Register & rt,unsigned bit_pos,Label * label)427 void Assembler::tbz(const Register& rt, unsigned bit_pos, Label* label) {
428 ptrdiff_t offset = LinkAndGetInstructionOffsetTo(label);
429 VIXL_ASSERT(Instruction::IsValidImmPCOffset(TestBranchType, offset));
430 tbz(rt, bit_pos, static_cast<int>(offset));
431 }
432
433
tbnz(const Register & rt,unsigned bit_pos,int64_t imm14)434 void Assembler::tbnz(const Register& rt, unsigned bit_pos, int64_t imm14) {
435 VIXL_ASSERT(rt.Is64Bits() || (rt.Is32Bits() && (bit_pos < kWRegSize)));
436 Emit(TBNZ | ImmTestBranchBit(bit_pos) | ImmTestBranch(imm14) | Rt(rt));
437 }
438
439
tbnz(const Register & rt,unsigned bit_pos,Label * label)440 void Assembler::tbnz(const Register& rt, unsigned bit_pos, Label* label) {
441 ptrdiff_t offset = LinkAndGetInstructionOffsetTo(label);
442 VIXL_ASSERT(Instruction::IsValidImmPCOffset(TestBranchType, offset));
443 tbnz(rt, bit_pos, static_cast<int>(offset));
444 }
445
446
adr(const Register & xd,int64_t imm21)447 void Assembler::adr(const Register& xd, int64_t imm21) {
448 VIXL_ASSERT(xd.Is64Bits());
449 Emit(ADR | ImmPCRelAddress(imm21) | Rd(xd));
450 }
451
452
adr(const Register & xd,Label * label)453 void Assembler::adr(const Register& xd, Label* label) {
454 adr(xd, static_cast<int>(LinkAndGetByteOffsetTo(label)));
455 }
456
457
adrp(const Register & xd,int64_t imm21)458 void Assembler::adrp(const Register& xd, int64_t imm21) {
459 VIXL_ASSERT(xd.Is64Bits());
460 Emit(ADRP | ImmPCRelAddress(imm21) | Rd(xd));
461 }
462
463
adrp(const Register & xd,Label * label)464 void Assembler::adrp(const Register& xd, Label* label) {
465 VIXL_ASSERT(AllowPageOffsetDependentCode());
466 adrp(xd, static_cast<int>(LinkAndGetPageOffsetTo(label)));
467 }
468
469
add(const Register & rd,const Register & rn,const Operand & operand)470 void Assembler::add(const Register& rd,
471 const Register& rn,
472 const Operand& operand) {
473 AddSub(rd, rn, operand, LeaveFlags, ADD);
474 }
475
476
adds(const Register & rd,const Register & rn,const Operand & operand)477 void Assembler::adds(const Register& rd,
478 const Register& rn,
479 const Operand& operand) {
480 AddSub(rd, rn, operand, SetFlags, ADD);
481 }
482
483
cmn(const Register & rn,const Operand & operand)484 void Assembler::cmn(const Register& rn, const Operand& operand) {
485 Register zr = AppropriateZeroRegFor(rn);
486 adds(zr, rn, operand);
487 }
488
489
sub(const Register & rd,const Register & rn,const Operand & operand)490 void Assembler::sub(const Register& rd,
491 const Register& rn,
492 const Operand& operand) {
493 AddSub(rd, rn, operand, LeaveFlags, SUB);
494 }
495
496
subs(const Register & rd,const Register & rn,const Operand & operand)497 void Assembler::subs(const Register& rd,
498 const Register& rn,
499 const Operand& operand) {
500 AddSub(rd, rn, operand, SetFlags, SUB);
501 }
502
503
cmp(const Register & rn,const Operand & operand)504 void Assembler::cmp(const Register& rn, const Operand& operand) {
505 Register zr = AppropriateZeroRegFor(rn);
506 subs(zr, rn, operand);
507 }
508
509
neg(const Register & rd,const Operand & operand)510 void Assembler::neg(const Register& rd, const Operand& operand) {
511 Register zr = AppropriateZeroRegFor(rd);
512 sub(rd, zr, operand);
513 }
514
515
negs(const Register & rd,const Operand & operand)516 void Assembler::negs(const Register& rd, const Operand& operand) {
517 Register zr = AppropriateZeroRegFor(rd);
518 subs(rd, zr, operand);
519 }
520
521
adc(const Register & rd,const Register & rn,const Operand & operand)522 void Assembler::adc(const Register& rd,
523 const Register& rn,
524 const Operand& operand) {
525 AddSubWithCarry(rd, rn, operand, LeaveFlags, ADC);
526 }
527
528
adcs(const Register & rd,const Register & rn,const Operand & operand)529 void Assembler::adcs(const Register& rd,
530 const Register& rn,
531 const Operand& operand) {
532 AddSubWithCarry(rd, rn, operand, SetFlags, ADC);
533 }
534
535
sbc(const Register & rd,const Register & rn,const Operand & operand)536 void Assembler::sbc(const Register& rd,
537 const Register& rn,
538 const Operand& operand) {
539 AddSubWithCarry(rd, rn, operand, LeaveFlags, SBC);
540 }
541
542
sbcs(const Register & rd,const Register & rn,const Operand & operand)543 void Assembler::sbcs(const Register& rd,
544 const Register& rn,
545 const Operand& operand) {
546 AddSubWithCarry(rd, rn, operand, SetFlags, SBC);
547 }
548
549
rmif(const Register & xn,unsigned rotation,StatusFlags flags)550 void Assembler::rmif(const Register& xn, unsigned rotation, StatusFlags flags) {
551 VIXL_ASSERT(CPUHas(CPUFeatures::kFlagM));
552 VIXL_ASSERT(xn.Is64Bits());
553 Emit(RMIF | Rn(xn) | ImmRMIFRotation(rotation) | Nzcv(flags));
554 }
555
556
setf8(const Register & rn)557 void Assembler::setf8(const Register& rn) {
558 VIXL_ASSERT(CPUHas(CPUFeatures::kFlagM));
559 Emit(SETF8 | Rn(rn));
560 }
561
562
setf16(const Register & rn)563 void Assembler::setf16(const Register& rn) {
564 VIXL_ASSERT(CPUHas(CPUFeatures::kFlagM));
565 Emit(SETF16 | Rn(rn));
566 }
567
568
ngc(const Register & rd,const Operand & operand)569 void Assembler::ngc(const Register& rd, const Operand& operand) {
570 Register zr = AppropriateZeroRegFor(rd);
571 sbc(rd, zr, operand);
572 }
573
574
ngcs(const Register & rd,const Operand & operand)575 void Assembler::ngcs(const Register& rd, const Operand& operand) {
576 Register zr = AppropriateZeroRegFor(rd);
577 sbcs(rd, zr, operand);
578 }
579
580
581 // Logical instructions.
and_(const Register & rd,const Register & rn,const Operand & operand)582 void Assembler::and_(const Register& rd,
583 const Register& rn,
584 const Operand& operand) {
585 Logical(rd, rn, operand, AND);
586 }
587
588
ands(const Register & rd,const Register & rn,const Operand & operand)589 void Assembler::ands(const Register& rd,
590 const Register& rn,
591 const Operand& operand) {
592 Logical(rd, rn, operand, ANDS);
593 }
594
595
tst(const Register & rn,const Operand & operand)596 void Assembler::tst(const Register& rn, const Operand& operand) {
597 ands(AppropriateZeroRegFor(rn), rn, operand);
598 }
599
600
bic(const Register & rd,const Register & rn,const Operand & operand)601 void Assembler::bic(const Register& rd,
602 const Register& rn,
603 const Operand& operand) {
604 Logical(rd, rn, operand, BIC);
605 }
606
607
bics(const Register & rd,const Register & rn,const Operand & operand)608 void Assembler::bics(const Register& rd,
609 const Register& rn,
610 const Operand& operand) {
611 Logical(rd, rn, operand, BICS);
612 }
613
614
orr(const Register & rd,const Register & rn,const Operand & operand)615 void Assembler::orr(const Register& rd,
616 const Register& rn,
617 const Operand& operand) {
618 Logical(rd, rn, operand, ORR);
619 }
620
621
orn(const Register & rd,const Register & rn,const Operand & operand)622 void Assembler::orn(const Register& rd,
623 const Register& rn,
624 const Operand& operand) {
625 Logical(rd, rn, operand, ORN);
626 }
627
628
eor(const Register & rd,const Register & rn,const Operand & operand)629 void Assembler::eor(const Register& rd,
630 const Register& rn,
631 const Operand& operand) {
632 Logical(rd, rn, operand, EOR);
633 }
634
635
eon(const Register & rd,const Register & rn,const Operand & operand)636 void Assembler::eon(const Register& rd,
637 const Register& rn,
638 const Operand& operand) {
639 Logical(rd, rn, operand, EON);
640 }
641
642
lslv(const Register & rd,const Register & rn,const Register & rm)643 void Assembler::lslv(const Register& rd,
644 const Register& rn,
645 const Register& rm) {
646 VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
647 VIXL_ASSERT(rd.GetSizeInBits() == rm.GetSizeInBits());
648 Emit(SF(rd) | LSLV | Rm(rm) | Rn(rn) | Rd(rd));
649 }
650
651
lsrv(const Register & rd,const Register & rn,const Register & rm)652 void Assembler::lsrv(const Register& rd,
653 const Register& rn,
654 const Register& rm) {
655 VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
656 VIXL_ASSERT(rd.GetSizeInBits() == rm.GetSizeInBits());
657 Emit(SF(rd) | LSRV | Rm(rm) | Rn(rn) | Rd(rd));
658 }
659
660
asrv(const Register & rd,const Register & rn,const Register & rm)661 void Assembler::asrv(const Register& rd,
662 const Register& rn,
663 const Register& rm) {
664 VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
665 VIXL_ASSERT(rd.GetSizeInBits() == rm.GetSizeInBits());
666 Emit(SF(rd) | ASRV | Rm(rm) | Rn(rn) | Rd(rd));
667 }
668
669
rorv(const Register & rd,const Register & rn,const Register & rm)670 void Assembler::rorv(const Register& rd,
671 const Register& rn,
672 const Register& rm) {
673 VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
674 VIXL_ASSERT(rd.GetSizeInBits() == rm.GetSizeInBits());
675 Emit(SF(rd) | RORV | Rm(rm) | Rn(rn) | Rd(rd));
676 }
677
678
679 // Bitfield operations.
bfm(const Register & rd,const Register & rn,unsigned immr,unsigned imms)680 void Assembler::bfm(const Register& rd,
681 const Register& rn,
682 unsigned immr,
683 unsigned imms) {
684 VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
685 Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset);
686 Emit(SF(rd) | BFM | N | ImmR(immr, rd.GetSizeInBits()) |
687 ImmS(imms, rn.GetSizeInBits()) | Rn(rn) | Rd(rd));
688 }
689
690
sbfm(const Register & rd,const Register & rn,unsigned immr,unsigned imms)691 void Assembler::sbfm(const Register& rd,
692 const Register& rn,
693 unsigned immr,
694 unsigned imms) {
695 VIXL_ASSERT(rd.Is64Bits() || rn.Is32Bits());
696 Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset);
697 Emit(SF(rd) | SBFM | N | ImmR(immr, rd.GetSizeInBits()) |
698 ImmS(imms, rn.GetSizeInBits()) | Rn(rn) | Rd(rd));
699 }
700
701
ubfm(const Register & rd,const Register & rn,unsigned immr,unsigned imms)702 void Assembler::ubfm(const Register& rd,
703 const Register& rn,
704 unsigned immr,
705 unsigned imms) {
706 VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
707 Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset);
708 Emit(SF(rd) | UBFM | N | ImmR(immr, rd.GetSizeInBits()) |
709 ImmS(imms, rn.GetSizeInBits()) | Rn(rn) | Rd(rd));
710 }
711
712
extr(const Register & rd,const Register & rn,const Register & rm,unsigned lsb)713 void Assembler::extr(const Register& rd,
714 const Register& rn,
715 const Register& rm,
716 unsigned lsb) {
717 VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
718 VIXL_ASSERT(rd.GetSizeInBits() == rm.GetSizeInBits());
719 Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset);
720 Emit(SF(rd) | EXTR | N | Rm(rm) | ImmS(lsb, rn.GetSizeInBits()) | Rn(rn) |
721 Rd(rd));
722 }
723
724
csel(const Register & rd,const Register & rn,const Register & rm,Condition cond)725 void Assembler::csel(const Register& rd,
726 const Register& rn,
727 const Register& rm,
728 Condition cond) {
729 ConditionalSelect(rd, rn, rm, cond, CSEL);
730 }
731
732
csinc(const Register & rd,const Register & rn,const Register & rm,Condition cond)733 void Assembler::csinc(const Register& rd,
734 const Register& rn,
735 const Register& rm,
736 Condition cond) {
737 ConditionalSelect(rd, rn, rm, cond, CSINC);
738 }
739
740
csinv(const Register & rd,const Register & rn,const Register & rm,Condition cond)741 void Assembler::csinv(const Register& rd,
742 const Register& rn,
743 const Register& rm,
744 Condition cond) {
745 ConditionalSelect(rd, rn, rm, cond, CSINV);
746 }
747
748
csneg(const Register & rd,const Register & rn,const Register & rm,Condition cond)749 void Assembler::csneg(const Register& rd,
750 const Register& rn,
751 const Register& rm,
752 Condition cond) {
753 ConditionalSelect(rd, rn, rm, cond, CSNEG);
754 }
755
756
cset(const Register & rd,Condition cond)757 void Assembler::cset(const Register& rd, Condition cond) {
758 VIXL_ASSERT((cond != al) && (cond != nv));
759 Register zr = AppropriateZeroRegFor(rd);
760 csinc(rd, zr, zr, InvertCondition(cond));
761 }
762
763
csetm(const Register & rd,Condition cond)764 void Assembler::csetm(const Register& rd, Condition cond) {
765 VIXL_ASSERT((cond != al) && (cond != nv));
766 Register zr = AppropriateZeroRegFor(rd);
767 csinv(rd, zr, zr, InvertCondition(cond));
768 }
769
770
cinc(const Register & rd,const Register & rn,Condition cond)771 void Assembler::cinc(const Register& rd, const Register& rn, Condition cond) {
772 VIXL_ASSERT((cond != al) && (cond != nv));
773 csinc(rd, rn, rn, InvertCondition(cond));
774 }
775
776
cinv(const Register & rd,const Register & rn,Condition cond)777 void Assembler::cinv(const Register& rd, const Register& rn, Condition cond) {
778 VIXL_ASSERT((cond != al) && (cond != nv));
779 csinv(rd, rn, rn, InvertCondition(cond));
780 }
781
782
cneg(const Register & rd,const Register & rn,Condition cond)783 void Assembler::cneg(const Register& rd, const Register& rn, Condition cond) {
784 VIXL_ASSERT((cond != al) && (cond != nv));
785 csneg(rd, rn, rn, InvertCondition(cond));
786 }
787
788
ConditionalSelect(const Register & rd,const Register & rn,const Register & rm,Condition cond,ConditionalSelectOp op)789 void Assembler::ConditionalSelect(const Register& rd,
790 const Register& rn,
791 const Register& rm,
792 Condition cond,
793 ConditionalSelectOp op) {
794 VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
795 VIXL_ASSERT(rd.GetSizeInBits() == rm.GetSizeInBits());
796 Emit(SF(rd) | op | Rm(rm) | Cond(cond) | Rn(rn) | Rd(rd));
797 }
798
799
ccmn(const Register & rn,const Operand & operand,StatusFlags nzcv,Condition cond)800 void Assembler::ccmn(const Register& rn,
801 const Operand& operand,
802 StatusFlags nzcv,
803 Condition cond) {
804 ConditionalCompare(rn, operand, nzcv, cond, CCMN);
805 }
806
807
ccmp(const Register & rn,const Operand & operand,StatusFlags nzcv,Condition cond)808 void Assembler::ccmp(const Register& rn,
809 const Operand& operand,
810 StatusFlags nzcv,
811 Condition cond) {
812 ConditionalCompare(rn, operand, nzcv, cond, CCMP);
813 }
814
815
DataProcessing3Source(const Register & rd,const Register & rn,const Register & rm,const Register & ra,DataProcessing3SourceOp op)816 void Assembler::DataProcessing3Source(const Register& rd,
817 const Register& rn,
818 const Register& rm,
819 const Register& ra,
820 DataProcessing3SourceOp op) {
821 Emit(SF(rd) | op | Rm(rm) | Ra(ra) | Rn(rn) | Rd(rd));
822 }
823
824
crc32b(const Register & wd,const Register & wn,const Register & wm)825 void Assembler::crc32b(const Register& wd,
826 const Register& wn,
827 const Register& wm) {
828 VIXL_ASSERT(CPUHas(CPUFeatures::kCRC32));
829 VIXL_ASSERT(wd.Is32Bits() && wn.Is32Bits() && wm.Is32Bits());
830 Emit(SF(wm) | Rm(wm) | CRC32B | Rn(wn) | Rd(wd));
831 }
832
833
crc32h(const Register & wd,const Register & wn,const Register & wm)834 void Assembler::crc32h(const Register& wd,
835 const Register& wn,
836 const Register& wm) {
837 VIXL_ASSERT(CPUHas(CPUFeatures::kCRC32));
838 VIXL_ASSERT(wd.Is32Bits() && wn.Is32Bits() && wm.Is32Bits());
839 Emit(SF(wm) | Rm(wm) | CRC32H | Rn(wn) | Rd(wd));
840 }
841
842
crc32w(const Register & wd,const Register & wn,const Register & wm)843 void Assembler::crc32w(const Register& wd,
844 const Register& wn,
845 const Register& wm) {
846 VIXL_ASSERT(CPUHas(CPUFeatures::kCRC32));
847 VIXL_ASSERT(wd.Is32Bits() && wn.Is32Bits() && wm.Is32Bits());
848 Emit(SF(wm) | Rm(wm) | CRC32W | Rn(wn) | Rd(wd));
849 }
850
851
crc32x(const Register & wd,const Register & wn,const Register & xm)852 void Assembler::crc32x(const Register& wd,
853 const Register& wn,
854 const Register& xm) {
855 VIXL_ASSERT(CPUHas(CPUFeatures::kCRC32));
856 VIXL_ASSERT(wd.Is32Bits() && wn.Is32Bits() && xm.Is64Bits());
857 Emit(SF(xm) | Rm(xm) | CRC32X | Rn(wn) | Rd(wd));
858 }
859
860
crc32cb(const Register & wd,const Register & wn,const Register & wm)861 void Assembler::crc32cb(const Register& wd,
862 const Register& wn,
863 const Register& wm) {
864 VIXL_ASSERT(CPUHas(CPUFeatures::kCRC32));
865 VIXL_ASSERT(wd.Is32Bits() && wn.Is32Bits() && wm.Is32Bits());
866 Emit(SF(wm) | Rm(wm) | CRC32CB | Rn(wn) | Rd(wd));
867 }
868
869
crc32ch(const Register & wd,const Register & wn,const Register & wm)870 void Assembler::crc32ch(const Register& wd,
871 const Register& wn,
872 const Register& wm) {
873 VIXL_ASSERT(CPUHas(CPUFeatures::kCRC32));
874 VIXL_ASSERT(wd.Is32Bits() && wn.Is32Bits() && wm.Is32Bits());
875 Emit(SF(wm) | Rm(wm) | CRC32CH | Rn(wn) | Rd(wd));
876 }
877
878
crc32cw(const Register & wd,const Register & wn,const Register & wm)879 void Assembler::crc32cw(const Register& wd,
880 const Register& wn,
881 const Register& wm) {
882 VIXL_ASSERT(CPUHas(CPUFeatures::kCRC32));
883 VIXL_ASSERT(wd.Is32Bits() && wn.Is32Bits() && wm.Is32Bits());
884 Emit(SF(wm) | Rm(wm) | CRC32CW | Rn(wn) | Rd(wd));
885 }
886
887
crc32cx(const Register & wd,const Register & wn,const Register & xm)888 void Assembler::crc32cx(const Register& wd,
889 const Register& wn,
890 const Register& xm) {
891 VIXL_ASSERT(CPUHas(CPUFeatures::kCRC32));
892 VIXL_ASSERT(wd.Is32Bits() && wn.Is32Bits() && xm.Is64Bits());
893 Emit(SF(xm) | Rm(xm) | CRC32CX | Rn(wn) | Rd(wd));
894 }
895
896
mul(const Register & rd,const Register & rn,const Register & rm)897 void Assembler::mul(const Register& rd,
898 const Register& rn,
899 const Register& rm) {
900 VIXL_ASSERT(AreSameSizeAndType(rd, rn, rm));
901 DataProcessing3Source(rd, rn, rm, AppropriateZeroRegFor(rd), MADD);
902 }
903
904
madd(const Register & rd,const Register & rn,const Register & rm,const Register & ra)905 void Assembler::madd(const Register& rd,
906 const Register& rn,
907 const Register& rm,
908 const Register& ra) {
909 DataProcessing3Source(rd, rn, rm, ra, MADD);
910 }
911
912
mneg(const Register & rd,const Register & rn,const Register & rm)913 void Assembler::mneg(const Register& rd,
914 const Register& rn,
915 const Register& rm) {
916 VIXL_ASSERT(AreSameSizeAndType(rd, rn, rm));
917 DataProcessing3Source(rd, rn, rm, AppropriateZeroRegFor(rd), MSUB);
918 }
919
920
msub(const Register & rd,const Register & rn,const Register & rm,const Register & ra)921 void Assembler::msub(const Register& rd,
922 const Register& rn,
923 const Register& rm,
924 const Register& ra) {
925 DataProcessing3Source(rd, rn, rm, ra, MSUB);
926 }
927
928
umaddl(const Register & xd,const Register & wn,const Register & wm,const Register & xa)929 void Assembler::umaddl(const Register& xd,
930 const Register& wn,
931 const Register& wm,
932 const Register& xa) {
933 VIXL_ASSERT(xd.Is64Bits() && xa.Is64Bits());
934 VIXL_ASSERT(wn.Is32Bits() && wm.Is32Bits());
935 DataProcessing3Source(xd, wn, wm, xa, UMADDL_x);
936 }
937
938
smaddl(const Register & xd,const Register & wn,const Register & wm,const Register & xa)939 void Assembler::smaddl(const Register& xd,
940 const Register& wn,
941 const Register& wm,
942 const Register& xa) {
943 VIXL_ASSERT(xd.Is64Bits() && xa.Is64Bits());
944 VIXL_ASSERT(wn.Is32Bits() && wm.Is32Bits());
945 DataProcessing3Source(xd, wn, wm, xa, SMADDL_x);
946 }
947
948
umsubl(const Register & xd,const Register & wn,const Register & wm,const Register & xa)949 void Assembler::umsubl(const Register& xd,
950 const Register& wn,
951 const Register& wm,
952 const Register& xa) {
953 VIXL_ASSERT(xd.Is64Bits() && xa.Is64Bits());
954 VIXL_ASSERT(wn.Is32Bits() && wm.Is32Bits());
955 DataProcessing3Source(xd, wn, wm, xa, UMSUBL_x);
956 }
957
958
smsubl(const Register & xd,const Register & wn,const Register & wm,const Register & xa)959 void Assembler::smsubl(const Register& xd,
960 const Register& wn,
961 const Register& wm,
962 const Register& xa) {
963 VIXL_ASSERT(xd.Is64Bits() && xa.Is64Bits());
964 VIXL_ASSERT(wn.Is32Bits() && wm.Is32Bits());
965 DataProcessing3Source(xd, wn, wm, xa, SMSUBL_x);
966 }
967
968
smull(const Register & xd,const Register & wn,const Register & wm)969 void Assembler::smull(const Register& xd,
970 const Register& wn,
971 const Register& wm) {
972 VIXL_ASSERT(xd.Is64Bits());
973 VIXL_ASSERT(wn.Is32Bits() && wm.Is32Bits());
974 DataProcessing3Source(xd, wn, wm, xzr, SMADDL_x);
975 }
976
977
sdiv(const Register & rd,const Register & rn,const Register & rm)978 void Assembler::sdiv(const Register& rd,
979 const Register& rn,
980 const Register& rm) {
981 VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
982 VIXL_ASSERT(rd.GetSizeInBits() == rm.GetSizeInBits());
983 Emit(SF(rd) | SDIV | Rm(rm) | Rn(rn) | Rd(rd));
984 }
985
986
smulh(const Register & xd,const Register & xn,const Register & xm)987 void Assembler::smulh(const Register& xd,
988 const Register& xn,
989 const Register& xm) {
990 VIXL_ASSERT(xd.Is64Bits() && xn.Is64Bits() && xm.Is64Bits());
991 DataProcessing3Source(xd, xn, xm, xzr, SMULH_x);
992 }
993
994
umulh(const Register & xd,const Register & xn,const Register & xm)995 void Assembler::umulh(const Register& xd,
996 const Register& xn,
997 const Register& xm) {
998 VIXL_ASSERT(xd.Is64Bits() && xn.Is64Bits() && xm.Is64Bits());
999 DataProcessing3Source(xd, xn, xm, xzr, UMULH_x);
1000 }
1001
1002
udiv(const Register & rd,const Register & rn,const Register & rm)1003 void Assembler::udiv(const Register& rd,
1004 const Register& rn,
1005 const Register& rm) {
1006 VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
1007 VIXL_ASSERT(rd.GetSizeInBits() == rm.GetSizeInBits());
1008 Emit(SF(rd) | UDIV | Rm(rm) | Rn(rn) | Rd(rd));
1009 }
1010
1011
rbit(const Register & rd,const Register & rn)1012 void Assembler::rbit(const Register& rd, const Register& rn) {
1013 DataProcessing1Source(rd, rn, RBIT);
1014 }
1015
1016
rev16(const Register & rd,const Register & rn)1017 void Assembler::rev16(const Register& rd, const Register& rn) {
1018 DataProcessing1Source(rd, rn, REV16);
1019 }
1020
1021
rev32(const Register & xd,const Register & xn)1022 void Assembler::rev32(const Register& xd, const Register& xn) {
1023 VIXL_ASSERT(xd.Is64Bits());
1024 DataProcessing1Source(xd, xn, REV);
1025 }
1026
1027
rev(const Register & rd,const Register & rn)1028 void Assembler::rev(const Register& rd, const Register& rn) {
1029 DataProcessing1Source(rd, rn, rd.Is64Bits() ? REV_x : REV_w);
1030 }
1031
1032
clz(const Register & rd,const Register & rn)1033 void Assembler::clz(const Register& rd, const Register& rn) {
1034 DataProcessing1Source(rd, rn, CLZ);
1035 }
1036
1037
cls(const Register & rd,const Register & rn)1038 void Assembler::cls(const Register& rd, const Register& rn) {
1039 DataProcessing1Source(rd, rn, CLS);
1040 }
1041
1042 #define PAUTH_VARIATIONS(V) \
1043 V(paci, PACI) \
1044 V(pacd, PACD) \
1045 V(auti, AUTI) \
1046 V(autd, AUTD)
1047
1048 #define VIXL_DEFINE_ASM_FUNC(PRE, OP) \
1049 void Assembler::PRE##a(const Register& xd, const Register& xn) { \
1050 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth)); \
1051 VIXL_ASSERT(xd.Is64Bits() && xn.Is64Bits()); \
1052 Emit(SF(xd) | OP##A | Rd(xd) | RnSP(xn)); \
1053 } \
1054 \
1055 void Assembler::PRE##za(const Register& xd) { \
1056 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth)); \
1057 VIXL_ASSERT(xd.Is64Bits()); \
1058 Emit(SF(xd) | OP##ZA | Rd(xd) | Rn(xzr)); \
1059 } \
1060 \
1061 void Assembler::PRE##b(const Register& xd, const Register& xn) { \
1062 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth)); \
1063 VIXL_ASSERT(xd.Is64Bits() && xn.Is64Bits()); \
1064 Emit(SF(xd) | OP##B | Rd(xd) | RnSP(xn)); \
1065 } \
1066 \
1067 void Assembler::PRE##zb(const Register& xd) { \
1068 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth)); \
1069 VIXL_ASSERT(xd.Is64Bits()); \
1070 Emit(SF(xd) | OP##ZB | Rd(xd) | Rn(xzr)); \
1071 }
1072
PAUTH_VARIATIONS(VIXL_DEFINE_ASM_FUNC)1073 PAUTH_VARIATIONS(VIXL_DEFINE_ASM_FUNC)
1074 #undef VIXL_DEFINE_ASM_FUNC
1075
1076 void Assembler::pacga(const Register& xd,
1077 const Register& xn,
1078 const Register& xm) {
1079 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth, CPUFeatures::kPAuthGeneric));
1080 VIXL_ASSERT(xd.Is64Bits() && xn.Is64Bits() && xm.Is64Bits());
1081 Emit(SF(xd) | PACGA | Rd(xd) | Rn(xn) | RmSP(xm));
1082 }
1083
xpaci(const Register & xd)1084 void Assembler::xpaci(const Register& xd) {
1085 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
1086 VIXL_ASSERT(xd.Is64Bits());
1087 Emit(SF(xd) | XPACI | Rd(xd) | Rn(xzr));
1088 }
1089
xpacd(const Register & xd)1090 void Assembler::xpacd(const Register& xd) {
1091 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
1092 VIXL_ASSERT(xd.Is64Bits());
1093 Emit(SF(xd) | XPACD | Rd(xd) | Rn(xzr));
1094 }
1095
1096
ldp(const CPURegister & rt,const CPURegister & rt2,const MemOperand & src)1097 void Assembler::ldp(const CPURegister& rt,
1098 const CPURegister& rt2,
1099 const MemOperand& src) {
1100 LoadStorePair(rt, rt2, src, LoadPairOpFor(rt, rt2));
1101 }
1102
1103
stp(const CPURegister & rt,const CPURegister & rt2,const MemOperand & dst)1104 void Assembler::stp(const CPURegister& rt,
1105 const CPURegister& rt2,
1106 const MemOperand& dst) {
1107 LoadStorePair(rt, rt2, dst, StorePairOpFor(rt, rt2));
1108 }
1109
1110
ldpsw(const Register & xt,const Register & xt2,const MemOperand & src)1111 void Assembler::ldpsw(const Register& xt,
1112 const Register& xt2,
1113 const MemOperand& src) {
1114 VIXL_ASSERT(xt.Is64Bits() && xt2.Is64Bits());
1115 LoadStorePair(xt, xt2, src, LDPSW_x);
1116 }
1117
1118
LoadStorePair(const CPURegister & rt,const CPURegister & rt2,const MemOperand & addr,LoadStorePairOp op)1119 void Assembler::LoadStorePair(const CPURegister& rt,
1120 const CPURegister& rt2,
1121 const MemOperand& addr,
1122 LoadStorePairOp op) {
1123 VIXL_ASSERT(CPUHas(rt, rt2));
1124
1125 // 'rt' and 'rt2' can only be aliased for stores.
1126 VIXL_ASSERT(((op & LoadStorePairLBit) == 0) || !rt.Is(rt2));
1127 VIXL_ASSERT(AreSameSizeAndType(rt, rt2));
1128 VIXL_ASSERT(IsImmLSPair(addr.GetOffset(), CalcLSPairDataSize(op)));
1129
1130 int offset = static_cast<int>(addr.GetOffset());
1131 Instr memop = op | Rt(rt) | Rt2(rt2) | RnSP(addr.GetBaseRegister()) |
1132 ImmLSPair(offset, CalcLSPairDataSize(op));
1133
1134 Instr addrmodeop;
1135 if (addr.IsImmediateOffset()) {
1136 addrmodeop = LoadStorePairOffsetFixed;
1137 } else {
1138 if (addr.IsImmediatePreIndex()) {
1139 addrmodeop = LoadStorePairPreIndexFixed;
1140 } else {
1141 VIXL_ASSERT(addr.IsImmediatePostIndex());
1142 addrmodeop = LoadStorePairPostIndexFixed;
1143 }
1144 }
1145
1146 Instr emitop = addrmodeop | memop;
1147
1148 // Only X registers may be specified for ldpsw.
1149 VIXL_ASSERT(((emitop & LoadStorePairMask) != LDPSW_x) || rt.IsX());
1150
1151 Emit(emitop);
1152 }
1153
1154
ldnp(const CPURegister & rt,const CPURegister & rt2,const MemOperand & src)1155 void Assembler::ldnp(const CPURegister& rt,
1156 const CPURegister& rt2,
1157 const MemOperand& src) {
1158 LoadStorePairNonTemporal(rt, rt2, src, LoadPairNonTemporalOpFor(rt, rt2));
1159 }
1160
1161
stnp(const CPURegister & rt,const CPURegister & rt2,const MemOperand & dst)1162 void Assembler::stnp(const CPURegister& rt,
1163 const CPURegister& rt2,
1164 const MemOperand& dst) {
1165 LoadStorePairNonTemporal(rt, rt2, dst, StorePairNonTemporalOpFor(rt, rt2));
1166 }
1167
1168
LoadStorePairNonTemporal(const CPURegister & rt,const CPURegister & rt2,const MemOperand & addr,LoadStorePairNonTemporalOp op)1169 void Assembler::LoadStorePairNonTemporal(const CPURegister& rt,
1170 const CPURegister& rt2,
1171 const MemOperand& addr,
1172 LoadStorePairNonTemporalOp op) {
1173 VIXL_ASSERT(CPUHas(rt, rt2));
1174
1175 VIXL_ASSERT(!rt.Is(rt2));
1176 VIXL_ASSERT(AreSameSizeAndType(rt, rt2));
1177 VIXL_ASSERT(addr.IsImmediateOffset());
1178
1179 unsigned size =
1180 CalcLSPairDataSize(static_cast<LoadStorePairOp>(op & LoadStorePairMask));
1181 VIXL_ASSERT(IsImmLSPair(addr.GetOffset(), size));
1182 int offset = static_cast<int>(addr.GetOffset());
1183 Emit(op | Rt(rt) | Rt2(rt2) | RnSP(addr.GetBaseRegister()) |
1184 ImmLSPair(offset, size));
1185 }
1186
1187
1188 // Memory instructions.
ldrb(const Register & rt,const MemOperand & src,LoadStoreScalingOption option)1189 void Assembler::ldrb(const Register& rt,
1190 const MemOperand& src,
1191 LoadStoreScalingOption option) {
1192 VIXL_ASSERT(option != RequireUnscaledOffset);
1193 VIXL_ASSERT(option != PreferUnscaledOffset);
1194 LoadStore(rt, src, LDRB_w, option);
1195 }
1196
1197
strb(const Register & rt,const MemOperand & dst,LoadStoreScalingOption option)1198 void Assembler::strb(const Register& rt,
1199 const MemOperand& dst,
1200 LoadStoreScalingOption option) {
1201 VIXL_ASSERT(option != RequireUnscaledOffset);
1202 VIXL_ASSERT(option != PreferUnscaledOffset);
1203 LoadStore(rt, dst, STRB_w, option);
1204 }
1205
1206
ldrsb(const Register & rt,const MemOperand & src,LoadStoreScalingOption option)1207 void Assembler::ldrsb(const Register& rt,
1208 const MemOperand& src,
1209 LoadStoreScalingOption option) {
1210 VIXL_ASSERT(option != RequireUnscaledOffset);
1211 VIXL_ASSERT(option != PreferUnscaledOffset);
1212 LoadStore(rt, src, rt.Is64Bits() ? LDRSB_x : LDRSB_w, option);
1213 }
1214
1215
ldrh(const Register & rt,const MemOperand & src,LoadStoreScalingOption option)1216 void Assembler::ldrh(const Register& rt,
1217 const MemOperand& src,
1218 LoadStoreScalingOption option) {
1219 VIXL_ASSERT(option != RequireUnscaledOffset);
1220 VIXL_ASSERT(option != PreferUnscaledOffset);
1221 LoadStore(rt, src, LDRH_w, option);
1222 }
1223
1224
strh(const Register & rt,const MemOperand & dst,LoadStoreScalingOption option)1225 void Assembler::strh(const Register& rt,
1226 const MemOperand& dst,
1227 LoadStoreScalingOption option) {
1228 VIXL_ASSERT(option != RequireUnscaledOffset);
1229 VIXL_ASSERT(option != PreferUnscaledOffset);
1230 LoadStore(rt, dst, STRH_w, option);
1231 }
1232
1233
ldrsh(const Register & rt,const MemOperand & src,LoadStoreScalingOption option)1234 void Assembler::ldrsh(const Register& rt,
1235 const MemOperand& src,
1236 LoadStoreScalingOption option) {
1237 VIXL_ASSERT(option != RequireUnscaledOffset);
1238 VIXL_ASSERT(option != PreferUnscaledOffset);
1239 LoadStore(rt, src, rt.Is64Bits() ? LDRSH_x : LDRSH_w, option);
1240 }
1241
1242
ldr(const CPURegister & rt,const MemOperand & src,LoadStoreScalingOption option)1243 void Assembler::ldr(const CPURegister& rt,
1244 const MemOperand& src,
1245 LoadStoreScalingOption option) {
1246 VIXL_ASSERT(option != RequireUnscaledOffset);
1247 VIXL_ASSERT(option != PreferUnscaledOffset);
1248 LoadStore(rt, src, LoadOpFor(rt), option);
1249 }
1250
1251
str(const CPURegister & rt,const MemOperand & dst,LoadStoreScalingOption option)1252 void Assembler::str(const CPURegister& rt,
1253 const MemOperand& dst,
1254 LoadStoreScalingOption option) {
1255 VIXL_ASSERT(option != RequireUnscaledOffset);
1256 VIXL_ASSERT(option != PreferUnscaledOffset);
1257 LoadStore(rt, dst, StoreOpFor(rt), option);
1258 }
1259
1260
ldrsw(const Register & xt,const MemOperand & src,LoadStoreScalingOption option)1261 void Assembler::ldrsw(const Register& xt,
1262 const MemOperand& src,
1263 LoadStoreScalingOption option) {
1264 VIXL_ASSERT(xt.Is64Bits());
1265 VIXL_ASSERT(option != RequireUnscaledOffset);
1266 VIXL_ASSERT(option != PreferUnscaledOffset);
1267 LoadStore(xt, src, LDRSW_x, option);
1268 }
1269
1270
ldurb(const Register & rt,const MemOperand & src,LoadStoreScalingOption option)1271 void Assembler::ldurb(const Register& rt,
1272 const MemOperand& src,
1273 LoadStoreScalingOption option) {
1274 VIXL_ASSERT(option != RequireScaledOffset);
1275 VIXL_ASSERT(option != PreferScaledOffset);
1276 LoadStore(rt, src, LDRB_w, option);
1277 }
1278
1279
sturb(const Register & rt,const MemOperand & dst,LoadStoreScalingOption option)1280 void Assembler::sturb(const Register& rt,
1281 const MemOperand& dst,
1282 LoadStoreScalingOption option) {
1283 VIXL_ASSERT(option != RequireScaledOffset);
1284 VIXL_ASSERT(option != PreferScaledOffset);
1285 LoadStore(rt, dst, STRB_w, option);
1286 }
1287
1288
ldursb(const Register & rt,const MemOperand & src,LoadStoreScalingOption option)1289 void Assembler::ldursb(const Register& rt,
1290 const MemOperand& src,
1291 LoadStoreScalingOption option) {
1292 VIXL_ASSERT(option != RequireScaledOffset);
1293 VIXL_ASSERT(option != PreferScaledOffset);
1294 LoadStore(rt, src, rt.Is64Bits() ? LDRSB_x : LDRSB_w, option);
1295 }
1296
1297
ldurh(const Register & rt,const MemOperand & src,LoadStoreScalingOption option)1298 void Assembler::ldurh(const Register& rt,
1299 const MemOperand& src,
1300 LoadStoreScalingOption option) {
1301 VIXL_ASSERT(option != RequireScaledOffset);
1302 VIXL_ASSERT(option != PreferScaledOffset);
1303 LoadStore(rt, src, LDRH_w, option);
1304 }
1305
1306
sturh(const Register & rt,const MemOperand & dst,LoadStoreScalingOption option)1307 void Assembler::sturh(const Register& rt,
1308 const MemOperand& dst,
1309 LoadStoreScalingOption option) {
1310 VIXL_ASSERT(option != RequireScaledOffset);
1311 VIXL_ASSERT(option != PreferScaledOffset);
1312 LoadStore(rt, dst, STRH_w, option);
1313 }
1314
1315
ldursh(const Register & rt,const MemOperand & src,LoadStoreScalingOption option)1316 void Assembler::ldursh(const Register& rt,
1317 const MemOperand& src,
1318 LoadStoreScalingOption option) {
1319 VIXL_ASSERT(option != RequireScaledOffset);
1320 VIXL_ASSERT(option != PreferScaledOffset);
1321 LoadStore(rt, src, rt.Is64Bits() ? LDRSH_x : LDRSH_w, option);
1322 }
1323
1324
ldur(const CPURegister & rt,const MemOperand & src,LoadStoreScalingOption option)1325 void Assembler::ldur(const CPURegister& rt,
1326 const MemOperand& src,
1327 LoadStoreScalingOption option) {
1328 VIXL_ASSERT(option != RequireScaledOffset);
1329 VIXL_ASSERT(option != PreferScaledOffset);
1330 LoadStore(rt, src, LoadOpFor(rt), option);
1331 }
1332
1333
stur(const CPURegister & rt,const MemOperand & dst,LoadStoreScalingOption option)1334 void Assembler::stur(const CPURegister& rt,
1335 const MemOperand& dst,
1336 LoadStoreScalingOption option) {
1337 VIXL_ASSERT(option != RequireScaledOffset);
1338 VIXL_ASSERT(option != PreferScaledOffset);
1339 LoadStore(rt, dst, StoreOpFor(rt), option);
1340 }
1341
1342
ldursw(const Register & xt,const MemOperand & src,LoadStoreScalingOption option)1343 void Assembler::ldursw(const Register& xt,
1344 const MemOperand& src,
1345 LoadStoreScalingOption option) {
1346 VIXL_ASSERT(xt.Is64Bits());
1347 VIXL_ASSERT(option != RequireScaledOffset);
1348 VIXL_ASSERT(option != PreferScaledOffset);
1349 LoadStore(xt, src, LDRSW_x, option);
1350 }
1351
1352
ldraa(const Register & xt,const MemOperand & src)1353 void Assembler::ldraa(const Register& xt, const MemOperand& src) {
1354 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
1355 LoadStorePAC(xt, src, LDRAA);
1356 }
1357
1358
ldrab(const Register & xt,const MemOperand & src)1359 void Assembler::ldrab(const Register& xt, const MemOperand& src) {
1360 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
1361 LoadStorePAC(xt, src, LDRAB);
1362 }
1363
1364
ldrsw(const Register & xt,RawLiteral * literal)1365 void Assembler::ldrsw(const Register& xt, RawLiteral* literal) {
1366 VIXL_ASSERT(xt.Is64Bits());
1367 VIXL_ASSERT(literal->GetSize() == kWRegSizeInBytes);
1368 ldrsw(xt, static_cast<int>(LinkAndGetWordOffsetTo(literal)));
1369 }
1370
1371
ldr(const CPURegister & rt,RawLiteral * literal)1372 void Assembler::ldr(const CPURegister& rt, RawLiteral* literal) {
1373 VIXL_ASSERT(CPUHas(rt));
1374 VIXL_ASSERT(literal->GetSize() == static_cast<size_t>(rt.GetSizeInBytes()));
1375 ldr(rt, static_cast<int>(LinkAndGetWordOffsetTo(literal)));
1376 }
1377
1378
ldrsw(const Register & rt,int64_t imm19)1379 void Assembler::ldrsw(const Register& rt, int64_t imm19) {
1380 Emit(LDRSW_x_lit | ImmLLiteral(imm19) | Rt(rt));
1381 }
1382
1383
ldr(const CPURegister & rt,int64_t imm19)1384 void Assembler::ldr(const CPURegister& rt, int64_t imm19) {
1385 VIXL_ASSERT(CPUHas(rt));
1386 LoadLiteralOp op = LoadLiteralOpFor(rt);
1387 Emit(op | ImmLLiteral(imm19) | Rt(rt));
1388 }
1389
1390
prfm(int op,int64_t imm19)1391 void Assembler::prfm(int op, int64_t imm19) {
1392 Emit(PRFM_lit | ImmPrefetchOperation(op) | ImmLLiteral(imm19));
1393 }
1394
prfm(PrefetchOperation op,int64_t imm19)1395 void Assembler::prfm(PrefetchOperation op, int64_t imm19) {
1396 // Passing unnamed values in 'op' is undefined behaviour in C++.
1397 VIXL_ASSERT(IsNamedPrefetchOperation(op));
1398 prfm(static_cast<int>(op), imm19);
1399 }
1400
1401
1402 // Exclusive-access instructions.
stxrb(const Register & rs,const Register & rt,const MemOperand & dst)1403 void Assembler::stxrb(const Register& rs,
1404 const Register& rt,
1405 const MemOperand& dst) {
1406 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1407 Emit(STXRB_w | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.GetBaseRegister()));
1408 }
1409
1410
stxrh(const Register & rs,const Register & rt,const MemOperand & dst)1411 void Assembler::stxrh(const Register& rs,
1412 const Register& rt,
1413 const MemOperand& dst) {
1414 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1415 Emit(STXRH_w | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.GetBaseRegister()));
1416 }
1417
1418
stxr(const Register & rs,const Register & rt,const MemOperand & dst)1419 void Assembler::stxr(const Register& rs,
1420 const Register& rt,
1421 const MemOperand& dst) {
1422 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1423 LoadStoreExclusive op = rt.Is64Bits() ? STXR_x : STXR_w;
1424 Emit(op | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.GetBaseRegister()));
1425 }
1426
1427
ldxrb(const Register & rt,const MemOperand & src)1428 void Assembler::ldxrb(const Register& rt, const MemOperand& src) {
1429 VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1430 Emit(LDXRB_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister()));
1431 }
1432
1433
ldxrh(const Register & rt,const MemOperand & src)1434 void Assembler::ldxrh(const Register& rt, const MemOperand& src) {
1435 VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1436 Emit(LDXRH_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister()));
1437 }
1438
1439
ldxr(const Register & rt,const MemOperand & src)1440 void Assembler::ldxr(const Register& rt, const MemOperand& src) {
1441 VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1442 LoadStoreExclusive op = rt.Is64Bits() ? LDXR_x : LDXR_w;
1443 Emit(op | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister()));
1444 }
1445
1446
stxp(const Register & rs,const Register & rt,const Register & rt2,const MemOperand & dst)1447 void Assembler::stxp(const Register& rs,
1448 const Register& rt,
1449 const Register& rt2,
1450 const MemOperand& dst) {
1451 VIXL_ASSERT(rt.GetSizeInBits() == rt2.GetSizeInBits());
1452 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1453 LoadStoreExclusive op = rt.Is64Bits() ? STXP_x : STXP_w;
1454 Emit(op | Rs(rs) | Rt(rt) | Rt2(rt2) | RnSP(dst.GetBaseRegister()));
1455 }
1456
1457
ldxp(const Register & rt,const Register & rt2,const MemOperand & src)1458 void Assembler::ldxp(const Register& rt,
1459 const Register& rt2,
1460 const MemOperand& src) {
1461 VIXL_ASSERT(rt.GetSizeInBits() == rt2.GetSizeInBits());
1462 VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1463 LoadStoreExclusive op = rt.Is64Bits() ? LDXP_x : LDXP_w;
1464 Emit(op | Rs_mask | Rt(rt) | Rt2(rt2) | RnSP(src.GetBaseRegister()));
1465 }
1466
1467
stlxrb(const Register & rs,const Register & rt,const MemOperand & dst)1468 void Assembler::stlxrb(const Register& rs,
1469 const Register& rt,
1470 const MemOperand& dst) {
1471 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1472 Emit(STLXRB_w | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.GetBaseRegister()));
1473 }
1474
1475
stlxrh(const Register & rs,const Register & rt,const MemOperand & dst)1476 void Assembler::stlxrh(const Register& rs,
1477 const Register& rt,
1478 const MemOperand& dst) {
1479 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1480 Emit(STLXRH_w | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.GetBaseRegister()));
1481 }
1482
1483
stlxr(const Register & rs,const Register & rt,const MemOperand & dst)1484 void Assembler::stlxr(const Register& rs,
1485 const Register& rt,
1486 const MemOperand& dst) {
1487 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1488 LoadStoreExclusive op = rt.Is64Bits() ? STLXR_x : STLXR_w;
1489 Emit(op | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.GetBaseRegister()));
1490 }
1491
1492
ldaxrb(const Register & rt,const MemOperand & src)1493 void Assembler::ldaxrb(const Register& rt, const MemOperand& src) {
1494 VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1495 Emit(LDAXRB_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister()));
1496 }
1497
1498
ldaxrh(const Register & rt,const MemOperand & src)1499 void Assembler::ldaxrh(const Register& rt, const MemOperand& src) {
1500 VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1501 Emit(LDAXRH_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister()));
1502 }
1503
1504
ldaxr(const Register & rt,const MemOperand & src)1505 void Assembler::ldaxr(const Register& rt, const MemOperand& src) {
1506 VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1507 LoadStoreExclusive op = rt.Is64Bits() ? LDAXR_x : LDAXR_w;
1508 Emit(op | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister()));
1509 }
1510
1511
stlxp(const Register & rs,const Register & rt,const Register & rt2,const MemOperand & dst)1512 void Assembler::stlxp(const Register& rs,
1513 const Register& rt,
1514 const Register& rt2,
1515 const MemOperand& dst) {
1516 VIXL_ASSERT(rt.GetSizeInBits() == rt2.GetSizeInBits());
1517 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1518 LoadStoreExclusive op = rt.Is64Bits() ? STLXP_x : STLXP_w;
1519 Emit(op | Rs(rs) | Rt(rt) | Rt2(rt2) | RnSP(dst.GetBaseRegister()));
1520 }
1521
1522
ldaxp(const Register & rt,const Register & rt2,const MemOperand & src)1523 void Assembler::ldaxp(const Register& rt,
1524 const Register& rt2,
1525 const MemOperand& src) {
1526 VIXL_ASSERT(rt.GetSizeInBits() == rt2.GetSizeInBits());
1527 VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1528 LoadStoreExclusive op = rt.Is64Bits() ? LDAXP_x : LDAXP_w;
1529 Emit(op | Rs_mask | Rt(rt) | Rt2(rt2) | RnSP(src.GetBaseRegister()));
1530 }
1531
1532
stlrb(const Register & rt,const MemOperand & dst)1533 void Assembler::stlrb(const Register& rt, const MemOperand& dst) {
1534 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1535 Emit(STLRB_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(dst.GetBaseRegister()));
1536 }
1537
stlurb(const Register & rt,const MemOperand & dst)1538 void Assembler::stlurb(const Register& rt, const MemOperand& dst) {
1539 VIXL_ASSERT(CPUHas(CPUFeatures::kRCpc, CPUFeatures::kRCpcImm));
1540 VIXL_ASSERT(dst.IsImmediateOffset() && IsImmLSUnscaled(dst.GetOffset()));
1541
1542 Instr base = RnSP(dst.GetBaseRegister());
1543 int64_t offset = dst.GetOffset();
1544 Emit(STLURB | Rt(rt) | base | ImmLS(static_cast<int>(offset)));
1545 }
1546
1547
stlrh(const Register & rt,const MemOperand & dst)1548 void Assembler::stlrh(const Register& rt, const MemOperand& dst) {
1549 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1550 Emit(STLRH_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(dst.GetBaseRegister()));
1551 }
1552
stlurh(const Register & rt,const MemOperand & dst)1553 void Assembler::stlurh(const Register& rt, const MemOperand& dst) {
1554 VIXL_ASSERT(CPUHas(CPUFeatures::kRCpc, CPUFeatures::kRCpcImm));
1555 VIXL_ASSERT(dst.IsImmediateOffset() && IsImmLSUnscaled(dst.GetOffset()));
1556
1557 Instr base = RnSP(dst.GetBaseRegister());
1558 int64_t offset = dst.GetOffset();
1559 Emit(STLURH | Rt(rt) | base | ImmLS(static_cast<int>(offset)));
1560 }
1561
1562
stlr(const Register & rt,const MemOperand & dst)1563 void Assembler::stlr(const Register& rt, const MemOperand& dst) {
1564 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1565 LoadStoreExclusive op = rt.Is64Bits() ? STLR_x : STLR_w;
1566 Emit(op | Rs_mask | Rt(rt) | Rt2_mask | RnSP(dst.GetBaseRegister()));
1567 }
1568
stlur(const Register & rt,const MemOperand & dst)1569 void Assembler::stlur(const Register& rt, const MemOperand& dst) {
1570 VIXL_ASSERT(CPUHas(CPUFeatures::kRCpc, CPUFeatures::kRCpcImm));
1571 VIXL_ASSERT(dst.IsImmediateOffset() && IsImmLSUnscaled(dst.GetOffset()));
1572
1573 Instr base = RnSP(dst.GetBaseRegister());
1574 int64_t offset = dst.GetOffset();
1575 Instr op = rt.Is64Bits() ? STLUR_x : STLUR_w;
1576 Emit(op | Rt(rt) | base | ImmLS(static_cast<int>(offset)));
1577 }
1578
1579
ldarb(const Register & rt,const MemOperand & src)1580 void Assembler::ldarb(const Register& rt, const MemOperand& src) {
1581 VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1582 Emit(LDARB_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister()));
1583 }
1584
1585
ldarh(const Register & rt,const MemOperand & src)1586 void Assembler::ldarh(const Register& rt, const MemOperand& src) {
1587 VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1588 Emit(LDARH_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister()));
1589 }
1590
1591
ldar(const Register & rt,const MemOperand & src)1592 void Assembler::ldar(const Register& rt, const MemOperand& src) {
1593 VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1594 LoadStoreExclusive op = rt.Is64Bits() ? LDAR_x : LDAR_w;
1595 Emit(op | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister()));
1596 }
1597
1598
stllrb(const Register & rt,const MemOperand & dst)1599 void Assembler::stllrb(const Register& rt, const MemOperand& dst) {
1600 VIXL_ASSERT(CPUHas(CPUFeatures::kLORegions));
1601 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1602 Emit(STLLRB | Rs_mask | Rt(rt) | Rt2_mask | RnSP(dst.GetBaseRegister()));
1603 }
1604
1605
stllrh(const Register & rt,const MemOperand & dst)1606 void Assembler::stllrh(const Register& rt, const MemOperand& dst) {
1607 VIXL_ASSERT(CPUHas(CPUFeatures::kLORegions));
1608 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1609 Emit(STLLRH | Rs_mask | Rt(rt) | Rt2_mask | RnSP(dst.GetBaseRegister()));
1610 }
1611
1612
stllr(const Register & rt,const MemOperand & dst)1613 void Assembler::stllr(const Register& rt, const MemOperand& dst) {
1614 VIXL_ASSERT(CPUHas(CPUFeatures::kLORegions));
1615 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1616 LoadStoreExclusive op = rt.Is64Bits() ? STLLR_x : STLLR_w;
1617 Emit(op | Rs_mask | Rt(rt) | Rt2_mask | RnSP(dst.GetBaseRegister()));
1618 }
1619
1620
ldlarb(const Register & rt,const MemOperand & src)1621 void Assembler::ldlarb(const Register& rt, const MemOperand& src) {
1622 VIXL_ASSERT(CPUHas(CPUFeatures::kLORegions));
1623 VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1624 Emit(LDLARB | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister()));
1625 }
1626
1627
ldlarh(const Register & rt,const MemOperand & src)1628 void Assembler::ldlarh(const Register& rt, const MemOperand& src) {
1629 VIXL_ASSERT(CPUHas(CPUFeatures::kLORegions));
1630 VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1631 Emit(LDLARH | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister()));
1632 }
1633
1634
ldlar(const Register & rt,const MemOperand & src)1635 void Assembler::ldlar(const Register& rt, const MemOperand& src) {
1636 VIXL_ASSERT(CPUHas(CPUFeatures::kLORegions));
1637 VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1638 LoadStoreExclusive op = rt.Is64Bits() ? LDLAR_x : LDLAR_w;
1639 Emit(op | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister()));
1640 }
1641
1642
1643 // clang-format off
1644 #define COMPARE_AND_SWAP_W_X_LIST(V) \
1645 V(cas, CAS) \
1646 V(casa, CASA) \
1647 V(casl, CASL) \
1648 V(casal, CASAL)
1649 // clang-format on
1650
1651 #define VIXL_DEFINE_ASM_FUNC(FN, OP) \
1652 void Assembler::FN(const Register& rs, \
1653 const Register& rt, \
1654 const MemOperand& src) { \
1655 VIXL_ASSERT(CPUHas(CPUFeatures::kAtomics)); \
1656 VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0)); \
1657 VIXL_ASSERT(AreSameFormat(rs, rt)); \
1658 LoadStoreExclusive op = rt.Is64Bits() ? OP##_x : OP##_w; \
1659 Emit(op | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister())); \
1660 }
1661 COMPARE_AND_SWAP_W_X_LIST(VIXL_DEFINE_ASM_FUNC)
1662 #undef VIXL_DEFINE_ASM_FUNC
1663
1664 // clang-format off
1665 #define COMPARE_AND_SWAP_W_LIST(V) \
1666 V(casb, CASB) \
1667 V(casab, CASAB) \
1668 V(caslb, CASLB) \
1669 V(casalb, CASALB) \
1670 V(cash, CASH) \
1671 V(casah, CASAH) \
1672 V(caslh, CASLH) \
1673 V(casalh, CASALH)
1674 // clang-format on
1675
1676 #define VIXL_DEFINE_ASM_FUNC(FN, OP) \
1677 void Assembler::FN(const Register& rs, \
1678 const Register& rt, \
1679 const MemOperand& src) { \
1680 VIXL_ASSERT(CPUHas(CPUFeatures::kAtomics)); \
1681 VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0)); \
1682 Emit(OP | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister())); \
1683 }
COMPARE_AND_SWAP_W_LIST(VIXL_DEFINE_ASM_FUNC)1684 COMPARE_AND_SWAP_W_LIST(VIXL_DEFINE_ASM_FUNC)
1685 #undef VIXL_DEFINE_ASM_FUNC
1686
1687
1688 // clang-format off
1689 #define COMPARE_AND_SWAP_PAIR_LIST(V) \
1690 V(casp, CASP) \
1691 V(caspa, CASPA) \
1692 V(caspl, CASPL) \
1693 V(caspal, CASPAL)
1694 // clang-format on
1695
1696 #define VIXL_DEFINE_ASM_FUNC(FN, OP) \
1697 void Assembler::FN(const Register& rs, \
1698 const Register& rs1, \
1699 const Register& rt, \
1700 const Register& rt1, \
1701 const MemOperand& src) { \
1702 VIXL_ASSERT(CPUHas(CPUFeatures::kAtomics)); \
1703 USE(rs1, rt1); \
1704 VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0)); \
1705 VIXL_ASSERT(AreEven(rs, rt)); \
1706 VIXL_ASSERT(AreConsecutive(rs, rs1)); \
1707 VIXL_ASSERT(AreConsecutive(rt, rt1)); \
1708 VIXL_ASSERT(AreSameFormat(rs, rs1, rt, rt1)); \
1709 LoadStoreExclusive op = rt.Is64Bits() ? OP##_x : OP##_w; \
1710 Emit(op | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister())); \
1711 }
1712 COMPARE_AND_SWAP_PAIR_LIST(VIXL_DEFINE_ASM_FUNC)
1713 #undef VIXL_DEFINE_ASM_FUNC
1714
1715 // These macros generate all the variations of the atomic memory operations,
1716 // e.g. ldadd, ldadda, ldaddb, staddl, etc.
1717 // For a full list of the methods with comments, see the assembler header file.
1718
1719 // clang-format off
1720 #define ATOMIC_MEMORY_SIMPLE_OPERATION_LIST(V, DEF) \
1721 V(DEF, add, LDADD) \
1722 V(DEF, clr, LDCLR) \
1723 V(DEF, eor, LDEOR) \
1724 V(DEF, set, LDSET) \
1725 V(DEF, smax, LDSMAX) \
1726 V(DEF, smin, LDSMIN) \
1727 V(DEF, umax, LDUMAX) \
1728 V(DEF, umin, LDUMIN)
1729
1730 #define ATOMIC_MEMORY_STORE_MODES(V, NAME, OP) \
1731 V(NAME, OP##_x, OP##_w) \
1732 V(NAME##l, OP##L_x, OP##L_w) \
1733 V(NAME##b, OP##B, OP##B) \
1734 V(NAME##lb, OP##LB, OP##LB) \
1735 V(NAME##h, OP##H, OP##H) \
1736 V(NAME##lh, OP##LH, OP##LH)
1737
1738 #define ATOMIC_MEMORY_LOAD_MODES(V, NAME, OP) \
1739 ATOMIC_MEMORY_STORE_MODES(V, NAME, OP) \
1740 V(NAME##a, OP##A_x, OP##A_w) \
1741 V(NAME##al, OP##AL_x, OP##AL_w) \
1742 V(NAME##ab, OP##AB, OP##AB) \
1743 V(NAME##alb, OP##ALB, OP##ALB) \
1744 V(NAME##ah, OP##AH, OP##AH) \
1745 V(NAME##alh, OP##ALH, OP##ALH)
1746 // clang-format on
1747
1748 #define DEFINE_ASM_LOAD_FUNC(FN, OP_X, OP_W) \
1749 void Assembler::ld##FN(const Register& rs, \
1750 const Register& rt, \
1751 const MemOperand& src) { \
1752 VIXL_ASSERT(CPUHas(CPUFeatures::kAtomics)); \
1753 VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0)); \
1754 AtomicMemoryOp op = rt.Is64Bits() ? OP_X : OP_W; \
1755 Emit(op | Rs(rs) | Rt(rt) | RnSP(src.GetBaseRegister())); \
1756 }
1757 #define DEFINE_ASM_STORE_FUNC(FN, OP_X, OP_W) \
1758 void Assembler::st##FN(const Register& rs, const MemOperand& src) { \
1759 VIXL_ASSERT(CPUHas(CPUFeatures::kAtomics)); \
1760 ld##FN(rs, AppropriateZeroRegFor(rs), src); \
1761 }
1762
1763 ATOMIC_MEMORY_SIMPLE_OPERATION_LIST(ATOMIC_MEMORY_LOAD_MODES,
1764 DEFINE_ASM_LOAD_FUNC)
1765 ATOMIC_MEMORY_SIMPLE_OPERATION_LIST(ATOMIC_MEMORY_STORE_MODES,
1766 DEFINE_ASM_STORE_FUNC)
1767
1768 #define DEFINE_ASM_SWP_FUNC(FN, OP_X, OP_W) \
1769 void Assembler::FN(const Register& rs, \
1770 const Register& rt, \
1771 const MemOperand& src) { \
1772 VIXL_ASSERT(CPUHas(CPUFeatures::kAtomics)); \
1773 VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0)); \
1774 AtomicMemoryOp op = rt.Is64Bits() ? OP_X : OP_W; \
1775 Emit(op | Rs(rs) | Rt(rt) | RnSP(src.GetBaseRegister())); \
1776 }
1777
1778 ATOMIC_MEMORY_LOAD_MODES(DEFINE_ASM_SWP_FUNC, swp, SWP)
1779
1780 #undef DEFINE_ASM_LOAD_FUNC
1781 #undef DEFINE_ASM_STORE_FUNC
1782 #undef DEFINE_ASM_SWP_FUNC
1783
1784
1785 void Assembler::ldaprb(const Register& rt, const MemOperand& src) {
1786 VIXL_ASSERT(CPUHas(CPUFeatures::kRCpc));
1787 VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1788 AtomicMemoryOp op = LDAPRB;
1789 Emit(op | Rs(xzr) | Rt(rt) | RnSP(src.GetBaseRegister()));
1790 }
1791
ldapurb(const Register & rt,const MemOperand & src)1792 void Assembler::ldapurb(const Register& rt, const MemOperand& src) {
1793 VIXL_ASSERT(CPUHas(CPUFeatures::kRCpc, CPUFeatures::kRCpcImm));
1794 VIXL_ASSERT(src.IsImmediateOffset() && IsImmLSUnscaled(src.GetOffset()));
1795
1796 Instr base = RnSP(src.GetBaseRegister());
1797 int64_t offset = src.GetOffset();
1798 Emit(LDAPURB | Rt(rt) | base | ImmLS(static_cast<int>(offset)));
1799 }
1800
ldapursb(const Register & rt,const MemOperand & src)1801 void Assembler::ldapursb(const Register& rt, const MemOperand& src) {
1802 VIXL_ASSERT(CPUHas(CPUFeatures::kRCpc, CPUFeatures::kRCpcImm));
1803 VIXL_ASSERT(src.IsImmediateOffset() && IsImmLSUnscaled(src.GetOffset()));
1804
1805 Instr base = RnSP(src.GetBaseRegister());
1806 int64_t offset = src.GetOffset();
1807 Instr op = rt.Is64Bits() ? LDAPURSB_x : LDAPURSB_w;
1808 Emit(op | Rt(rt) | base | ImmLS(static_cast<int>(offset)));
1809 }
1810
ldaprh(const Register & rt,const MemOperand & src)1811 void Assembler::ldaprh(const Register& rt, const MemOperand& src) {
1812 VIXL_ASSERT(CPUHas(CPUFeatures::kRCpc));
1813 VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1814 AtomicMemoryOp op = LDAPRH;
1815 Emit(op | Rs(xzr) | Rt(rt) | RnSP(src.GetBaseRegister()));
1816 }
1817
ldapurh(const Register & rt,const MemOperand & src)1818 void Assembler::ldapurh(const Register& rt, const MemOperand& src) {
1819 VIXL_ASSERT(CPUHas(CPUFeatures::kRCpc, CPUFeatures::kRCpcImm));
1820 VIXL_ASSERT(src.IsImmediateOffset() && IsImmLSUnscaled(src.GetOffset()));
1821
1822 Instr base = RnSP(src.GetBaseRegister());
1823 int64_t offset = src.GetOffset();
1824 Emit(LDAPURH | Rt(rt) | base | ImmLS(static_cast<int>(offset)));
1825 }
1826
ldapursh(const Register & rt,const MemOperand & src)1827 void Assembler::ldapursh(const Register& rt, const MemOperand& src) {
1828 VIXL_ASSERT(CPUHas(CPUFeatures::kRCpc, CPUFeatures::kRCpcImm));
1829 VIXL_ASSERT(src.IsImmediateOffset() && IsImmLSUnscaled(src.GetOffset()));
1830
1831 Instr base = RnSP(src.GetBaseRegister());
1832 int64_t offset = src.GetOffset();
1833 LoadStoreRCpcUnscaledOffsetOp op = rt.Is64Bits() ? LDAPURSH_x : LDAPURSH_w;
1834 Emit(op | Rt(rt) | base | ImmLS(static_cast<int>(offset)));
1835 }
1836
ldapr(const Register & rt,const MemOperand & src)1837 void Assembler::ldapr(const Register& rt, const MemOperand& src) {
1838 VIXL_ASSERT(CPUHas(CPUFeatures::kRCpc));
1839 VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1840 AtomicMemoryOp op = rt.Is64Bits() ? LDAPR_x : LDAPR_w;
1841 Emit(op | Rs(xzr) | Rt(rt) | RnSP(src.GetBaseRegister()));
1842 }
1843
ldapur(const Register & rt,const MemOperand & src)1844 void Assembler::ldapur(const Register& rt, const MemOperand& src) {
1845 VIXL_ASSERT(CPUHas(CPUFeatures::kRCpc, CPUFeatures::kRCpcImm));
1846 VIXL_ASSERT(src.IsImmediateOffset() && IsImmLSUnscaled(src.GetOffset()));
1847
1848 Instr base = RnSP(src.GetBaseRegister());
1849 int64_t offset = src.GetOffset();
1850 LoadStoreRCpcUnscaledOffsetOp op = rt.Is64Bits() ? LDAPUR_x : LDAPUR_w;
1851 Emit(op | Rt(rt) | base | ImmLS(static_cast<int>(offset)));
1852 }
1853
ldapursw(const Register & rt,const MemOperand & src)1854 void Assembler::ldapursw(const Register& rt, const MemOperand& src) {
1855 VIXL_ASSERT(CPUHas(CPUFeatures::kRCpc, CPUFeatures::kRCpcImm));
1856 VIXL_ASSERT(rt.Is64Bits());
1857 VIXL_ASSERT(src.IsImmediateOffset() && IsImmLSUnscaled(src.GetOffset()));
1858
1859 Instr base = RnSP(src.GetBaseRegister());
1860 int64_t offset = src.GetOffset();
1861 Emit(LDAPURSW | Rt(rt) | base | ImmLS(static_cast<int>(offset)));
1862 }
1863
prfm(int op,const MemOperand & address,LoadStoreScalingOption option)1864 void Assembler::prfm(int op,
1865 const MemOperand& address,
1866 LoadStoreScalingOption option) {
1867 VIXL_ASSERT(option != RequireUnscaledOffset);
1868 VIXL_ASSERT(option != PreferUnscaledOffset);
1869 Prefetch(op, address, option);
1870 }
1871
prfm(PrefetchOperation op,const MemOperand & address,LoadStoreScalingOption option)1872 void Assembler::prfm(PrefetchOperation op,
1873 const MemOperand& address,
1874 LoadStoreScalingOption option) {
1875 // Passing unnamed values in 'op' is undefined behaviour in C++.
1876 VIXL_ASSERT(IsNamedPrefetchOperation(op));
1877 prfm(static_cast<int>(op), address, option);
1878 }
1879
1880
prfum(int op,const MemOperand & address,LoadStoreScalingOption option)1881 void Assembler::prfum(int op,
1882 const MemOperand& address,
1883 LoadStoreScalingOption option) {
1884 VIXL_ASSERT(option != RequireScaledOffset);
1885 VIXL_ASSERT(option != PreferScaledOffset);
1886 Prefetch(op, address, option);
1887 }
1888
prfum(PrefetchOperation op,const MemOperand & address,LoadStoreScalingOption option)1889 void Assembler::prfum(PrefetchOperation op,
1890 const MemOperand& address,
1891 LoadStoreScalingOption option) {
1892 // Passing unnamed values in 'op' is undefined behaviour in C++.
1893 VIXL_ASSERT(IsNamedPrefetchOperation(op));
1894 prfum(static_cast<int>(op), address, option);
1895 }
1896
1897
prfm(int op,RawLiteral * literal)1898 void Assembler::prfm(int op, RawLiteral* literal) {
1899 prfm(op, static_cast<int>(LinkAndGetWordOffsetTo(literal)));
1900 }
1901
prfm(PrefetchOperation op,RawLiteral * literal)1902 void Assembler::prfm(PrefetchOperation op, RawLiteral* literal) {
1903 // Passing unnamed values in 'op' is undefined behaviour in C++.
1904 VIXL_ASSERT(IsNamedPrefetchOperation(op));
1905 prfm(static_cast<int>(op), literal);
1906 }
1907
1908
sys(int op1,int crn,int crm,int op2,const Register & xt)1909 void Assembler::sys(int op1, int crn, int crm, int op2, const Register& xt) {
1910 VIXL_ASSERT(xt.Is64Bits());
1911 Emit(SYS | ImmSysOp1(op1) | CRn(crn) | CRm(crm) | ImmSysOp2(op2) | Rt(xt));
1912 }
1913
1914
sys(int op,const Register & xt)1915 void Assembler::sys(int op, const Register& xt) {
1916 VIXL_ASSERT(xt.Is64Bits());
1917 Emit(SYS | SysOp(op) | Rt(xt));
1918 }
1919
1920
dc(DataCacheOp op,const Register & rt)1921 void Assembler::dc(DataCacheOp op, const Register& rt) {
1922 if (op == CVAP) VIXL_ASSERT(CPUHas(CPUFeatures::kDCPoP));
1923 if (op == CVADP) VIXL_ASSERT(CPUHas(CPUFeatures::kDCCVADP));
1924 sys(op, rt);
1925 }
1926
1927
ic(InstructionCacheOp op,const Register & rt)1928 void Assembler::ic(InstructionCacheOp op, const Register& rt) {
1929 VIXL_ASSERT(op == IVAU);
1930 sys(op, rt);
1931 }
1932
1933
hint(SystemHint code)1934 void Assembler::hint(SystemHint code) { hint(static_cast<int>(code)); }
1935
1936
hint(int imm7)1937 void Assembler::hint(int imm7) {
1938 VIXL_ASSERT(IsUint7(imm7));
1939 Emit(HINT | ImmHint(imm7) | Rt(xzr));
1940 }
1941
1942
1943 // MTE.
1944
addg(const Register & xd,const Register & xn,int offset,int tag_offset)1945 void Assembler::addg(const Register& xd,
1946 const Register& xn,
1947 int offset,
1948 int tag_offset) {
1949 VIXL_ASSERT(CPUHas(CPUFeatures::kMTE));
1950 VIXL_ASSERT(IsMultiple(offset, kMTETagGranuleInBytes));
1951
1952 Emit(0x91800000 | RdSP(xd) | RnSP(xn) |
1953 ImmUnsignedField<21, 16>(offset / kMTETagGranuleInBytes) |
1954 ImmUnsignedField<13, 10>(tag_offset));
1955 }
1956
gmi(const Register & xd,const Register & xn,const Register & xm)1957 void Assembler::gmi(const Register& xd,
1958 const Register& xn,
1959 const Register& xm) {
1960 VIXL_ASSERT(CPUHas(CPUFeatures::kMTE));
1961
1962 Emit(0x9ac01400 | Rd(xd) | RnSP(xn) | Rm(xm));
1963 }
1964
irg(const Register & xd,const Register & xn,const Register & xm)1965 void Assembler::irg(const Register& xd,
1966 const Register& xn,
1967 const Register& xm) {
1968 VIXL_ASSERT(CPUHas(CPUFeatures::kMTE));
1969
1970 Emit(0x9ac01000 | RdSP(xd) | RnSP(xn) | Rm(xm));
1971 }
1972
ldg(const Register & xt,const MemOperand & addr)1973 void Assembler::ldg(const Register& xt, const MemOperand& addr) {
1974 VIXL_ASSERT(CPUHas(CPUFeatures::kMTE));
1975 VIXL_ASSERT(addr.IsImmediateOffset());
1976 int offset = static_cast<int>(addr.GetOffset());
1977 VIXL_ASSERT(IsMultiple(offset, kMTETagGranuleInBytes));
1978
1979 Emit(0xd9600000 | Rt(xt) | RnSP(addr.GetBaseRegister()) |
1980 ImmField<20, 12>(offset / static_cast<int>(kMTETagGranuleInBytes)));
1981 }
1982
StoreTagHelper(const Register & xt,const MemOperand & addr,Instr op)1983 void Assembler::StoreTagHelper(const Register& xt,
1984 const MemOperand& addr,
1985 Instr op) {
1986 int offset = static_cast<int>(addr.GetOffset());
1987 VIXL_ASSERT(IsMultiple(offset, kMTETagGranuleInBytes));
1988
1989 Instr addr_mode;
1990 if (addr.IsImmediateOffset()) {
1991 addr_mode = 2;
1992 } else if (addr.IsImmediatePreIndex()) {
1993 addr_mode = 3;
1994 } else {
1995 VIXL_ASSERT(addr.IsImmediatePostIndex());
1996 addr_mode = 1;
1997 }
1998
1999 Emit(op | RdSP(xt) | RnSP(addr.GetBaseRegister()) | (addr_mode << 10) |
2000 ImmField<20, 12>(offset / static_cast<int>(kMTETagGranuleInBytes)));
2001 }
2002
st2g(const Register & xt,const MemOperand & addr)2003 void Assembler::st2g(const Register& xt, const MemOperand& addr) {
2004 VIXL_ASSERT(CPUHas(CPUFeatures::kMTE));
2005 StoreTagHelper(xt, addr, 0xd9a00000);
2006 }
2007
stg(const Register & xt,const MemOperand & addr)2008 void Assembler::stg(const Register& xt, const MemOperand& addr) {
2009 VIXL_ASSERT(CPUHas(CPUFeatures::kMTE));
2010 StoreTagHelper(xt, addr, 0xd9200000);
2011 }
2012
stgp(const Register & xt1,const Register & xt2,const MemOperand & addr)2013 void Assembler::stgp(const Register& xt1,
2014 const Register& xt2,
2015 const MemOperand& addr) {
2016 VIXL_ASSERT(CPUHas(CPUFeatures::kMTE));
2017 int offset = static_cast<int>(addr.GetOffset());
2018 VIXL_ASSERT(IsMultiple(offset, kMTETagGranuleInBytes));
2019
2020 Instr addr_mode;
2021 if (addr.IsImmediateOffset()) {
2022 addr_mode = 2;
2023 } else if (addr.IsImmediatePreIndex()) {
2024 addr_mode = 3;
2025 } else {
2026 VIXL_ASSERT(addr.IsImmediatePostIndex());
2027 addr_mode = 1;
2028 }
2029
2030 Emit(0x68000000 | RnSP(addr.GetBaseRegister()) | (addr_mode << 23) |
2031 ImmField<21, 15>(offset / static_cast<int>(kMTETagGranuleInBytes)) |
2032 Rt2(xt2) | Rt(xt1));
2033 }
2034
stz2g(const Register & xt,const MemOperand & addr)2035 void Assembler::stz2g(const Register& xt, const MemOperand& addr) {
2036 VIXL_ASSERT(CPUHas(CPUFeatures::kMTE));
2037 StoreTagHelper(xt, addr, 0xd9e00000);
2038 }
2039
stzg(const Register & xt,const MemOperand & addr)2040 void Assembler::stzg(const Register& xt, const MemOperand& addr) {
2041 VIXL_ASSERT(CPUHas(CPUFeatures::kMTE));
2042 StoreTagHelper(xt, addr, 0xd9600000);
2043 }
2044
subg(const Register & xd,const Register & xn,int offset,int tag_offset)2045 void Assembler::subg(const Register& xd,
2046 const Register& xn,
2047 int offset,
2048 int tag_offset) {
2049 VIXL_ASSERT(CPUHas(CPUFeatures::kMTE));
2050 VIXL_ASSERT(IsMultiple(offset, kMTETagGranuleInBytes));
2051
2052 Emit(0xd1800000 | RdSP(xd) | RnSP(xn) |
2053 ImmUnsignedField<21, 16>(offset / kMTETagGranuleInBytes) |
2054 ImmUnsignedField<13, 10>(tag_offset));
2055 }
2056
subp(const Register & xd,const Register & xn,const Register & xm)2057 void Assembler::subp(const Register& xd,
2058 const Register& xn,
2059 const Register& xm) {
2060 VIXL_ASSERT(CPUHas(CPUFeatures::kMTE));
2061
2062 Emit(0x9ac00000 | Rd(xd) | RnSP(xn) | RmSP(xm));
2063 }
2064
subps(const Register & xd,const Register & xn,const Register & xm)2065 void Assembler::subps(const Register& xd,
2066 const Register& xn,
2067 const Register& xm) {
2068 VIXL_ASSERT(CPUHas(CPUFeatures::kMTE));
2069
2070 Emit(0xbac00000 | Rd(xd) | RnSP(xn) | RmSP(xm));
2071 }
2072
cpye(const Register & rd,const Register & rs,const Register & rn)2073 void Assembler::cpye(const Register& rd,
2074 const Register& rs,
2075 const Register& rn) {
2076 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2077 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2078 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero() && !rs.IsZero());
2079
2080 Emit(0x1d800400 | Rd(rd) | Rn(rn) | Rs(rs));
2081 }
2082
cpyen(const Register & rd,const Register & rs,const Register & rn)2083 void Assembler::cpyen(const Register& rd,
2084 const Register& rs,
2085 const Register& rn) {
2086 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2087 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2088 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero() && !rs.IsZero());
2089
2090 Emit(0x1d80c400 | Rd(rd) | Rn(rn) | Rs(rs));
2091 }
2092
cpyern(const Register & rd,const Register & rs,const Register & rn)2093 void Assembler::cpyern(const Register& rd,
2094 const Register& rs,
2095 const Register& rn) {
2096 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2097 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2098 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero() && !rs.IsZero());
2099
2100 Emit(0x1d808400 | Rd(rd) | Rn(rn) | Rs(rs));
2101 }
2102
cpyewn(const Register & rd,const Register & rs,const Register & rn)2103 void Assembler::cpyewn(const Register& rd,
2104 const Register& rs,
2105 const Register& rn) {
2106 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2107 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2108 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero() && !rs.IsZero());
2109
2110 Emit(0x1d804400 | Rd(rd) | Rn(rn) | Rs(rs));
2111 }
2112
cpyfe(const Register & rd,const Register & rs,const Register & rn)2113 void Assembler::cpyfe(const Register& rd,
2114 const Register& rs,
2115 const Register& rn) {
2116 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2117 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2118 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero() && !rs.IsZero());
2119
2120 Emit(0x19800400 | Rd(rd) | Rn(rn) | Rs(rs));
2121 }
2122
cpyfen(const Register & rd,const Register & rs,const Register & rn)2123 void Assembler::cpyfen(const Register& rd,
2124 const Register& rs,
2125 const Register& rn) {
2126 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2127 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2128 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero() && !rs.IsZero());
2129
2130 Emit(0x1980c400 | Rd(rd) | Rn(rn) | Rs(rs));
2131 }
2132
cpyfern(const Register & rd,const Register & rs,const Register & rn)2133 void Assembler::cpyfern(const Register& rd,
2134 const Register& rs,
2135 const Register& rn) {
2136 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2137 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2138 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero() && !rs.IsZero());
2139
2140 Emit(0x19808400 | Rd(rd) | Rn(rn) | Rs(rs));
2141 }
2142
cpyfewn(const Register & rd,const Register & rs,const Register & rn)2143 void Assembler::cpyfewn(const Register& rd,
2144 const Register& rs,
2145 const Register& rn) {
2146 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2147 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2148 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero() && !rs.IsZero());
2149
2150 Emit(0x19804400 | Rd(rd) | Rn(rn) | Rs(rs));
2151 }
2152
cpyfm(const Register & rd,const Register & rs,const Register & rn)2153 void Assembler::cpyfm(const Register& rd,
2154 const Register& rs,
2155 const Register& rn) {
2156 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2157 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2158 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero() && !rs.IsZero());
2159
2160 Emit(0x19400400 | Rd(rd) | Rn(rn) | Rs(rs));
2161 }
2162
cpyfmn(const Register & rd,const Register & rs,const Register & rn)2163 void Assembler::cpyfmn(const Register& rd,
2164 const Register& rs,
2165 const Register& rn) {
2166 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2167 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2168 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero() && !rs.IsZero());
2169
2170 Emit(0x1940c400 | Rd(rd) | Rn(rn) | Rs(rs));
2171 }
2172
cpyfmrn(const Register & rd,const Register & rs,const Register & rn)2173 void Assembler::cpyfmrn(const Register& rd,
2174 const Register& rs,
2175 const Register& rn) {
2176 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2177 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2178 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero() && !rs.IsZero());
2179
2180 Emit(0x19408400 | Rd(rd) | Rn(rn) | Rs(rs));
2181 }
2182
cpyfmwn(const Register & rd,const Register & rs,const Register & rn)2183 void Assembler::cpyfmwn(const Register& rd,
2184 const Register& rs,
2185 const Register& rn) {
2186 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2187 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2188 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero() && !rs.IsZero());
2189
2190 Emit(0x19404400 | Rd(rd) | Rn(rn) | Rs(rs));
2191 }
2192
cpyfp(const Register & rd,const Register & rs,const Register & rn)2193 void Assembler::cpyfp(const Register& rd,
2194 const Register& rs,
2195 const Register& rn) {
2196 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2197 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2198 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero() && !rs.IsZero());
2199
2200 Emit(0x19000400 | Rd(rd) | Rn(rn) | Rs(rs));
2201 }
2202
cpyfpn(const Register & rd,const Register & rs,const Register & rn)2203 void Assembler::cpyfpn(const Register& rd,
2204 const Register& rs,
2205 const Register& rn) {
2206 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2207 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2208 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero() && !rs.IsZero());
2209
2210 Emit(0x1900c400 | Rd(rd) | Rn(rn) | Rs(rs));
2211 }
2212
cpyfprn(const Register & rd,const Register & rs,const Register & rn)2213 void Assembler::cpyfprn(const Register& rd,
2214 const Register& rs,
2215 const Register& rn) {
2216 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2217 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2218 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero() && !rs.IsZero());
2219
2220 Emit(0x19008400 | Rd(rd) | Rn(rn) | Rs(rs));
2221 }
2222
cpyfpwn(const Register & rd,const Register & rs,const Register & rn)2223 void Assembler::cpyfpwn(const Register& rd,
2224 const Register& rs,
2225 const Register& rn) {
2226 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2227 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2228 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero() && !rs.IsZero());
2229
2230 Emit(0x19004400 | Rd(rd) | Rn(rn) | Rs(rs));
2231 }
2232
cpym(const Register & rd,const Register & rs,const Register & rn)2233 void Assembler::cpym(const Register& rd,
2234 const Register& rs,
2235 const Register& rn) {
2236 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2237 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2238 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero() && !rs.IsZero());
2239
2240 Emit(0x1d400400 | Rd(rd) | Rn(rn) | Rs(rs));
2241 }
2242
cpymn(const Register & rd,const Register & rs,const Register & rn)2243 void Assembler::cpymn(const Register& rd,
2244 const Register& rs,
2245 const Register& rn) {
2246 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2247 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2248 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero() && !rs.IsZero());
2249
2250 Emit(0x1d40c400 | Rd(rd) | Rn(rn) | Rs(rs));
2251 }
2252
cpymrn(const Register & rd,const Register & rs,const Register & rn)2253 void Assembler::cpymrn(const Register& rd,
2254 const Register& rs,
2255 const Register& rn) {
2256 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2257 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2258 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero() && !rs.IsZero());
2259
2260 Emit(0x1d408400 | Rd(rd) | Rn(rn) | Rs(rs));
2261 }
2262
cpymwn(const Register & rd,const Register & rs,const Register & rn)2263 void Assembler::cpymwn(const Register& rd,
2264 const Register& rs,
2265 const Register& rn) {
2266 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2267 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2268 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero() && !rs.IsZero());
2269
2270 Emit(0x1d404400 | Rd(rd) | Rn(rn) | Rs(rs));
2271 }
2272
cpyp(const Register & rd,const Register & rs,const Register & rn)2273 void Assembler::cpyp(const Register& rd,
2274 const Register& rs,
2275 const Register& rn) {
2276 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2277 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2278 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero() && !rs.IsZero());
2279
2280 Emit(0x1d000400 | Rd(rd) | Rn(rn) | Rs(rs));
2281 }
2282
cpypn(const Register & rd,const Register & rs,const Register & rn)2283 void Assembler::cpypn(const Register& rd,
2284 const Register& rs,
2285 const Register& rn) {
2286 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2287 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2288 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero() && !rs.IsZero());
2289
2290 Emit(0x1d00c400 | Rd(rd) | Rn(rn) | Rs(rs));
2291 }
2292
cpyprn(const Register & rd,const Register & rs,const Register & rn)2293 void Assembler::cpyprn(const Register& rd,
2294 const Register& rs,
2295 const Register& rn) {
2296 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2297 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2298 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero() && !rs.IsZero());
2299
2300 Emit(0x1d008400 | Rd(rd) | Rn(rn) | Rs(rs));
2301 }
2302
cpypwn(const Register & rd,const Register & rs,const Register & rn)2303 void Assembler::cpypwn(const Register& rd,
2304 const Register& rs,
2305 const Register& rn) {
2306 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2307 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2308 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero() && !rs.IsZero());
2309
2310 Emit(0x1d004400 | Rd(rd) | Rn(rn) | Rs(rs));
2311 }
2312
sete(const Register & rd,const Register & rn,const Register & rs)2313 void Assembler::sete(const Register& rd,
2314 const Register& rn,
2315 const Register& rs) {
2316 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2317 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2318 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero());
2319
2320 Emit(0x19c08400 | Rd(rd) | Rn(rn) | Rs(rs));
2321 }
2322
seten(const Register & rd,const Register & rn,const Register & rs)2323 void Assembler::seten(const Register& rd,
2324 const Register& rn,
2325 const Register& rs) {
2326 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2327 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2328 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero());
2329
2330 Emit(0x19c0a400 | Rd(rd) | Rn(rn) | Rs(rs));
2331 }
2332
setge(const Register & rd,const Register & rn,const Register & rs)2333 void Assembler::setge(const Register& rd,
2334 const Register& rn,
2335 const Register& rs) {
2336 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2337 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2338 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero());
2339
2340 Emit(0x1dc08400 | Rd(rd) | Rn(rn) | Rs(rs));
2341 }
2342
setgen(const Register & rd,const Register & rn,const Register & rs)2343 void Assembler::setgen(const Register& rd,
2344 const Register& rn,
2345 const Register& rs) {
2346 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2347 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2348 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero());
2349
2350 Emit(0x1dc0a400 | Rd(rd) | Rn(rn) | Rs(rs));
2351 }
2352
setgm(const Register & rd,const Register & rn,const Register & rs)2353 void Assembler::setgm(const Register& rd,
2354 const Register& rn,
2355 const Register& rs) {
2356 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2357 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2358 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero());
2359
2360 Emit(0x1dc04400 | Rd(rd) | Rn(rn) | Rs(rs));
2361 }
2362
setgmn(const Register & rd,const Register & rn,const Register & rs)2363 void Assembler::setgmn(const Register& rd,
2364 const Register& rn,
2365 const Register& rs) {
2366 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2367 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2368 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero());
2369
2370 Emit(0x1dc06400 | Rd(rd) | Rn(rn) | Rs(rs));
2371 }
2372
setgp(const Register & rd,const Register & rn,const Register & rs)2373 void Assembler::setgp(const Register& rd,
2374 const Register& rn,
2375 const Register& rs) {
2376 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2377 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2378 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero());
2379
2380 Emit(0x1dc00400 | Rd(rd) | Rn(rn) | Rs(rs));
2381 }
2382
setgpn(const Register & rd,const Register & rn,const Register & rs)2383 void Assembler::setgpn(const Register& rd,
2384 const Register& rn,
2385 const Register& rs) {
2386 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2387 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2388 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero());
2389
2390 Emit(0x1dc02400 | Rd(rd) | Rn(rn) | Rs(rs));
2391 }
2392
setm(const Register & rd,const Register & rn,const Register & rs)2393 void Assembler::setm(const Register& rd,
2394 const Register& rn,
2395 const Register& rs) {
2396 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2397 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2398 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero());
2399
2400 Emit(0x19c04400 | Rd(rd) | Rn(rn) | Rs(rs));
2401 }
2402
setmn(const Register & rd,const Register & rn,const Register & rs)2403 void Assembler::setmn(const Register& rd,
2404 const Register& rn,
2405 const Register& rs) {
2406 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2407 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2408 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero());
2409
2410 Emit(0x19c06400 | Rd(rd) | Rn(rn) | Rs(rs));
2411 }
2412
setp(const Register & rd,const Register & rn,const Register & rs)2413 void Assembler::setp(const Register& rd,
2414 const Register& rn,
2415 const Register& rs) {
2416 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2417 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2418 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero());
2419
2420 Emit(0x19c00400 | Rd(rd) | Rn(rn) | Rs(rs));
2421 }
2422
setpn(const Register & rd,const Register & rn,const Register & rs)2423 void Assembler::setpn(const Register& rd,
2424 const Register& rn,
2425 const Register& rs) {
2426 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2427 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2428 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero());
2429
2430 Emit(0x19c02400 | Rd(rd) | Rn(rn) | Rs(rs));
2431 }
2432
abs(const Register & rd,const Register & rn)2433 void Assembler::abs(const Register& rd, const Register& rn) {
2434 VIXL_ASSERT(CPUHas(CPUFeatures::kCSSC));
2435 VIXL_ASSERT(rd.IsSameSizeAndType(rn));
2436
2437 Emit(0x5ac02000 | SF(rd) | Rd(rd) | Rn(rn));
2438 }
2439
cnt(const Register & rd,const Register & rn)2440 void Assembler::cnt(const Register& rd, const Register& rn) {
2441 VIXL_ASSERT(CPUHas(CPUFeatures::kCSSC));
2442 VIXL_ASSERT(rd.IsSameSizeAndType(rn));
2443
2444 Emit(0x5ac01c00 | SF(rd) | Rd(rd) | Rn(rn));
2445 }
2446
ctz(const Register & rd,const Register & rn)2447 void Assembler::ctz(const Register& rd, const Register& rn) {
2448 VIXL_ASSERT(CPUHas(CPUFeatures::kCSSC));
2449 VIXL_ASSERT(rd.IsSameSizeAndType(rn));
2450
2451 Emit(0x5ac01800 | SF(rd) | Rd(rd) | Rn(rn));
2452 }
2453
2454 #define MINMAX(V) \
2455 V(smax, 0x11c00000, 0x1ac06000, true) \
2456 V(smin, 0x11c80000, 0x1ac06800, true) \
2457 V(umax, 0x11c40000, 0x1ac06400, false) \
2458 V(umin, 0x11cc0000, 0x1ac06c00, false)
2459
2460 #define VIXL_DEFINE_ASM_FUNC(FN, IMMOP, REGOP, SIGNED) \
2461 void Assembler::FN(const Register& rd, \
2462 const Register& rn, \
2463 const Operand& op) { \
2464 VIXL_ASSERT(rd.IsSameSizeAndType(rn)); \
2465 Instr i = SF(rd) | Rd(rd) | Rn(rn); \
2466 if (op.IsImmediate()) { \
2467 int64_t imm = op.GetImmediate(); \
2468 i |= SIGNED ? ImmField<17, 10>(imm) : ImmUnsignedField<17, 10>(imm); \
2469 Emit(IMMOP | i); \
2470 } else { \
2471 VIXL_ASSERT(op.IsPlainRegister()); \
2472 VIXL_ASSERT(op.GetRegister().IsSameSizeAndType(rd)); \
2473 Emit(REGOP | i | Rm(op.GetRegister())); \
2474 } \
2475 }
MINMAX(VIXL_DEFINE_ASM_FUNC)2476 MINMAX(VIXL_DEFINE_ASM_FUNC)
2477 #undef VIXL_DEFINE_ASM_FUNC
2478
2479 // NEON structure loads and stores.
2480 Instr Assembler::LoadStoreStructAddrModeField(const MemOperand& addr) {
2481 Instr addr_field = RnSP(addr.GetBaseRegister());
2482
2483 if (addr.IsPostIndex()) {
2484 VIXL_STATIC_ASSERT(NEONLoadStoreMultiStructPostIndex ==
2485 static_cast<NEONLoadStoreMultiStructPostIndexOp>(
2486 NEONLoadStoreSingleStructPostIndex));
2487
2488 addr_field |= NEONLoadStoreMultiStructPostIndex;
2489 if (addr.GetOffset() == 0) {
2490 addr_field |= RmNot31(addr.GetRegisterOffset());
2491 } else {
2492 // The immediate post index addressing mode is indicated by rm = 31.
2493 // The immediate is implied by the number of vector registers used.
2494 addr_field |= (0x1f << Rm_offset);
2495 }
2496 } else {
2497 VIXL_ASSERT(addr.IsImmediateOffset() && (addr.GetOffset() == 0));
2498 }
2499 return addr_field;
2500 }
2501
LoadStoreStructVerify(const VRegister & vt,const MemOperand & addr,Instr op)2502 void Assembler::LoadStoreStructVerify(const VRegister& vt,
2503 const MemOperand& addr,
2504 Instr op) {
2505 #ifdef VIXL_DEBUG
2506 // Assert that addressing mode is either offset (with immediate 0), post
2507 // index by immediate of the size of the register list, or post index by a
2508 // value in a core register.
2509 VIXL_ASSERT(vt.HasSize() && vt.HasLaneSize());
2510 if (addr.IsImmediateOffset()) {
2511 VIXL_ASSERT(addr.GetOffset() == 0);
2512 } else {
2513 int offset = vt.GetSizeInBytes();
2514 switch (op) {
2515 case NEON_LD1_1v:
2516 case NEON_ST1_1v:
2517 offset *= 1;
2518 break;
2519 case NEONLoadStoreSingleStructLoad1:
2520 case NEONLoadStoreSingleStructStore1:
2521 case NEON_LD1R:
2522 offset = (offset / vt.GetLanes()) * 1;
2523 break;
2524
2525 case NEON_LD1_2v:
2526 case NEON_ST1_2v:
2527 case NEON_LD2:
2528 case NEON_ST2:
2529 offset *= 2;
2530 break;
2531 case NEONLoadStoreSingleStructLoad2:
2532 case NEONLoadStoreSingleStructStore2:
2533 case NEON_LD2R:
2534 offset = (offset / vt.GetLanes()) * 2;
2535 break;
2536
2537 case NEON_LD1_3v:
2538 case NEON_ST1_3v:
2539 case NEON_LD3:
2540 case NEON_ST3:
2541 offset *= 3;
2542 break;
2543 case NEONLoadStoreSingleStructLoad3:
2544 case NEONLoadStoreSingleStructStore3:
2545 case NEON_LD3R:
2546 offset = (offset / vt.GetLanes()) * 3;
2547 break;
2548
2549 case NEON_LD1_4v:
2550 case NEON_ST1_4v:
2551 case NEON_LD4:
2552 case NEON_ST4:
2553 offset *= 4;
2554 break;
2555 case NEONLoadStoreSingleStructLoad4:
2556 case NEONLoadStoreSingleStructStore4:
2557 case NEON_LD4R:
2558 offset = (offset / vt.GetLanes()) * 4;
2559 break;
2560 default:
2561 VIXL_UNREACHABLE();
2562 }
2563 VIXL_ASSERT(!addr.GetRegisterOffset().Is(NoReg) ||
2564 addr.GetOffset() == offset);
2565 }
2566 #else
2567 USE(vt, addr, op);
2568 #endif
2569 }
2570
LoadStoreStruct(const VRegister & vt,const MemOperand & addr,NEONLoadStoreMultiStructOp op)2571 void Assembler::LoadStoreStruct(const VRegister& vt,
2572 const MemOperand& addr,
2573 NEONLoadStoreMultiStructOp op) {
2574 LoadStoreStructVerify(vt, addr, op);
2575 VIXL_ASSERT(vt.IsVector() || vt.Is1D());
2576 Emit(op | LoadStoreStructAddrModeField(addr) | LSVFormat(vt) | Rt(vt));
2577 }
2578
2579
LoadStoreStructSingleAllLanes(const VRegister & vt,const MemOperand & addr,NEONLoadStoreSingleStructOp op)2580 void Assembler::LoadStoreStructSingleAllLanes(const VRegister& vt,
2581 const MemOperand& addr,
2582 NEONLoadStoreSingleStructOp op) {
2583 LoadStoreStructVerify(vt, addr, op);
2584 Emit(op | LoadStoreStructAddrModeField(addr) | LSVFormat(vt) | Rt(vt));
2585 }
2586
2587
ld1(const VRegister & vt,const MemOperand & src)2588 void Assembler::ld1(const VRegister& vt, const MemOperand& src) {
2589 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2590 LoadStoreStruct(vt, src, NEON_LD1_1v);
2591 }
2592
2593
ld1(const VRegister & vt,const VRegister & vt2,const MemOperand & src)2594 void Assembler::ld1(const VRegister& vt,
2595 const VRegister& vt2,
2596 const MemOperand& src) {
2597 USE(vt2);
2598 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2599 VIXL_ASSERT(AreSameFormat(vt, vt2));
2600 VIXL_ASSERT(AreConsecutive(vt, vt2));
2601 LoadStoreStruct(vt, src, NEON_LD1_2v);
2602 }
2603
2604
ld1(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const MemOperand & src)2605 void Assembler::ld1(const VRegister& vt,
2606 const VRegister& vt2,
2607 const VRegister& vt3,
2608 const MemOperand& src) {
2609 USE(vt2, vt3);
2610 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2611 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
2612 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
2613 LoadStoreStruct(vt, src, NEON_LD1_3v);
2614 }
2615
2616
ld1(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const VRegister & vt4,const MemOperand & src)2617 void Assembler::ld1(const VRegister& vt,
2618 const VRegister& vt2,
2619 const VRegister& vt3,
2620 const VRegister& vt4,
2621 const MemOperand& src) {
2622 USE(vt2, vt3, vt4);
2623 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2624 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
2625 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
2626 LoadStoreStruct(vt, src, NEON_LD1_4v);
2627 }
2628
2629
ld2(const VRegister & vt,const VRegister & vt2,const MemOperand & src)2630 void Assembler::ld2(const VRegister& vt,
2631 const VRegister& vt2,
2632 const MemOperand& src) {
2633 USE(vt2);
2634 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2635 VIXL_ASSERT(AreSameFormat(vt, vt2));
2636 VIXL_ASSERT(AreConsecutive(vt, vt2));
2637 LoadStoreStruct(vt, src, NEON_LD2);
2638 }
2639
2640
ld2(const VRegister & vt,const VRegister & vt2,int lane,const MemOperand & src)2641 void Assembler::ld2(const VRegister& vt,
2642 const VRegister& vt2,
2643 int lane,
2644 const MemOperand& src) {
2645 USE(vt2);
2646 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2647 VIXL_ASSERT(AreSameFormat(vt, vt2));
2648 VIXL_ASSERT(AreConsecutive(vt, vt2));
2649 LoadStoreStructSingle(vt, lane, src, NEONLoadStoreSingleStructLoad2);
2650 }
2651
2652
ld2r(const VRegister & vt,const VRegister & vt2,const MemOperand & src)2653 void Assembler::ld2r(const VRegister& vt,
2654 const VRegister& vt2,
2655 const MemOperand& src) {
2656 USE(vt2);
2657 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2658 VIXL_ASSERT(AreSameFormat(vt, vt2));
2659 VIXL_ASSERT(AreConsecutive(vt, vt2));
2660 LoadStoreStructSingleAllLanes(vt, src, NEON_LD2R);
2661 }
2662
2663
ld3(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const MemOperand & src)2664 void Assembler::ld3(const VRegister& vt,
2665 const VRegister& vt2,
2666 const VRegister& vt3,
2667 const MemOperand& src) {
2668 USE(vt2, vt3);
2669 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2670 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
2671 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
2672 LoadStoreStruct(vt, src, NEON_LD3);
2673 }
2674
2675
ld3(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,int lane,const MemOperand & src)2676 void Assembler::ld3(const VRegister& vt,
2677 const VRegister& vt2,
2678 const VRegister& vt3,
2679 int lane,
2680 const MemOperand& src) {
2681 USE(vt2, vt3);
2682 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2683 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
2684 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
2685 LoadStoreStructSingle(vt, lane, src, NEONLoadStoreSingleStructLoad3);
2686 }
2687
2688
ld3r(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const MemOperand & src)2689 void Assembler::ld3r(const VRegister& vt,
2690 const VRegister& vt2,
2691 const VRegister& vt3,
2692 const MemOperand& src) {
2693 USE(vt2, vt3);
2694 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2695 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
2696 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
2697 LoadStoreStructSingleAllLanes(vt, src, NEON_LD3R);
2698 }
2699
2700
ld4(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const VRegister & vt4,const MemOperand & src)2701 void Assembler::ld4(const VRegister& vt,
2702 const VRegister& vt2,
2703 const VRegister& vt3,
2704 const VRegister& vt4,
2705 const MemOperand& src) {
2706 USE(vt2, vt3, vt4);
2707 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2708 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
2709 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
2710 LoadStoreStruct(vt, src, NEON_LD4);
2711 }
2712
2713
ld4(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const VRegister & vt4,int lane,const MemOperand & src)2714 void Assembler::ld4(const VRegister& vt,
2715 const VRegister& vt2,
2716 const VRegister& vt3,
2717 const VRegister& vt4,
2718 int lane,
2719 const MemOperand& src) {
2720 USE(vt2, vt3, vt4);
2721 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2722 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
2723 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
2724 LoadStoreStructSingle(vt, lane, src, NEONLoadStoreSingleStructLoad4);
2725 }
2726
2727
ld4r(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const VRegister & vt4,const MemOperand & src)2728 void Assembler::ld4r(const VRegister& vt,
2729 const VRegister& vt2,
2730 const VRegister& vt3,
2731 const VRegister& vt4,
2732 const MemOperand& src) {
2733 USE(vt2, vt3, vt4);
2734 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2735 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
2736 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
2737 LoadStoreStructSingleAllLanes(vt, src, NEON_LD4R);
2738 }
2739
2740
st1(const VRegister & vt,const MemOperand & src)2741 void Assembler::st1(const VRegister& vt, const MemOperand& src) {
2742 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2743 LoadStoreStruct(vt, src, NEON_ST1_1v);
2744 }
2745
2746
st1(const VRegister & vt,const VRegister & vt2,const MemOperand & src)2747 void Assembler::st1(const VRegister& vt,
2748 const VRegister& vt2,
2749 const MemOperand& src) {
2750 USE(vt2);
2751 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2752 VIXL_ASSERT(AreSameFormat(vt, vt2));
2753 VIXL_ASSERT(AreConsecutive(vt, vt2));
2754 LoadStoreStruct(vt, src, NEON_ST1_2v);
2755 }
2756
2757
st1(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const MemOperand & src)2758 void Assembler::st1(const VRegister& vt,
2759 const VRegister& vt2,
2760 const VRegister& vt3,
2761 const MemOperand& src) {
2762 USE(vt2, vt3);
2763 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2764 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
2765 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
2766 LoadStoreStruct(vt, src, NEON_ST1_3v);
2767 }
2768
2769
st1(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const VRegister & vt4,const MemOperand & src)2770 void Assembler::st1(const VRegister& vt,
2771 const VRegister& vt2,
2772 const VRegister& vt3,
2773 const VRegister& vt4,
2774 const MemOperand& src) {
2775 USE(vt2, vt3, vt4);
2776 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2777 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
2778 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
2779 LoadStoreStruct(vt, src, NEON_ST1_4v);
2780 }
2781
2782
st2(const VRegister & vt,const VRegister & vt2,const MemOperand & dst)2783 void Assembler::st2(const VRegister& vt,
2784 const VRegister& vt2,
2785 const MemOperand& dst) {
2786 USE(vt2);
2787 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2788 VIXL_ASSERT(AreSameFormat(vt, vt2));
2789 VIXL_ASSERT(AreConsecutive(vt, vt2));
2790 LoadStoreStruct(vt, dst, NEON_ST2);
2791 }
2792
2793
st2(const VRegister & vt,const VRegister & vt2,int lane,const MemOperand & dst)2794 void Assembler::st2(const VRegister& vt,
2795 const VRegister& vt2,
2796 int lane,
2797 const MemOperand& dst) {
2798 USE(vt2);
2799 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2800 VIXL_ASSERT(AreSameFormat(vt, vt2));
2801 VIXL_ASSERT(AreConsecutive(vt, vt2));
2802 LoadStoreStructSingle(vt, lane, dst, NEONLoadStoreSingleStructStore2);
2803 }
2804
2805
st3(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const MemOperand & dst)2806 void Assembler::st3(const VRegister& vt,
2807 const VRegister& vt2,
2808 const VRegister& vt3,
2809 const MemOperand& dst) {
2810 USE(vt2, vt3);
2811 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2812 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
2813 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
2814 LoadStoreStruct(vt, dst, NEON_ST3);
2815 }
2816
2817
st3(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,int lane,const MemOperand & dst)2818 void Assembler::st3(const VRegister& vt,
2819 const VRegister& vt2,
2820 const VRegister& vt3,
2821 int lane,
2822 const MemOperand& dst) {
2823 USE(vt2, vt3);
2824 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2825 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
2826 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
2827 LoadStoreStructSingle(vt, lane, dst, NEONLoadStoreSingleStructStore3);
2828 }
2829
2830
st4(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const VRegister & vt4,const MemOperand & dst)2831 void Assembler::st4(const VRegister& vt,
2832 const VRegister& vt2,
2833 const VRegister& vt3,
2834 const VRegister& vt4,
2835 const MemOperand& dst) {
2836 USE(vt2, vt3, vt4);
2837 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2838 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
2839 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
2840 LoadStoreStruct(vt, dst, NEON_ST4);
2841 }
2842
2843
st4(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const VRegister & vt4,int lane,const MemOperand & dst)2844 void Assembler::st4(const VRegister& vt,
2845 const VRegister& vt2,
2846 const VRegister& vt3,
2847 const VRegister& vt4,
2848 int lane,
2849 const MemOperand& dst) {
2850 USE(vt2, vt3, vt4);
2851 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2852 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
2853 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
2854 LoadStoreStructSingle(vt, lane, dst, NEONLoadStoreSingleStructStore4);
2855 }
2856
2857
LoadStoreStructSingle(const VRegister & vt,uint32_t lane,const MemOperand & addr,NEONLoadStoreSingleStructOp op)2858 void Assembler::LoadStoreStructSingle(const VRegister& vt,
2859 uint32_t lane,
2860 const MemOperand& addr,
2861 NEONLoadStoreSingleStructOp op) {
2862 LoadStoreStructVerify(vt, addr, op);
2863
2864 // We support vt arguments of the form vt.VxT() or vt.T(), where x is the
2865 // number of lanes, and T is b, h, s or d.
2866 unsigned lane_size = vt.GetLaneSizeInBytes();
2867 VIXL_ASSERT(lane_size > 0);
2868 VIXL_ASSERT(lane < (kQRegSizeInBytes / lane_size));
2869
2870 // Lane size is encoded in the opcode field. Lane index is encoded in the Q,
2871 // S and size fields.
2872 lane *= lane_size;
2873 if (lane_size == 8) lane++;
2874
2875 Instr size = (lane << NEONLSSize_offset) & NEONLSSize_mask;
2876 Instr s = (lane << (NEONS_offset - 2)) & NEONS_mask;
2877 Instr q = (lane << (NEONQ_offset - 3)) & NEONQ_mask;
2878
2879 Instr instr = op;
2880 switch (lane_size) {
2881 case 1:
2882 instr |= NEONLoadStoreSingle_b;
2883 break;
2884 case 2:
2885 instr |= NEONLoadStoreSingle_h;
2886 break;
2887 case 4:
2888 instr |= NEONLoadStoreSingle_s;
2889 break;
2890 default:
2891 VIXL_ASSERT(lane_size == 8);
2892 instr |= NEONLoadStoreSingle_d;
2893 }
2894
2895 Emit(instr | LoadStoreStructAddrModeField(addr) | q | size | s | Rt(vt));
2896 }
2897
2898
ld1(const VRegister & vt,int lane,const MemOperand & src)2899 void Assembler::ld1(const VRegister& vt, int lane, const MemOperand& src) {
2900 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2901 LoadStoreStructSingle(vt, lane, src, NEONLoadStoreSingleStructLoad1);
2902 }
2903
2904
ld1r(const VRegister & vt,const MemOperand & src)2905 void Assembler::ld1r(const VRegister& vt, const MemOperand& src) {
2906 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2907 LoadStoreStructSingleAllLanes(vt, src, NEON_LD1R);
2908 }
2909
2910
st1(const VRegister & vt,int lane,const MemOperand & dst)2911 void Assembler::st1(const VRegister& vt, int lane, const MemOperand& dst) {
2912 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2913 LoadStoreStructSingle(vt, lane, dst, NEONLoadStoreSingleStructStore1);
2914 }
2915
2916
NEON3DifferentL(const VRegister & vd,const VRegister & vn,const VRegister & vm,NEON3DifferentOp vop)2917 void Assembler::NEON3DifferentL(const VRegister& vd,
2918 const VRegister& vn,
2919 const VRegister& vm,
2920 NEON3DifferentOp vop) {
2921 VIXL_ASSERT(AreSameFormat(vn, vm));
2922 VIXL_ASSERT((vn.Is1H() && vd.Is1S()) || (vn.Is1S() && vd.Is1D()) ||
2923 (vn.Is8B() && vd.Is8H()) || (vn.Is4H() && vd.Is4S()) ||
2924 (vn.Is2S() && vd.Is2D()) || (vn.Is16B() && vd.Is8H()) ||
2925 (vn.Is8H() && vd.Is4S()) || (vn.Is4S() && vd.Is2D()));
2926 Instr format, op = vop;
2927 if (vd.IsScalar()) {
2928 op |= NEON_Q | NEONScalar;
2929 format = SFormat(vn);
2930 } else {
2931 format = VFormat(vn);
2932 }
2933 Emit(format | op | Rm(vm) | Rn(vn) | Rd(vd));
2934 }
2935
2936
NEON3DifferentW(const VRegister & vd,const VRegister & vn,const VRegister & vm,NEON3DifferentOp vop)2937 void Assembler::NEON3DifferentW(const VRegister& vd,
2938 const VRegister& vn,
2939 const VRegister& vm,
2940 NEON3DifferentOp vop) {
2941 VIXL_ASSERT(AreSameFormat(vd, vn));
2942 VIXL_ASSERT((vm.Is8B() && vd.Is8H()) || (vm.Is4H() && vd.Is4S()) ||
2943 (vm.Is2S() && vd.Is2D()) || (vm.Is16B() && vd.Is8H()) ||
2944 (vm.Is8H() && vd.Is4S()) || (vm.Is4S() && vd.Is2D()));
2945 Emit(VFormat(vm) | vop | Rm(vm) | Rn(vn) | Rd(vd));
2946 }
2947
2948
NEON3DifferentHN(const VRegister & vd,const VRegister & vn,const VRegister & vm,NEON3DifferentOp vop)2949 void Assembler::NEON3DifferentHN(const VRegister& vd,
2950 const VRegister& vn,
2951 const VRegister& vm,
2952 NEON3DifferentOp vop) {
2953 VIXL_ASSERT(AreSameFormat(vm, vn));
2954 VIXL_ASSERT((vd.Is8B() && vn.Is8H()) || (vd.Is4H() && vn.Is4S()) ||
2955 (vd.Is2S() && vn.Is2D()) || (vd.Is16B() && vn.Is8H()) ||
2956 (vd.Is8H() && vn.Is4S()) || (vd.Is4S() && vn.Is2D()));
2957 Emit(VFormat(vd) | vop | Rm(vm) | Rn(vn) | Rd(vd));
2958 }
2959
2960
2961 // clang-format off
2962 #define NEON_3DIFF_LONG_LIST(V) \
2963 V(pmull, NEON_PMULL, vn.IsVector() && vn.Is8B()) \
2964 V(pmull2, NEON_PMULL2, vn.IsVector() && vn.Is16B()) \
2965 V(saddl, NEON_SADDL, vn.IsVector() && vn.IsD()) \
2966 V(saddl2, NEON_SADDL2, vn.IsVector() && vn.IsQ()) \
2967 V(sabal, NEON_SABAL, vn.IsVector() && vn.IsD()) \
2968 V(sabal2, NEON_SABAL2, vn.IsVector() && vn.IsQ()) \
2969 V(uabal, NEON_UABAL, vn.IsVector() && vn.IsD()) \
2970 V(uabal2, NEON_UABAL2, vn.IsVector() && vn.IsQ()) \
2971 V(sabdl, NEON_SABDL, vn.IsVector() && vn.IsD()) \
2972 V(sabdl2, NEON_SABDL2, vn.IsVector() && vn.IsQ()) \
2973 V(uabdl, NEON_UABDL, vn.IsVector() && vn.IsD()) \
2974 V(uabdl2, NEON_UABDL2, vn.IsVector() && vn.IsQ()) \
2975 V(smlal, NEON_SMLAL, vn.IsVector() && vn.IsD()) \
2976 V(smlal2, NEON_SMLAL2, vn.IsVector() && vn.IsQ()) \
2977 V(umlal, NEON_UMLAL, vn.IsVector() && vn.IsD()) \
2978 V(umlal2, NEON_UMLAL2, vn.IsVector() && vn.IsQ()) \
2979 V(smlsl, NEON_SMLSL, vn.IsVector() && vn.IsD()) \
2980 V(smlsl2, NEON_SMLSL2, vn.IsVector() && vn.IsQ()) \
2981 V(umlsl, NEON_UMLSL, vn.IsVector() && vn.IsD()) \
2982 V(umlsl2, NEON_UMLSL2, vn.IsVector() && vn.IsQ()) \
2983 V(smull, NEON_SMULL, vn.IsVector() && vn.IsD()) \
2984 V(smull2, NEON_SMULL2, vn.IsVector() && vn.IsQ()) \
2985 V(umull, NEON_UMULL, vn.IsVector() && vn.IsD()) \
2986 V(umull2, NEON_UMULL2, vn.IsVector() && vn.IsQ()) \
2987 V(ssubl, NEON_SSUBL, vn.IsVector() && vn.IsD()) \
2988 V(ssubl2, NEON_SSUBL2, vn.IsVector() && vn.IsQ()) \
2989 V(uaddl, NEON_UADDL, vn.IsVector() && vn.IsD()) \
2990 V(uaddl2, NEON_UADDL2, vn.IsVector() && vn.IsQ()) \
2991 V(usubl, NEON_USUBL, vn.IsVector() && vn.IsD()) \
2992 V(usubl2, NEON_USUBL2, vn.IsVector() && vn.IsQ()) \
2993 V(sqdmlal, NEON_SQDMLAL, vn.Is1H() || vn.Is1S() || vn.Is4H() || vn.Is2S()) \
2994 V(sqdmlal2, NEON_SQDMLAL2, vn.Is1H() || vn.Is1S() || vn.Is8H() || vn.Is4S()) \
2995 V(sqdmlsl, NEON_SQDMLSL, vn.Is1H() || vn.Is1S() || vn.Is4H() || vn.Is2S()) \
2996 V(sqdmlsl2, NEON_SQDMLSL2, vn.Is1H() || vn.Is1S() || vn.Is8H() || vn.Is4S()) \
2997 V(sqdmull, NEON_SQDMULL, vn.Is1H() || vn.Is1S() || vn.Is4H() || vn.Is2S()) \
2998 V(sqdmull2, NEON_SQDMULL2, vn.Is1H() || vn.Is1S() || vn.Is8H() || vn.Is4S()) \
2999 // clang-format on
3000
3001
3002 #define VIXL_DEFINE_ASM_FUNC(FN, OP, AS) \
3003 void Assembler::FN(const VRegister& vd, \
3004 const VRegister& vn, \
3005 const VRegister& vm) { \
3006 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON)); \
3007 VIXL_ASSERT(AS); \
3008 NEON3DifferentL(vd, vn, vm, OP); \
3009 }
3010 NEON_3DIFF_LONG_LIST(VIXL_DEFINE_ASM_FUNC)
3011 #undef VIXL_DEFINE_ASM_FUNC
3012
3013 // clang-format off
3014 #define NEON_3DIFF_HN_LIST(V) \
3015 V(addhn, NEON_ADDHN, vd.IsD()) \
3016 V(addhn2, NEON_ADDHN2, vd.IsQ()) \
3017 V(raddhn, NEON_RADDHN, vd.IsD()) \
3018 V(raddhn2, NEON_RADDHN2, vd.IsQ()) \
3019 V(subhn, NEON_SUBHN, vd.IsD()) \
3020 V(subhn2, NEON_SUBHN2, vd.IsQ()) \
3021 V(rsubhn, NEON_RSUBHN, vd.IsD()) \
3022 V(rsubhn2, NEON_RSUBHN2, vd.IsQ())
3023 // clang-format on
3024
3025 #define VIXL_DEFINE_ASM_FUNC(FN, OP, AS) \
3026 void Assembler::FN(const VRegister& vd, \
3027 const VRegister& vn, \
3028 const VRegister& vm) { \
3029 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON)); \
3030 VIXL_ASSERT(AS); \
3031 NEON3DifferentHN(vd, vn, vm, OP); \
3032 }
NEON_3DIFF_HN_LIST(VIXL_DEFINE_ASM_FUNC)3033 NEON_3DIFF_HN_LIST(VIXL_DEFINE_ASM_FUNC)
3034 #undef VIXL_DEFINE_ASM_FUNC
3035
3036 void Assembler::uaddw(const VRegister& vd,
3037 const VRegister& vn,
3038 const VRegister& vm) {
3039 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
3040 VIXL_ASSERT(vm.IsD());
3041 NEON3DifferentW(vd, vn, vm, NEON_UADDW);
3042 }
3043
3044
uaddw2(const VRegister & vd,const VRegister & vn,const VRegister & vm)3045 void Assembler::uaddw2(const VRegister& vd,
3046 const VRegister& vn,
3047 const VRegister& vm) {
3048 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
3049 VIXL_ASSERT(vm.IsQ());
3050 NEON3DifferentW(vd, vn, vm, NEON_UADDW2);
3051 }
3052
3053
saddw(const VRegister & vd,const VRegister & vn,const VRegister & vm)3054 void Assembler::saddw(const VRegister& vd,
3055 const VRegister& vn,
3056 const VRegister& vm) {
3057 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
3058 VIXL_ASSERT(vm.IsD());
3059 NEON3DifferentW(vd, vn, vm, NEON_SADDW);
3060 }
3061
3062
saddw2(const VRegister & vd,const VRegister & vn,const VRegister & vm)3063 void Assembler::saddw2(const VRegister& vd,
3064 const VRegister& vn,
3065 const VRegister& vm) {
3066 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
3067 VIXL_ASSERT(vm.IsQ());
3068 NEON3DifferentW(vd, vn, vm, NEON_SADDW2);
3069 }
3070
3071
usubw(const VRegister & vd,const VRegister & vn,const VRegister & vm)3072 void Assembler::usubw(const VRegister& vd,
3073 const VRegister& vn,
3074 const VRegister& vm) {
3075 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
3076 VIXL_ASSERT(vm.IsD());
3077 NEON3DifferentW(vd, vn, vm, NEON_USUBW);
3078 }
3079
3080
usubw2(const VRegister & vd,const VRegister & vn,const VRegister & vm)3081 void Assembler::usubw2(const VRegister& vd,
3082 const VRegister& vn,
3083 const VRegister& vm) {
3084 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
3085 VIXL_ASSERT(vm.IsQ());
3086 NEON3DifferentW(vd, vn, vm, NEON_USUBW2);
3087 }
3088
3089
ssubw(const VRegister & vd,const VRegister & vn,const VRegister & vm)3090 void Assembler::ssubw(const VRegister& vd,
3091 const VRegister& vn,
3092 const VRegister& vm) {
3093 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
3094 VIXL_ASSERT(vm.IsD());
3095 NEON3DifferentW(vd, vn, vm, NEON_SSUBW);
3096 }
3097
3098
ssubw2(const VRegister & vd,const VRegister & vn,const VRegister & vm)3099 void Assembler::ssubw2(const VRegister& vd,
3100 const VRegister& vn,
3101 const VRegister& vm) {
3102 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
3103 VIXL_ASSERT(vm.IsQ());
3104 NEON3DifferentW(vd, vn, vm, NEON_SSUBW2);
3105 }
3106
3107
mov(const Register & rd,const Register & rm)3108 void Assembler::mov(const Register& rd, const Register& rm) {
3109 // Moves involving the stack pointer are encoded as add immediate with
3110 // second operand of zero. Otherwise, orr with first operand zr is
3111 // used.
3112 if (rd.IsSP() || rm.IsSP()) {
3113 add(rd, rm, 0);
3114 } else {
3115 orr(rd, AppropriateZeroRegFor(rd), rm);
3116 }
3117 }
3118
xpaclri()3119 void Assembler::xpaclri() {
3120 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
3121 Emit(XPACLRI);
3122 }
3123
pacia1716()3124 void Assembler::pacia1716() {
3125 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
3126 Emit(PACIA1716);
3127 }
3128
pacib1716()3129 void Assembler::pacib1716() {
3130 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
3131 Emit(PACIB1716);
3132 }
3133
autia1716()3134 void Assembler::autia1716() {
3135 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
3136 Emit(AUTIA1716);
3137 }
3138
autib1716()3139 void Assembler::autib1716() {
3140 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
3141 Emit(AUTIB1716);
3142 }
3143
paciaz()3144 void Assembler::paciaz() {
3145 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
3146 Emit(PACIAZ);
3147 }
3148
pacibz()3149 void Assembler::pacibz() {
3150 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
3151 Emit(PACIBZ);
3152 }
3153
autiaz()3154 void Assembler::autiaz() {
3155 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
3156 Emit(AUTIAZ);
3157 }
3158
autibz()3159 void Assembler::autibz() {
3160 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
3161 Emit(AUTIBZ);
3162 }
3163
paciasp()3164 void Assembler::paciasp() {
3165 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
3166 Emit(PACIASP);
3167 }
3168
pacibsp()3169 void Assembler::pacibsp() {
3170 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
3171 Emit(PACIBSP);
3172 }
3173
autiasp()3174 void Assembler::autiasp() {
3175 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
3176 Emit(AUTIASP);
3177 }
3178
autibsp()3179 void Assembler::autibsp() {
3180 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
3181 Emit(AUTIBSP);
3182 }
3183
bti(BranchTargetIdentifier id)3184 void Assembler::bti(BranchTargetIdentifier id) {
3185 VIXL_ASSERT((id != EmitPACIASP) && (id != EmitPACIBSP)); // Not modes of Bti.
3186 VIXL_ASSERT(id != EmitBTI_none); // Always generate an instruction.
3187 VIXL_ASSERT(CPUHas(CPUFeatures::kBTI));
3188 hint(static_cast<SystemHint>(id));
3189 }
3190
mvn(const Register & rd,const Operand & operand)3191 void Assembler::mvn(const Register& rd, const Operand& operand) {
3192 orn(rd, AppropriateZeroRegFor(rd), operand);
3193 }
3194
3195
mrs(const Register & xt,SystemRegister sysreg)3196 void Assembler::mrs(const Register& xt, SystemRegister sysreg) {
3197 VIXL_ASSERT(xt.Is64Bits());
3198 VIXL_ASSERT(CPUHas(sysreg));
3199 Emit(MRS | ImmSystemRegister(sysreg) | Rt(xt));
3200 }
3201
3202
msr(SystemRegister sysreg,const Register & xt)3203 void Assembler::msr(SystemRegister sysreg, const Register& xt) {
3204 VIXL_ASSERT(xt.Is64Bits());
3205 VIXL_ASSERT(CPUHas(sysreg));
3206 Emit(MSR | Rt(xt) | ImmSystemRegister(sysreg));
3207 }
3208
3209
cfinv()3210 void Assembler::cfinv() {
3211 VIXL_ASSERT(CPUHas(CPUFeatures::kFlagM));
3212 Emit(CFINV);
3213 }
3214
3215
axflag()3216 void Assembler::axflag() {
3217 VIXL_ASSERT(CPUHas(CPUFeatures::kAXFlag));
3218 Emit(AXFLAG);
3219 }
3220
3221
xaflag()3222 void Assembler::xaflag() {
3223 VIXL_ASSERT(CPUHas(CPUFeatures::kAXFlag));
3224 Emit(XAFLAG);
3225 }
3226
3227
clrex(int imm4)3228 void Assembler::clrex(int imm4) { Emit(CLREX | CRm(imm4)); }
3229
3230
dmb(BarrierDomain domain,BarrierType type)3231 void Assembler::dmb(BarrierDomain domain, BarrierType type) {
3232 Emit(DMB | ImmBarrierDomain(domain) | ImmBarrierType(type));
3233 }
3234
3235
dsb(BarrierDomain domain,BarrierType type)3236 void Assembler::dsb(BarrierDomain domain, BarrierType type) {
3237 Emit(DSB | ImmBarrierDomain(domain) | ImmBarrierType(type));
3238 }
3239
3240
isb()3241 void Assembler::isb() {
3242 Emit(ISB | ImmBarrierDomain(FullSystem) | ImmBarrierType(BarrierAll));
3243 }
3244
esb()3245 void Assembler::esb() {
3246 VIXL_ASSERT(CPUHas(CPUFeatures::kRAS));
3247 hint(ESB);
3248 }
3249
csdb()3250 void Assembler::csdb() { hint(CSDB); }
3251
fmov(const VRegister & vd,double imm)3252 void Assembler::fmov(const VRegister& vd, double imm) {
3253 VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
3254 if (vd.IsScalar()) {
3255 VIXL_ASSERT(vd.Is1D());
3256 Emit(FMOV_d_imm | Rd(vd) | ImmFP64(imm));
3257 } else {
3258 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
3259 VIXL_ASSERT(vd.Is2D());
3260 Instr op = NEONModifiedImmediate_MOVI | NEONModifiedImmediateOpBit;
3261 Instr q = NEON_Q;
3262 uint32_t encoded_imm = FP64ToImm8(imm);
3263 Emit(q | op | ImmNEONabcdefgh(encoded_imm) | NEONCmode(0xf) | Rd(vd));
3264 }
3265 }
3266
3267
fmov(const VRegister & vd,float imm)3268 void Assembler::fmov(const VRegister& vd, float imm) {
3269 VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
3270 if (vd.IsScalar()) {
3271 VIXL_ASSERT(vd.Is1S());
3272 Emit(FMOV_s_imm | Rd(vd) | ImmFP32(imm));
3273 } else {
3274 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
3275 VIXL_ASSERT(vd.Is2S() || vd.Is4S());
3276 Instr op = NEONModifiedImmediate_MOVI;
3277 Instr q = vd.Is4S() ? NEON_Q : 0;
3278 uint32_t encoded_imm = FP32ToImm8(imm);
3279 Emit(q | op | ImmNEONabcdefgh(encoded_imm) | NEONCmode(0xf) | Rd(vd));
3280 }
3281 }
3282
3283
fmov(const VRegister & vd,Float16 imm)3284 void Assembler::fmov(const VRegister& vd, Float16 imm) {
3285 VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
3286 if (vd.IsScalar()) {
3287 VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
3288 VIXL_ASSERT(vd.Is1H());
3289 Emit(FMOV_h_imm | Rd(vd) | ImmFP16(imm));
3290 } else {
3291 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON, CPUFeatures::kNEONHalf));
3292 VIXL_ASSERT(vd.Is4H() || vd.Is8H());
3293 Instr q = vd.Is8H() ? NEON_Q : 0;
3294 uint32_t encoded_imm = FP16ToImm8(imm);
3295 Emit(q | NEONModifiedImmediate_FMOV | ImmNEONabcdefgh(encoded_imm) |
3296 NEONCmode(0xf) | Rd(vd));
3297 }
3298 }
3299
3300
fmov(const Register & rd,const VRegister & vn)3301 void Assembler::fmov(const Register& rd, const VRegister& vn) {
3302 VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
3303 VIXL_ASSERT(vn.Is1H() || vn.Is1S() || vn.Is1D());
3304 VIXL_ASSERT((rd.GetSizeInBits() == vn.GetSizeInBits()) || vn.Is1H());
3305 FPIntegerConvertOp op;
3306 switch (vn.GetSizeInBits()) {
3307 case 16:
3308 VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
3309 op = rd.Is64Bits() ? FMOV_xh : FMOV_wh;
3310 break;
3311 case 32:
3312 op = FMOV_ws;
3313 break;
3314 default:
3315 op = FMOV_xd;
3316 }
3317 Emit(op | Rd(rd) | Rn(vn));
3318 }
3319
3320
fmov(const VRegister & vd,const Register & rn)3321 void Assembler::fmov(const VRegister& vd, const Register& rn) {
3322 VIXL_ASSERT(CPUHas(CPUFeatures::kFP) ||
3323 (vd.Is1D() && CPUHas(CPUFeatures::kNEON)));
3324 VIXL_ASSERT(vd.Is1H() || vd.Is1S() || vd.Is1D());
3325 VIXL_ASSERT((vd.GetSizeInBits() == rn.GetSizeInBits()) || vd.Is1H());
3326 FPIntegerConvertOp op;
3327 switch (vd.GetSizeInBits()) {
3328 case 16:
3329 VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
3330 op = rn.Is64Bits() ? FMOV_hx : FMOV_hw;
3331 break;
3332 case 32:
3333 op = FMOV_sw;
3334 break;
3335 default:
3336 op = FMOV_dx;
3337 }
3338 Emit(op | Rd(vd) | Rn(rn));
3339 }
3340
3341
fmov(const VRegister & vd,const VRegister & vn)3342 void Assembler::fmov(const VRegister& vd, const VRegister& vn) {
3343 VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
3344 if (vd.Is1H()) {
3345 VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
3346 }
3347 VIXL_ASSERT(vd.Is1H() || vd.Is1S() || vd.Is1D());
3348 VIXL_ASSERT(vd.IsSameFormat(vn));
3349 Emit(FPType(vd) | FMOV | Rd(vd) | Rn(vn));
3350 }
3351
3352
fmov(const VRegister & vd,int index,const Register & rn)3353 void Assembler::fmov(const VRegister& vd, int index, const Register& rn) {
3354 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON, CPUFeatures::kFP));
3355 VIXL_ASSERT((index == 1) && vd.Is1D() && rn.IsX());
3356 USE(index);
3357 Emit(FMOV_d1_x | Rd(vd) | Rn(rn));
3358 }
3359
3360
fmov(const Register & rd,const VRegister & vn,int index)3361 void Assembler::fmov(const Register& rd, const VRegister& vn, int index) {
3362 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON, CPUFeatures::kFP));
3363 VIXL_ASSERT((index == 1) && vn.Is1D() && rd.IsX());
3364 USE(index);
3365 Emit(FMOV_x_d1 | Rd(rd) | Rn(vn));
3366 }
3367
3368
fmadd(const VRegister & vd,const VRegister & vn,const VRegister & vm,const VRegister & va)3369 void Assembler::fmadd(const VRegister& vd,
3370 const VRegister& vn,
3371 const VRegister& vm,
3372 const VRegister& va) {
3373 VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
3374 FPDataProcessing3SourceOp op;
3375 if (vd.Is1H()) {
3376 VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
3377 op = FMADD_h;
3378 } else if (vd.Is1S()) {
3379 op = FMADD_s;
3380 } else {
3381 VIXL_ASSERT(vd.Is1D());
3382 op = FMADD_d;
3383 }
3384 FPDataProcessing3Source(vd, vn, vm, va, op);
3385 }
3386
3387
fmsub(const VRegister & vd,const VRegister & vn,const VRegister & vm,const VRegister & va)3388 void Assembler::fmsub(const VRegister& vd,
3389 const VRegister& vn,
3390 const VRegister& vm,
3391 const VRegister& va) {
3392 VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
3393 FPDataProcessing3SourceOp op;
3394 if (vd.Is1H()) {
3395 VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
3396 op = FMSUB_h;
3397 } else if (vd.Is1S()) {
3398 op = FMSUB_s;
3399 } else {
3400 VIXL_ASSERT(vd.Is1D());
3401 op = FMSUB_d;
3402 }
3403 FPDataProcessing3Source(vd, vn, vm, va, op);
3404 }
3405
3406
fnmadd(const VRegister & vd,const VRegister & vn,const VRegister & vm,const VRegister & va)3407 void Assembler::fnmadd(const VRegister& vd,
3408 const VRegister& vn,
3409 const VRegister& vm,
3410 const VRegister& va) {
3411 VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
3412 FPDataProcessing3SourceOp op;
3413 if (vd.Is1H()) {
3414 VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
3415 op = FNMADD_h;
3416 } else if (vd.Is1S()) {
3417 op = FNMADD_s;
3418 } else {
3419 VIXL_ASSERT(vd.Is1D());
3420 op = FNMADD_d;
3421 }
3422 FPDataProcessing3Source(vd, vn, vm, va, op);
3423 }
3424
3425
fnmsub(const VRegister & vd,const VRegister & vn,const VRegister & vm,const VRegister & va)3426 void Assembler::fnmsub(const VRegister& vd,
3427 const VRegister& vn,
3428 const VRegister& vm,
3429 const VRegister& va) {
3430 VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
3431 FPDataProcessing3SourceOp op;
3432 if (vd.Is1H()) {
3433 VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
3434 op = FNMSUB_h;
3435 } else if (vd.Is1S()) {
3436 op = FNMSUB_s;
3437 } else {
3438 VIXL_ASSERT(vd.Is1D());
3439 op = FNMSUB_d;
3440 }
3441 FPDataProcessing3Source(vd, vn, vm, va, op);
3442 }
3443
3444
fnmul(const VRegister & vd,const VRegister & vn,const VRegister & vm)3445 void Assembler::fnmul(const VRegister& vd,
3446 const VRegister& vn,
3447 const VRegister& vm) {
3448 VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
3449 VIXL_ASSERT(AreSameSizeAndType(vd, vn, vm));
3450 Instr op;
3451 if (vd.Is1H()) {
3452 VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
3453 op = FNMUL_h;
3454 } else if (vd.Is1S()) {
3455 op = FNMUL_s;
3456 } else {
3457 VIXL_ASSERT(vd.Is1D());
3458 op = FNMUL_d;
3459 }
3460 Emit(FPType(vd) | op | Rm(vm) | Rn(vn) | Rd(vd));
3461 }
3462
3463
FPCompareMacro(const VRegister & vn,double value,FPTrapFlags trap)3464 void Assembler::FPCompareMacro(const VRegister& vn,
3465 double value,
3466 FPTrapFlags trap) {
3467 USE(value);
3468 // Although the fcmp{e} instructions can strictly only take an immediate
3469 // value of +0.0, we don't need to check for -0.0 because the sign of 0.0
3470 // doesn't affect the result of the comparison.
3471 VIXL_ASSERT(value == 0.0);
3472 VIXL_ASSERT(vn.Is1H() || vn.Is1S() || vn.Is1D());
3473 Instr op = (trap == EnableTrap) ? FCMPE_zero : FCMP_zero;
3474 Emit(FPType(vn) | op | Rn(vn));
3475 }
3476
3477
FPCompareMacro(const VRegister & vn,const VRegister & vm,FPTrapFlags trap)3478 void Assembler::FPCompareMacro(const VRegister& vn,
3479 const VRegister& vm,
3480 FPTrapFlags trap) {
3481 VIXL_ASSERT(vn.Is1H() || vn.Is1S() || vn.Is1D());
3482 VIXL_ASSERT(vn.IsSameSizeAndType(vm));
3483 Instr op = (trap == EnableTrap) ? FCMPE : FCMP;
3484 Emit(FPType(vn) | op | Rm(vm) | Rn(vn));
3485 }
3486
3487
fcmp(const VRegister & vn,const VRegister & vm)3488 void Assembler::fcmp(const VRegister& vn, const VRegister& vm) {
3489 VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
3490 if (vn.Is1H()) VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
3491 FPCompareMacro(vn, vm, DisableTrap);
3492 }
3493
3494
fcmpe(const VRegister & vn,const VRegister & vm)3495 void Assembler::fcmpe(const VRegister& vn, const VRegister& vm) {
3496 VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
3497 if (vn.Is1H()) VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
3498 FPCompareMacro(vn, vm, EnableTrap);
3499 }
3500
3501
fcmp(const VRegister & vn,double value)3502 void Assembler::fcmp(const VRegister& vn, double value) {
3503 VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
3504 if (vn.Is1H()) VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
3505 FPCompareMacro(vn, value, DisableTrap);
3506 }
3507
3508
fcmpe(const VRegister & vn,double value)3509 void Assembler::fcmpe(const VRegister& vn, double value) {
3510 VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
3511 if (vn.Is1H()) VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
3512 FPCompareMacro(vn, value, EnableTrap);
3513 }
3514
3515
FPCCompareMacro(const VRegister & vn,const VRegister & vm,StatusFlags nzcv,Condition cond,FPTrapFlags trap)3516 void Assembler::FPCCompareMacro(const VRegister& vn,
3517 const VRegister& vm,
3518 StatusFlags nzcv,
3519 Condition cond,
3520 FPTrapFlags trap) {
3521 VIXL_ASSERT(vn.Is1H() || vn.Is1S() || vn.Is1D());
3522 VIXL_ASSERT(vn.IsSameSizeAndType(vm));
3523 Instr op = (trap == EnableTrap) ? FCCMPE : FCCMP;
3524 Emit(FPType(vn) | op | Rm(vm) | Cond(cond) | Rn(vn) | Nzcv(nzcv));
3525 }
3526
fccmp(const VRegister & vn,const VRegister & vm,StatusFlags nzcv,Condition cond)3527 void Assembler::fccmp(const VRegister& vn,
3528 const VRegister& vm,
3529 StatusFlags nzcv,
3530 Condition cond) {
3531 VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
3532 if (vn.Is1H()) VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
3533 FPCCompareMacro(vn, vm, nzcv, cond, DisableTrap);
3534 }
3535
3536
fccmpe(const VRegister & vn,const VRegister & vm,StatusFlags nzcv,Condition cond)3537 void Assembler::fccmpe(const VRegister& vn,
3538 const VRegister& vm,
3539 StatusFlags nzcv,
3540 Condition cond) {
3541 VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
3542 if (vn.Is1H()) VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
3543 FPCCompareMacro(vn, vm, nzcv, cond, EnableTrap);
3544 }
3545
3546
fcsel(const VRegister & vd,const VRegister & vn,const VRegister & vm,Condition cond)3547 void Assembler::fcsel(const VRegister& vd,
3548 const VRegister& vn,
3549 const VRegister& vm,
3550 Condition cond) {
3551 VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
3552 if (vd.Is1H()) VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
3553 VIXL_ASSERT(vd.Is1H() || vd.Is1S() || vd.Is1D());
3554 VIXL_ASSERT(AreSameFormat(vd, vn, vm));
3555 Emit(FPType(vd) | FCSEL | Rm(vm) | Cond(cond) | Rn(vn) | Rd(vd));
3556 }
3557
3558
fcvt(const VRegister & vd,const VRegister & vn)3559 void Assembler::fcvt(const VRegister& vd, const VRegister& vn) {
3560 VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
3561 FPDataProcessing1SourceOp op;
3562 // The half-precision variants belong to base FP, and do not require kFPHalf.
3563 if (vd.Is1D()) {
3564 VIXL_ASSERT(vn.Is1S() || vn.Is1H());
3565 op = vn.Is1S() ? FCVT_ds : FCVT_dh;
3566 } else if (vd.Is1S()) {
3567 VIXL_ASSERT(vn.Is1D() || vn.Is1H());
3568 op = vn.Is1D() ? FCVT_sd : FCVT_sh;
3569 } else {
3570 VIXL_ASSERT(vd.Is1H());
3571 VIXL_ASSERT(vn.Is1D() || vn.Is1S());
3572 op = vn.Is1D() ? FCVT_hd : FCVT_hs;
3573 }
3574 FPDataProcessing1Source(vd, vn, op);
3575 }
3576
3577
fcvtl(const VRegister & vd,const VRegister & vn)3578 void Assembler::fcvtl(const VRegister& vd, const VRegister& vn) {
3579 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
3580 VIXL_ASSERT((vd.Is4S() && vn.Is4H()) || (vd.Is2D() && vn.Is2S()));
3581 // The half-precision variants belong to base FP, and do not require kFPHalf.
3582 Instr format = vd.Is2D() ? (1 << NEONSize_offset) : 0;
3583 Emit(format | NEON_FCVTL | Rn(vn) | Rd(vd));
3584 }
3585
3586
fcvtl2(const VRegister & vd,const VRegister & vn)3587 void Assembler::fcvtl2(const VRegister& vd, const VRegister& vn) {
3588 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
3589 VIXL_ASSERT((vd.Is4S() && vn.Is8H()) || (vd.Is2D() && vn.Is4S()));
3590 // The half-precision variants belong to base FP, and do not require kFPHalf.
3591 Instr format = vd.Is2D() ? (1 << NEONSize_offset) : 0;
3592 Emit(NEON_Q | format | NEON_FCVTL | Rn(vn) | Rd(vd));
3593 }
3594
3595
fcvtn(const VRegister & vd,const VRegister & vn)3596 void Assembler::fcvtn(const VRegister& vd, const VRegister& vn) {
3597 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
3598 VIXL_ASSERT((vn.Is4S() && vd.Is4H()) || (vn.Is2D() && vd.Is2S()));
3599 // The half-precision variants belong to base FP, and do not require kFPHalf.
3600 Instr format = vn.Is2D() ? (1 << NEONSize_offset) : 0;
3601 Emit(format | NEON_FCVTN | Rn(vn) | Rd(vd));
3602 }
3603
3604
fcvtn2(const VRegister & vd,const VRegister & vn)3605 void Assembler::fcvtn2(const VRegister& vd, const VRegister& vn) {
3606 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
3607 VIXL_ASSERT((vn.Is4S() && vd.Is8H()) || (vn.Is2D() && vd.Is4S()));
3608 // The half-precision variants belong to base FP, and do not require kFPHalf.
3609 Instr format = vn.Is2D() ? (1 << NEONSize_offset) : 0;
3610 Emit(NEON_Q | format | NEON_FCVTN | Rn(vn) | Rd(vd));
3611 }
3612
3613
fcvtxn(const VRegister & vd,const VRegister & vn)3614 void Assembler::fcvtxn(const VRegister& vd, const VRegister& vn) {
3615 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
3616 Instr format = 1 << NEONSize_offset;
3617 if (vd.IsScalar()) {
3618 VIXL_ASSERT(vd.Is1S() && vn.Is1D());
3619 Emit(format | NEON_FCVTXN_scalar | Rn(vn) | Rd(vd));
3620 } else {
3621 VIXL_ASSERT(vd.Is2S() && vn.Is2D());
3622 Emit(format | NEON_FCVTXN | Rn(vn) | Rd(vd));
3623 }
3624 }
3625
3626
fcvtxn2(const VRegister & vd,const VRegister & vn)3627 void Assembler::fcvtxn2(const VRegister& vd, const VRegister& vn) {
3628 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
3629 VIXL_ASSERT(vd.Is4S() && vn.Is2D());
3630 Instr format = 1 << NEONSize_offset;
3631 Emit(NEON_Q | format | NEON_FCVTXN | Rn(vn) | Rd(vd));
3632 }
3633
fjcvtzs(const Register & rd,const VRegister & vn)3634 void Assembler::fjcvtzs(const Register& rd, const VRegister& vn) {
3635 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kJSCVT));
3636 VIXL_ASSERT(rd.IsW() && vn.Is1D());
3637 Emit(FJCVTZS | Rn(vn) | Rd(rd));
3638 }
3639
3640
NEONFPConvertToInt(const Register & rd,const VRegister & vn,Instr op)3641 void Assembler::NEONFPConvertToInt(const Register& rd,
3642 const VRegister& vn,
3643 Instr op) {
3644 Emit(SF(rd) | FPType(vn) | op | Rn(vn) | Rd(rd));
3645 }
3646
3647
NEONFPConvertToInt(const VRegister & vd,const VRegister & vn,Instr op)3648 void Assembler::NEONFPConvertToInt(const VRegister& vd,
3649 const VRegister& vn,
3650 Instr op) {
3651 if (vn.IsScalar()) {
3652 VIXL_ASSERT((vd.Is1S() && vn.Is1S()) || (vd.Is1D() && vn.Is1D()));
3653 op |= NEON_Q | NEONScalar;
3654 }
3655 Emit(FPFormat(vn) | op | Rn(vn) | Rd(vd));
3656 }
3657
3658
NEONFP16ConvertToInt(const VRegister & vd,const VRegister & vn,Instr op)3659 void Assembler::NEONFP16ConvertToInt(const VRegister& vd,
3660 const VRegister& vn,
3661 Instr op) {
3662 VIXL_ASSERT(AreSameFormat(vd, vn));
3663 VIXL_ASSERT(vn.IsLaneSizeH());
3664 if (vn.IsScalar()) {
3665 op |= NEON_Q | NEONScalar;
3666 } else if (vn.Is8H()) {
3667 op |= NEON_Q;
3668 }
3669 Emit(op | Rn(vn) | Rd(vd));
3670 }
3671
3672
3673 #define NEON_FP2REGMISC_FCVT_LIST(V) \
3674 V(fcvtnu, NEON_FCVTNU, FCVTNU) \
3675 V(fcvtns, NEON_FCVTNS, FCVTNS) \
3676 V(fcvtpu, NEON_FCVTPU, FCVTPU) \
3677 V(fcvtps, NEON_FCVTPS, FCVTPS) \
3678 V(fcvtmu, NEON_FCVTMU, FCVTMU) \
3679 V(fcvtms, NEON_FCVTMS, FCVTMS) \
3680 V(fcvtau, NEON_FCVTAU, FCVTAU) \
3681 V(fcvtas, NEON_FCVTAS, FCVTAS)
3682
3683 #define VIXL_DEFINE_ASM_FUNC(FN, VEC_OP, SCA_OP) \
3684 void Assembler::FN(const Register& rd, const VRegister& vn) { \
3685 VIXL_ASSERT(CPUHas(CPUFeatures::kFP)); \
3686 if (vn.IsH()) VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf)); \
3687 NEONFPConvertToInt(rd, vn, SCA_OP); \
3688 } \
3689 void Assembler::FN(const VRegister& vd, const VRegister& vn) { \
3690 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON)); \
3691 if (vd.IsLaneSizeH()) { \
3692 VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf)); \
3693 NEONFP16ConvertToInt(vd, vn, VEC_OP##_H); \
3694 } else { \
3695 NEONFPConvertToInt(vd, vn, VEC_OP); \
3696 } \
3697 }
NEON_FP2REGMISC_FCVT_LIST(VIXL_DEFINE_ASM_FUNC)3698 NEON_FP2REGMISC_FCVT_LIST(VIXL_DEFINE_ASM_FUNC)
3699 #undef VIXL_DEFINE_ASM_FUNC
3700
3701
3702 void Assembler::fcvtzs(const Register& rd, const VRegister& vn, int fbits) {
3703 VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
3704 if (vn.Is1H()) VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
3705 VIXL_ASSERT(vn.Is1H() || vn.Is1S() || vn.Is1D());
3706 VIXL_ASSERT((fbits >= 0) && (fbits <= rd.GetSizeInBits()));
3707 if (fbits == 0) {
3708 Emit(SF(rd) | FPType(vn) | FCVTZS | Rn(vn) | Rd(rd));
3709 } else {
3710 Emit(SF(rd) | FPType(vn) | FCVTZS_fixed | FPScale(64 - fbits) | Rn(vn) |
3711 Rd(rd));
3712 }
3713 }
3714
3715
fcvtzs(const VRegister & vd,const VRegister & vn,int fbits)3716 void Assembler::fcvtzs(const VRegister& vd, const VRegister& vn, int fbits) {
3717 // This form is a NEON scalar FP instruction.
3718 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
3719 if (vn.IsLaneSizeH()) VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
3720 VIXL_ASSERT(fbits >= 0);
3721 if (fbits == 0) {
3722 if (vd.IsLaneSizeH()) {
3723 NEONFP2RegMiscFP16(vd, vn, NEON_FCVTZS_H);
3724 } else {
3725 NEONFP2RegMisc(vd, vn, NEON_FCVTZS);
3726 }
3727 } else {
3728 VIXL_ASSERT(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S() ||
3729 vd.Is1H() || vd.Is4H() || vd.Is8H());
3730 NEONShiftRightImmediate(vd, vn, fbits, NEON_FCVTZS_imm);
3731 }
3732 }
3733
3734
fcvtzu(const Register & rd,const VRegister & vn,int fbits)3735 void Assembler::fcvtzu(const Register& rd, const VRegister& vn, int fbits) {
3736 VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
3737 if (vn.Is1H()) VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
3738 VIXL_ASSERT(vn.Is1H() || vn.Is1S() || vn.Is1D());
3739 VIXL_ASSERT((fbits >= 0) && (fbits <= rd.GetSizeInBits()));
3740 if (fbits == 0) {
3741 Emit(SF(rd) | FPType(vn) | FCVTZU | Rn(vn) | Rd(rd));
3742 } else {
3743 Emit(SF(rd) | FPType(vn) | FCVTZU_fixed | FPScale(64 - fbits) | Rn(vn) |
3744 Rd(rd));
3745 }
3746 }
3747
3748
fcvtzu(const VRegister & vd,const VRegister & vn,int fbits)3749 void Assembler::fcvtzu(const VRegister& vd, const VRegister& vn, int fbits) {
3750 // This form is a NEON scalar FP instruction.
3751 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
3752 if (vn.IsLaneSizeH()) VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
3753 VIXL_ASSERT(fbits >= 0);
3754 if (fbits == 0) {
3755 if (vd.IsLaneSizeH()) {
3756 NEONFP2RegMiscFP16(vd, vn, NEON_FCVTZU_H);
3757 } else {
3758 NEONFP2RegMisc(vd, vn, NEON_FCVTZU);
3759 }
3760 } else {
3761 VIXL_ASSERT(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S() ||
3762 vd.Is1H() || vd.Is4H() || vd.Is8H());
3763 NEONShiftRightImmediate(vd, vn, fbits, NEON_FCVTZU_imm);
3764 }
3765 }
3766
ucvtf(const VRegister & vd,const VRegister & vn,int fbits)3767 void Assembler::ucvtf(const VRegister& vd, const VRegister& vn, int fbits) {
3768 // This form is a NEON scalar FP instruction.
3769 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
3770 if (vn.IsLaneSizeH()) VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
3771 VIXL_ASSERT(fbits >= 0);
3772 if (fbits == 0) {
3773 if (vd.IsLaneSizeH()) {
3774 NEONFP2RegMiscFP16(vd, vn, NEON_UCVTF_H);
3775 } else {
3776 NEONFP2RegMisc(vd, vn, NEON_UCVTF);
3777 }
3778 } else {
3779 VIXL_ASSERT(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S() ||
3780 vd.Is1H() || vd.Is4H() || vd.Is8H());
3781 NEONShiftRightImmediate(vd, vn, fbits, NEON_UCVTF_imm);
3782 }
3783 }
3784
scvtf(const VRegister & vd,const VRegister & vn,int fbits)3785 void Assembler::scvtf(const VRegister& vd, const VRegister& vn, int fbits) {
3786 // This form is a NEON scalar FP instruction.
3787 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
3788 if (vn.IsLaneSizeH()) VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
3789 VIXL_ASSERT(fbits >= 0);
3790 if (fbits == 0) {
3791 if (vd.IsLaneSizeH()) {
3792 NEONFP2RegMiscFP16(vd, vn, NEON_SCVTF_H);
3793 } else {
3794 NEONFP2RegMisc(vd, vn, NEON_SCVTF);
3795 }
3796 } else {
3797 VIXL_ASSERT(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S() ||
3798 vd.Is1H() || vd.Is4H() || vd.Is8H());
3799 NEONShiftRightImmediate(vd, vn, fbits, NEON_SCVTF_imm);
3800 }
3801 }
3802
3803
scvtf(const VRegister & vd,const Register & rn,int fbits)3804 void Assembler::scvtf(const VRegister& vd, const Register& rn, int fbits) {
3805 VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
3806 if (vd.Is1H()) VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
3807 VIXL_ASSERT(vd.Is1H() || vd.Is1S() || vd.Is1D());
3808 VIXL_ASSERT(fbits >= 0);
3809 if (fbits == 0) {
3810 Emit(SF(rn) | FPType(vd) | SCVTF | Rn(rn) | Rd(vd));
3811 } else {
3812 Emit(SF(rn) | FPType(vd) | SCVTF_fixed | FPScale(64 - fbits) | Rn(rn) |
3813 Rd(vd));
3814 }
3815 }
3816
3817
ucvtf(const VRegister & vd,const Register & rn,int fbits)3818 void Assembler::ucvtf(const VRegister& vd, const Register& rn, int fbits) {
3819 VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
3820 if (vd.Is1H()) VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
3821 VIXL_ASSERT(vd.Is1H() || vd.Is1S() || vd.Is1D());
3822 VIXL_ASSERT(fbits >= 0);
3823 if (fbits == 0) {
3824 Emit(SF(rn) | FPType(vd) | UCVTF | Rn(rn) | Rd(vd));
3825 } else {
3826 Emit(SF(rn) | FPType(vd) | UCVTF_fixed | FPScale(64 - fbits) | Rn(rn) |
3827 Rd(vd));
3828 }
3829 }
3830
3831
NEON3Same(const VRegister & vd,const VRegister & vn,const VRegister & vm,NEON3SameOp vop)3832 void Assembler::NEON3Same(const VRegister& vd,
3833 const VRegister& vn,
3834 const VRegister& vm,
3835 NEON3SameOp vop) {
3836 VIXL_ASSERT(AreSameFormat(vd, vn, vm));
3837 VIXL_ASSERT(vd.IsVector() || !vd.IsQ());
3838
3839 Instr format, op = vop;
3840 if (vd.IsScalar()) {
3841 op |= NEON_Q | NEONScalar;
3842 format = SFormat(vd);
3843 } else {
3844 format = VFormat(vd);
3845 }
3846
3847 Emit(format | op | Rm(vm) | Rn(vn) | Rd(vd));
3848 }
3849
3850
NEONFP3Same(const VRegister & vd,const VRegister & vn,const VRegister & vm,Instr op)3851 void Assembler::NEONFP3Same(const VRegister& vd,
3852 const VRegister& vn,
3853 const VRegister& vm,
3854 Instr op) {
3855 VIXL_ASSERT(AreSameFormat(vd, vn, vm));
3856 Emit(FPFormat(vd) | op | Rm(vm) | Rn(vn) | Rd(vd));
3857 }
3858
3859
NEON3SameFP16(const VRegister & vd,const VRegister & vn,const VRegister & vm,Instr op)3860 void Assembler::NEON3SameFP16(const VRegister& vd,
3861 const VRegister& vn,
3862 const VRegister& vm,
3863 Instr op) {
3864 VIXL_ASSERT(AreSameFormat(vd, vn, vm));
3865 VIXL_ASSERT(vd.GetLaneSizeInBytes() == kHRegSizeInBytes);
3866 if (vd.Is8H()) op |= NEON_Q;
3867 Emit(op | Rm(vm) | Rn(vn) | Rd(vd));
3868 }
3869
3870
3871 // clang-format off
3872 #define NEON_FP2REGMISC_LIST(V) \
3873 V(fabs, NEON_FABS, FABS, FABS_h) \
3874 V(fneg, NEON_FNEG, FNEG, FNEG_h) \
3875 V(fsqrt, NEON_FSQRT, FSQRT, FSQRT_h) \
3876 V(frintn, NEON_FRINTN, FRINTN, FRINTN_h) \
3877 V(frinta, NEON_FRINTA, FRINTA, FRINTA_h) \
3878 V(frintp, NEON_FRINTP, FRINTP, FRINTP_h) \
3879 V(frintm, NEON_FRINTM, FRINTM, FRINTM_h) \
3880 V(frintx, NEON_FRINTX, FRINTX, FRINTX_h) \
3881 V(frintz, NEON_FRINTZ, FRINTZ, FRINTZ_h) \
3882 V(frinti, NEON_FRINTI, FRINTI, FRINTI_h) \
3883 V(frsqrte, NEON_FRSQRTE, NEON_FRSQRTE_scalar, NEON_FRSQRTE_H_scalar) \
3884 V(frecpe, NEON_FRECPE, NEON_FRECPE_scalar, NEON_FRECPE_H_scalar)
3885 // clang-format on
3886
3887 #define VIXL_DEFINE_ASM_FUNC(FN, VEC_OP, SCA_OP, SCA_OP_H) \
3888 void Assembler::FN(const VRegister& vd, const VRegister& vn) { \
3889 VIXL_ASSERT(CPUHas(CPUFeatures::kFP)); \
3890 Instr op; \
3891 if (vd.IsScalar()) { \
3892 if (vd.Is1H()) { \
3893 if ((SCA_OP_H & NEONScalar2RegMiscFP16FMask) == \
3894 NEONScalar2RegMiscFP16Fixed) { \
3895 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON, CPUFeatures::kNEONHalf)); \
3896 } else { \
3897 VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf)); \
3898 } \
3899 op = SCA_OP_H; \
3900 } else { \
3901 if ((SCA_OP & NEONScalar2RegMiscFMask) == NEONScalar2RegMiscFixed) { \
3902 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON)); \
3903 } \
3904 VIXL_ASSERT(vd.Is1S() || vd.Is1D()); \
3905 op = SCA_OP; \
3906 } \
3907 } else { \
3908 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON)); \
3909 VIXL_ASSERT(vd.Is4H() || vd.Is8H() || vd.Is2S() || vd.Is2D() || \
3910 vd.Is4S()); \
3911 if (vd.IsLaneSizeH()) { \
3912 VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf)); \
3913 op = VEC_OP##_H; \
3914 if (vd.Is8H()) { \
3915 op |= NEON_Q; \
3916 } \
3917 } else { \
3918 op = VEC_OP; \
3919 } \
3920 } \
3921 if (vd.IsLaneSizeH()) { \
3922 NEONFP2RegMiscFP16(vd, vn, op); \
3923 } else { \
3924 NEONFP2RegMisc(vd, vn, op); \
3925 } \
3926 }
3927 NEON_FP2REGMISC_LIST(VIXL_DEFINE_ASM_FUNC)
3928 #undef VIXL_DEFINE_ASM_FUNC
3929
3930 // clang-format off
3931 #define NEON_FP2REGMISC_V85_LIST(V) \
3932 V(frint32x, NEON_FRINT32X, FRINT32X) \
3933 V(frint32z, NEON_FRINT32Z, FRINT32Z) \
3934 V(frint64x, NEON_FRINT64X, FRINT64X) \
3935 V(frint64z, NEON_FRINT64Z, FRINT64Z)
3936 // clang-format on
3937
3938 #define VIXL_DEFINE_ASM_FUNC(FN, VEC_OP, SCA_OP) \
3939 void Assembler::FN(const VRegister& vd, const VRegister& vn) { \
3940 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kFrintToFixedSizedInt)); \
3941 Instr op; \
3942 if (vd.IsScalar()) { \
3943 VIXL_ASSERT(vd.Is1S() || vd.Is1D()); \
3944 op = SCA_OP; \
3945 } else { \
3946 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON)); \
3947 VIXL_ASSERT(vd.Is2S() || vd.Is2D() || vd.Is4S()); \
3948 op = VEC_OP; \
3949 } \
3950 NEONFP2RegMisc(vd, vn, op); \
3951 }
NEON_FP2REGMISC_V85_LIST(VIXL_DEFINE_ASM_FUNC)3952 NEON_FP2REGMISC_V85_LIST(VIXL_DEFINE_ASM_FUNC)
3953 #undef VIXL_DEFINE_ASM_FUNC
3954
3955 void Assembler::NEONFP2RegMiscFP16(const VRegister& vd,
3956 const VRegister& vn,
3957 Instr op) {
3958 VIXL_ASSERT(AreSameFormat(vd, vn));
3959 Emit(op | Rn(vn) | Rd(vd));
3960 }
3961
3962
NEONFP2RegMisc(const VRegister & vd,const VRegister & vn,Instr op)3963 void Assembler::NEONFP2RegMisc(const VRegister& vd,
3964 const VRegister& vn,
3965 Instr op) {
3966 VIXL_ASSERT(AreSameFormat(vd, vn));
3967 Emit(FPFormat(vd) | op | Rn(vn) | Rd(vd));
3968 }
3969
3970
NEON2RegMisc(const VRegister & vd,const VRegister & vn,NEON2RegMiscOp vop,int value)3971 void Assembler::NEON2RegMisc(const VRegister& vd,
3972 const VRegister& vn,
3973 NEON2RegMiscOp vop,
3974 int value) {
3975 VIXL_ASSERT(AreSameFormat(vd, vn));
3976 VIXL_ASSERT(value == 0);
3977 USE(value);
3978
3979 Instr format, op = vop;
3980 if (vd.IsScalar()) {
3981 op |= NEON_Q | NEONScalar;
3982 format = SFormat(vd);
3983 } else {
3984 format = VFormat(vd);
3985 }
3986
3987 Emit(format | op | Rn(vn) | Rd(vd));
3988 }
3989
3990
cmeq(const VRegister & vd,const VRegister & vn,int value)3991 void Assembler::cmeq(const VRegister& vd, const VRegister& vn, int value) {
3992 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
3993 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
3994 NEON2RegMisc(vd, vn, NEON_CMEQ_zero, value);
3995 }
3996
3997
cmge(const VRegister & vd,const VRegister & vn,int value)3998 void Assembler::cmge(const VRegister& vd, const VRegister& vn, int value) {
3999 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4000 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
4001 NEON2RegMisc(vd, vn, NEON_CMGE_zero, value);
4002 }
4003
4004
cmgt(const VRegister & vd,const VRegister & vn,int value)4005 void Assembler::cmgt(const VRegister& vd, const VRegister& vn, int value) {
4006 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4007 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
4008 NEON2RegMisc(vd, vn, NEON_CMGT_zero, value);
4009 }
4010
4011
cmle(const VRegister & vd,const VRegister & vn,int value)4012 void Assembler::cmle(const VRegister& vd, const VRegister& vn, int value) {
4013 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
4014 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4015 NEON2RegMisc(vd, vn, NEON_CMLE_zero, value);
4016 }
4017
4018
cmlt(const VRegister & vd,const VRegister & vn,int value)4019 void Assembler::cmlt(const VRegister& vd, const VRegister& vn, int value) {
4020 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4021 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
4022 NEON2RegMisc(vd, vn, NEON_CMLT_zero, value);
4023 }
4024
4025
shll(const VRegister & vd,const VRegister & vn,int shift)4026 void Assembler::shll(const VRegister& vd, const VRegister& vn, int shift) {
4027 USE(shift);
4028 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4029 VIXL_ASSERT((vd.Is8H() && vn.Is8B() && shift == 8) ||
4030 (vd.Is4S() && vn.Is4H() && shift == 16) ||
4031 (vd.Is2D() && vn.Is2S() && shift == 32));
4032 Emit(VFormat(vn) | NEON_SHLL | Rn(vn) | Rd(vd));
4033 }
4034
4035
shll2(const VRegister & vd,const VRegister & vn,int shift)4036 void Assembler::shll2(const VRegister& vd, const VRegister& vn, int shift) {
4037 USE(shift);
4038 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4039 VIXL_ASSERT((vd.Is8H() && vn.Is16B() && shift == 8) ||
4040 (vd.Is4S() && vn.Is8H() && shift == 16) ||
4041 (vd.Is2D() && vn.Is4S() && shift == 32));
4042 Emit(VFormat(vn) | NEON_SHLL | Rn(vn) | Rd(vd));
4043 }
4044
4045
NEONFP2RegMisc(const VRegister & vd,const VRegister & vn,NEON2RegMiscOp vop,double value)4046 void Assembler::NEONFP2RegMisc(const VRegister& vd,
4047 const VRegister& vn,
4048 NEON2RegMiscOp vop,
4049 double value) {
4050 VIXL_ASSERT(AreSameFormat(vd, vn));
4051 VIXL_ASSERT(value == 0.0);
4052 USE(value);
4053
4054 Instr op = vop;
4055 if (vd.IsScalar()) {
4056 VIXL_ASSERT(vd.Is1S() || vd.Is1D());
4057 op |= NEON_Q | NEONScalar;
4058 } else {
4059 VIXL_ASSERT(vd.Is2S() || vd.Is2D() || vd.Is4S());
4060 }
4061
4062 Emit(FPFormat(vd) | op | Rn(vn) | Rd(vd));
4063 }
4064
4065
NEONFP2RegMiscFP16(const VRegister & vd,const VRegister & vn,NEON2RegMiscFP16Op vop,double value)4066 void Assembler::NEONFP2RegMiscFP16(const VRegister& vd,
4067 const VRegister& vn,
4068 NEON2RegMiscFP16Op vop,
4069 double value) {
4070 VIXL_ASSERT(AreSameFormat(vd, vn));
4071 VIXL_ASSERT(value == 0.0);
4072 USE(value);
4073
4074 Instr op = vop;
4075 if (vd.IsScalar()) {
4076 VIXL_ASSERT(vd.Is1H());
4077 op |= NEON_Q | NEONScalar;
4078 } else {
4079 VIXL_ASSERT(vd.Is4H() || vd.Is8H());
4080 if (vd.Is8H()) {
4081 op |= NEON_Q;
4082 }
4083 }
4084
4085 Emit(op | Rn(vn) | Rd(vd));
4086 }
4087
4088
fcmeq(const VRegister & vd,const VRegister & vn,double value)4089 void Assembler::fcmeq(const VRegister& vd, const VRegister& vn, double value) {
4090 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
4091 if (vd.IsLaneSizeH()) {
4092 VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
4093 NEONFP2RegMiscFP16(vd, vn, NEON_FCMEQ_H_zero, value);
4094 } else {
4095 NEONFP2RegMisc(vd, vn, NEON_FCMEQ_zero, value);
4096 }
4097 }
4098
4099
fcmge(const VRegister & vd,const VRegister & vn,double value)4100 void Assembler::fcmge(const VRegister& vd, const VRegister& vn, double value) {
4101 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
4102 if (vd.IsLaneSizeH()) {
4103 VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
4104 NEONFP2RegMiscFP16(vd, vn, NEON_FCMGE_H_zero, value);
4105 } else {
4106 NEONFP2RegMisc(vd, vn, NEON_FCMGE_zero, value);
4107 }
4108 }
4109
4110
fcmgt(const VRegister & vd,const VRegister & vn,double value)4111 void Assembler::fcmgt(const VRegister& vd, const VRegister& vn, double value) {
4112 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
4113 if (vd.IsLaneSizeH()) {
4114 VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
4115 NEONFP2RegMiscFP16(vd, vn, NEON_FCMGT_H_zero, value);
4116 } else {
4117 NEONFP2RegMisc(vd, vn, NEON_FCMGT_zero, value);
4118 }
4119 }
4120
4121
fcmle(const VRegister & vd,const VRegister & vn,double value)4122 void Assembler::fcmle(const VRegister& vd, const VRegister& vn, double value) {
4123 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
4124 if (vd.IsLaneSizeH()) {
4125 VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
4126 NEONFP2RegMiscFP16(vd, vn, NEON_FCMLE_H_zero, value);
4127 } else {
4128 NEONFP2RegMisc(vd, vn, NEON_FCMLE_zero, value);
4129 }
4130 }
4131
4132
fcmlt(const VRegister & vd,const VRegister & vn,double value)4133 void Assembler::fcmlt(const VRegister& vd, const VRegister& vn, double value) {
4134 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
4135 if (vd.IsLaneSizeH()) {
4136 VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
4137 NEONFP2RegMiscFP16(vd, vn, NEON_FCMLT_H_zero, value);
4138 } else {
4139 NEONFP2RegMisc(vd, vn, NEON_FCMLT_zero, value);
4140 }
4141 }
4142
4143
frecpx(const VRegister & vd,const VRegister & vn)4144 void Assembler::frecpx(const VRegister& vd, const VRegister& vn) {
4145 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
4146 VIXL_ASSERT(vd.IsScalar());
4147 VIXL_ASSERT(AreSameFormat(vd, vn));
4148 Instr op;
4149 if (vd.Is1H()) {
4150 VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
4151 op = NEON_FRECPX_H_scalar;
4152 } else {
4153 VIXL_ASSERT(vd.Is1S() || vd.Is1D());
4154 op = NEON_FRECPX_scalar;
4155 }
4156 Emit(FPFormat(vd) | op | Rn(vn) | Rd(vd));
4157 }
4158
4159
4160 // clang-format off
4161 #define NEON_3SAME_LIST(V) \
4162 V(add, NEON_ADD, vd.IsVector() || vd.Is1D()) \
4163 V(addp, NEON_ADDP, vd.IsVector() || vd.Is1D()) \
4164 V(sub, NEON_SUB, vd.IsVector() || vd.Is1D()) \
4165 V(cmeq, NEON_CMEQ, vd.IsVector() || vd.Is1D()) \
4166 V(cmge, NEON_CMGE, vd.IsVector() || vd.Is1D()) \
4167 V(cmgt, NEON_CMGT, vd.IsVector() || vd.Is1D()) \
4168 V(cmhi, NEON_CMHI, vd.IsVector() || vd.Is1D()) \
4169 V(cmhs, NEON_CMHS, vd.IsVector() || vd.Is1D()) \
4170 V(cmtst, NEON_CMTST, vd.IsVector() || vd.Is1D()) \
4171 V(sshl, NEON_SSHL, vd.IsVector() || vd.Is1D()) \
4172 V(ushl, NEON_USHL, vd.IsVector() || vd.Is1D()) \
4173 V(srshl, NEON_SRSHL, vd.IsVector() || vd.Is1D()) \
4174 V(urshl, NEON_URSHL, vd.IsVector() || vd.Is1D()) \
4175 V(sqdmulh, NEON_SQDMULH, vd.IsLaneSizeH() || vd.IsLaneSizeS()) \
4176 V(sqrdmulh, NEON_SQRDMULH, vd.IsLaneSizeH() || vd.IsLaneSizeS()) \
4177 V(shadd, NEON_SHADD, vd.IsVector() && !vd.IsLaneSizeD()) \
4178 V(uhadd, NEON_UHADD, vd.IsVector() && !vd.IsLaneSizeD()) \
4179 V(srhadd, NEON_SRHADD, vd.IsVector() && !vd.IsLaneSizeD()) \
4180 V(urhadd, NEON_URHADD, vd.IsVector() && !vd.IsLaneSizeD()) \
4181 V(shsub, NEON_SHSUB, vd.IsVector() && !vd.IsLaneSizeD()) \
4182 V(uhsub, NEON_UHSUB, vd.IsVector() && !vd.IsLaneSizeD()) \
4183 V(smax, NEON_SMAX, vd.IsVector() && !vd.IsLaneSizeD()) \
4184 V(smaxp, NEON_SMAXP, vd.IsVector() && !vd.IsLaneSizeD()) \
4185 V(smin, NEON_SMIN, vd.IsVector() && !vd.IsLaneSizeD()) \
4186 V(sminp, NEON_SMINP, vd.IsVector() && !vd.IsLaneSizeD()) \
4187 V(umax, NEON_UMAX, vd.IsVector() && !vd.IsLaneSizeD()) \
4188 V(umaxp, NEON_UMAXP, vd.IsVector() && !vd.IsLaneSizeD()) \
4189 V(umin, NEON_UMIN, vd.IsVector() && !vd.IsLaneSizeD()) \
4190 V(uminp, NEON_UMINP, vd.IsVector() && !vd.IsLaneSizeD()) \
4191 V(saba, NEON_SABA, vd.IsVector() && !vd.IsLaneSizeD()) \
4192 V(sabd, NEON_SABD, vd.IsVector() && !vd.IsLaneSizeD()) \
4193 V(uaba, NEON_UABA, vd.IsVector() && !vd.IsLaneSizeD()) \
4194 V(uabd, NEON_UABD, vd.IsVector() && !vd.IsLaneSizeD()) \
4195 V(mla, NEON_MLA, vd.IsVector() && !vd.IsLaneSizeD()) \
4196 V(mls, NEON_MLS, vd.IsVector() && !vd.IsLaneSizeD()) \
4197 V(mul, NEON_MUL, vd.IsVector() && !vd.IsLaneSizeD()) \
4198 V(and_, NEON_AND, vd.Is8B() || vd.Is16B()) \
4199 V(orr, NEON_ORR, vd.Is8B() || vd.Is16B()) \
4200 V(orn, NEON_ORN, vd.Is8B() || vd.Is16B()) \
4201 V(eor, NEON_EOR, vd.Is8B() || vd.Is16B()) \
4202 V(bic, NEON_BIC, vd.Is8B() || vd.Is16B()) \
4203 V(bit, NEON_BIT, vd.Is8B() || vd.Is16B()) \
4204 V(bif, NEON_BIF, vd.Is8B() || vd.Is16B()) \
4205 V(bsl, NEON_BSL, vd.Is8B() || vd.Is16B()) \
4206 V(pmul, NEON_PMUL, vd.Is8B() || vd.Is16B()) \
4207 V(uqadd, NEON_UQADD, true) \
4208 V(sqadd, NEON_SQADD, true) \
4209 V(uqsub, NEON_UQSUB, true) \
4210 V(sqsub, NEON_SQSUB, true) \
4211 V(sqshl, NEON_SQSHL, true) \
4212 V(uqshl, NEON_UQSHL, true) \
4213 V(sqrshl, NEON_SQRSHL, true) \
4214 V(uqrshl, NEON_UQRSHL, true)
4215 // clang-format on
4216
4217 #define VIXL_DEFINE_ASM_FUNC(FN, OP, AS) \
4218 void Assembler::FN(const VRegister& vd, \
4219 const VRegister& vn, \
4220 const VRegister& vm) { \
4221 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON)); \
4222 VIXL_ASSERT(AS); \
4223 NEON3Same(vd, vn, vm, OP); \
4224 }
4225 NEON_3SAME_LIST(VIXL_DEFINE_ASM_FUNC)
4226 #undef VIXL_DEFINE_ASM_FUNC
4227
4228 // clang-format off
4229 #define NEON_FP3SAME_OP_LIST(V) \
4230 V(fmulx, NEON_FMULX, NEON_FMULX_scalar, NEON_FMULX_H_scalar) \
4231 V(frecps, NEON_FRECPS, NEON_FRECPS_scalar, NEON_FRECPS_H_scalar) \
4232 V(frsqrts, NEON_FRSQRTS, NEON_FRSQRTS_scalar, NEON_FRSQRTS_H_scalar) \
4233 V(fabd, NEON_FABD, NEON_FABD_scalar, NEON_FABD_H_scalar) \
4234 V(fmla, NEON_FMLA, 0, 0) \
4235 V(fmls, NEON_FMLS, 0, 0) \
4236 V(facge, NEON_FACGE, NEON_FACGE_scalar, NEON_FACGE_H_scalar) \
4237 V(facgt, NEON_FACGT, NEON_FACGT_scalar, NEON_FACGT_H_scalar) \
4238 V(fcmeq, NEON_FCMEQ, NEON_FCMEQ_scalar, NEON_FCMEQ_H_scalar) \
4239 V(fcmge, NEON_FCMGE, NEON_FCMGE_scalar, NEON_FCMGE_H_scalar) \
4240 V(fcmgt, NEON_FCMGT, NEON_FCMGT_scalar, NEON_FCMGT_H_scalar) \
4241 V(faddp, NEON_FADDP, 0, 0) \
4242 V(fmaxp, NEON_FMAXP, 0, 0) \
4243 V(fminp, NEON_FMINP, 0, 0) \
4244 V(fmaxnmp, NEON_FMAXNMP, 0, 0) \
4245 V(fadd, NEON_FADD, FADD, 0) \
4246 V(fsub, NEON_FSUB, FSUB, 0) \
4247 V(fmul, NEON_FMUL, FMUL, 0) \
4248 V(fdiv, NEON_FDIV, FDIV, 0) \
4249 V(fmax, NEON_FMAX, FMAX, 0) \
4250 V(fmin, NEON_FMIN, FMIN, 0) \
4251 V(fmaxnm, NEON_FMAXNM, FMAXNM, 0) \
4252 V(fminnm, NEON_FMINNM, FMINNM, 0) \
4253 V(fminnmp, NEON_FMINNMP, 0, 0)
4254 // clang-format on
4255
4256 // TODO: This macro is complicated because it classifies the instructions in the
4257 // macro list above, and treats each case differently. It could be somewhat
4258 // simpler if we were to split the macro, at the cost of some duplication.
4259 #define VIXL_DEFINE_ASM_FUNC(FN, VEC_OP, SCA_OP, SCA_OP_H) \
4260 void Assembler::FN(const VRegister& vd, \
4261 const VRegister& vn, \
4262 const VRegister& vm) { \
4263 VIXL_ASSERT(CPUHas(CPUFeatures::kFP)); \
4264 Instr op; \
4265 bool is_fp16 = false; \
4266 if ((SCA_OP != 0) && vd.IsScalar()) { \
4267 if ((SCA_OP_H != 0) && vd.Is1H()) { \
4268 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON, CPUFeatures::kNEONHalf)); \
4269 is_fp16 = true; \
4270 op = SCA_OP_H; \
4271 } else { \
4272 VIXL_ASSERT(vd.Is1H() || vd.Is1S() || vd.Is1D()); \
4273 if ((SCA_OP & NEONScalar3SameFMask) == NEONScalar3SameFixed) { \
4274 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON)); \
4275 if (vd.Is1H()) VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf)); \
4276 } else if (vd.Is1H()) { \
4277 VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf)); \
4278 } \
4279 op = SCA_OP; \
4280 } \
4281 } else { \
4282 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON)); \
4283 VIXL_ASSERT(vd.IsVector()); \
4284 if (vd.Is4H() || vd.Is8H()) { \
4285 VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf)); \
4286 is_fp16 = true; \
4287 op = VEC_OP##_H; \
4288 } else { \
4289 VIXL_ASSERT(vd.Is2S() || vd.Is2D() || vd.Is4S()); \
4290 op = VEC_OP; \
4291 } \
4292 } \
4293 if (is_fp16) { \
4294 NEON3SameFP16(vd, vn, vm, op); \
4295 } else { \
4296 NEONFP3Same(vd, vn, vm, op); \
4297 } \
4298 }
NEON_FP3SAME_OP_LIST(VIXL_DEFINE_ASM_FUNC)4299 NEON_FP3SAME_OP_LIST(VIXL_DEFINE_ASM_FUNC)
4300 #undef VIXL_DEFINE_ASM_FUNC
4301
4302
4303 // clang-format off
4304 #define NEON_FHM_LIST(V) \
4305 V(fmlal, NEON_FMLAL) \
4306 V(fmlal2, NEON_FMLAL2) \
4307 V(fmlsl, NEON_FMLSL) \
4308 V(fmlsl2, NEON_FMLSL2)
4309 // clang-format on
4310
4311 #define VIXL_DEFINE_ASM_FUNC(FN, VEC_OP) \
4312 void Assembler::FN(const VRegister& vd, \
4313 const VRegister& vn, \
4314 const VRegister& vm) { \
4315 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON, \
4316 CPUFeatures::kFP, \
4317 CPUFeatures::kNEONHalf, \
4318 CPUFeatures::kFHM)); \
4319 VIXL_ASSERT((vd.Is2S() && vn.Is2H() && vm.Is2H()) || \
4320 (vd.Is4S() && vn.Is4H() && vm.Is4H())); \
4321 Emit(FPFormat(vd) | VEC_OP | Rm(vm) | Rn(vn) | Rd(vd)); \
4322 }
4323 NEON_FHM_LIST(VIXL_DEFINE_ASM_FUNC)
4324 #undef VIXL_DEFINE_ASM_FUNC
4325
4326
4327 void Assembler::addp(const VRegister& vd, const VRegister& vn) {
4328 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4329 VIXL_ASSERT((vd.Is1D() && vn.Is2D()));
4330 Emit(SFormat(vd) | NEON_ADDP_scalar | Rn(vn) | Rd(vd));
4331 }
4332
4333
sqrdmlah(const VRegister & vd,const VRegister & vn,const VRegister & vm)4334 void Assembler::sqrdmlah(const VRegister& vd,
4335 const VRegister& vn,
4336 const VRegister& vm) {
4337 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON, CPUFeatures::kRDM));
4338 VIXL_ASSERT(AreSameFormat(vd, vn, vm));
4339 VIXL_ASSERT(vd.IsVector() || !vd.IsQ());
4340
4341 Instr format, op = NEON_SQRDMLAH;
4342 if (vd.IsScalar()) {
4343 op |= NEON_Q | NEONScalar;
4344 format = SFormat(vd);
4345 } else {
4346 format = VFormat(vd);
4347 }
4348
4349 Emit(format | op | Rm(vm) | Rn(vn) | Rd(vd));
4350 }
4351
4352
sqrdmlsh(const VRegister & vd,const VRegister & vn,const VRegister & vm)4353 void Assembler::sqrdmlsh(const VRegister& vd,
4354 const VRegister& vn,
4355 const VRegister& vm) {
4356 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON, CPUFeatures::kRDM));
4357 VIXL_ASSERT(AreSameFormat(vd, vn, vm));
4358 VIXL_ASSERT(vd.IsVector() || !vd.IsQ());
4359
4360 Instr format, op = NEON_SQRDMLSH;
4361 if (vd.IsScalar()) {
4362 op |= NEON_Q | NEONScalar;
4363 format = SFormat(vd);
4364 } else {
4365 format = VFormat(vd);
4366 }
4367
4368 Emit(format | op | Rm(vm) | Rn(vn) | Rd(vd));
4369 }
4370
4371
sdot(const VRegister & vd,const VRegister & vn,const VRegister & vm)4372 void Assembler::sdot(const VRegister& vd,
4373 const VRegister& vn,
4374 const VRegister& vm) {
4375 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON, CPUFeatures::kDotProduct));
4376 VIXL_ASSERT(AreSameFormat(vn, vm));
4377 VIXL_ASSERT((vd.Is2S() && vn.Is8B()) || (vd.Is4S() && vn.Is16B()));
4378
4379 Emit(VFormat(vd) | NEON_SDOT | Rm(vm) | Rn(vn) | Rd(vd));
4380 }
4381
4382
udot(const VRegister & vd,const VRegister & vn,const VRegister & vm)4383 void Assembler::udot(const VRegister& vd,
4384 const VRegister& vn,
4385 const VRegister& vm) {
4386 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON, CPUFeatures::kDotProduct));
4387 VIXL_ASSERT(AreSameFormat(vn, vm));
4388 VIXL_ASSERT((vd.Is2S() && vn.Is8B()) || (vd.Is4S() && vn.Is16B()));
4389
4390 Emit(VFormat(vd) | NEON_UDOT | Rm(vm) | Rn(vn) | Rd(vd));
4391 }
4392
usdot(const VRegister & vd,const VRegister & vn,const VRegister & vm)4393 void Assembler::usdot(const VRegister& vd,
4394 const VRegister& vn,
4395 const VRegister& vm) {
4396 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON, CPUFeatures::kI8MM));
4397 VIXL_ASSERT(AreSameFormat(vn, vm));
4398 VIXL_ASSERT((vd.Is2S() && vn.Is8B()) || (vd.Is4S() && vn.Is16B()));
4399
4400 Emit(VFormat(vd) | 0x0e809c00 | Rm(vm) | Rn(vn) | Rd(vd));
4401 }
4402
faddp(const VRegister & vd,const VRegister & vn)4403 void Assembler::faddp(const VRegister& vd, const VRegister& vn) {
4404 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
4405 VIXL_ASSERT((vd.Is1S() && vn.Is2S()) || (vd.Is1D() && vn.Is2D()) ||
4406 (vd.Is1H() && vn.Is2H()));
4407 if (vd.Is1H()) {
4408 VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
4409 Emit(NEON_FADDP_h_scalar | Rn(vn) | Rd(vd));
4410 } else {
4411 Emit(FPFormat(vd) | NEON_FADDP_scalar | Rn(vn) | Rd(vd));
4412 }
4413 }
4414
4415
fmaxp(const VRegister & vd,const VRegister & vn)4416 void Assembler::fmaxp(const VRegister& vd, const VRegister& vn) {
4417 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
4418 VIXL_ASSERT((vd.Is1S() && vn.Is2S()) || (vd.Is1D() && vn.Is2D()) ||
4419 (vd.Is1H() && vn.Is2H()));
4420 if (vd.Is1H()) {
4421 VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
4422 Emit(NEON_FMAXP_h_scalar | Rn(vn) | Rd(vd));
4423 } else {
4424 Emit(FPFormat(vd) | NEON_FMAXP_scalar | Rn(vn) | Rd(vd));
4425 }
4426 }
4427
4428
fminp(const VRegister & vd,const VRegister & vn)4429 void Assembler::fminp(const VRegister& vd, const VRegister& vn) {
4430 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
4431 VIXL_ASSERT((vd.Is1S() && vn.Is2S()) || (vd.Is1D() && vn.Is2D()) ||
4432 (vd.Is1H() && vn.Is2H()));
4433 if (vd.Is1H()) {
4434 VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
4435 Emit(NEON_FMINP_h_scalar | Rn(vn) | Rd(vd));
4436 } else {
4437 Emit(FPFormat(vd) | NEON_FMINP_scalar | Rn(vn) | Rd(vd));
4438 }
4439 }
4440
4441
fmaxnmp(const VRegister & vd,const VRegister & vn)4442 void Assembler::fmaxnmp(const VRegister& vd, const VRegister& vn) {
4443 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
4444 VIXL_ASSERT((vd.Is1S() && vn.Is2S()) || (vd.Is1D() && vn.Is2D()) ||
4445 (vd.Is1H() && vn.Is2H()));
4446 if (vd.Is1H()) {
4447 VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
4448 Emit(NEON_FMAXNMP_h_scalar | Rn(vn) | Rd(vd));
4449 } else {
4450 Emit(FPFormat(vd) | NEON_FMAXNMP_scalar | Rn(vn) | Rd(vd));
4451 }
4452 }
4453
4454
fminnmp(const VRegister & vd,const VRegister & vn)4455 void Assembler::fminnmp(const VRegister& vd, const VRegister& vn) {
4456 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
4457 VIXL_ASSERT((vd.Is1S() && vn.Is2S()) || (vd.Is1D() && vn.Is2D()) ||
4458 (vd.Is1H() && vn.Is2H()));
4459 if (vd.Is1H()) {
4460 VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
4461 Emit(NEON_FMINNMP_h_scalar | Rn(vn) | Rd(vd));
4462 } else {
4463 Emit(FPFormat(vd) | NEON_FMINNMP_scalar | Rn(vn) | Rd(vd));
4464 }
4465 }
4466
4467
4468 // v8.3 complex numbers - floating-point complex multiply accumulate.
fcmla(const VRegister & vd,const VRegister & vn,const VRegister & vm,int vm_index,int rot)4469 void Assembler::fcmla(const VRegister& vd,
4470 const VRegister& vn,
4471 const VRegister& vm,
4472 int vm_index,
4473 int rot) {
4474 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON, CPUFeatures::kFcma));
4475 VIXL_ASSERT(vd.IsVector() && AreSameFormat(vd, vn));
4476 VIXL_ASSERT((vm.IsH() && (vd.Is8H() || vd.Is4H())) ||
4477 (vm.IsS() && vd.Is4S()));
4478 if (vd.IsLaneSizeH()) VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
4479 int index_num_bits = vd.Is4S() ? 1 : 2;
4480 Emit(VFormat(vd) | Rm(vm) | NEON_FCMLA_byelement |
4481 ImmNEONHLM(vm_index, index_num_bits) | ImmRotFcmlaSca(rot) | Rn(vn) |
4482 Rd(vd));
4483 }
4484
4485
fcmla(const VRegister & vd,const VRegister & vn,const VRegister & vm,int rot)4486 void Assembler::fcmla(const VRegister& vd,
4487 const VRegister& vn,
4488 const VRegister& vm,
4489 int rot) {
4490 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON, CPUFeatures::kFcma));
4491 VIXL_ASSERT(AreSameFormat(vd, vn, vm));
4492 VIXL_ASSERT(vd.IsVector() && !vd.IsLaneSizeB());
4493 if (vd.IsLaneSizeH()) VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
4494 Emit(VFormat(vd) | Rm(vm) | NEON_FCMLA | ImmRotFcmlaVec(rot) | Rn(vn) |
4495 Rd(vd));
4496 }
4497
4498
4499 // v8.3 complex numbers - floating-point complex add.
fcadd(const VRegister & vd,const VRegister & vn,const VRegister & vm,int rot)4500 void Assembler::fcadd(const VRegister& vd,
4501 const VRegister& vn,
4502 const VRegister& vm,
4503 int rot) {
4504 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON, CPUFeatures::kFcma));
4505 VIXL_ASSERT(AreSameFormat(vd, vn, vm));
4506 VIXL_ASSERT(vd.IsVector() && !vd.IsLaneSizeB());
4507 if (vd.IsLaneSizeH()) VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
4508 Emit(VFormat(vd) | Rm(vm) | NEON_FCADD | ImmRotFcadd(rot) | Rn(vn) | Rd(vd));
4509 }
4510
4511
orr(const VRegister & vd,const int imm8,const int left_shift)4512 void Assembler::orr(const VRegister& vd, const int imm8, const int left_shift) {
4513 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4514 NEONModifiedImmShiftLsl(vd, imm8, left_shift, NEONModifiedImmediate_ORR);
4515 }
4516
4517
mov(const VRegister & vd,const VRegister & vn)4518 void Assembler::mov(const VRegister& vd, const VRegister& vn) {
4519 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4520 VIXL_ASSERT(AreSameFormat(vd, vn));
4521 if (vd.IsD()) {
4522 orr(vd.V8B(), vn.V8B(), vn.V8B());
4523 } else {
4524 VIXL_ASSERT(vd.IsQ());
4525 orr(vd.V16B(), vn.V16B(), vn.V16B());
4526 }
4527 }
4528
4529
bic(const VRegister & vd,const int imm8,const int left_shift)4530 void Assembler::bic(const VRegister& vd, const int imm8, const int left_shift) {
4531 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4532 NEONModifiedImmShiftLsl(vd, imm8, left_shift, NEONModifiedImmediate_BIC);
4533 }
4534
4535
movi(const VRegister & vd,const uint64_t imm,Shift shift,const int shift_amount)4536 void Assembler::movi(const VRegister& vd,
4537 const uint64_t imm,
4538 Shift shift,
4539 const int shift_amount) {
4540 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4541 VIXL_ASSERT((shift == LSL) || (shift == MSL));
4542 if (vd.Is2D() || vd.Is1D()) {
4543 VIXL_ASSERT(shift_amount == 0);
4544 int imm8 = 0;
4545 for (int i = 0; i < 8; ++i) {
4546 int byte = (imm >> (i * 8)) & 0xff;
4547 VIXL_ASSERT((byte == 0) || (byte == 0xff));
4548 if (byte == 0xff) {
4549 imm8 |= (1 << i);
4550 }
4551 }
4552 int q = vd.Is2D() ? NEON_Q : 0;
4553 Emit(q | NEONModImmOp(1) | NEONModifiedImmediate_MOVI |
4554 ImmNEONabcdefgh(imm8) | NEONCmode(0xe) | Rd(vd));
4555 } else if (shift == LSL) {
4556 VIXL_ASSERT(IsUint8(imm));
4557 NEONModifiedImmShiftLsl(vd,
4558 static_cast<int>(imm),
4559 shift_amount,
4560 NEONModifiedImmediate_MOVI);
4561 } else {
4562 VIXL_ASSERT(IsUint8(imm));
4563 NEONModifiedImmShiftMsl(vd,
4564 static_cast<int>(imm),
4565 shift_amount,
4566 NEONModifiedImmediate_MOVI);
4567 }
4568 }
4569
4570
mvn(const VRegister & vd,const VRegister & vn)4571 void Assembler::mvn(const VRegister& vd, const VRegister& vn) {
4572 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4573 VIXL_ASSERT(AreSameFormat(vd, vn));
4574 if (vd.IsD()) {
4575 not_(vd.V8B(), vn.V8B());
4576 } else {
4577 VIXL_ASSERT(vd.IsQ());
4578 not_(vd.V16B(), vn.V16B());
4579 }
4580 }
4581
4582
mvni(const VRegister & vd,const int imm8,Shift shift,const int shift_amount)4583 void Assembler::mvni(const VRegister& vd,
4584 const int imm8,
4585 Shift shift,
4586 const int shift_amount) {
4587 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4588 VIXL_ASSERT((shift == LSL) || (shift == MSL));
4589 if (shift == LSL) {
4590 NEONModifiedImmShiftLsl(vd, imm8, shift_amount, NEONModifiedImmediate_MVNI);
4591 } else {
4592 NEONModifiedImmShiftMsl(vd, imm8, shift_amount, NEONModifiedImmediate_MVNI);
4593 }
4594 }
4595
4596
NEONFPByElement(const VRegister & vd,const VRegister & vn,const VRegister & vm,int vm_index,NEONByIndexedElementOp vop,NEONByIndexedElementOp vop_half)4597 void Assembler::NEONFPByElement(const VRegister& vd,
4598 const VRegister& vn,
4599 const VRegister& vm,
4600 int vm_index,
4601 NEONByIndexedElementOp vop,
4602 NEONByIndexedElementOp vop_half) {
4603 VIXL_ASSERT(AreSameFormat(vd, vn));
4604 VIXL_ASSERT((vd.Is2S() && vm.Is1S()) || (vd.Is4S() && vm.Is1S()) ||
4605 (vd.Is1S() && vm.Is1S()) || (vd.Is2D() && vm.Is1D()) ||
4606 (vd.Is1D() && vm.Is1D()) || (vd.Is4H() && vm.Is1H()) ||
4607 (vd.Is8H() && vm.Is1H()) || (vd.Is1H() && vm.Is1H()));
4608 VIXL_ASSERT((vm.Is1S() && (vm_index < 4)) || (vm.Is1D() && (vm_index < 2)) ||
4609 (vm.Is1H() && (vm.GetCode() < 16) && (vm_index < 8)));
4610
4611 Instr op = vop;
4612 int index_num_bits;
4613 if (vm.Is1D()) {
4614 index_num_bits = 1;
4615 } else if (vm.Is1S()) {
4616 index_num_bits = 2;
4617 } else {
4618 index_num_bits = 3;
4619 op = vop_half;
4620 }
4621
4622 if (vd.IsScalar()) {
4623 op |= NEON_Q | NEONScalar;
4624 }
4625
4626 if (!vm.Is1H()) {
4627 op |= FPFormat(vd);
4628 } else if (vd.Is8H()) {
4629 op |= NEON_Q;
4630 }
4631
4632 Emit(op | ImmNEONHLM(vm_index, index_num_bits) | Rm(vm) | Rn(vn) | Rd(vd));
4633 }
4634
4635
NEONByElement(const VRegister & vd,const VRegister & vn,const VRegister & vm,int vm_index,NEONByIndexedElementOp vop)4636 void Assembler::NEONByElement(const VRegister& vd,
4637 const VRegister& vn,
4638 const VRegister& vm,
4639 int vm_index,
4640 NEONByIndexedElementOp vop) {
4641 VIXL_ASSERT(AreSameFormat(vd, vn));
4642 VIXL_ASSERT((vd.Is4H() && vm.Is1H()) || (vd.Is8H() && vm.Is1H()) ||
4643 (vd.Is1H() && vm.Is1H()) || (vd.Is2S() && vm.Is1S()) ||
4644 (vd.Is4S() && vm.Is1S()) || (vd.Is1S() && vm.Is1S()));
4645 VIXL_ASSERT((vm.Is1H() && (vm.GetCode() < 16) && (vm_index < 8)) ||
4646 (vm.Is1S() && (vm_index < 4)));
4647
4648 Instr format, op = vop;
4649 int index_num_bits = vm.Is1H() ? 3 : 2;
4650 if (vd.IsScalar()) {
4651 op |= NEONScalar | NEON_Q;
4652 format = SFormat(vn);
4653 } else {
4654 format = VFormat(vn);
4655 }
4656 Emit(format | op | ImmNEONHLM(vm_index, index_num_bits) | Rm(vm) | Rn(vn) |
4657 Rd(vd));
4658 }
4659
4660
NEONByElementL(const VRegister & vd,const VRegister & vn,const VRegister & vm,int vm_index,NEONByIndexedElementOp vop)4661 void Assembler::NEONByElementL(const VRegister& vd,
4662 const VRegister& vn,
4663 const VRegister& vm,
4664 int vm_index,
4665 NEONByIndexedElementOp vop) {
4666 VIXL_ASSERT((vd.Is4S() && vn.Is4H() && vm.Is1H()) ||
4667 (vd.Is4S() && vn.Is8H() && vm.Is1H()) ||
4668 (vd.Is1S() && vn.Is1H() && vm.Is1H()) ||
4669 (vd.Is2D() && vn.Is2S() && vm.Is1S()) ||
4670 (vd.Is2D() && vn.Is4S() && vm.Is1S()) ||
4671 (vd.Is1D() && vn.Is1S() && vm.Is1S()));
4672
4673 VIXL_ASSERT((vm.Is1H() && (vm.GetCode() < 16) && (vm_index < 8)) ||
4674 (vm.Is1S() && (vm_index < 4)));
4675
4676 Instr format, op = vop;
4677 int index_num_bits = vm.Is1H() ? 3 : 2;
4678 if (vd.IsScalar()) {
4679 op |= NEONScalar | NEON_Q;
4680 format = SFormat(vn);
4681 } else {
4682 format = VFormat(vn);
4683 }
4684 Emit(format | op | ImmNEONHLM(vm_index, index_num_bits) | Rm(vm) | Rn(vn) |
4685 Rd(vd));
4686 }
4687
4688
sdot(const VRegister & vd,const VRegister & vn,const VRegister & vm,int vm_index)4689 void Assembler::sdot(const VRegister& vd,
4690 const VRegister& vn,
4691 const VRegister& vm,
4692 int vm_index) {
4693 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON, CPUFeatures::kDotProduct));
4694 VIXL_ASSERT((vd.Is2S() && vn.Is8B() && vm.Is1S4B()) ||
4695 (vd.Is4S() && vn.Is16B() && vm.Is1S4B()));
4696
4697 int index_num_bits = 2;
4698 Emit(VFormat(vd) | NEON_SDOT_byelement |
4699 ImmNEONHLM(vm_index, index_num_bits) | Rm(vm) | Rn(vn) | Rd(vd));
4700 }
4701
4702
udot(const VRegister & vd,const VRegister & vn,const VRegister & vm,int vm_index)4703 void Assembler::udot(const VRegister& vd,
4704 const VRegister& vn,
4705 const VRegister& vm,
4706 int vm_index) {
4707 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON, CPUFeatures::kDotProduct));
4708 VIXL_ASSERT((vd.Is2S() && vn.Is8B() && vm.Is1S4B()) ||
4709 (vd.Is4S() && vn.Is16B() && vm.Is1S4B()));
4710
4711 int index_num_bits = 2;
4712 Emit(VFormat(vd) | NEON_UDOT_byelement |
4713 ImmNEONHLM(vm_index, index_num_bits) | Rm(vm) | Rn(vn) | Rd(vd));
4714 }
4715
sudot(const VRegister & vd,const VRegister & vn,const VRegister & vm,int vm_index)4716 void Assembler::sudot(const VRegister& vd,
4717 const VRegister& vn,
4718 const VRegister& vm,
4719 int vm_index) {
4720 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON, CPUFeatures::kI8MM));
4721 VIXL_ASSERT((vd.Is2S() && vn.Is8B() && vm.Is1S4B()) ||
4722 (vd.Is4S() && vn.Is16B() && vm.Is1S4B()));
4723 int q = vd.Is4S() ? (1U << NEONQ_offset) : 0;
4724 int index_num_bits = 2;
4725 Emit(q | 0x0f00f000 | ImmNEONHLM(vm_index, index_num_bits) | Rm(vm) | Rn(vn) |
4726 Rd(vd));
4727 }
4728
4729
usdot(const VRegister & vd,const VRegister & vn,const VRegister & vm,int vm_index)4730 void Assembler::usdot(const VRegister& vd,
4731 const VRegister& vn,
4732 const VRegister& vm,
4733 int vm_index) {
4734 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON, CPUFeatures::kI8MM));
4735 VIXL_ASSERT((vd.Is2S() && vn.Is8B() && vm.Is1S4B()) ||
4736 (vd.Is4S() && vn.Is16B() && vm.Is1S4B()));
4737 int q = vd.Is4S() ? (1U << NEONQ_offset) : 0;
4738 int index_num_bits = 2;
4739 Emit(q | 0x0f80f000 | ImmNEONHLM(vm_index, index_num_bits) | Rm(vm) | Rn(vn) |
4740 Rd(vd));
4741 }
4742
4743 // clang-format off
4744 #define NEON_BYELEMENT_LIST(V) \
4745 V(mul, NEON_MUL_byelement, vn.IsVector()) \
4746 V(mla, NEON_MLA_byelement, vn.IsVector()) \
4747 V(mls, NEON_MLS_byelement, vn.IsVector()) \
4748 V(sqdmulh, NEON_SQDMULH_byelement, true) \
4749 V(sqrdmulh, NEON_SQRDMULH_byelement, true) \
4750 // clang-format on
4751
4752 #define VIXL_DEFINE_ASM_FUNC(FN, OP, AS) \
4753 void Assembler::FN(const VRegister& vd, \
4754 const VRegister& vn, \
4755 const VRegister& vm, \
4756 int vm_index) { \
4757 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON)); \
4758 VIXL_ASSERT(AS); \
4759 NEONByElement(vd, vn, vm, vm_index, OP); \
4760 }
4761 NEON_BYELEMENT_LIST(VIXL_DEFINE_ASM_FUNC)
4762 #undef VIXL_DEFINE_ASM_FUNC
4763
4764
4765 // clang-format off
4766 #define NEON_BYELEMENT_RDM_LIST(V) \
4767 V(sqrdmlah, NEON_SQRDMLAH_byelement) \
4768 V(sqrdmlsh, NEON_SQRDMLSH_byelement)
4769 // clang-format on
4770
4771 #define VIXL_DEFINE_ASM_FUNC(FN, OP) \
4772 void Assembler::FN(const VRegister& vd, \
4773 const VRegister& vn, \
4774 const VRegister& vm, \
4775 int vm_index) { \
4776 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON, CPUFeatures::kRDM)); \
4777 NEONByElement(vd, vn, vm, vm_index, OP); \
4778 }
NEON_BYELEMENT_RDM_LIST(VIXL_DEFINE_ASM_FUNC)4779 NEON_BYELEMENT_RDM_LIST(VIXL_DEFINE_ASM_FUNC)
4780 #undef VIXL_DEFINE_ASM_FUNC
4781
4782
4783 // clang-format off
4784 #define NEON_FPBYELEMENT_LIST(V) \
4785 V(fmul, NEON_FMUL_byelement, NEON_FMUL_H_byelement) \
4786 V(fmla, NEON_FMLA_byelement, NEON_FMLA_H_byelement) \
4787 V(fmls, NEON_FMLS_byelement, NEON_FMLS_H_byelement) \
4788 V(fmulx, NEON_FMULX_byelement, NEON_FMULX_H_byelement)
4789 // clang-format on
4790
4791 #define VIXL_DEFINE_ASM_FUNC(FN, OP, OP_H) \
4792 void Assembler::FN(const VRegister& vd, \
4793 const VRegister& vn, \
4794 const VRegister& vm, \
4795 int vm_index) { \
4796 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON)); \
4797 if (vd.IsLaneSizeH()) VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf)); \
4798 NEONFPByElement(vd, vn, vm, vm_index, OP, OP_H); \
4799 }
4800 NEON_FPBYELEMENT_LIST(VIXL_DEFINE_ASM_FUNC)
4801 #undef VIXL_DEFINE_ASM_FUNC
4802
4803
4804 // clang-format off
4805 #define NEON_BYELEMENT_LONG_LIST(V) \
4806 V(sqdmull, NEON_SQDMULL_byelement, vn.IsScalar() || vn.IsD()) \
4807 V(sqdmull2, NEON_SQDMULL_byelement, vn.IsVector() && vn.IsQ()) \
4808 V(sqdmlal, NEON_SQDMLAL_byelement, vn.IsScalar() || vn.IsD()) \
4809 V(sqdmlal2, NEON_SQDMLAL_byelement, vn.IsVector() && vn.IsQ()) \
4810 V(sqdmlsl, NEON_SQDMLSL_byelement, vn.IsScalar() || vn.IsD()) \
4811 V(sqdmlsl2, NEON_SQDMLSL_byelement, vn.IsVector() && vn.IsQ()) \
4812 V(smull, NEON_SMULL_byelement, vn.IsVector() && vn.IsD()) \
4813 V(smull2, NEON_SMULL_byelement, vn.IsVector() && vn.IsQ()) \
4814 V(umull, NEON_UMULL_byelement, vn.IsVector() && vn.IsD()) \
4815 V(umull2, NEON_UMULL_byelement, vn.IsVector() && vn.IsQ()) \
4816 V(smlal, NEON_SMLAL_byelement, vn.IsVector() && vn.IsD()) \
4817 V(smlal2, NEON_SMLAL_byelement, vn.IsVector() && vn.IsQ()) \
4818 V(umlal, NEON_UMLAL_byelement, vn.IsVector() && vn.IsD()) \
4819 V(umlal2, NEON_UMLAL_byelement, vn.IsVector() && vn.IsQ()) \
4820 V(smlsl, NEON_SMLSL_byelement, vn.IsVector() && vn.IsD()) \
4821 V(smlsl2, NEON_SMLSL_byelement, vn.IsVector() && vn.IsQ()) \
4822 V(umlsl, NEON_UMLSL_byelement, vn.IsVector() && vn.IsD()) \
4823 V(umlsl2, NEON_UMLSL_byelement, vn.IsVector() && vn.IsQ())
4824 // clang-format on
4825
4826
4827 #define VIXL_DEFINE_ASM_FUNC(FN, OP, AS) \
4828 void Assembler::FN(const VRegister& vd, \
4829 const VRegister& vn, \
4830 const VRegister& vm, \
4831 int vm_index) { \
4832 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON)); \
4833 VIXL_ASSERT(AS); \
4834 NEONByElementL(vd, vn, vm, vm_index, OP); \
4835 }
4836 NEON_BYELEMENT_LONG_LIST(VIXL_DEFINE_ASM_FUNC)
4837 #undef VIXL_DEFINE_ASM_FUNC
4838
4839
4840 // clang-format off
4841 #define NEON_BYELEMENT_FHM_LIST(V) \
4842 V(fmlal, NEON_FMLAL_H_byelement) \
4843 V(fmlal2, NEON_FMLAL2_H_byelement) \
4844 V(fmlsl, NEON_FMLSL_H_byelement) \
4845 V(fmlsl2, NEON_FMLSL2_H_byelement)
4846 // clang-format on
4847
4848
4849 #define VIXL_DEFINE_ASM_FUNC(FN, OP) \
4850 void Assembler::FN(const VRegister& vd, \
4851 const VRegister& vn, \
4852 const VRegister& vm, \
4853 int vm_index) { \
4854 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON, \
4855 CPUFeatures::kFP, \
4856 CPUFeatures::kNEONHalf, \
4857 CPUFeatures::kFHM)); \
4858 VIXL_ASSERT((vd.Is2S() && vn.Is2H()) || (vd.Is4S() && vn.Is4H())); \
4859 VIXL_ASSERT(vm.IsH()); \
4860 VIXL_ASSERT((vm_index >= 0) && (vm_index < 8)); \
4861 /* Vm itself can only be in the bottom 16 registers. */ \
4862 VIXL_ASSERT(vm.GetCode() < 16); \
4863 Emit(FPFormat(vd) | OP | Rd(vd) | Rn(vn) | Rm(vm) | \
4864 ImmNEONHLM(vm_index, 3)); \
4865 }
4866 NEON_BYELEMENT_FHM_LIST(VIXL_DEFINE_ASM_FUNC)
4867 #undef VIXL_DEFINE_ASM_FUNC
4868
4869 void Assembler::suqadd(const VRegister& vd, const VRegister& vn) {
4870 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4871 NEON2RegMisc(vd, vn, NEON_SUQADD);
4872 }
4873
4874
usqadd(const VRegister & vd,const VRegister & vn)4875 void Assembler::usqadd(const VRegister& vd, const VRegister& vn) {
4876 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4877 NEON2RegMisc(vd, vn, NEON_USQADD);
4878 }
4879
4880
abs(const VRegister & vd,const VRegister & vn)4881 void Assembler::abs(const VRegister& vd, const VRegister& vn) {
4882 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4883 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
4884 NEON2RegMisc(vd, vn, NEON_ABS);
4885 }
4886
4887
sqabs(const VRegister & vd,const VRegister & vn)4888 void Assembler::sqabs(const VRegister& vd, const VRegister& vn) {
4889 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4890 NEON2RegMisc(vd, vn, NEON_SQABS);
4891 }
4892
4893
neg(const VRegister & vd,const VRegister & vn)4894 void Assembler::neg(const VRegister& vd, const VRegister& vn) {
4895 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4896 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
4897 NEON2RegMisc(vd, vn, NEON_NEG);
4898 }
4899
4900
sqneg(const VRegister & vd,const VRegister & vn)4901 void Assembler::sqneg(const VRegister& vd, const VRegister& vn) {
4902 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4903 NEON2RegMisc(vd, vn, NEON_SQNEG);
4904 }
4905
4906
NEONXtn(const VRegister & vd,const VRegister & vn,NEON2RegMiscOp vop)4907 void Assembler::NEONXtn(const VRegister& vd,
4908 const VRegister& vn,
4909 NEON2RegMiscOp vop) {
4910 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4911 Instr format, op = vop;
4912 if (vd.IsScalar()) {
4913 VIXL_ASSERT((vd.Is1B() && vn.Is1H()) || (vd.Is1H() && vn.Is1S()) ||
4914 (vd.Is1S() && vn.Is1D()));
4915 op |= NEON_Q | NEONScalar;
4916 format = SFormat(vd);
4917 } else {
4918 VIXL_ASSERT((vd.Is8B() && vn.Is8H()) || (vd.Is4H() && vn.Is4S()) ||
4919 (vd.Is2S() && vn.Is2D()) || (vd.Is16B() && vn.Is8H()) ||
4920 (vd.Is8H() && vn.Is4S()) || (vd.Is4S() && vn.Is2D()));
4921 format = VFormat(vd);
4922 }
4923 Emit(format | op | Rn(vn) | Rd(vd));
4924 }
4925
4926
xtn(const VRegister & vd,const VRegister & vn)4927 void Assembler::xtn(const VRegister& vd, const VRegister& vn) {
4928 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4929 VIXL_ASSERT(vd.IsVector() && vd.IsD());
4930 NEONXtn(vd, vn, NEON_XTN);
4931 }
4932
4933
xtn2(const VRegister & vd,const VRegister & vn)4934 void Assembler::xtn2(const VRegister& vd, const VRegister& vn) {
4935 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4936 VIXL_ASSERT(vd.IsVector() && vd.IsQ());
4937 NEONXtn(vd, vn, NEON_XTN);
4938 }
4939
4940
sqxtn(const VRegister & vd,const VRegister & vn)4941 void Assembler::sqxtn(const VRegister& vd, const VRegister& vn) {
4942 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4943 VIXL_ASSERT(vd.IsScalar() || vd.IsD());
4944 NEONXtn(vd, vn, NEON_SQXTN);
4945 }
4946
4947
sqxtn2(const VRegister & vd,const VRegister & vn)4948 void Assembler::sqxtn2(const VRegister& vd, const VRegister& vn) {
4949 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4950 VIXL_ASSERT(vd.IsVector() && vd.IsQ());
4951 NEONXtn(vd, vn, NEON_SQXTN);
4952 }
4953
4954
sqxtun(const VRegister & vd,const VRegister & vn)4955 void Assembler::sqxtun(const VRegister& vd, const VRegister& vn) {
4956 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4957 VIXL_ASSERT(vd.IsScalar() || vd.IsD());
4958 NEONXtn(vd, vn, NEON_SQXTUN);
4959 }
4960
4961
sqxtun2(const VRegister & vd,const VRegister & vn)4962 void Assembler::sqxtun2(const VRegister& vd, const VRegister& vn) {
4963 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4964 VIXL_ASSERT(vd.IsVector() && vd.IsQ());
4965 NEONXtn(vd, vn, NEON_SQXTUN);
4966 }
4967
4968
uqxtn(const VRegister & vd,const VRegister & vn)4969 void Assembler::uqxtn(const VRegister& vd, const VRegister& vn) {
4970 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4971 VIXL_ASSERT(vd.IsScalar() || vd.IsD());
4972 NEONXtn(vd, vn, NEON_UQXTN);
4973 }
4974
4975
uqxtn2(const VRegister & vd,const VRegister & vn)4976 void Assembler::uqxtn2(const VRegister& vd, const VRegister& vn) {
4977 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4978 VIXL_ASSERT(vd.IsVector() && vd.IsQ());
4979 NEONXtn(vd, vn, NEON_UQXTN);
4980 }
4981
4982
4983 // NEON NOT and RBIT are distinguised by bit 22, the bottom bit of "size".
not_(const VRegister & vd,const VRegister & vn)4984 void Assembler::not_(const VRegister& vd, const VRegister& vn) {
4985 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4986 VIXL_ASSERT(AreSameFormat(vd, vn));
4987 VIXL_ASSERT(vd.Is8B() || vd.Is16B());
4988 Emit(VFormat(vd) | NEON_RBIT_NOT | Rn(vn) | Rd(vd));
4989 }
4990
4991
rbit(const VRegister & vd,const VRegister & vn)4992 void Assembler::rbit(const VRegister& vd, const VRegister& vn) {
4993 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4994 VIXL_ASSERT(AreSameFormat(vd, vn));
4995 VIXL_ASSERT(vd.Is8B() || vd.Is16B());
4996 Emit(VFormat(vn) | (1 << NEONSize_offset) | NEON_RBIT_NOT | Rn(vn) | Rd(vd));
4997 }
4998
4999
ext(const VRegister & vd,const VRegister & vn,const VRegister & vm,int index)5000 void Assembler::ext(const VRegister& vd,
5001 const VRegister& vn,
5002 const VRegister& vm,
5003 int index) {
5004 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5005 VIXL_ASSERT(AreSameFormat(vd, vn, vm));
5006 VIXL_ASSERT(vd.Is8B() || vd.Is16B());
5007 VIXL_ASSERT((0 <= index) && (index < vd.GetLanes()));
5008 Emit(VFormat(vd) | NEON_EXT | Rm(vm) | ImmNEONExt(index) | Rn(vn) | Rd(vd));
5009 }
5010
5011
dup(const VRegister & vd,const VRegister & vn,int vn_index)5012 void Assembler::dup(const VRegister& vd, const VRegister& vn, int vn_index) {
5013 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5014 Instr q, scalar;
5015
5016 // We support vn arguments of the form vn.VxT() or vn.T(), where x is the
5017 // number of lanes, and T is b, h, s or d.
5018 int lane_size = vn.GetLaneSizeInBytes();
5019 NEONFormatField format;
5020 switch (lane_size) {
5021 case 1:
5022 format = NEON_16B;
5023 break;
5024 case 2:
5025 format = NEON_8H;
5026 break;
5027 case 4:
5028 format = NEON_4S;
5029 break;
5030 default:
5031 VIXL_ASSERT(lane_size == 8);
5032 format = NEON_2D;
5033 break;
5034 }
5035
5036 if (vd.IsScalar()) {
5037 q = NEON_Q;
5038 scalar = NEONScalar;
5039 } else {
5040 VIXL_ASSERT(!vd.Is1D());
5041 q = vd.IsD() ? 0 : NEON_Q;
5042 scalar = 0;
5043 }
5044 Emit(q | scalar | NEON_DUP_ELEMENT | ImmNEON5(format, vn_index) | Rn(vn) |
5045 Rd(vd));
5046 }
5047
5048
mov(const VRegister & vd,const VRegister & vn,int vn_index)5049 void Assembler::mov(const VRegister& vd, const VRegister& vn, int vn_index) {
5050 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5051 VIXL_ASSERT(vd.IsScalar());
5052 dup(vd, vn, vn_index);
5053 }
5054
5055
dup(const VRegister & vd,const Register & rn)5056 void Assembler::dup(const VRegister& vd, const Register& rn) {
5057 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5058 VIXL_ASSERT(!vd.Is1D());
5059 VIXL_ASSERT(vd.Is2D() == rn.IsX());
5060 int q = vd.IsD() ? 0 : NEON_Q;
5061 Emit(q | NEON_DUP_GENERAL | ImmNEON5(VFormat(vd), 0) | Rn(rn) | Rd(vd));
5062 }
5063
5064
ins(const VRegister & vd,int vd_index,const VRegister & vn,int vn_index)5065 void Assembler::ins(const VRegister& vd,
5066 int vd_index,
5067 const VRegister& vn,
5068 int vn_index) {
5069 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5070 VIXL_ASSERT(AreSameFormat(vd, vn));
5071 // We support vd arguments of the form vd.VxT() or vd.T(), where x is the
5072 // number of lanes, and T is b, h, s or d.
5073 int lane_size = vd.GetLaneSizeInBytes();
5074 NEONFormatField format;
5075 switch (lane_size) {
5076 case 1:
5077 format = NEON_16B;
5078 break;
5079 case 2:
5080 format = NEON_8H;
5081 break;
5082 case 4:
5083 format = NEON_4S;
5084 break;
5085 default:
5086 VIXL_ASSERT(lane_size == 8);
5087 format = NEON_2D;
5088 break;
5089 }
5090
5091 VIXL_ASSERT(
5092 (0 <= vd_index) &&
5093 (vd_index < LaneCountFromFormat(static_cast<VectorFormat>(format))));
5094 VIXL_ASSERT(
5095 (0 <= vn_index) &&
5096 (vn_index < LaneCountFromFormat(static_cast<VectorFormat>(format))));
5097 Emit(NEON_INS_ELEMENT | ImmNEON5(format, vd_index) |
5098 ImmNEON4(format, vn_index) | Rn(vn) | Rd(vd));
5099 }
5100
5101
mov(const VRegister & vd,int vd_index,const VRegister & vn,int vn_index)5102 void Assembler::mov(const VRegister& vd,
5103 int vd_index,
5104 const VRegister& vn,
5105 int vn_index) {
5106 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5107 ins(vd, vd_index, vn, vn_index);
5108 }
5109
5110
ins(const VRegister & vd,int vd_index,const Register & rn)5111 void Assembler::ins(const VRegister& vd, int vd_index, const Register& rn) {
5112 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5113 // We support vd arguments of the form vd.VxT() or vd.T(), where x is the
5114 // number of lanes, and T is b, h, s or d.
5115 int lane_size = vd.GetLaneSizeInBytes();
5116 NEONFormatField format;
5117 switch (lane_size) {
5118 case 1:
5119 format = NEON_16B;
5120 VIXL_ASSERT(rn.IsW());
5121 break;
5122 case 2:
5123 format = NEON_8H;
5124 VIXL_ASSERT(rn.IsW());
5125 break;
5126 case 4:
5127 format = NEON_4S;
5128 VIXL_ASSERT(rn.IsW());
5129 break;
5130 default:
5131 VIXL_ASSERT(lane_size == 8);
5132 VIXL_ASSERT(rn.IsX());
5133 format = NEON_2D;
5134 break;
5135 }
5136
5137 VIXL_ASSERT(
5138 (0 <= vd_index) &&
5139 (vd_index < LaneCountFromFormat(static_cast<VectorFormat>(format))));
5140 Emit(NEON_INS_GENERAL | ImmNEON5(format, vd_index) | Rn(rn) | Rd(vd));
5141 }
5142
5143
mov(const VRegister & vd,int vd_index,const Register & rn)5144 void Assembler::mov(const VRegister& vd, int vd_index, const Register& rn) {
5145 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5146 ins(vd, vd_index, rn);
5147 }
5148
5149
umov(const Register & rd,const VRegister & vn,int vn_index)5150 void Assembler::umov(const Register& rd, const VRegister& vn, int vn_index) {
5151 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5152 // We support vn arguments of the form vn.VxT() or vn.T(), where x is the
5153 // number of lanes, and T is b, h, s or d.
5154 int lane_size = vn.GetLaneSizeInBytes();
5155 NEONFormatField format;
5156 Instr q = 0;
5157 switch (lane_size) {
5158 case 1:
5159 format = NEON_16B;
5160 VIXL_ASSERT(rd.IsW());
5161 break;
5162 case 2:
5163 format = NEON_8H;
5164 VIXL_ASSERT(rd.IsW());
5165 break;
5166 case 4:
5167 format = NEON_4S;
5168 VIXL_ASSERT(rd.IsW());
5169 break;
5170 default:
5171 VIXL_ASSERT(lane_size == 8);
5172 VIXL_ASSERT(rd.IsX());
5173 format = NEON_2D;
5174 q = NEON_Q;
5175 break;
5176 }
5177
5178 VIXL_ASSERT(
5179 (0 <= vn_index) &&
5180 (vn_index < LaneCountFromFormat(static_cast<VectorFormat>(format))));
5181 Emit(q | NEON_UMOV | ImmNEON5(format, vn_index) | Rn(vn) | Rd(rd));
5182 }
5183
5184
mov(const Register & rd,const VRegister & vn,int vn_index)5185 void Assembler::mov(const Register& rd, const VRegister& vn, int vn_index) {
5186 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5187 VIXL_ASSERT(vn.GetSizeInBytes() >= 4);
5188 umov(rd, vn, vn_index);
5189 }
5190
5191
smov(const Register & rd,const VRegister & vn,int vn_index)5192 void Assembler::smov(const Register& rd, const VRegister& vn, int vn_index) {
5193 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5194 // We support vn arguments of the form vn.VxT() or vn.T(), where x is the
5195 // number of lanes, and T is b, h, s.
5196 int lane_size = vn.GetLaneSizeInBytes();
5197 NEONFormatField format;
5198 Instr q = 0;
5199 VIXL_ASSERT(lane_size != 8);
5200 switch (lane_size) {
5201 case 1:
5202 format = NEON_16B;
5203 break;
5204 case 2:
5205 format = NEON_8H;
5206 break;
5207 default:
5208 VIXL_ASSERT(lane_size == 4);
5209 VIXL_ASSERT(rd.IsX());
5210 format = NEON_4S;
5211 break;
5212 }
5213 q = rd.IsW() ? 0 : NEON_Q;
5214 VIXL_ASSERT(
5215 (0 <= vn_index) &&
5216 (vn_index < LaneCountFromFormat(static_cast<VectorFormat>(format))));
5217 Emit(q | NEON_SMOV | ImmNEON5(format, vn_index) | Rn(vn) | Rd(rd));
5218 }
5219
5220
cls(const VRegister & vd,const VRegister & vn)5221 void Assembler::cls(const VRegister& vd, const VRegister& vn) {
5222 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5223 VIXL_ASSERT(AreSameFormat(vd, vn));
5224 VIXL_ASSERT(!vd.Is1D() && !vd.Is2D());
5225 Emit(VFormat(vn) | NEON_CLS | Rn(vn) | Rd(vd));
5226 }
5227
5228
clz(const VRegister & vd,const VRegister & vn)5229 void Assembler::clz(const VRegister& vd, const VRegister& vn) {
5230 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5231 VIXL_ASSERT(AreSameFormat(vd, vn));
5232 VIXL_ASSERT(!vd.Is1D() && !vd.Is2D());
5233 Emit(VFormat(vn) | NEON_CLZ | Rn(vn) | Rd(vd));
5234 }
5235
5236
cnt(const VRegister & vd,const VRegister & vn)5237 void Assembler::cnt(const VRegister& vd, const VRegister& vn) {
5238 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5239 VIXL_ASSERT(AreSameFormat(vd, vn));
5240 VIXL_ASSERT(vd.Is8B() || vd.Is16B());
5241 Emit(VFormat(vn) | NEON_CNT | Rn(vn) | Rd(vd));
5242 }
5243
5244
rev16(const VRegister & vd,const VRegister & vn)5245 void Assembler::rev16(const VRegister& vd, const VRegister& vn) {
5246 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5247 VIXL_ASSERT(AreSameFormat(vd, vn));
5248 VIXL_ASSERT(vd.Is8B() || vd.Is16B());
5249 Emit(VFormat(vn) | NEON_REV16 | Rn(vn) | Rd(vd));
5250 }
5251
5252
rev32(const VRegister & vd,const VRegister & vn)5253 void Assembler::rev32(const VRegister& vd, const VRegister& vn) {
5254 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5255 VIXL_ASSERT(AreSameFormat(vd, vn));
5256 VIXL_ASSERT(vd.Is8B() || vd.Is16B() || vd.Is4H() || vd.Is8H());
5257 Emit(VFormat(vn) | NEON_REV32 | Rn(vn) | Rd(vd));
5258 }
5259
5260
rev64(const VRegister & vd,const VRegister & vn)5261 void Assembler::rev64(const VRegister& vd, const VRegister& vn) {
5262 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5263 VIXL_ASSERT(AreSameFormat(vd, vn));
5264 VIXL_ASSERT(!vd.Is1D() && !vd.Is2D());
5265 Emit(VFormat(vn) | NEON_REV64 | Rn(vn) | Rd(vd));
5266 }
5267
5268
ursqrte(const VRegister & vd,const VRegister & vn)5269 void Assembler::ursqrte(const VRegister& vd, const VRegister& vn) {
5270 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5271 VIXL_ASSERT(AreSameFormat(vd, vn));
5272 VIXL_ASSERT(vd.Is2S() || vd.Is4S());
5273 Emit(VFormat(vn) | NEON_URSQRTE | Rn(vn) | Rd(vd));
5274 }
5275
5276
urecpe(const VRegister & vd,const VRegister & vn)5277 void Assembler::urecpe(const VRegister& vd, const VRegister& vn) {
5278 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5279 VIXL_ASSERT(AreSameFormat(vd, vn));
5280 VIXL_ASSERT(vd.Is2S() || vd.Is4S());
5281 Emit(VFormat(vn) | NEON_URECPE | Rn(vn) | Rd(vd));
5282 }
5283
5284
NEONAddlp(const VRegister & vd,const VRegister & vn,NEON2RegMiscOp op)5285 void Assembler::NEONAddlp(const VRegister& vd,
5286 const VRegister& vn,
5287 NEON2RegMiscOp op) {
5288 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5289 VIXL_ASSERT((op == NEON_SADDLP) || (op == NEON_UADDLP) ||
5290 (op == NEON_SADALP) || (op == NEON_UADALP));
5291
5292 VIXL_ASSERT((vn.Is8B() && vd.Is4H()) || (vn.Is4H() && vd.Is2S()) ||
5293 (vn.Is2S() && vd.Is1D()) || (vn.Is16B() && vd.Is8H()) ||
5294 (vn.Is8H() && vd.Is4S()) || (vn.Is4S() && vd.Is2D()));
5295 Emit(VFormat(vn) | op | Rn(vn) | Rd(vd));
5296 }
5297
5298
saddlp(const VRegister & vd,const VRegister & vn)5299 void Assembler::saddlp(const VRegister& vd, const VRegister& vn) {
5300 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5301 NEONAddlp(vd, vn, NEON_SADDLP);
5302 }
5303
5304
uaddlp(const VRegister & vd,const VRegister & vn)5305 void Assembler::uaddlp(const VRegister& vd, const VRegister& vn) {
5306 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5307 NEONAddlp(vd, vn, NEON_UADDLP);
5308 }
5309
5310
sadalp(const VRegister & vd,const VRegister & vn)5311 void Assembler::sadalp(const VRegister& vd, const VRegister& vn) {
5312 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5313 NEONAddlp(vd, vn, NEON_SADALP);
5314 }
5315
5316
uadalp(const VRegister & vd,const VRegister & vn)5317 void Assembler::uadalp(const VRegister& vd, const VRegister& vn) {
5318 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5319 NEONAddlp(vd, vn, NEON_UADALP);
5320 }
5321
5322
NEONAcrossLanesL(const VRegister & vd,const VRegister & vn,NEONAcrossLanesOp op)5323 void Assembler::NEONAcrossLanesL(const VRegister& vd,
5324 const VRegister& vn,
5325 NEONAcrossLanesOp op) {
5326 VIXL_ASSERT((vn.Is8B() && vd.Is1H()) || (vn.Is16B() && vd.Is1H()) ||
5327 (vn.Is4H() && vd.Is1S()) || (vn.Is8H() && vd.Is1S()) ||
5328 (vn.Is4S() && vd.Is1D()));
5329 Emit(VFormat(vn) | op | Rn(vn) | Rd(vd));
5330 }
5331
5332
saddlv(const VRegister & vd,const VRegister & vn)5333 void Assembler::saddlv(const VRegister& vd, const VRegister& vn) {
5334 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5335 NEONAcrossLanesL(vd, vn, NEON_SADDLV);
5336 }
5337
5338
uaddlv(const VRegister & vd,const VRegister & vn)5339 void Assembler::uaddlv(const VRegister& vd, const VRegister& vn) {
5340 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5341 NEONAcrossLanesL(vd, vn, NEON_UADDLV);
5342 }
5343
5344
NEONAcrossLanes(const VRegister & vd,const VRegister & vn,NEONAcrossLanesOp op,Instr op_half)5345 void Assembler::NEONAcrossLanes(const VRegister& vd,
5346 const VRegister& vn,
5347 NEONAcrossLanesOp op,
5348 Instr op_half) {
5349 VIXL_ASSERT((vn.Is8B() && vd.Is1B()) || (vn.Is16B() && vd.Is1B()) ||
5350 (vn.Is4H() && vd.Is1H()) || (vn.Is8H() && vd.Is1H()) ||
5351 (vn.Is4S() && vd.Is1S()));
5352 if ((op & NEONAcrossLanesFPFMask) == NEONAcrossLanesFPFixed) {
5353 if (vd.Is1H()) {
5354 VIXL_ASSERT(op_half != 0);
5355 Instr vop = op_half;
5356 if (vn.Is8H()) {
5357 vop |= NEON_Q;
5358 }
5359 Emit(vop | Rn(vn) | Rd(vd));
5360 } else {
5361 Emit(FPFormat(vn) | op | Rn(vn) | Rd(vd));
5362 }
5363 } else {
5364 Emit(VFormat(vn) | op | Rn(vn) | Rd(vd));
5365 }
5366 }
5367
5368 // clang-format off
5369 #define NEON_ACROSSLANES_LIST(V) \
5370 V(addv, NEON_ADDV) \
5371 V(smaxv, NEON_SMAXV) \
5372 V(sminv, NEON_SMINV) \
5373 V(umaxv, NEON_UMAXV) \
5374 V(uminv, NEON_UMINV)
5375 // clang-format on
5376
5377 #define VIXL_DEFINE_ASM_FUNC(FN, OP) \
5378 void Assembler::FN(const VRegister& vd, const VRegister& vn) { \
5379 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON)); \
5380 NEONAcrossLanes(vd, vn, OP, 0); \
5381 }
5382 NEON_ACROSSLANES_LIST(VIXL_DEFINE_ASM_FUNC)
5383 #undef VIXL_DEFINE_ASM_FUNC
5384
5385
5386 // clang-format off
5387 #define NEON_ACROSSLANES_FP_LIST(V) \
5388 V(fmaxv, NEON_FMAXV, NEON_FMAXV_H) \
5389 V(fminv, NEON_FMINV, NEON_FMINV_H) \
5390 V(fmaxnmv, NEON_FMAXNMV, NEON_FMAXNMV_H) \
5391 V(fminnmv, NEON_FMINNMV, NEON_FMINNMV_H) \
5392 // clang-format on
5393
5394 #define VIXL_DEFINE_ASM_FUNC(FN, OP, OP_H) \
5395 void Assembler::FN(const VRegister& vd, const VRegister& vn) { \
5396 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON)); \
5397 if (vd.Is1H()) VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf)); \
5398 VIXL_ASSERT(vd.Is1S() || vd.Is1H()); \
5399 NEONAcrossLanes(vd, vn, OP, OP_H); \
5400 }
NEON_ACROSSLANES_FP_LIST(VIXL_DEFINE_ASM_FUNC)5401 NEON_ACROSSLANES_FP_LIST(VIXL_DEFINE_ASM_FUNC)
5402 #undef VIXL_DEFINE_ASM_FUNC
5403
5404
5405 void Assembler::NEONPerm(const VRegister& vd,
5406 const VRegister& vn,
5407 const VRegister& vm,
5408 NEONPermOp op) {
5409 VIXL_ASSERT(AreSameFormat(vd, vn, vm));
5410 VIXL_ASSERT(!vd.Is1D());
5411 Emit(VFormat(vd) | op | Rm(vm) | Rn(vn) | Rd(vd));
5412 }
5413
5414
trn1(const VRegister & vd,const VRegister & vn,const VRegister & vm)5415 void Assembler::trn1(const VRegister& vd,
5416 const VRegister& vn,
5417 const VRegister& vm) {
5418 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5419 NEONPerm(vd, vn, vm, NEON_TRN1);
5420 }
5421
5422
trn2(const VRegister & vd,const VRegister & vn,const VRegister & vm)5423 void Assembler::trn2(const VRegister& vd,
5424 const VRegister& vn,
5425 const VRegister& vm) {
5426 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5427 NEONPerm(vd, vn, vm, NEON_TRN2);
5428 }
5429
5430
uzp1(const VRegister & vd,const VRegister & vn,const VRegister & vm)5431 void Assembler::uzp1(const VRegister& vd,
5432 const VRegister& vn,
5433 const VRegister& vm) {
5434 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5435 NEONPerm(vd, vn, vm, NEON_UZP1);
5436 }
5437
5438
uzp2(const VRegister & vd,const VRegister & vn,const VRegister & vm)5439 void Assembler::uzp2(const VRegister& vd,
5440 const VRegister& vn,
5441 const VRegister& vm) {
5442 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5443 NEONPerm(vd, vn, vm, NEON_UZP2);
5444 }
5445
5446
zip1(const VRegister & vd,const VRegister & vn,const VRegister & vm)5447 void Assembler::zip1(const VRegister& vd,
5448 const VRegister& vn,
5449 const VRegister& vm) {
5450 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5451 NEONPerm(vd, vn, vm, NEON_ZIP1);
5452 }
5453
5454
zip2(const VRegister & vd,const VRegister & vn,const VRegister & vm)5455 void Assembler::zip2(const VRegister& vd,
5456 const VRegister& vn,
5457 const VRegister& vm) {
5458 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5459 NEONPerm(vd, vn, vm, NEON_ZIP2);
5460 }
5461
5462
NEONShiftImmediate(const VRegister & vd,const VRegister & vn,NEONShiftImmediateOp op,int immh_immb)5463 void Assembler::NEONShiftImmediate(const VRegister& vd,
5464 const VRegister& vn,
5465 NEONShiftImmediateOp op,
5466 int immh_immb) {
5467 VIXL_ASSERT(AreSameFormat(vd, vn));
5468 Instr q, scalar;
5469 if (vn.IsScalar()) {
5470 q = NEON_Q;
5471 scalar = NEONScalar;
5472 } else {
5473 q = vd.IsD() ? 0 : NEON_Q;
5474 scalar = 0;
5475 }
5476 Emit(q | op | scalar | immh_immb | Rn(vn) | Rd(vd));
5477 }
5478
5479
NEONShiftLeftImmediate(const VRegister & vd,const VRegister & vn,int shift,NEONShiftImmediateOp op)5480 void Assembler::NEONShiftLeftImmediate(const VRegister& vd,
5481 const VRegister& vn,
5482 int shift,
5483 NEONShiftImmediateOp op) {
5484 int lane_size_in_bits = vn.GetLaneSizeInBits();
5485 VIXL_ASSERT((shift >= 0) && (shift < lane_size_in_bits));
5486 NEONShiftImmediate(vd, vn, op, (lane_size_in_bits + shift) << 16);
5487 }
5488
5489
NEONShiftRightImmediate(const VRegister & vd,const VRegister & vn,int shift,NEONShiftImmediateOp op)5490 void Assembler::NEONShiftRightImmediate(const VRegister& vd,
5491 const VRegister& vn,
5492 int shift,
5493 NEONShiftImmediateOp op) {
5494 int lane_size_in_bits = vn.GetLaneSizeInBits();
5495 VIXL_ASSERT((shift >= 1) && (shift <= lane_size_in_bits));
5496 NEONShiftImmediate(vd, vn, op, ((2 * lane_size_in_bits) - shift) << 16);
5497 }
5498
5499
NEONShiftImmediateL(const VRegister & vd,const VRegister & vn,int shift,NEONShiftImmediateOp op)5500 void Assembler::NEONShiftImmediateL(const VRegister& vd,
5501 const VRegister& vn,
5502 int shift,
5503 NEONShiftImmediateOp op) {
5504 int lane_size_in_bits = vn.GetLaneSizeInBits();
5505 VIXL_ASSERT((shift >= 0) && (shift < lane_size_in_bits));
5506 int immh_immb = (lane_size_in_bits + shift) << 16;
5507
5508 VIXL_ASSERT((vn.Is8B() && vd.Is8H()) || (vn.Is4H() && vd.Is4S()) ||
5509 (vn.Is2S() && vd.Is2D()) || (vn.Is16B() && vd.Is8H()) ||
5510 (vn.Is8H() && vd.Is4S()) || (vn.Is4S() && vd.Is2D()));
5511 Instr q;
5512 q = vn.IsD() ? 0 : NEON_Q;
5513 Emit(q | op | immh_immb | Rn(vn) | Rd(vd));
5514 }
5515
5516
NEONShiftImmediateN(const VRegister & vd,const VRegister & vn,int shift,NEONShiftImmediateOp op)5517 void Assembler::NEONShiftImmediateN(const VRegister& vd,
5518 const VRegister& vn,
5519 int shift,
5520 NEONShiftImmediateOp op) {
5521 Instr q, scalar;
5522 int lane_size_in_bits = vd.GetLaneSizeInBits();
5523 VIXL_ASSERT((shift >= 1) && (shift <= lane_size_in_bits));
5524 int immh_immb = (2 * lane_size_in_bits - shift) << 16;
5525
5526 if (vn.IsScalar()) {
5527 VIXL_ASSERT((vd.Is1B() && vn.Is1H()) || (vd.Is1H() && vn.Is1S()) ||
5528 (vd.Is1S() && vn.Is1D()));
5529 q = NEON_Q;
5530 scalar = NEONScalar;
5531 } else {
5532 VIXL_ASSERT((vd.Is8B() && vn.Is8H()) || (vd.Is4H() && vn.Is4S()) ||
5533 (vd.Is2S() && vn.Is2D()) || (vd.Is16B() && vn.Is8H()) ||
5534 (vd.Is8H() && vn.Is4S()) || (vd.Is4S() && vn.Is2D()));
5535 scalar = 0;
5536 q = vd.IsD() ? 0 : NEON_Q;
5537 }
5538 Emit(q | op | scalar | immh_immb | Rn(vn) | Rd(vd));
5539 }
5540
5541
shl(const VRegister & vd,const VRegister & vn,int shift)5542 void Assembler::shl(const VRegister& vd, const VRegister& vn, int shift) {
5543 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5544 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
5545 NEONShiftLeftImmediate(vd, vn, shift, NEON_SHL);
5546 }
5547
5548
sli(const VRegister & vd,const VRegister & vn,int shift)5549 void Assembler::sli(const VRegister& vd, const VRegister& vn, int shift) {
5550 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5551 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
5552 NEONShiftLeftImmediate(vd, vn, shift, NEON_SLI);
5553 }
5554
5555
sqshl(const VRegister & vd,const VRegister & vn,int shift)5556 void Assembler::sqshl(const VRegister& vd, const VRegister& vn, int shift) {
5557 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5558 NEONShiftLeftImmediate(vd, vn, shift, NEON_SQSHL_imm);
5559 }
5560
5561
sqshlu(const VRegister & vd,const VRegister & vn,int shift)5562 void Assembler::sqshlu(const VRegister& vd, const VRegister& vn, int shift) {
5563 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5564 NEONShiftLeftImmediate(vd, vn, shift, NEON_SQSHLU);
5565 }
5566
5567
uqshl(const VRegister & vd,const VRegister & vn,int shift)5568 void Assembler::uqshl(const VRegister& vd, const VRegister& vn, int shift) {
5569 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5570 NEONShiftLeftImmediate(vd, vn, shift, NEON_UQSHL_imm);
5571 }
5572
5573
sshll(const VRegister & vd,const VRegister & vn,int shift)5574 void Assembler::sshll(const VRegister& vd, const VRegister& vn, int shift) {
5575 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5576 VIXL_ASSERT(vn.IsD());
5577 NEONShiftImmediateL(vd, vn, shift, NEON_SSHLL);
5578 }
5579
5580
sshll2(const VRegister & vd,const VRegister & vn,int shift)5581 void Assembler::sshll2(const VRegister& vd, const VRegister& vn, int shift) {
5582 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5583 VIXL_ASSERT(vn.IsQ());
5584 NEONShiftImmediateL(vd, vn, shift, NEON_SSHLL);
5585 }
5586
5587
sxtl(const VRegister & vd,const VRegister & vn)5588 void Assembler::sxtl(const VRegister& vd, const VRegister& vn) {
5589 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5590 sshll(vd, vn, 0);
5591 }
5592
5593
sxtl2(const VRegister & vd,const VRegister & vn)5594 void Assembler::sxtl2(const VRegister& vd, const VRegister& vn) {
5595 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5596 sshll2(vd, vn, 0);
5597 }
5598
5599
ushll(const VRegister & vd,const VRegister & vn,int shift)5600 void Assembler::ushll(const VRegister& vd, const VRegister& vn, int shift) {
5601 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5602 VIXL_ASSERT(vn.IsD());
5603 NEONShiftImmediateL(vd, vn, shift, NEON_USHLL);
5604 }
5605
5606
ushll2(const VRegister & vd,const VRegister & vn,int shift)5607 void Assembler::ushll2(const VRegister& vd, const VRegister& vn, int shift) {
5608 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5609 VIXL_ASSERT(vn.IsQ());
5610 NEONShiftImmediateL(vd, vn, shift, NEON_USHLL);
5611 }
5612
5613
uxtl(const VRegister & vd,const VRegister & vn)5614 void Assembler::uxtl(const VRegister& vd, const VRegister& vn) {
5615 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5616 ushll(vd, vn, 0);
5617 }
5618
5619
uxtl2(const VRegister & vd,const VRegister & vn)5620 void Assembler::uxtl2(const VRegister& vd, const VRegister& vn) {
5621 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5622 ushll2(vd, vn, 0);
5623 }
5624
5625
sri(const VRegister & vd,const VRegister & vn,int shift)5626 void Assembler::sri(const VRegister& vd, const VRegister& vn, int shift) {
5627 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5628 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
5629 NEONShiftRightImmediate(vd, vn, shift, NEON_SRI);
5630 }
5631
5632
sshr(const VRegister & vd,const VRegister & vn,int shift)5633 void Assembler::sshr(const VRegister& vd, const VRegister& vn, int shift) {
5634 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5635 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
5636 NEONShiftRightImmediate(vd, vn, shift, NEON_SSHR);
5637 }
5638
5639
ushr(const VRegister & vd,const VRegister & vn,int shift)5640 void Assembler::ushr(const VRegister& vd, const VRegister& vn, int shift) {
5641 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5642 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
5643 NEONShiftRightImmediate(vd, vn, shift, NEON_USHR);
5644 }
5645
5646
srshr(const VRegister & vd,const VRegister & vn,int shift)5647 void Assembler::srshr(const VRegister& vd, const VRegister& vn, int shift) {
5648 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5649 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
5650 NEONShiftRightImmediate(vd, vn, shift, NEON_SRSHR);
5651 }
5652
5653
urshr(const VRegister & vd,const VRegister & vn,int shift)5654 void Assembler::urshr(const VRegister& vd, const VRegister& vn, int shift) {
5655 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5656 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
5657 NEONShiftRightImmediate(vd, vn, shift, NEON_URSHR);
5658 }
5659
5660
ssra(const VRegister & vd,const VRegister & vn,int shift)5661 void Assembler::ssra(const VRegister& vd, const VRegister& vn, int shift) {
5662 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5663 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
5664 NEONShiftRightImmediate(vd, vn, shift, NEON_SSRA);
5665 }
5666
5667
usra(const VRegister & vd,const VRegister & vn,int shift)5668 void Assembler::usra(const VRegister& vd, const VRegister& vn, int shift) {
5669 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5670 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
5671 NEONShiftRightImmediate(vd, vn, shift, NEON_USRA);
5672 }
5673
5674
srsra(const VRegister & vd,const VRegister & vn,int shift)5675 void Assembler::srsra(const VRegister& vd, const VRegister& vn, int shift) {
5676 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5677 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
5678 NEONShiftRightImmediate(vd, vn, shift, NEON_SRSRA);
5679 }
5680
5681
ursra(const VRegister & vd,const VRegister & vn,int shift)5682 void Assembler::ursra(const VRegister& vd, const VRegister& vn, int shift) {
5683 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5684 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
5685 NEONShiftRightImmediate(vd, vn, shift, NEON_URSRA);
5686 }
5687
5688
shrn(const VRegister & vd,const VRegister & vn,int shift)5689 void Assembler::shrn(const VRegister& vd, const VRegister& vn, int shift) {
5690 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5691 VIXL_ASSERT(vn.IsVector() && vd.IsD());
5692 NEONShiftImmediateN(vd, vn, shift, NEON_SHRN);
5693 }
5694
5695
shrn2(const VRegister & vd,const VRegister & vn,int shift)5696 void Assembler::shrn2(const VRegister& vd, const VRegister& vn, int shift) {
5697 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5698 VIXL_ASSERT(vn.IsVector() && vd.IsQ());
5699 NEONShiftImmediateN(vd, vn, shift, NEON_SHRN);
5700 }
5701
5702
rshrn(const VRegister & vd,const VRegister & vn,int shift)5703 void Assembler::rshrn(const VRegister& vd, const VRegister& vn, int shift) {
5704 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5705 VIXL_ASSERT(vn.IsVector() && vd.IsD());
5706 NEONShiftImmediateN(vd, vn, shift, NEON_RSHRN);
5707 }
5708
5709
rshrn2(const VRegister & vd,const VRegister & vn,int shift)5710 void Assembler::rshrn2(const VRegister& vd, const VRegister& vn, int shift) {
5711 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5712 VIXL_ASSERT(vn.IsVector() && vd.IsQ());
5713 NEONShiftImmediateN(vd, vn, shift, NEON_RSHRN);
5714 }
5715
5716
sqshrn(const VRegister & vd,const VRegister & vn,int shift)5717 void Assembler::sqshrn(const VRegister& vd, const VRegister& vn, int shift) {
5718 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5719 VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
5720 NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRN);
5721 }
5722
5723
sqshrn2(const VRegister & vd,const VRegister & vn,int shift)5724 void Assembler::sqshrn2(const VRegister& vd, const VRegister& vn, int shift) {
5725 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5726 VIXL_ASSERT(vn.IsVector() && vd.IsQ());
5727 NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRN);
5728 }
5729
5730
sqrshrn(const VRegister & vd,const VRegister & vn,int shift)5731 void Assembler::sqrshrn(const VRegister& vd, const VRegister& vn, int shift) {
5732 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5733 VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
5734 NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRN);
5735 }
5736
5737
sqrshrn2(const VRegister & vd,const VRegister & vn,int shift)5738 void Assembler::sqrshrn2(const VRegister& vd, const VRegister& vn, int shift) {
5739 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5740 VIXL_ASSERT(vn.IsVector() && vd.IsQ());
5741 NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRN);
5742 }
5743
5744
sqshrun(const VRegister & vd,const VRegister & vn,int shift)5745 void Assembler::sqshrun(const VRegister& vd, const VRegister& vn, int shift) {
5746 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5747 VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
5748 NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRUN);
5749 }
5750
5751
sqshrun2(const VRegister & vd,const VRegister & vn,int shift)5752 void Assembler::sqshrun2(const VRegister& vd, const VRegister& vn, int shift) {
5753 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5754 VIXL_ASSERT(vn.IsVector() && vd.IsQ());
5755 NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRUN);
5756 }
5757
5758
sqrshrun(const VRegister & vd,const VRegister & vn,int shift)5759 void Assembler::sqrshrun(const VRegister& vd, const VRegister& vn, int shift) {
5760 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5761 VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
5762 NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRUN);
5763 }
5764
5765
sqrshrun2(const VRegister & vd,const VRegister & vn,int shift)5766 void Assembler::sqrshrun2(const VRegister& vd, const VRegister& vn, int shift) {
5767 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5768 VIXL_ASSERT(vn.IsVector() && vd.IsQ());
5769 NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRUN);
5770 }
5771
5772
uqshrn(const VRegister & vd,const VRegister & vn,int shift)5773 void Assembler::uqshrn(const VRegister& vd, const VRegister& vn, int shift) {
5774 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5775 VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
5776 NEONShiftImmediateN(vd, vn, shift, NEON_UQSHRN);
5777 }
5778
5779
uqshrn2(const VRegister & vd,const VRegister & vn,int shift)5780 void Assembler::uqshrn2(const VRegister& vd, const VRegister& vn, int shift) {
5781 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5782 VIXL_ASSERT(vn.IsVector() && vd.IsQ());
5783 NEONShiftImmediateN(vd, vn, shift, NEON_UQSHRN);
5784 }
5785
5786
uqrshrn(const VRegister & vd,const VRegister & vn,int shift)5787 void Assembler::uqrshrn(const VRegister& vd, const VRegister& vn, int shift) {
5788 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5789 VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
5790 NEONShiftImmediateN(vd, vn, shift, NEON_UQRSHRN);
5791 }
5792
5793
uqrshrn2(const VRegister & vd,const VRegister & vn,int shift)5794 void Assembler::uqrshrn2(const VRegister& vd, const VRegister& vn, int shift) {
5795 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5796 VIXL_ASSERT(vn.IsVector() && vd.IsQ());
5797 NEONShiftImmediateN(vd, vn, shift, NEON_UQRSHRN);
5798 }
5799
smmla(const VRegister & vd,const VRegister & vn,const VRegister & vm)5800 void Assembler::smmla(const VRegister& vd, const VRegister& vn, const VRegister& vm) {
5801 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5802 VIXL_ASSERT(CPUHas(CPUFeatures::kI8MM));
5803 VIXL_ASSERT(vd.IsLaneSizeS());
5804 VIXL_ASSERT(vn.IsLaneSizeB() && vm.IsLaneSizeB());
5805
5806 Emit(0x4e80a400 | Rd(vd) | Rn(vn) | Rm(vm));
5807 }
5808
usmmla(const VRegister & vd,const VRegister & vn,const VRegister & vm)5809 void Assembler::usmmla(const VRegister& vd, const VRegister& vn, const VRegister& vm) {
5810 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5811 VIXL_ASSERT(CPUHas(CPUFeatures::kI8MM));
5812 VIXL_ASSERT(vd.IsLaneSizeS());
5813 VIXL_ASSERT(vn.IsLaneSizeB() && vm.IsLaneSizeB());
5814
5815 Emit(0x4e80ac00 | Rd(vd) | Rn(vn) | Rm(vm));
5816 }
5817
ummla(const VRegister & vd,const VRegister & vn,const VRegister & vm)5818 void Assembler::ummla(const VRegister& vd, const VRegister& vn, const VRegister& vm) {
5819 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5820 VIXL_ASSERT(CPUHas(CPUFeatures::kI8MM));
5821 VIXL_ASSERT(vd.IsLaneSizeS());
5822 VIXL_ASSERT(vn.IsLaneSizeB() && vm.IsLaneSizeB());
5823
5824 Emit(0x6e80a400 | Rd(vd) | Rn(vn) | Rm(vm));
5825 }
5826
5827 // Note:
5828 // For all ToImm instructions below, a difference in case
5829 // for the same letter indicates a negated bit.
5830 // If b is 1, then B is 0.
FP16ToImm8(Float16 imm)5831 uint32_t Assembler::FP16ToImm8(Float16 imm) {
5832 VIXL_ASSERT(IsImmFP16(imm));
5833 // Half: aBbb.cdef.gh00.0000 (16 bits)
5834 uint16_t bits = Float16ToRawbits(imm);
5835 // bit7: a000.0000
5836 uint16_t bit7 = ((bits >> 15) & 0x1) << 7;
5837 // bit6: 0b00.0000
5838 uint16_t bit6 = ((bits >> 13) & 0x1) << 6;
5839 // bit5_to_0: 00cd.efgh
5840 uint16_t bit5_to_0 = (bits >> 6) & 0x3f;
5841 uint32_t result = static_cast<uint32_t>(bit7 | bit6 | bit5_to_0);
5842 return result;
5843 }
5844
5845
ImmFP16(Float16 imm)5846 Instr Assembler::ImmFP16(Float16 imm) {
5847 return FP16ToImm8(imm) << ImmFP_offset;
5848 }
5849
5850
FP32ToImm8(float imm)5851 uint32_t Assembler::FP32ToImm8(float imm) {
5852 // bits: aBbb.bbbc.defg.h000.0000.0000.0000.0000
5853 uint32_t bits = FloatToRawbits(imm);
5854 VIXL_ASSERT(IsImmFP32(bits));
5855 // bit7: a000.0000
5856 uint32_t bit7 = ((bits >> 31) & 0x1) << 7;
5857 // bit6: 0b00.0000
5858 uint32_t bit6 = ((bits >> 29) & 0x1) << 6;
5859 // bit5_to_0: 00cd.efgh
5860 uint32_t bit5_to_0 = (bits >> 19) & 0x3f;
5861
5862 return bit7 | bit6 | bit5_to_0;
5863 }
5864
5865
ImmFP32(float imm)5866 Instr Assembler::ImmFP32(float imm) { return FP32ToImm8(imm) << ImmFP_offset; }
5867
5868
FP64ToImm8(double imm)5869 uint32_t Assembler::FP64ToImm8(double imm) {
5870 // bits: aBbb.bbbb.bbcd.efgh.0000.0000.0000.0000
5871 // 0000.0000.0000.0000.0000.0000.0000.0000
5872 uint64_t bits = DoubleToRawbits(imm);
5873 VIXL_ASSERT(IsImmFP64(bits));
5874 // bit7: a000.0000
5875 uint64_t bit7 = ((bits >> 63) & 0x1) << 7;
5876 // bit6: 0b00.0000
5877 uint64_t bit6 = ((bits >> 61) & 0x1) << 6;
5878 // bit5_to_0: 00cd.efgh
5879 uint64_t bit5_to_0 = (bits >> 48) & 0x3f;
5880
5881 return static_cast<uint32_t>(bit7 | bit6 | bit5_to_0);
5882 }
5883
5884
ImmFP64(double imm)5885 Instr Assembler::ImmFP64(double imm) { return FP64ToImm8(imm) << ImmFP_offset; }
5886
5887
5888 // Code generation helpers.
OneInstrMoveImmediateHelper(Assembler * assm,const Register & dst,uint64_t imm)5889 bool Assembler::OneInstrMoveImmediateHelper(Assembler* assm,
5890 const Register& dst,
5891 uint64_t imm) {
5892 bool emit_code = assm != NULL;
5893 unsigned n, imm_s, imm_r;
5894 int reg_size = dst.GetSizeInBits();
5895
5896 if (IsImmMovz(imm, reg_size) && !dst.IsSP()) {
5897 // Immediate can be represented in a move zero instruction. Movz can't write
5898 // to the stack pointer.
5899 if (emit_code) {
5900 assm->movz(dst, imm);
5901 }
5902 return true;
5903 } else if (IsImmMovn(imm, reg_size) && !dst.IsSP()) {
5904 // Immediate can be represented in a move negative instruction. Movn can't
5905 // write to the stack pointer.
5906 if (emit_code) {
5907 assm->movn(dst, dst.Is64Bits() ? ~imm : (~imm & kWRegMask));
5908 }
5909 return true;
5910 } else if (IsImmLogical(imm, reg_size, &n, &imm_s, &imm_r)) {
5911 // Immediate can be represented in a logical orr instruction.
5912 VIXL_ASSERT(!dst.IsZero());
5913 if (emit_code) {
5914 assm->LogicalImmediate(dst,
5915 AppropriateZeroRegFor(dst),
5916 n,
5917 imm_s,
5918 imm_r,
5919 ORR);
5920 }
5921 return true;
5922 }
5923 return false;
5924 }
5925
5926
MoveWide(const Register & rd,uint64_t imm,int shift,MoveWideImmediateOp mov_op)5927 void Assembler::MoveWide(const Register& rd,
5928 uint64_t imm,
5929 int shift,
5930 MoveWideImmediateOp mov_op) {
5931 // Ignore the top 32 bits of an immediate if we're moving to a W register.
5932 if (rd.Is32Bits()) {
5933 // Check that the top 32 bits are zero (a positive 32-bit number) or top
5934 // 33 bits are one (a negative 32-bit number, sign extended to 64 bits).
5935 VIXL_ASSERT(((imm >> kWRegSize) == 0) ||
5936 ((imm >> (kWRegSize - 1)) == 0x1ffffffff));
5937 imm &= kWRegMask;
5938 }
5939
5940 if (shift >= 0) {
5941 // Explicit shift specified.
5942 VIXL_ASSERT((shift == 0) || (shift == 16) || (shift == 32) ||
5943 (shift == 48));
5944 VIXL_ASSERT(rd.Is64Bits() || (shift == 0) || (shift == 16));
5945 shift /= 16;
5946 } else {
5947 // Calculate a new immediate and shift combination to encode the immediate
5948 // argument.
5949 VIXL_ASSERT(shift == -1);
5950 shift = 0;
5951 if ((imm & 0xffffffffffff0000) == 0) {
5952 // Nothing to do.
5953 } else if ((imm & 0xffffffff0000ffff) == 0) {
5954 imm >>= 16;
5955 shift = 1;
5956 } else if ((imm & 0xffff0000ffffffff) == 0) {
5957 VIXL_ASSERT(rd.Is64Bits());
5958 imm >>= 32;
5959 shift = 2;
5960 } else if ((imm & 0x0000ffffffffffff) == 0) {
5961 VIXL_ASSERT(rd.Is64Bits());
5962 imm >>= 48;
5963 shift = 3;
5964 }
5965 }
5966
5967 VIXL_ASSERT(IsUint16(imm));
5968
5969 Emit(SF(rd) | MoveWideImmediateFixed | mov_op | Rd(rd) | ImmMoveWide(imm) |
5970 ShiftMoveWide(shift));
5971 }
5972
5973
AddSub(const Register & rd,const Register & rn,const Operand & operand,FlagsUpdate S,AddSubOp op)5974 void Assembler::AddSub(const Register& rd,
5975 const Register& rn,
5976 const Operand& operand,
5977 FlagsUpdate S,
5978 AddSubOp op) {
5979 VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
5980 if (operand.IsImmediate()) {
5981 int64_t immediate = operand.GetImmediate();
5982 VIXL_ASSERT(IsImmAddSub(immediate));
5983 Instr dest_reg = (S == SetFlags) ? Rd(rd) : RdSP(rd);
5984 Emit(SF(rd) | AddSubImmediateFixed | op | Flags(S) |
5985 ImmAddSub(static_cast<int>(immediate)) | dest_reg | RnSP(rn));
5986 } else if (operand.IsShiftedRegister()) {
5987 VIXL_ASSERT(operand.GetRegister().GetSizeInBits() == rd.GetSizeInBits());
5988 VIXL_ASSERT(operand.GetShift() != ROR);
5989
5990 // For instructions of the form:
5991 // add/sub wsp, <Wn>, <Wm> [, LSL #0-3 ]
5992 // add/sub <Wd>, wsp, <Wm> [, LSL #0-3 ]
5993 // add/sub wsp, wsp, <Wm> [, LSL #0-3 ]
5994 // adds/subs <Wd>, wsp, <Wm> [, LSL #0-3 ]
5995 // or their 64-bit register equivalents, convert the operand from shifted to
5996 // extended register mode, and emit an add/sub extended instruction.
5997 if (rn.IsSP() || rd.IsSP()) {
5998 VIXL_ASSERT(!(rd.IsSP() && (S == SetFlags)));
5999 DataProcExtendedRegister(rd,
6000 rn,
6001 operand.ToExtendedRegister(),
6002 S,
6003 AddSubExtendedFixed | op);
6004 } else {
6005 DataProcShiftedRegister(rd, rn, operand, S, AddSubShiftedFixed | op);
6006 }
6007 } else {
6008 VIXL_ASSERT(operand.IsExtendedRegister());
6009 DataProcExtendedRegister(rd, rn, operand, S, AddSubExtendedFixed | op);
6010 }
6011 }
6012
6013
AddSubWithCarry(const Register & rd,const Register & rn,const Operand & operand,FlagsUpdate S,AddSubWithCarryOp op)6014 void Assembler::AddSubWithCarry(const Register& rd,
6015 const Register& rn,
6016 const Operand& operand,
6017 FlagsUpdate S,
6018 AddSubWithCarryOp op) {
6019 VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
6020 VIXL_ASSERT(rd.GetSizeInBits() == operand.GetRegister().GetSizeInBits());
6021 VIXL_ASSERT(operand.IsShiftedRegister() && (operand.GetShiftAmount() == 0));
6022 Emit(SF(rd) | op | Flags(S) | Rm(operand.GetRegister()) | Rn(rn) | Rd(rd));
6023 }
6024
6025
hlt(int code)6026 void Assembler::hlt(int code) {
6027 VIXL_ASSERT(IsUint16(code));
6028 Emit(HLT | ImmException(code));
6029 }
6030
6031
brk(int code)6032 void Assembler::brk(int code) {
6033 VIXL_ASSERT(IsUint16(code));
6034 Emit(BRK | ImmException(code));
6035 }
6036
6037
svc(int code)6038 void Assembler::svc(int code) { Emit(SVC | ImmException(code)); }
6039
udf(int code)6040 void Assembler::udf(int code) { Emit(UDF | ImmUdf(code)); }
6041
6042
6043 // TODO(all): The third parameter should be passed by reference but gcc 4.8.2
6044 // reports a bogus uninitialised warning then.
Logical(const Register & rd,const Register & rn,const Operand operand,LogicalOp op)6045 void Assembler::Logical(const Register& rd,
6046 const Register& rn,
6047 const Operand operand,
6048 LogicalOp op) {
6049 VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
6050 if (operand.IsImmediate()) {
6051 int64_t immediate = operand.GetImmediate();
6052 unsigned reg_size = rd.GetSizeInBits();
6053
6054 VIXL_ASSERT(immediate != 0);
6055 VIXL_ASSERT(immediate != -1);
6056 VIXL_ASSERT(rd.Is64Bits() || IsUint32(immediate));
6057
6058 // If the operation is NOT, invert the operation and immediate.
6059 if ((op & NOT) == NOT) {
6060 op = static_cast<LogicalOp>(op & ~NOT);
6061 immediate = rd.Is64Bits() ? ~immediate : (~immediate & kWRegMask);
6062 }
6063
6064 unsigned n, imm_s, imm_r;
6065 if (IsImmLogical(immediate, reg_size, &n, &imm_s, &imm_r)) {
6066 // Immediate can be encoded in the instruction.
6067 LogicalImmediate(rd, rn, n, imm_s, imm_r, op);
6068 } else {
6069 // This case is handled in the macro assembler.
6070 VIXL_UNREACHABLE();
6071 }
6072 } else {
6073 VIXL_ASSERT(operand.IsShiftedRegister());
6074 VIXL_ASSERT(operand.GetRegister().GetSizeInBits() == rd.GetSizeInBits());
6075 Instr dp_op = static_cast<Instr>(op | LogicalShiftedFixed);
6076 DataProcShiftedRegister(rd, rn, operand, LeaveFlags, dp_op);
6077 }
6078 }
6079
6080
LogicalImmediate(const Register & rd,const Register & rn,unsigned n,unsigned imm_s,unsigned imm_r,LogicalOp op)6081 void Assembler::LogicalImmediate(const Register& rd,
6082 const Register& rn,
6083 unsigned n,
6084 unsigned imm_s,
6085 unsigned imm_r,
6086 LogicalOp op) {
6087 unsigned reg_size = rd.GetSizeInBits();
6088 Instr dest_reg = (op == ANDS) ? Rd(rd) : RdSP(rd);
6089 Emit(SF(rd) | LogicalImmediateFixed | op | BitN(n, reg_size) |
6090 ImmSetBits(imm_s, reg_size) | ImmRotate(imm_r, reg_size) | dest_reg |
6091 Rn(rn));
6092 }
6093
6094
ConditionalCompare(const Register & rn,const Operand & operand,StatusFlags nzcv,Condition cond,ConditionalCompareOp op)6095 void Assembler::ConditionalCompare(const Register& rn,
6096 const Operand& operand,
6097 StatusFlags nzcv,
6098 Condition cond,
6099 ConditionalCompareOp op) {
6100 Instr ccmpop;
6101 if (operand.IsImmediate()) {
6102 int64_t immediate = operand.GetImmediate();
6103 VIXL_ASSERT(IsImmConditionalCompare(immediate));
6104 ccmpop = ConditionalCompareImmediateFixed | op |
6105 ImmCondCmp(static_cast<unsigned>(immediate));
6106 } else {
6107 VIXL_ASSERT(operand.IsShiftedRegister() && (operand.GetShiftAmount() == 0));
6108 ccmpop = ConditionalCompareRegisterFixed | op | Rm(operand.GetRegister());
6109 }
6110 Emit(SF(rn) | ccmpop | Cond(cond) | Rn(rn) | Nzcv(nzcv));
6111 }
6112
6113
DataProcessing1Source(const Register & rd,const Register & rn,DataProcessing1SourceOp op)6114 void Assembler::DataProcessing1Source(const Register& rd,
6115 const Register& rn,
6116 DataProcessing1SourceOp op) {
6117 VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
6118 Emit(SF(rn) | op | Rn(rn) | Rd(rd));
6119 }
6120
6121
FPDataProcessing1Source(const VRegister & vd,const VRegister & vn,FPDataProcessing1SourceOp op)6122 void Assembler::FPDataProcessing1Source(const VRegister& vd,
6123 const VRegister& vn,
6124 FPDataProcessing1SourceOp op) {
6125 VIXL_ASSERT(vd.Is1H() || vd.Is1S() || vd.Is1D());
6126 Emit(FPType(vn) | op | Rn(vn) | Rd(vd));
6127 }
6128
6129
FPDataProcessing3Source(const VRegister & vd,const VRegister & vn,const VRegister & vm,const VRegister & va,FPDataProcessing3SourceOp op)6130 void Assembler::FPDataProcessing3Source(const VRegister& vd,
6131 const VRegister& vn,
6132 const VRegister& vm,
6133 const VRegister& va,
6134 FPDataProcessing3SourceOp op) {
6135 VIXL_ASSERT(vd.Is1H() || vd.Is1S() || vd.Is1D());
6136 VIXL_ASSERT(AreSameSizeAndType(vd, vn, vm, va));
6137 Emit(FPType(vd) | op | Rm(vm) | Rn(vn) | Rd(vd) | Ra(va));
6138 }
6139
6140
NEONModifiedImmShiftLsl(const VRegister & vd,const int imm8,const int left_shift,NEONModifiedImmediateOp op)6141 void Assembler::NEONModifiedImmShiftLsl(const VRegister& vd,
6142 const int imm8,
6143 const int left_shift,
6144 NEONModifiedImmediateOp op) {
6145 VIXL_ASSERT(vd.Is8B() || vd.Is16B() || vd.Is4H() || vd.Is8H() || vd.Is2S() ||
6146 vd.Is4S());
6147 VIXL_ASSERT((left_shift == 0) || (left_shift == 8) || (left_shift == 16) ||
6148 (left_shift == 24));
6149 VIXL_ASSERT(IsUint8(imm8));
6150
6151 int cmode_1, cmode_2, cmode_3;
6152 if (vd.Is8B() || vd.Is16B()) {
6153 VIXL_ASSERT(op == NEONModifiedImmediate_MOVI);
6154 cmode_1 = 1;
6155 cmode_2 = 1;
6156 cmode_3 = 1;
6157 } else {
6158 cmode_1 = (left_shift >> 3) & 1;
6159 cmode_2 = left_shift >> 4;
6160 cmode_3 = 0;
6161 if (vd.Is4H() || vd.Is8H()) {
6162 VIXL_ASSERT((left_shift == 0) || (left_shift == 8));
6163 cmode_3 = 1;
6164 }
6165 }
6166 int cmode = (cmode_3 << 3) | (cmode_2 << 2) | (cmode_1 << 1);
6167
6168 int q = vd.IsQ() ? NEON_Q : 0;
6169
6170 Emit(q | op | ImmNEONabcdefgh(imm8) | NEONCmode(cmode) | Rd(vd));
6171 }
6172
6173
NEONModifiedImmShiftMsl(const VRegister & vd,const int imm8,const int shift_amount,NEONModifiedImmediateOp op)6174 void Assembler::NEONModifiedImmShiftMsl(const VRegister& vd,
6175 const int imm8,
6176 const int shift_amount,
6177 NEONModifiedImmediateOp op) {
6178 VIXL_ASSERT(vd.Is2S() || vd.Is4S());
6179 VIXL_ASSERT((shift_amount == 8) || (shift_amount == 16));
6180 VIXL_ASSERT(IsUint8(imm8));
6181
6182 int cmode_0 = (shift_amount >> 4) & 1;
6183 int cmode = 0xc | cmode_0;
6184
6185 int q = vd.IsQ() ? NEON_Q : 0;
6186
6187 Emit(q | op | ImmNEONabcdefgh(imm8) | NEONCmode(cmode) | Rd(vd));
6188 }
6189
6190
EmitShift(const Register & rd,const Register & rn,Shift shift,unsigned shift_amount)6191 void Assembler::EmitShift(const Register& rd,
6192 const Register& rn,
6193 Shift shift,
6194 unsigned shift_amount) {
6195 switch (shift) {
6196 case LSL:
6197 lsl(rd, rn, shift_amount);
6198 break;
6199 case LSR:
6200 lsr(rd, rn, shift_amount);
6201 break;
6202 case ASR:
6203 asr(rd, rn, shift_amount);
6204 break;
6205 case ROR:
6206 ror(rd, rn, shift_amount);
6207 break;
6208 default:
6209 VIXL_UNREACHABLE();
6210 }
6211 }
6212
6213
EmitExtendShift(const Register & rd,const Register & rn,Extend extend,unsigned left_shift)6214 void Assembler::EmitExtendShift(const Register& rd,
6215 const Register& rn,
6216 Extend extend,
6217 unsigned left_shift) {
6218 VIXL_ASSERT(rd.GetSizeInBits() >= rn.GetSizeInBits());
6219 unsigned reg_size = rd.GetSizeInBits();
6220 // Use the correct size of register.
6221 Register rn_ = Register(rn.GetCode(), rd.GetSizeInBits());
6222 // Bits extracted are high_bit:0.
6223 unsigned high_bit = (8 << (extend & 0x3)) - 1;
6224 // Number of bits left in the result that are not introduced by the shift.
6225 unsigned non_shift_bits = (reg_size - left_shift) & (reg_size - 1);
6226
6227 if ((non_shift_bits > high_bit) || (non_shift_bits == 0)) {
6228 switch (extend) {
6229 case UXTB:
6230 case UXTH:
6231 case UXTW:
6232 ubfm(rd, rn_, non_shift_bits, high_bit);
6233 break;
6234 case SXTB:
6235 case SXTH:
6236 case SXTW:
6237 sbfm(rd, rn_, non_shift_bits, high_bit);
6238 break;
6239 case UXTX:
6240 case SXTX: {
6241 VIXL_ASSERT(rn.GetSizeInBits() == kXRegSize);
6242 // Nothing to extend. Just shift.
6243 lsl(rd, rn_, left_shift);
6244 break;
6245 }
6246 default:
6247 VIXL_UNREACHABLE();
6248 }
6249 } else {
6250 // No need to extend as the extended bits would be shifted away.
6251 lsl(rd, rn_, left_shift);
6252 }
6253 }
6254
6255
DataProcShiftedRegister(const Register & rd,const Register & rn,const Operand & operand,FlagsUpdate S,Instr op)6256 void Assembler::DataProcShiftedRegister(const Register& rd,
6257 const Register& rn,
6258 const Operand& operand,
6259 FlagsUpdate S,
6260 Instr op) {
6261 VIXL_ASSERT(operand.IsShiftedRegister());
6262 VIXL_ASSERT(rn.Is64Bits() ||
6263 (rn.Is32Bits() && IsUint5(operand.GetShiftAmount())));
6264 Emit(SF(rd) | op | Flags(S) | ShiftDP(operand.GetShift()) |
6265 ImmDPShift(operand.GetShiftAmount()) | Rm(operand.GetRegister()) |
6266 Rn(rn) | Rd(rd));
6267 }
6268
6269
DataProcExtendedRegister(const Register & rd,const Register & rn,const Operand & operand,FlagsUpdate S,Instr op)6270 void Assembler::DataProcExtendedRegister(const Register& rd,
6271 const Register& rn,
6272 const Operand& operand,
6273 FlagsUpdate S,
6274 Instr op) {
6275 Instr dest_reg = (S == SetFlags) ? Rd(rd) : RdSP(rd);
6276 Emit(SF(rd) | op | Flags(S) | Rm(operand.GetRegister()) |
6277 ExtendMode(operand.GetExtend()) |
6278 ImmExtendShift(operand.GetShiftAmount()) | dest_reg | RnSP(rn));
6279 }
6280
6281
LoadStoreMemOperand(const MemOperand & addr,unsigned access_size_in_bytes_log2,LoadStoreScalingOption option)6282 Instr Assembler::LoadStoreMemOperand(const MemOperand& addr,
6283 unsigned access_size_in_bytes_log2,
6284 LoadStoreScalingOption option) {
6285 Instr base = RnSP(addr.GetBaseRegister());
6286 int64_t offset = addr.GetOffset();
6287
6288 if (addr.IsImmediateOffset()) {
6289 bool prefer_unscaled =
6290 (option == PreferUnscaledOffset) || (option == RequireUnscaledOffset);
6291 if (prefer_unscaled && IsImmLSUnscaled(offset)) {
6292 // Use the unscaled addressing mode.
6293 return base | LoadStoreUnscaledOffsetFixed | ImmLS(offset);
6294 }
6295
6296 if ((option != RequireUnscaledOffset) &&
6297 IsImmLSScaled(offset, access_size_in_bytes_log2)) {
6298 // We need `offset` to be positive for the shift to be well-defined.
6299 // IsImmLSScaled should check this.
6300 VIXL_ASSERT(offset >= 0);
6301 // Use the scaled addressing mode.
6302 return base | LoadStoreUnsignedOffsetFixed |
6303 ImmLSUnsigned(offset >> access_size_in_bytes_log2);
6304 }
6305
6306 if ((option != RequireScaledOffset) && IsImmLSUnscaled(offset)) {
6307 // Use the unscaled addressing mode.
6308 return base | LoadStoreUnscaledOffsetFixed | ImmLS(offset);
6309 }
6310 }
6311
6312 // All remaining addressing modes are register-offset, pre-indexed or
6313 // post-indexed modes.
6314 VIXL_ASSERT((option != RequireUnscaledOffset) &&
6315 (option != RequireScaledOffset));
6316
6317 if (addr.IsRegisterOffset()) {
6318 Extend ext = addr.GetExtend();
6319 Shift shift = addr.GetShift();
6320 unsigned shift_amount = addr.GetShiftAmount();
6321
6322 // LSL is encoded in the option field as UXTX.
6323 if (shift == LSL) {
6324 ext = UXTX;
6325 }
6326
6327 // Shifts are encoded in one bit, indicating a left shift by the memory
6328 // access size.
6329 VIXL_ASSERT((shift_amount == 0) || (shift_amount == access_size_in_bytes_log2));
6330 return base | LoadStoreRegisterOffsetFixed | Rm(addr.GetRegisterOffset()) |
6331 ExtendMode(ext) | ImmShiftLS((shift_amount > 0) ? 1 : 0);
6332 }
6333
6334 if (addr.IsImmediatePreIndex() && IsImmLSUnscaled(offset)) {
6335 return base | LoadStorePreIndexFixed | ImmLS(offset);
6336 }
6337
6338 if (addr.IsImmediatePostIndex() && IsImmLSUnscaled(offset)) {
6339 return base | LoadStorePostIndexFixed | ImmLS(offset);
6340 }
6341
6342 // If this point is reached, the MemOperand (addr) cannot be encoded.
6343 VIXL_UNREACHABLE();
6344 return 0;
6345 }
6346
6347
LoadStore(const CPURegister & rt,const MemOperand & addr,LoadStoreOp op,LoadStoreScalingOption option)6348 void Assembler::LoadStore(const CPURegister& rt,
6349 const MemOperand& addr,
6350 LoadStoreOp op,
6351 LoadStoreScalingOption option) {
6352 VIXL_ASSERT(CPUHas(rt));
6353 Emit(op | Rt(rt) | LoadStoreMemOperand(addr, CalcLSDataSize(op), option));
6354 }
6355
LoadStorePAC(const Register & xt,const MemOperand & addr,LoadStorePACOp op)6356 void Assembler::LoadStorePAC(const Register& xt,
6357 const MemOperand& addr,
6358 LoadStorePACOp op) {
6359 VIXL_ASSERT(xt.Is64Bits());
6360 VIXL_ASSERT(addr.IsImmediateOffset() || addr.IsImmediatePreIndex());
6361
6362 Instr pac_op = op;
6363 if (addr.IsImmediatePreIndex()) {
6364 pac_op |= LoadStorePACPreBit;
6365 }
6366
6367 Instr base = RnSP(addr.GetBaseRegister());
6368 int64_t offset = addr.GetOffset();
6369
6370 Emit(pac_op | Rt(xt) | base | ImmLSPAC(static_cast<int>(offset)));
6371 }
6372
6373
Prefetch(int op,const MemOperand & addr,LoadStoreScalingOption option)6374 void Assembler::Prefetch(int op,
6375 const MemOperand& addr,
6376 LoadStoreScalingOption option) {
6377 VIXL_ASSERT(addr.IsRegisterOffset() || addr.IsImmediateOffset());
6378
6379 Instr prfop = ImmPrefetchOperation(op);
6380 Emit(PRFM | prfop | LoadStoreMemOperand(addr, kXRegSizeInBytesLog2, option));
6381 }
6382
Prefetch(PrefetchOperation op,const MemOperand & addr,LoadStoreScalingOption option)6383 void Assembler::Prefetch(PrefetchOperation op,
6384 const MemOperand& addr,
6385 LoadStoreScalingOption option) {
6386 // Passing unnamed values in 'op' is undefined behaviour in C++.
6387 VIXL_ASSERT(IsNamedPrefetchOperation(op));
6388 Prefetch(static_cast<int>(op), addr, option);
6389 }
6390
6391
IsImmAddSub(int64_t immediate)6392 bool Assembler::IsImmAddSub(int64_t immediate) {
6393 return IsUint12(immediate) ||
6394 (IsUint12(immediate >> 12) && ((immediate & 0xfff) == 0));
6395 }
6396
6397
IsImmConditionalCompare(int64_t immediate)6398 bool Assembler::IsImmConditionalCompare(int64_t immediate) {
6399 return IsUint5(immediate);
6400 }
6401
6402
IsImmFP16(Float16 imm)6403 bool Assembler::IsImmFP16(Float16 imm) {
6404 // Valid values will have the form:
6405 // aBbb.cdef.gh00.000
6406 uint16_t bits = Float16ToRawbits(imm);
6407 // bits[6..0] are cleared.
6408 if ((bits & 0x3f) != 0) {
6409 return false;
6410 }
6411
6412 // bits[13..12] are all set or all cleared.
6413 uint16_t b_pattern = (bits >> 12) & 0x03;
6414 if (b_pattern != 0 && b_pattern != 0x03) {
6415 return false;
6416 }
6417
6418 // bit[15] and bit[14] are opposite.
6419 if (((bits ^ (bits << 1)) & 0x4000) == 0) {
6420 return false;
6421 }
6422
6423 return true;
6424 }
6425
6426
IsImmFP32(uint32_t bits)6427 bool Assembler::IsImmFP32(uint32_t bits) {
6428 // Valid values will have the form:
6429 // aBbb.bbbc.defg.h000.0000.0000.0000.0000
6430 // bits[19..0] are cleared.
6431 if ((bits & 0x7ffff) != 0) {
6432 return false;
6433 }
6434
6435 // bits[29..25] are all set or all cleared.
6436 uint32_t b_pattern = (bits >> 16) & 0x3e00;
6437 if (b_pattern != 0 && b_pattern != 0x3e00) {
6438 return false;
6439 }
6440
6441 // bit[30] and bit[29] are opposite.
6442 if (((bits ^ (bits << 1)) & 0x40000000) == 0) {
6443 return false;
6444 }
6445
6446 return true;
6447 }
6448
6449
IsImmFP64(uint64_t bits)6450 bool Assembler::IsImmFP64(uint64_t bits) {
6451 // Valid values will have the form:
6452 // aBbb.bbbb.bbcd.efgh.0000.0000.0000.0000
6453 // 0000.0000.0000.0000.0000.0000.0000.0000
6454 // bits[47..0] are cleared.
6455 if ((bits & 0x0000ffffffffffff) != 0) {
6456 return false;
6457 }
6458
6459 // bits[61..54] are all set or all cleared.
6460 uint32_t b_pattern = (bits >> 48) & 0x3fc0;
6461 if ((b_pattern != 0) && (b_pattern != 0x3fc0)) {
6462 return false;
6463 }
6464
6465 // bit[62] and bit[61] are opposite.
6466 if (((bits ^ (bits << 1)) & (UINT64_C(1) << 62)) == 0) {
6467 return false;
6468 }
6469
6470 return true;
6471 }
6472
6473
IsImmLSPair(int64_t offset,unsigned access_size_in_bytes_log2)6474 bool Assembler::IsImmLSPair(int64_t offset, unsigned access_size_in_bytes_log2) {
6475 const auto access_size_in_bytes = 1U << access_size_in_bytes_log2;
6476 VIXL_ASSERT(access_size_in_bytes_log2 <= kQRegSizeInBytesLog2);
6477 return IsMultiple(offset, access_size_in_bytes) &&
6478 IsInt7(offset / access_size_in_bytes);
6479 }
6480
6481
IsImmLSScaled(int64_t offset,unsigned access_size_in_bytes_log2)6482 bool Assembler::IsImmLSScaled(int64_t offset, unsigned access_size_in_bytes_log2) {
6483 const auto access_size_in_bytes = 1U << access_size_in_bytes_log2;
6484 VIXL_ASSERT(access_size_in_bytes_log2 <= kQRegSizeInBytesLog2);
6485 return IsMultiple(offset, access_size_in_bytes) &&
6486 IsUint12(offset / access_size_in_bytes);
6487 }
6488
6489
IsImmLSUnscaled(int64_t offset)6490 bool Assembler::IsImmLSUnscaled(int64_t offset) { return IsInt9(offset); }
6491
6492
6493 // The movn instruction can generate immediates containing an arbitrary 16-bit
6494 // value, with remaining bits set, eg. 0xffff1234, 0xffff1234ffffffff.
IsImmMovn(uint64_t imm,unsigned reg_size)6495 bool Assembler::IsImmMovn(uint64_t imm, unsigned reg_size) {
6496 return IsImmMovz(~imm, reg_size);
6497 }
6498
6499
6500 // The movz instruction can generate immediates containing an arbitrary 16-bit
6501 // value, with remaining bits clear, eg. 0x00001234, 0x0000123400000000.
IsImmMovz(uint64_t imm,unsigned reg_size)6502 bool Assembler::IsImmMovz(uint64_t imm, unsigned reg_size) {
6503 VIXL_ASSERT((reg_size == kXRegSize) || (reg_size == kWRegSize));
6504 return CountClearHalfWords(imm, reg_size) >= ((reg_size / 16) - 1);
6505 }
6506
6507
6508 // Test if a given value can be encoded in the immediate field of a logical
6509 // instruction.
6510 // If it can be encoded, the function returns true, and values pointed to by n,
6511 // imm_s and imm_r are updated with immediates encoded in the format required
6512 // by the corresponding fields in the logical instruction.
6513 // If it can not be encoded, the function returns false, and the values pointed
6514 // to by n, imm_s and imm_r are undefined.
IsImmLogical(uint64_t value,unsigned width,unsigned * n,unsigned * imm_s,unsigned * imm_r)6515 bool Assembler::IsImmLogical(uint64_t value,
6516 unsigned width,
6517 unsigned* n,
6518 unsigned* imm_s,
6519 unsigned* imm_r) {
6520 VIXL_ASSERT((width == kBRegSize) || (width == kHRegSize) ||
6521 (width == kSRegSize) || (width == kDRegSize));
6522
6523 bool negate = false;
6524
6525 // Logical immediates are encoded using parameters n, imm_s and imm_r using
6526 // the following table:
6527 //
6528 // N imms immr size S R
6529 // 1 ssssss rrrrrr 64 UInt(ssssss) UInt(rrrrrr)
6530 // 0 0sssss xrrrrr 32 UInt(sssss) UInt(rrrrr)
6531 // 0 10ssss xxrrrr 16 UInt(ssss) UInt(rrrr)
6532 // 0 110sss xxxrrr 8 UInt(sss) UInt(rrr)
6533 // 0 1110ss xxxxrr 4 UInt(ss) UInt(rr)
6534 // 0 11110s xxxxxr 2 UInt(s) UInt(r)
6535 // (s bits must not be all set)
6536 //
6537 // A pattern is constructed of size bits, where the least significant S+1 bits
6538 // are set. The pattern is rotated right by R, and repeated across a 32 or
6539 // 64-bit value, depending on destination register width.
6540 //
6541 // Put another way: the basic format of a logical immediate is a single
6542 // contiguous stretch of 1 bits, repeated across the whole word at intervals
6543 // given by a power of 2. To identify them quickly, we first locate the
6544 // lowest stretch of 1 bits, then the next 1 bit above that; that combination
6545 // is different for every logical immediate, so it gives us all the
6546 // information we need to identify the only logical immediate that our input
6547 // could be, and then we simply check if that's the value we actually have.
6548 //
6549 // (The rotation parameter does give the possibility of the stretch of 1 bits
6550 // going 'round the end' of the word. To deal with that, we observe that in
6551 // any situation where that happens the bitwise NOT of the value is also a
6552 // valid logical immediate. So we simply invert the input whenever its low bit
6553 // is set, and then we know that the rotated case can't arise.)
6554
6555 if (value & 1) {
6556 // If the low bit is 1, negate the value, and set a flag to remember that we
6557 // did (so that we can adjust the return values appropriately).
6558 negate = true;
6559 value = ~value;
6560 }
6561
6562 if (width <= kWRegSize) {
6563 // To handle 8/16/32-bit logical immediates, the very easiest thing is to repeat
6564 // the input value to fill a 64-bit word. The correct encoding of that as a
6565 // logical immediate will also be the correct encoding of the value.
6566
6567 // Avoid making the assumption that the most-significant 56/48/32 bits are zero by
6568 // shifting the value left and duplicating it.
6569 for (unsigned bits = width; bits <= kWRegSize; bits *= 2) {
6570 value <<= bits;
6571 uint64_t mask = (UINT64_C(1) << bits) - 1;
6572 value |= ((value >> bits) & mask);
6573 }
6574 }
6575
6576 // The basic analysis idea: imagine our input word looks like this.
6577 //
6578 // 0011111000111110001111100011111000111110001111100011111000111110
6579 // c b a
6580 // |<--d-->|
6581 //
6582 // We find the lowest set bit (as an actual power-of-2 value, not its index)
6583 // and call it a. Then we add a to our original number, which wipes out the
6584 // bottommost stretch of set bits and replaces it with a 1 carried into the
6585 // next zero bit. Then we look for the new lowest set bit, which is in
6586 // position b, and subtract it, so now our number is just like the original
6587 // but with the lowest stretch of set bits completely gone. Now we find the
6588 // lowest set bit again, which is position c in the diagram above. Then we'll
6589 // measure the distance d between bit positions a and c (using CLZ), and that
6590 // tells us that the only valid logical immediate that could possibly be equal
6591 // to this number is the one in which a stretch of bits running from a to just
6592 // below b is replicated every d bits.
6593 uint64_t a = LowestSetBit(value);
6594 uint64_t value_plus_a = value + a;
6595 uint64_t b = LowestSetBit(value_plus_a);
6596 uint64_t value_plus_a_minus_b = value_plus_a - b;
6597 uint64_t c = LowestSetBit(value_plus_a_minus_b);
6598
6599 int d, clz_a, out_n;
6600 uint64_t mask;
6601
6602 if (c != 0) {
6603 // The general case, in which there is more than one stretch of set bits.
6604 // Compute the repeat distance d, and set up a bitmask covering the basic
6605 // unit of repetition (i.e. a word with the bottom d bits set). Also, in all
6606 // of these cases the N bit of the output will be zero.
6607 clz_a = CountLeadingZeros(a, kXRegSize);
6608 int clz_c = CountLeadingZeros(c, kXRegSize);
6609 d = clz_a - clz_c;
6610 mask = ((UINT64_C(1) << d) - 1);
6611 out_n = 0;
6612 } else {
6613 // Handle degenerate cases.
6614 //
6615 // If any of those 'find lowest set bit' operations didn't find a set bit at
6616 // all, then the word will have been zero thereafter, so in particular the
6617 // last lowest_set_bit operation will have returned zero. So we can test for
6618 // all the special case conditions in one go by seeing if c is zero.
6619 if (a == 0) {
6620 // The input was zero (or all 1 bits, which will come to here too after we
6621 // inverted it at the start of the function), for which we just return
6622 // false.
6623 return false;
6624 } else {
6625 // Otherwise, if c was zero but a was not, then there's just one stretch
6626 // of set bits in our word, meaning that we have the trivial case of
6627 // d == 64 and only one 'repetition'. Set up all the same variables as in
6628 // the general case above, and set the N bit in the output.
6629 clz_a = CountLeadingZeros(a, kXRegSize);
6630 d = 64;
6631 mask = ~UINT64_C(0);
6632 out_n = 1;
6633 }
6634 }
6635
6636 // If the repeat period d is not a power of two, it can't be encoded.
6637 if (!IsPowerOf2(d)) {
6638 return false;
6639 }
6640
6641 if (((b - a) & ~mask) != 0) {
6642 // If the bit stretch (b - a) does not fit within the mask derived from the
6643 // repeat period, then fail.
6644 return false;
6645 }
6646
6647 // The only possible option is b - a repeated every d bits. Now we're going to
6648 // actually construct the valid logical immediate derived from that
6649 // specification, and see if it equals our original input.
6650 //
6651 // To repeat a value every d bits, we multiply it by a number of the form
6652 // (1 + 2^d + 2^(2d) + ...), i.e. 0x0001000100010001 or similar. These can
6653 // be derived using a table lookup on CLZ(d).
6654 static const uint64_t multipliers[] = {
6655 0x0000000000000001UL,
6656 0x0000000100000001UL,
6657 0x0001000100010001UL,
6658 0x0101010101010101UL,
6659 0x1111111111111111UL,
6660 0x5555555555555555UL,
6661 };
6662 uint64_t multiplier = multipliers[CountLeadingZeros(d, kXRegSize) - 57];
6663 uint64_t candidate = (b - a) * multiplier;
6664
6665 if (value != candidate) {
6666 // The candidate pattern doesn't match our input value, so fail.
6667 return false;
6668 }
6669
6670 // We have a match! This is a valid logical immediate, so now we have to
6671 // construct the bits and pieces of the instruction encoding that generates
6672 // it.
6673
6674 // Count the set bits in our basic stretch. The special case of clz(0) == -1
6675 // makes the answer come out right for stretches that reach the very top of
6676 // the word (e.g. numbers like 0xffffc00000000000).
6677 int clz_b = (b == 0) ? -1 : CountLeadingZeros(b, kXRegSize);
6678 int s = clz_a - clz_b;
6679
6680 // Decide how many bits to rotate right by, to put the low bit of that basic
6681 // stretch in position a.
6682 int r;
6683 if (negate) {
6684 // If we inverted the input right at the start of this function, here's
6685 // where we compensate: the number of set bits becomes the number of clear
6686 // bits, and the rotation count is based on position b rather than position
6687 // a (since b is the location of the 'lowest' 1 bit after inversion).
6688 s = d - s;
6689 r = (clz_b + 1) & (d - 1);
6690 } else {
6691 r = (clz_a + 1) & (d - 1);
6692 }
6693
6694 // Now we're done, except for having to encode the S output in such a way that
6695 // it gives both the number of set bits and the length of the repeated
6696 // segment. The s field is encoded like this:
6697 //
6698 // imms size S
6699 // ssssss 64 UInt(ssssss)
6700 // 0sssss 32 UInt(sssss)
6701 // 10ssss 16 UInt(ssss)
6702 // 110sss 8 UInt(sss)
6703 // 1110ss 4 UInt(ss)
6704 // 11110s 2 UInt(s)
6705 //
6706 // So we 'or' (2 * -d) with our computed s to form imms.
6707 if ((n != NULL) || (imm_s != NULL) || (imm_r != NULL)) {
6708 *n = out_n;
6709 *imm_s = ((2 * -d) | (s - 1)) & 0x3f;
6710 *imm_r = r;
6711 }
6712
6713 return true;
6714 }
6715
6716
LoadOpFor(const CPURegister & rt)6717 LoadStoreOp Assembler::LoadOpFor(const CPURegister& rt) {
6718 VIXL_ASSERT(rt.IsValid());
6719 if (rt.IsRegister()) {
6720 return rt.Is64Bits() ? LDR_x : LDR_w;
6721 } else {
6722 VIXL_ASSERT(rt.IsVRegister());
6723 switch (rt.GetSizeInBits()) {
6724 case kBRegSize:
6725 return LDR_b;
6726 case kHRegSize:
6727 return LDR_h;
6728 case kSRegSize:
6729 return LDR_s;
6730 case kDRegSize:
6731 return LDR_d;
6732 default:
6733 VIXL_ASSERT(rt.IsQ());
6734 return LDR_q;
6735 }
6736 }
6737 }
6738
6739
StoreOpFor(const CPURegister & rt)6740 LoadStoreOp Assembler::StoreOpFor(const CPURegister& rt) {
6741 VIXL_ASSERT(rt.IsValid());
6742 if (rt.IsRegister()) {
6743 return rt.Is64Bits() ? STR_x : STR_w;
6744 } else {
6745 VIXL_ASSERT(rt.IsVRegister());
6746 switch (rt.GetSizeInBits()) {
6747 case kBRegSize:
6748 return STR_b;
6749 case kHRegSize:
6750 return STR_h;
6751 case kSRegSize:
6752 return STR_s;
6753 case kDRegSize:
6754 return STR_d;
6755 default:
6756 VIXL_ASSERT(rt.IsQ());
6757 return STR_q;
6758 }
6759 }
6760 }
6761
6762
StorePairOpFor(const CPURegister & rt,const CPURegister & rt2)6763 LoadStorePairOp Assembler::StorePairOpFor(const CPURegister& rt,
6764 const CPURegister& rt2) {
6765 VIXL_ASSERT(AreSameSizeAndType(rt, rt2));
6766 USE(rt2);
6767 if (rt.IsRegister()) {
6768 return rt.Is64Bits() ? STP_x : STP_w;
6769 } else {
6770 VIXL_ASSERT(rt.IsVRegister());
6771 switch (rt.GetSizeInBytes()) {
6772 case kSRegSizeInBytes:
6773 return STP_s;
6774 case kDRegSizeInBytes:
6775 return STP_d;
6776 default:
6777 VIXL_ASSERT(rt.IsQ());
6778 return STP_q;
6779 }
6780 }
6781 }
6782
6783
LoadPairOpFor(const CPURegister & rt,const CPURegister & rt2)6784 LoadStorePairOp Assembler::LoadPairOpFor(const CPURegister& rt,
6785 const CPURegister& rt2) {
6786 VIXL_ASSERT((STP_w | LoadStorePairLBit) == LDP_w);
6787 return static_cast<LoadStorePairOp>(StorePairOpFor(rt, rt2) |
6788 LoadStorePairLBit);
6789 }
6790
6791
StorePairNonTemporalOpFor(const CPURegister & rt,const CPURegister & rt2)6792 LoadStorePairNonTemporalOp Assembler::StorePairNonTemporalOpFor(
6793 const CPURegister& rt, const CPURegister& rt2) {
6794 VIXL_ASSERT(AreSameSizeAndType(rt, rt2));
6795 USE(rt2);
6796 if (rt.IsRegister()) {
6797 return rt.Is64Bits() ? STNP_x : STNP_w;
6798 } else {
6799 VIXL_ASSERT(rt.IsVRegister());
6800 switch (rt.GetSizeInBytes()) {
6801 case kSRegSizeInBytes:
6802 return STNP_s;
6803 case kDRegSizeInBytes:
6804 return STNP_d;
6805 default:
6806 VIXL_ASSERT(rt.IsQ());
6807 return STNP_q;
6808 }
6809 }
6810 }
6811
6812
LoadPairNonTemporalOpFor(const CPURegister & rt,const CPURegister & rt2)6813 LoadStorePairNonTemporalOp Assembler::LoadPairNonTemporalOpFor(
6814 const CPURegister& rt, const CPURegister& rt2) {
6815 VIXL_ASSERT((STNP_w | LoadStorePairNonTemporalLBit) == LDNP_w);
6816 return static_cast<LoadStorePairNonTemporalOp>(
6817 StorePairNonTemporalOpFor(rt, rt2) | LoadStorePairNonTemporalLBit);
6818 }
6819
6820
LoadLiteralOpFor(const CPURegister & rt)6821 LoadLiteralOp Assembler::LoadLiteralOpFor(const CPURegister& rt) {
6822 if (rt.IsRegister()) {
6823 return rt.IsX() ? LDR_x_lit : LDR_w_lit;
6824 } else {
6825 VIXL_ASSERT(rt.IsVRegister());
6826 switch (rt.GetSizeInBytes()) {
6827 case kSRegSizeInBytes:
6828 return LDR_s_lit;
6829 case kDRegSizeInBytes:
6830 return LDR_d_lit;
6831 default:
6832 VIXL_ASSERT(rt.IsQ());
6833 return LDR_q_lit;
6834 }
6835 }
6836 }
6837
6838
CPUHas(const CPURegister & rt) const6839 bool Assembler::CPUHas(const CPURegister& rt) const {
6840 // Core registers are available without any particular CPU features.
6841 if (rt.IsRegister()) return true;
6842 VIXL_ASSERT(rt.IsVRegister());
6843 // The architecture does not allow FP and NEON to be implemented separately,
6844 // but we can crudely categorise them based on register size, since FP only
6845 // uses D, S and (occasionally) H registers.
6846 if (rt.IsH() || rt.IsS() || rt.IsD()) {
6847 return CPUHas(CPUFeatures::kFP) || CPUHas(CPUFeatures::kNEON);
6848 }
6849 VIXL_ASSERT(rt.IsB() || rt.IsQ());
6850 return CPUHas(CPUFeatures::kNEON);
6851 }
6852
6853
CPUHas(const CPURegister & rt,const CPURegister & rt2) const6854 bool Assembler::CPUHas(const CPURegister& rt, const CPURegister& rt2) const {
6855 // This is currently only used for loads and stores, where rt and rt2 must
6856 // have the same size and type. We could extend this to cover other cases if
6857 // necessary, but for now we can avoid checking both registers.
6858 VIXL_ASSERT(AreSameSizeAndType(rt, rt2));
6859 USE(rt2);
6860 return CPUHas(rt);
6861 }
6862
6863
CPUHas(SystemRegister sysreg) const6864 bool Assembler::CPUHas(SystemRegister sysreg) const {
6865 switch (sysreg) {
6866 case RNDR:
6867 case RNDRRS:
6868 return CPUHas(CPUFeatures::kRNG);
6869 case FPCR:
6870 case NZCV:
6871 break;
6872 }
6873 return true;
6874 }
6875
6876
6877 } // namespace aarch64
6878 } // namespace vixl
6879