1 // Copyright 2015, VIXL authors
2 // All rights reserved.
3 //
4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions are met:
6 //
7 // * Redistributions of source code must retain the above copyright notice,
8 // this list of conditions and the following disclaimer.
9 // * Redistributions in binary form must reproduce the above copyright notice,
10 // this list of conditions and the following disclaimer in the documentation
11 // and/or other materials provided with the distribution.
12 // * Neither the name of ARM Limited nor the names of its contributors may be
13 // used to endorse or promote products derived from this software without
14 // specific prior written permission.
15 //
16 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
17 // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
20 // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
22 // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
23 // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24 // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26
27
28 #include "assembler-aarch64.h"
29
30 #include <cmath>
31
32 #include "macro-assembler-aarch64.h"
33
34 namespace vixl {
35 namespace aarch64 {
36
RawLiteral(size_t size,LiteralPool * literal_pool,DeletionPolicy deletion_policy)37 RawLiteral::RawLiteral(size_t size,
38 LiteralPool* literal_pool,
39 DeletionPolicy deletion_policy)
40 : size_(size),
41 offset_(0),
42 low64_(0),
43 high64_(0),
44 literal_pool_(literal_pool),
45 deletion_policy_(deletion_policy) {
46 VIXL_ASSERT((deletion_policy == kManuallyDeleted) || (literal_pool_ != NULL));
47 if (deletion_policy == kDeletedOnPoolDestruction) {
48 literal_pool_->DeleteOnDestruction(this);
49 }
50 }
51
52
Reset()53 void Assembler::Reset() { GetBuffer()->Reset(); }
54
55
bind(Label * label)56 void Assembler::bind(Label* label) {
57 BindToOffset(label, GetBuffer()->GetCursorOffset());
58 }
59
60
BindToOffset(Label * label,ptrdiff_t offset)61 void Assembler::BindToOffset(Label* label, ptrdiff_t offset) {
62 VIXL_ASSERT((offset >= 0) && (offset <= GetBuffer()->GetCursorOffset()));
63 VIXL_ASSERT(offset % kInstructionSize == 0);
64
65 label->Bind(offset);
66
67 for (Label::LabelLinksIterator it(label); !it.Done(); it.Advance()) {
68 Instruction* link =
69 GetBuffer()->GetOffsetAddress<Instruction*>(*it.Current());
70 link->SetImmPCOffsetTarget(GetLabelAddress<Instruction*>(label));
71 }
72 label->ClearAllLinks();
73 }
74
75
76 // A common implementation for the LinkAndGet<Type>OffsetTo helpers.
77 //
78 // The offset is calculated by aligning the PC and label addresses down to a
79 // multiple of 1 << element_shift, then calculating the (scaled) offset between
80 // them. This matches the semantics of adrp, for example.
81 template <int element_shift>
LinkAndGetOffsetTo(Label * label)82 ptrdiff_t Assembler::LinkAndGetOffsetTo(Label* label) {
83 VIXL_STATIC_ASSERT(element_shift < (sizeof(ptrdiff_t) * 8));
84
85 if (label->IsBound()) {
86 uintptr_t pc_offset = GetCursorAddress<uintptr_t>() >> element_shift;
87 uintptr_t label_offset = GetLabelAddress<uintptr_t>(label) >> element_shift;
88 return label_offset - pc_offset;
89 } else {
90 label->AddLink(GetBuffer()->GetCursorOffset());
91 return 0;
92 }
93 }
94
95
LinkAndGetByteOffsetTo(Label * label)96 ptrdiff_t Assembler::LinkAndGetByteOffsetTo(Label* label) {
97 return LinkAndGetOffsetTo<0>(label);
98 }
99
100
LinkAndGetInstructionOffsetTo(Label * label)101 ptrdiff_t Assembler::LinkAndGetInstructionOffsetTo(Label* label) {
102 return LinkAndGetOffsetTo<kInstructionSizeLog2>(label);
103 }
104
105
LinkAndGetPageOffsetTo(Label * label)106 ptrdiff_t Assembler::LinkAndGetPageOffsetTo(Label* label) {
107 return LinkAndGetOffsetTo<kPageSizeLog2>(label);
108 }
109
110
place(RawLiteral * literal)111 void Assembler::place(RawLiteral* literal) {
112 VIXL_ASSERT(!literal->IsPlaced());
113
114 // Patch instructions using this literal.
115 if (literal->IsUsed()) {
116 Instruction* target = GetCursorAddress<Instruction*>();
117 ptrdiff_t offset = literal->GetLastUse();
118 bool done;
119 do {
120 Instruction* ldr = GetBuffer()->GetOffsetAddress<Instruction*>(offset);
121 VIXL_ASSERT(ldr->IsLoadLiteral());
122
123 ptrdiff_t imm19 = ldr->GetImmLLiteral();
124 VIXL_ASSERT(imm19 <= 0);
125 done = (imm19 == 0);
126 offset += imm19 * kLiteralEntrySize;
127
128 ldr->SetImmLLiteral(target);
129 } while (!done);
130 }
131
132 // "bind" the literal.
133 literal->SetOffset(GetCursorOffset());
134 // Copy the data into the pool.
135 switch (literal->GetSize()) {
136 case kSRegSizeInBytes:
137 dc32(literal->GetRawValue32());
138 break;
139 case kDRegSizeInBytes:
140 dc64(literal->GetRawValue64());
141 break;
142 default:
143 VIXL_ASSERT(literal->GetSize() == kQRegSizeInBytes);
144 dc64(literal->GetRawValue128Low64());
145 dc64(literal->GetRawValue128High64());
146 }
147
148 literal->literal_pool_ = NULL;
149 }
150
151
LinkAndGetWordOffsetTo(RawLiteral * literal)152 ptrdiff_t Assembler::LinkAndGetWordOffsetTo(RawLiteral* literal) {
153 VIXL_ASSERT(IsWordAligned(GetCursorOffset()));
154
155 bool register_first_use =
156 (literal->GetLiteralPool() != NULL) && !literal->IsUsed();
157
158 if (literal->IsPlaced()) {
159 // The literal is "behind", the offset will be negative.
160 VIXL_ASSERT((literal->GetOffset() - GetCursorOffset()) <= 0);
161 return (literal->GetOffset() - GetCursorOffset()) >> kLiteralEntrySizeLog2;
162 }
163
164 ptrdiff_t offset = 0;
165 // Link all uses together.
166 if (literal->IsUsed()) {
167 offset =
168 (literal->GetLastUse() - GetCursorOffset()) >> kLiteralEntrySizeLog2;
169 }
170 literal->SetLastUse(GetCursorOffset());
171
172 if (register_first_use) {
173 literal->GetLiteralPool()->AddEntry(literal);
174 }
175
176 return offset;
177 }
178
179
180 // Code generation.
br(const Register & xn)181 void Assembler::br(const Register& xn) {
182 VIXL_ASSERT(xn.Is64Bits());
183 Emit(BR | Rn(xn));
184 }
185
186
blr(const Register & xn)187 void Assembler::blr(const Register& xn) {
188 VIXL_ASSERT(xn.Is64Bits());
189 Emit(BLR | Rn(xn));
190 }
191
192
ret(const Register & xn)193 void Assembler::ret(const Register& xn) {
194 VIXL_ASSERT(xn.Is64Bits());
195 Emit(RET | Rn(xn));
196 }
197
198
braaz(const Register & xn)199 void Assembler::braaz(const Register& xn) {
200 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
201 VIXL_ASSERT(xn.Is64Bits());
202 Emit(BRAAZ | Rn(xn) | Rd_mask);
203 }
204
brabz(const Register & xn)205 void Assembler::brabz(const Register& xn) {
206 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
207 VIXL_ASSERT(xn.Is64Bits());
208 Emit(BRABZ | Rn(xn) | Rd_mask);
209 }
210
blraaz(const Register & xn)211 void Assembler::blraaz(const Register& xn) {
212 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
213 VIXL_ASSERT(xn.Is64Bits());
214 Emit(BLRAAZ | Rn(xn) | Rd_mask);
215 }
216
blrabz(const Register & xn)217 void Assembler::blrabz(const Register& xn) {
218 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
219 VIXL_ASSERT(xn.Is64Bits());
220 Emit(BLRABZ | Rn(xn) | Rd_mask);
221 }
222
retaa()223 void Assembler::retaa() {
224 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
225 Emit(RETAA | Rn_mask | Rd_mask);
226 }
227
retab()228 void Assembler::retab() {
229 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
230 Emit(RETAB | Rn_mask | Rd_mask);
231 }
232
233 // The Arm ARM names the register Xm but encodes it in the Xd bitfield.
braa(const Register & xn,const Register & xm)234 void Assembler::braa(const Register& xn, const Register& xm) {
235 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
236 VIXL_ASSERT(xn.Is64Bits() && xm.Is64Bits());
237 Emit(BRAA | Rn(xn) | RdSP(xm));
238 }
239
brab(const Register & xn,const Register & xm)240 void Assembler::brab(const Register& xn, const Register& xm) {
241 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
242 VIXL_ASSERT(xn.Is64Bits() && xm.Is64Bits());
243 Emit(BRAB | Rn(xn) | RdSP(xm));
244 }
245
blraa(const Register & xn,const Register & xm)246 void Assembler::blraa(const Register& xn, const Register& xm) {
247 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
248 VIXL_ASSERT(xn.Is64Bits() && xm.Is64Bits());
249 Emit(BLRAA | Rn(xn) | RdSP(xm));
250 }
251
blrab(const Register & xn,const Register & xm)252 void Assembler::blrab(const Register& xn, const Register& xm) {
253 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
254 VIXL_ASSERT(xn.Is64Bits() && xm.Is64Bits());
255 Emit(BLRAB | Rn(xn) | RdSP(xm));
256 }
257
258
b(int64_t imm26)259 void Assembler::b(int64_t imm26) { Emit(B | ImmUncondBranch(imm26)); }
260
261
b(int64_t imm19,Condition cond)262 void Assembler::b(int64_t imm19, Condition cond) {
263 Emit(B_cond | ImmCondBranch(imm19) | cond);
264 }
265
266
b(Label * label)267 void Assembler::b(Label* label) {
268 int64_t offset = LinkAndGetInstructionOffsetTo(label);
269 VIXL_ASSERT(Instruction::IsValidImmPCOffset(UncondBranchType, offset));
270 b(static_cast<int>(offset));
271 }
272
273
b(Label * label,Condition cond)274 void Assembler::b(Label* label, Condition cond) {
275 int64_t offset = LinkAndGetInstructionOffsetTo(label);
276 VIXL_ASSERT(Instruction::IsValidImmPCOffset(CondBranchType, offset));
277 b(static_cast<int>(offset), cond);
278 }
279
280
bl(int64_t imm26)281 void Assembler::bl(int64_t imm26) { Emit(BL | ImmUncondBranch(imm26)); }
282
283
bl(Label * label)284 void Assembler::bl(Label* label) {
285 int64_t offset = LinkAndGetInstructionOffsetTo(label);
286 VIXL_ASSERT(Instruction::IsValidImmPCOffset(UncondBranchType, offset));
287 bl(static_cast<int>(offset));
288 }
289
290
cbz(const Register & rt,int64_t imm19)291 void Assembler::cbz(const Register& rt, int64_t imm19) {
292 Emit(SF(rt) | CBZ | ImmCmpBranch(imm19) | Rt(rt));
293 }
294
295
cbz(const Register & rt,Label * label)296 void Assembler::cbz(const Register& rt, Label* label) {
297 int64_t offset = LinkAndGetInstructionOffsetTo(label);
298 VIXL_ASSERT(Instruction::IsValidImmPCOffset(CompareBranchType, offset));
299 cbz(rt, static_cast<int>(offset));
300 }
301
302
cbnz(const Register & rt,int64_t imm19)303 void Assembler::cbnz(const Register& rt, int64_t imm19) {
304 Emit(SF(rt) | CBNZ | ImmCmpBranch(imm19) | Rt(rt));
305 }
306
307
cbnz(const Register & rt,Label * label)308 void Assembler::cbnz(const Register& rt, Label* label) {
309 int64_t offset = LinkAndGetInstructionOffsetTo(label);
310 VIXL_ASSERT(Instruction::IsValidImmPCOffset(CompareBranchType, offset));
311 cbnz(rt, static_cast<int>(offset));
312 }
313
314
NEONTable(const VRegister & vd,const VRegister & vn,const VRegister & vm,NEONTableOp op)315 void Assembler::NEONTable(const VRegister& vd,
316 const VRegister& vn,
317 const VRegister& vm,
318 NEONTableOp op) {
319 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
320 VIXL_ASSERT(vd.Is16B() || vd.Is8B());
321 VIXL_ASSERT(vn.Is16B());
322 VIXL_ASSERT(AreSameFormat(vd, vm));
323 Emit(op | (vd.IsQ() ? NEON_Q : 0) | Rm(vm) | Rn(vn) | Rd(vd));
324 }
325
326
tbl(const VRegister & vd,const VRegister & vn,const VRegister & vm)327 void Assembler::tbl(const VRegister& vd,
328 const VRegister& vn,
329 const VRegister& vm) {
330 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
331 NEONTable(vd, vn, vm, NEON_TBL_1v);
332 }
333
334
tbl(const VRegister & vd,const VRegister & vn,const VRegister & vn2,const VRegister & vm)335 void Assembler::tbl(const VRegister& vd,
336 const VRegister& vn,
337 const VRegister& vn2,
338 const VRegister& vm) {
339 USE(vn2);
340 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
341 VIXL_ASSERT(AreSameFormat(vn, vn2));
342 VIXL_ASSERT(AreConsecutive(vn, vn2));
343 NEONTable(vd, vn, vm, NEON_TBL_2v);
344 }
345
346
tbl(const VRegister & vd,const VRegister & vn,const VRegister & vn2,const VRegister & vn3,const VRegister & vm)347 void Assembler::tbl(const VRegister& vd,
348 const VRegister& vn,
349 const VRegister& vn2,
350 const VRegister& vn3,
351 const VRegister& vm) {
352 USE(vn2, vn3);
353 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
354 VIXL_ASSERT(AreSameFormat(vn, vn2, vn3));
355 VIXL_ASSERT(AreConsecutive(vn, vn2, vn3));
356 NEONTable(vd, vn, vm, NEON_TBL_3v);
357 }
358
359
tbl(const VRegister & vd,const VRegister & vn,const VRegister & vn2,const VRegister & vn3,const VRegister & vn4,const VRegister & vm)360 void Assembler::tbl(const VRegister& vd,
361 const VRegister& vn,
362 const VRegister& vn2,
363 const VRegister& vn3,
364 const VRegister& vn4,
365 const VRegister& vm) {
366 USE(vn2, vn3, vn4);
367 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
368 VIXL_ASSERT(AreSameFormat(vn, vn2, vn3, vn4));
369 VIXL_ASSERT(AreConsecutive(vn, vn2, vn3, vn4));
370 NEONTable(vd, vn, vm, NEON_TBL_4v);
371 }
372
373
tbx(const VRegister & vd,const VRegister & vn,const VRegister & vm)374 void Assembler::tbx(const VRegister& vd,
375 const VRegister& vn,
376 const VRegister& vm) {
377 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
378 NEONTable(vd, vn, vm, NEON_TBX_1v);
379 }
380
381
tbx(const VRegister & vd,const VRegister & vn,const VRegister & vn2,const VRegister & vm)382 void Assembler::tbx(const VRegister& vd,
383 const VRegister& vn,
384 const VRegister& vn2,
385 const VRegister& vm) {
386 USE(vn2);
387 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
388 VIXL_ASSERT(AreSameFormat(vn, vn2));
389 VIXL_ASSERT(AreConsecutive(vn, vn2));
390 NEONTable(vd, vn, vm, NEON_TBX_2v);
391 }
392
393
tbx(const VRegister & vd,const VRegister & vn,const VRegister & vn2,const VRegister & vn3,const VRegister & vm)394 void Assembler::tbx(const VRegister& vd,
395 const VRegister& vn,
396 const VRegister& vn2,
397 const VRegister& vn3,
398 const VRegister& vm) {
399 USE(vn2, vn3);
400 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
401 VIXL_ASSERT(AreSameFormat(vn, vn2, vn3));
402 VIXL_ASSERT(AreConsecutive(vn, vn2, vn3));
403 NEONTable(vd, vn, vm, NEON_TBX_3v);
404 }
405
406
tbx(const VRegister & vd,const VRegister & vn,const VRegister & vn2,const VRegister & vn3,const VRegister & vn4,const VRegister & vm)407 void Assembler::tbx(const VRegister& vd,
408 const VRegister& vn,
409 const VRegister& vn2,
410 const VRegister& vn3,
411 const VRegister& vn4,
412 const VRegister& vm) {
413 USE(vn2, vn3, vn4);
414 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
415 VIXL_ASSERT(AreSameFormat(vn, vn2, vn3, vn4));
416 VIXL_ASSERT(AreConsecutive(vn, vn2, vn3, vn4));
417 NEONTable(vd, vn, vm, NEON_TBX_4v);
418 }
419
420
tbz(const Register & rt,unsigned bit_pos,int64_t imm14)421 void Assembler::tbz(const Register& rt, unsigned bit_pos, int64_t imm14) {
422 VIXL_ASSERT(rt.Is64Bits() || (rt.Is32Bits() && (bit_pos < kWRegSize)));
423 Emit(TBZ | ImmTestBranchBit(bit_pos) | ImmTestBranch(imm14) | Rt(rt));
424 }
425
426
tbz(const Register & rt,unsigned bit_pos,Label * label)427 void Assembler::tbz(const Register& rt, unsigned bit_pos, Label* label) {
428 ptrdiff_t offset = LinkAndGetInstructionOffsetTo(label);
429 VIXL_ASSERT(Instruction::IsValidImmPCOffset(TestBranchType, offset));
430 tbz(rt, bit_pos, static_cast<int>(offset));
431 }
432
433
tbnz(const Register & rt,unsigned bit_pos,int64_t imm14)434 void Assembler::tbnz(const Register& rt, unsigned bit_pos, int64_t imm14) {
435 VIXL_ASSERT(rt.Is64Bits() || (rt.Is32Bits() && (bit_pos < kWRegSize)));
436 Emit(TBNZ | ImmTestBranchBit(bit_pos) | ImmTestBranch(imm14) | Rt(rt));
437 }
438
439
tbnz(const Register & rt,unsigned bit_pos,Label * label)440 void Assembler::tbnz(const Register& rt, unsigned bit_pos, Label* label) {
441 ptrdiff_t offset = LinkAndGetInstructionOffsetTo(label);
442 VIXL_ASSERT(Instruction::IsValidImmPCOffset(TestBranchType, offset));
443 tbnz(rt, bit_pos, static_cast<int>(offset));
444 }
445
446
adr(const Register & xd,int64_t imm21)447 void Assembler::adr(const Register& xd, int64_t imm21) {
448 VIXL_ASSERT(xd.Is64Bits());
449 Emit(ADR | ImmPCRelAddress(imm21) | Rd(xd));
450 }
451
452
adr(const Register & xd,Label * label)453 void Assembler::adr(const Register& xd, Label* label) {
454 adr(xd, static_cast<int>(LinkAndGetByteOffsetTo(label)));
455 }
456
457
adrp(const Register & xd,int64_t imm21)458 void Assembler::adrp(const Register& xd, int64_t imm21) {
459 VIXL_ASSERT(xd.Is64Bits());
460 Emit(ADRP | ImmPCRelAddress(imm21) | Rd(xd));
461 }
462
463
adrp(const Register & xd,Label * label)464 void Assembler::adrp(const Register& xd, Label* label) {
465 VIXL_ASSERT(AllowPageOffsetDependentCode());
466 adrp(xd, static_cast<int>(LinkAndGetPageOffsetTo(label)));
467 }
468
469
add(const Register & rd,const Register & rn,const Operand & operand)470 void Assembler::add(const Register& rd,
471 const Register& rn,
472 const Operand& operand) {
473 AddSub(rd, rn, operand, LeaveFlags, ADD);
474 }
475
476
adds(const Register & rd,const Register & rn,const Operand & operand)477 void Assembler::adds(const Register& rd,
478 const Register& rn,
479 const Operand& operand) {
480 AddSub(rd, rn, operand, SetFlags, ADD);
481 }
482
483
cmn(const Register & rn,const Operand & operand)484 void Assembler::cmn(const Register& rn, const Operand& operand) {
485 Register zr = AppropriateZeroRegFor(rn);
486 adds(zr, rn, operand);
487 }
488
489
sub(const Register & rd,const Register & rn,const Operand & operand)490 void Assembler::sub(const Register& rd,
491 const Register& rn,
492 const Operand& operand) {
493 AddSub(rd, rn, operand, LeaveFlags, SUB);
494 }
495
496
subs(const Register & rd,const Register & rn,const Operand & operand)497 void Assembler::subs(const Register& rd,
498 const Register& rn,
499 const Operand& operand) {
500 AddSub(rd, rn, operand, SetFlags, SUB);
501 }
502
503
cmp(const Register & rn,const Operand & operand)504 void Assembler::cmp(const Register& rn, const Operand& operand) {
505 Register zr = AppropriateZeroRegFor(rn);
506 subs(zr, rn, operand);
507 }
508
509
neg(const Register & rd,const Operand & operand)510 void Assembler::neg(const Register& rd, const Operand& operand) {
511 Register zr = AppropriateZeroRegFor(rd);
512 sub(rd, zr, operand);
513 }
514
515
negs(const Register & rd,const Operand & operand)516 void Assembler::negs(const Register& rd, const Operand& operand) {
517 Register zr = AppropriateZeroRegFor(rd);
518 subs(rd, zr, operand);
519 }
520
521
adc(const Register & rd,const Register & rn,const Operand & operand)522 void Assembler::adc(const Register& rd,
523 const Register& rn,
524 const Operand& operand) {
525 AddSubWithCarry(rd, rn, operand, LeaveFlags, ADC);
526 }
527
528
adcs(const Register & rd,const Register & rn,const Operand & operand)529 void Assembler::adcs(const Register& rd,
530 const Register& rn,
531 const Operand& operand) {
532 AddSubWithCarry(rd, rn, operand, SetFlags, ADC);
533 }
534
535
sbc(const Register & rd,const Register & rn,const Operand & operand)536 void Assembler::sbc(const Register& rd,
537 const Register& rn,
538 const Operand& operand) {
539 AddSubWithCarry(rd, rn, operand, LeaveFlags, SBC);
540 }
541
542
sbcs(const Register & rd,const Register & rn,const Operand & operand)543 void Assembler::sbcs(const Register& rd,
544 const Register& rn,
545 const Operand& operand) {
546 AddSubWithCarry(rd, rn, operand, SetFlags, SBC);
547 }
548
549
rmif(const Register & xn,unsigned rotation,StatusFlags flags)550 void Assembler::rmif(const Register& xn, unsigned rotation, StatusFlags flags) {
551 VIXL_ASSERT(CPUHas(CPUFeatures::kFlagM));
552 VIXL_ASSERT(xn.Is64Bits());
553 Emit(RMIF | Rn(xn) | ImmRMIFRotation(rotation) | Nzcv(flags));
554 }
555
556
setf8(const Register & rn)557 void Assembler::setf8(const Register& rn) {
558 VIXL_ASSERT(CPUHas(CPUFeatures::kFlagM));
559 Emit(SETF8 | Rn(rn));
560 }
561
562
setf16(const Register & rn)563 void Assembler::setf16(const Register& rn) {
564 VIXL_ASSERT(CPUHas(CPUFeatures::kFlagM));
565 Emit(SETF16 | Rn(rn));
566 }
567
568
ngc(const Register & rd,const Operand & operand)569 void Assembler::ngc(const Register& rd, const Operand& operand) {
570 Register zr = AppropriateZeroRegFor(rd);
571 sbc(rd, zr, operand);
572 }
573
574
ngcs(const Register & rd,const Operand & operand)575 void Assembler::ngcs(const Register& rd, const Operand& operand) {
576 Register zr = AppropriateZeroRegFor(rd);
577 sbcs(rd, zr, operand);
578 }
579
580
581 // Logical instructions.
and_(const Register & rd,const Register & rn,const Operand & operand)582 void Assembler::and_(const Register& rd,
583 const Register& rn,
584 const Operand& operand) {
585 Logical(rd, rn, operand, AND);
586 }
587
588
ands(const Register & rd,const Register & rn,const Operand & operand)589 void Assembler::ands(const Register& rd,
590 const Register& rn,
591 const Operand& operand) {
592 Logical(rd, rn, operand, ANDS);
593 }
594
595
tst(const Register & rn,const Operand & operand)596 void Assembler::tst(const Register& rn, const Operand& operand) {
597 ands(AppropriateZeroRegFor(rn), rn, operand);
598 }
599
600
bic(const Register & rd,const Register & rn,const Operand & operand)601 void Assembler::bic(const Register& rd,
602 const Register& rn,
603 const Operand& operand) {
604 Logical(rd, rn, operand, BIC);
605 }
606
607
bics(const Register & rd,const Register & rn,const Operand & operand)608 void Assembler::bics(const Register& rd,
609 const Register& rn,
610 const Operand& operand) {
611 Logical(rd, rn, operand, BICS);
612 }
613
614
orr(const Register & rd,const Register & rn,const Operand & operand)615 void Assembler::orr(const Register& rd,
616 const Register& rn,
617 const Operand& operand) {
618 Logical(rd, rn, operand, ORR);
619 }
620
621
orn(const Register & rd,const Register & rn,const Operand & operand)622 void Assembler::orn(const Register& rd,
623 const Register& rn,
624 const Operand& operand) {
625 Logical(rd, rn, operand, ORN);
626 }
627
628
eor(const Register & rd,const Register & rn,const Operand & operand)629 void Assembler::eor(const Register& rd,
630 const Register& rn,
631 const Operand& operand) {
632 Logical(rd, rn, operand, EOR);
633 }
634
635
eon(const Register & rd,const Register & rn,const Operand & operand)636 void Assembler::eon(const Register& rd,
637 const Register& rn,
638 const Operand& operand) {
639 Logical(rd, rn, operand, EON);
640 }
641
642
lslv(const Register & rd,const Register & rn,const Register & rm)643 void Assembler::lslv(const Register& rd,
644 const Register& rn,
645 const Register& rm) {
646 VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
647 VIXL_ASSERT(rd.GetSizeInBits() == rm.GetSizeInBits());
648 Emit(SF(rd) | LSLV | Rm(rm) | Rn(rn) | Rd(rd));
649 }
650
651
lsrv(const Register & rd,const Register & rn,const Register & rm)652 void Assembler::lsrv(const Register& rd,
653 const Register& rn,
654 const Register& rm) {
655 VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
656 VIXL_ASSERT(rd.GetSizeInBits() == rm.GetSizeInBits());
657 Emit(SF(rd) | LSRV | Rm(rm) | Rn(rn) | Rd(rd));
658 }
659
660
asrv(const Register & rd,const Register & rn,const Register & rm)661 void Assembler::asrv(const Register& rd,
662 const Register& rn,
663 const Register& rm) {
664 VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
665 VIXL_ASSERT(rd.GetSizeInBits() == rm.GetSizeInBits());
666 Emit(SF(rd) | ASRV | Rm(rm) | Rn(rn) | Rd(rd));
667 }
668
669
rorv(const Register & rd,const Register & rn,const Register & rm)670 void Assembler::rorv(const Register& rd,
671 const Register& rn,
672 const Register& rm) {
673 VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
674 VIXL_ASSERT(rd.GetSizeInBits() == rm.GetSizeInBits());
675 Emit(SF(rd) | RORV | Rm(rm) | Rn(rn) | Rd(rd));
676 }
677
678
679 // Bitfield operations.
bfm(const Register & rd,const Register & rn,unsigned immr,unsigned imms)680 void Assembler::bfm(const Register& rd,
681 const Register& rn,
682 unsigned immr,
683 unsigned imms) {
684 VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
685 Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset);
686 Emit(SF(rd) | BFM | N | ImmR(immr, rd.GetSizeInBits()) |
687 ImmS(imms, rn.GetSizeInBits()) | Rn(rn) | Rd(rd));
688 }
689
690
sbfm(const Register & rd,const Register & rn,unsigned immr,unsigned imms)691 void Assembler::sbfm(const Register& rd,
692 const Register& rn,
693 unsigned immr,
694 unsigned imms) {
695 VIXL_ASSERT(rd.Is64Bits() || rn.Is32Bits());
696 Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset);
697 Emit(SF(rd) | SBFM | N | ImmR(immr, rd.GetSizeInBits()) |
698 ImmS(imms, rn.GetSizeInBits()) | Rn(rn) | Rd(rd));
699 }
700
701
ubfm(const Register & rd,const Register & rn,unsigned immr,unsigned imms)702 void Assembler::ubfm(const Register& rd,
703 const Register& rn,
704 unsigned immr,
705 unsigned imms) {
706 VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
707 Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset);
708 Emit(SF(rd) | UBFM | N | ImmR(immr, rd.GetSizeInBits()) |
709 ImmS(imms, rn.GetSizeInBits()) | Rn(rn) | Rd(rd));
710 }
711
712
extr(const Register & rd,const Register & rn,const Register & rm,unsigned lsb)713 void Assembler::extr(const Register& rd,
714 const Register& rn,
715 const Register& rm,
716 unsigned lsb) {
717 VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
718 VIXL_ASSERT(rd.GetSizeInBits() == rm.GetSizeInBits());
719 Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset);
720 Emit(SF(rd) | EXTR | N | Rm(rm) | ImmS(lsb, rn.GetSizeInBits()) | Rn(rn) |
721 Rd(rd));
722 }
723
724
csel(const Register & rd,const Register & rn,const Register & rm,Condition cond)725 void Assembler::csel(const Register& rd,
726 const Register& rn,
727 const Register& rm,
728 Condition cond) {
729 ConditionalSelect(rd, rn, rm, cond, CSEL);
730 }
731
732
csinc(const Register & rd,const Register & rn,const Register & rm,Condition cond)733 void Assembler::csinc(const Register& rd,
734 const Register& rn,
735 const Register& rm,
736 Condition cond) {
737 ConditionalSelect(rd, rn, rm, cond, CSINC);
738 }
739
740
csinv(const Register & rd,const Register & rn,const Register & rm,Condition cond)741 void Assembler::csinv(const Register& rd,
742 const Register& rn,
743 const Register& rm,
744 Condition cond) {
745 ConditionalSelect(rd, rn, rm, cond, CSINV);
746 }
747
748
csneg(const Register & rd,const Register & rn,const Register & rm,Condition cond)749 void Assembler::csneg(const Register& rd,
750 const Register& rn,
751 const Register& rm,
752 Condition cond) {
753 ConditionalSelect(rd, rn, rm, cond, CSNEG);
754 }
755
756
cset(const Register & rd,Condition cond)757 void Assembler::cset(const Register& rd, Condition cond) {
758 VIXL_ASSERT((cond != al) && (cond != nv));
759 Register zr = AppropriateZeroRegFor(rd);
760 csinc(rd, zr, zr, InvertCondition(cond));
761 }
762
763
csetm(const Register & rd,Condition cond)764 void Assembler::csetm(const Register& rd, Condition cond) {
765 VIXL_ASSERT((cond != al) && (cond != nv));
766 Register zr = AppropriateZeroRegFor(rd);
767 csinv(rd, zr, zr, InvertCondition(cond));
768 }
769
770
cinc(const Register & rd,const Register & rn,Condition cond)771 void Assembler::cinc(const Register& rd, const Register& rn, Condition cond) {
772 VIXL_ASSERT((cond != al) && (cond != nv));
773 csinc(rd, rn, rn, InvertCondition(cond));
774 }
775
776
cinv(const Register & rd,const Register & rn,Condition cond)777 void Assembler::cinv(const Register& rd, const Register& rn, Condition cond) {
778 VIXL_ASSERT((cond != al) && (cond != nv));
779 csinv(rd, rn, rn, InvertCondition(cond));
780 }
781
782
cneg(const Register & rd,const Register & rn,Condition cond)783 void Assembler::cneg(const Register& rd, const Register& rn, Condition cond) {
784 VIXL_ASSERT((cond != al) && (cond != nv));
785 csneg(rd, rn, rn, InvertCondition(cond));
786 }
787
788
ConditionalSelect(const Register & rd,const Register & rn,const Register & rm,Condition cond,ConditionalSelectOp op)789 void Assembler::ConditionalSelect(const Register& rd,
790 const Register& rn,
791 const Register& rm,
792 Condition cond,
793 ConditionalSelectOp op) {
794 VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
795 VIXL_ASSERT(rd.GetSizeInBits() == rm.GetSizeInBits());
796 Emit(SF(rd) | op | Rm(rm) | Cond(cond) | Rn(rn) | Rd(rd));
797 }
798
799
ccmn(const Register & rn,const Operand & operand,StatusFlags nzcv,Condition cond)800 void Assembler::ccmn(const Register& rn,
801 const Operand& operand,
802 StatusFlags nzcv,
803 Condition cond) {
804 ConditionalCompare(rn, operand, nzcv, cond, CCMN);
805 }
806
807
ccmp(const Register & rn,const Operand & operand,StatusFlags nzcv,Condition cond)808 void Assembler::ccmp(const Register& rn,
809 const Operand& operand,
810 StatusFlags nzcv,
811 Condition cond) {
812 ConditionalCompare(rn, operand, nzcv, cond, CCMP);
813 }
814
815
DataProcessing3Source(const Register & rd,const Register & rn,const Register & rm,const Register & ra,DataProcessing3SourceOp op)816 void Assembler::DataProcessing3Source(const Register& rd,
817 const Register& rn,
818 const Register& rm,
819 const Register& ra,
820 DataProcessing3SourceOp op) {
821 Emit(SF(rd) | op | Rm(rm) | Ra(ra) | Rn(rn) | Rd(rd));
822 }
823
824
crc32b(const Register & wd,const Register & wn,const Register & wm)825 void Assembler::crc32b(const Register& wd,
826 const Register& wn,
827 const Register& wm) {
828 VIXL_ASSERT(CPUHas(CPUFeatures::kCRC32));
829 VIXL_ASSERT(wd.Is32Bits() && wn.Is32Bits() && wm.Is32Bits());
830 Emit(SF(wm) | Rm(wm) | CRC32B | Rn(wn) | Rd(wd));
831 }
832
833
crc32h(const Register & wd,const Register & wn,const Register & wm)834 void Assembler::crc32h(const Register& wd,
835 const Register& wn,
836 const Register& wm) {
837 VIXL_ASSERT(CPUHas(CPUFeatures::kCRC32));
838 VIXL_ASSERT(wd.Is32Bits() && wn.Is32Bits() && wm.Is32Bits());
839 Emit(SF(wm) | Rm(wm) | CRC32H | Rn(wn) | Rd(wd));
840 }
841
842
crc32w(const Register & wd,const Register & wn,const Register & wm)843 void Assembler::crc32w(const Register& wd,
844 const Register& wn,
845 const Register& wm) {
846 VIXL_ASSERT(CPUHas(CPUFeatures::kCRC32));
847 VIXL_ASSERT(wd.Is32Bits() && wn.Is32Bits() && wm.Is32Bits());
848 Emit(SF(wm) | Rm(wm) | CRC32W | Rn(wn) | Rd(wd));
849 }
850
851
crc32x(const Register & wd,const Register & wn,const Register & xm)852 void Assembler::crc32x(const Register& wd,
853 const Register& wn,
854 const Register& xm) {
855 VIXL_ASSERT(CPUHas(CPUFeatures::kCRC32));
856 VIXL_ASSERT(wd.Is32Bits() && wn.Is32Bits() && xm.Is64Bits());
857 Emit(SF(xm) | Rm(xm) | CRC32X | Rn(wn) | Rd(wd));
858 }
859
860
crc32cb(const Register & wd,const Register & wn,const Register & wm)861 void Assembler::crc32cb(const Register& wd,
862 const Register& wn,
863 const Register& wm) {
864 VIXL_ASSERT(CPUHas(CPUFeatures::kCRC32));
865 VIXL_ASSERT(wd.Is32Bits() && wn.Is32Bits() && wm.Is32Bits());
866 Emit(SF(wm) | Rm(wm) | CRC32CB | Rn(wn) | Rd(wd));
867 }
868
869
crc32ch(const Register & wd,const Register & wn,const Register & wm)870 void Assembler::crc32ch(const Register& wd,
871 const Register& wn,
872 const Register& wm) {
873 VIXL_ASSERT(CPUHas(CPUFeatures::kCRC32));
874 VIXL_ASSERT(wd.Is32Bits() && wn.Is32Bits() && wm.Is32Bits());
875 Emit(SF(wm) | Rm(wm) | CRC32CH | Rn(wn) | Rd(wd));
876 }
877
878
crc32cw(const Register & wd,const Register & wn,const Register & wm)879 void Assembler::crc32cw(const Register& wd,
880 const Register& wn,
881 const Register& wm) {
882 VIXL_ASSERT(CPUHas(CPUFeatures::kCRC32));
883 VIXL_ASSERT(wd.Is32Bits() && wn.Is32Bits() && wm.Is32Bits());
884 Emit(SF(wm) | Rm(wm) | CRC32CW | Rn(wn) | Rd(wd));
885 }
886
887
crc32cx(const Register & wd,const Register & wn,const Register & xm)888 void Assembler::crc32cx(const Register& wd,
889 const Register& wn,
890 const Register& xm) {
891 VIXL_ASSERT(CPUHas(CPUFeatures::kCRC32));
892 VIXL_ASSERT(wd.Is32Bits() && wn.Is32Bits() && xm.Is64Bits());
893 Emit(SF(xm) | Rm(xm) | CRC32CX | Rn(wn) | Rd(wd));
894 }
895
896
mul(const Register & rd,const Register & rn,const Register & rm)897 void Assembler::mul(const Register& rd,
898 const Register& rn,
899 const Register& rm) {
900 VIXL_ASSERT(AreSameSizeAndType(rd, rn, rm));
901 DataProcessing3Source(rd, rn, rm, AppropriateZeroRegFor(rd), MADD);
902 }
903
904
madd(const Register & rd,const Register & rn,const Register & rm,const Register & ra)905 void Assembler::madd(const Register& rd,
906 const Register& rn,
907 const Register& rm,
908 const Register& ra) {
909 DataProcessing3Source(rd, rn, rm, ra, MADD);
910 }
911
912
mneg(const Register & rd,const Register & rn,const Register & rm)913 void Assembler::mneg(const Register& rd,
914 const Register& rn,
915 const Register& rm) {
916 VIXL_ASSERT(AreSameSizeAndType(rd, rn, rm));
917 DataProcessing3Source(rd, rn, rm, AppropriateZeroRegFor(rd), MSUB);
918 }
919
920
msub(const Register & rd,const Register & rn,const Register & rm,const Register & ra)921 void Assembler::msub(const Register& rd,
922 const Register& rn,
923 const Register& rm,
924 const Register& ra) {
925 DataProcessing3Source(rd, rn, rm, ra, MSUB);
926 }
927
928
umaddl(const Register & xd,const Register & wn,const Register & wm,const Register & xa)929 void Assembler::umaddl(const Register& xd,
930 const Register& wn,
931 const Register& wm,
932 const Register& xa) {
933 VIXL_ASSERT(xd.Is64Bits() && xa.Is64Bits());
934 VIXL_ASSERT(wn.Is32Bits() && wm.Is32Bits());
935 DataProcessing3Source(xd, wn, wm, xa, UMADDL_x);
936 }
937
938
smaddl(const Register & xd,const Register & wn,const Register & wm,const Register & xa)939 void Assembler::smaddl(const Register& xd,
940 const Register& wn,
941 const Register& wm,
942 const Register& xa) {
943 VIXL_ASSERT(xd.Is64Bits() && xa.Is64Bits());
944 VIXL_ASSERT(wn.Is32Bits() && wm.Is32Bits());
945 DataProcessing3Source(xd, wn, wm, xa, SMADDL_x);
946 }
947
948
umsubl(const Register & xd,const Register & wn,const Register & wm,const Register & xa)949 void Assembler::umsubl(const Register& xd,
950 const Register& wn,
951 const Register& wm,
952 const Register& xa) {
953 VIXL_ASSERT(xd.Is64Bits() && xa.Is64Bits());
954 VIXL_ASSERT(wn.Is32Bits() && wm.Is32Bits());
955 DataProcessing3Source(xd, wn, wm, xa, UMSUBL_x);
956 }
957
958
smsubl(const Register & xd,const Register & wn,const Register & wm,const Register & xa)959 void Assembler::smsubl(const Register& xd,
960 const Register& wn,
961 const Register& wm,
962 const Register& xa) {
963 VIXL_ASSERT(xd.Is64Bits() && xa.Is64Bits());
964 VIXL_ASSERT(wn.Is32Bits() && wm.Is32Bits());
965 DataProcessing3Source(xd, wn, wm, xa, SMSUBL_x);
966 }
967
968
smull(const Register & xd,const Register & wn,const Register & wm)969 void Assembler::smull(const Register& xd,
970 const Register& wn,
971 const Register& wm) {
972 VIXL_ASSERT(xd.Is64Bits());
973 VIXL_ASSERT(wn.Is32Bits() && wm.Is32Bits());
974 DataProcessing3Source(xd, wn, wm, xzr, SMADDL_x);
975 }
976
977
sdiv(const Register & rd,const Register & rn,const Register & rm)978 void Assembler::sdiv(const Register& rd,
979 const Register& rn,
980 const Register& rm) {
981 VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
982 VIXL_ASSERT(rd.GetSizeInBits() == rm.GetSizeInBits());
983 Emit(SF(rd) | SDIV | Rm(rm) | Rn(rn) | Rd(rd));
984 }
985
986
smulh(const Register & xd,const Register & xn,const Register & xm)987 void Assembler::smulh(const Register& xd,
988 const Register& xn,
989 const Register& xm) {
990 VIXL_ASSERT(xd.Is64Bits() && xn.Is64Bits() && xm.Is64Bits());
991 DataProcessing3Source(xd, xn, xm, xzr, SMULH_x);
992 }
993
994
umulh(const Register & xd,const Register & xn,const Register & xm)995 void Assembler::umulh(const Register& xd,
996 const Register& xn,
997 const Register& xm) {
998 VIXL_ASSERT(xd.Is64Bits() && xn.Is64Bits() && xm.Is64Bits());
999 DataProcessing3Source(xd, xn, xm, xzr, UMULH_x);
1000 }
1001
1002
udiv(const Register & rd,const Register & rn,const Register & rm)1003 void Assembler::udiv(const Register& rd,
1004 const Register& rn,
1005 const Register& rm) {
1006 VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
1007 VIXL_ASSERT(rd.GetSizeInBits() == rm.GetSizeInBits());
1008 Emit(SF(rd) | UDIV | Rm(rm) | Rn(rn) | Rd(rd));
1009 }
1010
1011
rbit(const Register & rd,const Register & rn)1012 void Assembler::rbit(const Register& rd, const Register& rn) {
1013 DataProcessing1Source(rd, rn, RBIT);
1014 }
1015
1016
rev16(const Register & rd,const Register & rn)1017 void Assembler::rev16(const Register& rd, const Register& rn) {
1018 DataProcessing1Source(rd, rn, REV16);
1019 }
1020
1021
rev32(const Register & xd,const Register & xn)1022 void Assembler::rev32(const Register& xd, const Register& xn) {
1023 VIXL_ASSERT(xd.Is64Bits());
1024 DataProcessing1Source(xd, xn, REV);
1025 }
1026
1027
rev(const Register & rd,const Register & rn)1028 void Assembler::rev(const Register& rd, const Register& rn) {
1029 DataProcessing1Source(rd, rn, rd.Is64Bits() ? REV_x : REV_w);
1030 }
1031
1032
clz(const Register & rd,const Register & rn)1033 void Assembler::clz(const Register& rd, const Register& rn) {
1034 DataProcessing1Source(rd, rn, CLZ);
1035 }
1036
1037
cls(const Register & rd,const Register & rn)1038 void Assembler::cls(const Register& rd, const Register& rn) {
1039 DataProcessing1Source(rd, rn, CLS);
1040 }
1041
1042 #define PAUTH_VARIATIONS(V) \
1043 V(paci, PACI) \
1044 V(pacd, PACD) \
1045 V(auti, AUTI) \
1046 V(autd, AUTD)
1047
1048 #define VIXL_DEFINE_ASM_FUNC(PRE, OP) \
1049 void Assembler::PRE##a(const Register& xd, const Register& xn) { \
1050 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth)); \
1051 VIXL_ASSERT(xd.Is64Bits() && xn.Is64Bits()); \
1052 Emit(SF(xd) | OP##A | Rd(xd) | RnSP(xn)); \
1053 } \
1054 \
1055 void Assembler::PRE##za(const Register& xd) { \
1056 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth)); \
1057 VIXL_ASSERT(xd.Is64Bits()); \
1058 Emit(SF(xd) | OP##ZA | Rd(xd) | Rn(xzr)); \
1059 } \
1060 \
1061 void Assembler::PRE##b(const Register& xd, const Register& xn) { \
1062 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth)); \
1063 VIXL_ASSERT(xd.Is64Bits() && xn.Is64Bits()); \
1064 Emit(SF(xd) | OP##B | Rd(xd) | RnSP(xn)); \
1065 } \
1066 \
1067 void Assembler::PRE##zb(const Register& xd) { \
1068 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth)); \
1069 VIXL_ASSERT(xd.Is64Bits()); \
1070 Emit(SF(xd) | OP##ZB | Rd(xd) | Rn(xzr)); \
1071 }
1072
PAUTH_VARIATIONS(VIXL_DEFINE_ASM_FUNC)1073 PAUTH_VARIATIONS(VIXL_DEFINE_ASM_FUNC)
1074 #undef VIXL_DEFINE_ASM_FUNC
1075
1076 void Assembler::pacga(const Register& xd,
1077 const Register& xn,
1078 const Register& xm) {
1079 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth, CPUFeatures::kPAuthGeneric));
1080 VIXL_ASSERT(xd.Is64Bits() && xn.Is64Bits() && xm.Is64Bits());
1081 Emit(SF(xd) | PACGA | Rd(xd) | Rn(xn) | RmSP(xm));
1082 }
1083
xpaci(const Register & xd)1084 void Assembler::xpaci(const Register& xd) {
1085 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
1086 VIXL_ASSERT(xd.Is64Bits());
1087 Emit(SF(xd) | XPACI | Rd(xd) | Rn(xzr));
1088 }
1089
xpacd(const Register & xd)1090 void Assembler::xpacd(const Register& xd) {
1091 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
1092 VIXL_ASSERT(xd.Is64Bits());
1093 Emit(SF(xd) | XPACD | Rd(xd) | Rn(xzr));
1094 }
1095
1096
ldp(const CPURegister & rt,const CPURegister & rt2,const MemOperand & src)1097 void Assembler::ldp(const CPURegister& rt,
1098 const CPURegister& rt2,
1099 const MemOperand& src) {
1100 LoadStorePair(rt, rt2, src, LoadPairOpFor(rt, rt2));
1101 }
1102
1103
stp(const CPURegister & rt,const CPURegister & rt2,const MemOperand & dst)1104 void Assembler::stp(const CPURegister& rt,
1105 const CPURegister& rt2,
1106 const MemOperand& dst) {
1107 LoadStorePair(rt, rt2, dst, StorePairOpFor(rt, rt2));
1108 }
1109
1110
ldpsw(const Register & xt,const Register & xt2,const MemOperand & src)1111 void Assembler::ldpsw(const Register& xt,
1112 const Register& xt2,
1113 const MemOperand& src) {
1114 VIXL_ASSERT(xt.Is64Bits() && xt2.Is64Bits());
1115 LoadStorePair(xt, xt2, src, LDPSW_x);
1116 }
1117
1118
LoadStorePair(const CPURegister & rt,const CPURegister & rt2,const MemOperand & addr,LoadStorePairOp op)1119 void Assembler::LoadStorePair(const CPURegister& rt,
1120 const CPURegister& rt2,
1121 const MemOperand& addr,
1122 LoadStorePairOp op) {
1123 VIXL_ASSERT(CPUHas(rt, rt2));
1124
1125 // 'rt' and 'rt2' can only be aliased for stores.
1126 VIXL_ASSERT(((op & LoadStorePairLBit) == 0) || !rt.Is(rt2));
1127 VIXL_ASSERT(AreSameSizeAndType(rt, rt2));
1128 VIXL_ASSERT(IsImmLSPair(addr.GetOffset(), CalcLSPairDataSize(op)));
1129
1130 int offset = static_cast<int>(addr.GetOffset());
1131 Instr memop = op | Rt(rt) | Rt2(rt2) | RnSP(addr.GetBaseRegister()) |
1132 ImmLSPair(offset, CalcLSPairDataSize(op));
1133
1134 Instr addrmodeop;
1135 if (addr.IsImmediateOffset()) {
1136 addrmodeop = LoadStorePairOffsetFixed;
1137 } else {
1138 if (addr.IsImmediatePreIndex()) {
1139 addrmodeop = LoadStorePairPreIndexFixed;
1140 } else {
1141 VIXL_ASSERT(addr.IsImmediatePostIndex());
1142 addrmodeop = LoadStorePairPostIndexFixed;
1143 }
1144 }
1145
1146 Instr emitop = addrmodeop | memop;
1147
1148 // Only X registers may be specified for ldpsw.
1149 VIXL_ASSERT(((emitop & LoadStorePairMask) != LDPSW_x) || rt.IsX());
1150
1151 Emit(emitop);
1152 }
1153
1154
ldnp(const CPURegister & rt,const CPURegister & rt2,const MemOperand & src)1155 void Assembler::ldnp(const CPURegister& rt,
1156 const CPURegister& rt2,
1157 const MemOperand& src) {
1158 LoadStorePairNonTemporal(rt, rt2, src, LoadPairNonTemporalOpFor(rt, rt2));
1159 }
1160
1161
stnp(const CPURegister & rt,const CPURegister & rt2,const MemOperand & dst)1162 void Assembler::stnp(const CPURegister& rt,
1163 const CPURegister& rt2,
1164 const MemOperand& dst) {
1165 LoadStorePairNonTemporal(rt, rt2, dst, StorePairNonTemporalOpFor(rt, rt2));
1166 }
1167
1168
LoadStorePairNonTemporal(const CPURegister & rt,const CPURegister & rt2,const MemOperand & addr,LoadStorePairNonTemporalOp op)1169 void Assembler::LoadStorePairNonTemporal(const CPURegister& rt,
1170 const CPURegister& rt2,
1171 const MemOperand& addr,
1172 LoadStorePairNonTemporalOp op) {
1173 VIXL_ASSERT(CPUHas(rt, rt2));
1174
1175 VIXL_ASSERT(!rt.Is(rt2));
1176 VIXL_ASSERT(AreSameSizeAndType(rt, rt2));
1177 VIXL_ASSERT(addr.IsImmediateOffset());
1178
1179 unsigned size =
1180 CalcLSPairDataSize(static_cast<LoadStorePairOp>(op & LoadStorePairMask));
1181 VIXL_ASSERT(IsImmLSPair(addr.GetOffset(), size));
1182 int offset = static_cast<int>(addr.GetOffset());
1183 Emit(op | Rt(rt) | Rt2(rt2) | RnSP(addr.GetBaseRegister()) |
1184 ImmLSPair(offset, size));
1185 }
1186
1187
1188 // Memory instructions.
ldrb(const Register & rt,const MemOperand & src,LoadStoreScalingOption option)1189 void Assembler::ldrb(const Register& rt,
1190 const MemOperand& src,
1191 LoadStoreScalingOption option) {
1192 VIXL_ASSERT(option != RequireUnscaledOffset);
1193 VIXL_ASSERT(option != PreferUnscaledOffset);
1194 LoadStore(rt, src, LDRB_w, option);
1195 }
1196
1197
strb(const Register & rt,const MemOperand & dst,LoadStoreScalingOption option)1198 void Assembler::strb(const Register& rt,
1199 const MemOperand& dst,
1200 LoadStoreScalingOption option) {
1201 VIXL_ASSERT(option != RequireUnscaledOffset);
1202 VIXL_ASSERT(option != PreferUnscaledOffset);
1203 LoadStore(rt, dst, STRB_w, option);
1204 }
1205
1206
ldrsb(const Register & rt,const MemOperand & src,LoadStoreScalingOption option)1207 void Assembler::ldrsb(const Register& rt,
1208 const MemOperand& src,
1209 LoadStoreScalingOption option) {
1210 VIXL_ASSERT(option != RequireUnscaledOffset);
1211 VIXL_ASSERT(option != PreferUnscaledOffset);
1212 LoadStore(rt, src, rt.Is64Bits() ? LDRSB_x : LDRSB_w, option);
1213 }
1214
1215
ldrh(const Register & rt,const MemOperand & src,LoadStoreScalingOption option)1216 void Assembler::ldrh(const Register& rt,
1217 const MemOperand& src,
1218 LoadStoreScalingOption option) {
1219 VIXL_ASSERT(option != RequireUnscaledOffset);
1220 VIXL_ASSERT(option != PreferUnscaledOffset);
1221 LoadStore(rt, src, LDRH_w, option);
1222 }
1223
1224
strh(const Register & rt,const MemOperand & dst,LoadStoreScalingOption option)1225 void Assembler::strh(const Register& rt,
1226 const MemOperand& dst,
1227 LoadStoreScalingOption option) {
1228 VIXL_ASSERT(option != RequireUnscaledOffset);
1229 VIXL_ASSERT(option != PreferUnscaledOffset);
1230 LoadStore(rt, dst, STRH_w, option);
1231 }
1232
1233
ldrsh(const Register & rt,const MemOperand & src,LoadStoreScalingOption option)1234 void Assembler::ldrsh(const Register& rt,
1235 const MemOperand& src,
1236 LoadStoreScalingOption option) {
1237 VIXL_ASSERT(option != RequireUnscaledOffset);
1238 VIXL_ASSERT(option != PreferUnscaledOffset);
1239 LoadStore(rt, src, rt.Is64Bits() ? LDRSH_x : LDRSH_w, option);
1240 }
1241
1242
ldr(const CPURegister & rt,const MemOperand & src,LoadStoreScalingOption option)1243 void Assembler::ldr(const CPURegister& rt,
1244 const MemOperand& src,
1245 LoadStoreScalingOption option) {
1246 VIXL_ASSERT(option != RequireUnscaledOffset);
1247 VIXL_ASSERT(option != PreferUnscaledOffset);
1248 LoadStore(rt, src, LoadOpFor(rt), option);
1249 }
1250
1251
str(const CPURegister & rt,const MemOperand & dst,LoadStoreScalingOption option)1252 void Assembler::str(const CPURegister& rt,
1253 const MemOperand& dst,
1254 LoadStoreScalingOption option) {
1255 VIXL_ASSERT(option != RequireUnscaledOffset);
1256 VIXL_ASSERT(option != PreferUnscaledOffset);
1257 LoadStore(rt, dst, StoreOpFor(rt), option);
1258 }
1259
1260
ldrsw(const Register & xt,const MemOperand & src,LoadStoreScalingOption option)1261 void Assembler::ldrsw(const Register& xt,
1262 const MemOperand& src,
1263 LoadStoreScalingOption option) {
1264 VIXL_ASSERT(xt.Is64Bits());
1265 VIXL_ASSERT(option != RequireUnscaledOffset);
1266 VIXL_ASSERT(option != PreferUnscaledOffset);
1267 LoadStore(xt, src, LDRSW_x, option);
1268 }
1269
1270
ldurb(const Register & rt,const MemOperand & src,LoadStoreScalingOption option)1271 void Assembler::ldurb(const Register& rt,
1272 const MemOperand& src,
1273 LoadStoreScalingOption option) {
1274 VIXL_ASSERT(option != RequireScaledOffset);
1275 VIXL_ASSERT(option != PreferScaledOffset);
1276 LoadStore(rt, src, LDRB_w, option);
1277 }
1278
1279
sturb(const Register & rt,const MemOperand & dst,LoadStoreScalingOption option)1280 void Assembler::sturb(const Register& rt,
1281 const MemOperand& dst,
1282 LoadStoreScalingOption option) {
1283 VIXL_ASSERT(option != RequireScaledOffset);
1284 VIXL_ASSERT(option != PreferScaledOffset);
1285 LoadStore(rt, dst, STRB_w, option);
1286 }
1287
1288
ldursb(const Register & rt,const MemOperand & src,LoadStoreScalingOption option)1289 void Assembler::ldursb(const Register& rt,
1290 const MemOperand& src,
1291 LoadStoreScalingOption option) {
1292 VIXL_ASSERT(option != RequireScaledOffset);
1293 VIXL_ASSERT(option != PreferScaledOffset);
1294 LoadStore(rt, src, rt.Is64Bits() ? LDRSB_x : LDRSB_w, option);
1295 }
1296
1297
ldurh(const Register & rt,const MemOperand & src,LoadStoreScalingOption option)1298 void Assembler::ldurh(const Register& rt,
1299 const MemOperand& src,
1300 LoadStoreScalingOption option) {
1301 VIXL_ASSERT(option != RequireScaledOffset);
1302 VIXL_ASSERT(option != PreferScaledOffset);
1303 LoadStore(rt, src, LDRH_w, option);
1304 }
1305
1306
sturh(const Register & rt,const MemOperand & dst,LoadStoreScalingOption option)1307 void Assembler::sturh(const Register& rt,
1308 const MemOperand& dst,
1309 LoadStoreScalingOption option) {
1310 VIXL_ASSERT(option != RequireScaledOffset);
1311 VIXL_ASSERT(option != PreferScaledOffset);
1312 LoadStore(rt, dst, STRH_w, option);
1313 }
1314
1315
ldursh(const Register & rt,const MemOperand & src,LoadStoreScalingOption option)1316 void Assembler::ldursh(const Register& rt,
1317 const MemOperand& src,
1318 LoadStoreScalingOption option) {
1319 VIXL_ASSERT(option != RequireScaledOffset);
1320 VIXL_ASSERT(option != PreferScaledOffset);
1321 LoadStore(rt, src, rt.Is64Bits() ? LDRSH_x : LDRSH_w, option);
1322 }
1323
1324
ldur(const CPURegister & rt,const MemOperand & src,LoadStoreScalingOption option)1325 void Assembler::ldur(const CPURegister& rt,
1326 const MemOperand& src,
1327 LoadStoreScalingOption option) {
1328 VIXL_ASSERT(option != RequireScaledOffset);
1329 VIXL_ASSERT(option != PreferScaledOffset);
1330 LoadStore(rt, src, LoadOpFor(rt), option);
1331 }
1332
1333
stur(const CPURegister & rt,const MemOperand & dst,LoadStoreScalingOption option)1334 void Assembler::stur(const CPURegister& rt,
1335 const MemOperand& dst,
1336 LoadStoreScalingOption option) {
1337 VIXL_ASSERT(option != RequireScaledOffset);
1338 VIXL_ASSERT(option != PreferScaledOffset);
1339 LoadStore(rt, dst, StoreOpFor(rt), option);
1340 }
1341
1342
ldursw(const Register & xt,const MemOperand & src,LoadStoreScalingOption option)1343 void Assembler::ldursw(const Register& xt,
1344 const MemOperand& src,
1345 LoadStoreScalingOption option) {
1346 VIXL_ASSERT(xt.Is64Bits());
1347 VIXL_ASSERT(option != RequireScaledOffset);
1348 VIXL_ASSERT(option != PreferScaledOffset);
1349 LoadStore(xt, src, LDRSW_x, option);
1350 }
1351
1352
ldraa(const Register & xt,const MemOperand & src)1353 void Assembler::ldraa(const Register& xt, const MemOperand& src) {
1354 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
1355 LoadStorePAC(xt, src, LDRAA);
1356 }
1357
1358
ldrab(const Register & xt,const MemOperand & src)1359 void Assembler::ldrab(const Register& xt, const MemOperand& src) {
1360 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
1361 LoadStorePAC(xt, src, LDRAB);
1362 }
1363
1364
ldrsw(const Register & xt,RawLiteral * literal)1365 void Assembler::ldrsw(const Register& xt, RawLiteral* literal) {
1366 VIXL_ASSERT(xt.Is64Bits());
1367 VIXL_ASSERT(literal->GetSize() == kWRegSizeInBytes);
1368 ldrsw(xt, static_cast<int>(LinkAndGetWordOffsetTo(literal)));
1369 }
1370
1371
ldr(const CPURegister & rt,RawLiteral * literal)1372 void Assembler::ldr(const CPURegister& rt, RawLiteral* literal) {
1373 VIXL_ASSERT(CPUHas(rt));
1374 VIXL_ASSERT(literal->GetSize() == static_cast<size_t>(rt.GetSizeInBytes()));
1375 ldr(rt, static_cast<int>(LinkAndGetWordOffsetTo(literal)));
1376 }
1377
1378
ldrsw(const Register & rt,int64_t imm19)1379 void Assembler::ldrsw(const Register& rt, int64_t imm19) {
1380 Emit(LDRSW_x_lit | ImmLLiteral(imm19) | Rt(rt));
1381 }
1382
1383
ldr(const CPURegister & rt,int64_t imm19)1384 void Assembler::ldr(const CPURegister& rt, int64_t imm19) {
1385 VIXL_ASSERT(CPUHas(rt));
1386 LoadLiteralOp op = LoadLiteralOpFor(rt);
1387 Emit(op | ImmLLiteral(imm19) | Rt(rt));
1388 }
1389
1390
prfm(int op,int64_t imm19)1391 void Assembler::prfm(int op, int64_t imm19) {
1392 Emit(PRFM_lit | ImmPrefetchOperation(op) | ImmLLiteral(imm19));
1393 }
1394
prfm(PrefetchOperation op,int64_t imm19)1395 void Assembler::prfm(PrefetchOperation op, int64_t imm19) {
1396 // Passing unnamed values in 'op' is undefined behaviour in C++.
1397 VIXL_ASSERT(IsNamedPrefetchOperation(op));
1398 prfm(static_cast<int>(op), imm19);
1399 }
1400
1401
1402 // Exclusive-access instructions.
stxrb(const Register & rs,const Register & rt,const MemOperand & dst)1403 void Assembler::stxrb(const Register& rs,
1404 const Register& rt,
1405 const MemOperand& dst) {
1406 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1407 Emit(STXRB_w | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.GetBaseRegister()));
1408 }
1409
1410
stxrh(const Register & rs,const Register & rt,const MemOperand & dst)1411 void Assembler::stxrh(const Register& rs,
1412 const Register& rt,
1413 const MemOperand& dst) {
1414 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1415 Emit(STXRH_w | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.GetBaseRegister()));
1416 }
1417
1418
stxr(const Register & rs,const Register & rt,const MemOperand & dst)1419 void Assembler::stxr(const Register& rs,
1420 const Register& rt,
1421 const MemOperand& dst) {
1422 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1423 LoadStoreExclusive op = rt.Is64Bits() ? STXR_x : STXR_w;
1424 Emit(op | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.GetBaseRegister()));
1425 }
1426
1427
ldxrb(const Register & rt,const MemOperand & src)1428 void Assembler::ldxrb(const Register& rt, const MemOperand& src) {
1429 VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1430 Emit(LDXRB_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister()));
1431 }
1432
1433
ldxrh(const Register & rt,const MemOperand & src)1434 void Assembler::ldxrh(const Register& rt, const MemOperand& src) {
1435 VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1436 Emit(LDXRH_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister()));
1437 }
1438
1439
ldxr(const Register & rt,const MemOperand & src)1440 void Assembler::ldxr(const Register& rt, const MemOperand& src) {
1441 VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1442 LoadStoreExclusive op = rt.Is64Bits() ? LDXR_x : LDXR_w;
1443 Emit(op | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister()));
1444 }
1445
1446
stxp(const Register & rs,const Register & rt,const Register & rt2,const MemOperand & dst)1447 void Assembler::stxp(const Register& rs,
1448 const Register& rt,
1449 const Register& rt2,
1450 const MemOperand& dst) {
1451 VIXL_ASSERT(rt.GetSizeInBits() == rt2.GetSizeInBits());
1452 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1453 LoadStoreExclusive op = rt.Is64Bits() ? STXP_x : STXP_w;
1454 Emit(op | Rs(rs) | Rt(rt) | Rt2(rt2) | RnSP(dst.GetBaseRegister()));
1455 }
1456
1457
ldxp(const Register & rt,const Register & rt2,const MemOperand & src)1458 void Assembler::ldxp(const Register& rt,
1459 const Register& rt2,
1460 const MemOperand& src) {
1461 VIXL_ASSERT(rt.GetSizeInBits() == rt2.GetSizeInBits());
1462 VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1463 LoadStoreExclusive op = rt.Is64Bits() ? LDXP_x : LDXP_w;
1464 Emit(op | Rs_mask | Rt(rt) | Rt2(rt2) | RnSP(src.GetBaseRegister()));
1465 }
1466
1467
stlxrb(const Register & rs,const Register & rt,const MemOperand & dst)1468 void Assembler::stlxrb(const Register& rs,
1469 const Register& rt,
1470 const MemOperand& dst) {
1471 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1472 Emit(STLXRB_w | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.GetBaseRegister()));
1473 }
1474
1475
stlxrh(const Register & rs,const Register & rt,const MemOperand & dst)1476 void Assembler::stlxrh(const Register& rs,
1477 const Register& rt,
1478 const MemOperand& dst) {
1479 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1480 Emit(STLXRH_w | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.GetBaseRegister()));
1481 }
1482
1483
stlxr(const Register & rs,const Register & rt,const MemOperand & dst)1484 void Assembler::stlxr(const Register& rs,
1485 const Register& rt,
1486 const MemOperand& dst) {
1487 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1488 LoadStoreExclusive op = rt.Is64Bits() ? STLXR_x : STLXR_w;
1489 Emit(op | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.GetBaseRegister()));
1490 }
1491
1492
ldaxrb(const Register & rt,const MemOperand & src)1493 void Assembler::ldaxrb(const Register& rt, const MemOperand& src) {
1494 VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1495 Emit(LDAXRB_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister()));
1496 }
1497
1498
ldaxrh(const Register & rt,const MemOperand & src)1499 void Assembler::ldaxrh(const Register& rt, const MemOperand& src) {
1500 VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1501 Emit(LDAXRH_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister()));
1502 }
1503
1504
ldaxr(const Register & rt,const MemOperand & src)1505 void Assembler::ldaxr(const Register& rt, const MemOperand& src) {
1506 VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1507 LoadStoreExclusive op = rt.Is64Bits() ? LDAXR_x : LDAXR_w;
1508 Emit(op | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister()));
1509 }
1510
1511
stlxp(const Register & rs,const Register & rt,const Register & rt2,const MemOperand & dst)1512 void Assembler::stlxp(const Register& rs,
1513 const Register& rt,
1514 const Register& rt2,
1515 const MemOperand& dst) {
1516 VIXL_ASSERT(rt.GetSizeInBits() == rt2.GetSizeInBits());
1517 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1518 LoadStoreExclusive op = rt.Is64Bits() ? STLXP_x : STLXP_w;
1519 Emit(op | Rs(rs) | Rt(rt) | Rt2(rt2) | RnSP(dst.GetBaseRegister()));
1520 }
1521
1522
ldaxp(const Register & rt,const Register & rt2,const MemOperand & src)1523 void Assembler::ldaxp(const Register& rt,
1524 const Register& rt2,
1525 const MemOperand& src) {
1526 VIXL_ASSERT(rt.GetSizeInBits() == rt2.GetSizeInBits());
1527 VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1528 LoadStoreExclusive op = rt.Is64Bits() ? LDAXP_x : LDAXP_w;
1529 Emit(op | Rs_mask | Rt(rt) | Rt2(rt2) | RnSP(src.GetBaseRegister()));
1530 }
1531
1532
stlrb(const Register & rt,const MemOperand & dst)1533 void Assembler::stlrb(const Register& rt, const MemOperand& dst) {
1534 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1535 Emit(STLRB_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(dst.GetBaseRegister()));
1536 }
1537
stlurb(const Register & rt,const MemOperand & dst)1538 void Assembler::stlurb(const Register& rt, const MemOperand& dst) {
1539 VIXL_ASSERT(CPUHas(CPUFeatures::kRCpc, CPUFeatures::kRCpcImm));
1540 VIXL_ASSERT(dst.IsImmediateOffset() && IsImmLSUnscaled(dst.GetOffset()));
1541
1542 Instr base = RnSP(dst.GetBaseRegister());
1543 int64_t offset = dst.GetOffset();
1544 Emit(STLURB | Rt(rt) | base | ImmLS(static_cast<int>(offset)));
1545 }
1546
1547
stlrh(const Register & rt,const MemOperand & dst)1548 void Assembler::stlrh(const Register& rt, const MemOperand& dst) {
1549 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1550 Emit(STLRH_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(dst.GetBaseRegister()));
1551 }
1552
stlurh(const Register & rt,const MemOperand & dst)1553 void Assembler::stlurh(const Register& rt, const MemOperand& dst) {
1554 VIXL_ASSERT(CPUHas(CPUFeatures::kRCpc, CPUFeatures::kRCpcImm));
1555 VIXL_ASSERT(dst.IsImmediateOffset() && IsImmLSUnscaled(dst.GetOffset()));
1556
1557 Instr base = RnSP(dst.GetBaseRegister());
1558 int64_t offset = dst.GetOffset();
1559 Emit(STLURH | Rt(rt) | base | ImmLS(static_cast<int>(offset)));
1560 }
1561
1562
stlr(const Register & rt,const MemOperand & dst)1563 void Assembler::stlr(const Register& rt, const MemOperand& dst) {
1564 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1565 LoadStoreExclusive op = rt.Is64Bits() ? STLR_x : STLR_w;
1566 Emit(op | Rs_mask | Rt(rt) | Rt2_mask | RnSP(dst.GetBaseRegister()));
1567 }
1568
stlur(const Register & rt,const MemOperand & dst)1569 void Assembler::stlur(const Register& rt, const MemOperand& dst) {
1570 VIXL_ASSERT(CPUHas(CPUFeatures::kRCpc, CPUFeatures::kRCpcImm));
1571 VIXL_ASSERT(dst.IsImmediateOffset() && IsImmLSUnscaled(dst.GetOffset()));
1572
1573 Instr base = RnSP(dst.GetBaseRegister());
1574 int64_t offset = dst.GetOffset();
1575 Instr op = rt.Is64Bits() ? STLUR_x : STLUR_w;
1576 Emit(op | Rt(rt) | base | ImmLS(static_cast<int>(offset)));
1577 }
1578
1579
ldarb(const Register & rt,const MemOperand & src)1580 void Assembler::ldarb(const Register& rt, const MemOperand& src) {
1581 VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1582 Emit(LDARB_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister()));
1583 }
1584
1585
ldarh(const Register & rt,const MemOperand & src)1586 void Assembler::ldarh(const Register& rt, const MemOperand& src) {
1587 VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1588 Emit(LDARH_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister()));
1589 }
1590
1591
ldar(const Register & rt,const MemOperand & src)1592 void Assembler::ldar(const Register& rt, const MemOperand& src) {
1593 VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1594 LoadStoreExclusive op = rt.Is64Bits() ? LDAR_x : LDAR_w;
1595 Emit(op | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister()));
1596 }
1597
1598
stllrb(const Register & rt,const MemOperand & dst)1599 void Assembler::stllrb(const Register& rt, const MemOperand& dst) {
1600 VIXL_ASSERT(CPUHas(CPUFeatures::kLORegions));
1601 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1602 Emit(STLLRB | Rs_mask | Rt(rt) | Rt2_mask | RnSP(dst.GetBaseRegister()));
1603 }
1604
1605
stllrh(const Register & rt,const MemOperand & dst)1606 void Assembler::stllrh(const Register& rt, const MemOperand& dst) {
1607 VIXL_ASSERT(CPUHas(CPUFeatures::kLORegions));
1608 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1609 Emit(STLLRH | Rs_mask | Rt(rt) | Rt2_mask | RnSP(dst.GetBaseRegister()));
1610 }
1611
1612
stllr(const Register & rt,const MemOperand & dst)1613 void Assembler::stllr(const Register& rt, const MemOperand& dst) {
1614 VIXL_ASSERT(CPUHas(CPUFeatures::kLORegions));
1615 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1616 LoadStoreExclusive op = rt.Is64Bits() ? STLLR_x : STLLR_w;
1617 Emit(op | Rs_mask | Rt(rt) | Rt2_mask | RnSP(dst.GetBaseRegister()));
1618 }
1619
1620
ldlarb(const Register & rt,const MemOperand & src)1621 void Assembler::ldlarb(const Register& rt, const MemOperand& src) {
1622 VIXL_ASSERT(CPUHas(CPUFeatures::kLORegions));
1623 VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1624 Emit(LDLARB | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister()));
1625 }
1626
1627
ldlarh(const Register & rt,const MemOperand & src)1628 void Assembler::ldlarh(const Register& rt, const MemOperand& src) {
1629 VIXL_ASSERT(CPUHas(CPUFeatures::kLORegions));
1630 VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1631 Emit(LDLARH | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister()));
1632 }
1633
1634
ldlar(const Register & rt,const MemOperand & src)1635 void Assembler::ldlar(const Register& rt, const MemOperand& src) {
1636 VIXL_ASSERT(CPUHas(CPUFeatures::kLORegions));
1637 VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1638 LoadStoreExclusive op = rt.Is64Bits() ? LDLAR_x : LDLAR_w;
1639 Emit(op | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister()));
1640 }
1641
1642
1643 // clang-format off
1644 #define COMPARE_AND_SWAP_W_X_LIST(V) \
1645 V(cas, CAS) \
1646 V(casa, CASA) \
1647 V(casl, CASL) \
1648 V(casal, CASAL)
1649 // clang-format on
1650
1651 #define VIXL_DEFINE_ASM_FUNC(FN, OP) \
1652 void Assembler::FN(const Register& rs, \
1653 const Register& rt, \
1654 const MemOperand& src) { \
1655 VIXL_ASSERT(CPUHas(CPUFeatures::kAtomics)); \
1656 VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0)); \
1657 VIXL_ASSERT(AreSameFormat(rs, rt)); \
1658 LoadStoreExclusive op = rt.Is64Bits() ? OP##_x : OP##_w; \
1659 Emit(op | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister())); \
1660 }
1661 COMPARE_AND_SWAP_W_X_LIST(VIXL_DEFINE_ASM_FUNC)
1662 #undef VIXL_DEFINE_ASM_FUNC
1663
1664 // clang-format off
1665 #define COMPARE_AND_SWAP_W_LIST(V) \
1666 V(casb, CASB) \
1667 V(casab, CASAB) \
1668 V(caslb, CASLB) \
1669 V(casalb, CASALB) \
1670 V(cash, CASH) \
1671 V(casah, CASAH) \
1672 V(caslh, CASLH) \
1673 V(casalh, CASALH)
1674 // clang-format on
1675
1676 #define VIXL_DEFINE_ASM_FUNC(FN, OP) \
1677 void Assembler::FN(const Register& rs, \
1678 const Register& rt, \
1679 const MemOperand& src) { \
1680 VIXL_ASSERT(CPUHas(CPUFeatures::kAtomics)); \
1681 VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0)); \
1682 Emit(OP | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister())); \
1683 }
COMPARE_AND_SWAP_W_LIST(VIXL_DEFINE_ASM_FUNC)1684 COMPARE_AND_SWAP_W_LIST(VIXL_DEFINE_ASM_FUNC)
1685 #undef VIXL_DEFINE_ASM_FUNC
1686
1687
1688 // clang-format off
1689 #define COMPARE_AND_SWAP_PAIR_LIST(V) \
1690 V(casp, CASP) \
1691 V(caspa, CASPA) \
1692 V(caspl, CASPL) \
1693 V(caspal, CASPAL)
1694 // clang-format on
1695
1696 #define VIXL_DEFINE_ASM_FUNC(FN, OP) \
1697 void Assembler::FN(const Register& rs, \
1698 const Register& rs1, \
1699 const Register& rt, \
1700 const Register& rt1, \
1701 const MemOperand& src) { \
1702 VIXL_ASSERT(CPUHas(CPUFeatures::kAtomics)); \
1703 USE(rs1, rt1); \
1704 VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0)); \
1705 VIXL_ASSERT(AreEven(rs, rt)); \
1706 VIXL_ASSERT(AreConsecutive(rs, rs1)); \
1707 VIXL_ASSERT(AreConsecutive(rt, rt1)); \
1708 VIXL_ASSERT(AreSameFormat(rs, rs1, rt, rt1)); \
1709 LoadStoreExclusive op = rt.Is64Bits() ? OP##_x : OP##_w; \
1710 Emit(op | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister())); \
1711 }
1712 COMPARE_AND_SWAP_PAIR_LIST(VIXL_DEFINE_ASM_FUNC)
1713 #undef VIXL_DEFINE_ASM_FUNC
1714
1715 // These macros generate all the variations of the atomic memory operations,
1716 // e.g. ldadd, ldadda, ldaddb, staddl, etc.
1717 // For a full list of the methods with comments, see the assembler header file.
1718
1719 // clang-format off
1720 #define ATOMIC_MEMORY_SIMPLE_OPERATION_LIST(V, DEF) \
1721 V(DEF, add, LDADD) \
1722 V(DEF, clr, LDCLR) \
1723 V(DEF, eor, LDEOR) \
1724 V(DEF, set, LDSET) \
1725 V(DEF, smax, LDSMAX) \
1726 V(DEF, smin, LDSMIN) \
1727 V(DEF, umax, LDUMAX) \
1728 V(DEF, umin, LDUMIN)
1729
1730 #define ATOMIC_MEMORY_STORE_MODES(V, NAME, OP) \
1731 V(NAME, OP##_x, OP##_w) \
1732 V(NAME##l, OP##L_x, OP##L_w) \
1733 V(NAME##b, OP##B, OP##B) \
1734 V(NAME##lb, OP##LB, OP##LB) \
1735 V(NAME##h, OP##H, OP##H) \
1736 V(NAME##lh, OP##LH, OP##LH)
1737
1738 #define ATOMIC_MEMORY_LOAD_MODES(V, NAME, OP) \
1739 ATOMIC_MEMORY_STORE_MODES(V, NAME, OP) \
1740 V(NAME##a, OP##A_x, OP##A_w) \
1741 V(NAME##al, OP##AL_x, OP##AL_w) \
1742 V(NAME##ab, OP##AB, OP##AB) \
1743 V(NAME##alb, OP##ALB, OP##ALB) \
1744 V(NAME##ah, OP##AH, OP##AH) \
1745 V(NAME##alh, OP##ALH, OP##ALH)
1746 // clang-format on
1747
1748 #define DEFINE_ASM_LOAD_FUNC(FN, OP_X, OP_W) \
1749 void Assembler::ld##FN(const Register& rs, \
1750 const Register& rt, \
1751 const MemOperand& src) { \
1752 VIXL_ASSERT(CPUHas(CPUFeatures::kAtomics)); \
1753 VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0)); \
1754 AtomicMemoryOp op = rt.Is64Bits() ? OP_X : OP_W; \
1755 Emit(op | Rs(rs) | Rt(rt) | RnSP(src.GetBaseRegister())); \
1756 }
1757 #define DEFINE_ASM_STORE_FUNC(FN, OP_X, OP_W) \
1758 void Assembler::st##FN(const Register& rs, const MemOperand& src) { \
1759 VIXL_ASSERT(CPUHas(CPUFeatures::kAtomics)); \
1760 ld##FN(rs, AppropriateZeroRegFor(rs), src); \
1761 }
1762
1763 ATOMIC_MEMORY_SIMPLE_OPERATION_LIST(ATOMIC_MEMORY_LOAD_MODES,
1764 DEFINE_ASM_LOAD_FUNC)
1765 ATOMIC_MEMORY_SIMPLE_OPERATION_LIST(ATOMIC_MEMORY_STORE_MODES,
1766 DEFINE_ASM_STORE_FUNC)
1767
1768 #define DEFINE_ASM_SWP_FUNC(FN, OP_X, OP_W) \
1769 void Assembler::FN(const Register& rs, \
1770 const Register& rt, \
1771 const MemOperand& src) { \
1772 VIXL_ASSERT(CPUHas(CPUFeatures::kAtomics)); \
1773 VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0)); \
1774 AtomicMemoryOp op = rt.Is64Bits() ? OP_X : OP_W; \
1775 Emit(op | Rs(rs) | Rt(rt) | RnSP(src.GetBaseRegister())); \
1776 }
1777
1778 ATOMIC_MEMORY_LOAD_MODES(DEFINE_ASM_SWP_FUNC, swp, SWP)
1779
1780 #undef DEFINE_ASM_LOAD_FUNC
1781 #undef DEFINE_ASM_STORE_FUNC
1782 #undef DEFINE_ASM_SWP_FUNC
1783
1784
1785 void Assembler::ldaprb(const Register& rt, const MemOperand& src) {
1786 VIXL_ASSERT(CPUHas(CPUFeatures::kRCpc));
1787 VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1788 AtomicMemoryOp op = LDAPRB;
1789 Emit(op | Rs(xzr) | Rt(rt) | RnSP(src.GetBaseRegister()));
1790 }
1791
ldapurb(const Register & rt,const MemOperand & src)1792 void Assembler::ldapurb(const Register& rt, const MemOperand& src) {
1793 VIXL_ASSERT(CPUHas(CPUFeatures::kRCpc, CPUFeatures::kRCpcImm));
1794 VIXL_ASSERT(src.IsImmediateOffset() && IsImmLSUnscaled(src.GetOffset()));
1795
1796 Instr base = RnSP(src.GetBaseRegister());
1797 int64_t offset = src.GetOffset();
1798 Emit(LDAPURB | Rt(rt) | base | ImmLS(static_cast<int>(offset)));
1799 }
1800
ldapursb(const Register & rt,const MemOperand & src)1801 void Assembler::ldapursb(const Register& rt, const MemOperand& src) {
1802 VIXL_ASSERT(CPUHas(CPUFeatures::kRCpc, CPUFeatures::kRCpcImm));
1803 VIXL_ASSERT(src.IsImmediateOffset() && IsImmLSUnscaled(src.GetOffset()));
1804
1805 Instr base = RnSP(src.GetBaseRegister());
1806 int64_t offset = src.GetOffset();
1807 Instr op = rt.Is64Bits() ? LDAPURSB_x : LDAPURSB_w;
1808 Emit(op | Rt(rt) | base | ImmLS(static_cast<int>(offset)));
1809 }
1810
ldaprh(const Register & rt,const MemOperand & src)1811 void Assembler::ldaprh(const Register& rt, const MemOperand& src) {
1812 VIXL_ASSERT(CPUHas(CPUFeatures::kRCpc));
1813 VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1814 AtomicMemoryOp op = LDAPRH;
1815 Emit(op | Rs(xzr) | Rt(rt) | RnSP(src.GetBaseRegister()));
1816 }
1817
ldapurh(const Register & rt,const MemOperand & src)1818 void Assembler::ldapurh(const Register& rt, const MemOperand& src) {
1819 VIXL_ASSERT(CPUHas(CPUFeatures::kRCpc, CPUFeatures::kRCpcImm));
1820 VIXL_ASSERT(src.IsImmediateOffset() && IsImmLSUnscaled(src.GetOffset()));
1821
1822 Instr base = RnSP(src.GetBaseRegister());
1823 int64_t offset = src.GetOffset();
1824 Emit(LDAPURH | Rt(rt) | base | ImmLS(static_cast<int>(offset)));
1825 }
1826
ldapursh(const Register & rt,const MemOperand & src)1827 void Assembler::ldapursh(const Register& rt, const MemOperand& src) {
1828 VIXL_ASSERT(CPUHas(CPUFeatures::kRCpc, CPUFeatures::kRCpcImm));
1829 VIXL_ASSERT(src.IsImmediateOffset() && IsImmLSUnscaled(src.GetOffset()));
1830
1831 Instr base = RnSP(src.GetBaseRegister());
1832 int64_t offset = src.GetOffset();
1833 LoadStoreRCpcUnscaledOffsetOp op = rt.Is64Bits() ? LDAPURSH_x : LDAPURSH_w;
1834 Emit(op | Rt(rt) | base | ImmLS(static_cast<int>(offset)));
1835 }
1836
ldapr(const Register & rt,const MemOperand & src)1837 void Assembler::ldapr(const Register& rt, const MemOperand& src) {
1838 VIXL_ASSERT(CPUHas(CPUFeatures::kRCpc));
1839 VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1840 AtomicMemoryOp op = rt.Is64Bits() ? LDAPR_x : LDAPR_w;
1841 Emit(op | Rs(xzr) | Rt(rt) | RnSP(src.GetBaseRegister()));
1842 }
1843
ldapur(const Register & rt,const MemOperand & src)1844 void Assembler::ldapur(const Register& rt, const MemOperand& src) {
1845 VIXL_ASSERT(CPUHas(CPUFeatures::kRCpc, CPUFeatures::kRCpcImm));
1846 VIXL_ASSERT(src.IsImmediateOffset() && IsImmLSUnscaled(src.GetOffset()));
1847
1848 Instr base = RnSP(src.GetBaseRegister());
1849 int64_t offset = src.GetOffset();
1850 LoadStoreRCpcUnscaledOffsetOp op = rt.Is64Bits() ? LDAPUR_x : LDAPUR_w;
1851 Emit(op | Rt(rt) | base | ImmLS(static_cast<int>(offset)));
1852 }
1853
ldapursw(const Register & rt,const MemOperand & src)1854 void Assembler::ldapursw(const Register& rt, const MemOperand& src) {
1855 VIXL_ASSERT(CPUHas(CPUFeatures::kRCpc, CPUFeatures::kRCpcImm));
1856 VIXL_ASSERT(rt.Is64Bits());
1857 VIXL_ASSERT(src.IsImmediateOffset() && IsImmLSUnscaled(src.GetOffset()));
1858
1859 Instr base = RnSP(src.GetBaseRegister());
1860 int64_t offset = src.GetOffset();
1861 Emit(LDAPURSW | Rt(rt) | base | ImmLS(static_cast<int>(offset)));
1862 }
1863
prfm(int op,const MemOperand & address,LoadStoreScalingOption option)1864 void Assembler::prfm(int op,
1865 const MemOperand& address,
1866 LoadStoreScalingOption option) {
1867 VIXL_ASSERT(option != RequireUnscaledOffset);
1868 VIXL_ASSERT(option != PreferUnscaledOffset);
1869 Prefetch(op, address, option);
1870 }
1871
prfm(PrefetchOperation op,const MemOperand & address,LoadStoreScalingOption option)1872 void Assembler::prfm(PrefetchOperation op,
1873 const MemOperand& address,
1874 LoadStoreScalingOption option) {
1875 // Passing unnamed values in 'op' is undefined behaviour in C++.
1876 VIXL_ASSERT(IsNamedPrefetchOperation(op));
1877 prfm(static_cast<int>(op), address, option);
1878 }
1879
1880
prfum(int op,const MemOperand & address,LoadStoreScalingOption option)1881 void Assembler::prfum(int op,
1882 const MemOperand& address,
1883 LoadStoreScalingOption option) {
1884 VIXL_ASSERT(option != RequireScaledOffset);
1885 VIXL_ASSERT(option != PreferScaledOffset);
1886 Prefetch(op, address, option);
1887 }
1888
prfum(PrefetchOperation op,const MemOperand & address,LoadStoreScalingOption option)1889 void Assembler::prfum(PrefetchOperation op,
1890 const MemOperand& address,
1891 LoadStoreScalingOption option) {
1892 // Passing unnamed values in 'op' is undefined behaviour in C++.
1893 VIXL_ASSERT(IsNamedPrefetchOperation(op));
1894 prfum(static_cast<int>(op), address, option);
1895 }
1896
1897
prfm(int op,RawLiteral * literal)1898 void Assembler::prfm(int op, RawLiteral* literal) {
1899 prfm(op, static_cast<int>(LinkAndGetWordOffsetTo(literal)));
1900 }
1901
prfm(PrefetchOperation op,RawLiteral * literal)1902 void Assembler::prfm(PrefetchOperation op, RawLiteral* literal) {
1903 // Passing unnamed values in 'op' is undefined behaviour in C++.
1904 VIXL_ASSERT(IsNamedPrefetchOperation(op));
1905 prfm(static_cast<int>(op), literal);
1906 }
1907
1908
sys(int op1,int crn,int crm,int op2,const Register & xt)1909 void Assembler::sys(int op1, int crn, int crm, int op2, const Register& xt) {
1910 VIXL_ASSERT(xt.Is64Bits());
1911 Emit(SYS | ImmSysOp1(op1) | CRn(crn) | CRm(crm) | ImmSysOp2(op2) | Rt(xt));
1912 }
1913
1914
sys(int op,const Register & xt)1915 void Assembler::sys(int op, const Register& xt) {
1916 VIXL_ASSERT(xt.Is64Bits());
1917 Emit(SYS | SysOp(op) | Rt(xt));
1918 }
1919
1920
sysl(int op,const Register & xt)1921 void Assembler::sysl(int op, const Register& xt) {
1922 VIXL_ASSERT(xt.Is64Bits());
1923 Emit(SYSL | SysOp(op) | Rt(xt));
1924 }
1925
1926
dc(DataCacheOp op,const Register & rt)1927 void Assembler::dc(DataCacheOp op, const Register& rt) {
1928 if (op == CVAP) VIXL_ASSERT(CPUHas(CPUFeatures::kDCPoP));
1929 if (op == CVADP) VIXL_ASSERT(CPUHas(CPUFeatures::kDCCVADP));
1930 sys(op, rt);
1931 }
1932
1933
ic(InstructionCacheOp op,const Register & rt)1934 void Assembler::ic(InstructionCacheOp op, const Register& rt) {
1935 VIXL_ASSERT(op == IVAU);
1936 sys(op, rt);
1937 }
1938
gcspushm(const Register & rt)1939 void Assembler::gcspushm(const Register& rt) {
1940 VIXL_ASSERT(CPUHas(CPUFeatures::kGCS));
1941 sys(GCSPUSHM, rt);
1942 }
1943
gcspopm(const Register & rt)1944 void Assembler::gcspopm(const Register& rt) {
1945 VIXL_ASSERT(CPUHas(CPUFeatures::kGCS));
1946 sysl(GCSPOPM, rt);
1947 }
1948
1949
gcsss1(const Register & rt)1950 void Assembler::gcsss1(const Register& rt) {
1951 VIXL_ASSERT(CPUHas(CPUFeatures::kGCS));
1952 sys(GCSSS1, rt);
1953 }
1954
1955
gcsss2(const Register & rt)1956 void Assembler::gcsss2(const Register& rt) {
1957 VIXL_ASSERT(CPUHas(CPUFeatures::kGCS));
1958 sysl(GCSSS2, rt);
1959 }
1960
1961
chkfeat(const Register & rd)1962 void Assembler::chkfeat(const Register& rd) {
1963 VIXL_ASSERT(rd.Is(x16));
1964 USE(rd);
1965 hint(CHKFEAT);
1966 }
1967
1968
hint(SystemHint code)1969 void Assembler::hint(SystemHint code) { hint(static_cast<int>(code)); }
1970
1971
hint(int imm7)1972 void Assembler::hint(int imm7) {
1973 VIXL_ASSERT(IsUint7(imm7));
1974 Emit(HINT | ImmHint(imm7) | Rt(xzr));
1975 }
1976
1977
1978 // MTE.
1979
addg(const Register & xd,const Register & xn,int offset,int tag_offset)1980 void Assembler::addg(const Register& xd,
1981 const Register& xn,
1982 int offset,
1983 int tag_offset) {
1984 VIXL_ASSERT(CPUHas(CPUFeatures::kMTE));
1985 VIXL_ASSERT(IsMultiple(offset, kMTETagGranuleInBytes));
1986
1987 Emit(0x91800000 | RdSP(xd) | RnSP(xn) |
1988 ImmUnsignedField<21, 16>(offset / kMTETagGranuleInBytes) |
1989 ImmUnsignedField<13, 10>(tag_offset));
1990 }
1991
gmi(const Register & xd,const Register & xn,const Register & xm)1992 void Assembler::gmi(const Register& xd,
1993 const Register& xn,
1994 const Register& xm) {
1995 VIXL_ASSERT(CPUHas(CPUFeatures::kMTE));
1996
1997 Emit(0x9ac01400 | Rd(xd) | RnSP(xn) | Rm(xm));
1998 }
1999
irg(const Register & xd,const Register & xn,const Register & xm)2000 void Assembler::irg(const Register& xd,
2001 const Register& xn,
2002 const Register& xm) {
2003 VIXL_ASSERT(CPUHas(CPUFeatures::kMTE));
2004
2005 Emit(0x9ac01000 | RdSP(xd) | RnSP(xn) | Rm(xm));
2006 }
2007
ldg(const Register & xt,const MemOperand & addr)2008 void Assembler::ldg(const Register& xt, const MemOperand& addr) {
2009 VIXL_ASSERT(CPUHas(CPUFeatures::kMTE));
2010 VIXL_ASSERT(addr.IsImmediateOffset());
2011 int offset = static_cast<int>(addr.GetOffset());
2012 VIXL_ASSERT(IsMultiple(offset, kMTETagGranuleInBytes));
2013
2014 Emit(0xd9600000 | Rt(xt) | RnSP(addr.GetBaseRegister()) |
2015 ImmField<20, 12>(offset / static_cast<int>(kMTETagGranuleInBytes)));
2016 }
2017
StoreTagHelper(const Register & xt,const MemOperand & addr,Instr op)2018 void Assembler::StoreTagHelper(const Register& xt,
2019 const MemOperand& addr,
2020 Instr op) {
2021 int offset = static_cast<int>(addr.GetOffset());
2022 VIXL_ASSERT(IsMultiple(offset, kMTETagGranuleInBytes));
2023
2024 Instr addr_mode;
2025 if (addr.IsImmediateOffset()) {
2026 addr_mode = 2;
2027 } else if (addr.IsImmediatePreIndex()) {
2028 addr_mode = 3;
2029 } else {
2030 VIXL_ASSERT(addr.IsImmediatePostIndex());
2031 addr_mode = 1;
2032 }
2033
2034 Emit(op | RdSP(xt) | RnSP(addr.GetBaseRegister()) | (addr_mode << 10) |
2035 ImmField<20, 12>(offset / static_cast<int>(kMTETagGranuleInBytes)));
2036 }
2037
st2g(const Register & xt,const MemOperand & addr)2038 void Assembler::st2g(const Register& xt, const MemOperand& addr) {
2039 VIXL_ASSERT(CPUHas(CPUFeatures::kMTE));
2040 StoreTagHelper(xt, addr, 0xd9a00000);
2041 }
2042
stg(const Register & xt,const MemOperand & addr)2043 void Assembler::stg(const Register& xt, const MemOperand& addr) {
2044 VIXL_ASSERT(CPUHas(CPUFeatures::kMTE));
2045 StoreTagHelper(xt, addr, 0xd9200000);
2046 }
2047
stgp(const Register & xt1,const Register & xt2,const MemOperand & addr)2048 void Assembler::stgp(const Register& xt1,
2049 const Register& xt2,
2050 const MemOperand& addr) {
2051 VIXL_ASSERT(CPUHas(CPUFeatures::kMTE));
2052 int offset = static_cast<int>(addr.GetOffset());
2053 VIXL_ASSERT(IsMultiple(offset, kMTETagGranuleInBytes));
2054
2055 Instr addr_mode;
2056 if (addr.IsImmediateOffset()) {
2057 addr_mode = 2;
2058 } else if (addr.IsImmediatePreIndex()) {
2059 addr_mode = 3;
2060 } else {
2061 VIXL_ASSERT(addr.IsImmediatePostIndex());
2062 addr_mode = 1;
2063 }
2064
2065 Emit(0x68000000 | RnSP(addr.GetBaseRegister()) | (addr_mode << 23) |
2066 ImmField<21, 15>(offset / static_cast<int>(kMTETagGranuleInBytes)) |
2067 Rt2(xt2) | Rt(xt1));
2068 }
2069
stz2g(const Register & xt,const MemOperand & addr)2070 void Assembler::stz2g(const Register& xt, const MemOperand& addr) {
2071 VIXL_ASSERT(CPUHas(CPUFeatures::kMTE));
2072 StoreTagHelper(xt, addr, 0xd9e00000);
2073 }
2074
stzg(const Register & xt,const MemOperand & addr)2075 void Assembler::stzg(const Register& xt, const MemOperand& addr) {
2076 VIXL_ASSERT(CPUHas(CPUFeatures::kMTE));
2077 StoreTagHelper(xt, addr, 0xd9600000);
2078 }
2079
subg(const Register & xd,const Register & xn,int offset,int tag_offset)2080 void Assembler::subg(const Register& xd,
2081 const Register& xn,
2082 int offset,
2083 int tag_offset) {
2084 VIXL_ASSERT(CPUHas(CPUFeatures::kMTE));
2085 VIXL_ASSERT(IsMultiple(offset, kMTETagGranuleInBytes));
2086
2087 Emit(0xd1800000 | RdSP(xd) | RnSP(xn) |
2088 ImmUnsignedField<21, 16>(offset / kMTETagGranuleInBytes) |
2089 ImmUnsignedField<13, 10>(tag_offset));
2090 }
2091
subp(const Register & xd,const Register & xn,const Register & xm)2092 void Assembler::subp(const Register& xd,
2093 const Register& xn,
2094 const Register& xm) {
2095 VIXL_ASSERT(CPUHas(CPUFeatures::kMTE));
2096
2097 Emit(0x9ac00000 | Rd(xd) | RnSP(xn) | RmSP(xm));
2098 }
2099
subps(const Register & xd,const Register & xn,const Register & xm)2100 void Assembler::subps(const Register& xd,
2101 const Register& xn,
2102 const Register& xm) {
2103 VIXL_ASSERT(CPUHas(CPUFeatures::kMTE));
2104
2105 Emit(0xbac00000 | Rd(xd) | RnSP(xn) | RmSP(xm));
2106 }
2107
cpye(const Register & rd,const Register & rs,const Register & rn)2108 void Assembler::cpye(const Register& rd,
2109 const Register& rs,
2110 const Register& rn) {
2111 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2112 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2113 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero() && !rs.IsZero());
2114
2115 Emit(0x1d800400 | Rd(rd) | Rn(rn) | Rs(rs));
2116 }
2117
cpyen(const Register & rd,const Register & rs,const Register & rn)2118 void Assembler::cpyen(const Register& rd,
2119 const Register& rs,
2120 const Register& rn) {
2121 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2122 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2123 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero() && !rs.IsZero());
2124
2125 Emit(0x1d80c400 | Rd(rd) | Rn(rn) | Rs(rs));
2126 }
2127
cpyern(const Register & rd,const Register & rs,const Register & rn)2128 void Assembler::cpyern(const Register& rd,
2129 const Register& rs,
2130 const Register& rn) {
2131 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2132 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2133 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero() && !rs.IsZero());
2134
2135 Emit(0x1d808400 | Rd(rd) | Rn(rn) | Rs(rs));
2136 }
2137
cpyewn(const Register & rd,const Register & rs,const Register & rn)2138 void Assembler::cpyewn(const Register& rd,
2139 const Register& rs,
2140 const Register& rn) {
2141 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2142 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2143 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero() && !rs.IsZero());
2144
2145 Emit(0x1d804400 | Rd(rd) | Rn(rn) | Rs(rs));
2146 }
2147
cpyfe(const Register & rd,const Register & rs,const Register & rn)2148 void Assembler::cpyfe(const Register& rd,
2149 const Register& rs,
2150 const Register& rn) {
2151 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2152 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2153 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero() && !rs.IsZero());
2154
2155 Emit(0x19800400 | Rd(rd) | Rn(rn) | Rs(rs));
2156 }
2157
cpyfen(const Register & rd,const Register & rs,const Register & rn)2158 void Assembler::cpyfen(const Register& rd,
2159 const Register& rs,
2160 const Register& rn) {
2161 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2162 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2163 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero() && !rs.IsZero());
2164
2165 Emit(0x1980c400 | Rd(rd) | Rn(rn) | Rs(rs));
2166 }
2167
cpyfern(const Register & rd,const Register & rs,const Register & rn)2168 void Assembler::cpyfern(const Register& rd,
2169 const Register& rs,
2170 const Register& rn) {
2171 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2172 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2173 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero() && !rs.IsZero());
2174
2175 Emit(0x19808400 | Rd(rd) | Rn(rn) | Rs(rs));
2176 }
2177
cpyfewn(const Register & rd,const Register & rs,const Register & rn)2178 void Assembler::cpyfewn(const Register& rd,
2179 const Register& rs,
2180 const Register& rn) {
2181 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2182 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2183 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero() && !rs.IsZero());
2184
2185 Emit(0x19804400 | Rd(rd) | Rn(rn) | Rs(rs));
2186 }
2187
cpyfm(const Register & rd,const Register & rs,const Register & rn)2188 void Assembler::cpyfm(const Register& rd,
2189 const Register& rs,
2190 const Register& rn) {
2191 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2192 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2193 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero() && !rs.IsZero());
2194
2195 Emit(0x19400400 | Rd(rd) | Rn(rn) | Rs(rs));
2196 }
2197
cpyfmn(const Register & rd,const Register & rs,const Register & rn)2198 void Assembler::cpyfmn(const Register& rd,
2199 const Register& rs,
2200 const Register& rn) {
2201 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2202 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2203 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero() && !rs.IsZero());
2204
2205 Emit(0x1940c400 | Rd(rd) | Rn(rn) | Rs(rs));
2206 }
2207
cpyfmrn(const Register & rd,const Register & rs,const Register & rn)2208 void Assembler::cpyfmrn(const Register& rd,
2209 const Register& rs,
2210 const Register& rn) {
2211 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2212 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2213 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero() && !rs.IsZero());
2214
2215 Emit(0x19408400 | Rd(rd) | Rn(rn) | Rs(rs));
2216 }
2217
cpyfmwn(const Register & rd,const Register & rs,const Register & rn)2218 void Assembler::cpyfmwn(const Register& rd,
2219 const Register& rs,
2220 const Register& rn) {
2221 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2222 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2223 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero() && !rs.IsZero());
2224
2225 Emit(0x19404400 | Rd(rd) | Rn(rn) | Rs(rs));
2226 }
2227
cpyfp(const Register & rd,const Register & rs,const Register & rn)2228 void Assembler::cpyfp(const Register& rd,
2229 const Register& rs,
2230 const Register& rn) {
2231 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2232 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2233 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero() && !rs.IsZero());
2234
2235 Emit(0x19000400 | Rd(rd) | Rn(rn) | Rs(rs));
2236 }
2237
cpyfpn(const Register & rd,const Register & rs,const Register & rn)2238 void Assembler::cpyfpn(const Register& rd,
2239 const Register& rs,
2240 const Register& rn) {
2241 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2242 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2243 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero() && !rs.IsZero());
2244
2245 Emit(0x1900c400 | Rd(rd) | Rn(rn) | Rs(rs));
2246 }
2247
cpyfprn(const Register & rd,const Register & rs,const Register & rn)2248 void Assembler::cpyfprn(const Register& rd,
2249 const Register& rs,
2250 const Register& rn) {
2251 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2252 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2253 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero() && !rs.IsZero());
2254
2255 Emit(0x19008400 | Rd(rd) | Rn(rn) | Rs(rs));
2256 }
2257
cpyfpwn(const Register & rd,const Register & rs,const Register & rn)2258 void Assembler::cpyfpwn(const Register& rd,
2259 const Register& rs,
2260 const Register& rn) {
2261 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2262 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2263 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero() && !rs.IsZero());
2264
2265 Emit(0x19004400 | Rd(rd) | Rn(rn) | Rs(rs));
2266 }
2267
cpym(const Register & rd,const Register & rs,const Register & rn)2268 void Assembler::cpym(const Register& rd,
2269 const Register& rs,
2270 const Register& rn) {
2271 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2272 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2273 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero() && !rs.IsZero());
2274
2275 Emit(0x1d400400 | Rd(rd) | Rn(rn) | Rs(rs));
2276 }
2277
cpymn(const Register & rd,const Register & rs,const Register & rn)2278 void Assembler::cpymn(const Register& rd,
2279 const Register& rs,
2280 const Register& rn) {
2281 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2282 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2283 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero() && !rs.IsZero());
2284
2285 Emit(0x1d40c400 | Rd(rd) | Rn(rn) | Rs(rs));
2286 }
2287
cpymrn(const Register & rd,const Register & rs,const Register & rn)2288 void Assembler::cpymrn(const Register& rd,
2289 const Register& rs,
2290 const Register& rn) {
2291 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2292 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2293 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero() && !rs.IsZero());
2294
2295 Emit(0x1d408400 | Rd(rd) | Rn(rn) | Rs(rs));
2296 }
2297
cpymwn(const Register & rd,const Register & rs,const Register & rn)2298 void Assembler::cpymwn(const Register& rd,
2299 const Register& rs,
2300 const Register& rn) {
2301 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2302 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2303 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero() && !rs.IsZero());
2304
2305 Emit(0x1d404400 | Rd(rd) | Rn(rn) | Rs(rs));
2306 }
2307
cpyp(const Register & rd,const Register & rs,const Register & rn)2308 void Assembler::cpyp(const Register& rd,
2309 const Register& rs,
2310 const Register& rn) {
2311 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2312 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2313 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero() && !rs.IsZero());
2314
2315 Emit(0x1d000400 | Rd(rd) | Rn(rn) | Rs(rs));
2316 }
2317
cpypn(const Register & rd,const Register & rs,const Register & rn)2318 void Assembler::cpypn(const Register& rd,
2319 const Register& rs,
2320 const Register& rn) {
2321 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2322 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2323 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero() && !rs.IsZero());
2324
2325 Emit(0x1d00c400 | Rd(rd) | Rn(rn) | Rs(rs));
2326 }
2327
cpyprn(const Register & rd,const Register & rs,const Register & rn)2328 void Assembler::cpyprn(const Register& rd,
2329 const Register& rs,
2330 const Register& rn) {
2331 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2332 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2333 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero() && !rs.IsZero());
2334
2335 Emit(0x1d008400 | Rd(rd) | Rn(rn) | Rs(rs));
2336 }
2337
cpypwn(const Register & rd,const Register & rs,const Register & rn)2338 void Assembler::cpypwn(const Register& rd,
2339 const Register& rs,
2340 const Register& rn) {
2341 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2342 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2343 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero() && !rs.IsZero());
2344
2345 Emit(0x1d004400 | Rd(rd) | Rn(rn) | Rs(rs));
2346 }
2347
sete(const Register & rd,const Register & rn,const Register & rs)2348 void Assembler::sete(const Register& rd,
2349 const Register& rn,
2350 const Register& rs) {
2351 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2352 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2353 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero());
2354
2355 Emit(0x19c08400 | Rd(rd) | Rn(rn) | Rs(rs));
2356 }
2357
seten(const Register & rd,const Register & rn,const Register & rs)2358 void Assembler::seten(const Register& rd,
2359 const Register& rn,
2360 const Register& rs) {
2361 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2362 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2363 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero());
2364
2365 Emit(0x19c0a400 | Rd(rd) | Rn(rn) | Rs(rs));
2366 }
2367
setge(const Register & rd,const Register & rn,const Register & rs)2368 void Assembler::setge(const Register& rd,
2369 const Register& rn,
2370 const Register& rs) {
2371 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2372 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2373 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero());
2374
2375 Emit(0x1dc08400 | Rd(rd) | Rn(rn) | Rs(rs));
2376 }
2377
setgen(const Register & rd,const Register & rn,const Register & rs)2378 void Assembler::setgen(const Register& rd,
2379 const Register& rn,
2380 const Register& rs) {
2381 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2382 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2383 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero());
2384
2385 Emit(0x1dc0a400 | Rd(rd) | Rn(rn) | Rs(rs));
2386 }
2387
setgm(const Register & rd,const Register & rn,const Register & rs)2388 void Assembler::setgm(const Register& rd,
2389 const Register& rn,
2390 const Register& rs) {
2391 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2392 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2393 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero());
2394
2395 Emit(0x1dc04400 | Rd(rd) | Rn(rn) | Rs(rs));
2396 }
2397
setgmn(const Register & rd,const Register & rn,const Register & rs)2398 void Assembler::setgmn(const Register& rd,
2399 const Register& rn,
2400 const Register& rs) {
2401 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2402 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2403 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero());
2404
2405 Emit(0x1dc06400 | Rd(rd) | Rn(rn) | Rs(rs));
2406 }
2407
setgp(const Register & rd,const Register & rn,const Register & rs)2408 void Assembler::setgp(const Register& rd,
2409 const Register& rn,
2410 const Register& rs) {
2411 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2412 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2413 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero());
2414
2415 Emit(0x1dc00400 | Rd(rd) | Rn(rn) | Rs(rs));
2416 }
2417
setgpn(const Register & rd,const Register & rn,const Register & rs)2418 void Assembler::setgpn(const Register& rd,
2419 const Register& rn,
2420 const Register& rs) {
2421 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2422 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2423 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero());
2424
2425 Emit(0x1dc02400 | Rd(rd) | Rn(rn) | Rs(rs));
2426 }
2427
setm(const Register & rd,const Register & rn,const Register & rs)2428 void Assembler::setm(const Register& rd,
2429 const Register& rn,
2430 const Register& rs) {
2431 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2432 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2433 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero());
2434
2435 Emit(0x19c04400 | Rd(rd) | Rn(rn) | Rs(rs));
2436 }
2437
setmn(const Register & rd,const Register & rn,const Register & rs)2438 void Assembler::setmn(const Register& rd,
2439 const Register& rn,
2440 const Register& rs) {
2441 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2442 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2443 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero());
2444
2445 Emit(0x19c06400 | Rd(rd) | Rn(rn) | Rs(rs));
2446 }
2447
setp(const Register & rd,const Register & rn,const Register & rs)2448 void Assembler::setp(const Register& rd,
2449 const Register& rn,
2450 const Register& rs) {
2451 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2452 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2453 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero());
2454
2455 Emit(0x19c00400 | Rd(rd) | Rn(rn) | Rs(rs));
2456 }
2457
setpn(const Register & rd,const Register & rn,const Register & rs)2458 void Assembler::setpn(const Register& rd,
2459 const Register& rn,
2460 const Register& rs) {
2461 VIXL_ASSERT(CPUHas(CPUFeatures::kMOPS));
2462 VIXL_ASSERT(!AreAliased(rd, rn, rs));
2463 VIXL_ASSERT(!rd.IsZero() && !rn.IsZero());
2464
2465 Emit(0x19c02400 | Rd(rd) | Rn(rn) | Rs(rs));
2466 }
2467
abs(const Register & rd,const Register & rn)2468 void Assembler::abs(const Register& rd, const Register& rn) {
2469 VIXL_ASSERT(CPUHas(CPUFeatures::kCSSC));
2470 VIXL_ASSERT(rd.IsSameSizeAndType(rn));
2471
2472 Emit(0x5ac02000 | SF(rd) | Rd(rd) | Rn(rn));
2473 }
2474
cnt(const Register & rd,const Register & rn)2475 void Assembler::cnt(const Register& rd, const Register& rn) {
2476 VIXL_ASSERT(CPUHas(CPUFeatures::kCSSC));
2477 VIXL_ASSERT(rd.IsSameSizeAndType(rn));
2478
2479 Emit(0x5ac01c00 | SF(rd) | Rd(rd) | Rn(rn));
2480 }
2481
ctz(const Register & rd,const Register & rn)2482 void Assembler::ctz(const Register& rd, const Register& rn) {
2483 VIXL_ASSERT(CPUHas(CPUFeatures::kCSSC));
2484 VIXL_ASSERT(rd.IsSameSizeAndType(rn));
2485
2486 Emit(0x5ac01800 | SF(rd) | Rd(rd) | Rn(rn));
2487 }
2488
2489 #define MINMAX(V) \
2490 V(smax, 0x11c00000, 0x1ac06000, true) \
2491 V(smin, 0x11c80000, 0x1ac06800, true) \
2492 V(umax, 0x11c40000, 0x1ac06400, false) \
2493 V(umin, 0x11cc0000, 0x1ac06c00, false)
2494
2495 #define VIXL_DEFINE_ASM_FUNC(FN, IMMOP, REGOP, SIGNED) \
2496 void Assembler::FN(const Register& rd, \
2497 const Register& rn, \
2498 const Operand& op) { \
2499 VIXL_ASSERT(rd.IsSameSizeAndType(rn)); \
2500 Instr i = SF(rd) | Rd(rd) | Rn(rn); \
2501 if (op.IsImmediate()) { \
2502 int64_t imm = op.GetImmediate(); \
2503 i |= SIGNED ? ImmField<17, 10>(imm) : ImmUnsignedField<17, 10>(imm); \
2504 Emit(IMMOP | i); \
2505 } else { \
2506 VIXL_ASSERT(op.IsPlainRegister()); \
2507 VIXL_ASSERT(op.GetRegister().IsSameSizeAndType(rd)); \
2508 Emit(REGOP | i | Rm(op.GetRegister())); \
2509 } \
2510 }
MINMAX(VIXL_DEFINE_ASM_FUNC)2511 MINMAX(VIXL_DEFINE_ASM_FUNC)
2512 #undef VIXL_DEFINE_ASM_FUNC
2513
2514 // NEON structure loads and stores.
2515 Instr Assembler::LoadStoreStructAddrModeField(const MemOperand& addr) {
2516 Instr addr_field = RnSP(addr.GetBaseRegister());
2517
2518 if (addr.IsPostIndex()) {
2519 VIXL_STATIC_ASSERT(NEONLoadStoreMultiStructPostIndex ==
2520 static_cast<NEONLoadStoreMultiStructPostIndexOp>(
2521 NEONLoadStoreSingleStructPostIndex));
2522
2523 addr_field |= NEONLoadStoreMultiStructPostIndex;
2524 if (addr.GetOffset() == 0) {
2525 addr_field |= RmNot31(addr.GetRegisterOffset());
2526 } else {
2527 // The immediate post index addressing mode is indicated by rm = 31.
2528 // The immediate is implied by the number of vector registers used.
2529 addr_field |= (0x1f << Rm_offset);
2530 }
2531 } else {
2532 VIXL_ASSERT(addr.IsImmediateOffset() && (addr.GetOffset() == 0));
2533 }
2534 return addr_field;
2535 }
2536
LoadStoreStructVerify(const VRegister & vt,const MemOperand & addr,Instr op)2537 void Assembler::LoadStoreStructVerify(const VRegister& vt,
2538 const MemOperand& addr,
2539 Instr op) {
2540 #ifdef VIXL_DEBUG
2541 // Assert that addressing mode is either offset (with immediate 0), post
2542 // index by immediate of the size of the register list, or post index by a
2543 // value in a core register.
2544 VIXL_ASSERT(vt.HasSize() && vt.HasLaneSize());
2545 if (addr.IsImmediateOffset()) {
2546 VIXL_ASSERT(addr.GetOffset() == 0);
2547 } else {
2548 int offset = vt.GetSizeInBytes();
2549 switch (op) {
2550 case NEON_LD1_1v:
2551 case NEON_ST1_1v:
2552 offset *= 1;
2553 break;
2554 case NEONLoadStoreSingleStructLoad1:
2555 case NEONLoadStoreSingleStructStore1:
2556 case NEON_LD1R:
2557 offset = (offset / vt.GetLanes()) * 1;
2558 break;
2559
2560 case NEON_LD1_2v:
2561 case NEON_ST1_2v:
2562 case NEON_LD2:
2563 case NEON_ST2:
2564 offset *= 2;
2565 break;
2566 case NEONLoadStoreSingleStructLoad2:
2567 case NEONLoadStoreSingleStructStore2:
2568 case NEON_LD2R:
2569 offset = (offset / vt.GetLanes()) * 2;
2570 break;
2571
2572 case NEON_LD1_3v:
2573 case NEON_ST1_3v:
2574 case NEON_LD3:
2575 case NEON_ST3:
2576 offset *= 3;
2577 break;
2578 case NEONLoadStoreSingleStructLoad3:
2579 case NEONLoadStoreSingleStructStore3:
2580 case NEON_LD3R:
2581 offset = (offset / vt.GetLanes()) * 3;
2582 break;
2583
2584 case NEON_LD1_4v:
2585 case NEON_ST1_4v:
2586 case NEON_LD4:
2587 case NEON_ST4:
2588 offset *= 4;
2589 break;
2590 case NEONLoadStoreSingleStructLoad4:
2591 case NEONLoadStoreSingleStructStore4:
2592 case NEON_LD4R:
2593 offset = (offset / vt.GetLanes()) * 4;
2594 break;
2595 default:
2596 VIXL_UNREACHABLE();
2597 }
2598 VIXL_ASSERT(!addr.GetRegisterOffset().Is(NoReg) ||
2599 addr.GetOffset() == offset);
2600 }
2601 #else
2602 USE(vt, addr, op);
2603 #endif
2604 }
2605
LoadStoreStruct(const VRegister & vt,const MemOperand & addr,NEONLoadStoreMultiStructOp op)2606 void Assembler::LoadStoreStruct(const VRegister& vt,
2607 const MemOperand& addr,
2608 NEONLoadStoreMultiStructOp op) {
2609 LoadStoreStructVerify(vt, addr, op);
2610 VIXL_ASSERT(vt.IsVector() || vt.Is1D());
2611 Emit(op | LoadStoreStructAddrModeField(addr) | LSVFormat(vt) | Rt(vt));
2612 }
2613
2614
LoadStoreStructSingleAllLanes(const VRegister & vt,const MemOperand & addr,NEONLoadStoreSingleStructOp op)2615 void Assembler::LoadStoreStructSingleAllLanes(const VRegister& vt,
2616 const MemOperand& addr,
2617 NEONLoadStoreSingleStructOp op) {
2618 LoadStoreStructVerify(vt, addr, op);
2619 Emit(op | LoadStoreStructAddrModeField(addr) | LSVFormat(vt) | Rt(vt));
2620 }
2621
2622
ld1(const VRegister & vt,const MemOperand & src)2623 void Assembler::ld1(const VRegister& vt, const MemOperand& src) {
2624 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2625 LoadStoreStruct(vt, src, NEON_LD1_1v);
2626 }
2627
2628
ld1(const VRegister & vt,const VRegister & vt2,const MemOperand & src)2629 void Assembler::ld1(const VRegister& vt,
2630 const VRegister& vt2,
2631 const MemOperand& src) {
2632 USE(vt2);
2633 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2634 VIXL_ASSERT(AreSameFormat(vt, vt2));
2635 VIXL_ASSERT(AreConsecutive(vt, vt2));
2636 LoadStoreStruct(vt, src, NEON_LD1_2v);
2637 }
2638
2639
ld1(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const MemOperand & src)2640 void Assembler::ld1(const VRegister& vt,
2641 const VRegister& vt2,
2642 const VRegister& vt3,
2643 const MemOperand& src) {
2644 USE(vt2, vt3);
2645 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2646 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
2647 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
2648 LoadStoreStruct(vt, src, NEON_LD1_3v);
2649 }
2650
2651
ld1(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const VRegister & vt4,const MemOperand & src)2652 void Assembler::ld1(const VRegister& vt,
2653 const VRegister& vt2,
2654 const VRegister& vt3,
2655 const VRegister& vt4,
2656 const MemOperand& src) {
2657 USE(vt2, vt3, vt4);
2658 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2659 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
2660 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
2661 LoadStoreStruct(vt, src, NEON_LD1_4v);
2662 }
2663
2664
ld2(const VRegister & vt,const VRegister & vt2,const MemOperand & src)2665 void Assembler::ld2(const VRegister& vt,
2666 const VRegister& vt2,
2667 const MemOperand& src) {
2668 USE(vt2);
2669 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2670 VIXL_ASSERT(AreSameFormat(vt, vt2));
2671 VIXL_ASSERT(AreConsecutive(vt, vt2));
2672 LoadStoreStruct(vt, src, NEON_LD2);
2673 }
2674
2675
ld2(const VRegister & vt,const VRegister & vt2,int lane,const MemOperand & src)2676 void Assembler::ld2(const VRegister& vt,
2677 const VRegister& vt2,
2678 int lane,
2679 const MemOperand& src) {
2680 USE(vt2);
2681 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2682 VIXL_ASSERT(AreSameFormat(vt, vt2));
2683 VIXL_ASSERT(AreConsecutive(vt, vt2));
2684 LoadStoreStructSingle(vt, lane, src, NEONLoadStoreSingleStructLoad2);
2685 }
2686
2687
ld2r(const VRegister & vt,const VRegister & vt2,const MemOperand & src)2688 void Assembler::ld2r(const VRegister& vt,
2689 const VRegister& vt2,
2690 const MemOperand& src) {
2691 USE(vt2);
2692 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2693 VIXL_ASSERT(AreSameFormat(vt, vt2));
2694 VIXL_ASSERT(AreConsecutive(vt, vt2));
2695 LoadStoreStructSingleAllLanes(vt, src, NEON_LD2R);
2696 }
2697
2698
ld3(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const MemOperand & src)2699 void Assembler::ld3(const VRegister& vt,
2700 const VRegister& vt2,
2701 const VRegister& vt3,
2702 const MemOperand& src) {
2703 USE(vt2, vt3);
2704 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2705 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
2706 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
2707 LoadStoreStruct(vt, src, NEON_LD3);
2708 }
2709
2710
ld3(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,int lane,const MemOperand & src)2711 void Assembler::ld3(const VRegister& vt,
2712 const VRegister& vt2,
2713 const VRegister& vt3,
2714 int lane,
2715 const MemOperand& src) {
2716 USE(vt2, vt3);
2717 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2718 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
2719 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
2720 LoadStoreStructSingle(vt, lane, src, NEONLoadStoreSingleStructLoad3);
2721 }
2722
2723
ld3r(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const MemOperand & src)2724 void Assembler::ld3r(const VRegister& vt,
2725 const VRegister& vt2,
2726 const VRegister& vt3,
2727 const MemOperand& src) {
2728 USE(vt2, vt3);
2729 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2730 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
2731 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
2732 LoadStoreStructSingleAllLanes(vt, src, NEON_LD3R);
2733 }
2734
2735
ld4(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const VRegister & vt4,const MemOperand & src)2736 void Assembler::ld4(const VRegister& vt,
2737 const VRegister& vt2,
2738 const VRegister& vt3,
2739 const VRegister& vt4,
2740 const MemOperand& src) {
2741 USE(vt2, vt3, vt4);
2742 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2743 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
2744 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
2745 LoadStoreStruct(vt, src, NEON_LD4);
2746 }
2747
2748
ld4(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const VRegister & vt4,int lane,const MemOperand & src)2749 void Assembler::ld4(const VRegister& vt,
2750 const VRegister& vt2,
2751 const VRegister& vt3,
2752 const VRegister& vt4,
2753 int lane,
2754 const MemOperand& src) {
2755 USE(vt2, vt3, vt4);
2756 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2757 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
2758 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
2759 LoadStoreStructSingle(vt, lane, src, NEONLoadStoreSingleStructLoad4);
2760 }
2761
2762
ld4r(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const VRegister & vt4,const MemOperand & src)2763 void Assembler::ld4r(const VRegister& vt,
2764 const VRegister& vt2,
2765 const VRegister& vt3,
2766 const VRegister& vt4,
2767 const MemOperand& src) {
2768 USE(vt2, vt3, vt4);
2769 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2770 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
2771 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
2772 LoadStoreStructSingleAllLanes(vt, src, NEON_LD4R);
2773 }
2774
2775
st1(const VRegister & vt,const MemOperand & src)2776 void Assembler::st1(const VRegister& vt, const MemOperand& src) {
2777 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2778 LoadStoreStruct(vt, src, NEON_ST1_1v);
2779 }
2780
2781
st1(const VRegister & vt,const VRegister & vt2,const MemOperand & src)2782 void Assembler::st1(const VRegister& vt,
2783 const VRegister& vt2,
2784 const MemOperand& src) {
2785 USE(vt2);
2786 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2787 VIXL_ASSERT(AreSameFormat(vt, vt2));
2788 VIXL_ASSERT(AreConsecutive(vt, vt2));
2789 LoadStoreStruct(vt, src, NEON_ST1_2v);
2790 }
2791
2792
st1(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const MemOperand & src)2793 void Assembler::st1(const VRegister& vt,
2794 const VRegister& vt2,
2795 const VRegister& vt3,
2796 const MemOperand& src) {
2797 USE(vt2, vt3);
2798 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2799 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
2800 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
2801 LoadStoreStruct(vt, src, NEON_ST1_3v);
2802 }
2803
2804
st1(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const VRegister & vt4,const MemOperand & src)2805 void Assembler::st1(const VRegister& vt,
2806 const VRegister& vt2,
2807 const VRegister& vt3,
2808 const VRegister& vt4,
2809 const MemOperand& src) {
2810 USE(vt2, vt3, vt4);
2811 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2812 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
2813 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
2814 LoadStoreStruct(vt, src, NEON_ST1_4v);
2815 }
2816
2817
st2(const VRegister & vt,const VRegister & vt2,const MemOperand & dst)2818 void Assembler::st2(const VRegister& vt,
2819 const VRegister& vt2,
2820 const MemOperand& dst) {
2821 USE(vt2);
2822 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2823 VIXL_ASSERT(AreSameFormat(vt, vt2));
2824 VIXL_ASSERT(AreConsecutive(vt, vt2));
2825 LoadStoreStruct(vt, dst, NEON_ST2);
2826 }
2827
2828
st2(const VRegister & vt,const VRegister & vt2,int lane,const MemOperand & dst)2829 void Assembler::st2(const VRegister& vt,
2830 const VRegister& vt2,
2831 int lane,
2832 const MemOperand& dst) {
2833 USE(vt2);
2834 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2835 VIXL_ASSERT(AreSameFormat(vt, vt2));
2836 VIXL_ASSERT(AreConsecutive(vt, vt2));
2837 LoadStoreStructSingle(vt, lane, dst, NEONLoadStoreSingleStructStore2);
2838 }
2839
2840
st3(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const MemOperand & dst)2841 void Assembler::st3(const VRegister& vt,
2842 const VRegister& vt2,
2843 const VRegister& vt3,
2844 const MemOperand& dst) {
2845 USE(vt2, vt3);
2846 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2847 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
2848 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
2849 LoadStoreStruct(vt, dst, NEON_ST3);
2850 }
2851
2852
st3(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,int lane,const MemOperand & dst)2853 void Assembler::st3(const VRegister& vt,
2854 const VRegister& vt2,
2855 const VRegister& vt3,
2856 int lane,
2857 const MemOperand& dst) {
2858 USE(vt2, vt3);
2859 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2860 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
2861 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
2862 LoadStoreStructSingle(vt, lane, dst, NEONLoadStoreSingleStructStore3);
2863 }
2864
2865
st4(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const VRegister & vt4,const MemOperand & dst)2866 void Assembler::st4(const VRegister& vt,
2867 const VRegister& vt2,
2868 const VRegister& vt3,
2869 const VRegister& vt4,
2870 const MemOperand& dst) {
2871 USE(vt2, vt3, vt4);
2872 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2873 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
2874 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
2875 LoadStoreStruct(vt, dst, NEON_ST4);
2876 }
2877
2878
st4(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const VRegister & vt4,int lane,const MemOperand & dst)2879 void Assembler::st4(const VRegister& vt,
2880 const VRegister& vt2,
2881 const VRegister& vt3,
2882 const VRegister& vt4,
2883 int lane,
2884 const MemOperand& dst) {
2885 USE(vt2, vt3, vt4);
2886 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2887 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
2888 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
2889 LoadStoreStructSingle(vt, lane, dst, NEONLoadStoreSingleStructStore4);
2890 }
2891
2892
LoadStoreStructSingle(const VRegister & vt,uint32_t lane,const MemOperand & addr,NEONLoadStoreSingleStructOp op)2893 void Assembler::LoadStoreStructSingle(const VRegister& vt,
2894 uint32_t lane,
2895 const MemOperand& addr,
2896 NEONLoadStoreSingleStructOp op) {
2897 LoadStoreStructVerify(vt, addr, op);
2898
2899 // We support vt arguments of the form vt.VxT() or vt.T(), where x is the
2900 // number of lanes, and T is b, h, s or d.
2901 unsigned lane_size = vt.GetLaneSizeInBytes();
2902 VIXL_ASSERT(lane_size > 0);
2903 VIXL_ASSERT(lane < (kQRegSizeInBytes / lane_size));
2904
2905 // Lane size is encoded in the opcode field. Lane index is encoded in the Q,
2906 // S and size fields.
2907 lane *= lane_size;
2908 if (lane_size == 8) lane++;
2909
2910 Instr size = (lane << NEONLSSize_offset) & NEONLSSize_mask;
2911 Instr s = (lane << (NEONS_offset - 2)) & NEONS_mask;
2912 Instr q = (lane << (NEONQ_offset - 3)) & NEONQ_mask;
2913
2914 Instr instr = op;
2915 switch (lane_size) {
2916 case 1:
2917 instr |= NEONLoadStoreSingle_b;
2918 break;
2919 case 2:
2920 instr |= NEONLoadStoreSingle_h;
2921 break;
2922 case 4:
2923 instr |= NEONLoadStoreSingle_s;
2924 break;
2925 default:
2926 VIXL_ASSERT(lane_size == 8);
2927 instr |= NEONLoadStoreSingle_d;
2928 }
2929
2930 Emit(instr | LoadStoreStructAddrModeField(addr) | q | size | s | Rt(vt));
2931 }
2932
2933
ld1(const VRegister & vt,int lane,const MemOperand & src)2934 void Assembler::ld1(const VRegister& vt, int lane, const MemOperand& src) {
2935 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2936 LoadStoreStructSingle(vt, lane, src, NEONLoadStoreSingleStructLoad1);
2937 }
2938
2939
ld1r(const VRegister & vt,const MemOperand & src)2940 void Assembler::ld1r(const VRegister& vt, const MemOperand& src) {
2941 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2942 LoadStoreStructSingleAllLanes(vt, src, NEON_LD1R);
2943 }
2944
2945
st1(const VRegister & vt,int lane,const MemOperand & dst)2946 void Assembler::st1(const VRegister& vt, int lane, const MemOperand& dst) {
2947 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2948 LoadStoreStructSingle(vt, lane, dst, NEONLoadStoreSingleStructStore1);
2949 }
2950
pmull(const VRegister & vd,const VRegister & vn,const VRegister & vm)2951 void Assembler::pmull(const VRegister& vd,
2952 const VRegister& vn,
2953 const VRegister& vm) {
2954 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2955 VIXL_ASSERT(AreSameFormat(vn, vm));
2956 VIXL_ASSERT((vn.Is8B() && vd.Is8H()) || (vn.Is1D() && vd.Is1Q()));
2957 VIXL_ASSERT(CPUHas(CPUFeatures::kPmull1Q) || vd.Is8H());
2958 Emit(VFormat(vn) | NEON_PMULL | Rm(vm) | Rn(vn) | Rd(vd));
2959 }
2960
pmull2(const VRegister & vd,const VRegister & vn,const VRegister & vm)2961 void Assembler::pmull2(const VRegister& vd,
2962 const VRegister& vn,
2963 const VRegister& vm) {
2964 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2965 VIXL_ASSERT(AreSameFormat(vn, vm));
2966 VIXL_ASSERT((vn.Is16B() && vd.Is8H()) || (vn.Is2D() && vd.Is1Q()));
2967 VIXL_ASSERT(CPUHas(CPUFeatures::kPmull1Q) || vd.Is8H());
2968 Emit(VFormat(vn) | NEON_PMULL2 | Rm(vm) | Rn(vn) | Rd(vd));
2969 }
2970
NEON3DifferentL(const VRegister & vd,const VRegister & vn,const VRegister & vm,NEON3DifferentOp vop)2971 void Assembler::NEON3DifferentL(const VRegister& vd,
2972 const VRegister& vn,
2973 const VRegister& vm,
2974 NEON3DifferentOp vop) {
2975 VIXL_ASSERT(AreSameFormat(vn, vm));
2976 VIXL_ASSERT((vn.Is1H() && vd.Is1S()) || (vn.Is1S() && vd.Is1D()) ||
2977 (vn.Is8B() && vd.Is8H()) || (vn.Is4H() && vd.Is4S()) ||
2978 (vn.Is2S() && vd.Is2D()) || (vn.Is16B() && vd.Is8H()) ||
2979 (vn.Is8H() && vd.Is4S()) || (vn.Is4S() && vd.Is2D()));
2980 Instr format, op = vop;
2981 if (vd.IsScalar()) {
2982 op |= NEON_Q | NEONScalar;
2983 format = SFormat(vn);
2984 } else {
2985 format = VFormat(vn);
2986 }
2987 Emit(format | op | Rm(vm) | Rn(vn) | Rd(vd));
2988 }
2989
2990
NEON3DifferentW(const VRegister & vd,const VRegister & vn,const VRegister & vm,NEON3DifferentOp vop)2991 void Assembler::NEON3DifferentW(const VRegister& vd,
2992 const VRegister& vn,
2993 const VRegister& vm,
2994 NEON3DifferentOp vop) {
2995 VIXL_ASSERT(AreSameFormat(vd, vn));
2996 VIXL_ASSERT((vm.Is8B() && vd.Is8H()) || (vm.Is4H() && vd.Is4S()) ||
2997 (vm.Is2S() && vd.Is2D()) || (vm.Is16B() && vd.Is8H()) ||
2998 (vm.Is8H() && vd.Is4S()) || (vm.Is4S() && vd.Is2D()));
2999 Emit(VFormat(vm) | vop | Rm(vm) | Rn(vn) | Rd(vd));
3000 }
3001
3002
NEON3DifferentHN(const VRegister & vd,const VRegister & vn,const VRegister & vm,NEON3DifferentOp vop)3003 void Assembler::NEON3DifferentHN(const VRegister& vd,
3004 const VRegister& vn,
3005 const VRegister& vm,
3006 NEON3DifferentOp vop) {
3007 VIXL_ASSERT(AreSameFormat(vm, vn));
3008 VIXL_ASSERT((vd.Is8B() && vn.Is8H()) || (vd.Is4H() && vn.Is4S()) ||
3009 (vd.Is2S() && vn.Is2D()) || (vd.Is16B() && vn.Is8H()) ||
3010 (vd.Is8H() && vn.Is4S()) || (vd.Is4S() && vn.Is2D()));
3011 Emit(VFormat(vd) | vop | Rm(vm) | Rn(vn) | Rd(vd));
3012 }
3013
3014
3015 // clang-format off
3016 #define NEON_3DIFF_LONG_LIST(V) \
3017 V(saddl, NEON_SADDL, vn.IsVector() && vn.IsD()) \
3018 V(saddl2, NEON_SADDL2, vn.IsVector() && vn.IsQ()) \
3019 V(sabal, NEON_SABAL, vn.IsVector() && vn.IsD()) \
3020 V(sabal2, NEON_SABAL2, vn.IsVector() && vn.IsQ()) \
3021 V(uabal, NEON_UABAL, vn.IsVector() && vn.IsD()) \
3022 V(uabal2, NEON_UABAL2, vn.IsVector() && vn.IsQ()) \
3023 V(sabdl, NEON_SABDL, vn.IsVector() && vn.IsD()) \
3024 V(sabdl2, NEON_SABDL2, vn.IsVector() && vn.IsQ()) \
3025 V(uabdl, NEON_UABDL, vn.IsVector() && vn.IsD()) \
3026 V(uabdl2, NEON_UABDL2, vn.IsVector() && vn.IsQ()) \
3027 V(smlal, NEON_SMLAL, vn.IsVector() && vn.IsD()) \
3028 V(smlal2, NEON_SMLAL2, vn.IsVector() && vn.IsQ()) \
3029 V(umlal, NEON_UMLAL, vn.IsVector() && vn.IsD()) \
3030 V(umlal2, NEON_UMLAL2, vn.IsVector() && vn.IsQ()) \
3031 V(smlsl, NEON_SMLSL, vn.IsVector() && vn.IsD()) \
3032 V(smlsl2, NEON_SMLSL2, vn.IsVector() && vn.IsQ()) \
3033 V(umlsl, NEON_UMLSL, vn.IsVector() && vn.IsD()) \
3034 V(umlsl2, NEON_UMLSL2, vn.IsVector() && vn.IsQ()) \
3035 V(smull, NEON_SMULL, vn.IsVector() && vn.IsD()) \
3036 V(smull2, NEON_SMULL2, vn.IsVector() && vn.IsQ()) \
3037 V(umull, NEON_UMULL, vn.IsVector() && vn.IsD()) \
3038 V(umull2, NEON_UMULL2, vn.IsVector() && vn.IsQ()) \
3039 V(ssubl, NEON_SSUBL, vn.IsVector() && vn.IsD()) \
3040 V(ssubl2, NEON_SSUBL2, vn.IsVector() && vn.IsQ()) \
3041 V(uaddl, NEON_UADDL, vn.IsVector() && vn.IsD()) \
3042 V(uaddl2, NEON_UADDL2, vn.IsVector() && vn.IsQ()) \
3043 V(usubl, NEON_USUBL, vn.IsVector() && vn.IsD()) \
3044 V(usubl2, NEON_USUBL2, vn.IsVector() && vn.IsQ()) \
3045 V(sqdmlal, NEON_SQDMLAL, vn.Is1H() || vn.Is1S() || vn.Is4H() || vn.Is2S()) \
3046 V(sqdmlal2, NEON_SQDMLAL2, vn.Is1H() || vn.Is1S() || vn.Is8H() || vn.Is4S()) \
3047 V(sqdmlsl, NEON_SQDMLSL, vn.Is1H() || vn.Is1S() || vn.Is4H() || vn.Is2S()) \
3048 V(sqdmlsl2, NEON_SQDMLSL2, vn.Is1H() || vn.Is1S() || vn.Is8H() || vn.Is4S()) \
3049 V(sqdmull, NEON_SQDMULL, vn.Is1H() || vn.Is1S() || vn.Is4H() || vn.Is2S()) \
3050 V(sqdmull2, NEON_SQDMULL2, vn.Is1H() || vn.Is1S() || vn.Is8H() || vn.Is4S()) \
3051 // clang-format on
3052
3053
3054 #define VIXL_DEFINE_ASM_FUNC(FN, OP, AS) \
3055 void Assembler::FN(const VRegister& vd, \
3056 const VRegister& vn, \
3057 const VRegister& vm) { \
3058 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON)); \
3059 VIXL_ASSERT(AS); \
3060 NEON3DifferentL(vd, vn, vm, OP); \
3061 }
3062 NEON_3DIFF_LONG_LIST(VIXL_DEFINE_ASM_FUNC)
3063 #undef VIXL_DEFINE_ASM_FUNC
3064
3065 // clang-format off
3066 #define NEON_3DIFF_HN_LIST(V) \
3067 V(addhn, NEON_ADDHN, vd.IsD()) \
3068 V(addhn2, NEON_ADDHN2, vd.IsQ()) \
3069 V(raddhn, NEON_RADDHN, vd.IsD()) \
3070 V(raddhn2, NEON_RADDHN2, vd.IsQ()) \
3071 V(subhn, NEON_SUBHN, vd.IsD()) \
3072 V(subhn2, NEON_SUBHN2, vd.IsQ()) \
3073 V(rsubhn, NEON_RSUBHN, vd.IsD()) \
3074 V(rsubhn2, NEON_RSUBHN2, vd.IsQ())
3075 // clang-format on
3076
3077 #define VIXL_DEFINE_ASM_FUNC(FN, OP, AS) \
3078 void Assembler::FN(const VRegister& vd, \
3079 const VRegister& vn, \
3080 const VRegister& vm) { \
3081 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON)); \
3082 VIXL_ASSERT(AS); \
3083 NEON3DifferentHN(vd, vn, vm, OP); \
3084 }
NEON_3DIFF_HN_LIST(VIXL_DEFINE_ASM_FUNC)3085 NEON_3DIFF_HN_LIST(VIXL_DEFINE_ASM_FUNC)
3086 #undef VIXL_DEFINE_ASM_FUNC
3087
3088 void Assembler::uaddw(const VRegister& vd,
3089 const VRegister& vn,
3090 const VRegister& vm) {
3091 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
3092 VIXL_ASSERT(vm.IsD());
3093 NEON3DifferentW(vd, vn, vm, NEON_UADDW);
3094 }
3095
3096
uaddw2(const VRegister & vd,const VRegister & vn,const VRegister & vm)3097 void Assembler::uaddw2(const VRegister& vd,
3098 const VRegister& vn,
3099 const VRegister& vm) {
3100 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
3101 VIXL_ASSERT(vm.IsQ());
3102 NEON3DifferentW(vd, vn, vm, NEON_UADDW2);
3103 }
3104
3105
saddw(const VRegister & vd,const VRegister & vn,const VRegister & vm)3106 void Assembler::saddw(const VRegister& vd,
3107 const VRegister& vn,
3108 const VRegister& vm) {
3109 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
3110 VIXL_ASSERT(vm.IsD());
3111 NEON3DifferentW(vd, vn, vm, NEON_SADDW);
3112 }
3113
3114
saddw2(const VRegister & vd,const VRegister & vn,const VRegister & vm)3115 void Assembler::saddw2(const VRegister& vd,
3116 const VRegister& vn,
3117 const VRegister& vm) {
3118 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
3119 VIXL_ASSERT(vm.IsQ());
3120 NEON3DifferentW(vd, vn, vm, NEON_SADDW2);
3121 }
3122
3123
usubw(const VRegister & vd,const VRegister & vn,const VRegister & vm)3124 void Assembler::usubw(const VRegister& vd,
3125 const VRegister& vn,
3126 const VRegister& vm) {
3127 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
3128 VIXL_ASSERT(vm.IsD());
3129 NEON3DifferentW(vd, vn, vm, NEON_USUBW);
3130 }
3131
3132
usubw2(const VRegister & vd,const VRegister & vn,const VRegister & vm)3133 void Assembler::usubw2(const VRegister& vd,
3134 const VRegister& vn,
3135 const VRegister& vm) {
3136 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
3137 VIXL_ASSERT(vm.IsQ());
3138 NEON3DifferentW(vd, vn, vm, NEON_USUBW2);
3139 }
3140
3141
ssubw(const VRegister & vd,const VRegister & vn,const VRegister & vm)3142 void Assembler::ssubw(const VRegister& vd,
3143 const VRegister& vn,
3144 const VRegister& vm) {
3145 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
3146 VIXL_ASSERT(vm.IsD());
3147 NEON3DifferentW(vd, vn, vm, NEON_SSUBW);
3148 }
3149
3150
ssubw2(const VRegister & vd,const VRegister & vn,const VRegister & vm)3151 void Assembler::ssubw2(const VRegister& vd,
3152 const VRegister& vn,
3153 const VRegister& vm) {
3154 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
3155 VIXL_ASSERT(vm.IsQ());
3156 NEON3DifferentW(vd, vn, vm, NEON_SSUBW2);
3157 }
3158
3159
mov(const Register & rd,const Register & rm)3160 void Assembler::mov(const Register& rd, const Register& rm) {
3161 // Moves involving the stack pointer are encoded as add immediate with
3162 // second operand of zero. Otherwise, orr with first operand zr is
3163 // used.
3164 if (rd.IsSP() || rm.IsSP()) {
3165 add(rd, rm, 0);
3166 } else {
3167 orr(rd, AppropriateZeroRegFor(rd), rm);
3168 }
3169 }
3170
xpaclri()3171 void Assembler::xpaclri() {
3172 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
3173 Emit(XPACLRI);
3174 }
3175
pacia1716()3176 void Assembler::pacia1716() {
3177 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
3178 Emit(PACIA1716);
3179 }
3180
pacib1716()3181 void Assembler::pacib1716() {
3182 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
3183 Emit(PACIB1716);
3184 }
3185
autia1716()3186 void Assembler::autia1716() {
3187 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
3188 Emit(AUTIA1716);
3189 }
3190
autib1716()3191 void Assembler::autib1716() {
3192 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
3193 Emit(AUTIB1716);
3194 }
3195
paciaz()3196 void Assembler::paciaz() {
3197 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
3198 Emit(PACIAZ);
3199 }
3200
pacibz()3201 void Assembler::pacibz() {
3202 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
3203 Emit(PACIBZ);
3204 }
3205
autiaz()3206 void Assembler::autiaz() {
3207 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
3208 Emit(AUTIAZ);
3209 }
3210
autibz()3211 void Assembler::autibz() {
3212 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
3213 Emit(AUTIBZ);
3214 }
3215
paciasp()3216 void Assembler::paciasp() {
3217 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
3218 Emit(PACIASP);
3219 }
3220
pacibsp()3221 void Assembler::pacibsp() {
3222 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
3223 Emit(PACIBSP);
3224 }
3225
autiasp()3226 void Assembler::autiasp() {
3227 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
3228 Emit(AUTIASP);
3229 }
3230
autibsp()3231 void Assembler::autibsp() {
3232 VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
3233 Emit(AUTIBSP);
3234 }
3235
bti(BranchTargetIdentifier id)3236 void Assembler::bti(BranchTargetIdentifier id) {
3237 VIXL_ASSERT((id != EmitPACIASP) && (id != EmitPACIBSP)); // Not modes of Bti.
3238 VIXL_ASSERT(id != EmitBTI_none); // Always generate an instruction.
3239 VIXL_ASSERT(CPUHas(CPUFeatures::kBTI));
3240 hint(static_cast<SystemHint>(id));
3241 }
3242
mvn(const Register & rd,const Operand & operand)3243 void Assembler::mvn(const Register& rd, const Operand& operand) {
3244 orn(rd, AppropriateZeroRegFor(rd), operand);
3245 }
3246
3247
mrs(const Register & xt,SystemRegister sysreg)3248 void Assembler::mrs(const Register& xt, SystemRegister sysreg) {
3249 VIXL_ASSERT(xt.Is64Bits());
3250 VIXL_ASSERT(CPUHas(sysreg));
3251 Emit(MRS | ImmSystemRegister(sysreg) | Rt(xt));
3252 }
3253
3254
msr(SystemRegister sysreg,const Register & xt)3255 void Assembler::msr(SystemRegister sysreg, const Register& xt) {
3256 VIXL_ASSERT(xt.Is64Bits());
3257 VIXL_ASSERT(CPUHas(sysreg));
3258 Emit(MSR | Rt(xt) | ImmSystemRegister(sysreg));
3259 }
3260
3261
cfinv()3262 void Assembler::cfinv() {
3263 VIXL_ASSERT(CPUHas(CPUFeatures::kFlagM));
3264 Emit(CFINV);
3265 }
3266
3267
axflag()3268 void Assembler::axflag() {
3269 VIXL_ASSERT(CPUHas(CPUFeatures::kAXFlag));
3270 Emit(AXFLAG);
3271 }
3272
3273
xaflag()3274 void Assembler::xaflag() {
3275 VIXL_ASSERT(CPUHas(CPUFeatures::kAXFlag));
3276 Emit(XAFLAG);
3277 }
3278
3279
clrex(int imm4)3280 void Assembler::clrex(int imm4) { Emit(CLREX | CRm(imm4)); }
3281
3282
dmb(BarrierDomain domain,BarrierType type)3283 void Assembler::dmb(BarrierDomain domain, BarrierType type) {
3284 Emit(DMB | ImmBarrierDomain(domain) | ImmBarrierType(type));
3285 }
3286
3287
dsb(BarrierDomain domain,BarrierType type)3288 void Assembler::dsb(BarrierDomain domain, BarrierType type) {
3289 Emit(DSB | ImmBarrierDomain(domain) | ImmBarrierType(type));
3290 }
3291
3292
isb()3293 void Assembler::isb() {
3294 Emit(ISB | ImmBarrierDomain(FullSystem) | ImmBarrierType(BarrierAll));
3295 }
3296
esb()3297 void Assembler::esb() {
3298 VIXL_ASSERT(CPUHas(CPUFeatures::kRAS));
3299 hint(ESB);
3300 }
3301
csdb()3302 void Assembler::csdb() { hint(CSDB); }
3303
fmov(const VRegister & vd,double imm)3304 void Assembler::fmov(const VRegister& vd, double imm) {
3305 VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
3306 if (vd.IsScalar()) {
3307 VIXL_ASSERT(vd.Is1D());
3308 Emit(FMOV_d_imm | Rd(vd) | ImmFP64(imm));
3309 } else {
3310 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
3311 VIXL_ASSERT(vd.Is2D());
3312 Instr op = NEONModifiedImmediate_MOVI | NEONModifiedImmediateOpBit;
3313 Instr q = NEON_Q;
3314 uint32_t encoded_imm = FP64ToImm8(imm);
3315 Emit(q | op | ImmNEONabcdefgh(encoded_imm) | NEONCmode(0xf) | Rd(vd));
3316 }
3317 }
3318
3319
fmov(const VRegister & vd,float imm)3320 void Assembler::fmov(const VRegister& vd, float imm) {
3321 VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
3322 if (vd.IsScalar()) {
3323 VIXL_ASSERT(vd.Is1S());
3324 Emit(FMOV_s_imm | Rd(vd) | ImmFP32(imm));
3325 } else {
3326 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
3327 VIXL_ASSERT(vd.Is2S() || vd.Is4S());
3328 Instr op = NEONModifiedImmediate_MOVI;
3329 Instr q = vd.Is4S() ? NEON_Q : 0;
3330 uint32_t encoded_imm = FP32ToImm8(imm);
3331 Emit(q | op | ImmNEONabcdefgh(encoded_imm) | NEONCmode(0xf) | Rd(vd));
3332 }
3333 }
3334
3335
fmov(const VRegister & vd,Float16 imm)3336 void Assembler::fmov(const VRegister& vd, Float16 imm) {
3337 VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
3338 if (vd.IsScalar()) {
3339 VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
3340 VIXL_ASSERT(vd.Is1H());
3341 Emit(FMOV_h_imm | Rd(vd) | ImmFP16(imm));
3342 } else {
3343 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON, CPUFeatures::kNEONHalf));
3344 VIXL_ASSERT(vd.Is4H() || vd.Is8H());
3345 Instr q = vd.Is8H() ? NEON_Q : 0;
3346 uint32_t encoded_imm = FP16ToImm8(imm);
3347 Emit(q | NEONModifiedImmediate_FMOV | ImmNEONabcdefgh(encoded_imm) |
3348 NEONCmode(0xf) | Rd(vd));
3349 }
3350 }
3351
3352
fmov(const Register & rd,const VRegister & vn)3353 void Assembler::fmov(const Register& rd, const VRegister& vn) {
3354 VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
3355 VIXL_ASSERT(vn.Is1H() || vn.Is1S() || vn.Is1D());
3356 VIXL_ASSERT((rd.GetSizeInBits() == vn.GetSizeInBits()) || vn.Is1H());
3357 FPIntegerConvertOp op;
3358 switch (vn.GetSizeInBits()) {
3359 case 16:
3360 VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
3361 op = rd.Is64Bits() ? FMOV_xh : FMOV_wh;
3362 break;
3363 case 32:
3364 op = FMOV_ws;
3365 break;
3366 default:
3367 op = FMOV_xd;
3368 }
3369 Emit(op | Rd(rd) | Rn(vn));
3370 }
3371
3372
fmov(const VRegister & vd,const Register & rn)3373 void Assembler::fmov(const VRegister& vd, const Register& rn) {
3374 VIXL_ASSERT(CPUHas(CPUFeatures::kFP) ||
3375 (vd.Is1D() && CPUHas(CPUFeatures::kNEON)));
3376 VIXL_ASSERT(vd.Is1H() || vd.Is1S() || vd.Is1D());
3377 VIXL_ASSERT((vd.GetSizeInBits() == rn.GetSizeInBits()) || vd.Is1H());
3378 FPIntegerConvertOp op;
3379 switch (vd.GetSizeInBits()) {
3380 case 16:
3381 VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
3382 op = rn.Is64Bits() ? FMOV_hx : FMOV_hw;
3383 break;
3384 case 32:
3385 op = FMOV_sw;
3386 break;
3387 default:
3388 op = FMOV_dx;
3389 }
3390 Emit(op | Rd(vd) | Rn(rn));
3391 }
3392
3393
fmov(const VRegister & vd,const VRegister & vn)3394 void Assembler::fmov(const VRegister& vd, const VRegister& vn) {
3395 VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
3396 if (vd.Is1H()) {
3397 VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
3398 }
3399 VIXL_ASSERT(vd.Is1H() || vd.Is1S() || vd.Is1D());
3400 VIXL_ASSERT(vd.IsSameFormat(vn));
3401 Emit(FPType(vd) | FMOV | Rd(vd) | Rn(vn));
3402 }
3403
3404
fmov(const VRegister & vd,int index,const Register & rn)3405 void Assembler::fmov(const VRegister& vd, int index, const Register& rn) {
3406 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON, CPUFeatures::kFP));
3407 VIXL_ASSERT((index == 1) && vd.Is1D() && rn.IsX());
3408 USE(index);
3409 Emit(FMOV_d1_x | Rd(vd) | Rn(rn));
3410 }
3411
3412
fmov(const Register & rd,const VRegister & vn,int index)3413 void Assembler::fmov(const Register& rd, const VRegister& vn, int index) {
3414 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON, CPUFeatures::kFP));
3415 VIXL_ASSERT((index == 1) && vn.Is1D() && rd.IsX());
3416 USE(index);
3417 Emit(FMOV_x_d1 | Rd(rd) | Rn(vn));
3418 }
3419
3420
fmadd(const VRegister & vd,const VRegister & vn,const VRegister & vm,const VRegister & va)3421 void Assembler::fmadd(const VRegister& vd,
3422 const VRegister& vn,
3423 const VRegister& vm,
3424 const VRegister& va) {
3425 VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
3426 FPDataProcessing3SourceOp op;
3427 if (vd.Is1H()) {
3428 VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
3429 op = FMADD_h;
3430 } else if (vd.Is1S()) {
3431 op = FMADD_s;
3432 } else {
3433 VIXL_ASSERT(vd.Is1D());
3434 op = FMADD_d;
3435 }
3436 FPDataProcessing3Source(vd, vn, vm, va, op);
3437 }
3438
3439
fmsub(const VRegister & vd,const VRegister & vn,const VRegister & vm,const VRegister & va)3440 void Assembler::fmsub(const VRegister& vd,
3441 const VRegister& vn,
3442 const VRegister& vm,
3443 const VRegister& va) {
3444 VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
3445 FPDataProcessing3SourceOp op;
3446 if (vd.Is1H()) {
3447 VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
3448 op = FMSUB_h;
3449 } else if (vd.Is1S()) {
3450 op = FMSUB_s;
3451 } else {
3452 VIXL_ASSERT(vd.Is1D());
3453 op = FMSUB_d;
3454 }
3455 FPDataProcessing3Source(vd, vn, vm, va, op);
3456 }
3457
3458
fnmadd(const VRegister & vd,const VRegister & vn,const VRegister & vm,const VRegister & va)3459 void Assembler::fnmadd(const VRegister& vd,
3460 const VRegister& vn,
3461 const VRegister& vm,
3462 const VRegister& va) {
3463 VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
3464 FPDataProcessing3SourceOp op;
3465 if (vd.Is1H()) {
3466 VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
3467 op = FNMADD_h;
3468 } else if (vd.Is1S()) {
3469 op = FNMADD_s;
3470 } else {
3471 VIXL_ASSERT(vd.Is1D());
3472 op = FNMADD_d;
3473 }
3474 FPDataProcessing3Source(vd, vn, vm, va, op);
3475 }
3476
3477
fnmsub(const VRegister & vd,const VRegister & vn,const VRegister & vm,const VRegister & va)3478 void Assembler::fnmsub(const VRegister& vd,
3479 const VRegister& vn,
3480 const VRegister& vm,
3481 const VRegister& va) {
3482 VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
3483 FPDataProcessing3SourceOp op;
3484 if (vd.Is1H()) {
3485 VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
3486 op = FNMSUB_h;
3487 } else if (vd.Is1S()) {
3488 op = FNMSUB_s;
3489 } else {
3490 VIXL_ASSERT(vd.Is1D());
3491 op = FNMSUB_d;
3492 }
3493 FPDataProcessing3Source(vd, vn, vm, va, op);
3494 }
3495
3496
fnmul(const VRegister & vd,const VRegister & vn,const VRegister & vm)3497 void Assembler::fnmul(const VRegister& vd,
3498 const VRegister& vn,
3499 const VRegister& vm) {
3500 VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
3501 VIXL_ASSERT(AreSameSizeAndType(vd, vn, vm));
3502 Instr op;
3503 if (vd.Is1H()) {
3504 VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
3505 op = FNMUL_h;
3506 } else if (vd.Is1S()) {
3507 op = FNMUL_s;
3508 } else {
3509 VIXL_ASSERT(vd.Is1D());
3510 op = FNMUL_d;
3511 }
3512 Emit(FPType(vd) | op | Rm(vm) | Rn(vn) | Rd(vd));
3513 }
3514
3515
FPCompareMacro(const VRegister & vn,double value,FPTrapFlags trap)3516 void Assembler::FPCompareMacro(const VRegister& vn,
3517 double value,
3518 FPTrapFlags trap) {
3519 USE(value);
3520 // Although the fcmp{e} instructions can strictly only take an immediate
3521 // value of +0.0, we don't need to check for -0.0 because the sign of 0.0
3522 // doesn't affect the result of the comparison.
3523 VIXL_ASSERT(value == 0.0);
3524 VIXL_ASSERT(vn.Is1H() || vn.Is1S() || vn.Is1D());
3525 Instr op = (trap == EnableTrap) ? FCMPE_zero : FCMP_zero;
3526 Emit(FPType(vn) | op | Rn(vn));
3527 }
3528
3529
FPCompareMacro(const VRegister & vn,const VRegister & vm,FPTrapFlags trap)3530 void Assembler::FPCompareMacro(const VRegister& vn,
3531 const VRegister& vm,
3532 FPTrapFlags trap) {
3533 VIXL_ASSERT(vn.Is1H() || vn.Is1S() || vn.Is1D());
3534 VIXL_ASSERT(vn.IsSameSizeAndType(vm));
3535 Instr op = (trap == EnableTrap) ? FCMPE : FCMP;
3536 Emit(FPType(vn) | op | Rm(vm) | Rn(vn));
3537 }
3538
3539
fcmp(const VRegister & vn,const VRegister & vm)3540 void Assembler::fcmp(const VRegister& vn, const VRegister& vm) {
3541 VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
3542 if (vn.Is1H()) VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
3543 FPCompareMacro(vn, vm, DisableTrap);
3544 }
3545
3546
fcmpe(const VRegister & vn,const VRegister & vm)3547 void Assembler::fcmpe(const VRegister& vn, const VRegister& vm) {
3548 VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
3549 if (vn.Is1H()) VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
3550 FPCompareMacro(vn, vm, EnableTrap);
3551 }
3552
3553
fcmp(const VRegister & vn,double value)3554 void Assembler::fcmp(const VRegister& vn, double value) {
3555 VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
3556 if (vn.Is1H()) VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
3557 FPCompareMacro(vn, value, DisableTrap);
3558 }
3559
3560
fcmpe(const VRegister & vn,double value)3561 void Assembler::fcmpe(const VRegister& vn, double value) {
3562 VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
3563 if (vn.Is1H()) VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
3564 FPCompareMacro(vn, value, EnableTrap);
3565 }
3566
3567
FPCCompareMacro(const VRegister & vn,const VRegister & vm,StatusFlags nzcv,Condition cond,FPTrapFlags trap)3568 void Assembler::FPCCompareMacro(const VRegister& vn,
3569 const VRegister& vm,
3570 StatusFlags nzcv,
3571 Condition cond,
3572 FPTrapFlags trap) {
3573 VIXL_ASSERT(vn.Is1H() || vn.Is1S() || vn.Is1D());
3574 VIXL_ASSERT(vn.IsSameSizeAndType(vm));
3575 Instr op = (trap == EnableTrap) ? FCCMPE : FCCMP;
3576 Emit(FPType(vn) | op | Rm(vm) | Cond(cond) | Rn(vn) | Nzcv(nzcv));
3577 }
3578
fccmp(const VRegister & vn,const VRegister & vm,StatusFlags nzcv,Condition cond)3579 void Assembler::fccmp(const VRegister& vn,
3580 const VRegister& vm,
3581 StatusFlags nzcv,
3582 Condition cond) {
3583 VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
3584 if (vn.Is1H()) VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
3585 FPCCompareMacro(vn, vm, nzcv, cond, DisableTrap);
3586 }
3587
3588
fccmpe(const VRegister & vn,const VRegister & vm,StatusFlags nzcv,Condition cond)3589 void Assembler::fccmpe(const VRegister& vn,
3590 const VRegister& vm,
3591 StatusFlags nzcv,
3592 Condition cond) {
3593 VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
3594 if (vn.Is1H()) VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
3595 FPCCompareMacro(vn, vm, nzcv, cond, EnableTrap);
3596 }
3597
3598
fcsel(const VRegister & vd,const VRegister & vn,const VRegister & vm,Condition cond)3599 void Assembler::fcsel(const VRegister& vd,
3600 const VRegister& vn,
3601 const VRegister& vm,
3602 Condition cond) {
3603 VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
3604 if (vd.Is1H()) VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
3605 VIXL_ASSERT(vd.Is1H() || vd.Is1S() || vd.Is1D());
3606 VIXL_ASSERT(AreSameFormat(vd, vn, vm));
3607 Emit(FPType(vd) | FCSEL | Rm(vm) | Cond(cond) | Rn(vn) | Rd(vd));
3608 }
3609
3610
fcvt(const VRegister & vd,const VRegister & vn)3611 void Assembler::fcvt(const VRegister& vd, const VRegister& vn) {
3612 VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
3613 FPDataProcessing1SourceOp op;
3614 // The half-precision variants belong to base FP, and do not require kFPHalf.
3615 if (vd.Is1D()) {
3616 VIXL_ASSERT(vn.Is1S() || vn.Is1H());
3617 op = vn.Is1S() ? FCVT_ds : FCVT_dh;
3618 } else if (vd.Is1S()) {
3619 VIXL_ASSERT(vn.Is1D() || vn.Is1H());
3620 op = vn.Is1D() ? FCVT_sd : FCVT_sh;
3621 } else {
3622 VIXL_ASSERT(vd.Is1H());
3623 VIXL_ASSERT(vn.Is1D() || vn.Is1S());
3624 op = vn.Is1D() ? FCVT_hd : FCVT_hs;
3625 }
3626 FPDataProcessing1Source(vd, vn, op);
3627 }
3628
3629
fcvtl(const VRegister & vd,const VRegister & vn)3630 void Assembler::fcvtl(const VRegister& vd, const VRegister& vn) {
3631 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
3632 VIXL_ASSERT((vd.Is4S() && vn.Is4H()) || (vd.Is2D() && vn.Is2S()));
3633 // The half-precision variants belong to base FP, and do not require kFPHalf.
3634 Instr format = vd.Is2D() ? (1 << NEONSize_offset) : 0;
3635 Emit(format | NEON_FCVTL | Rn(vn) | Rd(vd));
3636 }
3637
3638
fcvtl2(const VRegister & vd,const VRegister & vn)3639 void Assembler::fcvtl2(const VRegister& vd, const VRegister& vn) {
3640 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
3641 VIXL_ASSERT((vd.Is4S() && vn.Is8H()) || (vd.Is2D() && vn.Is4S()));
3642 // The half-precision variants belong to base FP, and do not require kFPHalf.
3643 Instr format = vd.Is2D() ? (1 << NEONSize_offset) : 0;
3644 Emit(NEON_Q | format | NEON_FCVTL | Rn(vn) | Rd(vd));
3645 }
3646
3647
fcvtn(const VRegister & vd,const VRegister & vn)3648 void Assembler::fcvtn(const VRegister& vd, const VRegister& vn) {
3649 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
3650 VIXL_ASSERT((vn.Is4S() && vd.Is4H()) || (vn.Is2D() && vd.Is2S()));
3651 // The half-precision variants belong to base FP, and do not require kFPHalf.
3652 Instr format = vn.Is2D() ? (1 << NEONSize_offset) : 0;
3653 Emit(format | NEON_FCVTN | Rn(vn) | Rd(vd));
3654 }
3655
3656
fcvtn2(const VRegister & vd,const VRegister & vn)3657 void Assembler::fcvtn2(const VRegister& vd, const VRegister& vn) {
3658 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
3659 VIXL_ASSERT((vn.Is4S() && vd.Is8H()) || (vn.Is2D() && vd.Is4S()));
3660 // The half-precision variants belong to base FP, and do not require kFPHalf.
3661 Instr format = vn.Is2D() ? (1 << NEONSize_offset) : 0;
3662 Emit(NEON_Q | format | NEON_FCVTN | Rn(vn) | Rd(vd));
3663 }
3664
3665
fcvtxn(const VRegister & vd,const VRegister & vn)3666 void Assembler::fcvtxn(const VRegister& vd, const VRegister& vn) {
3667 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
3668 Instr format = 1 << NEONSize_offset;
3669 if (vd.IsScalar()) {
3670 VIXL_ASSERT(vd.Is1S() && vn.Is1D());
3671 Emit(format | NEON_FCVTXN_scalar | Rn(vn) | Rd(vd));
3672 } else {
3673 VIXL_ASSERT(vd.Is2S() && vn.Is2D());
3674 Emit(format | NEON_FCVTXN | Rn(vn) | Rd(vd));
3675 }
3676 }
3677
3678
fcvtxn2(const VRegister & vd,const VRegister & vn)3679 void Assembler::fcvtxn2(const VRegister& vd, const VRegister& vn) {
3680 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
3681 VIXL_ASSERT(vd.Is4S() && vn.Is2D());
3682 Instr format = 1 << NEONSize_offset;
3683 Emit(NEON_Q | format | NEON_FCVTXN | Rn(vn) | Rd(vd));
3684 }
3685
fjcvtzs(const Register & rd,const VRegister & vn)3686 void Assembler::fjcvtzs(const Register& rd, const VRegister& vn) {
3687 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kJSCVT));
3688 VIXL_ASSERT(rd.IsW() && vn.Is1D());
3689 Emit(FJCVTZS | Rn(vn) | Rd(rd));
3690 }
3691
3692
NEONFPConvertToInt(const Register & rd,const VRegister & vn,Instr op)3693 void Assembler::NEONFPConvertToInt(const Register& rd,
3694 const VRegister& vn,
3695 Instr op) {
3696 Emit(SF(rd) | FPType(vn) | op | Rn(vn) | Rd(rd));
3697 }
3698
3699
NEONFPConvertToInt(const VRegister & vd,const VRegister & vn,Instr op)3700 void Assembler::NEONFPConvertToInt(const VRegister& vd,
3701 const VRegister& vn,
3702 Instr op) {
3703 if (vn.IsScalar()) {
3704 VIXL_ASSERT((vd.Is1S() && vn.Is1S()) || (vd.Is1D() && vn.Is1D()));
3705 op |= NEON_Q | NEONScalar;
3706 }
3707 Emit(FPFormat(vn) | op | Rn(vn) | Rd(vd));
3708 }
3709
3710
NEONFP16ConvertToInt(const VRegister & vd,const VRegister & vn,Instr op)3711 void Assembler::NEONFP16ConvertToInt(const VRegister& vd,
3712 const VRegister& vn,
3713 Instr op) {
3714 VIXL_ASSERT(AreSameFormat(vd, vn));
3715 VIXL_ASSERT(vn.IsLaneSizeH());
3716 if (vn.IsScalar()) {
3717 op |= NEON_Q | NEONScalar;
3718 } else if (vn.Is8H()) {
3719 op |= NEON_Q;
3720 }
3721 Emit(op | Rn(vn) | Rd(vd));
3722 }
3723
3724
3725 #define NEON_FP2REGMISC_FCVT_LIST(V) \
3726 V(fcvtnu, NEON_FCVTNU, FCVTNU) \
3727 V(fcvtns, NEON_FCVTNS, FCVTNS) \
3728 V(fcvtpu, NEON_FCVTPU, FCVTPU) \
3729 V(fcvtps, NEON_FCVTPS, FCVTPS) \
3730 V(fcvtmu, NEON_FCVTMU, FCVTMU) \
3731 V(fcvtms, NEON_FCVTMS, FCVTMS) \
3732 V(fcvtau, NEON_FCVTAU, FCVTAU) \
3733 V(fcvtas, NEON_FCVTAS, FCVTAS)
3734
3735 #define VIXL_DEFINE_ASM_FUNC(FN, VEC_OP, SCA_OP) \
3736 void Assembler::FN(const Register& rd, const VRegister& vn) { \
3737 VIXL_ASSERT(CPUHas(CPUFeatures::kFP)); \
3738 if (vn.IsH()) VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf)); \
3739 NEONFPConvertToInt(rd, vn, SCA_OP); \
3740 } \
3741 void Assembler::FN(const VRegister& vd, const VRegister& vn) { \
3742 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON)); \
3743 if (vd.IsLaneSizeH()) { \
3744 VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf)); \
3745 NEONFP16ConvertToInt(vd, vn, VEC_OP##_H); \
3746 } else { \
3747 NEONFPConvertToInt(vd, vn, VEC_OP); \
3748 } \
3749 }
NEON_FP2REGMISC_FCVT_LIST(VIXL_DEFINE_ASM_FUNC)3750 NEON_FP2REGMISC_FCVT_LIST(VIXL_DEFINE_ASM_FUNC)
3751 #undef VIXL_DEFINE_ASM_FUNC
3752
3753
3754 void Assembler::fcvtzs(const Register& rd, const VRegister& vn, int fbits) {
3755 VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
3756 if (vn.Is1H()) VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
3757 VIXL_ASSERT(vn.Is1H() || vn.Is1S() || vn.Is1D());
3758 VIXL_ASSERT((fbits >= 0) && (fbits <= rd.GetSizeInBits()));
3759 if (fbits == 0) {
3760 Emit(SF(rd) | FPType(vn) | FCVTZS | Rn(vn) | Rd(rd));
3761 } else {
3762 Emit(SF(rd) | FPType(vn) | FCVTZS_fixed | FPScale(64 - fbits) | Rn(vn) |
3763 Rd(rd));
3764 }
3765 }
3766
3767
fcvtzs(const VRegister & vd,const VRegister & vn,int fbits)3768 void Assembler::fcvtzs(const VRegister& vd, const VRegister& vn, int fbits) {
3769 // This form is a NEON scalar FP instruction.
3770 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
3771 if (vn.IsLaneSizeH()) VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
3772 VIXL_ASSERT(fbits >= 0);
3773 if (fbits == 0) {
3774 if (vd.IsLaneSizeH()) {
3775 NEONFP2RegMiscFP16(vd, vn, NEON_FCVTZS_H);
3776 } else {
3777 NEONFP2RegMisc(vd, vn, NEON_FCVTZS);
3778 }
3779 } else {
3780 VIXL_ASSERT(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S() ||
3781 vd.Is1H() || vd.Is4H() || vd.Is8H());
3782 NEONShiftRightImmediate(vd, vn, fbits, NEON_FCVTZS_imm);
3783 }
3784 }
3785
3786
fcvtzu(const Register & rd,const VRegister & vn,int fbits)3787 void Assembler::fcvtzu(const Register& rd, const VRegister& vn, int fbits) {
3788 VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
3789 if (vn.Is1H()) VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
3790 VIXL_ASSERT(vn.Is1H() || vn.Is1S() || vn.Is1D());
3791 VIXL_ASSERT((fbits >= 0) && (fbits <= rd.GetSizeInBits()));
3792 if (fbits == 0) {
3793 Emit(SF(rd) | FPType(vn) | FCVTZU | Rn(vn) | Rd(rd));
3794 } else {
3795 Emit(SF(rd) | FPType(vn) | FCVTZU_fixed | FPScale(64 - fbits) | Rn(vn) |
3796 Rd(rd));
3797 }
3798 }
3799
3800
fcvtzu(const VRegister & vd,const VRegister & vn,int fbits)3801 void Assembler::fcvtzu(const VRegister& vd, const VRegister& vn, int fbits) {
3802 // This form is a NEON scalar FP instruction.
3803 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
3804 if (vn.IsLaneSizeH()) VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
3805 VIXL_ASSERT(fbits >= 0);
3806 if (fbits == 0) {
3807 if (vd.IsLaneSizeH()) {
3808 NEONFP2RegMiscFP16(vd, vn, NEON_FCVTZU_H);
3809 } else {
3810 NEONFP2RegMisc(vd, vn, NEON_FCVTZU);
3811 }
3812 } else {
3813 VIXL_ASSERT(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S() ||
3814 vd.Is1H() || vd.Is4H() || vd.Is8H());
3815 NEONShiftRightImmediate(vd, vn, fbits, NEON_FCVTZU_imm);
3816 }
3817 }
3818
ucvtf(const VRegister & vd,const VRegister & vn,int fbits)3819 void Assembler::ucvtf(const VRegister& vd, const VRegister& vn, int fbits) {
3820 // This form is a NEON scalar FP instruction.
3821 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
3822 if (vn.IsLaneSizeH()) VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
3823 VIXL_ASSERT(fbits >= 0);
3824 if (fbits == 0) {
3825 if (vd.IsLaneSizeH()) {
3826 NEONFP2RegMiscFP16(vd, vn, NEON_UCVTF_H);
3827 } else {
3828 NEONFP2RegMisc(vd, vn, NEON_UCVTF);
3829 }
3830 } else {
3831 VIXL_ASSERT(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S() ||
3832 vd.Is1H() || vd.Is4H() || vd.Is8H());
3833 NEONShiftRightImmediate(vd, vn, fbits, NEON_UCVTF_imm);
3834 }
3835 }
3836
scvtf(const VRegister & vd,const VRegister & vn,int fbits)3837 void Assembler::scvtf(const VRegister& vd, const VRegister& vn, int fbits) {
3838 // This form is a NEON scalar FP instruction.
3839 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
3840 if (vn.IsLaneSizeH()) VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
3841 VIXL_ASSERT(fbits >= 0);
3842 if (fbits == 0) {
3843 if (vd.IsLaneSizeH()) {
3844 NEONFP2RegMiscFP16(vd, vn, NEON_SCVTF_H);
3845 } else {
3846 NEONFP2RegMisc(vd, vn, NEON_SCVTF);
3847 }
3848 } else {
3849 VIXL_ASSERT(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S() ||
3850 vd.Is1H() || vd.Is4H() || vd.Is8H());
3851 NEONShiftRightImmediate(vd, vn, fbits, NEON_SCVTF_imm);
3852 }
3853 }
3854
3855
scvtf(const VRegister & vd,const Register & rn,int fbits)3856 void Assembler::scvtf(const VRegister& vd, const Register& rn, int fbits) {
3857 VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
3858 if (vd.Is1H()) VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
3859 VIXL_ASSERT(vd.Is1H() || vd.Is1S() || vd.Is1D());
3860 VIXL_ASSERT(fbits >= 0);
3861 if (fbits == 0) {
3862 Emit(SF(rn) | FPType(vd) | SCVTF | Rn(rn) | Rd(vd));
3863 } else {
3864 Emit(SF(rn) | FPType(vd) | SCVTF_fixed | FPScale(64 - fbits) | Rn(rn) |
3865 Rd(vd));
3866 }
3867 }
3868
3869
ucvtf(const VRegister & vd,const Register & rn,int fbits)3870 void Assembler::ucvtf(const VRegister& vd, const Register& rn, int fbits) {
3871 VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
3872 if (vd.Is1H()) VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
3873 VIXL_ASSERT(vd.Is1H() || vd.Is1S() || vd.Is1D());
3874 VIXL_ASSERT(fbits >= 0);
3875 if (fbits == 0) {
3876 Emit(SF(rn) | FPType(vd) | UCVTF | Rn(rn) | Rd(vd));
3877 } else {
3878 Emit(SF(rn) | FPType(vd) | UCVTF_fixed | FPScale(64 - fbits) | Rn(rn) |
3879 Rd(vd));
3880 }
3881 }
3882
3883
NEON3Same(const VRegister & vd,const VRegister & vn,const VRegister & vm,NEON3SameOp vop)3884 void Assembler::NEON3Same(const VRegister& vd,
3885 const VRegister& vn,
3886 const VRegister& vm,
3887 NEON3SameOp vop) {
3888 VIXL_ASSERT(AreSameFormat(vd, vn, vm));
3889 VIXL_ASSERT(vd.IsVector() || !vd.IsQ());
3890
3891 Instr format, op = vop;
3892 if (vd.IsScalar()) {
3893 op |= NEON_Q | NEONScalar;
3894 format = SFormat(vd);
3895 } else {
3896 format = VFormat(vd);
3897 }
3898
3899 Emit(format | op | Rm(vm) | Rn(vn) | Rd(vd));
3900 }
3901
3902
NEONFP3Same(const VRegister & vd,const VRegister & vn,const VRegister & vm,Instr op)3903 void Assembler::NEONFP3Same(const VRegister& vd,
3904 const VRegister& vn,
3905 const VRegister& vm,
3906 Instr op) {
3907 VIXL_ASSERT(AreSameFormat(vd, vn, vm));
3908 Emit(FPFormat(vd) | op | Rm(vm) | Rn(vn) | Rd(vd));
3909 }
3910
3911
NEON3SameFP16(const VRegister & vd,const VRegister & vn,const VRegister & vm,Instr op)3912 void Assembler::NEON3SameFP16(const VRegister& vd,
3913 const VRegister& vn,
3914 const VRegister& vm,
3915 Instr op) {
3916 VIXL_ASSERT(AreSameFormat(vd, vn, vm));
3917 VIXL_ASSERT(vd.GetLaneSizeInBytes() == kHRegSizeInBytes);
3918 if (vd.Is8H()) op |= NEON_Q;
3919 Emit(op | Rm(vm) | Rn(vn) | Rd(vd));
3920 }
3921
3922
3923 // clang-format off
3924 #define NEON_FP2REGMISC_LIST(V) \
3925 V(fabs, NEON_FABS, FABS, FABS_h) \
3926 V(fneg, NEON_FNEG, FNEG, FNEG_h) \
3927 V(fsqrt, NEON_FSQRT, FSQRT, FSQRT_h) \
3928 V(frintn, NEON_FRINTN, FRINTN, FRINTN_h) \
3929 V(frinta, NEON_FRINTA, FRINTA, FRINTA_h) \
3930 V(frintp, NEON_FRINTP, FRINTP, FRINTP_h) \
3931 V(frintm, NEON_FRINTM, FRINTM, FRINTM_h) \
3932 V(frintx, NEON_FRINTX, FRINTX, FRINTX_h) \
3933 V(frintz, NEON_FRINTZ, FRINTZ, FRINTZ_h) \
3934 V(frinti, NEON_FRINTI, FRINTI, FRINTI_h) \
3935 V(frsqrte, NEON_FRSQRTE, NEON_FRSQRTE_scalar, NEON_FRSQRTE_H_scalar) \
3936 V(frecpe, NEON_FRECPE, NEON_FRECPE_scalar, NEON_FRECPE_H_scalar)
3937 // clang-format on
3938
3939 #define VIXL_DEFINE_ASM_FUNC(FN, VEC_OP, SCA_OP, SCA_OP_H) \
3940 void Assembler::FN(const VRegister& vd, const VRegister& vn) { \
3941 VIXL_ASSERT(CPUHas(CPUFeatures::kFP)); \
3942 Instr op; \
3943 if (vd.IsScalar()) { \
3944 if (vd.Is1H()) { \
3945 if ((SCA_OP_H & NEONScalar2RegMiscFP16FMask) == \
3946 NEONScalar2RegMiscFP16Fixed) { \
3947 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON, CPUFeatures::kNEONHalf)); \
3948 } else { \
3949 VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf)); \
3950 } \
3951 op = SCA_OP_H; \
3952 } else { \
3953 if ((SCA_OP & NEONScalar2RegMiscFMask) == NEONScalar2RegMiscFixed) { \
3954 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON)); \
3955 } \
3956 VIXL_ASSERT(vd.Is1S() || vd.Is1D()); \
3957 op = SCA_OP; \
3958 } \
3959 } else { \
3960 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON)); \
3961 VIXL_ASSERT(vd.Is4H() || vd.Is8H() || vd.Is2S() || vd.Is2D() || \
3962 vd.Is4S()); \
3963 if (vd.IsLaneSizeH()) { \
3964 VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf)); \
3965 op = VEC_OP##_H; \
3966 if (vd.Is8H()) { \
3967 op |= NEON_Q; \
3968 } \
3969 } else { \
3970 op = VEC_OP; \
3971 } \
3972 } \
3973 if (vd.IsLaneSizeH()) { \
3974 NEONFP2RegMiscFP16(vd, vn, op); \
3975 } else { \
3976 NEONFP2RegMisc(vd, vn, op); \
3977 } \
3978 }
3979 NEON_FP2REGMISC_LIST(VIXL_DEFINE_ASM_FUNC)
3980 #undef VIXL_DEFINE_ASM_FUNC
3981
3982 // clang-format off
3983 #define NEON_FP2REGMISC_V85_LIST(V) \
3984 V(frint32x, NEON_FRINT32X, FRINT32X) \
3985 V(frint32z, NEON_FRINT32Z, FRINT32Z) \
3986 V(frint64x, NEON_FRINT64X, FRINT64X) \
3987 V(frint64z, NEON_FRINT64Z, FRINT64Z)
3988 // clang-format on
3989
3990 #define VIXL_DEFINE_ASM_FUNC(FN, VEC_OP, SCA_OP) \
3991 void Assembler::FN(const VRegister& vd, const VRegister& vn) { \
3992 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kFrintToFixedSizedInt)); \
3993 Instr op; \
3994 if (vd.IsScalar()) { \
3995 VIXL_ASSERT(vd.Is1S() || vd.Is1D()); \
3996 op = SCA_OP; \
3997 } else { \
3998 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON)); \
3999 VIXL_ASSERT(vd.Is2S() || vd.Is2D() || vd.Is4S()); \
4000 op = VEC_OP; \
4001 } \
4002 NEONFP2RegMisc(vd, vn, op); \
4003 }
NEON_FP2REGMISC_V85_LIST(VIXL_DEFINE_ASM_FUNC)4004 NEON_FP2REGMISC_V85_LIST(VIXL_DEFINE_ASM_FUNC)
4005 #undef VIXL_DEFINE_ASM_FUNC
4006
4007 void Assembler::NEONFP2RegMiscFP16(const VRegister& vd,
4008 const VRegister& vn,
4009 Instr op) {
4010 VIXL_ASSERT(AreSameFormat(vd, vn));
4011 Emit(op | Rn(vn) | Rd(vd));
4012 }
4013
4014
NEONFP2RegMisc(const VRegister & vd,const VRegister & vn,Instr op)4015 void Assembler::NEONFP2RegMisc(const VRegister& vd,
4016 const VRegister& vn,
4017 Instr op) {
4018 VIXL_ASSERT(AreSameFormat(vd, vn));
4019 Emit(FPFormat(vd) | op | Rn(vn) | Rd(vd));
4020 }
4021
4022
NEON2RegMisc(const VRegister & vd,const VRegister & vn,NEON2RegMiscOp vop,int value)4023 void Assembler::NEON2RegMisc(const VRegister& vd,
4024 const VRegister& vn,
4025 NEON2RegMiscOp vop,
4026 int value) {
4027 VIXL_ASSERT(AreSameFormat(vd, vn));
4028 VIXL_ASSERT(value == 0);
4029 USE(value);
4030
4031 Instr format, op = vop;
4032 if (vd.IsScalar()) {
4033 op |= NEON_Q | NEONScalar;
4034 format = SFormat(vd);
4035 } else {
4036 format = VFormat(vd);
4037 }
4038
4039 Emit(format | op | Rn(vn) | Rd(vd));
4040 }
4041
4042
cmeq(const VRegister & vd,const VRegister & vn,int value)4043 void Assembler::cmeq(const VRegister& vd, const VRegister& vn, int value) {
4044 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4045 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
4046 NEON2RegMisc(vd, vn, NEON_CMEQ_zero, value);
4047 }
4048
4049
cmge(const VRegister & vd,const VRegister & vn,int value)4050 void Assembler::cmge(const VRegister& vd, const VRegister& vn, int value) {
4051 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4052 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
4053 NEON2RegMisc(vd, vn, NEON_CMGE_zero, value);
4054 }
4055
4056
cmgt(const VRegister & vd,const VRegister & vn,int value)4057 void Assembler::cmgt(const VRegister& vd, const VRegister& vn, int value) {
4058 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4059 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
4060 NEON2RegMisc(vd, vn, NEON_CMGT_zero, value);
4061 }
4062
4063
cmle(const VRegister & vd,const VRegister & vn,int value)4064 void Assembler::cmle(const VRegister& vd, const VRegister& vn, int value) {
4065 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
4066 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4067 NEON2RegMisc(vd, vn, NEON_CMLE_zero, value);
4068 }
4069
4070
cmlt(const VRegister & vd,const VRegister & vn,int value)4071 void Assembler::cmlt(const VRegister& vd, const VRegister& vn, int value) {
4072 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4073 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
4074 NEON2RegMisc(vd, vn, NEON_CMLT_zero, value);
4075 }
4076
4077
shll(const VRegister & vd,const VRegister & vn,int shift)4078 void Assembler::shll(const VRegister& vd, const VRegister& vn, int shift) {
4079 USE(shift);
4080 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4081 VIXL_ASSERT((vd.Is8H() && vn.Is8B() && shift == 8) ||
4082 (vd.Is4S() && vn.Is4H() && shift == 16) ||
4083 (vd.Is2D() && vn.Is2S() && shift == 32));
4084 Emit(VFormat(vn) | NEON_SHLL | Rn(vn) | Rd(vd));
4085 }
4086
4087
shll2(const VRegister & vd,const VRegister & vn,int shift)4088 void Assembler::shll2(const VRegister& vd, const VRegister& vn, int shift) {
4089 USE(shift);
4090 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4091 VIXL_ASSERT((vd.Is8H() && vn.Is16B() && shift == 8) ||
4092 (vd.Is4S() && vn.Is8H() && shift == 16) ||
4093 (vd.Is2D() && vn.Is4S() && shift == 32));
4094 Emit(VFormat(vn) | NEON_SHLL | Rn(vn) | Rd(vd));
4095 }
4096
4097
NEONFP2RegMisc(const VRegister & vd,const VRegister & vn,NEON2RegMiscOp vop,double value)4098 void Assembler::NEONFP2RegMisc(const VRegister& vd,
4099 const VRegister& vn,
4100 NEON2RegMiscOp vop,
4101 double value) {
4102 VIXL_ASSERT(AreSameFormat(vd, vn));
4103 VIXL_ASSERT(value == 0.0);
4104 USE(value);
4105
4106 Instr op = vop;
4107 if (vd.IsScalar()) {
4108 VIXL_ASSERT(vd.Is1S() || vd.Is1D());
4109 op |= NEON_Q | NEONScalar;
4110 } else {
4111 VIXL_ASSERT(vd.Is2S() || vd.Is2D() || vd.Is4S());
4112 }
4113
4114 Emit(FPFormat(vd) | op | Rn(vn) | Rd(vd));
4115 }
4116
4117
NEONFP2RegMiscFP16(const VRegister & vd,const VRegister & vn,NEON2RegMiscFP16Op vop,double value)4118 void Assembler::NEONFP2RegMiscFP16(const VRegister& vd,
4119 const VRegister& vn,
4120 NEON2RegMiscFP16Op vop,
4121 double value) {
4122 VIXL_ASSERT(AreSameFormat(vd, vn));
4123 VIXL_ASSERT(value == 0.0);
4124 USE(value);
4125
4126 Instr op = vop;
4127 if (vd.IsScalar()) {
4128 VIXL_ASSERT(vd.Is1H());
4129 op |= NEON_Q | NEONScalar;
4130 } else {
4131 VIXL_ASSERT(vd.Is4H() || vd.Is8H());
4132 if (vd.Is8H()) {
4133 op |= NEON_Q;
4134 }
4135 }
4136
4137 Emit(op | Rn(vn) | Rd(vd));
4138 }
4139
4140
fcmeq(const VRegister & vd,const VRegister & vn,double value)4141 void Assembler::fcmeq(const VRegister& vd, const VRegister& vn, double value) {
4142 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
4143 if (vd.IsLaneSizeH()) {
4144 VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
4145 NEONFP2RegMiscFP16(vd, vn, NEON_FCMEQ_H_zero, value);
4146 } else {
4147 NEONFP2RegMisc(vd, vn, NEON_FCMEQ_zero, value);
4148 }
4149 }
4150
4151
fcmge(const VRegister & vd,const VRegister & vn,double value)4152 void Assembler::fcmge(const VRegister& vd, const VRegister& vn, double value) {
4153 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
4154 if (vd.IsLaneSizeH()) {
4155 VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
4156 NEONFP2RegMiscFP16(vd, vn, NEON_FCMGE_H_zero, value);
4157 } else {
4158 NEONFP2RegMisc(vd, vn, NEON_FCMGE_zero, value);
4159 }
4160 }
4161
4162
fcmgt(const VRegister & vd,const VRegister & vn,double value)4163 void Assembler::fcmgt(const VRegister& vd, const VRegister& vn, double value) {
4164 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
4165 if (vd.IsLaneSizeH()) {
4166 VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
4167 NEONFP2RegMiscFP16(vd, vn, NEON_FCMGT_H_zero, value);
4168 } else {
4169 NEONFP2RegMisc(vd, vn, NEON_FCMGT_zero, value);
4170 }
4171 }
4172
4173
fcmle(const VRegister & vd,const VRegister & vn,double value)4174 void Assembler::fcmle(const VRegister& vd, const VRegister& vn, double value) {
4175 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
4176 if (vd.IsLaneSizeH()) {
4177 VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
4178 NEONFP2RegMiscFP16(vd, vn, NEON_FCMLE_H_zero, value);
4179 } else {
4180 NEONFP2RegMisc(vd, vn, NEON_FCMLE_zero, value);
4181 }
4182 }
4183
4184
fcmlt(const VRegister & vd,const VRegister & vn,double value)4185 void Assembler::fcmlt(const VRegister& vd, const VRegister& vn, double value) {
4186 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
4187 if (vd.IsLaneSizeH()) {
4188 VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
4189 NEONFP2RegMiscFP16(vd, vn, NEON_FCMLT_H_zero, value);
4190 } else {
4191 NEONFP2RegMisc(vd, vn, NEON_FCMLT_zero, value);
4192 }
4193 }
4194
4195
frecpx(const VRegister & vd,const VRegister & vn)4196 void Assembler::frecpx(const VRegister& vd, const VRegister& vn) {
4197 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
4198 VIXL_ASSERT(vd.IsScalar());
4199 VIXL_ASSERT(AreSameFormat(vd, vn));
4200 Instr op;
4201 if (vd.Is1H()) {
4202 VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
4203 op = NEON_FRECPX_H_scalar;
4204 } else {
4205 VIXL_ASSERT(vd.Is1S() || vd.Is1D());
4206 op = NEON_FRECPX_scalar;
4207 }
4208 Emit(FPFormat(vd) | op | Rn(vn) | Rd(vd));
4209 }
4210
4211
4212 // clang-format off
4213 #define NEON_3SAME_LIST(V) \
4214 V(add, NEON_ADD, vd.IsVector() || vd.Is1D()) \
4215 V(addp, NEON_ADDP, vd.IsVector() || vd.Is1D()) \
4216 V(sub, NEON_SUB, vd.IsVector() || vd.Is1D()) \
4217 V(cmeq, NEON_CMEQ, vd.IsVector() || vd.Is1D()) \
4218 V(cmge, NEON_CMGE, vd.IsVector() || vd.Is1D()) \
4219 V(cmgt, NEON_CMGT, vd.IsVector() || vd.Is1D()) \
4220 V(cmhi, NEON_CMHI, vd.IsVector() || vd.Is1D()) \
4221 V(cmhs, NEON_CMHS, vd.IsVector() || vd.Is1D()) \
4222 V(cmtst, NEON_CMTST, vd.IsVector() || vd.Is1D()) \
4223 V(sshl, NEON_SSHL, vd.IsVector() || vd.Is1D()) \
4224 V(ushl, NEON_USHL, vd.IsVector() || vd.Is1D()) \
4225 V(srshl, NEON_SRSHL, vd.IsVector() || vd.Is1D()) \
4226 V(urshl, NEON_URSHL, vd.IsVector() || vd.Is1D()) \
4227 V(sqdmulh, NEON_SQDMULH, vd.IsLaneSizeH() || vd.IsLaneSizeS()) \
4228 V(sqrdmulh, NEON_SQRDMULH, vd.IsLaneSizeH() || vd.IsLaneSizeS()) \
4229 V(shadd, NEON_SHADD, vd.IsVector() && !vd.IsLaneSizeD()) \
4230 V(uhadd, NEON_UHADD, vd.IsVector() && !vd.IsLaneSizeD()) \
4231 V(srhadd, NEON_SRHADD, vd.IsVector() && !vd.IsLaneSizeD()) \
4232 V(urhadd, NEON_URHADD, vd.IsVector() && !vd.IsLaneSizeD()) \
4233 V(shsub, NEON_SHSUB, vd.IsVector() && !vd.IsLaneSizeD()) \
4234 V(uhsub, NEON_UHSUB, vd.IsVector() && !vd.IsLaneSizeD()) \
4235 V(smax, NEON_SMAX, vd.IsVector() && !vd.IsLaneSizeD()) \
4236 V(smaxp, NEON_SMAXP, vd.IsVector() && !vd.IsLaneSizeD()) \
4237 V(smin, NEON_SMIN, vd.IsVector() && !vd.IsLaneSizeD()) \
4238 V(sminp, NEON_SMINP, vd.IsVector() && !vd.IsLaneSizeD()) \
4239 V(umax, NEON_UMAX, vd.IsVector() && !vd.IsLaneSizeD()) \
4240 V(umaxp, NEON_UMAXP, vd.IsVector() && !vd.IsLaneSizeD()) \
4241 V(umin, NEON_UMIN, vd.IsVector() && !vd.IsLaneSizeD()) \
4242 V(uminp, NEON_UMINP, vd.IsVector() && !vd.IsLaneSizeD()) \
4243 V(saba, NEON_SABA, vd.IsVector() && !vd.IsLaneSizeD()) \
4244 V(sabd, NEON_SABD, vd.IsVector() && !vd.IsLaneSizeD()) \
4245 V(uaba, NEON_UABA, vd.IsVector() && !vd.IsLaneSizeD()) \
4246 V(uabd, NEON_UABD, vd.IsVector() && !vd.IsLaneSizeD()) \
4247 V(mla, NEON_MLA, vd.IsVector() && !vd.IsLaneSizeD()) \
4248 V(mls, NEON_MLS, vd.IsVector() && !vd.IsLaneSizeD()) \
4249 V(mul, NEON_MUL, vd.IsVector() && !vd.IsLaneSizeD()) \
4250 V(and_, NEON_AND, vd.Is8B() || vd.Is16B()) \
4251 V(orr, NEON_ORR, vd.Is8B() || vd.Is16B()) \
4252 V(orn, NEON_ORN, vd.Is8B() || vd.Is16B()) \
4253 V(eor, NEON_EOR, vd.Is8B() || vd.Is16B()) \
4254 V(bic, NEON_BIC, vd.Is8B() || vd.Is16B()) \
4255 V(bit, NEON_BIT, vd.Is8B() || vd.Is16B()) \
4256 V(bif, NEON_BIF, vd.Is8B() || vd.Is16B()) \
4257 V(bsl, NEON_BSL, vd.Is8B() || vd.Is16B()) \
4258 V(pmul, NEON_PMUL, vd.Is8B() || vd.Is16B()) \
4259 V(uqadd, NEON_UQADD, true) \
4260 V(sqadd, NEON_SQADD, true) \
4261 V(uqsub, NEON_UQSUB, true) \
4262 V(sqsub, NEON_SQSUB, true) \
4263 V(sqshl, NEON_SQSHL, true) \
4264 V(uqshl, NEON_UQSHL, true) \
4265 V(sqrshl, NEON_SQRSHL, true) \
4266 V(uqrshl, NEON_UQRSHL, true)
4267 // clang-format on
4268
4269 #define VIXL_DEFINE_ASM_FUNC(FN, OP, AS) \
4270 void Assembler::FN(const VRegister& vd, \
4271 const VRegister& vn, \
4272 const VRegister& vm) { \
4273 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON)); \
4274 VIXL_ASSERT(AS); \
4275 NEON3Same(vd, vn, vm, OP); \
4276 }
4277 NEON_3SAME_LIST(VIXL_DEFINE_ASM_FUNC)
4278 #undef VIXL_DEFINE_ASM_FUNC
4279
4280 // clang-format off
4281 #define NEON_FP3SAME_OP_LIST(V) \
4282 V(fmulx, NEON_FMULX, NEON_FMULX_scalar, NEON_FMULX_H_scalar) \
4283 V(frecps, NEON_FRECPS, NEON_FRECPS_scalar, NEON_FRECPS_H_scalar) \
4284 V(frsqrts, NEON_FRSQRTS, NEON_FRSQRTS_scalar, NEON_FRSQRTS_H_scalar) \
4285 V(fabd, NEON_FABD, NEON_FABD_scalar, NEON_FABD_H_scalar) \
4286 V(fmla, NEON_FMLA, 0, 0) \
4287 V(fmls, NEON_FMLS, 0, 0) \
4288 V(facge, NEON_FACGE, NEON_FACGE_scalar, NEON_FACGE_H_scalar) \
4289 V(facgt, NEON_FACGT, NEON_FACGT_scalar, NEON_FACGT_H_scalar) \
4290 V(fcmeq, NEON_FCMEQ, NEON_FCMEQ_scalar, NEON_FCMEQ_H_scalar) \
4291 V(fcmge, NEON_FCMGE, NEON_FCMGE_scalar, NEON_FCMGE_H_scalar) \
4292 V(fcmgt, NEON_FCMGT, NEON_FCMGT_scalar, NEON_FCMGT_H_scalar) \
4293 V(faddp, NEON_FADDP, 0, 0) \
4294 V(fmaxp, NEON_FMAXP, 0, 0) \
4295 V(fminp, NEON_FMINP, 0, 0) \
4296 V(fmaxnmp, NEON_FMAXNMP, 0, 0) \
4297 V(fadd, NEON_FADD, FADD, 0) \
4298 V(fsub, NEON_FSUB, FSUB, 0) \
4299 V(fmul, NEON_FMUL, FMUL, 0) \
4300 V(fdiv, NEON_FDIV, FDIV, 0) \
4301 V(fmax, NEON_FMAX, FMAX, 0) \
4302 V(fmin, NEON_FMIN, FMIN, 0) \
4303 V(fmaxnm, NEON_FMAXNM, FMAXNM, 0) \
4304 V(fminnm, NEON_FMINNM, FMINNM, 0) \
4305 V(fminnmp, NEON_FMINNMP, 0, 0)
4306 // clang-format on
4307
4308 // TODO: This macro is complicated because it classifies the instructions in the
4309 // macro list above, and treats each case differently. It could be somewhat
4310 // simpler if we were to split the macro, at the cost of some duplication.
4311 #define VIXL_DEFINE_ASM_FUNC(FN, VEC_OP, SCA_OP, SCA_OP_H) \
4312 void Assembler::FN(const VRegister& vd, \
4313 const VRegister& vn, \
4314 const VRegister& vm) { \
4315 VIXL_ASSERT(CPUHas(CPUFeatures::kFP)); \
4316 Instr op; \
4317 bool is_fp16 = false; \
4318 if ((SCA_OP != 0) && vd.IsScalar()) { \
4319 if ((SCA_OP_H != 0) && vd.Is1H()) { \
4320 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON, CPUFeatures::kNEONHalf)); \
4321 is_fp16 = true; \
4322 op = SCA_OP_H; \
4323 } else { \
4324 VIXL_ASSERT(vd.Is1H() || vd.Is1S() || vd.Is1D()); \
4325 if ((SCA_OP & NEONScalar3SameFMask) == NEONScalar3SameFixed) { \
4326 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON)); \
4327 if (vd.Is1H()) VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf)); \
4328 } else if (vd.Is1H()) { \
4329 VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf)); \
4330 } \
4331 op = SCA_OP; \
4332 } \
4333 } else { \
4334 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON)); \
4335 VIXL_ASSERT(vd.IsVector()); \
4336 if (vd.Is4H() || vd.Is8H()) { \
4337 VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf)); \
4338 is_fp16 = true; \
4339 op = VEC_OP##_H; \
4340 } else { \
4341 VIXL_ASSERT(vd.Is2S() || vd.Is2D() || vd.Is4S()); \
4342 op = VEC_OP; \
4343 } \
4344 } \
4345 if (is_fp16) { \
4346 NEON3SameFP16(vd, vn, vm, op); \
4347 } else { \
4348 NEONFP3Same(vd, vn, vm, op); \
4349 } \
4350 }
NEON_FP3SAME_OP_LIST(VIXL_DEFINE_ASM_FUNC)4351 NEON_FP3SAME_OP_LIST(VIXL_DEFINE_ASM_FUNC)
4352 #undef VIXL_DEFINE_ASM_FUNC
4353
4354
4355 // clang-format off
4356 #define NEON_FHM_LIST(V) \
4357 V(fmlal, NEON_FMLAL) \
4358 V(fmlal2, NEON_FMLAL2) \
4359 V(fmlsl, NEON_FMLSL) \
4360 V(fmlsl2, NEON_FMLSL2)
4361 // clang-format on
4362
4363 #define VIXL_DEFINE_ASM_FUNC(FN, VEC_OP) \
4364 void Assembler::FN(const VRegister& vd, \
4365 const VRegister& vn, \
4366 const VRegister& vm) { \
4367 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON, \
4368 CPUFeatures::kFP, \
4369 CPUFeatures::kNEONHalf, \
4370 CPUFeatures::kFHM)); \
4371 VIXL_ASSERT((vd.Is2S() && vn.Is2H() && vm.Is2H()) || \
4372 (vd.Is4S() && vn.Is4H() && vm.Is4H())); \
4373 Emit(FPFormat(vd) | VEC_OP | Rm(vm) | Rn(vn) | Rd(vd)); \
4374 }
4375 NEON_FHM_LIST(VIXL_DEFINE_ASM_FUNC)
4376 #undef VIXL_DEFINE_ASM_FUNC
4377
4378
4379 void Assembler::addp(const VRegister& vd, const VRegister& vn) {
4380 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4381 VIXL_ASSERT((vd.Is1D() && vn.Is2D()));
4382 Emit(SFormat(vd) | NEON_ADDP_scalar | Rn(vn) | Rd(vd));
4383 }
4384
4385
sqrdmlah(const VRegister & vd,const VRegister & vn,const VRegister & vm)4386 void Assembler::sqrdmlah(const VRegister& vd,
4387 const VRegister& vn,
4388 const VRegister& vm) {
4389 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON, CPUFeatures::kRDM));
4390 VIXL_ASSERT(AreSameFormat(vd, vn, vm));
4391 VIXL_ASSERT(vd.IsLaneSizeH() || vd.IsLaneSizeS());
4392
4393 Instr format, op = NEON_SQRDMLAH;
4394 if (vd.IsScalar()) {
4395 op |= NEON_Q | NEONScalar;
4396 format = SFormat(vd);
4397 } else {
4398 format = VFormat(vd);
4399 }
4400
4401 Emit(format | op | Rm(vm) | Rn(vn) | Rd(vd));
4402 }
4403
4404
sqrdmlsh(const VRegister & vd,const VRegister & vn,const VRegister & vm)4405 void Assembler::sqrdmlsh(const VRegister& vd,
4406 const VRegister& vn,
4407 const VRegister& vm) {
4408 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON, CPUFeatures::kRDM));
4409 VIXL_ASSERT(AreSameFormat(vd, vn, vm));
4410 VIXL_ASSERT(vd.IsLaneSizeH() || vd.IsLaneSizeS());
4411
4412 Instr format, op = NEON_SQRDMLSH;
4413 if (vd.IsScalar()) {
4414 op |= NEON_Q | NEONScalar;
4415 format = SFormat(vd);
4416 } else {
4417 format = VFormat(vd);
4418 }
4419
4420 Emit(format | op | Rm(vm) | Rn(vn) | Rd(vd));
4421 }
4422
4423
sdot(const VRegister & vd,const VRegister & vn,const VRegister & vm)4424 void Assembler::sdot(const VRegister& vd,
4425 const VRegister& vn,
4426 const VRegister& vm) {
4427 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON, CPUFeatures::kDotProduct));
4428 VIXL_ASSERT(AreSameFormat(vn, vm));
4429 VIXL_ASSERT((vd.Is2S() && vn.Is8B()) || (vd.Is4S() && vn.Is16B()));
4430
4431 Emit(VFormat(vd) | NEON_SDOT | Rm(vm) | Rn(vn) | Rd(vd));
4432 }
4433
4434
udot(const VRegister & vd,const VRegister & vn,const VRegister & vm)4435 void Assembler::udot(const VRegister& vd,
4436 const VRegister& vn,
4437 const VRegister& vm) {
4438 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON, CPUFeatures::kDotProduct));
4439 VIXL_ASSERT(AreSameFormat(vn, vm));
4440 VIXL_ASSERT((vd.Is2S() && vn.Is8B()) || (vd.Is4S() && vn.Is16B()));
4441
4442 Emit(VFormat(vd) | NEON_UDOT | Rm(vm) | Rn(vn) | Rd(vd));
4443 }
4444
usdot(const VRegister & vd,const VRegister & vn,const VRegister & vm)4445 void Assembler::usdot(const VRegister& vd,
4446 const VRegister& vn,
4447 const VRegister& vm) {
4448 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON, CPUFeatures::kI8MM));
4449 VIXL_ASSERT(AreSameFormat(vn, vm));
4450 VIXL_ASSERT((vd.Is2S() && vn.Is8B()) || (vd.Is4S() && vn.Is16B()));
4451
4452 Emit(VFormat(vd) | 0x0e809c00 | Rm(vm) | Rn(vn) | Rd(vd));
4453 }
4454
faddp(const VRegister & vd,const VRegister & vn)4455 void Assembler::faddp(const VRegister& vd, const VRegister& vn) {
4456 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
4457 VIXL_ASSERT((vd.Is1S() && vn.Is2S()) || (vd.Is1D() && vn.Is2D()) ||
4458 (vd.Is1H() && vn.Is2H()));
4459 if (vd.Is1H()) {
4460 VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
4461 Emit(NEON_FADDP_h_scalar | Rn(vn) | Rd(vd));
4462 } else {
4463 Emit(FPFormat(vd) | NEON_FADDP_scalar | Rn(vn) | Rd(vd));
4464 }
4465 }
4466
4467
fmaxp(const VRegister & vd,const VRegister & vn)4468 void Assembler::fmaxp(const VRegister& vd, const VRegister& vn) {
4469 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
4470 VIXL_ASSERT((vd.Is1S() && vn.Is2S()) || (vd.Is1D() && vn.Is2D()) ||
4471 (vd.Is1H() && vn.Is2H()));
4472 if (vd.Is1H()) {
4473 VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
4474 Emit(NEON_FMAXP_h_scalar | Rn(vn) | Rd(vd));
4475 } else {
4476 Emit(FPFormat(vd) | NEON_FMAXP_scalar | Rn(vn) | Rd(vd));
4477 }
4478 }
4479
4480
fminp(const VRegister & vd,const VRegister & vn)4481 void Assembler::fminp(const VRegister& vd, const VRegister& vn) {
4482 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
4483 VIXL_ASSERT((vd.Is1S() && vn.Is2S()) || (vd.Is1D() && vn.Is2D()) ||
4484 (vd.Is1H() && vn.Is2H()));
4485 if (vd.Is1H()) {
4486 VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
4487 Emit(NEON_FMINP_h_scalar | Rn(vn) | Rd(vd));
4488 } else {
4489 Emit(FPFormat(vd) | NEON_FMINP_scalar | Rn(vn) | Rd(vd));
4490 }
4491 }
4492
4493
fmaxnmp(const VRegister & vd,const VRegister & vn)4494 void Assembler::fmaxnmp(const VRegister& vd, const VRegister& vn) {
4495 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
4496 VIXL_ASSERT((vd.Is1S() && vn.Is2S()) || (vd.Is1D() && vn.Is2D()) ||
4497 (vd.Is1H() && vn.Is2H()));
4498 if (vd.Is1H()) {
4499 VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
4500 Emit(NEON_FMAXNMP_h_scalar | Rn(vn) | Rd(vd));
4501 } else {
4502 Emit(FPFormat(vd) | NEON_FMAXNMP_scalar | Rn(vn) | Rd(vd));
4503 }
4504 }
4505
4506
fminnmp(const VRegister & vd,const VRegister & vn)4507 void Assembler::fminnmp(const VRegister& vd, const VRegister& vn) {
4508 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
4509 VIXL_ASSERT((vd.Is1S() && vn.Is2S()) || (vd.Is1D() && vn.Is2D()) ||
4510 (vd.Is1H() && vn.Is2H()));
4511 if (vd.Is1H()) {
4512 VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
4513 Emit(NEON_FMINNMP_h_scalar | Rn(vn) | Rd(vd));
4514 } else {
4515 Emit(FPFormat(vd) | NEON_FMINNMP_scalar | Rn(vn) | Rd(vd));
4516 }
4517 }
4518
4519
4520 // v8.3 complex numbers - floating-point complex multiply accumulate.
fcmla(const VRegister & vd,const VRegister & vn,const VRegister & vm,int vm_index,int rot)4521 void Assembler::fcmla(const VRegister& vd,
4522 const VRegister& vn,
4523 const VRegister& vm,
4524 int vm_index,
4525 int rot) {
4526 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON, CPUFeatures::kFcma));
4527 VIXL_ASSERT(vd.IsVector() && AreSameFormat(vd, vn));
4528 VIXL_ASSERT((vm.IsH() && (vd.Is8H() || vd.Is4H())) ||
4529 (vm.IsS() && vd.Is4S()));
4530 if (vd.IsLaneSizeH()) VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
4531 int index_num_bits = vd.Is4S() ? 1 : 2;
4532 Emit(VFormat(vd) | Rm(vm) | NEON_FCMLA_byelement |
4533 ImmNEONHLM(vm_index, index_num_bits) | ImmRotFcmlaSca(rot) | Rn(vn) |
4534 Rd(vd));
4535 }
4536
4537
fcmla(const VRegister & vd,const VRegister & vn,const VRegister & vm,int rot)4538 void Assembler::fcmla(const VRegister& vd,
4539 const VRegister& vn,
4540 const VRegister& vm,
4541 int rot) {
4542 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON, CPUFeatures::kFcma));
4543 VIXL_ASSERT(AreSameFormat(vd, vn, vm));
4544 VIXL_ASSERT(vd.IsVector() && !vd.IsLaneSizeB());
4545 if (vd.IsLaneSizeH()) VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
4546 Emit(VFormat(vd) | Rm(vm) | NEON_FCMLA | ImmRotFcmlaVec(rot) | Rn(vn) |
4547 Rd(vd));
4548 }
4549
4550
4551 // v8.3 complex numbers - floating-point complex add.
fcadd(const VRegister & vd,const VRegister & vn,const VRegister & vm,int rot)4552 void Assembler::fcadd(const VRegister& vd,
4553 const VRegister& vn,
4554 const VRegister& vm,
4555 int rot) {
4556 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON, CPUFeatures::kFcma));
4557 VIXL_ASSERT(AreSameFormat(vd, vn, vm));
4558 VIXL_ASSERT(vd.IsVector() && !vd.IsLaneSizeB());
4559 if (vd.IsLaneSizeH()) VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
4560 Emit(VFormat(vd) | Rm(vm) | NEON_FCADD | ImmRotFcadd(rot) | Rn(vn) | Rd(vd));
4561 }
4562
4563
orr(const VRegister & vd,const int imm8,const int left_shift)4564 void Assembler::orr(const VRegister& vd, const int imm8, const int left_shift) {
4565 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4566 NEONModifiedImmShiftLsl(vd, imm8, left_shift, NEONModifiedImmediate_ORR);
4567 }
4568
4569
mov(const VRegister & vd,const VRegister & vn)4570 void Assembler::mov(const VRegister& vd, const VRegister& vn) {
4571 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4572 VIXL_ASSERT(AreSameFormat(vd, vn));
4573 if (vd.IsD()) {
4574 orr(vd.V8B(), vn.V8B(), vn.V8B());
4575 } else {
4576 VIXL_ASSERT(vd.IsQ());
4577 orr(vd.V16B(), vn.V16B(), vn.V16B());
4578 }
4579 }
4580
4581
bic(const VRegister & vd,const int imm8,const int left_shift)4582 void Assembler::bic(const VRegister& vd, const int imm8, const int left_shift) {
4583 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4584 NEONModifiedImmShiftLsl(vd, imm8, left_shift, NEONModifiedImmediate_BIC);
4585 }
4586
4587
movi(const VRegister & vd,const uint64_t imm,Shift shift,const int shift_amount)4588 void Assembler::movi(const VRegister& vd,
4589 const uint64_t imm,
4590 Shift shift,
4591 const int shift_amount) {
4592 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4593 VIXL_ASSERT((shift == LSL) || (shift == MSL));
4594 if (vd.Is2D() || vd.Is1D()) {
4595 VIXL_ASSERT(shift_amount == 0);
4596 int imm8 = 0;
4597 for (int i = 0; i < 8; ++i) {
4598 int byte = (imm >> (i * 8)) & 0xff;
4599 VIXL_ASSERT((byte == 0) || (byte == 0xff));
4600 if (byte == 0xff) {
4601 imm8 |= (1 << i);
4602 }
4603 }
4604 int q = vd.Is2D() ? NEON_Q : 0;
4605 Emit(q | NEONModImmOp(1) | NEONModifiedImmediate_MOVI |
4606 ImmNEONabcdefgh(imm8) | NEONCmode(0xe) | Rd(vd));
4607 } else if (shift == LSL) {
4608 VIXL_ASSERT(IsUint8(imm));
4609 NEONModifiedImmShiftLsl(vd,
4610 static_cast<int>(imm),
4611 shift_amount,
4612 NEONModifiedImmediate_MOVI);
4613 } else {
4614 VIXL_ASSERT(IsUint8(imm));
4615 NEONModifiedImmShiftMsl(vd,
4616 static_cast<int>(imm),
4617 shift_amount,
4618 NEONModifiedImmediate_MOVI);
4619 }
4620 }
4621
4622
mvn(const VRegister & vd,const VRegister & vn)4623 void Assembler::mvn(const VRegister& vd, const VRegister& vn) {
4624 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4625 VIXL_ASSERT(AreSameFormat(vd, vn));
4626 if (vd.IsD()) {
4627 not_(vd.V8B(), vn.V8B());
4628 } else {
4629 VIXL_ASSERT(vd.IsQ());
4630 not_(vd.V16B(), vn.V16B());
4631 }
4632 }
4633
4634
mvni(const VRegister & vd,const int imm8,Shift shift,const int shift_amount)4635 void Assembler::mvni(const VRegister& vd,
4636 const int imm8,
4637 Shift shift,
4638 const int shift_amount) {
4639 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4640 VIXL_ASSERT((shift == LSL) || (shift == MSL));
4641 if (shift == LSL) {
4642 NEONModifiedImmShiftLsl(vd, imm8, shift_amount, NEONModifiedImmediate_MVNI);
4643 } else {
4644 NEONModifiedImmShiftMsl(vd, imm8, shift_amount, NEONModifiedImmediate_MVNI);
4645 }
4646 }
4647
4648
NEONFPByElement(const VRegister & vd,const VRegister & vn,const VRegister & vm,int vm_index,NEONByIndexedElementOp vop,NEONByIndexedElementOp vop_half)4649 void Assembler::NEONFPByElement(const VRegister& vd,
4650 const VRegister& vn,
4651 const VRegister& vm,
4652 int vm_index,
4653 NEONByIndexedElementOp vop,
4654 NEONByIndexedElementOp vop_half) {
4655 VIXL_ASSERT(AreSameFormat(vd, vn));
4656 VIXL_ASSERT((vd.Is2S() && vm.Is1S()) || (vd.Is4S() && vm.Is1S()) ||
4657 (vd.Is1S() && vm.Is1S()) || (vd.Is2D() && vm.Is1D()) ||
4658 (vd.Is1D() && vm.Is1D()) || (vd.Is4H() && vm.Is1H()) ||
4659 (vd.Is8H() && vm.Is1H()) || (vd.Is1H() && vm.Is1H()));
4660 VIXL_ASSERT((vm.Is1S() && (vm_index < 4)) || (vm.Is1D() && (vm_index < 2)) ||
4661 (vm.Is1H() && (vm.GetCode() < 16) && (vm_index < 8)));
4662
4663 Instr op = vop;
4664 int index_num_bits;
4665 if (vm.Is1D()) {
4666 index_num_bits = 1;
4667 } else if (vm.Is1S()) {
4668 index_num_bits = 2;
4669 } else {
4670 index_num_bits = 3;
4671 op = vop_half;
4672 }
4673
4674 if (vd.IsScalar()) {
4675 op |= NEON_Q | NEONScalar;
4676 }
4677
4678 if (!vm.Is1H()) {
4679 op |= FPFormat(vd);
4680 } else if (vd.Is8H()) {
4681 op |= NEON_Q;
4682 }
4683
4684 Emit(op | ImmNEONHLM(vm_index, index_num_bits) | Rm(vm) | Rn(vn) | Rd(vd));
4685 }
4686
4687
NEONByElement(const VRegister & vd,const VRegister & vn,const VRegister & vm,int vm_index,NEONByIndexedElementOp vop)4688 void Assembler::NEONByElement(const VRegister& vd,
4689 const VRegister& vn,
4690 const VRegister& vm,
4691 int vm_index,
4692 NEONByIndexedElementOp vop) {
4693 VIXL_ASSERT(AreSameFormat(vd, vn));
4694 VIXL_ASSERT((vd.Is4H() && vm.Is1H()) || (vd.Is8H() && vm.Is1H()) ||
4695 (vd.Is1H() && vm.Is1H()) || (vd.Is2S() && vm.Is1S()) ||
4696 (vd.Is4S() && vm.Is1S()) || (vd.Is1S() && vm.Is1S()));
4697 VIXL_ASSERT((vm.Is1H() && (vm.GetCode() < 16) && (vm_index < 8)) ||
4698 (vm.Is1S() && (vm_index < 4)));
4699
4700 Instr format, op = vop;
4701 int index_num_bits = vm.Is1H() ? 3 : 2;
4702 if (vd.IsScalar()) {
4703 op |= NEONScalar | NEON_Q;
4704 format = SFormat(vn);
4705 } else {
4706 format = VFormat(vn);
4707 }
4708 Emit(format | op | ImmNEONHLM(vm_index, index_num_bits) | Rm(vm) | Rn(vn) |
4709 Rd(vd));
4710 }
4711
4712
NEONByElementL(const VRegister & vd,const VRegister & vn,const VRegister & vm,int vm_index,NEONByIndexedElementOp vop)4713 void Assembler::NEONByElementL(const VRegister& vd,
4714 const VRegister& vn,
4715 const VRegister& vm,
4716 int vm_index,
4717 NEONByIndexedElementOp vop) {
4718 VIXL_ASSERT((vd.Is4S() && vn.Is4H() && vm.Is1H()) ||
4719 (vd.Is4S() && vn.Is8H() && vm.Is1H()) ||
4720 (vd.Is1S() && vn.Is1H() && vm.Is1H()) ||
4721 (vd.Is2D() && vn.Is2S() && vm.Is1S()) ||
4722 (vd.Is2D() && vn.Is4S() && vm.Is1S()) ||
4723 (vd.Is1D() && vn.Is1S() && vm.Is1S()));
4724
4725 VIXL_ASSERT((vm.Is1H() && (vm.GetCode() < 16) && (vm_index < 8)) ||
4726 (vm.Is1S() && (vm_index < 4)));
4727
4728 Instr format, op = vop;
4729 int index_num_bits = vm.Is1H() ? 3 : 2;
4730 if (vd.IsScalar()) {
4731 op |= NEONScalar | NEON_Q;
4732 format = SFormat(vn);
4733 } else {
4734 format = VFormat(vn);
4735 }
4736 Emit(format | op | ImmNEONHLM(vm_index, index_num_bits) | Rm(vm) | Rn(vn) |
4737 Rd(vd));
4738 }
4739
4740
sdot(const VRegister & vd,const VRegister & vn,const VRegister & vm,int vm_index)4741 void Assembler::sdot(const VRegister& vd,
4742 const VRegister& vn,
4743 const VRegister& vm,
4744 int vm_index) {
4745 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON, CPUFeatures::kDotProduct));
4746 VIXL_ASSERT((vd.Is2S() && vn.Is8B() && vm.Is1S4B()) ||
4747 (vd.Is4S() && vn.Is16B() && vm.Is1S4B()));
4748
4749 int index_num_bits = 2;
4750 Emit(VFormat(vd) | NEON_SDOT_byelement |
4751 ImmNEONHLM(vm_index, index_num_bits) | Rm(vm) | Rn(vn) | Rd(vd));
4752 }
4753
4754
udot(const VRegister & vd,const VRegister & vn,const VRegister & vm,int vm_index)4755 void Assembler::udot(const VRegister& vd,
4756 const VRegister& vn,
4757 const VRegister& vm,
4758 int vm_index) {
4759 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON, CPUFeatures::kDotProduct));
4760 VIXL_ASSERT((vd.Is2S() && vn.Is8B() && vm.Is1S4B()) ||
4761 (vd.Is4S() && vn.Is16B() && vm.Is1S4B()));
4762
4763 int index_num_bits = 2;
4764 Emit(VFormat(vd) | NEON_UDOT_byelement |
4765 ImmNEONHLM(vm_index, index_num_bits) | Rm(vm) | Rn(vn) | Rd(vd));
4766 }
4767
sudot(const VRegister & vd,const VRegister & vn,const VRegister & vm,int vm_index)4768 void Assembler::sudot(const VRegister& vd,
4769 const VRegister& vn,
4770 const VRegister& vm,
4771 int vm_index) {
4772 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON, CPUFeatures::kI8MM));
4773 VIXL_ASSERT((vd.Is2S() && vn.Is8B() && vm.Is1S4B()) ||
4774 (vd.Is4S() && vn.Is16B() && vm.Is1S4B()));
4775 int q = vd.Is4S() ? (1U << NEONQ_offset) : 0;
4776 int index_num_bits = 2;
4777 Emit(q | 0x0f00f000 | ImmNEONHLM(vm_index, index_num_bits) | Rm(vm) | Rn(vn) |
4778 Rd(vd));
4779 }
4780
4781
usdot(const VRegister & vd,const VRegister & vn,const VRegister & vm,int vm_index)4782 void Assembler::usdot(const VRegister& vd,
4783 const VRegister& vn,
4784 const VRegister& vm,
4785 int vm_index) {
4786 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON, CPUFeatures::kI8MM));
4787 VIXL_ASSERT((vd.Is2S() && vn.Is8B() && vm.Is1S4B()) ||
4788 (vd.Is4S() && vn.Is16B() && vm.Is1S4B()));
4789 int q = vd.Is4S() ? (1U << NEONQ_offset) : 0;
4790 int index_num_bits = 2;
4791 Emit(q | 0x0f80f000 | ImmNEONHLM(vm_index, index_num_bits) | Rm(vm) | Rn(vn) |
4792 Rd(vd));
4793 }
4794
4795 // clang-format off
4796 #define NEON_BYELEMENT_LIST(V) \
4797 V(mul, NEON_MUL_byelement, vn.IsVector()) \
4798 V(mla, NEON_MLA_byelement, vn.IsVector()) \
4799 V(mls, NEON_MLS_byelement, vn.IsVector()) \
4800 V(sqdmulh, NEON_SQDMULH_byelement, true) \
4801 V(sqrdmulh, NEON_SQRDMULH_byelement, true) \
4802 // clang-format on
4803
4804 #define VIXL_DEFINE_ASM_FUNC(FN, OP, AS) \
4805 void Assembler::FN(const VRegister& vd, \
4806 const VRegister& vn, \
4807 const VRegister& vm, \
4808 int vm_index) { \
4809 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON)); \
4810 VIXL_ASSERT(AS); \
4811 NEONByElement(vd, vn, vm, vm_index, OP); \
4812 }
4813 NEON_BYELEMENT_LIST(VIXL_DEFINE_ASM_FUNC)
4814 #undef VIXL_DEFINE_ASM_FUNC
4815
4816
4817 // clang-format off
4818 #define NEON_BYELEMENT_RDM_LIST(V) \
4819 V(sqrdmlah, NEON_SQRDMLAH_byelement) \
4820 V(sqrdmlsh, NEON_SQRDMLSH_byelement)
4821 // clang-format on
4822
4823 #define VIXL_DEFINE_ASM_FUNC(FN, OP) \
4824 void Assembler::FN(const VRegister& vd, \
4825 const VRegister& vn, \
4826 const VRegister& vm, \
4827 int vm_index) { \
4828 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON, CPUFeatures::kRDM)); \
4829 NEONByElement(vd, vn, vm, vm_index, OP); \
4830 }
NEON_BYELEMENT_RDM_LIST(VIXL_DEFINE_ASM_FUNC)4831 NEON_BYELEMENT_RDM_LIST(VIXL_DEFINE_ASM_FUNC)
4832 #undef VIXL_DEFINE_ASM_FUNC
4833
4834
4835 // clang-format off
4836 #define NEON_FPBYELEMENT_LIST(V) \
4837 V(fmul, NEON_FMUL_byelement, NEON_FMUL_H_byelement) \
4838 V(fmla, NEON_FMLA_byelement, NEON_FMLA_H_byelement) \
4839 V(fmls, NEON_FMLS_byelement, NEON_FMLS_H_byelement) \
4840 V(fmulx, NEON_FMULX_byelement, NEON_FMULX_H_byelement)
4841 // clang-format on
4842
4843 #define VIXL_DEFINE_ASM_FUNC(FN, OP, OP_H) \
4844 void Assembler::FN(const VRegister& vd, \
4845 const VRegister& vn, \
4846 const VRegister& vm, \
4847 int vm_index) { \
4848 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON)); \
4849 if (vd.IsLaneSizeH()) VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf)); \
4850 NEONFPByElement(vd, vn, vm, vm_index, OP, OP_H); \
4851 }
4852 NEON_FPBYELEMENT_LIST(VIXL_DEFINE_ASM_FUNC)
4853 #undef VIXL_DEFINE_ASM_FUNC
4854
4855
4856 // clang-format off
4857 #define NEON_BYELEMENT_LONG_LIST(V) \
4858 V(sqdmull, NEON_SQDMULL_byelement, vn.IsScalar() || vn.IsD()) \
4859 V(sqdmull2, NEON_SQDMULL_byelement, vn.IsVector() && vn.IsQ()) \
4860 V(sqdmlal, NEON_SQDMLAL_byelement, vn.IsScalar() || vn.IsD()) \
4861 V(sqdmlal2, NEON_SQDMLAL_byelement, vn.IsVector() && vn.IsQ()) \
4862 V(sqdmlsl, NEON_SQDMLSL_byelement, vn.IsScalar() || vn.IsD()) \
4863 V(sqdmlsl2, NEON_SQDMLSL_byelement, vn.IsVector() && vn.IsQ()) \
4864 V(smull, NEON_SMULL_byelement, vn.IsVector() && vn.IsD()) \
4865 V(smull2, NEON_SMULL_byelement, vn.IsVector() && vn.IsQ()) \
4866 V(umull, NEON_UMULL_byelement, vn.IsVector() && vn.IsD()) \
4867 V(umull2, NEON_UMULL_byelement, vn.IsVector() && vn.IsQ()) \
4868 V(smlal, NEON_SMLAL_byelement, vn.IsVector() && vn.IsD()) \
4869 V(smlal2, NEON_SMLAL_byelement, vn.IsVector() && vn.IsQ()) \
4870 V(umlal, NEON_UMLAL_byelement, vn.IsVector() && vn.IsD()) \
4871 V(umlal2, NEON_UMLAL_byelement, vn.IsVector() && vn.IsQ()) \
4872 V(smlsl, NEON_SMLSL_byelement, vn.IsVector() && vn.IsD()) \
4873 V(smlsl2, NEON_SMLSL_byelement, vn.IsVector() && vn.IsQ()) \
4874 V(umlsl, NEON_UMLSL_byelement, vn.IsVector() && vn.IsD()) \
4875 V(umlsl2, NEON_UMLSL_byelement, vn.IsVector() && vn.IsQ())
4876 // clang-format on
4877
4878
4879 #define VIXL_DEFINE_ASM_FUNC(FN, OP, AS) \
4880 void Assembler::FN(const VRegister& vd, \
4881 const VRegister& vn, \
4882 const VRegister& vm, \
4883 int vm_index) { \
4884 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON)); \
4885 VIXL_ASSERT(AS); \
4886 NEONByElementL(vd, vn, vm, vm_index, OP); \
4887 }
4888 NEON_BYELEMENT_LONG_LIST(VIXL_DEFINE_ASM_FUNC)
4889 #undef VIXL_DEFINE_ASM_FUNC
4890
4891
4892 // clang-format off
4893 #define NEON_BYELEMENT_FHM_LIST(V) \
4894 V(fmlal, NEON_FMLAL_H_byelement) \
4895 V(fmlal2, NEON_FMLAL2_H_byelement) \
4896 V(fmlsl, NEON_FMLSL_H_byelement) \
4897 V(fmlsl2, NEON_FMLSL2_H_byelement)
4898 // clang-format on
4899
4900
4901 #define VIXL_DEFINE_ASM_FUNC(FN, OP) \
4902 void Assembler::FN(const VRegister& vd, \
4903 const VRegister& vn, \
4904 const VRegister& vm, \
4905 int vm_index) { \
4906 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON, \
4907 CPUFeatures::kFP, \
4908 CPUFeatures::kNEONHalf, \
4909 CPUFeatures::kFHM)); \
4910 VIXL_ASSERT((vd.Is2S() && vn.Is2H()) || (vd.Is4S() && vn.Is4H())); \
4911 VIXL_ASSERT(vm.IsH()); \
4912 VIXL_ASSERT((vm_index >= 0) && (vm_index < 8)); \
4913 /* Vm itself can only be in the bottom 16 registers. */ \
4914 VIXL_ASSERT(vm.GetCode() < 16); \
4915 Emit(FPFormat(vd) | OP | Rd(vd) | Rn(vn) | Rm(vm) | \
4916 ImmNEONHLM(vm_index, 3)); \
4917 }
4918 NEON_BYELEMENT_FHM_LIST(VIXL_DEFINE_ASM_FUNC)
4919 #undef VIXL_DEFINE_ASM_FUNC
4920
4921 void Assembler::suqadd(const VRegister& vd, const VRegister& vn) {
4922 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4923 NEON2RegMisc(vd, vn, NEON_SUQADD);
4924 }
4925
4926
usqadd(const VRegister & vd,const VRegister & vn)4927 void Assembler::usqadd(const VRegister& vd, const VRegister& vn) {
4928 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4929 NEON2RegMisc(vd, vn, NEON_USQADD);
4930 }
4931
4932
abs(const VRegister & vd,const VRegister & vn)4933 void Assembler::abs(const VRegister& vd, const VRegister& vn) {
4934 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4935 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
4936 NEON2RegMisc(vd, vn, NEON_ABS);
4937 }
4938
4939
sqabs(const VRegister & vd,const VRegister & vn)4940 void Assembler::sqabs(const VRegister& vd, const VRegister& vn) {
4941 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4942 NEON2RegMisc(vd, vn, NEON_SQABS);
4943 }
4944
4945
neg(const VRegister & vd,const VRegister & vn)4946 void Assembler::neg(const VRegister& vd, const VRegister& vn) {
4947 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4948 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
4949 NEON2RegMisc(vd, vn, NEON_NEG);
4950 }
4951
4952
sqneg(const VRegister & vd,const VRegister & vn)4953 void Assembler::sqneg(const VRegister& vd, const VRegister& vn) {
4954 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4955 NEON2RegMisc(vd, vn, NEON_SQNEG);
4956 }
4957
4958
NEONXtn(const VRegister & vd,const VRegister & vn,NEON2RegMiscOp vop)4959 void Assembler::NEONXtn(const VRegister& vd,
4960 const VRegister& vn,
4961 NEON2RegMiscOp vop) {
4962 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4963 Instr format, op = vop;
4964 if (vd.IsScalar()) {
4965 VIXL_ASSERT((vd.Is1B() && vn.Is1H()) || (vd.Is1H() && vn.Is1S()) ||
4966 (vd.Is1S() && vn.Is1D()));
4967 op |= NEON_Q | NEONScalar;
4968 format = SFormat(vd);
4969 } else {
4970 VIXL_ASSERT((vd.Is8B() && vn.Is8H()) || (vd.Is4H() && vn.Is4S()) ||
4971 (vd.Is2S() && vn.Is2D()) || (vd.Is16B() && vn.Is8H()) ||
4972 (vd.Is8H() && vn.Is4S()) || (vd.Is4S() && vn.Is2D()));
4973 format = VFormat(vd);
4974 }
4975 Emit(format | op | Rn(vn) | Rd(vd));
4976 }
4977
4978
xtn(const VRegister & vd,const VRegister & vn)4979 void Assembler::xtn(const VRegister& vd, const VRegister& vn) {
4980 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4981 VIXL_ASSERT(vd.IsVector() && vd.IsD());
4982 NEONXtn(vd, vn, NEON_XTN);
4983 }
4984
4985
xtn2(const VRegister & vd,const VRegister & vn)4986 void Assembler::xtn2(const VRegister& vd, const VRegister& vn) {
4987 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4988 VIXL_ASSERT(vd.IsVector() && vd.IsQ());
4989 NEONXtn(vd, vn, NEON_XTN);
4990 }
4991
4992
sqxtn(const VRegister & vd,const VRegister & vn)4993 void Assembler::sqxtn(const VRegister& vd, const VRegister& vn) {
4994 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4995 VIXL_ASSERT(vd.IsScalar() || vd.IsD());
4996 NEONXtn(vd, vn, NEON_SQXTN);
4997 }
4998
4999
sqxtn2(const VRegister & vd,const VRegister & vn)5000 void Assembler::sqxtn2(const VRegister& vd, const VRegister& vn) {
5001 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5002 VIXL_ASSERT(vd.IsVector() && vd.IsQ());
5003 NEONXtn(vd, vn, NEON_SQXTN);
5004 }
5005
5006
sqxtun(const VRegister & vd,const VRegister & vn)5007 void Assembler::sqxtun(const VRegister& vd, const VRegister& vn) {
5008 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5009 VIXL_ASSERT(vd.IsScalar() || vd.IsD());
5010 NEONXtn(vd, vn, NEON_SQXTUN);
5011 }
5012
5013
sqxtun2(const VRegister & vd,const VRegister & vn)5014 void Assembler::sqxtun2(const VRegister& vd, const VRegister& vn) {
5015 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5016 VIXL_ASSERT(vd.IsVector() && vd.IsQ());
5017 NEONXtn(vd, vn, NEON_SQXTUN);
5018 }
5019
5020
uqxtn(const VRegister & vd,const VRegister & vn)5021 void Assembler::uqxtn(const VRegister& vd, const VRegister& vn) {
5022 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5023 VIXL_ASSERT(vd.IsScalar() || vd.IsD());
5024 NEONXtn(vd, vn, NEON_UQXTN);
5025 }
5026
5027
uqxtn2(const VRegister & vd,const VRegister & vn)5028 void Assembler::uqxtn2(const VRegister& vd, const VRegister& vn) {
5029 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5030 VIXL_ASSERT(vd.IsVector() && vd.IsQ());
5031 NEONXtn(vd, vn, NEON_UQXTN);
5032 }
5033
5034
5035 // NEON NOT and RBIT are distinguised by bit 22, the bottom bit of "size".
not_(const VRegister & vd,const VRegister & vn)5036 void Assembler::not_(const VRegister& vd, const VRegister& vn) {
5037 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5038 VIXL_ASSERT(AreSameFormat(vd, vn));
5039 VIXL_ASSERT(vd.Is8B() || vd.Is16B());
5040 Emit(VFormat(vd) | NEON_RBIT_NOT | Rn(vn) | Rd(vd));
5041 }
5042
5043
rbit(const VRegister & vd,const VRegister & vn)5044 void Assembler::rbit(const VRegister& vd, const VRegister& vn) {
5045 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5046 VIXL_ASSERT(AreSameFormat(vd, vn));
5047 VIXL_ASSERT(vd.Is8B() || vd.Is16B());
5048 Emit(VFormat(vn) | (1 << NEONSize_offset) | NEON_RBIT_NOT | Rn(vn) | Rd(vd));
5049 }
5050
5051
ext(const VRegister & vd,const VRegister & vn,const VRegister & vm,int index)5052 void Assembler::ext(const VRegister& vd,
5053 const VRegister& vn,
5054 const VRegister& vm,
5055 int index) {
5056 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5057 VIXL_ASSERT(AreSameFormat(vd, vn, vm));
5058 VIXL_ASSERT(vd.Is8B() || vd.Is16B());
5059 VIXL_ASSERT((0 <= index) && (index < vd.GetLanes()));
5060 Emit(VFormat(vd) | NEON_EXT | Rm(vm) | ImmNEONExt(index) | Rn(vn) | Rd(vd));
5061 }
5062
5063
dup(const VRegister & vd,const VRegister & vn,int vn_index)5064 void Assembler::dup(const VRegister& vd, const VRegister& vn, int vn_index) {
5065 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5066 Instr q, scalar;
5067
5068 // We support vn arguments of the form vn.VxT() or vn.T(), where x is the
5069 // number of lanes, and T is b, h, s or d.
5070 int lane_size = vn.GetLaneSizeInBytes();
5071 NEONFormatField format;
5072 switch (lane_size) {
5073 case 1:
5074 format = NEON_16B;
5075 break;
5076 case 2:
5077 format = NEON_8H;
5078 break;
5079 case 4:
5080 format = NEON_4S;
5081 break;
5082 default:
5083 VIXL_ASSERT(lane_size == 8);
5084 format = NEON_2D;
5085 break;
5086 }
5087
5088 if (vd.IsScalar()) {
5089 q = NEON_Q;
5090 scalar = NEONScalar;
5091 } else {
5092 VIXL_ASSERT(!vd.Is1D());
5093 q = vd.IsD() ? 0 : NEON_Q;
5094 scalar = 0;
5095 }
5096 Emit(q | scalar | NEON_DUP_ELEMENT | ImmNEON5(format, vn_index) | Rn(vn) |
5097 Rd(vd));
5098 }
5099
5100
mov(const VRegister & vd,const VRegister & vn,int vn_index)5101 void Assembler::mov(const VRegister& vd, const VRegister& vn, int vn_index) {
5102 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5103 VIXL_ASSERT(vd.IsScalar());
5104 dup(vd, vn, vn_index);
5105 }
5106
5107
dup(const VRegister & vd,const Register & rn)5108 void Assembler::dup(const VRegister& vd, const Register& rn) {
5109 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5110 VIXL_ASSERT(!vd.Is1D());
5111 VIXL_ASSERT(vd.Is2D() == rn.IsX());
5112 int q = vd.IsD() ? 0 : NEON_Q;
5113 Emit(q | NEON_DUP_GENERAL | ImmNEON5(VFormat(vd), 0) | Rn(rn) | Rd(vd));
5114 }
5115
5116
ins(const VRegister & vd,int vd_index,const VRegister & vn,int vn_index)5117 void Assembler::ins(const VRegister& vd,
5118 int vd_index,
5119 const VRegister& vn,
5120 int vn_index) {
5121 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5122 VIXL_ASSERT(AreSameFormat(vd, vn));
5123 // We support vd arguments of the form vd.VxT() or vd.T(), where x is the
5124 // number of lanes, and T is b, h, s or d.
5125 int lane_size = vd.GetLaneSizeInBytes();
5126 NEONFormatField format;
5127 switch (lane_size) {
5128 case 1:
5129 format = NEON_16B;
5130 break;
5131 case 2:
5132 format = NEON_8H;
5133 break;
5134 case 4:
5135 format = NEON_4S;
5136 break;
5137 default:
5138 VIXL_ASSERT(lane_size == 8);
5139 format = NEON_2D;
5140 break;
5141 }
5142
5143 VIXL_ASSERT(
5144 (0 <= vd_index) &&
5145 (vd_index < LaneCountFromFormat(static_cast<VectorFormat>(format))));
5146 VIXL_ASSERT(
5147 (0 <= vn_index) &&
5148 (vn_index < LaneCountFromFormat(static_cast<VectorFormat>(format))));
5149 Emit(NEON_INS_ELEMENT | ImmNEON5(format, vd_index) |
5150 ImmNEON4(format, vn_index) | Rn(vn) | Rd(vd));
5151 }
5152
5153
mov(const VRegister & vd,int vd_index,const VRegister & vn,int vn_index)5154 void Assembler::mov(const VRegister& vd,
5155 int vd_index,
5156 const VRegister& vn,
5157 int vn_index) {
5158 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5159 ins(vd, vd_index, vn, vn_index);
5160 }
5161
5162
ins(const VRegister & vd,int vd_index,const Register & rn)5163 void Assembler::ins(const VRegister& vd, int vd_index, const Register& rn) {
5164 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5165 // We support vd arguments of the form vd.VxT() or vd.T(), where x is the
5166 // number of lanes, and T is b, h, s or d.
5167 int lane_size = vd.GetLaneSizeInBytes();
5168 NEONFormatField format;
5169 switch (lane_size) {
5170 case 1:
5171 format = NEON_16B;
5172 VIXL_ASSERT(rn.IsW());
5173 break;
5174 case 2:
5175 format = NEON_8H;
5176 VIXL_ASSERT(rn.IsW());
5177 break;
5178 case 4:
5179 format = NEON_4S;
5180 VIXL_ASSERT(rn.IsW());
5181 break;
5182 default:
5183 VIXL_ASSERT(lane_size == 8);
5184 VIXL_ASSERT(rn.IsX());
5185 format = NEON_2D;
5186 break;
5187 }
5188
5189 VIXL_ASSERT(
5190 (0 <= vd_index) &&
5191 (vd_index < LaneCountFromFormat(static_cast<VectorFormat>(format))));
5192 Emit(NEON_INS_GENERAL | ImmNEON5(format, vd_index) | Rn(rn) | Rd(vd));
5193 }
5194
5195
mov(const VRegister & vd,int vd_index,const Register & rn)5196 void Assembler::mov(const VRegister& vd, int vd_index, const Register& rn) {
5197 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5198 ins(vd, vd_index, rn);
5199 }
5200
5201
umov(const Register & rd,const VRegister & vn,int vn_index)5202 void Assembler::umov(const Register& rd, const VRegister& vn, int vn_index) {
5203 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5204 // We support vn arguments of the form vn.VxT() or vn.T(), where x is the
5205 // number of lanes, and T is b, h, s or d.
5206 int lane_size = vn.GetLaneSizeInBytes();
5207 NEONFormatField format;
5208 Instr q = 0;
5209 switch (lane_size) {
5210 case 1:
5211 format = NEON_16B;
5212 VIXL_ASSERT(rd.IsW());
5213 break;
5214 case 2:
5215 format = NEON_8H;
5216 VIXL_ASSERT(rd.IsW());
5217 break;
5218 case 4:
5219 format = NEON_4S;
5220 VIXL_ASSERT(rd.IsW());
5221 break;
5222 default:
5223 VIXL_ASSERT(lane_size == 8);
5224 VIXL_ASSERT(rd.IsX());
5225 format = NEON_2D;
5226 q = NEON_Q;
5227 break;
5228 }
5229
5230 VIXL_ASSERT(
5231 (0 <= vn_index) &&
5232 (vn_index < LaneCountFromFormat(static_cast<VectorFormat>(format))));
5233 Emit(q | NEON_UMOV | ImmNEON5(format, vn_index) | Rn(vn) | Rd(rd));
5234 }
5235
5236
mov(const Register & rd,const VRegister & vn,int vn_index)5237 void Assembler::mov(const Register& rd, const VRegister& vn, int vn_index) {
5238 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5239 VIXL_ASSERT(vn.GetSizeInBytes() >= 4);
5240 umov(rd, vn, vn_index);
5241 }
5242
5243
smov(const Register & rd,const VRegister & vn,int vn_index)5244 void Assembler::smov(const Register& rd, const VRegister& vn, int vn_index) {
5245 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5246 // We support vn arguments of the form vn.VxT() or vn.T(), where x is the
5247 // number of lanes, and T is b, h, s.
5248 int lane_size = vn.GetLaneSizeInBytes();
5249 NEONFormatField format;
5250 Instr q = 0;
5251 VIXL_ASSERT(lane_size != 8);
5252 switch (lane_size) {
5253 case 1:
5254 format = NEON_16B;
5255 break;
5256 case 2:
5257 format = NEON_8H;
5258 break;
5259 default:
5260 VIXL_ASSERT(lane_size == 4);
5261 VIXL_ASSERT(rd.IsX());
5262 format = NEON_4S;
5263 break;
5264 }
5265 q = rd.IsW() ? 0 : NEON_Q;
5266 VIXL_ASSERT(
5267 (0 <= vn_index) &&
5268 (vn_index < LaneCountFromFormat(static_cast<VectorFormat>(format))));
5269 Emit(q | NEON_SMOV | ImmNEON5(format, vn_index) | Rn(vn) | Rd(rd));
5270 }
5271
5272
cls(const VRegister & vd,const VRegister & vn)5273 void Assembler::cls(const VRegister& vd, const VRegister& vn) {
5274 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5275 VIXL_ASSERT(AreSameFormat(vd, vn));
5276 VIXL_ASSERT(!vd.Is1D() && !vd.Is2D());
5277 Emit(VFormat(vn) | NEON_CLS | Rn(vn) | Rd(vd));
5278 }
5279
5280
clz(const VRegister & vd,const VRegister & vn)5281 void Assembler::clz(const VRegister& vd, const VRegister& vn) {
5282 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5283 VIXL_ASSERT(AreSameFormat(vd, vn));
5284 VIXL_ASSERT(!vd.Is1D() && !vd.Is2D());
5285 Emit(VFormat(vn) | NEON_CLZ | Rn(vn) | Rd(vd));
5286 }
5287
5288
cnt(const VRegister & vd,const VRegister & vn)5289 void Assembler::cnt(const VRegister& vd, const VRegister& vn) {
5290 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5291 VIXL_ASSERT(AreSameFormat(vd, vn));
5292 VIXL_ASSERT(vd.Is8B() || vd.Is16B());
5293 Emit(VFormat(vn) | NEON_CNT | Rn(vn) | Rd(vd));
5294 }
5295
5296
rev16(const VRegister & vd,const VRegister & vn)5297 void Assembler::rev16(const VRegister& vd, const VRegister& vn) {
5298 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5299 VIXL_ASSERT(AreSameFormat(vd, vn));
5300 VIXL_ASSERT(vd.Is8B() || vd.Is16B());
5301 Emit(VFormat(vn) | NEON_REV16 | Rn(vn) | Rd(vd));
5302 }
5303
5304
rev32(const VRegister & vd,const VRegister & vn)5305 void Assembler::rev32(const VRegister& vd, const VRegister& vn) {
5306 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5307 VIXL_ASSERT(AreSameFormat(vd, vn));
5308 VIXL_ASSERT(vd.Is8B() || vd.Is16B() || vd.Is4H() || vd.Is8H());
5309 Emit(VFormat(vn) | NEON_REV32 | Rn(vn) | Rd(vd));
5310 }
5311
5312
rev64(const VRegister & vd,const VRegister & vn)5313 void Assembler::rev64(const VRegister& vd, const VRegister& vn) {
5314 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5315 VIXL_ASSERT(AreSameFormat(vd, vn));
5316 VIXL_ASSERT(!vd.Is1D() && !vd.Is2D());
5317 Emit(VFormat(vn) | NEON_REV64 | Rn(vn) | Rd(vd));
5318 }
5319
5320
ursqrte(const VRegister & vd,const VRegister & vn)5321 void Assembler::ursqrte(const VRegister& vd, const VRegister& vn) {
5322 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5323 VIXL_ASSERT(AreSameFormat(vd, vn));
5324 VIXL_ASSERT(vd.Is2S() || vd.Is4S());
5325 Emit(VFormat(vn) | NEON_URSQRTE | Rn(vn) | Rd(vd));
5326 }
5327
5328
urecpe(const VRegister & vd,const VRegister & vn)5329 void Assembler::urecpe(const VRegister& vd, const VRegister& vn) {
5330 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5331 VIXL_ASSERT(AreSameFormat(vd, vn));
5332 VIXL_ASSERT(vd.Is2S() || vd.Is4S());
5333 Emit(VFormat(vn) | NEON_URECPE | Rn(vn) | Rd(vd));
5334 }
5335
5336
NEONAddlp(const VRegister & vd,const VRegister & vn,NEON2RegMiscOp op)5337 void Assembler::NEONAddlp(const VRegister& vd,
5338 const VRegister& vn,
5339 NEON2RegMiscOp op) {
5340 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5341 VIXL_ASSERT((op == NEON_SADDLP) || (op == NEON_UADDLP) ||
5342 (op == NEON_SADALP) || (op == NEON_UADALP));
5343
5344 VIXL_ASSERT((vn.Is8B() && vd.Is4H()) || (vn.Is4H() && vd.Is2S()) ||
5345 (vn.Is2S() && vd.Is1D()) || (vn.Is16B() && vd.Is8H()) ||
5346 (vn.Is8H() && vd.Is4S()) || (vn.Is4S() && vd.Is2D()));
5347 Emit(VFormat(vn) | op | Rn(vn) | Rd(vd));
5348 }
5349
5350
saddlp(const VRegister & vd,const VRegister & vn)5351 void Assembler::saddlp(const VRegister& vd, const VRegister& vn) {
5352 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5353 NEONAddlp(vd, vn, NEON_SADDLP);
5354 }
5355
5356
uaddlp(const VRegister & vd,const VRegister & vn)5357 void Assembler::uaddlp(const VRegister& vd, const VRegister& vn) {
5358 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5359 NEONAddlp(vd, vn, NEON_UADDLP);
5360 }
5361
5362
sadalp(const VRegister & vd,const VRegister & vn)5363 void Assembler::sadalp(const VRegister& vd, const VRegister& vn) {
5364 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5365 NEONAddlp(vd, vn, NEON_SADALP);
5366 }
5367
5368
uadalp(const VRegister & vd,const VRegister & vn)5369 void Assembler::uadalp(const VRegister& vd, const VRegister& vn) {
5370 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5371 NEONAddlp(vd, vn, NEON_UADALP);
5372 }
5373
5374
NEONAcrossLanesL(const VRegister & vd,const VRegister & vn,NEONAcrossLanesOp op)5375 void Assembler::NEONAcrossLanesL(const VRegister& vd,
5376 const VRegister& vn,
5377 NEONAcrossLanesOp op) {
5378 VIXL_ASSERT((vn.Is8B() && vd.Is1H()) || (vn.Is16B() && vd.Is1H()) ||
5379 (vn.Is4H() && vd.Is1S()) || (vn.Is8H() && vd.Is1S()) ||
5380 (vn.Is4S() && vd.Is1D()));
5381 Emit(VFormat(vn) | op | Rn(vn) | Rd(vd));
5382 }
5383
5384
saddlv(const VRegister & vd,const VRegister & vn)5385 void Assembler::saddlv(const VRegister& vd, const VRegister& vn) {
5386 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5387 NEONAcrossLanesL(vd, vn, NEON_SADDLV);
5388 }
5389
5390
uaddlv(const VRegister & vd,const VRegister & vn)5391 void Assembler::uaddlv(const VRegister& vd, const VRegister& vn) {
5392 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5393 NEONAcrossLanesL(vd, vn, NEON_UADDLV);
5394 }
5395
5396
NEONAcrossLanes(const VRegister & vd,const VRegister & vn,NEONAcrossLanesOp op,Instr op_half)5397 void Assembler::NEONAcrossLanes(const VRegister& vd,
5398 const VRegister& vn,
5399 NEONAcrossLanesOp op,
5400 Instr op_half) {
5401 VIXL_ASSERT((vn.Is8B() && vd.Is1B()) || (vn.Is16B() && vd.Is1B()) ||
5402 (vn.Is4H() && vd.Is1H()) || (vn.Is8H() && vd.Is1H()) ||
5403 (vn.Is4S() && vd.Is1S()));
5404 if ((op & NEONAcrossLanesFPFMask) == NEONAcrossLanesFPFixed) {
5405 if (vd.Is1H()) {
5406 VIXL_ASSERT(op_half != 0);
5407 Instr vop = op_half;
5408 if (vn.Is8H()) {
5409 vop |= NEON_Q;
5410 }
5411 Emit(vop | Rn(vn) | Rd(vd));
5412 } else {
5413 Emit(FPFormat(vn) | op | Rn(vn) | Rd(vd));
5414 }
5415 } else {
5416 Emit(VFormat(vn) | op | Rn(vn) | Rd(vd));
5417 }
5418 }
5419
5420 // clang-format off
5421 #define NEON_ACROSSLANES_LIST(V) \
5422 V(addv, NEON_ADDV) \
5423 V(smaxv, NEON_SMAXV) \
5424 V(sminv, NEON_SMINV) \
5425 V(umaxv, NEON_UMAXV) \
5426 V(uminv, NEON_UMINV)
5427 // clang-format on
5428
5429 #define VIXL_DEFINE_ASM_FUNC(FN, OP) \
5430 void Assembler::FN(const VRegister& vd, const VRegister& vn) { \
5431 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON)); \
5432 NEONAcrossLanes(vd, vn, OP, 0); \
5433 }
5434 NEON_ACROSSLANES_LIST(VIXL_DEFINE_ASM_FUNC)
5435 #undef VIXL_DEFINE_ASM_FUNC
5436
5437
5438 // clang-format off
5439 #define NEON_ACROSSLANES_FP_LIST(V) \
5440 V(fmaxv, NEON_FMAXV, NEON_FMAXV_H) \
5441 V(fminv, NEON_FMINV, NEON_FMINV_H) \
5442 V(fmaxnmv, NEON_FMAXNMV, NEON_FMAXNMV_H) \
5443 V(fminnmv, NEON_FMINNMV, NEON_FMINNMV_H) \
5444 // clang-format on
5445
5446 #define VIXL_DEFINE_ASM_FUNC(FN, OP, OP_H) \
5447 void Assembler::FN(const VRegister& vd, const VRegister& vn) { \
5448 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON)); \
5449 if (vd.Is1H()) VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf)); \
5450 VIXL_ASSERT(vd.Is1S() || vd.Is1H()); \
5451 NEONAcrossLanes(vd, vn, OP, OP_H); \
5452 }
NEON_ACROSSLANES_FP_LIST(VIXL_DEFINE_ASM_FUNC)5453 NEON_ACROSSLANES_FP_LIST(VIXL_DEFINE_ASM_FUNC)
5454 #undef VIXL_DEFINE_ASM_FUNC
5455
5456
5457 void Assembler::NEONPerm(const VRegister& vd,
5458 const VRegister& vn,
5459 const VRegister& vm,
5460 NEONPermOp op) {
5461 VIXL_ASSERT(AreSameFormat(vd, vn, vm));
5462 VIXL_ASSERT(!vd.Is1D());
5463 Emit(VFormat(vd) | op | Rm(vm) | Rn(vn) | Rd(vd));
5464 }
5465
5466
trn1(const VRegister & vd,const VRegister & vn,const VRegister & vm)5467 void Assembler::trn1(const VRegister& vd,
5468 const VRegister& vn,
5469 const VRegister& vm) {
5470 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5471 NEONPerm(vd, vn, vm, NEON_TRN1);
5472 }
5473
5474
trn2(const VRegister & vd,const VRegister & vn,const VRegister & vm)5475 void Assembler::trn2(const VRegister& vd,
5476 const VRegister& vn,
5477 const VRegister& vm) {
5478 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5479 NEONPerm(vd, vn, vm, NEON_TRN2);
5480 }
5481
5482
uzp1(const VRegister & vd,const VRegister & vn,const VRegister & vm)5483 void Assembler::uzp1(const VRegister& vd,
5484 const VRegister& vn,
5485 const VRegister& vm) {
5486 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5487 NEONPerm(vd, vn, vm, NEON_UZP1);
5488 }
5489
5490
uzp2(const VRegister & vd,const VRegister & vn,const VRegister & vm)5491 void Assembler::uzp2(const VRegister& vd,
5492 const VRegister& vn,
5493 const VRegister& vm) {
5494 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5495 NEONPerm(vd, vn, vm, NEON_UZP2);
5496 }
5497
5498
zip1(const VRegister & vd,const VRegister & vn,const VRegister & vm)5499 void Assembler::zip1(const VRegister& vd,
5500 const VRegister& vn,
5501 const VRegister& vm) {
5502 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5503 NEONPerm(vd, vn, vm, NEON_ZIP1);
5504 }
5505
5506
zip2(const VRegister & vd,const VRegister & vn,const VRegister & vm)5507 void Assembler::zip2(const VRegister& vd,
5508 const VRegister& vn,
5509 const VRegister& vm) {
5510 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5511 NEONPerm(vd, vn, vm, NEON_ZIP2);
5512 }
5513
5514
NEONShiftImmediate(const VRegister & vd,const VRegister & vn,NEONShiftImmediateOp op,int immh_immb)5515 void Assembler::NEONShiftImmediate(const VRegister& vd,
5516 const VRegister& vn,
5517 NEONShiftImmediateOp op,
5518 int immh_immb) {
5519 VIXL_ASSERT(AreSameFormat(vd, vn));
5520 Instr q, scalar;
5521 if (vn.IsScalar()) {
5522 q = NEON_Q;
5523 scalar = NEONScalar;
5524 } else {
5525 q = vd.IsD() ? 0 : NEON_Q;
5526 scalar = 0;
5527 }
5528 Emit(q | op | scalar | immh_immb | Rn(vn) | Rd(vd));
5529 }
5530
5531
NEONShiftLeftImmediate(const VRegister & vd,const VRegister & vn,int shift,NEONShiftImmediateOp op)5532 void Assembler::NEONShiftLeftImmediate(const VRegister& vd,
5533 const VRegister& vn,
5534 int shift,
5535 NEONShiftImmediateOp op) {
5536 int lane_size_in_bits = vn.GetLaneSizeInBits();
5537 VIXL_ASSERT((shift >= 0) && (shift < lane_size_in_bits));
5538 NEONShiftImmediate(vd, vn, op, (lane_size_in_bits + shift) << 16);
5539 }
5540
5541
NEONShiftRightImmediate(const VRegister & vd,const VRegister & vn,int shift,NEONShiftImmediateOp op)5542 void Assembler::NEONShiftRightImmediate(const VRegister& vd,
5543 const VRegister& vn,
5544 int shift,
5545 NEONShiftImmediateOp op) {
5546 int lane_size_in_bits = vn.GetLaneSizeInBits();
5547 VIXL_ASSERT((shift >= 1) && (shift <= lane_size_in_bits));
5548 NEONShiftImmediate(vd, vn, op, ((2 * lane_size_in_bits) - shift) << 16);
5549 }
5550
5551
NEONShiftImmediateL(const VRegister & vd,const VRegister & vn,int shift,NEONShiftImmediateOp op)5552 void Assembler::NEONShiftImmediateL(const VRegister& vd,
5553 const VRegister& vn,
5554 int shift,
5555 NEONShiftImmediateOp op) {
5556 int lane_size_in_bits = vn.GetLaneSizeInBits();
5557 VIXL_ASSERT((shift >= 0) && (shift < lane_size_in_bits));
5558 int immh_immb = (lane_size_in_bits + shift) << 16;
5559
5560 VIXL_ASSERT((vn.Is8B() && vd.Is8H()) || (vn.Is4H() && vd.Is4S()) ||
5561 (vn.Is2S() && vd.Is2D()) || (vn.Is16B() && vd.Is8H()) ||
5562 (vn.Is8H() && vd.Is4S()) || (vn.Is4S() && vd.Is2D()));
5563 Instr q;
5564 q = vn.IsD() ? 0 : NEON_Q;
5565 Emit(q | op | immh_immb | Rn(vn) | Rd(vd));
5566 }
5567
5568
NEONShiftImmediateN(const VRegister & vd,const VRegister & vn,int shift,NEONShiftImmediateOp op)5569 void Assembler::NEONShiftImmediateN(const VRegister& vd,
5570 const VRegister& vn,
5571 int shift,
5572 NEONShiftImmediateOp op) {
5573 Instr q, scalar;
5574 int lane_size_in_bits = vd.GetLaneSizeInBits();
5575 VIXL_ASSERT((shift >= 1) && (shift <= lane_size_in_bits));
5576 int immh_immb = (2 * lane_size_in_bits - shift) << 16;
5577
5578 if (vn.IsScalar()) {
5579 VIXL_ASSERT((vd.Is1B() && vn.Is1H()) || (vd.Is1H() && vn.Is1S()) ||
5580 (vd.Is1S() && vn.Is1D()));
5581 q = NEON_Q;
5582 scalar = NEONScalar;
5583 } else {
5584 VIXL_ASSERT((vd.Is8B() && vn.Is8H()) || (vd.Is4H() && vn.Is4S()) ||
5585 (vd.Is2S() && vn.Is2D()) || (vd.Is16B() && vn.Is8H()) ||
5586 (vd.Is8H() && vn.Is4S()) || (vd.Is4S() && vn.Is2D()));
5587 scalar = 0;
5588 q = vd.IsD() ? 0 : NEON_Q;
5589 }
5590 Emit(q | op | scalar | immh_immb | Rn(vn) | Rd(vd));
5591 }
5592
5593
shl(const VRegister & vd,const VRegister & vn,int shift)5594 void Assembler::shl(const VRegister& vd, const VRegister& vn, int shift) {
5595 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5596 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
5597 NEONShiftLeftImmediate(vd, vn, shift, NEON_SHL);
5598 }
5599
5600
sli(const VRegister & vd,const VRegister & vn,int shift)5601 void Assembler::sli(const VRegister& vd, const VRegister& vn, int shift) {
5602 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5603 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
5604 NEONShiftLeftImmediate(vd, vn, shift, NEON_SLI);
5605 }
5606
5607
sqshl(const VRegister & vd,const VRegister & vn,int shift)5608 void Assembler::sqshl(const VRegister& vd, const VRegister& vn, int shift) {
5609 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5610 NEONShiftLeftImmediate(vd, vn, shift, NEON_SQSHL_imm);
5611 }
5612
5613
sqshlu(const VRegister & vd,const VRegister & vn,int shift)5614 void Assembler::sqshlu(const VRegister& vd, const VRegister& vn, int shift) {
5615 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5616 NEONShiftLeftImmediate(vd, vn, shift, NEON_SQSHLU);
5617 }
5618
5619
uqshl(const VRegister & vd,const VRegister & vn,int shift)5620 void Assembler::uqshl(const VRegister& vd, const VRegister& vn, int shift) {
5621 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5622 NEONShiftLeftImmediate(vd, vn, shift, NEON_UQSHL_imm);
5623 }
5624
5625
sshll(const VRegister & vd,const VRegister & vn,int shift)5626 void Assembler::sshll(const VRegister& vd, const VRegister& vn, int shift) {
5627 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5628 VIXL_ASSERT(vn.IsD());
5629 NEONShiftImmediateL(vd, vn, shift, NEON_SSHLL);
5630 }
5631
5632
sshll2(const VRegister & vd,const VRegister & vn,int shift)5633 void Assembler::sshll2(const VRegister& vd, const VRegister& vn, int shift) {
5634 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5635 VIXL_ASSERT(vn.IsQ());
5636 NEONShiftImmediateL(vd, vn, shift, NEON_SSHLL);
5637 }
5638
5639
sxtl(const VRegister & vd,const VRegister & vn)5640 void Assembler::sxtl(const VRegister& vd, const VRegister& vn) {
5641 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5642 sshll(vd, vn, 0);
5643 }
5644
5645
sxtl2(const VRegister & vd,const VRegister & vn)5646 void Assembler::sxtl2(const VRegister& vd, const VRegister& vn) {
5647 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5648 sshll2(vd, vn, 0);
5649 }
5650
5651
ushll(const VRegister & vd,const VRegister & vn,int shift)5652 void Assembler::ushll(const VRegister& vd, const VRegister& vn, int shift) {
5653 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5654 VIXL_ASSERT(vn.IsD());
5655 NEONShiftImmediateL(vd, vn, shift, NEON_USHLL);
5656 }
5657
5658
ushll2(const VRegister & vd,const VRegister & vn,int shift)5659 void Assembler::ushll2(const VRegister& vd, const VRegister& vn, int shift) {
5660 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5661 VIXL_ASSERT(vn.IsQ());
5662 NEONShiftImmediateL(vd, vn, shift, NEON_USHLL);
5663 }
5664
5665
uxtl(const VRegister & vd,const VRegister & vn)5666 void Assembler::uxtl(const VRegister& vd, const VRegister& vn) {
5667 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5668 ushll(vd, vn, 0);
5669 }
5670
5671
uxtl2(const VRegister & vd,const VRegister & vn)5672 void Assembler::uxtl2(const VRegister& vd, const VRegister& vn) {
5673 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5674 ushll2(vd, vn, 0);
5675 }
5676
5677
sri(const VRegister & vd,const VRegister & vn,int shift)5678 void Assembler::sri(const VRegister& vd, const VRegister& vn, int shift) {
5679 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5680 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
5681 NEONShiftRightImmediate(vd, vn, shift, NEON_SRI);
5682 }
5683
5684
sshr(const VRegister & vd,const VRegister & vn,int shift)5685 void Assembler::sshr(const VRegister& vd, const VRegister& vn, int shift) {
5686 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5687 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
5688 NEONShiftRightImmediate(vd, vn, shift, NEON_SSHR);
5689 }
5690
5691
ushr(const VRegister & vd,const VRegister & vn,int shift)5692 void Assembler::ushr(const VRegister& vd, const VRegister& vn, int shift) {
5693 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5694 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
5695 NEONShiftRightImmediate(vd, vn, shift, NEON_USHR);
5696 }
5697
5698
srshr(const VRegister & vd,const VRegister & vn,int shift)5699 void Assembler::srshr(const VRegister& vd, const VRegister& vn, int shift) {
5700 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5701 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
5702 NEONShiftRightImmediate(vd, vn, shift, NEON_SRSHR);
5703 }
5704
5705
urshr(const VRegister & vd,const VRegister & vn,int shift)5706 void Assembler::urshr(const VRegister& vd, const VRegister& vn, int shift) {
5707 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5708 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
5709 NEONShiftRightImmediate(vd, vn, shift, NEON_URSHR);
5710 }
5711
5712
ssra(const VRegister & vd,const VRegister & vn,int shift)5713 void Assembler::ssra(const VRegister& vd, const VRegister& vn, int shift) {
5714 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5715 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
5716 NEONShiftRightImmediate(vd, vn, shift, NEON_SSRA);
5717 }
5718
5719
usra(const VRegister & vd,const VRegister & vn,int shift)5720 void Assembler::usra(const VRegister& vd, const VRegister& vn, int shift) {
5721 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5722 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
5723 NEONShiftRightImmediate(vd, vn, shift, NEON_USRA);
5724 }
5725
5726
srsra(const VRegister & vd,const VRegister & vn,int shift)5727 void Assembler::srsra(const VRegister& vd, const VRegister& vn, int shift) {
5728 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5729 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
5730 NEONShiftRightImmediate(vd, vn, shift, NEON_SRSRA);
5731 }
5732
5733
ursra(const VRegister & vd,const VRegister & vn,int shift)5734 void Assembler::ursra(const VRegister& vd, const VRegister& vn, int shift) {
5735 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5736 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
5737 NEONShiftRightImmediate(vd, vn, shift, NEON_URSRA);
5738 }
5739
5740
shrn(const VRegister & vd,const VRegister & vn,int shift)5741 void Assembler::shrn(const VRegister& vd, const VRegister& vn, int shift) {
5742 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5743 VIXL_ASSERT(vn.IsVector() && vd.IsD());
5744 NEONShiftImmediateN(vd, vn, shift, NEON_SHRN);
5745 }
5746
5747
shrn2(const VRegister & vd,const VRegister & vn,int shift)5748 void Assembler::shrn2(const VRegister& vd, const VRegister& vn, int shift) {
5749 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5750 VIXL_ASSERT(vn.IsVector() && vd.IsQ());
5751 NEONShiftImmediateN(vd, vn, shift, NEON_SHRN);
5752 }
5753
5754
rshrn(const VRegister & vd,const VRegister & vn,int shift)5755 void Assembler::rshrn(const VRegister& vd, const VRegister& vn, int shift) {
5756 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5757 VIXL_ASSERT(vn.IsVector() && vd.IsD());
5758 NEONShiftImmediateN(vd, vn, shift, NEON_RSHRN);
5759 }
5760
5761
rshrn2(const VRegister & vd,const VRegister & vn,int shift)5762 void Assembler::rshrn2(const VRegister& vd, const VRegister& vn, int shift) {
5763 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5764 VIXL_ASSERT(vn.IsVector() && vd.IsQ());
5765 NEONShiftImmediateN(vd, vn, shift, NEON_RSHRN);
5766 }
5767
5768
sqshrn(const VRegister & vd,const VRegister & vn,int shift)5769 void Assembler::sqshrn(const VRegister& vd, const VRegister& vn, int shift) {
5770 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5771 VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
5772 NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRN);
5773 }
5774
5775
sqshrn2(const VRegister & vd,const VRegister & vn,int shift)5776 void Assembler::sqshrn2(const VRegister& vd, const VRegister& vn, int shift) {
5777 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5778 VIXL_ASSERT(vn.IsVector() && vd.IsQ());
5779 NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRN);
5780 }
5781
5782
sqrshrn(const VRegister & vd,const VRegister & vn,int shift)5783 void Assembler::sqrshrn(const VRegister& vd, const VRegister& vn, int shift) {
5784 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5785 VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
5786 NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRN);
5787 }
5788
5789
sqrshrn2(const VRegister & vd,const VRegister & vn,int shift)5790 void Assembler::sqrshrn2(const VRegister& vd, const VRegister& vn, int shift) {
5791 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5792 VIXL_ASSERT(vn.IsVector() && vd.IsQ());
5793 NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRN);
5794 }
5795
5796
sqshrun(const VRegister & vd,const VRegister & vn,int shift)5797 void Assembler::sqshrun(const VRegister& vd, const VRegister& vn, int shift) {
5798 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5799 VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
5800 NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRUN);
5801 }
5802
5803
sqshrun2(const VRegister & vd,const VRegister & vn,int shift)5804 void Assembler::sqshrun2(const VRegister& vd, const VRegister& vn, int shift) {
5805 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5806 VIXL_ASSERT(vn.IsVector() && vd.IsQ());
5807 NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRUN);
5808 }
5809
5810
sqrshrun(const VRegister & vd,const VRegister & vn,int shift)5811 void Assembler::sqrshrun(const VRegister& vd, const VRegister& vn, int shift) {
5812 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5813 VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
5814 NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRUN);
5815 }
5816
5817
sqrshrun2(const VRegister & vd,const VRegister & vn,int shift)5818 void Assembler::sqrshrun2(const VRegister& vd, const VRegister& vn, int shift) {
5819 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5820 VIXL_ASSERT(vn.IsVector() && vd.IsQ());
5821 NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRUN);
5822 }
5823
5824
uqshrn(const VRegister & vd,const VRegister & vn,int shift)5825 void Assembler::uqshrn(const VRegister& vd, const VRegister& vn, int shift) {
5826 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5827 VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
5828 NEONShiftImmediateN(vd, vn, shift, NEON_UQSHRN);
5829 }
5830
5831
uqshrn2(const VRegister & vd,const VRegister & vn,int shift)5832 void Assembler::uqshrn2(const VRegister& vd, const VRegister& vn, int shift) {
5833 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5834 VIXL_ASSERT(vn.IsVector() && vd.IsQ());
5835 NEONShiftImmediateN(vd, vn, shift, NEON_UQSHRN);
5836 }
5837
5838
uqrshrn(const VRegister & vd,const VRegister & vn,int shift)5839 void Assembler::uqrshrn(const VRegister& vd, const VRegister& vn, int shift) {
5840 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5841 VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
5842 NEONShiftImmediateN(vd, vn, shift, NEON_UQRSHRN);
5843 }
5844
5845
uqrshrn2(const VRegister & vd,const VRegister & vn,int shift)5846 void Assembler::uqrshrn2(const VRegister& vd, const VRegister& vn, int shift) {
5847 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5848 VIXL_ASSERT(vn.IsVector() && vd.IsQ());
5849 NEONShiftImmediateN(vd, vn, shift, NEON_UQRSHRN);
5850 }
5851
smmla(const VRegister & vd,const VRegister & vn,const VRegister & vm)5852 void Assembler::smmla(const VRegister& vd, const VRegister& vn, const VRegister& vm) {
5853 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5854 VIXL_ASSERT(CPUHas(CPUFeatures::kI8MM));
5855 VIXL_ASSERT(vd.IsLaneSizeS());
5856 VIXL_ASSERT(vn.IsLaneSizeB() && vm.IsLaneSizeB());
5857
5858 Emit(0x4e80a400 | Rd(vd) | Rn(vn) | Rm(vm));
5859 }
5860
usmmla(const VRegister & vd,const VRegister & vn,const VRegister & vm)5861 void Assembler::usmmla(const VRegister& vd, const VRegister& vn, const VRegister& vm) {
5862 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5863 VIXL_ASSERT(CPUHas(CPUFeatures::kI8MM));
5864 VIXL_ASSERT(vd.IsLaneSizeS());
5865 VIXL_ASSERT(vn.IsLaneSizeB() && vm.IsLaneSizeB());
5866
5867 Emit(0x4e80ac00 | Rd(vd) | Rn(vn) | Rm(vm));
5868 }
5869
ummla(const VRegister & vd,const VRegister & vn,const VRegister & vm)5870 void Assembler::ummla(const VRegister& vd, const VRegister& vn, const VRegister& vm) {
5871 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5872 VIXL_ASSERT(CPUHas(CPUFeatures::kI8MM));
5873 VIXL_ASSERT(vd.IsLaneSizeS());
5874 VIXL_ASSERT(vn.IsLaneSizeB() && vm.IsLaneSizeB());
5875
5876 Emit(0x6e80a400 | Rd(vd) | Rn(vn) | Rm(vm));
5877 }
5878
bcax(const VRegister & vd,const VRegister & vn,const VRegister & vm,const VRegister & va)5879 void Assembler::bcax(const VRegister& vd, const VRegister& vn, const VRegister& vm, const VRegister& va) {
5880 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5881 VIXL_ASSERT(CPUHas(CPUFeatures::kSHA3));
5882 VIXL_ASSERT(vd.Is16B() && vn.Is16B() && vm.Is16B());
5883
5884 Emit(0xce200000 | Rd(vd) | Rn(vn) | Rm(vm) | Ra(va));
5885 }
5886
eor3(const VRegister & vd,const VRegister & vn,const VRegister & vm,const VRegister & va)5887 void Assembler::eor3(const VRegister& vd, const VRegister& vn, const VRegister& vm, const VRegister& va) {
5888 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5889 VIXL_ASSERT(CPUHas(CPUFeatures::kSHA3));
5890 VIXL_ASSERT(vd.Is16B() && vn.Is16B() && vm.Is16B() && va.Is16B());
5891
5892 Emit(0xce000000 | Rd(vd) | Rn(vn) | Rm(vm) | Ra(va));
5893 }
5894
xar(const VRegister & vd,const VRegister & vn,const VRegister & vm,int rotate)5895 void Assembler::xar(const VRegister& vd, const VRegister& vn, const VRegister& vm, int rotate) {
5896 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5897 VIXL_ASSERT(CPUHas(CPUFeatures::kSHA3));
5898 VIXL_ASSERT(vd.Is2D() && vn.Is2D() && vm.Is2D());
5899 VIXL_ASSERT(IsUint6(rotate));
5900
5901 Emit(0xce800000 | Rd(vd) | Rn(vn) | Rm(vm) | rotate << 10);
5902 }
5903
rax1(const VRegister & vd,const VRegister & vn,const VRegister & vm)5904 void Assembler::rax1(const VRegister& vd, const VRegister& vn, const VRegister& vm) {
5905 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5906 VIXL_ASSERT(CPUHas(CPUFeatures::kSHA3));
5907 VIXL_ASSERT(vd.Is2D() && vn.Is2D() && vm.Is2D());
5908
5909 Emit(0xce608c00 | Rd(vd) | Rn(vn) | Rm(vm));
5910 }
5911
sha1c(const VRegister & vd,const VRegister & vn,const VRegister & vm)5912 void Assembler::sha1c(const VRegister& vd, const VRegister& vn, const VRegister& vm) {
5913 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5914 VIXL_ASSERT(CPUHas(CPUFeatures::kSHA1));
5915 VIXL_ASSERT(vd.IsQ() && vn.IsS() && vm.Is4S());
5916
5917 Emit(0x5e000000 | Rd(vd) | Rn(vn) | Rm(vm));
5918 }
5919
sha1h(const VRegister & sd,const VRegister & sn)5920 void Assembler::sha1h(const VRegister& sd, const VRegister& sn) {
5921 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5922 VIXL_ASSERT(CPUHas(CPUFeatures::kSHA1));
5923 VIXL_ASSERT(sd.IsS() && sn.IsS());
5924
5925 Emit(0x5e280800 | Rd(sd) | Rn(sn));
5926 }
5927
sha1m(const VRegister & vd,const VRegister & vn,const VRegister & vm)5928 void Assembler::sha1m(const VRegister& vd, const VRegister& vn, const VRegister& vm) {
5929 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5930 VIXL_ASSERT(CPUHas(CPUFeatures::kSHA1));
5931 VIXL_ASSERT(vd.IsQ() && vn.IsS() && vm.Is4S());
5932
5933 Emit(0x5e002000 | Rd(vd) | Rn(vn) | Rm(vm));
5934 }
5935
sha1p(const VRegister & vd,const VRegister & vn,const VRegister & vm)5936 void Assembler::sha1p(const VRegister& vd, const VRegister& vn, const VRegister& vm) {
5937 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5938 VIXL_ASSERT(CPUHas(CPUFeatures::kSHA1));
5939 VIXL_ASSERT(vd.IsQ() && vn.IsS() && vm.Is4S());
5940
5941 Emit(0x5e001000 | Rd(vd) | Rn(vn) | Rm(vm));
5942 }
5943
sha1su0(const VRegister & vd,const VRegister & vn,const VRegister & vm)5944 void Assembler::sha1su0(const VRegister& vd, const VRegister& vn, const VRegister& vm) {
5945 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5946 VIXL_ASSERT(CPUHas(CPUFeatures::kSHA1));
5947 VIXL_ASSERT(vd.Is4S() && vn.Is4S() && vm.Is4S());
5948
5949 Emit(0x5e003000 | Rd(vd) | Rn(vn) | Rm(vm));
5950 }
5951
sha1su1(const VRegister & vd,const VRegister & vn)5952 void Assembler::sha1su1(const VRegister& vd, const VRegister& vn) {
5953 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5954 VIXL_ASSERT(CPUHas(CPUFeatures::kSHA1));
5955 VIXL_ASSERT(vd.Is4S() && vn.Is4S());
5956
5957 Emit(0x5e281800 | Rd(vd) | Rn(vn));
5958 }
5959
sha256h(const VRegister & vd,const VRegister & vn,const VRegister & vm)5960 void Assembler::sha256h(const VRegister& vd, const VRegister& vn, const VRegister& vm) {
5961 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5962 VIXL_ASSERT(CPUHas(CPUFeatures::kSHA2));
5963 VIXL_ASSERT(vd.IsQ() && vn.IsQ() && vm.Is4S());
5964
5965 Emit(0x5e004000 | Rd(vd) | Rn(vn) | Rm(vm));
5966 }
5967
sha256h2(const VRegister & vd,const VRegister & vn,const VRegister & vm)5968 void Assembler::sha256h2(const VRegister& vd, const VRegister& vn, const VRegister& vm) {
5969 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5970 VIXL_ASSERT(CPUHas(CPUFeatures::kSHA2));
5971 VIXL_ASSERT(vd.IsQ() && vn.IsQ() && vm.Is4S());
5972
5973 Emit(0x5e005000 | Rd(vd) | Rn(vn) | Rm(vm));
5974 }
5975
sha256su0(const VRegister & vd,const VRegister & vn)5976 void Assembler::sha256su0(const VRegister& vd, const VRegister& vn) {
5977 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5978 VIXL_ASSERT(CPUHas(CPUFeatures::kSHA2));
5979 VIXL_ASSERT(vd.Is4S() && vn.Is4S());
5980
5981 Emit(0x5e282800 | Rd(vd) | Rn(vn));
5982 }
5983
sha256su1(const VRegister & vd,const VRegister & vn,const VRegister & vm)5984 void Assembler::sha256su1(const VRegister& vd, const VRegister& vn, const VRegister& vm) {
5985 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5986 VIXL_ASSERT(CPUHas(CPUFeatures::kSHA2));
5987 VIXL_ASSERT(vd.Is4S() && vn.Is4S() && vm.Is4S());
5988
5989 Emit(0x5e006000 | Rd(vd) | Rn(vn) | Rm(vm));
5990 }
5991
sha512h(const VRegister & vd,const VRegister & vn,const VRegister & vm)5992 void Assembler::sha512h(const VRegister& vd, const VRegister& vn, const VRegister& vm) {
5993 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5994 VIXL_ASSERT(CPUHas(CPUFeatures::kSHA512));
5995 VIXL_ASSERT(vd.IsQ() && vn.IsQ() && vm.Is2D());
5996
5997 Emit(0xce608000 | Rd(vd) | Rn(vn) | Rm(vm));
5998 }
5999
sha512h2(const VRegister & vd,const VRegister & vn,const VRegister & vm)6000 void Assembler::sha512h2(const VRegister& vd, const VRegister& vn, const VRegister& vm) {
6001 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
6002 VIXL_ASSERT(CPUHas(CPUFeatures::kSHA512));
6003 VIXL_ASSERT(vd.IsQ() && vn.IsQ() && vm.Is2D());
6004
6005 Emit(0xce608400 | Rd(vd) | Rn(vn) | Rm(vm));
6006 }
6007
sha512su0(const VRegister & vd,const VRegister & vn)6008 void Assembler::sha512su0(const VRegister& vd, const VRegister& vn) {
6009 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
6010 VIXL_ASSERT(CPUHas(CPUFeatures::kSHA512));
6011 VIXL_ASSERT(vd.Is2D() && vn.Is2D());
6012
6013 Emit(0xcec08000 | Rd(vd) | Rn(vn));
6014 }
6015
sha512su1(const VRegister & vd,const VRegister & vn,const VRegister & vm)6016 void Assembler::sha512su1(const VRegister& vd, const VRegister& vn, const VRegister& vm) {
6017 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
6018 VIXL_ASSERT(CPUHas(CPUFeatures::kSHA512));
6019 VIXL_ASSERT(vd.Is2D() && vn.Is2D() && vm.Is2D());
6020
6021 Emit(0xce608800 | Rd(vd) | Rn(vn) | Rm(vm));
6022 }
6023
aesd(const VRegister & vd,const VRegister & vn)6024 void Assembler::aesd(const VRegister& vd, const VRegister& vn) {
6025 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
6026 VIXL_ASSERT(CPUHas(CPUFeatures::kAES));
6027 VIXL_ASSERT(vd.Is16B() && vn.Is16B());
6028
6029 Emit(0x4e285800 | Rd(vd) | Rn(vn));
6030 }
6031
aese(const VRegister & vd,const VRegister & vn)6032 void Assembler::aese(const VRegister& vd, const VRegister& vn) {
6033 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
6034 VIXL_ASSERT(CPUHas(CPUFeatures::kAES));
6035 VIXL_ASSERT(vd.Is16B() && vn.Is16B());
6036
6037 Emit(0x4e284800 | Rd(vd) | Rn(vn));
6038 }
6039
aesimc(const VRegister & vd,const VRegister & vn)6040 void Assembler::aesimc(const VRegister& vd, const VRegister& vn) {
6041 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
6042 VIXL_ASSERT(CPUHas(CPUFeatures::kAES));
6043 VIXL_ASSERT(vd.Is16B() && vn.Is16B());
6044
6045 Emit(0x4e287800 | Rd(vd) | Rn(vn));
6046 }
6047
aesmc(const VRegister & vd,const VRegister & vn)6048 void Assembler::aesmc(const VRegister& vd, const VRegister& vn) {
6049 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
6050 VIXL_ASSERT(CPUHas(CPUFeatures::kAES));
6051 VIXL_ASSERT(vd.Is16B() && vn.Is16B());
6052
6053 Emit(0x4e286800 | Rd(vd) | Rn(vn));
6054 }
6055
sm3partw1(const VRegister & vd,const VRegister & vn,const VRegister & vm)6056 void Assembler::sm3partw1(const VRegister& vd, const VRegister& vn, const VRegister& vm) {
6057 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
6058 VIXL_ASSERT(CPUHas(CPUFeatures::kSM3));
6059 VIXL_ASSERT(vd.Is4S() && vn.Is4S() && vm.Is4S());
6060
6061 Emit(0xce60c000 | Rd(vd) | Rn(vn) | Rm(vm));
6062 }
6063
sm3partw2(const VRegister & vd,const VRegister & vn,const VRegister & vm)6064 void Assembler::sm3partw2(const VRegister& vd, const VRegister& vn, const VRegister& vm) {
6065 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
6066 VIXL_ASSERT(CPUHas(CPUFeatures::kSM3));
6067 VIXL_ASSERT(vd.Is4S() && vn.Is4S() && vm.Is4S());
6068
6069 Emit(0xce60c400 | Rd(vd) | Rn(vn) | Rm(vm));
6070 }
6071
sm3ss1(const VRegister & vd,const VRegister & vn,const VRegister & vm,const VRegister & va)6072 void Assembler::sm3ss1(const VRegister& vd, const VRegister& vn, const VRegister& vm, const VRegister& va) {
6073 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
6074 VIXL_ASSERT(CPUHas(CPUFeatures::kSM3));
6075 VIXL_ASSERT(vd.Is4S() && vn.Is4S() && vm.Is4S() && va.Is4S());
6076
6077 Emit(0xce400000 | Rd(vd) | Rn(vn) | Rm(vm) | Ra(va));
6078 }
6079
sm3tt1a(const VRegister & vd,const VRegister & vn,const VRegister & vm,int index)6080 void Assembler::sm3tt1a(const VRegister& vd, const VRegister& vn, const VRegister& vm, int index) {
6081 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
6082 VIXL_ASSERT(CPUHas(CPUFeatures::kSM3));
6083 VIXL_ASSERT(vd.Is4S() && vn.Is4S() && vm.Is4S());
6084 VIXL_ASSERT(IsUint2(index));
6085
6086 Instr i = static_cast<uint32_t>(index) << 12;
6087 Emit(0xce408000 | Rd(vd) | Rn(vn) | Rm(vm) | i);
6088 }
6089
sm3tt1b(const VRegister & vd,const VRegister & vn,const VRegister & vm,int index)6090 void Assembler::sm3tt1b(const VRegister& vd, const VRegister& vn, const VRegister& vm, int index) {
6091 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
6092 VIXL_ASSERT(CPUHas(CPUFeatures::kSM3));
6093 VIXL_ASSERT(vd.Is4S() && vn.Is4S() && vm.Is4S());
6094 VIXL_ASSERT(IsUint2(index));
6095
6096 Instr i = static_cast<uint32_t>(index) << 12;
6097 Emit(0xce408400 | Rd(vd) | Rn(vn) | Rm(vm) | i);
6098 }
6099
sm3tt2a(const VRegister & vd,const VRegister & vn,const VRegister & vm,int index)6100 void Assembler::sm3tt2a(const VRegister& vd, const VRegister& vn, const VRegister& vm, int index) {
6101 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
6102 VIXL_ASSERT(CPUHas(CPUFeatures::kSM3));
6103 VIXL_ASSERT(vd.Is4S() && vn.Is4S() && vm.Is4S());
6104 VIXL_ASSERT(IsUint2(index));
6105
6106 Instr i = static_cast<uint32_t>(index) << 12;
6107 Emit(0xce408800 | Rd(vd) | Rn(vn) | Rm(vm) | i);
6108 }
6109
sm3tt2b(const VRegister & vd,const VRegister & vn,const VRegister & vm,int index)6110 void Assembler::sm3tt2b(const VRegister& vd, const VRegister& vn, const VRegister& vm, int index) {
6111 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
6112 VIXL_ASSERT(CPUHas(CPUFeatures::kSM3));
6113 VIXL_ASSERT(vd.Is4S() && vn.Is4S() && vm.Is4S());
6114 VIXL_ASSERT(IsUint2(index));
6115
6116 Instr i = static_cast<uint32_t>(index) << 12;
6117 Emit(0xce408c00 | Rd(vd) | Rn(vn) | Rm(vm) | i);
6118 }
6119
sm4e(const VRegister & vd,const VRegister & vn)6120 void Assembler::sm4e(const VRegister& vd, const VRegister& vn) {
6121 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
6122 VIXL_ASSERT(CPUHas(CPUFeatures::kSM4));
6123 VIXL_ASSERT(vd.Is4S() && vn.Is4S());
6124
6125 Emit(0xcec08400 | Rd(vd) | Rn(vn));
6126 }
6127
sm4ekey(const VRegister & vd,const VRegister & vn,const VRegister & vm)6128 void Assembler::sm4ekey(const VRegister& vd, const VRegister& vn, const VRegister& vm) {
6129 VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
6130 VIXL_ASSERT(CPUHas(CPUFeatures::kSM4));
6131 VIXL_ASSERT(vd.Is4S() && vn.Is4S() && vm.Is4S());
6132
6133 Emit(0xce60c800 | Rd(vd) | Rn(vn) | Rm(vm));
6134 }
6135
6136 // Note:
6137 // For all ToImm instructions below, a difference in case
6138 // for the same letter indicates a negated bit.
6139 // If b is 1, then B is 0.
FP16ToImm8(Float16 imm)6140 uint32_t Assembler::FP16ToImm8(Float16 imm) {
6141 VIXL_ASSERT(IsImmFP16(imm));
6142 // Half: aBbb.cdef.gh00.0000 (16 bits)
6143 uint16_t bits = Float16ToRawbits(imm);
6144 // bit7: a000.0000
6145 uint16_t bit7 = ((bits >> 15) & 0x1) << 7;
6146 // bit6: 0b00.0000
6147 uint16_t bit6 = ((bits >> 13) & 0x1) << 6;
6148 // bit5_to_0: 00cd.efgh
6149 uint16_t bit5_to_0 = (bits >> 6) & 0x3f;
6150 uint32_t result = static_cast<uint32_t>(bit7 | bit6 | bit5_to_0);
6151 return result;
6152 }
6153
6154
ImmFP16(Float16 imm)6155 Instr Assembler::ImmFP16(Float16 imm) {
6156 return FP16ToImm8(imm) << ImmFP_offset;
6157 }
6158
6159
FP32ToImm8(float imm)6160 uint32_t Assembler::FP32ToImm8(float imm) {
6161 // bits: aBbb.bbbc.defg.h000.0000.0000.0000.0000
6162 uint32_t bits = FloatToRawbits(imm);
6163 VIXL_ASSERT(IsImmFP32(bits));
6164 // bit7: a000.0000
6165 uint32_t bit7 = ((bits >> 31) & 0x1) << 7;
6166 // bit6: 0b00.0000
6167 uint32_t bit6 = ((bits >> 29) & 0x1) << 6;
6168 // bit5_to_0: 00cd.efgh
6169 uint32_t bit5_to_0 = (bits >> 19) & 0x3f;
6170
6171 return bit7 | bit6 | bit5_to_0;
6172 }
6173
6174
ImmFP32(float imm)6175 Instr Assembler::ImmFP32(float imm) { return FP32ToImm8(imm) << ImmFP_offset; }
6176
6177
FP64ToImm8(double imm)6178 uint32_t Assembler::FP64ToImm8(double imm) {
6179 // bits: aBbb.bbbb.bbcd.efgh.0000.0000.0000.0000
6180 // 0000.0000.0000.0000.0000.0000.0000.0000
6181 uint64_t bits = DoubleToRawbits(imm);
6182 VIXL_ASSERT(IsImmFP64(bits));
6183 // bit7: a000.0000
6184 uint64_t bit7 = ((bits >> 63) & 0x1) << 7;
6185 // bit6: 0b00.0000
6186 uint64_t bit6 = ((bits >> 61) & 0x1) << 6;
6187 // bit5_to_0: 00cd.efgh
6188 uint64_t bit5_to_0 = (bits >> 48) & 0x3f;
6189
6190 return static_cast<uint32_t>(bit7 | bit6 | bit5_to_0);
6191 }
6192
6193
ImmFP64(double imm)6194 Instr Assembler::ImmFP64(double imm) { return FP64ToImm8(imm) << ImmFP_offset; }
6195
6196
6197 // Code generation helpers.
OneInstrMoveImmediateHelper(Assembler * assm,const Register & dst,uint64_t imm)6198 bool Assembler::OneInstrMoveImmediateHelper(Assembler* assm,
6199 const Register& dst,
6200 uint64_t imm) {
6201 bool emit_code = assm != NULL;
6202 unsigned n, imm_s, imm_r;
6203 int reg_size = dst.GetSizeInBits();
6204
6205 if (IsImmMovz(imm, reg_size) && !dst.IsSP()) {
6206 // Immediate can be represented in a move zero instruction. Movz can't write
6207 // to the stack pointer.
6208 if (emit_code) {
6209 assm->movz(dst, imm);
6210 }
6211 return true;
6212 } else if (IsImmMovn(imm, reg_size) && !dst.IsSP()) {
6213 // Immediate can be represented in a move negative instruction. Movn can't
6214 // write to the stack pointer.
6215 if (emit_code) {
6216 assm->movn(dst, dst.Is64Bits() ? ~imm : (~imm & kWRegMask));
6217 }
6218 return true;
6219 } else if (IsImmLogical(imm, reg_size, &n, &imm_s, &imm_r)) {
6220 // Immediate can be represented in a logical orr instruction.
6221 VIXL_ASSERT(!dst.IsZero());
6222 if (emit_code) {
6223 assm->LogicalImmediate(dst,
6224 AppropriateZeroRegFor(dst),
6225 n,
6226 imm_s,
6227 imm_r,
6228 ORR);
6229 }
6230 return true;
6231 }
6232 return false;
6233 }
6234
6235
MoveWide(const Register & rd,uint64_t imm,int shift,MoveWideImmediateOp mov_op)6236 void Assembler::MoveWide(const Register& rd,
6237 uint64_t imm,
6238 int shift,
6239 MoveWideImmediateOp mov_op) {
6240 // Ignore the top 32 bits of an immediate if we're moving to a W register.
6241 if (rd.Is32Bits()) {
6242 // Check that the top 32 bits are zero (a positive 32-bit number) or top
6243 // 33 bits are one (a negative 32-bit number, sign extended to 64 bits).
6244 VIXL_ASSERT(((imm >> kWRegSize) == 0) ||
6245 ((imm >> (kWRegSize - 1)) == 0x1ffffffff));
6246 imm &= kWRegMask;
6247 }
6248
6249 if (shift >= 0) {
6250 // Explicit shift specified.
6251 VIXL_ASSERT((shift == 0) || (shift == 16) || (shift == 32) ||
6252 (shift == 48));
6253 VIXL_ASSERT(rd.Is64Bits() || (shift == 0) || (shift == 16));
6254 shift /= 16;
6255 } else {
6256 // Calculate a new immediate and shift combination to encode the immediate
6257 // argument.
6258 VIXL_ASSERT(shift == -1);
6259 shift = 0;
6260 if ((imm & 0xffffffffffff0000) == 0) {
6261 // Nothing to do.
6262 } else if ((imm & 0xffffffff0000ffff) == 0) {
6263 imm >>= 16;
6264 shift = 1;
6265 } else if ((imm & 0xffff0000ffffffff) == 0) {
6266 VIXL_ASSERT(rd.Is64Bits());
6267 imm >>= 32;
6268 shift = 2;
6269 } else if ((imm & 0x0000ffffffffffff) == 0) {
6270 VIXL_ASSERT(rd.Is64Bits());
6271 imm >>= 48;
6272 shift = 3;
6273 }
6274 }
6275
6276 VIXL_ASSERT(IsUint16(imm));
6277
6278 Emit(SF(rd) | MoveWideImmediateFixed | mov_op | Rd(rd) | ImmMoveWide(imm) |
6279 ShiftMoveWide(shift));
6280 }
6281
6282
AddSub(const Register & rd,const Register & rn,const Operand & operand,FlagsUpdate S,AddSubOp op)6283 void Assembler::AddSub(const Register& rd,
6284 const Register& rn,
6285 const Operand& operand,
6286 FlagsUpdate S,
6287 AddSubOp op) {
6288 VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
6289 if (operand.IsImmediate()) {
6290 int64_t immediate = operand.GetImmediate();
6291 VIXL_ASSERT(IsImmAddSub(immediate));
6292 Instr dest_reg = (S == SetFlags) ? Rd(rd) : RdSP(rd);
6293 Emit(SF(rd) | AddSubImmediateFixed | op | Flags(S) |
6294 ImmAddSub(static_cast<int>(immediate)) | dest_reg | RnSP(rn));
6295 } else if (operand.IsShiftedRegister()) {
6296 VIXL_ASSERT(operand.GetRegister().GetSizeInBits() == rd.GetSizeInBits());
6297 VIXL_ASSERT(operand.GetShift() != ROR);
6298
6299 // For instructions of the form:
6300 // add/sub wsp, <Wn>, <Wm> [, LSL #0-3 ]
6301 // add/sub <Wd>, wsp, <Wm> [, LSL #0-3 ]
6302 // add/sub wsp, wsp, <Wm> [, LSL #0-3 ]
6303 // adds/subs <Wd>, wsp, <Wm> [, LSL #0-3 ]
6304 // or their 64-bit register equivalents, convert the operand from shifted to
6305 // extended register mode, and emit an add/sub extended instruction.
6306 if (rn.IsSP() || rd.IsSP()) {
6307 VIXL_ASSERT(!(rd.IsSP() && (S == SetFlags)));
6308 DataProcExtendedRegister(rd,
6309 rn,
6310 operand.ToExtendedRegister(),
6311 S,
6312 AddSubExtendedFixed | op);
6313 } else {
6314 DataProcShiftedRegister(rd, rn, operand, S, AddSubShiftedFixed | op);
6315 }
6316 } else {
6317 VIXL_ASSERT(operand.IsExtendedRegister());
6318 DataProcExtendedRegister(rd, rn, operand, S, AddSubExtendedFixed | op);
6319 }
6320 }
6321
6322
AddSubWithCarry(const Register & rd,const Register & rn,const Operand & operand,FlagsUpdate S,AddSubWithCarryOp op)6323 void Assembler::AddSubWithCarry(const Register& rd,
6324 const Register& rn,
6325 const Operand& operand,
6326 FlagsUpdate S,
6327 AddSubWithCarryOp op) {
6328 VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
6329 VIXL_ASSERT(rd.GetSizeInBits() == operand.GetRegister().GetSizeInBits());
6330 VIXL_ASSERT(operand.IsShiftedRegister() && (operand.GetShiftAmount() == 0));
6331 Emit(SF(rd) | op | Flags(S) | Rm(operand.GetRegister()) | Rn(rn) | Rd(rd));
6332 }
6333
6334
hlt(int code)6335 void Assembler::hlt(int code) {
6336 VIXL_ASSERT(IsUint16(code));
6337 Emit(HLT | ImmException(code));
6338 }
6339
6340
brk(int code)6341 void Assembler::brk(int code) {
6342 VIXL_ASSERT(IsUint16(code));
6343 Emit(BRK | ImmException(code));
6344 }
6345
6346
svc(int code)6347 void Assembler::svc(int code) { Emit(SVC | ImmException(code)); }
6348
udf(int code)6349 void Assembler::udf(int code) { Emit(UDF | ImmUdf(code)); }
6350
6351
6352 // TODO(all): The third parameter should be passed by reference but gcc 4.8.2
6353 // reports a bogus uninitialised warning then.
Logical(const Register & rd,const Register & rn,const Operand operand,LogicalOp op)6354 void Assembler::Logical(const Register& rd,
6355 const Register& rn,
6356 const Operand operand,
6357 LogicalOp op) {
6358 VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
6359 if (operand.IsImmediate()) {
6360 int64_t immediate = operand.GetImmediate();
6361 unsigned reg_size = rd.GetSizeInBits();
6362
6363 VIXL_ASSERT(immediate != 0);
6364 VIXL_ASSERT(immediate != -1);
6365 VIXL_ASSERT(rd.Is64Bits() || IsUint32(immediate));
6366
6367 // If the operation is NOT, invert the operation and immediate.
6368 if ((op & NOT) == NOT) {
6369 op = static_cast<LogicalOp>(op & ~NOT);
6370 immediate = rd.Is64Bits() ? ~immediate : (~immediate & kWRegMask);
6371 }
6372
6373 unsigned n, imm_s, imm_r;
6374 if (IsImmLogical(immediate, reg_size, &n, &imm_s, &imm_r)) {
6375 // Immediate can be encoded in the instruction.
6376 LogicalImmediate(rd, rn, n, imm_s, imm_r, op);
6377 } else {
6378 // This case is handled in the macro assembler.
6379 VIXL_UNREACHABLE();
6380 }
6381 } else {
6382 VIXL_ASSERT(operand.IsShiftedRegister());
6383 VIXL_ASSERT(operand.GetRegister().GetSizeInBits() == rd.GetSizeInBits());
6384 Instr dp_op = static_cast<Instr>(op | LogicalShiftedFixed);
6385 DataProcShiftedRegister(rd, rn, operand, LeaveFlags, dp_op);
6386 }
6387 }
6388
6389
LogicalImmediate(const Register & rd,const Register & rn,unsigned n,unsigned imm_s,unsigned imm_r,LogicalOp op)6390 void Assembler::LogicalImmediate(const Register& rd,
6391 const Register& rn,
6392 unsigned n,
6393 unsigned imm_s,
6394 unsigned imm_r,
6395 LogicalOp op) {
6396 unsigned reg_size = rd.GetSizeInBits();
6397 Instr dest_reg = (op == ANDS) ? Rd(rd) : RdSP(rd);
6398 Emit(SF(rd) | LogicalImmediateFixed | op | BitN(n, reg_size) |
6399 ImmSetBits(imm_s, reg_size) | ImmRotate(imm_r, reg_size) | dest_reg |
6400 Rn(rn));
6401 }
6402
6403
ConditionalCompare(const Register & rn,const Operand & operand,StatusFlags nzcv,Condition cond,ConditionalCompareOp op)6404 void Assembler::ConditionalCompare(const Register& rn,
6405 const Operand& operand,
6406 StatusFlags nzcv,
6407 Condition cond,
6408 ConditionalCompareOp op) {
6409 Instr ccmpop;
6410 if (operand.IsImmediate()) {
6411 int64_t immediate = operand.GetImmediate();
6412 VIXL_ASSERT(IsImmConditionalCompare(immediate));
6413 ccmpop = ConditionalCompareImmediateFixed | op |
6414 ImmCondCmp(static_cast<unsigned>(immediate));
6415 } else {
6416 VIXL_ASSERT(operand.IsShiftedRegister() && (operand.GetShiftAmount() == 0));
6417 ccmpop = ConditionalCompareRegisterFixed | op | Rm(operand.GetRegister());
6418 }
6419 Emit(SF(rn) | ccmpop | Cond(cond) | Rn(rn) | Nzcv(nzcv));
6420 }
6421
6422
DataProcessing1Source(const Register & rd,const Register & rn,DataProcessing1SourceOp op)6423 void Assembler::DataProcessing1Source(const Register& rd,
6424 const Register& rn,
6425 DataProcessing1SourceOp op) {
6426 VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
6427 Emit(SF(rn) | op | Rn(rn) | Rd(rd));
6428 }
6429
6430
FPDataProcessing1Source(const VRegister & vd,const VRegister & vn,FPDataProcessing1SourceOp op)6431 void Assembler::FPDataProcessing1Source(const VRegister& vd,
6432 const VRegister& vn,
6433 FPDataProcessing1SourceOp op) {
6434 VIXL_ASSERT(vd.Is1H() || vd.Is1S() || vd.Is1D());
6435 Emit(FPType(vn) | op | Rn(vn) | Rd(vd));
6436 }
6437
6438
FPDataProcessing3Source(const VRegister & vd,const VRegister & vn,const VRegister & vm,const VRegister & va,FPDataProcessing3SourceOp op)6439 void Assembler::FPDataProcessing3Source(const VRegister& vd,
6440 const VRegister& vn,
6441 const VRegister& vm,
6442 const VRegister& va,
6443 FPDataProcessing3SourceOp op) {
6444 VIXL_ASSERT(vd.Is1H() || vd.Is1S() || vd.Is1D());
6445 VIXL_ASSERT(AreSameSizeAndType(vd, vn, vm, va));
6446 Emit(FPType(vd) | op | Rm(vm) | Rn(vn) | Rd(vd) | Ra(va));
6447 }
6448
6449
NEONModifiedImmShiftLsl(const VRegister & vd,const int imm8,const int left_shift,NEONModifiedImmediateOp op)6450 void Assembler::NEONModifiedImmShiftLsl(const VRegister& vd,
6451 const int imm8,
6452 const int left_shift,
6453 NEONModifiedImmediateOp op) {
6454 VIXL_ASSERT(vd.Is8B() || vd.Is16B() || vd.Is4H() || vd.Is8H() || vd.Is2S() ||
6455 vd.Is4S());
6456 VIXL_ASSERT((left_shift == 0) || (left_shift == 8) || (left_shift == 16) ||
6457 (left_shift == 24));
6458 VIXL_ASSERT(IsUint8(imm8));
6459
6460 int cmode_1, cmode_2, cmode_3;
6461 if (vd.Is8B() || vd.Is16B()) {
6462 VIXL_ASSERT(op == NEONModifiedImmediate_MOVI);
6463 cmode_1 = 1;
6464 cmode_2 = 1;
6465 cmode_3 = 1;
6466 } else {
6467 cmode_1 = (left_shift >> 3) & 1;
6468 cmode_2 = left_shift >> 4;
6469 cmode_3 = 0;
6470 if (vd.Is4H() || vd.Is8H()) {
6471 VIXL_ASSERT((left_shift == 0) || (left_shift == 8));
6472 cmode_3 = 1;
6473 }
6474 }
6475 int cmode = (cmode_3 << 3) | (cmode_2 << 2) | (cmode_1 << 1);
6476
6477 int q = vd.IsQ() ? NEON_Q : 0;
6478
6479 Emit(q | op | ImmNEONabcdefgh(imm8) | NEONCmode(cmode) | Rd(vd));
6480 }
6481
6482
NEONModifiedImmShiftMsl(const VRegister & vd,const int imm8,const int shift_amount,NEONModifiedImmediateOp op)6483 void Assembler::NEONModifiedImmShiftMsl(const VRegister& vd,
6484 const int imm8,
6485 const int shift_amount,
6486 NEONModifiedImmediateOp op) {
6487 VIXL_ASSERT(vd.Is2S() || vd.Is4S());
6488 VIXL_ASSERT((shift_amount == 8) || (shift_amount == 16));
6489 VIXL_ASSERT(IsUint8(imm8));
6490
6491 int cmode_0 = (shift_amount >> 4) & 1;
6492 int cmode = 0xc | cmode_0;
6493
6494 int q = vd.IsQ() ? NEON_Q : 0;
6495
6496 Emit(q | op | ImmNEONabcdefgh(imm8) | NEONCmode(cmode) | Rd(vd));
6497 }
6498
6499
EmitShift(const Register & rd,const Register & rn,Shift shift,unsigned shift_amount)6500 void Assembler::EmitShift(const Register& rd,
6501 const Register& rn,
6502 Shift shift,
6503 unsigned shift_amount) {
6504 switch (shift) {
6505 case LSL:
6506 lsl(rd, rn, shift_amount);
6507 break;
6508 case LSR:
6509 lsr(rd, rn, shift_amount);
6510 break;
6511 case ASR:
6512 asr(rd, rn, shift_amount);
6513 break;
6514 case ROR:
6515 ror(rd, rn, shift_amount);
6516 break;
6517 default:
6518 VIXL_UNREACHABLE();
6519 }
6520 }
6521
6522
EmitExtendShift(const Register & rd,const Register & rn,Extend extend,unsigned left_shift)6523 void Assembler::EmitExtendShift(const Register& rd,
6524 const Register& rn,
6525 Extend extend,
6526 unsigned left_shift) {
6527 VIXL_ASSERT(rd.GetSizeInBits() >= rn.GetSizeInBits());
6528 unsigned reg_size = rd.GetSizeInBits();
6529 // Use the correct size of register.
6530 Register rn_ = Register(rn.GetCode(), rd.GetSizeInBits());
6531 // Bits extracted are high_bit:0.
6532 unsigned high_bit = (8 << (extend & 0x3)) - 1;
6533 // Number of bits left in the result that are not introduced by the shift.
6534 unsigned non_shift_bits = (reg_size - left_shift) & (reg_size - 1);
6535
6536 if ((non_shift_bits > high_bit) || (non_shift_bits == 0)) {
6537 switch (extend) {
6538 case UXTB:
6539 case UXTH:
6540 case UXTW:
6541 ubfm(rd, rn_, non_shift_bits, high_bit);
6542 break;
6543 case SXTB:
6544 case SXTH:
6545 case SXTW:
6546 sbfm(rd, rn_, non_shift_bits, high_bit);
6547 break;
6548 case UXTX:
6549 case SXTX: {
6550 VIXL_ASSERT(rn.GetSizeInBits() == kXRegSize);
6551 // Nothing to extend. Just shift.
6552 lsl(rd, rn_, left_shift);
6553 break;
6554 }
6555 default:
6556 VIXL_UNREACHABLE();
6557 }
6558 } else {
6559 // No need to extend as the extended bits would be shifted away.
6560 lsl(rd, rn_, left_shift);
6561 }
6562 }
6563
6564
DataProcShiftedRegister(const Register & rd,const Register & rn,const Operand & operand,FlagsUpdate S,Instr op)6565 void Assembler::DataProcShiftedRegister(const Register& rd,
6566 const Register& rn,
6567 const Operand& operand,
6568 FlagsUpdate S,
6569 Instr op) {
6570 VIXL_ASSERT(operand.IsShiftedRegister());
6571 VIXL_ASSERT(rn.Is64Bits() ||
6572 (rn.Is32Bits() && IsUint5(operand.GetShiftAmount())));
6573 Emit(SF(rd) | op | Flags(S) | ShiftDP(operand.GetShift()) |
6574 ImmDPShift(operand.GetShiftAmount()) | Rm(operand.GetRegister()) |
6575 Rn(rn) | Rd(rd));
6576 }
6577
6578
DataProcExtendedRegister(const Register & rd,const Register & rn,const Operand & operand,FlagsUpdate S,Instr op)6579 void Assembler::DataProcExtendedRegister(const Register& rd,
6580 const Register& rn,
6581 const Operand& operand,
6582 FlagsUpdate S,
6583 Instr op) {
6584 Instr dest_reg = (S == SetFlags) ? Rd(rd) : RdSP(rd);
6585 Emit(SF(rd) | op | Flags(S) | Rm(operand.GetRegister()) |
6586 ExtendMode(operand.GetExtend()) |
6587 ImmExtendShift(operand.GetShiftAmount()) | dest_reg | RnSP(rn));
6588 }
6589
6590
LoadStoreMemOperand(const MemOperand & addr,unsigned access_size_in_bytes_log2,LoadStoreScalingOption option)6591 Instr Assembler::LoadStoreMemOperand(const MemOperand& addr,
6592 unsigned access_size_in_bytes_log2,
6593 LoadStoreScalingOption option) {
6594 Instr base = RnSP(addr.GetBaseRegister());
6595 int64_t offset = addr.GetOffset();
6596
6597 if (addr.IsImmediateOffset()) {
6598 bool prefer_unscaled =
6599 (option == PreferUnscaledOffset) || (option == RequireUnscaledOffset);
6600 if (prefer_unscaled && IsImmLSUnscaled(offset)) {
6601 // Use the unscaled addressing mode.
6602 return base | LoadStoreUnscaledOffsetFixed | ImmLS(offset);
6603 }
6604
6605 if ((option != RequireUnscaledOffset) &&
6606 IsImmLSScaled(offset, access_size_in_bytes_log2)) {
6607 // We need `offset` to be positive for the shift to be well-defined.
6608 // IsImmLSScaled should check this.
6609 VIXL_ASSERT(offset >= 0);
6610 // Use the scaled addressing mode.
6611 return base | LoadStoreUnsignedOffsetFixed |
6612 ImmLSUnsigned(offset >> access_size_in_bytes_log2);
6613 }
6614
6615 if ((option != RequireScaledOffset) && IsImmLSUnscaled(offset)) {
6616 // Use the unscaled addressing mode.
6617 return base | LoadStoreUnscaledOffsetFixed | ImmLS(offset);
6618 }
6619 }
6620
6621 // All remaining addressing modes are register-offset, pre-indexed or
6622 // post-indexed modes.
6623 VIXL_ASSERT((option != RequireUnscaledOffset) &&
6624 (option != RequireScaledOffset));
6625
6626 if (addr.IsRegisterOffset()) {
6627 Extend ext = addr.GetExtend();
6628 Shift shift = addr.GetShift();
6629 unsigned shift_amount = addr.GetShiftAmount();
6630
6631 // LSL is encoded in the option field as UXTX.
6632 if (shift == LSL) {
6633 ext = UXTX;
6634 }
6635
6636 // Shifts are encoded in one bit, indicating a left shift by the memory
6637 // access size.
6638 VIXL_ASSERT((shift_amount == 0) || (shift_amount == access_size_in_bytes_log2));
6639 return base | LoadStoreRegisterOffsetFixed | Rm(addr.GetRegisterOffset()) |
6640 ExtendMode(ext) | ImmShiftLS((shift_amount > 0) ? 1 : 0);
6641 }
6642
6643 if (addr.IsImmediatePreIndex() && IsImmLSUnscaled(offset)) {
6644 return base | LoadStorePreIndexFixed | ImmLS(offset);
6645 }
6646
6647 if (addr.IsImmediatePostIndex() && IsImmLSUnscaled(offset)) {
6648 return base | LoadStorePostIndexFixed | ImmLS(offset);
6649 }
6650
6651 // If this point is reached, the MemOperand (addr) cannot be encoded.
6652 VIXL_UNREACHABLE();
6653 return 0;
6654 }
6655
6656
LoadStore(const CPURegister & rt,const MemOperand & addr,LoadStoreOp op,LoadStoreScalingOption option)6657 void Assembler::LoadStore(const CPURegister& rt,
6658 const MemOperand& addr,
6659 LoadStoreOp op,
6660 LoadStoreScalingOption option) {
6661 VIXL_ASSERT(CPUHas(rt));
6662 Emit(op | Rt(rt) | LoadStoreMemOperand(addr, CalcLSDataSize(op), option));
6663 }
6664
LoadStorePAC(const Register & xt,const MemOperand & addr,LoadStorePACOp op)6665 void Assembler::LoadStorePAC(const Register& xt,
6666 const MemOperand& addr,
6667 LoadStorePACOp op) {
6668 VIXL_ASSERT(xt.Is64Bits());
6669 VIXL_ASSERT(addr.IsImmediateOffset() || addr.IsImmediatePreIndex());
6670
6671 Instr pac_op = op;
6672 if (addr.IsImmediatePreIndex()) {
6673 pac_op |= LoadStorePACPreBit;
6674 }
6675
6676 Instr base = RnSP(addr.GetBaseRegister());
6677 int64_t offset = addr.GetOffset();
6678
6679 Emit(pac_op | Rt(xt) | base | ImmLSPAC(static_cast<int>(offset)));
6680 }
6681
6682
Prefetch(int op,const MemOperand & addr,LoadStoreScalingOption option)6683 void Assembler::Prefetch(int op,
6684 const MemOperand& addr,
6685 LoadStoreScalingOption option) {
6686 VIXL_ASSERT(addr.IsRegisterOffset() || addr.IsImmediateOffset());
6687
6688 Instr prfop = ImmPrefetchOperation(op);
6689 Emit(PRFM | prfop | LoadStoreMemOperand(addr, kXRegSizeInBytesLog2, option));
6690 }
6691
Prefetch(PrefetchOperation op,const MemOperand & addr,LoadStoreScalingOption option)6692 void Assembler::Prefetch(PrefetchOperation op,
6693 const MemOperand& addr,
6694 LoadStoreScalingOption option) {
6695 // Passing unnamed values in 'op' is undefined behaviour in C++.
6696 VIXL_ASSERT(IsNamedPrefetchOperation(op));
6697 Prefetch(static_cast<int>(op), addr, option);
6698 }
6699
6700
IsImmAddSub(int64_t immediate)6701 bool Assembler::IsImmAddSub(int64_t immediate) {
6702 return IsUint12(immediate) ||
6703 (IsUint12(immediate >> 12) && ((immediate & 0xfff) == 0));
6704 }
6705
6706
IsImmConditionalCompare(int64_t immediate)6707 bool Assembler::IsImmConditionalCompare(int64_t immediate) {
6708 return IsUint5(immediate);
6709 }
6710
6711
IsImmFP16(Float16 imm)6712 bool Assembler::IsImmFP16(Float16 imm) {
6713 // Valid values will have the form:
6714 // aBbb.cdef.gh00.000
6715 uint16_t bits = Float16ToRawbits(imm);
6716 // bits[6..0] are cleared.
6717 if ((bits & 0x3f) != 0) {
6718 return false;
6719 }
6720
6721 // bits[13..12] are all set or all cleared.
6722 uint16_t b_pattern = (bits >> 12) & 0x03;
6723 if (b_pattern != 0 && b_pattern != 0x03) {
6724 return false;
6725 }
6726
6727 // bit[15] and bit[14] are opposite.
6728 if (((bits ^ (bits << 1)) & 0x4000) == 0) {
6729 return false;
6730 }
6731
6732 return true;
6733 }
6734
6735
IsImmFP32(uint32_t bits)6736 bool Assembler::IsImmFP32(uint32_t bits) {
6737 // Valid values will have the form:
6738 // aBbb.bbbc.defg.h000.0000.0000.0000.0000
6739 // bits[19..0] are cleared.
6740 if ((bits & 0x7ffff) != 0) {
6741 return false;
6742 }
6743
6744 // bits[29..25] are all set or all cleared.
6745 uint32_t b_pattern = (bits >> 16) & 0x3e00;
6746 if (b_pattern != 0 && b_pattern != 0x3e00) {
6747 return false;
6748 }
6749
6750 // bit[30] and bit[29] are opposite.
6751 if (((bits ^ (bits << 1)) & 0x40000000) == 0) {
6752 return false;
6753 }
6754
6755 return true;
6756 }
6757
6758
IsImmFP64(uint64_t bits)6759 bool Assembler::IsImmFP64(uint64_t bits) {
6760 // Valid values will have the form:
6761 // aBbb.bbbb.bbcd.efgh.0000.0000.0000.0000
6762 // 0000.0000.0000.0000.0000.0000.0000.0000
6763 // bits[47..0] are cleared.
6764 if ((bits & 0x0000ffffffffffff) != 0) {
6765 return false;
6766 }
6767
6768 // bits[61..54] are all set or all cleared.
6769 uint32_t b_pattern = (bits >> 48) & 0x3fc0;
6770 if ((b_pattern != 0) && (b_pattern != 0x3fc0)) {
6771 return false;
6772 }
6773
6774 // bit[62] and bit[61] are opposite.
6775 if (((bits ^ (bits << 1)) & (UINT64_C(1) << 62)) == 0) {
6776 return false;
6777 }
6778
6779 return true;
6780 }
6781
6782
IsImmLSPair(int64_t offset,unsigned access_size_in_bytes_log2)6783 bool Assembler::IsImmLSPair(int64_t offset, unsigned access_size_in_bytes_log2) {
6784 const auto access_size_in_bytes = 1U << access_size_in_bytes_log2;
6785 VIXL_ASSERT(access_size_in_bytes_log2 <= kQRegSizeInBytesLog2);
6786 return IsMultiple(offset, access_size_in_bytes) &&
6787 IsInt7(offset / access_size_in_bytes);
6788 }
6789
6790
IsImmLSScaled(int64_t offset,unsigned access_size_in_bytes_log2)6791 bool Assembler::IsImmLSScaled(int64_t offset, unsigned access_size_in_bytes_log2) {
6792 const auto access_size_in_bytes = 1U << access_size_in_bytes_log2;
6793 VIXL_ASSERT(access_size_in_bytes_log2 <= kQRegSizeInBytesLog2);
6794 return IsMultiple(offset, access_size_in_bytes) &&
6795 IsUint12(offset / access_size_in_bytes);
6796 }
6797
6798
IsImmLSUnscaled(int64_t offset)6799 bool Assembler::IsImmLSUnscaled(int64_t offset) { return IsInt9(offset); }
6800
6801
6802 // The movn instruction can generate immediates containing an arbitrary 16-bit
6803 // value, with remaining bits set, eg. 0xffff1234, 0xffff1234ffffffff.
IsImmMovn(uint64_t imm,unsigned reg_size)6804 bool Assembler::IsImmMovn(uint64_t imm, unsigned reg_size) {
6805 return IsImmMovz(~imm, reg_size);
6806 }
6807
6808
6809 // The movz instruction can generate immediates containing an arbitrary 16-bit
6810 // value, with remaining bits clear, eg. 0x00001234, 0x0000123400000000.
IsImmMovz(uint64_t imm,unsigned reg_size)6811 bool Assembler::IsImmMovz(uint64_t imm, unsigned reg_size) {
6812 VIXL_ASSERT((reg_size == kXRegSize) || (reg_size == kWRegSize));
6813 return CountClearHalfWords(imm, reg_size) >= ((reg_size / 16) - 1);
6814 }
6815
6816
6817 // Test if a given value can be encoded in the immediate field of a logical
6818 // instruction.
6819 // If it can be encoded, the function returns true, and values pointed to by n,
6820 // imm_s and imm_r are updated with immediates encoded in the format required
6821 // by the corresponding fields in the logical instruction.
6822 // If it can not be encoded, the function returns false, and the values pointed
6823 // to by n, imm_s and imm_r are undefined.
IsImmLogical(uint64_t value,unsigned width,unsigned * n,unsigned * imm_s,unsigned * imm_r)6824 bool Assembler::IsImmLogical(uint64_t value,
6825 unsigned width,
6826 unsigned* n,
6827 unsigned* imm_s,
6828 unsigned* imm_r) {
6829 VIXL_ASSERT((width == kBRegSize) || (width == kHRegSize) ||
6830 (width == kSRegSize) || (width == kDRegSize));
6831
6832 bool negate = false;
6833
6834 // Logical immediates are encoded using parameters n, imm_s and imm_r using
6835 // the following table:
6836 //
6837 // N imms immr size S R
6838 // 1 ssssss rrrrrr 64 UInt(ssssss) UInt(rrrrrr)
6839 // 0 0sssss xrrrrr 32 UInt(sssss) UInt(rrrrr)
6840 // 0 10ssss xxrrrr 16 UInt(ssss) UInt(rrrr)
6841 // 0 110sss xxxrrr 8 UInt(sss) UInt(rrr)
6842 // 0 1110ss xxxxrr 4 UInt(ss) UInt(rr)
6843 // 0 11110s xxxxxr 2 UInt(s) UInt(r)
6844 // (s bits must not be all set)
6845 //
6846 // A pattern is constructed of size bits, where the least significant S+1 bits
6847 // are set. The pattern is rotated right by R, and repeated across a 32 or
6848 // 64-bit value, depending on destination register width.
6849 //
6850 // Put another way: the basic format of a logical immediate is a single
6851 // contiguous stretch of 1 bits, repeated across the whole word at intervals
6852 // given by a power of 2. To identify them quickly, we first locate the
6853 // lowest stretch of 1 bits, then the next 1 bit above that; that combination
6854 // is different for every logical immediate, so it gives us all the
6855 // information we need to identify the only logical immediate that our input
6856 // could be, and then we simply check if that's the value we actually have.
6857 //
6858 // (The rotation parameter does give the possibility of the stretch of 1 bits
6859 // going 'round the end' of the word. To deal with that, we observe that in
6860 // any situation where that happens the bitwise NOT of the value is also a
6861 // valid logical immediate. So we simply invert the input whenever its low bit
6862 // is set, and then we know that the rotated case can't arise.)
6863
6864 if (value & 1) {
6865 // If the low bit is 1, negate the value, and set a flag to remember that we
6866 // did (so that we can adjust the return values appropriately).
6867 negate = true;
6868 value = ~value;
6869 }
6870
6871 if (width <= kWRegSize) {
6872 // To handle 8/16/32-bit logical immediates, the very easiest thing is to repeat
6873 // the input value to fill a 64-bit word. The correct encoding of that as a
6874 // logical immediate will also be the correct encoding of the value.
6875
6876 // Avoid making the assumption that the most-significant 56/48/32 bits are zero by
6877 // shifting the value left and duplicating it.
6878 for (unsigned bits = width; bits <= kWRegSize; bits *= 2) {
6879 value <<= bits;
6880 uint64_t mask = (UINT64_C(1) << bits) - 1;
6881 value |= ((value >> bits) & mask);
6882 }
6883 }
6884
6885 // The basic analysis idea: imagine our input word looks like this.
6886 //
6887 // 0011111000111110001111100011111000111110001111100011111000111110
6888 // c b a
6889 // |<--d-->|
6890 //
6891 // We find the lowest set bit (as an actual power-of-2 value, not its index)
6892 // and call it a. Then we add a to our original number, which wipes out the
6893 // bottommost stretch of set bits and replaces it with a 1 carried into the
6894 // next zero bit. Then we look for the new lowest set bit, which is in
6895 // position b, and subtract it, so now our number is just like the original
6896 // but with the lowest stretch of set bits completely gone. Now we find the
6897 // lowest set bit again, which is position c in the diagram above. Then we'll
6898 // measure the distance d between bit positions a and c (using CLZ), and that
6899 // tells us that the only valid logical immediate that could possibly be equal
6900 // to this number is the one in which a stretch of bits running from a to just
6901 // below b is replicated every d bits.
6902 uint64_t a = LowestSetBit(value);
6903 uint64_t value_plus_a = value + a;
6904 uint64_t b = LowestSetBit(value_plus_a);
6905 uint64_t value_plus_a_minus_b = value_plus_a - b;
6906 uint64_t c = LowestSetBit(value_plus_a_minus_b);
6907
6908 int d, clz_a, out_n;
6909 uint64_t mask;
6910
6911 if (c != 0) {
6912 // The general case, in which there is more than one stretch of set bits.
6913 // Compute the repeat distance d, and set up a bitmask covering the basic
6914 // unit of repetition (i.e. a word with the bottom d bits set). Also, in all
6915 // of these cases the N bit of the output will be zero.
6916 clz_a = CountLeadingZeros(a, kXRegSize);
6917 int clz_c = CountLeadingZeros(c, kXRegSize);
6918 d = clz_a - clz_c;
6919 mask = ((UINT64_C(1) << d) - 1);
6920 out_n = 0;
6921 } else {
6922 // Handle degenerate cases.
6923 //
6924 // If any of those 'find lowest set bit' operations didn't find a set bit at
6925 // all, then the word will have been zero thereafter, so in particular the
6926 // last lowest_set_bit operation will have returned zero. So we can test for
6927 // all the special case conditions in one go by seeing if c is zero.
6928 if (a == 0) {
6929 // The input was zero (or all 1 bits, which will come to here too after we
6930 // inverted it at the start of the function), for which we just return
6931 // false.
6932 return false;
6933 } else {
6934 // Otherwise, if c was zero but a was not, then there's just one stretch
6935 // of set bits in our word, meaning that we have the trivial case of
6936 // d == 64 and only one 'repetition'. Set up all the same variables as in
6937 // the general case above, and set the N bit in the output.
6938 clz_a = CountLeadingZeros(a, kXRegSize);
6939 d = 64;
6940 mask = ~UINT64_C(0);
6941 out_n = 1;
6942 }
6943 }
6944
6945 // If the repeat period d is not a power of two, it can't be encoded.
6946 if (!IsPowerOf2(d)) {
6947 return false;
6948 }
6949
6950 if (((b - a) & ~mask) != 0) {
6951 // If the bit stretch (b - a) does not fit within the mask derived from the
6952 // repeat period, then fail.
6953 return false;
6954 }
6955
6956 // The only possible option is b - a repeated every d bits. Now we're going to
6957 // actually construct the valid logical immediate derived from that
6958 // specification, and see if it equals our original input.
6959 //
6960 // To repeat a value every d bits, we multiply it by a number of the form
6961 // (1 + 2^d + 2^(2d) + ...), i.e. 0x0001000100010001 or similar. These can
6962 // be derived using a table lookup on CLZ(d).
6963 static const uint64_t multipliers[] = {
6964 0x0000000000000001UL,
6965 0x0000000100000001UL,
6966 0x0001000100010001UL,
6967 0x0101010101010101UL,
6968 0x1111111111111111UL,
6969 0x5555555555555555UL,
6970 };
6971 uint64_t multiplier = multipliers[CountLeadingZeros(d, kXRegSize) - 57];
6972 uint64_t candidate = (b - a) * multiplier;
6973
6974 if (value != candidate) {
6975 // The candidate pattern doesn't match our input value, so fail.
6976 return false;
6977 }
6978
6979 // We have a match! This is a valid logical immediate, so now we have to
6980 // construct the bits and pieces of the instruction encoding that generates
6981 // it.
6982
6983 // Count the set bits in our basic stretch. The special case of clz(0) == -1
6984 // makes the answer come out right for stretches that reach the very top of
6985 // the word (e.g. numbers like 0xffffc00000000000).
6986 int clz_b = (b == 0) ? -1 : CountLeadingZeros(b, kXRegSize);
6987 int s = clz_a - clz_b;
6988
6989 // Decide how many bits to rotate right by, to put the low bit of that basic
6990 // stretch in position a.
6991 int r;
6992 if (negate) {
6993 // If we inverted the input right at the start of this function, here's
6994 // where we compensate: the number of set bits becomes the number of clear
6995 // bits, and the rotation count is based on position b rather than position
6996 // a (since b is the location of the 'lowest' 1 bit after inversion).
6997 s = d - s;
6998 r = (clz_b + 1) & (d - 1);
6999 } else {
7000 r = (clz_a + 1) & (d - 1);
7001 }
7002
7003 // Now we're done, except for having to encode the S output in such a way that
7004 // it gives both the number of set bits and the length of the repeated
7005 // segment. The s field is encoded like this:
7006 //
7007 // imms size S
7008 // ssssss 64 UInt(ssssss)
7009 // 0sssss 32 UInt(sssss)
7010 // 10ssss 16 UInt(ssss)
7011 // 110sss 8 UInt(sss)
7012 // 1110ss 4 UInt(ss)
7013 // 11110s 2 UInt(s)
7014 //
7015 // So we 'or' (2 * -d) with our computed s to form imms.
7016 if ((n != NULL) || (imm_s != NULL) || (imm_r != NULL)) {
7017 *n = out_n;
7018 *imm_s = ((2 * -d) | (s - 1)) & 0x3f;
7019 *imm_r = r;
7020 }
7021
7022 return true;
7023 }
7024
7025
LoadOpFor(const CPURegister & rt)7026 LoadStoreOp Assembler::LoadOpFor(const CPURegister& rt) {
7027 VIXL_ASSERT(rt.IsValid());
7028 if (rt.IsRegister()) {
7029 return rt.Is64Bits() ? LDR_x : LDR_w;
7030 } else {
7031 VIXL_ASSERT(rt.IsVRegister());
7032 switch (rt.GetSizeInBits()) {
7033 case kBRegSize:
7034 return LDR_b;
7035 case kHRegSize:
7036 return LDR_h;
7037 case kSRegSize:
7038 return LDR_s;
7039 case kDRegSize:
7040 return LDR_d;
7041 default:
7042 VIXL_ASSERT(rt.IsQ());
7043 return LDR_q;
7044 }
7045 }
7046 }
7047
7048
StoreOpFor(const CPURegister & rt)7049 LoadStoreOp Assembler::StoreOpFor(const CPURegister& rt) {
7050 VIXL_ASSERT(rt.IsValid());
7051 if (rt.IsRegister()) {
7052 return rt.Is64Bits() ? STR_x : STR_w;
7053 } else {
7054 VIXL_ASSERT(rt.IsVRegister());
7055 switch (rt.GetSizeInBits()) {
7056 case kBRegSize:
7057 return STR_b;
7058 case kHRegSize:
7059 return STR_h;
7060 case kSRegSize:
7061 return STR_s;
7062 case kDRegSize:
7063 return STR_d;
7064 default:
7065 VIXL_ASSERT(rt.IsQ());
7066 return STR_q;
7067 }
7068 }
7069 }
7070
7071
StorePairOpFor(const CPURegister & rt,const CPURegister & rt2)7072 LoadStorePairOp Assembler::StorePairOpFor(const CPURegister& rt,
7073 const CPURegister& rt2) {
7074 VIXL_ASSERT(AreSameSizeAndType(rt, rt2));
7075 USE(rt2);
7076 if (rt.IsRegister()) {
7077 return rt.Is64Bits() ? STP_x : STP_w;
7078 } else {
7079 VIXL_ASSERT(rt.IsVRegister());
7080 switch (rt.GetSizeInBytes()) {
7081 case kSRegSizeInBytes:
7082 return STP_s;
7083 case kDRegSizeInBytes:
7084 return STP_d;
7085 default:
7086 VIXL_ASSERT(rt.IsQ());
7087 return STP_q;
7088 }
7089 }
7090 }
7091
7092
LoadPairOpFor(const CPURegister & rt,const CPURegister & rt2)7093 LoadStorePairOp Assembler::LoadPairOpFor(const CPURegister& rt,
7094 const CPURegister& rt2) {
7095 VIXL_ASSERT((STP_w | LoadStorePairLBit) == LDP_w);
7096 return static_cast<LoadStorePairOp>(StorePairOpFor(rt, rt2) |
7097 LoadStorePairLBit);
7098 }
7099
7100
StorePairNonTemporalOpFor(const CPURegister & rt,const CPURegister & rt2)7101 LoadStorePairNonTemporalOp Assembler::StorePairNonTemporalOpFor(
7102 const CPURegister& rt, const CPURegister& rt2) {
7103 VIXL_ASSERT(AreSameSizeAndType(rt, rt2));
7104 USE(rt2);
7105 if (rt.IsRegister()) {
7106 return rt.Is64Bits() ? STNP_x : STNP_w;
7107 } else {
7108 VIXL_ASSERT(rt.IsVRegister());
7109 switch (rt.GetSizeInBytes()) {
7110 case kSRegSizeInBytes:
7111 return STNP_s;
7112 case kDRegSizeInBytes:
7113 return STNP_d;
7114 default:
7115 VIXL_ASSERT(rt.IsQ());
7116 return STNP_q;
7117 }
7118 }
7119 }
7120
7121
LoadPairNonTemporalOpFor(const CPURegister & rt,const CPURegister & rt2)7122 LoadStorePairNonTemporalOp Assembler::LoadPairNonTemporalOpFor(
7123 const CPURegister& rt, const CPURegister& rt2) {
7124 VIXL_ASSERT((STNP_w | LoadStorePairNonTemporalLBit) == LDNP_w);
7125 return static_cast<LoadStorePairNonTemporalOp>(
7126 StorePairNonTemporalOpFor(rt, rt2) | LoadStorePairNonTemporalLBit);
7127 }
7128
7129
LoadLiteralOpFor(const CPURegister & rt)7130 LoadLiteralOp Assembler::LoadLiteralOpFor(const CPURegister& rt) {
7131 if (rt.IsRegister()) {
7132 return rt.IsX() ? LDR_x_lit : LDR_w_lit;
7133 } else {
7134 VIXL_ASSERT(rt.IsVRegister());
7135 switch (rt.GetSizeInBytes()) {
7136 case kSRegSizeInBytes:
7137 return LDR_s_lit;
7138 case kDRegSizeInBytes:
7139 return LDR_d_lit;
7140 default:
7141 VIXL_ASSERT(rt.IsQ());
7142 return LDR_q_lit;
7143 }
7144 }
7145 }
7146
7147
CPUHas(const CPURegister & rt) const7148 bool Assembler::CPUHas(const CPURegister& rt) const {
7149 // Core registers are available without any particular CPU features.
7150 if (rt.IsRegister()) return true;
7151 VIXL_ASSERT(rt.IsVRegister());
7152 // The architecture does not allow FP and NEON to be implemented separately,
7153 // but we can crudely categorise them based on register size, since FP only
7154 // uses D, S and (occasionally) H registers.
7155 if (rt.IsH() || rt.IsS() || rt.IsD()) {
7156 return CPUHas(CPUFeatures::kFP) || CPUHas(CPUFeatures::kNEON);
7157 }
7158 VIXL_ASSERT(rt.IsB() || rt.IsQ());
7159 return CPUHas(CPUFeatures::kNEON);
7160 }
7161
7162
CPUHas(const CPURegister & rt,const CPURegister & rt2) const7163 bool Assembler::CPUHas(const CPURegister& rt, const CPURegister& rt2) const {
7164 // This is currently only used for loads and stores, where rt and rt2 must
7165 // have the same size and type. We could extend this to cover other cases if
7166 // necessary, but for now we can avoid checking both registers.
7167 VIXL_ASSERT(AreSameSizeAndType(rt, rt2));
7168 USE(rt2);
7169 return CPUHas(rt);
7170 }
7171
7172
CPUHas(SystemRegister sysreg) const7173 bool Assembler::CPUHas(SystemRegister sysreg) const {
7174 switch (sysreg) {
7175 case RNDR:
7176 case RNDRRS:
7177 return CPUHas(CPUFeatures::kRNG);
7178 case FPCR:
7179 case NZCV:
7180 case DCZID_EL0:
7181 break;
7182 }
7183 return true;
7184 }
7185
7186
7187 } // namespace aarch64
7188 } // namespace vixl
7189