• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // Copyright 2015, VIXL authors
2 // All rights reserved.
3 //
4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions are met:
6 //
7 //   * Redistributions of source code must retain the above copyright notice,
8 //     this list of conditions and the following disclaimer.
9 //   * Redistributions in binary form must reproduce the above copyright notice,
10 //     this list of conditions and the following disclaimer in the documentation
11 //     and/or other materials provided with the distribution.
12 //   * Neither the name of ARM Limited nor the names of its contributors may be
13 //     used to endorse or promote products derived from this software without
14 //     specific prior written permission.
15 //
16 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
17 // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
20 // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
22 // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
23 // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24 // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 
27 
28 #include <cmath>
29 
30 #include "assembler-aarch64.h"
31 #include "macro-assembler-aarch64.h"
32 
33 namespace vixl {
34 namespace aarch64 {
35 
RawLiteral(size_t size,LiteralPool * literal_pool,DeletionPolicy deletion_policy)36 RawLiteral::RawLiteral(size_t size,
37                        LiteralPool* literal_pool,
38                        DeletionPolicy deletion_policy)
39     : size_(size),
40       offset_(0),
41       low64_(0),
42       high64_(0),
43       literal_pool_(literal_pool),
44       deletion_policy_(deletion_policy) {
45   VIXL_ASSERT((deletion_policy == kManuallyDeleted) || (literal_pool_ != NULL));
46   if (deletion_policy == kDeletedOnPoolDestruction) {
47     literal_pool_->DeleteOnDestruction(this);
48   }
49 }
50 
51 
Reset()52 void Assembler::Reset() { GetBuffer()->Reset(); }
53 
54 
bind(Label * label)55 void Assembler::bind(Label* label) {
56   BindToOffset(label, GetBuffer()->GetCursorOffset());
57 }
58 
59 
BindToOffset(Label * label,ptrdiff_t offset)60 void Assembler::BindToOffset(Label* label, ptrdiff_t offset) {
61   VIXL_ASSERT((offset >= 0) && (offset <= GetBuffer()->GetCursorOffset()));
62   VIXL_ASSERT(offset % kInstructionSize == 0);
63 
64   label->Bind(offset);
65 
66   for (Label::LabelLinksIterator it(label); !it.Done(); it.Advance()) {
67     Instruction* link =
68         GetBuffer()->GetOffsetAddress<Instruction*>(*it.Current());
69     link->SetImmPCOffsetTarget(GetLabelAddress<Instruction*>(label));
70   }
71   label->ClearAllLinks();
72 }
73 
74 
75 // A common implementation for the LinkAndGet<Type>OffsetTo helpers.
76 //
77 // The offset is calculated by aligning the PC and label addresses down to a
78 // multiple of 1 << element_shift, then calculating the (scaled) offset between
79 // them. This matches the semantics of adrp, for example.
80 template <int element_shift>
LinkAndGetOffsetTo(Label * label)81 ptrdiff_t Assembler::LinkAndGetOffsetTo(Label* label) {
82   VIXL_STATIC_ASSERT(element_shift < (sizeof(ptrdiff_t) * 8));
83 
84   if (label->IsBound()) {
85     uintptr_t pc_offset = GetCursorAddress<uintptr_t>() >> element_shift;
86     uintptr_t label_offset = GetLabelAddress<uintptr_t>(label) >> element_shift;
87     return label_offset - pc_offset;
88   } else {
89     label->AddLink(GetBuffer()->GetCursorOffset());
90     return 0;
91   }
92 }
93 
94 
LinkAndGetByteOffsetTo(Label * label)95 ptrdiff_t Assembler::LinkAndGetByteOffsetTo(Label* label) {
96   return LinkAndGetOffsetTo<0>(label);
97 }
98 
99 
LinkAndGetInstructionOffsetTo(Label * label)100 ptrdiff_t Assembler::LinkAndGetInstructionOffsetTo(Label* label) {
101   return LinkAndGetOffsetTo<kInstructionSizeLog2>(label);
102 }
103 
104 
LinkAndGetPageOffsetTo(Label * label)105 ptrdiff_t Assembler::LinkAndGetPageOffsetTo(Label* label) {
106   return LinkAndGetOffsetTo<kPageSizeLog2>(label);
107 }
108 
109 
place(RawLiteral * literal)110 void Assembler::place(RawLiteral* literal) {
111   VIXL_ASSERT(!literal->IsPlaced());
112 
113   // Patch instructions using this literal.
114   if (literal->IsUsed()) {
115     Instruction* target = GetCursorAddress<Instruction*>();
116     ptrdiff_t offset = literal->GetLastUse();
117     bool done;
118     do {
119       Instruction* ldr = GetBuffer()->GetOffsetAddress<Instruction*>(offset);
120       VIXL_ASSERT(ldr->IsLoadLiteral());
121 
122       ptrdiff_t imm19 = ldr->GetImmLLiteral();
123       VIXL_ASSERT(imm19 <= 0);
124       done = (imm19 == 0);
125       offset += imm19 * kLiteralEntrySize;
126 
127       ldr->SetImmLLiteral(target);
128     } while (!done);
129   }
130 
131   // "bind" the literal.
132   literal->SetOffset(GetCursorOffset());
133   // Copy the data into the pool.
134   switch (literal->GetSize()) {
135     case kSRegSizeInBytes:
136       dc32(literal->GetRawValue32());
137       break;
138     case kDRegSizeInBytes:
139       dc64(literal->GetRawValue64());
140       break;
141     default:
142       VIXL_ASSERT(literal->GetSize() == kQRegSizeInBytes);
143       dc64(literal->GetRawValue128Low64());
144       dc64(literal->GetRawValue128High64());
145   }
146 
147   literal->literal_pool_ = NULL;
148 }
149 
150 
LinkAndGetWordOffsetTo(RawLiteral * literal)151 ptrdiff_t Assembler::LinkAndGetWordOffsetTo(RawLiteral* literal) {
152   VIXL_ASSERT(IsWordAligned(GetCursorOffset()));
153 
154   bool register_first_use =
155       (literal->GetLiteralPool() != NULL) && !literal->IsUsed();
156 
157   if (literal->IsPlaced()) {
158     // The literal is "behind", the offset will be negative.
159     VIXL_ASSERT((literal->GetOffset() - GetCursorOffset()) <= 0);
160     return (literal->GetOffset() - GetCursorOffset()) >> kLiteralEntrySizeLog2;
161   }
162 
163   ptrdiff_t offset = 0;
164   // Link all uses together.
165   if (literal->IsUsed()) {
166     offset =
167         (literal->GetLastUse() - GetCursorOffset()) >> kLiteralEntrySizeLog2;
168   }
169   literal->SetLastUse(GetCursorOffset());
170 
171   if (register_first_use) {
172     literal->GetLiteralPool()->AddEntry(literal);
173   }
174 
175   return offset;
176 }
177 
178 
179 // Code generation.
br(const Register & xn)180 void Assembler::br(const Register& xn) {
181   VIXL_ASSERT(xn.Is64Bits());
182   Emit(BR | Rn(xn));
183 }
184 
185 
blr(const Register & xn)186 void Assembler::blr(const Register& xn) {
187   VIXL_ASSERT(xn.Is64Bits());
188   Emit(BLR | Rn(xn));
189 }
190 
191 
ret(const Register & xn)192 void Assembler::ret(const Register& xn) {
193   VIXL_ASSERT(xn.Is64Bits());
194   Emit(RET | Rn(xn));
195 }
196 
197 
braaz(const Register & xn)198 void Assembler::braaz(const Register& xn) {
199   VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
200   VIXL_ASSERT(xn.Is64Bits());
201   Emit(BRAAZ | Rn(xn) | Rd_mask);
202 }
203 
brabz(const Register & xn)204 void Assembler::brabz(const Register& xn) {
205   VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
206   VIXL_ASSERT(xn.Is64Bits());
207   Emit(BRABZ | Rn(xn) | Rd_mask);
208 }
209 
blraaz(const Register & xn)210 void Assembler::blraaz(const Register& xn) {
211   VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
212   VIXL_ASSERT(xn.Is64Bits());
213   Emit(BLRAAZ | Rn(xn) | Rd_mask);
214 }
215 
blrabz(const Register & xn)216 void Assembler::blrabz(const Register& xn) {
217   VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
218   VIXL_ASSERT(xn.Is64Bits());
219   Emit(BLRABZ | Rn(xn) | Rd_mask);
220 }
221 
retaa()222 void Assembler::retaa() {
223   VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
224   Emit(RETAA | Rn_mask | Rd_mask);
225 }
226 
retab()227 void Assembler::retab() {
228   VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
229   Emit(RETAB | Rn_mask | Rd_mask);
230 }
231 
232 // The Arm ARM names the register Xm but encodes it in the Xd bitfield.
braa(const Register & xn,const Register & xm)233 void Assembler::braa(const Register& xn, const Register& xm) {
234   VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
235   VIXL_ASSERT(xn.Is64Bits() && xm.Is64Bits());
236   Emit(BRAA | Rn(xn) | RdSP(xm));
237 }
238 
brab(const Register & xn,const Register & xm)239 void Assembler::brab(const Register& xn, const Register& xm) {
240   VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
241   VIXL_ASSERT(xn.Is64Bits() && xm.Is64Bits());
242   Emit(BRAB | Rn(xn) | RdSP(xm));
243 }
244 
blraa(const Register & xn,const Register & xm)245 void Assembler::blraa(const Register& xn, const Register& xm) {
246   VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
247   VIXL_ASSERT(xn.Is64Bits() && xm.Is64Bits());
248   Emit(BLRAA | Rn(xn) | RdSP(xm));
249 }
250 
blrab(const Register & xn,const Register & xm)251 void Assembler::blrab(const Register& xn, const Register& xm) {
252   VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
253   VIXL_ASSERT(xn.Is64Bits() && xm.Is64Bits());
254   Emit(BLRAB | Rn(xn) | RdSP(xm));
255 }
256 
257 
b(int64_t imm26)258 void Assembler::b(int64_t imm26) { Emit(B | ImmUncondBranch(imm26)); }
259 
260 
b(int64_t imm19,Condition cond)261 void Assembler::b(int64_t imm19, Condition cond) {
262   Emit(B_cond | ImmCondBranch(imm19) | cond);
263 }
264 
265 
b(Label * label)266 void Assembler::b(Label* label) {
267   int64_t offset = LinkAndGetInstructionOffsetTo(label);
268   VIXL_ASSERT(Instruction::IsValidImmPCOffset(UncondBranchType, offset));
269   b(static_cast<int>(offset));
270 }
271 
272 
b(Label * label,Condition cond)273 void Assembler::b(Label* label, Condition cond) {
274   int64_t offset = LinkAndGetInstructionOffsetTo(label);
275   VIXL_ASSERT(Instruction::IsValidImmPCOffset(CondBranchType, offset));
276   b(static_cast<int>(offset), cond);
277 }
278 
279 
bl(int64_t imm26)280 void Assembler::bl(int64_t imm26) { Emit(BL | ImmUncondBranch(imm26)); }
281 
282 
bl(Label * label)283 void Assembler::bl(Label* label) {
284   int64_t offset = LinkAndGetInstructionOffsetTo(label);
285   VIXL_ASSERT(Instruction::IsValidImmPCOffset(UncondBranchType, offset));
286   bl(static_cast<int>(offset));
287 }
288 
289 
cbz(const Register & rt,int64_t imm19)290 void Assembler::cbz(const Register& rt, int64_t imm19) {
291   Emit(SF(rt) | CBZ | ImmCmpBranch(imm19) | Rt(rt));
292 }
293 
294 
cbz(const Register & rt,Label * label)295 void Assembler::cbz(const Register& rt, Label* label) {
296   int64_t offset = LinkAndGetInstructionOffsetTo(label);
297   VIXL_ASSERT(Instruction::IsValidImmPCOffset(CompareBranchType, offset));
298   cbz(rt, static_cast<int>(offset));
299 }
300 
301 
cbnz(const Register & rt,int64_t imm19)302 void Assembler::cbnz(const Register& rt, int64_t imm19) {
303   Emit(SF(rt) | CBNZ | ImmCmpBranch(imm19) | Rt(rt));
304 }
305 
306 
cbnz(const Register & rt,Label * label)307 void Assembler::cbnz(const Register& rt, Label* label) {
308   int64_t offset = LinkAndGetInstructionOffsetTo(label);
309   VIXL_ASSERT(Instruction::IsValidImmPCOffset(CompareBranchType, offset));
310   cbnz(rt, static_cast<int>(offset));
311 }
312 
313 
NEONTable(const VRegister & vd,const VRegister & vn,const VRegister & vm,NEONTableOp op)314 void Assembler::NEONTable(const VRegister& vd,
315                           const VRegister& vn,
316                           const VRegister& vm,
317                           NEONTableOp op) {
318   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
319   VIXL_ASSERT(vd.Is16B() || vd.Is8B());
320   VIXL_ASSERT(vn.Is16B());
321   VIXL_ASSERT(AreSameFormat(vd, vm));
322   Emit(op | (vd.IsQ() ? NEON_Q : 0) | Rm(vm) | Rn(vn) | Rd(vd));
323 }
324 
325 
tbl(const VRegister & vd,const VRegister & vn,const VRegister & vm)326 void Assembler::tbl(const VRegister& vd,
327                     const VRegister& vn,
328                     const VRegister& vm) {
329   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
330   NEONTable(vd, vn, vm, NEON_TBL_1v);
331 }
332 
333 
tbl(const VRegister & vd,const VRegister & vn,const VRegister & vn2,const VRegister & vm)334 void Assembler::tbl(const VRegister& vd,
335                     const VRegister& vn,
336                     const VRegister& vn2,
337                     const VRegister& vm) {
338   USE(vn2);
339   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
340   VIXL_ASSERT(AreSameFormat(vn, vn2));
341   VIXL_ASSERT(AreConsecutive(vn, vn2));
342   NEONTable(vd, vn, vm, NEON_TBL_2v);
343 }
344 
345 
tbl(const VRegister & vd,const VRegister & vn,const VRegister & vn2,const VRegister & vn3,const VRegister & vm)346 void Assembler::tbl(const VRegister& vd,
347                     const VRegister& vn,
348                     const VRegister& vn2,
349                     const VRegister& vn3,
350                     const VRegister& vm) {
351   USE(vn2, vn3);
352   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
353   VIXL_ASSERT(AreSameFormat(vn, vn2, vn3));
354   VIXL_ASSERT(AreConsecutive(vn, vn2, vn3));
355   NEONTable(vd, vn, vm, NEON_TBL_3v);
356 }
357 
358 
tbl(const VRegister & vd,const VRegister & vn,const VRegister & vn2,const VRegister & vn3,const VRegister & vn4,const VRegister & vm)359 void Assembler::tbl(const VRegister& vd,
360                     const VRegister& vn,
361                     const VRegister& vn2,
362                     const VRegister& vn3,
363                     const VRegister& vn4,
364                     const VRegister& vm) {
365   USE(vn2, vn3, vn4);
366   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
367   VIXL_ASSERT(AreSameFormat(vn, vn2, vn3, vn4));
368   VIXL_ASSERT(AreConsecutive(vn, vn2, vn3, vn4));
369   NEONTable(vd, vn, vm, NEON_TBL_4v);
370 }
371 
372 
tbx(const VRegister & vd,const VRegister & vn,const VRegister & vm)373 void Assembler::tbx(const VRegister& vd,
374                     const VRegister& vn,
375                     const VRegister& vm) {
376   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
377   NEONTable(vd, vn, vm, NEON_TBX_1v);
378 }
379 
380 
tbx(const VRegister & vd,const VRegister & vn,const VRegister & vn2,const VRegister & vm)381 void Assembler::tbx(const VRegister& vd,
382                     const VRegister& vn,
383                     const VRegister& vn2,
384                     const VRegister& vm) {
385   USE(vn2);
386   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
387   VIXL_ASSERT(AreSameFormat(vn, vn2));
388   VIXL_ASSERT(AreConsecutive(vn, vn2));
389   NEONTable(vd, vn, vm, NEON_TBX_2v);
390 }
391 
392 
tbx(const VRegister & vd,const VRegister & vn,const VRegister & vn2,const VRegister & vn3,const VRegister & vm)393 void Assembler::tbx(const VRegister& vd,
394                     const VRegister& vn,
395                     const VRegister& vn2,
396                     const VRegister& vn3,
397                     const VRegister& vm) {
398   USE(vn2, vn3);
399   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
400   VIXL_ASSERT(AreSameFormat(vn, vn2, vn3));
401   VIXL_ASSERT(AreConsecutive(vn, vn2, vn3));
402   NEONTable(vd, vn, vm, NEON_TBX_3v);
403 }
404 
405 
tbx(const VRegister & vd,const VRegister & vn,const VRegister & vn2,const VRegister & vn3,const VRegister & vn4,const VRegister & vm)406 void Assembler::tbx(const VRegister& vd,
407                     const VRegister& vn,
408                     const VRegister& vn2,
409                     const VRegister& vn3,
410                     const VRegister& vn4,
411                     const VRegister& vm) {
412   USE(vn2, vn3, vn4);
413   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
414   VIXL_ASSERT(AreSameFormat(vn, vn2, vn3, vn4));
415   VIXL_ASSERT(AreConsecutive(vn, vn2, vn3, vn4));
416   NEONTable(vd, vn, vm, NEON_TBX_4v);
417 }
418 
419 
tbz(const Register & rt,unsigned bit_pos,int64_t imm14)420 void Assembler::tbz(const Register& rt, unsigned bit_pos, int64_t imm14) {
421   VIXL_ASSERT(rt.Is64Bits() || (rt.Is32Bits() && (bit_pos < kWRegSize)));
422   Emit(TBZ | ImmTestBranchBit(bit_pos) | ImmTestBranch(imm14) | Rt(rt));
423 }
424 
425 
tbz(const Register & rt,unsigned bit_pos,Label * label)426 void Assembler::tbz(const Register& rt, unsigned bit_pos, Label* label) {
427   ptrdiff_t offset = LinkAndGetInstructionOffsetTo(label);
428   VIXL_ASSERT(Instruction::IsValidImmPCOffset(TestBranchType, offset));
429   tbz(rt, bit_pos, static_cast<int>(offset));
430 }
431 
432 
tbnz(const Register & rt,unsigned bit_pos,int64_t imm14)433 void Assembler::tbnz(const Register& rt, unsigned bit_pos, int64_t imm14) {
434   VIXL_ASSERT(rt.Is64Bits() || (rt.Is32Bits() && (bit_pos < kWRegSize)));
435   Emit(TBNZ | ImmTestBranchBit(bit_pos) | ImmTestBranch(imm14) | Rt(rt));
436 }
437 
438 
tbnz(const Register & rt,unsigned bit_pos,Label * label)439 void Assembler::tbnz(const Register& rt, unsigned bit_pos, Label* label) {
440   ptrdiff_t offset = LinkAndGetInstructionOffsetTo(label);
441   VIXL_ASSERT(Instruction::IsValidImmPCOffset(TestBranchType, offset));
442   tbnz(rt, bit_pos, static_cast<int>(offset));
443 }
444 
445 
adr(const Register & xd,int64_t imm21)446 void Assembler::adr(const Register& xd, int64_t imm21) {
447   VIXL_ASSERT(xd.Is64Bits());
448   Emit(ADR | ImmPCRelAddress(imm21) | Rd(xd));
449 }
450 
451 
adr(const Register & xd,Label * label)452 void Assembler::adr(const Register& xd, Label* label) {
453   adr(xd, static_cast<int>(LinkAndGetByteOffsetTo(label)));
454 }
455 
456 
adrp(const Register & xd,int64_t imm21)457 void Assembler::adrp(const Register& xd, int64_t imm21) {
458   VIXL_ASSERT(xd.Is64Bits());
459   Emit(ADRP | ImmPCRelAddress(imm21) | Rd(xd));
460 }
461 
462 
adrp(const Register & xd,Label * label)463 void Assembler::adrp(const Register& xd, Label* label) {
464   VIXL_ASSERT(AllowPageOffsetDependentCode());
465   adrp(xd, static_cast<int>(LinkAndGetPageOffsetTo(label)));
466 }
467 
468 
add(const Register & rd,const Register & rn,const Operand & operand)469 void Assembler::add(const Register& rd,
470                     const Register& rn,
471                     const Operand& operand) {
472   AddSub(rd, rn, operand, LeaveFlags, ADD);
473 }
474 
475 
adds(const Register & rd,const Register & rn,const Operand & operand)476 void Assembler::adds(const Register& rd,
477                      const Register& rn,
478                      const Operand& operand) {
479   AddSub(rd, rn, operand, SetFlags, ADD);
480 }
481 
482 
cmn(const Register & rn,const Operand & operand)483 void Assembler::cmn(const Register& rn, const Operand& operand) {
484   Register zr = AppropriateZeroRegFor(rn);
485   adds(zr, rn, operand);
486 }
487 
488 
sub(const Register & rd,const Register & rn,const Operand & operand)489 void Assembler::sub(const Register& rd,
490                     const Register& rn,
491                     const Operand& operand) {
492   AddSub(rd, rn, operand, LeaveFlags, SUB);
493 }
494 
495 
subs(const Register & rd,const Register & rn,const Operand & operand)496 void Assembler::subs(const Register& rd,
497                      const Register& rn,
498                      const Operand& operand) {
499   AddSub(rd, rn, operand, SetFlags, SUB);
500 }
501 
502 
cmp(const Register & rn,const Operand & operand)503 void Assembler::cmp(const Register& rn, const Operand& operand) {
504   Register zr = AppropriateZeroRegFor(rn);
505   subs(zr, rn, operand);
506 }
507 
508 
neg(const Register & rd,const Operand & operand)509 void Assembler::neg(const Register& rd, const Operand& operand) {
510   Register zr = AppropriateZeroRegFor(rd);
511   sub(rd, zr, operand);
512 }
513 
514 
negs(const Register & rd,const Operand & operand)515 void Assembler::negs(const Register& rd, const Operand& operand) {
516   Register zr = AppropriateZeroRegFor(rd);
517   subs(rd, zr, operand);
518 }
519 
520 
adc(const Register & rd,const Register & rn,const Operand & operand)521 void Assembler::adc(const Register& rd,
522                     const Register& rn,
523                     const Operand& operand) {
524   AddSubWithCarry(rd, rn, operand, LeaveFlags, ADC);
525 }
526 
527 
adcs(const Register & rd,const Register & rn,const Operand & operand)528 void Assembler::adcs(const Register& rd,
529                      const Register& rn,
530                      const Operand& operand) {
531   AddSubWithCarry(rd, rn, operand, SetFlags, ADC);
532 }
533 
534 
sbc(const Register & rd,const Register & rn,const Operand & operand)535 void Assembler::sbc(const Register& rd,
536                     const Register& rn,
537                     const Operand& operand) {
538   AddSubWithCarry(rd, rn, operand, LeaveFlags, SBC);
539 }
540 
541 
sbcs(const Register & rd,const Register & rn,const Operand & operand)542 void Assembler::sbcs(const Register& rd,
543                      const Register& rn,
544                      const Operand& operand) {
545   AddSubWithCarry(rd, rn, operand, SetFlags, SBC);
546 }
547 
548 
rmif(const Register & xn,unsigned rotation,StatusFlags flags)549 void Assembler::rmif(const Register& xn, unsigned rotation, StatusFlags flags) {
550   VIXL_ASSERT(CPUHas(CPUFeatures::kFlagM));
551   VIXL_ASSERT(xn.Is64Bits());
552   Emit(RMIF | Rn(xn) | ImmRMIFRotation(rotation) | Nzcv(flags));
553 }
554 
555 
setf8(const Register & rn)556 void Assembler::setf8(const Register& rn) {
557   VIXL_ASSERT(CPUHas(CPUFeatures::kFlagM));
558   Emit(SETF8 | Rn(rn));
559 }
560 
561 
setf16(const Register & rn)562 void Assembler::setf16(const Register& rn) {
563   VIXL_ASSERT(CPUHas(CPUFeatures::kFlagM));
564   Emit(SETF16 | Rn(rn));
565 }
566 
567 
ngc(const Register & rd,const Operand & operand)568 void Assembler::ngc(const Register& rd, const Operand& operand) {
569   Register zr = AppropriateZeroRegFor(rd);
570   sbc(rd, zr, operand);
571 }
572 
573 
ngcs(const Register & rd,const Operand & operand)574 void Assembler::ngcs(const Register& rd, const Operand& operand) {
575   Register zr = AppropriateZeroRegFor(rd);
576   sbcs(rd, zr, operand);
577 }
578 
579 
580 // Logical instructions.
and_(const Register & rd,const Register & rn,const Operand & operand)581 void Assembler::and_(const Register& rd,
582                      const Register& rn,
583                      const Operand& operand) {
584   Logical(rd, rn, operand, AND);
585 }
586 
587 
ands(const Register & rd,const Register & rn,const Operand & operand)588 void Assembler::ands(const Register& rd,
589                      const Register& rn,
590                      const Operand& operand) {
591   Logical(rd, rn, operand, ANDS);
592 }
593 
594 
tst(const Register & rn,const Operand & operand)595 void Assembler::tst(const Register& rn, const Operand& operand) {
596   ands(AppropriateZeroRegFor(rn), rn, operand);
597 }
598 
599 
bic(const Register & rd,const Register & rn,const Operand & operand)600 void Assembler::bic(const Register& rd,
601                     const Register& rn,
602                     const Operand& operand) {
603   Logical(rd, rn, operand, BIC);
604 }
605 
606 
bics(const Register & rd,const Register & rn,const Operand & operand)607 void Assembler::bics(const Register& rd,
608                      const Register& rn,
609                      const Operand& operand) {
610   Logical(rd, rn, operand, BICS);
611 }
612 
613 
orr(const Register & rd,const Register & rn,const Operand & operand)614 void Assembler::orr(const Register& rd,
615                     const Register& rn,
616                     const Operand& operand) {
617   Logical(rd, rn, operand, ORR);
618 }
619 
620 
orn(const Register & rd,const Register & rn,const Operand & operand)621 void Assembler::orn(const Register& rd,
622                     const Register& rn,
623                     const Operand& operand) {
624   Logical(rd, rn, operand, ORN);
625 }
626 
627 
eor(const Register & rd,const Register & rn,const Operand & operand)628 void Assembler::eor(const Register& rd,
629                     const Register& rn,
630                     const Operand& operand) {
631   Logical(rd, rn, operand, EOR);
632 }
633 
634 
eon(const Register & rd,const Register & rn,const Operand & operand)635 void Assembler::eon(const Register& rd,
636                     const Register& rn,
637                     const Operand& operand) {
638   Logical(rd, rn, operand, EON);
639 }
640 
641 
lslv(const Register & rd,const Register & rn,const Register & rm)642 void Assembler::lslv(const Register& rd,
643                      const Register& rn,
644                      const Register& rm) {
645   VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
646   VIXL_ASSERT(rd.GetSizeInBits() == rm.GetSizeInBits());
647   Emit(SF(rd) | LSLV | Rm(rm) | Rn(rn) | Rd(rd));
648 }
649 
650 
lsrv(const Register & rd,const Register & rn,const Register & rm)651 void Assembler::lsrv(const Register& rd,
652                      const Register& rn,
653                      const Register& rm) {
654   VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
655   VIXL_ASSERT(rd.GetSizeInBits() == rm.GetSizeInBits());
656   Emit(SF(rd) | LSRV | Rm(rm) | Rn(rn) | Rd(rd));
657 }
658 
659 
asrv(const Register & rd,const Register & rn,const Register & rm)660 void Assembler::asrv(const Register& rd,
661                      const Register& rn,
662                      const Register& rm) {
663   VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
664   VIXL_ASSERT(rd.GetSizeInBits() == rm.GetSizeInBits());
665   Emit(SF(rd) | ASRV | Rm(rm) | Rn(rn) | Rd(rd));
666 }
667 
668 
rorv(const Register & rd,const Register & rn,const Register & rm)669 void Assembler::rorv(const Register& rd,
670                      const Register& rn,
671                      const Register& rm) {
672   VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
673   VIXL_ASSERT(rd.GetSizeInBits() == rm.GetSizeInBits());
674   Emit(SF(rd) | RORV | Rm(rm) | Rn(rn) | Rd(rd));
675 }
676 
677 
678 // Bitfield operations.
bfm(const Register & rd,const Register & rn,unsigned immr,unsigned imms)679 void Assembler::bfm(const Register& rd,
680                     const Register& rn,
681                     unsigned immr,
682                     unsigned imms) {
683   VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
684   Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset);
685   Emit(SF(rd) | BFM | N | ImmR(immr, rd.GetSizeInBits()) |
686        ImmS(imms, rn.GetSizeInBits()) | Rn(rn) | Rd(rd));
687 }
688 
689 
sbfm(const Register & rd,const Register & rn,unsigned immr,unsigned imms)690 void Assembler::sbfm(const Register& rd,
691                      const Register& rn,
692                      unsigned immr,
693                      unsigned imms) {
694   VIXL_ASSERT(rd.Is64Bits() || rn.Is32Bits());
695   Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset);
696   Emit(SF(rd) | SBFM | N | ImmR(immr, rd.GetSizeInBits()) |
697        ImmS(imms, rn.GetSizeInBits()) | Rn(rn) | Rd(rd));
698 }
699 
700 
ubfm(const Register & rd,const Register & rn,unsigned immr,unsigned imms)701 void Assembler::ubfm(const Register& rd,
702                      const Register& rn,
703                      unsigned immr,
704                      unsigned imms) {
705   VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
706   Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset);
707   Emit(SF(rd) | UBFM | N | ImmR(immr, rd.GetSizeInBits()) |
708        ImmS(imms, rn.GetSizeInBits()) | Rn(rn) | Rd(rd));
709 }
710 
711 
extr(const Register & rd,const Register & rn,const Register & rm,unsigned lsb)712 void Assembler::extr(const Register& rd,
713                      const Register& rn,
714                      const Register& rm,
715                      unsigned lsb) {
716   VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
717   VIXL_ASSERT(rd.GetSizeInBits() == rm.GetSizeInBits());
718   Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset);
719   Emit(SF(rd) | EXTR | N | Rm(rm) | ImmS(lsb, rn.GetSizeInBits()) | Rn(rn) |
720        Rd(rd));
721 }
722 
723 
csel(const Register & rd,const Register & rn,const Register & rm,Condition cond)724 void Assembler::csel(const Register& rd,
725                      const Register& rn,
726                      const Register& rm,
727                      Condition cond) {
728   ConditionalSelect(rd, rn, rm, cond, CSEL);
729 }
730 
731 
csinc(const Register & rd,const Register & rn,const Register & rm,Condition cond)732 void Assembler::csinc(const Register& rd,
733                       const Register& rn,
734                       const Register& rm,
735                       Condition cond) {
736   ConditionalSelect(rd, rn, rm, cond, CSINC);
737 }
738 
739 
csinv(const Register & rd,const Register & rn,const Register & rm,Condition cond)740 void Assembler::csinv(const Register& rd,
741                       const Register& rn,
742                       const Register& rm,
743                       Condition cond) {
744   ConditionalSelect(rd, rn, rm, cond, CSINV);
745 }
746 
747 
csneg(const Register & rd,const Register & rn,const Register & rm,Condition cond)748 void Assembler::csneg(const Register& rd,
749                       const Register& rn,
750                       const Register& rm,
751                       Condition cond) {
752   ConditionalSelect(rd, rn, rm, cond, CSNEG);
753 }
754 
755 
cset(const Register & rd,Condition cond)756 void Assembler::cset(const Register& rd, Condition cond) {
757   VIXL_ASSERT((cond != al) && (cond != nv));
758   Register zr = AppropriateZeroRegFor(rd);
759   csinc(rd, zr, zr, InvertCondition(cond));
760 }
761 
762 
csetm(const Register & rd,Condition cond)763 void Assembler::csetm(const Register& rd, Condition cond) {
764   VIXL_ASSERT((cond != al) && (cond != nv));
765   Register zr = AppropriateZeroRegFor(rd);
766   csinv(rd, zr, zr, InvertCondition(cond));
767 }
768 
769 
cinc(const Register & rd,const Register & rn,Condition cond)770 void Assembler::cinc(const Register& rd, const Register& rn, Condition cond) {
771   VIXL_ASSERT((cond != al) && (cond != nv));
772   csinc(rd, rn, rn, InvertCondition(cond));
773 }
774 
775 
cinv(const Register & rd,const Register & rn,Condition cond)776 void Assembler::cinv(const Register& rd, const Register& rn, Condition cond) {
777   VIXL_ASSERT((cond != al) && (cond != nv));
778   csinv(rd, rn, rn, InvertCondition(cond));
779 }
780 
781 
cneg(const Register & rd,const Register & rn,Condition cond)782 void Assembler::cneg(const Register& rd, const Register& rn, Condition cond) {
783   VIXL_ASSERT((cond != al) && (cond != nv));
784   csneg(rd, rn, rn, InvertCondition(cond));
785 }
786 
787 
ConditionalSelect(const Register & rd,const Register & rn,const Register & rm,Condition cond,ConditionalSelectOp op)788 void Assembler::ConditionalSelect(const Register& rd,
789                                   const Register& rn,
790                                   const Register& rm,
791                                   Condition cond,
792                                   ConditionalSelectOp op) {
793   VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
794   VIXL_ASSERT(rd.GetSizeInBits() == rm.GetSizeInBits());
795   Emit(SF(rd) | op | Rm(rm) | Cond(cond) | Rn(rn) | Rd(rd));
796 }
797 
798 
ccmn(const Register & rn,const Operand & operand,StatusFlags nzcv,Condition cond)799 void Assembler::ccmn(const Register& rn,
800                      const Operand& operand,
801                      StatusFlags nzcv,
802                      Condition cond) {
803   ConditionalCompare(rn, operand, nzcv, cond, CCMN);
804 }
805 
806 
ccmp(const Register & rn,const Operand & operand,StatusFlags nzcv,Condition cond)807 void Assembler::ccmp(const Register& rn,
808                      const Operand& operand,
809                      StatusFlags nzcv,
810                      Condition cond) {
811   ConditionalCompare(rn, operand, nzcv, cond, CCMP);
812 }
813 
814 
DataProcessing3Source(const Register & rd,const Register & rn,const Register & rm,const Register & ra,DataProcessing3SourceOp op)815 void Assembler::DataProcessing3Source(const Register& rd,
816                                       const Register& rn,
817                                       const Register& rm,
818                                       const Register& ra,
819                                       DataProcessing3SourceOp op) {
820   Emit(SF(rd) | op | Rm(rm) | Ra(ra) | Rn(rn) | Rd(rd));
821 }
822 
823 
crc32b(const Register & wd,const Register & wn,const Register & wm)824 void Assembler::crc32b(const Register& wd,
825                        const Register& wn,
826                        const Register& wm) {
827   VIXL_ASSERT(CPUHas(CPUFeatures::kCRC32));
828   VIXL_ASSERT(wd.Is32Bits() && wn.Is32Bits() && wm.Is32Bits());
829   Emit(SF(wm) | Rm(wm) | CRC32B | Rn(wn) | Rd(wd));
830 }
831 
832 
crc32h(const Register & wd,const Register & wn,const Register & wm)833 void Assembler::crc32h(const Register& wd,
834                        const Register& wn,
835                        const Register& wm) {
836   VIXL_ASSERT(CPUHas(CPUFeatures::kCRC32));
837   VIXL_ASSERT(wd.Is32Bits() && wn.Is32Bits() && wm.Is32Bits());
838   Emit(SF(wm) | Rm(wm) | CRC32H | Rn(wn) | Rd(wd));
839 }
840 
841 
crc32w(const Register & wd,const Register & wn,const Register & wm)842 void Assembler::crc32w(const Register& wd,
843                        const Register& wn,
844                        const Register& wm) {
845   VIXL_ASSERT(CPUHas(CPUFeatures::kCRC32));
846   VIXL_ASSERT(wd.Is32Bits() && wn.Is32Bits() && wm.Is32Bits());
847   Emit(SF(wm) | Rm(wm) | CRC32W | Rn(wn) | Rd(wd));
848 }
849 
850 
crc32x(const Register & wd,const Register & wn,const Register & xm)851 void Assembler::crc32x(const Register& wd,
852                        const Register& wn,
853                        const Register& xm) {
854   VIXL_ASSERT(CPUHas(CPUFeatures::kCRC32));
855   VIXL_ASSERT(wd.Is32Bits() && wn.Is32Bits() && xm.Is64Bits());
856   Emit(SF(xm) | Rm(xm) | CRC32X | Rn(wn) | Rd(wd));
857 }
858 
859 
crc32cb(const Register & wd,const Register & wn,const Register & wm)860 void Assembler::crc32cb(const Register& wd,
861                         const Register& wn,
862                         const Register& wm) {
863   VIXL_ASSERT(CPUHas(CPUFeatures::kCRC32));
864   VIXL_ASSERT(wd.Is32Bits() && wn.Is32Bits() && wm.Is32Bits());
865   Emit(SF(wm) | Rm(wm) | CRC32CB | Rn(wn) | Rd(wd));
866 }
867 
868 
crc32ch(const Register & wd,const Register & wn,const Register & wm)869 void Assembler::crc32ch(const Register& wd,
870                         const Register& wn,
871                         const Register& wm) {
872   VIXL_ASSERT(CPUHas(CPUFeatures::kCRC32));
873   VIXL_ASSERT(wd.Is32Bits() && wn.Is32Bits() && wm.Is32Bits());
874   Emit(SF(wm) | Rm(wm) | CRC32CH | Rn(wn) | Rd(wd));
875 }
876 
877 
crc32cw(const Register & wd,const Register & wn,const Register & wm)878 void Assembler::crc32cw(const Register& wd,
879                         const Register& wn,
880                         const Register& wm) {
881   VIXL_ASSERT(CPUHas(CPUFeatures::kCRC32));
882   VIXL_ASSERT(wd.Is32Bits() && wn.Is32Bits() && wm.Is32Bits());
883   Emit(SF(wm) | Rm(wm) | CRC32CW | Rn(wn) | Rd(wd));
884 }
885 
886 
crc32cx(const Register & wd,const Register & wn,const Register & xm)887 void Assembler::crc32cx(const Register& wd,
888                         const Register& wn,
889                         const Register& xm) {
890   VIXL_ASSERT(CPUHas(CPUFeatures::kCRC32));
891   VIXL_ASSERT(wd.Is32Bits() && wn.Is32Bits() && xm.Is64Bits());
892   Emit(SF(xm) | Rm(xm) | CRC32CX | Rn(wn) | Rd(wd));
893 }
894 
895 
mul(const Register & rd,const Register & rn,const Register & rm)896 void Assembler::mul(const Register& rd,
897                     const Register& rn,
898                     const Register& rm) {
899   VIXL_ASSERT(AreSameSizeAndType(rd, rn, rm));
900   DataProcessing3Source(rd, rn, rm, AppropriateZeroRegFor(rd), MADD);
901 }
902 
903 
madd(const Register & rd,const Register & rn,const Register & rm,const Register & ra)904 void Assembler::madd(const Register& rd,
905                      const Register& rn,
906                      const Register& rm,
907                      const Register& ra) {
908   DataProcessing3Source(rd, rn, rm, ra, MADD);
909 }
910 
911 
mneg(const Register & rd,const Register & rn,const Register & rm)912 void Assembler::mneg(const Register& rd,
913                      const Register& rn,
914                      const Register& rm) {
915   VIXL_ASSERT(AreSameSizeAndType(rd, rn, rm));
916   DataProcessing3Source(rd, rn, rm, AppropriateZeroRegFor(rd), MSUB);
917 }
918 
919 
msub(const Register & rd,const Register & rn,const Register & rm,const Register & ra)920 void Assembler::msub(const Register& rd,
921                      const Register& rn,
922                      const Register& rm,
923                      const Register& ra) {
924   DataProcessing3Source(rd, rn, rm, ra, MSUB);
925 }
926 
927 
umaddl(const Register & xd,const Register & wn,const Register & wm,const Register & xa)928 void Assembler::umaddl(const Register& xd,
929                        const Register& wn,
930                        const Register& wm,
931                        const Register& xa) {
932   VIXL_ASSERT(xd.Is64Bits() && xa.Is64Bits());
933   VIXL_ASSERT(wn.Is32Bits() && wm.Is32Bits());
934   DataProcessing3Source(xd, wn, wm, xa, UMADDL_x);
935 }
936 
937 
smaddl(const Register & xd,const Register & wn,const Register & wm,const Register & xa)938 void Assembler::smaddl(const Register& xd,
939                        const Register& wn,
940                        const Register& wm,
941                        const Register& xa) {
942   VIXL_ASSERT(xd.Is64Bits() && xa.Is64Bits());
943   VIXL_ASSERT(wn.Is32Bits() && wm.Is32Bits());
944   DataProcessing3Source(xd, wn, wm, xa, SMADDL_x);
945 }
946 
947 
umsubl(const Register & xd,const Register & wn,const Register & wm,const Register & xa)948 void Assembler::umsubl(const Register& xd,
949                        const Register& wn,
950                        const Register& wm,
951                        const Register& xa) {
952   VIXL_ASSERT(xd.Is64Bits() && xa.Is64Bits());
953   VIXL_ASSERT(wn.Is32Bits() && wm.Is32Bits());
954   DataProcessing3Source(xd, wn, wm, xa, UMSUBL_x);
955 }
956 
957 
smsubl(const Register & xd,const Register & wn,const Register & wm,const Register & xa)958 void Assembler::smsubl(const Register& xd,
959                        const Register& wn,
960                        const Register& wm,
961                        const Register& xa) {
962   VIXL_ASSERT(xd.Is64Bits() && xa.Is64Bits());
963   VIXL_ASSERT(wn.Is32Bits() && wm.Is32Bits());
964   DataProcessing3Source(xd, wn, wm, xa, SMSUBL_x);
965 }
966 
967 
smull(const Register & xd,const Register & wn,const Register & wm)968 void Assembler::smull(const Register& xd,
969                       const Register& wn,
970                       const Register& wm) {
971   VIXL_ASSERT(xd.Is64Bits());
972   VIXL_ASSERT(wn.Is32Bits() && wm.Is32Bits());
973   DataProcessing3Source(xd, wn, wm, xzr, SMADDL_x);
974 }
975 
976 
sdiv(const Register & rd,const Register & rn,const Register & rm)977 void Assembler::sdiv(const Register& rd,
978                      const Register& rn,
979                      const Register& rm) {
980   VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
981   VIXL_ASSERT(rd.GetSizeInBits() == rm.GetSizeInBits());
982   Emit(SF(rd) | SDIV | Rm(rm) | Rn(rn) | Rd(rd));
983 }
984 
985 
smulh(const Register & xd,const Register & xn,const Register & xm)986 void Assembler::smulh(const Register& xd,
987                       const Register& xn,
988                       const Register& xm) {
989   VIXL_ASSERT(xd.Is64Bits() && xn.Is64Bits() && xm.Is64Bits());
990   DataProcessing3Source(xd, xn, xm, xzr, SMULH_x);
991 }
992 
993 
umulh(const Register & xd,const Register & xn,const Register & xm)994 void Assembler::umulh(const Register& xd,
995                       const Register& xn,
996                       const Register& xm) {
997   VIXL_ASSERT(xd.Is64Bits() && xn.Is64Bits() && xm.Is64Bits());
998   DataProcessing3Source(xd, xn, xm, xzr, UMULH_x);
999 }
1000 
1001 
udiv(const Register & rd,const Register & rn,const Register & rm)1002 void Assembler::udiv(const Register& rd,
1003                      const Register& rn,
1004                      const Register& rm) {
1005   VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
1006   VIXL_ASSERT(rd.GetSizeInBits() == rm.GetSizeInBits());
1007   Emit(SF(rd) | UDIV | Rm(rm) | Rn(rn) | Rd(rd));
1008 }
1009 
1010 
rbit(const Register & rd,const Register & rn)1011 void Assembler::rbit(const Register& rd, const Register& rn) {
1012   DataProcessing1Source(rd, rn, RBIT);
1013 }
1014 
1015 
rev16(const Register & rd,const Register & rn)1016 void Assembler::rev16(const Register& rd, const Register& rn) {
1017   DataProcessing1Source(rd, rn, REV16);
1018 }
1019 
1020 
rev32(const Register & xd,const Register & xn)1021 void Assembler::rev32(const Register& xd, const Register& xn) {
1022   VIXL_ASSERT(xd.Is64Bits());
1023   DataProcessing1Source(xd, xn, REV);
1024 }
1025 
1026 
rev(const Register & rd,const Register & rn)1027 void Assembler::rev(const Register& rd, const Register& rn) {
1028   DataProcessing1Source(rd, rn, rd.Is64Bits() ? REV_x : REV_w);
1029 }
1030 
1031 
clz(const Register & rd,const Register & rn)1032 void Assembler::clz(const Register& rd, const Register& rn) {
1033   DataProcessing1Source(rd, rn, CLZ);
1034 }
1035 
1036 
cls(const Register & rd,const Register & rn)1037 void Assembler::cls(const Register& rd, const Register& rn) {
1038   DataProcessing1Source(rd, rn, CLS);
1039 }
1040 
1041 #define PAUTH_VARIATIONS(V) \
1042   V(paci, PACI)             \
1043   V(pacd, PACD)             \
1044   V(auti, AUTI)             \
1045   V(autd, AUTD)
1046 
1047 #define VIXL_DEFINE_ASM_FUNC(PRE, OP)                              \
1048   void Assembler::PRE##a(const Register& xd, const Register& xn) { \
1049     VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));                      \
1050     VIXL_ASSERT(xd.Is64Bits() && xn.Is64Bits());                   \
1051     Emit(SF(xd) | OP##A | Rd(xd) | RnSP(xn));                      \
1052   }                                                                \
1053                                                                    \
1054   void Assembler::PRE##za(const Register& xd) {                    \
1055     VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));                      \
1056     VIXL_ASSERT(xd.Is64Bits());                                    \
1057     Emit(SF(xd) | OP##ZA | Rd(xd) | Rn(xzr));                      \
1058   }                                                                \
1059                                                                    \
1060   void Assembler::PRE##b(const Register& xd, const Register& xn) { \
1061     VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));                      \
1062     VIXL_ASSERT(xd.Is64Bits() && xn.Is64Bits());                   \
1063     Emit(SF(xd) | OP##B | Rd(xd) | RnSP(xn));                      \
1064   }                                                                \
1065                                                                    \
1066   void Assembler::PRE##zb(const Register& xd) {                    \
1067     VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));                      \
1068     VIXL_ASSERT(xd.Is64Bits());                                    \
1069     Emit(SF(xd) | OP##ZB | Rd(xd) | Rn(xzr));                      \
1070   }
1071 
PAUTH_VARIATIONS(VIXL_DEFINE_ASM_FUNC)1072 PAUTH_VARIATIONS(VIXL_DEFINE_ASM_FUNC)
1073 #undef VIXL_DEFINE_ASM_FUNC
1074 
1075 void Assembler::pacga(const Register& xd,
1076                       const Register& xn,
1077                       const Register& xm) {
1078   VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth, CPUFeatures::kPAuthGeneric));
1079   VIXL_ASSERT(xd.Is64Bits() && xn.Is64Bits() && xm.Is64Bits());
1080   Emit(SF(xd) | PACGA | Rd(xd) | Rn(xn) | RmSP(xm));
1081 }
1082 
xpaci(const Register & xd)1083 void Assembler::xpaci(const Register& xd) {
1084   VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
1085   VIXL_ASSERT(xd.Is64Bits());
1086   Emit(SF(xd) | XPACI | Rd(xd) | Rn(xzr));
1087 }
1088 
xpacd(const Register & xd)1089 void Assembler::xpacd(const Register& xd) {
1090   VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
1091   VIXL_ASSERT(xd.Is64Bits());
1092   Emit(SF(xd) | XPACD | Rd(xd) | Rn(xzr));
1093 }
1094 
1095 
ldp(const CPURegister & rt,const CPURegister & rt2,const MemOperand & src)1096 void Assembler::ldp(const CPURegister& rt,
1097                     const CPURegister& rt2,
1098                     const MemOperand& src) {
1099   LoadStorePair(rt, rt2, src, LoadPairOpFor(rt, rt2));
1100 }
1101 
1102 
stp(const CPURegister & rt,const CPURegister & rt2,const MemOperand & dst)1103 void Assembler::stp(const CPURegister& rt,
1104                     const CPURegister& rt2,
1105                     const MemOperand& dst) {
1106   LoadStorePair(rt, rt2, dst, StorePairOpFor(rt, rt2));
1107 }
1108 
1109 
ldpsw(const Register & xt,const Register & xt2,const MemOperand & src)1110 void Assembler::ldpsw(const Register& xt,
1111                       const Register& xt2,
1112                       const MemOperand& src) {
1113   VIXL_ASSERT(xt.Is64Bits() && xt2.Is64Bits());
1114   LoadStorePair(xt, xt2, src, LDPSW_x);
1115 }
1116 
1117 
LoadStorePair(const CPURegister & rt,const CPURegister & rt2,const MemOperand & addr,LoadStorePairOp op)1118 void Assembler::LoadStorePair(const CPURegister& rt,
1119                               const CPURegister& rt2,
1120                               const MemOperand& addr,
1121                               LoadStorePairOp op) {
1122   VIXL_ASSERT(CPUHas(rt, rt2));
1123 
1124   // 'rt' and 'rt2' can only be aliased for stores.
1125   VIXL_ASSERT(((op & LoadStorePairLBit) == 0) || !rt.Is(rt2));
1126   VIXL_ASSERT(AreSameSizeAndType(rt, rt2));
1127   VIXL_ASSERT(IsImmLSPair(addr.GetOffset(), CalcLSPairDataSize(op)));
1128 
1129   int offset = static_cast<int>(addr.GetOffset());
1130   Instr memop = op | Rt(rt) | Rt2(rt2) | RnSP(addr.GetBaseRegister()) |
1131                 ImmLSPair(offset, CalcLSPairDataSize(op));
1132 
1133   Instr addrmodeop;
1134   if (addr.IsImmediateOffset()) {
1135     addrmodeop = LoadStorePairOffsetFixed;
1136   } else {
1137     if (addr.IsImmediatePreIndex()) {
1138       addrmodeop = LoadStorePairPreIndexFixed;
1139     } else {
1140       VIXL_ASSERT(addr.IsImmediatePostIndex());
1141       addrmodeop = LoadStorePairPostIndexFixed;
1142     }
1143   }
1144 
1145   Instr emitop = addrmodeop | memop;
1146 
1147   // Only X registers may be specified for ldpsw.
1148   VIXL_ASSERT(((emitop & LoadStorePairMask) != LDPSW_x) || rt.IsX());
1149 
1150   Emit(emitop);
1151 }
1152 
1153 
ldnp(const CPURegister & rt,const CPURegister & rt2,const MemOperand & src)1154 void Assembler::ldnp(const CPURegister& rt,
1155                      const CPURegister& rt2,
1156                      const MemOperand& src) {
1157   LoadStorePairNonTemporal(rt, rt2, src, LoadPairNonTemporalOpFor(rt, rt2));
1158 }
1159 
1160 
stnp(const CPURegister & rt,const CPURegister & rt2,const MemOperand & dst)1161 void Assembler::stnp(const CPURegister& rt,
1162                      const CPURegister& rt2,
1163                      const MemOperand& dst) {
1164   LoadStorePairNonTemporal(rt, rt2, dst, StorePairNonTemporalOpFor(rt, rt2));
1165 }
1166 
1167 
LoadStorePairNonTemporal(const CPURegister & rt,const CPURegister & rt2,const MemOperand & addr,LoadStorePairNonTemporalOp op)1168 void Assembler::LoadStorePairNonTemporal(const CPURegister& rt,
1169                                          const CPURegister& rt2,
1170                                          const MemOperand& addr,
1171                                          LoadStorePairNonTemporalOp op) {
1172   VIXL_ASSERT(CPUHas(rt, rt2));
1173 
1174   VIXL_ASSERT(!rt.Is(rt2));
1175   VIXL_ASSERT(AreSameSizeAndType(rt, rt2));
1176   VIXL_ASSERT(addr.IsImmediateOffset());
1177 
1178   unsigned size =
1179       CalcLSPairDataSize(static_cast<LoadStorePairOp>(op & LoadStorePairMask));
1180   VIXL_ASSERT(IsImmLSPair(addr.GetOffset(), size));
1181   int offset = static_cast<int>(addr.GetOffset());
1182   Emit(op | Rt(rt) | Rt2(rt2) | RnSP(addr.GetBaseRegister()) |
1183        ImmLSPair(offset, size));
1184 }
1185 
1186 
1187 // Memory instructions.
ldrb(const Register & rt,const MemOperand & src,LoadStoreScalingOption option)1188 void Assembler::ldrb(const Register& rt,
1189                      const MemOperand& src,
1190                      LoadStoreScalingOption option) {
1191   VIXL_ASSERT(option != RequireUnscaledOffset);
1192   VIXL_ASSERT(option != PreferUnscaledOffset);
1193   LoadStore(rt, src, LDRB_w, option);
1194 }
1195 
1196 
strb(const Register & rt,const MemOperand & dst,LoadStoreScalingOption option)1197 void Assembler::strb(const Register& rt,
1198                      const MemOperand& dst,
1199                      LoadStoreScalingOption option) {
1200   VIXL_ASSERT(option != RequireUnscaledOffset);
1201   VIXL_ASSERT(option != PreferUnscaledOffset);
1202   LoadStore(rt, dst, STRB_w, option);
1203 }
1204 
1205 
ldrsb(const Register & rt,const MemOperand & src,LoadStoreScalingOption option)1206 void Assembler::ldrsb(const Register& rt,
1207                       const MemOperand& src,
1208                       LoadStoreScalingOption option) {
1209   VIXL_ASSERT(option != RequireUnscaledOffset);
1210   VIXL_ASSERT(option != PreferUnscaledOffset);
1211   LoadStore(rt, src, rt.Is64Bits() ? LDRSB_x : LDRSB_w, option);
1212 }
1213 
1214 
ldrh(const Register & rt,const MemOperand & src,LoadStoreScalingOption option)1215 void Assembler::ldrh(const Register& rt,
1216                      const MemOperand& src,
1217                      LoadStoreScalingOption option) {
1218   VIXL_ASSERT(option != RequireUnscaledOffset);
1219   VIXL_ASSERT(option != PreferUnscaledOffset);
1220   LoadStore(rt, src, LDRH_w, option);
1221 }
1222 
1223 
strh(const Register & rt,const MemOperand & dst,LoadStoreScalingOption option)1224 void Assembler::strh(const Register& rt,
1225                      const MemOperand& dst,
1226                      LoadStoreScalingOption option) {
1227   VIXL_ASSERT(option != RequireUnscaledOffset);
1228   VIXL_ASSERT(option != PreferUnscaledOffset);
1229   LoadStore(rt, dst, STRH_w, option);
1230 }
1231 
1232 
ldrsh(const Register & rt,const MemOperand & src,LoadStoreScalingOption option)1233 void Assembler::ldrsh(const Register& rt,
1234                       const MemOperand& src,
1235                       LoadStoreScalingOption option) {
1236   VIXL_ASSERT(option != RequireUnscaledOffset);
1237   VIXL_ASSERT(option != PreferUnscaledOffset);
1238   LoadStore(rt, src, rt.Is64Bits() ? LDRSH_x : LDRSH_w, option);
1239 }
1240 
1241 
ldr(const CPURegister & rt,const MemOperand & src,LoadStoreScalingOption option)1242 void Assembler::ldr(const CPURegister& rt,
1243                     const MemOperand& src,
1244                     LoadStoreScalingOption option) {
1245   VIXL_ASSERT(option != RequireUnscaledOffset);
1246   VIXL_ASSERT(option != PreferUnscaledOffset);
1247   LoadStore(rt, src, LoadOpFor(rt), option);
1248 }
1249 
1250 
str(const CPURegister & rt,const MemOperand & dst,LoadStoreScalingOption option)1251 void Assembler::str(const CPURegister& rt,
1252                     const MemOperand& dst,
1253                     LoadStoreScalingOption option) {
1254   VIXL_ASSERT(option != RequireUnscaledOffset);
1255   VIXL_ASSERT(option != PreferUnscaledOffset);
1256   LoadStore(rt, dst, StoreOpFor(rt), option);
1257 }
1258 
1259 
ldrsw(const Register & xt,const MemOperand & src,LoadStoreScalingOption option)1260 void Assembler::ldrsw(const Register& xt,
1261                       const MemOperand& src,
1262                       LoadStoreScalingOption option) {
1263   VIXL_ASSERT(xt.Is64Bits());
1264   VIXL_ASSERT(option != RequireUnscaledOffset);
1265   VIXL_ASSERT(option != PreferUnscaledOffset);
1266   LoadStore(xt, src, LDRSW_x, option);
1267 }
1268 
1269 
ldurb(const Register & rt,const MemOperand & src,LoadStoreScalingOption option)1270 void Assembler::ldurb(const Register& rt,
1271                       const MemOperand& src,
1272                       LoadStoreScalingOption option) {
1273   VIXL_ASSERT(option != RequireScaledOffset);
1274   VIXL_ASSERT(option != PreferScaledOffset);
1275   LoadStore(rt, src, LDRB_w, option);
1276 }
1277 
1278 
sturb(const Register & rt,const MemOperand & dst,LoadStoreScalingOption option)1279 void Assembler::sturb(const Register& rt,
1280                       const MemOperand& dst,
1281                       LoadStoreScalingOption option) {
1282   VIXL_ASSERT(option != RequireScaledOffset);
1283   VIXL_ASSERT(option != PreferScaledOffset);
1284   LoadStore(rt, dst, STRB_w, option);
1285 }
1286 
1287 
ldursb(const Register & rt,const MemOperand & src,LoadStoreScalingOption option)1288 void Assembler::ldursb(const Register& rt,
1289                        const MemOperand& src,
1290                        LoadStoreScalingOption option) {
1291   VIXL_ASSERT(option != RequireScaledOffset);
1292   VIXL_ASSERT(option != PreferScaledOffset);
1293   LoadStore(rt, src, rt.Is64Bits() ? LDRSB_x : LDRSB_w, option);
1294 }
1295 
1296 
ldurh(const Register & rt,const MemOperand & src,LoadStoreScalingOption option)1297 void Assembler::ldurh(const Register& rt,
1298                       const MemOperand& src,
1299                       LoadStoreScalingOption option) {
1300   VIXL_ASSERT(option != RequireScaledOffset);
1301   VIXL_ASSERT(option != PreferScaledOffset);
1302   LoadStore(rt, src, LDRH_w, option);
1303 }
1304 
1305 
sturh(const Register & rt,const MemOperand & dst,LoadStoreScalingOption option)1306 void Assembler::sturh(const Register& rt,
1307                       const MemOperand& dst,
1308                       LoadStoreScalingOption option) {
1309   VIXL_ASSERT(option != RequireScaledOffset);
1310   VIXL_ASSERT(option != PreferScaledOffset);
1311   LoadStore(rt, dst, STRH_w, option);
1312 }
1313 
1314 
ldursh(const Register & rt,const MemOperand & src,LoadStoreScalingOption option)1315 void Assembler::ldursh(const Register& rt,
1316                        const MemOperand& src,
1317                        LoadStoreScalingOption option) {
1318   VIXL_ASSERT(option != RequireScaledOffset);
1319   VIXL_ASSERT(option != PreferScaledOffset);
1320   LoadStore(rt, src, rt.Is64Bits() ? LDRSH_x : LDRSH_w, option);
1321 }
1322 
1323 
ldur(const CPURegister & rt,const MemOperand & src,LoadStoreScalingOption option)1324 void Assembler::ldur(const CPURegister& rt,
1325                      const MemOperand& src,
1326                      LoadStoreScalingOption option) {
1327   VIXL_ASSERT(option != RequireScaledOffset);
1328   VIXL_ASSERT(option != PreferScaledOffset);
1329   LoadStore(rt, src, LoadOpFor(rt), option);
1330 }
1331 
1332 
stur(const CPURegister & rt,const MemOperand & dst,LoadStoreScalingOption option)1333 void Assembler::stur(const CPURegister& rt,
1334                      const MemOperand& dst,
1335                      LoadStoreScalingOption option) {
1336   VIXL_ASSERT(option != RequireScaledOffset);
1337   VIXL_ASSERT(option != PreferScaledOffset);
1338   LoadStore(rt, dst, StoreOpFor(rt), option);
1339 }
1340 
1341 
ldursw(const Register & xt,const MemOperand & src,LoadStoreScalingOption option)1342 void Assembler::ldursw(const Register& xt,
1343                        const MemOperand& src,
1344                        LoadStoreScalingOption option) {
1345   VIXL_ASSERT(xt.Is64Bits());
1346   VIXL_ASSERT(option != RequireScaledOffset);
1347   VIXL_ASSERT(option != PreferScaledOffset);
1348   LoadStore(xt, src, LDRSW_x, option);
1349 }
1350 
1351 
ldraa(const Register & xt,const MemOperand & src)1352 void Assembler::ldraa(const Register& xt, const MemOperand& src) {
1353   VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
1354   LoadStorePAC(xt, src, LDRAA);
1355 }
1356 
1357 
ldrab(const Register & xt,const MemOperand & src)1358 void Assembler::ldrab(const Register& xt, const MemOperand& src) {
1359   VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
1360   LoadStorePAC(xt, src, LDRAB);
1361 }
1362 
1363 
ldrsw(const Register & xt,RawLiteral * literal)1364 void Assembler::ldrsw(const Register& xt, RawLiteral* literal) {
1365   VIXL_ASSERT(xt.Is64Bits());
1366   VIXL_ASSERT(literal->GetSize() == kWRegSizeInBytes);
1367   ldrsw(xt, static_cast<int>(LinkAndGetWordOffsetTo(literal)));
1368 }
1369 
1370 
ldr(const CPURegister & rt,RawLiteral * literal)1371 void Assembler::ldr(const CPURegister& rt, RawLiteral* literal) {
1372   VIXL_ASSERT(CPUHas(rt));
1373   VIXL_ASSERT(literal->GetSize() == static_cast<size_t>(rt.GetSizeInBytes()));
1374   ldr(rt, static_cast<int>(LinkAndGetWordOffsetTo(literal)));
1375 }
1376 
1377 
ldrsw(const Register & rt,int64_t imm19)1378 void Assembler::ldrsw(const Register& rt, int64_t imm19) {
1379   Emit(LDRSW_x_lit | ImmLLiteral(imm19) | Rt(rt));
1380 }
1381 
1382 
ldr(const CPURegister & rt,int64_t imm19)1383 void Assembler::ldr(const CPURegister& rt, int64_t imm19) {
1384   VIXL_ASSERT(CPUHas(rt));
1385   LoadLiteralOp op = LoadLiteralOpFor(rt);
1386   Emit(op | ImmLLiteral(imm19) | Rt(rt));
1387 }
1388 
1389 
prfm(int op,int64_t imm19)1390 void Assembler::prfm(int op, int64_t imm19) {
1391   Emit(PRFM_lit | ImmPrefetchOperation(op) | ImmLLiteral(imm19));
1392 }
1393 
prfm(PrefetchOperation op,int64_t imm19)1394 void Assembler::prfm(PrefetchOperation op, int64_t imm19) {
1395   // Passing unnamed values in 'op' is undefined behaviour in C++.
1396   VIXL_ASSERT(IsNamedPrefetchOperation(op));
1397   prfm(static_cast<int>(op), imm19);
1398 }
1399 
1400 
1401 // Exclusive-access instructions.
stxrb(const Register & rs,const Register & rt,const MemOperand & dst)1402 void Assembler::stxrb(const Register& rs,
1403                       const Register& rt,
1404                       const MemOperand& dst) {
1405   VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1406   Emit(STXRB_w | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.GetBaseRegister()));
1407 }
1408 
1409 
stxrh(const Register & rs,const Register & rt,const MemOperand & dst)1410 void Assembler::stxrh(const Register& rs,
1411                       const Register& rt,
1412                       const MemOperand& dst) {
1413   VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1414   Emit(STXRH_w | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.GetBaseRegister()));
1415 }
1416 
1417 
stxr(const Register & rs,const Register & rt,const MemOperand & dst)1418 void Assembler::stxr(const Register& rs,
1419                      const Register& rt,
1420                      const MemOperand& dst) {
1421   VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1422   LoadStoreExclusive op = rt.Is64Bits() ? STXR_x : STXR_w;
1423   Emit(op | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.GetBaseRegister()));
1424 }
1425 
1426 
ldxrb(const Register & rt,const MemOperand & src)1427 void Assembler::ldxrb(const Register& rt, const MemOperand& src) {
1428   VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1429   Emit(LDXRB_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister()));
1430 }
1431 
1432 
ldxrh(const Register & rt,const MemOperand & src)1433 void Assembler::ldxrh(const Register& rt, const MemOperand& src) {
1434   VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1435   Emit(LDXRH_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister()));
1436 }
1437 
1438 
ldxr(const Register & rt,const MemOperand & src)1439 void Assembler::ldxr(const Register& rt, const MemOperand& src) {
1440   VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1441   LoadStoreExclusive op = rt.Is64Bits() ? LDXR_x : LDXR_w;
1442   Emit(op | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister()));
1443 }
1444 
1445 
stxp(const Register & rs,const Register & rt,const Register & rt2,const MemOperand & dst)1446 void Assembler::stxp(const Register& rs,
1447                      const Register& rt,
1448                      const Register& rt2,
1449                      const MemOperand& dst) {
1450   VIXL_ASSERT(rt.GetSizeInBits() == rt2.GetSizeInBits());
1451   VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1452   LoadStoreExclusive op = rt.Is64Bits() ? STXP_x : STXP_w;
1453   Emit(op | Rs(rs) | Rt(rt) | Rt2(rt2) | RnSP(dst.GetBaseRegister()));
1454 }
1455 
1456 
ldxp(const Register & rt,const Register & rt2,const MemOperand & src)1457 void Assembler::ldxp(const Register& rt,
1458                      const Register& rt2,
1459                      const MemOperand& src) {
1460   VIXL_ASSERT(rt.GetSizeInBits() == rt2.GetSizeInBits());
1461   VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1462   LoadStoreExclusive op = rt.Is64Bits() ? LDXP_x : LDXP_w;
1463   Emit(op | Rs_mask | Rt(rt) | Rt2(rt2) | RnSP(src.GetBaseRegister()));
1464 }
1465 
1466 
stlxrb(const Register & rs,const Register & rt,const MemOperand & dst)1467 void Assembler::stlxrb(const Register& rs,
1468                        const Register& rt,
1469                        const MemOperand& dst) {
1470   VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1471   Emit(STLXRB_w | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.GetBaseRegister()));
1472 }
1473 
1474 
stlxrh(const Register & rs,const Register & rt,const MemOperand & dst)1475 void Assembler::stlxrh(const Register& rs,
1476                        const Register& rt,
1477                        const MemOperand& dst) {
1478   VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1479   Emit(STLXRH_w | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.GetBaseRegister()));
1480 }
1481 
1482 
stlxr(const Register & rs,const Register & rt,const MemOperand & dst)1483 void Assembler::stlxr(const Register& rs,
1484                       const Register& rt,
1485                       const MemOperand& dst) {
1486   VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1487   LoadStoreExclusive op = rt.Is64Bits() ? STLXR_x : STLXR_w;
1488   Emit(op | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.GetBaseRegister()));
1489 }
1490 
1491 
ldaxrb(const Register & rt,const MemOperand & src)1492 void Assembler::ldaxrb(const Register& rt, const MemOperand& src) {
1493   VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1494   Emit(LDAXRB_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister()));
1495 }
1496 
1497 
ldaxrh(const Register & rt,const MemOperand & src)1498 void Assembler::ldaxrh(const Register& rt, const MemOperand& src) {
1499   VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1500   Emit(LDAXRH_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister()));
1501 }
1502 
1503 
ldaxr(const Register & rt,const MemOperand & src)1504 void Assembler::ldaxr(const Register& rt, const MemOperand& src) {
1505   VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1506   LoadStoreExclusive op = rt.Is64Bits() ? LDAXR_x : LDAXR_w;
1507   Emit(op | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister()));
1508 }
1509 
1510 
stlxp(const Register & rs,const Register & rt,const Register & rt2,const MemOperand & dst)1511 void Assembler::stlxp(const Register& rs,
1512                       const Register& rt,
1513                       const Register& rt2,
1514                       const MemOperand& dst) {
1515   VIXL_ASSERT(rt.GetSizeInBits() == rt2.GetSizeInBits());
1516   VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1517   LoadStoreExclusive op = rt.Is64Bits() ? STLXP_x : STLXP_w;
1518   Emit(op | Rs(rs) | Rt(rt) | Rt2(rt2) | RnSP(dst.GetBaseRegister()));
1519 }
1520 
1521 
ldaxp(const Register & rt,const Register & rt2,const MemOperand & src)1522 void Assembler::ldaxp(const Register& rt,
1523                       const Register& rt2,
1524                       const MemOperand& src) {
1525   VIXL_ASSERT(rt.GetSizeInBits() == rt2.GetSizeInBits());
1526   VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1527   LoadStoreExclusive op = rt.Is64Bits() ? LDAXP_x : LDAXP_w;
1528   Emit(op | Rs_mask | Rt(rt) | Rt2(rt2) | RnSP(src.GetBaseRegister()));
1529 }
1530 
1531 
stlrb(const Register & rt,const MemOperand & dst)1532 void Assembler::stlrb(const Register& rt, const MemOperand& dst) {
1533   VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1534   Emit(STLRB_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(dst.GetBaseRegister()));
1535 }
1536 
stlurb(const Register & rt,const MemOperand & dst)1537 void Assembler::stlurb(const Register& rt, const MemOperand& dst) {
1538   VIXL_ASSERT(CPUHas(CPUFeatures::kRCpc, CPUFeatures::kRCpcImm));
1539   VIXL_ASSERT(dst.IsImmediateOffset() && IsImmLSUnscaled(dst.GetOffset()));
1540 
1541   Instr base = RnSP(dst.GetBaseRegister());
1542   int64_t offset = dst.GetOffset();
1543   Emit(STLURB | Rt(rt) | base | ImmLS(static_cast<int>(offset)));
1544 }
1545 
1546 
stlrh(const Register & rt,const MemOperand & dst)1547 void Assembler::stlrh(const Register& rt, const MemOperand& dst) {
1548   VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1549   Emit(STLRH_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(dst.GetBaseRegister()));
1550 }
1551 
stlurh(const Register & rt,const MemOperand & dst)1552 void Assembler::stlurh(const Register& rt, const MemOperand& dst) {
1553   VIXL_ASSERT(CPUHas(CPUFeatures::kRCpc, CPUFeatures::kRCpcImm));
1554   VIXL_ASSERT(dst.IsImmediateOffset() && IsImmLSUnscaled(dst.GetOffset()));
1555 
1556   Instr base = RnSP(dst.GetBaseRegister());
1557   int64_t offset = dst.GetOffset();
1558   Emit(STLURH | Rt(rt) | base | ImmLS(static_cast<int>(offset)));
1559 }
1560 
1561 
stlr(const Register & rt,const MemOperand & dst)1562 void Assembler::stlr(const Register& rt, const MemOperand& dst) {
1563   VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1564   LoadStoreExclusive op = rt.Is64Bits() ? STLR_x : STLR_w;
1565   Emit(op | Rs_mask | Rt(rt) | Rt2_mask | RnSP(dst.GetBaseRegister()));
1566 }
1567 
stlur(const Register & rt,const MemOperand & dst)1568 void Assembler::stlur(const Register& rt, const MemOperand& dst) {
1569   VIXL_ASSERT(CPUHas(CPUFeatures::kRCpc, CPUFeatures::kRCpcImm));
1570   VIXL_ASSERT(dst.IsImmediateOffset() && IsImmLSUnscaled(dst.GetOffset()));
1571 
1572   Instr base = RnSP(dst.GetBaseRegister());
1573   int64_t offset = dst.GetOffset();
1574   Instr op = rt.Is64Bits() ? STLUR_x : STLUR_w;
1575   Emit(op | Rt(rt) | base | ImmLS(static_cast<int>(offset)));
1576 }
1577 
1578 
ldarb(const Register & rt,const MemOperand & src)1579 void Assembler::ldarb(const Register& rt, const MemOperand& src) {
1580   VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1581   Emit(LDARB_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister()));
1582 }
1583 
1584 
ldarh(const Register & rt,const MemOperand & src)1585 void Assembler::ldarh(const Register& rt, const MemOperand& src) {
1586   VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1587   Emit(LDARH_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister()));
1588 }
1589 
1590 
ldar(const Register & rt,const MemOperand & src)1591 void Assembler::ldar(const Register& rt, const MemOperand& src) {
1592   VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1593   LoadStoreExclusive op = rt.Is64Bits() ? LDAR_x : LDAR_w;
1594   Emit(op | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister()));
1595 }
1596 
1597 
stllrb(const Register & rt,const MemOperand & dst)1598 void Assembler::stllrb(const Register& rt, const MemOperand& dst) {
1599   VIXL_ASSERT(CPUHas(CPUFeatures::kLORegions));
1600   VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1601   Emit(STLLRB | Rs_mask | Rt(rt) | Rt2_mask | RnSP(dst.GetBaseRegister()));
1602 }
1603 
1604 
stllrh(const Register & rt,const MemOperand & dst)1605 void Assembler::stllrh(const Register& rt, const MemOperand& dst) {
1606   VIXL_ASSERT(CPUHas(CPUFeatures::kLORegions));
1607   VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1608   Emit(STLLRH | Rs_mask | Rt(rt) | Rt2_mask | RnSP(dst.GetBaseRegister()));
1609 }
1610 
1611 
stllr(const Register & rt,const MemOperand & dst)1612 void Assembler::stllr(const Register& rt, const MemOperand& dst) {
1613   VIXL_ASSERT(CPUHas(CPUFeatures::kLORegions));
1614   VIXL_ASSERT(dst.IsImmediateOffset() && (dst.GetOffset() == 0));
1615   LoadStoreExclusive op = rt.Is64Bits() ? STLLR_x : STLLR_w;
1616   Emit(op | Rs_mask | Rt(rt) | Rt2_mask | RnSP(dst.GetBaseRegister()));
1617 }
1618 
1619 
ldlarb(const Register & rt,const MemOperand & src)1620 void Assembler::ldlarb(const Register& rt, const MemOperand& src) {
1621   VIXL_ASSERT(CPUHas(CPUFeatures::kLORegions));
1622   VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1623   Emit(LDLARB | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister()));
1624 }
1625 
1626 
ldlarh(const Register & rt,const MemOperand & src)1627 void Assembler::ldlarh(const Register& rt, const MemOperand& src) {
1628   VIXL_ASSERT(CPUHas(CPUFeatures::kLORegions));
1629   VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1630   Emit(LDLARH | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister()));
1631 }
1632 
1633 
ldlar(const Register & rt,const MemOperand & src)1634 void Assembler::ldlar(const Register& rt, const MemOperand& src) {
1635   VIXL_ASSERT(CPUHas(CPUFeatures::kLORegions));
1636   VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1637   LoadStoreExclusive op = rt.Is64Bits() ? LDLAR_x : LDLAR_w;
1638   Emit(op | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister()));
1639 }
1640 
1641 
1642 // clang-format off
1643 #define COMPARE_AND_SWAP_W_X_LIST(V) \
1644   V(cas,   CAS)                      \
1645   V(casa,  CASA)                     \
1646   V(casl,  CASL)                     \
1647   V(casal, CASAL)
1648 // clang-format on
1649 
1650 #define VIXL_DEFINE_ASM_FUNC(FN, OP)                                     \
1651   void Assembler::FN(const Register& rs,                                 \
1652                      const Register& rt,                                 \
1653                      const MemOperand& src) {                            \
1654     VIXL_ASSERT(CPUHas(CPUFeatures::kAtomics));                          \
1655     VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));      \
1656     VIXL_ASSERT(AreSameFormat(rs, rt));                                  \
1657     LoadStoreExclusive op = rt.Is64Bits() ? OP##_x : OP##_w;             \
1658     Emit(op | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister())); \
1659   }
1660 COMPARE_AND_SWAP_W_X_LIST(VIXL_DEFINE_ASM_FUNC)
1661 #undef VIXL_DEFINE_ASM_FUNC
1662 
1663 // clang-format off
1664 #define COMPARE_AND_SWAP_W_LIST(V) \
1665   V(casb,   CASB)                  \
1666   V(casab,  CASAB)                 \
1667   V(caslb,  CASLB)                 \
1668   V(casalb, CASALB)                \
1669   V(cash,   CASH)                  \
1670   V(casah,  CASAH)                 \
1671   V(caslh,  CASLH)                 \
1672   V(casalh, CASALH)
1673 // clang-format on
1674 
1675 #define VIXL_DEFINE_ASM_FUNC(FN, OP)                                     \
1676   void Assembler::FN(const Register& rs,                                 \
1677                      const Register& rt,                                 \
1678                      const MemOperand& src) {                            \
1679     VIXL_ASSERT(CPUHas(CPUFeatures::kAtomics));                          \
1680     VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));      \
1681     Emit(OP | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister())); \
1682   }
COMPARE_AND_SWAP_W_LIST(VIXL_DEFINE_ASM_FUNC)1683 COMPARE_AND_SWAP_W_LIST(VIXL_DEFINE_ASM_FUNC)
1684 #undef VIXL_DEFINE_ASM_FUNC
1685 
1686 
1687 // clang-format off
1688 #define COMPARE_AND_SWAP_PAIR_LIST(V) \
1689   V(casp,   CASP)                     \
1690   V(caspa,  CASPA)                    \
1691   V(caspl,  CASPL)                    \
1692   V(caspal, CASPAL)
1693 // clang-format on
1694 
1695 #define VIXL_DEFINE_ASM_FUNC(FN, OP)                                     \
1696   void Assembler::FN(const Register& rs,                                 \
1697                      const Register& rs1,                                \
1698                      const Register& rt,                                 \
1699                      const Register& rt1,                                \
1700                      const MemOperand& src) {                            \
1701     VIXL_ASSERT(CPUHas(CPUFeatures::kAtomics));                          \
1702     USE(rs1, rt1);                                                       \
1703     VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));      \
1704     VIXL_ASSERT(AreEven(rs, rt));                                        \
1705     VIXL_ASSERT(AreConsecutive(rs, rs1));                                \
1706     VIXL_ASSERT(AreConsecutive(rt, rt1));                                \
1707     VIXL_ASSERT(AreSameFormat(rs, rs1, rt, rt1));                        \
1708     LoadStoreExclusive op = rt.Is64Bits() ? OP##_x : OP##_w;             \
1709     Emit(op | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(src.GetBaseRegister())); \
1710   }
1711 COMPARE_AND_SWAP_PAIR_LIST(VIXL_DEFINE_ASM_FUNC)
1712 #undef VIXL_DEFINE_ASM_FUNC
1713 
1714 // These macros generate all the variations of the atomic memory operations,
1715 // e.g. ldadd, ldadda, ldaddb, staddl, etc.
1716 // For a full list of the methods with comments, see the assembler header file.
1717 
1718 // clang-format off
1719 #define ATOMIC_MEMORY_SIMPLE_OPERATION_LIST(V, DEF) \
1720   V(DEF, add,  LDADD)                               \
1721   V(DEF, clr,  LDCLR)                               \
1722   V(DEF, eor,  LDEOR)                               \
1723   V(DEF, set,  LDSET)                               \
1724   V(DEF, smax, LDSMAX)                              \
1725   V(DEF, smin, LDSMIN)                              \
1726   V(DEF, umax, LDUMAX)                              \
1727   V(DEF, umin, LDUMIN)
1728 
1729 #define ATOMIC_MEMORY_STORE_MODES(V, NAME, OP) \
1730   V(NAME,     OP##_x,   OP##_w)                \
1731   V(NAME##l,  OP##L_x,  OP##L_w)               \
1732   V(NAME##b,  OP##B,    OP##B)                 \
1733   V(NAME##lb, OP##LB,   OP##LB)                \
1734   V(NAME##h,  OP##H,    OP##H)                 \
1735   V(NAME##lh, OP##LH,   OP##LH)
1736 
1737 #define ATOMIC_MEMORY_LOAD_MODES(V, NAME, OP) \
1738   ATOMIC_MEMORY_STORE_MODES(V, NAME, OP)      \
1739   V(NAME##a,   OP##A_x,  OP##A_w)             \
1740   V(NAME##al,  OP##AL_x, OP##AL_w)            \
1741   V(NAME##ab,  OP##AB,   OP##AB)              \
1742   V(NAME##alb, OP##ALB,  OP##ALB)             \
1743   V(NAME##ah,  OP##AH,   OP##AH)              \
1744   V(NAME##alh, OP##ALH,  OP##ALH)
1745 // clang-format on
1746 
1747 #define DEFINE_ASM_LOAD_FUNC(FN, OP_X, OP_W)                        \
1748   void Assembler::ld##FN(const Register& rs,                        \
1749                          const Register& rt,                        \
1750                          const MemOperand& src) {                   \
1751     VIXL_ASSERT(CPUHas(CPUFeatures::kAtomics));                     \
1752     VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0)); \
1753     AtomicMemoryOp op = rt.Is64Bits() ? OP_X : OP_W;                \
1754     Emit(op | Rs(rs) | Rt(rt) | RnSP(src.GetBaseRegister()));       \
1755   }
1756 #define DEFINE_ASM_STORE_FUNC(FN, OP_X, OP_W)                         \
1757   void Assembler::st##FN(const Register& rs, const MemOperand& src) { \
1758     VIXL_ASSERT(CPUHas(CPUFeatures::kAtomics));                       \
1759     ld##FN(rs, AppropriateZeroRegFor(rs), src);                       \
1760   }
1761 
1762 ATOMIC_MEMORY_SIMPLE_OPERATION_LIST(ATOMIC_MEMORY_LOAD_MODES,
1763                                     DEFINE_ASM_LOAD_FUNC)
1764 ATOMIC_MEMORY_SIMPLE_OPERATION_LIST(ATOMIC_MEMORY_STORE_MODES,
1765                                     DEFINE_ASM_STORE_FUNC)
1766 
1767 #define DEFINE_ASM_SWP_FUNC(FN, OP_X, OP_W)                         \
1768   void Assembler::FN(const Register& rs,                            \
1769                      const Register& rt,                            \
1770                      const MemOperand& src) {                       \
1771     VIXL_ASSERT(CPUHas(CPUFeatures::kAtomics));                     \
1772     VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0)); \
1773     AtomicMemoryOp op = rt.Is64Bits() ? OP_X : OP_W;                \
1774     Emit(op | Rs(rs) | Rt(rt) | RnSP(src.GetBaseRegister()));       \
1775   }
1776 
1777 ATOMIC_MEMORY_LOAD_MODES(DEFINE_ASM_SWP_FUNC, swp, SWP)
1778 
1779 #undef DEFINE_ASM_LOAD_FUNC
1780 #undef DEFINE_ASM_STORE_FUNC
1781 #undef DEFINE_ASM_SWP_FUNC
1782 
1783 
1784 void Assembler::ldaprb(const Register& rt, const MemOperand& src) {
1785   VIXL_ASSERT(CPUHas(CPUFeatures::kRCpc));
1786   VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1787   AtomicMemoryOp op = LDAPRB;
1788   Emit(op | Rs(xzr) | Rt(rt) | RnSP(src.GetBaseRegister()));
1789 }
1790 
ldapurb(const Register & rt,const MemOperand & src)1791 void Assembler::ldapurb(const Register& rt, const MemOperand& src) {
1792   VIXL_ASSERT(CPUHas(CPUFeatures::kRCpc, CPUFeatures::kRCpcImm));
1793   VIXL_ASSERT(src.IsImmediateOffset() && IsImmLSUnscaled(src.GetOffset()));
1794 
1795   Instr base = RnSP(src.GetBaseRegister());
1796   int64_t offset = src.GetOffset();
1797   Emit(LDAPURB | Rt(rt) | base | ImmLS(static_cast<int>(offset)));
1798 }
1799 
ldapursb(const Register & rt,const MemOperand & src)1800 void Assembler::ldapursb(const Register& rt, const MemOperand& src) {
1801   VIXL_ASSERT(CPUHas(CPUFeatures::kRCpc, CPUFeatures::kRCpcImm));
1802   VIXL_ASSERT(src.IsImmediateOffset() && IsImmLSUnscaled(src.GetOffset()));
1803 
1804   Instr base = RnSP(src.GetBaseRegister());
1805   int64_t offset = src.GetOffset();
1806   Instr op = rt.Is64Bits() ? LDAPURSB_x : LDAPURSB_w;
1807   Emit(op | Rt(rt) | base | ImmLS(static_cast<int>(offset)));
1808 }
1809 
ldaprh(const Register & rt,const MemOperand & src)1810 void Assembler::ldaprh(const Register& rt, const MemOperand& src) {
1811   VIXL_ASSERT(CPUHas(CPUFeatures::kRCpc));
1812   VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1813   AtomicMemoryOp op = LDAPRH;
1814   Emit(op | Rs(xzr) | Rt(rt) | RnSP(src.GetBaseRegister()));
1815 }
1816 
ldapurh(const Register & rt,const MemOperand & src)1817 void Assembler::ldapurh(const Register& rt, const MemOperand& src) {
1818   VIXL_ASSERT(CPUHas(CPUFeatures::kRCpc, CPUFeatures::kRCpcImm));
1819   VIXL_ASSERT(src.IsImmediateOffset() && IsImmLSUnscaled(src.GetOffset()));
1820 
1821   Instr base = RnSP(src.GetBaseRegister());
1822   int64_t offset = src.GetOffset();
1823   Emit(LDAPURH | Rt(rt) | base | ImmLS(static_cast<int>(offset)));
1824 }
1825 
ldapursh(const Register & rt,const MemOperand & src)1826 void Assembler::ldapursh(const Register& rt, const MemOperand& src) {
1827   VIXL_ASSERT(CPUHas(CPUFeatures::kRCpc, CPUFeatures::kRCpcImm));
1828   VIXL_ASSERT(src.IsImmediateOffset() && IsImmLSUnscaled(src.GetOffset()));
1829 
1830   Instr base = RnSP(src.GetBaseRegister());
1831   int64_t offset = src.GetOffset();
1832   LoadStoreRCpcUnscaledOffsetOp op = rt.Is64Bits() ? LDAPURSH_x : LDAPURSH_w;
1833   Emit(op | Rt(rt) | base | ImmLS(static_cast<int>(offset)));
1834 }
1835 
ldapr(const Register & rt,const MemOperand & src)1836 void Assembler::ldapr(const Register& rt, const MemOperand& src) {
1837   VIXL_ASSERT(CPUHas(CPUFeatures::kRCpc));
1838   VIXL_ASSERT(src.IsImmediateOffset() && (src.GetOffset() == 0));
1839   AtomicMemoryOp op = rt.Is64Bits() ? LDAPR_x : LDAPR_w;
1840   Emit(op | Rs(xzr) | Rt(rt) | RnSP(src.GetBaseRegister()));
1841 }
1842 
ldapur(const Register & rt,const MemOperand & src)1843 void Assembler::ldapur(const Register& rt, const MemOperand& src) {
1844   VIXL_ASSERT(CPUHas(CPUFeatures::kRCpc, CPUFeatures::kRCpcImm));
1845   VIXL_ASSERT(src.IsImmediateOffset() && IsImmLSUnscaled(src.GetOffset()));
1846 
1847   Instr base = RnSP(src.GetBaseRegister());
1848   int64_t offset = src.GetOffset();
1849   LoadStoreRCpcUnscaledOffsetOp op = rt.Is64Bits() ? LDAPUR_x : LDAPUR_w;
1850   Emit(op | Rt(rt) | base | ImmLS(static_cast<int>(offset)));
1851 }
1852 
ldapursw(const Register & rt,const MemOperand & src)1853 void Assembler::ldapursw(const Register& rt, const MemOperand& src) {
1854   VIXL_ASSERT(CPUHas(CPUFeatures::kRCpc, CPUFeatures::kRCpcImm));
1855   VIXL_ASSERT(rt.Is64Bits());
1856   VIXL_ASSERT(src.IsImmediateOffset() && IsImmLSUnscaled(src.GetOffset()));
1857 
1858   Instr base = RnSP(src.GetBaseRegister());
1859   int64_t offset = src.GetOffset();
1860   Emit(LDAPURSW | Rt(rt) | base | ImmLS(static_cast<int>(offset)));
1861 }
1862 
prfm(int op,const MemOperand & address,LoadStoreScalingOption option)1863 void Assembler::prfm(int op,
1864                      const MemOperand& address,
1865                      LoadStoreScalingOption option) {
1866   VIXL_ASSERT(option != RequireUnscaledOffset);
1867   VIXL_ASSERT(option != PreferUnscaledOffset);
1868   Prefetch(op, address, option);
1869 }
1870 
prfm(PrefetchOperation op,const MemOperand & address,LoadStoreScalingOption option)1871 void Assembler::prfm(PrefetchOperation op,
1872                      const MemOperand& address,
1873                      LoadStoreScalingOption option) {
1874   // Passing unnamed values in 'op' is undefined behaviour in C++.
1875   VIXL_ASSERT(IsNamedPrefetchOperation(op));
1876   prfm(static_cast<int>(op), address, option);
1877 }
1878 
1879 
prfum(int op,const MemOperand & address,LoadStoreScalingOption option)1880 void Assembler::prfum(int op,
1881                       const MemOperand& address,
1882                       LoadStoreScalingOption option) {
1883   VIXL_ASSERT(option != RequireScaledOffset);
1884   VIXL_ASSERT(option != PreferScaledOffset);
1885   Prefetch(op, address, option);
1886 }
1887 
prfum(PrefetchOperation op,const MemOperand & address,LoadStoreScalingOption option)1888 void Assembler::prfum(PrefetchOperation op,
1889                       const MemOperand& address,
1890                       LoadStoreScalingOption option) {
1891   // Passing unnamed values in 'op' is undefined behaviour in C++.
1892   VIXL_ASSERT(IsNamedPrefetchOperation(op));
1893   prfum(static_cast<int>(op), address, option);
1894 }
1895 
1896 
prfm(int op,RawLiteral * literal)1897 void Assembler::prfm(int op, RawLiteral* literal) {
1898   prfm(op, static_cast<int>(LinkAndGetWordOffsetTo(literal)));
1899 }
1900 
prfm(PrefetchOperation op,RawLiteral * literal)1901 void Assembler::prfm(PrefetchOperation op, RawLiteral* literal) {
1902   // Passing unnamed values in 'op' is undefined behaviour in C++.
1903   VIXL_ASSERT(IsNamedPrefetchOperation(op));
1904   prfm(static_cast<int>(op), literal);
1905 }
1906 
1907 
sys(int op1,int crn,int crm,int op2,const Register & xt)1908 void Assembler::sys(int op1, int crn, int crm, int op2, const Register& xt) {
1909   VIXL_ASSERT(xt.Is64Bits());
1910   Emit(SYS | ImmSysOp1(op1) | CRn(crn) | CRm(crm) | ImmSysOp2(op2) | Rt(xt));
1911 }
1912 
1913 
sys(int op,const Register & xt)1914 void Assembler::sys(int op, const Register& xt) {
1915   VIXL_ASSERT(xt.Is64Bits());
1916   Emit(SYS | SysOp(op) | Rt(xt));
1917 }
1918 
1919 
dc(DataCacheOp op,const Register & rt)1920 void Assembler::dc(DataCacheOp op, const Register& rt) {
1921   if (op == CVAP) VIXL_ASSERT(CPUHas(CPUFeatures::kDCPoP));
1922   if (op == CVADP) VIXL_ASSERT(CPUHas(CPUFeatures::kDCCVADP));
1923   sys(op, rt);
1924 }
1925 
1926 
ic(InstructionCacheOp op,const Register & rt)1927 void Assembler::ic(InstructionCacheOp op, const Register& rt) {
1928   VIXL_ASSERT(op == IVAU);
1929   sys(op, rt);
1930 }
1931 
1932 
hint(SystemHint code)1933 void Assembler::hint(SystemHint code) { hint(static_cast<int>(code)); }
1934 
1935 
hint(int imm7)1936 void Assembler::hint(int imm7) {
1937   VIXL_ASSERT(IsUint7(imm7));
1938   Emit(HINT | ImmHint(imm7) | Rt(xzr));
1939 }
1940 
1941 
1942 // NEON structure loads and stores.
LoadStoreStructAddrModeField(const MemOperand & addr)1943 Instr Assembler::LoadStoreStructAddrModeField(const MemOperand& addr) {
1944   Instr addr_field = RnSP(addr.GetBaseRegister());
1945 
1946   if (addr.IsPostIndex()) {
1947     VIXL_STATIC_ASSERT(NEONLoadStoreMultiStructPostIndex ==
1948                        static_cast<NEONLoadStoreMultiStructPostIndexOp>(
1949                            NEONLoadStoreSingleStructPostIndex));
1950 
1951     addr_field |= NEONLoadStoreMultiStructPostIndex;
1952     if (addr.GetOffset() == 0) {
1953       addr_field |= RmNot31(addr.GetRegisterOffset());
1954     } else {
1955       // The immediate post index addressing mode is indicated by rm = 31.
1956       // The immediate is implied by the number of vector registers used.
1957       addr_field |= (0x1f << Rm_offset);
1958     }
1959   } else {
1960     VIXL_ASSERT(addr.IsImmediateOffset() && (addr.GetOffset() == 0));
1961   }
1962   return addr_field;
1963 }
1964 
LoadStoreStructVerify(const VRegister & vt,const MemOperand & addr,Instr op)1965 void Assembler::LoadStoreStructVerify(const VRegister& vt,
1966                                       const MemOperand& addr,
1967                                       Instr op) {
1968 #ifdef VIXL_DEBUG
1969   // Assert that addressing mode is either offset (with immediate 0), post
1970   // index by immediate of the size of the register list, or post index by a
1971   // value in a core register.
1972   VIXL_ASSERT(vt.HasSize() && vt.HasLaneSize());
1973   if (addr.IsImmediateOffset()) {
1974     VIXL_ASSERT(addr.GetOffset() == 0);
1975   } else {
1976     int offset = vt.GetSizeInBytes();
1977     switch (op) {
1978       case NEON_LD1_1v:
1979       case NEON_ST1_1v:
1980         offset *= 1;
1981         break;
1982       case NEONLoadStoreSingleStructLoad1:
1983       case NEONLoadStoreSingleStructStore1:
1984       case NEON_LD1R:
1985         offset = (offset / vt.GetLanes()) * 1;
1986         break;
1987 
1988       case NEON_LD1_2v:
1989       case NEON_ST1_2v:
1990       case NEON_LD2:
1991       case NEON_ST2:
1992         offset *= 2;
1993         break;
1994       case NEONLoadStoreSingleStructLoad2:
1995       case NEONLoadStoreSingleStructStore2:
1996       case NEON_LD2R:
1997         offset = (offset / vt.GetLanes()) * 2;
1998         break;
1999 
2000       case NEON_LD1_3v:
2001       case NEON_ST1_3v:
2002       case NEON_LD3:
2003       case NEON_ST3:
2004         offset *= 3;
2005         break;
2006       case NEONLoadStoreSingleStructLoad3:
2007       case NEONLoadStoreSingleStructStore3:
2008       case NEON_LD3R:
2009         offset = (offset / vt.GetLanes()) * 3;
2010         break;
2011 
2012       case NEON_LD1_4v:
2013       case NEON_ST1_4v:
2014       case NEON_LD4:
2015       case NEON_ST4:
2016         offset *= 4;
2017         break;
2018       case NEONLoadStoreSingleStructLoad4:
2019       case NEONLoadStoreSingleStructStore4:
2020       case NEON_LD4R:
2021         offset = (offset / vt.GetLanes()) * 4;
2022         break;
2023       default:
2024         VIXL_UNREACHABLE();
2025     }
2026     VIXL_ASSERT(!addr.GetRegisterOffset().Is(NoReg) ||
2027                 addr.GetOffset() == offset);
2028   }
2029 #else
2030   USE(vt, addr, op);
2031 #endif
2032 }
2033 
LoadStoreStruct(const VRegister & vt,const MemOperand & addr,NEONLoadStoreMultiStructOp op)2034 void Assembler::LoadStoreStruct(const VRegister& vt,
2035                                 const MemOperand& addr,
2036                                 NEONLoadStoreMultiStructOp op) {
2037   LoadStoreStructVerify(vt, addr, op);
2038   VIXL_ASSERT(vt.IsVector() || vt.Is1D());
2039   Emit(op | LoadStoreStructAddrModeField(addr) | LSVFormat(vt) | Rt(vt));
2040 }
2041 
2042 
LoadStoreStructSingleAllLanes(const VRegister & vt,const MemOperand & addr,NEONLoadStoreSingleStructOp op)2043 void Assembler::LoadStoreStructSingleAllLanes(const VRegister& vt,
2044                                               const MemOperand& addr,
2045                                               NEONLoadStoreSingleStructOp op) {
2046   LoadStoreStructVerify(vt, addr, op);
2047   Emit(op | LoadStoreStructAddrModeField(addr) | LSVFormat(vt) | Rt(vt));
2048 }
2049 
2050 
ld1(const VRegister & vt,const MemOperand & src)2051 void Assembler::ld1(const VRegister& vt, const MemOperand& src) {
2052   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2053   LoadStoreStruct(vt, src, NEON_LD1_1v);
2054 }
2055 
2056 
ld1(const VRegister & vt,const VRegister & vt2,const MemOperand & src)2057 void Assembler::ld1(const VRegister& vt,
2058                     const VRegister& vt2,
2059                     const MemOperand& src) {
2060   USE(vt2);
2061   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2062   VIXL_ASSERT(AreSameFormat(vt, vt2));
2063   VIXL_ASSERT(AreConsecutive(vt, vt2));
2064   LoadStoreStruct(vt, src, NEON_LD1_2v);
2065 }
2066 
2067 
ld1(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const MemOperand & src)2068 void Assembler::ld1(const VRegister& vt,
2069                     const VRegister& vt2,
2070                     const VRegister& vt3,
2071                     const MemOperand& src) {
2072   USE(vt2, vt3);
2073   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2074   VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
2075   VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
2076   LoadStoreStruct(vt, src, NEON_LD1_3v);
2077 }
2078 
2079 
ld1(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const VRegister & vt4,const MemOperand & src)2080 void Assembler::ld1(const VRegister& vt,
2081                     const VRegister& vt2,
2082                     const VRegister& vt3,
2083                     const VRegister& vt4,
2084                     const MemOperand& src) {
2085   USE(vt2, vt3, vt4);
2086   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2087   VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
2088   VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
2089   LoadStoreStruct(vt, src, NEON_LD1_4v);
2090 }
2091 
2092 
ld2(const VRegister & vt,const VRegister & vt2,const MemOperand & src)2093 void Assembler::ld2(const VRegister& vt,
2094                     const VRegister& vt2,
2095                     const MemOperand& src) {
2096   USE(vt2);
2097   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2098   VIXL_ASSERT(AreSameFormat(vt, vt2));
2099   VIXL_ASSERT(AreConsecutive(vt, vt2));
2100   LoadStoreStruct(vt, src, NEON_LD2);
2101 }
2102 
2103 
ld2(const VRegister & vt,const VRegister & vt2,int lane,const MemOperand & src)2104 void Assembler::ld2(const VRegister& vt,
2105                     const VRegister& vt2,
2106                     int lane,
2107                     const MemOperand& src) {
2108   USE(vt2);
2109   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2110   VIXL_ASSERT(AreSameFormat(vt, vt2));
2111   VIXL_ASSERT(AreConsecutive(vt, vt2));
2112   LoadStoreStructSingle(vt, lane, src, NEONLoadStoreSingleStructLoad2);
2113 }
2114 
2115 
ld2r(const VRegister & vt,const VRegister & vt2,const MemOperand & src)2116 void Assembler::ld2r(const VRegister& vt,
2117                      const VRegister& vt2,
2118                      const MemOperand& src) {
2119   USE(vt2);
2120   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2121   VIXL_ASSERT(AreSameFormat(vt, vt2));
2122   VIXL_ASSERT(AreConsecutive(vt, vt2));
2123   LoadStoreStructSingleAllLanes(vt, src, NEON_LD2R);
2124 }
2125 
2126 
ld3(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const MemOperand & src)2127 void Assembler::ld3(const VRegister& vt,
2128                     const VRegister& vt2,
2129                     const VRegister& vt3,
2130                     const MemOperand& src) {
2131   USE(vt2, vt3);
2132   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2133   VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
2134   VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
2135   LoadStoreStruct(vt, src, NEON_LD3);
2136 }
2137 
2138 
ld3(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,int lane,const MemOperand & src)2139 void Assembler::ld3(const VRegister& vt,
2140                     const VRegister& vt2,
2141                     const VRegister& vt3,
2142                     int lane,
2143                     const MemOperand& src) {
2144   USE(vt2, vt3);
2145   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2146   VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
2147   VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
2148   LoadStoreStructSingle(vt, lane, src, NEONLoadStoreSingleStructLoad3);
2149 }
2150 
2151 
ld3r(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const MemOperand & src)2152 void Assembler::ld3r(const VRegister& vt,
2153                      const VRegister& vt2,
2154                      const VRegister& vt3,
2155                      const MemOperand& src) {
2156   USE(vt2, vt3);
2157   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2158   VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
2159   VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
2160   LoadStoreStructSingleAllLanes(vt, src, NEON_LD3R);
2161 }
2162 
2163 
ld4(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const VRegister & vt4,const MemOperand & src)2164 void Assembler::ld4(const VRegister& vt,
2165                     const VRegister& vt2,
2166                     const VRegister& vt3,
2167                     const VRegister& vt4,
2168                     const MemOperand& src) {
2169   USE(vt2, vt3, vt4);
2170   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2171   VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
2172   VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
2173   LoadStoreStruct(vt, src, NEON_LD4);
2174 }
2175 
2176 
ld4(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const VRegister & vt4,int lane,const MemOperand & src)2177 void Assembler::ld4(const VRegister& vt,
2178                     const VRegister& vt2,
2179                     const VRegister& vt3,
2180                     const VRegister& vt4,
2181                     int lane,
2182                     const MemOperand& src) {
2183   USE(vt2, vt3, vt4);
2184   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2185   VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
2186   VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
2187   LoadStoreStructSingle(vt, lane, src, NEONLoadStoreSingleStructLoad4);
2188 }
2189 
2190 
ld4r(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const VRegister & vt4,const MemOperand & src)2191 void Assembler::ld4r(const VRegister& vt,
2192                      const VRegister& vt2,
2193                      const VRegister& vt3,
2194                      const VRegister& vt4,
2195                      const MemOperand& src) {
2196   USE(vt2, vt3, vt4);
2197   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2198   VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
2199   VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
2200   LoadStoreStructSingleAllLanes(vt, src, NEON_LD4R);
2201 }
2202 
2203 
st1(const VRegister & vt,const MemOperand & src)2204 void Assembler::st1(const VRegister& vt, const MemOperand& src) {
2205   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2206   LoadStoreStruct(vt, src, NEON_ST1_1v);
2207 }
2208 
2209 
st1(const VRegister & vt,const VRegister & vt2,const MemOperand & src)2210 void Assembler::st1(const VRegister& vt,
2211                     const VRegister& vt2,
2212                     const MemOperand& src) {
2213   USE(vt2);
2214   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2215   VIXL_ASSERT(AreSameFormat(vt, vt2));
2216   VIXL_ASSERT(AreConsecutive(vt, vt2));
2217   LoadStoreStruct(vt, src, NEON_ST1_2v);
2218 }
2219 
2220 
st1(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const MemOperand & src)2221 void Assembler::st1(const VRegister& vt,
2222                     const VRegister& vt2,
2223                     const VRegister& vt3,
2224                     const MemOperand& src) {
2225   USE(vt2, vt3);
2226   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2227   VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
2228   VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
2229   LoadStoreStruct(vt, src, NEON_ST1_3v);
2230 }
2231 
2232 
st1(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const VRegister & vt4,const MemOperand & src)2233 void Assembler::st1(const VRegister& vt,
2234                     const VRegister& vt2,
2235                     const VRegister& vt3,
2236                     const VRegister& vt4,
2237                     const MemOperand& src) {
2238   USE(vt2, vt3, vt4);
2239   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2240   VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
2241   VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
2242   LoadStoreStruct(vt, src, NEON_ST1_4v);
2243 }
2244 
2245 
st2(const VRegister & vt,const VRegister & vt2,const MemOperand & dst)2246 void Assembler::st2(const VRegister& vt,
2247                     const VRegister& vt2,
2248                     const MemOperand& dst) {
2249   USE(vt2);
2250   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2251   VIXL_ASSERT(AreSameFormat(vt, vt2));
2252   VIXL_ASSERT(AreConsecutive(vt, vt2));
2253   LoadStoreStruct(vt, dst, NEON_ST2);
2254 }
2255 
2256 
st2(const VRegister & vt,const VRegister & vt2,int lane,const MemOperand & dst)2257 void Assembler::st2(const VRegister& vt,
2258                     const VRegister& vt2,
2259                     int lane,
2260                     const MemOperand& dst) {
2261   USE(vt2);
2262   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2263   VIXL_ASSERT(AreSameFormat(vt, vt2));
2264   VIXL_ASSERT(AreConsecutive(vt, vt2));
2265   LoadStoreStructSingle(vt, lane, dst, NEONLoadStoreSingleStructStore2);
2266 }
2267 
2268 
st3(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const MemOperand & dst)2269 void Assembler::st3(const VRegister& vt,
2270                     const VRegister& vt2,
2271                     const VRegister& vt3,
2272                     const MemOperand& dst) {
2273   USE(vt2, vt3);
2274   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2275   VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
2276   VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
2277   LoadStoreStruct(vt, dst, NEON_ST3);
2278 }
2279 
2280 
st3(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,int lane,const MemOperand & dst)2281 void Assembler::st3(const VRegister& vt,
2282                     const VRegister& vt2,
2283                     const VRegister& vt3,
2284                     int lane,
2285                     const MemOperand& dst) {
2286   USE(vt2, vt3);
2287   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2288   VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
2289   VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
2290   LoadStoreStructSingle(vt, lane, dst, NEONLoadStoreSingleStructStore3);
2291 }
2292 
2293 
st4(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const VRegister & vt4,const MemOperand & dst)2294 void Assembler::st4(const VRegister& vt,
2295                     const VRegister& vt2,
2296                     const VRegister& vt3,
2297                     const VRegister& vt4,
2298                     const MemOperand& dst) {
2299   USE(vt2, vt3, vt4);
2300   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2301   VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
2302   VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
2303   LoadStoreStruct(vt, dst, NEON_ST4);
2304 }
2305 
2306 
st4(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const VRegister & vt4,int lane,const MemOperand & dst)2307 void Assembler::st4(const VRegister& vt,
2308                     const VRegister& vt2,
2309                     const VRegister& vt3,
2310                     const VRegister& vt4,
2311                     int lane,
2312                     const MemOperand& dst) {
2313   USE(vt2, vt3, vt4);
2314   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2315   VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
2316   VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
2317   LoadStoreStructSingle(vt, lane, dst, NEONLoadStoreSingleStructStore4);
2318 }
2319 
2320 
LoadStoreStructSingle(const VRegister & vt,uint32_t lane,const MemOperand & addr,NEONLoadStoreSingleStructOp op)2321 void Assembler::LoadStoreStructSingle(const VRegister& vt,
2322                                       uint32_t lane,
2323                                       const MemOperand& addr,
2324                                       NEONLoadStoreSingleStructOp op) {
2325   LoadStoreStructVerify(vt, addr, op);
2326 
2327   // We support vt arguments of the form vt.VxT() or vt.T(), where x is the
2328   // number of lanes, and T is b, h, s or d.
2329   unsigned lane_size = vt.GetLaneSizeInBytes();
2330   VIXL_ASSERT(lane_size > 0);
2331   VIXL_ASSERT(lane < (kQRegSizeInBytes / lane_size));
2332 
2333   // Lane size is encoded in the opcode field. Lane index is encoded in the Q,
2334   // S and size fields.
2335   lane *= lane_size;
2336   if (lane_size == 8) lane++;
2337 
2338   Instr size = (lane << NEONLSSize_offset) & NEONLSSize_mask;
2339   Instr s = (lane << (NEONS_offset - 2)) & NEONS_mask;
2340   Instr q = (lane << (NEONQ_offset - 3)) & NEONQ_mask;
2341 
2342   Instr instr = op;
2343   switch (lane_size) {
2344     case 1:
2345       instr |= NEONLoadStoreSingle_b;
2346       break;
2347     case 2:
2348       instr |= NEONLoadStoreSingle_h;
2349       break;
2350     case 4:
2351       instr |= NEONLoadStoreSingle_s;
2352       break;
2353     default:
2354       VIXL_ASSERT(lane_size == 8);
2355       instr |= NEONLoadStoreSingle_d;
2356   }
2357 
2358   Emit(instr | LoadStoreStructAddrModeField(addr) | q | size | s | Rt(vt));
2359 }
2360 
2361 
ld1(const VRegister & vt,int lane,const MemOperand & src)2362 void Assembler::ld1(const VRegister& vt, int lane, const MemOperand& src) {
2363   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2364   LoadStoreStructSingle(vt, lane, src, NEONLoadStoreSingleStructLoad1);
2365 }
2366 
2367 
ld1r(const VRegister & vt,const MemOperand & src)2368 void Assembler::ld1r(const VRegister& vt, const MemOperand& src) {
2369   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2370   LoadStoreStructSingleAllLanes(vt, src, NEON_LD1R);
2371 }
2372 
2373 
st1(const VRegister & vt,int lane,const MemOperand & dst)2374 void Assembler::st1(const VRegister& vt, int lane, const MemOperand& dst) {
2375   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2376   LoadStoreStructSingle(vt, lane, dst, NEONLoadStoreSingleStructStore1);
2377 }
2378 
2379 
NEON3DifferentL(const VRegister & vd,const VRegister & vn,const VRegister & vm,NEON3DifferentOp vop)2380 void Assembler::NEON3DifferentL(const VRegister& vd,
2381                                 const VRegister& vn,
2382                                 const VRegister& vm,
2383                                 NEON3DifferentOp vop) {
2384   VIXL_ASSERT(AreSameFormat(vn, vm));
2385   VIXL_ASSERT((vn.Is1H() && vd.Is1S()) || (vn.Is1S() && vd.Is1D()) ||
2386               (vn.Is8B() && vd.Is8H()) || (vn.Is4H() && vd.Is4S()) ||
2387               (vn.Is2S() && vd.Is2D()) || (vn.Is16B() && vd.Is8H()) ||
2388               (vn.Is8H() && vd.Is4S()) || (vn.Is4S() && vd.Is2D()));
2389   Instr format, op = vop;
2390   if (vd.IsScalar()) {
2391     op |= NEON_Q | NEONScalar;
2392     format = SFormat(vn);
2393   } else {
2394     format = VFormat(vn);
2395   }
2396   Emit(format | op | Rm(vm) | Rn(vn) | Rd(vd));
2397 }
2398 
2399 
NEON3DifferentW(const VRegister & vd,const VRegister & vn,const VRegister & vm,NEON3DifferentOp vop)2400 void Assembler::NEON3DifferentW(const VRegister& vd,
2401                                 const VRegister& vn,
2402                                 const VRegister& vm,
2403                                 NEON3DifferentOp vop) {
2404   VIXL_ASSERT(AreSameFormat(vd, vn));
2405   VIXL_ASSERT((vm.Is8B() && vd.Is8H()) || (vm.Is4H() && vd.Is4S()) ||
2406               (vm.Is2S() && vd.Is2D()) || (vm.Is16B() && vd.Is8H()) ||
2407               (vm.Is8H() && vd.Is4S()) || (vm.Is4S() && vd.Is2D()));
2408   Emit(VFormat(vm) | vop | Rm(vm) | Rn(vn) | Rd(vd));
2409 }
2410 
2411 
NEON3DifferentHN(const VRegister & vd,const VRegister & vn,const VRegister & vm,NEON3DifferentOp vop)2412 void Assembler::NEON3DifferentHN(const VRegister& vd,
2413                                  const VRegister& vn,
2414                                  const VRegister& vm,
2415                                  NEON3DifferentOp vop) {
2416   VIXL_ASSERT(AreSameFormat(vm, vn));
2417   VIXL_ASSERT((vd.Is8B() && vn.Is8H()) || (vd.Is4H() && vn.Is4S()) ||
2418               (vd.Is2S() && vn.Is2D()) || (vd.Is16B() && vn.Is8H()) ||
2419               (vd.Is8H() && vn.Is4S()) || (vd.Is4S() && vn.Is2D()));
2420   Emit(VFormat(vd) | vop | Rm(vm) | Rn(vn) | Rd(vd));
2421 }
2422 
2423 
2424 // clang-format off
2425 #define NEON_3DIFF_LONG_LIST(V) \
2426   V(pmull,  NEON_PMULL,  vn.IsVector() && vn.Is8B())                           \
2427   V(pmull2, NEON_PMULL2, vn.IsVector() && vn.Is16B())                          \
2428   V(saddl,  NEON_SADDL,  vn.IsVector() && vn.IsD())                            \
2429   V(saddl2, NEON_SADDL2, vn.IsVector() && vn.IsQ())                            \
2430   V(sabal,  NEON_SABAL,  vn.IsVector() && vn.IsD())                            \
2431   V(sabal2, NEON_SABAL2, vn.IsVector() && vn.IsQ())                            \
2432   V(uabal,  NEON_UABAL,  vn.IsVector() && vn.IsD())                            \
2433   V(uabal2, NEON_UABAL2, vn.IsVector() && vn.IsQ())                            \
2434   V(sabdl,  NEON_SABDL,  vn.IsVector() && vn.IsD())                            \
2435   V(sabdl2, NEON_SABDL2, vn.IsVector() && vn.IsQ())                            \
2436   V(uabdl,  NEON_UABDL,  vn.IsVector() && vn.IsD())                            \
2437   V(uabdl2, NEON_UABDL2, vn.IsVector() && vn.IsQ())                            \
2438   V(smlal,  NEON_SMLAL,  vn.IsVector() && vn.IsD())                            \
2439   V(smlal2, NEON_SMLAL2, vn.IsVector() && vn.IsQ())                            \
2440   V(umlal,  NEON_UMLAL,  vn.IsVector() && vn.IsD())                            \
2441   V(umlal2, NEON_UMLAL2, vn.IsVector() && vn.IsQ())                            \
2442   V(smlsl,  NEON_SMLSL,  vn.IsVector() && vn.IsD())                            \
2443   V(smlsl2, NEON_SMLSL2, vn.IsVector() && vn.IsQ())                            \
2444   V(umlsl,  NEON_UMLSL,  vn.IsVector() && vn.IsD())                            \
2445   V(umlsl2, NEON_UMLSL2, vn.IsVector() && vn.IsQ())                            \
2446   V(smull,  NEON_SMULL,  vn.IsVector() && vn.IsD())                            \
2447   V(smull2, NEON_SMULL2, vn.IsVector() && vn.IsQ())                            \
2448   V(umull,  NEON_UMULL,  vn.IsVector() && vn.IsD())                            \
2449   V(umull2, NEON_UMULL2, vn.IsVector() && vn.IsQ())                            \
2450   V(ssubl,  NEON_SSUBL,  vn.IsVector() && vn.IsD())                            \
2451   V(ssubl2, NEON_SSUBL2, vn.IsVector() && vn.IsQ())                            \
2452   V(uaddl,  NEON_UADDL,  vn.IsVector() && vn.IsD())                            \
2453   V(uaddl2, NEON_UADDL2, vn.IsVector() && vn.IsQ())                            \
2454   V(usubl,  NEON_USUBL,  vn.IsVector() && vn.IsD())                            \
2455   V(usubl2, NEON_USUBL2, vn.IsVector() && vn.IsQ())                            \
2456   V(sqdmlal,  NEON_SQDMLAL,  vn.Is1H() || vn.Is1S() || vn.Is4H() || vn.Is2S()) \
2457   V(sqdmlal2, NEON_SQDMLAL2, vn.Is1H() || vn.Is1S() || vn.Is8H() || vn.Is4S()) \
2458   V(sqdmlsl,  NEON_SQDMLSL,  vn.Is1H() || vn.Is1S() || vn.Is4H() || vn.Is2S()) \
2459   V(sqdmlsl2, NEON_SQDMLSL2, vn.Is1H() || vn.Is1S() || vn.Is8H() || vn.Is4S()) \
2460   V(sqdmull,  NEON_SQDMULL,  vn.Is1H() || vn.Is1S() || vn.Is4H() || vn.Is2S()) \
2461   V(sqdmull2, NEON_SQDMULL2, vn.Is1H() || vn.Is1S() || vn.Is8H() || vn.Is4S()) \
2462 // clang-format on
2463 
2464 
2465 #define VIXL_DEFINE_ASM_FUNC(FN, OP, AS)                   \
2466 void Assembler::FN(const VRegister& vd,               \
2467                    const VRegister& vn,               \
2468                    const VRegister& vm) {             \
2469   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));            \
2470   VIXL_ASSERT(AS);                                    \
2471   NEON3DifferentL(vd, vn, vm, OP);                    \
2472 }
2473 NEON_3DIFF_LONG_LIST(VIXL_DEFINE_ASM_FUNC)
2474 #undef VIXL_DEFINE_ASM_FUNC
2475 
2476 // clang-format off
2477 #define NEON_3DIFF_HN_LIST(V)         \
2478   V(addhn,   NEON_ADDHN,   vd.IsD())  \
2479   V(addhn2,  NEON_ADDHN2,  vd.IsQ())  \
2480   V(raddhn,  NEON_RADDHN,  vd.IsD())  \
2481   V(raddhn2, NEON_RADDHN2, vd.IsQ())  \
2482   V(subhn,   NEON_SUBHN,   vd.IsD())  \
2483   V(subhn2,  NEON_SUBHN2,  vd.IsQ())  \
2484   V(rsubhn,  NEON_RSUBHN,  vd.IsD())  \
2485   V(rsubhn2, NEON_RSUBHN2, vd.IsQ())
2486 // clang-format on
2487 
2488 #define VIXL_DEFINE_ASM_FUNC(FN, OP, AS)     \
2489   void Assembler::FN(const VRegister& vd,    \
2490                      const VRegister& vn,    \
2491                      const VRegister& vm) {  \
2492     VIXL_ASSERT(CPUHas(CPUFeatures::kNEON)); \
2493     VIXL_ASSERT(AS);                         \
2494     NEON3DifferentHN(vd, vn, vm, OP);        \
2495   }
NEON_3DIFF_HN_LIST(VIXL_DEFINE_ASM_FUNC)2496 NEON_3DIFF_HN_LIST(VIXL_DEFINE_ASM_FUNC)
2497 #undef VIXL_DEFINE_ASM_FUNC
2498 
2499 void Assembler::uaddw(const VRegister& vd,
2500                       const VRegister& vn,
2501                       const VRegister& vm) {
2502   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2503   VIXL_ASSERT(vm.IsD());
2504   NEON3DifferentW(vd, vn, vm, NEON_UADDW);
2505 }
2506 
2507 
uaddw2(const VRegister & vd,const VRegister & vn,const VRegister & vm)2508 void Assembler::uaddw2(const VRegister& vd,
2509                        const VRegister& vn,
2510                        const VRegister& vm) {
2511   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2512   VIXL_ASSERT(vm.IsQ());
2513   NEON3DifferentW(vd, vn, vm, NEON_UADDW2);
2514 }
2515 
2516 
saddw(const VRegister & vd,const VRegister & vn,const VRegister & vm)2517 void Assembler::saddw(const VRegister& vd,
2518                       const VRegister& vn,
2519                       const VRegister& vm) {
2520   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2521   VIXL_ASSERT(vm.IsD());
2522   NEON3DifferentW(vd, vn, vm, NEON_SADDW);
2523 }
2524 
2525 
saddw2(const VRegister & vd,const VRegister & vn,const VRegister & vm)2526 void Assembler::saddw2(const VRegister& vd,
2527                        const VRegister& vn,
2528                        const VRegister& vm) {
2529   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2530   VIXL_ASSERT(vm.IsQ());
2531   NEON3DifferentW(vd, vn, vm, NEON_SADDW2);
2532 }
2533 
2534 
usubw(const VRegister & vd,const VRegister & vn,const VRegister & vm)2535 void Assembler::usubw(const VRegister& vd,
2536                       const VRegister& vn,
2537                       const VRegister& vm) {
2538   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2539   VIXL_ASSERT(vm.IsD());
2540   NEON3DifferentW(vd, vn, vm, NEON_USUBW);
2541 }
2542 
2543 
usubw2(const VRegister & vd,const VRegister & vn,const VRegister & vm)2544 void Assembler::usubw2(const VRegister& vd,
2545                        const VRegister& vn,
2546                        const VRegister& vm) {
2547   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2548   VIXL_ASSERT(vm.IsQ());
2549   NEON3DifferentW(vd, vn, vm, NEON_USUBW2);
2550 }
2551 
2552 
ssubw(const VRegister & vd,const VRegister & vn,const VRegister & vm)2553 void Assembler::ssubw(const VRegister& vd,
2554                       const VRegister& vn,
2555                       const VRegister& vm) {
2556   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2557   VIXL_ASSERT(vm.IsD());
2558   NEON3DifferentW(vd, vn, vm, NEON_SSUBW);
2559 }
2560 
2561 
ssubw2(const VRegister & vd,const VRegister & vn,const VRegister & vm)2562 void Assembler::ssubw2(const VRegister& vd,
2563                        const VRegister& vn,
2564                        const VRegister& vm) {
2565   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2566   VIXL_ASSERT(vm.IsQ());
2567   NEON3DifferentW(vd, vn, vm, NEON_SSUBW2);
2568 }
2569 
2570 
mov(const Register & rd,const Register & rm)2571 void Assembler::mov(const Register& rd, const Register& rm) {
2572   // Moves involving the stack pointer are encoded as add immediate with
2573   // second operand of zero. Otherwise, orr with first operand zr is
2574   // used.
2575   if (rd.IsSP() || rm.IsSP()) {
2576     add(rd, rm, 0);
2577   } else {
2578     orr(rd, AppropriateZeroRegFor(rd), rm);
2579   }
2580 }
2581 
xpaclri()2582 void Assembler::xpaclri() {
2583   VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
2584   Emit(XPACLRI);
2585 }
2586 
pacia1716()2587 void Assembler::pacia1716() {
2588   VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
2589   Emit(PACIA1716);
2590 }
2591 
pacib1716()2592 void Assembler::pacib1716() {
2593   VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
2594   Emit(PACIB1716);
2595 }
2596 
autia1716()2597 void Assembler::autia1716() {
2598   VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
2599   Emit(AUTIA1716);
2600 }
2601 
autib1716()2602 void Assembler::autib1716() {
2603   VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
2604   Emit(AUTIB1716);
2605 }
2606 
paciaz()2607 void Assembler::paciaz() {
2608   VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
2609   Emit(PACIAZ);
2610 }
2611 
pacibz()2612 void Assembler::pacibz() {
2613   VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
2614   Emit(PACIBZ);
2615 }
2616 
autiaz()2617 void Assembler::autiaz() {
2618   VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
2619   Emit(AUTIAZ);
2620 }
2621 
autibz()2622 void Assembler::autibz() {
2623   VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
2624   Emit(AUTIBZ);
2625 }
2626 
paciasp()2627 void Assembler::paciasp() {
2628   VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
2629   Emit(PACIASP);
2630 }
2631 
pacibsp()2632 void Assembler::pacibsp() {
2633   VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
2634   Emit(PACIBSP);
2635 }
2636 
autiasp()2637 void Assembler::autiasp() {
2638   VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
2639   Emit(AUTIASP);
2640 }
2641 
autibsp()2642 void Assembler::autibsp() {
2643   VIXL_ASSERT(CPUHas(CPUFeatures::kPAuth));
2644   Emit(AUTIBSP);
2645 }
2646 
bti(BranchTargetIdentifier id)2647 void Assembler::bti(BranchTargetIdentifier id) {
2648   VIXL_ASSERT((id != EmitPACIASP) && (id != EmitPACIBSP));  // Not modes of Bti.
2649   VIXL_ASSERT(id != EmitBTI_none);  // Always generate an instruction.
2650   VIXL_ASSERT(CPUHas(CPUFeatures::kBTI));
2651   hint(static_cast<SystemHint>(id));
2652 }
2653 
mvn(const Register & rd,const Operand & operand)2654 void Assembler::mvn(const Register& rd, const Operand& operand) {
2655   orn(rd, AppropriateZeroRegFor(rd), operand);
2656 }
2657 
2658 
mrs(const Register & xt,SystemRegister sysreg)2659 void Assembler::mrs(const Register& xt, SystemRegister sysreg) {
2660   VIXL_ASSERT(xt.Is64Bits());
2661   VIXL_ASSERT(CPUHas(sysreg));
2662   Emit(MRS | ImmSystemRegister(sysreg) | Rt(xt));
2663 }
2664 
2665 
msr(SystemRegister sysreg,const Register & xt)2666 void Assembler::msr(SystemRegister sysreg, const Register& xt) {
2667   VIXL_ASSERT(xt.Is64Bits());
2668   VIXL_ASSERT(CPUHas(sysreg));
2669   Emit(MSR | Rt(xt) | ImmSystemRegister(sysreg));
2670 }
2671 
2672 
cfinv()2673 void Assembler::cfinv() {
2674   VIXL_ASSERT(CPUHas(CPUFeatures::kFlagM));
2675   Emit(CFINV);
2676 }
2677 
2678 
axflag()2679 void Assembler::axflag() {
2680   VIXL_ASSERT(CPUHas(CPUFeatures::kAXFlag));
2681   Emit(AXFLAG);
2682 }
2683 
2684 
xaflag()2685 void Assembler::xaflag() {
2686   VIXL_ASSERT(CPUHas(CPUFeatures::kAXFlag));
2687   Emit(XAFLAG);
2688 }
2689 
2690 
clrex(int imm4)2691 void Assembler::clrex(int imm4) { Emit(CLREX | CRm(imm4)); }
2692 
2693 
dmb(BarrierDomain domain,BarrierType type)2694 void Assembler::dmb(BarrierDomain domain, BarrierType type) {
2695   Emit(DMB | ImmBarrierDomain(domain) | ImmBarrierType(type));
2696 }
2697 
2698 
dsb(BarrierDomain domain,BarrierType type)2699 void Assembler::dsb(BarrierDomain domain, BarrierType type) {
2700   Emit(DSB | ImmBarrierDomain(domain) | ImmBarrierType(type));
2701 }
2702 
2703 
isb()2704 void Assembler::isb() {
2705   Emit(ISB | ImmBarrierDomain(FullSystem) | ImmBarrierType(BarrierAll));
2706 }
2707 
esb()2708 void Assembler::esb() {
2709   VIXL_ASSERT(CPUHas(CPUFeatures::kRAS));
2710   hint(ESB);
2711 }
2712 
csdb()2713 void Assembler::csdb() { hint(CSDB); }
2714 
fmov(const VRegister & vd,double imm)2715 void Assembler::fmov(const VRegister& vd, double imm) {
2716   VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
2717   if (vd.IsScalar()) {
2718     VIXL_ASSERT(vd.Is1D());
2719     Emit(FMOV_d_imm | Rd(vd) | ImmFP64(imm));
2720   } else {
2721     VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2722     VIXL_ASSERT(vd.Is2D());
2723     Instr op = NEONModifiedImmediate_MOVI | NEONModifiedImmediateOpBit;
2724     Instr q = NEON_Q;
2725     uint32_t encoded_imm = FP64ToImm8(imm);
2726     Emit(q | op | ImmNEONabcdefgh(encoded_imm) | NEONCmode(0xf) | Rd(vd));
2727   }
2728 }
2729 
2730 
fmov(const VRegister & vd,float imm)2731 void Assembler::fmov(const VRegister& vd, float imm) {
2732   VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
2733   if (vd.IsScalar()) {
2734     VIXL_ASSERT(vd.Is1S());
2735     Emit(FMOV_s_imm | Rd(vd) | ImmFP32(imm));
2736   } else {
2737     VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
2738     VIXL_ASSERT(vd.Is2S() || vd.Is4S());
2739     Instr op = NEONModifiedImmediate_MOVI;
2740     Instr q = vd.Is4S() ? NEON_Q : 0;
2741     uint32_t encoded_imm = FP32ToImm8(imm);
2742     Emit(q | op | ImmNEONabcdefgh(encoded_imm) | NEONCmode(0xf) | Rd(vd));
2743   }
2744 }
2745 
2746 
fmov(const VRegister & vd,Float16 imm)2747 void Assembler::fmov(const VRegister& vd, Float16 imm) {
2748   VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
2749   if (vd.IsScalar()) {
2750     VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
2751     VIXL_ASSERT(vd.Is1H());
2752     Emit(FMOV_h_imm | Rd(vd) | ImmFP16(imm));
2753   } else {
2754     VIXL_ASSERT(CPUHas(CPUFeatures::kNEON, CPUFeatures::kNEONHalf));
2755     VIXL_ASSERT(vd.Is4H() || vd.Is8H());
2756     Instr q = vd.Is8H() ? NEON_Q : 0;
2757     uint32_t encoded_imm = FP16ToImm8(imm);
2758     Emit(q | NEONModifiedImmediate_FMOV | ImmNEONabcdefgh(encoded_imm) |
2759          NEONCmode(0xf) | Rd(vd));
2760   }
2761 }
2762 
2763 
fmov(const Register & rd,const VRegister & vn)2764 void Assembler::fmov(const Register& rd, const VRegister& vn) {
2765   VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
2766   VIXL_ASSERT(vn.Is1H() || vn.Is1S() || vn.Is1D());
2767   VIXL_ASSERT((rd.GetSizeInBits() == vn.GetSizeInBits()) || vn.Is1H());
2768   FPIntegerConvertOp op;
2769   switch (vn.GetSizeInBits()) {
2770     case 16:
2771       VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
2772       op = rd.Is64Bits() ? FMOV_xh : FMOV_wh;
2773       break;
2774     case 32:
2775       op = FMOV_ws;
2776       break;
2777     default:
2778       op = FMOV_xd;
2779   }
2780   Emit(op | Rd(rd) | Rn(vn));
2781 }
2782 
2783 
fmov(const VRegister & vd,const Register & rn)2784 void Assembler::fmov(const VRegister& vd, const Register& rn) {
2785   VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
2786   VIXL_ASSERT(vd.Is1H() || vd.Is1S() || vd.Is1D());
2787   VIXL_ASSERT((vd.GetSizeInBits() == rn.GetSizeInBits()) || vd.Is1H());
2788   FPIntegerConvertOp op;
2789   switch (vd.GetSizeInBits()) {
2790     case 16:
2791       VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
2792       op = rn.Is64Bits() ? FMOV_hx : FMOV_hw;
2793       break;
2794     case 32:
2795       op = FMOV_sw;
2796       break;
2797     default:
2798       op = FMOV_dx;
2799   }
2800   Emit(op | Rd(vd) | Rn(rn));
2801 }
2802 
2803 
fmov(const VRegister & vd,const VRegister & vn)2804 void Assembler::fmov(const VRegister& vd, const VRegister& vn) {
2805   VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
2806   if (vd.Is1H()) {
2807     VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
2808   }
2809   VIXL_ASSERT(vd.Is1H() || vd.Is1S() || vd.Is1D());
2810   VIXL_ASSERT(vd.IsSameFormat(vn));
2811   Emit(FPType(vd) | FMOV | Rd(vd) | Rn(vn));
2812 }
2813 
2814 
fmov(const VRegister & vd,int index,const Register & rn)2815 void Assembler::fmov(const VRegister& vd, int index, const Register& rn) {
2816   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON, CPUFeatures::kFP));
2817   VIXL_ASSERT((index == 1) && vd.Is1D() && rn.IsX());
2818   USE(index);
2819   Emit(FMOV_d1_x | Rd(vd) | Rn(rn));
2820 }
2821 
2822 
fmov(const Register & rd,const VRegister & vn,int index)2823 void Assembler::fmov(const Register& rd, const VRegister& vn, int index) {
2824   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON, CPUFeatures::kFP));
2825   VIXL_ASSERT((index == 1) && vn.Is1D() && rd.IsX());
2826   USE(index);
2827   Emit(FMOV_x_d1 | Rd(rd) | Rn(vn));
2828 }
2829 
2830 
fmadd(const VRegister & vd,const VRegister & vn,const VRegister & vm,const VRegister & va)2831 void Assembler::fmadd(const VRegister& vd,
2832                       const VRegister& vn,
2833                       const VRegister& vm,
2834                       const VRegister& va) {
2835   VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
2836   FPDataProcessing3SourceOp op;
2837   if (vd.Is1H()) {
2838     VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
2839     op = FMADD_h;
2840   } else if (vd.Is1S()) {
2841     op = FMADD_s;
2842   } else {
2843     VIXL_ASSERT(vd.Is1D());
2844     op = FMADD_d;
2845   }
2846   FPDataProcessing3Source(vd, vn, vm, va, op);
2847 }
2848 
2849 
fmsub(const VRegister & vd,const VRegister & vn,const VRegister & vm,const VRegister & va)2850 void Assembler::fmsub(const VRegister& vd,
2851                       const VRegister& vn,
2852                       const VRegister& vm,
2853                       const VRegister& va) {
2854   VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
2855   FPDataProcessing3SourceOp op;
2856   if (vd.Is1H()) {
2857     VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
2858     op = FMSUB_h;
2859   } else if (vd.Is1S()) {
2860     op = FMSUB_s;
2861   } else {
2862     VIXL_ASSERT(vd.Is1D());
2863     op = FMSUB_d;
2864   }
2865   FPDataProcessing3Source(vd, vn, vm, va, op);
2866 }
2867 
2868 
fnmadd(const VRegister & vd,const VRegister & vn,const VRegister & vm,const VRegister & va)2869 void Assembler::fnmadd(const VRegister& vd,
2870                        const VRegister& vn,
2871                        const VRegister& vm,
2872                        const VRegister& va) {
2873   VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
2874   FPDataProcessing3SourceOp op;
2875   if (vd.Is1H()) {
2876     VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
2877     op = FNMADD_h;
2878   } else if (vd.Is1S()) {
2879     op = FNMADD_s;
2880   } else {
2881     VIXL_ASSERT(vd.Is1D());
2882     op = FNMADD_d;
2883   }
2884   FPDataProcessing3Source(vd, vn, vm, va, op);
2885 }
2886 
2887 
fnmsub(const VRegister & vd,const VRegister & vn,const VRegister & vm,const VRegister & va)2888 void Assembler::fnmsub(const VRegister& vd,
2889                        const VRegister& vn,
2890                        const VRegister& vm,
2891                        const VRegister& va) {
2892   VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
2893   FPDataProcessing3SourceOp op;
2894   if (vd.Is1H()) {
2895     VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
2896     op = FNMSUB_h;
2897   } else if (vd.Is1S()) {
2898     op = FNMSUB_s;
2899   } else {
2900     VIXL_ASSERT(vd.Is1D());
2901     op = FNMSUB_d;
2902   }
2903   FPDataProcessing3Source(vd, vn, vm, va, op);
2904 }
2905 
2906 
fnmul(const VRegister & vd,const VRegister & vn,const VRegister & vm)2907 void Assembler::fnmul(const VRegister& vd,
2908                       const VRegister& vn,
2909                       const VRegister& vm) {
2910   VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
2911   VIXL_ASSERT(AreSameSizeAndType(vd, vn, vm));
2912   Instr op;
2913   if (vd.Is1H()) {
2914     VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
2915     op = FNMUL_h;
2916   } else if (vd.Is1S()) {
2917     op = FNMUL_s;
2918   } else {
2919     VIXL_ASSERT(vd.Is1D());
2920     op = FNMUL_d;
2921   }
2922   Emit(FPType(vd) | op | Rm(vm) | Rn(vn) | Rd(vd));
2923 }
2924 
2925 
FPCompareMacro(const VRegister & vn,double value,FPTrapFlags trap)2926 void Assembler::FPCompareMacro(const VRegister& vn,
2927                                double value,
2928                                FPTrapFlags trap) {
2929   USE(value);
2930   // Although the fcmp{e} instructions can strictly only take an immediate
2931   // value of +0.0, we don't need to check for -0.0 because the sign of 0.0
2932   // doesn't affect the result of the comparison.
2933   VIXL_ASSERT(value == 0.0);
2934   VIXL_ASSERT(vn.Is1H() || vn.Is1S() || vn.Is1D());
2935   Instr op = (trap == EnableTrap) ? FCMPE_zero : FCMP_zero;
2936   Emit(FPType(vn) | op | Rn(vn));
2937 }
2938 
2939 
FPCompareMacro(const VRegister & vn,const VRegister & vm,FPTrapFlags trap)2940 void Assembler::FPCompareMacro(const VRegister& vn,
2941                                const VRegister& vm,
2942                                FPTrapFlags trap) {
2943   VIXL_ASSERT(vn.Is1H() || vn.Is1S() || vn.Is1D());
2944   VIXL_ASSERT(vn.IsSameSizeAndType(vm));
2945   Instr op = (trap == EnableTrap) ? FCMPE : FCMP;
2946   Emit(FPType(vn) | op | Rm(vm) | Rn(vn));
2947 }
2948 
2949 
fcmp(const VRegister & vn,const VRegister & vm)2950 void Assembler::fcmp(const VRegister& vn, const VRegister& vm) {
2951   VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
2952   if (vn.Is1H()) VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
2953   FPCompareMacro(vn, vm, DisableTrap);
2954 }
2955 
2956 
fcmpe(const VRegister & vn,const VRegister & vm)2957 void Assembler::fcmpe(const VRegister& vn, const VRegister& vm) {
2958   VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
2959   if (vn.Is1H()) VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
2960   FPCompareMacro(vn, vm, EnableTrap);
2961 }
2962 
2963 
fcmp(const VRegister & vn,double value)2964 void Assembler::fcmp(const VRegister& vn, double value) {
2965   VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
2966   if (vn.Is1H()) VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
2967   FPCompareMacro(vn, value, DisableTrap);
2968 }
2969 
2970 
fcmpe(const VRegister & vn,double value)2971 void Assembler::fcmpe(const VRegister& vn, double value) {
2972   VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
2973   if (vn.Is1H()) VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
2974   FPCompareMacro(vn, value, EnableTrap);
2975 }
2976 
2977 
FPCCompareMacro(const VRegister & vn,const VRegister & vm,StatusFlags nzcv,Condition cond,FPTrapFlags trap)2978 void Assembler::FPCCompareMacro(const VRegister& vn,
2979                                 const VRegister& vm,
2980                                 StatusFlags nzcv,
2981                                 Condition cond,
2982                                 FPTrapFlags trap) {
2983   VIXL_ASSERT(vn.Is1H() || vn.Is1S() || vn.Is1D());
2984   VIXL_ASSERT(vn.IsSameSizeAndType(vm));
2985   Instr op = (trap == EnableTrap) ? FCCMPE : FCCMP;
2986   Emit(FPType(vn) | op | Rm(vm) | Cond(cond) | Rn(vn) | Nzcv(nzcv));
2987 }
2988 
fccmp(const VRegister & vn,const VRegister & vm,StatusFlags nzcv,Condition cond)2989 void Assembler::fccmp(const VRegister& vn,
2990                       const VRegister& vm,
2991                       StatusFlags nzcv,
2992                       Condition cond) {
2993   VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
2994   if (vn.Is1H()) VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
2995   FPCCompareMacro(vn, vm, nzcv, cond, DisableTrap);
2996 }
2997 
2998 
fccmpe(const VRegister & vn,const VRegister & vm,StatusFlags nzcv,Condition cond)2999 void Assembler::fccmpe(const VRegister& vn,
3000                        const VRegister& vm,
3001                        StatusFlags nzcv,
3002                        Condition cond) {
3003   VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
3004   if (vn.Is1H()) VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
3005   FPCCompareMacro(vn, vm, nzcv, cond, EnableTrap);
3006 }
3007 
3008 
fcsel(const VRegister & vd,const VRegister & vn,const VRegister & vm,Condition cond)3009 void Assembler::fcsel(const VRegister& vd,
3010                       const VRegister& vn,
3011                       const VRegister& vm,
3012                       Condition cond) {
3013   VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
3014   if (vd.Is1H()) VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
3015   VIXL_ASSERT(vd.Is1H() || vd.Is1S() || vd.Is1D());
3016   VIXL_ASSERT(AreSameFormat(vd, vn, vm));
3017   Emit(FPType(vd) | FCSEL | Rm(vm) | Cond(cond) | Rn(vn) | Rd(vd));
3018 }
3019 
3020 
fcvt(const VRegister & vd,const VRegister & vn)3021 void Assembler::fcvt(const VRegister& vd, const VRegister& vn) {
3022   VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
3023   FPDataProcessing1SourceOp op;
3024   // The half-precision variants belong to base FP, and do not require kFPHalf.
3025   if (vd.Is1D()) {
3026     VIXL_ASSERT(vn.Is1S() || vn.Is1H());
3027     op = vn.Is1S() ? FCVT_ds : FCVT_dh;
3028   } else if (vd.Is1S()) {
3029     VIXL_ASSERT(vn.Is1D() || vn.Is1H());
3030     op = vn.Is1D() ? FCVT_sd : FCVT_sh;
3031   } else {
3032     VIXL_ASSERT(vd.Is1H());
3033     VIXL_ASSERT(vn.Is1D() || vn.Is1S());
3034     op = vn.Is1D() ? FCVT_hd : FCVT_hs;
3035   }
3036   FPDataProcessing1Source(vd, vn, op);
3037 }
3038 
3039 
fcvtl(const VRegister & vd,const VRegister & vn)3040 void Assembler::fcvtl(const VRegister& vd, const VRegister& vn) {
3041   VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
3042   VIXL_ASSERT((vd.Is4S() && vn.Is4H()) || (vd.Is2D() && vn.Is2S()));
3043   // The half-precision variants belong to base FP, and do not require kFPHalf.
3044   Instr format = vd.Is2D() ? (1 << NEONSize_offset) : 0;
3045   Emit(format | NEON_FCVTL | Rn(vn) | Rd(vd));
3046 }
3047 
3048 
fcvtl2(const VRegister & vd,const VRegister & vn)3049 void Assembler::fcvtl2(const VRegister& vd, const VRegister& vn) {
3050   VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
3051   VIXL_ASSERT((vd.Is4S() && vn.Is8H()) || (vd.Is2D() && vn.Is4S()));
3052   // The half-precision variants belong to base FP, and do not require kFPHalf.
3053   Instr format = vd.Is2D() ? (1 << NEONSize_offset) : 0;
3054   Emit(NEON_Q | format | NEON_FCVTL | Rn(vn) | Rd(vd));
3055 }
3056 
3057 
fcvtn(const VRegister & vd,const VRegister & vn)3058 void Assembler::fcvtn(const VRegister& vd, const VRegister& vn) {
3059   VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
3060   VIXL_ASSERT((vn.Is4S() && vd.Is4H()) || (vn.Is2D() && vd.Is2S()));
3061   // The half-precision variants belong to base FP, and do not require kFPHalf.
3062   Instr format = vn.Is2D() ? (1 << NEONSize_offset) : 0;
3063   Emit(format | NEON_FCVTN | Rn(vn) | Rd(vd));
3064 }
3065 
3066 
fcvtn2(const VRegister & vd,const VRegister & vn)3067 void Assembler::fcvtn2(const VRegister& vd, const VRegister& vn) {
3068   VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
3069   VIXL_ASSERT((vn.Is4S() && vd.Is8H()) || (vn.Is2D() && vd.Is4S()));
3070   // The half-precision variants belong to base FP, and do not require kFPHalf.
3071   Instr format = vn.Is2D() ? (1 << NEONSize_offset) : 0;
3072   Emit(NEON_Q | format | NEON_FCVTN | Rn(vn) | Rd(vd));
3073 }
3074 
3075 
fcvtxn(const VRegister & vd,const VRegister & vn)3076 void Assembler::fcvtxn(const VRegister& vd, const VRegister& vn) {
3077   VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
3078   Instr format = 1 << NEONSize_offset;
3079   if (vd.IsScalar()) {
3080     VIXL_ASSERT(vd.Is1S() && vn.Is1D());
3081     Emit(format | NEON_FCVTXN_scalar | Rn(vn) | Rd(vd));
3082   } else {
3083     VIXL_ASSERT(vd.Is2S() && vn.Is2D());
3084     Emit(format | NEON_FCVTXN | Rn(vn) | Rd(vd));
3085   }
3086 }
3087 
3088 
fcvtxn2(const VRegister & vd,const VRegister & vn)3089 void Assembler::fcvtxn2(const VRegister& vd, const VRegister& vn) {
3090   VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
3091   VIXL_ASSERT(vd.Is4S() && vn.Is2D());
3092   Instr format = 1 << NEONSize_offset;
3093   Emit(NEON_Q | format | NEON_FCVTXN | Rn(vn) | Rd(vd));
3094 }
3095 
fjcvtzs(const Register & rd,const VRegister & vn)3096 void Assembler::fjcvtzs(const Register& rd, const VRegister& vn) {
3097   VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kJSCVT));
3098   VIXL_ASSERT(rd.IsW() && vn.Is1D());
3099   Emit(FJCVTZS | Rn(vn) | Rd(rd));
3100 }
3101 
3102 
NEONFPConvertToInt(const Register & rd,const VRegister & vn,Instr op)3103 void Assembler::NEONFPConvertToInt(const Register& rd,
3104                                    const VRegister& vn,
3105                                    Instr op) {
3106   Emit(SF(rd) | FPType(vn) | op | Rn(vn) | Rd(rd));
3107 }
3108 
3109 
NEONFPConvertToInt(const VRegister & vd,const VRegister & vn,Instr op)3110 void Assembler::NEONFPConvertToInt(const VRegister& vd,
3111                                    const VRegister& vn,
3112                                    Instr op) {
3113   if (vn.IsScalar()) {
3114     VIXL_ASSERT((vd.Is1S() && vn.Is1S()) || (vd.Is1D() && vn.Is1D()));
3115     op |= NEON_Q | NEONScalar;
3116   }
3117   Emit(FPFormat(vn) | op | Rn(vn) | Rd(vd));
3118 }
3119 
3120 
NEONFP16ConvertToInt(const VRegister & vd,const VRegister & vn,Instr op)3121 void Assembler::NEONFP16ConvertToInt(const VRegister& vd,
3122                                      const VRegister& vn,
3123                                      Instr op) {
3124   VIXL_ASSERT(AreSameFormat(vd, vn));
3125   VIXL_ASSERT(vn.IsLaneSizeH());
3126   if (vn.IsScalar()) {
3127     op |= NEON_Q | NEONScalar;
3128   } else if (vn.Is8H()) {
3129     op |= NEON_Q;
3130   }
3131   Emit(op | Rn(vn) | Rd(vd));
3132 }
3133 
3134 
3135 #define NEON_FP2REGMISC_FCVT_LIST(V) \
3136   V(fcvtnu, NEON_FCVTNU, FCVTNU)     \
3137   V(fcvtns, NEON_FCVTNS, FCVTNS)     \
3138   V(fcvtpu, NEON_FCVTPU, FCVTPU)     \
3139   V(fcvtps, NEON_FCVTPS, FCVTPS)     \
3140   V(fcvtmu, NEON_FCVTMU, FCVTMU)     \
3141   V(fcvtms, NEON_FCVTMS, FCVTMS)     \
3142   V(fcvtau, NEON_FCVTAU, FCVTAU)     \
3143   V(fcvtas, NEON_FCVTAS, FCVTAS)
3144 
3145 #define VIXL_DEFINE_ASM_FUNC(FN, VEC_OP, SCA_OP)                 \
3146   void Assembler::FN(const Register& rd, const VRegister& vn) {  \
3147     VIXL_ASSERT(CPUHas(CPUFeatures::kFP));                       \
3148     if (vn.IsH()) VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));     \
3149     NEONFPConvertToInt(rd, vn, SCA_OP);                          \
3150   }                                                              \
3151   void Assembler::FN(const VRegister& vd, const VRegister& vn) { \
3152     VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));   \
3153     if (vd.IsLaneSizeH()) {                                      \
3154       VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));               \
3155       NEONFP16ConvertToInt(vd, vn, VEC_OP##_H);                  \
3156     } else {                                                     \
3157       NEONFPConvertToInt(vd, vn, VEC_OP);                        \
3158     }                                                            \
3159   }
NEON_FP2REGMISC_FCVT_LIST(VIXL_DEFINE_ASM_FUNC)3160 NEON_FP2REGMISC_FCVT_LIST(VIXL_DEFINE_ASM_FUNC)
3161 #undef VIXL_DEFINE_ASM_FUNC
3162 
3163 
3164 void Assembler::fcvtzs(const Register& rd, const VRegister& vn, int fbits) {
3165   VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
3166   if (vn.Is1H()) VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
3167   VIXL_ASSERT(vn.Is1H() || vn.Is1S() || vn.Is1D());
3168   VIXL_ASSERT((fbits >= 0) && (fbits <= rd.GetSizeInBits()));
3169   if (fbits == 0) {
3170     Emit(SF(rd) | FPType(vn) | FCVTZS | Rn(vn) | Rd(rd));
3171   } else {
3172     Emit(SF(rd) | FPType(vn) | FCVTZS_fixed | FPScale(64 - fbits) | Rn(vn) |
3173          Rd(rd));
3174   }
3175 }
3176 
3177 
fcvtzs(const VRegister & vd,const VRegister & vn,int fbits)3178 void Assembler::fcvtzs(const VRegister& vd, const VRegister& vn, int fbits) {
3179   // This form is a NEON scalar FP instruction.
3180   VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
3181   if (vn.IsLaneSizeH()) VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
3182   VIXL_ASSERT(fbits >= 0);
3183   if (fbits == 0) {
3184     if (vd.IsLaneSizeH()) {
3185       NEONFP2RegMiscFP16(vd, vn, NEON_FCVTZS_H);
3186     } else {
3187       NEONFP2RegMisc(vd, vn, NEON_FCVTZS);
3188     }
3189   } else {
3190     VIXL_ASSERT(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S() ||
3191                 vd.Is1H() || vd.Is4H() || vd.Is8H());
3192     NEONShiftRightImmediate(vd, vn, fbits, NEON_FCVTZS_imm);
3193   }
3194 }
3195 
3196 
fcvtzu(const Register & rd,const VRegister & vn,int fbits)3197 void Assembler::fcvtzu(const Register& rd, const VRegister& vn, int fbits) {
3198   VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
3199   if (vn.Is1H()) VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
3200   VIXL_ASSERT(vn.Is1H() || vn.Is1S() || vn.Is1D());
3201   VIXL_ASSERT((fbits >= 0) && (fbits <= rd.GetSizeInBits()));
3202   if (fbits == 0) {
3203     Emit(SF(rd) | FPType(vn) | FCVTZU | Rn(vn) | Rd(rd));
3204   } else {
3205     Emit(SF(rd) | FPType(vn) | FCVTZU_fixed | FPScale(64 - fbits) | Rn(vn) |
3206          Rd(rd));
3207   }
3208 }
3209 
3210 
fcvtzu(const VRegister & vd,const VRegister & vn,int fbits)3211 void Assembler::fcvtzu(const VRegister& vd, const VRegister& vn, int fbits) {
3212   // This form is a NEON scalar FP instruction.
3213   VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
3214   if (vn.IsLaneSizeH()) VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
3215   VIXL_ASSERT(fbits >= 0);
3216   if (fbits == 0) {
3217     if (vd.IsLaneSizeH()) {
3218       NEONFP2RegMiscFP16(vd, vn, NEON_FCVTZU_H);
3219     } else {
3220       NEONFP2RegMisc(vd, vn, NEON_FCVTZU);
3221     }
3222   } else {
3223     VIXL_ASSERT(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S() ||
3224                 vd.Is1H() || vd.Is4H() || vd.Is8H());
3225     NEONShiftRightImmediate(vd, vn, fbits, NEON_FCVTZU_imm);
3226   }
3227 }
3228 
ucvtf(const VRegister & vd,const VRegister & vn,int fbits)3229 void Assembler::ucvtf(const VRegister& vd, const VRegister& vn, int fbits) {
3230   // This form is a NEON scalar FP instruction.
3231   VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
3232   if (vn.IsLaneSizeH()) VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
3233   VIXL_ASSERT(fbits >= 0);
3234   if (fbits == 0) {
3235     if (vd.IsLaneSizeH()) {
3236       NEONFP2RegMiscFP16(vd, vn, NEON_UCVTF_H);
3237     } else {
3238       NEONFP2RegMisc(vd, vn, NEON_UCVTF);
3239     }
3240   } else {
3241     VIXL_ASSERT(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S() ||
3242                 vd.Is1H() || vd.Is4H() || vd.Is8H());
3243     NEONShiftRightImmediate(vd, vn, fbits, NEON_UCVTF_imm);
3244   }
3245 }
3246 
scvtf(const VRegister & vd,const VRegister & vn,int fbits)3247 void Assembler::scvtf(const VRegister& vd, const VRegister& vn, int fbits) {
3248   // This form is a NEON scalar FP instruction.
3249   VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
3250   if (vn.IsLaneSizeH()) VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
3251   VIXL_ASSERT(fbits >= 0);
3252   if (fbits == 0) {
3253     if (vd.IsLaneSizeH()) {
3254       NEONFP2RegMiscFP16(vd, vn, NEON_SCVTF_H);
3255     } else {
3256       NEONFP2RegMisc(vd, vn, NEON_SCVTF);
3257     }
3258   } else {
3259     VIXL_ASSERT(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S() ||
3260                 vd.Is1H() || vd.Is4H() || vd.Is8H());
3261     NEONShiftRightImmediate(vd, vn, fbits, NEON_SCVTF_imm);
3262   }
3263 }
3264 
3265 
scvtf(const VRegister & vd,const Register & rn,int fbits)3266 void Assembler::scvtf(const VRegister& vd, const Register& rn, int fbits) {
3267   VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
3268   if (vd.Is1H()) VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
3269   VIXL_ASSERT(vd.Is1H() || vd.Is1S() || vd.Is1D());
3270   VIXL_ASSERT(fbits >= 0);
3271   if (fbits == 0) {
3272     Emit(SF(rn) | FPType(vd) | SCVTF | Rn(rn) | Rd(vd));
3273   } else {
3274     Emit(SF(rn) | FPType(vd) | SCVTF_fixed | FPScale(64 - fbits) | Rn(rn) |
3275          Rd(vd));
3276   }
3277 }
3278 
3279 
ucvtf(const VRegister & vd,const Register & rn,int fbits)3280 void Assembler::ucvtf(const VRegister& vd, const Register& rn, int fbits) {
3281   VIXL_ASSERT(CPUHas(CPUFeatures::kFP));
3282   if (vd.Is1H()) VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
3283   VIXL_ASSERT(vd.Is1H() || vd.Is1S() || vd.Is1D());
3284   VIXL_ASSERT(fbits >= 0);
3285   if (fbits == 0) {
3286     Emit(SF(rn) | FPType(vd) | UCVTF | Rn(rn) | Rd(vd));
3287   } else {
3288     Emit(SF(rn) | FPType(vd) | UCVTF_fixed | FPScale(64 - fbits) | Rn(rn) |
3289          Rd(vd));
3290   }
3291 }
3292 
3293 
NEON3Same(const VRegister & vd,const VRegister & vn,const VRegister & vm,NEON3SameOp vop)3294 void Assembler::NEON3Same(const VRegister& vd,
3295                           const VRegister& vn,
3296                           const VRegister& vm,
3297                           NEON3SameOp vop) {
3298   VIXL_ASSERT(AreSameFormat(vd, vn, vm));
3299   VIXL_ASSERT(vd.IsVector() || !vd.IsQ());
3300 
3301   Instr format, op = vop;
3302   if (vd.IsScalar()) {
3303     op |= NEON_Q | NEONScalar;
3304     format = SFormat(vd);
3305   } else {
3306     format = VFormat(vd);
3307   }
3308 
3309   Emit(format | op | Rm(vm) | Rn(vn) | Rd(vd));
3310 }
3311 
3312 
NEONFP3Same(const VRegister & vd,const VRegister & vn,const VRegister & vm,Instr op)3313 void Assembler::NEONFP3Same(const VRegister& vd,
3314                             const VRegister& vn,
3315                             const VRegister& vm,
3316                             Instr op) {
3317   VIXL_ASSERT(AreSameFormat(vd, vn, vm));
3318   Emit(FPFormat(vd) | op | Rm(vm) | Rn(vn) | Rd(vd));
3319 }
3320 
3321 
NEON3SameFP16(const VRegister & vd,const VRegister & vn,const VRegister & vm,Instr op)3322 void Assembler::NEON3SameFP16(const VRegister& vd,
3323                               const VRegister& vn,
3324                               const VRegister& vm,
3325                               Instr op) {
3326   VIXL_ASSERT(AreSameFormat(vd, vn, vm));
3327   VIXL_ASSERT(vd.GetLaneSizeInBytes() == kHRegSizeInBytes);
3328   if (vd.Is8H()) op |= NEON_Q;
3329   Emit(op | Rm(vm) | Rn(vn) | Rd(vd));
3330 }
3331 
3332 
3333 // clang-format off
3334 #define NEON_FP2REGMISC_LIST(V)                                        \
3335   V(fabs,    NEON_FABS,    FABS,                FABS_h)                \
3336   V(fneg,    NEON_FNEG,    FNEG,                FNEG_h)                \
3337   V(fsqrt,   NEON_FSQRT,   FSQRT,               FSQRT_h)               \
3338   V(frintn,  NEON_FRINTN,  FRINTN,              FRINTN_h)              \
3339   V(frinta,  NEON_FRINTA,  FRINTA,              FRINTA_h)              \
3340   V(frintp,  NEON_FRINTP,  FRINTP,              FRINTP_h)              \
3341   V(frintm,  NEON_FRINTM,  FRINTM,              FRINTM_h)              \
3342   V(frintx,  NEON_FRINTX,  FRINTX,              FRINTX_h)              \
3343   V(frintz,  NEON_FRINTZ,  FRINTZ,              FRINTZ_h)              \
3344   V(frinti,  NEON_FRINTI,  FRINTI,              FRINTI_h)              \
3345   V(frsqrte, NEON_FRSQRTE, NEON_FRSQRTE_scalar, NEON_FRSQRTE_H_scalar) \
3346   V(frecpe,  NEON_FRECPE,  NEON_FRECPE_scalar,  NEON_FRECPE_H_scalar)
3347 // clang-format on
3348 
3349 #define VIXL_DEFINE_ASM_FUNC(FN, VEC_OP, SCA_OP, SCA_OP_H)                   \
3350   void Assembler::FN(const VRegister& vd, const VRegister& vn) {             \
3351     VIXL_ASSERT(CPUHas(CPUFeatures::kFP));                                   \
3352     Instr op;                                                                \
3353     if (vd.IsScalar()) {                                                     \
3354       if (vd.Is1H()) {                                                       \
3355         if ((SCA_OP_H & NEONScalar2RegMiscFP16FMask) ==                      \
3356             NEONScalar2RegMiscFP16Fixed) {                                   \
3357           VIXL_ASSERT(CPUHas(CPUFeatures::kNEON, CPUFeatures::kNEONHalf));   \
3358         } else {                                                             \
3359           VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));                         \
3360         }                                                                    \
3361         op = SCA_OP_H;                                                       \
3362       } else {                                                               \
3363         if ((SCA_OP & NEONScalar2RegMiscFMask) == NEONScalar2RegMiscFixed) { \
3364           VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));                           \
3365         }                                                                    \
3366         VIXL_ASSERT(vd.Is1S() || vd.Is1D());                                 \
3367         op = SCA_OP;                                                         \
3368       }                                                                      \
3369     } else {                                                                 \
3370       VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));                               \
3371       VIXL_ASSERT(vd.Is4H() || vd.Is8H() || vd.Is2S() || vd.Is2D() ||        \
3372                   vd.Is4S());                                                \
3373       if (vd.IsLaneSizeH()) {                                                \
3374         VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));                         \
3375         op = VEC_OP##_H;                                                     \
3376         if (vd.Is8H()) {                                                     \
3377           op |= NEON_Q;                                                      \
3378         }                                                                    \
3379       } else {                                                               \
3380         op = VEC_OP;                                                         \
3381       }                                                                      \
3382     }                                                                        \
3383     if (vd.IsLaneSizeH()) {                                                  \
3384       NEONFP2RegMiscFP16(vd, vn, op);                                        \
3385     } else {                                                                 \
3386       NEONFP2RegMisc(vd, vn, op);                                            \
3387     }                                                                        \
3388   }
3389 NEON_FP2REGMISC_LIST(VIXL_DEFINE_ASM_FUNC)
3390 #undef VIXL_DEFINE_ASM_FUNC
3391 
3392 // clang-format off
3393 #define NEON_FP2REGMISC_V85_LIST(V)       \
3394   V(frint32x,  NEON_FRINT32X,  FRINT32X)  \
3395   V(frint32z,  NEON_FRINT32Z,  FRINT32Z)  \
3396   V(frint64x,  NEON_FRINT64X,  FRINT64X)  \
3397   V(frint64z,  NEON_FRINT64Z,  FRINT64Z)
3398 // clang-format on
3399 
3400 #define VIXL_DEFINE_ASM_FUNC(FN, VEC_OP, SCA_OP)                               \
3401   void Assembler::FN(const VRegister& vd, const VRegister& vn) {               \
3402     VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kFrintToFixedSizedInt)); \
3403     Instr op;                                                                  \
3404     if (vd.IsScalar()) {                                                       \
3405       VIXL_ASSERT(vd.Is1S() || vd.Is1D());                                     \
3406       op = SCA_OP;                                                             \
3407     } else {                                                                   \
3408       VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));                                 \
3409       VIXL_ASSERT(vd.Is2S() || vd.Is2D() || vd.Is4S());                        \
3410       op = VEC_OP;                                                             \
3411     }                                                                          \
3412     NEONFP2RegMisc(vd, vn, op);                                                \
3413   }
NEON_FP2REGMISC_V85_LIST(VIXL_DEFINE_ASM_FUNC)3414 NEON_FP2REGMISC_V85_LIST(VIXL_DEFINE_ASM_FUNC)
3415 #undef VIXL_DEFINE_ASM_FUNC
3416 
3417 void Assembler::NEONFP2RegMiscFP16(const VRegister& vd,
3418                                    const VRegister& vn,
3419                                    Instr op) {
3420   VIXL_ASSERT(AreSameFormat(vd, vn));
3421   Emit(op | Rn(vn) | Rd(vd));
3422 }
3423 
3424 
NEONFP2RegMisc(const VRegister & vd,const VRegister & vn,Instr op)3425 void Assembler::NEONFP2RegMisc(const VRegister& vd,
3426                                const VRegister& vn,
3427                                Instr op) {
3428   VIXL_ASSERT(AreSameFormat(vd, vn));
3429   Emit(FPFormat(vd) | op | Rn(vn) | Rd(vd));
3430 }
3431 
3432 
NEON2RegMisc(const VRegister & vd,const VRegister & vn,NEON2RegMiscOp vop,int value)3433 void Assembler::NEON2RegMisc(const VRegister& vd,
3434                              const VRegister& vn,
3435                              NEON2RegMiscOp vop,
3436                              int value) {
3437   VIXL_ASSERT(AreSameFormat(vd, vn));
3438   VIXL_ASSERT(value == 0);
3439   USE(value);
3440 
3441   Instr format, op = vop;
3442   if (vd.IsScalar()) {
3443     op |= NEON_Q | NEONScalar;
3444     format = SFormat(vd);
3445   } else {
3446     format = VFormat(vd);
3447   }
3448 
3449   Emit(format | op | Rn(vn) | Rd(vd));
3450 }
3451 
3452 
cmeq(const VRegister & vd,const VRegister & vn,int value)3453 void Assembler::cmeq(const VRegister& vd, const VRegister& vn, int value) {
3454   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
3455   VIXL_ASSERT(vd.IsVector() || vd.Is1D());
3456   NEON2RegMisc(vd, vn, NEON_CMEQ_zero, value);
3457 }
3458 
3459 
cmge(const VRegister & vd,const VRegister & vn,int value)3460 void Assembler::cmge(const VRegister& vd, const VRegister& vn, int value) {
3461   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
3462   VIXL_ASSERT(vd.IsVector() || vd.Is1D());
3463   NEON2RegMisc(vd, vn, NEON_CMGE_zero, value);
3464 }
3465 
3466 
cmgt(const VRegister & vd,const VRegister & vn,int value)3467 void Assembler::cmgt(const VRegister& vd, const VRegister& vn, int value) {
3468   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
3469   VIXL_ASSERT(vd.IsVector() || vd.Is1D());
3470   NEON2RegMisc(vd, vn, NEON_CMGT_zero, value);
3471 }
3472 
3473 
cmle(const VRegister & vd,const VRegister & vn,int value)3474 void Assembler::cmle(const VRegister& vd, const VRegister& vn, int value) {
3475   VIXL_ASSERT(vd.IsVector() || vd.Is1D());
3476   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
3477   NEON2RegMisc(vd, vn, NEON_CMLE_zero, value);
3478 }
3479 
3480 
cmlt(const VRegister & vd,const VRegister & vn,int value)3481 void Assembler::cmlt(const VRegister& vd, const VRegister& vn, int value) {
3482   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
3483   VIXL_ASSERT(vd.IsVector() || vd.Is1D());
3484   NEON2RegMisc(vd, vn, NEON_CMLT_zero, value);
3485 }
3486 
3487 
shll(const VRegister & vd,const VRegister & vn,int shift)3488 void Assembler::shll(const VRegister& vd, const VRegister& vn, int shift) {
3489   USE(shift);
3490   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
3491   VIXL_ASSERT((vd.Is8H() && vn.Is8B() && shift == 8) ||
3492               (vd.Is4S() && vn.Is4H() && shift == 16) ||
3493               (vd.Is2D() && vn.Is2S() && shift == 32));
3494   Emit(VFormat(vn) | NEON_SHLL | Rn(vn) | Rd(vd));
3495 }
3496 
3497 
shll2(const VRegister & vd,const VRegister & vn,int shift)3498 void Assembler::shll2(const VRegister& vd, const VRegister& vn, int shift) {
3499   USE(shift);
3500   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
3501   VIXL_ASSERT((vd.Is8H() && vn.Is16B() && shift == 8) ||
3502               (vd.Is4S() && vn.Is8H() && shift == 16) ||
3503               (vd.Is2D() && vn.Is4S() && shift == 32));
3504   Emit(VFormat(vn) | NEON_SHLL | Rn(vn) | Rd(vd));
3505 }
3506 
3507 
NEONFP2RegMisc(const VRegister & vd,const VRegister & vn,NEON2RegMiscOp vop,double value)3508 void Assembler::NEONFP2RegMisc(const VRegister& vd,
3509                                const VRegister& vn,
3510                                NEON2RegMiscOp vop,
3511                                double value) {
3512   VIXL_ASSERT(AreSameFormat(vd, vn));
3513   VIXL_ASSERT(value == 0.0);
3514   USE(value);
3515 
3516   Instr op = vop;
3517   if (vd.IsScalar()) {
3518     VIXL_ASSERT(vd.Is1S() || vd.Is1D());
3519     op |= NEON_Q | NEONScalar;
3520   } else {
3521     VIXL_ASSERT(vd.Is2S() || vd.Is2D() || vd.Is4S());
3522   }
3523 
3524   Emit(FPFormat(vd) | op | Rn(vn) | Rd(vd));
3525 }
3526 
3527 
NEONFP2RegMiscFP16(const VRegister & vd,const VRegister & vn,NEON2RegMiscFP16Op vop,double value)3528 void Assembler::NEONFP2RegMiscFP16(const VRegister& vd,
3529                                    const VRegister& vn,
3530                                    NEON2RegMiscFP16Op vop,
3531                                    double value) {
3532   VIXL_ASSERT(AreSameFormat(vd, vn));
3533   VIXL_ASSERT(value == 0.0);
3534   USE(value);
3535 
3536   Instr op = vop;
3537   if (vd.IsScalar()) {
3538     VIXL_ASSERT(vd.Is1H());
3539     op |= NEON_Q | NEONScalar;
3540   } else {
3541     VIXL_ASSERT(vd.Is4H() || vd.Is8H());
3542     if (vd.Is8H()) {
3543       op |= NEON_Q;
3544     }
3545   }
3546 
3547   Emit(op | Rn(vn) | Rd(vd));
3548 }
3549 
3550 
fcmeq(const VRegister & vd,const VRegister & vn,double value)3551 void Assembler::fcmeq(const VRegister& vd, const VRegister& vn, double value) {
3552   VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
3553   if (vd.IsLaneSizeH()) {
3554     VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
3555     NEONFP2RegMiscFP16(vd, vn, NEON_FCMEQ_H_zero, value);
3556   } else {
3557     NEONFP2RegMisc(vd, vn, NEON_FCMEQ_zero, value);
3558   }
3559 }
3560 
3561 
fcmge(const VRegister & vd,const VRegister & vn,double value)3562 void Assembler::fcmge(const VRegister& vd, const VRegister& vn, double value) {
3563   VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
3564   if (vd.IsLaneSizeH()) {
3565     VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
3566     NEONFP2RegMiscFP16(vd, vn, NEON_FCMGE_H_zero, value);
3567   } else {
3568     NEONFP2RegMisc(vd, vn, NEON_FCMGE_zero, value);
3569   }
3570 }
3571 
3572 
fcmgt(const VRegister & vd,const VRegister & vn,double value)3573 void Assembler::fcmgt(const VRegister& vd, const VRegister& vn, double value) {
3574   VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
3575   if (vd.IsLaneSizeH()) {
3576     VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
3577     NEONFP2RegMiscFP16(vd, vn, NEON_FCMGT_H_zero, value);
3578   } else {
3579     NEONFP2RegMisc(vd, vn, NEON_FCMGT_zero, value);
3580   }
3581 }
3582 
3583 
fcmle(const VRegister & vd,const VRegister & vn,double value)3584 void Assembler::fcmle(const VRegister& vd, const VRegister& vn, double value) {
3585   VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
3586   if (vd.IsLaneSizeH()) {
3587     VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
3588     NEONFP2RegMiscFP16(vd, vn, NEON_FCMLE_H_zero, value);
3589   } else {
3590     NEONFP2RegMisc(vd, vn, NEON_FCMLE_zero, value);
3591   }
3592 }
3593 
3594 
fcmlt(const VRegister & vd,const VRegister & vn,double value)3595 void Assembler::fcmlt(const VRegister& vd, const VRegister& vn, double value) {
3596   VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
3597   if (vd.IsLaneSizeH()) {
3598     VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
3599     NEONFP2RegMiscFP16(vd, vn, NEON_FCMLT_H_zero, value);
3600   } else {
3601     NEONFP2RegMisc(vd, vn, NEON_FCMLT_zero, value);
3602   }
3603 }
3604 
3605 
frecpx(const VRegister & vd,const VRegister & vn)3606 void Assembler::frecpx(const VRegister& vd, const VRegister& vn) {
3607   VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
3608   VIXL_ASSERT(vd.IsScalar());
3609   VIXL_ASSERT(AreSameFormat(vd, vn));
3610   Instr op;
3611   if (vd.Is1H()) {
3612     VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
3613     op = NEON_FRECPX_H_scalar;
3614   } else {
3615     VIXL_ASSERT(vd.Is1S() || vd.Is1D());
3616     op = NEON_FRECPX_scalar;
3617   }
3618   Emit(FPFormat(vd) | op | Rn(vn) | Rd(vd));
3619 }
3620 
3621 
3622 // clang-format off
3623 #define NEON_3SAME_LIST(V) \
3624   V(add,      NEON_ADD,      vd.IsVector() || vd.Is1D())            \
3625   V(addp,     NEON_ADDP,     vd.IsVector() || vd.Is1D())            \
3626   V(sub,      NEON_SUB,      vd.IsVector() || vd.Is1D())            \
3627   V(cmeq,     NEON_CMEQ,     vd.IsVector() || vd.Is1D())            \
3628   V(cmge,     NEON_CMGE,     vd.IsVector() || vd.Is1D())            \
3629   V(cmgt,     NEON_CMGT,     vd.IsVector() || vd.Is1D())            \
3630   V(cmhi,     NEON_CMHI,     vd.IsVector() || vd.Is1D())            \
3631   V(cmhs,     NEON_CMHS,     vd.IsVector() || vd.Is1D())            \
3632   V(cmtst,    NEON_CMTST,    vd.IsVector() || vd.Is1D())            \
3633   V(sshl,     NEON_SSHL,     vd.IsVector() || vd.Is1D())            \
3634   V(ushl,     NEON_USHL,     vd.IsVector() || vd.Is1D())            \
3635   V(srshl,    NEON_SRSHL,    vd.IsVector() || vd.Is1D())            \
3636   V(urshl,    NEON_URSHL,    vd.IsVector() || vd.Is1D())            \
3637   V(sqdmulh,  NEON_SQDMULH,  vd.IsLaneSizeH() || vd.IsLaneSizeS())  \
3638   V(sqrdmulh, NEON_SQRDMULH, vd.IsLaneSizeH() || vd.IsLaneSizeS())  \
3639   V(shadd,    NEON_SHADD,    vd.IsVector() && !vd.IsLaneSizeD())    \
3640   V(uhadd,    NEON_UHADD,    vd.IsVector() && !vd.IsLaneSizeD())    \
3641   V(srhadd,   NEON_SRHADD,   vd.IsVector() && !vd.IsLaneSizeD())    \
3642   V(urhadd,   NEON_URHADD,   vd.IsVector() && !vd.IsLaneSizeD())    \
3643   V(shsub,    NEON_SHSUB,    vd.IsVector() && !vd.IsLaneSizeD())    \
3644   V(uhsub,    NEON_UHSUB,    vd.IsVector() && !vd.IsLaneSizeD())    \
3645   V(smax,     NEON_SMAX,     vd.IsVector() && !vd.IsLaneSizeD())    \
3646   V(smaxp,    NEON_SMAXP,    vd.IsVector() && !vd.IsLaneSizeD())    \
3647   V(smin,     NEON_SMIN,     vd.IsVector() && !vd.IsLaneSizeD())    \
3648   V(sminp,    NEON_SMINP,    vd.IsVector() && !vd.IsLaneSizeD())    \
3649   V(umax,     NEON_UMAX,     vd.IsVector() && !vd.IsLaneSizeD())    \
3650   V(umaxp,    NEON_UMAXP,    vd.IsVector() && !vd.IsLaneSizeD())    \
3651   V(umin,     NEON_UMIN,     vd.IsVector() && !vd.IsLaneSizeD())    \
3652   V(uminp,    NEON_UMINP,    vd.IsVector() && !vd.IsLaneSizeD())    \
3653   V(saba,     NEON_SABA,     vd.IsVector() && !vd.IsLaneSizeD())    \
3654   V(sabd,     NEON_SABD,     vd.IsVector() && !vd.IsLaneSizeD())    \
3655   V(uaba,     NEON_UABA,     vd.IsVector() && !vd.IsLaneSizeD())    \
3656   V(uabd,     NEON_UABD,     vd.IsVector() && !vd.IsLaneSizeD())    \
3657   V(mla,      NEON_MLA,      vd.IsVector() && !vd.IsLaneSizeD())    \
3658   V(mls,      NEON_MLS,      vd.IsVector() && !vd.IsLaneSizeD())    \
3659   V(mul,      NEON_MUL,      vd.IsVector() && !vd.IsLaneSizeD())    \
3660   V(and_,     NEON_AND,      vd.Is8B() || vd.Is16B())               \
3661   V(orr,      NEON_ORR,      vd.Is8B() || vd.Is16B())               \
3662   V(orn,      NEON_ORN,      vd.Is8B() || vd.Is16B())               \
3663   V(eor,      NEON_EOR,      vd.Is8B() || vd.Is16B())               \
3664   V(bic,      NEON_BIC,      vd.Is8B() || vd.Is16B())               \
3665   V(bit,      NEON_BIT,      vd.Is8B() || vd.Is16B())               \
3666   V(bif,      NEON_BIF,      vd.Is8B() || vd.Is16B())               \
3667   V(bsl,      NEON_BSL,      vd.Is8B() || vd.Is16B())               \
3668   V(pmul,     NEON_PMUL,     vd.Is8B() || vd.Is16B())               \
3669   V(uqadd,    NEON_UQADD,    true)                                  \
3670   V(sqadd,    NEON_SQADD,    true)                                  \
3671   V(uqsub,    NEON_UQSUB,    true)                                  \
3672   V(sqsub,    NEON_SQSUB,    true)                                  \
3673   V(sqshl,    NEON_SQSHL,    true)                                  \
3674   V(uqshl,    NEON_UQSHL,    true)                                  \
3675   V(sqrshl,   NEON_SQRSHL,   true)                                  \
3676   V(uqrshl,   NEON_UQRSHL,   true)
3677 // clang-format on
3678 
3679 #define VIXL_DEFINE_ASM_FUNC(FN, OP, AS)     \
3680   void Assembler::FN(const VRegister& vd,    \
3681                      const VRegister& vn,    \
3682                      const VRegister& vm) {  \
3683     VIXL_ASSERT(CPUHas(CPUFeatures::kNEON)); \
3684     VIXL_ASSERT(AS);                         \
3685     NEON3Same(vd, vn, vm, OP);               \
3686   }
3687 NEON_3SAME_LIST(VIXL_DEFINE_ASM_FUNC)
3688 #undef VIXL_DEFINE_ASM_FUNC
3689 
3690 // clang-format off
3691 #define NEON_FP3SAME_OP_LIST(V)                                        \
3692   V(fmulx,   NEON_FMULX,   NEON_FMULX_scalar,   NEON_FMULX_H_scalar)   \
3693   V(frecps,  NEON_FRECPS,  NEON_FRECPS_scalar,  NEON_FRECPS_H_scalar)  \
3694   V(frsqrts, NEON_FRSQRTS, NEON_FRSQRTS_scalar, NEON_FRSQRTS_H_scalar) \
3695   V(fabd,    NEON_FABD,    NEON_FABD_scalar,    NEON_FABD_H_scalar)    \
3696   V(fmla,    NEON_FMLA,    0,                   0)                     \
3697   V(fmls,    NEON_FMLS,    0,                   0)                     \
3698   V(facge,   NEON_FACGE,   NEON_FACGE_scalar,   NEON_FACGE_H_scalar)   \
3699   V(facgt,   NEON_FACGT,   NEON_FACGT_scalar,   NEON_FACGT_H_scalar)   \
3700   V(fcmeq,   NEON_FCMEQ,   NEON_FCMEQ_scalar,   NEON_FCMEQ_H_scalar)   \
3701   V(fcmge,   NEON_FCMGE,   NEON_FCMGE_scalar,   NEON_FCMGE_H_scalar)   \
3702   V(fcmgt,   NEON_FCMGT,   NEON_FCMGT_scalar,   NEON_FCMGT_H_scalar)   \
3703   V(faddp,   NEON_FADDP,   0,                   0)                     \
3704   V(fmaxp,   NEON_FMAXP,   0,                   0)                     \
3705   V(fminp,   NEON_FMINP,   0,                   0)                     \
3706   V(fmaxnmp, NEON_FMAXNMP, 0,                   0)                     \
3707   V(fadd,    NEON_FADD,    FADD,                0)                     \
3708   V(fsub,    NEON_FSUB,    FSUB,                0)                     \
3709   V(fmul,    NEON_FMUL,    FMUL,                0)                     \
3710   V(fdiv,    NEON_FDIV,    FDIV,                0)                     \
3711   V(fmax,    NEON_FMAX,    FMAX,                0)                     \
3712   V(fmin,    NEON_FMIN,    FMIN,                0)                     \
3713   V(fmaxnm,  NEON_FMAXNM,  FMAXNM,              0)                     \
3714   V(fminnm,  NEON_FMINNM,  FMINNM,              0)                     \
3715   V(fminnmp, NEON_FMINNMP, 0,                   0)
3716 // clang-format on
3717 
3718 // TODO: This macro is complicated because it classifies the instructions in the
3719 // macro list above, and treats each case differently. It could be somewhat
3720 // simpler if we were to split the macro, at the cost of some duplication.
3721 #define VIXL_DEFINE_ASM_FUNC(FN, VEC_OP, SCA_OP, SCA_OP_H)               \
3722   void Assembler::FN(const VRegister& vd,                                \
3723                      const VRegister& vn,                                \
3724                      const VRegister& vm) {                              \
3725     VIXL_ASSERT(CPUHas(CPUFeatures::kFP));                               \
3726     Instr op;                                                            \
3727     bool is_fp16 = false;                                                \
3728     if ((SCA_OP != 0) && vd.IsScalar()) {                                \
3729       if ((SCA_OP_H != 0) && vd.Is1H()) {                                \
3730         VIXL_ASSERT(CPUHas(CPUFeatures::kNEON, CPUFeatures::kNEONHalf)); \
3731         is_fp16 = true;                                                  \
3732         op = SCA_OP_H;                                                   \
3733       } else {                                                           \
3734         VIXL_ASSERT(vd.Is1H() || vd.Is1S() || vd.Is1D());                \
3735         if ((SCA_OP & NEONScalar3SameFMask) == NEONScalar3SameFixed) {   \
3736           VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));                       \
3737           if (vd.Is1H()) VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));    \
3738         } else if (vd.Is1H()) {                                          \
3739           VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));                     \
3740         }                                                                \
3741         op = SCA_OP;                                                     \
3742       }                                                                  \
3743     } else {                                                             \
3744       VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));                           \
3745       VIXL_ASSERT(vd.IsVector());                                        \
3746       if (vd.Is4H() || vd.Is8H()) {                                      \
3747         VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));                     \
3748         is_fp16 = true;                                                  \
3749         op = VEC_OP##_H;                                                 \
3750       } else {                                                           \
3751         VIXL_ASSERT(vd.Is2S() || vd.Is2D() || vd.Is4S());                \
3752         op = VEC_OP;                                                     \
3753       }                                                                  \
3754     }                                                                    \
3755     if (is_fp16) {                                                       \
3756       NEON3SameFP16(vd, vn, vm, op);                                     \
3757     } else {                                                             \
3758       NEONFP3Same(vd, vn, vm, op);                                       \
3759     }                                                                    \
3760   }
NEON_FP3SAME_OP_LIST(VIXL_DEFINE_ASM_FUNC)3761 NEON_FP3SAME_OP_LIST(VIXL_DEFINE_ASM_FUNC)
3762 #undef VIXL_DEFINE_ASM_FUNC
3763 
3764 
3765 // clang-format off
3766 #define NEON_FHM_LIST(V) \
3767   V(fmlal,   NEON_FMLAL)   \
3768   V(fmlal2,  NEON_FMLAL2)  \
3769   V(fmlsl,   NEON_FMLSL)   \
3770   V(fmlsl2,  NEON_FMLSL2)
3771 // clang-format on
3772 
3773 #define VIXL_DEFINE_ASM_FUNC(FN, VEC_OP)                    \
3774   void Assembler::FN(const VRegister& vd,                   \
3775                      const VRegister& vn,                   \
3776                      const VRegister& vm) {                 \
3777     VIXL_ASSERT(CPUHas(CPUFeatures::kNEON,                  \
3778                        CPUFeatures::kFP,                    \
3779                        CPUFeatures::kNEONHalf,              \
3780                        CPUFeatures::kFHM));                 \
3781     VIXL_ASSERT((vd.Is2S() && vn.Is2H() && vm.Is2H()) ||    \
3782                 (vd.Is4S() && vn.Is4H() && vm.Is4H()));     \
3783     Emit(FPFormat(vd) | VEC_OP | Rm(vm) | Rn(vn) | Rd(vd)); \
3784   }
3785 NEON_FHM_LIST(VIXL_DEFINE_ASM_FUNC)
3786 #undef VIXL_DEFINE_ASM_FUNC
3787 
3788 
3789 void Assembler::addp(const VRegister& vd, const VRegister& vn) {
3790   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
3791   VIXL_ASSERT((vd.Is1D() && vn.Is2D()));
3792   Emit(SFormat(vd) | NEON_ADDP_scalar | Rn(vn) | Rd(vd));
3793 }
3794 
3795 
sqrdmlah(const VRegister & vd,const VRegister & vn,const VRegister & vm)3796 void Assembler::sqrdmlah(const VRegister& vd,
3797                          const VRegister& vn,
3798                          const VRegister& vm) {
3799   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON, CPUFeatures::kRDM));
3800   VIXL_ASSERT(AreSameFormat(vd, vn, vm));
3801   VIXL_ASSERT(vd.IsVector() || !vd.IsQ());
3802 
3803   Instr format, op = NEON_SQRDMLAH;
3804   if (vd.IsScalar()) {
3805     op |= NEON_Q | NEONScalar;
3806     format = SFormat(vd);
3807   } else {
3808     format = VFormat(vd);
3809   }
3810 
3811   Emit(format | op | Rm(vm) | Rn(vn) | Rd(vd));
3812 }
3813 
3814 
sqrdmlsh(const VRegister & vd,const VRegister & vn,const VRegister & vm)3815 void Assembler::sqrdmlsh(const VRegister& vd,
3816                          const VRegister& vn,
3817                          const VRegister& vm) {
3818   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON, CPUFeatures::kRDM));
3819   VIXL_ASSERT(AreSameFormat(vd, vn, vm));
3820   VIXL_ASSERT(vd.IsVector() || !vd.IsQ());
3821 
3822   Instr format, op = NEON_SQRDMLSH;
3823   if (vd.IsScalar()) {
3824     op |= NEON_Q | NEONScalar;
3825     format = SFormat(vd);
3826   } else {
3827     format = VFormat(vd);
3828   }
3829 
3830   Emit(format | op | Rm(vm) | Rn(vn) | Rd(vd));
3831 }
3832 
3833 
sdot(const VRegister & vd,const VRegister & vn,const VRegister & vm)3834 void Assembler::sdot(const VRegister& vd,
3835                      const VRegister& vn,
3836                      const VRegister& vm) {
3837   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON, CPUFeatures::kDotProduct));
3838   VIXL_ASSERT(AreSameFormat(vn, vm));
3839   VIXL_ASSERT((vd.Is2S() && vn.Is8B()) || (vd.Is4S() && vn.Is16B()));
3840 
3841   Emit(VFormat(vd) | NEON_SDOT | Rm(vm) | Rn(vn) | Rd(vd));
3842 }
3843 
3844 
udot(const VRegister & vd,const VRegister & vn,const VRegister & vm)3845 void Assembler::udot(const VRegister& vd,
3846                      const VRegister& vn,
3847                      const VRegister& vm) {
3848   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON, CPUFeatures::kDotProduct));
3849   VIXL_ASSERT(AreSameFormat(vn, vm));
3850   VIXL_ASSERT((vd.Is2S() && vn.Is8B()) || (vd.Is4S() && vn.Is16B()));
3851 
3852   Emit(VFormat(vd) | NEON_UDOT | Rm(vm) | Rn(vn) | Rd(vd));
3853 }
3854 
usdot(const VRegister & vd,const VRegister & vn,const VRegister & vm)3855 void Assembler::usdot(const VRegister& vd,
3856                       const VRegister& vn,
3857                       const VRegister& vm) {
3858   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON, CPUFeatures::kI8MM));
3859   VIXL_ASSERT(AreSameFormat(vn, vm));
3860   VIXL_ASSERT((vd.Is2S() && vn.Is8B()) || (vd.Is4S() && vn.Is16B()));
3861 
3862   Emit(VFormat(vd) | 0x0e809c00 | Rm(vm) | Rn(vn) | Rd(vd));
3863 }
3864 
faddp(const VRegister & vd,const VRegister & vn)3865 void Assembler::faddp(const VRegister& vd, const VRegister& vn) {
3866   VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
3867   VIXL_ASSERT((vd.Is1S() && vn.Is2S()) || (vd.Is1D() && vn.Is2D()) ||
3868               (vd.Is1H() && vn.Is2H()));
3869   if (vd.Is1H()) {
3870     VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
3871     Emit(NEON_FADDP_h_scalar | Rn(vn) | Rd(vd));
3872   } else {
3873     Emit(FPFormat(vd) | NEON_FADDP_scalar | Rn(vn) | Rd(vd));
3874   }
3875 }
3876 
3877 
fmaxp(const VRegister & vd,const VRegister & vn)3878 void Assembler::fmaxp(const VRegister& vd, const VRegister& vn) {
3879   VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
3880   VIXL_ASSERT((vd.Is1S() && vn.Is2S()) || (vd.Is1D() && vn.Is2D()) ||
3881               (vd.Is1H() && vn.Is2H()));
3882   if (vd.Is1H()) {
3883     VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
3884     Emit(NEON_FMAXP_h_scalar | Rn(vn) | Rd(vd));
3885   } else {
3886     Emit(FPFormat(vd) | NEON_FMAXP_scalar | Rn(vn) | Rd(vd));
3887   }
3888 }
3889 
3890 
fminp(const VRegister & vd,const VRegister & vn)3891 void Assembler::fminp(const VRegister& vd, const VRegister& vn) {
3892   VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
3893   VIXL_ASSERT((vd.Is1S() && vn.Is2S()) || (vd.Is1D() && vn.Is2D()) ||
3894               (vd.Is1H() && vn.Is2H()));
3895   if (vd.Is1H()) {
3896     VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
3897     Emit(NEON_FMINP_h_scalar | Rn(vn) | Rd(vd));
3898   } else {
3899     Emit(FPFormat(vd) | NEON_FMINP_scalar | Rn(vn) | Rd(vd));
3900   }
3901 }
3902 
3903 
fmaxnmp(const VRegister & vd,const VRegister & vn)3904 void Assembler::fmaxnmp(const VRegister& vd, const VRegister& vn) {
3905   VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
3906   VIXL_ASSERT((vd.Is1S() && vn.Is2S()) || (vd.Is1D() && vn.Is2D()) ||
3907               (vd.Is1H() && vn.Is2H()));
3908   if (vd.Is1H()) {
3909     VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
3910     Emit(NEON_FMAXNMP_h_scalar | Rn(vn) | Rd(vd));
3911   } else {
3912     Emit(FPFormat(vd) | NEON_FMAXNMP_scalar | Rn(vn) | Rd(vd));
3913   }
3914 }
3915 
3916 
fminnmp(const VRegister & vd,const VRegister & vn)3917 void Assembler::fminnmp(const VRegister& vd, const VRegister& vn) {
3918   VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));
3919   VIXL_ASSERT((vd.Is1S() && vn.Is2S()) || (vd.Is1D() && vn.Is2D()) ||
3920               (vd.Is1H() && vn.Is2H()));
3921   if (vd.Is1H()) {
3922     VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
3923     Emit(NEON_FMINNMP_h_scalar | Rn(vn) | Rd(vd));
3924   } else {
3925     Emit(FPFormat(vd) | NEON_FMINNMP_scalar | Rn(vn) | Rd(vd));
3926   }
3927 }
3928 
3929 
3930 // v8.3 complex numbers - floating-point complex multiply accumulate.
fcmla(const VRegister & vd,const VRegister & vn,const VRegister & vm,int vm_index,int rot)3931 void Assembler::fcmla(const VRegister& vd,
3932                       const VRegister& vn,
3933                       const VRegister& vm,
3934                       int vm_index,
3935                       int rot) {
3936   VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON, CPUFeatures::kFcma));
3937   VIXL_ASSERT(vd.IsVector() && AreSameFormat(vd, vn));
3938   VIXL_ASSERT((vm.IsH() && (vd.Is8H() || vd.Is4H())) ||
3939               (vm.IsS() && vd.Is4S()));
3940   if (vd.IsLaneSizeH()) VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
3941   int index_num_bits = vd.Is4S() ? 1 : 2;
3942   Emit(VFormat(vd) | Rm(vm) | NEON_FCMLA_byelement |
3943        ImmNEONHLM(vm_index, index_num_bits) | ImmRotFcmlaSca(rot) | Rn(vn) |
3944        Rd(vd));
3945 }
3946 
3947 
fcmla(const VRegister & vd,const VRegister & vn,const VRegister & vm,int rot)3948 void Assembler::fcmla(const VRegister& vd,
3949                       const VRegister& vn,
3950                       const VRegister& vm,
3951                       int rot) {
3952   VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON, CPUFeatures::kFcma));
3953   VIXL_ASSERT(AreSameFormat(vd, vn, vm));
3954   VIXL_ASSERT(vd.IsVector() && !vd.IsLaneSizeB());
3955   if (vd.IsLaneSizeH()) VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
3956   Emit(VFormat(vd) | Rm(vm) | NEON_FCMLA | ImmRotFcmlaVec(rot) | Rn(vn) |
3957        Rd(vd));
3958 }
3959 
3960 
3961 // v8.3 complex numbers - floating-point complex add.
fcadd(const VRegister & vd,const VRegister & vn,const VRegister & vm,int rot)3962 void Assembler::fcadd(const VRegister& vd,
3963                       const VRegister& vn,
3964                       const VRegister& vm,
3965                       int rot) {
3966   VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON, CPUFeatures::kFcma));
3967   VIXL_ASSERT(AreSameFormat(vd, vn, vm));
3968   VIXL_ASSERT(vd.IsVector() && !vd.IsLaneSizeB());
3969   if (vd.IsLaneSizeH()) VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
3970   Emit(VFormat(vd) | Rm(vm) | NEON_FCADD | ImmRotFcadd(rot) | Rn(vn) | Rd(vd));
3971 }
3972 
3973 
orr(const VRegister & vd,const int imm8,const int left_shift)3974 void Assembler::orr(const VRegister& vd, const int imm8, const int left_shift) {
3975   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
3976   NEONModifiedImmShiftLsl(vd, imm8, left_shift, NEONModifiedImmediate_ORR);
3977 }
3978 
3979 
mov(const VRegister & vd,const VRegister & vn)3980 void Assembler::mov(const VRegister& vd, const VRegister& vn) {
3981   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
3982   VIXL_ASSERT(AreSameFormat(vd, vn));
3983   if (vd.IsD()) {
3984     orr(vd.V8B(), vn.V8B(), vn.V8B());
3985   } else {
3986     VIXL_ASSERT(vd.IsQ());
3987     orr(vd.V16B(), vn.V16B(), vn.V16B());
3988   }
3989 }
3990 
3991 
bic(const VRegister & vd,const int imm8,const int left_shift)3992 void Assembler::bic(const VRegister& vd, const int imm8, const int left_shift) {
3993   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
3994   NEONModifiedImmShiftLsl(vd, imm8, left_shift, NEONModifiedImmediate_BIC);
3995 }
3996 
3997 
movi(const VRegister & vd,const uint64_t imm,Shift shift,const int shift_amount)3998 void Assembler::movi(const VRegister& vd,
3999                      const uint64_t imm,
4000                      Shift shift,
4001                      const int shift_amount) {
4002   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4003   VIXL_ASSERT((shift == LSL) || (shift == MSL));
4004   if (vd.Is2D() || vd.Is1D()) {
4005     VIXL_ASSERT(shift_amount == 0);
4006     int imm8 = 0;
4007     for (int i = 0; i < 8; ++i) {
4008       int byte = (imm >> (i * 8)) & 0xff;
4009       VIXL_ASSERT((byte == 0) || (byte == 0xff));
4010       if (byte == 0xff) {
4011         imm8 |= (1 << i);
4012       }
4013     }
4014     int q = vd.Is2D() ? NEON_Q : 0;
4015     Emit(q | NEONModImmOp(1) | NEONModifiedImmediate_MOVI |
4016          ImmNEONabcdefgh(imm8) | NEONCmode(0xe) | Rd(vd));
4017   } else if (shift == LSL) {
4018     VIXL_ASSERT(IsUint8(imm));
4019     NEONModifiedImmShiftLsl(vd,
4020                             static_cast<int>(imm),
4021                             shift_amount,
4022                             NEONModifiedImmediate_MOVI);
4023   } else {
4024     VIXL_ASSERT(IsUint8(imm));
4025     NEONModifiedImmShiftMsl(vd,
4026                             static_cast<int>(imm),
4027                             shift_amount,
4028                             NEONModifiedImmediate_MOVI);
4029   }
4030 }
4031 
4032 
mvn(const VRegister & vd,const VRegister & vn)4033 void Assembler::mvn(const VRegister& vd, const VRegister& vn) {
4034   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4035   VIXL_ASSERT(AreSameFormat(vd, vn));
4036   if (vd.IsD()) {
4037     not_(vd.V8B(), vn.V8B());
4038   } else {
4039     VIXL_ASSERT(vd.IsQ());
4040     not_(vd.V16B(), vn.V16B());
4041   }
4042 }
4043 
4044 
mvni(const VRegister & vd,const int imm8,Shift shift,const int shift_amount)4045 void Assembler::mvni(const VRegister& vd,
4046                      const int imm8,
4047                      Shift shift,
4048                      const int shift_amount) {
4049   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4050   VIXL_ASSERT((shift == LSL) || (shift == MSL));
4051   if (shift == LSL) {
4052     NEONModifiedImmShiftLsl(vd, imm8, shift_amount, NEONModifiedImmediate_MVNI);
4053   } else {
4054     NEONModifiedImmShiftMsl(vd, imm8, shift_amount, NEONModifiedImmediate_MVNI);
4055   }
4056 }
4057 
4058 
NEONFPByElement(const VRegister & vd,const VRegister & vn,const VRegister & vm,int vm_index,NEONByIndexedElementOp vop,NEONByIndexedElementOp vop_half)4059 void Assembler::NEONFPByElement(const VRegister& vd,
4060                                 const VRegister& vn,
4061                                 const VRegister& vm,
4062                                 int vm_index,
4063                                 NEONByIndexedElementOp vop,
4064                                 NEONByIndexedElementOp vop_half) {
4065   VIXL_ASSERT(AreSameFormat(vd, vn));
4066   VIXL_ASSERT((vd.Is2S() && vm.Is1S()) || (vd.Is4S() && vm.Is1S()) ||
4067               (vd.Is1S() && vm.Is1S()) || (vd.Is2D() && vm.Is1D()) ||
4068               (vd.Is1D() && vm.Is1D()) || (vd.Is4H() && vm.Is1H()) ||
4069               (vd.Is8H() && vm.Is1H()) || (vd.Is1H() && vm.Is1H()));
4070   VIXL_ASSERT((vm.Is1S() && (vm_index < 4)) || (vm.Is1D() && (vm_index < 2)) ||
4071               (vm.Is1H() && (vm.GetCode() < 16) && (vm_index < 8)));
4072 
4073   Instr op = vop;
4074   int index_num_bits;
4075   if (vm.Is1D()) {
4076     index_num_bits = 1;
4077   } else if (vm.Is1S()) {
4078     index_num_bits = 2;
4079   } else {
4080     index_num_bits = 3;
4081     op = vop_half;
4082   }
4083 
4084   if (vd.IsScalar()) {
4085     op |= NEON_Q | NEONScalar;
4086   }
4087 
4088   if (!vm.Is1H()) {
4089     op |= FPFormat(vd);
4090   } else if (vd.Is8H()) {
4091     op |= NEON_Q;
4092   }
4093 
4094   Emit(op | ImmNEONHLM(vm_index, index_num_bits) | Rm(vm) | Rn(vn) | Rd(vd));
4095 }
4096 
4097 
NEONByElement(const VRegister & vd,const VRegister & vn,const VRegister & vm,int vm_index,NEONByIndexedElementOp vop)4098 void Assembler::NEONByElement(const VRegister& vd,
4099                               const VRegister& vn,
4100                               const VRegister& vm,
4101                               int vm_index,
4102                               NEONByIndexedElementOp vop) {
4103   VIXL_ASSERT(AreSameFormat(vd, vn));
4104   VIXL_ASSERT((vd.Is4H() && vm.Is1H()) || (vd.Is8H() && vm.Is1H()) ||
4105               (vd.Is1H() && vm.Is1H()) || (vd.Is2S() && vm.Is1S()) ||
4106               (vd.Is4S() && vm.Is1S()) || (vd.Is1S() && vm.Is1S()));
4107   VIXL_ASSERT((vm.Is1H() && (vm.GetCode() < 16) && (vm_index < 8)) ||
4108               (vm.Is1S() && (vm_index < 4)));
4109 
4110   Instr format, op = vop;
4111   int index_num_bits = vm.Is1H() ? 3 : 2;
4112   if (vd.IsScalar()) {
4113     op |= NEONScalar | NEON_Q;
4114     format = SFormat(vn);
4115   } else {
4116     format = VFormat(vn);
4117   }
4118   Emit(format | op | ImmNEONHLM(vm_index, index_num_bits) | Rm(vm) | Rn(vn) |
4119        Rd(vd));
4120 }
4121 
4122 
NEONByElementL(const VRegister & vd,const VRegister & vn,const VRegister & vm,int vm_index,NEONByIndexedElementOp vop)4123 void Assembler::NEONByElementL(const VRegister& vd,
4124                                const VRegister& vn,
4125                                const VRegister& vm,
4126                                int vm_index,
4127                                NEONByIndexedElementOp vop) {
4128   VIXL_ASSERT((vd.Is4S() && vn.Is4H() && vm.Is1H()) ||
4129               (vd.Is4S() && vn.Is8H() && vm.Is1H()) ||
4130               (vd.Is1S() && vn.Is1H() && vm.Is1H()) ||
4131               (vd.Is2D() && vn.Is2S() && vm.Is1S()) ||
4132               (vd.Is2D() && vn.Is4S() && vm.Is1S()) ||
4133               (vd.Is1D() && vn.Is1S() && vm.Is1S()));
4134 
4135   VIXL_ASSERT((vm.Is1H() && (vm.GetCode() < 16) && (vm_index < 8)) ||
4136               (vm.Is1S() && (vm_index < 4)));
4137 
4138   Instr format, op = vop;
4139   int index_num_bits = vm.Is1H() ? 3 : 2;
4140   if (vd.IsScalar()) {
4141     op |= NEONScalar | NEON_Q;
4142     format = SFormat(vn);
4143   } else {
4144     format = VFormat(vn);
4145   }
4146   Emit(format | op | ImmNEONHLM(vm_index, index_num_bits) | Rm(vm) | Rn(vn) |
4147        Rd(vd));
4148 }
4149 
4150 
sdot(const VRegister & vd,const VRegister & vn,const VRegister & vm,int vm_index)4151 void Assembler::sdot(const VRegister& vd,
4152                      const VRegister& vn,
4153                      const VRegister& vm,
4154                      int vm_index) {
4155   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON, CPUFeatures::kDotProduct));
4156   VIXL_ASSERT((vd.Is2S() && vn.Is8B() && vm.Is1S4B()) ||
4157               (vd.Is4S() && vn.Is16B() && vm.Is1S4B()));
4158 
4159   int index_num_bits = 2;
4160   Emit(VFormat(vd) | NEON_SDOT_byelement |
4161        ImmNEONHLM(vm_index, index_num_bits) | Rm(vm) | Rn(vn) | Rd(vd));
4162 }
4163 
4164 
udot(const VRegister & vd,const VRegister & vn,const VRegister & vm,int vm_index)4165 void Assembler::udot(const VRegister& vd,
4166                      const VRegister& vn,
4167                      const VRegister& vm,
4168                      int vm_index) {
4169   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON, CPUFeatures::kDotProduct));
4170   VIXL_ASSERT((vd.Is2S() && vn.Is8B() && vm.Is1S4B()) ||
4171               (vd.Is4S() && vn.Is16B() && vm.Is1S4B()));
4172 
4173   int index_num_bits = 2;
4174   Emit(VFormat(vd) | NEON_UDOT_byelement |
4175        ImmNEONHLM(vm_index, index_num_bits) | Rm(vm) | Rn(vn) | Rd(vd));
4176 }
4177 
sudot(const VRegister & vd,const VRegister & vn,const VRegister & vm,int vm_index)4178 void Assembler::sudot(const VRegister& vd,
4179                       const VRegister& vn,
4180                       const VRegister& vm,
4181                       int vm_index) {
4182   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON, CPUFeatures::kI8MM));
4183   VIXL_ASSERT((vd.Is2S() && vn.Is8B() && vm.Is1S4B()) ||
4184               (vd.Is4S() && vn.Is16B() && vm.Is1S4B()));
4185   int q = vd.Is4S() ? (1U << NEONQ_offset) : 0;
4186   int index_num_bits = 2;
4187   Emit(q | 0x0f00f000 | ImmNEONHLM(vm_index, index_num_bits) | Rm(vm) | Rn(vn) |
4188        Rd(vd));
4189 }
4190 
4191 
usdot(const VRegister & vd,const VRegister & vn,const VRegister & vm,int vm_index)4192 void Assembler::usdot(const VRegister& vd,
4193                       const VRegister& vn,
4194                       const VRegister& vm,
4195                       int vm_index) {
4196   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON, CPUFeatures::kI8MM));
4197   VIXL_ASSERT((vd.Is2S() && vn.Is8B() && vm.Is1S4B()) ||
4198               (vd.Is4S() && vn.Is16B() && vm.Is1S4B()));
4199   int q = vd.Is4S() ? (1U << NEONQ_offset) : 0;
4200   int index_num_bits = 2;
4201   Emit(q | 0x0f80f000 | ImmNEONHLM(vm_index, index_num_bits) | Rm(vm) | Rn(vn) |
4202        Rd(vd));
4203 }
4204 
4205 // clang-format off
4206 #define NEON_BYELEMENT_LIST(V)                        \
4207   V(mul,      NEON_MUL_byelement,      vn.IsVector()) \
4208   V(mla,      NEON_MLA_byelement,      vn.IsVector()) \
4209   V(mls,      NEON_MLS_byelement,      vn.IsVector()) \
4210   V(sqdmulh,  NEON_SQDMULH_byelement,  true)          \
4211   V(sqrdmulh, NEON_SQRDMULH_byelement, true)          \
4212 // clang-format on
4213 
4214 #define VIXL_DEFINE_ASM_FUNC(FN, OP, AS)                     \
4215   void Assembler::FN(const VRegister& vd,               \
4216                      const VRegister& vn,               \
4217                      const VRegister& vm,               \
4218                      int vm_index) {                    \
4219     VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));            \
4220     VIXL_ASSERT(AS);                                    \
4221     NEONByElement(vd, vn, vm, vm_index, OP);            \
4222   }
4223 NEON_BYELEMENT_LIST(VIXL_DEFINE_ASM_FUNC)
4224 #undef VIXL_DEFINE_ASM_FUNC
4225 
4226 
4227 // clang-format off
4228 #define NEON_BYELEMENT_RDM_LIST(V)     \
4229   V(sqrdmlah, NEON_SQRDMLAH_byelement) \
4230   V(sqrdmlsh, NEON_SQRDMLSH_byelement)
4231 // clang-format on
4232 
4233 #define VIXL_DEFINE_ASM_FUNC(FN, OP)                            \
4234   void Assembler::FN(const VRegister& vd,                       \
4235                      const VRegister& vn,                       \
4236                      const VRegister& vm,                       \
4237                      int vm_index) {                            \
4238     VIXL_ASSERT(CPUHas(CPUFeatures::kNEON, CPUFeatures::kRDM)); \
4239     NEONByElement(vd, vn, vm, vm_index, OP);                    \
4240   }
NEON_BYELEMENT_RDM_LIST(VIXL_DEFINE_ASM_FUNC)4241 NEON_BYELEMENT_RDM_LIST(VIXL_DEFINE_ASM_FUNC)
4242 #undef VIXL_DEFINE_ASM_FUNC
4243 
4244 
4245 // clang-format off
4246 #define NEON_FPBYELEMENT_LIST(V) \
4247   V(fmul,  NEON_FMUL_byelement,  NEON_FMUL_H_byelement)  \
4248   V(fmla,  NEON_FMLA_byelement,  NEON_FMLA_H_byelement)  \
4249   V(fmls,  NEON_FMLS_byelement,  NEON_FMLS_H_byelement)  \
4250   V(fmulx, NEON_FMULX_byelement, NEON_FMULX_H_byelement)
4251 // clang-format on
4252 
4253 #define VIXL_DEFINE_ASM_FUNC(FN, OP, OP_H)                             \
4254   void Assembler::FN(const VRegister& vd,                              \
4255                      const VRegister& vn,                              \
4256                      const VRegister& vm,                              \
4257                      int vm_index) {                                   \
4258     VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));         \
4259     if (vd.IsLaneSizeH()) VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf)); \
4260     NEONFPByElement(vd, vn, vm, vm_index, OP, OP_H);                   \
4261   }
4262 NEON_FPBYELEMENT_LIST(VIXL_DEFINE_ASM_FUNC)
4263 #undef VIXL_DEFINE_ASM_FUNC
4264 
4265 
4266 // clang-format off
4267 #define NEON_BYELEMENT_LONG_LIST(V)                               \
4268   V(sqdmull,  NEON_SQDMULL_byelement, vn.IsScalar() || vn.IsD())  \
4269   V(sqdmull2, NEON_SQDMULL_byelement, vn.IsVector() && vn.IsQ())  \
4270   V(sqdmlal,  NEON_SQDMLAL_byelement, vn.IsScalar() || vn.IsD())  \
4271   V(sqdmlal2, NEON_SQDMLAL_byelement, vn.IsVector() && vn.IsQ())  \
4272   V(sqdmlsl,  NEON_SQDMLSL_byelement, vn.IsScalar() || vn.IsD())  \
4273   V(sqdmlsl2, NEON_SQDMLSL_byelement, vn.IsVector() && vn.IsQ())  \
4274   V(smull,    NEON_SMULL_byelement,   vn.IsVector() && vn.IsD())  \
4275   V(smull2,   NEON_SMULL_byelement,   vn.IsVector() && vn.IsQ())  \
4276   V(umull,    NEON_UMULL_byelement,   vn.IsVector() && vn.IsD())  \
4277   V(umull2,   NEON_UMULL_byelement,   vn.IsVector() && vn.IsQ())  \
4278   V(smlal,    NEON_SMLAL_byelement,   vn.IsVector() && vn.IsD())  \
4279   V(smlal2,   NEON_SMLAL_byelement,   vn.IsVector() && vn.IsQ())  \
4280   V(umlal,    NEON_UMLAL_byelement,   vn.IsVector() && vn.IsD())  \
4281   V(umlal2,   NEON_UMLAL_byelement,   vn.IsVector() && vn.IsQ())  \
4282   V(smlsl,    NEON_SMLSL_byelement,   vn.IsVector() && vn.IsD())  \
4283   V(smlsl2,   NEON_SMLSL_byelement,   vn.IsVector() && vn.IsQ())  \
4284   V(umlsl,    NEON_UMLSL_byelement,   vn.IsVector() && vn.IsD())  \
4285   V(umlsl2,   NEON_UMLSL_byelement,   vn.IsVector() && vn.IsQ())
4286 // clang-format on
4287 
4288 
4289 #define VIXL_DEFINE_ASM_FUNC(FN, OP, AS)      \
4290   void Assembler::FN(const VRegister& vd,     \
4291                      const VRegister& vn,     \
4292                      const VRegister& vm,     \
4293                      int vm_index) {          \
4294     VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));  \
4295     VIXL_ASSERT(AS);                          \
4296     NEONByElementL(vd, vn, vm, vm_index, OP); \
4297   }
4298 NEON_BYELEMENT_LONG_LIST(VIXL_DEFINE_ASM_FUNC)
4299 #undef VIXL_DEFINE_ASM_FUNC
4300 
4301 
4302 // clang-format off
4303 #define NEON_BYELEMENT_FHM_LIST(V)    \
4304   V(fmlal, NEON_FMLAL_H_byelement)    \
4305   V(fmlal2, NEON_FMLAL2_H_byelement)  \
4306   V(fmlsl, NEON_FMLSL_H_byelement)    \
4307   V(fmlsl2, NEON_FMLSL2_H_byelement)
4308 // clang-format on
4309 
4310 
4311 #define VIXL_DEFINE_ASM_FUNC(FN, OP)                                   \
4312   void Assembler::FN(const VRegister& vd,                              \
4313                      const VRegister& vn,                              \
4314                      const VRegister& vm,                              \
4315                      int vm_index) {                                   \
4316     VIXL_ASSERT(CPUHas(CPUFeatures::kNEON,                             \
4317                        CPUFeatures::kFP,                               \
4318                        CPUFeatures::kNEONHalf,                         \
4319                        CPUFeatures::kFHM));                            \
4320     VIXL_ASSERT((vd.Is2S() && vn.Is2H()) || (vd.Is4S() && vn.Is4H())); \
4321     VIXL_ASSERT(vm.IsH());                                             \
4322     VIXL_ASSERT((vm_index >= 0) && (vm_index < 8));                    \
4323     /* Vm itself can only be in the bottom 16 registers. */            \
4324     VIXL_ASSERT(vm.GetCode() < 16);                                    \
4325     Emit(FPFormat(vd) | OP | Rd(vd) | Rn(vn) | Rm(vm) |                \
4326          ImmNEONHLM(vm_index, 3));                                     \
4327   }
4328 NEON_BYELEMENT_FHM_LIST(VIXL_DEFINE_ASM_FUNC)
4329 #undef VIXL_DEFINE_ASM_FUNC
4330 
4331 void Assembler::suqadd(const VRegister& vd, const VRegister& vn) {
4332   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4333   NEON2RegMisc(vd, vn, NEON_SUQADD);
4334 }
4335 
4336 
usqadd(const VRegister & vd,const VRegister & vn)4337 void Assembler::usqadd(const VRegister& vd, const VRegister& vn) {
4338   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4339   NEON2RegMisc(vd, vn, NEON_USQADD);
4340 }
4341 
4342 
abs(const VRegister & vd,const VRegister & vn)4343 void Assembler::abs(const VRegister& vd, const VRegister& vn) {
4344   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4345   VIXL_ASSERT(vd.IsVector() || vd.Is1D());
4346   NEON2RegMisc(vd, vn, NEON_ABS);
4347 }
4348 
4349 
sqabs(const VRegister & vd,const VRegister & vn)4350 void Assembler::sqabs(const VRegister& vd, const VRegister& vn) {
4351   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4352   NEON2RegMisc(vd, vn, NEON_SQABS);
4353 }
4354 
4355 
neg(const VRegister & vd,const VRegister & vn)4356 void Assembler::neg(const VRegister& vd, const VRegister& vn) {
4357   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4358   VIXL_ASSERT(vd.IsVector() || vd.Is1D());
4359   NEON2RegMisc(vd, vn, NEON_NEG);
4360 }
4361 
4362 
sqneg(const VRegister & vd,const VRegister & vn)4363 void Assembler::sqneg(const VRegister& vd, const VRegister& vn) {
4364   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4365   NEON2RegMisc(vd, vn, NEON_SQNEG);
4366 }
4367 
4368 
NEONXtn(const VRegister & vd,const VRegister & vn,NEON2RegMiscOp vop)4369 void Assembler::NEONXtn(const VRegister& vd,
4370                         const VRegister& vn,
4371                         NEON2RegMiscOp vop) {
4372   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4373   Instr format, op = vop;
4374   if (vd.IsScalar()) {
4375     VIXL_ASSERT((vd.Is1B() && vn.Is1H()) || (vd.Is1H() && vn.Is1S()) ||
4376                 (vd.Is1S() && vn.Is1D()));
4377     op |= NEON_Q | NEONScalar;
4378     format = SFormat(vd);
4379   } else {
4380     VIXL_ASSERT((vd.Is8B() && vn.Is8H()) || (vd.Is4H() && vn.Is4S()) ||
4381                 (vd.Is2S() && vn.Is2D()) || (vd.Is16B() && vn.Is8H()) ||
4382                 (vd.Is8H() && vn.Is4S()) || (vd.Is4S() && vn.Is2D()));
4383     format = VFormat(vd);
4384   }
4385   Emit(format | op | Rn(vn) | Rd(vd));
4386 }
4387 
4388 
xtn(const VRegister & vd,const VRegister & vn)4389 void Assembler::xtn(const VRegister& vd, const VRegister& vn) {
4390   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4391   VIXL_ASSERT(vd.IsVector() && vd.IsD());
4392   NEONXtn(vd, vn, NEON_XTN);
4393 }
4394 
4395 
xtn2(const VRegister & vd,const VRegister & vn)4396 void Assembler::xtn2(const VRegister& vd, const VRegister& vn) {
4397   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4398   VIXL_ASSERT(vd.IsVector() && vd.IsQ());
4399   NEONXtn(vd, vn, NEON_XTN);
4400 }
4401 
4402 
sqxtn(const VRegister & vd,const VRegister & vn)4403 void Assembler::sqxtn(const VRegister& vd, const VRegister& vn) {
4404   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4405   VIXL_ASSERT(vd.IsScalar() || vd.IsD());
4406   NEONXtn(vd, vn, NEON_SQXTN);
4407 }
4408 
4409 
sqxtn2(const VRegister & vd,const VRegister & vn)4410 void Assembler::sqxtn2(const VRegister& vd, const VRegister& vn) {
4411   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4412   VIXL_ASSERT(vd.IsVector() && vd.IsQ());
4413   NEONXtn(vd, vn, NEON_SQXTN);
4414 }
4415 
4416 
sqxtun(const VRegister & vd,const VRegister & vn)4417 void Assembler::sqxtun(const VRegister& vd, const VRegister& vn) {
4418   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4419   VIXL_ASSERT(vd.IsScalar() || vd.IsD());
4420   NEONXtn(vd, vn, NEON_SQXTUN);
4421 }
4422 
4423 
sqxtun2(const VRegister & vd,const VRegister & vn)4424 void Assembler::sqxtun2(const VRegister& vd, const VRegister& vn) {
4425   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4426   VIXL_ASSERT(vd.IsVector() && vd.IsQ());
4427   NEONXtn(vd, vn, NEON_SQXTUN);
4428 }
4429 
4430 
uqxtn(const VRegister & vd,const VRegister & vn)4431 void Assembler::uqxtn(const VRegister& vd, const VRegister& vn) {
4432   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4433   VIXL_ASSERT(vd.IsScalar() || vd.IsD());
4434   NEONXtn(vd, vn, NEON_UQXTN);
4435 }
4436 
4437 
uqxtn2(const VRegister & vd,const VRegister & vn)4438 void Assembler::uqxtn2(const VRegister& vd, const VRegister& vn) {
4439   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4440   VIXL_ASSERT(vd.IsVector() && vd.IsQ());
4441   NEONXtn(vd, vn, NEON_UQXTN);
4442 }
4443 
4444 
4445 // NEON NOT and RBIT are distinguised by bit 22, the bottom bit of "size".
not_(const VRegister & vd,const VRegister & vn)4446 void Assembler::not_(const VRegister& vd, const VRegister& vn) {
4447   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4448   VIXL_ASSERT(AreSameFormat(vd, vn));
4449   VIXL_ASSERT(vd.Is8B() || vd.Is16B());
4450   Emit(VFormat(vd) | NEON_RBIT_NOT | Rn(vn) | Rd(vd));
4451 }
4452 
4453 
rbit(const VRegister & vd,const VRegister & vn)4454 void Assembler::rbit(const VRegister& vd, const VRegister& vn) {
4455   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4456   VIXL_ASSERT(AreSameFormat(vd, vn));
4457   VIXL_ASSERT(vd.Is8B() || vd.Is16B());
4458   Emit(VFormat(vn) | (1 << NEONSize_offset) | NEON_RBIT_NOT | Rn(vn) | Rd(vd));
4459 }
4460 
4461 
ext(const VRegister & vd,const VRegister & vn,const VRegister & vm,int index)4462 void Assembler::ext(const VRegister& vd,
4463                     const VRegister& vn,
4464                     const VRegister& vm,
4465                     int index) {
4466   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4467   VIXL_ASSERT(AreSameFormat(vd, vn, vm));
4468   VIXL_ASSERT(vd.Is8B() || vd.Is16B());
4469   VIXL_ASSERT((0 <= index) && (index < vd.GetLanes()));
4470   Emit(VFormat(vd) | NEON_EXT | Rm(vm) | ImmNEONExt(index) | Rn(vn) | Rd(vd));
4471 }
4472 
4473 
dup(const VRegister & vd,const VRegister & vn,int vn_index)4474 void Assembler::dup(const VRegister& vd, const VRegister& vn, int vn_index) {
4475   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4476   Instr q, scalar;
4477 
4478   // We support vn arguments of the form vn.VxT() or vn.T(), where x is the
4479   // number of lanes, and T is b, h, s or d.
4480   int lane_size = vn.GetLaneSizeInBytes();
4481   NEONFormatField format;
4482   switch (lane_size) {
4483     case 1:
4484       format = NEON_16B;
4485       break;
4486     case 2:
4487       format = NEON_8H;
4488       break;
4489     case 4:
4490       format = NEON_4S;
4491       break;
4492     default:
4493       VIXL_ASSERT(lane_size == 8);
4494       format = NEON_2D;
4495       break;
4496   }
4497 
4498   if (vd.IsScalar()) {
4499     q = NEON_Q;
4500     scalar = NEONScalar;
4501   } else {
4502     VIXL_ASSERT(!vd.Is1D());
4503     q = vd.IsD() ? 0 : NEON_Q;
4504     scalar = 0;
4505   }
4506   Emit(q | scalar | NEON_DUP_ELEMENT | ImmNEON5(format, vn_index) | Rn(vn) |
4507        Rd(vd));
4508 }
4509 
4510 
mov(const VRegister & vd,const VRegister & vn,int vn_index)4511 void Assembler::mov(const VRegister& vd, const VRegister& vn, int vn_index) {
4512   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4513   VIXL_ASSERT(vd.IsScalar());
4514   dup(vd, vn, vn_index);
4515 }
4516 
4517 
dup(const VRegister & vd,const Register & rn)4518 void Assembler::dup(const VRegister& vd, const Register& rn) {
4519   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4520   VIXL_ASSERT(!vd.Is1D());
4521   VIXL_ASSERT(vd.Is2D() == rn.IsX());
4522   int q = vd.IsD() ? 0 : NEON_Q;
4523   Emit(q | NEON_DUP_GENERAL | ImmNEON5(VFormat(vd), 0) | Rn(rn) | Rd(vd));
4524 }
4525 
4526 
ins(const VRegister & vd,int vd_index,const VRegister & vn,int vn_index)4527 void Assembler::ins(const VRegister& vd,
4528                     int vd_index,
4529                     const VRegister& vn,
4530                     int vn_index) {
4531   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4532   VIXL_ASSERT(AreSameFormat(vd, vn));
4533   // We support vd arguments of the form vd.VxT() or vd.T(), where x is the
4534   // number of lanes, and T is b, h, s or d.
4535   int lane_size = vd.GetLaneSizeInBytes();
4536   NEONFormatField format;
4537   switch (lane_size) {
4538     case 1:
4539       format = NEON_16B;
4540       break;
4541     case 2:
4542       format = NEON_8H;
4543       break;
4544     case 4:
4545       format = NEON_4S;
4546       break;
4547     default:
4548       VIXL_ASSERT(lane_size == 8);
4549       format = NEON_2D;
4550       break;
4551   }
4552 
4553   VIXL_ASSERT(
4554       (0 <= vd_index) &&
4555       (vd_index < LaneCountFromFormat(static_cast<VectorFormat>(format))));
4556   VIXL_ASSERT(
4557       (0 <= vn_index) &&
4558       (vn_index < LaneCountFromFormat(static_cast<VectorFormat>(format))));
4559   Emit(NEON_INS_ELEMENT | ImmNEON5(format, vd_index) |
4560        ImmNEON4(format, vn_index) | Rn(vn) | Rd(vd));
4561 }
4562 
4563 
mov(const VRegister & vd,int vd_index,const VRegister & vn,int vn_index)4564 void Assembler::mov(const VRegister& vd,
4565                     int vd_index,
4566                     const VRegister& vn,
4567                     int vn_index) {
4568   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4569   ins(vd, vd_index, vn, vn_index);
4570 }
4571 
4572 
ins(const VRegister & vd,int vd_index,const Register & rn)4573 void Assembler::ins(const VRegister& vd, int vd_index, const Register& rn) {
4574   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4575   // We support vd arguments of the form vd.VxT() or vd.T(), where x is the
4576   // number of lanes, and T is b, h, s or d.
4577   int lane_size = vd.GetLaneSizeInBytes();
4578   NEONFormatField format;
4579   switch (lane_size) {
4580     case 1:
4581       format = NEON_16B;
4582       VIXL_ASSERT(rn.IsW());
4583       break;
4584     case 2:
4585       format = NEON_8H;
4586       VIXL_ASSERT(rn.IsW());
4587       break;
4588     case 4:
4589       format = NEON_4S;
4590       VIXL_ASSERT(rn.IsW());
4591       break;
4592     default:
4593       VIXL_ASSERT(lane_size == 8);
4594       VIXL_ASSERT(rn.IsX());
4595       format = NEON_2D;
4596       break;
4597   }
4598 
4599   VIXL_ASSERT(
4600       (0 <= vd_index) &&
4601       (vd_index < LaneCountFromFormat(static_cast<VectorFormat>(format))));
4602   Emit(NEON_INS_GENERAL | ImmNEON5(format, vd_index) | Rn(rn) | Rd(vd));
4603 }
4604 
4605 
mov(const VRegister & vd,int vd_index,const Register & rn)4606 void Assembler::mov(const VRegister& vd, int vd_index, const Register& rn) {
4607   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4608   ins(vd, vd_index, rn);
4609 }
4610 
4611 
umov(const Register & rd,const VRegister & vn,int vn_index)4612 void Assembler::umov(const Register& rd, const VRegister& vn, int vn_index) {
4613   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4614   // We support vn arguments of the form vn.VxT() or vn.T(), where x is the
4615   // number of lanes, and T is b, h, s or d.
4616   int lane_size = vn.GetLaneSizeInBytes();
4617   NEONFormatField format;
4618   Instr q = 0;
4619   switch (lane_size) {
4620     case 1:
4621       format = NEON_16B;
4622       VIXL_ASSERT(rd.IsW());
4623       break;
4624     case 2:
4625       format = NEON_8H;
4626       VIXL_ASSERT(rd.IsW());
4627       break;
4628     case 4:
4629       format = NEON_4S;
4630       VIXL_ASSERT(rd.IsW());
4631       break;
4632     default:
4633       VIXL_ASSERT(lane_size == 8);
4634       VIXL_ASSERT(rd.IsX());
4635       format = NEON_2D;
4636       q = NEON_Q;
4637       break;
4638   }
4639 
4640   VIXL_ASSERT(
4641       (0 <= vn_index) &&
4642       (vn_index < LaneCountFromFormat(static_cast<VectorFormat>(format))));
4643   Emit(q | NEON_UMOV | ImmNEON5(format, vn_index) | Rn(vn) | Rd(rd));
4644 }
4645 
4646 
mov(const Register & rd,const VRegister & vn,int vn_index)4647 void Assembler::mov(const Register& rd, const VRegister& vn, int vn_index) {
4648   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4649   VIXL_ASSERT(vn.GetSizeInBytes() >= 4);
4650   umov(rd, vn, vn_index);
4651 }
4652 
4653 
smov(const Register & rd,const VRegister & vn,int vn_index)4654 void Assembler::smov(const Register& rd, const VRegister& vn, int vn_index) {
4655   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4656   // We support vn arguments of the form vn.VxT() or vn.T(), where x is the
4657   // number of lanes, and T is b, h, s.
4658   int lane_size = vn.GetLaneSizeInBytes();
4659   NEONFormatField format;
4660   Instr q = 0;
4661   VIXL_ASSERT(lane_size != 8);
4662   switch (lane_size) {
4663     case 1:
4664       format = NEON_16B;
4665       break;
4666     case 2:
4667       format = NEON_8H;
4668       break;
4669     default:
4670       VIXL_ASSERT(lane_size == 4);
4671       VIXL_ASSERT(rd.IsX());
4672       format = NEON_4S;
4673       break;
4674   }
4675   q = rd.IsW() ? 0 : NEON_Q;
4676   VIXL_ASSERT(
4677       (0 <= vn_index) &&
4678       (vn_index < LaneCountFromFormat(static_cast<VectorFormat>(format))));
4679   Emit(q | NEON_SMOV | ImmNEON5(format, vn_index) | Rn(vn) | Rd(rd));
4680 }
4681 
4682 
cls(const VRegister & vd,const VRegister & vn)4683 void Assembler::cls(const VRegister& vd, const VRegister& vn) {
4684   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4685   VIXL_ASSERT(AreSameFormat(vd, vn));
4686   VIXL_ASSERT(!vd.Is1D() && !vd.Is2D());
4687   Emit(VFormat(vn) | NEON_CLS | Rn(vn) | Rd(vd));
4688 }
4689 
4690 
clz(const VRegister & vd,const VRegister & vn)4691 void Assembler::clz(const VRegister& vd, const VRegister& vn) {
4692   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4693   VIXL_ASSERT(AreSameFormat(vd, vn));
4694   VIXL_ASSERT(!vd.Is1D() && !vd.Is2D());
4695   Emit(VFormat(vn) | NEON_CLZ | Rn(vn) | Rd(vd));
4696 }
4697 
4698 
cnt(const VRegister & vd,const VRegister & vn)4699 void Assembler::cnt(const VRegister& vd, const VRegister& vn) {
4700   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4701   VIXL_ASSERT(AreSameFormat(vd, vn));
4702   VIXL_ASSERT(vd.Is8B() || vd.Is16B());
4703   Emit(VFormat(vn) | NEON_CNT | Rn(vn) | Rd(vd));
4704 }
4705 
4706 
rev16(const VRegister & vd,const VRegister & vn)4707 void Assembler::rev16(const VRegister& vd, const VRegister& vn) {
4708   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4709   VIXL_ASSERT(AreSameFormat(vd, vn));
4710   VIXL_ASSERT(vd.Is8B() || vd.Is16B());
4711   Emit(VFormat(vn) | NEON_REV16 | Rn(vn) | Rd(vd));
4712 }
4713 
4714 
rev32(const VRegister & vd,const VRegister & vn)4715 void Assembler::rev32(const VRegister& vd, const VRegister& vn) {
4716   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4717   VIXL_ASSERT(AreSameFormat(vd, vn));
4718   VIXL_ASSERT(vd.Is8B() || vd.Is16B() || vd.Is4H() || vd.Is8H());
4719   Emit(VFormat(vn) | NEON_REV32 | Rn(vn) | Rd(vd));
4720 }
4721 
4722 
rev64(const VRegister & vd,const VRegister & vn)4723 void Assembler::rev64(const VRegister& vd, const VRegister& vn) {
4724   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4725   VIXL_ASSERT(AreSameFormat(vd, vn));
4726   VIXL_ASSERT(!vd.Is1D() && !vd.Is2D());
4727   Emit(VFormat(vn) | NEON_REV64 | Rn(vn) | Rd(vd));
4728 }
4729 
4730 
ursqrte(const VRegister & vd,const VRegister & vn)4731 void Assembler::ursqrte(const VRegister& vd, const VRegister& vn) {
4732   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4733   VIXL_ASSERT(AreSameFormat(vd, vn));
4734   VIXL_ASSERT(vd.Is2S() || vd.Is4S());
4735   Emit(VFormat(vn) | NEON_URSQRTE | Rn(vn) | Rd(vd));
4736 }
4737 
4738 
urecpe(const VRegister & vd,const VRegister & vn)4739 void Assembler::urecpe(const VRegister& vd, const VRegister& vn) {
4740   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4741   VIXL_ASSERT(AreSameFormat(vd, vn));
4742   VIXL_ASSERT(vd.Is2S() || vd.Is4S());
4743   Emit(VFormat(vn) | NEON_URECPE | Rn(vn) | Rd(vd));
4744 }
4745 
4746 
NEONAddlp(const VRegister & vd,const VRegister & vn,NEON2RegMiscOp op)4747 void Assembler::NEONAddlp(const VRegister& vd,
4748                           const VRegister& vn,
4749                           NEON2RegMiscOp op) {
4750   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4751   VIXL_ASSERT((op == NEON_SADDLP) || (op == NEON_UADDLP) ||
4752               (op == NEON_SADALP) || (op == NEON_UADALP));
4753 
4754   VIXL_ASSERT((vn.Is8B() && vd.Is4H()) || (vn.Is4H() && vd.Is2S()) ||
4755               (vn.Is2S() && vd.Is1D()) || (vn.Is16B() && vd.Is8H()) ||
4756               (vn.Is8H() && vd.Is4S()) || (vn.Is4S() && vd.Is2D()));
4757   Emit(VFormat(vn) | op | Rn(vn) | Rd(vd));
4758 }
4759 
4760 
saddlp(const VRegister & vd,const VRegister & vn)4761 void Assembler::saddlp(const VRegister& vd, const VRegister& vn) {
4762   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4763   NEONAddlp(vd, vn, NEON_SADDLP);
4764 }
4765 
4766 
uaddlp(const VRegister & vd,const VRegister & vn)4767 void Assembler::uaddlp(const VRegister& vd, const VRegister& vn) {
4768   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4769   NEONAddlp(vd, vn, NEON_UADDLP);
4770 }
4771 
4772 
sadalp(const VRegister & vd,const VRegister & vn)4773 void Assembler::sadalp(const VRegister& vd, const VRegister& vn) {
4774   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4775   NEONAddlp(vd, vn, NEON_SADALP);
4776 }
4777 
4778 
uadalp(const VRegister & vd,const VRegister & vn)4779 void Assembler::uadalp(const VRegister& vd, const VRegister& vn) {
4780   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4781   NEONAddlp(vd, vn, NEON_UADALP);
4782 }
4783 
4784 
NEONAcrossLanesL(const VRegister & vd,const VRegister & vn,NEONAcrossLanesOp op)4785 void Assembler::NEONAcrossLanesL(const VRegister& vd,
4786                                  const VRegister& vn,
4787                                  NEONAcrossLanesOp op) {
4788   VIXL_ASSERT((vn.Is8B() && vd.Is1H()) || (vn.Is16B() && vd.Is1H()) ||
4789               (vn.Is4H() && vd.Is1S()) || (vn.Is8H() && vd.Is1S()) ||
4790               (vn.Is4S() && vd.Is1D()));
4791   Emit(VFormat(vn) | op | Rn(vn) | Rd(vd));
4792 }
4793 
4794 
saddlv(const VRegister & vd,const VRegister & vn)4795 void Assembler::saddlv(const VRegister& vd, const VRegister& vn) {
4796   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4797   NEONAcrossLanesL(vd, vn, NEON_SADDLV);
4798 }
4799 
4800 
uaddlv(const VRegister & vd,const VRegister & vn)4801 void Assembler::uaddlv(const VRegister& vd, const VRegister& vn) {
4802   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4803   NEONAcrossLanesL(vd, vn, NEON_UADDLV);
4804 }
4805 
4806 
NEONAcrossLanes(const VRegister & vd,const VRegister & vn,NEONAcrossLanesOp op,Instr op_half)4807 void Assembler::NEONAcrossLanes(const VRegister& vd,
4808                                 const VRegister& vn,
4809                                 NEONAcrossLanesOp op,
4810                                 Instr op_half) {
4811   VIXL_ASSERT((vn.Is8B() && vd.Is1B()) || (vn.Is16B() && vd.Is1B()) ||
4812               (vn.Is4H() && vd.Is1H()) || (vn.Is8H() && vd.Is1H()) ||
4813               (vn.Is4S() && vd.Is1S()));
4814   if ((op & NEONAcrossLanesFPFMask) == NEONAcrossLanesFPFixed) {
4815     if (vd.Is1H()) {
4816       VIXL_ASSERT(op_half != 0);
4817       Instr vop = op_half;
4818       if (vn.Is8H()) {
4819         vop |= NEON_Q;
4820       }
4821       Emit(vop | Rn(vn) | Rd(vd));
4822     } else {
4823       Emit(FPFormat(vn) | op | Rn(vn) | Rd(vd));
4824     }
4825   } else {
4826     Emit(VFormat(vn) | op | Rn(vn) | Rd(vd));
4827   }
4828 }
4829 
4830 // clang-format off
4831 #define NEON_ACROSSLANES_LIST(V)           \
4832   V(addv,    NEON_ADDV)                    \
4833   V(smaxv,   NEON_SMAXV)                   \
4834   V(sminv,   NEON_SMINV)                   \
4835   V(umaxv,   NEON_UMAXV)                   \
4836   V(uminv,   NEON_UMINV)
4837 // clang-format on
4838 
4839 #define VIXL_DEFINE_ASM_FUNC(FN, OP)                             \
4840   void Assembler::FN(const VRegister& vd, const VRegister& vn) { \
4841     VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));                     \
4842     NEONAcrossLanes(vd, vn, OP, 0);                              \
4843   }
4844 NEON_ACROSSLANES_LIST(VIXL_DEFINE_ASM_FUNC)
4845 #undef VIXL_DEFINE_ASM_FUNC
4846 
4847 
4848 // clang-format off
4849 #define NEON_ACROSSLANES_FP_LIST(V)   \
4850   V(fmaxv,   NEON_FMAXV,   NEON_FMAXV_H) \
4851   V(fminv,   NEON_FMINV,   NEON_FMINV_H) \
4852   V(fmaxnmv, NEON_FMAXNMV, NEON_FMAXNMV_H) \
4853   V(fminnmv, NEON_FMINNMV, NEON_FMINNMV_H) \
4854 // clang-format on
4855 
4856 #define VIXL_DEFINE_ASM_FUNC(FN, OP, OP_H)                            \
4857   void Assembler::FN(const VRegister& vd, const VRegister& vn) { \
4858     VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kNEON));   \
4859     if (vd.Is1H()) VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));  \
4860     VIXL_ASSERT(vd.Is1S() || vd.Is1H());                         \
4861     NEONAcrossLanes(vd, vn, OP, OP_H);                           \
4862   }
NEON_ACROSSLANES_FP_LIST(VIXL_DEFINE_ASM_FUNC)4863 NEON_ACROSSLANES_FP_LIST(VIXL_DEFINE_ASM_FUNC)
4864 #undef VIXL_DEFINE_ASM_FUNC
4865 
4866 
4867 void Assembler::NEONPerm(const VRegister& vd,
4868                          const VRegister& vn,
4869                          const VRegister& vm,
4870                          NEONPermOp op) {
4871   VIXL_ASSERT(AreSameFormat(vd, vn, vm));
4872   VIXL_ASSERT(!vd.Is1D());
4873   Emit(VFormat(vd) | op | Rm(vm) | Rn(vn) | Rd(vd));
4874 }
4875 
4876 
trn1(const VRegister & vd,const VRegister & vn,const VRegister & vm)4877 void Assembler::trn1(const VRegister& vd,
4878                      const VRegister& vn,
4879                      const VRegister& vm) {
4880   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4881   NEONPerm(vd, vn, vm, NEON_TRN1);
4882 }
4883 
4884 
trn2(const VRegister & vd,const VRegister & vn,const VRegister & vm)4885 void Assembler::trn2(const VRegister& vd,
4886                      const VRegister& vn,
4887                      const VRegister& vm) {
4888   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4889   NEONPerm(vd, vn, vm, NEON_TRN2);
4890 }
4891 
4892 
uzp1(const VRegister & vd,const VRegister & vn,const VRegister & vm)4893 void Assembler::uzp1(const VRegister& vd,
4894                      const VRegister& vn,
4895                      const VRegister& vm) {
4896   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4897   NEONPerm(vd, vn, vm, NEON_UZP1);
4898 }
4899 
4900 
uzp2(const VRegister & vd,const VRegister & vn,const VRegister & vm)4901 void Assembler::uzp2(const VRegister& vd,
4902                      const VRegister& vn,
4903                      const VRegister& vm) {
4904   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4905   NEONPerm(vd, vn, vm, NEON_UZP2);
4906 }
4907 
4908 
zip1(const VRegister & vd,const VRegister & vn,const VRegister & vm)4909 void Assembler::zip1(const VRegister& vd,
4910                      const VRegister& vn,
4911                      const VRegister& vm) {
4912   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4913   NEONPerm(vd, vn, vm, NEON_ZIP1);
4914 }
4915 
4916 
zip2(const VRegister & vd,const VRegister & vn,const VRegister & vm)4917 void Assembler::zip2(const VRegister& vd,
4918                      const VRegister& vn,
4919                      const VRegister& vm) {
4920   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
4921   NEONPerm(vd, vn, vm, NEON_ZIP2);
4922 }
4923 
4924 
NEONShiftImmediate(const VRegister & vd,const VRegister & vn,NEONShiftImmediateOp op,int immh_immb)4925 void Assembler::NEONShiftImmediate(const VRegister& vd,
4926                                    const VRegister& vn,
4927                                    NEONShiftImmediateOp op,
4928                                    int immh_immb) {
4929   VIXL_ASSERT(AreSameFormat(vd, vn));
4930   Instr q, scalar;
4931   if (vn.IsScalar()) {
4932     q = NEON_Q;
4933     scalar = NEONScalar;
4934   } else {
4935     q = vd.IsD() ? 0 : NEON_Q;
4936     scalar = 0;
4937   }
4938   Emit(q | op | scalar | immh_immb | Rn(vn) | Rd(vd));
4939 }
4940 
4941 
NEONShiftLeftImmediate(const VRegister & vd,const VRegister & vn,int shift,NEONShiftImmediateOp op)4942 void Assembler::NEONShiftLeftImmediate(const VRegister& vd,
4943                                        const VRegister& vn,
4944                                        int shift,
4945                                        NEONShiftImmediateOp op) {
4946   int lane_size_in_bits = vn.GetLaneSizeInBits();
4947   VIXL_ASSERT((shift >= 0) && (shift < lane_size_in_bits));
4948   NEONShiftImmediate(vd, vn, op, (lane_size_in_bits + shift) << 16);
4949 }
4950 
4951 
NEONShiftRightImmediate(const VRegister & vd,const VRegister & vn,int shift,NEONShiftImmediateOp op)4952 void Assembler::NEONShiftRightImmediate(const VRegister& vd,
4953                                         const VRegister& vn,
4954                                         int shift,
4955                                         NEONShiftImmediateOp op) {
4956   int lane_size_in_bits = vn.GetLaneSizeInBits();
4957   VIXL_ASSERT((shift >= 1) && (shift <= lane_size_in_bits));
4958   NEONShiftImmediate(vd, vn, op, ((2 * lane_size_in_bits) - shift) << 16);
4959 }
4960 
4961 
NEONShiftImmediateL(const VRegister & vd,const VRegister & vn,int shift,NEONShiftImmediateOp op)4962 void Assembler::NEONShiftImmediateL(const VRegister& vd,
4963                                     const VRegister& vn,
4964                                     int shift,
4965                                     NEONShiftImmediateOp op) {
4966   int lane_size_in_bits = vn.GetLaneSizeInBits();
4967   VIXL_ASSERT((shift >= 0) && (shift < lane_size_in_bits));
4968   int immh_immb = (lane_size_in_bits + shift) << 16;
4969 
4970   VIXL_ASSERT((vn.Is8B() && vd.Is8H()) || (vn.Is4H() && vd.Is4S()) ||
4971               (vn.Is2S() && vd.Is2D()) || (vn.Is16B() && vd.Is8H()) ||
4972               (vn.Is8H() && vd.Is4S()) || (vn.Is4S() && vd.Is2D()));
4973   Instr q;
4974   q = vn.IsD() ? 0 : NEON_Q;
4975   Emit(q | op | immh_immb | Rn(vn) | Rd(vd));
4976 }
4977 
4978 
NEONShiftImmediateN(const VRegister & vd,const VRegister & vn,int shift,NEONShiftImmediateOp op)4979 void Assembler::NEONShiftImmediateN(const VRegister& vd,
4980                                     const VRegister& vn,
4981                                     int shift,
4982                                     NEONShiftImmediateOp op) {
4983   Instr q, scalar;
4984   int lane_size_in_bits = vd.GetLaneSizeInBits();
4985   VIXL_ASSERT((shift >= 1) && (shift <= lane_size_in_bits));
4986   int immh_immb = (2 * lane_size_in_bits - shift) << 16;
4987 
4988   if (vn.IsScalar()) {
4989     VIXL_ASSERT((vd.Is1B() && vn.Is1H()) || (vd.Is1H() && vn.Is1S()) ||
4990                 (vd.Is1S() && vn.Is1D()));
4991     q = NEON_Q;
4992     scalar = NEONScalar;
4993   } else {
4994     VIXL_ASSERT((vd.Is8B() && vn.Is8H()) || (vd.Is4H() && vn.Is4S()) ||
4995                 (vd.Is2S() && vn.Is2D()) || (vd.Is16B() && vn.Is8H()) ||
4996                 (vd.Is8H() && vn.Is4S()) || (vd.Is4S() && vn.Is2D()));
4997     scalar = 0;
4998     q = vd.IsD() ? 0 : NEON_Q;
4999   }
5000   Emit(q | op | scalar | immh_immb | Rn(vn) | Rd(vd));
5001 }
5002 
5003 
shl(const VRegister & vd,const VRegister & vn,int shift)5004 void Assembler::shl(const VRegister& vd, const VRegister& vn, int shift) {
5005   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5006   VIXL_ASSERT(vd.IsVector() || vd.Is1D());
5007   NEONShiftLeftImmediate(vd, vn, shift, NEON_SHL);
5008 }
5009 
5010 
sli(const VRegister & vd,const VRegister & vn,int shift)5011 void Assembler::sli(const VRegister& vd, const VRegister& vn, int shift) {
5012   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5013   VIXL_ASSERT(vd.IsVector() || vd.Is1D());
5014   NEONShiftLeftImmediate(vd, vn, shift, NEON_SLI);
5015 }
5016 
5017 
sqshl(const VRegister & vd,const VRegister & vn,int shift)5018 void Assembler::sqshl(const VRegister& vd, const VRegister& vn, int shift) {
5019   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5020   NEONShiftLeftImmediate(vd, vn, shift, NEON_SQSHL_imm);
5021 }
5022 
5023 
sqshlu(const VRegister & vd,const VRegister & vn,int shift)5024 void Assembler::sqshlu(const VRegister& vd, const VRegister& vn, int shift) {
5025   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5026   NEONShiftLeftImmediate(vd, vn, shift, NEON_SQSHLU);
5027 }
5028 
5029 
uqshl(const VRegister & vd,const VRegister & vn,int shift)5030 void Assembler::uqshl(const VRegister& vd, const VRegister& vn, int shift) {
5031   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5032   NEONShiftLeftImmediate(vd, vn, shift, NEON_UQSHL_imm);
5033 }
5034 
5035 
sshll(const VRegister & vd,const VRegister & vn,int shift)5036 void Assembler::sshll(const VRegister& vd, const VRegister& vn, int shift) {
5037   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5038   VIXL_ASSERT(vn.IsD());
5039   NEONShiftImmediateL(vd, vn, shift, NEON_SSHLL);
5040 }
5041 
5042 
sshll2(const VRegister & vd,const VRegister & vn,int shift)5043 void Assembler::sshll2(const VRegister& vd, const VRegister& vn, int shift) {
5044   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5045   VIXL_ASSERT(vn.IsQ());
5046   NEONShiftImmediateL(vd, vn, shift, NEON_SSHLL);
5047 }
5048 
5049 
sxtl(const VRegister & vd,const VRegister & vn)5050 void Assembler::sxtl(const VRegister& vd, const VRegister& vn) {
5051   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5052   sshll(vd, vn, 0);
5053 }
5054 
5055 
sxtl2(const VRegister & vd,const VRegister & vn)5056 void Assembler::sxtl2(const VRegister& vd, const VRegister& vn) {
5057   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5058   sshll2(vd, vn, 0);
5059 }
5060 
5061 
ushll(const VRegister & vd,const VRegister & vn,int shift)5062 void Assembler::ushll(const VRegister& vd, const VRegister& vn, int shift) {
5063   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5064   VIXL_ASSERT(vn.IsD());
5065   NEONShiftImmediateL(vd, vn, shift, NEON_USHLL);
5066 }
5067 
5068 
ushll2(const VRegister & vd,const VRegister & vn,int shift)5069 void Assembler::ushll2(const VRegister& vd, const VRegister& vn, int shift) {
5070   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5071   VIXL_ASSERT(vn.IsQ());
5072   NEONShiftImmediateL(vd, vn, shift, NEON_USHLL);
5073 }
5074 
5075 
uxtl(const VRegister & vd,const VRegister & vn)5076 void Assembler::uxtl(const VRegister& vd, const VRegister& vn) {
5077   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5078   ushll(vd, vn, 0);
5079 }
5080 
5081 
uxtl2(const VRegister & vd,const VRegister & vn)5082 void Assembler::uxtl2(const VRegister& vd, const VRegister& vn) {
5083   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5084   ushll2(vd, vn, 0);
5085 }
5086 
5087 
sri(const VRegister & vd,const VRegister & vn,int shift)5088 void Assembler::sri(const VRegister& vd, const VRegister& vn, int shift) {
5089   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5090   VIXL_ASSERT(vd.IsVector() || vd.Is1D());
5091   NEONShiftRightImmediate(vd, vn, shift, NEON_SRI);
5092 }
5093 
5094 
sshr(const VRegister & vd,const VRegister & vn,int shift)5095 void Assembler::sshr(const VRegister& vd, const VRegister& vn, int shift) {
5096   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5097   VIXL_ASSERT(vd.IsVector() || vd.Is1D());
5098   NEONShiftRightImmediate(vd, vn, shift, NEON_SSHR);
5099 }
5100 
5101 
ushr(const VRegister & vd,const VRegister & vn,int shift)5102 void Assembler::ushr(const VRegister& vd, const VRegister& vn, int shift) {
5103   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5104   VIXL_ASSERT(vd.IsVector() || vd.Is1D());
5105   NEONShiftRightImmediate(vd, vn, shift, NEON_USHR);
5106 }
5107 
5108 
srshr(const VRegister & vd,const VRegister & vn,int shift)5109 void Assembler::srshr(const VRegister& vd, const VRegister& vn, int shift) {
5110   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5111   VIXL_ASSERT(vd.IsVector() || vd.Is1D());
5112   NEONShiftRightImmediate(vd, vn, shift, NEON_SRSHR);
5113 }
5114 
5115 
urshr(const VRegister & vd,const VRegister & vn,int shift)5116 void Assembler::urshr(const VRegister& vd, const VRegister& vn, int shift) {
5117   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5118   VIXL_ASSERT(vd.IsVector() || vd.Is1D());
5119   NEONShiftRightImmediate(vd, vn, shift, NEON_URSHR);
5120 }
5121 
5122 
ssra(const VRegister & vd,const VRegister & vn,int shift)5123 void Assembler::ssra(const VRegister& vd, const VRegister& vn, int shift) {
5124   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5125   VIXL_ASSERT(vd.IsVector() || vd.Is1D());
5126   NEONShiftRightImmediate(vd, vn, shift, NEON_SSRA);
5127 }
5128 
5129 
usra(const VRegister & vd,const VRegister & vn,int shift)5130 void Assembler::usra(const VRegister& vd, const VRegister& vn, int shift) {
5131   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5132   VIXL_ASSERT(vd.IsVector() || vd.Is1D());
5133   NEONShiftRightImmediate(vd, vn, shift, NEON_USRA);
5134 }
5135 
5136 
srsra(const VRegister & vd,const VRegister & vn,int shift)5137 void Assembler::srsra(const VRegister& vd, const VRegister& vn, int shift) {
5138   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5139   VIXL_ASSERT(vd.IsVector() || vd.Is1D());
5140   NEONShiftRightImmediate(vd, vn, shift, NEON_SRSRA);
5141 }
5142 
5143 
ursra(const VRegister & vd,const VRegister & vn,int shift)5144 void Assembler::ursra(const VRegister& vd, const VRegister& vn, int shift) {
5145   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5146   VIXL_ASSERT(vd.IsVector() || vd.Is1D());
5147   NEONShiftRightImmediate(vd, vn, shift, NEON_URSRA);
5148 }
5149 
5150 
shrn(const VRegister & vd,const VRegister & vn,int shift)5151 void Assembler::shrn(const VRegister& vd, const VRegister& vn, int shift) {
5152   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5153   VIXL_ASSERT(vn.IsVector() && vd.IsD());
5154   NEONShiftImmediateN(vd, vn, shift, NEON_SHRN);
5155 }
5156 
5157 
shrn2(const VRegister & vd,const VRegister & vn,int shift)5158 void Assembler::shrn2(const VRegister& vd, const VRegister& vn, int shift) {
5159   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5160   VIXL_ASSERT(vn.IsVector() && vd.IsQ());
5161   NEONShiftImmediateN(vd, vn, shift, NEON_SHRN);
5162 }
5163 
5164 
rshrn(const VRegister & vd,const VRegister & vn,int shift)5165 void Assembler::rshrn(const VRegister& vd, const VRegister& vn, int shift) {
5166   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5167   VIXL_ASSERT(vn.IsVector() && vd.IsD());
5168   NEONShiftImmediateN(vd, vn, shift, NEON_RSHRN);
5169 }
5170 
5171 
rshrn2(const VRegister & vd,const VRegister & vn,int shift)5172 void Assembler::rshrn2(const VRegister& vd, const VRegister& vn, int shift) {
5173   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5174   VIXL_ASSERT(vn.IsVector() && vd.IsQ());
5175   NEONShiftImmediateN(vd, vn, shift, NEON_RSHRN);
5176 }
5177 
5178 
sqshrn(const VRegister & vd,const VRegister & vn,int shift)5179 void Assembler::sqshrn(const VRegister& vd, const VRegister& vn, int shift) {
5180   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5181   VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
5182   NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRN);
5183 }
5184 
5185 
sqshrn2(const VRegister & vd,const VRegister & vn,int shift)5186 void Assembler::sqshrn2(const VRegister& vd, const VRegister& vn, int shift) {
5187   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5188   VIXL_ASSERT(vn.IsVector() && vd.IsQ());
5189   NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRN);
5190 }
5191 
5192 
sqrshrn(const VRegister & vd,const VRegister & vn,int shift)5193 void Assembler::sqrshrn(const VRegister& vd, const VRegister& vn, int shift) {
5194   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5195   VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
5196   NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRN);
5197 }
5198 
5199 
sqrshrn2(const VRegister & vd,const VRegister & vn,int shift)5200 void Assembler::sqrshrn2(const VRegister& vd, const VRegister& vn, int shift) {
5201   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5202   VIXL_ASSERT(vn.IsVector() && vd.IsQ());
5203   NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRN);
5204 }
5205 
5206 
sqshrun(const VRegister & vd,const VRegister & vn,int shift)5207 void Assembler::sqshrun(const VRegister& vd, const VRegister& vn, int shift) {
5208   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5209   VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
5210   NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRUN);
5211 }
5212 
5213 
sqshrun2(const VRegister & vd,const VRegister & vn,int shift)5214 void Assembler::sqshrun2(const VRegister& vd, const VRegister& vn, int shift) {
5215   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5216   VIXL_ASSERT(vn.IsVector() && vd.IsQ());
5217   NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRUN);
5218 }
5219 
5220 
sqrshrun(const VRegister & vd,const VRegister & vn,int shift)5221 void Assembler::sqrshrun(const VRegister& vd, const VRegister& vn, int shift) {
5222   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5223   VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
5224   NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRUN);
5225 }
5226 
5227 
sqrshrun2(const VRegister & vd,const VRegister & vn,int shift)5228 void Assembler::sqrshrun2(const VRegister& vd, const VRegister& vn, int shift) {
5229   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5230   VIXL_ASSERT(vn.IsVector() && vd.IsQ());
5231   NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRUN);
5232 }
5233 
5234 
uqshrn(const VRegister & vd,const VRegister & vn,int shift)5235 void Assembler::uqshrn(const VRegister& vd, const VRegister& vn, int shift) {
5236   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5237   VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
5238   NEONShiftImmediateN(vd, vn, shift, NEON_UQSHRN);
5239 }
5240 
5241 
uqshrn2(const VRegister & vd,const VRegister & vn,int shift)5242 void Assembler::uqshrn2(const VRegister& vd, const VRegister& vn, int shift) {
5243   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5244   VIXL_ASSERT(vn.IsVector() && vd.IsQ());
5245   NEONShiftImmediateN(vd, vn, shift, NEON_UQSHRN);
5246 }
5247 
5248 
uqrshrn(const VRegister & vd,const VRegister & vn,int shift)5249 void Assembler::uqrshrn(const VRegister& vd, const VRegister& vn, int shift) {
5250   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5251   VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
5252   NEONShiftImmediateN(vd, vn, shift, NEON_UQRSHRN);
5253 }
5254 
5255 
uqrshrn2(const VRegister & vd,const VRegister & vn,int shift)5256 void Assembler::uqrshrn2(const VRegister& vd, const VRegister& vn, int shift) {
5257   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5258   VIXL_ASSERT(vn.IsVector() && vd.IsQ());
5259   NEONShiftImmediateN(vd, vn, shift, NEON_UQRSHRN);
5260 }
5261 
smmla(const VRegister & vd,const VRegister & vn,const VRegister & vm)5262 void Assembler::smmla(const VRegister& vd, const VRegister& vn, const VRegister& vm) {
5263   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5264   VIXL_ASSERT(CPUHas(CPUFeatures::kI8MM));
5265   VIXL_ASSERT(vd.IsLaneSizeS());
5266   VIXL_ASSERT(vn.IsLaneSizeB() && vm.IsLaneSizeB());
5267 
5268   Emit(0x4e80a400 | Rd(vd) | Rn(vn) | Rm(vm));
5269 }
5270 
usmmla(const VRegister & vd,const VRegister & vn,const VRegister & vm)5271 void Assembler::usmmla(const VRegister& vd, const VRegister& vn, const VRegister& vm) {
5272   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5273   VIXL_ASSERT(CPUHas(CPUFeatures::kI8MM));
5274   VIXL_ASSERT(vd.IsLaneSizeS());
5275   VIXL_ASSERT(vn.IsLaneSizeB() && vm.IsLaneSizeB());
5276 
5277   Emit(0x4e80ac00 | Rd(vd) | Rn(vn) | Rm(vm));
5278 }
5279 
ummla(const VRegister & vd,const VRegister & vn,const VRegister & vm)5280 void Assembler::ummla(const VRegister& vd, const VRegister& vn, const VRegister& vm) {
5281   VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
5282   VIXL_ASSERT(CPUHas(CPUFeatures::kI8MM));
5283   VIXL_ASSERT(vd.IsLaneSizeS());
5284   VIXL_ASSERT(vn.IsLaneSizeB() && vm.IsLaneSizeB());
5285 
5286   Emit(0x6e80a400 | Rd(vd) | Rn(vn) | Rm(vm));
5287 }
5288 
5289 // Note:
5290 // For all ToImm instructions below, a difference in case
5291 // for the same letter indicates a negated bit.
5292 // If b is 1, then B is 0.
FP16ToImm8(Float16 imm)5293 uint32_t Assembler::FP16ToImm8(Float16 imm) {
5294   VIXL_ASSERT(IsImmFP16(imm));
5295   // Half: aBbb.cdef.gh00.0000 (16 bits)
5296   uint16_t bits = Float16ToRawbits(imm);
5297   // bit7: a000.0000
5298   uint16_t bit7 = ((bits >> 15) & 0x1) << 7;
5299   // bit6: 0b00.0000
5300   uint16_t bit6 = ((bits >> 13) & 0x1) << 6;
5301   // bit5_to_0: 00cd.efgh
5302   uint16_t bit5_to_0 = (bits >> 6) & 0x3f;
5303   uint32_t result = static_cast<uint32_t>(bit7 | bit6 | bit5_to_0);
5304   return result;
5305 }
5306 
5307 
ImmFP16(Float16 imm)5308 Instr Assembler::ImmFP16(Float16 imm) {
5309   return FP16ToImm8(imm) << ImmFP_offset;
5310 }
5311 
5312 
FP32ToImm8(float imm)5313 uint32_t Assembler::FP32ToImm8(float imm) {
5314   VIXL_ASSERT(IsImmFP32(imm));
5315   // bits: aBbb.bbbc.defg.h000.0000.0000.0000.0000
5316   uint32_t bits = FloatToRawbits(imm);
5317   // bit7: a000.0000
5318   uint32_t bit7 = ((bits >> 31) & 0x1) << 7;
5319   // bit6: 0b00.0000
5320   uint32_t bit6 = ((bits >> 29) & 0x1) << 6;
5321   // bit5_to_0: 00cd.efgh
5322   uint32_t bit5_to_0 = (bits >> 19) & 0x3f;
5323 
5324   return bit7 | bit6 | bit5_to_0;
5325 }
5326 
5327 
ImmFP32(float imm)5328 Instr Assembler::ImmFP32(float imm) { return FP32ToImm8(imm) << ImmFP_offset; }
5329 
5330 
FP64ToImm8(double imm)5331 uint32_t Assembler::FP64ToImm8(double imm) {
5332   VIXL_ASSERT(IsImmFP64(imm));
5333   // bits: aBbb.bbbb.bbcd.efgh.0000.0000.0000.0000
5334   //       0000.0000.0000.0000.0000.0000.0000.0000
5335   uint64_t bits = DoubleToRawbits(imm);
5336   // bit7: a000.0000
5337   uint64_t bit7 = ((bits >> 63) & 0x1) << 7;
5338   // bit6: 0b00.0000
5339   uint64_t bit6 = ((bits >> 61) & 0x1) << 6;
5340   // bit5_to_0: 00cd.efgh
5341   uint64_t bit5_to_0 = (bits >> 48) & 0x3f;
5342 
5343   return static_cast<uint32_t>(bit7 | bit6 | bit5_to_0);
5344 }
5345 
5346 
ImmFP64(double imm)5347 Instr Assembler::ImmFP64(double imm) { return FP64ToImm8(imm) << ImmFP_offset; }
5348 
5349 
5350 // Code generation helpers.
OneInstrMoveImmediateHelper(Assembler * assm,const Register & dst,uint64_t imm)5351 bool Assembler::OneInstrMoveImmediateHelper(Assembler* assm,
5352                                             const Register& dst,
5353                                             uint64_t imm) {
5354   bool emit_code = assm != NULL;
5355   unsigned n, imm_s, imm_r;
5356   int reg_size = dst.GetSizeInBits();
5357 
5358   if (IsImmMovz(imm, reg_size) && !dst.IsSP()) {
5359     // Immediate can be represented in a move zero instruction. Movz can't write
5360     // to the stack pointer.
5361     if (emit_code) {
5362       assm->movz(dst, imm);
5363     }
5364     return true;
5365   } else if (IsImmMovn(imm, reg_size) && !dst.IsSP()) {
5366     // Immediate can be represented in a move negative instruction. Movn can't
5367     // write to the stack pointer.
5368     if (emit_code) {
5369       assm->movn(dst, dst.Is64Bits() ? ~imm : (~imm & kWRegMask));
5370     }
5371     return true;
5372   } else if (IsImmLogical(imm, reg_size, &n, &imm_s, &imm_r)) {
5373     // Immediate can be represented in a logical orr instruction.
5374     VIXL_ASSERT(!dst.IsZero());
5375     if (emit_code) {
5376       assm->LogicalImmediate(dst,
5377                              AppropriateZeroRegFor(dst),
5378                              n,
5379                              imm_s,
5380                              imm_r,
5381                              ORR);
5382     }
5383     return true;
5384   }
5385   return false;
5386 }
5387 
5388 
MoveWide(const Register & rd,uint64_t imm,int shift,MoveWideImmediateOp mov_op)5389 void Assembler::MoveWide(const Register& rd,
5390                          uint64_t imm,
5391                          int shift,
5392                          MoveWideImmediateOp mov_op) {
5393   // Ignore the top 32 bits of an immediate if we're moving to a W register.
5394   if (rd.Is32Bits()) {
5395     // Check that the top 32 bits are zero (a positive 32-bit number) or top
5396     // 33 bits are one (a negative 32-bit number, sign extended to 64 bits).
5397     VIXL_ASSERT(((imm >> kWRegSize) == 0) ||
5398                 ((imm >> (kWRegSize - 1)) == 0x1ffffffff));
5399     imm &= kWRegMask;
5400   }
5401 
5402   if (shift >= 0) {
5403     // Explicit shift specified.
5404     VIXL_ASSERT((shift == 0) || (shift == 16) || (shift == 32) ||
5405                 (shift == 48));
5406     VIXL_ASSERT(rd.Is64Bits() || (shift == 0) || (shift == 16));
5407     shift /= 16;
5408   } else {
5409     // Calculate a new immediate and shift combination to encode the immediate
5410     // argument.
5411     VIXL_ASSERT(shift == -1);
5412     shift = 0;
5413     if ((imm & 0xffffffffffff0000) == 0) {
5414       // Nothing to do.
5415     } else if ((imm & 0xffffffff0000ffff) == 0) {
5416       imm >>= 16;
5417       shift = 1;
5418     } else if ((imm & 0xffff0000ffffffff) == 0) {
5419       VIXL_ASSERT(rd.Is64Bits());
5420       imm >>= 32;
5421       shift = 2;
5422     } else if ((imm & 0x0000ffffffffffff) == 0) {
5423       VIXL_ASSERT(rd.Is64Bits());
5424       imm >>= 48;
5425       shift = 3;
5426     }
5427   }
5428 
5429   VIXL_ASSERT(IsUint16(imm));
5430 
5431   Emit(SF(rd) | MoveWideImmediateFixed | mov_op | Rd(rd) | ImmMoveWide(imm) |
5432        ShiftMoveWide(shift));
5433 }
5434 
5435 
AddSub(const Register & rd,const Register & rn,const Operand & operand,FlagsUpdate S,AddSubOp op)5436 void Assembler::AddSub(const Register& rd,
5437                        const Register& rn,
5438                        const Operand& operand,
5439                        FlagsUpdate S,
5440                        AddSubOp op) {
5441   VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
5442   if (operand.IsImmediate()) {
5443     int64_t immediate = operand.GetImmediate();
5444     VIXL_ASSERT(IsImmAddSub(immediate));
5445     Instr dest_reg = (S == SetFlags) ? Rd(rd) : RdSP(rd);
5446     Emit(SF(rd) | AddSubImmediateFixed | op | Flags(S) |
5447          ImmAddSub(static_cast<int>(immediate)) | dest_reg | RnSP(rn));
5448   } else if (operand.IsShiftedRegister()) {
5449     VIXL_ASSERT(operand.GetRegister().GetSizeInBits() == rd.GetSizeInBits());
5450     VIXL_ASSERT(operand.GetShift() != ROR);
5451 
5452     // For instructions of the form:
5453     //   add/sub   wsp, <Wn>, <Wm> [, LSL #0-3 ]
5454     //   add/sub   <Wd>, wsp, <Wm> [, LSL #0-3 ]
5455     //   add/sub   wsp, wsp, <Wm> [, LSL #0-3 ]
5456     //   adds/subs <Wd>, wsp, <Wm> [, LSL #0-3 ]
5457     // or their 64-bit register equivalents, convert the operand from shifted to
5458     // extended register mode, and emit an add/sub extended instruction.
5459     if (rn.IsSP() || rd.IsSP()) {
5460       VIXL_ASSERT(!(rd.IsSP() && (S == SetFlags)));
5461       DataProcExtendedRegister(rd,
5462                                rn,
5463                                operand.ToExtendedRegister(),
5464                                S,
5465                                AddSubExtendedFixed | op);
5466     } else {
5467       DataProcShiftedRegister(rd, rn, operand, S, AddSubShiftedFixed | op);
5468     }
5469   } else {
5470     VIXL_ASSERT(operand.IsExtendedRegister());
5471     DataProcExtendedRegister(rd, rn, operand, S, AddSubExtendedFixed | op);
5472   }
5473 }
5474 
5475 
AddSubWithCarry(const Register & rd,const Register & rn,const Operand & operand,FlagsUpdate S,AddSubWithCarryOp op)5476 void Assembler::AddSubWithCarry(const Register& rd,
5477                                 const Register& rn,
5478                                 const Operand& operand,
5479                                 FlagsUpdate S,
5480                                 AddSubWithCarryOp op) {
5481   VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
5482   VIXL_ASSERT(rd.GetSizeInBits() == operand.GetRegister().GetSizeInBits());
5483   VIXL_ASSERT(operand.IsShiftedRegister() && (operand.GetShiftAmount() == 0));
5484   Emit(SF(rd) | op | Flags(S) | Rm(operand.GetRegister()) | Rn(rn) | Rd(rd));
5485 }
5486 
5487 
hlt(int code)5488 void Assembler::hlt(int code) {
5489   VIXL_ASSERT(IsUint16(code));
5490   Emit(HLT | ImmException(code));
5491 }
5492 
5493 
brk(int code)5494 void Assembler::brk(int code) {
5495   VIXL_ASSERT(IsUint16(code));
5496   Emit(BRK | ImmException(code));
5497 }
5498 
5499 
svc(int code)5500 void Assembler::svc(int code) { Emit(SVC | ImmException(code)); }
5501 
udf(int code)5502 void Assembler::udf(int code) { Emit(UDF | ImmUdf(code)); }
5503 
5504 
5505 // TODO(all): The third parameter should be passed by reference but gcc 4.8.2
5506 // reports a bogus uninitialised warning then.
Logical(const Register & rd,const Register & rn,const Operand operand,LogicalOp op)5507 void Assembler::Logical(const Register& rd,
5508                         const Register& rn,
5509                         const Operand operand,
5510                         LogicalOp op) {
5511   VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
5512   if (operand.IsImmediate()) {
5513     int64_t immediate = operand.GetImmediate();
5514     unsigned reg_size = rd.GetSizeInBits();
5515 
5516     VIXL_ASSERT(immediate != 0);
5517     VIXL_ASSERT(immediate != -1);
5518     VIXL_ASSERT(rd.Is64Bits() || IsUint32(immediate));
5519 
5520     // If the operation is NOT, invert the operation and immediate.
5521     if ((op & NOT) == NOT) {
5522       op = static_cast<LogicalOp>(op & ~NOT);
5523       immediate = rd.Is64Bits() ? ~immediate : (~immediate & kWRegMask);
5524     }
5525 
5526     unsigned n, imm_s, imm_r;
5527     if (IsImmLogical(immediate, reg_size, &n, &imm_s, &imm_r)) {
5528       // Immediate can be encoded in the instruction.
5529       LogicalImmediate(rd, rn, n, imm_s, imm_r, op);
5530     } else {
5531       // This case is handled in the macro assembler.
5532       VIXL_UNREACHABLE();
5533     }
5534   } else {
5535     VIXL_ASSERT(operand.IsShiftedRegister());
5536     VIXL_ASSERT(operand.GetRegister().GetSizeInBits() == rd.GetSizeInBits());
5537     Instr dp_op = static_cast<Instr>(op | LogicalShiftedFixed);
5538     DataProcShiftedRegister(rd, rn, operand, LeaveFlags, dp_op);
5539   }
5540 }
5541 
5542 
LogicalImmediate(const Register & rd,const Register & rn,unsigned n,unsigned imm_s,unsigned imm_r,LogicalOp op)5543 void Assembler::LogicalImmediate(const Register& rd,
5544                                  const Register& rn,
5545                                  unsigned n,
5546                                  unsigned imm_s,
5547                                  unsigned imm_r,
5548                                  LogicalOp op) {
5549   unsigned reg_size = rd.GetSizeInBits();
5550   Instr dest_reg = (op == ANDS) ? Rd(rd) : RdSP(rd);
5551   Emit(SF(rd) | LogicalImmediateFixed | op | BitN(n, reg_size) |
5552        ImmSetBits(imm_s, reg_size) | ImmRotate(imm_r, reg_size) | dest_reg |
5553        Rn(rn));
5554 }
5555 
5556 
ConditionalCompare(const Register & rn,const Operand & operand,StatusFlags nzcv,Condition cond,ConditionalCompareOp op)5557 void Assembler::ConditionalCompare(const Register& rn,
5558                                    const Operand& operand,
5559                                    StatusFlags nzcv,
5560                                    Condition cond,
5561                                    ConditionalCompareOp op) {
5562   Instr ccmpop;
5563   if (operand.IsImmediate()) {
5564     int64_t immediate = operand.GetImmediate();
5565     VIXL_ASSERT(IsImmConditionalCompare(immediate));
5566     ccmpop = ConditionalCompareImmediateFixed | op |
5567              ImmCondCmp(static_cast<unsigned>(immediate));
5568   } else {
5569     VIXL_ASSERT(operand.IsShiftedRegister() && (operand.GetShiftAmount() == 0));
5570     ccmpop = ConditionalCompareRegisterFixed | op | Rm(operand.GetRegister());
5571   }
5572   Emit(SF(rn) | ccmpop | Cond(cond) | Rn(rn) | Nzcv(nzcv));
5573 }
5574 
5575 
DataProcessing1Source(const Register & rd,const Register & rn,DataProcessing1SourceOp op)5576 void Assembler::DataProcessing1Source(const Register& rd,
5577                                       const Register& rn,
5578                                       DataProcessing1SourceOp op) {
5579   VIXL_ASSERT(rd.GetSizeInBits() == rn.GetSizeInBits());
5580   Emit(SF(rn) | op | Rn(rn) | Rd(rd));
5581 }
5582 
5583 
FPDataProcessing1Source(const VRegister & vd,const VRegister & vn,FPDataProcessing1SourceOp op)5584 void Assembler::FPDataProcessing1Source(const VRegister& vd,
5585                                         const VRegister& vn,
5586                                         FPDataProcessing1SourceOp op) {
5587   VIXL_ASSERT(vd.Is1H() || vd.Is1S() || vd.Is1D());
5588   Emit(FPType(vn) | op | Rn(vn) | Rd(vd));
5589 }
5590 
5591 
FPDataProcessing3Source(const VRegister & vd,const VRegister & vn,const VRegister & vm,const VRegister & va,FPDataProcessing3SourceOp op)5592 void Assembler::FPDataProcessing3Source(const VRegister& vd,
5593                                         const VRegister& vn,
5594                                         const VRegister& vm,
5595                                         const VRegister& va,
5596                                         FPDataProcessing3SourceOp op) {
5597   VIXL_ASSERT(vd.Is1H() || vd.Is1S() || vd.Is1D());
5598   VIXL_ASSERT(AreSameSizeAndType(vd, vn, vm, va));
5599   Emit(FPType(vd) | op | Rm(vm) | Rn(vn) | Rd(vd) | Ra(va));
5600 }
5601 
5602 
NEONModifiedImmShiftLsl(const VRegister & vd,const int imm8,const int left_shift,NEONModifiedImmediateOp op)5603 void Assembler::NEONModifiedImmShiftLsl(const VRegister& vd,
5604                                         const int imm8,
5605                                         const int left_shift,
5606                                         NEONModifiedImmediateOp op) {
5607   VIXL_ASSERT(vd.Is8B() || vd.Is16B() || vd.Is4H() || vd.Is8H() || vd.Is2S() ||
5608               vd.Is4S());
5609   VIXL_ASSERT((left_shift == 0) || (left_shift == 8) || (left_shift == 16) ||
5610               (left_shift == 24));
5611   VIXL_ASSERT(IsUint8(imm8));
5612 
5613   int cmode_1, cmode_2, cmode_3;
5614   if (vd.Is8B() || vd.Is16B()) {
5615     VIXL_ASSERT(op == NEONModifiedImmediate_MOVI);
5616     cmode_1 = 1;
5617     cmode_2 = 1;
5618     cmode_3 = 1;
5619   } else {
5620     cmode_1 = (left_shift >> 3) & 1;
5621     cmode_2 = left_shift >> 4;
5622     cmode_3 = 0;
5623     if (vd.Is4H() || vd.Is8H()) {
5624       VIXL_ASSERT((left_shift == 0) || (left_shift == 8));
5625       cmode_3 = 1;
5626     }
5627   }
5628   int cmode = (cmode_3 << 3) | (cmode_2 << 2) | (cmode_1 << 1);
5629 
5630   int q = vd.IsQ() ? NEON_Q : 0;
5631 
5632   Emit(q | op | ImmNEONabcdefgh(imm8) | NEONCmode(cmode) | Rd(vd));
5633 }
5634 
5635 
NEONModifiedImmShiftMsl(const VRegister & vd,const int imm8,const int shift_amount,NEONModifiedImmediateOp op)5636 void Assembler::NEONModifiedImmShiftMsl(const VRegister& vd,
5637                                         const int imm8,
5638                                         const int shift_amount,
5639                                         NEONModifiedImmediateOp op) {
5640   VIXL_ASSERT(vd.Is2S() || vd.Is4S());
5641   VIXL_ASSERT((shift_amount == 8) || (shift_amount == 16));
5642   VIXL_ASSERT(IsUint8(imm8));
5643 
5644   int cmode_0 = (shift_amount >> 4) & 1;
5645   int cmode = 0xc | cmode_0;
5646 
5647   int q = vd.IsQ() ? NEON_Q : 0;
5648 
5649   Emit(q | op | ImmNEONabcdefgh(imm8) | NEONCmode(cmode) | Rd(vd));
5650 }
5651 
5652 
EmitShift(const Register & rd,const Register & rn,Shift shift,unsigned shift_amount)5653 void Assembler::EmitShift(const Register& rd,
5654                           const Register& rn,
5655                           Shift shift,
5656                           unsigned shift_amount) {
5657   switch (shift) {
5658     case LSL:
5659       lsl(rd, rn, shift_amount);
5660       break;
5661     case LSR:
5662       lsr(rd, rn, shift_amount);
5663       break;
5664     case ASR:
5665       asr(rd, rn, shift_amount);
5666       break;
5667     case ROR:
5668       ror(rd, rn, shift_amount);
5669       break;
5670     default:
5671       VIXL_UNREACHABLE();
5672   }
5673 }
5674 
5675 
EmitExtendShift(const Register & rd,const Register & rn,Extend extend,unsigned left_shift)5676 void Assembler::EmitExtendShift(const Register& rd,
5677                                 const Register& rn,
5678                                 Extend extend,
5679                                 unsigned left_shift) {
5680   VIXL_ASSERT(rd.GetSizeInBits() >= rn.GetSizeInBits());
5681   unsigned reg_size = rd.GetSizeInBits();
5682   // Use the correct size of register.
5683   Register rn_ = Register(rn.GetCode(), rd.GetSizeInBits());
5684   // Bits extracted are high_bit:0.
5685   unsigned high_bit = (8 << (extend & 0x3)) - 1;
5686   // Number of bits left in the result that are not introduced by the shift.
5687   unsigned non_shift_bits = (reg_size - left_shift) & (reg_size - 1);
5688 
5689   if ((non_shift_bits > high_bit) || (non_shift_bits == 0)) {
5690     switch (extend) {
5691       case UXTB:
5692       case UXTH:
5693       case UXTW:
5694         ubfm(rd, rn_, non_shift_bits, high_bit);
5695         break;
5696       case SXTB:
5697       case SXTH:
5698       case SXTW:
5699         sbfm(rd, rn_, non_shift_bits, high_bit);
5700         break;
5701       case UXTX:
5702       case SXTX: {
5703         VIXL_ASSERT(rn.GetSizeInBits() == kXRegSize);
5704         // Nothing to extend. Just shift.
5705         lsl(rd, rn_, left_shift);
5706         break;
5707       }
5708       default:
5709         VIXL_UNREACHABLE();
5710     }
5711   } else {
5712     // No need to extend as the extended bits would be shifted away.
5713     lsl(rd, rn_, left_shift);
5714   }
5715 }
5716 
5717 
DataProcShiftedRegister(const Register & rd,const Register & rn,const Operand & operand,FlagsUpdate S,Instr op)5718 void Assembler::DataProcShiftedRegister(const Register& rd,
5719                                         const Register& rn,
5720                                         const Operand& operand,
5721                                         FlagsUpdate S,
5722                                         Instr op) {
5723   VIXL_ASSERT(operand.IsShiftedRegister());
5724   VIXL_ASSERT(rn.Is64Bits() ||
5725               (rn.Is32Bits() && IsUint5(operand.GetShiftAmount())));
5726   Emit(SF(rd) | op | Flags(S) | ShiftDP(operand.GetShift()) |
5727        ImmDPShift(operand.GetShiftAmount()) | Rm(operand.GetRegister()) |
5728        Rn(rn) | Rd(rd));
5729 }
5730 
5731 
DataProcExtendedRegister(const Register & rd,const Register & rn,const Operand & operand,FlagsUpdate S,Instr op)5732 void Assembler::DataProcExtendedRegister(const Register& rd,
5733                                          const Register& rn,
5734                                          const Operand& operand,
5735                                          FlagsUpdate S,
5736                                          Instr op) {
5737   Instr dest_reg = (S == SetFlags) ? Rd(rd) : RdSP(rd);
5738   Emit(SF(rd) | op | Flags(S) | Rm(operand.GetRegister()) |
5739        ExtendMode(operand.GetExtend()) |
5740        ImmExtendShift(operand.GetShiftAmount()) | dest_reg | RnSP(rn));
5741 }
5742 
5743 
LoadStoreMemOperand(const MemOperand & addr,unsigned access_size_in_bytes_log2,LoadStoreScalingOption option)5744 Instr Assembler::LoadStoreMemOperand(const MemOperand& addr,
5745                                      unsigned access_size_in_bytes_log2,
5746                                      LoadStoreScalingOption option) {
5747   Instr base = RnSP(addr.GetBaseRegister());
5748   int64_t offset = addr.GetOffset();
5749 
5750   if (addr.IsImmediateOffset()) {
5751     bool prefer_unscaled =
5752         (option == PreferUnscaledOffset) || (option == RequireUnscaledOffset);
5753     if (prefer_unscaled && IsImmLSUnscaled(offset)) {
5754       // Use the unscaled addressing mode.
5755       return base | LoadStoreUnscaledOffsetFixed | ImmLS(offset);
5756     }
5757 
5758     if ((option != RequireUnscaledOffset) &&
5759         IsImmLSScaled(offset, access_size_in_bytes_log2)) {
5760       // We need `offset` to be positive for the shift to be well-defined.
5761       // IsImmLSScaled should check this.
5762       VIXL_ASSERT(offset >= 0);
5763       // Use the scaled addressing mode.
5764       return base | LoadStoreUnsignedOffsetFixed |
5765              ImmLSUnsigned(offset >> access_size_in_bytes_log2);
5766     }
5767 
5768     if ((option != RequireScaledOffset) && IsImmLSUnscaled(offset)) {
5769       // Use the unscaled addressing mode.
5770       return base | LoadStoreUnscaledOffsetFixed | ImmLS(offset);
5771     }
5772   }
5773 
5774   // All remaining addressing modes are register-offset, pre-indexed or
5775   // post-indexed modes.
5776   VIXL_ASSERT((option != RequireUnscaledOffset) &&
5777               (option != RequireScaledOffset));
5778 
5779   if (addr.IsRegisterOffset()) {
5780     Extend ext = addr.GetExtend();
5781     Shift shift = addr.GetShift();
5782     unsigned shift_amount = addr.GetShiftAmount();
5783 
5784     // LSL is encoded in the option field as UXTX.
5785     if (shift == LSL) {
5786       ext = UXTX;
5787     }
5788 
5789     // Shifts are encoded in one bit, indicating a left shift by the memory
5790     // access size.
5791     VIXL_ASSERT((shift_amount == 0) || (shift_amount == access_size_in_bytes_log2));
5792     return base | LoadStoreRegisterOffsetFixed | Rm(addr.GetRegisterOffset()) |
5793            ExtendMode(ext) | ImmShiftLS((shift_amount > 0) ? 1 : 0);
5794   }
5795 
5796   if (addr.IsImmediatePreIndex() && IsImmLSUnscaled(offset)) {
5797     return base | LoadStorePreIndexFixed | ImmLS(offset);
5798   }
5799 
5800   if (addr.IsImmediatePostIndex() && IsImmLSUnscaled(offset)) {
5801     return base | LoadStorePostIndexFixed | ImmLS(offset);
5802   }
5803 
5804   // If this point is reached, the MemOperand (addr) cannot be encoded.
5805   VIXL_UNREACHABLE();
5806   return 0;
5807 }
5808 
5809 
LoadStore(const CPURegister & rt,const MemOperand & addr,LoadStoreOp op,LoadStoreScalingOption option)5810 void Assembler::LoadStore(const CPURegister& rt,
5811                           const MemOperand& addr,
5812                           LoadStoreOp op,
5813                           LoadStoreScalingOption option) {
5814   VIXL_ASSERT(CPUHas(rt));
5815   Emit(op | Rt(rt) | LoadStoreMemOperand(addr, CalcLSDataSize(op), option));
5816 }
5817 
LoadStorePAC(const Register & xt,const MemOperand & addr,LoadStorePACOp op)5818 void Assembler::LoadStorePAC(const Register& xt,
5819                              const MemOperand& addr,
5820                              LoadStorePACOp op) {
5821   VIXL_ASSERT(xt.Is64Bits());
5822   VIXL_ASSERT(addr.IsImmediateOffset() || addr.IsImmediatePreIndex());
5823 
5824   Instr pac_op = op;
5825   if (addr.IsImmediatePreIndex()) {
5826     pac_op |= LoadStorePACPreBit;
5827   }
5828 
5829   Instr base = RnSP(addr.GetBaseRegister());
5830   int64_t offset = addr.GetOffset();
5831 
5832   Emit(pac_op | Rt(xt) | base | ImmLSPAC(static_cast<int>(offset)));
5833 }
5834 
5835 
Prefetch(int op,const MemOperand & addr,LoadStoreScalingOption option)5836 void Assembler::Prefetch(int op,
5837                          const MemOperand& addr,
5838                          LoadStoreScalingOption option) {
5839   VIXL_ASSERT(addr.IsRegisterOffset() || addr.IsImmediateOffset());
5840 
5841   Instr prfop = ImmPrefetchOperation(op);
5842   Emit(PRFM | prfop | LoadStoreMemOperand(addr, kXRegSizeInBytesLog2, option));
5843 }
5844 
Prefetch(PrefetchOperation op,const MemOperand & addr,LoadStoreScalingOption option)5845 void Assembler::Prefetch(PrefetchOperation op,
5846                          const MemOperand& addr,
5847                          LoadStoreScalingOption option) {
5848   // Passing unnamed values in 'op' is undefined behaviour in C++.
5849   VIXL_ASSERT(IsNamedPrefetchOperation(op));
5850   Prefetch(static_cast<int>(op), addr, option);
5851 }
5852 
5853 
IsImmAddSub(int64_t immediate)5854 bool Assembler::IsImmAddSub(int64_t immediate) {
5855   return IsUint12(immediate) ||
5856          (IsUint12(immediate >> 12) && ((immediate & 0xfff) == 0));
5857 }
5858 
5859 
IsImmConditionalCompare(int64_t immediate)5860 bool Assembler::IsImmConditionalCompare(int64_t immediate) {
5861   return IsUint5(immediate);
5862 }
5863 
5864 
IsImmFP16(Float16 imm)5865 bool Assembler::IsImmFP16(Float16 imm) {
5866   // Valid values will have the form:
5867   // aBbb.cdef.gh00.000
5868   uint16_t bits = Float16ToRawbits(imm);
5869   // bits[6..0] are cleared.
5870   if ((bits & 0x3f) != 0) {
5871     return false;
5872   }
5873 
5874   // bits[13..12] are all set or all cleared.
5875   uint16_t b_pattern = (bits >> 12) & 0x03;
5876   if (b_pattern != 0 && b_pattern != 0x03) {
5877     return false;
5878   }
5879 
5880   // bit[15] and bit[14] are opposite.
5881   if (((bits ^ (bits << 1)) & 0x4000) == 0) {
5882     return false;
5883   }
5884 
5885   return true;
5886 }
5887 
5888 
IsImmFP32(float imm)5889 bool Assembler::IsImmFP32(float imm) {
5890   // Valid values will have the form:
5891   // aBbb.bbbc.defg.h000.0000.0000.0000.0000
5892   uint32_t bits = FloatToRawbits(imm);
5893   // bits[19..0] are cleared.
5894   if ((bits & 0x7ffff) != 0) {
5895     return false;
5896   }
5897 
5898   // bits[29..25] are all set or all cleared.
5899   uint32_t b_pattern = (bits >> 16) & 0x3e00;
5900   if (b_pattern != 0 && b_pattern != 0x3e00) {
5901     return false;
5902   }
5903 
5904   // bit[30] and bit[29] are opposite.
5905   if (((bits ^ (bits << 1)) & 0x40000000) == 0) {
5906     return false;
5907   }
5908 
5909   return true;
5910 }
5911 
5912 
IsImmFP64(double imm)5913 bool Assembler::IsImmFP64(double imm) {
5914   // Valid values will have the form:
5915   // aBbb.bbbb.bbcd.efgh.0000.0000.0000.0000
5916   // 0000.0000.0000.0000.0000.0000.0000.0000
5917   uint64_t bits = DoubleToRawbits(imm);
5918   // bits[47..0] are cleared.
5919   if ((bits & 0x0000ffffffffffff) != 0) {
5920     return false;
5921   }
5922 
5923   // bits[61..54] are all set or all cleared.
5924   uint32_t b_pattern = (bits >> 48) & 0x3fc0;
5925   if ((b_pattern != 0) && (b_pattern != 0x3fc0)) {
5926     return false;
5927   }
5928 
5929   // bit[62] and bit[61] are opposite.
5930   if (((bits ^ (bits << 1)) & (UINT64_C(1) << 62)) == 0) {
5931     return false;
5932   }
5933 
5934   return true;
5935 }
5936 
5937 
IsImmLSPair(int64_t offset,unsigned access_size_in_bytes_log2)5938 bool Assembler::IsImmLSPair(int64_t offset, unsigned access_size_in_bytes_log2) {
5939   VIXL_ASSERT(access_size_in_bytes_log2 <= kQRegSizeInBytesLog2);
5940   return IsMultiple(offset, 1 << access_size_in_bytes_log2) &&
5941          IsInt7(offset / (1 << access_size_in_bytes_log2));
5942 }
5943 
5944 
IsImmLSScaled(int64_t offset,unsigned access_size_in_bytes_log2)5945 bool Assembler::IsImmLSScaled(int64_t offset, unsigned access_size_in_bytes_log2) {
5946   VIXL_ASSERT(access_size_in_bytes_log2 <= kQRegSizeInBytesLog2);
5947   return IsMultiple(offset, 1 << access_size_in_bytes_log2) &&
5948          IsUint12(offset / (1 << access_size_in_bytes_log2));
5949 }
5950 
5951 
IsImmLSUnscaled(int64_t offset)5952 bool Assembler::IsImmLSUnscaled(int64_t offset) { return IsInt9(offset); }
5953 
5954 
5955 // The movn instruction can generate immediates containing an arbitrary 16-bit
5956 // value, with remaining bits set, eg. 0xffff1234, 0xffff1234ffffffff.
IsImmMovn(uint64_t imm,unsigned reg_size)5957 bool Assembler::IsImmMovn(uint64_t imm, unsigned reg_size) {
5958   return IsImmMovz(~imm, reg_size);
5959 }
5960 
5961 
5962 // The movz instruction can generate immediates containing an arbitrary 16-bit
5963 // value, with remaining bits clear, eg. 0x00001234, 0x0000123400000000.
IsImmMovz(uint64_t imm,unsigned reg_size)5964 bool Assembler::IsImmMovz(uint64_t imm, unsigned reg_size) {
5965   VIXL_ASSERT((reg_size == kXRegSize) || (reg_size == kWRegSize));
5966   return CountClearHalfWords(imm, reg_size) >= ((reg_size / 16) - 1);
5967 }
5968 
5969 
5970 // Test if a given value can be encoded in the immediate field of a logical
5971 // instruction.
5972 // If it can be encoded, the function returns true, and values pointed to by n,
5973 // imm_s and imm_r are updated with immediates encoded in the format required
5974 // by the corresponding fields in the logical instruction.
5975 // If it can not be encoded, the function returns false, and the values pointed
5976 // to by n, imm_s and imm_r are undefined.
IsImmLogical(uint64_t value,unsigned width,unsigned * n,unsigned * imm_s,unsigned * imm_r)5977 bool Assembler::IsImmLogical(uint64_t value,
5978                              unsigned width,
5979                              unsigned* n,
5980                              unsigned* imm_s,
5981                              unsigned* imm_r) {
5982   VIXL_ASSERT((width == kBRegSize) || (width == kHRegSize) ||
5983               (width == kSRegSize) || (width == kDRegSize));
5984 
5985   bool negate = false;
5986 
5987   // Logical immediates are encoded using parameters n, imm_s and imm_r using
5988   // the following table:
5989   //
5990   //    N   imms    immr    size        S             R
5991   //    1  ssssss  rrrrrr    64    UInt(ssssss)  UInt(rrrrrr)
5992   //    0  0sssss  xrrrrr    32    UInt(sssss)   UInt(rrrrr)
5993   //    0  10ssss  xxrrrr    16    UInt(ssss)    UInt(rrrr)
5994   //    0  110sss  xxxrrr     8    UInt(sss)     UInt(rrr)
5995   //    0  1110ss  xxxxrr     4    UInt(ss)      UInt(rr)
5996   //    0  11110s  xxxxxr     2    UInt(s)       UInt(r)
5997   // (s bits must not be all set)
5998   //
5999   // A pattern is constructed of size bits, where the least significant S+1 bits
6000   // are set. The pattern is rotated right by R, and repeated across a 32 or
6001   // 64-bit value, depending on destination register width.
6002   //
6003   // Put another way: the basic format of a logical immediate is a single
6004   // contiguous stretch of 1 bits, repeated across the whole word at intervals
6005   // given by a power of 2. To identify them quickly, we first locate the
6006   // lowest stretch of 1 bits, then the next 1 bit above that; that combination
6007   // is different for every logical immediate, so it gives us all the
6008   // information we need to identify the only logical immediate that our input
6009   // could be, and then we simply check if that's the value we actually have.
6010   //
6011   // (The rotation parameter does give the possibility of the stretch of 1 bits
6012   // going 'round the end' of the word. To deal with that, we observe that in
6013   // any situation where that happens the bitwise NOT of the value is also a
6014   // valid logical immediate. So we simply invert the input whenever its low bit
6015   // is set, and then we know that the rotated case can't arise.)
6016 
6017   if (value & 1) {
6018     // If the low bit is 1, negate the value, and set a flag to remember that we
6019     // did (so that we can adjust the return values appropriately).
6020     negate = true;
6021     value = ~value;
6022   }
6023 
6024   if (width <= kWRegSize) {
6025     // To handle 8/16/32-bit logical immediates, the very easiest thing is to repeat
6026     // the input value to fill a 64-bit word. The correct encoding of that as a
6027     // logical immediate will also be the correct encoding of the value.
6028 
6029     // Avoid making the assumption that the most-significant 56/48/32 bits are zero by
6030     // shifting the value left and duplicating it.
6031     for (unsigned bits = width; bits <= kWRegSize; bits *= 2) {
6032       value <<= bits;
6033       uint64_t mask = (UINT64_C(1) << bits) - 1;
6034       value |= ((value >> bits) & mask);
6035     }
6036   }
6037 
6038   // The basic analysis idea: imagine our input word looks like this.
6039   //
6040   //    0011111000111110001111100011111000111110001111100011111000111110
6041   //                                                          c  b    a
6042   //                                                          |<--d-->|
6043   //
6044   // We find the lowest set bit (as an actual power-of-2 value, not its index)
6045   // and call it a. Then we add a to our original number, which wipes out the
6046   // bottommost stretch of set bits and replaces it with a 1 carried into the
6047   // next zero bit. Then we look for the new lowest set bit, which is in
6048   // position b, and subtract it, so now our number is just like the original
6049   // but with the lowest stretch of set bits completely gone. Now we find the
6050   // lowest set bit again, which is position c in the diagram above. Then we'll
6051   // measure the distance d between bit positions a and c (using CLZ), and that
6052   // tells us that the only valid logical immediate that could possibly be equal
6053   // to this number is the one in which a stretch of bits running from a to just
6054   // below b is replicated every d bits.
6055   uint64_t a = LowestSetBit(value);
6056   uint64_t value_plus_a = value + a;
6057   uint64_t b = LowestSetBit(value_plus_a);
6058   uint64_t value_plus_a_minus_b = value_plus_a - b;
6059   uint64_t c = LowestSetBit(value_plus_a_minus_b);
6060 
6061   int d, clz_a, out_n;
6062   uint64_t mask;
6063 
6064   if (c != 0) {
6065     // The general case, in which there is more than one stretch of set bits.
6066     // Compute the repeat distance d, and set up a bitmask covering the basic
6067     // unit of repetition (i.e. a word with the bottom d bits set). Also, in all
6068     // of these cases the N bit of the output will be zero.
6069     clz_a = CountLeadingZeros(a, kXRegSize);
6070     int clz_c = CountLeadingZeros(c, kXRegSize);
6071     d = clz_a - clz_c;
6072     mask = ((UINT64_C(1) << d) - 1);
6073     out_n = 0;
6074   } else {
6075     // Handle degenerate cases.
6076     //
6077     // If any of those 'find lowest set bit' operations didn't find a set bit at
6078     // all, then the word will have been zero thereafter, so in particular the
6079     // last lowest_set_bit operation will have returned zero. So we can test for
6080     // all the special case conditions in one go by seeing if c is zero.
6081     if (a == 0) {
6082       // The input was zero (or all 1 bits, which will come to here too after we
6083       // inverted it at the start of the function), for which we just return
6084       // false.
6085       return false;
6086     } else {
6087       // Otherwise, if c was zero but a was not, then there's just one stretch
6088       // of set bits in our word, meaning that we have the trivial case of
6089       // d == 64 and only one 'repetition'. Set up all the same variables as in
6090       // the general case above, and set the N bit in the output.
6091       clz_a = CountLeadingZeros(a, kXRegSize);
6092       d = 64;
6093       mask = ~UINT64_C(0);
6094       out_n = 1;
6095     }
6096   }
6097 
6098   // If the repeat period d is not a power of two, it can't be encoded.
6099   if (!IsPowerOf2(d)) {
6100     return false;
6101   }
6102 
6103   if (((b - a) & ~mask) != 0) {
6104     // If the bit stretch (b - a) does not fit within the mask derived from the
6105     // repeat period, then fail.
6106     return false;
6107   }
6108 
6109   // The only possible option is b - a repeated every d bits. Now we're going to
6110   // actually construct the valid logical immediate derived from that
6111   // specification, and see if it equals our original input.
6112   //
6113   // To repeat a value every d bits, we multiply it by a number of the form
6114   // (1 + 2^d + 2^(2d) + ...), i.e. 0x0001000100010001 or similar. These can
6115   // be derived using a table lookup on CLZ(d).
6116   static const uint64_t multipliers[] = {
6117       0x0000000000000001UL,
6118       0x0000000100000001UL,
6119       0x0001000100010001UL,
6120       0x0101010101010101UL,
6121       0x1111111111111111UL,
6122       0x5555555555555555UL,
6123   };
6124   uint64_t multiplier = multipliers[CountLeadingZeros(d, kXRegSize) - 57];
6125   uint64_t candidate = (b - a) * multiplier;
6126 
6127   if (value != candidate) {
6128     // The candidate pattern doesn't match our input value, so fail.
6129     return false;
6130   }
6131 
6132   // We have a match! This is a valid logical immediate, so now we have to
6133   // construct the bits and pieces of the instruction encoding that generates
6134   // it.
6135 
6136   // Count the set bits in our basic stretch. The special case of clz(0) == -1
6137   // makes the answer come out right for stretches that reach the very top of
6138   // the word (e.g. numbers like 0xffffc00000000000).
6139   int clz_b = (b == 0) ? -1 : CountLeadingZeros(b, kXRegSize);
6140   int s = clz_a - clz_b;
6141 
6142   // Decide how many bits to rotate right by, to put the low bit of that basic
6143   // stretch in position a.
6144   int r;
6145   if (negate) {
6146     // If we inverted the input right at the start of this function, here's
6147     // where we compensate: the number of set bits becomes the number of clear
6148     // bits, and the rotation count is based on position b rather than position
6149     // a (since b is the location of the 'lowest' 1 bit after inversion).
6150     s = d - s;
6151     r = (clz_b + 1) & (d - 1);
6152   } else {
6153     r = (clz_a + 1) & (d - 1);
6154   }
6155 
6156   // Now we're done, except for having to encode the S output in such a way that
6157   // it gives both the number of set bits and the length of the repeated
6158   // segment. The s field is encoded like this:
6159   //
6160   //     imms    size        S
6161   //    ssssss    64    UInt(ssssss)
6162   //    0sssss    32    UInt(sssss)
6163   //    10ssss    16    UInt(ssss)
6164   //    110sss     8    UInt(sss)
6165   //    1110ss     4    UInt(ss)
6166   //    11110s     2    UInt(s)
6167   //
6168   // So we 'or' (2 * -d) with our computed s to form imms.
6169   if ((n != NULL) || (imm_s != NULL) || (imm_r != NULL)) {
6170     *n = out_n;
6171     *imm_s = ((2 * -d) | (s - 1)) & 0x3f;
6172     *imm_r = r;
6173   }
6174 
6175   return true;
6176 }
6177 
6178 
LoadOpFor(const CPURegister & rt)6179 LoadStoreOp Assembler::LoadOpFor(const CPURegister& rt) {
6180   VIXL_ASSERT(rt.IsValid());
6181   if (rt.IsRegister()) {
6182     return rt.Is64Bits() ? LDR_x : LDR_w;
6183   } else {
6184     VIXL_ASSERT(rt.IsVRegister());
6185     switch (rt.GetSizeInBits()) {
6186       case kBRegSize:
6187         return LDR_b;
6188       case kHRegSize:
6189         return LDR_h;
6190       case kSRegSize:
6191         return LDR_s;
6192       case kDRegSize:
6193         return LDR_d;
6194       default:
6195         VIXL_ASSERT(rt.IsQ());
6196         return LDR_q;
6197     }
6198   }
6199 }
6200 
6201 
StoreOpFor(const CPURegister & rt)6202 LoadStoreOp Assembler::StoreOpFor(const CPURegister& rt) {
6203   VIXL_ASSERT(rt.IsValid());
6204   if (rt.IsRegister()) {
6205     return rt.Is64Bits() ? STR_x : STR_w;
6206   } else {
6207     VIXL_ASSERT(rt.IsVRegister());
6208     switch (rt.GetSizeInBits()) {
6209       case kBRegSize:
6210         return STR_b;
6211       case kHRegSize:
6212         return STR_h;
6213       case kSRegSize:
6214         return STR_s;
6215       case kDRegSize:
6216         return STR_d;
6217       default:
6218         VIXL_ASSERT(rt.IsQ());
6219         return STR_q;
6220     }
6221   }
6222 }
6223 
6224 
StorePairOpFor(const CPURegister & rt,const CPURegister & rt2)6225 LoadStorePairOp Assembler::StorePairOpFor(const CPURegister& rt,
6226                                           const CPURegister& rt2) {
6227   VIXL_ASSERT(AreSameSizeAndType(rt, rt2));
6228   USE(rt2);
6229   if (rt.IsRegister()) {
6230     return rt.Is64Bits() ? STP_x : STP_w;
6231   } else {
6232     VIXL_ASSERT(rt.IsVRegister());
6233     switch (rt.GetSizeInBytes()) {
6234       case kSRegSizeInBytes:
6235         return STP_s;
6236       case kDRegSizeInBytes:
6237         return STP_d;
6238       default:
6239         VIXL_ASSERT(rt.IsQ());
6240         return STP_q;
6241     }
6242   }
6243 }
6244 
6245 
LoadPairOpFor(const CPURegister & rt,const CPURegister & rt2)6246 LoadStorePairOp Assembler::LoadPairOpFor(const CPURegister& rt,
6247                                          const CPURegister& rt2) {
6248   VIXL_ASSERT((STP_w | LoadStorePairLBit) == LDP_w);
6249   return static_cast<LoadStorePairOp>(StorePairOpFor(rt, rt2) |
6250                                       LoadStorePairLBit);
6251 }
6252 
6253 
StorePairNonTemporalOpFor(const CPURegister & rt,const CPURegister & rt2)6254 LoadStorePairNonTemporalOp Assembler::StorePairNonTemporalOpFor(
6255     const CPURegister& rt, const CPURegister& rt2) {
6256   VIXL_ASSERT(AreSameSizeAndType(rt, rt2));
6257   USE(rt2);
6258   if (rt.IsRegister()) {
6259     return rt.Is64Bits() ? STNP_x : STNP_w;
6260   } else {
6261     VIXL_ASSERT(rt.IsVRegister());
6262     switch (rt.GetSizeInBytes()) {
6263       case kSRegSizeInBytes:
6264         return STNP_s;
6265       case kDRegSizeInBytes:
6266         return STNP_d;
6267       default:
6268         VIXL_ASSERT(rt.IsQ());
6269         return STNP_q;
6270     }
6271   }
6272 }
6273 
6274 
LoadPairNonTemporalOpFor(const CPURegister & rt,const CPURegister & rt2)6275 LoadStorePairNonTemporalOp Assembler::LoadPairNonTemporalOpFor(
6276     const CPURegister& rt, const CPURegister& rt2) {
6277   VIXL_ASSERT((STNP_w | LoadStorePairNonTemporalLBit) == LDNP_w);
6278   return static_cast<LoadStorePairNonTemporalOp>(
6279       StorePairNonTemporalOpFor(rt, rt2) | LoadStorePairNonTemporalLBit);
6280 }
6281 
6282 
LoadLiteralOpFor(const CPURegister & rt)6283 LoadLiteralOp Assembler::LoadLiteralOpFor(const CPURegister& rt) {
6284   if (rt.IsRegister()) {
6285     return rt.IsX() ? LDR_x_lit : LDR_w_lit;
6286   } else {
6287     VIXL_ASSERT(rt.IsVRegister());
6288     switch (rt.GetSizeInBytes()) {
6289       case kSRegSizeInBytes:
6290         return LDR_s_lit;
6291       case kDRegSizeInBytes:
6292         return LDR_d_lit;
6293       default:
6294         VIXL_ASSERT(rt.IsQ());
6295         return LDR_q_lit;
6296     }
6297   }
6298 }
6299 
6300 
CPUHas(const CPURegister & rt) const6301 bool Assembler::CPUHas(const CPURegister& rt) const {
6302   // Core registers are available without any particular CPU features.
6303   if (rt.IsRegister()) return true;
6304   VIXL_ASSERT(rt.IsVRegister());
6305   // The architecture does not allow FP and NEON to be implemented separately,
6306   // but we can crudely categorise them based on register size, since FP only
6307   // uses D, S and (occasionally) H registers.
6308   if (rt.IsH() || rt.IsS() || rt.IsD()) {
6309     return CPUHas(CPUFeatures::kFP) || CPUHas(CPUFeatures::kNEON);
6310   }
6311   VIXL_ASSERT(rt.IsB() || rt.IsQ());
6312   return CPUHas(CPUFeatures::kNEON);
6313 }
6314 
6315 
CPUHas(const CPURegister & rt,const CPURegister & rt2) const6316 bool Assembler::CPUHas(const CPURegister& rt, const CPURegister& rt2) const {
6317   // This is currently only used for loads and stores, where rt and rt2 must
6318   // have the same size and type. We could extend this to cover other cases if
6319   // necessary, but for now we can avoid checking both registers.
6320   VIXL_ASSERT(AreSameSizeAndType(rt, rt2));
6321   USE(rt2);
6322   return CPUHas(rt);
6323 }
6324 
6325 
CPUHas(SystemRegister sysreg) const6326 bool Assembler::CPUHas(SystemRegister sysreg) const {
6327   switch (sysreg) {
6328     case RNDR:
6329     case RNDRRS:
6330       return CPUHas(CPUFeatures::kRNG);
6331     case FPCR:
6332     case NZCV:
6333       break;
6334   }
6335   return true;
6336 }
6337 
6338 
6339 }  // namespace aarch64
6340 }  // namespace vixl
6341