1 // Copyright 2018, VIXL authors
2 // All rights reserved.
3 //
4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions are met:
6 //
7 // * Redistributions of source code must retain the above copyright notice,
8 // this list of conditions and the following disclaimer.
9 // * Redistributions in binary form must reproduce the above copyright notice,
10 // this list of conditions and the following disclaimer in the documentation
11 // and/or other materials provided with the distribution.
12 // * Neither the name of Arm Limited nor the names of its contributors may be
13 // used to endorse or promote products derived from this software without
14 // specific prior written permission.
15 //
16 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
17 // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
20 // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
22 // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
23 // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24 // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26
27 #include "cpu-features.h"
28 #include "globals-vixl.h"
29 #include "utils-vixl.h"
30 #include "decoder-aarch64.h"
31
32 #include "cpu-features-auditor-aarch64.h"
33
34 namespace vixl {
35 namespace aarch64 {
36
37 const CPUFeaturesAuditor::FormToVisitorFnMap CPUFeaturesAuditor::FORM_TO_VISITOR = {
38 DEFAULT_FORM_TO_VISITOR_MAP(CPUFeaturesAuditor),
39 SIM_AUD_VISITOR_MAP(CPUFeaturesAuditor),
40 {"fcmla_asimdelem_c_h"_h, &CPUFeaturesAuditor::VisitNEONByIndexedElement},
41 {"fcmla_asimdelem_c_s"_h, &CPUFeaturesAuditor::VisitNEONByIndexedElement},
42 {"fmlal2_asimdelem_lh"_h, &CPUFeaturesAuditor::VisitNEONByIndexedElement},
43 {"fmlal_asimdelem_lh"_h, &CPUFeaturesAuditor::VisitNEONByIndexedElement},
44 {"fmla_asimdelem_rh_h"_h, &CPUFeaturesAuditor::VisitNEONByIndexedElement},
45 {"fmla_asimdelem_r_sd"_h, &CPUFeaturesAuditor::VisitNEONByIndexedElement},
46 {"fmlsl2_asimdelem_lh"_h, &CPUFeaturesAuditor::VisitNEONByIndexedElement},
47 {"fmlsl_asimdelem_lh"_h, &CPUFeaturesAuditor::VisitNEONByIndexedElement},
48 {"fmls_asimdelem_rh_h"_h, &CPUFeaturesAuditor::VisitNEONByIndexedElement},
49 {"fmls_asimdelem_r_sd"_h, &CPUFeaturesAuditor::VisitNEONByIndexedElement},
50 {"fmulx_asimdelem_rh_h"_h, &CPUFeaturesAuditor::VisitNEONByIndexedElement},
51 {"fmulx_asimdelem_r_sd"_h, &CPUFeaturesAuditor::VisitNEONByIndexedElement},
52 {"fmul_asimdelem_rh_h"_h, &CPUFeaturesAuditor::VisitNEONByIndexedElement},
53 {"fmul_asimdelem_r_sd"_h, &CPUFeaturesAuditor::VisitNEONByIndexedElement},
54 {"sdot_asimdelem_d"_h, &CPUFeaturesAuditor::VisitNEONByIndexedElement},
55 {"smlal_asimdelem_l"_h, &CPUFeaturesAuditor::VisitNEONByIndexedElement},
56 {"smlsl_asimdelem_l"_h, &CPUFeaturesAuditor::VisitNEONByIndexedElement},
57 {"smull_asimdelem_l"_h, &CPUFeaturesAuditor::VisitNEONByIndexedElement},
58 {"sqdmlal_asimdelem_l"_h, &CPUFeaturesAuditor::VisitNEONByIndexedElement},
59 {"sqdmlsl_asimdelem_l"_h, &CPUFeaturesAuditor::VisitNEONByIndexedElement},
60 {"sqdmull_asimdelem_l"_h, &CPUFeaturesAuditor::VisitNEONByIndexedElement},
61 {"udot_asimdelem_d"_h, &CPUFeaturesAuditor::VisitNEONByIndexedElement},
62 {"umlal_asimdelem_l"_h, &CPUFeaturesAuditor::VisitNEONByIndexedElement},
63 {"umlsl_asimdelem_l"_h, &CPUFeaturesAuditor::VisitNEONByIndexedElement},
64 {"umull_asimdelem_l"_h, &CPUFeaturesAuditor::VisitNEONByIndexedElement},
65 };
66
67 const CPUFeaturesAuditor::FormToVisitorFnMap*
GetFormToVisitorFnMap()68 CPUFeaturesAuditor::GetFormToVisitorFnMap() {
69 return &FORM_TO_VISITOR;
70 }
71
72 // Every instruction must update last_instruction_, even if only to clear it,
73 // and every instruction must also update seen_ once it has been fully handled.
74 // This scope makes that simple, and allows early returns in the decode logic.
75 class CPUFeaturesAuditor::RecordInstructionFeaturesScope {
76 public:
RecordInstructionFeaturesScope(CPUFeaturesAuditor * auditor)77 explicit RecordInstructionFeaturesScope(CPUFeaturesAuditor* auditor)
78 : auditor_(auditor) {
79 auditor_->last_instruction_ = CPUFeatures::None();
80 }
~RecordInstructionFeaturesScope()81 ~RecordInstructionFeaturesScope() {
82 auditor_->seen_.Combine(auditor_->last_instruction_);
83 }
84
Record(const CPUFeatures & features)85 void Record(const CPUFeatures& features) {
86 auditor_->last_instruction_.Combine(features);
87 }
88
Record(CPUFeatures::Feature feature0,CPUFeatures::Feature feature1=CPUFeatures::kNone,CPUFeatures::Feature feature2=CPUFeatures::kNone,CPUFeatures::Feature feature3=CPUFeatures::kNone)89 void Record(CPUFeatures::Feature feature0,
90 CPUFeatures::Feature feature1 = CPUFeatures::kNone,
91 CPUFeatures::Feature feature2 = CPUFeatures::kNone,
92 CPUFeatures::Feature feature3 = CPUFeatures::kNone) {
93 auditor_->last_instruction_.Combine(feature0, feature1, feature2, feature3);
94 }
95
96 // If exactly one of a or b is known to be available, record it. Otherwise,
97 // record both. This is intended for encodings that can be provided by two
98 // different features.
RecordOneOrBothOf(CPUFeatures::Feature a,CPUFeatures::Feature b)99 void RecordOneOrBothOf(CPUFeatures::Feature a, CPUFeatures::Feature b) {
100 bool hint_a = auditor_->available_.Has(a);
101 bool hint_b = auditor_->available_.Has(b);
102 if (hint_a && !hint_b) {
103 Record(a);
104 } else if (hint_b && !hint_a) {
105 Record(b);
106 } else {
107 Record(a, b);
108 }
109 }
110
111 private:
112 CPUFeaturesAuditor* auditor_;
113 };
114
LoadStoreHelper(const Instruction * instr)115 void CPUFeaturesAuditor::LoadStoreHelper(const Instruction* instr) {
116 RecordInstructionFeaturesScope scope(this);
117 switch (instr->Mask(LoadStoreMask)) {
118 case LDR_b:
119 case LDR_q:
120 case STR_b:
121 case STR_q:
122 scope.Record(CPUFeatures::kNEON);
123 return;
124 case LDR_h:
125 case LDR_s:
126 case LDR_d:
127 case STR_h:
128 case STR_s:
129 case STR_d:
130 scope.RecordOneOrBothOf(CPUFeatures::kFP, CPUFeatures::kNEON);
131 return;
132 default:
133 // No special CPU features.
134 return;
135 }
136 }
137
LoadStorePairHelper(const Instruction * instr)138 void CPUFeaturesAuditor::LoadStorePairHelper(const Instruction* instr) {
139 RecordInstructionFeaturesScope scope(this);
140 switch (instr->Mask(LoadStorePairMask)) {
141 case LDP_q:
142 case STP_q:
143 scope.Record(CPUFeatures::kNEON);
144 return;
145 case LDP_s:
146 case LDP_d:
147 case STP_s:
148 case STP_d: {
149 scope.RecordOneOrBothOf(CPUFeatures::kFP, CPUFeatures::kNEON);
150 return;
151 }
152 default:
153 // No special CPU features.
154 return;
155 }
156 }
157
VisitAddSubExtended(const Instruction * instr)158 void CPUFeaturesAuditor::VisitAddSubExtended(const Instruction* instr) {
159 RecordInstructionFeaturesScope scope(this);
160 USE(instr);
161 }
162
VisitAddSubImmediate(const Instruction * instr)163 void CPUFeaturesAuditor::VisitAddSubImmediate(const Instruction* instr) {
164 RecordInstructionFeaturesScope scope(this);
165 USE(instr);
166 }
167
VisitAddSubShifted(const Instruction * instr)168 void CPUFeaturesAuditor::VisitAddSubShifted(const Instruction* instr) {
169 RecordInstructionFeaturesScope scope(this);
170 USE(instr);
171 }
172
VisitAddSubWithCarry(const Instruction * instr)173 void CPUFeaturesAuditor::VisitAddSubWithCarry(const Instruction* instr) {
174 RecordInstructionFeaturesScope scope(this);
175 USE(instr);
176 }
177
VisitRotateRightIntoFlags(const Instruction * instr)178 void CPUFeaturesAuditor::VisitRotateRightIntoFlags(const Instruction* instr) {
179 RecordInstructionFeaturesScope scope(this);
180 switch (instr->Mask(RotateRightIntoFlagsMask)) {
181 case RMIF:
182 scope.Record(CPUFeatures::kFlagM);
183 return;
184 }
185 }
186
VisitEvaluateIntoFlags(const Instruction * instr)187 void CPUFeaturesAuditor::VisitEvaluateIntoFlags(const Instruction* instr) {
188 RecordInstructionFeaturesScope scope(this);
189 switch (instr->Mask(EvaluateIntoFlagsMask)) {
190 case SETF8:
191 case SETF16:
192 scope.Record(CPUFeatures::kFlagM);
193 return;
194 }
195 }
196
VisitAtomicMemory(const Instruction * instr)197 void CPUFeaturesAuditor::VisitAtomicMemory(const Instruction* instr) {
198 RecordInstructionFeaturesScope scope(this);
199 switch (instr->Mask(AtomicMemoryMask)) {
200 case LDAPRB:
201 case LDAPRH:
202 case LDAPR_w:
203 case LDAPR_x:
204 scope.Record(CPUFeatures::kRCpc);
205 return;
206 default:
207 // Everything else belongs to the Atomics extension.
208 scope.Record(CPUFeatures::kAtomics);
209 return;
210 }
211 }
212
VisitBitfield(const Instruction * instr)213 void CPUFeaturesAuditor::VisitBitfield(const Instruction* instr) {
214 RecordInstructionFeaturesScope scope(this);
215 USE(instr);
216 }
217
VisitCompareBranch(const Instruction * instr)218 void CPUFeaturesAuditor::VisitCompareBranch(const Instruction* instr) {
219 RecordInstructionFeaturesScope scope(this);
220 USE(instr);
221 }
222
VisitConditionalBranch(const Instruction * instr)223 void CPUFeaturesAuditor::VisitConditionalBranch(const Instruction* instr) {
224 RecordInstructionFeaturesScope scope(this);
225 USE(instr);
226 }
227
VisitConditionalCompareImmediate(const Instruction * instr)228 void CPUFeaturesAuditor::VisitConditionalCompareImmediate(
229 const Instruction* instr) {
230 RecordInstructionFeaturesScope scope(this);
231 USE(instr);
232 }
233
VisitConditionalCompareRegister(const Instruction * instr)234 void CPUFeaturesAuditor::VisitConditionalCompareRegister(
235 const Instruction* instr) {
236 RecordInstructionFeaturesScope scope(this);
237 USE(instr);
238 }
239
VisitConditionalSelect(const Instruction * instr)240 void CPUFeaturesAuditor::VisitConditionalSelect(const Instruction* instr) {
241 RecordInstructionFeaturesScope scope(this);
242 USE(instr);
243 }
244
VisitCrypto2RegSHA(const Instruction * instr)245 void CPUFeaturesAuditor::VisitCrypto2RegSHA(const Instruction* instr) {
246 RecordInstructionFeaturesScope scope(this);
247 USE(instr);
248 }
249
VisitCrypto3RegSHA(const Instruction * instr)250 void CPUFeaturesAuditor::VisitCrypto3RegSHA(const Instruction* instr) {
251 RecordInstructionFeaturesScope scope(this);
252 USE(instr);
253 }
254
VisitCryptoAES(const Instruction * instr)255 void CPUFeaturesAuditor::VisitCryptoAES(const Instruction* instr) {
256 RecordInstructionFeaturesScope scope(this);
257 USE(instr);
258 }
259
VisitDataProcessing1Source(const Instruction * instr)260 void CPUFeaturesAuditor::VisitDataProcessing1Source(const Instruction* instr) {
261 RecordInstructionFeaturesScope scope(this);
262 switch (instr->Mask(DataProcessing1SourceMask)) {
263 case PACIA:
264 case PACIB:
265 case PACDA:
266 case PACDB:
267 case AUTIA:
268 case AUTIB:
269 case AUTDA:
270 case AUTDB:
271 case PACIZA:
272 case PACIZB:
273 case PACDZA:
274 case PACDZB:
275 case AUTIZA:
276 case AUTIZB:
277 case AUTDZA:
278 case AUTDZB:
279 case XPACI:
280 case XPACD:
281 scope.Record(CPUFeatures::kPAuth);
282 return;
283 default:
284 // No special CPU features.
285 return;
286 }
287 }
288
VisitDataProcessing2Source(const Instruction * instr)289 void CPUFeaturesAuditor::VisitDataProcessing2Source(const Instruction* instr) {
290 RecordInstructionFeaturesScope scope(this);
291 switch (instr->Mask(DataProcessing2SourceMask)) {
292 case CRC32B:
293 case CRC32H:
294 case CRC32W:
295 case CRC32X:
296 case CRC32CB:
297 case CRC32CH:
298 case CRC32CW:
299 case CRC32CX:
300 scope.Record(CPUFeatures::kCRC32);
301 return;
302 case PACGA:
303 scope.Record(CPUFeatures::kPAuth, CPUFeatures::kPAuthGeneric);
304 return;
305 default:
306 // No special CPU features.
307 return;
308 }
309 }
310
VisitLoadStoreRCpcUnscaledOffset(const Instruction * instr)311 void CPUFeaturesAuditor::VisitLoadStoreRCpcUnscaledOffset(
312 const Instruction* instr) {
313 RecordInstructionFeaturesScope scope(this);
314 switch (instr->Mask(LoadStoreRCpcUnscaledOffsetMask)) {
315 case LDAPURB:
316 case LDAPURSB_w:
317 case LDAPURSB_x:
318 case LDAPURH:
319 case LDAPURSH_w:
320 case LDAPURSH_x:
321 case LDAPUR_w:
322 case LDAPURSW:
323 case LDAPUR_x:
324
325 // These stores don't actually have RCpc semantics but they're included with
326 // the RCpc extensions.
327 case STLURB:
328 case STLURH:
329 case STLUR_w:
330 case STLUR_x:
331 scope.Record(CPUFeatures::kRCpc, CPUFeatures::kRCpcImm);
332 return;
333 }
334 }
335
VisitLoadStorePAC(const Instruction * instr)336 void CPUFeaturesAuditor::VisitLoadStorePAC(const Instruction* instr) {
337 RecordInstructionFeaturesScope scope(this);
338 USE(instr);
339 scope.Record(CPUFeatures::kPAuth);
340 }
341
VisitDataProcessing3Source(const Instruction * instr)342 void CPUFeaturesAuditor::VisitDataProcessing3Source(const Instruction* instr) {
343 RecordInstructionFeaturesScope scope(this);
344 USE(instr);
345 }
346
VisitException(const Instruction * instr)347 void CPUFeaturesAuditor::VisitException(const Instruction* instr) {
348 RecordInstructionFeaturesScope scope(this);
349 USE(instr);
350 }
351
VisitExtract(const Instruction * instr)352 void CPUFeaturesAuditor::VisitExtract(const Instruction* instr) {
353 RecordInstructionFeaturesScope scope(this);
354 USE(instr);
355 }
356
VisitFPCompare(const Instruction * instr)357 void CPUFeaturesAuditor::VisitFPCompare(const Instruction* instr) {
358 RecordInstructionFeaturesScope scope(this);
359 // All of these instructions require FP.
360 scope.Record(CPUFeatures::kFP);
361 switch (instr->Mask(FPCompareMask)) {
362 case FCMP_h:
363 case FCMP_h_zero:
364 case FCMPE_h:
365 case FCMPE_h_zero:
366 scope.Record(CPUFeatures::kFPHalf);
367 return;
368 default:
369 // No special CPU features.
370 return;
371 }
372 }
373
VisitFPConditionalCompare(const Instruction * instr)374 void CPUFeaturesAuditor::VisitFPConditionalCompare(const Instruction* instr) {
375 RecordInstructionFeaturesScope scope(this);
376 // All of these instructions require FP.
377 scope.Record(CPUFeatures::kFP);
378 switch (instr->Mask(FPConditionalCompareMask)) {
379 case FCCMP_h:
380 case FCCMPE_h:
381 scope.Record(CPUFeatures::kFPHalf);
382 return;
383 default:
384 // No special CPU features.
385 return;
386 }
387 }
388
VisitFPConditionalSelect(const Instruction * instr)389 void CPUFeaturesAuditor::VisitFPConditionalSelect(const Instruction* instr) {
390 RecordInstructionFeaturesScope scope(this);
391 // All of these instructions require FP.
392 scope.Record(CPUFeatures::kFP);
393 if (instr->Mask(FPConditionalSelectMask) == FCSEL_h) {
394 scope.Record(CPUFeatures::kFPHalf);
395 }
396 }
397
VisitFPDataProcessing1Source(const Instruction * instr)398 void CPUFeaturesAuditor::VisitFPDataProcessing1Source(
399 const Instruction* instr) {
400 RecordInstructionFeaturesScope scope(this);
401 // All of these instructions require FP.
402 scope.Record(CPUFeatures::kFP);
403 switch (instr->Mask(FPDataProcessing1SourceMask)) {
404 case FMOV_h:
405 case FABS_h:
406 case FNEG_h:
407 case FSQRT_h:
408 case FRINTN_h:
409 case FRINTP_h:
410 case FRINTM_h:
411 case FRINTZ_h:
412 case FRINTA_h:
413 case FRINTX_h:
414 case FRINTI_h:
415 scope.Record(CPUFeatures::kFPHalf);
416 return;
417 case FRINT32X_s:
418 case FRINT32X_d:
419 case FRINT32Z_s:
420 case FRINT32Z_d:
421 case FRINT64X_s:
422 case FRINT64X_d:
423 case FRINT64Z_s:
424 case FRINT64Z_d:
425 scope.Record(CPUFeatures::kFrintToFixedSizedInt);
426 return;
427 default:
428 // No special CPU features.
429 // This category includes some half-precision FCVT instructions that do
430 // not require FPHalf.
431 return;
432 }
433 }
434
VisitFPDataProcessing2Source(const Instruction * instr)435 void CPUFeaturesAuditor::VisitFPDataProcessing2Source(
436 const Instruction* instr) {
437 RecordInstructionFeaturesScope scope(this);
438 // All of these instructions require FP.
439 scope.Record(CPUFeatures::kFP);
440 switch (instr->Mask(FPDataProcessing2SourceMask)) {
441 case FMUL_h:
442 case FDIV_h:
443 case FADD_h:
444 case FSUB_h:
445 case FMAX_h:
446 case FMIN_h:
447 case FMAXNM_h:
448 case FMINNM_h:
449 case FNMUL_h:
450 scope.Record(CPUFeatures::kFPHalf);
451 return;
452 default:
453 // No special CPU features.
454 return;
455 }
456 }
457
VisitFPDataProcessing3Source(const Instruction * instr)458 void CPUFeaturesAuditor::VisitFPDataProcessing3Source(
459 const Instruction* instr) {
460 RecordInstructionFeaturesScope scope(this);
461 // All of these instructions require FP.
462 scope.Record(CPUFeatures::kFP);
463 switch (instr->Mask(FPDataProcessing3SourceMask)) {
464 case FMADD_h:
465 case FMSUB_h:
466 case FNMADD_h:
467 case FNMSUB_h:
468 scope.Record(CPUFeatures::kFPHalf);
469 return;
470 default:
471 // No special CPU features.
472 return;
473 }
474 }
475
VisitFPFixedPointConvert(const Instruction * instr)476 void CPUFeaturesAuditor::VisitFPFixedPointConvert(const Instruction* instr) {
477 RecordInstructionFeaturesScope scope(this);
478 // All of these instructions require FP.
479 scope.Record(CPUFeatures::kFP);
480 switch (instr->Mask(FPFixedPointConvertMask)) {
481 case FCVTZS_wh_fixed:
482 case FCVTZS_xh_fixed:
483 case FCVTZU_wh_fixed:
484 case FCVTZU_xh_fixed:
485 case SCVTF_hw_fixed:
486 case SCVTF_hx_fixed:
487 case UCVTF_hw_fixed:
488 case UCVTF_hx_fixed:
489 scope.Record(CPUFeatures::kFPHalf);
490 return;
491 default:
492 // No special CPU features.
493 return;
494 }
495 }
496
VisitFPImmediate(const Instruction * instr)497 void CPUFeaturesAuditor::VisitFPImmediate(const Instruction* instr) {
498 RecordInstructionFeaturesScope scope(this);
499 // All of these instructions require FP.
500 scope.Record(CPUFeatures::kFP);
501 if (instr->Mask(FPImmediateMask) == FMOV_h_imm) {
502 scope.Record(CPUFeatures::kFPHalf);
503 }
504 }
505
VisitFPIntegerConvert(const Instruction * instr)506 void CPUFeaturesAuditor::VisitFPIntegerConvert(const Instruction* instr) {
507 RecordInstructionFeaturesScope scope(this);
508 // All of these instructions require FP.
509 scope.Record(CPUFeatures::kFP);
510 switch (instr->Mask(FPIntegerConvertMask)) {
511 case FCVTAS_wh:
512 case FCVTAS_xh:
513 case FCVTAU_wh:
514 case FCVTAU_xh:
515 case FCVTMS_wh:
516 case FCVTMS_xh:
517 case FCVTMU_wh:
518 case FCVTMU_xh:
519 case FCVTNS_wh:
520 case FCVTNS_xh:
521 case FCVTNU_wh:
522 case FCVTNU_xh:
523 case FCVTPS_wh:
524 case FCVTPS_xh:
525 case FCVTPU_wh:
526 case FCVTPU_xh:
527 case FCVTZS_wh:
528 case FCVTZS_xh:
529 case FCVTZU_wh:
530 case FCVTZU_xh:
531 case FMOV_hw:
532 case FMOV_hx:
533 case FMOV_wh:
534 case FMOV_xh:
535 case SCVTF_hw:
536 case SCVTF_hx:
537 case UCVTF_hw:
538 case UCVTF_hx:
539 scope.Record(CPUFeatures::kFPHalf);
540 return;
541 case FMOV_d1_x:
542 case FMOV_x_d1:
543 scope.Record(CPUFeatures::kNEON);
544 return;
545 case FJCVTZS:
546 scope.Record(CPUFeatures::kJSCVT);
547 return;
548 default:
549 // No special CPU features.
550 return;
551 }
552 }
553
VisitLoadLiteral(const Instruction * instr)554 void CPUFeaturesAuditor::VisitLoadLiteral(const Instruction* instr) {
555 RecordInstructionFeaturesScope scope(this);
556 switch (instr->Mask(LoadLiteralMask)) {
557 case LDR_s_lit:
558 case LDR_d_lit:
559 scope.RecordOneOrBothOf(CPUFeatures::kFP, CPUFeatures::kNEON);
560 return;
561 case LDR_q_lit:
562 scope.Record(CPUFeatures::kNEON);
563 return;
564 default:
565 // No special CPU features.
566 return;
567 }
568 }
569
VisitLoadStoreExclusive(const Instruction * instr)570 void CPUFeaturesAuditor::VisitLoadStoreExclusive(const Instruction* instr) {
571 RecordInstructionFeaturesScope scope(this);
572 switch (instr->Mask(LoadStoreExclusiveMask)) {
573 case CAS_w:
574 case CASA_w:
575 case CASL_w:
576 case CASAL_w:
577 case CAS_x:
578 case CASA_x:
579 case CASL_x:
580 case CASAL_x:
581 case CASB:
582 case CASAB:
583 case CASLB:
584 case CASALB:
585 case CASH:
586 case CASAH:
587 case CASLH:
588 case CASALH:
589 case CASP_w:
590 case CASPA_w:
591 case CASPL_w:
592 case CASPAL_w:
593 case CASP_x:
594 case CASPA_x:
595 case CASPL_x:
596 case CASPAL_x:
597 scope.Record(CPUFeatures::kAtomics);
598 return;
599 case STLLRB:
600 case LDLARB:
601 case STLLRH:
602 case LDLARH:
603 case STLLR_w:
604 case LDLAR_w:
605 case STLLR_x:
606 case LDLAR_x:
607 scope.Record(CPUFeatures::kLORegions);
608 return;
609 default:
610 // No special CPU features.
611 return;
612 }
613 }
614
VisitLoadStorePairNonTemporal(const Instruction * instr)615 void CPUFeaturesAuditor::VisitLoadStorePairNonTemporal(
616 const Instruction* instr) {
617 LoadStorePairHelper(instr);
618 }
619
VisitLoadStorePairOffset(const Instruction * instr)620 void CPUFeaturesAuditor::VisitLoadStorePairOffset(const Instruction* instr) {
621 LoadStorePairHelper(instr);
622 }
623
VisitLoadStorePairPostIndex(const Instruction * instr)624 void CPUFeaturesAuditor::VisitLoadStorePairPostIndex(const Instruction* instr) {
625 LoadStorePairHelper(instr);
626 }
627
VisitLoadStorePairPreIndex(const Instruction * instr)628 void CPUFeaturesAuditor::VisitLoadStorePairPreIndex(const Instruction* instr) {
629 LoadStorePairHelper(instr);
630 }
631
VisitLoadStorePostIndex(const Instruction * instr)632 void CPUFeaturesAuditor::VisitLoadStorePostIndex(const Instruction* instr) {
633 LoadStoreHelper(instr);
634 }
635
VisitLoadStorePreIndex(const Instruction * instr)636 void CPUFeaturesAuditor::VisitLoadStorePreIndex(const Instruction* instr) {
637 LoadStoreHelper(instr);
638 }
639
VisitLoadStoreRegisterOffset(const Instruction * instr)640 void CPUFeaturesAuditor::VisitLoadStoreRegisterOffset(
641 const Instruction* instr) {
642 LoadStoreHelper(instr);
643 }
644
VisitLoadStoreUnscaledOffset(const Instruction * instr)645 void CPUFeaturesAuditor::VisitLoadStoreUnscaledOffset(
646 const Instruction* instr) {
647 LoadStoreHelper(instr);
648 }
649
VisitLoadStoreUnsignedOffset(const Instruction * instr)650 void CPUFeaturesAuditor::VisitLoadStoreUnsignedOffset(
651 const Instruction* instr) {
652 LoadStoreHelper(instr);
653 }
654
VisitLogicalImmediate(const Instruction * instr)655 void CPUFeaturesAuditor::VisitLogicalImmediate(const Instruction* instr) {
656 RecordInstructionFeaturesScope scope(this);
657 USE(instr);
658 }
659
VisitLogicalShifted(const Instruction * instr)660 void CPUFeaturesAuditor::VisitLogicalShifted(const Instruction* instr) {
661 RecordInstructionFeaturesScope scope(this);
662 USE(instr);
663 }
664
VisitMoveWideImmediate(const Instruction * instr)665 void CPUFeaturesAuditor::VisitMoveWideImmediate(const Instruction* instr) {
666 RecordInstructionFeaturesScope scope(this);
667 USE(instr);
668 }
669
VisitNEON2RegMisc(const Instruction * instr)670 void CPUFeaturesAuditor::VisitNEON2RegMisc(const Instruction* instr) {
671 RecordInstructionFeaturesScope scope(this);
672 // All of these instructions require NEON.
673 scope.Record(CPUFeatures::kNEON);
674 switch (instr->Mask(NEON2RegMiscFPMask)) {
675 case NEON_FABS:
676 case NEON_FNEG:
677 case NEON_FSQRT:
678 case NEON_FCVTL:
679 case NEON_FCVTN:
680 case NEON_FCVTXN:
681 case NEON_FRINTI:
682 case NEON_FRINTX:
683 case NEON_FRINTA:
684 case NEON_FRINTM:
685 case NEON_FRINTN:
686 case NEON_FRINTP:
687 case NEON_FRINTZ:
688 case NEON_FCVTNS:
689 case NEON_FCVTNU:
690 case NEON_FCVTPS:
691 case NEON_FCVTPU:
692 case NEON_FCVTMS:
693 case NEON_FCVTMU:
694 case NEON_FCVTZS:
695 case NEON_FCVTZU:
696 case NEON_FCVTAS:
697 case NEON_FCVTAU:
698 case NEON_SCVTF:
699 case NEON_UCVTF:
700 case NEON_FRSQRTE:
701 case NEON_FRECPE:
702 case NEON_FCMGT_zero:
703 case NEON_FCMGE_zero:
704 case NEON_FCMEQ_zero:
705 case NEON_FCMLE_zero:
706 case NEON_FCMLT_zero:
707 scope.Record(CPUFeatures::kFP);
708 return;
709 case NEON_FRINT32X:
710 case NEON_FRINT32Z:
711 case NEON_FRINT64X:
712 case NEON_FRINT64Z:
713 scope.Record(CPUFeatures::kFP, CPUFeatures::kFrintToFixedSizedInt);
714 return;
715 default:
716 // No additional features.
717 return;
718 }
719 }
720
VisitNEON2RegMiscFP16(const Instruction * instr)721 void CPUFeaturesAuditor::VisitNEON2RegMiscFP16(const Instruction* instr) {
722 RecordInstructionFeaturesScope scope(this);
723 // All of these instructions require NEONHalf.
724 scope.Record(CPUFeatures::kFP, CPUFeatures::kNEON, CPUFeatures::kNEONHalf);
725 USE(instr);
726 }
727
VisitNEON3Different(const Instruction * instr)728 void CPUFeaturesAuditor::VisitNEON3Different(const Instruction* instr) {
729 RecordInstructionFeaturesScope scope(this);
730 // All of these instructions require NEON.
731 scope.Record(CPUFeatures::kNEON);
732 USE(instr);
733 }
734
VisitNEON3Same(const Instruction * instr)735 void CPUFeaturesAuditor::VisitNEON3Same(const Instruction* instr) {
736 RecordInstructionFeaturesScope scope(this);
737 // All of these instructions require NEON.
738 scope.Record(CPUFeatures::kNEON);
739 if (instr->Mask(NEON3SameFPFMask) == NEON3SameFPFixed) {
740 scope.Record(CPUFeatures::kFP);
741 }
742 switch (instr->Mask(NEON3SameFHMMask)) {
743 case NEON_FMLAL:
744 case NEON_FMLAL2:
745 case NEON_FMLSL:
746 case NEON_FMLSL2:
747 scope.Record(CPUFeatures::kFP, CPUFeatures::kNEONHalf, CPUFeatures::kFHM);
748 return;
749 default:
750 // No additional features.
751 return;
752 }
753 }
754
VisitNEON3SameExtra(const Instruction * instr)755 void CPUFeaturesAuditor::VisitNEON3SameExtra(const Instruction* instr) {
756 RecordInstructionFeaturesScope scope(this);
757 // All of these instructions require NEON.
758 scope.Record(CPUFeatures::kNEON);
759 if ((instr->Mask(NEON3SameExtraFCMLAMask) == NEON_FCMLA) ||
760 (instr->Mask(NEON3SameExtraFCADDMask) == NEON_FCADD)) {
761 scope.Record(CPUFeatures::kFP, CPUFeatures::kFcma);
762 if (instr->GetNEONSize() == 1) scope.Record(CPUFeatures::kNEONHalf);
763 } else {
764 switch (instr->Mask(NEON3SameExtraMask)) {
765 case NEON_SDOT:
766 case NEON_UDOT:
767 scope.Record(CPUFeatures::kDotProduct);
768 return;
769 case NEON_SQRDMLAH:
770 case NEON_SQRDMLSH:
771 scope.Record(CPUFeatures::kRDM);
772 return;
773 default:
774 // No additional features.
775 return;
776 }
777 }
778 }
779
VisitNEON3SameFP16(const Instruction * instr)780 void CPUFeaturesAuditor::VisitNEON3SameFP16(const Instruction* instr) {
781 RecordInstructionFeaturesScope scope(this);
782 // All of these instructions require NEON FP16 support.
783 scope.Record(CPUFeatures::kFP, CPUFeatures::kNEON, CPUFeatures::kNEONHalf);
784 USE(instr);
785 }
786
VisitNEONAcrossLanes(const Instruction * instr)787 void CPUFeaturesAuditor::VisitNEONAcrossLanes(const Instruction* instr) {
788 RecordInstructionFeaturesScope scope(this);
789 // All of these instructions require NEON.
790 scope.Record(CPUFeatures::kNEON);
791 if (instr->Mask(NEONAcrossLanesFP16FMask) == NEONAcrossLanesFP16Fixed) {
792 // FMAXV_H, FMINV_H, FMAXNMV_H, FMINNMV_H
793 scope.Record(CPUFeatures::kFP, CPUFeatures::kNEONHalf);
794 } else if (instr->Mask(NEONAcrossLanesFPFMask) == NEONAcrossLanesFPFixed) {
795 // FMAXV, FMINV, FMAXNMV, FMINNMV
796 scope.Record(CPUFeatures::kFP);
797 }
798 }
799
VisitNEONByIndexedElement(const Instruction * instr)800 void CPUFeaturesAuditor::VisitNEONByIndexedElement(const Instruction* instr) {
801 RecordInstructionFeaturesScope scope(this);
802 // All of these instructions require NEON.
803 scope.Record(CPUFeatures::kNEON);
804 switch (instr->Mask(NEONByIndexedElementMask)) {
805 case NEON_SDOT_byelement:
806 case NEON_UDOT_byelement:
807 scope.Record(CPUFeatures::kDotProduct);
808 return;
809 case NEON_SQRDMLAH_byelement:
810 case NEON_SQRDMLSH_byelement:
811 scope.Record(CPUFeatures::kRDM);
812 return;
813 default:
814 // Fall through to check other instructions.
815 break;
816 }
817 switch (instr->Mask(NEONByIndexedElementFPLongMask)) {
818 case NEON_FMLAL_H_byelement:
819 case NEON_FMLAL2_H_byelement:
820 case NEON_FMLSL_H_byelement:
821 case NEON_FMLSL2_H_byelement:
822 scope.Record(CPUFeatures::kFP, CPUFeatures::kNEONHalf, CPUFeatures::kFHM);
823 return;
824 default:
825 // Fall through to check other instructions.
826 break;
827 }
828 switch (instr->Mask(NEONByIndexedElementFPMask)) {
829 case NEON_FMLA_H_byelement:
830 case NEON_FMLS_H_byelement:
831 case NEON_FMUL_H_byelement:
832 case NEON_FMULX_H_byelement:
833 scope.Record(CPUFeatures::kNEONHalf);
834 VIXL_FALLTHROUGH();
835 case NEON_FMLA_byelement:
836 case NEON_FMLS_byelement:
837 case NEON_FMUL_byelement:
838 case NEON_FMULX_byelement:
839 scope.Record(CPUFeatures::kFP);
840 return;
841 default:
842 switch (instr->Mask(NEONByIndexedElementFPComplexMask)) {
843 case NEON_FCMLA_byelement:
844 scope.Record(CPUFeatures::kFP, CPUFeatures::kFcma);
845 if (instr->GetNEONSize() == 1) scope.Record(CPUFeatures::kNEONHalf);
846 return;
847 }
848 // No additional features.
849 return;
850 }
851 }
852
VisitNEONCopy(const Instruction * instr)853 void CPUFeaturesAuditor::VisitNEONCopy(const Instruction* instr) {
854 RecordInstructionFeaturesScope scope(this);
855 // All of these instructions require NEON.
856 scope.Record(CPUFeatures::kNEON);
857 USE(instr);
858 }
859
VisitNEONExtract(const Instruction * instr)860 void CPUFeaturesAuditor::VisitNEONExtract(const Instruction* instr) {
861 RecordInstructionFeaturesScope scope(this);
862 // All of these instructions require NEON.
863 scope.Record(CPUFeatures::kNEON);
864 USE(instr);
865 }
866
VisitNEONLoadStoreMultiStruct(const Instruction * instr)867 void CPUFeaturesAuditor::VisitNEONLoadStoreMultiStruct(
868 const Instruction* instr) {
869 RecordInstructionFeaturesScope scope(this);
870 // All of these instructions require NEON.
871 scope.Record(CPUFeatures::kNEON);
872 USE(instr);
873 }
874
VisitNEONLoadStoreMultiStructPostIndex(const Instruction * instr)875 void CPUFeaturesAuditor::VisitNEONLoadStoreMultiStructPostIndex(
876 const Instruction* instr) {
877 RecordInstructionFeaturesScope scope(this);
878 // All of these instructions require NEON.
879 scope.Record(CPUFeatures::kNEON);
880 USE(instr);
881 }
882
VisitNEONLoadStoreSingleStruct(const Instruction * instr)883 void CPUFeaturesAuditor::VisitNEONLoadStoreSingleStruct(
884 const Instruction* instr) {
885 RecordInstructionFeaturesScope scope(this);
886 // All of these instructions require NEON.
887 scope.Record(CPUFeatures::kNEON);
888 USE(instr);
889 }
890
VisitNEONLoadStoreSingleStructPostIndex(const Instruction * instr)891 void CPUFeaturesAuditor::VisitNEONLoadStoreSingleStructPostIndex(
892 const Instruction* instr) {
893 RecordInstructionFeaturesScope scope(this);
894 // All of these instructions require NEON.
895 scope.Record(CPUFeatures::kNEON);
896 USE(instr);
897 }
898
VisitNEONModifiedImmediate(const Instruction * instr)899 void CPUFeaturesAuditor::VisitNEONModifiedImmediate(const Instruction* instr) {
900 RecordInstructionFeaturesScope scope(this);
901 // All of these instructions require NEON.
902 scope.Record(CPUFeatures::kNEON);
903 if (instr->GetNEONCmode() == 0xf) {
904 // FMOV (vector, immediate), double-, single- or half-precision.
905 scope.Record(CPUFeatures::kFP);
906 if (instr->ExtractBit(11)) scope.Record(CPUFeatures::kNEONHalf);
907 }
908 }
909
VisitNEONPerm(const Instruction * instr)910 void CPUFeaturesAuditor::VisitNEONPerm(const Instruction* instr) {
911 RecordInstructionFeaturesScope scope(this);
912 // All of these instructions require NEON.
913 scope.Record(CPUFeatures::kNEON);
914 USE(instr);
915 }
916
VisitNEONScalar2RegMisc(const Instruction * instr)917 void CPUFeaturesAuditor::VisitNEONScalar2RegMisc(const Instruction* instr) {
918 RecordInstructionFeaturesScope scope(this);
919 // All of these instructions require NEON.
920 scope.Record(CPUFeatures::kNEON);
921 switch (instr->Mask(NEONScalar2RegMiscFPMask)) {
922 case NEON_FRECPE_scalar:
923 case NEON_FRECPX_scalar:
924 case NEON_FRSQRTE_scalar:
925 case NEON_FCMGT_zero_scalar:
926 case NEON_FCMGE_zero_scalar:
927 case NEON_FCMEQ_zero_scalar:
928 case NEON_FCMLE_zero_scalar:
929 case NEON_FCMLT_zero_scalar:
930 case NEON_SCVTF_scalar:
931 case NEON_UCVTF_scalar:
932 case NEON_FCVTNS_scalar:
933 case NEON_FCVTNU_scalar:
934 case NEON_FCVTPS_scalar:
935 case NEON_FCVTPU_scalar:
936 case NEON_FCVTMS_scalar:
937 case NEON_FCVTMU_scalar:
938 case NEON_FCVTZS_scalar:
939 case NEON_FCVTZU_scalar:
940 case NEON_FCVTAS_scalar:
941 case NEON_FCVTAU_scalar:
942 case NEON_FCVTXN_scalar:
943 scope.Record(CPUFeatures::kFP);
944 return;
945 default:
946 // No additional features.
947 return;
948 }
949 }
950
VisitNEONScalar2RegMiscFP16(const Instruction * instr)951 void CPUFeaturesAuditor::VisitNEONScalar2RegMiscFP16(const Instruction* instr) {
952 RecordInstructionFeaturesScope scope(this);
953 // All of these instructions require NEONHalf.
954 scope.Record(CPUFeatures::kFP, CPUFeatures::kNEON, CPUFeatures::kNEONHalf);
955 USE(instr);
956 }
957
VisitNEONScalar3Diff(const Instruction * instr)958 void CPUFeaturesAuditor::VisitNEONScalar3Diff(const Instruction* instr) {
959 RecordInstructionFeaturesScope scope(this);
960 // All of these instructions require NEON.
961 scope.Record(CPUFeatures::kNEON);
962 USE(instr);
963 }
964
VisitNEONScalar3Same(const Instruction * instr)965 void CPUFeaturesAuditor::VisitNEONScalar3Same(const Instruction* instr) {
966 RecordInstructionFeaturesScope scope(this);
967 // All of these instructions require NEON.
968 scope.Record(CPUFeatures::kNEON);
969 if (instr->Mask(NEONScalar3SameFPFMask) == NEONScalar3SameFPFixed) {
970 scope.Record(CPUFeatures::kFP);
971 }
972 }
973
VisitNEONScalar3SameExtra(const Instruction * instr)974 void CPUFeaturesAuditor::VisitNEONScalar3SameExtra(const Instruction* instr) {
975 RecordInstructionFeaturesScope scope(this);
976 // All of these instructions require NEON and RDM.
977 scope.Record(CPUFeatures::kNEON, CPUFeatures::kRDM);
978 USE(instr);
979 }
980
VisitNEONScalar3SameFP16(const Instruction * instr)981 void CPUFeaturesAuditor::VisitNEONScalar3SameFP16(const Instruction* instr) {
982 RecordInstructionFeaturesScope scope(this);
983 // All of these instructions require NEONHalf.
984 scope.Record(CPUFeatures::kFP, CPUFeatures::kNEON, CPUFeatures::kNEONHalf);
985 USE(instr);
986 }
987
VisitNEONScalarByIndexedElement(const Instruction * instr)988 void CPUFeaturesAuditor::VisitNEONScalarByIndexedElement(
989 const Instruction* instr) {
990 RecordInstructionFeaturesScope scope(this);
991 // All of these instructions require NEON.
992 scope.Record(CPUFeatures::kNEON);
993 switch (instr->Mask(NEONScalarByIndexedElementMask)) {
994 case NEON_SQRDMLAH_byelement_scalar:
995 case NEON_SQRDMLSH_byelement_scalar:
996 scope.Record(CPUFeatures::kRDM);
997 return;
998 default:
999 switch (instr->Mask(NEONScalarByIndexedElementFPMask)) {
1000 case NEON_FMLA_H_byelement_scalar:
1001 case NEON_FMLS_H_byelement_scalar:
1002 case NEON_FMUL_H_byelement_scalar:
1003 case NEON_FMULX_H_byelement_scalar:
1004 scope.Record(CPUFeatures::kNEONHalf);
1005 VIXL_FALLTHROUGH();
1006 case NEON_FMLA_byelement_scalar:
1007 case NEON_FMLS_byelement_scalar:
1008 case NEON_FMUL_byelement_scalar:
1009 case NEON_FMULX_byelement_scalar:
1010 scope.Record(CPUFeatures::kFP);
1011 return;
1012 }
1013 // No additional features.
1014 return;
1015 }
1016 }
1017
VisitNEONScalarCopy(const Instruction * instr)1018 void CPUFeaturesAuditor::VisitNEONScalarCopy(const Instruction* instr) {
1019 RecordInstructionFeaturesScope scope(this);
1020 // All of these instructions require NEON.
1021 scope.Record(CPUFeatures::kNEON);
1022 USE(instr);
1023 }
1024
VisitNEONScalarPairwise(const Instruction * instr)1025 void CPUFeaturesAuditor::VisitNEONScalarPairwise(const Instruction* instr) {
1026 RecordInstructionFeaturesScope scope(this);
1027 // All of these instructions require NEON.
1028 scope.Record(CPUFeatures::kNEON);
1029 switch (instr->Mask(NEONScalarPairwiseMask)) {
1030 case NEON_FMAXNMP_h_scalar:
1031 case NEON_FADDP_h_scalar:
1032 case NEON_FMAXP_h_scalar:
1033 case NEON_FMINNMP_h_scalar:
1034 case NEON_FMINP_h_scalar:
1035 scope.Record(CPUFeatures::kNEONHalf);
1036 VIXL_FALLTHROUGH();
1037 case NEON_FADDP_scalar:
1038 case NEON_FMAXP_scalar:
1039 case NEON_FMAXNMP_scalar:
1040 case NEON_FMINP_scalar:
1041 case NEON_FMINNMP_scalar:
1042 scope.Record(CPUFeatures::kFP);
1043 return;
1044 default:
1045 // No additional features.
1046 return;
1047 }
1048 }
1049
VisitNEONScalarShiftImmediate(const Instruction * instr)1050 void CPUFeaturesAuditor::VisitNEONScalarShiftImmediate(
1051 const Instruction* instr) {
1052 RecordInstructionFeaturesScope scope(this);
1053 // All of these instructions require NEON.
1054 scope.Record(CPUFeatures::kNEON);
1055 switch (instr->Mask(NEONScalarShiftImmediateMask)) {
1056 case NEON_FCVTZS_imm_scalar:
1057 case NEON_FCVTZU_imm_scalar:
1058 case NEON_SCVTF_imm_scalar:
1059 case NEON_UCVTF_imm_scalar:
1060 scope.Record(CPUFeatures::kFP);
1061 // If immh is 0b001x then the data type is FP16, and requires kNEONHalf.
1062 if ((instr->GetImmNEONImmh() & 0xe) == 0x2) {
1063 scope.Record(CPUFeatures::kNEONHalf);
1064 }
1065 return;
1066 default:
1067 // No additional features.
1068 return;
1069 }
1070 }
1071
VisitNEONShiftImmediate(const Instruction * instr)1072 void CPUFeaturesAuditor::VisitNEONShiftImmediate(const Instruction* instr) {
1073 RecordInstructionFeaturesScope scope(this);
1074 // All of these instructions require NEON.
1075 scope.Record(CPUFeatures::kNEON);
1076 switch (instr->Mask(NEONShiftImmediateMask)) {
1077 case NEON_SCVTF_imm:
1078 case NEON_UCVTF_imm:
1079 case NEON_FCVTZS_imm:
1080 case NEON_FCVTZU_imm:
1081 scope.Record(CPUFeatures::kFP);
1082 // If immh is 0b001x then the data type is FP16, and requires kNEONHalf.
1083 if ((instr->GetImmNEONImmh() & 0xe) == 0x2) {
1084 scope.Record(CPUFeatures::kNEONHalf);
1085 }
1086 return;
1087 default:
1088 // No additional features.
1089 return;
1090 }
1091 }
1092
VisitNEONTable(const Instruction * instr)1093 void CPUFeaturesAuditor::VisitNEONTable(const Instruction* instr) {
1094 RecordInstructionFeaturesScope scope(this);
1095 // All of these instructions require NEON.
1096 scope.Record(CPUFeatures::kNEON);
1097 USE(instr);
1098 }
1099
VisitPCRelAddressing(const Instruction * instr)1100 void CPUFeaturesAuditor::VisitPCRelAddressing(const Instruction* instr) {
1101 RecordInstructionFeaturesScope scope(this);
1102 USE(instr);
1103 }
1104
1105 // Most SVE visitors require only SVE.
1106 #define VIXL_SIMPLE_SVE_VISITOR_LIST(V) \
1107 V(SVE32BitGatherLoad_ScalarPlus32BitUnscaledOffsets) \
1108 V(SVE32BitGatherLoad_VectorPlusImm) \
1109 V(SVE32BitGatherLoadHalfwords_ScalarPlus32BitScaledOffsets) \
1110 V(SVE32BitGatherLoadWords_ScalarPlus32BitScaledOffsets) \
1111 V(SVE32BitGatherPrefetch_ScalarPlus32BitScaledOffsets) \
1112 V(SVE32BitGatherPrefetch_VectorPlusImm) \
1113 V(SVE32BitScatterStore_ScalarPlus32BitScaledOffsets) \
1114 V(SVE32BitScatterStore_ScalarPlus32BitUnscaledOffsets) \
1115 V(SVE32BitScatterStore_VectorPlusImm) \
1116 V(SVE64BitGatherLoad_ScalarPlus32BitUnpackedScaledOffsets) \
1117 V(SVE64BitGatherLoad_ScalarPlus64BitScaledOffsets) \
1118 V(SVE64BitGatherLoad_ScalarPlus64BitUnscaledOffsets) \
1119 V(SVE64BitGatherLoad_ScalarPlusUnpacked32BitUnscaledOffsets) \
1120 V(SVE64BitGatherLoad_VectorPlusImm) \
1121 V(SVE64BitGatherPrefetch_ScalarPlus64BitScaledOffsets) \
1122 V(SVE64BitGatherPrefetch_ScalarPlusUnpacked32BitScaledOffsets) \
1123 V(SVE64BitGatherPrefetch_VectorPlusImm) \
1124 V(SVE64BitScatterStore_ScalarPlus64BitScaledOffsets) \
1125 V(SVE64BitScatterStore_ScalarPlus64BitUnscaledOffsets) \
1126 V(SVE64BitScatterStore_ScalarPlusUnpacked32BitScaledOffsets) \
1127 V(SVE64BitScatterStore_ScalarPlusUnpacked32BitUnscaledOffsets) \
1128 V(SVE64BitScatterStore_VectorPlusImm) \
1129 V(SVEAddressGeneration) \
1130 V(SVEBitwiseLogicalUnpredicated) \
1131 V(SVEBitwiseShiftUnpredicated) \
1132 V(SVEFFRInitialise) \
1133 V(SVEFFRWriteFromPredicate) \
1134 V(SVEFPAccumulatingReduction) \
1135 V(SVEFPArithmeticUnpredicated) \
1136 V(SVEFPCompareVectors) \
1137 V(SVEFPCompareWithZero) \
1138 V(SVEFPComplexAddition) \
1139 V(SVEFPComplexMulAdd) \
1140 V(SVEFPComplexMulAddIndex) \
1141 V(SVEFPFastReduction) \
1142 V(SVEFPMulIndex) \
1143 V(SVEFPMulAdd) \
1144 V(SVEFPMulAddIndex) \
1145 V(SVEFPUnaryOpUnpredicated) \
1146 V(SVEIncDecByPredicateCount) \
1147 V(SVEIndexGeneration) \
1148 V(SVEIntArithmeticUnpredicated) \
1149 V(SVEIntCompareSignedImm) \
1150 V(SVEIntCompareUnsignedImm) \
1151 V(SVEIntCompareVectors) \
1152 V(SVEIntMulAddPredicated) \
1153 V(SVEIntMulAddUnpredicated) \
1154 V(SVEIntReduction) \
1155 V(SVEIntUnaryArithmeticPredicated) \
1156 V(SVEMovprfx) \
1157 V(SVEMulIndex) \
1158 V(SVEPermuteVectorExtract) \
1159 V(SVEPermuteVectorInterleaving) \
1160 V(SVEPredicateCount) \
1161 V(SVEPredicateLogical) \
1162 V(SVEPropagateBreak) \
1163 V(SVEStackFrameAdjustment) \
1164 V(SVEStackFrameSize) \
1165 V(SVEVectorSelect) \
1166 V(SVEBitwiseLogical_Predicated) \
1167 V(SVEBitwiseLogicalWithImm_Unpredicated) \
1168 V(SVEBitwiseShiftByImm_Predicated) \
1169 V(SVEBitwiseShiftByVector_Predicated) \
1170 V(SVEBitwiseShiftByWideElements_Predicated) \
1171 V(SVEBroadcastBitmaskImm) \
1172 V(SVEBroadcastFPImm_Unpredicated) \
1173 V(SVEBroadcastGeneralRegister) \
1174 V(SVEBroadcastIndexElement) \
1175 V(SVEBroadcastIntImm_Unpredicated) \
1176 V(SVECompressActiveElements) \
1177 V(SVEConditionallyBroadcastElementToVector) \
1178 V(SVEConditionallyExtractElementToSIMDFPScalar) \
1179 V(SVEConditionallyExtractElementToGeneralRegister) \
1180 V(SVEConditionallyTerminateScalars) \
1181 V(SVEConstructivePrefix_Unpredicated) \
1182 V(SVEContiguousFirstFaultLoad_ScalarPlusScalar) \
1183 V(SVEContiguousLoad_ScalarPlusImm) \
1184 V(SVEContiguousLoad_ScalarPlusScalar) \
1185 V(SVEContiguousNonFaultLoad_ScalarPlusImm) \
1186 V(SVEContiguousNonTemporalLoad_ScalarPlusImm) \
1187 V(SVEContiguousNonTemporalLoad_ScalarPlusScalar) \
1188 V(SVEContiguousNonTemporalStore_ScalarPlusImm) \
1189 V(SVEContiguousNonTemporalStore_ScalarPlusScalar) \
1190 V(SVEContiguousPrefetch_ScalarPlusImm) \
1191 V(SVEContiguousPrefetch_ScalarPlusScalar) \
1192 V(SVEContiguousStore_ScalarPlusImm) \
1193 V(SVEContiguousStore_ScalarPlusScalar) \
1194 V(SVECopySIMDFPScalarRegisterToVector_Predicated) \
1195 V(SVECopyFPImm_Predicated) \
1196 V(SVECopyGeneralRegisterToVector_Predicated) \
1197 V(SVECopyIntImm_Predicated) \
1198 V(SVEElementCount) \
1199 V(SVEExtractElementToSIMDFPScalarRegister) \
1200 V(SVEExtractElementToGeneralRegister) \
1201 V(SVEFPArithmetic_Predicated) \
1202 V(SVEFPArithmeticWithImm_Predicated) \
1203 V(SVEFPConvertPrecision) \
1204 V(SVEFPConvertToInt) \
1205 V(SVEFPExponentialAccelerator) \
1206 V(SVEFPRoundToIntegralValue) \
1207 V(SVEFPTrigMulAddCoefficient) \
1208 V(SVEFPTrigSelectCoefficient) \
1209 V(SVEFPUnaryOp) \
1210 V(SVEIncDecRegisterByElementCount) \
1211 V(SVEIncDecVectorByElementCount) \
1212 V(SVEInsertSIMDFPScalarRegister) \
1213 V(SVEInsertGeneralRegister) \
1214 V(SVEIntAddSubtractImm_Unpredicated) \
1215 V(SVEIntAddSubtractVectors_Predicated) \
1216 V(SVEIntCompareScalarCountAndLimit) \
1217 V(SVEIntConvertToFP) \
1218 V(SVEIntDivideVectors_Predicated) \
1219 V(SVEIntMinMaxImm_Unpredicated) \
1220 V(SVEIntMinMaxDifference_Predicated) \
1221 V(SVEIntMulImm_Unpredicated) \
1222 V(SVEIntMulVectors_Predicated) \
1223 V(SVELoadAndBroadcastElement) \
1224 V(SVELoadAndBroadcastQOWord_ScalarPlusImm) \
1225 V(SVELoadAndBroadcastQOWord_ScalarPlusScalar) \
1226 V(SVELoadMultipleStructures_ScalarPlusImm) \
1227 V(SVELoadMultipleStructures_ScalarPlusScalar) \
1228 V(SVELoadPredicateRegister) \
1229 V(SVELoadVectorRegister) \
1230 V(SVEPartitionBreakCondition) \
1231 V(SVEPermutePredicateElements) \
1232 V(SVEPredicateFirstActive) \
1233 V(SVEPredicateInitialize) \
1234 V(SVEPredicateNextActive) \
1235 V(SVEPredicateReadFromFFR_Predicated) \
1236 V(SVEPredicateReadFromFFR_Unpredicated) \
1237 V(SVEPredicateTest) \
1238 V(SVEPredicateZero) \
1239 V(SVEPropagateBreakToNextPartition) \
1240 V(SVEReversePredicateElements) \
1241 V(SVEReverseVectorElements) \
1242 V(SVEReverseWithinElements) \
1243 V(SVESaturatingIncDecRegisterByElementCount) \
1244 V(SVESaturatingIncDecVectorByElementCount) \
1245 V(SVEStoreMultipleStructures_ScalarPlusImm) \
1246 V(SVEStoreMultipleStructures_ScalarPlusScalar) \
1247 V(SVEStorePredicateRegister) \
1248 V(SVEStoreVectorRegister) \
1249 V(SVETableLookup) \
1250 V(SVEUnpackPredicateElements) \
1251 V(SVEUnpackVectorElements) \
1252 V(SVEVectorSplice)
1253
1254 #define VIXL_DEFINE_SIMPLE_SVE_VISITOR(NAME) \
1255 void CPUFeaturesAuditor::Visit##NAME(const Instruction* instr) { \
1256 RecordInstructionFeaturesScope scope(this); \
1257 scope.Record(CPUFeatures::kSVE); \
1258 USE(instr); \
1259 }
VIXL_SIMPLE_SVE_VISITOR_LIST(VIXL_DEFINE_SIMPLE_SVE_VISITOR)1260 VIXL_SIMPLE_SVE_VISITOR_LIST(VIXL_DEFINE_SIMPLE_SVE_VISITOR)
1261 #undef VIXL_DEFINE_SIMPLE_SVE_VISITOR
1262 #undef VIXL_SIMPLE_SVE_VISITOR_LIST
1263
1264 void CPUFeaturesAuditor::VisitSystem(const Instruction* instr) {
1265 RecordInstructionFeaturesScope scope(this);
1266 if (instr->Mask(SystemHintFMask) == SystemHintFixed) {
1267 CPUFeatures required;
1268 switch (instr->GetInstructionBits()) {
1269 case PACIA1716:
1270 case PACIB1716:
1271 case AUTIA1716:
1272 case AUTIB1716:
1273 case PACIAZ:
1274 case PACIASP:
1275 case PACIBZ:
1276 case PACIBSP:
1277 case AUTIAZ:
1278 case AUTIASP:
1279 case AUTIBZ:
1280 case AUTIBSP:
1281 case XPACLRI:
1282 required.Combine(CPUFeatures::kPAuth);
1283 break;
1284 default:
1285 switch (instr->GetImmHint()) {
1286 case ESB:
1287 required.Combine(CPUFeatures::kRAS);
1288 break;
1289 case BTI:
1290 case BTI_j:
1291 case BTI_c:
1292 case BTI_jc:
1293 required.Combine(CPUFeatures::kBTI);
1294 break;
1295 default:
1296 break;
1297 }
1298 break;
1299 }
1300
1301 // These are all HINT instructions, and behave as NOPs if the corresponding
1302 // features are not implemented, so we record the corresponding features
1303 // only if they are available.
1304 if (available_.Has(required)) scope.Record(required);
1305 } else if (instr->Mask(SystemSysMask) == SYS) {
1306 switch (instr->GetSysOp()) {
1307 // DC instruction variants.
1308 case CVAP:
1309 scope.Record(CPUFeatures::kDCPoP);
1310 break;
1311 case CVADP:
1312 scope.Record(CPUFeatures::kDCCVADP);
1313 break;
1314 case IVAU:
1315 case CVAC:
1316 case CVAU:
1317 case CIVAC:
1318 // No special CPU features.
1319 break;
1320 }
1321 } else if (instr->Mask(SystemPStateFMask) == SystemPStateFixed) {
1322 switch (instr->Mask(SystemPStateMask)) {
1323 case CFINV:
1324 scope.Record(CPUFeatures::kFlagM);
1325 break;
1326 case AXFLAG:
1327 case XAFLAG:
1328 scope.Record(CPUFeatures::kAXFlag);
1329 break;
1330 }
1331 } else if (instr->Mask(SystemSysRegFMask) == SystemSysRegFixed) {
1332 if (instr->Mask(SystemSysRegMask) == MRS) {
1333 switch (instr->GetImmSystemRegister()) {
1334 case RNDR:
1335 case RNDRRS:
1336 scope.Record(CPUFeatures::kRNG);
1337 break;
1338 }
1339 }
1340 }
1341 }
1342
VisitTestBranch(const Instruction * instr)1343 void CPUFeaturesAuditor::VisitTestBranch(const Instruction* instr) {
1344 RecordInstructionFeaturesScope scope(this);
1345 USE(instr);
1346 }
1347
VisitUnallocated(const Instruction * instr)1348 void CPUFeaturesAuditor::VisitUnallocated(const Instruction* instr) {
1349 RecordInstructionFeaturesScope scope(this);
1350 USE(instr);
1351 }
1352
VisitUnconditionalBranch(const Instruction * instr)1353 void CPUFeaturesAuditor::VisitUnconditionalBranch(const Instruction* instr) {
1354 RecordInstructionFeaturesScope scope(this);
1355 USE(instr);
1356 }
1357
VisitUnconditionalBranchToRegister(const Instruction * instr)1358 void CPUFeaturesAuditor::VisitUnconditionalBranchToRegister(
1359 const Instruction* instr) {
1360 RecordInstructionFeaturesScope scope(this);
1361 switch (instr->Mask(UnconditionalBranchToRegisterMask)) {
1362 case BRAAZ:
1363 case BRABZ:
1364 case BLRAAZ:
1365 case BLRABZ:
1366 case RETAA:
1367 case RETAB:
1368 case BRAA:
1369 case BRAB:
1370 case BLRAA:
1371 case BLRAB:
1372 scope.Record(CPUFeatures::kPAuth);
1373 return;
1374 default:
1375 // No additional features.
1376 return;
1377 }
1378 }
1379
VisitReserved(const Instruction * instr)1380 void CPUFeaturesAuditor::VisitReserved(const Instruction* instr) {
1381 RecordInstructionFeaturesScope scope(this);
1382 USE(instr);
1383 }
1384
VisitUnimplemented(const Instruction * instr)1385 void CPUFeaturesAuditor::VisitUnimplemented(const Instruction* instr) {
1386 RecordInstructionFeaturesScope scope(this);
1387 USE(instr);
1388 }
1389
Visit(Metadata * metadata,const Instruction * instr)1390 void CPUFeaturesAuditor::Visit(Metadata* metadata, const Instruction* instr) {
1391 VIXL_ASSERT(metadata->count("form") > 0);
1392 const std::string& form = (*metadata)["form"];
1393 uint32_t form_hash = Hash(form.c_str());
1394 const FormToVisitorFnMap* fv = CPUFeaturesAuditor::GetFormToVisitorFnMap();
1395 FormToVisitorFnMap::const_iterator it = fv->find(form_hash);
1396 if (it == fv->end()) {
1397 RecordInstructionFeaturesScope scope(this);
1398 std::map<uint32_t, const CPUFeatures> features = {
1399 {"adclb_z_zzz"_h, CPUFeatures::kSVE2},
1400 {"adclt_z_zzz"_h, CPUFeatures::kSVE2},
1401 {"addhnb_z_zz"_h, CPUFeatures::kSVE2},
1402 {"addhnt_z_zz"_h, CPUFeatures::kSVE2},
1403 {"addp_z_p_zz"_h, CPUFeatures::kSVE2},
1404 {"bcax_z_zzz"_h, CPUFeatures::kSVE2},
1405 {"bdep_z_zz"_h,
1406 CPUFeatures(CPUFeatures::kSVE2, CPUFeatures::kSVEBitPerm)},
1407 {"bext_z_zz"_h,
1408 CPUFeatures(CPUFeatures::kSVE2, CPUFeatures::kSVEBitPerm)},
1409 {"bgrp_z_zz"_h,
1410 CPUFeatures(CPUFeatures::kSVE2, CPUFeatures::kSVEBitPerm)},
1411 {"bsl1n_z_zzz"_h, CPUFeatures::kSVE2},
1412 {"bsl2n_z_zzz"_h, CPUFeatures::kSVE2},
1413 {"bsl_z_zzz"_h, CPUFeatures::kSVE2},
1414 {"cadd_z_zz"_h, CPUFeatures::kSVE2},
1415 {"cdot_z_zzz"_h, CPUFeatures::kSVE2},
1416 {"cdot_z_zzzi_d"_h, CPUFeatures::kSVE2},
1417 {"cdot_z_zzzi_s"_h, CPUFeatures::kSVE2},
1418 {"cmla_z_zzz"_h, CPUFeatures::kSVE2},
1419 {"cmla_z_zzzi_h"_h, CPUFeatures::kSVE2},
1420 {"cmla_z_zzzi_s"_h, CPUFeatures::kSVE2},
1421 {"eor3_z_zzz"_h, CPUFeatures::kSVE2},
1422 {"eorbt_z_zz"_h, CPUFeatures::kSVE2},
1423 {"eortb_z_zz"_h, CPUFeatures::kSVE2},
1424 {"ext_z_zi_con"_h, CPUFeatures::kSVE2},
1425 {"faddp_z_p_zz"_h, CPUFeatures::kSVE2},
1426 {"fcvtlt_z_p_z_h2s"_h, CPUFeatures::kSVE2},
1427 {"fcvtlt_z_p_z_s2d"_h, CPUFeatures::kSVE2},
1428 {"fcvtnt_z_p_z_d2s"_h, CPUFeatures::kSVE2},
1429 {"fcvtnt_z_p_z_s2h"_h, CPUFeatures::kSVE2},
1430 {"fcvtx_z_p_z_d2s"_h, CPUFeatures::kSVE2},
1431 {"fcvtxnt_z_p_z_d2s"_h, CPUFeatures::kSVE2},
1432 {"flogb_z_p_z"_h, CPUFeatures::kSVE2},
1433 {"fmaxnmp_z_p_zz"_h, CPUFeatures::kSVE2},
1434 {"fmaxp_z_p_zz"_h, CPUFeatures::kSVE2},
1435 {"fminnmp_z_p_zz"_h, CPUFeatures::kSVE2},
1436 {"fminp_z_p_zz"_h, CPUFeatures::kSVE2},
1437 {"fmlalb_z_zzz"_h, CPUFeatures::kSVE2},
1438 {"fmlalb_z_zzzi_s"_h, CPUFeatures::kSVE2},
1439 {"fmlalt_z_zzz"_h, CPUFeatures::kSVE2},
1440 {"fmlalt_z_zzzi_s"_h, CPUFeatures::kSVE2},
1441 {"fmlslb_z_zzz"_h, CPUFeatures::kSVE2},
1442 {"fmlslb_z_zzzi_s"_h, CPUFeatures::kSVE2},
1443 {"fmlslt_z_zzz"_h, CPUFeatures::kSVE2},
1444 {"fmlslt_z_zzzi_s"_h, CPUFeatures::kSVE2},
1445 {"histcnt_z_p_zz"_h, CPUFeatures::kSVE2},
1446 {"histseg_z_zz"_h, CPUFeatures::kSVE2},
1447 {"ldnt1b_z_p_ar_d_64_unscaled"_h, CPUFeatures::kSVE2},
1448 {"ldnt1b_z_p_ar_s_x32_unscaled"_h, CPUFeatures::kSVE2},
1449 {"ldnt1d_z_p_ar_d_64_unscaled"_h, CPUFeatures::kSVE2},
1450 {"ldnt1h_z_p_ar_d_64_unscaled"_h, CPUFeatures::kSVE2},
1451 {"ldnt1h_z_p_ar_s_x32_unscaled"_h, CPUFeatures::kSVE2},
1452 {"ldnt1sb_z_p_ar_d_64_unscaled"_h, CPUFeatures::kSVE2},
1453 {"ldnt1sb_z_p_ar_s_x32_unscaled"_h, CPUFeatures::kSVE2},
1454 {"ldnt1sh_z_p_ar_d_64_unscaled"_h, CPUFeatures::kSVE2},
1455 {"ldnt1sh_z_p_ar_s_x32_unscaled"_h, CPUFeatures::kSVE2},
1456 {"ldnt1sw_z_p_ar_d_64_unscaled"_h, CPUFeatures::kSVE2},
1457 {"ldnt1w_z_p_ar_d_64_unscaled"_h, CPUFeatures::kSVE2},
1458 {"ldnt1w_z_p_ar_s_x32_unscaled"_h, CPUFeatures::kSVE2},
1459 {"match_p_p_zz"_h, CPUFeatures::kSVE2},
1460 {"mla_z_zzzi_d"_h, CPUFeatures::kSVE2},
1461 {"mla_z_zzzi_h"_h, CPUFeatures::kSVE2},
1462 {"mla_z_zzzi_s"_h, CPUFeatures::kSVE2},
1463 {"mls_z_zzzi_d"_h, CPUFeatures::kSVE2},
1464 {"mls_z_zzzi_h"_h, CPUFeatures::kSVE2},
1465 {"mls_z_zzzi_s"_h, CPUFeatures::kSVE2},
1466 {"mul_z_zz"_h, CPUFeatures::kSVE2},
1467 {"mul_z_zzi_d"_h, CPUFeatures::kSVE2},
1468 {"mul_z_zzi_h"_h, CPUFeatures::kSVE2},
1469 {"mul_z_zzi_s"_h, CPUFeatures::kSVE2},
1470 {"nbsl_z_zzz"_h, CPUFeatures::kSVE2},
1471 {"nmatch_p_p_zz"_h, CPUFeatures::kSVE2},
1472 {"pmul_z_zz"_h, CPUFeatures::kSVE2},
1473 {"pmullb_z_zz"_h, CPUFeatures::kSVE2},
1474 {"pmullt_z_zz"_h, CPUFeatures::kSVE2},
1475 {"raddhnb_z_zz"_h, CPUFeatures::kSVE2},
1476 {"raddhnt_z_zz"_h, CPUFeatures::kSVE2},
1477 {"rshrnb_z_zi"_h, CPUFeatures::kSVE2},
1478 {"rshrnt_z_zi"_h, CPUFeatures::kSVE2},
1479 {"rsubhnb_z_zz"_h, CPUFeatures::kSVE2},
1480 {"rsubhnt_z_zz"_h, CPUFeatures::kSVE2},
1481 {"saba_z_zzz"_h, CPUFeatures::kSVE2},
1482 {"sabalb_z_zzz"_h, CPUFeatures::kSVE2},
1483 {"sabalt_z_zzz"_h, CPUFeatures::kSVE2},
1484 {"sabdlb_z_zz"_h, CPUFeatures::kSVE2},
1485 {"sabdlt_z_zz"_h, CPUFeatures::kSVE2},
1486 {"sadalp_z_p_z"_h, CPUFeatures::kSVE2},
1487 {"saddlb_z_zz"_h, CPUFeatures::kSVE2},
1488 {"saddlbt_z_zz"_h, CPUFeatures::kSVE2},
1489 {"saddlt_z_zz"_h, CPUFeatures::kSVE2},
1490 {"saddwb_z_zz"_h, CPUFeatures::kSVE2},
1491 {"saddwt_z_zz"_h, CPUFeatures::kSVE2},
1492 {"sbclb_z_zzz"_h, CPUFeatures::kSVE2},
1493 {"sbclt_z_zzz"_h, CPUFeatures::kSVE2},
1494 {"shadd_z_p_zz"_h, CPUFeatures::kSVE2},
1495 {"shrnb_z_zi"_h, CPUFeatures::kSVE2},
1496 {"shrnt_z_zi"_h, CPUFeatures::kSVE2},
1497 {"shsub_z_p_zz"_h, CPUFeatures::kSVE2},
1498 {"shsubr_z_p_zz"_h, CPUFeatures::kSVE2},
1499 {"sli_z_zzi"_h, CPUFeatures::kSVE2},
1500 {"smaxp_z_p_zz"_h, CPUFeatures::kSVE2},
1501 {"sminp_z_p_zz"_h, CPUFeatures::kSVE2},
1502 {"smlalb_z_zzz"_h, CPUFeatures::kSVE2},
1503 {"smlalb_z_zzzi_d"_h, CPUFeatures::kSVE2},
1504 {"smlalb_z_zzzi_s"_h, CPUFeatures::kSVE2},
1505 {"smlalt_z_zzz"_h, CPUFeatures::kSVE2},
1506 {"smlalt_z_zzzi_d"_h, CPUFeatures::kSVE2},
1507 {"smlalt_z_zzzi_s"_h, CPUFeatures::kSVE2},
1508 {"smlslb_z_zzz"_h, CPUFeatures::kSVE2},
1509 {"smlslb_z_zzzi_d"_h, CPUFeatures::kSVE2},
1510 {"smlslb_z_zzzi_s"_h, CPUFeatures::kSVE2},
1511 {"smlslt_z_zzz"_h, CPUFeatures::kSVE2},
1512 {"smlslt_z_zzzi_d"_h, CPUFeatures::kSVE2},
1513 {"smlslt_z_zzzi_s"_h, CPUFeatures::kSVE2},
1514 {"smulh_z_zz"_h, CPUFeatures::kSVE2},
1515 {"smullb_z_zz"_h, CPUFeatures::kSVE2},
1516 {"smullb_z_zzi_d"_h, CPUFeatures::kSVE2},
1517 {"smullb_z_zzi_s"_h, CPUFeatures::kSVE2},
1518 {"smullt_z_zz"_h, CPUFeatures::kSVE2},
1519 {"smullt_z_zzi_d"_h, CPUFeatures::kSVE2},
1520 {"smullt_z_zzi_s"_h, CPUFeatures::kSVE2},
1521 {"splice_z_p_zz_con"_h, CPUFeatures::kSVE2},
1522 {"sqabs_z_p_z"_h, CPUFeatures::kSVE2},
1523 {"sqadd_z_p_zz"_h, CPUFeatures::kSVE2},
1524 {"sqcadd_z_zz"_h, CPUFeatures::kSVE2},
1525 {"sqdmlalb_z_zzz"_h, CPUFeatures::kSVE2},
1526 {"sqdmlalb_z_zzzi_d"_h, CPUFeatures::kSVE2},
1527 {"sqdmlalb_z_zzzi_s"_h, CPUFeatures::kSVE2},
1528 {"sqdmlalbt_z_zzz"_h, CPUFeatures::kSVE2},
1529 {"sqdmlalt_z_zzz"_h, CPUFeatures::kSVE2},
1530 {"sqdmlalt_z_zzzi_d"_h, CPUFeatures::kSVE2},
1531 {"sqdmlalt_z_zzzi_s"_h, CPUFeatures::kSVE2},
1532 {"sqdmlslb_z_zzz"_h, CPUFeatures::kSVE2},
1533 {"sqdmlslb_z_zzzi_d"_h, CPUFeatures::kSVE2},
1534 {"sqdmlslb_z_zzzi_s"_h, CPUFeatures::kSVE2},
1535 {"sqdmlslbt_z_zzz"_h, CPUFeatures::kSVE2},
1536 {"sqdmlslt_z_zzz"_h, CPUFeatures::kSVE2},
1537 {"sqdmlslt_z_zzzi_d"_h, CPUFeatures::kSVE2},
1538 {"sqdmlslt_z_zzzi_s"_h, CPUFeatures::kSVE2},
1539 {"sqdmulh_z_zz"_h, CPUFeatures::kSVE2},
1540 {"sqdmulh_z_zzi_d"_h, CPUFeatures::kSVE2},
1541 {"sqdmulh_z_zzi_h"_h, CPUFeatures::kSVE2},
1542 {"sqdmulh_z_zzi_s"_h, CPUFeatures::kSVE2},
1543 {"sqdmullb_z_zz"_h, CPUFeatures::kSVE2},
1544 {"sqdmullb_z_zzi_d"_h, CPUFeatures::kSVE2},
1545 {"sqdmullb_z_zzi_s"_h, CPUFeatures::kSVE2},
1546 {"sqdmullt_z_zz"_h, CPUFeatures::kSVE2},
1547 {"sqdmullt_z_zzi_d"_h, CPUFeatures::kSVE2},
1548 {"sqdmullt_z_zzi_s"_h, CPUFeatures::kSVE2},
1549 {"sqneg_z_p_z"_h, CPUFeatures::kSVE2},
1550 {"sqrdcmlah_z_zzz"_h, CPUFeatures::kSVE2},
1551 {"sqrdcmlah_z_zzzi_h"_h, CPUFeatures::kSVE2},
1552 {"sqrdcmlah_z_zzzi_s"_h, CPUFeatures::kSVE2},
1553 {"sqrdmlah_z_zzz"_h, CPUFeatures::kSVE2},
1554 {"sqrdmlah_z_zzzi_d"_h, CPUFeatures::kSVE2},
1555 {"sqrdmlah_z_zzzi_h"_h, CPUFeatures::kSVE2},
1556 {"sqrdmlah_z_zzzi_s"_h, CPUFeatures::kSVE2},
1557 {"sqrdmlsh_z_zzz"_h, CPUFeatures::kSVE2},
1558 {"sqrdmlsh_z_zzzi_d"_h, CPUFeatures::kSVE2},
1559 {"sqrdmlsh_z_zzzi_h"_h, CPUFeatures::kSVE2},
1560 {"sqrdmlsh_z_zzzi_s"_h, CPUFeatures::kSVE2},
1561 {"sqrdmulh_z_zz"_h, CPUFeatures::kSVE2},
1562 {"sqrdmulh_z_zzi_d"_h, CPUFeatures::kSVE2},
1563 {"sqrdmulh_z_zzi_h"_h, CPUFeatures::kSVE2},
1564 {"sqrdmulh_z_zzi_s"_h, CPUFeatures::kSVE2},
1565 {"sqrshl_z_p_zz"_h, CPUFeatures::kSVE2},
1566 {"sqrshlr_z_p_zz"_h, CPUFeatures::kSVE2},
1567 {"sqrshrnb_z_zi"_h, CPUFeatures::kSVE2},
1568 {"sqrshrnt_z_zi"_h, CPUFeatures::kSVE2},
1569 {"sqrshrunb_z_zi"_h, CPUFeatures::kSVE2},
1570 {"sqrshrunt_z_zi"_h, CPUFeatures::kSVE2},
1571 {"sqshl_z_p_zi"_h, CPUFeatures::kSVE2},
1572 {"sqshl_z_p_zz"_h, CPUFeatures::kSVE2},
1573 {"sqshlr_z_p_zz"_h, CPUFeatures::kSVE2},
1574 {"sqshlu_z_p_zi"_h, CPUFeatures::kSVE2},
1575 {"sqshrnb_z_zi"_h, CPUFeatures::kSVE2},
1576 {"sqshrnt_z_zi"_h, CPUFeatures::kSVE2},
1577 {"sqshrunb_z_zi"_h, CPUFeatures::kSVE2},
1578 {"sqshrunt_z_zi"_h, CPUFeatures::kSVE2},
1579 {"sqsub_z_p_zz"_h, CPUFeatures::kSVE2},
1580 {"sqsubr_z_p_zz"_h, CPUFeatures::kSVE2},
1581 {"sqxtnb_z_zz"_h, CPUFeatures::kSVE2},
1582 {"sqxtnt_z_zz"_h, CPUFeatures::kSVE2},
1583 {"sqxtunb_z_zz"_h, CPUFeatures::kSVE2},
1584 {"sqxtunt_z_zz"_h, CPUFeatures::kSVE2},
1585 {"srhadd_z_p_zz"_h, CPUFeatures::kSVE2},
1586 {"sri_z_zzi"_h, CPUFeatures::kSVE2},
1587 {"srshl_z_p_zz"_h, CPUFeatures::kSVE2},
1588 {"srshlr_z_p_zz"_h, CPUFeatures::kSVE2},
1589 {"srshr_z_p_zi"_h, CPUFeatures::kSVE2},
1590 {"srsra_z_zi"_h, CPUFeatures::kSVE2},
1591 {"sshllb_z_zi"_h, CPUFeatures::kSVE2},
1592 {"sshllt_z_zi"_h, CPUFeatures::kSVE2},
1593 {"ssra_z_zi"_h, CPUFeatures::kSVE2},
1594 {"ssublb_z_zz"_h, CPUFeatures::kSVE2},
1595 {"ssublbt_z_zz"_h, CPUFeatures::kSVE2},
1596 {"ssublt_z_zz"_h, CPUFeatures::kSVE2},
1597 {"ssubltb_z_zz"_h, CPUFeatures::kSVE2},
1598 {"ssubwb_z_zz"_h, CPUFeatures::kSVE2},
1599 {"ssubwt_z_zz"_h, CPUFeatures::kSVE2},
1600 {"stnt1b_z_p_ar_d_64_unscaled"_h, CPUFeatures::kSVE2},
1601 {"stnt1b_z_p_ar_s_x32_unscaled"_h, CPUFeatures::kSVE2},
1602 {"stnt1d_z_p_ar_d_64_unscaled"_h, CPUFeatures::kSVE2},
1603 {"stnt1h_z_p_ar_d_64_unscaled"_h, CPUFeatures::kSVE2},
1604 {"stnt1h_z_p_ar_s_x32_unscaled"_h, CPUFeatures::kSVE2},
1605 {"stnt1w_z_p_ar_d_64_unscaled"_h, CPUFeatures::kSVE2},
1606 {"stnt1w_z_p_ar_s_x32_unscaled"_h, CPUFeatures::kSVE2},
1607 {"subhnb_z_zz"_h, CPUFeatures::kSVE2},
1608 {"subhnt_z_zz"_h, CPUFeatures::kSVE2},
1609 {"suqadd_z_p_zz"_h, CPUFeatures::kSVE2},
1610 {"tbl_z_zz_2"_h, CPUFeatures::kSVE2},
1611 {"tbx_z_zz"_h, CPUFeatures::kSVE2},
1612 {"uaba_z_zzz"_h, CPUFeatures::kSVE2},
1613 {"uabalb_z_zzz"_h, CPUFeatures::kSVE2},
1614 {"uabalt_z_zzz"_h, CPUFeatures::kSVE2},
1615 {"uabdlb_z_zz"_h, CPUFeatures::kSVE2},
1616 {"uabdlt_z_zz"_h, CPUFeatures::kSVE2},
1617 {"uadalp_z_p_z"_h, CPUFeatures::kSVE2},
1618 {"uaddlb_z_zz"_h, CPUFeatures::kSVE2},
1619 {"uaddlt_z_zz"_h, CPUFeatures::kSVE2},
1620 {"uaddwb_z_zz"_h, CPUFeatures::kSVE2},
1621 {"uaddwt_z_zz"_h, CPUFeatures::kSVE2},
1622 {"uhadd_z_p_zz"_h, CPUFeatures::kSVE2},
1623 {"uhsub_z_p_zz"_h, CPUFeatures::kSVE2},
1624 {"uhsubr_z_p_zz"_h, CPUFeatures::kSVE2},
1625 {"umaxp_z_p_zz"_h, CPUFeatures::kSVE2},
1626 {"uminp_z_p_zz"_h, CPUFeatures::kSVE2},
1627 {"umlalb_z_zzz"_h, CPUFeatures::kSVE2},
1628 {"umlalb_z_zzzi_d"_h, CPUFeatures::kSVE2},
1629 {"umlalb_z_zzzi_s"_h, CPUFeatures::kSVE2},
1630 {"umlalt_z_zzz"_h, CPUFeatures::kSVE2},
1631 {"umlalt_z_zzzi_d"_h, CPUFeatures::kSVE2},
1632 {"umlalt_z_zzzi_s"_h, CPUFeatures::kSVE2},
1633 {"umlslb_z_zzz"_h, CPUFeatures::kSVE2},
1634 {"umlslb_z_zzzi_d"_h, CPUFeatures::kSVE2},
1635 {"umlslb_z_zzzi_s"_h, CPUFeatures::kSVE2},
1636 {"umlslt_z_zzz"_h, CPUFeatures::kSVE2},
1637 {"umlslt_z_zzzi_d"_h, CPUFeatures::kSVE2},
1638 {"umlslt_z_zzzi_s"_h, CPUFeatures::kSVE2},
1639 {"umulh_z_zz"_h, CPUFeatures::kSVE2},
1640 {"umullb_z_zz"_h, CPUFeatures::kSVE2},
1641 {"umullb_z_zzi_d"_h, CPUFeatures::kSVE2},
1642 {"umullb_z_zzi_s"_h, CPUFeatures::kSVE2},
1643 {"umullt_z_zz"_h, CPUFeatures::kSVE2},
1644 {"umullt_z_zzi_d"_h, CPUFeatures::kSVE2},
1645 {"umullt_z_zzi_s"_h, CPUFeatures::kSVE2},
1646 {"uqadd_z_p_zz"_h, CPUFeatures::kSVE2},
1647 {"uqrshl_z_p_zz"_h, CPUFeatures::kSVE2},
1648 {"uqrshlr_z_p_zz"_h, CPUFeatures::kSVE2},
1649 {"uqrshrnb_z_zi"_h, CPUFeatures::kSVE2},
1650 {"uqrshrnt_z_zi"_h, CPUFeatures::kSVE2},
1651 {"uqshl_z_p_zi"_h, CPUFeatures::kSVE2},
1652 {"uqshl_z_p_zz"_h, CPUFeatures::kSVE2},
1653 {"uqshlr_z_p_zz"_h, CPUFeatures::kSVE2},
1654 {"uqshrnb_z_zi"_h, CPUFeatures::kSVE2},
1655 {"uqshrnt_z_zi"_h, CPUFeatures::kSVE2},
1656 {"uqsub_z_p_zz"_h, CPUFeatures::kSVE2},
1657 {"uqsubr_z_p_zz"_h, CPUFeatures::kSVE2},
1658 {"uqxtnb_z_zz"_h, CPUFeatures::kSVE2},
1659 {"uqxtnt_z_zz"_h, CPUFeatures::kSVE2},
1660 {"urecpe_z_p_z"_h, CPUFeatures::kSVE2},
1661 {"urhadd_z_p_zz"_h, CPUFeatures::kSVE2},
1662 {"urshl_z_p_zz"_h, CPUFeatures::kSVE2},
1663 {"urshlr_z_p_zz"_h, CPUFeatures::kSVE2},
1664 {"urshr_z_p_zi"_h, CPUFeatures::kSVE2},
1665 {"ursqrte_z_p_z"_h, CPUFeatures::kSVE2},
1666 {"ursra_z_zi"_h, CPUFeatures::kSVE2},
1667 {"ushllb_z_zi"_h, CPUFeatures::kSVE2},
1668 {"ushllt_z_zi"_h, CPUFeatures::kSVE2},
1669 {"usqadd_z_p_zz"_h, CPUFeatures::kSVE2},
1670 {"usra_z_zi"_h, CPUFeatures::kSVE2},
1671 {"usublb_z_zz"_h, CPUFeatures::kSVE2},
1672 {"usublt_z_zz"_h, CPUFeatures::kSVE2},
1673 {"usubwb_z_zz"_h, CPUFeatures::kSVE2},
1674 {"usubwt_z_zz"_h, CPUFeatures::kSVE2},
1675 {"whilege_p_p_rr"_h, CPUFeatures::kSVE2},
1676 {"whilegt_p_p_rr"_h, CPUFeatures::kSVE2},
1677 {"whilehi_p_p_rr"_h, CPUFeatures::kSVE2},
1678 {"whilehs_p_p_rr"_h, CPUFeatures::kSVE2},
1679 {"whilerw_p_rr"_h, CPUFeatures::kSVE2},
1680 {"whilewr_p_rr"_h, CPUFeatures::kSVE2},
1681 {"xar_z_zzi"_h, CPUFeatures::kSVE2},
1682 {"smmla_z_zzz"_h,
1683 CPUFeatures(CPUFeatures::kSVE, CPUFeatures::kSVEI8MM)},
1684 {"ummla_z_zzz"_h,
1685 CPUFeatures(CPUFeatures::kSVE, CPUFeatures::kSVEI8MM)},
1686 {"usmmla_z_zzz"_h,
1687 CPUFeatures(CPUFeatures::kSVE, CPUFeatures::kSVEI8MM)},
1688 {"fmmla_z_zzz_s"_h,
1689 CPUFeatures(CPUFeatures::kSVE, CPUFeatures::kSVEF32MM)},
1690 {"fmmla_z_zzz_d"_h,
1691 CPUFeatures(CPUFeatures::kSVE, CPUFeatures::kSVEF64MM)},
1692 {"smmla_asimdsame2_g"_h,
1693 CPUFeatures(CPUFeatures::kNEON, CPUFeatures::kI8MM)},
1694 {"ummla_asimdsame2_g"_h,
1695 CPUFeatures(CPUFeatures::kNEON, CPUFeatures::kI8MM)},
1696 {"usmmla_asimdsame2_g"_h,
1697 CPUFeatures(CPUFeatures::kNEON, CPUFeatures::kI8MM)},
1698 {"ld1row_z_p_bi_u32"_h,
1699 CPUFeatures(CPUFeatures::kSVE, CPUFeatures::kSVEF64MM)},
1700 {"ld1row_z_p_br_contiguous"_h,
1701 CPUFeatures(CPUFeatures::kSVE, CPUFeatures::kSVEF64MM)},
1702 {"ld1rod_z_p_bi_u64"_h,
1703 CPUFeatures(CPUFeatures::kSVE, CPUFeatures::kSVEF64MM)},
1704 {"ld1rod_z_p_br_contiguous"_h,
1705 CPUFeatures(CPUFeatures::kSVE, CPUFeatures::kSVEF64MM)},
1706 {"ld1rob_z_p_bi_u8"_h,
1707 CPUFeatures(CPUFeatures::kSVE, CPUFeatures::kSVEF64MM)},
1708 {"ld1rob_z_p_br_contiguous"_h,
1709 CPUFeatures(CPUFeatures::kSVE, CPUFeatures::kSVEF64MM)},
1710 {"ld1roh_z_p_bi_u16"_h,
1711 CPUFeatures(CPUFeatures::kSVE, CPUFeatures::kSVEF64MM)},
1712 {"ld1roh_z_p_br_contiguous"_h,
1713 CPUFeatures(CPUFeatures::kSVE, CPUFeatures::kSVEF64MM)},
1714 {"usdot_asimdsame2_d"_h,
1715 CPUFeatures(CPUFeatures::kNEON, CPUFeatures::kI8MM)},
1716 {"sudot_asimdelem_d"_h,
1717 CPUFeatures(CPUFeatures::kNEON, CPUFeatures::kI8MM)},
1718 {"usdot_asimdelem_d"_h,
1719 CPUFeatures(CPUFeatures::kNEON, CPUFeatures::kI8MM)},
1720 {"usdot_z_zzz_s"_h,
1721 CPUFeatures(CPUFeatures::kSVE, CPUFeatures::kSVEI8MM)},
1722 {"usdot_z_zzzi_s"_h,
1723 CPUFeatures(CPUFeatures::kSVE, CPUFeatures::kSVEI8MM)},
1724 {"sudot_z_zzzi_s"_h,
1725 CPUFeatures(CPUFeatures::kSVE, CPUFeatures::kSVEI8MM)},
1726 };
1727
1728 if (features.count(form_hash) > 0) {
1729 scope.Record(features[form_hash]);
1730 }
1731 } else {
1732 (it->second)(this, instr);
1733 }
1734 }
1735
1736 } // namespace aarch64
1737 } // namespace vixl
1738